LCOV - code coverage report
Current view: top level - kernel - fork.c (source / functions) Hit Total Coverage
Test: landlock.info Lines: 913 1289 70.8 %
Date: 2021-04-22 12:43:58 Functions: 64 88 72.7 %

          Line data    Source code
       1             : // SPDX-License-Identifier: GPL-2.0-only
       2             : /*
       3             :  *  linux/kernel/fork.c
       4             :  *
       5             :  *  Copyright (C) 1991, 1992  Linus Torvalds
       6             :  */
       7             : 
       8             : /*
       9             :  *  'fork.c' contains the help-routines for the 'fork' system call
      10             :  * (see also entry.S and others).
      11             :  * Fork is rather simple, once you get the hang of it, but the memory
      12             :  * management can be a bitch. See 'mm/memory.c': 'copy_page_range()'
      13             :  */
      14             : 
      15             : #include <linux/anon_inodes.h>
      16             : #include <linux/slab.h>
      17             : #include <linux/sched/autogroup.h>
      18             : #include <linux/sched/mm.h>
      19             : #include <linux/sched/coredump.h>
      20             : #include <linux/sched/user.h>
      21             : #include <linux/sched/numa_balancing.h>
      22             : #include <linux/sched/stat.h>
      23             : #include <linux/sched/task.h>
      24             : #include <linux/sched/task_stack.h>
      25             : #include <linux/sched/cputime.h>
      26             : #include <linux/seq_file.h>
      27             : #include <linux/rtmutex.h>
      28             : #include <linux/init.h>
      29             : #include <linux/unistd.h>
      30             : #include <linux/module.h>
      31             : #include <linux/vmalloc.h>
      32             : #include <linux/completion.h>
      33             : #include <linux/personality.h>
      34             : #include <linux/mempolicy.h>
      35             : #include <linux/sem.h>
      36             : #include <linux/file.h>
      37             : #include <linux/fdtable.h>
      38             : #include <linux/iocontext.h>
      39             : #include <linux/key.h>
      40             : #include <linux/binfmts.h>
      41             : #include <linux/mman.h>
      42             : #include <linux/mmu_notifier.h>
      43             : #include <linux/fs.h>
      44             : #include <linux/mm.h>
      45             : #include <linux/vmacache.h>
      46             : #include <linux/nsproxy.h>
      47             : #include <linux/capability.h>
      48             : #include <linux/cpu.h>
      49             : #include <linux/cgroup.h>
      50             : #include <linux/security.h>
      51             : #include <linux/hugetlb.h>
      52             : #include <linux/seccomp.h>
      53             : #include <linux/swap.h>
      54             : #include <linux/syscalls.h>
      55             : #include <linux/jiffies.h>
      56             : #include <linux/futex.h>
      57             : #include <linux/compat.h>
      58             : #include <linux/kthread.h>
      59             : #include <linux/task_io_accounting_ops.h>
      60             : #include <linux/rcupdate.h>
      61             : #include <linux/ptrace.h>
      62             : #include <linux/mount.h>
      63             : #include <linux/audit.h>
      64             : #include <linux/memcontrol.h>
      65             : #include <linux/ftrace.h>
      66             : #include <linux/proc_fs.h>
      67             : #include <linux/profile.h>
      68             : #include <linux/rmap.h>
      69             : #include <linux/ksm.h>
      70             : #include <linux/acct.h>
      71             : #include <linux/userfaultfd_k.h>
      72             : #include <linux/tsacct_kern.h>
      73             : #include <linux/cn_proc.h>
      74             : #include <linux/freezer.h>
      75             : #include <linux/delayacct.h>
      76             : #include <linux/taskstats_kern.h>
      77             : #include <linux/random.h>
      78             : #include <linux/tty.h>
      79             : #include <linux/blkdev.h>
      80             : #include <linux/fs_struct.h>
      81             : #include <linux/magic.h>
      82             : #include <linux/perf_event.h>
      83             : #include <linux/posix-timers.h>
      84             : #include <linux/user-return-notifier.h>
      85             : #include <linux/oom.h>
      86             : #include <linux/khugepaged.h>
      87             : #include <linux/signalfd.h>
      88             : #include <linux/uprobes.h>
      89             : #include <linux/aio.h>
      90             : #include <linux/compiler.h>
      91             : #include <linux/sysctl.h>
      92             : #include <linux/kcov.h>
      93             : #include <linux/livepatch.h>
      94             : #include <linux/thread_info.h>
      95             : #include <linux/stackleak.h>
      96             : #include <linux/kasan.h>
      97             : #include <linux/scs.h>
      98             : #include <linux/io_uring.h>
      99             : 
     100             : #include <asm/pgalloc.h>
     101             : #include <linux/uaccess.h>
     102             : #include <asm/mmu_context.h>
     103             : #include <asm/cacheflush.h>
     104             : #include <asm/tlbflush.h>
     105             : 
     106             : #include <trace/events/sched.h>
     107             : 
     108             : #define CREATE_TRACE_POINTS
     109             : #include <trace/events/task.h>
     110             : 
     111             : /*
     112             :  * Minimum number of threads to boot the kernel
     113             :  */
     114             : #define MIN_THREADS 20
     115             : 
     116             : /*
     117             :  * Maximum number of threads
     118             :  */
     119             : #define MAX_THREADS FUTEX_TID_MASK
     120             : 
     121             : /*
     122             :  * Protected counters by write_lock_irq(&tasklist_lock)
     123             :  */
     124             : unsigned long total_forks;      /* Handle normal Linux uptimes. */
     125             : int nr_threads;                 /* The idle threads do not count.. */
     126             : 
     127             : static int max_threads;         /* tunable limit on nr_threads */
     128             : 
     129             : #define NAMED_ARRAY_INDEX(x)    [x] = __stringify(x)
     130             : 
     131             : static const char * const resident_page_types[] = {
     132             :         NAMED_ARRAY_INDEX(MM_FILEPAGES),
     133             :         NAMED_ARRAY_INDEX(MM_ANONPAGES),
     134             :         NAMED_ARRAY_INDEX(MM_SWAPENTS),
     135             :         NAMED_ARRAY_INDEX(MM_SHMEMPAGES),
     136             : };
     137             : 
     138             : DEFINE_PER_CPU(unsigned long, process_counts) = 0;
     139             : 
     140             : __cacheline_aligned DEFINE_RWLOCK(tasklist_lock);  /* outer */
     141             : 
     142             : #ifdef CONFIG_PROVE_RCU
     143        8143 : int lockdep_tasklist_lock_is_held(void)
     144             : {
     145        8143 :         return lockdep_is_held(&tasklist_lock);
     146             : }
     147             : EXPORT_SYMBOL_GPL(lockdep_tasklist_lock_is_held);
     148             : #endif /* #ifdef CONFIG_PROVE_RCU */
     149             : 
     150         153 : int nr_processes(void)
     151             : {
     152         153 :         int cpu;
     153         153 :         int total = 0;
     154             : 
     155         765 :         for_each_possible_cpu(cpu)
     156         612 :                 total += per_cpu(process_counts, cpu);
     157             : 
     158         153 :         return total;
     159             : }
     160             : 
     161         834 : void __weak arch_release_task_struct(struct task_struct *tsk)
     162             : {
     163         834 : }
     164             : 
     165             : #ifndef CONFIG_ARCH_TASK_STRUCT_ALLOCATOR
     166             : static struct kmem_cache *task_struct_cachep;
     167             : 
     168         916 : static inline struct task_struct *alloc_task_struct_node(int node)
     169             : {
     170         916 :         return kmem_cache_alloc_node(task_struct_cachep, GFP_KERNEL, node);
     171             : }
     172             : 
     173         834 : static inline void free_task_struct(struct task_struct *tsk)
     174             : {
     175         834 :         kmem_cache_free(task_struct_cachep, tsk);
     176             : }
     177             : #endif
     178             : 
     179             : #ifndef CONFIG_ARCH_THREAD_STACK_ALLOCATOR
     180             : 
     181             : /*
     182             :  * Allocate pages if THREAD_SIZE is >= PAGE_SIZE, otherwise use a
     183             :  * kmemcache based allocator.
     184             :  */
     185             : # if THREAD_SIZE >= PAGE_SIZE || defined(CONFIG_VMAP_STACK)
     186             : 
     187             : #ifdef CONFIG_VMAP_STACK
     188             : /*
     189             :  * vmalloc() is a bit slow, and calling vfree() enough times will force a TLB
     190             :  * flush.  Try to minimize the number of calls by caching stacks.
     191             :  */
     192             : #define NR_CACHED_STACKS 2
     193             : static DEFINE_PER_CPU(struct vm_struct *, cached_stacks[NR_CACHED_STACKS]);
     194             : 
     195             : static int free_vm_stack_cache(unsigned int cpu)
     196             : {
     197             :         struct vm_struct **cached_vm_stacks = per_cpu_ptr(cached_stacks, cpu);
     198             :         int i;
     199             : 
     200             :         for (i = 0; i < NR_CACHED_STACKS; i++) {
     201             :                 struct vm_struct *vm_stack = cached_vm_stacks[i];
     202             : 
     203             :                 if (!vm_stack)
     204             :                         continue;
     205             : 
     206             :                 vfree(vm_stack->addr);
     207             :                 cached_vm_stacks[i] = NULL;
     208             :         }
     209             : 
     210             :         return 0;
     211             : }
     212             : #endif
     213             : 
     214         916 : static unsigned long *alloc_thread_stack_node(struct task_struct *tsk, int node)
     215             : {
     216             : #ifdef CONFIG_VMAP_STACK
     217             :         void *stack;
     218             :         int i;
     219             : 
     220             :         for (i = 0; i < NR_CACHED_STACKS; i++) {
     221             :                 struct vm_struct *s;
     222             : 
     223             :                 s = this_cpu_xchg(cached_stacks[i], NULL);
     224             : 
     225             :                 if (!s)
     226             :                         continue;
     227             : 
     228             :                 /* Mark stack accessible for KASAN. */
     229             :                 kasan_unpoison_range(s->addr, THREAD_SIZE);
     230             : 
     231             :                 /* Clear stale pointers from reused stack. */
     232             :                 memset(s->addr, 0, THREAD_SIZE);
     233             : 
     234             :                 tsk->stack_vm_area = s;
     235             :                 tsk->stack = s->addr;
     236             :                 return s->addr;
     237             :         }
     238             : 
     239             :         /*
     240             :          * Allocated stacks are cached and later reused by new threads,
     241             :          * so memcg accounting is performed manually on assigning/releasing
     242             :          * stacks to tasks. Drop __GFP_ACCOUNT.
     243             :          */
     244             :         stack = __vmalloc_node_range(THREAD_SIZE, THREAD_ALIGN,
     245             :                                      VMALLOC_START, VMALLOC_END,
     246             :                                      THREADINFO_GFP & ~__GFP_ACCOUNT,
     247             :                                      PAGE_KERNEL,
     248             :                                      0, node, __builtin_return_address(0));
     249             : 
     250             :         /*
     251             :          * We can't call find_vm_area() in interrupt context, and
     252             :          * free_thread_stack() can be called in interrupt context,
     253             :          * so cache the vm_struct.
     254             :          */
     255             :         if (stack) {
     256             :                 tsk->stack_vm_area = find_vm_area(stack);
     257             :                 tsk->stack = stack;
     258             :         }
     259             :         return stack;
     260             : #else
     261         916 :         struct page *page = alloc_pages_node(node, THREADINFO_GFP,
     262             :                                              THREAD_SIZE_ORDER);
     263             : 
     264         916 :         if (likely(page)) {
     265         916 :                 tsk->stack = kasan_reset_tag(page_address(page));
     266         916 :                 return tsk->stack;
     267             :         }
     268             :         return NULL;
     269             : #endif
     270             : }
     271             : 
     272         836 : static inline void free_thread_stack(struct task_struct *tsk)
     273             : {
     274             : #ifdef CONFIG_VMAP_STACK
     275             :         struct vm_struct *vm = task_stack_vm_area(tsk);
     276             : 
     277             :         if (vm) {
     278             :                 int i;
     279             : 
     280             :                 for (i = 0; i < THREAD_SIZE / PAGE_SIZE; i++)
     281             :                         memcg_kmem_uncharge_page(vm->pages[i], 0);
     282             : 
     283             :                 for (i = 0; i < NR_CACHED_STACKS; i++) {
     284             :                         if (this_cpu_cmpxchg(cached_stacks[i],
     285             :                                         NULL, tsk->stack_vm_area) != NULL)
     286             :                                 continue;
     287             : 
     288             :                         return;
     289             :                 }
     290             : 
     291             :                 vfree_atomic(tsk->stack);
     292             :                 return;
     293             :         }
     294             : #endif
     295             : 
     296         836 :         __free_pages(virt_to_page(tsk->stack), THREAD_SIZE_ORDER);
     297         836 : }
     298             : # else
     299             : static struct kmem_cache *thread_stack_cache;
     300             : 
     301             : static unsigned long *alloc_thread_stack_node(struct task_struct *tsk,
     302             :                                                   int node)
     303             : {
     304             :         unsigned long *stack;
     305             :         stack = kmem_cache_alloc_node(thread_stack_cache, THREADINFO_GFP, node);
     306             :         stack = kasan_reset_tag(stack);
     307             :         tsk->stack = stack;
     308             :         return stack;
     309             : }
     310             : 
     311             : static void free_thread_stack(struct task_struct *tsk)
     312             : {
     313             :         kmem_cache_free(thread_stack_cache, tsk->stack);
     314             : }
     315             : 
     316             : void thread_stack_cache_init(void)
     317             : {
     318             :         thread_stack_cache = kmem_cache_create_usercopy("thread_stack",
     319             :                                         THREAD_SIZE, THREAD_SIZE, 0, 0,
     320             :                                         THREAD_SIZE, NULL);
     321             :         BUG_ON(thread_stack_cache == NULL);
     322             : }
     323             : # endif
     324             : #endif
     325             : 
     326             : /* SLAB cache for signal_struct structures (tsk->signal) */
     327             : static struct kmem_cache *signal_cachep;
     328             : 
     329             : /* SLAB cache for sighand_struct structures (tsk->sighand) */
     330             : struct kmem_cache *sighand_cachep;
     331             : 
     332             : /* SLAB cache for files_struct structures (tsk->files) */
     333             : struct kmem_cache *files_cachep;
     334             : 
     335             : /* SLAB cache for fs_struct structures (tsk->fs) */
     336             : struct kmem_cache *fs_cachep;
     337             : 
     338             : /* SLAB cache for vm_area_struct structures */
     339             : static struct kmem_cache *vm_area_cachep;
     340             : 
     341             : /* SLAB cache for mm_struct structures (tsk->mm) */
     342             : static struct kmem_cache *mm_cachep;
     343             : 
     344       23143 : struct vm_area_struct *vm_area_alloc(struct mm_struct *mm)
     345             : {
     346       23143 :         struct vm_area_struct *vma;
     347             : 
     348       23143 :         vma = kmem_cache_alloc(vm_area_cachep, GFP_KERNEL);
     349       23146 :         if (vma)
     350       23146 :                 vma_init(vma, mm);
     351       23146 :         return vma;
     352             : }
     353             : 
     354       66368 : struct vm_area_struct *vm_area_dup(struct vm_area_struct *orig)
     355             : {
     356       66368 :         struct vm_area_struct *new = kmem_cache_alloc(vm_area_cachep, GFP_KERNEL);
     357             : 
     358       66369 :         if (new) {
     359       66369 :                 ASSERT_EXCLUSIVE_WRITER(orig->vm_flags);
     360       66369 :                 ASSERT_EXCLUSIVE_WRITER(orig->vm_file);
     361             :                 /*
     362             :                  * orig->shared.rb may be modified concurrently, but the clone
     363             :                  * will be reinitialized.
     364             :                  */
     365       66369 :                 *new = data_race(*orig);
     366       66369 :                 INIT_LIST_HEAD(&new->anon_vma_chain);
     367       66369 :                 new->vm_next = new->vm_prev = NULL;
     368             :         }
     369       66369 :         return new;
     370             : }
     371             : 
     372       86616 : void vm_area_free(struct vm_area_struct *vma)
     373             : {
     374       86616 :         kmem_cache_free(vm_area_cachep, vma);
     375           0 : }
     376             : 
     377        1752 : static void account_kernel_stack(struct task_struct *tsk, int account)
     378             : {
     379        1752 :         void *stack = task_stack_page(tsk);
     380        1752 :         struct vm_struct *vm = task_stack_vm_area(tsk);
     381             : 
     382             : 
     383             :         /* All stack pages are in the same node. */
     384        1752 :         if (vm)
     385             :                 mod_lruvec_page_state(vm->pages[0], NR_KERNEL_STACK_KB,
     386             :                                       account * (THREAD_SIZE / 1024));
     387             :         else
     388         836 :                 mod_lruvec_kmem_state(stack, NR_KERNEL_STACK_KB,
     389             :                                       account * (THREAD_SIZE / 1024));
     390             : }
     391             : 
     392         916 : static int memcg_charge_kernel_stack(struct task_struct *tsk)
     393             : {
     394             : #ifdef CONFIG_VMAP_STACK
     395             :         struct vm_struct *vm = task_stack_vm_area(tsk);
     396             :         int ret;
     397             : 
     398             :         BUILD_BUG_ON(IS_ENABLED(CONFIG_VMAP_STACK) && PAGE_SIZE % 1024 != 0);
     399             : 
     400             :         if (vm) {
     401             :                 int i;
     402             : 
     403             :                 BUG_ON(vm->nr_pages != THREAD_SIZE / PAGE_SIZE);
     404             : 
     405             :                 for (i = 0; i < THREAD_SIZE / PAGE_SIZE; i++) {
     406             :                         /*
     407             :                          * If memcg_kmem_charge_page() fails, page's
     408             :                          * memory cgroup pointer is NULL, and
     409             :                          * memcg_kmem_uncharge_page() in free_thread_stack()
     410             :                          * will ignore this page.
     411             :                          */
     412             :                         ret = memcg_kmem_charge_page(vm->pages[i], GFP_KERNEL,
     413             :                                                      0);
     414             :                         if (ret)
     415             :                                 return ret;
     416             :                 }
     417             :         }
     418             : #endif
     419         916 :         return 0;
     420             : }
     421             : 
     422         836 : static void release_task_stack(struct task_struct *tsk)
     423             : {
     424         836 :         if (WARN_ON(tsk->state != TASK_DEAD))
     425             :                 return;  /* Better to leak the stack than to free prematurely */
     426             : 
     427         836 :         account_kernel_stack(tsk, -1);
     428         836 :         free_thread_stack(tsk);
     429         836 :         tsk->stack = NULL;
     430             : #ifdef CONFIG_VMAP_STACK
     431             :         tsk->stack_vm_area = NULL;
     432             : #endif
     433             : }
     434             : 
     435             : #ifdef CONFIG_THREAD_INFO_IN_TASK
     436         890 : void put_task_stack(struct task_struct *tsk)
     437             : {
     438         890 :         if (refcount_dec_and_test(&tsk->stack_refcount))
     439         836 :                 release_task_stack(tsk);
     440         890 : }
     441             : #endif
     442             : 
     443         834 : void free_task(struct task_struct *tsk)
     444             : {
     445         834 :         scs_release(tsk);
     446             : 
     447             : #ifndef CONFIG_THREAD_INFO_IN_TASK
     448             :         /*
     449             :          * The task is finally done with both the stack and thread_info,
     450             :          * so free both.
     451             :          */
     452             :         release_task_stack(tsk);
     453             : #else
     454             :         /*
     455             :          * If the task had a separate stack allocation, it should be gone
     456             :          * by now.
     457             :          */
     458         834 :         WARN_ON_ONCE(refcount_read(&tsk->stack_refcount) != 0);
     459             : #endif
     460         834 :         rt_mutex_debug_task_free(tsk);
     461         834 :         ftrace_graph_exit_task(tsk);
     462         834 :         arch_release_task_struct(tsk);
     463         834 :         if (tsk->flags & PF_KTHREAD)
     464           0 :                 free_kthread_struct(tsk);
     465         834 :         free_task_struct(tsk);
     466         834 : }
     467             : EXPORT_SYMBOL(free_task);
     468             : 
     469             : #ifdef CONFIG_MMU
     470         858 : static __latent_entropy int dup_mmap(struct mm_struct *mm,
     471             :                                         struct mm_struct *oldmm)
     472             : {
     473         858 :         struct vm_area_struct *mpnt, *tmp, *prev, **pprev;
     474         858 :         struct rb_node **rb_link, *rb_parent;
     475         858 :         int retval;
     476         858 :         unsigned long charge;
     477         858 :         LIST_HEAD(uf);
     478             : 
     479         858 :         uprobe_start_dup_mmap();
     480         858 :         if (mmap_write_lock_killable(oldmm)) {
     481           0 :                 retval = -EINTR;
     482           0 :                 goto fail_uprobe_end;
     483             :         }
     484         858 :         flush_cache_dup_mm(oldmm);
     485         858 :         uprobe_dup_mmap(oldmm, mm);
     486             :         /*
     487             :          * Not linked in yet - no deadlock potential:
     488             :          */
     489         858 :         mmap_write_lock_nested(mm, SINGLE_DEPTH_NESTING);
     490             : 
     491             :         /* No ordering required: file already has been exposed. */
     492         858 :         RCU_INIT_POINTER(mm->exe_file, get_mm_exe_file(oldmm));
     493             : 
     494         858 :         mm->total_vm = oldmm->total_vm;
     495         858 :         mm->data_vm = oldmm->data_vm;
     496         858 :         mm->exec_vm = oldmm->exec_vm;
     497         858 :         mm->stack_vm = oldmm->stack_vm;
     498             : 
     499         858 :         rb_link = &mm->mm_rb.rb_node;
     500         858 :         rb_parent = NULL;
     501         858 :         pprev = &mm->mmap;
     502         858 :         retval = ksm_fork(mm, oldmm);
     503         858 :         if (retval)
     504           0 :                 goto out;
     505         858 :         retval = khugepaged_fork(mm, oldmm);
     506         858 :         if (retval)
     507           0 :                 goto out;
     508             : 
     509         858 :         prev = NULL;
     510       45845 :         for (mpnt = oldmm->mmap; mpnt; mpnt = mpnt->vm_next) {
     511       44987 :                 struct file *file;
     512             : 
     513       44987 :                 if (mpnt->vm_flags & VM_DONTCOPY) {
     514           0 :                         vm_stat_account(mm, mpnt->vm_flags, -vma_pages(mpnt));
     515           0 :                         continue;
     516             :                 }
     517       44987 :                 charge = 0;
     518             :                 /*
     519             :                  * Don't duplicate many vmas if we've been oom-killed (for
     520             :                  * example)
     521             :                  */
     522       44987 :                 if (fatal_signal_pending(current)) {
     523           0 :                         retval = -EINTR;
     524           0 :                         goto out;
     525             :                 }
     526       44988 :                 if (mpnt->vm_flags & VM_ACCOUNT) {
     527       20315 :                         unsigned long len = vma_pages(mpnt);
     528             : 
     529       20315 :                         if (security_vm_enough_memory_mm(oldmm, len)) /* sic */
     530           0 :                                 goto fail_nomem;
     531             :                         charge = len;
     532             :                 }
     533       44988 :                 tmp = vm_area_dup(mpnt);
     534       44988 :                 if (!tmp)
     535           0 :                         goto fail_nomem;
     536       44988 :                 retval = vma_dup_policy(mpnt, tmp);
     537       44988 :                 if (retval)
     538           0 :                         goto fail_nomem_policy;
     539       44988 :                 tmp->vm_mm = mm;
     540       44988 :                 retval = dup_userfaultfd(tmp, &uf);
     541       44988 :                 if (retval)
     542             :                         goto fail_nomem_anon_vma_fork;
     543       44988 :                 if (tmp->vm_flags & VM_WIPEONFORK) {
     544             :                         /*
     545             :                          * VM_WIPEONFORK gets a clean slate in the child.
     546             :                          * Don't prepare anon_vma until fault since we don't
     547             :                          * copy page for current vma.
     548             :                          */
     549           0 :                         tmp->anon_vma = NULL;
     550       44988 :                 } else if (anon_vma_fork(tmp, mpnt))
     551           0 :                         goto fail_nomem_anon_vma_fork;
     552       44988 :                 tmp->vm_flags &= ~(VM_LOCKED | VM_LOCKONFAULT);
     553       44988 :                 file = tmp->vm_file;
     554       44988 :                 if (file) {
     555       36915 :                         struct inode *inode = file_inode(file);
     556       36915 :                         struct address_space *mapping = file->f_mapping;
     557             : 
     558       36915 :                         get_file(file);
     559       36915 :                         if (tmp->vm_flags & VM_DENYWRITE)
     560        8570 :                                 put_write_access(inode);
     561       36915 :                         i_mmap_lock_write(mapping);
     562       36914 :                         if (tmp->vm_flags & VM_SHARED)
     563         120 :                                 mapping_allow_writable(mapping);
     564       36914 :                         flush_dcache_mmap_lock(mapping);
     565             :                         /* insert tmp into the share list, just after mpnt */
     566       36914 :                         vma_interval_tree_insert_after(tmp, mpnt,
     567             :                                         &mapping->i_mmap);
     568       36915 :                         flush_dcache_mmap_unlock(mapping);
     569       36915 :                         i_mmap_unlock_write(mapping);
     570             :                 }
     571             : 
     572             :                 /*
     573             :                  * Clear hugetlb-related page reserves for children. This only
     574             :                  * affects MAP_PRIVATE mappings. Faults generated by the child
     575             :                  * are not guaranteed to succeed, even if read-only
     576             :                  */
     577       44988 :                 if (is_vm_hugetlb_page(tmp))
     578       44988 :                         reset_vma_resv_huge_pages(tmp);
     579             : 
     580             :                 /*
     581             :                  * Link in the new vma and copy the page table entries.
     582             :                  */
     583       44988 :                 *pprev = tmp;
     584       44988 :                 pprev = &tmp->vm_next;
     585       44988 :                 tmp->vm_prev = prev;
     586       44988 :                 prev = tmp;
     587             : 
     588       44988 :                 __vma_link_rb(mm, tmp, rb_link, rb_parent);
     589       44988 :                 rb_link = &tmp->vm_rb.rb_right;
     590       44988 :                 rb_parent = &tmp->vm_rb;
     591             : 
     592       44988 :                 mm->map_count++;
     593       44988 :                 if (!(tmp->vm_flags & VM_WIPEONFORK))
     594       44988 :                         retval = copy_page_range(tmp, mpnt);
     595             : 
     596       44987 :                 if (tmp->vm_ops && tmp->vm_ops->open)
     597           0 :                         tmp->vm_ops->open(tmp);
     598             : 
     599       44987 :                 if (retval)
     600           0 :                         goto out;
     601             :         }
     602             :         /* a new mm has just been created */
     603         858 :         retval = arch_dup_mmap(oldmm, mm);
     604         858 : out:
     605         858 :         mmap_write_unlock(mm);
     606         858 :         flush_tlb_mm(oldmm);
     607         858 :         mmap_write_unlock(oldmm);
     608         858 :         dup_userfaultfd_complete(&uf);
     609         858 : fail_uprobe_end:
     610         858 :         uprobe_end_dup_mmap();
     611         858 :         return retval;
     612           0 : fail_nomem_anon_vma_fork:
     613           0 :         mpol_put(vma_policy(tmp));
     614           0 : fail_nomem_policy:
     615           0 :         vm_area_free(tmp);
     616           0 : fail_nomem:
     617           0 :         retval = -ENOMEM;
     618           0 :         vm_unacct_memory(charge);
     619           0 :         goto out;
     620             : }
     621             : 
     622        2019 : static inline int mm_alloc_pgd(struct mm_struct *mm)
     623             : {
     624        4038 :         mm->pgd = pgd_alloc(mm);
     625        2019 :         if (unlikely(!mm->pgd))
     626           0 :                 return -ENOMEM;
     627             :         return 0;
     628             : }
     629             : 
     630        1992 : static inline void mm_free_pgd(struct mm_struct *mm)
     631             : {
     632        1992 :         pgd_free(mm, mm->pgd);
     633           0 : }
     634             : #else
     635             : static int dup_mmap(struct mm_struct *mm, struct mm_struct *oldmm)
     636             : {
     637             :         mmap_write_lock(oldmm);
     638             :         RCU_INIT_POINTER(mm->exe_file, get_mm_exe_file(oldmm));
     639             :         mmap_write_unlock(oldmm);
     640             :         return 0;
     641             : }
     642             : #define mm_alloc_pgd(mm)        (0)
     643             : #define mm_free_pgd(mm)
     644             : #endif /* CONFIG_MMU */
     645             : 
     646        1992 : static void check_mm(struct mm_struct *mm)
     647             : {
     648        1992 :         int i;
     649             : 
     650        1992 :         BUILD_BUG_ON_MSG(ARRAY_SIZE(resident_page_types) != NR_MM_COUNTERS,
     651             :                          "Please make sure 'struct resident_page_types[]' is updated as well");
     652             : 
     653        9960 :         for (i = 0; i < NR_MM_COUNTERS; i++) {
     654        7968 :                 long x = atomic_long_read(&mm->rss_stat.count[i]);
     655             : 
     656        7968 :                 if (unlikely(x))
     657           0 :                         pr_alert("BUG: Bad rss-counter state mm:%p type:%s val:%ld\n",
     658             :                                  mm, resident_page_types[i], x);
     659             :         }
     660             : 
     661        1992 :         if (mm_pgtables_bytes(mm))
     662           0 :                 pr_alert("BUG: non-zero pgtables_bytes on freeing mm: %ld\n",
     663             :                                 mm_pgtables_bytes(mm));
     664             : 
     665             : #if defined(CONFIG_TRANSPARENT_HUGEPAGE) && !USE_SPLIT_PMD_PTLOCKS
     666             :         VM_BUG_ON_MM(mm->pmd_huge_pte, mm);
     667             : #endif
     668        1992 : }
     669             : 
     670             : #define allocate_mm()   (kmem_cache_alloc(mm_cachep, GFP_KERNEL))
     671             : #define free_mm(mm)     (kmem_cache_free(mm_cachep, (mm)))
     672             : 
     673             : /*
     674             :  * Called when the last reference to the mm
     675             :  * is dropped: either by a lazy thread or by
     676             :  * mmput. Free the page directory and the mm.
     677             :  */
     678        1992 : void __mmdrop(struct mm_struct *mm)
     679             : {
     680        1992 :         BUG_ON(mm == &init_mm);
     681        1992 :         WARN_ON_ONCE(mm == current->mm);
     682        1992 :         WARN_ON_ONCE(mm == current->active_mm);
     683        1992 :         mm_free_pgd(mm);
     684        1992 :         destroy_context(mm);
     685        1992 :         mmu_notifier_subscriptions_destroy(mm);
     686        1992 :         check_mm(mm);
     687        1992 :         put_user_ns(mm->user_ns);
     688        1992 :         free_mm(mm);
     689        1992 : }
     690             : EXPORT_SYMBOL_GPL(__mmdrop);
     691             : 
     692           0 : static void mmdrop_async_fn(struct work_struct *work)
     693             : {
     694           0 :         struct mm_struct *mm;
     695             : 
     696           0 :         mm = container_of(work, struct mm_struct, async_put_work);
     697           0 :         __mmdrop(mm);
     698           0 : }
     699             : 
     700           0 : static void mmdrop_async(struct mm_struct *mm)
     701             : {
     702           0 :         if (unlikely(atomic_dec_and_test(&mm->mm_count))) {
     703           0 :                 INIT_WORK(&mm->async_put_work, mmdrop_async_fn);
     704           0 :                 schedule_work(&mm->async_put_work);
     705             :         }
     706           0 : }
     707             : 
     708         832 : static inline void free_signal_struct(struct signal_struct *sig)
     709             : {
     710         832 :         taskstats_tgid_free(sig);
     711         832 :         sched_autogroup_exit(sig);
     712             :         /*
     713             :          * __mmdrop is not safe to call from softirq context on x86 due to
     714             :          * pgd_dtor so postpone it to the async context
     715             :          */
     716         832 :         if (sig->oom_mm)
     717           0 :                 mmdrop_async(sig->oom_mm);
     718         832 :         kmem_cache_free(signal_cachep, sig);
     719         832 : }
     720             : 
     721         834 : static inline void put_signal_struct(struct signal_struct *sig)
     722             : {
     723         834 :         if (refcount_dec_and_test(&sig->sigcnt))
     724         832 :                 free_signal_struct(sig);
     725         834 : }
     726             : 
     727         834 : void __put_task_struct(struct task_struct *tsk)
     728             : {
     729         834 :         WARN_ON(!tsk->exit_state);
     730         834 :         WARN_ON(refcount_read(&tsk->usage));
     731         834 :         WARN_ON(tsk == current);
     732             : 
     733         834 :         io_uring_free(tsk);
     734         834 :         cgroup_free(tsk);
     735         834 :         task_numa_free(tsk, true);
     736         834 :         security_task_free(tsk);
     737         834 :         exit_creds(tsk);
     738         834 :         delayacct_tsk_free(tsk);
     739         834 :         put_signal_struct(tsk->signal);
     740             : 
     741         834 :         if (!profile_handoff_task(tsk))
     742         834 :                 free_task(tsk);
     743         834 : }
     744             : EXPORT_SYMBOL_GPL(__put_task_struct);
     745             : 
     746           1 : void __init __weak arch_task_cache_init(void) { }
     747             : 
     748             : /*
     749             :  * set_max_threads
     750             :  */
     751           1 : static void set_max_threads(unsigned int max_threads_suggested)
     752             : {
     753           1 :         u64 threads;
     754           1 :         unsigned long nr_pages = totalram_pages();
     755             : 
     756             :         /*
     757             :          * The number of threads shall be limited such that the thread
     758             :          * structures may only consume a small part of the available memory.
     759             :          */
     760           1 :         if (fls64(nr_pages) + fls64(PAGE_SIZE) > 64)
     761             :                 threads = MAX_THREADS;
     762             :         else
     763           1 :                 threads = div64_u64((u64) nr_pages * (u64) PAGE_SIZE,
     764             :                                     (u64) THREAD_SIZE * 8UL);
     765             : 
     766           1 :         if (threads > max_threads_suggested)
     767             :                 threads = max_threads_suggested;
     768             : 
     769           1 :         max_threads = clamp_t(u64, threads, MIN_THREADS, MAX_THREADS);
     770           1 : }
     771             : 
     772             : #ifdef CONFIG_ARCH_WANTS_DYNAMIC_TASK_STRUCT
     773             : /* Initialized by the architecture: */
     774             : int arch_task_struct_size __read_mostly;
     775             : #endif
     776             : 
     777             : #ifndef CONFIG_ARCH_TASK_STRUCT_ALLOCATOR
     778           1 : static void task_struct_whitelist(unsigned long *offset, unsigned long *size)
     779             : {
     780             :         /* Fetch thread_struct whitelist for the architecture. */
     781           1 :         arch_thread_struct_whitelist(offset, size);
     782             : 
     783             :         /*
     784             :          * Handle zero-sized whitelist or empty thread_struct, otherwise
     785             :          * adjust offset to position of thread_struct in task_struct.
     786             :          */
     787           1 :         if (unlikely(*size == 0))
     788             :                 *offset = 0;
     789             :         else
     790           1 :                 *offset += offsetof(struct task_struct, thread);
     791             : }
     792             : #endif /* CONFIG_ARCH_TASK_STRUCT_ALLOCATOR */
     793             : 
     794           1 : void __init fork_init(void)
     795             : {
     796           1 :         int i;
     797             : #ifndef CONFIG_ARCH_TASK_STRUCT_ALLOCATOR
     798             : #ifndef ARCH_MIN_TASKALIGN
     799             : #define ARCH_MIN_TASKALIGN      0
     800             : #endif
     801           1 :         int align = max_t(int, L1_CACHE_BYTES, ARCH_MIN_TASKALIGN);
     802           1 :         unsigned long useroffset, usersize;
     803             : 
     804             :         /* create a slab on which task_structs can be allocated */
     805           1 :         task_struct_whitelist(&useroffset, &usersize);
     806           1 :         task_struct_cachep = kmem_cache_create_usercopy("task_struct",
     807             :                         arch_task_struct_size, align,
     808             :                         SLAB_PANIC|SLAB_ACCOUNT,
     809             :                         useroffset, usersize, NULL);
     810             : #endif
     811             : 
     812             :         /* do the arch specific task caches init */
     813           1 :         arch_task_cache_init();
     814             : 
     815           1 :         set_max_threads(MAX_THREADS);
     816             : 
     817           1 :         init_task.signal->rlim[RLIMIT_NPROC].rlim_cur = max_threads/2;
     818           1 :         init_task.signal->rlim[RLIMIT_NPROC].rlim_max = max_threads/2;
     819           1 :         init_task.signal->rlim[RLIMIT_SIGPENDING] =
     820           1 :                 init_task.signal->rlim[RLIMIT_NPROC];
     821             : 
     822          11 :         for (i = 0; i < UCOUNT_COUNTS; i++)
     823          10 :                 init_user_ns.ucount_max[i] = max_threads/2;
     824             : 
     825             : #ifdef CONFIG_VMAP_STACK
     826             :         cpuhp_setup_state(CPUHP_BP_PREPARE_DYN, "fork:vm_stack_cache",
     827             :                           NULL, free_vm_stack_cache);
     828             : #endif
     829             : 
     830           1 :         scs_init();
     831             : 
     832           1 :         lockdep_init_task(&init_task);
     833           1 :         uprobes_init();
     834           1 : }
     835             : 
     836           0 : int __weak arch_dup_task_struct(struct task_struct *dst,
     837             :                                                struct task_struct *src)
     838             : {
     839           0 :         *dst = *src;
     840           0 :         return 0;
     841             : }
     842             : 
     843         917 : void set_task_stack_end_magic(struct task_struct *tsk)
     844             : {
     845         917 :         unsigned long *stackend;
     846             : 
     847         917 :         stackend = end_of_stack(tsk);
     848         917 :         *stackend = STACK_END_MAGIC;    /* for overflow detection */
     849           1 : }
     850             : 
     851         916 : static struct task_struct *dup_task_struct(struct task_struct *orig, int node)
     852             : {
     853         916 :         struct task_struct *tsk;
     854         916 :         unsigned long *stack;
     855         916 :         struct vm_struct *stack_vm_area __maybe_unused;
     856         916 :         int err;
     857             : 
     858         916 :         if (node == NUMA_NO_NODE)
     859         913 :                 node = tsk_fork_get_node(orig);
     860         916 :         tsk = alloc_task_struct_node(node);
     861         916 :         if (!tsk)
     862             :                 return NULL;
     863             : 
     864         916 :         stack = alloc_thread_stack_node(tsk, node);
     865         916 :         if (!stack)
     866           0 :                 goto free_tsk;
     867             : 
     868         916 :         if (memcg_charge_kernel_stack(tsk))
     869             :                 goto free_stack;
     870             : 
     871         916 :         stack_vm_area = task_stack_vm_area(tsk);
     872             : 
     873         916 :         err = arch_dup_task_struct(tsk, orig);
     874             : 
     875             :         /*
     876             :          * arch_dup_task_struct() clobbers the stack-related fields.  Make
     877             :          * sure they're properly initialized before using any stack-related
     878             :          * functions again.
     879             :          */
     880         916 :         tsk->stack = stack;
     881             : #ifdef CONFIG_VMAP_STACK
     882             :         tsk->stack_vm_area = stack_vm_area;
     883             : #endif
     884             : #ifdef CONFIG_THREAD_INFO_IN_TASK
     885         916 :         refcount_set(&tsk->stack_refcount, 1);
     886             : #endif
     887             : 
     888         916 :         if (err)
     889           0 :                 goto free_stack;
     890             : 
     891         916 :         err = scs_prepare(tsk, node);
     892         916 :         if (err)
     893             :                 goto free_stack;
     894             : 
     895             : #ifdef CONFIG_SECCOMP
     896             :         /*
     897             :          * We must handle setting up seccomp filters once we're under
     898             :          * the sighand lock in case orig has changed between now and
     899             :          * then. Until then, filter must be NULL to avoid messing up
     900             :          * the usage counts on the error path calling free_task.
     901             :          */
     902             :         tsk->seccomp.filter = NULL;
     903             : #endif
     904             : 
     905         916 :         setup_thread_stack(tsk, orig);
     906         916 :         clear_user_return_notifier(tsk);
     907         916 :         clear_tsk_need_resched(tsk);
     908         916 :         set_task_stack_end_magic(tsk);
     909         916 :         clear_syscall_work_syscall_user_dispatch(tsk);
     910             : 
     911             : #ifdef CONFIG_STACKPROTECTOR
     912             :         tsk->stack_canary = get_random_canary();
     913             : #endif
     914         916 :         if (orig->cpus_ptr == &orig->cpus_mask)
     915         916 :                 tsk->cpus_ptr = &tsk->cpus_mask;
     916             : 
     917             :         /*
     918             :          * One for the user space visible state that goes away when reaped.
     919             :          * One for the scheduler.
     920             :          */
     921         916 :         refcount_set(&tsk->rcu_users, 2);
     922             :         /* One for the rcu users */
     923         916 :         refcount_set(&tsk->usage, 1);
     924             : #ifdef CONFIG_BLK_DEV_IO_TRACE
     925             :         tsk->btrace_seq = 0;
     926             : #endif
     927         916 :         tsk->splice_pipe = NULL;
     928         916 :         tsk->task_frag.page = NULL;
     929         916 :         tsk->wake_q.next = NULL;
     930             : 
     931         916 :         account_kernel_stack(tsk, 1);
     932             : 
     933         916 :         kcov_task_init(tsk);
     934         916 :         kmap_local_fork(tsk);
     935             : 
     936             : #ifdef CONFIG_FAULT_INJECTION
     937             :         tsk->fail_nth = 0;
     938             : #endif
     939             : 
     940             : #ifdef CONFIG_BLK_CGROUP
     941             :         tsk->throttle_queue = NULL;
     942             :         tsk->use_memdelay = 0;
     943             : #endif
     944             : 
     945             : #ifdef CONFIG_MEMCG
     946             :         tsk->active_memcg = NULL;
     947             : #endif
     948         916 :         return tsk;
     949             : 
     950           0 : free_stack:
     951           0 :         free_thread_stack(tsk);
     952           0 : free_tsk:
     953           0 :         free_task_struct(tsk);
     954           0 :         return NULL;
     955             : }
     956             : 
     957             : __cacheline_aligned_in_smp DEFINE_SPINLOCK(mmlist_lock);
     958             : 
     959             : static unsigned long default_dump_filter = MMF_DUMP_FILTER_DEFAULT;
     960             : 
     961           0 : static int __init coredump_filter_setup(char *s)
     962             : {
     963           0 :         default_dump_filter =
     964           0 :                 (simple_strtoul(s, NULL, 0) << MMF_DUMP_FILTER_SHIFT) &
     965             :                 MMF_DUMP_FILTER_MASK;
     966           0 :         return 1;
     967             : }
     968             : 
     969             : __setup("coredump_filter=", coredump_filter_setup);
     970             : 
     971             : #include <linux/init_task.h>
     972             : 
     973        2019 : static void mm_init_aio(struct mm_struct *mm)
     974             : {
     975             : #ifdef CONFIG_AIO
     976             :         spin_lock_init(&mm->ioctx_lock);
     977             :         mm->ioctx_table = NULL;
     978             : #endif
     979        2019 : }
     980             : 
     981           0 : static __always_inline void mm_clear_owner(struct mm_struct *mm,
     982             :                                            struct task_struct *p)
     983             : {
     984             : #ifdef CONFIG_MEMCG
     985             :         if (mm->owner == p)
     986             :                 WRITE_ONCE(mm->owner, NULL);
     987             : #endif
     988           0 : }
     989             : 
     990        2019 : static void mm_init_owner(struct mm_struct *mm, struct task_struct *p)
     991             : {
     992             : #ifdef CONFIG_MEMCG
     993             :         mm->owner = p;
     994             : #endif
     995        2019 : }
     996             : 
     997        2019 : static void mm_init_pasid(struct mm_struct *mm)
     998             : {
     999             : #ifdef CONFIG_IOMMU_SUPPORT
    1000             :         mm->pasid = INIT_PASID;
    1001             : #endif
    1002        2019 : }
    1003             : 
    1004        2019 : static void mm_init_uprobes_state(struct mm_struct *mm)
    1005             : {
    1006             : #ifdef CONFIG_UPROBES
    1007             :         mm->uprobes_state.xol_area = NULL;
    1008             : #endif
    1009        2019 : }
    1010             : 
    1011        2019 : static struct mm_struct *mm_init(struct mm_struct *mm, struct task_struct *p,
    1012             :         struct user_namespace *user_ns)
    1013             : {
    1014        2019 :         mm->mmap = NULL;
    1015        2019 :         mm->mm_rb = RB_ROOT;
    1016        2019 :         mm->vmacache_seqnum = 0;
    1017        2019 :         atomic_set(&mm->mm_users, 1);
    1018        2019 :         atomic_set(&mm->mm_count, 1);
    1019        2019 :         seqcount_init(&mm->write_protect_seq);
    1020        2019 :         mmap_init_lock(mm);
    1021        2019 :         INIT_LIST_HEAD(&mm->mmlist);
    1022        2019 :         mm->core_state = NULL;
    1023        2019 :         mm_pgtables_bytes_init(mm);
    1024        2019 :         mm->map_count = 0;
    1025        2019 :         mm->locked_vm = 0;
    1026        2019 :         atomic_set(&mm->has_pinned, 0);
    1027        2019 :         atomic64_set(&mm->pinned_vm, 0);
    1028        2019 :         memset(&mm->rss_stat, 0, sizeof(mm->rss_stat));
    1029        2019 :         spin_lock_init(&mm->page_table_lock);
    1030        2019 :         spin_lock_init(&mm->arg_lock);
    1031        2019 :         mm_init_cpumask(mm);
    1032        2019 :         mm_init_aio(mm);
    1033        2019 :         mm_init_owner(mm, p);
    1034        2019 :         mm_init_pasid(mm);
    1035        2019 :         RCU_INIT_POINTER(mm->exe_file, NULL);
    1036        2019 :         mmu_notifier_subscriptions_init(mm);
    1037        2019 :         init_tlb_flush_pending(mm);
    1038             : #if defined(CONFIG_TRANSPARENT_HUGEPAGE) && !USE_SPLIT_PMD_PTLOCKS
    1039             :         mm->pmd_huge_pte = NULL;
    1040             : #endif
    1041        2019 :         mm_init_uprobes_state(mm);
    1042             : 
    1043        2019 :         if (current->mm) {
    1044        2017 :                 mm->flags = current->mm->flags & MMF_INIT_MASK;
    1045        2017 :                 mm->def_flags = current->mm->def_flags & VM_INIT_DEF_MASK;
    1046             :         } else {
    1047           2 :                 mm->flags = default_dump_filter;
    1048           2 :                 mm->def_flags = 0;
    1049             :         }
    1050             : 
    1051        2019 :         if (mm_alloc_pgd(mm))
    1052           0 :                 goto fail_nopgd;
    1053             : 
    1054        2019 :         if (init_new_context(p, mm))
    1055           0 :                 goto fail_nocontext;
    1056             : 
    1057        2019 :         mm->user_ns = get_user_ns(user_ns);
    1058        2019 :         return mm;
    1059             : 
    1060           0 : fail_nocontext:
    1061           0 :         mm_free_pgd(mm);
    1062           0 : fail_nopgd:
    1063           0 :         free_mm(mm);
    1064           0 :         return NULL;
    1065             : }
    1066             : 
    1067             : /*
    1068             :  * Allocate and initialize an mm_struct.
    1069             :  */
    1070        1161 : struct mm_struct *mm_alloc(void)
    1071             : {
    1072        1161 :         struct mm_struct *mm;
    1073             : 
    1074        1161 :         mm = allocate_mm();
    1075        1161 :         if (!mm)
    1076             :                 return NULL;
    1077             : 
    1078        1161 :         memset(mm, 0, sizeof(*mm));
    1079        1161 :         return mm_init(mm, current, current_user_ns());
    1080             : }
    1081             : 
    1082        1994 : static inline void __mmput(struct mm_struct *mm)
    1083             : {
    1084        1994 :         VM_BUG_ON(atomic_read(&mm->mm_users));
    1085             : 
    1086        1994 :         uprobe_clear_state(mm);
    1087        1994 :         exit_aio(mm);
    1088        1994 :         ksm_exit(mm);
    1089        1994 :         khugepaged_exit(mm); /* must run before exit_mmap */
    1090        1994 :         exit_mmap(mm);
    1091        1994 :         mm_put_huge_zero_page(mm);
    1092        1994 :         set_mm_exe_file(mm, NULL);
    1093        1994 :         if (!list_empty(&mm->mmlist)) {
    1094           0 :                 spin_lock(&mmlist_lock);
    1095           0 :                 list_del(&mm->mmlist);
    1096           0 :                 spin_unlock(&mmlist_lock);
    1097             :         }
    1098        1994 :         if (mm->binfmt)
    1099        1994 :                 module_put(mm->binfmt->module);
    1100        1994 :         mmdrop(mm);
    1101        1994 : }
    1102             : 
    1103             : /*
    1104             :  * Decrement the use count and release all resources for an mm.
    1105             :  */
    1106        2402 : void mmput(struct mm_struct *mm)
    1107             : {
    1108        2402 :         might_sleep();
    1109             : 
    1110        4804 :         if (atomic_dec_and_test(&mm->mm_users))
    1111        1994 :                 __mmput(mm);
    1112        2402 : }
    1113             : EXPORT_SYMBOL_GPL(mmput);
    1114             : 
    1115             : #ifdef CONFIG_MMU
    1116           0 : static void mmput_async_fn(struct work_struct *work)
    1117             : {
    1118           0 :         struct mm_struct *mm = container_of(work, struct mm_struct,
    1119             :                                             async_put_work);
    1120             : 
    1121           0 :         __mmput(mm);
    1122           0 : }
    1123             : 
    1124           0 : void mmput_async(struct mm_struct *mm)
    1125             : {
    1126           0 :         if (atomic_dec_and_test(&mm->mm_users)) {
    1127           0 :                 INIT_WORK(&mm->async_put_work, mmput_async_fn);
    1128           0 :                 schedule_work(&mm->async_put_work);
    1129             :         }
    1130           0 : }
    1131             : #endif
    1132             : 
    1133             : /**
    1134             :  * set_mm_exe_file - change a reference to the mm's executable file
    1135             :  *
    1136             :  * This changes mm's executable file (shown as symlink /proc/[pid]/exe).
    1137             :  *
    1138             :  * Main users are mmput() and sys_execve(). Callers prevent concurrent
    1139             :  * invocations: in mmput() nobody alive left, in execve task is single
    1140             :  * threaded. sys_prctl(PR_SET_MM_MAP/EXE_FILE) also needs to set the
    1141             :  * mm->exe_file, but does so without using set_mm_exe_file() in order
    1142             :  * to do avoid the need for any locks.
    1143             :  */
    1144        2545 : void set_mm_exe_file(struct mm_struct *mm, struct file *new_exe_file)
    1145             : {
    1146        2545 :         struct file *old_exe_file;
    1147             : 
    1148             :         /*
    1149             :          * It is safe to dereference the exe_file without RCU as
    1150             :          * this function is only called if nobody else can access
    1151             :          * this mm -- see comment above for justification.
    1152             :          */
    1153        2545 :         old_exe_file = rcu_dereference_raw(mm->exe_file);
    1154             : 
    1155        2545 :         if (new_exe_file)
    1156         551 :                 get_file(new_exe_file);
    1157        2545 :         rcu_assign_pointer(mm->exe_file, new_exe_file);
    1158        2545 :         if (old_exe_file)
    1159        1384 :                 fput(old_exe_file);
    1160        2545 : }
    1161             : 
    1162             : /**
    1163             :  * get_mm_exe_file - acquire a reference to the mm's executable file
    1164             :  *
    1165             :  * Returns %NULL if mm has no associated executable file.
    1166             :  * User must release file via fput().
    1167             :  */
    1168         904 : struct file *get_mm_exe_file(struct mm_struct *mm)
    1169             : {
    1170         904 :         struct file *exe_file;
    1171             : 
    1172         904 :         rcu_read_lock();
    1173         904 :         exe_file = rcu_dereference(mm->exe_file);
    1174        1807 :         if (exe_file && !get_file_rcu(exe_file))
    1175           0 :                 exe_file = NULL;
    1176         904 :         rcu_read_unlock();
    1177         904 :         return exe_file;
    1178             : }
    1179             : EXPORT_SYMBOL(get_mm_exe_file);
    1180             : 
    1181             : /**
    1182             :  * get_task_exe_file - acquire a reference to the task's executable file
    1183             :  *
    1184             :  * Returns %NULL if task's mm (if any) has no associated executable file or
    1185             :  * this is a kernel thread with borrowed mm (see the comment above get_task_mm).
    1186             :  * User must release file via fput().
    1187             :  */
    1188          46 : struct file *get_task_exe_file(struct task_struct *task)
    1189             : {
    1190          46 :         struct file *exe_file = NULL;
    1191          46 :         struct mm_struct *mm;
    1192             : 
    1193          46 :         task_lock(task);
    1194          46 :         mm = task->mm;
    1195          46 :         if (mm) {
    1196          46 :                 if (!(task->flags & PF_KTHREAD))
    1197          46 :                         exe_file = get_mm_exe_file(mm);
    1198             :         }
    1199          46 :         task_unlock(task);
    1200          46 :         return exe_file;
    1201             : }
    1202             : EXPORT_SYMBOL(get_task_exe_file);
    1203             : 
    1204             : /**
    1205             :  * get_task_mm - acquire a reference to the task's mm
    1206             :  *
    1207             :  * Returns %NULL if the task has no mm.  Checks PF_KTHREAD (meaning
    1208             :  * this kernel workthread has transiently adopted a user mm with use_mm,
    1209             :  * to do its AIO) is not set and if so returns a reference to it, after
    1210             :  * bumping up the use count.  User must release the mm via mmput()
    1211             :  * after use.  Typically used by /proc and ptrace.
    1212             :  */
    1213         362 : struct mm_struct *get_task_mm(struct task_struct *task)
    1214             : {
    1215         362 :         struct mm_struct *mm;
    1216             : 
    1217         362 :         task_lock(task);
    1218         362 :         mm = task->mm;
    1219         362 :         if (mm) {
    1220         316 :                 if (task->flags & PF_KTHREAD)
    1221             :                         mm = NULL;
    1222             :                 else
    1223         316 :                         mmget(mm);
    1224             :         }
    1225         362 :         task_unlock(task);
    1226         362 :         return mm;
    1227             : }
    1228             : EXPORT_SYMBOL_GPL(get_task_mm);
    1229             : 
    1230          62 : struct mm_struct *mm_access(struct task_struct *task, unsigned int mode)
    1231             : {
    1232          62 :         struct mm_struct *mm;
    1233          62 :         int err;
    1234             : 
    1235          62 :         err =  down_read_killable(&task->signal->exec_update_lock);
    1236          62 :         if (err)
    1237           0 :                 return ERR_PTR(err);
    1238             : 
    1239          62 :         mm = get_task_mm(task);
    1240         124 :         if (mm && mm != current->mm &&
    1241          62 :                         !ptrace_may_access(task, mode)) {
    1242           9 :                 mmput(mm);
    1243           9 :                 mm = ERR_PTR(-EACCES);
    1244             :         }
    1245          62 :         up_read(&task->signal->exec_update_lock);
    1246             : 
    1247          62 :         return mm;
    1248             : }
    1249             : 
    1250           0 : static void complete_vfork_done(struct task_struct *tsk)
    1251             : {
    1252           0 :         struct completion *vfork;
    1253             : 
    1254           0 :         task_lock(tsk);
    1255           0 :         vfork = tsk->vfork_done;
    1256           0 :         if (likely(vfork)) {
    1257           0 :                 tsk->vfork_done = NULL;
    1258           0 :                 complete(vfork);
    1259             :         }
    1260           0 :         task_unlock(tsk);
    1261           0 : }
    1262             : 
    1263           0 : static int wait_for_vfork_done(struct task_struct *child,
    1264             :                                 struct completion *vfork)
    1265             : {
    1266           0 :         int killed;
    1267             : 
    1268           0 :         freezer_do_not_count();
    1269           0 :         cgroup_enter_frozen();
    1270           0 :         killed = wait_for_completion_killable(vfork);
    1271           0 :         cgroup_leave_frozen(false);
    1272           0 :         freezer_count();
    1273             : 
    1274           0 :         if (killed) {
    1275           0 :                 task_lock(child);
    1276           0 :                 child->vfork_done = NULL;
    1277           0 :                 task_unlock(child);
    1278             :         }
    1279             : 
    1280           0 :         put_task_struct(child);
    1281           0 :         return killed;
    1282             : }
    1283             : 
    1284             : /* Please note the differences between mmput and mm_release.
    1285             :  * mmput is called whenever we stop holding onto a mm_struct,
    1286             :  * error success whatever.
    1287             :  *
    1288             :  * mm_release is called after a mm_struct has been removed
    1289             :  * from the current process.
    1290             :  *
    1291             :  * This difference is important for error handling, when we
    1292             :  * only half set up a mm_struct for a new process and need to restore
    1293             :  * the old one.  Because we mmput the new mm_struct before
    1294             :  * restoring the old one. . .
    1295             :  * Eric Biederman 10 January 1998
    1296             :  */
    1297        1387 : static void mm_release(struct task_struct *tsk, struct mm_struct *mm)
    1298             : {
    1299        1387 :         uprobe_free_utask(tsk);
    1300             : 
    1301             :         /* Get rid of any cached register state */
    1302        1387 :         deactivate_mm(tsk, mm);
    1303             : 
    1304             :         /*
    1305             :          * Signal userspace if we're not exiting with a core dump
    1306             :          * because we want to leave the value intact for debugging
    1307             :          * purposes.
    1308             :          */
    1309        1387 :         if (tsk->clear_child_tid) {
    1310        1159 :                 if (!(tsk->signal->flags & SIGNAL_GROUP_COREDUMP) &&
    1311        1159 :                     atomic_read(&mm->mm_users) > 1) {
    1312             :                         /*
    1313             :                          * We don't check the error code - if userspace has
    1314             :                          * not set up a proper pointer then tough luck.
    1315             :                          */
    1316           2 :                         put_user(0, tsk->clear_child_tid);
    1317           2 :                         do_futex(tsk->clear_child_tid, FUTEX_WAKE,
    1318             :                                         1, NULL, NULL, 0, 0);
    1319             :                 }
    1320        1159 :                 tsk->clear_child_tid = NULL;
    1321             :         }
    1322             : 
    1323             :         /*
    1324             :          * All done, finally we can wake up parent and return this mm to him.
    1325             :          * Also kthread_stop() uses this completion for synchronization.
    1326             :          */
    1327        1387 :         if (tsk->vfork_done)
    1328           0 :                 complete_vfork_done(tsk);
    1329        1387 : }
    1330             : 
    1331         836 : void exit_mm_release(struct task_struct *tsk, struct mm_struct *mm)
    1332             : {
    1333         836 :         futex_exit_release(tsk);
    1334         836 :         mm_release(tsk, mm);
    1335         836 : }
    1336             : 
    1337         551 : void exec_mm_release(struct task_struct *tsk, struct mm_struct *mm)
    1338             : {
    1339         551 :         futex_exec_release(tsk);
    1340         551 :         mm_release(tsk, mm);
    1341         551 : }
    1342             : 
    1343             : /**
    1344             :  * dup_mm() - duplicates an existing mm structure
    1345             :  * @tsk: the task_struct with which the new mm will be associated.
    1346             :  * @oldmm: the mm to duplicate.
    1347             :  *
    1348             :  * Allocates a new mm structure and duplicates the provided @oldmm structure
    1349             :  * content into it.
    1350             :  *
    1351             :  * Return: the duplicated mm or NULL on failure.
    1352             :  */
    1353         858 : static struct mm_struct *dup_mm(struct task_struct *tsk,
    1354             :                                 struct mm_struct *oldmm)
    1355             : {
    1356         858 :         struct mm_struct *mm;
    1357         858 :         int err;
    1358             : 
    1359         858 :         mm = allocate_mm();
    1360         858 :         if (!mm)
    1361           0 :                 goto fail_nomem;
    1362             : 
    1363         858 :         memcpy(mm, oldmm, sizeof(*mm));
    1364             : 
    1365         858 :         if (!mm_init(mm, tsk, mm->user_ns))
    1366           0 :                 goto fail_nomem;
    1367             : 
    1368         858 :         err = dup_mmap(mm, oldmm);
    1369         858 :         if (err)
    1370           0 :                 goto free_pt;
    1371             : 
    1372         858 :         mm->hiwater_rss = get_mm_rss(mm);
    1373         858 :         mm->hiwater_vm = mm->total_vm;
    1374             : 
    1375         858 :         if (mm->binfmt && !try_module_get(mm->binfmt->module))
    1376             :                 goto free_pt;
    1377             : 
    1378             :         return mm;
    1379             : 
    1380           0 : free_pt:
    1381             :         /* don't put binfmt in mmput, we haven't got module yet */
    1382           0 :         mm->binfmt = NULL;
    1383           0 :         mm_init_owner(mm, NULL);
    1384           0 :         mmput(mm);
    1385             : 
    1386             : fail_nomem:
    1387             :         return NULL;
    1388             : }
    1389             : 
    1390         916 : static int copy_mm(unsigned long clone_flags, struct task_struct *tsk)
    1391             : {
    1392         916 :         struct mm_struct *mm, *oldmm;
    1393         916 :         int retval;
    1394             : 
    1395         916 :         tsk->min_flt = tsk->maj_flt = 0;
    1396         916 :         tsk->nvcsw = tsk->nivcsw = 0;
    1397             : #ifdef CONFIG_DETECT_HUNG_TASK
    1398             :         tsk->last_switch_count = tsk->nvcsw + tsk->nivcsw;
    1399             :         tsk->last_switch_time = 0;
    1400             : #endif
    1401             : 
    1402         916 :         tsk->mm = NULL;
    1403         916 :         tsk->active_mm = NULL;
    1404             : 
    1405             :         /*
    1406             :          * Are we cloning a kernel thread?
    1407             :          *
    1408             :          * We need to steal a active VM for that..
    1409             :          */
    1410         916 :         oldmm = current->mm;
    1411         916 :         if (!oldmm)
    1412             :                 return 0;
    1413             : 
    1414             :         /* initialize the new vmacache entries */
    1415         863 :         vmacache_flush(tsk);
    1416             : 
    1417         863 :         if (clone_flags & CLONE_VM) {
    1418           6 :                 mmget(oldmm);
    1419           6 :                 mm = oldmm;
    1420           6 :                 goto good_mm;
    1421             :         }
    1422             : 
    1423         857 :         retval = -ENOMEM;
    1424         857 :         mm = dup_mm(tsk, current->mm);
    1425         857 :         if (!mm)
    1426           0 :                 goto fail_nomem;
    1427             : 
    1428         857 : good_mm:
    1429         863 :         tsk->mm = mm;
    1430         863 :         tsk->active_mm = mm;
    1431         863 :         return 0;
    1432             : 
    1433           0 : fail_nomem:
    1434           0 :         return retval;
    1435             : }
    1436             : 
    1437         916 : static int copy_fs(unsigned long clone_flags, struct task_struct *tsk)
    1438             : {
    1439         916 :         struct fs_struct *fs = current->fs;
    1440         916 :         if (clone_flags & CLONE_FS) {
    1441             :                 /* tsk->fs is already what we want */
    1442          56 :                 spin_lock(&fs->lock);
    1443          56 :                 if (fs->in_exec) {
    1444           0 :                         spin_unlock(&fs->lock);
    1445           0 :                         return -EAGAIN;
    1446             :                 }
    1447          56 :                 fs->users++;
    1448          56 :                 spin_unlock(&fs->lock);
    1449          56 :                 return 0;
    1450             :         }
    1451         860 :         tsk->fs = copy_fs_struct(fs);
    1452         860 :         if (!tsk->fs)
    1453           0 :                 return -ENOMEM;
    1454             :         return 0;
    1455             : }
    1456             : 
    1457         916 : static int copy_files(unsigned long clone_flags, struct task_struct *tsk)
    1458             : {
    1459         916 :         struct files_struct *oldf, *newf;
    1460         916 :         int error = 0;
    1461             : 
    1462             :         /*
    1463             :          * A background process may not have any files ...
    1464             :          */
    1465         916 :         oldf = current->files;
    1466         916 :         if (!oldf)
    1467           0 :                 goto out;
    1468             : 
    1469         916 :         if (clone_flags & CLONE_FILES) {
    1470          55 :                 atomic_inc(&oldf->count);
    1471          55 :                 goto out;
    1472             :         }
    1473             : 
    1474         861 :         newf = dup_fd(oldf, NR_OPEN_MAX, &error);
    1475         861 :         if (!newf)
    1476           0 :                 goto out;
    1477             : 
    1478         861 :         tsk->files = newf;
    1479         861 :         error = 0;
    1480         916 : out:
    1481         916 :         return error;
    1482             : }
    1483             : 
    1484         916 : static int copy_io(unsigned long clone_flags, struct task_struct *tsk)
    1485             : {
    1486             : #ifdef CONFIG_BLOCK
    1487         916 :         struct io_context *ioc = current->io_context;
    1488         916 :         struct io_context *new_ioc;
    1489             : 
    1490         916 :         if (!ioc)
    1491             :                 return 0;
    1492             :         /*
    1493             :          * Share io context with parent, if CLONE_IO is set
    1494             :          */
    1495         232 :         if (clone_flags & CLONE_IO) {
    1496           0 :                 ioc_task_link(ioc);
    1497           0 :                 tsk->io_context = ioc;
    1498         232 :         } else if (ioprio_valid(ioc->ioprio)) {
    1499          17 :                 new_ioc = get_task_io_context(tsk, GFP_KERNEL, NUMA_NO_NODE);
    1500          17 :                 if (unlikely(!new_ioc))
    1501             :                         return -ENOMEM;
    1502             : 
    1503          17 :                 new_ioc->ioprio = ioc->ioprio;
    1504          17 :                 put_io_context(new_ioc);
    1505             :         }
    1506             : #endif
    1507             :         return 0;
    1508             : }
    1509             : 
    1510         916 : static int copy_sighand(unsigned long clone_flags, struct task_struct *tsk)
    1511             : {
    1512         916 :         struct sighand_struct *sig;
    1513             : 
    1514         916 :         if (clone_flags & CLONE_SIGHAND) {
    1515           6 :                 refcount_inc(&current->sighand->count);
    1516           6 :                 return 0;
    1517             :         }
    1518         910 :         sig = kmem_cache_alloc(sighand_cachep, GFP_KERNEL);
    1519         910 :         RCU_INIT_POINTER(tsk->sighand, sig);
    1520         910 :         if (!sig)
    1521             :                 return -ENOMEM;
    1522             : 
    1523         910 :         refcount_set(&sig->count, 1);
    1524         910 :         spin_lock_irq(&current->sighand->siglock);
    1525         910 :         memcpy(sig->action, current->sighand->action, sizeof(sig->action));
    1526         910 :         spin_unlock_irq(&current->sighand->siglock);
    1527             : 
    1528             :         /* Reset all signal handler not set to SIG_IGN to SIG_DFL. */
    1529         910 :         if (clone_flags & CLONE_CLEAR_SIGHAND)
    1530           0 :                 flush_signal_handlers(tsk, 0);
    1531             : 
    1532             :         return 0;
    1533             : }
    1534             : 
    1535         836 : void __cleanup_sighand(struct sighand_struct *sighand)
    1536             : {
    1537         836 :         if (refcount_dec_and_test(&sighand->count)) {
    1538         834 :                 signalfd_cleanup(sighand);
    1539             :                 /*
    1540             :                  * sighand_cachep is SLAB_TYPESAFE_BY_RCU so we can free it
    1541             :                  * without an RCU grace period, see __lock_task_sighand().
    1542             :                  */
    1543         834 :                 kmem_cache_free(sighand_cachep, sighand);
    1544             :         }
    1545         836 : }
    1546             : 
    1547             : /*
    1548             :  * Initialize POSIX timer handling for a thread group.
    1549             :  */
    1550         910 : static void posix_cpu_timers_init_group(struct signal_struct *sig)
    1551             : {
    1552         910 :         struct posix_cputimers *pct = &sig->posix_cputimers;
    1553         910 :         unsigned long cpu_limit;
    1554             : 
    1555         910 :         cpu_limit = READ_ONCE(sig->rlim[RLIMIT_CPU].rlim_cur);
    1556         910 :         posix_cputimers_group_init(pct, cpu_limit);
    1557             : }
    1558             : 
    1559         916 : static int copy_signal(unsigned long clone_flags, struct task_struct *tsk)
    1560             : {
    1561         916 :         struct signal_struct *sig;
    1562             : 
    1563         916 :         if (clone_flags & CLONE_THREAD)
    1564             :                 return 0;
    1565             : 
    1566         910 :         sig = kmem_cache_zalloc(signal_cachep, GFP_KERNEL);
    1567         910 :         tsk->signal = sig;
    1568         910 :         if (!sig)
    1569             :                 return -ENOMEM;
    1570             : 
    1571         910 :         sig->nr_threads = 1;
    1572         910 :         atomic_set(&sig->live, 1);
    1573         910 :         refcount_set(&sig->sigcnt, 1);
    1574             : 
    1575             :         /* list_add(thread_node, thread_head) without INIT_LIST_HEAD() */
    1576         910 :         sig->thread_head = (struct list_head)LIST_HEAD_INIT(tsk->thread_node);
    1577         910 :         tsk->thread_node = (struct list_head)LIST_HEAD_INIT(sig->thread_head);
    1578             : 
    1579         910 :         init_waitqueue_head(&sig->wait_chldexit);
    1580         910 :         sig->curr_target = tsk;
    1581         910 :         init_sigpending(&sig->shared_pending);
    1582         910 :         INIT_HLIST_HEAD(&sig->multiprocess);
    1583         910 :         seqlock_init(&sig->stats_lock);
    1584         910 :         prev_cputime_init(&sig->prev_cputime);
    1585             : 
    1586             : #ifdef CONFIG_POSIX_TIMERS
    1587         910 :         INIT_LIST_HEAD(&sig->posix_timers);
    1588         910 :         hrtimer_init(&sig->real_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
    1589         910 :         sig->real_timer.function = it_real_fn;
    1590             : #endif
    1591             : 
    1592         910 :         task_lock(current->group_leader);
    1593         910 :         memcpy(sig->rlim, current->signal->rlim, sizeof sig->rlim);
    1594         910 :         task_unlock(current->group_leader);
    1595             : 
    1596         910 :         posix_cpu_timers_init_group(sig);
    1597             : 
    1598         910 :         tty_audit_fork(sig);
    1599         910 :         sched_autogroup_fork(sig);
    1600             : 
    1601         910 :         sig->oom_score_adj = current->signal->oom_score_adj;
    1602         910 :         sig->oom_score_adj_min = current->signal->oom_score_adj_min;
    1603             : 
    1604         910 :         mutex_init(&sig->cred_guard_mutex);
    1605         910 :         init_rwsem(&sig->exec_update_lock);
    1606             : 
    1607         910 :         return 0;
    1608             : }
    1609             : 
    1610         916 : static void copy_seccomp(struct task_struct *p)
    1611             : {
    1612             : #ifdef CONFIG_SECCOMP
    1613             :         /*
    1614             :          * Must be called with sighand->lock held, which is common to
    1615             :          * all threads in the group. Holding cred_guard_mutex is not
    1616             :          * needed because this new task is not yet running and cannot
    1617             :          * be racing exec.
    1618             :          */
    1619             :         assert_spin_locked(&current->sighand->siglock);
    1620             : 
    1621             :         /* Ref-count the new filter user, and assign it. */
    1622             :         get_seccomp_filter(current);
    1623             :         p->seccomp = current->seccomp;
    1624             : 
    1625             :         /*
    1626             :          * Explicitly enable no_new_privs here in case it got set
    1627             :          * between the task_struct being duplicated and holding the
    1628             :          * sighand lock. The seccomp state and nnp must be in sync.
    1629             :          */
    1630             :         if (task_no_new_privs(current))
    1631             :                 task_set_no_new_privs(p);
    1632             : 
    1633             :         /*
    1634             :          * If the parent gained a seccomp mode after copying thread
    1635             :          * flags and between before we held the sighand lock, we have
    1636             :          * to manually enable the seccomp thread flag here.
    1637             :          */
    1638             :         if (p->seccomp.mode != SECCOMP_MODE_DISABLED)
    1639             :                 set_task_syscall_work(p, SECCOMP);
    1640             : #endif
    1641         916 : }
    1642             : 
    1643         644 : SYSCALL_DEFINE1(set_tid_address, int __user *, tidptr)
    1644             : {
    1645         322 :         current->clear_child_tid = tidptr;
    1646             : 
    1647         322 :         return task_pid_vnr(current);
    1648             : }
    1649             : 
    1650         916 : static void rt_mutex_init_task(struct task_struct *p)
    1651             : {
    1652         916 :         raw_spin_lock_init(&p->pi_lock);
    1653             : #ifdef CONFIG_RT_MUTEXES
    1654         916 :         p->pi_waiters = RB_ROOT_CACHED;
    1655         916 :         p->pi_top_task = NULL;
    1656         916 :         p->pi_blocked_on = NULL;
    1657             : #endif
    1658         916 : }
    1659             : 
    1660             : static inline void init_task_pid_links(struct task_struct *task)
    1661             : {
    1662             :         enum pid_type type;
    1663             : 
    1664        3664 :         for (type = PIDTYPE_PID; type < PIDTYPE_MAX; ++type)
    1665        3664 :                 INIT_HLIST_NODE(&task->pid_links[type]);
    1666             : }
    1667             : 
    1668             : static inline void
    1669        1832 : init_task_pid(struct task_struct *task, enum pid_type type, struct pid *pid)
    1670             : {
    1671        1832 :         if (type == PIDTYPE_PID)
    1672           3 :                 task->thread_pid = pid;
    1673             :         else
    1674           9 :                 task->signal->pids[type] = pid;
    1675             : }
    1676             : 
    1677         916 : static inline void rcu_copy_process(struct task_struct *p)
    1678             : {
    1679             : #ifdef CONFIG_PREEMPT_RCU
    1680             :         p->rcu_read_lock_nesting = 0;
    1681             :         p->rcu_read_unlock_special.s = 0;
    1682             :         p->rcu_blocked_node = NULL;
    1683             :         INIT_LIST_HEAD(&p->rcu_node_entry);
    1684             : #endif /* #ifdef CONFIG_PREEMPT_RCU */
    1685             : #ifdef CONFIG_TASKS_RCU
    1686             :         p->rcu_tasks_holdout = false;
    1687             :         INIT_LIST_HEAD(&p->rcu_tasks_holdout_list);
    1688             :         p->rcu_tasks_idle_cpu = -1;
    1689             : #endif /* #ifdef CONFIG_TASKS_RCU */
    1690             : #ifdef CONFIG_TASKS_TRACE_RCU
    1691             :         p->trc_reader_nesting = 0;
    1692             :         p->trc_reader_special.s = 0;
    1693             :         INIT_LIST_HEAD(&p->trc_holdout_list);
    1694             : #endif /* #ifdef CONFIG_TASKS_TRACE_RCU */
    1695         916 : }
    1696             : 
    1697           0 : struct pid *pidfd_pid(const struct file *file)
    1698             : {
    1699           0 :         if (file->f_op == &pidfd_fops)
    1700           0 :                 return file->private_data;
    1701             : 
    1702           0 :         return ERR_PTR(-EBADF);
    1703             : }
    1704             : 
    1705           0 : static int pidfd_release(struct inode *inode, struct file *file)
    1706             : {
    1707           0 :         struct pid *pid = file->private_data;
    1708             : 
    1709           0 :         file->private_data = NULL;
    1710           0 :         put_pid(pid);
    1711           0 :         return 0;
    1712             : }
    1713             : 
    1714             : #ifdef CONFIG_PROC_FS
    1715             : /**
    1716             :  * pidfd_show_fdinfo - print information about a pidfd
    1717             :  * @m: proc fdinfo file
    1718             :  * @f: file referencing a pidfd
    1719             :  *
    1720             :  * Pid:
    1721             :  * This function will print the pid that a given pidfd refers to in the
    1722             :  * pid namespace of the procfs instance.
    1723             :  * If the pid namespace of the process is not a descendant of the pid
    1724             :  * namespace of the procfs instance 0 will be shown as its pid. This is
    1725             :  * similar to calling getppid() on a process whose parent is outside of
    1726             :  * its pid namespace.
    1727             :  *
    1728             :  * NSpid:
    1729             :  * If pid namespaces are supported then this function will also print
    1730             :  * the pid of a given pidfd refers to for all descendant pid namespaces
    1731             :  * starting from the current pid namespace of the instance, i.e. the
    1732             :  * Pid field and the first entry in the NSpid field will be identical.
    1733             :  * If the pid namespace of the process is not a descendant of the pid
    1734             :  * namespace of the procfs instance 0 will be shown as its first NSpid
    1735             :  * entry and no others will be shown.
    1736             :  * Note that this differs from the Pid and NSpid fields in
    1737             :  * /proc/<pid>/status where Pid and NSpid are always shown relative to
    1738             :  * the  pid namespace of the procfs instance. The difference becomes
    1739             :  * obvious when sending around a pidfd between pid namespaces from a
    1740             :  * different branch of the tree, i.e. where no ancestoral relation is
    1741             :  * present between the pid namespaces:
    1742             :  * - create two new pid namespaces ns1 and ns2 in the initial pid
    1743             :  *   namespace (also take care to create new mount namespaces in the
    1744             :  *   new pid namespace and mount procfs)
    1745             :  * - create a process with a pidfd in ns1
    1746             :  * - send pidfd from ns1 to ns2
    1747             :  * - read /proc/self/fdinfo/<pidfd> and observe that both Pid and NSpid
    1748             :  *   have exactly one entry, which is 0
    1749             :  */
    1750           0 : static void pidfd_show_fdinfo(struct seq_file *m, struct file *f)
    1751             : {
    1752           0 :         struct pid *pid = f->private_data;
    1753           0 :         struct pid_namespace *ns;
    1754           0 :         pid_t nr = -1;
    1755             : 
    1756           0 :         if (likely(pid_has_task(pid, PIDTYPE_PID))) {
    1757           0 :                 ns = proc_pid_ns(file_inode(m->file)->i_sb);
    1758           0 :                 nr = pid_nr_ns(pid, ns);
    1759             :         }
    1760             : 
    1761           0 :         seq_put_decimal_ll(m, "Pid:\t", nr);
    1762             : 
    1763             : #ifdef CONFIG_PID_NS
    1764             :         seq_put_decimal_ll(m, "\nNSpid:\t", nr);
    1765             :         if (nr > 0) {
    1766             :                 int i;
    1767             : 
    1768             :                 /* If nr is non-zero it means that 'pid' is valid and that
    1769             :                  * ns, i.e. the pid namespace associated with the procfs
    1770             :                  * instance, is in the pid namespace hierarchy of pid.
    1771             :                  * Start at one below the already printed level.
    1772             :                  */
    1773             :                 for (i = ns->level + 1; i <= pid->level; i++)
    1774             :                         seq_put_decimal_ll(m, "\t", pid->numbers[i].nr);
    1775             :         }
    1776             : #endif
    1777           0 :         seq_putc(m, '\n');
    1778           0 : }
    1779             : #endif
    1780             : 
    1781             : /*
    1782             :  * Poll support for process exit notification.
    1783             :  */
    1784           0 : static __poll_t pidfd_poll(struct file *file, struct poll_table_struct *pts)
    1785             : {
    1786           0 :         struct pid *pid = file->private_data;
    1787           0 :         __poll_t poll_flags = 0;
    1788             : 
    1789           0 :         poll_wait(file, &pid->wait_pidfd, pts);
    1790             : 
    1791             :         /*
    1792             :          * Inform pollers only when the whole thread group exits.
    1793             :          * If the thread group leader exits before all other threads in the
    1794             :          * group, then poll(2) should block, similar to the wait(2) family.
    1795             :          */
    1796           0 :         if (thread_group_exited(pid))
    1797           0 :                 poll_flags = EPOLLIN | EPOLLRDNORM;
    1798             : 
    1799           0 :         return poll_flags;
    1800             : }
    1801             : 
    1802             : const struct file_operations pidfd_fops = {
    1803             :         .release = pidfd_release,
    1804             :         .poll = pidfd_poll,
    1805             : #ifdef CONFIG_PROC_FS
    1806             :         .show_fdinfo = pidfd_show_fdinfo,
    1807             : #endif
    1808             : };
    1809             : 
    1810             : static void __delayed_free_task(struct rcu_head *rhp)
    1811             : {
    1812             :         struct task_struct *tsk = container_of(rhp, struct task_struct, rcu);
    1813             : 
    1814             :         free_task(tsk);
    1815             : }
    1816             : 
    1817           0 : static __always_inline void delayed_free_task(struct task_struct *tsk)
    1818             : {
    1819           0 :         if (IS_ENABLED(CONFIG_MEMCG))
    1820             :                 call_rcu(&tsk->rcu, __delayed_free_task);
    1821             :         else
    1822           0 :                 free_task(tsk);
    1823           0 : }
    1824             : 
    1825         916 : static void copy_oom_score_adj(u64 clone_flags, struct task_struct *tsk)
    1826             : {
    1827             :         /* Skip if kernel thread */
    1828         916 :         if (!tsk->mm)
    1829             :                 return;
    1830             : 
    1831             :         /* Skip if spawning a thread or using vfork */
    1832         863 :         if ((clone_flags & (CLONE_VM | CLONE_THREAD | CLONE_VFORK)) != CLONE_VM)
    1833             :                 return;
    1834             : 
    1835             :         /* We need to synchronize with __set_oom_adj */
    1836           0 :         mutex_lock(&oom_adj_mutex);
    1837           0 :         set_bit(MMF_MULTIPROCESS, &tsk->mm->flags);
    1838             :         /* Update the values in case they were changed after copy_signal */
    1839           0 :         tsk->signal->oom_score_adj = current->signal->oom_score_adj;
    1840           0 :         tsk->signal->oom_score_adj_min = current->signal->oom_score_adj_min;
    1841           0 :         mutex_unlock(&oom_adj_mutex);
    1842             : }
    1843             : 
    1844             : /*
    1845             :  * This creates a new process as a copy of the old one,
    1846             :  * but does not actually start it yet.
    1847             :  *
    1848             :  * It copies the registers, and all the appropriate
    1849             :  * parts of the process environment (as per the clone
    1850             :  * flags). The actual kick-off is left to the caller.
    1851             :  */
    1852         916 : static __latent_entropy struct task_struct *copy_process(
    1853             :                                         struct pid *pid,
    1854             :                                         int trace,
    1855             :                                         int node,
    1856             :                                         struct kernel_clone_args *args)
    1857             : {
    1858         916 :         int pidfd = -1, retval;
    1859         916 :         struct task_struct *p;
    1860         916 :         struct multiprocess_signals delayed;
    1861         916 :         struct file *pidfile = NULL;
    1862         916 :         u64 clone_flags = args->flags;
    1863         916 :         struct nsproxy *nsp = current->nsproxy;
    1864             : 
    1865             :         /*
    1866             :          * Don't allow sharing the root directory with processes in a different
    1867             :          * namespace
    1868             :          */
    1869         916 :         if ((clone_flags & (CLONE_NEWNS|CLONE_FS)) == (CLONE_NEWNS|CLONE_FS))
    1870         916 :                 return ERR_PTR(-EINVAL);
    1871             : 
    1872         916 :         if ((clone_flags & (CLONE_NEWUSER|CLONE_FS)) == (CLONE_NEWUSER|CLONE_FS))
    1873         916 :                 return ERR_PTR(-EINVAL);
    1874             : 
    1875             :         /*
    1876             :          * Thread groups must share signals as well, and detached threads
    1877             :          * can only be started up within the thread group.
    1878             :          */
    1879         916 :         if ((clone_flags & CLONE_THREAD) && !(clone_flags & CLONE_SIGHAND))
    1880         916 :                 return ERR_PTR(-EINVAL);
    1881             : 
    1882             :         /*
    1883             :          * Shared signal handlers imply shared VM. By way of the above,
    1884             :          * thread groups also imply shared VM. Blocking this case allows
    1885             :          * for various simplifications in other code.
    1886             :          */
    1887         916 :         if ((clone_flags & CLONE_SIGHAND) && !(clone_flags & CLONE_VM))
    1888         916 :                 return ERR_PTR(-EINVAL);
    1889             : 
    1890             :         /*
    1891             :          * Siblings of global init remain as zombies on exit since they are
    1892             :          * not reaped by their parent (swapper). To solve this and to avoid
    1893             :          * multi-rooted process trees, prevent global and container-inits
    1894             :          * from creating siblings.
    1895             :          */
    1896         916 :         if ((clone_flags & CLONE_PARENT) &&
    1897           0 :                                 current->signal->flags & SIGNAL_UNKILLABLE)
    1898         916 :                 return ERR_PTR(-EINVAL);
    1899             : 
    1900             :         /*
    1901             :          * If the new process will be in a different pid or user namespace
    1902             :          * do not allow it to share a thread group with the forking task.
    1903             :          */
    1904         916 :         if (clone_flags & CLONE_THREAD) {
    1905           6 :                 if ((clone_flags & (CLONE_NEWUSER | CLONE_NEWPID)) ||
    1906           6 :                     (task_active_pid_ns(current) != nsp->pid_ns_for_children))
    1907           0 :                         return ERR_PTR(-EINVAL);
    1908             :         }
    1909             : 
    1910             :         /*
    1911             :          * If the new process will be in a different time namespace
    1912             :          * do not allow it to share VM or a thread group with the forking task.
    1913             :          */
    1914         916 :         if (clone_flags & (CLONE_THREAD | CLONE_VM)) {
    1915          59 :                 if (nsp->time_ns != nsp->time_ns_for_children)
    1916         916 :                         return ERR_PTR(-EINVAL);
    1917             :         }
    1918             : 
    1919         916 :         if (clone_flags & CLONE_PIDFD) {
    1920             :                 /*
    1921             :                  * - CLONE_DETACHED is blocked so that we can potentially
    1922             :                  *   reuse it later for CLONE_PIDFD.
    1923             :                  * - CLONE_THREAD is blocked until someone really needs it.
    1924             :                  */
    1925           0 :                 if (clone_flags & (CLONE_DETACHED | CLONE_THREAD))
    1926         916 :                         return ERR_PTR(-EINVAL);
    1927             :         }
    1928             : 
    1929             :         /*
    1930             :          * Force any signals received before this point to be delivered
    1931             :          * before the fork happens.  Collect up signals sent to multiple
    1932             :          * processes that happen during the fork and delay them so that
    1933             :          * they appear to happen after the fork.
    1934             :          */
    1935         916 :         sigemptyset(&delayed.signal);
    1936         916 :         INIT_HLIST_NODE(&delayed.node);
    1937             : 
    1938         916 :         spin_lock_irq(&current->sighand->siglock);
    1939         916 :         if (!(clone_flags & CLONE_THREAD))
    1940         910 :                 hlist_add_head(&delayed.node, &current->signal->multiprocess);
    1941         916 :         recalc_sigpending();
    1942         916 :         spin_unlock_irq(&current->sighand->siglock);
    1943         916 :         retval = -ERESTARTNOINTR;
    1944         916 :         if (signal_pending(current))
    1945           0 :                 goto fork_out;
    1946             : 
    1947         916 :         retval = -ENOMEM;
    1948         916 :         p = dup_task_struct(current, node);
    1949         916 :         if (!p)
    1950           0 :                 goto fork_out;
    1951         916 :         if (args->io_thread)
    1952           0 :                 p->flags |= PF_IO_WORKER;
    1953             : 
    1954             :         /*
    1955             :          * This _must_ happen before we call free_task(), i.e. before we jump
    1956             :          * to any of the bad_fork_* labels. This is to avoid freeing
    1957             :          * p->set_child_tid which is (ab)used as a kthread's data pointer for
    1958             :          * kernel threads (PF_KTHREAD).
    1959             :          */
    1960         916 :         p->set_child_tid = (clone_flags & CLONE_CHILD_SETTID) ? args->child_tid : NULL;
    1961             :         /*
    1962             :          * Clear TID on mm_release()?
    1963             :          */
    1964         916 :         p->clear_child_tid = (clone_flags & CLONE_CHILD_CLEARTID) ? args->child_tid : NULL;
    1965             : 
    1966         916 :         ftrace_graph_init_task(p);
    1967             : 
    1968         916 :         rt_mutex_init_task(p);
    1969             : 
    1970        1832 :         lockdep_assert_irqs_enabled();
    1971             : #ifdef CONFIG_PROVE_LOCKING
    1972         916 :         DEBUG_LOCKS_WARN_ON(!p->softirqs_enabled);
    1973             : #endif
    1974         916 :         retval = -EAGAIN;
    1975         916 :         if (atomic_read(&p->real_cred->user->processes) >=
    1976         916 :                         task_rlimit(p, RLIMIT_NPROC)) {
    1977           0 :                 if (p->real_cred->user != INIT_USER &&
    1978           0 :                     !capable(CAP_SYS_RESOURCE) && !capable(CAP_SYS_ADMIN))
    1979           0 :                         goto bad_fork_free;
    1980             :         }
    1981         916 :         current->flags &= ~PF_NPROC_EXCEEDED;
    1982             : 
    1983         916 :         retval = copy_creds(p, clone_flags);
    1984         916 :         if (retval < 0)
    1985           0 :                 goto bad_fork_free;
    1986             : 
    1987             :         /*
    1988             :          * If multiple threads are within copy_process(), then this check
    1989             :          * triggers too late. This doesn't hurt, the check is only there
    1990             :          * to stop root fork bombs.
    1991             :          */
    1992         916 :         retval = -EAGAIN;
    1993         916 :         if (data_race(nr_threads >= max_threads))
    1994           0 :                 goto bad_fork_cleanup_count;
    1995             : 
    1996         916 :         delayacct_tsk_init(p);  /* Must remain after dup_task_struct() */
    1997         916 :         p->flags &= ~(PF_SUPERPRIV | PF_WQ_WORKER | PF_IDLE);
    1998         916 :         p->flags |= PF_FORKNOEXEC;
    1999         916 :         INIT_LIST_HEAD(&p->children);
    2000         916 :         INIT_LIST_HEAD(&p->sibling);
    2001         916 :         rcu_copy_process(p);
    2002         916 :         p->vfork_done = NULL;
    2003         916 :         spin_lock_init(&p->alloc_lock);
    2004             : 
    2005         916 :         init_sigpending(&p->pending);
    2006             : 
    2007         916 :         p->utime = p->stime = p->gtime = 0;
    2008             : #ifdef CONFIG_ARCH_HAS_SCALED_CPUTIME
    2009             :         p->utimescaled = p->stimescaled = 0;
    2010             : #endif
    2011         916 :         prev_cputime_init(&p->prev_cputime);
    2012             : 
    2013             : #ifdef CONFIG_VIRT_CPU_ACCOUNTING_GEN
    2014             :         seqcount_init(&p->vtime.seqcount);
    2015             :         p->vtime.starttime = 0;
    2016             :         p->vtime.state = VTIME_INACTIVE;
    2017             : #endif
    2018             : 
    2019             : #ifdef CONFIG_IO_URING
    2020             :         p->io_uring = NULL;
    2021             : #endif
    2022             : 
    2023             : #if defined(SPLIT_RSS_COUNTING)
    2024         916 :         memset(&p->rss_stat, 0, sizeof(p->rss_stat));
    2025             : #endif
    2026             : 
    2027         916 :         p->default_timer_slack_ns = current->timer_slack_ns;
    2028             : 
    2029             : #ifdef CONFIG_PSI
    2030             :         p->psi_flags = 0;
    2031             : #endif
    2032             : 
    2033         916 :         task_io_accounting_init(&p->ioac);
    2034         916 :         acct_clear_integrals(p);
    2035             : 
    2036         916 :         posix_cputimers_init(&p->posix_cputimers);
    2037             : 
    2038         916 :         p->io_context = NULL;
    2039         916 :         audit_set_context(p, NULL);
    2040         916 :         cgroup_fork(p);
    2041             : #ifdef CONFIG_NUMA
    2042         916 :         p->mempolicy = mpol_dup(p->mempolicy);
    2043         916 :         if (IS_ERR(p->mempolicy)) {
    2044           0 :                 retval = PTR_ERR(p->mempolicy);
    2045           0 :                 p->mempolicy = NULL;
    2046           0 :                 goto bad_fork_cleanup_threadgroup_lock;
    2047             :         }
    2048             : #endif
    2049             : #ifdef CONFIG_CPUSETS
    2050             :         p->cpuset_mem_spread_rotor = NUMA_NO_NODE;
    2051             :         p->cpuset_slab_spread_rotor = NUMA_NO_NODE;
    2052             :         seqcount_spinlock_init(&p->mems_allowed_seq, &p->alloc_lock);
    2053             : #endif
    2054             : #ifdef CONFIG_TRACE_IRQFLAGS
    2055         916 :         memset(&p->irqtrace, 0, sizeof(p->irqtrace));
    2056         916 :         p->irqtrace.hardirq_disable_ip       = _THIS_IP_;
    2057         916 :         p->irqtrace.softirq_enable_ip        = _THIS_IP_;
    2058         916 :         p->softirqs_enabled          = 1;
    2059         916 :         p->softirq_context           = 0;
    2060             : #endif
    2061             : 
    2062         916 :         p->pagefault_disabled = 0;
    2063             : 
    2064             : #ifdef CONFIG_LOCKDEP
    2065         916 :         lockdep_init_task(p);
    2066             : #endif
    2067             : 
    2068             : #ifdef CONFIG_DEBUG_MUTEXES
    2069         916 :         p->blocked_on = NULL; /* not blocked yet */
    2070             : #endif
    2071             : #ifdef CONFIG_BCACHE
    2072             :         p->sequential_io     = 0;
    2073             :         p->sequential_io_avg = 0;
    2074             : #endif
    2075             : 
    2076             :         /* Perform scheduler related setup. Assign this task to a CPU. */
    2077         916 :         retval = sched_fork(clone_flags, p);
    2078         916 :         if (retval)
    2079           0 :                 goto bad_fork_cleanup_policy;
    2080             : 
    2081         916 :         retval = perf_event_init_task(p);
    2082         916 :         if (retval)
    2083           0 :                 goto bad_fork_cleanup_policy;
    2084         916 :         retval = audit_alloc(p);
    2085         916 :         if (retval)
    2086             :                 goto bad_fork_cleanup_perf;
    2087             :         /* copy all the process information */
    2088         916 :         shm_init_task(p);
    2089         916 :         retval = security_task_alloc(p, clone_flags);
    2090         916 :         if (retval)
    2091           0 :                 goto bad_fork_cleanup_audit;
    2092         916 :         retval = copy_semundo(clone_flags, p);
    2093         916 :         if (retval)
    2094           0 :                 goto bad_fork_cleanup_security;
    2095         916 :         retval = copy_files(clone_flags, p);
    2096         916 :         if (retval)
    2097           0 :                 goto bad_fork_cleanup_semundo;
    2098         916 :         retval = copy_fs(clone_flags, p);
    2099         916 :         if (retval)
    2100           0 :                 goto bad_fork_cleanup_files;
    2101         916 :         retval = copy_sighand(clone_flags, p);
    2102         916 :         if (retval)
    2103           0 :                 goto bad_fork_cleanup_fs;
    2104         916 :         retval = copy_signal(clone_flags, p);
    2105         916 :         if (retval)
    2106           0 :                 goto bad_fork_cleanup_sighand;
    2107         916 :         retval = copy_mm(clone_flags, p);
    2108         916 :         if (retval)
    2109           0 :                 goto bad_fork_cleanup_signal;
    2110         916 :         retval = copy_namespaces(clone_flags, p);
    2111         916 :         if (retval)
    2112           0 :                 goto bad_fork_cleanup_mm;
    2113         916 :         retval = copy_io(clone_flags, p);
    2114         916 :         if (retval)
    2115           0 :                 goto bad_fork_cleanup_namespaces;
    2116         916 :         retval = copy_thread(clone_flags, args->stack, args->stack_size, p, args->tls);
    2117         916 :         if (retval)
    2118           0 :                 goto bad_fork_cleanup_io;
    2119             : 
    2120         916 :         stackleak_task_init(p);
    2121             : 
    2122         916 :         if (pid != &init_struct_pid) {
    2123         913 :                 pid = alloc_pid(p->nsproxy->pid_ns_for_children, args->set_tid,
    2124             :                                 args->set_tid_size);
    2125         913 :                 if (IS_ERR(pid)) {
    2126           0 :                         retval = PTR_ERR(pid);
    2127           0 :                         goto bad_fork_cleanup_thread;
    2128             :                 }
    2129             :         }
    2130             : 
    2131             :         /*
    2132             :          * This has to happen after we've potentially unshared the file
    2133             :          * descriptor table (so that the pidfd doesn't leak into the child
    2134             :          * if the fd table isn't shared).
    2135             :          */
    2136         916 :         if (clone_flags & CLONE_PIDFD) {
    2137           0 :                 retval = get_unused_fd_flags(O_RDWR | O_CLOEXEC);
    2138           0 :                 if (retval < 0)
    2139           0 :                         goto bad_fork_free_pid;
    2140             : 
    2141           0 :                 pidfd = retval;
    2142             : 
    2143           0 :                 pidfile = anon_inode_getfile("[pidfd]", &pidfd_fops, pid,
    2144             :                                               O_RDWR | O_CLOEXEC);
    2145           0 :                 if (IS_ERR(pidfile)) {
    2146           0 :                         put_unused_fd(pidfd);
    2147           0 :                         retval = PTR_ERR(pidfile);
    2148           0 :                         goto bad_fork_free_pid;
    2149             :                 }
    2150           0 :                 get_pid(pid);   /* held by pidfile now */
    2151             : 
    2152           0 :                 retval = put_user(pidfd, args->pidfd);
    2153           0 :                 if (retval)
    2154           0 :                         goto bad_fork_put_pidfd;
    2155             :         }
    2156             : 
    2157             : #ifdef CONFIG_BLOCK
    2158         916 :         p->plug = NULL;
    2159             : #endif
    2160         916 :         futex_init_task(p);
    2161             : 
    2162             :         /*
    2163             :          * sigaltstack should be cleared when sharing the same VM
    2164             :          */
    2165         916 :         if ((clone_flags & (CLONE_VM|CLONE_VFORK)) == CLONE_VM)
    2166          59 :                 sas_ss_reset(p);
    2167             : 
    2168             :         /*
    2169             :          * Syscall tracing and stepping should be turned off in the
    2170             :          * child regardless of CLONE_PTRACE.
    2171             :          */
    2172         916 :         user_disable_single_step(p);
    2173         916 :         clear_task_syscall_work(p, SYSCALL_TRACE);
    2174             : #if defined(CONFIG_GENERIC_ENTRY) || defined(TIF_SYSCALL_EMU)
    2175         916 :         clear_task_syscall_work(p, SYSCALL_EMU);
    2176             : #endif
    2177         916 :         clear_tsk_latency_tracing(p);
    2178             : 
    2179             :         /* ok, now we should be set up.. */
    2180         916 :         p->pid = pid_nr(pid);
    2181         916 :         if (clone_flags & CLONE_THREAD) {
    2182           6 :                 p->group_leader = current->group_leader;
    2183           6 :                 p->tgid = current->tgid;
    2184             :         } else {
    2185         910 :                 p->group_leader = p;
    2186         910 :                 p->tgid = p->pid;
    2187             :         }
    2188             : 
    2189         916 :         p->nr_dirtied = 0;
    2190         916 :         p->nr_dirtied_pause = 128 >> (PAGE_SHIFT - 10);
    2191         916 :         p->dirty_paused_when = 0;
    2192             : 
    2193         916 :         p->pdeath_signal = 0;
    2194         916 :         INIT_LIST_HEAD(&p->thread_group);
    2195         916 :         p->task_works = NULL;
    2196             : 
    2197             : #ifdef CONFIG_KRETPROBES
    2198             :         p->kretprobe_instances.first = NULL;
    2199             : #endif
    2200             : 
    2201             :         /*
    2202             :          * Ensure that the cgroup subsystem policies allow the new process to be
    2203             :          * forked. It should be noted that the new process's css_set can be changed
    2204             :          * between here and cgroup_post_fork() if an organisation operation is in
    2205             :          * progress.
    2206             :          */
    2207         916 :         retval = cgroup_can_fork(p, args);
    2208         916 :         if (retval)
    2209           0 :                 goto bad_fork_put_pidfd;
    2210             : 
    2211             :         /*
    2212             :          * From this point on we must avoid any synchronous user-space
    2213             :          * communication until we take the tasklist-lock. In particular, we do
    2214             :          * not want user-space to be able to predict the process start-time by
    2215             :          * stalling fork(2) after we recorded the start_time but before it is
    2216             :          * visible to the system.
    2217             :          */
    2218             : 
    2219         916 :         p->start_time = ktime_get_ns();
    2220         916 :         p->start_boottime = ktime_get_boottime_ns();
    2221             : 
    2222             :         /*
    2223             :          * Make it visible to the rest of the system, but dont wake it up yet.
    2224             :          * Need tasklist lock for parent etc handling!
    2225             :          */
    2226         916 :         write_lock_irq(&tasklist_lock);
    2227             : 
    2228             :         /* CLONE_PARENT re-uses the old parent */
    2229         916 :         if (clone_flags & (CLONE_PARENT|CLONE_THREAD)) {
    2230           6 :                 p->real_parent = current->real_parent;
    2231           6 :                 p->parent_exec_id = current->parent_exec_id;
    2232           6 :                 if (clone_flags & CLONE_THREAD)
    2233           6 :                         p->exit_signal = -1;
    2234             :                 else
    2235           0 :                         p->exit_signal = current->group_leader->exit_signal;
    2236             :         } else {
    2237         910 :                 p->real_parent = current;
    2238         910 :                 p->parent_exec_id = current->self_exec_id;
    2239         910 :                 p->exit_signal = args->exit_signal;
    2240             :         }
    2241             : 
    2242         916 :         klp_copy_process(p);
    2243             : 
    2244         916 :         spin_lock(&current->sighand->siglock);
    2245             : 
    2246             :         /*
    2247             :          * Copy seccomp details explicitly here, in case they were changed
    2248             :          * before holding sighand lock.
    2249             :          */
    2250         916 :         copy_seccomp(p);
    2251             : 
    2252         916 :         rseq_fork(p, clone_flags);
    2253             : 
    2254             :         /* Don't start children in a dying pid namespace */
    2255        1832 :         if (unlikely(!(ns_of_pid(pid)->pid_allocated & PIDNS_ADDING))) {
    2256           0 :                 retval = -ENOMEM;
    2257           0 :                 goto bad_fork_cancel_cgroup;
    2258             :         }
    2259             : 
    2260             :         /* Let kill terminate clone/fork in the middle */
    2261         916 :         if (fatal_signal_pending(current)) {
    2262           0 :                 retval = -EINTR;
    2263           0 :                 goto bad_fork_cancel_cgroup;
    2264             :         }
    2265             : 
    2266             :         /* past the last point of failure */
    2267         916 :         if (pidfile)
    2268           0 :                 fd_install(pidfd, pidfile);
    2269             : 
    2270        4580 :         init_task_pid_links(p);
    2271         916 :         if (likely(p->pid)) {
    2272        1826 :                 ptrace_init_task(p, (clone_flags & CLONE_PTRACE) || trace);
    2273             : 
    2274         913 :                 init_task_pid(p, PIDTYPE_PID, pid);
    2275         913 :                 if (thread_group_leader(p)) {
    2276         907 :                         init_task_pid(p, PIDTYPE_TGID, pid);
    2277         907 :                         init_task_pid(p, PIDTYPE_PGID, task_pgrp(current));
    2278         907 :                         init_task_pid(p, PIDTYPE_SID, task_session(current));
    2279             : 
    2280         907 :                         if (is_child_reaper(pid)) {
    2281           1 :                                 ns_of_pid(pid)->child_reaper = p;
    2282           1 :                                 p->signal->flags |= SIGNAL_UNKILLABLE;
    2283             :                         }
    2284         907 :                         p->signal->shared_pending.signal = delayed.signal;
    2285         907 :                         p->signal->tty = tty_kref_get(current->signal->tty);
    2286             :                         /*
    2287             :                          * Inherit has_child_subreaper flag under the same
    2288             :                          * tasklist_lock with adding child to the process tree
    2289             :                          * for propagate_has_child_subreaper optimization.
    2290             :                          */
    2291         907 :                         p->signal->has_child_subreaper = p->real_parent->signal->has_child_subreaper ||
    2292             :                                                          p->real_parent->signal->is_child_subreaper;
    2293         907 :                         list_add_tail(&p->sibling, &p->real_parent->children);
    2294         907 :                         list_add_tail_rcu(&p->tasks, &init_task.tasks);
    2295         907 :                         attach_pid(p, PIDTYPE_TGID);
    2296         907 :                         attach_pid(p, PIDTYPE_PGID);
    2297         907 :                         attach_pid(p, PIDTYPE_SID);
    2298         907 :                         __this_cpu_inc(process_counts);
    2299             :                 } else {
    2300           6 :                         current->signal->nr_threads++;
    2301           6 :                         atomic_inc(&current->signal->live);
    2302           6 :                         refcount_inc(&current->signal->sigcnt);
    2303           6 :                         task_join_group_stop(p);
    2304          12 :                         list_add_tail_rcu(&p->thread_group,
    2305           6 :                                           &p->group_leader->thread_group);
    2306         919 :                         list_add_tail_rcu(&p->thread_node,
    2307           6 :                                           &p->signal->thread_head);
    2308             :                 }
    2309         913 :                 attach_pid(p, PIDTYPE_PID);
    2310         913 :                 nr_threads++;
    2311             :         }
    2312         916 :         total_forks++;
    2313         916 :         hlist_del_init(&delayed.node);
    2314         916 :         spin_unlock(&current->sighand->siglock);
    2315         916 :         syscall_tracepoint_update(p);
    2316         916 :         write_unlock_irq(&tasklist_lock);
    2317             : 
    2318         916 :         proc_fork_connector(p);
    2319         916 :         sched_post_fork(p);
    2320         916 :         cgroup_post_fork(p, args);
    2321         916 :         perf_event_fork(p);
    2322             : 
    2323         916 :         trace_task_newtask(p, clone_flags);
    2324         916 :         uprobe_copy_process(p, clone_flags);
    2325             : 
    2326         916 :         copy_oom_score_adj(clone_flags, p);
    2327             : 
    2328         916 :         return p;
    2329             : 
    2330           0 : bad_fork_cancel_cgroup:
    2331           0 :         spin_unlock(&current->sighand->siglock);
    2332           0 :         write_unlock_irq(&tasklist_lock);
    2333           0 :         cgroup_cancel_fork(p, args);
    2334           0 : bad_fork_put_pidfd:
    2335           0 :         if (clone_flags & CLONE_PIDFD) {
    2336           0 :                 fput(pidfile);
    2337           0 :                 put_unused_fd(pidfd);
    2338             :         }
    2339           0 : bad_fork_free_pid:
    2340           0 :         if (pid != &init_struct_pid)
    2341           0 :                 free_pid(pid);
    2342           0 : bad_fork_cleanup_thread:
    2343           0 :         exit_thread(p);
    2344           0 : bad_fork_cleanup_io:
    2345           0 :         if (p->io_context)
    2346           0 :                 exit_io_context(p);
    2347           0 : bad_fork_cleanup_namespaces:
    2348           0 :         exit_task_namespaces(p);
    2349           0 : bad_fork_cleanup_mm:
    2350           0 :         if (p->mm) {
    2351           0 :                 mm_clear_owner(p->mm, p);
    2352           0 :                 mmput(p->mm);
    2353             :         }
    2354           0 : bad_fork_cleanup_signal:
    2355           0 :         if (!(clone_flags & CLONE_THREAD))
    2356           0 :                 free_signal_struct(p->signal);
    2357           0 : bad_fork_cleanup_sighand:
    2358           0 :         __cleanup_sighand(p->sighand);
    2359           0 : bad_fork_cleanup_fs:
    2360           0 :         exit_fs(p); /* blocking */
    2361           0 : bad_fork_cleanup_files:
    2362           0 :         exit_files(p); /* blocking */
    2363           0 : bad_fork_cleanup_semundo:
    2364           0 :         exit_sem(p);
    2365           0 : bad_fork_cleanup_security:
    2366           0 :         security_task_free(p);
    2367           0 : bad_fork_cleanup_audit:
    2368           0 :         audit_free(p);
    2369           0 : bad_fork_cleanup_perf:
    2370           0 :         perf_event_free_task(p);
    2371           0 : bad_fork_cleanup_policy:
    2372           0 :         lockdep_free_task(p);
    2373             : #ifdef CONFIG_NUMA
    2374           0 :         mpol_put(p->mempolicy);
    2375           0 : bad_fork_cleanup_threadgroup_lock:
    2376             : #endif
    2377           0 :         delayacct_tsk_free(p);
    2378           0 : bad_fork_cleanup_count:
    2379           0 :         atomic_dec(&p->cred->user->processes);
    2380           0 :         exit_creds(p);
    2381           0 : bad_fork_free:
    2382           0 :         p->state = TASK_DEAD;
    2383           0 :         put_task_stack(p);
    2384           0 :         delayed_free_task(p);
    2385           0 : fork_out:
    2386           0 :         spin_lock_irq(&current->sighand->siglock);
    2387           0 :         hlist_del_init(&delayed.node);
    2388           0 :         spin_unlock_irq(&current->sighand->siglock);
    2389           0 :         return ERR_PTR(retval);
    2390             : }
    2391             : 
    2392           3 : static inline void init_idle_pids(struct task_struct *idle)
    2393             : {
    2394           3 :         enum pid_type type;
    2395             : 
    2396          15 :         for (type = PIDTYPE_PID; type < PIDTYPE_MAX; ++type) {
    2397          12 :                 INIT_HLIST_NODE(&idle->pid_links[type]); /* not really needed */
    2398          24 :                 init_task_pid(idle, type, &init_struct_pid);
    2399             :         }
    2400           3 : }
    2401             : 
    2402           3 : struct task_struct *fork_idle(int cpu)
    2403             : {
    2404           3 :         struct task_struct *task;
    2405           3 :         struct kernel_clone_args args = {
    2406             :                 .flags = CLONE_VM,
    2407             :         };
    2408             : 
    2409           3 :         task = copy_process(&init_struct_pid, 0, cpu_to_node(cpu), &args);
    2410           3 :         if (!IS_ERR(task)) {
    2411           3 :                 init_idle_pids(task);
    2412           3 :                 init_idle(task, cpu);
    2413             :         }
    2414             : 
    2415           3 :         return task;
    2416             : }
    2417             : 
    2418           1 : struct mm_struct *copy_init_mm(void)
    2419             : {
    2420           1 :         return dup_mm(NULL, &init_mm);
    2421             : }
    2422             : 
    2423             : /*
    2424             :  * This is like kernel_clone(), but shaved down and tailored to just
    2425             :  * creating io_uring workers. It returns a created task, or an error pointer.
    2426             :  * The returned task is inactive, and the caller must fire it up through
    2427             :  * wake_up_new_task(p). All signals are blocked in the created task.
    2428             :  */
    2429           0 : struct task_struct *create_io_thread(int (*fn)(void *), void *arg, int node)
    2430             : {
    2431           0 :         unsigned long flags = CLONE_FS|CLONE_FILES|CLONE_SIGHAND|CLONE_THREAD|
    2432             :                                 CLONE_IO;
    2433           0 :         struct kernel_clone_args args = {
    2434             :                 .flags          = ((lower_32_bits(flags) | CLONE_VM |
    2435             :                                     CLONE_UNTRACED) & ~CSIGNAL),
    2436             :                 .exit_signal    = (lower_32_bits(flags) & CSIGNAL),
    2437           0 :                 .stack          = (unsigned long)fn,
    2438           0 :                 .stack_size     = (unsigned long)arg,
    2439             :                 .io_thread      = 1,
    2440             :         };
    2441           0 :         struct task_struct *tsk;
    2442             : 
    2443           0 :         tsk = copy_process(NULL, 0, node, &args);
    2444           0 :         if (!IS_ERR(tsk)) {
    2445           0 :                 sigfillset(&tsk->blocked);
    2446           0 :                 sigdelsetmask(&tsk->blocked, sigmask(SIGKILL));
    2447           0 :                 tsk->flags |= PF_NOFREEZE;
    2448             :         }
    2449           0 :         return tsk;
    2450             : }
    2451             : 
    2452             : /*
    2453             :  *  Ok, this is the main fork-routine.
    2454             :  *
    2455             :  * It copies the process, and if successful kick-starts
    2456             :  * it and waits for it to finish using the VM if required.
    2457             :  *
    2458             :  * args->exit_signal is expected to be checked for sanity by the caller.
    2459             :  */
    2460         913 : pid_t kernel_clone(struct kernel_clone_args *args)
    2461             : {
    2462         913 :         u64 clone_flags = args->flags;
    2463         913 :         struct completion vfork;
    2464         913 :         struct pid *pid;
    2465         913 :         struct task_struct *p;
    2466         913 :         int trace = 0;
    2467         913 :         pid_t nr;
    2468             : 
    2469             :         /*
    2470             :          * For legacy clone() calls, CLONE_PIDFD uses the parent_tid argument
    2471             :          * to return the pidfd. Hence, CLONE_PIDFD and CLONE_PARENT_SETTID are
    2472             :          * mutually exclusive. With clone3() CLONE_PIDFD has grown a separate
    2473             :          * field in struct clone_args and it still doesn't make sense to have
    2474             :          * them both point at the same memory location. Performing this check
    2475             :          * here has the advantage that we don't need to have a separate helper
    2476             :          * to check for legacy clone().
    2477             :          */
    2478         913 :         if ((args->flags & CLONE_PIDFD) &&
    2479           0 :             (args->flags & CLONE_PARENT_SETTID) &&
    2480           0 :             (args->pidfd == args->parent_tid))
    2481             :                 return -EINVAL;
    2482             : 
    2483             :         /*
    2484             :          * Determine whether and which event to report to ptracer.  When
    2485             :          * called from kernel_thread or CLONE_UNTRACED is explicitly
    2486             :          * requested, no event is reported; otherwise, report if the event
    2487             :          * for the type of forking is enabled.
    2488             :          */
    2489         913 :         if (!(clone_flags & CLONE_UNTRACED)) {
    2490         863 :                 if (clone_flags & CLONE_VFORK)
    2491             :                         trace = PTRACE_EVENT_VFORK;
    2492         863 :                 else if (args->exit_signal != SIGCHLD)
    2493             :                         trace = PTRACE_EVENT_CLONE;
    2494             :                 else
    2495         857 :                         trace = PTRACE_EVENT_FORK;
    2496             : 
    2497         863 :                 if (likely(!ptrace_event_enabled(current, trace)))
    2498         863 :                         trace = 0;
    2499             :         }
    2500             : 
    2501         913 :         p = copy_process(NULL, trace, NUMA_NO_NODE, args);
    2502         913 :         add_latent_entropy();
    2503             : 
    2504         913 :         if (IS_ERR(p))
    2505           0 :                 return PTR_ERR(p);
    2506             : 
    2507             :         /*
    2508             :          * Do this prior waking up the new thread - the thread pointer
    2509             :          * might get invalid after that point, if the thread exits quickly.
    2510             :          */
    2511         913 :         trace_sched_process_fork(current, p);
    2512             : 
    2513         913 :         pid = get_task_pid(p, PIDTYPE_PID);
    2514         913 :         nr = pid_vnr(pid);
    2515             : 
    2516         913 :         if (clone_flags & CLONE_PARENT_SETTID)
    2517          10 :                 put_user(nr, args->parent_tid);
    2518             : 
    2519         913 :         if (clone_flags & CLONE_VFORK) {
    2520           0 :                 p->vfork_done = &vfork;
    2521           0 :                 init_completion(&vfork);
    2522           0 :                 get_task_struct(p);
    2523             :         }
    2524             : 
    2525         913 :         wake_up_new_task(p);
    2526             : 
    2527             :         /* forking complete and child started to run, tell ptracer */
    2528         913 :         if (unlikely(trace))
    2529           0 :                 ptrace_event_pid(trace, pid);
    2530             : 
    2531         913 :         if (clone_flags & CLONE_VFORK) {
    2532           0 :                 if (!wait_for_vfork_done(p, &vfork))
    2533           0 :                         ptrace_event_pid(PTRACE_EVENT_VFORK_DONE, pid);
    2534             :         }
    2535             : 
    2536         913 :         put_pid(pid);
    2537         913 :         return nr;
    2538             : }
    2539             : 
    2540             : /*
    2541             :  * Create a kernel thread.
    2542             :  */
    2543          50 : pid_t kernel_thread(int (*fn)(void *), void *arg, unsigned long flags)
    2544             : {
    2545          50 :         struct kernel_clone_args args = {
    2546          50 :                 .flags          = ((lower_32_bits(flags) | CLONE_VM |
    2547          50 :                                     CLONE_UNTRACED) & ~CSIGNAL),
    2548          50 :                 .exit_signal    = (lower_32_bits(flags) & CSIGNAL),
    2549          50 :                 .stack          = (unsigned long)fn,
    2550          50 :                 .stack_size     = (unsigned long)arg,
    2551             :         };
    2552             : 
    2553          50 :         return kernel_clone(&args);
    2554             : }
    2555             : 
    2556             : #ifdef __ARCH_WANT_SYS_FORK
    2557           0 : SYSCALL_DEFINE0(fork)
    2558             : {
    2559             : #ifdef CONFIG_MMU
    2560           0 :         struct kernel_clone_args args = {
    2561             :                 .exit_signal = SIGCHLD,
    2562             :         };
    2563             : 
    2564           0 :         return kernel_clone(&args);
    2565             : #else
    2566             :         /* can not support in nommu mode */
    2567             :         return -EINVAL;
    2568             : #endif
    2569             : }
    2570             : #endif
    2571             : 
    2572             : #ifdef __ARCH_WANT_SYS_VFORK
    2573           0 : SYSCALL_DEFINE0(vfork)
    2574             : {
    2575           0 :         struct kernel_clone_args args = {
    2576             :                 .flags          = CLONE_VFORK | CLONE_VM,
    2577             :                 .exit_signal    = SIGCHLD,
    2578             :         };
    2579             : 
    2580           0 :         return kernel_clone(&args);
    2581             : }
    2582             : #endif
    2583             : 
    2584             : #ifdef __ARCH_WANT_SYS_CLONE
    2585             : #ifdef CONFIG_CLONE_BACKWARDS
    2586             : SYSCALL_DEFINE5(clone, unsigned long, clone_flags, unsigned long, newsp,
    2587             :                  int __user *, parent_tidptr,
    2588             :                  unsigned long, tls,
    2589             :                  int __user *, child_tidptr)
    2590             : #elif defined(CONFIG_CLONE_BACKWARDS2)
    2591             : SYSCALL_DEFINE5(clone, unsigned long, newsp, unsigned long, clone_flags,
    2592             :                  int __user *, parent_tidptr,
    2593             :                  int __user *, child_tidptr,
    2594             :                  unsigned long, tls)
    2595             : #elif defined(CONFIG_CLONE_BACKWARDS3)
    2596             : SYSCALL_DEFINE6(clone, unsigned long, clone_flags, unsigned long, newsp,
    2597             :                 int, stack_size,
    2598             :                 int __user *, parent_tidptr,
    2599             :                 int __user *, child_tidptr,
    2600             :                 unsigned long, tls)
    2601             : #else
    2602        1726 : SYSCALL_DEFINE5(clone, unsigned long, clone_flags, unsigned long, newsp,
    2603             :                  int __user *, parent_tidptr,
    2604             :                  int __user *, child_tidptr,
    2605             :                  unsigned long, tls)
    2606             : #endif
    2607             : {
    2608         863 :         struct kernel_clone_args args = {
    2609         863 :                 .flags          = (lower_32_bits(clone_flags) & ~CSIGNAL),
    2610             :                 .pidfd          = parent_tidptr,
    2611             :                 .child_tid      = child_tidptr,
    2612             :                 .parent_tid     = parent_tidptr,
    2613         863 :                 .exit_signal    = (lower_32_bits(clone_flags) & CSIGNAL),
    2614             :                 .stack          = newsp,
    2615             :                 .tls            = tls,
    2616             :         };
    2617             : 
    2618         863 :         return kernel_clone(&args);
    2619             : }
    2620             : #endif
    2621             : 
    2622             : #ifdef __ARCH_WANT_SYS_CLONE3
    2623             : 
    2624           0 : noinline static int copy_clone_args_from_user(struct kernel_clone_args *kargs,
    2625             :                                               struct clone_args __user *uargs,
    2626             :                                               size_t usize)
    2627             : {
    2628           0 :         int err;
    2629           0 :         struct clone_args args;
    2630           0 :         pid_t *kset_tid = kargs->set_tid;
    2631             : 
    2632           0 :         BUILD_BUG_ON(offsetofend(struct clone_args, tls) !=
    2633             :                      CLONE_ARGS_SIZE_VER0);
    2634           0 :         BUILD_BUG_ON(offsetofend(struct clone_args, set_tid_size) !=
    2635             :                      CLONE_ARGS_SIZE_VER1);
    2636           0 :         BUILD_BUG_ON(offsetofend(struct clone_args, cgroup) !=
    2637             :                      CLONE_ARGS_SIZE_VER2);
    2638           0 :         BUILD_BUG_ON(sizeof(struct clone_args) != CLONE_ARGS_SIZE_VER2);
    2639             : 
    2640           0 :         if (unlikely(usize > PAGE_SIZE))
    2641             :                 return -E2BIG;
    2642           0 :         if (unlikely(usize < CLONE_ARGS_SIZE_VER0))
    2643             :                 return -EINVAL;
    2644             : 
    2645           0 :         err = copy_struct_from_user(&args, sizeof(args), uargs, usize);
    2646           0 :         if (err)
    2647           0 :                 return err;
    2648             : 
    2649           0 :         if (unlikely(args.set_tid_size > MAX_PID_NS_LEVEL))
    2650             :                 return -EINVAL;
    2651             : 
    2652           0 :         if (unlikely(!args.set_tid && args.set_tid_size > 0))
    2653             :                 return -EINVAL;
    2654             : 
    2655           0 :         if (unlikely(args.set_tid && args.set_tid_size == 0))
    2656             :                 return -EINVAL;
    2657             : 
    2658             :         /*
    2659             :          * Verify that higher 32bits of exit_signal are unset and that
    2660             :          * it is a valid signal
    2661             :          */
    2662           0 :         if (unlikely((args.exit_signal & ~((u64)CSIGNAL)) ||
    2663             :                      !valid_signal(args.exit_signal)))
    2664             :                 return -EINVAL;
    2665             : 
    2666           0 :         if ((args.flags & CLONE_INTO_CGROUP) &&
    2667           0 :             (args.cgroup > INT_MAX || usize < CLONE_ARGS_SIZE_VER2))
    2668             :                 return -EINVAL;
    2669             : 
    2670           0 :         *kargs = (struct kernel_clone_args){
    2671             :                 .flags          = args.flags,
    2672           0 :                 .pidfd          = u64_to_user_ptr(args.pidfd),
    2673           0 :                 .child_tid      = u64_to_user_ptr(args.child_tid),
    2674           0 :                 .parent_tid     = u64_to_user_ptr(args.parent_tid),
    2675             :                 .exit_signal    = args.exit_signal,
    2676           0 :                 .stack          = args.stack,
    2677           0 :                 .stack_size     = args.stack_size,
    2678           0 :                 .tls            = args.tls,
    2679             :                 .set_tid_size   = args.set_tid_size,
    2680           0 :                 .cgroup         = args.cgroup,
    2681             :         };
    2682             : 
    2683           0 :         if (args.set_tid &&
    2684           0 :                 copy_from_user(kset_tid, u64_to_user_ptr(args.set_tid),
    2685             :                         (kargs->set_tid_size * sizeof(pid_t))))
    2686             :                 return -EFAULT;
    2687             : 
    2688           0 :         kargs->set_tid = kset_tid;
    2689             : 
    2690           0 :         return 0;
    2691             : }
    2692             : 
    2693             : /**
    2694             :  * clone3_stack_valid - check and prepare stack
    2695             :  * @kargs: kernel clone args
    2696             :  *
    2697             :  * Verify that the stack arguments userspace gave us are sane.
    2698             :  * In addition, set the stack direction for userspace since it's easy for us to
    2699             :  * determine.
    2700             :  */
    2701           0 : static inline bool clone3_stack_valid(struct kernel_clone_args *kargs)
    2702             : {
    2703           0 :         if (kargs->stack == 0) {
    2704           0 :                 if (kargs->stack_size > 0)
    2705           0 :                         return false;
    2706             :         } else {
    2707           0 :                 if (kargs->stack_size == 0)
    2708             :                         return false;
    2709             : 
    2710           0 :                 if (!access_ok((void __user *)kargs->stack, kargs->stack_size))
    2711             :                         return false;
    2712             : 
    2713             : #if !defined(CONFIG_STACK_GROWSUP) && !defined(CONFIG_IA64)
    2714           0 :                 kargs->stack += kargs->stack_size;
    2715             : #endif
    2716             :         }
    2717             : 
    2718             :         return true;
    2719             : }
    2720             : 
    2721           0 : static bool clone3_args_valid(struct kernel_clone_args *kargs)
    2722             : {
    2723             :         /* Verify that no unknown flags are passed along. */
    2724           0 :         if (kargs->flags &
    2725             :             ~(CLONE_LEGACY_FLAGS | CLONE_CLEAR_SIGHAND | CLONE_INTO_CGROUP))
    2726             :                 return false;
    2727             : 
    2728             :         /*
    2729             :          * - make the CLONE_DETACHED bit reuseable for clone3
    2730             :          * - make the CSIGNAL bits reuseable for clone3
    2731             :          */
    2732           0 :         if (kargs->flags & (CLONE_DETACHED | CSIGNAL))
    2733             :                 return false;
    2734             : 
    2735           0 :         if ((kargs->flags & (CLONE_SIGHAND | CLONE_CLEAR_SIGHAND)) ==
    2736             :             (CLONE_SIGHAND | CLONE_CLEAR_SIGHAND))
    2737             :                 return false;
    2738             : 
    2739           0 :         if ((kargs->flags & (CLONE_THREAD | CLONE_PARENT)) &&
    2740           0 :             kargs->exit_signal)
    2741             :                 return false;
    2742             : 
    2743           0 :         if (!clone3_stack_valid(kargs))
    2744           0 :                 return false;
    2745             : 
    2746             :         return true;
    2747             : }
    2748             : 
    2749             : /**
    2750             :  * clone3 - create a new process with specific properties
    2751             :  * @uargs: argument structure
    2752             :  * @size:  size of @uargs
    2753             :  *
    2754             :  * clone3() is the extensible successor to clone()/clone2().
    2755             :  * It takes a struct as argument that is versioned by its size.
    2756             :  *
    2757             :  * Return: On success, a positive PID for the child process.
    2758             :  *         On error, a negative errno number.
    2759             :  */
    2760           0 : SYSCALL_DEFINE2(clone3, struct clone_args __user *, uargs, size_t, size)
    2761             : {
    2762           0 :         int err;
    2763             : 
    2764           0 :         struct kernel_clone_args kargs;
    2765           0 :         pid_t set_tid[MAX_PID_NS_LEVEL];
    2766             : 
    2767           0 :         kargs.set_tid = set_tid;
    2768             : 
    2769           0 :         err = copy_clone_args_from_user(&kargs, uargs, size);
    2770           0 :         if (err)
    2771           0 :                 return err;
    2772             : 
    2773           0 :         if (!clone3_args_valid(&kargs))
    2774             :                 return -EINVAL;
    2775             : 
    2776           0 :         return kernel_clone(&kargs);
    2777             : }
    2778             : #endif
    2779             : 
    2780           1 : void walk_process_tree(struct task_struct *top, proc_visitor visitor, void *data)
    2781             : {
    2782           1 :         struct task_struct *leader, *parent, *child;
    2783           1 :         int res;
    2784             : 
    2785           1 :         read_lock(&tasklist_lock);
    2786           1 :         leader = top = top->group_leader;
    2787           2 : down:
    2788           4 :         for_each_thread(leader, parent) {
    2789           3 :                 list_for_each_entry(child, &parent->children, sibling) {
    2790           1 :                         res = visitor(child, data);
    2791           1 :                         if (res) {
    2792           1 :                                 if (res < 0)
    2793           0 :                                         goto out;
    2794           1 :                                 leader = child;
    2795           1 :                                 goto down;
    2796             :                         }
    2797           0 : up:
    2798           1 :                         ;
    2799             :                 }
    2800             :         }
    2801             : 
    2802           2 :         if (leader != top) {
    2803           1 :                 child = leader;
    2804           1 :                 parent = child->real_parent;
    2805           1 :                 leader = parent->group_leader;
    2806           1 :                 goto up;
    2807             :         }
    2808           1 : out:
    2809           1 :         read_unlock(&tasklist_lock);
    2810           1 : }
    2811             : 
    2812             : #ifndef ARCH_MIN_MMSTRUCT_ALIGN
    2813             : #define ARCH_MIN_MMSTRUCT_ALIGN 0
    2814             : #endif
    2815             : 
    2816         117 : static void sighand_ctor(void *data)
    2817             : {
    2818         117 :         struct sighand_struct *sighand = data;
    2819             : 
    2820         117 :         spin_lock_init(&sighand->siglock);
    2821         117 :         init_waitqueue_head(&sighand->signalfd_wqh);
    2822         117 : }
    2823             : 
    2824           1 : void __init proc_caches_init(void)
    2825             : {
    2826           1 :         unsigned int mm_size;
    2827             : 
    2828           1 :         sighand_cachep = kmem_cache_create("sighand_cache",
    2829             :                         sizeof(struct sighand_struct), 0,
    2830             :                         SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_TYPESAFE_BY_RCU|
    2831             :                         SLAB_ACCOUNT, sighand_ctor);
    2832           1 :         signal_cachep = kmem_cache_create("signal_cache",
    2833             :                         sizeof(struct signal_struct), 0,
    2834             :                         SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_ACCOUNT,
    2835             :                         NULL);
    2836           1 :         files_cachep = kmem_cache_create("files_cache",
    2837             :                         sizeof(struct files_struct), 0,
    2838             :                         SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_ACCOUNT,
    2839             :                         NULL);
    2840           1 :         fs_cachep = kmem_cache_create("fs_cache",
    2841             :                         sizeof(struct fs_struct), 0,
    2842             :                         SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_ACCOUNT,
    2843             :                         NULL);
    2844             : 
    2845             :         /*
    2846             :          * The mm_cpumask is located at the end of mm_struct, and is
    2847             :          * dynamically sized based on the maximum CPU number this system
    2848             :          * can have, taking hotplug into account (nr_cpu_ids).
    2849             :          */
    2850           1 :         mm_size = sizeof(struct mm_struct) + cpumask_size();
    2851             : 
    2852           1 :         mm_cachep = kmem_cache_create_usercopy("mm_struct",
    2853             :                         mm_size, ARCH_MIN_MMSTRUCT_ALIGN,
    2854             :                         SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_ACCOUNT,
    2855             :                         offsetof(struct mm_struct, saved_auxv),
    2856             :                         sizeof_field(struct mm_struct, saved_auxv),
    2857             :                         NULL);
    2858           1 :         vm_area_cachep = KMEM_CACHE(vm_area_struct, SLAB_PANIC|SLAB_ACCOUNT);
    2859           1 :         mmap_init();
    2860           1 :         nsproxy_cache_init();
    2861           1 : }
    2862             : 
    2863             : /*
    2864             :  * Check constraints on flags passed to the unshare system call.
    2865             :  */
    2866          50 : static int check_unshare_flags(unsigned long unshare_flags)
    2867             : {
    2868          50 :         if (unshare_flags & ~(CLONE_THREAD|CLONE_FS|CLONE_NEWNS|CLONE_SIGHAND|
    2869             :                                 CLONE_VM|CLONE_FILES|CLONE_SYSVSEM|
    2870             :                                 CLONE_NEWUTS|CLONE_NEWIPC|CLONE_NEWNET|
    2871             :                                 CLONE_NEWUSER|CLONE_NEWPID|CLONE_NEWCGROUP|
    2872             :                                 CLONE_NEWTIME))
    2873             :                 return -EINVAL;
    2874             :         /*
    2875             :          * Not implemented, but pretend it works if there is nothing
    2876             :          * to unshare.  Note that unsharing the address space or the
    2877             :          * signal handlers also need to unshare the signal queues (aka
    2878             :          * CLONE_THREAD).
    2879             :          */
    2880          50 :         if (unshare_flags & (CLONE_THREAD | CLONE_SIGHAND | CLONE_VM)) {
    2881           0 :                 if (!thread_group_empty(current))
    2882             :                         return -EINVAL;
    2883             :         }
    2884          50 :         if (unshare_flags & (CLONE_SIGHAND | CLONE_VM)) {
    2885           0 :                 if (refcount_read(&current->sighand->count) > 1)
    2886             :                         return -EINVAL;
    2887             :         }
    2888          50 :         if (unshare_flags & CLONE_VM) {
    2889           0 :                 if (!current_is_single_threaded())
    2890           0 :                         return -EINVAL;
    2891             :         }
    2892             : 
    2893             :         return 0;
    2894             : }
    2895             : 
    2896             : /*
    2897             :  * Unshare the filesystem structure if it is being shared
    2898             :  */
    2899          50 : static int unshare_fs(unsigned long unshare_flags, struct fs_struct **new_fsp)
    2900             : {
    2901          50 :         struct fs_struct *fs = current->fs;
    2902             : 
    2903          50 :         if (!(unshare_flags & CLONE_FS) || !fs)
    2904             :                 return 0;
    2905             : 
    2906             :         /* don't need lock here; in the worst case we'll do useless copy */
    2907          50 :         if (fs->users == 1)
    2908             :                 return 0;
    2909             : 
    2910           1 :         *new_fsp = copy_fs_struct(fs);
    2911           1 :         if (!*new_fsp)
    2912           0 :                 return -ENOMEM;
    2913             : 
    2914             :         return 0;
    2915             : }
    2916             : 
    2917             : /*
    2918             :  * Unshare file descriptor table if it is being shared
    2919             :  */
    2920         601 : int unshare_fd(unsigned long unshare_flags, unsigned int max_fds,
    2921             :                struct files_struct **new_fdp)
    2922             : {
    2923         601 :         struct files_struct *fd = current->files;
    2924         601 :         int error = 0;
    2925             : 
    2926         601 :         if ((unshare_flags & CLONE_FILES) &&
    2927         551 :             (fd && atomic_read(&fd->count) > 1)) {
    2928           0 :                 *new_fdp = dup_fd(fd, max_fds, &error);
    2929           0 :                 if (!*new_fdp)
    2930           0 :                         return error;
    2931             :         }
    2932             : 
    2933             :         return 0;
    2934             : }
    2935             : 
    2936             : /*
    2937             :  * unshare allows a process to 'unshare' part of the process
    2938             :  * context which was originally shared using clone.  copy_*
    2939             :  * functions used by kernel_clone() cannot be used here directly
    2940             :  * because they modify an inactive task_struct that is being
    2941             :  * constructed. Here we are modifying the current, active,
    2942             :  * task_struct.
    2943             :  */
    2944          50 : int ksys_unshare(unsigned long unshare_flags)
    2945             : {
    2946          50 :         struct fs_struct *fs, *new_fs = NULL;
    2947          50 :         struct files_struct *fd, *new_fd = NULL;
    2948          50 :         struct cred *new_cred = NULL;
    2949          50 :         struct nsproxy *new_nsproxy = NULL;
    2950          50 :         int do_sysvsem = 0;
    2951          50 :         int err;
    2952             : 
    2953             :         /*
    2954             :          * If unsharing a user namespace must also unshare the thread group
    2955             :          * and unshare the filesystem root and working directories.
    2956             :          */
    2957          50 :         if (unshare_flags & CLONE_NEWUSER)
    2958           0 :                 unshare_flags |= CLONE_THREAD | CLONE_FS;
    2959             :         /*
    2960             :          * If unsharing vm, must also unshare signal handlers.
    2961             :          */
    2962          50 :         if (unshare_flags & CLONE_VM)
    2963           0 :                 unshare_flags |= CLONE_SIGHAND;
    2964             :         /*
    2965             :          * If unsharing a signal handlers, must also unshare the signal queues.
    2966             :          */
    2967          50 :         if (unshare_flags & CLONE_SIGHAND)
    2968           0 :                 unshare_flags |= CLONE_THREAD;
    2969             :         /*
    2970             :          * If unsharing namespace, must also unshare filesystem information.
    2971             :          */
    2972          50 :         if (unshare_flags & CLONE_NEWNS)
    2973          50 :                 unshare_flags |= CLONE_FS;
    2974             : 
    2975          50 :         err = check_unshare_flags(unshare_flags);
    2976          50 :         if (err)
    2977           0 :                 goto bad_unshare_out;
    2978             :         /*
    2979             :          * CLONE_NEWIPC must also detach from the undolist: after switching
    2980             :          * to a new ipc namespace, the semaphore arrays from the old
    2981             :          * namespace are unreachable.
    2982             :          */
    2983          50 :         if (unshare_flags & (CLONE_NEWIPC|CLONE_SYSVSEM))
    2984           0 :                 do_sysvsem = 1;
    2985          50 :         err = unshare_fs(unshare_flags, &new_fs);
    2986          50 :         if (err)
    2987           0 :                 goto bad_unshare_out;
    2988          50 :         err = unshare_fd(unshare_flags, NR_OPEN_MAX, &new_fd);
    2989          50 :         if (err)
    2990           0 :                 goto bad_unshare_cleanup_fs;
    2991          50 :         err = unshare_userns(unshare_flags, &new_cred);
    2992          50 :         if (err)
    2993           0 :                 goto bad_unshare_cleanup_fd;
    2994          50 :         err = unshare_nsproxy_namespaces(unshare_flags, &new_nsproxy,
    2995             :                                          new_cred, new_fs);
    2996          50 :         if (err)
    2997           0 :                 goto bad_unshare_cleanup_cred;
    2998             : 
    2999          50 :         if (new_fs || new_fd || do_sysvsem || new_cred || new_nsproxy) {
    3000          50 :                 if (do_sysvsem) {
    3001             :                         /*
    3002             :                          * CLONE_SYSVSEM is equivalent to sys_exit().
    3003             :                          */
    3004           0 :                         exit_sem(current);
    3005             :                 }
    3006          50 :                 if (unshare_flags & CLONE_NEWIPC) {
    3007             :                         /* Orphan segments in old ns (see sem above). */
    3008           0 :                         exit_shm(current);
    3009           0 :                         shm_init_task(current);
    3010             :                 }
    3011             : 
    3012          50 :                 if (new_nsproxy)
    3013          50 :                         switch_task_namespaces(current, new_nsproxy);
    3014             : 
    3015          50 :                 task_lock(current);
    3016             : 
    3017          50 :                 if (new_fs) {
    3018           1 :                         fs = current->fs;
    3019           1 :                         spin_lock(&fs->lock);
    3020           1 :                         current->fs = new_fs;
    3021           1 :                         if (--fs->users)
    3022           1 :                                 new_fs = NULL;
    3023             :                         else
    3024           0 :                                 new_fs = fs;
    3025           1 :                         spin_unlock(&fs->lock);
    3026             :                 }
    3027             : 
    3028          50 :                 if (new_fd) {
    3029           0 :                         fd = current->files;
    3030           0 :                         current->files = new_fd;
    3031           0 :                         new_fd = fd;
    3032             :                 }
    3033             : 
    3034          50 :                 task_unlock(current);
    3035             : 
    3036          50 :                 if (new_cred) {
    3037             :                         /* Install the new user namespace */
    3038             :                         commit_creds(new_cred);
    3039             :                         new_cred = NULL;
    3040             :                 }
    3041             :         }
    3042             : 
    3043          50 :         perf_event_namespaces(current);
    3044             : 
    3045             : bad_unshare_cleanup_cred:
    3046             :         if (new_cred)
    3047             :                 put_cred(new_cred);
    3048          50 : bad_unshare_cleanup_fd:
    3049          50 :         if (new_fd)
    3050           0 :                 put_files_struct(new_fd);
    3051             : 
    3052          50 : bad_unshare_cleanup_fs:
    3053          50 :         if (new_fs)
    3054           0 :                 free_fs_struct(new_fs);
    3055             : 
    3056          50 : bad_unshare_out:
    3057          50 :         return err;
    3058             : }
    3059             : 
    3060          98 : SYSCALL_DEFINE1(unshare, unsigned long, unshare_flags)
    3061             : {
    3062          49 :         return ksys_unshare(unshare_flags);
    3063             : }
    3064             : 
    3065             : /*
    3066             :  *      Helper to unshare the files of the current task.
    3067             :  *      We don't want to expose copy_files internals to
    3068             :  *      the exec layer of the kernel.
    3069             :  */
    3070             : 
    3071         551 : int unshare_files(void)
    3072             : {
    3073         551 :         struct task_struct *task = current;
    3074         551 :         struct files_struct *old, *copy = NULL;
    3075         551 :         int error;
    3076             : 
    3077         551 :         error = unshare_fd(CLONE_FILES, NR_OPEN_MAX, &copy);
    3078         551 :         if (error || !copy)
    3079             :                 return error;
    3080             : 
    3081           0 :         old = task->files;
    3082           0 :         task_lock(task);
    3083           0 :         task->files = copy;
    3084           0 :         task_unlock(task);
    3085           0 :         put_files_struct(old);
    3086           0 :         return 0;
    3087             : }
    3088             : 
    3089           8 : int sysctl_max_threads(struct ctl_table *table, int write,
    3090             :                        void *buffer, size_t *lenp, loff_t *ppos)
    3091             : {
    3092           8 :         struct ctl_table t;
    3093           8 :         int ret;
    3094           8 :         int threads = max_threads;
    3095           8 :         int min = 1;
    3096           8 :         int max = MAX_THREADS;
    3097             : 
    3098           8 :         t = *table;
    3099           8 :         t.data = &threads;
    3100           8 :         t.extra1 = &min;
    3101           8 :         t.extra2 = &max;
    3102             : 
    3103           8 :         ret = proc_dointvec_minmax(&t, write, buffer, lenp, ppos);
    3104           8 :         if (ret || !write)
    3105             :                 return ret;
    3106             : 
    3107           0 :         max_threads = threads;
    3108             : 
    3109           0 :         return 0;
    3110             : }

Generated by: LCOV version 1.14