projects/linux·blob v7.3-rc6 · refs

3427 lines · 84.3 KiB

1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * linux/kernel/fork.c
4 *
5 * Copyright (C) 1991, 1992 Linus Torvalds
6 */
7
8/*
9 * 'fork.c' contains the help-routines for the 'fork' system call
10 * (see also entry.S and others).
11 * Fork is rather simple, once you get the hang of it, but the memory
12 * management can be a bitch. See 'mm/memory.c': 'copy_page_range()'
13 */
14
15#include <linux/anon_inodes.h>
16#include <linux/slab.h>
17#include <linux/sched/autogroup.h>
18#include <linux/sched/mm.h>
19#include <linux/sched/user.h>
20#include <linux/sched/numa_balancing.h>
21#include <linux/sched/stat.h>
22#include <linux/sched/task.h>
23#include <linux/sched/task_stack.h>
24#include <linux/sched/cputime.h>
25#include <linux/sched/ext.h>
26#include <linux/sched/exec_state.h>
27#include <linux/seq_file.h>
28#include <linux/rtmutex.h>
29#include <linux/init.h>
30#include <linux/unistd.h>
31#include <linux/module.h>
32#include <linux/vmalloc.h>
33#include <linux/completion.h>
34#include <linux/personality.h>
35#include <linux/mempolicy.h>
36#include <linux/sem.h>
37#include <linux/file.h>
38#include <linux/fdtable.h>
39#include <linux/iocontext.h>
40#include <linux/key.h>
41#include <linux/kmsan.h>
42#include <linux/binfmts.h>
43#include <linux/mman.h>
44#include <linux/mmu_notifier.h>
45#include <linux/fs.h>
46#include <linux/mm.h>
47#include <linux/mm_inline.h>
48#include <linux/memblock.h>
49#include <linux/nsproxy.h>
50#include <linux/ns/ns_common_types.h>
51#include <linux/capability.h>
52#include <linux/cpu.h>
53#include <linux/cgroup.h>
54#include <linux/security.h>
55#include <linux/hugetlb.h>
56#include <linux/seccomp.h>
57#include <linux/swap.h>
58#include <linux/syscalls.h>
59#include <linux/syscall_user_dispatch.h>
60#include <linux/jiffies.h>
61#include <linux/futex.h>
62#include <linux/compat.h>
63#include <linux/kthread.h>
64#include <linux/task_io_accounting_ops.h>
65#include <linux/rcupdate.h>
66#include <linux/ptrace.h>
67#include <linux/mount.h>
68#include <linux/audit.h>
69#include <linux/memcontrol.h>
70#include <linux/ftrace.h>
71#include <linux/proc_fs.h>
72#include <linux/profile.h>
73#include <linux/rmap.h>
74#include <linux/ksm.h>
75#include <linux/acct.h>
76#include <linux/userfaultfd_k.h>
77#include <linux/tsacct_kern.h>
78#include <linux/cn_proc.h>
79#include <linux/freezer.h>
80#include <linux/delayacct.h>
81#include <linux/taskstats_kern.h>
82#include <linux/tty.h>
83#include <linux/fs_struct.h>
84#include <linux/magic.h>
85#include <linux/perf_event.h>
86#include <linux/posix-timers.h>
87#include <linux/user-return-notifier.h>
88#include <linux/oom.h>
89#include <linux/khugepaged.h>
90#include <linux/signalfd.h>
91#include <linux/uprobes.h>
92#include <linux/aio.h>
93#include <linux/compiler.h>
94#include <linux/sysctl.h>
95#include <linux/kcov.h>
96#include <linux/livepatch.h>
97#include <linux/thread_info.h>
98#include <linux/kstack_erase.h>
99#include <linux/kasan.h>
100#include <linux/randomize_kstack.h>
101#include <linux/scs.h>
102#include <linux/io_uring.h>
103#include <linux/io_uring_types.h>
104#include <linux/bpf.h>
105#include <linux/stackprotector.h>
106#include <linux/user_events.h>
107#include <linux/iommu.h>
108#include <linux/rseq.h>
109#include <uapi/linux/pidfd.h>
110#include <linux/pidfs.h>
111#include <linux/tick.h>
112#include <linux/unwind_deferred.h>
113#include <linux/pgalloc.h>
114#include <linux/uaccess.h>
115
116#include <asm/mmu_context.h>
117#include <asm/cacheflush.h>
118#include <asm/tlbflush.h>
119
120/* For dup_mmap(). */
121#include "../mm/internal.h"
122
123#include <trace/events/sched.h>
124
125#define CREATE_TRACE_POINTS
126#include <trace/events/task.h>
127
128#include <kunit/visibility.h>
129
130/*
131 * Minimum number of threads to boot the kernel
132 */
133#define MIN_THREADS 20
134
135/*
136 * Maximum number of threads
137 */
138#define MAX_THREADS FUTEX_TID_MASK
139
140/*
141 * Protected counters by write_lock_irq(&tasklist_lock)
142 */
143unsigned long total_forks; /* Handle normal Linux uptimes. */
144int nr_threads; /* The idle threads do not count.. */
145
146static int max_threads __read_mostly; /* tunable limit on nr_threads */
147
148#define NAMED_ARRAY_INDEX(x) [x] = __stringify(x)
149
150static const char * const resident_page_types[] = {
151 NAMED_ARRAY_INDEX(MM_FILEPAGES),
152 NAMED_ARRAY_INDEX(MM_ANONPAGES),
153 NAMED_ARRAY_INDEX(MM_SWAPENTS),
154 NAMED_ARRAY_INDEX(MM_SHMEMPAGES),
155};
156
157DEFINE_PER_CPU(unsigned long, process_counts) = 0;
158
159__cacheline_aligned DEFINE_RWLOCK(tasklist_lock); /* outer */
160
161#ifdef CONFIG_PROVE_RCU
162int lockdep_tasklist_lock_is_held(void)
163{
164 return lockdep_is_held(&tasklist_lock);
165}
166EXPORT_SYMBOL_GPL(lockdep_tasklist_lock_is_held);
167#endif /* #ifdef CONFIG_PROVE_RCU */
168
169int nr_processes(void)
170{
171 int cpu;
172 int total = 0;
173
174 for_each_possible_cpu(cpu)
175 total += per_cpu(process_counts, cpu);
176
177 return total;
178}
179
180void __weak arch_release_task_struct(struct task_struct *tsk)
181{
182}
183
184static struct kmem_cache *task_struct_cachep;
185
186static inline struct task_struct *alloc_task_struct_node(int node)
187{
188 return kmem_cache_alloc_node(task_struct_cachep, GFP_KERNEL, node);
189}
190
191static inline void free_task_struct(struct task_struct *tsk)
192{
193 kmem_cache_free(task_struct_cachep, tsk);
194}
195
196#ifdef CONFIG_VMAP_STACK
197/*
198 * vmalloc() is a bit slow, and calling vfree() enough times will force a TLB
199 * flush. Try to minimize the number of calls by caching stacks.
200 */
201#define NR_CACHED_STACKS 2
202static DEFINE_PER_CPU(struct vm_struct *, cached_stacks[NR_CACHED_STACKS]);
203/*
204 * Allocated stacks are cached and later reused by new threads, so memcg
205 * accounting is performed by the code assigning/releasing stacks to tasks.
206 * We need a zeroed memory without __GFP_ACCOUNT.
207 */
208#define GFP_VMAP_STACK (GFP_KERNEL | __GFP_ZERO | __GFP_SKIP_KASAN)
209
210struct vm_stack {
211 struct rcu_head rcu;
212 struct vm_struct *stack_vm_area;
213};
214
215static struct vm_struct *alloc_thread_stack_node_from_cache(struct task_struct *tsk, int node)
216{
217 struct vm_struct *vm_area;
218 unsigned int i;
219
220 /*
221 * If the node has memory, we are guaranteed the stacks are backed by local pages.
222 * Otherwise the pages are arbitrary.
223 *
224 * Note that depending on cpuset it is possible we will get migrated to a different
225 * node immediately after allocating here, so this does *not* guarantee locality for
226 * arbitrary callers.
227 */
228 scoped_guard(preempt) {
229 if (node != NUMA_NO_NODE && numa_node_id() != node)
230 return NULL;
231
232 for (i = 0; i < NR_CACHED_STACKS; i++) {
233 vm_area = this_cpu_xchg(cached_stacks[i], NULL);
234 if (vm_area)
235 return vm_area;
236 }
237 }
238
239 return NULL;
240}
241
242static bool try_release_thread_stack_to_cache(struct vm_struct *vm_area)
243{
244 unsigned int i;
245 int nid;
246
247 /*
248 * Don't cache stacks if any of the pages don't match the local domain, unless
249 * there is no local memory to begin with.
250 *
251 * Note that lack of local memory does not automatically mean it makes no difference
252 * performance-wise which other domain backs the stack. In this case we are merely
253 * trying to avoid constantly going to vmalloc.
254 */
255 scoped_guard(preempt) {
256 nid = numa_node_id();
257 if (node_state(nid, N_MEMORY)) {
258 for (i = 0; i < vm_area->nr_pages; i++) {
259 struct page *page = vm_area->pages[i];
260 if (page_to_nid(page) != nid)
261 return false;
262 }
263 }
264
265 for (i = 0; i < NR_CACHED_STACKS; i++) {
266 struct vm_struct *tmp = NULL;
267
268 if (this_cpu_try_cmpxchg(cached_stacks[i], &tmp, vm_area))
269 return true;
270 }
271 }
272 return false;
273}
274
275static void thread_stack_free_rcu(struct rcu_head *rh)
276{
277 struct vm_stack *vm_stack = container_of(rh, struct vm_stack, rcu);
278 struct vm_struct *vm_area = vm_stack->stack_vm_area;
279
280 if (try_release_thread_stack_to_cache(vm_stack->stack_vm_area))
281 return;
282
283 vfree(vm_area->addr);
284}
285
286static void thread_stack_delayed_free(struct task_struct *tsk)
287{
288 struct vm_stack *vm_stack = tsk->stack;
289
290 vm_stack->stack_vm_area = tsk->stack_vm_area;
291 call_rcu(&vm_stack->rcu, thread_stack_free_rcu);
292}
293
294static int free_vm_stack_cache(unsigned int cpu)
295{
296 struct vm_struct **cached_vm_stack_areas = per_cpu_ptr(cached_stacks, cpu);
297 int i;
298
299 for (i = 0; i < NR_CACHED_STACKS; i++) {
300 struct vm_struct *vm_area = cached_vm_stack_areas[i];
301
302 if (!vm_area)
303 continue;
304
305 vfree(vm_area->addr);
306 cached_vm_stack_areas[i] = NULL;
307 }
308
309 return 0;
310}
311
312static int memcg_charge_kernel_stack(struct vm_struct *vm_area)
313{
314 int i;
315 int ret;
316 int nr_charged = 0;
317
318 BUG_ON(vm_area->nr_pages != THREAD_SIZE / PAGE_SIZE);
319
320 for (i = 0; i < THREAD_SIZE / PAGE_SIZE; i++) {
321 ret = memcg_kmem_charge_page(vm_area->pages[i], GFP_KERNEL, 0);
322 if (ret)
323 goto err;
324 nr_charged++;
325 }
326 return 0;
327err:
328 for (i = 0; i < nr_charged; i++)
329 memcg_kmem_uncharge_page(vm_area->pages[i], 0);
330 return ret;
331}
332
333static int alloc_thread_stack_node(struct task_struct *tsk, int node)
334{
335 struct vm_struct *vm_area;
336 void *stack;
337
338 vm_area = alloc_thread_stack_node_from_cache(tsk, node);
339 if (vm_area) {
340 if (memcg_charge_kernel_stack(vm_area)) {
341 vfree(vm_area->addr);
342 return -ENOMEM;
343 }
344
345 /* Reset stack metadata. */
346 if (!kasan_hw_tags_enabled())
347 kasan_unpoison_range(vm_area->addr, THREAD_SIZE);
348
349 stack = kasan_reset_tag(vm_area->addr);
350
351 /* Clear stale pointers from reused stack. */
352 clear_pages(vm_area->addr, vm_area->nr_pages);
353
354 tsk->stack_vm_area = vm_area;
355 tsk->stack = stack;
356 return 0;
357 }
358
359 stack = __vmalloc_node(THREAD_SIZE, THREAD_ALIGN,
360 GFP_VMAP_STACK,
361 node, __builtin_return_address(0));
362 if (!stack)
363 return -ENOMEM;
364
365 vm_area = find_vm_area(stack);
366 if (memcg_charge_kernel_stack(vm_area)) {
367 vfree(stack);
368 return -ENOMEM;
369 }
370 /*
371 * We can't call find_vm_area() in interrupt context, and
372 * free_thread_stack() can be called in interrupt context,
373 * so cache the vm_struct.
374 */
375 tsk->stack_vm_area = vm_area;
376 stack = kasan_reset_tag(stack);
377 tsk->stack = stack;
378 return 0;
379}
380
381static void free_thread_stack(struct task_struct *tsk)
382{
383 if (!try_release_thread_stack_to_cache(tsk->stack_vm_area))
384 thread_stack_delayed_free(tsk);
385
386 tsk->stack = NULL;
387 tsk->stack_vm_area = NULL;
388}
389
390#else /* !CONFIG_VMAP_STACK */
391
392/*
393 * Allocate pages if THREAD_SIZE is >= PAGE_SIZE, otherwise use a
394 * kmemcache based allocator.
395 */
396#if THREAD_SIZE >= PAGE_SIZE
397
398static void thread_stack_free_rcu(struct rcu_head *rh)
399{
400 __free_pages(virt_to_page(rh), THREAD_SIZE_ORDER);
401}
402
403static void thread_stack_delayed_free(struct task_struct *tsk)
404{
405 struct rcu_head *rh = tsk->stack;
406
407 call_rcu(rh, thread_stack_free_rcu);
408}
409
410static int alloc_thread_stack_node(struct task_struct *tsk, int node)
411{
412 struct page *page = alloc_pages_node(node, THREADINFO_GFP,
413 THREAD_SIZE_ORDER);
414
415 if (likely(page)) {
416 tsk->stack = kasan_reset_tag(page_address(page));
417 return 0;
418 }
419 return -ENOMEM;
420}
421
422static void free_thread_stack(struct task_struct *tsk)
423{
424 thread_stack_delayed_free(tsk);
425 tsk->stack = NULL;
426}
427
428#else /* !(THREAD_SIZE >= PAGE_SIZE) */
429
430static struct kmem_cache *thread_stack_cache;
431
432static void thread_stack_free_rcu(struct rcu_head *rh)
433{
434 kmem_cache_free(thread_stack_cache, rh);
435}
436
437static void thread_stack_delayed_free(struct task_struct *tsk)
438{
439 struct rcu_head *rh = tsk->stack;
440
441 call_rcu(rh, thread_stack_free_rcu);
442}
443
444static int alloc_thread_stack_node(struct task_struct *tsk, int node)
445{
446 unsigned long *stack;
447 stack = kmem_cache_alloc_node(thread_stack_cache, THREADINFO_GFP, node);
448 stack = kasan_reset_tag(stack);
449 tsk->stack = stack;
450 return stack ? 0 : -ENOMEM;
451}
452
453static void free_thread_stack(struct task_struct *tsk)
454{
455 thread_stack_delayed_free(tsk);
456 tsk->stack = NULL;
457}
458
459void thread_stack_cache_init(void)
460{
461 thread_stack_cache = kmem_cache_create_usercopy("thread_stack",
462 THREAD_SIZE, THREAD_SIZE, 0, 0,
463 THREAD_SIZE, NULL);
464 BUG_ON(thread_stack_cache == NULL);
465}
466
467#endif /* THREAD_SIZE >= PAGE_SIZE */
468#endif /* CONFIG_VMAP_STACK */
469
470/* SLAB cache for signal_struct structures (tsk->signal) */
471static struct kmem_cache *signal_cachep;
472
473/* SLAB cache for sighand_struct structures (tsk->sighand) */
474struct kmem_cache *sighand_cachep;
475
476/* SLAB cache for files_struct structures (tsk->files) */
477struct kmem_cache *files_cachep;
478
479/* SLAB cache for fs_struct structures (tsk->fs) */
480struct kmem_cache *fs_cachep;
481
482/* SLAB cache for mm_struct structures (tsk->mm) */
483static struct kmem_cache *mm_cachep;
484
485static void account_kernel_stack(struct task_struct *tsk, int account)
486{
487 if (IS_ENABLED(CONFIG_VMAP_STACK)) {
488 struct vm_struct *vm_area = task_stack_vm_area(tsk);
489 int i;
490
491 for (i = 0; i < THREAD_SIZE / PAGE_SIZE; i++)
492 mod_lruvec_page_state(vm_area->pages[i], NR_KERNEL_STACK_KB,
493 account * (PAGE_SIZE / 1024));
494 } else {
495 void *stack = task_stack_page(tsk);
496
497 /* All stack pages are in the same node. */
498 mod_lruvec_kmem_state(stack, NR_KERNEL_STACK_KB,
499 account * (THREAD_SIZE / 1024));
500 }
501}
502
503void exit_task_stack_account(struct task_struct *tsk)
504{
505 account_kernel_stack(tsk, -1);
506
507 if (IS_ENABLED(CONFIG_VMAP_STACK)) {
508 struct vm_struct *vm_area;
509 int i;
510
511 vm_area = task_stack_vm_area(tsk);
512 for (i = 0; i < THREAD_SIZE / PAGE_SIZE; i++)
513 memcg_kmem_uncharge_page(vm_area->pages[i], 0);
514 }
515}
516
517static void release_task_stack(struct task_struct *tsk)
518{
519 if (WARN_ON(READ_ONCE(tsk->__state) != TASK_DEAD))
520 return; /* Better to leak the stack than to free prematurely */
521
522 free_thread_stack(tsk);
523}
524
525#ifdef CONFIG_THREAD_INFO_IN_TASK
526void put_task_stack(struct task_struct *tsk)
527{
528 if (refcount_dec_and_test(&tsk->stack_refcount))
529 release_task_stack(tsk);
530}
531#endif
532
533void free_task(struct task_struct *tsk)
534{
535#ifdef CONFIG_SECCOMP
536 WARN_ON_ONCE(tsk->seccomp.filter);
537#endif
538 release_user_cpus_ptr(tsk);
539 scs_release(tsk);
540 smp_task_ipi_mask_free(tsk);
541
542#ifndef CONFIG_THREAD_INFO_IN_TASK
543 /*
544 * The task is finally done with both the stack and thread_info,
545 * so free both.
546 */
547 release_task_stack(tsk);
548#else
549 /*
550 * If the task had a separate stack allocation, it should be gone
551 * by now.
552 */
553 WARN_ON_ONCE(refcount_read(&tsk->stack_refcount) != 0);
554#endif
555 rt_mutex_debug_task_free(tsk);
556 ftrace_graph_exit_task(tsk);
557 arch_release_task_struct(tsk);
558 if (tsk->flags & PF_KTHREAD)
559 free_kthread_struct(tsk);
560 bpf_task_storage_free(tsk);
561 put_task_exec_state(rcu_access_pointer(tsk->exec_state));
562 free_task_struct(tsk);
563}
564EXPORT_SYMBOL(free_task);
565
566void dup_mm_exe_file(struct mm_struct *mm, struct mm_struct *oldmm)
567{
568 struct file *exe_file;
569
570 exe_file = get_mm_exe_file(oldmm);
571 RCU_INIT_POINTER(mm->exe_file, exe_file);
572 /*
573 * We depend on the oldmm having properly denied write access to the
574 * exe_file already.
575 */
576 if (exe_file && exe_file_deny_write_access(exe_file))
577 pr_warn_once("exe_file_deny_write_access() failed in %s\n", __func__);
578}
579
580#ifdef CONFIG_MMU
581static inline int mm_alloc_pgd(struct mm_struct *mm)
582{
583 mm->pgd = pgd_alloc(mm);
584 if (unlikely(!mm->pgd))
585 return -ENOMEM;
586 return 0;
587}
588
589static inline void mm_free_pgd(struct mm_struct *mm)
590{
591 pgd_free(mm, mm->pgd);
592}
593#else
594#define mm_alloc_pgd(mm) (0)
595#define mm_free_pgd(mm)
596#endif /* CONFIG_MMU */
597
598#ifdef CONFIG_MM_ID
599static DEFINE_IDA(mm_ida);
600
601static inline int mm_alloc_id(struct mm_struct *mm)
602{
603 int ret;
604
605 ret = ida_alloc_range(&mm_ida, MM_ID_MIN, MM_ID_MAX, GFP_KERNEL);
606 if (ret < 0)
607 return ret;
608 mm->mm_id = ret;
609 return 0;
610}
611
612static inline void mm_free_id(struct mm_struct *mm)
613{
614 const mm_id_t id = mm->mm_id;
615
616 mm->mm_id = MM_ID_DUMMY;
617 if (id == MM_ID_DUMMY)
618 return;
619 if (WARN_ON_ONCE(id < MM_ID_MIN || id > MM_ID_MAX))
620 return;
621 ida_free(&mm_ida, id);
622}
623#else /* !CONFIG_MM_ID */
624static inline int mm_alloc_id(struct mm_struct *mm) { return 0; }
625static inline void mm_free_id(struct mm_struct *mm) {}
626#endif /* CONFIG_MM_ID */
627
628static void check_mm(struct mm_struct *mm)
629{
630 int i;
631
632 BUILD_BUG_ON_MSG(ARRAY_SIZE(resident_page_types) != NR_MM_COUNTERS,
633 "Please make sure 'struct resident_page_types[]' is updated as well");
634
635 for (i = 0; i < NR_MM_COUNTERS; i++) {
636 long x = percpu_counter_sum(&mm->rss_stat[i]);
637
638 if (unlikely(x)) {
639 pr_alert("BUG: Bad rss-counter state mm:%p type:%s val:%ld Comm:%s Pid:%d\n",
640 mm, resident_page_types[i], x,
641 current->comm,
642 task_pid_nr(current));
643 }
644 }
645
646 if (mm_pgtables_bytes(mm))
647 pr_alert("BUG: non-zero pgtables_bytes on freeing mm: %ld\n",
648 mm_pgtables_bytes(mm));
649
650#if defined(CONFIG_TRANSPARENT_HUGEPAGE) && !defined(CONFIG_SPLIT_PMD_PTLOCKS)
651 VM_BUG_ON_MM(mm->pmd_huge_pte, mm);
652#endif
653}
654
655#define allocate_mm() (kmem_cache_alloc(mm_cachep, GFP_KERNEL))
656#define free_mm(mm) (kmem_cache_free(mm_cachep, (mm)))
657
658static void do_check_lazy_tlb(void *arg)
659{
660 struct mm_struct *mm = arg;
661
662 WARN_ON_ONCE(current->active_mm == mm);
663}
664
665static void do_shoot_lazy_tlb(void *arg)
666{
667 struct mm_struct *mm = arg;
668
669 if (current->active_mm == mm) {
670 WARN_ON_ONCE(current->mm);
671 current->active_mm = &init_mm;
672 switch_mm(mm, &init_mm, current);
673 }
674}
675
676static void cleanup_lazy_tlbs(struct mm_struct *mm)
677{
678 if (!IS_ENABLED(CONFIG_MMU_LAZY_TLB_SHOOTDOWN)) {
679 /*
680 * In this case, lazy tlb mms are refounted and would not reach
681 * __mmdrop until all CPUs have switched away and mmdrop()ed.
682 */
683 return;
684 }
685
686 /*
687 * Lazy mm shootdown does not refcount "lazy tlb mm" usage, rather it
688 * requires lazy mm users to switch to another mm when the refcount
689 * drops to zero, before the mm is freed. This requires IPIs here to
690 * switch kernel threads to init_mm.
691 *
692 * archs that use IPIs to flush TLBs can piggy-back that lazy tlb mm
693 * switch with the final userspace teardown TLB flush which leaves the
694 * mm lazy on this CPU but no others, reducing the need for additional
695 * IPIs here. There are cases where a final IPI is still required here,
696 * such as the final mmdrop being performed on a different CPU than the
697 * one exiting, or kernel threads using the mm when userspace exits.
698 *
699 * IPI overheads have not found to be expensive, but they could be
700 * reduced in a number of possible ways, for example (roughly
701 * increasing order of complexity):
702 * - The last lazy reference created by exit_mm() could instead switch
703 * to init_mm, however it's probable this will run on the same CPU
704 * immediately afterwards, so this may not reduce IPIs much.
705 * - A batch of mms requiring IPIs could be gathered and freed at once.
706 * - CPUs store active_mm where it can be remotely checked without a
707 * lock, to filter out false-positives in the cpumask.
708 * - After mm_users or mm_count reaches zero, switching away from the
709 * mm could clear mm_cpumask to reduce some IPIs, perhaps together
710 * with some batching or delaying of the final IPIs.
711 * - A delayed freeing and RCU-like quiescing sequence based on mm
712 * switching to avoid IPIs completely.
713 */
714 on_each_cpu_mask(mm_cpumask(mm), do_shoot_lazy_tlb, (void *)mm, 1);
715 if (IS_ENABLED(CONFIG_DEBUG_VM_SHOOT_LAZIES))
716 on_each_cpu(do_check_lazy_tlb, (void *)mm, 1);
717}
718
719/*
720 * Called when the last reference to the mm
721 * is dropped: either by a lazy thread or by
722 * mmput. Free the page directory and the mm.
723 */
724void __mmdrop(struct mm_struct *mm)
725{
726 BUG_ON(mm == &init_mm);
727 WARN_ON_ONCE(mm == current->mm);
728
729 /* Ensure no CPUs are using this as their lazy tlb mm */
730 cleanup_lazy_tlbs(mm);
731
732 WARN_ON_ONCE(mm == current->active_mm);
733 mm_destroy_sched(mm);
734 mm_free_pgd(mm);
735 mm_free_id(mm);
736 destroy_context(mm);
737 mmu_notifier_subscriptions_destroy(mm);
738 check_mm(mm);
739 mm_pasid_drop(mm);
740 mm_destroy_cid(mm);
741 percpu_counter_destroy_many(mm->rss_stat, NR_MM_COUNTERS);
742
743 free_mm(mm);
744}
745EXPORT_SYMBOL_GPL(__mmdrop);
746
747static void mmdrop_async_fn(struct work_struct *work)
748{
749 struct mm_struct *mm;
750
751 mm = container_of(work, struct mm_struct, async_put_work);
752 __mmdrop(mm);
753}
754
755static void mmdrop_async(struct mm_struct *mm)
756{
757 if (unlikely(atomic_dec_and_test(&mm->mm_count))) {
758 INIT_WORK(&mm->async_put_work, mmdrop_async_fn);
759 schedule_work(&mm->async_put_work);
760 }
761}
762
763static inline void free_signal_struct(struct signal_struct *sig)
764{
765 taskstats_tgid_free(sig);
766 sched_autogroup_exit(sig);
767 /*
768 * __mmdrop is not safe to call from softirq context on x86 due to
769 * pgd_dtor so postpone it to the async context
770 */
771 if (sig->oom_mm)
772 mmdrop_async(sig->oom_mm);
773 kmem_cache_free(signal_cachep, sig);
774}
775
776static inline void put_signal_struct(struct signal_struct *sig)
777{
778 if (refcount_dec_and_test(&sig->sigcnt))
779 free_signal_struct(sig);
780}
781
782void __put_task_struct(struct task_struct *tsk)
783{
784 WARN_ON(!tsk->exit_state);
785 WARN_ON(refcount_read(&tsk->usage));
786 WARN_ON(tsk == current);
787
788 unwind_task_free(tsk);
789 io_uring_free(tsk);
790 cgroup_task_free(tsk);
791 task_numa_free(tsk, true);
792 security_task_free(tsk);
793 exit_creds(tsk);
794 delayacct_tsk_free(tsk);
795 put_signal_struct(tsk->signal);
796 sched_core_free(tsk);
797 free_task(tsk);
798}
799EXPORT_SYMBOL_GPL(__put_task_struct);
800
801void __put_task_struct_rcu_cb(struct rcu_head *rhp)
802{
803 struct task_struct *task = container_of(rhp, struct task_struct, rcu);
804
805 __put_task_struct(task);
806}
807EXPORT_SYMBOL_GPL(__put_task_struct_rcu_cb);
808
809void __init __weak arch_task_cache_init(void) { }
810
811/*
812 * set_max_threads
813 */
814static void __init set_max_threads(unsigned int max_threads_suggested)
815{
816 u64 threads;
817 unsigned long nr_pages = memblock_estimated_nr_free_pages();
818
819 /*
820 * The number of threads shall be limited such that the thread
821 * structures may only consume a small part of the available memory.
822 */
823 if (fls64(nr_pages) + fls64(PAGE_SIZE) > 64)
824 threads = MAX_THREADS;
825 else
826 threads = div64_u64((u64) nr_pages * (u64) PAGE_SIZE,
827 (u64) THREAD_SIZE * 8UL);
828
829 if (threads > max_threads_suggested)
830 threads = max_threads_suggested;
831
832 max_threads = clamp_t(u64, threads, MIN_THREADS, MAX_THREADS);
833}
834
835#ifdef CONFIG_ARCH_WANTS_DYNAMIC_TASK_STRUCT
836/* Initialized by the architecture: */
837int arch_task_struct_size __read_mostly;
838#endif
839
840static void __init task_struct_whitelist(unsigned long *offset, unsigned long *size)
841{
842 /* Fetch thread_struct whitelist for the architecture. */
843 arch_thread_struct_whitelist(offset, size);
844
845 /*
846 * Handle zero-sized whitelist or empty thread_struct, otherwise
847 * adjust offset to position of thread_struct in task_struct.
848 */
849 if (unlikely(*size == 0))
850 *offset = 0;
851 else
852 *offset += offsetof(struct task_struct, thread);
853}
854
855void __init fork_init(void)
856{
857 int i;
858#ifndef ARCH_MIN_TASKALIGN
859#define ARCH_MIN_TASKALIGN 0
860#endif
861 int align = max_t(int, L1_CACHE_BYTES, ARCH_MIN_TASKALIGN);
862 unsigned long useroffset, usersize;
863
864 /* create a slab on which task_structs can be allocated */
865 task_struct_whitelist(&useroffset, &usersize);
866 task_struct_cachep = kmem_cache_create_usercopy("task_struct",
867 arch_task_struct_size, align,
868 SLAB_PANIC|SLAB_ACCOUNT,
869 useroffset, usersize, NULL);
870
871 /* do the arch specific task caches init */
872 arch_task_cache_init();
873
874 set_max_threads(MAX_THREADS);
875
876 init_task.signal->rlim[RLIMIT_NPROC].rlim_cur = max_threads/2;
877 init_task.signal->rlim[RLIMIT_NPROC].rlim_max = max_threads/2;
878 init_task.signal->rlim[RLIMIT_SIGPENDING] =
879 init_task.signal->rlim[RLIMIT_NPROC];
880
881 for (i = 0; i < UCOUNT_COUNTS; i++)
882 init_user_ns.ucount_max[i] = max_threads/2;
883
884 set_userns_rlimit_max(&init_user_ns, UCOUNT_RLIMIT_NPROC, RLIM_INFINITY);
885 set_userns_rlimit_max(&init_user_ns, UCOUNT_RLIMIT_MSGQUEUE, RLIM_INFINITY);
886 set_userns_rlimit_max(&init_user_ns, UCOUNT_RLIMIT_SIGPENDING, RLIM_INFINITY);
887 set_userns_rlimit_max(&init_user_ns, UCOUNT_RLIMIT_MEMLOCK, RLIM_INFINITY);
888
889#ifdef CONFIG_VMAP_STACK
890 cpuhp_setup_state(CPUHP_BP_PREPARE_DYN, "fork:vm_stack_cache",
891 NULL, free_vm_stack_cache);
892#endif
893
894 scs_init();
895
896 lockdep_init_task(&init_task);
897 uprobes_init();
898}
899
900int __weak arch_dup_task_struct(struct task_struct *dst,
901 struct task_struct *src)
902{
903 *dst = *src;
904 return 0;
905}
906
907void set_task_stack_end_magic(struct task_struct *tsk)
908{
909 unsigned long *stackend;
910
911 stackend = end_of_stack(tsk);
912 *stackend = STACK_END_MAGIC; /* for overflow detection */
913}
914
915static struct task_struct *dup_task_struct(struct task_struct *orig, int node)
916{
917 struct task_struct *tsk;
918 int err;
919
920 if (node == NUMA_NO_NODE)
921 node = tsk_fork_get_node(orig);
922 tsk = alloc_task_struct_node(node);
923 if (!tsk)
924 return NULL;
925
926 err = arch_dup_task_struct(tsk, orig);
927 if (err)
928 goto free_tsk;
929
930 err = alloc_thread_stack_node(tsk, node);
931 if (err)
932 goto free_tsk;
933
934#ifdef CONFIG_THREAD_INFO_IN_TASK
935 refcount_set(&tsk->stack_refcount, 1);
936#endif
937 account_kernel_stack(tsk, 1);
938
939 err = smp_task_ipi_mask_alloc(tsk);
940 if (err)
941 goto free_stack;
942
943 err = scs_prepare(tsk, node);
944 if (err)
945 goto free_ipi_mask;
946
947#ifdef CONFIG_SECCOMP
948 /*
949 * We must handle setting up seccomp filters once we're under
950 * the sighand lock in case orig has changed between now and
951 * then. Until then, filter must be NULL to avoid messing up
952 * the usage counts on the error path calling free_task.
953 */
954 tsk->seccomp.filter = NULL;
955#endif
956
957 RCU_INIT_POINTER(tsk->exec_state, NULL);
958
959 setup_thread_stack(tsk, orig);
960 clear_user_return_notifier(tsk);
961 clear_tsk_need_resched(tsk);
962 set_task_stack_end_magic(tsk);
963 clear_syscall_work_syscall_user_dispatch(tsk);
964
965#ifdef CONFIG_STACKPROTECTOR
966 tsk->stack_canary = get_random_canary();
967#endif
968 if (orig->cpus_ptr == &orig->cpus_mask)
969 tsk->cpus_ptr = &tsk->cpus_mask;
970 dup_user_cpus_ptr(tsk, orig, node);
971
972 /*
973 * One for the user space visible state that goes away when reaped.
974 * One for the scheduler.
975 */
976 refcount_set(&tsk->rcu_users, 2);
977 /* One for the rcu users */
978 refcount_set(&tsk->usage, 1);
979#ifdef CONFIG_BLK_DEV_IO_TRACE
980 tsk->btrace_seq = 0;
981#endif
982 tsk->splice_pipe = NULL;
983 tsk->task_frag.page = NULL;
984 tsk->wake_q.next = NULL;
985 tsk->worker_private = NULL;
986
987 kcov_task_init(tsk);
988 kmsan_task_create(tsk);
989 kmap_local_fork(tsk);
990
991#ifdef CONFIG_FAULT_INJECTION
992 tsk->fail_nth = 0;
993#endif
994
995#ifdef CONFIG_BLK_CGROUP
996 tsk->throttle_disk = NULL;
997 tsk->use_memdelay = 0;
998#endif
999
1000#ifdef CONFIG_ARCH_HAS_CPU_PASID
1001 tsk->pasid_activated = 0;
1002#endif
1003
1004#ifdef CONFIG_MEMCG
1005 tsk->active_memcg = NULL;
1006#endif
1007
1008#ifdef CONFIG_X86_BUS_LOCK_DETECT
1009 tsk->reported_split_lock = 0;
1010#endif
1011
1012#ifdef CONFIG_SCHED_MM_CID
1013 tsk->mm_cid.cid = MM_CID_UNSET;
1014 tsk->mm_cid.active = 0;
1015 INIT_HLIST_NODE(&tsk->mm_cid.node);
1016#endif
1017
1018#ifdef CONFIG_BPF_SYSCALL
1019 RCU_INIT_POINTER(tsk->bpf_storage, NULL);
1020 tsk->bpf_ctx = NULL;
1021#endif
1022 return tsk;
1023
1024free_ipi_mask:
1025 smp_task_ipi_mask_free(tsk);
1026free_stack:
1027 exit_task_stack_account(tsk);
1028 free_thread_stack(tsk);
1029free_tsk:
1030 free_task_struct(tsk);
1031 return NULL;
1032}
1033
1034__cacheline_aligned_in_smp DEFINE_SPINLOCK(mmlist_lock);
1035
1036static unsigned long coredump_filter = MMF_DUMP_FILTER_DEFAULT;
1037
1038static int __init coredump_filter_setup(char *s)
1039{
1040 if (kstrtoul(s, 0, &coredump_filter))
1041 return 0;
1042 coredump_filter <<= MMF_DUMP_FILTER_SHIFT;
1043 coredump_filter &= MMF_DUMP_FILTER_MASK;
1044 return 1;
1045}
1046
1047__setup("coredump_filter=", coredump_filter_setup);
1048
1049#include <linux/init_task.h>
1050
1051static void mm_init_aio(struct mm_struct *mm)
1052{
1053#ifdef CONFIG_AIO
1054 spin_lock_init(&mm->ioctx_lock);
1055 mm->ioctx_table = NULL;
1056#endif
1057}
1058
1059static __always_inline void mm_clear_owner(struct mm_struct *mm,
1060 struct task_struct *p)
1061{
1062#ifdef CONFIG_MEMCG
1063 if (mm->owner == p)
1064 WRITE_ONCE(mm->owner, NULL);
1065#endif
1066}
1067
1068static void mm_init_owner(struct mm_struct *mm, struct task_struct *p)
1069{
1070#ifdef CONFIG_MEMCG
1071 mm->owner = p;
1072#endif
1073}
1074
1075static void mm_init_uprobes_state(struct mm_struct *mm)
1076{
1077#ifdef CONFIG_UPROBES
1078 mm->uprobes_state.xol_area = NULL;
1079#endif
1080}
1081
1082static void mmap_init_lock(struct mm_struct *mm)
1083{
1084 init_rwsem(&mm->mmap_lock);
1085 mm_lock_seqcount_init(mm);
1086#ifdef CONFIG_PER_VMA_LOCK
1087 rcuwait_init(&mm->vma_writer_wait);
1088#endif
1089}
1090
1091static struct mm_struct *mm_init(struct mm_struct *mm, struct task_struct *p)
1092{
1093 mt_init_flags(&mm->mm_mt, MM_MT_FLAGS);
1094 mt_set_external_lock(&mm->mm_mt, &mm->mmap_lock);
1095 atomic_set(&mm->mm_users, 1);
1096 atomic_set(&mm->mm_count, 1);
1097 seqcount_init(&mm->write_protect_seq);
1098 mmap_init_lock(mm);
1099 INIT_LIST_HEAD(&mm->mmlist);
1100 mm_pgtables_bytes_init(mm);
1101 mm->map_count = 0;
1102 mm->locked_vm = 0;
1103 atomic64_set(&mm->pinned_vm, 0);
1104 memset(&mm->rss_stat, 0, sizeof(mm->rss_stat));
1105 spin_lock_init(&mm->page_table_lock);
1106 spin_lock_init(&mm->arg_lock);
1107 mm_init_cpumask(mm);
1108 mm_init_aio(mm);
1109 mm_init_owner(mm, p);
1110 mm_pasid_init(mm);
1111 RCU_INIT_POINTER(mm->exe_file, NULL);
1112 mmu_notifier_subscriptions_init(mm);
1113 init_tlb_flush_pending(mm);
1114#if defined(CONFIG_TRANSPARENT_HUGEPAGE) && !defined(CONFIG_SPLIT_PMD_PTLOCKS)
1115 mm->pmd_huge_pte = NULL;
1116#endif
1117 mm_init_uprobes_state(mm);
1118 hugetlb_count_init(mm);
1119 futex_mm_init(mm);
1120
1121 mm_flags_clear_all(mm);
1122 if (current->mm) {
1123 unsigned long flags = __mm_flags_get_word(current->mm);
1124
1125 __mm_flags_overwrite_word(mm, mmf_init_legacy_flags(flags));
1126 mm->def_flags = current->mm->def_flags & VM_INIT_DEF_MASK;
1127 } else {
1128 __mm_flags_overwrite_word(mm, coredump_filter);
1129 mm->def_flags = 0;
1130 }
1131
1132 if (mm_alloc_pgd(mm))
1133 goto fail_mm_init;
1134
1135 if (mm_alloc_id(mm))
1136 goto fail_noid;
1137
1138 if (init_new_context(p, mm))
1139 goto fail_nocontext;
1140
1141 if (mm_alloc_cid(mm, p))
1142 goto fail_cid;
1143
1144 if (mm_alloc_sched(mm))
1145 goto fail_sched;
1146
1147 if (percpu_counter_init_many(mm->rss_stat, 0, GFP_KERNEL_ACCOUNT,
1148 NR_MM_COUNTERS))
1149 goto fail_pcpu;
1150
1151 lru_gen_init_mm(mm);
1152 return mm;
1153
1154fail_pcpu:
1155 mm_destroy_sched(mm);
1156fail_sched:
1157 mm_destroy_cid(mm);
1158fail_cid:
1159 destroy_context(mm);
1160fail_nocontext:
1161 mm_free_id(mm);
1162fail_noid:
1163 mm_free_pgd(mm);
1164fail_mm_init:
1165 free_mm(mm);
1166 return NULL;
1167}
1168
1169/*
1170 * Allocate and initialize an mm_struct.
1171 */
1172struct mm_struct *mm_alloc(void)
1173{
1174 struct mm_struct *mm;
1175
1176 mm = allocate_mm();
1177 if (!mm)
1178 return NULL;
1179
1180 memset(mm, 0, sizeof(*mm));
1181 return mm_init(mm, current);
1182}
1183EXPORT_SYMBOL_IF_KUNIT(mm_alloc);
1184
1185static inline void __mmput(struct mm_struct *mm)
1186{
1187 VM_BUG_ON(atomic_read(&mm->mm_users));
1188
1189 uprobe_clear_state(mm);
1190 exit_aio(mm);
1191 ksm_exit(mm);
1192 khugepaged_exit(mm); /* must run before exit_mmap */
1193 exit_mmap(mm);
1194 mm_put_huge_zero_folio(mm);
1195 set_mm_exe_file(mm, NULL);
1196 if (!list_empty(&mm->mmlist)) {
1197 spin_lock(&mmlist_lock);
1198 list_del(&mm->mmlist);
1199 spin_unlock(&mmlist_lock);
1200 }
1201 if (mm->binfmt)
1202 module_put(mm->binfmt->module);
1203 lru_gen_del_mm(mm);
1204 futex_hash_free(mm);
1205 mmdrop(mm);
1206}
1207
1208/*
1209 * Decrement the use count and release all resources for an mm.
1210 */
1211void mmput(struct mm_struct *mm)
1212{
1213 might_sleep();
1214
1215 if (atomic_dec_and_test(&mm->mm_users))
1216 __mmput(mm);
1217}
1218EXPORT_SYMBOL_GPL(mmput);
1219
1220#if defined(CONFIG_MMU) || defined(CONFIG_FUTEX_PRIVATE_HASH)
1221static void mmput_async_fn(struct work_struct *work)
1222{
1223 struct mm_struct *mm = container_of(work, struct mm_struct,
1224 async_put_work);
1225
1226 __mmput(mm);
1227}
1228
1229void mmput_async(struct mm_struct *mm)
1230{
1231 if (atomic_dec_and_test(&mm->mm_users)) {
1232 INIT_WORK(&mm->async_put_work, mmput_async_fn);
1233 schedule_work(&mm->async_put_work);
1234 }
1235}
1236EXPORT_SYMBOL_GPL(mmput_async);
1237#endif
1238
1239/**
1240 * set_mm_exe_file - change a reference to the mm's executable file
1241 * @mm: The mm to change.
1242 * @new_exe_file: The new file to use.
1243 *
1244 * This changes mm's executable file (shown as symlink /proc/[pid]/exe).
1245 *
1246 * Main users are mmput() and sys_execve(). Callers prevent concurrent
1247 * invocations: in mmput() nobody alive left, in execve it happens before
1248 * the new mm is made visible to anyone.
1249 *
1250 * Can only fail if new_exe_file != NULL.
1251 */
1252int set_mm_exe_file(struct mm_struct *mm, struct file *new_exe_file)
1253{
1254 struct file *old_exe_file;
1255
1256 /*
1257 * It is safe to dereference the exe_file without RCU as
1258 * this function is only called if nobody else can access
1259 * this mm -- see comment above for justification.
1260 */
1261 old_exe_file = rcu_dereference_raw(mm->exe_file);
1262
1263 if (new_exe_file) {
1264 /*
1265 * We expect the caller (i.e., sys_execve) to already denied
1266 * write access, so this is unlikely to fail.
1267 */
1268 if (unlikely(exe_file_deny_write_access(new_exe_file)))
1269 return -EACCES;
1270 get_file(new_exe_file);
1271 }
1272 rcu_assign_pointer(mm->exe_file, new_exe_file);
1273 if (old_exe_file) {
1274 exe_file_allow_write_access(old_exe_file);
1275 fput(old_exe_file);
1276 }
1277 return 0;
1278}
1279
1280/**
1281 * replace_mm_exe_file - replace a reference to the mm's executable file
1282 * @mm: The mm to change.
1283 * @new_exe_file: The new file to use.
1284 *
1285 * This changes mm's executable file (shown as symlink /proc/[pid]/exe).
1286 *
1287 * Main user is sys_prctl(PR_SET_MM_MAP/EXE_FILE).
1288 */
1289int replace_mm_exe_file(struct mm_struct *mm, struct file *new_exe_file)
1290{
1291 struct vm_area_struct *vma;
1292 struct file *old_exe_file;
1293 int ret = 0;
1294
1295 /* Forbid mm->exe_file change if old file still mapped. */
1296 old_exe_file = get_mm_exe_file(mm);
1297 if (old_exe_file) {
1298 VMA_ITERATOR(vmi, mm, 0);
1299 mmap_read_lock(mm);
1300 for_each_vma(vmi, vma) {
1301 if (!vma->vm_file)
1302 continue;
1303 if (path_equal(&vma->vm_file->f_path,
1304 &old_exe_file->f_path)) {
1305 ret = -EBUSY;
1306 break;
1307 }
1308 }
1309 mmap_read_unlock(mm);
1310 fput(old_exe_file);
1311 if (ret)
1312 return ret;
1313 }
1314
1315 ret = exe_file_deny_write_access(new_exe_file);
1316 if (ret)
1317 return -EACCES;
1318 get_file(new_exe_file);
1319
1320 /* set the new file */
1321 mmap_write_lock(mm);
1322 old_exe_file = rcu_dereference_raw(mm->exe_file);
1323 rcu_assign_pointer(mm->exe_file, new_exe_file);
1324 mmap_write_unlock(mm);
1325
1326 if (old_exe_file) {
1327 exe_file_allow_write_access(old_exe_file);
1328 fput(old_exe_file);
1329 }
1330 return 0;
1331}
1332
1333/**
1334 * get_mm_exe_file - acquire a reference to the mm's executable file
1335 * @mm: The mm of interest.
1336 *
1337 * Returns %NULL if mm has no associated executable file.
1338 * User must release file via fput().
1339 */
1340struct file *get_mm_exe_file(struct mm_struct *mm)
1341{
1342 struct file *exe_file;
1343
1344 rcu_read_lock();
1345 exe_file = get_file_rcu(&mm->exe_file);
1346 rcu_read_unlock();
1347 return exe_file;
1348}
1349
1350/**
1351 * get_task_exe_file - acquire a reference to the task's executable file
1352 * @task: The task.
1353 *
1354 * Returns %NULL if task's mm (if any) has no associated executable file or
1355 * this is a kernel thread with borrowed mm (see the comment above get_task_mm).
1356 * User must release file via fput().
1357 */
1358struct file *get_task_exe_file(struct task_struct *task)
1359{
1360 struct file *exe_file = NULL;
1361 struct mm_struct *mm;
1362
1363 if (task->flags & PF_KTHREAD)
1364 return NULL;
1365
1366 task_lock(task);
1367 mm = task->mm;
1368 if (mm)
1369 exe_file = get_mm_exe_file(mm);
1370 task_unlock(task);
1371 return exe_file;
1372}
1373
1374/**
1375 * get_task_mm - acquire a reference to the task's mm
1376 * @task: The task.
1377 *
1378 * Returns %NULL if the task has no mm. Checks PF_KTHREAD (meaning
1379 * this kernel workthread has transiently adopted a user mm with kthread_use_mm,
1380 * to do its AIO) is not set and if so returns a reference to it, after
1381 * bumping up the use count. User must release the mm via mmput()
1382 * after use. Typically used by /proc and ptrace.
1383 */
1384struct mm_struct *get_task_mm(struct task_struct *task)
1385{
1386 struct mm_struct *mm;
1387
1388 if (task->flags & PF_KTHREAD)
1389 return NULL;
1390
1391 task_lock(task);
1392 mm = task->mm;
1393 if (mm)
1394 mmget(mm);
1395 task_unlock(task);
1396 return mm;
1397}
1398EXPORT_SYMBOL_GPL(get_task_mm);
1399
1400static bool may_access_mm(struct mm_struct *mm, struct task_struct *task, unsigned int mode)
1401{
1402 if (mm == current->mm)
1403 return true;
1404 if (ptrace_may_access(task, mode))
1405 return true;
1406 if ((mode & PTRACE_MODE_READ) && perfmon_capable())
1407 return true;
1408 return false;
1409}
1410
1411struct mm_struct *mm_access(struct task_struct *task, unsigned int mode)
1412{
1413 struct mm_struct *mm;
1414 int err;
1415
1416 err = down_read_killable(&task->signal->exec_update_lock);
1417 if (err)
1418 return ERR_PTR(err);
1419
1420 mm = get_task_mm(task);
1421 if (!mm) {
1422 mm = ERR_PTR(-ESRCH);
1423 } else if (!may_access_mm(mm, task, mode)) {
1424 mmput(mm);
1425 mm = ERR_PTR(-EACCES);
1426 }
1427 up_read(&task->signal->exec_update_lock);
1428
1429 return mm;
1430}
1431
1432static void complete_vfork_done(struct task_struct *tsk)
1433{
1434 struct completion *vfork;
1435
1436 task_lock(tsk);
1437 vfork = tsk->vfork_done;
1438 if (likely(vfork)) {
1439 tsk->vfork_done = NULL;
1440 complete(vfork);
1441 }
1442 task_unlock(tsk);
1443}
1444
1445static int wait_for_vfork_done(struct task_struct *child,
1446 struct completion *vfork)
1447{
1448 unsigned int state = TASK_KILLABLE|TASK_FREEZABLE;
1449 int killed;
1450
1451 cgroup_enter_frozen();
1452 killed = wait_for_completion_state(vfork, state);
1453 cgroup_leave_frozen(false);
1454
1455 if (killed) {
1456 task_lock(child);
1457 child->vfork_done = NULL;
1458 task_unlock(child);
1459 }
1460
1461 put_task_struct(child);
1462 return killed;
1463}
1464
1465/* Please note the differences between mmput and mm_release.
1466 * mmput is called whenever we stop holding onto a mm_struct,
1467 * error success whatever.
1468 *
1469 * mm_release is called after a mm_struct has been removed
1470 * from the current process.
1471 *
1472 * This difference is important for error handling, when we
1473 * only half set up a mm_struct for a new process and need to restore
1474 * the old one. Because we mmput the new mm_struct before
1475 * restoring the old one. . .
1476 * Eric Biederman 10 January 1998
1477 */
1478static void mm_release(struct task_struct *tsk, struct mm_struct *mm)
1479{
1480 uprobe_free_utask(tsk);
1481
1482 /* Get rid of any cached register state */
1483 deactivate_mm(tsk, mm);
1484
1485 /*
1486 * Signal userspace if we're not exiting with a core dump
1487 * because we want to leave the value intact for debugging
1488 * purposes.
1489 */
1490 if (tsk->clear_child_tid) {
1491 if (atomic_read(&mm->mm_users) > 1) {
1492 /*
1493 * We don't check the error code - if userspace has
1494 * not set up a proper pointer then tough luck.
1495 */
1496 put_user(0, tsk->clear_child_tid);
1497 do_futex(tsk->clear_child_tid, FUTEX_WAKE,
1498 1, NULL, NULL, 0, 0);
1499 }
1500 tsk->clear_child_tid = NULL;
1501 }
1502
1503 /*
1504 * All done, finally we can wake up parent and return this mm to him.
1505 * Also kthread_stop() uses this completion for synchronization.
1506 */
1507 if (tsk->vfork_done)
1508 complete_vfork_done(tsk);
1509}
1510
1511void mm_exit_exec_release(struct task_struct *tsk, struct mm_struct *mm)
1512{
1513 futex_exit_exec_release(tsk);
1514 mm_release(tsk, mm);
1515}
1516
1517/**
1518 * dup_mm() - duplicates an existing mm structure
1519 * @tsk: the task_struct with which the new mm will be associated.
1520 * @oldmm: the mm to duplicate.
1521 *
1522 * Allocates a new mm structure and duplicates the provided @oldmm structure
1523 * content into it.
1524 *
1525 * Return: the duplicated mm or NULL on failure.
1526 */
1527static struct mm_struct *dup_mm(struct task_struct *tsk,
1528 struct mm_struct *oldmm)
1529{
1530 struct mm_struct *mm;
1531 int err;
1532
1533 mm = allocate_mm();
1534 if (!mm)
1535 goto fail_nomem;
1536
1537 memcpy(mm, oldmm, sizeof(*mm));
1538
1539 if (!mm_init(mm, tsk))
1540 goto fail_nomem;
1541
1542 uprobe_start_dup_mmap();
1543 err = dup_mmap(mm, oldmm);
1544 if (err)
1545 goto free_pt;
1546 uprobe_end_dup_mmap();
1547
1548 mm->hiwater_rss = get_mm_rss(mm);
1549 mm->hiwater_vm = mm->total_vm;
1550
1551 if (mm->binfmt && !try_module_get(mm->binfmt->module))
1552 goto free_pt;
1553
1554 return mm;
1555
1556free_pt:
1557 /* don't put binfmt in mmput, we haven't got module yet */
1558 mm->binfmt = NULL;
1559 mm_init_owner(mm, NULL);
1560 mmput(mm);
1561 if (err)
1562 uprobe_end_dup_mmap();
1563
1564fail_nomem:
1565 return NULL;
1566}
1567
1568static int copy_mm(u64 clone_flags, struct task_struct *tsk)
1569{
1570 struct mm_struct *mm, *oldmm;
1571
1572 tsk->min_flt = tsk->maj_flt = 0;
1573 tsk->nvcsw = tsk->nivcsw = 0;
1574#ifdef CONFIG_DETECT_HUNG_TASK
1575 tsk->last_switch_count = tsk->nvcsw + tsk->nivcsw;
1576 tsk->last_switch_time = 0;
1577#endif
1578
1579 tsk->mm = NULL;
1580 tsk->active_mm = NULL;
1581
1582 /*
1583 * Are we cloning a kernel thread?
1584 *
1585 * We need to steal a active VM for that..
1586 */
1587 oldmm = current->mm;
1588 if (!oldmm)
1589 return 0;
1590
1591 if (clone_flags & CLONE_VM) {
1592 mmget(oldmm);
1593 mm = oldmm;
1594 } else {
1595 mm = dup_mm(tsk, current->mm);
1596 if (!mm)
1597 return -ENOMEM;
1598 }
1599
1600 tsk->mm = mm;
1601 tsk->active_mm = mm;
1602 sched_cache_fork(tsk);
1603 return 0;
1604}
1605
1606static int copy_exec_state(u64 clone_flags, struct task_struct *tsk)
1607{
1608 struct task_exec_state *exec_state;
1609
1610 /* CLONE_VM siblings refcount-share the parent's exec_state. */
1611 if (clone_flags & CLONE_VM) {
1612 exec_state = rcu_dereference_protected(current->exec_state, true);
1613 refcount_inc(&exec_state->count);
1614 rcu_assign_pointer(tsk->exec_state, exec_state);
1615 return 0;
1616 }
1617
1618 /* Everyone else inherits a fresh copy. */
1619 return task_exec_state_copy(tsk);
1620}
1621
1622static int copy_fs(u64 clone_flags, struct task_struct *tsk, bool umh)
1623{
1624 struct fs_struct *fs;
1625
1626 /*
1627 * Usermodehelper may copy userspace_init_fs filesystem state but
1628 * they don't get to create mount namespaces, share the
1629 * filesystem state, or be started from a non-initial mount
1630 * namespace.
1631 */
1632 if (umh) {
1633 if (clone_flags & (CLONE_NEWNS | CLONE_FS))
1634 return -EINVAL;
1635 if (current->nsproxy->mnt_ns != &init_mnt_ns)
1636 return -EINVAL;
1637 fs = userspace_init_fs;
1638 } else {
1639 fs = current->fs;
1640 VFS_WARN_ON_ONCE(current->fs != current->real_fs);
1641 }
1642
1643 if (clone_flags & CLONE_FS) {
1644 /* tsk->fs is already what we want */
1645 read_seqlock_excl(&fs->seq);
1646 /* "users" and "in_exec" locked for check_unsafe_exec() */
1647 if (fs->in_exec) {
1648 read_sequnlock_excl(&fs->seq);
1649 return -EAGAIN;
1650 }
1651 fs->users++;
1652 read_sequnlock_excl(&fs->seq);
1653 return 0;
1654 }
1655 tsk->real_fs = tsk->fs = copy_fs_struct(fs);
1656 if (!tsk->fs)
1657 return -ENOMEM;
1658 return 0;
1659}
1660
1661static int copy_files(u64 clone_flags, struct task_struct *tsk,
1662 int no_files)
1663{
1664 struct files_struct *oldf, *newf;
1665
1666 /*
1667 * A background process may not have any files ...
1668 */
1669 oldf = current->files;
1670 if (!oldf)
1671 return 0;
1672
1673 if (no_files) {
1674 tsk->files = NULL;
1675 return 0;
1676 }
1677
1678 if (clone_flags & CLONE_FILES) {
1679 atomic_inc(&oldf->count);
1680 return 0;
1681 }
1682
1683 newf = dup_fd(oldf, NULL);
1684 if (IS_ERR(newf))
1685 return PTR_ERR(newf);
1686
1687 tsk->files = newf;
1688 return 0;
1689}
1690
1691static int copy_sighand(u64 clone_flags, struct task_struct *tsk)
1692{
1693 struct sighand_struct *sig;
1694
1695 if (clone_flags & CLONE_SIGHAND) {
1696 refcount_inc(&current->sighand->count);
1697 return 0;
1698 }
1699 sig = kmem_cache_alloc(sighand_cachep, GFP_KERNEL);
1700 RCU_INIT_POINTER(tsk->sighand, sig);
1701 if (!sig)
1702 return -ENOMEM;
1703
1704 refcount_set(&sig->count, 1);
1705 spin_lock_irq(&current->sighand->siglock);
1706 memcpy(sig->action, current->sighand->action, sizeof(sig->action));
1707 spin_unlock_irq(&current->sighand->siglock);
1708
1709 /* Reset all signal handler not set to SIG_IGN to SIG_DFL. */
1710 if (clone_flags & CLONE_CLEAR_SIGHAND)
1711 flush_signal_handlers(tsk, 0);
1712
1713 return 0;
1714}
1715
1716void __cleanup_sighand(struct sighand_struct *sighand)
1717{
1718 if (refcount_dec_and_test(&sighand->count)) {
1719 signalfd_cleanup(sighand);
1720 /*
1721 * sighand_cachep is SLAB_TYPESAFE_BY_RCU so we can free it
1722 * without an RCU grace period, see __lock_task_sighand().
1723 */
1724 kmem_cache_free(sighand_cachep, sighand);
1725 }
1726}
1727
1728/*
1729 * Initialize POSIX timer handling for a thread group.
1730 */
1731static void posix_cpu_timers_init_group(struct signal_struct *sig)
1732{
1733 struct posix_cputimers *pct = &sig->posix_cputimers;
1734 unsigned long cpu_limit;
1735
1736 cpu_limit = READ_ONCE(sig->rlim[RLIMIT_CPU].rlim_cur);
1737 posix_cputimers_group_init(pct, cpu_limit);
1738}
1739
1740static int copy_signal(u64 clone_flags, struct task_struct *tsk)
1741{
1742 struct signal_struct *sig;
1743
1744 if (clone_flags & CLONE_THREAD)
1745 return 0;
1746
1747 sig = kmem_cache_zalloc(signal_cachep, GFP_KERNEL);
1748 tsk->signal = sig;
1749 if (!sig)
1750 return -ENOMEM;
1751
1752 sig->nr_threads = 1;
1753 sig->quick_threads = 1;
1754 atomic_set(&sig->live, 1);
1755 refcount_set(&sig->sigcnt, 1);
1756
1757 /* list_add(thread_node, thread_head) without INIT_LIST_HEAD() */
1758 sig->thread_head = (struct list_head)LIST_HEAD_INIT(tsk->thread_node);
1759 tsk->thread_node = (struct list_head)LIST_HEAD_INIT(sig->thread_head);
1760
1761 init_waitqueue_head(&sig->wait_chldexit);
1762 sig->curr_target = tsk;
1763 init_sigpending(&sig->shared_pending);
1764 INIT_HLIST_HEAD(&sig->multiprocess);
1765 seqlock_init(&sig->stats_lock);
1766 prev_cputime_init(&sig->prev_cputime);
1767
1768#ifdef CONFIG_POSIX_TIMERS
1769 INIT_HLIST_HEAD(&sig->posix_timers);
1770 INIT_HLIST_HEAD(&sig->ignored_posix_timers);
1771 hrtimer_setup(&sig->real_timer, it_real_fn, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
1772#endif
1773
1774 task_lock(current->group_leader);
1775 memcpy(sig->rlim, current->signal->rlim, sizeof sig->rlim);
1776 task_unlock(current->group_leader);
1777
1778 posix_cpu_timers_init_group(sig);
1779
1780 tty_audit_fork(sig);
1781 sched_autogroup_fork(sig);
1782
1783#ifdef CONFIG_CGROUPS
1784 init_rwsem(&sig->cgroup_threadgroup_rwsem);
1785#endif
1786
1787 sig->oom_score_adj = current->signal->oom_score_adj;
1788 sig->oom_score_adj_min = current->signal->oom_score_adj_min;
1789
1790 mutex_init(&sig->cred_guard_mutex);
1791 init_rwsem(&sig->exec_update_lock);
1792
1793 return 0;
1794}
1795
1796static void copy_seccomp(struct task_struct *p)
1797{
1798#ifdef CONFIG_SECCOMP
1799 /*
1800 * Must be called with sighand->lock held, which is common to
1801 * all threads in the group. Holding cred_guard_mutex is not
1802 * needed because this new task is not yet running and cannot
1803 * be racing exec.
1804 */
1805 assert_spin_locked(&current->sighand->siglock);
1806
1807 /* Ref-count the new filter user, and assign it. */
1808 get_seccomp_filter(current);
1809 p->seccomp = current->seccomp;
1810
1811 /*
1812 * Explicitly enable no_new_privs here in case it got set
1813 * between the task_struct being duplicated and holding the
1814 * sighand lock. The seccomp state and nnp must be in sync.
1815 */
1816 if (task_no_new_privs(current))
1817 task_set_no_new_privs(p);
1818
1819 /*
1820 * If the parent gained a seccomp mode after copying thread
1821 * flags and between before we held the sighand lock, we have
1822 * to manually enable the seccomp thread flag here.
1823 */
1824 if (p->seccomp.mode != SECCOMP_MODE_DISABLED)
1825 set_task_syscall_work(p, SECCOMP);
1826#endif
1827}
1828
1829SYSCALL_DEFINE1(set_tid_address, int __user *, tidptr)
1830{
1831 current->clear_child_tid = tidptr;
1832
1833 return task_pid_vnr(current);
1834}
1835
1836static void rt_mutex_init_task(struct task_struct *p)
1837{
1838 raw_spin_lock_init(&p->pi_lock);
1839#ifdef CONFIG_RT_MUTEXES
1840 p->pi_waiters = RB_ROOT_CACHED;
1841 p->pi_top_task = NULL;
1842 p->pi_blocked_on = NULL;
1843#endif
1844}
1845
1846static inline void init_task_pid_links(struct task_struct *task)
1847{
1848 enum pid_type type;
1849
1850 for (type = PIDTYPE_PID; type < PIDTYPE_MAX; ++type)
1851 INIT_HLIST_NODE(&task->pid_links[type]);
1852}
1853
1854static inline void
1855init_task_pid(struct task_struct *task, enum pid_type type, struct pid *pid)
1856{
1857 if (type == PIDTYPE_PID)
1858 task->thread_pid = pid;
1859 else
1860 task->signal->pids[type] = pid;
1861}
1862
1863static inline void rcu_copy_process(struct task_struct *p)
1864{
1865#ifdef CONFIG_PREEMPT_RCU
1866 p->rcu_read_lock_nesting = 0;
1867 p->rcu_read_unlock_special.s = 0;
1868 p->rcu_blocked_node = NULL;
1869 INIT_LIST_HEAD(&p->rcu_node_entry);
1870#endif /* #ifdef CONFIG_PREEMPT_RCU */
1871#ifdef CONFIG_TASKS_RCU
1872 p->rcu_tasks_holdout = false;
1873 INIT_LIST_HEAD(&p->rcu_tasks_holdout_list);
1874 p->rcu_tasks_idle_cpu = -1;
1875 INIT_LIST_HEAD(&p->rcu_tasks_exit_list);
1876#endif /* #ifdef CONFIG_TASKS_RCU */
1877#ifdef CONFIG_TASKS_TRACE_RCU
1878 p->trc_reader_nesting = 0;
1879#endif /* #ifdef CONFIG_TASKS_TRACE_RCU */
1880}
1881
1882/**
1883 * pidfd_prepare - allocate a new pidfd_file and reserve a pidfd
1884 * @pid: the struct pid for which to create a pidfd
1885 * @flags: flags of the new @pidfd
1886 * @ret_file: return the new pidfs file
1887 *
1888 * Allocate a new file that stashes @pid and reserve a new pidfd number in the
1889 * caller's file descriptor table. The pidfd is reserved but not installed yet.
1890 *
1891 * The helper verifies that @pid is still in use, without PIDFD_THREAD the
1892 * task identified by @pid must be a thread-group leader.
1893 *
1894 * If this function returns successfully the caller is responsible to either
1895 * call fd_install() passing the returned pidfd and pidfd file as arguments in
1896 * order to install the pidfd into its file descriptor table or they must use
1897 * put_unused_fd() and fput() on the returned pidfd and pidfd file
1898 * respectively.
1899 *
1900 * This function is useful when a pidfd must already be reserved but there
1901 * might still be points of failure afterwards and the caller wants to ensure
1902 * that no pidfd is leaked into its file descriptor table.
1903 *
1904 * Return: On success, a reserved pidfd is returned from the function and a new
1905 * pidfd file is returned in the last argument to the function. On
1906 * error, a negative error code is returned from the function and the
1907 * last argument remains unchanged.
1908 */
1909int pidfd_prepare(struct pid *pid, unsigned int flags, struct file **ret_file)
1910{
1911 struct file *pidfs_file;
1912
1913 /*
1914 * PIDFD_STALE is only allowed to be passed if the caller knows
1915 * that @pid is already registered in pidfs and thus
1916 * PIDFD_INFO_EXIT information is guaranteed to be available.
1917 */
1918 if (!(flags & PIDFD_STALE)) {
1919 /*
1920 * While holding the pidfd waitqueue lock removing the
1921 * task linkage for the thread-group leader pid
1922 * (PIDTYPE_TGID) isn't possible. Thus, if there's still
1923 * task linkage for PIDTYPE_PID not having thread-group
1924 * leader linkage for the pid means it wasn't a
1925 * thread-group leader in the first place.
1926 */
1927 guard(spinlock_irq)(&pid->wait_pidfd.lock);
1928
1929 /* Task has already been reaped. */
1930 if (!pid_has_task(pid, PIDTYPE_PID))
1931 return -ESRCH;
1932 /*
1933 * If this struct pid isn't used as a thread-group
1934 * leader but the caller requested to create a
1935 * thread-group leader pidfd then report ENOENT.
1936 */
1937 if (!(flags & PIDFD_THREAD) && !pid_has_task(pid, PIDTYPE_TGID))
1938 return -ENOENT;
1939 }
1940
1941 CLASS(get_unused_fd, pidfd)(O_CLOEXEC);
1942 if (pidfd < 0)
1943 return pidfd;
1944
1945 pidfs_file = pidfs_alloc_file(pid, flags | O_RDWR);
1946 if (IS_ERR(pidfs_file))
1947 return PTR_ERR(pidfs_file);
1948
1949 *ret_file = pidfs_file;
1950 return take_fd(pidfd);
1951}
1952
1953static void __delayed_free_task(struct rcu_head *rhp)
1954{
1955 struct task_struct *tsk = container_of(rhp, struct task_struct, rcu);
1956
1957 free_task(tsk);
1958}
1959
1960static __always_inline void delayed_free_task(struct task_struct *tsk)
1961{
1962 if (IS_ENABLED(CONFIG_MEMCG))
1963 call_rcu(&tsk->rcu, __delayed_free_task);
1964 else
1965 free_task(tsk);
1966}
1967
1968static void copy_oom_score_adj(u64 clone_flags, struct task_struct *tsk)
1969{
1970 /* Skip if kernel thread */
1971 if (!tsk->mm)
1972 return;
1973
1974 /* Skip if spawning a thread or using vfork */
1975 if ((clone_flags & (CLONE_VM | CLONE_THREAD | CLONE_VFORK)) != CLONE_VM)
1976 return;
1977
1978 /* We need to synchronize with __set_oom_adj */
1979 mutex_lock(&oom_adj_mutex);
1980 mm_flags_set(MMF_MULTIPROCESS, tsk->mm);
1981 /* Update the values in case they were changed after copy_signal */
1982 tsk->signal->oom_score_adj = current->signal->oom_score_adj;
1983 tsk->signal->oom_score_adj_min = current->signal->oom_score_adj_min;
1984 mutex_unlock(&oom_adj_mutex);
1985}
1986
1987#ifdef CONFIG_RV
1988static void rv_task_fork(struct task_struct *p)
1989{
1990 memset(&p->rv, 0, sizeof(p->rv));
1991}
1992#else
1993#define rv_task_fork(p) do {} while (0)
1994#endif
1995
1996static bool need_futex_hash_allocate_default(u64 clone_flags)
1997{
1998 /*
1999 * Allocate a default futex hash for any sibling that will
2000 * share the parent's mm.
2001 */
2002 return clone_flags & CLONE_VM;
2003}
2004
2005/*
2006 * This creates a new process as a copy of the old one,
2007 * but does not actually start it yet.
2008 *
2009 * It copies the registers, and all the appropriate
2010 * parts of the process environment (as per the clone
2011 * flags). The actual kick-off is left to the caller.
2012 */
2013__latent_entropy struct task_struct *copy_process(
2014 struct pid *pid,
2015 int trace,
2016 int node,
2017 struct kernel_clone_args *args)
2018{
2019 int pidfd = -1, retval;
2020 struct task_struct *p;
2021 struct multiprocess_signals delayed;
2022 struct file *pidfile = NULL;
2023 const u64 clone_flags = args->flags;
2024 struct nsproxy *nsp = current->nsproxy;
2025
2026 /*
2027 * Don't allow sharing the root directory with processes in a different
2028 * namespace
2029 */
2030 if ((clone_flags & (CLONE_NEWNS|CLONE_FS)) == (CLONE_NEWNS|CLONE_FS))
2031 return ERR_PTR(-EINVAL);
2032
2033 if ((clone_flags & (CLONE_NEWUSER|CLONE_FS)) == (CLONE_NEWUSER|CLONE_FS))
2034 return ERR_PTR(-EINVAL);
2035
2036 /*
2037 * Thread groups must share signals as well, and detached threads
2038 * can only be started up within the thread group.
2039 */
2040 if ((clone_flags & CLONE_THREAD) && !(clone_flags & CLONE_SIGHAND))
2041 return ERR_PTR(-EINVAL);
2042
2043 /*
2044 * Shared signal handlers imply shared VM. By way of the above,
2045 * thread groups also imply shared VM. Blocking this case allows
2046 * for various simplifications in other code.
2047 */
2048 if ((clone_flags & CLONE_SIGHAND) && !(clone_flags & CLONE_VM))
2049 return ERR_PTR(-EINVAL);
2050
2051 /*
2052 * Siblings of global init remain as zombies on exit since they are
2053 * not reaped by their parent (swapper). To solve this and to avoid
2054 * multi-rooted process trees, prevent global and container-inits
2055 * from creating siblings.
2056 */
2057 if ((clone_flags & CLONE_PARENT) &&
2058 current->signal->flags & SIGNAL_UNKILLABLE)
2059 return ERR_PTR(-EINVAL);
2060
2061 /*
2062 * If the new process will be in a different pid or user namespace
2063 * do not allow it to share a thread group with the forking task.
2064 */
2065 if (clone_flags & CLONE_THREAD) {
2066 if ((clone_flags & (CLONE_NEWUSER | CLONE_NEWPID)) ||
2067 (task_active_pid_ns(current) != nsp->pid_ns_for_children))
2068 return ERR_PTR(-EINVAL);
2069 }
2070
2071 if (clone_flags & CLONE_PIDFD) {
2072 /*
2073 * - CLONE_DETACHED is blocked so that we can potentially
2074 * reuse it later for CLONE_PIDFD.
2075 */
2076 if (clone_flags & CLONE_DETACHED)
2077 return ERR_PTR(-EINVAL);
2078 }
2079
2080 if (clone_flags & CLONE_AUTOREAP) {
2081 if (clone_flags & CLONE_THREAD)
2082 return ERR_PTR(-EINVAL);
2083 if (clone_flags & CLONE_PARENT)
2084 return ERR_PTR(-EINVAL);
2085 if (args->exit_signal)
2086 return ERR_PTR(-EINVAL);
2087 }
2088
2089 if ((clone_flags & CLONE_PARENT) && current->signal->autoreap)
2090 return ERR_PTR(-EINVAL);
2091
2092 if (clone_flags & CLONE_NNP) {
2093 if (clone_flags & CLONE_THREAD)
2094 return ERR_PTR(-EINVAL);
2095 }
2096
2097 if (clone_flags & CLONE_PIDFD_AUTOKILL) {
2098 if (!(clone_flags & CLONE_PIDFD))
2099 return ERR_PTR(-EINVAL);
2100 if (!(clone_flags & CLONE_AUTOREAP))
2101 return ERR_PTR(-EINVAL);
2102 if (clone_flags & CLONE_THREAD)
2103 return ERR_PTR(-EINVAL);
2104 /*
2105 * Without CLONE_NNP the child could escalate privileges
2106 * after being spawned, so require CAP_SYS_ADMIN.
2107 * With CLONE_NNP the child can't gain new privileges,
2108 * so allow unprivileged usage.
2109 */
2110 if (!(clone_flags & CLONE_NNP) &&
2111 !ns_capable(current_user_ns(), CAP_SYS_ADMIN))
2112 return ERR_PTR(-EPERM);
2113 }
2114
2115 /*
2116 * Force any signals received before this point to be delivered
2117 * before the fork happens. Collect up signals sent to multiple
2118 * processes that happen during the fork and delay them so that
2119 * they appear to happen after the fork.
2120 */
2121 sigemptyset(&delayed.signal);
2122 INIT_HLIST_NODE(&delayed.node);
2123
2124 spin_lock_irq(&current->sighand->siglock);
2125 if (!(clone_flags & CLONE_THREAD))
2126 hlist_add_head(&delayed.node, &current->signal->multiprocess);
2127 recalc_sigpending();
2128 spin_unlock_irq(&current->sighand->siglock);
2129 retval = -ERESTARTNOINTR;
2130 if (task_sigpending(current))
2131 goto fork_out;
2132
2133 retval = -ENOMEM;
2134 p = dup_task_struct(current, node);
2135 if (!p)
2136 goto fork_out;
2137 /*
2138 * Must run before the first fallible op, so error paths never
2139 * free the parent's ret_stack.
2140 */
2141 ftrace_graph_init_task(p);
2142 retval = copy_exec_state(clone_flags, p);
2143 if (retval)
2144 goto bad_fork_free;
2145 p->flags &= ~PF_KTHREAD;
2146 if (args->kthread)
2147 p->flags |= PF_KTHREAD;
2148 if (args->user_worker) {
2149 /*
2150 * Mark us a user worker, and block any signal that isn't
2151 * fatal or STOP
2152 */
2153 p->flags |= PF_USER_WORKER;
2154 siginitsetinv(&p->blocked, sigmask(SIGKILL)|sigmask(SIGSTOP));
2155 }
2156 if (args->io_thread)
2157 p->flags |= PF_IO_WORKER;
2158
2159 if (args->name)
2160 strscpy_pad(p->comm, args->name, sizeof(p->comm));
2161
2162 p->set_child_tid = (clone_flags & CLONE_CHILD_SETTID) ? args->child_tid : NULL;
2163 /*
2164 * TID is cleared in mm_release() when the task exits
2165 */
2166 p->clear_child_tid = (clone_flags & CLONE_CHILD_CLEARTID) ? args->child_tid : NULL;
2167
2168 rt_mutex_init_task(p);
2169 raw_spin_lock_init(&p->blocked_lock);
2170
2171 lockdep_assert_irqs_enabled();
2172#ifdef CONFIG_PROVE_LOCKING
2173 DEBUG_LOCKS_WARN_ON(!p->softirqs_enabled);
2174#endif
2175 retval = copy_creds(p, clone_flags);
2176 if (retval < 0)
2177 goto bad_fork_free;
2178
2179 retval = -EAGAIN;
2180 if (is_rlimit_overlimit(task_ucounts(p), UCOUNT_RLIMIT_NPROC, rlimit(RLIMIT_NPROC))) {
2181 if (p->real_cred->user != INIT_USER &&
2182 !capable(CAP_SYS_RESOURCE) && !capable(CAP_SYS_ADMIN))
2183 goto bad_fork_cleanup_count;
2184 }
2185 current->flags &= ~PF_NPROC_EXCEEDED;
2186
2187 /*
2188 * If multiple threads are within copy_process(), then this check
2189 * triggers too late. This doesn't hurt, the check is only there
2190 * to stop root fork bombs.
2191 */
2192 retval = -EAGAIN;
2193 if (data_race(nr_threads >= max_threads))
2194 goto bad_fork_cleanup_count;
2195
2196 delayacct_tsk_init(p); /* Must remain after dup_task_struct() */
2197 p->flags &= ~(PF_SUPERPRIV | PF_WQ_WORKER | PF_IDLE | PF_NO_SETAFFINITY);
2198 p->flags |= PF_FORKNOEXEC;
2199 INIT_LIST_HEAD(&p->children);
2200 INIT_LIST_HEAD(&p->sibling);
2201 rcu_copy_process(p);
2202 p->vfork_done = NULL;
2203 spin_lock_init(&p->alloc_lock);
2204
2205 init_sigpending(&p->pending);
2206
2207 p->utime = p->stime = p->gtime = 0;
2208#ifdef CONFIG_ARCH_HAS_SCALED_CPUTIME
2209 p->utimescaled = p->stimescaled = 0;
2210#endif
2211 prev_cputime_init(&p->prev_cputime);
2212
2213#ifdef CONFIG_VIRT_CPU_ACCOUNTING_GEN
2214 seqcount_init(&p->vtime.seqcount);
2215 p->vtime.starttime = 0;
2216 p->vtime.state = VTIME_INACTIVE;
2217#endif
2218
2219#ifdef CONFIG_IO_URING
2220 p->io_uring = NULL;
2221 retval = io_uring_fork(p);
2222 if (unlikely(retval))
2223 goto bad_fork_cleanup_delayacct;
2224 retval = -EAGAIN;
2225#endif
2226
2227 p->default_timer_slack_ns = current->timer_slack_ns;
2228
2229#ifdef CONFIG_PSI
2230 p->psi_flags = 0;
2231#endif
2232
2233 task_io_accounting_init(&p->ioac);
2234 acct_clear_integrals(p);
2235
2236 posix_cputimers_init(&p->posix_cputimers);
2237 tick_dep_init_task(p);
2238
2239 p->io_context = NULL;
2240 audit_set_context(p, NULL);
2241 cgroup_fork(p);
2242 if (args->kthread) {
2243 if (!set_kthread_struct(p))
2244 goto bad_fork_cleanup_delayacct;
2245 }
2246#ifdef CONFIG_NUMA
2247 p->mempolicy = mpol_dup(p->mempolicy);
2248 if (IS_ERR(p->mempolicy)) {
2249 retval = PTR_ERR(p->mempolicy);
2250 p->mempolicy = NULL;
2251 goto bad_fork_cleanup_delayacct;
2252 }
2253#endif
2254#ifdef CONFIG_CPUSETS
2255 p->cpuset_mem_spread_rotor = NUMA_NO_NODE;
2256 seqcount_spinlock_init(&p->mems_allowed_seq, &p->alloc_lock);
2257#endif
2258#ifdef CONFIG_TRACE_IRQFLAGS
2259 memset(&p->irqtrace, 0, sizeof(p->irqtrace));
2260 p->irqtrace.hardirq_disable_ip = _THIS_IP_;
2261 p->irqtrace.softirq_enable_ip = _THIS_IP_;
2262 p->softirqs_enabled = 1;
2263 p->softirq_context = 0;
2264#endif
2265
2266 p->pagefault_disabled = 0;
2267
2268 lockdep_init_task(p);
2269
2270 p->blocked_on = NULL; /* not blocked yet */
2271 p->blocked_donor = NULL; /* nobody is boosting p yet */
2272
2273#ifdef CONFIG_BCACHE
2274 p->sequential_io = 0;
2275 p->sequential_io_avg = 0;
2276#endif
2277
2278 unwind_task_init(p);
2279
2280 /* Perform scheduler related setup. Assign this task to a CPU. */
2281 retval = sched_fork(clone_flags, p);
2282 if (retval)
2283 goto bad_fork_cleanup_policy;
2284
2285 retval = perf_event_init_task(p, clone_flags);
2286 if (retval)
2287 goto bad_fork_sched_cancel_fork;
2288 retval = audit_alloc(p);
2289 if (retval)
2290 goto bad_fork_cleanup_perf;
2291 /* copy all the process information */
2292 shm_init_task(p);
2293 retval = security_task_alloc(p, clone_flags);
2294 if (retval)
2295 goto bad_fork_cleanup_audit;
2296 retval = copy_semundo(clone_flags, p);
2297 if (retval)
2298 goto bad_fork_cleanup_security;
2299 retval = copy_files(clone_flags, p, args->no_files);
2300 if (retval)
2301 goto bad_fork_cleanup_semundo;
2302 retval = copy_fs(clone_flags, p, args->umh);
2303 if (retval)
2304 goto bad_fork_cleanup_files;
2305 retval = copy_sighand(clone_flags, p);
2306 if (retval)
2307 goto bad_fork_cleanup_fs;
2308 retval = copy_signal(clone_flags, p);
2309 if (retval)
2310 goto bad_fork_cleanup_sighand;
2311 retval = copy_mm(clone_flags, p);
2312 if (retval)
2313 goto bad_fork_cleanup_signal;
2314 retval = copy_namespaces(clone_flags, p);
2315 if (retval)
2316 goto bad_fork_cleanup_mm;
2317 retval = copy_io(clone_flags, p);
2318 if (retval)
2319 goto bad_fork_cleanup_namespaces;
2320 retval = copy_thread(p, args);
2321 if (retval)
2322 goto bad_fork_cleanup_io;
2323
2324 stackleak_task_init(p);
2325
2326 if (pid != &init_struct_pid) {
2327 pid = alloc_pid(p->nsproxy->pid_ns_for_children, args->set_tid,
2328 args->set_tid_size);
2329 if (IS_ERR(pid)) {
2330 retval = PTR_ERR(pid);
2331 goto bad_fork_cleanup_thread;
2332 }
2333 }
2334
2335 /*
2336 * This has to happen after we've potentially unshared the file
2337 * descriptor table (so that the pidfd doesn't leak into the child
2338 * if the fd table isn't shared).
2339 */
2340 if (clone_flags & CLONE_PIDFD) {
2341 unsigned flags = PIDFD_STALE;
2342
2343 if (clone_flags & CLONE_THREAD)
2344 flags |= PIDFD_THREAD;
2345 if (clone_flags & CLONE_PIDFD_AUTOKILL)
2346 flags |= PIDFD_AUTOKILL;
2347
2348 /*
2349 * Note that no task has been attached to @pid yet indicate
2350 * that via CLONE_PIDFD.
2351 */
2352 retval = pidfd_prepare(pid, flags, &pidfile);
2353 if (retval < 0)
2354 goto bad_fork_free_pid;
2355 pidfd = retval;
2356
2357 retval = put_user(pidfd, args->pidfd);
2358 if (retval)
2359 goto bad_fork_put_pidfd;
2360 }
2361
2362#ifdef CONFIG_BLOCK
2363 p->plug = NULL;
2364 p->flags &= ~PF_BLOCK_TS;
2365#endif
2366 futex_init_task(p);
2367
2368 /*
2369 * sigaltstack should be cleared when sharing the same VM
2370 */
2371 if ((clone_flags & (CLONE_VM|CLONE_VFORK)) == CLONE_VM)
2372 sas_ss_reset(p);
2373
2374 /*
2375 * Syscall tracing and stepping should be turned off in the
2376 * child regardless of CLONE_PTRACE.
2377 */
2378 user_disable_single_step(p);
2379 clear_task_syscall_work(p, SYSCALL_TRACE);
2380#if defined(CONFIG_GENERIC_ENTRY) || defined(TIF_SYSCALL_EMU)
2381 clear_task_syscall_work(p, SYSCALL_EMU);
2382#endif
2383 clear_tsk_latency_tracing(p);
2384
2385 /* ok, now we should be set up.. */
2386 p->pid = pid_nr(pid);
2387 if (clone_flags & CLONE_THREAD) {
2388 p->group_leader = current->group_leader;
2389 p->tgid = current->tgid;
2390 } else {
2391 p->group_leader = p;
2392 p->tgid = p->pid;
2393 }
2394
2395 p->nr_dirtied = 0;
2396 p->nr_dirtied_pause = 128 >> (PAGE_SHIFT - 10);
2397 p->dirty_paused_when = 0;
2398
2399 p->pdeath_signal = 0;
2400 p->task_works = NULL;
2401 clear_posix_cputimers_work(p);
2402
2403#ifdef CONFIG_KRETPROBES
2404 p->kretprobe_instances.first = NULL;
2405#endif
2406#ifdef CONFIG_RETHOOK
2407 p->rethooks.first = NULL;
2408#endif
2409
2410 /*
2411 * Ensure that the cgroup subsystem policies allow the new process to be
2412 * forked. It should be noted that the new process's css_set can be changed
2413 * between here and cgroup_post_fork() if an organisation operation is in
2414 * progress.
2415 */
2416 retval = cgroup_can_fork(p, args);
2417 if (retval)
2418 goto bad_fork_put_pidfd;
2419
2420 /*
2421 * Now that the cgroups are pinned, re-clone the parent cgroup and put
2422 * the new task on the correct runqueue. All this *before* the task
2423 * becomes visible.
2424 *
2425 * This isn't part of ->can_fork() because while the re-cloning is
2426 * cgroup specific, it unconditionally needs to place the task on a
2427 * runqueue.
2428 */
2429 retval = sched_cgroup_fork(p, args);
2430 if (retval)
2431 goto bad_fork_cancel_cgroup;
2432
2433 if (need_futex_hash_allocate_default(clone_flags)) {
2434 retval = futex_hash_allocate_default();
2435 if (retval)
2436 goto bad_fork_cancel_cgroup;
2437 /*
2438 * If we fail beyond this point we don't free the allocated
2439 * futex hash map. We assume that another thread will be created
2440 * and makes use of it. The hash map will be freed once the main
2441 * thread terminates.
2442 */
2443 }
2444 /*
2445 * From this point on we must avoid any synchronous user-space
2446 * communication until we take the tasklist-lock. In particular, we do
2447 * not want user-space to be able to predict the process start-time by
2448 * stalling fork(2) after we recorded the start_time but before it is
2449 * visible to the system.
2450 */
2451
2452 p->start_time = ktime_get_ns();
2453 p->start_boottime = ktime_get_boottime_ns();
2454
2455 /*
2456 * Make it visible to the rest of the system, but dont wake it up yet.
2457 * Need tasklist lock for parent etc handling!
2458 */
2459 write_lock_irq(&tasklist_lock);
2460
2461 /* CLONE_PARENT re-uses the old parent */
2462 if (clone_flags & (CLONE_PARENT|CLONE_THREAD)) {
2463 p->real_parent = current->real_parent;
2464 p->parent_exec_id = current->parent_exec_id;
2465 if (clone_flags & CLONE_THREAD)
2466 p->exit_signal = -1;
2467 else
2468 p->exit_signal = current->group_leader->exit_signal;
2469 } else {
2470 p->real_parent = current;
2471 p->parent_exec_id = current->self_exec_id;
2472 p->exit_signal = args->exit_signal;
2473 }
2474
2475 klp_copy_process(p);
2476
2477 sched_core_fork(p);
2478
2479 spin_lock(&current->sighand->siglock);
2480
2481 rv_task_fork(p);
2482
2483 rseq_fork(p, clone_flags);
2484
2485 /*
2486 * If zap_pid_ns_processes() was called after alloc_pid(), the new
2487 * child missed SIGKILL. If current is not in the same namespace,
2488 * we can't rely on fatal_signal_pending() below.
2489 */
2490 if (unlikely(!(ns_of_pid(pid)->pid_allocated & PIDNS_ADDING))) {
2491 retval = -ENOMEM;
2492 goto bad_fork_core_free;
2493 }
2494
2495 /* Let kill terminate clone/fork in the middle */
2496 if (fatal_signal_pending(current)) {
2497 retval = -EINTR;
2498 goto bad_fork_core_free;
2499 }
2500
2501 /* No more failure paths after this point. */
2502
2503 /*
2504 * Copy seccomp details explicitly here, in case they were changed
2505 * before holding sighand lock.
2506 */
2507 copy_seccomp(p);
2508
2509 if (clone_flags & CLONE_NNP)
2510 task_set_no_new_privs(p);
2511
2512 init_task_pid_links(p);
2513 if (likely(p->pid)) {
2514 ptrace_init_task(p, (clone_flags & CLONE_PTRACE) || trace);
2515
2516 init_task_pid(p, PIDTYPE_PID, pid);
2517 if (thread_group_leader(p)) {
2518 init_task_pid(p, PIDTYPE_TGID, pid);
2519 init_task_pid(p, PIDTYPE_PGID, task_pgrp(current));
2520 init_task_pid(p, PIDTYPE_SID, task_session(current));
2521
2522 if (is_child_reaper(pid)) {
2523 struct pid_namespace *ns = ns_of_pid(pid);
2524
2525 ASSERT_EXCLUSIVE_WRITER(ns->child_reaper);
2526 WRITE_ONCE(ns->child_reaper, p);
2527 p->signal->flags |= SIGNAL_UNKILLABLE;
2528 }
2529 p->signal->shared_pending.signal = delayed.signal;
2530 p->signal->tty = tty_kref_get(current->signal->tty);
2531 /*
2532 * Inherit has_child_subreaper flag under the same
2533 * tasklist_lock with adding child to the process tree
2534 * for propagate_has_child_subreaper optimization.
2535 */
2536 p->signal->has_child_subreaper = p->real_parent->signal->has_child_subreaper ||
2537 p->real_parent->signal->is_child_subreaper;
2538 if (clone_flags & CLONE_AUTOREAP)
2539 p->signal->autoreap = 1;
2540 list_add_tail(&p->sibling, &p->real_parent->children);
2541 list_add_tail_rcu(&p->tasks, &init_task.tasks);
2542 attach_pid(p, PIDTYPE_TGID);
2543 attach_pid(p, PIDTYPE_PGID);
2544 attach_pid(p, PIDTYPE_SID);
2545 __this_cpu_inc(process_counts);
2546 } else {
2547 current->signal->nr_threads++;
2548 current->signal->quick_threads++;
2549 atomic_inc(&current->signal->live);
2550 refcount_inc(&current->signal->sigcnt);
2551 task_join_group_stop(p);
2552 list_add_tail_rcu(&p->thread_node,
2553 &p->signal->thread_head);
2554 }
2555 attach_pid(p, PIDTYPE_PID);
2556 nr_threads++;
2557 }
2558 total_forks++;
2559 hlist_del_init(&delayed.node);
2560 spin_unlock(&current->sighand->siglock);
2561 syscall_tracepoint_update(p);
2562 write_unlock_irq(&tasklist_lock);
2563
2564 if (pidfile)
2565 fd_install(pidfd, pidfile);
2566
2567 proc_fork_connector(p);
2568 /*
2569 * sched_ext needs @p to be associated with its cgroup in its post_fork
2570 * hook. cgroup_post_fork() should come before sched_post_fork().
2571 */
2572 cgroup_post_fork(p, args);
2573 sched_post_fork(p);
2574 perf_event_fork(p);
2575
2576 trace_task_newtask(p, clone_flags);
2577 uprobe_copy_process(p, clone_flags);
2578 user_events_fork(p, clone_flags);
2579
2580 copy_oom_score_adj(clone_flags, p);
2581
2582 return p;
2583
2584bad_fork_core_free:
2585 sched_core_free(p);
2586 spin_unlock(&current->sighand->siglock);
2587 write_unlock_irq(&tasklist_lock);
2588bad_fork_cancel_cgroup:
2589 cgroup_cancel_fork(p, args);
2590bad_fork_put_pidfd:
2591 if (clone_flags & CLONE_PIDFD) {
2592 fput(pidfile);
2593 put_unused_fd(pidfd);
2594 }
2595bad_fork_free_pid:
2596 if (pid != &init_struct_pid)
2597 free_pid(pid);
2598bad_fork_cleanup_thread:
2599 exit_thread(p);
2600bad_fork_cleanup_io:
2601 if (p->io_context)
2602 exit_io_context(p);
2603bad_fork_cleanup_namespaces:
2604 exit_nsproxy_namespaces(p);
2605bad_fork_cleanup_mm:
2606 sched_cache_fork_cleanup(p);
2607 if (p->mm) {
2608 mm_clear_owner(p->mm, p);
2609 mmput(p->mm);
2610 }
2611bad_fork_cleanup_signal:
2612 if (!(clone_flags & CLONE_THREAD))
2613 free_signal_struct(p->signal);
2614bad_fork_cleanup_sighand:
2615 __cleanup_sighand(p->sighand);
2616bad_fork_cleanup_fs:
2617 exit_fs(p); /* blocking */
2618bad_fork_cleanup_files:
2619 exit_files(p); /* blocking */
2620bad_fork_cleanup_semundo:
2621 exit_sem(p);
2622bad_fork_cleanup_security:
2623 security_task_free(p);
2624bad_fork_cleanup_audit:
2625 audit_free(p);
2626bad_fork_cleanup_perf:
2627 perf_event_free_task(p);
2628bad_fork_sched_cancel_fork:
2629 sched_cancel_fork(p);
2630bad_fork_cleanup_policy:
2631 lockdep_free_task(p);
2632#ifdef CONFIG_NUMA
2633 mpol_put(p->mempolicy);
2634#endif
2635bad_fork_cleanup_delayacct:
2636 io_uring_free(p);
2637 delayacct_tsk_free(p);
2638bad_fork_cleanup_count:
2639 dec_rlimit_ucounts(task_ucounts(p), UCOUNT_RLIMIT_NPROC, 1);
2640 exit_cred_namespaces(p);
2641 exit_creds(p);
2642bad_fork_free:
2643 WRITE_ONCE(p->__state, TASK_DEAD);
2644 exit_task_stack_account(p);
2645 put_task_stack(p);
2646 delayed_free_task(p);
2647fork_out:
2648 spin_lock_irq(&current->sighand->siglock);
2649 hlist_del_init(&delayed.node);
2650 spin_unlock_irq(&current->sighand->siglock);
2651 return ERR_PTR(retval);
2652}
2653
2654static inline void init_idle_pids(struct task_struct *idle)
2655{
2656 enum pid_type type;
2657
2658 for (type = PIDTYPE_PID; type < PIDTYPE_MAX; ++type) {
2659 INIT_HLIST_NODE(&idle->pid_links[type]); /* not really needed */
2660 init_task_pid(idle, type, &init_struct_pid);
2661 }
2662}
2663
2664static int idle_dummy(void *dummy)
2665{
2666 /* This function is never called */
2667 return 0;
2668}
2669
2670struct task_struct * __init fork_idle(int cpu)
2671{
2672 struct task_struct *task;
2673 struct kernel_clone_args args = {
2674 .flags = CLONE_VM,
2675 .fn = &idle_dummy,
2676 .fn_arg = NULL,
2677 .kthread = 1,
2678 .idle = 1,
2679 };
2680
2681 task = copy_process(&init_struct_pid, 0, cpu_to_node(cpu), &args);
2682 if (!IS_ERR(task)) {
2683 init_idle_pids(task);
2684 init_idle(task, cpu);
2685 }
2686
2687 return task;
2688}
2689
2690/*
2691 * This is like kernel_clone(), but shaved down and tailored to just
2692 * creating io_uring workers. It returns a created task, or an error pointer.
2693 * The returned task is inactive, and the caller must fire it up through
2694 * wake_up_new_task(p). All signals are blocked in the created task.
2695 */
2696struct task_struct *create_io_thread(int (*fn)(void *), void *arg, int node)
2697{
2698 unsigned long flags = CLONE_FS|CLONE_FILES|CLONE_SIGHAND|CLONE_THREAD|
2699 CLONE_IO|CLONE_VM|CLONE_UNTRACED;
2700 struct kernel_clone_args args = {
2701 .flags = flags,
2702 .fn = fn,
2703 .fn_arg = arg,
2704 .io_thread = 1,
2705 .user_worker = 1,
2706 };
2707
2708 return copy_process(NULL, 0, node, &args);
2709}
2710
2711/*
2712 * Ok, this is the main fork-routine.
2713 *
2714 * It copies the process, and if successful kick-starts
2715 * it and waits for it to finish using the VM if required.
2716 */
2717pid_t kernel_clone(struct kernel_clone_args *args)
2718{
2719 u64 clone_flags = args->flags;
2720 struct completion vfork;
2721 struct pid *pid;
2722 struct task_struct *p;
2723 int trace = 0;
2724 pid_t nr;
2725
2726 /*
2727 * Creating an empty mount namespace implies creating a new mount
2728 * namespace. Set this before copy_process() so that the
2729 * CLONE_NEWNS|CLONE_FS mutual exclusion check works correctly.
2730 */
2731 if (clone_flags & CLONE_EMPTY_MNTNS) {
2732 clone_flags |= CLONE_NEWNS;
2733 args->flags = clone_flags;
2734 }
2735
2736 /*
2737 * For legacy clone() calls, CLONE_PIDFD uses the parent_tid argument
2738 * to return the pidfd. Hence, CLONE_PIDFD and CLONE_PARENT_SETTID are
2739 * mutually exclusive. With clone3() CLONE_PIDFD has grown a separate
2740 * field in struct clone_args and it still doesn't make sense to have
2741 * them both point at the same memory location. Performing this check
2742 * here has the advantage that we don't need to have a separate helper
2743 * to check for legacy clone().
2744 */
2745 if ((clone_flags & CLONE_PIDFD) &&
2746 (clone_flags & CLONE_PARENT_SETTID) &&
2747 (args->pidfd == args->parent_tid))
2748 return -EINVAL;
2749
2750 if (!valid_signal(args->exit_signal))
2751 return -EINVAL;
2752
2753 /*
2754 * Determine whether and which event to report to ptracer. When
2755 * called from kernel_thread or CLONE_UNTRACED is explicitly
2756 * requested, no event is reported; otherwise, report if the event
2757 * for the type of forking is enabled.
2758 */
2759 if (!(clone_flags & CLONE_UNTRACED)) {
2760 if (clone_flags & CLONE_VFORK)
2761 trace = PTRACE_EVENT_VFORK;
2762 else if (args->exit_signal != SIGCHLD)
2763 trace = PTRACE_EVENT_CLONE;
2764 else
2765 trace = PTRACE_EVENT_FORK;
2766
2767 if (likely(!ptrace_event_enabled(current, trace)))
2768 trace = 0;
2769 }
2770
2771 p = copy_process(NULL, trace, NUMA_NO_NODE, args);
2772 add_latent_entropy();
2773
2774 if (IS_ERR(p))
2775 return PTR_ERR(p);
2776
2777 /*
2778 * Do this prior waking up the new thread - the thread pointer
2779 * might get invalid after that point, if the thread exits quickly.
2780 */
2781 trace_sched_process_fork(current, p);
2782
2783 pid = get_task_pid(p, PIDTYPE_PID);
2784 nr = pid_vnr(pid);
2785
2786 if (clone_flags & CLONE_PARENT_SETTID)
2787 put_user(nr, args->parent_tid);
2788
2789 if (clone_flags & CLONE_VFORK) {
2790 p->vfork_done = &vfork;
2791 init_completion(&vfork);
2792 get_task_struct(p);
2793 }
2794
2795 if (IS_ENABLED(CONFIG_LRU_GEN_WALKS_MMU) && !(clone_flags & CLONE_VM)) {
2796 /* lock the task to synchronize with memcg migration */
2797 task_lock(p);
2798 lru_gen_add_mm(p->mm);
2799 task_unlock(p);
2800 }
2801
2802 wake_up_new_task(p);
2803
2804 /* forking complete and child started to run, tell ptracer */
2805 if (unlikely(trace))
2806 ptrace_event_pid(trace, pid);
2807
2808 if (clone_flags & CLONE_VFORK) {
2809 if (!wait_for_vfork_done(p, &vfork))
2810 ptrace_event_pid(PTRACE_EVENT_VFORK_DONE, pid);
2811 }
2812
2813 put_pid(pid);
2814 return nr;
2815}
2816
2817/*
2818 * Create a kernel thread.
2819 */
2820pid_t kernel_thread(int (*fn)(void *), void *arg, const char *name,
2821 unsigned long flags)
2822{
2823 struct kernel_clone_args args = {
2824 .flags = ((flags | CLONE_VM | CLONE_UNTRACED) & ~CSIGNAL),
2825 .exit_signal = (flags & CSIGNAL),
2826 .fn = fn,
2827 .fn_arg = arg,
2828 .name = name,
2829 .kthread = 1,
2830 };
2831
2832 return kernel_clone(&args);
2833}
2834
2835/*
2836 * Create a user mode thread.
2837 */
2838pid_t user_mode_thread(int (*fn)(void *), void *arg, unsigned long flags)
2839{
2840 struct kernel_clone_args args = {
2841 .flags = ((flags | CLONE_VM | CLONE_UNTRACED) & ~CSIGNAL),
2842 .exit_signal = (flags & CSIGNAL),
2843 .fn = fn,
2844 .fn_arg = arg,
2845 .umh = 1,
2846 };
2847
2848 return kernel_clone(&args);
2849}
2850
2851#ifdef __ARCH_WANT_SYS_FORK
2852SYSCALL_DEFINE0(fork)
2853{
2854#ifdef CONFIG_MMU
2855 struct kernel_clone_args args = {
2856 .exit_signal = SIGCHLD,
2857 };
2858
2859 return kernel_clone(&args);
2860#else
2861 /* can not support in nommu mode */
2862 return -EINVAL;
2863#endif
2864}
2865#endif
2866
2867#ifdef __ARCH_WANT_SYS_VFORK
2868SYSCALL_DEFINE0(vfork)
2869{
2870 struct kernel_clone_args args = {
2871 .flags = CLONE_VFORK | CLONE_VM,
2872 .exit_signal = SIGCHLD,
2873 };
2874
2875 return kernel_clone(&args);
2876}
2877#endif
2878
2879#ifdef __ARCH_WANT_SYS_CLONE
2880#ifdef CONFIG_CLONE_BACKWARDS
2881SYSCALL_DEFINE5(clone, unsigned long, clone_flags, unsigned long, newsp,
2882 int __user *, parent_tidptr,
2883 unsigned long, tls,
2884 int __user *, child_tidptr)
2885#elif defined(CONFIG_CLONE_BACKWARDS2)
2886SYSCALL_DEFINE5(clone, unsigned long, newsp, unsigned long, clone_flags,
2887 int __user *, parent_tidptr,
2888 int __user *, child_tidptr,
2889 unsigned long, tls)
2890#elif defined(CONFIG_CLONE_BACKWARDS3)
2891SYSCALL_DEFINE6(clone, unsigned long, clone_flags, unsigned long, newsp,
2892 int, stack_size,
2893 int __user *, parent_tidptr,
2894 int __user *, child_tidptr,
2895 unsigned long, tls)
2896#else
2897SYSCALL_DEFINE5(clone, unsigned long, clone_flags, unsigned long, newsp,
2898 int __user *, parent_tidptr,
2899 int __user *, child_tidptr,
2900 unsigned long, tls)
2901#endif
2902{
2903 struct kernel_clone_args args = {
2904 .flags = (lower_32_bits(clone_flags) & ~CSIGNAL),
2905 .pidfd = parent_tidptr,
2906 .child_tid = child_tidptr,
2907 .parent_tid = parent_tidptr,
2908 .exit_signal = (lower_32_bits(clone_flags) & CSIGNAL),
2909 .stack = newsp,
2910 .tls = tls,
2911 };
2912
2913 return kernel_clone(&args);
2914}
2915#endif
2916
2917static noinline int copy_clone_args_from_user(struct kernel_clone_args *kargs,
2918 struct clone_args __user *uargs,
2919 size_t usize)
2920{
2921 int err;
2922 struct clone_args args;
2923 pid_t *kset_tid = kargs->set_tid;
2924
2925 BUILD_BUG_ON(offsetofend(struct clone_args, tls) !=
2926 CLONE_ARGS_SIZE_VER0);
2927 BUILD_BUG_ON(offsetofend(struct clone_args, set_tid_size) !=
2928 CLONE_ARGS_SIZE_VER1);
2929 BUILD_BUG_ON(offsetofend(struct clone_args, cgroup) !=
2930 CLONE_ARGS_SIZE_VER2);
2931 BUILD_BUG_ON(sizeof(struct clone_args) != CLONE_ARGS_SIZE_VER2);
2932
2933 if (unlikely(usize > PAGE_SIZE))
2934 return -E2BIG;
2935 if (unlikely(usize < CLONE_ARGS_SIZE_VER0))
2936 return -EINVAL;
2937
2938 err = copy_struct_from_user(&args, sizeof(args), uargs, usize);
2939 if (err)
2940 return err;
2941
2942 if (unlikely(args.set_tid_size > MAX_PID_NS_LEVEL))
2943 return -EINVAL;
2944
2945 if (unlikely(!args.set_tid && args.set_tid_size > 0))
2946 return -EINVAL;
2947
2948 if (unlikely(args.set_tid && args.set_tid_size == 0))
2949 return -EINVAL;
2950
2951 /*
2952 * Verify that higher 32bits of exit_signal are unset
2953 */
2954 if (unlikely(args.exit_signal & ~((u64)CSIGNAL)))
2955 return -EINVAL;
2956
2957 if ((args.flags & CLONE_INTO_CGROUP) &&
2958 (args.cgroup > INT_MAX || usize < CLONE_ARGS_SIZE_VER2))
2959 return -EINVAL;
2960
2961 *kargs = (struct kernel_clone_args){
2962 .flags = args.flags,
2963 .pidfd = u64_to_user_ptr(args.pidfd),
2964 .child_tid = u64_to_user_ptr(args.child_tid),
2965 .parent_tid = u64_to_user_ptr(args.parent_tid),
2966 .exit_signal = args.exit_signal,
2967 .stack = args.stack,
2968 .stack_size = args.stack_size,
2969 .tls = args.tls,
2970 .set_tid_size = args.set_tid_size,
2971 .cgroup = args.cgroup,
2972 };
2973
2974 if (args.set_tid &&
2975 copy_from_user(kset_tid, u64_to_user_ptr(args.set_tid),
2976 (kargs->set_tid_size * sizeof(pid_t))))
2977 return -EFAULT;
2978
2979 kargs->set_tid = kset_tid;
2980
2981 return 0;
2982}
2983
2984/**
2985 * clone3_stack_valid - check and prepare stack
2986 * @kargs: kernel clone args
2987 *
2988 * Verify that the stack arguments userspace gave us are sane.
2989 * In addition, set the stack direction for userspace since it's easy for us to
2990 * determine.
2991 */
2992static inline bool clone3_stack_valid(struct kernel_clone_args *kargs)
2993{
2994 if (kargs->stack == 0) {
2995 if (kargs->stack_size > 0)
2996 return false;
2997 } else {
2998 if (kargs->stack_size == 0)
2999 return false;
3000
3001 if (!access_ok((void __user *)kargs->stack, kargs->stack_size))
3002 return false;
3003
3004#if !defined(CONFIG_STACK_GROWSUP)
3005 kargs->stack += kargs->stack_size;
3006#endif
3007 }
3008
3009 return true;
3010}
3011
3012static bool clone3_args_valid(struct kernel_clone_args *kargs)
3013{
3014 /* Verify that no unknown flags are passed along. */
3015 if (kargs->flags &
3016 ~(CLONE_LEGACY_FLAGS | CLONE_CLEAR_SIGHAND |
3017 CLONE_INTO_CGROUP | CLONE_AUTOREAP | CLONE_NNP |
3018 CLONE_PIDFD_AUTOKILL | CLONE_EMPTY_MNTNS))
3019 return false;
3020
3021 /*
3022 * - make the CLONE_DETACHED bit reusable for clone3
3023 * - make the CSIGNAL bits reusable for clone3
3024 */
3025 if (kargs->flags & (CLONE_DETACHED | (CSIGNAL & (~CLONE_NEWTIME))))
3026 return false;
3027
3028 if ((kargs->flags & (CLONE_SIGHAND | CLONE_CLEAR_SIGHAND)) ==
3029 (CLONE_SIGHAND | CLONE_CLEAR_SIGHAND))
3030 return false;
3031
3032 if ((kargs->flags & (CLONE_THREAD | CLONE_PARENT)) &&
3033 kargs->exit_signal)
3034 return false;
3035
3036 if (!clone3_stack_valid(kargs))
3037 return false;
3038
3039 return true;
3040}
3041
3042/**
3043 * sys_clone3 - create a new process with specific properties
3044 * @uargs: argument structure
3045 * @size: size of @uargs
3046 *
3047 * clone3() is the extensible successor to clone()/clone2().
3048 * It takes a struct as argument that is versioned by its size.
3049 *
3050 * Return: On success, a positive PID for the child process.
3051 * On error, a negative errno number.
3052 */
3053SYSCALL_DEFINE2(clone3, struct clone_args __user *, uargs, size_t, size)
3054{
3055 int err;
3056
3057 struct kernel_clone_args kargs;
3058 pid_t set_tid[MAX_PID_NS_LEVEL];
3059
3060#ifdef __ARCH_BROKEN_SYS_CLONE3
3061#warning clone3() entry point is missing, please fix
3062 return -ENOSYS;
3063#endif
3064
3065 kargs.set_tid = set_tid;
3066
3067 err = copy_clone_args_from_user(&kargs, uargs, size);
3068 if (err)
3069 return err;
3070
3071 if (!clone3_args_valid(&kargs))
3072 return -EINVAL;
3073
3074 return kernel_clone(&kargs);
3075}
3076
3077void walk_process_tree(struct task_struct *top, proc_visitor visitor, void *data)
3078{
3079 struct task_struct *leader, *parent, *child;
3080 int res;
3081
3082 read_lock(&tasklist_lock);
3083 leader = top = top->group_leader;
3084down:
3085 for_each_thread(leader, parent) {
3086 list_for_each_entry(child, &parent->children, sibling) {
3087 res = visitor(child, data);
3088 if (res) {
3089 if (res < 0)
3090 goto out;
3091 leader = child;
3092 goto down;
3093 }
3094up:
3095 ;
3096 }
3097 }
3098
3099 if (leader != top) {
3100 child = leader;
3101 parent = child->real_parent;
3102 leader = parent->group_leader;
3103 goto up;
3104 }
3105out:
3106 read_unlock(&tasklist_lock);
3107}
3108
3109#ifndef ARCH_MIN_MMSTRUCT_ALIGN
3110#define ARCH_MIN_MMSTRUCT_ALIGN 0
3111#endif
3112
3113static void sighand_ctor(void *data)
3114{
3115 struct sighand_struct *sighand = data;
3116
3117 spin_lock_init(&sighand->siglock);
3118 init_waitqueue_head(&sighand->signalfd_wqh);
3119}
3120
3121void __init mm_cache_init(void)
3122{
3123 unsigned int mm_size;
3124
3125 /*
3126 * The mm_cpumask is located at the end of mm_struct, and is
3127 * dynamically sized based on the maximum CPU number this system
3128 * can have, taking hotplug into account (nr_cpu_ids).
3129 */
3130 mm_size = sizeof(struct mm_struct) + cpumask_size() + mm_cid_size();
3131
3132 mm_cachep = kmem_cache_create_usercopy("mm_struct",
3133 mm_size, ARCH_MIN_MMSTRUCT_ALIGN,
3134 SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_ACCOUNT,
3135 offsetof(struct mm_struct, saved_auxv),
3136 sizeof_field(struct mm_struct, saved_auxv),
3137 NULL);
3138}
3139
3140void __init proc_caches_init(void)
3141{
3142 sighand_cachep = kmem_cache_create("sighand_cache",
3143 sizeof(struct sighand_struct), 0,
3144 SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_TYPESAFE_BY_RCU|
3145 SLAB_ACCOUNT, sighand_ctor);
3146 signal_cachep = kmem_cache_create("signal_cache",
3147 sizeof(struct signal_struct), 0,
3148 SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_ACCOUNT,
3149 NULL);
3150 exec_state_init();
3151 files_cachep = kmem_cache_create("files_cache",
3152 sizeof(struct files_struct), 0,
3153 SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_ACCOUNT,
3154 NULL);
3155 fs_cachep = kmem_cache_create("fs_cache",
3156 sizeof(struct fs_struct), 0,
3157 SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_ACCOUNT,
3158 NULL);
3159 mmap_init();
3160 nsproxy_cache_init();
3161}
3162
3163/*
3164 * Check constraints on flags passed to the unshare system call.
3165 */
3166static int check_unshare_flags(unsigned long unshare_flags)
3167{
3168 if (unshare_flags & ~(CLONE_THREAD|CLONE_FS|CLONE_SIGHAND|
3169 CLONE_VM|CLONE_FILES|CLONE_SYSVSEM|
3170 CLONE_NS_ALL | UNSHARE_EMPTY_MNTNS))
3171 return -EINVAL;
3172 /*
3173 * Not implemented, but pretend it works if there is nothing
3174 * to unshare. Note that unsharing the address space or the
3175 * signal handlers also need to unshare the signal queues (aka
3176 * CLONE_THREAD).
3177 */
3178 if (unshare_flags & (CLONE_THREAD | CLONE_SIGHAND | CLONE_VM)) {
3179 if (!thread_group_empty(current))
3180 return -EINVAL;
3181 }
3182 if (unshare_flags & (CLONE_SIGHAND | CLONE_VM)) {
3183 if (refcount_read(&current->sighand->count) > 1)
3184 return -EINVAL;
3185 }
3186 if (unshare_flags & CLONE_VM) {
3187 if (!current_is_single_threaded())
3188 return -EINVAL;
3189 }
3190
3191 return 0;
3192}
3193
3194/*
3195 * Unshare the filesystem structure if it is being shared
3196 */
3197static int unshare_fs(unsigned long unshare_flags, struct fs_struct **new_fsp)
3198{
3199 struct fs_struct *fs = current->fs;
3200
3201 if (!(unshare_flags & CLONE_FS) || !fs)
3202 return 0;
3203
3204 /* don't need lock here; in the worst case we'll do useless copy */
3205 if (!(unshare_flags & CLONE_NEWNS) && fs->users == 1)
3206 return 0;
3207
3208 *new_fsp = copy_fs_struct(fs);
3209 if (!*new_fsp)
3210 return -ENOMEM;
3211
3212 return 0;
3213}
3214
3215/*
3216 * Unshare file descriptor table if it is being shared
3217 */
3218static int unshare_fd(unsigned long unshare_flags, struct files_struct **new_fdp)
3219{
3220 struct files_struct *fd = current->files;
3221
3222 if ((unshare_flags & CLONE_FILES) &&
3223 (fd && atomic_read(&fd->count) > 1)) {
3224 fd = dup_fd(fd, NULL);
3225 if (IS_ERR(fd))
3226 return PTR_ERR(fd);
3227 *new_fdp = fd;
3228 }
3229
3230 return 0;
3231}
3232
3233/*
3234 * unshare allows a process to 'unshare' part of the process
3235 * context which was originally shared using clone. copy_*
3236 * functions used by kernel_clone() cannot be used here directly
3237 * because they modify an inactive task_struct that is being
3238 * constructed. Here we are modifying the current, active,
3239 * task_struct.
3240 */
3241int ksys_unshare(unsigned long unshare_flags)
3242{
3243 struct fs_struct *new_fs = NULL;
3244 struct files_struct *new_fd = NULL;
3245 struct cred *new_cred = NULL;
3246 struct nsproxy *new_nsproxy = NULL;
3247 int do_sysvsem = 0;
3248 int err;
3249
3250 /*
3251 * If unsharing a user namespace must also unshare the thread group
3252 * and unshare the filesystem root and working directories.
3253 */
3254 if (unshare_flags & CLONE_NEWUSER)
3255 unshare_flags |= CLONE_THREAD | CLONE_FS;
3256 /*
3257 * If unsharing vm, must also unshare signal handlers.
3258 */
3259 if (unshare_flags & CLONE_VM)
3260 unshare_flags |= CLONE_SIGHAND;
3261 /*
3262 * If unsharing a signal handlers, must also unshare the signal queues.
3263 */
3264 if (unshare_flags & CLONE_SIGHAND)
3265 unshare_flags |= CLONE_THREAD;
3266 /*
3267 * If unsharing namespace, must also unshare filesystem information.
3268 */
3269 if (unshare_flags & UNSHARE_EMPTY_MNTNS)
3270 unshare_flags |= CLONE_NEWNS;
3271 if (unshare_flags & CLONE_NEWNS)
3272 unshare_flags |= CLONE_FS;
3273
3274 /* No unsharing with overriden fs state */
3275 VFS_WARN_ON_ONCE(unshare_flags & (CLONE_NEWNS | CLONE_FS) &&
3276 current->fs != current->real_fs);
3277
3278 err = check_unshare_flags(unshare_flags);
3279 if (err)
3280 goto bad_unshare_out;
3281 /*
3282 * CLONE_NEWIPC must also detach from the undolist: after switching
3283 * to a new ipc namespace, the semaphore arrays from the old
3284 * namespace are unreachable.
3285 */
3286 if (unshare_flags & (CLONE_NEWIPC|CLONE_SYSVSEM))
3287 do_sysvsem = 1;
3288 err = unshare_fs(unshare_flags, &new_fs);
3289 if (err)
3290 goto bad_unshare_out;
3291 err = unshare_fd(unshare_flags, &new_fd);
3292 if (err)
3293 goto bad_unshare_cleanup_fs;
3294 err = unshare_userns(unshare_flags, &new_cred);
3295 if (err)
3296 goto bad_unshare_cleanup_fd;
3297 err = unshare_nsproxy_namespaces(unshare_flags, &new_nsproxy,
3298 new_cred, new_fs);
3299 if (err)
3300 goto bad_unshare_cleanup_cred;
3301 if (new_cred) {
3302 err = set_cred_ucounts(new_cred);
3303 if (err)
3304 goto bad_unshare_cleanup_nsproxy;
3305 }
3306
3307 if (new_fs || new_fd || do_sysvsem || new_cred || new_nsproxy) {
3308 if (do_sysvsem) {
3309 /*
3310 * CLONE_SYSVSEM is equivalent to sys_exit().
3311 */
3312 exit_sem(current);
3313 }
3314 if (unshare_flags & CLONE_NEWIPC) {
3315 /* Orphan segments in old ns (see sem above). */
3316 exit_shm(current);
3317 shm_init_task(current);
3318 }
3319
3320 if (new_nsproxy) {
3321 switch_task_namespaces(current, new_nsproxy);
3322 new_nsproxy = NULL;
3323 }
3324
3325 if (new_fs)
3326 new_fs = switch_fs_struct(new_fs);
3327
3328 if (new_fd) {
3329 guard(task_lock)(current);
3330 swap(current->files, new_fd);
3331 }
3332
3333 if (new_cred) {
3334 /* Install the new user namespace */
3335 commit_creds(new_cred);
3336 new_cred = NULL;
3337 }
3338 }
3339
3340 perf_event_namespaces(current);
3341
3342bad_unshare_cleanup_nsproxy:
3343 if (new_nsproxy)
3344 put_nsproxy(new_nsproxy);
3345bad_unshare_cleanup_cred:
3346 if (new_cred)
3347 put_cred(new_cred);
3348bad_unshare_cleanup_fd:
3349 if (new_fd)
3350 put_files_struct(new_fd);
3351bad_unshare_cleanup_fs:
3352 if (new_fs)
3353 free_fs_struct(new_fs);
3354
3355bad_unshare_out:
3356 return err;
3357}
3358
3359SYSCALL_DEFINE1(unshare, unsigned long, unshare_flags)
3360{
3361 return ksys_unshare(unshare_flags);
3362}
3363
3364/*
3365 * Helper to unshare the files of the current task.
3366 * We don't want to expose copy_files internals to
3367 * the exec layer of the kernel.
3368 */
3369
3370int unshare_files(void)
3371{
3372 struct task_struct *task = current;
3373 struct files_struct *old, *copy = NULL;
3374 int error;
3375
3376 error = unshare_fd(CLONE_FILES, &copy);
3377 if (error || !copy)
3378 return error;
3379
3380 old = task->files;
3381 task_lock(task);
3382 task->files = copy;
3383 task_unlock(task);
3384 put_files_struct(old);
3385 return 0;
3386}
3387
3388static int sysctl_max_threads(const struct ctl_table *table, int write,
3389 void *buffer, size_t *lenp, loff_t *ppos)
3390{
3391 struct ctl_table t;
3392 int ret;
3393 int threads = max_threads;
3394 int min = 1;
3395 int max = MAX_THREADS;
3396
3397 t = *table;
3398 t.data = &threads;
3399 t.extra1 = &min;
3400 t.extra2 = &max;
3401
3402 ret = proc_dointvec_minmax(&t, write, buffer, lenp, ppos);
3403 if (ret || !write)
3404 return ret;
3405
3406 max_threads = threads;
3407
3408 return 0;
3409}
3410
3411static const struct ctl_table fork_sysctl_table[] = {
3412 {
3413 .procname = "threads-max",
3414 .data = NULL,
3415 .maxlen = sizeof(int),
3416 .mode = 0644,
3417 .proc_handler = sysctl_max_threads,
3418 },
3419};
3420
3421static int __init init_fork_sysctl(void)
3422{
3423 register_sysctl_init("kernel", fork_sysctl_table);
3424 return 0;
3425}
3426
3427subsys_initcall(init_fork_sysctl);