linux - kernel/fork.c

3422 lines · 84.2 KiB

1	// SPDX-License-Identifier: GPL-2.0-only
2	/*
3	* linux/kernel/fork.c
4	*
5	* Copyright (C) 1991, 1992 Linus Torvalds
6	*/
7
8	/*
9	* 'fork.c' contains the help-routines for the 'fork' system call
10	* (see also entry.S and others).
11	* Fork is rather simple, once you get the hang of it, but the memory
12	* management can be a bitch. See 'mm/memory.c': 'copy_page_range()'
13	*/
14
15	#include <linux/anon_inodes.h>
16	#include <linux/slab.h>
17	#include <linux/sched/autogroup.h>
18	#include <linux/sched/mm.h>
19	#include <linux/sched/user.h>
20	#include <linux/sched/numa_balancing.h>
21	#include <linux/sched/stat.h>
22	#include <linux/sched/task.h>
23	#include <linux/sched/task_stack.h>
24	#include <linux/sched/cputime.h>
25	#include <linux/sched/ext.h>
26	#include <linux/sched/exec_state.h>
27	#include <linux/seq_file.h>
28	#include <linux/rtmutex.h>
29	#include <linux/init.h>
30	#include <linux/unistd.h>
31	#include <linux/module.h>
32	#include <linux/vmalloc.h>
33	#include <linux/completion.h>
34	#include <linux/personality.h>
35	#include <linux/mempolicy.h>
36	#include <linux/sem.h>
37	#include <linux/file.h>
38	#include <linux/fdtable.h>
39	#include <linux/iocontext.h>
40	#include <linux/key.h>
41	#include <linux/kmsan.h>
42	#include <linux/binfmts.h>
43	#include <linux/mman.h>
44	#include <linux/mmu_notifier.h>
45	#include <linux/fs.h>
46	#include <linux/mm.h>
47	#include <linux/mm_inline.h>
48	#include <linux/memblock.h>
49	#include <linux/nsproxy.h>
50	#include <linux/ns/ns_common_types.h>
51	#include <linux/capability.h>
52	#include <linux/cpu.h>
53	#include <linux/cgroup.h>
54	#include <linux/security.h>
55	#include <linux/hugetlb.h>
56	#include <linux/seccomp.h>
57	#include <linux/swap.h>
58	#include <linux/syscalls.h>
59	#include <linux/syscall_user_dispatch.h>
60	#include <linux/jiffies.h>
61	#include <linux/futex.h>
62	#include <linux/compat.h>
63	#include <linux/kthread.h>
64	#include <linux/task_io_accounting_ops.h>
65	#include <linux/rcupdate.h>
66	#include <linux/ptrace.h>
67	#include <linux/mount.h>
68	#include <linux/audit.h>
69	#include <linux/memcontrol.h>
70	#include <linux/ftrace.h>
71	#include <linux/proc_fs.h>
72	#include <linux/profile.h>
73	#include <linux/rmap.h>
74	#include <linux/ksm.h>
75	#include <linux/acct.h>
76	#include <linux/userfaultfd_k.h>
77	#include <linux/tsacct_kern.h>
78	#include <linux/cn_proc.h>
79	#include <linux/freezer.h>
80	#include <linux/delayacct.h>
81	#include <linux/taskstats_kern.h>
82	#include <linux/tty.h>
83	#include <linux/fs_struct.h>
84	#include <linux/magic.h>
85	#include <linux/perf_event.h>
86	#include <linux/posix-timers.h>
87	#include <linux/user-return-notifier.h>
88	#include <linux/oom.h>
89	#include <linux/khugepaged.h>
90	#include <linux/signalfd.h>
91	#include <linux/uprobes.h>
92	#include <linux/aio.h>
93	#include <linux/compiler.h>
94	#include <linux/sysctl.h>
95	#include <linux/kcov.h>
96	#include <linux/livepatch.h>
97	#include <linux/thread_info.h>
98	#include <linux/kstack_erase.h>
99	#include <linux/kasan.h>
100	#include <linux/randomize_kstack.h>
101	#include <linux/scs.h>
102	#include <linux/io_uring.h>
103	#include <linux/io_uring_types.h>
104	#include <linux/bpf.h>
105	#include <linux/stackprotector.h>
106	#include <linux/user_events.h>
107	#include <linux/iommu.h>
108	#include <linux/rseq.h>
109	#include <uapi/linux/pidfd.h>
110	#include <linux/pidfs.h>
111	#include <linux/tick.h>
112	#include <linux/unwind_deferred.h>
113	#include <linux/pgalloc.h>
114	#include <linux/uaccess.h>
115
116	#include <asm/mmu_context.h>
117	#include <asm/cacheflush.h>
118	#include <asm/tlbflush.h>
119
120	/* For dup_mmap(). */
121	#include "../mm/internal.h"
122
123	#include <trace/events/sched.h>
124
125	#define CREATE_TRACE_POINTS
126	#include <trace/events/task.h>
127
128	#include <kunit/visibility.h>
129
130	/*
131	* Minimum number of threads to boot the kernel
132	*/
133	#define MIN_THREADS 20
134
135	/*
136	* Maximum number of threads
137	*/
138	#define MAX_THREADS FUTEX_TID_MASK
139
140	/*
141	* Protected counters by write_lock_irq(&tasklist_lock)
142	*/
143	unsigned long total_forks; /* Handle normal Linux uptimes. */
144	int nr_threads; /* The idle threads do not count.. */
145
146	static int max_threads __read_mostly; /* tunable limit on nr_threads */
147
148	#define NAMED_ARRAY_INDEX(x) [x] = __stringify(x)
149
150	static const char * const resident_page_types[] = {
151	NAMED_ARRAY_INDEX(MM_FILEPAGES),
152	NAMED_ARRAY_INDEX(MM_ANONPAGES),
153	NAMED_ARRAY_INDEX(MM_SWAPENTS),
154	NAMED_ARRAY_INDEX(MM_SHMEMPAGES),
155	};
156
157	DEFINE_PER_CPU(unsigned long, process_counts) = 0;
158
159	__cacheline_aligned DEFINE_RWLOCK(tasklist_lock); /* outer */
160
161	#ifdef CONFIG_PROVE_RCU
162	int lockdep_tasklist_lock_is_held(void)
163	{
164	return lockdep_is_held(&tasklist_lock);
165	}
166	EXPORT_SYMBOL_GPL(lockdep_tasklist_lock_is_held);
167	#endif /* #ifdef CONFIG_PROVE_RCU */
168
169	int nr_processes(void)
170	{
171	int cpu;
172	int total = 0;
173
174	for_each_possible_cpu(cpu)
175	total += per_cpu(process_counts, cpu);
176
177	return total;
178	}
179
180	void __weak arch_release_task_struct(struct task_struct *tsk)
181	{
182	}
183
184	static struct kmem_cache *task_struct_cachep;
185
186	static inline struct task_struct *alloc_task_struct_node(int node)
187	{
188	return kmem_cache_alloc_node(task_struct_cachep, GFP_KERNEL, node);
189	}
190
191	static inline void free_task_struct(struct task_struct *tsk)
192	{
193	kmem_cache_free(task_struct_cachep, tsk);
194	}
195
196	#ifdef CONFIG_VMAP_STACK
197	/*
198	* vmalloc() is a bit slow, and calling vfree() enough times will force a TLB
199	* flush. Try to minimize the number of calls by caching stacks.
200	*/
201	#define NR_CACHED_STACKS 2
202	static DEFINE_PER_CPU(struct vm_struct *, cached_stacks[NR_CACHED_STACKS]);
203	/*
204	* Allocated stacks are cached and later reused by new threads, so memcg
205	* accounting is performed by the code assigning/releasing stacks to tasks.
206	* We need a zeroed memory without __GFP_ACCOUNT.
207	*/
208	#define GFP_VMAP_STACK (GFP_KERNEL \| __GFP_ZERO \| __GFP_SKIP_KASAN)
209
210	struct vm_stack {
211	struct rcu_head rcu;
212	struct vm_struct *stack_vm_area;
213	};
214
215	static struct vm_struct alloc_thread_stack_node_from_cache(struct task_struct tsk, int node)
216	{
217	struct vm_struct *vm_area;
218	unsigned int i;
219
220	/*
221	* If the node has memory, we are guaranteed the stacks are backed by local pages.
222	* Otherwise the pages are arbitrary.
223	*
224	* Note that depending on cpuset it is possible we will get migrated to a different
225	* node immediately after allocating here, so this does not guarantee locality for
226	* arbitrary callers.
227	*/
228	scoped_guard(preempt) {
229	if (node != NUMA_NO_NODE && numa_node_id() != node)
230	return NULL;
231
232	for (i = 0; i < NR_CACHED_STACKS; i++) {
233	vm_area = this_cpu_xchg(cached_stacks[i], NULL);
234	if (vm_area)
235	return vm_area;
236	}
237	}
238
239	return NULL;
240	}
241
242	static bool try_release_thread_stack_to_cache(struct vm_struct *vm_area)
243	{
244	unsigned int i;
245	int nid;
246
247	/*
248	* Don't cache stacks if any of the pages don't match the local domain, unless
249	* there is no local memory to begin with.
250	*
251	* Note that lack of local memory does not automatically mean it makes no difference
252	* performance-wise which other domain backs the stack. In this case we are merely
253	* trying to avoid constantly going to vmalloc.
254	*/
255	scoped_guard(preempt) {
256	nid = numa_node_id();
257	if (node_state(nid, N_MEMORY)) {
258	for (i = 0; i < vm_area->nr_pages; i++) {
259	struct page *page = vm_area->pages[i];
260	if (page_to_nid(page) != nid)
261	return false;
262	}
263	}
264
265	for (i = 0; i < NR_CACHED_STACKS; i++) {
266	struct vm_struct *tmp = NULL;
267
268	if (this_cpu_try_cmpxchg(cached_stacks[i], &tmp, vm_area))
269	return true;
270	}
271	}
272	return false;
273	}
274
275	static void thread_stack_free_rcu(struct rcu_head *rh)
276	{
277	struct vm_stack *vm_stack = container_of(rh, struct vm_stack, rcu);
278	struct vm_struct *vm_area = vm_stack->stack_vm_area;
279
280	if (try_release_thread_stack_to_cache(vm_stack->stack_vm_area))
281	return;
282
283	vfree(vm_area->addr);
284	}
285
286	static void thread_stack_delayed_free(struct task_struct *tsk)
287	{
288	struct vm_stack *vm_stack = tsk->stack;
289
290	vm_stack->stack_vm_area = tsk->stack_vm_area;
291	call_rcu(&vm_stack->rcu, thread_stack_free_rcu);
292	}
293
294	static int free_vm_stack_cache(unsigned int cpu)
295	{
296	struct vm_struct **cached_vm_stack_areas = per_cpu_ptr(cached_stacks, cpu);
297	int i;
298
299	for (i = 0; i < NR_CACHED_STACKS; i++) {
300	struct vm_struct *vm_area = cached_vm_stack_areas[i];
301
302	if (!vm_area)
303	continue;
304
305	vfree(vm_area->addr);
306	cached_vm_stack_areas[i] = NULL;
307	}
308
309	return 0;
310	}
311
312	static int memcg_charge_kernel_stack(struct vm_struct *vm_area)
313	{
314	int i;
315	int ret;
316	int nr_charged = 0;
317
318	BUG_ON(vm_area->nr_pages != THREAD_SIZE / PAGE_SIZE);
319
320	for (i = 0; i < THREAD_SIZE / PAGE_SIZE; i++) {
321	ret = memcg_kmem_charge_page(vm_area->pages[i], GFP_KERNEL, 0);
322	if (ret)
323	goto err;
324	nr_charged++;
325	}
326	return 0;
327	err:
328	for (i = 0; i < nr_charged; i++)
329	memcg_kmem_uncharge_page(vm_area->pages[i], 0);
330	return ret;
331	}
332
333	static int alloc_thread_stack_node(struct task_struct *tsk, int node)
334	{
335	struct vm_struct *vm_area;
336	void *stack;
337
338	vm_area = alloc_thread_stack_node_from_cache(tsk, node);
339	if (vm_area) {
340	if (memcg_charge_kernel_stack(vm_area)) {
341	vfree(vm_area->addr);
342	return -ENOMEM;
343	}
344
345	/* Reset stack metadata. */
346	if (!kasan_hw_tags_enabled())
347	kasan_unpoison_range(vm_area->addr, THREAD_SIZE);
348
349	stack = kasan_reset_tag(vm_area->addr);
350
351	/* Clear stale pointers from reused stack. */
352	clear_pages(vm_area->addr, vm_area->nr_pages);
353
354	tsk->stack_vm_area = vm_area;
355	tsk->stack = stack;
356	return 0;
357	}
358
359	stack = __vmalloc_node(THREAD_SIZE, THREAD_ALIGN,
360	GFP_VMAP_STACK,
361	node, __builtin_return_address(0));
362	if (!stack)
363	return -ENOMEM;
364
365	vm_area = find_vm_area(stack);
366	if (memcg_charge_kernel_stack(vm_area)) {
367	vfree(stack);
368	return -ENOMEM;
369	}
370	/*
371	* We can't call find_vm_area() in interrupt context, and
372	* free_thread_stack() can be called in interrupt context,
373	* so cache the vm_struct.
374	*/
375	tsk->stack_vm_area = vm_area;
376	stack = kasan_reset_tag(stack);
377	tsk->stack = stack;
378	return 0;
379	}
380
381	static void free_thread_stack(struct task_struct *tsk)
382	{
383	if (!try_release_thread_stack_to_cache(tsk->stack_vm_area))
384	thread_stack_delayed_free(tsk);
385
386	tsk->stack = NULL;
387	tsk->stack_vm_area = NULL;
388	}
389
390	#else /* !CONFIG_VMAP_STACK */
391
392	/*
393	* Allocate pages if THREAD_SIZE is >= PAGE_SIZE, otherwise use a
394	* kmemcache based allocator.
395	*/
396	#if THREAD_SIZE >= PAGE_SIZE
397
398	static void thread_stack_free_rcu(struct rcu_head *rh)
399	{
400	__free_pages(virt_to_page(rh), THREAD_SIZE_ORDER);
401	}
402
403	static void thread_stack_delayed_free(struct task_struct *tsk)
404	{
405	struct rcu_head *rh = tsk->stack;
406
407	call_rcu(rh, thread_stack_free_rcu);
408	}
409
410	static int alloc_thread_stack_node(struct task_struct *tsk, int node)
411	{
412	struct page *page = alloc_pages_node(node, THREADINFO_GFP,
413	THREAD_SIZE_ORDER);
414
415	if (likely(page)) {
416	tsk->stack = kasan_reset_tag(page_address(page));
417	return 0;
418	}
419	return -ENOMEM;
420	}
421
422	static void free_thread_stack(struct task_struct *tsk)
423	{
424	thread_stack_delayed_free(tsk);
425	tsk->stack = NULL;
426	}
427
428	#else /* !(THREAD_SIZE >= PAGE_SIZE) */
429
430	static struct kmem_cache *thread_stack_cache;
431
432	static void thread_stack_free_rcu(struct rcu_head *rh)
433	{
434	kmem_cache_free(thread_stack_cache, rh);
435	}
436
437	static void thread_stack_delayed_free(struct task_struct *tsk)
438	{
439	struct rcu_head *rh = tsk->stack;
440
441	call_rcu(rh, thread_stack_free_rcu);
442	}
443
444	static int alloc_thread_stack_node(struct task_struct *tsk, int node)
445	{
446	unsigned long *stack;
447	stack = kmem_cache_alloc_node(thread_stack_cache, THREADINFO_GFP, node);
448	stack = kasan_reset_tag(stack);
449	tsk->stack = stack;
450	return stack ? 0 : -ENOMEM;
451	}
452
453	static void free_thread_stack(struct task_struct *tsk)
454	{
455	thread_stack_delayed_free(tsk);
456	tsk->stack = NULL;
457	}
458
459	void thread_stack_cache_init(void)
460	{
461	thread_stack_cache = kmem_cache_create_usercopy("thread_stack",
462	THREAD_SIZE, THREAD_SIZE, 0, 0,
463	THREAD_SIZE, NULL);
464	BUG_ON(thread_stack_cache == NULL);
465	}
466
467	#endif /* THREAD_SIZE >= PAGE_SIZE */
468	#endif /* CONFIG_VMAP_STACK */
469
470	/* SLAB cache for signal_struct structures (tsk->signal) */
471	static struct kmem_cache *signal_cachep;
472
473	/* SLAB cache for sighand_struct structures (tsk->sighand) */
474	struct kmem_cache *sighand_cachep;
475
476	/* SLAB cache for files_struct structures (tsk->files) */
477	struct kmem_cache *files_cachep;
478
479	/* SLAB cache for fs_struct structures (tsk->fs) */
480	struct kmem_cache *fs_cachep;
481
482	/* SLAB cache for mm_struct structures (tsk->mm) */
483	static struct kmem_cache *mm_cachep;
484
485	static void account_kernel_stack(struct task_struct *tsk, int account)
486	{
487	if (IS_ENABLED(CONFIG_VMAP_STACK)) {
488	struct vm_struct *vm_area = task_stack_vm_area(tsk);
489	int i;
490
491	for (i = 0; i < THREAD_SIZE / PAGE_SIZE; i++)
492	mod_lruvec_page_state(vm_area->pages[i], NR_KERNEL_STACK_KB,
493	account * (PAGE_SIZE / 1024));
494	} else {
495	void *stack = task_stack_page(tsk);
496
497	/* All stack pages are in the same node. */
498	mod_lruvec_kmem_state(stack, NR_KERNEL_STACK_KB,
499	account * (THREAD_SIZE / 1024));
500	}
501	}
502
503	void exit_task_stack_account(struct task_struct *tsk)
504	{
505	account_kernel_stack(tsk, -1);
506
507	if (IS_ENABLED(CONFIG_VMAP_STACK)) {
508	struct vm_struct *vm_area;
509	int i;
510
511	vm_area = task_stack_vm_area(tsk);
512	for (i = 0; i < THREAD_SIZE / PAGE_SIZE; i++)
513	memcg_kmem_uncharge_page(vm_area->pages[i], 0);
514	}
515	}
516
517	static void release_task_stack(struct task_struct *tsk)
518	{
519	if (WARN_ON(READ_ONCE(tsk->__state) != TASK_DEAD))
520	return; /* Better to leak the stack than to free prematurely */
521
522	free_thread_stack(tsk);
523	}
524
525	#ifdef CONFIG_THREAD_INFO_IN_TASK
526	void put_task_stack(struct task_struct *tsk)
527	{
528	if (refcount_dec_and_test(&tsk->stack_refcount))
529	release_task_stack(tsk);
530	}
531	#endif
532
533	void free_task(struct task_struct *tsk)
534	{
535	#ifdef CONFIG_SECCOMP
536	WARN_ON_ONCE(tsk->seccomp.filter);
537	#endif
538	release_user_cpus_ptr(tsk);
539	scs_release(tsk);
540	smp_task_ipi_mask_free(tsk);
541
542	#ifndef CONFIG_THREAD_INFO_IN_TASK
543	/*
544	* The task is finally done with both the stack and thread_info,
545	* so free both.
546	*/
547	release_task_stack(tsk);
548	#else
549	/*
550	* If the task had a separate stack allocation, it should be gone
551	* by now.
552	*/
553	WARN_ON_ONCE(refcount_read(&tsk->stack_refcount) != 0);
554	#endif
555	rt_mutex_debug_task_free(tsk);
556	ftrace_graph_exit_task(tsk);
557	arch_release_task_struct(tsk);
558	if (tsk->flags & PF_KTHREAD)
559	free_kthread_struct(tsk);
560	bpf_task_storage_free(tsk);
561	put_task_exec_state(rcu_access_pointer(tsk->exec_state));
562	free_task_struct(tsk);
563	}
564	EXPORT_SYMBOL(free_task);
565
566	void dup_mm_exe_file(struct mm_struct mm, struct mm_struct oldmm)
567	{
568	struct file *exe_file;
569
570	exe_file = get_mm_exe_file(oldmm);
571	RCU_INIT_POINTER(mm->exe_file, exe_file);
572	/*
573	* We depend on the oldmm having properly denied write access to the
574	* exe_file already.
575	*/
576	if (exe_file && exe_file_deny_write_access(exe_file))
577	pr_warn_once("exe_file_deny_write_access() failed in %s\n", __func__);
578	}
579
580	#ifdef CONFIG_MMU
581	static inline int mm_alloc_pgd(struct mm_struct *mm)
582	{
583	mm->pgd = pgd_alloc(mm);
584	if (unlikely(!mm->pgd))
585	return -ENOMEM;
586	return 0;
587	}
588
589	static inline void mm_free_pgd(struct mm_struct *mm)
590	{
591	pgd_free(mm, mm->pgd);
592	}
593	#else
594	#define mm_alloc_pgd(mm) (0)
595	#define mm_free_pgd(mm)
596	#endif /* CONFIG_MMU */
597
598	#ifdef CONFIG_MM_ID
599	static DEFINE_IDA(mm_ida);
600
601	static inline int mm_alloc_id(struct mm_struct *mm)
602	{
603	int ret;
604
605	ret = ida_alloc_range(&mm_ida, MM_ID_MIN, MM_ID_MAX, GFP_KERNEL);
606	if (ret < 0)
607	return ret;
608	mm->mm_id = ret;
609	return 0;
610	}
611
612	static inline void mm_free_id(struct mm_struct *mm)
613	{
614	const mm_id_t id = mm->mm_id;
615
616	mm->mm_id = MM_ID_DUMMY;
617	if (id == MM_ID_DUMMY)
618	return;
619	if (WARN_ON_ONCE(id < MM_ID_MIN \|\| id > MM_ID_MAX))
620	return;
621	ida_free(&mm_ida, id);
622	}
623	#else /* !CONFIG_MM_ID */
624	static inline int mm_alloc_id(struct mm_struct *mm) { return 0; }
625	static inline void mm_free_id(struct mm_struct *mm) {}
626	#endif /* CONFIG_MM_ID */
627
628	static void check_mm(struct mm_struct *mm)
629	{
630	int i;
631
632	BUILD_BUG_ON_MSG(ARRAY_SIZE(resident_page_types) != NR_MM_COUNTERS,
633	"Please make sure 'struct resident_page_types[]' is updated as well");
634
635	for (i = 0; i < NR_MM_COUNTERS; i++) {
636	long x = percpu_counter_sum(&mm->rss_stat[i]);
637
638	if (unlikely(x)) {
639	pr_alert("BUG: Bad rss-counter state mm:%p type:%s val:%ld Comm:%s Pid:%d\n",
640	mm, resident_page_types[i], x,
641	current->comm,
642	task_pid_nr(current));
643	}
644	}
645
646	if (mm_pgtables_bytes(mm))
647	pr_alert("BUG: non-zero pgtables_bytes on freeing mm: %ld\n",
648	mm_pgtables_bytes(mm));
649
650	#if defined(CONFIG_TRANSPARENT_HUGEPAGE) && !defined(CONFIG_SPLIT_PMD_PTLOCKS)
651	VM_BUG_ON_MM(mm->pmd_huge_pte, mm);
652	#endif
653	}
654
655	#define allocate_mm() (kmem_cache_alloc(mm_cachep, GFP_KERNEL))
656	#define free_mm(mm) (kmem_cache_free(mm_cachep, (mm)))
657
658	static void do_check_lazy_tlb(void *arg)
659	{
660	struct mm_struct *mm = arg;
661
662	WARN_ON_ONCE(current->active_mm == mm);
663	}
664
665	static void do_shoot_lazy_tlb(void *arg)
666	{
667	struct mm_struct *mm = arg;
668
669	if (current->active_mm == mm) {
670	WARN_ON_ONCE(current->mm);
671	current->active_mm = &init_mm;
672	switch_mm(mm, &init_mm, current);
673	}
674	}
675
676	static void cleanup_lazy_tlbs(struct mm_struct *mm)
677	{
678	if (!IS_ENABLED(CONFIG_MMU_LAZY_TLB_SHOOTDOWN)) {
679	/*
680	* In this case, lazy tlb mms are refounted and would not reach
681	* __mmdrop until all CPUs have switched away and mmdrop()ed.
682	*/
683	return;
684	}
685
686	/*
687	* Lazy mm shootdown does not refcount "lazy tlb mm" usage, rather it
688	* requires lazy mm users to switch to another mm when the refcount
689	* drops to zero, before the mm is freed. This requires IPIs here to
690	* switch kernel threads to init_mm.
691	*
692	* archs that use IPIs to flush TLBs can piggy-back that lazy tlb mm
693	* switch with the final userspace teardown TLB flush which leaves the
694	* mm lazy on this CPU but no others, reducing the need for additional
695	* IPIs here. There are cases where a final IPI is still required here,
696	* such as the final mmdrop being performed on a different CPU than the
697	* one exiting, or kernel threads using the mm when userspace exits.
698	*
699	* IPI overheads have not found to be expensive, but they could be
700	* reduced in a number of possible ways, for example (roughly
701	* increasing order of complexity):
702	* - The last lazy reference created by exit_mm() could instead switch
703	* to init_mm, however it's probable this will run on the same CPU
704	* immediately afterwards, so this may not reduce IPIs much.
705	* - A batch of mms requiring IPIs could be gathered and freed at once.
706	* - CPUs store active_mm where it can be remotely checked without a
707	* lock, to filter out false-positives in the cpumask.
708	* - After mm_users or mm_count reaches zero, switching away from the
709	* mm could clear mm_cpumask to reduce some IPIs, perhaps together
710	* with some batching or delaying of the final IPIs.
711	* - A delayed freeing and RCU-like quiescing sequence based on mm
712	* switching to avoid IPIs completely.
713	*/
714	on_each_cpu_mask(mm_cpumask(mm), do_shoot_lazy_tlb, (void *)mm, 1);
715	if (IS_ENABLED(CONFIG_DEBUG_VM_SHOOT_LAZIES))
716	on_each_cpu(do_check_lazy_tlb, (void *)mm, 1);
717	}
718
719	/*
720	* Called when the last reference to the mm
721	* is dropped: either by a lazy thread or by
722	* mmput. Free the page directory and the mm.
723	*/
724	void __mmdrop(struct mm_struct *mm)
725	{
726	BUG_ON(mm == &init_mm);
727	WARN_ON_ONCE(mm == current->mm);
728
729	/* Ensure no CPUs are using this as their lazy tlb mm */
730	cleanup_lazy_tlbs(mm);
731
732	WARN_ON_ONCE(mm == current->active_mm);
733	mm_destroy_sched(mm);
734	mm_free_pgd(mm);
735	mm_free_id(mm);
736	destroy_context(mm);
737	mmu_notifier_subscriptions_destroy(mm);
738	check_mm(mm);
739	mm_pasid_drop(mm);
740	mm_destroy_cid(mm);
741	percpu_counter_destroy_many(mm->rss_stat, NR_MM_COUNTERS);
742
743	free_mm(mm);
744	}
745	EXPORT_SYMBOL_GPL(__mmdrop);
746
747	static void mmdrop_async_fn(struct work_struct *work)
748	{
749	struct mm_struct *mm;
750
751	mm = container_of(work, struct mm_struct, async_put_work);
752	__mmdrop(mm);
753	}
754
755	static void mmdrop_async(struct mm_struct *mm)
756	{
757	if (unlikely(atomic_dec_and_test(&mm->mm_count))) {
758	INIT_WORK(&mm->async_put_work, mmdrop_async_fn);
759	schedule_work(&mm->async_put_work);
760	}
761	}
762
763	static inline void free_signal_struct(struct signal_struct *sig)
764	{
765	taskstats_tgid_free(sig);
766	sched_autogroup_exit(sig);
767	/*
768	* __mmdrop is not safe to call from softirq context on x86 due to
769	* pgd_dtor so postpone it to the async context
770	*/
771	if (sig->oom_mm)
772	mmdrop_async(sig->oom_mm);
773	kmem_cache_free(signal_cachep, sig);
774	}
775
776	static inline void put_signal_struct(struct signal_struct *sig)
777	{
778	if (refcount_dec_and_test(&sig->sigcnt))
779	free_signal_struct(sig);
780	}
781
782	void __put_task_struct(struct task_struct *tsk)
783	{
784	WARN_ON(!tsk->exit_state);
785	WARN_ON(refcount_read(&tsk->usage));
786	WARN_ON(tsk == current);
787
788	unwind_task_free(tsk);
789	io_uring_free(tsk);
790	cgroup_task_free(tsk);
791	task_numa_free(tsk, true);
792	security_task_free(tsk);
793	exit_creds(tsk);
794	delayacct_tsk_free(tsk);
795	put_signal_struct(tsk->signal);
796	sched_core_free(tsk);
797	free_task(tsk);
798	}
799	EXPORT_SYMBOL_GPL(__put_task_struct);
800
801	void __put_task_struct_rcu_cb(struct rcu_head *rhp)
802	{
803	struct task_struct *task = container_of(rhp, struct task_struct, rcu);
804
805	__put_task_struct(task);
806	}
807	EXPORT_SYMBOL_GPL(__put_task_struct_rcu_cb);
808
809	void __init __weak arch_task_cache_init(void) { }
810
811	/*
812	* set_max_threads
813	*/
814	static void __init set_max_threads(unsigned int max_threads_suggested)
815	{
816	u64 threads;
817	unsigned long nr_pages = memblock_estimated_nr_free_pages();
818
819	/*
820	* The number of threads shall be limited such that the thread
821	* structures may only consume a small part of the available memory.
822	*/
823	if (fls64(nr_pages) + fls64(PAGE_SIZE) > 64)
824	threads = MAX_THREADS;
825	else
826	threads = div64_u64((u64) nr_pages * (u64) PAGE_SIZE,
827	(u64) THREAD_SIZE * 8UL);
828
829	if (threads > max_threads_suggested)
830	threads = max_threads_suggested;
831
832	max_threads = clamp_t(u64, threads, MIN_THREADS, MAX_THREADS);
833	}
834
835	#ifdef CONFIG_ARCH_WANTS_DYNAMIC_TASK_STRUCT
836	/* Initialized by the architecture: */
837	int arch_task_struct_size __read_mostly;
838	#endif
839
840	static void __init task_struct_whitelist(unsigned long offset, unsigned long size)
841	{
842	/* Fetch thread_struct whitelist for the architecture. */
843	arch_thread_struct_whitelist(offset, size);
844
845	/*
846	* Handle zero-sized whitelist or empty thread_struct, otherwise
847	* adjust offset to position of thread_struct in task_struct.
848	*/
849	if (unlikely(*size == 0))
850	*offset = 0;
851	else
852	*offset += offsetof(struct task_struct, thread);
853	}
854
855	void __init fork_init(void)
856	{
857	int i;
858	#ifndef ARCH_MIN_TASKALIGN
859	#define ARCH_MIN_TASKALIGN 0
860	#endif
861	int align = max_t(int, L1_CACHE_BYTES, ARCH_MIN_TASKALIGN);
862	unsigned long useroffset, usersize;
863
864	/* create a slab on which task_structs can be allocated */
865	task_struct_whitelist(&useroffset, &usersize);
866	task_struct_cachep = kmem_cache_create_usercopy("task_struct",
867	arch_task_struct_size, align,
868	SLAB_PANIC\|SLAB_ACCOUNT,
869	useroffset, usersize, NULL);
870
871	/* do the arch specific task caches init */
872	arch_task_cache_init();
873
874	set_max_threads(MAX_THREADS);
875
876	init_task.signal->rlim[RLIMIT_NPROC].rlim_cur = max_threads/2;
877	init_task.signal->rlim[RLIMIT_NPROC].rlim_max = max_threads/2;
878	init_task.signal->rlim[RLIMIT_SIGPENDING] =
879	init_task.signal->rlim[RLIMIT_NPROC];
880
881	for (i = 0; i < UCOUNT_COUNTS; i++)
882	init_user_ns.ucount_max[i] = max_threads/2;
883
884	set_userns_rlimit_max(&init_user_ns, UCOUNT_RLIMIT_NPROC, RLIM_INFINITY);
885	set_userns_rlimit_max(&init_user_ns, UCOUNT_RLIMIT_MSGQUEUE, RLIM_INFINITY);
886	set_userns_rlimit_max(&init_user_ns, UCOUNT_RLIMIT_SIGPENDING, RLIM_INFINITY);
887	set_userns_rlimit_max(&init_user_ns, UCOUNT_RLIMIT_MEMLOCK, RLIM_INFINITY);
888
889	#ifdef CONFIG_VMAP_STACK
890	cpuhp_setup_state(CPUHP_BP_PREPARE_DYN, "fork:vm_stack_cache",
891	NULL, free_vm_stack_cache);
892	#endif
893
894	scs_init();
895
896	lockdep_init_task(&init_task);
897	uprobes_init();
898	}
899
900	int __weak arch_dup_task_struct(struct task_struct *dst,
901	struct task_struct *src)
902	{
903	dst = src;
904	return 0;
905	}
906
907	void set_task_stack_end_magic(struct task_struct *tsk)
908	{
909	unsigned long *stackend;
910
911	stackend = end_of_stack(tsk);
912	stackend = STACK_END_MAGIC; / for overflow detection */
913	}
914
915	static struct task_struct dup_task_struct(struct task_struct orig, int node)
916	{
917	struct task_struct *tsk;
918	int err;
919
920	if (node == NUMA_NO_NODE)
921	node = tsk_fork_get_node(orig);
922	tsk = alloc_task_struct_node(node);
923	if (!tsk)
924	return NULL;
925
926	err = arch_dup_task_struct(tsk, orig);
927	if (err)
928	goto free_tsk;
929
930	err = alloc_thread_stack_node(tsk, node);
931	if (err)
932	goto free_tsk;
933
934	#ifdef CONFIG_THREAD_INFO_IN_TASK
935	refcount_set(&tsk->stack_refcount, 1);
936	#endif
937	account_kernel_stack(tsk, 1);
938
939	err = smp_task_ipi_mask_alloc(tsk);
940	if (err)
941	goto free_stack;
942
943	err = scs_prepare(tsk, node);
944	if (err)
945	goto free_ipi_mask;
946
947	#ifdef CONFIG_SECCOMP
948	/*
949	* We must handle setting up seccomp filters once we're under
950	* the sighand lock in case orig has changed between now and
951	* then. Until then, filter must be NULL to avoid messing up
952	* the usage counts on the error path calling free_task.
953	*/
954	tsk->seccomp.filter = NULL;
955	#endif
956
957	RCU_INIT_POINTER(tsk->exec_state, NULL);
958
959	setup_thread_stack(tsk, orig);
960	clear_user_return_notifier(tsk);
961	clear_tsk_need_resched(tsk);
962	set_task_stack_end_magic(tsk);
963	clear_syscall_work_syscall_user_dispatch(tsk);
964
965	#ifdef CONFIG_STACKPROTECTOR
966	tsk->stack_canary = get_random_canary();
967	#endif
968	if (orig->cpus_ptr == &orig->cpus_mask)
969	tsk->cpus_ptr = &tsk->cpus_mask;
970	dup_user_cpus_ptr(tsk, orig, node);
971
972	/*
973	* One for the user space visible state that goes away when reaped.
974	* One for the scheduler.
975	*/
976	refcount_set(&tsk->rcu_users, 2);
977	/* One for the rcu users */
978	refcount_set(&tsk->usage, 1);
979	#ifdef CONFIG_BLK_DEV_IO_TRACE
980	tsk->btrace_seq = 0;
981	#endif
982	tsk->splice_pipe = NULL;
983	tsk->task_frag.page = NULL;
984	tsk->wake_q.next = NULL;
985	tsk->worker_private = NULL;
986
987	kcov_task_init(tsk);
988	kmsan_task_create(tsk);
989	kmap_local_fork(tsk);
990
991	#ifdef CONFIG_FAULT_INJECTION
992	tsk->fail_nth = 0;
993	#endif
994
995	#ifdef CONFIG_BLK_CGROUP
996	tsk->throttle_disk = NULL;
997	tsk->use_memdelay = 0;
998	#endif
999
1000	#ifdef CONFIG_ARCH_HAS_CPU_PASID
1001	tsk->pasid_activated = 0;
1002	#endif
1003
1004	#ifdef CONFIG_MEMCG
1005	tsk->active_memcg = NULL;
1006	#endif
1007
1008	#ifdef CONFIG_X86_BUS_LOCK_DETECT
1009	tsk->reported_split_lock = 0;
1010	#endif
1011
1012	#ifdef CONFIG_SCHED_MM_CID
1013	tsk->mm_cid.cid = MM_CID_UNSET;
1014	tsk->mm_cid.active = 0;
1015	INIT_HLIST_NODE(&tsk->mm_cid.node);
1016	#endif
1017
1018	#ifdef CONFIG_BPF_SYSCALL
1019	RCU_INIT_POINTER(tsk->bpf_storage, NULL);
1020	tsk->bpf_ctx = NULL;
1021	#endif
1022	return tsk;
1023
1024	free_ipi_mask:
1025	smp_task_ipi_mask_free(tsk);
1026	free_stack:
1027	exit_task_stack_account(tsk);
1028	free_thread_stack(tsk);
1029	free_tsk:
1030	free_task_struct(tsk);
1031	return NULL;
1032	}
1033
1034	__cacheline_aligned_in_smp DEFINE_SPINLOCK(mmlist_lock);
1035
1036	static unsigned long coredump_filter = MMF_DUMP_FILTER_DEFAULT;
1037
1038	static int __init coredump_filter_setup(char *s)
1039	{
1040	if (kstrtoul(s, 0, &coredump_filter))
1041	return 0;
1042	coredump_filter <<= MMF_DUMP_FILTER_SHIFT;
1043	coredump_filter &= MMF_DUMP_FILTER_MASK;
1044	return 1;
1045	}
1046
1047	__setup("coredump_filter=", coredump_filter_setup);
1048
1049	#include <linux/init_task.h>
1050
1051	static void mm_init_aio(struct mm_struct *mm)
1052	{
1053	#ifdef CONFIG_AIO
1054	spin_lock_init(&mm->ioctx_lock);
1055	mm->ioctx_table = NULL;
1056	#endif
1057	}
1058
1059	static __always_inline void mm_clear_owner(struct mm_struct *mm,
1060	struct task_struct *p)
1061	{
1062	#ifdef CONFIG_MEMCG
1063	if (mm->owner == p)
1064	WRITE_ONCE(mm->owner, NULL);
1065	#endif
1066	}
1067
1068	static void mm_init_owner(struct mm_struct mm, struct task_struct p)
1069	{
1070	#ifdef CONFIG_MEMCG
1071	mm->owner = p;
1072	#endif
1073	}
1074
1075	static void mm_init_uprobes_state(struct mm_struct *mm)
1076	{
1077	#ifdef CONFIG_UPROBES
1078	mm->uprobes_state.xol_area = NULL;
1079	#endif
1080	}
1081
1082	static void mmap_init_lock(struct mm_struct *mm)
1083	{
1084	init_rwsem(&mm->mmap_lock);
1085	mm_lock_seqcount_init(mm);
1086	#ifdef CONFIG_PER_VMA_LOCK
1087	rcuwait_init(&mm->vma_writer_wait);
1088	#endif
1089	}
1090
1091	static struct mm_struct mm_init(struct mm_struct mm, struct task_struct *p)
1092	{
1093	mt_init_flags(&mm->mm_mt, MM_MT_FLAGS);
1094	mt_set_external_lock(&mm->mm_mt, &mm->mmap_lock);
1095	atomic_set(&mm->mm_users, 1);
1096	atomic_set(&mm->mm_count, 1);
1097	seqcount_init(&mm->write_protect_seq);
1098	mmap_init_lock(mm);
1099	INIT_LIST_HEAD(&mm->mmlist);
1100	mm_pgtables_bytes_init(mm);
1101	mm->map_count = 0;
1102	mm->locked_vm = 0;
1103	atomic64_set(&mm->pinned_vm, 0);
1104	memset(&mm->rss_stat, 0, sizeof(mm->rss_stat));
1105	spin_lock_init(&mm->page_table_lock);
1106	spin_lock_init(&mm->arg_lock);
1107	mm_init_cpumask(mm);
1108	mm_init_aio(mm);
1109	mm_init_owner(mm, p);
1110	mm_pasid_init(mm);
1111	RCU_INIT_POINTER(mm->exe_file, NULL);
1112	mmu_notifier_subscriptions_init(mm);
1113	init_tlb_flush_pending(mm);
1114	#if defined(CONFIG_TRANSPARENT_HUGEPAGE) && !defined(CONFIG_SPLIT_PMD_PTLOCKS)
1115	mm->pmd_huge_pte = NULL;
1116	#endif
1117	mm_init_uprobes_state(mm);
1118	hugetlb_count_init(mm);
1119	futex_mm_init(mm);
1120
1121	mm_flags_clear_all(mm);
1122	if (current->mm) {
1123	unsigned long flags = __mm_flags_get_word(current->mm);
1124
1125	__mm_flags_overwrite_word(mm, mmf_init_legacy_flags(flags));
1126	mm->def_flags = current->mm->def_flags & VM_INIT_DEF_MASK;
1127	} else {
1128	__mm_flags_overwrite_word(mm, coredump_filter);
1129	mm->def_flags = 0;
1130	}
1131
1132	if (mm_alloc_pgd(mm))
1133	goto fail_mm_init;
1134
1135	if (mm_alloc_id(mm))
1136	goto fail_noid;
1137
1138	if (init_new_context(p, mm))
1139	goto fail_nocontext;
1140
1141	if (mm_alloc_cid(mm, p))
1142	goto fail_cid;
1143
1144	if (mm_alloc_sched(mm))
1145	goto fail_sched;
1146
1147	if (percpu_counter_init_many(mm->rss_stat, 0, GFP_KERNEL_ACCOUNT,
1148	NR_MM_COUNTERS))
1149	goto fail_pcpu;
1150
1151	lru_gen_init_mm(mm);
1152	return mm;
1153
1154	fail_pcpu:
1155	mm_destroy_sched(mm);
1156	fail_sched:
1157	mm_destroy_cid(mm);
1158	fail_cid:
1159	destroy_context(mm);
1160	fail_nocontext:
1161	mm_free_id(mm);
1162	fail_noid:
1163	mm_free_pgd(mm);
1164	fail_mm_init:
1165	free_mm(mm);
1166	return NULL;
1167	}
1168
1169	/*
1170	* Allocate and initialize an mm_struct.
1171	*/
1172	struct mm_struct *mm_alloc(void)
1173	{
1174	struct mm_struct *mm;
1175
1176	mm = allocate_mm();
1177	if (!mm)
1178	return NULL;
1179
1180	memset(mm, 0, sizeof(*mm));
1181	return mm_init(mm, current);
1182	}
1183	EXPORT_SYMBOL_IF_KUNIT(mm_alloc);
1184
1185	static inline void __mmput(struct mm_struct *mm)
1186	{
1187	VM_BUG_ON(atomic_read(&mm->mm_users));
1188
1189	uprobe_clear_state(mm);
1190	exit_aio(mm);
1191	ksm_exit(mm);
1192	khugepaged_exit(mm); /* must run before exit_mmap */
1193	exit_mmap(mm);
1194	mm_put_huge_zero_folio(mm);
1195	set_mm_exe_file(mm, NULL);
1196	if (!list_empty(&mm->mmlist)) {
1197	spin_lock(&mmlist_lock);
1198	list_del(&mm->mmlist);
1199	spin_unlock(&mmlist_lock);
1200	}
1201	if (mm->binfmt)
1202	module_put(mm->binfmt->module);
1203	lru_gen_del_mm(mm);
1204	futex_hash_free(mm);
1205	mmdrop(mm);
1206	}
1207
1208	/*
1209	* Decrement the use count and release all resources for an mm.
1210	*/
1211	void mmput(struct mm_struct *mm)
1212	{
1213	might_sleep();
1214
1215	if (atomic_dec_and_test(&mm->mm_users))
1216	__mmput(mm);
1217	}
1218	EXPORT_SYMBOL_GPL(mmput);
1219
1220	#if defined(CONFIG_MMU) \|\| defined(CONFIG_FUTEX_PRIVATE_HASH)
1221	static void mmput_async_fn(struct work_struct *work)
1222	{
1223	struct mm_struct *mm = container_of(work, struct mm_struct,
1224	async_put_work);
1225
1226	__mmput(mm);
1227	}
1228
1229	void mmput_async(struct mm_struct *mm)
1230	{
1231	if (atomic_dec_and_test(&mm->mm_users)) {
1232	INIT_WORK(&mm->async_put_work, mmput_async_fn);
1233	schedule_work(&mm->async_put_work);
1234	}
1235	}
1236	EXPORT_SYMBOL_GPL(mmput_async);
1237	#endif
1238
1239	/**
1240	* set_mm_exe_file - change a reference to the mm's executable file
1241	* @mm: The mm to change.
1242	* @new_exe_file: The new file to use.
1243	*
1244	* This changes mm's executable file (shown as symlink /proc/[pid]/exe).
1245	*
1246	* Main users are mmput() and sys_execve(). Callers prevent concurrent
1247	* invocations: in mmput() nobody alive left, in execve it happens before
1248	* the new mm is made visible to anyone.
1249	*
1250	* Can only fail if new_exe_file != NULL.
1251	*/
1252	int set_mm_exe_file(struct mm_struct mm, struct file new_exe_file)
1253	{
1254	struct file *old_exe_file;
1255
1256	/*
1257	* It is safe to dereference the exe_file without RCU as
1258	* this function is only called if nobody else can access
1259	* this mm -- see comment above for justification.
1260	*/
1261	old_exe_file = rcu_dereference_raw(mm->exe_file);
1262
1263	if (new_exe_file) {
1264	/*
1265	* We expect the caller (i.e., sys_execve) to already denied
1266	* write access, so this is unlikely to fail.
1267	*/
1268	if (unlikely(exe_file_deny_write_access(new_exe_file)))
1269	return -EACCES;
1270	get_file(new_exe_file);
1271	}
1272	rcu_assign_pointer(mm->exe_file, new_exe_file);
1273	if (old_exe_file) {
1274	exe_file_allow_write_access(old_exe_file);
1275	fput(old_exe_file);
1276	}
1277	return 0;
1278	}
1279
1280	/**
1281	* replace_mm_exe_file - replace a reference to the mm's executable file
1282	* @mm: The mm to change.
1283	* @new_exe_file: The new file to use.
1284	*
1285	* This changes mm's executable file (shown as symlink /proc/[pid]/exe).
1286	*
1287	* Main user is sys_prctl(PR_SET_MM_MAP/EXE_FILE).
1288	*/
1289	int replace_mm_exe_file(struct mm_struct mm, struct file new_exe_file)
1290	{
1291	struct vm_area_struct *vma;
1292	struct file *old_exe_file;
1293	int ret = 0;
1294
1295	/* Forbid mm->exe_file change if old file still mapped. */
1296	old_exe_file = get_mm_exe_file(mm);
1297	if (old_exe_file) {
1298	VMA_ITERATOR(vmi, mm, 0);
1299	mmap_read_lock(mm);
1300	for_each_vma(vmi, vma) {
1301	if (!vma->vm_file)
1302	continue;
1303	if (path_equal(&vma->vm_file->f_path,
1304	&old_exe_file->f_path)) {
1305	ret = -EBUSY;
1306	break;
1307	}
1308	}
1309	mmap_read_unlock(mm);
1310	fput(old_exe_file);
1311	if (ret)
1312	return ret;
1313	}
1314
1315	ret = exe_file_deny_write_access(new_exe_file);
1316	if (ret)
1317	return -EACCES;
1318	get_file(new_exe_file);
1319
1320	/* set the new file */
1321	mmap_write_lock(mm);
1322	old_exe_file = rcu_dereference_raw(mm->exe_file);
1323	rcu_assign_pointer(mm->exe_file, new_exe_file);
1324	mmap_write_unlock(mm);
1325
1326	if (old_exe_file) {
1327	exe_file_allow_write_access(old_exe_file);
1328	fput(old_exe_file);
1329	}
1330	return 0;
1331	}
1332
1333	/**
1334	* get_mm_exe_file - acquire a reference to the mm's executable file
1335	* @mm: The mm of interest.
1336	*
1337	* Returns %NULL if mm has no associated executable file.
1338	* User must release file via fput().
1339	*/
1340	struct file get_mm_exe_file(struct mm_struct mm)
1341	{
1342	struct file *exe_file;
1343
1344	rcu_read_lock();
1345	exe_file = get_file_rcu(&mm->exe_file);
1346	rcu_read_unlock();
1347	return exe_file;
1348	}
1349
1350	/**
1351	* get_task_exe_file - acquire a reference to the task's executable file
1352	* @task: The task.
1353	*
1354	* Returns %NULL if task's mm (if any) has no associated executable file or
1355	* this is a kernel thread with borrowed mm (see the comment above get_task_mm).
1356	* User must release file via fput().
1357	*/
1358	struct file get_task_exe_file(struct task_struct task)
1359	{
1360	struct file *exe_file = NULL;
1361	struct mm_struct *mm;
1362
1363	if (task->flags & PF_KTHREAD)
1364	return NULL;
1365
1366	task_lock(task);
1367	mm = task->mm;
1368	if (mm)
1369	exe_file = get_mm_exe_file(mm);
1370	task_unlock(task);
1371	return exe_file;
1372	}
1373
1374	/**
1375	* get_task_mm - acquire a reference to the task's mm
1376	* @task: The task.
1377	*
1378	* Returns %NULL if the task has no mm. Checks PF_KTHREAD (meaning
1379	* this kernel workthread has transiently adopted a user mm with kthread_use_mm,
1380	* to do its AIO) is not set and if so returns a reference to it, after
1381	* bumping up the use count. User must release the mm via mmput()
1382	* after use. Typically used by /proc and ptrace.
1383	*/
1384	struct mm_struct get_task_mm(struct task_struct task)
1385	{
1386	struct mm_struct *mm;
1387
1388	if (task->flags & PF_KTHREAD)
1389	return NULL;
1390
1391	task_lock(task);
1392	mm = task->mm;
1393	if (mm)
1394	mmget(mm);
1395	task_unlock(task);
1396	return mm;
1397	}
1398	EXPORT_SYMBOL_GPL(get_task_mm);
1399
1400	static bool may_access_mm(struct mm_struct mm, struct task_struct task, unsigned int mode)
1401	{
1402	if (mm == current->mm)
1403	return true;
1404	if (ptrace_may_access(task, mode))
1405	return true;
1406	if ((mode & PTRACE_MODE_READ) && perfmon_capable())
1407	return true;
1408	return false;
1409	}
1410
1411	struct mm_struct mm_access(struct task_struct task, unsigned int mode)
1412	{
1413	struct mm_struct *mm;
1414	int err;
1415
1416	err = down_read_killable(&task->signal->exec_update_lock);
1417	if (err)
1418	return ERR_PTR(err);
1419
1420	mm = get_task_mm(task);
1421	if (!mm) {
1422	mm = ERR_PTR(-ESRCH);
1423	} else if (!may_access_mm(mm, task, mode)) {
1424	mmput(mm);
1425	mm = ERR_PTR(-EACCES);
1426	}
1427	up_read(&task->signal->exec_update_lock);
1428
1429	return mm;
1430	}
1431
1432	static void complete_vfork_done(struct task_struct *tsk)
1433	{
1434	struct completion *vfork;
1435
1436	task_lock(tsk);
1437	vfork = tsk->vfork_done;
1438	if (likely(vfork)) {
1439	tsk->vfork_done = NULL;
1440	complete(vfork);
1441	}
1442	task_unlock(tsk);
1443	}
1444
1445	static int wait_for_vfork_done(struct task_struct *child,
1446	struct completion *vfork)
1447	{
1448	unsigned int state = TASK_KILLABLE\|TASK_FREEZABLE;
1449	int killed;
1450
1451	cgroup_enter_frozen();
1452	killed = wait_for_completion_state(vfork, state);
1453	cgroup_leave_frozen(false);
1454
1455	if (killed) {
1456	task_lock(child);
1457	child->vfork_done = NULL;
1458	task_unlock(child);
1459	}
1460
1461	put_task_struct(child);
1462	return killed;
1463	}
1464
1465	/* Please note the differences between mmput and mm_release.
1466	* mmput is called whenever we stop holding onto a mm_struct,
1467	* error success whatever.
1468	*
1469	* mm_release is called after a mm_struct has been removed
1470	* from the current process.
1471	*
1472	* This difference is important for error handling, when we
1473	* only half set up a mm_struct for a new process and need to restore
1474	* the old one. Because we mmput the new mm_struct before
1475	* restoring the old one. . .
1476	* Eric Biederman 10 January 1998
1477	*/
1478	static void mm_release(struct task_struct tsk, struct mm_struct mm)
1479	{
1480	uprobe_free_utask(tsk);
1481
1482	/* Get rid of any cached register state */
1483	deactivate_mm(tsk, mm);
1484
1485	/*
1486	* Signal userspace if we're not exiting with a core dump
1487	* because we want to leave the value intact for debugging
1488	* purposes.
1489	*/
1490	if (tsk->clear_child_tid) {
1491	if (atomic_read(&mm->mm_users) > 1) {
1492	/*
1493	* We don't check the error code - if userspace has
1494	* not set up a proper pointer then tough luck.
1495	*/
1496	put_user(0, tsk->clear_child_tid);
1497	do_futex(tsk->clear_child_tid, FUTEX_WAKE,
1498	1, NULL, NULL, 0, 0);
1499	}
1500	tsk->clear_child_tid = NULL;
1501	}
1502
1503	/*
1504	* All done, finally we can wake up parent and return this mm to him.
1505	* Also kthread_stop() uses this completion for synchronization.
1506	*/
1507	if (tsk->vfork_done)
1508	complete_vfork_done(tsk);
1509	}
1510
1511	void mm_exit_exec_release(struct task_struct tsk, struct mm_struct mm)
1512	{
1513	futex_exit_exec_release(tsk);
1514	mm_release(tsk, mm);
1515	}
1516
1517	/**
1518	* dup_mm() - duplicates an existing mm structure
1519	* @tsk: the task_struct with which the new mm will be associated.
1520	* @oldmm: the mm to duplicate.
1521	*
1522	* Allocates a new mm structure and duplicates the provided @oldmm structure
1523	* content into it.
1524	*
1525	* Return: the duplicated mm or NULL on failure.
1526	*/
1527	static struct mm_struct dup_mm(struct task_struct tsk,
1528	struct mm_struct *oldmm)
1529	{
1530	struct mm_struct *mm;
1531	int err;
1532
1533	mm = allocate_mm();
1534	if (!mm)
1535	goto fail_nomem;
1536
1537	memcpy(mm, oldmm, sizeof(*mm));
1538
1539	if (!mm_init(mm, tsk))
1540	goto fail_nomem;
1541
1542	uprobe_start_dup_mmap();
1543	err = dup_mmap(mm, oldmm);
1544	if (err)
1545	goto free_pt;
1546	uprobe_end_dup_mmap();
1547
1548	mm->hiwater_rss = get_mm_rss(mm);
1549	mm->hiwater_vm = mm->total_vm;
1550
1551	if (mm->binfmt && !try_module_get(mm->binfmt->module))
1552	goto free_pt;
1553
1554	return mm;
1555
1556	free_pt:
1557	/* don't put binfmt in mmput, we haven't got module yet */
1558	mm->binfmt = NULL;
1559	mm_init_owner(mm, NULL);
1560	mmput(mm);
1561	if (err)
1562	uprobe_end_dup_mmap();
1563
1564	fail_nomem:
1565	return NULL;
1566	}
1567
1568	static int copy_mm(u64 clone_flags, struct task_struct *tsk)
1569	{
1570	struct mm_struct mm, oldmm;
1571
1572	tsk->min_flt = tsk->maj_flt = 0;
1573	tsk->nvcsw = tsk->nivcsw = 0;
1574	#ifdef CONFIG_DETECT_HUNG_TASK
1575	tsk->last_switch_count = tsk->nvcsw + tsk->nivcsw;
1576	tsk->last_switch_time = 0;
1577	#endif
1578
1579	tsk->mm = NULL;
1580	tsk->active_mm = NULL;
1581
1582	/*
1583	* Are we cloning a kernel thread?
1584	*
1585	* We need to steal a active VM for that..
1586	*/
1587	oldmm = current->mm;
1588	if (!oldmm)
1589	return 0;
1590
1591	if (clone_flags & CLONE_VM) {
1592	mmget(oldmm);
1593	mm = oldmm;
1594	} else {
1595	mm = dup_mm(tsk, current->mm);
1596	if (!mm)
1597	return -ENOMEM;
1598	}
1599
1600	tsk->mm = mm;
1601	tsk->active_mm = mm;
1602	return 0;
1603	}
1604
1605	static int copy_exec_state(u64 clone_flags, struct task_struct *tsk)
1606	{
1607	struct task_exec_state *exec_state;
1608
1609	/* CLONE_VM siblings refcount-share the parent's exec_state. */
1610	if (clone_flags & CLONE_VM) {
1611	exec_state = rcu_dereference_protected(current->exec_state, true);
1612	refcount_inc(&exec_state->count);
1613	rcu_assign_pointer(tsk->exec_state, exec_state);
1614	return 0;
1615	}
1616
1617	/* Everyone else inherits a fresh copy. */
1618	return task_exec_state_copy(tsk);
1619	}
1620
1621	static int copy_fs(u64 clone_flags, struct task_struct *tsk, bool umh)
1622	{
1623	struct fs_struct *fs;
1624
1625	/*
1626	* Usermodehelper may copy userspace_init_fs filesystem state but
1627	* they don't get to create mount namespaces, share the
1628	* filesystem state, or be started from a non-initial mount
1629	* namespace.
1630	*/
1631	if (umh) {
1632	if (clone_flags & (CLONE_NEWNS \| CLONE_FS))
1633	return -EINVAL;
1634	if (current->nsproxy->mnt_ns != &init_mnt_ns)
1635	return -EINVAL;
1636	fs = userspace_init_fs;
1637	} else {
1638	fs = current->fs;
1639	VFS_WARN_ON_ONCE(current->fs != current->real_fs);
1640	}
1641
1642	if (clone_flags & CLONE_FS) {
1643	/* tsk->fs is already what we want */
1644	read_seqlock_excl(&fs->seq);
1645	/* "users" and "in_exec" locked for check_unsafe_exec() */
1646	if (fs->in_exec) {
1647	read_sequnlock_excl(&fs->seq);
1648	return -EAGAIN;
1649	}
1650	fs->users++;
1651	read_sequnlock_excl(&fs->seq);
1652	return 0;
1653	}
1654	tsk->real_fs = tsk->fs = copy_fs_struct(fs);
1655	if (!tsk->fs)
1656	return -ENOMEM;
1657	return 0;
1658	}
1659
1660	static int copy_files(u64 clone_flags, struct task_struct *tsk,
1661	int no_files)
1662	{
1663	struct files_struct oldf, newf;
1664
1665	/*
1666	* A background process may not have any files ...
1667	*/
1668	oldf = current->files;
1669	if (!oldf)
1670	return 0;
1671
1672	if (no_files) {
1673	tsk->files = NULL;
1674	return 0;
1675	}
1676
1677	if (clone_flags & CLONE_FILES) {
1678	atomic_inc(&oldf->count);
1679	return 0;
1680	}
1681
1682	newf = dup_fd(oldf, NULL);
1683	if (IS_ERR(newf))
1684	return PTR_ERR(newf);
1685
1686	tsk->files = newf;
1687	return 0;
1688	}
1689
1690	static int copy_sighand(u64 clone_flags, struct task_struct *tsk)
1691	{
1692	struct sighand_struct *sig;
1693
1694	if (clone_flags & CLONE_SIGHAND) {
1695	refcount_inc(&current->sighand->count);
1696	return 0;
1697	}
1698	sig = kmem_cache_alloc(sighand_cachep, GFP_KERNEL);
1699	RCU_INIT_POINTER(tsk->sighand, sig);
1700	if (!sig)
1701	return -ENOMEM;
1702
1703	refcount_set(&sig->count, 1);
1704	spin_lock_irq(&current->sighand->siglock);
1705	memcpy(sig->action, current->sighand->action, sizeof(sig->action));
1706	spin_unlock_irq(&current->sighand->siglock);
1707
1708	/* Reset all signal handler not set to SIG_IGN to SIG_DFL. */
1709	if (clone_flags & CLONE_CLEAR_SIGHAND)
1710	flush_signal_handlers(tsk, 0);
1711
1712	return 0;
1713	}
1714
1715	void __cleanup_sighand(struct sighand_struct *sighand)
1716	{
1717	if (refcount_dec_and_test(&sighand->count)) {
1718	signalfd_cleanup(sighand);
1719	/*
1720	* sighand_cachep is SLAB_TYPESAFE_BY_RCU so we can free it
1721	* without an RCU grace period, see __lock_task_sighand().
1722	*/
1723	kmem_cache_free(sighand_cachep, sighand);
1724	}
1725	}
1726
1727	/*
1728	* Initialize POSIX timer handling for a thread group.
1729	*/
1730	static void posix_cpu_timers_init_group(struct signal_struct *sig)
1731	{
1732	struct posix_cputimers *pct = &sig->posix_cputimers;
1733	unsigned long cpu_limit;
1734
1735	cpu_limit = READ_ONCE(sig->rlim[RLIMIT_CPU].rlim_cur);
1736	posix_cputimers_group_init(pct, cpu_limit);
1737	}
1738
1739	static int copy_signal(u64 clone_flags, struct task_struct *tsk)
1740	{
1741	struct signal_struct *sig;
1742
1743	if (clone_flags & CLONE_THREAD)
1744	return 0;
1745
1746	sig = kmem_cache_zalloc(signal_cachep, GFP_KERNEL);
1747	tsk->signal = sig;
1748	if (!sig)
1749	return -ENOMEM;
1750
1751	sig->nr_threads = 1;
1752	sig->quick_threads = 1;
1753	atomic_set(&sig->live, 1);
1754	refcount_set(&sig->sigcnt, 1);
1755
1756	/* list_add(thread_node, thread_head) without INIT_LIST_HEAD() */
1757	sig->thread_head = (struct list_head)LIST_HEAD_INIT(tsk->thread_node);
1758	tsk->thread_node = (struct list_head)LIST_HEAD_INIT(sig->thread_head);
1759
1760	init_waitqueue_head(&sig->wait_chldexit);
1761	sig->curr_target = tsk;
1762	init_sigpending(&sig->shared_pending);
1763	INIT_HLIST_HEAD(&sig->multiprocess);
1764	seqlock_init(&sig->stats_lock);
1765	prev_cputime_init(&sig->prev_cputime);
1766
1767	#ifdef CONFIG_POSIX_TIMERS
1768	INIT_HLIST_HEAD(&sig->posix_timers);
1769	INIT_HLIST_HEAD(&sig->ignored_posix_timers);
1770	hrtimer_setup(&sig->real_timer, it_real_fn, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
1771	#endif
1772
1773	task_lock(current->group_leader);
1774	memcpy(sig->rlim, current->signal->rlim, sizeof sig->rlim);
1775	task_unlock(current->group_leader);
1776
1777	posix_cpu_timers_init_group(sig);
1778
1779	tty_audit_fork(sig);
1780	sched_autogroup_fork(sig);
1781
1782	#ifdef CONFIG_CGROUPS
1783	init_rwsem(&sig->cgroup_threadgroup_rwsem);
1784	#endif
1785
1786	sig->oom_score_adj = current->signal->oom_score_adj;
1787	sig->oom_score_adj_min = current->signal->oom_score_adj_min;
1788
1789	mutex_init(&sig->cred_guard_mutex);
1790	init_rwsem(&sig->exec_update_lock);
1791
1792	return 0;
1793	}
1794
1795	static void copy_seccomp(struct task_struct *p)
1796	{
1797	#ifdef CONFIG_SECCOMP
1798	/*
1799	* Must be called with sighand->lock held, which is common to
1800	* all threads in the group. Holding cred_guard_mutex is not
1801	* needed because this new task is not yet running and cannot
1802	* be racing exec.
1803	*/
1804	assert_spin_locked(&current->sighand->siglock);
1805
1806	/* Ref-count the new filter user, and assign it. */
1807	get_seccomp_filter(current);
1808	p->seccomp = current->seccomp;
1809
1810	/*
1811	* Explicitly enable no_new_privs here in case it got set
1812	* between the task_struct being duplicated and holding the
1813	* sighand lock. The seccomp state and nnp must be in sync.
1814	*/
1815	if (task_no_new_privs(current))
1816	task_set_no_new_privs(p);
1817
1818	/*
1819	* If the parent gained a seccomp mode after copying thread
1820	* flags and between before we held the sighand lock, we have
1821	* to manually enable the seccomp thread flag here.
1822	*/
1823	if (p->seccomp.mode != SECCOMP_MODE_DISABLED)
1824	set_task_syscall_work(p, SECCOMP);
1825	#endif
1826	}
1827
1828	SYSCALL_DEFINE1(set_tid_address, int __user *, tidptr)
1829	{
1830	current->clear_child_tid = tidptr;
1831
1832	return task_pid_vnr(current);
1833	}
1834
1835	static void rt_mutex_init_task(struct task_struct *p)
1836	{
1837	raw_spin_lock_init(&p->pi_lock);
1838	#ifdef CONFIG_RT_MUTEXES
1839	p->pi_waiters = RB_ROOT_CACHED;
1840	p->pi_top_task = NULL;
1841	p->pi_blocked_on = NULL;
1842	#endif
1843	}
1844
1845	static inline void init_task_pid_links(struct task_struct *task)
1846	{
1847	enum pid_type type;
1848
1849	for (type = PIDTYPE_PID; type < PIDTYPE_MAX; ++type)
1850	INIT_HLIST_NODE(&task->pid_links[type]);
1851	}
1852
1853	static inline void
1854	init_task_pid(struct task_struct task, enum pid_type type, struct pid pid)
1855	{
1856	if (type == PIDTYPE_PID)
1857	task->thread_pid = pid;
1858	else
1859	task->signal->pids[type] = pid;
1860	}
1861
1862	static inline void rcu_copy_process(struct task_struct *p)
1863	{
1864	#ifdef CONFIG_PREEMPT_RCU
1865	p->rcu_read_lock_nesting = 0;
1866	p->rcu_read_unlock_special.s = 0;
1867	p->rcu_blocked_node = NULL;
1868	INIT_LIST_HEAD(&p->rcu_node_entry);
1869	#endif /* #ifdef CONFIG_PREEMPT_RCU */
1870	#ifdef CONFIG_TASKS_RCU
1871	p->rcu_tasks_holdout = false;
1872	INIT_LIST_HEAD(&p->rcu_tasks_holdout_list);
1873	p->rcu_tasks_idle_cpu = -1;
1874	INIT_LIST_HEAD(&p->rcu_tasks_exit_list);
1875	#endif /* #ifdef CONFIG_TASKS_RCU */
1876	#ifdef CONFIG_TASKS_TRACE_RCU
1877	p->trc_reader_nesting = 0;
1878	#endif /* #ifdef CONFIG_TASKS_TRACE_RCU */
1879	}
1880
1881	/**
1882	* pidfd_prepare - allocate a new pidfd_file and reserve a pidfd
1883	* @pid: the struct pid for which to create a pidfd
1884	* @flags: flags of the new @pidfd
1885	* @ret_file: return the new pidfs file
1886	*
1887	* Allocate a new file that stashes @pid and reserve a new pidfd number in the
1888	* caller's file descriptor table. The pidfd is reserved but not installed yet.
1889	*
1890	* The helper verifies that @pid is still in use, without PIDFD_THREAD the
1891	* task identified by @pid must be a thread-group leader.
1892	*
1893	* If this function returns successfully the caller is responsible to either
1894	* call fd_install() passing the returned pidfd and pidfd file as arguments in
1895	* order to install the pidfd into its file descriptor table or they must use
1896	* put_unused_fd() and fput() on the returned pidfd and pidfd file
1897	* respectively.
1898	*
1899	* This function is useful when a pidfd must already be reserved but there
1900	* might still be points of failure afterwards and the caller wants to ensure
1901	* that no pidfd is leaked into its file descriptor table.
1902	*
1903	* Return: On success, a reserved pidfd is returned from the function and a new
1904	* pidfd file is returned in the last argument to the function. On
1905	* error, a negative error code is returned from the function and the
1906	* last argument remains unchanged.
1907	*/
1908	int pidfd_prepare(struct pid pid, unsigned int flags, struct file *ret_file)
1909	{
1910	struct file *pidfs_file;
1911
1912	/*
1913	* PIDFD_STALE is only allowed to be passed if the caller knows
1914	* that @pid is already registered in pidfs and thus
1915	* PIDFD_INFO_EXIT information is guaranteed to be available.
1916	*/
1917	if (!(flags & PIDFD_STALE)) {
1918	/*
1919	* While holding the pidfd waitqueue lock removing the
1920	* task linkage for the thread-group leader pid
1921	* (PIDTYPE_TGID) isn't possible. Thus, if there's still
1922	* task linkage for PIDTYPE_PID not having thread-group
1923	* leader linkage for the pid means it wasn't a
1924	* thread-group leader in the first place.
1925	*/
1926	guard(spinlock_irq)(&pid->wait_pidfd.lock);
1927
1928	/* Task has already been reaped. */
1929	if (!pid_has_task(pid, PIDTYPE_PID))
1930	return -ESRCH;
1931	/*
1932	* If this struct pid isn't used as a thread-group
1933	* leader but the caller requested to create a
1934	* thread-group leader pidfd then report ENOENT.
1935	*/
1936	if (!(flags & PIDFD_THREAD) && !pid_has_task(pid, PIDTYPE_TGID))
1937	return -ENOENT;
1938	}
1939
1940	CLASS(get_unused_fd, pidfd)(O_CLOEXEC);
1941	if (pidfd < 0)
1942	return pidfd;
1943
1944	pidfs_file = pidfs_alloc_file(pid, flags \| O_RDWR);
1945	if (IS_ERR(pidfs_file))
1946	return PTR_ERR(pidfs_file);
1947
1948	*ret_file = pidfs_file;
1949	return take_fd(pidfd);
1950	}
1951
1952	static void __delayed_free_task(struct rcu_head *rhp)
1953	{
1954	struct task_struct *tsk = container_of(rhp, struct task_struct, rcu);
1955
1956	free_task(tsk);
1957	}
1958
1959	static __always_inline void delayed_free_task(struct task_struct *tsk)
1960	{
1961	if (IS_ENABLED(CONFIG_MEMCG))
1962	call_rcu(&tsk->rcu, __delayed_free_task);
1963	else
1964	free_task(tsk);
1965	}
1966
1967	static void copy_oom_score_adj(u64 clone_flags, struct task_struct *tsk)
1968	{
1969	/* Skip if kernel thread */
1970	if (!tsk->mm)
1971	return;
1972
1973	/* Skip if spawning a thread or using vfork */
1974	if ((clone_flags & (CLONE_VM \| CLONE_THREAD \| CLONE_VFORK)) != CLONE_VM)
1975	return;
1976
1977	/* We need to synchronize with __set_oom_adj */
1978	mutex_lock(&oom_adj_mutex);
1979	mm_flags_set(MMF_MULTIPROCESS, tsk->mm);
1980	/* Update the values in case they were changed after copy_signal */
1981	tsk->signal->oom_score_adj = current->signal->oom_score_adj;
1982	tsk->signal->oom_score_adj_min = current->signal->oom_score_adj_min;
1983	mutex_unlock(&oom_adj_mutex);
1984	}
1985
1986	#ifdef CONFIG_RV
1987	static void rv_task_fork(struct task_struct *p)
1988	{
1989	memset(&p->rv, 0, sizeof(p->rv));
1990	}
1991	#else
1992	#define rv_task_fork(p) do {} while (0)
1993	#endif
1994
1995	static bool need_futex_hash_allocate_default(u64 clone_flags)
1996	{
1997	/*
1998	* Allocate a default futex hash for any sibling that will
1999	* share the parent's mm, except vfork.
2000	*/
2001	return (clone_flags & (CLONE_VM \| CLONE_VFORK)) == CLONE_VM;
2002	}
2003
2004	/*
2005	* This creates a new process as a copy of the old one,
2006	* but does not actually start it yet.
2007	*
2008	* It copies the registers, and all the appropriate
2009	* parts of the process environment (as per the clone
2010	* flags). The actual kick-off is left to the caller.
2011	*/
2012	__latent_entropy struct task_struct *copy_process(
2013	struct pid *pid,
2014	int trace,
2015	int node,
2016	struct kernel_clone_args *args)
2017	{
2018	int pidfd = -1, retval;
2019	struct task_struct *p;
2020	struct multiprocess_signals delayed;
2021	struct file *pidfile = NULL;
2022	const u64 clone_flags = args->flags;
2023	struct nsproxy *nsp = current->nsproxy;
2024
2025	/*
2026	* Don't allow sharing the root directory with processes in a different
2027	* namespace
2028	*/
2029	if ((clone_flags & (CLONE_NEWNS\|CLONE_FS)) == (CLONE_NEWNS\|CLONE_FS))
2030	return ERR_PTR(-EINVAL);
2031
2032	if ((clone_flags & (CLONE_NEWUSER\|CLONE_FS)) == (CLONE_NEWUSER\|CLONE_FS))
2033	return ERR_PTR(-EINVAL);
2034
2035	/*
2036	* Thread groups must share signals as well, and detached threads
2037	* can only be started up within the thread group.
2038	*/
2039	if ((clone_flags & CLONE_THREAD) && !(clone_flags & CLONE_SIGHAND))
2040	return ERR_PTR(-EINVAL);
2041
2042	/*
2043	* Shared signal handlers imply shared VM. By way of the above,
2044	* thread groups also imply shared VM. Blocking this case allows
2045	* for various simplifications in other code.
2046	*/
2047	if ((clone_flags & CLONE_SIGHAND) && !(clone_flags & CLONE_VM))
2048	return ERR_PTR(-EINVAL);
2049
2050	/*
2051	* Siblings of global init remain as zombies on exit since they are
2052	* not reaped by their parent (swapper). To solve this and to avoid
2053	* multi-rooted process trees, prevent global and container-inits
2054	* from creating siblings.
2055	*/
2056	if ((clone_flags & CLONE_PARENT) &&
2057	current->signal->flags & SIGNAL_UNKILLABLE)
2058	return ERR_PTR(-EINVAL);
2059
2060	/*
2061	* If the new process will be in a different pid or user namespace
2062	* do not allow it to share a thread group with the forking task.
2063	*/
2064	if (clone_flags & CLONE_THREAD) {
2065	if ((clone_flags & (CLONE_NEWUSER \| CLONE_NEWPID)) \|\|
2066	(task_active_pid_ns(current) != nsp->pid_ns_for_children))
2067	return ERR_PTR(-EINVAL);
2068	}
2069
2070	if (clone_flags & CLONE_PIDFD) {
2071	/*
2072	* - CLONE_DETACHED is blocked so that we can potentially
2073	* reuse it later for CLONE_PIDFD.
2074	*/
2075	if (clone_flags & CLONE_DETACHED)
2076	return ERR_PTR(-EINVAL);
2077	}
2078
2079	if (clone_flags & CLONE_AUTOREAP) {
2080	if (clone_flags & CLONE_THREAD)
2081	return ERR_PTR(-EINVAL);
2082	if (clone_flags & CLONE_PARENT)
2083	return ERR_PTR(-EINVAL);
2084	if (args->exit_signal)
2085	return ERR_PTR(-EINVAL);
2086	}
2087
2088	if ((clone_flags & CLONE_PARENT) && current->signal->autoreap)
2089	return ERR_PTR(-EINVAL);
2090
2091	if (clone_flags & CLONE_NNP) {
2092	if (clone_flags & CLONE_THREAD)
2093	return ERR_PTR(-EINVAL);
2094	}
2095
2096	if (clone_flags & CLONE_PIDFD_AUTOKILL) {
2097	if (!(clone_flags & CLONE_PIDFD))
2098	return ERR_PTR(-EINVAL);
2099	if (!(clone_flags & CLONE_AUTOREAP))
2100	return ERR_PTR(-EINVAL);
2101	if (clone_flags & CLONE_THREAD)
2102	return ERR_PTR(-EINVAL);
2103	/*
2104	* Without CLONE_NNP the child could escalate privileges
2105	* after being spawned, so require CAP_SYS_ADMIN.
2106	* With CLONE_NNP the child can't gain new privileges,
2107	* so allow unprivileged usage.
2108	*/
2109	if (!(clone_flags & CLONE_NNP) &&
2110	!ns_capable(current_user_ns(), CAP_SYS_ADMIN))
2111	return ERR_PTR(-EPERM);
2112	}
2113
2114	/*
2115	* Force any signals received before this point to be delivered
2116	* before the fork happens. Collect up signals sent to multiple
2117	* processes that happen during the fork and delay them so that
2118	* they appear to happen after the fork.
2119	*/
2120	sigemptyset(&delayed.signal);
2121	INIT_HLIST_NODE(&delayed.node);
2122
2123	spin_lock_irq(&current->sighand->siglock);
2124	if (!(clone_flags & CLONE_THREAD))
2125	hlist_add_head(&delayed.node, &current->signal->multiprocess);
2126	recalc_sigpending();
2127	spin_unlock_irq(&current->sighand->siglock);
2128	retval = -ERESTARTNOINTR;
2129	if (task_sigpending(current))
2130	goto fork_out;
2131
2132	retval = -ENOMEM;
2133	p = dup_task_struct(current, node);
2134	if (!p)
2135	goto fork_out;
2136	retval = copy_exec_state(clone_flags, p);
2137	if (retval)
2138	goto bad_fork_free;
2139	p->flags &= ~PF_KTHREAD;
2140	if (args->kthread)
2141	p->flags \|= PF_KTHREAD;
2142	if (args->user_worker) {
2143	/*
2144	* Mark us a user worker, and block any signal that isn't
2145	* fatal or STOP
2146	*/
2147	p->flags \|= PF_USER_WORKER;
2148	siginitsetinv(&p->blocked, sigmask(SIGKILL)\|sigmask(SIGSTOP));
2149	}
2150	if (args->io_thread)
2151	p->flags \|= PF_IO_WORKER;
2152
2153	if (args->name)
2154	strscpy_pad(p->comm, args->name, sizeof(p->comm));
2155
2156	p->set_child_tid = (clone_flags & CLONE_CHILD_SETTID) ? args->child_tid : NULL;
2157	/*
2158	* TID is cleared in mm_release() when the task exits
2159	*/
2160	p->clear_child_tid = (clone_flags & CLONE_CHILD_CLEARTID) ? args->child_tid : NULL;
2161
2162	ftrace_graph_init_task(p);
2163
2164	rt_mutex_init_task(p);
2165	raw_spin_lock_init(&p->blocked_lock);
2166
2167	lockdep_assert_irqs_enabled();
2168	#ifdef CONFIG_PROVE_LOCKING
2169	DEBUG_LOCKS_WARN_ON(!p->softirqs_enabled);
2170	#endif
2171	retval = copy_creds(p, clone_flags);
2172	if (retval < 0)
2173	goto bad_fork_free;
2174
2175	retval = -EAGAIN;
2176	if (is_rlimit_overlimit(task_ucounts(p), UCOUNT_RLIMIT_NPROC, rlimit(RLIMIT_NPROC))) {
2177	if (p->real_cred->user != INIT_USER &&
2178	!capable(CAP_SYS_RESOURCE) && !capable(CAP_SYS_ADMIN))
2179	goto bad_fork_cleanup_count;
2180	}
2181	current->flags &= ~PF_NPROC_EXCEEDED;
2182
2183	/*
2184	* If multiple threads are within copy_process(), then this check
2185	* triggers too late. This doesn't hurt, the check is only there
2186	* to stop root fork bombs.
2187	*/
2188	retval = -EAGAIN;
2189	if (data_race(nr_threads >= max_threads))
2190	goto bad_fork_cleanup_count;
2191
2192	delayacct_tsk_init(p); /* Must remain after dup_task_struct() */
2193	p->flags &= ~(PF_SUPERPRIV \| PF_WQ_WORKER \| PF_IDLE \| PF_NO_SETAFFINITY);
2194	p->flags \|= PF_FORKNOEXEC;
2195	INIT_LIST_HEAD(&p->children);
2196	INIT_LIST_HEAD(&p->sibling);
2197	rcu_copy_process(p);
2198	p->vfork_done = NULL;
2199	spin_lock_init(&p->alloc_lock);
2200
2201	init_sigpending(&p->pending);
2202
2203	p->utime = p->stime = p->gtime = 0;
2204	#ifdef CONFIG_ARCH_HAS_SCALED_CPUTIME
2205	p->utimescaled = p->stimescaled = 0;
2206	#endif
2207	prev_cputime_init(&p->prev_cputime);
2208
2209	#ifdef CONFIG_VIRT_CPU_ACCOUNTING_GEN
2210	seqcount_init(&p->vtime.seqcount);
2211	p->vtime.starttime = 0;
2212	p->vtime.state = VTIME_INACTIVE;
2213	#endif
2214
2215	#ifdef CONFIG_IO_URING
2216	p->io_uring = NULL;
2217	retval = io_uring_fork(p);
2218	if (unlikely(retval))
2219	goto bad_fork_cleanup_delayacct;
2220	retval = -EAGAIN;
2221	#endif
2222
2223	p->default_timer_slack_ns = current->timer_slack_ns;
2224
2225	#ifdef CONFIG_PSI
2226	p->psi_flags = 0;
2227	#endif
2228
2229	task_io_accounting_init(&p->ioac);
2230	acct_clear_integrals(p);
2231
2232	posix_cputimers_init(&p->posix_cputimers);
2233	tick_dep_init_task(p);
2234
2235	p->io_context = NULL;
2236	audit_set_context(p, NULL);
2237	cgroup_fork(p);
2238	if (args->kthread) {
2239	if (!set_kthread_struct(p))
2240	goto bad_fork_cleanup_delayacct;
2241	}
2242	#ifdef CONFIG_NUMA
2243	p->mempolicy = mpol_dup(p->mempolicy);
2244	if (IS_ERR(p->mempolicy)) {
2245	retval = PTR_ERR(p->mempolicy);
2246	p->mempolicy = NULL;
2247	goto bad_fork_cleanup_delayacct;
2248	}
2249	#endif
2250	#ifdef CONFIG_CPUSETS
2251	p->cpuset_mem_spread_rotor = NUMA_NO_NODE;
2252	seqcount_spinlock_init(&p->mems_allowed_seq, &p->alloc_lock);
2253	#endif
2254	#ifdef CONFIG_TRACE_IRQFLAGS
2255	memset(&p->irqtrace, 0, sizeof(p->irqtrace));
2256	p->irqtrace.hardirq_disable_ip = _THIS_IP_;
2257	p->irqtrace.softirq_enable_ip = _THIS_IP_;
2258	p->softirqs_enabled = 1;
2259	p->softirq_context = 0;
2260	#endif
2261
2262	p->pagefault_disabled = 0;
2263
2264	lockdep_init_task(p);
2265
2266	p->blocked_on = NULL; /* not blocked yet */
2267	p->blocked_donor = NULL; /* nobody is boosting p yet */
2268
2269	#ifdef CONFIG_BCACHE
2270	p->sequential_io = 0;
2271	p->sequential_io_avg = 0;
2272	#endif
2273
2274	unwind_task_init(p);
2275
2276	/* Perform scheduler related setup. Assign this task to a CPU. */
2277	retval = sched_fork(clone_flags, p);
2278	if (retval)
2279	goto bad_fork_cleanup_policy;
2280
2281	retval = perf_event_init_task(p, clone_flags);
2282	if (retval)
2283	goto bad_fork_sched_cancel_fork;
2284	retval = audit_alloc(p);
2285	if (retval)
2286	goto bad_fork_cleanup_perf;
2287	/* copy all the process information */
2288	shm_init_task(p);
2289	retval = security_task_alloc(p, clone_flags);
2290	if (retval)
2291	goto bad_fork_cleanup_audit;
2292	retval = copy_semundo(clone_flags, p);
2293	if (retval)
2294	goto bad_fork_cleanup_security;
2295	retval = copy_files(clone_flags, p, args->no_files);
2296	if (retval)
2297	goto bad_fork_cleanup_semundo;
2298	retval = copy_fs(clone_flags, p, args->umh);
2299	if (retval)
2300	goto bad_fork_cleanup_files;
2301	retval = copy_sighand(clone_flags, p);
2302	if (retval)
2303	goto bad_fork_cleanup_fs;
2304	retval = copy_signal(clone_flags, p);
2305	if (retval)
2306	goto bad_fork_cleanup_sighand;
2307	retval = copy_mm(clone_flags, p);
2308	if (retval)
2309	goto bad_fork_cleanup_signal;
2310	retval = copy_namespaces(clone_flags, p);
2311	if (retval)
2312	goto bad_fork_cleanup_mm;
2313	retval = copy_io(clone_flags, p);
2314	if (retval)
2315	goto bad_fork_cleanup_namespaces;
2316	retval = copy_thread(p, args);
2317	if (retval)
2318	goto bad_fork_cleanup_io;
2319
2320	stackleak_task_init(p);
2321
2322	if (pid != &init_struct_pid) {
2323	pid = alloc_pid(p->nsproxy->pid_ns_for_children, args->set_tid,
2324	args->set_tid_size);
2325	if (IS_ERR(pid)) {
2326	retval = PTR_ERR(pid);
2327	goto bad_fork_cleanup_thread;
2328	}
2329	}
2330
2331	/*
2332	* This has to happen after we've potentially unshared the file
2333	* descriptor table (so that the pidfd doesn't leak into the child
2334	* if the fd table isn't shared).
2335	*/
2336	if (clone_flags & CLONE_PIDFD) {
2337	unsigned flags = PIDFD_STALE;
2338
2339	if (clone_flags & CLONE_THREAD)
2340	flags \|= PIDFD_THREAD;
2341	if (clone_flags & CLONE_PIDFD_AUTOKILL)
2342	flags \|= PIDFD_AUTOKILL;
2343
2344	/*
2345	* Note that no task has been attached to @pid yet indicate
2346	* that via CLONE_PIDFD.
2347	*/
2348	retval = pidfd_prepare(pid, flags, &pidfile);
2349	if (retval < 0)
2350	goto bad_fork_free_pid;
2351	pidfd = retval;
2352
2353	retval = put_user(pidfd, args->pidfd);
2354	if (retval)
2355	goto bad_fork_put_pidfd;
2356	}
2357
2358	#ifdef CONFIG_BLOCK
2359	p->plug = NULL;
2360	p->flags &= ~PF_BLOCK_TS;
2361	#endif
2362	futex_init_task(p);
2363
2364	/*
2365	* sigaltstack should be cleared when sharing the same VM
2366	*/
2367	if ((clone_flags & (CLONE_VM\|CLONE_VFORK)) == CLONE_VM)
2368	sas_ss_reset(p);
2369
2370	/*
2371	* Syscall tracing and stepping should be turned off in the
2372	* child regardless of CLONE_PTRACE.
2373	*/
2374	user_disable_single_step(p);
2375	clear_task_syscall_work(p, SYSCALL_TRACE);
2376	#if defined(CONFIG_GENERIC_ENTRY) \|\| defined(TIF_SYSCALL_EMU)
2377	clear_task_syscall_work(p, SYSCALL_EMU);
2378	#endif
2379	clear_tsk_latency_tracing(p);
2380
2381	/* ok, now we should be set up.. */
2382	p->pid = pid_nr(pid);
2383	if (clone_flags & CLONE_THREAD) {
2384	p->group_leader = current->group_leader;
2385	p->tgid = current->tgid;
2386	} else {
2387	p->group_leader = p;
2388	p->tgid = p->pid;
2389	}
2390
2391	p->nr_dirtied = 0;
2392	p->nr_dirtied_pause = 128 >> (PAGE_SHIFT - 10);
2393	p->dirty_paused_when = 0;
2394
2395	p->pdeath_signal = 0;
2396	p->task_works = NULL;
2397	clear_posix_cputimers_work(p);
2398
2399	#ifdef CONFIG_KRETPROBES
2400	p->kretprobe_instances.first = NULL;
2401	#endif
2402	#ifdef CONFIG_RETHOOK
2403	p->rethooks.first = NULL;
2404	#endif
2405
2406	/*
2407	* Ensure that the cgroup subsystem policies allow the new process to be
2408	* forked. It should be noted that the new process's css_set can be changed
2409	* between here and cgroup_post_fork() if an organisation operation is in
2410	* progress.
2411	*/
2412	retval = cgroup_can_fork(p, args);
2413	if (retval)
2414	goto bad_fork_put_pidfd;
2415
2416	/*
2417	* Now that the cgroups are pinned, re-clone the parent cgroup and put
2418	* the new task on the correct runqueue. All this before the task
2419	* becomes visible.
2420	*
2421	* This isn't part of ->can_fork() because while the re-cloning is
2422	* cgroup specific, it unconditionally needs to place the task on a
2423	* runqueue.
2424	*/
2425	retval = sched_cgroup_fork(p, args);
2426	if (retval)
2427	goto bad_fork_cancel_cgroup;
2428
2429	if (need_futex_hash_allocate_default(clone_flags)) {
2430	retval = futex_hash_allocate_default();
2431	if (retval)
2432	goto bad_fork_cancel_cgroup;
2433	/*
2434	* If we fail beyond this point we don't free the allocated
2435	* futex hash map. We assume that another thread will be created
2436	* and makes use of it. The hash map will be freed once the main
2437	* thread terminates.
2438	*/
2439	}
2440	/*
2441	* From this point on we must avoid any synchronous user-space
2442	* communication until we take the tasklist-lock. In particular, we do
2443	* not want user-space to be able to predict the process start-time by
2444	* stalling fork(2) after we recorded the start_time but before it is
2445	* visible to the system.
2446	*/
2447
2448	p->start_time = ktime_get_ns();
2449	p->start_boottime = ktime_get_boottime_ns();
2450
2451	/*
2452	* Make it visible to the rest of the system, but dont wake it up yet.
2453	* Need tasklist lock for parent etc handling!
2454	*/
2455	write_lock_irq(&tasklist_lock);
2456
2457	/* CLONE_PARENT re-uses the old parent */
2458	if (clone_flags & (CLONE_PARENT\|CLONE_THREAD)) {
2459	p->real_parent = current->real_parent;
2460	p->parent_exec_id = current->parent_exec_id;
2461	if (clone_flags & CLONE_THREAD)
2462	p->exit_signal = -1;
2463	else
2464	p->exit_signal = current->group_leader->exit_signal;
2465	} else {
2466	p->real_parent = current;
2467	p->parent_exec_id = current->self_exec_id;
2468	p->exit_signal = args->exit_signal;
2469	}
2470
2471	klp_copy_process(p);
2472
2473	sched_core_fork(p);
2474
2475	spin_lock(&current->sighand->siglock);
2476
2477	rv_task_fork(p);
2478
2479	rseq_fork(p, clone_flags);
2480
2481	/*
2482	* If zap_pid_ns_processes() was called after alloc_pid(), the new
2483	* child missed SIGKILL. If current is not in the same namespace,
2484	* we can't rely on fatal_signal_pending() below.
2485	*/
2486	if (unlikely(!(ns_of_pid(pid)->pid_allocated & PIDNS_ADDING))) {
2487	retval = -ENOMEM;
2488	goto bad_fork_core_free;
2489	}
2490
2491	/* Let kill terminate clone/fork in the middle */
2492	if (fatal_signal_pending(current)) {
2493	retval = -EINTR;
2494	goto bad_fork_core_free;
2495	}
2496
2497	/* No more failure paths after this point. */
2498
2499	/*
2500	* Copy seccomp details explicitly here, in case they were changed
2501	* before holding sighand lock.
2502	*/
2503	copy_seccomp(p);
2504
2505	if (clone_flags & CLONE_NNP)
2506	task_set_no_new_privs(p);
2507
2508	init_task_pid_links(p);
2509	if (likely(p->pid)) {
2510	ptrace_init_task(p, (clone_flags & CLONE_PTRACE) \|\| trace);
2511
2512	init_task_pid(p, PIDTYPE_PID, pid);
2513	if (thread_group_leader(p)) {
2514	init_task_pid(p, PIDTYPE_TGID, pid);
2515	init_task_pid(p, PIDTYPE_PGID, task_pgrp(current));
2516	init_task_pid(p, PIDTYPE_SID, task_session(current));
2517
2518	if (is_child_reaper(pid)) {
2519	struct pid_namespace *ns = ns_of_pid(pid);
2520
2521	ASSERT_EXCLUSIVE_WRITER(ns->child_reaper);
2522	WRITE_ONCE(ns->child_reaper, p);
2523	p->signal->flags \|= SIGNAL_UNKILLABLE;
2524	}
2525	p->signal->shared_pending.signal = delayed.signal;
2526	p->signal->tty = tty_kref_get(current->signal->tty);
2527	/*
2528	* Inherit has_child_subreaper flag under the same
2529	* tasklist_lock with adding child to the process tree
2530	* for propagate_has_child_subreaper optimization.
2531	*/
2532	p->signal->has_child_subreaper = p->real_parent->signal->has_child_subreaper \|\|
2533	p->real_parent->signal->is_child_subreaper;
2534	if (clone_flags & CLONE_AUTOREAP)
2535	p->signal->autoreap = 1;
2536	list_add_tail(&p->sibling, &p->real_parent->children);
2537	list_add_tail_rcu(&p->tasks, &init_task.tasks);
2538	attach_pid(p, PIDTYPE_TGID);
2539	attach_pid(p, PIDTYPE_PGID);
2540	attach_pid(p, PIDTYPE_SID);
2541	__this_cpu_inc(process_counts);
2542	} else {
2543	current->signal->nr_threads++;
2544	current->signal->quick_threads++;
2545	atomic_inc(&current->signal->live);
2546	refcount_inc(&current->signal->sigcnt);
2547	task_join_group_stop(p);
2548	list_add_tail_rcu(&p->thread_node,
2549	&p->signal->thread_head);
2550	}
2551	attach_pid(p, PIDTYPE_PID);
2552	nr_threads++;
2553	}
2554	total_forks++;
2555	hlist_del_init(&delayed.node);
2556	spin_unlock(&current->sighand->siglock);
2557	syscall_tracepoint_update(p);
2558	write_unlock_irq(&tasklist_lock);
2559
2560	if (pidfile)
2561	fd_install(pidfd, pidfile);
2562
2563	proc_fork_connector(p);
2564	/*
2565	* sched_ext needs @p to be associated with its cgroup in its post_fork
2566	* hook. cgroup_post_fork() should come before sched_post_fork().
2567	*/
2568	cgroup_post_fork(p, args);
2569	sched_post_fork(p);
2570	perf_event_fork(p);
2571
2572	trace_task_newtask(p, clone_flags);
2573	uprobe_copy_process(p, clone_flags);
2574	user_events_fork(p, clone_flags);
2575
2576	copy_oom_score_adj(clone_flags, p);
2577
2578	return p;
2579
2580	bad_fork_core_free:
2581	sched_core_free(p);
2582	spin_unlock(&current->sighand->siglock);
2583	write_unlock_irq(&tasklist_lock);
2584	bad_fork_cancel_cgroup:
2585	cgroup_cancel_fork(p, args);
2586	bad_fork_put_pidfd:
2587	if (clone_flags & CLONE_PIDFD) {
2588	fput(pidfile);
2589	put_unused_fd(pidfd);
2590	}
2591	bad_fork_free_pid:
2592	if (pid != &init_struct_pid)
2593	free_pid(pid);
2594	bad_fork_cleanup_thread:
2595	exit_thread(p);
2596	bad_fork_cleanup_io:
2597	if (p->io_context)
2598	exit_io_context(p);
2599	bad_fork_cleanup_namespaces:
2600	exit_nsproxy_namespaces(p);
2601	bad_fork_cleanup_mm:
2602	if (p->mm) {
2603	mm_clear_owner(p->mm, p);
2604	mmput(p->mm);
2605	}
2606	bad_fork_cleanup_signal:
2607	if (!(clone_flags & CLONE_THREAD))
2608	free_signal_struct(p->signal);
2609	bad_fork_cleanup_sighand:
2610	__cleanup_sighand(p->sighand);
2611	bad_fork_cleanup_fs:
2612	exit_fs(p); /* blocking */
2613	bad_fork_cleanup_files:
2614	exit_files(p); /* blocking */
2615	bad_fork_cleanup_semundo:
2616	exit_sem(p);
2617	bad_fork_cleanup_security:
2618	security_task_free(p);
2619	bad_fork_cleanup_audit:
2620	audit_free(p);
2621	bad_fork_cleanup_perf:
2622	perf_event_free_task(p);
2623	bad_fork_sched_cancel_fork:
2624	sched_cancel_fork(p);
2625	bad_fork_cleanup_policy:
2626	lockdep_free_task(p);
2627	#ifdef CONFIG_NUMA
2628	mpol_put(p->mempolicy);
2629	#endif
2630	bad_fork_cleanup_delayacct:
2631	io_uring_free(p);
2632	delayacct_tsk_free(p);
2633	bad_fork_cleanup_count:
2634	dec_rlimit_ucounts(task_ucounts(p), UCOUNT_RLIMIT_NPROC, 1);
2635	exit_cred_namespaces(p);
2636	exit_creds(p);
2637	bad_fork_free:
2638	WRITE_ONCE(p->__state, TASK_DEAD);
2639	exit_task_stack_account(p);
2640	put_task_stack(p);
2641	delayed_free_task(p);
2642	fork_out:
2643	spin_lock_irq(&current->sighand->siglock);
2644	hlist_del_init(&delayed.node);
2645	spin_unlock_irq(&current->sighand->siglock);
2646	return ERR_PTR(retval);
2647	}
2648
2649	static inline void init_idle_pids(struct task_struct *idle)
2650	{
2651	enum pid_type type;
2652
2653	for (type = PIDTYPE_PID; type < PIDTYPE_MAX; ++type) {
2654	INIT_HLIST_NODE(&idle->pid_links[type]); /* not really needed */
2655	init_task_pid(idle, type, &init_struct_pid);
2656	}
2657	}
2658
2659	static int idle_dummy(void *dummy)
2660	{
2661	/* This function is never called */
2662	return 0;
2663	}
2664
2665	struct task_struct * __init fork_idle(int cpu)
2666	{
2667	struct task_struct *task;
2668	struct kernel_clone_args args = {
2669	.flags = CLONE_VM,
2670	.fn = &idle_dummy,
2671	.fn_arg = NULL,
2672	.kthread = 1,
2673	.idle = 1,
2674	};
2675
2676	task = copy_process(&init_struct_pid, 0, cpu_to_node(cpu), &args);
2677	if (!IS_ERR(task)) {
2678	init_idle_pids(task);
2679	init_idle(task, cpu);
2680	}
2681
2682	return task;
2683	}
2684
2685	/*
2686	* This is like kernel_clone(), but shaved down and tailored to just
2687	* creating io_uring workers. It returns a created task, or an error pointer.
2688	* The returned task is inactive, and the caller must fire it up through
2689	* wake_up_new_task(p). All signals are blocked in the created task.
2690	*/
2691	struct task_struct create_io_thread(int (fn)(void ), void arg, int node)
2692	{
2693	unsigned long flags = CLONE_FS\|CLONE_FILES\|CLONE_SIGHAND\|CLONE_THREAD\|
2694	CLONE_IO\|CLONE_VM\|CLONE_UNTRACED;
2695	struct kernel_clone_args args = {
2696	.flags = flags,
2697	.fn = fn,
2698	.fn_arg = arg,
2699	.io_thread = 1,
2700	.user_worker = 1,
2701	};
2702
2703	return copy_process(NULL, 0, node, &args);
2704	}
2705
2706	/*
2707	* Ok, this is the main fork-routine.
2708	*
2709	* It copies the process, and if successful kick-starts
2710	* it and waits for it to finish using the VM if required.
2711	*/
2712	pid_t kernel_clone(struct kernel_clone_args *args)
2713	{
2714	u64 clone_flags = args->flags;
2715	struct completion vfork;
2716	struct pid *pid;
2717	struct task_struct *p;
2718	int trace = 0;
2719	pid_t nr;
2720
2721	/*
2722	* Creating an empty mount namespace implies creating a new mount
2723	* namespace. Set this before copy_process() so that the
2724	* CLONE_NEWNS\|CLONE_FS mutual exclusion check works correctly.
2725	*/
2726	if (clone_flags & CLONE_EMPTY_MNTNS) {
2727	clone_flags \|= CLONE_NEWNS;
2728	args->flags = clone_flags;
2729	}
2730
2731	/*
2732	* For legacy clone() calls, CLONE_PIDFD uses the parent_tid argument
2733	* to return the pidfd. Hence, CLONE_PIDFD and CLONE_PARENT_SETTID are
2734	* mutually exclusive. With clone3() CLONE_PIDFD has grown a separate
2735	* field in struct clone_args and it still doesn't make sense to have
2736	* them both point at the same memory location. Performing this check
2737	* here has the advantage that we don't need to have a separate helper
2738	* to check for legacy clone().
2739	*/
2740	if ((clone_flags & CLONE_PIDFD) &&
2741	(clone_flags & CLONE_PARENT_SETTID) &&
2742	(args->pidfd == args->parent_tid))
2743	return -EINVAL;
2744
2745	if (!valid_signal(args->exit_signal))
2746	return -EINVAL;
2747
2748	/*
2749	* Determine whether and which event to report to ptracer. When
2750	* called from kernel_thread or CLONE_UNTRACED is explicitly
2751	* requested, no event is reported; otherwise, report if the event
2752	* for the type of forking is enabled.
2753	*/
2754	if (!(clone_flags & CLONE_UNTRACED)) {
2755	if (clone_flags & CLONE_VFORK)
2756	trace = PTRACE_EVENT_VFORK;
2757	else if (args->exit_signal != SIGCHLD)
2758	trace = PTRACE_EVENT_CLONE;
2759	else
2760	trace = PTRACE_EVENT_FORK;
2761
2762	if (likely(!ptrace_event_enabled(current, trace)))
2763	trace = 0;
2764	}
2765
2766	p = copy_process(NULL, trace, NUMA_NO_NODE, args);
2767	add_latent_entropy();
2768
2769	if (IS_ERR(p))
2770	return PTR_ERR(p);
2771
2772	/*
2773	* Do this prior waking up the new thread - the thread pointer
2774	* might get invalid after that point, if the thread exits quickly.
2775	*/
2776	trace_sched_process_fork(current, p);
2777
2778	pid = get_task_pid(p, PIDTYPE_PID);
2779	nr = pid_vnr(pid);
2780
2781	if (clone_flags & CLONE_PARENT_SETTID)
2782	put_user(nr, args->parent_tid);
2783
2784	if (clone_flags & CLONE_VFORK) {
2785	p->vfork_done = &vfork;
2786	init_completion(&vfork);
2787	get_task_struct(p);
2788	}
2789
2790	if (IS_ENABLED(CONFIG_LRU_GEN_WALKS_MMU) && !(clone_flags & CLONE_VM)) {
2791	/* lock the task to synchronize with memcg migration */
2792	task_lock(p);
2793	lru_gen_add_mm(p->mm);
2794	task_unlock(p);
2795	}
2796
2797	wake_up_new_task(p);
2798
2799	/* forking complete and child started to run, tell ptracer */
2800	if (unlikely(trace))
2801	ptrace_event_pid(trace, pid);
2802
2803	if (clone_flags & CLONE_VFORK) {
2804	if (!wait_for_vfork_done(p, &vfork))
2805	ptrace_event_pid(PTRACE_EVENT_VFORK_DONE, pid);
2806	}
2807
2808	put_pid(pid);
2809	return nr;
2810	}
2811
2812	/*
2813	* Create a kernel thread.
2814	*/
2815	pid_t kernel_thread(int (fn)(void ), void arg, const char name,
2816	unsigned long flags)
2817	{
2818	struct kernel_clone_args args = {
2819	.flags = ((flags \| CLONE_VM \| CLONE_UNTRACED) & ~CSIGNAL),
2820	.exit_signal = (flags & CSIGNAL),
2821	.fn = fn,
2822	.fn_arg = arg,
2823	.name = name,
2824	.kthread = 1,
2825	};
2826
2827	return kernel_clone(&args);
2828	}
2829
2830	/*
2831	* Create a user mode thread.
2832	*/
2833	pid_t user_mode_thread(int (fn)(void ), void *arg, unsigned long flags)
2834	{
2835	struct kernel_clone_args args = {
2836	.flags = ((flags \| CLONE_VM \| CLONE_UNTRACED) & ~CSIGNAL),
2837	.exit_signal = (flags & CSIGNAL),
2838	.fn = fn,
2839	.fn_arg = arg,
2840	.umh = 1,
2841	};
2842
2843	return kernel_clone(&args);
2844	}
2845
2846	#ifdef __ARCH_WANT_SYS_FORK
2847	SYSCALL_DEFINE0(fork)
2848	{
2849	#ifdef CONFIG_MMU
2850	struct kernel_clone_args args = {
2851	.exit_signal = SIGCHLD,
2852	};
2853
2854	return kernel_clone(&args);
2855	#else
2856	/* can not support in nommu mode */
2857	return -EINVAL;
2858	#endif
2859	}
2860	#endif
2861
2862	#ifdef __ARCH_WANT_SYS_VFORK
2863	SYSCALL_DEFINE0(vfork)
2864	{
2865	struct kernel_clone_args args = {
2866	.flags = CLONE_VFORK \| CLONE_VM,
2867	.exit_signal = SIGCHLD,
2868	};
2869
2870	return kernel_clone(&args);
2871	}
2872	#endif
2873
2874	#ifdef __ARCH_WANT_SYS_CLONE
2875	#ifdef CONFIG_CLONE_BACKWARDS
2876	SYSCALL_DEFINE5(clone, unsigned long, clone_flags, unsigned long, newsp,
2877	int __user *, parent_tidptr,
2878	unsigned long, tls,
2879	int __user *, child_tidptr)
2880	#elif defined(CONFIG_CLONE_BACKWARDS2)
2881	SYSCALL_DEFINE5(clone, unsigned long, newsp, unsigned long, clone_flags,
2882	int __user *, parent_tidptr,
2883	int __user *, child_tidptr,
2884	unsigned long, tls)
2885	#elif defined(CONFIG_CLONE_BACKWARDS3)
2886	SYSCALL_DEFINE6(clone, unsigned long, clone_flags, unsigned long, newsp,
2887	int, stack_size,
2888	int __user *, parent_tidptr,
2889	int __user *, child_tidptr,
2890	unsigned long, tls)
2891	#else
2892	SYSCALL_DEFINE5(clone, unsigned long, clone_flags, unsigned long, newsp,
2893	int __user *, parent_tidptr,
2894	int __user *, child_tidptr,
2895	unsigned long, tls)
2896	#endif
2897	{
2898	struct kernel_clone_args args = {
2899	.flags = (lower_32_bits(clone_flags) & ~CSIGNAL),
2900	.pidfd = parent_tidptr,
2901	.child_tid = child_tidptr,
2902	.parent_tid = parent_tidptr,
2903	.exit_signal = (lower_32_bits(clone_flags) & CSIGNAL),
2904	.stack = newsp,
2905	.tls = tls,
2906	};
2907
2908	return kernel_clone(&args);
2909	}
2910	#endif
2911
2912	static noinline int copy_clone_args_from_user(struct kernel_clone_args *kargs,
2913	struct clone_args __user *uargs,
2914	size_t usize)
2915	{
2916	int err;
2917	struct clone_args args;
2918	pid_t *kset_tid = kargs->set_tid;
2919
2920	BUILD_BUG_ON(offsetofend(struct clone_args, tls) !=
2921	CLONE_ARGS_SIZE_VER0);
2922	BUILD_BUG_ON(offsetofend(struct clone_args, set_tid_size) !=
2923	CLONE_ARGS_SIZE_VER1);
2924	BUILD_BUG_ON(offsetofend(struct clone_args, cgroup) !=
2925	CLONE_ARGS_SIZE_VER2);
2926	BUILD_BUG_ON(sizeof(struct clone_args) != CLONE_ARGS_SIZE_VER2);
2927
2928	if (unlikely(usize > PAGE_SIZE))
2929	return -E2BIG;
2930	if (unlikely(usize < CLONE_ARGS_SIZE_VER0))
2931	return -EINVAL;
2932
2933	err = copy_struct_from_user(&args, sizeof(args), uargs, usize);
2934	if (err)
2935	return err;
2936
2937	if (unlikely(args.set_tid_size > MAX_PID_NS_LEVEL))
2938	return -EINVAL;
2939
2940	if (unlikely(!args.set_tid && args.set_tid_size > 0))
2941	return -EINVAL;
2942
2943	if (unlikely(args.set_tid && args.set_tid_size == 0))
2944	return -EINVAL;
2945
2946	/*
2947	* Verify that higher 32bits of exit_signal are unset
2948	*/
2949	if (unlikely(args.exit_signal & ~((u64)CSIGNAL)))
2950	return -EINVAL;
2951
2952	if ((args.flags & CLONE_INTO_CGROUP) &&
2953	(args.cgroup > INT_MAX \|\| usize < CLONE_ARGS_SIZE_VER2))
2954	return -EINVAL;
2955
2956	*kargs = (struct kernel_clone_args){
2957	.flags = args.flags,
2958	.pidfd = u64_to_user_ptr(args.pidfd),
2959	.child_tid = u64_to_user_ptr(args.child_tid),
2960	.parent_tid = u64_to_user_ptr(args.parent_tid),
2961	.exit_signal = args.exit_signal,
2962	.stack = args.stack,
2963	.stack_size = args.stack_size,
2964	.tls = args.tls,
2965	.set_tid_size = args.set_tid_size,
2966	.cgroup = args.cgroup,
2967	};
2968
2969	if (args.set_tid &&
2970	copy_from_user(kset_tid, u64_to_user_ptr(args.set_tid),
2971	(kargs->set_tid_size * sizeof(pid_t))))
2972	return -EFAULT;
2973
2974	kargs->set_tid = kset_tid;
2975
2976	return 0;
2977	}
2978
2979	/**
2980	* clone3_stack_valid - check and prepare stack
2981	* @kargs: kernel clone args
2982	*
2983	* Verify that the stack arguments userspace gave us are sane.
2984	* In addition, set the stack direction for userspace since it's easy for us to
2985	* determine.
2986	*/
2987	static inline bool clone3_stack_valid(struct kernel_clone_args *kargs)
2988	{
2989	if (kargs->stack == 0) {
2990	if (kargs->stack_size > 0)
2991	return false;
2992	} else {
2993	if (kargs->stack_size == 0)
2994	return false;
2995
2996	if (!access_ok((void __user *)kargs->stack, kargs->stack_size))
2997	return false;
2998
2999	#if !defined(CONFIG_STACK_GROWSUP)
3000	kargs->stack += kargs->stack_size;
3001	#endif
3002	}
3003
3004	return true;
3005	}
3006
3007	static bool clone3_args_valid(struct kernel_clone_args *kargs)
3008	{
3009	/* Verify that no unknown flags are passed along. */
3010	if (kargs->flags &
3011	~(CLONE_LEGACY_FLAGS \| CLONE_CLEAR_SIGHAND \|
3012	CLONE_INTO_CGROUP \| CLONE_AUTOREAP \| CLONE_NNP \|
3013	CLONE_PIDFD_AUTOKILL \| CLONE_EMPTY_MNTNS))
3014	return false;
3015
3016	/*
3017	* - make the CLONE_DETACHED bit reusable for clone3
3018	* - make the CSIGNAL bits reusable for clone3
3019	*/
3020	if (kargs->flags & (CLONE_DETACHED \| (CSIGNAL & (~CLONE_NEWTIME))))
3021	return false;
3022
3023	if ((kargs->flags & (CLONE_SIGHAND \| CLONE_CLEAR_SIGHAND)) ==
3024	(CLONE_SIGHAND \| CLONE_CLEAR_SIGHAND))
3025	return false;
3026
3027	if ((kargs->flags & (CLONE_THREAD \| CLONE_PARENT)) &&
3028	kargs->exit_signal)
3029	return false;
3030
3031	if (!clone3_stack_valid(kargs))
3032	return false;
3033
3034	return true;
3035	}
3036
3037	/**
3038	* sys_clone3 - create a new process with specific properties
3039	* @uargs: argument structure
3040	* @size: size of @uargs
3041	*
3042	* clone3() is the extensible successor to clone()/clone2().
3043	* It takes a struct as argument that is versioned by its size.
3044	*
3045	* Return: On success, a positive PID for the child process.
3046	* On error, a negative errno number.
3047	*/
3048	SYSCALL_DEFINE2(clone3, struct clone_args __user *, uargs, size_t, size)
3049	{
3050	int err;
3051
3052	struct kernel_clone_args kargs;
3053	pid_t set_tid[MAX_PID_NS_LEVEL];
3054
3055	#ifdef __ARCH_BROKEN_SYS_CLONE3
3056	#warning clone3() entry point is missing, please fix
3057	return -ENOSYS;
3058	#endif
3059
3060	kargs.set_tid = set_tid;
3061
3062	err = copy_clone_args_from_user(&kargs, uargs, size);
3063	if (err)
3064	return err;
3065
3066	if (!clone3_args_valid(&kargs))
3067	return -EINVAL;
3068
3069	return kernel_clone(&kargs);
3070	}
3071
3072	void walk_process_tree(struct task_struct top, proc_visitor visitor, void data)
3073	{
3074	struct task_struct leader, parent, *child;
3075	int res;
3076
3077	read_lock(&tasklist_lock);
3078	leader = top = top->group_leader;
3079	down:
3080	for_each_thread(leader, parent) {
3081	list_for_each_entry(child, &parent->children, sibling) {
3082	res = visitor(child, data);
3083	if (res) {
3084	if (res < 0)
3085	goto out;
3086	leader = child;
3087	goto down;
3088	}
3089	up:
3090	;
3091	}
3092	}
3093
3094	if (leader != top) {
3095	child = leader;
3096	parent = child->real_parent;
3097	leader = parent->group_leader;
3098	goto up;
3099	}
3100	out:
3101	read_unlock(&tasklist_lock);
3102	}
3103
3104	#ifndef ARCH_MIN_MMSTRUCT_ALIGN
3105	#define ARCH_MIN_MMSTRUCT_ALIGN 0
3106	#endif
3107
3108	static void sighand_ctor(void *data)
3109	{
3110	struct sighand_struct *sighand = data;
3111
3112	spin_lock_init(&sighand->siglock);
3113	init_waitqueue_head(&sighand->signalfd_wqh);
3114	}
3115
3116	void __init mm_cache_init(void)
3117	{
3118	unsigned int mm_size;
3119
3120	/*
3121	* The mm_cpumask is located at the end of mm_struct, and is
3122	* dynamically sized based on the maximum CPU number this system
3123	* can have, taking hotplug into account (nr_cpu_ids).
3124	*/
3125	mm_size = sizeof(struct mm_struct) + cpumask_size() + mm_cid_size();
3126
3127	mm_cachep = kmem_cache_create_usercopy("mm_struct",
3128	mm_size, ARCH_MIN_MMSTRUCT_ALIGN,
3129	SLAB_HWCACHE_ALIGN\|SLAB_PANIC\|SLAB_ACCOUNT,
3130	offsetof(struct mm_struct, saved_auxv),
3131	sizeof_field(struct mm_struct, saved_auxv),
3132	NULL);
3133	}
3134
3135	void __init proc_caches_init(void)
3136	{
3137	sighand_cachep = kmem_cache_create("sighand_cache",
3138	sizeof(struct sighand_struct), 0,
3139	SLAB_HWCACHE_ALIGN\|SLAB_PANIC\|SLAB_TYPESAFE_BY_RCU\|
3140	SLAB_ACCOUNT, sighand_ctor);
3141	signal_cachep = kmem_cache_create("signal_cache",
3142	sizeof(struct signal_struct), 0,
3143	SLAB_HWCACHE_ALIGN\|SLAB_PANIC\|SLAB_ACCOUNT,
3144	NULL);
3145	exec_state_init();
3146	files_cachep = kmem_cache_create("files_cache",
3147	sizeof(struct files_struct), 0,
3148	SLAB_HWCACHE_ALIGN\|SLAB_PANIC\|SLAB_ACCOUNT,
3149	NULL);
3150	fs_cachep = kmem_cache_create("fs_cache",
3151	sizeof(struct fs_struct), 0,
3152	SLAB_HWCACHE_ALIGN\|SLAB_PANIC\|SLAB_ACCOUNT,
3153	NULL);
3154	mmap_init();
3155	nsproxy_cache_init();
3156	}
3157
3158	/*
3159	* Check constraints on flags passed to the unshare system call.
3160	*/
3161	static int check_unshare_flags(unsigned long unshare_flags)
3162	{
3163	if (unshare_flags & ~(CLONE_THREAD\|CLONE_FS\|CLONE_SIGHAND\|
3164	CLONE_VM\|CLONE_FILES\|CLONE_SYSVSEM\|
3165	CLONE_NS_ALL \| UNSHARE_EMPTY_MNTNS))
3166	return -EINVAL;
3167	/*
3168	* Not implemented, but pretend it works if there is nothing
3169	* to unshare. Note that unsharing the address space or the
3170	* signal handlers also need to unshare the signal queues (aka
3171	* CLONE_THREAD).
3172	*/
3173	if (unshare_flags & (CLONE_THREAD \| CLONE_SIGHAND \| CLONE_VM)) {
3174	if (!thread_group_empty(current))
3175	return -EINVAL;
3176	}
3177	if (unshare_flags & (CLONE_SIGHAND \| CLONE_VM)) {
3178	if (refcount_read(&current->sighand->count) > 1)
3179	return -EINVAL;
3180	}
3181	if (unshare_flags & CLONE_VM) {
3182	if (!current_is_single_threaded())
3183	return -EINVAL;
3184	}
3185
3186	return 0;
3187	}
3188
3189	/*
3190	* Unshare the filesystem structure if it is being shared
3191	*/
3192	static int unshare_fs(unsigned long unshare_flags, struct fs_struct **new_fsp)
3193	{
3194	struct fs_struct *fs = current->fs;
3195
3196	if (!(unshare_flags & CLONE_FS) \|\| !fs)
3197	return 0;
3198
3199	/* don't need lock here; in the worst case we'll do useless copy */
3200	if (!(unshare_flags & CLONE_NEWNS) && fs->users == 1)
3201	return 0;
3202
3203	*new_fsp = copy_fs_struct(fs);
3204	if (!*new_fsp)
3205	return -ENOMEM;
3206
3207	return 0;
3208	}
3209
3210	/*
3211	* Unshare file descriptor table if it is being shared
3212	*/
3213	static int unshare_fd(unsigned long unshare_flags, struct files_struct **new_fdp)
3214	{
3215	struct files_struct *fd = current->files;
3216
3217	if ((unshare_flags & CLONE_FILES) &&
3218	(fd && atomic_read(&fd->count) > 1)) {
3219	fd = dup_fd(fd, NULL);
3220	if (IS_ERR(fd))
3221	return PTR_ERR(fd);
3222	*new_fdp = fd;
3223	}
3224
3225	return 0;
3226	}
3227
3228	/*
3229	* unshare allows a process to 'unshare' part of the process
3230	* context which was originally shared using clone. copy_*
3231	* functions used by kernel_clone() cannot be used here directly
3232	* because they modify an inactive task_struct that is being
3233	* constructed. Here we are modifying the current, active,
3234	* task_struct.
3235	*/
3236	int ksys_unshare(unsigned long unshare_flags)
3237	{
3238	struct fs_struct *new_fs = NULL;
3239	struct files_struct *new_fd = NULL;
3240	struct cred *new_cred = NULL;
3241	struct nsproxy *new_nsproxy = NULL;
3242	int do_sysvsem = 0;
3243	int err;
3244
3245	/*
3246	* If unsharing a user namespace must also unshare the thread group
3247	* and unshare the filesystem root and working directories.
3248	*/
3249	if (unshare_flags & CLONE_NEWUSER)
3250	unshare_flags \|= CLONE_THREAD \| CLONE_FS;
3251	/*
3252	* If unsharing vm, must also unshare signal handlers.
3253	*/
3254	if (unshare_flags & CLONE_VM)
3255	unshare_flags \|= CLONE_SIGHAND;
3256	/*
3257	* If unsharing a signal handlers, must also unshare the signal queues.
3258	*/
3259	if (unshare_flags & CLONE_SIGHAND)
3260	unshare_flags \|= CLONE_THREAD;
3261	/*
3262	* If unsharing namespace, must also unshare filesystem information.
3263	*/
3264	if (unshare_flags & UNSHARE_EMPTY_MNTNS)
3265	unshare_flags \|= CLONE_NEWNS;
3266	if (unshare_flags & CLONE_NEWNS)
3267	unshare_flags \|= CLONE_FS;
3268
3269	/* No unsharing with overriden fs state */
3270	VFS_WARN_ON_ONCE(unshare_flags & (CLONE_NEWNS \| CLONE_FS) &&
3271	current->fs != current->real_fs);
3272
3273	err = check_unshare_flags(unshare_flags);
3274	if (err)
3275	goto bad_unshare_out;
3276	/*
3277	* CLONE_NEWIPC must also detach from the undolist: after switching
3278	* to a new ipc namespace, the semaphore arrays from the old
3279	* namespace are unreachable.
3280	*/
3281	if (unshare_flags & (CLONE_NEWIPC\|CLONE_SYSVSEM))
3282	do_sysvsem = 1;
3283	err = unshare_fs(unshare_flags, &new_fs);
3284	if (err)
3285	goto bad_unshare_out;
3286	err = unshare_fd(unshare_flags, &new_fd);
3287	if (err)
3288	goto bad_unshare_cleanup_fs;
3289	err = unshare_userns(unshare_flags, &new_cred);
3290	if (err)
3291	goto bad_unshare_cleanup_fd;
3292	err = unshare_nsproxy_namespaces(unshare_flags, &new_nsproxy,
3293	new_cred, new_fs);
3294	if (err)
3295	goto bad_unshare_cleanup_cred;
3296	if (new_cred) {
3297	err = set_cred_ucounts(new_cred);
3298	if (err)
3299	goto bad_unshare_cleanup_nsproxy;
3300	}
3301
3302	if (new_fs \|\| new_fd \|\| do_sysvsem \|\| new_cred \|\| new_nsproxy) {
3303	if (do_sysvsem) {
3304	/*
3305	* CLONE_SYSVSEM is equivalent to sys_exit().
3306	*/
3307	exit_sem(current);
3308	}
3309	if (unshare_flags & CLONE_NEWIPC) {
3310	/* Orphan segments in old ns (see sem above). */
3311	exit_shm(current);
3312	shm_init_task(current);
3313	}
3314
3315	if (new_nsproxy) {
3316	switch_task_namespaces(current, new_nsproxy);
3317	new_nsproxy = NULL;
3318	}
3319
3320	if (new_fs)
3321	new_fs = switch_fs_struct(new_fs);
3322
3323	if (new_fd) {
3324	guard(task_lock)(current);
3325	swap(current->files, new_fd);
3326	}
3327
3328	if (new_cred) {
3329	/* Install the new user namespace */
3330	commit_creds(new_cred);
3331	new_cred = NULL;
3332	}
3333	}
3334
3335	perf_event_namespaces(current);
3336
3337	bad_unshare_cleanup_nsproxy:
3338	if (new_nsproxy)
3339	put_nsproxy(new_nsproxy);
3340	bad_unshare_cleanup_cred:
3341	if (new_cred)
3342	put_cred(new_cred);
3343	bad_unshare_cleanup_fd:
3344	if (new_fd)
3345	put_files_struct(new_fd);
3346	bad_unshare_cleanup_fs:
3347	if (new_fs)
3348	free_fs_struct(new_fs);
3349
3350	bad_unshare_out:
3351	return err;
3352	}
3353
3354	SYSCALL_DEFINE1(unshare, unsigned long, unshare_flags)
3355	{
3356	return ksys_unshare(unshare_flags);
3357	}
3358
3359	/*
3360	* Helper to unshare the files of the current task.
3361	* We don't want to expose copy_files internals to
3362	* the exec layer of the kernel.
3363	*/
3364
3365	int unshare_files(void)
3366	{
3367	struct task_struct *task = current;
3368	struct files_struct old, copy = NULL;
3369	int error;
3370
3371	error = unshare_fd(CLONE_FILES, &copy);
3372	if (error \|\| !copy)
3373	return error;
3374
3375	old = task->files;
3376	task_lock(task);
3377	task->files = copy;
3378	task_unlock(task);
3379	put_files_struct(old);
3380	return 0;
3381	}
3382
3383	static int sysctl_max_threads(const struct ctl_table *table, int write,
3384	void buffer, size_t lenp, loff_t *ppos)
3385	{
3386	struct ctl_table t;
3387	int ret;
3388	int threads = max_threads;
3389	int min = 1;
3390	int max = MAX_THREADS;
3391
3392	t = *table;
3393	t.data = &threads;
3394	t.extra1 = &min;
3395	t.extra2 = &max;
3396
3397	ret = proc_dointvec_minmax(&t, write, buffer, lenp, ppos);
3398	if (ret \|\| !write)
3399	return ret;
3400
3401	max_threads = threads;
3402
3403	return 0;
3404	}
3405
3406	static const struct ctl_table fork_sysctl_table[] = {
3407	{
3408	.procname = "threads-max",
3409	.data = NULL,
3410	.maxlen = sizeof(int),
3411	.mode = 0644,
3412	.proc_handler = sysctl_max_threads,
3413	},
3414	};
3415
3416	static int __init init_fork_sysctl(void)
3417	{
3418	register_sysctl_init("kernel", fork_sysctl_table);
3419	return 0;
3420	}
3421
3422	subsys_initcall(init_fork_sysctl);