projects/linux·blob assoc-array-fixes-v2 · refs

2015 lines · 50.1 KiB

1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * linux/kernel/exit.c
4 *
5 * Copyright (C) 1991, 1992 Linus Torvalds
6 */
7
8#include <linux/mm.h>
9#include <linux/slab.h>
10#include <linux/sched/autogroup.h>
11#include <linux/sched/mm.h>
12#include <linux/sched/stat.h>
13#include <linux/sched/task.h>
14#include <linux/sched/task_stack.h>
15#include <linux/sched/cputime.h>
16#include <linux/interrupt.h>
17#include <linux/module.h>
18#include <linux/capability.h>
19#include <linux/completion.h>
20#include <linux/personality.h>
21#include <linux/tty.h>
22#include <linux/iocontext.h>
23#include <linux/key.h>
24#include <linux/cpu.h>
25#include <linux/acct.h>
26#include <linux/tsacct_kern.h>
27#include <linux/file.h>
28#include <linux/freezer.h>
29#include <linux/binfmts.h>
30#include <linux/nsproxy.h>
31#include <linux/pid_namespace.h>
32#include <linux/ptrace.h>
33#include <linux/profile.h>
34#include <linux/mount.h>
35#include <linux/proc_fs.h>
36#include <linux/kthread.h>
37#include <linux/mempolicy.h>
38#include <linux/taskstats_kern.h>
39#include <linux/delayacct.h>
40#include <linux/cgroup.h>
41#include <linux/syscalls.h>
42#include <linux/signal.h>
43#include <linux/posix-timers.h>
44#include <linux/cn_proc.h>
45#include <linux/mutex.h>
46#include <linux/futex.h>
47#include <linux/pipe_fs_i.h>
48#include <linux/audit.h> /* for audit_free() */
49#include <linux/resource.h>
50#include <linux/task_io_accounting_ops.h>
51#include <linux/task_work.h>
52#include <linux/fs_struct.h>
53#include <linux/init_task.h>
54#include <linux/perf_event.h>
55#include <trace/events/sched.h>
56#include <linux/hw_breakpoint.h>
57#include <linux/oom.h>
58#include <linux/writeback.h>
59#include <linux/shm.h>
60#include <linux/kcov.h>
61#include <linux/kmsan.h>
62#include <linux/random.h>
63#include <linux/rcuwait.h>
64#include <linux/compat.h>
65#include <linux/io_uring.h>
66#include <linux/kprobes.h>
67#include <linux/rethook.h>
68#include <linux/sysfs.h>
69#include <linux/user_events.h>
70#include <linux/unwind_deferred.h>
71#include <linux/uaccess.h>
72#include <linux/pidfs.h>
73
74#include <uapi/linux/wait.h>
75
76#include <asm/unistd.h>
77#include <asm/mmu_context.h>
78
79#include "exit.h"
80
81/*
82 * The default value should be high enough to not crash a system that randomly
83 * crashes its kernel from time to time, but low enough to at least not permit
84 * overflowing 32-bit refcounts or the ldsem writer count.
85 */
86static unsigned int oops_limit = 10000;
87
88#ifdef CONFIG_SYSCTL
89static const struct ctl_table kern_exit_table[] = {
90 {
91 .procname = "oops_limit",
92 .data = &oops_limit,
93 .maxlen = sizeof(oops_limit),
94 .mode = 0644,
95 .proc_handler = proc_douintvec,
96 },
97};
98
99static __init int kernel_exit_sysctls_init(void)
100{
101 register_sysctl_init("kernel", kern_exit_table);
102 return 0;
103}
104late_initcall(kernel_exit_sysctls_init);
105#endif
106
107static atomic_t oops_count = ATOMIC_INIT(0);
108
109#ifdef CONFIG_SYSFS
110static ssize_t oops_count_show(struct kobject *kobj, struct kobj_attribute *attr,
111 char *page)
112{
113 return sysfs_emit(page, "%d\n", atomic_read(&oops_count));
114}
115
116static struct kobj_attribute oops_count_attr = __ATTR_RO(oops_count);
117
118static __init int kernel_exit_sysfs_init(void)
119{
120 sysfs_add_file_to_group(kernel_kobj, &oops_count_attr.attr, NULL);
121 return 0;
122}
123late_initcall(kernel_exit_sysfs_init);
124#endif
125
126/*
127 * For things release_task() would like to do *after* tasklist_lock is released.
128 */
129struct release_task_post {
130 struct pid *pids[PIDTYPE_MAX];
131};
132
133static void __unhash_process(struct release_task_post *post, struct task_struct *p,
134 bool group_dead)
135{
136 struct pid *pid = task_pid(p);
137
138 nr_threads--;
139
140 detach_pid(post->pids, p, PIDTYPE_PID);
141 wake_up_all(&pid->wait_pidfd);
142
143 if (group_dead) {
144 detach_pid(post->pids, p, PIDTYPE_TGID);
145 detach_pid(post->pids, p, PIDTYPE_PGID);
146 detach_pid(post->pids, p, PIDTYPE_SID);
147
148 list_del_rcu(&p->tasks);
149 list_del_init(&p->sibling);
150 __this_cpu_dec(process_counts);
151 }
152 list_del_rcu(&p->thread_node);
153}
154
155/*
156 * This function expects the tasklist_lock write-locked.
157 */
158static void __exit_signal(struct release_task_post *post, struct task_struct *tsk)
159{
160 struct signal_struct *sig = tsk->signal;
161 bool group_dead = thread_group_leader(tsk);
162 struct sighand_struct *sighand;
163 struct tty_struct *tty;
164 u64 utime, stime;
165
166 sighand = rcu_dereference_check(tsk->sighand,
167 lockdep_tasklist_lock_is_held());
168 spin_lock(&sighand->siglock);
169
170#ifdef CONFIG_POSIX_TIMERS
171 posix_cpu_timers_exit(tsk);
172 if (group_dead)
173 posix_cpu_timers_exit_group(tsk);
174#endif
175
176 if (group_dead) {
177 tty = sig->tty;
178 sig->tty = NULL;
179 } else {
180 /*
181 * If there is any task waiting for the group exit
182 * then notify it:
183 */
184 if (sig->notify_count > 0 && !--sig->notify_count)
185 wake_up_process(sig->group_exec_task);
186
187 if (tsk == sig->curr_target)
188 sig->curr_target = next_thread(tsk);
189 }
190
191 /*
192 * Accumulate here the counters for all threads as they die. We could
193 * skip the group leader because it is the last user of signal_struct,
194 * but we want to avoid the race with thread_group_cputime() which can
195 * see the empty ->thread_head list.
196 */
197 task_cputime(tsk, &utime, &stime);
198 write_seqlock(&sig->stats_lock);
199 sig->utime += utime;
200 sig->stime += stime;
201 sig->gtime += task_gtime(tsk);
202 sig->min_flt += tsk->min_flt;
203 sig->maj_flt += tsk->maj_flt;
204 sig->nvcsw += tsk->nvcsw;
205 sig->nivcsw += tsk->nivcsw;
206 sig->inblock += task_io_get_inblock(tsk);
207 sig->oublock += task_io_get_oublock(tsk);
208 task_io_accounting_add(&sig->ioac, &tsk->ioac);
209 sig->sum_sched_runtime += tsk->se.sum_exec_runtime;
210 sig->nr_threads--;
211 __unhash_process(post, tsk, group_dead);
212 write_sequnlock(&sig->stats_lock);
213
214 /*
215 * Ensure that all preceeding state is visible. Pairs with
216 * the smp_acquire__after_ctrl_dep() in the sighand == NULL
217 * path of lock_task_sighand().
218 */
219 smp_store_release(&tsk->sighand, NULL);
220 spin_unlock(&sighand->siglock);
221
222 __cleanup_sighand(sighand);
223 if (group_dead)
224 tty_kref_put(tty);
225}
226
227static void delayed_put_task_struct(struct rcu_head *rhp)
228{
229 struct task_struct *tsk = container_of(rhp, struct task_struct, rcu);
230
231 kprobe_flush_task(tsk);
232 rethook_flush_task(tsk);
233 perf_event_delayed_put(tsk);
234 trace_sched_process_free(tsk);
235 put_task_struct(tsk);
236}
237
238void put_task_struct_rcu_user(struct task_struct *task)
239{
240 if (refcount_dec_and_test(&task->rcu_users))
241 call_rcu(&task->rcu, delayed_put_task_struct);
242}
243
244void __weak release_thread(struct task_struct *dead_task)
245{
246}
247
248void release_task(struct task_struct *p)
249{
250 struct release_task_post post;
251 struct task_struct *leader;
252 struct pid *thread_pid;
253 int zap_leader;
254repeat:
255 memset(&post, 0, sizeof(post));
256
257 /* don't need to get the RCU readlock here - the process is dead and
258 * can't be modifying its own credentials. */
259 dec_rlimit_ucounts(task_ucounts(p), UCOUNT_RLIMIT_NPROC, 1);
260
261 pidfs_exit(p);
262 cgroup_task_release(p);
263
264 /*
265 * Pin @thread_pid before __unhash_process() clears it. The last
266 * PIDTYPE detach can otherwise free it before proc_flush_pid().
267 */
268 thread_pid = get_pid(task_pid(p));
269
270 write_lock_irq(&tasklist_lock);
271 ptrace_release_task(p);
272 __exit_signal(&post, p);
273
274 /*
275 * If we are the last non-leader member of the thread
276 * group, and the leader is zombie, then notify the
277 * group leader's parent process. (if it wants notification.)
278 */
279 zap_leader = 0;
280 leader = p->group_leader;
281 if (leader != p && thread_group_empty(leader)
282 && leader->exit_state == EXIT_ZOMBIE) {
283 /* for pidfs_exit() and do_notify_parent() */
284 if (leader->signal->flags & SIGNAL_GROUP_EXIT)
285 leader->exit_code = leader->signal->group_exit_code;
286 /*
287 * If we were the last child thread and the leader has
288 * exited already, and the leader's parent ignores SIGCHLD,
289 * then we are the one who should release the leader.
290 */
291 zap_leader = do_notify_parent(leader, leader->exit_signal);
292 if (zap_leader)
293 leader->exit_state = EXIT_DEAD;
294 }
295
296 write_unlock_irq(&tasklist_lock);
297 proc_flush_pid(thread_pid);
298 put_pid(thread_pid);
299 exit_cred_namespaces(p);
300 add_device_randomness(&p->se.sum_exec_runtime,
301 sizeof(p->se.sum_exec_runtime));
302 free_pids(post.pids);
303 release_thread(p);
304 /*
305 * This task was already removed from the process/thread/pid lists and
306 * lock_task_sighand(p) can't succeed. If it's the group leader then
307 * flush tsk->signal->shared_pending. tsk->pending has been flushed
308 * already in exit_signals(). Nothing else can touch
309 * signal->shared_pending anymore, so flush_sigqueue() can be invoked
310 * lockless.
311 */
312 if (thread_group_leader(p))
313 flush_sigqueue(&p->signal->shared_pending);
314
315 put_task_struct_rcu_user(p);
316
317 p = leader;
318 if (unlikely(zap_leader))
319 goto repeat;
320}
321
322int rcuwait_wake_up(struct rcuwait *w)
323{
324 int ret = 0;
325 struct task_struct *task;
326
327 rcu_read_lock();
328
329 /*
330 * Order condition vs @task, such that everything prior to the load
331 * of @task is visible. This is the condition as to why the user called
332 * rcuwait_wake() in the first place. Pairs with set_current_state()
333 * barrier (A) in rcuwait_wait_event().
334 *
335 * WAIT WAKE
336 * [S] tsk = current [S] cond = true
337 * MB (A) MB (B)
338 * [L] cond [L] tsk
339 */
340 smp_mb(); /* (B) */
341
342 task = rcu_dereference(w->task);
343 if (task)
344 ret = wake_up_process(task);
345 rcu_read_unlock();
346
347 return ret;
348}
349EXPORT_SYMBOL_GPL(rcuwait_wake_up);
350
351/*
352 * Determine if a process group is "orphaned", according to the POSIX
353 * definition in 2.2.2.52. Orphaned process groups are not to be affected
354 * by terminal-generated stop signals. Newly orphaned process groups are
355 * to receive a SIGHUP and a SIGCONT.
356 *
357 * "I ask you, have you ever known what it is to be an orphan?"
358 */
359static int will_become_orphaned_pgrp(struct pid *pgrp,
360 struct task_struct *ignored_task)
361{
362 struct task_struct *p;
363
364 do_each_pid_task(pgrp, PIDTYPE_PGID, p) {
365 if ((p == ignored_task) ||
366 (p->exit_state && thread_group_empty(p)) ||
367 is_global_init(p->real_parent))
368 continue;
369
370 if (task_pgrp(p->real_parent) != pgrp &&
371 task_session(p->real_parent) == task_session(p))
372 return 0;
373 } while_each_pid_task(pgrp, PIDTYPE_PGID, p);
374
375 return 1;
376}
377
378int is_current_pgrp_orphaned(void)
379{
380 int retval;
381
382 read_lock(&tasklist_lock);
383 retval = will_become_orphaned_pgrp(task_pgrp(current), NULL);
384 read_unlock(&tasklist_lock);
385
386 return retval;
387}
388
389static bool has_stopped_jobs(struct pid *pgrp)
390{
391 struct task_struct *p;
392
393 do_each_pid_task(pgrp, PIDTYPE_PGID, p) {
394 if (p->signal->flags & SIGNAL_STOP_STOPPED)
395 return true;
396 } while_each_pid_task(pgrp, PIDTYPE_PGID, p);
397
398 return false;
399}
400
401/*
402 * Check to see if any process groups have become orphaned as
403 * a result of our exiting, and if they have any stopped jobs,
404 * send them a SIGHUP and then a SIGCONT. (POSIX 3.2.2.2)
405 */
406static void
407kill_orphaned_pgrp(struct task_struct *tsk, struct task_struct *parent)
408{
409 struct pid *pgrp = task_pgrp(tsk);
410 struct task_struct *ignored_task = tsk;
411
412 if (!parent)
413 /* exit: our father is in a different pgrp than
414 * we are and we were the only connection outside.
415 */
416 parent = tsk->real_parent;
417 else
418 /* reparent: our child is in a different pgrp than
419 * we are, and it was the only connection outside.
420 */
421 ignored_task = NULL;
422
423 if (task_pgrp(parent) != pgrp &&
424 task_session(parent) == task_session(tsk) &&
425 will_become_orphaned_pgrp(pgrp, ignored_task) &&
426 has_stopped_jobs(pgrp)) {
427 __kill_pgrp_info(SIGHUP, SEND_SIG_PRIV, pgrp);
428 __kill_pgrp_info(SIGCONT, SEND_SIG_PRIV, pgrp);
429 }
430}
431
432static void coredump_task_exit(struct task_struct *tsk,
433 struct core_state *core_state)
434{
435 struct core_thread self;
436
437 self.task = tsk;
438 if (self.task->flags & PF_SIGNALED)
439 self.next = xchg(&core_state->dumper.next, &self);
440 else
441 self.task = NULL;
442 /*
443 * Implies mb(), the result of xchg() must be visible
444 * to core_state->dumper.
445 */
446 if (atomic_dec_and_test(&core_state->nr_threads))
447 complete(&core_state->startup);
448
449 for (;;) {
450 set_current_state(TASK_IDLE|TASK_FREEZABLE);
451 if (!self.task) /* see coredump_finish() */
452 break;
453 schedule();
454 }
455 __set_current_state(TASK_RUNNING);
456}
457
458#ifdef CONFIG_MEMCG
459/* drops tasklist_lock if succeeds */
460static bool __try_to_set_owner(struct task_struct *tsk, struct mm_struct *mm)
461{
462 bool ret = false;
463
464 task_lock(tsk);
465 if (likely(tsk->mm == mm)) {
466 /* tsk can't pass exit_mm/exec_mmap and exit */
467 read_unlock(&tasklist_lock);
468 WRITE_ONCE(mm->owner, tsk);
469 lru_gen_migrate_mm(mm);
470 ret = true;
471 }
472 task_unlock(tsk);
473 return ret;
474}
475
476static bool try_to_set_owner(struct task_struct *g, struct mm_struct *mm)
477{
478 struct task_struct *t;
479
480 for_each_thread(g, t) {
481 struct mm_struct *t_mm = READ_ONCE(t->mm);
482 if (t_mm == mm) {
483 if (__try_to_set_owner(t, mm))
484 return true;
485 } else if (t_mm)
486 break;
487 }
488
489 return false;
490}
491
492/*
493 * A task is exiting. If it owned this mm, find a new owner for the mm.
494 */
495void mm_update_next_owner(struct mm_struct *mm)
496{
497 struct task_struct *g, *p = current;
498
499 /*
500 * If the exiting or execing task is not the owner, it's
501 * someone else's problem.
502 */
503 if (mm->owner != p)
504 return;
505 /*
506 * The current owner is exiting/execing and there are no other
507 * candidates. Do not leave the mm pointing to a possibly
508 * freed task structure.
509 */
510 if (atomic_read(&mm->mm_users) <= 1) {
511 WRITE_ONCE(mm->owner, NULL);
512 return;
513 }
514
515 read_lock(&tasklist_lock);
516 /*
517 * Search in the children
518 */
519 list_for_each_entry(g, &p->children, sibling) {
520 if (try_to_set_owner(g, mm))
521 goto ret;
522 }
523 /*
524 * Search in the siblings
525 */
526 list_for_each_entry(g, &p->real_parent->children, sibling) {
527 if (try_to_set_owner(g, mm))
528 goto ret;
529 }
530 /*
531 * Search through everything else, we should not get here often.
532 */
533 for_each_process(g) {
534 if (atomic_read(&mm->mm_users) <= 1)
535 break;
536 if (g->flags & PF_KTHREAD)
537 continue;
538 if (try_to_set_owner(g, mm))
539 goto ret;
540 }
541 read_unlock(&tasklist_lock);
542 /*
543 * We found no owner yet mm_users > 1: this implies that we are
544 * most likely racing with swapoff (try_to_unuse()) or /proc or
545 * ptrace or page migration (get_task_mm()). Mark owner as NULL.
546 */
547 WRITE_ONCE(mm->owner, NULL);
548 ret:
549 return;
550
551}
552#endif /* CONFIG_MEMCG */
553
554/*
555 * Turn us into a lazy TLB process if we
556 * aren't already..
557 */
558static void exit_mm(void)
559{
560 struct mm_struct *mm = current->mm;
561
562 mm_exit_exec_release(current, mm);
563 if (!mm)
564 return;
565
566 sched_cache_exit_mm(current);
567
568 mmap_read_lock(mm);
569 mmgrab_lazy_tlb(mm);
570 BUG_ON(mm != current->active_mm);
571 /* more a memory barrier than a real lock */
572 task_lock(current);
573 /*
574 * When a thread stops operating on an address space, the loop
575 * in membarrier_private_expedited() may not observe that
576 * tsk->mm, and the loop in membarrier_global_expedited() may
577 * not observe a MEMBARRIER_STATE_GLOBAL_EXPEDITED
578 * rq->membarrier_state, so those would not issue an IPI.
579 * Membarrier requires a memory barrier after accessing
580 * user-space memory, before clearing tsk->mm or the
581 * rq->membarrier_state.
582 */
583 smp_mb__after_spinlock();
584 local_irq_disable();
585 current->mm = NULL;
586 membarrier_update_current_mm(NULL);
587 enter_lazy_tlb(mm, current);
588 local_irq_enable();
589 task_unlock(current);
590 mmap_read_unlock(mm);
591 mm_update_next_owner(mm);
592 mmput(mm);
593 if (test_thread_flag(TIF_MEMDIE))
594 exit_oom_victim();
595}
596
597static struct task_struct *find_alive_thread(struct task_struct *p)
598{
599 struct task_struct *t;
600
601 for_each_thread(p, t) {
602 if (!(t->flags & PF_EXITING))
603 return t;
604 }
605 return NULL;
606}
607
608static struct task_struct *find_child_reaper(struct task_struct *father,
609 struct list_head *dead)
610 __releases(&tasklist_lock)
611 __acquires(&tasklist_lock)
612{
613 struct pid_namespace *pid_ns = task_active_pid_ns(father);
614 struct task_struct *reaper = pid_ns->child_reaper;
615 struct task_struct *p, *n;
616
617 if (likely(reaper != father))
618 return reaper;
619
620 reaper = find_alive_thread(father);
621 if (reaper) {
622 ASSERT_EXCLUSIVE_WRITER(pid_ns->child_reaper);
623 WRITE_ONCE(pid_ns->child_reaper, reaper);
624 return reaper;
625 }
626
627 write_unlock_irq(&tasklist_lock);
628
629 list_for_each_entry_safe(p, n, dead, ptrace_entry) {
630 list_del_init(&p->ptrace_entry);
631 release_task(p);
632 }
633
634 zap_pid_ns_processes(pid_ns);
635 write_lock_irq(&tasklist_lock);
636
637 return father;
638}
639
640/*
641 * When we die, we re-parent all our children, and try to:
642 * 1. give them to another thread in our thread group, if such a member exists
643 * 2. give it to the first ancestor process which prctl'd itself as a
644 * child_subreaper for its children (like a service manager)
645 * 3. give it to the init process (PID 1) in our pid namespace
646 */
647static struct task_struct *find_new_reaper(struct task_struct *father,
648 struct task_struct *child_reaper)
649{
650 struct task_struct *thread, *reaper;
651
652 thread = find_alive_thread(father);
653 if (thread)
654 return thread;
655
656 if (father->signal->has_child_subreaper) {
657 unsigned int ns_level = task_pid(father)->level;
658 /*
659 * Find the first ->is_child_subreaper ancestor in our pid_ns.
660 * We can't check reaper != child_reaper to ensure we do not
661 * cross the namespaces, the exiting parent could be injected
662 * by setns() + fork().
663 * We check pid->level, this is slightly more efficient than
664 * task_active_pid_ns(reaper) != task_active_pid_ns(father).
665 */
666 for (reaper = father->real_parent;
667 task_pid(reaper)->level == ns_level;
668 reaper = reaper->real_parent) {
669 if (reaper == &init_task)
670 break;
671 if (!reaper->signal->is_child_subreaper)
672 continue;
673 thread = find_alive_thread(reaper);
674 if (thread)
675 return thread;
676 }
677 }
678
679 return child_reaper;
680}
681
682/*
683* Any that need to be release_task'd are put on the @dead list.
684 */
685static void reparent_leader(struct task_struct *father, struct task_struct *p,
686 struct list_head *dead)
687{
688 if (unlikely(p->exit_state == EXIT_DEAD))
689 return;
690
691 /* We don't want people slaying init. */
692 p->exit_signal = SIGCHLD;
693
694 /* If it has exited notify the new parent about this child's death. */
695 if (!p->ptrace &&
696 p->exit_state == EXIT_ZOMBIE && thread_group_empty(p)) {
697 if (do_notify_parent(p, p->exit_signal)) {
698 p->exit_state = EXIT_DEAD;
699 list_add(&p->ptrace_entry, dead);
700 }
701 }
702
703 kill_orphaned_pgrp(p, father);
704}
705
706/*
707 * Make init inherit all the child processes
708 */
709static void forget_original_parent(struct task_struct *father,
710 struct list_head *dead)
711{
712 struct task_struct *p, *t, *reaper;
713
714 if (unlikely(!list_empty(&father->ptraced)))
715 exit_ptrace(father, dead);
716
717 /* Can drop and reacquire tasklist_lock */
718 reaper = find_child_reaper(father, dead);
719 if (list_empty(&father->children))
720 return;
721
722 reaper = find_new_reaper(father, reaper);
723 list_for_each_entry(p, &father->children, sibling) {
724 for_each_thread(p, t) {
725 RCU_INIT_POINTER(t->real_parent, reaper);
726 BUG_ON((!t->ptrace) != (rcu_access_pointer(t->parent) == father));
727 if (likely(!t->ptrace))
728 t->parent = t->real_parent;
729 if (t->pdeath_signal)
730 group_send_sig_info(t->pdeath_signal,
731 SEND_SIG_NOINFO, t,
732 PIDTYPE_TGID);
733 }
734 /*
735 * If this is a threaded reparent there is no need to
736 * notify anyone anything has happened.
737 */
738 if (!same_thread_group(reaper, father))
739 reparent_leader(father, p, dead);
740 }
741 list_splice_tail_init(&father->children, &reaper->children);
742}
743
744/*
745 * Send signals to all our closest relatives so that they know
746 * to properly mourn us..
747 */
748static void exit_notify(struct task_struct *tsk, int group_dead)
749{
750 bool autoreap;
751 struct task_struct *p, *n;
752 LIST_HEAD(dead);
753
754 write_lock_irq(&tasklist_lock);
755 forget_original_parent(tsk, &dead);
756
757 if (group_dead)
758 kill_orphaned_pgrp(tsk->group_leader, NULL);
759
760 tsk->exit_state = EXIT_ZOMBIE;
761
762 if (unlikely(tsk->ptrace)) {
763 int sig = thread_group_empty(tsk) && !ptrace_reparented(tsk)
764 ? tsk->exit_signal : SIGCHLD;
765 autoreap = do_notify_parent(tsk, sig);
766 } else if (thread_group_leader(tsk)) {
767 autoreap = thread_group_empty(tsk) &&
768 do_notify_parent(tsk, tsk->exit_signal);
769 } else {
770 autoreap = true;
771 /* untraced sub-thread */
772 do_notify_pidfd(tsk);
773 }
774
775 if (autoreap) {
776 tsk->exit_state = EXIT_DEAD;
777 list_add(&tsk->ptrace_entry, &dead);
778 }
779
780 /* mt-exec, de_thread() is waiting for group leader */
781 if (unlikely(tsk->signal->notify_count < 0))
782 wake_up_process(tsk->signal->group_exec_task);
783 write_unlock_irq(&tasklist_lock);
784
785 list_for_each_entry_safe(p, n, &dead, ptrace_entry) {
786 list_del_init(&p->ptrace_entry);
787 release_task(p);
788 }
789}
790
791#ifdef CONFIG_DEBUG_STACK_USAGE
792#ifdef CONFIG_STACK_GROWSUP
793unsigned long stack_not_used(struct task_struct *p)
794{
795 unsigned long *n = end_of_stack(p);
796
797 do { /* Skip over canary */
798 n--;
799 } while (!*n);
800
801 return (unsigned long)end_of_stack(p) - (unsigned long)n;
802}
803#else /* !CONFIG_STACK_GROWSUP */
804unsigned long stack_not_used(struct task_struct *p)
805{
806 unsigned long *n = end_of_stack(p);
807
808 do { /* Skip over canary */
809 n++;
810 } while (!*n);
811
812 return (unsigned long)n - (unsigned long)end_of_stack(p);
813}
814#endif /* CONFIG_STACK_GROWSUP */
815
816/* Count the maximum pages reached in kernel stacks */
817static inline void kstack_histogram(unsigned long used_stack)
818{
819#ifdef CONFIG_VM_EVENT_COUNTERS
820 if (used_stack <= 1024)
821 count_vm_event(KSTACK_1K);
822#if THREAD_SIZE > 1024
823 else if (used_stack <= 2048)
824 count_vm_event(KSTACK_2K);
825#endif
826#if THREAD_SIZE > 2048
827 else if (used_stack <= 4096)
828 count_vm_event(KSTACK_4K);
829#endif
830#if THREAD_SIZE > 4096
831 else if (used_stack <= 8192)
832 count_vm_event(KSTACK_8K);
833#endif
834#if THREAD_SIZE > 8192
835 else if (used_stack <= 16384)
836 count_vm_event(KSTACK_16K);
837#endif
838#if THREAD_SIZE > 16384
839 else if (used_stack <= 32768)
840 count_vm_event(KSTACK_32K);
841#endif
842#if THREAD_SIZE > 32768
843 else if (used_stack <= 65536)
844 count_vm_event(KSTACK_64K);
845#endif
846#if THREAD_SIZE > 65536
847 else
848 count_vm_event(KSTACK_REST);
849#endif
850#endif /* CONFIG_VM_EVENT_COUNTERS */
851}
852
853static void check_stack_usage(void)
854{
855 static DEFINE_SPINLOCK(low_water_lock);
856 static int lowest_to_date = THREAD_SIZE;
857 unsigned long free;
858
859 free = stack_not_used(current);
860 kstack_histogram(THREAD_SIZE - free);
861
862 if (free >= lowest_to_date)
863 return;
864
865 spin_lock(&low_water_lock);
866 if (free < lowest_to_date) {
867 pr_info("%s (%d) used greatest stack depth: %lu bytes left\n",
868 current->comm, task_pid_nr(current), free);
869 lowest_to_date = free;
870 }
871 spin_unlock(&low_water_lock);
872}
873#else /* !CONFIG_DEBUG_STACK_USAGE */
874static inline void check_stack_usage(void) {}
875#endif /* CONFIG_DEBUG_STACK_USAGE */
876
877static void synchronize_group_exit(struct task_struct *tsk, long code)
878{
879 struct sighand_struct *sighand = tsk->sighand;
880 struct signal_struct *signal = tsk->signal;
881 struct core_state *core_state;
882
883 spin_lock_irq(&sighand->siglock);
884 signal->quick_threads--;
885 if ((signal->quick_threads == 0) &&
886 !(signal->flags & SIGNAL_GROUP_EXIT)) {
887 signal->flags = SIGNAL_GROUP_EXIT;
888 signal->group_exit_code = code;
889 signal->group_stop_count = 0;
890 }
891 /*
892 * Serialize with any possible pending coredump.
893 * We must hold siglock around checking core_state
894 * and setting PF_POSTCOREDUMP. The core-inducing thread
895 * will increment ->nr_threads for each thread in the
896 * group without PF_POSTCOREDUMP set.
897 */
898 tsk->flags |= PF_POSTCOREDUMP;
899 core_state = signal->core_state;
900 spin_unlock_irq(&sighand->siglock);
901
902 if (unlikely(core_state))
903 coredump_task_exit(tsk, core_state);
904}
905
906void __noreturn do_exit(long code)
907{
908 struct task_struct *tsk = current;
909 struct kthread *kthread;
910 int group_dead;
911
912 WARN_ON(irqs_disabled());
913 WARN_ON(tsk->plug);
914
915 kthread = tsk_is_kthread(tsk);
916 if (unlikely(kthread))
917 kthread_do_exit(kthread, code);
918
919 kcov_task_exit(tsk);
920 kmsan_task_exit(tsk);
921
922 synchronize_group_exit(tsk, code);
923 ptrace_event(PTRACE_EVENT_EXIT, code);
924 user_events_exit(tsk);
925
926 io_uring_files_cancel();
927 sched_mm_cid_exit(tsk);
928 exit_signals(tsk); /* sets PF_EXITING */
929
930 seccomp_filter_release(tsk);
931
932 acct_update_integrals(tsk);
933 group_dead = atomic_dec_and_test(&tsk->signal->live);
934 if (group_dead) {
935 /*
936 * If the last thread of global init has exited, panic
937 * immediately to get a useable coredump.
938 */
939 if (unlikely(is_global_init(tsk)))
940 panic("Attempted to kill init! exitcode=0x%08x\n",
941 tsk->signal->group_exit_code ?: (int)code);
942
943#ifdef CONFIG_POSIX_TIMERS
944 hrtimer_cancel(&tsk->signal->real_timer);
945 exit_itimers(tsk);
946#endif
947 if (tsk->mm)
948 setmax_mm_hiwater_rss(&tsk->signal->maxrss, tsk->mm);
949 }
950 acct_collect(code, group_dead);
951 if (group_dead)
952 tty_audit_exit();
953 audit_free(tsk);
954
955 tsk->exit_code = code;
956 taskstats_exit(tsk, group_dead);
957 trace_sched_process_exit(tsk, group_dead);
958
959 /*
960 * Since sampling can touch ->mm, make sure to stop everything before we
961 * tear it down.
962 *
963 * Also flushes inherited counters to the parent - before the parent
964 * gets woken up by child-exit notifications.
965 */
966 perf_event_exit_task(tsk);
967 /*
968 * PF_EXITING (above) ensures unwind_deferred_request() will no
969 * longer add new unwinds. While exit_mm() (below) will destroy the
970 * abaility to do unwinds. So flush any pending unwinds here.
971 */
972 unwind_deferred_task_exit(tsk);
973
974 exit_mm();
975
976 if (group_dead)
977 acct_process();
978
979 exit_sem(tsk);
980 exit_shm(tsk);
981 exit_files(tsk);
982 exit_fs(tsk);
983 if (group_dead)
984 disassociate_ctty(1);
985 exit_nsproxy_namespaces(tsk);
986 exit_task_work(tsk);
987 exit_thread(tsk);
988
989 sched_autogroup_exit_task(tsk);
990 cgroup_task_exit(tsk);
991
992 /*
993 * FIXME: do that only when needed, using sched_exit tracepoint
994 */
995 flush_ptrace_hw_breakpoint(tsk);
996
997 exit_tasks_rcu_start();
998 exit_notify(tsk, group_dead);
999 proc_exit_connector(tsk);
1000 mpol_put_task_policy(tsk);
1001#ifdef CONFIG_FUTEX
1002 if (unlikely(current->futex.pi_state_cache))
1003 kfree(current->futex.pi_state_cache);
1004#endif
1005 /*
1006 * Make sure we are holding no locks:
1007 */
1008 debug_check_no_locks_held();
1009
1010 if (tsk->io_context)
1011 exit_io_context(tsk);
1012
1013 if (tsk->splice_pipe)
1014 free_pipe_info(tsk->splice_pipe);
1015
1016 if (tsk->task_frag.page)
1017 put_page(tsk->task_frag.page);
1018
1019 exit_task_stack_account(tsk);
1020
1021 check_stack_usage();
1022 preempt_disable();
1023 if (tsk->nr_dirtied)
1024 __this_cpu_add(dirty_throttle_leaks, tsk->nr_dirtied);
1025 exit_rcu();
1026 exit_tasks_rcu_finish();
1027
1028 lockdep_free_task(tsk);
1029 do_task_dead();
1030}
1031EXPORT_SYMBOL(do_exit);
1032
1033void __noreturn make_task_dead(int signr)
1034{
1035 /*
1036 * Take the task off the cpu after something catastrophic has
1037 * happened.
1038 *
1039 * We can get here from a kernel oops, sometimes with preemption off.
1040 * Start by checking for critical errors.
1041 * Then fix up important state like USER_DS and preemption.
1042 * Then do everything else.
1043 */
1044 struct task_struct *tsk = current;
1045 unsigned int limit;
1046
1047 if (unlikely(in_interrupt()))
1048 panic("Aiee, killing interrupt handler!");
1049 if (unlikely(!tsk->pid))
1050 panic("Attempted to kill the idle task!");
1051
1052 if (unlikely(irqs_disabled())) {
1053 pr_info("note: %s[%d] exited with irqs disabled\n",
1054 current->comm, task_pid_nr(current));
1055 local_irq_enable();
1056 }
1057 if (unlikely(in_atomic())) {
1058 pr_info("note: %s[%d] exited with preempt_count %d\n",
1059 current->comm, task_pid_nr(current),
1060 preempt_count());
1061 preempt_count_set(PREEMPT_ENABLED);
1062 }
1063
1064 /*
1065 * Every time the system oopses, if the oops happens while a reference
1066 * to an object was held, the reference leaks.
1067 * If the oops doesn't also leak memory, repeated oopsing can cause
1068 * reference counters to wrap around (if they're not using refcount_t).
1069 * This means that repeated oopsing can make unexploitable-looking bugs
1070 * exploitable through repeated oopsing.
1071 * To make sure this can't happen, place an upper bound on how often the
1072 * kernel may oops without panic().
1073 */
1074 limit = READ_ONCE(oops_limit);
1075 if (atomic_inc_return(&oops_count) >= limit && limit)
1076 panic("Oopsed too often (kernel.oops_limit is %d)", limit);
1077
1078 /*
1079 * We're taking recursive faults here in make_task_dead. Safest is to just
1080 * leave this task alone and wait for reboot.
1081 */
1082 if (unlikely(tsk->flags & PF_EXITING)) {
1083 pr_alert("Fixing recursive fault but reboot is needed!\n");
1084 futex_exit_recursive(tsk);
1085 tsk->exit_state = EXIT_DEAD;
1086 refcount_inc(&tsk->rcu_users);
1087 preempt_disable();
1088 do_task_dead();
1089 }
1090
1091 do_exit(signr);
1092}
1093
1094SYSCALL_DEFINE1(exit, int, error_code)
1095{
1096 do_exit((error_code & 0xff) << 8);
1097}
1098
1099/*
1100 * Take down every thread in the group. This is called by fatal signals
1101 * as well as by sys_exit_group (below).
1102 */
1103void __noreturn
1104do_group_exit(int exit_code)
1105{
1106 struct signal_struct *sig = current->signal;
1107
1108 if (sig->flags & SIGNAL_GROUP_EXIT)
1109 exit_code = sig->group_exit_code;
1110 else if (sig->group_exec_task)
1111 exit_code = 0;
1112 else {
1113 struct sighand_struct *const sighand = current->sighand;
1114
1115 spin_lock_irq(&sighand->siglock);
1116 if (sig->flags & SIGNAL_GROUP_EXIT)
1117 /* Another thread got here before we took the lock. */
1118 exit_code = sig->group_exit_code;
1119 else if (sig->group_exec_task)
1120 exit_code = 0;
1121 else {
1122 sig->group_exit_code = exit_code;
1123 sig->flags = SIGNAL_GROUP_EXIT;
1124 zap_other_threads(current);
1125 }
1126 spin_unlock_irq(&sighand->siglock);
1127 }
1128
1129 do_exit(exit_code);
1130 /* NOTREACHED */
1131}
1132
1133/*
1134 * this kills every thread in the thread group. Note that any externally
1135 * wait4()-ing process will get the correct exit code - even if this
1136 * thread is not the thread group leader.
1137 */
1138SYSCALL_DEFINE1(exit_group, int, error_code)
1139{
1140 do_group_exit((error_code & 0xff) << 8);
1141 /* NOTREACHED */
1142 return 0;
1143}
1144
1145static int eligible_pid(struct wait_opts *wo, struct task_struct *p)
1146{
1147 return wo->wo_type == PIDTYPE_MAX ||
1148 task_pid_type(p, wo->wo_type) == wo->wo_pid;
1149}
1150
1151static int
1152eligible_child(struct wait_opts *wo, bool ptrace, struct task_struct *p)
1153{
1154 if (!eligible_pid(wo, p))
1155 return 0;
1156
1157 /*
1158 * Wait for all children (clone and not) if __WALL is set or
1159 * if it is traced by us.
1160 */
1161 if (ptrace || (wo->wo_flags & __WALL))
1162 return 1;
1163
1164 /*
1165 * Otherwise, wait for clone children *only* if __WCLONE is set;
1166 * otherwise, wait for non-clone children *only*.
1167 *
1168 * Note: a "clone" child here is one that reports to its parent
1169 * using a signal other than SIGCHLD, or a non-leader thread which
1170 * we can only see if it is traced by us.
1171 */
1172 if ((p->exit_signal != SIGCHLD) ^ !!(wo->wo_flags & __WCLONE))
1173 return 0;
1174
1175 return 1;
1176}
1177
1178/*
1179 * Handle sys_wait4 work for one task in state EXIT_ZOMBIE. We hold
1180 * read_lock(&tasklist_lock) on entry. If we return zero, we still hold
1181 * the lock and this task is uninteresting. If we return nonzero, we have
1182 * released the lock and the system call should return.
1183 */
1184static int wait_task_zombie(struct wait_opts *wo, struct task_struct *p)
1185{
1186 int state, status;
1187 pid_t pid = task_pid_vnr(p);
1188 uid_t uid = from_kuid_munged(current_user_ns(), task_uid(p));
1189 struct waitid_info *infop;
1190
1191 if (!likely(wo->wo_flags & WEXITED))
1192 return 0;
1193
1194 if (unlikely(wo->wo_flags & WNOWAIT)) {
1195 status = (p->signal->flags & SIGNAL_GROUP_EXIT)
1196 ? p->signal->group_exit_code : p->exit_code;
1197 get_task_struct(p);
1198 read_unlock(&tasklist_lock);
1199 sched_annotate_sleep();
1200 if (wo->wo_rusage)
1201 getrusage(p, RUSAGE_BOTH, wo->wo_rusage);
1202 put_task_struct(p);
1203 goto out_info;
1204 }
1205 /*
1206 * Move the task's state to DEAD/TRACE, only one thread can do this.
1207 */
1208 state = (ptrace_reparented(p) && thread_group_leader(p)) ?
1209 EXIT_TRACE : EXIT_DEAD;
1210 if (cmpxchg(&p->exit_state, EXIT_ZOMBIE, state) != EXIT_ZOMBIE)
1211 return 0;
1212 /*
1213 * We own this thread, nobody else can reap it.
1214 */
1215 read_unlock(&tasklist_lock);
1216 sched_annotate_sleep();
1217
1218 /*
1219 * Check thread_group_leader() to exclude the traced sub-threads.
1220 */
1221 if (state == EXIT_DEAD && thread_group_leader(p)) {
1222 struct signal_struct *sig = p->signal;
1223 struct signal_struct *psig = current->signal;
1224 unsigned long maxrss;
1225 u64 tgutime, tgstime;
1226
1227 /*
1228 * The resource counters for the group leader are in its
1229 * own task_struct. Those for dead threads in the group
1230 * are in its signal_struct, as are those for the child
1231 * processes it has previously reaped. All these
1232 * accumulate in the parent's signal_struct c* fields.
1233 *
1234 * We don't bother to take a lock here to protect these
1235 * p->signal fields because the whole thread group is dead
1236 * and nobody can change them.
1237 *
1238 * psig->stats_lock also protects us from our sub-threads
1239 * which can reap other children at the same time.
1240 *
1241 * We use thread_group_cputime_adjusted() to get times for
1242 * the thread group, which consolidates times for all threads
1243 * in the group including the group leader.
1244 */
1245 thread_group_cputime_adjusted(p, &tgutime, &tgstime);
1246 write_seqlock_irq(&psig->stats_lock);
1247 psig->cutime += tgutime + sig->cutime;
1248 psig->cstime += tgstime + sig->cstime;
1249 psig->cgtime += task_gtime(p) + sig->gtime + sig->cgtime;
1250 psig->cmin_flt +=
1251 p->min_flt + sig->min_flt + sig->cmin_flt;
1252 psig->cmaj_flt +=
1253 p->maj_flt + sig->maj_flt + sig->cmaj_flt;
1254 psig->cnvcsw +=
1255 p->nvcsw + sig->nvcsw + sig->cnvcsw;
1256 psig->cnivcsw +=
1257 p->nivcsw + sig->nivcsw + sig->cnivcsw;
1258 psig->cinblock +=
1259 task_io_get_inblock(p) +
1260 sig->inblock + sig->cinblock;
1261 psig->coublock +=
1262 task_io_get_oublock(p) +
1263 sig->oublock + sig->coublock;
1264 maxrss = max(sig->maxrss, sig->cmaxrss);
1265 if (psig->cmaxrss < maxrss)
1266 psig->cmaxrss = maxrss;
1267 task_io_accounting_add(&psig->ioac, &p->ioac);
1268 task_io_accounting_add(&psig->ioac, &sig->ioac);
1269 write_sequnlock_irq(&psig->stats_lock);
1270 }
1271
1272 if (wo->wo_rusage)
1273 getrusage(p, RUSAGE_BOTH, wo->wo_rusage);
1274 status = (p->signal->flags & SIGNAL_GROUP_EXIT)
1275 ? p->signal->group_exit_code : p->exit_code;
1276 wo->wo_stat = status;
1277
1278 if (state == EXIT_TRACE) {
1279 write_lock_irq(&tasklist_lock);
1280 /* We dropped tasklist, ptracer could die and untrace */
1281 ptrace_unlink(p);
1282
1283 /* If parent wants a zombie, don't release it now */
1284 state = EXIT_ZOMBIE;
1285 if (do_notify_parent(p, p->exit_signal))
1286 state = EXIT_DEAD;
1287 p->exit_state = state;
1288 write_unlock_irq(&tasklist_lock);
1289 }
1290 if (state == EXIT_DEAD)
1291 release_task(p);
1292
1293out_info:
1294 infop = wo->wo_info;
1295 if (infop) {
1296 if ((status & 0x7f) == 0) {
1297 infop->cause = CLD_EXITED;
1298 infop->status = status >> 8;
1299 } else {
1300 infop->cause = (status & 0x80) ? CLD_DUMPED : CLD_KILLED;
1301 infop->status = status & 0x7f;
1302 }
1303 infop->pid = pid;
1304 infop->uid = uid;
1305 }
1306
1307 return pid;
1308}
1309
1310static int *task_stopped_code(struct task_struct *p, bool ptrace)
1311{
1312 if (ptrace) {
1313 if (task_is_traced(p) && !(p->jobctl & JOBCTL_LISTENING))
1314 return &p->exit_code;
1315 } else {
1316 if (p->signal->flags & SIGNAL_STOP_STOPPED)
1317 return &p->signal->group_exit_code;
1318 }
1319 return NULL;
1320}
1321
1322/**
1323 * wait_task_stopped - Wait for %TASK_STOPPED or %TASK_TRACED
1324 * @wo: wait options
1325 * @ptrace: is the wait for ptrace
1326 * @p: task to wait for
1327 *
1328 * Handle sys_wait4() work for %p in state %TASK_STOPPED or %TASK_TRACED.
1329 *
1330 * CONTEXT:
1331 * read_lock(&tasklist_lock), which is released if return value is
1332 * non-zero. Also, grabs and releases @p->sighand->siglock.
1333 *
1334 * RETURNS:
1335 * 0 if wait condition didn't exist and search for other wait conditions
1336 * should continue. Non-zero return, -errno on failure and @p's pid on
1337 * success, implies that tasklist_lock is released and wait condition
1338 * search should terminate.
1339 */
1340static int wait_task_stopped(struct wait_opts *wo,
1341 int ptrace, struct task_struct *p)
1342{
1343 struct waitid_info *infop;
1344 int exit_code, *p_code, why;
1345 uid_t uid = 0; /* unneeded, required by compiler */
1346 pid_t pid;
1347
1348 /*
1349 * Traditionally we see ptrace'd stopped tasks regardless of options.
1350 */
1351 if (!ptrace && !(wo->wo_flags & WUNTRACED))
1352 return 0;
1353
1354 if (!task_stopped_code(p, ptrace))
1355 return 0;
1356
1357 exit_code = 0;
1358 spin_lock_irq(&p->sighand->siglock);
1359
1360 p_code = task_stopped_code(p, ptrace);
1361 if (unlikely(!p_code))
1362 goto unlock_sig;
1363
1364 exit_code = *p_code;
1365 if (!exit_code)
1366 goto unlock_sig;
1367
1368 if (!unlikely(wo->wo_flags & WNOWAIT))
1369 *p_code = 0;
1370
1371 uid = from_kuid_munged(current_user_ns(), task_uid(p));
1372unlock_sig:
1373 spin_unlock_irq(&p->sighand->siglock);
1374 if (!exit_code)
1375 return 0;
1376
1377 /*
1378 * Now we are pretty sure this task is interesting.
1379 * Make sure it doesn't get reaped out from under us while we
1380 * give up the lock and then examine it below. We don't want to
1381 * keep holding onto the tasklist_lock while we call getrusage and
1382 * possibly take page faults for user memory.
1383 */
1384 get_task_struct(p);
1385 pid = task_pid_vnr(p);
1386 why = ptrace ? CLD_TRAPPED : CLD_STOPPED;
1387 read_unlock(&tasklist_lock);
1388 sched_annotate_sleep();
1389 if (wo->wo_rusage)
1390 getrusage(p, RUSAGE_BOTH, wo->wo_rusage);
1391 put_task_struct(p);
1392
1393 if (likely(!(wo->wo_flags & WNOWAIT)))
1394 wo->wo_stat = (exit_code << 8) | 0x7f;
1395
1396 infop = wo->wo_info;
1397 if (infop) {
1398 infop->cause = why;
1399 infop->status = exit_code;
1400 infop->pid = pid;
1401 infop->uid = uid;
1402 }
1403 return pid;
1404}
1405
1406/*
1407 * Handle do_wait work for one task in a live, non-stopped state.
1408 * read_lock(&tasklist_lock) on entry. If we return zero, we still hold
1409 * the lock and this task is uninteresting. If we return nonzero, we have
1410 * released the lock and the system call should return.
1411 */
1412static int wait_task_continued(struct wait_opts *wo, struct task_struct *p)
1413{
1414 struct waitid_info *infop;
1415 pid_t pid;
1416 uid_t uid;
1417
1418 if (!unlikely(wo->wo_flags & WCONTINUED))
1419 return 0;
1420
1421 if (!(p->signal->flags & SIGNAL_STOP_CONTINUED))
1422 return 0;
1423
1424 spin_lock_irq(&p->sighand->siglock);
1425 /* Re-check with the lock held. */
1426 if (!(p->signal->flags & SIGNAL_STOP_CONTINUED)) {
1427 spin_unlock_irq(&p->sighand->siglock);
1428 return 0;
1429 }
1430 if (!unlikely(wo->wo_flags & WNOWAIT))
1431 p->signal->flags &= ~SIGNAL_STOP_CONTINUED;
1432 uid = from_kuid_munged(current_user_ns(), task_uid(p));
1433 spin_unlock_irq(&p->sighand->siglock);
1434
1435 pid = task_pid_vnr(p);
1436 get_task_struct(p);
1437 read_unlock(&tasklist_lock);
1438 sched_annotate_sleep();
1439 if (wo->wo_rusage)
1440 getrusage(p, RUSAGE_BOTH, wo->wo_rusage);
1441 put_task_struct(p);
1442
1443 infop = wo->wo_info;
1444 if (!infop) {
1445 wo->wo_stat = 0xffff;
1446 } else {
1447 infop->cause = CLD_CONTINUED;
1448 infop->pid = pid;
1449 infop->uid = uid;
1450 infop->status = SIGCONT;
1451 }
1452 return pid;
1453}
1454
1455/*
1456 * Consider @p for a wait by @parent.
1457 *
1458 * -ECHILD should be in ->notask_error before the first call.
1459 * Returns nonzero for a final return, when we have unlocked tasklist_lock.
1460 * Returns zero if the search for a child should continue;
1461 * then ->notask_error is 0 if @p is an eligible child,
1462 * or still -ECHILD.
1463 */
1464static int wait_consider_task(struct wait_opts *wo, int ptrace,
1465 struct task_struct *p)
1466{
1467 /*
1468 * We can race with wait_task_zombie() from another thread.
1469 * Ensure that EXIT_ZOMBIE -> EXIT_DEAD/EXIT_TRACE transition
1470 * can't confuse the checks below.
1471 */
1472 int exit_state = READ_ONCE(p->exit_state);
1473 int ret;
1474
1475 if (unlikely(exit_state == EXIT_DEAD))
1476 return 0;
1477
1478 ret = eligible_child(wo, ptrace, p);
1479 if (!ret)
1480 return ret;
1481
1482 if (unlikely(exit_state == EXIT_TRACE)) {
1483 /*
1484 * ptrace == 0 means we are the natural parent. In this case
1485 * we should clear notask_error, debugger will notify us.
1486 */
1487 if (likely(!ptrace))
1488 wo->notask_error = 0;
1489 return 0;
1490 }
1491
1492 if (likely(!ptrace) && unlikely(p->ptrace)) {
1493 /*
1494 * If it is traced by its real parent's group, just pretend
1495 * the caller is ptrace_do_wait() and reap this child if it
1496 * is zombie.
1497 *
1498 * This also hides group stop state from real parent; otherwise
1499 * a single stop can be reported twice as group and ptrace stop.
1500 * If a ptracer wants to distinguish these two events for its
1501 * own children it should create a separate process which takes
1502 * the role of real parent.
1503 */
1504 if (!ptrace_reparented(p))
1505 ptrace = 1;
1506 }
1507
1508 /* slay zombie? */
1509 if (exit_state == EXIT_ZOMBIE) {
1510 /* we don't reap group leaders with subthreads */
1511 if (!delay_group_leader(p)) {
1512 /*
1513 * A zombie ptracee is only visible to its ptracer.
1514 * Notification and reaping will be cascaded to the
1515 * real parent when the ptracer detaches.
1516 */
1517 if (unlikely(ptrace) || likely(!p->ptrace))
1518 return wait_task_zombie(wo, p);
1519 }
1520
1521 /*
1522 * Allow access to stopped/continued state via zombie by
1523 * falling through. Clearing of notask_error is complex.
1524 *
1525 * When !@ptrace:
1526 *
1527 * If WEXITED is set, notask_error should naturally be
1528 * cleared. If not, subset of WSTOPPED|WCONTINUED is set,
1529 * so, if there are live subthreads, there are events to
1530 * wait for. If all subthreads are dead, it's still safe
1531 * to clear - this function will be called again in finite
1532 * amount time once all the subthreads are released and
1533 * will then return without clearing.
1534 *
1535 * When @ptrace:
1536 *
1537 * Stopped state is per-task and thus can't change once the
1538 * target task dies. Only continued and exited can happen.
1539 * Clear notask_error if WCONTINUED | WEXITED.
1540 */
1541 if (likely(!ptrace) || (wo->wo_flags & (WCONTINUED | WEXITED)))
1542 wo->notask_error = 0;
1543 } else {
1544 /*
1545 * @p is alive and it's gonna stop, continue or exit, so
1546 * there always is something to wait for.
1547 */
1548 wo->notask_error = 0;
1549 }
1550
1551 /*
1552 * Wait for stopped. Depending on @ptrace, different stopped state
1553 * is used and the two don't interact with each other.
1554 */
1555 ret = wait_task_stopped(wo, ptrace, p);
1556 if (ret)
1557 return ret;
1558
1559 /*
1560 * Wait for continued. There's only one continued state and the
1561 * ptracer can consume it which can confuse the real parent. Don't
1562 * use WCONTINUED from ptracer. You don't need or want it.
1563 */
1564 return wait_task_continued(wo, p);
1565}
1566
1567/*
1568 * Do the work of do_wait() for one thread in the group, @tsk.
1569 *
1570 * -ECHILD should be in ->notask_error before the first call.
1571 * Returns nonzero for a final return, when we have unlocked tasklist_lock.
1572 * Returns zero if the search for a child should continue; then
1573 * ->notask_error is 0 if there were any eligible children,
1574 * or still -ECHILD.
1575 */
1576static int do_wait_thread(struct wait_opts *wo, struct task_struct *tsk)
1577{
1578 struct task_struct *p;
1579
1580 list_for_each_entry(p, &tsk->children, sibling) {
1581 int ret = wait_consider_task(wo, 0, p);
1582
1583 if (ret)
1584 return ret;
1585 }
1586
1587 return 0;
1588}
1589
1590static int ptrace_do_wait(struct wait_opts *wo, struct task_struct *tsk)
1591{
1592 struct task_struct *p;
1593
1594 list_for_each_entry(p, &tsk->ptraced, ptrace_entry) {
1595 int ret = wait_consider_task(wo, 1, p);
1596
1597 if (ret)
1598 return ret;
1599 }
1600
1601 return 0;
1602}
1603
1604bool pid_child_should_wake(struct wait_opts *wo, struct task_struct *p)
1605{
1606 if (!eligible_pid(wo, p))
1607 return false;
1608
1609 if ((wo->wo_flags & __WNOTHREAD) && wo->child_wait.private != p->parent)
1610 return false;
1611
1612 return true;
1613}
1614
1615static int child_wait_callback(wait_queue_entry_t *wait, unsigned mode,
1616 int sync, void *key)
1617{
1618 struct wait_opts *wo = container_of(wait, struct wait_opts,
1619 child_wait);
1620 struct task_struct *p = key;
1621
1622 if (pid_child_should_wake(wo, p))
1623 return default_wake_function(wait, mode, sync, key);
1624
1625 return 0;
1626}
1627
1628void __wake_up_parent(struct task_struct *p, struct task_struct *parent)
1629{
1630 __wake_up_sync_key(&parent->signal->wait_chldexit,
1631 TASK_INTERRUPTIBLE, p);
1632}
1633
1634static bool is_effectively_child(struct wait_opts *wo, bool ptrace,
1635 struct task_struct *target)
1636{
1637 struct task_struct *parent =
1638 !ptrace ? target->real_parent : target->parent;
1639
1640 return current == parent || (!(wo->wo_flags & __WNOTHREAD) &&
1641 same_thread_group(current, parent));
1642}
1643
1644/*
1645 * Optimization for waiting on PIDTYPE_PID. No need to iterate through child
1646 * and tracee lists to find the target task.
1647 */
1648static int do_wait_pid(struct wait_opts *wo)
1649{
1650 bool ptrace;
1651 struct task_struct *target;
1652 int retval;
1653
1654 ptrace = false;
1655 target = pid_task(wo->wo_pid, PIDTYPE_TGID);
1656 if (target && is_effectively_child(wo, ptrace, target)) {
1657 retval = wait_consider_task(wo, ptrace, target);
1658 if (retval)
1659 return retval;
1660 }
1661
1662 ptrace = true;
1663 target = pid_task(wo->wo_pid, PIDTYPE_PID);
1664 if (target && target->ptrace &&
1665 is_effectively_child(wo, ptrace, target)) {
1666 retval = wait_consider_task(wo, ptrace, target);
1667 if (retval)
1668 return retval;
1669 }
1670
1671 return 0;
1672}
1673
1674long __do_wait(struct wait_opts *wo)
1675{
1676 long retval;
1677
1678 /*
1679 * If there is nothing that can match our criteria, just get out.
1680 * We will clear ->notask_error to zero if we see any child that
1681 * might later match our criteria, even if we are not able to reap
1682 * it yet.
1683 */
1684 wo->notask_error = -ECHILD;
1685 if ((wo->wo_type < PIDTYPE_MAX) &&
1686 (!wo->wo_pid || !pid_has_task(wo->wo_pid, wo->wo_type)))
1687 goto notask;
1688
1689 read_lock(&tasklist_lock);
1690
1691 if (wo->wo_type == PIDTYPE_PID) {
1692 retval = do_wait_pid(wo);
1693 if (retval)
1694 return retval;
1695 } else {
1696 struct task_struct *tsk = current;
1697
1698 do {
1699 retval = do_wait_thread(wo, tsk);
1700 if (retval)
1701 return retval;
1702
1703 retval = ptrace_do_wait(wo, tsk);
1704 if (retval)
1705 return retval;
1706
1707 if (wo->wo_flags & __WNOTHREAD)
1708 break;
1709 } while_each_thread(current, tsk);
1710 }
1711 read_unlock(&tasklist_lock);
1712
1713notask:
1714 retval = wo->notask_error;
1715 if (!retval && !(wo->wo_flags & WNOHANG))
1716 return -ERESTARTSYS;
1717
1718 return retval;
1719}
1720
1721static long do_wait(struct wait_opts *wo)
1722{
1723 int retval;
1724
1725 trace_sched_process_wait(wo->wo_pid);
1726
1727 init_waitqueue_func_entry(&wo->child_wait, child_wait_callback);
1728 wo->child_wait.private = current;
1729 add_wait_queue(&current->signal->wait_chldexit, &wo->child_wait);
1730
1731 do {
1732 set_current_state(TASK_INTERRUPTIBLE);
1733 retval = __do_wait(wo);
1734 if (retval != -ERESTARTSYS)
1735 break;
1736 if (signal_pending(current))
1737 break;
1738 schedule();
1739 } while (1);
1740
1741 __set_current_state(TASK_RUNNING);
1742 remove_wait_queue(&current->signal->wait_chldexit, &wo->child_wait);
1743 return retval;
1744}
1745
1746int kernel_waitid_prepare(struct wait_opts *wo, int which, pid_t upid,
1747 struct waitid_info *infop, int options,
1748 struct rusage *ru)
1749{
1750 unsigned int f_flags = 0;
1751 struct pid *pid = NULL;
1752 enum pid_type type;
1753
1754 if (options & ~(WNOHANG|WNOWAIT|WEXITED|WSTOPPED|WCONTINUED|
1755 __WNOTHREAD|__WCLONE|__WALL))
1756 return -EINVAL;
1757 if (!(options & (WEXITED|WSTOPPED|WCONTINUED)))
1758 return -EINVAL;
1759
1760 switch (which) {
1761 case P_ALL:
1762 type = PIDTYPE_MAX;
1763 break;
1764 case P_PID:
1765 type = PIDTYPE_PID;
1766 if (upid <= 0)
1767 return -EINVAL;
1768
1769 pid = find_get_pid(upid);
1770 break;
1771 case P_PGID:
1772 type = PIDTYPE_PGID;
1773 if (upid < 0)
1774 return -EINVAL;
1775
1776 if (upid)
1777 pid = find_get_pid(upid);
1778 else
1779 pid = get_task_pid(current, PIDTYPE_PGID);
1780 break;
1781 case P_PIDFD:
1782 type = PIDTYPE_PID;
1783 if (upid < 0)
1784 return -EINVAL;
1785
1786 pid = pidfd_get_pid(upid, &f_flags);
1787 if (IS_ERR(pid))
1788 return PTR_ERR(pid);
1789
1790 break;
1791 default:
1792 return -EINVAL;
1793 }
1794
1795 wo->wo_type = type;
1796 wo->wo_pid = pid;
1797 wo->wo_flags = options;
1798 wo->wo_info = infop;
1799 wo->wo_rusage = ru;
1800 if (f_flags & O_NONBLOCK)
1801 wo->wo_flags |= WNOHANG;
1802
1803 return 0;
1804}
1805
1806static long kernel_waitid(int which, pid_t upid, struct waitid_info *infop,
1807 int options, struct rusage *ru)
1808{
1809 struct wait_opts wo;
1810 long ret;
1811
1812 ret = kernel_waitid_prepare(&wo, which, upid, infop, options, ru);
1813 if (ret)
1814 return ret;
1815
1816 ret = do_wait(&wo);
1817 if (!ret && !(options & WNOHANG) && (wo.wo_flags & WNOHANG))
1818 ret = -EAGAIN;
1819
1820 put_pid(wo.wo_pid);
1821 return ret;
1822}
1823
1824SYSCALL_DEFINE5(waitid, int, which, pid_t, upid, struct siginfo __user *,
1825 infop, int, options, struct rusage __user *, ru)
1826{
1827 struct rusage r;
1828 struct waitid_info info = {.status = 0};
1829 long err = kernel_waitid(which, upid, &info, options, ru ? &r : NULL);
1830 int signo = 0;
1831
1832 if (err > 0) {
1833 signo = SIGCHLD;
1834 err = 0;
1835 if (ru && copy_to_user(ru, &r, sizeof(struct rusage)))
1836 return -EFAULT;
1837 }
1838 if (!infop)
1839 return err;
1840
1841 if (!user_write_access_begin(infop, sizeof(*infop)))
1842 return -EFAULT;
1843
1844 unsafe_put_user(signo, &infop->si_signo, Efault);
1845 unsafe_put_user(0, &infop->si_errno, Efault);
1846 unsafe_put_user(info.cause, &infop->si_code, Efault);
1847 unsafe_put_user(info.pid, &infop->si_pid, Efault);
1848 unsafe_put_user(info.uid, &infop->si_uid, Efault);
1849 unsafe_put_user(info.status, &infop->si_status, Efault);
1850 user_write_access_end();
1851 return err;
1852Efault:
1853 user_write_access_end();
1854 return -EFAULT;
1855}
1856
1857long kernel_wait4(pid_t upid, int __user *stat_addr, int options,
1858 struct rusage *ru)
1859{
1860 struct wait_opts wo;
1861 struct pid *pid = NULL;
1862 enum pid_type type;
1863 long ret;
1864
1865 if (options & ~(WNOHANG|WUNTRACED|WCONTINUED|
1866 __WNOTHREAD|__WCLONE|__WALL))
1867 return -EINVAL;
1868
1869 /* -INT_MIN is not defined */
1870 if (upid == INT_MIN)
1871 return -ESRCH;
1872
1873 if (upid == -1)
1874 type = PIDTYPE_MAX;
1875 else if (upid < 0) {
1876 type = PIDTYPE_PGID;
1877 pid = find_get_pid(-upid);
1878 } else if (upid == 0) {
1879 type = PIDTYPE_PGID;
1880 pid = get_task_pid(current, PIDTYPE_PGID);
1881 } else /* upid > 0 */ {
1882 type = PIDTYPE_PID;
1883 pid = find_get_pid(upid);
1884 }
1885
1886 wo.wo_type = type;
1887 wo.wo_pid = pid;
1888 wo.wo_flags = options | WEXITED;
1889 wo.wo_info = NULL;
1890 wo.wo_stat = 0;
1891 wo.wo_rusage = ru;
1892 ret = do_wait(&wo);
1893 put_pid(pid);
1894 if (ret > 0 && stat_addr && put_user(wo.wo_stat, stat_addr))
1895 ret = -EFAULT;
1896
1897 return ret;
1898}
1899
1900int kernel_wait(pid_t pid, int *stat)
1901{
1902 struct wait_opts wo = {
1903 .wo_type = PIDTYPE_PID,
1904 .wo_pid = find_get_pid(pid),
1905 .wo_flags = WEXITED,
1906 };
1907 int ret;
1908
1909 ret = do_wait(&wo);
1910 if (ret > 0 && wo.wo_stat)
1911 *stat = wo.wo_stat;
1912 put_pid(wo.wo_pid);
1913 return ret;
1914}
1915
1916SYSCALL_DEFINE4(wait4, pid_t, upid, int __user *, stat_addr,
1917 int, options, struct rusage __user *, ru)
1918{
1919 struct rusage r;
1920 long err = kernel_wait4(upid, stat_addr, options, ru ? &r : NULL);
1921
1922 if (err > 0) {
1923 if (ru && copy_to_user(ru, &r, sizeof(struct rusage)))
1924 return -EFAULT;
1925 }
1926 return err;
1927}
1928
1929#ifdef __ARCH_WANT_SYS_WAITPID
1930
1931/*
1932 * sys_waitpid() remains for compatibility. waitpid() should be
1933 * implemented by calling sys_wait4() from libc.a.
1934 */
1935SYSCALL_DEFINE3(waitpid, pid_t, pid, int __user *, stat_addr, int, options)
1936{
1937 return kernel_wait4(pid, stat_addr, options, NULL);
1938}
1939
1940#endif
1941
1942#ifdef CONFIG_COMPAT
1943COMPAT_SYSCALL_DEFINE4(wait4,
1944 compat_pid_t, pid,
1945 compat_uint_t __user *, stat_addr,
1946 int, options,
1947 struct compat_rusage __user *, ru)
1948{
1949 struct rusage r;
1950 long err = kernel_wait4(pid, stat_addr, options, ru ? &r : NULL);
1951 if (err > 0) {
1952 if (ru && put_compat_rusage(&r, ru))
1953 return -EFAULT;
1954 }
1955 return err;
1956}
1957
1958COMPAT_SYSCALL_DEFINE5(waitid,
1959 int, which, compat_pid_t, pid,
1960 struct compat_siginfo __user *, infop, int, options,
1961 struct compat_rusage __user *, uru)
1962{
1963 struct rusage ru;
1964 struct waitid_info info = {.status = 0};
1965 long err = kernel_waitid(which, pid, &info, options, uru ? &ru : NULL);
1966 int signo = 0;
1967 if (err > 0) {
1968 signo = SIGCHLD;
1969 err = 0;
1970 if (uru) {
1971 /* kernel_waitid() overwrites everything in ru */
1972 if (COMPAT_USE_64BIT_TIME)
1973 err = copy_to_user(uru, &ru, sizeof(ru));
1974 else
1975 err = put_compat_rusage(&ru, uru);
1976 if (err)
1977 return -EFAULT;
1978 }
1979 }
1980
1981 if (!infop)
1982 return err;
1983
1984 if (!user_write_access_begin(infop, sizeof(*infop)))
1985 return -EFAULT;
1986
1987 unsafe_put_user(signo, &infop->si_signo, Efault);
1988 unsafe_put_user(0, &infop->si_errno, Efault);
1989 unsafe_put_user(info.cause, &infop->si_code, Efault);
1990 unsafe_put_user(info.pid, &infop->si_pid, Efault);
1991 unsafe_put_user(info.uid, &infop->si_uid, Efault);
1992 unsafe_put_user(info.status, &infop->si_status, Efault);
1993 user_write_access_end();
1994 return err;
1995Efault:
1996 user_write_access_end();
1997 return -EFAULT;
1998}
1999#endif
2000
2001/*
2002 * This needs to be __function_aligned as GCC implicitly makes any
2003 * implementation of abort() cold and drops alignment specified by
2004 * -falign-functions=N.
2005 *
2006 * See https://gcc.gnu.org/bugzilla/show_bug.cgi?id=88345#c11
2007 */
2008__weak __function_aligned void abort(void)
2009{
2010 BUG();
2011
2012 /* if that doesn't kill us, halt */
2013 panic("Oops failed to kill thread");
2014}
2015EXPORT_SYMBOL(abort);