projects/linux·blob master · refs

1218 lines · 31.2 KiB

1// SPDX-License-Identifier: GPL-2.0
2#include <linux/anon_inodes.h>
3#include <linux/compat.h>
4#include <linux/exportfs.h>
5#include <linux/file.h>
6#include <linux/fs.h>
7#include <linux/cgroup.h>
8#include <linux/magic.h>
9#include <linux/mount.h>
10#include <linux/pid.h>
11#include <linux/pidfs.h>
12#include <linux/sched/signal.h>
13#include <linux/signal.h>
14#include <linux/pid_namespace.h>
15#include <linux/poll.h>
16#include <linux/proc_fs.h>
17#include <linux/proc_ns.h>
18#include <linux/pseudo_fs.h>
19#include <linux/ptrace.h>
20#include <linux/seq_file.h>
21#include <uapi/linux/pidfd.h>
22#include <linux/ipc_namespace.h>
23#include <linux/time_namespace.h>
24#include <linux/utsname.h>
25#include <net/net_namespace.h>
26#include <linux/coredump.h>
27#include <linux/rhashtable.h>
28#include <linux/llist.h>
29#include <linux/xattr.h>
30#include <linux/cookie.h>
31
32#include "internal.h"
33#include "mount.h"
34
35#define PIDFS_PID_DEAD ERR_PTR(-ESRCH)
36
37static struct kmem_cache *pidfs_attr_cachep __ro_after_init;
38
39static struct path pidfs_root_path = {};
40
41static struct simple_xattr_cache pidfs_xa_cache;
42
43void pidfs_get_root(struct path *path)
44{
45 *path = pidfs_root_path;
46 path_get(path);
47}
48
49enum pidfs_attr_mask_bits {
50 PIDFS_ATTR_BIT_EXIT = 0,
51 PIDFS_ATTR_BIT_COREDUMP = 1,
52};
53
54struct pidfs_anon_attr {
55 unsigned long attr_mask;
56 struct /* exit info */ {
57 __u64 cgroupid;
58 __s32 exit_code;
59 };
60 __u32 coredump_mask;
61 __u32 coredump_signal;
62 __u32 coredump_code;
63};
64
65static struct rhashtable pidfs_ino_ht;
66
67static const struct rhashtable_params pidfs_ino_ht_params = {
68 .key_offset = offsetof(struct pid, ino),
69 .key_len = sizeof(u64),
70 .head_offset = offsetof(struct pid, pidfs_hash),
71 .automatic_shrinking = true,
72};
73
74/*
75 * inode number handling
76 *
77 * On 64 bit nothing special happens. The 64bit number assigned
78 * to struct pid is the inode number.
79 *
80 * On 32 bit the 64 bit number assigned to struct pid is split
81 * into two 32 bit numbers. The lower 32 bits are used as the
82 * inode number and the upper 32 bits are used as the inode
83 * generation number.
84 *
85 * On 32 bit pidfs_ino() will return the lower 32 bit. When
86 * pidfs_ino() returns zero a wrap around happened. When a
87 * wraparound happens the 64 bit number will be incremented by 1
88 * so inode numbering starts at 1 again.
89 *
90 * On 64 bit comparing two pidfds is as simple as comparing
91 * inode numbers.
92 *
93 * When a wraparound happens on 32 bit multiple pidfds with the
94 * same inode number are likely to exist (This isn't a problem
95 * since before pidfs pidfds used the anonymous inode meaning
96 * all pidfds had the same inode number.). Userspace can
97 * reconstruct the 64 bit identifier by retrieving both the
98 * inode number and the inode generation number to compare or
99 * use file handles.
100 */
101struct pidfs_attr {
102 struct list_head xattrs;
103 union {
104 struct pidfs_anon_attr;
105 struct llist_node pidfs_llist;
106 };
107};
108
109#if BITS_PER_LONG == 32
110
111static DEFINE_SPINLOCK(pidfs_ino_lock);
112static u64 pidfs_ino_nr = 1;
113
114static inline unsigned long pidfs_ino(u64 ino)
115{
116 return lower_32_bits(ino);
117}
118
119/* On 32 bit the generation number are the upper 32 bits. */
120static inline u32 pidfs_gen(u64 ino)
121{
122 return upper_32_bits(ino);
123}
124
125static inline u64 pidfs_alloc_ino(void)
126{
127 u64 ino;
128
129 spin_lock(&pidfs_ino_lock);
130 if (pidfs_ino(pidfs_ino_nr) == 0)
131 pidfs_ino_nr++;
132 ino = pidfs_ino_nr++;
133 spin_unlock(&pidfs_ino_lock);
134 return ino;
135}
136
137#else
138
139/* On 64 bit simply return ino. */
140static inline unsigned long pidfs_ino(u64 ino)
141{
142 return ino;
143}
144
145/* On 64 bit the generation number is 0. */
146static inline u32 pidfs_gen(u64 ino)
147{
148 return 0;
149}
150
151DEFINE_COOKIE(pidfs_ino_cookie);
152
153static u64 pidfs_alloc_ino(void)
154{
155 u64 ino;
156
157 preempt_disable();
158 ino = gen_cookie_next(&pidfs_ino_cookie);
159 preempt_enable();
160
161 VFS_WARN_ON_ONCE(ino < 1);
162 return ino;
163}
164
165#endif
166
167void pidfs_prepare_pid(struct pid *pid)
168{
169 pid->stashed = NULL;
170 pid->attr = NULL;
171 pid->ino = 0;
172}
173
174int pidfs_add_pid(struct pid *pid)
175{
176 int ret;
177
178 pid->ino = pidfs_alloc_ino();
179 ret = rhashtable_insert_fast(&pidfs_ino_ht, &pid->pidfs_hash,
180 pidfs_ino_ht_params);
181 if (unlikely(ret))
182 pid->ino = 0;
183 return ret;
184}
185
186void pidfs_remove_pid(struct pid *pid)
187{
188 if (likely(pid->ino))
189 rhashtable_remove_fast(&pidfs_ino_ht, &pid->pidfs_hash,
190 pidfs_ino_ht_params);
191}
192
193static LLIST_HEAD(pidfs_free_list);
194
195static void pidfs_free_attr_work(struct work_struct *work)
196{
197 struct pidfs_attr *attr, *next;
198 struct llist_node *head;
199
200 head = llist_del_all(&pidfs_free_list);
201 llist_for_each_entry_safe(attr, next, head, pidfs_llist) {
202 simple_xattrs_free(&pidfs_xa_cache, &attr->xattrs, NULL);
203 kfree(attr);
204 }
205}
206
207static DECLARE_WORK(pidfs_free_work, pidfs_free_attr_work);
208
209void pidfs_free_pid(struct pid *pid)
210{
211 struct pidfs_attr *attr = pid->attr;
212
213 /*
214 * Any dentry must've been wiped from the pid by now.
215 * Otherwise there's a reference count bug.
216 */
217 VFS_WARN_ON_ONCE(pid->stashed);
218
219 /*
220 * This if an error occurred during e.g., task creation that
221 * causes us to never go through the exit path.
222 */
223 if (unlikely(!attr))
224 return;
225
226 /* This never had a pidfd created. */
227 if (IS_ERR(attr))
228 return;
229
230 if (likely(list_empty(&attr->xattrs)))
231 kfree(attr);
232 else if (llist_add(&attr->pidfs_llist, &pidfs_free_list))
233 schedule_work(&pidfs_free_work);
234}
235
236#ifdef CONFIG_PROC_FS
237/**
238 * pidfd_show_fdinfo - print information about a pidfd
239 * @m: proc fdinfo file
240 * @f: file referencing a pidfd
241 *
242 * Pid:
243 * This function will print the pid that a given pidfd refers to in the
244 * pid namespace of the procfs instance.
245 * If the pid namespace of the process is not a descendant of the pid
246 * namespace of the procfs instance 0 will be shown as its pid. This is
247 * similar to calling getppid() on a process whose parent is outside of
248 * its pid namespace.
249 *
250 * NSpid:
251 * If pid namespaces are supported then this function will also print
252 * the pid of a given pidfd refers to for all descendant pid namespaces
253 * starting from the current pid namespace of the instance, i.e. the
254 * Pid field and the first entry in the NSpid field will be identical.
255 * If the pid namespace of the process is not a descendant of the pid
256 * namespace of the procfs instance 0 will be shown as its first NSpid
257 * entry and no others will be shown.
258 * Note that this differs from the Pid and NSpid fields in
259 * /proc/<pid>/status where Pid and NSpid are always shown relative to
260 * the pid namespace of the procfs instance. The difference becomes
261 * obvious when sending around a pidfd between pid namespaces from a
262 * different branch of the tree, i.e. where no ancestral relation is
263 * present between the pid namespaces:
264 * - create two new pid namespaces ns1 and ns2 in the initial pid
265 * namespace (also take care to create new mount namespaces in the
266 * new pid namespace and mount procfs)
267 * - create a process with a pidfd in ns1
268 * - send pidfd from ns1 to ns2
269 * - read /proc/self/fdinfo/<pidfd> and observe that both Pid and NSpid
270 * have exactly one entry, which is 0
271 */
272static void pidfd_show_fdinfo(struct seq_file *m, struct file *f)
273{
274 struct pid *pid = pidfd_pid(f);
275 struct pid_namespace *ns;
276 pid_t nr = -1;
277
278 if (likely(pid_has_task(pid, PIDTYPE_PID))) {
279 ns = proc_pid_ns(file_inode(m->file)->i_sb);
280 nr = pid_nr_ns(pid, ns);
281 }
282
283 seq_put_decimal_ll(m, "Pid:\t", nr);
284
285#ifdef CONFIG_PID_NS
286 seq_put_decimal_ll(m, "\nNSpid:\t", nr);
287 if (nr > 0) {
288 int i;
289
290 /* If nr is non-zero it means that 'pid' is valid and that
291 * ns, i.e. the pid namespace associated with the procfs
292 * instance, is in the pid namespace hierarchy of pid.
293 * Start at one below the already printed level.
294 */
295 for (i = ns->level + 1; i <= pid->level; i++)
296 seq_put_decimal_ll(m, "\t", pid->numbers[i].nr);
297 }
298#endif
299 seq_putc(m, '\n');
300}
301#endif
302
303/*
304 * Poll support for process exit notification.
305 */
306static __poll_t pidfd_poll(struct file *file, struct poll_table_struct *pts)
307{
308 struct pid *pid = pidfd_pid(file);
309 struct task_struct *task;
310 __poll_t poll_flags = 0;
311
312 poll_wait(file, &pid->wait_pidfd, pts);
313 /*
314 * Don't wake waiters if the thread-group leader exited
315 * prematurely. They either get notified when the last subthread
316 * exits or not at all if one of the remaining subthreads execs
317 * and assumes the struct pid of the old thread-group leader.
318 */
319 guard(rcu)();
320 task = pid_task(pid, PIDTYPE_PID);
321 if (!task)
322 poll_flags = EPOLLIN | EPOLLRDNORM | EPOLLHUP;
323 else if (task->exit_state && !delay_group_leader(task))
324 poll_flags = EPOLLIN | EPOLLRDNORM;
325
326 return poll_flags;
327}
328
329static inline bool pid_in_current_pidns(const struct pid *pid)
330{
331 const struct pid_namespace *ns = task_active_pid_ns(current);
332
333 if (ns->level <= pid->level)
334 return pid->numbers[ns->level].ns == ns;
335
336 return false;
337}
338
339static __u32 pidfs_coredump_mask(enum task_dumpable dumpable)
340{
341 switch (dumpable) {
342 case TASK_DUMPABLE_OWNER:
343 return PIDFD_COREDUMP_USER;
344 case TASK_DUMPABLE_ROOT:
345 return PIDFD_COREDUMP_ROOT;
346 case TASK_DUMPABLE_OFF:
347 return PIDFD_COREDUMP_SKIP;
348 default:
349 WARN_ON_ONCE(true);
350 }
351
352 return 0;
353}
354
355/* This must be updated whenever a new flag is added */
356#define PIDFD_INFO_SUPPORTED (PIDFD_INFO_PID | \
357 PIDFD_INFO_CREDS | \
358 PIDFD_INFO_CGROUPID | \
359 PIDFD_INFO_EXIT | \
360 PIDFD_INFO_COREDUMP | \
361 PIDFD_INFO_SUPPORTED_MASK | \
362 PIDFD_INFO_COREDUMP_SIGNAL | \
363 PIDFD_INFO_COREDUMP_CODE)
364
365static long pidfd_info(struct file *file, unsigned int cmd, unsigned long arg)
366{
367 struct pidfd_info __user *uinfo = (struct pidfd_info __user *)arg;
368 struct task_struct *task __free(put_task) = NULL;
369 struct pid *pid = pidfd_pid(file);
370 size_t usize = _IOC_SIZE(cmd);
371 struct pidfd_info kinfo = {};
372 struct user_namespace *user_ns;
373 struct pidfs_attr *attr;
374 const struct cred *c;
375 __u64 mask;
376
377 BUILD_BUG_ON(sizeof(struct pidfd_info) != PIDFD_INFO_SIZE_VER3);
378
379 if (!uinfo)
380 return -EINVAL;
381 if (usize < PIDFD_INFO_SIZE_VER0)
382 return -EINVAL; /* First version, no smaller struct possible */
383
384 if (copy_from_user(&mask, &uinfo->mask, sizeof(mask)))
385 return -EFAULT;
386
387 /*
388 * Restrict information retrieval to tasks within the caller's pid
389 * namespace hierarchy.
390 */
391 if (!pid_in_current_pidns(pid))
392 return -EREMOTE;
393
394 attr = READ_ONCE(pid->attr);
395 if (mask & PIDFD_INFO_EXIT) {
396 if (test_bit(PIDFS_ATTR_BIT_EXIT, &attr->attr_mask)) {
397 smp_rmb();
398 kinfo.mask |= PIDFD_INFO_EXIT;
399#ifdef CONFIG_CGROUPS
400 kinfo.cgroupid = attr->cgroupid;
401 kinfo.mask |= PIDFD_INFO_CGROUPID;
402#endif
403 kinfo.exit_code = attr->exit_code;
404 }
405 }
406
407 if (mask & PIDFD_INFO_COREDUMP) {
408 if (test_bit(PIDFS_ATTR_BIT_COREDUMP, &attr->attr_mask)) {
409 smp_rmb();
410 kinfo.mask |= PIDFD_INFO_COREDUMP | PIDFD_INFO_COREDUMP_SIGNAL | PIDFD_INFO_COREDUMP_CODE;
411 kinfo.coredump_mask = attr->coredump_mask;
412 kinfo.coredump_signal = attr->coredump_signal;
413 kinfo.coredump_code = attr->coredump_code;
414 }
415 }
416
417 task = get_pid_task(pid, PIDTYPE_PID);
418 if (!task) {
419 /*
420 * If the task has already been reaped, only exit
421 * information is available
422 */
423 if (!(mask & PIDFD_INFO_EXIT))
424 return -ESRCH;
425
426 goto copy_out;
427 }
428
429 c = get_task_cred(task);
430 if (!c)
431 return -ESRCH;
432
433 if ((mask & PIDFD_INFO_COREDUMP) && !kinfo.coredump_mask) {
434 kinfo.coredump_mask = pidfs_coredump_mask(task_exec_state_get_dumpable(task));
435 kinfo.mask |= PIDFD_INFO_COREDUMP;
436 /* No coredump actually took place, so no coredump signal. */
437 }
438
439 /* Unconditionally return identifiers and credentials, the rest only on request */
440
441 user_ns = current_user_ns();
442 kinfo.ruid = from_kuid_munged(user_ns, c->uid);
443 kinfo.rgid = from_kgid_munged(user_ns, c->gid);
444 kinfo.euid = from_kuid_munged(user_ns, c->euid);
445 kinfo.egid = from_kgid_munged(user_ns, c->egid);
446 kinfo.suid = from_kuid_munged(user_ns, c->suid);
447 kinfo.sgid = from_kgid_munged(user_ns, c->sgid);
448 kinfo.fsuid = from_kuid_munged(user_ns, c->fsuid);
449 kinfo.fsgid = from_kgid_munged(user_ns, c->fsgid);
450 kinfo.mask |= PIDFD_INFO_CREDS;
451 put_cred(c);
452
453#ifdef CONFIG_CGROUPS
454 if (!kinfo.cgroupid) {
455 struct cgroup *cgrp;
456
457 rcu_read_lock();
458 cgrp = task_dfl_cgroup(task);
459 kinfo.cgroupid = cgroup_id(cgrp);
460 kinfo.mask |= PIDFD_INFO_CGROUPID;
461 rcu_read_unlock();
462 }
463#endif
464
465 /*
466 * Copy pid/tgid last, to reduce the chances the information might be
467 * stale. Note that it is not possible to ensure it will be valid as the
468 * task might return as soon as the copy_to_user finishes, but that's ok
469 * and userspace expects that might happen and can act accordingly, so
470 * this is just best-effort. What we can do however is checking that all
471 * the fields are set correctly, or return ESRCH to avoid providing
472 * incomplete information. */
473
474 kinfo.ppid = task_ppid_vnr(task);
475 kinfo.tgid = task_tgid_vnr(task);
476 kinfo.pid = task_pid_vnr(task);
477 kinfo.mask |= PIDFD_INFO_PID;
478
479 if (kinfo.pid == 0 || kinfo.tgid == 0)
480 return -ESRCH;
481
482copy_out:
483 if (mask & PIDFD_INFO_SUPPORTED_MASK) {
484 kinfo.mask |= PIDFD_INFO_SUPPORTED_MASK;
485 kinfo.supported_mask = PIDFD_INFO_SUPPORTED;
486 }
487
488 /* Are there bits in the return mask not present in PIDFD_INFO_SUPPORTED? */
489 WARN_ON_ONCE(~PIDFD_INFO_SUPPORTED & kinfo.mask);
490 /*
491 * If userspace and the kernel have the same struct size it can just
492 * be copied. If userspace provides an older struct, only the bits that
493 * userspace knows about will be copied. If userspace provides a new
494 * struct, only the bits that the kernel knows about will be copied.
495 */
496 return copy_struct_to_user(uinfo, usize, &kinfo, sizeof(kinfo), NULL);
497}
498
499static bool pidfs_ioctl_valid(unsigned int cmd)
500{
501 switch (cmd) {
502 case FS_IOC_GETVERSION:
503 case PIDFD_GET_CGROUP_NAMESPACE:
504 case PIDFD_GET_IPC_NAMESPACE:
505 case PIDFD_GET_MNT_NAMESPACE:
506 case PIDFD_GET_NET_NAMESPACE:
507 case PIDFD_GET_PID_FOR_CHILDREN_NAMESPACE:
508 case PIDFD_GET_TIME_NAMESPACE:
509 case PIDFD_GET_TIME_FOR_CHILDREN_NAMESPACE:
510 case PIDFD_GET_UTS_NAMESPACE:
511 case PIDFD_GET_USER_NAMESPACE:
512 case PIDFD_GET_PID_NAMESPACE:
513 return true;
514 }
515
516 /* Extensible ioctls require some more careful checks. */
517 switch (_IOC_NR(cmd)) {
518 case _IOC_NR(PIDFD_GET_INFO):
519 /*
520 * Try to prevent performing a pidfd ioctl when someone
521 * erronously mistook the file descriptor for a pidfd.
522 * This is not perfect but will catch most cases.
523 */
524 return extensible_ioctl_valid(cmd, PIDFD_GET_INFO, PIDFD_INFO_SIZE_VER0);
525 }
526
527 return false;
528}
529
530static long pidfd_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
531{
532 struct task_struct *task __free(put_task) = NULL;
533 struct nsproxy *nsp __free(put_nsproxy) = NULL;
534 struct ns_common *ns_common = NULL;
535 int error;
536
537 if (!pidfs_ioctl_valid(cmd))
538 return -ENOIOCTLCMD;
539
540 if (cmd == FS_IOC_GETVERSION) {
541 if (!arg)
542 return -EINVAL;
543
544 __u32 __user *argp = (__u32 __user *)arg;
545 return put_user(file_inode(file)->i_generation, argp);
546 }
547
548 /* Extensible IOCTL that does not open namespace FDs, take a shortcut */
549 if (_IOC_NR(cmd) == _IOC_NR(PIDFD_GET_INFO))
550 return pidfd_info(file, cmd, arg);
551
552 task = get_pid_task(pidfd_pid(file), PIDTYPE_PID);
553 if (!task)
554 return -ESRCH;
555
556 if (arg)
557 return -EINVAL;
558
559 /*
560 * We're trying to open a file descriptor to the namespace so perform a
561 * filesystem cred ptrace check. Hold @task's exec_update_lock for the
562 * duration of the ptrace check and the namespace lookup so that the
563 * credentials used for the access decision match those of @task at the
564 * time its namespace is read, preventing a concurrent execve() from
565 * swapping the task's credentials in between the check and the use. We
566 * mirror nsfs behavior.
567 */
568 error = down_read_killable(&task->signal->exec_update_lock);
569 if (error)
570 return error;
571
572 if (!ptrace_may_access(task, PTRACE_MODE_READ_FSCREDS)) {
573 error = -EACCES;
574 goto out_unlock;
575 }
576
577 scoped_guard(task_lock, task) {
578 nsp = task->nsproxy;
579 if (nsp)
580 get_nsproxy(nsp);
581 }
582 if (!nsp) {
583 error = -ESRCH; /* just pretend it didn't exist */
584 goto out_unlock;
585 }
586
587 switch (cmd) {
588 /* Namespaces that hang of nsproxy. */
589 case PIDFD_GET_CGROUP_NAMESPACE:
590#ifdef CONFIG_CGROUPS
591 if (!ns_ref_get(nsp->cgroup_ns))
592 break;
593 ns_common = to_ns_common(nsp->cgroup_ns);
594#endif
595 break;
596 case PIDFD_GET_IPC_NAMESPACE:
597#ifdef CONFIG_IPC_NS
598 if (!ns_ref_get(nsp->ipc_ns))
599 break;
600 ns_common = to_ns_common(nsp->ipc_ns);
601#endif
602 break;
603 case PIDFD_GET_MNT_NAMESPACE:
604 if (!ns_ref_get(nsp->mnt_ns))
605 break;
606 ns_common = to_ns_common(nsp->mnt_ns);
607 break;
608 case PIDFD_GET_NET_NAMESPACE:
609#ifdef CONFIG_NET_NS
610 if (!ns_ref_get(nsp->net_ns))
611 break;
612 ns_common = to_ns_common(nsp->net_ns);
613#endif
614 break;
615 case PIDFD_GET_PID_FOR_CHILDREN_NAMESPACE:
616#ifdef CONFIG_PID_NS
617 if (!ns_ref_get(nsp->pid_ns_for_children))
618 break;
619 ns_common = to_ns_common(nsp->pid_ns_for_children);
620#endif
621 break;
622 case PIDFD_GET_TIME_NAMESPACE:
623#ifdef CONFIG_TIME_NS
624 if (!ns_ref_get(nsp->time_ns))
625 break;
626 ns_common = to_ns_common(nsp->time_ns);
627#endif
628 break;
629 case PIDFD_GET_TIME_FOR_CHILDREN_NAMESPACE:
630#ifdef CONFIG_TIME_NS
631 if (!ns_ref_get(nsp->time_ns_for_children))
632 break;
633 ns_common = to_ns_common(nsp->time_ns_for_children);
634#endif
635 break;
636 case PIDFD_GET_UTS_NAMESPACE:
637#ifdef CONFIG_UTS_NS
638 if (!ns_ref_get(nsp->uts_ns))
639 break;
640 ns_common = to_ns_common(nsp->uts_ns);
641#endif
642 break;
643 /* Namespaces that don't hang of nsproxy. */
644 case PIDFD_GET_USER_NAMESPACE:
645#ifdef CONFIG_USER_NS
646 scoped_guard(rcu) {
647 struct user_namespace *user_ns;
648
649 user_ns = task_cred_xxx(task, user_ns);
650 if (ns_ref_get(user_ns))
651 ns_common = to_ns_common(user_ns);
652 }
653#endif
654 break;
655 case PIDFD_GET_PID_NAMESPACE:
656#ifdef CONFIG_PID_NS
657 scoped_guard(rcu) {
658 struct pid_namespace *pid_ns;
659
660 pid_ns = task_active_pid_ns(task);
661 if (ns_ref_get(pid_ns))
662 ns_common = to_ns_common(pid_ns);
663 }
664#endif
665 break;
666 default:
667 error = -ENOIOCTLCMD;
668 }
669
670 if (!error && !ns_common)
671 error = -EOPNOTSUPP;
672
673out_unlock:
674 up_read(&task->signal->exec_update_lock);
675 if (error)
676 return error;
677
678 /* open_namespace() unconditionally consumes the reference */
679 return open_namespace(ns_common);
680}
681
682#ifdef CONFIG_COMPAT
683static long pidfd_compat_ioctl(struct file *file, unsigned int cmd,
684 unsigned long arg)
685{
686 if (cmd == FS_IOC32_GETVERSION)
687 cmd = FS_IOC_GETVERSION;
688
689 return pidfd_ioctl(file, cmd, (unsigned long)compat_ptr(arg));
690}
691#endif
692
693static int pidfs_file_release(struct inode *inode, struct file *file)
694{
695 struct pid *pid = inode->i_private;
696 struct task_struct *task;
697
698 if (!(file->f_flags & PIDFD_AUTOKILL))
699 return 0;
700
701 guard(rcu)();
702 task = pid_task(pid, PIDTYPE_TGID);
703 if (!task)
704 return 0;
705
706 /* Not available for kthreads or user workers for now. */
707 if (WARN_ON_ONCE(task->flags & (PF_KTHREAD | PF_USER_WORKER)))
708 return 0;
709 do_send_sig_info(SIGKILL, SEND_SIG_PRIV, task, PIDTYPE_TGID);
710 return 0;
711}
712
713static const struct file_operations pidfs_file_operations = {
714 .release = pidfs_file_release,
715 .poll = pidfd_poll,
716#ifdef CONFIG_PROC_FS
717 .show_fdinfo = pidfd_show_fdinfo,
718#endif
719 .unlocked_ioctl = pidfd_ioctl,
720#ifdef CONFIG_COMPAT
721 .compat_ioctl = pidfd_compat_ioctl,
722#endif
723};
724
725struct pid *pidfd_pid(const struct file *file)
726{
727 if (file->f_op != &pidfs_file_operations)
728 return ERR_PTR(-EBADF);
729 return file_inode(file)->i_private;
730}
731
732/*
733 * We're called from release_task(). We know there's at least one
734 * reference to struct pid being held that won't be released until the
735 * task has been reaped which cannot happen until we're out of
736 * release_task().
737 *
738 * If this struct pid has at least once been referred to by a pidfd then
739 * pid->attr will be allocated. If not we mark the struct pid as dead so
740 * anyone who is trying to register it with pidfs will fail to do so.
741 * Otherwise we would hand out pidfs for reaped tasks without having
742 * exit information available.
743 *
744 * Worst case is that we've filled in the info and the pid gets freed
745 * right away in free_pid() when no one holds a pidfd anymore. Since
746 * pidfs_exit() currently is placed after exit_task_work() we know that
747 * it cannot be us aka the exiting task holding a pidfd to itself.
748 */
749void pidfs_exit(struct task_struct *tsk)
750{
751 struct pid *pid = task_pid(tsk);
752 struct pidfs_attr *attr;
753#ifdef CONFIG_CGROUPS
754 struct cgroup *cgrp;
755#endif
756
757 might_sleep();
758
759 /* Synchronize with pidfs_register_pid(). */
760 scoped_guard(spinlock_irq, &pid->wait_pidfd.lock) {
761 attr = pid->attr;
762 if (!attr) {
763 /*
764 * No one ever held a pidfd for this struct pid.
765 * Mark it as dead so no one can add a pidfs
766 * entry anymore. We're about to be reaped and
767 * so no exit information would be available.
768 */
769 pid->attr = PIDFS_PID_DEAD;
770 return;
771 }
772 }
773
774 /*
775 * If @pid->attr is set someone might still legitimately hold a
776 * pidfd to @pid or someone might concurrently still be getting
777 * a reference to an already stashed dentry from @pid->stashed.
778 * So defer cleaning @pid->attr until the last reference to @pid
779 * is put
780 */
781
782#ifdef CONFIG_CGROUPS
783 rcu_read_lock();
784 cgrp = task_dfl_cgroup(tsk);
785 attr->cgroupid = cgroup_id(cgrp);
786 rcu_read_unlock();
787#endif
788 attr->exit_code = tsk->exit_code;
789
790 /* Ensure that PIDFD_GET_INFO sees either all or nothing. */
791 smp_wmb();
792 set_bit(PIDFS_ATTR_BIT_EXIT, &attr->attr_mask);
793}
794
795#ifdef CONFIG_COREDUMP
796static void pidfs_coredump_pid(struct pid *pid,
797 const struct coredump_params *cprm)
798{
799 struct pidfs_attr *attr;
800
801 attr = READ_ONCE(pid->attr);
802
803 VFS_WARN_ON_ONCE(!attr);
804 VFS_WARN_ON_ONCE(attr == PIDFS_PID_DEAD);
805
806 /* Note how we were coredumped and that we coredumped. */
807 attr->coredump_mask = pidfs_coredump_mask(cprm->dumpable) |
808 PIDFD_COREDUMPED;
809 /* If coredumping is set to skip we should never end up here. */
810 VFS_WARN_ON_ONCE(attr->coredump_mask & PIDFD_COREDUMP_SKIP);
811 /* Expose the signal number and code that caused the coredump. */
812 attr->coredump_signal = cprm->siginfo->si_signo;
813 attr->coredump_code = cprm->siginfo->si_code;
814 smp_wmb();
815 set_bit(PIDFS_ATTR_BIT_COREDUMP, &attr->attr_mask);
816}
817
818void pidfs_coredump(const struct coredump_params *cprm)
819{
820 /* The dumping thread's pidfd reports the coredump as well. */
821 for (enum pid_type type = PIDTYPE_PID; type <= pids_last(cprm->pid); type++)
822 pidfs_coredump_pid(cprm->pid[type], cprm);
823}
824#endif
825
826static struct vfsmount *pidfs_mnt __ro_after_init;
827
828/*
829 * The vfs falls back to simple_setattr() if i_op->setattr() isn't
830 * implemented. Let's reject it completely until we have a clean
831 * permission concept for pidfds.
832 */
833static int pidfs_setattr(const struct mnt_idmap *idmap, struct dentry *dentry,
834 struct iattr *attr)
835{
836 return anon_inode_setattr(idmap, dentry, attr);
837}
838
839static int pidfs_getattr(const struct mnt_idmap *idmap, const struct path *path,
840 struct kstat *stat, u32 request_mask,
841 unsigned int query_flags)
842{
843 return anon_inode_getattr(idmap, path, stat, request_mask, query_flags);
844}
845
846static ssize_t pidfs_listxattr(struct dentry *dentry, char *buf, size_t size)
847{
848 struct inode *inode = d_inode(dentry);
849 struct pid *pid = inode->i_private;
850
851 return simple_xattr_list(inode, &pid->attr->xattrs, buf, size);
852}
853
854static const struct inode_operations pidfs_inode_operations = {
855 .getattr = pidfs_getattr,
856 .setattr = pidfs_setattr,
857 .listxattr = pidfs_listxattr,
858};
859
860static void pidfs_evict_inode(struct inode *inode)
861{
862 struct pid *pid = inode->i_private;
863
864 clear_inode(inode);
865 put_pid(pid);
866}
867
868static const struct super_operations pidfs_sops = {
869 .drop_inode = inode_just_drop,
870 .evict_inode = pidfs_evict_inode,
871 .statfs = simple_statfs,
872};
873
874/*
875 * 'lsof' has knowledge of out historical anon_inode use, and expects
876 * the pidfs dentry name to start with 'anon_inode'.
877 */
878static char *pidfs_dname(struct dentry *dentry, char *buffer, int buflen)
879{
880 return dynamic_dname(buffer, buflen, "anon_inode:[pidfd]");
881}
882
883const struct dentry_operations pidfs_dentry_operations = {
884 .d_dname = pidfs_dname,
885 .d_prune = stashed_dentry_prune,
886};
887
888static int pidfs_encode_fh(struct inode *inode, u32 *fh, int *max_len,
889 struct inode *parent)
890{
891 const struct pid *pid = inode->i_private;
892
893 if (*max_len < 2) {
894 *max_len = 2;
895 return FILEID_INVALID;
896 }
897
898 *max_len = 2;
899 *(u64 *)fh = pid->ino;
900 return FILEID_KERNFS;
901}
902
903/* Find a struct pid based on the inode number. */
904static struct pid *pidfs_ino_get_pid(u64 ino)
905{
906 struct pid *pid;
907 struct pidfs_attr *attr;
908
909 guard(rcu)();
910 pid = rhashtable_lookup(&pidfs_ino_ht, &ino, pidfs_ino_ht_params);
911 if (!pid)
912 return NULL;
913 attr = READ_ONCE(pid->attr);
914 if (IS_ERR_OR_NULL(attr))
915 return NULL;
916 if (test_bit(PIDFS_ATTR_BIT_EXIT, &attr->attr_mask))
917 return NULL;
918 /* Within our pid namespace hierarchy? */
919 if (pid_vnr(pid) == 0)
920 return NULL;
921 return get_pid(pid);
922}
923
924static struct dentry *pidfs_fh_to_dentry(struct super_block *sb,
925 struct fid *fid, int fh_len,
926 int fh_type)
927{
928 int ret;
929 u64 pid_ino;
930 struct path path;
931 struct pid *pid;
932
933 if (fh_len < 2)
934 return NULL;
935
936 switch (fh_type) {
937 case FILEID_KERNFS:
938 pid_ino = *(u64 *)fid;
939 break;
940 default:
941 return NULL;
942 }
943
944 pid = pidfs_ino_get_pid(pid_ino);
945 if (!pid)
946 return NULL;
947
948 ret = path_from_stashed(&pid->stashed, pidfs_mnt, pid, &path);
949 if (ret < 0)
950 return ERR_PTR(ret);
951
952 VFS_WARN_ON_ONCE(!pid->attr);
953
954 mntput(path.mnt);
955 return path.dentry;
956}
957
958static struct file *pidfs_dentry_open(const struct path *path,
959 unsigned int flags,
960 const struct cred *cred)
961{
962 struct file *file;
963
964 /* pidfds are always O_RDWR. */
965 file = dentry_open(path, flags | O_RDWR, cred);
966 /* do_dentry_open() strips O_EXCL and O_TRUNC. */
967 if (!IS_ERR(file))
968 file->f_flags |= flags & (PIDFD_THREAD | PIDFD_AUTOKILL);
969 return file;
970}
971
972/*
973 * Make sure that we reject any nonsensical flags that users pass via
974 * open_by_handle_at(). Note that PIDFD_THREAD is defined as O_EXCL, and
975 * PIDFD_NONBLOCK as O_NONBLOCK.
976 */
977#define VALID_FILE_HANDLE_OPEN_FLAGS \
978 (O_RDONLY | O_WRONLY | O_RDWR | O_NONBLOCK | O_CLOEXEC | O_EXCL)
979
980static int pidfs_export_permission(struct handle_to_path_ctx *ctx,
981 unsigned int oflags)
982{
983 if (oflags & ~(VALID_FILE_HANDLE_OPEN_FLAGS | O_LARGEFILE))
984 return -EINVAL;
985
986 /*
987 * pidfd_ino_get_pid() will verify that the struct pid is part
988 * of the caller's pid namespace hierarchy. No further
989 * permission checks are needed.
990 */
991 return 0;
992}
993
994static struct file *pidfs_export_open(const struct path *path, unsigned int oflags)
995{
996 /*
997 * Opening via file handle may never raise PIDFD_AUTOKILL. That can
998 * only be done at task creation!
999 */
1000 if (WARN_ON_ONCE(oflags & PIDFD_AUTOKILL))
1001 return ERR_PTR(-EINVAL);
1002 /* Clear O_LARGEFILE as open_by_handle_at() forces it. */
1003 return pidfs_dentry_open(path, oflags & ~O_LARGEFILE, current_cred());
1004}
1005
1006static const struct export_operations pidfs_export_operations = {
1007 .encode_fh = pidfs_encode_fh,
1008 .fh_to_dentry = pidfs_fh_to_dentry,
1009 .open = pidfs_export_open,
1010 .permission = pidfs_export_permission,
1011};
1012
1013static int pidfs_init_inode(struct inode *inode, void *data)
1014{
1015 const struct pid *pid = data;
1016
1017 inode->i_private = data;
1018 inode->i_flags |= S_PRIVATE | S_ANON_INODE;
1019 /* We allow to set xattrs. */
1020 inode->i_flags &= ~S_IMMUTABLE;
1021 inode->i_mode |= S_IRWXU;
1022 inode->i_op = &pidfs_inode_operations;
1023 inode->i_fop = &pidfs_file_operations;
1024 inode->i_ino = pidfs_ino(pid->ino);
1025 inode->i_generation = pidfs_gen(pid->ino);
1026 return 0;
1027}
1028
1029static void pidfs_put_data(void *data)
1030{
1031 struct pid *pid = data;
1032 put_pid(pid);
1033}
1034
1035/**
1036 * pidfs_register_pid_gfp - register a struct pid in pidfs with custom GFP
1037 * flags
1038 * @pid: pid to pin
1039 * @gfp: GFP flags for memory allocation
1040 *
1041 * Register a struct pid in pidfs with custom GFP flags.
1042 *
1043 * Return: On success zero, on error a negative error code is returned.
1044 */
1045int pidfs_register_pid_gfp(struct pid *pid, gfp_t gfp)
1046{
1047 struct pidfs_attr *new_attr __free(kfree) = NULL;
1048 struct pidfs_attr *attr;
1049
1050 might_sleep();
1051
1052 if (!pid)
1053 return 0;
1054
1055 attr = READ_ONCE(pid->attr);
1056 if (unlikely(attr == PIDFS_PID_DEAD))
1057 return PTR_ERR(PIDFS_PID_DEAD);
1058 if (attr)
1059 return 0;
1060
1061 new_attr = kmem_cache_zalloc(pidfs_attr_cachep, gfp);
1062 if (!new_attr)
1063 return -ENOMEM;
1064
1065 INIT_LIST_HEAD_RCU(&new_attr->xattrs);
1066
1067 /* Synchronize with pidfs_exit(). */
1068 guard(spinlock_irq)(&pid->wait_pidfd.lock);
1069
1070 attr = pid->attr;
1071 if (unlikely(attr == PIDFS_PID_DEAD))
1072 return PTR_ERR(PIDFS_PID_DEAD);
1073 if (unlikely(attr))
1074 return 0;
1075
1076 pid->attr = no_free_ptr(new_attr);
1077 return 0;
1078}
1079
1080/* Register the pids up to pid type @last of @pids in pidfs. */
1081int __pidfs_register_pids(struct pid *const *pids, enum pid_type last)
1082{
1083 if (WARN_ON_ONCE(last >= PIDTYPE_MAX))
1084 return -EINVAL;
1085
1086 for (enum pid_type type = PIDTYPE_PID; type <= last; type++) {
1087 int ret = pidfs_register_pid(pids[type]);
1088
1089 if (unlikely(ret))
1090 return ret;
1091 }
1092
1093 return 0;
1094}
1095
1096static struct dentry *pidfs_stash_dentry(struct dentry **stashed,
1097 struct dentry *dentry)
1098{
1099 int ret;
1100 struct pid *pid = d_inode(dentry)->i_private;
1101
1102 VFS_WARN_ON_ONCE(stashed != &pid->stashed);
1103
1104 ret = pidfs_register_pid(pid);
1105 if (ret)
1106 return ERR_PTR(ret);
1107
1108 return stash_dentry(stashed, dentry);
1109}
1110
1111static const struct stashed_operations pidfs_stashed_ops = {
1112 .stash_dentry = pidfs_stash_dentry,
1113 .init_inode = pidfs_init_inode,
1114 .put_data = pidfs_put_data,
1115};
1116
1117static int pidfs_xattr_get(const struct xattr_handler *handler,
1118 struct dentry *unused, struct inode *inode,
1119 const char *suffix, void *value, size_t size)
1120{
1121 struct pid *pid = inode->i_private;
1122 const char *name = xattr_full_name(handler, suffix);
1123
1124 return simple_xattr_get(&pidfs_xa_cache, &pid->attr->xattrs, name, value, size);
1125}
1126
1127static int pidfs_xattr_set(const struct xattr_handler *handler,
1128 const struct mnt_idmap *idmap, struct dentry *unused,
1129 struct inode *inode, const char *suffix,
1130 const void *value, size_t size, int flags)
1131{
1132 struct pid *pid = inode->i_private;
1133 const char *name = xattr_full_name(handler, suffix);
1134 struct simple_xattr *old_xattr;
1135
1136 /* Ensure we're the only one to set @attr->xattrs. */
1137 WARN_ON_ONCE(!inode_is_locked(inode));
1138
1139 old_xattr = simple_xattr_set(&pidfs_xa_cache, &pid->attr->xattrs, name, value, size, flags);
1140 if (IS_ERR(old_xattr))
1141 return PTR_ERR(old_xattr);
1142
1143 simple_xattr_free_rcu(old_xattr);
1144 return 0;
1145}
1146
1147static const struct xattr_handler pidfs_trusted_xattr_handler = {
1148 .prefix = XATTR_TRUSTED_PREFIX,
1149 .get = pidfs_xattr_get,
1150 .set = pidfs_xattr_set,
1151};
1152
1153static const struct xattr_handler *const pidfs_xattr_handlers[] = {
1154 &pidfs_trusted_xattr_handler,
1155 NULL
1156};
1157
1158static int pidfs_init_fs_context(struct fs_context *fc)
1159{
1160 struct pseudo_fs_context *ctx;
1161
1162 ctx = init_pseudo(fc, PID_FS_MAGIC);
1163 if (!ctx)
1164 return -ENOMEM;
1165
1166 ctx->s_d_flags |= DCACHE_DONTCACHE;
1167 ctx->ops = &pidfs_sops;
1168 ctx->eops = &pidfs_export_operations;
1169 ctx->dops = &pidfs_dentry_operations;
1170 ctx->xattr = pidfs_xattr_handlers;
1171 fc->s_fs_info = (void *)&pidfs_stashed_ops;
1172 return 0;
1173}
1174
1175static struct file_system_type pidfs_type = {
1176 .name = "pidfs",
1177 .init_fs_context = pidfs_init_fs_context,
1178 .kill_sb = kill_anon_super,
1179};
1180
1181struct file *pidfs_alloc_file(struct pid *pid, unsigned int flags)
1182{
1183 struct path path __free(path_put) = {};
1184 int ret;
1185
1186 /*
1187 * Ensure that internal pidfd flags don't overlap with each
1188 * other or with uapi pidfd flags.
1189 */
1190 BUILD_BUG_ON(hweight32(PIDFD_THREAD | PIDFD_NONBLOCK |
1191 PIDFD_STALE | PIDFD_AUTOKILL) != 4);
1192
1193 ret = path_from_stashed(&pid->stashed, pidfs_mnt, get_pid(pid), &path);
1194 if (ret < 0)
1195 return ERR_PTR(ret);
1196
1197 VFS_WARN_ON_ONCE(!pid->attr);
1198
1199 flags &= ~PIDFD_STALE;
1200 return pidfs_dentry_open(&path, flags, current_cred());
1201}
1202
1203void __init pidfs_init(void)
1204{
1205 if (rhashtable_init(&pidfs_ino_ht, &pidfs_ino_ht_params))
1206 panic("Failed to initialize pidfs hashtable");
1207
1208 pidfs_attr_cachep = kmem_cache_create("pidfs_attr_cache", sizeof(struct pidfs_attr), 0,
1209 (SLAB_HWCACHE_ALIGN | SLAB_RECLAIM_ACCOUNT |
1210 SLAB_ACCOUNT | SLAB_PANIC), NULL);
1211
1212 pidfs_mnt = kern_mount(&pidfs_type);
1213 if (IS_ERR(pidfs_mnt))
1214 panic("Failed to mount pidfs pseudo filesystem");
1215
1216 pidfs_root_path.mnt = pidfs_mnt;
1217 pidfs_root_path.dentry = pidfs_mnt->mnt_root;
1218}