projects/linux·blob master · refs

1106 lines · 28.0 KiB

1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * fs/kernfs/file.c - kernfs file implementation
4 *
5 * Copyright (c) 2001-3 Patrick Mochel
6 * Copyright (c) 2007 SUSE Linux Products GmbH
7 * Copyright (c) 2007, 2013 Tejun Heo <[email protected]>
8 */
9
10#include <linux/fs.h>
11#include <linux/seq_file.h>
12#include <linux/slab.h>
13#include <linux/poll.h>
14#include <linux/pagemap.h>
15#include <linux/sched/mm.h>
16#include <linux/fsnotify.h>
17#include <linux/uio.h>
18
19#include "kernfs-internal.h"
20
21struct kernfs_open_node {
22 struct rcu_head rcu_head;
23 atomic_t event;
24 wait_queue_head_t poll;
25 struct list_head files; /* goes through kernfs_open_file.list */
26 unsigned int nr_mmapped;
27 unsigned int nr_to_release;
28};
29
30/*
31 * kernfs_notify() may be called from any context and bounces notifications
32 * through a work item. To minimize space overhead in kernfs_node, the
33 * pending queue is implemented as a singly linked list of kernfs_nodes.
34 * The list is terminated with the self pointer so that whether a
35 * kernfs_node is on the list or not can be determined by testing the next
36 * pointer for %NULL.
37 */
38#define KERNFS_NOTIFY_EOL ((void *)&kernfs_notify_list)
39
40static DEFINE_SPINLOCK(kernfs_notify_lock);
41static struct kernfs_node *kernfs_notify_list = KERNFS_NOTIFY_EOL;
42
43/* Compatibility wrappers - use the common hashed node lock */
44static inline struct mutex *kernfs_open_file_mutex_ptr(struct kernfs_node *kn)
45{
46 return kernfs_node_lock_ptr(kn);
47}
48
49static inline struct mutex *kernfs_open_file_mutex_lock(struct kernfs_node *kn)
50{
51 return kernfs_node_lock(kn);
52}
53
54/**
55 * of_on - Get the kernfs_open_node of the specified kernfs_open_file
56 * @of: target kernfs_open_file
57 *
58 * Return: the kernfs_open_node of the kernfs_open_file
59 */
60static struct kernfs_open_node *of_on(struct kernfs_open_file *of)
61{
62 return rcu_dereference_protected(of->kn->attr.open,
63 !list_empty(&of->list));
64}
65
66/* Get active reference to kernfs node for an open file */
67static struct kernfs_open_file *kernfs_get_active_of(struct kernfs_open_file *of)
68{
69 /* Skip if file was already released */
70 if (unlikely(of->released))
71 return NULL;
72
73 if (!kernfs_get_active(of->kn))
74 return NULL;
75
76 return of;
77}
78
79static void kernfs_put_active_of(struct kernfs_open_file *of)
80{
81 return kernfs_put_active(of->kn);
82}
83
84/**
85 * kernfs_deref_open_node_locked - Get kernfs_open_node corresponding to @kn
86 *
87 * @kn: target kernfs_node.
88 *
89 * Fetch and return ->attr.open of @kn when caller holds the
90 * kernfs_open_file_mutex_ptr(kn).
91 *
92 * Update of ->attr.open happens under kernfs_open_file_mutex_ptr(kn). So when
93 * the caller guarantees that this mutex is being held, other updaters can't
94 * change ->attr.open and this means that we can safely deref ->attr.open
95 * outside RCU read-side critical section.
96 *
97 * The caller needs to make sure that kernfs_open_file_mutex is held.
98 *
99 * Return: @kn->attr.open when kernfs_open_file_mutex is held.
100 */
101static struct kernfs_open_node *
102kernfs_deref_open_node_locked(struct kernfs_node *kn)
103{
104 return rcu_dereference_protected(kn->attr.open,
105 lockdep_is_held(kernfs_open_file_mutex_ptr(kn)));
106}
107
108static struct kernfs_open_file *kernfs_of(struct file *file)
109{
110 return ((struct seq_file *)file->private_data)->private;
111}
112
113/*
114 * Determine the kernfs_ops for the given kernfs_node. This function must
115 * be called while holding an active reference.
116 */
117static const struct kernfs_ops *kernfs_ops(struct kernfs_node *kn)
118{
119 if (kn->flags & KERNFS_LOCKDEP)
120 lockdep_assert_held(kn);
121 return kn->attr.ops;
122}
123
124/*
125 * As kernfs_seq_stop() is also called after kernfs_seq_start() or
126 * kernfs_seq_next() failure, it needs to distinguish whether it's stopping
127 * a seq_file iteration which is fully initialized with an active reference
128 * or an aborted kernfs_seq_start() due to get_active failure. The
129 * position pointer is the only context for each seq_file iteration and
130 * thus the stop condition should be encoded in it. As the return value is
131 * directly visible to userland, ERR_PTR(-ENODEV) is the only acceptable
132 * choice to indicate get_active failure.
133 *
134 * Unfortunately, this is complicated due to the optional custom seq_file
135 * operations which may return ERR_PTR(-ENODEV) too. kernfs_seq_stop()
136 * can't distinguish whether ERR_PTR(-ENODEV) is from get_active failure or
137 * custom seq_file operations and thus can't decide whether put_active
138 * should be performed or not only on ERR_PTR(-ENODEV).
139 *
140 * This is worked around by factoring out the custom seq_stop() and
141 * put_active part into kernfs_seq_stop_active(), skipping it from
142 * kernfs_seq_stop() if ERR_PTR(-ENODEV) while invoking it directly after
143 * custom seq_file operations fail with ERR_PTR(-ENODEV) - this ensures
144 * that kernfs_seq_stop_active() is skipped only after get_active failure.
145 */
146static void kernfs_seq_stop_active(struct seq_file *sf, void *v)
147{
148 struct kernfs_open_file *of = sf->private;
149 const struct kernfs_ops *ops = kernfs_ops(of->kn);
150
151 if (ops->seq_stop)
152 ops->seq_stop(sf, v);
153 kernfs_put_active_of(of);
154}
155
156static void *kernfs_seq_start(struct seq_file *sf, loff_t *ppos)
157{
158 struct kernfs_open_file *of = sf->private;
159 const struct kernfs_ops *ops;
160
161 /*
162 * @of->mutex nests outside active ref and is primarily to ensure that
163 * the ops aren't called concurrently for the same open file.
164 */
165 mutex_lock(&of->mutex);
166 if (!kernfs_get_active_of(of))
167 return ERR_PTR(-ENODEV);
168
169 ops = kernfs_ops(of->kn);
170 if (ops->seq_start) {
171 void *next = ops->seq_start(sf, ppos);
172 /* see the comment above kernfs_seq_stop_active() */
173 if (next == ERR_PTR(-ENODEV))
174 kernfs_seq_stop_active(sf, next);
175 return next;
176 }
177 return single_start(sf, ppos);
178}
179
180static void *kernfs_seq_next(struct seq_file *sf, void *v, loff_t *ppos)
181{
182 struct kernfs_open_file *of = sf->private;
183 const struct kernfs_ops *ops = kernfs_ops(of->kn);
184
185 if (ops->seq_next) {
186 void *next = ops->seq_next(sf, v, ppos);
187 /* see the comment above kernfs_seq_stop_active() */
188 if (next == ERR_PTR(-ENODEV))
189 kernfs_seq_stop_active(sf, next);
190 return next;
191 } else {
192 /*
193 * The same behavior and code as single_open(), always
194 * terminate after the initial read.
195 */
196 ++*ppos;
197 return NULL;
198 }
199}
200
201static void kernfs_seq_stop(struct seq_file *sf, void *v)
202{
203 struct kernfs_open_file *of = sf->private;
204
205 if (v != ERR_PTR(-ENODEV))
206 kernfs_seq_stop_active(sf, v);
207 mutex_unlock(&of->mutex);
208}
209
210static int kernfs_seq_show(struct seq_file *sf, void *v)
211{
212 struct kernfs_open_file *of = sf->private;
213
214 of->event = atomic_read(&of_on(of)->event);
215
216 return of->kn->attr.ops->seq_show(sf, v);
217}
218
219static const struct seq_operations kernfs_seq_ops = {
220 .start = kernfs_seq_start,
221 .next = kernfs_seq_next,
222 .stop = kernfs_seq_stop,
223 .show = kernfs_seq_show,
224};
225
226/*
227 * As reading a bin file can have side-effects, the exact offset and bytes
228 * specified in read(2) call should be passed to the read callback making
229 * it difficult to use seq_file. Implement simplistic custom buffering for
230 * bin files.
231 */
232static ssize_t kernfs_file_read_iter(struct kiocb *iocb, struct iov_iter *iter)
233{
234 struct kernfs_open_file *of = kernfs_of(iocb->ki_filp);
235 ssize_t len = min_t(size_t, iov_iter_count(iter), PAGE_SIZE);
236 const struct kernfs_ops *ops;
237 char *buf;
238
239 buf = of->prealloc_buf;
240 if (buf)
241 mutex_lock(&of->prealloc_mutex);
242 else
243 buf = kmalloc(len, GFP_KERNEL);
244 if (!buf)
245 return -ENOMEM;
246
247 /*
248 * @of->mutex nests outside active ref and is used both to ensure that
249 * the ops aren't called concurrently for the same open file.
250 */
251 mutex_lock(&of->mutex);
252 if (!kernfs_get_active_of(of)) {
253 len = -ENODEV;
254 mutex_unlock(&of->mutex);
255 goto out_free;
256 }
257
258 of->event = atomic_read(&of_on(of)->event);
259
260 ops = kernfs_ops(of->kn);
261 if (ops->read)
262 len = ops->read(of, buf, len, iocb->ki_pos);
263 else
264 len = -EINVAL;
265
266 kernfs_put_active_of(of);
267 mutex_unlock(&of->mutex);
268
269 if (len < 0)
270 goto out_free;
271
272 if (copy_to_iter(buf, len, iter) != len) {
273 len = -EFAULT;
274 goto out_free;
275 }
276
277 iocb->ki_pos += len;
278
279 out_free:
280 if (buf == of->prealloc_buf)
281 mutex_unlock(&of->prealloc_mutex);
282 else
283 kfree(buf);
284 return len;
285}
286
287static ssize_t kernfs_fop_read_iter(struct kiocb *iocb, struct iov_iter *iter)
288{
289 if (kernfs_of(iocb->ki_filp)->kn->flags & KERNFS_HAS_SEQ_SHOW)
290 return seq_read_iter(iocb, iter);
291 return kernfs_file_read_iter(iocb, iter);
292}
293
294/*
295 * Copy data in from userland and pass it to the matching kernfs write
296 * operation.
297 *
298 * There is no easy way for us to know if userspace is only doing a partial
299 * write, so we don't support them. We expect the entire buffer to come on
300 * the first write. Hint: if you're writing a value, first read the file,
301 * modify only the value you're changing, then write entire buffer
302 * back.
303 */
304static ssize_t kernfs_fop_write_iter(struct kiocb *iocb, struct iov_iter *iter)
305{
306 struct kernfs_open_file *of = kernfs_of(iocb->ki_filp);
307 ssize_t len = iov_iter_count(iter);
308 const struct kernfs_ops *ops;
309 char *buf;
310
311 if (of->atomic_write_len) {
312 if (len > of->atomic_write_len)
313 return -E2BIG;
314 } else {
315 len = min_t(size_t, len, PAGE_SIZE);
316 }
317
318 buf = of->prealloc_buf;
319 if (buf)
320 mutex_lock(&of->prealloc_mutex);
321 else
322 buf = kmalloc(len + 1, GFP_KERNEL);
323 if (!buf)
324 return -ENOMEM;
325
326 if (copy_from_iter(buf, len, iter) != len) {
327 len = -EFAULT;
328 goto out_free;
329 }
330 buf[len] = '\0'; /* guarantee string termination */
331
332 /*
333 * @of->mutex nests outside active ref and is used both to ensure that
334 * the ops aren't called concurrently for the same open file.
335 */
336 mutex_lock(&of->mutex);
337 if (!kernfs_get_active_of(of)) {
338 mutex_unlock(&of->mutex);
339 len = -ENODEV;
340 goto out_free;
341 }
342
343 ops = kernfs_ops(of->kn);
344 if (ops->write)
345 len = ops->write(of, buf, len, iocb->ki_pos);
346 else
347 len = -EINVAL;
348
349 kernfs_put_active_of(of);
350 mutex_unlock(&of->mutex);
351
352 if (len > 0)
353 iocb->ki_pos += len;
354
355out_free:
356 if (buf == of->prealloc_buf)
357 mutex_unlock(&of->prealloc_mutex);
358 else
359 kfree(buf);
360 return len;
361}
362
363static void kernfs_vma_open(struct vm_area_struct *vma)
364{
365 struct file *file = vma->vm_file;
366 struct kernfs_open_file *of = kernfs_of(file);
367
368 if (!of->vm_ops)
369 return;
370
371 if (!kernfs_get_active_of(of))
372 return;
373
374 if (of->vm_ops->open)
375 of->vm_ops->open(vma);
376
377 kernfs_put_active_of(of);
378}
379
380static vm_fault_t kernfs_vma_fault(struct vm_fault *vmf)
381{
382 struct file *file = vmf->vma->vm_file;
383 struct kernfs_open_file *of = kernfs_of(file);
384 vm_fault_t ret;
385
386 if (!of->vm_ops)
387 return VM_FAULT_SIGBUS;
388
389 if (!kernfs_get_active_of(of))
390 return VM_FAULT_SIGBUS;
391
392 ret = VM_FAULT_SIGBUS;
393 if (of->vm_ops->fault)
394 ret = of->vm_ops->fault(vmf);
395
396 kernfs_put_active_of(of);
397 return ret;
398}
399
400static vm_fault_t kernfs_vma_page_mkwrite(struct vm_fault *vmf)
401{
402 struct file *file = vmf->vma->vm_file;
403 struct kernfs_open_file *of = kernfs_of(file);
404 vm_fault_t ret;
405
406 if (!of->vm_ops)
407 return VM_FAULT_SIGBUS;
408
409 if (!kernfs_get_active_of(of))
410 return VM_FAULT_SIGBUS;
411
412 ret = 0;
413 if (of->vm_ops->page_mkwrite)
414 ret = of->vm_ops->page_mkwrite(vmf);
415 else
416 file_update_time(file);
417
418 kernfs_put_active_of(of);
419 return ret;
420}
421
422static int kernfs_vma_access(struct vm_area_struct *vma, unsigned long addr,
423 void *buf, int len, int write)
424{
425 struct file *file = vma->vm_file;
426 struct kernfs_open_file *of = kernfs_of(file);
427 int ret;
428
429 if (!of->vm_ops)
430 return -EINVAL;
431
432 if (!kernfs_get_active_of(of))
433 return -EINVAL;
434
435 ret = -EINVAL;
436 if (of->vm_ops->access)
437 ret = of->vm_ops->access(vma, addr, buf, len, write);
438
439 kernfs_put_active_of(of);
440 return ret;
441}
442
443static const struct vm_operations_struct kernfs_vm_ops = {
444 .open = kernfs_vma_open,
445 .fault = kernfs_vma_fault,
446 .page_mkwrite = kernfs_vma_page_mkwrite,
447 .access = kernfs_vma_access,
448};
449
450static int kernfs_fop_mmap(struct file *file, struct vm_area_struct *vma)
451{
452 struct kernfs_open_file *of = kernfs_of(file);
453 const struct kernfs_ops *ops;
454 int rc;
455
456 /*
457 * mmap path and of->mutex are prone to triggering spurious lockdep
458 * warnings and we don't want to add spurious locking dependency
459 * between the two. Check whether mmap is actually implemented
460 * without grabbing @of->mutex by testing HAS_MMAP flag. See the
461 * comment in kernfs_fop_open() for more details.
462 */
463 if (!(of->kn->flags & KERNFS_HAS_MMAP))
464 return -ENODEV;
465
466 mutex_lock(&of->mutex);
467
468 rc = -ENODEV;
469 if (!kernfs_get_active_of(of))
470 goto out_unlock;
471
472 ops = kernfs_ops(of->kn);
473 rc = ops->mmap(of, vma);
474 if (rc)
475 goto out_put;
476
477 /*
478 * PowerPC's pci_mmap of legacy_mem uses shmem_zero_setup()
479 * to satisfy versions of X which crash if the mmap fails: that
480 * substitutes a new vm_file, and we don't then want bin_vm_ops.
481 */
482 if (vma->vm_file != file)
483 goto out_put;
484
485 rc = -EINVAL;
486 if (of->mmapped && of->vm_ops != vma->vm_ops)
487 goto out_put;
488
489 /*
490 * It is not possible to successfully wrap close.
491 * So error if someone is trying to use close.
492 */
493 if (vma->vm_ops && vma->vm_ops->close)
494 goto out_put;
495
496 rc = 0;
497 if (!of->mmapped) {
498 of->mmapped = true;
499 of_on(of)->nr_mmapped++;
500 of->vm_ops = vma->vm_ops;
501 }
502 vma->vm_ops = &kernfs_vm_ops;
503out_put:
504 kernfs_put_active_of(of);
505out_unlock:
506 mutex_unlock(&of->mutex);
507
508 return rc;
509}
510
511/**
512 * kernfs_get_open_node - get or create kernfs_open_node
513 * @kn: target kernfs_node
514 * @of: kernfs_open_file for this instance of open
515 *
516 * If @kn->attr.open exists, increment its reference count; otherwise,
517 * create one. @of is chained to the files list.
518 *
519 * Locking:
520 * Kernel thread context (may sleep).
521 *
522 * Return:
523 * %0 on success, -errno on failure.
524 */
525static int kernfs_get_open_node(struct kernfs_node *kn,
526 struct kernfs_open_file *of)
527{
528 struct kernfs_open_node *on, *new_on = NULL;
529 struct mutex *mutex;
530
531 /*
532 * Peek without the mutex: if nothing has this open, we will need a
533 * node and can allocate before taking a mutex shared by every node
534 * hashing to it.
535 */
536 if (!rcu_access_pointer(kn->attr.open))
537 new_on = kzalloc_obj(*new_on);
538
539 mutex = kernfs_open_file_mutex_lock(kn);
540 on = kernfs_deref_open_node_locked(kn);
541
542 if (!on) {
543 /* not there, initialize a new one */
544 on = new_on;
545 new_on = NULL;
546 if (!on) {
547 /* the peek raced; rare, so allocate here */
548 on = kzalloc_obj(*on);
549 if (!on) {
550 mutex_unlock(mutex);
551 return -ENOMEM;
552 }
553 }
554 atomic_set(&on->event, 1);
555 init_waitqueue_head(&on->poll);
556 INIT_LIST_HEAD(&on->files);
557 rcu_assign_pointer(kn->attr.open, on);
558 }
559
560 list_add_tail(&of->list, &on->files);
561 if (kn->flags & KERNFS_HAS_RELEASE)
562 on->nr_to_release++;
563
564 mutex_unlock(mutex);
565 kfree(new_on);
566 return 0;
567}
568
569/**
570 * kernfs_unlink_open_file - Unlink @of from @kn.
571 *
572 * @kn: target kernfs_node
573 * @of: associated kernfs_open_file
574 * @open_failed: ->open() failed, cancel ->release()
575 *
576 * Unlink @of from list of @kn's associated open files. If list of
577 * associated open files becomes empty, disassociate and free
578 * kernfs_open_node.
579 *
580 * LOCKING:
581 * None.
582 */
583static void kernfs_unlink_open_file(struct kernfs_node *kn,
584 struct kernfs_open_file *of,
585 bool open_failed)
586{
587 struct kernfs_open_node *on;
588 struct mutex *mutex;
589
590 mutex = kernfs_open_file_mutex_lock(kn);
591
592 on = kernfs_deref_open_node_locked(kn);
593 if (!on) {
594 mutex_unlock(mutex);
595 return;
596 }
597
598 if (of) {
599 if (kn->flags & KERNFS_HAS_RELEASE) {
600 WARN_ON_ONCE(of->released == open_failed);
601 if (open_failed)
602 on->nr_to_release--;
603 }
604 if (of->mmapped)
605 on->nr_mmapped--;
606 list_del(&of->list);
607 }
608
609 if (list_empty(&on->files)) {
610 rcu_assign_pointer(kn->attr.open, NULL);
611 kfree_rcu(on, rcu_head);
612 }
613
614 mutex_unlock(mutex);
615}
616
617static int kernfs_fop_open(struct inode *inode, struct file *file)
618{
619 struct kernfs_node *kn = inode->i_private;
620 struct kernfs_root *root = kernfs_root(kn);
621 const struct kernfs_ops *ops;
622 struct kernfs_open_file *of;
623 bool has_read, has_write, has_mmap;
624 int error = -EACCES;
625
626 if (!kernfs_get_active(kn))
627 return -ENODEV;
628
629 ops = kernfs_ops(kn);
630
631 has_read = ops->seq_show || ops->read || ops->mmap;
632 has_write = ops->write || ops->mmap;
633 has_mmap = ops->mmap;
634
635 /* see the flag definition for details */
636 if (root->flags & KERNFS_ROOT_EXTRA_OPEN_PERM_CHECK) {
637 if ((file->f_mode & FMODE_WRITE) &&
638 (!(inode->i_mode & S_IWUGO) || !has_write))
639 goto err_out;
640
641 if ((file->f_mode & FMODE_READ) &&
642 (!(inode->i_mode & S_IRUGO) || !has_read))
643 goto err_out;
644 }
645
646 /* allocate a kernfs_open_file for the file */
647 error = -ENOMEM;
648 of = kzalloc_obj(struct kernfs_open_file);
649 if (!of)
650 goto err_out;
651
652 /*
653 * The following is done to give a different lockdep key to
654 * @of->mutex for files which implement mmap. This is a rather
655 * crude way to avoid false positive lockdep warning around
656 * mm->mmap_lock - mmap nests @of->mutex under mm->mmap_lock and
657 * reading /sys/block/sda/trace/act_mask grabs sr_mutex, under
658 * which mm->mmap_lock nests, while holding @of->mutex. As each
659 * open file has a separate mutex, it's okay as long as those don't
660 * happen on the same file. At this point, we can't easily give
661 * each file a separate locking class. Let's differentiate on
662 * whether the file has mmap or not for now.
663 *
664 * For similar reasons, writable and readonly files are given different
665 * lockdep key, because the writable file /sys/power/resume may call vfs
666 * lookup helpers for arbitrary paths and readonly files can be read by
667 * overlayfs from vfs helpers when sysfs is a lower layer of overalyfs.
668 *
669 * All three cases look the same. They're supposed to
670 * look that way and give @of->mutex different static lockdep keys.
671 */
672 if (has_mmap)
673 mutex_init(&of->mutex);
674 else if (file->f_mode & FMODE_WRITE)
675 mutex_init(&of->mutex);
676 else
677 mutex_init(&of->mutex);
678
679 of->kn = kn;
680 of->file = file;
681
682 /*
683 * Write path needs to atomic_write_len outside active reference.
684 * Cache it in open_file. See kernfs_fop_write_iter() for details.
685 */
686 of->atomic_write_len = ops->atomic_write_len;
687
688 error = -EINVAL;
689 /*
690 * ->seq_show is incompatible with ->prealloc,
691 * as seq_read does its own allocation.
692 * ->read must be used instead.
693 */
694 if (ops->prealloc && ops->seq_show)
695 goto err_free;
696 if (ops->prealloc) {
697 int len = of->atomic_write_len ?: PAGE_SIZE;
698 of->prealloc_buf = kmalloc(len + 1, GFP_KERNEL);
699 error = -ENOMEM;
700 if (!of->prealloc_buf)
701 goto err_free;
702 mutex_init(&of->prealloc_mutex);
703 }
704
705 /*
706 * Always instantiate seq_file even if read access doesn't use
707 * seq_file or is not requested. This unifies private data access
708 * and readable regular files are the vast majority anyway.
709 */
710 if (ops->seq_show)
711 error = seq_open(file, &kernfs_seq_ops);
712 else
713 error = seq_open(file, NULL);
714 if (error)
715 goto err_free;
716
717 of->seq_file = file->private_data;
718 of->seq_file->private = of;
719
720 /* seq_file clears PWRITE unconditionally, restore it if WRITE */
721 if (file->f_mode & FMODE_WRITE)
722 file->f_mode |= FMODE_PWRITE;
723
724 /* make sure we have open node struct */
725 error = kernfs_get_open_node(kn, of);
726 if (error)
727 goto err_seq_release;
728
729 if (ops->open) {
730 /* nobody has access to @of yet, skip @of->mutex */
731 error = ops->open(of);
732 if (error)
733 goto err_put_node;
734 }
735
736 /* open succeeded, put active references */
737 kernfs_put_active(kn);
738 return 0;
739
740err_put_node:
741 kernfs_unlink_open_file(kn, of, true);
742err_seq_release:
743 seq_release(inode, file);
744err_free:
745 kfree(of->prealloc_buf);
746 kfree(of);
747err_out:
748 kernfs_put_active(kn);
749 return error;
750}
751
752/* used from release/drain to ensure that ->release() is called exactly once */
753static void kernfs_release_file(struct kernfs_node *kn,
754 struct kernfs_open_file *of)
755{
756 /*
757 * @of is guaranteed to have no other file operations in flight and
758 * we just want to synchronize release and drain paths.
759 * @kernfs_open_file_mutex_ptr(kn) is enough. @of->mutex can't be used
760 * here because drain path may be called from places which can
761 * cause circular dependency.
762 */
763 lockdep_assert_held(kernfs_open_file_mutex_ptr(kn));
764
765 if (!of->released) {
766 /*
767 * A file is never detached without being released and we
768 * need to be able to release files which are deactivated
769 * and being drained. Don't use kernfs_ops().
770 */
771 kn->attr.ops->release(of);
772 of->released = true;
773 of_on(of)->nr_to_release--;
774 }
775}
776
777static int kernfs_fop_release(struct inode *inode, struct file *filp)
778{
779 struct kernfs_node *kn = inode->i_private;
780 struct kernfs_open_file *of = kernfs_of(filp);
781
782 if (kn->flags & KERNFS_HAS_RELEASE) {
783 struct mutex *mutex;
784
785 mutex = kernfs_open_file_mutex_lock(kn);
786 kernfs_release_file(kn, of);
787 mutex_unlock(mutex);
788 }
789
790 kernfs_unlink_open_file(kn, of, false);
791 seq_release(inode, filp);
792 kfree(of->prealloc_buf);
793 kfree(of);
794
795 return 0;
796}
797
798bool kernfs_should_drain_open_files(struct kernfs_node *kn)
799{
800 struct kernfs_open_node *on;
801 bool ret;
802
803 /*
804 * @kn being deactivated guarantees that @kn->attr.open can't change
805 * beneath us making the lockless test below safe.
806 * Callers post kernfs_unbreak_active_protection may be counted in
807 * kn->active by now, do not WARN_ON because of them.
808 */
809
810 rcu_read_lock();
811 on = rcu_dereference(kn->attr.open);
812 ret = on && (on->nr_mmapped || on->nr_to_release);
813 rcu_read_unlock();
814
815 return ret;
816}
817
818void kernfs_drain_open_files(struct kernfs_node *kn)
819{
820 struct kernfs_open_node *on;
821 struct kernfs_open_file *of;
822 struct mutex *mutex;
823
824 mutex = kernfs_open_file_mutex_lock(kn);
825 on = kernfs_deref_open_node_locked(kn);
826 if (!on) {
827 mutex_unlock(mutex);
828 return;
829 }
830
831 list_for_each_entry(of, &on->files, list) {
832 struct inode *inode = file_inode(of->file);
833
834 if (of->mmapped) {
835 unmap_mapping_range(inode->i_mapping, 0, 0, 1);
836 of->mmapped = false;
837 on->nr_mmapped--;
838 }
839
840 if (kn->flags & KERNFS_HAS_RELEASE)
841 kernfs_release_file(kn, of);
842 }
843
844 WARN_ON_ONCE(on->nr_mmapped || on->nr_to_release);
845 mutex_unlock(mutex);
846}
847
848/*
849 * Kernfs attribute files are pollable. The idea is that you read
850 * the content and then you use 'poll' or 'select' to wait for
851 * the content to change. When the content changes (assuming the
852 * manager for the kobject supports notification), poll will
853 * return EPOLLERR|EPOLLPRI, and select will return the fd whether
854 * it is waiting for read, write, or exceptions.
855 * Once poll/select indicates that the value has changed, you
856 * need to close and re-open the file, or seek to 0 and read again.
857 * Reminder: this only works for attributes which actively support
858 * it, and it is not possible to test an attribute from userspace
859 * to see if it supports poll (Neither 'poll' nor 'select' return
860 * an appropriate error code). When in doubt, set a suitable timeout value.
861 */
862__poll_t kernfs_generic_poll(struct kernfs_open_file *of, poll_table *wait)
863{
864 struct kernfs_open_node *on = of_on(of);
865
866 poll_wait(of->file, &on->poll, wait);
867
868 if (of->event != atomic_read(&on->event))
869 return DEFAULT_POLLMASK|EPOLLERR|EPOLLPRI;
870
871 return DEFAULT_POLLMASK;
872}
873
874static __poll_t kernfs_fop_poll(struct file *filp, poll_table *wait)
875{
876 struct kernfs_open_file *of = kernfs_of(filp);
877 struct kernfs_node *kn = kernfs_dentry_node(filp->f_path.dentry);
878 __poll_t ret;
879
880 if (!kernfs_get_active_of(of))
881 return DEFAULT_POLLMASK|EPOLLERR|EPOLLPRI;
882
883 if (kn->attr.ops->poll)
884 ret = kn->attr.ops->poll(of, wait);
885 else
886 ret = kernfs_generic_poll(of, wait);
887
888 kernfs_put_active_of(of);
889 return ret;
890}
891
892static loff_t kernfs_fop_llseek(struct file *file, loff_t offset, int whence)
893{
894 struct kernfs_open_file *of = kernfs_of(file);
895 const struct kernfs_ops *ops;
896 loff_t ret;
897
898 /*
899 * @of->mutex nests outside active ref and is primarily to ensure that
900 * the ops aren't called concurrently for the same open file.
901 */
902 mutex_lock(&of->mutex);
903 if (!kernfs_get_active_of(of)) {
904 mutex_unlock(&of->mutex);
905 return -ENODEV;
906 }
907
908 ops = kernfs_ops(of->kn);
909 if (ops->llseek)
910 ret = ops->llseek(of, offset, whence);
911 else
912 ret = generic_file_llseek(file, offset, whence);
913
914 kernfs_put_active_of(of);
915 mutex_unlock(&of->mutex);
916 return ret;
917}
918
919static void kernfs_notify_workfn(struct work_struct *work)
920{
921 char name_buf[NAME_MAX + 1];
922 struct kernfs_super_info *info;
923 struct kernfs_node *kn;
924 struct kernfs_root *root;
925 struct qstr name;
926 bool have_name;
927repeat:
928 /* pop one off the notify_list */
929 spin_lock_irq(&kernfs_notify_lock);
930 kn = kernfs_notify_list;
931 if (kn == KERNFS_NOTIFY_EOL) {
932 spin_unlock_irq(&kernfs_notify_lock);
933 return;
934 }
935 kernfs_notify_list = kn->attr.notify_next;
936 kn->attr.notify_next = NULL;
937 spin_unlock_irq(&kernfs_notify_lock);
938
939 root = kernfs_root(kn);
940 /* kick fsnotify */
941
942 /*
943 * Sample the name once so kernfs_rwsem need not be held across the
944 * loop. A name that does not fit is reported without one; fsnotify()
945 * takes the name as optional, so a watcher loses the name and not the
946 * event.
947 */
948 have_name = kernfs_name(kn, name_buf, sizeof(name_buf)) >= 0;
949 name = QSTR(name_buf);
950
951 down_read(&root->kernfs_supers_rwsem);
952 list_for_each_entry(info, &root->supers, node) {
953 struct kernfs_node *parent;
954 struct inode *p_inode = NULL;
955 struct inode *inode;
956
957 /*
958 * We want fsnotify_modify() on @kn but as the
959 * modifications aren't originating from userland don't
960 * have the matching @file available. Look up the inodes
961 * and generate the events manually.
962 */
963 inode = ilookup(info->sb, kernfs_ino(kn));
964 if (!inode)
965 continue;
966
967 parent = kernfs_get_parent(kn);
968 if (parent) {
969 p_inode = ilookup(info->sb, kernfs_ino(parent));
970 if (p_inode) {
971 fsnotify(FS_MODIFY | FS_EVENT_ON_CHILD,
972 inode, FSNOTIFY_EVENT_INODE,
973 p_inode, have_name ? &name : NULL,
974 inode, 0);
975 iput(p_inode);
976 }
977
978 kernfs_put(parent);
979 }
980
981 if (!p_inode)
982 fsnotify_inode(inode, FS_MODIFY);
983
984 iput(inode);
985 }
986
987 up_read(&root->kernfs_supers_rwsem);
988 kernfs_put(kn);
989 goto repeat;
990}
991
992/**
993 * kernfs_notify - notify a kernfs file
994 * @kn: file to notify
995 *
996 * Notify @kn such that poll(2) on @kn wakes up. Maybe be called from any
997 * context.
998 */
999void kernfs_notify(struct kernfs_node *kn)
1000{
1001 static DECLARE_WORK(kernfs_notify_work, kernfs_notify_workfn);
1002 unsigned long flags;
1003 struct kernfs_open_node *on;
1004
1005 if (WARN_ON(kernfs_type(kn) != KERNFS_FILE))
1006 return;
1007
1008 /* kick poll immediately */
1009 rcu_read_lock();
1010 on = rcu_dereference(kn->attr.open);
1011 if (on) {
1012 atomic_inc(&on->event);
1013 wake_up_interruptible(&on->poll);
1014 }
1015 rcu_read_unlock();
1016
1017 /* schedule work to kick fsnotify */
1018 spin_lock_irqsave(&kernfs_notify_lock, flags);
1019 if (!kn->attr.notify_next) {
1020 kernfs_get(kn);
1021 kn->attr.notify_next = kernfs_notify_list;
1022 kernfs_notify_list = kn;
1023 schedule_work(&kernfs_notify_work);
1024 }
1025 spin_unlock_irqrestore(&kernfs_notify_lock, flags);
1026}
1027EXPORT_SYMBOL_GPL(kernfs_notify);
1028
1029const struct file_operations kernfs_file_fops = {
1030 .read_iter = kernfs_fop_read_iter,
1031 .write_iter = kernfs_fop_write_iter,
1032 .llseek = kernfs_fop_llseek,
1033 .mmap = kernfs_fop_mmap,
1034 .open = kernfs_fop_open,
1035 .release = kernfs_fop_release,
1036 .poll = kernfs_fop_poll,
1037 .fsync = noop_fsync,
1038 .splice_read = copy_splice_read,
1039 .splice_write = iter_file_splice_write,
1040};
1041
1042/**
1043 * __kernfs_create_file - kernfs internal function to create a file
1044 * @parent: directory to create the file in
1045 * @name: name of the file
1046 * @mode: mode of the file
1047 * @uid: uid of the file
1048 * @gid: gid of the file
1049 * @size: size of the file
1050 * @ops: kernfs operations for the file
1051 * @priv: private data for the file
1052 * @ns: optional namespace tag of the file
1053 * @key: lockdep key for the file's active_ref, %NULL to disable lockdep
1054 *
1055 * Return: the created node on success, ERR_PTR() value on error.
1056 */
1057struct kernfs_node *__kernfs_create_file(struct kernfs_node *parent,
1058 const char *name,
1059 umode_t mode, kuid_t uid, kgid_t gid,
1060 loff_t size,
1061 const struct kernfs_ops *ops,
1062 void *priv, const struct ns_common *ns,
1063 struct lock_class_key *key)
1064{
1065 struct kernfs_node *kn;
1066 unsigned flags;
1067 int rc;
1068
1069 flags = KERNFS_FILE;
1070
1071 kn = kernfs_new_node(parent, name, (mode & S_IALLUGO) | S_IFREG,
1072 uid, gid, flags);
1073 if (!kn)
1074 return ERR_PTR(-ENOMEM);
1075
1076 kn->attr.ops = ops;
1077 kn->attr.size = size;
1078 kn->ns = ns;
1079 kn->priv = priv;
1080
1081#ifdef CONFIG_DEBUG_LOCK_ALLOC
1082 if (key) {
1083 lockdep_init_map(&kn->dep_map, "kn->active", key, 0);
1084 kn->flags |= KERNFS_LOCKDEP;
1085 }
1086#endif
1087
1088 /*
1089 * kn->attr.ops is accessible only while holding active ref. We
1090 * need to know whether some ops are implemented outside active
1091 * ref. Cache their existence in flags.
1092 */
1093 if (ops->seq_show)
1094 kn->flags |= KERNFS_HAS_SEQ_SHOW;
1095 if (ops->mmap)
1096 kn->flags |= KERNFS_HAS_MMAP;
1097 if (ops->release)
1098 kn->flags |= KERNFS_HAS_RELEASE;
1099
1100 rc = kernfs_add_one(kn);
1101 if (rc) {
1102 kernfs_put(kn);
1103 return ERR_PTR(rc);
1104 }
1105 return kn;
1106}