projects/linux·blob master · refs

1209 lines · 35.0 KiB

1// SPDX-License-Identifier: GPL-2.0-or-later
2/*
3 * Copyright (C) 2008 Red Hat, Inc., Eric Paris <[email protected]>
4 */
5
6/*
7 * fsnotify inode mark locking/lifetime/and refcnting
8 *
9 * REFCNT:
10 * The group->recnt and mark->refcnt tell how many "things" in the kernel
11 * currently are referencing the objects. Both kind of objects typically will
12 * live inside the kernel with a refcnt of 2, one for its creation and one for
13 * the reference a group and a mark hold to each other.
14 * If you are holding the appropriate locks, you can take a reference and the
15 * object itself is guaranteed to survive until the reference is dropped.
16 *
17 * LOCKING:
18 * There are 3 locks involved with fsnotify inode marks and they MUST be taken
19 * in order as follows:
20 *
21 * group->mark_mutex
22 * mark->lock
23 * mark->connector->lock
24 *
25 * group->mark_mutex protects the marks_list anchored inside a given group and
26 * each mark is hooked via the g_list. It also protects the groups private
27 * data (i.e group limits).
28
29 * mark->lock protects the marks attributes like its masks and flags.
30 * Furthermore it protects the access to a reference of the group that the mark
31 * is assigned to as well as the access to a reference of the inode/vfsmount
32 * that is being watched by the mark.
33 *
34 * mark->connector->lock protects the list of marks anchored inside an
35 * inode / vfsmount and each mark is hooked via the i_list.
36 *
37 * A list of notification marks relating to inode / mnt is contained in
38 * fsnotify_mark_connector. That structure is alive as long as there are any
39 * marks in the list and is also protected by fsnotify_mark_srcu. A mark gets
40 * detached from fsnotify_mark_connector when last reference to the mark is
41 * dropped. Thus having mark reference is enough to protect mark->connector
42 * pointer and to make sure fsnotify_mark_connector cannot disappear. Also
43 * because we remove mark from g_list before dropping mark reference associated
44 * with that, any mark found through g_list is guaranteed to have
45 * mark->connector set until we drop group->mark_mutex.
46 *
47 * LIFETIME:
48 * Inode marks survive between when they are added to an inode and when their
49 * refcnt==0. Marks are also protected by fsnotify_mark_srcu.
50 *
51 * The inode mark can be cleared for a number of different reasons including:
52 * - The inode is unlinked for the last time. (fsnotify_inode_remove)
53 * - The inode is being evicted from cache. (fsnotify_inode_delete)
54 * - The fs the inode is on is unmounted. (fsnotify_inode_delete/fsnotify_unmount_inodes)
55 * - Something explicitly requests that it be removed. (fsnotify_destroy_mark)
56 * - The fsnotify_group associated with the mark is going away and all such marks
57 * need to be cleaned up. (fsnotify_clear_marks_by_group)
58 *
59 * This has the very interesting property of being able to run concurrently with
60 * any (or all) other directions.
61 */
62
63#include <linux/fs.h>
64#include <linux/init.h>
65#include <linux/kernel.h>
66#include <linux/kthread.h>
67#include <linux/module.h>
68#include <linux/mutex.h>
69#include <linux/slab.h>
70#include <linux/spinlock.h>
71#include <linux/srcu.h>
72#include <linux/ratelimit.h>
73
74#include <linux/atomic.h>
75
76#include <linux/fsnotify_backend.h>
77#include "fsnotify.h"
78
79#define FSNOTIFY_REAPER_DELAY (1) /* 1 jiffy */
80
81struct srcu_struct fsnotify_mark_srcu;
82static struct kmem_cache *fsnotify_mark_connector_cachep;
83static struct kmem_cache *fsnotify_inode_mark_connector_cachep;
84
85static DEFINE_SPINLOCK(destroy_lock);
86static LIST_HEAD(destroy_list);
87static struct fsnotify_mark_connector *connector_destroy_list;
88
89static void fsnotify_mark_destroy_workfn(struct work_struct *work);
90static DECLARE_DELAYED_WORK(reaper_work, fsnotify_mark_destroy_workfn);
91
92static void fsnotify_connector_destroy_workfn(struct work_struct *work);
93static DECLARE_WORK(connector_reaper_work, fsnotify_connector_destroy_workfn);
94
95void fsnotify_get_mark(struct fsnotify_mark *mark)
96{
97 WARN_ON_ONCE(!refcount_read(&mark->refcnt));
98 refcount_inc(&mark->refcnt);
99}
100
101static fsnotify_connp_t *fsnotify_object_connp(void *obj,
102 enum fsnotify_obj_type obj_type)
103{
104 switch (obj_type) {
105 case FSNOTIFY_OBJ_TYPE_INODE:
106 return &((struct inode *)obj)->i_fsnotify_marks;
107 case FSNOTIFY_OBJ_TYPE_VFSMOUNT:
108 return &real_mount(obj)->mnt_fsnotify_marks;
109 case FSNOTIFY_OBJ_TYPE_SB:
110 return fsnotify_sb_marks(obj);
111 case FSNOTIFY_OBJ_TYPE_MNTNS:
112 return &((struct mnt_namespace *)obj)->n_fsnotify_marks;
113 default:
114 return NULL;
115 }
116}
117
118static __u32 *fsnotify_conn_mask_p(struct fsnotify_mark_connector *conn)
119{
120 if (conn->type == FSNOTIFY_OBJ_TYPE_INODE)
121 return &fsnotify_conn_inode(conn)->i_fsnotify_mask;
122 else if (conn->type == FSNOTIFY_OBJ_TYPE_VFSMOUNT)
123 return &fsnotify_conn_mount(conn)->mnt_fsnotify_mask;
124 else if (conn->type == FSNOTIFY_OBJ_TYPE_SB)
125 return &fsnotify_conn_sb(conn)->s_fsnotify_mask;
126 else if (conn->type == FSNOTIFY_OBJ_TYPE_MNTNS)
127 return &fsnotify_conn_mntns(conn)->n_fsnotify_mask;
128 return NULL;
129}
130
131__u32 fsnotify_conn_mask(struct fsnotify_mark_connector *conn)
132{
133 if (WARN_ON(!fsnotify_valid_obj_type(conn->type)))
134 return 0;
135
136 return READ_ONCE(*fsnotify_conn_mask_p(conn));
137}
138
139static void fsnotify_get_sb_watched_objects(struct super_block *sb)
140{
141 atomic_long_inc(fsnotify_sb_watched_objects(sb));
142}
143
144static void fsnotify_put_sb_watched_objects(struct super_block *sb)
145{
146 atomic_long_t *watched_objects = fsnotify_sb_watched_objects(sb);
147
148 /* the superblock can go away after this decrement */
149 if (atomic_long_dec_and_test(watched_objects))
150 wake_up_var(watched_objects);
151}
152
153static void fsnotify_get_inode_ref(struct inode *inode)
154{
155 ihold(inode);
156 fsnotify_get_sb_watched_objects(inode->i_sb);
157}
158
159static void fsnotify_put_inode_ref(struct inode *inode)
160{
161 /* read ->i_sb before the inode can go away */
162 struct super_block *sb = inode->i_sb;
163
164 iput(inode);
165 fsnotify_put_sb_watched_objects(sb);
166}
167
168/*
169 * Grab or drop watched objects reference depending on whether the connector
170 * is attached and has any marks attached.
171 */
172static void fsnotify_update_sb_watchers(struct super_block *sb,
173 struct fsnotify_mark_connector *conn)
174{
175 struct fsnotify_sb_info *sbinfo = fsnotify_sb_info(sb);
176 bool is_watched = conn->flags & FSNOTIFY_CONN_FLAG_IS_WATCHED;
177 struct fsnotify_mark *first_mark = NULL;
178 unsigned int highest_prio = 0;
179
180 if (conn->obj)
181 first_mark = hlist_entry_safe(conn->list.first,
182 struct fsnotify_mark, obj_list);
183 if (first_mark)
184 highest_prio = first_mark->group->priority;
185 if (WARN_ON(highest_prio >= __FSNOTIFY_PRIO_NUM))
186 highest_prio = 0;
187
188 /*
189 * If the highest priority of group watching this object is prio,
190 * then watched object has a reference on counters [0..prio].
191 * Update priority >= 1 watched objects counters.
192 */
193 for (unsigned int p = conn->prio + 1; p <= highest_prio; p++)
194 atomic_long_inc(&sbinfo->watched_objects[p]);
195 for (unsigned int p = conn->prio; p > highest_prio; p--)
196 atomic_long_dec(&sbinfo->watched_objects[p]);
197 conn->prio = highest_prio;
198
199 /* Update priority >= 0 (a.k.a total) watched objects counter */
200 BUILD_BUG_ON(FSNOTIFY_PRIO_NORMAL != 0);
201 if (first_mark && !is_watched) {
202 conn->flags |= FSNOTIFY_CONN_FLAG_IS_WATCHED;
203 fsnotify_get_sb_watched_objects(sb);
204 } else if (!first_mark && is_watched) {
205 conn->flags &= ~FSNOTIFY_CONN_FLAG_IS_WATCHED;
206 fsnotify_put_sb_watched_objects(sb);
207 }
208}
209
210/*
211 * Grab or drop inode reference for the connector if needed.
212 *
213 * When it's time to drop the reference, we only clear the HAS_IREF flag and
214 * return the inode object. fsnotify_drop_object() will be resonsible for doing
215 * iput() outside of spinlocks. This happens when last mark that wanted iref is
216 * detached.
217 */
218static struct inode *fsnotify_update_iref(struct fsnotify_mark_connector *conn,
219 bool want_iref)
220{
221 bool has_iref = conn->flags & FSNOTIFY_CONN_FLAG_HAS_IREF;
222 struct inode *inode = NULL;
223
224 if (conn->type != FSNOTIFY_OBJ_TYPE_INODE ||
225 want_iref == has_iref)
226 return NULL;
227
228 if (want_iref) {
229 /* Pin inode if any mark wants inode refcount held */
230 fsnotify_get_inode_ref(fsnotify_conn_inode(conn));
231 conn->flags |= FSNOTIFY_CONN_FLAG_HAS_IREF;
232 } else {
233 /* Unpin inode after detach of last mark that wanted iref */
234 inode = fsnotify_conn_inode(conn);
235 conn->flags &= ~FSNOTIFY_CONN_FLAG_HAS_IREF;
236 }
237
238 return inode;
239}
240
241/*
242 * Calculate mask of events for a list of marks.
243 *
244 * Return true if any of the attached marks want to hold an inode reference.
245 */
246static bool __fsnotify_recalc_mask(struct fsnotify_mark_connector *conn)
247{
248 u32 new_mask = 0;
249 bool want_iref = false;
250 struct fsnotify_mark *mark;
251
252 assert_spin_locked(&conn->lock);
253 /* We can get detached connector here when inode is getting unlinked. */
254 if (!fsnotify_valid_obj_type(conn->type))
255 return NULL;
256 hlist_for_each_entry(mark, &conn->list, obj_list) {
257 if (!(mark->flags & FSNOTIFY_MARK_FLAG_ATTACHED))
258 continue;
259 new_mask |= fsnotify_calc_mask(mark);
260 if (conn->type == FSNOTIFY_OBJ_TYPE_INODE &&
261 !(mark->flags & FSNOTIFY_MARK_FLAG_NO_IREF))
262 want_iref = true;
263 }
264 /*
265 * We use WRITE_ONCE() to prevent silly compiler optimizations from
266 * confusing readers not holding conn->lock with partial updates.
267 */
268 WRITE_ONCE(*fsnotify_conn_mask_p(conn), new_mask);
269
270 return want_iref;
271}
272
273/*
274 * Calculate mask of events for a list of marks after attach/modify mark
275 * and get an inode reference for the connector if needed.
276 *
277 * A concurrent add of evictable mark and detach of non-evictable mark can
278 * lead to __fsnotify_recalc_mask() returning false want_iref, but in this
279 * case we defer clearing iref to fsnotify_recalc_mask_clear_iref() called
280 * from fsnotify_put_mark().
281 */
282static void fsnotify_recalc_mask_set_iref(struct fsnotify_mark_connector *conn)
283{
284 bool has_iref = conn->flags & FSNOTIFY_CONN_FLAG_HAS_IREF;
285 bool want_iref = __fsnotify_recalc_mask(conn) || has_iref;
286
287 (void) fsnotify_update_iref(conn, want_iref);
288}
289
290/*
291 * Calculate mask of events for a list of marks after detach mark
292 * and return the inode object if its reference is no longer needed.
293 */
294static void *fsnotify_recalc_mask_clear_iref(struct fsnotify_mark_connector *conn)
295{
296 bool want_iref = __fsnotify_recalc_mask(conn);
297
298 return fsnotify_update_iref(conn, want_iref);
299}
300
301static bool fsnotify_conn_watches_children(
302 struct fsnotify_mark_connector *conn)
303{
304 if (conn->type != FSNOTIFY_OBJ_TYPE_INODE)
305 return false;
306
307 return fsnotify_inode_watches_children(fsnotify_conn_inode(conn));
308}
309
310static void fsnotify_conn_set_children_dentry_flags(
311 struct fsnotify_mark_connector *conn)
312{
313 if (conn->type != FSNOTIFY_OBJ_TYPE_INODE)
314 return;
315
316 fsnotify_set_children_dentry_flags(fsnotify_conn_inode(conn));
317}
318
319/*
320 * Calculate mask of events for a list of marks. The caller must make sure
321 * connector and connector->obj cannot disappear under us. Callers achieve
322 * this by holding a mark->lock or mark->group->mark_mutex for a mark on this
323 * list.
324 */
325void fsnotify_recalc_mask(struct fsnotify_mark_connector *conn)
326{
327 bool update_children;
328
329 if (!conn)
330 return;
331
332 spin_lock(&conn->lock);
333 update_children = !fsnotify_conn_watches_children(conn);
334 fsnotify_recalc_mask_set_iref(conn);
335 update_children &= fsnotify_conn_watches_children(conn);
336 spin_unlock(&conn->lock);
337 /*
338 * Set children's PARENT_WATCHED flags only if parent started watching.
339 * When parent stops watching, we clear false positive PARENT_WATCHED
340 * flags lazily in __fsnotify_parent().
341 */
342 if (update_children)
343 fsnotify_conn_set_children_dentry_flags(conn);
344}
345
346/**
347 * fsnotify_modify_mark_mask - set and/or clear flags in a mark's mask
348 * @mark: mark to be modified
349 * @set: bits to be set in mask
350 * @clear: bits to be cleared in mask
351 *
352 * Modify a fsnotify_mark mask as directed, and update its associated conn.
353 * The caller is expected to hold a reference to the mark.
354 */
355void fsnotify_modify_mark_mask(struct fsnotify_mark *mark, u32 set, u32 clear)
356{
357 bool recalc = false;
358 u32 mask;
359
360 WARN_ON_ONCE(clear & set);
361
362 spin_lock(&mark->lock);
363 mask = mark->mask;
364 mark->mask |= set;
365 mark->mask &= ~clear;
366 if (mark->mask != mask)
367 recalc = true;
368 spin_unlock(&mark->lock);
369
370 if (recalc)
371 fsnotify_recalc_mask(mark->connector);
372}
373EXPORT_SYMBOL_GPL(fsnotify_modify_mark_mask);
374
375/* Free all connectors queued for freeing once SRCU period ends */
376static void fsnotify_connector_destroy_workfn(struct work_struct *work)
377{
378 struct fsnotify_mark_connector *conn, *free;
379
380 spin_lock(&destroy_lock);
381 conn = connector_destroy_list;
382 connector_destroy_list = NULL;
383 spin_unlock(&destroy_lock);
384
385 synchronize_srcu(&fsnotify_mark_srcu);
386 while (conn) {
387 free = conn;
388 conn = conn->destroy_next;
389 kfree(free);
390 }
391}
392
393static void fsnotify_untrack_connector(struct fsnotify_mark_connector *conn);
394
395static void *fsnotify_detach_connector_from_object(
396 struct fsnotify_mark_connector *conn,
397 unsigned int *type)
398{
399 fsnotify_connp_t *connp = fsnotify_object_connp(conn->obj, conn->type);
400 struct super_block *sb = fsnotify_connector_sb(conn);
401 struct inode *inode = NULL;
402
403 *type = conn->type;
404 if (conn->type == FSNOTIFY_OBJ_TYPE_DETACHED)
405 return NULL;
406
407 if (conn->type == FSNOTIFY_OBJ_TYPE_INODE) {
408 inode = fsnotify_conn_inode(conn);
409 inode->i_fsnotify_mask = 0;
410 fsnotify_untrack_connector(conn);
411
412 /* Unpin inode when detaching from connector */
413 if (!(conn->flags & FSNOTIFY_CONN_FLAG_HAS_IREF))
414 inode = NULL;
415 } else if (conn->type == FSNOTIFY_OBJ_TYPE_VFSMOUNT) {
416 fsnotify_conn_mount(conn)->mnt_fsnotify_mask = 0;
417 } else if (conn->type == FSNOTIFY_OBJ_TYPE_SB) {
418 fsnotify_conn_sb(conn)->s_fsnotify_mask = 0;
419 } else if (conn->type == FSNOTIFY_OBJ_TYPE_MNTNS) {
420 fsnotify_conn_mntns(conn)->n_fsnotify_mask = 0;
421 }
422
423 rcu_assign_pointer(*connp, NULL);
424 conn->obj = NULL;
425 conn->type = FSNOTIFY_OBJ_TYPE_DETACHED;
426 if (sb)
427 fsnotify_update_sb_watchers(sb, conn);
428
429 return inode;
430}
431
432static void fsnotify_final_mark_destroy(struct fsnotify_mark *mark)
433{
434 struct fsnotify_group *group = mark->group;
435
436 if (WARN_ON_ONCE(!group))
437 return;
438 group->ops->free_mark(mark);
439 fsnotify_put_group(group);
440}
441
442/* Drop object reference originally held by a connector */
443static void fsnotify_drop_object(unsigned int type, void *objp)
444{
445 if (!objp)
446 return;
447 /* Currently only inode references are passed to be dropped */
448 if (WARN_ON_ONCE(type != FSNOTIFY_OBJ_TYPE_INODE))
449 return;
450 fsnotify_put_inode_ref(objp);
451}
452
453void fsnotify_put_mark(struct fsnotify_mark *mark)
454{
455 struct fsnotify_mark_connector *conn = READ_ONCE(mark->connector);
456 void *objp = NULL;
457 unsigned int type = FSNOTIFY_OBJ_TYPE_DETACHED;
458 bool free_conn = false;
459
460 /* Catch marks that were actually never attached to object */
461 if (!conn) {
462 if (refcount_dec_and_test(&mark->refcnt))
463 fsnotify_final_mark_destroy(mark);
464 return;
465 }
466
467 /*
468 * We have to be careful so that traversals of obj_list under lock can
469 * safely grab mark reference.
470 */
471 if (!refcount_dec_and_lock(&mark->refcnt, &conn->lock))
472 return;
473
474 hlist_del_init_rcu(&mark->obj_list);
475 if (hlist_empty(&conn->list)) {
476 objp = fsnotify_detach_connector_from_object(conn, &type);
477 free_conn = true;
478 } else {
479 struct super_block *sb = fsnotify_connector_sb(conn);
480
481 /* Update watched objects after detaching mark */
482 if (sb)
483 fsnotify_update_sb_watchers(sb, conn);
484 objp = fsnotify_recalc_mask_clear_iref(conn);
485 type = conn->type;
486 }
487 WRITE_ONCE(mark->connector, NULL);
488 spin_unlock(&conn->lock);
489
490 fsnotify_drop_object(type, objp);
491
492 if (free_conn) {
493 spin_lock(&destroy_lock);
494 conn->destroy_next = connector_destroy_list;
495 connector_destroy_list = conn;
496 spin_unlock(&destroy_lock);
497 queue_work(system_dfl_wq, &connector_reaper_work);
498 }
499 /*
500 * Note that we didn't update flags telling whether inode cares about
501 * what's happening with children. We update these flags from
502 * __fsnotify_parent() lazily when next event happens on one of our
503 * children.
504 */
505 spin_lock(&destroy_lock);
506 list_add(&mark->g_list, &destroy_list);
507 spin_unlock(&destroy_lock);
508 queue_delayed_work(system_dfl_wq, &reaper_work,
509 FSNOTIFY_REAPER_DELAY);
510}
511EXPORT_SYMBOL_GPL(fsnotify_put_mark);
512
513/*
514 * Get mark reference when we found the mark via lockless traversal of object
515 * list. Mark can be already removed from the list by now and on its way to be
516 * destroyed once SRCU period ends.
517 *
518 * Also pin the group so it doesn't disappear under us.
519 */
520static bool fsnotify_get_mark_safe(struct fsnotify_mark *mark)
521{
522 if (refcount_inc_not_zero(&mark->refcnt)) {
523 spin_lock(&mark->lock);
524 if (mark->flags & FSNOTIFY_MARK_FLAG_ATTACHED) {
525 /* mark is attached, group is still alive then */
526 atomic_inc(&mark->group->user_waits);
527 spin_unlock(&mark->lock);
528 return true;
529 }
530 spin_unlock(&mark->lock);
531 fsnotify_put_mark(mark);
532 }
533 return false;
534}
535
536/*
537 * Puts marks and wakes up group destruction if necessary.
538 *
539 * Pairs with fsnotify_get_mark_safe()
540 */
541static void fsnotify_put_mark_wake(struct fsnotify_mark *mark)
542{
543 if (mark) {
544 struct fsnotify_group *group = mark->group;
545
546 fsnotify_put_mark(mark);
547 /*
548 * We abuse notification_waitq on group shutdown for waiting for
549 * all marks pinned when waiting for userspace.
550 */
551 if (atomic_dec_and_test(&group->user_waits) && group->shutdown)
552 wake_up(&group->notification_waitq);
553 }
554}
555
556bool fsnotify_prepare_user_wait(struct fsnotify_iter_info *iter_info)
557 __releases(&fsnotify_mark_srcu)
558{
559 int type;
560
561 fsnotify_foreach_iter_type(type) {
562 struct fsnotify_mark *mark = iter_info->marks[type];
563
564 /* This can fail if mark is being removed */
565 while (mark && !fsnotify_get_mark_safe(mark)) {
566 if (mark->group == iter_info->current_group) {
567 __release(&fsnotify_mark_srcu);
568 goto fail;
569 }
570 /* This is a mark in an unrelated group, skip */
571 mark = fsnotify_next_mark(mark);
572 iter_info->marks[type] = mark;
573 }
574 }
575
576 /*
577 * Now that all marks are pinned by refcount in the inode / vfsmount / etc
578 * lists, we can drop SRCU lock, and safely resume the list iteration
579 * once userspace returns.
580 */
581 srcu_read_unlock(&fsnotify_mark_srcu, iter_info->srcu_idx);
582
583 return true;
584
585fail:
586 for (type--; type >= 0; type--)
587 fsnotify_put_mark_wake(iter_info->marks[type]);
588 return false;
589}
590
591void fsnotify_finish_user_wait(struct fsnotify_iter_info *iter_info)
592 __acquires(&fsnotify_mark_srcu)
593{
594 int type;
595
596 iter_info->srcu_idx = srcu_read_lock(&fsnotify_mark_srcu);
597 fsnotify_foreach_iter_type(type)
598 fsnotify_put_mark_wake(iter_info->marks[type]);
599}
600
601/*
602 * Mark mark as detached, remove it from group list. Mark still stays in object
603 * list until its last reference is dropped. Note that we rely on mark being
604 * removed from group list before corresponding reference to it is dropped. In
605 * particular we rely on mark->connector being valid while we hold
606 * group->mark_mutex if we found the mark through g_list.
607 *
608 * Must be called with group->mark_mutex held. The caller must either hold
609 * reference to the mark or be protected by fsnotify_mark_srcu.
610 */
611void fsnotify_detach_mark(struct fsnotify_mark *mark)
612{
613 fsnotify_group_assert_locked(mark->group);
614 WARN_ON_ONCE(!srcu_read_lock_held(&fsnotify_mark_srcu) &&
615 refcount_read(&mark->refcnt) < 1 +
616 !!(mark->flags & FSNOTIFY_MARK_FLAG_ATTACHED));
617
618 spin_lock(&mark->lock);
619 /* something else already called this function on this mark */
620 if (!(mark->flags & FSNOTIFY_MARK_FLAG_ATTACHED)) {
621 spin_unlock(&mark->lock);
622 return;
623 }
624 mark->flags &= ~FSNOTIFY_MARK_FLAG_ATTACHED;
625 list_del_init(&mark->g_list);
626 spin_unlock(&mark->lock);
627
628 /* Drop mark reference acquired in fsnotify_add_mark_locked() */
629 fsnotify_put_mark(mark);
630}
631
632/*
633 * Free fsnotify mark. The mark is actually only marked as being freed. The
634 * freeing is actually happening only once last reference to the mark is
635 * dropped from a workqueue which first waits for srcu period end.
636 *
637 * Caller must have a reference to the mark or be protected by
638 * fsnotify_mark_srcu.
639 */
640void fsnotify_free_mark(struct fsnotify_mark *mark)
641{
642 struct fsnotify_group *group = mark->group;
643
644 spin_lock(&mark->lock);
645 /* something else already called this function on this mark */
646 if (!(mark->flags & FSNOTIFY_MARK_FLAG_ALIVE)) {
647 spin_unlock(&mark->lock);
648 return;
649 }
650 mark->flags &= ~FSNOTIFY_MARK_FLAG_ALIVE;
651 spin_unlock(&mark->lock);
652
653 /*
654 * Some groups like to know that marks are being freed. This is a
655 * callback to the group function to let it know that this mark
656 * is being freed.
657 */
658 if (group->ops->freeing_mark)
659 group->ops->freeing_mark(mark, group);
660}
661
662void fsnotify_destroy_mark(struct fsnotify_mark *mark,
663 struct fsnotify_group *group)
664{
665 fsnotify_group_lock(group);
666 fsnotify_detach_mark(mark);
667 fsnotify_group_unlock(group);
668 fsnotify_free_mark(mark);
669}
670EXPORT_SYMBOL_GPL(fsnotify_destroy_mark);
671
672/*
673 * Sorting function for lists of fsnotify marks.
674 *
675 * Fanotify supports different notification classes (reflected as priority of
676 * notification group). Events shall be passed to notification groups in
677 * decreasing priority order. To achieve this marks in notification lists for
678 * inodes and vfsmounts are sorted so that priorities of corresponding groups
679 * are descending.
680 *
681 * Furthermore correct handling of the ignore mask requires processing inode
682 * and vfsmount marks of each group together. Using the group address as
683 * further sort criterion provides a unique sorting order and thus we can
684 * merge inode and vfsmount lists of marks in linear time and find groups
685 * present in both lists.
686 *
687 * A return value of 1 signifies that b has priority over a.
688 * A return value of 0 signifies that the two marks have to be handled together.
689 * A return value of -1 signifies that a has priority over b.
690 */
691int fsnotify_compare_groups(struct fsnotify_group *a, struct fsnotify_group *b)
692{
693 if (a == b)
694 return 0;
695 if (!a)
696 return 1;
697 if (!b)
698 return -1;
699 if (a->priority < b->priority)
700 return 1;
701 if (a->priority > b->priority)
702 return -1;
703 if (a < b)
704 return 1;
705 return -1;
706}
707
708static int fsnotify_attach_info_to_sb(struct super_block *sb)
709{
710 struct fsnotify_sb_info *sbinfo;
711
712 /* sb info is freed on fsnotify_sb_delete() */
713 sbinfo = kzalloc_obj(*sbinfo);
714 if (!sbinfo)
715 return -ENOMEM;
716
717 INIT_LIST_HEAD(&sbinfo->inode_conn_list);
718 spin_lock_init(&sbinfo->list_lock);
719 /*
720 * cmpxchg() provides the barrier so that callers of fsnotify_sb_info()
721 * will observe an initialized structure
722 */
723 if (cmpxchg(&sb->s_fsnotify_info, NULL, sbinfo)) {
724 /* Someone else created sbinfo for us */
725 kfree(sbinfo);
726 }
727 return 0;
728}
729
730struct fsnotify_inode_mark_connector {
731 struct fsnotify_mark_connector common;
732 struct list_head conns_list;
733};
734
735static struct inode *fsnotify_get_living_inode(struct fsnotify_sb_info *sbinfo)
736{
737 struct fsnotify_inode_mark_connector *iconn;
738 struct inode *inode;
739
740 spin_lock(&sbinfo->list_lock);
741 /* Find the first non-evicting inode */
742 list_for_each_entry(iconn, &sbinfo->inode_conn_list, conns_list) {
743 /* All connectors on the list are still attached to an inode */
744 inode = iconn->common.obj;
745 /*
746 * For connectors without FSNOTIFY_CONN_FLAG_HAS_IREF
747 * (evictable marks) corresponding inode may well have 0
748 * refcount and can be undergoing eviction. OTOH list_lock
749 * protects us from the connector getting detached and inode
750 * freed. So we can poke around the inode safely.
751 */
752 spin_lock(&inode->i_lock);
753 if (likely(
754 !(inode_state_read(inode) & (I_FREEING | I_WILL_FREE)))) {
755 __iget(inode);
756 spin_unlock(&inode->i_lock);
757 spin_unlock(&sbinfo->list_lock);
758 return inode;
759 }
760 spin_unlock(&inode->i_lock);
761 }
762 spin_unlock(&sbinfo->list_lock);
763
764 return NULL;
765}
766
767/**
768 * fsnotify_unmount_inodes - an sb is unmounting. Handle any watched inodes.
769 * @sbinfo: fsnotify info for superblock being unmounted.
770 *
771 * Walk all inode connectors for the superblock and free all associated marks.
772 */
773void fsnotify_unmount_inodes(struct fsnotify_sb_info *sbinfo)
774{
775 struct inode *inode;
776
777 while ((inode = fsnotify_get_living_inode(sbinfo))) {
778 fsnotify_inode(inode, FS_UNMOUNT);
779 fsnotify_clear_marks_by_inode(inode);
780 iput(inode);
781 cond_resched();
782 }
783}
784
785static void fsnotify_init_connector(struct fsnotify_mark_connector *conn,
786 void *obj, unsigned int obj_type)
787{
788 spin_lock_init(&conn->lock);
789 INIT_HLIST_HEAD(&conn->list);
790 conn->flags = 0;
791 conn->prio = 0;
792 conn->type = obj_type;
793 conn->obj = obj;
794}
795
796static struct fsnotify_mark_connector *
797fsnotify_alloc_inode_connector(struct inode *inode)
798{
799 struct fsnotify_inode_mark_connector *iconn;
800 struct fsnotify_sb_info *sbinfo = fsnotify_sb_info(inode->i_sb);
801
802 iconn = kmem_cache_alloc(fsnotify_inode_mark_connector_cachep,
803 GFP_KERNEL);
804 if (!iconn)
805 return NULL;
806
807 fsnotify_init_connector(&iconn->common, inode, FSNOTIFY_OBJ_TYPE_INODE);
808 spin_lock(&sbinfo->list_lock);
809 list_add(&iconn->conns_list, &sbinfo->inode_conn_list);
810 spin_unlock(&sbinfo->list_lock);
811
812 return &iconn->common;
813}
814
815static void fsnotify_untrack_connector(struct fsnotify_mark_connector *conn)
816{
817 struct fsnotify_inode_mark_connector *iconn;
818 struct fsnotify_sb_info *sbinfo;
819
820 if (conn->type != FSNOTIFY_OBJ_TYPE_INODE)
821 return;
822
823 iconn = container_of(conn, struct fsnotify_inode_mark_connector, common);
824 sbinfo = fsnotify_sb_info(fsnotify_conn_inode(conn)->i_sb);
825 spin_lock(&sbinfo->list_lock);
826 list_del(&iconn->conns_list);
827 spin_unlock(&sbinfo->list_lock);
828}
829
830static int fsnotify_attach_connector_to_object(fsnotify_connp_t *connp,
831 void *obj, unsigned int obj_type)
832{
833 struct fsnotify_mark_connector *conn;
834
835 if (obj_type == FSNOTIFY_OBJ_TYPE_INODE) {
836 struct inode *inode = obj;
837
838 conn = fsnotify_alloc_inode_connector(inode);
839 } else {
840 conn = kmem_cache_alloc(fsnotify_mark_connector_cachep,
841 GFP_KERNEL);
842 if (conn)
843 fsnotify_init_connector(conn, obj, obj_type);
844 }
845 if (!conn)
846 return -ENOMEM;
847
848 /*
849 * cmpxchg() provides the barrier so that readers of *connp can see
850 * only initialized structure
851 */
852 if (cmpxchg(connp, NULL, conn)) {
853 /* Someone else created list structure for us */
854 fsnotify_untrack_connector(conn);
855 kfree(conn);
856 }
857 return 0;
858}
859
860/*
861 * Get mark connector, make sure it is alive and return with its lock held.
862 * This is for users that get connector pointer from inode or mount. Users that
863 * hold reference to a mark on the list may directly lock connector->lock as
864 * they are sure list cannot go away under them.
865 */
866static struct fsnotify_mark_connector *fsnotify_grab_connector(
867 fsnotify_connp_t *connp)
868{
869 struct fsnotify_mark_connector *conn;
870 int idx;
871
872 idx = srcu_read_lock(&fsnotify_mark_srcu);
873 conn = srcu_dereference(*connp, &fsnotify_mark_srcu);
874 if (!conn)
875 goto out;
876 spin_lock(&conn->lock);
877 if (conn->type == FSNOTIFY_OBJ_TYPE_DETACHED) {
878 spin_unlock(&conn->lock);
879 srcu_read_unlock(&fsnotify_mark_srcu, idx);
880 return NULL;
881 }
882out:
883 srcu_read_unlock(&fsnotify_mark_srcu, idx);
884 return conn;
885}
886
887/*
888 * Add mark into proper place in given list of marks. These marks may be used
889 * for the fsnotify backend to determine which event types should be delivered
890 * to which group and for which inodes. These marks are ordered according to
891 * priority, highest number first, and then by the group's location in memory.
892 */
893static int fsnotify_add_mark_list(struct fsnotify_mark *mark, void *obj,
894 unsigned int obj_type, int add_flags)
895{
896 struct super_block *sb = fsnotify_object_sb(obj, obj_type);
897 struct fsnotify_mark *lmark, *last = NULL;
898 struct fsnotify_mark_connector *conn;
899 fsnotify_connp_t *connp;
900 int cmp;
901 int err = 0;
902
903 if (WARN_ON(!fsnotify_valid_obj_type(obj_type)))
904 return -EINVAL;
905
906 /* the filesystem doesn't want its objects watched */
907 if (sb && (sb->s_type->fs_flags & FS_DISALLOW_NOTIFY))
908 return -EINVAL;
909
910 /*
911 * Attach the sb info before attaching a connector to any object on sb.
912 * The sb info will remain attached as long as sb lives.
913 */
914 if (sb && !fsnotify_sb_info(sb)) {
915 err = fsnotify_attach_info_to_sb(sb);
916 if (err)
917 return err;
918 }
919
920 connp = fsnotify_object_connp(obj, obj_type);
921restart:
922 spin_lock(&mark->lock);
923 conn = fsnotify_grab_connector(connp);
924 if (!conn) {
925 spin_unlock(&mark->lock);
926 err = fsnotify_attach_connector_to_object(connp, obj, obj_type);
927 if (err)
928 return err;
929 goto restart;
930 }
931
932 if (conn->flags & FSNOTIFY_CONN_FLAG_DESTROYING) {
933 err = -ENOENT;
934 goto out_err;
935 }
936
937 /* is mark the first mark? */
938 if (hlist_empty(&conn->list)) {
939 hlist_add_head_rcu(&mark->obj_list, &conn->list);
940 goto added;
941 }
942
943 /* should mark be in the middle of the current list? */
944 hlist_for_each_entry(lmark, &conn->list, obj_list) {
945 last = lmark;
946
947 if ((lmark->group == mark->group) &&
948 (lmark->flags & FSNOTIFY_MARK_FLAG_ATTACHED) &&
949 !(mark->group->flags & FSNOTIFY_GROUP_DUPS)) {
950 err = -EEXIST;
951 goto out_err;
952 }
953
954 cmp = fsnotify_compare_groups(lmark->group, mark->group);
955 if (cmp >= 0) {
956 hlist_add_before_rcu(&mark->obj_list, &lmark->obj_list);
957 goto added;
958 }
959 }
960
961 BUG_ON(last == NULL);
962 /* mark should be the last entry. last is the current last entry */
963 hlist_add_behind_rcu(&mark->obj_list, &last->obj_list);
964added:
965 if (sb)
966 fsnotify_update_sb_watchers(sb, conn);
967 /*
968 * Since connector is attached to object using cmpxchg() we are
969 * guaranteed that connector initialization is fully visible by anyone
970 * seeing mark->connector set.
971 */
972 WRITE_ONCE(mark->connector, conn);
973out_err:
974 spin_unlock(&conn->lock);
975 spin_unlock(&mark->lock);
976 return err;
977}
978
979/*
980 * Attach an initialized mark to a given group and fs object.
981 * These marks may be used for the fsnotify backend to determine which
982 * event types should be delivered to which group.
983 */
984int fsnotify_add_mark_locked(struct fsnotify_mark *mark,
985 void *obj, unsigned int obj_type,
986 int add_flags)
987{
988 struct fsnotify_group *group = mark->group;
989 int ret = 0;
990
991 fsnotify_group_assert_locked(group);
992
993 /*
994 * LOCKING ORDER!!!!
995 * group->mark_mutex
996 * mark->lock
997 * mark->connector->lock
998 */
999 spin_lock(&mark->lock);
1000 mark->flags |= FSNOTIFY_MARK_FLAG_ALIVE | FSNOTIFY_MARK_FLAG_ATTACHED;
1001
1002 list_add(&mark->g_list, &group->marks_list);
1003 fsnotify_get_mark(mark); /* for g_list */
1004 spin_unlock(&mark->lock);
1005
1006 ret = fsnotify_add_mark_list(mark, obj, obj_type, add_flags);
1007 if (ret)
1008 goto err;
1009
1010 fsnotify_recalc_mask(mark->connector);
1011
1012 return ret;
1013err:
1014 spin_lock(&mark->lock);
1015 mark->flags &= ~(FSNOTIFY_MARK_FLAG_ALIVE |
1016 FSNOTIFY_MARK_FLAG_ATTACHED);
1017 list_del_init(&mark->g_list);
1018 spin_unlock(&mark->lock);
1019
1020 fsnotify_put_mark(mark);
1021 return ret;
1022}
1023
1024int fsnotify_add_mark(struct fsnotify_mark *mark, void *obj,
1025 unsigned int obj_type, int add_flags)
1026{
1027 int ret;
1028 struct fsnotify_group *group = mark->group;
1029
1030 fsnotify_group_lock(group);
1031 ret = fsnotify_add_mark_locked(mark, obj, obj_type, add_flags);
1032 fsnotify_group_unlock(group);
1033 return ret;
1034}
1035EXPORT_SYMBOL_GPL(fsnotify_add_mark);
1036
1037/*
1038 * Given a list of marks, find the mark associated with given group. If found
1039 * take a reference to that mark and return it, else return NULL.
1040 */
1041struct fsnotify_mark *fsnotify_find_mark(void *obj, unsigned int obj_type,
1042 struct fsnotify_group *group)
1043{
1044 fsnotify_connp_t *connp = fsnotify_object_connp(obj, obj_type);
1045 struct fsnotify_mark_connector *conn;
1046 struct fsnotify_mark *mark;
1047
1048 if (!connp)
1049 return NULL;
1050
1051 conn = fsnotify_grab_connector(connp);
1052 if (!conn)
1053 return NULL;
1054
1055 hlist_for_each_entry(mark, &conn->list, obj_list) {
1056 if (mark->group == group &&
1057 (mark->flags & FSNOTIFY_MARK_FLAG_ATTACHED)) {
1058 fsnotify_get_mark(mark);
1059 spin_unlock(&conn->lock);
1060 return mark;
1061 }
1062 }
1063 spin_unlock(&conn->lock);
1064 return NULL;
1065}
1066EXPORT_SYMBOL_GPL(fsnotify_find_mark);
1067
1068/* Clear any marks in a group with given type mask */
1069void fsnotify_clear_marks_by_group(struct fsnotify_group *group,
1070 unsigned int obj_type)
1071{
1072 struct fsnotify_mark *lmark, *mark;
1073 LIST_HEAD(to_free);
1074 struct list_head *head = &to_free;
1075
1076 /* Skip selection step if we want to clear all marks. */
1077 if (obj_type == FSNOTIFY_OBJ_TYPE_ANY) {
1078 head = &group->marks_list;
1079 goto clear;
1080 }
1081 /*
1082 * We have to be really careful here. Anytime we drop mark_mutex, e.g.
1083 * fsnotify_clear_marks_by_inode() can come and free marks. Even in our
1084 * to_free list so we have to use mark_mutex even when accessing that
1085 * list. And freeing mark requires us to drop mark_mutex. So we can
1086 * reliably free only the first mark in the list. That's why we first
1087 * move marks to free to to_free list in one go and then free marks in
1088 * to_free list one by one.
1089 */
1090 fsnotify_group_lock(group);
1091 list_for_each_entry_safe(mark, lmark, &group->marks_list, g_list) {
1092 if (mark->connector->type == obj_type)
1093 list_move(&mark->g_list, &to_free);
1094 }
1095 fsnotify_group_unlock(group);
1096
1097clear:
1098 while (1) {
1099 fsnotify_group_lock(group);
1100 if (list_empty(head)) {
1101 fsnotify_group_unlock(group);
1102 break;
1103 }
1104 mark = list_first_entry(head, struct fsnotify_mark, g_list);
1105 fsnotify_get_mark(mark);
1106 fsnotify_detach_mark(mark);
1107 fsnotify_group_unlock(group);
1108 fsnotify_free_mark(mark);
1109 fsnotify_put_mark(mark);
1110 }
1111}
1112
1113/* Destroy all marks attached to an object via connector */
1114void fsnotify_destroy_marks(fsnotify_connp_t *connp)
1115{
1116 struct fsnotify_mark_connector *conn;
1117 struct fsnotify_mark *mark, *old_mark = NULL;
1118 void *objp;
1119 unsigned int type;
1120
1121 conn = fsnotify_grab_connector(connp);
1122 if (!conn)
1123 return;
1124 /*
1125 * New marks could be inserted before our cursor and survive the final
1126 * detachment of the connector from the object. Prevent additions to
1127 * this connector before dropping its lock.
1128 */
1129 conn->flags |= FSNOTIFY_CONN_FLAG_DESTROYING;
1130 /*
1131 * We have to be careful since we can race with e.g.
1132 * fsnotify_clear_marks_by_group() and once we drop the conn->lock, the
1133 * list can get modified. However we are holding mark reference and
1134 * thus our mark cannot be removed from obj_list so we can continue
1135 * iteration after regaining conn->lock.
1136 */
1137 hlist_for_each_entry(mark, &conn->list, obj_list) {
1138 fsnotify_get_mark(mark);
1139 spin_unlock(&conn->lock);
1140 if (old_mark)
1141 fsnotify_put_mark(old_mark);
1142 old_mark = mark;
1143 fsnotify_destroy_mark(mark, mark->group);
1144 spin_lock(&conn->lock);
1145 }
1146 /*
1147 * Detach list from object now so that we don't pin inode until all
1148 * mark references get dropped. It would lead to strange results such
1149 * as delaying inode deletion or blocking unmount.
1150 */
1151 objp = fsnotify_detach_connector_from_object(conn, &type);
1152 spin_unlock(&conn->lock);
1153 if (old_mark)
1154 fsnotify_put_mark(old_mark);
1155 fsnotify_drop_object(type, objp);
1156}
1157
1158/*
1159 * Nothing fancy, just initialize lists and locks and counters.
1160 */
1161void fsnotify_init_mark(struct fsnotify_mark *mark,
1162 struct fsnotify_group *group)
1163{
1164 memset(mark, 0, sizeof(*mark));
1165 spin_lock_init(&mark->lock);
1166 refcount_set(&mark->refcnt, 1);
1167 fsnotify_get_group(group);
1168 mark->group = group;
1169 WRITE_ONCE(mark->connector, NULL);
1170}
1171EXPORT_SYMBOL_GPL(fsnotify_init_mark);
1172
1173/*
1174 * Destroy all marks in destroy_list, waits for SRCU period to finish before
1175 * actually freeing marks.
1176 */
1177static void fsnotify_mark_destroy_workfn(struct work_struct *work)
1178{
1179 struct fsnotify_mark *mark, *next;
1180 struct list_head private_destroy_list;
1181
1182 spin_lock(&destroy_lock);
1183 /* exchange the list head */
1184 list_replace_init(&destroy_list, &private_destroy_list);
1185 spin_unlock(&destroy_lock);
1186
1187 synchronize_srcu(&fsnotify_mark_srcu);
1188
1189 list_for_each_entry_safe(mark, next, &private_destroy_list, g_list) {
1190 list_del_init(&mark->g_list);
1191 fsnotify_final_mark_destroy(mark);
1192 }
1193}
1194
1195/* Wait for all marks queued for destruction to be actually destroyed */
1196void fsnotify_wait_marks_destroyed(void)
1197{
1198 flush_delayed_work(&reaper_work);
1199}
1200EXPORT_SYMBOL_GPL(fsnotify_wait_marks_destroyed);
1201
1202__init void fsnotify_init_connector_caches(void)
1203{
1204 fsnotify_mark_connector_cachep = KMEM_CACHE(fsnotify_mark_connector,
1205 SLAB_PANIC);
1206 fsnotify_inode_mark_connector_cachep = KMEM_CACHE(
1207 fsnotify_inode_mark_connector,
1208 SLAB_PANIC);
1209}