projects/linux·blob master · refs

1565 lines · 42.8 KiB

1// SPDX-License-Identifier: GPL-2.0
2/*
3 * The NFSD open file cache.
4 *
5 * (c) 2015 - Jeff Layton <[email protected]>
6 *
7 * An nfsd_file object is a per-file collection of open state that binds
8 * together:
9 * - a struct file *
10 * - a user credential
11 * - a network namespace
12 * - a read-ahead context
13 * - monitoring for writeback errors
14 *
15 * nfsd_file objects are reference-counted. Consumers acquire a new
16 * object via the nfsd_file_acquire API. They manage their interest in
17 * the acquired object, and hence the object's reference count, via
18 * nfsd_file_get and nfsd_file_put. There are two varieties of nfsd_file
19 * object:
20 *
21 * * non-garbage-collected: When a consumer wants to precisely control
22 * the lifetime of a file's open state, it acquires a non-garbage-
23 * collected nfsd_file. The final nfsd_file_put releases the open
24 * state immediately.
25 *
26 * * garbage-collected: When a consumer does not control the lifetime
27 * of open state, it acquires a garbage-collected nfsd_file. The
28 * final nfsd_file_put allows the open state to linger for a period
29 * during which it may be re-used.
30 */
31
32#include <linux/hash.h>
33#include <linux/slab.h>
34#include <linux/file.h>
35#include <linux/pagemap.h>
36#include <linux/sched.h>
37#include <linux/list_lru.h>
38#include <linux/fsnotify_backend.h>
39#include <linux/fsnotify.h>
40#include <linux/seq_file.h>
41#include <linux/rhashtable.h>
42#include <linux/nfslocalio.h>
43
44#include "vfs.h"
45#include "nfsd.h"
46#include "nfsfh.h"
47#include "netns.h"
48#include "filecache.h"
49#include "trace.h"
50
51#define NFSD_LAUNDRETTE_DELAY (2 * HZ)
52
53#define NFSD_FILE_CACHE_UP (0)
54
55/* We only care about NFSD_MAY_READ/WRITE for this cache */
56#define NFSD_FILE_MAY_MASK (NFSD_MAY_READ|NFSD_MAY_WRITE|NFSD_MAY_LOCALIO)
57
58/* If the shrinker runs between calls to list_lru_walk_node() in
59 * nfsd_file_gc(), the "remaining" count will be wrong. This could
60 * result in premature freeing of some files. This may not matter much
61 * but is easy to fix with this spinlock which temporarily disables
62 * the shrinker.
63 *
64 * It also serializes callers of nfsd_file_dispose_list_delayed()
65 * against per-net shutdown.
66 */
67static DEFINE_SPINLOCK(nfsd_gc_lock);
68
69static DEFINE_PER_CPU(unsigned long, nfsd_file_cache_hits);
70static DEFINE_PER_CPU(unsigned long, nfsd_file_acquisitions);
71static DEFINE_PER_CPU(unsigned long, nfsd_file_allocations);
72static DEFINE_PER_CPU(unsigned long, nfsd_file_releases);
73static DEFINE_PER_CPU(unsigned long, nfsd_file_total_age);
74static DEFINE_PER_CPU(unsigned long, nfsd_file_evictions);
75
76static struct kmem_cache *nfsd_file_slab;
77static struct kmem_cache *nfsd_file_mark_slab;
78static struct list_lru nfsd_file_lru;
79static unsigned long nfsd_file_flags;
80static struct fsnotify_group *nfsd_file_fsnotify_group;
81static struct fsnotify_group *nfsd_dir_fsnotify_group;
82static struct delayed_work nfsd_filecache_laundrette;
83static struct rhltable nfsd_file_rhltable
84 ____cacheline_aligned_in_smp;
85
86static bool
87nfsd_match_cred(const struct cred *c1, const struct cred *c2)
88{
89 int i;
90
91 if (!uid_eq(c1->fsuid, c2->fsuid))
92 return false;
93 if (!gid_eq(c1->fsgid, c2->fsgid))
94 return false;
95 if (c1->group_info == NULL || c2->group_info == NULL)
96 return c1->group_info == c2->group_info;
97 if (c1->group_info->ngroups != c2->group_info->ngroups)
98 return false;
99 for (i = 0; i < c1->group_info->ngroups; i++) {
100 if (!gid_eq(c1->group_info->gid[i], c2->group_info->gid[i]))
101 return false;
102 }
103 return true;
104}
105
106static const struct rhashtable_params nfsd_file_rhash_params = {
107 .key_len = sizeof_field(struct nfsd_file, nf_inode),
108 .key_offset = offsetof(struct nfsd_file, nf_inode),
109 .head_offset = offsetof(struct nfsd_file, nf_rlist),
110
111 /*
112 * Start with a single page hash table to reduce resizing churn
113 * on light workloads.
114 */
115 .min_size = 256,
116 .automatic_shrinking = true,
117};
118
119static void
120nfsd_file_schedule_laundrette(void)
121{
122 if (test_bit(NFSD_FILE_CACHE_UP, &nfsd_file_flags))
123 queue_delayed_work(system_dfl_wq, &nfsd_filecache_laundrette,
124 NFSD_LAUNDRETTE_DELAY);
125}
126
127static void
128nfsd_file_slab_free(struct rcu_head *rcu)
129{
130 struct nfsd_file *nf = container_of(rcu, struct nfsd_file, nf_rcu);
131
132 put_cred(nf->nf_cred);
133 kmem_cache_free(nfsd_file_slab, nf);
134}
135
136static void
137nfsd_file_mark_free(struct fsnotify_mark *mark)
138{
139 struct nfsd_file_mark *nfm = container_of(mark, struct nfsd_file_mark,
140 nfm_mark);
141
142 kmem_cache_free(nfsd_file_mark_slab, nfm);
143}
144
145static struct nfsd_file_mark *
146nfsd_file_mark_get(struct nfsd_file_mark *nfm)
147{
148 if (!refcount_inc_not_zero(&nfm->nfm_ref))
149 return NULL;
150 return nfm;
151}
152
153static void
154nfsd_file_mark_put(struct nfsd_file_mark *nfm)
155{
156 if (refcount_dec_and_test(&nfm->nfm_ref)) {
157 fsnotify_destroy_mark(&nfm->nfm_mark, nfm->nfm_mark.group);
158 fsnotify_put_mark(&nfm->nfm_mark);
159 }
160}
161
162static struct nfsd_file_mark *
163nfsd_file_mark_find_or_create(struct inode *inode)
164{
165 struct nfsd_file_mark *nfm = NULL, *new;
166 struct fsnotify_group *group;
167 struct fsnotify_mark *mark;
168 int err;
169
170 group = S_ISDIR(inode->i_mode) ? nfsd_dir_fsnotify_group : nfsd_file_fsnotify_group;
171
172 do {
173 fsnotify_group_lock(group);
174 mark = fsnotify_find_inode_mark(inode, group);
175 if (mark) {
176 nfm = nfsd_file_mark_get(container_of(mark,
177 struct nfsd_file_mark,
178 nfm_mark));
179 fsnotify_group_unlock(group);
180 if (nfm) {
181 fsnotify_put_mark(mark);
182 break;
183 }
184 /* Avoid soft lockup race with nfsd_file_mark_put() */
185 fsnotify_destroy_mark(mark, group);
186 fsnotify_put_mark(mark);
187 } else {
188 fsnotify_group_unlock(group);
189 }
190
191 /* allocate a new nfm */
192 new = kmem_cache_alloc(nfsd_file_mark_slab, GFP_KERNEL);
193 if (!new)
194 return NULL;
195 fsnotify_init_mark(&new->nfm_mark, group);
196 new->nfm_mark.mask = FS_ATTRIB|FS_DELETE_SELF;
197 refcount_set(&new->nfm_ref, 1);
198 mutex_init(&new->nfm_recalc_mutex);
199
200 err = fsnotify_add_inode_mark(&new->nfm_mark, inode, 0);
201
202 /*
203 * If the add was successful, then return the object.
204 * Otherwise, we need to put the reference we hold on the
205 * nfm_mark. The fsnotify code will take a reference and put
206 * it on failure, so we can't just free it directly. It's also
207 * not safe to call fsnotify_destroy_mark on it as the
208 * mark->group will be NULL. Thus, we can't let the nfm_ref
209 * counter drive the destruction at this point.
210 */
211 if (likely(!err))
212 nfm = new;
213 else
214 fsnotify_put_mark(&new->nfm_mark);
215 } while (unlikely(err == -EEXIST));
216
217 return nfm;
218}
219
220static struct nfsd_file *
221nfsd_file_alloc(struct net *net, struct inode *inode, unsigned char need,
222 bool want_gc)
223{
224 struct nfsd_file *nf;
225
226 nf = kmem_cache_alloc(nfsd_file_slab, GFP_KERNEL);
227 if (unlikely(!nf))
228 return NULL;
229
230 this_cpu_inc(nfsd_file_allocations);
231 INIT_LIST_HEAD(&nf->nf_lru);
232 INIT_LIST_HEAD(&nf->nf_gc);
233 nf->nf_birthtime = ktime_get();
234 nf->nf_file = NULL;
235 nf->nf_cred = get_current_cred();
236 nf->nf_net = net;
237 nf->nf_flags = want_gc ?
238 BIT(NFSD_FILE_HASHED) | BIT(NFSD_FILE_PENDING) | BIT(NFSD_FILE_GC) :
239 BIT(NFSD_FILE_HASHED) | BIT(NFSD_FILE_PENDING);
240 nf->nf_inode = inode;
241 refcount_set(&nf->nf_ref, 1);
242 nf->nf_may = need;
243 nf->nf_mark = NULL;
244 nf->nf_dio_mem_align = 0;
245 nf->nf_dio_offset_align = 0;
246 nf->nf_dio_read_offset_align = 0;
247 return nf;
248}
249
250/**
251 * nfsd_file_check_write_error - check for writeback errors on a file
252 * @nf: nfsd_file to check for writeback errors
253 *
254 * Check whether a nfsd_file has an unseen error. Reset the write
255 * verifier if so.
256 */
257static void
258nfsd_file_check_write_error(struct nfsd_file *nf)
259{
260 struct file *file = nf->nf_file;
261
262 if ((file->f_mode & FMODE_WRITE) &&
263 filemap_check_wb_err(file->f_mapping, READ_ONCE(file->f_wb_err)))
264 nfsd_reset_write_verifier(net_generic(nf->nf_net, nfsd_net_id));
265}
266
267static void
268nfsd_file_hash_remove(struct nfsd_file *nf)
269{
270 trace_nfsd_file_unhash(nf);
271 rhltable_remove(&nfsd_file_rhltable, &nf->nf_rlist,
272 nfsd_file_rhash_params);
273}
274
275static bool
276nfsd_file_unhash(struct nfsd_file *nf)
277{
278 if (test_and_clear_bit(NFSD_FILE_HASHED, &nf->nf_flags)) {
279 nfsd_file_hash_remove(nf);
280 return true;
281 }
282 return false;
283}
284
285static void
286nfsd_file_free(struct nfsd_file *nf)
287{
288 s64 age = ktime_to_ms(ktime_sub(ktime_get(), nf->nf_birthtime));
289
290 trace_nfsd_file_free(nf);
291
292 this_cpu_inc(nfsd_file_releases);
293 this_cpu_add(nfsd_file_total_age, age);
294
295 nfsd_file_unhash(nf);
296 if (nf->nf_mark)
297 nfsd_file_mark_put(nf->nf_mark);
298 if (nf->nf_file) {
299 nfsd_file_check_write_error(nf);
300 nfsd_filp_close(nf->nf_file);
301 }
302
303 /*
304 * If this item is still linked via nf_lru, that's a bug.
305 * WARN and leak it to preserve system stability.
306 */
307 if (WARN_ON_ONCE(!list_empty(&nf->nf_lru)))
308 return;
309
310 call_rcu(&nf->nf_rcu, nfsd_file_slab_free);
311}
312
313static bool
314nfsd_file_check_writeback(struct nfsd_file *nf)
315{
316 struct file *file = nf->nf_file;
317 struct address_space *mapping;
318
319 /* File not open for write? */
320 if (!(file->f_mode & FMODE_WRITE))
321 return false;
322
323 /*
324 * Some filesystems (e.g. NFS) flush all dirty data on close.
325 * On others, there is no need to wait for writeback.
326 */
327 if (!(file_inode(file)->i_sb->s_export_op->flags & EXPORT_OP_FLUSH_ON_CLOSE))
328 return false;
329
330 mapping = file->f_mapping;
331 return mapping_tagged(mapping, PAGECACHE_TAG_DIRTY) ||
332 mapping_tagged(mapping, PAGECACHE_TAG_WRITEBACK);
333}
334
335static void nfsd_file_lru_add(struct nfsd_file *nf)
336{
337 refcount_inc(&nf->nf_ref);
338 if (list_lru_add_obj(&nfsd_file_lru, &nf->nf_lru))
339 trace_nfsd_file_lru_add(nf);
340 else {
341 refcount_dec(&nf->nf_ref);
342 WARN_ON_ONCE(1);
343 return;
344 }
345 nfsd_file_schedule_laundrette();
346}
347
348static bool nfsd_file_lru_remove(struct nfsd_file *nf)
349{
350 if (list_lru_del_obj(&nfsd_file_lru, &nf->nf_lru)) {
351 trace_nfsd_file_lru_del(nf);
352 return true;
353 }
354 return false;
355}
356
357struct nfsd_file *
358nfsd_file_get(struct nfsd_file *nf)
359{
360 if (nf && refcount_inc_not_zero(&nf->nf_ref))
361 return nf;
362 return NULL;
363}
364
365/**
366 * nfsd_file_put - put the reference to a nfsd_file
367 * @nf: nfsd_file of which to put the reference
368 *
369 * Put a reference to a nfsd_file. In the non-GC case, we just put the
370 * reference immediately. In the GC case, if the reference would be
371 * the last one, the put it on the LRU instead to be cleaned up later.
372 */
373void
374nfsd_file_put(struct nfsd_file *nf)
375{
376 might_sleep();
377 trace_nfsd_file_put(nf);
378
379 if (test_bit(NFSD_FILE_GC, &nf->nf_flags) &&
380 test_bit(NFSD_FILE_HASHED, &nf->nf_flags)) {
381 set_bit(NFSD_FILE_REFERENCED, &nf->nf_flags);
382 set_bit(NFSD_FILE_RECENT, &nf->nf_flags);
383 }
384
385 if (refcount_dec_and_test(&nf->nf_ref))
386 nfsd_file_free(nf);
387}
388
389/**
390 * nfsd_file_put_local - put nfsd_file reference and arm nfsd_net_put in caller
391 * @pnf: nfsd_file of which to put the reference
392 *
393 * First save the associated net to return to caller, then put
394 * the reference of the nfsd_file.
395 */
396struct net *
397nfsd_file_put_local(struct nfsd_file __rcu **pnf)
398{
399 struct nfsd_file *nf;
400 struct net *net = NULL;
401
402 nf = unrcu_pointer(xchg(pnf, NULL));
403 if (nf) {
404 net = nf->nf_net;
405 nfsd_file_put(nf);
406 }
407 return net;
408}
409
410/**
411 * nfsd_file_file - get the backing file of an nfsd_file
412 * @nf: nfsd_file of which to access the backing file.
413 *
414 * Return backing file for @nf.
415 */
416struct file *
417nfsd_file_file(struct nfsd_file *nf)
418{
419 return nf->nf_file;
420}
421
422static void
423nfsd_file_dispose_list(struct list_head *dispose)
424{
425 struct nfsd_file *nf;
426
427 while (!list_empty(dispose)) {
428 nf = list_first_entry(dispose, struct nfsd_file, nf_gc);
429 list_del_init(&nf->nf_gc);
430 nfsd_file_free(nf);
431 }
432}
433
434/**
435 * nfsd_file_dispose_list_delayed - queue dead files for nfsd thread disposal
436 * @dispose: list of nfsd_files to be disposed
437 *
438 * Transfers each file to the dispose list in its nfsd_net and wakes an nfsd
439 * thread to do the actual close. This keeps the cost of fput() in the nfsd
440 * threads rather than in the shrinker or GC worker.
441 *
442 * All callers must hold nfsd_gc_lock, so that nfsd_file_cache_shutdown_net()
443 * can synchronize against them before draining the per-net dispose list.
444 * This guarantees nf_net is still live when we call net_generic().
445 */
446static void
447nfsd_file_dispose_list_delayed(struct list_head *dispose)
448{
449 lockdep_assert_held(&nfsd_gc_lock);
450
451 while (!list_empty(dispose)) {
452 struct nfsd_file *nf = list_first_entry(dispose,
453 struct nfsd_file, nf_gc);
454 struct nfsd_net *nn = net_generic(nf->nf_net, nfsd_net_id);
455 struct svc_serv *serv;
456
457 spin_lock(&nn->fcache_dispose_lock);
458 list_move_tail(&nf->nf_gc, &nn->fcache_dispose_list);
459 spin_unlock(&nn->fcache_dispose_lock);
460
461 /*
462 * The filecache laundrette is shut down after the
463 * nn->nfsd_serv pointer is cleared, but before the
464 * svc_serv is freed.
465 */
466 serv = nn->nfsd_serv;
467 if (serv)
468 svc_wake_up(serv);
469 }
470}
471
472/**
473 * nfsd_file_net_dispose - deal with nfsd_files waiting to be disposed.
474 * @nn: nfsd_net in which to find files to be disposed.
475 *
476 * When files held open for nfsv3 are removed from the filecache, whether
477 * due to memory pressure or garbage collection, they are queued to
478 * a per-net-ns queue. This function completes the disposal, either
479 * directly or by waking another nfsd thread to help with the work.
480 */
481void nfsd_file_net_dispose(struct nfsd_net *nn)
482{
483 if (!list_empty(&nn->fcache_dispose_list)) {
484 LIST_HEAD(dispose);
485 int i;
486
487 spin_lock(&nn->fcache_dispose_lock);
488 for (i = 0; i < 8 && !list_empty(&nn->fcache_dispose_list); i++)
489 list_move(nn->fcache_dispose_list.next, &dispose);
490 spin_unlock(&nn->fcache_dispose_lock);
491 if (!list_empty(&nn->fcache_dispose_list)) {
492 /*
493 * Wake up another thread to share the work
494 * *before* doing any actual disposing.
495 *
496 * The filecache laundrette is shut down after
497 * the nn->nfsd_serv pointer is cleared, but
498 * before the svc_serv is freed.
499 */
500 struct svc_serv *serv = nn->nfsd_serv;
501
502 if (serv)
503 svc_wake_up(serv);
504 }
505 nfsd_file_dispose_list(&dispose);
506 }
507}
508
509/**
510 * nfsd_file_lru_cb - Examine an entry on the LRU list
511 * @item: LRU entry to examine
512 * @lru: controlling LRU
513 * @arg: dispose list
514 *
515 * Return values:
516 * %LRU_REMOVED: @item was removed from the LRU
517 * %LRU_ROTATE: @item is to be moved to the LRU tail
518 * %LRU_SKIP: @item cannot be evicted
519 */
520static enum lru_status
521nfsd_file_lru_cb(struct list_head *item, struct list_lru_one *lru,
522 void *arg)
523{
524 struct list_head *head = arg;
525 struct nfsd_file *nf = list_entry(item, struct nfsd_file, nf_lru);
526
527 /* We should only be dealing with GC entries here */
528 WARN_ON_ONCE(!test_bit(NFSD_FILE_GC, &nf->nf_flags));
529
530 /*
531 * Don't throw out files that are still undergoing I/O or
532 * that have uncleared errors pending.
533 */
534 if (nfsd_file_check_writeback(nf)) {
535 trace_nfsd_file_gc_writeback(nf);
536 return LRU_SKIP;
537 }
538
539 /* If it was recently added to the list, skip it */
540 if (test_and_clear_bit(NFSD_FILE_REFERENCED, &nf->nf_flags)) {
541 trace_nfsd_file_gc_referenced(nf);
542 return LRU_ROTATE;
543 }
544
545 /*
546 * Put the reference held on behalf of the LRU if it is the last
547 * reference, else rotate.
548 */
549 if (!refcount_dec_if_one(&nf->nf_ref)) {
550 trace_nfsd_file_gc_in_use(nf);
551 return LRU_ROTATE;
552 }
553
554 /* Refcount went to zero. Unhash it and queue it to the dispose list */
555 nfsd_file_unhash(nf);
556 list_lru_isolate(lru, &nf->nf_lru);
557 list_add(&nf->nf_gc, head);
558 this_cpu_inc(nfsd_file_evictions);
559 trace_nfsd_file_gc_disposed(nf);
560 return LRU_REMOVED;
561}
562
563static enum lru_status
564nfsd_file_gc_cb(struct list_head *item, struct list_lru_one *lru,
565 void *arg)
566{
567 struct nfsd_file *nf = list_entry(item, struct nfsd_file, nf_lru);
568
569 if (test_and_clear_bit(NFSD_FILE_RECENT, &nf->nf_flags)) {
570 /*
571 * "REFERENCED" really means "should be at the end of the
572 * LRU. As we are putting it there we can clear the flag.
573 */
574 clear_bit(NFSD_FILE_REFERENCED, &nf->nf_flags);
575 trace_nfsd_file_gc_aged(nf);
576 return LRU_ROTATE;
577 }
578 return nfsd_file_lru_cb(item, lru, arg);
579}
580
581static void
582nfsd_file_gc(void)
583{
584 unsigned long ret = 0;
585 LIST_HEAD(dispose);
586 int nid;
587
588 spin_lock(&nfsd_gc_lock);
589 for_each_node_state(nid, N_NORMAL_MEMORY) {
590 unsigned long remaining = list_lru_count_node(&nfsd_file_lru, nid);
591
592 while (remaining > 0) {
593 unsigned long nr = min(remaining, NFSD_FILE_GC_BATCH);
594
595 remaining -= nr;
596 ret += list_lru_walk_node(&nfsd_file_lru, nid, nfsd_file_gc_cb,
597 &dispose, &nr);
598 if (nr)
599 /* walk aborted early */
600 remaining = 0;
601 }
602 }
603 nfsd_file_dispose_list_delayed(&dispose);
604 spin_unlock(&nfsd_gc_lock);
605 trace_nfsd_file_gc_removed(ret, list_lru_count(&nfsd_file_lru));
606}
607
608static void
609nfsd_file_gc_worker(struct work_struct *work)
610{
611 if (list_lru_count(&nfsd_file_lru))
612 nfsd_file_gc();
613 nfsd_file_schedule_laundrette();
614}
615
616static unsigned long
617nfsd_file_lru_count(struct shrinker *s, struct shrink_control *sc)
618{
619 return list_lru_count(&nfsd_file_lru);
620}
621
622static unsigned long
623nfsd_file_lru_scan(struct shrinker *s, struct shrink_control *sc)
624{
625 LIST_HEAD(dispose);
626 unsigned long ret;
627
628 if (!spin_trylock(&nfsd_gc_lock))
629 return SHRINK_STOP;
630
631 ret = list_lru_shrink_walk(&nfsd_file_lru, sc,
632 nfsd_file_lru_cb, &dispose);
633 nfsd_file_dispose_list_delayed(&dispose);
634 spin_unlock(&nfsd_gc_lock);
635 trace_nfsd_file_shrinker_removed(ret, list_lru_count(&nfsd_file_lru));
636 return ret;
637}
638
639static struct shrinker *nfsd_file_shrinker;
640
641/**
642 * nfsd_file_cond_queue - conditionally unhash and queue a nfsd_file
643 * @nf: nfsd_file to attempt to queue
644 * @dispose: private list to queue successfully-put objects
645 *
646 * Unhash an nfsd_file, try to get a reference to it, and then put that
647 * reference. If it's the last reference, queue it to the dispose list.
648 */
649static void
650nfsd_file_cond_queue(struct nfsd_file *nf, struct list_head *dispose)
651 __must_hold(RCU)
652{
653 int decrement = 1;
654
655 /* If we raced with someone else unhashing, ignore it */
656 if (!nfsd_file_unhash(nf))
657 return;
658
659 /* If we can't get a reference, ignore it */
660 if (!nfsd_file_get(nf))
661 return;
662
663 /* Extra decrement if we remove from the LRU */
664 if (nfsd_file_lru_remove(nf))
665 ++decrement;
666
667 /* If refcount goes to 0, then put on the dispose list */
668 if (refcount_sub_and_test(decrement, &nf->nf_ref)) {
669 list_add(&nf->nf_gc, dispose);
670 trace_nfsd_file_closing(nf);
671 }
672}
673
674/**
675 * nfsd_file_queue_for_close: try to close out any open nfsd_files for an inode
676 * @inode: inode on which to close out nfsd_files
677 * @dispose: list on which to gather nfsd_files to close out
678 *
679 * An nfsd_file represents a struct file being held open on behalf of nfsd.
680 * An open file however can block other activity (such as leases), or cause
681 * undesirable behavior (e.g. spurious silly-renames when reexporting NFS).
682 *
683 * This function is intended to find open nfsd_files when this sort of
684 * conflicting access occurs and then attempt to close those files out.
685 *
686 * Populates the dispose list with entries that have already had their
687 * refcounts go to zero. The actual free of an nfsd_file can be expensive,
688 * so we leave it up to the caller whether it wants to wait or not.
689 */
690static void
691nfsd_file_queue_for_close(struct inode *inode, struct list_head *dispose)
692{
693 struct rhlist_head *tmp, *list;
694 struct nfsd_file *nf;
695
696 rcu_read_lock();
697 list = rhltable_lookup(&nfsd_file_rhltable, &inode,
698 nfsd_file_rhash_params);
699 rhl_for_each_entry_rcu(nf, tmp, list, nf_rlist) {
700 if (!test_bit(NFSD_FILE_GC, &nf->nf_flags))
701 continue;
702 nfsd_file_cond_queue(nf, dispose);
703 }
704 rcu_read_unlock();
705}
706
707/**
708 * nfsd_file_close_inode - attempt a deferred close of a nfsd_file
709 * @inode: inode of the file to attempt to remove
710 *
711 * Close out any open nfsd_files that can be reaped for @inode. The
712 * actual freeing is deferred to the nfsd service threads.
713 *
714 * This is used by the fsnotify callbacks and setlease notifier.
715 */
716static void
717nfsd_file_close_inode(struct inode *inode)
718{
719 LIST_HEAD(dispose);
720
721 spin_lock(&nfsd_gc_lock);
722 nfsd_file_queue_for_close(inode, &dispose);
723 nfsd_file_dispose_list_delayed(&dispose);
724 spin_unlock(&nfsd_gc_lock);
725}
726
727/**
728 * nfsd_file_close_inode_sync - attempt to forcibly close a nfsd_file
729 * @inode: inode of the file to attempt to remove
730 *
731 * Close out any open nfsd_files that can be reaped for @inode. The
732 * nfsd_files are closed out synchronously.
733 *
734 * This is called from nfsd_rename and nfsd_unlink to avoid silly-renames
735 * when reexporting NFS.
736 */
737void
738nfsd_file_close_inode_sync(struct inode *inode)
739{
740 LIST_HEAD(dispose);
741
742 trace_nfsd_file_close(inode);
743
744 nfsd_file_queue_for_close(inode, &dispose);
745 nfsd_file_dispose_list(&dispose);
746}
747
748/**
749 * nfsd_file_close_export - close cached file handles for an export
750 * @net: net namespace in which to operate
751 * @path: export path whose cached files should be closed
752 *
753 * Close out GC-managed nfsd_file entries whose underlying file is on
754 * the same filesystem as, and a descendant of, @path. nfsd_file
755 * entries do not carry an export reference, so the check uses the
756 * file's dentry ancestry. False positives (closing a cached handle
757 * that did not originate from the target export) are harmless -- the
758 * handle is simply reopened on the next access.
759 *
760 * Called from the NFSD_CMD_UNLOCK_EXPORT handler before revoking
761 * NFSv4 state, to ensure that cached file handles do not hold the
762 * filesystem busy.
763 */
764void nfsd_file_close_export(struct net *net, const struct path *path)
765{
766 struct rhashtable_iter iter;
767 struct nfsd_file *nf;
768 LIST_HEAD(dispose);
769
770 rhltable_walk_enter(&nfsd_file_rhltable, &iter);
771 do {
772 rhashtable_walk_start(&iter);
773
774 nf = rhashtable_walk_next(&iter);
775 while (!IS_ERR_OR_NULL(nf)) {
776 if (nf->nf_net == net &&
777 test_bit(NFSD_FILE_GC, &nf->nf_flags) &&
778 nf->nf_file &&
779 file_inode(nf->nf_file)->i_sb ==
780 path->dentry->d_sb &&
781 is_subdir(nf->nf_file->f_path.dentry,
782 path->dentry))
783 nfsd_file_cond_queue(nf, &dispose);
784 nf = rhashtable_walk_next(&iter);
785 }
786
787 rhashtable_walk_stop(&iter);
788 } while (nf == ERR_PTR(-EAGAIN));
789 rhashtable_walk_exit(&iter);
790
791 nfsd_file_dispose_list(&dispose);
792}
793
794static int
795nfsd_file_lease_notifier_call(struct notifier_block *nb, unsigned long arg,
796 void *data)
797{
798 struct file_lease *fl = data;
799
800 /* Only close files for F_SETLEASE leases */
801 if (fl->c.flc_flags & FL_LEASE)
802 nfsd_file_close_inode(file_inode(fl->c.flc_file));
803 return 0;
804}
805
806static struct notifier_block nfsd_file_lease_notifier = {
807 .notifier_call = nfsd_file_lease_notifier_call,
808};
809
810static int
811nfsd_file_fsnotify_handle_event(struct fsnotify_mark *mark, u32 mask,
812 struct inode *inode, struct inode *dir,
813 const struct qstr *name, u32 cookie)
814{
815 if (WARN_ON_ONCE(!inode))
816 return 0;
817
818 trace_nfsd_file_fsnotify_handle_event(inode, mask);
819
820 /* Should be no marks on non-regular files */
821 if (!S_ISREG(inode->i_mode)) {
822 WARN_ON_ONCE(1);
823 return 0;
824 }
825
826 /* don't close files if this was not the last link */
827 if (mask & FS_ATTRIB) {
828 if (inode->i_nlink)
829 return 0;
830 }
831
832 nfsd_file_close_inode(inode);
833 return 0;
834}
835
836#ifdef CONFIG_NFSD_V4
837static int
838nfsd_dir_fsnotify_handle_event(struct fsnotify_group *group, u32 mask,
839 const void *data, int data_type, struct inode *dir,
840 const struct qstr *name, u32 cookie,
841 struct fsnotify_iter_info *iter_info)
842{
843 return nfsd_handle_dir_event(mask, dir, data, data_type, name);
844}
845#else
846static int
847nfsd_dir_fsnotify_handle_event(struct fsnotify_group *group, u32 mask,
848 const void *data, int data_type, struct inode *dir,
849 const struct qstr *name, u32 cookie,
850 struct fsnotify_iter_info *iter_info)
851{
852 return 0;
853}
854#endif
855
856static const struct fsnotify_ops nfsd_file_fsnotify_ops = {
857 .handle_inode_event = nfsd_file_fsnotify_handle_event,
858 .free_mark = nfsd_file_mark_free,
859};
860
861static const struct fsnotify_ops nfsd_dir_fsnotify_ops = {
862 .handle_event = nfsd_dir_fsnotify_handle_event,
863 .free_mark = nfsd_file_mark_free,
864};
865
866int
867nfsd_file_cache_init(void)
868{
869 int ret;
870
871 lockdep_assert_held(&nfsd_mutex);
872 if (test_and_set_bit(NFSD_FILE_CACHE_UP, &nfsd_file_flags) == 1)
873 return 0;
874
875 ret = rhltable_init(&nfsd_file_rhltable, &nfsd_file_rhash_params);
876 if (ret)
877 goto out;
878
879 ret = -ENOMEM;
880 nfsd_file_slab = KMEM_CACHE(nfsd_file, 0);
881 if (!nfsd_file_slab) {
882 pr_err("nfsd: unable to create nfsd_file_slab\n");
883 goto out_err;
884 }
885
886 nfsd_file_mark_slab = KMEM_CACHE(nfsd_file_mark, 0);
887 if (!nfsd_file_mark_slab) {
888 pr_err("nfsd: unable to create nfsd_file_mark_slab\n");
889 goto out_err;
890 }
891
892 ret = list_lru_init(&nfsd_file_lru);
893 if (ret) {
894 pr_err("nfsd: failed to init nfsd_file_lru: %d\n", ret);
895 goto out_err;
896 }
897
898 nfsd_file_shrinker = shrinker_alloc(0, "nfsd-filecache");
899 if (!nfsd_file_shrinker) {
900 ret = -ENOMEM;
901 pr_err("nfsd: failed to allocate nfsd_file_shrinker\n");
902 goto out_lru;
903 }
904
905 nfsd_file_shrinker->count_objects = nfsd_file_lru_count;
906 nfsd_file_shrinker->scan_objects = nfsd_file_lru_scan;
907 nfsd_file_shrinker->seeks = 1;
908
909 shrinker_register(nfsd_file_shrinker);
910
911 ret = lease_register_notifier(&nfsd_file_lease_notifier);
912 if (ret) {
913 pr_err("nfsd: unable to register lease notifier: %d\n", ret);
914 goto out_shrinker;
915 }
916
917 nfsd_file_fsnotify_group = fsnotify_alloc_group(&nfsd_file_fsnotify_ops, 0);
918 if (IS_ERR(nfsd_file_fsnotify_group)) {
919 pr_err("nfsd: unable to create fsnotify group: %ld\n",
920 PTR_ERR(nfsd_file_fsnotify_group));
921 ret = PTR_ERR(nfsd_file_fsnotify_group);
922 nfsd_file_fsnotify_group = NULL;
923 goto out_notifier;
924 }
925
926 nfsd_dir_fsnotify_group = fsnotify_alloc_group(&nfsd_dir_fsnotify_ops, 0);
927 if (IS_ERR(nfsd_dir_fsnotify_group)) {
928 pr_err("nfsd: unable to create fsnotify group: %ld\n",
929 PTR_ERR(nfsd_dir_fsnotify_group));
930 ret = PTR_ERR(nfsd_dir_fsnotify_group);
931 nfsd_dir_fsnotify_group = NULL;
932 goto out_notify_group;
933 }
934
935 INIT_DELAYED_WORK(&nfsd_filecache_laundrette, nfsd_file_gc_worker);
936out:
937 if (ret)
938 clear_bit(NFSD_FILE_CACHE_UP, &nfsd_file_flags);
939 return ret;
940out_notify_group:
941 fsnotify_put_group(nfsd_file_fsnotify_group);
942 nfsd_file_fsnotify_group = NULL;
943out_notifier:
944 lease_unregister_notifier(&nfsd_file_lease_notifier);
945out_shrinker:
946 shrinker_free(nfsd_file_shrinker);
947out_lru:
948 list_lru_destroy(&nfsd_file_lru);
949out_err:
950 kmem_cache_destroy(nfsd_file_slab);
951 nfsd_file_slab = NULL;
952 kmem_cache_destroy(nfsd_file_mark_slab);
953 nfsd_file_mark_slab = NULL;
954 rhltable_destroy(&nfsd_file_rhltable);
955 goto out;
956}
957
958/**
959 * __nfsd_file_cache_purge: clean out the cache for shutdown
960 * @net: net-namespace to shut down the cache (may be NULL)
961 *
962 * Walk the nfsd_file cache and close out any that match @net. If @net is NULL,
963 * then close out everything. Called when an nfsd instance is being shut down,
964 * and when the exports table is flushed.
965 */
966static void
967__nfsd_file_cache_purge(struct net *net)
968{
969 struct rhashtable_iter iter;
970 struct nfsd_file *nf;
971 LIST_HEAD(dispose);
972
973#if IS_ENABLED(CONFIG_NFS_LOCALIO)
974 if (net) {
975 struct nfsd_net *nn = net_generic(net, nfsd_net_id);
976 nfs_localio_invalidate_clients(&nn->local_clients,
977 &nn->local_clients_lock);
978 }
979#endif
980
981 rhltable_walk_enter(&nfsd_file_rhltable, &iter);
982 do {
983 rhashtable_walk_start(&iter);
984
985 nf = rhashtable_walk_next(&iter);
986 while (!IS_ERR_OR_NULL(nf)) {
987 if (!net || nf->nf_net == net)
988 nfsd_file_cond_queue(nf, &dispose);
989 nf = rhashtable_walk_next(&iter);
990 }
991
992 rhashtable_walk_stop(&iter);
993 } while (nf == ERR_PTR(-EAGAIN));
994 rhashtable_walk_exit(&iter);
995
996 nfsd_file_dispose_list(&dispose);
997}
998
999int
1000nfsd_file_cache_start_net(struct net *net)
1001{
1002 struct nfsd_net *nn = net_generic(net, nfsd_net_id);
1003
1004 spin_lock_init(&nn->fcache_dispose_lock);
1005 INIT_LIST_HEAD(&nn->fcache_dispose_list);
1006 return 0;
1007}
1008
1009/**
1010 * nfsd_file_cache_purge - Remove all cache items associated with @net
1011 * @net: target net namespace
1012 *
1013 */
1014void
1015nfsd_file_cache_purge(struct net *net)
1016{
1017 lockdep_assert_held(&nfsd_mutex);
1018 if (test_bit(NFSD_FILE_CACHE_UP, &nfsd_file_flags) == 1)
1019 __nfsd_file_cache_purge(net);
1020}
1021
1022void
1023nfsd_file_cache_shutdown_net(struct net *net)
1024{
1025 struct nfsd_net *nn = net_generic(net, nfsd_net_id);
1026
1027 nfsd_file_cache_purge(net);
1028 /*
1029 * Ensure any in-progress shrinker, GC, or fsnotify/lease callback
1030 * (all of which hold nfsd_gc_lock while calling
1031 * nfsd_file_dispose_list_delayed()) has fully completed before
1032 * draining the per-net dispose list.
1033 */
1034 spin_lock(&nfsd_gc_lock);
1035 spin_unlock(&nfsd_gc_lock);
1036 nfsd_file_dispose_list(&nn->fcache_dispose_list);
1037}
1038
1039void
1040nfsd_file_cache_shutdown(void)
1041{
1042 int i;
1043
1044 lockdep_assert_held(&nfsd_mutex);
1045 if (test_and_clear_bit(NFSD_FILE_CACHE_UP, &nfsd_file_flags) == 0)
1046 return;
1047
1048 lease_unregister_notifier(&nfsd_file_lease_notifier);
1049 shrinker_free(nfsd_file_shrinker);
1050 /*
1051 * make sure all callers of nfsd_file_lru_cb are done before
1052 * calling nfsd_file_cache_purge
1053 */
1054 cancel_delayed_work_sync(&nfsd_filecache_laundrette);
1055 __nfsd_file_cache_purge(NULL);
1056 list_lru_destroy(&nfsd_file_lru);
1057 rcu_barrier();
1058 fsnotify_put_group(nfsd_file_fsnotify_group);
1059 nfsd_file_fsnotify_group = NULL;
1060 fsnotify_put_group(nfsd_dir_fsnotify_group);
1061 nfsd_dir_fsnotify_group = NULL;
1062 kmem_cache_destroy(nfsd_file_slab);
1063 nfsd_file_slab = NULL;
1064 fsnotify_wait_marks_destroyed();
1065 kmem_cache_destroy(nfsd_file_mark_slab);
1066 nfsd_file_mark_slab = NULL;
1067 rhltable_destroy(&nfsd_file_rhltable);
1068
1069 for_each_possible_cpu(i) {
1070 per_cpu(nfsd_file_cache_hits, i) = 0;
1071 per_cpu(nfsd_file_acquisitions, i) = 0;
1072 per_cpu(nfsd_file_allocations, i) = 0;
1073 per_cpu(nfsd_file_releases, i) = 0;
1074 per_cpu(nfsd_file_total_age, i) = 0;
1075 per_cpu(nfsd_file_evictions, i) = 0;
1076 }
1077}
1078
1079static struct nfsd_file *
1080nfsd_file_lookup_locked(const struct net *net, const struct cred *cred,
1081 struct inode *inode, unsigned char need,
1082 bool want_gc)
1083{
1084 struct rhlist_head *tmp, *list;
1085 struct nfsd_file *nf;
1086
1087 list = rhltable_lookup(&nfsd_file_rhltable, &inode,
1088 nfsd_file_rhash_params);
1089 rhl_for_each_entry_rcu(nf, tmp, list, nf_rlist) {
1090 if (nf->nf_may != need)
1091 continue;
1092 if (nf->nf_net != net)
1093 continue;
1094 if (!nfsd_match_cred(nf->nf_cred, cred))
1095 continue;
1096 if (test_bit(NFSD_FILE_GC, &nf->nf_flags) != want_gc)
1097 continue;
1098 if (test_bit(NFSD_FILE_HASHED, &nf->nf_flags) == 0)
1099 continue;
1100
1101 if (!nfsd_file_get(nf))
1102 continue;
1103 return nf;
1104 }
1105 return NULL;
1106}
1107
1108/**
1109 * nfsd_file_is_cached - are there any cached open files for this inode?
1110 * @inode: inode to check
1111 *
1112 * The lookup matches inodes in all net namespaces and is atomic wrt
1113 * nfsd_file_acquire().
1114 *
1115 * Return values:
1116 * %true: filecache contains at least one file matching this inode
1117 * %false: filecache contains no files matching this inode
1118 */
1119bool
1120nfsd_file_is_cached(struct inode *inode)
1121{
1122 struct rhlist_head *tmp, *list;
1123 struct nfsd_file *nf;
1124 bool ret = false;
1125
1126 rcu_read_lock();
1127 list = rhltable_lookup(&nfsd_file_rhltable, &inode,
1128 nfsd_file_rhash_params);
1129 rhl_for_each_entry_rcu(nf, tmp, list, nf_rlist)
1130 if (test_bit(NFSD_FILE_GC, &nf->nf_flags)) {
1131 ret = true;
1132 break;
1133 }
1134 rcu_read_unlock();
1135
1136 trace_nfsd_file_is_cached(inode, (int)ret);
1137 return ret;
1138}
1139
1140static __be32
1141nfsd_file_get_dio_attrs(const struct svc_fh *fhp, struct nfsd_file *nf)
1142{
1143 struct inode *inode = file_inode(nf->nf_file);
1144 struct kstat stat;
1145 __be32 status;
1146
1147 /* Currently only need to get DIO alignment info for regular files */
1148 if (!S_ISREG(inode->i_mode))
1149 return nfs_ok;
1150
1151 status = fh_getattr(fhp, &stat);
1152 if (status != nfs_ok)
1153 return status;
1154
1155 trace_nfsd_file_get_dio_attrs(inode, &stat);
1156
1157 if (stat.result_mask & STATX_DIOALIGN) {
1158 nf->nf_dio_mem_align = stat.dio_mem_align;
1159 nf->nf_dio_offset_align = stat.dio_offset_align;
1160 }
1161 if (stat.result_mask & STATX_DIO_READ_ALIGN)
1162 nf->nf_dio_read_offset_align = stat.dio_read_offset_align;
1163 else
1164 nf->nf_dio_read_offset_align = nf->nf_dio_offset_align;
1165
1166 return nfs_ok;
1167}
1168
1169static __be32
1170nfsd_file_do_acquire(struct svc_rqst *rqstp, struct net *net,
1171 struct svc_cred *cred,
1172 struct auth_domain *client,
1173 struct svc_fh *fhp,
1174 unsigned int may_flags, struct file *file,
1175 umode_t type, bool want_gc, struct nfsd_file **pnf)
1176{
1177 unsigned char need = may_flags & NFSD_FILE_MAY_MASK;
1178 struct nfsd_file *new, *nf;
1179 bool stale_retry = true;
1180 bool open_retry = true;
1181 struct inode *inode;
1182 __be32 status;
1183 int ret;
1184
1185retry:
1186 if (rqstp)
1187 status = fh_verify(rqstp, fhp, type,
1188 may_flags|NFSD_MAY_OWNER_OVERRIDE);
1189 else
1190 status = fh_verify_local(net, cred, client, fhp, type,
1191 may_flags|NFSD_MAY_OWNER_OVERRIDE);
1192
1193 if (status != nfs_ok)
1194 return status;
1195 inode = d_inode(fhp->fh_dentry);
1196
1197 rcu_read_lock();
1198 nf = nfsd_file_lookup_locked(net, current_cred(), inode, need, want_gc);
1199 rcu_read_unlock();
1200
1201 if (nf)
1202 goto wait_for_construction;
1203
1204 new = nfsd_file_alloc(net, inode, need, want_gc);
1205 if (!new) {
1206 status = nfserr_jukebox;
1207 goto out;
1208 }
1209
1210 rcu_read_lock();
1211 spin_lock(&inode->i_lock);
1212 nf = nfsd_file_lookup_locked(net, current_cred(), inode, need, want_gc);
1213 if (unlikely(nf)) {
1214 spin_unlock(&inode->i_lock);
1215 rcu_read_unlock();
1216 nfsd_file_free(new);
1217 goto wait_for_construction;
1218 }
1219 nf = new;
1220 ret = rhltable_insert(&nfsd_file_rhltable, &nf->nf_rlist,
1221 nfsd_file_rhash_params);
1222 spin_unlock(&inode->i_lock);
1223 rcu_read_unlock();
1224 if (likely(ret == 0))
1225 goto open_file;
1226
1227 trace_nfsd_file_insert_err(rqstp, inode, may_flags, ret);
1228 status = nfserr_jukebox;
1229 goto construction_err;
1230
1231wait_for_construction:
1232 wait_on_bit(&nf->nf_flags, NFSD_FILE_PENDING, TASK_UNINTERRUPTIBLE);
1233
1234 /* Did construction of this file fail? */
1235 if (!test_bit(NFSD_FILE_HASHED, &nf->nf_flags)) {
1236 trace_nfsd_file_cons_err(rqstp, inode, may_flags, nf);
1237 if (!open_retry) {
1238 status = nfserr_jukebox;
1239 goto construction_err;
1240 }
1241 nfsd_file_put(nf);
1242 open_retry = false;
1243 fh_put(fhp);
1244 goto retry;
1245 }
1246 this_cpu_inc(nfsd_file_cache_hits);
1247
1248 status = nfserrno(nfsd_open_break_lease(file_inode(nf->nf_file), may_flags));
1249 if (status != nfs_ok) {
1250 nfsd_file_put(nf);
1251 nf = NULL;
1252 }
1253
1254out:
1255 if (status == nfs_ok) {
1256 this_cpu_inc(nfsd_file_acquisitions);
1257 nfsd_file_check_write_error(nf);
1258 *pnf = nf;
1259 }
1260 trace_nfsd_file_acquire(rqstp, inode, may_flags, nf, status);
1261 return status;
1262
1263open_file:
1264 trace_nfsd_file_alloc(nf);
1265
1266 nf->nf_mark = nfsd_file_mark_find_or_create(inode);
1267 if (nf->nf_mark) {
1268 if (file && (file->f_mode & FMODE_OPENED)) {
1269 get_file(file);
1270 nf->nf_file = file;
1271 status = nfs_ok;
1272 trace_nfsd_file_opened(nf, status);
1273 } else {
1274 ret = nfsd_open_verified(fhp, type, may_flags, &nf->nf_file);
1275 if (ret == -EOPENSTALE && stale_retry) {
1276 stale_retry = false;
1277 nfsd_file_unhash(nf);
1278 clear_and_wake_up_bit(NFSD_FILE_PENDING,
1279 &nf->nf_flags);
1280 if (refcount_dec_and_test(&nf->nf_ref))
1281 nfsd_file_free(nf);
1282 nf = NULL;
1283 fh_put(fhp);
1284 goto retry;
1285 }
1286 status = nfserrno(ret);
1287 trace_nfsd_file_open(nf, status);
1288 if (status == nfs_ok)
1289 status = nfsd_file_get_dio_attrs(fhp, nf);
1290 }
1291 } else
1292 status = nfserr_jukebox;
1293 /*
1294 * If construction failed, or we raced with a call to unlink()
1295 * then unhash.
1296 */
1297 if (status != nfs_ok || inode->i_nlink == 0)
1298 nfsd_file_unhash(nf);
1299 else if (want_gc)
1300 nfsd_file_lru_add(nf);
1301
1302 clear_and_wake_up_bit(NFSD_FILE_PENDING, &nf->nf_flags);
1303 if (status == nfs_ok)
1304 goto out;
1305
1306construction_err:
1307 if (refcount_dec_and_test(&nf->nf_ref))
1308 nfsd_file_free(nf);
1309 nf = NULL;
1310 goto out;
1311}
1312
1313/**
1314 * nfsd_file_acquire_gc - Get a struct nfsd_file with an open file
1315 * @rqstp: the RPC transaction being executed
1316 * @fhp: the NFS filehandle of the file to be opened
1317 * @may_flags: NFSD_MAY_ settings for the file
1318 * @pnf: OUT: new or found "struct nfsd_file" object
1319 *
1320 * The nfsd_file object returned by this API is reference-counted
1321 * and garbage-collected. The object is retained for a few
1322 * seconds after the final nfsd_file_put() in case the caller
1323 * wants to re-use it.
1324 *
1325 * Return values:
1326 * %nfs_ok - @pnf points to an nfsd_file with its reference
1327 * count boosted.
1328 *
1329 * On error, an nfsstat value in network byte order is returned.
1330 */
1331__be32
1332nfsd_file_acquire_gc(struct svc_rqst *rqstp, struct svc_fh *fhp,
1333 unsigned int may_flags, struct nfsd_file **pnf)
1334{
1335 return nfsd_file_do_acquire(rqstp, SVC_NET(rqstp), NULL, NULL,
1336 fhp, may_flags, NULL, S_IFREG, true, pnf);
1337}
1338
1339/**
1340 * nfsd_file_acquire - Get a struct nfsd_file with an open file
1341 * @rqstp: the RPC transaction being executed
1342 * @fhp: the NFS filehandle of the file to be opened
1343 * @may_flags: NFSD_MAY_ settings for the file
1344 * @pnf: OUT: new or found "struct nfsd_file" object
1345 *
1346 * The nfsd_file_object returned by this API is reference-counted
1347 * but not garbage-collected. The object is unhashed after the
1348 * final nfsd_file_put().
1349 *
1350 * Return values:
1351 * %nfs_ok - @pnf points to an nfsd_file with its reference
1352 * count boosted.
1353 *
1354 * On error, an nfsstat value in network byte order is returned.
1355 */
1356__be32
1357nfsd_file_acquire(struct svc_rqst *rqstp, struct svc_fh *fhp,
1358 unsigned int may_flags, struct nfsd_file **pnf)
1359{
1360 return nfsd_file_do_acquire(rqstp, SVC_NET(rqstp), NULL, NULL,
1361 fhp, may_flags, NULL, S_IFREG, false, pnf);
1362}
1363
1364/**
1365 * nfsd_file_acquire_local - Get a struct nfsd_file with an open file for localio
1366 * @net: The network namespace in which to perform a lookup
1367 * @cred: the user credential with which to validate access
1368 * @client: the auth_domain for LOCALIO lookup
1369 * @fhp: the NFS filehandle of the file to be opened
1370 * @may_flags: NFSD_MAY_ settings for the file
1371 * @pnf: OUT: new or found "struct nfsd_file" object
1372 *
1373 * This file lookup interface provide access to a file given the
1374 * filehandle and credential. No connection-based authorisation
1375 * is performed and in that way it is quite different to other
1376 * file access mediated by nfsd. It allows a kernel module such as the NFS
1377 * client to reach across network and filesystem namespaces to access
1378 * a file. The security implications of this should be carefully
1379 * considered before use.
1380 *
1381 * The nfsd_file_object returned by this API is reference-counted
1382 * but not garbage-collected. The object is unhashed after the
1383 * final nfsd_file_put().
1384 *
1385 * Return values:
1386 * %nfs_ok - @pnf points to an nfsd_file with its reference
1387 * count boosted.
1388 *
1389 * On error, an nfsstat value in network byte order is returned.
1390 */
1391__be32
1392nfsd_file_acquire_local(struct net *net, struct svc_cred *cred,
1393 struct auth_domain *client, struct svc_fh *fhp,
1394 unsigned int may_flags, struct nfsd_file **pnf)
1395{
1396 /*
1397 * Save creds before calling nfsd_file_do_acquire() (which calls
1398 * nfsd_setuser). Important because caller (LOCALIO) is from
1399 * client context.
1400 */
1401 const struct cred *save_cred = get_current_cred();
1402 __be32 beres;
1403
1404 beres = nfsd_file_do_acquire(NULL, net, cred, client, fhp, may_flags,
1405 NULL, S_IFREG, false, pnf);
1406 put_cred(revert_creds(save_cred));
1407 return beres;
1408}
1409
1410/**
1411 * nfsd_file_acquire_opened - Get a struct nfsd_file using existing open file
1412 * @rqstp: the RPC transaction being executed
1413 * @fhp: the NFS filehandle of the file just created
1414 * @may_flags: NFSD_MAY_ settings for the file
1415 * @file: cached, already-open file (may be NULL or not yet opened)
1416 * @pnf: OUT: new or found "struct nfsd_file" object
1417 *
1418 * Acquire a nfsd_file object that is not GC'ed. If one doesn't already exist,
1419 * and @file has FMODE_OPENED set, use it to instantiate a new nfsd_file
1420 * instead of opening a new one.
1421 *
1422 * Return values:
1423 * %nfs_ok - @pnf points to an nfsd_file with its reference
1424 * count boosted.
1425 *
1426 * On error, an nfsstat value in network byte order is returned.
1427 */
1428__be32
1429nfsd_file_acquire_opened(struct svc_rqst *rqstp, struct svc_fh *fhp,
1430 unsigned int may_flags, struct file *file,
1431 struct nfsd_file **pnf)
1432{
1433 return nfsd_file_do_acquire(rqstp, SVC_NET(rqstp), NULL, NULL,
1434 fhp, may_flags, file, S_IFREG, false, pnf);
1435}
1436
1437/**
1438 * nfsd_file_acquire_dir - Get a struct nfsd_file with an open directory
1439 * @rqstp: the RPC transaction being executed
1440 * @fhp: the NFS filehandle of the file to be opened
1441 * @pnf: OUT: new or found "struct nfsd_file" object
1442 *
1443 * The nfsd_file_object returned by this API is reference-counted
1444 * but not garbage-collected. The object is unhashed after the
1445 * final nfsd_file_put(). This opens directories only, and only
1446 * in O_RDONLY mode.
1447 *
1448 * Return values:
1449 * %nfs_ok - @pnf points to an nfsd_file with its reference
1450 * count boosted.
1451 *
1452 * On error, an nfsstat value in network byte order is returned.
1453 */
1454__be32
1455nfsd_file_acquire_dir(struct svc_rqst *rqstp, struct svc_fh *fhp,
1456 struct nfsd_file **pnf)
1457{
1458 return nfsd_file_do_acquire(rqstp, SVC_NET(rqstp), NULL, NULL, fhp,
1459 NFSD_MAY_READ|NFSD_MAY_64BIT_COOKIE,
1460 NULL, S_IFDIR, false, pnf);
1461}
1462
1463/*
1464 * Note that fields may be added, removed or reordered in the future. Programs
1465 * scraping this file for info should test the labels to ensure they're
1466 * getting the correct field.
1467 */
1468int nfsd_file_cache_stats_show(struct seq_file *m, void *v)
1469{
1470 unsigned long allocations = 0, releases = 0, evictions = 0;
1471 unsigned long hits = 0, acquisitions = 0;
1472 unsigned int i, count = 0, buckets = 0;
1473 unsigned long lru = 0, total_age = 0;
1474
1475 /* Serialize with server shutdown */
1476 mutex_lock(&nfsd_mutex);
1477 if (test_bit(NFSD_FILE_CACHE_UP, &nfsd_file_flags) == 1) {
1478 struct bucket_table *tbl;
1479 struct rhashtable *ht;
1480
1481 lru = list_lru_count(&nfsd_file_lru);
1482
1483 rcu_read_lock();
1484 ht = &nfsd_file_rhltable.ht;
1485 count = atomic_read(&ht->nelems);
1486 tbl = rht_dereference_rcu(ht->tbl, ht);
1487 buckets = tbl->size;
1488 rcu_read_unlock();
1489 }
1490 mutex_unlock(&nfsd_mutex);
1491
1492 for_each_possible_cpu(i) {
1493 hits += per_cpu(nfsd_file_cache_hits, i);
1494 acquisitions += per_cpu(nfsd_file_acquisitions, i);
1495 allocations += per_cpu(nfsd_file_allocations, i);
1496 releases += per_cpu(nfsd_file_releases, i);
1497 total_age += per_cpu(nfsd_file_total_age, i);
1498 evictions += per_cpu(nfsd_file_evictions, i);
1499 }
1500
1501 seq_printf(m, "total inodes: %u\n", count);
1502 seq_printf(m, "hash buckets: %u\n", buckets);
1503 seq_printf(m, "lru entries: %lu\n", lru);
1504 seq_printf(m, "cache hits: %lu\n", hits);
1505 seq_printf(m, "acquisitions: %lu\n", acquisitions);
1506 seq_printf(m, "allocations: %lu\n", allocations);
1507 seq_printf(m, "releases: %lu\n", releases);
1508 seq_printf(m, "evictions: %lu\n", evictions);
1509 if (releases)
1510 seq_printf(m, "mean age (ms): %ld\n", total_age / releases);
1511 else
1512 seq_printf(m, "mean age (ms): -\n");
1513 return 0;
1514}
1515
1516/**
1517 * nfsd_fsnotify_recalc_mask - recalculate the fsnotify mask for a nfsd_file
1518 * @nf: nfsd_file to recalculate the mask on
1519 *
1520 * When a directory nfsd_file has a delegation added or removed, that may
1521 * change the events that nfsd requires from the VFS layer. This function
1522 * recalculates the fsnotify mask based on the leases present.
1523 */
1524void nfsd_fsnotify_recalc_mask(struct nfsd_file *nf)
1525{
1526 struct inode *inode = file_inode(nf->nf_file);
1527 u32 lease_mask, set = 0, clear = 0;
1528 struct fsnotify_mark *mark;
1529
1530 /* This is only needed when adding or removing dir delegs */
1531 if (!S_ISDIR(inode->i_mode) || !nf->nf_mark)
1532 return;
1533
1534 mark = &nf->nf_mark->nfm_mark;
1535
1536 /*
1537 * The mark is shared by every nfsd_file on this inode, so concurrent
1538 * delegation add/remove on the same directory can recalc it in
1539 * parallel. Serialize the read of the lease state and the update of
1540 * the mark so that a recalc working from a stale snapshot of the
1541 * lease list can't clobber a concurrent recalc's update.
1542 */
1543 mutex_lock(&nf->nf_mark->nfm_recalc_mutex);
1544
1545 /* Set up notifications for any ignored delegation events */
1546 lease_mask = inode_lease_ignore_mask(inode);
1547
1548 if (lease_mask & FL_IGN_DIR_CREATE)
1549 set |= FS_CREATE | FS_MOVED_TO;
1550 else
1551 clear |= FS_CREATE | FS_MOVED_TO;
1552
1553 if (lease_mask & FL_IGN_DIR_DELETE)
1554 set |= FS_DELETE | FS_MOVED_FROM;
1555 else
1556 clear |= FS_DELETE | FS_MOVED_FROM;
1557
1558 if (lease_mask & FL_IGN_DIR_RENAME)
1559 set |= FS_RENAME;
1560 else
1561 clear |= FS_RENAME;
1562
1563 fsnotify_modify_mark_mask(mark, set, clear);
1564 mutex_unlock(&nf->nf_mark->nfm_recalc_mutex);
1565}