projects/linux·blob master · refs

2008 lines · 47.8 KiB

1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * net/sunrpc/cache.c
4 *
5 * Generic code for various authentication-related caches
6 * used by sunrpc clients and servers.
7 *
8 * Copyright (C) 2002 Neil Brown <[email protected]>
9 */
10
11#include <linux/types.h>
12#include <linux/fs.h>
13#include <linux/file.h>
14#include <linux/hex.h>
15#include <linux/slab.h>
16#include <linux/signal.h>
17#include <linux/sched.h>
18#include <linux/kmod.h>
19#include <linux/list.h>
20#include <linux/module.h>
21#include <linux/ctype.h>
22#include <linux/string_helpers.h>
23#include <linux/uaccess.h>
24#include <linux/poll.h>
25#include <linux/seq_file.h>
26#include <linux/proc_fs.h>
27#include <linux/net.h>
28#include <linux/workqueue.h>
29#include <linux/mutex.h>
30#include <linux/pagemap.h>
31#include <asm/ioctls.h>
32#include <linux/sunrpc/types.h>
33#include <linux/sunrpc/cache.h>
34#include <linux/sunrpc/stats.h>
35#include <linux/sunrpc/rpc_pipe_fs.h>
36#include <net/genetlink.h>
37#include <trace/events/sunrpc.h>
38
39#include "netns.h"
40#include "netlink.h"
41#include "fail.h"
42
43#define RPCDBG_FACILITY RPCDBG_CACHE
44
45static bool cache_defer_req(struct cache_req *req, struct cache_head *item);
46static void cache_revisit_request(struct cache_head *item);
47
48static void cache_init(struct cache_head *h, struct cache_detail *detail)
49{
50 time64_t now = seconds_since_boot();
51 INIT_HLIST_NODE(&h->cache_list);
52 h->flags = 0;
53 kref_init(&h->ref);
54 h->expiry_time = now + CACHE_NEW_EXPIRY;
55 if (now <= detail->flush_time)
56 /* ensure it isn't already expired */
57 now = detail->flush_time + 1;
58 h->last_refresh = now;
59}
60
61static void cache_fresh_unlocked(struct cache_head *head,
62 struct cache_detail *detail);
63
64static struct cache_head *sunrpc_cache_find_rcu(struct cache_detail *detail,
65 struct cache_head *key,
66 int hash)
67{
68 struct hlist_head *head = &detail->hash_table[hash];
69 struct cache_head *tmp;
70
71 rcu_read_lock();
72 hlist_for_each_entry_rcu(tmp, head, cache_list) {
73 if (!detail->match(tmp, key))
74 continue;
75 if (test_bit(CACHE_VALID, &tmp->flags) &&
76 cache_is_expired(detail, tmp))
77 continue;
78 tmp = cache_get_rcu(tmp);
79 rcu_read_unlock();
80 return tmp;
81 }
82 rcu_read_unlock();
83 return NULL;
84}
85
86static void sunrpc_begin_cache_remove_entry(struct cache_head *ch,
87 struct cache_detail *cd)
88{
89 /* Must be called under cd->hash_lock */
90 hlist_del_init_rcu(&ch->cache_list);
91 set_bit(CACHE_CLEANED, &ch->flags);
92 cd->entries --;
93}
94
95static void sunrpc_end_cache_remove_entry(struct cache_head *ch,
96 struct cache_detail *cd)
97{
98 cache_fresh_unlocked(ch, cd);
99 cache_put(ch, cd);
100}
101
102static struct cache_head *sunrpc_cache_add_entry(struct cache_detail *detail,
103 struct cache_head *key,
104 int hash)
105{
106 struct cache_head *new, *tmp, *freeme = NULL;
107 struct hlist_head *head = &detail->hash_table[hash];
108
109 new = detail->alloc();
110 if (!new)
111 return NULL;
112 /* must fully initialise 'new', else
113 * we might get lose if we need to
114 * cache_put it soon.
115 */
116 cache_init(new, detail);
117 detail->init(new, key);
118
119 spin_lock(&detail->hash_lock);
120
121 /* check if entry appeared while we slept */
122 hlist_for_each_entry_rcu(tmp, head, cache_list,
123 lockdep_is_held(&detail->hash_lock)) {
124 if (!detail->match(tmp, key))
125 continue;
126 if (test_bit(CACHE_VALID, &tmp->flags) &&
127 cache_is_expired(detail, tmp)) {
128 sunrpc_begin_cache_remove_entry(tmp, detail);
129 trace_cache_entry_expired(detail, tmp);
130 freeme = tmp;
131 break;
132 }
133 cache_get(tmp);
134 spin_unlock(&detail->hash_lock);
135 cache_put(new, detail);
136 return tmp;
137 }
138
139 cache_get(new);
140 hlist_add_head_rcu(&new->cache_list, head);
141 detail->entries++;
142 if (detail->nextcheck > new->expiry_time)
143 detail->nextcheck = new->expiry_time + 1;
144 spin_unlock(&detail->hash_lock);
145
146 if (freeme)
147 sunrpc_end_cache_remove_entry(freeme, detail);
148 return new;
149}
150
151struct cache_head *sunrpc_cache_lookup_rcu(struct cache_detail *detail,
152 struct cache_head *key, int hash)
153{
154 struct cache_head *ret;
155
156 ret = sunrpc_cache_find_rcu(detail, key, hash);
157 if (ret)
158 return ret;
159 /* Didn't find anything, insert an empty entry */
160 return sunrpc_cache_add_entry(detail, key, hash);
161}
162EXPORT_SYMBOL_GPL(sunrpc_cache_lookup_rcu);
163
164static void cache_dequeue(struct cache_detail *detail, struct cache_head *ch);
165
166static void cache_fresh_locked(struct cache_head *head, time64_t expiry,
167 struct cache_detail *detail)
168{
169 time64_t now = seconds_since_boot();
170 if (now <= detail->flush_time)
171 /* ensure it isn't immediately treated as expired */
172 now = detail->flush_time + 1;
173 head->expiry_time = expiry;
174 head->last_refresh = now;
175 smp_wmb(); /* paired with smp_rmb() in cache_is_valid() */
176 set_bit(CACHE_VALID, &head->flags);
177}
178
179static void cache_fresh_unlocked(struct cache_head *head,
180 struct cache_detail *detail)
181{
182 if (test_and_clear_bit(CACHE_PENDING, &head->flags)) {
183 cache_revisit_request(head);
184 cache_dequeue(detail, head);
185 }
186}
187
188static void cache_make_negative(struct cache_detail *detail,
189 struct cache_head *h)
190{
191 set_bit(CACHE_NEGATIVE, &h->flags);
192 trace_cache_entry_make_negative(detail, h);
193}
194
195static void cache_entry_update(struct cache_detail *detail,
196 struct cache_head *h,
197 struct cache_head *new)
198{
199 if (!test_bit(CACHE_NEGATIVE, &new->flags)) {
200 detail->update(h, new);
201 trace_cache_entry_update(detail, h);
202 } else {
203 cache_make_negative(detail, h);
204 }
205}
206
207struct cache_head *sunrpc_cache_update(struct cache_detail *detail,
208 struct cache_head *new, struct cache_head *old, int hash)
209{
210 /* The 'old' entry is to be replaced by 'new'.
211 * If 'old' is not VALID, we update it directly,
212 * otherwise we need to replace it
213 */
214 struct cache_head *tmp;
215
216 if (!test_bit(CACHE_VALID, &old->flags)) {
217 spin_lock(&detail->hash_lock);
218 if (!test_bit(CACHE_VALID, &old->flags)) {
219 cache_entry_update(detail, old, new);
220 cache_fresh_locked(old, new->expiry_time, detail);
221 spin_unlock(&detail->hash_lock);
222 cache_fresh_unlocked(old, detail);
223 return old;
224 }
225 spin_unlock(&detail->hash_lock);
226 }
227 /* We need to insert a new entry */
228 tmp = detail->alloc();
229 if (!tmp) {
230 cache_put(old, detail);
231 return NULL;
232 }
233 cache_init(tmp, detail);
234 detail->init(tmp, old);
235
236 spin_lock(&detail->hash_lock);
237 cache_entry_update(detail, tmp, new);
238 cache_get(tmp);
239 hlist_add_head_rcu(&tmp->cache_list, &detail->hash_table[hash]);
240 detail->entries++;
241 cache_fresh_locked(tmp, new->expiry_time, detail);
242 cache_fresh_locked(old, 0, detail);
243 spin_unlock(&detail->hash_lock);
244 cache_fresh_unlocked(tmp, detail);
245 cache_fresh_unlocked(old, detail);
246 cache_put(old, detail);
247 return tmp;
248}
249EXPORT_SYMBOL_GPL(sunrpc_cache_update);
250
251static inline int cache_is_valid(struct cache_head *h)
252{
253 if (!test_bit(CACHE_VALID, &h->flags))
254 return -EAGAIN;
255 else {
256 /* entry is valid */
257 if (test_bit(CACHE_NEGATIVE, &h->flags))
258 return -ENOENT;
259 else {
260 /*
261 * In combination with write barrier in
262 * sunrpc_cache_update, ensures that anyone
263 * using the cache entry after this sees the
264 * updated contents:
265 */
266 smp_rmb();
267 return 0;
268 }
269 }
270}
271
272static int try_to_negate_entry(struct cache_detail *detail, struct cache_head *h)
273{
274 int rv;
275
276 spin_lock(&detail->hash_lock);
277 rv = cache_is_valid(h);
278 if (rv == -EAGAIN) {
279 cache_make_negative(detail, h);
280 cache_fresh_locked(h, seconds_since_boot()+CACHE_NEW_EXPIRY,
281 detail);
282 rv = -ENOENT;
283 }
284 spin_unlock(&detail->hash_lock);
285 cache_fresh_unlocked(h, detail);
286 return rv;
287}
288
289int cache_check_rcu(struct cache_detail *detail,
290 struct cache_head *h, struct cache_req *rqstp)
291{
292 int rv;
293 time64_t refresh_age, age;
294
295 /* First decide return status as best we can */
296 rv = cache_is_valid(h);
297
298 /* now see if we want to start an upcall */
299 refresh_age = (h->expiry_time - h->last_refresh);
300 age = seconds_since_boot() - h->last_refresh;
301
302 if (rqstp == NULL) {
303 if (rv == -EAGAIN)
304 rv = -ENOENT;
305 } else if (rv == -EAGAIN ||
306 (h->expiry_time != 0 && age > refresh_age/2)) {
307 dprintk("RPC: Want update, refage=%lld, age=%lld\n",
308 refresh_age, age);
309 switch (detail->cache_upcall(detail, h)) {
310 case -EINVAL:
311 rv = try_to_negate_entry(detail, h);
312 break;
313 case -EAGAIN:
314 cache_fresh_unlocked(h, detail);
315 break;
316 }
317 }
318
319 if (rv == -EAGAIN) {
320 if (!cache_defer_req(rqstp, h)) {
321 /*
322 * Request was not deferred; handle it as best
323 * we can ourselves:
324 */
325 rv = cache_is_valid(h);
326 if (rv == -EAGAIN)
327 rv = -ETIMEDOUT;
328 }
329 }
330
331 return rv;
332}
333EXPORT_SYMBOL_GPL(cache_check_rcu);
334
335/*
336 * This is the generic cache management routine for all
337 * the authentication caches.
338 * It checks the currency of a cache item and will (later)
339 * initiate an upcall to fill it if needed.
340 *
341 *
342 * Returns 0 if the cache_head can be used, or cache_puts it and returns
343 * -EAGAIN if upcall is pending and request has been queued
344 * -ETIMEDOUT if upcall failed or request could not be queue or
345 * upcall completed but item is still invalid (implying that
346 * the cache item has been replaced with a newer one).
347 * -ENOENT if cache entry was negative
348 */
349int cache_check(struct cache_detail *detail,
350 struct cache_head *h, struct cache_req *rqstp)
351{
352 int rv;
353
354 rv = cache_check_rcu(detail, h, rqstp);
355 if (rv)
356 cache_put(h, detail);
357 return rv;
358}
359EXPORT_SYMBOL_GPL(cache_check);
360
361/*
362 * caches need to be periodically cleaned.
363 * For this we maintain a list of cache_detail and
364 * a current pointer into that list and into the table
365 * for that entry.
366 *
367 * Each time cache_clean is called it finds the next non-empty entry
368 * in the current table and walks the list in that entry
369 * looking for entries that can be removed.
370 *
371 * An entry gets removed if:
372 * - The expiry is before current time
373 * - The last_refresh time is before the flush_time for that cache
374 *
375 * later we might drop old entries with non-NEVER expiry if that table
376 * is getting 'full' for some definition of 'full'
377 *
378 * The question of "how often to scan a table" is an interesting one
379 * and is answered in part by the use of the "nextcheck" field in the
380 * cache_detail.
381 * When a scan of a table begins, the nextcheck field is set to a time
382 * that is well into the future.
383 * While scanning, if an expiry time is found that is earlier than the
384 * current nextcheck time, nextcheck is set to that expiry time.
385 * If the flush_time is ever set to a time earlier than the nextcheck
386 * time, the nextcheck time is then set to that flush_time.
387 *
388 * A table is then only scanned if the current time is at least
389 * the nextcheck time.
390 *
391 */
392
393static LIST_HEAD(cache_list);
394static DEFINE_SPINLOCK(cache_list_lock);
395static struct cache_detail *current_detail;
396static int current_index;
397
398static void do_cache_clean(struct work_struct *work);
399static struct delayed_work cache_cleaner;
400
401void sunrpc_init_cache_detail(struct cache_detail *cd)
402{
403 spin_lock_init(&cd->hash_lock);
404 INIT_LIST_HEAD(&cd->requests);
405 INIT_LIST_HEAD(&cd->readers);
406 spin_lock_init(&cd->queue_lock);
407 init_waitqueue_head(&cd->queue_wait);
408 cd->next_seqno = 1;
409 spin_lock(&cache_list_lock);
410 cd->nextcheck = 0;
411 cd->entries = 0;
412 atomic_set(&cd->writers, 0);
413 cd->last_close = 0;
414 cd->last_warn = -1;
415 list_add(&cd->others, &cache_list);
416 spin_unlock(&cache_list_lock);
417
418 /* start the cleaning process */
419 queue_delayed_work(system_power_efficient_wq, &cache_cleaner, 0);
420}
421EXPORT_SYMBOL_GPL(sunrpc_init_cache_detail);
422
423void sunrpc_destroy_cache_detail(struct cache_detail *cd)
424{
425 cache_purge(cd);
426 spin_lock(&cache_list_lock);
427 spin_lock(&cd->hash_lock);
428 if (current_detail == cd)
429 current_detail = NULL;
430 list_del_init(&cd->others);
431 spin_unlock(&cd->hash_lock);
432 spin_unlock(&cache_list_lock);
433 cancel_delayed_work_sync(&cache_cleaner);
434 if (!list_empty(&cache_list))
435 queue_delayed_work(system_power_efficient_wq, &cache_cleaner, 0);
436}
437EXPORT_SYMBOL_GPL(sunrpc_destroy_cache_detail);
438
439/* clean cache tries to find something to clean
440 * and cleans it.
441 * It returns 1 if it cleaned something,
442 * 0 if it didn't find anything this time
443 * -1 if it fell off the end of the list.
444 */
445static int cache_clean(void)
446{
447 int rv = 0;
448 struct list_head *next;
449
450 spin_lock(&cache_list_lock);
451
452 /* find a suitable table if we don't already have one */
453 while (current_detail == NULL ||
454 current_index >= current_detail->hash_size) {
455 if (current_detail)
456 next = current_detail->others.next;
457 else
458 next = cache_list.next;
459 if (next == &cache_list) {
460 current_detail = NULL;
461 spin_unlock(&cache_list_lock);
462 return -1;
463 }
464 current_detail = list_entry(next, struct cache_detail, others);
465 if (current_detail->nextcheck > seconds_since_boot())
466 current_index = current_detail->hash_size;
467 else {
468 current_index = 0;
469 current_detail->nextcheck = seconds_since_boot()+30*60;
470 }
471 }
472
473 spin_lock(&current_detail->hash_lock);
474
475 /* find a non-empty bucket in the table */
476 while (current_index < current_detail->hash_size &&
477 hlist_empty(&current_detail->hash_table[current_index]))
478 current_index++;
479
480 /* find a cleanable entry in the bucket and clean it, or set to next bucket */
481 if (current_index < current_detail->hash_size) {
482 struct cache_head *ch = NULL;
483 struct cache_detail *d;
484 struct hlist_head *head;
485 struct hlist_node *tmp;
486
487 /* Ok, now to clean this strand */
488 head = &current_detail->hash_table[current_index];
489 hlist_for_each_entry_safe(ch, tmp, head, cache_list) {
490 if (current_detail->nextcheck > ch->expiry_time)
491 current_detail->nextcheck = ch->expiry_time+1;
492 if (!cache_is_expired(current_detail, ch))
493 continue;
494
495 sunrpc_begin_cache_remove_entry(ch, current_detail);
496 trace_cache_entry_expired(current_detail, ch);
497 rv = 1;
498 break;
499 }
500
501 spin_unlock(&current_detail->hash_lock);
502 d = current_detail;
503 if (!ch)
504 current_index ++;
505 spin_unlock(&cache_list_lock);
506 if (ch)
507 sunrpc_end_cache_remove_entry(ch, d);
508 } else {
509 spin_unlock(&current_detail->hash_lock);
510 spin_unlock(&cache_list_lock);
511 }
512
513 return rv;
514}
515
516/*
517 * We want to regularly clean the cache, so we need to schedule some work ...
518 */
519static void do_cache_clean(struct work_struct *work)
520{
521 int delay;
522
523 if (list_empty(&cache_list))
524 return;
525
526 if (cache_clean() == -1)
527 delay = round_jiffies_relative(30*HZ);
528 else
529 delay = 5;
530
531 queue_delayed_work(system_power_efficient_wq, &cache_cleaner, delay);
532}
533
534
535/*
536 * Clean all caches promptly. This just calls cache_clean
537 * repeatedly until we are sure that every cache has had a chance to
538 * be fully cleaned
539 */
540void cache_flush(void)
541{
542 while (cache_clean() != -1)
543 cond_resched();
544 while (cache_clean() != -1)
545 cond_resched();
546}
547EXPORT_SYMBOL_GPL(cache_flush);
548
549void cache_purge(struct cache_detail *detail)
550{
551 struct cache_head *ch = NULL;
552 struct hlist_head *head = NULL;
553 int i = 0;
554
555 spin_lock(&detail->hash_lock);
556 if (!detail->entries) {
557 spin_unlock(&detail->hash_lock);
558 return;
559 }
560
561 dprintk("RPC: %d entries in %s cache\n", detail->entries, detail->name);
562 for (i = 0; i < detail->hash_size; i++) {
563 head = &detail->hash_table[i];
564 while (!hlist_empty(head)) {
565 ch = hlist_entry(head->first, struct cache_head,
566 cache_list);
567 sunrpc_begin_cache_remove_entry(ch, detail);
568 spin_unlock(&detail->hash_lock);
569 sunrpc_end_cache_remove_entry(ch, detail);
570 spin_lock(&detail->hash_lock);
571 }
572 }
573 spin_unlock(&detail->hash_lock);
574}
575EXPORT_SYMBOL_GPL(cache_purge);
576
577
578/*
579 * Deferral and Revisiting of Requests.
580 *
581 * If a cache lookup finds a pending entry, we
582 * need to defer the request and revisit it later.
583 * All deferred requests are stored in a hash table,
584 * indexed by "struct cache_head *".
585 * As it may be wasteful to store a whole request
586 * structure, we allow the request to provide a
587 * deferred form, which must contain a
588 * 'struct cache_deferred_req'
589 * This cache_deferred_req contains a method to allow
590 * it to be revisited when cache info is available
591 */
592
593#define DFR_HASHSIZE (PAGE_SIZE/sizeof(struct list_head))
594#define DFR_HASH(item) ((((long)item)>>4 ^ (((long)item)>>13)) % DFR_HASHSIZE)
595
596#define DFR_MAX 300 /* ??? */
597
598static DEFINE_SPINLOCK(cache_defer_lock);
599static LIST_HEAD(cache_defer_list);
600static struct hlist_head cache_defer_hash[DFR_HASHSIZE];
601static int cache_defer_cnt;
602
603static void __unhash_deferred_req(struct cache_deferred_req *dreq)
604{
605 hlist_del_init(&dreq->hash);
606 if (!list_empty(&dreq->recent)) {
607 list_del_init(&dreq->recent);
608 cache_defer_cnt--;
609 }
610}
611
612static void __hash_deferred_req(struct cache_deferred_req *dreq, struct cache_head *item)
613{
614 int hash = DFR_HASH(item);
615
616 INIT_LIST_HEAD(&dreq->recent);
617 hlist_add_head(&dreq->hash, &cache_defer_hash[hash]);
618}
619
620static void setup_deferral(struct cache_deferred_req *dreq,
621 struct cache_head *item,
622 int count_me)
623{
624
625 dreq->item = item;
626
627 spin_lock(&cache_defer_lock);
628
629 __hash_deferred_req(dreq, item);
630
631 if (count_me) {
632 cache_defer_cnt++;
633 list_add(&dreq->recent, &cache_defer_list);
634 }
635
636 spin_unlock(&cache_defer_lock);
637
638}
639
640struct thread_deferred_req {
641 struct cache_deferred_req handle;
642 struct completion completion;
643};
644
645static void cache_restart_thread(struct cache_deferred_req *dreq, int too_many)
646{
647 struct thread_deferred_req *dr =
648 container_of(dreq, struct thread_deferred_req, handle);
649 complete(&dr->completion);
650}
651
652static void cache_wait_req(struct cache_req *req, struct cache_head *item)
653{
654 struct thread_deferred_req sleeper;
655 struct cache_deferred_req *dreq = &sleeper.handle;
656
657 sleeper.completion = COMPLETION_INITIALIZER_ONSTACK(sleeper.completion);
658 dreq->revisit = cache_restart_thread;
659
660 setup_deferral(dreq, item, 0);
661
662 if (!test_bit(CACHE_PENDING, &item->flags) ||
663 wait_for_completion_interruptible_timeout(
664 &sleeper.completion, req->thread_wait) <= 0) {
665 /* The completion wasn't completed, so we need
666 * to clean up
667 */
668 spin_lock(&cache_defer_lock);
669 if (!hlist_unhashed(&sleeper.handle.hash)) {
670 __unhash_deferred_req(&sleeper.handle);
671 spin_unlock(&cache_defer_lock);
672 } else {
673 /* cache_revisit_request already removed
674 * this from the hash table, but hasn't
675 * called ->revisit yet. It will very soon
676 * and we need to wait for it.
677 */
678 spin_unlock(&cache_defer_lock);
679 wait_for_completion(&sleeper.completion);
680 }
681 }
682}
683
684static void cache_limit_defers(void)
685{
686 /* Make sure we haven't exceed the limit of allowed deferred
687 * requests.
688 */
689 struct cache_deferred_req *discard = NULL;
690
691 if (cache_defer_cnt <= DFR_MAX)
692 return;
693
694 spin_lock(&cache_defer_lock);
695
696 /* Consider removing either the first or the last */
697 if (cache_defer_cnt > DFR_MAX) {
698 if (get_random_u32_below(2))
699 discard = list_entry(cache_defer_list.next,
700 struct cache_deferred_req, recent);
701 else
702 discard = list_entry(cache_defer_list.prev,
703 struct cache_deferred_req, recent);
704 __unhash_deferred_req(discard);
705 }
706 spin_unlock(&cache_defer_lock);
707 if (discard)
708 discard->revisit(discard, 1);
709}
710
711#if IS_ENABLED(CONFIG_FAIL_SUNRPC)
712static inline bool cache_defer_immediately(void)
713{
714 return !fail_sunrpc.ignore_cache_wait &&
715 should_fail(&fail_sunrpc.attr, 1);
716}
717#else
718static inline bool cache_defer_immediately(void)
719{
720 return false;
721}
722#endif
723
724/* Return true if and only if a deferred request is queued. */
725static bool cache_defer_req(struct cache_req *req, struct cache_head *item)
726{
727 struct cache_deferred_req *dreq;
728
729 if (!cache_defer_immediately()) {
730 cache_wait_req(req, item);
731 if (!test_bit(CACHE_PENDING, &item->flags))
732 return false;
733 }
734
735 dreq = req->defer(req);
736 if (dreq == NULL)
737 return false;
738 setup_deferral(dreq, item, 1);
739 if (!test_bit(CACHE_PENDING, &item->flags))
740 /* Bit could have been cleared before we managed to
741 * set up the deferral, so need to revisit just in case
742 */
743 cache_revisit_request(item);
744
745 cache_limit_defers();
746 return true;
747}
748
749static void cache_revisit_request(struct cache_head *item)
750{
751 struct cache_deferred_req *dreq;
752 struct hlist_node *tmp;
753 int hash = DFR_HASH(item);
754 LIST_HEAD(pending);
755
756 spin_lock(&cache_defer_lock);
757
758 hlist_for_each_entry_safe(dreq, tmp, &cache_defer_hash[hash], hash)
759 if (dreq->item == item) {
760 __unhash_deferred_req(dreq);
761 list_add(&dreq->recent, &pending);
762 }
763
764 spin_unlock(&cache_defer_lock);
765
766 while (!list_empty(&pending)) {
767 dreq = list_entry(pending.next, struct cache_deferred_req, recent);
768 list_del_init(&dreq->recent);
769 dreq->revisit(dreq, 0);
770 }
771}
772
773void cache_clean_deferred(void *owner)
774{
775 struct cache_deferred_req *dreq, *tmp;
776 LIST_HEAD(pending);
777
778 spin_lock(&cache_defer_lock);
779
780 list_for_each_entry_safe(dreq, tmp, &cache_defer_list, recent) {
781 if (dreq->owner == owner) {
782 __unhash_deferred_req(dreq);
783 list_add(&dreq->recent, &pending);
784 }
785 }
786 spin_unlock(&cache_defer_lock);
787
788 while (!list_empty(&pending)) {
789 dreq = list_entry(pending.next, struct cache_deferred_req, recent);
790 list_del_init(&dreq->recent);
791 dreq->revisit(dreq, 1);
792 }
793}
794
795/*
796 * communicate with user-space
797 *
798 * We have a magic /proc file - /proc/net/rpc/<cachename>/channel.
799 * On read, you get a full request, or block.
800 * On write, an update request is processed.
801 * Poll works if anything to read, and always allows write.
802 */
803
804struct cache_request {
805 struct list_head list;
806 struct cache_head *item;
807 char *buf;
808 int len;
809 int readers;
810 u64 seqno;
811};
812struct cache_reader {
813 struct list_head list;
814 int offset; /* if non-0, we have a refcnt on next request */
815 u64 next_seqno;
816};
817
818static int cache_request(struct cache_detail *detail,
819 struct cache_request *crq)
820{
821 char *bp = crq->buf;
822 int len = PAGE_SIZE;
823
824 detail->cache_request(detail, crq->item, &bp, &len);
825 if (len < 0)
826 return -E2BIG;
827 return PAGE_SIZE - len;
828}
829
830static struct cache_request *
831cache_next_request(struct cache_detail *cd, u64 seqno)
832{
833 struct cache_request *rq;
834
835 list_for_each_entry(rq, &cd->requests, list)
836 if (rq->seqno >= seqno)
837 return rq;
838 return NULL;
839}
840
841static ssize_t cache_read(struct file *filp, char __user *buf, size_t count,
842 loff_t *ppos, struct cache_detail *cd)
843{
844 struct cache_reader *rp = filp->private_data;
845 struct cache_request *rq;
846 struct inode *inode = file_inode(filp);
847 int err;
848
849 if (count == 0)
850 return 0;
851
852 inode_lock(inode); /* protect against multiple concurrent
853 * readers on this file */
854 again:
855 spin_lock(&cd->queue_lock);
856 /* need to find next request */
857 rq = cache_next_request(cd, rp->next_seqno);
858 if (!rq) {
859 spin_unlock(&cd->queue_lock);
860 inode_unlock(inode);
861 WARN_ON_ONCE(rp->offset);
862 return 0;
863 }
864 if (rp->offset == 0)
865 rq->readers++;
866 spin_unlock(&cd->queue_lock);
867
868 if (rq->len == 0) {
869 err = cache_request(cd, rq);
870 if (err < 0)
871 goto out;
872 rq->len = err;
873 }
874
875 if (rp->offset == 0 && !test_bit(CACHE_PENDING, &rq->item->flags)) {
876 err = -EAGAIN;
877 rp->next_seqno = rq->seqno + 1;
878 } else {
879 if (rp->offset + count > rq->len)
880 count = rq->len - rp->offset;
881 err = -EFAULT;
882 if (copy_to_user(buf, rq->buf + rp->offset, count))
883 goto out;
884 rp->offset += count;
885 if (rp->offset >= rq->len) {
886 rp->offset = 0;
887 rp->next_seqno = rq->seqno + 1;
888 }
889 err = 0;
890 }
891 out:
892 if (rp->offset == 0) {
893 /* need to release rq */
894 spin_lock(&cd->queue_lock);
895 rq->readers--;
896 if (rq->readers == 0 &&
897 !test_bit(CACHE_PENDING, &rq->item->flags)) {
898 list_del(&rq->list);
899 spin_unlock(&cd->queue_lock);
900 cache_put(rq->item, cd);
901 kfree(rq->buf);
902 kfree(rq);
903 } else
904 spin_unlock(&cd->queue_lock);
905 }
906 if (err == -EAGAIN)
907 goto again;
908 inode_unlock(inode);
909 return err ? err : count;
910}
911
912static ssize_t cache_do_downcall(char *kaddr, const char __user *buf,
913 size_t count, struct cache_detail *cd)
914{
915 ssize_t ret;
916
917 if (count == 0)
918 return -EINVAL;
919 if (copy_from_user(kaddr, buf, count))
920 return -EFAULT;
921 kaddr[count] = '\0';
922 ret = cd->cache_parse(cd, kaddr, count);
923 if (!ret)
924 ret = count;
925 return ret;
926}
927
928static ssize_t cache_downcall(struct address_space *mapping,
929 const char __user *buf,
930 size_t count, struct cache_detail *cd)
931{
932 char *write_buf;
933 ssize_t ret = -ENOMEM;
934
935 if (count >= 32768) { /* 32k is max userland buffer, lets check anyway */
936 ret = -EINVAL;
937 goto out;
938 }
939
940 write_buf = kvmalloc(count + 1, GFP_KERNEL);
941 if (!write_buf)
942 goto out;
943
944 ret = cache_do_downcall(write_buf, buf, count, cd);
945 kvfree(write_buf);
946out:
947 return ret;
948}
949
950static ssize_t cache_write(struct file *filp, const char __user *buf,
951 size_t count, loff_t *ppos,
952 struct cache_detail *cd)
953{
954 struct address_space *mapping = filp->f_mapping;
955 struct inode *inode = file_inode(filp);
956 ssize_t ret = -EINVAL;
957
958 if (!cd->cache_parse)
959 goto out;
960
961 inode_lock(inode);
962 ret = cache_downcall(mapping, buf, count, cd);
963 inode_unlock(inode);
964out:
965 return ret;
966}
967
968static __poll_t cache_poll(struct file *filp, poll_table *wait,
969 struct cache_detail *cd)
970{
971 __poll_t mask;
972 struct cache_reader *rp = filp->private_data;
973
974 poll_wait(filp, &cd->queue_wait, wait);
975
976 /* alway allow write */
977 mask = EPOLLOUT | EPOLLWRNORM;
978
979 if (!rp)
980 return mask;
981
982 spin_lock(&cd->queue_lock);
983
984 if (cache_next_request(cd, rp->next_seqno))
985 mask |= EPOLLIN | EPOLLRDNORM;
986 spin_unlock(&cd->queue_lock);
987 return mask;
988}
989
990static int cache_ioctl(struct inode *ino, struct file *filp,
991 unsigned int cmd, unsigned long arg,
992 struct cache_detail *cd)
993{
994 int len = 0;
995 struct cache_reader *rp = filp->private_data;
996 struct cache_request *rq;
997
998 if (cmd != FIONREAD || !rp)
999 return -EINVAL;
1000
1001 spin_lock(&cd->queue_lock);
1002
1003 /* only find the length remaining in current request,
1004 * or the length of the next request
1005 */
1006 rq = cache_next_request(cd, rp->next_seqno);
1007 if (rq)
1008 len = rq->len - rp->offset;
1009 spin_unlock(&cd->queue_lock);
1010
1011 return put_user(len, (int __user *)arg);
1012}
1013
1014static int cache_open(struct inode *inode, struct file *filp,
1015 struct cache_detail *cd)
1016{
1017 struct cache_reader *rp = NULL;
1018
1019 if (!cd || !try_module_get(cd->owner))
1020 return -EACCES;
1021 nonseekable_open(inode, filp);
1022 if (filp->f_mode & FMODE_READ) {
1023 rp = kmalloc_obj(*rp);
1024 if (!rp) {
1025 module_put(cd->owner);
1026 return -ENOMEM;
1027 }
1028 rp->offset = 0;
1029 rp->next_seqno = 0;
1030
1031 spin_lock(&cd->queue_lock);
1032 list_add(&rp->list, &cd->readers);
1033 spin_unlock(&cd->queue_lock);
1034 }
1035 if (filp->f_mode & FMODE_WRITE)
1036 atomic_inc(&cd->writers);
1037 filp->private_data = rp;
1038 return 0;
1039}
1040
1041static int cache_release(struct inode *inode, struct file *filp,
1042 struct cache_detail *cd)
1043{
1044 struct cache_reader *rp = filp->private_data;
1045
1046 if (rp) {
1047 struct cache_request *rq = NULL;
1048
1049 spin_lock(&cd->queue_lock);
1050 if (rp->offset) {
1051 struct cache_request *cr;
1052
1053 cr = cache_next_request(cd, rp->next_seqno);
1054 if (cr) {
1055 cr->readers--;
1056 if (cr->readers == 0 &&
1057 !test_bit(CACHE_PENDING,
1058 &cr->item->flags)) {
1059 list_del(&cr->list);
1060 rq = cr;
1061 }
1062 }
1063 rp->offset = 0;
1064 }
1065 list_del(&rp->list);
1066 spin_unlock(&cd->queue_lock);
1067
1068 if (rq) {
1069 cache_put(rq->item, cd);
1070 kfree(rq->buf);
1071 kfree(rq);
1072 }
1073
1074 filp->private_data = NULL;
1075 kfree(rp);
1076 }
1077 if (filp->f_mode & FMODE_WRITE) {
1078 atomic_dec(&cd->writers);
1079 cd->last_close = seconds_since_boot();
1080 }
1081 module_put(cd->owner);
1082 return 0;
1083}
1084
1085
1086
1087static void cache_dequeue(struct cache_detail *detail, struct cache_head *ch)
1088{
1089 struct cache_request *cr, *tmp;
1090 LIST_HEAD(dequeued);
1091
1092 spin_lock(&detail->queue_lock);
1093 list_for_each_entry_safe(cr, tmp, &detail->requests, list) {
1094 if (cr->item != ch)
1095 continue;
1096 if (test_bit(CACHE_PENDING, &ch->flags))
1097 /* Lost a race and it is pending again */
1098 break;
1099 if (cr->readers != 0)
1100 continue;
1101 list_move(&cr->list, &dequeued);
1102 }
1103 spin_unlock(&detail->queue_lock);
1104 while (!list_empty(&dequeued)) {
1105 cr = list_entry(dequeued.next, struct cache_request, list);
1106 list_del(&cr->list);
1107 cache_put(cr->item, detail);
1108 kfree(cr->buf);
1109 kfree(cr);
1110 }
1111}
1112
1113/*
1114 * Support routines for text-based upcalls.
1115 * Fields are separated by spaces.
1116 * Fields are either mangled to quote space tab newline slosh with slosh
1117 * or a hexified with a leading \x
1118 * Record is terminated with newline.
1119 *
1120 */
1121
1122void qword_add(char **bpp, int *lp, char *str)
1123{
1124 char *bp = *bpp;
1125 int len = *lp;
1126 int ret;
1127
1128 if (len < 0) return;
1129
1130 ret = string_escape_str(str, bp, len, ESCAPE_OCTAL, "\\ \n\t");
1131 if (ret >= len) {
1132 bp += len;
1133 len = -1;
1134 } else {
1135 bp += ret;
1136 len -= ret;
1137 *bp++ = ' ';
1138 len--;
1139 }
1140 *bpp = bp;
1141 *lp = len;
1142}
1143EXPORT_SYMBOL_GPL(qword_add);
1144
1145void qword_addhex(char **bpp, int *lp, char *buf, int blen)
1146{
1147 char *bp = *bpp;
1148 int len = *lp;
1149
1150 if (len < 0) return;
1151
1152 if (len > 2) {
1153 *bp++ = '\\';
1154 *bp++ = 'x';
1155 len -= 2;
1156 while (blen && len >= 2) {
1157 bp = hex_byte_pack(bp, *buf++);
1158 len -= 2;
1159 blen--;
1160 }
1161 }
1162 if (blen || len<1) len = -1;
1163 else {
1164 *bp++ = ' ';
1165 len--;
1166 }
1167 *bpp = bp;
1168 *lp = len;
1169}
1170EXPORT_SYMBOL_GPL(qword_addhex);
1171
1172static void warn_no_listener(struct cache_detail *detail)
1173{
1174 if (detail->last_warn != detail->last_close) {
1175 detail->last_warn = detail->last_close;
1176 if (detail->warn_no_listener)
1177 detail->warn_no_listener(detail, detail->last_close != 0);
1178 }
1179}
1180
1181static bool cache_listeners_exist(struct cache_detail *detail)
1182{
1183 if (atomic_read(&detail->writers))
1184 return true;
1185 if (detail->last_close == 0)
1186 /* This cache was never opened */
1187 return false;
1188 if (detail->last_close < seconds_since_boot() - 30)
1189 /*
1190 * We allow for the possibility that someone might
1191 * restart a userspace daemon without restarting the
1192 * server; but after 30 seconds, we give up.
1193 */
1194 return false;
1195 return true;
1196}
1197
1198/*
1199 * register an upcall request to user-space and queue it up to be fetched by
1200 * the upcall daemon.
1201 *
1202 * Each request is at most one page long.
1203 */
1204static int cache_do_upcall(struct cache_detail *detail, struct cache_head *h)
1205{
1206 char *buf;
1207 struct cache_request *crq;
1208 int ret = 0;
1209
1210 if (test_bit(CACHE_CLEANED, &h->flags))
1211 /* Too late to make an upcall */
1212 return -EAGAIN;
1213
1214 buf = kmalloc(PAGE_SIZE, GFP_KERNEL);
1215 if (!buf)
1216 return -EAGAIN;
1217
1218 crq = kmalloc_obj(*crq);
1219 if (!crq) {
1220 kfree(buf);
1221 return -EAGAIN;
1222 }
1223
1224 crq->buf = buf;
1225 crq->len = 0;
1226 crq->readers = 0;
1227 spin_lock(&detail->queue_lock);
1228 if (test_bit(CACHE_PENDING, &h->flags)) {
1229 crq->item = cache_get(h);
1230 crq->seqno = detail->next_seqno++;
1231 list_add_tail(&crq->list, &detail->requests);
1232 trace_cache_entry_upcall(detail, h);
1233 } else
1234 /* Lost a race, no longer PENDING, so don't enqueue */
1235 ret = -EAGAIN;
1236 spin_unlock(&detail->queue_lock);
1237 if (ret != -EAGAIN && detail->cache_notify)
1238 detail->cache_notify(detail, h);
1239 wake_up(&detail->queue_wait);
1240 if (ret == -EAGAIN) {
1241 kfree(buf);
1242 kfree(crq);
1243 }
1244 return ret;
1245}
1246
1247int sunrpc_cache_upcall(struct cache_detail *detail, struct cache_head *h)
1248{
1249 if (test_and_set_bit(CACHE_PENDING, &h->flags))
1250 return 0;
1251 return cache_do_upcall(detail, h);
1252}
1253EXPORT_SYMBOL_GPL(sunrpc_cache_upcall);
1254
1255int sunrpc_cache_upcall_warn(struct cache_detail *detail,
1256 struct cache_head *h)
1257{
1258 if (!cache_listeners_exist(detail)) {
1259 warn_no_listener(detail);
1260 trace_cache_entry_no_listener(detail, h);
1261 return -EINVAL;
1262 }
1263 return sunrpc_cache_upcall(detail, h);
1264}
1265EXPORT_SYMBOL_GPL(sunrpc_cache_upcall_warn);
1266
1267/*
1268 * parse a message from user-space and pass it
1269 * to an appropriate cache
1270 * Messages are, like requests, separated into fields by
1271 * spaces and dequotes as \xHEXSTRING or embedded \nnn octal
1272 *
1273 * Message is
1274 * reply cachename expiry key ... content....
1275 *
1276 * key and content are both parsed by cache
1277 */
1278
1279int qword_get(char **bpp, char *dest, int bufsize)
1280{
1281 /* return bytes copied, or -1 on error */
1282 char *bp = *bpp;
1283 int len = 0;
1284
1285 while (*bp == ' ') bp++;
1286
1287 if (bp[0] == '\\' && bp[1] == 'x') {
1288 /* HEX STRING */
1289 bp += 2;
1290 while (len < bufsize - 1) {
1291 int h, l;
1292
1293 h = hex_to_bin(bp[0]);
1294 if (h < 0)
1295 break;
1296
1297 l = hex_to_bin(bp[1]);
1298 if (l < 0)
1299 break;
1300
1301 *dest++ = (h << 4) | l;
1302 bp += 2;
1303 len++;
1304 }
1305 } else {
1306 /* text with \nnn octal quoting */
1307 while (*bp != ' ' && *bp != '\n' && *bp && len < bufsize-1) {
1308 if (*bp == '\\' &&
1309 isodigit(bp[1]) && (bp[1] <= '3') &&
1310 isodigit(bp[2]) &&
1311 isodigit(bp[3])) {
1312 int byte = (*++bp -'0');
1313 bp++;
1314 byte = (byte << 3) | (*bp++ - '0');
1315 byte = (byte << 3) | (*bp++ - '0');
1316 *dest++ = byte;
1317 len++;
1318 } else {
1319 *dest++ = *bp++;
1320 len++;
1321 }
1322 }
1323 }
1324
1325 if (*bp != ' ' && *bp != '\n' && *bp != '\0')
1326 return -1;
1327 while (*bp == ' ') bp++;
1328 *bpp = bp;
1329 *dest = '\0';
1330 return len;
1331}
1332EXPORT_SYMBOL_GPL(qword_get);
1333
1334
1335/*
1336 * support /proc/net/rpc/$CACHENAME/content
1337 * as a seqfile.
1338 * We call ->cache_show passing NULL for the item to
1339 * get a header, then pass each real item in the cache
1340 */
1341
1342static void *__cache_seq_start(struct seq_file *m, loff_t *pos)
1343{
1344 loff_t n = *pos;
1345 unsigned int hash, entry;
1346 struct cache_head *ch;
1347 struct cache_detail *cd = m->private;
1348
1349 if (!n--)
1350 return SEQ_START_TOKEN;
1351 hash = n >> 32;
1352 entry = n & ((1LL<<32) - 1);
1353
1354 if (hash >= cd->hash_size)
1355 return NULL;
1356
1357 hlist_for_each_entry_rcu(ch, &cd->hash_table[hash], cache_list)
1358 if (!entry--)
1359 return ch;
1360 ch = NULL;
1361 while (!ch && ++hash < cd->hash_size)
1362 ch = hlist_entry_safe(rcu_dereference(
1363 hlist_first_rcu(&cd->hash_table[hash])),
1364 struct cache_head, cache_list);
1365
1366 *pos = ((long long)hash << 32) + 1;
1367 return ch;
1368}
1369
1370static void *cache_seq_next(struct seq_file *m, void *p, loff_t *pos)
1371{
1372 struct cache_head *ch = p;
1373 int hash = (*pos >> 32);
1374 struct cache_detail *cd = m->private;
1375
1376 if (p == SEQ_START_TOKEN) {
1377 hash = 0;
1378 ch = NULL;
1379 }
1380 while (hash < cd->hash_size) {
1381 if (ch)
1382 ch = hlist_entry_safe(
1383 rcu_dereference(
1384 hlist_next_rcu(&ch->cache_list)),
1385 struct cache_head, cache_list);
1386 else
1387 ch = hlist_entry_safe(
1388 rcu_dereference(
1389 hlist_first_rcu(&cd->hash_table[hash])),
1390 struct cache_head, cache_list);
1391 if (ch) {
1392 ++*pos;
1393 return ch;
1394 }
1395 hash++;
1396 *pos = (long long)hash << 32;
1397 }
1398 return NULL;
1399}
1400
1401void *cache_seq_start_rcu(struct seq_file *m, loff_t *pos)
1402 __acquires(RCU)
1403{
1404 rcu_read_lock();
1405 return __cache_seq_start(m, pos);
1406}
1407EXPORT_SYMBOL_GPL(cache_seq_start_rcu);
1408
1409void *cache_seq_next_rcu(struct seq_file *file, void *p, loff_t *pos)
1410{
1411 return cache_seq_next(file, p, pos);
1412}
1413EXPORT_SYMBOL_GPL(cache_seq_next_rcu);
1414
1415void cache_seq_stop_rcu(struct seq_file *m, void *p)
1416 __releases(RCU)
1417{
1418 rcu_read_unlock();
1419}
1420EXPORT_SYMBOL_GPL(cache_seq_stop_rcu);
1421
1422static int c_show(struct seq_file *m, void *p)
1423{
1424 struct cache_head *cp = p;
1425 struct cache_detail *cd = m->private;
1426
1427 if (p == SEQ_START_TOKEN)
1428 return cd->cache_show(m, cd, NULL);
1429
1430 ifdebug(CACHE)
1431 seq_printf(m, "# expiry=%lld refcnt=%d flags=%lx\n",
1432 convert_to_wallclock(cp->expiry_time),
1433 kref_read(&cp->ref), cp->flags);
1434
1435 if (cache_check_rcu(cd, cp, NULL))
1436 seq_puts(m, "# ");
1437 else if (cache_is_expired(cd, cp))
1438 seq_puts(m, "# ");
1439
1440 return cd->cache_show(m, cd, cp);
1441}
1442
1443static const struct seq_operations cache_content_op = {
1444 .start = cache_seq_start_rcu,
1445 .next = cache_seq_next_rcu,
1446 .stop = cache_seq_stop_rcu,
1447 .show = c_show,
1448};
1449
1450static int content_open(struct inode *inode, struct file *file,
1451 struct cache_detail *cd)
1452{
1453 struct seq_file *seq;
1454 int err;
1455
1456 if (!cd || !try_module_get(cd->owner))
1457 return -EACCES;
1458
1459 err = seq_open(file, &cache_content_op);
1460 if (err) {
1461 module_put(cd->owner);
1462 return err;
1463 }
1464
1465 seq = file->private_data;
1466 seq->private = cd;
1467 return 0;
1468}
1469
1470static int content_release(struct inode *inode, struct file *file,
1471 struct cache_detail *cd)
1472{
1473 int ret = seq_release(inode, file);
1474 module_put(cd->owner);
1475 return ret;
1476}
1477
1478static int open_flush(struct inode *inode, struct file *file,
1479 struct cache_detail *cd)
1480{
1481 if (!cd || !try_module_get(cd->owner))
1482 return -EACCES;
1483 return nonseekable_open(inode, file);
1484}
1485
1486static int release_flush(struct inode *inode, struct file *file,
1487 struct cache_detail *cd)
1488{
1489 module_put(cd->owner);
1490 return 0;
1491}
1492
1493static ssize_t read_flush(struct file *file, char __user *buf,
1494 size_t count, loff_t *ppos,
1495 struct cache_detail *cd)
1496{
1497 char tbuf[22];
1498 size_t len;
1499
1500 len = snprintf(tbuf, sizeof(tbuf), "%llu\n",
1501 convert_to_wallclock(cd->flush_time));
1502 return simple_read_from_buffer(buf, count, ppos, tbuf, len);
1503}
1504
1505static ssize_t write_flush(struct file *file, const char __user *buf,
1506 size_t count, loff_t *ppos,
1507 struct cache_detail *cd)
1508{
1509 char tbuf[20];
1510 char *ep;
1511 time64_t now;
1512
1513 if (*ppos || count > sizeof(tbuf)-1)
1514 return -EINVAL;
1515 if (copy_from_user(tbuf, buf, count))
1516 return -EFAULT;
1517 tbuf[count] = 0;
1518 simple_strtoul(tbuf, &ep, 0);
1519 if (*ep && *ep != '\n')
1520 return -EINVAL;
1521 /* Note that while we check that 'buf' holds a valid number,
1522 * we always ignore the value and just flush everything.
1523 * Making use of the number leads to races.
1524 */
1525
1526 now = seconds_since_boot();
1527 /* Always flush everything, so behave like cache_purge()
1528 * Do this by advancing flush_time to the current time,
1529 * or by one second if it has already reached the current time.
1530 * Newly added cache entries will always have ->last_refresh greater
1531 * that ->flush_time, so they don't get flushed prematurely.
1532 */
1533
1534 if (cd->flush_time >= now)
1535 now = cd->flush_time + 1;
1536
1537 cd->flush_time = now;
1538 cd->nextcheck = now;
1539 cache_flush();
1540
1541 if (cd->flush)
1542 cd->flush();
1543
1544 *ppos += count;
1545 return count;
1546}
1547
1548static ssize_t cache_read_procfs(struct file *filp, char __user *buf,
1549 size_t count, loff_t *ppos)
1550{
1551 struct cache_detail *cd = pde_data(file_inode(filp));
1552
1553 return cache_read(filp, buf, count, ppos, cd);
1554}
1555
1556static ssize_t cache_write_procfs(struct file *filp, const char __user *buf,
1557 size_t count, loff_t *ppos)
1558{
1559 struct cache_detail *cd = pde_data(file_inode(filp));
1560
1561 return cache_write(filp, buf, count, ppos, cd);
1562}
1563
1564static __poll_t cache_poll_procfs(struct file *filp, poll_table *wait)
1565{
1566 struct cache_detail *cd = pde_data(file_inode(filp));
1567
1568 return cache_poll(filp, wait, cd);
1569}
1570
1571static long cache_ioctl_procfs(struct file *filp,
1572 unsigned int cmd, unsigned long arg)
1573{
1574 struct inode *inode = file_inode(filp);
1575 struct cache_detail *cd = pde_data(inode);
1576
1577 return cache_ioctl(inode, filp, cmd, arg, cd);
1578}
1579
1580static int cache_open_procfs(struct inode *inode, struct file *filp)
1581{
1582 struct cache_detail *cd = pde_data(inode);
1583
1584 return cache_open(inode, filp, cd);
1585}
1586
1587static int cache_release_procfs(struct inode *inode, struct file *filp)
1588{
1589 struct cache_detail *cd = pde_data(inode);
1590
1591 return cache_release(inode, filp, cd);
1592}
1593
1594static const struct proc_ops cache_channel_proc_ops = {
1595 .proc_read = cache_read_procfs,
1596 .proc_write = cache_write_procfs,
1597 .proc_poll = cache_poll_procfs,
1598 .proc_ioctl = cache_ioctl_procfs, /* for FIONREAD */
1599 .proc_open = cache_open_procfs,
1600 .proc_release = cache_release_procfs,
1601};
1602
1603static int content_open_procfs(struct inode *inode, struct file *filp)
1604{
1605 struct cache_detail *cd = pde_data(inode);
1606
1607 return content_open(inode, filp, cd);
1608}
1609
1610static int content_release_procfs(struct inode *inode, struct file *filp)
1611{
1612 struct cache_detail *cd = pde_data(inode);
1613
1614 return content_release(inode, filp, cd);
1615}
1616
1617static const struct proc_ops content_proc_ops = {
1618 .proc_open = content_open_procfs,
1619 .proc_read = seq_read,
1620 .proc_lseek = seq_lseek,
1621 .proc_release = content_release_procfs,
1622};
1623
1624static int open_flush_procfs(struct inode *inode, struct file *filp)
1625{
1626 struct cache_detail *cd = pde_data(inode);
1627
1628 return open_flush(inode, filp, cd);
1629}
1630
1631static int release_flush_procfs(struct inode *inode, struct file *filp)
1632{
1633 struct cache_detail *cd = pde_data(inode);
1634
1635 return release_flush(inode, filp, cd);
1636}
1637
1638static ssize_t read_flush_procfs(struct file *filp, char __user *buf,
1639 size_t count, loff_t *ppos)
1640{
1641 struct cache_detail *cd = pde_data(file_inode(filp));
1642
1643 return read_flush(filp, buf, count, ppos, cd);
1644}
1645
1646static ssize_t write_flush_procfs(struct file *filp,
1647 const char __user *buf,
1648 size_t count, loff_t *ppos)
1649{
1650 struct cache_detail *cd = pde_data(file_inode(filp));
1651
1652 return write_flush(filp, buf, count, ppos, cd);
1653}
1654
1655static const struct proc_ops cache_flush_proc_ops = {
1656 .proc_open = open_flush_procfs,
1657 .proc_read = read_flush_procfs,
1658 .proc_write = write_flush_procfs,
1659 .proc_release = release_flush_procfs,
1660};
1661
1662static void remove_cache_proc_entries(struct cache_detail *cd)
1663{
1664 if (cd->procfs) {
1665 proc_remove(cd->procfs);
1666 cd->procfs = NULL;
1667 }
1668}
1669
1670static int create_cache_proc_entries(struct cache_detail *cd, struct net *net)
1671{
1672 struct proc_dir_entry *p;
1673 struct sunrpc_net *sn;
1674
1675 if (!IS_ENABLED(CONFIG_PROC_FS))
1676 return 0;
1677
1678 sn = net_generic(net, sunrpc_net_id);
1679 cd->procfs = proc_mkdir(cd->name, sn->proc_net_rpc);
1680 if (cd->procfs == NULL)
1681 goto out_nomem;
1682
1683 p = proc_create_data("flush", S_IFREG | 0600,
1684 cd->procfs, &cache_flush_proc_ops, cd);
1685 if (p == NULL)
1686 goto out_nomem;
1687
1688 if (cd->cache_request || cd->cache_parse) {
1689 p = proc_create_data("channel", S_IFREG | 0600, cd->procfs,
1690 &cache_channel_proc_ops, cd);
1691 if (p == NULL)
1692 goto out_nomem;
1693 }
1694 if (cd->cache_show) {
1695 p = proc_create_data("content", S_IFREG | 0400, cd->procfs,
1696 &content_proc_ops, cd);
1697 if (p == NULL)
1698 goto out_nomem;
1699 }
1700 return 0;
1701out_nomem:
1702 remove_cache_proc_entries(cd);
1703 return -ENOMEM;
1704}
1705
1706void __init cache_initialize(void)
1707{
1708 INIT_DEFERRABLE_WORK(&cache_cleaner, do_cache_clean);
1709}
1710
1711int cache_register_net(struct cache_detail *cd, struct net *net)
1712{
1713 int ret;
1714
1715 sunrpc_init_cache_detail(cd);
1716 ret = create_cache_proc_entries(cd, net);
1717 if (ret)
1718 sunrpc_destroy_cache_detail(cd);
1719 return ret;
1720}
1721EXPORT_SYMBOL_GPL(cache_register_net);
1722
1723void cache_unregister_net(struct cache_detail *cd, struct net *net)
1724{
1725 remove_cache_proc_entries(cd);
1726 sunrpc_destroy_cache_detail(cd);
1727}
1728EXPORT_SYMBOL_GPL(cache_unregister_net);
1729
1730struct cache_detail *cache_create_net(const struct cache_detail *tmpl, struct net *net)
1731{
1732 struct cache_detail *cd;
1733 int i;
1734
1735 cd = kmemdup(tmpl, sizeof(struct cache_detail), GFP_KERNEL);
1736 if (cd == NULL)
1737 return ERR_PTR(-ENOMEM);
1738
1739 cd->hash_table = kzalloc_objs(struct hlist_head, cd->hash_size);
1740 if (cd->hash_table == NULL) {
1741 kfree(cd);
1742 return ERR_PTR(-ENOMEM);
1743 }
1744
1745 for (i = 0; i < cd->hash_size; i++)
1746 INIT_HLIST_HEAD(&cd->hash_table[i]);
1747 cd->net = net;
1748 return cd;
1749}
1750EXPORT_SYMBOL_GPL(cache_create_net);
1751
1752void cache_destroy_net(struct cache_detail *cd, struct net *net)
1753{
1754 kfree(cd->hash_table);
1755 kfree(cd);
1756}
1757EXPORT_SYMBOL_GPL(cache_destroy_net);
1758
1759static ssize_t cache_read_pipefs(struct file *filp, char __user *buf,
1760 size_t count, loff_t *ppos)
1761{
1762 struct cache_detail *cd = RPC_I(file_inode(filp))->private;
1763
1764 return cache_read(filp, buf, count, ppos, cd);
1765}
1766
1767static ssize_t cache_write_pipefs(struct file *filp, const char __user *buf,
1768 size_t count, loff_t *ppos)
1769{
1770 struct cache_detail *cd = RPC_I(file_inode(filp))->private;
1771
1772 return cache_write(filp, buf, count, ppos, cd);
1773}
1774
1775static __poll_t cache_poll_pipefs(struct file *filp, poll_table *wait)
1776{
1777 struct cache_detail *cd = RPC_I(file_inode(filp))->private;
1778
1779 return cache_poll(filp, wait, cd);
1780}
1781
1782static long cache_ioctl_pipefs(struct file *filp,
1783 unsigned int cmd, unsigned long arg)
1784{
1785 struct inode *inode = file_inode(filp);
1786 struct cache_detail *cd = RPC_I(inode)->private;
1787
1788 return cache_ioctl(inode, filp, cmd, arg, cd);
1789}
1790
1791static int cache_open_pipefs(struct inode *inode, struct file *filp)
1792{
1793 struct cache_detail *cd = RPC_I(inode)->private;
1794
1795 return cache_open(inode, filp, cd);
1796}
1797
1798static int cache_release_pipefs(struct inode *inode, struct file *filp)
1799{
1800 struct cache_detail *cd = RPC_I(inode)->private;
1801
1802 return cache_release(inode, filp, cd);
1803}
1804
1805const struct file_operations cache_file_operations_pipefs = {
1806 .owner = THIS_MODULE,
1807 .read = cache_read_pipefs,
1808 .write = cache_write_pipefs,
1809 .poll = cache_poll_pipefs,
1810 .unlocked_ioctl = cache_ioctl_pipefs, /* for FIONREAD */
1811 .open = cache_open_pipefs,
1812 .release = cache_release_pipefs,
1813};
1814
1815static int content_open_pipefs(struct inode *inode, struct file *filp)
1816{
1817 struct cache_detail *cd = RPC_I(inode)->private;
1818
1819 return content_open(inode, filp, cd);
1820}
1821
1822static int content_release_pipefs(struct inode *inode, struct file *filp)
1823{
1824 struct cache_detail *cd = RPC_I(inode)->private;
1825
1826 return content_release(inode, filp, cd);
1827}
1828
1829const struct file_operations content_file_operations_pipefs = {
1830 .open = content_open_pipefs,
1831 .read = seq_read,
1832 .llseek = seq_lseek,
1833 .release = content_release_pipefs,
1834};
1835
1836static int open_flush_pipefs(struct inode *inode, struct file *filp)
1837{
1838 struct cache_detail *cd = RPC_I(inode)->private;
1839
1840 return open_flush(inode, filp, cd);
1841}
1842
1843static int release_flush_pipefs(struct inode *inode, struct file *filp)
1844{
1845 struct cache_detail *cd = RPC_I(inode)->private;
1846
1847 return release_flush(inode, filp, cd);
1848}
1849
1850static ssize_t read_flush_pipefs(struct file *filp, char __user *buf,
1851 size_t count, loff_t *ppos)
1852{
1853 struct cache_detail *cd = RPC_I(file_inode(filp))->private;
1854
1855 return read_flush(filp, buf, count, ppos, cd);
1856}
1857
1858static ssize_t write_flush_pipefs(struct file *filp,
1859 const char __user *buf,
1860 size_t count, loff_t *ppos)
1861{
1862 struct cache_detail *cd = RPC_I(file_inode(filp))->private;
1863
1864 return write_flush(filp, buf, count, ppos, cd);
1865}
1866
1867const struct file_operations cache_flush_operations_pipefs = {
1868 .open = open_flush_pipefs,
1869 .read = read_flush_pipefs,
1870 .write = write_flush_pipefs,
1871 .release = release_flush_pipefs,
1872};
1873
1874int sunrpc_cache_register_pipefs(struct dentry *parent,
1875 const char *name, umode_t umode,
1876 struct cache_detail *cd)
1877{
1878 struct dentry *dir = rpc_create_cache_dir(parent, name, umode, cd);
1879 if (IS_ERR(dir))
1880 return PTR_ERR(dir);
1881 cd->pipefs = dir;
1882 return 0;
1883}
1884EXPORT_SYMBOL_GPL(sunrpc_cache_register_pipefs);
1885
1886void sunrpc_cache_unregister_pipefs(struct cache_detail *cd)
1887{
1888 if (cd->pipefs) {
1889 rpc_remove_cache_dir(cd->pipefs);
1890 cd->pipefs = NULL;
1891 }
1892}
1893EXPORT_SYMBOL_GPL(sunrpc_cache_unregister_pipefs);
1894
1895void sunrpc_cache_unhash(struct cache_detail *cd, struct cache_head *h)
1896{
1897 spin_lock(&cd->hash_lock);
1898 if (!hlist_unhashed(&h->cache_list)){
1899 sunrpc_begin_cache_remove_entry(h, cd);
1900 spin_unlock(&cd->hash_lock);
1901 sunrpc_end_cache_remove_entry(h, cd);
1902 } else
1903 spin_unlock(&cd->hash_lock);
1904}
1905EXPORT_SYMBOL_GPL(sunrpc_cache_unhash);
1906
1907/**
1908 * sunrpc_cache_requests_count - count pending upcall requests
1909 * @cd: cache_detail to query
1910 *
1911 * Returns the number of requests on the cache's request list that
1912 * still have CACHE_PENDING set.
1913 */
1914int sunrpc_cache_requests_count(struct cache_detail *cd)
1915{
1916 struct cache_request *crq;
1917 int cnt = 0;
1918
1919 spin_lock(&cd->queue_lock);
1920 list_for_each_entry(crq, &cd->requests, list) {
1921 if (test_bit(CACHE_PENDING, &crq->item->flags))
1922 cnt++;
1923 }
1924 spin_unlock(&cd->queue_lock);
1925 return cnt;
1926}
1927EXPORT_SYMBOL_GPL(sunrpc_cache_requests_count);
1928
1929/**
1930 * sunrpc_cache_requests_snapshot - snapshot pending upcall requests
1931 * @cd: cache_detail to query
1932 * @items: array to fill with cache_head pointers (caller-allocated)
1933 * @seqnos: array to fill with sequence numbers (caller-allocated)
1934 * @max: size of the arrays
1935 * @min_seqno: only include entries with seqno > min_seqno (0 for all)
1936 *
1937 * Only entries with CACHE_PENDING set are included. Takes a reference
1938 * on each cache_head via cache_get(). Caller must call cache_put()
1939 * on each returned item when done.
1940 *
1941 * Returns the number of entries filled.
1942 */
1943int sunrpc_cache_requests_snapshot(struct cache_detail *cd,
1944 struct cache_head **items,
1945 u64 *seqnos, int max,
1946 u64 min_seqno)
1947{
1948 struct cache_request *crq;
1949 int i = 0;
1950
1951 spin_lock(&cd->queue_lock);
1952 list_for_each_entry(crq, &cd->requests, list) {
1953 if (i >= max)
1954 break;
1955 if (!test_bit(CACHE_PENDING, &crq->item->flags))
1956 continue;
1957 if (crq->seqno <= min_seqno)
1958 continue;
1959 items[i] = cache_get(crq->item);
1960 seqnos[i] = crq->seqno;
1961 i++;
1962 }
1963 spin_unlock(&cd->queue_lock);
1964 return i;
1965}
1966EXPORT_SYMBOL_GPL(sunrpc_cache_requests_snapshot);
1967
1968/**
1969 * sunrpc_cache_notify - send a netlink notification for a cache event
1970 * @cd: cache_detail for the cache
1971 * @h: cache_head entry (unused, reserved for future use)
1972 * @cache_type: cache type identifier (e.g. SUNRPC_CACHE_TYPE_UNIX_GID)
1973 *
1974 * Sends a SUNRPC_CMD_CACHE_NOTIFY multicast message on the "exportd"
1975 * group if any listeners are present. Returns 0 on success or a
1976 * negative errno.
1977 */
1978int sunrpc_cache_notify(struct cache_detail *cd, struct cache_head *h,
1979 u32 cache_type)
1980{
1981 struct genlmsghdr *hdr;
1982 struct sk_buff *msg;
1983
1984 if (!genl_has_listeners(&sunrpc_nl_family, cd->net,
1985 SUNRPC_NLGRP_EXPORTD))
1986 return -ENOLINK;
1987
1988 msg = genlmsg_new(nla_total_size(sizeof(u32)), GFP_KERNEL);
1989 if (!msg)
1990 return -ENOMEM;
1991
1992 hdr = genlmsg_put(msg, 0, 0, &sunrpc_nl_family, 0,
1993 SUNRPC_CMD_CACHE_NOTIFY);
1994 if (!hdr) {
1995 nlmsg_free(msg);
1996 return -ENOMEM;
1997 }
1998
1999 if (nla_put_u32(msg, SUNRPC_A_CACHE_NOTIFY_CACHE_TYPE, cache_type)) {
2000 nlmsg_free(msg);
2001 return -ENOMEM;
2002 }
2003
2004 genlmsg_end(msg, hdr);
2005 return genlmsg_multicast_netns(&sunrpc_nl_family, cd->net, msg, 0,
2006 SUNRPC_NLGRP_EXPORTD, GFP_KERNEL);
2007}
2008EXPORT_SYMBOL_GPL(sunrpc_cache_notify);