projects/linux·blob master · refs

532 lines · 14.9 KiB

1// SPDX-License-Identifier: GPL-2.0-or-later
2/* Volume-level cache cookie handling.
3 *
4 * Copyright (C) 2021 Red Hat, Inc. All Rights Reserved.
5 * Written by David Howells ([email protected])
6 */
7
8#define FSCACHE_DEBUG_LEVEL COOKIE
9#include <linux/export.h>
10#include <linux/slab.h>
11#include "internal.h"
12
13#define fscache_volume_hash_shift 10
14static struct hlist_bl_head fscache_volume_hash[1 << fscache_volume_hash_shift];
15static atomic_t fscache_volume_debug_id;
16static LIST_HEAD(fscache_volumes);
17
18static void fscache_create_volume_work(struct work_struct *work);
19
20struct fscache_volume *fscache_get_volume(struct fscache_volume *volume,
21 enum fscache_volume_trace where)
22{
23 int ref;
24
25 __refcount_inc(&volume->ref, &ref);
26 trace_fscache_volume(volume->debug_id, ref + 1, where);
27 return volume;
28}
29
30struct fscache_volume *fscache_try_get_volume(struct fscache_volume *volume,
31 enum fscache_volume_trace where)
32{
33 int ref;
34
35 if (!__refcount_inc_not_zero(&volume->ref, &ref))
36 return NULL;
37
38 trace_fscache_volume(volume->debug_id, ref + 1, where);
39 return volume;
40}
41EXPORT_SYMBOL(fscache_try_get_volume);
42
43static void fscache_see_volume(struct fscache_volume *volume,
44 enum fscache_volume_trace where)
45{
46 int ref = refcount_read(&volume->ref);
47
48 trace_fscache_volume(volume->debug_id, ref, where);
49}
50
51/*
52 * Pin the cache behind a volume so that we can access it.
53 */
54static void __fscache_begin_volume_access(struct fscache_volume *volume,
55 struct fscache_cookie *cookie,
56 enum fscache_access_trace why)
57{
58 int n_accesses;
59
60 n_accesses = atomic_inc_return(&volume->n_accesses);
61 smp_mb__after_atomic();
62 trace_fscache_access_volume(volume->debug_id, cookie ? cookie->debug_id : 0,
63 refcount_read(&volume->ref),
64 n_accesses, why);
65}
66
67/**
68 * fscache_begin_volume_access - Pin a cache so a volume can be accessed
69 * @volume: The volume cookie
70 * @cookie: A datafile cookie for a tracing reference (or NULL)
71 * @why: An indication of the circumstances of the access for tracing
72 *
73 * Attempt to pin the cache to prevent it from going away whilst we're
74 * accessing a volume and returns true if successful. This works as follows:
75 *
76 * (1) If the cache tests as not live (state is not FSCACHE_CACHE_IS_ACTIVE),
77 * then we return false to indicate access was not permitted.
78 *
79 * (2) If the cache tests as live, then we increment the volume's n_accesses
80 * count and then recheck the cache liveness, ending the access if it
81 * ceased to be live.
82 *
83 * (3) When we end the access, we decrement the volume's n_accesses and wake
84 * up the any waiters if it reaches 0.
85 *
86 * (4) Whilst the cache is caching, the volume's n_accesses is kept
87 * artificially incremented to prevent wakeups from happening.
88 *
89 * (5) When the cache is taken offline, the state is changed to prevent new
90 * accesses, the volume's n_accesses is decremented and we wait for it to
91 * become 0.
92 *
93 * The datafile @cookie and the @why indicator are merely provided for tracing
94 * purposes.
95 */
96bool fscache_begin_volume_access(struct fscache_volume *volume,
97 struct fscache_cookie *cookie,
98 enum fscache_access_trace why)
99{
100 if (!fscache_cache_is_live(volume->cache))
101 return false;
102 __fscache_begin_volume_access(volume, cookie, why);
103 if (!fscache_cache_is_live(volume->cache)) {
104 fscache_end_volume_access(volume, cookie, fscache_access_unlive);
105 return false;
106 }
107 return true;
108}
109
110/**
111 * fscache_end_volume_access - Unpin a cache at the end of an access.
112 * @volume: The volume cookie
113 * @cookie: A datafile cookie for a tracing reference (or NULL)
114 * @why: An indication of the circumstances of the access for tracing
115 *
116 * Unpin a cache volume after we've accessed it. The datafile @cookie and the
117 * @why indicator are merely provided for tracing purposes.
118 */
119void fscache_end_volume_access(struct fscache_volume *volume,
120 struct fscache_cookie *cookie,
121 enum fscache_access_trace why)
122{
123 int n_accesses;
124
125 smp_mb__before_atomic();
126 n_accesses = atomic_dec_return(&volume->n_accesses);
127 trace_fscache_access_volume(volume->debug_id, cookie ? cookie->debug_id : 0,
128 refcount_read(&volume->ref),
129 n_accesses, why);
130 if (n_accesses == 0)
131 wake_up_var(&volume->n_accesses);
132}
133EXPORT_SYMBOL(fscache_end_volume_access);
134
135static bool fscache_volume_same(const struct fscache_volume *a,
136 const struct fscache_volume *b)
137{
138 size_t klen;
139
140 if (a->key_hash != b->key_hash ||
141 a->cache != b->cache ||
142 a->key[0] != b->key[0])
143 return false;
144
145 klen = round_up(a->key[0] + 1, sizeof(__le32));
146 return memcmp(a->key, b->key, klen) == 0;
147}
148
149static bool fscache_is_acquire_pending(struct fscache_volume *volume)
150{
151 return test_bit(FSCACHE_VOLUME_ACQUIRE_PENDING, &volume->flags);
152}
153
154static void fscache_wait_on_volume_collision(struct fscache_volume *candidate,
155 unsigned int collidee_debug_id)
156{
157 wait_on_bit_timeout(&candidate->flags, FSCACHE_VOLUME_ACQUIRE_PENDING,
158 TASK_UNINTERRUPTIBLE, 20 * HZ);
159 if (fscache_is_acquire_pending(candidate)) {
160 pr_notice("Potential volume collision new=%08x old=%08x",
161 candidate->debug_id, collidee_debug_id);
162 fscache_stat(&fscache_n_volumes_collision);
163 wait_on_bit(&candidate->flags, FSCACHE_VOLUME_ACQUIRE_PENDING,
164 TASK_UNINTERRUPTIBLE);
165 }
166}
167
168/*
169 * Attempt to insert the new volume into the hash. If there's a collision, we
170 * wait for the old volume to complete if it's being relinquished and an error
171 * otherwise.
172 */
173static bool fscache_hash_volume(struct fscache_volume *candidate)
174{
175 struct fscache_volume *cursor;
176 struct hlist_bl_head *h;
177 struct hlist_bl_node *p;
178 unsigned int bucket, collidee_debug_id = 0;
179
180 bucket = candidate->key_hash & (ARRAY_SIZE(fscache_volume_hash) - 1);
181 h = &fscache_volume_hash[bucket];
182
183 hlist_bl_lock(h);
184 hlist_bl_for_each_entry(cursor, p, h, hash_link) {
185 if (fscache_volume_same(candidate, cursor)) {
186 if (!test_bit(FSCACHE_VOLUME_RELINQUISHED, &cursor->flags))
187 goto collision;
188 fscache_see_volume(cursor, fscache_volume_get_hash_collision);
189 set_bit(FSCACHE_VOLUME_COLLIDED_WITH, &cursor->flags);
190 set_bit(FSCACHE_VOLUME_ACQUIRE_PENDING, &candidate->flags);
191 collidee_debug_id = cursor->debug_id;
192 break;
193 }
194 }
195
196 hlist_bl_add_head(&candidate->hash_link, h);
197 hlist_bl_unlock(h);
198
199 if (fscache_is_acquire_pending(candidate))
200 fscache_wait_on_volume_collision(candidate, collidee_debug_id);
201 return true;
202
203collision:
204 fscache_see_volume(cursor, fscache_volume_collision);
205 hlist_bl_unlock(h);
206 return false;
207}
208
209/*
210 * Allocate and initialise a volume representation cookie.
211 */
212static struct fscache_volume *fscache_alloc_volume(const char *volume_key,
213 const char *cache_name,
214 const void *coherency_data,
215 size_t coherency_len)
216{
217 struct fscache_volume *volume;
218 struct fscache_cache *cache;
219 size_t klen, hlen;
220 u8 *key;
221
222 klen = strlen(volume_key);
223 if (klen > NAME_MAX)
224 return NULL;
225
226 if (!coherency_data)
227 coherency_len = 0;
228
229 cache = fscache_lookup_cache(cache_name, false);
230 if (IS_ERR(cache))
231 return NULL;
232
233 volume = kzalloc_flex(*volume, coherency, coherency_len);
234 if (!volume)
235 goto err_cache;
236
237 volume->cache = cache;
238 volume->coherency_len = coherency_len;
239 if (coherency_data)
240 memcpy(volume->coherency, coherency_data, coherency_len);
241 INIT_LIST_HEAD(&volume->proc_link);
242 INIT_WORK(&volume->work, fscache_create_volume_work);
243 refcount_set(&volume->ref, 1);
244 spin_lock_init(&volume->lock);
245
246 /* Stick the length on the front of the key and pad it out to make
247 * hashing easier.
248 */
249 hlen = round_up(1 + klen + 1, sizeof(__le32));
250 key = kzalloc(hlen, GFP_KERNEL);
251 if (!key)
252 goto err_vol;
253 key[0] = klen;
254 memcpy(key + 1, volume_key, klen);
255
256 volume->key = key;
257 volume->key_hash = fscache_hash(0, key, hlen);
258
259 volume->debug_id = atomic_inc_return(&fscache_volume_debug_id);
260 down_write(&fscache_addremove_sem);
261 atomic_inc(&cache->n_volumes);
262 list_add_tail(&volume->proc_link, &fscache_volumes);
263 fscache_see_volume(volume, fscache_volume_new_acquire);
264 fscache_stat(&fscache_n_volumes);
265 up_write(&fscache_addremove_sem);
266 _leave(" = v=%x", volume->debug_id);
267 return volume;
268
269err_vol:
270 kfree(volume);
271err_cache:
272 fscache_put_cache(cache, fscache_cache_put_alloc_volume);
273 fscache_stat(&fscache_n_volumes_nomem);
274 return NULL;
275}
276
277/*
278 * Create a volume's representation on disk. Have a volume ref and a cache
279 * access we have to release.
280 */
281static void fscache_create_volume_work(struct work_struct *work)
282{
283 const struct fscache_cache_ops *ops;
284 struct fscache_volume *volume =
285 container_of(work, struct fscache_volume, work);
286
287 fscache_see_volume(volume, fscache_volume_see_create_work);
288
289 ops = volume->cache->ops;
290 if (ops->acquire_volume)
291 ops->acquire_volume(volume);
292 fscache_end_cache_access(volume->cache,
293 fscache_access_acquire_volume_end);
294
295 clear_and_wake_up_bit(FSCACHE_VOLUME_CREATING, &volume->flags);
296 fscache_put_volume(volume, fscache_volume_put_create_work);
297}
298
299/*
300 * Dispatch a worker thread to create a volume's representation on disk.
301 */
302void fscache_create_volume(struct fscache_volume *volume, bool wait)
303{
304 if (test_and_set_bit(FSCACHE_VOLUME_CREATING, &volume->flags))
305 goto maybe_wait;
306 if (volume->cache_priv)
307 goto no_wait; /* We raced */
308 if (!fscache_begin_cache_access(volume->cache,
309 fscache_access_acquire_volume))
310 goto no_wait;
311
312 fscache_get_volume(volume, fscache_volume_get_create_work);
313 if (!schedule_work(&volume->work))
314 fscache_put_volume(volume, fscache_volume_put_create_work);
315
316maybe_wait:
317 if (wait) {
318 fscache_see_volume(volume, fscache_volume_wait_create_work);
319 wait_on_bit(&volume->flags, FSCACHE_VOLUME_CREATING,
320 TASK_UNINTERRUPTIBLE);
321 }
322 return;
323no_wait:
324 clear_and_wake_up_bit(FSCACHE_VOLUME_CREATING, &volume->flags);
325}
326
327/*
328 * Acquire a volume representation cookie and link it to a (proposed) cache.
329 */
330struct fscache_volume *__fscache_acquire_volume(const char *volume_key,
331 const char *cache_name,
332 const void *coherency_data,
333 size_t coherency_len)
334{
335 struct fscache_volume *volume;
336
337 volume = fscache_alloc_volume(volume_key, cache_name,
338 coherency_data, coherency_len);
339 if (!volume)
340 return ERR_PTR(-ENOMEM);
341
342 if (!fscache_hash_volume(volume)) {
343 fscache_put_volume(volume, fscache_volume_put_hash_collision);
344 return ERR_PTR(-EBUSY);
345 }
346
347 fscache_create_volume(volume, false);
348 return volume;
349}
350EXPORT_SYMBOL(__fscache_acquire_volume);
351
352static void fscache_wake_pending_volume(struct fscache_volume *volume,
353 struct hlist_bl_head *h)
354{
355 struct fscache_volume *cursor;
356 struct hlist_bl_node *p;
357
358 hlist_bl_for_each_entry(cursor, p, h, hash_link) {
359 if (fscache_volume_same(cursor, volume)) {
360 fscache_see_volume(cursor, fscache_volume_see_hash_wake);
361 clear_and_wake_up_bit(FSCACHE_VOLUME_ACQUIRE_PENDING,
362 &cursor->flags);
363 return;
364 }
365 }
366}
367
368/*
369 * Remove a volume cookie from the hash table.
370 */
371static void fscache_unhash_volume(struct fscache_volume *volume)
372{
373 struct hlist_bl_head *h;
374 unsigned int bucket;
375
376 bucket = volume->key_hash & (ARRAY_SIZE(fscache_volume_hash) - 1);
377 h = &fscache_volume_hash[bucket];
378
379 hlist_bl_lock(h);
380 hlist_bl_del(&volume->hash_link);
381 if (test_bit(FSCACHE_VOLUME_COLLIDED_WITH, &volume->flags))
382 fscache_wake_pending_volume(volume, h);
383 hlist_bl_unlock(h);
384}
385
386/*
387 * Drop a cache's volume attachments.
388 */
389static void fscache_free_volume(struct fscache_volume *volume)
390{
391 struct fscache_cache *cache = volume->cache;
392
393 if (volume->cache_priv) {
394 __fscache_begin_volume_access(volume, NULL,
395 fscache_access_relinquish_volume);
396 if (volume->cache_priv)
397 cache->ops->free_volume(volume);
398 fscache_end_volume_access(volume, NULL,
399 fscache_access_relinquish_volume_end);
400 }
401
402 down_write(&fscache_addremove_sem);
403 list_del_init(&volume->proc_link);
404 atomic_dec(&volume->cache->n_volumes);
405 up_write(&fscache_addremove_sem);
406
407 if (!hlist_bl_unhashed(&volume->hash_link))
408 fscache_unhash_volume(volume);
409
410 trace_fscache_volume(volume->debug_id, 0, fscache_volume_free);
411 kfree(volume->key);
412 kfree(volume);
413 fscache_stat_d(&fscache_n_volumes);
414 fscache_put_cache(cache, fscache_cache_put_volume);
415}
416
417/*
418 * Drop a reference to a volume cookie.
419 */
420void fscache_put_volume(struct fscache_volume *volume,
421 enum fscache_volume_trace where)
422{
423 if (volume) {
424 unsigned int debug_id = volume->debug_id;
425 bool zero;
426 int ref;
427
428 zero = __refcount_dec_and_test(&volume->ref, &ref);
429 trace_fscache_volume(debug_id, ref - 1, where);
430 if (zero)
431 fscache_free_volume(volume);
432 }
433}
434EXPORT_SYMBOL(fscache_put_volume);
435
436/*
437 * Relinquish a volume representation cookie.
438 */
439void __fscache_relinquish_volume(struct fscache_volume *volume,
440 const void *coherency_data,
441 bool invalidate)
442{
443 if (WARN_ON(test_and_set_bit(FSCACHE_VOLUME_RELINQUISHED, &volume->flags)))
444 return;
445
446 if (invalidate) {
447 set_bit(FSCACHE_VOLUME_INVALIDATE, &volume->flags);
448 } else if (coherency_data) {
449 memcpy(volume->coherency, coherency_data, volume->coherency_len);
450 }
451
452 fscache_put_volume(volume, fscache_volume_put_relinquish);
453}
454EXPORT_SYMBOL(__fscache_relinquish_volume);
455
456/**
457 * fscache_withdraw_volume - Withdraw a volume from being cached
458 * @volume: Volume cookie
459 *
460 * Withdraw a cache volume from service, waiting for all accesses to complete
461 * before returning.
462 */
463void fscache_withdraw_volume(struct fscache_volume *volume)
464{
465 int n_accesses;
466
467 _debug("withdraw V=%x", volume->debug_id);
468
469 /* Allow wakeups on dec-to-0 */
470 n_accesses = atomic_dec_return(&volume->n_accesses);
471 trace_fscache_access_volume(volume->debug_id, 0,
472 refcount_read(&volume->ref),
473 n_accesses, fscache_access_cache_unpin);
474
475 wait_var_event(&volume->n_accesses,
476 atomic_read(&volume->n_accesses) == 0);
477}
478EXPORT_SYMBOL(fscache_withdraw_volume);
479
480#ifdef CONFIG_PROC_FS
481/*
482 * Generate a list of volumes in /proc/fs/fscache/volumes
483 */
484static int fscache_volumes_seq_show(struct seq_file *m, void *v)
485{
486 struct fscache_volume *volume;
487
488 if (v == &fscache_volumes) {
489 seq_puts(m,
490 "VOLUME REF nCOOK ACC FL CACHE KEY\n"
491 "======== ===== ===== === == =============== ================\n");
492 return 0;
493 }
494
495 volume = list_entry(v, struct fscache_volume, proc_link);
496 seq_printf(m,
497 "%08x %5d %5d %3d %02lx %-15.15s %s\n",
498 volume->debug_id,
499 refcount_read(&volume->ref),
500 atomic_read(&volume->n_cookies),
501 atomic_read(&volume->n_accesses),
502 volume->flags,
503 volume->cache->name ?: "-",
504 volume->key + 1);
505 return 0;
506}
507
508static void *fscache_volumes_seq_start(struct seq_file *m, loff_t *_pos)
509 __acquires(&fscache_addremove_sem)
510{
511 down_read(&fscache_addremove_sem);
512 return seq_list_start_head(&fscache_volumes, *_pos);
513}
514
515static void *fscache_volumes_seq_next(struct seq_file *m, void *v, loff_t *_pos)
516{
517 return seq_list_next(v, &fscache_volumes, _pos);
518}
519
520static void fscache_volumes_seq_stop(struct seq_file *m, void *v)
521 __releases(&fscache_addremove_sem)
522{
523 up_read(&fscache_addremove_sem);
524}
525
526const struct seq_operations fscache_volumes_seq_ops = {
527 .start = fscache_volumes_seq_start,
528 .next = fscache_volumes_seq_next,
529 .stop = fscache_volumes_seq_stop,
530 .show = fscache_volumes_seq_show,
531};
532#endif /* CONFIG_PROC_FS */