projects/linux·blob master · refs

720 lines · 18.5 KiB

1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Primary bucket allocation code
4 *
5 * Copyright 2012 Google, Inc.
6 *
7 * Allocation in bcache is done in terms of buckets:
8 *
9 * Each bucket has associated an 8 bit gen; this gen corresponds to the gen in
10 * btree pointers - they must match for the pointer to be considered valid.
11 *
12 * Thus (assuming a bucket has no dirty data or metadata in it) we can reuse a
13 * bucket simply by incrementing its gen.
14 *
15 * The gens (along with the priorities; it's really the gens are important but
16 * the code is named as if it's the priorities) are written in an arbitrary list
17 * of buckets on disk, with a pointer to them in the journal header.
18 *
19 * When we invalidate a bucket, we have to write its new gen to disk and wait
20 * for that write to complete before we use it - otherwise after a crash we
21 * could have pointers that appeared to be good but pointed to data that had
22 * been overwritten.
23 *
24 * Since the gens and priorities are all stored contiguously on disk, we can
25 * batch this up: We fill up the free_inc list with freshly invalidated buckets,
26 * call prio_write(), and when prio_write() finishes we pull buckets off the
27 * free_inc list.
28 *
29 * free_inc isn't the only freelist - if it was, we'd often to sleep while
30 * priorities and gens were being written before we could allocate. c->free is a
31 * smaller freelist, and buckets on that list are always ready to be used.
32 *
33 * There is another freelist, because sometimes we have buckets that we know
34 * have nothing pointing into them - these we can reuse without waiting for
35 * priorities to be rewritten. These come from freed btree nodes and buckets
36 * that garbage collection discovered no longer had valid keys pointing into
37 * them (because they were overwritten). That's the unused list - buckets on the
38 * unused list move to the free list.
39 *
40 * It's also important to ensure that gens don't wrap around - with respect to
41 * either the oldest gen in the btree or the gen on disk. This is quite
42 * difficult to do in practice, but we explicitly guard against it anyways - if
43 * a bucket is in danger of wrapping around we simply skip invalidating it that
44 * time around, and we garbage collect or rewrite the priorities sooner than we
45 * would have otherwise.
46 *
47 * bch_bucket_alloc() allocates a single bucket from a specific cache.
48 *
49 * bch_bucket_alloc_set() allocates one bucket from different caches
50 * out of a cache set.
51 *
52 * free_some_buckets() drives all the processes described above. It's called
53 * from bch_bucket_alloc() and a few other places that need to make sure free
54 * buckets are ready.
55 *
56 * invalidate_buckets_(lru|fifo)() find buckets that are available to be
57 * invalidated, and then invalidate them and stick them on the free_inc list -
58 * in either lru or fifo order.
59 */
60
61#include "bcache.h"
62#include "btree.h"
63
64#include <linux/blkdev.h>
65#include <linux/kthread.h>
66#include <linux/random.h>
67#include <trace/events/bcache.h>
68
69#define MAX_OPEN_BUCKETS 128
70
71/* Bucket heap / gen */
72
73uint8_t bch_inc_gen(struct cache *ca, struct bucket *b)
74{
75 uint8_t ret = ++b->gen;
76
77 ca->set->need_gc = max(ca->set->need_gc, bucket_gc_gen(b));
78 WARN_ON_ONCE(ca->set->need_gc > BUCKET_GC_GEN_MAX);
79
80 return ret;
81}
82
83void bch_rescale_priorities(struct cache_set *c, int sectors)
84{
85 struct cache *ca;
86 struct bucket *b;
87 unsigned long next = c->nbuckets * c->cache->sb.bucket_size / 1024;
88 int r;
89
90 atomic_sub(sectors, &c->rescale);
91
92 do {
93 r = atomic_read(&c->rescale);
94
95 if (r >= 0)
96 return;
97 } while (atomic_cmpxchg(&c->rescale, r, r + next) != r);
98
99 mutex_lock(&c->bucket_lock);
100
101 c->min_prio = USHRT_MAX;
102
103 ca = c->cache;
104 for_each_bucket(b, ca)
105 if (b->prio &&
106 b->prio != BTREE_PRIO &&
107 !atomic_read(&b->pin)) {
108 b->prio--;
109 c->min_prio = min(c->min_prio, b->prio);
110 }
111
112 mutex_unlock(&c->bucket_lock);
113}
114
115/*
116 * Background allocation thread: scans for buckets to be invalidated,
117 * invalidates them, rewrites prios/gens (marking them as invalidated on disk),
118 * then puts them on the various freelists.
119 */
120
121static inline bool can_inc_bucket_gen(struct bucket *b)
122{
123 return bucket_gc_gen(b) < BUCKET_GC_GEN_MAX;
124}
125
126bool bch_can_invalidate_bucket(struct cache *ca, struct bucket *b)
127{
128 return (ca->set->gc_mark_valid || b->reclaimable_in_gc) &&
129 ((!GC_MARK(b) || GC_MARK(b) == GC_MARK_RECLAIMABLE) &&
130 !atomic_read(&b->pin) && can_inc_bucket_gen(b));
131}
132
133void __bch_invalidate_one_bucket(struct cache *ca, struct bucket *b)
134{
135 lockdep_assert_held(&ca->set->bucket_lock);
136 BUG_ON(GC_MARK(b) && GC_MARK(b) != GC_MARK_RECLAIMABLE);
137
138 if (GC_SECTORS_USED(b))
139 trace_bcache_invalidate(ca, b - ca->buckets);
140
141 bch_inc_gen(ca, b);
142 b->prio = INITIAL_PRIO;
143 atomic_inc(&b->pin);
144 b->reclaimable_in_gc = 0;
145}
146
147static void bch_invalidate_one_bucket(struct cache *ca, struct bucket *b)
148{
149 __bch_invalidate_one_bucket(ca, b);
150
151 fifo_push(&ca->free_inc, b - ca->buckets);
152}
153
154/*
155 * Determines what order we're going to reuse buckets, smallest bucket_prio()
156 * first: we also take into account the number of sectors of live data in that
157 * bucket, and in order for that multiply to make sense we have to scale bucket
158 *
159 * Thus, we scale the bucket priorities so that the bucket with the smallest
160 * prio is worth 1/8th of what INITIAL_PRIO is worth.
161 */
162
163#define bucket_prio(b) \
164({ \
165 unsigned int min_prio = (INITIAL_PRIO - ca->set->min_prio) / 8; \
166 \
167 (b->prio - ca->set->min_prio + min_prio) * GC_SECTORS_USED(b); \
168})
169
170#define bucket_max_cmp(l, r) (bucket_prio(l) < bucket_prio(r))
171#define bucket_min_cmp(l, r) (bucket_prio(l) > bucket_prio(r))
172
173static void invalidate_buckets_lru(struct cache *ca)
174{
175 struct bucket *b;
176 ssize_t i;
177
178 ca->heap.used = 0;
179
180 for_each_bucket(b, ca) {
181 if (!bch_can_invalidate_bucket(ca, b))
182 continue;
183
184 if (!heap_full(&ca->heap))
185 heap_add(&ca->heap, b, bucket_max_cmp);
186 else if (bucket_max_cmp(b, heap_peek(&ca->heap))) {
187 ca->heap.data[0] = b;
188 heap_sift(&ca->heap, 0, bucket_max_cmp);
189 }
190 }
191
192 for (i = ca->heap.used / 2 - 1; i >= 0; --i)
193 heap_sift(&ca->heap, i, bucket_min_cmp);
194
195 while (!fifo_full(&ca->free_inc)) {
196 if (!heap_pop(&ca->heap, b, bucket_min_cmp)) {
197 /*
198 * We don't want to be calling invalidate_buckets()
199 * multiple times when it can't do anything
200 */
201 ca->invalidate_needs_gc = 1;
202 wake_up_gc(ca->set);
203 return;
204 }
205
206 bch_invalidate_one_bucket(ca, b);
207 }
208}
209
210static void invalidate_buckets_fifo(struct cache *ca)
211{
212 struct bucket *b;
213 size_t checked = 0;
214
215 while (!fifo_full(&ca->free_inc)) {
216 if (ca->fifo_last_bucket < ca->sb.first_bucket ||
217 ca->fifo_last_bucket >= ca->sb.nbuckets)
218 ca->fifo_last_bucket = ca->sb.first_bucket;
219
220 b = ca->buckets + ca->fifo_last_bucket++;
221
222 if (bch_can_invalidate_bucket(ca, b))
223 bch_invalidate_one_bucket(ca, b);
224
225 if (++checked >= ca->sb.nbuckets) {
226 ca->invalidate_needs_gc = 1;
227 wake_up_gc(ca->set);
228 return;
229 }
230 }
231}
232
233static void invalidate_buckets_random(struct cache *ca)
234{
235 struct bucket *b;
236 size_t checked = 0;
237
238 while (!fifo_full(&ca->free_inc)) {
239 size_t n;
240
241 get_random_bytes(&n, sizeof(n));
242
243 n %= (size_t) (ca->sb.nbuckets - ca->sb.first_bucket);
244 n += ca->sb.first_bucket;
245
246 b = ca->buckets + n;
247
248 if (bch_can_invalidate_bucket(ca, b))
249 bch_invalidate_one_bucket(ca, b);
250
251 if (++checked >= ca->sb.nbuckets / 2) {
252 ca->invalidate_needs_gc = 1;
253 wake_up_gc(ca->set);
254 return;
255 }
256 }
257}
258
259static void invalidate_buckets(struct cache *ca)
260{
261 BUG_ON(ca->invalidate_needs_gc);
262
263 switch (CACHE_REPLACEMENT(&ca->sb)) {
264 case CACHE_REPLACEMENT_LRU:
265 invalidate_buckets_lru(ca);
266 break;
267 case CACHE_REPLACEMENT_FIFO:
268 invalidate_buckets_fifo(ca);
269 break;
270 case CACHE_REPLACEMENT_RANDOM:
271 invalidate_buckets_random(ca);
272 break;
273 }
274}
275
276#define allocator_wait(ca, cond) \
277do { \
278 while (1) { \
279 set_current_state(TASK_INTERRUPTIBLE); \
280 if (cond) \
281 break; \
282 \
283 mutex_unlock(&(ca)->set->bucket_lock); \
284 if (kthread_should_stop() || \
285 test_bit(CACHE_SET_IO_DISABLE, &ca->set->flags)) { \
286 set_current_state(TASK_RUNNING); \
287 goto out; \
288 } \
289 \
290 schedule(); \
291 mutex_lock(&(ca)->set->bucket_lock); \
292 } \
293 __set_current_state(TASK_RUNNING); \
294} while (0)
295
296static int bch_allocator_push(struct cache *ca, long bucket)
297{
298 unsigned int i;
299
300 /* Prios/gens are actually the most important reserve */
301 if (fifo_push(&ca->free[RESERVE_PRIO], bucket))
302 return true;
303
304 for (i = 0; i < RESERVE_NR; i++)
305 if (fifo_push(&ca->free[i], bucket))
306 return true;
307
308 return false;
309}
310
311static int bch_allocator_thread(void *arg)
312{
313 struct cache *ca = arg;
314
315 mutex_lock(&ca->set->bucket_lock);
316
317 while (1) {
318 /*
319 * First, we pull buckets off of the unused and free_inc lists,
320 * then we add the bucket to the free list:
321 */
322 while (1) {
323 long bucket;
324
325 if (!fifo_pop(&ca->free_inc, bucket))
326 break;
327
328 allocator_wait(ca, bch_allocator_push(ca, bucket));
329 wake_up(&ca->set->btree_cache_wait);
330 wake_up(&ca->set->bucket_wait);
331 }
332
333 /*
334 * We've run out of free buckets, we need to find some buckets
335 * we can invalidate. First, invalidate them in memory and add
336 * them to the free_inc list:
337 */
338
339retry_invalidate:
340 allocator_wait(ca, !ca->invalidate_needs_gc);
341 invalidate_buckets(ca);
342
343 /*
344 * Now, we write their new gens to disk so we can start writing
345 * new stuff to them:
346 */
347 allocator_wait(ca, !atomic_read(&ca->set->prio_blocked));
348 if (CACHE_SYNC(&ca->sb)) {
349 /*
350 * This could deadlock if an allocation with a btree
351 * node locked ever blocked - having the btree node
352 * locked would block garbage collection, but here we're
353 * waiting on garbage collection before we invalidate
354 * and free anything.
355 *
356 * But this should be safe since the btree code always
357 * uses btree_check_reserve() before allocating now, and
358 * if it fails it blocks without btree nodes locked.
359 */
360 if (!fifo_full(&ca->free_inc))
361 goto retry_invalidate;
362
363 if (bch_prio_write(ca, false) < 0) {
364 ca->invalidate_needs_gc = 1;
365 wake_up_gc(ca->set);
366 }
367 }
368 }
369out:
370 wait_for_kthread_stop();
371 return 0;
372}
373
374/* Allocation */
375
376long bch_bucket_alloc(struct cache *ca, unsigned int reserve, bool wait)
377{
378 DEFINE_WAIT(w);
379 struct bucket *b;
380 long r;
381
382
383 /* No allocation if CACHE_SET_IO_DISABLE bit is set */
384 if (unlikely(test_bit(CACHE_SET_IO_DISABLE, &ca->set->flags)))
385 return -1;
386
387 /* fastpath */
388 if (fifo_pop(&ca->free[RESERVE_NONE], r) ||
389 fifo_pop(&ca->free[reserve], r))
390 goto out;
391
392 if (!wait) {
393 trace_bcache_alloc_fail(ca, reserve);
394 return -1;
395 }
396
397 do {
398 prepare_to_wait(&ca->set->bucket_wait, &w,
399 TASK_UNINTERRUPTIBLE);
400
401 mutex_unlock(&ca->set->bucket_lock);
402
403 atomic_inc(&ca->set->bucket_wait_cnt);
404 schedule();
405 atomic_dec(&ca->set->bucket_wait_cnt);
406
407 mutex_lock(&ca->set->bucket_lock);
408 } while (!fifo_pop(&ca->free[RESERVE_NONE], r) &&
409 !fifo_pop(&ca->free[reserve], r));
410
411 finish_wait(&ca->set->bucket_wait, &w);
412out:
413 if (ca->alloc_thread)
414 wake_up_process(ca->alloc_thread);
415
416 trace_bcache_alloc(ca, reserve);
417
418 if (expensive_debug_checks(ca->set)) {
419 size_t iter;
420 long i;
421 unsigned int j;
422
423 for (iter = 0; iter < prio_buckets(ca) * 2; iter++)
424 BUG_ON(ca->prio_buckets[iter] == (uint64_t) r);
425
426 for (j = 0; j < RESERVE_NR; j++)
427 fifo_for_each(i, &ca->free[j], iter)
428 BUG_ON(i == r);
429 fifo_for_each(i, &ca->free_inc, iter)
430 BUG_ON(i == r);
431 }
432
433 b = ca->buckets + r;
434
435 BUG_ON(atomic_read(&b->pin) != 1);
436
437 SET_GC_SECTORS_USED(b, ca->sb.bucket_size);
438
439 if (reserve <= RESERVE_PRIO) {
440 SET_GC_MARK(b, GC_MARK_METADATA);
441 SET_GC_MOVE(b, 0);
442 b->prio = BTREE_PRIO;
443 } else {
444 SET_GC_MARK(b, GC_MARK_RECLAIMABLE);
445 SET_GC_MOVE(b, 0);
446 b->prio = INITIAL_PRIO;
447 }
448
449 if (ca->set->avail_nbuckets > 0) {
450 ca->set->avail_nbuckets--;
451 bch_update_bucket_in_use(ca->set, &ca->set->gc_stats);
452 }
453
454 return r;
455}
456
457void __bch_bucket_free(struct cache *ca, struct bucket *b)
458{
459 SET_GC_MARK(b, 0);
460 SET_GC_SECTORS_USED(b, 0);
461
462 if (ca->set->avail_nbuckets < ca->set->nbuckets) {
463 ca->set->avail_nbuckets++;
464 bch_update_bucket_in_use(ca->set, &ca->set->gc_stats);
465 }
466}
467
468void bch_bucket_free(struct cache_set *c, struct bkey *k)
469{
470 unsigned int i;
471
472 for (i = 0; i < KEY_PTRS(k); i++)
473 __bch_bucket_free(c->cache, PTR_BUCKET(c, k, i));
474}
475
476int __bch_bucket_alloc_set(struct cache_set *c, unsigned int reserve,
477 struct bkey *k, bool wait)
478{
479 struct cache *ca;
480 long b;
481
482 /* No allocation if CACHE_SET_IO_DISABLE bit is set */
483 if (unlikely(test_bit(CACHE_SET_IO_DISABLE, &c->flags)))
484 return -1;
485
486 lockdep_assert_held(&c->bucket_lock);
487
488 bkey_init(k);
489
490 ca = c->cache;
491 b = bch_bucket_alloc(ca, reserve, wait);
492 if (b < 0)
493 return -1;
494
495 k->ptr[0] = MAKE_PTR(ca->buckets[b].gen,
496 bucket_to_sector(c, b),
497 ca->sb.nr_this_dev);
498
499 SET_KEY_PTRS(k, 1);
500
501 return 0;
502}
503
504int bch_bucket_alloc_set(struct cache_set *c, unsigned int reserve,
505 struct bkey *k, bool wait)
506{
507 int ret;
508
509 mutex_lock(&c->bucket_lock);
510 ret = __bch_bucket_alloc_set(c, reserve, k, wait);
511 mutex_unlock(&c->bucket_lock);
512 return ret;
513}
514
515/* Sector allocator */
516
517struct open_bucket {
518 struct list_head list;
519 unsigned int last_write_point;
520 unsigned int sectors_free;
521 BKEY_PADDED(key);
522};
523
524/*
525 * We keep multiple buckets open for writes, and try to segregate different
526 * write streams for better cache utilization: first we try to segregate flash
527 * only volume write streams from cached devices, secondly we look for a bucket
528 * where the last write to it was sequential with the current write, and
529 * failing that we look for a bucket that was last used by the same task.
530 *
531 * The ideas is if you've got multiple tasks pulling data into the cache at the
532 * same time, you'll get better cache utilization if you try to segregate their
533 * data and preserve locality.
534 *
535 * For example, dirty sectors of flash only volume is not reclaimable, if their
536 * dirty sectors mixed with dirty sectors of cached device, such buckets will
537 * be marked as dirty and won't be reclaimed, though the dirty data of cached
538 * device have been written back to backend device.
539 *
540 * And say you've starting Firefox at the same time you're copying a
541 * bunch of files. Firefox will likely end up being fairly hot and stay in the
542 * cache awhile, but the data you copied might not be; if you wrote all that
543 * data to the same buckets it'd get invalidated at the same time.
544 *
545 * Both of those tasks will be doing fairly random IO so we can't rely on
546 * detecting sequential IO to segregate their data, but going off of the task
547 * should be a sane heuristic.
548 */
549static struct open_bucket *pick_data_bucket(struct cache_set *c,
550 const struct bkey *search,
551 unsigned int write_point,
552 struct bkey *alloc)
553{
554 struct open_bucket *ret, *ret_task = NULL;
555
556 list_for_each_entry_reverse(ret, &c->data_buckets, list)
557 if (UUID_FLASH_ONLY(&c->uuids[KEY_INODE(&ret->key)]) !=
558 UUID_FLASH_ONLY(&c->uuids[KEY_INODE(search)]))
559 continue;
560 else if (!bkey_cmp(&ret->key, search))
561 goto found;
562 else if (ret->last_write_point == write_point)
563 ret_task = ret;
564
565 ret = ret_task ?: list_first_entry(&c->data_buckets,
566 struct open_bucket, list);
567found:
568 if (!ret->sectors_free && KEY_PTRS(alloc)) {
569 ret->sectors_free = c->cache->sb.bucket_size;
570 bkey_copy(&ret->key, alloc);
571 bkey_init(alloc);
572 }
573
574 if (!ret->sectors_free)
575 ret = NULL;
576
577 return ret;
578}
579
580/*
581 * Allocates some space in the cache to write to, and k to point to the newly
582 * allocated space, and updates KEY_SIZE(k) and KEY_OFFSET(k) (to point to the
583 * end of the newly allocated space).
584 *
585 * May allocate fewer sectors than @sectors, KEY_SIZE(k) indicates how many
586 * sectors were actually allocated.
587 *
588 * If s->writeback is true, will not fail.
589 */
590bool bch_alloc_sectors(struct cache_set *c,
591 struct bkey *k,
592 unsigned int sectors,
593 unsigned int write_point,
594 unsigned int write_prio,
595 bool wait)
596{
597 struct open_bucket *b;
598 BKEY_PADDED(key) alloc;
599 unsigned int i;
600
601 /*
602 * We might have to allocate a new bucket, which we can't do with a
603 * spinlock held. So if we have to allocate, we drop the lock, allocate
604 * and then retry. KEY_PTRS() indicates whether alloc points to
605 * allocated bucket(s).
606 */
607
608 bkey_init(&alloc.key);
609 spin_lock(&c->data_bucket_lock);
610
611 while (!(b = pick_data_bucket(c, k, write_point, &alloc.key))) {
612 unsigned int watermark = write_prio
613 ? RESERVE_MOVINGGC
614 : RESERVE_NONE;
615
616 spin_unlock(&c->data_bucket_lock);
617
618 if (bch_bucket_alloc_set(c, watermark, &alloc.key, wait))
619 return false;
620
621 spin_lock(&c->data_bucket_lock);
622 }
623
624 /*
625 * If we had to allocate, we might race and not need to allocate the
626 * second time we call pick_data_bucket(). If we allocated a bucket but
627 * didn't use it, drop the refcount bch_bucket_alloc_set() took:
628 */
629 if (KEY_PTRS(&alloc.key))
630 bkey_put(c, &alloc.key);
631
632 for (i = 0; i < KEY_PTRS(&b->key); i++)
633 EBUG_ON(ptr_stale(c, &b->key, i));
634
635 /* Set up the pointer to the space we're allocating: */
636
637 for (i = 0; i < KEY_PTRS(&b->key); i++)
638 k->ptr[i] = b->key.ptr[i];
639
640 sectors = min(sectors, b->sectors_free);
641
642 SET_KEY_OFFSET(k, KEY_OFFSET(k) + sectors);
643 SET_KEY_SIZE(k, sectors);
644 SET_KEY_PTRS(k, KEY_PTRS(&b->key));
645
646 /*
647 * Move b to the end of the lru, and keep track of what this bucket was
648 * last used for:
649 */
650 list_move_tail(&b->list, &c->data_buckets);
651 bkey_copy_key(&b->key, k);
652 b->last_write_point = write_point;
653
654 b->sectors_free -= sectors;
655
656 for (i = 0; i < KEY_PTRS(&b->key); i++) {
657 SET_PTR_OFFSET(&b->key, i, PTR_OFFSET(&b->key, i) + sectors);
658
659 atomic_long_add(sectors,
660 &c->cache->sectors_written);
661 }
662
663 if (b->sectors_free < c->cache->sb.block_size)
664 b->sectors_free = 0;
665
666 /*
667 * k takes refcounts on the buckets it points to until it's inserted
668 * into the btree, but if we're done with this bucket we just transfer
669 * get_data_bucket()'s refcount.
670 */
671 if (b->sectors_free)
672 for (i = 0; i < KEY_PTRS(&b->key); i++)
673 atomic_inc(&PTR_BUCKET(c, &b->key, i)->pin);
674
675 spin_unlock(&c->data_bucket_lock);
676 return true;
677}
678
679/* Init */
680
681void bch_open_buckets_free(struct cache_set *c)
682{
683 struct open_bucket *b;
684
685 while (!list_empty(&c->data_buckets)) {
686 b = list_first_entry(&c->data_buckets,
687 struct open_bucket, list);
688 list_del(&b->list);
689 kfree(b);
690 }
691}
692
693int bch_open_buckets_alloc(struct cache_set *c)
694{
695 int i;
696
697 spin_lock_init(&c->data_bucket_lock);
698
699 for (i = 0; i < MAX_OPEN_BUCKETS; i++) {
700 struct open_bucket *b = kzalloc_obj(*b);
701
702 if (!b)
703 return -ENOMEM;
704
705 list_add(&b->list, &c->data_buckets);
706 }
707
708 return 0;
709}
710
711int bch_cache_allocator_start(struct cache *ca)
712{
713 struct task_struct *k = kthread_run(bch_allocator_thread,
714 ca, "bcache_allocator");
715 if (IS_ERR(k))
716 return PTR_ERR(k);
717
718 ca->alloc_thread = k;
719 return 0;
720}