projects/linux·blob master · refs

2421 lines · 66.3 KiB

1// SPDX-License-Identifier: GPL-2.0-or-later
2
3#include <linux/blkdev.h>
4#include <linux/module.h>
5#include <linux/errno.h>
6#include <linux/slab.h>
7#include <linux/init.h>
8#include <linux/timer.h>
9#include <linux/sched.h>
10#include <linux/list.h>
11#include <linux/file.h>
12#include <linux/math64.h>
13#include <linux/seq_file.h>
14#include <trace/events/block.h>
15
16#include "md.h"
17#include "md-bitmap.h"
18
19/*
20 * #### Background
21 *
22 * Redundant data is used to enhance data fault tolerance, and the storage
23 * methods for redundant data vary depending on the RAID levels. And it's
24 * important to maintain the consistency of redundant data.
25 *
26 * Bitmap is used to record which data blocks have been synchronized and which
27 * ones need to be resynchronized or recovered. Each bit in the bitmap
28 * represents a segment of data in the array. When a bit is set, it indicates
29 * that the multiple redundant copies of that data segment may not be
30 * consistent. Data synchronization can be performed based on the bitmap after
31 * power failure or readding a disk. If there is no bitmap, a full disk
32 * synchronization is required.
33 *
34 * #### Key Features
35 *
36 * - IO fastpath is lockless, if user issues lots of write IO to the same
37 * bitmap bit in a short time, only the first write has additional overhead
38 * to update bitmap bit, no additional overhead for the following writes;
39 * - support only resync or recover written data, means in the case creating
40 * new array or replacing with a new disk, there is no need to do a full disk
41 * resync/recovery;
42 *
43 * #### Key Concept
44 *
45 * ##### State Machine
46 *
47 * Each bit is one byte, contain 6 different states, see llbitmap_state. And
48 * there are total 8 different actions, see llbitmap_action, can change state:
49 *
50 * llbitmap state machine: transitions between states
51 *
52 * | | Startwrite | Startsync | Endsync | Abortsync|
53 * | --------- | ---------- | --------- | ------- | ------- |
54 * | Unwritten | Dirty | x | x | x |
55 * | Clean | Dirty | x | x | x |
56 * | Dirty | x | x | x | x |
57 * | NeedSync | x | Syncing | x | x |
58 * | Syncing | x | Syncing | Dirty | NeedSync |
59 *
60 * | | Reload | Daemon | Discard | Stale |
61 * | --------- | -------- | ------ | --------- | --------- |
62 * | Unwritten | x | x | x | x |
63 * | Clean | x | x | Unwritten | NeedSync |
64 * | Dirty | NeedSync | Clean | Unwritten | NeedSync |
65 * | NeedSync | x | x | Unwritten | x |
66 * | Syncing | NeedSync | x | Unwritten | NeedSync |
67 *
68 * Typical scenarios:
69 *
70 * 1) Create new array
71 * All bits will be set to Unwritten by default, if --assume-clean is set,
72 * all bits will be set to Clean instead.
73 *
74 * 2) write data, raid1/raid10 have full copy of data, while raid456 doesn't and
75 * rely on xor data
76 *
77 * 2.1) write new data to raid1/raid10:
78 * Unwritten --StartWrite--> Dirty
79 *
80 * 2.2) write new data to raid456:
81 * Unwritten --StartWrite--> NeedSync
82 *
83 * Because the initial recover for raid456 is skipped, the xor data is not built
84 * yet, the bit must be set to NeedSync first and after lazy initial recover is
85 * finished, the bit will finally set to Dirty(see 5.1 and 5.4);
86 *
87 * 2.3) cover write
88 * Clean --StartWrite--> Dirty
89 *
90 * 3) daemon, if the array is not degraded:
91 * Dirty --Daemon--> Clean
92 *
93 * 4) discard
94 * {Clean, Dirty, NeedSync, Syncing} --Discard--> Unwritten
95 *
96 * 5) resync and recover
97 *
98 * 5.1) common process
99 * NeedSync --Startsync--> Syncing --Endsync--> Dirty --Daemon--> Clean
100 *
101 * 5.2) resync after power failure
102 * Dirty --Reload--> NeedSync
103 *
104 * 5.3) recover while replacing with a new disk
105 * By default, the old bitmap framework will recover all data, and llbitmap
106 * implements this by a new helper, see llbitmap_skip_sync_blocks:
107 *
108 * skip recover for bits other than dirty or clean;
109 *
110 * 5.4) lazy initial recover for raid5:
111 * By default, the old bitmap framework will only allow new recover when there
112 * are spares(new disk), a new recovery flag MD_RECOVERY_LAZY_RECOVER is added
113 * to perform raid456 lazy recover for set bits(from 2.2).
114 *
115 * 6. special handling for degraded array:
116 *
117 * - Dirty bits will never be cleared, daemon will just do nothing, so that if
118 * a disk is readded, Clean bits can be skipped with recovery;
119 * - Dirty bits will convert to Syncing from start write, to do data recovery
120 * for new added disks;
121 * - New write will convert bits to NeedSync directly;
122 *
123 * ##### Bitmap IO
124 *
125 * ##### Chunksize
126 *
127 * The default bitmap size is 128k, incluing 1k bitmap super block, and
128 * the default size of segment of data in the array each bit(chunksize) is 64k,
129 * and chunksize will adjust to twice the old size each time if the total number
130 * bits is not less than 127k.(see llbitmap_init)
131 *
132 * ##### READ
133 *
134 * While creating bitmap, all pages will be allocated and read for llbitmap,
135 * there won't be read afterwards
136 *
137 * ##### WRITE
138 *
139 * WRITE IO is divided into logical_block_size of the array, the dirty state
140 * of each block is tracked independently, for example:
141 *
142 * each page is 4k, contain 8 blocks; each block is 512 bytes contain 512 bit;
143 *
144 * | page0 | page1 | ... | page 31 |
145 * | |
146 * | \-----------------------\
147 * | |
148 * | block0 | block1 | ... | block 8|
149 * | |
150 * | \-----------------\
151 * | |
152 * | bit0 | bit1 | ... | bit511 |
153 *
154 * From IO path, if one bit is changed to Dirty or NeedSync, the corresponding
155 * subpage will be marked dirty, such block must write first before the IO is
156 * issued. This behaviour will affect IO performance, to reduce the impact, if
157 * multiple bits are changed in the same block in a short time, all bits in this
158 * block will be changed to Dirty/NeedSync, so that there won't be any overhead
159 * until daemon clears dirty bits.
160 *
161 * ##### Dirty Bits synchronization
162 *
163 * IO fast path will set bits to dirty, and those dirty bits will be cleared
164 * by daemon after IO is done. llbitmap_page_ctl is used to synchronize between
165 * IO path and daemon;
166 *
167 * IO path:
168 * 1) try to grab a reference, if succeed, set expire time after 5s and return;
169 * 2) if failed to grab a reference, wait for daemon to finish clearing dirty
170 * bits;
171 *
172 * Daemon (Daemon will be woken up every daemon_sleep seconds):
173 * For each page:
174 * 1) check if page expired, if not skip this page; for expired page:
175 * 2) suspend the page and wait for inflight write IO to be done;
176 * 3) change dirty page to clean;
177 * 4) resume the page;
178 */
179
180#define BITMAP_DATA_OFFSET 1024
181
182/* 64k is the max IO size of sync IO for raid1/raid10 */
183#define MIN_CHUNK_SIZE (64 * 2)
184
185/* By default, daemon will be woken up every 30s */
186#define DEFAULT_DAEMON_SLEEP 30
187
188/*
189 * Dirtied bits that have not been accessed for more than 5s will be cleared
190 * by daemon.
191 */
192#define DEFAULT_BARRIER_IDLE 5
193
194enum llbitmap_state {
195 /* No valid data, init state after assemble the array */
196 BitUnwritten = 0,
197 /* data is consistent */
198 BitClean,
199 /* data will be consistent after IO is done, set directly for writes */
200 BitDirty,
201 /*
202 * data need to be resynchronized:
203 * 1) set directly for writes if array is degraded, prevent full disk
204 * synchronization after readding a disk;
205 * 2) reassemble the array after power failure, and dirty bits are
206 * found after reloading the bitmap;
207 * 3) set for first write for raid5, to build initial xor data lazily
208 */
209 BitNeedSync,
210 /* data is synchronizing */
211 BitSyncing,
212 /*
213 * Proactive sync requested for unwritten region (raid456 only).
214 * Triggered via sysfs when user wants to pre-build XOR parity
215 * for regions that have never been written.
216 */
217 BitNeedSyncUnwritten,
218 /* Proactive sync in progress for unwritten region */
219 BitSyncingUnwritten,
220 /*
221 * XOR parity has been pre-built for a region that has never had
222 * user data written. When user writes to this region, it transitions
223 * to BitDirty.
224 */
225 BitCleanUnwritten,
226 BitStateCount,
227 BitNone = 0xff,
228};
229
230enum llbitmap_action {
231 /* User write new data, this is the only action from IO fast path */
232 BitmapActionStartwrite = 0,
233 /* Start recovery */
234 BitmapActionStartsync,
235 /* Finish recovery */
236 BitmapActionEndsync,
237 /* Failed recovery */
238 BitmapActionAbortsync,
239 /* Reassemble the array */
240 BitmapActionReload,
241 /* Daemon thread is trying to clear dirty bits */
242 BitmapActionDaemon,
243 /* Data is deleted */
244 BitmapActionDiscard,
245 /*
246 * Bitmap is stale, mark all bits in addition to BitUnwritten to
247 * BitNeedSync.
248 */
249 BitmapActionStale,
250 /*
251 * Proactive sync trigger for raid456 - builds XOR parity for
252 * Unwritten regions without requiring user data write first.
253 */
254 BitmapActionProactiveSync,
255 BitmapActionClearUnwritten,
256 BitmapActionCount,
257 /* Init state is BitUnwritten */
258 BitmapActionInit,
259};
260
261enum llbitmap_page_state {
262 LLPageFlush = 0,
263 LLPageDirty,
264};
265
266struct llbitmap_page_ctl {
267 char *state;
268 struct page *page;
269 unsigned long expire;
270 unsigned long flags;
271 wait_queue_head_t wait;
272 struct percpu_ref active;
273 /* Per block size dirty state, maximum 64k page / 1 sector = 128 */
274 unsigned long dirty[];
275};
276
277struct llbitmap {
278 struct mddev *mddev;
279 struct llbitmap_page_ctl **pctl;
280
281 unsigned int nr_pages;
282 unsigned int io_size;
283 unsigned int blocks_per_page;
284
285 /* shift of one chunk */
286 unsigned long chunkshift;
287 /* size of one chunk in sector */
288 unsigned long chunksize;
289 /* total number of chunks */
290 unsigned long chunks;
291 /* total number of sectors tracked by current bitmap geometry */
292 sector_t sync_size;
293 unsigned long reshape_chunksize;
294 unsigned long reshape_chunks;
295 sector_t reshape_sync_size;
296 unsigned long last_end_sync;
297 /*
298 * time in seconds that dirty bits will be cleared if the page is not
299 * accessed.
300 */
301 unsigned long barrier_idle;
302 /* fires on first BitDirty state */
303 struct timer_list pending_timer;
304 struct work_struct daemon_work;
305 /*
306 * Serialize reshape checkpoint remapping against normal I/O bitmap
307 * updates without blocking concurrent I/O updates on each other.
308 */
309 rwlock_t reshape_lock;
310
311 unsigned long flags;
312 __u64 events_cleared;
313
314 /* for slow disks */
315 atomic_t behind_writes;
316 wait_queue_head_t behind_wait;
317};
318
319struct llbitmap_unplug_work {
320 struct work_struct work;
321 struct llbitmap *llbitmap;
322 struct completion *done;
323};
324
325static struct workqueue_struct *md_llbitmap_io_wq;
326static struct workqueue_struct *md_llbitmap_unplug_wq;
327
328static char state_machine[BitStateCount][BitmapActionCount] = {
329 [BitUnwritten] = {
330 [BitmapActionStartwrite] = BitDirty,
331 [BitmapActionStartsync] = BitNone,
332 [BitmapActionEndsync] = BitNone,
333 [BitmapActionAbortsync] = BitNone,
334 [BitmapActionReload] = BitNone,
335 [BitmapActionDaemon] = BitNone,
336 [BitmapActionDiscard] = BitNone,
337 [BitmapActionStale] = BitNone,
338 [BitmapActionProactiveSync] = BitNeedSyncUnwritten,
339 [BitmapActionClearUnwritten] = BitNone,
340 },
341 [BitClean] = {
342 [BitmapActionStartwrite] = BitDirty,
343 [BitmapActionStartsync] = BitNone,
344 [BitmapActionEndsync] = BitNone,
345 [BitmapActionAbortsync] = BitNone,
346 [BitmapActionReload] = BitNone,
347 [BitmapActionDaemon] = BitNone,
348 [BitmapActionDiscard] = BitUnwritten,
349 [BitmapActionStale] = BitNeedSync,
350 [BitmapActionProactiveSync] = BitNone,
351 [BitmapActionClearUnwritten] = BitNone,
352 },
353 [BitDirty] = {
354 [BitmapActionStartwrite] = BitNone,
355 [BitmapActionStartsync] = BitNone,
356 [BitmapActionEndsync] = BitNone,
357 [BitmapActionAbortsync] = BitNone,
358 [BitmapActionReload] = BitNeedSync,
359 [BitmapActionDaemon] = BitClean,
360 [BitmapActionDiscard] = BitUnwritten,
361 [BitmapActionStale] = BitNeedSync,
362 [BitmapActionProactiveSync] = BitNone,
363 [BitmapActionClearUnwritten] = BitNone,
364 },
365 [BitNeedSync] = {
366 [BitmapActionStartwrite] = BitNone,
367 [BitmapActionStartsync] = BitSyncing,
368 [BitmapActionEndsync] = BitNone,
369 [BitmapActionAbortsync] = BitNone,
370 [BitmapActionReload] = BitNone,
371 [BitmapActionDaemon] = BitNone,
372 [BitmapActionDiscard] = BitUnwritten,
373 [BitmapActionStale] = BitNone,
374 [BitmapActionProactiveSync] = BitNone,
375 [BitmapActionClearUnwritten] = BitNone,
376 },
377 [BitSyncing] = {
378 [BitmapActionStartwrite] = BitNone,
379 [BitmapActionStartsync] = BitSyncing,
380 [BitmapActionEndsync] = BitDirty,
381 [BitmapActionAbortsync] = BitNeedSync,
382 [BitmapActionReload] = BitNeedSync,
383 [BitmapActionDaemon] = BitNone,
384 [BitmapActionDiscard] = BitUnwritten,
385 [BitmapActionStale] = BitNeedSync,
386 [BitmapActionProactiveSync] = BitNone,
387 [BitmapActionClearUnwritten] = BitNone,
388 },
389 [BitNeedSyncUnwritten] = {
390 [BitmapActionStartwrite] = BitNeedSync,
391 [BitmapActionStartsync] = BitSyncingUnwritten,
392 [BitmapActionEndsync] = BitNone,
393 [BitmapActionAbortsync] = BitUnwritten,
394 [BitmapActionReload] = BitUnwritten,
395 [BitmapActionDaemon] = BitNone,
396 [BitmapActionDiscard] = BitUnwritten,
397 [BitmapActionStale] = BitUnwritten,
398 [BitmapActionProactiveSync] = BitNone,
399 [BitmapActionClearUnwritten] = BitUnwritten,
400 },
401 [BitSyncingUnwritten] = {
402 [BitmapActionStartwrite] = BitSyncing,
403 [BitmapActionStartsync] = BitSyncingUnwritten,
404 [BitmapActionEndsync] = BitCleanUnwritten,
405 [BitmapActionAbortsync] = BitUnwritten,
406 [BitmapActionReload] = BitUnwritten,
407 [BitmapActionDaemon] = BitNone,
408 [BitmapActionDiscard] = BitUnwritten,
409 [BitmapActionStale] = BitUnwritten,
410 [BitmapActionProactiveSync] = BitNone,
411 [BitmapActionClearUnwritten] = BitUnwritten,
412 },
413 [BitCleanUnwritten] = {
414 [BitmapActionStartwrite] = BitDirty,
415 [BitmapActionStartsync] = BitNone,
416 [BitmapActionEndsync] = BitNone,
417 [BitmapActionAbortsync] = BitNone,
418 [BitmapActionReload] = BitNone,
419 [BitmapActionDaemon] = BitNone,
420 [BitmapActionDiscard] = BitUnwritten,
421 [BitmapActionStale] = BitUnwritten,
422 [BitmapActionProactiveSync] = BitNone,
423 [BitmapActionClearUnwritten] = BitUnwritten,
424 },
425};
426
427static void __llbitmap_flush(struct mddev *mddev);
428static void llbitmap_flush(struct mddev *mddev);
429static void llbitmap_update_sb(void *data);
430
431static void llbitmap_calculate_chunks(struct mddev *mddev, sector_t blocks,
432 unsigned long *chunksize,
433 unsigned long *chunks)
434{
435 *chunks = DIV_ROUND_UP_SECTOR_T(blocks, *chunksize);
436 while (*chunks > mddev->bitmap_info.space << SECTOR_SHIFT) {
437 *chunksize = *chunksize << 1;
438 *chunks = DIV_ROUND_UP_SECTOR_T(blocks, *chunksize);
439 }
440}
441
442static sector_t llbitmap_personality_sync_size(struct llbitmap *llbitmap,
443 bool previous)
444{
445 struct mddev *mddev = llbitmap->mddev;
446
447 if (READ_ONCE(mddev->reshape_position) == MaxSector ||
448 !mddev->private || !mddev->pers ||
449 !mddev->pers->bitmap_sync_size)
450 return llbitmap->sync_size;
451 return mddev->pers->bitmap_sync_size(mddev, previous);
452}
453
454static sector_t llbitmap_logical_size(struct llbitmap *llbitmap, bool previous)
455{
456 struct mddev *mddev = llbitmap->mddev;
457
458 if (!mddev->private || !mddev->pers ||
459 !mddev->pers->bitmap_array_sectors)
460 return llbitmap_personality_sync_size(llbitmap, previous);
461 return mddev->pers->bitmap_array_sectors(mddev, previous);
462}
463
464static void llbitmap_refresh_reshape(struct llbitmap *llbitmap)
465{
466 unsigned long old_chunks = DIV_ROUND_UP_SECTOR_T(llbitmap->sync_size,
467 llbitmap->chunksize);
468 sector_t blocks = llbitmap_personality_sync_size(llbitmap, false);
469 unsigned long chunksize = llbitmap->chunksize;
470 unsigned long chunks = DIV_ROUND_UP_SECTOR_T(blocks, chunksize);
471
472 llbitmap->reshape_sync_size = blocks;
473 llbitmap->reshape_chunksize = chunksize;
474 llbitmap->reshape_chunks = chunks;
475 llbitmap_calculate_chunks(llbitmap->mddev, blocks,
476 &llbitmap->reshape_chunksize,
477 &llbitmap->reshape_chunks);
478 llbitmap->chunks = max(old_chunks, llbitmap->reshape_chunks);
479}
480
481static void llbitmap_map_layout(struct llbitmap *llbitmap, sector_t *offset,
482 unsigned long *sectors, bool previous)
483{
484 sector_t limit = llbitmap_logical_size(llbitmap, previous);
485 sector_t start = *offset;
486 sector_t end = start + *sectors;
487
488 if (start >= limit) {
489 *sectors = 0;
490 return;
491 }
492 if (end > limit)
493 end = limit;
494
495 *offset = start;
496 *sectors = end - start;
497 if (!*sectors)
498 return;
499
500 if (llbitmap->mddev->pers->bitmap_sector_map)
501 llbitmap->mddev->pers->bitmap_sector_map(llbitmap->mddev, offset,
502 sectors, previous);
503 else if (!previous && llbitmap->mddev->pers->bitmap_sector)
504 llbitmap->mddev->pers->bitmap_sector(llbitmap->mddev, offset,
505 sectors);
506
507 limit = llbitmap_personality_sync_size(llbitmap, previous);
508 start = *offset;
509 end = start + *sectors;
510 if (start >= limit)
511 *sectors = 0;
512 else if (end > limit)
513 *sectors = limit - start;
514}
515
516static void llbitmap_encode_range(struct llbitmap *llbitmap, sector_t *offset,
517 unsigned long *sectors, bool previous)
518{
519 unsigned long chunksize = previous ? llbitmap->chunksize :
520 llbitmap->reshape_chunksize;
521 u64 start;
522 u64 end;
523
524 if (!*sectors) {
525 *offset = 0;
526 return;
527 }
528
529 start = div64_u64(*offset, chunksize);
530 end = div64_u64(*offset + *sectors - 1, chunksize);
531 *offset = (sector_t)start << llbitmap->chunkshift;
532 *sectors = (end - start + 1) << llbitmap->chunkshift;
533}
534
535static void llbitmap_encode_discard_range(struct llbitmap *llbitmap,
536 sector_t *offset,
537 unsigned long *sectors,
538 bool previous)
539{
540 unsigned long chunksize = previous ? llbitmap->chunksize :
541 llbitmap->reshape_chunksize;
542 sector_t end = *offset + *sectors;
543 u64 start;
544 u64 last;
545
546 if (!*sectors) {
547 *offset = 0;
548 return;
549 }
550
551 start = DIV_ROUND_UP_SECTOR_T(*offset, chunksize);
552 last = div64_u64(end, chunksize);
553 if (start >= last) {
554 *offset = 0;
555 *sectors = 0;
556 return;
557 }
558
559 *offset = (sector_t)start << llbitmap->chunkshift;
560 *sectors = (last - start) << llbitmap->chunkshift;
561}
562
563static enum llbitmap_state llbitmap_read(struct llbitmap *llbitmap, loff_t pos)
564{
565 unsigned int idx;
566 unsigned int offset;
567
568 pos += BITMAP_DATA_OFFSET;
569 idx = pos >> PAGE_SHIFT;
570 offset = offset_in_page(pos);
571
572 return llbitmap->pctl[idx]->state[offset];
573}
574
575/* set all the bits in the subpage as dirty */
576static void llbitmap_infect_dirty_bits(struct llbitmap *llbitmap,
577 struct llbitmap_page_ctl *pctl,
578 unsigned int block)
579{
580 bool level_456 = raid_is_456(llbitmap->mddev);
581 unsigned int io_size = llbitmap->io_size;
582 int pos;
583
584 for (pos = block * io_size; pos < (block + 1) * io_size; pos++) {
585 switch (pctl->state[pos]) {
586 case BitUnwritten:
587 pctl->state[pos] = level_456 ? BitNeedSync : BitDirty;
588 break;
589 case BitClean:
590 case BitCleanUnwritten:
591 pctl->state[pos] = BitDirty;
592 break;
593 }
594 }
595}
596
597static void llbitmap_set_page_dirty(struct llbitmap *llbitmap, int idx,
598 int offset, bool infect)
599{
600 struct llbitmap_page_ctl *pctl = llbitmap->pctl[idx];
601 unsigned int io_size = llbitmap->io_size;
602 int block = offset / io_size;
603 int pos;
604
605 if (!test_bit(LLPageDirty, &pctl->flags))
606 set_bit(LLPageDirty, &pctl->flags);
607
608 /*
609 * For degraded array, dirty bits will never be cleared, and we must
610 * resync all the dirty bits, hence skip infect new dirty bits to
611 * prevent resync unnecessary data.
612 */
613 if (llbitmap->mddev->degraded || !infect) {
614 set_bit(block, pctl->dirty);
615 return;
616 }
617
618 /*
619 * The subpage usually contains a total of 512 bits. If any single bit
620 * within the subpage is marked as dirty, the entire sector will be
621 * written. To avoid impacting write performance, when multiple bits
622 * within the same sector are modified within llbitmap->barrier_idle,
623 * all bits in the sector will be collectively marked as dirty at once.
624 */
625 if (test_and_set_bit(block, pctl->dirty)) {
626 llbitmap_infect_dirty_bits(llbitmap, pctl, block);
627 return;
628 }
629
630 for (pos = block * io_size; pos < (block + 1) * io_size; pos++) {
631 if (pos == offset)
632 continue;
633 if (pctl->state[pos] == BitDirty ||
634 pctl->state[pos] == BitNeedSync) {
635 llbitmap_infect_dirty_bits(llbitmap, pctl, block);
636 return;
637 }
638 }
639}
640
641static void llbitmap_write(struct llbitmap *llbitmap, enum llbitmap_state state,
642 loff_t pos)
643{
644 unsigned int idx;
645 unsigned int bit;
646
647 pos += BITMAP_DATA_OFFSET;
648 idx = pos >> PAGE_SHIFT;
649 bit = offset_in_page(pos);
650
651 llbitmap->pctl[idx]->state[bit] = state;
652 if (state == BitDirty || state == BitNeedSync)
653 llbitmap_set_page_dirty(llbitmap, idx, bit, true);
654 else if (state == BitNeedSyncUnwritten)
655 llbitmap_set_page_dirty(llbitmap, idx, bit, false);
656}
657
658static unsigned int llbitmap_used_pages(struct llbitmap *llbitmap,
659 unsigned long chunks)
660{
661 return DIV_ROUND_UP(chunks + BITMAP_DATA_OFFSET, PAGE_SIZE);
662}
663
664static struct page *llbitmap_read_page(struct llbitmap *llbitmap, int idx)
665{
666 struct mddev *mddev = llbitmap->mddev;
667 struct page *page = NULL;
668 struct md_rdev *rdev;
669
670 if (llbitmap->pctl && idx < llbitmap->nr_pages && llbitmap->pctl[idx])
671 page = llbitmap->pctl[idx]->page;
672 if (page)
673 return page;
674
675 page = alloc_page(GFP_NOIO | __GFP_ZERO);
676 if (!page)
677 return ERR_PTR(-ENOMEM);
678 if (idx >= llbitmap_used_pages(llbitmap, llbitmap->chunks))
679 return page;
680
681 rdev_for_each(rdev, mddev) {
682 sector_t sector;
683
684 if (rdev->raid_disk < 0 || test_bit(Faulty, &rdev->flags) ||
685 !test_bit(In_sync, &rdev->flags))
686 continue;
687
688 sector = mddev->bitmap_info.offset +
689 (idx << PAGE_SECTORS_SHIFT);
690
691 if (sync_page_io(rdev, sector, PAGE_SIZE, page, REQ_OP_READ,
692 true))
693 return page;
694
695 md_error(mddev, rdev);
696 }
697
698 __free_page(page);
699 return ERR_PTR(-EIO);
700}
701
702static void llbitmap_write_page(struct llbitmap *llbitmap, int idx)
703{
704 struct page *page = llbitmap->pctl[idx]->page;
705 struct mddev *mddev = llbitmap->mddev;
706 struct md_rdev *rdev;
707 int block;
708
709 for (block = 0; block < llbitmap->blocks_per_page; block++) {
710 struct llbitmap_page_ctl *pctl = llbitmap->pctl[idx];
711
712 if (!test_and_clear_bit(block, pctl->dirty))
713 continue;
714
715 rdev_for_each(rdev, mddev) {
716 sector_t sector;
717 sector_t bit_sector = llbitmap->io_size >> SECTOR_SHIFT;
718
719 if (rdev->raid_disk < 0 || test_bit(Faulty, &rdev->flags))
720 continue;
721
722 sector = mddev->bitmap_info.offset + rdev->sb_start +
723 (idx << PAGE_SECTORS_SHIFT) +
724 block * bit_sector;
725 md_write_metadata(mddev, rdev, sector,
726 llbitmap->io_size, page,
727 block * llbitmap->io_size);
728 }
729 }
730}
731
732static void active_release(struct percpu_ref *ref)
733{
734 struct llbitmap_page_ctl *pctl =
735 container_of(ref, struct llbitmap_page_ctl, active);
736
737 wake_up(&pctl->wait);
738}
739
740static void llbitmap_free_pages(struct llbitmap *llbitmap)
741{
742 int i;
743
744 if (!llbitmap->pctl)
745 return;
746
747 for (i = 0; i < llbitmap->nr_pages; i++) {
748 struct llbitmap_page_ctl *pctl = llbitmap->pctl[i];
749
750 if (!pctl)
751 continue;
752 if (pctl->page)
753 __free_page(pctl->page);
754 percpu_ref_exit(&pctl->active);
755 kfree(pctl);
756 }
757
758 kfree(llbitmap->pctl);
759 llbitmap->pctl = NULL;
760}
761
762static struct llbitmap_page_ctl *
763llbitmap_alloc_page_ctl(struct llbitmap *llbitmap, int idx)
764{
765 struct llbitmap_page_ctl *pctl;
766 struct page *page;
767 unsigned int size = struct_size(pctl, dirty, BITS_TO_LONGS(
768 llbitmap->blocks_per_page));
769
770 size = round_up(size, cache_line_size());
771 pctl = kzalloc(size, GFP_NOIO);
772 if (!pctl)
773 return ERR_PTR(-ENOMEM);
774
775 page = llbitmap_read_page(llbitmap, idx);
776
777 if (IS_ERR(page)) {
778 kfree(pctl);
779 return ERR_CAST(page);
780 }
781
782 if (percpu_ref_init(&pctl->active, active_release,
783 PERCPU_REF_ALLOW_REINIT, GFP_NOIO)) {
784 __free_page(page);
785 kfree(pctl);
786 return ERR_PTR(-ENOMEM);
787 }
788
789 pctl->page = page;
790 pctl->state = page_address(page);
791 init_waitqueue_head(&pctl->wait);
792 return pctl;
793}
794
795static unsigned int llbitmap_reserved_pages(struct llbitmap *llbitmap)
796{
797 return DIV_ROUND_UP(llbitmap->mddev->bitmap_info.space << SECTOR_SHIFT,
798 PAGE_SIZE);
799}
800
801static int llbitmap_expand_pages(struct llbitmap *llbitmap,
802 unsigned long chunks)
803{
804 struct llbitmap_page_ctl **pctl;
805 unsigned int old_nr_pages = llbitmap->nr_pages;
806 unsigned int nr_pages = llbitmap_used_pages(llbitmap, chunks);
807 unsigned int i;
808 int ret;
809
810 if (nr_pages <= old_nr_pages)
811 return 0;
812
813 pctl = kzalloc_objs(*pctl, nr_pages, GFP_NOIO);
814 if (!pctl)
815 return -ENOMEM;
816
817 if (llbitmap->pctl)
818 memcpy(pctl, llbitmap->pctl,
819 array_size(old_nr_pages, sizeof(*pctl)));
820
821 for (i = old_nr_pages; i < nr_pages; i++) {
822 pctl[i] = llbitmap_alloc_page_ctl(llbitmap, i);
823 if (IS_ERR(pctl[i]))
824 goto err_alloc_ptr;
825 }
826
827 kfree(llbitmap->pctl);
828 llbitmap->pctl = pctl;
829 llbitmap->nr_pages = nr_pages;
830 return 0;
831
832err_alloc_ptr:
833 ret = PTR_ERR(pctl[i]);
834 while (i-- > old_nr_pages) {
835 __free_page(pctl[i]->page);
836 percpu_ref_exit(&pctl[i]->active);
837 kfree(pctl[i]);
838 }
839 kfree(pctl);
840 return ret;
841}
842
843static int llbitmap_alloc_pages(struct llbitmap *llbitmap)
844{
845 unsigned int used_pages = llbitmap_used_pages(llbitmap, llbitmap->chunks);
846 unsigned int nr_pages = max(used_pages, llbitmap_reserved_pages(llbitmap));
847 int i;
848
849 llbitmap->pctl = kzalloc_objs(*llbitmap->pctl, nr_pages, GFP_NOIO);
850 if (!llbitmap->pctl)
851 return -ENOMEM;
852
853 llbitmap->nr_pages = nr_pages;
854
855 for (i = 0; i < nr_pages; i++) {
856 llbitmap->pctl[i] = llbitmap_alloc_page_ctl(llbitmap, i);
857 if (IS_ERR(llbitmap->pctl[i])) {
858 int ret = PTR_ERR(llbitmap->pctl[i]);
859
860 llbitmap->pctl[i] = NULL;
861 llbitmap_free_pages(llbitmap);
862 return ret;
863 }
864 }
865
866 return 0;
867}
868
869/*
870 * Check if all underlying disks support write_zeroes with unmap.
871 */
872static bool llbitmap_all_disks_support_wzeroes_unmap(struct llbitmap *llbitmap)
873{
874 struct mddev *mddev = llbitmap->mddev;
875 struct md_rdev *rdev;
876
877 rdev_for_each(rdev, mddev) {
878 if (rdev->raid_disk < 0 || test_bit(Faulty, &rdev->flags))
879 continue;
880
881 if (bdev_write_zeroes_unmap_sectors(rdev->bdev) == 0)
882 return false;
883 }
884
885 return true;
886}
887
888/*
889 * Issue write_zeroes to all underlying disks to zero their data regions.
890 * This ensures parity consistency for RAID-456 (0 XOR 0 = 0).
891 * Returns true if all disks were successfully zeroed.
892 */
893static bool llbitmap_zero_all_disks(struct llbitmap *llbitmap)
894{
895 struct mddev *mddev = llbitmap->mddev;
896 struct md_rdev *rdev;
897 sector_t dev_sectors = mddev->dev_sectors;
898 int ret;
899
900 rdev_for_each(rdev, mddev) {
901 if (rdev->raid_disk < 0 || test_bit(Faulty, &rdev->flags))
902 continue;
903
904 ret = blkdev_issue_zeroout(rdev->bdev,
905 rdev->data_offset,
906 dev_sectors,
907 GFP_KERNEL, 0);
908 if (ret) {
909 pr_warn("md/llbitmap: failed to zero disk %pg: %d\n",
910 rdev->bdev, ret);
911 return false;
912 }
913 }
914
915 return true;
916}
917
918static void llbitmap_mark_range(struct llbitmap *llbitmap,
919 unsigned long start,
920 unsigned long end,
921 enum llbitmap_state state)
922{
923 while (start <= end) {
924 llbitmap_write(llbitmap, state, start);
925 start++;
926 }
927}
928
929static int llbitmap_prepare_resize(struct llbitmap *llbitmap,
930 unsigned long old_chunks,
931 unsigned long new_chunks,
932 unsigned long cache_chunks)
933{
934 int ret;
935
936 llbitmap_flush(llbitmap->mddev);
937 ret = llbitmap_expand_pages(llbitmap, cache_chunks);
938 if (ret)
939 return ret;
940 if (new_chunks > old_chunks)
941 llbitmap_mark_range(llbitmap, old_chunks, new_chunks - 1,
942 BitUnwritten);
943 return 0;
944}
945
946static enum llbitmap_state
947llbitmap_rmerge_state(struct llbitmap *llbitmap,
948 enum llbitmap_state dst,
949 enum llbitmap_state src)
950{
951 bool level_456 = raid_is_456(llbitmap->mddev);
952
953 if (dst == BitNeedSync || dst == BitSyncing ||
954 src == BitNeedSync || src == BitSyncing)
955 return BitNeedSync;
956
957 if (dst == BitDirty || src == BitDirty)
958 return BitDirty;
959
960 /*
961 * Reshape generates valid target parity/data for both already-written
962 * and not-yet-written regions in the checkpointed range, so a mix of
963 * clean and unwritten still results in a clean destination bit.
964 */
965 if (level_456 && ((dst == BitClean && src == BitUnwritten) ||
966 (src == BitClean && dst == BitUnwritten)))
967 return BitClean;
968 if (dst == BitClean || src == BitClean)
969 return BitClean;
970 return BitUnwritten;
971}
972
973static void llbitmap_init_state(struct llbitmap *llbitmap)
974{
975 struct mddev *mddev = llbitmap->mddev;
976 enum llbitmap_state state = BitUnwritten;
977 unsigned long i;
978
979 if (test_and_clear_bit(BITMAP_CLEAN, &llbitmap->flags)) {
980 state = BitClean;
981 } else if (raid_is_456(mddev) &&
982 llbitmap_all_disks_support_wzeroes_unmap(llbitmap)) {
983 /*
984 * All disks support write_zeroes with unmap. Zero all disks
985 * to ensure parity consistency, then set BitCleanUnwritten
986 * to skip initial sync.
987 */
988 if (llbitmap_zero_all_disks(llbitmap))
989 state = BitCleanUnwritten;
990 }
991
992 for (i = 0; i < llbitmap->chunks; i++)
993 llbitmap_write(llbitmap, state, i);
994}
995
996/* The return value is only used from resync, where @start == @end. */
997static enum llbitmap_state llbitmap_state_machine(struct llbitmap *llbitmap,
998 unsigned long start,
999 unsigned long end,
1000 enum llbitmap_action action)
1001{
1002 struct mddev *mddev = llbitmap->mddev;
1003 enum llbitmap_state state = BitNone;
1004 bool level_456 = raid_is_456(llbitmap->mddev);
1005 bool need_resync = false;
1006 bool need_recovery = false;
1007
1008 if (test_bit(BITMAP_WRITE_ERROR, &llbitmap->flags))
1009 return BitNone;
1010
1011 if (action == BitmapActionInit) {
1012 llbitmap_init_state(llbitmap);
1013 return BitNone;
1014 }
1015 if (start >= llbitmap->chunks)
1016 return BitNone;
1017 if (end >= llbitmap->chunks)
1018 end = llbitmap->chunks - 1;
1019 while (start <= end) {
1020 enum llbitmap_state c = llbitmap_read(llbitmap, start);
1021
1022 if (c < 0 || c >= BitStateCount) {
1023 pr_err("%s: invalid bit %lu state %d action %d, forcing resync\n",
1024 __func__, start, c, action);
1025 state = BitNeedSync;
1026 goto write_bitmap;
1027 }
1028
1029 if (c == BitNeedSync || c == BitNeedSyncUnwritten)
1030 need_resync = !mddev->degraded;
1031
1032 state = state_machine[c][action];
1033write_bitmap:
1034 if (unlikely(mddev->degraded)) {
1035 /* For degraded array, mark new data as need sync. */
1036 if (state == BitDirty &&
1037 action == BitmapActionStartwrite)
1038 state = BitNeedSync;
1039 /*
1040 * For degraded array, resync dirty data as well, noted
1041 * if array is still degraded after resync is done, all
1042 * new data will still be dirty until array is clean.
1043 */
1044 else if (c == BitDirty &&
1045 action == BitmapActionStartsync)
1046 state = BitSyncing;
1047 } else if (c == BitUnwritten && state == BitDirty &&
1048 action == BitmapActionStartwrite && level_456) {
1049 /* Delay raid456 initial recovery to first write. */
1050 state = BitNeedSync;
1051 }
1052
1053 if (state == BitNone) {
1054 start++;
1055 continue;
1056 }
1057
1058 llbitmap_write(llbitmap, state, start);
1059 if (state == BitNeedSync || state == BitNeedSyncUnwritten)
1060 need_resync = !mddev->degraded;
1061 else if (state == BitDirty &&
1062 !test_bit(BITMAP_SHUTDOWN, &llbitmap->flags) &&
1063 !timer_pending(&llbitmap->pending_timer))
1064 mod_timer(&llbitmap->pending_timer,
1065 jiffies + mddev->bitmap_info.daemon_sleep * HZ);
1066
1067 start++;
1068 }
1069
1070 if (need_resync && level_456)
1071 need_recovery = true;
1072
1073 if (need_recovery) {
1074 set_bit(MD_RECOVERY_NEEDED, &mddev->recovery);
1075 set_bit(MD_RECOVERY_LAZY_RECOVER, &mddev->recovery);
1076 md_wakeup_thread(mddev->thread);
1077 } else if (need_resync) {
1078 set_bit(MD_RECOVERY_NEEDED, &mddev->recovery);
1079 set_bit(MD_RECOVERY_SYNC, &mddev->recovery);
1080 md_wakeup_thread(mddev->thread);
1081 }
1082
1083 return state;
1084}
1085
1086static void llbitmap_raise_barrier(struct llbitmap *llbitmap, int page_idx)
1087{
1088 struct llbitmap_page_ctl *pctl = llbitmap->pctl[page_idx];
1089
1090retry:
1091 if (likely(percpu_ref_tryget_live(&pctl->active))) {
1092 WRITE_ONCE(pctl->expire, jiffies + llbitmap->barrier_idle * HZ);
1093 return;
1094 }
1095
1096 wait_event(pctl->wait, !percpu_ref_is_dying(&pctl->active));
1097 goto retry;
1098}
1099
1100static void llbitmap_release_barrier(struct llbitmap *llbitmap, int page_idx)
1101{
1102 struct llbitmap_page_ctl *pctl = llbitmap->pctl[page_idx];
1103
1104 percpu_ref_put(&pctl->active);
1105}
1106
1107static int llbitmap_suspend_timeout(struct llbitmap *llbitmap, int page_idx)
1108{
1109 struct llbitmap_page_ctl *pctl = llbitmap->pctl[page_idx];
1110
1111 percpu_ref_kill(&pctl->active);
1112
1113 if (!wait_event_timeout(pctl->wait, percpu_ref_is_zero(&pctl->active),
1114 llbitmap->mddev->bitmap_info.daemon_sleep * HZ)) {
1115 percpu_ref_resurrect(&pctl->active);
1116 return -ETIMEDOUT;
1117 }
1118
1119 return 0;
1120}
1121
1122static void llbitmap_resume(struct llbitmap *llbitmap, int page_idx)
1123{
1124 struct llbitmap_page_ctl *pctl = llbitmap->pctl[page_idx];
1125
1126 pctl->expire = LONG_MAX;
1127 percpu_ref_resurrect(&pctl->active);
1128 wake_up(&pctl->wait);
1129}
1130
1131static int llbitmap_check_support(struct mddev *mddev)
1132{
1133 if (test_bit(MD_HAS_JOURNAL, &mddev->flags)) {
1134 pr_notice("md/llbitmap: %s: array with journal cannot have bitmap\n",
1135 mdname(mddev));
1136 return -EBUSY;
1137 }
1138
1139 if (mddev->bitmap_info.space == 0) {
1140 if (mddev->bitmap_info.default_space == 0) {
1141 pr_notice("md/llbitmap: %s: no space for bitmap\n",
1142 mdname(mddev));
1143 return -ENOSPC;
1144 }
1145 }
1146
1147 if (!mddev->persistent) {
1148 pr_notice("md/llbitmap: %s: array must be persistent\n",
1149 mdname(mddev));
1150 return -EOPNOTSUPP;
1151 }
1152
1153 if (mddev->bitmap_info.file) {
1154 pr_notice("md/llbitmap: %s: doesn't support bitmap file\n",
1155 mdname(mddev));
1156 return -EOPNOTSUPP;
1157 }
1158
1159 if (mddev->bitmap_info.external) {
1160 pr_notice("md/llbitmap: %s: doesn't support external metadata\n",
1161 mdname(mddev));
1162 return -EOPNOTSUPP;
1163 }
1164
1165 if (mddev_is_dm(mddev)) {
1166 pr_notice("md/llbitmap: %s: doesn't support dm-raid\n",
1167 mdname(mddev));
1168 return -EOPNOTSUPP;
1169 }
1170
1171 return 0;
1172}
1173
1174static int llbitmap_init(struct llbitmap *llbitmap)
1175{
1176 struct mddev *mddev = llbitmap->mddev;
1177 sector_t blocks = mddev->resync_max_sectors;
1178 unsigned long chunksize = MIN_CHUNK_SIZE;
1179 unsigned long chunks = DIV_ROUND_UP(blocks, chunksize);
1180 unsigned long space = mddev->bitmap_info.space << SECTOR_SHIFT;
1181 int ret;
1182
1183 while (chunks > space) {
1184 chunksize = chunksize << 1;
1185 chunks = DIV_ROUND_UP_SECTOR_T(blocks, chunksize);
1186 }
1187
1188 llbitmap->barrier_idle = DEFAULT_BARRIER_IDLE;
1189 llbitmap->chunkshift = ffz(~chunksize);
1190 llbitmap->chunksize = chunksize;
1191 llbitmap->chunks = chunks;
1192 llbitmap->sync_size = blocks;
1193 llbitmap_refresh_reshape(llbitmap);
1194 mddev->bitmap_info.daemon_sleep = DEFAULT_DAEMON_SLEEP;
1195
1196 ret = llbitmap_alloc_pages(llbitmap);
1197 if (ret)
1198 return ret;
1199
1200 llbitmap_state_machine(llbitmap, 0, llbitmap->chunks - 1,
1201 BitmapActionInit);
1202 /* flush initial llbitmap to disk */
1203 __llbitmap_flush(mddev);
1204
1205 return 0;
1206}
1207
1208static int llbitmap_read_sb(struct llbitmap *llbitmap)
1209{
1210 struct mddev *mddev = llbitmap->mddev;
1211 unsigned long daemon_sleep;
1212 unsigned long chunksize;
1213 unsigned long events;
1214 sector_t sync_size;
1215 struct page *sb_page;
1216 bitmap_super_t *sb;
1217 int ret = -EINVAL;
1218
1219 if (!mddev->bitmap_info.offset) {
1220 pr_err("md/llbitmap: %s: no super block found", mdname(mddev));
1221 return -EINVAL;
1222 }
1223
1224 sb_page = llbitmap_read_page(llbitmap, 0);
1225 if (IS_ERR(sb_page)) {
1226 pr_err("md/llbitmap: %s: read super block failed",
1227 mdname(mddev));
1228 return -EIO;
1229 }
1230
1231 sb = kmap_local_page(sb_page);
1232 if (sb->magic != cpu_to_le32(BITMAP_MAGIC)) {
1233 pr_err("md/llbitmap: %s: invalid super block magic number",
1234 mdname(mddev));
1235 goto out_put_page;
1236 }
1237
1238 if (sb->version != cpu_to_le32(BITMAP_MAJOR_LOCKLESS)) {
1239 pr_err("md/llbitmap: %s: invalid super block version",
1240 mdname(mddev));
1241 goto out_put_page;
1242 }
1243
1244 if (memcmp(sb->uuid, mddev->uuid, 16)) {
1245 pr_err("md/llbitmap: %s: bitmap superblock UUID mismatch\n",
1246 mdname(mddev));
1247 goto out_put_page;
1248 }
1249
1250 if (mddev->bitmap_info.space == 0) {
1251 int room = le32_to_cpu(sb->sectors_reserved);
1252
1253 if (room)
1254 mddev->bitmap_info.space = room;
1255 else
1256 mddev->bitmap_info.space = mddev->bitmap_info.default_space;
1257 }
1258 llbitmap->flags = le32_to_cpu(sb->state) & ~BIT(BITMAP_SHUTDOWN);
1259 if (test_and_clear_bit(BITMAP_FIRST_USE, &llbitmap->flags)) {
1260 ret = llbitmap_init(llbitmap);
1261 goto out_put_page;
1262 }
1263
1264 sync_size = le64_to_cpu(sb->sync_size);
1265 if (!sync_size)
1266 sync_size = mddev->resync_max_sectors;
1267 if (sync_size > mddev->resync_max_sectors) {
1268 pr_err("md/llbitmap: %s: sync_size %llu exceeds array sync size %llu",
1269 mdname(mddev), sync_size, mddev->resync_max_sectors);
1270 goto out_put_page;
1271 }
1272 chunksize = le32_to_cpu(sb->chunksize);
1273 if (!is_power_of_2(chunksize)) {
1274 pr_err("md/llbitmap: %s: chunksize not a power of 2",
1275 mdname(mddev));
1276 goto out_put_page;
1277 }
1278
1279 if (chunksize < DIV_ROUND_UP_SECTOR_T(sync_size,
1280 mddev->bitmap_info.space << SECTOR_SHIFT)) {
1281 pr_err("md/llbitmap: %s: chunksize too small %lu < %llu / %lu",
1282 mdname(mddev), chunksize, sync_size,
1283 mddev->bitmap_info.space);
1284 goto out_put_page;
1285 }
1286
1287 daemon_sleep = le32_to_cpu(sb->daemon_sleep);
1288 if (daemon_sleep < 1 || daemon_sleep > MAX_SCHEDULE_TIMEOUT / HZ) {
1289 pr_err("md/llbitmap: %s: daemon sleep %lu period out of range",
1290 mdname(mddev), daemon_sleep);
1291 goto out_put_page;
1292 }
1293
1294 events = le64_to_cpu(sb->events);
1295 if (events < mddev->events) {
1296 pr_warn("md/llbitmap :%s: bitmap file is out of date (%lu < %llu) -- forcing full recovery",
1297 mdname(mddev), events, mddev->events);
1298 set_bit(BITMAP_STALE, &llbitmap->flags);
1299 }
1300
1301 sb->sync_size = cpu_to_le64(mddev->resync_max_sectors);
1302 mddev->bitmap_info.chunksize = chunksize;
1303 mddev->bitmap_info.daemon_sleep = daemon_sleep;
1304
1305 llbitmap->barrier_idle = DEFAULT_BARRIER_IDLE;
1306 llbitmap->chunksize = chunksize;
1307 llbitmap->chunks = DIV_ROUND_UP_SECTOR_T(sync_size, chunksize);
1308 llbitmap->chunkshift = ffz(~chunksize);
1309 llbitmap->sync_size = sync_size;
1310 llbitmap_refresh_reshape(llbitmap);
1311 ret = llbitmap_alloc_pages(llbitmap);
1312
1313out_put_page:
1314 __free_page(sb_page);
1315 kunmap_local(sb);
1316 return ret;
1317}
1318
1319static void llbitmap_pending_timer_fn(struct timer_list *pending_timer)
1320{
1321 struct llbitmap *llbitmap =
1322 container_of(pending_timer, struct llbitmap, pending_timer);
1323
1324 if (test_bit(BITMAP_SHUTDOWN, &llbitmap->flags))
1325 return;
1326
1327 if (work_busy(&llbitmap->daemon_work)) {
1328 pr_warn("md/llbitmap: %s daemon_work not finished in %lu seconds\n",
1329 mdname(llbitmap->mddev),
1330 llbitmap->mddev->bitmap_info.daemon_sleep);
1331 set_bit(BITMAP_DAEMON_BUSY, &llbitmap->flags);
1332 return;
1333 }
1334
1335 queue_work(md_llbitmap_io_wq, &llbitmap->daemon_work);
1336}
1337
1338static void md_llbitmap_daemon_fn(struct work_struct *work)
1339{
1340 struct llbitmap *llbitmap =
1341 container_of(work, struct llbitmap, daemon_work);
1342 unsigned long start;
1343 unsigned long end;
1344 bool restart;
1345 int idx;
1346
1347 if (test_bit(BITMAP_SHUTDOWN, &llbitmap->flags))
1348 return;
1349
1350 if (llbitmap->mddev->degraded)
1351 return;
1352
1353retry:
1354 start = 0;
1355 end = min(llbitmap->chunks, PAGE_SIZE - BITMAP_DATA_OFFSET) - 1;
1356 restart = false;
1357
1358 for (idx = 0; idx < llbitmap->nr_pages; idx++) {
1359 struct llbitmap_page_ctl *pctl = llbitmap->pctl[idx];
1360 bool flush = test_and_clear_bit(LLPageFlush, &pctl->flags);
1361
1362 if (idx > 0) {
1363 start = end + 1;
1364 end = min(end + PAGE_SIZE, llbitmap->chunks - 1);
1365 }
1366
1367 if (!flush && time_before(jiffies, pctl->expire)) {
1368 restart = true;
1369 continue;
1370 }
1371
1372 if (llbitmap_suspend_timeout(llbitmap, idx) < 0) {
1373 pr_warn("md/llbitmap: %s: %s waiting for page %d timeout\n",
1374 mdname(llbitmap->mddev), __func__, idx);
1375 continue;
1376 }
1377
1378 llbitmap_state_machine(llbitmap, start, end, BitmapActionDaemon);
1379 llbitmap_resume(llbitmap, idx);
1380 }
1381
1382 /*
1383 * If the daemon took a long time to finish, retry to prevent missing
1384 * clearing dirty bits.
1385 */
1386 if (test_and_clear_bit(BITMAP_DAEMON_BUSY, &llbitmap->flags))
1387 goto retry;
1388
1389 /* If some page is dirty but not expired, setup timer again */
1390 if (restart && !test_bit(BITMAP_SHUTDOWN, &llbitmap->flags))
1391 mod_timer(&llbitmap->pending_timer,
1392 jiffies + llbitmap->mddev->bitmap_info.daemon_sleep * HZ);
1393}
1394
1395static int llbitmap_create(struct mddev *mddev)
1396{
1397 struct llbitmap *llbitmap;
1398 int ret;
1399
1400 ret = llbitmap_check_support(mddev);
1401 if (ret)
1402 return ret;
1403
1404 llbitmap = kzalloc_obj(*llbitmap, GFP_NOIO);
1405 if (!llbitmap)
1406 return -ENOMEM;
1407
1408 llbitmap->mddev = mddev;
1409 llbitmap->io_size = bdev_logical_block_size(mddev->gendisk->part0);
1410 llbitmap->blocks_per_page = PAGE_SIZE / llbitmap->io_size;
1411
1412 timer_setup(&llbitmap->pending_timer, llbitmap_pending_timer_fn, 0);
1413 INIT_WORK(&llbitmap->daemon_work, md_llbitmap_daemon_fn);
1414 rwlock_init(&llbitmap->reshape_lock);
1415 atomic_set(&llbitmap->behind_writes, 0);
1416 init_waitqueue_head(&llbitmap->behind_wait);
1417
1418 mutex_lock(&mddev->bitmap_info.mutex);
1419 mddev->bitmap = llbitmap;
1420 ret = llbitmap_read_sb(llbitmap);
1421 if (ret)
1422 mddev->bitmap = NULL;
1423 mutex_unlock(&mddev->bitmap_info.mutex);
1424 if (ret) {
1425 kfree(llbitmap);
1426 }
1427
1428 return ret;
1429}
1430
1431static int llbitmap_resize(struct mddev *mddev, sector_t blocks, int chunksize)
1432{
1433 struct llbitmap *llbitmap = mddev->bitmap;
1434 sector_t old_blocks = llbitmap->sync_size;
1435 unsigned long old_chunks = llbitmap->chunks;
1436 unsigned long chunks;
1437 unsigned long cache_chunks;
1438 int ret = 0;
1439 unsigned long bitmap_chunksize;
1440 bool reshape;
1441 bool quiesced = false;
1442
1443 if (chunksize == 0)
1444 chunksize = llbitmap->chunksize;
1445
1446 bitmap_chunksize = chunksize;
1447 llbitmap_calculate_chunks(mddev, blocks, &bitmap_chunksize, &chunks);
1448
1449 reshape = mddev->delta_disks || mddev->new_level != mddev->level ||
1450 mddev->new_layout != mddev->layout ||
1451 mddev->new_chunk_sectors != mddev->chunk_sectors;
1452 if (!reshape && bitmap_chunksize != llbitmap->chunksize)
1453 return -EOPNOTSUPP;
1454 if (blocks == old_blocks && chunks == llbitmap->chunks)
1455 return 0;
1456
1457 if (mddev->pers->quiesce) {
1458 mddev->pers->quiesce(mddev, 1);
1459 quiesced = true;
1460 }
1461
1462 mutex_lock(&mddev->bitmap_info.mutex);
1463 cache_chunks = reshape ? max(old_chunks, chunks) : chunks;
1464 ret = llbitmap_prepare_resize(llbitmap, old_chunks, chunks, cache_chunks);
1465 if (ret)
1466 goto out;
1467
1468 if (reshape) {
1469 llbitmap->reshape_sync_size = blocks;
1470 llbitmap->reshape_chunksize = bitmap_chunksize;
1471 llbitmap->reshape_chunks = chunks;
1472 llbitmap->chunks = max(old_chunks, chunks);
1473 } else {
1474 if (blocks < old_blocks && chunks < old_chunks)
1475 llbitmap_mark_range(llbitmap, chunks, old_chunks - 1,
1476 BitUnwritten);
1477 mddev->bitmap_info.chunksize = bitmap_chunksize;
1478 llbitmap->chunks = chunks;
1479 llbitmap->sync_size = blocks;
1480 llbitmap_refresh_reshape(llbitmap);
1481 llbitmap_update_sb(llbitmap);
1482 }
1483 __llbitmap_flush(mddev);
1484 mutex_unlock(&mddev->bitmap_info.mutex);
1485 if (quiesced)
1486 mddev->pers->quiesce(mddev, 0);
1487 return 0;
1488
1489out:
1490 mutex_unlock(&mddev->bitmap_info.mutex);
1491 if (quiesced)
1492 mddev->pers->quiesce(mddev, 0);
1493 return ret;
1494}
1495
1496static int llbitmap_load(struct mddev *mddev)
1497{
1498 enum llbitmap_action action = BitmapActionReload;
1499 struct llbitmap *llbitmap = mddev->bitmap;
1500 int ret;
1501
1502 if (test_and_clear_bit(BITMAP_STALE, &llbitmap->flags))
1503 action = BitmapActionStale;
1504
1505 mutex_lock(&mddev->bitmap_info.mutex);
1506 llbitmap_refresh_reshape(llbitmap);
1507 ret = llbitmap_expand_pages(llbitmap, llbitmap->chunks);
1508 if (ret) {
1509 mutex_unlock(&mddev->bitmap_info.mutex);
1510 return ret;
1511 }
1512 llbitmap_state_machine(llbitmap, 0, llbitmap->chunks - 1, action);
1513 mutex_unlock(&mddev->bitmap_info.mutex);
1514 return 0;
1515}
1516
1517static void llbitmap_destroy(struct mddev *mddev)
1518{
1519 struct llbitmap *llbitmap = mddev->bitmap;
1520
1521 if (!llbitmap)
1522 return;
1523
1524 mutex_lock(&mddev->bitmap_info.mutex);
1525
1526 set_bit(BITMAP_SHUTDOWN, &llbitmap->flags);
1527 timer_shutdown_sync(&llbitmap->pending_timer);
1528 cancel_work_sync(&llbitmap->daemon_work);
1529 flush_workqueue(md_llbitmap_io_wq);
1530 flush_workqueue(md_llbitmap_unplug_wq);
1531
1532 mddev->bitmap = NULL;
1533 llbitmap_free_pages(llbitmap);
1534 kfree(llbitmap);
1535 mutex_unlock(&mddev->bitmap_info.mutex);
1536}
1537
1538static bool llbitmap_map_previous(struct llbitmap *llbitmap, sector_t offset,
1539 unsigned long sectors)
1540{
1541 struct mddev *mddev = llbitmap->mddev;
1542 sector_t boundary = READ_ONCE(mddev->reshape_position);
1543
1544 if (boundary == MaxSector)
1545 return false;
1546
1547 WARN_ON_ONCE(sectors && offset < boundary && offset + sectors > boundary);
1548
1549 return mddev->reshape_backwards ? offset < boundary : offset >= boundary;
1550}
1551
1552static void llbitmap_prepare_range(struct mddev *mddev, sector_t *offset,
1553 unsigned long *sectors, bool discard)
1554{
1555 struct llbitmap *llbitmap = mddev->bitmap;
1556 bool previous;
1557
1558 if (!llbitmap)
1559 return;
1560
1561 previous = llbitmap_map_previous(llbitmap, *offset, *sectors);
1562 llbitmap_map_layout(llbitmap, offset, sectors, previous);
1563 if (discard)
1564 llbitmap_encode_discard_range(llbitmap, offset, sectors, previous);
1565 else
1566 llbitmap_encode_range(llbitmap, offset, sectors, previous);
1567}
1568
1569static void llbitmap_start_write(struct mddev *mddev, sector_t offset,
1570 unsigned long sectors)
1571{
1572 struct llbitmap *llbitmap = mddev->bitmap;
1573 unsigned long start = offset >> llbitmap->chunkshift;
1574 unsigned long end = (offset + sectors - 1) >> llbitmap->chunkshift;
1575 int page_start = (start + BITMAP_DATA_OFFSET) >> PAGE_SHIFT;
1576 int page_end = (end + BITMAP_DATA_OFFSET) >> PAGE_SHIFT;
1577
1578 while (page_start <= page_end) {
1579 llbitmap_raise_barrier(llbitmap, page_start);
1580 page_start++;
1581 }
1582
1583 read_lock(&llbitmap->reshape_lock);
1584 llbitmap_state_machine(llbitmap, start, end, BitmapActionStartwrite);
1585 read_unlock(&llbitmap->reshape_lock);
1586}
1587
1588static void llbitmap_end_write(struct mddev *mddev, sector_t offset,
1589 unsigned long sectors)
1590{
1591 struct llbitmap *llbitmap = mddev->bitmap;
1592 unsigned long start = offset >> llbitmap->chunkshift;
1593 unsigned long end = (offset + sectors - 1) >> llbitmap->chunkshift;
1594 int page_start = (start + BITMAP_DATA_OFFSET) >> PAGE_SHIFT;
1595 int page_end = (end + BITMAP_DATA_OFFSET) >> PAGE_SHIFT;
1596
1597 while (page_start <= page_end) {
1598 llbitmap_release_barrier(llbitmap, page_start);
1599 page_start++;
1600 }
1601}
1602
1603static void llbitmap_start_discard(struct mddev *mddev, sector_t offset,
1604 unsigned long sectors)
1605{
1606 struct llbitmap *llbitmap = mddev->bitmap;
1607 unsigned long start = DIV_ROUND_UP_SECTOR_T(offset, llbitmap->chunksize);
1608 unsigned long end = (offset + sectors - 1) >> llbitmap->chunkshift;
1609 int page_start = (start + BITMAP_DATA_OFFSET) >> PAGE_SHIFT;
1610 int page_end = (end + BITMAP_DATA_OFFSET) >> PAGE_SHIFT;
1611
1612 while (page_start <= page_end) {
1613 llbitmap_raise_barrier(llbitmap, page_start);
1614 page_start++;
1615 }
1616
1617 read_lock(&llbitmap->reshape_lock);
1618 llbitmap_state_machine(llbitmap, start, end, BitmapActionDiscard);
1619 read_unlock(&llbitmap->reshape_lock);
1620}
1621
1622static void llbitmap_end_discard(struct mddev *mddev, sector_t offset,
1623 unsigned long sectors)
1624{
1625 struct llbitmap *llbitmap = mddev->bitmap;
1626 unsigned long start = DIV_ROUND_UP_SECTOR_T(offset, llbitmap->chunksize);
1627 unsigned long end = (offset + sectors - 1) >> llbitmap->chunkshift;
1628 int page_start = (start + BITMAP_DATA_OFFSET) >> PAGE_SHIFT;
1629 int page_end = (end + BITMAP_DATA_OFFSET) >> PAGE_SHIFT;
1630
1631 while (page_start <= page_end) {
1632 llbitmap_release_barrier(llbitmap, page_start);
1633 page_start++;
1634 }
1635}
1636
1637static void llbitmap_unplug_fn(struct work_struct *work)
1638{
1639 struct llbitmap_unplug_work *unplug_work =
1640 container_of(work, struct llbitmap_unplug_work, work);
1641 struct llbitmap *llbitmap = unplug_work->llbitmap;
1642 struct blk_plug plug;
1643 int i;
1644
1645 blk_start_plug(&plug);
1646
1647 for (i = 0; i < llbitmap->nr_pages; i++) {
1648 if (!test_bit(LLPageDirty, &llbitmap->pctl[i]->flags) ||
1649 !test_and_clear_bit(LLPageDirty, &llbitmap->pctl[i]->flags))
1650 continue;
1651
1652 llbitmap_write_page(llbitmap, i);
1653 }
1654
1655 blk_finish_plug(&plug);
1656 md_super_wait(llbitmap->mddev);
1657 complete(unplug_work->done);
1658}
1659
1660static bool llbitmap_dirty(struct llbitmap *llbitmap)
1661{
1662 int i;
1663
1664 for (i = 0; i < llbitmap->nr_pages; i++)
1665 if (test_bit(LLPageDirty, &llbitmap->pctl[i]->flags))
1666 return true;
1667
1668 return false;
1669}
1670
1671static void llbitmap_unplug(struct mddev *mddev, bool sync)
1672{
1673 DECLARE_COMPLETION_ONSTACK(done);
1674 struct llbitmap *llbitmap = mddev->bitmap;
1675 struct llbitmap_unplug_work unplug_work = {
1676 .llbitmap = llbitmap,
1677 .done = &done,
1678 };
1679
1680 if (!llbitmap_dirty(llbitmap))
1681 return;
1682
1683 /*
1684 * Issue new bitmap IO under submit_bio() context will deadlock:
1685 * - the bio will wait for bitmap bio to be done, before it can be
1686 * issued;
1687 * - bitmap bio will be added to current->bio_list and wait for this
1688 * bio to be issued;
1689 */
1690 INIT_WORK_ONSTACK(&unplug_work.work, llbitmap_unplug_fn);
1691 queue_work(md_llbitmap_unplug_wq, &unplug_work.work);
1692 wait_for_completion(&done);
1693 destroy_work_on_stack(&unplug_work.work);
1694}
1695
1696/*
1697 * Force to write all bitmap pages to disk, called when stopping the array, or
1698 * every daemon_sleep seconds when sync_thread is running.
1699 */
1700static void __llbitmap_flush(struct mddev *mddev)
1701{
1702 struct llbitmap *llbitmap = mddev->bitmap;
1703 struct blk_plug plug;
1704 int i;
1705
1706 blk_start_plug(&plug);
1707 for (i = 0; i < llbitmap->nr_pages; i++) {
1708 struct llbitmap_page_ctl *pctl = llbitmap->pctl[i];
1709
1710 /* mark all blocks as dirty */
1711 set_bit(LLPageDirty, &pctl->flags);
1712 bitmap_fill(pctl->dirty, llbitmap->blocks_per_page);
1713 llbitmap_write_page(llbitmap, i);
1714 }
1715 blk_finish_plug(&plug);
1716 md_super_wait(llbitmap->mddev);
1717}
1718
1719static void llbitmap_flush(struct mddev *mddev)
1720{
1721 struct llbitmap *llbitmap = mddev->bitmap;
1722 int i;
1723
1724 for (i = 0; i < llbitmap->nr_pages; i++)
1725 set_bit(LLPageFlush, &llbitmap->pctl[i]->flags);
1726
1727 timer_delete_sync(&llbitmap->pending_timer);
1728 queue_work(md_llbitmap_io_wq, &llbitmap->daemon_work);
1729 flush_work(&llbitmap->daemon_work);
1730
1731 __llbitmap_flush(mddev);
1732}
1733
1734/* This is used for raid5 lazy initial recovery */
1735static bool llbitmap_blocks_synced(struct mddev *mddev, sector_t offset)
1736{
1737 struct llbitmap *llbitmap = mddev->bitmap;
1738 unsigned long p = offset >> llbitmap->chunkshift;
1739 enum llbitmap_state c;
1740
1741 if (p >= llbitmap->chunks)
1742 return false;
1743 c = llbitmap_read(llbitmap, p);
1744
1745 return c == BitClean || c == BitDirty || c == BitCleanUnwritten;
1746}
1747
1748static sector_t llbitmap_skip_sync_blocks(struct mddev *mddev, sector_t offset)
1749{
1750 struct llbitmap *llbitmap = mddev->bitmap;
1751 unsigned long p = offset >> llbitmap->chunkshift;
1752 int blocks = llbitmap->chunksize - (offset & (llbitmap->chunksize - 1));
1753 enum llbitmap_state c;
1754
1755 if (p >= llbitmap->chunks)
1756 return 0;
1757 c = llbitmap_read(llbitmap, p);
1758
1759 /*
1760 * Reshape progress is tracked by array metadata rather than llbitmap.
1761 * Skipping reshape ranges from stale bitmap state can lose data after a
1762 * restart before the corresponding bits are checkpointed to disk.
1763 */
1764 if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery))
1765 return 0;
1766
1767 /* always skip unwritten blocks */
1768 if (c == BitUnwritten)
1769 return blocks;
1770
1771 /* Skip CleanUnwritten - no user data, will be reset after recovery */
1772 if (c == BitCleanUnwritten)
1773 return blocks;
1774
1775 /* For degraded array, don't skip */
1776 if (mddev->degraded)
1777 return 0;
1778
1779 /* For resync also skip clean/dirty blocks */
1780 if ((c == BitClean || c == BitDirty) &&
1781 test_bit(MD_RECOVERY_SYNC, &mddev->recovery) &&
1782 !test_bit(MD_RECOVERY_REQUESTED, &mddev->recovery))
1783 return blocks;
1784
1785 return 0;
1786}
1787
1788static bool llbitmap_start_sync(struct mddev *mddev, sector_t offset,
1789 sector_t *blocks, bool degraded)
1790{
1791 struct llbitmap *llbitmap = mddev->bitmap;
1792 unsigned long p = offset >> llbitmap->chunkshift;
1793 enum llbitmap_state state;
1794
1795 /*
1796 * Before recovery starts, convert CleanUnwritten to Unwritten.
1797 * This ensures the new disk won't have stale parity data.
1798 */
1799 if (offset == 0 && test_bit(MD_RECOVERY_RECOVER, &mddev->recovery) &&
1800 !test_bit(MD_RECOVERY_LAZY_RECOVER, &mddev->recovery))
1801 llbitmap_state_machine(llbitmap, 0, llbitmap->chunks - 1,
1802 BitmapActionClearUnwritten);
1803
1804
1805 /*
1806 * Handle one bit at a time, this is much simpler. And it doesn't matter
1807 * if md_do_sync() loop more times.
1808 */
1809 *blocks = llbitmap->chunksize - (offset & (llbitmap->chunksize - 1));
1810 if (p >= llbitmap->chunks)
1811 return false;
1812 state = llbitmap_state_machine(llbitmap, p, p, BitmapActionStartsync);
1813 return state == BitSyncing || state == BitSyncingUnwritten;
1814}
1815
1816/* Something is wrong, sync_thread stop at @offset */
1817static void llbitmap_end_sync(struct mddev *mddev, sector_t offset,
1818 sector_t *blocks)
1819{
1820 struct llbitmap *llbitmap = mddev->bitmap;
1821 unsigned long p = offset >> llbitmap->chunkshift;
1822
1823 *blocks = llbitmap->chunksize - (offset & (llbitmap->chunksize - 1));
1824 llbitmap_state_machine(llbitmap, p, llbitmap->chunks - 1,
1825 BitmapActionAbortsync);
1826}
1827
1828/* A full sync_thread is finished */
1829static void llbitmap_close_sync(struct mddev *mddev)
1830{
1831 struct llbitmap *llbitmap = mddev->bitmap;
1832 int i;
1833
1834 for (i = 0; i < llbitmap->nr_pages; i++) {
1835 struct llbitmap_page_ctl *pctl = llbitmap->pctl[i];
1836
1837 /* let daemon_fn clear dirty bits immediately */
1838 WRITE_ONCE(pctl->expire, jiffies);
1839 }
1840
1841 llbitmap_state_machine(llbitmap, 0, llbitmap->chunks - 1,
1842 BitmapActionEndsync);
1843}
1844
1845/*
1846 * sync_thread have reached @sector, update metadata every daemon_sleep seconds,
1847 * just in case sync_thread have to restart after power failure.
1848 */
1849static void llbitmap_cond_end_sync(struct mddev *mddev, sector_t sector,
1850 bool force)
1851{
1852 struct llbitmap *llbitmap = mddev->bitmap;
1853 sector_t complete;
1854
1855 if (sector == 0) {
1856 llbitmap->last_end_sync = jiffies;
1857 return;
1858 }
1859
1860 if (!force && time_before(jiffies, llbitmap->last_end_sync +
1861 HZ * mddev->bitmap_info.daemon_sleep))
1862 return;
1863
1864 wait_event(mddev->recovery_wait, !atomic_read(&mddev->recovery_active));
1865
1866 mddev->curr_resync_completed = sector;
1867 set_bit(MD_SB_CHANGE_CLEAN, &mddev->sb_flags);
1868
1869 complete = round_down(sector, llbitmap->chunksize);
1870 if (complete)
1871 llbitmap_state_machine(llbitmap, 0,
1872 (complete >> llbitmap->chunkshift) - 1,
1873 BitmapActionEndsync);
1874 __llbitmap_flush(mddev);
1875
1876 llbitmap->last_end_sync = jiffies;
1877 sysfs_notify_dirent_safe(mddev->sysfs_completed);
1878}
1879
1880static bool llbitmap_enabled(void *data, bool flush)
1881{
1882 struct llbitmap *llbitmap = data;
1883
1884 return llbitmap && !test_bit(BITMAP_WRITE_ERROR, &llbitmap->flags);
1885}
1886
1887static void llbitmap_dirty_bits(struct mddev *mddev, unsigned long s,
1888 unsigned long e)
1889{
1890 llbitmap_state_machine(mddev->bitmap, s, e, BitmapActionStartwrite);
1891}
1892
1893static int llbitmap_reshape_can_start(struct mddev *mddev)
1894{
1895 struct llbitmap *llbitmap = mddev->bitmap;
1896 unsigned long chunk;
1897 int ret = 0;
1898
1899 if (!llbitmap)
1900 return 0;
1901
1902 mutex_lock(&mddev->bitmap_info.mutex);
1903 for (chunk = 0; chunk < llbitmap->chunks; chunk++) {
1904 enum llbitmap_state state = llbitmap_read(llbitmap, chunk);
1905
1906 if (state == BitNeedSync || state == BitSyncing) {
1907 ret = -EBUSY;
1908 break;
1909 }
1910 }
1911 mutex_unlock(&mddev->bitmap_info.mutex);
1912
1913 return ret;
1914}
1915
1916struct llbitmap_reshape_range {
1917 sector_t offset;
1918 unsigned long sectors;
1919 sector_t start;
1920 sector_t end;
1921};
1922
1923static enum llbitmap_state
1924llbitmap_reshape_init_dst(struct llbitmap *llbitmap, unsigned long dst,
1925 const struct llbitmap_reshape_range *new)
1926{
1927 u64 bit_start = (u64)dst * llbitmap->reshape_chunksize;
1928 u64 bit_end = bit_start + llbitmap->reshape_chunksize;
1929
1930 if (!llbitmap->mddev->reshape_backwards)
1931 return bit_start < new->offset ? llbitmap_read(llbitmap, dst) :
1932 BitUnwritten;
1933 return bit_end > new->end ? llbitmap_read(llbitmap, dst) : BitUnwritten;
1934}
1935
1936static void llbitmap_reshape_dst_range(struct llbitmap *llbitmap,
1937 unsigned long dst,
1938 const struct llbitmap_reshape_range *new,
1939 struct llbitmap_reshape_range *dst_range)
1940{
1941 sector_t dst_bit_start = (sector_t)dst * llbitmap->reshape_chunksize;
1942
1943 dst_range->start = max(dst_bit_start, new->offset);
1944 dst_range->end = min(dst_bit_start + llbitmap->reshape_chunksize,
1945 new->end);
1946 dst_range->offset = dst_range->start;
1947 dst_range->sectors = dst_range->end - dst_range->start;
1948}
1949
1950static void llbitmap_reshape_map_range(struct llbitmap *llbitmap,
1951 sector_t lo, sector_t hi,
1952 bool previous,
1953 struct llbitmap_reshape_range *range)
1954{
1955 range->offset = lo;
1956 range->sectors = hi - lo;
1957 llbitmap_map_layout(llbitmap, &range->offset, &range->sectors, previous);
1958 range->start = range->offset;
1959 range->end = range->offset + range->sectors;
1960}
1961
1962static bool llbitmap_reshape_src_range(const struct llbitmap_reshape_range *old,
1963 const struct llbitmap_reshape_range *new,
1964 const struct llbitmap_reshape_range *dst,
1965 struct llbitmap_reshape_range *src)
1966{
1967 if (!old->sectors)
1968 return false;
1969
1970 src->start = old->offset +
1971 mul_u64_u64_div_u64(dst->start - new->offset,
1972 old->sectors, new->sectors);
1973 src->end = old->offset +
1974 mul_u64_u64_div_u64_roundup(dst->end - new->offset,
1975 old->sectors, new->sectors);
1976 if (src->end > old->end)
1977 src->end = old->end;
1978 src->offset = src->start;
1979 src->sectors = src->end - src->start;
1980
1981 return src->sectors;
1982}
1983
1984static enum llbitmap_state llbitmap_rmerge_src(struct llbitmap *llbitmap,
1985 enum llbitmap_state state,
1986 const struct llbitmap_reshape_range *src)
1987{
1988 unsigned long bit = div64_u64(src->start, llbitmap->chunksize);
1989 unsigned long end = div64_u64(src->end - 1, llbitmap->chunksize);
1990
1991 while (bit <= end) {
1992 enum llbitmap_state src_state = llbitmap_read(llbitmap, bit);
1993
1994 state = llbitmap_rmerge_state(llbitmap, state, src_state);
1995 bit++;
1996 }
1997
1998 return state;
1999}
2000
2001static void llbitmap_reshape_merge(struct llbitmap *llbitmap,
2002 const struct llbitmap_reshape_range *old,
2003 const struct llbitmap_reshape_range *new)
2004{
2005 unsigned long dst_start;
2006 unsigned long dst_end;
2007 unsigned long dst;
2008 bool backwards = false;
2009
2010 if (!new->sectors)
2011 return;
2012
2013 dst_start = div64_u64(new->offset, llbitmap->reshape_chunksize);
2014 dst_end = div64_u64(new->end - 1, llbitmap->reshape_chunksize);
2015 if (old->sectors) {
2016 unsigned long src_start = div64_u64(old->offset,
2017 llbitmap->chunksize);
2018 unsigned long src_end = div64_u64(old->end - 1,
2019 llbitmap->chunksize);
2020
2021 backwards = src_start < dst_start && src_end >= dst_start;
2022 }
2023
2024 dst = backwards ? dst_end : dst_start;
2025 while (true) {
2026 struct llbitmap_reshape_range dst_range;
2027 struct llbitmap_reshape_range src;
2028 enum llbitmap_state state;
2029
2030 llbitmap_reshape_dst_range(llbitmap, dst, new, &dst_range);
2031 state = llbitmap_reshape_init_dst(llbitmap, dst, new);
2032 if (llbitmap_reshape_src_range(old, new, &dst_range, &src))
2033 state = llbitmap_rmerge_src(llbitmap, state, &src);
2034 else
2035 state = llbitmap_rmerge_state(llbitmap, state, BitUnwritten);
2036 llbitmap_write(llbitmap, state, dst);
2037 if (dst == (backwards ? dst_start : dst_end))
2038 break;
2039 if (backwards)
2040 dst--;
2041 else
2042 dst++;
2043 }
2044}
2045
2046static void llbitmap_reshape_finish(struct mddev *mddev)
2047{
2048 struct llbitmap *llbitmap = mddev->bitmap;
2049
2050 if (mddev->pers->quiesce)
2051 mddev->pers->quiesce(mddev, 1);
2052
2053 mutex_lock(&mddev->bitmap_info.mutex);
2054 llbitmap_flush(mddev);
2055
2056 llbitmap->chunksize = llbitmap->reshape_chunksize;
2057 llbitmap->chunkshift = ffz(~llbitmap->chunksize);
2058 llbitmap->chunks = llbitmap->reshape_chunks;
2059 llbitmap->sync_size = llbitmap->reshape_sync_size;
2060 llbitmap_refresh_reshape(llbitmap);
2061 mddev->bitmap_info.chunksize = llbitmap->chunksize;
2062 llbitmap_update_sb(llbitmap);
2063 __llbitmap_flush(mddev);
2064 mutex_unlock(&mddev->bitmap_info.mutex);
2065
2066 if (mddev->pers->quiesce)
2067 mddev->pers->quiesce(mddev, 0);
2068}
2069
2070static void llbitmap_reshape_mark(struct mddev *mddev, sector_t old_pos,
2071 sector_t new_pos)
2072{
2073 struct llbitmap *llbitmap = mddev->bitmap;
2074 sector_t lo;
2075 sector_t hi;
2076 struct llbitmap_reshape_range old;
2077 struct llbitmap_reshape_range new;
2078
2079 if (!llbitmap || old_pos == new_pos)
2080 return;
2081
2082 lo = min(old_pos, new_pos);
2083 hi = max(old_pos, new_pos);
2084 if (!hi)
2085 return;
2086
2087 llbitmap_reshape_map_range(llbitmap, lo, hi, true, &old);
2088 llbitmap_reshape_map_range(llbitmap, lo, hi, false, &new);
2089 if (!new.sectors)
2090 return;
2091
2092 write_lock(&llbitmap->reshape_lock);
2093 llbitmap_reshape_merge(llbitmap, &old, &new);
2094 write_unlock(&llbitmap->reshape_lock);
2095}
2096
2097static void llbitmap_write_sb(struct llbitmap *llbitmap)
2098{
2099 int nr_blocks = DIV_ROUND_UP(BITMAP_DATA_OFFSET, llbitmap->io_size);
2100
2101 bitmap_fill(llbitmap->pctl[0]->dirty, nr_blocks);
2102 llbitmap_write_page(llbitmap, 0);
2103 md_super_wait(llbitmap->mddev);
2104}
2105
2106static void llbitmap_update_sb(void *data)
2107{
2108 struct llbitmap *llbitmap = data;
2109 struct mddev *mddev = llbitmap->mddev;
2110 struct page *sb_page;
2111 bitmap_super_t *sb;
2112
2113 if (test_bit(BITMAP_WRITE_ERROR, &llbitmap->flags))
2114 return;
2115
2116 sb_page = llbitmap_read_page(llbitmap, 0);
2117 if (IS_ERR(sb_page)) {
2118 pr_err("%s: %s: read super block failed", __func__,
2119 mdname(mddev));
2120 set_bit(BITMAP_WRITE_ERROR, &llbitmap->flags);
2121 return;
2122 }
2123
2124 if (mddev->events < llbitmap->events_cleared)
2125 llbitmap->events_cleared = mddev->events;
2126
2127 sb = kmap_local_page(sb_page);
2128 sb->events = cpu_to_le64(mddev->events);
2129 sb->state = cpu_to_le32(llbitmap->flags & ~BIT(BITMAP_SHUTDOWN));
2130 sb->chunksize = cpu_to_le32(llbitmap->chunksize);
2131 sb->sync_size = cpu_to_le64(llbitmap->sync_size);
2132 sb->events_cleared = cpu_to_le64(llbitmap->events_cleared);
2133 sb->sectors_reserved = cpu_to_le32(mddev->bitmap_info.space);
2134 sb->daemon_sleep = cpu_to_le32(mddev->bitmap_info.daemon_sleep);
2135
2136 kunmap_local(sb);
2137 llbitmap_write_sb(llbitmap);
2138}
2139
2140static int llbitmap_get_stats(void *data, struct md_bitmap_stats *stats)
2141{
2142 struct llbitmap *llbitmap = data;
2143
2144 memset(stats, 0, sizeof(*stats));
2145
2146 stats->missing_pages = 0;
2147 stats->pages = llbitmap->nr_pages;
2148 stats->file_pages = llbitmap->nr_pages;
2149 stats->sync_size = llbitmap->sync_size;
2150
2151 stats->behind_writes = atomic_read(&llbitmap->behind_writes);
2152 stats->behind_wait = wq_has_sleeper(&llbitmap->behind_wait);
2153 stats->events_cleared = llbitmap->events_cleared;
2154
2155 return 0;
2156}
2157
2158/* just flag all pages as needing to be written */
2159static void llbitmap_write_all(struct mddev *mddev)
2160{
2161 int i;
2162 struct llbitmap *llbitmap = mddev->bitmap;
2163
2164 for (i = 0; i < llbitmap->nr_pages; i++) {
2165 struct llbitmap_page_ctl *pctl = llbitmap->pctl[i];
2166
2167 set_bit(LLPageDirty, &pctl->flags);
2168 bitmap_fill(pctl->dirty, llbitmap->blocks_per_page);
2169 }
2170}
2171
2172static void llbitmap_start_behind_write(struct mddev *mddev)
2173{
2174 struct llbitmap *llbitmap = mddev->bitmap;
2175
2176 atomic_inc(&llbitmap->behind_writes);
2177}
2178
2179static void llbitmap_end_behind_write(struct mddev *mddev)
2180{
2181 struct llbitmap *llbitmap = mddev->bitmap;
2182
2183 if (atomic_dec_and_test(&llbitmap->behind_writes))
2184 wake_up(&llbitmap->behind_wait);
2185}
2186
2187static void llbitmap_wait_behind_writes(struct mddev *mddev)
2188{
2189 struct llbitmap *llbitmap = mddev->bitmap;
2190
2191 if (llbitmap && atomic_read(&llbitmap->behind_writes) > 0)
2192 wait_event(llbitmap->behind_wait,
2193 atomic_read(&llbitmap->behind_writes) == 0);
2194}
2195
2196static ssize_t bits_show(struct mddev *mddev, char *page)
2197{
2198 struct llbitmap *llbitmap;
2199 int bits[BitStateCount] = {0};
2200 loff_t start = 0;
2201
2202 mutex_lock(&mddev->bitmap_info.mutex);
2203 llbitmap = mddev->bitmap;
2204 if (!llbitmap || !llbitmap->pctl) {
2205 mutex_unlock(&mddev->bitmap_info.mutex);
2206 return sprintf(page, "no bitmap\n");
2207 }
2208
2209 if (test_bit(BITMAP_WRITE_ERROR, &llbitmap->flags)) {
2210 mutex_unlock(&mddev->bitmap_info.mutex);
2211 return sprintf(page, "bitmap io error\n");
2212 }
2213
2214 while (start < llbitmap->chunks) {
2215 enum llbitmap_state c = llbitmap_read(llbitmap, start);
2216
2217 if (c < 0 || c >= BitStateCount)
2218 pr_err("%s: invalid bit %llu state %d\n",
2219 __func__, start, c);
2220 else
2221 bits[c]++;
2222 start++;
2223 }
2224
2225 mutex_unlock(&mddev->bitmap_info.mutex);
2226 return sprintf(page,
2227 "unwritten %d\nclean %d\ndirty %d\n"
2228 "need sync %d\nsyncing %d\n"
2229 "need sync unwritten %d\nsyncing unwritten %d\n"
2230 "clean unwritten %d\n",
2231 bits[BitUnwritten], bits[BitClean], bits[BitDirty],
2232 bits[BitNeedSync], bits[BitSyncing],
2233 bits[BitNeedSyncUnwritten], bits[BitSyncingUnwritten],
2234 bits[BitCleanUnwritten]);
2235}
2236
2237static struct md_sysfs_entry llbitmap_bits = __ATTR_RO(bits);
2238
2239static ssize_t metadata_show(struct mddev *mddev, char *page)
2240{
2241 struct llbitmap *llbitmap;
2242 ssize_t ret;
2243
2244 mutex_lock(&mddev->bitmap_info.mutex);
2245 llbitmap = mddev->bitmap;
2246 if (!llbitmap) {
2247 mutex_unlock(&mddev->bitmap_info.mutex);
2248 return sprintf(page, "no bitmap\n");
2249 }
2250
2251 ret = sprintf(page, "chunksize %lu\nchunkshift %lu\nchunks %lu\noffset %llu\ndaemon_sleep %lu\n",
2252 llbitmap->chunksize, llbitmap->chunkshift,
2253 llbitmap->chunks, mddev->bitmap_info.offset,
2254 llbitmap->mddev->bitmap_info.daemon_sleep);
2255 mutex_unlock(&mddev->bitmap_info.mutex);
2256
2257 return ret;
2258}
2259
2260static struct md_sysfs_entry llbitmap_metadata = __ATTR_RO(metadata);
2261
2262static ssize_t
2263daemon_sleep_show(struct mddev *mddev, char *page)
2264{
2265 return sprintf(page, "%lu\n", mddev->bitmap_info.daemon_sleep);
2266}
2267
2268static ssize_t
2269daemon_sleep_store(struct mddev *mddev, const char *buf, size_t len)
2270{
2271 unsigned long timeout;
2272 int rv = kstrtoul(buf, 10, &timeout);
2273
2274 if (rv)
2275 return rv;
2276
2277 mddev->bitmap_info.daemon_sleep = timeout;
2278 return len;
2279}
2280
2281static struct md_sysfs_entry llbitmap_daemon_sleep = __ATTR_RW(daemon_sleep);
2282
2283static ssize_t
2284barrier_idle_show(struct mddev *mddev, char *page)
2285{
2286 struct llbitmap *llbitmap = mddev->bitmap;
2287
2288 return sprintf(page, "%lu\n", llbitmap->barrier_idle);
2289}
2290
2291static ssize_t
2292barrier_idle_store(struct mddev *mddev, const char *buf, size_t len)
2293{
2294 struct llbitmap *llbitmap = mddev->bitmap;
2295 unsigned long timeout;
2296 int rv = kstrtoul(buf, 10, &timeout);
2297
2298 if (rv)
2299 return rv;
2300
2301 llbitmap->barrier_idle = timeout;
2302 return len;
2303}
2304
2305static struct md_sysfs_entry llbitmap_barrier_idle = __ATTR_RW(barrier_idle);
2306
2307static ssize_t
2308proactive_sync_store(struct mddev *mddev, const char *buf, size_t len)
2309{
2310 struct llbitmap *llbitmap;
2311
2312 /* Only for RAID-456 */
2313 if (!raid_is_456(mddev))
2314 return -EINVAL;
2315
2316 mutex_lock(&mddev->bitmap_info.mutex);
2317 llbitmap = mddev->bitmap;
2318 if (!llbitmap || !llbitmap->pctl) {
2319 mutex_unlock(&mddev->bitmap_info.mutex);
2320 return -ENODEV;
2321 }
2322
2323 /* Trigger proactive sync on all Unwritten regions */
2324 llbitmap_state_machine(llbitmap, 0, llbitmap->chunks - 1,
2325 BitmapActionProactiveSync);
2326
2327 mutex_unlock(&mddev->bitmap_info.mutex);
2328 return len;
2329}
2330
2331static struct md_sysfs_entry llbitmap_proactive_sync =
2332 __ATTR(proactive_sync, 0200, NULL, proactive_sync_store);
2333
2334static struct attribute *md_llbitmap_attrs[] = {
2335 &llbitmap_bits.attr,
2336 &llbitmap_metadata.attr,
2337 &llbitmap_daemon_sleep.attr,
2338 &llbitmap_barrier_idle.attr,
2339 &llbitmap_proactive_sync.attr,
2340 NULL
2341};
2342
2343static struct attribute_group md_llbitmap_group = {
2344 .name = "llbitmap",
2345 .attrs = md_llbitmap_attrs,
2346};
2347
2348static const struct attribute_group *md_llbitmap_groups[] = {
2349 &md_llbitmap_group,
2350 NULL,
2351};
2352
2353static struct bitmap_operations llbitmap_ops = {
2354 .head = {
2355 .type = MD_BITMAP,
2356 .id = ID_LLBITMAP,
2357 .name = "llbitmap",
2358 },
2359
2360 .enabled = llbitmap_enabled,
2361 .create = llbitmap_create,
2362 .resize = llbitmap_resize,
2363 .load = llbitmap_load,
2364 .destroy = llbitmap_destroy,
2365
2366 .start_write = llbitmap_start_write,
2367 .end_write = llbitmap_end_write,
2368 .start_discard = llbitmap_start_discard,
2369 .end_discard = llbitmap_end_discard,
2370 .unplug = llbitmap_unplug,
2371 .flush = llbitmap_flush,
2372
2373 .start_behind_write = llbitmap_start_behind_write,
2374 .end_behind_write = llbitmap_end_behind_write,
2375 .wait_behind_writes = llbitmap_wait_behind_writes,
2376
2377 .blocks_synced = llbitmap_blocks_synced,
2378 .skip_sync_blocks = llbitmap_skip_sync_blocks,
2379 .start_sync = llbitmap_start_sync,
2380 .end_sync = llbitmap_end_sync,
2381 .close_sync = llbitmap_close_sync,
2382 .cond_end_sync = llbitmap_cond_end_sync,
2383
2384 .update_sb = llbitmap_update_sb,
2385 .get_stats = llbitmap_get_stats,
2386 .dirty_bits = llbitmap_dirty_bits,
2387 .prepare_range = llbitmap_prepare_range,
2388 .reshape_finish = llbitmap_reshape_finish,
2389 .reshape_can_start = llbitmap_reshape_can_start,
2390 .reshape_mark = llbitmap_reshape_mark,
2391 .write_all = llbitmap_write_all,
2392
2393 .groups = md_llbitmap_groups,
2394};
2395
2396int md_llbitmap_init(void)
2397{
2398 md_llbitmap_io_wq = alloc_workqueue("md_llbitmap_io",
2399 WQ_MEM_RECLAIM | WQ_UNBOUND, 0);
2400 if (!md_llbitmap_io_wq)
2401 return -ENOMEM;
2402
2403 md_llbitmap_unplug_wq = alloc_workqueue("md_llbitmap_unplug",
2404 WQ_MEM_RECLAIM | WQ_UNBOUND, 0);
2405 if (!md_llbitmap_unplug_wq) {
2406 destroy_workqueue(md_llbitmap_io_wq);
2407 md_llbitmap_io_wq = NULL;
2408 return -ENOMEM;
2409 }
2410
2411 return register_md_submodule(&llbitmap_ops.head);
2412}
2413
2414void md_llbitmap_exit(void)
2415{
2416 destroy_workqueue(md_llbitmap_io_wq);
2417 md_llbitmap_io_wq = NULL;
2418 destroy_workqueue(md_llbitmap_unplug_wq);
2419 md_llbitmap_unplug_wq = NULL;
2420 unregister_md_submodule(&llbitmap_ops.head);
2421}