projects/linux·blob master · refs

3152 lines · 83.4 KiB

1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * bitmap.c two-level bitmap (C) Peter T. Breuer ([email protected]) 2003
4 *
5 * bitmap_create - sets up the bitmap structure
6 * bitmap_destroy - destroys the bitmap structure
7 *
8 * additions, Copyright (C) 2003-2004, Paul Clements, SteelEye Technology, Inc.:
9 * - added disk storage for bitmap
10 * - changes to allow various bitmap chunk sizes
11 */
12
13/*
14 * Still to do:
15 *
16 * flush after percent set rather than just time based. (maybe both).
17 */
18
19#include <linux/blkdev.h>
20#include <linux/module.h>
21#include <linux/errno.h>
22#include <linux/slab.h>
23#include <linux/init.h>
24#include <linux/timer.h>
25#include <linux/sched.h>
26#include <linux/list.h>
27#include <linux/file.h>
28#include <linux/mount.h>
29#include <linux/buffer_head.h>
30#include <linux/seq_file.h>
31#include <trace/events/block.h>
32
33#include "md.h"
34#include "md-bitmap.h"
35#include "md-cluster.h"
36
37/*
38 * in-memory bitmap:
39 *
40 * Use 16 bit block counters to track pending writes to each "chunk".
41 * The 2 high order bits are special-purpose, the first is a flag indicating
42 * whether a resync is needed. The second is a flag indicating whether a
43 * resync is active.
44 * This means that the counter is actually 14 bits:
45 *
46 * +--------+--------+------------------------------------------------+
47 * | resync | resync | counter |
48 * | needed | active | |
49 * | (0-1) | (0-1) | (0-16383) |
50 * +--------+--------+------------------------------------------------+
51 *
52 * The "resync needed" bit is set when:
53 * a '1' bit is read from storage at startup.
54 * a write request fails on some drives
55 * a resync is aborted on a chunk with 'resync active' set
56 * It is cleared (and resync-active set) when a resync starts across all drives
57 * of the chunk.
58 *
59 *
60 * The "resync active" bit is set when:
61 * a resync is started on all drives, and resync_needed is set.
62 * resync_needed will be cleared (as long as resync_active wasn't already set).
63 * It is cleared when a resync completes.
64 *
65 * The counter counts pending write requests, plus the on-disk bit.
66 * When the counter is '1' and the resync bits are clear, the on-disk
67 * bit can be cleared as well, thus setting the counter to 0.
68 * When we set a bit, or in the counter (to start a write), if the fields is
69 * 0, we first set the disk bit and set the counter to 1.
70 *
71 * If the counter is 0, the on-disk bit is clear and the stripe is clean
72 * Anything that dirties the stripe pushes the counter to 2 (at least)
73 * and sets the on-disk bit (lazily).
74 * If a periodic sweep find the counter at 2, it is decremented to 1.
75 * If the sweep find the counter at 1, the on-disk bit is cleared and the
76 * counter goes to zero.
77 *
78 * Also, we'll hijack the "map" pointer itself and use it as two 16 bit block
79 * counters as a fallback when "page" memory cannot be allocated:
80 *
81 * Normal case (page memory allocated):
82 *
83 * page pointer (32-bit)
84 *
85 * [ ] ------+
86 * |
87 * +-------> [ ][ ]..[ ] (4096 byte page == 2048 counters)
88 * c1 c2 c2048
89 *
90 * Hijacked case (page memory allocation failed):
91 *
92 * hijacked page pointer (32-bit)
93 *
94 * [ ][ ] (no page memory allocated)
95 * counter #1 (16-bit) counter #2 (16-bit)
96 *
97 */
98
99typedef __u16 bitmap_counter_t;
100
101#define PAGE_BITS (PAGE_SIZE << 3)
102#define PAGE_BIT_SHIFT (PAGE_SHIFT + 3)
103
104#define COUNTER_BITS 16
105#define COUNTER_BIT_SHIFT 4
106#define COUNTER_BYTE_SHIFT (COUNTER_BIT_SHIFT - 3)
107
108#define NEEDED_MASK ((bitmap_counter_t) (1 << (COUNTER_BITS - 1)))
109#define RESYNC_MASK ((bitmap_counter_t) (1 << (COUNTER_BITS - 2)))
110#define COUNTER_MAX ((bitmap_counter_t) RESYNC_MASK - 1)
111
112#define NEEDED(x) (((bitmap_counter_t) x) & NEEDED_MASK)
113#define RESYNC(x) (((bitmap_counter_t) x) & RESYNC_MASK)
114#define COUNTER(x) (((bitmap_counter_t) x) & COUNTER_MAX)
115
116/* how many counters per page? */
117#define PAGE_COUNTER_RATIO (PAGE_BITS / COUNTER_BITS)
118/* same, except a shift value for more efficient bitops */
119#define PAGE_COUNTER_SHIFT (PAGE_BIT_SHIFT - COUNTER_BIT_SHIFT)
120/* same, except a mask value for more efficient bitops */
121#define PAGE_COUNTER_MASK (PAGE_COUNTER_RATIO - 1)
122
123#define BITMAP_BLOCK_SHIFT 9
124
125/*
126 * bitmap structures:
127 */
128
129/* the in-memory bitmap is represented by bitmap_pages */
130struct bitmap_page {
131 /*
132 * map points to the actual memory page
133 */
134 char *map;
135 /*
136 * in emergencies (when map cannot be alloced), hijack the map
137 * pointer and use it as two counters itself
138 */
139 unsigned int hijacked:1;
140 /*
141 * If any counter in this page is '1' or '2' - and so could be
142 * cleared then that page is marked as 'pending'
143 */
144 unsigned int pending:1;
145 /*
146 * count of dirty bits on the page
147 */
148 unsigned int count:30;
149};
150
151/* the main bitmap structure - one per mddev */
152struct bitmap {
153
154 struct bitmap_counts {
155 spinlock_t lock;
156 struct bitmap_page *bp;
157 /* total number of pages in the bitmap */
158 unsigned long pages;
159 /* number of pages not yet allocated */
160 unsigned long missing_pages;
161 /* chunksize = 2^chunkshift (for bitops) */
162 unsigned long chunkshift;
163 /* total number of data chunks for the array */
164 unsigned long chunks;
165 } counts;
166
167 struct mddev *mddev; /* the md device that the bitmap is for */
168
169 __u64 events_cleared;
170 int need_sync;
171
172 struct bitmap_storage {
173 /* backing disk file */
174 struct file *file;
175 /* cached copy of the bitmap file superblock */
176 struct page *sb_page;
177 unsigned long sb_index;
178 /* list of cache pages for the file */
179 struct page **filemap;
180 /* attributes associated filemap pages */
181 unsigned long *filemap_attr;
182 /* number of pages in the file */
183 unsigned long file_pages;
184 /* total bytes in the bitmap */
185 unsigned long bytes;
186 } storage;
187
188 unsigned long flags;
189
190 int allclean;
191
192 atomic_t behind_writes;
193 /* highest actual value at runtime */
194 unsigned long behind_writes_used;
195
196 /*
197 * the bitmap daemon - periodically wakes up and sweeps the bitmap
198 * file, cleaning up bits and flushing out pages to disk as necessary
199 */
200 unsigned long daemon_lastrun; /* jiffies of last run */
201 /*
202 * when we lasted called end_sync to update bitmap with resync
203 * progress.
204 */
205 unsigned long last_end_sync;
206
207 /* pending writes to the bitmap file */
208 atomic_t pending_writes;
209 wait_queue_head_t write_wait;
210 wait_queue_head_t overflow_wait;
211 wait_queue_head_t behind_wait;
212
213 struct kernfs_node *sysfs_can_clear;
214 /* slot offset for clustered env */
215 int cluster_slot;
216};
217
218static struct workqueue_struct *md_bitmap_wq;
219static struct attribute_group md_bitmap_internal_group;
220
221static int __bitmap_resize(struct bitmap *bitmap, sector_t blocks,
222 int chunksize, bool init);
223
224static inline char *bmname(struct bitmap *bitmap)
225{
226 return bitmap->mddev ? mdname(bitmap->mddev) : "mdX";
227}
228
229static bool bitmap_enabled(void *data, bool flush)
230{
231 struct bitmap *bitmap = data;
232
233 if (!flush)
234 return true;
235
236 /*
237 * If caller want to flush bitmap pages to underlying disks, check if
238 * there are cached pages in filemap.
239 */
240 return !test_bit(BITMAP_STALE, &bitmap->flags) &&
241 bitmap->storage.filemap != NULL;
242}
243
244/*
245 * check a page and, if necessary, allocate it (or hijack it if the alloc fails)
246 *
247 * 1) check to see if this page is allocated, if it's not then try to alloc
248 * 2) if the alloc fails, set the page's hijacked flag so we'll use the
249 * page pointer directly as a counter
250 *
251 * if we find our page, we increment the page's refcount so that it stays
252 * allocated while we're using it
253 */
254static int md_bitmap_checkpage(struct bitmap_counts *bitmap,
255 unsigned long page, int create, int no_hijack)
256__releases(bitmap->lock)
257__acquires(bitmap->lock)
258{
259 unsigned char *mappage;
260
261 WARN_ON_ONCE(page >= bitmap->pages);
262 if (bitmap->bp[page].hijacked) /* it's hijacked, don't try to alloc */
263 return 0;
264
265 if (bitmap->bp[page].map) /* page is already allocated, just return */
266 return 0;
267
268 if (!create)
269 return -ENOENT;
270
271 /* this page has not been allocated yet */
272
273 spin_unlock_irq(&bitmap->lock);
274 /* It is possible that this is being called inside a
275 * prepare_to_wait/finish_wait loop from raid5c:make_request().
276 * In general it is not permitted to sleep in that context as it
277 * can cause the loop to spin freely.
278 * That doesn't apply here as we can only reach this point
279 * once with any loop.
280 * When this function completes, either bp[page].map or
281 * bp[page].hijacked. In either case, this function will
282 * abort before getting to this point again. So there is
283 * no risk of a free-spin, and so it is safe to assert
284 * that sleeping here is allowed.
285 */
286 sched_annotate_sleep();
287 mappage = kzalloc(PAGE_SIZE, GFP_NOIO);
288 spin_lock_irq(&bitmap->lock);
289
290 if (mappage == NULL) {
291 pr_debug("md/bitmap: map page allocation failed, hijacking\n");
292 /* We don't support hijack for cluster raid */
293 if (no_hijack)
294 return -ENOMEM;
295 /* failed - set the hijacked flag so that we can use the
296 * pointer as a counter */
297 if (!bitmap->bp[page].map)
298 bitmap->bp[page].hijacked = 1;
299 } else if (bitmap->bp[page].map ||
300 bitmap->bp[page].hijacked) {
301 /* somebody beat us to getting the page */
302 kfree(mappage);
303 } else {
304
305 /* no page was in place and we have one, so install it */
306
307 bitmap->bp[page].map = mappage;
308 bitmap->missing_pages--;
309 }
310 return 0;
311}
312
313/* if page is completely empty, put it back on the free list, or dealloc it */
314/* if page was hijacked, unmark the flag so it might get alloced next time */
315/* Note: lock should be held when calling this */
316static void md_bitmap_checkfree(struct bitmap_counts *bitmap, unsigned long page)
317{
318 char *ptr;
319
320 if (bitmap->bp[page].count) /* page is still busy */
321 return;
322
323 /* page is no longer in use, it can be released */
324
325 if (bitmap->bp[page].hijacked) { /* page was hijacked, undo this now */
326 bitmap->bp[page].hijacked = 0;
327 bitmap->bp[page].map = NULL;
328 } else {
329 /* normal case, free the page */
330 ptr = bitmap->bp[page].map;
331 bitmap->bp[page].map = NULL;
332 bitmap->missing_pages++;
333 kfree(ptr);
334 }
335}
336
337/*
338 * bitmap file handling - read and write the bitmap file and its superblock
339 */
340
341/*
342 * basic page I/O operations
343 */
344
345/* IO operations when bitmap is stored near all superblocks */
346
347/* choose a good rdev and read the page from there */
348static int read_sb_page(struct mddev *mddev, loff_t offset,
349 struct page *page, unsigned long index, int size)
350{
351
352 sector_t sector = mddev->bitmap_info.offset + offset +
353 index * (PAGE_SIZE / SECTOR_SIZE);
354 struct md_rdev *rdev;
355
356 rdev_for_each(rdev, mddev) {
357 u32 iosize = roundup(size, bdev_logical_block_size(rdev->bdev));
358
359 if (!test_bit(In_sync, &rdev->flags) ||
360 test_bit(Faulty, &rdev->flags) ||
361 test_bit(Bitmap_sync, &rdev->flags))
362 continue;
363
364 if (sync_page_io(rdev, sector, iosize, page, REQ_OP_READ, true))
365 return 0;
366 }
367 return -EIO;
368}
369
370static struct md_rdev *next_active_rdev(struct md_rdev *rdev, struct mddev *mddev)
371{
372 /* Iterate the disks of an mddev, using rcu to protect access to the
373 * linked list, and raising the refcount of devices we return to ensure
374 * they don't disappear while in use.
375 * As devices are only added or removed when raid_disk is < 0 and
376 * nr_pending is 0 and In_sync is clear, the entries we return will
377 * still be in the same position on the list when we re-enter
378 * list_for_each_entry_continue_rcu.
379 *
380 * Note that if entered with 'rdev == NULL' to start at the
381 * beginning, we temporarily assign 'rdev' to an address which
382 * isn't really an rdev, but which can be used by
383 * list_for_each_entry_continue_rcu() to find the first entry.
384 */
385 rcu_read_lock();
386 if (rdev == NULL)
387 /* start at the beginning */
388 rdev = list_entry(&mddev->disks, struct md_rdev, same_set);
389 else {
390 /* release the previous rdev and start from there. */
391 rdev_dec_pending(rdev, mddev);
392 }
393 list_for_each_entry_continue_rcu(rdev, &mddev->disks, same_set) {
394 if (rdev->raid_disk >= 0 &&
395 !test_bit(Faulty, &rdev->flags)) {
396 /* this is a usable devices */
397 atomic_inc(&rdev->nr_pending);
398 rcu_read_unlock();
399 return rdev;
400 }
401 }
402 rcu_read_unlock();
403 return NULL;
404}
405
406static unsigned int optimal_io_size(struct block_device *bdev,
407 unsigned int last_page_size,
408 unsigned int io_size)
409{
410 if (bdev_io_opt(bdev) > bdev_logical_block_size(bdev))
411 return roundup(last_page_size, bdev_io_opt(bdev));
412 return io_size;
413}
414
415static unsigned int bitmap_io_size(unsigned int io_size, unsigned int opt_size,
416 loff_t start, loff_t boundary)
417{
418 if (io_size != opt_size &&
419 start + opt_size / SECTOR_SIZE <= boundary)
420 return opt_size;
421 if (start + io_size / SECTOR_SIZE <= boundary)
422 return io_size;
423
424 /* Overflows boundary */
425 return 0;
426}
427
428static int __write_sb_page(struct md_rdev *rdev, struct bitmap *bitmap,
429 unsigned long pg_index, struct page *page)
430{
431 struct block_device *bdev;
432 struct mddev *mddev = bitmap->mddev;
433 struct bitmap_storage *store = &bitmap->storage;
434 unsigned long num_pages = bitmap->storage.file_pages;
435 unsigned int bitmap_limit = (num_pages - pg_index % num_pages) << PAGE_SHIFT;
436 loff_t sboff, offset = mddev->bitmap_info.offset;
437 sector_t ps = pg_index * PAGE_SIZE / SECTOR_SIZE;
438 unsigned int size = PAGE_SIZE;
439 unsigned int opt_size = PAGE_SIZE;
440 sector_t doff;
441
442 bdev = (rdev->meta_bdev) ? rdev->meta_bdev : rdev->bdev;
443 /* we compare length (page numbers), not page offset. */
444 if ((pg_index - store->sb_index) == num_pages - 1) {
445 unsigned int last_page_size = store->bytes & (PAGE_SIZE - 1);
446
447 if (last_page_size == 0)
448 last_page_size = PAGE_SIZE;
449 size = roundup(last_page_size, bdev_logical_block_size(bdev));
450 opt_size = optimal_io_size(bdev, last_page_size, size);
451 }
452
453 sboff = rdev->sb_start + offset;
454 doff = rdev->data_offset;
455
456 /* Just make sure we aren't corrupting data or metadata */
457 if (mddev->external) {
458 /* Bitmap could be anywhere. */
459 if (sboff + ps > doff &&
460 sboff < (doff + mddev->dev_sectors + PAGE_SIZE / SECTOR_SIZE))
461 return -EINVAL;
462 } else if (offset < 0) {
463 /* DATA BITMAP METADATA */
464 size = bitmap_io_size(size, opt_size, offset + ps, 0);
465 if (size == 0)
466 /* bitmap runs in to metadata */
467 return -EINVAL;
468
469 if (doff + mddev->dev_sectors > sboff)
470 /* data runs in to bitmap */
471 return -EINVAL;
472 } else if (rdev->sb_start < rdev->data_offset) {
473 /* METADATA BITMAP DATA */
474 size = bitmap_io_size(size, opt_size, sboff + ps, doff);
475 if (size == 0)
476 /* bitmap runs in to data */
477 return -EINVAL;
478 }
479
480 md_write_metadata(mddev, rdev, sboff + ps, (int)min(size, bitmap_limit),
481 page, 0);
482 return 0;
483}
484
485static void write_sb_page(struct bitmap *bitmap, unsigned long pg_index,
486 struct page *page, bool wait)
487{
488 struct mddev *mddev = bitmap->mddev;
489
490 do {
491 struct md_rdev *rdev = NULL;
492
493 while ((rdev = next_active_rdev(rdev, mddev)) != NULL) {
494 if (__write_sb_page(rdev, bitmap, pg_index, page) < 0) {
495 set_bit(BITMAP_WRITE_ERROR, &bitmap->flags);
496 return;
497 }
498 }
499 } while (wait && md_super_wait(mddev) < 0);
500}
501
502static void md_bitmap_file_kick(struct bitmap *bitmap);
503
504#ifdef CONFIG_MD_BITMAP_FILE
505static void end_bitmap_write(struct bio *bio)
506{
507 struct buffer_head *bh;
508 bool uptodate = bio_endio_bh(bio, &bh);
509 struct bitmap *bitmap = bh->b_private;
510
511 if (!uptodate)
512 set_bit(BITMAP_WRITE_ERROR, &bitmap->flags);
513 if (atomic_dec_and_test(&bitmap->pending_writes))
514 wake_up(&bitmap->write_wait);
515}
516
517static void write_file_page(struct bitmap *bitmap, struct page *page, int wait)
518{
519 struct buffer_head *bh = page_buffers(page);
520
521 while (bh && bh->b_blocknr) {
522 atomic_inc(&bitmap->pending_writes);
523 set_buffer_locked(bh);
524 set_buffer_mapped(bh);
525 bh_submit(bh, REQ_OP_WRITE | REQ_SYNC, end_bitmap_write);
526 bh = bh->b_this_page;
527 }
528
529 if (wait)
530 wait_event(bitmap->write_wait,
531 atomic_read(&bitmap->pending_writes) == 0);
532}
533
534static void free_buffers(struct page *page)
535{
536 struct buffer_head *bh;
537
538 if (!PagePrivate(page))
539 return;
540
541 bh = page_buffers(page);
542 while (bh) {
543 struct buffer_head *next = bh->b_this_page;
544 free_buffer_head(bh);
545 bh = next;
546 }
547 detach_page_private(page);
548 put_page(page);
549}
550
551/* read a page from a file.
552 * We both read the page, and attach buffers to the page to record the
553 * address of each block (using bmap). These addresses will be used
554 * to write the block later, completely bypassing the filesystem.
555 * This usage is similar to how swap files are handled, and allows us
556 * to write to a file with no concerns of memory allocation failing.
557 */
558static int read_file_page(struct file *file, unsigned long index,
559 struct bitmap *bitmap, unsigned long count, struct page *page)
560{
561 int ret = 0;
562 struct inode *inode = file_inode(file);
563 struct buffer_head *bh;
564 sector_t block, blk_cur;
565 unsigned long blocksize = i_blocksize(inode);
566
567 pr_debug("read bitmap file (%dB @ %llu)\n", (int)PAGE_SIZE,
568 (unsigned long long)index << PAGE_SHIFT);
569
570 bh = alloc_page_buffers(page, blocksize);
571 if (!bh) {
572 ret = -ENOMEM;
573 goto out;
574 }
575 attach_page_private(page, bh);
576 blk_cur = index << (PAGE_SHIFT - inode->i_blkbits);
577 while (bh) {
578 block = blk_cur;
579
580 if (count == 0)
581 bh->b_blocknr = 0;
582 else {
583 ret = bmap(inode, &block);
584 if (ret || !block) {
585 ret = -EINVAL;
586 bh->b_blocknr = 0;
587 goto out;
588 }
589
590 bh->b_blocknr = block;
591 bh->b_bdev = inode->i_sb->s_bdev;
592 if (count < blocksize)
593 count = 0;
594 else
595 count -= blocksize;
596
597 bh->b_private = bitmap;
598 atomic_inc(&bitmap->pending_writes);
599 set_buffer_locked(bh);
600 set_buffer_mapped(bh);
601 bh_submit(bh, REQ_OP_READ, end_bitmap_write);
602 }
603 blk_cur++;
604 bh = bh->b_this_page;
605 }
606
607 wait_event(bitmap->write_wait,
608 atomic_read(&bitmap->pending_writes)==0);
609 if (test_bit(BITMAP_WRITE_ERROR, &bitmap->flags))
610 ret = -EIO;
611out:
612 if (ret)
613 pr_err("md: bitmap read error: (%dB @ %llu): %d\n",
614 (int)PAGE_SIZE,
615 (unsigned long long)index << PAGE_SHIFT,
616 ret);
617 return ret;
618}
619#else /* CONFIG_MD_BITMAP_FILE */
620static void write_file_page(struct bitmap *bitmap, struct page *page, int wait)
621{
622}
623static int read_file_page(struct file *file, unsigned long index,
624 struct bitmap *bitmap, unsigned long count, struct page *page)
625{
626 return -EIO;
627}
628static void free_buffers(struct page *page)
629{
630 put_page(page);
631}
632#endif /* CONFIG_MD_BITMAP_FILE */
633
634/*
635 * bitmap file superblock operations
636 */
637
638/*
639 * write out a page to a file
640 */
641static void filemap_write_page(struct bitmap *bitmap, unsigned long pg_index,
642 bool wait)
643{
644 struct bitmap_storage *store = &bitmap->storage;
645 struct page *page = store->filemap[pg_index];
646
647 if (mddev_is_clustered(bitmap->mddev)) {
648 /* go to node bitmap area starting point */
649 pg_index += store->sb_index;
650 }
651
652 if (store->file)
653 write_file_page(bitmap, page, wait);
654 else
655 write_sb_page(bitmap, pg_index, page, wait);
656}
657
658/*
659 * md_bitmap_wait_writes() should be called before writing any bitmap
660 * blocks, to ensure previous writes, particularly from
661 * md_bitmap_daemon_work(), have completed.
662 */
663static void md_bitmap_wait_writes(struct bitmap *bitmap)
664{
665 if (bitmap->storage.file)
666 wait_event(bitmap->write_wait,
667 atomic_read(&bitmap->pending_writes)==0);
668 else
669 /* Note that we ignore the return value. The writes
670 * might have failed, but that would just mean that
671 * some bits which should be cleared haven't been,
672 * which is safe. The relevant bitmap blocks will
673 * probably get written again, but there is no great
674 * loss if they aren't.
675 */
676 md_super_wait(bitmap->mddev);
677}
678
679
680/* update the event counter and sync the superblock to disk */
681static void bitmap_update_sb(void *data)
682{
683 bitmap_super_t *sb;
684 struct bitmap *bitmap = data;
685
686 if (!bitmap || !bitmap->mddev) /* no bitmap for this array */
687 return;
688 if (bitmap->mddev->bitmap_info.external)
689 return;
690 if (!bitmap->storage.sb_page) /* no superblock */
691 return;
692 sb = kmap_local_page(bitmap->storage.sb_page);
693 sb->events = cpu_to_le64(bitmap->mddev->events);
694 if (bitmap->mddev->events < bitmap->events_cleared)
695 /* rocking back to read-only */
696 bitmap->events_cleared = bitmap->mddev->events;
697 sb->events_cleared = cpu_to_le64(bitmap->events_cleared);
698 /*
699 * clear BITMAP_WRITE_ERROR bit to protect against the case that
700 * a bitmap write error occurred but the later writes succeeded.
701 */
702 sb->state = cpu_to_le32(bitmap->flags & ~BIT(BITMAP_WRITE_ERROR));
703 /* Just in case these have been changed via sysfs: */
704 sb->daemon_sleep = cpu_to_le32(bitmap->mddev->bitmap_info.daemon_sleep/HZ);
705 sb->write_behind = cpu_to_le32(bitmap->mddev->bitmap_info.max_write_behind);
706 /* This might have been changed by a reshape */
707 sb->sync_size = cpu_to_le64(bitmap->mddev->resync_max_sectors);
708 sb->chunksize = cpu_to_le32(bitmap->mddev->bitmap_info.chunksize);
709 sb->nodes = cpu_to_le32(bitmap->mddev->bitmap_info.nodes);
710 sb->sectors_reserved = cpu_to_le32(bitmap->mddev->
711 bitmap_info.space);
712 kunmap_local(sb);
713
714 if (bitmap->storage.file)
715 write_file_page(bitmap, bitmap->storage.sb_page, 1);
716 else
717 write_sb_page(bitmap, bitmap->storage.sb_index,
718 bitmap->storage.sb_page, 1);
719}
720
721static void bitmap_print_sb(struct bitmap *bitmap)
722{
723 bitmap_super_t *sb;
724
725 if (!bitmap || !bitmap->storage.sb_page)
726 return;
727 sb = kmap_local_page(bitmap->storage.sb_page);
728 pr_debug("%s: bitmap file superblock:\n", bmname(bitmap));
729 pr_debug(" magic: %08x\n", le32_to_cpu(sb->magic));
730 pr_debug(" version: %u\n", le32_to_cpu(sb->version));
731 pr_debug(" uuid: %08x.%08x.%08x.%08x\n",
732 le32_to_cpu(*(__le32 *)(sb->uuid+0)),
733 le32_to_cpu(*(__le32 *)(sb->uuid+4)),
734 le32_to_cpu(*(__le32 *)(sb->uuid+8)),
735 le32_to_cpu(*(__le32 *)(sb->uuid+12)));
736 pr_debug(" events: %llu\n",
737 (unsigned long long) le64_to_cpu(sb->events));
738 pr_debug("events cleared: %llu\n",
739 (unsigned long long) le64_to_cpu(sb->events_cleared));
740 pr_debug(" state: %08x\n", le32_to_cpu(sb->state));
741 pr_debug(" chunksize: %u B\n", le32_to_cpu(sb->chunksize));
742 pr_debug(" daemon sleep: %us\n", le32_to_cpu(sb->daemon_sleep));
743 pr_debug(" sync size: %llu KB\n",
744 (unsigned long long)le64_to_cpu(sb->sync_size)/2);
745 pr_debug("max write behind: %u\n", le32_to_cpu(sb->write_behind));
746 kunmap_local(sb);
747}
748
749/*
750 * bitmap_new_disk_sb
751 * @bitmap
752 *
753 * This function is somewhat the reverse of bitmap_read_sb. bitmap_read_sb
754 * reads and verifies the on-disk bitmap superblock and populates bitmap_info.
755 * This function verifies 'bitmap_info' and populates the on-disk bitmap
756 * structure, which is to be written to disk.
757 *
758 * Returns: 0 on success, -Exxx on error
759 */
760static int md_bitmap_new_disk_sb(struct bitmap *bitmap)
761{
762 bitmap_super_t *sb;
763 unsigned long chunksize, daemon_sleep, write_behind;
764
765 bitmap->storage.sb_page = alloc_page(GFP_KERNEL | __GFP_ZERO);
766 if (bitmap->storage.sb_page == NULL)
767 return -ENOMEM;
768 bitmap->storage.sb_index = 0;
769
770 sb = kmap_local_page(bitmap->storage.sb_page);
771
772 sb->magic = cpu_to_le32(BITMAP_MAGIC);
773 sb->version = cpu_to_le32(BITMAP_MAJOR_HI);
774
775 chunksize = bitmap->mddev->bitmap_info.chunksize;
776 BUG_ON(!chunksize);
777 if (!is_power_of_2(chunksize)) {
778 kunmap_local(sb);
779 pr_warn("bitmap chunksize not a power of 2\n");
780 return -EINVAL;
781 }
782 sb->chunksize = cpu_to_le32(chunksize);
783
784 daemon_sleep = bitmap->mddev->bitmap_info.daemon_sleep;
785 if (!daemon_sleep || (daemon_sleep > MAX_SCHEDULE_TIMEOUT)) {
786 pr_debug("Choosing daemon_sleep default (5 sec)\n");
787 daemon_sleep = 5 * HZ;
788 }
789 sb->daemon_sleep = cpu_to_le32(daemon_sleep);
790 bitmap->mddev->bitmap_info.daemon_sleep = daemon_sleep;
791
792 /*
793 * FIXME: write_behind for RAID1. If not specified, what
794 * is a good choice? We choose COUNTER_MAX / 2 arbitrarily.
795 */
796 write_behind = bitmap->mddev->bitmap_info.max_write_behind;
797 if (write_behind > COUNTER_MAX / 2)
798 write_behind = COUNTER_MAX / 2;
799 sb->write_behind = cpu_to_le32(write_behind);
800 bitmap->mddev->bitmap_info.max_write_behind = write_behind;
801
802 /* keep the array size field of the bitmap superblock up to date */
803 sb->sync_size = cpu_to_le64(bitmap->mddev->resync_max_sectors);
804
805 memcpy(sb->uuid, bitmap->mddev->uuid, 16);
806
807 set_bit(BITMAP_STALE, &bitmap->flags);
808 sb->state = cpu_to_le32(bitmap->flags);
809 bitmap->events_cleared = bitmap->mddev->events;
810 sb->events_cleared = cpu_to_le64(bitmap->mddev->events);
811 bitmap->mddev->bitmap_info.nodes = 0;
812
813 kunmap_local(sb);
814
815 return 0;
816}
817
818/* read the superblock from the bitmap file and initialize some bitmap fields */
819static int md_bitmap_read_sb(struct bitmap *bitmap)
820{
821 char *reason = NULL;
822 bitmap_super_t *sb;
823 unsigned long chunksize, daemon_sleep, write_behind;
824 unsigned long long events;
825 int nodes = 0;
826 unsigned long sectors_reserved = 0;
827 int err = -EINVAL;
828 struct page *sb_page;
829 loff_t offset = 0;
830
831 if (!bitmap->storage.file && !bitmap->mddev->bitmap_info.offset) {
832 chunksize = 128 * 1024 * 1024;
833 daemon_sleep = 5 * HZ;
834 write_behind = 0;
835 set_bit(BITMAP_STALE, &bitmap->flags);
836 err = 0;
837 goto out_no_sb;
838 }
839 /* page 0 is the superblock, read it... */
840 sb_page = alloc_page(GFP_KERNEL);
841 if (!sb_page)
842 return -ENOMEM;
843 bitmap->storage.sb_page = sb_page;
844
845re_read:
846 /* If cluster_slot is set, the cluster is setup */
847 if (bitmap->cluster_slot >= 0) {
848 sector_t bm_blocks = bitmap->mddev->resync_max_sectors;
849
850 bm_blocks = DIV_ROUND_UP_SECTOR_T(bm_blocks,
851 (bitmap->mddev->bitmap_info.chunksize >> 9));
852 /* bits to bytes */
853 bm_blocks = ((bm_blocks+7) >> 3) + sizeof(bitmap_super_t);
854 /* to 4k blocks */
855 bm_blocks = DIV_ROUND_UP_SECTOR_T(bm_blocks, 4096);
856 offset = bitmap->cluster_slot * (bm_blocks << 3);
857 pr_debug("%s:%d bm slot: %d offset: %llu\n", __func__, __LINE__,
858 bitmap->cluster_slot, offset);
859 }
860
861 if (bitmap->storage.file) {
862 loff_t isize = i_size_read(bitmap->storage.file->f_mapping->host);
863 int bytes = isize > PAGE_SIZE ? PAGE_SIZE : isize;
864
865 err = read_file_page(bitmap->storage.file, 0,
866 bitmap, bytes, sb_page);
867 } else {
868 err = read_sb_page(bitmap->mddev, offset, sb_page, 0,
869 sizeof(bitmap_super_t));
870 }
871 if (err)
872 return err;
873
874 err = -EINVAL;
875 sb = kmap_local_page(sb_page);
876
877 chunksize = le32_to_cpu(sb->chunksize);
878 daemon_sleep = le32_to_cpu(sb->daemon_sleep) * HZ;
879 write_behind = le32_to_cpu(sb->write_behind);
880 sectors_reserved = le32_to_cpu(sb->sectors_reserved);
881
882 /* verify that the bitmap-specific fields are valid */
883 if (sb->magic != cpu_to_le32(BITMAP_MAGIC))
884 reason = "bad magic";
885 else if (le32_to_cpu(sb->version) < BITMAP_MAJOR_LO ||
886 le32_to_cpu(sb->version) > BITMAP_MAJOR_CLUSTERED)
887 reason = "unrecognized superblock version";
888 else if (chunksize < 512)
889 reason = "bitmap chunksize too small";
890 else if (!is_power_of_2(chunksize))
891 reason = "bitmap chunksize not a power of 2";
892 else if (daemon_sleep < 1 || daemon_sleep > MAX_SCHEDULE_TIMEOUT)
893 reason = "daemon sleep period out of range";
894 else if (write_behind > COUNTER_MAX)
895 reason = "write-behind limit out of range (0 - 16383)";
896 if (reason) {
897 pr_warn("%s: invalid bitmap file superblock: %s\n",
898 bmname(bitmap), reason);
899 goto out;
900 }
901
902 /*
903 * Setup nodes/clustername only if bitmap version is
904 * cluster-compatible
905 */
906 if (sb->version == cpu_to_le32(BITMAP_MAJOR_CLUSTERED)) {
907 nodes = le32_to_cpu(sb->nodes);
908 strscpy(bitmap->mddev->bitmap_info.cluster_name,
909 sb->cluster_name, 64);
910 }
911
912 /* keep the array size field of the bitmap superblock up to date */
913 sb->sync_size = cpu_to_le64(bitmap->mddev->resync_max_sectors);
914
915 if (bitmap->mddev->persistent) {
916 /*
917 * We have a persistent array superblock, so compare the
918 * bitmap's UUID and event counter to the mddev's
919 */
920 if (memcmp(sb->uuid, bitmap->mddev->uuid, 16)) {
921 pr_warn("%s: bitmap superblock UUID mismatch\n",
922 bmname(bitmap));
923 goto out;
924 }
925 events = le64_to_cpu(sb->events);
926 if (!nodes && (events < bitmap->mddev->events)) {
927 pr_warn("%s: bitmap file is out of date (%llu < %llu) -- forcing full recovery\n",
928 bmname(bitmap), events,
929 (unsigned long long) bitmap->mddev->events);
930 set_bit(BITMAP_STALE, &bitmap->flags);
931 }
932 }
933
934 /* assign fields using values from superblock */
935 bitmap->flags |= le32_to_cpu(sb->state);
936 if (le32_to_cpu(sb->version) == BITMAP_MAJOR_HOSTENDIAN)
937 set_bit(BITMAP_HOSTENDIAN, &bitmap->flags);
938 bitmap->events_cleared = le64_to_cpu(sb->events_cleared);
939 err = 0;
940
941out:
942 kunmap_local(sb);
943 if (err == 0 && nodes && (bitmap->cluster_slot < 0)) {
944 /* Assigning chunksize is required for "re_read" */
945 bitmap->mddev->bitmap_info.chunksize = chunksize;
946 err = md_setup_cluster(bitmap->mddev, nodes);
947 if (err) {
948 pr_warn("%s: Could not setup cluster service (%d)\n",
949 bmname(bitmap), err);
950 goto out_no_sb;
951 }
952 bitmap->cluster_slot = bitmap->mddev->cluster_ops->slot_number(bitmap->mddev);
953 goto re_read;
954 }
955
956out_no_sb:
957 if (err == 0) {
958 if (test_bit(BITMAP_STALE, &bitmap->flags))
959 bitmap->events_cleared = bitmap->mddev->events;
960 bitmap->mddev->bitmap_info.chunksize = chunksize;
961 bitmap->mddev->bitmap_info.daemon_sleep = daemon_sleep;
962 bitmap->mddev->bitmap_info.max_write_behind = write_behind;
963 bitmap->mddev->bitmap_info.nodes = nodes;
964 if (bitmap->mddev->bitmap_info.space == 0 ||
965 bitmap->mddev->bitmap_info.space > sectors_reserved)
966 bitmap->mddev->bitmap_info.space = sectors_reserved;
967 } else {
968 bitmap_print_sb(bitmap);
969 if (bitmap->cluster_slot < 0)
970 md_cluster_stop(bitmap->mddev);
971 }
972 return err;
973}
974
975/*
976 * general bitmap file operations
977 */
978
979/*
980 * on-disk bitmap:
981 *
982 * Use one bit per "chunk" (block set). We do the disk I/O on the bitmap
983 * file a page at a time. There's a superblock at the start of the file.
984 */
985/* calculate the index of the page that contains this bit */
986static inline unsigned long file_page_index(struct bitmap_storage *store,
987 unsigned long chunk)
988{
989 if (store->sb_page)
990 chunk += sizeof(bitmap_super_t) << 3;
991 return chunk >> PAGE_BIT_SHIFT;
992}
993
994/* calculate the (bit) offset of this bit within a page */
995static inline unsigned long file_page_offset(struct bitmap_storage *store,
996 unsigned long chunk)
997{
998 if (store->sb_page)
999 chunk += sizeof(bitmap_super_t) << 3;
1000 return chunk & (PAGE_BITS - 1);
1001}
1002
1003/*
1004 * return a pointer to the page in the filemap that contains the given bit
1005 *
1006 */
1007static inline struct page *filemap_get_page(struct bitmap_storage *store,
1008 unsigned long chunk)
1009{
1010 if (file_page_index(store, chunk) >= store->file_pages)
1011 return NULL;
1012 return store->filemap[file_page_index(store, chunk)];
1013}
1014
1015static int md_bitmap_storage_alloc(struct bitmap_storage *store,
1016 unsigned long chunks, int with_super,
1017 int slot_number)
1018{
1019 int pnum, offset = 0;
1020 unsigned long num_pages;
1021 unsigned long bytes;
1022
1023 bytes = DIV_ROUND_UP(chunks, 8);
1024 if (with_super)
1025 bytes += sizeof(bitmap_super_t);
1026
1027 num_pages = DIV_ROUND_UP(bytes, PAGE_SIZE);
1028 offset = slot_number * num_pages;
1029
1030 store->filemap = kmalloc_objs(struct page *, num_pages);
1031 if (!store->filemap)
1032 return -ENOMEM;
1033
1034 if (with_super && !store->sb_page) {
1035 store->sb_page = alloc_page(GFP_KERNEL|__GFP_ZERO);
1036 if (store->sb_page == NULL)
1037 return -ENOMEM;
1038 }
1039
1040 pnum = 0;
1041 if (store->sb_page) {
1042 store->filemap[0] = store->sb_page;
1043 pnum = 1;
1044 store->sb_index = offset;
1045 }
1046
1047 for ( ; pnum < num_pages; pnum++) {
1048 store->filemap[pnum] = alloc_page(GFP_KERNEL|__GFP_ZERO);
1049 if (!store->filemap[pnum]) {
1050 store->file_pages = pnum;
1051 return -ENOMEM;
1052 }
1053 }
1054 store->file_pages = pnum;
1055
1056 /* We need 4 bits per page, rounded up to a multiple
1057 * of sizeof(unsigned long) */
1058 store->filemap_attr = kzalloc(
1059 roundup(DIV_ROUND_UP(num_pages*4, 8), sizeof(unsigned long)),
1060 GFP_KERNEL);
1061 if (!store->filemap_attr)
1062 return -ENOMEM;
1063
1064 store->bytes = bytes;
1065
1066 return 0;
1067}
1068
1069static void md_bitmap_file_unmap(struct bitmap_storage *store)
1070{
1071 struct file *file = store->file;
1072 struct page *sb_page = store->sb_page;
1073 struct page **map = store->filemap;
1074 int pages = store->file_pages;
1075
1076 while (pages--)
1077 if (map[pages] != sb_page) /* 0 is sb_page, release it below */
1078 free_buffers(map[pages]);
1079 kfree(map);
1080 kfree(store->filemap_attr);
1081
1082 if (sb_page)
1083 free_buffers(sb_page);
1084
1085 if (file) {
1086 struct inode *inode = file_inode(file);
1087 invalidate_mapping_pages(inode->i_mapping, 0, -1);
1088 fput(file);
1089 }
1090}
1091
1092/*
1093 * bitmap_file_kick - if an error occurs while manipulating the bitmap file
1094 * then it is no longer reliable, so we stop using it and we mark the file
1095 * as failed in the superblock
1096 */
1097static void md_bitmap_file_kick(struct bitmap *bitmap)
1098{
1099 if (!test_and_set_bit(BITMAP_STALE, &bitmap->flags)) {
1100 bitmap_update_sb(bitmap);
1101
1102 if (bitmap->storage.file) {
1103 pr_warn("%s: kicking failed bitmap file %pD4 from array!\n",
1104 bmname(bitmap), bitmap->storage.file);
1105
1106 } else
1107 pr_warn("%s: disabling internal bitmap due to errors\n",
1108 bmname(bitmap));
1109 }
1110}
1111
1112enum bitmap_page_attr {
1113 BITMAP_PAGE_DIRTY = 0, /* there are set bits that need to be synced */
1114 BITMAP_PAGE_PENDING = 1, /* there are bits that are being cleaned.
1115 * i.e. counter is 1 or 2. */
1116 BITMAP_PAGE_NEEDWRITE = 2, /* there are cleared bits that need to be synced */
1117};
1118
1119static inline void set_page_attr(struct bitmap *bitmap, int pnum,
1120 enum bitmap_page_attr attr)
1121{
1122 set_bit((pnum<<2) + attr, bitmap->storage.filemap_attr);
1123}
1124
1125static inline void clear_page_attr(struct bitmap *bitmap, int pnum,
1126 enum bitmap_page_attr attr)
1127{
1128 clear_bit((pnum<<2) + attr, bitmap->storage.filemap_attr);
1129}
1130
1131static inline int test_page_attr(struct bitmap *bitmap, int pnum,
1132 enum bitmap_page_attr attr)
1133{
1134 return test_bit((pnum<<2) + attr, bitmap->storage.filemap_attr);
1135}
1136
1137static inline int test_and_clear_page_attr(struct bitmap *bitmap, int pnum,
1138 enum bitmap_page_attr attr)
1139{
1140 return test_and_clear_bit((pnum<<2) + attr,
1141 bitmap->storage.filemap_attr);
1142}
1143/*
1144 * bitmap_file_set_bit -- called before performing a write to the md device
1145 * to set (and eventually sync) a particular bit in the bitmap file
1146 *
1147 * we set the bit immediately, then we record the page number so that
1148 * when an unplug occurs, we can flush the dirty pages out to disk
1149 */
1150static void md_bitmap_file_set_bit(struct bitmap *bitmap, sector_t block)
1151{
1152 unsigned long bit;
1153 struct page *page;
1154 void *kaddr;
1155 unsigned long chunk = block >> bitmap->counts.chunkshift;
1156 struct bitmap_storage *store = &bitmap->storage;
1157 unsigned long index = file_page_index(store, chunk);
1158 unsigned long node_offset = 0;
1159
1160 index += store->sb_index;
1161 if (mddev_is_clustered(bitmap->mddev))
1162 node_offset = bitmap->cluster_slot * store->file_pages;
1163
1164 page = filemap_get_page(&bitmap->storage, chunk);
1165 if (!page)
1166 return;
1167 bit = file_page_offset(&bitmap->storage, chunk);
1168
1169 /* set the bit */
1170 kaddr = kmap_local_page(page);
1171 if (test_bit(BITMAP_HOSTENDIAN, &bitmap->flags))
1172 set_bit(bit, kaddr);
1173 else
1174 set_bit_le(bit, kaddr);
1175 kunmap_local(kaddr);
1176 pr_debug("set file bit %lu page %lu\n", bit, index);
1177 /* record page number so it gets flushed to disk when unplug occurs */
1178 set_page_attr(bitmap, index - node_offset, BITMAP_PAGE_DIRTY);
1179}
1180
1181static void md_bitmap_file_clear_bit(struct bitmap *bitmap, sector_t block)
1182{
1183 unsigned long bit;
1184 struct page *page;
1185 void *paddr;
1186 unsigned long chunk = block >> bitmap->counts.chunkshift;
1187 struct bitmap_storage *store = &bitmap->storage;
1188 unsigned long index = file_page_index(store, chunk);
1189 unsigned long node_offset = 0;
1190
1191 index += store->sb_index;
1192 if (mddev_is_clustered(bitmap->mddev))
1193 node_offset = bitmap->cluster_slot * store->file_pages;
1194
1195 page = filemap_get_page(&bitmap->storage, chunk);
1196 if (!page)
1197 return;
1198 bit = file_page_offset(&bitmap->storage, chunk);
1199 paddr = kmap_local_page(page);
1200 if (test_bit(BITMAP_HOSTENDIAN, &bitmap->flags))
1201 clear_bit(bit, paddr);
1202 else
1203 clear_bit_le(bit, paddr);
1204 kunmap_local(paddr);
1205 if (!test_page_attr(bitmap, index - node_offset, BITMAP_PAGE_NEEDWRITE)) {
1206 set_page_attr(bitmap, index - node_offset, BITMAP_PAGE_PENDING);
1207 bitmap->allclean = 0;
1208 }
1209}
1210
1211static int md_bitmap_file_test_bit(struct bitmap *bitmap, sector_t block)
1212{
1213 unsigned long bit;
1214 struct page *page;
1215 void *paddr;
1216 unsigned long chunk = block >> bitmap->counts.chunkshift;
1217 int set = 0;
1218
1219 page = filemap_get_page(&bitmap->storage, chunk);
1220 if (!page)
1221 return -EINVAL;
1222 bit = file_page_offset(&bitmap->storage, chunk);
1223 paddr = kmap_local_page(page);
1224 if (test_bit(BITMAP_HOSTENDIAN, &bitmap->flags))
1225 set = test_bit(bit, paddr);
1226 else
1227 set = test_bit_le(bit, paddr);
1228 kunmap_local(paddr);
1229 return set;
1230}
1231
1232/* this gets called when the md device is ready to unplug its underlying
1233 * (slave) device queues -- before we let any writes go down, we need to
1234 * sync the dirty pages of the bitmap file to disk */
1235static void __bitmap_unplug(struct bitmap *bitmap)
1236{
1237 unsigned long i;
1238 int dirty, need_write;
1239 int writing = 0;
1240
1241 if (!bitmap_enabled(bitmap, true))
1242 return;
1243
1244 /* look at each page to see if there are any set bits that need to be
1245 * flushed out to disk */
1246 for (i = 0; i < bitmap->storage.file_pages; i++) {
1247 dirty = test_and_clear_page_attr(bitmap, i, BITMAP_PAGE_DIRTY);
1248 need_write = test_and_clear_page_attr(bitmap, i,
1249 BITMAP_PAGE_NEEDWRITE);
1250 if (dirty || need_write) {
1251 if (!writing) {
1252 md_bitmap_wait_writes(bitmap);
1253 mddev_add_trace_msg(bitmap->mddev,
1254 "md bitmap_unplug");
1255 }
1256 clear_page_attr(bitmap, i, BITMAP_PAGE_PENDING);
1257 filemap_write_page(bitmap, i, false);
1258 writing = 1;
1259 }
1260 }
1261 if (writing)
1262 md_bitmap_wait_writes(bitmap);
1263
1264 if (test_bit(BITMAP_WRITE_ERROR, &bitmap->flags))
1265 md_bitmap_file_kick(bitmap);
1266}
1267
1268struct bitmap_unplug_work {
1269 struct work_struct work;
1270 struct bitmap *bitmap;
1271 struct completion *done;
1272};
1273
1274static void md_bitmap_unplug_fn(struct work_struct *work)
1275{
1276 struct bitmap_unplug_work *unplug_work =
1277 container_of(work, struct bitmap_unplug_work, work);
1278
1279 __bitmap_unplug(unplug_work->bitmap);
1280 complete(unplug_work->done);
1281}
1282
1283static void bitmap_unplug_async(struct bitmap *bitmap)
1284{
1285 DECLARE_COMPLETION_ONSTACK(done);
1286 struct bitmap_unplug_work unplug_work;
1287
1288 INIT_WORK_ONSTACK(&unplug_work.work, md_bitmap_unplug_fn);
1289 unplug_work.bitmap = bitmap;
1290 unplug_work.done = &done;
1291
1292 queue_work(md_bitmap_wq, &unplug_work.work);
1293 wait_for_completion(&done);
1294 destroy_work_on_stack(&unplug_work.work);
1295}
1296
1297static void bitmap_unplug(struct mddev *mddev, bool sync)
1298{
1299 struct bitmap *bitmap = mddev->bitmap;
1300
1301 if (!bitmap)
1302 return;
1303
1304 if (sync)
1305 __bitmap_unplug(bitmap);
1306 else
1307 bitmap_unplug_async(bitmap);
1308}
1309
1310static void md_bitmap_set_memory_bits(struct bitmap *bitmap, sector_t offset, int needed);
1311
1312/*
1313 * Initialize the in-memory bitmap from the on-disk bitmap and set up the memory
1314 * mapping of the bitmap file.
1315 *
1316 * Special case: If there's no bitmap file, or if the bitmap file had been
1317 * previously kicked from the array, we mark all the bits as 1's in order to
1318 * cause a full resync.
1319 *
1320 * We ignore all bits for sectors that end earlier than 'start'.
1321 * This is used when reading an out-of-date bitmap.
1322 */
1323static int md_bitmap_init_from_disk(struct bitmap *bitmap, sector_t start)
1324{
1325 bool outofdate = test_bit(BITMAP_STALE, &bitmap->flags);
1326 struct mddev *mddev = bitmap->mddev;
1327 unsigned long chunks = bitmap->counts.chunks;
1328 struct bitmap_storage *store = &bitmap->storage;
1329 struct file *file = store->file;
1330 unsigned long node_offset = 0;
1331 unsigned long bit_cnt = 0;
1332 unsigned long i;
1333 int ret;
1334
1335 if (!file && !mddev->bitmap_info.offset) {
1336 /* No permanent bitmap - fill with '1s'. */
1337 store->filemap = NULL;
1338 store->file_pages = 0;
1339 for (i = 0; i < chunks ; i++) {
1340 /* if the disk bit is set, set the memory bit */
1341 int needed = ((sector_t)(i+1) << (bitmap->counts.chunkshift)
1342 >= start);
1343 md_bitmap_set_memory_bits(bitmap,
1344 (sector_t)i << bitmap->counts.chunkshift,
1345 needed);
1346 }
1347 return 0;
1348 }
1349
1350 if (file && i_size_read(file->f_mapping->host) < store->bytes) {
1351 pr_warn("%s: bitmap file too short %lu < %lu\n",
1352 bmname(bitmap),
1353 (unsigned long) i_size_read(file->f_mapping->host),
1354 store->bytes);
1355 ret = -ENOSPC;
1356 goto err;
1357 }
1358
1359 if (mddev_is_clustered(mddev))
1360 node_offset = bitmap->cluster_slot * (DIV_ROUND_UP(store->bytes, PAGE_SIZE));
1361
1362 for (i = 0; i < store->file_pages; i++) {
1363 struct page *page = store->filemap[i];
1364 int count;
1365
1366 /* unmap the old page, we're done with it */
1367 if (i == store->file_pages - 1)
1368 count = store->bytes - i * PAGE_SIZE;
1369 else
1370 count = PAGE_SIZE;
1371
1372 if (file)
1373 ret = read_file_page(file, i, bitmap, count, page);
1374 else
1375 ret = read_sb_page(mddev, 0, page, i + node_offset,
1376 count);
1377 if (ret)
1378 goto err;
1379 }
1380
1381 if (outofdate) {
1382 pr_warn("%s: bitmap file is out of date, doing full recovery\n",
1383 bmname(bitmap));
1384
1385 for (i = 0; i < store->file_pages; i++) {
1386 struct page *page = store->filemap[i];
1387 unsigned long offset = 0;
1388 void *paddr;
1389
1390 if (i == 0 && !mddev->bitmap_info.external)
1391 offset = sizeof(bitmap_super_t);
1392
1393 /*
1394 * If the bitmap is out of date, dirty the whole page
1395 * and write it out
1396 */
1397 paddr = kmap_local_page(page);
1398 memset(paddr + offset, 0xff, PAGE_SIZE - offset);
1399 kunmap_local(paddr);
1400
1401 filemap_write_page(bitmap, i, true);
1402 if (test_bit(BITMAP_WRITE_ERROR, &bitmap->flags)) {
1403 ret = -EIO;
1404 goto err;
1405 }
1406 }
1407 }
1408
1409 for (i = 0; i < chunks; i++) {
1410 struct page *page = filemap_get_page(&bitmap->storage, i);
1411 unsigned long bit = file_page_offset(&bitmap->storage, i);
1412 void *paddr;
1413 bool was_set;
1414
1415 paddr = kmap_local_page(page);
1416 if (test_bit(BITMAP_HOSTENDIAN, &bitmap->flags))
1417 was_set = test_bit(bit, paddr);
1418 else
1419 was_set = test_bit_le(bit, paddr);
1420 kunmap_local(paddr);
1421
1422 if (was_set) {
1423 /* if the disk bit is set, set the memory bit */
1424 int needed = ((sector_t)(i+1) << bitmap->counts.chunkshift
1425 >= start);
1426 md_bitmap_set_memory_bits(bitmap,
1427 (sector_t)i << bitmap->counts.chunkshift,
1428 needed);
1429 bit_cnt++;
1430 }
1431 }
1432
1433 pr_debug("%s: bitmap initialized from disk: read %lu pages, set %lu of %lu bits\n",
1434 bmname(bitmap), store->file_pages,
1435 bit_cnt, chunks);
1436
1437 return 0;
1438
1439 err:
1440 pr_warn("%s: bitmap initialisation failed: %d\n",
1441 bmname(bitmap), ret);
1442 return ret;
1443}
1444
1445/* just flag bitmap pages as needing to be written. */
1446static void bitmap_write_all(struct mddev *mddev)
1447{
1448 int i;
1449 struct bitmap *bitmap = mddev->bitmap;
1450
1451 if (!bitmap || !bitmap->storage.filemap)
1452 return;
1453
1454 /* Only one copy, so nothing needed */
1455 if (bitmap->storage.file)
1456 return;
1457
1458 for (i = 0; i < bitmap->storage.file_pages; i++)
1459 set_page_attr(bitmap, i, BITMAP_PAGE_NEEDWRITE);
1460 bitmap->allclean = 0;
1461}
1462
1463static void md_bitmap_count_page(struct bitmap_counts *bitmap,
1464 sector_t offset, int inc)
1465{
1466 sector_t chunk = offset >> bitmap->chunkshift;
1467 unsigned long page = chunk >> PAGE_COUNTER_SHIFT;
1468 bitmap->bp[page].count += inc;
1469 md_bitmap_checkfree(bitmap, page);
1470}
1471
1472static void md_bitmap_set_pending(struct bitmap_counts *bitmap, sector_t offset)
1473{
1474 sector_t chunk = offset >> bitmap->chunkshift;
1475 unsigned long page = chunk >> PAGE_COUNTER_SHIFT;
1476 struct bitmap_page *bp = &bitmap->bp[page];
1477
1478 if (!bp->pending)
1479 bp->pending = 1;
1480}
1481
1482static bitmap_counter_t *md_bitmap_get_counter(struct bitmap_counts *bitmap,
1483 sector_t offset, sector_t *blocks,
1484 int create);
1485
1486static void mddev_set_timeout(struct mddev *mddev, unsigned long timeout,
1487 bool force)
1488{
1489 struct md_thread *thread;
1490
1491 rcu_read_lock();
1492 thread = rcu_dereference(mddev->thread);
1493
1494 if (!thread)
1495 goto out;
1496
1497 if (force || thread->timeout < MAX_SCHEDULE_TIMEOUT)
1498 thread->timeout = timeout;
1499
1500out:
1501 rcu_read_unlock();
1502}
1503
1504/*
1505 * bitmap daemon -- periodically wakes up to clean bits and flush pages
1506 * out to disk
1507 */
1508static void bitmap_daemon_work(struct mddev *mddev)
1509{
1510 struct bitmap *bitmap;
1511 unsigned long j;
1512 unsigned long nextpage;
1513 sector_t blocks;
1514 struct bitmap_counts *counts;
1515
1516 /* Use a mutex to guard daemon_work against
1517 * bitmap_destroy.
1518 */
1519 mutex_lock(&mddev->bitmap_info.mutex);
1520 bitmap = mddev->bitmap;
1521 if (bitmap == NULL) {
1522 mutex_unlock(&mddev->bitmap_info.mutex);
1523 return;
1524 }
1525 if (time_before(jiffies, bitmap->daemon_lastrun
1526 + mddev->bitmap_info.daemon_sleep))
1527 goto done;
1528
1529 bitmap->daemon_lastrun = jiffies;
1530 if (bitmap->allclean) {
1531 mddev_set_timeout(mddev, MAX_SCHEDULE_TIMEOUT, true);
1532 goto done;
1533 }
1534 bitmap->allclean = 1;
1535
1536 mddev_add_trace_msg(bitmap->mddev, "md bitmap_daemon_work");
1537
1538 /* Any file-page which is PENDING now needs to be written.
1539 * So set NEEDWRITE now, then after we make any last-minute changes
1540 * we will write it.
1541 */
1542 for (j = 0; j < bitmap->storage.file_pages; j++)
1543 if (test_and_clear_page_attr(bitmap, j,
1544 BITMAP_PAGE_PENDING))
1545 set_page_attr(bitmap, j,
1546 BITMAP_PAGE_NEEDWRITE);
1547
1548 if (bitmap->need_sync &&
1549 mddev->bitmap_info.external == 0) {
1550 /* Arrange for superblock update as well as
1551 * other changes */
1552 bitmap_super_t *sb;
1553 bitmap->need_sync = 0;
1554 if (bitmap->storage.filemap) {
1555 sb = kmap_local_page(bitmap->storage.sb_page);
1556 sb->events_cleared =
1557 cpu_to_le64(bitmap->events_cleared);
1558 kunmap_local(sb);
1559 set_page_attr(bitmap, 0,
1560 BITMAP_PAGE_NEEDWRITE);
1561 }
1562 }
1563 /* Now look at the bitmap counters and if any are '2' or '1',
1564 * decrement and handle accordingly.
1565 */
1566 counts = &bitmap->counts;
1567 spin_lock_irq(&counts->lock);
1568 nextpage = 0;
1569 for (j = 0; j < counts->chunks; j++) {
1570 bitmap_counter_t *bmc;
1571 sector_t block = (sector_t)j << counts->chunkshift;
1572
1573 if (j == nextpage) {
1574 nextpage += PAGE_COUNTER_RATIO;
1575 if (!counts->bp[j >> PAGE_COUNTER_SHIFT].pending) {
1576 j |= PAGE_COUNTER_MASK;
1577 continue;
1578 }
1579 counts->bp[j >> PAGE_COUNTER_SHIFT].pending = 0;
1580 }
1581
1582 bmc = md_bitmap_get_counter(counts, block, &blocks, 0);
1583 if (!bmc) {
1584 j |= PAGE_COUNTER_MASK;
1585 continue;
1586 }
1587 if (*bmc == 1 && !bitmap->need_sync) {
1588 /* We can clear the bit */
1589 *bmc = 0;
1590 md_bitmap_count_page(counts, block, -1);
1591 md_bitmap_file_clear_bit(bitmap, block);
1592 } else if (*bmc && *bmc <= 2) {
1593 *bmc = 1;
1594 md_bitmap_set_pending(counts, block);
1595 bitmap->allclean = 0;
1596 }
1597 }
1598 spin_unlock_irq(&counts->lock);
1599
1600 md_bitmap_wait_writes(bitmap);
1601 /* Now start writeout on any page in NEEDWRITE that isn't DIRTY.
1602 * DIRTY pages need to be written by bitmap_unplug so it can wait
1603 * for them.
1604 * If we find any DIRTY page we stop there and let bitmap_unplug
1605 * handle all the rest. This is important in the case where
1606 * the first blocking holds the superblock and it has been updated.
1607 * We mustn't write any other blocks before the superblock.
1608 */
1609 for (j = 0;
1610 j < bitmap->storage.file_pages
1611 && !test_bit(BITMAP_STALE, &bitmap->flags);
1612 j++) {
1613 if (test_page_attr(bitmap, j,
1614 BITMAP_PAGE_DIRTY))
1615 /* bitmap_unplug will handle the rest */
1616 break;
1617 if (bitmap->storage.filemap &&
1618 test_and_clear_page_attr(bitmap, j,
1619 BITMAP_PAGE_NEEDWRITE))
1620 filemap_write_page(bitmap, j, false);
1621 }
1622
1623 done:
1624 if (bitmap->allclean == 0)
1625 mddev_set_timeout(mddev, mddev->bitmap_info.daemon_sleep, true);
1626 mutex_unlock(&mddev->bitmap_info.mutex);
1627}
1628
1629static bitmap_counter_t *md_bitmap_get_counter(struct bitmap_counts *bitmap,
1630 sector_t offset, sector_t *blocks,
1631 int create)
1632__releases(bitmap->lock)
1633__acquires(bitmap->lock)
1634{
1635 /* If 'create', we might release the lock and reclaim it.
1636 * The lock must have been taken with interrupts enabled.
1637 * If !create, we don't release the lock.
1638 */
1639 sector_t chunk = offset >> bitmap->chunkshift;
1640 unsigned long page = chunk >> PAGE_COUNTER_SHIFT;
1641 unsigned long pageoff = (chunk & PAGE_COUNTER_MASK) << COUNTER_BYTE_SHIFT;
1642 sector_t csize = ((sector_t)1) << bitmap->chunkshift;
1643 int err;
1644
1645 if (page >= bitmap->pages) {
1646 /*
1647 * This can happen if bitmap_start_sync goes beyond
1648 * End-of-device while looking for a whole page or
1649 * user set a huge number to sysfs bitmap_set_bits.
1650 */
1651 *blocks = csize - (offset & (csize - 1));
1652 return NULL;
1653 }
1654 err = md_bitmap_checkpage(bitmap, page, create, 0);
1655
1656 if (bitmap->bp[page].hijacked ||
1657 bitmap->bp[page].map == NULL)
1658 csize = ((sector_t)1) << (bitmap->chunkshift +
1659 PAGE_COUNTER_SHIFT);
1660
1661 *blocks = csize - (offset & (csize - 1));
1662
1663 if (err < 0)
1664 return NULL;
1665
1666 /* now locked ... */
1667
1668 if (bitmap->bp[page].hijacked) { /* hijacked pointer */
1669 /* should we use the first or second counter field
1670 * of the hijacked pointer? */
1671 int hi = (pageoff > PAGE_COUNTER_MASK);
1672 return &((bitmap_counter_t *)
1673 &bitmap->bp[page].map)[hi];
1674 } else /* page is allocated */
1675 return (bitmap_counter_t *)
1676 &(bitmap->bp[page].map[pageoff]);
1677}
1678
1679static void bitmap_start_write(struct mddev *mddev, sector_t offset,
1680 unsigned long sectors)
1681{
1682 struct bitmap *bitmap = mddev->bitmap;
1683
1684 if (!bitmap)
1685 return;
1686
1687 while (sectors) {
1688 sector_t blocks;
1689 bitmap_counter_t *bmc;
1690
1691 spin_lock_irq(&bitmap->counts.lock);
1692 bmc = md_bitmap_get_counter(&bitmap->counts, offset, &blocks, 1);
1693 if (!bmc) {
1694 spin_unlock_irq(&bitmap->counts.lock);
1695 return;
1696 }
1697
1698 if (unlikely(COUNTER(*bmc) == COUNTER_MAX)) {
1699 DEFINE_WAIT(__wait);
1700 /* note that it is safe to do the prepare_to_wait
1701 * after the test as long as we do it before dropping
1702 * the spinlock.
1703 */
1704 prepare_to_wait(&bitmap->overflow_wait, &__wait,
1705 TASK_UNINTERRUPTIBLE);
1706 spin_unlock_irq(&bitmap->counts.lock);
1707 schedule();
1708 finish_wait(&bitmap->overflow_wait, &__wait);
1709 continue;
1710 }
1711
1712 switch (*bmc) {
1713 case 0:
1714 md_bitmap_file_set_bit(bitmap, offset);
1715 md_bitmap_count_page(&bitmap->counts, offset, 1);
1716 fallthrough;
1717 case 1:
1718 *bmc = 2;
1719 }
1720
1721 (*bmc)++;
1722
1723 spin_unlock_irq(&bitmap->counts.lock);
1724
1725 offset += blocks;
1726 if (sectors > blocks)
1727 sectors -= blocks;
1728 else
1729 sectors = 0;
1730 }
1731}
1732
1733static void bitmap_prepare_range(struct mddev *mddev, sector_t *offset,
1734 unsigned long *sectors, bool discard)
1735{
1736 if (mddev->pers->bitmap_sector)
1737 mddev->pers->bitmap_sector(mddev, offset, sectors);
1738}
1739
1740static void bitmap_end_write(struct mddev *mddev, sector_t offset,
1741 unsigned long sectors)
1742{
1743 struct bitmap *bitmap = mddev->bitmap;
1744
1745 if (!bitmap)
1746 return;
1747
1748 while (sectors) {
1749 sector_t blocks;
1750 unsigned long flags;
1751 bitmap_counter_t *bmc;
1752
1753 spin_lock_irqsave(&bitmap->counts.lock, flags);
1754 bmc = md_bitmap_get_counter(&bitmap->counts, offset, &blocks, 0);
1755 if (!bmc) {
1756 spin_unlock_irqrestore(&bitmap->counts.lock, flags);
1757 return;
1758 }
1759
1760 if (!bitmap->mddev->degraded) {
1761 if (bitmap->events_cleared < bitmap->mddev->events) {
1762 bitmap->events_cleared = bitmap->mddev->events;
1763 bitmap->need_sync = 1;
1764 sysfs_notify_dirent_safe(
1765 bitmap->sysfs_can_clear);
1766 }
1767 } else if (!NEEDED(*bmc)) {
1768 *bmc |= NEEDED_MASK;
1769 }
1770
1771 if (COUNTER(*bmc) == COUNTER_MAX)
1772 wake_up(&bitmap->overflow_wait);
1773
1774 (*bmc)--;
1775 if (*bmc <= 2) {
1776 md_bitmap_set_pending(&bitmap->counts, offset);
1777 bitmap->allclean = 0;
1778 }
1779 spin_unlock_irqrestore(&bitmap->counts.lock, flags);
1780 offset += blocks;
1781 if (sectors > blocks)
1782 sectors -= blocks;
1783 else
1784 sectors = 0;
1785 }
1786}
1787
1788static bool __bitmap_start_sync(struct bitmap *bitmap, sector_t offset,
1789 sector_t *blocks, bool degraded)
1790{
1791 bitmap_counter_t *bmc;
1792 bool rv = false;
1793
1794 spin_lock_irq(&bitmap->counts.lock);
1795 bmc = md_bitmap_get_counter(&bitmap->counts, offset, blocks, 0);
1796 if (bmc) {
1797 /* locked */
1798 if (RESYNC(*bmc)) {
1799 rv = true;
1800 } else if (NEEDED(*bmc)) {
1801 rv = true;
1802 if (!degraded) { /* don't set/clear bits if degraded */
1803 *bmc |= RESYNC_MASK;
1804 *bmc &= ~NEEDED_MASK;
1805 }
1806 }
1807 }
1808 spin_unlock_irq(&bitmap->counts.lock);
1809
1810 return rv;
1811}
1812
1813static bool bitmap_start_sync(struct mddev *mddev, sector_t offset,
1814 sector_t *blocks, bool degraded)
1815{
1816 /* bitmap_start_sync must always report on multiples of whole
1817 * pages, otherwise resync (which is very PAGE_SIZE based) will
1818 * get confused.
1819 * So call __bitmap_start_sync repeatedly (if needed) until
1820 * At least PAGE_SIZE>>9 blocks are covered.
1821 * Return the 'or' of the result.
1822 */
1823 bool rv = false;
1824 sector_t blocks1;
1825
1826 *blocks = 0;
1827 while (*blocks < (PAGE_SIZE>>9)) {
1828 rv |= __bitmap_start_sync(mddev->bitmap, offset,
1829 &blocks1, degraded);
1830 offset += blocks1;
1831 *blocks += blocks1;
1832 }
1833
1834 return rv;
1835}
1836
1837static void __bitmap_end_sync(struct bitmap *bitmap, sector_t offset,
1838 sector_t *blocks, bool aborted)
1839{
1840 bitmap_counter_t *bmc;
1841 unsigned long flags;
1842
1843 spin_lock_irqsave(&bitmap->counts.lock, flags);
1844 bmc = md_bitmap_get_counter(&bitmap->counts, offset, blocks, 0);
1845 if (bmc == NULL)
1846 goto unlock;
1847 /* locked */
1848 if (RESYNC(*bmc)) {
1849 *bmc &= ~RESYNC_MASK;
1850
1851 if (!NEEDED(*bmc) && aborted)
1852 *bmc |= NEEDED_MASK;
1853 else {
1854 if (*bmc <= 2) {
1855 md_bitmap_set_pending(&bitmap->counts, offset);
1856 bitmap->allclean = 0;
1857 }
1858 }
1859 }
1860 unlock:
1861 spin_unlock_irqrestore(&bitmap->counts.lock, flags);
1862}
1863
1864static void bitmap_end_sync(struct mddev *mddev, sector_t offset,
1865 sector_t *blocks)
1866{
1867 __bitmap_end_sync(mddev->bitmap, offset, blocks, true);
1868}
1869
1870static void bitmap_close_sync(struct mddev *mddev)
1871{
1872 /* Sync has finished, and any bitmap chunks that weren't synced
1873 * properly have been aborted. It remains to us to clear the
1874 * RESYNC bit wherever it is still on
1875 */
1876 sector_t sector = 0;
1877 sector_t blocks;
1878 struct bitmap *bitmap = mddev->bitmap;
1879
1880 if (!bitmap)
1881 return;
1882
1883 while (sector < bitmap->mddev->resync_max_sectors) {
1884 __bitmap_end_sync(bitmap, sector, &blocks, false);
1885 sector += blocks;
1886 }
1887}
1888
1889static void bitmap_cond_end_sync(struct mddev *mddev, sector_t sector,
1890 bool force)
1891{
1892 sector_t s = 0;
1893 sector_t blocks;
1894 struct bitmap *bitmap = mddev->bitmap;
1895
1896 if (!bitmap)
1897 return;
1898 if (sector == 0) {
1899 bitmap->last_end_sync = jiffies;
1900 return;
1901 }
1902 if (!force && time_before(jiffies, (bitmap->last_end_sync
1903 + bitmap->mddev->bitmap_info.daemon_sleep)))
1904 return;
1905 wait_event(bitmap->mddev->recovery_wait,
1906 atomic_read(&bitmap->mddev->recovery_active) == 0);
1907
1908 bitmap->mddev->curr_resync_completed = sector;
1909 set_bit(MD_SB_CHANGE_CLEAN, &bitmap->mddev->sb_flags);
1910 sector &= ~((1ULL << bitmap->counts.chunkshift) - 1);
1911 s = 0;
1912 while (s < sector && s < bitmap->mddev->resync_max_sectors) {
1913 __bitmap_end_sync(bitmap, s, &blocks, false);
1914 s += blocks;
1915 }
1916 bitmap->last_end_sync = jiffies;
1917 sysfs_notify_dirent_safe(bitmap->mddev->sysfs_completed);
1918}
1919
1920static void bitmap_sync_with_cluster(struct mddev *mddev,
1921 sector_t old_lo, sector_t old_hi,
1922 sector_t new_lo, sector_t new_hi)
1923{
1924 struct bitmap *bitmap = mddev->bitmap;
1925 sector_t sector, blocks = 0;
1926
1927 for (sector = old_lo; sector < new_lo; ) {
1928 __bitmap_end_sync(bitmap, sector, &blocks, false);
1929 sector += blocks;
1930 }
1931 WARN((blocks > new_lo) && old_lo, "alignment is not correct for lo\n");
1932
1933 for (sector = old_hi; sector < new_hi; ) {
1934 bitmap_start_sync(mddev, sector, &blocks, false);
1935 sector += blocks;
1936 }
1937 WARN((blocks > new_hi) && old_hi, "alignment is not correct for hi\n");
1938}
1939
1940static void md_bitmap_set_memory_bits(struct bitmap *bitmap, sector_t offset, int needed)
1941{
1942 /* For each chunk covered by any of these sectors, set the
1943 * counter to 2 and possibly set resync_needed. They should all
1944 * be 0 at this point
1945 */
1946
1947 sector_t secs;
1948 bitmap_counter_t *bmc;
1949 spin_lock_irq(&bitmap->counts.lock);
1950 bmc = md_bitmap_get_counter(&bitmap->counts, offset, &secs, 1);
1951 if (!bmc) {
1952 spin_unlock_irq(&bitmap->counts.lock);
1953 return;
1954 }
1955 if (!*bmc) {
1956 *bmc = 2;
1957 md_bitmap_count_page(&bitmap->counts, offset, 1);
1958 md_bitmap_set_pending(&bitmap->counts, offset);
1959 bitmap->allclean = 0;
1960 }
1961 if (needed)
1962 *bmc |= NEEDED_MASK;
1963 spin_unlock_irq(&bitmap->counts.lock);
1964}
1965
1966/* dirty the memory and file bits for bitmap chunks "s" to "e" */
1967static void bitmap_dirty_bits(struct mddev *mddev, unsigned long s,
1968 unsigned long e)
1969{
1970 unsigned long chunk;
1971 struct bitmap *bitmap = mddev->bitmap;
1972
1973 if (!bitmap)
1974 return;
1975
1976 for (chunk = s; chunk <= e; chunk++) {
1977 sector_t sec = (sector_t)chunk << bitmap->counts.chunkshift;
1978
1979 md_bitmap_set_memory_bits(bitmap, sec, 1);
1980 md_bitmap_file_set_bit(bitmap, sec);
1981 if (sec < bitmap->mddev->resync_offset)
1982 /* We are asserting that the array is dirty,
1983 * so move the resync_offset address back so
1984 * that it is obvious that it is dirty
1985 */
1986 bitmap->mddev->resync_offset = sec;
1987 }
1988}
1989
1990static void bitmap_flush(struct mddev *mddev)
1991{
1992 struct bitmap *bitmap = mddev->bitmap;
1993 long sleep;
1994
1995 if (!bitmap) /* there was no bitmap */
1996 return;
1997
1998 /* run the daemon_work three time to ensure everything is flushed
1999 * that can be
2000 */
2001 sleep = mddev->bitmap_info.daemon_sleep * 2;
2002 bitmap->daemon_lastrun -= sleep;
2003 bitmap_daemon_work(mddev);
2004 bitmap->daemon_lastrun -= sleep;
2005 bitmap_daemon_work(mddev);
2006 bitmap->daemon_lastrun -= sleep;
2007 bitmap_daemon_work(mddev);
2008 if (mddev->bitmap_info.external)
2009 md_super_wait(mddev);
2010 bitmap_update_sb(bitmap);
2011}
2012
2013static void md_bitmap_free(void *data)
2014{
2015 unsigned long k, pages;
2016 struct bitmap_page *bp;
2017 struct bitmap *bitmap = data;
2018
2019 if (!bitmap) /* there was no bitmap */
2020 return;
2021
2022 if (bitmap->sysfs_can_clear)
2023 sysfs_put(bitmap->sysfs_can_clear);
2024
2025 if (mddev_is_clustered(bitmap->mddev) && bitmap->mddev->cluster_info &&
2026 bitmap->cluster_slot == bitmap->mddev->cluster_ops->slot_number(bitmap->mddev))
2027 md_cluster_stop(bitmap->mddev);
2028
2029 /* Shouldn't be needed - but just in case.... */
2030 wait_event(bitmap->write_wait,
2031 atomic_read(&bitmap->pending_writes) == 0);
2032
2033 /* release the bitmap file */
2034 md_bitmap_file_unmap(&bitmap->storage);
2035
2036 bp = bitmap->counts.bp;
2037 pages = bitmap->counts.pages;
2038
2039 /* free all allocated memory */
2040
2041 if (bp) /* deallocate the page memory */
2042 for (k = 0; k < pages; k++)
2043 if (bp[k].map && !bp[k].hijacked)
2044 kfree(bp[k].map);
2045 kfree(bp);
2046 kfree(bitmap);
2047}
2048
2049static void bitmap_start_behind_write(struct mddev *mddev)
2050{
2051 struct bitmap *bitmap = mddev->bitmap;
2052 int bw;
2053
2054 atomic_inc(&bitmap->behind_writes);
2055 bw = atomic_read(&bitmap->behind_writes);
2056 if (bw > bitmap->behind_writes_used)
2057 bitmap->behind_writes_used = bw;
2058
2059 pr_debug("inc write-behind count %d/%lu\n",
2060 bw, bitmap->mddev->bitmap_info.max_write_behind);
2061}
2062
2063static void bitmap_end_behind_write(struct mddev *mddev)
2064{
2065 struct bitmap *bitmap = mddev->bitmap;
2066
2067 if (atomic_dec_and_test(&bitmap->behind_writes))
2068 wake_up(&bitmap->behind_wait);
2069 pr_debug("dec write-behind count %d/%lu\n",
2070 atomic_read(&bitmap->behind_writes),
2071 bitmap->mddev->bitmap_info.max_write_behind);
2072}
2073
2074static void bitmap_wait_behind_writes(struct mddev *mddev)
2075{
2076 struct bitmap *bitmap = mddev->bitmap;
2077
2078 /* wait for behind writes to complete */
2079 if (bitmap && atomic_read(&bitmap->behind_writes) > 0) {
2080 pr_debug("md:%s: behind writes in progress - waiting to stop.\n",
2081 mdname(mddev));
2082 /* need to kick something here to make sure I/O goes? */
2083 wait_event(bitmap->behind_wait,
2084 atomic_read(&bitmap->behind_writes) == 0);
2085 }
2086}
2087
2088static void bitmap_destroy(struct mddev *mddev)
2089{
2090 struct bitmap *bitmap = mddev->bitmap;
2091
2092 if (!bitmap) /* there was no bitmap */
2093 return;
2094
2095 bitmap_wait_behind_writes(mddev);
2096 if (!test_bit(MD_SERIALIZE_POLICY, &mddev->flags))
2097 mddev_destroy_serial_pool(mddev, NULL);
2098
2099 mutex_lock(&mddev->bitmap_info.mutex);
2100 spin_lock(&mddev->lock);
2101 mddev->bitmap = NULL; /* disconnect from the md device */
2102 spin_unlock(&mddev->lock);
2103 mutex_unlock(&mddev->bitmap_info.mutex);
2104 mddev_set_timeout(mddev, MAX_SCHEDULE_TIMEOUT, true);
2105
2106 md_bitmap_free(bitmap);
2107}
2108
2109/*
2110 * initialize the bitmap structure
2111 * if this returns an error, bitmap_destroy must be called to do clean up
2112 * once mddev->bitmap is set
2113 */
2114static struct bitmap *__bitmap_create(struct mddev *mddev, int slot)
2115{
2116 struct bitmap *bitmap;
2117 sector_t blocks = mddev->resync_max_sectors;
2118 struct file *file = mddev->bitmap_info.file;
2119 int err;
2120 struct kernfs_node *bm = NULL;
2121
2122 BUILD_BUG_ON(sizeof(bitmap_super_t) != 256);
2123
2124 BUG_ON(file && mddev->bitmap_info.offset);
2125
2126 if (test_bit(MD_HAS_JOURNAL, &mddev->flags)) {
2127 pr_notice("md/raid:%s: array with journal cannot have bitmap\n",
2128 mdname(mddev));
2129 return ERR_PTR(-EBUSY);
2130 }
2131
2132 bitmap = kzalloc_obj(*bitmap);
2133 if (!bitmap)
2134 return ERR_PTR(-ENOMEM);
2135
2136 spin_lock_init(&bitmap->counts.lock);
2137 atomic_set(&bitmap->pending_writes, 0);
2138 init_waitqueue_head(&bitmap->write_wait);
2139 init_waitqueue_head(&bitmap->overflow_wait);
2140 init_waitqueue_head(&bitmap->behind_wait);
2141
2142 bitmap->mddev = mddev;
2143 bitmap->cluster_slot = slot;
2144
2145 if (mddev->kobj.sd)
2146 bm = sysfs_get_dirent(mddev->kobj.sd, "bitmap");
2147 if (bm) {
2148 bitmap->sysfs_can_clear = sysfs_get_dirent(bm, "can_clear");
2149 sysfs_put(bm);
2150 } else
2151 bitmap->sysfs_can_clear = NULL;
2152
2153 bitmap->storage.file = file;
2154 if (file) {
2155 get_file(file);
2156 /* As future accesses to this file will use bmap,
2157 * and bypass the page cache, we must sync the file
2158 * first.
2159 */
2160 vfs_fsync(file, 1);
2161 }
2162 /* read superblock from bitmap file (this sets mddev->bitmap_info.chunksize) */
2163 if (!mddev->bitmap_info.external) {
2164 /*
2165 * If 'MD_ARRAY_FIRST_USE' is set, then device-mapper is
2166 * instructing us to create a new on-disk bitmap instance.
2167 */
2168 if (test_and_clear_bit(MD_ARRAY_FIRST_USE, &mddev->flags))
2169 err = md_bitmap_new_disk_sb(bitmap);
2170 else
2171 err = md_bitmap_read_sb(bitmap);
2172 } else {
2173 err = 0;
2174 if (mddev->bitmap_info.chunksize == 0 ||
2175 mddev->bitmap_info.daemon_sleep == 0)
2176 /* chunksize and time_base need to be
2177 * set first. */
2178 err = -EINVAL;
2179 }
2180 if (err)
2181 goto error;
2182
2183 bitmap->daemon_lastrun = jiffies;
2184 err = __bitmap_resize(bitmap, blocks, mddev->bitmap_info.chunksize,
2185 true);
2186 if (err)
2187 goto error;
2188
2189 pr_debug("created bitmap (%lu pages) for device %s\n",
2190 bitmap->counts.pages, bmname(bitmap));
2191
2192 err = test_bit(BITMAP_WRITE_ERROR, &bitmap->flags) ? -EIO : 0;
2193 if (err)
2194 goto error;
2195
2196 return bitmap;
2197 error:
2198 md_bitmap_free(bitmap);
2199 return ERR_PTR(err);
2200}
2201
2202static int bitmap_create(struct mddev *mddev)
2203{
2204 struct bitmap *bitmap = __bitmap_create(mddev, -1);
2205
2206 if (IS_ERR(bitmap))
2207 return PTR_ERR(bitmap);
2208
2209 mddev->bitmap = bitmap;
2210 return 0;
2211}
2212
2213static int bitmap_load(struct mddev *mddev)
2214{
2215 int err = 0;
2216 sector_t start = 0;
2217 sector_t sector = 0;
2218 struct bitmap *bitmap = mddev->bitmap;
2219 struct md_rdev *rdev;
2220
2221 if (!bitmap)
2222 goto out;
2223
2224 rdev_for_each(rdev, mddev)
2225 mddev_create_serial_pool(mddev, rdev);
2226
2227 if (mddev_is_clustered(mddev))
2228 mddev->cluster_ops->load_bitmaps(mddev, mddev->bitmap_info.nodes);
2229
2230 /* Clear out old bitmap info first: Either there is none, or we
2231 * are resuming after someone else has possibly changed things,
2232 * so we should forget old cached info.
2233 * All chunks should be clean, but some might need_sync.
2234 */
2235 while (sector < mddev->resync_max_sectors) {
2236 sector_t blocks;
2237 bitmap_start_sync(mddev, sector, &blocks, false);
2238 sector += blocks;
2239 }
2240 bitmap_close_sync(mddev);
2241
2242 if (mddev->degraded == 0
2243 || bitmap->events_cleared == mddev->events)
2244 /* no need to keep dirty bits to optimise a
2245 * re-add of a missing device */
2246 start = mddev->resync_offset;
2247
2248 mutex_lock(&mddev->bitmap_info.mutex);
2249 err = md_bitmap_init_from_disk(bitmap, start);
2250 mutex_unlock(&mddev->bitmap_info.mutex);
2251
2252 if (err)
2253 goto out;
2254 clear_bit(BITMAP_STALE, &bitmap->flags);
2255
2256 /* Kick recovery in case any bits were set */
2257 set_bit(MD_RECOVERY_NEEDED, &bitmap->mddev->recovery);
2258
2259 mddev_set_timeout(mddev, mddev->bitmap_info.daemon_sleep, true);
2260 md_wakeup_thread(mddev->thread);
2261
2262 bitmap_update_sb(bitmap);
2263
2264 if (test_bit(BITMAP_WRITE_ERROR, &bitmap->flags))
2265 err = -EIO;
2266out:
2267 return err;
2268}
2269
2270/* caller need to free returned bitmap with md_bitmap_free() */
2271static void *bitmap_get_from_slot(struct mddev *mddev, int slot)
2272{
2273 int rv = 0;
2274 struct bitmap *bitmap;
2275
2276 bitmap = __bitmap_create(mddev, slot);
2277 if (IS_ERR(bitmap)) {
2278 rv = PTR_ERR(bitmap);
2279 return ERR_PTR(rv);
2280 }
2281
2282 rv = md_bitmap_init_from_disk(bitmap, 0);
2283 if (rv) {
2284 md_bitmap_free(bitmap);
2285 return ERR_PTR(rv);
2286 }
2287
2288 return bitmap;
2289}
2290
2291/* Loads the bitmap associated with slot and copies the resync information
2292 * to our bitmap
2293 */
2294static int bitmap_copy_from_slot(struct mddev *mddev, int slot, sector_t *low,
2295 sector_t *high, bool clear_bits)
2296{
2297 int rv = 0, i, j;
2298 sector_t block, lo = 0, hi = 0;
2299 struct bitmap_counts *counts;
2300 struct bitmap *bitmap;
2301
2302 bitmap = bitmap_get_from_slot(mddev, slot);
2303 if (IS_ERR(bitmap)) {
2304 pr_err("%s can't get bitmap from slot %d\n", __func__, slot);
2305 return -1;
2306 }
2307
2308 counts = &bitmap->counts;
2309 for (j = 0; j < counts->chunks; j++) {
2310 block = (sector_t)j << counts->chunkshift;
2311 if (md_bitmap_file_test_bit(bitmap, block)) {
2312 if (!lo)
2313 lo = block;
2314 hi = block;
2315 md_bitmap_file_clear_bit(bitmap, block);
2316 md_bitmap_set_memory_bits(mddev->bitmap, block, 1);
2317 md_bitmap_file_set_bit(mddev->bitmap, block);
2318 }
2319 }
2320
2321 if (clear_bits) {
2322 bitmap_update_sb(bitmap);
2323 /* BITMAP_PAGE_PENDING is set, but bitmap_unplug needs
2324 * BITMAP_PAGE_DIRTY or _NEEDWRITE to write ... */
2325 for (i = 0; i < bitmap->storage.file_pages; i++)
2326 if (test_page_attr(bitmap, i, BITMAP_PAGE_PENDING))
2327 set_page_attr(bitmap, i, BITMAP_PAGE_NEEDWRITE);
2328 __bitmap_unplug(bitmap);
2329 }
2330 __bitmap_unplug(mddev->bitmap);
2331 *low = lo;
2332 *high = hi;
2333 md_bitmap_free(bitmap);
2334
2335 return rv;
2336}
2337
2338static void bitmap_set_pages(void *data, unsigned long pages)
2339{
2340 struct bitmap *bitmap = data;
2341
2342 bitmap->counts.pages = pages;
2343}
2344
2345static int bitmap_get_stats(void *data, struct md_bitmap_stats *stats)
2346{
2347 struct bitmap_storage *storage;
2348 struct bitmap_counts *counts;
2349 struct bitmap *bitmap = data;
2350 bitmap_super_t *sb;
2351
2352 if (!bitmap)
2353 return -ENOENT;
2354 if (!bitmap->storage.sb_page)
2355 return -EINVAL;
2356 sb = kmap_local_page(bitmap->storage.sb_page);
2357 stats->sync_size = le64_to_cpu(sb->sync_size);
2358 kunmap_local(sb);
2359
2360 counts = &bitmap->counts;
2361 stats->missing_pages = counts->missing_pages;
2362 stats->pages = counts->pages;
2363
2364 storage = &bitmap->storage;
2365 stats->file_pages = storage->file_pages;
2366 stats->file = storage->file;
2367
2368 stats->behind_writes = atomic_read(&bitmap->behind_writes);
2369 stats->behind_wait = wq_has_sleeper(&bitmap->behind_wait);
2370 stats->events_cleared = bitmap->events_cleared;
2371 return 0;
2372}
2373
2374static int __bitmap_resize(struct bitmap *bitmap, sector_t blocks,
2375 int chunksize, bool init)
2376{
2377 /* If chunk_size is 0, choose an appropriate chunk size.
2378 * Then possibly allocate new storage space.
2379 * Then quiesce, copy bits, replace bitmap, and re-start
2380 *
2381 * This function is called both to set up the initial bitmap
2382 * and to resize the bitmap while the array is active.
2383 * If this happens as a result of the array being resized,
2384 * chunksize will be zero, and we need to choose a suitable
2385 * chunksize, otherwise we use what we are given.
2386 */
2387 struct bitmap_storage store;
2388 struct bitmap_counts old_counts;
2389 unsigned long chunks;
2390 sector_t block;
2391 sector_t old_blocks, new_blocks;
2392 int chunkshift;
2393 int ret = 0;
2394 long pages;
2395 struct bitmap_page *new_bp;
2396
2397 if (bitmap->storage.file && !init) {
2398 pr_info("md: cannot resize file-based bitmap\n");
2399 return -EINVAL;
2400 }
2401
2402 if (chunksize == 0) {
2403 /* If there is enough space, leave the chunk size unchanged,
2404 * else increase by factor of two until there is enough space.
2405 */
2406 long bytes;
2407 long space = bitmap->mddev->bitmap_info.space;
2408
2409 if (space == 0) {
2410 /* We don't know how much space there is, so limit
2411 * to current size - in sectors.
2412 */
2413 bytes = DIV_ROUND_UP(bitmap->counts.chunks, 8);
2414 if (!bitmap->mddev->bitmap_info.external)
2415 bytes += sizeof(bitmap_super_t);
2416 space = DIV_ROUND_UP(bytes, 512);
2417 bitmap->mddev->bitmap_info.space = space;
2418 }
2419 chunkshift = bitmap->counts.chunkshift;
2420 chunkshift--;
2421 do {
2422 /* 'chunkshift' is shift from block size to chunk size */
2423 chunkshift++;
2424 chunks = DIV_ROUND_UP_SECTOR_T(blocks, 1 << chunkshift);
2425 bytes = DIV_ROUND_UP(chunks, 8);
2426 if (!bitmap->mddev->bitmap_info.external)
2427 bytes += sizeof(bitmap_super_t);
2428 } while (bytes > (space << 9) && (chunkshift + BITMAP_BLOCK_SHIFT) <
2429 (BITS_PER_BYTE * sizeof(((bitmap_super_t *)0)->chunksize) - 1));
2430 } else
2431 chunkshift = ffz(~chunksize) - BITMAP_BLOCK_SHIFT;
2432
2433 chunks = DIV_ROUND_UP_SECTOR_T(blocks, 1 << chunkshift);
2434 memset(&store, 0, sizeof(store));
2435 if (bitmap->mddev->bitmap_info.offset || bitmap->mddev->bitmap_info.file)
2436 ret = md_bitmap_storage_alloc(&store, chunks,
2437 !bitmap->mddev->bitmap_info.external,
2438 mddev_is_clustered(bitmap->mddev)
2439 ? bitmap->cluster_slot : 0);
2440 if (ret) {
2441 md_bitmap_file_unmap(&store);
2442 goto err;
2443 }
2444
2445 pages = DIV_ROUND_UP(chunks, PAGE_COUNTER_RATIO);
2446
2447 new_bp = kzalloc_objs(*new_bp, pages);
2448 ret = -ENOMEM;
2449 if (!new_bp) {
2450 md_bitmap_file_unmap(&store);
2451 goto err;
2452 }
2453
2454 if (!init)
2455 bitmap->mddev->pers->quiesce(bitmap->mddev, 1);
2456
2457 store.file = bitmap->storage.file;
2458 bitmap->storage.file = NULL;
2459
2460 if (store.sb_page && bitmap->storage.sb_page)
2461 memcpy(page_address(store.sb_page),
2462 page_address(bitmap->storage.sb_page),
2463 sizeof(bitmap_super_t));
2464 mutex_lock(&bitmap->mddev->bitmap_info.mutex);
2465 spin_lock_irq(&bitmap->counts.lock);
2466 md_bitmap_file_unmap(&bitmap->storage);
2467 bitmap->storage = store;
2468
2469 old_counts = bitmap->counts;
2470 bitmap->counts.bp = new_bp;
2471 bitmap->counts.pages = pages;
2472 bitmap->counts.missing_pages = pages;
2473 bitmap->counts.chunkshift = chunkshift;
2474 bitmap->counts.chunks = chunks;
2475 bitmap->mddev->bitmap_info.chunksize = 1UL << (chunkshift +
2476 BITMAP_BLOCK_SHIFT);
2477
2478 blocks = min(old_counts.chunks << old_counts.chunkshift,
2479 chunks << chunkshift);
2480
2481 /* For cluster raid, need to pre-allocate bitmap */
2482 if (mddev_is_clustered(bitmap->mddev)) {
2483 unsigned long page;
2484 for (page = 0; page < pages; page++) {
2485 ret = md_bitmap_checkpage(&bitmap->counts, page, 1, 1);
2486 if (ret) {
2487 unsigned long k;
2488
2489 /* deallocate the page memory */
2490 for (k = 0; k < page; k++) {
2491 kfree(new_bp[k].map);
2492 }
2493 kfree(new_bp);
2494
2495 /* restore some fields from old_counts */
2496 bitmap->counts.bp = old_counts.bp;
2497 bitmap->counts.pages = old_counts.pages;
2498 bitmap->counts.missing_pages = old_counts.pages;
2499 bitmap->counts.chunkshift = old_counts.chunkshift;
2500 bitmap->counts.chunks = old_counts.chunks;
2501 bitmap->mddev->bitmap_info.chunksize =
2502 1UL << (old_counts.chunkshift + BITMAP_BLOCK_SHIFT);
2503 blocks = old_counts.chunks << old_counts.chunkshift;
2504 pr_warn("Could not pre-allocate in-memory bitmap for cluster raid\n");
2505 break;
2506 } else
2507 bitmap->counts.bp[page].count += 1;
2508 }
2509 }
2510
2511 for (block = 0; block < blocks; ) {
2512 bitmap_counter_t *bmc_old, *bmc_new;
2513 int set;
2514
2515 bmc_old = md_bitmap_get_counter(&old_counts, block, &old_blocks, 0);
2516 set = bmc_old && NEEDED(*bmc_old);
2517
2518 if (set) {
2519 bmc_new = md_bitmap_get_counter(&bitmap->counts, block, &new_blocks, 1);
2520 if (bmc_new) {
2521 if (*bmc_new == 0) {
2522 /* need to set on-disk bits too. */
2523 sector_t end = block + new_blocks;
2524 sector_t start = block >> chunkshift;
2525
2526 start <<= chunkshift;
2527 while (start < end) {
2528 md_bitmap_file_set_bit(bitmap, block);
2529 start += 1 << chunkshift;
2530 }
2531 *bmc_new = 2;
2532 md_bitmap_count_page(&bitmap->counts, block, 1);
2533 md_bitmap_set_pending(&bitmap->counts, block);
2534 }
2535 *bmc_new |= NEEDED_MASK;
2536 }
2537 if (new_blocks < old_blocks)
2538 old_blocks = new_blocks;
2539 }
2540 block += old_blocks;
2541 }
2542
2543 if (bitmap->counts.bp != old_counts.bp) {
2544 unsigned long k;
2545 for (k = 0; k < old_counts.pages; k++)
2546 if (!old_counts.bp[k].hijacked)
2547 kfree(old_counts.bp[k].map);
2548 kfree(old_counts.bp);
2549 }
2550
2551 if (!init) {
2552 int i;
2553 while (block < (chunks << chunkshift)) {
2554 bitmap_counter_t *bmc;
2555 bmc = md_bitmap_get_counter(&bitmap->counts, block, &new_blocks, 1);
2556 if (bmc) {
2557 /* new space. It needs to be resynced, so
2558 * we set NEEDED_MASK.
2559 */
2560 if (*bmc == 0) {
2561 *bmc = NEEDED_MASK | 2;
2562 md_bitmap_count_page(&bitmap->counts, block, 1);
2563 md_bitmap_set_pending(&bitmap->counts, block);
2564 }
2565 }
2566 block += new_blocks;
2567 }
2568 for (i = 0; i < bitmap->storage.file_pages; i++)
2569 set_page_attr(bitmap, i, BITMAP_PAGE_DIRTY);
2570 }
2571 spin_unlock_irq(&bitmap->counts.lock);
2572 mutex_unlock(&bitmap->mddev->bitmap_info.mutex);
2573 if (!init) {
2574 __bitmap_unplug(bitmap);
2575 bitmap->mddev->pers->quiesce(bitmap->mddev, 0);
2576 }
2577 ret = 0;
2578err:
2579 return ret;
2580}
2581
2582static int bitmap_resize(struct mddev *mddev, sector_t blocks, int chunksize)
2583{
2584 struct bitmap *bitmap = mddev->bitmap;
2585
2586 if (!bitmap)
2587 return 0;
2588
2589 return __bitmap_resize(bitmap, blocks, chunksize, false);
2590}
2591
2592static bool bitmap_none_enabled(void *data, bool flush)
2593{
2594 return false;
2595}
2596
2597static int bitmap_none_create(struct mddev *mddev)
2598{
2599 return 0;
2600}
2601
2602static int bitmap_none_load(struct mddev *mddev)
2603{
2604 return 0;
2605}
2606
2607static void bitmap_none_destroy(struct mddev *mddev)
2608{
2609}
2610
2611static int bitmap_none_get_stats(void *data, struct md_bitmap_stats *stats)
2612{
2613 return -ENOENT;
2614}
2615
2616static ssize_t
2617location_show(struct mddev *mddev, char *page)
2618{
2619 ssize_t len;
2620 if (mddev->bitmap_info.file)
2621 len = sprintf(page, "file");
2622 else if (mddev->bitmap_info.offset)
2623 len = sprintf(page, "%+lld", (long long)mddev->bitmap_info.offset);
2624 else
2625 len = sprintf(page, "none");
2626 len += sprintf(page+len, "\n");
2627 return len;
2628}
2629
2630static ssize_t
2631location_store(struct mddev *mddev, const char *buf, size_t len)
2632{
2633 int rv;
2634 unsigned int noio_flags;
2635
2636 rv = mddev_suspend_and_lock(mddev);
2637 if (rv)
2638 return rv;
2639 noio_flags = memalloc_noio_save();
2640
2641 if (mddev->pers) {
2642 if (mddev->recovery || mddev->sync_thread) {
2643 rv = -EBUSY;
2644 goto out;
2645 }
2646 }
2647
2648 if (mddev->bitmap || mddev->bitmap_info.file ||
2649 mddev->bitmap_info.offset) {
2650 /* bitmap already configured. Only option is to clear it */
2651 if (strncmp(buf, "none", 4) != 0) {
2652 rv = -EBUSY;
2653 goto out;
2654 }
2655
2656 sysfs_unmerge_group(&mddev->kobj, &md_bitmap_internal_group);
2657 md_bitmap_destroy_nosysfs(mddev);
2658 mddev->bitmap_id = ID_BITMAP_NONE;
2659 if (!mddev_set_bitmap_ops_nosysfs(mddev))
2660 goto none_err;
2661 mddev->bitmap_info.offset = 0;
2662 if (mddev->bitmap_info.file) {
2663 struct file *f = mddev->bitmap_info.file;
2664 mddev->bitmap_info.file = NULL;
2665 fput(f);
2666 }
2667 } else {
2668 /* No bitmap, OK to set a location */
2669 long long offset;
2670
2671 if (strncmp(buf, "none", 4) == 0)
2672 /* nothing to be done */;
2673 else if (strncmp(buf, "file:", 5) == 0) {
2674 /* Not supported yet */
2675 rv = -EINVAL;
2676 goto out;
2677 } else {
2678 if (buf[0] == '+')
2679 rv = kstrtoll(buf+1, 10, &offset);
2680 else
2681 rv = kstrtoll(buf, 10, &offset);
2682 if (rv)
2683 goto out;
2684 if (offset == 0) {
2685 rv = -EINVAL;
2686 goto out;
2687 }
2688 if (mddev->bitmap_info.external == 0 &&
2689 mddev->major_version == 0 &&
2690 offset != mddev->bitmap_info.default_offset) {
2691 rv = -EINVAL;
2692 goto out;
2693 }
2694
2695 mddev->bitmap_info.offset = offset;
2696 md_bitmap_destroy_nosysfs(mddev);
2697 mddev->bitmap_id = ID_BITMAP;
2698 if (!mddev_set_bitmap_ops_nosysfs(mddev))
2699 goto bitmap_err;
2700
2701 rv = md_bitmap_create_nosysfs(mddev);
2702 if (rv)
2703 goto create_err;
2704
2705 rv = mddev->bitmap_ops->load(mddev);
2706 if (rv) {
2707 mddev->bitmap_info.offset = 0;
2708 goto load_err;
2709 }
2710
2711 rv = sysfs_merge_group(&mddev->kobj,
2712 &md_bitmap_internal_group);
2713 if (rv)
2714 goto merge_err;
2715 }
2716 }
2717 if (!mddev->external) {
2718 /* Ensure new bitmap info is stored in
2719 * metadata promptly.
2720 */
2721 set_bit(MD_SB_CHANGE_DEVS, &mddev->sb_flags);
2722 md_wakeup_thread(mddev->thread);
2723 }
2724 rv = 0;
2725out:
2726 memalloc_noio_restore(noio_flags);
2727 mddev_unlock_and_resume(mddev);
2728 if (rv)
2729 return rv;
2730 return len;
2731
2732merge_err:
2733 mddev->bitmap_info.offset = 0;
2734load_err:
2735 md_bitmap_destroy_nosysfs(mddev);
2736create_err:
2737 mddev->bitmap_info.offset = 0;
2738 mddev->bitmap_id = ID_BITMAP_NONE;
2739 if (!mddev_set_bitmap_ops_nosysfs(mddev))
2740 rv = -ENOENT;
2741 goto out;
2742bitmap_err:
2743 rv = -ENOENT;
2744none_err:
2745 mddev->bitmap_info.offset = 0;
2746 goto out;
2747}
2748
2749static struct md_sysfs_entry bitmap_location =
2750__ATTR(location, S_IRUGO|S_IWUSR, location_show, location_store);
2751
2752/* 'bitmap/space' is the space available at 'location' for the
2753 * bitmap. This allows the kernel to know when it is safe to
2754 * resize the bitmap to match a resized array.
2755 */
2756static ssize_t
2757space_show(struct mddev *mddev, char *page)
2758{
2759 return sprintf(page, "%lu\n", mddev->bitmap_info.space);
2760}
2761
2762static ssize_t
2763space_store(struct mddev *mddev, const char *buf, size_t len)
2764{
2765 struct bitmap *bitmap;
2766 unsigned long sectors;
2767 int rv;
2768
2769 rv = kstrtoul(buf, 10, &sectors);
2770 if (rv)
2771 return rv;
2772
2773 if (sectors == 0)
2774 return -EINVAL;
2775
2776 bitmap = mddev->bitmap;
2777 if (bitmap && sectors < (bitmap->storage.bytes + 511) >> 9)
2778 return -EFBIG; /* Bitmap is too big for this small space */
2779
2780 /* could make sure it isn't too big, but that isn't really
2781 * needed - user-space should be careful.
2782 */
2783 mddev->bitmap_info.space = sectors;
2784 return len;
2785}
2786
2787static struct md_sysfs_entry bitmap_space =
2788__ATTR(space, S_IRUGO|S_IWUSR, space_show, space_store);
2789
2790static ssize_t
2791timeout_show(struct mddev *mddev, char *page)
2792{
2793 ssize_t len;
2794 unsigned long secs = mddev->bitmap_info.daemon_sleep / HZ;
2795 unsigned long jifs = mddev->bitmap_info.daemon_sleep % HZ;
2796
2797 len = sprintf(page, "%lu", secs);
2798 if (jifs)
2799 len += sprintf(page+len, ".%03u", jiffies_to_msecs(jifs));
2800 len += sprintf(page+len, "\n");
2801 return len;
2802}
2803
2804static ssize_t
2805timeout_store(struct mddev *mddev, const char *buf, size_t len)
2806{
2807 /* timeout can be set at any time */
2808 unsigned long timeout;
2809 int rv = strict_strtoul_scaled(buf, &timeout, 4);
2810 if (rv)
2811 return rv;
2812
2813 /* just to make sure we don't overflow... */
2814 if (timeout >= LONG_MAX / HZ)
2815 return -EINVAL;
2816
2817 timeout = timeout * HZ / 10000;
2818
2819 if (timeout >= MAX_SCHEDULE_TIMEOUT)
2820 timeout = MAX_SCHEDULE_TIMEOUT-1;
2821 if (timeout < 1)
2822 timeout = 1;
2823
2824 mddev->bitmap_info.daemon_sleep = timeout;
2825 mddev_set_timeout(mddev, timeout, false);
2826 md_wakeup_thread(mddev->thread);
2827
2828 return len;
2829}
2830
2831static struct md_sysfs_entry bitmap_timeout =
2832__ATTR(time_base, S_IRUGO|S_IWUSR, timeout_show, timeout_store);
2833
2834static ssize_t
2835backlog_show(struct mddev *mddev, char *page)
2836{
2837 return sprintf(page, "%lu\n", mddev->bitmap_info.max_write_behind);
2838}
2839
2840static ssize_t
2841backlog_store(struct mddev *mddev, const char *buf, size_t len)
2842{
2843 unsigned long backlog;
2844 unsigned long old_mwb = mddev->bitmap_info.max_write_behind;
2845 struct md_rdev *rdev;
2846 bool has_write_mostly = false;
2847 int rv = kstrtoul(buf, 10, &backlog);
2848 if (rv)
2849 return rv;
2850 if (backlog > COUNTER_MAX)
2851 return -EINVAL;
2852
2853 rv = mddev_suspend_and_lock(mddev);
2854 if (rv)
2855 return rv;
2856
2857 /*
2858 * Without write mostly device, it doesn't make sense to set
2859 * backlog for max_write_behind.
2860 */
2861 rdev_for_each(rdev, mddev) {
2862 if (test_bit(WriteMostly, &rdev->flags)) {
2863 has_write_mostly = true;
2864 break;
2865 }
2866 }
2867 if (!has_write_mostly) {
2868 pr_warn_ratelimited("%s: can't set backlog, no write mostly device available\n",
2869 mdname(mddev));
2870 mddev_unlock_and_resume(mddev);
2871 return -EINVAL;
2872 }
2873
2874 mddev->bitmap_info.max_write_behind = backlog;
2875 if (!backlog && mddev->serial_info_pool) {
2876 /* serial_info_pool is not needed if backlog is zero */
2877 if (!test_bit(MD_SERIALIZE_POLICY, &mddev->flags))
2878 mddev_destroy_serial_pool(mddev, NULL);
2879 } else if (backlog && !mddev->serial_info_pool) {
2880 /* serial_info_pool is needed since backlog is not zero */
2881 rdev_for_each(rdev, mddev)
2882 mddev_create_serial_pool(mddev, rdev);
2883 }
2884 if (old_mwb != backlog)
2885 bitmap_update_sb(mddev->bitmap);
2886
2887 mddev_unlock_and_resume(mddev);
2888 return len;
2889}
2890
2891static struct md_sysfs_entry bitmap_backlog =
2892__ATTR(backlog, S_IRUGO|S_IWUSR, backlog_show, backlog_store);
2893
2894static ssize_t
2895chunksize_show(struct mddev *mddev, char *page)
2896{
2897 return sprintf(page, "%lu\n", mddev->bitmap_info.chunksize);
2898}
2899
2900static ssize_t
2901chunksize_store(struct mddev *mddev, const char *buf, size_t len)
2902{
2903 /* Can only be changed when no bitmap is active */
2904 int rv;
2905 unsigned long csize;
2906 if (mddev->bitmap)
2907 return -EBUSY;
2908 rv = kstrtoul(buf, 10, &csize);
2909 if (rv)
2910 return rv;
2911 if (csize < 512 ||
2912 !is_power_of_2(csize))
2913 return -EINVAL;
2914 if (BITS_PER_LONG > 32 && csize >= (1ULL << (BITS_PER_BYTE *
2915 sizeof(((bitmap_super_t *)0)->chunksize))))
2916 return -EOVERFLOW;
2917 mddev->bitmap_info.chunksize = csize;
2918 return len;
2919}
2920
2921static struct md_sysfs_entry bitmap_chunksize =
2922__ATTR(chunksize, S_IRUGO|S_IWUSR, chunksize_show, chunksize_store);
2923
2924static ssize_t metadata_show(struct mddev *mddev, char *page)
2925{
2926 if (mddev_is_clustered(mddev))
2927 return sprintf(page, "clustered\n");
2928 return sprintf(page, "%s\n", (mddev->bitmap_info.external
2929 ? "external" : "internal"));
2930}
2931
2932static ssize_t metadata_store(struct mddev *mddev, const char *buf, size_t len)
2933{
2934 if (mddev->bitmap ||
2935 mddev->bitmap_info.file ||
2936 mddev->bitmap_info.offset)
2937 return -EBUSY;
2938 if (strncmp(buf, "external", 8) == 0)
2939 mddev->bitmap_info.external = 1;
2940 else if ((strncmp(buf, "internal", 8) == 0) ||
2941 (strncmp(buf, "clustered", 9) == 0))
2942 mddev->bitmap_info.external = 0;
2943 else
2944 return -EINVAL;
2945 return len;
2946}
2947
2948static struct md_sysfs_entry bitmap_metadata =
2949__ATTR(metadata, S_IRUGO|S_IWUSR, metadata_show, metadata_store);
2950
2951static ssize_t can_clear_show(struct mddev *mddev, char *page)
2952{
2953 int len;
2954 struct bitmap *bitmap;
2955
2956 spin_lock(&mddev->lock);
2957 bitmap = mddev->bitmap;
2958 if (bitmap)
2959 len = sprintf(page, "%s\n", (bitmap->need_sync ? "false" :
2960 "true"));
2961 else
2962 len = sprintf(page, "\n");
2963 spin_unlock(&mddev->lock);
2964 return len;
2965}
2966
2967static ssize_t can_clear_store(struct mddev *mddev, const char *buf, size_t len)
2968{
2969 struct bitmap *bitmap = mddev->bitmap;
2970
2971 if (!bitmap)
2972 return -ENOENT;
2973
2974 if (strncmp(buf, "false", 5) == 0) {
2975 bitmap->need_sync = 1;
2976 return len;
2977 }
2978
2979 if (strncmp(buf, "true", 4) == 0) {
2980 if (mddev->degraded)
2981 return -EBUSY;
2982 bitmap->need_sync = 0;
2983 return len;
2984 }
2985
2986 return -EINVAL;
2987}
2988
2989static struct md_sysfs_entry bitmap_can_clear =
2990__ATTR(can_clear, S_IRUGO|S_IWUSR, can_clear_show, can_clear_store);
2991
2992static ssize_t
2993behind_writes_used_show(struct mddev *mddev, char *page)
2994{
2995 ssize_t ret;
2996 struct bitmap *bitmap;
2997
2998 spin_lock(&mddev->lock);
2999 bitmap = mddev->bitmap;
3000 if (!bitmap)
3001 ret = sprintf(page, "0\n");
3002 else
3003 ret = sprintf(page, "%lu\n", bitmap->behind_writes_used);
3004 spin_unlock(&mddev->lock);
3005
3006 return ret;
3007}
3008
3009static ssize_t
3010behind_writes_used_reset(struct mddev *mddev, const char *buf, size_t len)
3011{
3012 struct bitmap *bitmap = mddev->bitmap;
3013
3014 if (bitmap)
3015 bitmap->behind_writes_used = 0;
3016 return len;
3017}
3018
3019static struct md_sysfs_entry max_backlog_used =
3020__ATTR(max_backlog_used, S_IRUGO | S_IWUSR,
3021 behind_writes_used_show, behind_writes_used_reset);
3022
3023static struct attribute *md_bitmap_common_attrs[] = {
3024 &bitmap_location.attr,
3025 NULL
3026};
3027
3028static struct attribute *md_bitmap_internal_attrs[] = {
3029 &bitmap_space.attr,
3030 &bitmap_timeout.attr,
3031 &bitmap_backlog.attr,
3032 &bitmap_chunksize.attr,
3033 &bitmap_metadata.attr,
3034 &bitmap_can_clear.attr,
3035 &max_backlog_used.attr,
3036 NULL
3037};
3038
3039static struct attribute_group md_bitmap_common_group = {
3040 .name = "bitmap",
3041 .attrs = md_bitmap_common_attrs,
3042};
3043
3044static struct attribute_group md_bitmap_internal_group = {
3045 .name = "bitmap",
3046 .attrs = md_bitmap_internal_attrs,
3047};
3048
3049static const struct attribute_group *bitmap_groups[] = {
3050 &md_bitmap_common_group,
3051 &md_bitmap_internal_group,
3052 NULL,
3053};
3054
3055static const struct attribute_group *bitmap_none_groups[] = {
3056 &md_bitmap_common_group,
3057 NULL,
3058};
3059
3060static struct bitmap_operations bitmap_none_ops = {
3061 .head = {
3062 .type = MD_BITMAP,
3063 .id = ID_BITMAP_NONE,
3064 .name = "none",
3065 },
3066
3067 .enabled = bitmap_none_enabled,
3068 .create = bitmap_none_create,
3069 .load = bitmap_none_load,
3070 .destroy = bitmap_none_destroy,
3071 .get_stats = bitmap_none_get_stats,
3072
3073 .groups = bitmap_none_groups,
3074};
3075
3076static struct bitmap_operations bitmap_ops = {
3077 .head = {
3078 .type = MD_BITMAP,
3079 .id = ID_BITMAP,
3080 .name = "bitmap",
3081 },
3082
3083 .enabled = bitmap_enabled,
3084 .create = bitmap_create,
3085 .resize = bitmap_resize,
3086 .load = bitmap_load,
3087 .destroy = bitmap_destroy,
3088 .flush = bitmap_flush,
3089 .write_all = bitmap_write_all,
3090 .dirty_bits = bitmap_dirty_bits,
3091 .prepare_range = bitmap_prepare_range,
3092 .unplug = bitmap_unplug,
3093 .daemon_work = bitmap_daemon_work,
3094
3095 .start_behind_write = bitmap_start_behind_write,
3096 .end_behind_write = bitmap_end_behind_write,
3097 .wait_behind_writes = bitmap_wait_behind_writes,
3098
3099 .start_write = bitmap_start_write,
3100 .end_write = bitmap_end_write,
3101 .start_discard = bitmap_start_write,
3102 .end_discard = bitmap_end_write,
3103
3104 .start_sync = bitmap_start_sync,
3105 .end_sync = bitmap_end_sync,
3106 .cond_end_sync = bitmap_cond_end_sync,
3107 .close_sync = bitmap_close_sync,
3108
3109 .update_sb = bitmap_update_sb,
3110 .get_stats = bitmap_get_stats,
3111
3112 .sync_with_cluster = bitmap_sync_with_cluster,
3113 .get_from_slot = bitmap_get_from_slot,
3114 .copy_from_slot = bitmap_copy_from_slot,
3115 .set_pages = bitmap_set_pages,
3116 .free = md_bitmap_free,
3117
3118 .groups = bitmap_groups,
3119};
3120
3121int md_bitmap_init(void)
3122{
3123 int err;
3124
3125 md_bitmap_wq = alloc_workqueue("md_bitmap", WQ_MEM_RECLAIM | WQ_UNBOUND,
3126 0);
3127 if (!md_bitmap_wq)
3128 return -ENOMEM;
3129
3130 err = register_md_submodule(&bitmap_none_ops.head);
3131 if (err)
3132 goto err_wq;
3133
3134 err = register_md_submodule(&bitmap_ops.head);
3135 if (err)
3136 goto err_none;
3137
3138 return 0;
3139
3140err_none:
3141 unregister_md_submodule(&bitmap_none_ops.head);
3142err_wq:
3143 destroy_workqueue(md_bitmap_wq);
3144 return err;
3145}
3146
3147void md_bitmap_exit(void)
3148{
3149 unregister_md_submodule(&bitmap_ops.head);
3150 unregister_md_submodule(&bitmap_none_ops.head);
3151 destroy_workqueue(md_bitmap_wq);
3152}