projects/linux·blob master · refs

2773 lines · 69.2 KiB

1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Copyright (C) 2018 Red Hat. All rights reserved.
4 *
5 * This file is released under the GPL.
6 */
7
8#include <linux/device-mapper.h>
9#include <linux/module.h>
10#include <linux/init.h>
11#include <linux/vmalloc.h>
12#include <linux/kthread.h>
13#include <linux/dm-io.h>
14#include <linux/dm-kcopyd.h>
15#include <linux/dax.h>
16#include <linux/libnvdimm.h>
17#include <linux/delay.h>
18#include "dm-io-tracker.h"
19
20#define DM_MSG_PREFIX "writecache"
21
22#define HIGH_WATERMARK 50
23#define LOW_WATERMARK 45
24#define MAX_WRITEBACK_JOBS min(0x10000000 / PAGE_SIZE, totalram_pages() / 16)
25#define ENDIO_LATENCY 16
26#define WRITEBACK_LATENCY 64
27#define AUTOCOMMIT_BLOCKS_SSD 65536
28#define AUTOCOMMIT_BLOCKS_PMEM 64
29#define AUTOCOMMIT_MSEC 1000
30#define MAX_AGE_DIV 16
31#define MAX_AGE_UNSPECIFIED -1UL
32#define PAUSE_WRITEBACK (HZ * 3)
33
34#define BITMAP_GRANULARITY 65536
35#if BITMAP_GRANULARITY < PAGE_SIZE
36#undef BITMAP_GRANULARITY
37#define BITMAP_GRANULARITY PAGE_SIZE
38#endif
39
40#if IS_ENABLED(CONFIG_ARCH_HAS_PMEM_API) && IS_ENABLED(CONFIG_FS_DAX)
41#define DM_WRITECACHE_HAS_PMEM
42#endif
43
44#ifdef DM_WRITECACHE_HAS_PMEM
45#define pmem_assign(dest, src) \
46do { \
47 typeof(dest) uniq = (src); \
48 memcpy_flushcache(&(dest), &uniq, sizeof(dest)); \
49} while (0)
50#else
51#define pmem_assign(dest, src) ((dest) = (src))
52#endif
53
54#if IS_ENABLED(CONFIG_ARCH_HAS_COPY_MC) && defined(DM_WRITECACHE_HAS_PMEM)
55#define DM_WRITECACHE_HANDLE_HARDWARE_ERRORS
56#endif
57
58#define MEMORY_SUPERBLOCK_MAGIC 0x23489321
59#define MEMORY_SUPERBLOCK_VERSION 1
60
61struct wc_memory_entry {
62 __le64 original_sector;
63 __le64 seq_count;
64};
65
66struct wc_memory_superblock {
67 union {
68 struct {
69 __le32 magic;
70 __le32 version;
71 __le32 block_size;
72 __le32 pad;
73 __le64 n_blocks;
74 __le64 seq_count;
75 };
76 __le64 padding[8];
77 };
78 struct wc_memory_entry entries[];
79};
80
81struct wc_entry {
82 struct rb_node rb_node;
83 struct list_head lru;
84 unsigned short wc_list_contiguous;
85#if BITS_PER_LONG == 64
86 bool write_in_progress : 1;
87 unsigned long index : 47;
88#else
89 bool write_in_progress;
90 unsigned long index;
91#endif
92 unsigned long age;
93#ifdef DM_WRITECACHE_HANDLE_HARDWARE_ERRORS
94 uint64_t original_sector;
95 uint64_t seq_count;
96#endif
97};
98
99#ifdef DM_WRITECACHE_HAS_PMEM
100#define WC_MODE_PMEM(wc) ((wc)->pmem_mode)
101#define WC_MODE_FUA(wc) ((wc)->writeback_fua)
102#else
103#define WC_MODE_PMEM(wc) false
104#define WC_MODE_FUA(wc) false
105#endif
106#define WC_MODE_SORT_FREELIST(wc) (!WC_MODE_PMEM(wc))
107
108struct dm_writecache {
109 struct mutex lock;
110 struct list_head lru;
111 union {
112 struct list_head freelist;
113 struct {
114 struct rb_root freetree;
115 struct wc_entry *current_free;
116 };
117 };
118 struct rb_root tree;
119
120 size_t freelist_size;
121 size_t writeback_size;
122 size_t freelist_high_watermark;
123 size_t freelist_low_watermark;
124 unsigned long max_age;
125 unsigned long pause;
126
127 unsigned int uncommitted_blocks;
128 unsigned int autocommit_blocks;
129 unsigned int max_writeback_jobs;
130
131 int error;
132
133 unsigned long autocommit_jiffies;
134 struct timer_list autocommit_timer;
135 struct wait_queue_head freelist_wait;
136
137 struct timer_list max_age_timer;
138
139 atomic_t bio_in_progress[2];
140 struct wait_queue_head bio_in_progress_wait[2];
141
142 struct dm_target *ti;
143 struct dm_dev *dev;
144 struct dm_dev *ssd_dev;
145 sector_t start_sector;
146 void *memory_map;
147 uint64_t memory_map_size;
148 size_t metadata_sectors;
149 size_t n_blocks;
150 uint64_t seq_count;
151 sector_t data_device_sectors;
152 void *block_start;
153 struct wc_entry *entries;
154 unsigned int block_size;
155 unsigned char block_size_bits;
156
157 bool pmem_mode:1;
158 bool writeback_fua:1;
159
160 bool overwrote_committed:1;
161 bool memory_vmapped:1;
162
163 bool start_sector_set:1;
164 bool high_wm_percent_set:1;
165 bool low_wm_percent_set:1;
166 bool max_writeback_jobs_set:1;
167 bool autocommit_blocks_set:1;
168 bool autocommit_time_set:1;
169 bool max_age_set:1;
170 bool writeback_fua_set:1;
171 bool flush_on_suspend:1;
172 bool cleaner:1;
173 bool cleaner_set:1;
174 bool metadata_only:1;
175 bool pause_set:1;
176
177 unsigned int high_wm_percent_value;
178 unsigned int low_wm_percent_value;
179 unsigned int autocommit_time_value;
180 unsigned int max_age_value;
181 unsigned int pause_value;
182
183 unsigned int writeback_all;
184 struct workqueue_struct *writeback_wq;
185 struct work_struct writeback_work;
186 struct work_struct flush_work;
187
188 struct dm_io_tracker iot;
189
190 struct dm_io_client *dm_io;
191
192 raw_spinlock_t endio_list_lock;
193 struct list_head endio_list;
194 struct task_struct *endio_thread;
195
196 struct task_struct *flush_thread;
197 struct bio_list flush_list;
198
199 struct dm_kcopyd_client *dm_kcopyd;
200 unsigned long *dirty_bitmap;
201 unsigned int dirty_bitmap_size;
202
203 struct bio_set bio_set;
204 mempool_t copy_pool;
205
206 struct {
207 unsigned long long reads;
208 unsigned long long read_hits;
209 unsigned long long writes;
210 unsigned long long write_hits_uncommitted;
211 unsigned long long write_hits_committed;
212 unsigned long long writes_around;
213 unsigned long long writes_allocate;
214 unsigned long long writes_blocked_on_freelist;
215 unsigned long long flushes;
216 unsigned long long discards;
217 } stats;
218};
219
220#define WB_LIST_INLINE 16
221
222struct writeback_struct {
223 struct list_head endio_entry;
224 struct dm_writecache *wc;
225 struct wc_entry **wc_list;
226 unsigned int wc_list_n;
227 struct wc_entry *wc_list_inline[WB_LIST_INLINE];
228 struct bio bio;
229};
230
231struct copy_struct {
232 struct list_head endio_entry;
233 struct dm_writecache *wc;
234 struct wc_entry *e;
235 unsigned int n_entries;
236 int error;
237};
238
239DECLARE_DM_KCOPYD_THROTTLE_WITH_MODULE_PARM(dm_writecache_throttle,
240 "A percentage of time allocated for data copying");
241
242static void wc_lock(struct dm_writecache *wc)
243{
244 mutex_lock(&wc->lock);
245}
246
247static void wc_unlock(struct dm_writecache *wc)
248{
249 mutex_unlock(&wc->lock);
250}
251
252#ifdef DM_WRITECACHE_HAS_PMEM
253static int persistent_memory_claim(struct dm_writecache *wc)
254{
255 int r;
256 loff_t s;
257 long p, da;
258 unsigned long pfn;
259 int id;
260 struct page **pages;
261 sector_t offset;
262
263 wc->memory_vmapped = false;
264
265 s = wc->memory_map_size;
266 p = s >> PAGE_SHIFT;
267 if (!p) {
268 r = -EINVAL;
269 goto err1;
270 }
271 if (p != s >> PAGE_SHIFT) {
272 r = -EOVERFLOW;
273 goto err1;
274 }
275
276 offset = get_start_sect(wc->ssd_dev->bdev);
277 if (offset & (PAGE_SIZE / 512 - 1)) {
278 r = -EINVAL;
279 goto err1;
280 }
281 offset >>= PAGE_SHIFT - 9;
282
283 id = dax_read_lock();
284
285 da = dax_direct_access(wc->ssd_dev->dax_dev, offset, p, DAX_ACCESS,
286 &wc->memory_map, &pfn);
287 if (da < 0) {
288 wc->memory_map = NULL;
289 r = da;
290 goto err2;
291 }
292 if (!pfn_valid(pfn)) {
293 wc->memory_map = NULL;
294 r = -EOPNOTSUPP;
295 goto err2;
296 }
297 if (da != p) {
298 long i;
299
300 wc->memory_map = NULL;
301 pages = vmalloc_array(p, sizeof(struct page *));
302 if (!pages) {
303 r = -ENOMEM;
304 goto err2;
305 }
306 i = 0;
307 do {
308 long daa;
309
310 daa = dax_direct_access(wc->ssd_dev->dax_dev, offset + i,
311 p - i, DAX_ACCESS, NULL, &pfn);
312 if (daa <= 0) {
313 r = daa ? daa : -EINVAL;
314 goto err3;
315 }
316 if (!pfn_valid(pfn)) {
317 r = -EOPNOTSUPP;
318 goto err3;
319 }
320 while (daa-- && i < p) {
321 pages[i++] = pfn_to_page(pfn);
322 pfn++;
323 if (!(i & 15))
324 cond_resched();
325 }
326 } while (i < p);
327 wc->memory_map = vmap(pages, p, VM_MAP, PAGE_KERNEL);
328 if (!wc->memory_map) {
329 r = -ENOMEM;
330 goto err3;
331 }
332 vfree(pages);
333 wc->memory_vmapped = true;
334 }
335
336 dax_read_unlock(id);
337
338 wc->memory_map += (size_t)wc->start_sector << SECTOR_SHIFT;
339 wc->memory_map_size -= (size_t)wc->start_sector << SECTOR_SHIFT;
340
341 return 0;
342err3:
343 vfree(pages);
344err2:
345 dax_read_unlock(id);
346err1:
347 return r;
348}
349#else
350static int persistent_memory_claim(struct dm_writecache *wc)
351{
352 return -EOPNOTSUPP;
353}
354#endif
355
356static void persistent_memory_release(struct dm_writecache *wc)
357{
358 if (wc->memory_vmapped)
359 vunmap(wc->memory_map - ((size_t)wc->start_sector << SECTOR_SHIFT));
360}
361
362static struct page *persistent_memory_page(void *addr)
363{
364 if (is_vmalloc_addr(addr))
365 return vmalloc_to_page(addr);
366 else
367 return virt_to_page(addr);
368}
369
370static unsigned int persistent_memory_page_offset(void *addr)
371{
372 return (unsigned long)addr & (PAGE_SIZE - 1);
373}
374
375static void persistent_memory_flush_cache(void *ptr, size_t size)
376{
377 if (is_vmalloc_addr(ptr))
378 flush_kernel_vmap_range(ptr, size);
379}
380
381static void persistent_memory_invalidate_cache(void *ptr, size_t size)
382{
383 if (is_vmalloc_addr(ptr))
384 invalidate_kernel_vmap_range(ptr, size);
385}
386
387static struct wc_memory_superblock *sb(struct dm_writecache *wc)
388{
389 return wc->memory_map;
390}
391
392static struct wc_memory_entry *memory_entry(struct dm_writecache *wc, struct wc_entry *e)
393{
394 return &sb(wc)->entries[e->index];
395}
396
397static void *memory_data(struct dm_writecache *wc, struct wc_entry *e)
398{
399 return (char *)wc->block_start + (e->index << wc->block_size_bits);
400}
401
402static sector_t cache_sector(struct dm_writecache *wc, struct wc_entry *e)
403{
404 return wc->start_sector + wc->metadata_sectors +
405 ((sector_t)e->index << (wc->block_size_bits - SECTOR_SHIFT));
406}
407
408static uint64_t read_original_sector(struct dm_writecache *wc, struct wc_entry *e)
409{
410#ifdef DM_WRITECACHE_HANDLE_HARDWARE_ERRORS
411 return e->original_sector;
412#else
413 return le64_to_cpu(memory_entry(wc, e)->original_sector);
414#endif
415}
416
417static uint64_t read_seq_count(struct dm_writecache *wc, struct wc_entry *e)
418{
419#ifdef DM_WRITECACHE_HANDLE_HARDWARE_ERRORS
420 return e->seq_count;
421#else
422 return le64_to_cpu(memory_entry(wc, e)->seq_count);
423#endif
424}
425
426static void clear_seq_count(struct dm_writecache *wc, struct wc_entry *e)
427{
428#ifdef DM_WRITECACHE_HANDLE_HARDWARE_ERRORS
429 e->seq_count = -1;
430#endif
431 pmem_assign(memory_entry(wc, e)->seq_count, cpu_to_le64(-1));
432}
433
434static void write_original_sector_seq_count(struct dm_writecache *wc, struct wc_entry *e,
435 uint64_t original_sector, uint64_t seq_count)
436{
437 struct wc_memory_entry me;
438#ifdef DM_WRITECACHE_HANDLE_HARDWARE_ERRORS
439 e->original_sector = original_sector;
440 e->seq_count = seq_count;
441#endif
442 me.original_sector = cpu_to_le64(original_sector);
443 me.seq_count = cpu_to_le64(seq_count);
444 pmem_assign(*memory_entry(wc, e), me);
445}
446
447#define writecache_error(wc, err, msg, arg...) \
448do { \
449 if (!cmpxchg(&(wc)->error, 0, err)) \
450 DMERR(msg, ##arg); \
451 wake_up(&(wc)->freelist_wait); \
452} while (0)
453
454#define writecache_has_error(wc) (unlikely(READ_ONCE((wc)->error)))
455
456static void writecache_flush_all_metadata(struct dm_writecache *wc)
457{
458 if (!WC_MODE_PMEM(wc))
459 memset(wc->dirty_bitmap, -1, wc->dirty_bitmap_size);
460}
461
462static void writecache_flush_region(struct dm_writecache *wc, void *ptr, size_t size)
463{
464 if (!WC_MODE_PMEM(wc))
465 __set_bit(((char *)ptr - (char *)wc->memory_map) / BITMAP_GRANULARITY,
466 wc->dirty_bitmap);
467}
468
469static void writecache_disk_flush(struct dm_writecache *wc, struct dm_dev *dev);
470
471struct io_notify {
472 struct dm_writecache *wc;
473 struct completion c;
474 atomic_t count;
475};
476
477static void writecache_notify_io(unsigned long error, unsigned long unsup, void *context)
478{
479 struct io_notify *endio = context;
480
481 if (unlikely(error != 0))
482 writecache_error(endio->wc, -EIO, "error writing metadata");
483 else if (unlikely(unsup != 0))
484 writecache_error(endio->wc, -EOPNOTSUPP, "error writing metadata");
485 BUG_ON(atomic_read(&endio->count) <= 0);
486 if (atomic_dec_and_test(&endio->count))
487 complete(&endio->c);
488}
489
490static void writecache_wait_for_ios(struct dm_writecache *wc, int direction)
491{
492 wait_event(wc->bio_in_progress_wait[direction],
493 !atomic_read(&wc->bio_in_progress[direction]));
494}
495
496static void ssd_commit_flushed(struct dm_writecache *wc, bool wait_for_ios)
497{
498 struct dm_io_region region;
499 struct dm_io_request req;
500 struct io_notify endio = {
501 wc,
502 COMPLETION_INITIALIZER_ONSTACK(endio.c),
503 ATOMIC_INIT(1),
504 };
505 unsigned int bitmap_bits = wc->dirty_bitmap_size * 8;
506 unsigned int i = 0;
507
508 while (1) {
509 unsigned int j;
510
511 i = find_next_bit(wc->dirty_bitmap, bitmap_bits, i);
512 if (unlikely(i == bitmap_bits))
513 break;
514 j = find_next_zero_bit(wc->dirty_bitmap, bitmap_bits, i);
515
516 region.bdev = wc->ssd_dev->bdev;
517 region.sector = (sector_t)i * (BITMAP_GRANULARITY >> SECTOR_SHIFT);
518 region.count = (sector_t)(j - i) * (BITMAP_GRANULARITY >> SECTOR_SHIFT);
519
520 if (unlikely(region.sector >= wc->metadata_sectors))
521 break;
522 if (unlikely(region.sector + region.count > wc->metadata_sectors))
523 region.count = wc->metadata_sectors - region.sector;
524
525 region.sector += wc->start_sector;
526 atomic_inc(&endio.count);
527 req.bi_opf = REQ_OP_WRITE | REQ_SYNC;
528 req.mem.type = DM_IO_VMA;
529 req.mem.ptr.vma = (char *)wc->memory_map + (size_t)i * BITMAP_GRANULARITY;
530 req.client = wc->dm_io;
531 req.notify.fn = writecache_notify_io;
532 req.notify.context = &endio;
533
534 /* writing via async dm-io (implied by notify.fn above) won't return an error */
535 (void) dm_io(&req, 1, &region, NULL, NULL, IOPRIO_DEFAULT);
536 i = j;
537 }
538
539 writecache_notify_io(0, 0, &endio);
540 wait_for_completion_io(&endio.c);
541
542 if (wait_for_ios)
543 writecache_wait_for_ios(wc, WRITE);
544
545 writecache_disk_flush(wc, wc->ssd_dev);
546
547 memset(wc->dirty_bitmap, 0, wc->dirty_bitmap_size);
548}
549
550static void ssd_commit_superblock(struct dm_writecache *wc)
551{
552 int r;
553 struct dm_io_region region;
554 struct dm_io_request req;
555
556 region.bdev = wc->ssd_dev->bdev;
557 region.sector = 0;
558 region.count = max(4096U, wc->block_size) >> SECTOR_SHIFT;
559
560 if (unlikely(region.sector + region.count > wc->metadata_sectors))
561 region.count = wc->metadata_sectors - region.sector;
562
563 region.sector += wc->start_sector;
564
565 req.bi_opf = REQ_OP_WRITE | REQ_SYNC | REQ_FUA;
566 req.mem.type = DM_IO_VMA;
567 req.mem.ptr.vma = (char *)wc->memory_map;
568 req.client = wc->dm_io;
569 req.notify.fn = NULL;
570 req.notify.context = NULL;
571
572 r = dm_io(&req, 1, &region, NULL, NULL, IOPRIO_DEFAULT);
573 if (unlikely(r))
574 writecache_error(wc, r, "error writing superblock");
575}
576
577static void writecache_commit_flushed(struct dm_writecache *wc, bool wait_for_ios)
578{
579 if (WC_MODE_PMEM(wc))
580 pmem_wmb();
581 else
582 ssd_commit_flushed(wc, wait_for_ios);
583}
584
585static void writecache_disk_flush(struct dm_writecache *wc, struct dm_dev *dev)
586{
587 int r;
588 struct dm_io_region region;
589 struct dm_io_request req;
590
591 region.bdev = dev->bdev;
592 region.sector = 0;
593 region.count = 0;
594 req.bi_opf = REQ_OP_WRITE | REQ_PREFLUSH;
595 req.mem.type = DM_IO_KMEM;
596 req.mem.ptr.addr = NULL;
597 req.client = wc->dm_io;
598 req.notify.fn = NULL;
599
600 r = dm_io(&req, 1, &region, NULL, NULL, IOPRIO_DEFAULT);
601 if (unlikely(r))
602 writecache_error(wc, r, "error flushing metadata: %d", r);
603}
604
605#define WFE_RETURN_FOLLOWING 1
606#define WFE_LOWEST_SEQ 2
607
608static struct wc_entry *writecache_find_entry(struct dm_writecache *wc,
609 uint64_t block, int flags)
610{
611 struct wc_entry *e;
612 struct rb_node *node = wc->tree.rb_node;
613
614 if (unlikely(!node))
615 return NULL;
616
617 while (1) {
618 e = container_of(node, struct wc_entry, rb_node);
619 if (read_original_sector(wc, e) == block)
620 break;
621
622 node = (read_original_sector(wc, e) >= block ?
623 e->rb_node.rb_left : e->rb_node.rb_right);
624 if (unlikely(!node)) {
625 if (!(flags & WFE_RETURN_FOLLOWING))
626 return NULL;
627 if (read_original_sector(wc, e) >= block)
628 return e;
629
630 node = rb_next(&e->rb_node);
631 if (unlikely(!node))
632 return NULL;
633
634 e = container_of(node, struct wc_entry, rb_node);
635 return e;
636 }
637 }
638
639 while (1) {
640 struct wc_entry *e2;
641
642 if (flags & WFE_LOWEST_SEQ)
643 node = rb_prev(&e->rb_node);
644 else
645 node = rb_next(&e->rb_node);
646 if (unlikely(!node))
647 return e;
648 e2 = container_of(node, struct wc_entry, rb_node);
649 if (read_original_sector(wc, e2) != block)
650 return e;
651 e = e2;
652 }
653}
654
655static void writecache_insert_entry(struct dm_writecache *wc, struct wc_entry *ins)
656{
657 struct wc_entry *e;
658 struct rb_node **node = &wc->tree.rb_node, *parent = NULL;
659
660 while (*node) {
661 e = container_of(*node, struct wc_entry, rb_node);
662 parent = &e->rb_node;
663 if (read_original_sector(wc, e) > read_original_sector(wc, ins))
664 node = &parent->rb_left;
665 else
666 node = &parent->rb_right;
667 }
668 rb_link_node(&ins->rb_node, parent, node);
669 rb_insert_color(&ins->rb_node, &wc->tree);
670 list_add(&ins->lru, &wc->lru);
671 ins->age = jiffies;
672}
673
674static void writecache_unlink(struct dm_writecache *wc, struct wc_entry *e)
675{
676 list_del(&e->lru);
677 rb_erase(&e->rb_node, &wc->tree);
678}
679
680static void writecache_add_to_freelist(struct dm_writecache *wc, struct wc_entry *e)
681{
682 if (WC_MODE_SORT_FREELIST(wc)) {
683 struct rb_node **node = &wc->freetree.rb_node, *parent = NULL;
684
685 if (unlikely(!*node))
686 wc->current_free = e;
687 while (*node) {
688 parent = *node;
689 if (&e->rb_node < *node)
690 node = &parent->rb_left;
691 else
692 node = &parent->rb_right;
693 }
694 rb_link_node(&e->rb_node, parent, node);
695 rb_insert_color(&e->rb_node, &wc->freetree);
696 } else {
697 list_add_tail(&e->lru, &wc->freelist);
698 }
699 wc->freelist_size++;
700}
701
702static inline void writecache_verify_watermark(struct dm_writecache *wc)
703{
704 if (unlikely(wc->freelist_size + wc->writeback_size <= wc->freelist_high_watermark))
705 queue_work(wc->writeback_wq, &wc->writeback_work);
706}
707
708static void writecache_max_age_timer(struct timer_list *t)
709{
710 struct dm_writecache *wc = timer_container_of(wc, t, max_age_timer);
711
712 if (!dm_suspended(wc->ti) && !writecache_has_error(wc)) {
713 queue_work(wc->writeback_wq, &wc->writeback_work);
714 mod_timer(&wc->max_age_timer, jiffies + wc->max_age / MAX_AGE_DIV);
715 }
716}
717
718static struct wc_entry *writecache_pop_from_freelist(struct dm_writecache *wc, sector_t expected_sector)
719{
720 struct wc_entry *e;
721
722 if (WC_MODE_SORT_FREELIST(wc)) {
723 struct rb_node *next;
724
725 if (unlikely(!wc->current_free))
726 return NULL;
727 e = wc->current_free;
728 if (expected_sector != (sector_t)-1 && unlikely(cache_sector(wc, e) != expected_sector))
729 return NULL;
730 next = rb_next(&e->rb_node);
731 rb_erase(&e->rb_node, &wc->freetree);
732 if (unlikely(!next))
733 next = rb_first(&wc->freetree);
734 wc->current_free = next ? container_of(next, struct wc_entry, rb_node) : NULL;
735 } else {
736 if (unlikely(list_empty(&wc->freelist)))
737 return NULL;
738 e = container_of(wc->freelist.next, struct wc_entry, lru);
739 if (expected_sector != (sector_t)-1 && unlikely(cache_sector(wc, e) != expected_sector))
740 return NULL;
741 list_del(&e->lru);
742 }
743 wc->freelist_size--;
744
745 writecache_verify_watermark(wc);
746
747 return e;
748}
749
750static void writecache_free_entry(struct dm_writecache *wc, struct wc_entry *e)
751{
752 writecache_unlink(wc, e);
753 writecache_add_to_freelist(wc, e);
754 clear_seq_count(wc, e);
755 writecache_flush_region(wc, memory_entry(wc, e), sizeof(struct wc_memory_entry));
756 if (unlikely(waitqueue_active(&wc->freelist_wait)))
757 wake_up(&wc->freelist_wait);
758}
759
760static void writecache_wait_on_freelist(struct dm_writecache *wc)
761{
762 DEFINE_WAIT(wait);
763
764 prepare_to_wait(&wc->freelist_wait, &wait, TASK_UNINTERRUPTIBLE);
765 wc_unlock(wc);
766 io_schedule();
767 finish_wait(&wc->freelist_wait, &wait);
768 wc_lock(wc);
769}
770
771static void writecache_poison_lists(struct dm_writecache *wc)
772{
773 /*
774 * Catch incorrect access to these values while the device is suspended.
775 */
776 memset(&wc->tree, -1, sizeof(wc->tree));
777 wc->lru.next = LIST_POISON1;
778 wc->lru.prev = LIST_POISON2;
779 wc->freelist.next = LIST_POISON1;
780 wc->freelist.prev = LIST_POISON2;
781}
782
783static void writecache_flush_entry(struct dm_writecache *wc, struct wc_entry *e)
784{
785 writecache_flush_region(wc, memory_entry(wc, e), sizeof(struct wc_memory_entry));
786 if (WC_MODE_PMEM(wc))
787 writecache_flush_region(wc, memory_data(wc, e), wc->block_size);
788}
789
790static bool writecache_entry_is_committed(struct dm_writecache *wc, struct wc_entry *e)
791{
792 return read_seq_count(wc, e) < wc->seq_count;
793}
794
795static void writecache_flush(struct dm_writecache *wc)
796{
797 struct wc_entry *e, *e2;
798 bool need_flush_after_free;
799
800 wc->uncommitted_blocks = 0;
801 timer_delete(&wc->autocommit_timer);
802
803 if (list_empty(&wc->lru))
804 return;
805
806 e = container_of(wc->lru.next, struct wc_entry, lru);
807 if (writecache_entry_is_committed(wc, e)) {
808 if (wc->overwrote_committed) {
809 writecache_wait_for_ios(wc, WRITE);
810 writecache_disk_flush(wc, wc->ssd_dev);
811 wc->overwrote_committed = false;
812 }
813 return;
814 }
815 while (1) {
816 writecache_flush_entry(wc, e);
817 if (unlikely(e->lru.next == &wc->lru))
818 break;
819 e2 = container_of(e->lru.next, struct wc_entry, lru);
820 if (writecache_entry_is_committed(wc, e2))
821 break;
822 e = e2;
823 cond_resched();
824 }
825 writecache_commit_flushed(wc, true);
826
827 wc->seq_count++;
828 pmem_assign(sb(wc)->seq_count, cpu_to_le64(wc->seq_count));
829 if (WC_MODE_PMEM(wc))
830 writecache_commit_flushed(wc, false);
831 else
832 ssd_commit_superblock(wc);
833
834 wc->overwrote_committed = false;
835
836 need_flush_after_free = false;
837 while (1) {
838 /* Free another committed entry with lower seq-count */
839 struct rb_node *rb_node = rb_prev(&e->rb_node);
840
841 if (rb_node) {
842 e2 = container_of(rb_node, struct wc_entry, rb_node);
843 if (read_original_sector(wc, e2) == read_original_sector(wc, e) &&
844 likely(!e2->write_in_progress)) {
845 writecache_free_entry(wc, e2);
846 need_flush_after_free = true;
847 }
848 }
849 if (unlikely(e->lru.prev == &wc->lru))
850 break;
851 e = container_of(e->lru.prev, struct wc_entry, lru);
852 cond_resched();
853 }
854
855 if (need_flush_after_free)
856 writecache_commit_flushed(wc, false);
857}
858
859static void writecache_flush_work(struct work_struct *work)
860{
861 struct dm_writecache *wc = container_of(work, struct dm_writecache, flush_work);
862
863 wc_lock(wc);
864 writecache_flush(wc);
865 wc_unlock(wc);
866}
867
868static void writecache_autocommit_timer(struct timer_list *t)
869{
870 struct dm_writecache *wc = timer_container_of(wc, t, autocommit_timer);
871
872 if (!writecache_has_error(wc))
873 queue_work(wc->writeback_wq, &wc->flush_work);
874}
875
876static void writecache_schedule_autocommit(struct dm_writecache *wc)
877{
878 if (!timer_pending(&wc->autocommit_timer))
879 mod_timer(&wc->autocommit_timer, jiffies + wc->autocommit_jiffies);
880}
881
882static void writecache_discard(struct dm_writecache *wc, sector_t start, sector_t end)
883{
884 struct wc_entry *e;
885 bool discarded_something = false;
886
887 e = writecache_find_entry(wc, start, WFE_RETURN_FOLLOWING | WFE_LOWEST_SEQ);
888 if (unlikely(!e))
889 return;
890
891 while (read_original_sector(wc, e) < end) {
892 struct rb_node *node = rb_next(&e->rb_node);
893
894 if (likely(!e->write_in_progress)) {
895 if (!discarded_something) {
896 if (!WC_MODE_PMEM(wc)) {
897 writecache_wait_for_ios(wc, READ);
898 writecache_wait_for_ios(wc, WRITE);
899 }
900 discarded_something = true;
901 }
902 if (!writecache_entry_is_committed(wc, e))
903 wc->uncommitted_blocks--;
904 writecache_free_entry(wc, e);
905 }
906
907 if (unlikely(!node))
908 break;
909
910 e = container_of(node, struct wc_entry, rb_node);
911 }
912
913 if (discarded_something)
914 writecache_commit_flushed(wc, false);
915}
916
917static bool writecache_wait_for_writeback(struct dm_writecache *wc)
918{
919 if (wc->writeback_size) {
920 writecache_wait_on_freelist(wc);
921 return true;
922 }
923 return false;
924}
925
926static void writecache_suspend(struct dm_target *ti)
927{
928 struct dm_writecache *wc = ti->private;
929 bool flush_on_suspend;
930
931 timer_delete_sync(&wc->autocommit_timer);
932 timer_delete_sync(&wc->max_age_timer);
933
934 wc_lock(wc);
935 writecache_flush(wc);
936 flush_on_suspend = wc->flush_on_suspend;
937 if (flush_on_suspend) {
938 wc->flush_on_suspend = false;
939 wc->writeback_all++;
940 queue_work(wc->writeback_wq, &wc->writeback_work);
941 }
942 wc_unlock(wc);
943
944 drain_workqueue(wc->writeback_wq);
945
946 wc_lock(wc);
947 if (flush_on_suspend)
948 wc->writeback_all--;
949 while (writecache_wait_for_writeback(wc))
950 ;
951
952 if (WC_MODE_PMEM(wc))
953 persistent_memory_flush_cache(wc->memory_map, wc->memory_map_size);
954
955 writecache_poison_lists(wc);
956
957 wc_unlock(wc);
958}
959
960static int writecache_alloc_entries(struct dm_writecache *wc)
961{
962 size_t b;
963
964 if (wc->entries)
965 return 0;
966 wc->entries = vmalloc_array(wc->n_blocks, sizeof(struct wc_entry));
967 if (!wc->entries)
968 return -ENOMEM;
969 for (b = 0; b < wc->n_blocks; b++) {
970 struct wc_entry *e = &wc->entries[b];
971
972 e->index = b;
973 e->write_in_progress = false;
974 cond_resched();
975 }
976
977 return 0;
978}
979
980static int writecache_read_metadata(struct dm_writecache *wc, sector_t n_sectors)
981{
982 struct dm_io_region region;
983 struct dm_io_request req;
984
985 region.bdev = wc->ssd_dev->bdev;
986 region.sector = wc->start_sector;
987 region.count = n_sectors;
988 req.bi_opf = REQ_OP_READ | REQ_SYNC;
989 req.mem.type = DM_IO_VMA;
990 req.mem.ptr.vma = (char *)wc->memory_map;
991 req.client = wc->dm_io;
992 req.notify.fn = NULL;
993
994 return dm_io(&req, 1, &region, NULL, NULL, IOPRIO_DEFAULT);
995}
996
997static void writecache_resume(struct dm_target *ti)
998{
999 struct dm_writecache *wc = ti->private;
1000 size_t b;
1001 bool need_flush = false;
1002 __le64 sb_seq_count;
1003 int r;
1004
1005 wc_lock(wc);
1006
1007 wc->data_device_sectors = bdev_nr_sectors(wc->dev->bdev);
1008
1009 if (WC_MODE_PMEM(wc)) {
1010 persistent_memory_invalidate_cache(wc->memory_map, wc->memory_map_size);
1011 } else {
1012 r = writecache_read_metadata(wc, wc->metadata_sectors);
1013 if (r) {
1014 size_t sb_entries_offset;
1015
1016 writecache_error(wc, r, "unable to read metadata: %d", r);
1017 sb_entries_offset = offsetof(struct wc_memory_superblock, entries);
1018 memset((char *)wc->memory_map + sb_entries_offset, -1,
1019 (wc->metadata_sectors << SECTOR_SHIFT) - sb_entries_offset);
1020 }
1021 }
1022
1023 wc->tree = RB_ROOT;
1024 INIT_LIST_HEAD(&wc->lru);
1025 if (WC_MODE_SORT_FREELIST(wc)) {
1026 wc->freetree = RB_ROOT;
1027 wc->current_free = NULL;
1028 } else {
1029 INIT_LIST_HEAD(&wc->freelist);
1030 }
1031 wc->freelist_size = 0;
1032
1033 r = copy_mc_to_kernel(&sb_seq_count, &sb(wc)->seq_count,
1034 sizeof(uint64_t));
1035 if (r) {
1036 writecache_error(wc, r, "hardware memory error when reading superblock: %d", r);
1037 sb_seq_count = cpu_to_le64(0);
1038 }
1039 wc->seq_count = le64_to_cpu(sb_seq_count);
1040
1041#ifdef DM_WRITECACHE_HANDLE_HARDWARE_ERRORS
1042 for (b = 0; b < wc->n_blocks; b++) {
1043 struct wc_entry *e = &wc->entries[b];
1044 struct wc_memory_entry wme;
1045
1046 if (writecache_has_error(wc)) {
1047 e->original_sector = -1;
1048 e->seq_count = -1;
1049 continue;
1050 }
1051 r = copy_mc_to_kernel(&wme, memory_entry(wc, e),
1052 sizeof(struct wc_memory_entry));
1053 if (r) {
1054 writecache_error(wc, r, "hardware memory error when reading metadata entry %lu: %d",
1055 (unsigned long)b, r);
1056 e->original_sector = -1;
1057 e->seq_count = -1;
1058 } else {
1059 e->original_sector = le64_to_cpu(wme.original_sector);
1060 e->seq_count = le64_to_cpu(wme.seq_count);
1061 }
1062 cond_resched();
1063 }
1064#endif
1065 for (b = 0; b < wc->n_blocks; b++) {
1066 struct wc_entry *e = &wc->entries[b];
1067
1068 if (!writecache_entry_is_committed(wc, e)) {
1069 if (read_seq_count(wc, e) != -1) {
1070erase_this:
1071 clear_seq_count(wc, e);
1072 need_flush = true;
1073 }
1074 writecache_add_to_freelist(wc, e);
1075 } else {
1076 struct wc_entry *old;
1077
1078 old = writecache_find_entry(wc, read_original_sector(wc, e), 0);
1079 if (!old) {
1080 writecache_insert_entry(wc, e);
1081 } else {
1082 if (read_seq_count(wc, old) == read_seq_count(wc, e)) {
1083 writecache_error(wc, -EINVAL,
1084 "two identical entries, position %llu, sector %llu, sequence %llu",
1085 (unsigned long long)b, (unsigned long long)read_original_sector(wc, e),
1086 (unsigned long long)read_seq_count(wc, e));
1087 }
1088 if (read_seq_count(wc, old) > read_seq_count(wc, e)) {
1089 goto erase_this;
1090 } else {
1091 writecache_free_entry(wc, old);
1092 writecache_insert_entry(wc, e);
1093 need_flush = true;
1094 }
1095 }
1096 }
1097 cond_resched();
1098 }
1099
1100 if (need_flush) {
1101 writecache_flush_all_metadata(wc);
1102 writecache_commit_flushed(wc, false);
1103 }
1104
1105 writecache_verify_watermark(wc);
1106
1107 if (wc->max_age != MAX_AGE_UNSPECIFIED)
1108 mod_timer(&wc->max_age_timer, jiffies + wc->max_age / MAX_AGE_DIV);
1109
1110 wc_unlock(wc);
1111}
1112
1113static int process_flush_mesg(unsigned int argc, char **argv, struct dm_writecache *wc)
1114{
1115 if (argc != 1)
1116 return -EINVAL;
1117
1118 wc_lock(wc);
1119 if (dm_suspended(wc->ti)) {
1120 wc_unlock(wc);
1121 return -EBUSY;
1122 }
1123 if (writecache_has_error(wc)) {
1124 wc_unlock(wc);
1125 return -EIO;
1126 }
1127
1128 writecache_flush(wc);
1129 wc->writeback_all++;
1130 queue_work(wc->writeback_wq, &wc->writeback_work);
1131 wc_unlock(wc);
1132
1133 flush_workqueue(wc->writeback_wq);
1134
1135 wc_lock(wc);
1136 wc->writeback_all--;
1137 if (writecache_has_error(wc)) {
1138 wc_unlock(wc);
1139 return -EIO;
1140 }
1141 wc_unlock(wc);
1142
1143 return 0;
1144}
1145
1146static int process_flush_on_suspend_mesg(unsigned int argc, char **argv, struct dm_writecache *wc)
1147{
1148 if (argc != 1)
1149 return -EINVAL;
1150
1151 wc_lock(wc);
1152 wc->flush_on_suspend = true;
1153 wc_unlock(wc);
1154
1155 return 0;
1156}
1157
1158static void activate_cleaner(struct dm_writecache *wc)
1159{
1160 wc->flush_on_suspend = true;
1161 wc->cleaner = true;
1162 wc->freelist_high_watermark = wc->n_blocks;
1163 wc->freelist_low_watermark = wc->n_blocks;
1164}
1165
1166static int process_cleaner_mesg(unsigned int argc, char **argv, struct dm_writecache *wc)
1167{
1168 if (argc != 1)
1169 return -EINVAL;
1170
1171 wc_lock(wc);
1172 activate_cleaner(wc);
1173 if (!dm_suspended(wc->ti))
1174 writecache_verify_watermark(wc);
1175 wc_unlock(wc);
1176
1177 return 0;
1178}
1179
1180static int process_clear_stats_mesg(unsigned int argc, char **argv, struct dm_writecache *wc)
1181{
1182 if (argc != 1)
1183 return -EINVAL;
1184
1185 wc_lock(wc);
1186 memset(&wc->stats, 0, sizeof(wc->stats));
1187 wc_unlock(wc);
1188
1189 return 0;
1190}
1191
1192static int writecache_message(struct dm_target *ti, unsigned int argc, char **argv,
1193 char *result, unsigned int maxlen)
1194{
1195 int r = -EINVAL;
1196 struct dm_writecache *wc = ti->private;
1197
1198 if (!strcasecmp(argv[0], "flush"))
1199 r = process_flush_mesg(argc, argv, wc);
1200 else if (!strcasecmp(argv[0], "flush_on_suspend"))
1201 r = process_flush_on_suspend_mesg(argc, argv, wc);
1202 else if (!strcasecmp(argv[0], "cleaner"))
1203 r = process_cleaner_mesg(argc, argv, wc);
1204 else if (!strcasecmp(argv[0], "clear_stats"))
1205 r = process_clear_stats_mesg(argc, argv, wc);
1206 else
1207 DMERR("unrecognised message received: %s", argv[0]);
1208
1209 return r;
1210}
1211
1212static void memcpy_flushcache_optimized(void *dest, void *source, size_t size)
1213{
1214 /*
1215 * clflushopt performs better with block size 1024, 2048, 4096
1216 * non-temporal stores perform better with block size 512
1217 *
1218 * block size 512 1024 2048 4096
1219 * movnti 496 MB/s 642 MB/s 725 MB/s 744 MB/s
1220 * clflushopt 373 MB/s 688 MB/s 1.1 GB/s 1.2 GB/s
1221 *
1222 * We see that movnti performs better for 512-byte blocks, and
1223 * clflushopt performs better for 1024-byte and larger blocks. So, we
1224 * prefer clflushopt for sizes >= 768.
1225 *
1226 * NOTE: this happens to be the case now (with dm-writecache's single
1227 * threaded model) but re-evaluate this once memcpy_flushcache() is
1228 * enabled to use movdir64b which might invalidate this performance
1229 * advantage seen with cache-allocating-writes plus flushing.
1230 */
1231#ifdef CONFIG_X86
1232 if (cpu_feature_enabled(X86_FEATURE_CLFLUSHOPT) &&
1233 likely(boot_cpu_data.x86_clflush_size == 64) &&
1234 likely(size >= 768)) {
1235 do {
1236 memcpy((void *)dest, (void *)source, 64);
1237 clflushopt((void *)dest);
1238 dest += 64;
1239 source += 64;
1240 size -= 64;
1241 } while (size >= 64);
1242 return;
1243 }
1244#endif
1245 memcpy_flushcache(dest, source, size);
1246}
1247
1248static void bio_copy_block(struct dm_writecache *wc, struct bio *bio, void *data)
1249{
1250 void *buf;
1251 unsigned int size;
1252 int rw = bio_data_dir(bio);
1253 unsigned int remaining_size = wc->block_size;
1254
1255 do {
1256 struct bio_vec bv = bio_iter_iovec(bio, bio->bi_iter);
1257
1258 buf = bvec_kmap_local(&bv);
1259 size = bv.bv_len;
1260 if (unlikely(size > remaining_size))
1261 size = remaining_size;
1262
1263 if (rw == READ) {
1264 int r;
1265
1266 r = copy_mc_to_kernel(buf, data, size);
1267 flush_dcache_page(bio_page(bio));
1268 if (unlikely(r)) {
1269 writecache_error(wc, r, "hardware memory error when reading data: %d", r);
1270 bio->bi_status = BLK_STS_IOERR;
1271 }
1272 } else {
1273 flush_dcache_page(bio_page(bio));
1274 memcpy_flushcache_optimized(data, buf, size);
1275 }
1276
1277 kunmap_local(buf);
1278
1279 data = (char *)data + size;
1280 remaining_size -= size;
1281 bio_advance(bio, size);
1282 } while (unlikely(remaining_size));
1283}
1284
1285static int writecache_flush_thread(void *data)
1286{
1287 struct dm_writecache *wc = data;
1288
1289 while (1) {
1290 struct bio *bio;
1291
1292 wc_lock(wc);
1293 bio = bio_list_pop(&wc->flush_list);
1294 if (!bio) {
1295 set_current_state(TASK_INTERRUPTIBLE);
1296 wc_unlock(wc);
1297
1298 if (unlikely(kthread_should_stop())) {
1299 set_current_state(TASK_RUNNING);
1300 break;
1301 }
1302
1303 schedule();
1304 continue;
1305 }
1306
1307 if (bio_op(bio) == REQ_OP_DISCARD) {
1308 writecache_discard(wc, bio->bi_iter.bi_sector,
1309 bio_end_sector(bio));
1310 wc_unlock(wc);
1311 bio_set_dev(bio, wc->dev->bdev);
1312 submit_bio_noacct(bio);
1313 } else {
1314 writecache_flush(wc);
1315 wc_unlock(wc);
1316 if (writecache_has_error(wc))
1317 bio->bi_status = BLK_STS_IOERR;
1318 bio_endio(bio);
1319 }
1320 }
1321
1322 return 0;
1323}
1324
1325static void writecache_offload_bio(struct dm_writecache *wc, struct bio *bio)
1326{
1327 if (bio_list_empty(&wc->flush_list))
1328 wake_up_process(wc->flush_thread);
1329 bio_list_add(&wc->flush_list, bio);
1330}
1331
1332enum wc_map_op {
1333 WC_MAP_SUBMIT,
1334 WC_MAP_REMAP,
1335 WC_MAP_REMAP_ORIGIN,
1336 WC_MAP_RETURN,
1337 WC_MAP_ERROR,
1338};
1339
1340static void writecache_map_remap_origin(struct dm_writecache *wc, struct bio *bio,
1341 struct wc_entry *e)
1342{
1343 if (e) {
1344 sector_t next_boundary =
1345 read_original_sector(wc, e) - bio->bi_iter.bi_sector;
1346 if (next_boundary < bio->bi_iter.bi_size >> SECTOR_SHIFT)
1347 dm_accept_partial_bio(bio, next_boundary);
1348 }
1349}
1350
1351static enum wc_map_op writecache_map_read(struct dm_writecache *wc, struct bio *bio)
1352{
1353 enum wc_map_op map_op;
1354 struct wc_entry *e;
1355
1356read_next_block:
1357 wc->stats.reads++;
1358 e = writecache_find_entry(wc, bio->bi_iter.bi_sector, WFE_RETURN_FOLLOWING);
1359 if (e && read_original_sector(wc, e) == bio->bi_iter.bi_sector) {
1360 wc->stats.read_hits++;
1361 if (WC_MODE_PMEM(wc)) {
1362 bio_copy_block(wc, bio, memory_data(wc, e));
1363 if (bio->bi_iter.bi_size)
1364 goto read_next_block;
1365 map_op = WC_MAP_SUBMIT;
1366 } else {
1367 dm_accept_partial_bio(bio, wc->block_size >> SECTOR_SHIFT);
1368 bio_set_dev(bio, wc->ssd_dev->bdev);
1369 bio->bi_iter.bi_sector = cache_sector(wc, e);
1370 if (!writecache_entry_is_committed(wc, e))
1371 writecache_wait_for_ios(wc, WRITE);
1372 map_op = WC_MAP_REMAP;
1373 }
1374 } else {
1375 writecache_map_remap_origin(wc, bio, e);
1376 wc->stats.reads += (bio->bi_iter.bi_size - wc->block_size) >> wc->block_size_bits;
1377 map_op = WC_MAP_REMAP_ORIGIN;
1378 }
1379
1380 return map_op;
1381}
1382
1383static void writecache_bio_copy_ssd(struct dm_writecache *wc, struct bio *bio,
1384 struct wc_entry *e, bool search_used)
1385{
1386 unsigned int bio_size = wc->block_size;
1387 sector_t start_cache_sec = cache_sector(wc, e);
1388 sector_t current_cache_sec = start_cache_sec + (bio_size >> SECTOR_SHIFT);
1389
1390 while (bio_size < bio->bi_iter.bi_size) {
1391 if (!search_used) {
1392 struct wc_entry *f = writecache_pop_from_freelist(wc, current_cache_sec);
1393
1394 if (!f)
1395 break;
1396 write_original_sector_seq_count(wc, f, bio->bi_iter.bi_sector +
1397 (bio_size >> SECTOR_SHIFT), wc->seq_count);
1398 writecache_insert_entry(wc, f);
1399 wc->uncommitted_blocks++;
1400 } else {
1401 struct wc_entry *f;
1402 struct rb_node *next = rb_next(&e->rb_node);
1403
1404 if (!next)
1405 break;
1406 f = container_of(next, struct wc_entry, rb_node);
1407 if (f != e + 1)
1408 break;
1409 if (read_original_sector(wc, f) !=
1410 read_original_sector(wc, e) + (wc->block_size >> SECTOR_SHIFT))
1411 break;
1412 if (unlikely(f->write_in_progress))
1413 break;
1414 if (writecache_entry_is_committed(wc, f))
1415 wc->overwrote_committed = true;
1416 e = f;
1417 }
1418 bio_size += wc->block_size;
1419 current_cache_sec += wc->block_size >> SECTOR_SHIFT;
1420 }
1421
1422 bio_set_dev(bio, wc->ssd_dev->bdev);
1423 bio->bi_iter.bi_sector = start_cache_sec;
1424 dm_accept_partial_bio(bio, bio_size >> SECTOR_SHIFT);
1425
1426 wc->stats.writes += bio->bi_iter.bi_size >> wc->block_size_bits;
1427 wc->stats.writes_allocate += (bio->bi_iter.bi_size - wc->block_size) >> wc->block_size_bits;
1428
1429 if (unlikely(wc->uncommitted_blocks >= wc->autocommit_blocks)) {
1430 wc->uncommitted_blocks = 0;
1431 queue_work(wc->writeback_wq, &wc->flush_work);
1432 } else {
1433 writecache_schedule_autocommit(wc);
1434 }
1435}
1436
1437static enum wc_map_op writecache_map_write(struct dm_writecache *wc, struct bio *bio)
1438{
1439 struct wc_entry *e;
1440
1441 do {
1442 bool found_entry = false;
1443 bool search_used = false;
1444
1445 if (writecache_has_error(wc)) {
1446 wc->stats.writes += bio->bi_iter.bi_size >> wc->block_size_bits;
1447 return WC_MAP_ERROR;
1448 }
1449 e = writecache_find_entry(wc, bio->bi_iter.bi_sector, 0);
1450 if (e) {
1451 if (!writecache_entry_is_committed(wc, e)) {
1452 wc->stats.write_hits_uncommitted++;
1453 search_used = true;
1454 goto bio_copy;
1455 }
1456 wc->stats.write_hits_committed++;
1457 if (!WC_MODE_PMEM(wc) && !e->write_in_progress) {
1458 wc->overwrote_committed = true;
1459 search_used = true;
1460 goto bio_copy;
1461 }
1462 found_entry = true;
1463 } else {
1464 if (unlikely(wc->cleaner) ||
1465 (wc->metadata_only && !(bio->bi_opf & REQ_META)))
1466 goto direct_write;
1467 }
1468 e = writecache_pop_from_freelist(wc, (sector_t)-1);
1469 if (unlikely(!e)) {
1470 if (!WC_MODE_PMEM(wc) && !found_entry) {
1471direct_write:
1472 e = writecache_find_entry(wc, bio->bi_iter.bi_sector, WFE_RETURN_FOLLOWING);
1473 writecache_map_remap_origin(wc, bio, e);
1474 wc->stats.writes_around += bio->bi_iter.bi_size >> wc->block_size_bits;
1475 wc->stats.writes += bio->bi_iter.bi_size >> wc->block_size_bits;
1476 return WC_MAP_REMAP_ORIGIN;
1477 }
1478 wc->stats.writes_blocked_on_freelist++;
1479 writecache_wait_on_freelist(wc);
1480 continue;
1481 }
1482 write_original_sector_seq_count(wc, e, bio->bi_iter.bi_sector, wc->seq_count);
1483 writecache_insert_entry(wc, e);
1484 wc->uncommitted_blocks++;
1485 wc->stats.writes_allocate++;
1486bio_copy:
1487 if (WC_MODE_PMEM(wc)) {
1488 bio_copy_block(wc, bio, memory_data(wc, e));
1489 wc->stats.writes++;
1490 } else {
1491 writecache_bio_copy_ssd(wc, bio, e, search_used);
1492 return WC_MAP_REMAP;
1493 }
1494 } while (bio->bi_iter.bi_size);
1495
1496 if (unlikely(bio->bi_opf & REQ_FUA || wc->uncommitted_blocks >= wc->autocommit_blocks))
1497 writecache_flush(wc);
1498 else
1499 writecache_schedule_autocommit(wc);
1500
1501 return WC_MAP_SUBMIT;
1502}
1503
1504static enum wc_map_op writecache_map_flush(struct dm_writecache *wc, struct bio *bio)
1505{
1506 if (writecache_has_error(wc))
1507 return WC_MAP_ERROR;
1508
1509 if (WC_MODE_PMEM(wc)) {
1510 wc->stats.flushes++;
1511 writecache_flush(wc);
1512 if (writecache_has_error(wc))
1513 return WC_MAP_ERROR;
1514 else if (unlikely(wc->cleaner) || unlikely(wc->metadata_only))
1515 return WC_MAP_REMAP_ORIGIN;
1516 return WC_MAP_SUBMIT;
1517 }
1518 /* SSD: */
1519 if (dm_bio_get_target_bio_nr(bio))
1520 return WC_MAP_REMAP_ORIGIN;
1521 wc->stats.flushes++;
1522 writecache_offload_bio(wc, bio);
1523 return WC_MAP_RETURN;
1524}
1525
1526static enum wc_map_op writecache_map_discard(struct dm_writecache *wc, struct bio *bio)
1527{
1528 wc->stats.discards += bio->bi_iter.bi_size >> wc->block_size_bits;
1529
1530 if (writecache_has_error(wc))
1531 return WC_MAP_ERROR;
1532
1533 if (WC_MODE_PMEM(wc)) {
1534 writecache_discard(wc, bio->bi_iter.bi_sector, bio_end_sector(bio));
1535 return WC_MAP_REMAP_ORIGIN;
1536 }
1537 /* SSD: */
1538 writecache_offload_bio(wc, bio);
1539 return WC_MAP_RETURN;
1540}
1541
1542static int writecache_map(struct dm_target *ti, struct bio *bio)
1543{
1544 struct dm_writecache *wc = ti->private;
1545 enum wc_map_op map_op;
1546
1547 bio->bi_private = NULL;
1548
1549 wc_lock(wc);
1550
1551 if (unlikely(bio->bi_opf & REQ_PREFLUSH)) {
1552 map_op = writecache_map_flush(wc, bio);
1553 goto done;
1554 }
1555
1556 bio->bi_iter.bi_sector = dm_target_offset(ti, bio->bi_iter.bi_sector);
1557
1558 if (unlikely((((unsigned int)bio->bi_iter.bi_sector | bio_sectors(bio)) &
1559 (wc->block_size / 512 - 1)) != 0)) {
1560 DMERR("I/O is not aligned, sector %llu, size %u, block size %u",
1561 (unsigned long long)bio->bi_iter.bi_sector,
1562 bio->bi_iter.bi_size, wc->block_size);
1563 map_op = WC_MAP_ERROR;
1564 goto done;
1565 }
1566
1567 if (unlikely(bio_op(bio) == REQ_OP_DISCARD)) {
1568 map_op = writecache_map_discard(wc, bio);
1569 goto done;
1570 }
1571
1572 if (bio_data_dir(bio) == READ)
1573 map_op = writecache_map_read(wc, bio);
1574 else
1575 map_op = writecache_map_write(wc, bio);
1576done:
1577 switch (map_op) {
1578 case WC_MAP_REMAP_ORIGIN:
1579 if (likely(wc->pause != 0)) {
1580 if (bio_op(bio) == REQ_OP_WRITE) {
1581 dm_iot_io_begin(&wc->iot, 1);
1582 bio->bi_private = (void *)2;
1583 }
1584 }
1585 bio_set_dev(bio, wc->dev->bdev);
1586 wc_unlock(wc);
1587 return DM_MAPIO_REMAPPED;
1588
1589 case WC_MAP_REMAP:
1590 /* make sure that writecache_end_io decrements bio_in_progress: */
1591 bio->bi_private = (void *)1;
1592 atomic_inc(&wc->bio_in_progress[bio_data_dir(bio)]);
1593 wc_unlock(wc);
1594 return DM_MAPIO_REMAPPED;
1595
1596 case WC_MAP_SUBMIT:
1597 wc_unlock(wc);
1598 bio_endio(bio);
1599 return DM_MAPIO_SUBMITTED;
1600
1601 case WC_MAP_RETURN:
1602 wc_unlock(wc);
1603 return DM_MAPIO_SUBMITTED;
1604
1605 case WC_MAP_ERROR:
1606 wc_unlock(wc);
1607 bio_io_error(bio);
1608 return DM_MAPIO_SUBMITTED;
1609
1610 default:
1611 BUG();
1612 wc_unlock(wc);
1613 return DM_MAPIO_KILL;
1614 }
1615}
1616
1617static int writecache_end_io(struct dm_target *ti, struct bio *bio, blk_status_t *status)
1618{
1619 struct dm_writecache *wc = ti->private;
1620
1621 if (bio->bi_private == (void *)1) {
1622 int dir = bio_data_dir(bio);
1623
1624 if (atomic_dec_and_test(&wc->bio_in_progress[dir]))
1625 if (unlikely(waitqueue_active(&wc->bio_in_progress_wait[dir])))
1626 wake_up(&wc->bio_in_progress_wait[dir]);
1627 } else if (bio->bi_private == (void *)2) {
1628 dm_iot_io_end(&wc->iot, 1);
1629 }
1630 return 0;
1631}
1632
1633static int writecache_iterate_devices(struct dm_target *ti,
1634 iterate_devices_callout_fn fn, void *data)
1635{
1636 struct dm_writecache *wc = ti->private;
1637
1638 return fn(ti, wc->dev, 0, ti->len, data);
1639}
1640
1641static void writecache_io_hints(struct dm_target *ti, struct queue_limits *limits)
1642{
1643 struct dm_writecache *wc = ti->private;
1644
1645 dm_stack_bs_limits(limits, wc->block_size);
1646}
1647
1648static void writecache_writeback_endio(struct bio *bio)
1649{
1650 struct writeback_struct *wb = container_of(bio, struct writeback_struct, bio);
1651 struct dm_writecache *wc = wb->wc;
1652 unsigned long flags;
1653
1654 raw_spin_lock_irqsave(&wc->endio_list_lock, flags);
1655 if (unlikely(list_empty(&wc->endio_list)))
1656 wake_up_process(wc->endio_thread);
1657 list_add_tail(&wb->endio_entry, &wc->endio_list);
1658 raw_spin_unlock_irqrestore(&wc->endio_list_lock, flags);
1659}
1660
1661static void writecache_copy_endio(int read_err, unsigned long write_err, void *ptr)
1662{
1663 struct copy_struct *c = ptr;
1664 struct dm_writecache *wc = c->wc;
1665
1666 c->error = likely(!(read_err | write_err)) ? 0 : -EIO;
1667
1668 raw_spin_lock_irq(&wc->endio_list_lock);
1669 if (unlikely(list_empty(&wc->endio_list)))
1670 wake_up_process(wc->endio_thread);
1671 list_add_tail(&c->endio_entry, &wc->endio_list);
1672 raw_spin_unlock_irq(&wc->endio_list_lock);
1673}
1674
1675static void __writecache_endio_pmem(struct dm_writecache *wc, struct list_head *list)
1676{
1677 unsigned int i;
1678 struct writeback_struct *wb;
1679 struct wc_entry *e;
1680 unsigned long n_walked = 0;
1681
1682 do {
1683 wb = list_entry(list->next, struct writeback_struct, endio_entry);
1684 list_del(&wb->endio_entry);
1685
1686 if (unlikely(wb->bio.bi_status != BLK_STS_OK))
1687 writecache_error(wc, blk_status_to_errno(wb->bio.bi_status),
1688 "write error %d", wb->bio.bi_status);
1689 i = 0;
1690 do {
1691 e = wb->wc_list[i];
1692 BUG_ON(!e->write_in_progress);
1693 e->write_in_progress = false;
1694 INIT_LIST_HEAD(&e->lru);
1695 if (!writecache_has_error(wc))
1696 writecache_free_entry(wc, e);
1697 BUG_ON(!wc->writeback_size);
1698 wc->writeback_size--;
1699 n_walked++;
1700 if (unlikely(n_walked >= ENDIO_LATENCY)) {
1701 writecache_commit_flushed(wc, false);
1702 wc_unlock(wc);
1703 wc_lock(wc);
1704 n_walked = 0;
1705 }
1706 } while (++i < wb->wc_list_n);
1707
1708 if (wb->wc_list != wb->wc_list_inline)
1709 kfree(wb->wc_list);
1710 bio_put(&wb->bio);
1711 } while (!list_empty(list));
1712}
1713
1714static void __writecache_endio_ssd(struct dm_writecache *wc, struct list_head *list)
1715{
1716 struct copy_struct *c;
1717 struct wc_entry *e;
1718
1719 do {
1720 c = list_entry(list->next, struct copy_struct, endio_entry);
1721 list_del(&c->endio_entry);
1722
1723 if (unlikely(c->error))
1724 writecache_error(wc, c->error, "copy error");
1725
1726 e = c->e;
1727 do {
1728 BUG_ON(!e->write_in_progress);
1729 e->write_in_progress = false;
1730 INIT_LIST_HEAD(&e->lru);
1731 if (!writecache_has_error(wc))
1732 writecache_free_entry(wc, e);
1733
1734 BUG_ON(!wc->writeback_size);
1735 wc->writeback_size--;
1736 e++;
1737 } while (--c->n_entries);
1738 mempool_free(c, &wc->copy_pool);
1739 } while (!list_empty(list));
1740}
1741
1742static int writecache_endio_thread(void *data)
1743{
1744 struct dm_writecache *wc = data;
1745
1746 while (1) {
1747 struct list_head list;
1748
1749 raw_spin_lock_irq(&wc->endio_list_lock);
1750 if (!list_empty(&wc->endio_list))
1751 goto pop_from_list;
1752 set_current_state(TASK_INTERRUPTIBLE);
1753 raw_spin_unlock_irq(&wc->endio_list_lock);
1754
1755 if (unlikely(kthread_should_stop())) {
1756 set_current_state(TASK_RUNNING);
1757 break;
1758 }
1759
1760 schedule();
1761
1762 continue;
1763
1764pop_from_list:
1765 list = wc->endio_list;
1766 list.next->prev = list.prev->next = &list;
1767 INIT_LIST_HEAD(&wc->endio_list);
1768 raw_spin_unlock_irq(&wc->endio_list_lock);
1769
1770 if (!WC_MODE_FUA(wc))
1771 writecache_disk_flush(wc, wc->dev);
1772
1773 wc_lock(wc);
1774
1775 if (WC_MODE_PMEM(wc)) {
1776 __writecache_endio_pmem(wc, &list);
1777 } else {
1778 __writecache_endio_ssd(wc, &list);
1779 writecache_wait_for_ios(wc, READ);
1780 }
1781
1782 writecache_commit_flushed(wc, false);
1783
1784 wc_unlock(wc);
1785 }
1786
1787 return 0;
1788}
1789
1790static bool wc_add_block(struct writeback_struct *wb, struct wc_entry *e)
1791{
1792 struct dm_writecache *wc = wb->wc;
1793 unsigned int block_size = wc->block_size;
1794 void *address = memory_data(wc, e);
1795
1796 persistent_memory_flush_cache(address, block_size);
1797
1798 if (unlikely(bio_end_sector(&wb->bio) >= wc->data_device_sectors))
1799 return true;
1800
1801 return bio_add_page(&wb->bio, persistent_memory_page(address),
1802 block_size, persistent_memory_page_offset(address)) != 0;
1803}
1804
1805struct writeback_list {
1806 struct list_head list;
1807 size_t size;
1808};
1809
1810static void __writeback_throttle(struct dm_writecache *wc, struct writeback_list *wbl)
1811{
1812 if (unlikely(wc->max_writeback_jobs)) {
1813 if (READ_ONCE(wc->writeback_size) - wbl->size >= wc->max_writeback_jobs) {
1814 wc_lock(wc);
1815 while (wc->writeback_size - wbl->size >= wc->max_writeback_jobs)
1816 writecache_wait_on_freelist(wc);
1817 wc_unlock(wc);
1818 }
1819 }
1820 cond_resched();
1821}
1822
1823static void __writecache_writeback_pmem(struct dm_writecache *wc, struct writeback_list *wbl)
1824{
1825 struct wc_entry *e, *f;
1826 struct bio *bio;
1827 struct writeback_struct *wb;
1828 unsigned int max_pages;
1829
1830 while (wbl->size) {
1831 wbl->size--;
1832 e = container_of(wbl->list.prev, struct wc_entry, lru);
1833 list_del(&e->lru);
1834
1835 max_pages = e->wc_list_contiguous;
1836
1837 bio = bio_alloc_bioset(wc->dev->bdev, max_pages, REQ_OP_WRITE,
1838 GFP_NOIO, &wc->bio_set);
1839 wb = container_of(bio, struct writeback_struct, bio);
1840 wb->wc = wc;
1841 bio->bi_end_io = writecache_writeback_endio;
1842 bio->bi_iter.bi_sector = read_original_sector(wc, e);
1843
1844 if (unlikely(max_pages > WB_LIST_INLINE))
1845 wb->wc_list = kmalloc_objs(struct wc_entry *, max_pages,
1846 GFP_NOIO | __GFP_NORETRY | __GFP_NOMEMALLOC | __GFP_NOWARN);
1847
1848 if (likely(max_pages <= WB_LIST_INLINE) || unlikely(!wb->wc_list)) {
1849 wb->wc_list = wb->wc_list_inline;
1850 max_pages = WB_LIST_INLINE;
1851 }
1852
1853 BUG_ON(!wc_add_block(wb, e));
1854
1855 wb->wc_list[0] = e;
1856 wb->wc_list_n = 1;
1857
1858 while (wbl->size && wb->wc_list_n < max_pages) {
1859 f = container_of(wbl->list.prev, struct wc_entry, lru);
1860 if (read_original_sector(wc, f) !=
1861 read_original_sector(wc, e) + (wc->block_size >> SECTOR_SHIFT))
1862 break;
1863 if (!wc_add_block(wb, f))
1864 break;
1865 wbl->size--;
1866 list_del(&f->lru);
1867 wb->wc_list[wb->wc_list_n++] = f;
1868 e = f;
1869 }
1870 if (WC_MODE_FUA(wc))
1871 bio->bi_opf |= REQ_FUA;
1872 if (writecache_has_error(wc)) {
1873 bio->bi_status = BLK_STS_IOERR;
1874 bio_endio(bio);
1875 } else if (unlikely(!bio_sectors(bio))) {
1876 bio->bi_status = BLK_STS_OK;
1877 bio_endio(bio);
1878 } else {
1879 submit_bio(bio);
1880 }
1881
1882 __writeback_throttle(wc, wbl);
1883 }
1884}
1885
1886static void __writecache_writeback_ssd(struct dm_writecache *wc, struct writeback_list *wbl)
1887{
1888 struct wc_entry *e, *f;
1889 struct dm_io_region from, to;
1890 struct copy_struct *c;
1891
1892 while (wbl->size) {
1893 unsigned int n_sectors;
1894
1895 wbl->size--;
1896 e = container_of(wbl->list.prev, struct wc_entry, lru);
1897 list_del(&e->lru);
1898
1899 n_sectors = e->wc_list_contiguous << (wc->block_size_bits - SECTOR_SHIFT);
1900
1901 from.bdev = wc->ssd_dev->bdev;
1902 from.sector = cache_sector(wc, e);
1903 from.count = n_sectors;
1904 to.bdev = wc->dev->bdev;
1905 to.sector = read_original_sector(wc, e);
1906 to.count = n_sectors;
1907
1908 c = mempool_alloc(&wc->copy_pool, GFP_NOIO);
1909 c->wc = wc;
1910 c->e = e;
1911 c->n_entries = e->wc_list_contiguous;
1912
1913 while ((n_sectors -= wc->block_size >> SECTOR_SHIFT)) {
1914 wbl->size--;
1915 f = container_of(wbl->list.prev, struct wc_entry, lru);
1916 BUG_ON(f != e + 1);
1917 list_del(&f->lru);
1918 e = f;
1919 }
1920
1921 if (unlikely(to.sector + to.count > wc->data_device_sectors)) {
1922 if (to.sector >= wc->data_device_sectors) {
1923 writecache_copy_endio(0, 0, c);
1924 continue;
1925 }
1926 from.count = to.count = wc->data_device_sectors - to.sector;
1927 }
1928
1929 dm_kcopyd_copy(wc->dm_kcopyd, &from, 1, &to, 0, writecache_copy_endio, c);
1930
1931 __writeback_throttle(wc, wbl);
1932 }
1933}
1934
1935static void writecache_writeback(struct work_struct *work)
1936{
1937 struct dm_writecache *wc = container_of(work, struct dm_writecache, writeback_work);
1938 struct blk_plug plug;
1939 struct wc_entry *f, *g, *e = NULL;
1940 struct rb_node *node, *next_node;
1941 struct list_head skipped;
1942 struct writeback_list wbl;
1943 unsigned long n_walked;
1944
1945 if (!WC_MODE_PMEM(wc)) {
1946 /* Wait for any active kcopyd work on behalf of ssd writeback */
1947 dm_kcopyd_client_flush(wc->dm_kcopyd);
1948 }
1949
1950 if (likely(wc->pause != 0)) {
1951 while (1) {
1952 unsigned long idle;
1953
1954 if (unlikely(wc->cleaner) || unlikely(wc->writeback_all) ||
1955 unlikely(dm_suspended(wc->ti)))
1956 break;
1957 idle = dm_iot_idle_time(&wc->iot);
1958 if (idle >= wc->pause)
1959 break;
1960 idle = wc->pause - idle;
1961 if (idle > HZ)
1962 idle = HZ;
1963 schedule_timeout_idle(idle);
1964 }
1965 }
1966
1967 wc_lock(wc);
1968restart:
1969 if (writecache_has_error(wc)) {
1970 wc_unlock(wc);
1971 return;
1972 }
1973
1974 if (unlikely(wc->writeback_all)) {
1975 if (writecache_wait_for_writeback(wc))
1976 goto restart;
1977 }
1978
1979 if (wc->overwrote_committed)
1980 writecache_wait_for_ios(wc, WRITE);
1981
1982 n_walked = 0;
1983 INIT_LIST_HEAD(&skipped);
1984 INIT_LIST_HEAD(&wbl.list);
1985 wbl.size = 0;
1986 while (!list_empty(&wc->lru) &&
1987 (wc->writeback_all ||
1988 wc->freelist_size + wc->writeback_size <= wc->freelist_low_watermark ||
1989 (jiffies - container_of(wc->lru.prev, struct wc_entry, lru)->age >=
1990 wc->max_age - wc->max_age / MAX_AGE_DIV))) {
1991
1992 n_walked++;
1993 if (unlikely(n_walked > WRITEBACK_LATENCY) &&
1994 likely(!wc->writeback_all)) {
1995 if (likely(!dm_suspended(wc->ti)))
1996 queue_work(wc->writeback_wq, &wc->writeback_work);
1997 break;
1998 }
1999
2000 if (unlikely(wc->writeback_all)) {
2001 if (unlikely(!e)) {
2002 writecache_flush(wc);
2003 e = container_of(rb_first(&wc->tree), struct wc_entry, rb_node);
2004 } else
2005 e = g;
2006 } else
2007 e = container_of(wc->lru.prev, struct wc_entry, lru);
2008 BUG_ON(e->write_in_progress);
2009 if (unlikely(!writecache_entry_is_committed(wc, e)))
2010 writecache_flush(wc);
2011
2012 node = rb_prev(&e->rb_node);
2013 if (node) {
2014 f = container_of(node, struct wc_entry, rb_node);
2015 if (unlikely(read_original_sector(wc, f) ==
2016 read_original_sector(wc, e))) {
2017 BUG_ON(!f->write_in_progress);
2018 list_move(&e->lru, &skipped);
2019 cond_resched();
2020 continue;
2021 }
2022 }
2023 wc->writeback_size++;
2024 list_move(&e->lru, &wbl.list);
2025 wbl.size++;
2026 e->write_in_progress = true;
2027 e->wc_list_contiguous = 1;
2028
2029 f = e;
2030
2031 while (1) {
2032 next_node = rb_next(&f->rb_node);
2033 if (unlikely(!next_node))
2034 break;
2035 g = container_of(next_node, struct wc_entry, rb_node);
2036 if (unlikely(read_original_sector(wc, g) ==
2037 read_original_sector(wc, f))) {
2038 f = g;
2039 continue;
2040 }
2041 if (read_original_sector(wc, g) !=
2042 read_original_sector(wc, f) + (wc->block_size >> SECTOR_SHIFT))
2043 break;
2044 if (unlikely(g->write_in_progress))
2045 break;
2046 if (unlikely(!writecache_entry_is_committed(wc, g)))
2047 break;
2048
2049 if (!WC_MODE_PMEM(wc)) {
2050 if (g != f + 1)
2051 break;
2052 }
2053
2054 n_walked++;
2055 //if (unlikely(n_walked > WRITEBACK_LATENCY) && likely(!wc->writeback_all))
2056 // break;
2057
2058 wc->writeback_size++;
2059 list_move(&g->lru, &wbl.list);
2060 wbl.size++;
2061 g->write_in_progress = true;
2062 g->wc_list_contiguous = BIO_MAX_VECS;
2063 f = g;
2064 e->wc_list_contiguous++;
2065 if (unlikely(e->wc_list_contiguous == BIO_MAX_VECS)) {
2066 if (unlikely(wc->writeback_all)) {
2067 next_node = rb_next(&f->rb_node);
2068 if (likely(next_node))
2069 g = container_of(next_node, struct wc_entry, rb_node);
2070 }
2071 break;
2072 }
2073 }
2074 cond_resched();
2075 }
2076
2077 if (!list_empty(&skipped)) {
2078 list_splice_tail(&skipped, &wc->lru);
2079 /*
2080 * If we didn't do any progress, we must wait until some
2081 * writeback finishes to avoid burning CPU in a loop
2082 */
2083 if (unlikely(!wbl.size))
2084 writecache_wait_for_writeback(wc);
2085 }
2086
2087 wc_unlock(wc);
2088
2089 blk_start_plug(&plug);
2090
2091 if (WC_MODE_PMEM(wc))
2092 __writecache_writeback_pmem(wc, &wbl);
2093 else
2094 __writecache_writeback_ssd(wc, &wbl);
2095
2096 blk_finish_plug(&plug);
2097
2098 if (unlikely(wc->writeback_all)) {
2099 wc_lock(wc);
2100 while (writecache_wait_for_writeback(wc))
2101 ;
2102 wc_unlock(wc);
2103 }
2104}
2105
2106static int calculate_memory_size(uint64_t device_size, unsigned int block_size,
2107 size_t *n_blocks_p, size_t *n_metadata_blocks_p)
2108{
2109 uint64_t n_blocks, offset;
2110 struct wc_entry e;
2111
2112 n_blocks = device_size;
2113 do_div(n_blocks, block_size + sizeof(struct wc_memory_entry));
2114
2115 while (1) {
2116 if (!n_blocks)
2117 return -ENOSPC;
2118 /* Verify the following entries[n_blocks] won't overflow */
2119 if (n_blocks >= ((size_t)-sizeof(struct wc_memory_superblock) /
2120 sizeof(struct wc_memory_entry)))
2121 return -EFBIG;
2122 offset = offsetof(struct wc_memory_superblock, entries[n_blocks]);
2123 offset = (offset + block_size - 1) & ~(uint64_t)(block_size - 1);
2124 if (offset + n_blocks * block_size <= device_size)
2125 break;
2126 n_blocks--;
2127 }
2128
2129 /* check if the bit field overflows */
2130 e.index = n_blocks;
2131 if (e.index != n_blocks)
2132 return -EFBIG;
2133
2134 if (n_blocks_p)
2135 *n_blocks_p = n_blocks;
2136 if (n_metadata_blocks_p)
2137 *n_metadata_blocks_p = offset >> __ffs(block_size);
2138 return 0;
2139}
2140
2141static int init_memory(struct dm_writecache *wc)
2142{
2143 size_t b;
2144 int r;
2145
2146 r = calculate_memory_size(wc->memory_map_size, wc->block_size, &wc->n_blocks, NULL);
2147 if (r)
2148 return r;
2149
2150 r = writecache_alloc_entries(wc);
2151 if (r)
2152 return r;
2153
2154 for (b = 0; b < ARRAY_SIZE(sb(wc)->padding); b++)
2155 pmem_assign(sb(wc)->padding[b], cpu_to_le64(0));
2156 pmem_assign(sb(wc)->version, cpu_to_le32(MEMORY_SUPERBLOCK_VERSION));
2157 pmem_assign(sb(wc)->block_size, cpu_to_le32(wc->block_size));
2158 pmem_assign(sb(wc)->n_blocks, cpu_to_le64(wc->n_blocks));
2159 pmem_assign(sb(wc)->seq_count, cpu_to_le64(0));
2160
2161 for (b = 0; b < wc->n_blocks; b++) {
2162 write_original_sector_seq_count(wc, &wc->entries[b], -1, -1);
2163 cond_resched();
2164 }
2165
2166 writecache_flush_all_metadata(wc);
2167 writecache_commit_flushed(wc, false);
2168 pmem_assign(sb(wc)->magic, cpu_to_le32(MEMORY_SUPERBLOCK_MAGIC));
2169 writecache_flush_region(wc, &sb(wc)->magic, sizeof(sb(wc)->magic));
2170 writecache_commit_flushed(wc, false);
2171
2172 return 0;
2173}
2174
2175static void writecache_dtr(struct dm_target *ti)
2176{
2177 struct dm_writecache *wc = ti->private;
2178
2179 if (!wc)
2180 return;
2181
2182 if (wc->endio_thread)
2183 kthread_stop(wc->endio_thread);
2184
2185 if (wc->flush_thread)
2186 kthread_stop(wc->flush_thread);
2187
2188 bioset_exit(&wc->bio_set);
2189
2190 mempool_exit(&wc->copy_pool);
2191
2192 if (wc->writeback_wq)
2193 destroy_workqueue(wc->writeback_wq);
2194
2195 if (wc->dev)
2196 dm_put_device(ti, wc->dev);
2197
2198 if (wc->ssd_dev)
2199 dm_put_device(ti, wc->ssd_dev);
2200
2201 vfree(wc->entries);
2202
2203 if (wc->memory_map) {
2204 if (WC_MODE_PMEM(wc))
2205 persistent_memory_release(wc);
2206 else
2207 vfree(wc->memory_map);
2208 }
2209
2210 if (wc->dm_kcopyd)
2211 dm_kcopyd_client_destroy(wc->dm_kcopyd);
2212
2213 if (wc->dm_io)
2214 dm_io_client_destroy(wc->dm_io);
2215
2216 vfree(wc->dirty_bitmap);
2217
2218 kfree(wc);
2219}
2220
2221static int writecache_ctr(struct dm_target *ti, unsigned int argc, char **argv)
2222{
2223 struct dm_writecache *wc;
2224 struct dm_arg_set as;
2225 const char *string;
2226 unsigned int opt_params;
2227 size_t offset, data_size;
2228 int i, r;
2229 char dummy;
2230 int high_wm_percent = HIGH_WATERMARK;
2231 int low_wm_percent = LOW_WATERMARK;
2232 uint64_t x;
2233 struct wc_memory_superblock s;
2234
2235 static struct dm_arg _args[] = {
2236 {0, 18, "Invalid number of feature args"},
2237 };
2238
2239 as.argc = argc;
2240 as.argv = argv;
2241
2242 wc = kzalloc_obj(struct dm_writecache);
2243 if (!wc) {
2244 ti->error = "Cannot allocate writecache structure";
2245 r = -ENOMEM;
2246 goto bad;
2247 }
2248 ti->private = wc;
2249 wc->ti = ti;
2250
2251 mutex_init(&wc->lock);
2252 wc->max_age = MAX_AGE_UNSPECIFIED;
2253 writecache_poison_lists(wc);
2254 init_waitqueue_head(&wc->freelist_wait);
2255 timer_setup(&wc->autocommit_timer, writecache_autocommit_timer, 0);
2256 timer_setup(&wc->max_age_timer, writecache_max_age_timer, 0);
2257
2258 for (i = 0; i < 2; i++) {
2259 atomic_set(&wc->bio_in_progress[i], 0);
2260 init_waitqueue_head(&wc->bio_in_progress_wait[i]);
2261 }
2262
2263 wc->dm_io = dm_io_client_create();
2264 if (IS_ERR(wc->dm_io)) {
2265 r = PTR_ERR(wc->dm_io);
2266 ti->error = "Unable to allocate dm-io client";
2267 wc->dm_io = NULL;
2268 goto bad;
2269 }
2270
2271 wc->writeback_wq = alloc_workqueue("writecache-writeback",
2272 WQ_MEM_RECLAIM | WQ_PERCPU, 1);
2273 if (!wc->writeback_wq) {
2274 r = -ENOMEM;
2275 ti->error = "Could not allocate writeback workqueue";
2276 goto bad;
2277 }
2278 INIT_WORK(&wc->writeback_work, writecache_writeback);
2279 INIT_WORK(&wc->flush_work, writecache_flush_work);
2280
2281 dm_iot_init(&wc->iot);
2282
2283 raw_spin_lock_init(&wc->endio_list_lock);
2284 INIT_LIST_HEAD(&wc->endio_list);
2285 wc->endio_thread = kthread_run(writecache_endio_thread, wc, "writecache_endio");
2286 if (IS_ERR(wc->endio_thread)) {
2287 r = PTR_ERR(wc->endio_thread);
2288 wc->endio_thread = NULL;
2289 ti->error = "Couldn't spawn endio thread";
2290 goto bad;
2291 }
2292
2293 /*
2294 * Parse the mode (pmem or ssd)
2295 */
2296 string = dm_shift_arg(&as);
2297 if (!string)
2298 goto bad_arguments;
2299
2300 if (!strcasecmp(string, "s")) {
2301 wc->pmem_mode = false;
2302 } else if (!strcasecmp(string, "p")) {
2303#ifdef DM_WRITECACHE_HAS_PMEM
2304 wc->pmem_mode = true;
2305 wc->writeback_fua = true;
2306#else
2307 /*
2308 * If the architecture doesn't support persistent memory or
2309 * the kernel doesn't support any DAX drivers, this driver can
2310 * only be used in SSD-only mode.
2311 */
2312 r = -EOPNOTSUPP;
2313 ti->error = "Persistent memory or DAX not supported on this system";
2314 goto bad;
2315#endif
2316 } else {
2317 goto bad_arguments;
2318 }
2319
2320 if (WC_MODE_PMEM(wc)) {
2321 r = bioset_init(&wc->bio_set, BIO_POOL_SIZE,
2322 offsetof(struct writeback_struct, bio),
2323 BIOSET_NEED_BVECS);
2324 if (r) {
2325 ti->error = "Could not allocate bio set";
2326 goto bad;
2327 }
2328 } else {
2329 wc->pause = PAUSE_WRITEBACK;
2330 r = mempool_init_kmalloc_pool(&wc->copy_pool, 1, sizeof(struct copy_struct));
2331 if (r) {
2332 ti->error = "Could not allocate mempool";
2333 goto bad;
2334 }
2335 }
2336
2337 /*
2338 * Parse the origin data device
2339 */
2340 string = dm_shift_arg(&as);
2341 if (!string)
2342 goto bad_arguments;
2343 r = dm_get_device(ti, string, dm_table_get_mode(ti->table), &wc->dev);
2344 if (r) {
2345 ti->error = "Origin data device lookup failed";
2346 goto bad;
2347 }
2348
2349 /*
2350 * Parse cache data device (be it pmem or ssd)
2351 */
2352 string = dm_shift_arg(&as);
2353 if (!string)
2354 goto bad_arguments;
2355
2356 r = dm_get_device(ti, string, dm_table_get_mode(ti->table), &wc->ssd_dev);
2357 if (r) {
2358 ti->error = "Cache data device lookup failed";
2359 goto bad;
2360 }
2361 wc->memory_map_size = bdev_nr_bytes(wc->ssd_dev->bdev);
2362
2363 /*
2364 * Parse the cache block size
2365 */
2366 string = dm_shift_arg(&as);
2367 if (!string)
2368 goto bad_arguments;
2369 if (sscanf(string, "%u%c", &wc->block_size, &dummy) != 1 ||
2370 wc->block_size < 512 || wc->block_size > PAGE_SIZE ||
2371 (wc->block_size & (wc->block_size - 1))) {
2372 r = -EINVAL;
2373 ti->error = "Invalid block size";
2374 goto bad;
2375 }
2376 if (wc->block_size < bdev_logical_block_size(wc->dev->bdev) ||
2377 wc->block_size < bdev_logical_block_size(wc->ssd_dev->bdev)) {
2378 r = -EINVAL;
2379 ti->error = "Block size is smaller than device logical block size";
2380 goto bad;
2381 }
2382 wc->block_size_bits = __ffs(wc->block_size);
2383
2384 wc->max_writeback_jobs = MAX_WRITEBACK_JOBS;
2385 wc->autocommit_blocks = !WC_MODE_PMEM(wc) ? AUTOCOMMIT_BLOCKS_SSD : AUTOCOMMIT_BLOCKS_PMEM;
2386 wc->autocommit_jiffies = msecs_to_jiffies(AUTOCOMMIT_MSEC);
2387
2388 /*
2389 * Parse optional arguments
2390 */
2391 r = dm_read_arg_group(_args, &as, &opt_params, &ti->error);
2392 if (r)
2393 goto bad;
2394
2395 while (opt_params) {
2396 string = dm_shift_arg(&as), opt_params--;
2397 if (!strcasecmp(string, "start_sector") && opt_params >= 1) {
2398 unsigned long long start_sector;
2399
2400 string = dm_shift_arg(&as), opt_params--;
2401 if (sscanf(string, "%llu%c", &start_sector, &dummy) != 1)
2402 goto invalid_optional;
2403 wc->start_sector = start_sector;
2404 wc->start_sector_set = true;
2405 if (wc->start_sector != start_sector ||
2406 wc->start_sector >= wc->memory_map_size >> SECTOR_SHIFT)
2407 goto invalid_optional;
2408 } else if (!strcasecmp(string, "high_watermark") && opt_params >= 1) {
2409 string = dm_shift_arg(&as), opt_params--;
2410 if (sscanf(string, "%d%c", &high_wm_percent, &dummy) != 1)
2411 goto invalid_optional;
2412 if (high_wm_percent < 0 || high_wm_percent > 100)
2413 goto invalid_optional;
2414 wc->high_wm_percent_value = high_wm_percent;
2415 wc->high_wm_percent_set = true;
2416 } else if (!strcasecmp(string, "low_watermark") && opt_params >= 1) {
2417 string = dm_shift_arg(&as), opt_params--;
2418 if (sscanf(string, "%d%c", &low_wm_percent, &dummy) != 1)
2419 goto invalid_optional;
2420 if (low_wm_percent < 0 || low_wm_percent > 100)
2421 goto invalid_optional;
2422 wc->low_wm_percent_value = low_wm_percent;
2423 wc->low_wm_percent_set = true;
2424 } else if (!strcasecmp(string, "writeback_jobs") && opt_params >= 1) {
2425 string = dm_shift_arg(&as), opt_params--;
2426 if (sscanf(string, "%u%c", &wc->max_writeback_jobs, &dummy) != 1)
2427 goto invalid_optional;
2428 wc->max_writeback_jobs_set = true;
2429 } else if (!strcasecmp(string, "autocommit_blocks") && opt_params >= 1) {
2430 string = dm_shift_arg(&as), opt_params--;
2431 if (sscanf(string, "%u%c", &wc->autocommit_blocks, &dummy) != 1)
2432 goto invalid_optional;
2433 wc->autocommit_blocks_set = true;
2434 } else if (!strcasecmp(string, "autocommit_time") && opt_params >= 1) {
2435 unsigned int autocommit_msecs;
2436
2437 string = dm_shift_arg(&as), opt_params--;
2438 if (sscanf(string, "%u%c", &autocommit_msecs, &dummy) != 1)
2439 goto invalid_optional;
2440 if (autocommit_msecs > 3600000)
2441 goto invalid_optional;
2442 wc->autocommit_jiffies = msecs_to_jiffies(autocommit_msecs);
2443 wc->autocommit_time_value = autocommit_msecs;
2444 wc->autocommit_time_set = true;
2445 } else if (!strcasecmp(string, "max_age") && opt_params >= 1) {
2446 unsigned int max_age_msecs;
2447
2448 string = dm_shift_arg(&as), opt_params--;
2449 if (sscanf(string, "%u%c", &max_age_msecs, &dummy) != 1)
2450 goto invalid_optional;
2451 if (max_age_msecs > 86400000)
2452 goto invalid_optional;
2453 wc->max_age = msecs_to_jiffies(max_age_msecs);
2454 wc->max_age_set = true;
2455 wc->max_age_value = max_age_msecs;
2456 } else if (!strcasecmp(string, "cleaner")) {
2457 wc->cleaner_set = true;
2458 wc->cleaner = true;
2459 } else if (!strcasecmp(string, "fua")) {
2460 if (WC_MODE_PMEM(wc)) {
2461 wc->writeback_fua = true;
2462 wc->writeback_fua_set = true;
2463 } else
2464 goto invalid_optional;
2465 } else if (!strcasecmp(string, "nofua")) {
2466 if (WC_MODE_PMEM(wc)) {
2467 wc->writeback_fua = false;
2468 wc->writeback_fua_set = true;
2469 } else
2470 goto invalid_optional;
2471 } else if (!strcasecmp(string, "metadata_only")) {
2472 wc->metadata_only = true;
2473 } else if (!strcasecmp(string, "pause_writeback") && opt_params >= 1) {
2474 unsigned int pause_msecs;
2475
2476 if (WC_MODE_PMEM(wc))
2477 goto invalid_optional;
2478 string = dm_shift_arg(&as), opt_params--;
2479 if (sscanf(string, "%u%c", &pause_msecs, &dummy) != 1)
2480 goto invalid_optional;
2481 if (pause_msecs > 60000)
2482 goto invalid_optional;
2483 wc->pause = msecs_to_jiffies(pause_msecs);
2484 wc->pause_set = true;
2485 wc->pause_value = pause_msecs;
2486 } else {
2487invalid_optional:
2488 r = -EINVAL;
2489 ti->error = "Invalid optional argument";
2490 goto bad;
2491 }
2492 }
2493
2494 if (high_wm_percent < low_wm_percent) {
2495 r = -EINVAL;
2496 ti->error = "High watermark must be greater than or equal to low watermark";
2497 goto bad;
2498 }
2499
2500 if (WC_MODE_PMEM(wc)) {
2501 if (!dax_synchronous(wc->ssd_dev->dax_dev)) {
2502 r = -EOPNOTSUPP;
2503 ti->error = "Asynchronous persistent memory not supported as pmem cache";
2504 goto bad;
2505 }
2506
2507 r = persistent_memory_claim(wc);
2508 if (r) {
2509 ti->error = "Unable to map persistent memory for cache";
2510 goto bad;
2511 }
2512 } else {
2513 size_t n_blocks, n_metadata_blocks;
2514 uint64_t n_bitmap_bits;
2515
2516 wc->memory_map_size -= (uint64_t)wc->start_sector << SECTOR_SHIFT;
2517
2518 bio_list_init(&wc->flush_list);
2519 wc->flush_thread = kthread_run(writecache_flush_thread, wc, "dm_writecache_flush");
2520 if (IS_ERR(wc->flush_thread)) {
2521 r = PTR_ERR(wc->flush_thread);
2522 wc->flush_thread = NULL;
2523 ti->error = "Couldn't spawn flush thread";
2524 goto bad;
2525 }
2526
2527 r = calculate_memory_size(wc->memory_map_size, wc->block_size,
2528 &n_blocks, &n_metadata_blocks);
2529 if (r) {
2530 ti->error = "Invalid device size";
2531 goto bad;
2532 }
2533
2534 n_bitmap_bits = (((uint64_t)n_metadata_blocks << wc->block_size_bits) +
2535 BITMAP_GRANULARITY - 1) / BITMAP_GRANULARITY;
2536 /* this is limitation of test_bit functions */
2537 if (n_bitmap_bits > 1U << 31) {
2538 r = -EFBIG;
2539 ti->error = "Invalid device size";
2540 goto bad;
2541 }
2542
2543 wc->memory_map = vmalloc(n_metadata_blocks << wc->block_size_bits);
2544 if (!wc->memory_map) {
2545 r = -ENOMEM;
2546 ti->error = "Unable to allocate memory for metadata";
2547 goto bad;
2548 }
2549
2550 wc->dm_kcopyd = dm_kcopyd_client_create(&dm_kcopyd_throttle);
2551 if (IS_ERR(wc->dm_kcopyd)) {
2552 r = PTR_ERR(wc->dm_kcopyd);
2553 ti->error = "Unable to allocate dm-kcopyd client";
2554 wc->dm_kcopyd = NULL;
2555 goto bad;
2556 }
2557
2558 wc->metadata_sectors = n_metadata_blocks << (wc->block_size_bits - SECTOR_SHIFT);
2559 wc->dirty_bitmap_size = (n_bitmap_bits + BITS_PER_LONG - 1) /
2560 BITS_PER_LONG * sizeof(unsigned long);
2561 wc->dirty_bitmap = vzalloc(wc->dirty_bitmap_size);
2562 if (!wc->dirty_bitmap) {
2563 r = -ENOMEM;
2564 ti->error = "Unable to allocate dirty bitmap";
2565 goto bad;
2566 }
2567
2568 r = writecache_read_metadata(wc, wc->block_size >> SECTOR_SHIFT);
2569 if (r) {
2570 ti->error = "Unable to read first block of metadata";
2571 goto bad;
2572 }
2573 }
2574
2575 r = copy_mc_to_kernel(&s, sb(wc), sizeof(struct wc_memory_superblock));
2576 if (r) {
2577 ti->error = "Hardware memory error when reading superblock";
2578 goto bad;
2579 }
2580 if (!le32_to_cpu(s.magic) && !le32_to_cpu(s.version)) {
2581 r = init_memory(wc);
2582 if (r) {
2583 ti->error = "Unable to initialize device";
2584 goto bad;
2585 }
2586 r = copy_mc_to_kernel(&s, sb(wc),
2587 sizeof(struct wc_memory_superblock));
2588 if (r) {
2589 ti->error = "Hardware memory error when reading superblock";
2590 goto bad;
2591 }
2592 }
2593
2594 if (le32_to_cpu(s.magic) != MEMORY_SUPERBLOCK_MAGIC) {
2595 ti->error = "Invalid magic in the superblock";
2596 r = -EINVAL;
2597 goto bad;
2598 }
2599
2600 if (le32_to_cpu(s.version) != MEMORY_SUPERBLOCK_VERSION) {
2601 ti->error = "Invalid version in the superblock";
2602 r = -EINVAL;
2603 goto bad;
2604 }
2605
2606 if (le32_to_cpu(s.block_size) != wc->block_size) {
2607 ti->error = "Block size does not match superblock";
2608 r = -EINVAL;
2609 goto bad;
2610 }
2611
2612 wc->n_blocks = le64_to_cpu(s.n_blocks);
2613
2614 offset = wc->n_blocks * sizeof(struct wc_memory_entry);
2615 if (offset / sizeof(struct wc_memory_entry) != le64_to_cpu(sb(wc)->n_blocks)) {
2616overflow:
2617 ti->error = "Overflow in size calculation";
2618 r = -EINVAL;
2619 goto bad;
2620 }
2621 offset += sizeof(struct wc_memory_superblock);
2622 if (offset < sizeof(struct wc_memory_superblock))
2623 goto overflow;
2624 offset = (offset + wc->block_size - 1) & ~(size_t)(wc->block_size - 1);
2625 data_size = wc->n_blocks * (size_t)wc->block_size;
2626 if (!offset || (data_size / wc->block_size != wc->n_blocks) ||
2627 (offset + data_size < offset))
2628 goto overflow;
2629 if (offset + data_size > wc->memory_map_size) {
2630 ti->error = "Memory area is too small";
2631 r = -EINVAL;
2632 goto bad;
2633 }
2634
2635 wc->metadata_sectors = offset >> SECTOR_SHIFT;
2636 wc->block_start = (char *)sb(wc) + offset;
2637
2638 x = (uint64_t)wc->n_blocks * (100 - high_wm_percent);
2639 x += 50;
2640 do_div(x, 100);
2641 wc->freelist_high_watermark = x;
2642 x = (uint64_t)wc->n_blocks * (100 - low_wm_percent);
2643 x += 50;
2644 do_div(x, 100);
2645 wc->freelist_low_watermark = x;
2646
2647 if (wc->cleaner)
2648 activate_cleaner(wc);
2649
2650 r = writecache_alloc_entries(wc);
2651 if (r) {
2652 ti->error = "Cannot allocate memory";
2653 goto bad;
2654 }
2655
2656 ti->num_flush_bios = WC_MODE_PMEM(wc) ? 1 : 2;
2657 ti->flush_supported = true;
2658 ti->num_discard_bios = 1;
2659
2660 if (WC_MODE_PMEM(wc))
2661 persistent_memory_flush_cache(wc->memory_map, wc->memory_map_size);
2662
2663 return 0;
2664
2665bad_arguments:
2666 r = -EINVAL;
2667 ti->error = "Bad arguments";
2668bad:
2669 writecache_dtr(ti);
2670 return r;
2671}
2672
2673static void writecache_status(struct dm_target *ti, status_type_t type,
2674 unsigned int status_flags, char *result, unsigned int maxlen)
2675{
2676 struct dm_writecache *wc = ti->private;
2677 unsigned int extra_args;
2678 unsigned int sz = 0;
2679
2680 switch (type) {
2681 case STATUSTYPE_INFO:
2682 DMEMIT("%ld %llu %llu %llu %llu %llu %llu %llu %llu %llu %llu %llu %llu %llu",
2683 writecache_has_error(wc),
2684 (unsigned long long)wc->n_blocks, (unsigned long long)wc->freelist_size,
2685 (unsigned long long)wc->writeback_size,
2686 wc->stats.reads,
2687 wc->stats.read_hits,
2688 wc->stats.writes,
2689 wc->stats.write_hits_uncommitted,
2690 wc->stats.write_hits_committed,
2691 wc->stats.writes_around,
2692 wc->stats.writes_allocate,
2693 wc->stats.writes_blocked_on_freelist,
2694 wc->stats.flushes,
2695 wc->stats.discards);
2696 break;
2697 case STATUSTYPE_TABLE:
2698 DMEMIT("%c %s %s %u ", WC_MODE_PMEM(wc) ? 'p' : 's',
2699 wc->dev->name, wc->ssd_dev->name, wc->block_size);
2700 extra_args = 0;
2701 if (wc->start_sector_set)
2702 extra_args += 2;
2703 if (wc->high_wm_percent_set)
2704 extra_args += 2;
2705 if (wc->low_wm_percent_set)
2706 extra_args += 2;
2707 if (wc->max_writeback_jobs_set)
2708 extra_args += 2;
2709 if (wc->autocommit_blocks_set)
2710 extra_args += 2;
2711 if (wc->autocommit_time_set)
2712 extra_args += 2;
2713 if (wc->max_age_set)
2714 extra_args += 2;
2715 if (wc->cleaner_set)
2716 extra_args++;
2717 if (wc->writeback_fua_set)
2718 extra_args++;
2719 if (wc->metadata_only)
2720 extra_args++;
2721 if (wc->pause_set)
2722 extra_args += 2;
2723
2724 DMEMIT("%u", extra_args);
2725 if (wc->start_sector_set)
2726 DMEMIT(" start_sector %llu", (unsigned long long)wc->start_sector);
2727 if (wc->high_wm_percent_set)
2728 DMEMIT(" high_watermark %u", wc->high_wm_percent_value);
2729 if (wc->low_wm_percent_set)
2730 DMEMIT(" low_watermark %u", wc->low_wm_percent_value);
2731 if (wc->max_writeback_jobs_set)
2732 DMEMIT(" writeback_jobs %u", wc->max_writeback_jobs);
2733 if (wc->autocommit_blocks_set)
2734 DMEMIT(" autocommit_blocks %u", wc->autocommit_blocks);
2735 if (wc->autocommit_time_set)
2736 DMEMIT(" autocommit_time %u", wc->autocommit_time_value);
2737 if (wc->max_age_set)
2738 DMEMIT(" max_age %u", wc->max_age_value);
2739 if (wc->cleaner_set)
2740 DMEMIT(" cleaner");
2741 if (wc->writeback_fua_set)
2742 DMEMIT(" %sfua", wc->writeback_fua ? "" : "no");
2743 if (wc->metadata_only)
2744 DMEMIT(" metadata_only");
2745 if (wc->pause_set)
2746 DMEMIT(" pause_writeback %u", wc->pause_value);
2747 break;
2748 case STATUSTYPE_IMA:
2749 *result = '\0';
2750 break;
2751 }
2752}
2753
2754static struct target_type writecache_target = {
2755 .name = "writecache",
2756 .version = {1, 6, 0},
2757 .module = THIS_MODULE,
2758 .ctr = writecache_ctr,
2759 .dtr = writecache_dtr,
2760 .status = writecache_status,
2761 .postsuspend = writecache_suspend,
2762 .resume = writecache_resume,
2763 .message = writecache_message,
2764 .map = writecache_map,
2765 .end_io = writecache_end_io,
2766 .iterate_devices = writecache_iterate_devices,
2767 .io_hints = writecache_io_hints,
2768};
2769module_dm(writecache);
2770
2771MODULE_DESCRIPTION(DM_NAME " writecache target");
2772MODULE_AUTHOR("Mikulas Patocka <[email protected]>");
2773MODULE_LICENSE("GPL");