projects/linux·blob master · refs

323 lines · 7.6 KiB

1// SPDX-License-Identifier: GPL-2.0-or-later
2
3#include <linux/blkdev.h>
4#include <linux/dax.h>
5#include <linux/vmalloc.h>
6#include <linux/parser.h>
7
8#include "cache_dev.h"
9#include "backing_dev.h"
10#include "cache.h"
11#include "dm_pcache.h"
12
13static void cache_dev_dax_exit(struct pcache_cache_dev *cache_dev)
14{
15 if (cache_dev->use_vmap)
16 vunmap(cache_dev->mapping);
17}
18
19static int build_vmap(struct dax_device *dax_dev, long total_pages, void **vaddr)
20{
21 struct page **pages;
22 long i = 0, chunk;
23 unsigned long pfn;
24 int ret;
25
26 pages = vmalloc_array(total_pages, sizeof(struct page *));
27 if (!pages)
28 return -ENOMEM;
29
30 do {
31 chunk = dax_direct_access(dax_dev, i, total_pages - i,
32 DAX_ACCESS, NULL, &pfn);
33 if (chunk <= 0) {
34 ret = chunk ? chunk : -EINVAL;
35 goto out_free;
36 }
37
38 if (!pfn_valid(pfn)) {
39 ret = -EOPNOTSUPP;
40 goto out_free;
41 }
42
43 while (chunk-- && i < total_pages) {
44 pages[i++] = pfn_to_page(pfn);
45 pfn++;
46 if (!(i & 15))
47 cond_resched();
48 }
49 } while (i < total_pages);
50
51 *vaddr = vmap(pages, total_pages, VM_MAP, PAGE_KERNEL);
52 if (!*vaddr) {
53 ret = -ENOMEM;
54 goto out_free;
55 }
56
57 ret = 0;
58
59out_free:
60 vfree(pages);
61 return ret;
62}
63
64static int cache_dev_dax_init(struct pcache_cache_dev *cache_dev)
65{
66 struct dm_pcache *pcache = CACHE_DEV_TO_PCACHE(cache_dev);
67 struct dax_device *dax_dev;
68 long total_pages, mapped_pages;
69 u64 bdev_size;
70 void *vaddr;
71 int ret;
72 int id;
73 unsigned long pfn;
74
75 dax_dev = cache_dev->dm_dev->dax_dev;
76 /* total size check */
77 bdev_size = bdev_nr_bytes(cache_dev->dm_dev->bdev);
78 if (bdev_size < PCACHE_CACHE_DEV_SIZE_MIN) {
79 pcache_dev_err(pcache, "dax device is too small, required at least %llu",
80 PCACHE_CACHE_DEV_SIZE_MIN);
81 ret = -ENOSPC;
82 goto out;
83 }
84
85 total_pages = bdev_size >> PAGE_SHIFT;
86 /* attempt: direct-map the whole range */
87 id = dax_read_lock();
88 mapped_pages = dax_direct_access(dax_dev, 0, total_pages,
89 DAX_ACCESS, &vaddr, &pfn);
90 if (mapped_pages < 0) {
91 pcache_dev_err(pcache, "dax_direct_access failed: %ld\n", mapped_pages);
92 ret = mapped_pages;
93 goto unlock;
94 }
95
96 if (!pfn_valid(pfn)) {
97 ret = -EOPNOTSUPP;
98 goto unlock;
99 }
100
101 if (mapped_pages == total_pages) {
102 /* success: contiguous direct mapping */
103 cache_dev->mapping = vaddr;
104 } else {
105 /* need vmap fallback */
106 ret = build_vmap(dax_dev, total_pages, &vaddr);
107 if (ret) {
108 pcache_dev_err(pcache, "vmap fallback failed: %d\n", ret);
109 goto unlock;
110 }
111
112 cache_dev->mapping = vaddr;
113 cache_dev->use_vmap = true;
114 }
115 dax_read_unlock(id);
116
117 return 0;
118unlock:
119 dax_read_unlock(id);
120out:
121 return ret;
122}
123
124void cache_dev_zero_range(struct pcache_cache_dev *cache_dev, void *pos, u32 size)
125{
126 memset(pos, 0, size);
127 dax_flush(cache_dev->dm_dev->dax_dev, pos, size);
128}
129
130static int sb_read(struct pcache_cache_dev *cache_dev, struct pcache_sb *sb)
131{
132 struct pcache_sb *sb_addr = CACHE_DEV_SB(cache_dev);
133
134 if (copy_mc_to_kernel(sb, sb_addr, sizeof(struct pcache_sb)))
135 return -EIO;
136
137 return 0;
138}
139
140static void sb_write(struct pcache_cache_dev *cache_dev, struct pcache_sb *sb)
141{
142 struct pcache_sb *sb_addr = CACHE_DEV_SB(cache_dev);
143
144 memcpy_flushcache(sb_addr, sb, sizeof(struct pcache_sb));
145 pmem_wmb();
146}
147
148static int sb_init(struct pcache_cache_dev *cache_dev, struct pcache_sb *sb)
149{
150 struct dm_pcache *pcache = CACHE_DEV_TO_PCACHE(cache_dev);
151 u64 nr_segs;
152 u64 cache_dev_size;
153 u64 magic;
154 u32 flags = 0;
155
156 magic = le64_to_cpu(sb->magic);
157 if (magic)
158 return -EEXIST;
159
160 cache_dev_size = bdev_nr_bytes(file_bdev(cache_dev->dm_dev->bdev_file));
161 if (cache_dev_size < PCACHE_CACHE_DEV_SIZE_MIN) {
162 pcache_dev_err(pcache, "dax device is too small, required at least %llu",
163 PCACHE_CACHE_DEV_SIZE_MIN);
164 return -ENOSPC;
165 }
166
167 nr_segs = (cache_dev_size - PCACHE_SEGMENTS_OFF) / ((PCACHE_SEG_SIZE));
168
169#if defined(__BYTE_ORDER) ? (__BIG_ENDIAN == __BYTE_ORDER) : defined(__BIG_ENDIAN)
170 flags |= PCACHE_SB_F_BIGENDIAN;
171#endif
172 sb->flags = cpu_to_le32(flags);
173 sb->magic = cpu_to_le64(PCACHE_MAGIC);
174 sb->seg_num = cpu_to_le32(nr_segs);
175 sb->crc = cpu_to_le32(crc32c(PCACHE_CRC_SEED, (void *)(sb) + 4, sizeof(struct pcache_sb) - 4));
176
177 cache_dev_zero_range(cache_dev, CACHE_DEV_CACHE_INFO(cache_dev),
178 PCACHE_CACHE_INFO_SIZE * PCACHE_META_INDEX_MAX +
179 PCACHE_CACHE_CTRL_SIZE);
180
181 return 0;
182}
183
184static int sb_validate(struct pcache_cache_dev *cache_dev, struct pcache_sb *sb)
185{
186 struct dm_pcache *pcache = CACHE_DEV_TO_PCACHE(cache_dev);
187 u32 flags;
188 u32 crc;
189
190 if (le64_to_cpu(sb->magic) != PCACHE_MAGIC) {
191 pcache_dev_err(pcache, "unexpected magic: %llx\n",
192 le64_to_cpu(sb->magic));
193 return -EINVAL;
194 }
195
196 crc = crc32c(PCACHE_CRC_SEED, (void *)(sb) + 4, sizeof(struct pcache_sb) - 4);
197 if (crc != le32_to_cpu(sb->crc)) {
198 pcache_dev_err(pcache, "corrupted sb: %u, expected: %u\n", crc, le32_to_cpu(sb->crc));
199 return -EINVAL;
200 }
201
202 flags = le32_to_cpu(sb->flags);
203#if defined(__BYTE_ORDER) ? (__BIG_ENDIAN == __BYTE_ORDER) : defined(__BIG_ENDIAN)
204 if (!(flags & PCACHE_SB_F_BIGENDIAN)) {
205 pcache_dev_err(pcache, "cache_dev is not big endian\n");
206 return -EINVAL;
207 }
208#else
209 if (flags & PCACHE_SB_F_BIGENDIAN) {
210 pcache_dev_err(pcache, "cache_dev is big endian\n");
211 return -EINVAL;
212 }
213#endif
214 return 0;
215}
216
217static int cache_dev_init(struct pcache_cache_dev *cache_dev, u32 seg_num)
218{
219 cache_dev->seg_num = seg_num;
220 cache_dev->seg_bitmap = kvcalloc(BITS_TO_LONGS(cache_dev->seg_num), sizeof(unsigned long), GFP_KERNEL);
221 if (!cache_dev->seg_bitmap)
222 return -ENOMEM;
223
224 return 0;
225}
226
227static void cache_dev_exit(struct pcache_cache_dev *cache_dev)
228{
229 kvfree(cache_dev->seg_bitmap);
230}
231
232void cache_dev_stop(struct dm_pcache *pcache)
233{
234 struct pcache_cache_dev *cache_dev = &pcache->cache_dev;
235
236 cache_dev_exit(cache_dev);
237 cache_dev_dax_exit(cache_dev);
238}
239
240int cache_dev_start(struct dm_pcache *pcache)
241{
242 struct pcache_cache_dev *cache_dev = &pcache->cache_dev;
243 struct pcache_sb sb;
244 bool format = false;
245 u32 seg_num;
246 u64 max_segs;
247 int ret;
248
249 mutex_init(&cache_dev->seg_lock);
250
251 ret = cache_dev_dax_init(cache_dev);
252 if (ret) {
253 pcache_dev_err(pcache, "failed to init cache_dev %s via dax way: %d.",
254 cache_dev->dm_dev->name, ret);
255 goto err;
256 }
257
258 ret = sb_read(cache_dev, &sb);
259 if (ret)
260 goto dax_release;
261
262 if (le64_to_cpu(sb.magic) == 0) {
263 format = true;
264 ret = sb_init(cache_dev, &sb);
265 if (ret < 0)
266 goto dax_release;
267 }
268
269 ret = sb_validate(cache_dev, &sb);
270 if (ret)
271 goto dax_release;
272
273 cache_dev->sb_flags = le32_to_cpu(sb.flags);
274
275 /*
276 * seg_num is read from the crc32c-only superblock, so whoever supplies
277 * the cache device controls it. It is the ceiling every later on-media
278 * segment id is validated against, so bound it against what the device
279 * physically holds before it is trusted, or a forged seg_num lets a
280 * segment id address past the DAX mapping.
281 */
282 seg_num = le32_to_cpu(sb.seg_num);
283 max_segs = (bdev_nr_bytes(cache_dev->dm_dev->bdev) - PCACHE_SEGMENTS_OFF) /
284 PCACHE_SEG_SIZE;
285 if (seg_num == 0 || seg_num > max_segs || seg_num > PCACHE_CACHE_SEGS_MAX) {
286 pcache_dev_err(pcache, "invalid seg_num %u from cache device (device holds %llu, max %u)\n",
287 seg_num, max_segs, (u32)PCACHE_CACHE_SEGS_MAX);
288 ret = -EIO;
289 goto dax_release;
290 }
291
292 ret = cache_dev_init(cache_dev, seg_num);
293 if (ret)
294 goto dax_release;
295
296 if (format)
297 sb_write(cache_dev, &sb);
298
299 return 0;
300
301dax_release:
302 cache_dev_dax_exit(cache_dev);
303err:
304 return ret;
305}
306
307int cache_dev_get_empty_segment_id(struct pcache_cache_dev *cache_dev, u32 *seg_id)
308{
309 int ret;
310
311 mutex_lock(&cache_dev->seg_lock);
312 *seg_id = find_next_zero_bit(cache_dev->seg_bitmap, cache_dev->seg_num, 0);
313 if (*seg_id == cache_dev->seg_num) {
314 ret = -ENOSPC;
315 goto unlock;
316 }
317
318 __set_bit(*seg_id, cache_dev->seg_bitmap);
319 ret = 0;
320unlock:
321 mutex_unlock(&cache_dev->seg_lock);
322 return ret;
323}