projects/linux·blob master · refs

409 lines · 10.9 KiB

1// SPDX-License-Identifier: GPL-2.0
2/* Copyright(c) 2026 Micron Technology, Inc. */
3#include <linux/memremap.h>
4#include <linux/pagemap.h>
5#include <linux/module.h>
6#include <linux/device.h>
7#include <linux/cdev.h>
8#include <linux/slab.h>
9#include <linux/dax.h>
10#include <linux/uio.h>
11#include <linux/fs.h>
12#include <linux/mm.h>
13#include "dax-private.h"
14#include "bus.h"
15
16/*
17 * FS-DAX compatible devdax driver
18 *
19 * Unlike drivers/dax/device.c which pre-initializes compound folios based
20 * on device alignment (via vmemmap_shift), this driver leaves folios
21 * uninitialized similar to pmem. This allows fs-dax filesystems like famfs
22 * to work without needing special handling for pre-initialized folios.
23 *
24 * Key differences from device.c:
25 * - pgmap type is MEMORY_DEVICE_FS_DAX (not MEMORY_DEVICE_GENERIC)
26 * - vmemmap_shift is NOT set (folios remain order-0)
27 * - fs-dax can dynamically create compound folios as needed
28 * - No mmap support - all access is through fs-dax/iomap
29 */
30
31static void fsdev_write_dax(void *addr, struct page *page,
32 unsigned int off, unsigned int len)
33{
34 while (len) {
35 void *mem = kmap_local_page(page);
36 unsigned int chunk = min_t(unsigned int, len, PAGE_SIZE - off);
37
38 memcpy_flushcache(addr, mem + off, chunk);
39 kunmap_local(mem);
40 len -= chunk;
41 off = 0;
42 page++;
43 addr += chunk;
44 }
45}
46
47static long __fsdev_dax_direct_access(struct dax_device *dax_dev, pgoff_t pgoff,
48 long nr_pages, enum dax_access_mode mode, void **kaddr,
49 unsigned long *pfn)
50{
51 struct dev_dax *dev_dax = dax_get_private(dax_dev);
52 size_t size = nr_pages << PAGE_SHIFT;
53 size_t offset = pgoff << PAGE_SHIFT;
54 phys_addr_t phys;
55
56 phys = dax_pgoff_to_phys(dev_dax, pgoff, size);
57 if (phys == -1) {
58 dev_dbg(&dev_dax->dev,
59 "pgoff (%#lx) out of range\n", pgoff);
60 return -EFAULT;
61 }
62
63 if (kaddr)
64 *kaddr = __va(phys);
65
66 if (pfn)
67 *pfn = PHYS_PFN(phys);
68
69 /*
70 * Use cached_size which was computed at probe time. The size cannot
71 * change while the driver is bound (resize returns -EBUSY).
72 */
73 return PHYS_PFN(min(size, dev_dax->cached_size - offset));
74}
75
76static int fsdev_dax_zero_page_range(struct dax_device *dax_dev,
77 pgoff_t pgoff, size_t nr_pages)
78{
79 void *kaddr;
80 long rc;
81
82 WARN_ONCE(nr_pages > 1, "%s: nr_pages > 1\n", __func__);
83 rc = __fsdev_dax_direct_access(dax_dev, pgoff, 1, DAX_ACCESS,
84 &kaddr, NULL);
85 if (rc < 0)
86 return rc;
87 fsdev_write_dax(kaddr, ZERO_PAGE(0), 0, PAGE_SIZE);
88 return 0;
89}
90
91static long fsdev_dax_direct_access(struct dax_device *dax_dev,
92 pgoff_t pgoff, long nr_pages, enum dax_access_mode mode,
93 void **kaddr, unsigned long *pfn)
94{
95 return __fsdev_dax_direct_access(dax_dev, pgoff, nr_pages, mode,
96 kaddr, pfn);
97}
98
99static size_t fsdev_dax_recovery_write(struct dax_device *dax_dev,
100 pgoff_t pgoff, void *addr, size_t bytes, struct iov_iter *i)
101{
102 return _copy_from_iter_flushcache(addr, bytes, i);
103}
104
105static const struct dax_operations dev_dax_ops = {
106 .direct_access = fsdev_dax_direct_access,
107 .zero_page_range = fsdev_dax_zero_page_range,
108 .recovery_write = fsdev_dax_recovery_write,
109};
110
111static void fsdev_cdev_del(void *cdev)
112{
113 cdev_del(cdev);
114}
115
116static void fsdev_kill(void *dev_dax)
117{
118 kill_dev_dax(dev_dax);
119}
120
121static void fsdev_clear_ops(void *data)
122{
123 struct dev_dax *dev_dax = data;
124
125 dax_set_ops(dev_dax->dax_dev, NULL);
126}
127
128static void fsdev_clear_pgmap_ops(void *data)
129{
130 struct dev_pagemap *pgmap = data;
131
132 /*
133 * fsdev installs pgmap->ops and ->owner at probe. For a static device
134 * the pgmap is shared and long-lived (owned by the dax bus), so
135 * leaving fsdev's ops behind on unbind would let a later
136 * memory_failure -- after rebind to another driver, or after this
137 * module is unloaded -- dispatch through a stale or freed
138 * ->memory_failure handler. Clear them so the pgmap carries no fsdev
139 * state once we are unbound.
140 */
141 pgmap->ops = NULL;
142 pgmap->owner = NULL;
143}
144
145/*
146 * Page map operations for FS-DAX mode
147 * Similar to fsdax_pagemap_ops in drivers/nvdimm/pmem.c
148 *
149 * Note: folio_free callback is not needed for MEMORY_DEVICE_FS_DAX.
150 * The core mm code in free_zone_device_folio() handles the wake_up_var()
151 * directly for this memory type.
152 */
153static u64 fsdev_pfn_to_offset(struct dev_dax *dev_dax, unsigned long pfn)
154{
155 phys_addr_t phys = PFN_PHYS(pfn);
156 u64 offset = 0;
157
158 for (int i = 0; i < dev_dax->nr_range; i++) {
159 struct range *range = &dev_dax->ranges[i].range;
160
161 if (phys >= range->start && phys <= range->end)
162 return offset + (phys - range->start);
163 offset += range_len(range);
164 }
165 return -1ULL;
166}
167
168static int fsdev_pagemap_memory_failure(struct dev_pagemap *pgmap,
169 unsigned long pfn, unsigned long nr_pages, int mf_flags)
170{
171 struct dev_dax *dev_dax = pgmap->owner;
172 u64 offset = fsdev_pfn_to_offset(dev_dax, pfn);
173 u64 len = nr_pages << PAGE_SHIFT;
174
175 return dax_holder_notify_failure(dev_dax->dax_dev, offset,
176 len, mf_flags);
177}
178
179static const struct dev_pagemap_ops fsdev_pagemap_ops = {
180 .memory_failure = fsdev_pagemap_memory_failure,
181};
182
183/*
184 * Clear any stale folio state from pages in the given range.
185 * This is necessary because device_dax pre-initializes compound folios
186 * based on vmemmap_shift, and that state may persist after driver unbind.
187 * Since fsdev_dax uses MEMORY_DEVICE_FS_DAX without vmemmap_shift, fs-dax
188 * expects to find clean order-0 folios that it can build into compound
189 * folios on demand.
190 *
191 * At probe time, no filesystem should be mounted yet, so all mappings
192 * are stale and must be cleared along with compound state.
193 */
194static void fsdev_clear_folio_state(struct dev_dax *dev_dax)
195{
196 for (int i = 0; i < dev_dax->nr_range; i++) {
197 struct range *range = &dev_dax->ranges[i].range;
198 unsigned long pfn = PHYS_PFN(range->start);
199 unsigned long end_pfn = PHYS_PFN(range->end) + 1;
200
201 while (pfn < end_pfn) {
202 struct folio *folio = pfn_folio(pfn);
203 int order = dax_folio_reset_order(folio);
204
205 pfn += 1UL << order;
206 }
207 }
208}
209
210static void fsdev_clear_folio_state_action(void *data)
211{
212 fsdev_clear_folio_state(data);
213}
214
215static int fsdev_open(struct inode *inode, struct file *filp)
216{
217 struct dax_device *dax_dev = inode_dax(inode);
218 struct dev_dax *dev_dax = dax_get_private(dax_dev);
219
220 filp->private_data = dev_dax;
221
222 return 0;
223}
224
225static int fsdev_release(struct inode *inode, struct file *filp)
226{
227 return 0;
228}
229
230static const struct file_operations fsdev_fops = {
231 .llseek = noop_llseek,
232 .owner = THIS_MODULE,
233 .open = fsdev_open,
234 .release = fsdev_release,
235};
236
237/*
238 * Acquire the dev_pagemap for probe: the static (pre-populated) one if
239 * present, or a devm-allocated one for the dynamic case. Note that
240 * dev_dax->pgmap is not set here; fsdev_dax_probe() sets it only once
241 * probe succeeds, so a failed probe never leaves a dangling pointer
242 * to a devres-freed pgmap.
243 */
244static struct dev_pagemap *fsdev_acquire_pgmap(struct dev_dax *dev_dax)
245{
246 struct device *dev = &dev_dax->dev;
247 struct dev_pagemap *pgmap;
248 size_t pgmap_size;
249
250 if (static_dev_dax(dev_dax)) {
251 if (dev_dax->nr_range > 1) {
252 dev_warn(dev,
253 "static pgmap / multi-range device conflict\n");
254 return ERR_PTR(-EINVAL);
255 }
256
257 pgmap = dev_dax->pgmap;
258 pgmap->vmemmap_shift = 0;
259 return pgmap;
260 }
261
262 if (dev_dax->pgmap) {
263 dev_warn(dev, "dynamic-dax with pre-populated page map\n");
264 return ERR_PTR(-EINVAL);
265 }
266
267 pgmap_size = struct_size(pgmap, ranges, dev_dax->nr_range - 1);
268 pgmap = devm_kzalloc(dev, pgmap_size, GFP_KERNEL);
269 if (!pgmap)
270 return ERR_PTR(-ENOMEM);
271
272 pgmap->nr_range = dev_dax->nr_range;
273 for (int i = 0; i < dev_dax->nr_range; i++)
274 pgmap->ranges[i] = dev_dax->ranges[i].range;
275
276 return pgmap;
277}
278
279static int fsdev_dax_probe(struct dev_dax *dev_dax)
280{
281 struct dax_device *dax_dev = dev_dax->dax_dev;
282 struct device *dev = &dev_dax->dev;
283 struct dev_pagemap *pgmap;
284 struct inode *inode;
285 u64 data_offset = 0;
286 struct cdev *cdev;
287 void *addr;
288 int rc, i;
289
290 pgmap = fsdev_acquire_pgmap(dev_dax);
291 if (IS_ERR(pgmap))
292 return PTR_ERR(pgmap);
293
294 for (i = 0; i < dev_dax->nr_range; i++) {
295 struct range *range = &dev_dax->ranges[i].range;
296
297 if (!devm_request_mem_region(dev, range->start,
298 range_len(range), dev_name(dev))) {
299 dev_warn(dev, "mapping%d: %#llx-%#llx could not reserve range\n",
300 i, range->start, range->end);
301 return -EBUSY;
302 }
303 }
304
305 /* Cache size now; it cannot change while driver is bound */
306 dev_dax->cached_size = 0;
307 for (i = 0; i < dev_dax->nr_range; i++)
308 dev_dax->cached_size += range_len(&dev_dax->ranges[i].range);
309
310 /*
311 * Use MEMORY_DEVICE_FS_DAX without setting vmemmap_shift, leaving
312 * folios at order-0. Unlike device.c (MEMORY_DEVICE_GENERIC), this
313 * lets fs-dax dynamically build compound folios as needed, similar
314 * to pmem behavior.
315 */
316 pgmap->type = MEMORY_DEVICE_FS_DAX;
317 pgmap->ops = &fsdev_pagemap_ops;
318 pgmap->owner = dev_dax;
319
320 addr = devm_memremap_pages(dev, pgmap);
321 if (IS_ERR(addr))
322 return PTR_ERR(addr);
323
324 /* Drop fsdev's pgmap->ops/owner on unbind so no stale ops survive. */
325 rc = devm_add_action_or_reset(dev, fsdev_clear_pgmap_ops, pgmap);
326 if (rc)
327 return rc;
328
329 /*
330 * Clear any stale compound folio state left over from a previous
331 * driver (e.g., device_dax with vmemmap_shift). Also register this
332 * as a devm action so folio state is cleared on unbind, ensuring
333 * clean pages for subsequent drivers (e.g., kmem for system-ram).
334 */
335 fsdev_clear_folio_state(dev_dax);
336 rc = devm_add_action_or_reset(dev, fsdev_clear_folio_state_action,
337 dev_dax);
338 if (rc)
339 return rc;
340
341 /* Detect whether the data is at a non-zero offset into the memory */
342 if (pgmap->range.start != dev_dax->ranges[0].range.start) {
343 u64 phys = dev_dax->ranges[0].range.start;
344 u64 pgmap_phys = pgmap[0].range.start;
345
346 if (pgmap_phys > phys) {
347 dev_err(dev, "pgmap start %#llx exceeds data start %#llx\n",
348 pgmap_phys, phys);
349 return -EINVAL;
350 }
351 data_offset = phys - pgmap_phys;
352
353 pr_debug("%s: offset detected phys=%llx pgmap_phys=%llx offset=%llx\n",
354 __func__, phys, pgmap_phys, data_offset);
355 }
356
357 inode = dax_inode(dax_dev);
358 cdev = inode->i_cdev;
359 cdev_init(cdev, &fsdev_fops);
360 cdev->owner = dev->driver->owner;
361 cdev_set_parent(cdev, &dev->kobj);
362 rc = cdev_add(cdev, dev->devt, 1);
363 if (rc)
364 return rc;
365
366 rc = devm_add_action_or_reset(dev, fsdev_cdev_del, cdev);
367 if (rc)
368 return rc;
369
370 /* Set the dax operations for fs-dax access path */
371 rc = dax_set_ops(dax_dev, &dev_dax_ops);
372 if (rc)
373 return rc;
374
375 rc = devm_add_action_or_reset(dev, fsdev_clear_ops, dev_dax);
376 if (rc)
377 return rc;
378
379 run_dax(dax_dev);
380 rc = devm_add_action_or_reset(dev, fsdev_kill, dev_dax);
381 if (rc)
382 return rc;
383
384 /* Probe can no longer fail; expose the pgmap via dev_dax */
385 dev_dax->pgmap = pgmap;
386 return 0;
387}
388
389static struct dax_device_driver fsdev_dax_driver = {
390 .probe = fsdev_dax_probe,
391 .type = DAXDRV_FSDEV_TYPE,
392};
393
394static int __init dax_init(void)
395{
396 return dax_driver_register(&fsdev_dax_driver);
397}
398
399static void __exit dax_exit(void)
400{
401 dax_driver_unregister(&fsdev_dax_driver);
402}
403
404MODULE_AUTHOR("John Groves");
405MODULE_DESCRIPTION("FS-DAX Device: fs-dax compatible devdax driver");
406MODULE_LICENSE("GPL");
407module_init(dax_init);
408module_exit(dax_exit);
409MODULE_ALIAS_DAX_DEVICE(0);