projects/linux·blob master · refs

1356 lines · 35.3 KiB

1// SPDX-License-Identifier: GPL-2.0
2/*
3 * dax: direct host memory access
4 * Copyright (C) 2020 Red Hat, Inc.
5 */
6
7#include "fuse_i.h"
8
9#include <linux/delay.h>
10#include <linux/dax.h>
11#include <linux/uio.h>
12#include <linux/pagemap.h>
13#include <linux/iomap.h>
14#include <linux/interval_tree.h>
15
16/*
17 * Default memory range size. A power of 2 so it agrees with common FUSE_INIT
18 * map_alignment values 4KB and 64KB.
19 */
20#define FUSE_DAX_SHIFT 21
21#define FUSE_DAX_SZ (1 << FUSE_DAX_SHIFT)
22#define FUSE_DAX_PAGES (FUSE_DAX_SZ / PAGE_SIZE)
23
24/* Number of ranges reclaimer will try to free in one invocation */
25#define FUSE_DAX_RECLAIM_CHUNK (10)
26
27/*
28 * Dax memory reclaim threshold in percetage of total ranges. When free
29 * number of free ranges drops below this threshold, reclaim can trigger
30 * Default is 20%
31 */
32#define FUSE_DAX_RECLAIM_THRESHOLD (20)
33
34/** Translation information for file offsets to DAX window offsets */
35struct fuse_dax_mapping {
36 /* Pointer to inode where this memory range is mapped */
37 struct inode *inode;
38
39 /* Will connect in fcd->free_ranges to keep track of free memory */
40 struct list_head list;
41
42 /* For interval tree in file/inode */
43 struct interval_tree_node itn;
44
45 /* Will connect in fc->busy_ranges to keep track busy memory */
46 struct list_head busy_list;
47
48 /** Position in DAX window */
49 u64 window_offset;
50
51 /** Length of mapping, in bytes */
52 loff_t length;
53
54 /* Is this mapping read-only or read-write */
55 bool writable;
56
57 /* reference count when the mapping is used by dax iomap. */
58 refcount_t refcnt;
59};
60
61/* Per-inode dax map */
62struct fuse_inode_dax {
63 /* Semaphore to protect modifications to the dmap tree */
64 struct rw_semaphore sem;
65
66 /* Sorted rb tree of struct fuse_dax_mapping elements */
67 struct rb_root_cached tree;
68 unsigned long nr;
69};
70
71struct fuse_conn_dax {
72 /* DAX device */
73 struct dax_device *dev;
74
75 /* Lock protecting accessess to members of this structure */
76 spinlock_t lock;
77
78 /* List of memory ranges which are busy */
79 unsigned long nr_busy_ranges;
80 struct list_head busy_ranges;
81
82 /* Worker to free up memory ranges */
83 struct delayed_work free_work;
84
85 /* Wait queue for a dax range to become free */
86 wait_queue_head_t range_waitq;
87
88 /* DAX Window Free Ranges */
89 long nr_free_ranges;
90 struct list_head free_ranges;
91
92 unsigned long nr_ranges;
93};
94
95static inline struct fuse_dax_mapping *
96node_to_dmap(struct interval_tree_node *node)
97{
98 if (!node)
99 return NULL;
100
101 return container_of(node, struct fuse_dax_mapping, itn);
102}
103
104static struct fuse_dax_mapping *
105alloc_dax_mapping_reclaim(struct fuse_conn_dax *fcd, struct inode *inode);
106
107static void
108__kick_dmap_free_worker(struct fuse_conn_dax *fcd, unsigned long delay_ms)
109{
110 unsigned long free_threshold;
111
112 /* If number of free ranges are below threshold, start reclaim */
113 free_threshold = max_t(unsigned long, fcd->nr_ranges * FUSE_DAX_RECLAIM_THRESHOLD / 100,
114 1);
115 if (fcd->nr_free_ranges < free_threshold)
116 queue_delayed_work(system_dfl_long_wq, &fcd->free_work,
117 msecs_to_jiffies(delay_ms));
118}
119
120static void kick_dmap_free_worker(struct fuse_conn_dax *fcd,
121 unsigned long delay_ms)
122{
123 spin_lock(&fcd->lock);
124 __kick_dmap_free_worker(fcd, delay_ms);
125 spin_unlock(&fcd->lock);
126}
127
128static struct fuse_dax_mapping *alloc_dax_mapping(struct fuse_conn_dax *fcd)
129{
130 struct fuse_dax_mapping *dmap;
131
132 spin_lock(&fcd->lock);
133 dmap = list_first_entry_or_null(&fcd->free_ranges,
134 struct fuse_dax_mapping, list);
135 if (dmap) {
136 list_del_init(&dmap->list);
137 WARN_ON(fcd->nr_free_ranges <= 0);
138 fcd->nr_free_ranges--;
139 }
140 __kick_dmap_free_worker(fcd, 0);
141 spin_unlock(&fcd->lock);
142
143 return dmap;
144}
145
146/* This assumes fcd->lock is held */
147static void __dmap_remove_busy_list(struct fuse_conn_dax *fcd,
148 struct fuse_dax_mapping *dmap)
149{
150 list_del_init(&dmap->busy_list);
151 WARN_ON(fcd->nr_busy_ranges == 0);
152 fcd->nr_busy_ranges--;
153}
154
155static void dmap_remove_busy_list(struct fuse_conn_dax *fcd,
156 struct fuse_dax_mapping *dmap)
157{
158 spin_lock(&fcd->lock);
159 __dmap_remove_busy_list(fcd, dmap);
160 spin_unlock(&fcd->lock);
161}
162
163/* This assumes fcd->lock is held */
164static void __dmap_add_to_free_pool(struct fuse_conn_dax *fcd,
165 struct fuse_dax_mapping *dmap)
166{
167 list_add_tail(&dmap->list, &fcd->free_ranges);
168 fcd->nr_free_ranges++;
169 wake_up(&fcd->range_waitq);
170}
171
172static void dmap_add_to_free_pool(struct fuse_conn_dax *fcd,
173 struct fuse_dax_mapping *dmap)
174{
175 /* Return fuse_dax_mapping to free list */
176 spin_lock(&fcd->lock);
177 __dmap_add_to_free_pool(fcd, dmap);
178 spin_unlock(&fcd->lock);
179}
180
181static int fuse_setup_one_mapping(struct inode *inode, unsigned long start_idx,
182 struct fuse_dax_mapping *dmap, bool writable,
183 bool upgrade)
184{
185 struct fuse_mount *fm = get_fuse_mount(inode);
186 struct fuse_conn_dax *fcd = fm->fc->dax;
187 struct fuse_inode *fi = get_fuse_inode(inode);
188 struct fuse_setupmapping_in inarg;
189 loff_t offset = start_idx << FUSE_DAX_SHIFT;
190 FUSE_ARGS(args);
191 ssize_t err;
192
193 WARN_ON(fcd->nr_free_ranges < 0);
194
195 /* Ask fuse daemon to setup mapping */
196 memset(&inarg, 0, sizeof(inarg));
197 inarg.foffset = offset;
198 inarg.fh = -1;
199 inarg.moffset = dmap->window_offset;
200 inarg.len = FUSE_DAX_SZ;
201 inarg.flags |= FUSE_SETUPMAPPING_FLAG_READ;
202 if (writable)
203 inarg.flags |= FUSE_SETUPMAPPING_FLAG_WRITE;
204 args.opcode = FUSE_SETUPMAPPING;
205 args.nodeid = fi->nodeid;
206 args.in_numargs = 1;
207 args.in_args[0].size = sizeof(inarg);
208 args.in_args[0].value = &inarg;
209 err = fuse_simple_request(fm, &args);
210 if (err < 0)
211 return err;
212 dmap->writable = writable;
213 if (!upgrade) {
214 /*
215 * We don't take a reference on inode. inode is valid right now
216 * and when inode is going away, cleanup logic should first
217 * cleanup dmap entries.
218 */
219 dmap->inode = inode;
220 dmap->itn.start = dmap->itn.last = start_idx;
221 /* Protected by fi->dax->sem */
222 interval_tree_insert(&dmap->itn, &fi->dax->tree);
223 fi->dax->nr++;
224 spin_lock(&fcd->lock);
225 list_add_tail(&dmap->busy_list, &fcd->busy_ranges);
226 fcd->nr_busy_ranges++;
227 spin_unlock(&fcd->lock);
228 }
229 return 0;
230}
231
232static int fuse_send_removemapping(struct inode *inode,
233 struct fuse_removemapping_in *inargp,
234 struct fuse_removemapping_one *remove_one)
235{
236 struct fuse_inode *fi = get_fuse_inode(inode);
237 struct fuse_mount *fm = get_fuse_mount(inode);
238 FUSE_ARGS(args);
239
240 args.opcode = FUSE_REMOVEMAPPING;
241 args.nodeid = fi->nodeid;
242 args.in_numargs = 3;
243 fuse_set_zero_arg0(&args);
244 args.in_args[1].size = sizeof(*inargp);
245 args.in_args[1].value = inargp;
246 args.in_args[2].size = inargp->count * sizeof(*remove_one);
247 args.in_args[2].value = remove_one;
248 return fuse_simple_request(fm, &args);
249}
250
251static int dmap_removemapping_list(struct inode *inode, unsigned int num,
252 struct list_head *to_remove)
253{
254 struct fuse_removemapping_one *remove_one, *ptr;
255 struct fuse_removemapping_in inarg;
256 struct fuse_dax_mapping *dmap;
257 int ret, i = 0, nr_alloc;
258
259 nr_alloc = min_t(unsigned int, num, FUSE_REMOVEMAPPING_MAX_ENTRY);
260 remove_one = kmalloc_objs(*remove_one, nr_alloc, GFP_NOFS);
261 if (!remove_one)
262 return -ENOMEM;
263
264 ptr = remove_one;
265 list_for_each_entry(dmap, to_remove, list) {
266 ptr->moffset = dmap->window_offset;
267 ptr->len = dmap->length;
268 ptr++;
269 i++;
270 num--;
271 if (i >= nr_alloc || num == 0) {
272 memset(&inarg, 0, sizeof(inarg));
273 inarg.count = i;
274 ret = fuse_send_removemapping(inode, &inarg,
275 remove_one);
276 if (ret)
277 goto out;
278 ptr = remove_one;
279 i = 0;
280 }
281 }
282out:
283 kfree(remove_one);
284 return ret;
285}
286
287/*
288 * Cleanup dmap entry and add back to free list. This should be called with
289 * fcd->lock held.
290 */
291static void dmap_reinit_add_to_free_pool(struct fuse_conn_dax *fcd,
292 struct fuse_dax_mapping *dmap)
293{
294 pr_debug("fuse: freeing memory range start_idx=0x%lx end_idx=0x%lx window_offset=0x%llx length=0x%llx\n",
295 dmap->itn.start, dmap->itn.last, dmap->window_offset,
296 dmap->length);
297 __dmap_remove_busy_list(fcd, dmap);
298 dmap->inode = NULL;
299 dmap->itn.start = dmap->itn.last = 0;
300 __dmap_add_to_free_pool(fcd, dmap);
301}
302
303/*
304 * Free inode dmap entries whose range falls inside [start, end].
305 * Does not take any locks. At this point of time it should only be
306 * called from evict_inode() path where we know all dmap entries can be
307 * reclaimed.
308 */
309static void inode_reclaim_dmap_range(struct fuse_conn_dax *fcd,
310 struct inode *inode,
311 loff_t start, loff_t end)
312{
313 struct fuse_inode *fi = get_fuse_inode(inode);
314 struct fuse_dax_mapping *dmap, *n;
315 int err, num = 0;
316 LIST_HEAD(to_remove);
317 unsigned long start_idx = start >> FUSE_DAX_SHIFT;
318 unsigned long end_idx = end >> FUSE_DAX_SHIFT;
319 struct interval_tree_node *node;
320
321 while (1) {
322 node = interval_tree_iter_first(&fi->dax->tree, start_idx,
323 end_idx);
324 if (!node)
325 break;
326 dmap = node_to_dmap(node);
327 /* inode is going away. There should not be any users of dmap */
328 WARN_ON(refcount_read(&dmap->refcnt) > 1);
329 interval_tree_remove(&dmap->itn, &fi->dax->tree);
330 num++;
331 list_add(&dmap->list, &to_remove);
332 }
333
334 /* Nothing to remove */
335 if (list_empty(&to_remove))
336 return;
337
338 WARN_ON(fi->dax->nr < num);
339 fi->dax->nr -= num;
340 err = dmap_removemapping_list(inode, num, &to_remove);
341 if (err && err != -ENOTCONN) {
342 pr_warn("Failed to removemappings. start=0x%llx end=0x%llx\n",
343 start, end);
344 }
345 spin_lock(&fcd->lock);
346 list_for_each_entry_safe(dmap, n, &to_remove, list) {
347 list_del_init(&dmap->list);
348 dmap_reinit_add_to_free_pool(fcd, dmap);
349 }
350 spin_unlock(&fcd->lock);
351}
352
353static int dmap_removemapping_one(struct inode *inode,
354 struct fuse_dax_mapping *dmap)
355{
356 struct fuse_removemapping_one forget_one;
357 struct fuse_removemapping_in inarg;
358
359 memset(&inarg, 0, sizeof(inarg));
360 inarg.count = 1;
361 memset(&forget_one, 0, sizeof(forget_one));
362 forget_one.moffset = dmap->window_offset;
363 forget_one.len = dmap->length;
364
365 return fuse_send_removemapping(inode, &inarg, &forget_one);
366}
367
368/*
369 * It is called from evict_inode() and by that time inode is going away. So
370 * this function does not take any locks like fi->dax->sem for traversing
371 * that fuse inode interval tree. If that lock is taken then lock validator
372 * complains of deadlock situation w.r.t fs_reclaim lock.
373 */
374void fuse_dax_inode_cleanup(struct inode *inode)
375{
376 struct fuse_conn *fc = get_fuse_conn(inode);
377 struct fuse_inode *fi = get_fuse_inode(inode);
378
379 /*
380 * fuse_evict_inode() has already called truncate_inode_pages_final()
381 * before we arrive here. So we should not have to worry about any
382 * pages/exception entries still associated with inode.
383 */
384 inode_reclaim_dmap_range(fc->dax, inode, 0, -1);
385 WARN_ON(fi->dax->nr);
386}
387
388static void fuse_fill_iomap_hole(struct iomap *iomap, loff_t length)
389{
390 iomap->addr = IOMAP_NULL_ADDR;
391 iomap->length = length;
392 iomap->type = IOMAP_HOLE;
393}
394
395static void fuse_fill_iomap(struct inode *inode, loff_t pos, loff_t length,
396 struct iomap *iomap, struct fuse_dax_mapping *dmap,
397 unsigned int flags)
398{
399 loff_t offset, len;
400 loff_t i_size = i_size_read(inode);
401
402 offset = pos - (dmap->itn.start << FUSE_DAX_SHIFT);
403 len = min(length, dmap->length - offset);
404
405 /* If length is beyond end of file, truncate further */
406 if (pos + len > i_size)
407 len = i_size - pos;
408
409 if (len > 0) {
410 iomap->addr = dmap->window_offset + offset;
411 iomap->length = len;
412 if (flags & IOMAP_FAULT)
413 iomap->length = ALIGN(len, PAGE_SIZE);
414 iomap->type = IOMAP_MAPPED;
415 /*
416 * increace refcnt so that reclaim code knows this dmap is in
417 * use. This assumes fi->dax->sem mutex is held either
418 * shared/exclusive.
419 */
420 refcount_inc(&dmap->refcnt);
421
422 /* iomap->private should be NULL */
423 WARN_ON_ONCE(iomap->private);
424 iomap->private = dmap;
425 } else {
426 /* Mapping beyond end of file is hole */
427 fuse_fill_iomap_hole(iomap, length);
428 }
429}
430
431static int fuse_setup_new_dax_mapping(struct inode *inode, loff_t pos,
432 loff_t length, unsigned int flags,
433 struct iomap *iomap)
434{
435 struct fuse_inode *fi = get_fuse_inode(inode);
436 struct fuse_conn *fc = get_fuse_conn(inode);
437 struct fuse_conn_dax *fcd = fc->dax;
438 struct fuse_dax_mapping *dmap, *alloc_dmap = NULL;
439 int ret;
440 bool writable = flags & IOMAP_WRITE;
441 unsigned long start_idx = pos >> FUSE_DAX_SHIFT;
442 struct interval_tree_node *node;
443
444 /*
445 * Can't do inline reclaim in fault path. We call
446 * dax_layout_busy_page() before we free a range. And
447 * fuse_wait_dax_page() drops mapping->invalidate_lock and requires it.
448 * In fault path we enter with mapping->invalidate_lock held and can't
449 * drop it. Also in fault path we hold mapping->invalidate_lock shared
450 * and not exclusive, so that creates further issues with
451 * fuse_wait_dax_page(). Hence return -EAGAIN and fuse_dax_fault()
452 * will wait for a memory range to become free and retry.
453 */
454 if (flags & IOMAP_FAULT) {
455 alloc_dmap = alloc_dax_mapping(fcd);
456 if (!alloc_dmap)
457 return -EAGAIN;
458 } else {
459 alloc_dmap = alloc_dax_mapping_reclaim(fcd, inode);
460 if (IS_ERR(alloc_dmap))
461 return PTR_ERR(alloc_dmap);
462 }
463
464 /* If we are here, we should have memory allocated */
465 if (WARN_ON(!alloc_dmap))
466 return -EIO;
467
468 /*
469 * Take write lock so that only one caller can try to setup mapping
470 * and other waits.
471 */
472 down_write(&fi->dax->sem);
473 /*
474 * We dropped lock. Check again if somebody else setup
475 * mapping already.
476 */
477 node = interval_tree_iter_first(&fi->dax->tree, start_idx, start_idx);
478 if (node) {
479 dmap = node_to_dmap(node);
480 fuse_fill_iomap(inode, pos, length, iomap, dmap, flags);
481 dmap_add_to_free_pool(fcd, alloc_dmap);
482 up_write(&fi->dax->sem);
483 return 0;
484 }
485
486 /* Setup one mapping */
487 ret = fuse_setup_one_mapping(inode, pos >> FUSE_DAX_SHIFT, alloc_dmap,
488 writable, false);
489 if (ret < 0) {
490 dmap_add_to_free_pool(fcd, alloc_dmap);
491 up_write(&fi->dax->sem);
492 return ret;
493 }
494 fuse_fill_iomap(inode, pos, length, iomap, alloc_dmap, flags);
495 up_write(&fi->dax->sem);
496 return 0;
497}
498
499static int fuse_upgrade_dax_mapping(struct inode *inode, loff_t pos,
500 loff_t length, unsigned int flags,
501 struct iomap *iomap)
502{
503 struct fuse_inode *fi = get_fuse_inode(inode);
504 struct fuse_dax_mapping *dmap;
505 int ret;
506 unsigned long idx = pos >> FUSE_DAX_SHIFT;
507 struct interval_tree_node *node;
508
509 /*
510 * Take exclusive lock so that only one caller can try to setup
511 * mapping and others wait.
512 */
513 down_write(&fi->dax->sem);
514 node = interval_tree_iter_first(&fi->dax->tree, idx, idx);
515
516 /* We are holding either inode lock or invalidate_lock, and that should
517 * ensure that dmap can't be truncated. We are holding a reference
518 * on dmap and that should make sure it can't be reclaimed. So dmap
519 * should still be there in tree despite the fact we dropped and
520 * re-acquired the fi->dax->sem lock.
521 */
522 ret = -EIO;
523 if (WARN_ON(!node))
524 goto out_err;
525
526 dmap = node_to_dmap(node);
527
528 /* We took an extra reference on dmap to make sure its not reclaimd.
529 * Now we hold fi->dax->sem lock and that reference is not needed
530 * anymore. Drop it.
531 */
532 if (refcount_dec_and_test(&dmap->refcnt)) {
533 /* refcount should not hit 0. This object only goes
534 * away when fuse connection goes away
535 */
536 WARN_ON_ONCE(1);
537 }
538
539 /* Maybe another thread already upgraded mapping while we were not
540 * holding lock.
541 */
542 if (dmap->writable) {
543 ret = 0;
544 goto out_fill_iomap;
545 }
546
547 ret = fuse_setup_one_mapping(inode, pos >> FUSE_DAX_SHIFT, dmap, true,
548 true);
549 if (ret < 0)
550 goto out_err;
551out_fill_iomap:
552 fuse_fill_iomap(inode, pos, length, iomap, dmap, flags);
553out_err:
554 up_write(&fi->dax->sem);
555 return ret;
556}
557
558/* This is just for DAX and the mapping is ephemeral, do not use it for other
559 * purposes since there is no block device with a permanent mapping.
560 */
561static int fuse_iomap_begin(struct inode *inode, loff_t pos, loff_t length,
562 unsigned int flags, struct iomap *iomap,
563 struct iomap *srcmap)
564{
565 struct fuse_inode *fi = get_fuse_inode(inode);
566 struct fuse_conn *fc = get_fuse_conn(inode);
567 struct fuse_dax_mapping *dmap;
568 bool writable = flags & IOMAP_WRITE;
569 unsigned long start_idx = pos >> FUSE_DAX_SHIFT;
570 struct interval_tree_node *node;
571
572 /* We don't support FIEMAP */
573 if (WARN_ON(flags & IOMAP_REPORT))
574 return -EIO;
575
576 iomap->offset = pos;
577 iomap->flags = 0;
578 iomap->bdev = NULL;
579 iomap->dax_dev = fc->dax->dev;
580
581 /*
582 * Both read/write and mmap path can race here. So we need something
583 * to make sure if we are setting up mapping, then other path waits
584 *
585 * For now, use a semaphore for this. It probably needs to be
586 * optimized later.
587 */
588 down_read(&fi->dax->sem);
589 node = interval_tree_iter_first(&fi->dax->tree, start_idx, start_idx);
590 if (node) {
591 dmap = node_to_dmap(node);
592 if (writable && !dmap->writable) {
593 /* Upgrade read-only mapping to read-write. This will
594 * require exclusive fi->dax->sem lock as we don't want
595 * two threads to be trying to this simultaneously
596 * for same dmap. So drop shared lock and acquire
597 * exclusive lock.
598 *
599 * Before dropping fi->dax->sem lock, take reference
600 * on dmap so that its not freed by range reclaim.
601 */
602 refcount_inc(&dmap->refcnt);
603 up_read(&fi->dax->sem);
604 pr_debug("%s: Upgrading mapping at offset 0x%llx length 0x%llx\n",
605 __func__, pos, length);
606 return fuse_upgrade_dax_mapping(inode, pos, length,
607 flags, iomap);
608 } else {
609 fuse_fill_iomap(inode, pos, length, iomap, dmap, flags);
610 up_read(&fi->dax->sem);
611 return 0;
612 }
613 } else {
614 up_read(&fi->dax->sem);
615 pr_debug("%s: no mapping at offset 0x%llx length 0x%llx\n",
616 __func__, pos, length);
617 if (pos >= i_size_read(inode))
618 goto iomap_hole;
619
620 return fuse_setup_new_dax_mapping(inode, pos, length, flags,
621 iomap);
622 }
623
624 /*
625 * If read beyond end of file happens, fs code seems to return
626 * it as hole
627 */
628iomap_hole:
629 fuse_fill_iomap_hole(iomap, length);
630 pr_debug("%s returning hole mapping. pos=0x%llx length_asked=0x%llx length_returned=0x%llx\n",
631 __func__, pos, length, iomap->length);
632 return 0;
633}
634
635static int fuse_iomap_end(struct inode *inode, loff_t pos, loff_t length,
636 ssize_t written, unsigned int flags,
637 struct iomap *iomap)
638{
639 struct fuse_dax_mapping *dmap = iomap->private;
640
641 if (dmap) {
642 if (refcount_dec_and_test(&dmap->refcnt)) {
643 /* refcount should not hit 0. This object only goes
644 * away when fuse connection goes away
645 */
646 WARN_ON_ONCE(1);
647 }
648 }
649
650 /* DAX writes beyond end-of-file aren't handled using iomap, so the
651 * file size is unchanged and there is nothing to do here.
652 */
653 return 0;
654}
655
656static DEFINE_IOMAP_ITER_NEXT_END(fuse_iomap_next, fuse_iomap_begin,
657 fuse_iomap_end);
658
659static const struct iomap_ops fuse_iomap_ops = {
660 .iomap_next = fuse_iomap_next,
661};
662
663static void fuse_wait_dax_page(struct inode *inode)
664{
665 filemap_invalidate_unlock(inode->i_mapping);
666 schedule();
667 filemap_invalidate_lock(inode->i_mapping);
668}
669
670/* Should be called with mapping->invalidate_lock held exclusively. */
671int fuse_dax_break_layouts(struct inode *inode, u64 dmap_start,
672 u64 dmap_end)
673{
674 return dax_break_layout(inode, dmap_start, dmap_end,
675 fuse_wait_dax_page);
676}
677
678ssize_t fuse_dax_read_iter(struct kiocb *iocb, struct iov_iter *to)
679{
680 struct inode *inode = file_inode(iocb->ki_filp);
681 ssize_t ret;
682
683 if (iocb->ki_flags & IOCB_NOWAIT) {
684 if (!inode_trylock_shared(inode))
685 return -EAGAIN;
686 } else {
687 inode_lock_shared(inode);
688 }
689
690 ret = dax_iomap_rw(iocb, to, &fuse_iomap_ops);
691 inode_unlock_shared(inode);
692
693 /* TODO file_accessed(iocb->f_filp) */
694 return ret;
695}
696
697static bool file_extending_write(struct kiocb *iocb, struct iov_iter *from)
698{
699 struct inode *inode = file_inode(iocb->ki_filp);
700
701 return (iov_iter_rw(from) == WRITE &&
702 ((iocb->ki_pos) >= i_size_read(inode) ||
703 (iocb->ki_pos + iov_iter_count(from) > i_size_read(inode))));
704}
705
706static ssize_t fuse_dax_direct_write(struct kiocb *iocb, struct iov_iter *from)
707{
708 struct inode *inode = file_inode(iocb->ki_filp);
709 struct fuse_io_priv io = FUSE_IO_PRIV_SYNC(iocb);
710 ssize_t ret;
711
712 ret = fuse_direct_io(&io, from, &iocb->ki_pos, FUSE_DIO_WRITE);
713
714 fuse_write_update_attr(inode, iocb->ki_pos, ret);
715 return ret;
716}
717
718ssize_t fuse_dax_write_iter(struct kiocb *iocb, struct iov_iter *from)
719{
720 struct inode *inode = file_inode(iocb->ki_filp);
721 ssize_t ret;
722
723 if (iocb->ki_flags & IOCB_NOWAIT) {
724 if (!inode_trylock(inode))
725 return -EAGAIN;
726 } else {
727 inode_lock(inode);
728 }
729
730 ret = generic_write_checks(iocb, from);
731 if (ret <= 0)
732 goto out;
733
734 ret = file_remove_privs(iocb->ki_filp);
735 if (ret)
736 goto out;
737 /* TODO file_update_time() but we don't want metadata I/O */
738
739 /* Do not use dax for file extending writes as write and on
740 * disk i_size increase are not atomic otherwise.
741 */
742 if (file_extending_write(iocb, from))
743 ret = fuse_dax_direct_write(iocb, from);
744 else
745 ret = dax_iomap_rw(iocb, from, &fuse_iomap_ops);
746
747out:
748 inode_unlock(inode);
749
750 if (ret > 0)
751 ret = generic_write_sync(iocb, ret);
752 return ret;
753}
754
755static vm_fault_t __fuse_dax_fault(struct vm_fault *vmf, unsigned int order,
756 bool write)
757{
758 vm_fault_t ret;
759 struct inode *inode = file_inode(vmf->vma->vm_file);
760 struct super_block *sb = inode->i_sb;
761 unsigned long pfn;
762 int error = 0;
763 struct fuse_conn *fc = get_fuse_conn(inode);
764 struct fuse_conn_dax *fcd = fc->dax;
765 bool retry = false;
766
767 if (write)
768 sb_start_pagefault(sb);
769retry:
770 if (retry && !(fcd->nr_free_ranges > 0))
771 wait_event(fcd->range_waitq, (fcd->nr_free_ranges > 0));
772
773 /*
774 * We need to serialize against not only truncate but also against
775 * fuse dax memory range reclaim. While a range is being reclaimed,
776 * we do not want any read/write/mmap to make progress and try
777 * to populate page cache or access memory we are trying to free.
778 */
779 filemap_invalidate_lock_shared(inode->i_mapping);
780 ret = dax_iomap_fault(vmf, order, &pfn, &error, &fuse_iomap_ops);
781 if ((ret & VM_FAULT_ERROR) && error == -EAGAIN) {
782 error = 0;
783 retry = true;
784 filemap_invalidate_unlock_shared(inode->i_mapping);
785 goto retry;
786 }
787
788 if (ret & VM_FAULT_NEEDDSYNC)
789 ret = dax_finish_sync_fault(vmf, order, pfn);
790 filemap_invalidate_unlock_shared(inode->i_mapping);
791
792 if (write)
793 sb_end_pagefault(sb);
794
795 return ret;
796}
797
798static vm_fault_t fuse_dax_fault(struct vm_fault *vmf)
799{
800 return __fuse_dax_fault(vmf, 0, vmf->flags & FAULT_FLAG_WRITE);
801}
802
803static vm_fault_t fuse_dax_huge_fault(struct vm_fault *vmf, unsigned int order)
804{
805 return __fuse_dax_fault(vmf, order, vmf->flags & FAULT_FLAG_WRITE);
806}
807
808static vm_fault_t fuse_dax_page_mkwrite(struct vm_fault *vmf)
809{
810 return __fuse_dax_fault(vmf, 0, true);
811}
812
813static vm_fault_t fuse_dax_pfn_mkwrite(struct vm_fault *vmf)
814{
815 return __fuse_dax_fault(vmf, 0, true);
816}
817
818static const struct vm_operations_struct fuse_dax_vm_ops = {
819 .fault = fuse_dax_fault,
820 .huge_fault = fuse_dax_huge_fault,
821 .page_mkwrite = fuse_dax_page_mkwrite,
822 .pfn_mkwrite = fuse_dax_pfn_mkwrite,
823};
824
825int fuse_dax_mmap(struct file *file, struct vm_area_struct *vma)
826{
827 file_accessed(file);
828 vma->vm_ops = &fuse_dax_vm_ops;
829 vm_flags_set(vma, VM_MIXEDMAP | VM_HUGEPAGE);
830 return 0;
831}
832
833static int dmap_writeback_invalidate(struct inode *inode,
834 struct fuse_dax_mapping *dmap)
835{
836 int ret;
837 loff_t start_pos = dmap->itn.start << FUSE_DAX_SHIFT;
838 loff_t end_pos = (start_pos + FUSE_DAX_SZ - 1);
839
840 ret = filemap_fdatawrite_range(inode->i_mapping, start_pos, end_pos);
841 if (ret) {
842 pr_debug("fuse: filemap_fdatawrite_range() failed. err=%d start_pos=0x%llx, end_pos=0x%llx\n",
843 ret, start_pos, end_pos);
844 return ret;
845 }
846
847 ret = invalidate_inode_pages2_range(inode->i_mapping,
848 start_pos >> PAGE_SHIFT,
849 end_pos >> PAGE_SHIFT);
850 if (ret)
851 pr_debug("fuse: invalidate_inode_pages2_range() failed err=%d\n",
852 ret);
853
854 return ret;
855}
856
857static int reclaim_one_dmap_locked(struct inode *inode,
858 struct fuse_dax_mapping *dmap)
859{
860 int ret;
861 struct fuse_inode *fi = get_fuse_inode(inode);
862
863 /*
864 * igrab() was done to make sure inode won't go under us, and this
865 * further avoids the race with evict().
866 */
867 ret = dmap_writeback_invalidate(inode, dmap);
868 if (ret)
869 return ret;
870
871 /* Remove dax mapping from inode interval tree now */
872 interval_tree_remove(&dmap->itn, &fi->dax->tree);
873 fi->dax->nr--;
874
875 /* It is possible that umount/shutdown has killed the fuse connection
876 * and worker thread is trying to reclaim memory in parallel. Don't
877 * warn in that case.
878 */
879 ret = dmap_removemapping_one(inode, dmap);
880 if (ret && ret != -ENOTCONN) {
881 pr_warn("Failed to remove mapping. offset=0x%llx len=0x%llx ret=%d\n",
882 dmap->window_offset, dmap->length, ret);
883 }
884 return 0;
885}
886
887/* Find first mapped dmap for an inode and return file offset. Caller needs
888 * to hold fi->dax->sem lock either shared or exclusive.
889 */
890static struct fuse_dax_mapping *inode_lookup_first_dmap(struct inode *inode)
891{
892 struct fuse_inode *fi = get_fuse_inode(inode);
893 struct fuse_dax_mapping *dmap;
894 struct interval_tree_node *node;
895
896 for (node = interval_tree_iter_first(&fi->dax->tree, 0, -1); node;
897 node = interval_tree_iter_next(node, 0, -1)) {
898 dmap = node_to_dmap(node);
899 /* still in use. */
900 if (refcount_read(&dmap->refcnt) > 1)
901 continue;
902
903 return dmap;
904 }
905
906 return NULL;
907}
908
909/*
910 * Find first mapping in the tree and free it and return it. Do not add
911 * it back to free pool.
912 */
913static struct fuse_dax_mapping *
914inode_inline_reclaim_one_dmap(struct fuse_conn_dax *fcd, struct inode *inode,
915 bool *retry)
916{
917 struct fuse_inode *fi = get_fuse_inode(inode);
918 struct fuse_dax_mapping *dmap;
919 u64 dmap_start, dmap_end;
920 unsigned long start_idx;
921 int ret;
922 struct interval_tree_node *node;
923
924 filemap_invalidate_lock(inode->i_mapping);
925
926 /* Lookup a dmap and corresponding file offset to reclaim. */
927 down_read(&fi->dax->sem);
928 dmap = inode_lookup_first_dmap(inode);
929 if (dmap) {
930 start_idx = dmap->itn.start;
931 dmap_start = start_idx << FUSE_DAX_SHIFT;
932 dmap_end = dmap_start + FUSE_DAX_SZ - 1;
933 }
934 up_read(&fi->dax->sem);
935
936 if (!dmap)
937 goto out_mmap_sem;
938 /*
939 * Make sure there are no references to inode pages using
940 * get_user_pages()
941 */
942 ret = fuse_dax_break_layouts(inode, dmap_start, dmap_end);
943 if (ret) {
944 pr_debug("fuse: fuse_dax_break_layouts() failed. err=%d\n",
945 ret);
946 dmap = ERR_PTR(ret);
947 goto out_mmap_sem;
948 }
949
950 down_write(&fi->dax->sem);
951 node = interval_tree_iter_first(&fi->dax->tree, start_idx, start_idx);
952 /* Range already got reclaimed by somebody else */
953 if (!node) {
954 if (retry)
955 *retry = true;
956 goto out_write_dmap_sem;
957 }
958
959 dmap = node_to_dmap(node);
960 /* still in use. */
961 if (refcount_read(&dmap->refcnt) > 1) {
962 dmap = NULL;
963 if (retry)
964 *retry = true;
965 goto out_write_dmap_sem;
966 }
967
968 ret = reclaim_one_dmap_locked(inode, dmap);
969 if (ret < 0) {
970 dmap = ERR_PTR(ret);
971 goto out_write_dmap_sem;
972 }
973
974 /* Clean up dmap. Do not add back to free list */
975 dmap_remove_busy_list(fcd, dmap);
976 dmap->inode = NULL;
977 dmap->itn.start = dmap->itn.last = 0;
978
979 pr_debug("fuse: %s: inline reclaimed memory range. inode=%p, window_offset=0x%llx, length=0x%llx\n",
980 __func__, inode, dmap->window_offset, dmap->length);
981
982out_write_dmap_sem:
983 up_write(&fi->dax->sem);
984out_mmap_sem:
985 filemap_invalidate_unlock(inode->i_mapping);
986 return dmap;
987}
988
989static struct fuse_dax_mapping *
990alloc_dax_mapping_reclaim(struct fuse_conn_dax *fcd, struct inode *inode)
991{
992 struct fuse_dax_mapping *dmap;
993 struct fuse_inode *fi = get_fuse_inode(inode);
994
995 while (1) {
996 bool retry = false;
997
998 dmap = alloc_dax_mapping(fcd);
999 if (dmap)
1000 return dmap;
1001
1002 dmap = inode_inline_reclaim_one_dmap(fcd, inode, &retry);
1003 /*
1004 * Either we got a mapping or it is an error, return in both
1005 * the cases.
1006 */
1007 if (dmap)
1008 return dmap;
1009
1010 /* If we could not reclaim a mapping because it
1011 * had a reference or some other temporary failure,
1012 * Try again. We want to give up inline reclaim only
1013 * if there is no range assigned to this node. Otherwise
1014 * if a deadlock is possible if we sleep with
1015 * mapping->invalidate_lock held and worker to free memory
1016 * can't make progress due to unavailability of
1017 * mapping->invalidate_lock. So sleep only if fi->dax->nr=0
1018 */
1019 if (retry)
1020 continue;
1021 /*
1022 * There are no mappings which can be reclaimed. Wait for one.
1023 * We are not holding fi->dax->sem. So it is possible
1024 * that range gets added now. But as we are not holding
1025 * mapping->invalidate_lock, worker should still be able to
1026 * free up a range and wake us up.
1027 */
1028 if (!fi->dax->nr && !(fcd->nr_free_ranges > 0)) {
1029 if (wait_event_killable_exclusive(fcd->range_waitq,
1030 (fcd->nr_free_ranges > 0))) {
1031 return ERR_PTR(-EINTR);
1032 }
1033 }
1034 }
1035}
1036
1037static int lookup_and_reclaim_dmap_locked(struct fuse_conn_dax *fcd,
1038 struct inode *inode,
1039 unsigned long start_idx)
1040{
1041 int ret;
1042 struct fuse_inode *fi = get_fuse_inode(inode);
1043 struct fuse_dax_mapping *dmap;
1044 struct interval_tree_node *node;
1045
1046 /* Find fuse dax mapping at file offset inode. */
1047 node = interval_tree_iter_first(&fi->dax->tree, start_idx, start_idx);
1048
1049 /* Range already got cleaned up by somebody else */
1050 if (!node)
1051 return 0;
1052 dmap = node_to_dmap(node);
1053
1054 /* still in use. */
1055 if (refcount_read(&dmap->refcnt) > 1)
1056 return 0;
1057
1058 ret = reclaim_one_dmap_locked(inode, dmap);
1059 if (ret < 0)
1060 return ret;
1061
1062 /* Cleanup dmap entry and add back to free list */
1063 spin_lock(&fcd->lock);
1064 dmap_reinit_add_to_free_pool(fcd, dmap);
1065 spin_unlock(&fcd->lock);
1066 return ret;
1067}
1068
1069/*
1070 * Free a range of memory.
1071 * Locking:
1072 * 1. Take mapping->invalidate_lock to block dax faults.
1073 * 2. Take fi->dax->sem to protect interval tree and also to make sure
1074 * read/write can not reuse a dmap which we might be freeing.
1075 */
1076static int lookup_and_reclaim_dmap(struct fuse_conn_dax *fcd,
1077 struct inode *inode,
1078 unsigned long start_idx,
1079 unsigned long end_idx)
1080{
1081 int ret;
1082 struct fuse_inode *fi = get_fuse_inode(inode);
1083 loff_t dmap_start = start_idx << FUSE_DAX_SHIFT;
1084 loff_t dmap_end = (dmap_start + FUSE_DAX_SZ) - 1;
1085
1086 filemap_invalidate_lock(inode->i_mapping);
1087 ret = fuse_dax_break_layouts(inode, dmap_start, dmap_end);
1088 if (ret) {
1089 pr_debug("virtio_fs: fuse_dax_break_layouts() failed. err=%d\n",
1090 ret);
1091 goto out_mmap_sem;
1092 }
1093
1094 down_write(&fi->dax->sem);
1095 ret = lookup_and_reclaim_dmap_locked(fcd, inode, start_idx);
1096 up_write(&fi->dax->sem);
1097out_mmap_sem:
1098 filemap_invalidate_unlock(inode->i_mapping);
1099 return ret;
1100}
1101
1102static int try_to_free_dmap_chunks(struct fuse_conn_dax *fcd,
1103 unsigned long nr_to_free)
1104{
1105 struct fuse_dax_mapping *dmap, *pos, *temp;
1106 int ret, nr_freed = 0;
1107 unsigned long start_idx = 0, end_idx = 0;
1108 struct inode *inode = NULL;
1109
1110 /* Pick first busy range and free it for now*/
1111 while (1) {
1112 if (nr_freed >= nr_to_free)
1113 break;
1114
1115 dmap = NULL;
1116 spin_lock(&fcd->lock);
1117
1118 if (!fcd->nr_busy_ranges) {
1119 spin_unlock(&fcd->lock);
1120 return 0;
1121 }
1122
1123 list_for_each_entry_safe(pos, temp, &fcd->busy_ranges,
1124 busy_list) {
1125 /* skip this range if it's in use. */
1126 if (refcount_read(&pos->refcnt) > 1)
1127 continue;
1128
1129 inode = igrab(pos->inode);
1130 /*
1131 * This inode is going away. That will free
1132 * up all the ranges anyway, continue to
1133 * next range.
1134 */
1135 if (!inode)
1136 continue;
1137 /*
1138 * Take this element off list and add it tail. If
1139 * this element can't be freed, it will help with
1140 * selecting new element in next iteration of loop.
1141 */
1142 dmap = pos;
1143 list_move_tail(&dmap->busy_list, &fcd->busy_ranges);
1144 start_idx = end_idx = dmap->itn.start;
1145 break;
1146 }
1147 spin_unlock(&fcd->lock);
1148 if (!dmap)
1149 return 0;
1150
1151 ret = lookup_and_reclaim_dmap(fcd, inode, start_idx, end_idx);
1152 iput(inode);
1153 if (ret)
1154 return ret;
1155 nr_freed++;
1156 }
1157 return 0;
1158}
1159
1160static void fuse_dax_free_mem_worker(struct work_struct *work)
1161{
1162 int ret;
1163 struct fuse_conn_dax *fcd = container_of(work, struct fuse_conn_dax,
1164 free_work.work);
1165 ret = try_to_free_dmap_chunks(fcd, FUSE_DAX_RECLAIM_CHUNK);
1166 if (ret) {
1167 pr_debug("fuse: try_to_free_dmap_chunks() failed with err=%d\n",
1168 ret);
1169 }
1170
1171 /* If number of free ranges are still below threshold, requeue */
1172 kick_dmap_free_worker(fcd, 1);
1173}
1174
1175static void fuse_free_dax_mem_ranges(struct list_head *mem_list)
1176{
1177 struct fuse_dax_mapping *range, *temp;
1178
1179 /* Free All allocated elements */
1180 list_for_each_entry_safe(range, temp, mem_list, list) {
1181 list_del(&range->list);
1182 if (!list_empty(&range->busy_list))
1183 list_del(&range->busy_list);
1184 kfree(range);
1185 }
1186}
1187
1188void fuse_dax_conn_free(struct fuse_conn *fc)
1189{
1190 if (fc->dax) {
1191 fuse_free_dax_mem_ranges(&fc->dax->free_ranges);
1192 kfree(fc->dax);
1193 fc->dax = NULL;
1194 }
1195}
1196
1197static int fuse_dax_mem_range_init(struct fuse_conn_dax *fcd)
1198{
1199 long nr_pages, nr_ranges;
1200 struct fuse_dax_mapping *range;
1201 int ret, id;
1202 size_t dax_size = -1;
1203 unsigned long i;
1204
1205 init_waitqueue_head(&fcd->range_waitq);
1206 INIT_LIST_HEAD(&fcd->free_ranges);
1207 INIT_LIST_HEAD(&fcd->busy_ranges);
1208 INIT_DELAYED_WORK(&fcd->free_work, fuse_dax_free_mem_worker);
1209
1210 id = dax_read_lock();
1211 nr_pages = dax_direct_access(fcd->dev, 0, PHYS_PFN(dax_size),
1212 DAX_ACCESS, NULL, NULL);
1213 dax_read_unlock(id);
1214 if (nr_pages < 0) {
1215 pr_debug("dax_direct_access() returned %ld\n", nr_pages);
1216 return nr_pages;
1217 }
1218
1219 nr_ranges = nr_pages/FUSE_DAX_PAGES;
1220 pr_debug("%s: dax mapped %ld pages. nr_ranges=%ld\n",
1221 __func__, nr_pages, nr_ranges);
1222
1223 for (i = 0; i < nr_ranges; i++) {
1224 range = kzalloc_obj(struct fuse_dax_mapping);
1225 ret = -ENOMEM;
1226 if (!range)
1227 goto out_err;
1228
1229 /* TODO: This offset only works if virtio-fs driver is not
1230 * having some memory hidden at the beginning. This needs
1231 * better handling
1232 */
1233 range->window_offset = i * FUSE_DAX_SZ;
1234 range->length = FUSE_DAX_SZ;
1235 INIT_LIST_HEAD(&range->busy_list);
1236 refcount_set(&range->refcnt, 1);
1237 list_add_tail(&range->list, &fcd->free_ranges);
1238 }
1239
1240 fcd->nr_free_ranges = nr_ranges;
1241 fcd->nr_ranges = nr_ranges;
1242 return 0;
1243out_err:
1244 /* Free All allocated elements */
1245 fuse_free_dax_mem_ranges(&fcd->free_ranges);
1246 return ret;
1247}
1248
1249int fuse_dax_conn_alloc(struct fuse_conn *fc, enum fuse_dax_mode dax_mode,
1250 struct dax_device *dax_dev)
1251{
1252 struct fuse_conn_dax *fcd;
1253 int err;
1254
1255 fc->dax_mode = dax_mode;
1256
1257 if (!dax_dev)
1258 return 0;
1259
1260 fcd = kzalloc_obj(*fcd);
1261 if (!fcd)
1262 return -ENOMEM;
1263
1264 spin_lock_init(&fcd->lock);
1265 fcd->dev = dax_dev;
1266 err = fuse_dax_mem_range_init(fcd);
1267 if (err) {
1268 kfree(fcd);
1269 return err;
1270 }
1271
1272 fc->dax = fcd;
1273 return 0;
1274}
1275
1276bool fuse_dax_inode_alloc(struct super_block *sb, struct fuse_inode *fi)
1277{
1278 struct fuse_conn *fc = get_fuse_conn_super(sb);
1279
1280 fi->dax = NULL;
1281 if (fc->dax) {
1282 fi->dax = kzalloc_obj(*fi->dax, GFP_KERNEL_ACCOUNT);
1283 if (!fi->dax)
1284 return false;
1285
1286 init_rwsem(&fi->dax->sem);
1287 fi->dax->tree = RB_ROOT_CACHED;
1288 }
1289
1290 return true;
1291}
1292
1293static const struct address_space_operations fuse_dax_file_aops = {
1294 .direct_IO = noop_direct_IO,
1295 .dirty_folio = noop_dirty_folio,
1296};
1297
1298static bool fuse_should_enable_dax(struct inode *inode, unsigned int flags)
1299{
1300 struct fuse_conn *fc = get_fuse_conn(inode);
1301 enum fuse_dax_mode dax_mode = fc->dax_mode;
1302
1303 if (dax_mode == FUSE_DAX_NEVER)
1304 return false;
1305
1306 /*
1307 * fc->dax may be NULL in 'inode' mode when filesystem device doesn't
1308 * support DAX, in which case it will silently fallback to 'never' mode.
1309 */
1310 if (!fc->dax)
1311 return false;
1312
1313 if (dax_mode == FUSE_DAX_ALWAYS)
1314 return true;
1315
1316 /* dax_mode is FUSE_DAX_INODE* */
1317 return fc->inode_dax && (flags & FUSE_ATTR_DAX);
1318}
1319
1320void fuse_dax_inode_init(struct inode *inode, unsigned int flags)
1321{
1322 if (!fuse_should_enable_dax(inode, flags))
1323 return;
1324
1325 inode->i_flags |= S_DAX;
1326 inode->i_data.a_ops = &fuse_dax_file_aops;
1327}
1328
1329void fuse_dax_dontcache(struct inode *inode, unsigned int flags)
1330{
1331 struct fuse_conn *fc = get_fuse_conn(inode);
1332
1333 if (fuse_is_inode_dax_mode(fc->dax_mode) &&
1334 ((bool) IS_DAX(inode) != (bool) (flags & FUSE_ATTR_DAX)))
1335 d_mark_dontcache(inode);
1336}
1337
1338bool fuse_dax_check_alignment(struct fuse_conn *fc, unsigned int map_alignment)
1339{
1340 if (fc->dax && (map_alignment > FUSE_DAX_SHIFT)) {
1341 pr_warn("FUSE: map_alignment %u incompatible with dax mem range size %u\n",
1342 map_alignment, FUSE_DAX_SZ);
1343 return false;
1344 }
1345 return true;
1346}
1347
1348void fuse_dax_cancel_work(struct fuse_conn *fc)
1349{
1350 struct fuse_conn_dax *fcd = fc->dax;
1351
1352 if (fcd)
1353 cancel_delayed_work_sync(&fcd->free_work);
1354
1355}
1356EXPORT_SYMBOL_GPL(fuse_dax_cancel_work);