projects/linux·blob master · refs

2271 lines · 60.5 KiB

1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * fs/dax.c - Direct Access filesystem code
4 * Copyright (c) 2013-2014 Intel Corporation
5 * Author: Matthew Wilcox <[email protected]>
6 * Author: Ross Zwisler <[email protected]>
7 */
8
9#include <linux/atomic.h>
10#include <linux/blkdev.h>
11#include <linux/buffer_head.h>
12#include <linux/dax.h>
13#include <linux/fs.h>
14#include <linux/highmem.h>
15#include <linux/memcontrol.h>
16#include <linux/mm.h>
17#include <linux/mutex.h>
18#include <linux/sched.h>
19#include <linux/sched/signal.h>
20#include <linux/uio.h>
21#include <linux/vmstat.h>
22#include <linux/sizes.h>
23#include <linux/mmu_notifier.h>
24#include <linux/iomap.h>
25#include <linux/rmap.h>
26#include <linux/pgalloc.h>
27
28#define CREATE_TRACE_POINTS
29#include <trace/events/fs_dax.h>
30
31/* We choose 4096 entries - same as per-zone page wait tables */
32#define DAX_WAIT_TABLE_BITS 12
33#define DAX_WAIT_TABLE_ENTRIES (1 << DAX_WAIT_TABLE_BITS)
34
35/* The 'colour' (ie low bits) within a PMD of a page offset. */
36#define PG_PMD_COLOUR ((PMD_SIZE >> PAGE_SHIFT) - 1)
37#define PG_PMD_NR (PMD_SIZE >> PAGE_SHIFT)
38
39static wait_queue_head_t wait_table[DAX_WAIT_TABLE_ENTRIES];
40
41static int __init init_dax_wait_table(void)
42{
43 int i;
44
45 for (i = 0; i < DAX_WAIT_TABLE_ENTRIES; i++)
46 init_waitqueue_head(wait_table + i);
47 return 0;
48}
49fs_initcall(init_dax_wait_table);
50
51/*
52 * DAX pagecache entries use XArray value entries so they can't be mistaken
53 * for pages. We use one bit for locking, one bit for the entry size (PMD)
54 * and two more to tell us if the entry is a zero page or an empty entry that
55 * is just used for locking. In total four special bits.
56 *
57 * If the PMD bit isn't set the entry has size PAGE_SIZE, and if the ZERO_PAGE
58 * and EMPTY bits aren't set the entry is a normal DAX entry with a filesystem
59 * block allocation.
60 */
61#define DAX_SHIFT (4)
62#define DAX_LOCKED (1UL << 0)
63#define DAX_PMD (1UL << 1)
64#define DAX_ZERO_PAGE (1UL << 2)
65#define DAX_EMPTY (1UL << 3)
66
67static unsigned long dax_to_pfn(void *entry)
68{
69 return xa_to_value(entry) >> DAX_SHIFT;
70}
71
72static struct folio *dax_to_folio(void *entry)
73{
74 return page_folio(pfn_to_page(dax_to_pfn(entry)));
75}
76
77static void *dax_make_entry(unsigned long pfn, unsigned long flags)
78{
79 return xa_mk_value(flags | (pfn << DAX_SHIFT));
80}
81
82static bool dax_is_locked(void *entry)
83{
84 return xa_to_value(entry) & DAX_LOCKED;
85}
86
87static unsigned int dax_entry_order(void *entry)
88{
89 if (xa_to_value(entry) & DAX_PMD)
90 return PMD_ORDER;
91 return 0;
92}
93
94static unsigned long dax_is_pmd_entry(void *entry)
95{
96 return xa_to_value(entry) & DAX_PMD;
97}
98
99static bool dax_is_pte_entry(void *entry)
100{
101 return !(xa_to_value(entry) & DAX_PMD);
102}
103
104static int dax_is_zero_entry(void *entry)
105{
106 return xa_to_value(entry) & DAX_ZERO_PAGE;
107}
108
109static int dax_is_empty_entry(void *entry)
110{
111 return xa_to_value(entry) & DAX_EMPTY;
112}
113
114/*
115 * true if the entry that was found is of a smaller order than the entry
116 * we were looking for
117 */
118static bool dax_is_conflict(void *entry)
119{
120 return entry == XA_RETRY_ENTRY;
121}
122
123/*
124 * DAX page cache entry locking
125 */
126struct exceptional_entry_key {
127 struct xarray *xa;
128 pgoff_t entry_start;
129};
130
131struct wait_exceptional_entry_queue {
132 wait_queue_entry_t wait;
133 struct exceptional_entry_key key;
134};
135
136/**
137 * enum dax_wake_mode: waitqueue wakeup behaviour
138 * @WAKE_ALL: wake all waiters in the waitqueue
139 * @WAKE_NEXT: wake only the first waiter in the waitqueue
140 */
141enum dax_wake_mode {
142 WAKE_ALL,
143 WAKE_NEXT,
144};
145
146static wait_queue_head_t *dax_entry_waitqueue(struct xa_state *xas,
147 void *entry, struct exceptional_entry_key *key)
148{
149 unsigned long hash;
150 unsigned long index = xas->xa_index;
151
152 /*
153 * If 'entry' is a PMD, align the 'index' that we use for the wait
154 * queue to the start of that PMD. This ensures that all offsets in
155 * the range covered by the PMD map to the same bit lock.
156 */
157 if (dax_is_pmd_entry(entry))
158 index &= ~PG_PMD_COLOUR;
159 key->xa = xas->xa;
160 key->entry_start = index;
161
162 hash = hash_long((unsigned long)xas->xa ^ index, DAX_WAIT_TABLE_BITS);
163 return wait_table + hash;
164}
165
166static int wake_exceptional_entry_func(wait_queue_entry_t *wait,
167 unsigned int mode, int sync, void *keyp)
168{
169 struct exceptional_entry_key *key = keyp;
170 struct wait_exceptional_entry_queue *ewait =
171 container_of(wait, struct wait_exceptional_entry_queue, wait);
172
173 if (key->xa != ewait->key.xa ||
174 key->entry_start != ewait->key.entry_start)
175 return 0;
176 return autoremove_wake_function(wait, mode, sync, NULL);
177}
178
179/*
180 * @entry may no longer be the entry at the index in the mapping.
181 * The important information it's conveying is whether the entry at
182 * this index used to be a PMD entry.
183 */
184static void dax_wake_entry(struct xa_state *xas, void *entry,
185 enum dax_wake_mode mode)
186{
187 struct exceptional_entry_key key;
188 wait_queue_head_t *wq;
189
190 wq = dax_entry_waitqueue(xas, entry, &key);
191
192 /*
193 * Checking for locked entry and prepare_to_wait_exclusive() happens
194 * under the i_pages lock, ditto for entry handling in our callers.
195 * So at this point all tasks that could have seen our entry locked
196 * must be in the waitqueue and the following check will see them.
197 */
198 if (waitqueue_active(wq))
199 __wake_up(wq, TASK_NORMAL, mode == WAKE_ALL ? 0 : 1, &key);
200}
201
202/*
203 * Look up entry in page cache, wait for it to become unlocked if it
204 * is a DAX entry and return it. The caller must subsequently call
205 * put_unlocked_entry() if it did not lock the entry or dax_unlock_entry()
206 * if it did. The entry returned may have a larger order than @order.
207 * If @order is larger than the order of the entry found in i_pages, this
208 * function returns a dax_is_conflict entry.
209 *
210 * Must be called with the i_pages lock held.
211 */
212static void *get_next_unlocked_entry(struct xa_state *xas, unsigned int order)
213{
214 void *entry;
215 struct wait_exceptional_entry_queue ewait;
216 wait_queue_head_t *wq;
217
218 init_wait(&ewait.wait);
219 ewait.wait.func = wake_exceptional_entry_func;
220
221 for (;;) {
222 entry = xas_find_conflict(xas);
223 if (!entry || WARN_ON_ONCE(!xa_is_value(entry)))
224 return entry;
225 if (dax_entry_order(entry) < order)
226 return XA_RETRY_ENTRY;
227 if (!dax_is_locked(entry))
228 return entry;
229
230 wq = dax_entry_waitqueue(xas, entry, &ewait.key);
231 prepare_to_wait_exclusive(wq, &ewait.wait,
232 TASK_UNINTERRUPTIBLE);
233 xas_unlock_irq(xas);
234 xas_reset(xas);
235 schedule();
236 finish_wait(wq, &ewait.wait);
237 xas_lock_irq(xas);
238 }
239}
240
241/*
242 * Wait for the given entry to become unlocked. Caller must hold the i_pages
243 * lock and call either put_unlocked_entry() if it did not lock the entry or
244 * dax_unlock_entry() if it did. Returns an unlocked entry if still present.
245 */
246static void *wait_entry_unlocked_exclusive(struct xa_state *xas, void *entry)
247{
248 struct wait_exceptional_entry_queue ewait;
249 wait_queue_head_t *wq;
250
251 init_wait(&ewait.wait);
252 ewait.wait.func = wake_exceptional_entry_func;
253
254 while (unlikely(dax_is_locked(entry))) {
255 wq = dax_entry_waitqueue(xas, entry, &ewait.key);
256 prepare_to_wait_exclusive(wq, &ewait.wait,
257 TASK_UNINTERRUPTIBLE);
258 xas_reset(xas);
259 xas_unlock_irq(xas);
260 schedule();
261 finish_wait(wq, &ewait.wait);
262 xas_lock_irq(xas);
263 entry = xas_load(xas);
264 }
265
266 if (xa_is_internal(entry))
267 return NULL;
268
269 return entry;
270}
271
272/*
273 * The only thing keeping the address space around is the i_pages lock
274 * (it's cycled in clear_inode() after removing the entries from i_pages)
275 * After we call xas_unlock_irq(), we cannot touch xas->xa.
276 */
277static void wait_entry_unlocked(struct xa_state *xas, void *entry)
278{
279 struct wait_exceptional_entry_queue ewait;
280 wait_queue_head_t *wq;
281
282 init_wait(&ewait.wait);
283 ewait.wait.func = wake_exceptional_entry_func;
284
285 wq = dax_entry_waitqueue(xas, entry, &ewait.key);
286 /*
287 * Unlike get_next_unlocked_entry() there is no guarantee that this
288 * path ever successfully retrieves an unlocked entry before an
289 * inode dies. Perform a non-exclusive wait in case this path
290 * never successfully performs its own wake up.
291 */
292 prepare_to_wait(wq, &ewait.wait, TASK_UNINTERRUPTIBLE);
293 xas_unlock_irq(xas);
294 schedule();
295 finish_wait(wq, &ewait.wait);
296}
297
298static void put_unlocked_entry(struct xa_state *xas, void *entry,
299 enum dax_wake_mode mode)
300{
301 if (entry && !dax_is_conflict(entry))
302 dax_wake_entry(xas, entry, mode);
303}
304
305/*
306 * We used the xa_state to get the entry, but then we locked the entry and
307 * dropped the xa_lock, so we know the xa_state is stale and must be reset
308 * before use.
309 */
310static void dax_unlock_entry(struct xa_state *xas, void *entry)
311{
312 void *old;
313
314 BUG_ON(dax_is_locked(entry));
315 xas_reset(xas);
316 xas_lock_irq(xas);
317 old = xas_store(xas, entry);
318 xas_unlock_irq(xas);
319 BUG_ON(!dax_is_locked(old));
320 dax_wake_entry(xas, entry, WAKE_NEXT);
321}
322
323/*
324 * Return: The entry stored at this location before it was locked.
325 */
326static void *dax_lock_entry(struct xa_state *xas, void *entry)
327{
328 unsigned long v = xa_to_value(entry);
329 return xas_store(xas, xa_mk_value(v | DAX_LOCKED));
330}
331
332static unsigned long dax_entry_size(void *entry)
333{
334 if (dax_is_zero_entry(entry))
335 return 0;
336 else if (dax_is_empty_entry(entry))
337 return 0;
338 else if (dax_is_pmd_entry(entry))
339 return PMD_SIZE;
340 else
341 return PAGE_SIZE;
342}
343
344/*
345 * A DAX folio is considered shared if it has no mapping set and ->share (which
346 * shares the ->index field) is non-zero. Note this may return false even if the
347 * page is shared between multiple files but has not yet actually been mapped
348 * into multiple address spaces.
349 */
350static inline bool dax_folio_is_shared(struct folio *folio)
351{
352 return !folio->mapping && folio->share;
353}
354
355/*
356 * When it is called by dax_insert_entry(), the shared flag will indicate
357 * whether this entry is shared by multiple files. If the page has not
358 * previously been associated with any mappings the ->mapping and ->index
359 * fields will be set. If it has already been associated with a mapping
360 * the mapping will be cleared and the share count set. It's then up to
361 * reverse map users like memory_failure() to call back into the filesystem to
362 * recover ->mapping and ->index information. For example by implementing
363 * dax_holder_operations.
364 */
365static void dax_folio_make_shared(struct folio *folio)
366{
367 /*
368 * folio is not currently shared so mark it as shared by clearing
369 * folio->mapping.
370 */
371 folio->mapping = NULL;
372
373 /*
374 * folio has previously been mapped into one address space so set the
375 * share count.
376 */
377 folio->share = 1;
378}
379
380/**
381 * dax_folio_reset_order - Reset a compound DAX folio to order-0 pages
382 * @folio: The folio to reset
383 *
384 * Splits a compound folio back into individual order-0 pages,
385 * clearing compound state and restoring pgmap pointers.
386 *
387 * Returns: the original folio order (0 if already order-0)
388 */
389int dax_folio_reset_order(struct folio *folio)
390{
391 struct dev_pagemap *pgmap = page_pgmap(&folio->page);
392 int order = folio_order(folio);
393
394 /*
395 * Clear the mapping and the index/share union word. folio->share
396 * and folio->index occupy the same union in struct folio. For
397 * non-shared folios (mapping != NULL), the union holds folio->index
398 * (file page offset); for shared folios (mapping == NULL), it holds
399 * folio->share (reference count). Either way, we are releasing the
400 * folio and both fields should be zeroed.
401 */
402 folio->mapping = NULL;
403 folio->share = 0;
404
405 if (!order) {
406 /*
407 * Restore pgmap explicitly even for order-0 folios. For the
408 * dax_folio_put() caller this is a no-op (same value), but
409 * fsdev_clear_folio_state() may call this on folios that
410 * were previously compound and need pgmap re-established.
411 */
412 folio->pgmap = pgmap;
413 return 0;
414 }
415
416 folio_reset_order(folio);
417
418 for (int i = 0; i < (1UL << order); i++) {
419 struct page *page = folio_page(folio, i);
420 struct folio *f = (struct folio *)page;
421
422 ClearPageHead(page);
423 clear_compound_head(page);
424 f->mapping = NULL;
425 f->share = 0;
426 f->pgmap = pgmap;
427 }
428
429 return order;
430}
431EXPORT_SYMBOL_GPL(dax_folio_reset_order);
432
433static inline unsigned long dax_folio_put(struct folio *folio)
434{
435 unsigned long ref;
436 int order, i;
437
438 if (!dax_folio_is_shared(folio))
439 ref = 0;
440 else
441 ref = --folio->share;
442
443 if (ref)
444 return ref;
445
446 order = dax_folio_reset_order(folio);
447
448 /* Debug check: verify refcounts are zero for all sub-folios */
449 for (i = 0; i < (1UL << order); i++) {
450 struct page *page = folio_page(folio, i);
451
452 WARN_ON_ONCE(folio_ref_count((struct folio *)page));
453 }
454
455 return ref;
456}
457
458static void dax_folio_init(void *entry)
459{
460 struct folio *folio = dax_to_folio(entry);
461 int order = dax_entry_order(entry);
462
463 /*
464 * Folio should have been split back to order-0 pages in
465 * dax_folio_put() when they were removed from their
466 * final mapping.
467 */
468 WARN_ON_ONCE(folio_order(folio));
469
470 if (order > 0) {
471 prep_compound_page(&folio->page, order);
472 WARN_ON_ONCE(folio_ref_count(folio));
473 }
474}
475
476static void dax_associate_entry(void *entry, struct address_space *mapping,
477 struct vm_area_struct *vma,
478 unsigned long address, bool shared)
479{
480 unsigned long size = dax_entry_size(entry), index;
481 struct folio *folio;
482
483 if (dax_is_zero_entry(entry) || dax_is_empty_entry(entry))
484 return;
485
486 folio = dax_to_folio(entry);
487 index = linear_page_index(vma, address & ~(size - 1));
488 if (shared && (folio->mapping || dax_folio_is_shared(folio))) {
489 if (folio->mapping)
490 dax_folio_make_shared(folio);
491
492 WARN_ON_ONCE(!folio->share);
493 WARN_ON_ONCE(dax_entry_order(entry) != folio_order(folio));
494 folio->share++;
495 } else {
496 WARN_ON_ONCE(folio->mapping);
497 dax_folio_init(entry);
498 folio = dax_to_folio(entry);
499 folio->mapping = mapping;
500 folio->index = index;
501 }
502}
503
504static void dax_disassociate_entry(void *entry, struct address_space *mapping,
505 bool trunc)
506{
507 struct folio *folio;
508
509 if (dax_is_zero_entry(entry) || dax_is_empty_entry(entry))
510 return;
511
512 folio = dax_to_folio(entry);
513 dax_folio_put(folio);
514}
515
516static struct page *dax_busy_page(void *entry)
517{
518 struct folio *folio;
519
520 if (dax_is_zero_entry(entry) || dax_is_empty_entry(entry))
521 return NULL;
522
523 folio = dax_to_folio(entry);
524 if (folio_ref_count(folio) - folio_mapcount(folio))
525 return &folio->page;
526 else
527 return NULL;
528}
529
530/**
531 * dax_lock_folio - Lock the DAX entry corresponding to a folio
532 * @folio: The folio whose entry we want to lock
533 *
534 * Context: Process context.
535 * Return: A cookie to pass to dax_unlock_folio() or 0 if the entry could
536 * not be locked.
537 */
538dax_entry_t dax_lock_folio(struct folio *folio)
539{
540 XA_STATE(xas, NULL, 0);
541 void *entry;
542
543 /* Ensure folio->mapping isn't freed while we look at it */
544 rcu_read_lock();
545 for (;;) {
546 struct address_space *mapping = READ_ONCE(folio->mapping);
547
548 entry = NULL;
549 if (!mapping || !dax_mapping(mapping))
550 break;
551
552 /*
553 * In the device-dax case there's no need to lock, a
554 * struct dev_pagemap pin is sufficient to keep the
555 * inode alive, and we assume we have dev_pagemap pin
556 * otherwise we would not have a valid pfn_to_page()
557 * translation.
558 */
559 entry = (void *)~0UL;
560 if (S_ISCHR(mapping->host->i_mode))
561 break;
562
563 xas.xa = &mapping->i_pages;
564 xas_lock_irq(&xas);
565 if (mapping != folio->mapping) {
566 xas_unlock_irq(&xas);
567 continue;
568 }
569 xas_set(&xas, folio->index);
570 entry = xas_load(&xas);
571 if (dax_is_locked(entry)) {
572 rcu_read_unlock();
573 wait_entry_unlocked(&xas, entry);
574 rcu_read_lock();
575 continue;
576 }
577 dax_lock_entry(&xas, entry);
578 xas_unlock_irq(&xas);
579 break;
580 }
581 rcu_read_unlock();
582 return (dax_entry_t)entry;
583}
584
585void dax_unlock_folio(struct folio *folio, dax_entry_t cookie)
586{
587 struct address_space *mapping = folio->mapping;
588 XA_STATE(xas, &mapping->i_pages, folio->index);
589
590 if (S_ISCHR(mapping->host->i_mode))
591 return;
592
593 dax_unlock_entry(&xas, (void *)cookie);
594}
595
596/*
597 * dax_lock_mapping_entry - Lock the DAX entry corresponding to a mapping
598 * @mapping: the file's mapping whose entry we want to lock
599 * @index: the offset within this file
600 * @page: output the dax page corresponding to this dax entry
601 *
602 * Return: A cookie to pass to dax_unlock_mapping_entry() or 0 if the entry
603 * could not be locked.
604 */
605dax_entry_t dax_lock_mapping_entry(struct address_space *mapping, pgoff_t index,
606 struct page **page)
607{
608 XA_STATE(xas, NULL, 0);
609 void *entry;
610
611 rcu_read_lock();
612 for (;;) {
613 entry = NULL;
614 if (!dax_mapping(mapping))
615 break;
616
617 xas.xa = &mapping->i_pages;
618 xas_lock_irq(&xas);
619 xas_set(&xas, index);
620 entry = xas_load(&xas);
621 if (dax_is_locked(entry)) {
622 rcu_read_unlock();
623 wait_entry_unlocked(&xas, entry);
624 rcu_read_lock();
625 continue;
626 }
627 if (!entry ||
628 dax_is_zero_entry(entry) || dax_is_empty_entry(entry)) {
629 /*
630 * Because we are looking for entry from file's mapping
631 * and index, so the entry may not be inserted for now,
632 * or even a zero/empty entry. We don't think this is
633 * an error case. So, return a special value and do
634 * not output @page.
635 */
636 entry = (void *)~0UL;
637 } else {
638 *page = pfn_to_page(dax_to_pfn(entry));
639 dax_lock_entry(&xas, entry);
640 }
641 xas_unlock_irq(&xas);
642 break;
643 }
644 rcu_read_unlock();
645 return (dax_entry_t)entry;
646}
647
648void dax_unlock_mapping_entry(struct address_space *mapping, pgoff_t index,
649 dax_entry_t cookie)
650{
651 XA_STATE(xas, &mapping->i_pages, index);
652
653 if (cookie == ~0UL)
654 return;
655
656 dax_unlock_entry(&xas, (void *)cookie);
657}
658
659/*
660 * Find page cache entry at given index. If it is a DAX entry, return it
661 * with the entry locked. If the page cache doesn't contain an entry at
662 * that index, add a locked empty entry.
663 *
664 * When requesting an entry with size DAX_PMD, grab_mapping_entry() will
665 * either return that locked entry or will return VM_FAULT_FALLBACK.
666 * This will happen if there are any PTE entries within the PMD range
667 * that we are requesting.
668 *
669 * We always favor PTE entries over PMD entries. There isn't a flow where we
670 * evict PTE entries in order to 'upgrade' them to a PMD entry. A PMD
671 * insertion will fail if it finds any PTE entries already in the tree, and a
672 * PTE insertion will cause an existing PMD entry to be unmapped and
673 * downgraded to PTE entries. This happens for both PMD zero pages as
674 * well as PMD empty entries.
675 *
676 * The exception to this downgrade path is for PMD entries that have
677 * real storage backing them. We will leave these real PMD entries in
678 * the tree, and PTE writes will simply dirty the entire PMD entry.
679 *
680 * Note: Unlike filemap_fault() we don't honor FAULT_FLAG_RETRY flags. For
681 * persistent memory the benefit is doubtful. We can add that later if we can
682 * show it helps.
683 *
684 * On error, this function does not return an ERR_PTR. Instead it returns
685 * a VM_FAULT code, encoded as an xarray internal entry. The ERR_PTR values
686 * overlap with xarray value entries.
687 */
688static void *grab_mapping_entry(struct xa_state *xas,
689 struct address_space *mapping, unsigned int order)
690{
691 unsigned long index = xas->xa_index;
692 bool pmd_downgrade; /* splitting PMD entry into PTE entries? */
693 void *entry;
694
695retry:
696 pmd_downgrade = false;
697 xas_lock_irq(xas);
698 entry = get_next_unlocked_entry(xas, order);
699
700 if (entry) {
701 if (dax_is_conflict(entry))
702 goto fallback;
703 if (!xa_is_value(entry)) {
704 xas_set_err(xas, -EIO);
705 goto out_unlock;
706 }
707
708 if (order == 0) {
709 if (dax_is_pmd_entry(entry) &&
710 (dax_is_zero_entry(entry) ||
711 dax_is_empty_entry(entry))) {
712 pmd_downgrade = true;
713 }
714 }
715 }
716
717 if (pmd_downgrade) {
718 /*
719 * Make sure 'entry' remains valid while we drop
720 * the i_pages lock.
721 */
722 dax_lock_entry(xas, entry);
723
724 /*
725 * Besides huge zero pages the only other thing that gets
726 * downgraded are empty entries which don't need to be
727 * unmapped.
728 */
729 if (dax_is_zero_entry(entry)) {
730 xas_unlock_irq(xas);
731 unmap_mapping_pages(mapping,
732 xas->xa_index & ~PG_PMD_COLOUR,
733 PG_PMD_NR, false);
734 xas_reset(xas);
735 xas_lock_irq(xas);
736 }
737
738 dax_disassociate_entry(entry, mapping, false);
739 xas_store(xas, NULL); /* undo the PMD join */
740 dax_wake_entry(xas, entry, WAKE_ALL);
741 mapping->nrpages -= PG_PMD_NR;
742 entry = NULL;
743 xas_set(xas, index);
744 }
745
746 if (entry) {
747 dax_lock_entry(xas, entry);
748 } else {
749 unsigned long flags = DAX_EMPTY;
750
751 if (order > 0)
752 flags |= DAX_PMD;
753 entry = dax_make_entry(0, flags);
754 dax_lock_entry(xas, entry);
755 if (xas_error(xas))
756 goto out_unlock;
757 mapping->nrpages += 1UL << order;
758 }
759
760out_unlock:
761 xas_unlock_irq(xas);
762 if (xas_nomem(xas, mapping_gfp_mask(mapping) & ~__GFP_HIGHMEM))
763 goto retry;
764 if (xas->xa_node == XA_ERROR(-ENOMEM))
765 return xa_mk_internal(VM_FAULT_OOM);
766 if (xas_error(xas))
767 return xa_mk_internal(VM_FAULT_SIGBUS);
768 return entry;
769fallback:
770 xas_unlock_irq(xas);
771 return xa_mk_internal(VM_FAULT_FALLBACK);
772}
773
774/**
775 * dax_layout_busy_page_range - find first pinned page in @mapping
776 * @mapping: address space to scan for a pinned page
777 * @start: Starting offset. Page containing 'start' is included.
778 * @end: End offset. Page containing 'end' is included. If 'end' is LLONG_MAX,
779 * pages from 'start' till the end of file are included.
780 *
781 * DAX requires ZONE_DEVICE mapped pages. A page is considered busy when
782 * folio_ref_count(folio) exceeds folio_mapcount(folio). This helper is
783 * used to determine if any page in the mapping is busy, i.e. for DMA,
784 * or other get_user_pages() usages.
785 *
786 * It is expected that the filesystem is holding locks to block the
787 * establishment of new mappings in this address_space. I.e. it expects
788 * to be able to run unmap_mapping_pages() and subsequently not race
789 * mapping_mapped() becoming true.
790 */
791static struct page *dax_layout_busy_page_range(struct address_space *mapping,
792 loff_t start, loff_t end)
793{
794 void *entry;
795 unsigned int scanned = 0;
796 struct page *page = NULL;
797 pgoff_t start_idx = start >> PAGE_SHIFT;
798 pgoff_t end_idx;
799 XA_STATE(xas, &mapping->i_pages, start_idx);
800
801 if (!dax_mapping(mapping))
802 return NULL;
803
804 /* If end == LLONG_MAX, all pages from start to till end of file */
805 if (end == LLONG_MAX)
806 end_idx = ULONG_MAX;
807 else
808 end_idx = end >> PAGE_SHIFT;
809 /*
810 * If we race get_user_pages_fast() here either we'll see the
811 * elevated page count in the iteration and wait, or
812 * get_user_pages_fast() will see that the page it took a reference
813 * against is no longer mapped in the page tables and bail to the
814 * get_user_pages() slow path. The slow path is protected by
815 * pte_lock() and pmd_lock(). New references are not taken without
816 * holding those locks, and unmap_mapping_pages() will not zero the
817 * pte or pmd without holding the respective lock, so we are
818 * guaranteed to either see new references or prevent new
819 * references from being established.
820 */
821 unmap_mapping_pages(mapping, start_idx, end_idx - start_idx + 1, 0);
822
823 xas_lock_irq(&xas);
824 xas_for_each(&xas, entry, end_idx) {
825 if (WARN_ON_ONCE(!xa_is_value(entry)))
826 continue;
827 entry = wait_entry_unlocked_exclusive(&xas, entry);
828 if (entry)
829 page = dax_busy_page(entry);
830 put_unlocked_entry(&xas, entry, WAKE_NEXT);
831 if (page)
832 break;
833 if (++scanned % XA_CHECK_SCHED)
834 continue;
835
836 xas_pause(&xas);
837 xas_unlock_irq(&xas);
838 cond_resched();
839 xas_lock_irq(&xas);
840 }
841 xas_unlock_irq(&xas);
842 return page;
843}
844
845static int __dax_invalidate_entry(struct address_space *mapping,
846 pgoff_t index, bool trunc)
847{
848 XA_STATE(xas, &mapping->i_pages, index);
849 int ret = 0;
850 void *entry;
851
852 xas_lock_irq(&xas);
853 entry = get_next_unlocked_entry(&xas, 0);
854 if (!entry || WARN_ON_ONCE(!xa_is_value(entry)))
855 goto out;
856 if (!trunc &&
857 (xas_get_mark(&xas, PAGECACHE_TAG_DIRTY) ||
858 xas_get_mark(&xas, PAGECACHE_TAG_TOWRITE)))
859 goto out;
860 dax_disassociate_entry(entry, mapping, trunc);
861 xas_store(&xas, NULL);
862 mapping->nrpages -= 1UL << dax_entry_order(entry);
863 ret = 1;
864out:
865 put_unlocked_entry(&xas, entry, WAKE_ALL);
866 xas_unlock_irq(&xas);
867 return ret;
868}
869
870static int __dax_clear_dirty_range(struct address_space *mapping,
871 pgoff_t start, pgoff_t end)
872{
873 XA_STATE(xas, &mapping->i_pages, start);
874 unsigned int scanned = 0;
875 void *entry;
876
877 xas_lock_irq(&xas);
878 xas_for_each(&xas, entry, end) {
879 entry = wait_entry_unlocked_exclusive(&xas, entry);
880 if (!entry)
881 continue;
882 xas_clear_mark(&xas, PAGECACHE_TAG_DIRTY);
883 xas_clear_mark(&xas, PAGECACHE_TAG_TOWRITE);
884 put_unlocked_entry(&xas, entry, WAKE_NEXT);
885
886 if (++scanned % XA_CHECK_SCHED)
887 continue;
888
889 xas_pause(&xas);
890 xas_unlock_irq(&xas);
891 cond_resched();
892 xas_lock_irq(&xas);
893 }
894 xas_unlock_irq(&xas);
895
896 return 0;
897}
898
899/*
900 * Delete DAX entry at @index from @mapping. Wait for it
901 * to be unlocked before deleting it.
902 */
903int dax_delete_mapping_entry(struct address_space *mapping, pgoff_t index)
904{
905 int ret = __dax_invalidate_entry(mapping, index, true);
906
907 /*
908 * This gets called from truncate / punch_hole path. As such, the caller
909 * must hold locks protecting against concurrent modifications of the
910 * page cache (usually fs-private i_mmap_sem for writing). Since the
911 * caller has seen a DAX entry for this index, we better find it
912 * at that index as well...
913 */
914 WARN_ON_ONCE(!ret);
915 return ret;
916}
917
918void dax_delete_mapping_range(struct address_space *mapping,
919 loff_t start, loff_t end)
920{
921 void *entry;
922 pgoff_t start_idx = start >> PAGE_SHIFT;
923 pgoff_t end_idx;
924 XA_STATE(xas, &mapping->i_pages, start_idx);
925
926 /* If end == LLONG_MAX, all pages from start to till end of file */
927 if (end == LLONG_MAX)
928 end_idx = ULONG_MAX;
929 else
930 end_idx = end >> PAGE_SHIFT;
931
932 xas_lock_irq(&xas);
933 xas_for_each(&xas, entry, end_idx) {
934 if (!xa_is_value(entry))
935 continue;
936 entry = wait_entry_unlocked_exclusive(&xas, entry);
937 if (!entry)
938 continue;
939 dax_disassociate_entry(entry, mapping, true);
940 xas_store(&xas, NULL);
941 mapping->nrpages -= 1UL << dax_entry_order(entry);
942 put_unlocked_entry(&xas, entry, WAKE_ALL);
943 }
944 xas_unlock_irq(&xas);
945}
946EXPORT_SYMBOL_GPL(dax_delete_mapping_range);
947
948static int wait_page_idle(struct page *page,
949 void (cb)(struct inode *),
950 struct inode *inode)
951{
952 return ___wait_var_event(page, dax_page_is_idle(page),
953 TASK_INTERRUPTIBLE, 0, 0, cb(inode));
954}
955
956static void wait_page_idle_uninterruptible(struct page *page,
957 struct inode *inode)
958{
959 ___wait_var_event(page, dax_page_is_idle(page),
960 TASK_UNINTERRUPTIBLE, 0, 0, schedule());
961}
962
963/*
964 * Unmaps the inode and waits for any DMA to complete prior to deleting the
965 * DAX mapping entries for the range.
966 *
967 * For NOWAIT behavior, pass @cb as NULL to early-exit on first found
968 * busy page
969 */
970int dax_break_layout(struct inode *inode, loff_t start, loff_t end,
971 void (cb)(struct inode *))
972{
973 struct page *page;
974 int error = 0;
975
976 if (!dax_mapping(inode->i_mapping))
977 return 0;
978
979 do {
980 page = dax_layout_busy_page_range(inode->i_mapping, start, end);
981 if (!page)
982 break;
983 if (!cb) {
984 error = -ERESTARTSYS;
985 break;
986 }
987
988 error = wait_page_idle(page, cb, inode);
989 } while (error == 0);
990
991 if (!page)
992 dax_delete_mapping_range(inode->i_mapping, start, end);
993
994 return error;
995}
996EXPORT_SYMBOL_GPL(dax_break_layout);
997
998void dax_break_layout_final(struct inode *inode)
999{
1000 struct page *page;
1001
1002 if (!dax_mapping(inode->i_mapping))
1003 return;
1004
1005 do {
1006 page = dax_layout_busy_page_range(inode->i_mapping, 0,
1007 LLONG_MAX);
1008 if (!page)
1009 break;
1010
1011 wait_page_idle_uninterruptible(page, inode);
1012 } while (true);
1013
1014 if (!page)
1015 dax_delete_mapping_range(inode->i_mapping, 0, LLONG_MAX);
1016}
1017EXPORT_SYMBOL_GPL(dax_break_layout_final);
1018
1019/*
1020 * Invalidate DAX entry if it is clean.
1021 */
1022int dax_invalidate_mapping_entry_sync(struct address_space *mapping,
1023 pgoff_t index)
1024{
1025 return __dax_invalidate_entry(mapping, index, false);
1026}
1027
1028static pgoff_t dax_iomap_pgoff(const struct iomap *iomap, loff_t pos)
1029{
1030 return PHYS_PFN(iomap->addr + (pos & PAGE_MASK) - iomap->offset);
1031}
1032
1033static int copy_cow_page_dax(struct vm_fault *vmf, const struct iomap_iter *iter)
1034{
1035 pgoff_t pgoff = dax_iomap_pgoff(&iter->iomap, iter->pos);
1036 void *vto, *kaddr;
1037 long rc;
1038 int id;
1039
1040 id = dax_read_lock();
1041 rc = dax_direct_access(iter->iomap.dax_dev, pgoff, 1, DAX_ACCESS,
1042 &kaddr, NULL);
1043 if (rc < 0) {
1044 dax_read_unlock(id);
1045 return rc;
1046 }
1047 vto = kmap_atomic(vmf->cow_page);
1048 copy_user_page(vto, kaddr, vmf->address, vmf->cow_page);
1049 kunmap_atomic(vto);
1050 dax_read_unlock(id);
1051 return 0;
1052}
1053
1054/*
1055 * MAP_SYNC on a dax mapping guarantees dirty metadata is
1056 * flushed on write-faults (non-cow), but not read-faults.
1057 */
1058static bool dax_fault_is_synchronous(const struct iomap_iter *iter,
1059 struct vm_area_struct *vma)
1060{
1061 return (iter->flags & IOMAP_WRITE) && (vma->vm_flags & VM_SYNC) &&
1062 (iter->iomap.flags & IOMAP_F_DIRTY);
1063}
1064
1065/*
1066 * By this point grab_mapping_entry() has ensured that we have a locked entry
1067 * of the appropriate size so we don't have to worry about downgrading PMDs to
1068 * PTEs. If we happen to be trying to insert a PTE and there is a PMD
1069 * already in the tree, we will skip the insertion and just dirty the PMD as
1070 * appropriate.
1071 */
1072static void *dax_insert_entry(struct xa_state *xas, struct vm_fault *vmf,
1073 const struct iomap_iter *iter, void *entry, unsigned long pfn,
1074 unsigned long flags)
1075{
1076 struct address_space *mapping = vmf->vma->vm_file->f_mapping;
1077 void *new_entry = dax_make_entry(pfn, flags);
1078 bool write = iter->flags & IOMAP_WRITE;
1079 bool dirty = write && !dax_fault_is_synchronous(iter, vmf->vma);
1080 bool shared = iter->iomap.flags & IOMAP_F_SHARED;
1081
1082 if (dirty)
1083 __mark_inode_dirty(mapping->host, I_DIRTY_PAGES);
1084
1085 if (shared || (dax_is_zero_entry(entry) && !(flags & DAX_ZERO_PAGE))) {
1086 unsigned long index = xas->xa_index;
1087 /* we are replacing a zero page with block mapping */
1088 if (dax_is_pmd_entry(entry))
1089 unmap_mapping_pages(mapping, index & ~PG_PMD_COLOUR,
1090 PG_PMD_NR, false);
1091 else /* pte entry */
1092 unmap_mapping_pages(mapping, index, 1, false);
1093 }
1094
1095 xas_reset(xas);
1096 xas_lock_irq(xas);
1097 if (shared || dax_is_zero_entry(entry) || dax_is_empty_entry(entry)) {
1098 void *old;
1099
1100 dax_disassociate_entry(entry, mapping, false);
1101 dax_associate_entry(new_entry, mapping, vmf->vma,
1102 vmf->address, shared);
1103
1104 /*
1105 * Only swap our new entry into the page cache if the current
1106 * entry is a zero page or an empty entry. If a normal PTE or
1107 * PMD entry is already in the cache, we leave it alone. This
1108 * means that if we are trying to insert a PTE and the
1109 * existing entry is a PMD, we will just leave the PMD in the
1110 * tree and dirty it if necessary.
1111 */
1112 old = dax_lock_entry(xas, new_entry);
1113 WARN_ON_ONCE(old != xa_mk_value(xa_to_value(entry) |
1114 DAX_LOCKED));
1115 entry = new_entry;
1116 } else {
1117 xas_load(xas); /* Walk the xa_state */
1118 }
1119
1120 if (dirty)
1121 xas_set_mark(xas, PAGECACHE_TAG_DIRTY);
1122
1123 if (write && shared)
1124 xas_set_mark(xas, PAGECACHE_TAG_TOWRITE);
1125
1126 xas_unlock_irq(xas);
1127 return entry;
1128}
1129
1130static int dax_writeback_one(struct xa_state *xas, struct dax_device *dax_dev,
1131 struct address_space *mapping, void *entry)
1132{
1133 unsigned long pfn, index, count, end;
1134 long ret = 0;
1135 struct vm_area_struct *vma;
1136
1137 /*
1138 * A page got tagged dirty in DAX mapping? Something is seriously
1139 * wrong.
1140 */
1141 if (WARN_ON(!xa_is_value(entry)))
1142 return -EIO;
1143
1144 if (unlikely(dax_is_locked(entry))) {
1145 void *old_entry = entry;
1146
1147 entry = get_next_unlocked_entry(xas, 0);
1148
1149 /* Entry got punched out / reallocated? */
1150 if (!entry || WARN_ON_ONCE(!xa_is_value(entry)))
1151 goto put_unlocked;
1152 /*
1153 * Entry got reallocated elsewhere? No need to writeback.
1154 * We have to compare pfns as we must not bail out due to
1155 * difference in lockbit or entry type.
1156 */
1157 if (dax_to_pfn(old_entry) != dax_to_pfn(entry))
1158 goto put_unlocked;
1159 if (WARN_ON_ONCE(dax_is_empty_entry(entry) ||
1160 dax_is_zero_entry(entry))) {
1161 ret = -EIO;
1162 goto put_unlocked;
1163 }
1164
1165 /* Another fsync thread may have already done this entry */
1166 if (!xas_get_mark(xas, PAGECACHE_TAG_TOWRITE))
1167 goto put_unlocked;
1168 }
1169
1170 /* Lock the entry to serialize with page faults */
1171 dax_lock_entry(xas, entry);
1172
1173 /*
1174 * We can clear the tag now but we have to be careful so that concurrent
1175 * dax_writeback_one() calls for the same index cannot finish before we
1176 * actually flush the caches. This is achieved as the calls will look
1177 * at the entry only under the i_pages lock and once they do that
1178 * they will see the entry locked and wait for it to unlock.
1179 */
1180 xas_clear_mark(xas, PAGECACHE_TAG_TOWRITE);
1181 xas_unlock_irq(xas);
1182
1183 /*
1184 * If dax_writeback_mapping_range() was given a wbc->range_start
1185 * in the middle of a PMD, the 'index' we use needs to be
1186 * aligned to the start of the PMD.
1187 * This allows us to flush for PMD_SIZE and not have to worry about
1188 * partial PMD writebacks.
1189 */
1190 pfn = dax_to_pfn(entry);
1191 count = 1UL << dax_entry_order(entry);
1192 index = xas->xa_index & ~(count - 1);
1193 end = index + count - 1;
1194
1195 /* Walk all mappings of a given index of a file and writeprotect them */
1196 i_mmap_lock_read(mapping);
1197 mapping_rmap_tree_foreach(vma, mapping, index, end) {
1198 pfn_mkclean_range(pfn, count, index, vma);
1199 cond_resched();
1200 }
1201 i_mmap_unlock_read(mapping);
1202
1203 dax_flush(dax_dev, page_address(pfn_to_page(pfn)), count * PAGE_SIZE);
1204 /*
1205 * After we have flushed the cache, we can clear the dirty tag. There
1206 * cannot be new dirty data in the pfn after the flush has completed as
1207 * the pfn mappings are writeprotected and fault waits for mapping
1208 * entry lock.
1209 */
1210 xas_reset(xas);
1211 xas_lock_irq(xas);
1212 xas_store(xas, entry);
1213 xas_clear_mark(xas, PAGECACHE_TAG_DIRTY);
1214 dax_wake_entry(xas, entry, WAKE_NEXT);
1215
1216 trace_dax_writeback_one(mapping->host, index, count);
1217 return ret;
1218
1219 put_unlocked:
1220 put_unlocked_entry(xas, entry, WAKE_NEXT);
1221 return ret;
1222}
1223
1224/*
1225 * Flush the mapping to the persistent domain within the byte range of [start,
1226 * end]. This is required by data integrity operations to ensure file data is
1227 * on persistent storage prior to completion of the operation.
1228 */
1229int dax_writeback_mapping_range(struct address_space *mapping,
1230 struct dax_device *dax_dev, struct writeback_control *wbc)
1231{
1232 XA_STATE(xas, &mapping->i_pages, wbc->range_start >> PAGE_SHIFT);
1233 struct inode *inode = mapping->host;
1234 pgoff_t end_index = wbc->range_end >> PAGE_SHIFT;
1235 void *entry;
1236 int ret = 0;
1237 unsigned int scanned = 0;
1238
1239 if (WARN_ON_ONCE(inode->i_blkbits != PAGE_SHIFT))
1240 return -EIO;
1241
1242 if (mapping_empty(mapping) || wbc->sync_mode != WB_SYNC_ALL)
1243 return 0;
1244
1245 trace_dax_writeback_range(inode, xas.xa_index, end_index);
1246
1247 tag_pages_for_writeback(mapping, xas.xa_index, end_index);
1248
1249 xas_lock_irq(&xas);
1250 xas_for_each_marked(&xas, entry, end_index, PAGECACHE_TAG_TOWRITE) {
1251 ret = dax_writeback_one(&xas, dax_dev, mapping, entry);
1252 if (ret < 0) {
1253 mapping_set_error(mapping, ret);
1254 break;
1255 }
1256 if (++scanned % XA_CHECK_SCHED)
1257 continue;
1258
1259 xas_pause(&xas);
1260 xas_unlock_irq(&xas);
1261 cond_resched();
1262 xas_lock_irq(&xas);
1263 }
1264 xas_unlock_irq(&xas);
1265 trace_dax_writeback_range_done(inode, xas.xa_index, end_index);
1266 return ret;
1267}
1268EXPORT_SYMBOL_GPL(dax_writeback_mapping_range);
1269
1270static int dax_iomap_direct_access(const struct iomap *iomap, loff_t pos,
1271 size_t size, void **kaddr, unsigned long *pfnp)
1272{
1273 pgoff_t pgoff = dax_iomap_pgoff(iomap, pos);
1274 int id, rc = 0;
1275 long length;
1276
1277 id = dax_read_lock();
1278 length = dax_direct_access(iomap->dax_dev, pgoff, PHYS_PFN(size),
1279 DAX_ACCESS, kaddr, pfnp);
1280 if (length < 0) {
1281 rc = length;
1282 goto out;
1283 }
1284 if (!pfnp)
1285 goto out_check_addr;
1286 rc = -EINVAL;
1287 if (PFN_PHYS(length) < size)
1288 goto out;
1289 if (*pfnp & (PHYS_PFN(size)-1))
1290 goto out;
1291
1292 rc = 0;
1293
1294out_check_addr:
1295 if (!kaddr)
1296 goto out;
1297 if (!*kaddr)
1298 rc = -EFAULT;
1299out:
1300 dax_read_unlock(id);
1301 return rc;
1302}
1303
1304/**
1305 * dax_iomap_copy_around - Prepare for an unaligned write to a shared/cow page
1306 * by copying the data before and after the range to be written.
1307 * @pos: address to do copy from.
1308 * @length: size of copy operation.
1309 * @align_size: aligned w.r.t align_size (either PMD_SIZE or PAGE_SIZE)
1310 * @srcmap: iomap srcmap
1311 * @daddr: destination address to copy to.
1312 *
1313 * This can be called from two places. Either during DAX write fault (page
1314 * aligned), to copy the length size data to daddr. Or, while doing normal DAX
1315 * write operation, dax_iomap_iter() might call this to do the copy of either
1316 * start or end unaligned address. In the latter case the rest of the copy of
1317 * aligned ranges is taken care by dax_iomap_iter() itself.
1318 * If the srcmap contains invalid data, such as HOLE and UNWRITTEN, zero the
1319 * area to make sure no old data remains.
1320 */
1321static int dax_iomap_copy_around(loff_t pos, uint64_t length, size_t align_size,
1322 const struct iomap *srcmap, void *daddr)
1323{
1324 loff_t head_off = pos & (align_size - 1);
1325 size_t size = ALIGN(head_off + length, align_size);
1326 loff_t end = pos + length;
1327 loff_t pg_end = round_up(end, align_size);
1328 /* copy_all is usually in page fault case */
1329 bool copy_all = head_off == 0 && end == pg_end;
1330 /* zero the edges if srcmap is a HOLE or IOMAP_UNWRITTEN */
1331 bool zero_edge = srcmap->flags & IOMAP_F_SHARED ||
1332 srcmap->type == IOMAP_UNWRITTEN;
1333 void *saddr = NULL;
1334 int ret = 0;
1335
1336 if (!zero_edge) {
1337 ret = dax_iomap_direct_access(srcmap, pos, size, &saddr, NULL);
1338 if (ret)
1339 return dax_mem2blk_err(ret);
1340 }
1341
1342 if (copy_all) {
1343 if (zero_edge)
1344 memset(daddr, 0, size);
1345 else
1346 ret = copy_mc_to_kernel(daddr, saddr, length);
1347 goto out;
1348 }
1349
1350 /* Copy the head part of the range */
1351 if (head_off) {
1352 if (zero_edge)
1353 memset(daddr, 0, head_off);
1354 else {
1355 ret = copy_mc_to_kernel(daddr, saddr, head_off);
1356 if (ret)
1357 return -EIO;
1358 }
1359 }
1360
1361 /* Copy the tail part of the range */
1362 if (end < pg_end) {
1363 loff_t tail_off = head_off + length;
1364 loff_t tail_len = pg_end - end;
1365
1366 if (zero_edge)
1367 memset(daddr + tail_off, 0, tail_len);
1368 else {
1369 ret = copy_mc_to_kernel(daddr + tail_off,
1370 saddr + tail_off, tail_len);
1371 if (ret)
1372 return -EIO;
1373 }
1374 }
1375out:
1376 if (zero_edge)
1377 dax_flush(srcmap->dax_dev, daddr, size);
1378 return ret ? -EIO : 0;
1379}
1380
1381/*
1382 * The user has performed a load from a hole in the file. Allocating a new
1383 * page in the file would cause excessive storage usage for workloads with
1384 * sparse files. Instead we insert a read-only mapping of the 4k zero page.
1385 * If this page is ever written to we will re-fault and change the mapping to
1386 * point to real DAX storage instead.
1387 */
1388static vm_fault_t dax_load_hole(struct xa_state *xas, struct vm_fault *vmf,
1389 const struct iomap_iter *iter, void **entry)
1390{
1391 struct inode *inode = iter->inode;
1392 unsigned long vaddr = vmf->address;
1393 unsigned long pfn = zero_pfn(vaddr);
1394 vm_fault_t ret;
1395
1396 *entry = dax_insert_entry(xas, vmf, iter, *entry, pfn, DAX_ZERO_PAGE);
1397
1398 ret = vmf_insert_page_mkwrite(vmf, pfn_to_page(pfn), false);
1399 trace_dax_load_hole(inode, vmf, ret);
1400 return ret;
1401}
1402
1403#ifdef CONFIG_FS_DAX_PMD
1404static vm_fault_t dax_pmd_load_hole(struct xa_state *xas, struct vm_fault *vmf,
1405 const struct iomap_iter *iter, void **entry)
1406{
1407 struct address_space *mapping = vmf->vma->vm_file->f_mapping;
1408 struct inode *inode = mapping->host;
1409 struct folio *zero_folio;
1410 vm_fault_t ret;
1411
1412 zero_folio = mm_get_huge_zero_folio(vmf->vma->vm_mm);
1413
1414 if (unlikely(!zero_folio)) {
1415 trace_dax_pmd_load_hole_fallback(inode, vmf, zero_folio, *entry);
1416 return VM_FAULT_FALLBACK;
1417 }
1418
1419 *entry = dax_insert_entry(xas, vmf, iter, *entry, folio_pfn(zero_folio),
1420 DAX_PMD | DAX_ZERO_PAGE);
1421
1422 ret = vmf_insert_folio_pmd(vmf, zero_folio, false);
1423 if (ret == VM_FAULT_NOPAGE)
1424 trace_dax_pmd_load_hole(inode, vmf, zero_folio, *entry);
1425 return ret;
1426}
1427#else
1428static vm_fault_t dax_pmd_load_hole(struct xa_state *xas, struct vm_fault *vmf,
1429 const struct iomap_iter *iter, void **entry)
1430{
1431 return VM_FAULT_FALLBACK;
1432}
1433#endif /* CONFIG_FS_DAX_PMD */
1434
1435static int dax_unshare_iter(struct iomap_iter *iter)
1436{
1437 struct iomap *iomap = &iter->iomap;
1438 const struct iomap *srcmap = iomap_iter_srcmap(iter);
1439 loff_t copy_pos = iter->pos;
1440 u64 copy_len = iomap_length(iter);
1441 u32 mod;
1442 int id = 0;
1443 s64 ret;
1444 void *daddr = NULL, *saddr = NULL;
1445
1446 if (!iomap_want_unshare_iter(iter))
1447 return iomap_iter_advance_full(iter);
1448
1449 /*
1450 * Extend the file range to be aligned to fsblock/pagesize, because
1451 * we need to copy entire blocks, not just the byte range specified.
1452 * Invalidate the mapping because we're about to CoW.
1453 */
1454 mod = offset_in_page(copy_pos);
1455 if (mod) {
1456 copy_len += mod;
1457 copy_pos -= mod;
1458 }
1459
1460 mod = offset_in_page(copy_pos + copy_len);
1461 if (mod)
1462 copy_len += PAGE_SIZE - mod;
1463
1464 invalidate_inode_pages2_range(iter->inode->i_mapping,
1465 copy_pos >> PAGE_SHIFT,
1466 (copy_pos + copy_len - 1) >> PAGE_SHIFT);
1467
1468 id = dax_read_lock();
1469 ret = dax_iomap_direct_access(iomap, copy_pos, copy_len, &daddr, NULL);
1470 if (ret < 0)
1471 goto out_unlock;
1472
1473 ret = dax_iomap_direct_access(srcmap, copy_pos, copy_len, &saddr, NULL);
1474 if (ret < 0)
1475 goto out_unlock;
1476
1477 if (copy_mc_to_kernel(daddr, saddr, copy_len) != 0)
1478 ret = -EIO;
1479
1480out_unlock:
1481 dax_read_unlock(id);
1482 if (ret < 0)
1483 return dax_mem2blk_err(ret);
1484 return iomap_iter_advance_full(iter);
1485}
1486
1487int dax_file_unshare(struct inode *inode, loff_t pos, loff_t len,
1488 const struct iomap_ops *ops)
1489{
1490 struct iomap_iter iter = {
1491 .inode = inode,
1492 .pos = pos,
1493 .flags = IOMAP_WRITE | IOMAP_UNSHARE | IOMAP_DAX,
1494 };
1495 loff_t size = i_size_read(inode);
1496 int ret;
1497
1498 if (pos < 0 || pos >= size)
1499 return 0;
1500
1501 iter.len = min(len, size - pos);
1502 while ((ret = iomap_iter(&iter, ops)) > 0)
1503 iter.status = dax_unshare_iter(&iter);
1504 return ret;
1505}
1506EXPORT_SYMBOL_GPL(dax_file_unshare);
1507
1508static int dax_memzero(struct iomap_iter *iter, loff_t pos, size_t size)
1509{
1510 const struct iomap *iomap = &iter->iomap;
1511 const struct iomap *srcmap = iomap_iter_srcmap(iter);
1512 unsigned offset = offset_in_page(pos);
1513 pgoff_t pgoff = dax_iomap_pgoff(iomap, pos);
1514 void *kaddr;
1515 long ret;
1516
1517 ret = dax_direct_access(iomap->dax_dev, pgoff, 1, DAX_ACCESS, &kaddr,
1518 NULL);
1519 if (ret < 0)
1520 return dax_mem2blk_err(ret);
1521
1522 memset(kaddr + offset, 0, size);
1523 if (iomap->flags & IOMAP_F_SHARED)
1524 ret = dax_iomap_copy_around(pos, size, PAGE_SIZE, srcmap,
1525 kaddr);
1526 else
1527 dax_flush(iomap->dax_dev, kaddr + offset, size);
1528 return ret;
1529}
1530
1531static int dax_zero_iter(struct iomap_iter *iter, bool *did_zero)
1532{
1533 const struct iomap *iomap = &iter->iomap;
1534 const struct iomap *srcmap = iomap_iter_srcmap(iter);
1535 u64 length = iomap_length(iter);
1536 int ret;
1537
1538 /* already zeroed? we're done. */
1539 if (srcmap->type == IOMAP_HOLE || srcmap->type == IOMAP_UNWRITTEN)
1540 return iomap_iter_advance(iter, length);
1541
1542 /*
1543 * invalidate the pages whose sharing state is to be changed
1544 * because of CoW.
1545 */
1546 if (iomap->flags & IOMAP_F_SHARED)
1547 invalidate_inode_pages2_range(iter->inode->i_mapping,
1548 iter->pos >> PAGE_SHIFT,
1549 (iter->pos + length - 1) >> PAGE_SHIFT);
1550
1551 do {
1552 loff_t pos = iter->pos;
1553 unsigned offset = offset_in_page(pos);
1554 pgoff_t pgoff = dax_iomap_pgoff(iomap, pos);
1555 int id;
1556
1557 length = min_t(u64, PAGE_SIZE - offset, length);
1558
1559 id = dax_read_lock();
1560 if (IS_ALIGNED(pos, PAGE_SIZE) && length == PAGE_SIZE)
1561 ret = dax_zero_page_range(iomap->dax_dev, pgoff, 1);
1562 else
1563 ret = dax_memzero(iter, pos, length);
1564 dax_read_unlock(id);
1565
1566 if (ret < 0)
1567 return ret;
1568
1569 ret = iomap_iter_advance(iter, length);
1570 if (ret)
1571 return ret;
1572 } while ((length = iomap_length(iter)) > 0);
1573
1574 if (did_zero)
1575 *did_zero = true;
1576 return ret;
1577}
1578
1579int dax_zero_range(struct inode *inode, loff_t pos, loff_t len, bool *did_zero,
1580 const struct iomap_ops *ops)
1581{
1582 struct iomap_iter iter = {
1583 .inode = inode,
1584 .pos = pos,
1585 .len = len,
1586 .flags = IOMAP_DAX | IOMAP_ZERO,
1587 };
1588 int ret;
1589
1590 while ((ret = iomap_iter(&iter, ops)) > 0)
1591 iter.status = dax_zero_iter(&iter, did_zero);
1592 return ret;
1593}
1594EXPORT_SYMBOL_GPL(dax_zero_range);
1595
1596int dax_truncate_page(struct inode *inode, loff_t pos, bool *did_zero,
1597 const struct iomap_ops *ops)
1598{
1599 unsigned int blocksize = i_blocksize(inode);
1600 unsigned int off = pos & (blocksize - 1);
1601
1602 /* Block boundary? Nothing to do */
1603 if (!off)
1604 return 0;
1605 return dax_zero_range(inode, pos, blocksize - off, did_zero, ops);
1606}
1607EXPORT_SYMBOL_GPL(dax_truncate_page);
1608
1609static int dax_iomap_iter(struct iomap_iter *iomi, struct iov_iter *iter)
1610{
1611 const struct iomap *iomap = &iomi->iomap;
1612 const struct iomap *srcmap = iomap_iter_srcmap(iomi);
1613 loff_t length = iomap_length(iomi);
1614 loff_t pos = iomi->pos;
1615 struct dax_device *dax_dev = iomap->dax_dev;
1616 loff_t end = pos + length, done = 0;
1617 bool write = iov_iter_rw(iter) == WRITE;
1618 bool cow = write && iomap->flags & IOMAP_F_SHARED;
1619 ssize_t ret = 0;
1620 size_t xfer;
1621 int id;
1622
1623 if (!write) {
1624 end = min(end, i_size_read(iomi->inode));
1625 if (pos >= end)
1626 return 0;
1627
1628 if (iomap->type == IOMAP_HOLE || iomap->type == IOMAP_UNWRITTEN) {
1629 done = iov_iter_zero(min(length, end - pos), iter);
1630 return iomap_iter_advance(iomi, done);
1631 }
1632 }
1633
1634 /*
1635 * In DAX mode, enforce either pure overwrites of written extents, or
1636 * writes to unwritten extents as part of a copy-on-write operation.
1637 */
1638 if (WARN_ON_ONCE(iomap->type != IOMAP_MAPPED &&
1639 !(iomap->flags & IOMAP_F_SHARED)))
1640 return -EIO;
1641
1642 /*
1643 * Write can allocate block for an area which has a hole page mapped
1644 * into page tables. We have to tear down these mappings so that data
1645 * written by write(2) is visible in mmap.
1646 */
1647 if (iomap->flags & IOMAP_F_NEW || cow) {
1648 /*
1649 * Filesystem allows CoW on non-shared extents. The src extents
1650 * may have been mmapped with dirty mark before. To be able to
1651 * invalidate its dax entries, we need to clear the dirty mark
1652 * in advance.
1653 */
1654 if (cow)
1655 __dax_clear_dirty_range(iomi->inode->i_mapping,
1656 pos >> PAGE_SHIFT,
1657 (end - 1) >> PAGE_SHIFT);
1658 invalidate_inode_pages2_range(iomi->inode->i_mapping,
1659 pos >> PAGE_SHIFT,
1660 (end - 1) >> PAGE_SHIFT);
1661 }
1662
1663 id = dax_read_lock();
1664 while ((pos = iomi->pos) < end) {
1665 unsigned offset = pos & (PAGE_SIZE - 1);
1666 const size_t size = ALIGN(length + offset, PAGE_SIZE);
1667 pgoff_t pgoff = dax_iomap_pgoff(iomap, pos);
1668 ssize_t map_len;
1669 bool recovery = false;
1670 void *kaddr;
1671
1672 if (fatal_signal_pending(current)) {
1673 ret = -EINTR;
1674 break;
1675 }
1676
1677 map_len = dax_direct_access(dax_dev, pgoff, PHYS_PFN(size),
1678 DAX_ACCESS, &kaddr, NULL);
1679 if (map_len == -EHWPOISON && iov_iter_rw(iter) == WRITE) {
1680 map_len = dax_direct_access(dax_dev, pgoff,
1681 PHYS_PFN(size), DAX_RECOVERY_WRITE,
1682 &kaddr, NULL);
1683 if (map_len > 0)
1684 recovery = true;
1685 }
1686 if (map_len < 0) {
1687 ret = dax_mem2blk_err(map_len);
1688 break;
1689 }
1690
1691 if (cow) {
1692 ret = dax_iomap_copy_around(pos, length, PAGE_SIZE,
1693 srcmap, kaddr);
1694 if (ret)
1695 break;
1696 }
1697
1698 map_len = PFN_PHYS(map_len);
1699 kaddr += offset;
1700 map_len -= offset;
1701 if (map_len > end - pos)
1702 map_len = end - pos;
1703
1704 if (recovery)
1705 xfer = dax_recovery_write(dax_dev, pgoff, kaddr,
1706 map_len, iter);
1707 else if (write)
1708 xfer = dax_copy_from_iter(dax_dev, pgoff, kaddr,
1709 map_len, iter);
1710 else
1711 xfer = dax_copy_to_iter(dax_dev, pgoff, kaddr,
1712 map_len, iter);
1713
1714 ret = iomap_iter_advance(iomi, xfer);
1715 if (!ret && xfer == 0)
1716 ret = -EFAULT;
1717 if (xfer < map_len)
1718 break;
1719 length = iomap_length(iomi);
1720 }
1721 dax_read_unlock(id);
1722
1723 return ret;
1724}
1725
1726/**
1727 * dax_iomap_rw - Perform I/O to a DAX file
1728 * @iocb: The control block for this I/O
1729 * @iter: The addresses to do I/O from or to
1730 * @ops: iomap ops passed from the file system
1731 *
1732 * This function performs read and write operations to directly mapped
1733 * persistent memory. The callers needs to take care of read/write exclusion
1734 * and evicting any page cache pages in the region under I/O.
1735 */
1736ssize_t
1737dax_iomap_rw(struct kiocb *iocb, struct iov_iter *iter,
1738 const struct iomap_ops *ops)
1739{
1740 struct iomap_iter iomi = {
1741 .inode = iocb->ki_filp->f_mapping->host,
1742 .pos = iocb->ki_pos,
1743 .len = iov_iter_count(iter),
1744 .flags = IOMAP_DAX,
1745 };
1746 loff_t done = 0;
1747 int ret;
1748
1749 if (WARN_ON_ONCE(iocb->ki_flags & IOCB_ATOMIC))
1750 return -EIO;
1751
1752 if (!iomi.len)
1753 return 0;
1754
1755 if (iov_iter_rw(iter) == WRITE) {
1756 lockdep_assert_held_write(&iomi.inode->i_rwsem);
1757 iomi.flags |= IOMAP_WRITE;
1758 } else if (!sb_rdonly(iomi.inode->i_sb)) {
1759 lockdep_assert_held(&iomi.inode->i_rwsem);
1760 }
1761
1762 if (iocb->ki_flags & IOCB_NOWAIT)
1763 iomi.flags |= IOMAP_NOWAIT;
1764
1765 while ((ret = iomap_iter(&iomi, ops)) > 0)
1766 iomi.status = dax_iomap_iter(&iomi, iter);
1767
1768 done = iomi.pos - iocb->ki_pos;
1769 iocb->ki_pos = iomi.pos;
1770 return done ? done : ret;
1771}
1772EXPORT_SYMBOL_GPL(dax_iomap_rw);
1773
1774static vm_fault_t dax_fault_return(int error)
1775{
1776 if (error == 0)
1777 return VM_FAULT_NOPAGE;
1778 return vmf_error(error);
1779}
1780
1781/*
1782 * When handling a synchronous page fault and the inode need a fsync, we can
1783 * insert the PTE/PMD into page tables only after that fsync happened. Skip
1784 * insertion for now and return the pfn so that caller can insert it after the
1785 * fsync is done.
1786 */
1787static vm_fault_t dax_fault_synchronous_pfnp(unsigned long *pfnp,
1788 unsigned long pfn)
1789{
1790 if (WARN_ON_ONCE(!pfnp))
1791 return VM_FAULT_SIGBUS;
1792 *pfnp = pfn;
1793 return VM_FAULT_NEEDDSYNC;
1794}
1795
1796static vm_fault_t dax_fault_cow_page(struct vm_fault *vmf,
1797 const struct iomap_iter *iter)
1798{
1799 vm_fault_t ret;
1800 int error = 0;
1801
1802 switch (iter->iomap.type) {
1803 case IOMAP_HOLE:
1804 case IOMAP_UNWRITTEN:
1805 clear_user_highpage(vmf->cow_page, vmf->address);
1806 break;
1807 case IOMAP_MAPPED:
1808 error = copy_cow_page_dax(vmf, iter);
1809 break;
1810 default:
1811 WARN_ON_ONCE(1);
1812 error = -EIO;
1813 break;
1814 }
1815
1816 if (error)
1817 return dax_fault_return(error);
1818
1819 __SetPageUptodate(vmf->cow_page);
1820 ret = finish_fault(vmf);
1821 if (!ret)
1822 return VM_FAULT_DONE_COW;
1823 return ret;
1824}
1825
1826/**
1827 * dax_fault_iter - Common actor to handle pfn insertion in PTE/PMD fault.
1828 * @vmf: vm fault instance
1829 * @iter: iomap iter
1830 * @pfnp: pfn to be returned
1831 * @xas: the dax mapping tree of a file
1832 * @entry: an unlocked dax entry to be inserted
1833 * @pmd: distinguish whether it is a pmd fault
1834 */
1835static vm_fault_t dax_fault_iter(struct vm_fault *vmf,
1836 const struct iomap_iter *iter, unsigned long *pfnp,
1837 struct xa_state *xas, void **entry, bool pmd)
1838{
1839 const struct iomap *iomap = &iter->iomap;
1840 const struct iomap *srcmap = iomap_iter_srcmap(iter);
1841 size_t size = pmd ? PMD_SIZE : PAGE_SIZE;
1842 loff_t pos = (loff_t)xas->xa_index << PAGE_SHIFT;
1843 bool write = iter->flags & IOMAP_WRITE;
1844 unsigned long entry_flags = pmd ? DAX_PMD : 0;
1845 struct folio *folio;
1846 int ret, err = 0;
1847 unsigned long pfn;
1848 void *kaddr;
1849
1850 if (!pmd && vmf->cow_page)
1851 return dax_fault_cow_page(vmf, iter);
1852
1853 /* if we are reading UNWRITTEN and HOLE, return a hole. */
1854 if (!write &&
1855 (iomap->type == IOMAP_UNWRITTEN || iomap->type == IOMAP_HOLE)) {
1856 if (!pmd)
1857 return dax_load_hole(xas, vmf, iter, entry);
1858 return dax_pmd_load_hole(xas, vmf, iter, entry);
1859 }
1860
1861 if (iomap->type != IOMAP_MAPPED && !(iomap->flags & IOMAP_F_SHARED)) {
1862 WARN_ON_ONCE(1);
1863 return pmd ? VM_FAULT_FALLBACK : VM_FAULT_SIGBUS;
1864 }
1865
1866 err = dax_iomap_direct_access(iomap, pos, size, &kaddr, &pfn);
1867 if (err)
1868 return pmd ? VM_FAULT_FALLBACK : dax_fault_return(err);
1869
1870 *entry = dax_insert_entry(xas, vmf, iter, *entry, pfn, entry_flags);
1871
1872 if (write && iomap->flags & IOMAP_F_SHARED) {
1873 err = dax_iomap_copy_around(pos, size, size, srcmap, kaddr);
1874 if (err)
1875 return dax_fault_return(err);
1876 }
1877
1878 folio = dax_to_folio(*entry);
1879 if (dax_fault_is_synchronous(iter, vmf->vma))
1880 return dax_fault_synchronous_pfnp(pfnp, pfn);
1881
1882 folio_ref_inc(folio);
1883 if (pmd)
1884 ret = vmf_insert_folio_pmd(vmf, pfn_folio(pfn), write);
1885 else
1886 ret = vmf_insert_page_mkwrite(vmf, pfn_to_page(pfn), write);
1887 folio_put(folio);
1888
1889 return ret;
1890}
1891
1892static vm_fault_t dax_iomap_pte_fault(struct vm_fault *vmf, unsigned long *pfnp,
1893 int *iomap_errp, const struct iomap_ops *ops)
1894{
1895 struct address_space *mapping = vmf->vma->vm_file->f_mapping;
1896 XA_STATE(xas, &mapping->i_pages, vmf->pgoff);
1897 struct iomap_iter iter = {
1898 .inode = mapping->host,
1899 .pos = (loff_t)vmf->pgoff << PAGE_SHIFT,
1900 .len = PAGE_SIZE,
1901 .flags = IOMAP_DAX | IOMAP_FAULT,
1902 };
1903 vm_fault_t ret = 0;
1904 void *entry;
1905 int error;
1906
1907 trace_dax_pte_fault(iter.inode, vmf, ret);
1908 /*
1909 * Check whether offset isn't beyond end of file now. Caller is supposed
1910 * to hold locks serializing us with truncate / punch hole so this is
1911 * a reliable test.
1912 */
1913 if (iter.pos >= i_size_read(iter.inode)) {
1914 ret = VM_FAULT_SIGBUS;
1915 goto out;
1916 }
1917
1918 if ((vmf->flags & FAULT_FLAG_WRITE) && !vmf->cow_page)
1919 iter.flags |= IOMAP_WRITE;
1920
1921 entry = grab_mapping_entry(&xas, mapping, 0);
1922 if (xa_is_internal(entry)) {
1923 ret = xa_to_internal(entry);
1924 goto out;
1925 }
1926
1927 /*
1928 * It is possible, particularly with mixed reads & writes to private
1929 * mappings, that we have raced with a PMD fault that overlaps with
1930 * the PTE we need to set up. If so just return and the fault will be
1931 * retried.
1932 */
1933 if (pmd_trans_huge(*vmf->pmd)) {
1934 ret = VM_FAULT_NOPAGE;
1935 goto unlock_entry;
1936 }
1937
1938 while ((error = iomap_iter(&iter, ops)) > 0) {
1939 if (WARN_ON_ONCE(iomap_length(&iter) < PAGE_SIZE)) {
1940 iter.status = -EIO; /* fs corruption? */
1941 continue;
1942 }
1943
1944 ret = dax_fault_iter(vmf, &iter, pfnp, &xas, &entry, false);
1945 if (ret != VM_FAULT_SIGBUS &&
1946 (iter.iomap.flags & IOMAP_F_NEW)) {
1947 count_vm_event(PGMAJFAULT);
1948 count_memcg_event_mm(vmf->vma->vm_mm, PGMAJFAULT);
1949 ret |= VM_FAULT_MAJOR;
1950 }
1951
1952 if (!(ret & VM_FAULT_ERROR))
1953 iter.status = iomap_iter_advance(&iter, PAGE_SIZE);
1954 }
1955
1956 if (iomap_errp)
1957 *iomap_errp = error;
1958 if (!ret && error)
1959 ret = dax_fault_return(error);
1960
1961unlock_entry:
1962 dax_unlock_entry(&xas, entry);
1963out:
1964 trace_dax_pte_fault_done(iter.inode, vmf, ret);
1965 return ret;
1966}
1967
1968#ifdef CONFIG_FS_DAX_PMD
1969static bool dax_fault_check_fallback(struct vm_fault *vmf, struct xa_state *xas,
1970 pgoff_t max_pgoff)
1971{
1972 unsigned long pmd_addr = vmf->address & PMD_MASK;
1973 bool write = vmf->flags & FAULT_FLAG_WRITE;
1974
1975 /*
1976 * Make sure that the faulting address's PMD offset (color) matches
1977 * the PMD offset from the start of the file. This is necessary so
1978 * that a PMD range in the page table overlaps exactly with a PMD
1979 * range in the page cache.
1980 */
1981 if ((vmf->pgoff & PG_PMD_COLOUR) !=
1982 ((vmf->address >> PAGE_SHIFT) & PG_PMD_COLOUR))
1983 return true;
1984
1985 /* Fall back to PTEs if we're going to COW */
1986 if (write && !(vmf->vma->vm_flags & VM_SHARED))
1987 return true;
1988
1989 /* If the PMD would extend outside the VMA */
1990 if (pmd_addr < vmf->vma->vm_start)
1991 return true;
1992 if ((pmd_addr + PMD_SIZE) > vmf->vma->vm_end)
1993 return true;
1994
1995 /* If the PMD would extend beyond the file size */
1996 if ((xas->xa_index | PG_PMD_COLOUR) >= max_pgoff)
1997 return true;
1998
1999 return false;
2000}
2001
2002static vm_fault_t dax_iomap_pmd_fault(struct vm_fault *vmf, unsigned long *pfnp,
2003 const struct iomap_ops *ops)
2004{
2005 struct address_space *mapping = vmf->vma->vm_file->f_mapping;
2006 XA_STATE_ORDER(xas, &mapping->i_pages, vmf->pgoff, PMD_ORDER);
2007 struct iomap_iter iter = {
2008 .inode = mapping->host,
2009 .len = PMD_SIZE,
2010 .flags = IOMAP_DAX | IOMAP_FAULT,
2011 };
2012 vm_fault_t ret = VM_FAULT_FALLBACK;
2013 pgoff_t max_pgoff;
2014 void *entry;
2015
2016 if (vmf->flags & FAULT_FLAG_WRITE)
2017 iter.flags |= IOMAP_WRITE;
2018
2019 /*
2020 * Check whether offset isn't beyond end of file now. Caller is
2021 * supposed to hold locks serializing us with truncate / punch hole so
2022 * this is a reliable test.
2023 */
2024 max_pgoff = DIV_ROUND_UP(i_size_read(iter.inode), PAGE_SIZE);
2025
2026 trace_dax_pmd_fault(iter.inode, vmf, max_pgoff, 0);
2027
2028 if (xas.xa_index >= max_pgoff) {
2029 ret = VM_FAULT_SIGBUS;
2030 goto out;
2031 }
2032
2033 if (dax_fault_check_fallback(vmf, &xas, max_pgoff))
2034 goto fallback;
2035
2036 /*
2037 * grab_mapping_entry() will make sure we get an empty PMD entry,
2038 * a zero PMD entry or a DAX PMD. If it can't (because a PTE
2039 * entry is already in the array, for instance), it will return
2040 * VM_FAULT_FALLBACK.
2041 */
2042 entry = grab_mapping_entry(&xas, mapping, PMD_ORDER);
2043 if (xa_is_internal(entry)) {
2044 ret = xa_to_internal(entry);
2045 goto fallback;
2046 }
2047
2048 /*
2049 * It is possible, particularly with mixed reads & writes to private
2050 * mappings, that we have raced with a PTE fault that overlaps with
2051 * the PMD we need to set up. If so just return and the fault will be
2052 * retried.
2053 */
2054 if (!pmd_none(*vmf->pmd) && !pmd_trans_huge(*vmf->pmd)) {
2055 ret = 0;
2056 goto unlock_entry;
2057 }
2058
2059 iter.pos = (loff_t)xas.xa_index << PAGE_SHIFT;
2060 while (iomap_iter(&iter, ops) > 0) {
2061 if (iomap_length(&iter) < PMD_SIZE)
2062 continue; /* actually breaks out of the loop */
2063
2064 ret = dax_fault_iter(vmf, &iter, pfnp, &xas, &entry, true);
2065 if (ret != VM_FAULT_FALLBACK)
2066 iter.status = iomap_iter_advance(&iter, PMD_SIZE);
2067 }
2068
2069unlock_entry:
2070 dax_unlock_entry(&xas, entry);
2071fallback:
2072 if (ret == VM_FAULT_FALLBACK) {
2073 split_huge_pmd(vmf->vma, vmf->pmd, vmf->address);
2074 count_vm_event(THP_FAULT_FALLBACK);
2075 }
2076out:
2077 trace_dax_pmd_fault_done(iter.inode, vmf, max_pgoff, ret);
2078 return ret;
2079}
2080#else
2081static vm_fault_t dax_iomap_pmd_fault(struct vm_fault *vmf, unsigned long *pfnp,
2082 const struct iomap_ops *ops)
2083{
2084 return VM_FAULT_FALLBACK;
2085}
2086#endif /* CONFIG_FS_DAX_PMD */
2087
2088/**
2089 * dax_iomap_fault - handle a page fault on a DAX file
2090 * @vmf: The description of the fault
2091 * @order: Order of the page to fault in
2092 * @pfnp: PFN to insert for synchronous faults if fsync is required
2093 * @iomap_errp: Storage for detailed error code in case of error
2094 * @ops: Iomap ops passed from the file system
2095 *
2096 * When a page fault occurs, filesystems may call this helper in
2097 * their fault handler for DAX files. dax_iomap_fault() assumes the caller
2098 * has done all the necessary locking for page fault to proceed
2099 * successfully.
2100 */
2101vm_fault_t dax_iomap_fault(struct vm_fault *vmf, unsigned int order,
2102 unsigned long *pfnp, int *iomap_errp,
2103 const struct iomap_ops *ops)
2104{
2105 if (order == 0)
2106 return dax_iomap_pte_fault(vmf, pfnp, iomap_errp, ops);
2107 else if (order == PMD_ORDER)
2108 return dax_iomap_pmd_fault(vmf, pfnp, ops);
2109 else
2110 return VM_FAULT_FALLBACK;
2111}
2112EXPORT_SYMBOL_GPL(dax_iomap_fault);
2113
2114/*
2115 * dax_insert_pfn_mkwrite - insert PTE or PMD entry into page tables
2116 * @vmf: The description of the fault
2117 * @pfn: PFN to insert
2118 * @order: Order of entry to insert.
2119 *
2120 * This function inserts a writeable PTE or PMD entry into the page tables
2121 * for an mmaped DAX file. It also marks the page cache entry as dirty.
2122 */
2123static vm_fault_t dax_insert_pfn_mkwrite(struct vm_fault *vmf,
2124 unsigned long pfn, unsigned int order)
2125{
2126 struct address_space *mapping = vmf->vma->vm_file->f_mapping;
2127 XA_STATE_ORDER(xas, &mapping->i_pages, vmf->pgoff, order);
2128 struct folio *folio;
2129 void *entry;
2130 vm_fault_t ret;
2131
2132 xas_lock_irq(&xas);
2133 entry = get_next_unlocked_entry(&xas, order);
2134 /* Did we race with someone splitting entry or so? */
2135 if (!entry || dax_is_conflict(entry) ||
2136 (order == 0 && !dax_is_pte_entry(entry))) {
2137 put_unlocked_entry(&xas, entry, WAKE_NEXT);
2138 xas_unlock_irq(&xas);
2139 trace_dax_insert_pfn_mkwrite_no_entry(mapping->host, vmf,
2140 VM_FAULT_NOPAGE);
2141 return VM_FAULT_NOPAGE;
2142 }
2143 xas_set_mark(&xas, PAGECACHE_TAG_DIRTY);
2144 dax_lock_entry(&xas, entry);
2145 xas_unlock_irq(&xas);
2146 folio = pfn_folio(pfn);
2147 folio_ref_inc(folio);
2148 if (order == 0)
2149 ret = vmf_insert_page_mkwrite(vmf, &folio->page, true);
2150#ifdef CONFIG_FS_DAX_PMD
2151 else if (order == PMD_ORDER)
2152 ret = vmf_insert_folio_pmd(vmf, folio, FAULT_FLAG_WRITE);
2153#endif
2154 else
2155 ret = VM_FAULT_FALLBACK;
2156 folio_put(folio);
2157 dax_unlock_entry(&xas, entry);
2158 trace_dax_insert_pfn_mkwrite(mapping->host, vmf, ret);
2159 return ret;
2160}
2161
2162/**
2163 * dax_finish_sync_fault - finish synchronous page fault
2164 * @vmf: The description of the fault
2165 * @order: Order of entry to be inserted
2166 * @pfn: PFN to insert
2167 *
2168 * This function ensures that the file range touched by the page fault is
2169 * stored persistently on the media and handles inserting of appropriate page
2170 * table entry.
2171 */
2172vm_fault_t dax_finish_sync_fault(struct vm_fault *vmf, unsigned int order,
2173 unsigned long pfn)
2174{
2175 int err;
2176 loff_t start = ((loff_t)vmf->pgoff) << PAGE_SHIFT;
2177 size_t len = PAGE_SIZE << order;
2178
2179 err = vfs_fsync_range(vmf->vma->vm_file, start, start + len - 1, 1);
2180 if (err)
2181 return VM_FAULT_SIGBUS;
2182 return dax_insert_pfn_mkwrite(vmf, pfn, order);
2183}
2184EXPORT_SYMBOL_GPL(dax_finish_sync_fault);
2185
2186static int dax_range_compare_iter(struct iomap_iter *it_src,
2187 struct iomap_iter *it_dest, u64 len, bool *same)
2188{
2189 const struct iomap *smap = &it_src->iomap;
2190 const struct iomap *dmap = &it_dest->iomap;
2191 loff_t pos1 = it_src->pos, pos2 = it_dest->pos;
2192 void *saddr, *daddr;
2193 int id, ret;
2194
2195 len = min(len, min(smap->length, dmap->length));
2196
2197 if (smap->type == IOMAP_HOLE && dmap->type == IOMAP_HOLE) {
2198 *same = true;
2199 goto advance;
2200 }
2201
2202 if (smap->type == IOMAP_HOLE || dmap->type == IOMAP_HOLE) {
2203 *same = false;
2204 return 0;
2205 }
2206
2207 id = dax_read_lock();
2208 ret = dax_iomap_direct_access(smap, pos1, ALIGN(pos1 + len, PAGE_SIZE),
2209 &saddr, NULL);
2210 if (ret < 0)
2211 goto out_unlock;
2212
2213 ret = dax_iomap_direct_access(dmap, pos2, ALIGN(pos2 + len, PAGE_SIZE),
2214 &daddr, NULL);
2215 if (ret < 0)
2216 goto out_unlock;
2217
2218 *same = !memcmp(saddr, daddr, len);
2219 if (!*same)
2220 len = 0;
2221 dax_read_unlock(id);
2222
2223advance:
2224 ret = iomap_iter_advance(it_src, len);
2225 if (!ret)
2226 ret = iomap_iter_advance(it_dest, len);
2227 return ret;
2228
2229out_unlock:
2230 dax_read_unlock(id);
2231 return -EIO;
2232}
2233
2234int dax_dedupe_file_range_compare(struct inode *src, loff_t srcoff,
2235 struct inode *dst, loff_t dstoff, loff_t len, bool *same,
2236 const struct iomap_ops *ops)
2237{
2238 struct iomap_iter src_iter = {
2239 .inode = src,
2240 .pos = srcoff,
2241 .len = len,
2242 .flags = IOMAP_DAX,
2243 };
2244 struct iomap_iter dst_iter = {
2245 .inode = dst,
2246 .pos = dstoff,
2247 .len = len,
2248 .flags = IOMAP_DAX,
2249 };
2250 int ret, status;
2251
2252 while ((ret = iomap_iter(&src_iter, ops)) > 0 &&
2253 (ret = iomap_iter(&dst_iter, ops)) > 0) {
2254 status = dax_range_compare_iter(&src_iter, &dst_iter,
2255 min(src_iter.len, dst_iter.len), same);
2256 if (status < 0)
2257 return ret;
2258 src_iter.status = dst_iter.status = status;
2259 }
2260 return ret;
2261}
2262
2263int dax_remap_file_range_prep(struct file *file_in, loff_t pos_in,
2264 struct file *file_out, loff_t pos_out,
2265 loff_t *len, unsigned int remap_flags,
2266 const struct iomap_ops *ops)
2267{
2268 return __generic_remap_file_range_prep(file_in, pos_in, file_out,
2269 pos_out, len, remap_flags, ops);
2270}
2271EXPORT_SYMBOL_GPL(dax_remap_file_range_prep);