projects/linux·blob master · refs

1923 lines · 50.5 KiB

1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Copyright (C) 2018 HUAWEI, Inc.
4 * https://www.huawei.com/
5 * Copyright (C) 2022 Alibaba Cloud
6 */
7#include "compress.h"
8#include <linux/psi.h>
9#include <linux/cpuhotplug.h>
10#include <trace/events/erofs.h>
11
12#define Z_EROFS_MAX_SYNC_DECOMPRESS_BYTES 12288
13#define Z_EROFS_PCLUSTER_MAX_PAGES (Z_EROFS_PCLUSTER_MAX_SIZE / PAGE_SIZE)
14#define Z_EROFS_INLINE_BVECS 2
15
16struct z_erofs_bvec {
17 struct page *page;
18 int offset;
19 unsigned int end;
20};
21
22#define __Z_EROFS_BVSET(name, total) \
23struct name { \
24 /* point to the next page which contains the following bvecs */ \
25 struct page *nextpage; \
26 struct z_erofs_bvec bvec[total]; \
27}
28__Z_EROFS_BVSET(z_erofs_bvset,);
29__Z_EROFS_BVSET(z_erofs_bvset_inline, Z_EROFS_INLINE_BVECS);
30
31/*
32 * Structure fields follow one of the following exclusion rules.
33 *
34 * I: Modifiable by initialization/destruction paths and read-only
35 * for everyone else;
36 *
37 * L: Field should be protected by the pcluster lock;
38 *
39 * A: Field should be accessed / updated in atomic for parallelized code.
40 */
41struct z_erofs_pcluster {
42 struct mutex lock;
43 struct lockref lockref;
44
45 /* A: point to next chained pcluster or TAILs */
46 struct z_erofs_pcluster *next;
47
48 /* I: start physical position of this pcluster */
49 erofs_off_t pos;
50
51 /* L: the maximum decompression size of this round */
52 unsigned int length;
53
54 /* L: total number of bvecs */
55 unsigned int vcnt;
56
57 /* I: pcluster size (compressed size) in bytes */
58 unsigned int pclustersize;
59
60 /* I: page offset of start position of decompression */
61 unsigned short pageofs_out;
62
63 /* I: page offset of inline compressed data */
64 unsigned short pageofs_in;
65
66 union {
67 /* L: inline a certain number of bvec for bootstrap */
68 struct z_erofs_bvset_inline bvset;
69
70 /* I: can be used to free the pcluster by RCU. */
71 struct rcu_head rcu;
72 };
73
74 /* I: compression algorithm format */
75 unsigned char algorithmformat;
76
77 /* I: whether compressed data is in-lined or not */
78 bool from_meta;
79
80 /* L: whether partial decompression or not */
81 bool partial;
82
83 /* L: whether extra buffer allocations are best-effort */
84 bool besteffort;
85
86 /* A: compressed bvecs (can be cached or inplaced pages) */
87 struct z_erofs_bvec compressed_bvecs[];
88};
89
90/* the end of a chain of pclusters */
91#define Z_EROFS_PCLUSTER_TAIL ((void *) 0x700 + POISON_POINTER_DELTA)
92
93struct z_erofs_decompressqueue {
94 struct super_block *sb;
95 struct z_erofs_pcluster *head;
96 atomic_t pending_bios;
97
98 union {
99 struct completion done;
100 struct work_struct work;
101 struct kthread_work kthread_work;
102 } u;
103 bool eio, sync;
104};
105
106static inline unsigned int z_erofs_pclusterpages(struct z_erofs_pcluster *pcl)
107{
108 return PAGE_ALIGN(pcl->pageofs_in + pcl->pclustersize) >> PAGE_SHIFT;
109}
110
111static bool erofs_folio_is_managed(struct erofs_sb_info *sbi, struct folio *fo)
112{
113 return fo->mapping == MNGD_MAPPING(sbi);
114}
115
116#define Z_EROFS_ONSTACK_PAGES 32
117
118/*
119 * since pclustersize is variable for big pcluster feature, introduce slab
120 * pools implementation for different pcluster sizes.
121 */
122struct z_erofs_pcluster_slab {
123 struct kmem_cache *slab;
124 unsigned int maxpages;
125 char name[48];
126};
127
128#define _PCLP(n) { .maxpages = n }
129
130static struct z_erofs_pcluster_slab pcluster_pool[] __read_mostly = {
131 _PCLP(1),
132 _PCLP(4),
133#if Z_EROFS_PCLUSTER_MAX_PAGES > 16
134 _PCLP(16),
135#endif
136#if Z_EROFS_PCLUSTER_MAX_PAGES > 64
137 _PCLP(64),
138#endif
139#if Z_EROFS_PCLUSTER_MAX_PAGES > 128
140 _PCLP(128),
141#endif
142 _PCLP(Z_EROFS_PCLUSTER_MAX_PAGES + 1)
143};
144
145struct z_erofs_bvec_iter {
146 struct page *bvpage;
147 struct z_erofs_bvset *bvset;
148 unsigned int nr, cur;
149};
150
151static struct page *z_erofs_bvec_iter_end(struct z_erofs_bvec_iter *iter)
152{
153 if (iter->bvpage)
154 kunmap_local(iter->bvset);
155 return iter->bvpage;
156}
157
158static struct page *z_erofs_bvset_flip(struct z_erofs_bvec_iter *iter)
159{
160 unsigned long base = (unsigned long)((struct z_erofs_bvset *)0)->bvec;
161 /* have to access nextpage in advance, otherwise it will be unmapped */
162 struct page *nextpage = iter->bvset->nextpage;
163 struct page *oldpage;
164
165 DBG_BUGON(!nextpage);
166 oldpage = z_erofs_bvec_iter_end(iter);
167 iter->bvpage = nextpage;
168 iter->bvset = kmap_local_page(nextpage);
169 iter->nr = (PAGE_SIZE - base) / sizeof(struct z_erofs_bvec);
170 iter->cur = 0;
171 return oldpage;
172}
173
174static void z_erofs_bvec_iter_begin(struct z_erofs_bvec_iter *iter,
175 struct z_erofs_bvset_inline *bvset,
176 unsigned int bootstrap_nr,
177 unsigned int cur)
178{
179 *iter = (struct z_erofs_bvec_iter) {
180 .nr = bootstrap_nr,
181 .bvset = (struct z_erofs_bvset *)bvset,
182 };
183
184 while (cur > iter->nr) {
185 cur -= iter->nr;
186 z_erofs_bvset_flip(iter);
187 }
188 iter->cur = cur;
189}
190
191static int z_erofs_bvec_enqueue(struct z_erofs_bvec_iter *iter,
192 struct z_erofs_bvec *bvec,
193 struct page **candidate_bvpage,
194 struct page **pagepool)
195{
196 if (iter->cur >= iter->nr) {
197 struct page *nextpage = *candidate_bvpage;
198
199 if (!nextpage) {
200 nextpage = __erofs_allocpage(pagepool, GFP_KERNEL,
201 true);
202 if (!nextpage)
203 return -ENOMEM;
204 set_page_private(nextpage, Z_EROFS_SHORTLIVED_PAGE);
205 }
206 DBG_BUGON(iter->bvset->nextpage);
207 iter->bvset->nextpage = nextpage;
208 z_erofs_bvset_flip(iter);
209
210 iter->bvset->nextpage = NULL;
211 *candidate_bvpage = NULL;
212 }
213 iter->bvset->bvec[iter->cur++] = *bvec;
214 return 0;
215}
216
217static void z_erofs_bvec_dequeue(struct z_erofs_bvec_iter *iter,
218 struct z_erofs_bvec *bvec,
219 struct page **old_bvpage)
220{
221 if (iter->cur == iter->nr)
222 *old_bvpage = z_erofs_bvset_flip(iter);
223 else
224 *old_bvpage = NULL;
225 *bvec = iter->bvset->bvec[iter->cur++];
226}
227
228static void z_erofs_destroy_pcluster_pool(void)
229{
230 int i;
231
232 for (i = 0; i < ARRAY_SIZE(pcluster_pool); ++i) {
233 if (!pcluster_pool[i].slab)
234 continue;
235 kmem_cache_destroy(pcluster_pool[i].slab);
236 pcluster_pool[i].slab = NULL;
237 }
238}
239
240static int z_erofs_create_pcluster_pool(void)
241{
242 struct z_erofs_pcluster_slab *pcs;
243 struct z_erofs_pcluster *a;
244 unsigned int size;
245
246 for (pcs = pcluster_pool;
247 pcs < pcluster_pool + ARRAY_SIZE(pcluster_pool); ++pcs) {
248 size = struct_size(a, compressed_bvecs, pcs->maxpages);
249
250 sprintf(pcs->name, "erofs_pcluster-%u", pcs->maxpages);
251 pcs->slab = kmem_cache_create(pcs->name, size, 0,
252 SLAB_RECLAIM_ACCOUNT, NULL);
253 if (pcs->slab)
254 continue;
255
256 z_erofs_destroy_pcluster_pool();
257 return -ENOMEM;
258 }
259 return 0;
260}
261
262static struct z_erofs_pcluster *z_erofs_alloc_pcluster(unsigned int size)
263{
264 unsigned int nrpages = PAGE_ALIGN(size) >> PAGE_SHIFT;
265 struct z_erofs_pcluster_slab *pcs = pcluster_pool;
266
267 for (; pcs < pcluster_pool + ARRAY_SIZE(pcluster_pool); ++pcs) {
268 struct z_erofs_pcluster *pcl;
269
270 if (nrpages > pcs->maxpages)
271 continue;
272
273 pcl = kmem_cache_zalloc(pcs->slab, GFP_KERNEL);
274 if (!pcl)
275 return ERR_PTR(-ENOMEM);
276 return pcl;
277 }
278 return ERR_PTR(-EINVAL);
279}
280
281static void z_erofs_free_pcluster(struct z_erofs_pcluster *pcl)
282{
283 unsigned int pclusterpages = z_erofs_pclusterpages(pcl);
284 int i;
285
286 for (i = 0; i < ARRAY_SIZE(pcluster_pool); ++i) {
287 struct z_erofs_pcluster_slab *pcs = pcluster_pool + i;
288
289 if (pclusterpages > pcs->maxpages)
290 continue;
291
292 kmem_cache_free(pcs->slab, pcl);
293 return;
294 }
295 DBG_BUGON(1);
296}
297
298static struct workqueue_struct *z_erofs_workqueue __read_mostly;
299
300#ifdef CONFIG_EROFS_FS_PCPU_KTHREAD
301static struct kthread_worker __rcu **z_erofs_pcpu_workers;
302static atomic_t erofs_percpu_workers_initialized = ATOMIC_INIT(0);
303
304static void erofs_destroy_percpu_workers(void)
305{
306 struct kthread_worker *worker;
307 unsigned int cpu;
308
309 for_each_possible_cpu(cpu) {
310 worker = rcu_dereference_protected(
311 z_erofs_pcpu_workers[cpu], 1);
312 rcu_assign_pointer(z_erofs_pcpu_workers[cpu], NULL);
313 if (worker)
314 kthread_destroy_worker(worker);
315 }
316 kfree(z_erofs_pcpu_workers);
317}
318
319static struct kthread_worker *erofs_init_percpu_worker(int cpu)
320{
321 struct kthread_worker *worker =
322 kthread_run_worker_on_cpu(cpu, 0, "erofs_worker/%u");
323
324 if (IS_ERR(worker))
325 return worker;
326 if (IS_ENABLED(CONFIG_EROFS_FS_PCPU_KTHREAD_HIPRI))
327 sched_set_fifo_low(worker->task);
328 return worker;
329}
330
331static int erofs_init_percpu_workers(void)
332{
333 struct kthread_worker *worker;
334 unsigned int cpu;
335
336 z_erofs_pcpu_workers = kzalloc_objs(struct kthread_worker *,
337 num_possible_cpus(), GFP_ATOMIC);
338 if (!z_erofs_pcpu_workers)
339 return -ENOMEM;
340
341 for_each_online_cpu(cpu) { /* could miss cpu{off,on}line? */
342 worker = erofs_init_percpu_worker(cpu);
343 if (!IS_ERR(worker))
344 rcu_assign_pointer(z_erofs_pcpu_workers[cpu], worker);
345 }
346 return 0;
347}
348
349#ifdef CONFIG_HOTPLUG_CPU
350static DEFINE_SPINLOCK(z_erofs_pcpu_worker_lock);
351static enum cpuhp_state erofs_cpuhp_state;
352
353static int erofs_cpu_online(unsigned int cpu)
354{
355 struct kthread_worker *worker, *old;
356
357 worker = erofs_init_percpu_worker(cpu);
358 if (IS_ERR(worker))
359 return PTR_ERR(worker);
360
361 spin_lock(&z_erofs_pcpu_worker_lock);
362 old = rcu_dereference_protected(z_erofs_pcpu_workers[cpu],
363 lockdep_is_held(&z_erofs_pcpu_worker_lock));
364 if (!old)
365 rcu_assign_pointer(z_erofs_pcpu_workers[cpu], worker);
366 spin_unlock(&z_erofs_pcpu_worker_lock);
367 if (old)
368 kthread_destroy_worker(worker);
369 return 0;
370}
371
372static int erofs_cpu_offline(unsigned int cpu)
373{
374 struct kthread_worker *worker;
375
376 spin_lock(&z_erofs_pcpu_worker_lock);
377 worker = rcu_dereference_protected(z_erofs_pcpu_workers[cpu],
378 lockdep_is_held(&z_erofs_pcpu_worker_lock));
379 rcu_assign_pointer(z_erofs_pcpu_workers[cpu], NULL);
380 spin_unlock(&z_erofs_pcpu_worker_lock);
381
382 synchronize_rcu();
383 if (worker)
384 kthread_destroy_worker(worker);
385 return 0;
386}
387
388static int erofs_cpu_hotplug_init(void)
389{
390 int state;
391
392 state = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN,
393 "fs/erofs:online", erofs_cpu_online, erofs_cpu_offline);
394 if (state < 0)
395 return state;
396
397 erofs_cpuhp_state = state;
398 return 0;
399}
400
401static void erofs_cpu_hotplug_destroy(void)
402{
403 if (erofs_cpuhp_state)
404 cpuhp_remove_state_nocalls(erofs_cpuhp_state);
405}
406#else /* !CONFIG_HOTPLUG_CPU */
407static inline int erofs_cpu_hotplug_init(void) { return 0; }
408static inline void erofs_cpu_hotplug_destroy(void) {}
409#endif/* CONFIG_HOTPLUG_CPU */
410static int z_erofs_init_pcpu_workers(struct super_block *sb)
411{
412 int err;
413
414 if (atomic_xchg(&erofs_percpu_workers_initialized, 1))
415 return 0;
416
417 err = erofs_init_percpu_workers();
418 if (err) {
419 erofs_err(sb, "per-cpu workers: failed to allocate.");
420 goto err_init_percpu_workers;
421 }
422
423 err = erofs_cpu_hotplug_init();
424 if (err < 0) {
425 erofs_err(sb, "per-cpu workers: failed CPU hotplug init.");
426 goto err_cpuhp_init;
427 }
428 erofs_info(sb, "initialized per-cpu workers successfully.");
429 return err;
430
431err_cpuhp_init:
432 erofs_destroy_percpu_workers();
433err_init_percpu_workers:
434 atomic_set(&erofs_percpu_workers_initialized, 0);
435 return err;
436}
437
438static void z_erofs_destroy_pcpu_workers(void)
439{
440 if (!atomic_xchg(&erofs_percpu_workers_initialized, 0))
441 return;
442 erofs_cpu_hotplug_destroy();
443 erofs_destroy_percpu_workers();
444}
445#else /* !CONFIG_EROFS_FS_PCPU_KTHREAD */
446static inline int z_erofs_init_pcpu_workers(struct super_block *sb) { return 0; }
447static inline void z_erofs_destroy_pcpu_workers(void) {}
448#endif/* CONFIG_EROFS_FS_PCPU_KTHREAD */
449
450void z_erofs_exit_subsystem(void)
451{
452 z_erofs_destroy_pcpu_workers();
453 destroy_workqueue(z_erofs_workqueue);
454 z_erofs_destroy_pcluster_pool();
455 z_erofs_crypto_disable_all_engines();
456 z_erofs_exit_decompressor();
457}
458
459int __init z_erofs_init_subsystem(void)
460{
461 int err = z_erofs_init_decompressor();
462
463 if (err)
464 goto err_decompressor;
465
466 err = z_erofs_create_pcluster_pool();
467 if (err)
468 goto err_pcluster_pool;
469
470 z_erofs_workqueue = alloc_workqueue("erofs_worker",
471 WQ_UNBOUND | WQ_HIGHPRI, num_possible_cpus());
472 if (!z_erofs_workqueue) {
473 err = -ENOMEM;
474 goto err_workqueue_init;
475 }
476
477 return err;
478
479err_workqueue_init:
480 z_erofs_destroy_pcluster_pool();
481err_pcluster_pool:
482 z_erofs_exit_decompressor();
483err_decompressor:
484 return err;
485}
486
487enum z_erofs_pclustermode {
488 /* It has previously been linked into another processing chain */
489 Z_EROFS_PCLUSTER_INFLIGHT,
490 /*
491 * A weaker form of Z_EROFS_PCLUSTER_FOLLOWED; the difference is that it
492 * may be dispatched to the bypass queue later due to uptodated managed
493 * folios. All file-backed folios related to this pcluster cannot be
494 * reused for in-place I/O (or bvpage) since the pcluster may be decoded
495 * in a separate queue (and thus out of order).
496 */
497 Z_EROFS_PCLUSTER_FOLLOWED_NOINPLACE,
498 /*
499 * The pcluster has just been linked to our processing chain.
500 * File-backed folios (except for the head page) related to it can be
501 * used for in-place I/O (or bvpage).
502 */
503 Z_EROFS_PCLUSTER_FOLLOWED,
504};
505
506struct z_erofs_frontend {
507 struct inode *inode, *sharedinode;
508 struct erofs_map_blocks map;
509 struct z_erofs_bvec_iter biter;
510
511 struct page *pagepool;
512 struct page *candidate_bvpage;
513 struct z_erofs_pcluster *pcl, *head;
514 enum z_erofs_pclustermode mode;
515
516 erofs_off_t headoffset;
517
518 /* a pointer used to pick up inplace I/O pages */
519 unsigned int icur;
520};
521
522#define Z_EROFS_DEFINE_FRONTEND(fe, i, si, ho) struct z_erofs_frontend fe = { \
523 .inode = i, .sharedinode = si, .head = Z_EROFS_PCLUSTER_TAIL, \
524 .mode = Z_EROFS_PCLUSTER_FOLLOWED, .headoffset = ho }
525
526static bool z_erofs_should_alloc_cache(struct z_erofs_frontend *fe)
527{
528 unsigned int cachestrategy = EROFS_I_SB(fe->inode)->opt.cache_strategy;
529
530 if (cachestrategy <= EROFS_ZIP_CACHE_DISABLED)
531 return false;
532
533 if (fe->map.m_flags & EROFS_MAP_PARTIAL_MAPPED)
534 return true;
535
536 if (cachestrategy >= EROFS_ZIP_CACHE_READAROUND &&
537 fe->map.m_la < fe->headoffset)
538 return true;
539
540 return false;
541}
542
543static void z_erofs_bind_cache(struct z_erofs_frontend *fe)
544{
545 struct address_space *mc = MNGD_MAPPING(EROFS_I_SB(fe->inode));
546 struct z_erofs_pcluster *pcl = fe->pcl;
547 unsigned int pclusterpages = z_erofs_pclusterpages(pcl);
548 bool shouldalloc = z_erofs_should_alloc_cache(fe);
549 pgoff_t poff = pcl->pos >> PAGE_SHIFT;
550 bool may_bypass = true;
551 /* Optimistic allocation, as in-place I/O can be used as a fallback */
552 gfp_t gfp = (mapping_gfp_mask(mc) & ~__GFP_DIRECT_RECLAIM) |
553 __GFP_NOMEMALLOC | __GFP_NORETRY | __GFP_NOWARN;
554 struct folio *folio, *newfolio;
555 unsigned int i;
556
557 if (i_blocksize(fe->inode) != PAGE_SIZE ||
558 fe->mode < Z_EROFS_PCLUSTER_FOLLOWED)
559 return;
560
561 for (i = 0; i < pclusterpages; ++i) {
562 /* Inaccurate check w/o locking to avoid unneeded lookups */
563 if (READ_ONCE(pcl->compressed_bvecs[i].page))
564 continue;
565
566 folio = filemap_get_folio(mc, poff + i);
567 if (IS_ERR(folio)) {
568 may_bypass = false;
569 if (!shouldalloc)
570 continue;
571
572 /*
573 * Allocate a managed folio for cached I/O, or it may be
574 * then filled with a file-backed folio for in-place I/O
575 */
576 newfolio = filemap_alloc_folio(gfp, 0, NULL);
577 if (!newfolio)
578 continue;
579 newfolio->private = Z_EROFS_PREALLOCATED_FOLIO;
580 folio = NULL;
581 }
582 spin_lock(&pcl->lockref.lock);
583 if (!pcl->compressed_bvecs[i].page) {
584 pcl->compressed_bvecs[i].page =
585 folio_page(folio ?: newfolio, 0);
586 spin_unlock(&pcl->lockref.lock);
587 continue;
588 }
589 spin_unlock(&pcl->lockref.lock);
590 folio_put(folio ?: newfolio);
591 }
592
593 /*
594 * Don't perform in-place I/O if all compressed pages are available in
595 * the managed cache, as the pcluster can be moved to the bypass queue.
596 */
597 if (may_bypass)
598 fe->mode = Z_EROFS_PCLUSTER_FOLLOWED_NOINPLACE;
599}
600
601/* (erofs_shrinker) disconnect cached encoded data with pclusters */
602static int erofs_try_to_free_all_cached_folios(struct erofs_sb_info *sbi,
603 struct z_erofs_pcluster *pcl)
604{
605 unsigned int pclusterpages = z_erofs_pclusterpages(pcl);
606 struct folio *folio;
607 int i;
608
609 DBG_BUGON(pcl->from_meta);
610 /* Each cached folio contains one page unless bs > ps is supported */
611 for (i = 0; i < pclusterpages; ++i) {
612 if (pcl->compressed_bvecs[i].page) {
613 folio = page_folio(pcl->compressed_bvecs[i].page);
614 /* Avoid reclaiming or migrating this folio */
615 if (!folio_trylock(folio))
616 return -EBUSY;
617
618 DBG_BUGON(!erofs_folio_is_managed(sbi, folio));
619 pcl->compressed_bvecs[i].page = NULL;
620 folio_detach_private(folio);
621 folio_unlock(folio);
622 }
623 }
624 return 0;
625}
626
627static bool z_erofs_cache_release_folio(struct folio *folio, gfp_t gfp)
628{
629 struct z_erofs_pcluster *pcl = folio_get_private(folio);
630 struct z_erofs_bvec *bvec = pcl->compressed_bvecs;
631 struct z_erofs_bvec *end = bvec + z_erofs_pclusterpages(pcl);
632 bool ret;
633
634 if (!folio_test_private(folio))
635 return true;
636
637 ret = false;
638 spin_lock(&pcl->lockref.lock);
639 if (pcl->lockref.count <= 0) {
640 DBG_BUGON(pcl->from_meta);
641 for (; bvec < end; ++bvec) {
642 if (bvec->page && page_folio(bvec->page) == folio) {
643 bvec->page = NULL;
644 folio_detach_private(folio);
645 ret = true;
646 break;
647 }
648 }
649 }
650 spin_unlock(&pcl->lockref.lock);
651 return ret;
652}
653
654/*
655 * It will be called only on inode eviction. In case that there are still some
656 * decompression requests in progress, wait with rescheduling for a bit here.
657 * An extra lock could be introduced instead but it seems unnecessary.
658 */
659static void z_erofs_cache_invalidate_folio(struct folio *folio,
660 size_t offset, size_t length)
661{
662 const size_t stop = length + offset;
663
664 /* Check for potential overflow in debug mode */
665 DBG_BUGON(stop > folio_size(folio) || stop < length);
666
667 if (offset == 0 && stop == folio_size(folio))
668 while (!z_erofs_cache_release_folio(folio, 0))
669 cond_resched();
670}
671
672static const struct address_space_operations z_erofs_cache_aops = {
673 .release_folio = z_erofs_cache_release_folio,
674 .invalidate_folio = z_erofs_cache_invalidate_folio,
675};
676
677int z_erofs_init_super(struct super_block *sb)
678{
679 int err;
680
681 err = z_erofs_init_pcpu_workers(sb);
682 if (err)
683 return err;
684
685 err = erofs_setup_managed_cache(sb);
686 if (err)
687 return err;
688 EROFS_SB(sb)->managed_cache->i_mapping->a_ops = &z_erofs_cache_aops;
689 xa_init(&EROFS_SB(sb)->managed_pslots);
690 return 0;
691}
692
693/* callers must be with pcluster lock held */
694static int z_erofs_attach_page(struct z_erofs_frontend *fe,
695 struct z_erofs_bvec *bvec, bool exclusive)
696{
697 struct z_erofs_pcluster *pcl = fe->pcl;
698 int ret;
699
700 if (exclusive) {
701 /* Inplace I/O is limited to one page for uncompressed data */
702 if (pcl->algorithmformat < Z_EROFS_COMPRESSION_MAX ||
703 fe->icur <= 1) {
704 /* Try to prioritize inplace I/O here */
705 spin_lock(&pcl->lockref.lock);
706 while (fe->icur > 0) {
707 if (pcl->compressed_bvecs[--fe->icur].page)
708 continue;
709 pcl->compressed_bvecs[fe->icur] = *bvec;
710 spin_unlock(&pcl->lockref.lock);
711 return 0;
712 }
713 spin_unlock(&pcl->lockref.lock);
714 }
715
716 /* otherwise, check if it can be used as a bvpage */
717 if (fe->mode >= Z_EROFS_PCLUSTER_FOLLOWED &&
718 !fe->candidate_bvpage)
719 fe->candidate_bvpage = bvec->page;
720 }
721 ret = z_erofs_bvec_enqueue(&fe->biter, bvec, &fe->candidate_bvpage,
722 &fe->pagepool);
723 fe->pcl->vcnt += (ret >= 0);
724 return ret;
725}
726
727static bool z_erofs_get_pcluster(struct z_erofs_pcluster *pcl)
728{
729 if (lockref_get_not_zero(&pcl->lockref))
730 return true;
731
732 spin_lock(&pcl->lockref.lock);
733 if (lockref_is_dead(&pcl->lockref)) {
734 spin_unlock(&pcl->lockref.lock);
735 return false;
736 }
737
738 if (!pcl->lockref.count++)
739 atomic_long_dec(&erofs_global_shrink_cnt);
740 spin_unlock(&pcl->lockref.lock);
741 return true;
742}
743
744static int z_erofs_register_pcluster(struct z_erofs_frontend *fe)
745{
746 struct erofs_map_blocks *map = &fe->map;
747 struct super_block *sb = fe->inode->i_sb;
748 struct erofs_sb_info *sbi = EROFS_SB(sb);
749 struct z_erofs_pcluster *pcl, *pre;
750 unsigned int pageofs_in;
751 int err;
752
753 pageofs_in = erofs_blkoff(sb, map->m_pa);
754 pcl = z_erofs_alloc_pcluster(pageofs_in + map->m_plen);
755 if (IS_ERR(pcl))
756 return PTR_ERR(pcl);
757
758 lockref_init(&pcl->lockref); /* one ref for this request */
759 pcl->algorithmformat = map->m_algorithmformat;
760 pcl->pclustersize = map->m_plen;
761 pcl->length = 0;
762 pcl->partial = true;
763 pcl->next = fe->head;
764 pcl->pos = map->m_pa;
765 pcl->pageofs_in = pageofs_in;
766 pcl->pageofs_out = map->m_la & ~PAGE_MASK;
767 pcl->from_meta = map->m_flags & EROFS_MAP_META;
768 fe->mode = Z_EROFS_PCLUSTER_FOLLOWED;
769
770 /*
771 * lock all primary followed works before visible to others
772 * and mutex_trylock *never* fails for a new pcluster.
773 */
774 mutex_init(&pcl->lock);
775 DBG_BUGON(!mutex_trylock(&pcl->lock));
776
777 if (!pcl->from_meta) {
778 while (1) {
779 xa_lock(&sbi->managed_pslots);
780 pre = __xa_cmpxchg(&sbi->managed_pslots, pcl->pos,
781 NULL, pcl, GFP_KERNEL);
782 if (!pre || xa_is_err(pre) || z_erofs_get_pcluster(pre)) {
783 xa_unlock(&sbi->managed_pslots);
784 break;
785 }
786 /* try to legitimize the current in-tree one */
787 xa_unlock(&sbi->managed_pslots);
788 cond_resched();
789 }
790 if (xa_is_err(pre)) {
791 err = xa_err(pre);
792 goto err_out;
793 } else if (pre) {
794 fe->pcl = pre;
795 err = -EEXIST;
796 goto err_out;
797 }
798 }
799 fe->head = fe->pcl = pcl;
800 return 0;
801
802err_out:
803 mutex_unlock(&pcl->lock);
804 z_erofs_free_pcluster(pcl);
805 return err;
806}
807
808static int z_erofs_pcluster_begin(struct z_erofs_frontend *fe)
809{
810 struct erofs_map_blocks *map = &fe->map;
811 struct super_block *sb = fe->inode->i_sb;
812 struct z_erofs_pcluster *pcl = NULL;
813 void *ptr = NULL;
814 bool needretry;
815 int ret;
816
817 DBG_BUGON(fe->pcl);
818 /* must be Z_EROFS_PCLUSTER_TAIL or pointed to previous pcluster */
819 DBG_BUGON(!fe->head);
820
821 if (map->m_flags & EROFS_MAP_META) {
822 ret = erofs_init_metabuf(&map->buf, sb,
823 erofs_inode_in_metabox(fe->inode));
824 if (ret)
825 return ret;
826 ptr = erofs_bread(&map->buf, map->m_pa, false);
827 if (IS_ERR(ptr)) {
828 erofs_err(sb, "failed to read inline data %pe @ pa %llu of nid %llu",
829 ptr, map->m_pa, EROFS_I(fe->inode)->nid);
830 return PTR_ERR(ptr);
831 }
832 ptr = map->buf.page;
833 } else {
834 do {
835 rcu_read_lock();
836 pcl = xa_load(&EROFS_SB(sb)->managed_pslots, map->m_pa);
837 needretry = pcl && !z_erofs_get_pcluster(pcl);
838 rcu_read_unlock();
839 } while (needretry);
840 }
841
842 if (pcl) {
843 DBG_BUGON(map->m_pa != pcl->pos);
844 fe->pcl = pcl;
845 ret = -EEXIST;
846 } else {
847 ret = z_erofs_register_pcluster(fe);
848 }
849
850 if (ret == -EEXIST) {
851 mutex_lock(&fe->pcl->lock);
852 /* check if this pcluster hasn't been linked into any chain. */
853 if (!cmpxchg(&fe->pcl->next, NULL, fe->head)) {
854 /* .. so it can be attached to our submission chain */
855 fe->head = fe->pcl;
856 fe->mode = Z_EROFS_PCLUSTER_FOLLOWED;
857 } else { /* otherwise, it belongs to an inflight chain */
858 fe->mode = Z_EROFS_PCLUSTER_INFLIGHT;
859 }
860 } else if (ret) {
861 return ret;
862 }
863
864 z_erofs_bvec_iter_begin(&fe->biter, &fe->pcl->bvset,
865 Z_EROFS_INLINE_BVECS, fe->pcl->vcnt);
866 if (!fe->pcl->from_meta) {
867 /* bind cache first when cached decompression is preferred */
868 z_erofs_bind_cache(fe);
869 } else {
870 folio_get(page_folio((struct page *)ptr));
871 WRITE_ONCE(fe->pcl->compressed_bvecs[0].page, ptr);
872 fe->pcl->pageofs_in = map->m_pa & ~PAGE_MASK;
873 fe->mode = Z_EROFS_PCLUSTER_FOLLOWED_NOINPLACE;
874 }
875 /* file-backed inplace I/O pages are traversed in reverse order */
876 fe->icur = z_erofs_pclusterpages(fe->pcl);
877 return 0;
878}
879
880static void z_erofs_rcu_callback(struct rcu_head *head)
881{
882 z_erofs_free_pcluster(container_of(head, struct z_erofs_pcluster, rcu));
883}
884
885static bool __erofs_try_to_release_pcluster(struct erofs_sb_info *sbi,
886 struct z_erofs_pcluster *pcl)
887{
888 if (pcl->lockref.count)
889 return false;
890
891 /*
892 * Note that all cached folios should be detached before deleted from
893 * the XArray. Otherwise some folios could be still attached to the
894 * orphan old pcluster when the new one is available in the tree.
895 */
896 if (erofs_try_to_free_all_cached_folios(sbi, pcl))
897 return false;
898
899 /*
900 * It's impossible to fail after the pcluster is freezed, but in order
901 * to avoid some race conditions, add a DBG_BUGON to observe this.
902 */
903 DBG_BUGON(__xa_erase(&sbi->managed_pslots, pcl->pos) != pcl);
904
905 lockref_mark_dead(&pcl->lockref);
906 return true;
907}
908
909static bool erofs_try_to_release_pcluster(struct erofs_sb_info *sbi,
910 struct z_erofs_pcluster *pcl)
911{
912 bool free;
913
914 spin_lock(&pcl->lockref.lock);
915 free = __erofs_try_to_release_pcluster(sbi, pcl);
916 spin_unlock(&pcl->lockref.lock);
917 if (free) {
918 atomic_long_dec(&erofs_global_shrink_cnt);
919 call_rcu(&pcl->rcu, z_erofs_rcu_callback);
920 }
921 return free;
922}
923
924unsigned long z_erofs_shrink_scan(struct erofs_sb_info *sbi, unsigned long nr)
925{
926 struct z_erofs_pcluster *pcl;
927 unsigned long index, freed = 0;
928
929 xa_lock(&sbi->managed_pslots);
930 xa_for_each(&sbi->managed_pslots, index, pcl) {
931 /* try to shrink each valid pcluster */
932 if (!erofs_try_to_release_pcluster(sbi, pcl))
933 continue;
934 xa_unlock(&sbi->managed_pslots);
935
936 ++freed;
937 if (!--nr)
938 return freed;
939 xa_lock(&sbi->managed_pslots);
940 }
941 xa_unlock(&sbi->managed_pslots);
942 return freed;
943}
944
945static void z_erofs_put_pcluster(struct erofs_sb_info *sbi,
946 struct z_erofs_pcluster *pcl, bool try_free)
947{
948 bool free = false;
949
950 if (lockref_put_or_lock(&pcl->lockref))
951 return;
952
953 DBG_BUGON(lockref_is_dead(&pcl->lockref));
954 if (!--pcl->lockref.count) {
955 if (try_free && xa_trylock(&sbi->managed_pslots)) {
956 free = __erofs_try_to_release_pcluster(sbi, pcl);
957 xa_unlock(&sbi->managed_pslots);
958 }
959 atomic_long_add(!free, &erofs_global_shrink_cnt);
960 }
961 spin_unlock(&pcl->lockref.lock);
962 if (free)
963 call_rcu(&pcl->rcu, z_erofs_rcu_callback);
964}
965
966static void z_erofs_pcluster_end(struct z_erofs_frontend *fe)
967{
968 struct z_erofs_pcluster *pcl = fe->pcl;
969
970 if (!pcl)
971 return;
972
973 z_erofs_bvec_iter_end(&fe->biter);
974 mutex_unlock(&pcl->lock);
975
976 if (fe->candidate_bvpage)
977 fe->candidate_bvpage = NULL;
978
979 /* Drop refcount if it doesn't belong to our processing chain */
980 if (fe->mode < Z_EROFS_PCLUSTER_FOLLOWED_NOINPLACE)
981 z_erofs_put_pcluster(EROFS_I_SB(fe->inode), pcl, false);
982 fe->pcl = NULL;
983}
984
985static int z_erofs_read_fragment(struct super_block *sb, struct folio *folio,
986 unsigned int cur, unsigned int end, erofs_off_t pos)
987{
988 struct inode *packed_inode = EROFS_SB(sb)->packed_inode;
989 struct erofs_buf buf = __EROFS_BUF_INITIALIZER;
990 unsigned int cnt;
991 u8 *src;
992
993 if (!packed_inode)
994 return -EFSCORRUPTED;
995
996 buf.mapping = packed_inode->i_mapping;
997 for (; cur < end; cur += cnt, pos += cnt) {
998 cnt = min(end - cur, sb->s_blocksize - erofs_blkoff(sb, pos));
999 src = erofs_bread(&buf, pos, true);
1000 if (IS_ERR(src)) {
1001 erofs_put_metabuf(&buf);
1002 return PTR_ERR(src);
1003 }
1004 memcpy_to_folio(folio, cur, src, cnt);
1005 }
1006 erofs_put_metabuf(&buf);
1007 return 0;
1008}
1009
1010static int z_erofs_scan_folio(struct z_erofs_frontend *f,
1011 struct folio *folio, bool ra)
1012{
1013 struct inode *const inode = f->inode;
1014 struct erofs_map_blocks *const map = &f->map;
1015 const loff_t offset = folio_pos(folio);
1016 const unsigned int bs = i_blocksize(inode);
1017 unsigned int end = folio_size(folio), split = 0, cur, pgs;
1018 bool tight, excl;
1019 int err = 0;
1020
1021 tight = (bs == PAGE_SIZE);
1022 erofs_onlinefolio_init(folio);
1023 do {
1024 if (offset + end - 1 < map->m_la ||
1025 offset + end - 1 >= map->m_la + map->m_llen) {
1026 z_erofs_pcluster_end(f);
1027 map->m_la = offset + end - 1;
1028 map->m_llen = 0;
1029 err = z_erofs_map_blocks_iter(inode, map, 0);
1030 if (err)
1031 break;
1032 }
1033
1034 cur = offset > map->m_la ? 0 : map->m_la - offset;
1035 pgs = round_down(cur, PAGE_SIZE);
1036 /* bump split parts first to avoid several separate cases */
1037 ++split;
1038
1039 if (map->m_flags & EROFS_MAP_FRAGMENT) {
1040 erofs_off_t fpos = offset + cur - map->m_la;
1041
1042 err = z_erofs_read_fragment(inode->i_sb, folio, cur,
1043 cur + min(map->m_llen - fpos, end - cur),
1044 EROFS_I(inode)->z_fragmentoff + fpos);
1045 if (err)
1046 break;
1047 tight = false;
1048 } else if (!(map->m_flags & EROFS_MAP_MAPPED)) {
1049 folio_zero_segment(folio, cur, end);
1050 tight = false;
1051 } else {
1052 if (!f->pcl) {
1053 err = z_erofs_pcluster_begin(f);
1054 if (err)
1055 break;
1056 f->pcl->besteffort |= !ra;
1057 }
1058
1059 pgs = round_down(end - 1, PAGE_SIZE);
1060 /*
1061 * Ensure this partial page belongs to this submit chain
1062 * rather than other concurrent submit chains or
1063 * noio(bypass) chains since those chains are handled
1064 * asynchronously thus it cannot be used for inplace I/O
1065 * or bvpage (should be processed in the strict order.)
1066 */
1067 tight &= (f->mode >= Z_EROFS_PCLUSTER_FOLLOWED);
1068 excl = false;
1069 if (cur <= pgs) {
1070 excl = (split <= 1) || tight;
1071 cur = pgs;
1072 }
1073
1074 err = z_erofs_attach_page(f, &((struct z_erofs_bvec) {
1075 .page = folio_page(folio, pgs >> PAGE_SHIFT),
1076 .offset = offset + pgs - map->m_la,
1077 .end = end - pgs, }), excl);
1078 if (err)
1079 break;
1080
1081 erofs_onlinefolio_split(folio);
1082 if (f->pcl->length < offset + end - map->m_la) {
1083 f->pcl->length = offset + end - map->m_la;
1084 f->pcl->pageofs_out = map->m_la & ~PAGE_MASK;
1085 }
1086 if (EROFS_MAP_FULL(map->m_flags) &&
1087 f->pcl->length == map->m_llen)
1088 f->pcl->partial = false;
1089 }
1090 /* shorten the remaining extent to update progress */
1091 map->m_llen = offset + cur - map->m_la;
1092 map->m_flags |= EROFS_MAP_PARTIAL_MAPPED;
1093 if (cur <= pgs) {
1094 split = cur < pgs;
1095 tight = (bs == PAGE_SIZE);
1096 }
1097 } while ((end = cur) > 0);
1098 erofs_onlinefolio_end(folio, err, false);
1099 return err;
1100}
1101
1102static bool z_erofs_page_is_invalidated(struct page *page)
1103{
1104 return !page_folio(page)->mapping && !z_erofs_is_shortlived_page(page);
1105}
1106
1107struct z_erofs_backend {
1108 struct page *onstack_pages[Z_EROFS_ONSTACK_PAGES];
1109 struct super_block *sb;
1110 struct z_erofs_pcluster *pcl;
1111 /* pages with the longest decompressed length for deduplication */
1112 struct page **decompressed_pages;
1113 /* pages to keep the compressed data */
1114 struct page **compressed_pages;
1115
1116 struct list_head decompressed_secondary_bvecs;
1117 struct page **pagepool;
1118 unsigned int onstack_used, nr_pages;
1119 /* indicate if temporary copies should be preserved for later use */
1120 bool keepxcpy;
1121};
1122
1123struct z_erofs_bvec_item {
1124 struct z_erofs_bvec bvec;
1125 struct list_head list;
1126};
1127
1128static void z_erofs_do_decompressed_bvec(struct z_erofs_backend *be,
1129 struct z_erofs_bvec *bvec)
1130{
1131 int poff = bvec->offset + be->pcl->pageofs_out;
1132 struct z_erofs_bvec_item *item;
1133 struct page **page;
1134
1135 if (!(poff & ~PAGE_MASK) && (bvec->end == PAGE_SIZE ||
1136 bvec->offset + bvec->end == be->pcl->length)) {
1137 DBG_BUGON((poff >> PAGE_SHIFT) >= be->nr_pages);
1138 page = be->decompressed_pages + (poff >> PAGE_SHIFT);
1139 if (!*page) {
1140 *page = bvec->page;
1141 return;
1142 }
1143 } else {
1144 be->keepxcpy = true;
1145 }
1146
1147 /* (cold path) one pcluster is requested multiple times */
1148 item = kmalloc_obj(*item, GFP_KERNEL | __GFP_NOFAIL);
1149 item->bvec = *bvec;
1150 list_add(&item->list, &be->decompressed_secondary_bvecs);
1151}
1152
1153static void z_erofs_fill_other_copies(struct z_erofs_backend *be, int err)
1154{
1155 unsigned int off0 = be->pcl->pageofs_out;
1156 struct list_head *p, *n;
1157
1158 list_for_each_safe(p, n, &be->decompressed_secondary_bvecs) {
1159 struct z_erofs_bvec_item *bvi;
1160 unsigned int end, cur;
1161 void *dst, *src;
1162
1163 bvi = container_of(p, struct z_erofs_bvec_item, list);
1164 cur = bvi->bvec.offset < 0 ? -bvi->bvec.offset : 0;
1165 end = min_t(unsigned int, be->pcl->length - bvi->bvec.offset,
1166 bvi->bvec.end);
1167 dst = kmap_local_page(bvi->bvec.page);
1168 while (cur < end) {
1169 unsigned int pgnr, scur, len;
1170
1171 pgnr = (bvi->bvec.offset + cur + off0) >> PAGE_SHIFT;
1172 DBG_BUGON(pgnr >= be->nr_pages);
1173
1174 scur = bvi->bvec.offset + cur -
1175 ((pgnr << PAGE_SHIFT) - off0);
1176 len = min_t(unsigned int, end - cur, PAGE_SIZE - scur);
1177 if (!be->decompressed_pages[pgnr]) {
1178 err = -EFSCORRUPTED;
1179 cur += len;
1180 continue;
1181 }
1182 src = kmap_local_page(be->decompressed_pages[pgnr]);
1183 memcpy(dst + cur, src + scur, len);
1184 kunmap_local(src);
1185 cur += len;
1186 }
1187 kunmap_local(dst);
1188 erofs_onlinefolio_end(page_folio(bvi->bvec.page), err, true);
1189 list_del(p);
1190 kfree(bvi);
1191 }
1192}
1193
1194static void z_erofs_parse_out_bvecs(struct z_erofs_backend *be)
1195{
1196 struct z_erofs_pcluster *pcl = be->pcl;
1197 struct z_erofs_bvec_iter biter;
1198 struct page *old_bvpage;
1199 int i;
1200
1201 z_erofs_bvec_iter_begin(&biter, &pcl->bvset, Z_EROFS_INLINE_BVECS, 0);
1202 for (i = 0; i < pcl->vcnt; ++i) {
1203 struct z_erofs_bvec bvec;
1204
1205 z_erofs_bvec_dequeue(&biter, &bvec, &old_bvpage);
1206
1207 if (old_bvpage)
1208 z_erofs_put_shortlivedpage(be->pagepool, old_bvpage);
1209
1210 DBG_BUGON(z_erofs_page_is_invalidated(bvec.page));
1211 z_erofs_do_decompressed_bvec(be, &bvec);
1212 }
1213
1214 old_bvpage = z_erofs_bvec_iter_end(&biter);
1215 if (old_bvpage)
1216 z_erofs_put_shortlivedpage(be->pagepool, old_bvpage);
1217}
1218
1219static int z_erofs_parse_in_bvecs(struct z_erofs_backend *be, bool *overlapped)
1220{
1221 struct z_erofs_pcluster *pcl = be->pcl;
1222 unsigned int pclusterpages = z_erofs_pclusterpages(pcl);
1223 int i, err = 0;
1224
1225 *overlapped = false;
1226 for (i = 0; i < pclusterpages; ++i) {
1227 struct z_erofs_bvec *bvec = &pcl->compressed_bvecs[i];
1228 struct page *page = bvec->page;
1229
1230 /* compressed data ought to be valid when decompressing */
1231 if (IS_ERR(page) || !page) {
1232 bvec->page = NULL; /* clear the failure reason */
1233 err = page ? PTR_ERR(page) : -EIO;
1234 continue;
1235 }
1236 be->compressed_pages[i] = page;
1237
1238 if (pcl->from_meta ||
1239 erofs_folio_is_managed(EROFS_SB(be->sb), page_folio(page))) {
1240 if (!PageUptodate(page))
1241 err = -EIO;
1242 continue;
1243 }
1244
1245 DBG_BUGON(z_erofs_page_is_invalidated(page));
1246 if (z_erofs_is_shortlived_page(page))
1247 continue;
1248 z_erofs_do_decompressed_bvec(be, bvec);
1249 *overlapped = true;
1250 }
1251 return err;
1252}
1253
1254static int z_erofs_decompress_pcluster(struct z_erofs_backend *be, bool eio)
1255{
1256 struct erofs_sb_info *const sbi = EROFS_SB(be->sb);
1257 struct z_erofs_pcluster *pcl = be->pcl;
1258 unsigned int pclusterpages = z_erofs_pclusterpages(pcl);
1259 const struct z_erofs_decompressor *alg =
1260 z_erofs_decomp[pcl->algorithmformat];
1261 bool try_free = true;
1262 int i, err2, err = eio ? -EIO : 0;
1263 struct page *page;
1264 bool overlapped;
1265 const char *reason;
1266
1267 mutex_lock(&pcl->lock);
1268 be->nr_pages = PAGE_ALIGN(pcl->length + pcl->pageofs_out) >> PAGE_SHIFT;
1269
1270 /* allocate (de)compressed page arrays if cannot be kept on stack */
1271 be->decompressed_pages = NULL;
1272 be->compressed_pages = NULL;
1273 be->onstack_used = 0;
1274 if (be->nr_pages <= Z_EROFS_ONSTACK_PAGES) {
1275 be->decompressed_pages = be->onstack_pages;
1276 be->onstack_used = be->nr_pages;
1277 memset(be->decompressed_pages, 0,
1278 sizeof(struct page *) * be->nr_pages);
1279 }
1280
1281 if (pclusterpages + be->onstack_used <= Z_EROFS_ONSTACK_PAGES)
1282 be->compressed_pages = be->onstack_pages + be->onstack_used;
1283
1284 if (!be->decompressed_pages)
1285 be->decompressed_pages =
1286 kvzalloc_objs(struct page *, be->nr_pages,
1287 GFP_KERNEL | __GFP_NOFAIL);
1288 if (!be->compressed_pages)
1289 be->compressed_pages =
1290 kvzalloc_objs(struct page *, pclusterpages,
1291 GFP_KERNEL | __GFP_NOFAIL);
1292
1293 z_erofs_parse_out_bvecs(be);
1294 err2 = z_erofs_parse_in_bvecs(be, &overlapped);
1295 if (err2)
1296 err = err2;
1297 if (!err) {
1298 reason = alg->decompress(&(struct z_erofs_decompress_req) {
1299 .sb = be->sb,
1300 .in = be->compressed_pages,
1301 .out = be->decompressed_pages,
1302 .inpages = pclusterpages,
1303 .outpages = be->nr_pages,
1304 .pageofs_in = pcl->pageofs_in,
1305 .pageofs_out = pcl->pageofs_out,
1306 .inputsize = pcl->pclustersize,
1307 .outputsize = pcl->length,
1308 .alg = pcl->algorithmformat,
1309 .inplace_io = overlapped,
1310 .partial_decoding = pcl->partial,
1311 .fillgaps = be->keepxcpy,
1312 .gfp = pcl->besteffort ? GFP_KERNEL :
1313 GFP_NOWAIT | __GFP_NORETRY
1314 }, be->pagepool);
1315 if (IS_ERR(reason)) {
1316 if (pcl->besteffort || reason != ERR_PTR(-ENOMEM))
1317 erofs_err(be->sb, "failed to decompress (%s) %pe @ pa %llu size %u => %u",
1318 alg->name, reason, pcl->pos,
1319 pcl->pclustersize, pcl->length);
1320 err = PTR_ERR(reason);
1321 } else if (unlikely(reason)) {
1322 erofs_err(be->sb, "failed to decompress (%s) %s @ pa %llu size %u => %u",
1323 alg->name, reason, pcl->pos,
1324 pcl->pclustersize, pcl->length);
1325 err = -EFSCORRUPTED;
1326 }
1327 }
1328
1329 /* must handle all compressed pages before actual file pages */
1330 if (pcl->from_meta) {
1331 folio_put(page_folio(pcl->compressed_bvecs[0].page));
1332 WRITE_ONCE(pcl->compressed_bvecs[0].page, NULL);
1333 } else {
1334 /* managed folios are still left in compressed_bvecs[] */
1335 for (i = 0; i < pclusterpages; ++i) {
1336 page = be->compressed_pages[i];
1337 if (!page)
1338 continue;
1339 if (erofs_folio_is_managed(sbi, page_folio(page))) {
1340 try_free = false;
1341 continue;
1342 }
1343 (void)z_erofs_put_shortlivedpage(be->pagepool, page);
1344 WRITE_ONCE(pcl->compressed_bvecs[i].page, NULL);
1345 }
1346 }
1347 if (be->compressed_pages < be->onstack_pages ||
1348 be->compressed_pages >= be->onstack_pages + Z_EROFS_ONSTACK_PAGES)
1349 kvfree(be->compressed_pages);
1350
1351 z_erofs_fill_other_copies(be, err);
1352 for (i = 0; i < be->nr_pages; ++i) {
1353 page = be->decompressed_pages[i];
1354 if (!page)
1355 continue;
1356
1357 DBG_BUGON(z_erofs_page_is_invalidated(page));
1358 if (!z_erofs_is_shortlived_page(page))
1359 erofs_onlinefolio_end(page_folio(page), err, true);
1360 else
1361 erofs_pagepool_add(be->pagepool, page);
1362 }
1363 if (be->decompressed_pages != be->onstack_pages)
1364 kvfree(be->decompressed_pages);
1365
1366 pcl->length = 0;
1367 pcl->partial = true;
1368 pcl->besteffort = false;
1369 pcl->bvset.nextpage = NULL;
1370 pcl->vcnt = 0;
1371
1372 /* pcluster lock MUST be taken before the following line */
1373 WRITE_ONCE(pcl->next, NULL);
1374 mutex_unlock(&pcl->lock);
1375
1376 if (pcl->from_meta)
1377 z_erofs_free_pcluster(pcl);
1378 else
1379 z_erofs_put_pcluster(sbi, pcl, try_free);
1380 return err;
1381}
1382
1383static int z_erofs_decompress_queue(const struct z_erofs_decompressqueue *io,
1384 struct page **pagepool)
1385{
1386 struct z_erofs_backend be = {
1387 .sb = io->sb,
1388 .pagepool = pagepool,
1389 .decompressed_secondary_bvecs =
1390 LIST_HEAD_INIT(be.decompressed_secondary_bvecs),
1391 .pcl = io->head,
1392 };
1393 struct z_erofs_pcluster *next;
1394 int err = 0;
1395
1396 for (; be.pcl != Z_EROFS_PCLUSTER_TAIL; be.pcl = next) {
1397 DBG_BUGON(!be.pcl);
1398 next = READ_ONCE(be.pcl->next);
1399 err = z_erofs_decompress_pcluster(&be, io->eio) ?: err;
1400 }
1401 return err;
1402}
1403
1404static void z_erofs_decompressqueue_work(struct work_struct *work)
1405{
1406 struct z_erofs_decompressqueue *bgq =
1407 container_of(work, struct z_erofs_decompressqueue, u.work);
1408 struct page *pagepool = NULL;
1409
1410 DBG_BUGON(bgq->head == Z_EROFS_PCLUSTER_TAIL);
1411 z_erofs_decompress_queue(bgq, &pagepool);
1412 erofs_release_pages(&pagepool);
1413 kvfree(bgq);
1414}
1415
1416#ifdef CONFIG_EROFS_FS_PCPU_KTHREAD
1417static void z_erofs_decompressqueue_kthread_work(struct kthread_work *work)
1418{
1419 z_erofs_decompressqueue_work((struct work_struct *)work);
1420}
1421#endif
1422
1423
1424static void z_erofs_decompress_kickoff(struct z_erofs_decompressqueue *io,
1425 int bios)
1426{
1427 struct erofs_sb_info *const sbi = EROFS_SB(io->sb);
1428 int gfp_flag;
1429
1430 /* wake up the caller thread for sync decompression */
1431 if (io->sync) {
1432 if (!atomic_add_return(bios, &io->pending_bios))
1433 complete(&io->u.done);
1434 return;
1435 }
1436
1437 if (atomic_add_return(bios, &io->pending_bios))
1438 return;
1439 if (bio_in_atomic()) {
1440 /* See `sync_decompress` in sysfs-fs-erofs for more details */
1441 if (sbi->sync_decompress == EROFS_SYNC_DECOMPRESS_AUTO)
1442 sbi->sync_decompress = EROFS_SYNC_DECOMPRESS_FORCE_ON;
1443#ifdef CONFIG_EROFS_FS_PCPU_KTHREAD
1444 scoped_guard(rcu) {
1445 struct kthread_worker *worker;
1446
1447 worker = rcu_dereference(
1448 z_erofs_pcpu_workers[raw_smp_processor_id()]);
1449 if (worker) {
1450 kthread_queue_work(worker, &io->u.kthread_work);
1451 return;
1452 }
1453 }
1454 INIT_WORK(&io->u.work, z_erofs_decompressqueue_work);
1455#endif
1456 queue_work(z_erofs_workqueue, &io->u.work);
1457 return;
1458 }
1459 gfp_flag = memalloc_noio_save();
1460 z_erofs_decompressqueue_work(&io->u.work);
1461 memalloc_noio_restore(gfp_flag);
1462}
1463
1464static void z_erofs_fill_bio_vec(struct bio_vec *bvec,
1465 struct z_erofs_frontend *f,
1466 struct z_erofs_pcluster *pcl,
1467 unsigned int nr,
1468 struct address_space *mc)
1469{
1470 gfp_t gfp = mapping_gfp_mask(mc);
1471 bool tocache = false;
1472 struct z_erofs_bvec zbv;
1473 struct address_space *mapping;
1474 struct folio *folio;
1475 struct page *page;
1476 int bs = i_blocksize(f->inode);
1477
1478 /* Except for inplace folios, the entire folio can be used for I/Os */
1479 bvec->bv_offset = 0;
1480 bvec->bv_len = PAGE_SIZE;
1481repeat:
1482 spin_lock(&pcl->lockref.lock);
1483 zbv = pcl->compressed_bvecs[nr];
1484 spin_unlock(&pcl->lockref.lock);
1485 if (!zbv.page)
1486 goto out_allocfolio;
1487
1488 bvec->bv_page = zbv.page;
1489 DBG_BUGON(z_erofs_is_shortlived_page(bvec->bv_page));
1490
1491 folio = page_folio(zbv.page);
1492 /*
1493 * Preallocated folios are added to the managed cache here rather than
1494 * in z_erofs_bind_cache() in order to keep these folios locked in
1495 * increasing (physical) address order.
1496 * Clear folio->private before these folios become visible to others in
1497 * the managed cache to avoid duplicate additions for unaligned extents.
1498 */
1499 if (folio->private == Z_EROFS_PREALLOCATED_FOLIO) {
1500 folio->private = NULL;
1501 tocache = true;
1502 goto out_tocache;
1503 }
1504
1505 mapping = READ_ONCE(folio->mapping);
1506 /*
1507 * File-backed folios for inplace I/Os are all locked steady,
1508 * therefore it is impossible for `mapping` to be NULL.
1509 */
1510 if (mapping && mapping != mc) {
1511 if (zbv.offset < 0)
1512 bvec->bv_offset = round_up(-zbv.offset, bs);
1513 bvec->bv_len = round_up(zbv.end, bs) - bvec->bv_offset;
1514 return;
1515 }
1516
1517 folio_lock(folio);
1518 if (likely(folio->mapping == mc)) {
1519 /*
1520 * The cached folio is still in managed cache but without
1521 * a valid `->private` pcluster hint. Let's reconnect them.
1522 */
1523 if (!folio_test_private(folio)) {
1524 folio_attach_private(folio, pcl);
1525 /* compressed_bvecs[] already takes a ref before */
1526 folio_put(folio);
1527 }
1528 if (likely(folio->private == pcl)) {
1529 /* don't submit cache I/Os again if already uptodate */
1530 if (folio_test_uptodate(folio)) {
1531 folio_unlock(folio);
1532 bvec->bv_page = NULL;
1533 }
1534 return;
1535 }
1536 } else {
1537 DBG_BUGON(1); /* referenced managed folios can't be truncated */
1538 }
1539 folio_unlock(folio);
1540 folio_put(folio);
1541out_allocfolio:
1542 page = __erofs_allocpage(&f->pagepool, gfp, true);
1543 spin_lock(&pcl->lockref.lock);
1544 if (unlikely(pcl->compressed_bvecs[nr].page != zbv.page)) {
1545 if (page)
1546 erofs_pagepool_add(&f->pagepool, page);
1547 spin_unlock(&pcl->lockref.lock);
1548 cond_resched();
1549 goto repeat;
1550 }
1551 pcl->compressed_bvecs[nr].page = page ? page : ERR_PTR(-ENOMEM);
1552 spin_unlock(&pcl->lockref.lock);
1553 bvec->bv_page = page;
1554 if (!page)
1555 return;
1556 folio = page_folio(page);
1557out_tocache:
1558 if (!tocache || bs != PAGE_SIZE ||
1559 filemap_add_folio(mc, folio, (pcl->pos >> PAGE_SHIFT) + nr, gfp)) {
1560 /* turn into a temporary shortlived folio (1 ref) */
1561 folio->private = (void *)Z_EROFS_SHORTLIVED_PAGE;
1562 return;
1563 }
1564 folio_attach_private(folio, pcl);
1565 /* drop a refcount added by allocpage (then 2 refs in total here) */
1566 folio_put(folio);
1567}
1568
1569static struct z_erofs_decompressqueue *jobqueue_init(struct super_block *sb,
1570 struct z_erofs_decompressqueue *fgq, bool *fg)
1571{
1572 struct z_erofs_decompressqueue *q;
1573
1574 if (fg && !*fg) {
1575 q = kvzalloc_obj(*q, GFP_KERNEL | __GFP_NOWARN);
1576 if (!q) {
1577 *fg = true;
1578 goto fg_out;
1579 }
1580#ifdef CONFIG_EROFS_FS_PCPU_KTHREAD
1581 kthread_init_work(&q->u.kthread_work,
1582 z_erofs_decompressqueue_kthread_work);
1583#else
1584 INIT_WORK(&q->u.work, z_erofs_decompressqueue_work);
1585#endif
1586 } else {
1587fg_out:
1588 q = fgq;
1589 init_completion(&fgq->u.done);
1590 atomic_set(&fgq->pending_bios, 0);
1591 q->eio = false;
1592 q->sync = true;
1593 }
1594 q->sb = sb;
1595 q->head = Z_EROFS_PCLUSTER_TAIL;
1596 return q;
1597}
1598
1599/* define decompression jobqueue types */
1600enum {
1601 JQ_BYPASS,
1602 JQ_SUBMIT,
1603 NR_JOBQUEUES,
1604};
1605
1606static void z_erofs_move_to_bypass_queue(struct z_erofs_pcluster *pcl,
1607 struct z_erofs_pcluster *next,
1608 struct z_erofs_pcluster **qtail[])
1609{
1610 WRITE_ONCE(pcl->next, Z_EROFS_PCLUSTER_TAIL);
1611 WRITE_ONCE(*qtail[JQ_SUBMIT], next);
1612 WRITE_ONCE(*qtail[JQ_BYPASS], pcl);
1613 qtail[JQ_BYPASS] = &pcl->next;
1614}
1615
1616static void z_erofs_endio(struct bio *bio)
1617{
1618 struct z_erofs_decompressqueue *q = bio->bi_private;
1619 blk_status_t err = bio->bi_status;
1620 struct folio_iter fi;
1621
1622 bio_for_each_folio_all(fi, bio) {
1623 struct folio *folio = fi.folio;
1624
1625 DBG_BUGON(folio_test_uptodate(folio));
1626 DBG_BUGON(z_erofs_page_is_invalidated(&folio->page));
1627 if (!erofs_folio_is_managed(EROFS_SB(q->sb), folio))
1628 continue;
1629
1630 if (!err)
1631 folio_mark_uptodate(folio);
1632 folio_unlock(folio);
1633 }
1634 if (err)
1635 q->eio = true;
1636 z_erofs_decompress_kickoff(q, -1);
1637 if (bio->bi_bdev)
1638 bio_put(bio);
1639}
1640
1641static void z_erofs_submit_queue(struct z_erofs_frontend *f,
1642 struct z_erofs_decompressqueue *fgq,
1643 bool *force_fg, bool readahead)
1644{
1645 struct super_block *sb = f->inode->i_sb;
1646 struct address_space *mc = MNGD_MAPPING(EROFS_SB(sb));
1647 struct z_erofs_pcluster **qtail[NR_JOBQUEUES];
1648 struct z_erofs_decompressqueue *q[NR_JOBQUEUES];
1649 struct z_erofs_pcluster *pcl, *next;
1650 /* bio is NULL initially, so no need to initialize last_{index,bdev} */
1651 erofs_off_t last_pa;
1652 unsigned int nr_bios = 0;
1653 struct bio *bio = NULL;
1654 unsigned long pflags;
1655 int memstall = 0;
1656
1657 /* No need to read from device for pclusters in the bypass queue. */
1658 q[JQ_BYPASS] = jobqueue_init(sb, fgq + JQ_BYPASS, NULL);
1659 q[JQ_SUBMIT] = jobqueue_init(sb, fgq + JQ_SUBMIT, force_fg);
1660
1661 qtail[JQ_BYPASS] = &q[JQ_BYPASS]->head;
1662 qtail[JQ_SUBMIT] = &q[JQ_SUBMIT]->head;
1663
1664 /* by default, all need io submission */
1665 q[JQ_SUBMIT]->head = next = f->head;
1666
1667 do {
1668 struct erofs_map_dev mdev;
1669 erofs_off_t cur, end;
1670 struct bio_vec bvec;
1671 unsigned int i = 0;
1672 bool bypass = true;
1673
1674 pcl = next;
1675 next = READ_ONCE(pcl->next);
1676 if (pcl->from_meta) {
1677 z_erofs_move_to_bypass_queue(pcl, next, qtail);
1678 continue;
1679 }
1680
1681 /* no device id here, thus it will always succeed */
1682 mdev = (struct erofs_map_dev) {
1683 .m_pa = round_down(pcl->pos, sb->s_blocksize),
1684 };
1685 (void)erofs_map_dev(sb, &mdev);
1686
1687 cur = mdev.m_pa;
1688 end = round_up(cur + pcl->pageofs_in + pcl->pclustersize,
1689 sb->s_blocksize);
1690 do {
1691 bvec.bv_page = NULL;
1692 if (bio && (cur != last_pa ||
1693 bio->bi_bdev != mdev.m_bdev)) {
1694drain_io:
1695 if (erofs_is_fileio_mode(EROFS_SB(sb)))
1696 erofs_fileio_submit_bio(bio);
1697 else
1698 submit_bio(bio);
1699
1700 if (memstall) {
1701 psi_memstall_leave(&pflags);
1702 memstall = 0;
1703 }
1704 bio = NULL;
1705 }
1706
1707 if (!bvec.bv_page) {
1708 z_erofs_fill_bio_vec(&bvec, f, pcl, i++, mc);
1709 if (!bvec.bv_page)
1710 continue;
1711 if (cur + bvec.bv_len > end)
1712 bvec.bv_len = end - cur;
1713 DBG_BUGON(bvec.bv_len < sb->s_blocksize);
1714 }
1715
1716 if (unlikely(PageWorkingset(bvec.bv_page)) &&
1717 !memstall) {
1718 psi_memstall_enter(&pflags);
1719 memstall = 1;
1720 }
1721
1722 if (!bio) {
1723 if (erofs_is_fileio_mode(EROFS_SB(sb)))
1724 bio = erofs_fileio_bio_alloc(&mdev);
1725 else
1726 bio = bio_alloc(mdev.m_bdev, BIO_MAX_VECS,
1727 REQ_OP_READ, GFP_NOIO);
1728 bio->bi_end_io = z_erofs_endio;
1729 bio->bi_iter.bi_sector =
1730 (mdev.m_dif->fsoff + cur) >> 9;
1731 bio->bi_private = q[JQ_SUBMIT];
1732 if (readahead)
1733 bio->bi_opf |= REQ_RAHEAD;
1734 ++nr_bios;
1735 }
1736
1737 if (!bio_add_page(bio, bvec.bv_page, bvec.bv_len,
1738 bvec.bv_offset))
1739 goto drain_io;
1740 last_pa = cur + bvec.bv_len;
1741 bypass = false;
1742 } while ((cur += bvec.bv_len) < end);
1743
1744 if (!bypass)
1745 qtail[JQ_SUBMIT] = &pcl->next;
1746 else
1747 z_erofs_move_to_bypass_queue(pcl, next, qtail);
1748 } while (next != Z_EROFS_PCLUSTER_TAIL);
1749
1750 if (bio) {
1751 if (erofs_is_fileio_mode(EROFS_SB(sb)))
1752 erofs_fileio_submit_bio(bio);
1753 else
1754 submit_bio(bio);
1755 }
1756 if (memstall)
1757 psi_memstall_leave(&pflags);
1758
1759 /*
1760 * although background is preferred, no one is pending for submission.
1761 * don't issue decompression but drop it directly instead.
1762 */
1763 if (!*force_fg && !nr_bios) {
1764 kvfree(q[JQ_SUBMIT]);
1765 return;
1766 }
1767 z_erofs_decompress_kickoff(q[JQ_SUBMIT], nr_bios);
1768}
1769
1770static int z_erofs_runqueue(struct z_erofs_frontend *f, unsigned int rabytes)
1771{
1772 struct z_erofs_decompressqueue io[NR_JOBQUEUES];
1773 struct erofs_sb_info *sbi = EROFS_I_SB(f->inode);
1774 int syncmode = sbi->sync_decompress;
1775 bool force_fg;
1776 int err;
1777
1778 force_fg = (syncmode == EROFS_SYNC_DECOMPRESS_AUTO && !rabytes) ||
1779 (syncmode == EROFS_SYNC_DECOMPRESS_FORCE_ON &&
1780 (rabytes <= Z_EROFS_MAX_SYNC_DECOMPRESS_BYTES));
1781
1782 if (f->head == Z_EROFS_PCLUSTER_TAIL)
1783 return 0;
1784 z_erofs_submit_queue(f, io, &force_fg, !!rabytes);
1785
1786 /* handle bypass queue (no i/o pclusters) immediately */
1787 err = z_erofs_decompress_queue(&io[JQ_BYPASS], &f->pagepool);
1788 if (!force_fg)
1789 return err;
1790
1791 /* wait until all bios are completed */
1792 wait_for_completion_io(&io[JQ_SUBMIT].u.done);
1793
1794 /* handle synchronous decompress queue in the caller context */
1795 return z_erofs_decompress_queue(&io[JQ_SUBMIT], &f->pagepool) ?: err;
1796}
1797
1798/*
1799 * Since partial uptodate is still unimplemented for now, we have to use
1800 * approximate readmore strategies as a start.
1801 */
1802static void z_erofs_pcluster_readmore(struct z_erofs_frontend *f,
1803 struct readahead_control *rac, bool backmost)
1804{
1805 struct inode *inode = f->inode;
1806 struct erofs_map_blocks *map = &f->map;
1807 erofs_off_t cur, end, headoffset = f->headoffset;
1808 int err;
1809
1810 if (backmost) {
1811 if (rac)
1812 end = headoffset + readahead_length(rac) - 1;
1813 else
1814 end = headoffset + PAGE_SIZE - 1;
1815 map->m_la = end;
1816 err = z_erofs_map_blocks_iter(inode, map,
1817 EROFS_GET_BLOCKS_READMORE);
1818 if (err || !(map->m_flags & EROFS_MAP_MAPPED))
1819 return;
1820
1821 /* expand ra for the trailing edge if readahead */
1822 if (rac) {
1823 cur = round_up(map->m_la + map->m_llen, PAGE_SIZE);
1824 readahead_expand(rac, headoffset, cur - headoffset);
1825 return;
1826 }
1827 end = round_up(end, PAGE_SIZE);
1828 } else {
1829 end = round_up(map->m_la, PAGE_SIZE);
1830 if (!(map->m_flags & EROFS_MAP_MAPPED) || !map->m_llen)
1831 return;
1832 }
1833
1834 cur = map->m_la + map->m_llen - 1;
1835 while ((cur >= end) && (cur < i_size_read(inode))) {
1836 pgoff_t index = cur >> PAGE_SHIFT;
1837 struct folio *folio;
1838
1839 folio = erofs_grab_folio_nowait(f->sharedinode->i_mapping, index);
1840 if (!IS_ERR_OR_NULL(folio)) {
1841 if (folio_test_uptodate(folio))
1842 folio_unlock(folio);
1843 else
1844 z_erofs_scan_folio(f, folio, !!rac);
1845 folio_put(folio);
1846 }
1847
1848 if (cur < PAGE_SIZE)
1849 break;
1850 cur = ((loff_t)index << PAGE_SHIFT) - 1;
1851 }
1852}
1853
1854static int z_erofs_read_folio(struct file *file, struct folio *folio)
1855{
1856 struct inode *sharedinode = folio->mapping->host;
1857 bool need_iput;
1858 struct inode *realinode = erofs_real_inode(sharedinode, &need_iput);
1859 Z_EROFS_DEFINE_FRONTEND(f, realinode, sharedinode, folio_pos(folio));
1860 int err;
1861
1862 trace_erofs_read_folio(realinode, folio, false);
1863 z_erofs_pcluster_readmore(&f, NULL, true);
1864 err = z_erofs_scan_folio(&f, folio, false);
1865 z_erofs_pcluster_readmore(&f, NULL, false);
1866 z_erofs_pcluster_end(&f);
1867
1868 /* if some pclusters are ready, need submit them anyway */
1869 err = z_erofs_runqueue(&f, 0) ?: err;
1870 if (err && err != -EINTR)
1871 erofs_err(realinode->i_sb, "read error %d @ %lu of nid %llu",
1872 err, folio->index, EROFS_I(realinode)->nid);
1873
1874 erofs_put_metabuf(&f.map.buf);
1875 erofs_release_pages(&f.pagepool);
1876
1877 if (need_iput)
1878 iput(realinode);
1879 return err;
1880}
1881
1882static void z_erofs_readahead(struct readahead_control *rac)
1883{
1884 struct inode *sharedinode = rac->mapping->host;
1885 bool need_iput;
1886 struct inode *realinode = erofs_real_inode(sharedinode, &need_iput);
1887 Z_EROFS_DEFINE_FRONTEND(f, realinode, sharedinode, readahead_pos(rac));
1888 unsigned int nrpages = readahead_count(rac);
1889 struct folio *head = NULL, *folio;
1890 int err;
1891
1892 trace_erofs_readahead(realinode, readahead_index(rac), nrpages, false);
1893 z_erofs_pcluster_readmore(&f, rac, true);
1894 while ((folio = readahead_folio(rac))) {
1895 folio->private = head;
1896 head = folio;
1897 }
1898
1899 /* traverse in reverse order for best metadata I/O performance */
1900 while (head) {
1901 folio = head;
1902 head = folio_get_private(folio);
1903
1904 err = z_erofs_scan_folio(&f, folio, true);
1905 if (err && err != -EINTR)
1906 erofs_err(realinode->i_sb, "readahead error at folio %lu @ nid %llu",
1907 folio->index, EROFS_I(realinode)->nid);
1908 }
1909 z_erofs_pcluster_readmore(&f, rac, false);
1910 z_erofs_pcluster_end(&f);
1911
1912 (void)z_erofs_runqueue(&f, nrpages << PAGE_SHIFT);
1913 erofs_put_metabuf(&f.map.buf);
1914 erofs_release_pages(&f.pagepool);
1915
1916 if (need_iput)
1917 iput(realinode);
1918}
1919
1920const struct address_space_operations z_erofs_aops = {
1921 .read_folio = z_erofs_read_folio,
1922 .readahead = z_erofs_readahead,
1923};