projects/linux·blob master · refs

2090 lines · 49.3 KiB

1// SPDX-License-Identifier: GPL-2.0
2/*
3 * f2fs compress support
4 *
5 * Copyright (c) 2019 Chao Yu <[email protected]>
6 */
7
8#include <linux/fs.h>
9#include <linux/f2fs_fs.h>
10#include <linux/moduleparam.h>
11#include <linux/writeback.h>
12#include <linux/backing-dev.h>
13#include <linux/lzo.h>
14#include <linux/lz4.h>
15#include <linux/zstd.h>
16#include <linux/folio_batch.h>
17#include <linux/fserror.h>
18
19#include "f2fs.h"
20#include "node.h"
21#include "segment.h"
22#include <trace/events/f2fs.h>
23
24static struct kmem_cache *cic_entry_slab;
25static struct kmem_cache *dic_entry_slab;
26
27static void *page_array_alloc(struct f2fs_sb_info *sbi, int nr)
28{
29 unsigned int size = sizeof(struct page *) * nr;
30
31 if (likely(size <= sbi->page_array_slab_size))
32 return f2fs_kmem_cache_alloc(sbi->page_array_slab,
33 GFP_F2FS_ZERO, false, sbi);
34 return f2fs_kzalloc(sbi, size, GFP_NOFS);
35}
36
37static void page_array_free(struct f2fs_sb_info *sbi, void *pages, int nr)
38{
39 unsigned int size = sizeof(struct page *) * nr;
40
41 if (!pages)
42 return;
43
44 if (likely(size <= sbi->page_array_slab_size))
45 kmem_cache_free(sbi->page_array_slab, pages);
46 else
47 kfree(pages);
48}
49
50struct f2fs_compress_ops {
51 int (*init_compress_ctx)(struct compress_ctx *cc);
52 void (*destroy_compress_ctx)(struct compress_ctx *cc);
53 int (*compress_pages)(struct compress_ctx *cc);
54 int (*init_decompress_ctx)(struct decompress_io_ctx *dic);
55 void (*destroy_decompress_ctx)(struct decompress_io_ctx *dic);
56 int (*decompress_pages)(struct decompress_io_ctx *dic);
57 bool (*is_level_valid)(int level);
58};
59
60static unsigned int offset_in_cluster(struct compress_ctx *cc, pgoff_t index)
61{
62 return index & (cc->cluster_size - 1);
63}
64
65static pgoff_t cluster_idx(struct compress_ctx *cc, pgoff_t index)
66{
67 return index >> cc->log_cluster_size;
68}
69
70static pgoff_t start_idx_of_cluster(struct compress_ctx *cc)
71{
72 return cc->cluster_idx << cc->log_cluster_size;
73}
74
75bool f2fs_is_compressed_page(struct folio *folio)
76{
77 if (!folio->private)
78 return false;
79 if (folio_test_f2fs_nonpointer(folio))
80 return false;
81
82 f2fs_bug_on(F2FS_F_SB(folio),
83 *((u32 *)folio->private) != F2FS_COMPRESSED_PAGE_MAGIC);
84 return true;
85}
86
87static void f2fs_set_compressed_page(struct page *page,
88 struct inode *inode, pgoff_t index, void *data)
89{
90 struct folio *folio = page_folio(page);
91
92 folio_attach_private(folio, (void *)data);
93
94 /* i_crypto_info and iv index */
95 folio->index = index;
96 folio->mapping = inode->i_mapping;
97}
98
99static void f2fs_drop_rpages(struct compress_ctx *cc, int len, bool unlock)
100{
101 int i;
102
103 for (i = 0; i < len; i++) {
104 if (!cc->rpages[i])
105 continue;
106 if (unlock)
107 unlock_page(cc->rpages[i]);
108 else
109 put_page(cc->rpages[i]);
110 }
111}
112
113static void f2fs_put_rpages(struct compress_ctx *cc)
114{
115 f2fs_drop_rpages(cc, cc->cluster_size, false);
116}
117
118static void f2fs_unlock_rpages(struct compress_ctx *cc, int len)
119{
120 f2fs_drop_rpages(cc, len, true);
121}
122
123static void f2fs_put_rpages_wbc(struct compress_ctx *cc,
124 struct writeback_control *wbc, bool redirty, bool unlock)
125{
126 unsigned int i;
127
128 for (i = 0; i < cc->cluster_size; i++) {
129 if (!cc->rpages[i])
130 continue;
131 if (redirty)
132 redirty_page_for_writepage(wbc, cc->rpages[i]);
133 f2fs_put_page(cc->rpages[i], unlock);
134 }
135}
136
137struct folio *f2fs_compress_control_folio(struct folio *folio)
138{
139 struct compress_io_ctx *ctx = folio->private;
140
141 return page_folio(ctx->rpages[0]);
142}
143
144int f2fs_init_compress_ctx(struct compress_ctx *cc)
145{
146 if (cc->rpages)
147 return 0;
148
149 cc->rpages = page_array_alloc(F2FS_I_SB(cc->inode), cc->cluster_size);
150 return cc->rpages ? 0 : -ENOMEM;
151}
152
153void f2fs_destroy_compress_ctx(struct compress_ctx *cc, bool reuse)
154{
155 page_array_free(F2FS_I_SB(cc->inode), cc->rpages, cc->cluster_size);
156 cc->rpages = NULL;
157 cc->nr_rpages = 0;
158 cc->nr_cpages = 0;
159 cc->valid_nr_cpages = 0;
160 if (!reuse)
161 cc->cluster_idx = NULL_CLUSTER;
162}
163
164void f2fs_compress_ctx_add_page(struct compress_ctx *cc, struct folio *folio)
165{
166 unsigned int cluster_ofs;
167
168 if (!f2fs_cluster_can_merge_page(cc, folio->index))
169 f2fs_bug_on(F2FS_I_SB(cc->inode), 1);
170
171 cluster_ofs = offset_in_cluster(cc, folio->index);
172 cc->rpages[cluster_ofs] = folio_page(folio, 0);
173 cc->nr_rpages++;
174 cc->cluster_idx = cluster_idx(cc, folio->index);
175}
176
177#ifdef CONFIG_F2FS_FS_LZO
178static int lzo_init_compress_ctx(struct compress_ctx *cc)
179{
180 cc->private = f2fs_vmalloc(F2FS_I_SB(cc->inode),
181 LZO1X_MEM_COMPRESS);
182 if (!cc->private)
183 return -ENOMEM;
184
185 cc->clen = lzo1x_worst_compress(PAGE_SIZE << cc->log_cluster_size);
186 return 0;
187}
188
189static void lzo_destroy_compress_ctx(struct compress_ctx *cc)
190{
191 vfree(cc->private);
192 cc->private = NULL;
193}
194
195static int lzo_compress_pages(struct compress_ctx *cc)
196{
197 int ret;
198
199 ret = lzo1x_1_compress(cc->rbuf, cc->rlen, cc->cbuf->cdata,
200 &cc->clen, cc->private);
201 if (ret != LZO_E_OK) {
202 f2fs_err_ratelimited(F2FS_I_SB(cc->inode),
203 "lzo compress failed, ret:%d", ret);
204 return -EIO;
205 }
206 return 0;
207}
208
209static int lzo_decompress_pages(struct decompress_io_ctx *dic)
210{
211 int ret;
212
213 ret = lzo1x_decompress_safe(dic->cbuf->cdata, dic->clen,
214 dic->rbuf, &dic->rlen);
215 if (ret != LZO_E_OK) {
216 f2fs_err_ratelimited(dic->sbi,
217 "lzo decompress failed, ret:%d", ret);
218 return -EIO;
219 }
220
221 if (dic->rlen != PAGE_SIZE << dic->log_cluster_size) {
222 f2fs_err_ratelimited(dic->sbi,
223 "lzo invalid rlen:%zu, expected:%lu",
224 dic->rlen, PAGE_SIZE << dic->log_cluster_size);
225 return -EIO;
226 }
227 return 0;
228}
229
230static const struct f2fs_compress_ops f2fs_lzo_ops = {
231 .init_compress_ctx = lzo_init_compress_ctx,
232 .destroy_compress_ctx = lzo_destroy_compress_ctx,
233 .compress_pages = lzo_compress_pages,
234 .decompress_pages = lzo_decompress_pages,
235};
236#endif
237
238#ifdef CONFIG_F2FS_FS_LZ4
239static int lz4_init_compress_ctx(struct compress_ctx *cc)
240{
241 unsigned int size = LZ4_MEM_COMPRESS;
242
243#ifdef CONFIG_F2FS_FS_LZ4HC
244 if (F2FS_I(cc->inode)->i_compress_level)
245 size = LZ4HC_MEM_COMPRESS;
246#endif
247
248 cc->private = f2fs_vmalloc(F2FS_I_SB(cc->inode), size);
249 if (!cc->private)
250 return -ENOMEM;
251
252 /*
253 * we do not change cc->clen to LZ4_compressBound(inputsize) to
254 * adapt worst compress case, because lz4 compressor can handle
255 * output budget properly.
256 */
257 cc->clen = cc->rlen - PAGE_SIZE - COMPRESS_HEADER_SIZE;
258 return 0;
259}
260
261static void lz4_destroy_compress_ctx(struct compress_ctx *cc)
262{
263 vfree(cc->private);
264 cc->private = NULL;
265}
266
267static int lz4_compress_pages(struct compress_ctx *cc)
268{
269 int len = -EINVAL;
270 unsigned char level = F2FS_I(cc->inode)->i_compress_level;
271
272 if (!level)
273 len = LZ4_compress_default(cc->rbuf, cc->cbuf->cdata, cc->rlen,
274 cc->clen, cc->private);
275#ifdef CONFIG_F2FS_FS_LZ4HC
276 else
277 len = LZ4_compress_HC(cc->rbuf, cc->cbuf->cdata, cc->rlen,
278 cc->clen, level, cc->private);
279#endif
280 if (len < 0)
281 return len;
282 if (!len)
283 return -EAGAIN;
284
285 cc->clen = len;
286 return 0;
287}
288
289static int lz4_decompress_pages(struct decompress_io_ctx *dic)
290{
291 int ret;
292
293 ret = LZ4_decompress_safe(dic->cbuf->cdata, dic->rbuf,
294 dic->clen, dic->rlen);
295 if (ret < 0) {
296 f2fs_err_ratelimited(dic->sbi,
297 "lz4 decompress failed, ret:%d", ret);
298 return -EIO;
299 }
300
301 if (ret != PAGE_SIZE << dic->log_cluster_size) {
302 f2fs_err_ratelimited(dic->sbi,
303 "lz4 invalid ret:%d, expected:%lu",
304 ret, PAGE_SIZE << dic->log_cluster_size);
305 return -EIO;
306 }
307 return 0;
308}
309
310static bool lz4_is_level_valid(int lvl)
311{
312#ifdef CONFIG_F2FS_FS_LZ4HC
313 return !lvl || (lvl >= LZ4HC_MIN_CLEVEL && lvl <= LZ4HC_MAX_CLEVEL);
314#else
315 return lvl == 0;
316#endif
317}
318
319static const struct f2fs_compress_ops f2fs_lz4_ops = {
320 .init_compress_ctx = lz4_init_compress_ctx,
321 .destroy_compress_ctx = lz4_destroy_compress_ctx,
322 .compress_pages = lz4_compress_pages,
323 .decompress_pages = lz4_decompress_pages,
324 .is_level_valid = lz4_is_level_valid,
325};
326#endif
327
328#ifdef CONFIG_F2FS_FS_ZSTD
329static int zstd_init_compress_ctx(struct compress_ctx *cc)
330{
331 zstd_parameters params;
332 zstd_cstream *stream;
333 void *workspace;
334 unsigned int workspace_size;
335 unsigned char level = F2FS_I(cc->inode)->i_compress_level;
336
337 /* Need to remain this for backward compatibility */
338 if (!level)
339 level = F2FS_ZSTD_DEFAULT_CLEVEL;
340
341 params = zstd_get_params(level, cc->rlen);
342 workspace_size = zstd_cstream_workspace_bound(&params.cParams);
343
344 workspace = f2fs_vmalloc(F2FS_I_SB(cc->inode), workspace_size);
345 if (!workspace)
346 return -ENOMEM;
347
348 stream = zstd_init_cstream(&params, 0, workspace, workspace_size);
349 if (!stream) {
350 f2fs_err_ratelimited(F2FS_I_SB(cc->inode),
351 "%s zstd_init_cstream failed", __func__);
352 vfree(workspace);
353 return -EIO;
354 }
355
356 cc->private = workspace;
357 cc->private2 = stream;
358
359 cc->clen = cc->rlen - PAGE_SIZE - COMPRESS_HEADER_SIZE;
360 return 0;
361}
362
363static void zstd_destroy_compress_ctx(struct compress_ctx *cc)
364{
365 vfree(cc->private);
366 cc->private = NULL;
367 cc->private2 = NULL;
368}
369
370static int zstd_compress_pages(struct compress_ctx *cc)
371{
372 zstd_cstream *stream = cc->private2;
373 zstd_in_buffer inbuf;
374 zstd_out_buffer outbuf;
375 int src_size = cc->rlen;
376 int dst_size = src_size - PAGE_SIZE - COMPRESS_HEADER_SIZE;
377 int ret;
378
379 inbuf.pos = 0;
380 inbuf.src = cc->rbuf;
381 inbuf.size = src_size;
382
383 outbuf.pos = 0;
384 outbuf.dst = cc->cbuf->cdata;
385 outbuf.size = dst_size;
386
387 ret = zstd_compress_stream(stream, &outbuf, &inbuf);
388 if (zstd_is_error(ret)) {
389 f2fs_err_ratelimited(F2FS_I_SB(cc->inode),
390 "%s zstd_compress_stream failed, ret: %d",
391 __func__, zstd_get_error_code(ret));
392 return -EIO;
393 }
394
395 ret = zstd_end_stream(stream, &outbuf);
396 if (zstd_is_error(ret)) {
397 f2fs_err_ratelimited(F2FS_I_SB(cc->inode),
398 "%s zstd_end_stream returned %d",
399 __func__, zstd_get_error_code(ret));
400 return -EIO;
401 }
402
403 /*
404 * there is compressed data remained in intermediate buffer due to
405 * no more space in cbuf.cdata
406 */
407 if (ret)
408 return -EAGAIN;
409
410 cc->clen = outbuf.pos;
411 return 0;
412}
413
414static int zstd_init_decompress_ctx(struct decompress_io_ctx *dic)
415{
416 zstd_dstream *stream;
417 void *workspace;
418 unsigned int workspace_size;
419 unsigned int max_window_size =
420 MAX_COMPRESS_WINDOW_SIZE(dic->log_cluster_size);
421
422 workspace_size = zstd_dstream_workspace_bound(max_window_size);
423
424 workspace = f2fs_vmalloc(dic->sbi, workspace_size);
425 if (!workspace)
426 return -ENOMEM;
427
428 stream = zstd_init_dstream(max_window_size, workspace, workspace_size);
429 if (!stream) {
430 f2fs_err_ratelimited(dic->sbi,
431 "%s zstd_init_dstream failed", __func__);
432 vfree(workspace);
433 return -EIO;
434 }
435
436 dic->private = workspace;
437 dic->private2 = stream;
438
439 return 0;
440}
441
442static void zstd_destroy_decompress_ctx(struct decompress_io_ctx *dic)
443{
444 vfree(dic->private);
445 dic->private = NULL;
446 dic->private2 = NULL;
447}
448
449static int zstd_decompress_pages(struct decompress_io_ctx *dic)
450{
451 zstd_dstream *stream = dic->private2;
452 zstd_in_buffer inbuf;
453 zstd_out_buffer outbuf;
454 int ret;
455
456 inbuf.pos = 0;
457 inbuf.src = dic->cbuf->cdata;
458 inbuf.size = dic->clen;
459
460 outbuf.pos = 0;
461 outbuf.dst = dic->rbuf;
462 outbuf.size = dic->rlen;
463
464 ret = zstd_decompress_stream(stream, &outbuf, &inbuf);
465 if (zstd_is_error(ret)) {
466 f2fs_err_ratelimited(dic->sbi,
467 "%s zstd_decompress_stream failed, ret: %d",
468 __func__, zstd_get_error_code(ret));
469 return -EIO;
470 }
471
472 if (dic->rlen != outbuf.pos) {
473 f2fs_err_ratelimited(dic->sbi,
474 "%s ZSTD invalid rlen:%zu, expected:%lu",
475 __func__, dic->rlen,
476 PAGE_SIZE << dic->log_cluster_size);
477 return -EIO;
478 }
479
480 return 0;
481}
482
483static bool zstd_is_level_valid(int lvl)
484{
485 return lvl >= zstd_min_clevel() && lvl <= zstd_max_clevel();
486}
487
488static const struct f2fs_compress_ops f2fs_zstd_ops = {
489 .init_compress_ctx = zstd_init_compress_ctx,
490 .destroy_compress_ctx = zstd_destroy_compress_ctx,
491 .compress_pages = zstd_compress_pages,
492 .init_decompress_ctx = zstd_init_decompress_ctx,
493 .destroy_decompress_ctx = zstd_destroy_decompress_ctx,
494 .decompress_pages = zstd_decompress_pages,
495 .is_level_valid = zstd_is_level_valid,
496};
497#endif
498
499#ifdef CONFIG_F2FS_FS_LZO
500#ifdef CONFIG_F2FS_FS_LZORLE
501static int lzorle_compress_pages(struct compress_ctx *cc)
502{
503 int ret;
504
505 ret = lzorle1x_1_compress(cc->rbuf, cc->rlen, cc->cbuf->cdata,
506 &cc->clen, cc->private);
507 if (ret != LZO_E_OK) {
508 f2fs_err_ratelimited(F2FS_I_SB(cc->inode),
509 "lzo-rle compress failed, ret:%d", ret);
510 return -EIO;
511 }
512 return 0;
513}
514
515static const struct f2fs_compress_ops f2fs_lzorle_ops = {
516 .init_compress_ctx = lzo_init_compress_ctx,
517 .destroy_compress_ctx = lzo_destroy_compress_ctx,
518 .compress_pages = lzorle_compress_pages,
519 .decompress_pages = lzo_decompress_pages,
520};
521#endif
522#endif
523
524static const struct f2fs_compress_ops *f2fs_cops[COMPRESS_MAX] = {
525#ifdef CONFIG_F2FS_FS_LZO
526 &f2fs_lzo_ops,
527#else
528 NULL,
529#endif
530#ifdef CONFIG_F2FS_FS_LZ4
531 &f2fs_lz4_ops,
532#else
533 NULL,
534#endif
535#ifdef CONFIG_F2FS_FS_ZSTD
536 &f2fs_zstd_ops,
537#else
538 NULL,
539#endif
540#if defined(CONFIG_F2FS_FS_LZO) && defined(CONFIG_F2FS_FS_LZORLE)
541 &f2fs_lzorle_ops,
542#else
543 NULL,
544#endif
545};
546
547bool f2fs_is_compress_backend_ready(struct inode *inode)
548{
549 if (!f2fs_compressed_file(inode))
550 return true;
551 return f2fs_cops[F2FS_I(inode)->i_compress_algorithm];
552}
553
554bool f2fs_is_compress_level_valid(int alg, int lvl)
555{
556 const struct f2fs_compress_ops *cops = f2fs_cops[alg];
557
558 if (cops->is_level_valid)
559 return cops->is_level_valid(lvl);
560
561 return lvl == 0;
562}
563
564static mempool_t *compress_page_pool;
565static int num_compress_pages = 512;
566module_param(num_compress_pages, uint, 0444);
567MODULE_PARM_DESC(num_compress_pages,
568 "Number of intermediate compress pages to preallocate");
569
570int __init f2fs_init_compress_mempool(void)
571{
572 compress_page_pool = mempool_create_page_pool(num_compress_pages, 0);
573 return compress_page_pool ? 0 : -ENOMEM;
574}
575
576void f2fs_destroy_compress_mempool(void)
577{
578 mempool_destroy(compress_page_pool);
579}
580
581static struct page *f2fs_compress_alloc_page(void)
582{
583 struct page *page;
584
585 page = mempool_alloc(compress_page_pool, GFP_NOFS);
586 lock_page(page);
587
588 return page;
589}
590
591static void f2fs_compress_free_page(struct page *page)
592{
593 struct folio *folio;
594
595 if (!page)
596 return;
597 folio = page_folio(page);
598 folio_detach_private(folio);
599 folio->mapping = NULL;
600 folio_unlock(folio);
601 mempool_free(page, compress_page_pool);
602}
603
604#define MAX_VMAP_RETRIES 3
605
606static void *f2fs_vmap(struct page **pages, unsigned int count)
607{
608 int i;
609 void *buf = NULL;
610
611 for (i = 0; i < MAX_VMAP_RETRIES; i++) {
612 buf = vm_map_ram(pages, count, -1);
613 if (buf)
614 break;
615 vm_unmap_aliases();
616 }
617 return buf;
618}
619
620static int f2fs_compress_pages(struct compress_ctx *cc)
621{
622 struct f2fs_sb_info *sbi = F2FS_I_SB(cc->inode);
623 struct f2fs_inode_info *fi = F2FS_I(cc->inode);
624 const struct f2fs_compress_ops *cops =
625 f2fs_cops[fi->i_compress_algorithm];
626 unsigned int max_len, new_nr_cpages;
627 u32 chksum = 0;
628 int i, ret;
629
630 trace_f2fs_compress_pages_start(cc->inode, cc->cluster_idx,
631 cc->cluster_size, fi->i_compress_algorithm);
632
633 if (cops->init_compress_ctx) {
634 ret = cops->init_compress_ctx(cc);
635 if (ret)
636 goto out;
637 }
638
639 max_len = COMPRESS_HEADER_SIZE + cc->clen;
640 cc->nr_cpages = DIV_ROUND_UP(max_len, PAGE_SIZE);
641 cc->valid_nr_cpages = cc->nr_cpages;
642
643 cc->cpages = page_array_alloc(sbi, cc->nr_cpages);
644 if (!cc->cpages) {
645 ret = -ENOMEM;
646 goto destroy_compress_ctx;
647 }
648
649 for (i = 0; i < cc->nr_cpages; i++)
650 cc->cpages[i] = f2fs_compress_alloc_page();
651
652 cc->rbuf = f2fs_vmap(cc->rpages, cc->cluster_size);
653 if (!cc->rbuf) {
654 ret = -ENOMEM;
655 goto out_free_cpages;
656 }
657
658 cc->cbuf = f2fs_vmap(cc->cpages, cc->nr_cpages);
659 if (!cc->cbuf) {
660 ret = -ENOMEM;
661 goto out_vunmap_rbuf;
662 }
663
664 ret = cops->compress_pages(cc);
665 if (ret)
666 goto out_vunmap_cbuf;
667
668 max_len = PAGE_SIZE * (cc->cluster_size - 1) - COMPRESS_HEADER_SIZE;
669
670 if (cc->clen > max_len) {
671 ret = -EAGAIN;
672 goto out_vunmap_cbuf;
673 }
674
675 cc->cbuf->clen = cpu_to_le32(cc->clen);
676
677 if (fi->i_compress_flag & BIT(COMPRESS_CHKSUM))
678 chksum = f2fs_crc32(cc->cbuf->cdata, cc->clen);
679 cc->cbuf->chksum = cpu_to_le32(chksum);
680
681 for (i = 0; i < COMPRESS_DATA_RESERVED_SIZE; i++)
682 cc->cbuf->reserved[i] = cpu_to_le32(0);
683
684 new_nr_cpages = DIV_ROUND_UP(cc->clen + COMPRESS_HEADER_SIZE, PAGE_SIZE);
685
686 /* zero out any unused part of the last page */
687 memset(&cc->cbuf->cdata[cc->clen], 0,
688 (new_nr_cpages * PAGE_SIZE) -
689 (cc->clen + COMPRESS_HEADER_SIZE));
690
691 vm_unmap_ram(cc->cbuf, cc->nr_cpages);
692 vm_unmap_ram(cc->rbuf, cc->cluster_size);
693
694 for (i = new_nr_cpages; i < cc->nr_cpages; i++) {
695 f2fs_compress_free_page(cc->cpages[i]);
696 cc->cpages[i] = NULL;
697 }
698
699 if (cops->destroy_compress_ctx)
700 cops->destroy_compress_ctx(cc);
701
702 cc->valid_nr_cpages = new_nr_cpages;
703
704 trace_f2fs_compress_pages_end(cc->inode, cc->cluster_idx,
705 cc->clen, ret);
706 return 0;
707
708out_vunmap_cbuf:
709 vm_unmap_ram(cc->cbuf, cc->nr_cpages);
710out_vunmap_rbuf:
711 vm_unmap_ram(cc->rbuf, cc->cluster_size);
712out_free_cpages:
713 for (i = 0; i < cc->nr_cpages; i++) {
714 if (cc->cpages[i])
715 f2fs_compress_free_page(cc->cpages[i]);
716 }
717 page_array_free(sbi, cc->cpages, cc->nr_cpages);
718 cc->cpages = NULL;
719destroy_compress_ctx:
720 if (cops->destroy_compress_ctx)
721 cops->destroy_compress_ctx(cc);
722out:
723 trace_f2fs_compress_pages_end(cc->inode, cc->cluster_idx,
724 cc->clen, ret);
725 return ret;
726}
727
728static int f2fs_prepare_decomp_mem(struct decompress_io_ctx *dic,
729 bool pre_alloc);
730static void f2fs_release_decomp_mem(struct decompress_io_ctx *dic,
731 bool bypass_destroy_callback, bool pre_alloc);
732
733void f2fs_decompress_cluster(struct decompress_io_ctx *dic, bool in_task)
734{
735 struct f2fs_sb_info *sbi = dic->sbi;
736 struct f2fs_inode_info *fi = F2FS_I(dic->inode);
737 const struct f2fs_compress_ops *cops =
738 f2fs_cops[fi->i_compress_algorithm];
739 bool bypass_callback = false;
740 int ret;
741
742 trace_f2fs_decompress_pages_start(dic->inode, dic->cluster_idx,
743 dic->cluster_size, fi->i_compress_algorithm);
744
745 if (dic->failed) {
746 ret = -EIO;
747 goto out_end_io;
748 }
749
750 ret = f2fs_prepare_decomp_mem(dic, false);
751 if (ret) {
752 bypass_callback = true;
753 goto out_release;
754 }
755
756 dic->clen = le32_to_cpu(dic->cbuf->clen);
757 dic->rlen = PAGE_SIZE << dic->log_cluster_size;
758
759 if (dic->clen > PAGE_SIZE * dic->nr_cpages - COMPRESS_HEADER_SIZE) {
760 ret = -EFSCORRUPTED;
761
762 /* Avoid f2fs_commit_super in irq context */
763 f2fs_handle_error(sbi, ERROR_FAIL_DECOMPRESSION);
764 fserror_report_file_metadata(dic->inode, ret, GFP_NOFS);
765 goto out_release;
766 }
767
768 ret = cops->decompress_pages(dic);
769
770 if (!ret && (fi->i_compress_flag & BIT(COMPRESS_CHKSUM))) {
771 u32 provided = le32_to_cpu(dic->cbuf->chksum);
772 u32 calculated = f2fs_crc32(dic->cbuf->cdata, dic->clen);
773
774 if (provided != calculated) {
775 if (!is_inode_flag_set(dic->inode, FI_COMPRESS_CORRUPT)) {
776 set_inode_flag(dic->inode, FI_COMPRESS_CORRUPT);
777 f2fs_info_ratelimited(sbi,
778 "checksum invalid, nid = %llu, %x vs %x",
779 dic->inode->i_ino,
780 provided, calculated);
781 }
782 set_sbi_flag(sbi, SBI_NEED_FSCK);
783 }
784 }
785
786out_release:
787 f2fs_release_decomp_mem(dic, bypass_callback, false);
788
789out_end_io:
790 trace_f2fs_decompress_pages_end(dic->inode, dic->cluster_idx,
791 dic->clen, ret);
792 f2fs_decompress_end_io(dic, ret, in_task);
793}
794
795static void f2fs_cache_compressed_page(struct f2fs_sb_info *sbi,
796 struct folio *folio, nid_t ino, block_t blkaddr);
797
798/*
799 * This is called when a page of a compressed cluster has been read from disk
800 * (or failed to be read from disk). It checks whether this page was the last
801 * page being waited on in the cluster, and if so, it decompresses the cluster
802 * (or in the case of a failure, cleans up without actually decompressing).
803 */
804void f2fs_end_read_compressed_page(struct folio *folio, bool failed,
805 block_t blkaddr, bool in_task)
806{
807 struct decompress_io_ctx *dic = folio->private;
808 struct f2fs_sb_info *sbi = dic->sbi;
809
810 dec_page_count(sbi, F2FS_RD_DATA);
811
812 if (failed)
813 WRITE_ONCE(dic->failed, true);
814 else if (blkaddr && in_task)
815 f2fs_cache_compressed_page(sbi, folio,
816 dic->inode->i_ino, blkaddr);
817
818 if (atomic_dec_and_test(&dic->remaining_pages))
819 f2fs_decompress_cluster(dic, in_task);
820}
821
822static bool is_page_in_cluster(struct compress_ctx *cc, pgoff_t index)
823{
824 if (cc->cluster_idx == NULL_CLUSTER)
825 return true;
826 return cc->cluster_idx == cluster_idx(cc, index);
827}
828
829bool f2fs_cluster_is_empty(struct compress_ctx *cc)
830{
831 return cc->nr_rpages == 0;
832}
833
834static bool f2fs_cluster_is_full(struct compress_ctx *cc)
835{
836 return cc->cluster_size == cc->nr_rpages;
837}
838
839bool f2fs_cluster_can_merge_page(struct compress_ctx *cc, pgoff_t index)
840{
841 if (f2fs_cluster_is_empty(cc))
842 return true;
843 return is_page_in_cluster(cc, index);
844}
845
846bool f2fs_all_cluster_page_ready(struct compress_ctx *cc, struct page **pages,
847 int index, int nr_pages, bool uptodate)
848{
849 unsigned long pgidx = page_folio(pages[index])->index;
850 int i = uptodate ? 0 : 1;
851
852 /*
853 * when uptodate set to true, try to check all pages in cluster is
854 * uptodate or not.
855 */
856 if (uptodate && (pgidx % cc->cluster_size))
857 return false;
858
859 if (nr_pages - index < cc->cluster_size)
860 return false;
861
862 for (; i < cc->cluster_size; i++) {
863 struct folio *folio = page_folio(pages[index + i]);
864
865 if (folio->index != pgidx + i)
866 return false;
867 if (uptodate && !folio_test_uptodate(folio))
868 return false;
869 }
870
871 return true;
872}
873
874static bool cluster_has_invalid_data(struct compress_ctx *cc)
875{
876 loff_t i_size = i_size_read(cc->inode);
877 unsigned nr_pages = DIV_ROUND_UP(i_size, PAGE_SIZE);
878 int i;
879
880 for (i = 0; i < cc->cluster_size; i++) {
881 struct page *page = cc->rpages[i];
882
883 f2fs_bug_on(F2FS_I_SB(cc->inode), !page);
884
885 /* beyond EOF */
886 if (page_folio(page)->index >= nr_pages)
887 return true;
888 }
889 return false;
890}
891
892bool f2fs_sanity_check_cluster(struct dnode_of_data *dn)
893{
894#ifdef CONFIG_F2FS_CHECK_FS
895 struct f2fs_sb_info *sbi = F2FS_I_SB(dn->inode);
896 unsigned int cluster_size = F2FS_I(dn->inode)->i_cluster_size;
897 int cluster_end = 0;
898 unsigned int count;
899 int i;
900 char *reason = "";
901
902 if (dn->data_blkaddr != COMPRESS_ADDR)
903 return false;
904
905 /* [..., COMPR_ADDR, ...] */
906 if (dn->ofs_in_node % cluster_size) {
907 reason = "[*|C|*|*]";
908 goto out;
909 }
910
911 for (i = 1, count = 1; i < cluster_size; i++, count++) {
912 block_t blkaddr = data_blkaddr(dn->inode, dn->node_folio,
913 dn->ofs_in_node + i);
914
915 /* [COMPR_ADDR, ..., COMPR_ADDR] */
916 if (blkaddr == COMPRESS_ADDR) {
917 reason = "[C|*|C|*]";
918 goto out;
919 }
920 if (!__is_valid_data_blkaddr(blkaddr)) {
921 if (!cluster_end)
922 cluster_end = i;
923 continue;
924 }
925 /* [COMPR_ADDR, NULL_ADDR or NEW_ADDR, valid_blkaddr] */
926 if (cluster_end) {
927 reason = "[C|N|N|V]";
928 goto out;
929 }
930 }
931
932 f2fs_bug_on(F2FS_I_SB(dn->inode), count != cluster_size &&
933 !is_inode_flag_set(dn->inode, FI_COMPRESS_RELEASED));
934
935 return false;
936out:
937 f2fs_warn(sbi, "access invalid cluster, ino:%llu, nid:%u, ofs_in_node:%u, reason:%s",
938 dn->inode->i_ino, dn->nid, dn->ofs_in_node, reason);
939 set_sbi_flag(sbi, SBI_NEED_FSCK);
940 return true;
941#else
942 return false;
943#endif
944}
945
946static int __f2fs_get_cluster_blocks(struct inode *inode,
947 struct dnode_of_data *dn)
948{
949 unsigned int cluster_size = F2FS_I(inode)->i_cluster_size;
950 int count, i;
951
952 for (i = 0, count = 0; i < cluster_size; i++) {
953 block_t blkaddr = data_blkaddr(dn->inode, dn->node_folio,
954 dn->ofs_in_node + i);
955
956 if (__is_valid_data_blkaddr(blkaddr))
957 count++;
958 }
959
960 return count;
961}
962
963static int __f2fs_cluster_blocks(struct inode *inode, unsigned int cluster_idx,
964 enum cluster_check_type type)
965{
966 struct dnode_of_data dn;
967 unsigned int start_idx = cluster_idx <<
968 F2FS_I(inode)->i_log_cluster_size;
969 int ret;
970
971 set_new_dnode(&dn, inode, NULL, NULL, 0);
972 ret = f2fs_get_dnode_of_data(&dn, start_idx, LOOKUP_NODE);
973 if (ret) {
974 if (ret == -ENOENT)
975 ret = 0;
976 goto fail;
977 }
978
979 if (f2fs_sanity_check_cluster(&dn)) {
980 ret = -EFSCORRUPTED;
981 goto fail;
982 }
983
984 if (dn.data_blkaddr == COMPRESS_ADDR) {
985 if (type == CLUSTER_COMPR_BLKS)
986 ret = 1 + __f2fs_get_cluster_blocks(inode, &dn);
987 else if (type == CLUSTER_IS_COMPR)
988 ret = 1;
989 } else if (type == CLUSTER_RAW_BLKS) {
990 ret = __f2fs_get_cluster_blocks(inode, &dn);
991 }
992fail:
993 f2fs_put_dnode(&dn);
994 return ret;
995}
996
997/* return # of compressed blocks in compressed cluster */
998static int f2fs_compressed_blocks(struct compress_ctx *cc)
999{
1000 return __f2fs_cluster_blocks(cc->inode, cc->cluster_idx,
1001 CLUSTER_COMPR_BLKS);
1002}
1003
1004/* return # of raw blocks in non-compressed cluster */
1005static int f2fs_decompressed_blocks(struct inode *inode,
1006 unsigned int cluster_idx)
1007{
1008 return __f2fs_cluster_blocks(inode, cluster_idx,
1009 CLUSTER_RAW_BLKS);
1010}
1011
1012/* return whether cluster is compressed one or not */
1013int f2fs_is_compressed_cluster(struct inode *inode, pgoff_t index)
1014{
1015 return __f2fs_cluster_blocks(inode,
1016 index >> F2FS_I(inode)->i_log_cluster_size,
1017 CLUSTER_IS_COMPR);
1018}
1019
1020/* return whether cluster contains non raw blocks or not */
1021bool f2fs_is_sparse_cluster(struct inode *inode, pgoff_t index)
1022{
1023 unsigned int cluster_idx = index >> F2FS_I(inode)->i_log_cluster_size;
1024
1025 return f2fs_decompressed_blocks(inode, cluster_idx) !=
1026 F2FS_I(inode)->i_cluster_size;
1027}
1028
1029static bool cluster_may_compress(struct compress_ctx *cc)
1030{
1031 if (!f2fs_need_compress_data(cc->inode))
1032 return false;
1033 if (f2fs_is_atomic_file(cc->inode))
1034 return false;
1035 if (!f2fs_cluster_is_full(cc))
1036 return false;
1037 if (unlikely(f2fs_cp_error(F2FS_I_SB(cc->inode))))
1038 return false;
1039 return !cluster_has_invalid_data(cc);
1040}
1041
1042static void set_cluster_writeback(struct compress_ctx *cc)
1043{
1044 int i;
1045
1046 for (i = 0; i < cc->cluster_size; i++) {
1047 if (cc->rpages[i])
1048 set_page_writeback(cc->rpages[i]);
1049 }
1050}
1051
1052static void cancel_cluster_writeback(struct compress_ctx *cc,
1053 struct compress_io_ctx *cic, int submitted)
1054{
1055 int i;
1056
1057 /* Wait for submitted IOs. */
1058 if (submitted > 1) {
1059 f2fs_submit_merged_write(F2FS_I_SB(cc->inode), DATA);
1060 while (atomic_read(&cic->pending_pages) !=
1061 (cc->valid_nr_cpages - submitted + 1))
1062 f2fs_io_schedule_timeout(DEFAULT_SCHEDULE_TIMEOUT);
1063 }
1064
1065 /* Cancel writeback and stay locked. */
1066 for (i = 0; i < cc->cluster_size; i++) {
1067 if (i < submitted) {
1068 inode_inc_dirty_pages(cc->inode);
1069 lock_page(cc->rpages[i]);
1070 }
1071 clear_page_private_gcing(cc->rpages[i]);
1072 if (folio_test_writeback(page_folio(cc->rpages[i])))
1073 end_page_writeback(cc->rpages[i]);
1074 }
1075}
1076
1077static void set_cluster_dirty(struct compress_ctx *cc)
1078{
1079 int i;
1080
1081 for (i = 0; i < cc->cluster_size; i++)
1082 if (cc->rpages[i]) {
1083 set_page_dirty(cc->rpages[i]);
1084 set_page_private_gcing(cc->rpages[i]);
1085 }
1086}
1087
1088static int prepare_compress_overwrite(struct compress_ctx *cc,
1089 struct page **pagep, pgoff_t index, void **fsdata)
1090{
1091 struct f2fs_sb_info *sbi = F2FS_I_SB(cc->inode);
1092 struct address_space *mapping = cc->inode->i_mapping;
1093 struct folio *folio;
1094 sector_t last_block_in_bio;
1095 fgf_t fgp_flag = FGP_LOCK | FGP_WRITE | FGP_CREAT;
1096 pgoff_t start_idx = start_idx_of_cluster(cc);
1097 int i, ret;
1098
1099retry:
1100 ret = f2fs_is_compressed_cluster(cc->inode, start_idx);
1101 if (ret <= 0)
1102 return ret;
1103
1104 ret = f2fs_init_compress_ctx(cc);
1105 if (ret)
1106 return ret;
1107
1108 /* keep folio reference to avoid page reclaim */
1109 for (i = 0; i < cc->cluster_size; i++) {
1110 folio = f2fs_filemap_get_folio(mapping, start_idx + i,
1111 fgp_flag, GFP_NOFS);
1112 if (IS_ERR(folio)) {
1113 ret = PTR_ERR(folio);
1114 goto unlock_pages;
1115 }
1116
1117 if (folio_test_uptodate(folio))
1118 f2fs_folio_put(folio, true);
1119 else
1120 f2fs_compress_ctx_add_page(cc, folio);
1121 }
1122
1123 if (!f2fs_cluster_is_empty(cc)) {
1124 struct bio *bio = NULL;
1125
1126 ret = f2fs_read_multi_pages(cc, &bio, cc->cluster_size,
1127 &last_block_in_bio, NULL, true);
1128 f2fs_put_rpages(cc);
1129 f2fs_destroy_compress_ctx(cc, true);
1130 if (ret)
1131 goto out;
1132 if (bio)
1133 f2fs_submit_read_bio(sbi, bio, DATA);
1134
1135 ret = f2fs_init_compress_ctx(cc);
1136 if (ret)
1137 goto out;
1138 }
1139
1140 for (i = 0; i < cc->cluster_size; i++) {
1141 f2fs_bug_on(sbi, cc->rpages[i]);
1142
1143 folio = filemap_lock_folio(mapping, start_idx + i);
1144 if (IS_ERR(folio)) {
1145 /* folio could be truncated */
1146 goto release_and_retry;
1147 }
1148
1149 f2fs_folio_wait_writeback(folio, DATA, true, true);
1150 f2fs_compress_ctx_add_page(cc, folio);
1151
1152 if (!folio_test_uptodate(folio)) {
1153 f2fs_handle_page_eio(sbi, folio, DATA);
1154release_and_retry:
1155 f2fs_put_rpages(cc);
1156 f2fs_unlock_rpages(cc, i + 1);
1157 f2fs_destroy_compress_ctx(cc, true);
1158 goto retry;
1159 }
1160 }
1161
1162 if (likely(!ret)) {
1163 *fsdata = cc->rpages;
1164 *pagep = cc->rpages[offset_in_cluster(cc, index)];
1165 return cc->cluster_size;
1166 }
1167
1168unlock_pages:
1169 f2fs_put_rpages(cc);
1170 f2fs_unlock_rpages(cc, i);
1171 f2fs_destroy_compress_ctx(cc, true);
1172out:
1173 return ret;
1174}
1175
1176int f2fs_prepare_compress_overwrite(struct inode *inode,
1177 struct page **pagep, pgoff_t index, void **fsdata)
1178{
1179 struct compress_ctx cc = {
1180 .inode = inode,
1181 .log_cluster_size = F2FS_I(inode)->i_log_cluster_size,
1182 .cluster_size = F2FS_I(inode)->i_cluster_size,
1183 .cluster_idx = index >> F2FS_I(inode)->i_log_cluster_size,
1184 .rpages = NULL,
1185 .nr_rpages = 0,
1186 .vi = NULL, /* can't write to fsverity files */
1187 };
1188
1189 return prepare_compress_overwrite(&cc, pagep, index, fsdata);
1190}
1191
1192bool f2fs_compress_write_end(struct inode *inode, void *fsdata,
1193 pgoff_t index, unsigned copied)
1194
1195{
1196 struct compress_ctx cc = {
1197 .inode = inode,
1198 .log_cluster_size = F2FS_I(inode)->i_log_cluster_size,
1199 .cluster_size = F2FS_I(inode)->i_cluster_size,
1200 .rpages = fsdata,
1201 };
1202 struct folio *folio = page_folio(cc.rpages[0]);
1203 bool first_index = (index == folio->index);
1204
1205 if (copied)
1206 set_cluster_dirty(&cc);
1207
1208 f2fs_put_rpages_wbc(&cc, NULL, false, true);
1209 f2fs_destroy_compress_ctx(&cc, false);
1210
1211 return first_index;
1212}
1213
1214int f2fs_truncate_partial_cluster(struct inode *inode, u64 from, bool lock)
1215{
1216 void *fsdata = NULL;
1217 struct page *pagep;
1218 struct page **rpages;
1219 int log_cluster_size = F2FS_I(inode)->i_log_cluster_size;
1220 pgoff_t start_idx = from >> (PAGE_SHIFT + log_cluster_size) <<
1221 log_cluster_size;
1222 int i;
1223 int err;
1224
1225 err = f2fs_is_compressed_cluster(inode, start_idx);
1226 if (err < 0)
1227 return err;
1228
1229 /* truncate normal cluster */
1230 if (!err)
1231 return f2fs_do_truncate_blocks(inode, from, lock);
1232
1233 /* truncate compressed cluster */
1234 err = f2fs_prepare_compress_overwrite(inode, &pagep,
1235 start_idx, &fsdata);
1236
1237 /* should not be a normal cluster */
1238 f2fs_bug_on(F2FS_I_SB(inode), err == 0);
1239
1240 if (err <= 0)
1241 return err;
1242
1243 rpages = fsdata;
1244
1245 for (i = (1 << log_cluster_size) - 1; i >= 0; i--) {
1246 struct folio *folio = page_folio(rpages[i]);
1247 loff_t start = (loff_t)folio->index << PAGE_SHIFT;
1248 loff_t offset = from > start ? from - start : 0;
1249
1250 folio_zero_segment(folio, offset, folio_size(folio));
1251
1252 if (from >= start)
1253 break;
1254 }
1255
1256 f2fs_compress_write_end(inode, fsdata, start_idx, true);
1257
1258 err = filemap_write_and_wait_range(inode->i_mapping,
1259 round_down(from, 1 << log_cluster_size << PAGE_SHIFT),
1260 LLONG_MAX);
1261 if (err)
1262 return err;
1263
1264 truncate_pagecache(inode, from);
1265
1266 return f2fs_do_truncate_blocks(inode, round_up(from, PAGE_SIZE), lock);
1267}
1268
1269static int f2fs_write_compressed_pages(struct compress_ctx *cc,
1270 int *submitted,
1271 struct writeback_control *wbc,
1272 enum iostat_type io_type)
1273{
1274 struct inode *inode = cc->inode;
1275 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
1276 struct f2fs_inode_info *fi = F2FS_I(inode);
1277 struct f2fs_io_info fio = {
1278 .sbi = sbi,
1279 .ino = cc->inode->i_ino,
1280 .type = DATA,
1281 .op = REQ_OP_WRITE,
1282 .op_flags = wbc_to_write_flags(wbc),
1283 .old_blkaddr = NEW_ADDR,
1284 .page = NULL,
1285 .encrypted_page = NULL,
1286 .compressed_page = NULL,
1287 .io_type = io_type,
1288 .io_wbc = wbc,
1289 };
1290 struct folio *folio;
1291 struct dnode_of_data dn;
1292 struct node_info ni;
1293 struct compress_io_ctx *cic;
1294 struct f2fs_lock_context lc;
1295 pgoff_t start_idx = start_idx_of_cluster(cc);
1296 unsigned int last_index = cc->cluster_size - 1;
1297 loff_t psize;
1298 int i, err;
1299 bool quota_inode = IS_NOQUOTA(inode);
1300
1301 /* we should bypass data pages to proceed the kworker jobs */
1302 if (unlikely(f2fs_cp_error(sbi))) {
1303 mapping_set_error(inode->i_mapping, -EIO);
1304 goto out_free;
1305 }
1306
1307 if (quota_inode) {
1308 /*
1309 * We need to wait for node_write to avoid block allocation during
1310 * checkpoint. This can only happen to quota writes which can cause
1311 * the below discard race condition.
1312 */
1313 f2fs_down_read_trace(&sbi->node_write, &lc);
1314 } else if (!f2fs_trylock_op(sbi, &lc)) {
1315 goto out_free;
1316 }
1317
1318 set_new_dnode(&dn, cc->inode, NULL, NULL, 0);
1319
1320 err = f2fs_get_dnode_of_data(&dn, start_idx, LOOKUP_NODE);
1321 if (err)
1322 goto out_unlock_op;
1323
1324 for (i = 0; i < cc->cluster_size; i++) {
1325 if (data_blkaddr(dn.inode, dn.node_folio,
1326 dn.ofs_in_node + i) == NULL_ADDR)
1327 goto out_put_dnode;
1328 }
1329
1330 folio = page_folio(cc->rpages[last_index]);
1331 psize = folio_next_pos(folio);
1332
1333 err = f2fs_get_node_info(fio.sbi, dn.nid, &ni, false);
1334 if (err)
1335 goto out_put_dnode;
1336
1337 fio.version = ni.version;
1338
1339 cic = f2fs_kmem_cache_alloc(cic_entry_slab, GFP_F2FS_ZERO, false, sbi);
1340 if (!cic)
1341 goto out_put_dnode;
1342
1343 cic->magic = F2FS_COMPRESSED_PAGE_MAGIC;
1344 cic->inode = inode;
1345 atomic_set(&cic->pending_pages, cc->valid_nr_cpages);
1346 cic->rpages = page_array_alloc(sbi, cc->cluster_size);
1347 if (!cic->rpages)
1348 goto out_put_cic;
1349
1350 cic->nr_rpages = cc->cluster_size;
1351
1352 for (i = 0; i < cc->valid_nr_cpages; i++) {
1353 f2fs_set_compressed_page(cc->cpages[i], inode,
1354 page_folio(cc->rpages[i + 1])->index, cic);
1355 fio.compressed_page = cc->cpages[i];
1356
1357 fio.old_blkaddr = data_blkaddr(dn.inode, dn.node_folio,
1358 dn.ofs_in_node + i + 1);
1359
1360 /* wait for GCed page writeback via META_MAPPING */
1361 f2fs_wait_on_block_writeback(inode, fio.old_blkaddr);
1362 }
1363
1364 set_cluster_writeback(cc);
1365
1366 for (i = 0; i < cc->cluster_size; i++)
1367 cic->rpages[i] = cc->rpages[i];
1368
1369 for (i = 0; i < cc->cluster_size; i++, dn.ofs_in_node++) {
1370 block_t blkaddr;
1371
1372 blkaddr = f2fs_data_blkaddr(&dn);
1373 fio.page = cc->rpages[i];
1374 fio.old_blkaddr = blkaddr;
1375
1376 /* cluster header */
1377 if (i == 0) {
1378 if (blkaddr == COMPRESS_ADDR)
1379 fio.compr_blocks++;
1380 if (__is_valid_data_blkaddr(blkaddr))
1381 f2fs_invalidate_blocks(sbi, blkaddr, 1);
1382 f2fs_update_data_blkaddr(&dn, COMPRESS_ADDR);
1383 goto unlock_continue;
1384 }
1385
1386 if (fio.compr_blocks && __is_valid_data_blkaddr(blkaddr))
1387 fio.compr_blocks++;
1388
1389 if (i > cc->valid_nr_cpages) {
1390 if (__is_valid_data_blkaddr(blkaddr)) {
1391 f2fs_invalidate_blocks(sbi, blkaddr, 1);
1392 f2fs_update_data_blkaddr(&dn, NEW_ADDR);
1393 }
1394 goto unlock_continue;
1395 }
1396
1397 f2fs_bug_on(fio.sbi, blkaddr == NULL_ADDR);
1398
1399 fio.compressed_page = cc->cpages[i - 1];
1400
1401 cc->cpages[i - 1] = NULL;
1402 fio.submitted = 0;
1403 f2fs_outplace_write_data(&dn, &fio);
1404 if (unlikely(!fio.submitted)) {
1405 cancel_cluster_writeback(cc, cic, i);
1406 *submitted = 0;
1407 goto out_free_page_array;
1408 }
1409 (*submitted)++;
1410unlock_continue:
1411 inode_dec_dirty_pages(cc->inode);
1412 folio_unlock(fio.folio);
1413 }
1414
1415 if (fio.compr_blocks)
1416 f2fs_i_compr_blocks_update(inode, fio.compr_blocks - 1, false);
1417 f2fs_i_compr_blocks_update(inode, cc->valid_nr_cpages, true);
1418 add_compr_block_stat(inode, cc->valid_nr_cpages);
1419
1420 set_inode_flag(cc->inode, FI_APPEND_WRITE);
1421
1422 f2fs_put_dnode(&dn);
1423 if (quota_inode)
1424 f2fs_up_read_trace(&sbi->node_write, &lc);
1425 else
1426 f2fs_unlock_op(sbi, &lc);
1427
1428 spin_lock(&fi->i_size_lock);
1429 if (fi->last_disk_size < psize)
1430 fi->last_disk_size = psize;
1431 spin_unlock(&fi->i_size_lock);
1432
1433 f2fs_put_rpages(cc);
1434 page_array_free(sbi, cc->cpages, cc->nr_cpages);
1435 cc->cpages = NULL;
1436 f2fs_destroy_compress_ctx(cc, false);
1437 return 0;
1438
1439out_free_page_array:
1440 page_array_free(sbi, cic->rpages, cc->cluster_size);
1441out_put_cic:
1442 kmem_cache_free(cic_entry_slab, cic);
1443out_put_dnode:
1444 f2fs_put_dnode(&dn);
1445out_unlock_op:
1446 if (quota_inode)
1447 f2fs_up_read_trace(&sbi->node_write, &lc);
1448 else
1449 f2fs_unlock_op(sbi, &lc);
1450out_free:
1451 for (i = 0; i < cc->valid_nr_cpages; i++) {
1452 f2fs_compress_free_page(cc->cpages[i]);
1453 cc->cpages[i] = NULL;
1454 }
1455 page_array_free(sbi, cc->cpages, cc->nr_cpages);
1456 cc->cpages = NULL;
1457 return -EAGAIN;
1458}
1459
1460void f2fs_compress_write_end_io(struct bio *bio, struct folio *folio)
1461{
1462 struct page *page = &folio->page;
1463 struct f2fs_sb_info *sbi = bio->bi_private;
1464 struct compress_io_ctx *cic = folio->private;
1465 enum count_type type = WB_DATA_TYPE(folio, true);
1466 int i;
1467
1468 if (unlikely(bio->bi_status != BLK_STS_OK))
1469 mapping_set_error(cic->inode->i_mapping, -EIO);
1470
1471 f2fs_compress_free_page(page);
1472
1473 if (atomic_dec_return(&cic->pending_pages)) {
1474 dec_page_count(sbi, type);
1475 return;
1476 }
1477
1478 for (i = 0; i < cic->nr_rpages; i++) {
1479 WARN_ON(!cic->rpages[i]);
1480 clear_page_private_gcing(cic->rpages[i]);
1481 end_page_writeback(cic->rpages[i]);
1482 }
1483
1484 page_array_free(sbi, cic->rpages, cic->nr_rpages);
1485 kmem_cache_free(cic_entry_slab, cic);
1486
1487 /*
1488 * Make sure dec_page_count() is the last access to sbi.
1489 * Once it drops the F2FS_WB_CP_DATA counter to zero, the
1490 * unmount thread can proceed to destroy sbi and
1491 * sbi->page_array_slab.
1492 */
1493 dec_page_count(sbi, type);
1494}
1495
1496static int f2fs_write_raw_pages(struct compress_ctx *cc,
1497 int *submitted_p,
1498 struct writeback_control *wbc,
1499 enum iostat_type io_type)
1500{
1501 struct address_space *mapping = cc->inode->i_mapping;
1502 struct f2fs_sb_info *sbi = F2FS_M_SB(mapping);
1503 struct f2fs_lock_context lc;
1504 int submitted, compr_blocks, i;
1505 int ret = 0;
1506
1507 compr_blocks = f2fs_compressed_blocks(cc);
1508
1509 for (i = 0; i < cc->cluster_size; i++) {
1510 if (!cc->rpages[i])
1511 continue;
1512
1513 redirty_page_for_writepage(wbc, cc->rpages[i]);
1514 unlock_page(cc->rpages[i]);
1515 }
1516
1517 if (compr_blocks < 0)
1518 return compr_blocks;
1519
1520 /* overwrite compressed cluster w/ normal cluster */
1521 if (compr_blocks > 0)
1522 f2fs_lock_op(sbi, &lc);
1523
1524 for (i = 0; i < cc->cluster_size; i++) {
1525 struct folio *folio;
1526
1527 if (!cc->rpages[i])
1528 continue;
1529 folio = page_folio(cc->rpages[i]);
1530retry_write:
1531 folio_lock(folio);
1532
1533 if (folio->mapping != mapping) {
1534continue_unlock:
1535 folio_unlock(folio);
1536 continue;
1537 }
1538
1539 if (!folio_test_dirty(folio))
1540 goto continue_unlock;
1541
1542 if (folio_test_writeback(folio)) {
1543 if (wbc->sync_mode == WB_SYNC_NONE)
1544 goto continue_unlock;
1545 f2fs_folio_wait_writeback(folio, DATA, true, true);
1546 }
1547
1548 if (!folio_clear_dirty_for_io(folio))
1549 goto continue_unlock;
1550
1551 submitted = 0;
1552 ret = f2fs_write_single_data_page(folio, &submitted,
1553 NULL, NULL, wbc, io_type,
1554 compr_blocks, false);
1555 if (ret) {
1556 if (ret == 1) {
1557 ret = 0;
1558 } else if (ret == -EAGAIN) {
1559 ret = 0;
1560 /*
1561 * for quota file, just redirty left pages to
1562 * avoid deadlock caused by cluster update race
1563 * from foreground operation.
1564 */
1565 if (IS_NOQUOTA(cc->inode))
1566 goto out;
1567 f2fs_schedule_timeout(DEFAULT_SCHEDULE_TIMEOUT);
1568 goto retry_write;
1569 }
1570 goto out;
1571 }
1572
1573 *submitted_p += submitted;
1574 }
1575
1576out:
1577 if (compr_blocks > 0)
1578 f2fs_unlock_op(sbi, &lc);
1579
1580 f2fs_balance_fs(sbi, true);
1581 return ret;
1582}
1583
1584int f2fs_write_multi_pages(struct compress_ctx *cc,
1585 int *submitted,
1586 struct writeback_control *wbc,
1587 enum iostat_type io_type)
1588{
1589 int err;
1590
1591 *submitted = 0;
1592 if (cluster_may_compress(cc)) {
1593 err = f2fs_compress_pages(cc);
1594 if (err == -EAGAIN) {
1595 add_compr_block_stat(cc->inode, cc->cluster_size);
1596 goto write;
1597 } else if (err) {
1598 f2fs_put_rpages_wbc(cc, wbc, true, true);
1599 goto destroy_out;
1600 }
1601
1602 err = f2fs_write_compressed_pages(cc, submitted,
1603 wbc, io_type);
1604 if (!err)
1605 return 0;
1606 f2fs_bug_on(F2FS_I_SB(cc->inode), err != -EAGAIN);
1607 }
1608write:
1609 f2fs_bug_on(F2FS_I_SB(cc->inode), *submitted);
1610
1611 err = f2fs_write_raw_pages(cc, submitted, wbc, io_type);
1612 f2fs_put_rpages_wbc(cc, wbc, false, false);
1613destroy_out:
1614 f2fs_destroy_compress_ctx(cc, false);
1615 return err;
1616}
1617
1618static inline bool allow_memalloc_for_decomp(struct f2fs_sb_info *sbi,
1619 bool pre_alloc)
1620{
1621 return pre_alloc ^ f2fs_low_mem_mode(sbi);
1622}
1623
1624static int f2fs_prepare_decomp_mem(struct decompress_io_ctx *dic,
1625 bool pre_alloc)
1626{
1627 const struct f2fs_compress_ops *cops = f2fs_cops[dic->compress_algorithm];
1628 int i;
1629
1630 if (!allow_memalloc_for_decomp(dic->sbi, pre_alloc))
1631 return 0;
1632
1633 dic->tpages = page_array_alloc(dic->sbi, dic->cluster_size);
1634 if (!dic->tpages)
1635 return -ENOMEM;
1636
1637 for (i = 0; i < dic->cluster_size; i++) {
1638 if (dic->rpages[i]) {
1639 dic->tpages[i] = dic->rpages[i];
1640 continue;
1641 }
1642
1643 dic->tpages[i] = f2fs_compress_alloc_page();
1644 }
1645
1646 dic->rbuf = f2fs_vmap(dic->tpages, dic->cluster_size);
1647 if (!dic->rbuf)
1648 return -ENOMEM;
1649
1650 dic->cbuf = f2fs_vmap(dic->cpages, dic->nr_cpages);
1651 if (!dic->cbuf)
1652 return -ENOMEM;
1653
1654 if (cops->init_decompress_ctx)
1655 return cops->init_decompress_ctx(dic);
1656
1657 return 0;
1658}
1659
1660static void f2fs_release_decomp_mem(struct decompress_io_ctx *dic,
1661 bool bypass_destroy_callback, bool pre_alloc)
1662{
1663 const struct f2fs_compress_ops *cops = f2fs_cops[dic->compress_algorithm];
1664
1665 if (!allow_memalloc_for_decomp(dic->sbi, pre_alloc))
1666 return;
1667
1668 if (!bypass_destroy_callback && cops->destroy_decompress_ctx)
1669 cops->destroy_decompress_ctx(dic);
1670
1671 if (dic->cbuf)
1672 vm_unmap_ram(dic->cbuf, dic->nr_cpages);
1673
1674 if (dic->rbuf)
1675 vm_unmap_ram(dic->rbuf, dic->cluster_size);
1676}
1677
1678static void f2fs_free_dic(struct decompress_io_ctx *dic,
1679 bool bypass_destroy_callback);
1680
1681struct decompress_io_ctx *f2fs_alloc_dic(struct compress_ctx *cc)
1682{
1683 struct decompress_io_ctx *dic;
1684 pgoff_t start_idx = start_idx_of_cluster(cc);
1685 struct f2fs_sb_info *sbi = F2FS_I_SB(cc->inode);
1686 int i, ret;
1687
1688 dic = f2fs_kmem_cache_alloc(dic_entry_slab, GFP_F2FS_ZERO, false, sbi);
1689 if (!dic)
1690 return ERR_PTR(-ENOMEM);
1691
1692 dic->rpages = page_array_alloc(sbi, cc->cluster_size);
1693 if (!dic->rpages) {
1694 kmem_cache_free(dic_entry_slab, dic);
1695 return ERR_PTR(-ENOMEM);
1696 }
1697
1698 dic->magic = F2FS_COMPRESSED_PAGE_MAGIC;
1699 dic->inode = cc->inode;
1700 dic->sbi = sbi;
1701 dic->compress_algorithm = F2FS_I(cc->inode)->i_compress_algorithm;
1702 atomic_set(&dic->remaining_pages, cc->nr_cpages);
1703 dic->cluster_idx = cc->cluster_idx;
1704 dic->cluster_size = cc->cluster_size;
1705 dic->log_cluster_size = cc->log_cluster_size;
1706 dic->nr_cpages = cc->nr_cpages;
1707 refcount_set(&dic->refcnt, 1);
1708 dic->failed = false;
1709 dic->vi = cc->vi;
1710
1711 for (i = 0; i < dic->cluster_size; i++)
1712 dic->rpages[i] = cc->rpages[i];
1713 dic->nr_rpages = cc->cluster_size;
1714
1715 dic->cpages = page_array_alloc(sbi, dic->nr_cpages);
1716 if (!dic->cpages) {
1717 ret = -ENOMEM;
1718 goto out_free;
1719 }
1720
1721 for (i = 0; i < dic->nr_cpages; i++) {
1722 struct page *page;
1723
1724 page = f2fs_compress_alloc_page();
1725 f2fs_set_compressed_page(page, cc->inode,
1726 start_idx + i + 1, dic);
1727 dic->cpages[i] = page;
1728 }
1729
1730 ret = f2fs_prepare_decomp_mem(dic, true);
1731 if (ret)
1732 goto out_free;
1733
1734 return dic;
1735
1736out_free:
1737 f2fs_free_dic(dic, true);
1738 return ERR_PTR(ret);
1739}
1740
1741static void f2fs_free_dic(struct decompress_io_ctx *dic,
1742 bool bypass_destroy_callback)
1743{
1744 int i;
1745 /* use sbi in dic to avoid UFA of dic->inode*/
1746 struct f2fs_sb_info *sbi = dic->sbi;
1747
1748 f2fs_release_decomp_mem(dic, bypass_destroy_callback, true);
1749
1750 if (dic->tpages) {
1751 for (i = 0; i < dic->cluster_size; i++) {
1752 if (dic->rpages[i])
1753 continue;
1754 if (!dic->tpages[i])
1755 continue;
1756 f2fs_compress_free_page(dic->tpages[i]);
1757 }
1758 page_array_free(sbi, dic->tpages, dic->cluster_size);
1759 }
1760
1761 if (dic->cpages) {
1762 for (i = 0; i < dic->nr_cpages; i++) {
1763 if (!dic->cpages[i])
1764 continue;
1765 f2fs_compress_free_page(dic->cpages[i]);
1766 }
1767 page_array_free(sbi, dic->cpages, dic->nr_cpages);
1768 }
1769
1770 page_array_free(sbi, dic->rpages, dic->nr_rpages);
1771 kmem_cache_free(dic_entry_slab, dic);
1772}
1773
1774static void f2fs_late_free_dic(struct work_struct *work)
1775{
1776 struct decompress_io_ctx *dic =
1777 container_of(work, struct decompress_io_ctx, free_work);
1778
1779 f2fs_free_dic(dic, false);
1780}
1781
1782static void f2fs_put_dic(struct decompress_io_ctx *dic, bool in_task)
1783{
1784 if (refcount_dec_and_test(&dic->refcnt)) {
1785 if (in_task) {
1786 f2fs_free_dic(dic, false);
1787 } else {
1788 INIT_WORK(&dic->free_work, f2fs_late_free_dic);
1789 queue_work(dic->sbi->wq, &dic->free_work);
1790 }
1791 }
1792}
1793
1794static void f2fs_verify_cluster(struct work_struct *work)
1795{
1796 struct decompress_io_ctx *dic =
1797 container_of(work, struct decompress_io_ctx, verity_work);
1798 int i;
1799
1800 /* Verify, update, and unlock the decompressed pages. */
1801 for (i = 0; i < dic->cluster_size; i++) {
1802 struct page *rpage = dic->rpages[i];
1803 struct folio *rfolio;
1804
1805 if (!rpage)
1806 continue;
1807 rfolio = page_folio(rpage);
1808 if (fsverity_verify_folio(dic->vi, rfolio))
1809 folio_mark_uptodate(rfolio);
1810 folio_unlock(rfolio);
1811 }
1812
1813 f2fs_put_dic(dic, true);
1814}
1815
1816/*
1817 * This is called when a compressed cluster has been decompressed
1818 * (or failed to be read and/or decompressed).
1819 */
1820void f2fs_decompress_end_io(struct decompress_io_ctx *dic, bool failed,
1821 bool in_task)
1822{
1823 int i;
1824
1825 if (IS_ENABLED(CONFIG_FS_VERITY) && !failed && dic->vi) {
1826 /*
1827 * Note that to avoid deadlocks, the verity work can't be done
1828 * on the decompression workqueue. This is because verifying
1829 * the data pages can involve reading metadata pages from the
1830 * file, and these metadata pages may be compressed.
1831 */
1832 INIT_WORK(&dic->verity_work, f2fs_verify_cluster);
1833 fsverity_enqueue_verify_work(&dic->verity_work);
1834 return;
1835 }
1836
1837 /* Update and unlock the cluster's pagecache pages. */
1838 for (i = 0; i < dic->cluster_size; i++) {
1839 struct page *rpage = dic->rpages[i];
1840
1841 if (!rpage)
1842 continue;
1843
1844 if (failed)
1845 ClearPageUptodate(rpage);
1846 else
1847 SetPageUptodate(rpage);
1848 unlock_page(rpage);
1849 }
1850
1851 /*
1852 * Release the reference to the decompress_io_ctx that was being held
1853 * for I/O completion.
1854 */
1855 f2fs_put_dic(dic, in_task);
1856}
1857
1858/*
1859 * Put a reference to a compressed folio's decompress_io_ctx.
1860 *
1861 * This is called when the folio is no longer needed and can be freed.
1862 */
1863void f2fs_put_folio_dic(struct folio *folio, bool in_task)
1864{
1865 struct decompress_io_ctx *dic = folio->private;
1866
1867 f2fs_put_dic(dic, in_task);
1868}
1869
1870/*
1871 * check whether cluster blocks are contiguous, and add extent cache entry
1872 * only if cluster blocks are logically and physically contiguous.
1873 */
1874unsigned int f2fs_cluster_blocks_are_contiguous(struct dnode_of_data *dn,
1875 unsigned int ofs_in_node)
1876{
1877 bool compressed = data_blkaddr(dn->inode, dn->node_folio,
1878 ofs_in_node) == COMPRESS_ADDR;
1879 int i = compressed ? 1 : 0;
1880 block_t first_blkaddr = data_blkaddr(dn->inode, dn->node_folio,
1881 ofs_in_node + i);
1882
1883 for (i += 1; i < F2FS_I(dn->inode)->i_cluster_size; i++) {
1884 block_t blkaddr = data_blkaddr(dn->inode, dn->node_folio,
1885 ofs_in_node + i);
1886
1887 if (!__is_valid_data_blkaddr(blkaddr))
1888 break;
1889 if (first_blkaddr + i - (compressed ? 1 : 0) != blkaddr)
1890 return 0;
1891 }
1892
1893 return compressed ? i - 1 : i;
1894}
1895
1896const struct address_space_operations f2fs_compress_aops = {
1897 .release_folio = f2fs_release_folio,
1898 .invalidate_folio = f2fs_invalidate_folio,
1899 .migrate_folio = filemap_migrate_folio,
1900};
1901
1902struct address_space *COMPRESS_MAPPING(struct f2fs_sb_info *sbi)
1903{
1904 return sbi->compress_inode->i_mapping;
1905}
1906
1907void f2fs_invalidate_compress_pages_range(struct f2fs_sb_info *sbi,
1908 block_t blkaddr, unsigned int len)
1909{
1910 if (!sbi->compress_inode)
1911 return;
1912 invalidate_mapping_pages(COMPRESS_MAPPING(sbi), blkaddr, blkaddr + len - 1);
1913}
1914
1915static void f2fs_cache_compressed_page(struct f2fs_sb_info *sbi,
1916 struct folio *folio, nid_t ino, block_t blkaddr)
1917{
1918 struct folio *cfolio;
1919 int ret;
1920
1921 if (!test_opt(sbi, COMPRESS_CACHE))
1922 return;
1923
1924 if (!f2fs_is_valid_blkaddr(sbi, blkaddr, DATA_GENERIC_ENHANCE_READ))
1925 return;
1926
1927 if (!f2fs_available_free_memory(sbi, COMPRESS_PAGE))
1928 return;
1929
1930 cfolio = filemap_get_folio(COMPRESS_MAPPING(sbi), blkaddr);
1931 if (!IS_ERR(cfolio)) {
1932 f2fs_folio_put(cfolio, false);
1933 return;
1934 }
1935
1936 cfolio = filemap_alloc_folio(__GFP_NOWARN | __GFP_IO, 0, NULL);
1937 if (!cfolio)
1938 return;
1939
1940 ret = filemap_add_folio(COMPRESS_MAPPING(sbi), cfolio,
1941 blkaddr, GFP_NOFS);
1942 if (ret) {
1943 f2fs_folio_put(cfolio, false);
1944 return;
1945 }
1946
1947 folio_set_f2fs_data(cfolio, ino);
1948
1949 memcpy(folio_address(cfolio), folio_address(folio), PAGE_SIZE);
1950 folio_mark_uptodate(cfolio);
1951 f2fs_folio_put(cfolio, true);
1952}
1953
1954bool f2fs_load_compressed_folio(struct f2fs_sb_info *sbi, struct folio *folio,
1955 block_t blkaddr)
1956{
1957 struct folio *cfolio;
1958 bool hitted = false;
1959
1960 if (!test_opt(sbi, COMPRESS_CACHE))
1961 return false;
1962
1963 cfolio = f2fs_filemap_get_folio(COMPRESS_MAPPING(sbi),
1964 blkaddr, FGP_LOCK | FGP_NOWAIT, GFP_NOFS);
1965 if (!IS_ERR(cfolio)) {
1966 if (folio_test_uptodate(cfolio)) {
1967 atomic_inc(&sbi->compress_page_hit);
1968 memcpy(folio_address(folio),
1969 folio_address(cfolio), folio_size(folio));
1970 hitted = true;
1971 }
1972 f2fs_folio_put(cfolio, true);
1973 }
1974
1975 return hitted;
1976}
1977
1978void f2fs_invalidate_compress_pages(struct f2fs_sb_info *sbi, nid_t ino)
1979{
1980 struct address_space *mapping = COMPRESS_MAPPING(sbi);
1981 struct folio_batch fbatch;
1982 pgoff_t index = 0;
1983 pgoff_t end = MAX_BLKADDR(sbi);
1984
1985 if (!mapping->nrpages)
1986 return;
1987
1988 folio_batch_init(&fbatch);
1989
1990 do {
1991 unsigned int nr, i;
1992
1993 nr = filemap_get_folios(mapping, &index, end - 1, &fbatch);
1994 if (!nr)
1995 break;
1996
1997 for (i = 0; i < nr; i++) {
1998 struct folio *folio = fbatch.folios[i];
1999
2000 folio_lock(folio);
2001 if (folio->mapping != mapping) {
2002 folio_unlock(folio);
2003 continue;
2004 }
2005
2006 if (ino != folio_get_f2fs_data(folio)) {
2007 folio_unlock(folio);
2008 continue;
2009 }
2010
2011 generic_error_remove_folio(mapping, folio);
2012 folio_unlock(folio);
2013 }
2014 folio_batch_release(&fbatch);
2015 cond_resched();
2016 } while (index < end);
2017}
2018
2019int f2fs_init_compress_inode(struct f2fs_sb_info *sbi)
2020{
2021 struct inode *inode;
2022
2023 if (!test_opt(sbi, COMPRESS_CACHE))
2024 return 0;
2025
2026 inode = f2fs_iget(sbi->sb, F2FS_COMPRESS_INO(sbi));
2027 if (IS_ERR(inode))
2028 return PTR_ERR(inode);
2029 sbi->compress_inode = inode;
2030
2031 sbi->compress_percent = COMPRESS_PERCENT;
2032 sbi->compress_watermark = COMPRESS_WATERMARK;
2033
2034 atomic_set(&sbi->compress_page_hit, 0);
2035
2036 return 0;
2037}
2038
2039void f2fs_destroy_compress_inode(struct f2fs_sb_info *sbi)
2040{
2041 if (!sbi->compress_inode)
2042 return;
2043 iput(sbi->compress_inode);
2044 sbi->compress_inode = NULL;
2045}
2046
2047int f2fs_init_page_array_cache(struct f2fs_sb_info *sbi)
2048{
2049 dev_t dev = sbi->sb->s_bdev->bd_dev;
2050 char slab_name[35];
2051
2052 if (!f2fs_sb_has_compression(sbi))
2053 return 0;
2054
2055 sprintf(slab_name, "f2fs_page_array_entry-%u:%u", MAJOR(dev), MINOR(dev));
2056
2057 sbi->page_array_slab_size = sizeof(struct page *) <<
2058 F2FS_OPTION(sbi).compress_log_size;
2059
2060 sbi->page_array_slab = f2fs_kmem_cache_create(slab_name,
2061 sbi->page_array_slab_size);
2062 return sbi->page_array_slab ? 0 : -ENOMEM;
2063}
2064
2065void f2fs_destroy_page_array_cache(struct f2fs_sb_info *sbi)
2066{
2067 kmem_cache_destroy(sbi->page_array_slab);
2068}
2069
2070int __init f2fs_init_compress_cache(void)
2071{
2072 cic_entry_slab = f2fs_kmem_cache_create("f2fs_cic_entry",
2073 sizeof(struct compress_io_ctx));
2074 if (!cic_entry_slab)
2075 return -ENOMEM;
2076 dic_entry_slab = f2fs_kmem_cache_create("f2fs_dic_entry",
2077 sizeof(struct decompress_io_ctx));
2078 if (!dic_entry_slab)
2079 goto free_cic;
2080 return 0;
2081free_cic:
2082 kmem_cache_destroy(cic_entry_slab);
2083 return -ENOMEM;
2084}
2085
2086void f2fs_destroy_compress_cache(void)
2087{
2088 kmem_cache_destroy(dic_entry_slab);
2089 kmem_cache_destroy(cic_entry_slab);
2090}