projects/linux·blob master · refs

486 lines · 13.2 KiB

1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Copyright (C) 2019 HUAWEI, Inc.
4 * https://www.huawei.com/
5 * Copyright (C) 2024 Alibaba Cloud
6 */
7#include "compress.h"
8#include <linux/lz4.h>
9
10static int z_erofs_load_lz4_config(struct super_block *sb,
11 struct erofs_super_block *dsb, void *data, int size)
12{
13 struct erofs_sb_info *sbi = EROFS_SB(sb);
14 struct z_erofs_lz4_cfgs *lz4 = data;
15 u16 distance;
16
17 if (lz4) {
18 if (size < sizeof(struct z_erofs_lz4_cfgs)) {
19 erofs_err(sb, "invalid lz4 cfgs, size=%u", size);
20 return -EINVAL;
21 }
22 sbi->lz4.max_pclusterblks = le16_to_cpu(lz4->max_pclusterblks);
23 if (!sbi->lz4.max_pclusterblks) {
24 sbi->lz4.max_pclusterblks = 1; /* reserved case */
25 } else if (sbi->lz4.max_pclusterblks >
26 Z_EROFS_PCLUSTER_MAX_SIZE >> sbi->blkszbits) {
27 erofs_err(sb, "too large lz4 pclusterblks %u",
28 sbi->lz4.max_pclusterblks);
29 return -EINVAL;
30 }
31 } else {
32 distance = le16_to_cpu(dsb->u1.lz4_max_distance);
33 if (!distance && !erofs_sb_has_lz4_0padding(sbi))
34 return 0;
35 sbi->lz4.max_pclusterblks = 1;
36 sbi->available_compr_algs = 1 << Z_EROFS_COMPRESSION_LZ4;
37 }
38 return z_erofs_gbuf_growsize(sbi->lz4.max_pclusterblks);
39}
40
41/*
42 * Fill all gaps with bounce pages if it's a sparse page list (for example some
43 * folios are already uptodate and thus can be mapped into userspace). Also
44 * check if pages are physically consecutive, which can be seen for moderate CR.
45 */
46static int z_erofs_oneshot_prepare_dstpages(struct z_erofs_decompress_req *rq,
47 struct page **pagepool)
48{
49 void *kaddr = NULL;
50 unsigned int i;
51
52 for (i = 0; i < rq->outpages; ++i) {
53 struct page *page, *victim;
54
55 page = rq->out[i];
56 if (page) {
57 if (!PageHighMem(page)) {
58 if (!i) {
59 kaddr = page_address(page);
60 continue;
61 }
62 if (kaddr &&
63 kaddr + PAGE_SIZE == page_address(page)) {
64 kaddr += PAGE_SIZE;
65 continue;
66 }
67 }
68 } else {
69 victim = __erofs_allocpage(pagepool, rq->gfp, true);
70 if (!victim)
71 return -ENOMEM;
72 set_page_private(victim, Z_EROFS_SHORTLIVED_PAGE);
73 rq->out[i] = victim;
74 }
75 kaddr = NULL;
76 }
77 return kaddr ? 1 : 0;
78}
79
80static void *z_erofs_lz4_handle_overlap(const struct z_erofs_decompress_req *rq,
81 void *inpage, void *out, unsigned int *inputmargin,
82 int *maptype, bool may_inplace)
83{
84 unsigned int oend, omargin, cnt, i;
85 struct page **in;
86 void *src;
87
88 /*
89 * If in-place I/O isn't used, for example, the bounce compressed cache
90 * can hold data for incomplete read requests. Just map the compressed
91 * buffer as well and decompress directly.
92 */
93 if (!rq->inplace_io) {
94 if (rq->inpages <= 1) {
95 *maptype = 0;
96 return inpage;
97 }
98 kunmap_local(inpage);
99 src = erofs_vm_map_ram(rq->in, rq->inpages);
100 if (!src)
101 return ERR_PTR(-ENOMEM);
102 *maptype = 1;
103 return src;
104 }
105 /*
106 * Then, deal with in-place I/Os. The reasons why in-place I/O is useful
107 * are: (1) It minimizes memory footprint during the I/O submission,
108 * which is useful for slow storage (including network devices and
109 * low-end HDDs/eMMCs) but with a lot inflight I/Os; (2) If in-place
110 * decompression can also be applied, it will reuse the unique buffer so
111 * that no extra CPU D-cache is polluted with temporary compressed data
112 * for extreme performance.
113 */
114 oend = rq->pageofs_out + rq->outputsize;
115 omargin = PAGE_ALIGN(oend) - oend;
116 if (!rq->partial_decoding && may_inplace &&
117 omargin >= LZ4_DECOMPRESS_INPLACE_MARGIN(rq->inputsize)) {
118 for (i = 0; i < rq->inpages; ++i)
119 if (rq->out[rq->outpages - rq->inpages + i] !=
120 rq->in[i])
121 break;
122 if (i >= rq->inpages) {
123 kunmap_local(inpage);
124 *maptype = 3;
125 return out + ((rq->outpages - rq->inpages) << PAGE_SHIFT);
126 }
127 }
128 /*
129 * If in-place decompression can't be applied, copy compressed data that
130 * may potentially overlap during decompression to a per-CPU buffer.
131 */
132 src = z_erofs_get_gbuf(rq->inpages);
133 if (!src) {
134 DBG_BUGON(1);
135 kunmap_local(inpage);
136 return ERR_PTR(-EFAULT);
137 }
138
139 for (i = 0, in = rq->in; i < rq->inputsize; i += cnt, ++in) {
140 cnt = min_t(u32, rq->inputsize - i, PAGE_SIZE - *inputmargin);
141 if (!inpage)
142 inpage = kmap_local_page(*in);
143 memcpy(src + i, inpage + *inputmargin, cnt);
144 kunmap_local(inpage);
145 inpage = NULL;
146 *inputmargin = 0;
147 }
148 *maptype = 2;
149 return src;
150}
151
152/*
153 * Get the exact on-disk size of the compressed data:
154 * - For LZ4, it should apply if the zero_padding feature is on (5.3+);
155 * - For others, zero_padding is enabled all the time.
156 */
157const char *z_erofs_fixup_insize(struct z_erofs_decompress_req *rq,
158 const char *padbuf, unsigned int padbufsize)
159{
160 const char *padend;
161
162 padend = memchr_inv(padbuf, 0, padbufsize);
163 if (!padend)
164 return "compressed data start not found";
165 rq->inputsize -= padend - padbuf;
166 rq->pageofs_in += padend - padbuf;
167 return NULL;
168}
169
170static const char *__z_erofs_lz4_decompress(struct z_erofs_decompress_req *rq,
171 u8 *dst)
172{
173 bool may_inplace = false;
174 unsigned int inputmargin;
175 u8 *out, *headpage, *src;
176 const char *reason;
177 int ret, maptype;
178
179 headpage = kmap_local_page(*rq->in);
180 reason = z_erofs_fixup_insize(rq, headpage + rq->pageofs_in,
181 min_t(unsigned int, rq->inputsize,
182 rq->sb->s_blocksize - rq->pageofs_in));
183 if (reason) {
184 kunmap_local(headpage);
185 return reason;
186 }
187 may_inplace = !((rq->pageofs_in + rq->inputsize) &
188 (rq->sb->s_blocksize - 1));
189
190 inputmargin = rq->pageofs_in;
191 src = z_erofs_lz4_handle_overlap(rq, headpage, dst, &inputmargin,
192 &maptype, may_inplace);
193 if (IS_ERR(src))
194 return ERR_CAST(src);
195
196 out = dst + rq->pageofs_out;
197 if (rq->partial_decoding)
198 ret = LZ4_decompress_safe_partial(src + inputmargin, out,
199 rq->inputsize, rq->outputsize, rq->outputsize);
200 else
201 ret = LZ4_decompress_safe(src + inputmargin, out,
202 rq->inputsize, rq->outputsize);
203 if (ret == rq->outputsize)
204 reason = NULL;
205 else if (ret < 0)
206 reason = "corrupted compressed data";
207 else
208 reason = "unexpected end of stream";
209
210 if (!maptype) {
211 kunmap_local(headpage);
212 } else if (maptype == 1) {
213 vm_unmap_ram(src, rq->inpages);
214 } else if (maptype == 2) {
215 z_erofs_put_gbuf(src);
216 } else if (maptype != 3) {
217 DBG_BUGON(1);
218 return ERR_PTR(-EFAULT);
219 }
220 return reason;
221}
222
223static const char *z_erofs_lz4_decompress(struct z_erofs_decompress_req *rq,
224 struct page **pagepool)
225{
226 unsigned int dst_maptype;
227 const char *reason;
228 void *dst;
229 int ret;
230
231 /* one optimized fast path only for non bigpcluster cases yet */
232 if (rq->inpages == 1 && rq->outpages == 1 && !rq->inplace_io) {
233 DBG_BUGON(!*rq->out);
234 dst = kmap_local_page(*rq->out);
235 dst_maptype = 0;
236 } else {
237 /* general decoding path which can be used for all cases */
238 ret = z_erofs_oneshot_prepare_dstpages(rq, pagepool);
239 if (ret < 0)
240 return ERR_PTR(ret);
241 if (ret > 0) {
242 dst = page_address(*rq->out);
243 dst_maptype = 1;
244 } else {
245 dst = erofs_vm_map_ram(rq->out, rq->outpages);
246 if (!dst)
247 return ERR_PTR(-ENOMEM);
248 dst_maptype = 2;
249 }
250 }
251 reason = __z_erofs_lz4_decompress(rq, dst);
252 if (!dst_maptype)
253 kunmap_local(dst);
254 else if (dst_maptype == 2)
255 vm_unmap_ram(dst, rq->outpages);
256 return reason;
257}
258
259static const char *z_erofs_transform_plain(struct z_erofs_decompress_req *rq,
260 struct page **pagepool)
261{
262 const unsigned int nrpages_in = rq->inpages, nrpages_out = rq->outpages;
263 const unsigned int bs = rq->sb->s_blocksize;
264 unsigned int cur = 0, ni = 0, no, pi, po, insz, cnt;
265 u8 *kin;
266
267 if (rq->outputsize > rq->inputsize)
268 return ERR_PTR(-EOPNOTSUPP);
269 if (rq->alg == Z_EROFS_COMPRESSION_INTERLACED) {
270 cur = bs - (rq->pageofs_out & (bs - 1));
271 DBG_BUGON(rq->pageofs_in & (bs - 1));
272 pi = (rq->pageofs_in + rq->inputsize - cur) & ~PAGE_MASK;
273 cur = min(cur, rq->outputsize);
274 if (cur && rq->out[0]) {
275 kin = kmap_local_page(rq->in[nrpages_in - 1]);
276 if (rq->out[0] == rq->in[nrpages_in - 1])
277 memmove(kin + rq->pageofs_out, kin + pi, cur);
278 else
279 memcpy_to_page(rq->out[0], rq->pageofs_out,
280 kin + pi, cur);
281 kunmap_local(kin);
282 }
283 rq->outputsize -= cur;
284 }
285
286 for (; rq->outputsize; rq->pageofs_in = 0, cur += insz, ni++) {
287 insz = min(PAGE_SIZE - rq->pageofs_in, rq->outputsize);
288 rq->outputsize -= insz;
289 if (!rq->in[ni])
290 continue;
291 kin = kmap_local_page(rq->in[ni]);
292 pi = 0;
293 do {
294 no = (rq->pageofs_out + cur + pi) >> PAGE_SHIFT;
295 po = (rq->pageofs_out + cur + pi) & ~PAGE_MASK;
296 DBG_BUGON(no >= nrpages_out);
297 cnt = min(insz - pi, PAGE_SIZE - po);
298 if (rq->out[no] == rq->in[ni])
299 memmove(kin + po,
300 kin + rq->pageofs_in + pi, cnt);
301 else if (rq->out[no])
302 memcpy_to_page(rq->out[no], po,
303 kin + rq->pageofs_in + pi, cnt);
304 pi += cnt;
305 } while (pi < insz);
306 kunmap_local(kin);
307 }
308 DBG_BUGON(ni > nrpages_in);
309 return NULL;
310}
311
312const char *z_erofs_stream_switch_bufs(struct z_erofs_stream_dctx *dctx,
313 void **dst, void **src, struct page **pgpl)
314{
315 struct z_erofs_decompress_req *rq = dctx->rq;
316 struct page **pgo, *tmppage;
317 unsigned int j;
318
319 if (!dctx->avail_out) {
320 if (++dctx->no >= rq->outpages || !rq->outputsize)
321 return "insufficient space for decompressed data";
322
323 if (dctx->kout)
324 kunmap_local(dctx->kout);
325 dctx->avail_out = min(rq->outputsize, PAGE_SIZE - rq->pageofs_out);
326 rq->outputsize -= dctx->avail_out;
327 pgo = &rq->out[dctx->no];
328 if (!*pgo && rq->fillgaps) { /* deduped */
329 *pgo = erofs_allocpage(pgpl, rq->gfp);
330 if (!*pgo) {
331 dctx->kout = NULL;
332 return ERR_PTR(-ENOMEM);
333 }
334 set_page_private(*pgo, Z_EROFS_SHORTLIVED_PAGE);
335 }
336 if (*pgo) {
337 dctx->kout = kmap_local_page(*pgo);
338 *dst = dctx->kout + rq->pageofs_out;
339 } else {
340 *dst = dctx->kout = NULL;
341 }
342 rq->pageofs_out = 0;
343 }
344
345 if (dctx->inbuf_pos == dctx->inbuf_sz && rq->inputsize) {
346 if (++dctx->ni >= rq->inpages)
347 return "invalid compressed data";
348 if (dctx->kout) /* unlike kmap(), take care of the orders */
349 kunmap_local(dctx->kout);
350 kunmap_local(dctx->kin);
351
352 dctx->inbuf_sz = min_t(u32, rq->inputsize, PAGE_SIZE);
353 rq->inputsize -= dctx->inbuf_sz;
354 dctx->kin = kmap_local_page(rq->in[dctx->ni]);
355 *src = dctx->kin;
356 dctx->bounced = false;
357 if (dctx->kout) {
358 j = (u8 *)*dst - dctx->kout;
359 dctx->kout = kmap_local_page(rq->out[dctx->no]);
360 *dst = dctx->kout + j;
361 }
362 dctx->inbuf_pos = 0;
363 }
364
365 /*
366 * Handle overlapping: Use the given bounce buffer if the input data is
367 * under processing; Or utilize short-lived pages from the on-stack page
368 * pool, where pages are shared among the same request. Note that only
369 * a few inplace I/O pages need to be doubled.
370 */
371 if (!dctx->bounced && rq->out[dctx->no] == rq->in[dctx->ni]) {
372 memcpy(dctx->bounce, *src, dctx->inbuf_sz);
373 *src = dctx->bounce;
374 dctx->bounced = true;
375 }
376
377 for (j = dctx->ni + 1; j < rq->inpages; ++j) {
378 if (rq->out[dctx->no] != rq->in[j])
379 continue;
380 tmppage = erofs_allocpage(pgpl, rq->gfp);
381 if (!tmppage)
382 return ERR_PTR(-ENOMEM);
383 set_page_private(tmppage, Z_EROFS_SHORTLIVED_PAGE);
384 copy_highpage(tmppage, rq->in[j]);
385 rq->in[j] = tmppage;
386 }
387 return NULL;
388}
389
390const struct z_erofs_decompressor *z_erofs_decomp[] = {
391 [Z_EROFS_COMPRESSION_SHIFTED] = &(const struct z_erofs_decompressor) {
392 .decompress = z_erofs_transform_plain,
393 .name = "shifted"
394 },
395 [Z_EROFS_COMPRESSION_INTERLACED] = &(const struct z_erofs_decompressor) {
396 .decompress = z_erofs_transform_plain,
397 .name = "interlaced"
398 },
399 [Z_EROFS_COMPRESSION_LZ4] = &(const struct z_erofs_decompressor) {
400 .config = z_erofs_load_lz4_config,
401 .decompress = z_erofs_lz4_decompress,
402 .init = z_erofs_gbuf_init,
403 .exit = z_erofs_gbuf_exit,
404 .name = "lz4"
405 },
406#ifdef CONFIG_EROFS_FS_ZIP_LZMA
407 [Z_EROFS_COMPRESSION_LZMA] = &z_erofs_lzma_decomp,
408#endif
409#ifdef CONFIG_EROFS_FS_ZIP_DEFLATE
410 [Z_EROFS_COMPRESSION_DEFLATE] = &z_erofs_deflate_decomp,
411#endif
412#ifdef CONFIG_EROFS_FS_ZIP_ZSTD
413 [Z_EROFS_COMPRESSION_ZSTD] = &z_erofs_zstd_decomp,
414#endif
415};
416
417int z_erofs_parse_cfgs(struct super_block *sb, struct erofs_super_block *dsb)
418{
419 struct erofs_sb_info *sbi = EROFS_SB(sb);
420 struct erofs_buf buf = __EROFS_BUF_INITIALIZER;
421 unsigned long algs, alg;
422 erofs_off_t offset;
423 int size, ret = 0;
424
425 if (!erofs_sb_has_compr_cfgs(sbi))
426 return z_erofs_load_lz4_config(sb, dsb, NULL, 0);
427
428 algs = le16_to_cpu(dsb->u1.available_compr_algs);
429 sbi->available_compr_algs = algs;
430 if (algs & ~Z_EROFS_ALL_COMPR_ALGS) {
431 erofs_err(sb, "unidentified algorithms %lx, please upgrade kernel",
432 algs & ~Z_EROFS_ALL_COMPR_ALGS);
433 return -EOPNOTSUPP;
434 }
435
436 (void)erofs_init_metabuf(&buf, sb, false);
437 offset = EROFS_SUPER_OFFSET + sbi->sb_size;
438 for_each_set_bit(alg, &algs, Z_EROFS_COMPRESSION_MAX) {
439 const struct z_erofs_decompressor *dec = z_erofs_decomp[alg];
440 void *data;
441
442 data = erofs_read_metadata(sb, &buf, &offset, &size);
443 if (IS_ERR(data)) {
444 ret = PTR_ERR(data);
445 break;
446 }
447
448 if (dec && dec->config) {
449 ret = dec->config(sb, dsb, data, size);
450 } else {
451 erofs_err(sb, "algorithm %ld isn't enabled on this kernel",
452 alg);
453 ret = -EOPNOTSUPP;
454 }
455 kfree(data);
456 if (ret)
457 break;
458 }
459 erofs_put_metabuf(&buf);
460 return ret;
461}
462
463int __init z_erofs_init_decompressor(void)
464{
465 int i, err;
466
467 for (i = 0; i < Z_EROFS_COMPRESSION_MAX; ++i) {
468 err = z_erofs_decomp[i] ? z_erofs_decomp[i]->init() : 0;
469 if (err) {
470 while (i--)
471 if (z_erofs_decomp[i])
472 z_erofs_decomp[i]->exit();
473 return err;
474 }
475 }
476 return 0;
477}
478
479void z_erofs_exit_decompressor(void)
480{
481 int i;
482
483 for (i = 0; i < Z_EROFS_COMPRESSION_MAX; ++i)
484 if (z_erofs_decomp[i])
485 z_erofs_decomp[i]->exit();
486}