projects/linux·blob master · refs

220 lines · 5.8 KiB

1// SPDX-License-Identifier: GPL-2.0-or-later
2#include <linux/zstd.h>
3#include "compress.h"
4
5struct z_erofs_zstd {
6 struct z_erofs_zstd *next;
7 u8 bounce[PAGE_SIZE];
8 void *wksp;
9 unsigned int wkspsz;
10};
11
12static DEFINE_SPINLOCK(z_erofs_zstd_lock);
13static unsigned int z_erofs_zstd_max_dictsize;
14static unsigned int z_erofs_zstd_nstrms, z_erofs_zstd_avail_strms;
15static struct z_erofs_zstd *z_erofs_zstd_head;
16static DECLARE_WAIT_QUEUE_HEAD(z_erofs_zstd_wq);
17
18module_param_named(zstd_streams, z_erofs_zstd_nstrms, uint, 0444);
19
20static struct z_erofs_zstd *z_erofs_isolate_strms(bool all)
21{
22 struct z_erofs_zstd *strm;
23
24again:
25 spin_lock(&z_erofs_zstd_lock);
26 strm = z_erofs_zstd_head;
27 if (!strm) {
28 spin_unlock(&z_erofs_zstd_lock);
29 wait_event(z_erofs_zstd_wq, READ_ONCE(z_erofs_zstd_head));
30 goto again;
31 }
32 z_erofs_zstd_head = all ? NULL : strm->next;
33 spin_unlock(&z_erofs_zstd_lock);
34 return strm;
35}
36
37static void z_erofs_zstd_exit(void)
38{
39 while (z_erofs_zstd_avail_strms) {
40 struct z_erofs_zstd *strm, *n;
41
42 for (strm = z_erofs_isolate_strms(true); strm; strm = n) {
43 n = strm->next;
44
45 kvfree(strm->wksp);
46 kfree(strm);
47 --z_erofs_zstd_avail_strms;
48 }
49 }
50}
51
52static int __init z_erofs_zstd_init(void)
53{
54 /* by default, use # of possible CPUs instead */
55 if (!z_erofs_zstd_nstrms)
56 z_erofs_zstd_nstrms = num_possible_cpus();
57
58 for (; z_erofs_zstd_avail_strms < z_erofs_zstd_nstrms;
59 ++z_erofs_zstd_avail_strms) {
60 struct z_erofs_zstd *strm;
61
62 strm = kzalloc_obj(*strm);
63 if (!strm) {
64 z_erofs_zstd_exit();
65 return -ENOMEM;
66 }
67 spin_lock(&z_erofs_zstd_lock);
68 strm->next = z_erofs_zstd_head;
69 z_erofs_zstd_head = strm;
70 spin_unlock(&z_erofs_zstd_lock);
71 }
72 return 0;
73}
74
75static int z_erofs_load_zstd_config(struct super_block *sb,
76 struct erofs_super_block *dsb, void *data, int size)
77{
78 static DEFINE_MUTEX(zstd_resize_mutex);
79 struct z_erofs_zstd_cfgs *zstd = data;
80 unsigned int dict_size, wkspsz;
81 struct z_erofs_zstd *strm, *head = NULL;
82 void *wksp;
83
84 if (!zstd || size < sizeof(struct z_erofs_zstd_cfgs) || zstd->format) {
85 erofs_err(sb, "unsupported zstd format, size=%u", size);
86 return -EINVAL;
87 }
88
89 if (zstd->windowlog > ilog2(Z_EROFS_ZSTD_MAX_DICT_SIZE) - 10) {
90 erofs_err(sb, "unsupported zstd window log %u", zstd->windowlog);
91 return -EINVAL;
92 }
93 dict_size = 1U << (zstd->windowlog + 10);
94
95 /* in case 2 z_erofs_load_zstd_config() race to avoid deadlock */
96 mutex_lock(&zstd_resize_mutex);
97 if (z_erofs_zstd_max_dictsize >= dict_size) {
98 mutex_unlock(&zstd_resize_mutex);
99 return 0;
100 }
101
102 /* 1. collect/isolate all streams for the following check */
103 while (z_erofs_zstd_avail_strms) {
104 struct z_erofs_zstd *n;
105
106 for (strm = z_erofs_isolate_strms(true); strm; strm = n) {
107 n = strm->next;
108 strm->next = head;
109 head = strm;
110 --z_erofs_zstd_avail_strms;
111 }
112 }
113
114 /* 2. walk each isolated stream and grow max dict_size if needed */
115 wkspsz = zstd_dstream_workspace_bound(dict_size);
116 for (strm = head; strm; strm = strm->next) {
117 wksp = kvmalloc(wkspsz, GFP_KERNEL);
118 if (!wksp)
119 break;
120 kvfree(strm->wksp);
121 strm->wksp = wksp;
122 strm->wkspsz = wkspsz;
123 }
124
125 /* 3. push back all to the global list and update max dict_size */
126 spin_lock(&z_erofs_zstd_lock);
127 DBG_BUGON(z_erofs_zstd_head);
128 z_erofs_zstd_head = head;
129 spin_unlock(&z_erofs_zstd_lock);
130 z_erofs_zstd_avail_strms = z_erofs_zstd_nstrms;
131 wake_up_all(&z_erofs_zstd_wq);
132 if (!strm)
133 z_erofs_zstd_max_dictsize = dict_size;
134 mutex_unlock(&zstd_resize_mutex);
135 return strm ? -ENOMEM : 0;
136}
137
138static const char *z_erofs_zstd_decompress(struct z_erofs_decompress_req *rq,
139 struct page **pgpl)
140{
141 struct super_block *sb = rq->sb;
142 struct z_erofs_stream_dctx dctx = { .rq = rq, .no = -1, .ni = 0 };
143 zstd_in_buffer in_buf = { NULL, 0, 0 };
144 zstd_out_buffer out_buf = { NULL, 0, 0 };
145 struct z_erofs_zstd *strm;
146 zstd_dstream *stream;
147 const char *reason;
148 int zerr;
149
150 /* 1. get the exact compressed size */
151 dctx.kin = kmap_local_page(*rq->in);
152 reason = z_erofs_fixup_insize(rq, dctx.kin + rq->pageofs_in,
153 min(rq->inputsize, sb->s_blocksize - rq->pageofs_in));
154 if (reason) {
155 kunmap_local(dctx.kin);
156 return reason;
157 }
158
159 /* 2. get an available ZSTD context */
160 strm = z_erofs_isolate_strms(false);
161
162 /* 3. multi-call decompress */
163 stream = zstd_init_dstream(z_erofs_zstd_max_dictsize, strm->wksp, strm->wkspsz);
164 if (!stream) {
165 reason = ERR_PTR(-ENOMEM);
166 goto failed_zinit;
167 }
168
169 rq->fillgaps = true; /* ZSTD doesn't support NULL output buffer */
170 in_buf.size = min_t(u32, rq->inputsize, PAGE_SIZE - rq->pageofs_in);
171 rq->inputsize -= in_buf.size;
172 in_buf.src = dctx.kin + rq->pageofs_in;
173 dctx.bounce = strm->bounce;
174
175 do {
176 dctx.inbuf_sz = in_buf.size;
177 dctx.inbuf_pos = in_buf.pos;
178 reason = z_erofs_stream_switch_bufs(&dctx, &out_buf.dst,
179 (void **)&in_buf.src, pgpl);
180 if (reason)
181 break;
182
183 if (out_buf.size == out_buf.pos) {
184 out_buf.size = dctx.avail_out;
185 out_buf.pos = 0;
186 }
187 in_buf.size = dctx.inbuf_sz;
188 in_buf.pos = dctx.inbuf_pos;
189
190 zerr = zstd_decompress_stream(stream, &out_buf, &in_buf);
191 dctx.avail_out = out_buf.size - out_buf.pos;
192 if (zstd_is_error(zerr) ||
193 ((rq->outputsize + dctx.avail_out) && (!zerr || (zerr > 0 &&
194 !(rq->inputsize + in_buf.size - in_buf.pos))))) {
195 reason = zstd_is_error(zerr) ? zstd_get_error_name(zerr) :
196 "unexpected end of stream";
197 break;
198 }
199 } while (rq->outputsize + dctx.avail_out);
200
201 if (dctx.kout)
202 kunmap_local(dctx.kout);
203failed_zinit:
204 kunmap_local(dctx.kin);
205 /* 4. push back ZSTD stream context to the global list */
206 spin_lock(&z_erofs_zstd_lock);
207 strm->next = z_erofs_zstd_head;
208 z_erofs_zstd_head = strm;
209 spin_unlock(&z_erofs_zstd_lock);
210 wake_up(&z_erofs_zstd_wq);
211 return reason;
212}
213
214const struct z_erofs_decompressor z_erofs_zstd_decomp = {
215 .config = z_erofs_load_zstd_config,
216 .decompress = z_erofs_zstd_decompress,
217 .init = z_erofs_zstd_init,
218 .exit = z_erofs_zstd_exit,
219 .name = "zstd",
220};