projects/linux·blob master · refs

402 lines · 9.1 KiB

1// SPDX-License-Identifier: GPL-2.0-or-later
2/*
3 * Synchronous Compression operations
4 *
5 * Copyright 2015 LG Electronics Inc.
6 * Copyright (c) 2016, Intel Corporation
7 * Author: Giovanni Cabiddu <[email protected]>
8 */
9
10#include <crypto/internal/scompress.h>
11#include <crypto/scatterwalk.h>
12#include <linux/cpumask.h>
13#include <linux/cryptouser.h>
14#include <linux/err.h>
15#include <linux/highmem.h>
16#include <linux/kernel.h>
17#include <linux/module.h>
18#include <linux/overflow.h>
19#include <linux/scatterlist.h>
20#include <linux/seq_file.h>
21#include <linux/slab.h>
22#include <linux/string.h>
23#include <linux/workqueue.h>
24#include <net/netlink.h>
25
26#include "compress.h"
27
28struct scomp_scratch {
29 spinlock_t lock;
30 union {
31 void *src __guarded_by(&lock);
32 unsigned long saddr __guarded_by(&lock);
33 };
34};
35
36static DEFINE_PER_CPU(struct scomp_scratch, scomp_scratch) = {
37 .lock = __SPIN_LOCK_UNLOCKED(scomp_scratch.lock),
38};
39
40static const struct crypto_type crypto_scomp_type;
41static DEFINE_MUTEX(scomp_lock);
42static int scomp_scratch_users __guarded_by(&scomp_lock);
43
44static cpumask_t scomp_scratch_want;
45static void scomp_scratch_workfn(struct work_struct *work);
46static DECLARE_WORK(scomp_scratch_work, scomp_scratch_workfn);
47
48static int __maybe_unused crypto_scomp_report(
49 struct sk_buff *skb, struct crypto_alg *alg)
50{
51 struct crypto_report_comp rscomp = {
52 .type = "scomp",
53 };
54
55 return nla_put(skb, CRYPTOCFGA_REPORT_COMPRESS,
56 sizeof(rscomp), &rscomp);
57}
58
59static void __maybe_unused crypto_scomp_show(struct seq_file *m,
60 struct crypto_alg *alg)
61{
62 seq_puts(m, "type : scomp\n");
63}
64
65static void crypto_scomp_free_scratches(void)
66 __context_unsafe(/* frees @scratch */)
67{
68 struct scomp_scratch *scratch;
69 int i;
70
71 for_each_possible_cpu(i) {
72 scratch = per_cpu_ptr(&scomp_scratch, i);
73
74 free_page(scratch->saddr);
75 scratch->src = NULL;
76 }
77}
78
79static int scomp_alloc_scratch(struct scomp_scratch *scratch, int cpu)
80{
81 int node = cpu_to_node(cpu);
82 struct page *page;
83
84 page = alloc_pages_node(node, GFP_KERNEL, 0);
85 if (!page)
86 return -ENOMEM;
87 spin_lock_bh(&scratch->lock);
88 scratch->src = page_address(page);
89 spin_unlock_bh(&scratch->lock);
90 return 0;
91}
92
93static void scomp_scratch_workfn(struct work_struct *work)
94{
95 int cpu;
96
97 for_each_cpu(cpu, &scomp_scratch_want) {
98 struct scomp_scratch *scratch;
99
100 scratch = per_cpu_ptr(&scomp_scratch, cpu);
101 if (context_unsafe(scratch->src))
102 continue;
103 if (scomp_alloc_scratch(scratch, cpu))
104 break;
105
106 cpumask_clear_cpu(cpu, &scomp_scratch_want);
107 }
108}
109
110static int crypto_scomp_alloc_scratches(void)
111 __context_unsafe(/* allocates @scratch */)
112{
113 unsigned int i = cpumask_first(cpu_possible_mask);
114 struct scomp_scratch *scratch;
115
116 scratch = per_cpu_ptr(&scomp_scratch, i);
117 return scomp_alloc_scratch(scratch, i);
118}
119
120static int crypto_scomp_init_tfm(struct crypto_tfm *tfm)
121{
122 struct scomp_alg *alg = crypto_scomp_alg(__crypto_scomp_tfm(tfm));
123 int ret = 0;
124
125 mutex_lock(&scomp_lock);
126 ret = crypto_acomp_alloc_streams(&alg->streams);
127 if (ret)
128 goto unlock;
129 if (!scomp_scratch_users++) {
130 ret = crypto_scomp_alloc_scratches();
131 if (ret)
132 scomp_scratch_users--;
133 }
134unlock:
135 mutex_unlock(&scomp_lock);
136
137 return ret;
138}
139
140#define scomp_lock_scratch(...) __acquire_ret(_scomp_lock_scratch(__VA_ARGS__), &__ret->lock)
141static struct scomp_scratch *_scomp_lock_scratch(void) __acquires_ret
142{
143 int cpu = raw_smp_processor_id();
144 struct scomp_scratch *scratch;
145
146 scratch = per_cpu_ptr(&scomp_scratch, cpu);
147 spin_lock(&scratch->lock);
148 if (likely(scratch->src))
149 return scratch;
150 spin_unlock(&scratch->lock);
151
152 cpumask_set_cpu(cpu, &scomp_scratch_want);
153 schedule_work(&scomp_scratch_work);
154
155 scratch = per_cpu_ptr(&scomp_scratch, cpumask_first(cpu_possible_mask));
156 spin_lock(&scratch->lock);
157 return scratch;
158}
159
160static inline void scomp_unlock_scratch(struct scomp_scratch *scratch)
161 __releases(&scratch->lock)
162{
163 spin_unlock(&scratch->lock);
164}
165
166static int scomp_acomp_comp_decomp(struct acomp_req *req, int dir)
167{
168 struct crypto_acomp *tfm = crypto_acomp_reqtfm(req);
169 struct crypto_scomp **tfm_ctx = acomp_tfm_ctx(tfm);
170 bool src_isvirt = acomp_request_src_isvirt(req);
171 bool dst_isvirt = acomp_request_dst_isvirt(req);
172 struct crypto_scomp *scomp = *tfm_ctx;
173 unsigned int slen = req->slen;
174 unsigned int dlen = req->dlen;
175 struct page *spage, *dpage;
176 unsigned int n;
177 const u8 *src;
178 size_t soff;
179 size_t doff;
180 u8 *dst;
181 int ret;
182
183 if (!req->src || !slen)
184 return -EINVAL;
185
186 if (!req->dst || !dlen)
187 return -EINVAL;
188
189 if (dst_isvirt)
190 dst = req->dvirt;
191 else {
192 if (dlen <= req->dst->length) {
193 dpage = sg_page(req->dst);
194 doff = req->dst->offset;
195 } else
196 return -ENOSYS;
197
198 dpage += doff / PAGE_SIZE;
199 doff = offset_in_page(doff);
200
201 n = (dlen - 1) / PAGE_SIZE;
202 n += (offset_in_page(dlen - 1) + doff) / PAGE_SIZE;
203 if (PageHighMem(dpage + n) &&
204 size_add(doff, dlen) > PAGE_SIZE)
205 return -ENOSYS;
206 dst = kmap_local_page(dpage) + doff;
207 }
208
209 if (src_isvirt)
210 src = req->svirt;
211 else {
212 src = NULL;
213 do {
214 if (slen <= req->src->length) {
215 spage = sg_page(req->src);
216 soff = req->src->offset;
217 } else
218 break;
219
220 spage = spage + soff / PAGE_SIZE;
221 soff = offset_in_page(soff);
222
223 n = (slen - 1) / PAGE_SIZE;
224 n += (offset_in_page(slen - 1) + soff) / PAGE_SIZE;
225 if (PageHighMem(spage + n) &&
226 size_add(soff, slen) > PAGE_SIZE)
227 break;
228 src = kmap_local_page(spage) + soff;
229 } while (0);
230 }
231
232 struct crypto_acomp_stream *stream = crypto_acomp_lock_stream_bh(&crypto_scomp_alg(scomp)->streams);
233
234 if (!src_isvirt && !src) {
235 struct scomp_scratch *scratch = scomp_lock_scratch();
236 const u8 *src = scratch->src;
237
238 memcpy_from_sglist(scratch->src, req->src, 0, slen);
239
240 if (dir)
241 ret = crypto_scomp_compress(scomp, src, slen,
242 dst, &dlen, stream->ctx);
243 else
244 ret = crypto_scomp_decompress(scomp, src, slen,
245 dst, &dlen, stream->ctx);
246
247 scomp_unlock_scratch(scratch);
248 } else if (dir)
249 ret = crypto_scomp_compress(scomp, src, slen,
250 dst, &dlen, stream->ctx);
251 else
252 ret = crypto_scomp_decompress(scomp, src, slen,
253 dst, &dlen, stream->ctx);
254
255 crypto_acomp_unlock_stream_bh(stream);
256
257 req->dlen = dlen;
258
259 if (!src_isvirt && src)
260 kunmap_local(src);
261 if (!dst_isvirt) {
262 kunmap_local(dst);
263 dlen += doff;
264 for (;;) {
265 flush_dcache_page(dpage);
266 if (dlen <= PAGE_SIZE)
267 break;
268 dlen -= PAGE_SIZE;
269 dpage++;
270 }
271 }
272
273 return ret;
274}
275
276static int scomp_acomp_compress(struct acomp_req *req)
277{
278 return scomp_acomp_comp_decomp(req, 1);
279}
280
281static int scomp_acomp_decompress(struct acomp_req *req)
282{
283 return scomp_acomp_comp_decomp(req, 0);
284}
285
286static void crypto_exit_scomp_ops_async(struct crypto_tfm *tfm)
287{
288 struct crypto_scomp **ctx = crypto_tfm_ctx(tfm);
289
290 crypto_free_scomp(*ctx);
291
292 flush_work(&scomp_scratch_work);
293 mutex_lock(&scomp_lock);
294 if (!--scomp_scratch_users)
295 crypto_scomp_free_scratches();
296 mutex_unlock(&scomp_lock);
297}
298
299int crypto_init_scomp_ops_async(struct crypto_tfm *tfm)
300{
301 struct crypto_alg *calg = tfm->__crt_alg;
302 struct crypto_acomp *crt = __crypto_acomp_tfm(tfm);
303 struct crypto_scomp **ctx = crypto_tfm_ctx(tfm);
304 struct crypto_scomp *scomp;
305
306 if (!crypto_mod_get(calg))
307 return -EAGAIN;
308
309 scomp = crypto_create_tfm(calg, &crypto_scomp_type);
310 if (IS_ERR(scomp)) {
311 crypto_mod_put(calg);
312 return PTR_ERR(scomp);
313 }
314
315 *ctx = scomp;
316 tfm->exit = crypto_exit_scomp_ops_async;
317
318 crt->compress = scomp_acomp_compress;
319 crt->decompress = scomp_acomp_decompress;
320
321 return 0;
322}
323
324static void crypto_scomp_destroy(struct crypto_alg *alg)
325{
326 struct scomp_alg *scomp = __crypto_scomp_alg(alg);
327
328 crypto_acomp_free_streams(&scomp->streams);
329}
330
331static const struct crypto_type crypto_scomp_type = {
332 .extsize = crypto_alg_extsize,
333 .init_tfm = crypto_scomp_init_tfm,
334 .destroy = crypto_scomp_destroy,
335#ifdef CONFIG_PROC_FS
336 .show = crypto_scomp_show,
337#endif
338#if IS_ENABLED(CONFIG_CRYPTO_USER)
339 .report = crypto_scomp_report,
340#endif
341 .maskclear = ~CRYPTO_ALG_TYPE_MASK,
342 .maskset = CRYPTO_ALG_TYPE_MASK,
343 .type = CRYPTO_ALG_TYPE_SCOMPRESS,
344 .tfmsize = offsetof(struct crypto_scomp, base),
345 .algsize = offsetof(struct scomp_alg, base),
346};
347
348static void scomp_prepare_alg(struct scomp_alg *alg)
349{
350 struct crypto_alg *base = &alg->calg.base;
351
352 comp_prepare_alg(&alg->calg);
353
354 base->cra_flags |= CRYPTO_ALG_REQ_VIRT;
355}
356
357int crypto_register_scomp(struct scomp_alg *alg)
358{
359 struct crypto_alg *base = &alg->calg.base;
360
361 scomp_prepare_alg(alg);
362
363 base->cra_type = &crypto_scomp_type;
364 base->cra_flags |= CRYPTO_ALG_TYPE_SCOMPRESS;
365
366 return crypto_register_alg(base);
367}
368EXPORT_SYMBOL_GPL(crypto_register_scomp);
369
370void crypto_unregister_scomp(struct scomp_alg *alg)
371{
372 crypto_unregister_alg(&alg->base);
373}
374EXPORT_SYMBOL_GPL(crypto_unregister_scomp);
375
376int crypto_register_scomps(struct scomp_alg *algs, int count)
377{
378 int i, ret;
379
380 for (i = 0; i < count; i++) {
381 ret = crypto_register_scomp(&algs[i]);
382 if (ret) {
383 crypto_unregister_scomps(algs, i);
384 return ret;
385 }
386 }
387
388 return 0;
389}
390EXPORT_SYMBOL_GPL(crypto_register_scomps);
391
392void crypto_unregister_scomps(struct scomp_alg *algs, int count)
393{
394 int i;
395
396 for (i = count - 1; i >= 0; --i)
397 crypto_unregister_scomp(&algs[i]);
398}
399EXPORT_SYMBOL_GPL(crypto_unregister_scomps);
400
401MODULE_LICENSE("GPL");
402MODULE_DESCRIPTION("Synchronous compression type");