projects/linux·blob master · refs

588 lines · 13.5 KiB

1// SPDX-License-Identifier: GPL-2.0-or-later
2/*
3 * Asynchronous Compression operations
4 *
5 * Copyright (c) 2016, Intel Corporation
6 * Authors: Weigang Li <[email protected]>
7 * Giovanni Cabiddu <[email protected]>
8 */
9
10#include <crypto/internal/acompress.h>
11#include <crypto/scatterwalk.h>
12#include <linux/cryptouser.h>
13#include <linux/cpumask.h>
14#include <linux/err.h>
15#include <linux/kernel.h>
16#include <linux/module.h>
17#include <linux/percpu.h>
18#include <linux/scatterlist.h>
19#include <linux/sched.h>
20#include <linux/seq_file.h>
21#include <linux/smp.h>
22#include <linux/spinlock.h>
23#include <linux/string.h>
24#include <linux/workqueue.h>
25#include <net/netlink.h>
26
27#include "compress.h"
28
29struct crypto_scomp;
30
31enum {
32 ACOMP_WALK_SLEEP = 1 << 0,
33 ACOMP_WALK_SRC_LINEAR = 1 << 1,
34 ACOMP_WALK_DST_LINEAR = 1 << 2,
35};
36
37static const struct crypto_type crypto_acomp_type;
38
39static void acomp_reqchain_done(void *data, int err);
40
41static inline struct acomp_alg *__crypto_acomp_alg(struct crypto_alg *alg)
42{
43 return container_of(alg, struct acomp_alg, calg.base);
44}
45
46static inline struct acomp_alg *crypto_acomp_alg(struct crypto_acomp *tfm)
47{
48 return __crypto_acomp_alg(crypto_acomp_tfm(tfm)->__crt_alg);
49}
50
51static int __maybe_unused crypto_acomp_report(
52 struct sk_buff *skb, struct crypto_alg *alg)
53{
54 struct crypto_report_acomp racomp = {
55 .type = "acomp",
56 };
57
58 return nla_put(skb, CRYPTOCFGA_REPORT_ACOMP, sizeof(racomp), &racomp);
59}
60
61static void __maybe_unused crypto_acomp_show(struct seq_file *m,
62 struct crypto_alg *alg)
63{
64 seq_puts(m, "type : acomp\n");
65}
66
67static void crypto_acomp_exit_tfm(struct crypto_tfm *tfm)
68{
69 struct crypto_acomp *acomp = __crypto_acomp_tfm(tfm);
70 struct acomp_alg *alg = crypto_acomp_alg(acomp);
71
72 if (alg->exit)
73 alg->exit(acomp);
74
75 if (acomp_is_async(acomp))
76 crypto_free_acomp(crypto_acomp_fb(acomp));
77}
78
79static int crypto_acomp_init_tfm(struct crypto_tfm *tfm)
80{
81 struct crypto_acomp *acomp = __crypto_acomp_tfm(tfm);
82 struct acomp_alg *alg = crypto_acomp_alg(acomp);
83 struct crypto_acomp *fb = NULL;
84 int err;
85
86 if (tfm->__crt_alg->cra_type != &crypto_acomp_type)
87 return crypto_init_scomp_ops_async(tfm);
88
89 if (acomp_is_async(acomp)) {
90 fb = crypto_alloc_acomp(crypto_acomp_alg_name(acomp), 0,
91 CRYPTO_ALG_ASYNC);
92 if (IS_ERR(fb))
93 return PTR_ERR(fb);
94
95 err = -EINVAL;
96 if (crypto_acomp_reqsize(fb) > MAX_SYNC_COMP_REQSIZE)
97 goto out_free_fb;
98
99 tfm->fb = crypto_acomp_tfm(fb);
100 }
101
102 acomp->compress = alg->compress;
103 acomp->decompress = alg->decompress;
104 acomp->reqsize = alg->base.cra_reqsize;
105
106 acomp->base.exit = crypto_acomp_exit_tfm;
107
108 if (!alg->init)
109 return 0;
110
111 err = alg->init(acomp);
112 if (err)
113 goto out_free_fb;
114
115 return 0;
116
117out_free_fb:
118 crypto_free_acomp(fb);
119 return err;
120}
121
122static unsigned int crypto_acomp_extsize(struct crypto_alg *alg)
123{
124 int extsize = crypto_alg_extsize(alg);
125
126 if (alg->cra_type != &crypto_acomp_type)
127 extsize += sizeof(struct crypto_scomp *);
128
129 return extsize;
130}
131
132static const struct crypto_type crypto_acomp_type = {
133 .extsize = crypto_acomp_extsize,
134 .init_tfm = crypto_acomp_init_tfm,
135#ifdef CONFIG_PROC_FS
136 .show = crypto_acomp_show,
137#endif
138#if IS_ENABLED(CONFIG_CRYPTO_USER)
139 .report = crypto_acomp_report,
140#endif
141 .maskclear = ~CRYPTO_ALG_TYPE_MASK,
142 .maskset = CRYPTO_ALG_TYPE_ACOMPRESS_MASK,
143 .type = CRYPTO_ALG_TYPE_ACOMPRESS,
144 .tfmsize = offsetof(struct crypto_acomp, base),
145 .algsize = offsetof(struct acomp_alg, base),
146};
147
148struct crypto_acomp *crypto_alloc_acomp(const char *alg_name, u32 type,
149 u32 mask)
150{
151 return crypto_alloc_tfm(alg_name, &crypto_acomp_type, type, mask);
152}
153EXPORT_SYMBOL_GPL(crypto_alloc_acomp);
154
155struct crypto_acomp *crypto_alloc_acomp_node(const char *alg_name, u32 type,
156 u32 mask, int node)
157{
158 return crypto_alloc_tfm_node(alg_name, &crypto_acomp_type, type, mask,
159 node);
160}
161EXPORT_SYMBOL_GPL(crypto_alloc_acomp_node);
162
163static void acomp_save_req(struct acomp_req *req, crypto_completion_t cplt)
164{
165 struct acomp_req_chain *state = &req->chain;
166
167 state->compl = req->base.complete;
168 state->data = req->base.data;
169 req->base.complete = cplt;
170 req->base.data = req;
171}
172
173static void acomp_restore_req(struct acomp_req *req)
174{
175 req->base.complete = req->chain.compl;
176 req->base.data = req->chain.data;
177}
178
179static void acomp_reqchain_virt(struct acomp_req *req)
180{
181 struct acomp_req_chain *state = &req->chain;
182 unsigned int slen = req->slen;
183 unsigned int dlen = req->dlen;
184
185 if (state->flags & CRYPTO_ACOMP_REQ_SRC_VIRT)
186 acomp_request_set_src_dma(req, state->src, slen);
187 if (state->flags & CRYPTO_ACOMP_REQ_DST_VIRT)
188 acomp_request_set_dst_dma(req, state->dst, dlen);
189}
190
191static void acomp_virt_to_sg(struct acomp_req *req)
192{
193 struct acomp_req_chain *state = &req->chain;
194
195 state->flags = req->base.flags & (CRYPTO_ACOMP_REQ_SRC_VIRT |
196 CRYPTO_ACOMP_REQ_DST_VIRT);
197
198 if (acomp_request_src_isvirt(req)) {
199 unsigned int slen = req->slen;
200 const u8 *svirt = req->svirt;
201
202 state->src = svirt;
203 sg_init_one(&state->ssg, svirt, slen);
204 acomp_request_set_src_sg(req, &state->ssg, slen);
205 }
206
207 if (acomp_request_dst_isvirt(req)) {
208 unsigned int dlen = req->dlen;
209 u8 *dvirt = req->dvirt;
210
211 state->dst = dvirt;
212 sg_init_one(&state->dsg, dvirt, dlen);
213 acomp_request_set_dst_sg(req, &state->dsg, dlen);
214 }
215}
216
217static int acomp_do_nondma(struct acomp_req *req, bool comp)
218{
219 ACOMP_FBREQ_ON_STACK(fbreq, req);
220 int err;
221
222 if (comp)
223 err = crypto_acomp_compress(fbreq);
224 else
225 err = crypto_acomp_decompress(fbreq);
226
227 req->dlen = fbreq->dlen;
228 return err;
229}
230
231static int acomp_do_one_req(struct acomp_req *req, bool comp)
232{
233 if (acomp_request_isnondma(req))
234 return acomp_do_nondma(req, comp);
235
236 acomp_virt_to_sg(req);
237 return comp ? crypto_acomp_reqtfm(req)->compress(req) :
238 crypto_acomp_reqtfm(req)->decompress(req);
239}
240
241static int acomp_reqchain_finish(struct acomp_req *req, int err)
242{
243 acomp_reqchain_virt(req);
244 acomp_restore_req(req);
245 return err;
246}
247
248static void acomp_reqchain_done(void *data, int err)
249{
250 struct acomp_req *req = data;
251 crypto_completion_t compl;
252
253 compl = req->chain.compl;
254 data = req->chain.data;
255
256 if (err == -EINPROGRESS)
257 goto notify;
258
259 err = acomp_reqchain_finish(req, err);
260
261notify:
262 compl(data, err);
263}
264
265static int acomp_do_req_chain(struct acomp_req *req, bool comp)
266{
267 int err;
268
269 acomp_save_req(req, acomp_reqchain_done);
270
271 err = acomp_do_one_req(req, comp);
272 if (err == -EBUSY || err == -EINPROGRESS)
273 return err;
274
275 return acomp_reqchain_finish(req, err);
276}
277
278int crypto_acomp_compress(struct acomp_req *req)
279{
280 struct crypto_acomp *tfm = crypto_acomp_reqtfm(req);
281
282 if (acomp_req_on_stack(req) && acomp_is_async(tfm))
283 return -EAGAIN;
284 if (crypto_acomp_req_virt(tfm) || acomp_request_issg(req))
285 return crypto_acomp_reqtfm(req)->compress(req);
286 return acomp_do_req_chain(req, true);
287}
288EXPORT_SYMBOL_GPL(crypto_acomp_compress);
289
290int crypto_acomp_decompress(struct acomp_req *req)
291{
292 struct crypto_acomp *tfm = crypto_acomp_reqtfm(req);
293
294 if (acomp_req_on_stack(req) && acomp_is_async(tfm))
295 return -EAGAIN;
296 if (crypto_acomp_req_virt(tfm) || acomp_request_issg(req))
297 return crypto_acomp_reqtfm(req)->decompress(req);
298 return acomp_do_req_chain(req, false);
299}
300EXPORT_SYMBOL_GPL(crypto_acomp_decompress);
301
302void comp_prepare_alg(struct comp_alg_common *alg)
303{
304 struct crypto_alg *base = &alg->base;
305
306 base->cra_flags &= ~CRYPTO_ALG_TYPE_MASK;
307}
308
309int crypto_register_acomp(struct acomp_alg *alg)
310{
311 struct crypto_alg *base = &alg->calg.base;
312
313 comp_prepare_alg(&alg->calg);
314
315 base->cra_type = &crypto_acomp_type;
316 base->cra_flags |= CRYPTO_ALG_TYPE_ACOMPRESS;
317
318 return crypto_register_alg(base);
319}
320EXPORT_SYMBOL_GPL(crypto_register_acomp);
321
322void crypto_unregister_acomp(struct acomp_alg *alg)
323{
324 crypto_unregister_alg(&alg->base);
325}
326EXPORT_SYMBOL_GPL(crypto_unregister_acomp);
327
328int crypto_register_acomps(struct acomp_alg *algs, int count)
329{
330 int i, ret;
331
332 for (i = 0; i < count; i++) {
333 ret = crypto_register_acomp(&algs[i]);
334 if (ret) {
335 crypto_unregister_acomps(algs, i);
336 return ret;
337 }
338 }
339
340 return 0;
341}
342EXPORT_SYMBOL_GPL(crypto_register_acomps);
343
344void crypto_unregister_acomps(struct acomp_alg *algs, int count)
345{
346 int i;
347
348 for (i = count - 1; i >= 0; --i)
349 crypto_unregister_acomp(&algs[i]);
350}
351EXPORT_SYMBOL_GPL(crypto_unregister_acomps);
352
353static void acomp_stream_workfn(struct work_struct *work)
354{
355 struct crypto_acomp_streams *s =
356 container_of(work, struct crypto_acomp_streams, stream_work);
357 struct crypto_acomp_stream __percpu *streams = s->streams;
358 int cpu;
359
360 for_each_cpu(cpu, &s->stream_want) {
361 struct crypto_acomp_stream *ps;
362 void *ctx;
363
364 ps = per_cpu_ptr(streams, cpu);
365 if (ps->ctx)
366 continue;
367
368 ctx = s->alloc_ctx();
369 if (IS_ERR(ctx))
370 break;
371
372 spin_lock_bh(&ps->lock);
373 ps->ctx = ctx;
374 spin_unlock_bh(&ps->lock);
375
376 cpumask_clear_cpu(cpu, &s->stream_want);
377 }
378}
379
380void crypto_acomp_free_streams(struct crypto_acomp_streams *s)
381{
382 struct crypto_acomp_stream __percpu *streams = s->streams;
383 void (*free_ctx)(void *);
384 int i;
385
386 s->streams = NULL;
387 if (!streams)
388 return;
389
390 cancel_work_sync(&s->stream_work);
391 free_ctx = s->free_ctx;
392
393 for_each_possible_cpu(i) {
394 struct crypto_acomp_stream *ps = per_cpu_ptr(streams, i);
395
396 if (!ps->ctx)
397 continue;
398
399 free_ctx(ps->ctx);
400 }
401
402 free_percpu(streams);
403}
404EXPORT_SYMBOL_GPL(crypto_acomp_free_streams);
405
406int crypto_acomp_alloc_streams(struct crypto_acomp_streams *s)
407{
408 struct crypto_acomp_stream __percpu *streams;
409 struct crypto_acomp_stream *ps;
410 unsigned int i;
411 void *ctx;
412
413 if (s->streams)
414 return 0;
415
416 streams = alloc_percpu(struct crypto_acomp_stream);
417 if (!streams)
418 return -ENOMEM;
419
420 ctx = s->alloc_ctx();
421 if (IS_ERR(ctx)) {
422 free_percpu(streams);
423 return PTR_ERR(ctx);
424 }
425
426 i = cpumask_first(cpu_possible_mask);
427 ps = per_cpu_ptr(streams, i);
428 ps->ctx = ctx;
429
430 for_each_possible_cpu(i) {
431 ps = per_cpu_ptr(streams, i);
432 spin_lock_init(&ps->lock);
433 }
434
435 s->streams = streams;
436
437 INIT_WORK(&s->stream_work, acomp_stream_workfn);
438 return 0;
439}
440EXPORT_SYMBOL_GPL(crypto_acomp_alloc_streams);
441
442struct crypto_acomp_stream *_crypto_acomp_lock_stream_bh(
443 struct crypto_acomp_streams *s)
444{
445 struct crypto_acomp_stream __percpu *streams = s->streams;
446 int cpu = raw_smp_processor_id();
447 struct crypto_acomp_stream *ps;
448
449 ps = per_cpu_ptr(streams, cpu);
450 spin_lock_bh(&ps->lock);
451 if (likely(ps->ctx))
452 return ps;
453 spin_unlock(&ps->lock);
454
455 cpumask_set_cpu(cpu, &s->stream_want);
456 schedule_work(&s->stream_work);
457
458 ps = per_cpu_ptr(streams, cpumask_first(cpu_possible_mask));
459 spin_lock(&ps->lock);
460 return ps;
461}
462EXPORT_SYMBOL_GPL(_crypto_acomp_lock_stream_bh);
463
464void acomp_walk_done_src(struct acomp_walk *walk, int used)
465{
466 walk->slen -= used;
467 if ((walk->flags & ACOMP_WALK_SRC_LINEAR))
468 scatterwalk_advance(&walk->in, used);
469 else
470 scatterwalk_done_src(&walk->in, used);
471
472 if ((walk->flags & ACOMP_WALK_SLEEP))
473 cond_resched();
474}
475EXPORT_SYMBOL_GPL(acomp_walk_done_src);
476
477void acomp_walk_done_dst(struct acomp_walk *walk, int used)
478{
479 walk->dlen -= used;
480 if ((walk->flags & ACOMP_WALK_DST_LINEAR))
481 scatterwalk_advance(&walk->out, used);
482 else
483 scatterwalk_done_dst(&walk->out, used);
484
485 if ((walk->flags & ACOMP_WALK_SLEEP))
486 cond_resched();
487}
488EXPORT_SYMBOL_GPL(acomp_walk_done_dst);
489
490int acomp_walk_next_src(struct acomp_walk *walk)
491{
492 unsigned int slen = walk->slen;
493 unsigned int max = UINT_MAX;
494
495 if (!preempt_model_preemptible() && (walk->flags & ACOMP_WALK_SLEEP))
496 max = PAGE_SIZE;
497 if ((walk->flags & ACOMP_WALK_SRC_LINEAR)) {
498 walk->in.__addr = (void *)(((u8 *)walk->in.sg) +
499 walk->in.offset);
500 return min(slen, max);
501 }
502
503 return slen ? scatterwalk_next(&walk->in, slen) : 0;
504}
505EXPORT_SYMBOL_GPL(acomp_walk_next_src);
506
507int acomp_walk_next_dst(struct acomp_walk *walk)
508{
509 unsigned int dlen = walk->dlen;
510 unsigned int max = UINT_MAX;
511
512 if (!preempt_model_preemptible() && (walk->flags & ACOMP_WALK_SLEEP))
513 max = PAGE_SIZE;
514 if ((walk->flags & ACOMP_WALK_DST_LINEAR)) {
515 walk->out.__addr = (void *)(((u8 *)walk->out.sg) +
516 walk->out.offset);
517 return min(dlen, max);
518 }
519
520 return dlen ? scatterwalk_next(&walk->out, dlen) : 0;
521}
522EXPORT_SYMBOL_GPL(acomp_walk_next_dst);
523
524int acomp_walk_virt(struct acomp_walk *__restrict walk,
525 struct acomp_req *__restrict req, bool atomic)
526{
527 struct scatterlist *src = req->src;
528 struct scatterlist *dst = req->dst;
529
530 walk->slen = req->slen;
531 walk->dlen = req->dlen;
532
533 if (!walk->slen || !walk->dlen)
534 return -EINVAL;
535
536 walk->flags = 0;
537 if ((req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) && !atomic)
538 walk->flags |= ACOMP_WALK_SLEEP;
539 if ((req->base.flags & CRYPTO_ACOMP_REQ_SRC_VIRT))
540 walk->flags |= ACOMP_WALK_SRC_LINEAR;
541 if ((req->base.flags & CRYPTO_ACOMP_REQ_DST_VIRT))
542 walk->flags |= ACOMP_WALK_DST_LINEAR;
543
544 if ((walk->flags & ACOMP_WALK_SRC_LINEAR)) {
545 walk->in.sg = (void *)req->svirt;
546 walk->in.offset = 0;
547 } else
548 scatterwalk_start(&walk->in, src);
549 if ((walk->flags & ACOMP_WALK_DST_LINEAR)) {
550 walk->out.sg = (void *)req->dvirt;
551 walk->out.offset = 0;
552 } else
553 scatterwalk_start(&walk->out, dst);
554
555 return 0;
556}
557EXPORT_SYMBOL_GPL(acomp_walk_virt);
558
559struct acomp_req *acomp_request_clone(struct acomp_req *req,
560 size_t total, gfp_t gfp)
561{
562 struct crypto_tfm *tfm = req->base.tfm;
563 struct acomp_req *nreq;
564 size_t len;
565
566 len = sizeof(*req) +
567 crypto_acomp_reqsize(crypto_acomp_reqtfm(req));
568 len = ALIGN(len, CRYPTO_MINALIGN);
569
570 nreq = kzalloc(len, gfp);
571 if (!nreq) {
572 req->base.tfm = tfm->fb;
573 return req;
574 }
575
576 memcpy(nreq, req, sizeof(*req));
577 nreq->base.flags &= ~CRYPTO_TFM_REQ_ON_STACK;
578
579 if (req->src == &req->chain.ssg)
580 nreq->src = &nreq->chain.ssg;
581 if (req->dst == &req->chain.dsg)
582 nreq->dst = &nreq->chain.dsg;
583 return nreq;
584}
585EXPORT_SYMBOL_GPL(acomp_request_clone);
586
587MODULE_LICENSE("GPL");
588MODULE_DESCRIPTION("Asynchronous compression type");