projects/linux·blob master · refs

687 lines · 15.8 KiB

1// SPDX-License-Identifier: GPL-2.0-or-later
2/*
3 * Scatterlist Cryptographic API.
4 *
5 * Copyright (c) 2002 James Morris <[email protected]>
6 * Copyright (c) 2002 David S. Miller ([email protected])
7 * Copyright (c) 2005 Herbert Xu <[email protected]>
8 *
9 * Portions derived from Cryptoapi, by Alexander Kjeldaas <[email protected]>
10 * and Nettle, by Niels Möller.
11 */
12
13#include <linux/err.h>
14#include <linux/errno.h>
15#include <linux/jump_label.h>
16#include <linux/kernel.h>
17#include <linux/kmod.h>
18#include <linux/module.h>
19#include <linux/param.h>
20#include <linux/sched/signal.h>
21#include <linux/slab.h>
22#include <linux/string.h>
23#include <linux/completion.h>
24#include "internal.h"
25
26LIST_HEAD(crypto_alg_list);
27EXPORT_SYMBOL_GPL(crypto_alg_list);
28DECLARE_RWSEM(crypto_alg_sem);
29EXPORT_SYMBOL_GPL(crypto_alg_sem);
30
31BLOCKING_NOTIFIER_HEAD(crypto_chain);
32EXPORT_SYMBOL_GPL(crypto_chain);
33
34#if IS_BUILTIN(CONFIG_CRYPTO_ALGAPI) && IS_ENABLED(CONFIG_CRYPTO_SELFTESTS)
35DEFINE_STATIC_KEY_FALSE(__crypto_boot_test_finished);
36#endif
37
38static struct crypto_alg *crypto_larval_wait(struct crypto_alg *alg,
39 u32 type, u32 mask);
40static struct crypto_alg *crypto_alg_lookup(const char *name, u32 type,
41 u32 mask);
42
43struct crypto_alg *crypto_mod_get(struct crypto_alg *alg)
44{
45 return try_module_get(alg->cra_module) ? crypto_alg_get(alg) : NULL;
46}
47EXPORT_SYMBOL_GPL(crypto_mod_get);
48
49void crypto_mod_put(struct crypto_alg *alg)
50{
51 struct module *module = alg->cra_module;
52
53 crypto_alg_put(alg);
54 module_put(module);
55}
56EXPORT_SYMBOL_GPL(crypto_mod_put);
57
58static struct crypto_alg *__crypto_alg_lookup(const char *name, u32 type,
59 u32 mask)
60 __must_hold_shared(&crypto_alg_sem)
61{
62 struct crypto_alg *q, *alg = NULL;
63 int best = -2;
64
65 list_for_each_entry(q, &crypto_alg_list, cra_list) {
66 int exact, fuzzy;
67
68 if (crypto_is_moribund(q))
69 continue;
70
71 if ((q->cra_flags ^ type) & mask)
72 continue;
73
74 exact = !strcmp(q->cra_driver_name, name);
75 fuzzy = !strcmp(q->cra_name, name);
76 if (!exact && !(fuzzy && q->cra_priority > best))
77 continue;
78
79 if (unlikely(!crypto_mod_get(q)))
80 continue;
81
82 best = q->cra_priority;
83 if (alg)
84 crypto_mod_put(alg);
85 alg = q;
86
87 if (exact)
88 break;
89 }
90
91 return alg;
92}
93
94static void crypto_larval_destroy(struct crypto_alg *alg)
95{
96 struct crypto_larval *larval = (void *)alg;
97
98 BUG_ON(!crypto_is_larval(alg));
99 if (!IS_ERR_OR_NULL(larval->adult))
100 crypto_mod_put(larval->adult);
101 kfree(larval);
102}
103
104struct crypto_larval *crypto_larval_alloc(const char *name, u32 type, u32 mask)
105{
106 struct crypto_larval *larval;
107
108 larval = kzalloc_obj(*larval);
109 if (!larval)
110 return ERR_PTR(-ENOMEM);
111
112 type &= ~CRYPTO_ALG_TYPE_MASK | (mask ?: CRYPTO_ALG_TYPE_MASK);
113
114 larval->mask = mask;
115 larval->alg.cra_flags = CRYPTO_ALG_LARVAL | type;
116 larval->alg.cra_priority = -1;
117 larval->alg.cra_destroy = crypto_larval_destroy;
118
119 strscpy(larval->alg.cra_name, name);
120 init_completion(&larval->completion);
121
122 return larval;
123}
124EXPORT_SYMBOL_GPL(crypto_larval_alloc);
125
126static struct crypto_alg *crypto_larval_add(const char *name, u32 type,
127 u32 mask)
128{
129 struct crypto_alg *alg;
130 struct crypto_larval *larval;
131
132 larval = crypto_larval_alloc(name, type, mask);
133 if (IS_ERR(larval))
134 return ERR_CAST(larval);
135
136 refcount_set(&larval->alg.cra_refcnt, 2);
137
138 down_write(&crypto_alg_sem);
139 alg = __crypto_alg_lookup(name, type, mask);
140 if (!alg) {
141 alg = &larval->alg;
142 list_add(&alg->cra_list, &crypto_alg_list);
143 }
144 up_write(&crypto_alg_sem);
145
146 if (alg != &larval->alg) {
147 kfree(larval);
148 if (crypto_is_larval(alg))
149 alg = crypto_larval_wait(alg, type, mask);
150 }
151
152 return alg;
153}
154
155static void crypto_larval_kill(struct crypto_larval *larval)
156{
157 bool unlinked;
158
159 down_write(&crypto_alg_sem);
160 unlinked = list_empty(&larval->alg.cra_list);
161 if (!unlinked)
162 list_del_init(&larval->alg.cra_list);
163 up_write(&crypto_alg_sem);
164
165 if (unlinked)
166 return;
167
168 complete_all(&larval->completion);
169 crypto_alg_put(&larval->alg);
170}
171
172void crypto_schedule_test(struct crypto_larval *larval)
173{
174 int err;
175
176 err = crypto_probing_notify(CRYPTO_MSG_ALG_REGISTER, larval->adult);
177 WARN_ON_ONCE(err != NOTIFY_STOP);
178}
179EXPORT_SYMBOL_GPL(crypto_schedule_test);
180
181static void crypto_start_test(struct crypto_larval *larval)
182{
183 if (!crypto_is_test_larval(larval))
184 return;
185
186 if (larval->test_started)
187 return;
188
189 down_write(&crypto_alg_sem);
190 if (larval->test_started) {
191 up_write(&crypto_alg_sem);
192 return;
193 }
194
195 larval->test_started = true;
196 up_write(&crypto_alg_sem);
197
198 crypto_schedule_test(larval);
199}
200
201static struct crypto_alg *crypto_larval_wait(struct crypto_alg *alg,
202 u32 type, u32 mask)
203{
204 struct crypto_larval *larval;
205 long time_left;
206
207again:
208 larval = container_of(alg, struct crypto_larval, alg);
209
210 if (!crypto_boot_test_finished())
211 crypto_start_test(larval);
212
213 time_left = wait_for_completion_killable_timeout(
214 &larval->completion, 60 * HZ);
215
216 alg = larval->adult;
217 if (time_left < 0)
218 alg = ERR_PTR(-EINTR);
219 else if (!time_left) {
220 if (crypto_is_test_larval(larval))
221 crypto_larval_kill(larval);
222 alg = ERR_PTR(-ETIMEDOUT);
223 } else if (!alg || PTR_ERR(alg) == -EEXIST) {
224 int err = alg ? -EEXIST : -EAGAIN;
225
226 /*
227 * EEXIST is expected because two probes can be scheduled
228 * at the same time with one using alg_name and the other
229 * using driver_name. Do a re-lookup but do not retry in
230 * case we hit a quirk like gcm_base(ctr(aes),...) which
231 * will never match.
232 */
233 alg = &larval->alg;
234 alg = crypto_alg_lookup(alg->cra_name, type, mask) ?:
235 ERR_PTR(err);
236 } else if (IS_ERR(alg))
237 ;
238 else if (crypto_is_test_larval(larval) &&
239 !(alg->cra_flags & CRYPTO_ALG_TESTED))
240 alg = ERR_PTR(-EAGAIN);
241 else if (alg->cra_flags & CRYPTO_ALG_FIPS_INTERNAL)
242 alg = ERR_PTR(-EAGAIN);
243 else if (!crypto_mod_get(alg))
244 alg = ERR_PTR(-EAGAIN);
245 crypto_mod_put(&larval->alg);
246
247 if (!IS_ERR(alg) && crypto_is_larval(alg))
248 goto again;
249
250 return alg;
251}
252
253static struct crypto_alg *crypto_alg_lookup(const char *name, u32 type,
254 u32 mask)
255{
256 const u32 fips = CRYPTO_ALG_FIPS_INTERNAL;
257 struct crypto_alg *alg;
258 u32 test = 0;
259
260 if (!((type | mask) & CRYPTO_ALG_TESTED))
261 test |= CRYPTO_ALG_TESTED;
262
263 down_read(&crypto_alg_sem);
264 alg = __crypto_alg_lookup(name, (type | test) & ~fips,
265 (mask | test) & ~fips);
266 if (alg) {
267 if (((type | mask) ^ fips) & fips)
268 mask |= fips;
269 mask &= fips;
270
271 if (!crypto_is_larval(alg) &&
272 ((type ^ alg->cra_flags) & mask)) {
273 /* Algorithm is disallowed in FIPS mode. */
274 crypto_mod_put(alg);
275 alg = ERR_PTR(-ENOENT);
276 }
277 } else if (test) {
278 alg = __crypto_alg_lookup(name, type, mask);
279 if (alg && !crypto_is_larval(alg)) {
280 /* Test failed */
281 crypto_mod_put(alg);
282 alg = ERR_PTR(-ELIBBAD);
283 }
284 }
285 up_read(&crypto_alg_sem);
286
287 return alg;
288}
289
290static struct crypto_alg *crypto_larval_lookup(const char *name, u32 type,
291 u32 mask)
292{
293 struct crypto_alg *alg;
294
295 if (!name)
296 return ERR_PTR(-ENOENT);
297
298 type &= ~(CRYPTO_ALG_LARVAL | CRYPTO_ALG_DEAD);
299 mask &= ~(CRYPTO_ALG_LARVAL | CRYPTO_ALG_DEAD);
300
301 alg = crypto_alg_lookup(name, type, mask);
302 if (!alg && !(mask & CRYPTO_NOLOAD)) {
303 request_module("crypto-%s", name);
304
305 if (!((type ^ CRYPTO_ALG_NEED_FALLBACK) & mask &
306 CRYPTO_ALG_NEED_FALLBACK))
307 request_module("crypto-%s-all", name);
308
309 alg = crypto_alg_lookup(name, type, mask);
310 }
311
312 if (!IS_ERR_OR_NULL(alg) && crypto_is_larval(alg))
313 alg = crypto_larval_wait(alg, type, mask);
314 else if (alg)
315 ;
316 else if (!(mask & CRYPTO_ALG_TESTED))
317 alg = crypto_larval_add(name, type, mask);
318 else
319 alg = ERR_PTR(-ENOENT);
320
321 return alg;
322}
323
324int crypto_probing_notify(unsigned long val, void *v)
325{
326 int ok;
327
328 ok = blocking_notifier_call_chain(&crypto_chain, val, v);
329 if (ok == NOTIFY_DONE) {
330 request_module("cryptomgr");
331 ok = blocking_notifier_call_chain(&crypto_chain, val, v);
332 }
333
334 return ok;
335}
336EXPORT_SYMBOL_GPL(crypto_probing_notify);
337
338struct crypto_alg *crypto_alg_mod_lookup(const char *name, u32 type, u32 mask)
339{
340 struct crypto_alg *alg;
341 struct crypto_alg *larval;
342 int ok;
343
344 /*
345 * If the internal flag is set for a cipher, require a caller to
346 * invoke the cipher with the internal flag to use that cipher.
347 * Also, if a caller wants to allocate a cipher that may or may
348 * not be an internal cipher, use type | CRYPTO_ALG_INTERNAL and
349 * !(mask & CRYPTO_ALG_INTERNAL).
350 */
351 if (!((type | mask) & CRYPTO_ALG_INTERNAL))
352 mask |= CRYPTO_ALG_INTERNAL;
353
354 larval = crypto_larval_lookup(name, type, mask);
355 if (IS_ERR(larval) || !crypto_is_larval(larval))
356 return larval;
357
358 ok = crypto_probing_notify(CRYPTO_MSG_ALG_REQUEST, larval);
359
360 if (ok == NOTIFY_STOP)
361 alg = crypto_larval_wait(larval, type, mask);
362 else {
363 crypto_mod_put(larval);
364 alg = ERR_PTR(-ENOENT);
365 }
366 crypto_larval_kill(container_of(larval, struct crypto_larval, alg));
367 return alg;
368}
369EXPORT_SYMBOL_GPL(crypto_alg_mod_lookup);
370
371static void crypto_exit_ops(struct crypto_tfm *tfm)
372{
373 const struct crypto_type *type = tfm->__crt_alg->cra_type;
374
375 if (type && tfm->exit)
376 tfm->exit(tfm);
377}
378
379static unsigned int crypto_ctxsize(struct crypto_alg *alg, u32 type, u32 mask)
380{
381 const struct crypto_type *type_obj = alg->cra_type;
382 unsigned int len;
383
384 len = alg->cra_alignmask & ~(crypto_tfm_ctx_alignment() - 1);
385 if (type_obj)
386 return len + type_obj->ctxsize(alg, type, mask);
387
388 switch (alg->cra_flags & CRYPTO_ALG_TYPE_MASK) {
389 default:
390 BUG();
391
392 case CRYPTO_ALG_TYPE_CIPHER:
393 len += crypto_cipher_ctxsize(alg);
394 break;
395 }
396
397 return len;
398}
399
400void crypto_shoot_alg(struct crypto_alg *alg)
401{
402 down_write(&crypto_alg_sem);
403 alg->cra_flags |= CRYPTO_ALG_DYING;
404 up_write(&crypto_alg_sem);
405}
406EXPORT_SYMBOL_GPL(crypto_shoot_alg);
407
408struct crypto_tfm *__crypto_alloc_tfm(struct crypto_alg *alg, u32 type,
409 u32 mask)
410{
411 struct crypto_tfm *tfm;
412 unsigned int tfm_size;
413 int err = -ENOMEM;
414
415 tfm_size = sizeof(*tfm) + crypto_ctxsize(alg, type, mask);
416 tfm = kzalloc(tfm_size, GFP_KERNEL);
417 if (tfm == NULL)
418 goto out_err;
419
420 tfm->__crt_alg = alg;
421
422 if (!tfm->exit && alg->cra_init && (err = alg->cra_init(tfm)))
423 goto cra_init_failed;
424
425 goto out;
426
427cra_init_failed:
428 crypto_exit_ops(tfm);
429 if (err == -EAGAIN)
430 crypto_shoot_alg(alg);
431 kfree(tfm);
432out_err:
433 tfm = ERR_PTR(err);
434out:
435 return tfm;
436}
437EXPORT_SYMBOL_GPL(__crypto_alloc_tfm);
438
439/*
440 * crypto_alloc_base - Locate algorithm and allocate transform
441 * @alg_name: Name of algorithm
442 * @type: Type of algorithm
443 * @mask: Mask for type comparison
444 *
445 * This function should not be used by new algorithm types.
446 * Please use crypto_alloc_tfm instead.
447 *
448 * crypto_alloc_base() will first attempt to locate an already loaded
449 * algorithm. If that fails and the kernel supports dynamically loadable
450 * modules, it will then attempt to load a module of the same name or
451 * alias. If that fails it will send a query to any loaded crypto manager
452 * to construct an algorithm on the fly. A refcount is grabbed on the
453 * algorithm which is then associated with the new transform.
454 *
455 * The returned transform is of a non-determinate type. Most people
456 * should use one of the more specific allocation functions such as
457 * crypto_alloc_skcipher().
458 *
459 * In case of error the return value is an error pointer.
460 */
461struct crypto_tfm *crypto_alloc_base(const char *alg_name, u32 type, u32 mask)
462{
463 struct crypto_tfm *tfm;
464 int err;
465
466 for (;;) {
467 struct crypto_alg *alg;
468
469 alg = crypto_alg_mod_lookup(alg_name, type, mask);
470 if (IS_ERR(alg)) {
471 err = PTR_ERR(alg);
472 goto err;
473 }
474
475 tfm = __crypto_alloc_tfm(alg, type, mask);
476 if (!IS_ERR(tfm))
477 return tfm;
478
479 crypto_mod_put(alg);
480 err = PTR_ERR(tfm);
481
482err:
483 if (err != -EAGAIN)
484 break;
485 if (fatal_signal_pending(current)) {
486 err = -EINTR;
487 break;
488 }
489 }
490
491 return ERR_PTR(err);
492}
493EXPORT_SYMBOL_GPL(crypto_alloc_base);
494
495void *crypto_create_tfm_node(struct crypto_alg *alg,
496 const struct crypto_type *frontend,
497 int node)
498{
499 struct crypto_tfm *tfm;
500 size_t size;
501 char *mem;
502 int err;
503
504 size = frontend->tfmsize + sizeof(*tfm) + frontend->extsize(alg);
505 mem = kzalloc_node(size, GFP_KERNEL, node);
506 if (!mem)
507 return ERR_PTR(-ENOMEM);
508
509 tfm = (struct crypto_tfm *)(mem + frontend->tfmsize);
510 tfm->__crt_alg = alg;
511 tfm->node = node;
512 tfm->fb = tfm;
513
514 err = frontend->init_tfm(tfm);
515 if (err)
516 goto out_free_tfm;
517
518 if (!tfm->exit && alg->cra_init && (err = alg->cra_init(tfm)))
519 goto cra_init_failed;
520
521 goto out;
522
523cra_init_failed:
524 crypto_exit_ops(tfm);
525out_free_tfm:
526 if (err == -EAGAIN)
527 crypto_shoot_alg(alg);
528 kfree(mem);
529 mem = ERR_PTR(err);
530out:
531 return mem;
532}
533EXPORT_SYMBOL_GPL(crypto_create_tfm_node);
534
535struct crypto_alg *crypto_find_alg(const char *alg_name,
536 const struct crypto_type *frontend,
537 u32 type, u32 mask)
538{
539 if (frontend) {
540 type &= frontend->maskclear;
541 mask &= frontend->maskclear;
542 type |= frontend->type;
543 mask |= frontend->maskset;
544 }
545
546 return crypto_alg_mod_lookup(alg_name, type, mask);
547}
548EXPORT_SYMBOL_GPL(crypto_find_alg);
549
550/*
551 * crypto_alloc_tfm_node - Locate algorithm and allocate transform
552 * @alg_name: Name of algorithm
553 * @frontend: Frontend algorithm type
554 * @type: Type of algorithm
555 * @mask: Mask for type comparison
556 * @node: NUMA node in which users desire to put requests, if node is
557 * NUMA_NO_NODE, it means users have no special requirement.
558 *
559 * crypto_alloc_tfm() will first attempt to locate an already loaded
560 * algorithm. If that fails and the kernel supports dynamically loadable
561 * modules, it will then attempt to load a module of the same name or
562 * alias. If that fails it will send a query to any loaded crypto manager
563 * to construct an algorithm on the fly. A refcount is grabbed on the
564 * algorithm which is then associated with the new transform.
565 *
566 * The returned transform is of a non-determinate type. Most people
567 * should use one of the more specific allocation functions such as
568 * crypto_alloc_skcipher().
569 *
570 * In case of error the return value is an error pointer.
571 */
572
573void *crypto_alloc_tfm_node(const char *alg_name,
574 const struct crypto_type *frontend, u32 type, u32 mask,
575 int node)
576{
577 void *tfm;
578 int err;
579
580 for (;;) {
581 struct crypto_alg *alg;
582
583 alg = crypto_find_alg(alg_name, frontend, type, mask);
584 if (IS_ERR(alg)) {
585 err = PTR_ERR(alg);
586 goto err;
587 }
588
589 tfm = crypto_create_tfm_node(alg, frontend, node);
590 if (!IS_ERR(tfm))
591 return tfm;
592
593 crypto_mod_put(alg);
594 err = PTR_ERR(tfm);
595
596err:
597 if (err != -EAGAIN)
598 break;
599 if (fatal_signal_pending(current)) {
600 err = -EINTR;
601 break;
602 }
603 }
604
605 return ERR_PTR(err);
606}
607EXPORT_SYMBOL_GPL(crypto_alloc_tfm_node);
608
609/*
610 * crypto_destroy_tfm - Free crypto transform
611 * @mem: Start of tfm slab
612 * @tfm: Transform to free
613 *
614 * This function frees up the transform and any associated resources,
615 * then drops the refcount on the associated algorithm.
616 */
617void crypto_destroy_tfm(void *mem, struct crypto_tfm *tfm)
618{
619 struct crypto_alg *alg;
620
621 if (IS_ERR_OR_NULL(mem))
622 return;
623
624 alg = tfm->__crt_alg;
625
626 if (!tfm->exit && alg->cra_exit)
627 alg->cra_exit(tfm);
628 crypto_exit_ops(tfm);
629 crypto_mod_put(alg);
630 kfree_sensitive(mem);
631}
632EXPORT_SYMBOL_GPL(crypto_destroy_tfm);
633
634int crypto_has_alg(const char *name, u32 type, u32 mask)
635{
636 int ret = 0;
637 struct crypto_alg *alg = crypto_alg_mod_lookup(name, type, mask);
638
639 if (!IS_ERR(alg)) {
640 crypto_mod_put(alg);
641 ret = 1;
642 }
643
644 return ret;
645}
646EXPORT_SYMBOL_GPL(crypto_has_alg);
647
648void crypto_req_done(void *data, int err)
649{
650 struct crypto_wait *wait = data;
651
652 if (err == -EINPROGRESS)
653 return;
654
655 wait->err = err;
656 complete(&wait->completion);
657}
658EXPORT_SYMBOL_GPL(crypto_req_done);
659
660void crypto_destroy_alg(struct crypto_alg *alg)
661{
662 if (alg->cra_type && alg->cra_type->destroy)
663 alg->cra_type->destroy(alg);
664 if (alg->cra_destroy)
665 alg->cra_destroy(alg);
666}
667EXPORT_SYMBOL_GPL(crypto_destroy_alg);
668
669struct crypto_async_request *crypto_request_clone(
670 struct crypto_async_request *req, size_t total, gfp_t gfp)
671{
672 struct crypto_tfm *tfm = req->tfm;
673 struct crypto_async_request *nreq;
674
675 nreq = kmemdup(req, total, gfp);
676 if (!nreq) {
677 req->tfm = tfm->fb;
678 return req;
679 }
680
681 nreq->flags &= ~CRYPTO_TFM_REQ_ON_STACK;
682 return nreq;
683}
684EXPORT_SYMBOL_GPL(crypto_request_clone);
685
686MODULE_DESCRIPTION("Cryptographic core API");
687MODULE_LICENSE("GPL");