projects/linux·blob master · refs

1471 lines · 39.8 KiB

1// SPDX-License-Identifier: GPL-2.0
2/* Copyright (C) 2012-2019 ARM Limited (or its affiliates). */
3
4#include <linux/kernel.h>
5#include <linux/module.h>
6#include <linux/string.h>
7#include <crypto/algapi.h>
8#include <crypto/internal/skcipher.h>
9#include <crypto/internal/des.h>
10#include <crypto/xts.h>
11#include <crypto/sm4.h>
12#include <crypto/scatterwalk.h>
13
14#include "cc_driver.h"
15#include "cc_lli_defs.h"
16#include "cc_buffer_mgr.h"
17#include "cc_cipher.h"
18#include "cc_request_mgr.h"
19
20#define MAX_SKCIPHER_SEQ_LEN 6
21
22#define template_skcipher template_u.skcipher
23
24struct cc_user_key_info {
25 u8 *key;
26 dma_addr_t key_dma_addr;
27};
28
29struct cc_hw_key_info {
30 enum cc_hw_crypto_key key1_slot;
31 enum cc_hw_crypto_key key2_slot;
32};
33
34struct cc_cpp_key_info {
35 u8 slot;
36 enum cc_cpp_alg alg;
37};
38
39enum cc_key_type {
40 CC_UNPROTECTED_KEY, /* User key */
41 CC_HW_PROTECTED_KEY, /* HW (FDE) key */
42 CC_POLICY_PROTECTED_KEY, /* CPP key */
43 CC_INVALID_PROTECTED_KEY /* Invalid key */
44};
45
46struct cc_cipher_ctx {
47 struct cc_drvdata *drvdata;
48 int keylen;
49 int cipher_mode;
50 int flow_mode;
51 unsigned int flags;
52 enum cc_key_type key_type;
53 struct cc_user_key_info user;
54 union {
55 struct cc_hw_key_info hw;
56 struct cc_cpp_key_info cpp;
57 };
58 struct crypto_shash *shash_tfm;
59 struct crypto_skcipher *fallback_tfm;
60 bool fallback_on;
61};
62
63static void cc_cipher_complete(struct device *dev, void *cc_req, int err);
64
65static inline enum cc_key_type cc_key_type(struct crypto_tfm *tfm)
66{
67 struct cc_cipher_ctx *ctx_p = crypto_tfm_ctx(tfm);
68
69 return ctx_p->key_type;
70}
71
72static int validate_keys_sizes(struct cc_cipher_ctx *ctx_p, u32 size)
73{
74 switch (ctx_p->flow_mode) {
75 case S_DIN_to_AES:
76 switch (size) {
77 case CC_AES_128_BIT_KEY_SIZE:
78 case CC_AES_192_BIT_KEY_SIZE:
79 if (ctx_p->cipher_mode != DRV_CIPHER_XTS)
80 return 0;
81 break;
82 case CC_AES_256_BIT_KEY_SIZE:
83 return 0;
84 case (CC_AES_192_BIT_KEY_SIZE * 2):
85 case (CC_AES_256_BIT_KEY_SIZE * 2):
86 if (ctx_p->cipher_mode == DRV_CIPHER_XTS ||
87 ctx_p->cipher_mode == DRV_CIPHER_ESSIV)
88 return 0;
89 break;
90 default:
91 break;
92 }
93 break;
94 case S_DIN_to_DES:
95 if (size == DES3_EDE_KEY_SIZE || size == DES_KEY_SIZE)
96 return 0;
97 break;
98 case S_DIN_to_SM4:
99 if (size == SM4_KEY_SIZE)
100 return 0;
101 break;
102 default:
103 break;
104 }
105 return -EINVAL;
106}
107
108static int validate_data_size(struct cc_cipher_ctx *ctx_p,
109 unsigned int size)
110{
111 switch (ctx_p->flow_mode) {
112 case S_DIN_to_AES:
113 switch (ctx_p->cipher_mode) {
114 case DRV_CIPHER_XTS:
115 case DRV_CIPHER_CBC_CTS:
116 if (size >= AES_BLOCK_SIZE)
117 return 0;
118 break;
119 case DRV_CIPHER_OFB:
120 case DRV_CIPHER_CTR:
121 return 0;
122 case DRV_CIPHER_ECB:
123 case DRV_CIPHER_CBC:
124 case DRV_CIPHER_ESSIV:
125 if (IS_ALIGNED(size, AES_BLOCK_SIZE))
126 return 0;
127 break;
128 default:
129 break;
130 }
131 break;
132 case S_DIN_to_DES:
133 if (IS_ALIGNED(size, DES_BLOCK_SIZE))
134 return 0;
135 break;
136 case S_DIN_to_SM4:
137 switch (ctx_p->cipher_mode) {
138 case DRV_CIPHER_CTR:
139 return 0;
140 case DRV_CIPHER_ECB:
141 case DRV_CIPHER_CBC:
142 if (IS_ALIGNED(size, SM4_BLOCK_SIZE))
143 return 0;
144 break;
145 default:
146 break;
147 }
148 break;
149 default:
150 break;
151 }
152 return -EINVAL;
153}
154
155static int cc_cipher_init(struct crypto_tfm *tfm)
156{
157 struct cc_cipher_ctx *ctx_p = crypto_tfm_ctx(tfm);
158 struct cc_crypto_alg *cc_alg =
159 container_of(tfm->__crt_alg, struct cc_crypto_alg,
160 skcipher_alg.base);
161 struct device *dev = drvdata_to_dev(cc_alg->drvdata);
162 unsigned int max_key_buf_size = cc_alg->skcipher_alg.max_keysize;
163 unsigned int fallback_req_size = 0;
164
165 dev_dbg(dev, "Initializing context @%p for %s\n", ctx_p,
166 crypto_tfm_alg_name(tfm));
167
168 ctx_p->cipher_mode = cc_alg->cipher_mode;
169 ctx_p->flow_mode = cc_alg->flow_mode;
170 ctx_p->drvdata = cc_alg->drvdata;
171
172 if (ctx_p->cipher_mode == DRV_CIPHER_ESSIV) {
173 const char *name = crypto_tfm_alg_name(tfm);
174
175 /* Alloc hash tfm for essiv */
176 ctx_p->shash_tfm = crypto_alloc_shash("sha256", 0, 0);
177 if (IS_ERR(ctx_p->shash_tfm)) {
178 dev_err(dev, "Error allocating hash tfm for ESSIV.\n");
179 return PTR_ERR(ctx_p->shash_tfm);
180 }
181 max_key_buf_size <<= 1;
182
183 /* Alloc fallback tfm or essiv when key size != 256 bit */
184 ctx_p->fallback_tfm =
185 crypto_alloc_skcipher(name, 0, CRYPTO_ALG_NEED_FALLBACK | CRYPTO_ALG_ASYNC);
186
187 if (IS_ERR(ctx_p->fallback_tfm)) {
188 /* Note we're still allowing registration with no fallback since it's
189 * better to have most modes supported than none at all.
190 */
191 dev_warn(dev, "Error allocating fallback algo %s. Some modes may be available.\n",
192 name);
193 ctx_p->fallback_tfm = NULL;
194 } else {
195 fallback_req_size = crypto_skcipher_reqsize(ctx_p->fallback_tfm);
196 }
197 }
198
199 crypto_skcipher_set_reqsize(__crypto_skcipher_cast(tfm),
200 sizeof(struct cipher_req_ctx) + fallback_req_size);
201
202 /* Allocate key buffer, cache line aligned */
203 ctx_p->user.key = kzalloc(max_key_buf_size, GFP_KERNEL);
204 if (!ctx_p->user.key)
205 goto free_fallback;
206
207 dev_dbg(dev, "Allocated key buffer in context. key=@%p\n",
208 ctx_p->user.key);
209
210 /* Map key buffer */
211 ctx_p->user.key_dma_addr = dma_map_single(dev, ctx_p->user.key,
212 max_key_buf_size,
213 DMA_TO_DEVICE);
214 if (dma_mapping_error(dev, ctx_p->user.key_dma_addr)) {
215 dev_err(dev, "Mapping Key %u B at va=%p for DMA failed\n",
216 max_key_buf_size, ctx_p->user.key);
217 goto free_key;
218 }
219 dev_dbg(dev, "Mapped key %u B at va=%p to dma=%pad\n",
220 max_key_buf_size, ctx_p->user.key, &ctx_p->user.key_dma_addr);
221
222 return 0;
223
224free_key:
225 kfree(ctx_p->user.key);
226free_fallback:
227 crypto_free_skcipher(ctx_p->fallback_tfm);
228 crypto_free_shash(ctx_p->shash_tfm);
229
230 return -ENOMEM;
231}
232
233static void cc_cipher_exit(struct crypto_tfm *tfm)
234{
235 struct crypto_alg *alg = tfm->__crt_alg;
236 struct cc_crypto_alg *cc_alg =
237 container_of(alg, struct cc_crypto_alg,
238 skcipher_alg.base);
239 unsigned int max_key_buf_size = cc_alg->skcipher_alg.max_keysize;
240 struct cc_cipher_ctx *ctx_p = crypto_tfm_ctx(tfm);
241 struct device *dev = drvdata_to_dev(ctx_p->drvdata);
242
243 dev_dbg(dev, "Clearing context @%p for %s\n",
244 crypto_tfm_ctx(tfm), crypto_tfm_alg_name(tfm));
245
246 if (ctx_p->cipher_mode == DRV_CIPHER_ESSIV) {
247 /* Free hash tfm for essiv */
248 crypto_free_shash(ctx_p->shash_tfm);
249 ctx_p->shash_tfm = NULL;
250 crypto_free_skcipher(ctx_p->fallback_tfm);
251 ctx_p->fallback_tfm = NULL;
252 }
253
254 /* Unmap key buffer */
255 dma_unmap_single(dev, ctx_p->user.key_dma_addr, max_key_buf_size,
256 DMA_TO_DEVICE);
257 dev_dbg(dev, "Unmapped key buffer key_dma_addr=%pad\n",
258 &ctx_p->user.key_dma_addr);
259
260 /* Free key buffer in context */
261 dev_dbg(dev, "Free key buffer in context. key=@%p\n", ctx_p->user.key);
262 kfree_sensitive(ctx_p->user.key);
263}
264
265static enum cc_hw_crypto_key cc_slot_to_hw_key(u8 slot_num)
266{
267 switch (slot_num) {
268 case 0:
269 return KFDE0_KEY;
270 case 1:
271 return KFDE1_KEY;
272 case 2:
273 return KFDE2_KEY;
274 case 3:
275 return KFDE3_KEY;
276 }
277 return END_OF_KEYS;
278}
279
280static u8 cc_slot_to_cpp_key(u8 slot_num)
281{
282 return (slot_num - CC_FIRST_CPP_KEY_SLOT);
283}
284
285static inline enum cc_key_type cc_slot_to_key_type(u8 slot_num)
286{
287 if (slot_num >= CC_FIRST_HW_KEY_SLOT && slot_num <= CC_LAST_HW_KEY_SLOT)
288 return CC_HW_PROTECTED_KEY;
289 else if (slot_num >= CC_FIRST_CPP_KEY_SLOT &&
290 slot_num <= CC_LAST_CPP_KEY_SLOT)
291 return CC_POLICY_PROTECTED_KEY;
292 else
293 return CC_INVALID_PROTECTED_KEY;
294}
295
296static int cc_cipher_sethkey(struct crypto_skcipher *sktfm, const u8 *key,
297 unsigned int keylen)
298{
299 struct crypto_tfm *tfm = crypto_skcipher_tfm(sktfm);
300 struct cc_cipher_ctx *ctx_p = crypto_tfm_ctx(tfm);
301 struct device *dev = drvdata_to_dev(ctx_p->drvdata);
302 struct cc_hkey_info hki;
303
304 dev_dbg(dev, "Setting HW key in context @%p for %s. keylen=%u\n",
305 ctx_p, crypto_tfm_alg_name(tfm), keylen);
306 dump_byte_array("key", key, keylen);
307
308 /* STAT_PHASE_0: Init and sanity checks */
309
310 /* This check the size of the protected key token */
311 if (keylen != sizeof(hki)) {
312 dev_err(dev, "Unsupported protected key size %d.\n", keylen);
313 return -EINVAL;
314 }
315
316 memcpy(&hki, key, keylen);
317
318 /* The real key len for crypto op is the size of the HW key
319 * referenced by the HW key slot, not the hardware key token
320 */
321 keylen = hki.keylen;
322
323 if (validate_keys_sizes(ctx_p, keylen)) {
324 dev_dbg(dev, "Unsupported key size %d.\n", keylen);
325 return -EINVAL;
326 }
327
328 ctx_p->keylen = keylen;
329 ctx_p->fallback_on = false;
330
331 switch (cc_slot_to_key_type(hki.hw_key1)) {
332 case CC_HW_PROTECTED_KEY:
333 if (ctx_p->flow_mode == S_DIN_to_SM4) {
334 dev_err(dev, "Only AES HW protected keys are supported\n");
335 return -EINVAL;
336 }
337
338 ctx_p->hw.key1_slot = cc_slot_to_hw_key(hki.hw_key1);
339 if (ctx_p->hw.key1_slot == END_OF_KEYS) {
340 dev_err(dev, "Unsupported hw key1 number (%d)\n",
341 hki.hw_key1);
342 return -EINVAL;
343 }
344
345 if (ctx_p->cipher_mode == DRV_CIPHER_XTS ||
346 ctx_p->cipher_mode == DRV_CIPHER_ESSIV) {
347 if (hki.hw_key1 == hki.hw_key2) {
348 dev_err(dev, "Illegal hw key numbers (%d,%d)\n",
349 hki.hw_key1, hki.hw_key2);
350 return -EINVAL;
351 }
352
353 ctx_p->hw.key2_slot = cc_slot_to_hw_key(hki.hw_key2);
354 if (ctx_p->hw.key2_slot == END_OF_KEYS) {
355 dev_err(dev, "Unsupported hw key2 number (%d)\n",
356 hki.hw_key2);
357 return -EINVAL;
358 }
359 }
360
361 ctx_p->key_type = CC_HW_PROTECTED_KEY;
362 dev_dbg(dev, "HW protected key %d/%d set\n.",
363 ctx_p->hw.key1_slot, ctx_p->hw.key2_slot);
364 break;
365
366 case CC_POLICY_PROTECTED_KEY:
367 if (ctx_p->drvdata->hw_rev < CC_HW_REV_713) {
368 dev_err(dev, "CPP keys not supported in this hardware revision.\n");
369 return -EINVAL;
370 }
371
372 if (ctx_p->cipher_mode != DRV_CIPHER_CBC &&
373 ctx_p->cipher_mode != DRV_CIPHER_CTR) {
374 dev_err(dev, "CPP keys only supported in CBC or CTR modes.\n");
375 return -EINVAL;
376 }
377
378 ctx_p->cpp.slot = cc_slot_to_cpp_key(hki.hw_key1);
379 if (ctx_p->flow_mode == S_DIN_to_AES)
380 ctx_p->cpp.alg = CC_CPP_AES;
381 else /* Must be SM4 since due to sethkey registration */
382 ctx_p->cpp.alg = CC_CPP_SM4;
383 ctx_p->key_type = CC_POLICY_PROTECTED_KEY;
384 dev_dbg(dev, "policy protected key alg: %d slot: %d.\n",
385 ctx_p->cpp.alg, ctx_p->cpp.slot);
386 break;
387
388 default:
389 dev_err(dev, "Unsupported protected key (%d)\n", hki.hw_key1);
390 return -EINVAL;
391 }
392
393 return 0;
394}
395
396static int cc_cipher_setkey(struct crypto_skcipher *sktfm, const u8 *key,
397 unsigned int keylen)
398{
399 struct crypto_tfm *tfm = crypto_skcipher_tfm(sktfm);
400 struct cc_cipher_ctx *ctx_p = crypto_tfm_ctx(tfm);
401 struct device *dev = drvdata_to_dev(ctx_p->drvdata);
402 struct cc_crypto_alg *cc_alg =
403 container_of(tfm->__crt_alg, struct cc_crypto_alg,
404 skcipher_alg.base);
405 unsigned int max_key_buf_size = cc_alg->skcipher_alg.max_keysize;
406
407 dev_dbg(dev, "Setting key in context @%p for %s. keylen=%u\n",
408 ctx_p, crypto_tfm_alg_name(tfm), keylen);
409 dump_byte_array("key", key, keylen);
410
411 /* STAT_PHASE_0: Init and sanity checks */
412
413 if (validate_keys_sizes(ctx_p, keylen)) {
414 dev_dbg(dev, "Invalid key size %d.\n", keylen);
415 return -EINVAL;
416 }
417
418 if (ctx_p->cipher_mode == DRV_CIPHER_ESSIV) {
419
420 /* We only support 256 bit ESSIV-CBC-AES keys */
421 if (keylen != AES_KEYSIZE_256) {
422 unsigned int flags = crypto_tfm_get_flags(tfm) & CRYPTO_TFM_REQ_MASK;
423
424 if (likely(ctx_p->fallback_tfm)) {
425 ctx_p->fallback_on = true;
426 crypto_skcipher_clear_flags(ctx_p->fallback_tfm,
427 CRYPTO_TFM_REQ_MASK);
428 crypto_skcipher_clear_flags(ctx_p->fallback_tfm, flags);
429 return crypto_skcipher_setkey(ctx_p->fallback_tfm, key, keylen);
430 }
431
432 dev_dbg(dev, "Unsupported key size %d and no fallback.\n", keylen);
433 return -EINVAL;
434 }
435
436 /* Internal ESSIV key buffer is double sized */
437 max_key_buf_size <<= 1;
438 }
439
440 ctx_p->fallback_on = false;
441 ctx_p->key_type = CC_UNPROTECTED_KEY;
442
443 /*
444 * Verify DES weak keys
445 * Note that we're dropping the expanded key since the
446 * HW does the expansion on its own.
447 */
448 if (ctx_p->flow_mode == S_DIN_to_DES) {
449 if ((keylen == DES3_EDE_KEY_SIZE &&
450 verify_skcipher_des3_key(sktfm, key)) ||
451 verify_skcipher_des_key(sktfm, key)) {
452 dev_dbg(dev, "weak DES key");
453 return -EINVAL;
454 }
455 }
456
457 if (ctx_p->cipher_mode == DRV_CIPHER_XTS &&
458 xts_verify_key(sktfm, key, keylen)) {
459 dev_dbg(dev, "weak XTS key");
460 return -EINVAL;
461 }
462
463 /* STAT_PHASE_1: Copy key to ctx */
464 dma_sync_single_for_cpu(dev, ctx_p->user.key_dma_addr,
465 max_key_buf_size, DMA_TO_DEVICE);
466
467 memcpy(ctx_p->user.key, key, keylen);
468
469 if (ctx_p->cipher_mode == DRV_CIPHER_ESSIV) {
470 /* sha256 for key2 - use sw implementation */
471 int err;
472
473 err = crypto_shash_tfm_digest(ctx_p->shash_tfm,
474 ctx_p->user.key, keylen,
475 ctx_p->user.key + keylen);
476 if (err) {
477 dev_err(dev, "Failed to hash ESSIV key.\n");
478 return err;
479 }
480
481 keylen <<= 1;
482 }
483 dma_sync_single_for_device(dev, ctx_p->user.key_dma_addr,
484 max_key_buf_size, DMA_TO_DEVICE);
485 ctx_p->keylen = keylen;
486
487 dev_dbg(dev, "return safely");
488 return 0;
489}
490
491static int cc_out_setup_mode(struct cc_cipher_ctx *ctx_p)
492{
493 switch (ctx_p->flow_mode) {
494 case S_DIN_to_AES:
495 return S_AES_to_DOUT;
496 case S_DIN_to_DES:
497 return S_DES_to_DOUT;
498 case S_DIN_to_SM4:
499 return S_SM4_to_DOUT;
500 default:
501 return ctx_p->flow_mode;
502 }
503}
504
505static void cc_setup_readiv_desc(struct crypto_tfm *tfm,
506 struct cipher_req_ctx *req_ctx,
507 unsigned int ivsize, struct cc_hw_desc desc[],
508 unsigned int *seq_size)
509{
510 struct cc_cipher_ctx *ctx_p = crypto_tfm_ctx(tfm);
511 struct device *dev = drvdata_to_dev(ctx_p->drvdata);
512 int cipher_mode = ctx_p->cipher_mode;
513 int flow_mode = cc_out_setup_mode(ctx_p);
514 int direction = req_ctx->gen_ctx.op_type;
515 dma_addr_t iv_dma_addr = req_ctx->gen_ctx.iv_dma_addr;
516
517 if (ctx_p->key_type == CC_POLICY_PROTECTED_KEY)
518 return;
519
520 switch (cipher_mode) {
521 case DRV_CIPHER_ECB:
522 break;
523 case DRV_CIPHER_CBC:
524 case DRV_CIPHER_CBC_CTS:
525 case DRV_CIPHER_CTR:
526 case DRV_CIPHER_OFB:
527 /* Read next IV */
528 hw_desc_init(&desc[*seq_size]);
529 set_dout_dlli(&desc[*seq_size], iv_dma_addr, ivsize, NS_BIT, 1);
530 set_cipher_config0(&desc[*seq_size], direction);
531 set_flow_mode(&desc[*seq_size], flow_mode);
532 set_cipher_mode(&desc[*seq_size], cipher_mode);
533 if (cipher_mode == DRV_CIPHER_CTR ||
534 cipher_mode == DRV_CIPHER_OFB) {
535 set_setup_mode(&desc[*seq_size], SETUP_WRITE_STATE1);
536 } else {
537 set_setup_mode(&desc[*seq_size], SETUP_WRITE_STATE0);
538 }
539 set_queue_last_ind(ctx_p->drvdata, &desc[*seq_size]);
540 (*seq_size)++;
541 break;
542 case DRV_CIPHER_XTS:
543 case DRV_CIPHER_ESSIV:
544 /* IV */
545 hw_desc_init(&desc[*seq_size]);
546 set_setup_mode(&desc[*seq_size], SETUP_WRITE_STATE1);
547 set_cipher_mode(&desc[*seq_size], cipher_mode);
548 set_cipher_config0(&desc[*seq_size], direction);
549 set_flow_mode(&desc[*seq_size], flow_mode);
550 set_dout_dlli(&desc[*seq_size], iv_dma_addr, CC_AES_BLOCK_SIZE,
551 NS_BIT, 1);
552 set_queue_last_ind(ctx_p->drvdata, &desc[*seq_size]);
553 (*seq_size)++;
554 break;
555 default:
556 dev_err(dev, "Unsupported cipher mode (%d)\n", cipher_mode);
557 }
558}
559
560
561static void cc_setup_state_desc(struct crypto_tfm *tfm,
562 struct cipher_req_ctx *req_ctx,
563 unsigned int ivsize, unsigned int nbytes,
564 struct cc_hw_desc desc[],
565 unsigned int *seq_size)
566{
567 struct cc_cipher_ctx *ctx_p = crypto_tfm_ctx(tfm);
568 struct device *dev = drvdata_to_dev(ctx_p->drvdata);
569 int cipher_mode = ctx_p->cipher_mode;
570 int flow_mode = ctx_p->flow_mode;
571 int direction = req_ctx->gen_ctx.op_type;
572 dma_addr_t iv_dma_addr = req_ctx->gen_ctx.iv_dma_addr;
573
574 switch (cipher_mode) {
575 case DRV_CIPHER_ECB:
576 break;
577 case DRV_CIPHER_CBC:
578 case DRV_CIPHER_CBC_CTS:
579 case DRV_CIPHER_CTR:
580 case DRV_CIPHER_OFB:
581 /* Load IV */
582 hw_desc_init(&desc[*seq_size]);
583 set_din_type(&desc[*seq_size], DMA_DLLI, iv_dma_addr, ivsize,
584 NS_BIT);
585 set_cipher_config0(&desc[*seq_size], direction);
586 set_flow_mode(&desc[*seq_size], flow_mode);
587 set_cipher_mode(&desc[*seq_size], cipher_mode);
588 if (cipher_mode == DRV_CIPHER_CTR ||
589 cipher_mode == DRV_CIPHER_OFB) {
590 set_setup_mode(&desc[*seq_size], SETUP_LOAD_STATE1);
591 } else {
592 set_setup_mode(&desc[*seq_size], SETUP_LOAD_STATE0);
593 }
594 (*seq_size)++;
595 break;
596 case DRV_CIPHER_XTS:
597 case DRV_CIPHER_ESSIV:
598 break;
599 default:
600 dev_err(dev, "Unsupported cipher mode (%d)\n", cipher_mode);
601 }
602}
603
604
605static void cc_setup_xex_state_desc(struct crypto_tfm *tfm,
606 struct cipher_req_ctx *req_ctx,
607 unsigned int ivsize, unsigned int nbytes,
608 struct cc_hw_desc desc[],
609 unsigned int *seq_size)
610{
611 struct cc_cipher_ctx *ctx_p = crypto_tfm_ctx(tfm);
612 struct device *dev = drvdata_to_dev(ctx_p->drvdata);
613 int cipher_mode = ctx_p->cipher_mode;
614 int flow_mode = ctx_p->flow_mode;
615 int direction = req_ctx->gen_ctx.op_type;
616 dma_addr_t key_dma_addr = ctx_p->user.key_dma_addr;
617 unsigned int key_len = (ctx_p->keylen / 2);
618 dma_addr_t iv_dma_addr = req_ctx->gen_ctx.iv_dma_addr;
619 unsigned int key_offset = key_len;
620
621 switch (cipher_mode) {
622 case DRV_CIPHER_ECB:
623 break;
624 case DRV_CIPHER_CBC:
625 case DRV_CIPHER_CBC_CTS:
626 case DRV_CIPHER_CTR:
627 case DRV_CIPHER_OFB:
628 break;
629 case DRV_CIPHER_XTS:
630 case DRV_CIPHER_ESSIV:
631
632 if (cipher_mode == DRV_CIPHER_ESSIV)
633 key_len = SHA256_DIGEST_SIZE;
634
635 /* load XEX key */
636 hw_desc_init(&desc[*seq_size]);
637 set_cipher_mode(&desc[*seq_size], cipher_mode);
638 set_cipher_config0(&desc[*seq_size], direction);
639 if (cc_key_type(tfm) == CC_HW_PROTECTED_KEY) {
640 set_hw_crypto_key(&desc[*seq_size],
641 ctx_p->hw.key2_slot);
642 } else {
643 set_din_type(&desc[*seq_size], DMA_DLLI,
644 (key_dma_addr + key_offset),
645 key_len, NS_BIT);
646 }
647 set_xex_data_unit_size(&desc[*seq_size], nbytes);
648 set_flow_mode(&desc[*seq_size], S_DIN_to_AES2);
649 set_key_size_aes(&desc[*seq_size], key_len);
650 set_setup_mode(&desc[*seq_size], SETUP_LOAD_XEX_KEY);
651 (*seq_size)++;
652
653 /* Load IV */
654 hw_desc_init(&desc[*seq_size]);
655 set_setup_mode(&desc[*seq_size], SETUP_LOAD_STATE1);
656 set_cipher_mode(&desc[*seq_size], cipher_mode);
657 set_cipher_config0(&desc[*seq_size], direction);
658 set_key_size_aes(&desc[*seq_size], key_len);
659 set_flow_mode(&desc[*seq_size], flow_mode);
660 set_din_type(&desc[*seq_size], DMA_DLLI, iv_dma_addr,
661 CC_AES_BLOCK_SIZE, NS_BIT);
662 (*seq_size)++;
663 break;
664 default:
665 dev_err(dev, "Unsupported cipher mode (%d)\n", cipher_mode);
666 }
667}
668
669static int cc_out_flow_mode(struct cc_cipher_ctx *ctx_p)
670{
671 switch (ctx_p->flow_mode) {
672 case S_DIN_to_AES:
673 return DIN_AES_DOUT;
674 case S_DIN_to_DES:
675 return DIN_DES_DOUT;
676 case S_DIN_to_SM4:
677 return DIN_SM4_DOUT;
678 default:
679 return ctx_p->flow_mode;
680 }
681}
682
683static void cc_setup_key_desc(struct crypto_tfm *tfm,
684 struct cipher_req_ctx *req_ctx,
685 unsigned int nbytes, struct cc_hw_desc desc[],
686 unsigned int *seq_size)
687{
688 struct cc_cipher_ctx *ctx_p = crypto_tfm_ctx(tfm);
689 struct device *dev = drvdata_to_dev(ctx_p->drvdata);
690 int cipher_mode = ctx_p->cipher_mode;
691 int flow_mode = ctx_p->flow_mode;
692 int direction = req_ctx->gen_ctx.op_type;
693 dma_addr_t key_dma_addr = ctx_p->user.key_dma_addr;
694 unsigned int key_len = ctx_p->keylen;
695 unsigned int din_size;
696
697 switch (cipher_mode) {
698 case DRV_CIPHER_CBC:
699 case DRV_CIPHER_CBC_CTS:
700 case DRV_CIPHER_CTR:
701 case DRV_CIPHER_OFB:
702 case DRV_CIPHER_ECB:
703 /* Load key */
704 hw_desc_init(&desc[*seq_size]);
705 set_cipher_mode(&desc[*seq_size], cipher_mode);
706 set_cipher_config0(&desc[*seq_size], direction);
707
708 if (cc_key_type(tfm) == CC_POLICY_PROTECTED_KEY) {
709 /* We use the AES key size coding for all CPP algs */
710 set_key_size_aes(&desc[*seq_size], key_len);
711 set_cpp_crypto_key(&desc[*seq_size], ctx_p->cpp.slot);
712 flow_mode = cc_out_flow_mode(ctx_p);
713 } else {
714 if (flow_mode == S_DIN_to_AES) {
715 if (cc_key_type(tfm) == CC_HW_PROTECTED_KEY) {
716 set_hw_crypto_key(&desc[*seq_size],
717 ctx_p->hw.key1_slot);
718 } else {
719 /* CC_POLICY_UNPROTECTED_KEY
720 * Invalid keys are filtered out in
721 * sethkey()
722 */
723 din_size = (key_len == 24) ?
724 AES_MAX_KEY_SIZE : key_len;
725
726 set_din_type(&desc[*seq_size], DMA_DLLI,
727 key_dma_addr, din_size,
728 NS_BIT);
729 }
730 set_key_size_aes(&desc[*seq_size], key_len);
731 } else {
732 /*des*/
733 set_din_type(&desc[*seq_size], DMA_DLLI,
734 key_dma_addr, key_len, NS_BIT);
735 set_key_size_des(&desc[*seq_size], key_len);
736 }
737 set_setup_mode(&desc[*seq_size], SETUP_LOAD_KEY0);
738 }
739 set_flow_mode(&desc[*seq_size], flow_mode);
740 (*seq_size)++;
741 break;
742 case DRV_CIPHER_XTS:
743 case DRV_CIPHER_ESSIV:
744 /* Load AES key */
745 hw_desc_init(&desc[*seq_size]);
746 set_cipher_mode(&desc[*seq_size], cipher_mode);
747 set_cipher_config0(&desc[*seq_size], direction);
748 if (cc_key_type(tfm) == CC_HW_PROTECTED_KEY) {
749 set_hw_crypto_key(&desc[*seq_size],
750 ctx_p->hw.key1_slot);
751 } else {
752 set_din_type(&desc[*seq_size], DMA_DLLI, key_dma_addr,
753 (key_len / 2), NS_BIT);
754 }
755 set_key_size_aes(&desc[*seq_size], (key_len / 2));
756 set_flow_mode(&desc[*seq_size], flow_mode);
757 set_setup_mode(&desc[*seq_size], SETUP_LOAD_KEY0);
758 (*seq_size)++;
759 break;
760 default:
761 dev_err(dev, "Unsupported cipher mode (%d)\n", cipher_mode);
762 }
763}
764
765static void cc_setup_mlli_desc(struct crypto_tfm *tfm,
766 struct cipher_req_ctx *req_ctx,
767 struct scatterlist *dst, struct scatterlist *src,
768 unsigned int nbytes, void *areq,
769 struct cc_hw_desc desc[], unsigned int *seq_size)
770{
771 struct cc_cipher_ctx *ctx_p = crypto_tfm_ctx(tfm);
772 struct device *dev = drvdata_to_dev(ctx_p->drvdata);
773
774 if (req_ctx->dma_buf_type == CC_DMA_BUF_MLLI) {
775 /* bypass */
776 dev_dbg(dev, " bypass params addr %pad length 0x%X addr 0x%08X\n",
777 &req_ctx->mlli_params.mlli_dma_addr,
778 req_ctx->mlli_params.mlli_len,
779 ctx_p->drvdata->mlli_sram_addr);
780 hw_desc_init(&desc[*seq_size]);
781 set_din_type(&desc[*seq_size], DMA_DLLI,
782 req_ctx->mlli_params.mlli_dma_addr,
783 req_ctx->mlli_params.mlli_len, NS_BIT);
784 set_dout_sram(&desc[*seq_size],
785 ctx_p->drvdata->mlli_sram_addr,
786 req_ctx->mlli_params.mlli_len);
787 set_flow_mode(&desc[*seq_size], BYPASS);
788 (*seq_size)++;
789 }
790}
791
792static void cc_setup_flow_desc(struct crypto_tfm *tfm,
793 struct cipher_req_ctx *req_ctx,
794 struct scatterlist *dst, struct scatterlist *src,
795 unsigned int nbytes, struct cc_hw_desc desc[],
796 unsigned int *seq_size)
797{
798 struct cc_cipher_ctx *ctx_p = crypto_tfm_ctx(tfm);
799 struct device *dev = drvdata_to_dev(ctx_p->drvdata);
800 unsigned int flow_mode = cc_out_flow_mode(ctx_p);
801 bool last_desc = (ctx_p->key_type == CC_POLICY_PROTECTED_KEY ||
802 ctx_p->cipher_mode == DRV_CIPHER_ECB);
803
804 /* Process */
805 if (req_ctx->dma_buf_type == CC_DMA_BUF_DLLI) {
806 dev_dbg(dev, " data params addr %pad length 0x%X\n",
807 &sg_dma_address(src), nbytes);
808 dev_dbg(dev, " data params addr %pad length 0x%X\n",
809 &sg_dma_address(dst), nbytes);
810 hw_desc_init(&desc[*seq_size]);
811 set_din_type(&desc[*seq_size], DMA_DLLI, sg_dma_address(src),
812 nbytes, NS_BIT);
813 set_dout_dlli(&desc[*seq_size], sg_dma_address(dst),
814 nbytes, NS_BIT, (!last_desc ? 0 : 1));
815 if (last_desc)
816 set_queue_last_ind(ctx_p->drvdata, &desc[*seq_size]);
817
818 set_flow_mode(&desc[*seq_size], flow_mode);
819 (*seq_size)++;
820 } else {
821 hw_desc_init(&desc[*seq_size]);
822 set_din_type(&desc[*seq_size], DMA_MLLI,
823 ctx_p->drvdata->mlli_sram_addr,
824 req_ctx->in_mlli_nents, NS_BIT);
825 if (req_ctx->out_nents == 0) {
826 dev_dbg(dev, " din/dout params addr 0x%08X addr 0x%08X\n",
827 ctx_p->drvdata->mlli_sram_addr,
828 ctx_p->drvdata->mlli_sram_addr);
829 set_dout_mlli(&desc[*seq_size],
830 ctx_p->drvdata->mlli_sram_addr,
831 req_ctx->in_mlli_nents, NS_BIT,
832 (!last_desc ? 0 : 1));
833 } else {
834 dev_dbg(dev, " din/dout params addr 0x%08X addr 0x%08X\n",
835 ctx_p->drvdata->mlli_sram_addr,
836 ctx_p->drvdata->mlli_sram_addr +
837 (u32)LLI_ENTRY_BYTE_SIZE * req_ctx->in_nents);
838 set_dout_mlli(&desc[*seq_size],
839 (ctx_p->drvdata->mlli_sram_addr +
840 (LLI_ENTRY_BYTE_SIZE *
841 req_ctx->in_mlli_nents)),
842 req_ctx->out_mlli_nents, NS_BIT,
843 (!last_desc ? 0 : 1));
844 }
845 if (last_desc)
846 set_queue_last_ind(ctx_p->drvdata, &desc[*seq_size]);
847
848 set_flow_mode(&desc[*seq_size], flow_mode);
849 (*seq_size)++;
850 }
851}
852
853static void cc_cipher_complete(struct device *dev, void *cc_req, int err)
854{
855 struct skcipher_request *req = (struct skcipher_request *)cc_req;
856 struct scatterlist *dst = req->dst;
857 struct scatterlist *src = req->src;
858 struct cipher_req_ctx *req_ctx = skcipher_request_ctx(req);
859 struct crypto_skcipher *sk_tfm = crypto_skcipher_reqtfm(req);
860 unsigned int ivsize = crypto_skcipher_ivsize(sk_tfm);
861
862 if (err != -EINPROGRESS) {
863 /* Not a BACKLOG notification */
864 cc_unmap_cipher_request(dev, req_ctx, ivsize, src, dst);
865 memcpy(req->iv, req_ctx->iv, ivsize);
866 kfree_sensitive(req_ctx->iv);
867 }
868
869 skcipher_request_complete(req, err);
870}
871
872static int cc_cipher_process(struct skcipher_request *req,
873 enum drv_crypto_direction direction)
874{
875 struct crypto_skcipher *sk_tfm = crypto_skcipher_reqtfm(req);
876 struct crypto_tfm *tfm = crypto_skcipher_tfm(sk_tfm);
877 struct cipher_req_ctx *req_ctx = skcipher_request_ctx(req);
878 unsigned int ivsize = crypto_skcipher_ivsize(sk_tfm);
879 struct scatterlist *dst = req->dst;
880 struct scatterlist *src = req->src;
881 unsigned int nbytes = req->cryptlen;
882 void *iv = req->iv;
883 struct cc_cipher_ctx *ctx_p = crypto_tfm_ctx(tfm);
884 struct device *dev = drvdata_to_dev(ctx_p->drvdata);
885 struct cc_hw_desc desc[MAX_SKCIPHER_SEQ_LEN];
886 struct cc_crypto_req cc_req = {};
887 int rc;
888 unsigned int seq_len = 0;
889 gfp_t flags = cc_gfp_flags(&req->base);
890
891 dev_dbg(dev, "%s req=%p iv=%p nbytes=%d\n",
892 ((direction == DRV_CRYPTO_DIRECTION_ENCRYPT) ?
893 "Encrypt" : "Decrypt"), req, iv, nbytes);
894
895 /* STAT_PHASE_0: Init and sanity checks */
896
897 if (validate_data_size(ctx_p, nbytes)) {
898 dev_dbg(dev, "Unsupported data size %d.\n", nbytes);
899 rc = -EINVAL;
900 goto exit_process;
901 }
902 if (nbytes == 0) {
903 /* No data to process is valid */
904 rc = 0;
905 goto exit_process;
906 }
907
908 if (ctx_p->fallback_on) {
909 struct skcipher_request *subreq = skcipher_request_ctx(req);
910
911 *subreq = *req;
912 skcipher_request_set_tfm(subreq, ctx_p->fallback_tfm);
913 if (direction == DRV_CRYPTO_DIRECTION_ENCRYPT)
914 return crypto_skcipher_encrypt(subreq);
915 else
916 return crypto_skcipher_decrypt(subreq);
917 }
918
919 /* The IV we are handed may be allocated from the stack so
920 * we must copy it to a DMAable buffer before use.
921 */
922 req_ctx->iv = kmemdup(iv, ivsize, flags);
923 if (!req_ctx->iv) {
924 rc = -ENOMEM;
925 goto exit_process;
926 }
927
928 /* Setup request structure */
929 cc_req.user_cb = cc_cipher_complete;
930 cc_req.user_arg = req;
931
932 /* Setup CPP operation details */
933 if (ctx_p->key_type == CC_POLICY_PROTECTED_KEY) {
934 cc_req.cpp.is_cpp = true;
935 cc_req.cpp.alg = ctx_p->cpp.alg;
936 cc_req.cpp.slot = ctx_p->cpp.slot;
937 }
938
939 /* Setup request context */
940 req_ctx->gen_ctx.op_type = direction;
941
942 /* STAT_PHASE_1: Map buffers */
943
944 rc = cc_map_cipher_request(ctx_p->drvdata, req_ctx, ivsize, nbytes,
945 req_ctx->iv, src, dst, flags);
946 if (rc) {
947 dev_err(dev, "map_request() failed\n");
948 goto exit_process;
949 }
950
951 /* STAT_PHASE_2: Create sequence */
952
953 /* Setup state (IV) */
954 cc_setup_state_desc(tfm, req_ctx, ivsize, nbytes, desc, &seq_len);
955 /* Setup MLLI line, if needed */
956 cc_setup_mlli_desc(tfm, req_ctx, dst, src, nbytes, req, desc, &seq_len);
957 /* Setup key */
958 cc_setup_key_desc(tfm, req_ctx, nbytes, desc, &seq_len);
959 /* Setup state (IV and XEX key) */
960 cc_setup_xex_state_desc(tfm, req_ctx, ivsize, nbytes, desc, &seq_len);
961 /* Data processing */
962 cc_setup_flow_desc(tfm, req_ctx, dst, src, nbytes, desc, &seq_len);
963 /* Read next IV */
964 cc_setup_readiv_desc(tfm, req_ctx, ivsize, desc, &seq_len);
965
966 /* STAT_PHASE_3: Lock HW and push sequence */
967
968 rc = cc_send_request(ctx_p->drvdata, &cc_req, desc, seq_len,
969 &req->base);
970 if (rc != -EINPROGRESS && rc != -EBUSY) {
971 /* Failed to send the request or request completed
972 * synchronously
973 */
974 cc_unmap_cipher_request(dev, req_ctx, ivsize, src, dst);
975 }
976
977exit_process:
978 if (rc != -EINPROGRESS && rc != -EBUSY) {
979 kfree_sensitive(req_ctx->iv);
980 }
981
982 return rc;
983}
984
985static int cc_cipher_encrypt(struct skcipher_request *req)
986{
987 struct cipher_req_ctx *req_ctx = skcipher_request_ctx(req);
988
989 memset(req_ctx, 0, sizeof(*req_ctx));
990
991 return cc_cipher_process(req, DRV_CRYPTO_DIRECTION_ENCRYPT);
992}
993
994static int cc_cipher_decrypt(struct skcipher_request *req)
995{
996 struct cipher_req_ctx *req_ctx = skcipher_request_ctx(req);
997
998 memset(req_ctx, 0, sizeof(*req_ctx));
999
1000 return cc_cipher_process(req, DRV_CRYPTO_DIRECTION_DECRYPT);
1001}
1002
1003/* Block cipher alg */
1004static const struct cc_alg_template skcipher_algs[] = {
1005 {
1006 .name = "xts(paes)",
1007 .driver_name = "xts-paes-ccree",
1008 .blocksize = 1,
1009 .template_skcipher = {
1010 .setkey = cc_cipher_sethkey,
1011 .encrypt = cc_cipher_encrypt,
1012 .decrypt = cc_cipher_decrypt,
1013 .min_keysize = CC_HW_KEY_SIZE,
1014 .max_keysize = CC_HW_KEY_SIZE,
1015 .ivsize = AES_BLOCK_SIZE,
1016 },
1017 .cipher_mode = DRV_CIPHER_XTS,
1018 .flow_mode = S_DIN_to_AES,
1019 .min_hw_rev = CC_HW_REV_630,
1020 .std_body = CC_STD_NIST,
1021 .sec_func = true,
1022 },
1023 {
1024 .name = "essiv(cbc(paes),sha256)",
1025 .driver_name = "essiv-paes-ccree",
1026 .blocksize = AES_BLOCK_SIZE,
1027 .template_skcipher = {
1028 .setkey = cc_cipher_sethkey,
1029 .encrypt = cc_cipher_encrypt,
1030 .decrypt = cc_cipher_decrypt,
1031 .min_keysize = CC_HW_KEY_SIZE,
1032 .max_keysize = CC_HW_KEY_SIZE,
1033 .ivsize = AES_BLOCK_SIZE,
1034 },
1035 .cipher_mode = DRV_CIPHER_ESSIV,
1036 .flow_mode = S_DIN_to_AES,
1037 .min_hw_rev = CC_HW_REV_712,
1038 .std_body = CC_STD_NIST,
1039 .sec_func = true,
1040 },
1041 {
1042 .name = "ecb(paes)",
1043 .driver_name = "ecb-paes-ccree",
1044 .blocksize = AES_BLOCK_SIZE,
1045 .template_skcipher = {
1046 .setkey = cc_cipher_sethkey,
1047 .encrypt = cc_cipher_encrypt,
1048 .decrypt = cc_cipher_decrypt,
1049 .min_keysize = CC_HW_KEY_SIZE,
1050 .max_keysize = CC_HW_KEY_SIZE,
1051 .ivsize = 0,
1052 },
1053 .cipher_mode = DRV_CIPHER_ECB,
1054 .flow_mode = S_DIN_to_AES,
1055 .min_hw_rev = CC_HW_REV_712,
1056 .std_body = CC_STD_NIST,
1057 .sec_func = true,
1058 },
1059 {
1060 .name = "cbc(paes)",
1061 .driver_name = "cbc-paes-ccree",
1062 .blocksize = AES_BLOCK_SIZE,
1063 .template_skcipher = {
1064 .setkey = cc_cipher_sethkey,
1065 .encrypt = cc_cipher_encrypt,
1066 .decrypt = cc_cipher_decrypt,
1067 .min_keysize = CC_HW_KEY_SIZE,
1068 .max_keysize = CC_HW_KEY_SIZE,
1069 .ivsize = AES_BLOCK_SIZE,
1070 },
1071 .cipher_mode = DRV_CIPHER_CBC,
1072 .flow_mode = S_DIN_to_AES,
1073 .min_hw_rev = CC_HW_REV_712,
1074 .std_body = CC_STD_NIST,
1075 .sec_func = true,
1076 },
1077 {
1078 .name = "cts(cbc(paes))",
1079 .driver_name = "cts-cbc-paes-ccree",
1080 .blocksize = AES_BLOCK_SIZE,
1081 .template_skcipher = {
1082 .setkey = cc_cipher_sethkey,
1083 .encrypt = cc_cipher_encrypt,
1084 .decrypt = cc_cipher_decrypt,
1085 .min_keysize = CC_HW_KEY_SIZE,
1086 .max_keysize = CC_HW_KEY_SIZE,
1087 .ivsize = AES_BLOCK_SIZE,
1088 },
1089 .cipher_mode = DRV_CIPHER_CBC_CTS,
1090 .flow_mode = S_DIN_to_AES,
1091 .min_hw_rev = CC_HW_REV_712,
1092 .std_body = CC_STD_NIST,
1093 .sec_func = true,
1094 },
1095 {
1096 .name = "ctr(paes)",
1097 .driver_name = "ctr-paes-ccree",
1098 .blocksize = 1,
1099 .template_skcipher = {
1100 .setkey = cc_cipher_sethkey,
1101 .encrypt = cc_cipher_encrypt,
1102 .decrypt = cc_cipher_decrypt,
1103 .min_keysize = CC_HW_KEY_SIZE,
1104 .max_keysize = CC_HW_KEY_SIZE,
1105 .ivsize = AES_BLOCK_SIZE,
1106 },
1107 .cipher_mode = DRV_CIPHER_CTR,
1108 .flow_mode = S_DIN_to_AES,
1109 .min_hw_rev = CC_HW_REV_712,
1110 .std_body = CC_STD_NIST,
1111 .sec_func = true,
1112 },
1113 {
1114 /* See https://www.mail-archive.com/[email protected]/msg40576.html
1115 * for the reason why this differs from the generic
1116 * implementation.
1117 */
1118 .name = "xts(aes)",
1119 .driver_name = "xts-aes-ccree",
1120 .blocksize = 1,
1121 .template_skcipher = {
1122 .setkey = cc_cipher_setkey,
1123 .encrypt = cc_cipher_encrypt,
1124 .decrypt = cc_cipher_decrypt,
1125 .min_keysize = AES_MIN_KEY_SIZE * 2,
1126 .max_keysize = AES_MAX_KEY_SIZE * 2,
1127 .ivsize = AES_BLOCK_SIZE,
1128 },
1129 .cipher_mode = DRV_CIPHER_XTS,
1130 .flow_mode = S_DIN_to_AES,
1131 .min_hw_rev = CC_HW_REV_630,
1132 .std_body = CC_STD_NIST,
1133 },
1134 {
1135 .name = "essiv(cbc(aes),sha256)",
1136 .driver_name = "essiv-aes-ccree",
1137 .blocksize = AES_BLOCK_SIZE,
1138 .template_skcipher = {
1139 .setkey = cc_cipher_setkey,
1140 .encrypt = cc_cipher_encrypt,
1141 .decrypt = cc_cipher_decrypt,
1142 .min_keysize = AES_MIN_KEY_SIZE,
1143 .max_keysize = AES_MAX_KEY_SIZE,
1144 .ivsize = AES_BLOCK_SIZE,
1145 },
1146 .cipher_mode = DRV_CIPHER_ESSIV,
1147 .flow_mode = S_DIN_to_AES,
1148 .min_hw_rev = CC_HW_REV_712,
1149 .std_body = CC_STD_NIST,
1150 },
1151 {
1152 .name = "ecb(aes)",
1153 .driver_name = "ecb-aes-ccree",
1154 .blocksize = AES_BLOCK_SIZE,
1155 .template_skcipher = {
1156 .setkey = cc_cipher_setkey,
1157 .encrypt = cc_cipher_encrypt,
1158 .decrypt = cc_cipher_decrypt,
1159 .min_keysize = AES_MIN_KEY_SIZE,
1160 .max_keysize = AES_MAX_KEY_SIZE,
1161 .ivsize = 0,
1162 },
1163 .cipher_mode = DRV_CIPHER_ECB,
1164 .flow_mode = S_DIN_to_AES,
1165 .min_hw_rev = CC_HW_REV_630,
1166 .std_body = CC_STD_NIST,
1167 },
1168 {
1169 .name = "cbc(aes)",
1170 .driver_name = "cbc-aes-ccree",
1171 .blocksize = AES_BLOCK_SIZE,
1172 .template_skcipher = {
1173 .setkey = cc_cipher_setkey,
1174 .encrypt = cc_cipher_encrypt,
1175 .decrypt = cc_cipher_decrypt,
1176 .min_keysize = AES_MIN_KEY_SIZE,
1177 .max_keysize = AES_MAX_KEY_SIZE,
1178 .ivsize = AES_BLOCK_SIZE,
1179 },
1180 .cipher_mode = DRV_CIPHER_CBC,
1181 .flow_mode = S_DIN_to_AES,
1182 .min_hw_rev = CC_HW_REV_630,
1183 .std_body = CC_STD_NIST,
1184 },
1185 {
1186 .name = "cts(cbc(aes))",
1187 .driver_name = "cts-cbc-aes-ccree",
1188 .blocksize = AES_BLOCK_SIZE,
1189 .template_skcipher = {
1190 .setkey = cc_cipher_setkey,
1191 .encrypt = cc_cipher_encrypt,
1192 .decrypt = cc_cipher_decrypt,
1193 .min_keysize = AES_MIN_KEY_SIZE,
1194 .max_keysize = AES_MAX_KEY_SIZE,
1195 .ivsize = AES_BLOCK_SIZE,
1196 },
1197 .cipher_mode = DRV_CIPHER_CBC_CTS,
1198 .flow_mode = S_DIN_to_AES,
1199 .min_hw_rev = CC_HW_REV_630,
1200 .std_body = CC_STD_NIST,
1201 },
1202 {
1203 .name = "ctr(aes)",
1204 .driver_name = "ctr-aes-ccree",
1205 .blocksize = 1,
1206 .template_skcipher = {
1207 .setkey = cc_cipher_setkey,
1208 .encrypt = cc_cipher_encrypt,
1209 .decrypt = cc_cipher_decrypt,
1210 .min_keysize = AES_MIN_KEY_SIZE,
1211 .max_keysize = AES_MAX_KEY_SIZE,
1212 .ivsize = AES_BLOCK_SIZE,
1213 },
1214 .cipher_mode = DRV_CIPHER_CTR,
1215 .flow_mode = S_DIN_to_AES,
1216 .min_hw_rev = CC_HW_REV_630,
1217 .std_body = CC_STD_NIST,
1218 },
1219 {
1220 .name = "cbc(des3_ede)",
1221 .driver_name = "cbc-3des-ccree",
1222 .blocksize = DES3_EDE_BLOCK_SIZE,
1223 .template_skcipher = {
1224 .setkey = cc_cipher_setkey,
1225 .encrypt = cc_cipher_encrypt,
1226 .decrypt = cc_cipher_decrypt,
1227 .min_keysize = DES3_EDE_KEY_SIZE,
1228 .max_keysize = DES3_EDE_KEY_SIZE,
1229 .ivsize = DES3_EDE_BLOCK_SIZE,
1230 },
1231 .cipher_mode = DRV_CIPHER_CBC,
1232 .flow_mode = S_DIN_to_DES,
1233 .min_hw_rev = CC_HW_REV_630,
1234 .std_body = CC_STD_NIST,
1235 },
1236 {
1237 .name = "ecb(des3_ede)",
1238 .driver_name = "ecb-3des-ccree",
1239 .blocksize = DES3_EDE_BLOCK_SIZE,
1240 .template_skcipher = {
1241 .setkey = cc_cipher_setkey,
1242 .encrypt = cc_cipher_encrypt,
1243 .decrypt = cc_cipher_decrypt,
1244 .min_keysize = DES3_EDE_KEY_SIZE,
1245 .max_keysize = DES3_EDE_KEY_SIZE,
1246 .ivsize = 0,
1247 },
1248 .cipher_mode = DRV_CIPHER_ECB,
1249 .flow_mode = S_DIN_to_DES,
1250 .min_hw_rev = CC_HW_REV_630,
1251 .std_body = CC_STD_NIST,
1252 },
1253 {
1254 .name = "cbc(des)",
1255 .driver_name = "cbc-des-ccree",
1256 .blocksize = DES_BLOCK_SIZE,
1257 .template_skcipher = {
1258 .setkey = cc_cipher_setkey,
1259 .encrypt = cc_cipher_encrypt,
1260 .decrypt = cc_cipher_decrypt,
1261 .min_keysize = DES_KEY_SIZE,
1262 .max_keysize = DES_KEY_SIZE,
1263 .ivsize = DES_BLOCK_SIZE,
1264 },
1265 .cipher_mode = DRV_CIPHER_CBC,
1266 .flow_mode = S_DIN_to_DES,
1267 .min_hw_rev = CC_HW_REV_630,
1268 .std_body = CC_STD_NIST,
1269 },
1270 {
1271 .name = "ecb(des)",
1272 .driver_name = "ecb-des-ccree",
1273 .blocksize = DES_BLOCK_SIZE,
1274 .template_skcipher = {
1275 .setkey = cc_cipher_setkey,
1276 .encrypt = cc_cipher_encrypt,
1277 .decrypt = cc_cipher_decrypt,
1278 .min_keysize = DES_KEY_SIZE,
1279 .max_keysize = DES_KEY_SIZE,
1280 .ivsize = 0,
1281 },
1282 .cipher_mode = DRV_CIPHER_ECB,
1283 .flow_mode = S_DIN_to_DES,
1284 .min_hw_rev = CC_HW_REV_630,
1285 .std_body = CC_STD_NIST,
1286 },
1287 {
1288 .name = "cbc(sm4)",
1289 .driver_name = "cbc-sm4-ccree",
1290 .blocksize = SM4_BLOCK_SIZE,
1291 .template_skcipher = {
1292 .setkey = cc_cipher_setkey,
1293 .encrypt = cc_cipher_encrypt,
1294 .decrypt = cc_cipher_decrypt,
1295 .min_keysize = SM4_KEY_SIZE,
1296 .max_keysize = SM4_KEY_SIZE,
1297 .ivsize = SM4_BLOCK_SIZE,
1298 },
1299 .cipher_mode = DRV_CIPHER_CBC,
1300 .flow_mode = S_DIN_to_SM4,
1301 .min_hw_rev = CC_HW_REV_713,
1302 .std_body = CC_STD_OSCCA,
1303 },
1304 {
1305 .name = "ecb(sm4)",
1306 .driver_name = "ecb-sm4-ccree",
1307 .blocksize = SM4_BLOCK_SIZE,
1308 .template_skcipher = {
1309 .setkey = cc_cipher_setkey,
1310 .encrypt = cc_cipher_encrypt,
1311 .decrypt = cc_cipher_decrypt,
1312 .min_keysize = SM4_KEY_SIZE,
1313 .max_keysize = SM4_KEY_SIZE,
1314 .ivsize = 0,
1315 },
1316 .cipher_mode = DRV_CIPHER_ECB,
1317 .flow_mode = S_DIN_to_SM4,
1318 .min_hw_rev = CC_HW_REV_713,
1319 .std_body = CC_STD_OSCCA,
1320 },
1321 {
1322 .name = "ctr(sm4)",
1323 .driver_name = "ctr-sm4-ccree",
1324 .blocksize = 1,
1325 .template_skcipher = {
1326 .setkey = cc_cipher_setkey,
1327 .encrypt = cc_cipher_encrypt,
1328 .decrypt = cc_cipher_decrypt,
1329 .min_keysize = SM4_KEY_SIZE,
1330 .max_keysize = SM4_KEY_SIZE,
1331 .ivsize = SM4_BLOCK_SIZE,
1332 },
1333 .cipher_mode = DRV_CIPHER_CTR,
1334 .flow_mode = S_DIN_to_SM4,
1335 .min_hw_rev = CC_HW_REV_713,
1336 .std_body = CC_STD_OSCCA,
1337 },
1338 {
1339 .name = "cbc(psm4)",
1340 .driver_name = "cbc-psm4-ccree",
1341 .blocksize = SM4_BLOCK_SIZE,
1342 .template_skcipher = {
1343 .setkey = cc_cipher_sethkey,
1344 .encrypt = cc_cipher_encrypt,
1345 .decrypt = cc_cipher_decrypt,
1346 .min_keysize = CC_HW_KEY_SIZE,
1347 .max_keysize = CC_HW_KEY_SIZE,
1348 .ivsize = SM4_BLOCK_SIZE,
1349 },
1350 .cipher_mode = DRV_CIPHER_CBC,
1351 .flow_mode = S_DIN_to_SM4,
1352 .min_hw_rev = CC_HW_REV_713,
1353 .std_body = CC_STD_OSCCA,
1354 .sec_func = true,
1355 },
1356 {
1357 .name = "ctr(psm4)",
1358 .driver_name = "ctr-psm4-ccree",
1359 .blocksize = SM4_BLOCK_SIZE,
1360 .template_skcipher = {
1361 .setkey = cc_cipher_sethkey,
1362 .encrypt = cc_cipher_encrypt,
1363 .decrypt = cc_cipher_decrypt,
1364 .min_keysize = CC_HW_KEY_SIZE,
1365 .max_keysize = CC_HW_KEY_SIZE,
1366 .ivsize = SM4_BLOCK_SIZE,
1367 },
1368 .cipher_mode = DRV_CIPHER_CTR,
1369 .flow_mode = S_DIN_to_SM4,
1370 .min_hw_rev = CC_HW_REV_713,
1371 .std_body = CC_STD_OSCCA,
1372 .sec_func = true,
1373 },
1374};
1375
1376static struct cc_crypto_alg *cc_create_alg(const struct cc_alg_template *tmpl,
1377 struct device *dev)
1378{
1379 struct cc_crypto_alg *t_alg;
1380 struct skcipher_alg *alg;
1381
1382 t_alg = devm_kzalloc(dev, sizeof(*t_alg), GFP_KERNEL);
1383 if (!t_alg)
1384 return ERR_PTR(-ENOMEM);
1385
1386 alg = &t_alg->skcipher_alg;
1387
1388 memcpy(alg, &tmpl->template_skcipher, sizeof(*alg));
1389
1390 if (strscpy(alg->base.cra_name, tmpl->name) < 0)
1391 return ERR_PTR(-EINVAL);
1392 if (strscpy(alg->base.cra_driver_name, tmpl->driver_name) < 0)
1393 return ERR_PTR(-EINVAL);
1394
1395 alg->base.cra_module = THIS_MODULE;
1396 alg->base.cra_priority = CC_CRA_PRIO;
1397 alg->base.cra_blocksize = tmpl->blocksize;
1398 alg->base.cra_alignmask = 0;
1399 alg->base.cra_ctxsize = sizeof(struct cc_cipher_ctx);
1400
1401 alg->base.cra_init = cc_cipher_init;
1402 alg->base.cra_exit = cc_cipher_exit;
1403 alg->base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY;
1404
1405 t_alg->cipher_mode = tmpl->cipher_mode;
1406 t_alg->flow_mode = tmpl->flow_mode;
1407
1408 return t_alg;
1409}
1410
1411int cc_cipher_free(struct cc_drvdata *drvdata)
1412{
1413 struct cc_crypto_alg *t_alg, *n;
1414
1415 /* Remove registered algs */
1416 list_for_each_entry_safe(t_alg, n, &drvdata->alg_list, entry) {
1417 crypto_unregister_skcipher(&t_alg->skcipher_alg);
1418 list_del(&t_alg->entry);
1419 }
1420 return 0;
1421}
1422
1423int cc_cipher_alloc(struct cc_drvdata *drvdata)
1424{
1425 struct cc_crypto_alg *t_alg;
1426 struct device *dev = drvdata_to_dev(drvdata);
1427 int rc = -ENOMEM;
1428 int alg;
1429
1430 INIT_LIST_HEAD(&drvdata->alg_list);
1431
1432 /* Linux crypto */
1433 dev_dbg(dev, "Number of algorithms = %zu\n",
1434 ARRAY_SIZE(skcipher_algs));
1435 for (alg = 0; alg < ARRAY_SIZE(skcipher_algs); alg++) {
1436 if ((skcipher_algs[alg].min_hw_rev > drvdata->hw_rev) ||
1437 !(drvdata->std_bodies & skcipher_algs[alg].std_body) ||
1438 (drvdata->sec_disabled && skcipher_algs[alg].sec_func))
1439 continue;
1440
1441 dev_dbg(dev, "creating %s\n", skcipher_algs[alg].driver_name);
1442 t_alg = cc_create_alg(&skcipher_algs[alg], dev);
1443 if (IS_ERR(t_alg)) {
1444 rc = PTR_ERR(t_alg);
1445 dev_err(dev, "%s alg allocation failed\n",
1446 skcipher_algs[alg].driver_name);
1447 goto fail0;
1448 }
1449 t_alg->drvdata = drvdata;
1450
1451 dev_dbg(dev, "registering %s\n",
1452 skcipher_algs[alg].driver_name);
1453 rc = crypto_register_skcipher(&t_alg->skcipher_alg);
1454 dev_dbg(dev, "%s alg registration rc = %x\n",
1455 t_alg->skcipher_alg.base.cra_driver_name, rc);
1456 if (rc) {
1457 dev_err(dev, "%s alg registration failed\n",
1458 t_alg->skcipher_alg.base.cra_driver_name);
1459 goto fail0;
1460 }
1461
1462 list_add_tail(&t_alg->entry, &drvdata->alg_list);
1463 dev_dbg(dev, "Registered %s\n",
1464 t_alg->skcipher_alg.base.cra_driver_name);
1465 }
1466 return 0;
1467
1468fail0:
1469 cc_cipher_free(drvdata);
1470 return rc;
1471}