projects/linux·blob master · refs

2474 lines · 66.2 KiB

1// SPDX-License-Identifier: GPL-2.0
2/*
3 * K3 SA2UL crypto accelerator driver
4 *
5 * Copyright (C) 2018-2020 Texas Instruments Incorporated - http://www.ti.com
6 *
7 * Authors: Keerthy
8 * Vitaly Andrianov
9 * Tero Kristo
10 */
11#include <linux/bitfield.h>
12#include <linux/clk.h>
13#include <linux/dma-mapping.h>
14#include <linux/dmaengine.h>
15#include <linux/dmapool.h>
16#include <linux/kernel.h>
17#include <linux/module.h>
18#include <linux/of.h>
19#include <linux/of_platform.h>
20#include <linux/platform_device.h>
21#include <linux/pm_runtime.h>
22
23#include <crypto/aes.h>
24#include <crypto/authenc.h>
25#include <crypto/utils.h>
26#include <crypto/des.h>
27#include <crypto/internal/aead.h>
28#include <crypto/internal/hash.h>
29#include <crypto/internal/skcipher.h>
30#include <crypto/scatterwalk.h>
31#include <crypto/sha1.h>
32#include <crypto/sha2.h>
33
34#include "sa2ul.h"
35
36/* Byte offset for key in encryption security context */
37#define SC_ENC_KEY_OFFSET (1 + 27 + 4)
38/* Byte offset for Aux-1 in encryption security context */
39#define SC_ENC_AUX1_OFFSET (1 + 27 + 4 + 32)
40
41#define SA_CMDL_UPD_ENC 0x0001
42#define SA_CMDL_UPD_AUTH 0x0002
43#define SA_CMDL_UPD_ENC_IV 0x0004
44#define SA_CMDL_UPD_AUTH_IV 0x0008
45#define SA_CMDL_UPD_AUX_KEY 0x0010
46
47#define SA_AUTH_SUBKEY_LEN 16
48#define SA_CMDL_PAYLOAD_LENGTH_MASK 0xFFFF
49#define SA_CMDL_SOP_BYPASS_LEN_MASK 0xFF000000
50
51#define MODE_CONTROL_BYTES 27
52#define SA_HASH_PROCESSING 0
53#define SA_CRYPTO_PROCESSING 0
54#define SA_UPLOAD_HASH_TO_TLR BIT(6)
55
56#define SA_SW0_FLAGS_MASK 0xF0000
57#define SA_SW0_CMDL_INFO_MASK 0x1F00000
58#define SA_SW0_CMDL_PRESENT BIT(4)
59#define SA_SW0_ENG_ID_MASK 0x3E000000
60#define SA_SW0_DEST_INFO_PRESENT BIT(30)
61#define SA_SW2_EGRESS_LENGTH 0xFF000000
62#define SA_BASIC_HASH 0x10
63
64#define SHA256_DIGEST_WORDS 8
65/* Make 32-bit word from 4 bytes */
66#define SA_MK_U32(b0, b1, b2, b3) (((b0) << 24) | ((b1) << 16) | \
67 ((b2) << 8) | (b3))
68
69/* size of SCCTL structure in bytes */
70#define SA_SCCTL_SZ 16
71
72/* Max Authentication tag size */
73#define SA_MAX_AUTH_TAG_SZ 64
74
75enum sa_algo_id {
76 SA_ALG_CBC_AES = 0,
77 SA_ALG_EBC_AES,
78 SA_ALG_CBC_DES3,
79 SA_ALG_ECB_DES3,
80 SA_ALG_SHA1,
81 SA_ALG_SHA256,
82 SA_ALG_SHA512,
83 SA_ALG_AUTHENC_SHA1_AES,
84 SA_ALG_AUTHENC_SHA256_AES,
85};
86
87struct sa_match_data {
88 u8 priv;
89 u8 priv_id;
90 u32 supported_algos;
91};
92
93static struct device *sa_k3_dev;
94
95/**
96 * struct sa_cmdl_cfg - Command label configuration descriptor
97 * @aalg: authentication algorithm ID
98 * @enc_eng_id: Encryption Engine ID supported by the SA hardware
99 * @auth_eng_id: Authentication Engine ID
100 * @iv_size: Initialization Vector size
101 * @akey: Authentication key
102 * @akey_len: Authentication key length
103 * @enc: True, if this is an encode request
104 */
105struct sa_cmdl_cfg {
106 int aalg;
107 u8 enc_eng_id;
108 u8 auth_eng_id;
109 u8 iv_size;
110 const u8 *akey;
111 u16 akey_len;
112 bool enc;
113};
114
115/**
116 * struct algo_data - Crypto algorithm specific data
117 * @enc_eng: Encryption engine info structure
118 * @auth_eng: Authentication engine info structure
119 * @auth_ctrl: Authentication control word
120 * @hash_size: Size of digest
121 * @iv_idx: iv index in psdata
122 * @iv_out_size: iv out size
123 * @ealg_id: Encryption Algorithm ID
124 * @aalg_id: Authentication algorithm ID
125 * @mci_enc: Mode Control Instruction for Encryption algorithm
126 * @mci_dec: Mode Control Instruction for Decryption
127 * @inv_key: Whether the encryption algorithm demands key inversion
128 * @ctx: Pointer to the algorithm context
129 * @keyed_mac: Whether the authentication algorithm has key
130 * @prep_iopad: Function pointer to generate intermediate ipad/opad
131 */
132struct algo_data {
133 struct sa_eng_info enc_eng;
134 struct sa_eng_info auth_eng;
135 u8 auth_ctrl;
136 u8 hash_size;
137 u8 iv_idx;
138 u8 iv_out_size;
139 u8 ealg_id;
140 u8 aalg_id;
141 u8 *mci_enc;
142 u8 *mci_dec;
143 bool inv_key;
144 struct sa_tfm_ctx *ctx;
145 bool keyed_mac;
146 void (*prep_iopad)(struct algo_data *algo, const u8 *key,
147 u16 key_sz, __be32 *ipad, __be32 *opad);
148};
149
150/**
151 * struct sa_alg_tmpl: A generic template encompassing crypto/aead algorithms
152 * @type: Type of the crypto algorithm.
153 * @alg: Union of crypto algorithm definitions.
154 * @registered: Flag indicating if the crypto algorithm is already registered
155 */
156struct sa_alg_tmpl {
157 u32 type; /* CRYPTO_ALG_TYPE from <linux/crypto.h> */
158 union {
159 struct skcipher_alg skcipher;
160 struct ahash_alg ahash;
161 struct aead_alg aead;
162 } alg;
163 bool registered;
164};
165
166/**
167 * struct sa_mapped_sg: scatterlist information for tx and rx
168 * @mapped: Set to true if the @sgt is mapped
169 * @dir: mapping direction used for @sgt
170 * @split_sg: Set if the sg is split and needs to be freed up
171 * @static_sg: Static scatterlist entry for overriding data
172 * @sgt: scatterlist table for DMA API use
173 */
174struct sa_mapped_sg {
175 bool mapped;
176 enum dma_data_direction dir;
177 struct scatterlist static_sg;
178 struct scatterlist *split_sg;
179 struct sg_table sgt;
180};
181/**
182 * struct sa_rx_data: RX Packet miscellaneous data place holder
183 * @req: crypto request data pointer
184 * @ddev: pointer to the DMA device
185 * @tx_in: dma_async_tx_descriptor pointer for rx channel
186 * @mapped_sg: Information on tx (0) and rx (1) scatterlist DMA mapping
187 * @enc: Flag indicating either encryption or decryption
188 * @enc_iv_size: Initialisation vector size
189 * @iv_idx: Initialisation vector index
190 */
191struct sa_rx_data {
192 void *req;
193 struct device *ddev;
194 struct dma_async_tx_descriptor *tx_in;
195 struct sa_mapped_sg mapped_sg[2];
196 u8 enc;
197 u8 enc_iv_size;
198 u8 iv_idx;
199};
200
201/**
202 * struct sa_req: SA request definition
203 * @dev: device for the request
204 * @size: total data to the xmitted via DMA
205 * @enc_offset: offset of cipher data
206 * @enc_size: data to be passed to cipher engine
207 * @enc_iv: cipher IV
208 * @auth_offset: offset of the authentication data
209 * @auth_size: size of the authentication data
210 * @auth_iv: authentication IV
211 * @type: algorithm type for the request
212 * @cmdl: command label pointer
213 * @base: pointer to the base request
214 * @ctx: pointer to the algorithm context data
215 * @enc: true if this is an encode request
216 * @src: source data
217 * @dst: destination data
218 * @callback: DMA callback for the request
219 * @mdata_size: metadata size passed to DMA
220 */
221struct sa_req {
222 struct device *dev;
223 u16 size;
224 u8 enc_offset;
225 u16 enc_size;
226 u8 *enc_iv;
227 u8 auth_offset;
228 u16 auth_size;
229 u8 *auth_iv;
230 u32 type;
231 u32 *cmdl;
232 struct crypto_async_request *base;
233 struct sa_tfm_ctx *ctx;
234 bool enc;
235 struct scatterlist *src;
236 struct scatterlist *dst;
237 dma_async_tx_callback callback;
238 u16 mdata_size;
239};
240
241/*
242 * Mode Control Instructions for various Key lengths 128, 192, 256
243 * For CBC (Cipher Block Chaining) mode for encryption
244 */
245static u8 mci_cbc_enc_array[3][MODE_CONTROL_BYTES] = {
246 { 0x61, 0x00, 0x00, 0x18, 0x88, 0x0a, 0xaa, 0x4b, 0x7e, 0x00,
247 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
248 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
249 { 0x61, 0x00, 0x00, 0x18, 0x88, 0x4a, 0xaa, 0x4b, 0x7e, 0x00,
250 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
251 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
252 { 0x61, 0x00, 0x00, 0x18, 0x88, 0x8a, 0xaa, 0x4b, 0x7e, 0x00,
253 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
254 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
255};
256
257/*
258 * Mode Control Instructions for various Key lengths 128, 192, 256
259 * For CBC (Cipher Block Chaining) mode for decryption
260 */
261static u8 mci_cbc_dec_array[3][MODE_CONTROL_BYTES] = {
262 { 0x71, 0x00, 0x00, 0x80, 0x8a, 0xca, 0x98, 0xf4, 0x40, 0xc0,
263 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
264 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
265 { 0x71, 0x00, 0x00, 0x84, 0x8a, 0xca, 0x98, 0xf4, 0x40, 0xc0,
266 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
267 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
268 { 0x71, 0x00, 0x00, 0x88, 0x8a, 0xca, 0x98, 0xf4, 0x40, 0xc0,
269 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
270 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
271};
272
273/*
274 * Mode Control Instructions for various Key lengths 128, 192, 256
275 * For CBC (Cipher Block Chaining) mode for encryption
276 */
277static u8 mci_cbc_enc_no_iv_array[3][MODE_CONTROL_BYTES] = {
278 { 0x21, 0x00, 0x00, 0x18, 0x88, 0x0a, 0xaa, 0x4b, 0x7e, 0x00,
279 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
280 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
281 { 0x21, 0x00, 0x00, 0x18, 0x88, 0x4a, 0xaa, 0x4b, 0x7e, 0x00,
282 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
283 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
284 { 0x21, 0x00, 0x00, 0x18, 0x88, 0x8a, 0xaa, 0x4b, 0x7e, 0x00,
285 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
286 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
287};
288
289/*
290 * Mode Control Instructions for various Key lengths 128, 192, 256
291 * For CBC (Cipher Block Chaining) mode for decryption
292 */
293static u8 mci_cbc_dec_no_iv_array[3][MODE_CONTROL_BYTES] = {
294 { 0x31, 0x00, 0x00, 0x80, 0x8a, 0xca, 0x98, 0xf4, 0x40, 0xc0,
295 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
296 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
297 { 0x31, 0x00, 0x00, 0x84, 0x8a, 0xca, 0x98, 0xf4, 0x40, 0xc0,
298 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
299 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
300 { 0x31, 0x00, 0x00, 0x88, 0x8a, 0xca, 0x98, 0xf4, 0x40, 0xc0,
301 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
302 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
303};
304
305/*
306 * Mode Control Instructions for various Key lengths 128, 192, 256
307 * For ECB (Electronic Code Book) mode for encryption
308 */
309static u8 mci_ecb_enc_array[3][27] = {
310 { 0x21, 0x00, 0x00, 0x80, 0x8a, 0x04, 0xb7, 0x90, 0x00, 0x00,
311 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
312 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
313 { 0x21, 0x00, 0x00, 0x84, 0x8a, 0x04, 0xb7, 0x90, 0x00, 0x00,
314 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
315 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
316 { 0x21, 0x00, 0x00, 0x88, 0x8a, 0x04, 0xb7, 0x90, 0x00, 0x00,
317 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
318 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
319};
320
321/*
322 * Mode Control Instructions for various Key lengths 128, 192, 256
323 * For ECB (Electronic Code Book) mode for decryption
324 */
325static u8 mci_ecb_dec_array[3][27] = {
326 { 0x31, 0x00, 0x00, 0x80, 0x8a, 0x04, 0xb7, 0x90, 0x00, 0x00,
327 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
328 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
329 { 0x31, 0x00, 0x00, 0x84, 0x8a, 0x04, 0xb7, 0x90, 0x00, 0x00,
330 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
331 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
332 { 0x31, 0x00, 0x00, 0x88, 0x8a, 0x04, 0xb7, 0x90, 0x00, 0x00,
333 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
334 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
335};
336
337/*
338 * Mode Control Instructions for DES algorithm
339 * For CBC (Cipher Block Chaining) mode and ECB mode
340 * encryption and for decryption respectively
341 */
342static u8 mci_cbc_3des_enc_array[MODE_CONTROL_BYTES] = {
343 0x60, 0x00, 0x00, 0x18, 0x88, 0x52, 0xaa, 0x4b, 0x7e, 0x00, 0x00, 0x00,
344 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
345 0x00, 0x00, 0x00,
346};
347
348static u8 mci_cbc_3des_dec_array[MODE_CONTROL_BYTES] = {
349 0x70, 0x00, 0x00, 0x85, 0x0a, 0xca, 0x98, 0xf4, 0x40, 0xc0, 0x00, 0x00,
350 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
351 0x00, 0x00, 0x00,
352};
353
354static u8 mci_ecb_3des_enc_array[MODE_CONTROL_BYTES] = {
355 0x20, 0x00, 0x00, 0x85, 0x0a, 0x04, 0xb7, 0x90, 0x00, 0x00, 0x00, 0x00,
356 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
357 0x00, 0x00, 0x00,
358};
359
360static u8 mci_ecb_3des_dec_array[MODE_CONTROL_BYTES] = {
361 0x30, 0x00, 0x00, 0x85, 0x0a, 0x04, 0xb7, 0x90, 0x00, 0x00, 0x00, 0x00,
362 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
363 0x00, 0x00, 0x00,
364};
365
366/*
367 * Perform 16 byte or 128 bit swizzling
368 * The SA2UL Expects the security context to
369 * be in little Endian and the bus width is 128 bits or 16 bytes
370 * Hence swap 16 bytes at a time from higher to lower address
371 */
372static void sa_swiz_128(u8 *in, u16 len)
373{
374 u8 data[16];
375 int i, j;
376
377 for (i = 0; i < len; i += 16) {
378 memcpy(data, &in[i], 16);
379 for (j = 0; j < 16; j++)
380 in[i + j] = data[15 - j];
381 }
382}
383
384/* Prepare the ipad and opad from key as per SHA algorithm step 1*/
385static void prepare_kipad(u8 *k_ipad, const u8 *key, u16 key_sz)
386{
387 int i;
388
389 for (i = 0; i < key_sz; i++)
390 k_ipad[i] = key[i] ^ 0x36;
391
392 /* Instead of XOR with 0 */
393 for (; i < SHA1_BLOCK_SIZE; i++)
394 k_ipad[i] = 0x36;
395}
396
397static void prepare_kopad(u8 *k_opad, const u8 *key, u16 key_sz)
398{
399 int i;
400
401 for (i = 0; i < key_sz; i++)
402 k_opad[i] = key[i] ^ 0x5c;
403
404 /* Instead of XOR with 0 */
405 for (; i < SHA1_BLOCK_SIZE; i++)
406 k_opad[i] = 0x5c;
407}
408
409static void sa_export_shash(void *state, struct shash_desc *hash,
410 int digest_size, __be32 *out)
411{
412 struct sha1_state *sha1;
413 struct sha256_state *sha256;
414 u32 *result;
415
416 switch (digest_size) {
417 case SHA1_DIGEST_SIZE:
418 sha1 = state;
419 result = sha1->state;
420 break;
421 case SHA256_DIGEST_SIZE:
422 sha256 = state;
423 result = sha256->state;
424 break;
425 default:
426 dev_err(sa_k3_dev, "%s: bad digest_size=%d\n", __func__,
427 digest_size);
428 return;
429 }
430
431 crypto_shash_export(hash, state);
432
433 cpu_to_be32_array(out, result, digest_size / 4);
434}
435
436static void sa_prepare_iopads(struct algo_data *data, const u8 *key,
437 u16 key_sz, __be32 *ipad, __be32 *opad)
438{
439 SHASH_DESC_ON_STACK(shash, data->ctx->shash);
440 int block_size = crypto_shash_blocksize(data->ctx->shash);
441 int digest_size = crypto_shash_digestsize(data->ctx->shash);
442 union {
443 struct sha1_state sha1;
444 struct sha256_state sha256;
445 u8 k_pad[SHA1_BLOCK_SIZE];
446 } sha;
447
448 shash->tfm = data->ctx->shash;
449
450 prepare_kipad(sha.k_pad, key, key_sz);
451
452 crypto_shash_init(shash);
453 crypto_shash_update(shash, sha.k_pad, block_size);
454 sa_export_shash(&sha, shash, digest_size, ipad);
455
456 prepare_kopad(sha.k_pad, key, key_sz);
457
458 crypto_shash_init(shash);
459 crypto_shash_update(shash, sha.k_pad, block_size);
460
461 sa_export_shash(&sha, shash, digest_size, opad);
462
463 memzero_explicit(&sha, sizeof(sha));
464}
465
466/* Derive the inverse key used in AES-CBC decryption operation */
467static inline int sa_aes_inv_key(u8 *inv_key, const u8 *key, u16 key_sz)
468{
469 struct crypto_aes_ctx ctx;
470 int key_pos;
471
472 if (aes_expandkey(&ctx, key, key_sz)) {
473 dev_err(sa_k3_dev, "%s: bad key len(%d)\n", __func__, key_sz);
474 return -EINVAL;
475 }
476
477 /* work around to get the right inverse for AES_KEYSIZE_192 size keys */
478 if (key_sz == AES_KEYSIZE_192) {
479 ctx.key_enc[52] = ctx.key_enc[51] ^ ctx.key_enc[46];
480 ctx.key_enc[53] = ctx.key_enc[52] ^ ctx.key_enc[47];
481 }
482
483 /* Based crypto_aes_expand_key logic */
484 switch (key_sz) {
485 case AES_KEYSIZE_128:
486 case AES_KEYSIZE_192:
487 key_pos = key_sz + 24;
488 break;
489
490 case AES_KEYSIZE_256:
491 key_pos = key_sz + 24 - 4;
492 break;
493
494 default:
495 dev_err(sa_k3_dev, "%s: bad key len(%d)\n", __func__, key_sz);
496 return -EINVAL;
497 }
498
499 memcpy(inv_key, &ctx.key_enc[key_pos], key_sz);
500 return 0;
501}
502
503/* Set Security context for the encryption engine */
504static int sa_set_sc_enc(struct algo_data *ad, const u8 *key, u16 key_sz,
505 u8 enc, u8 *sc_buf)
506{
507 const u8 *mci = NULL;
508
509 /* Set Encryption mode selector to crypto processing */
510 sc_buf[0] = SA_CRYPTO_PROCESSING;
511
512 if (enc)
513 mci = ad->mci_enc;
514 else
515 mci = ad->mci_dec;
516 /* Set the mode control instructions in security context */
517 if (mci)
518 memcpy(&sc_buf[1], mci, MODE_CONTROL_BYTES);
519
520 /* For AES-CBC decryption get the inverse key */
521 if (ad->inv_key && !enc) {
522 if (sa_aes_inv_key(&sc_buf[SC_ENC_KEY_OFFSET], key, key_sz))
523 return -EINVAL;
524 /* For all other cases: key is used */
525 } else {
526 memcpy(&sc_buf[SC_ENC_KEY_OFFSET], key, key_sz);
527 }
528
529 return 0;
530}
531
532/* Set Security context for the authentication engine */
533static void sa_set_sc_auth(struct algo_data *ad, const u8 *key, u16 key_sz,
534 u8 *sc_buf)
535{
536 __be32 *ipad = (void *)(sc_buf + 32);
537 __be32 *opad = (void *)(sc_buf + 64);
538
539 /* Set Authentication mode selector to hash processing */
540 sc_buf[0] = SA_HASH_PROCESSING;
541 /* Auth SW ctrl word: bit[6]=1 (upload computed hash to TLR section) */
542 sc_buf[1] = SA_UPLOAD_HASH_TO_TLR;
543 sc_buf[1] |= ad->auth_ctrl;
544
545 /* Copy the keys or ipad/opad */
546 if (ad->keyed_mac)
547 ad->prep_iopad(ad, key, key_sz, ipad, opad);
548 else {
549 /* basic hash */
550 sc_buf[1] |= SA_BASIC_HASH;
551 }
552}
553
554static inline void sa_copy_iv(__be32 *out, const u8 *iv, bool size16)
555{
556 int j;
557
558 for (j = 0; j < ((size16) ? 4 : 2); j++) {
559 *out = cpu_to_be32(*((u32 *)iv));
560 iv += 4;
561 out++;
562 }
563}
564
565/* Format general command label */
566static int sa_format_cmdl_gen(struct sa_cmdl_cfg *cfg, u8 *cmdl,
567 struct sa_cmdl_upd_info *upd_info)
568{
569 u8 enc_offset = 0, auth_offset = 0, total = 0;
570 u8 enc_next_eng = SA_ENG_ID_OUTPORT2;
571 u8 auth_next_eng = SA_ENG_ID_OUTPORT2;
572 u32 *word_ptr = (u32 *)cmdl;
573 int i;
574
575 /* Clear the command label */
576 memzero_explicit(cmdl, (SA_MAX_CMDL_WORDS * sizeof(u32)));
577
578 /* Initialize the command update structure */
579 memzero_explicit(upd_info, sizeof(*upd_info));
580
581 if (cfg->enc_eng_id && cfg->auth_eng_id) {
582 if (cfg->enc) {
583 auth_offset = SA_CMDL_HEADER_SIZE_BYTES;
584 enc_next_eng = cfg->auth_eng_id;
585
586 if (cfg->iv_size)
587 auth_offset += cfg->iv_size;
588 } else {
589 enc_offset = SA_CMDL_HEADER_SIZE_BYTES;
590 auth_next_eng = cfg->enc_eng_id;
591 }
592 }
593
594 if (cfg->enc_eng_id) {
595 upd_info->flags |= SA_CMDL_UPD_ENC;
596 upd_info->enc_size.index = enc_offset >> 2;
597 upd_info->enc_offset.index = upd_info->enc_size.index + 1;
598 /* Encryption command label */
599 cmdl[enc_offset + SA_CMDL_OFFSET_NESC] = enc_next_eng;
600
601 /* Encryption modes requiring IV */
602 if (cfg->iv_size) {
603 upd_info->flags |= SA_CMDL_UPD_ENC_IV;
604 upd_info->enc_iv.index =
605 (enc_offset + SA_CMDL_HEADER_SIZE_BYTES) >> 2;
606 upd_info->enc_iv.size = cfg->iv_size;
607
608 cmdl[enc_offset + SA_CMDL_OFFSET_LABEL_LEN] =
609 SA_CMDL_HEADER_SIZE_BYTES + cfg->iv_size;
610
611 cmdl[enc_offset + SA_CMDL_OFFSET_OPTION_CTRL1] =
612 (SA_CTX_ENC_AUX2_OFFSET | (cfg->iv_size >> 3));
613 total += SA_CMDL_HEADER_SIZE_BYTES + cfg->iv_size;
614 } else {
615 cmdl[enc_offset + SA_CMDL_OFFSET_LABEL_LEN] =
616 SA_CMDL_HEADER_SIZE_BYTES;
617 total += SA_CMDL_HEADER_SIZE_BYTES;
618 }
619 }
620
621 if (cfg->auth_eng_id) {
622 upd_info->flags |= SA_CMDL_UPD_AUTH;
623 upd_info->auth_size.index = auth_offset >> 2;
624 upd_info->auth_offset.index = upd_info->auth_size.index + 1;
625 cmdl[auth_offset + SA_CMDL_OFFSET_NESC] = auth_next_eng;
626 cmdl[auth_offset + SA_CMDL_OFFSET_LABEL_LEN] =
627 SA_CMDL_HEADER_SIZE_BYTES;
628 total += SA_CMDL_HEADER_SIZE_BYTES;
629 }
630
631 total = roundup(total, 8);
632
633 for (i = 0; i < total / 4; i++)
634 word_ptr[i] = swab32(word_ptr[i]);
635
636 return total;
637}
638
639/* Update Command label */
640static inline void sa_update_cmdl(struct sa_req *req, u32 *cmdl,
641 struct sa_cmdl_upd_info *upd_info)
642{
643 int i = 0, j;
644
645 if (likely(upd_info->flags & SA_CMDL_UPD_ENC)) {
646 cmdl[upd_info->enc_size.index] &= ~SA_CMDL_PAYLOAD_LENGTH_MASK;
647 cmdl[upd_info->enc_size.index] |= req->enc_size;
648 cmdl[upd_info->enc_offset.index] &=
649 ~SA_CMDL_SOP_BYPASS_LEN_MASK;
650 cmdl[upd_info->enc_offset.index] |=
651 FIELD_PREP(SA_CMDL_SOP_BYPASS_LEN_MASK,
652 req->enc_offset);
653
654 if (likely(upd_info->flags & SA_CMDL_UPD_ENC_IV)) {
655 __be32 *data = (__be32 *)&cmdl[upd_info->enc_iv.index];
656 u32 *enc_iv = (u32 *)req->enc_iv;
657
658 for (j = 0; i < upd_info->enc_iv.size; i += 4, j++) {
659 data[j] = cpu_to_be32(*enc_iv);
660 enc_iv++;
661 }
662 }
663 }
664
665 if (likely(upd_info->flags & SA_CMDL_UPD_AUTH)) {
666 cmdl[upd_info->auth_size.index] &= ~SA_CMDL_PAYLOAD_LENGTH_MASK;
667 cmdl[upd_info->auth_size.index] |= req->auth_size;
668 cmdl[upd_info->auth_offset.index] &=
669 ~SA_CMDL_SOP_BYPASS_LEN_MASK;
670 cmdl[upd_info->auth_offset.index] |=
671 FIELD_PREP(SA_CMDL_SOP_BYPASS_LEN_MASK,
672 req->auth_offset);
673 if (upd_info->flags & SA_CMDL_UPD_AUTH_IV) {
674 sa_copy_iv((void *)&cmdl[upd_info->auth_iv.index],
675 req->auth_iv,
676 (upd_info->auth_iv.size > 8));
677 }
678 if (upd_info->flags & SA_CMDL_UPD_AUX_KEY) {
679 int offset = (req->auth_size & 0xF) ? 4 : 0;
680
681 memcpy(&cmdl[upd_info->aux_key_info.index],
682 &upd_info->aux_key[offset], 16);
683 }
684 }
685}
686
687/* Format SWINFO words to be sent to SA */
688static
689void sa_set_swinfo(u8 eng_id, u16 sc_id, dma_addr_t sc_phys,
690 u8 cmdl_present, u8 cmdl_offset, u8 flags,
691 u8 hash_size, u32 *swinfo)
692{
693 swinfo[0] = sc_id;
694 swinfo[0] |= FIELD_PREP(SA_SW0_FLAGS_MASK, flags);
695 if (likely(cmdl_present))
696 swinfo[0] |= FIELD_PREP(SA_SW0_CMDL_INFO_MASK,
697 cmdl_offset | SA_SW0_CMDL_PRESENT);
698 swinfo[0] |= FIELD_PREP(SA_SW0_ENG_ID_MASK, eng_id);
699
700 swinfo[0] |= SA_SW0_DEST_INFO_PRESENT;
701 swinfo[1] = (u32)(sc_phys & 0xFFFFFFFFULL);
702 swinfo[2] = (u32)((sc_phys & 0xFFFFFFFF00000000ULL) >> 32);
703 swinfo[2] |= FIELD_PREP(SA_SW2_EGRESS_LENGTH, hash_size);
704}
705
706/* Dump the security context */
707static void sa_dump_sc(u8 *buf, dma_addr_t dma_addr)
708{
709#ifdef DEBUG
710 dev_info(sa_k3_dev, "Security context dump:: 0x%pad\n", &dma_addr);
711 print_hex_dump(KERN_CONT, "", DUMP_PREFIX_OFFSET,
712 16, 1, buf, SA_CTX_MAX_SZ, false);
713#endif
714}
715
716static
717int sa_init_sc(struct sa_ctx_info *ctx, const struct sa_match_data *match_data,
718 const u8 *enc_key, u16 enc_key_sz,
719 const u8 *auth_key, u16 auth_key_sz,
720 struct algo_data *ad, u8 enc, u32 *swinfo)
721{
722 int enc_sc_offset = 0;
723 int auth_sc_offset = 0;
724 u8 *sc_buf = ctx->sc;
725 u16 sc_id = ctx->sc_id;
726 u8 first_engine = 0;
727
728 memzero_explicit(sc_buf, SA_CTX_MAX_SZ);
729
730 if (ad->auth_eng.eng_id) {
731 if (enc)
732 first_engine = ad->enc_eng.eng_id;
733 else
734 first_engine = ad->auth_eng.eng_id;
735
736 enc_sc_offset = SA_CTX_PHP_PE_CTX_SZ;
737 auth_sc_offset = enc_sc_offset + ad->enc_eng.sc_size;
738 sc_buf[1] = SA_SCCTL_FE_AUTH_ENC;
739 if (!ad->hash_size)
740 return -EINVAL;
741 ad->hash_size = roundup(ad->hash_size, 8);
742
743 } else if (ad->enc_eng.eng_id && !ad->auth_eng.eng_id) {
744 enc_sc_offset = SA_CTX_PHP_PE_CTX_SZ;
745 first_engine = ad->enc_eng.eng_id;
746 sc_buf[1] = SA_SCCTL_FE_ENC;
747 ad->hash_size = ad->iv_out_size;
748 }
749
750 /* SCCTL Owner info: 0=host, 1=CP_ACE */
751 sc_buf[SA_CTX_SCCTL_OWNER_OFFSET] = 0;
752 memcpy(&sc_buf[2], &sc_id, 2);
753 sc_buf[4] = 0x0;
754 sc_buf[5] = match_data->priv_id;
755 sc_buf[6] = match_data->priv;
756 sc_buf[7] = 0x0;
757
758 /* Prepare context for encryption engine */
759 if (ad->enc_eng.sc_size) {
760 if (sa_set_sc_enc(ad, enc_key, enc_key_sz, enc,
761 &sc_buf[enc_sc_offset]))
762 return -EINVAL;
763 }
764
765 /* Prepare context for authentication engine */
766 if (ad->auth_eng.sc_size)
767 sa_set_sc_auth(ad, auth_key, auth_key_sz,
768 &sc_buf[auth_sc_offset]);
769
770 /* Set the ownership of context to CP_ACE */
771 sc_buf[SA_CTX_SCCTL_OWNER_OFFSET] = 0x80;
772
773 /* swizzle the security context */
774 sa_swiz_128(sc_buf, SA_CTX_MAX_SZ);
775
776 sa_set_swinfo(first_engine, ctx->sc_id, ctx->sc_phys, 1, 0,
777 SA_SW_INFO_FLAG_EVICT, ad->hash_size, swinfo);
778
779 sa_dump_sc(sc_buf, ctx->sc_phys);
780
781 return 0;
782}
783
784/* Free the per direction context memory */
785static void sa_free_ctx_info(struct sa_ctx_info *ctx,
786 struct sa_crypto_data *data)
787{
788 unsigned long bn;
789
790 bn = ctx->sc_id - data->sc_id_start;
791 spin_lock(&data->scid_lock);
792 __clear_bit(bn, data->ctx_bm);
793 data->sc_id--;
794 spin_unlock(&data->scid_lock);
795
796 if (ctx->sc) {
797 dma_pool_free(data->sc_pool, ctx->sc, ctx->sc_phys);
798 ctx->sc = NULL;
799 }
800}
801
802static int sa_init_ctx_info(struct sa_ctx_info *ctx,
803 struct sa_crypto_data *data)
804{
805 unsigned long bn;
806 int err;
807
808 spin_lock(&data->scid_lock);
809 bn = find_first_zero_bit(data->ctx_bm, SA_MAX_NUM_CTX);
810 __set_bit(bn, data->ctx_bm);
811 data->sc_id++;
812 spin_unlock(&data->scid_lock);
813
814 ctx->sc_id = (u16)(data->sc_id_start + bn);
815
816 ctx->sc = dma_pool_alloc(data->sc_pool, GFP_KERNEL, &ctx->sc_phys);
817 if (!ctx->sc) {
818 dev_err(&data->pdev->dev, "Failed to allocate SC memory\n");
819 err = -ENOMEM;
820 goto scid_rollback;
821 }
822
823 return 0;
824
825scid_rollback:
826 spin_lock(&data->scid_lock);
827 __clear_bit(bn, data->ctx_bm);
828 data->sc_id--;
829 spin_unlock(&data->scid_lock);
830
831 return err;
832}
833
834static void sa_cipher_cra_exit(struct crypto_skcipher *tfm)
835{
836 struct sa_tfm_ctx *ctx = crypto_skcipher_ctx(tfm);
837 struct sa_crypto_data *data = dev_get_drvdata(sa_k3_dev);
838
839 dev_dbg(sa_k3_dev, "%s(0x%p) sc-ids(0x%x(0x%pad), 0x%x(0x%pad))\n",
840 __func__, tfm, ctx->enc.sc_id, &ctx->enc.sc_phys,
841 ctx->dec.sc_id, &ctx->dec.sc_phys);
842
843 sa_free_ctx_info(&ctx->enc, data);
844 sa_free_ctx_info(&ctx->dec, data);
845
846 crypto_free_skcipher(ctx->fallback.skcipher);
847}
848
849static int sa_cipher_cra_init(struct crypto_skcipher *tfm)
850{
851 struct sa_tfm_ctx *ctx = crypto_skcipher_ctx(tfm);
852 struct sa_crypto_data *data = dev_get_drvdata(sa_k3_dev);
853 const char *name = crypto_tfm_alg_name(&tfm->base);
854 struct crypto_skcipher *child;
855 int ret;
856
857 memzero_explicit(ctx, sizeof(*ctx));
858 ctx->dev_data = data;
859
860 ret = sa_init_ctx_info(&ctx->enc, data);
861 if (ret)
862 return ret;
863 ret = sa_init_ctx_info(&ctx->dec, data);
864 if (ret) {
865 sa_free_ctx_info(&ctx->enc, data);
866 return ret;
867 }
868
869 child = crypto_alloc_skcipher(name, 0, CRYPTO_ALG_NEED_FALLBACK);
870
871 if (IS_ERR(child)) {
872 dev_err(sa_k3_dev, "Error allocating fallback algo %s\n", name);
873 return PTR_ERR(child);
874 }
875
876 ctx->fallback.skcipher = child;
877 crypto_skcipher_set_reqsize(tfm, crypto_skcipher_reqsize(child) +
878 sizeof(struct skcipher_request));
879
880 dev_dbg(sa_k3_dev, "%s(0x%p) sc-ids(0x%x(0x%pad), 0x%x(0x%pad))\n",
881 __func__, tfm, ctx->enc.sc_id, &ctx->enc.sc_phys,
882 ctx->dec.sc_id, &ctx->dec.sc_phys);
883 return 0;
884}
885
886static int sa_cipher_setkey(struct crypto_skcipher *tfm, const u8 *key,
887 unsigned int keylen, struct algo_data *ad)
888{
889 struct sa_tfm_ctx *ctx = crypto_skcipher_ctx(tfm);
890 struct crypto_skcipher *child = ctx->fallback.skcipher;
891 int cmdl_len;
892 struct sa_cmdl_cfg cfg;
893 int ret;
894
895 if (keylen != AES_KEYSIZE_128 && keylen != AES_KEYSIZE_192 &&
896 keylen != AES_KEYSIZE_256)
897 return -EINVAL;
898
899 ad->enc_eng.eng_id = SA_ENG_ID_EM1;
900 ad->enc_eng.sc_size = SA_CTX_ENC_TYPE1_SZ;
901
902 memzero_explicit(&cfg, sizeof(cfg));
903 cfg.enc_eng_id = ad->enc_eng.eng_id;
904 cfg.iv_size = crypto_skcipher_ivsize(tfm);
905
906 crypto_skcipher_clear_flags(child, CRYPTO_TFM_REQ_MASK);
907 crypto_skcipher_set_flags(child, tfm->base.crt_flags &
908 CRYPTO_TFM_REQ_MASK);
909 ret = crypto_skcipher_setkey(child, key, keylen);
910 if (ret)
911 return ret;
912
913 /* Setup Encryption Security Context & Command label template */
914 if (sa_init_sc(&ctx->enc, ctx->dev_data->match_data, key, keylen, NULL, 0,
915 ad, 1, &ctx->enc.epib[1]))
916 goto badkey;
917
918 cmdl_len = sa_format_cmdl_gen(&cfg,
919 (u8 *)ctx->enc.cmdl,
920 &ctx->enc.cmdl_upd_info);
921 if (cmdl_len <= 0 || (cmdl_len > SA_MAX_CMDL_WORDS * sizeof(u32)))
922 goto badkey;
923
924 ctx->enc.cmdl_size = cmdl_len;
925
926 /* Setup Decryption Security Context & Command label template */
927 if (sa_init_sc(&ctx->dec, ctx->dev_data->match_data, key, keylen, NULL, 0,
928 ad, 0, &ctx->dec.epib[1]))
929 goto badkey;
930
931 cfg.enc_eng_id = ad->enc_eng.eng_id;
932 cmdl_len = sa_format_cmdl_gen(&cfg, (u8 *)ctx->dec.cmdl,
933 &ctx->dec.cmdl_upd_info);
934
935 if (cmdl_len <= 0 || (cmdl_len > SA_MAX_CMDL_WORDS * sizeof(u32)))
936 goto badkey;
937
938 ctx->dec.cmdl_size = cmdl_len;
939 ctx->iv_idx = ad->iv_idx;
940
941 return 0;
942
943badkey:
944 dev_err(sa_k3_dev, "%s: badkey\n", __func__);
945 return -EINVAL;
946}
947
948static int sa_aes_cbc_setkey(struct crypto_skcipher *tfm, const u8 *key,
949 unsigned int keylen)
950{
951 struct algo_data ad = { 0 };
952 /* Convert the key size (16/24/32) to the key size index (0/1/2) */
953 int key_idx = (keylen >> 3) - 2;
954
955 if (key_idx >= 3)
956 return -EINVAL;
957
958 ad.mci_enc = mci_cbc_enc_array[key_idx];
959 ad.mci_dec = mci_cbc_dec_array[key_idx];
960 ad.inv_key = true;
961 ad.ealg_id = SA_EALG_ID_AES_CBC;
962 ad.iv_idx = 4;
963 ad.iv_out_size = 16;
964
965 return sa_cipher_setkey(tfm, key, keylen, &ad);
966}
967
968static int sa_aes_ecb_setkey(struct crypto_skcipher *tfm, const u8 *key,
969 unsigned int keylen)
970{
971 struct algo_data ad = { 0 };
972 /* Convert the key size (16/24/32) to the key size index (0/1/2) */
973 int key_idx = (keylen >> 3) - 2;
974
975 if (key_idx >= 3)
976 return -EINVAL;
977
978 ad.mci_enc = mci_ecb_enc_array[key_idx];
979 ad.mci_dec = mci_ecb_dec_array[key_idx];
980 ad.inv_key = true;
981 ad.ealg_id = SA_EALG_ID_AES_ECB;
982
983 return sa_cipher_setkey(tfm, key, keylen, &ad);
984}
985
986static int sa_3des_cbc_setkey(struct crypto_skcipher *tfm, const u8 *key,
987 unsigned int keylen)
988{
989 struct algo_data ad = { 0 };
990
991 ad.mci_enc = mci_cbc_3des_enc_array;
992 ad.mci_dec = mci_cbc_3des_dec_array;
993 ad.ealg_id = SA_EALG_ID_3DES_CBC;
994 ad.iv_idx = 6;
995 ad.iv_out_size = 8;
996
997 return sa_cipher_setkey(tfm, key, keylen, &ad);
998}
999
1000static int sa_3des_ecb_setkey(struct crypto_skcipher *tfm, const u8 *key,
1001 unsigned int keylen)
1002{
1003 struct algo_data ad = { 0 };
1004
1005 ad.mci_enc = mci_ecb_3des_enc_array;
1006 ad.mci_dec = mci_ecb_3des_dec_array;
1007
1008 return sa_cipher_setkey(tfm, key, keylen, &ad);
1009}
1010
1011static void sa_sync_from_device(struct sa_rx_data *rxd)
1012{
1013 struct sg_table *sgt;
1014
1015 if (rxd->mapped_sg[0].dir == DMA_BIDIRECTIONAL)
1016 sgt = &rxd->mapped_sg[0].sgt;
1017 else
1018 sgt = &rxd->mapped_sg[1].sgt;
1019
1020 dma_sync_sgtable_for_cpu(rxd->ddev, sgt, DMA_FROM_DEVICE);
1021}
1022
1023static void sa_free_sa_rx_data(struct sa_rx_data *rxd)
1024{
1025 int i;
1026
1027 for (i = 0; i < ARRAY_SIZE(rxd->mapped_sg); i++) {
1028 struct sa_mapped_sg *mapped_sg = &rxd->mapped_sg[i];
1029
1030 if (mapped_sg->mapped) {
1031 dma_unmap_sgtable(rxd->ddev, &mapped_sg->sgt,
1032 mapped_sg->dir, 0);
1033 kfree(mapped_sg->split_sg);
1034 }
1035 }
1036
1037 kfree(rxd);
1038}
1039
1040static void sa_aes_dma_in_callback(void *data)
1041{
1042 struct sa_rx_data *rxd = data;
1043 struct skcipher_request *req;
1044 u32 *result;
1045 __be32 *mdptr;
1046 size_t ml, pl;
1047 int i;
1048
1049 sa_sync_from_device(rxd);
1050 req = container_of(rxd->req, struct skcipher_request, base);
1051
1052 if (req->iv) {
1053 mdptr = (__be32 *)dmaengine_desc_get_metadata_ptr(rxd->tx_in, &pl,
1054 &ml);
1055 result = (u32 *)req->iv;
1056
1057 for (i = 0; i < (rxd->enc_iv_size / 4); i++)
1058 result[i] = be32_to_cpu(mdptr[i + rxd->iv_idx]);
1059 }
1060
1061 sa_free_sa_rx_data(rxd);
1062
1063 skcipher_request_complete(req, 0);
1064}
1065
1066static void
1067sa_prepare_tx_desc(u32 *mdptr, u32 pslen, u32 *psdata, u32 epiblen, u32 *epib)
1068{
1069 u32 *out, *in;
1070 int i;
1071
1072 for (out = mdptr, in = epib, i = 0; i < epiblen / sizeof(u32); i++)
1073 *out++ = *in++;
1074
1075 mdptr[4] = (0xFFFF << 16);
1076 for (out = &mdptr[5], in = psdata, i = 0;
1077 i < pslen / sizeof(u32); i++)
1078 *out++ = *in++;
1079}
1080
1081static int sa_run(struct sa_req *req)
1082{
1083 struct sa_rx_data *rxd;
1084 gfp_t gfp_flags;
1085 u32 cmdl[SA_MAX_CMDL_WORDS];
1086 struct sa_crypto_data *pdata = dev_get_drvdata(sa_k3_dev);
1087 struct device *ddev;
1088 struct dma_chan *dma_rx;
1089 int sg_nents, src_nents, dst_nents;
1090 struct scatterlist *src, *dst;
1091 size_t pl, ml, split_size;
1092 struct sa_ctx_info *sa_ctx = req->enc ? &req->ctx->enc : &req->ctx->dec;
1093 int ret;
1094 struct dma_async_tx_descriptor *tx_out;
1095 u32 *mdptr;
1096 bool diff_dst;
1097 enum dma_data_direction dir_src;
1098 struct sa_mapped_sg *mapped_sg;
1099
1100 gfp_flags = req->base->flags & CRYPTO_TFM_REQ_MAY_SLEEP ?
1101 GFP_KERNEL : GFP_ATOMIC;
1102
1103 rxd = kzalloc_obj(*rxd, gfp_flags);
1104 if (!rxd)
1105 return -ENOMEM;
1106
1107 if (req->src != req->dst) {
1108 diff_dst = true;
1109 dir_src = DMA_TO_DEVICE;
1110 } else {
1111 diff_dst = false;
1112 dir_src = DMA_BIDIRECTIONAL;
1113 }
1114
1115 /*
1116 * SA2UL has an interesting feature where the receive DMA channel
1117 * is selected based on the data passed to the engine. Within the
1118 * transition range, there is also a space where it is impossible
1119 * to determine where the data will end up, and this should be
1120 * avoided. This will be handled by the SW fallback mechanism by
1121 * the individual algorithm implementations.
1122 */
1123 if (req->size >= 256)
1124 dma_rx = pdata->dma_rx2;
1125 else
1126 dma_rx = pdata->dma_rx1;
1127
1128 ddev = dmaengine_get_dma_device(pdata->dma_tx);
1129 rxd->ddev = ddev;
1130
1131 memcpy(cmdl, sa_ctx->cmdl, sa_ctx->cmdl_size);
1132
1133 sa_update_cmdl(req, cmdl, &sa_ctx->cmdl_upd_info);
1134
1135 if (req->type != CRYPTO_ALG_TYPE_AHASH) {
1136 if (req->enc)
1137 req->type |=
1138 (SA_REQ_SUBTYPE_ENC << SA_REQ_SUBTYPE_SHIFT);
1139 else
1140 req->type |=
1141 (SA_REQ_SUBTYPE_DEC << SA_REQ_SUBTYPE_SHIFT);
1142 }
1143
1144 cmdl[sa_ctx->cmdl_size / sizeof(u32)] = req->type;
1145
1146 /*
1147 * Map the packets, first we check if the data fits into a single
1148 * sg entry and use that if possible. If it does not fit, we check
1149 * if we need to do sg_split to align the scatterlist data on the
1150 * actual data size being processed by the crypto engine.
1151 */
1152 src = req->src;
1153 sg_nents = sg_nents_for_len(src, req->size);
1154
1155 split_size = req->size;
1156
1157 mapped_sg = &rxd->mapped_sg[0];
1158 if (sg_nents == 1 && split_size <= req->src->length) {
1159 src = &mapped_sg->static_sg;
1160 src_nents = 1;
1161 sg_init_table(src, 1);
1162 sg_set_page(src, sg_page(req->src), split_size,
1163 req->src->offset);
1164
1165 mapped_sg->sgt.sgl = src;
1166 mapped_sg->sgt.orig_nents = src_nents;
1167 ret = dma_map_sgtable(ddev, &mapped_sg->sgt, dir_src, 0);
1168 if (ret) {
1169 kfree(rxd);
1170 return ret;
1171 }
1172
1173 mapped_sg->dir = dir_src;
1174 mapped_sg->mapped = true;
1175 } else {
1176 mapped_sg->sgt.sgl = req->src;
1177 mapped_sg->sgt.orig_nents = sg_nents;
1178 ret = dma_map_sgtable(ddev, &mapped_sg->sgt, dir_src, 0);
1179 if (ret) {
1180 kfree(rxd);
1181 return ret;
1182 }
1183
1184 mapped_sg->dir = dir_src;
1185 mapped_sg->mapped = true;
1186
1187 ret = sg_split(mapped_sg->sgt.sgl, mapped_sg->sgt.nents, 0, 1,
1188 &split_size, &src, &src_nents, gfp_flags);
1189 if (ret) {
1190 src_nents = mapped_sg->sgt.nents;
1191 src = mapped_sg->sgt.sgl;
1192 } else {
1193 mapped_sg->split_sg = src;
1194 }
1195 }
1196
1197 dma_sync_sgtable_for_device(ddev, &mapped_sg->sgt, DMA_TO_DEVICE);
1198
1199 if (!diff_dst) {
1200 dst_nents = src_nents;
1201 dst = src;
1202 } else {
1203 dst_nents = sg_nents_for_len(req->dst, req->size);
1204 mapped_sg = &rxd->mapped_sg[1];
1205
1206 if (dst_nents == 1 && split_size <= req->dst->length) {
1207 dst = &mapped_sg->static_sg;
1208 dst_nents = 1;
1209 sg_init_table(dst, 1);
1210 sg_set_page(dst, sg_page(req->dst), split_size,
1211 req->dst->offset);
1212
1213 mapped_sg->sgt.sgl = dst;
1214 mapped_sg->sgt.orig_nents = dst_nents;
1215 ret = dma_map_sgtable(ddev, &mapped_sg->sgt,
1216 DMA_FROM_DEVICE, 0);
1217 if (ret)
1218 goto err_cleanup;
1219
1220 mapped_sg->dir = DMA_FROM_DEVICE;
1221 mapped_sg->mapped = true;
1222 } else {
1223 mapped_sg->sgt.sgl = req->dst;
1224 mapped_sg->sgt.orig_nents = dst_nents;
1225 ret = dma_map_sgtable(ddev, &mapped_sg->sgt,
1226 DMA_FROM_DEVICE, 0);
1227 if (ret)
1228 goto err_cleanup;
1229
1230 mapped_sg->dir = DMA_FROM_DEVICE;
1231 mapped_sg->mapped = true;
1232
1233 ret = sg_split(mapped_sg->sgt.sgl, mapped_sg->sgt.nents,
1234 0, 1, &split_size, &dst, &dst_nents,
1235 gfp_flags);
1236 if (ret) {
1237 dst_nents = mapped_sg->sgt.nents;
1238 dst = mapped_sg->sgt.sgl;
1239 } else {
1240 mapped_sg->split_sg = dst;
1241 }
1242 }
1243 }
1244
1245 rxd->tx_in = dmaengine_prep_slave_sg(dma_rx, dst, dst_nents,
1246 DMA_DEV_TO_MEM,
1247 DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
1248 if (!rxd->tx_in) {
1249 dev_err(pdata->dev, "IN prep_slave_sg() failed\n");
1250 ret = -EINVAL;
1251 goto err_cleanup;
1252 }
1253
1254 rxd->req = (void *)req->base;
1255 rxd->enc = req->enc;
1256 rxd->iv_idx = req->ctx->iv_idx;
1257 rxd->enc_iv_size = sa_ctx->cmdl_upd_info.enc_iv.size;
1258 rxd->tx_in->callback = req->callback;
1259 rxd->tx_in->callback_param = rxd;
1260
1261 tx_out = dmaengine_prep_slave_sg(pdata->dma_tx, src,
1262 src_nents, DMA_MEM_TO_DEV,
1263 DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
1264
1265 if (!tx_out) {
1266 dev_err(pdata->dev, "OUT prep_slave_sg() failed\n");
1267 ret = -EINVAL;
1268 goto err_cleanup;
1269 }
1270
1271 /*
1272 * Prepare metadata for DMA engine. This essentially describes the
1273 * crypto algorithm to be used, data sizes, different keys etc.
1274 */
1275 mdptr = (u32 *)dmaengine_desc_get_metadata_ptr(tx_out, &pl, &ml);
1276
1277 sa_prepare_tx_desc(mdptr, (sa_ctx->cmdl_size + (SA_PSDATA_CTX_WORDS *
1278 sizeof(u32))), cmdl, sizeof(sa_ctx->epib),
1279 sa_ctx->epib);
1280
1281 ml = sa_ctx->cmdl_size + (SA_PSDATA_CTX_WORDS * sizeof(u32));
1282 dmaengine_desc_set_metadata_len(tx_out, req->mdata_size);
1283
1284 dmaengine_submit(tx_out);
1285 dmaengine_submit(rxd->tx_in);
1286
1287 dma_async_issue_pending(dma_rx);
1288 dma_async_issue_pending(pdata->dma_tx);
1289
1290 return -EINPROGRESS;
1291
1292err_cleanup:
1293 sa_free_sa_rx_data(rxd);
1294
1295 return ret;
1296}
1297
1298static int sa_cipher_run(struct skcipher_request *req, u8 *iv, int enc)
1299{
1300 struct sa_tfm_ctx *ctx =
1301 crypto_skcipher_ctx(crypto_skcipher_reqtfm(req));
1302 struct crypto_alg *alg = req->base.tfm->__crt_alg;
1303 struct sa_req sa_req = { 0 };
1304
1305 if (!req->cryptlen)
1306 return 0;
1307
1308 if (req->cryptlen % alg->cra_blocksize)
1309 return -EINVAL;
1310
1311 /* Use SW fallback if the data size is not supported */
1312 if (req->cryptlen > SA_MAX_DATA_SZ ||
1313 (req->cryptlen >= SA_UNSAFE_DATA_SZ_MIN &&
1314 req->cryptlen <= SA_UNSAFE_DATA_SZ_MAX)) {
1315 struct skcipher_request *subreq = skcipher_request_ctx(req);
1316
1317 skcipher_request_set_tfm(subreq, ctx->fallback.skcipher);
1318 skcipher_request_set_callback(subreq, req->base.flags,
1319 req->base.complete,
1320 req->base.data);
1321 skcipher_request_set_crypt(subreq, req->src, req->dst,
1322 req->cryptlen, req->iv);
1323 if (enc)
1324 return crypto_skcipher_encrypt(subreq);
1325 else
1326 return crypto_skcipher_decrypt(subreq);
1327 }
1328
1329 sa_req.size = req->cryptlen;
1330 sa_req.enc_size = req->cryptlen;
1331 sa_req.src = req->src;
1332 sa_req.dst = req->dst;
1333 sa_req.enc_iv = iv;
1334 sa_req.type = CRYPTO_ALG_TYPE_SKCIPHER;
1335 sa_req.enc = enc;
1336 sa_req.callback = sa_aes_dma_in_callback;
1337 sa_req.mdata_size = 44;
1338 sa_req.base = &req->base;
1339 sa_req.ctx = ctx;
1340
1341 return sa_run(&sa_req);
1342}
1343
1344static int sa_encrypt(struct skcipher_request *req)
1345{
1346 return sa_cipher_run(req, req->iv, 1);
1347}
1348
1349static int sa_decrypt(struct skcipher_request *req)
1350{
1351 return sa_cipher_run(req, req->iv, 0);
1352}
1353
1354static void sa_sha_dma_in_callback(void *data)
1355{
1356 struct sa_rx_data *rxd = data;
1357 struct ahash_request *req;
1358 struct crypto_ahash *tfm;
1359 unsigned int authsize;
1360 int i;
1361 size_t ml, pl;
1362 u32 *result;
1363 __be32 *mdptr;
1364
1365 sa_sync_from_device(rxd);
1366 req = container_of(rxd->req, struct ahash_request, base);
1367 tfm = crypto_ahash_reqtfm(req);
1368 authsize = crypto_ahash_digestsize(tfm);
1369
1370 mdptr = (__be32 *)dmaengine_desc_get_metadata_ptr(rxd->tx_in, &pl, &ml);
1371 result = (u32 *)req->result;
1372
1373 for (i = 0; i < (authsize / 4); i++)
1374 result[i] = be32_to_cpu(mdptr[i + 4]);
1375
1376 sa_free_sa_rx_data(rxd);
1377
1378 ahash_request_complete(req, 0);
1379}
1380
1381static int zero_message_process(struct ahash_request *req)
1382{
1383 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
1384 int sa_digest_size = crypto_ahash_digestsize(tfm);
1385
1386 switch (sa_digest_size) {
1387 case SHA1_DIGEST_SIZE:
1388 memcpy(req->result, sha1_zero_message_hash, sa_digest_size);
1389 break;
1390 case SHA256_DIGEST_SIZE:
1391 memcpy(req->result, sha256_zero_message_hash, sa_digest_size);
1392 break;
1393 case SHA512_DIGEST_SIZE:
1394 memcpy(req->result, sha512_zero_message_hash, sa_digest_size);
1395 break;
1396 default:
1397 return -EINVAL;
1398 }
1399
1400 return 0;
1401}
1402
1403static int sa_sha_run(struct ahash_request *req)
1404{
1405 struct sa_tfm_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(req));
1406 struct sa_sha_req_ctx *rctx = ahash_request_ctx(req);
1407 struct sa_req sa_req = { 0 };
1408 size_t auth_len;
1409
1410 auth_len = req->nbytes;
1411
1412 if (!auth_len)
1413 return zero_message_process(req);
1414
1415 if (auth_len > SA_MAX_DATA_SZ ||
1416 (auth_len >= SA_UNSAFE_DATA_SZ_MIN &&
1417 auth_len <= SA_UNSAFE_DATA_SZ_MAX)) {
1418 struct ahash_request *subreq = &rctx->fallback_req;
1419 int ret;
1420
1421 ahash_request_set_tfm(subreq, ctx->fallback.ahash);
1422 ahash_request_set_callback(subreq, req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, NULL, NULL);
1423 ahash_request_set_crypt(subreq, req->src, req->result, auth_len);
1424
1425 ret = crypto_ahash_digest(subreq);
1426
1427 return ret;
1428 }
1429
1430 sa_req.size = auth_len;
1431 sa_req.auth_size = auth_len;
1432 sa_req.src = req->src;
1433 sa_req.dst = req->src;
1434 sa_req.enc = true;
1435 sa_req.type = CRYPTO_ALG_TYPE_AHASH;
1436 sa_req.callback = sa_sha_dma_in_callback;
1437 sa_req.mdata_size = 28;
1438 sa_req.ctx = ctx;
1439 sa_req.base = &req->base;
1440
1441 return sa_run(&sa_req);
1442}
1443
1444static int sa_sha_setup(struct sa_tfm_ctx *ctx, struct algo_data *ad)
1445{
1446 int bs = crypto_shash_blocksize(ctx->shash);
1447 int cmdl_len;
1448 struct sa_cmdl_cfg cfg;
1449
1450 ad->enc_eng.sc_size = SA_CTX_ENC_TYPE1_SZ;
1451 ad->auth_eng.eng_id = SA_ENG_ID_AM1;
1452 ad->auth_eng.sc_size = SA_CTX_AUTH_TYPE2_SZ;
1453
1454 memset(ctx->authkey, 0, bs);
1455 memset(&cfg, 0, sizeof(cfg));
1456 cfg.aalg = ad->aalg_id;
1457 cfg.enc_eng_id = ad->enc_eng.eng_id;
1458 cfg.auth_eng_id = ad->auth_eng.eng_id;
1459 cfg.iv_size = 0;
1460 cfg.akey = NULL;
1461 cfg.akey_len = 0;
1462
1463 ctx->dev_data = dev_get_drvdata(sa_k3_dev);
1464 /* Setup Encryption Security Context & Command label template */
1465 if (sa_init_sc(&ctx->enc, ctx->dev_data->match_data, NULL, 0, NULL, 0,
1466 ad, 0, &ctx->enc.epib[1]))
1467 goto badkey;
1468
1469 cmdl_len = sa_format_cmdl_gen(&cfg,
1470 (u8 *)ctx->enc.cmdl,
1471 &ctx->enc.cmdl_upd_info);
1472 if (cmdl_len <= 0 || (cmdl_len > SA_MAX_CMDL_WORDS * sizeof(u32)))
1473 goto badkey;
1474
1475 ctx->enc.cmdl_size = cmdl_len;
1476
1477 return 0;
1478
1479badkey:
1480 dev_err(sa_k3_dev, "%s: badkey\n", __func__);
1481 return -EINVAL;
1482}
1483
1484static int sa_sha_cra_init_alg(struct crypto_tfm *tfm, const char *alg_base)
1485{
1486 struct sa_tfm_ctx *ctx = crypto_tfm_ctx(tfm);
1487 struct sa_crypto_data *data = dev_get_drvdata(sa_k3_dev);
1488 int ret;
1489
1490 memset(ctx, 0, sizeof(*ctx));
1491 ctx->dev_data = data;
1492 ret = sa_init_ctx_info(&ctx->enc, data);
1493 if (ret)
1494 return ret;
1495
1496 if (alg_base) {
1497 ctx->shash = crypto_alloc_shash(alg_base, 0, 0);
1498 if (IS_ERR(ctx->shash)) {
1499 dev_err(sa_k3_dev, "base driver %s couldn't be loaded\n",
1500 alg_base);
1501 return PTR_ERR(ctx->shash);
1502 }
1503 /* for fallback */
1504 ctx->fallback.ahash =
1505 crypto_alloc_ahash(alg_base, 0, CRYPTO_ALG_ASYNC);
1506 if (IS_ERR(ctx->fallback.ahash)) {
1507 dev_err(ctx->dev_data->dev,
1508 "Could not load fallback driver\n");
1509 return PTR_ERR(ctx->fallback.ahash);
1510 }
1511 }
1512
1513 dev_dbg(sa_k3_dev, "%s(0x%p) sc-ids(0x%x(0x%pad), 0x%x(0x%pad))\n",
1514 __func__, tfm, ctx->enc.sc_id, &ctx->enc.sc_phys,
1515 ctx->dec.sc_id, &ctx->dec.sc_phys);
1516
1517 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
1518 sizeof(struct sa_sha_req_ctx) +
1519 crypto_ahash_reqsize(ctx->fallback.ahash));
1520
1521 return 0;
1522}
1523
1524static int sa_sha_digest(struct ahash_request *req)
1525{
1526 return sa_sha_run(req);
1527}
1528
1529static int sa_sha_init(struct ahash_request *req)
1530{
1531 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
1532 struct sa_sha_req_ctx *rctx = ahash_request_ctx(req);
1533 struct sa_tfm_ctx *ctx = crypto_ahash_ctx(tfm);
1534
1535 dev_dbg(sa_k3_dev, "init: digest size: %u, rctx=%p\n",
1536 crypto_ahash_digestsize(tfm), rctx);
1537
1538 ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback.ahash);
1539 ahash_request_set_callback(&rctx->fallback_req, req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, NULL, NULL);
1540 ahash_request_set_crypt(&rctx->fallback_req, NULL, NULL, 0);
1541
1542 return crypto_ahash_init(&rctx->fallback_req);
1543}
1544
1545static int sa_sha_update(struct ahash_request *req)
1546{
1547 struct sa_sha_req_ctx *rctx = ahash_request_ctx(req);
1548
1549 ahash_request_set_callback(&rctx->fallback_req, req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, NULL, NULL);
1550 ahash_request_set_crypt(&rctx->fallback_req, req->src, NULL, req->nbytes);
1551
1552 return crypto_ahash_update(&rctx->fallback_req);
1553}
1554
1555static int sa_sha_final(struct ahash_request *req)
1556{
1557 struct sa_sha_req_ctx *rctx = ahash_request_ctx(req);
1558
1559 ahash_request_set_callback(&rctx->fallback_req, req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, NULL, NULL);
1560 ahash_request_set_crypt(&rctx->fallback_req, NULL, req->result, 0);
1561
1562 return crypto_ahash_final(&rctx->fallback_req);
1563}
1564
1565static int sa_sha_finup(struct ahash_request *req)
1566{
1567 struct sa_sha_req_ctx *rctx = ahash_request_ctx(req);
1568
1569 ahash_request_set_callback(&rctx->fallback_req, req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, NULL, NULL);
1570 ahash_request_set_crypt(&rctx->fallback_req, req->src, req->result, req->nbytes);
1571
1572 return crypto_ahash_finup(&rctx->fallback_req);
1573}
1574
1575static int sa_sha_import(struct ahash_request *req, const void *in)
1576{
1577 struct sa_sha_req_ctx *rctx = ahash_request_ctx(req);
1578 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
1579 struct sa_tfm_ctx *ctx = crypto_ahash_ctx(tfm);
1580
1581 ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback.ahash);
1582 ahash_request_set_callback(&rctx->fallback_req, req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, NULL, NULL);
1583
1584 return crypto_ahash_import(&rctx->fallback_req, in);
1585}
1586
1587static int sa_sha_export(struct ahash_request *req, void *out)
1588{
1589 struct sa_sha_req_ctx *rctx = ahash_request_ctx(req);
1590 struct ahash_request *subreq = &rctx->fallback_req;
1591
1592 ahash_request_set_callback(subreq, req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, NULL, NULL);
1593
1594 return crypto_ahash_export(subreq, out);
1595}
1596
1597static int sa_sha1_cra_init(struct crypto_tfm *tfm)
1598{
1599 struct algo_data ad = { 0 };
1600 struct sa_tfm_ctx *ctx = crypto_tfm_ctx(tfm);
1601
1602 sa_sha_cra_init_alg(tfm, "sha1");
1603
1604 ad.aalg_id = SA_AALG_ID_SHA1;
1605 ad.hash_size = SHA1_DIGEST_SIZE;
1606 ad.auth_ctrl = SA_AUTH_SW_CTRL_SHA1;
1607
1608 sa_sha_setup(ctx, &ad);
1609
1610 return 0;
1611}
1612
1613static int sa_sha256_cra_init(struct crypto_tfm *tfm)
1614{
1615 struct algo_data ad = { 0 };
1616 struct sa_tfm_ctx *ctx = crypto_tfm_ctx(tfm);
1617
1618 sa_sha_cra_init_alg(tfm, "sha256");
1619
1620 ad.aalg_id = SA_AALG_ID_SHA2_256;
1621 ad.hash_size = SHA256_DIGEST_SIZE;
1622 ad.auth_ctrl = SA_AUTH_SW_CTRL_SHA256;
1623
1624 sa_sha_setup(ctx, &ad);
1625
1626 return 0;
1627}
1628
1629static int sa_sha512_cra_init(struct crypto_tfm *tfm)
1630{
1631 struct algo_data ad = { 0 };
1632 struct sa_tfm_ctx *ctx = crypto_tfm_ctx(tfm);
1633
1634 sa_sha_cra_init_alg(tfm, "sha512");
1635
1636 ad.aalg_id = SA_AALG_ID_SHA2_512;
1637 ad.hash_size = SHA512_DIGEST_SIZE;
1638 ad.auth_ctrl = SA_AUTH_SW_CTRL_SHA512;
1639
1640 sa_sha_setup(ctx, &ad);
1641
1642 return 0;
1643}
1644
1645static void sa_sha_cra_exit(struct crypto_tfm *tfm)
1646{
1647 struct sa_tfm_ctx *ctx = crypto_tfm_ctx(tfm);
1648 struct sa_crypto_data *data = dev_get_drvdata(sa_k3_dev);
1649
1650 dev_dbg(sa_k3_dev, "%s(0x%p) sc-ids(0x%x(0x%pad), 0x%x(0x%pad))\n",
1651 __func__, tfm, ctx->enc.sc_id, &ctx->enc.sc_phys,
1652 ctx->dec.sc_id, &ctx->dec.sc_phys);
1653
1654 if (crypto_tfm_alg_type(tfm) == CRYPTO_ALG_TYPE_AHASH)
1655 sa_free_ctx_info(&ctx->enc, data);
1656
1657 crypto_free_shash(ctx->shash);
1658 crypto_free_ahash(ctx->fallback.ahash);
1659}
1660
1661static void sa_aead_dma_in_callback(void *data)
1662{
1663 struct sa_rx_data *rxd = data;
1664 struct aead_request *req;
1665 struct crypto_aead *tfm;
1666 unsigned int start;
1667 unsigned int authsize;
1668 u8 auth_tag[SA_MAX_AUTH_TAG_SZ];
1669 size_t pl, ml;
1670 int i;
1671 int err = 0;
1672 u32 *mdptr;
1673
1674 sa_sync_from_device(rxd);
1675 req = container_of(rxd->req, struct aead_request, base);
1676 tfm = crypto_aead_reqtfm(req);
1677 start = req->assoclen + req->cryptlen;
1678 authsize = crypto_aead_authsize(tfm);
1679
1680 mdptr = (u32 *)dmaengine_desc_get_metadata_ptr(rxd->tx_in, &pl, &ml);
1681 for (i = 0; i < (authsize / 4); i++)
1682 mdptr[i + 4] = swab32(mdptr[i + 4]);
1683
1684 if (rxd->enc) {
1685 scatterwalk_map_and_copy(&mdptr[4], req->dst, start, authsize,
1686 1);
1687 } else {
1688 start -= authsize;
1689 scatterwalk_map_and_copy(auth_tag, req->src, start, authsize,
1690 0);
1691
1692 err = crypto_memneq(&mdptr[4], auth_tag, authsize) ? -EBADMSG : 0;
1693 }
1694
1695 sa_free_sa_rx_data(rxd);
1696
1697 aead_request_complete(req, err);
1698}
1699
1700static int sa_cra_init_aead(struct crypto_aead *tfm, const char *hash,
1701 const char *fallback)
1702{
1703 struct sa_tfm_ctx *ctx = crypto_aead_ctx(tfm);
1704 struct sa_crypto_data *data = dev_get_drvdata(sa_k3_dev);
1705 int ret;
1706
1707 memzero_explicit(ctx, sizeof(*ctx));
1708 ctx->dev_data = data;
1709
1710 ctx->shash = crypto_alloc_shash(hash, 0, CRYPTO_ALG_NEED_FALLBACK);
1711 if (IS_ERR(ctx->shash)) {
1712 dev_err(sa_k3_dev, "base driver %s couldn't be loaded\n", hash);
1713 return PTR_ERR(ctx->shash);
1714 }
1715
1716 ctx->fallback.aead = crypto_alloc_aead(fallback, 0,
1717 CRYPTO_ALG_NEED_FALLBACK);
1718
1719 if (IS_ERR(ctx->fallback.aead)) {
1720 dev_err(sa_k3_dev, "fallback driver %s couldn't be loaded\n",
1721 fallback);
1722 return PTR_ERR(ctx->fallback.aead);
1723 }
1724
1725 crypto_aead_set_reqsize(tfm, sizeof(struct aead_request) +
1726 crypto_aead_reqsize(ctx->fallback.aead));
1727
1728 ret = sa_init_ctx_info(&ctx->enc, data);
1729 if (ret)
1730 return ret;
1731
1732 ret = sa_init_ctx_info(&ctx->dec, data);
1733 if (ret) {
1734 sa_free_ctx_info(&ctx->enc, data);
1735 return ret;
1736 }
1737
1738 dev_dbg(sa_k3_dev, "%s(0x%p) sc-ids(0x%x(0x%pad), 0x%x(0x%pad))\n",
1739 __func__, tfm, ctx->enc.sc_id, &ctx->enc.sc_phys,
1740 ctx->dec.sc_id, &ctx->dec.sc_phys);
1741
1742 return ret;
1743}
1744
1745static int sa_cra_init_aead_sha1(struct crypto_aead *tfm)
1746{
1747 return sa_cra_init_aead(tfm, "sha1",
1748 "authenc(hmac(sha1),cbc(aes))");
1749}
1750
1751static int sa_cra_init_aead_sha256(struct crypto_aead *tfm)
1752{
1753 return sa_cra_init_aead(tfm, "sha256",
1754 "authenc(hmac(sha256),cbc(aes))");
1755}
1756
1757static void sa_exit_tfm_aead(struct crypto_aead *tfm)
1758{
1759 struct sa_tfm_ctx *ctx = crypto_aead_ctx(tfm);
1760 struct sa_crypto_data *data = dev_get_drvdata(sa_k3_dev);
1761
1762 crypto_free_shash(ctx->shash);
1763 crypto_free_aead(ctx->fallback.aead);
1764
1765 sa_free_ctx_info(&ctx->enc, data);
1766 sa_free_ctx_info(&ctx->dec, data);
1767}
1768
1769/* AEAD algorithm configuration interface function */
1770static int sa_aead_setkey(struct crypto_aead *authenc,
1771 const u8 *key, unsigned int keylen,
1772 struct algo_data *ad)
1773{
1774 struct sa_tfm_ctx *ctx = crypto_aead_ctx(authenc);
1775 struct crypto_authenc_keys keys;
1776 int cmdl_len;
1777 struct sa_cmdl_cfg cfg;
1778 int key_idx;
1779
1780 if (crypto_authenc_extractkeys(&keys, key, keylen) != 0)
1781 return -EINVAL;
1782
1783 /* Convert the key size (16/24/32) to the key size index (0/1/2) */
1784 key_idx = (keys.enckeylen >> 3) - 2;
1785 if (key_idx >= 3)
1786 return -EINVAL;
1787
1788 ad->ctx = ctx;
1789 ad->enc_eng.eng_id = SA_ENG_ID_EM1;
1790 ad->enc_eng.sc_size = SA_CTX_ENC_TYPE1_SZ;
1791 ad->auth_eng.eng_id = SA_ENG_ID_AM1;
1792 ad->auth_eng.sc_size = SA_CTX_AUTH_TYPE2_SZ;
1793 ad->mci_enc = mci_cbc_enc_no_iv_array[key_idx];
1794 ad->mci_dec = mci_cbc_dec_no_iv_array[key_idx];
1795 ad->inv_key = true;
1796 ad->keyed_mac = true;
1797 ad->ealg_id = SA_EALG_ID_AES_CBC;
1798 ad->prep_iopad = sa_prepare_iopads;
1799
1800 memset(&cfg, 0, sizeof(cfg));
1801 cfg.enc = true;
1802 cfg.aalg = ad->aalg_id;
1803 cfg.enc_eng_id = ad->enc_eng.eng_id;
1804 cfg.auth_eng_id = ad->auth_eng.eng_id;
1805 cfg.iv_size = crypto_aead_ivsize(authenc);
1806 cfg.akey = keys.authkey;
1807 cfg.akey_len = keys.authkeylen;
1808
1809 /* Setup Encryption Security Context & Command label template */
1810 if (sa_init_sc(&ctx->enc, ctx->dev_data->match_data, keys.enckey,
1811 keys.enckeylen, keys.authkey, keys.authkeylen,
1812 ad, 1, &ctx->enc.epib[1]))
1813 return -EINVAL;
1814
1815 cmdl_len = sa_format_cmdl_gen(&cfg,
1816 (u8 *)ctx->enc.cmdl,
1817 &ctx->enc.cmdl_upd_info);
1818 if (cmdl_len <= 0 || (cmdl_len > SA_MAX_CMDL_WORDS * sizeof(u32)))
1819 return -EINVAL;
1820
1821 ctx->enc.cmdl_size = cmdl_len;
1822
1823 /* Setup Decryption Security Context & Command label template */
1824 if (sa_init_sc(&ctx->dec, ctx->dev_data->match_data, keys.enckey,
1825 keys.enckeylen, keys.authkey, keys.authkeylen,
1826 ad, 0, &ctx->dec.epib[1]))
1827 return -EINVAL;
1828
1829 cfg.enc = false;
1830 cmdl_len = sa_format_cmdl_gen(&cfg, (u8 *)ctx->dec.cmdl,
1831 &ctx->dec.cmdl_upd_info);
1832
1833 if (cmdl_len <= 0 || (cmdl_len > SA_MAX_CMDL_WORDS * sizeof(u32)))
1834 return -EINVAL;
1835
1836 ctx->dec.cmdl_size = cmdl_len;
1837
1838 crypto_aead_clear_flags(ctx->fallback.aead, CRYPTO_TFM_REQ_MASK);
1839 crypto_aead_set_flags(ctx->fallback.aead,
1840 crypto_aead_get_flags(authenc) &
1841 CRYPTO_TFM_REQ_MASK);
1842
1843 return crypto_aead_setkey(ctx->fallback.aead, key, keylen);
1844}
1845
1846static int sa_aead_setauthsize(struct crypto_aead *tfm, unsigned int authsize)
1847{
1848 struct sa_tfm_ctx *ctx = crypto_tfm_ctx(crypto_aead_tfm(tfm));
1849
1850 return crypto_aead_setauthsize(ctx->fallback.aead, authsize);
1851}
1852
1853static int sa_aead_cbc_sha1_setkey(struct crypto_aead *authenc,
1854 const u8 *key, unsigned int keylen)
1855{
1856 struct algo_data ad = { 0 };
1857
1858 ad.ealg_id = SA_EALG_ID_AES_CBC;
1859 ad.aalg_id = SA_AALG_ID_HMAC_SHA1;
1860 ad.hash_size = SHA1_DIGEST_SIZE;
1861 ad.auth_ctrl = SA_AUTH_SW_CTRL_SHA1;
1862
1863 return sa_aead_setkey(authenc, key, keylen, &ad);
1864}
1865
1866static int sa_aead_cbc_sha256_setkey(struct crypto_aead *authenc,
1867 const u8 *key, unsigned int keylen)
1868{
1869 struct algo_data ad = { 0 };
1870
1871 ad.ealg_id = SA_EALG_ID_AES_CBC;
1872 ad.aalg_id = SA_AALG_ID_HMAC_SHA2_256;
1873 ad.hash_size = SHA256_DIGEST_SIZE;
1874 ad.auth_ctrl = SA_AUTH_SW_CTRL_SHA256;
1875
1876 return sa_aead_setkey(authenc, key, keylen, &ad);
1877}
1878
1879static int sa_aead_run(struct aead_request *req, u8 *iv, int enc)
1880{
1881 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1882 struct sa_tfm_ctx *ctx = crypto_aead_ctx(tfm);
1883 struct sa_req sa_req = { 0 };
1884 size_t auth_size, enc_size;
1885
1886 enc_size = req->cryptlen;
1887 auth_size = req->assoclen + req->cryptlen;
1888
1889 if (!enc) {
1890 enc_size -= crypto_aead_authsize(tfm);
1891 auth_size -= crypto_aead_authsize(tfm);
1892 }
1893
1894 if (auth_size > SA_MAX_DATA_SZ ||
1895 (auth_size >= SA_UNSAFE_DATA_SZ_MIN &&
1896 auth_size <= SA_UNSAFE_DATA_SZ_MAX)) {
1897 struct aead_request *subreq = aead_request_ctx(req);
1898 int ret;
1899
1900 aead_request_set_tfm(subreq, ctx->fallback.aead);
1901 aead_request_set_callback(subreq, req->base.flags,
1902 req->base.complete, req->base.data);
1903 aead_request_set_crypt(subreq, req->src, req->dst,
1904 req->cryptlen, req->iv);
1905 aead_request_set_ad(subreq, req->assoclen);
1906
1907 ret = enc ? crypto_aead_encrypt(subreq) :
1908 crypto_aead_decrypt(subreq);
1909 return ret;
1910 }
1911
1912 sa_req.enc_offset = req->assoclen;
1913 sa_req.enc_size = enc_size;
1914 sa_req.auth_size = auth_size;
1915 sa_req.size = auth_size;
1916 sa_req.enc_iv = iv;
1917 sa_req.type = CRYPTO_ALG_TYPE_AEAD;
1918 sa_req.enc = enc;
1919 sa_req.callback = sa_aead_dma_in_callback;
1920 sa_req.mdata_size = 52;
1921 sa_req.base = &req->base;
1922 sa_req.ctx = ctx;
1923 sa_req.src = req->src;
1924 sa_req.dst = req->dst;
1925
1926 return sa_run(&sa_req);
1927}
1928
1929/* AEAD algorithm encrypt interface function */
1930static int sa_aead_encrypt(struct aead_request *req)
1931{
1932 return sa_aead_run(req, req->iv, 1);
1933}
1934
1935/* AEAD algorithm decrypt interface function */
1936static int sa_aead_decrypt(struct aead_request *req)
1937{
1938 return sa_aead_run(req, req->iv, 0);
1939}
1940
1941static struct sa_alg_tmpl sa_algs[] = {
1942 [SA_ALG_CBC_AES] = {
1943 .type = CRYPTO_ALG_TYPE_SKCIPHER,
1944 .alg.skcipher = {
1945 .base.cra_name = "cbc(aes)",
1946 .base.cra_driver_name = "cbc-aes-sa2ul",
1947 .base.cra_priority = 30000,
1948 .base.cra_flags = CRYPTO_ALG_TYPE_SKCIPHER |
1949 CRYPTO_ALG_KERN_DRIVER_ONLY |
1950 CRYPTO_ALG_ASYNC |
1951 CRYPTO_ALG_NEED_FALLBACK,
1952 .base.cra_blocksize = AES_BLOCK_SIZE,
1953 .base.cra_ctxsize = sizeof(struct sa_tfm_ctx),
1954 .base.cra_module = THIS_MODULE,
1955 .init = sa_cipher_cra_init,
1956 .exit = sa_cipher_cra_exit,
1957 .min_keysize = AES_MIN_KEY_SIZE,
1958 .max_keysize = AES_MAX_KEY_SIZE,
1959 .ivsize = AES_BLOCK_SIZE,
1960 .setkey = sa_aes_cbc_setkey,
1961 .encrypt = sa_encrypt,
1962 .decrypt = sa_decrypt,
1963 }
1964 },
1965 [SA_ALG_EBC_AES] = {
1966 .type = CRYPTO_ALG_TYPE_SKCIPHER,
1967 .alg.skcipher = {
1968 .base.cra_name = "ecb(aes)",
1969 .base.cra_driver_name = "ecb-aes-sa2ul",
1970 .base.cra_priority = 30000,
1971 .base.cra_flags = CRYPTO_ALG_TYPE_SKCIPHER |
1972 CRYPTO_ALG_KERN_DRIVER_ONLY |
1973 CRYPTO_ALG_ASYNC |
1974 CRYPTO_ALG_NEED_FALLBACK,
1975 .base.cra_blocksize = AES_BLOCK_SIZE,
1976 .base.cra_ctxsize = sizeof(struct sa_tfm_ctx),
1977 .base.cra_module = THIS_MODULE,
1978 .init = sa_cipher_cra_init,
1979 .exit = sa_cipher_cra_exit,
1980 .min_keysize = AES_MIN_KEY_SIZE,
1981 .max_keysize = AES_MAX_KEY_SIZE,
1982 .setkey = sa_aes_ecb_setkey,
1983 .encrypt = sa_encrypt,
1984 .decrypt = sa_decrypt,
1985 }
1986 },
1987 [SA_ALG_CBC_DES3] = {
1988 .type = CRYPTO_ALG_TYPE_SKCIPHER,
1989 .alg.skcipher = {
1990 .base.cra_name = "cbc(des3_ede)",
1991 .base.cra_driver_name = "cbc-des3-sa2ul",
1992 .base.cra_priority = 30000,
1993 .base.cra_flags = CRYPTO_ALG_TYPE_SKCIPHER |
1994 CRYPTO_ALG_KERN_DRIVER_ONLY |
1995 CRYPTO_ALG_ASYNC |
1996 CRYPTO_ALG_NEED_FALLBACK,
1997 .base.cra_blocksize = DES_BLOCK_SIZE,
1998 .base.cra_ctxsize = sizeof(struct sa_tfm_ctx),
1999 .base.cra_module = THIS_MODULE,
2000 .init = sa_cipher_cra_init,
2001 .exit = sa_cipher_cra_exit,
2002 .min_keysize = 3 * DES_KEY_SIZE,
2003 .max_keysize = 3 * DES_KEY_SIZE,
2004 .ivsize = DES_BLOCK_SIZE,
2005 .setkey = sa_3des_cbc_setkey,
2006 .encrypt = sa_encrypt,
2007 .decrypt = sa_decrypt,
2008 }
2009 },
2010 [SA_ALG_ECB_DES3] = {
2011 .type = CRYPTO_ALG_TYPE_SKCIPHER,
2012 .alg.skcipher = {
2013 .base.cra_name = "ecb(des3_ede)",
2014 .base.cra_driver_name = "ecb-des3-sa2ul",
2015 .base.cra_priority = 30000,
2016 .base.cra_flags = CRYPTO_ALG_TYPE_SKCIPHER |
2017 CRYPTO_ALG_KERN_DRIVER_ONLY |
2018 CRYPTO_ALG_ASYNC |
2019 CRYPTO_ALG_NEED_FALLBACK,
2020 .base.cra_blocksize = DES_BLOCK_SIZE,
2021 .base.cra_ctxsize = sizeof(struct sa_tfm_ctx),
2022 .base.cra_module = THIS_MODULE,
2023 .init = sa_cipher_cra_init,
2024 .exit = sa_cipher_cra_exit,
2025 .min_keysize = 3 * DES_KEY_SIZE,
2026 .max_keysize = 3 * DES_KEY_SIZE,
2027 .setkey = sa_3des_ecb_setkey,
2028 .encrypt = sa_encrypt,
2029 .decrypt = sa_decrypt,
2030 }
2031 },
2032 [SA_ALG_SHA1] = {
2033 .type = CRYPTO_ALG_TYPE_AHASH,
2034 .alg.ahash = {
2035 .halg.base = {
2036 .cra_name = "sha1",
2037 .cra_driver_name = "sha1-sa2ul",
2038 .cra_priority = 400,
2039 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2040 CRYPTO_ALG_ASYNC |
2041 CRYPTO_ALG_KERN_DRIVER_ONLY |
2042 CRYPTO_ALG_NEED_FALLBACK,
2043 .cra_blocksize = SHA1_BLOCK_SIZE,
2044 .cra_ctxsize = sizeof(struct sa_tfm_ctx),
2045 .cra_module = THIS_MODULE,
2046 .cra_init = sa_sha1_cra_init,
2047 .cra_exit = sa_sha_cra_exit,
2048 },
2049 .halg.digestsize = SHA1_DIGEST_SIZE,
2050 .halg.statesize = sizeof(struct sa_sha_req_ctx) +
2051 sizeof(struct sha1_state),
2052 .init = sa_sha_init,
2053 .update = sa_sha_update,
2054 .final = sa_sha_final,
2055 .finup = sa_sha_finup,
2056 .digest = sa_sha_digest,
2057 .export = sa_sha_export,
2058 .import = sa_sha_import,
2059 },
2060 },
2061 [SA_ALG_SHA256] = {
2062 .type = CRYPTO_ALG_TYPE_AHASH,
2063 .alg.ahash = {
2064 .halg.base = {
2065 .cra_name = "sha256",
2066 .cra_driver_name = "sha256-sa2ul",
2067 .cra_priority = 400,
2068 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2069 CRYPTO_ALG_ASYNC |
2070 CRYPTO_ALG_KERN_DRIVER_ONLY |
2071 CRYPTO_ALG_NEED_FALLBACK,
2072 .cra_blocksize = SHA256_BLOCK_SIZE,
2073 .cra_ctxsize = sizeof(struct sa_tfm_ctx),
2074 .cra_module = THIS_MODULE,
2075 .cra_init = sa_sha256_cra_init,
2076 .cra_exit = sa_sha_cra_exit,
2077 },
2078 .halg.digestsize = SHA256_DIGEST_SIZE,
2079 .halg.statesize = sizeof(struct sa_sha_req_ctx) +
2080 sizeof(struct sha256_state),
2081 .init = sa_sha_init,
2082 .update = sa_sha_update,
2083 .final = sa_sha_final,
2084 .finup = sa_sha_finup,
2085 .digest = sa_sha_digest,
2086 .export = sa_sha_export,
2087 .import = sa_sha_import,
2088 },
2089 },
2090 [SA_ALG_SHA512] = {
2091 .type = CRYPTO_ALG_TYPE_AHASH,
2092 .alg.ahash = {
2093 .halg.base = {
2094 .cra_name = "sha512",
2095 .cra_driver_name = "sha512-sa2ul",
2096 .cra_priority = 400,
2097 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2098 CRYPTO_ALG_ASYNC |
2099 CRYPTO_ALG_KERN_DRIVER_ONLY |
2100 CRYPTO_ALG_NEED_FALLBACK,
2101 .cra_blocksize = SHA512_BLOCK_SIZE,
2102 .cra_ctxsize = sizeof(struct sa_tfm_ctx),
2103 .cra_module = THIS_MODULE,
2104 .cra_init = sa_sha512_cra_init,
2105 .cra_exit = sa_sha_cra_exit,
2106 },
2107 .halg.digestsize = SHA512_DIGEST_SIZE,
2108 .halg.statesize = sizeof(struct sa_sha_req_ctx) +
2109 sizeof(struct sha512_state),
2110 .init = sa_sha_init,
2111 .update = sa_sha_update,
2112 .final = sa_sha_final,
2113 .finup = sa_sha_finup,
2114 .digest = sa_sha_digest,
2115 .export = sa_sha_export,
2116 .import = sa_sha_import,
2117 },
2118 },
2119 [SA_ALG_AUTHENC_SHA1_AES] = {
2120 .type = CRYPTO_ALG_TYPE_AEAD,
2121 .alg.aead = {
2122 .base = {
2123 .cra_name = "authenc(hmac(sha1),cbc(aes))",
2124 .cra_driver_name =
2125 "authenc(hmac(sha1),cbc(aes))-sa2ul",
2126 .cra_blocksize = AES_BLOCK_SIZE,
2127 .cra_flags = CRYPTO_ALG_TYPE_AEAD |
2128 CRYPTO_ALG_KERN_DRIVER_ONLY |
2129 CRYPTO_ALG_ASYNC |
2130 CRYPTO_ALG_NEED_FALLBACK,
2131 .cra_ctxsize = sizeof(struct sa_tfm_ctx),
2132 .cra_module = THIS_MODULE,
2133 .cra_priority = 3000,
2134 },
2135 .ivsize = AES_BLOCK_SIZE,
2136 .maxauthsize = SHA1_DIGEST_SIZE,
2137
2138 .init = sa_cra_init_aead_sha1,
2139 .exit = sa_exit_tfm_aead,
2140 .setkey = sa_aead_cbc_sha1_setkey,
2141 .setauthsize = sa_aead_setauthsize,
2142 .encrypt = sa_aead_encrypt,
2143 .decrypt = sa_aead_decrypt,
2144 },
2145 },
2146 [SA_ALG_AUTHENC_SHA256_AES] = {
2147 .type = CRYPTO_ALG_TYPE_AEAD,
2148 .alg.aead = {
2149 .base = {
2150 .cra_name = "authenc(hmac(sha256),cbc(aes))",
2151 .cra_driver_name =
2152 "authenc(hmac(sha256),cbc(aes))-sa2ul",
2153 .cra_blocksize = AES_BLOCK_SIZE,
2154 .cra_flags = CRYPTO_ALG_TYPE_AEAD |
2155 CRYPTO_ALG_KERN_DRIVER_ONLY |
2156 CRYPTO_ALG_ASYNC |
2157 CRYPTO_ALG_NEED_FALLBACK,
2158 .cra_ctxsize = sizeof(struct sa_tfm_ctx),
2159 .cra_module = THIS_MODULE,
2160 .cra_alignmask = 0,
2161 .cra_priority = 3000,
2162 },
2163 .ivsize = AES_BLOCK_SIZE,
2164 .maxauthsize = SHA256_DIGEST_SIZE,
2165
2166 .init = sa_cra_init_aead_sha256,
2167 .exit = sa_exit_tfm_aead,
2168 .setkey = sa_aead_cbc_sha256_setkey,
2169 .setauthsize = sa_aead_setauthsize,
2170 .encrypt = sa_aead_encrypt,
2171 .decrypt = sa_aead_decrypt,
2172 },
2173 },
2174};
2175
2176/* Register the algorithms in crypto framework */
2177static void sa_register_algos(struct sa_crypto_data *dev_data)
2178{
2179 const struct sa_match_data *match_data = dev_data->match_data;
2180 struct device *dev = dev_data->dev;
2181 char *alg_name;
2182 u32 type;
2183 int i, err;
2184
2185 for (i = 0; i < ARRAY_SIZE(sa_algs); i++) {
2186 /* Skip unsupported algos */
2187 if (!(match_data->supported_algos & BIT(i)))
2188 continue;
2189
2190 type = sa_algs[i].type;
2191 if (type == CRYPTO_ALG_TYPE_SKCIPHER) {
2192 alg_name = sa_algs[i].alg.skcipher.base.cra_name;
2193 err = crypto_register_skcipher(&sa_algs[i].alg.skcipher);
2194 } else if (type == CRYPTO_ALG_TYPE_AHASH) {
2195 alg_name = sa_algs[i].alg.ahash.halg.base.cra_name;
2196 err = crypto_register_ahash(&sa_algs[i].alg.ahash);
2197 } else if (type == CRYPTO_ALG_TYPE_AEAD) {
2198 alg_name = sa_algs[i].alg.aead.base.cra_name;
2199 err = crypto_register_aead(&sa_algs[i].alg.aead);
2200 } else {
2201 dev_err(dev,
2202 "un-supported crypto algorithm (%d)",
2203 sa_algs[i].type);
2204 continue;
2205 }
2206
2207 if (err)
2208 dev_err(dev, "Failed to register '%s'\n", alg_name);
2209 else
2210 sa_algs[i].registered = true;
2211 }
2212}
2213
2214/* Unregister the algorithms in crypto framework */
2215static void sa_unregister_algos(const struct device *dev)
2216{
2217 u32 type;
2218 int i;
2219
2220 for (i = 0; i < ARRAY_SIZE(sa_algs); i++) {
2221 type = sa_algs[i].type;
2222 if (!sa_algs[i].registered)
2223 continue;
2224 if (type == CRYPTO_ALG_TYPE_SKCIPHER)
2225 crypto_unregister_skcipher(&sa_algs[i].alg.skcipher);
2226 else if (type == CRYPTO_ALG_TYPE_AHASH)
2227 crypto_unregister_ahash(&sa_algs[i].alg.ahash);
2228 else if (type == CRYPTO_ALG_TYPE_AEAD)
2229 crypto_unregister_aead(&sa_algs[i].alg.aead);
2230
2231 sa_algs[i].registered = false;
2232 }
2233}
2234
2235static int sa_init_mem(struct sa_crypto_data *dev_data)
2236{
2237 struct device *dev = &dev_data->pdev->dev;
2238 /* Setup dma pool for security context buffers */
2239 dev_data->sc_pool = dma_pool_create("keystone-sc", dev,
2240 SA_CTX_MAX_SZ, 64, 0);
2241 if (!dev_data->sc_pool) {
2242 dev_err(dev, "Failed to create dma pool");
2243 return -ENOMEM;
2244 }
2245
2246 return 0;
2247}
2248
2249static int sa_dma_init(struct sa_crypto_data *dd)
2250{
2251 int ret;
2252 struct dma_slave_config cfg;
2253
2254 dd->dma_rx1 = NULL;
2255 dd->dma_tx = NULL;
2256 dd->dma_rx2 = NULL;
2257
2258 ret = dma_coerce_mask_and_coherent(dd->dev, DMA_BIT_MASK(48));
2259 if (ret)
2260 return ret;
2261
2262 dd->dma_rx1 = dma_request_chan(dd->dev, "rx1");
2263 if (IS_ERR(dd->dma_rx1))
2264 return dev_err_probe(dd->dev, PTR_ERR(dd->dma_rx1),
2265 "Unable to request rx1 DMA channel\n");
2266
2267 dd->dma_rx2 = dma_request_chan(dd->dev, "rx2");
2268 if (IS_ERR(dd->dma_rx2)) {
2269 ret = dev_err_probe(dd->dev, PTR_ERR(dd->dma_rx2),
2270 "Unable to request rx2 DMA channel\n");
2271 goto err_dma_rx2;
2272 }
2273
2274 dd->dma_tx = dma_request_chan(dd->dev, "tx");
2275 if (IS_ERR(dd->dma_tx)) {
2276 ret = dev_err_probe(dd->dev, PTR_ERR(dd->dma_tx),
2277 "Unable to request tx DMA channel\n");
2278 goto err_dma_tx;
2279 }
2280
2281 memzero_explicit(&cfg, sizeof(cfg));
2282
2283 cfg.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
2284 cfg.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
2285 cfg.src_maxburst = 4;
2286 cfg.dst_maxburst = 4;
2287
2288 ret = dmaengine_slave_config(dd->dma_rx1, &cfg);
2289 if (ret) {
2290 dev_err(dd->dev, "can't configure IN dmaengine slave: %d\n",
2291 ret);
2292 goto err_dma_config;
2293 }
2294
2295 ret = dmaengine_slave_config(dd->dma_rx2, &cfg);
2296 if (ret) {
2297 dev_err(dd->dev, "can't configure IN dmaengine slave: %d\n",
2298 ret);
2299 goto err_dma_config;
2300 }
2301
2302 ret = dmaengine_slave_config(dd->dma_tx, &cfg);
2303 if (ret) {
2304 dev_err(dd->dev, "can't configure OUT dmaengine slave: %d\n",
2305 ret);
2306 goto err_dma_config;
2307 }
2308
2309 return 0;
2310
2311err_dma_config:
2312 dma_release_channel(dd->dma_tx);
2313err_dma_tx:
2314 dma_release_channel(dd->dma_rx2);
2315err_dma_rx2:
2316 dma_release_channel(dd->dma_rx1);
2317
2318 return ret;
2319}
2320
2321static int sa_link_child(struct device *dev, void *data)
2322{
2323 struct device *parent = data;
2324
2325 device_link_add(dev, parent, DL_FLAG_AUTOPROBE_CONSUMER);
2326
2327 return 0;
2328}
2329
2330static struct sa_match_data am654_match_data = {
2331 .priv = 1,
2332 .priv_id = 1,
2333 .supported_algos = BIT(SA_ALG_CBC_AES) |
2334 BIT(SA_ALG_EBC_AES) |
2335 BIT(SA_ALG_CBC_DES3) |
2336 BIT(SA_ALG_ECB_DES3) |
2337 BIT(SA_ALG_SHA1) |
2338 BIT(SA_ALG_SHA256) |
2339 BIT(SA_ALG_SHA512) |
2340 BIT(SA_ALG_AUTHENC_SHA1_AES) |
2341 BIT(SA_ALG_AUTHENC_SHA256_AES),
2342};
2343
2344static struct sa_match_data am64_match_data = {
2345 .priv = 0,
2346 .priv_id = 0,
2347 .supported_algos = BIT(SA_ALG_CBC_AES) |
2348 BIT(SA_ALG_EBC_AES) |
2349 BIT(SA_ALG_SHA256) |
2350 BIT(SA_ALG_SHA512) |
2351 BIT(SA_ALG_AUTHENC_SHA256_AES),
2352};
2353
2354static const struct of_device_id of_match[] = {
2355 { .compatible = "ti,j721e-sa2ul", .data = &am654_match_data, },
2356 { .compatible = "ti,am654-sa2ul", .data = &am654_match_data, },
2357 { .compatible = "ti,am64-sa2ul", .data = &am64_match_data, },
2358 { .compatible = "ti,am62-sa3ul", .data = &am64_match_data, },
2359 {},
2360};
2361MODULE_DEVICE_TABLE(of, of_match);
2362
2363static int sa_ul_probe(struct platform_device *pdev)
2364{
2365 struct device *dev = &pdev->dev;
2366 struct device_node *node = dev->of_node;
2367 static void __iomem *saul_base;
2368 struct sa_crypto_data *dev_data;
2369 u32 status, val;
2370 int ret;
2371
2372 dev_data = devm_kzalloc(dev, sizeof(*dev_data), GFP_KERNEL);
2373 if (!dev_data)
2374 return -ENOMEM;
2375
2376 dev_data->match_data = of_device_get_match_data(dev);
2377 if (!dev_data->match_data)
2378 return -ENODEV;
2379
2380 saul_base = devm_platform_ioremap_resource(pdev, 0);
2381 if (IS_ERR(saul_base))
2382 return PTR_ERR(saul_base);
2383
2384 sa_k3_dev = dev;
2385 dev_data->dev = dev;
2386 dev_data->pdev = pdev;
2387 dev_data->base = saul_base;
2388 platform_set_drvdata(pdev, dev_data);
2389 dev_set_drvdata(sa_k3_dev, dev_data);
2390
2391 pm_runtime_enable(dev);
2392 ret = pm_runtime_resume_and_get(dev);
2393 if (ret < 0) {
2394 dev_err(dev, "%s: failed to get sync: %d\n", __func__, ret);
2395 pm_runtime_disable(dev);
2396 return ret;
2397 }
2398
2399 ret = sa_init_mem(dev_data);
2400 if (ret)
2401 goto disable_pm;
2402
2403 ret = sa_dma_init(dev_data);
2404 if (ret)
2405 goto destroy_dma_pool;
2406
2407 spin_lock_init(&dev_data->scid_lock);
2408
2409 val = SA_EEC_ENCSS_EN | SA_EEC_AUTHSS_EN | SA_EEC_CTXCACH_EN |
2410 SA_EEC_CPPI_PORT_IN_EN | SA_EEC_CPPI_PORT_OUT_EN |
2411 SA_EEC_TRNG_EN;
2412 status = readl_relaxed(saul_base + SA_ENGINE_STATUS);
2413 /* Only enable engines if all are not already enabled */
2414 if (val & ~status)
2415 writel_relaxed(val, saul_base + SA_ENGINE_ENABLE_CONTROL);
2416
2417 sa_register_algos(dev_data);
2418
2419 ret = of_platform_populate(node, NULL, NULL, dev);
2420 if (ret)
2421 goto release_dma;
2422
2423 device_for_each_child(dev, dev, sa_link_child);
2424
2425 return 0;
2426
2427release_dma:
2428 sa_unregister_algos(dev);
2429
2430 dma_release_channel(dev_data->dma_rx2);
2431 dma_release_channel(dev_data->dma_rx1);
2432 dma_release_channel(dev_data->dma_tx);
2433
2434destroy_dma_pool:
2435 dma_pool_destroy(dev_data->sc_pool);
2436
2437disable_pm:
2438 pm_runtime_put_sync(dev);
2439 pm_runtime_disable(dev);
2440
2441 return ret;
2442}
2443
2444static void sa_ul_remove(struct platform_device *pdev)
2445{
2446 struct sa_crypto_data *dev_data = platform_get_drvdata(pdev);
2447
2448 of_platform_depopulate(&pdev->dev);
2449
2450 sa_unregister_algos(&pdev->dev);
2451
2452 dma_release_channel(dev_data->dma_rx2);
2453 dma_release_channel(dev_data->dma_rx1);
2454 dma_release_channel(dev_data->dma_tx);
2455
2456 dma_pool_destroy(dev_data->sc_pool);
2457
2458 platform_set_drvdata(pdev, NULL);
2459
2460 pm_runtime_put_sync(&pdev->dev);
2461 pm_runtime_disable(&pdev->dev);
2462}
2463
2464static struct platform_driver sa_ul_driver = {
2465 .probe = sa_ul_probe,
2466 .remove = sa_ul_remove,
2467 .driver = {
2468 .name = "saul-crypto",
2469 .of_match_table = of_match,
2470 },
2471};
2472module_platform_driver(sa_ul_driver);
2473MODULE_DESCRIPTION("K3 SA2UL crypto accelerator driver");
2474MODULE_LICENSE("GPL v2");