linux - drivers/crypto/bcm/cipher.c

4709 lines · 128.8 KiB

1	// SPDX-License-Identifier: GPL-2.0-only
2	/*
3	* Copyright 2016 Broadcom
4	*/
5
6	#include <linux/err.h>
7	#include <linux/module.h>
8	#include <linux/init.h>
9	#include <linux/errno.h>
10	#include <linux/kernel.h>
11	#include <linux/interrupt.h>
12	#include <linux/platform_device.h>
13	#include <linux/scatterlist.h>
14	#include <linux/crypto.h>
15	#include <linux/kthread.h>
16	#include <linux/rtnetlink.h>
17	#include <linux/sched.h>
18	#include <linux/string_choices.h>
19	#include <linux/of.h>
20	#include <linux/io.h>
21	#include <linux/bitops.h>
22
23	#include <crypto/algapi.h>
24	#include <crypto/aead.h>
25	#include <crypto/internal/aead.h>
26	#include <crypto/aes.h>
27	#include <crypto/internal/des.h>
28	#include <crypto/hmac.h>
29	#include <crypto/md5.h>
30	#include <crypto/authenc.h>
31	#include <crypto/skcipher.h>
32	#include <crypto/hash.h>
33	#include <crypto/sha1.h>
34	#include <crypto/sha2.h>
35	#include <crypto/sha3.h>
36
37	#include "util.h"
38	#include "cipher.h"
39	#include "spu.h"
40	#include "spum.h"
41	#include "spu2.h"
42
43	/* ================= Device Structure ================== */
44
45	struct bcm_device_private iproc_priv;
46
47	/* ==================== Parameters ===================== */
48
49	int flow_debug_logging;
50	module_param(flow_debug_logging, int, 0644);
51	MODULE_PARM_DESC(flow_debug_logging, "Enable Flow Debug Logging");
52
53	int packet_debug_logging;
54	module_param(packet_debug_logging, int, 0644);
55	MODULE_PARM_DESC(packet_debug_logging, "Enable Packet Debug Logging");
56
57	int debug_logging_sleep;
58	module_param(debug_logging_sleep, int, 0644);
59	MODULE_PARM_DESC(debug_logging_sleep, "Packet Debug Logging Sleep");
60
61	/*
62	* The value of these module parameters is used to set the priority for each
63	* algo type when this driver registers algos with the kernel crypto API.
64	* To use a priority other than the default, set the priority in the insmod or
65	* modprobe. Changing the module priority after init time has no effect.
66	*
67	* The default priorities are chosen to be lower (less preferred) than ARMv8 CE
68	* algos, but more preferred than generic software algos.
69	*/
70	static int cipher_pri = 150;
71	module_param(cipher_pri, int, 0644);
72	MODULE_PARM_DESC(cipher_pri, "Priority for cipher algos");
73
74	static int hash_pri = 100;
75	module_param(hash_pri, int, 0644);
76	MODULE_PARM_DESC(hash_pri, "Priority for hash algos");
77
78	static int aead_pri = 150;
79	module_param(aead_pri, int, 0644);
80	MODULE_PARM_DESC(aead_pri, "Priority for AEAD algos");
81
82	/* A type 3 BCM header, expected to precede the SPU header for SPU-M.
83	* Bits 3 and 4 in the first byte encode the channel number (the dma ringset).
84	* 0x60 - ring 0
85	* 0x68 - ring 1
86	* 0x70 - ring 2
87	* 0x78 - ring 3
88	*/
89	static char BCMHEADER[] = { 0x60, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x28 };
90	/*
91	* Some SPU hw does not use BCM header on SPU messages. So BCM_HDR_LEN
92	* is set dynamically after reading SPU type from device tree.
93	*/
94	#define BCM_HDR_LEN iproc_priv.bcm_hdr_len
95
96	/* min and max time to sleep before retrying when mbox queue is full. usec */
97	#define MBOX_SLEEP_MIN 800
98	#define MBOX_SLEEP_MAX 1000
99
100	/**
101	* select_channel() - Select a SPU channel to handle a crypto request. Selects
102	* channel in round robin order.
103	*
104	* Return: channel index
105	*/
106	static u8 select_channel(void)
107	{
108	u8 chan_idx = atomic_inc_return(&iproc_priv.next_chan);
109
110	return chan_idx % iproc_priv.spu.num_chan;
111	}
112
113	/**
114	* spu_skcipher_rx_sg_create() - Build up the scatterlist of buffers used to
115	* receive a SPU response message for an skcipher request. Includes buffers to
116	* catch SPU message headers and the response data.
117	* @mssg: mailbox message containing the receive sg
118	* @rctx: crypto request context
119	* @rx_frag_num: number of scatterlist elements required to hold the
120	* SPU response message
121	* @chunksize: Number of bytes of response data expected
122	* @stat_pad_len: Number of bytes required to pad the STAT field to
123	* a 4-byte boundary
124	*
125	* The scatterlist that gets allocated here is freed in spu_chunk_cleanup()
126	* when the request completes, whether the request is handled successfully or
127	* there is an error.
128	*
129	* Returns:
130	* 0 if successful
131	* < 0 if an error
132	*/
133	static int
134	spu_skcipher_rx_sg_create(struct brcm_message *mssg,
135	struct iproc_reqctx_s *rctx,
136	u8 rx_frag_num,
137	unsigned int chunksize, u32 stat_pad_len)
138	{
139	struct spu_hw *spu = &iproc_priv.spu;
140	struct scatterlist sg; / used to build sgs in mbox message */
141	struct iproc_ctx_s *ctx = rctx->ctx;
142	u32 datalen; /* Number of bytes of response data expected */
143
144	mssg->spu.dst = kmalloc_objs(struct scatterlist, rx_frag_num, rctx->gfp);
145	if (!mssg->spu.dst)
146	return -ENOMEM;
147
148	sg = mssg->spu.dst;
149	sg_init_table(sg, rx_frag_num);
150	/* Space for SPU message header */
151	sg_set_buf(sg++, rctx->msg_buf.spu_resp_hdr, ctx->spu_resp_hdr_len);
152
153	/* If XTS tweak in payload, add buffer to receive encrypted tweak */
154	if ((ctx->cipher.mode == CIPHER_MODE_XTS) &&
155	spu->spu_xts_tweak_in_payload())
156	sg_set_buf(sg++, rctx->msg_buf.c.supdt_tweak,
157	SPU_XTS_TWEAK_SIZE);
158
159	/* Copy in each dst sg entry from request, up to chunksize */
160	datalen = spu_msg_sg_add(&sg, &rctx->dst_sg, &rctx->dst_skip,
161	rctx->dst_nents, chunksize);
162	if (datalen < chunksize) {
163	pr_err("%s(): failed to copy dst sg to mbox msg. chunksize %u, datalen %u",
164	__func__, chunksize, datalen);
165	return -EFAULT;
166	}
167
168	if (stat_pad_len)
169	sg_set_buf(sg++, rctx->msg_buf.rx_stat_pad, stat_pad_len);
170
171	memset(rctx->msg_buf.rx_stat, 0, SPU_RX_STATUS_LEN);
172	sg_set_buf(sg, rctx->msg_buf.rx_stat, spu->spu_rx_status_len());
173
174	return 0;
175	}
176
177	/**
178	* spu_skcipher_tx_sg_create() - Build up the scatterlist of buffers used to
179	* send a SPU request message for an skcipher request. Includes SPU message
180	* headers and the request data.
181	* @mssg: mailbox message containing the transmit sg
182	* @rctx: crypto request context
183	* @tx_frag_num: number of scatterlist elements required to construct the
184	* SPU request message
185	* @chunksize: Number of bytes of request data
186	* @pad_len: Number of pad bytes
187	*
188	* The scatterlist that gets allocated here is freed in spu_chunk_cleanup()
189	* when the request completes, whether the request is handled successfully or
190	* there is an error.
191	*
192	* Returns:
193	* 0 if successful
194	* < 0 if an error
195	*/
196	static int
197	spu_skcipher_tx_sg_create(struct brcm_message *mssg,
198	struct iproc_reqctx_s *rctx,
199	u8 tx_frag_num, unsigned int chunksize, u32 pad_len)
200	{
201	struct spu_hw *spu = &iproc_priv.spu;
202	struct scatterlist sg; / used to build sgs in mbox message */
203	struct iproc_ctx_s *ctx = rctx->ctx;
204	u32 datalen; /* Number of bytes of response data expected */
205	u32 stat_len;
206
207	mssg->spu.src = kmalloc_objs(struct scatterlist, tx_frag_num, rctx->gfp);
208	if (unlikely(!mssg->spu.src))
209	return -ENOMEM;
210
211	sg = mssg->spu.src;
212	sg_init_table(sg, tx_frag_num);
213
214	sg_set_buf(sg++, rctx->msg_buf.bcm_spu_req_hdr,
215	BCM_HDR_LEN + ctx->spu_req_hdr_len);
216
217	/* if XTS tweak in payload, copy from IV (where crypto API puts it) */
218	if ((ctx->cipher.mode == CIPHER_MODE_XTS) &&
219	spu->spu_xts_tweak_in_payload())
220	sg_set_buf(sg++, rctx->msg_buf.iv_ctr, SPU_XTS_TWEAK_SIZE);
221
222	/* Copy in each src sg entry from request, up to chunksize */
223	datalen = spu_msg_sg_add(&sg, &rctx->src_sg, &rctx->src_skip,
224	rctx->src_nents, chunksize);
225	if (unlikely(datalen < chunksize)) {
226	pr_err("%s(): failed to copy src sg to mbox msg",
227	__func__);
228	return -EFAULT;
229	}
230
231	if (pad_len)
232	sg_set_buf(sg++, rctx->msg_buf.spu_req_pad, pad_len);
233
234	stat_len = spu->spu_tx_status_len();
235	if (stat_len) {
236	memset(rctx->msg_buf.tx_stat, 0, stat_len);
237	sg_set_buf(sg, rctx->msg_buf.tx_stat, stat_len);
238	}
239	return 0;
240	}
241
242	static int mailbox_send_message(struct brcm_message *mssg, u32 flags,
243	u8 chan_idx)
244	{
245	int err;
246	int retry_cnt = 0;
247	struct device *dev = &(iproc_priv.pdev->dev);
248
249	err = mbox_send_message(iproc_priv.mbox[chan_idx], mssg);
250	if (flags & CRYPTO_TFM_REQ_MAY_SLEEP) {
251	while ((err == -ENOBUFS) && (retry_cnt < SPU_MB_RETRY_MAX)) {
252	/*
253	* Mailbox queue is full. Since MAY_SLEEP is set, assume
254	* not in atomic context and we can wait and try again.
255	*/
256	retry_cnt++;
257	usleep_range(MBOX_SLEEP_MIN, MBOX_SLEEP_MAX);
258	err = mbox_send_message(iproc_priv.mbox[chan_idx],
259	mssg);
260	atomic_inc(&iproc_priv.mb_no_spc);
261	}
262	}
263	if (err < 0) {
264	atomic_inc(&iproc_priv.mb_send_fail);
265	return err;
266	}
267
268	/* Check error returned by mailbox controller */
269	err = mssg->error;
270	if (unlikely(err < 0)) {
271	dev_err(dev, "message error %d", err);
272	/* Signal txdone for mailbox channel */
273	}
274
275	/* Signal txdone for mailbox channel */
276	mbox_client_txdone(iproc_priv.mbox[chan_idx], err);
277	return err;
278	}
279
280	/**
281	* handle_skcipher_req() - Submit as much of a block cipher request as fits in
282	* a single SPU request message, starting at the current position in the request
283	* data.
284	* @rctx: Crypto request context
285	*
286	* This may be called on the crypto API thread, or, when a request is so large
287	* it must be broken into multiple SPU messages, on the thread used to invoke
288	* the response callback. When requests are broken into multiple SPU
289	* messages, we assume subsequent messages depend on previous results, and
290	* thus always wait for previous results before submitting the next message.
291	* Because requests are submitted in lock step like this, there is no need
292	* to synchronize access to request data structures.
293	*
294	* Return: -EINPROGRESS: request has been accepted and result will be returned
295	* asynchronously
296	* Any other value indicates an error
297	*/
298	static int handle_skcipher_req(struct iproc_reqctx_s *rctx)
299	{
300	struct spu_hw *spu = &iproc_priv.spu;
301	struct crypto_async_request *areq = rctx->parent;
302	struct skcipher_request *req =
303	container_of(areq, struct skcipher_request, base);
304	struct iproc_ctx_s *ctx = rctx->ctx;
305	struct spu_cipher_parms cipher_parms;
306	int err;
307	unsigned int chunksize; /* Num bytes of request to submit */
308	int remaining; /* Bytes of request still to process */
309	int chunk_start; /* Beginning of data for current SPU msg */
310
311	/* IV or ctr value to use in this SPU msg */
312	u8 local_iv_ctr[MAX_IV_SIZE];
313	u32 stat_pad_len; /* num bytes to align status field */
314	u32 pad_len; /* total length of all padding */
315	struct brcm_message mssg; / mailbox message */
316
317	/* number of entries in src and dst sg in mailbox message. */
318	u8 rx_frag_num = 2; /* response header and STATUS */
319	u8 tx_frag_num = 1; /* request header */
320
321	flow_log("%s\n", __func__);
322
323	cipher_parms.alg = ctx->cipher.alg;
324	cipher_parms.mode = ctx->cipher.mode;
325	cipher_parms.type = ctx->cipher_type;
326	cipher_parms.key_len = ctx->enckeylen;
327	cipher_parms.key_buf = ctx->enckey;
328	cipher_parms.iv_buf = local_iv_ctr;
329	cipher_parms.iv_len = rctx->iv_ctr_len;
330
331	mssg = &rctx->mb_mssg;
332	chunk_start = rctx->src_sent;
333	remaining = rctx->total_todo - chunk_start;
334
335	/* determine the chunk we are breaking off and update the indexes */
336	if ((ctx->max_payload != SPU_MAX_PAYLOAD_INF) &&
337	(remaining > ctx->max_payload))
338	chunksize = ctx->max_payload;
339	else
340	chunksize = remaining;
341
342	rctx->src_sent += chunksize;
343	rctx->total_sent = rctx->src_sent;
344
345	/* Count number of sg entries to be included in this request */
346	rctx->src_nents = spu_sg_count(rctx->src_sg, rctx->src_skip, chunksize);
347	rctx->dst_nents = spu_sg_count(rctx->dst_sg, rctx->dst_skip, chunksize);
348
349	if ((ctx->cipher.mode == CIPHER_MODE_CBC) &&
350	rctx->is_encrypt && chunk_start)
351	/*
352	* Encrypting non-first first chunk. Copy last block of
353	* previous result to IV for this chunk.
354	*/
355	sg_copy_part_to_buf(req->dst, rctx->msg_buf.iv_ctr,
356	rctx->iv_ctr_len,
357	chunk_start - rctx->iv_ctr_len);
358
359	if (rctx->iv_ctr_len) {
360	/* get our local copy of the iv */
361	__builtin_memcpy(local_iv_ctr, rctx->msg_buf.iv_ctr,
362	rctx->iv_ctr_len);
363
364	/* generate the next IV if possible */
365	if ((ctx->cipher.mode == CIPHER_MODE_CBC) &&
366	!rctx->is_encrypt) {
367	/*
368	* CBC Decrypt: next IV is the last ciphertext block in
369	* this chunk
370	*/
371	sg_copy_part_to_buf(req->src, rctx->msg_buf.iv_ctr,
372	rctx->iv_ctr_len,
373	rctx->src_sent - rctx->iv_ctr_len);
374	} else if (ctx->cipher.mode == CIPHER_MODE_CTR) {
375	/*
376	* The SPU hardware increments the counter once for
377	* each AES block of 16 bytes. So update the counter
378	* for the next chunk, if there is one. Note that for
379	* this chunk, the counter has already been copied to
380	* local_iv_ctr. We can assume a block size of 16,
381	* because we only support CTR mode for AES, not for
382	* any other cipher alg.
383	*/
384	add_to_ctr(rctx->msg_buf.iv_ctr, chunksize >> 4);
385	}
386	}
387
388	if (ctx->max_payload == SPU_MAX_PAYLOAD_INF)
389	flow_log("max_payload infinite\n");
390	else
391	flow_log("max_payload %u\n", ctx->max_payload);
392
393	flow_log("sent:%u start:%u remains:%u size:%u\n",
394	rctx->src_sent, chunk_start, remaining, chunksize);
395
396	/* Copy SPU header template created at setkey time */
397	memcpy(rctx->msg_buf.bcm_spu_req_hdr, ctx->bcm_spu_req_hdr,
398	sizeof(rctx->msg_buf.bcm_spu_req_hdr));
399
400	spu->spu_cipher_req_finish(rctx->msg_buf.bcm_spu_req_hdr + BCM_HDR_LEN,
401	ctx->spu_req_hdr_len, !(rctx->is_encrypt),
402	&cipher_parms, chunksize);
403
404	atomic64_add(chunksize, &iproc_priv.bytes_out);
405
406	stat_pad_len = spu->spu_wordalign_padlen(chunksize);
407	if (stat_pad_len)
408	rx_frag_num++;
409	pad_len = stat_pad_len;
410	if (pad_len) {
411	tx_frag_num++;
412	spu->spu_request_pad(rctx->msg_buf.spu_req_pad, 0,
413	0, ctx->auth.alg, ctx->auth.mode,
414	rctx->total_sent, stat_pad_len);
415	}
416
417	spu->spu_dump_msg_hdr(rctx->msg_buf.bcm_spu_req_hdr + BCM_HDR_LEN,
418	ctx->spu_req_hdr_len);
419	packet_log("payload:\n");
420	dump_sg(rctx->src_sg, rctx->src_skip, chunksize);
421	packet_dump(" pad: ", rctx->msg_buf.spu_req_pad, pad_len);
422
423	/*
424	* Build mailbox message containing SPU request msg and rx buffers
425	* to catch response message
426	*/
427	memset(mssg, 0, sizeof(*mssg));
428	mssg->type = BRCM_MESSAGE_SPU;
429	mssg->ctx = rctx; /* Will be returned in response */
430
431	/* Create rx scatterlist to catch result */
432	rx_frag_num += rctx->dst_nents;
433
434	if ((ctx->cipher.mode == CIPHER_MODE_XTS) &&
435	spu->spu_xts_tweak_in_payload())
436	rx_frag_num++; /* extra sg to insert tweak */
437
438	err = spu_skcipher_rx_sg_create(mssg, rctx, rx_frag_num, chunksize,
439	stat_pad_len);
440	if (err)
441	return err;
442
443	/* Create tx scatterlist containing SPU request message */
444	tx_frag_num += rctx->src_nents;
445	if (spu->spu_tx_status_len())
446	tx_frag_num++;
447
448	if ((ctx->cipher.mode == CIPHER_MODE_XTS) &&
449	spu->spu_xts_tweak_in_payload())
450	tx_frag_num++; /* extra sg to insert tweak */
451
452	err = spu_skcipher_tx_sg_create(mssg, rctx, tx_frag_num, chunksize,
453	pad_len);
454	if (err)
455	return err;
456
457	err = mailbox_send_message(mssg, req->base.flags, rctx->chan_idx);
458	if (unlikely(err < 0))
459	return err;
460
461	return -EINPROGRESS;
462	}
463
464	/**
465	* handle_skcipher_resp() - Process a block cipher SPU response. Updates the
466	* total received count for the request and updates global stats.
467	* @rctx: Crypto request context
468	*/
469	static void handle_skcipher_resp(struct iproc_reqctx_s *rctx)
470	{
471	struct spu_hw *spu = &iproc_priv.spu;
472	struct crypto_async_request *areq = rctx->parent;
473	struct skcipher_request *req = skcipher_request_cast(areq);
474	struct iproc_ctx_s *ctx = rctx->ctx;
475	u32 payload_len;
476
477	/* See how much data was returned */
478	payload_len = spu->spu_payload_length(rctx->msg_buf.spu_resp_hdr);
479
480	/*
481	* In XTS mode, the first SPU_XTS_TWEAK_SIZE bytes may be the
482	* encrypted tweak ("i") value; we don't count those.
483	*/
484	if ((ctx->cipher.mode == CIPHER_MODE_XTS) &&
485	spu->spu_xts_tweak_in_payload() &&
486	(payload_len >= SPU_XTS_TWEAK_SIZE))
487	payload_len -= SPU_XTS_TWEAK_SIZE;
488
489	atomic64_add(payload_len, &iproc_priv.bytes_in);
490
491	flow_log("%s() offset: %u, bd_len: %u BD:\n",
492	__func__, rctx->total_received, payload_len);
493
494	dump_sg(req->dst, rctx->total_received, payload_len);
495
496	rctx->total_received += payload_len;
497	if (rctx->total_received == rctx->total_todo) {
498	atomic_inc(&iproc_priv.op_counts[SPU_OP_CIPHER]);
499	atomic_inc(
500	&iproc_priv.cipher_cnt[ctx->cipher.alg][ctx->cipher.mode]);
501	}
502	}
503
504	/**
505	* spu_ahash_rx_sg_create() - Build up the scatterlist of buffers used to
506	* receive a SPU response message for an ahash request.
507	* @mssg: mailbox message containing the receive sg
508	* @rctx: crypto request context
509	* @rx_frag_num: number of scatterlist elements required to hold the
510	* SPU response message
511	* @digestsize: length of hash digest, in bytes
512	* @stat_pad_len: Number of bytes required to pad the STAT field to
513	* a 4-byte boundary
514	*
515	* The scatterlist that gets allocated here is freed in spu_chunk_cleanup()
516	* when the request completes, whether the request is handled successfully or
517	* there is an error.
518	*
519	* Return:
520	* 0 if successful
521	* < 0 if an error
522	*/
523	static int
524	spu_ahash_rx_sg_create(struct brcm_message *mssg,
525	struct iproc_reqctx_s *rctx,
526	u8 rx_frag_num, unsigned int digestsize,
527	u32 stat_pad_len)
528	{
529	struct spu_hw *spu = &iproc_priv.spu;
530	struct scatterlist sg; / used to build sgs in mbox message */
531	struct iproc_ctx_s *ctx = rctx->ctx;
532
533	mssg->spu.dst = kmalloc_objs(struct scatterlist, rx_frag_num, rctx->gfp);
534	if (!mssg->spu.dst)
535	return -ENOMEM;
536
537	sg = mssg->spu.dst;
538	sg_init_table(sg, rx_frag_num);
539	/* Space for SPU message header */
540	sg_set_buf(sg++, rctx->msg_buf.spu_resp_hdr, ctx->spu_resp_hdr_len);
541
542	/* Space for digest */
543	sg_set_buf(sg++, rctx->msg_buf.digest, digestsize);
544
545	if (stat_pad_len)
546	sg_set_buf(sg++, rctx->msg_buf.rx_stat_pad, stat_pad_len);
547
548	memset(rctx->msg_buf.rx_stat, 0, SPU_RX_STATUS_LEN);
549	sg_set_buf(sg, rctx->msg_buf.rx_stat, spu->spu_rx_status_len());
550	return 0;
551	}
552
553	/**
554	* spu_ahash_tx_sg_create() - Build up the scatterlist of buffers used to send
555	* a SPU request message for an ahash request. Includes SPU message headers and
556	* the request data.
557	* @mssg: mailbox message containing the transmit sg
558	* @rctx: crypto request context
559	* @tx_frag_num: number of scatterlist elements required to construct the
560	* SPU request message
561	* @spu_hdr_len: length in bytes of SPU message header
562	* @hash_carry_len: Number of bytes of data carried over from previous req
563	* @new_data_len: Number of bytes of new request data
564	* @pad_len: Number of pad bytes
565	*
566	* The scatterlist that gets allocated here is freed in spu_chunk_cleanup()
567	* when the request completes, whether the request is handled successfully or
568	* there is an error.
569	*
570	* Return:
571	* 0 if successful
572	* < 0 if an error
573	*/
574	static int
575	spu_ahash_tx_sg_create(struct brcm_message *mssg,
576	struct iproc_reqctx_s *rctx,
577	u8 tx_frag_num,
578	u32 spu_hdr_len,
579	unsigned int hash_carry_len,
580	unsigned int new_data_len, u32 pad_len)
581	{
582	struct spu_hw *spu = &iproc_priv.spu;
583	struct scatterlist sg; / used to build sgs in mbox message */
584	u32 datalen; /* Number of bytes of response data expected */
585	u32 stat_len;
586
587	mssg->spu.src = kmalloc_objs(struct scatterlist, tx_frag_num, rctx->gfp);
588	if (!mssg->spu.src)
589	return -ENOMEM;
590
591	sg = mssg->spu.src;
592	sg_init_table(sg, tx_frag_num);
593
594	sg_set_buf(sg++, rctx->msg_buf.bcm_spu_req_hdr,
595	BCM_HDR_LEN + spu_hdr_len);
596
597	if (hash_carry_len)
598	sg_set_buf(sg++, rctx->hash_carry, hash_carry_len);
599
600	if (new_data_len) {
601	/* Copy in each src sg entry from request, up to chunksize */
602	datalen = spu_msg_sg_add(&sg, &rctx->src_sg, &rctx->src_skip,
603	rctx->src_nents, new_data_len);
604	if (datalen < new_data_len) {
605	pr_err("%s(): failed to copy src sg to mbox msg",
606	__func__);
607	return -EFAULT;
608	}
609	}
610
611	if (pad_len)
612	sg_set_buf(sg++, rctx->msg_buf.spu_req_pad, pad_len);
613
614	stat_len = spu->spu_tx_status_len();
615	if (stat_len) {
616	memset(rctx->msg_buf.tx_stat, 0, stat_len);
617	sg_set_buf(sg, rctx->msg_buf.tx_stat, stat_len);
618	}
619
620	return 0;
621	}
622
623	/**
624	* handle_ahash_req() - Process an asynchronous hash request from the crypto
625	* API.
626	* @rctx: Crypto request context
627	*
628	* Builds a SPU request message embedded in a mailbox message and submits the
629	* mailbox message on a selected mailbox channel. The SPU request message is
630	* constructed as a scatterlist, including entries from the crypto API's
631	* src scatterlist to avoid copying the data to be hashed. This function is
632	* called either on the thread from the crypto API, or, in the case that the
633	* crypto API request is too large to fit in a single SPU request message,
634	* on the thread that invokes the receive callback with a response message.
635	* Because some operations require the response from one chunk before the next
636	* chunk can be submitted, we always wait for the response for the previous
637	* chunk before submitting the next chunk. Because requests are submitted in
638	* lock step like this, there is no need to synchronize access to request data
639	* structures.
640	*
641	* Return:
642	* -EINPROGRESS: request has been submitted to SPU and response will be
643	* returned asynchronously
644	* -EAGAIN: non-final request included a small amount of data, which for
645	* efficiency we did not submit to the SPU, but instead stored
646	* to be submitted to the SPU with the next part of the request
647	* other: an error code
648	*/
649	static int handle_ahash_req(struct iproc_reqctx_s *rctx)
650	{
651	struct spu_hw *spu = &iproc_priv.spu;
652	struct crypto_async_request *areq = rctx->parent;
653	struct ahash_request *req = ahash_request_cast(areq);
654	struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
655	struct crypto_tfm *tfm = crypto_ahash_tfm(ahash);
656	unsigned int blocksize = crypto_tfm_alg_blocksize(tfm);
657	struct iproc_ctx_s *ctx = rctx->ctx;
658
659	/* number of bytes still to be hashed in this req */
660	unsigned int nbytes_to_hash = 0;
661	int err;
662	unsigned int chunksize = 0; /* length of hash carry + new data */
663	/*
664	* length of new data, not from hash carry, to be submitted in
665	* this hw request
666	*/
667	unsigned int new_data_len;
668
669	unsigned int __maybe_unused chunk_start = 0;
670	u32 db_size; /* Length of data field, incl gcm and hash padding */
671	int pad_len = 0; /* total pad len, including gcm, hash, stat padding */
672	u32 data_pad_len = 0; /* length of GCM/CCM padding */
673	u32 stat_pad_len = 0; /* length of padding to align STATUS word */
674	struct brcm_message mssg; / mailbox message */
675	struct spu_request_opts req_opts;
676	struct spu_cipher_parms cipher_parms;
677	struct spu_hash_parms hash_parms;
678	struct spu_aead_parms aead_parms;
679	unsigned int local_nbuf;
680	u32 spu_hdr_len;
681	unsigned int digestsize;
682	u16 rem = 0;
683
684	/*
685	* number of entries in src and dst sg. Always includes SPU msg header.
686	* rx always includes a buffer to catch digest and STATUS.
687	*/
688	u8 rx_frag_num = 3;
689	u8 tx_frag_num = 1;
690
691	flow_log("total_todo %u, total_sent %u\n",
692	rctx->total_todo, rctx->total_sent);
693
694	memset(&req_opts, 0, sizeof(req_opts));
695	memset(&cipher_parms, 0, sizeof(cipher_parms));
696	memset(&hash_parms, 0, sizeof(hash_parms));
697	memset(&aead_parms, 0, sizeof(aead_parms));
698
699	req_opts.bd_suppress = true;
700	hash_parms.alg = ctx->auth.alg;
701	hash_parms.mode = ctx->auth.mode;
702	hash_parms.type = HASH_TYPE_NONE;
703	hash_parms.key_buf = (u8 *)ctx->authkey;
704	hash_parms.key_len = ctx->authkeylen;
705
706	/*
707	* For hash algorithms below assignment looks bit odd but
708	* it's needed for AES-XCBC and AES-CMAC hash algorithms
709	* to differentiate between 128, 192, 256 bit key values.
710	* Based on the key values, hash algorithm is selected.
711	* For example for 128 bit key, hash algorithm is AES-128.
712	*/
713	cipher_parms.type = ctx->cipher_type;
714
715	mssg = &rctx->mb_mssg;
716	chunk_start = rctx->src_sent;
717
718	/*
719	* Compute the amount remaining to hash. This may include data
720	* carried over from previous requests.
721	*/
722	nbytes_to_hash = rctx->total_todo - rctx->total_sent;
723	chunksize = nbytes_to_hash;
724	if ((ctx->max_payload != SPU_MAX_PAYLOAD_INF) &&
725	(chunksize > ctx->max_payload))
726	chunksize = ctx->max_payload;
727
728	/*
729	* If this is not a final request and the request data is not a multiple
730	* of a full block, then simply park the extra data and prefix it to the
731	* data for the next request.
732	*/
733	if (!rctx->is_final) {
734	u8 *dest = rctx->hash_carry + rctx->hash_carry_len;
735	u16 new_len; /* len of data to add to hash carry */
736
737	rem = chunksize % blocksize; /* remainder */
738	if (rem) {
739	/* chunksize not a multiple of blocksize */
740	chunksize -= rem;
741	if (chunksize == 0) {
742	/* Don't have a full block to submit to hw */
743	new_len = rem - rctx->hash_carry_len;
744	sg_copy_part_to_buf(req->src, dest, new_len,
745	rctx->src_sent);
746	rctx->hash_carry_len = rem;
747	flow_log("Exiting with hash carry len: %u\n",
748	rctx->hash_carry_len);
749	packet_dump(" buf: ",
750	rctx->hash_carry,
751	rctx->hash_carry_len);
752	return -EAGAIN;
753	}
754	}
755	}
756
757	/* if we have hash carry, then prefix it to the data in this request */
758	local_nbuf = rctx->hash_carry_len;
759	rctx->hash_carry_len = 0;
760	if (local_nbuf)
761	tx_frag_num++;
762	new_data_len = chunksize - local_nbuf;
763
764	/* Count number of sg entries to be used in this request */
765	rctx->src_nents = spu_sg_count(rctx->src_sg, rctx->src_skip,
766	new_data_len);
767
768	/* AES hashing keeps key size in type field, so need to copy it here */
769	if (hash_parms.alg == HASH_ALG_AES)
770	hash_parms.type = (enum hash_type)cipher_parms.type;
771	else
772	hash_parms.type = spu->spu_hash_type(rctx->total_sent);
773
774	digestsize = spu->spu_digest_size(ctx->digestsize, ctx->auth.alg,
775	hash_parms.type);
776	hash_parms.digestsize = digestsize;
777
778	/* update the indexes */
779	rctx->total_sent += chunksize;
780	/* if you sent a prebuf then that wasn't from this req->src */
781	rctx->src_sent += new_data_len;
782
783	if ((rctx->total_sent == rctx->total_todo) && rctx->is_final)
784	hash_parms.pad_len = spu->spu_hash_pad_len(hash_parms.alg,
785	hash_parms.mode,
786	chunksize,
787	blocksize);
788
789	/*
790	* If a non-first chunk, then include the digest returned from the
791	* previous chunk so that hw can add to it (except for AES types).
792	*/
793	if ((hash_parms.type == HASH_TYPE_UPDT) &&
794	(hash_parms.alg != HASH_ALG_AES)) {
795	hash_parms.key_buf = rctx->incr_hash;
796	hash_parms.key_len = digestsize;
797	}
798
799	atomic64_add(chunksize, &iproc_priv.bytes_out);
800
801	flow_log("%s() final: %u nbuf: %u ",
802	__func__, rctx->is_final, local_nbuf);
803
804	if (ctx->max_payload == SPU_MAX_PAYLOAD_INF)
805	flow_log("max_payload infinite\n");
806	else
807	flow_log("max_payload %u\n", ctx->max_payload);
808
809	flow_log("chunk_start: %u chunk_size: %u\n", chunk_start, chunksize);
810
811	/* Prepend SPU header with type 3 BCM header */
812	memcpy(rctx->msg_buf.bcm_spu_req_hdr, BCMHEADER, BCM_HDR_LEN);
813
814	hash_parms.prebuf_len = local_nbuf;
815	spu_hdr_len = spu->spu_create_request(rctx->msg_buf.bcm_spu_req_hdr +
816	BCM_HDR_LEN,
817	&req_opts, &cipher_parms,
818	&hash_parms, &aead_parms,
819	new_data_len);
820
821	if (spu_hdr_len == 0) {
822	pr_err("Failed to create SPU request header\n");
823	return -EFAULT;
824	}
825
826	/*
827	* Determine total length of padding required. Put all padding in one
828	* buffer.
829	*/
830	data_pad_len = spu->spu_gcm_ccm_pad_len(ctx->cipher.mode, chunksize);
831	db_size = spu_real_db_size(0, 0, local_nbuf, new_data_len,
832	0, 0, hash_parms.pad_len);
833	if (spu->spu_tx_status_len())
834	stat_pad_len = spu->spu_wordalign_padlen(db_size);
835	if (stat_pad_len)
836	rx_frag_num++;
837	pad_len = hash_parms.pad_len + data_pad_len + stat_pad_len;
838	if (pad_len) {
839	tx_frag_num++;
840	spu->spu_request_pad(rctx->msg_buf.spu_req_pad, data_pad_len,
841	hash_parms.pad_len, ctx->auth.alg,
842	ctx->auth.mode, rctx->total_sent,
843	stat_pad_len);
844	}
845
846	spu->spu_dump_msg_hdr(rctx->msg_buf.bcm_spu_req_hdr + BCM_HDR_LEN,
847	spu_hdr_len);
848	packet_dump(" prebuf: ", rctx->hash_carry, local_nbuf);
849	flow_log("Data:\n");
850	dump_sg(rctx->src_sg, rctx->src_skip, new_data_len);
851	packet_dump(" pad: ", rctx->msg_buf.spu_req_pad, pad_len);
852
853	/*
854	* Build mailbox message containing SPU request msg and rx buffers
855	* to catch response message
856	*/
857	memset(mssg, 0, sizeof(*mssg));
858	mssg->type = BRCM_MESSAGE_SPU;
859	mssg->ctx = rctx; /* Will be returned in response */
860
861	/* Create rx scatterlist to catch result */
862	err = spu_ahash_rx_sg_create(mssg, rctx, rx_frag_num, digestsize,
863	stat_pad_len);
864	if (err)
865	return err;
866
867	/* Create tx scatterlist containing SPU request message */
868	tx_frag_num += rctx->src_nents;
869	if (spu->spu_tx_status_len())
870	tx_frag_num++;
871	err = spu_ahash_tx_sg_create(mssg, rctx, tx_frag_num, spu_hdr_len,
872	local_nbuf, new_data_len, pad_len);
873	if (err)
874	return err;
875
876	err = mailbox_send_message(mssg, req->base.flags, rctx->chan_idx);
877	if (unlikely(err < 0))
878	return err;
879
880	return -EINPROGRESS;
881	}
882
883	/**
884	* spu_hmac_outer_hash() - Request synchonous software compute of the outer hash
885	* for an HMAC request.
886	* @req: The HMAC request from the crypto API
887	* @ctx: The session context
888	*
889	* Return: 0 if synchronous hash operation successful
890	* -EINVAL if the hash algo is unrecognized
891	* any other value indicates an error
892	*/
893	static int spu_hmac_outer_hash(struct ahash_request *req,
894	struct iproc_ctx_s *ctx)
895	{
896	struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
897	unsigned int blocksize =
898	crypto_tfm_alg_blocksize(crypto_ahash_tfm(ahash));
899	int rc;
900
901	switch (ctx->auth.alg) {
902	case HASH_ALG_MD5:
903	rc = do_shash("md5", req->result, ctx->opad, blocksize,
904	req->result, ctx->digestsize, NULL, 0);
905	break;
906	case HASH_ALG_SHA1:
907	rc = do_shash("sha1", req->result, ctx->opad, blocksize,
908	req->result, ctx->digestsize, NULL, 0);
909	break;
910	case HASH_ALG_SHA224:
911	rc = do_shash("sha224", req->result, ctx->opad, blocksize,
912	req->result, ctx->digestsize, NULL, 0);
913	break;
914	case HASH_ALG_SHA256:
915	rc = do_shash("sha256", req->result, ctx->opad, blocksize,
916	req->result, ctx->digestsize, NULL, 0);
917	break;
918	case HASH_ALG_SHA384:
919	rc = do_shash("sha384", req->result, ctx->opad, blocksize,
920	req->result, ctx->digestsize, NULL, 0);
921	break;
922	case HASH_ALG_SHA512:
923	rc = do_shash("sha512", req->result, ctx->opad, blocksize,
924	req->result, ctx->digestsize, NULL, 0);
925	break;
926	default:
927	pr_err("%s() Error : unknown hmac type\n", __func__);
928	rc = -EINVAL;
929	}
930	return rc;
931	}
932
933	/**
934	* ahash_req_done() - Process a hash result from the SPU hardware.
935	* @rctx: Crypto request context
936	*
937	* Return: 0 if successful
938	* < 0 if an error
939	*/
940	static int ahash_req_done(struct iproc_reqctx_s *rctx)
941	{
942	struct spu_hw *spu = &iproc_priv.spu;
943	struct crypto_async_request *areq = rctx->parent;
944	struct ahash_request *req = ahash_request_cast(areq);
945	struct iproc_ctx_s *ctx = rctx->ctx;
946	int err;
947
948	memcpy(req->result, rctx->msg_buf.digest, ctx->digestsize);
949
950	if (spu->spu_type == SPU_TYPE_SPUM) {
951	/* byte swap the output from the UPDT function to network byte
952	* order
953	*/
954	if (ctx->auth.alg == HASH_ALG_MD5) {
955	__swab32s((u32 *)req->result);
956	__swab32s(((u32 *)req->result) + 1);
957	__swab32s(((u32 *)req->result) + 2);
958	__swab32s(((u32 *)req->result) + 3);
959	__swab32s(((u32 *)req->result) + 4);
960	}
961	}
962
963	flow_dump(" digest ", req->result, ctx->digestsize);
964
965	/* if this an HMAC then do the outer hash */
966	if (rctx->is_sw_hmac) {
967	err = spu_hmac_outer_hash(req, ctx);
968	if (err < 0)
969	return err;
970	flow_dump(" hmac: ", req->result, ctx->digestsize);
971	}
972
973	if (rctx->is_sw_hmac \|\| ctx->auth.mode == HASH_MODE_HMAC) {
974	atomic_inc(&iproc_priv.op_counts[SPU_OP_HMAC]);
975	atomic_inc(&iproc_priv.hmac_cnt[ctx->auth.alg]);
976	} else {
977	atomic_inc(&iproc_priv.op_counts[SPU_OP_HASH]);
978	atomic_inc(&iproc_priv.hash_cnt[ctx->auth.alg]);
979	}
980
981	return 0;
982	}
983
984	/**
985	* handle_ahash_resp() - Process a SPU response message for a hash request.
986	* Checks if the entire crypto API request has been processed, and if so,
987	* invokes post processing on the result.
988	* @rctx: Crypto request context
989	*/
990	static void handle_ahash_resp(struct iproc_reqctx_s *rctx)
991	{
992	struct iproc_ctx_s *ctx = rctx->ctx;
993	struct crypto_async_request *areq = rctx->parent;
994	struct ahash_request *req = ahash_request_cast(areq);
995	struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
996	unsigned int blocksize =
997	crypto_tfm_alg_blocksize(crypto_ahash_tfm(ahash));
998	/*
999	* Save hash to use as input to next op if incremental. Might be copying
1000	* too much, but that's easier than figuring out actual digest size here
1001	*/
1002	memcpy(rctx->incr_hash, rctx->msg_buf.digest, MAX_DIGEST_SIZE);
1003
1004	flow_log("%s() blocksize:%u digestsize:%u\n",
1005	__func__, blocksize, ctx->digestsize);
1006
1007	atomic64_add(ctx->digestsize, &iproc_priv.bytes_in);
1008
1009	if (rctx->is_final && (rctx->total_sent == rctx->total_todo))
1010	ahash_req_done(rctx);
1011	}
1012
1013	/**
1014	* spu_aead_rx_sg_create() - Build up the scatterlist of buffers used to receive
1015	* a SPU response message for an AEAD request. Includes buffers to catch SPU
1016	* message headers and the response data.
1017	* @mssg: mailbox message containing the receive sg
1018	* @req: Crypto API request
1019	* @rctx: crypto request context
1020	* @rx_frag_num: number of scatterlist elements required to hold the
1021	* SPU response message
1022	* @assoc_len: Length of associated data included in the crypto request
1023	* @ret_iv_len: Length of IV returned in response
1024	* @resp_len: Number of bytes of response data expected to be written to
1025	* dst buffer from crypto API
1026	* @digestsize: Length of hash digest, in bytes
1027	* @stat_pad_len: Number of bytes required to pad the STAT field to
1028	* a 4-byte boundary
1029	*
1030	* The scatterlist that gets allocated here is freed in spu_chunk_cleanup()
1031	* when the request completes, whether the request is handled successfully or
1032	* there is an error.
1033	*
1034	* Returns:
1035	* 0 if successful
1036	* < 0 if an error
1037	*/
1038	static int spu_aead_rx_sg_create(struct brcm_message *mssg,
1039	struct aead_request *req,
1040	struct iproc_reqctx_s *rctx,
1041	u8 rx_frag_num,
1042	unsigned int assoc_len,
1043	u32 ret_iv_len, unsigned int resp_len,
1044	unsigned int digestsize, u32 stat_pad_len)
1045	{
1046	struct spu_hw *spu = &iproc_priv.spu;
1047	struct scatterlist sg; / used to build sgs in mbox message */
1048	struct iproc_ctx_s *ctx = rctx->ctx;
1049	u32 datalen; /* Number of bytes of response data expected */
1050	u32 assoc_buf_len;
1051	u8 data_padlen = 0;
1052
1053	if (ctx->is_rfc4543) {
1054	/* RFC4543: only pad after data, not after AAD */
1055	data_padlen = spu->spu_gcm_ccm_pad_len(ctx->cipher.mode,
1056	assoc_len + resp_len);
1057	assoc_buf_len = assoc_len;
1058	} else {
1059	data_padlen = spu->spu_gcm_ccm_pad_len(ctx->cipher.mode,
1060	resp_len);
1061	assoc_buf_len = spu->spu_assoc_resp_len(ctx->cipher.mode,
1062	assoc_len, ret_iv_len,
1063	rctx->is_encrypt);
1064	}
1065
1066	if (ctx->cipher.mode == CIPHER_MODE_CCM)
1067	/* ICV (after data) must be in the next 32-bit word for CCM */
1068	data_padlen += spu->spu_wordalign_padlen(assoc_buf_len +
1069	resp_len +
1070	data_padlen);
1071
1072	if (data_padlen)
1073	/* have to catch gcm pad in separate buffer */
1074	rx_frag_num++;
1075
1076	mssg->spu.dst = kmalloc_objs(struct scatterlist, rx_frag_num, rctx->gfp);
1077	if (!mssg->spu.dst)
1078	return -ENOMEM;
1079
1080	sg = mssg->spu.dst;
1081	sg_init_table(sg, rx_frag_num);
1082
1083	/* Space for SPU message header */
1084	sg_set_buf(sg++, rctx->msg_buf.spu_resp_hdr, ctx->spu_resp_hdr_len);
1085
1086	if (assoc_buf_len) {
1087	/*
1088	* Don't write directly to req->dst, because SPU may pad the
1089	* assoc data in the response
1090	*/
1091	memset(rctx->msg_buf.a.resp_aad, 0, assoc_buf_len);
1092	sg_set_buf(sg++, rctx->msg_buf.a.resp_aad, assoc_buf_len);
1093	}
1094
1095	if (resp_len) {
1096	/*
1097	* Copy in each dst sg entry from request, up to chunksize.
1098	* dst sg catches just the data. digest caught in separate buf.
1099	*/
1100	datalen = spu_msg_sg_add(&sg, &rctx->dst_sg, &rctx->dst_skip,
1101	rctx->dst_nents, resp_len);
1102	if (datalen < (resp_len)) {
1103	pr_err("%s(): failed to copy dst sg to mbox msg. expected len %u, datalen %u",
1104	__func__, resp_len, datalen);
1105	return -EFAULT;
1106	}
1107	}
1108
1109	/* If GCM/CCM data is padded, catch padding in separate buffer */
1110	if (data_padlen) {
1111	memset(rctx->msg_buf.a.gcmpad, 0, data_padlen);
1112	sg_set_buf(sg++, rctx->msg_buf.a.gcmpad, data_padlen);
1113	}
1114
1115	/* Always catch ICV in separate buffer */
1116	sg_set_buf(sg++, rctx->msg_buf.digest, digestsize);
1117
1118	flow_log("stat_pad_len %u\n", stat_pad_len);
1119	if (stat_pad_len) {
1120	memset(rctx->msg_buf.rx_stat_pad, 0, stat_pad_len);
1121	sg_set_buf(sg++, rctx->msg_buf.rx_stat_pad, stat_pad_len);
1122	}
1123
1124	memset(rctx->msg_buf.rx_stat, 0, SPU_RX_STATUS_LEN);
1125	sg_set_buf(sg, rctx->msg_buf.rx_stat, spu->spu_rx_status_len());
1126
1127	return 0;
1128	}
1129
1130	/**
1131	* spu_aead_tx_sg_create() - Build up the scatterlist of buffers used to send a
1132	* SPU request message for an AEAD request. Includes SPU message headers and the
1133	* request data.
1134	* @mssg: mailbox message containing the transmit sg
1135	* @rctx: crypto request context
1136	* @tx_frag_num: number of scatterlist elements required to construct the
1137	* SPU request message
1138	* @spu_hdr_len: length of SPU message header in bytes
1139	* @assoc: crypto API associated data scatterlist
1140	* @assoc_len: length of associated data
1141	* @assoc_nents: number of scatterlist entries containing assoc data
1142	* @aead_iv_len: length of AEAD IV, if included
1143	* @chunksize: Number of bytes of request data
1144	* @aad_pad_len: Number of bytes of padding at end of AAD. For GCM/CCM.
1145	* @pad_len: Number of pad bytes
1146	* @incl_icv: If true, write separate ICV buffer after data and
1147	* any padding
1148	*
1149	* The scatterlist that gets allocated here is freed in spu_chunk_cleanup()
1150	* when the request completes, whether the request is handled successfully or
1151	* there is an error.
1152	*
1153	* Return:
1154	* 0 if successful
1155	* < 0 if an error
1156	*/
1157	static int spu_aead_tx_sg_create(struct brcm_message *mssg,
1158	struct iproc_reqctx_s *rctx,
1159	u8 tx_frag_num,
1160	u32 spu_hdr_len,
1161	struct scatterlist *assoc,
1162	unsigned int assoc_len,
1163	int assoc_nents,
1164	unsigned int aead_iv_len,
1165	unsigned int chunksize,
1166	u32 aad_pad_len, u32 pad_len, bool incl_icv)
1167	{
1168	struct spu_hw *spu = &iproc_priv.spu;
1169	struct scatterlist sg; / used to build sgs in mbox message */
1170	struct scatterlist *assoc_sg = assoc;
1171	struct iproc_ctx_s *ctx = rctx->ctx;
1172	u32 datalen; /* Number of bytes of data to write */
1173	u32 written; /* Number of bytes of data written */
1174	u32 assoc_offset = 0;
1175	u32 stat_len;
1176
1177	mssg->spu.src = kmalloc_objs(struct scatterlist, tx_frag_num, rctx->gfp);
1178	if (!mssg->spu.src)
1179	return -ENOMEM;
1180
1181	sg = mssg->spu.src;
1182	sg_init_table(sg, tx_frag_num);
1183
1184	sg_set_buf(sg++, rctx->msg_buf.bcm_spu_req_hdr,
1185	BCM_HDR_LEN + spu_hdr_len);
1186
1187	if (assoc_len) {
1188	/* Copy in each associated data sg entry from request */
1189	written = spu_msg_sg_add(&sg, &assoc_sg, &assoc_offset,
1190	assoc_nents, assoc_len);
1191	if (written < assoc_len) {
1192	pr_err("%s(): failed to copy assoc sg to mbox msg",
1193	__func__);
1194	return -EFAULT;
1195	}
1196	}
1197
1198	if (aead_iv_len)
1199	sg_set_buf(sg++, rctx->msg_buf.iv_ctr, aead_iv_len);
1200
1201	if (aad_pad_len) {
1202	memset(rctx->msg_buf.a.req_aad_pad, 0, aad_pad_len);
1203	sg_set_buf(sg++, rctx->msg_buf.a.req_aad_pad, aad_pad_len);
1204	}
1205
1206	datalen = chunksize;
1207	if ((chunksize > ctx->digestsize) && incl_icv)
1208	datalen -= ctx->digestsize;
1209	if (datalen) {
1210	/* For aead, a single msg should consume the entire src sg */
1211	written = spu_msg_sg_add(&sg, &rctx->src_sg, &rctx->src_skip,
1212	rctx->src_nents, datalen);
1213	if (written < datalen) {
1214	pr_err("%s(): failed to copy src sg to mbox msg",
1215	__func__);
1216	return -EFAULT;
1217	}
1218	}
1219
1220	if (pad_len) {
1221	memset(rctx->msg_buf.spu_req_pad, 0, pad_len);
1222	sg_set_buf(sg++, rctx->msg_buf.spu_req_pad, pad_len);
1223	}
1224
1225	if (incl_icv)
1226	sg_set_buf(sg++, rctx->msg_buf.digest, ctx->digestsize);
1227
1228	stat_len = spu->spu_tx_status_len();
1229	if (stat_len) {
1230	memset(rctx->msg_buf.tx_stat, 0, stat_len);
1231	sg_set_buf(sg, rctx->msg_buf.tx_stat, stat_len);
1232	}
1233	return 0;
1234	}
1235
1236	/**
1237	* handle_aead_req() - Submit a SPU request message for the next chunk of the
1238	* current AEAD request.
1239	* @rctx: Crypto request context
1240	*
1241	* Unlike other operation types, we assume the length of the request fits in
1242	* a single SPU request message. aead_enqueue() makes sure this is true.
1243	* Comments for other op types regarding threads applies here as well.
1244	*
1245	* Unlike incremental hash ops, where the spu returns the entire hash for
1246	* truncated algs like sha-224, the SPU returns just the truncated hash in
1247	* response to aead requests. So digestsize is always ctx->digestsize here.
1248	*
1249	* Return: -EINPROGRESS: crypto request has been accepted and result will be
1250	* returned asynchronously
1251	* Any other value indicates an error
1252	*/
1253	static int handle_aead_req(struct iproc_reqctx_s *rctx)
1254	{
1255	struct spu_hw *spu = &iproc_priv.spu;
1256	struct crypto_async_request *areq = rctx->parent;
1257	struct aead_request *req = container_of(areq,
1258	struct aead_request, base);
1259	struct iproc_ctx_s *ctx = rctx->ctx;
1260	int err;
1261	unsigned int chunksize;
1262	unsigned int resp_len;
1263	u32 spu_hdr_len;
1264	u32 db_size;
1265	u32 stat_pad_len;
1266	u32 pad_len;
1267	struct brcm_message mssg; / mailbox message */
1268	struct spu_request_opts req_opts;
1269	struct spu_cipher_parms cipher_parms;
1270	struct spu_hash_parms hash_parms;
1271	struct spu_aead_parms aead_parms;
1272	int assoc_nents = 0;
1273	bool incl_icv = false;
1274	unsigned int digestsize = ctx->digestsize;
1275
1276	/* number of entries in src and dst sg. Always includes SPU msg header.
1277	*/
1278	u8 rx_frag_num = 2; /* and STATUS */
1279	u8 tx_frag_num = 1;
1280
1281	/* doing the whole thing at once */
1282	chunksize = rctx->total_todo;
1283
1284	flow_log("%s: chunksize %u\n", __func__, chunksize);
1285
1286	memset(&req_opts, 0, sizeof(req_opts));
1287	memset(&hash_parms, 0, sizeof(hash_parms));
1288	memset(&aead_parms, 0, sizeof(aead_parms));
1289
1290	req_opts.is_inbound = !(rctx->is_encrypt);
1291	req_opts.auth_first = ctx->auth_first;
1292	req_opts.is_aead = true;
1293	req_opts.is_esp = ctx->is_esp;
1294
1295	cipher_parms.alg = ctx->cipher.alg;
1296	cipher_parms.mode = ctx->cipher.mode;
1297	cipher_parms.type = ctx->cipher_type;
1298	cipher_parms.key_buf = ctx->enckey;
1299	cipher_parms.key_len = ctx->enckeylen;
1300	cipher_parms.iv_buf = rctx->msg_buf.iv_ctr;
1301	cipher_parms.iv_len = rctx->iv_ctr_len;
1302
1303	hash_parms.alg = ctx->auth.alg;
1304	hash_parms.mode = ctx->auth.mode;
1305	hash_parms.type = HASH_TYPE_NONE;
1306	hash_parms.key_buf = (u8 *)ctx->authkey;
1307	hash_parms.key_len = ctx->authkeylen;
1308	hash_parms.digestsize = digestsize;
1309
1310	if ((ctx->auth.alg == HASH_ALG_SHA224) &&
1311	(ctx->authkeylen < SHA224_DIGEST_SIZE))
1312	hash_parms.key_len = SHA224_DIGEST_SIZE;
1313
1314	aead_parms.assoc_size = req->assoclen;
1315	if (ctx->is_esp && !ctx->is_rfc4543) {
1316	/*
1317	* 8-byte IV is included assoc data in request. SPU2
1318	* expects AAD to include just SPI and seqno. So
1319	* subtract off the IV len.
1320	*/
1321	aead_parms.assoc_size -= GCM_RFC4106_IV_SIZE;
1322
1323	if (rctx->is_encrypt) {
1324	aead_parms.return_iv = true;
1325	aead_parms.ret_iv_len = GCM_RFC4106_IV_SIZE;
1326	aead_parms.ret_iv_off = GCM_ESP_SALT_SIZE;
1327	}
1328	} else {
1329	aead_parms.ret_iv_len = 0;
1330	}
1331
1332	/*
1333	* Count number of sg entries from the crypto API request that are to
1334	* be included in this mailbox message. For dst sg, don't count space
1335	* for digest. Digest gets caught in a separate buffer and copied back
1336	* to dst sg when processing response.
1337	*/
1338	rctx->src_nents = spu_sg_count(rctx->src_sg, rctx->src_skip, chunksize);
1339	rctx->dst_nents = spu_sg_count(rctx->dst_sg, rctx->dst_skip, chunksize);
1340	if (aead_parms.assoc_size)
1341	assoc_nents = spu_sg_count(rctx->assoc, 0,
1342	aead_parms.assoc_size);
1343
1344	mssg = &rctx->mb_mssg;
1345
1346	rctx->total_sent = chunksize;
1347	rctx->src_sent = chunksize;
1348	if (spu->spu_assoc_resp_len(ctx->cipher.mode,
1349	aead_parms.assoc_size,
1350	aead_parms.ret_iv_len,
1351	rctx->is_encrypt))
1352	rx_frag_num++;
1353
1354	aead_parms.iv_len = spu->spu_aead_ivlen(ctx->cipher.mode,
1355	rctx->iv_ctr_len);
1356
1357	if (ctx->auth.alg == HASH_ALG_AES)
1358	hash_parms.type = (enum hash_type)ctx->cipher_type;
1359
1360	/* General case AAD padding (CCM and RFC4543 special cases below) */
1361	aead_parms.aad_pad_len = spu->spu_gcm_ccm_pad_len(ctx->cipher.mode,
1362	aead_parms.assoc_size);
1363
1364	/* General case data padding (CCM decrypt special case below) */
1365	aead_parms.data_pad_len = spu->spu_gcm_ccm_pad_len(ctx->cipher.mode,
1366	chunksize);
1367
1368	if (ctx->cipher.mode == CIPHER_MODE_CCM) {
1369	/*
1370	* for CCM, AAD len + 2 (rather than AAD len) needs to be
1371	* 128-bit aligned
1372	*/
1373	aead_parms.aad_pad_len = spu->spu_gcm_ccm_pad_len(
1374	ctx->cipher.mode,
1375	aead_parms.assoc_size + 2);
1376
1377	/*
1378	* And when decrypting CCM, need to pad without including
1379	* size of ICV which is tacked on to end of chunk
1380	*/
1381	if (!rctx->is_encrypt)
1382	aead_parms.data_pad_len =
1383	spu->spu_gcm_ccm_pad_len(ctx->cipher.mode,
1384	chunksize - digestsize);
1385
1386	/* CCM also requires software to rewrite portions of IV: */
1387	spu->spu_ccm_update_iv(digestsize, &cipher_parms, req->assoclen,
1388	chunksize, rctx->is_encrypt,
1389	ctx->is_esp);
1390	}
1391
1392	if (ctx->is_rfc4543) {
1393	/*
1394	* RFC4543: data is included in AAD, so don't pad after AAD
1395	* and pad data based on both AAD + data size
1396	*/
1397	aead_parms.aad_pad_len = 0;
1398	if (!rctx->is_encrypt)
1399	aead_parms.data_pad_len = spu->spu_gcm_ccm_pad_len(
1400	ctx->cipher.mode,
1401	aead_parms.assoc_size + chunksize -
1402	digestsize);
1403	else
1404	aead_parms.data_pad_len = spu->spu_gcm_ccm_pad_len(
1405	ctx->cipher.mode,
1406	aead_parms.assoc_size + chunksize);
1407
1408	req_opts.is_rfc4543 = true;
1409	}
1410
1411	if (spu_req_incl_icv(ctx->cipher.mode, rctx->is_encrypt)) {
1412	incl_icv = true;
1413	tx_frag_num++;
1414	/* Copy ICV from end of src scatterlist to digest buf */
1415	sg_copy_part_to_buf(req->src, rctx->msg_buf.digest, digestsize,
1416	req->assoclen + rctx->total_sent -
1417	digestsize);
1418	}
1419
1420	atomic64_add(chunksize, &iproc_priv.bytes_out);
1421
1422	flow_log("%s()-sent chunksize:%u\n", __func__, chunksize);
1423
1424	/* Prepend SPU header with type 3 BCM header */
1425	memcpy(rctx->msg_buf.bcm_spu_req_hdr, BCMHEADER, BCM_HDR_LEN);
1426
1427	spu_hdr_len = spu->spu_create_request(rctx->msg_buf.bcm_spu_req_hdr +
1428	BCM_HDR_LEN, &req_opts,
1429	&cipher_parms, &hash_parms,
1430	&aead_parms, chunksize);
1431
1432	/* Determine total length of padding. Put all padding in one buffer. */
1433	db_size = spu_real_db_size(aead_parms.assoc_size, aead_parms.iv_len, 0,
1434	chunksize, aead_parms.aad_pad_len,
1435	aead_parms.data_pad_len, 0);
1436
1437	stat_pad_len = spu->spu_wordalign_padlen(db_size);
1438
1439	if (stat_pad_len)
1440	rx_frag_num++;
1441	pad_len = aead_parms.data_pad_len + stat_pad_len;
1442	if (pad_len) {
1443	tx_frag_num++;
1444	spu->spu_request_pad(rctx->msg_buf.spu_req_pad,
1445	aead_parms.data_pad_len, 0,
1446	ctx->auth.alg, ctx->auth.mode,
1447	rctx->total_sent, stat_pad_len);
1448	}
1449
1450	spu->spu_dump_msg_hdr(rctx->msg_buf.bcm_spu_req_hdr + BCM_HDR_LEN,
1451	spu_hdr_len);
1452	dump_sg(rctx->assoc, 0, aead_parms.assoc_size);
1453	packet_dump(" aead iv: ", rctx->msg_buf.iv_ctr, aead_parms.iv_len);
1454	packet_log("BD:\n");
1455	dump_sg(rctx->src_sg, rctx->src_skip, chunksize);
1456	packet_dump(" pad: ", rctx->msg_buf.spu_req_pad, pad_len);
1457
1458	/*
1459	* Build mailbox message containing SPU request msg and rx buffers
1460	* to catch response message
1461	*/
1462	memset(mssg, 0, sizeof(*mssg));
1463	mssg->type = BRCM_MESSAGE_SPU;
1464	mssg->ctx = rctx; /* Will be returned in response */
1465
1466	/* Create rx scatterlist to catch result */
1467	rx_frag_num += rctx->dst_nents;
1468	resp_len = chunksize;
1469
1470	/*
1471	* Always catch ICV in separate buffer. Have to for GCM/CCM because of
1472	* padding. Have to for SHA-224 and other truncated SHAs because SPU
1473	* sends entire digest back.
1474	*/
1475	rx_frag_num++;
1476
1477	if (((ctx->cipher.mode == CIPHER_MODE_GCM) \|\|
1478	(ctx->cipher.mode == CIPHER_MODE_CCM)) && !rctx->is_encrypt) {
1479	/*
1480	* Input is ciphertxt plus ICV, but ICV not incl
1481	* in output.
1482	*/
1483	resp_len -= ctx->digestsize;
1484	if (resp_len == 0)
1485	/* no rx frags to catch output data */
1486	rx_frag_num -= rctx->dst_nents;
1487	}
1488
1489	err = spu_aead_rx_sg_create(mssg, req, rctx, rx_frag_num,
1490	aead_parms.assoc_size,
1491	aead_parms.ret_iv_len, resp_len, digestsize,
1492	stat_pad_len);
1493	if (err)
1494	return err;
1495
1496	/* Create tx scatterlist containing SPU request message */
1497	tx_frag_num += rctx->src_nents;
1498	tx_frag_num += assoc_nents;
1499	if (aead_parms.aad_pad_len)
1500	tx_frag_num++;
1501	if (aead_parms.iv_len)
1502	tx_frag_num++;
1503	if (spu->spu_tx_status_len())
1504	tx_frag_num++;
1505	err = spu_aead_tx_sg_create(mssg, rctx, tx_frag_num, spu_hdr_len,
1506	rctx->assoc, aead_parms.assoc_size,
1507	assoc_nents, aead_parms.iv_len, chunksize,
1508	aead_parms.aad_pad_len, pad_len, incl_icv);
1509	if (err)
1510	return err;
1511
1512	err = mailbox_send_message(mssg, req->base.flags, rctx->chan_idx);
1513	if (unlikely(err < 0))
1514	return err;
1515
1516	return -EINPROGRESS;
1517	}
1518
1519	/**
1520	* handle_aead_resp() - Process a SPU response message for an AEAD request.
1521	* @rctx: Crypto request context
1522	*/
1523	static void handle_aead_resp(struct iproc_reqctx_s *rctx)
1524	{
1525	struct spu_hw *spu = &iproc_priv.spu;
1526	struct crypto_async_request *areq = rctx->parent;
1527	struct aead_request *req = container_of(areq,
1528	struct aead_request, base);
1529	struct iproc_ctx_s *ctx = rctx->ctx;
1530	u32 payload_len;
1531	unsigned int icv_offset;
1532	u32 result_len;
1533
1534	/* See how much data was returned */
1535	payload_len = spu->spu_payload_length(rctx->msg_buf.spu_resp_hdr);
1536	flow_log("payload_len %u\n", payload_len);
1537
1538	/* only count payload */
1539	atomic64_add(payload_len, &iproc_priv.bytes_in);
1540
1541	if (req->assoclen)
1542	packet_dump(" assoc_data ", rctx->msg_buf.a.resp_aad,
1543	req->assoclen);
1544
1545	/*
1546	* Copy the ICV back to the destination
1547	* buffer. In decrypt case, SPU gives us back the digest, but crypto
1548	* API doesn't expect ICV in dst buffer.
1549	*/
1550	result_len = req->cryptlen;
1551	if (rctx->is_encrypt) {
1552	icv_offset = req->assoclen + rctx->total_sent;
1553	packet_dump(" ICV: ", rctx->msg_buf.digest, ctx->digestsize);
1554	flow_log("copying ICV to dst sg at offset %u\n", icv_offset);
1555	sg_copy_part_from_buf(req->dst, rctx->msg_buf.digest,
1556	ctx->digestsize, icv_offset);
1557	result_len += ctx->digestsize;
1558	}
1559
1560	packet_log("response data: ");
1561	dump_sg(req->dst, req->assoclen, result_len);
1562
1563	atomic_inc(&iproc_priv.op_counts[SPU_OP_AEAD]);
1564	if (ctx->cipher.alg == CIPHER_ALG_AES) {
1565	if (ctx->cipher.mode == CIPHER_MODE_CCM)
1566	atomic_inc(&iproc_priv.aead_cnt[AES_CCM]);
1567	else if (ctx->cipher.mode == CIPHER_MODE_GCM)
1568	atomic_inc(&iproc_priv.aead_cnt[AES_GCM]);
1569	else
1570	atomic_inc(&iproc_priv.aead_cnt[AUTHENC]);
1571	} else {
1572	atomic_inc(&iproc_priv.aead_cnt[AUTHENC]);
1573	}
1574	}
1575
1576	/**
1577	* spu_chunk_cleanup() - Do cleanup after processing one chunk of a request
1578	* @rctx: request context
1579	*
1580	* Mailbox scatterlists are allocated for each chunk. So free them after
1581	* processing each chunk.
1582	*/
1583	static void spu_chunk_cleanup(struct iproc_reqctx_s *rctx)
1584	{
1585	/* mailbox message used to tx request */
1586	struct brcm_message *mssg = &rctx->mb_mssg;
1587
1588	kfree(mssg->spu.src);
1589	kfree(mssg->spu.dst);
1590	memset(mssg, 0, sizeof(struct brcm_message));
1591	}
1592
1593	/**
1594	* finish_req() - Used to invoke the complete callback from the requester when
1595	* a request has been handled asynchronously.
1596	* @rctx: Request context
1597	* @err: Indicates whether the request was successful or not
1598	*
1599	* Ensures that cleanup has been done for request
1600	*/
1601	static void finish_req(struct iproc_reqctx_s *rctx, int err)
1602	{
1603	struct crypto_async_request *areq = rctx->parent;
1604
1605	flow_log("%s() err:%d\n\n", __func__, err);
1606
1607	/* No harm done if already called */
1608	spu_chunk_cleanup(rctx);
1609
1610	if (areq)
1611	crypto_request_complete(areq, err);
1612	}
1613
1614	/**
1615	* spu_rx_callback() - Callback from mailbox framework with a SPU response.
1616	* @cl: mailbox client structure for SPU driver
1617	* @msg: mailbox message containing SPU response
1618	*/
1619	static void spu_rx_callback(struct mbox_client cl, void msg)
1620	{
1621	struct spu_hw *spu = &iproc_priv.spu;
1622	struct brcm_message *mssg = msg;
1623	struct iproc_reqctx_s *rctx;
1624	int err;
1625
1626	rctx = mssg->ctx;
1627	if (unlikely(!rctx)) {
1628	/* This is fatal */
1629	pr_err("%s(): no request context", __func__);
1630	err = -EFAULT;
1631	goto cb_finish;
1632	}
1633
1634	/* process the SPU status */
1635	err = spu->spu_status_process(rctx->msg_buf.rx_stat);
1636	if (err != 0) {
1637	if (err == SPU_INVALID_ICV)
1638	atomic_inc(&iproc_priv.bad_icv);
1639	err = -EBADMSG;
1640	goto cb_finish;
1641	}
1642
1643	/* Process the SPU response message */
1644	switch (rctx->ctx->alg->type) {
1645	case CRYPTO_ALG_TYPE_SKCIPHER:
1646	handle_skcipher_resp(rctx);
1647	break;
1648	case CRYPTO_ALG_TYPE_AHASH:
1649	handle_ahash_resp(rctx);
1650	break;
1651	case CRYPTO_ALG_TYPE_AEAD:
1652	handle_aead_resp(rctx);
1653	break;
1654	default:
1655	err = -EINVAL;
1656	goto cb_finish;
1657	}
1658
1659	/*
1660	* If this response does not complete the request, then send the next
1661	* request chunk.
1662	*/
1663	if (rctx->total_sent < rctx->total_todo) {
1664	/* Deallocate anything specific to previous chunk */
1665	spu_chunk_cleanup(rctx);
1666
1667	switch (rctx->ctx->alg->type) {
1668	case CRYPTO_ALG_TYPE_SKCIPHER:
1669	err = handle_skcipher_req(rctx);
1670	break;
1671	case CRYPTO_ALG_TYPE_AHASH:
1672	err = handle_ahash_req(rctx);
1673	if (err == -EAGAIN)
1674	/*
1675	* we saved data in hash carry, but tell crypto
1676	* API we successfully completed request.
1677	*/
1678	err = 0;
1679	break;
1680	case CRYPTO_ALG_TYPE_AEAD:
1681	err = handle_aead_req(rctx);
1682	break;
1683	default:
1684	err = -EINVAL;
1685	}
1686
1687	if (err == -EINPROGRESS)
1688	/* Successfully submitted request for next chunk */
1689	return;
1690	}
1691
1692	cb_finish:
1693	finish_req(rctx, err);
1694	}
1695
1696	/* ==================== Kernel Cryptographic API ==================== */
1697
1698	/**
1699	* skcipher_enqueue() - Handle skcipher encrypt or decrypt request.
1700	* @req: Crypto API request
1701	* @encrypt: true if encrypting; false if decrypting
1702	*
1703	* Return: -EINPROGRESS if request accepted and result will be returned
1704	* asynchronously
1705	* < 0 if an error
1706	*/
1707	static int skcipher_enqueue(struct skcipher_request *req, bool encrypt)
1708	{
1709	struct iproc_reqctx_s *rctx = skcipher_request_ctx(req);
1710	struct iproc_ctx_s *ctx =
1711	crypto_skcipher_ctx(crypto_skcipher_reqtfm(req));
1712	int err;
1713
1714	flow_log("%s() enc:%u\n", __func__, encrypt);
1715
1716	rctx->gfp = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG \|
1717	CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC;
1718	rctx->parent = &req->base;
1719	rctx->is_encrypt = encrypt;
1720	rctx->bd_suppress = false;
1721	rctx->total_todo = req->cryptlen;
1722	rctx->src_sent = 0;
1723	rctx->total_sent = 0;
1724	rctx->total_received = 0;
1725	rctx->ctx = ctx;
1726
1727	/* Initialize current position in src and dst scatterlists */
1728	rctx->src_sg = req->src;
1729	rctx->src_nents = 0;
1730	rctx->src_skip = 0;
1731	rctx->dst_sg = req->dst;
1732	rctx->dst_nents = 0;
1733	rctx->dst_skip = 0;
1734
1735	if (ctx->cipher.mode == CIPHER_MODE_CBC \|\|
1736	ctx->cipher.mode == CIPHER_MODE_CTR \|\|
1737	ctx->cipher.mode == CIPHER_MODE_OFB \|\|
1738	ctx->cipher.mode == CIPHER_MODE_XTS \|\|
1739	ctx->cipher.mode == CIPHER_MODE_GCM \|\|
1740	ctx->cipher.mode == CIPHER_MODE_CCM) {
1741	rctx->iv_ctr_len =
1742	crypto_skcipher_ivsize(crypto_skcipher_reqtfm(req));
1743	memcpy(rctx->msg_buf.iv_ctr, req->iv, rctx->iv_ctr_len);
1744	} else {
1745	rctx->iv_ctr_len = 0;
1746	}
1747
1748	/* Choose a SPU to process this request */
1749	rctx->chan_idx = select_channel();
1750	err = handle_skcipher_req(rctx);
1751	if (err != -EINPROGRESS)
1752	/* synchronous result */
1753	spu_chunk_cleanup(rctx);
1754
1755	return err;
1756	}
1757
1758	static int des_setkey(struct crypto_skcipher cipher, const u8 key,
1759	unsigned int keylen)
1760	{
1761	struct iproc_ctx_s *ctx = crypto_skcipher_ctx(cipher);
1762	int err;
1763
1764	err = verify_skcipher_des_key(cipher, key);
1765	if (err)
1766	return err;
1767
1768	ctx->cipher_type = CIPHER_TYPE_DES;
1769	return 0;
1770	}
1771
1772	static int threedes_setkey(struct crypto_skcipher cipher, const u8 key,
1773	unsigned int keylen)
1774	{
1775	struct iproc_ctx_s *ctx = crypto_skcipher_ctx(cipher);
1776	int err;
1777
1778	err = verify_skcipher_des3_key(cipher, key);
1779	if (err)
1780	return err;
1781
1782	ctx->cipher_type = CIPHER_TYPE_3DES;
1783	return 0;
1784	}
1785
1786	static int aes_setkey(struct crypto_skcipher cipher, const u8 key,
1787	unsigned int keylen)
1788	{
1789	struct iproc_ctx_s *ctx = crypto_skcipher_ctx(cipher);
1790
1791	if (ctx->cipher.mode == CIPHER_MODE_XTS)
1792	/* XTS includes two keys of equal length */
1793	keylen = keylen / 2;
1794
1795	switch (keylen) {
1796	case AES_KEYSIZE_128:
1797	ctx->cipher_type = CIPHER_TYPE_AES128;
1798	break;
1799	case AES_KEYSIZE_192:
1800	ctx->cipher_type = CIPHER_TYPE_AES192;
1801	break;
1802	case AES_KEYSIZE_256:
1803	ctx->cipher_type = CIPHER_TYPE_AES256;
1804	break;
1805	default:
1806	return -EINVAL;
1807	}
1808	WARN_ON((ctx->max_payload != SPU_MAX_PAYLOAD_INF) &&
1809	((ctx->max_payload % AES_BLOCK_SIZE) != 0));
1810	return 0;
1811	}
1812
1813	static int skcipher_setkey(struct crypto_skcipher cipher, const u8 key,
1814	unsigned int keylen)
1815	{
1816	struct spu_hw *spu = &iproc_priv.spu;
1817	struct iproc_ctx_s *ctx = crypto_skcipher_ctx(cipher);
1818	struct spu_cipher_parms cipher_parms;
1819	u32 alloc_len = 0;
1820	int err;
1821
1822	flow_log("skcipher_setkey() keylen: %d\n", keylen);
1823	flow_dump(" key: ", key, keylen);
1824
1825	switch (ctx->cipher.alg) {
1826	case CIPHER_ALG_DES:
1827	err = des_setkey(cipher, key, keylen);
1828	break;
1829	case CIPHER_ALG_3DES:
1830	err = threedes_setkey(cipher, key, keylen);
1831	break;
1832	case CIPHER_ALG_AES:
1833	err = aes_setkey(cipher, key, keylen);
1834	break;
1835	default:
1836	pr_err("%s() Error: unknown cipher alg\n", __func__);
1837	err = -EINVAL;
1838	}
1839	if (err)
1840	return err;
1841
1842	memcpy(ctx->enckey, key, keylen);
1843	ctx->enckeylen = keylen;
1844
1845	/* SPU needs XTS keys in the reverse order the crypto API presents */
1846	if ((ctx->cipher.alg == CIPHER_ALG_AES) &&
1847	(ctx->cipher.mode == CIPHER_MODE_XTS)) {
1848	unsigned int xts_keylen = keylen / 2;
1849
1850	memcpy(ctx->enckey, key + xts_keylen, xts_keylen);
1851	memcpy(ctx->enckey + xts_keylen, key, xts_keylen);
1852	}
1853
1854	if (spu->spu_type == SPU_TYPE_SPUM)
1855	alloc_len = BCM_HDR_LEN + SPU_HEADER_ALLOC_LEN;
1856	else if (spu->spu_type == SPU_TYPE_SPU2)
1857	alloc_len = BCM_HDR_LEN + SPU2_HEADER_ALLOC_LEN;
1858	memset(ctx->bcm_spu_req_hdr, 0, alloc_len);
1859	cipher_parms.iv_buf = NULL;
1860	cipher_parms.iv_len = crypto_skcipher_ivsize(cipher);
1861	flow_log("%s: iv_len %u\n", __func__, cipher_parms.iv_len);
1862
1863	cipher_parms.alg = ctx->cipher.alg;
1864	cipher_parms.mode = ctx->cipher.mode;
1865	cipher_parms.type = ctx->cipher_type;
1866	cipher_parms.key_buf = ctx->enckey;
1867	cipher_parms.key_len = ctx->enckeylen;
1868
1869	/* Prepend SPU request message with BCM header */
1870	memcpy(ctx->bcm_spu_req_hdr, BCMHEADER, BCM_HDR_LEN);
1871	ctx->spu_req_hdr_len =
1872	spu->spu_cipher_req_init(ctx->bcm_spu_req_hdr + BCM_HDR_LEN,
1873	&cipher_parms);
1874
1875	ctx->spu_resp_hdr_len = spu->spu_response_hdr_len(ctx->authkeylen,
1876	ctx->enckeylen,
1877	false);
1878
1879	atomic_inc(&iproc_priv.setkey_cnt[SPU_OP_CIPHER]);
1880
1881	return 0;
1882	}
1883
1884	static int skcipher_encrypt(struct skcipher_request *req)
1885	{
1886	flow_log("skcipher_encrypt() nbytes:%u\n", req->cryptlen);
1887
1888	return skcipher_enqueue(req, true);
1889	}
1890
1891	static int skcipher_decrypt(struct skcipher_request *req)
1892	{
1893	flow_log("skcipher_decrypt() nbytes:%u\n", req->cryptlen);
1894	return skcipher_enqueue(req, false);
1895	}
1896
1897	static int ahash_enqueue(struct ahash_request *req)
1898	{
1899	struct iproc_reqctx_s *rctx = ahash_request_ctx(req);
1900	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
1901	struct iproc_ctx_s *ctx = crypto_ahash_ctx(tfm);
1902	int err;
1903	const char *alg_name;
1904
1905	flow_log("ahash_enqueue() nbytes:%u\n", req->nbytes);
1906
1907	rctx->gfp = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG \|
1908	CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC;
1909	rctx->parent = &req->base;
1910	rctx->ctx = ctx;
1911	rctx->bd_suppress = true;
1912	memset(&rctx->mb_mssg, 0, sizeof(struct brcm_message));
1913
1914	/* Initialize position in src scatterlist */
1915	rctx->src_sg = req->src;
1916	rctx->src_skip = 0;
1917	rctx->src_nents = 0;
1918	rctx->dst_sg = NULL;
1919	rctx->dst_skip = 0;
1920	rctx->dst_nents = 0;
1921
1922	/* SPU2 hardware does not compute hash of zero length data */
1923	if ((rctx->is_final == 1) && (rctx->total_todo == 0) &&
1924	(iproc_priv.spu.spu_type == SPU_TYPE_SPU2)) {
1925	alg_name = crypto_ahash_alg_name(tfm);
1926	flow_log("Doing %sfinal %s zero-len hash request in software\n",
1927	rctx->is_final ? "" : "non-", alg_name);
1928	err = do_shash((unsigned char *)alg_name, req->result,
1929	NULL, 0, NULL, 0, ctx->authkey,
1930	ctx->authkeylen);
1931	if (err < 0)
1932	flow_log("Hash request failed with error %d\n", err);
1933	return err;
1934	}
1935	/* Choose a SPU to process this request */
1936	rctx->chan_idx = select_channel();
1937
1938	err = handle_ahash_req(rctx);
1939	if (err != -EINPROGRESS)
1940	/* synchronous result */
1941	spu_chunk_cleanup(rctx);
1942
1943	if (err == -EAGAIN)
1944	/*
1945	* we saved data in hash carry, but tell crypto API
1946	* we successfully completed request.
1947	*/
1948	err = 0;
1949
1950	return err;
1951	}
1952
1953	static int __ahash_init(struct ahash_request *req)
1954	{
1955	struct spu_hw *spu = &iproc_priv.spu;
1956	struct iproc_reqctx_s *rctx = ahash_request_ctx(req);
1957	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
1958	struct iproc_ctx_s *ctx = crypto_ahash_ctx(tfm);
1959
1960	flow_log("%s()\n", __func__);
1961
1962	/* Initialize the context */
1963	rctx->hash_carry_len = 0;
1964	rctx->is_final = 0;
1965
1966	rctx->total_todo = 0;
1967	rctx->src_sent = 0;
1968	rctx->total_sent = 0;
1969	rctx->total_received = 0;
1970
1971	ctx->digestsize = crypto_ahash_digestsize(tfm);
1972	/* If we add a hash whose digest is larger, catch it here. */
1973	WARN_ON(ctx->digestsize > MAX_DIGEST_SIZE);
1974
1975	rctx->is_sw_hmac = false;
1976
1977	ctx->spu_resp_hdr_len = spu->spu_response_hdr_len(ctx->authkeylen, 0,
1978	true);
1979
1980	return 0;
1981	}
1982
1983	/**
1984	* spu_no_incr_hash() - Determine whether incremental hashing is supported.
1985	* @ctx: Crypto session context
1986	*
1987	* SPU-2 does not support incremental hashing (we'll have to revisit and
1988	* condition based on chip revision or device tree entry if future versions do
1989	* support incremental hash)
1990	*
1991	* SPU-M also doesn't support incremental hashing of AES-XCBC
1992	*
1993	* Return: true if incremental hashing is not supported
1994	* false otherwise
1995	*/
1996	static bool spu_no_incr_hash(struct iproc_ctx_s *ctx)
1997	{
1998	struct spu_hw *spu = &iproc_priv.spu;
1999
2000	if (spu->spu_type == SPU_TYPE_SPU2)
2001	return true;
2002
2003	if ((ctx->auth.alg == HASH_ALG_AES) &&
2004	(ctx->auth.mode == HASH_MODE_XCBC))
2005	return true;
2006
2007	/* Otherwise, incremental hashing is supported */
2008	return false;
2009	}
2010
2011	static int ahash_init(struct ahash_request *req)
2012	{
2013	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
2014	struct iproc_ctx_s *ctx = crypto_ahash_ctx(tfm);
2015	const char *alg_name;
2016	struct crypto_shash *hash;
2017	int ret;
2018	gfp_t gfp;
2019
2020	if (spu_no_incr_hash(ctx)) {
2021	/*
2022	* If we get an incremental hashing request and it's not
2023	* supported by the hardware, we need to handle it in software
2024	* by calling synchronous hash functions.
2025	*/
2026	alg_name = crypto_ahash_alg_name(tfm);
2027	hash = crypto_alloc_shash(alg_name, 0, 0);
2028	if (IS_ERR(hash)) {
2029	ret = PTR_ERR(hash);
2030	goto err;
2031	}
2032
2033	gfp = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG \|
2034	CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC;
2035	ctx->shash = kmalloc(sizeof(*ctx->shash) +
2036	crypto_shash_descsize(hash), gfp);
2037	if (!ctx->shash) {
2038	ret = -ENOMEM;
2039	goto err_hash;
2040	}
2041	ctx->shash->tfm = hash;
2042
2043	/* Set the key using data we already have from setkey */
2044	if (ctx->authkeylen > 0) {
2045	ret = crypto_shash_setkey(hash, ctx->authkey,
2046	ctx->authkeylen);
2047	if (ret)
2048	goto err_shash;
2049	}
2050
2051	/* Initialize hash w/ this key and other params */
2052	ret = crypto_shash_init(ctx->shash);
2053	if (ret)
2054	goto err_shash;
2055	} else {
2056	/* Otherwise call the internal function which uses SPU hw */
2057	ret = __ahash_init(req);
2058	}
2059
2060	return ret;
2061
2062	err_shash:
2063	kfree(ctx->shash);
2064	err_hash:
2065	crypto_free_shash(hash);
2066	err:
2067	return ret;
2068	}
2069
2070	static int __ahash_update(struct ahash_request *req)
2071	{
2072	struct iproc_reqctx_s *rctx = ahash_request_ctx(req);
2073
2074	flow_log("ahash_update() nbytes:%u\n", req->nbytes);
2075
2076	if (!req->nbytes)
2077	return 0;
2078	rctx->total_todo += req->nbytes;
2079	rctx->src_sent = 0;
2080
2081	return ahash_enqueue(req);
2082	}
2083
2084	static int ahash_update(struct ahash_request *req)
2085	{
2086	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
2087	struct iproc_ctx_s *ctx = crypto_ahash_ctx(tfm);
2088	u8 *tmpbuf;
2089	int ret;
2090	int nents;
2091	gfp_t gfp;
2092
2093	if (spu_no_incr_hash(ctx)) {
2094	/*
2095	* If we get an incremental hashing request and it's not
2096	* supported by the hardware, we need to handle it in software
2097	* by calling synchronous hash functions.
2098	*/
2099	if (req->src)
2100	nents = sg_nents(req->src);
2101	else
2102	return -EINVAL;
2103
2104	/* Copy data from req scatterlist to tmp buffer */
2105	gfp = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG \|
2106	CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC;
2107	tmpbuf = kmalloc(req->nbytes, gfp);
2108	if (!tmpbuf)
2109	return -ENOMEM;
2110
2111	if (sg_copy_to_buffer(req->src, nents, tmpbuf, req->nbytes) !=
2112	req->nbytes) {
2113	kfree(tmpbuf);
2114	return -EINVAL;
2115	}
2116
2117	/* Call synchronous update */
2118	ret = crypto_shash_update(ctx->shash, tmpbuf, req->nbytes);
2119	kfree(tmpbuf);
2120	} else {
2121	/* Otherwise call the internal function which uses SPU hw */
2122	ret = __ahash_update(req);
2123	}
2124
2125	return ret;
2126	}
2127
2128	static int __ahash_final(struct ahash_request *req)
2129	{
2130	struct iproc_reqctx_s *rctx = ahash_request_ctx(req);
2131
2132	flow_log("ahash_final() nbytes:%u\n", req->nbytes);
2133
2134	rctx->is_final = 1;
2135
2136	return ahash_enqueue(req);
2137	}
2138
2139	static int ahash_final(struct ahash_request *req)
2140	{
2141	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
2142	struct iproc_ctx_s *ctx = crypto_ahash_ctx(tfm);
2143	int ret;
2144
2145	if (spu_no_incr_hash(ctx)) {
2146	/*
2147	* If we get an incremental hashing request and it's not
2148	* supported by the hardware, we need to handle it in software
2149	* by calling synchronous hash functions.
2150	*/
2151	ret = crypto_shash_final(ctx->shash, req->result);
2152
2153	/* Done with hash, can deallocate it now */
2154	crypto_free_shash(ctx->shash->tfm);
2155	kfree(ctx->shash);
2156
2157	} else {
2158	/* Otherwise call the internal function which uses SPU hw */
2159	ret = __ahash_final(req);
2160	}
2161
2162	return ret;
2163	}
2164
2165	static int __ahash_finup(struct ahash_request *req)
2166	{
2167	struct iproc_reqctx_s *rctx = ahash_request_ctx(req);
2168
2169	flow_log("ahash_finup() nbytes:%u\n", req->nbytes);
2170
2171	rctx->total_todo += req->nbytes;
2172	rctx->src_sent = 0;
2173	rctx->is_final = 1;
2174
2175	return ahash_enqueue(req);
2176	}
2177
2178	static int ahash_finup(struct ahash_request *req)
2179	{
2180	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
2181	struct iproc_ctx_s *ctx = crypto_ahash_ctx(tfm);
2182	u8 *tmpbuf;
2183	int ret;
2184	int nents;
2185	gfp_t gfp;
2186
2187	if (spu_no_incr_hash(ctx)) {
2188	/*
2189	* If we get an incremental hashing request and it's not
2190	* supported by the hardware, we need to handle it in software
2191	* by calling synchronous hash functions.
2192	*/
2193	if (req->src) {
2194	nents = sg_nents(req->src);
2195	} else {
2196	ret = -EINVAL;
2197	goto ahash_finup_exit;
2198	}
2199
2200	/* Copy data from req scatterlist to tmp buffer */
2201	gfp = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG \|
2202	CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC;
2203	tmpbuf = kmalloc(req->nbytes, gfp);
2204	if (!tmpbuf) {
2205	ret = -ENOMEM;
2206	goto ahash_finup_exit;
2207	}
2208
2209	if (sg_copy_to_buffer(req->src, nents, tmpbuf, req->nbytes) !=
2210	req->nbytes) {
2211	ret = -EINVAL;
2212	goto ahash_finup_free;
2213	}
2214
2215	/* Call synchronous update */
2216	ret = crypto_shash_finup(ctx->shash, tmpbuf, req->nbytes,
2217	req->result);
2218	} else {
2219	/* Otherwise call the internal function which uses SPU hw */
2220	return __ahash_finup(req);
2221	}
2222	ahash_finup_free:
2223	kfree(tmpbuf);
2224
2225	ahash_finup_exit:
2226	/* Done with hash, can deallocate it now */
2227	crypto_free_shash(ctx->shash->tfm);
2228	kfree(ctx->shash);
2229	return ret;
2230	}
2231
2232	static int ahash_digest(struct ahash_request *req)
2233	{
2234	int err;
2235
2236	flow_log("ahash_digest() nbytes:%u\n", req->nbytes);
2237
2238	/* whole thing at once */
2239	err = __ahash_init(req);
2240	if (!err)
2241	err = __ahash_finup(req);
2242
2243	return err;
2244	}
2245
2246	static int ahash_setkey(struct crypto_ahash ahash, const u8 key,
2247	unsigned int keylen)
2248	{
2249	struct iproc_ctx_s *ctx = crypto_ahash_ctx(ahash);
2250
2251	flow_log("%s() ahash:%p key:%p keylen:%u\n",
2252	__func__, ahash, key, keylen);
2253	flow_dump(" key: ", key, keylen);
2254
2255	if (ctx->auth.alg == HASH_ALG_AES) {
2256	switch (keylen) {
2257	case AES_KEYSIZE_128:
2258	ctx->cipher_type = CIPHER_TYPE_AES128;
2259	break;
2260	case AES_KEYSIZE_192:
2261	ctx->cipher_type = CIPHER_TYPE_AES192;
2262	break;
2263	case AES_KEYSIZE_256:
2264	ctx->cipher_type = CIPHER_TYPE_AES256;
2265	break;
2266	default:
2267	pr_err("%s() Error: Invalid key length\n", __func__);
2268	return -EINVAL;
2269	}
2270	} else {
2271	pr_err("%s() Error: unknown hash alg\n", __func__);
2272	return -EINVAL;
2273	}
2274	memcpy(ctx->authkey, key, keylen);
2275	ctx->authkeylen = keylen;
2276
2277	return 0;
2278	}
2279
2280	static int ahash_export(struct ahash_request req, void out)
2281	{
2282	const struct iproc_reqctx_s *rctx = ahash_request_ctx(req);
2283	struct spu_hash_export_s spu_exp = (struct spu_hash_export_s )out;
2284
2285	spu_exp->total_todo = rctx->total_todo;
2286	spu_exp->total_sent = rctx->total_sent;
2287	spu_exp->is_sw_hmac = rctx->is_sw_hmac;
2288	memcpy(spu_exp->hash_carry, rctx->hash_carry, sizeof(rctx->hash_carry));
2289	spu_exp->hash_carry_len = rctx->hash_carry_len;
2290	memcpy(spu_exp->incr_hash, rctx->incr_hash, sizeof(rctx->incr_hash));
2291
2292	return 0;
2293	}
2294
2295	static int ahash_import(struct ahash_request req, const void in)
2296	{
2297	struct iproc_reqctx_s *rctx = ahash_request_ctx(req);
2298	struct spu_hash_export_s spu_exp = (struct spu_hash_export_s )in;
2299
2300	rctx->total_todo = spu_exp->total_todo;
2301	rctx->total_sent = spu_exp->total_sent;
2302	rctx->is_sw_hmac = spu_exp->is_sw_hmac;
2303	memcpy(rctx->hash_carry, spu_exp->hash_carry, sizeof(rctx->hash_carry));
2304	rctx->hash_carry_len = spu_exp->hash_carry_len;
2305	memcpy(rctx->incr_hash, spu_exp->incr_hash, sizeof(rctx->incr_hash));
2306
2307	return 0;
2308	}
2309
2310	static int ahash_hmac_setkey(struct crypto_ahash ahash, const u8 key,
2311	unsigned int keylen)
2312	{
2313	struct iproc_ctx_s *ctx = crypto_ahash_ctx(ahash);
2314	unsigned int blocksize =
2315	crypto_tfm_alg_blocksize(crypto_ahash_tfm(ahash));
2316	unsigned int digestsize = crypto_ahash_digestsize(ahash);
2317	unsigned int index;
2318	int rc;
2319
2320	flow_log("%s() ahash:%p key:%p keylen:%u blksz:%u digestsz:%u\n",
2321	__func__, ahash, key, keylen, blocksize, digestsize);
2322	flow_dump(" key: ", key, keylen);
2323
2324	if (keylen > blocksize) {
2325	switch (ctx->auth.alg) {
2326	case HASH_ALG_MD5:
2327	rc = do_shash("md5", ctx->authkey, key, keylen, NULL,
2328	0, NULL, 0);
2329	break;
2330	case HASH_ALG_SHA1:
2331	rc = do_shash("sha1", ctx->authkey, key, keylen, NULL,
2332	0, NULL, 0);
2333	break;
2334	case HASH_ALG_SHA224:
2335	rc = do_shash("sha224", ctx->authkey, key, keylen, NULL,
2336	0, NULL, 0);
2337	break;
2338	case HASH_ALG_SHA256:
2339	rc = do_shash("sha256", ctx->authkey, key, keylen, NULL,
2340	0, NULL, 0);
2341	break;
2342	case HASH_ALG_SHA384:
2343	rc = do_shash("sha384", ctx->authkey, key, keylen, NULL,
2344	0, NULL, 0);
2345	break;
2346	case HASH_ALG_SHA512:
2347	rc = do_shash("sha512", ctx->authkey, key, keylen, NULL,
2348	0, NULL, 0);
2349	break;
2350	case HASH_ALG_SHA3_224:
2351	rc = do_shash("sha3-224", ctx->authkey, key, keylen,
2352	NULL, 0, NULL, 0);
2353	break;
2354	case HASH_ALG_SHA3_256:
2355	rc = do_shash("sha3-256", ctx->authkey, key, keylen,
2356	NULL, 0, NULL, 0);
2357	break;
2358	case HASH_ALG_SHA3_384:
2359	rc = do_shash("sha3-384", ctx->authkey, key, keylen,
2360	NULL, 0, NULL, 0);
2361	break;
2362	case HASH_ALG_SHA3_512:
2363	rc = do_shash("sha3-512", ctx->authkey, key, keylen,
2364	NULL, 0, NULL, 0);
2365	break;
2366	default:
2367	pr_err("%s() Error: unknown hash alg\n", __func__);
2368	return -EINVAL;
2369	}
2370	if (rc < 0) {
2371	pr_err("%s() Error %d computing shash for %s\n",
2372	__func__, rc, hash_alg_name[ctx->auth.alg]);
2373	return rc;
2374	}
2375	ctx->authkeylen = digestsize;
2376
2377	flow_log(" keylen > digestsize... hashed\n");
2378	flow_dump(" newkey: ", ctx->authkey, ctx->authkeylen);
2379	} else {
2380	memcpy(ctx->authkey, key, keylen);
2381	ctx->authkeylen = keylen;
2382	}
2383
2384	/*
2385	* Full HMAC operation in SPUM is not verified,
2386	* So keeping the generation of IPAD, OPAD and
2387	* outer hashing in software.
2388	*/
2389	if (iproc_priv.spu.spu_type == SPU_TYPE_SPUM) {
2390	memcpy_and_pad(ctx->ipad, blocksize, ctx->authkey, ctx->authkeylen, 0);
2391	ctx->authkeylen = 0;
2392	unsafe_memcpy(ctx->opad, ctx->ipad, blocksize,
2393	"fortified memcpy causes -Wrestrict warning");
2394
2395	for (index = 0; index < blocksize; index++) {
2396	ctx->ipad[index] ^= HMAC_IPAD_VALUE;
2397	ctx->opad[index] ^= HMAC_OPAD_VALUE;
2398	}
2399
2400	flow_dump(" ipad: ", ctx->ipad, blocksize);
2401	flow_dump(" opad: ", ctx->opad, blocksize);
2402	}
2403	ctx->digestsize = digestsize;
2404	atomic_inc(&iproc_priv.setkey_cnt[SPU_OP_HMAC]);
2405
2406	return 0;
2407	}
2408
2409	static int ahash_hmac_init(struct ahash_request *req)
2410	{
2411	int ret;
2412	struct iproc_reqctx_s *rctx = ahash_request_ctx(req);
2413	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
2414	struct iproc_ctx_s *ctx = crypto_ahash_ctx(tfm);
2415	unsigned int blocksize =
2416	crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm));
2417
2418	flow_log("ahash_hmac_init()\n");
2419
2420	/* init the context as a hash */
2421	ret = ahash_init(req);
2422	if (ret)
2423	return ret;
2424
2425	if (!spu_no_incr_hash(ctx)) {
2426	/* SPU-M can do incr hashing but needs sw for outer HMAC */
2427	rctx->is_sw_hmac = true;
2428	ctx->auth.mode = HASH_MODE_HASH;
2429	/* start with a prepended ipad */
2430	memcpy(rctx->hash_carry, ctx->ipad, blocksize);
2431	rctx->hash_carry_len = blocksize;
2432	rctx->total_todo += blocksize;
2433	}
2434
2435	return 0;
2436	}
2437
2438	static int ahash_hmac_update(struct ahash_request *req)
2439	{
2440	flow_log("ahash_hmac_update() nbytes:%u\n", req->nbytes);
2441
2442	if (!req->nbytes)
2443	return 0;
2444
2445	return ahash_update(req);
2446	}
2447
2448	static int ahash_hmac_final(struct ahash_request *req)
2449	{
2450	flow_log("ahash_hmac_final() nbytes:%u\n", req->nbytes);
2451
2452	return ahash_final(req);
2453	}
2454
2455	static int ahash_hmac_finup(struct ahash_request *req)
2456	{
2457	flow_log("ahash_hmac_finupl() nbytes:%u\n", req->nbytes);
2458
2459	return ahash_finup(req);
2460	}
2461
2462	static int ahash_hmac_digest(struct ahash_request *req)
2463	{
2464	struct iproc_reqctx_s *rctx = ahash_request_ctx(req);
2465	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
2466	struct iproc_ctx_s *ctx = crypto_ahash_ctx(tfm);
2467	unsigned int blocksize =
2468	crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm));
2469
2470	flow_log("ahash_hmac_digest() nbytes:%u\n", req->nbytes);
2471
2472	/* Perform initialization and then call finup */
2473	__ahash_init(req);
2474
2475	if (iproc_priv.spu.spu_type == SPU_TYPE_SPU2) {
2476	/*
2477	* SPU2 supports full HMAC implementation in the
2478	* hardware, need not to generate IPAD, OPAD and
2479	* outer hash in software.
2480	* Only for hash key len > hash block size, SPU2
2481	* expects to perform hashing on the key, shorten
2482	* it to digest size and feed it as hash key.
2483	*/
2484	rctx->is_sw_hmac = false;
2485	ctx->auth.mode = HASH_MODE_HMAC;
2486	} else {
2487	rctx->is_sw_hmac = true;
2488	ctx->auth.mode = HASH_MODE_HASH;
2489	/* start with a prepended ipad */
2490	memcpy(rctx->hash_carry, ctx->ipad, blocksize);
2491	rctx->hash_carry_len = blocksize;
2492	rctx->total_todo += blocksize;
2493	}
2494
2495	return __ahash_finup(req);
2496	}
2497
2498	/* aead helpers */
2499
2500	static int aead_need_fallback(struct aead_request *req)
2501	{
2502	struct iproc_reqctx_s *rctx = aead_request_ctx(req);
2503	struct spu_hw *spu = &iproc_priv.spu;
2504	struct crypto_aead *aead = crypto_aead_reqtfm(req);
2505	struct iproc_ctx_s *ctx = crypto_aead_ctx(aead);
2506	u32 payload_len;
2507
2508	/*
2509	* SPU hardware cannot handle the AES-GCM/CCM case where plaintext
2510	* and AAD are both 0 bytes long. So use fallback in this case.
2511	*/
2512	if (((ctx->cipher.mode == CIPHER_MODE_GCM) \|\|
2513	(ctx->cipher.mode == CIPHER_MODE_CCM)) &&
2514	(req->assoclen == 0)) {
2515	if ((rctx->is_encrypt && (req->cryptlen == 0)) \|\|
2516	(!rctx->is_encrypt && (req->cryptlen == ctx->digestsize))) {
2517	flow_log("AES GCM/CCM needs fallback for 0 len req\n");
2518	return 1;
2519	}
2520	}
2521
2522	/* SPU-M hardware only supports CCM digest size of 8, 12, or 16 bytes */
2523	if ((ctx->cipher.mode == CIPHER_MODE_CCM) &&
2524	(spu->spu_type == SPU_TYPE_SPUM) &&
2525	(ctx->digestsize != 8) && (ctx->digestsize != 12) &&
2526	(ctx->digestsize != 16)) {
2527	flow_log("%s() AES CCM needs fallback for digest size %d\n",
2528	__func__, ctx->digestsize);
2529	return 1;
2530	}
2531
2532	/*
2533	* SPU-M on NSP has an issue where AES-CCM hash is not correct
2534	* when AAD size is 0
2535	*/
2536	if ((ctx->cipher.mode == CIPHER_MODE_CCM) &&
2537	(spu->spu_subtype == SPU_SUBTYPE_SPUM_NSP) &&
2538	(req->assoclen == 0)) {
2539	flow_log("%s() AES_CCM needs fallback for 0 len AAD on NSP\n",
2540	__func__);
2541	return 1;
2542	}
2543
2544	/*
2545	* RFC4106 and RFC4543 cannot handle the case where AAD is other than
2546	* 16 or 20 bytes long. So use fallback in this case.
2547	*/
2548	if (ctx->cipher.mode == CIPHER_MODE_GCM &&
2549	ctx->cipher.alg == CIPHER_ALG_AES &&
2550	rctx->iv_ctr_len == GCM_RFC4106_IV_SIZE &&
2551	req->assoclen != 16 && req->assoclen != 20) {
2552	flow_log("RFC4106/RFC4543 needs fallback for assoclen"
2553	" other than 16 or 20 bytes\n");
2554	return 1;
2555	}
2556
2557	payload_len = req->cryptlen;
2558	if (spu->spu_type == SPU_TYPE_SPUM)
2559	payload_len += req->assoclen;
2560
2561	flow_log("%s() payload len: %u\n", __func__, payload_len);
2562
2563	if (ctx->max_payload == SPU_MAX_PAYLOAD_INF)
2564	return 0;
2565	else
2566	return payload_len > ctx->max_payload;
2567	}
2568
2569	static int aead_do_fallback(struct aead_request *req, bool is_encrypt)
2570	{
2571	struct crypto_aead *aead = crypto_aead_reqtfm(req);
2572	struct crypto_tfm *tfm = crypto_aead_tfm(aead);
2573	struct iproc_reqctx_s *rctx = aead_request_ctx(req);
2574	struct iproc_ctx_s *ctx = crypto_tfm_ctx(tfm);
2575	struct aead_request *subreq;
2576
2577	flow_log("%s() enc:%u\n", __func__, is_encrypt);
2578
2579	if (!ctx->fallback_cipher)
2580	return -EINVAL;
2581
2582	subreq = &rctx->req;
2583	aead_request_set_tfm(subreq, ctx->fallback_cipher);
2584	aead_request_set_callback(subreq, aead_request_flags(req),
2585	req->base.complete, req->base.data);
2586	aead_request_set_crypt(subreq, req->src, req->dst, req->cryptlen,
2587	req->iv);
2588	aead_request_set_ad(subreq, req->assoclen);
2589
2590	return is_encrypt ? crypto_aead_encrypt(req) :
2591	crypto_aead_decrypt(req);
2592	}
2593
2594	static int aead_enqueue(struct aead_request *req, bool is_encrypt)
2595	{
2596	struct iproc_reqctx_s *rctx = aead_request_ctx(req);
2597	struct crypto_aead *aead = crypto_aead_reqtfm(req);
2598	struct iproc_ctx_s *ctx = crypto_aead_ctx(aead);
2599	int err;
2600
2601	flow_log("%s() enc:%u\n", __func__, is_encrypt);
2602
2603	if (req->assoclen > MAX_ASSOC_SIZE) {
2604	pr_err
2605	("%s() Error: associated data too long. (%u > %u bytes)\n",
2606	__func__, req->assoclen, MAX_ASSOC_SIZE);
2607	return -EINVAL;
2608	}
2609
2610	rctx->gfp = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG \|
2611	CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC;
2612	rctx->parent = &req->base;
2613	rctx->is_encrypt = is_encrypt;
2614	rctx->bd_suppress = false;
2615	rctx->total_todo = req->cryptlen;
2616	rctx->src_sent = 0;
2617	rctx->total_sent = 0;
2618	rctx->total_received = 0;
2619	rctx->is_sw_hmac = false;
2620	rctx->ctx = ctx;
2621	memset(&rctx->mb_mssg, 0, sizeof(struct brcm_message));
2622
2623	/* assoc data is at start of src sg */
2624	rctx->assoc = req->src;
2625
2626	/*
2627	* Init current position in src scatterlist to be after assoc data.
2628	* src_skip set to buffer offset where data begins. (Assoc data could
2629	* end in the middle of a buffer.)
2630	*/
2631	if (spu_sg_at_offset(req->src, req->assoclen, &rctx->src_sg,
2632	&rctx->src_skip) < 0) {
2633	pr_err("%s() Error: Unable to find start of src data\n",
2634	__func__);
2635	return -EINVAL;
2636	}
2637
2638	rctx->src_nents = 0;
2639	rctx->dst_nents = 0;
2640	if (req->dst == req->src) {
2641	rctx->dst_sg = rctx->src_sg;
2642	rctx->dst_skip = rctx->src_skip;
2643	} else {
2644	/*
2645	* Expect req->dst to have room for assoc data followed by
2646	* output data and ICV, if encrypt. So initialize dst_sg
2647	* to point beyond assoc len offset.
2648	*/
2649	if (spu_sg_at_offset(req->dst, req->assoclen, &rctx->dst_sg,
2650	&rctx->dst_skip) < 0) {
2651	pr_err("%s() Error: Unable to find start of dst data\n",
2652	__func__);
2653	return -EINVAL;
2654	}
2655	}
2656
2657	if (ctx->cipher.mode == CIPHER_MODE_CBC \|\|
2658	ctx->cipher.mode == CIPHER_MODE_CTR \|\|
2659	ctx->cipher.mode == CIPHER_MODE_OFB \|\|
2660	ctx->cipher.mode == CIPHER_MODE_XTS \|\|
2661	ctx->cipher.mode == CIPHER_MODE_GCM) {
2662	rctx->iv_ctr_len =
2663	ctx->salt_len +
2664	crypto_aead_ivsize(crypto_aead_reqtfm(req));
2665	} else if (ctx->cipher.mode == CIPHER_MODE_CCM) {
2666	rctx->iv_ctr_len = CCM_AES_IV_SIZE;
2667	} else {
2668	rctx->iv_ctr_len = 0;
2669	}
2670
2671	rctx->hash_carry_len = 0;
2672
2673	flow_log(" src sg: %p\n", req->src);
2674	flow_log(" rctx->src_sg: %p, src_skip %u\n",
2675	rctx->src_sg, rctx->src_skip);
2676	flow_log(" assoc: %p, assoclen %u\n", rctx->assoc, req->assoclen);
2677	flow_log(" dst sg: %p\n", req->dst);
2678	flow_log(" rctx->dst_sg: %p, dst_skip %u\n",
2679	rctx->dst_sg, rctx->dst_skip);
2680	flow_log(" iv_ctr_len:%u\n", rctx->iv_ctr_len);
2681	flow_dump(" iv: ", req->iv, rctx->iv_ctr_len);
2682	flow_log(" authkeylen:%u\n", ctx->authkeylen);
2683	flow_log(" is_esp: %s\n", str_yes_no(ctx->is_esp));
2684
2685	if (ctx->max_payload == SPU_MAX_PAYLOAD_INF)
2686	flow_log(" max_payload infinite");
2687	else
2688	flow_log(" max_payload: %u\n", ctx->max_payload);
2689
2690	if (unlikely(aead_need_fallback(req)))
2691	return aead_do_fallback(req, is_encrypt);
2692
2693	/*
2694	* Do memory allocations for request after fallback check, because if we
2695	* do fallback, we won't call finish_req() to dealloc.
2696	*/
2697	if (rctx->iv_ctr_len) {
2698	if (ctx->salt_len)
2699	memcpy(rctx->msg_buf.iv_ctr + ctx->salt_offset,
2700	ctx->salt, ctx->salt_len);
2701	memcpy(rctx->msg_buf.iv_ctr + ctx->salt_offset + ctx->salt_len,
2702	req->iv,
2703	rctx->iv_ctr_len - ctx->salt_len - ctx->salt_offset);
2704	}
2705
2706	rctx->chan_idx = select_channel();
2707	err = handle_aead_req(rctx);
2708	if (err != -EINPROGRESS)
2709	/* synchronous result */
2710	spu_chunk_cleanup(rctx);
2711
2712	return err;
2713	}
2714
2715	static int aead_authenc_setkey(struct crypto_aead *cipher,
2716	const u8 *key, unsigned int keylen)
2717	{
2718	struct spu_hw *spu = &iproc_priv.spu;
2719	struct iproc_ctx_s *ctx = crypto_aead_ctx(cipher);
2720	struct crypto_tfm *tfm = crypto_aead_tfm(cipher);
2721	struct crypto_authenc_keys keys;
2722	int ret;
2723
2724	flow_log("%s() aead:%p key:%p keylen:%u\n", __func__, cipher, key,
2725	keylen);
2726	flow_dump(" key: ", key, keylen);
2727
2728	ret = crypto_authenc_extractkeys(&keys, key, keylen);
2729	if (ret)
2730	goto badkey;
2731
2732	if (keys.enckeylen > MAX_KEY_SIZE \|\|
2733	keys.authkeylen > MAX_KEY_SIZE)
2734	goto badkey;
2735
2736	ctx->enckeylen = keys.enckeylen;
2737	ctx->authkeylen = keys.authkeylen;
2738
2739	memcpy(ctx->enckey, keys.enckey, keys.enckeylen);
2740	/* May end up padding auth key. So make sure it's zeroed. */
2741	memset(ctx->authkey, 0, sizeof(ctx->authkey));
2742	memcpy(ctx->authkey, keys.authkey, keys.authkeylen);
2743
2744	switch (ctx->alg->cipher_info.alg) {
2745	case CIPHER_ALG_DES:
2746	if (verify_aead_des_key(cipher, keys.enckey, keys.enckeylen))
2747	return -EINVAL;
2748
2749	ctx->cipher_type = CIPHER_TYPE_DES;
2750	break;
2751	case CIPHER_ALG_3DES:
2752	if (verify_aead_des3_key(cipher, keys.enckey, keys.enckeylen))
2753	return -EINVAL;
2754
2755	ctx->cipher_type = CIPHER_TYPE_3DES;
2756	break;
2757	case CIPHER_ALG_AES:
2758	switch (ctx->enckeylen) {
2759	case AES_KEYSIZE_128:
2760	ctx->cipher_type = CIPHER_TYPE_AES128;
2761	break;
2762	case AES_KEYSIZE_192:
2763	ctx->cipher_type = CIPHER_TYPE_AES192;
2764	break;
2765	case AES_KEYSIZE_256:
2766	ctx->cipher_type = CIPHER_TYPE_AES256;
2767	break;
2768	default:
2769	goto badkey;
2770	}
2771	break;
2772	default:
2773	pr_err("%s() Error: Unknown cipher alg\n", __func__);
2774	return -EINVAL;
2775	}
2776
2777	flow_log(" enckeylen:%u authkeylen:%u\n", ctx->enckeylen,
2778	ctx->authkeylen);
2779	flow_dump(" enc: ", ctx->enckey, ctx->enckeylen);
2780	flow_dump(" auth: ", ctx->authkey, ctx->authkeylen);
2781
2782	/* setkey the fallback just in case we needto use it */
2783	if (ctx->fallback_cipher) {
2784	flow_log(" running fallback setkey()\n");
2785
2786	ctx->fallback_cipher->base.crt_flags &= ~CRYPTO_TFM_REQ_MASK;
2787	ctx->fallback_cipher->base.crt_flags \|=
2788	tfm->crt_flags & CRYPTO_TFM_REQ_MASK;
2789	ret = crypto_aead_setkey(ctx->fallback_cipher, key, keylen);
2790	if (ret)
2791	flow_log(" fallback setkey() returned:%d\n", ret);
2792	}
2793
2794	ctx->spu_resp_hdr_len = spu->spu_response_hdr_len(ctx->authkeylen,
2795	ctx->enckeylen,
2796	false);
2797
2798	atomic_inc(&iproc_priv.setkey_cnt[SPU_OP_AEAD]);
2799
2800	return ret;
2801
2802	badkey:
2803	ctx->enckeylen = 0;
2804	ctx->authkeylen = 0;
2805	ctx->digestsize = 0;
2806
2807	return -EINVAL;
2808	}
2809
2810	static int aead_gcm_ccm_setkey(struct crypto_aead *cipher,
2811	const u8 *key, unsigned int keylen)
2812	{
2813	struct spu_hw *spu = &iproc_priv.spu;
2814	struct iproc_ctx_s *ctx = crypto_aead_ctx(cipher);
2815	struct crypto_tfm *tfm = crypto_aead_tfm(cipher);
2816
2817	int ret = 0;
2818
2819	flow_log("%s() keylen:%u\n", __func__, keylen);
2820	flow_dump(" key: ", key, keylen);
2821
2822	if (!ctx->is_esp)
2823	ctx->digestsize = keylen;
2824
2825	ctx->enckeylen = keylen;
2826	ctx->authkeylen = 0;
2827
2828	switch (ctx->enckeylen) {
2829	case AES_KEYSIZE_128:
2830	ctx->cipher_type = CIPHER_TYPE_AES128;
2831	break;
2832	case AES_KEYSIZE_192:
2833	ctx->cipher_type = CIPHER_TYPE_AES192;
2834	break;
2835	case AES_KEYSIZE_256:
2836	ctx->cipher_type = CIPHER_TYPE_AES256;
2837	break;
2838	default:
2839	goto badkey;
2840	}
2841
2842	memcpy(ctx->enckey, key, ctx->enckeylen);
2843
2844	flow_log(" enckeylen:%u authkeylen:%u\n", ctx->enckeylen,
2845	ctx->authkeylen);
2846	flow_dump(" enc: ", ctx->enckey, ctx->enckeylen);
2847	flow_dump(" auth: ", ctx->authkey, ctx->authkeylen);
2848
2849	/* setkey the fallback just in case we need to use it */
2850	if (ctx->fallback_cipher) {
2851	flow_log(" running fallback setkey()\n");
2852
2853	ctx->fallback_cipher->base.crt_flags &= ~CRYPTO_TFM_REQ_MASK;
2854	ctx->fallback_cipher->base.crt_flags \|=
2855	tfm->crt_flags & CRYPTO_TFM_REQ_MASK;
2856	ret = crypto_aead_setkey(ctx->fallback_cipher, key,
2857	keylen + ctx->salt_len);
2858	if (ret)
2859	flow_log(" fallback setkey() returned:%d\n", ret);
2860	}
2861
2862	ctx->spu_resp_hdr_len = spu->spu_response_hdr_len(ctx->authkeylen,
2863	ctx->enckeylen,
2864	false);
2865
2866	atomic_inc(&iproc_priv.setkey_cnt[SPU_OP_AEAD]);
2867
2868	flow_log(" enckeylen:%u authkeylen:%u\n", ctx->enckeylen,
2869	ctx->authkeylen);
2870
2871	return ret;
2872
2873	badkey:
2874	ctx->enckeylen = 0;
2875	ctx->authkeylen = 0;
2876	ctx->digestsize = 0;
2877
2878	return -EINVAL;
2879	}
2880
2881	/**
2882	* aead_gcm_esp_setkey() - setkey() operation for ESP variant of GCM AES.
2883	* @cipher: AEAD structure
2884	* @key: Key followed by 4 bytes of salt
2885	* @keylen: Length of key plus salt, in bytes
2886	*
2887	* Extracts salt from key and stores it to be prepended to IV on each request.
2888	* Digest is always 16 bytes
2889	*
2890	* Return: Value from generic gcm setkey.
2891	*/
2892	static int aead_gcm_esp_setkey(struct crypto_aead *cipher,
2893	const u8 *key, unsigned int keylen)
2894	{
2895	struct iproc_ctx_s *ctx = crypto_aead_ctx(cipher);
2896
2897	flow_log("%s\n", __func__);
2898
2899	if (keylen < GCM_ESP_SALT_SIZE)
2900	return -EINVAL;
2901
2902	ctx->salt_len = GCM_ESP_SALT_SIZE;
2903	ctx->salt_offset = GCM_ESP_SALT_OFFSET;
2904	memcpy(ctx->salt, key + keylen - GCM_ESP_SALT_SIZE, GCM_ESP_SALT_SIZE);
2905	keylen -= GCM_ESP_SALT_SIZE;
2906	ctx->digestsize = GCM_ESP_DIGESTSIZE;
2907	ctx->is_esp = true;
2908	flow_dump("salt: ", ctx->salt, GCM_ESP_SALT_SIZE);
2909
2910	return aead_gcm_ccm_setkey(cipher, key, keylen);
2911	}
2912
2913	/**
2914	* rfc4543_gcm_esp_setkey() - setkey operation for RFC4543 variant of GCM/GMAC.
2915	* @cipher: AEAD structure
2916	* @key: Key followed by 4 bytes of salt
2917	* @keylen: Length of key plus salt, in bytes
2918	*
2919	* Extracts salt from key and stores it to be prepended to IV on each request.
2920	* Digest is always 16 bytes
2921	*
2922	* Return: Value from generic gcm setkey.
2923	*/
2924	static int rfc4543_gcm_esp_setkey(struct crypto_aead *cipher,
2925	const u8 *key, unsigned int keylen)
2926	{
2927	struct iproc_ctx_s *ctx = crypto_aead_ctx(cipher);
2928
2929	flow_log("%s\n", __func__);
2930
2931	if (keylen < GCM_ESP_SALT_SIZE)
2932	return -EINVAL;
2933
2934	ctx->salt_len = GCM_ESP_SALT_SIZE;
2935	ctx->salt_offset = GCM_ESP_SALT_OFFSET;
2936	memcpy(ctx->salt, key + keylen - GCM_ESP_SALT_SIZE, GCM_ESP_SALT_SIZE);
2937	keylen -= GCM_ESP_SALT_SIZE;
2938	ctx->digestsize = GCM_ESP_DIGESTSIZE;
2939	ctx->is_esp = true;
2940	ctx->is_rfc4543 = true;
2941	flow_dump("salt: ", ctx->salt, GCM_ESP_SALT_SIZE);
2942
2943	return aead_gcm_ccm_setkey(cipher, key, keylen);
2944	}
2945
2946	/**
2947	* aead_ccm_esp_setkey() - setkey() operation for ESP variant of CCM AES.
2948	* @cipher: AEAD structure
2949	* @key: Key followed by 4 bytes of salt
2950	* @keylen: Length of key plus salt, in bytes
2951	*
2952	* Extracts salt from key and stores it to be prepended to IV on each request.
2953	* Digest is always 16 bytes
2954	*
2955	* Return: Value from generic ccm setkey.
2956	*/
2957	static int aead_ccm_esp_setkey(struct crypto_aead *cipher,
2958	const u8 *key, unsigned int keylen)
2959	{
2960	struct iproc_ctx_s *ctx = crypto_aead_ctx(cipher);
2961
2962	flow_log("%s\n", __func__);
2963
2964	if (keylen < CCM_ESP_SALT_SIZE)
2965	return -EINVAL;
2966
2967	ctx->salt_len = CCM_ESP_SALT_SIZE;
2968	ctx->salt_offset = CCM_ESP_SALT_OFFSET;
2969	memcpy(ctx->salt, key + keylen - CCM_ESP_SALT_SIZE, CCM_ESP_SALT_SIZE);
2970	keylen -= CCM_ESP_SALT_SIZE;
2971	ctx->is_esp = true;
2972	flow_dump("salt: ", ctx->salt, CCM_ESP_SALT_SIZE);
2973
2974	return aead_gcm_ccm_setkey(cipher, key, keylen);
2975	}
2976
2977	static int aead_setauthsize(struct crypto_aead *cipher, unsigned int authsize)
2978	{
2979	struct iproc_ctx_s *ctx = crypto_aead_ctx(cipher);
2980	int ret = 0;
2981
2982	flow_log("%s() authkeylen:%u authsize:%u\n",
2983	__func__, ctx->authkeylen, authsize);
2984
2985	ctx->digestsize = authsize;
2986
2987	/* setkey the fallback just in case we needto use it */
2988	if (ctx->fallback_cipher) {
2989	flow_log(" running fallback setauth()\n");
2990
2991	ret = crypto_aead_setauthsize(ctx->fallback_cipher, authsize);
2992	if (ret)
2993	flow_log(" fallback setauth() returned:%d\n", ret);
2994	}
2995
2996	return ret;
2997	}
2998
2999	static int aead_encrypt(struct aead_request *req)
3000	{
3001	flow_log("%s() cryptlen:%u %08x\n", __func__, req->cryptlen,
3002	req->cryptlen);
3003	dump_sg(req->src, 0, req->cryptlen + req->assoclen);
3004	flow_log(" assoc_len:%u\n", req->assoclen);
3005
3006	return aead_enqueue(req, true);
3007	}
3008
3009	static int aead_decrypt(struct aead_request *req)
3010	{
3011	flow_log("%s() cryptlen:%u\n", __func__, req->cryptlen);
3012	dump_sg(req->src, 0, req->cryptlen + req->assoclen);
3013	flow_log(" assoc_len:%u\n", req->assoclen);
3014
3015	return aead_enqueue(req, false);
3016	}
3017
3018	/* ==================== Supported Cipher Algorithms ==================== */
3019
3020	static struct iproc_alg_s driver_algs[] = {
3021	{
3022	.type = CRYPTO_ALG_TYPE_AEAD,
3023	.alg.aead = {
3024	.base = {
3025	.cra_name = "gcm(aes)",
3026	.cra_driver_name = "gcm-aes-iproc",
3027	.cra_blocksize = AES_BLOCK_SIZE,
3028	.cra_flags = CRYPTO_ALG_NEED_FALLBACK
3029	},
3030	.setkey = aead_gcm_ccm_setkey,
3031	.ivsize = GCM_AES_IV_SIZE,
3032	.maxauthsize = AES_BLOCK_SIZE,
3033	},
3034	.cipher_info = {
3035	.alg = CIPHER_ALG_AES,
3036	.mode = CIPHER_MODE_GCM,
3037	},
3038	.auth_info = {
3039	.alg = HASH_ALG_AES,
3040	.mode = HASH_MODE_GCM,
3041	},
3042	.auth_first = 0,
3043	},
3044	{
3045	.type = CRYPTO_ALG_TYPE_AEAD,
3046	.alg.aead = {
3047	.base = {
3048	.cra_name = "ccm(aes)",
3049	.cra_driver_name = "ccm-aes-iproc",
3050	.cra_blocksize = AES_BLOCK_SIZE,
3051	.cra_flags = CRYPTO_ALG_NEED_FALLBACK
3052	},
3053	.setkey = aead_gcm_ccm_setkey,
3054	.ivsize = CCM_AES_IV_SIZE,
3055	.maxauthsize = AES_BLOCK_SIZE,
3056	},
3057	.cipher_info = {
3058	.alg = CIPHER_ALG_AES,
3059	.mode = CIPHER_MODE_CCM,
3060	},
3061	.auth_info = {
3062	.alg = HASH_ALG_AES,
3063	.mode = HASH_MODE_CCM,
3064	},
3065	.auth_first = 0,
3066	},
3067	{
3068	.type = CRYPTO_ALG_TYPE_AEAD,
3069	.alg.aead = {
3070	.base = {
3071	.cra_name = "rfc4106(gcm(aes))",
3072	.cra_driver_name = "gcm-aes-esp-iproc",
3073	.cra_blocksize = AES_BLOCK_SIZE,
3074	.cra_flags = CRYPTO_ALG_NEED_FALLBACK
3075	},
3076	.setkey = aead_gcm_esp_setkey,
3077	.ivsize = GCM_RFC4106_IV_SIZE,
3078	.maxauthsize = AES_BLOCK_SIZE,
3079	},
3080	.cipher_info = {
3081	.alg = CIPHER_ALG_AES,
3082	.mode = CIPHER_MODE_GCM,
3083	},
3084	.auth_info = {
3085	.alg = HASH_ALG_AES,
3086	.mode = HASH_MODE_GCM,
3087	},
3088	.auth_first = 0,
3089	},
3090	{
3091	.type = CRYPTO_ALG_TYPE_AEAD,
3092	.alg.aead = {
3093	.base = {
3094	.cra_name = "rfc4309(ccm(aes))",
3095	.cra_driver_name = "ccm-aes-esp-iproc",
3096	.cra_blocksize = AES_BLOCK_SIZE,
3097	.cra_flags = CRYPTO_ALG_NEED_FALLBACK
3098	},
3099	.setkey = aead_ccm_esp_setkey,
3100	.ivsize = CCM_AES_IV_SIZE,
3101	.maxauthsize = AES_BLOCK_SIZE,
3102	},
3103	.cipher_info = {
3104	.alg = CIPHER_ALG_AES,
3105	.mode = CIPHER_MODE_CCM,
3106	},
3107	.auth_info = {
3108	.alg = HASH_ALG_AES,
3109	.mode = HASH_MODE_CCM,
3110	},
3111	.auth_first = 0,
3112	},
3113	{
3114	.type = CRYPTO_ALG_TYPE_AEAD,
3115	.alg.aead = {
3116	.base = {
3117	.cra_name = "rfc4543(gcm(aes))",
3118	.cra_driver_name = "gmac-aes-esp-iproc",
3119	.cra_blocksize = AES_BLOCK_SIZE,
3120	.cra_flags = CRYPTO_ALG_NEED_FALLBACK
3121	},
3122	.setkey = rfc4543_gcm_esp_setkey,
3123	.ivsize = GCM_RFC4106_IV_SIZE,
3124	.maxauthsize = AES_BLOCK_SIZE,
3125	},
3126	.cipher_info = {
3127	.alg = CIPHER_ALG_AES,
3128	.mode = CIPHER_MODE_GCM,
3129	},
3130	.auth_info = {
3131	.alg = HASH_ALG_AES,
3132	.mode = HASH_MODE_GCM,
3133	},
3134	.auth_first = 0,
3135	},
3136	{
3137	.type = CRYPTO_ALG_TYPE_AEAD,
3138	.alg.aead = {
3139	.base = {
3140	.cra_name = "authenc(hmac(md5),cbc(aes))",
3141	.cra_driver_name = "authenc-hmac-md5-cbc-aes-iproc",
3142	.cra_blocksize = AES_BLOCK_SIZE,
3143	.cra_flags = CRYPTO_ALG_NEED_FALLBACK \|
3144	CRYPTO_ALG_ASYNC \|
3145	CRYPTO_ALG_ALLOCATES_MEMORY
3146	},
3147	.setkey = aead_authenc_setkey,
3148	.ivsize = AES_BLOCK_SIZE,
3149	.maxauthsize = MD5_DIGEST_SIZE,
3150	},
3151	.cipher_info = {
3152	.alg = CIPHER_ALG_AES,
3153	.mode = CIPHER_MODE_CBC,
3154	},
3155	.auth_info = {
3156	.alg = HASH_ALG_MD5,
3157	.mode = HASH_MODE_HMAC,
3158	},
3159	.auth_first = 0,
3160	},
3161	{
3162	.type = CRYPTO_ALG_TYPE_AEAD,
3163	.alg.aead = {
3164	.base = {
3165	.cra_name = "authenc(hmac(sha1),cbc(aes))",
3166	.cra_driver_name = "authenc-hmac-sha1-cbc-aes-iproc",
3167	.cra_blocksize = AES_BLOCK_SIZE,
3168	.cra_flags = CRYPTO_ALG_NEED_FALLBACK \|
3169	CRYPTO_ALG_ASYNC \|
3170	CRYPTO_ALG_ALLOCATES_MEMORY
3171	},
3172	.setkey = aead_authenc_setkey,
3173	.ivsize = AES_BLOCK_SIZE,
3174	.maxauthsize = SHA1_DIGEST_SIZE,
3175	},
3176	.cipher_info = {
3177	.alg = CIPHER_ALG_AES,
3178	.mode = CIPHER_MODE_CBC,
3179	},
3180	.auth_info = {
3181	.alg = HASH_ALG_SHA1,
3182	.mode = HASH_MODE_HMAC,
3183	},
3184	.auth_first = 0,
3185	},
3186	{
3187	.type = CRYPTO_ALG_TYPE_AEAD,
3188	.alg.aead = {
3189	.base = {
3190	.cra_name = "authenc(hmac(sha256),cbc(aes))",
3191	.cra_driver_name = "authenc-hmac-sha256-cbc-aes-iproc",
3192	.cra_blocksize = AES_BLOCK_SIZE,
3193	.cra_flags = CRYPTO_ALG_NEED_FALLBACK \|
3194	CRYPTO_ALG_ASYNC \|
3195	CRYPTO_ALG_ALLOCATES_MEMORY
3196	},
3197	.setkey = aead_authenc_setkey,
3198	.ivsize = AES_BLOCK_SIZE,
3199	.maxauthsize = SHA256_DIGEST_SIZE,
3200	},
3201	.cipher_info = {
3202	.alg = CIPHER_ALG_AES,
3203	.mode = CIPHER_MODE_CBC,
3204	},
3205	.auth_info = {
3206	.alg = HASH_ALG_SHA256,
3207	.mode = HASH_MODE_HMAC,
3208	},
3209	.auth_first = 0,
3210	},
3211	{
3212	.type = CRYPTO_ALG_TYPE_AEAD,
3213	.alg.aead = {
3214	.base = {
3215	.cra_name = "authenc(hmac(md5),cbc(des))",
3216	.cra_driver_name = "authenc-hmac-md5-cbc-des-iproc",
3217	.cra_blocksize = DES_BLOCK_SIZE,
3218	.cra_flags = CRYPTO_ALG_NEED_FALLBACK \|
3219	CRYPTO_ALG_ASYNC \|
3220	CRYPTO_ALG_ALLOCATES_MEMORY
3221	},
3222	.setkey = aead_authenc_setkey,
3223	.ivsize = DES_BLOCK_SIZE,
3224	.maxauthsize = MD5_DIGEST_SIZE,
3225	},
3226	.cipher_info = {
3227	.alg = CIPHER_ALG_DES,
3228	.mode = CIPHER_MODE_CBC,
3229	},
3230	.auth_info = {
3231	.alg = HASH_ALG_MD5,
3232	.mode = HASH_MODE_HMAC,
3233	},
3234	.auth_first = 0,
3235	},
3236	{
3237	.type = CRYPTO_ALG_TYPE_AEAD,
3238	.alg.aead = {
3239	.base = {
3240	.cra_name = "authenc(hmac(sha1),cbc(des))",
3241	.cra_driver_name = "authenc-hmac-sha1-cbc-des-iproc",
3242	.cra_blocksize = DES_BLOCK_SIZE,
3243	.cra_flags = CRYPTO_ALG_NEED_FALLBACK \|
3244	CRYPTO_ALG_ASYNC \|
3245	CRYPTO_ALG_ALLOCATES_MEMORY
3246	},
3247	.setkey = aead_authenc_setkey,
3248	.ivsize = DES_BLOCK_SIZE,
3249	.maxauthsize = SHA1_DIGEST_SIZE,
3250	},
3251	.cipher_info = {
3252	.alg = CIPHER_ALG_DES,
3253	.mode = CIPHER_MODE_CBC,
3254	},
3255	.auth_info = {
3256	.alg = HASH_ALG_SHA1,
3257	.mode = HASH_MODE_HMAC,
3258	},
3259	.auth_first = 0,
3260	},
3261	{
3262	.type = CRYPTO_ALG_TYPE_AEAD,
3263	.alg.aead = {
3264	.base = {
3265	.cra_name = "authenc(hmac(sha224),cbc(des))",
3266	.cra_driver_name = "authenc-hmac-sha224-cbc-des-iproc",
3267	.cra_blocksize = DES_BLOCK_SIZE,
3268	.cra_flags = CRYPTO_ALG_NEED_FALLBACK \|
3269	CRYPTO_ALG_ASYNC \|
3270	CRYPTO_ALG_ALLOCATES_MEMORY
3271	},
3272	.setkey = aead_authenc_setkey,
3273	.ivsize = DES_BLOCK_SIZE,
3274	.maxauthsize = SHA224_DIGEST_SIZE,
3275	},
3276	.cipher_info = {
3277	.alg = CIPHER_ALG_DES,
3278	.mode = CIPHER_MODE_CBC,
3279	},
3280	.auth_info = {
3281	.alg = HASH_ALG_SHA224,
3282	.mode = HASH_MODE_HMAC,
3283	},
3284	.auth_first = 0,
3285	},
3286	{
3287	.type = CRYPTO_ALG_TYPE_AEAD,
3288	.alg.aead = {
3289	.base = {
3290	.cra_name = "authenc(hmac(sha256),cbc(des))",
3291	.cra_driver_name = "authenc-hmac-sha256-cbc-des-iproc",
3292	.cra_blocksize = DES_BLOCK_SIZE,
3293	.cra_flags = CRYPTO_ALG_NEED_FALLBACK \|
3294	CRYPTO_ALG_ASYNC \|
3295	CRYPTO_ALG_ALLOCATES_MEMORY
3296	},
3297	.setkey = aead_authenc_setkey,
3298	.ivsize = DES_BLOCK_SIZE,
3299	.maxauthsize = SHA256_DIGEST_SIZE,
3300	},
3301	.cipher_info = {
3302	.alg = CIPHER_ALG_DES,
3303	.mode = CIPHER_MODE_CBC,
3304	},
3305	.auth_info = {
3306	.alg = HASH_ALG_SHA256,
3307	.mode = HASH_MODE_HMAC,
3308	},
3309	.auth_first = 0,
3310	},
3311	{
3312	.type = CRYPTO_ALG_TYPE_AEAD,
3313	.alg.aead = {
3314	.base = {
3315	.cra_name = "authenc(hmac(sha384),cbc(des))",
3316	.cra_driver_name = "authenc-hmac-sha384-cbc-des-iproc",
3317	.cra_blocksize = DES_BLOCK_SIZE,
3318	.cra_flags = CRYPTO_ALG_NEED_FALLBACK \|
3319	CRYPTO_ALG_ASYNC \|
3320	CRYPTO_ALG_ALLOCATES_MEMORY
3321	},
3322	.setkey = aead_authenc_setkey,
3323	.ivsize = DES_BLOCK_SIZE,
3324	.maxauthsize = SHA384_DIGEST_SIZE,
3325	},
3326	.cipher_info = {
3327	.alg = CIPHER_ALG_DES,
3328	.mode = CIPHER_MODE_CBC,
3329	},
3330	.auth_info = {
3331	.alg = HASH_ALG_SHA384,
3332	.mode = HASH_MODE_HMAC,
3333	},
3334	.auth_first = 0,
3335	},
3336	{
3337	.type = CRYPTO_ALG_TYPE_AEAD,
3338	.alg.aead = {
3339	.base = {
3340	.cra_name = "authenc(hmac(sha512),cbc(des))",
3341	.cra_driver_name = "authenc-hmac-sha512-cbc-des-iproc",
3342	.cra_blocksize = DES_BLOCK_SIZE,
3343	.cra_flags = CRYPTO_ALG_NEED_FALLBACK \|
3344	CRYPTO_ALG_ASYNC \|
3345	CRYPTO_ALG_ALLOCATES_MEMORY
3346	},
3347	.setkey = aead_authenc_setkey,
3348	.ivsize = DES_BLOCK_SIZE,
3349	.maxauthsize = SHA512_DIGEST_SIZE,
3350	},
3351	.cipher_info = {
3352	.alg = CIPHER_ALG_DES,
3353	.mode = CIPHER_MODE_CBC,
3354	},
3355	.auth_info = {
3356	.alg = HASH_ALG_SHA512,
3357	.mode = HASH_MODE_HMAC,
3358	},
3359	.auth_first = 0,
3360	},
3361	{
3362	.type = CRYPTO_ALG_TYPE_AEAD,
3363	.alg.aead = {
3364	.base = {
3365	.cra_name = "authenc(hmac(md5),cbc(des3_ede))",
3366	.cra_driver_name = "authenc-hmac-md5-cbc-des3-iproc",
3367	.cra_blocksize = DES3_EDE_BLOCK_SIZE,
3368	.cra_flags = CRYPTO_ALG_NEED_FALLBACK \|
3369	CRYPTO_ALG_ASYNC \|
3370	CRYPTO_ALG_ALLOCATES_MEMORY
3371	},
3372	.setkey = aead_authenc_setkey,
3373	.ivsize = DES3_EDE_BLOCK_SIZE,
3374	.maxauthsize = MD5_DIGEST_SIZE,
3375	},
3376	.cipher_info = {
3377	.alg = CIPHER_ALG_3DES,
3378	.mode = CIPHER_MODE_CBC,
3379	},
3380	.auth_info = {
3381	.alg = HASH_ALG_MD5,
3382	.mode = HASH_MODE_HMAC,
3383	},
3384	.auth_first = 0,
3385	},
3386	{
3387	.type = CRYPTO_ALG_TYPE_AEAD,
3388	.alg.aead = {
3389	.base = {
3390	.cra_name = "authenc(hmac(sha1),cbc(des3_ede))",
3391	.cra_driver_name = "authenc-hmac-sha1-cbc-des3-iproc",
3392	.cra_blocksize = DES3_EDE_BLOCK_SIZE,
3393	.cra_flags = CRYPTO_ALG_NEED_FALLBACK \|
3394	CRYPTO_ALG_ASYNC \|
3395	CRYPTO_ALG_ALLOCATES_MEMORY
3396	},
3397	.setkey = aead_authenc_setkey,
3398	.ivsize = DES3_EDE_BLOCK_SIZE,
3399	.maxauthsize = SHA1_DIGEST_SIZE,
3400	},
3401	.cipher_info = {
3402	.alg = CIPHER_ALG_3DES,
3403	.mode = CIPHER_MODE_CBC,
3404	},
3405	.auth_info = {
3406	.alg = HASH_ALG_SHA1,
3407	.mode = HASH_MODE_HMAC,
3408	},
3409	.auth_first = 0,
3410	},
3411	{
3412	.type = CRYPTO_ALG_TYPE_AEAD,
3413	.alg.aead = {
3414	.base = {
3415	.cra_name = "authenc(hmac(sha224),cbc(des3_ede))",
3416	.cra_driver_name = "authenc-hmac-sha224-cbc-des3-iproc",
3417	.cra_blocksize = DES3_EDE_BLOCK_SIZE,
3418	.cra_flags = CRYPTO_ALG_NEED_FALLBACK \|
3419	CRYPTO_ALG_ASYNC \|
3420	CRYPTO_ALG_ALLOCATES_MEMORY
3421	},
3422	.setkey = aead_authenc_setkey,
3423	.ivsize = DES3_EDE_BLOCK_SIZE,
3424	.maxauthsize = SHA224_DIGEST_SIZE,
3425	},
3426	.cipher_info = {
3427	.alg = CIPHER_ALG_3DES,
3428	.mode = CIPHER_MODE_CBC,
3429	},
3430	.auth_info = {
3431	.alg = HASH_ALG_SHA224,
3432	.mode = HASH_MODE_HMAC,
3433	},
3434	.auth_first = 0,
3435	},
3436	{
3437	.type = CRYPTO_ALG_TYPE_AEAD,
3438	.alg.aead = {
3439	.base = {
3440	.cra_name = "authenc(hmac(sha256),cbc(des3_ede))",
3441	.cra_driver_name = "authenc-hmac-sha256-cbc-des3-iproc",
3442	.cra_blocksize = DES3_EDE_BLOCK_SIZE,
3443	.cra_flags = CRYPTO_ALG_NEED_FALLBACK \|
3444	CRYPTO_ALG_ASYNC \|
3445	CRYPTO_ALG_ALLOCATES_MEMORY
3446	},
3447	.setkey = aead_authenc_setkey,
3448	.ivsize = DES3_EDE_BLOCK_SIZE,
3449	.maxauthsize = SHA256_DIGEST_SIZE,
3450	},
3451	.cipher_info = {
3452	.alg = CIPHER_ALG_3DES,
3453	.mode = CIPHER_MODE_CBC,
3454	},
3455	.auth_info = {
3456	.alg = HASH_ALG_SHA256,
3457	.mode = HASH_MODE_HMAC,
3458	},
3459	.auth_first = 0,
3460	},
3461	{
3462	.type = CRYPTO_ALG_TYPE_AEAD,
3463	.alg.aead = {
3464	.base = {
3465	.cra_name = "authenc(hmac(sha384),cbc(des3_ede))",
3466	.cra_driver_name = "authenc-hmac-sha384-cbc-des3-iproc",
3467	.cra_blocksize = DES3_EDE_BLOCK_SIZE,
3468	.cra_flags = CRYPTO_ALG_NEED_FALLBACK \|
3469	CRYPTO_ALG_ASYNC \|
3470	CRYPTO_ALG_ALLOCATES_MEMORY
3471	},
3472	.setkey = aead_authenc_setkey,
3473	.ivsize = DES3_EDE_BLOCK_SIZE,
3474	.maxauthsize = SHA384_DIGEST_SIZE,
3475	},
3476	.cipher_info = {
3477	.alg = CIPHER_ALG_3DES,
3478	.mode = CIPHER_MODE_CBC,
3479	},
3480	.auth_info = {
3481	.alg = HASH_ALG_SHA384,
3482	.mode = HASH_MODE_HMAC,
3483	},
3484	.auth_first = 0,
3485	},
3486	{
3487	.type = CRYPTO_ALG_TYPE_AEAD,
3488	.alg.aead = {
3489	.base = {
3490	.cra_name = "authenc(hmac(sha512),cbc(des3_ede))",
3491	.cra_driver_name = "authenc-hmac-sha512-cbc-des3-iproc",
3492	.cra_blocksize = DES3_EDE_BLOCK_SIZE,
3493	.cra_flags = CRYPTO_ALG_NEED_FALLBACK \|
3494	CRYPTO_ALG_ASYNC \|
3495	CRYPTO_ALG_ALLOCATES_MEMORY
3496	},
3497	.setkey = aead_authenc_setkey,
3498	.ivsize = DES3_EDE_BLOCK_SIZE,
3499	.maxauthsize = SHA512_DIGEST_SIZE,
3500	},
3501	.cipher_info = {
3502	.alg = CIPHER_ALG_3DES,
3503	.mode = CIPHER_MODE_CBC,
3504	},
3505	.auth_info = {
3506	.alg = HASH_ALG_SHA512,
3507	.mode = HASH_MODE_HMAC,
3508	},
3509	.auth_first = 0,
3510	},
3511
3512	/* SKCIPHER algorithms. */
3513	{
3514	.type = CRYPTO_ALG_TYPE_SKCIPHER,
3515	.alg.skcipher = {
3516	.base.cra_name = "cbc(des)",
3517	.base.cra_driver_name = "cbc-des-iproc",
3518	.base.cra_blocksize = DES_BLOCK_SIZE,
3519	.min_keysize = DES_KEY_SIZE,
3520	.max_keysize = DES_KEY_SIZE,
3521	.ivsize = DES_BLOCK_SIZE,
3522	},
3523	.cipher_info = {
3524	.alg = CIPHER_ALG_DES,
3525	.mode = CIPHER_MODE_CBC,
3526	},
3527	.auth_info = {
3528	.alg = HASH_ALG_NONE,
3529	.mode = HASH_MODE_NONE,
3530	},
3531	},
3532	{
3533	.type = CRYPTO_ALG_TYPE_SKCIPHER,
3534	.alg.skcipher = {
3535	.base.cra_name = "ecb(des)",
3536	.base.cra_driver_name = "ecb-des-iproc",
3537	.base.cra_blocksize = DES_BLOCK_SIZE,
3538	.min_keysize = DES_KEY_SIZE,
3539	.max_keysize = DES_KEY_SIZE,
3540	.ivsize = 0,
3541	},
3542	.cipher_info = {
3543	.alg = CIPHER_ALG_DES,
3544	.mode = CIPHER_MODE_ECB,
3545	},
3546	.auth_info = {
3547	.alg = HASH_ALG_NONE,
3548	.mode = HASH_MODE_NONE,
3549	},
3550	},
3551	{
3552	.type = CRYPTO_ALG_TYPE_SKCIPHER,
3553	.alg.skcipher = {
3554	.base.cra_name = "cbc(des3_ede)",
3555	.base.cra_driver_name = "cbc-des3-iproc",
3556	.base.cra_blocksize = DES3_EDE_BLOCK_SIZE,
3557	.min_keysize = DES3_EDE_KEY_SIZE,
3558	.max_keysize = DES3_EDE_KEY_SIZE,
3559	.ivsize = DES3_EDE_BLOCK_SIZE,
3560	},
3561	.cipher_info = {
3562	.alg = CIPHER_ALG_3DES,
3563	.mode = CIPHER_MODE_CBC,
3564	},
3565	.auth_info = {
3566	.alg = HASH_ALG_NONE,
3567	.mode = HASH_MODE_NONE,
3568	},
3569	},
3570	{
3571	.type = CRYPTO_ALG_TYPE_SKCIPHER,
3572	.alg.skcipher = {
3573	.base.cra_name = "ecb(des3_ede)",
3574	.base.cra_driver_name = "ecb-des3-iproc",
3575	.base.cra_blocksize = DES3_EDE_BLOCK_SIZE,
3576	.min_keysize = DES3_EDE_KEY_SIZE,
3577	.max_keysize = DES3_EDE_KEY_SIZE,
3578	.ivsize = 0,
3579	},
3580	.cipher_info = {
3581	.alg = CIPHER_ALG_3DES,
3582	.mode = CIPHER_MODE_ECB,
3583	},
3584	.auth_info = {
3585	.alg = HASH_ALG_NONE,
3586	.mode = HASH_MODE_NONE,
3587	},
3588	},
3589	{
3590	.type = CRYPTO_ALG_TYPE_SKCIPHER,
3591	.alg.skcipher = {
3592	.base.cra_name = "cbc(aes)",
3593	.base.cra_driver_name = "cbc-aes-iproc",
3594	.base.cra_blocksize = AES_BLOCK_SIZE,
3595	.min_keysize = AES_MIN_KEY_SIZE,
3596	.max_keysize = AES_MAX_KEY_SIZE,
3597	.ivsize = AES_BLOCK_SIZE,
3598	},
3599	.cipher_info = {
3600	.alg = CIPHER_ALG_AES,
3601	.mode = CIPHER_MODE_CBC,
3602	},
3603	.auth_info = {
3604	.alg = HASH_ALG_NONE,
3605	.mode = HASH_MODE_NONE,
3606	},
3607	},
3608	{
3609	.type = CRYPTO_ALG_TYPE_SKCIPHER,
3610	.alg.skcipher = {
3611	.base.cra_name = "ecb(aes)",
3612	.base.cra_driver_name = "ecb-aes-iproc",
3613	.base.cra_blocksize = AES_BLOCK_SIZE,
3614	.min_keysize = AES_MIN_KEY_SIZE,
3615	.max_keysize = AES_MAX_KEY_SIZE,
3616	.ivsize = 0,
3617	},
3618	.cipher_info = {
3619	.alg = CIPHER_ALG_AES,
3620	.mode = CIPHER_MODE_ECB,
3621	},
3622	.auth_info = {
3623	.alg = HASH_ALG_NONE,
3624	.mode = HASH_MODE_NONE,
3625	},
3626	},
3627	{
3628	.type = CRYPTO_ALG_TYPE_SKCIPHER,
3629	.alg.skcipher = {
3630	.base.cra_name = "ctr(aes)",
3631	.base.cra_driver_name = "ctr-aes-iproc",
3632	.base.cra_blocksize = AES_BLOCK_SIZE,
3633	.min_keysize = AES_MIN_KEY_SIZE,
3634	.max_keysize = AES_MAX_KEY_SIZE,
3635	.ivsize = AES_BLOCK_SIZE,
3636	},
3637	.cipher_info = {
3638	.alg = CIPHER_ALG_AES,
3639	.mode = CIPHER_MODE_CTR,
3640	},
3641	.auth_info = {
3642	.alg = HASH_ALG_NONE,
3643	.mode = HASH_MODE_NONE,
3644	},
3645	},
3646	{
3647	.type = CRYPTO_ALG_TYPE_SKCIPHER,
3648	.alg.skcipher = {
3649	.base.cra_name = "xts(aes)",
3650	.base.cra_driver_name = "xts-aes-iproc",
3651	.base.cra_blocksize = AES_BLOCK_SIZE,
3652	.min_keysize = 2 * AES_MIN_KEY_SIZE,
3653	.max_keysize = 2 * AES_MAX_KEY_SIZE,
3654	.ivsize = AES_BLOCK_SIZE,
3655	},
3656	.cipher_info = {
3657	.alg = CIPHER_ALG_AES,
3658	.mode = CIPHER_MODE_XTS,
3659	},
3660	.auth_info = {
3661	.alg = HASH_ALG_NONE,
3662	.mode = HASH_MODE_NONE,
3663	},
3664	},
3665
3666	/* AHASH algorithms. */
3667	{
3668	.type = CRYPTO_ALG_TYPE_AHASH,
3669	.alg.hash = {
3670	.halg.digestsize = MD5_DIGEST_SIZE,
3671	.halg.base = {
3672	.cra_name = "md5",
3673	.cra_driver_name = "md5-iproc",
3674	.cra_blocksize = MD5_BLOCK_WORDS * 4,
3675	.cra_flags = CRYPTO_ALG_ASYNC \|
3676	CRYPTO_ALG_ALLOCATES_MEMORY,
3677	}
3678	},
3679	.cipher_info = {
3680	.alg = CIPHER_ALG_NONE,
3681	.mode = CIPHER_MODE_NONE,
3682	},
3683	.auth_info = {
3684	.alg = HASH_ALG_MD5,
3685	.mode = HASH_MODE_HASH,
3686	},
3687	},
3688	{
3689	.type = CRYPTO_ALG_TYPE_AHASH,
3690	.alg.hash = {
3691	.halg.digestsize = MD5_DIGEST_SIZE,
3692	.halg.base = {
3693	.cra_name = "hmac(md5)",
3694	.cra_driver_name = "hmac-md5-iproc",
3695	.cra_blocksize = MD5_BLOCK_WORDS * 4,
3696	}
3697	},
3698	.cipher_info = {
3699	.alg = CIPHER_ALG_NONE,
3700	.mode = CIPHER_MODE_NONE,
3701	},
3702	.auth_info = {
3703	.alg = HASH_ALG_MD5,
3704	.mode = HASH_MODE_HMAC,
3705	},
3706	},
3707	{.type = CRYPTO_ALG_TYPE_AHASH,
3708	.alg.hash = {
3709	.halg.digestsize = SHA1_DIGEST_SIZE,
3710	.halg.base = {
3711	.cra_name = "sha1",
3712	.cra_driver_name = "sha1-iproc",
3713	.cra_blocksize = SHA1_BLOCK_SIZE,
3714	}
3715	},
3716	.cipher_info = {
3717	.alg = CIPHER_ALG_NONE,
3718	.mode = CIPHER_MODE_NONE,
3719	},
3720	.auth_info = {
3721	.alg = HASH_ALG_SHA1,
3722	.mode = HASH_MODE_HASH,
3723	},
3724	},
3725	{.type = CRYPTO_ALG_TYPE_AHASH,
3726	.alg.hash = {
3727	.halg.digestsize = SHA1_DIGEST_SIZE,
3728	.halg.base = {
3729	.cra_name = "hmac(sha1)",
3730	.cra_driver_name = "hmac-sha1-iproc",
3731	.cra_blocksize = SHA1_BLOCK_SIZE,
3732	}
3733	},
3734	.cipher_info = {
3735	.alg = CIPHER_ALG_NONE,
3736	.mode = CIPHER_MODE_NONE,
3737	},
3738	.auth_info = {
3739	.alg = HASH_ALG_SHA1,
3740	.mode = HASH_MODE_HMAC,
3741	},
3742	},
3743	{.type = CRYPTO_ALG_TYPE_AHASH,
3744	.alg.hash = {
3745	.halg.digestsize = SHA224_DIGEST_SIZE,
3746	.halg.base = {
3747	.cra_name = "sha224",
3748	.cra_driver_name = "sha224-iproc",
3749	.cra_blocksize = SHA224_BLOCK_SIZE,
3750	}
3751	},
3752	.cipher_info = {
3753	.alg = CIPHER_ALG_NONE,
3754	.mode = CIPHER_MODE_NONE,
3755	},
3756	.auth_info = {
3757	.alg = HASH_ALG_SHA224,
3758	.mode = HASH_MODE_HASH,
3759	},
3760	},
3761	{.type = CRYPTO_ALG_TYPE_AHASH,
3762	.alg.hash = {
3763	.halg.digestsize = SHA224_DIGEST_SIZE,
3764	.halg.base = {
3765	.cra_name = "hmac(sha224)",
3766	.cra_driver_name = "hmac-sha224-iproc",
3767	.cra_blocksize = SHA224_BLOCK_SIZE,
3768	}
3769	},
3770	.cipher_info = {
3771	.alg = CIPHER_ALG_NONE,
3772	.mode = CIPHER_MODE_NONE,
3773	},
3774	.auth_info = {
3775	.alg = HASH_ALG_SHA224,
3776	.mode = HASH_MODE_HMAC,
3777	},
3778	},
3779	{.type = CRYPTO_ALG_TYPE_AHASH,
3780	.alg.hash = {
3781	.halg.digestsize = SHA256_DIGEST_SIZE,
3782	.halg.base = {
3783	.cra_name = "sha256",
3784	.cra_driver_name = "sha256-iproc",
3785	.cra_blocksize = SHA256_BLOCK_SIZE,
3786	}
3787	},
3788	.cipher_info = {
3789	.alg = CIPHER_ALG_NONE,
3790	.mode = CIPHER_MODE_NONE,
3791	},
3792	.auth_info = {
3793	.alg = HASH_ALG_SHA256,
3794	.mode = HASH_MODE_HASH,
3795	},
3796	},
3797	{.type = CRYPTO_ALG_TYPE_AHASH,
3798	.alg.hash = {
3799	.halg.digestsize = SHA256_DIGEST_SIZE,
3800	.halg.base = {
3801	.cra_name = "hmac(sha256)",
3802	.cra_driver_name = "hmac-sha256-iproc",
3803	.cra_blocksize = SHA256_BLOCK_SIZE,
3804	}
3805	},
3806	.cipher_info = {
3807	.alg = CIPHER_ALG_NONE,
3808	.mode = CIPHER_MODE_NONE,
3809	},
3810	.auth_info = {
3811	.alg = HASH_ALG_SHA256,
3812	.mode = HASH_MODE_HMAC,
3813	},
3814	},
3815	{
3816	.type = CRYPTO_ALG_TYPE_AHASH,
3817	.alg.hash = {
3818	.halg.digestsize = SHA384_DIGEST_SIZE,
3819	.halg.base = {
3820	.cra_name = "sha384",
3821	.cra_driver_name = "sha384-iproc",
3822	.cra_blocksize = SHA384_BLOCK_SIZE,
3823	}
3824	},
3825	.cipher_info = {
3826	.alg = CIPHER_ALG_NONE,
3827	.mode = CIPHER_MODE_NONE,
3828	},
3829	.auth_info = {
3830	.alg = HASH_ALG_SHA384,
3831	.mode = HASH_MODE_HASH,
3832	},
3833	},
3834	{
3835	.type = CRYPTO_ALG_TYPE_AHASH,
3836	.alg.hash = {
3837	.halg.digestsize = SHA384_DIGEST_SIZE,
3838	.halg.base = {
3839	.cra_name = "hmac(sha384)",
3840	.cra_driver_name = "hmac-sha384-iproc",
3841	.cra_blocksize = SHA384_BLOCK_SIZE,
3842	}
3843	},
3844	.cipher_info = {
3845	.alg = CIPHER_ALG_NONE,
3846	.mode = CIPHER_MODE_NONE,
3847	},
3848	.auth_info = {
3849	.alg = HASH_ALG_SHA384,
3850	.mode = HASH_MODE_HMAC,
3851	},
3852	},
3853	{
3854	.type = CRYPTO_ALG_TYPE_AHASH,
3855	.alg.hash = {
3856	.halg.digestsize = SHA512_DIGEST_SIZE,
3857	.halg.base = {
3858	.cra_name = "sha512",
3859	.cra_driver_name = "sha512-iproc",
3860	.cra_blocksize = SHA512_BLOCK_SIZE,
3861	}
3862	},
3863	.cipher_info = {
3864	.alg = CIPHER_ALG_NONE,
3865	.mode = CIPHER_MODE_NONE,
3866	},
3867	.auth_info = {
3868	.alg = HASH_ALG_SHA512,
3869	.mode = HASH_MODE_HASH,
3870	},
3871	},
3872	{
3873	.type = CRYPTO_ALG_TYPE_AHASH,
3874	.alg.hash = {
3875	.halg.digestsize = SHA512_DIGEST_SIZE,
3876	.halg.base = {
3877	.cra_name = "hmac(sha512)",
3878	.cra_driver_name = "hmac-sha512-iproc",
3879	.cra_blocksize = SHA512_BLOCK_SIZE,
3880	}
3881	},
3882	.cipher_info = {
3883	.alg = CIPHER_ALG_NONE,
3884	.mode = CIPHER_MODE_NONE,
3885	},
3886	.auth_info = {
3887	.alg = HASH_ALG_SHA512,
3888	.mode = HASH_MODE_HMAC,
3889	},
3890	},
3891	{
3892	.type = CRYPTO_ALG_TYPE_AHASH,
3893	.alg.hash = {
3894	.halg.digestsize = SHA3_224_DIGEST_SIZE,
3895	.halg.base = {
3896	.cra_name = "sha3-224",
3897	.cra_driver_name = "sha3-224-iproc",
3898	.cra_blocksize = SHA3_224_BLOCK_SIZE,
3899	}
3900	},
3901	.cipher_info = {
3902	.alg = CIPHER_ALG_NONE,
3903	.mode = CIPHER_MODE_NONE,
3904	},
3905	.auth_info = {
3906	.alg = HASH_ALG_SHA3_224,
3907	.mode = HASH_MODE_HASH,
3908	},
3909	},
3910	{
3911	.type = CRYPTO_ALG_TYPE_AHASH,
3912	.alg.hash = {
3913	.halg.digestsize = SHA3_224_DIGEST_SIZE,
3914	.halg.base = {
3915	.cra_name = "hmac(sha3-224)",
3916	.cra_driver_name = "hmac-sha3-224-iproc",
3917	.cra_blocksize = SHA3_224_BLOCK_SIZE,
3918	}
3919	},
3920	.cipher_info = {
3921	.alg = CIPHER_ALG_NONE,
3922	.mode = CIPHER_MODE_NONE,
3923	},
3924	.auth_info = {
3925	.alg = HASH_ALG_SHA3_224,
3926	.mode = HASH_MODE_HMAC
3927	},
3928	},
3929	{
3930	.type = CRYPTO_ALG_TYPE_AHASH,
3931	.alg.hash = {
3932	.halg.digestsize = SHA3_256_DIGEST_SIZE,
3933	.halg.base = {
3934	.cra_name = "sha3-256",
3935	.cra_driver_name = "sha3-256-iproc",
3936	.cra_blocksize = SHA3_256_BLOCK_SIZE,
3937	}
3938	},
3939	.cipher_info = {
3940	.alg = CIPHER_ALG_NONE,
3941	.mode = CIPHER_MODE_NONE,
3942	},
3943	.auth_info = {
3944	.alg = HASH_ALG_SHA3_256,
3945	.mode = HASH_MODE_HASH,
3946	},
3947	},
3948	{
3949	.type = CRYPTO_ALG_TYPE_AHASH,
3950	.alg.hash = {
3951	.halg.digestsize = SHA3_256_DIGEST_SIZE,
3952	.halg.base = {
3953	.cra_name = "hmac(sha3-256)",
3954	.cra_driver_name = "hmac-sha3-256-iproc",
3955	.cra_blocksize = SHA3_256_BLOCK_SIZE,
3956	}
3957	},
3958	.cipher_info = {
3959	.alg = CIPHER_ALG_NONE,
3960	.mode = CIPHER_MODE_NONE,
3961	},
3962	.auth_info = {
3963	.alg = HASH_ALG_SHA3_256,
3964	.mode = HASH_MODE_HMAC,
3965	},
3966	},
3967	{
3968	.type = CRYPTO_ALG_TYPE_AHASH,
3969	.alg.hash = {
3970	.halg.digestsize = SHA3_384_DIGEST_SIZE,
3971	.halg.base = {
3972	.cra_name = "sha3-384",
3973	.cra_driver_name = "sha3-384-iproc",
3974	.cra_blocksize = SHA3_224_BLOCK_SIZE,
3975	}
3976	},
3977	.cipher_info = {
3978	.alg = CIPHER_ALG_NONE,
3979	.mode = CIPHER_MODE_NONE,
3980	},
3981	.auth_info = {
3982	.alg = HASH_ALG_SHA3_384,
3983	.mode = HASH_MODE_HASH,
3984	},
3985	},
3986	{
3987	.type = CRYPTO_ALG_TYPE_AHASH,
3988	.alg.hash = {
3989	.halg.digestsize = SHA3_384_DIGEST_SIZE,
3990	.halg.base = {
3991	.cra_name = "hmac(sha3-384)",
3992	.cra_driver_name = "hmac-sha3-384-iproc",
3993	.cra_blocksize = SHA3_384_BLOCK_SIZE,
3994	}
3995	},
3996	.cipher_info = {
3997	.alg = CIPHER_ALG_NONE,
3998	.mode = CIPHER_MODE_NONE,
3999	},
4000	.auth_info = {
4001	.alg = HASH_ALG_SHA3_384,
4002	.mode = HASH_MODE_HMAC,
4003	},
4004	},
4005	{
4006	.type = CRYPTO_ALG_TYPE_AHASH,
4007	.alg.hash = {
4008	.halg.digestsize = SHA3_512_DIGEST_SIZE,
4009	.halg.base = {
4010	.cra_name = "sha3-512",
4011	.cra_driver_name = "sha3-512-iproc",
4012	.cra_blocksize = SHA3_512_BLOCK_SIZE,
4013	}
4014	},
4015	.cipher_info = {
4016	.alg = CIPHER_ALG_NONE,
4017	.mode = CIPHER_MODE_NONE,
4018	},
4019	.auth_info = {
4020	.alg = HASH_ALG_SHA3_512,
4021	.mode = HASH_MODE_HASH,
4022	},
4023	},
4024	{
4025	.type = CRYPTO_ALG_TYPE_AHASH,
4026	.alg.hash = {
4027	.halg.digestsize = SHA3_512_DIGEST_SIZE,
4028	.halg.base = {
4029	.cra_name = "hmac(sha3-512)",
4030	.cra_driver_name = "hmac-sha3-512-iproc",
4031	.cra_blocksize = SHA3_512_BLOCK_SIZE,
4032	}
4033	},
4034	.cipher_info = {
4035	.alg = CIPHER_ALG_NONE,
4036	.mode = CIPHER_MODE_NONE,
4037	},
4038	.auth_info = {
4039	.alg = HASH_ALG_SHA3_512,
4040	.mode = HASH_MODE_HMAC,
4041	},
4042	},
4043	{
4044	.type = CRYPTO_ALG_TYPE_AHASH,
4045	.alg.hash = {
4046	.halg.digestsize = AES_BLOCK_SIZE,
4047	.halg.base = {
4048	.cra_name = "xcbc(aes)",
4049	.cra_driver_name = "xcbc-aes-iproc",
4050	.cra_blocksize = AES_BLOCK_SIZE,
4051	}
4052	},
4053	.cipher_info = {
4054	.alg = CIPHER_ALG_NONE,
4055	.mode = CIPHER_MODE_NONE,
4056	},
4057	.auth_info = {
4058	.alg = HASH_ALG_AES,
4059	.mode = HASH_MODE_XCBC,
4060	},
4061	},
4062	{
4063	.type = CRYPTO_ALG_TYPE_AHASH,
4064	.alg.hash = {
4065	.halg.digestsize = AES_BLOCK_SIZE,
4066	.halg.base = {
4067	.cra_name = "cmac(aes)",
4068	.cra_driver_name = "cmac-aes-iproc",
4069	.cra_blocksize = AES_BLOCK_SIZE,
4070	}
4071	},
4072	.cipher_info = {
4073	.alg = CIPHER_ALG_NONE,
4074	.mode = CIPHER_MODE_NONE,
4075	},
4076	.auth_info = {
4077	.alg = HASH_ALG_AES,
4078	.mode = HASH_MODE_CMAC,
4079	},
4080	},
4081	};
4082
4083	static int generic_cra_init(struct crypto_tfm *tfm,
4084	struct iproc_alg_s *cipher_alg)
4085	{
4086	struct spu_hw *spu = &iproc_priv.spu;
4087	struct iproc_ctx_s *ctx = crypto_tfm_ctx(tfm);
4088	unsigned int blocksize = crypto_tfm_alg_blocksize(tfm);
4089
4090	flow_log("%s()\n", __func__);
4091
4092	ctx->alg = cipher_alg;
4093	ctx->cipher = cipher_alg->cipher_info;
4094	ctx->auth = cipher_alg->auth_info;
4095	ctx->auth_first = cipher_alg->auth_first;
4096	ctx->max_payload = spu->spu_ctx_max_payload(ctx->cipher.alg,
4097	ctx->cipher.mode,
4098	blocksize);
4099	ctx->fallback_cipher = NULL;
4100
4101	ctx->enckeylen = 0;
4102	ctx->authkeylen = 0;
4103
4104	atomic_inc(&iproc_priv.stream_count);
4105	atomic_inc(&iproc_priv.session_count);
4106
4107	return 0;
4108	}
4109
4110	static int skcipher_init_tfm(struct crypto_skcipher *skcipher)
4111	{
4112	struct crypto_tfm *tfm = crypto_skcipher_tfm(skcipher);
4113	struct skcipher_alg *alg = crypto_skcipher_alg(skcipher);
4114	struct iproc_alg_s *cipher_alg;
4115
4116	flow_log("%s()\n", __func__);
4117
4118	crypto_skcipher_set_reqsize(skcipher, sizeof(struct iproc_reqctx_s));
4119
4120	cipher_alg = container_of(alg, struct iproc_alg_s, alg.skcipher);
4121	return generic_cra_init(tfm, cipher_alg);
4122	}
4123
4124	static int ahash_cra_init(struct crypto_tfm *tfm)
4125	{
4126	int err;
4127	struct crypto_alg *alg = tfm->__crt_alg;
4128	struct iproc_alg_s *cipher_alg;
4129
4130	cipher_alg = container_of(__crypto_ahash_alg(alg), struct iproc_alg_s,
4131	alg.hash);
4132
4133	err = generic_cra_init(tfm, cipher_alg);
4134	flow_log("%s()\n", __func__);
4135
4136	/*
4137	* export state size has to be < 512 bytes. So don't include msg bufs
4138	* in state size.
4139	*/
4140	crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
4141	sizeof(struct iproc_reqctx_s));
4142
4143	return err;
4144	}
4145
4146	static int aead_cra_init(struct crypto_aead *aead)
4147	{
4148	unsigned int reqsize = sizeof(struct iproc_reqctx_s);
4149	struct crypto_tfm *tfm = crypto_aead_tfm(aead);
4150	struct iproc_ctx_s *ctx = crypto_tfm_ctx(tfm);
4151	struct crypto_alg *alg = tfm->__crt_alg;
4152	struct aead_alg *aalg = container_of(alg, struct aead_alg, base);
4153	struct iproc_alg_s *cipher_alg = container_of(aalg, struct iproc_alg_s,
4154	alg.aead);
4155
4156	int err = generic_cra_init(tfm, cipher_alg);
4157
4158	flow_log("%s()\n", __func__);
4159
4160	ctx->is_esp = false;
4161	ctx->salt_len = 0;
4162	ctx->salt_offset = 0;
4163
4164	/* random first IV */
4165	get_random_bytes(ctx->iv, MAX_IV_SIZE);
4166	flow_dump(" iv: ", ctx->iv, MAX_IV_SIZE);
4167
4168	if (err)
4169	goto out;
4170
4171	if (!(alg->cra_flags & CRYPTO_ALG_NEED_FALLBACK))
4172	goto reqsize;
4173
4174	flow_log("%s() creating fallback cipher\n", __func__);
4175
4176	ctx->fallback_cipher = crypto_alloc_aead(alg->cra_name, 0,
4177	CRYPTO_ALG_ASYNC \|
4178	CRYPTO_ALG_NEED_FALLBACK);
4179	if (IS_ERR(ctx->fallback_cipher)) {
4180	pr_err("%s() Error: failed to allocate fallback for %s\n",
4181	__func__, alg->cra_name);
4182	return PTR_ERR(ctx->fallback_cipher);
4183	}
4184
4185	reqsize += crypto_aead_reqsize(ctx->fallback_cipher);
4186
4187	reqsize:
4188	crypto_aead_set_reqsize(aead, reqsize);
4189
4190	out:
4191	return err;
4192	}
4193
4194	static void generic_cra_exit(struct crypto_tfm *tfm)
4195	{
4196	atomic_dec(&iproc_priv.session_count);
4197	}
4198
4199	static void skcipher_exit_tfm(struct crypto_skcipher *tfm)
4200	{
4201	generic_cra_exit(crypto_skcipher_tfm(tfm));
4202	}
4203
4204	static void aead_cra_exit(struct crypto_aead *aead)
4205	{
4206	struct crypto_tfm *tfm = crypto_aead_tfm(aead);
4207	struct iproc_ctx_s *ctx = crypto_tfm_ctx(tfm);
4208
4209	generic_cra_exit(tfm);
4210
4211	if (ctx->fallback_cipher) {
4212	crypto_free_aead(ctx->fallback_cipher);
4213	ctx->fallback_cipher = NULL;
4214	}
4215	}
4216
4217	/**
4218	* spu_functions_register() - Specify hardware-specific SPU functions based on
4219	* SPU type read from device tree.
4220	* @dev: device structure
4221	* @spu_type: SPU hardware generation
4222	* @spu_subtype: SPU hardware version
4223	*/
4224	static void spu_functions_register(struct device *dev,
4225	enum spu_spu_type spu_type,
4226	enum spu_spu_subtype spu_subtype)
4227	{
4228	struct spu_hw *spu = &iproc_priv.spu;
4229
4230	if (spu_type == SPU_TYPE_SPUM) {
4231	dev_dbg(dev, "Registering SPUM functions");
4232	spu->spu_dump_msg_hdr = spum_dump_msg_hdr;
4233	spu->spu_payload_length = spum_payload_length;
4234	spu->spu_response_hdr_len = spum_response_hdr_len;
4235	spu->spu_hash_pad_len = spum_hash_pad_len;
4236	spu->spu_gcm_ccm_pad_len = spum_gcm_ccm_pad_len;
4237	spu->spu_assoc_resp_len = spum_assoc_resp_len;
4238	spu->spu_aead_ivlen = spum_aead_ivlen;
4239	spu->spu_hash_type = spum_hash_type;
4240	spu->spu_digest_size = spum_digest_size;
4241	spu->spu_create_request = spum_create_request;
4242	spu->spu_cipher_req_init = spum_cipher_req_init;
4243	spu->spu_cipher_req_finish = spum_cipher_req_finish;
4244	spu->spu_request_pad = spum_request_pad;
4245	spu->spu_tx_status_len = spum_tx_status_len;
4246	spu->spu_rx_status_len = spum_rx_status_len;
4247	spu->spu_status_process = spum_status_process;
4248	spu->spu_xts_tweak_in_payload = spum_xts_tweak_in_payload;
4249	spu->spu_ccm_update_iv = spum_ccm_update_iv;
4250	spu->spu_wordalign_padlen = spum_wordalign_padlen;
4251	if (spu_subtype == SPU_SUBTYPE_SPUM_NS2)
4252	spu->spu_ctx_max_payload = spum_ns2_ctx_max_payload;
4253	else
4254	spu->spu_ctx_max_payload = spum_nsp_ctx_max_payload;
4255	} else {
4256	dev_dbg(dev, "Registering SPU2 functions");
4257	spu->spu_dump_msg_hdr = spu2_dump_msg_hdr;
4258	spu->spu_ctx_max_payload = spu2_ctx_max_payload;
4259	spu->spu_payload_length = spu2_payload_length;
4260	spu->spu_response_hdr_len = spu2_response_hdr_len;
4261	spu->spu_hash_pad_len = spu2_hash_pad_len;
4262	spu->spu_gcm_ccm_pad_len = spu2_gcm_ccm_pad_len;
4263	spu->spu_assoc_resp_len = spu2_assoc_resp_len;
4264	spu->spu_aead_ivlen = spu2_aead_ivlen;
4265	spu->spu_hash_type = spu2_hash_type;
4266	spu->spu_digest_size = spu2_digest_size;
4267	spu->spu_create_request = spu2_create_request;
4268	spu->spu_cipher_req_init = spu2_cipher_req_init;
4269	spu->spu_cipher_req_finish = spu2_cipher_req_finish;
4270	spu->spu_request_pad = spu2_request_pad;
4271	spu->spu_tx_status_len = spu2_tx_status_len;
4272	spu->spu_rx_status_len = spu2_rx_status_len;
4273	spu->spu_status_process = spu2_status_process;
4274	spu->spu_xts_tweak_in_payload = spu2_xts_tweak_in_payload;
4275	spu->spu_ccm_update_iv = spu2_ccm_update_iv;
4276	spu->spu_wordalign_padlen = spu2_wordalign_padlen;
4277	}
4278	}
4279
4280	/**
4281	* spu_mb_init() - Initialize mailbox client. Request ownership of a mailbox
4282	* channel for the SPU being probed.
4283	* @dev: SPU driver device structure
4284	*
4285	* Return: 0 if successful
4286	* < 0 otherwise
4287	*/
4288	static int spu_mb_init(struct device *dev)
4289	{
4290	struct mbox_client *mcl = &iproc_priv.mcl;
4291	int err, i;
4292
4293	iproc_priv.mbox = devm_kcalloc(dev, iproc_priv.spu.num_chan,
4294	sizeof(struct mbox_chan *), GFP_KERNEL);
4295	if (!iproc_priv.mbox)
4296	return -ENOMEM;
4297
4298	mcl->dev = dev;
4299	mcl->tx_block = false;
4300	mcl->tx_tout = 0;
4301	mcl->knows_txdone = true;
4302	mcl->rx_callback = spu_rx_callback;
4303	mcl->tx_done = NULL;
4304
4305	for (i = 0; i < iproc_priv.spu.num_chan; i++) {
4306	iproc_priv.mbox[i] = mbox_request_channel(mcl, i);
4307	if (IS_ERR(iproc_priv.mbox[i])) {
4308	err = PTR_ERR(iproc_priv.mbox[i]);
4309	dev_err(dev,
4310	"Mbox channel %d request failed with err %d",
4311	i, err);
4312	iproc_priv.mbox[i] = NULL;
4313	goto free_channels;
4314	}
4315	}
4316
4317	return 0;
4318	free_channels:
4319	for (i = 0; i < iproc_priv.spu.num_chan; i++) {
4320	if (iproc_priv.mbox[i])
4321	mbox_free_channel(iproc_priv.mbox[i]);
4322	}
4323
4324	return err;
4325	}
4326
4327	static void spu_mb_release(struct platform_device *pdev)
4328	{
4329	int i;
4330
4331	for (i = 0; i < iproc_priv.spu.num_chan; i++)
4332	mbox_free_channel(iproc_priv.mbox[i]);
4333	}
4334
4335	static void spu_counters_init(void)
4336	{
4337	int i;
4338	int j;
4339
4340	atomic_set(&iproc_priv.session_count, 0);
4341	atomic_set(&iproc_priv.stream_count, 0);
4342	atomic_set(&iproc_priv.next_chan, (int)iproc_priv.spu.num_chan);
4343	atomic64_set(&iproc_priv.bytes_in, 0);
4344	atomic64_set(&iproc_priv.bytes_out, 0);
4345	for (i = 0; i < SPU_OP_NUM; i++) {
4346	atomic_set(&iproc_priv.op_counts[i], 0);
4347	atomic_set(&iproc_priv.setkey_cnt[i], 0);
4348	}
4349	for (i = 0; i < CIPHER_ALG_LAST; i++)
4350	for (j = 0; j < CIPHER_MODE_LAST; j++)
4351	atomic_set(&iproc_priv.cipher_cnt[i][j], 0);
4352
4353	for (i = 0; i < HASH_ALG_LAST; i++) {
4354	atomic_set(&iproc_priv.hash_cnt[i], 0);
4355	atomic_set(&iproc_priv.hmac_cnt[i], 0);
4356	}
4357	for (i = 0; i < AEAD_TYPE_LAST; i++)
4358	atomic_set(&iproc_priv.aead_cnt[i], 0);
4359
4360	atomic_set(&iproc_priv.mb_no_spc, 0);
4361	atomic_set(&iproc_priv.mb_send_fail, 0);
4362	atomic_set(&iproc_priv.bad_icv, 0);
4363	}
4364
4365	static int spu_register_skcipher(struct iproc_alg_s *driver_alg)
4366	{
4367	struct skcipher_alg *crypto = &driver_alg->alg.skcipher;
4368	int err;
4369
4370	crypto->base.cra_module = THIS_MODULE;
4371	crypto->base.cra_priority = cipher_pri;
4372	crypto->base.cra_alignmask = 0;
4373	crypto->base.cra_ctxsize = sizeof(struct iproc_ctx_s);
4374	crypto->base.cra_flags = CRYPTO_ALG_ASYNC \|
4375	CRYPTO_ALG_ALLOCATES_MEMORY \|
4376	CRYPTO_ALG_KERN_DRIVER_ONLY;
4377
4378	crypto->init = skcipher_init_tfm;
4379	crypto->exit = skcipher_exit_tfm;
4380	crypto->setkey = skcipher_setkey;
4381	crypto->encrypt = skcipher_encrypt;
4382	crypto->decrypt = skcipher_decrypt;
4383
4384	err = crypto_register_skcipher(crypto);
4385	/* Mark alg as having been registered, if successful */
4386	if (err == 0)
4387	driver_alg->registered = true;
4388	pr_debug(" registered skcipher %s\n", crypto->base.cra_driver_name);
4389	return err;
4390	}
4391
4392	static int spu_register_ahash(struct iproc_alg_s *driver_alg)
4393	{
4394	struct spu_hw *spu = &iproc_priv.spu;
4395	struct ahash_alg *hash = &driver_alg->alg.hash;
4396	int err;
4397
4398	/* AES-XCBC is the only AES hash type currently supported on SPU-M */
4399	if ((driver_alg->auth_info.alg == HASH_ALG_AES) &&
4400	(driver_alg->auth_info.mode != HASH_MODE_XCBC) &&
4401	(spu->spu_type == SPU_TYPE_SPUM))
4402	return 0;
4403
4404	/* SHA3 algorithm variants are not registered for SPU-M or SPU2. */
4405	if ((driver_alg->auth_info.alg >= HASH_ALG_SHA3_224) &&
4406	(spu->spu_subtype != SPU_SUBTYPE_SPU2_V2))
4407	return 0;
4408
4409	hash->halg.base.cra_module = THIS_MODULE;
4410	hash->halg.base.cra_priority = hash_pri;
4411	hash->halg.base.cra_alignmask = 0;
4412	hash->halg.base.cra_ctxsize = sizeof(struct iproc_ctx_s);
4413	hash->halg.base.cra_init = ahash_cra_init;
4414	hash->halg.base.cra_exit = generic_cra_exit;
4415	hash->halg.base.cra_flags = CRYPTO_ALG_ASYNC \|
4416	CRYPTO_ALG_ALLOCATES_MEMORY;
4417	hash->halg.statesize = sizeof(struct spu_hash_export_s);
4418
4419	if (driver_alg->auth_info.mode != HASH_MODE_HMAC) {
4420	hash->init = ahash_init;
4421	hash->update = ahash_update;
4422	hash->final = ahash_final;
4423	hash->finup = ahash_finup;
4424	hash->digest = ahash_digest;
4425	if ((driver_alg->auth_info.alg == HASH_ALG_AES) &&
4426	((driver_alg->auth_info.mode == HASH_MODE_XCBC) \|\|
4427	(driver_alg->auth_info.mode == HASH_MODE_CMAC))) {
4428	hash->setkey = ahash_setkey;
4429	}
4430	} else {
4431	hash->setkey = ahash_hmac_setkey;
4432	hash->init = ahash_hmac_init;
4433	hash->update = ahash_hmac_update;
4434	hash->final = ahash_hmac_final;
4435	hash->finup = ahash_hmac_finup;
4436	hash->digest = ahash_hmac_digest;
4437	}
4438	hash->export = ahash_export;
4439	hash->import = ahash_import;
4440
4441	err = crypto_register_ahash(hash);
4442	/* Mark alg as having been registered, if successful */
4443	if (err == 0)
4444	driver_alg->registered = true;
4445	pr_debug(" registered ahash %s\n",
4446	hash->halg.base.cra_driver_name);
4447	return err;
4448	}
4449
4450	static int spu_register_aead(struct iproc_alg_s *driver_alg)
4451	{
4452	struct aead_alg *aead = &driver_alg->alg.aead;
4453	int err;
4454
4455	aead->base.cra_module = THIS_MODULE;
4456	aead->base.cra_priority = aead_pri;
4457	aead->base.cra_alignmask = 0;
4458	aead->base.cra_ctxsize = sizeof(struct iproc_ctx_s);
4459
4460	aead->base.cra_flags \|= CRYPTO_ALG_ASYNC \| CRYPTO_ALG_ALLOCATES_MEMORY;
4461	/* setkey set in alg initialization */
4462	aead->setauthsize = aead_setauthsize;
4463	aead->encrypt = aead_encrypt;
4464	aead->decrypt = aead_decrypt;
4465	aead->init = aead_cra_init;
4466	aead->exit = aead_cra_exit;
4467
4468	err = crypto_register_aead(aead);
4469	/* Mark alg as having been registered, if successful */
4470	if (err == 0)
4471	driver_alg->registered = true;
4472	pr_debug(" registered aead %s\n", aead->base.cra_driver_name);
4473	return err;
4474	}
4475
4476	/* register crypto algorithms the device supports */
4477	static int spu_algs_register(struct device *dev)
4478	{
4479	int i, j;
4480	int err;
4481
4482	for (i = 0; i < ARRAY_SIZE(driver_algs); i++) {
4483	switch (driver_algs[i].type) {
4484	case CRYPTO_ALG_TYPE_SKCIPHER:
4485	err = spu_register_skcipher(&driver_algs[i]);
4486	break;
4487	case CRYPTO_ALG_TYPE_AHASH:
4488	err = spu_register_ahash(&driver_algs[i]);
4489	break;
4490	case CRYPTO_ALG_TYPE_AEAD:
4491	err = spu_register_aead(&driver_algs[i]);
4492	break;
4493	default:
4494	dev_err(dev,
4495	"iproc-crypto: unknown alg type: %d",
4496	driver_algs[i].type);
4497	err = -EINVAL;
4498	}
4499
4500	if (err) {
4501	dev_err(dev, "alg registration failed with error %d\n",
4502	err);
4503	goto err_algs;
4504	}
4505	}
4506
4507	return 0;
4508
4509	err_algs:
4510	for (j = 0; j < i; j++) {
4511	/* Skip any algorithm not registered */
4512	if (!driver_algs[j].registered)
4513	continue;
4514	switch (driver_algs[j].type) {
4515	case CRYPTO_ALG_TYPE_SKCIPHER:
4516	crypto_unregister_skcipher(&driver_algs[j].alg.skcipher);
4517	driver_algs[j].registered = false;
4518	break;
4519	case CRYPTO_ALG_TYPE_AHASH:
4520	crypto_unregister_ahash(&driver_algs[j].alg.hash);
4521	driver_algs[j].registered = false;
4522	break;
4523	case CRYPTO_ALG_TYPE_AEAD:
4524	crypto_unregister_aead(&driver_algs[j].alg.aead);
4525	driver_algs[j].registered = false;
4526	break;
4527	}
4528	}
4529	return err;
4530	}
4531
4532	/* ==================== Kernel Platform API ==================== */
4533
4534	static struct spu_type_subtype spum_ns2_types = {
4535	SPU_TYPE_SPUM, SPU_SUBTYPE_SPUM_NS2
4536	};
4537
4538	static struct spu_type_subtype spum_nsp_types = {
4539	SPU_TYPE_SPUM, SPU_SUBTYPE_SPUM_NSP
4540	};
4541
4542	static struct spu_type_subtype spu2_types = {
4543	SPU_TYPE_SPU2, SPU_SUBTYPE_SPU2_V1
4544	};
4545
4546	static struct spu_type_subtype spu2_v2_types = {
4547	SPU_TYPE_SPU2, SPU_SUBTYPE_SPU2_V2
4548	};
4549
4550	static const struct of_device_id bcm_spu_dt_ids[] = {
4551	{
4552	.compatible = "brcm,spum-crypto",
4553	.data = &spum_ns2_types,
4554	},
4555	{
4556	.compatible = "brcm,spum-nsp-crypto",
4557	.data = &spum_nsp_types,
4558	},
4559	{
4560	.compatible = "brcm,spu2-crypto",
4561	.data = &spu2_types,
4562	},
4563	{
4564	.compatible = "brcm,spu2-v2-crypto",
4565	.data = &spu2_v2_types,
4566	},
4567	{ /* sentinel */ }
4568	};
4569
4570	MODULE_DEVICE_TABLE(of, bcm_spu_dt_ids);
4571
4572	static int spu_dt_read(struct platform_device *pdev)
4573	{
4574	struct device *dev = &pdev->dev;
4575	struct spu_hw *spu = &iproc_priv.spu;
4576	struct resource *spu_ctrl_regs;
4577	const struct spu_type_subtype *matched_spu_type;
4578	struct device_node *dn = pdev->dev.of_node;
4579	int err, i;
4580
4581	/* Count number of mailbox channels */
4582	spu->num_chan = of_count_phandle_with_args(dn, "mboxes", "#mbox-cells");
4583
4584	matched_spu_type = of_device_get_match_data(dev);
4585	if (!matched_spu_type) {
4586	dev_err(dev, "Failed to match device\n");
4587	return -ENODEV;
4588	}
4589
4590	spu->spu_type = matched_spu_type->type;
4591	spu->spu_subtype = matched_spu_type->subtype;
4592
4593	for (i = 0; (i < MAX_SPUS) && ((spu_ctrl_regs =
4594	platform_get_resource(pdev, IORESOURCE_MEM, i)) != NULL); i++) {
4595
4596	spu->reg_vbase[i] = devm_ioremap_resource(dev, spu_ctrl_regs);
4597	if (IS_ERR(spu->reg_vbase[i])) {
4598	err = PTR_ERR(spu->reg_vbase[i]);
4599	dev_err(dev, "Failed to map registers: %d\n",
4600	err);
4601	spu->reg_vbase[i] = NULL;
4602	return err;
4603	}
4604	}
4605	spu->num_spu = i;
4606	dev_dbg(dev, "Device has %d SPUs", spu->num_spu);
4607
4608	return 0;
4609	}
4610
4611	static int bcm_spu_probe(struct platform_device *pdev)
4612	{
4613	struct device *dev = &pdev->dev;
4614	struct spu_hw *spu = &iproc_priv.spu;
4615	int err;
4616
4617	iproc_priv.pdev = pdev;
4618	platform_set_drvdata(iproc_priv.pdev,
4619	&iproc_priv);
4620
4621	err = spu_dt_read(pdev);
4622	if (err < 0)
4623	goto failure;
4624
4625	err = spu_mb_init(dev);
4626	if (err < 0)
4627	goto failure;
4628
4629	if (spu->spu_type == SPU_TYPE_SPUM)
4630	iproc_priv.bcm_hdr_len = 8;
4631	else if (spu->spu_type == SPU_TYPE_SPU2)
4632	iproc_priv.bcm_hdr_len = 0;
4633
4634	spu_functions_register(dev, spu->spu_type, spu->spu_subtype);
4635
4636	spu_counters_init();
4637
4638	spu_setup_debugfs();
4639
4640	err = spu_algs_register(dev);
4641	if (err < 0)
4642	goto fail_reg;
4643
4644	return 0;
4645
4646	fail_reg:
4647	spu_free_debugfs();
4648	failure:
4649	spu_mb_release(pdev);
4650	dev_err(dev, "%s failed with error %d.\n", __func__, err);
4651
4652	return err;
4653	}
4654
4655	static void bcm_spu_remove(struct platform_device *pdev)
4656	{
4657	int i;
4658	struct device *dev = &pdev->dev;
4659	char *cdn;
4660
4661	for (i = 0; i < ARRAY_SIZE(driver_algs); i++) {
4662	/*
4663	* Not all algorithms were registered, depending on whether
4664	* hardware is SPU or SPU2. So here we make sure to skip
4665	* those algorithms that were not previously registered.
4666	*/
4667	if (!driver_algs[i].registered)
4668	continue;
4669
4670	switch (driver_algs[i].type) {
4671	case CRYPTO_ALG_TYPE_SKCIPHER:
4672	crypto_unregister_skcipher(&driver_algs[i].alg.skcipher);
4673	dev_dbg(dev, " unregistered cipher %s\n",
4674	driver_algs[i].alg.skcipher.base.cra_driver_name);
4675	driver_algs[i].registered = false;
4676	break;
4677	case CRYPTO_ALG_TYPE_AHASH:
4678	crypto_unregister_ahash(&driver_algs[i].alg.hash);
4679	cdn = driver_algs[i].alg.hash.halg.base.cra_driver_name;
4680	dev_dbg(dev, " unregistered hash %s\n", cdn);
4681	driver_algs[i].registered = false;
4682	break;
4683	case CRYPTO_ALG_TYPE_AEAD:
4684	crypto_unregister_aead(&driver_algs[i].alg.aead);
4685	dev_dbg(dev, " unregistered aead %s\n",
4686	driver_algs[i].alg.aead.base.cra_driver_name);
4687	driver_algs[i].registered = false;
4688	break;
4689	}
4690	}
4691	spu_free_debugfs();
4692	spu_mb_release(pdev);
4693	}
4694
4695	/* ===== Kernel Module API ===== */
4696
4697	static struct platform_driver bcm_spu_pdriver = {
4698	.driver = {
4699	.name = "brcm-spu-crypto",
4700	.of_match_table = bcm_spu_dt_ids,
4701	},
4702	.probe = bcm_spu_probe,
4703	.remove = bcm_spu_remove,
4704	};
4705	module_platform_driver(bcm_spu_pdriver);
4706
4707	MODULE_AUTHOR("Rob Rice <[email protected]>");
4708	MODULE_DESCRIPTION("Broadcom symmetric crypto offload driver");
4709	MODULE_LICENSE("GPL v2");