projects/linux·blob master · refs

3635 lines · 104.4 KiB

1// SPDX-License-Identifier: GPL-2.0-or-later
2/*
3 * talitos - Freescale Integrated Security Engine (SEC) device driver
4 *
5 * Copyright (c) 2008-2011 Freescale Semiconductor, Inc.
6 *
7 * Scatterlist Crypto API glue code copied from files with the following:
8 * Copyright (c) 2006-2007 Herbert Xu <[email protected]>
9 *
10 * Crypto algorithm registration code copied from hifn driver:
11 * 2007+ Copyright (c) Evgeniy Polyakov <[email protected]>
12 * All rights reserved.
13 */
14
15#include <linux/kernel.h>
16#include <linux/module.h>
17#include <linux/device.h>
18#include <linux/interrupt.h>
19#include <linux/crypto.h>
20#include <linux/hw_random.h>
21#include <linux/of.h>
22#include <linux/of_irq.h>
23#include <linux/platform_device.h>
24#include <linux/dma-mapping.h>
25#include <linux/io.h>
26#include <linux/spinlock.h>
27#include <linux/rtnetlink.h>
28#include <linux/slab.h>
29
30#include <crypto/algapi.h>
31#include <crypto/aes.h>
32#include <crypto/internal/des.h>
33#include <crypto/sha1.h>
34#include <crypto/sha2.h>
35#include <crypto/md5.h>
36#include <crypto/internal/aead.h>
37#include <crypto/authenc.h>
38#include <crypto/internal/skcipher.h>
39#include <crypto/hash.h>
40#include <crypto/internal/hash.h>
41#include <crypto/scatterwalk.h>
42
43#include "talitos.h"
44
45static void to_talitos_ptr(struct talitos_ptr *ptr, dma_addr_t dma_addr,
46 unsigned int len, bool is_sec1)
47{
48 ptr->ptr = cpu_to_be32(lower_32_bits(dma_addr));
49 if (is_sec1) {
50 ptr->len1 = cpu_to_be16(len);
51 } else {
52 ptr->len = cpu_to_be16(len);
53 ptr->eptr = upper_32_bits(dma_addr);
54 }
55}
56
57static void copy_talitos_ptr(struct talitos_ptr *dst_ptr,
58 struct talitos_ptr *src_ptr, bool is_sec1)
59{
60 dst_ptr->ptr = src_ptr->ptr;
61 if (is_sec1) {
62 dst_ptr->len1 = src_ptr->len1;
63 } else {
64 dst_ptr->len = src_ptr->len;
65 dst_ptr->eptr = src_ptr->eptr;
66 }
67}
68
69static unsigned short from_talitos_ptr_len(struct talitos_ptr *ptr,
70 bool is_sec1)
71{
72 if (is_sec1)
73 return be16_to_cpu(ptr->len1);
74 else
75 return be16_to_cpu(ptr->len);
76}
77
78static void to_talitos_ptr_ext_set(struct talitos_ptr *ptr, u8 val,
79 bool is_sec1)
80{
81 if (!is_sec1)
82 ptr->j_extent = val;
83}
84
85static void to_talitos_ptr_ext_or(struct talitos_ptr *ptr, u8 val, bool is_sec1)
86{
87 if (!is_sec1)
88 ptr->j_extent |= val;
89}
90
91/*
92 * map virtual single (contiguous) pointer to h/w descriptor pointer
93 */
94static void __map_single_talitos_ptr(struct device *dev,
95 struct talitos_ptr *ptr,
96 unsigned int len, void *data,
97 enum dma_data_direction dir,
98 unsigned long attrs)
99{
100 dma_addr_t dma_addr = dma_map_single_attrs(dev, data, len, dir, attrs);
101 struct talitos_private *priv = dev_get_drvdata(dev);
102 bool is_sec1 = has_ftr_sec1(priv);
103
104 to_talitos_ptr(ptr, dma_addr, len, is_sec1);
105}
106
107static void map_single_talitos_ptr(struct device *dev,
108 struct talitos_ptr *ptr,
109 unsigned int len, void *data,
110 enum dma_data_direction dir)
111{
112 __map_single_talitos_ptr(dev, ptr, len, data, dir, 0);
113}
114
115static void map_single_talitos_ptr_nosync(struct device *dev,
116 struct talitos_ptr *ptr,
117 unsigned int len, void *data,
118 enum dma_data_direction dir)
119{
120 __map_single_talitos_ptr(dev, ptr, len, data, dir,
121 DMA_ATTR_SKIP_CPU_SYNC);
122}
123
124/*
125 * unmap bus single (contiguous) h/w descriptor pointer
126 */
127static void unmap_single_talitos_ptr(struct device *dev,
128 struct talitos_ptr *ptr,
129 enum dma_data_direction dir)
130{
131 struct talitos_private *priv = dev_get_drvdata(dev);
132 bool is_sec1 = has_ftr_sec1(priv);
133
134 dma_unmap_single(dev, be32_to_cpu(ptr->ptr),
135 from_talitos_ptr_len(ptr, is_sec1), dir);
136}
137
138static int reset_channel(struct device *dev, int ch)
139{
140 struct talitos_private *priv = dev_get_drvdata(dev);
141 unsigned int timeout = TALITOS_TIMEOUT;
142 bool is_sec1 = has_ftr_sec1(priv);
143
144 if (is_sec1) {
145 setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO,
146 TALITOS1_CCCR_LO_RESET);
147
148 while ((in_be32(priv->chan[ch].reg + TALITOS_CCCR_LO) &
149 TALITOS1_CCCR_LO_RESET) && --timeout)
150 cpu_relax();
151 } else {
152 setbits32(priv->chan[ch].reg + TALITOS_CCCR,
153 TALITOS2_CCCR_RESET);
154
155 while ((in_be32(priv->chan[ch].reg + TALITOS_CCCR) &
156 TALITOS2_CCCR_RESET) && --timeout)
157 cpu_relax();
158 }
159
160 if (timeout == 0) {
161 dev_err(dev, "failed to reset channel %d\n", ch);
162 return -EIO;
163 }
164
165 /* set 36-bit addressing, done writeback enable and done IRQ enable */
166 setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO, TALITOS_CCCR_LO_EAE |
167 TALITOS_CCCR_LO_CDWE | TALITOS_CCCR_LO_CDIE);
168 /* enable chaining descriptors */
169 if (is_sec1)
170 setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO,
171 TALITOS_CCCR_LO_NE);
172
173 /* and ICCR writeback, if available */
174 if (priv->features & TALITOS_FTR_HW_AUTH_CHECK)
175 setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO,
176 TALITOS_CCCR_LO_IWSE);
177
178 return 0;
179}
180
181static int reset_device(struct device *dev)
182{
183 struct talitos_private *priv = dev_get_drvdata(dev);
184 unsigned int timeout = TALITOS_TIMEOUT;
185 bool is_sec1 = has_ftr_sec1(priv);
186 u32 mcr = is_sec1 ? TALITOS1_MCR_SWR : TALITOS2_MCR_SWR;
187
188 setbits32(priv->reg + TALITOS_MCR, mcr);
189
190 while ((in_be32(priv->reg + TALITOS_MCR) & mcr)
191 && --timeout)
192 cpu_relax();
193
194 if (priv->irq[1] > 0) {
195 mcr = TALITOS_MCR_RCA1 | TALITOS_MCR_RCA3;
196 setbits32(priv->reg + TALITOS_MCR, mcr);
197 }
198
199 if (timeout == 0) {
200 dev_err(dev, "failed to reset device\n");
201 return -EIO;
202 }
203
204 return 0;
205}
206
207/*
208 * Reset and initialize the device
209 */
210static int init_device(struct device *dev)
211{
212 struct talitos_private *priv = dev_get_drvdata(dev);
213 int ch, err;
214 bool is_sec1 = has_ftr_sec1(priv);
215
216 /*
217 * Master reset
218 * errata documentation: warning: certain SEC interrupts
219 * are not fully cleared by writing the MCR:SWR bit,
220 * set bit twice to completely reset
221 */
222 err = reset_device(dev);
223 if (err)
224 return err;
225
226 err = reset_device(dev);
227 if (err)
228 return err;
229
230 /* reset channels */
231 for (ch = 0; ch < priv->num_channels; ch++) {
232 err = reset_channel(dev, ch);
233 if (err)
234 return err;
235 }
236
237 /* enable channel done and error interrupts */
238 if (is_sec1) {
239 clrbits32(priv->reg + TALITOS_IMR, TALITOS1_IMR_INIT);
240 clrbits32(priv->reg + TALITOS_IMR_LO, TALITOS1_IMR_LO_INIT);
241 /* disable parity error check in DEU (erroneous? test vect.) */
242 setbits32(priv->reg_deu + TALITOS_EUICR, TALITOS1_DEUICR_KPE);
243 } else {
244 setbits32(priv->reg + TALITOS_IMR, TALITOS2_IMR_INIT);
245 setbits32(priv->reg + TALITOS_IMR_LO, TALITOS2_IMR_LO_INIT);
246 }
247
248 /* disable integrity check error interrupts (use writeback instead) */
249 if (priv->features & TALITOS_FTR_HW_AUTH_CHECK)
250 setbits32(priv->reg_mdeu + TALITOS_EUICR_LO,
251 TALITOS_MDEUICR_LO_ICE);
252
253 return 0;
254}
255
256static void dma_map_request(struct device *dev, struct talitos_request *request,
257 struct talitos_desc *desc, bool is_sec1)
258{
259 struct talitos_edesc *edesc =
260 container_of(desc, struct talitos_edesc, desc);
261 dma_addr_t dma_desc, prev_dma_desc;
262 struct talitos_edesc *prev_edesc = NULL;
263
264 if (is_sec1) {
265 while (edesc) {
266 edesc->desc.hdr1 = edesc->desc.hdr;
267
268 dma_desc = dma_map_single(dev, &edesc->desc.hdr1,
269 TALITOS_DESC_SIZE,
270 DMA_BIDIRECTIONAL);
271
272 if (!prev_edesc) {
273 request->dma_desc = dma_desc;
274 goto next;
275 }
276
277 /* Chain in any previous descriptors. */
278
279 prev_edesc->desc.next_desc = cpu_to_be32(dma_desc);
280
281 dma_sync_single_for_device(dev, prev_dma_desc,
282 TALITOS_DESC_SIZE,
283 DMA_TO_DEVICE);
284
285next:
286 prev_edesc = edesc;
287 prev_dma_desc = dma_desc;
288 edesc = edesc->next_desc;
289 }
290 } else {
291 request->dma_desc = dma_map_single(dev, desc, TALITOS_DESC_SIZE,
292 DMA_BIDIRECTIONAL);
293 }
294}
295
296/**
297 * talitos_submit - submits a descriptor to the device for processing
298 * @dev: the SEC device to be used
299 * @ch: the SEC device channel to be used
300 * @desc: the descriptor to be processed by the device
301 * @callback: whom to call when processing is complete
302 * @context: a handle for use by caller (optional)
303 *
304 * desc must contain valid dma-mapped (bus physical) address pointers.
305 * callback must check err and feedback in descriptor header
306 * for device processing status.
307 */
308static int talitos_submit(struct device *dev, int ch, struct talitos_desc *desc,
309 void (*callback)(struct device *dev,
310 struct talitos_desc *desc,
311 void *context, int error),
312 void *context)
313{
314 struct talitos_private *priv = dev_get_drvdata(dev);
315 struct talitos_request *request;
316 unsigned long flags;
317 int head;
318 bool is_sec1 = has_ftr_sec1(priv);
319
320 spin_lock_irqsave(&priv->chan[ch].head_lock, flags);
321
322 if (!atomic_inc_not_zero(&priv->chan[ch].submit_count)) {
323 /* h/w fifo is full */
324 spin_unlock_irqrestore(&priv->chan[ch].head_lock, flags);
325 return -EAGAIN;
326 }
327
328 head = priv->chan[ch].head;
329 request = &priv->chan[ch].fifo[head];
330
331 /* map descriptor and save caller data */
332 dma_map_request(dev, request, desc, is_sec1);
333 request->callback = callback;
334 request->context = context;
335
336 /* increment fifo head */
337 priv->chan[ch].head = (priv->chan[ch].head + 1) & (priv->fifo_len - 1);
338
339 smp_wmb();
340 request->desc = desc;
341
342 /* GO! */
343 wmb();
344 out_be32(priv->chan[ch].reg + TALITOS_FF,
345 upper_32_bits(request->dma_desc));
346 out_be32(priv->chan[ch].reg + TALITOS_FF_LO,
347 lower_32_bits(request->dma_desc));
348
349 spin_unlock_irqrestore(&priv->chan[ch].head_lock, flags);
350
351 return -EINPROGRESS;
352}
353
354static __be32 get_request_hdr(struct device *dev,
355 struct talitos_request *request, bool is_sec1)
356{
357 struct talitos_edesc *edesc;
358 dma_addr_t dma_desc;
359
360 if (!is_sec1) {
361 dma_sync_single_for_cpu(dev, request->dma_desc,
362 TALITOS_DESC_SIZE, DMA_BIDIRECTIONAL);
363
364 return request->desc->hdr;
365 }
366
367 edesc = container_of(request->desc, struct talitos_edesc, desc);
368 dma_desc = request->dma_desc;
369 while (edesc->next_desc) {
370 dma_desc = be32_to_cpu(edesc->desc.next_desc);
371 edesc = edesc->next_desc;
372 }
373
374 dma_sync_single_for_cpu(dev, dma_desc, TALITOS_DESC_SIZE,
375 DMA_BIDIRECTIONAL);
376
377 return edesc->desc.hdr1;
378}
379
380static void dma_unmap_request(struct device *dev,
381 struct talitos_request *request, bool is_sec1)
382{
383 struct talitos_edesc *edesc;
384
385 if (is_sec1) {
386 dma_unmap_single(dev, request->dma_desc, TALITOS_DESC_SIZE,
387 DMA_BIDIRECTIONAL);
388 edesc = container_of(request->desc, struct talitos_edesc, desc);
389 while (edesc->next_desc) {
390 dma_unmap_single(dev,
391 be32_to_cpu(edesc->desc.next_desc),
392 TALITOS_DESC_SIZE, DMA_BIDIRECTIONAL);
393 edesc = edesc->next_desc;
394 }
395 } else {
396 dma_unmap_single(dev, request->dma_desc, TALITOS_DESC_SIZE,
397 DMA_BIDIRECTIONAL);
398 }
399}
400
401/*
402 * process what was done, notify callback of error if not
403 */
404static void flush_channel(struct device *dev, int ch, int error, int reset_ch)
405{
406 struct talitos_private *priv = dev_get_drvdata(dev);
407 struct talitos_request *request, saved_req;
408 unsigned long flags;
409 int tail, status;
410 bool is_sec1 = has_ftr_sec1(priv);
411
412 spin_lock_irqsave(&priv->chan[ch].tail_lock, flags);
413
414 tail = priv->chan[ch].tail;
415 while (priv->chan[ch].fifo[tail].desc) {
416 __be32 hdr;
417
418 request = &priv->chan[ch].fifo[tail];
419
420 /* descriptors with their done bits set don't get the error */
421 rmb();
422 hdr = get_request_hdr(dev, request, is_sec1);
423
424 if ((hdr & DESC_HDR_DONE) == DESC_HDR_DONE)
425 status = 0;
426 else
427 if (!error)
428 break;
429 else
430 status = error;
431
432 dma_unmap_request(dev, request, is_sec1);
433
434 /* copy entries so we can call callback outside lock */
435 saved_req.desc = request->desc;
436 saved_req.callback = request->callback;
437 saved_req.context = request->context;
438
439 /* release request entry in fifo */
440 smp_wmb();
441 request->desc = NULL;
442
443 /* increment fifo tail */
444 priv->chan[ch].tail = (tail + 1) & (priv->fifo_len - 1);
445
446 spin_unlock_irqrestore(&priv->chan[ch].tail_lock, flags);
447
448 atomic_dec(&priv->chan[ch].submit_count);
449
450 saved_req.callback(dev, saved_req.desc, saved_req.context,
451 status);
452 /* channel may resume processing in single desc error case */
453 if (error && !reset_ch && status == error)
454 return;
455 spin_lock_irqsave(&priv->chan[ch].tail_lock, flags);
456 tail = priv->chan[ch].tail;
457 }
458
459 spin_unlock_irqrestore(&priv->chan[ch].tail_lock, flags);
460}
461
462/*
463 * process completed requests for channels that have done status
464 */
465#define DEF_TALITOS1_DONE(name, ch_done_mask) \
466static void talitos1_done_##name(unsigned long data) \
467{ \
468 struct device *dev = (struct device *)data; \
469 struct talitos_private *priv = dev_get_drvdata(dev); \
470 unsigned long flags; \
471 \
472 if (ch_done_mask & 0x10000000) \
473 flush_channel(dev, 0, 0, 0); \
474 if (ch_done_mask & 0x40000000) \
475 flush_channel(dev, 1, 0, 0); \
476 if (ch_done_mask & 0x00010000) \
477 flush_channel(dev, 2, 0, 0); \
478 if (ch_done_mask & 0x00040000) \
479 flush_channel(dev, 3, 0, 0); \
480 \
481 /* At this point, all completed channels have been processed */ \
482 /* Unmask done interrupts for channels completed later on. */ \
483 spin_lock_irqsave(&priv->reg_lock, flags); \
484 clrbits32(priv->reg + TALITOS_IMR, ch_done_mask); \
485 clrbits32(priv->reg + TALITOS_IMR_LO, TALITOS1_IMR_LO_INIT); \
486 spin_unlock_irqrestore(&priv->reg_lock, flags); \
487}
488
489DEF_TALITOS1_DONE(4ch, TALITOS1_ISR_4CHDONE)
490DEF_TALITOS1_DONE(ch0, TALITOS1_ISR_CH_0_DONE)
491
492#define DEF_TALITOS2_DONE(name, ch_done_mask) \
493static void talitos2_done_##name(unsigned long data) \
494{ \
495 struct device *dev = (struct device *)data; \
496 struct talitos_private *priv = dev_get_drvdata(dev); \
497 unsigned long flags; \
498 \
499 if (ch_done_mask & 1) \
500 flush_channel(dev, 0, 0, 0); \
501 if (ch_done_mask & (1 << 2)) \
502 flush_channel(dev, 1, 0, 0); \
503 if (ch_done_mask & (1 << 4)) \
504 flush_channel(dev, 2, 0, 0); \
505 if (ch_done_mask & (1 << 6)) \
506 flush_channel(dev, 3, 0, 0); \
507 \
508 /* At this point, all completed channels have been processed */ \
509 /* Unmask done interrupts for channels completed later on. */ \
510 spin_lock_irqsave(&priv->reg_lock, flags); \
511 setbits32(priv->reg + TALITOS_IMR, ch_done_mask); \
512 setbits32(priv->reg + TALITOS_IMR_LO, TALITOS2_IMR_LO_INIT); \
513 spin_unlock_irqrestore(&priv->reg_lock, flags); \
514}
515
516DEF_TALITOS2_DONE(4ch, TALITOS2_ISR_4CHDONE)
517DEF_TALITOS2_DONE(ch0, TALITOS2_ISR_CH_0_DONE)
518DEF_TALITOS2_DONE(ch0_2, TALITOS2_ISR_CH_0_2_DONE)
519DEF_TALITOS2_DONE(ch1_3, TALITOS2_ISR_CH_1_3_DONE)
520
521static __be32 search_desc_hdr_in_request(struct talitos_request *request,
522 dma_addr_t cur_desc, bool is_sec1)
523{
524 struct talitos_edesc *edesc;
525
526 if (request->dma_desc == cur_desc) {
527 return request->desc->hdr;
528 } else if (is_sec1) {
529 edesc = container_of(request->desc, struct talitos_edesc, desc);
530 while (edesc->next_desc) {
531 if (edesc->desc.next_desc == cpu_to_be32(cur_desc))
532 return edesc->next_desc->desc.hdr1;
533 edesc = edesc->next_desc;
534 }
535 }
536 return 0;
537}
538
539/*
540 * locate current (offending) descriptor
541 */
542static __be32 current_desc_hdr(struct device *dev, int ch)
543{
544 struct talitos_private *priv = dev_get_drvdata(dev);
545 bool is_sec1 = has_ftr_sec1(priv);
546 struct talitos_request *request;
547 int tail, iter;
548 dma_addr_t cur_desc;
549 __be32 hdr = 0;
550
551 cur_desc = ((u64)in_be32(priv->chan[ch].reg + TALITOS_CDPR)) << 32;
552 cur_desc |= in_be32(priv->chan[ch].reg + TALITOS_CDPR_LO);
553
554 if (!cur_desc) {
555 dev_err(dev, "CDPR is NULL, giving up search for offending descriptor\n");
556 return 0;
557 }
558
559 tail = priv->chan[ch].tail;
560 iter = tail;
561 do {
562 request = &priv->chan[ch].fifo[iter];
563
564 hdr = search_desc_hdr_in_request(request, cur_desc, is_sec1);
565 if (hdr)
566 break;
567
568 iter = (iter + 1) & (priv->fifo_len - 1);
569 } while (iter != tail);
570
571 if (!hdr)
572 dev_err(dev, "couldn't locate current descriptor\n");
573
574 return hdr;
575}
576
577/*
578 * user diagnostics; report root cause of error based on execution unit status
579 */
580static void report_eu_error(struct device *dev, int ch, __be32 desc_hdr)
581{
582 struct talitos_private *priv = dev_get_drvdata(dev);
583 int i;
584
585 if (!desc_hdr)
586 desc_hdr = cpu_to_be32(in_be32(priv->chan[ch].reg + TALITOS_DESCBUF));
587
588 switch (desc_hdr & DESC_HDR_SEL0_MASK) {
589 case DESC_HDR_SEL0_AFEU:
590 dev_err(dev, "AFEUISR 0x%08x_%08x\n",
591 in_be32(priv->reg_afeu + TALITOS_EUISR),
592 in_be32(priv->reg_afeu + TALITOS_EUISR_LO));
593 break;
594 case DESC_HDR_SEL0_DEU:
595 dev_err(dev, "DEUISR 0x%08x_%08x\n",
596 in_be32(priv->reg_deu + TALITOS_EUISR),
597 in_be32(priv->reg_deu + TALITOS_EUISR_LO));
598 break;
599 case DESC_HDR_SEL0_MDEUA:
600 case DESC_HDR_SEL0_MDEUB:
601 dev_err(dev, "MDEUISR 0x%08x_%08x\n",
602 in_be32(priv->reg_mdeu + TALITOS_EUISR),
603 in_be32(priv->reg_mdeu + TALITOS_EUISR_LO));
604 break;
605 case DESC_HDR_SEL0_RNG:
606 dev_err(dev, "RNGUISR 0x%08x_%08x\n",
607 in_be32(priv->reg_rngu + TALITOS_ISR),
608 in_be32(priv->reg_rngu + TALITOS_ISR_LO));
609 break;
610 case DESC_HDR_SEL0_PKEU:
611 dev_err(dev, "PKEUISR 0x%08x_%08x\n",
612 in_be32(priv->reg_pkeu + TALITOS_EUISR),
613 in_be32(priv->reg_pkeu + TALITOS_EUISR_LO));
614 break;
615 case DESC_HDR_SEL0_AESU:
616 dev_err(dev, "AESUISR 0x%08x_%08x\n",
617 in_be32(priv->reg_aesu + TALITOS_EUISR),
618 in_be32(priv->reg_aesu + TALITOS_EUISR_LO));
619 break;
620 case DESC_HDR_SEL0_CRCU:
621 dev_err(dev, "CRCUISR 0x%08x_%08x\n",
622 in_be32(priv->reg_crcu + TALITOS_EUISR),
623 in_be32(priv->reg_crcu + TALITOS_EUISR_LO));
624 break;
625 case DESC_HDR_SEL0_KEU:
626 dev_err(dev, "KEUISR 0x%08x_%08x\n",
627 in_be32(priv->reg_pkeu + TALITOS_EUISR),
628 in_be32(priv->reg_pkeu + TALITOS_EUISR_LO));
629 break;
630 }
631
632 switch (desc_hdr & DESC_HDR_SEL1_MASK) {
633 case DESC_HDR_SEL1_MDEUA:
634 case DESC_HDR_SEL1_MDEUB:
635 dev_err(dev, "MDEUISR 0x%08x_%08x\n",
636 in_be32(priv->reg_mdeu + TALITOS_EUISR),
637 in_be32(priv->reg_mdeu + TALITOS_EUISR_LO));
638 break;
639 case DESC_HDR_SEL1_CRCU:
640 dev_err(dev, "CRCUISR 0x%08x_%08x\n",
641 in_be32(priv->reg_crcu + TALITOS_EUISR),
642 in_be32(priv->reg_crcu + TALITOS_EUISR_LO));
643 break;
644 }
645
646 for (i = 0; i < 8; i++)
647 dev_err(dev, "DESCBUF 0x%08x_%08x\n",
648 in_be32(priv->chan[ch].reg + TALITOS_DESCBUF + 8*i),
649 in_be32(priv->chan[ch].reg + TALITOS_DESCBUF_LO + 8*i));
650}
651
652/*
653 * recover from error interrupts
654 */
655static void talitos_error(struct device *dev, u32 isr, u32 isr_lo)
656{
657 struct talitos_private *priv = dev_get_drvdata(dev);
658 unsigned int timeout = TALITOS_TIMEOUT;
659 int ch, error, reset_dev = 0;
660 u32 v_lo;
661 bool is_sec1 = has_ftr_sec1(priv);
662 int reset_ch = is_sec1 ? 1 : 0; /* only SEC2 supports continuation */
663
664 for (ch = 0; ch < priv->num_channels; ch++) {
665 /* skip channels without errors */
666 if (is_sec1) {
667 /* bits 29, 31, 17, 19 */
668 if (!(isr & (1 << (29 + (ch & 1) * 2 - (ch & 2) * 6))))
669 continue;
670 } else {
671 if (!(isr & (1 << (ch * 2 + 1))))
672 continue;
673 }
674
675 error = -EINVAL;
676
677 v_lo = in_be32(priv->chan[ch].reg + TALITOS_CCPSR_LO);
678
679 if (v_lo & TALITOS_CCPSR_LO_DOF) {
680 dev_err(dev, "double fetch fifo overflow error\n");
681 error = -EAGAIN;
682 reset_ch = 1;
683 }
684 if (v_lo & TALITOS_CCPSR_LO_SOF) {
685 /* h/w dropped descriptor */
686 dev_err(dev, "single fetch fifo overflow error\n");
687 error = -EAGAIN;
688 }
689 if (v_lo & TALITOS_CCPSR_LO_MDTE)
690 dev_err(dev, "master data transfer error\n");
691 if (v_lo & TALITOS_CCPSR_LO_SGDLZ)
692 dev_err(dev, is_sec1 ? "pointer not complete error\n"
693 : "s/g data length zero error\n");
694 if (v_lo & TALITOS_CCPSR_LO_FPZ)
695 dev_err(dev, is_sec1 ? "parity error\n"
696 : "fetch pointer zero error\n");
697 if (v_lo & TALITOS_CCPSR_LO_IDH)
698 dev_err(dev, "illegal descriptor header error\n");
699 if (v_lo & TALITOS_CCPSR_LO_IEU)
700 dev_err(dev, is_sec1 ? "static assignment error\n"
701 : "invalid exec unit error\n");
702 if (v_lo & TALITOS_CCPSR_LO_EU)
703 report_eu_error(dev, ch, current_desc_hdr(dev, ch));
704 if (!is_sec1) {
705 if (v_lo & TALITOS_CCPSR_LO_GB)
706 dev_err(dev, "gather boundary error\n");
707 if (v_lo & TALITOS_CCPSR_LO_GRL)
708 dev_err(dev, "gather return/length error\n");
709 if (v_lo & TALITOS_CCPSR_LO_SB)
710 dev_err(dev, "scatter boundary error\n");
711 if (v_lo & TALITOS_CCPSR_LO_SRL)
712 dev_err(dev, "scatter return/length error\n");
713 }
714
715 flush_channel(dev, ch, error, reset_ch);
716
717 if (reset_ch) {
718 reset_channel(dev, ch);
719 } else {
720 setbits32(priv->chan[ch].reg + TALITOS_CCCR,
721 TALITOS2_CCCR_CONT);
722 setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO, 0);
723 while ((in_be32(priv->chan[ch].reg + TALITOS_CCCR) &
724 TALITOS2_CCCR_CONT) && --timeout)
725 cpu_relax();
726 if (timeout == 0) {
727 dev_err(dev, "failed to restart channel %d\n",
728 ch);
729 reset_dev = 1;
730 }
731 }
732 }
733 if (reset_dev || (is_sec1 && isr & ~TALITOS1_ISR_4CHERR) ||
734 (!is_sec1 && isr & ~TALITOS2_ISR_4CHERR) || isr_lo) {
735 if (is_sec1 && (isr_lo & TALITOS1_ISR_TEA_ERR))
736 dev_err(dev, "TEA error: ISR 0x%08x_%08x\n",
737 isr, isr_lo);
738 else
739 dev_err(dev, "done overflow, internal time out, or "
740 "rngu error: ISR 0x%08x_%08x\n", isr, isr_lo);
741
742 /* purge request queues */
743 for (ch = 0; ch < priv->num_channels; ch++)
744 flush_channel(dev, ch, -EIO, 1);
745
746 /* reset and reinitialize the device */
747 init_device(dev);
748 }
749}
750
751#define DEF_TALITOS1_INTERRUPT(name, ch_done_mask, ch_err_mask, tlet) \
752static irqreturn_t talitos1_interrupt_##name(int irq, void *data) \
753{ \
754 struct device *dev = data; \
755 struct talitos_private *priv = dev_get_drvdata(dev); \
756 u32 isr, isr_lo; \
757 unsigned long flags; \
758 \
759 spin_lock_irqsave(&priv->reg_lock, flags); \
760 isr = in_be32(priv->reg + TALITOS_ISR); \
761 isr_lo = in_be32(priv->reg + TALITOS_ISR_LO); \
762 /* Acknowledge interrupt */ \
763 out_be32(priv->reg + TALITOS_ICR, isr & (ch_done_mask | ch_err_mask)); \
764 out_be32(priv->reg + TALITOS_ICR_LO, isr_lo); \
765 \
766 if (unlikely(isr & ch_err_mask || isr_lo & TALITOS1_IMR_LO_INIT)) { \
767 spin_unlock_irqrestore(&priv->reg_lock, flags); \
768 talitos_error(dev, isr & ch_err_mask, isr_lo); \
769 } \
770 else { \
771 if (likely(isr & ch_done_mask)) { \
772 /* mask further done interrupts. */ \
773 setbits32(priv->reg + TALITOS_IMR, ch_done_mask); \
774 /* done_task will unmask done interrupts at exit */ \
775 tasklet_schedule(&priv->done_task[tlet]); \
776 } \
777 spin_unlock_irqrestore(&priv->reg_lock, flags); \
778 } \
779 \
780 return (isr & (ch_done_mask | ch_err_mask) || isr_lo) ? IRQ_HANDLED : \
781 IRQ_NONE; \
782}
783
784DEF_TALITOS1_INTERRUPT(4ch, TALITOS1_ISR_4CHDONE, TALITOS1_ISR_4CHERR, 0)
785
786#define DEF_TALITOS2_INTERRUPT(name, ch_done_mask, ch_err_mask, tlet) \
787static irqreturn_t talitos2_interrupt_##name(int irq, void *data) \
788{ \
789 struct device *dev = data; \
790 struct talitos_private *priv = dev_get_drvdata(dev); \
791 u32 isr, isr_lo; \
792 unsigned long flags; \
793 \
794 spin_lock_irqsave(&priv->reg_lock, flags); \
795 isr = in_be32(priv->reg + TALITOS_ISR); \
796 isr_lo = in_be32(priv->reg + TALITOS_ISR_LO); \
797 /* Acknowledge interrupt */ \
798 out_be32(priv->reg + TALITOS_ICR, isr & (ch_done_mask | ch_err_mask)); \
799 out_be32(priv->reg + TALITOS_ICR_LO, isr_lo); \
800 \
801 if (unlikely(isr & ch_err_mask || isr_lo)) { \
802 spin_unlock_irqrestore(&priv->reg_lock, flags); \
803 talitos_error(dev, isr & ch_err_mask, isr_lo); \
804 } \
805 else { \
806 if (likely(isr & ch_done_mask)) { \
807 /* mask further done interrupts. */ \
808 clrbits32(priv->reg + TALITOS_IMR, ch_done_mask); \
809 /* done_task will unmask done interrupts at exit */ \
810 tasklet_schedule(&priv->done_task[tlet]); \
811 } \
812 spin_unlock_irqrestore(&priv->reg_lock, flags); \
813 } \
814 \
815 return (isr & (ch_done_mask | ch_err_mask) || isr_lo) ? IRQ_HANDLED : \
816 IRQ_NONE; \
817}
818
819DEF_TALITOS2_INTERRUPT(4ch, TALITOS2_ISR_4CHDONE, TALITOS2_ISR_4CHERR, 0)
820DEF_TALITOS2_INTERRUPT(ch0_2, TALITOS2_ISR_CH_0_2_DONE, TALITOS2_ISR_CH_0_2_ERR,
821 0)
822DEF_TALITOS2_INTERRUPT(ch1_3, TALITOS2_ISR_CH_1_3_DONE, TALITOS2_ISR_CH_1_3_ERR,
823 1)
824
825/*
826 * hwrng
827 */
828static int talitos_rng_data_present(struct hwrng *rng, int wait)
829{
830 struct device *dev = (struct device *)rng->priv;
831 struct talitos_private *priv = dev_get_drvdata(dev);
832 u32 ofl;
833 int i;
834
835 for (i = 0; i < 20; i++) {
836 ofl = in_be32(priv->reg_rngu + TALITOS_EUSR_LO) &
837 TALITOS_RNGUSR_LO_OFL;
838 if (ofl || !wait)
839 break;
840 udelay(10);
841 }
842
843 return !!ofl;
844}
845
846static int talitos_rng_data_read(struct hwrng *rng, u32 *data)
847{
848 struct device *dev = (struct device *)rng->priv;
849 struct talitos_private *priv = dev_get_drvdata(dev);
850
851 /* rng fifo requires 64-bit accesses */
852 *data = in_be32(priv->reg_rngu + TALITOS_EU_FIFO);
853 *data = in_be32(priv->reg_rngu + TALITOS_EU_FIFO_LO);
854
855 return sizeof(u32);
856}
857
858static int talitos_rng_init(struct hwrng *rng)
859{
860 struct device *dev = (struct device *)rng->priv;
861 struct talitos_private *priv = dev_get_drvdata(dev);
862 unsigned int timeout = TALITOS_TIMEOUT;
863
864 setbits32(priv->reg_rngu + TALITOS_EURCR_LO, TALITOS_RNGURCR_LO_SR);
865 while (!(in_be32(priv->reg_rngu + TALITOS_EUSR_LO)
866 & TALITOS_RNGUSR_LO_RD)
867 && --timeout)
868 cpu_relax();
869 if (timeout == 0) {
870 dev_err(dev, "failed to reset rng hw\n");
871 return -ENODEV;
872 }
873
874 /* start generating */
875 setbits32(priv->reg_rngu + TALITOS_EUDSR_LO, 0);
876
877 return 0;
878}
879
880static int talitos_register_rng(struct device *dev)
881{
882 struct talitos_private *priv = dev_get_drvdata(dev);
883 int err;
884
885 priv->rng.name = dev_driver_string(dev);
886 priv->rng.init = talitos_rng_init;
887 priv->rng.data_present = talitos_rng_data_present;
888 priv->rng.data_read = talitos_rng_data_read;
889 priv->rng.priv = (unsigned long)dev;
890
891 err = hwrng_register(&priv->rng);
892 if (!err)
893 priv->rng_registered = true;
894
895 return err;
896}
897
898static void talitos_unregister_rng(struct device *dev)
899{
900 struct talitos_private *priv = dev_get_drvdata(dev);
901
902 if (!priv->rng_registered)
903 return;
904
905 hwrng_unregister(&priv->rng);
906 priv->rng_registered = false;
907}
908
909/*
910 * crypto alg
911 */
912#define TALITOS_CRA_PRIORITY 3000
913/*
914 * Defines a priority for doing AEAD with descriptors type
915 * HMAC_SNOOP_NO_AFEA (HSNA) instead of type IPSEC_ESP
916 */
917#define TALITOS_CRA_PRIORITY_AEAD_HSNA (TALITOS_CRA_PRIORITY - 1)
918#ifdef CONFIG_CRYPTO_DEV_TALITOS2
919#define TALITOS_MAX_KEY_SIZE (AES_MAX_KEY_SIZE + SHA512_BLOCK_SIZE)
920#else
921#define TALITOS_MAX_KEY_SIZE (AES_MAX_KEY_SIZE + SHA256_BLOCK_SIZE)
922#endif
923#define TALITOS_MAX_IV_LENGTH 16 /* max of AES_BLOCK_SIZE, DES3_EDE_BLOCK_SIZE */
924
925struct talitos_ctx {
926 struct device *dev;
927 int ch;
928 __be32 desc_hdr_template;
929 u8 key[TALITOS_MAX_KEY_SIZE];
930 u8 iv[TALITOS_MAX_IV_LENGTH];
931 dma_addr_t dma_key;
932 unsigned int keylen;
933 unsigned int enckeylen;
934 unsigned int authkeylen;
935};
936
937#define HASH_MAX_BLOCK_SIZE SHA512_BLOCK_SIZE
938#define TALITOS_MDEU_MAX_CONTEXT_SIZE TALITOS_MDEU_CONTEXT_SIZE_SHA384_SHA512
939
940struct talitos_ahash_req_ctx {
941 u32 hw_context[TALITOS_MDEU_MAX_CONTEXT_SIZE / sizeof(u32)];
942 unsigned int hw_context_size;
943 u8 buf[2][HASH_MAX_BLOCK_SIZE];
944 int buf_idx;
945 unsigned int swinit;
946 unsigned int first_request;
947 unsigned int last_request;
948 unsigned int to_hash_later;
949 unsigned int nbuf;
950 struct scatterlist bufsl[2];
951 struct scatterlist *psrc;
952};
953
954struct talitos_export_state {
955 u32 hw_context[TALITOS_MDEU_MAX_CONTEXT_SIZE / sizeof(u32)];
956 u8 buf[HASH_MAX_BLOCK_SIZE];
957 unsigned int swinit;
958 unsigned int first_request;
959 unsigned int last_request;
960 unsigned int to_hash_later;
961 unsigned int nbuf;
962};
963
964static int aead_setkey(struct crypto_aead *authenc,
965 const u8 *key, unsigned int keylen)
966{
967 struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
968 struct device *dev = ctx->dev;
969 struct crypto_authenc_keys keys;
970
971 if (crypto_authenc_extractkeys(&keys, key, keylen) != 0)
972 goto badkey;
973
974 if (keys.authkeylen + keys.enckeylen > TALITOS_MAX_KEY_SIZE)
975 goto badkey;
976
977 if (ctx->keylen)
978 dma_unmap_single(dev, ctx->dma_key, ctx->keylen, DMA_TO_DEVICE);
979
980 memcpy(ctx->key, keys.authkey, keys.authkeylen);
981 memcpy(&ctx->key[keys.authkeylen], keys.enckey, keys.enckeylen);
982
983 ctx->keylen = keys.authkeylen + keys.enckeylen;
984 ctx->enckeylen = keys.enckeylen;
985 ctx->authkeylen = keys.authkeylen;
986 ctx->dma_key = dma_map_single(dev, ctx->key, ctx->keylen,
987 DMA_TO_DEVICE);
988
989 memzero_explicit(&keys, sizeof(keys));
990 return 0;
991
992badkey:
993 memzero_explicit(&keys, sizeof(keys));
994 return -EINVAL;
995}
996
997static int aead_des3_setkey(struct crypto_aead *authenc,
998 const u8 *key, unsigned int keylen)
999{
1000 struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
1001 struct device *dev = ctx->dev;
1002 struct crypto_authenc_keys keys;
1003 int err;
1004
1005 err = crypto_authenc_extractkeys(&keys, key, keylen);
1006 if (unlikely(err))
1007 goto out;
1008
1009 err = -EINVAL;
1010 if (keys.authkeylen + keys.enckeylen > TALITOS_MAX_KEY_SIZE)
1011 goto out;
1012
1013 err = verify_aead_des3_key(authenc, keys.enckey, keys.enckeylen);
1014 if (err)
1015 goto out;
1016
1017 if (ctx->keylen)
1018 dma_unmap_single(dev, ctx->dma_key, ctx->keylen, DMA_TO_DEVICE);
1019
1020 memcpy(ctx->key, keys.authkey, keys.authkeylen);
1021 memcpy(&ctx->key[keys.authkeylen], keys.enckey, keys.enckeylen);
1022
1023 ctx->keylen = keys.authkeylen + keys.enckeylen;
1024 ctx->enckeylen = keys.enckeylen;
1025 ctx->authkeylen = keys.authkeylen;
1026 ctx->dma_key = dma_map_single(dev, ctx->key, ctx->keylen,
1027 DMA_TO_DEVICE);
1028
1029out:
1030 memzero_explicit(&keys, sizeof(keys));
1031 return err;
1032}
1033
1034static void talitos_sg_unmap(struct device *dev,
1035 struct talitos_edesc *edesc,
1036 struct scatterlist *src,
1037 struct scatterlist *dst,
1038 unsigned int len, unsigned int offset)
1039{
1040 struct talitos_private *priv = dev_get_drvdata(dev);
1041 bool is_sec1 = has_ftr_sec1(priv);
1042 unsigned int src_nents = edesc->src_nents ? : 1;
1043 unsigned int dst_nents = edesc->dst_nents ? : 1;
1044
1045 if (is_sec1 && dst && dst_nents > 1) {
1046 dma_sync_single_for_device(dev, edesc->dma_link_tbl + offset,
1047 len, DMA_FROM_DEVICE);
1048 sg_pcopy_from_buffer(dst, dst_nents, edesc->buf + offset, len,
1049 offset);
1050 }
1051 if (src != dst) {
1052 if (src_nents == 1 || !is_sec1)
1053 dma_unmap_sg(dev, src, src_nents, DMA_TO_DEVICE);
1054
1055 if (dst && (dst_nents == 1 || !is_sec1))
1056 dma_unmap_sg(dev, dst, dst_nents, DMA_FROM_DEVICE);
1057 } else if (src_nents == 1 || !is_sec1) {
1058 dma_unmap_sg(dev, src, src_nents, DMA_BIDIRECTIONAL);
1059 }
1060}
1061
1062static void ipsec_esp_unmap(struct device *dev,
1063 struct talitos_edesc *edesc,
1064 struct aead_request *areq, bool encrypt)
1065{
1066 struct crypto_aead *aead = crypto_aead_reqtfm(areq);
1067 struct talitos_ctx *ctx = crypto_aead_ctx(aead);
1068 unsigned int ivsize = crypto_aead_ivsize(aead);
1069 unsigned int authsize = crypto_aead_authsize(aead);
1070 unsigned int cryptlen = areq->cryptlen - (encrypt ? 0 : authsize);
1071 bool is_ipsec_esp = edesc->desc.hdr & DESC_HDR_TYPE_IPSEC_ESP;
1072 struct talitos_ptr *civ_ptr = &edesc->desc.ptr[is_ipsec_esp ? 2 : 3];
1073
1074 if (is_ipsec_esp)
1075 unmap_single_talitos_ptr(dev, &edesc->desc.ptr[6],
1076 DMA_FROM_DEVICE);
1077 unmap_single_talitos_ptr(dev, civ_ptr, DMA_TO_DEVICE);
1078
1079 talitos_sg_unmap(dev, edesc, areq->src, areq->dst,
1080 cryptlen + authsize, areq->assoclen);
1081
1082 if (edesc->dma_len)
1083 dma_unmap_single(dev, edesc->dma_link_tbl, edesc->dma_len,
1084 DMA_BIDIRECTIONAL);
1085
1086 if (!is_ipsec_esp) {
1087 unsigned int dst_nents = edesc->dst_nents ? : 1;
1088
1089 sg_pcopy_to_buffer(areq->dst, dst_nents, ctx->iv, ivsize,
1090 areq->assoclen + cryptlen - ivsize);
1091 }
1092}
1093
1094/*
1095 * ipsec_esp descriptor callbacks
1096 */
1097static void ipsec_esp_encrypt_done(struct device *dev,
1098 struct talitos_desc *desc, void *context,
1099 int err)
1100{
1101 struct aead_request *areq = context;
1102 struct crypto_aead *authenc = crypto_aead_reqtfm(areq);
1103 unsigned int ivsize = crypto_aead_ivsize(authenc);
1104 struct talitos_edesc *edesc;
1105
1106 edesc = container_of(desc, struct talitos_edesc, desc);
1107
1108 ipsec_esp_unmap(dev, edesc, areq, true);
1109
1110 dma_unmap_single(dev, edesc->iv_dma, ivsize, DMA_TO_DEVICE);
1111
1112 kfree(edesc);
1113
1114 aead_request_complete(areq, err);
1115}
1116
1117static void ipsec_esp_decrypt_swauth_done(struct device *dev,
1118 struct talitos_desc *desc,
1119 void *context, int err)
1120{
1121 struct aead_request *req = context;
1122 struct crypto_aead *authenc = crypto_aead_reqtfm(req);
1123 unsigned int authsize = crypto_aead_authsize(authenc);
1124 struct talitos_edesc *edesc;
1125 char *oicv, *icv;
1126
1127 edesc = container_of(desc, struct talitos_edesc, desc);
1128
1129 ipsec_esp_unmap(dev, edesc, req, false);
1130
1131 if (!err) {
1132 /* auth check */
1133 oicv = edesc->buf + edesc->dma_len;
1134 icv = oicv - authsize;
1135
1136 err = crypto_memneq(oicv, icv, authsize) ? -EBADMSG : 0;
1137 }
1138
1139 kfree(edesc);
1140
1141 aead_request_complete(req, err);
1142}
1143
1144static void ipsec_esp_decrypt_hwauth_done(struct device *dev,
1145 struct talitos_desc *desc,
1146 void *context, int err)
1147{
1148 struct aead_request *req = context;
1149 struct talitos_edesc *edesc;
1150
1151 edesc = container_of(desc, struct talitos_edesc, desc);
1152
1153 ipsec_esp_unmap(dev, edesc, req, false);
1154
1155 /* check ICV auth status */
1156 if (!err && ((desc->hdr_lo & DESC_HDR_LO_ICCR1_MASK) !=
1157 DESC_HDR_LO_ICCR1_PASS))
1158 err = -EBADMSG;
1159
1160 kfree(edesc);
1161
1162 aead_request_complete(req, err);
1163}
1164
1165/*
1166 * convert scatterlist to SEC h/w link table format
1167 * stop at cryptlen bytes
1168 */
1169static int sg_to_link_tbl_offset(struct scatterlist *sg, int sg_count,
1170 unsigned int offset, int datalen, int elen,
1171 struct talitos_ptr *link_tbl_ptr, int align)
1172{
1173 int n_sg = elen ? sg_count + 1 : sg_count;
1174 int count = 0;
1175 int cryptlen = datalen + elen;
1176 int padding = ALIGN(cryptlen, align) - cryptlen;
1177
1178 while (cryptlen && sg && n_sg--) {
1179 unsigned int len = sg_dma_len(sg);
1180
1181 if (offset >= len) {
1182 offset -= len;
1183 goto next;
1184 }
1185
1186 len -= offset;
1187
1188 if (len > cryptlen)
1189 len = cryptlen;
1190
1191 if (datalen > 0 && len > datalen) {
1192 to_talitos_ptr(link_tbl_ptr + count,
1193 sg_dma_address(sg) + offset, datalen, 0);
1194 to_talitos_ptr_ext_set(link_tbl_ptr + count, 0, 0);
1195 count++;
1196 len -= datalen;
1197 offset += datalen;
1198 }
1199 to_talitos_ptr(link_tbl_ptr + count,
1200 sg_dma_address(sg) + offset, sg_next(sg) ? len : len + padding, 0);
1201 to_talitos_ptr_ext_set(link_tbl_ptr + count, 0, 0);
1202 count++;
1203 cryptlen -= len;
1204 datalen -= len;
1205 offset = 0;
1206
1207next:
1208 sg = sg_next(sg);
1209 }
1210
1211 /* tag end of link table */
1212 if (count > 0)
1213 to_talitos_ptr_ext_set(link_tbl_ptr + count - 1,
1214 DESC_PTR_LNKTBL_RET, 0);
1215
1216 return count;
1217}
1218
1219static int talitos_sg_map_ext(struct device *dev, struct scatterlist *src,
1220 unsigned int len, struct talitos_edesc *edesc,
1221 struct talitos_ptr *ptr, int sg_count,
1222 unsigned int offset, int tbl_off, int elen,
1223 bool force, int align)
1224{
1225 struct talitos_private *priv = dev_get_drvdata(dev);
1226 bool is_sec1 = has_ftr_sec1(priv);
1227 int aligned_len = ALIGN(len, align);
1228
1229 if (!src) {
1230 to_talitos_ptr(ptr, 0, 0, is_sec1);
1231 return 1;
1232 }
1233 to_talitos_ptr_ext_set(ptr, elen, is_sec1);
1234 if (sg_count == 1 && !force) {
1235 to_talitos_ptr(ptr, sg_dma_address(src) + offset, aligned_len, is_sec1);
1236 return sg_count;
1237 }
1238 if (is_sec1) {
1239 to_talitos_ptr(ptr, edesc->dma_link_tbl + offset, aligned_len, is_sec1);
1240 return sg_count;
1241 }
1242 sg_count = sg_to_link_tbl_offset(src, sg_count, offset, len, elen,
1243 &edesc->link_tbl[tbl_off], align);
1244 if (sg_count == 1 && !force) {
1245 /* Only one segment now, so no link tbl needed*/
1246 copy_talitos_ptr(ptr, &edesc->link_tbl[tbl_off], is_sec1);
1247 return sg_count;
1248 }
1249 to_talitos_ptr(ptr, edesc->dma_link_tbl +
1250 tbl_off * sizeof(struct talitos_ptr), aligned_len, is_sec1);
1251 to_talitos_ptr_ext_or(ptr, DESC_PTR_LNKTBL_JUMP, is_sec1);
1252
1253 return sg_count;
1254}
1255
1256static int talitos_sg_map(struct device *dev, struct scatterlist *src,
1257 unsigned int len, struct talitos_edesc *edesc,
1258 struct talitos_ptr *ptr, int sg_count,
1259 unsigned int offset, int tbl_off)
1260{
1261 return talitos_sg_map_ext(dev, src, len, edesc, ptr, sg_count, offset,
1262 tbl_off, 0, false, 1);
1263}
1264
1265/*
1266 * fill in and submit ipsec_esp descriptor
1267 */
1268static int ipsec_esp(struct talitos_edesc *edesc, struct aead_request *areq,
1269 bool encrypt,
1270 void (*callback)(struct device *dev,
1271 struct talitos_desc *desc,
1272 void *context, int error))
1273{
1274 struct crypto_aead *aead = crypto_aead_reqtfm(areq);
1275 unsigned int authsize = crypto_aead_authsize(aead);
1276 struct talitos_ctx *ctx = crypto_aead_ctx(aead);
1277 struct device *dev = ctx->dev;
1278 struct talitos_desc *desc = &edesc->desc;
1279 unsigned int cryptlen = areq->cryptlen - (encrypt ? 0 : authsize);
1280 unsigned int ivsize = crypto_aead_ivsize(aead);
1281 int tbl_off = 0;
1282 int sg_count, ret;
1283 int elen = 0;
1284 bool sync_needed = false;
1285 struct talitos_private *priv = dev_get_drvdata(dev);
1286 bool is_sec1 = has_ftr_sec1(priv);
1287 bool is_ipsec_esp = desc->hdr & DESC_HDR_TYPE_IPSEC_ESP;
1288 struct talitos_ptr *civ_ptr = &desc->ptr[is_ipsec_esp ? 2 : 3];
1289 struct talitos_ptr *ckey_ptr = &desc->ptr[is_ipsec_esp ? 3 : 2];
1290 dma_addr_t dma_icv = edesc->dma_link_tbl + edesc->dma_len - authsize;
1291
1292 /* hmac key */
1293 to_talitos_ptr(&desc->ptr[0], ctx->dma_key, ctx->authkeylen, is_sec1);
1294
1295 sg_count = edesc->src_nents ?: 1;
1296 if (is_sec1 && sg_count > 1)
1297 sg_copy_to_buffer(areq->src, sg_count, edesc->buf,
1298 areq->assoclen + cryptlen);
1299 else
1300 sg_count = dma_map_sg(dev, areq->src, sg_count,
1301 (areq->src == areq->dst) ?
1302 DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
1303
1304 /* hmac data */
1305 ret = talitos_sg_map(dev, areq->src, areq->assoclen, edesc,
1306 &desc->ptr[1], sg_count, 0, tbl_off);
1307
1308 if (ret > 1) {
1309 tbl_off += ret;
1310 sync_needed = true;
1311 }
1312
1313 /* cipher iv */
1314 to_talitos_ptr(civ_ptr, edesc->iv_dma, ivsize, is_sec1);
1315
1316 /* cipher key */
1317 to_talitos_ptr(ckey_ptr, ctx->dma_key + ctx->authkeylen,
1318 ctx->enckeylen, is_sec1);
1319
1320 /*
1321 * cipher in
1322 * map and adjust cipher len to aead request cryptlen.
1323 * extent is bytes of HMAC postpended to ciphertext,
1324 * typically 12 for ipsec
1325 */
1326 if (is_ipsec_esp && (desc->hdr & DESC_HDR_MODE1_MDEU_CICV))
1327 elen = authsize;
1328
1329 ret = talitos_sg_map_ext(dev, areq->src, cryptlen, edesc, &desc->ptr[4],
1330 sg_count, areq->assoclen, tbl_off, elen,
1331 false, 1);
1332
1333 if (ret > 1) {
1334 tbl_off += ret;
1335 sync_needed = true;
1336 }
1337
1338 /* cipher out */
1339 if (areq->src != areq->dst) {
1340 sg_count = edesc->dst_nents ? : 1;
1341 if (!is_sec1 || sg_count == 1)
1342 dma_map_sg(dev, areq->dst, sg_count, DMA_FROM_DEVICE);
1343 }
1344
1345 if (is_ipsec_esp && encrypt)
1346 elen = authsize;
1347 else
1348 elen = 0;
1349 ret = talitos_sg_map_ext(dev, areq->dst, cryptlen, edesc, &desc->ptr[5],
1350 sg_count, areq->assoclen, tbl_off, elen,
1351 is_ipsec_esp && !encrypt, 1);
1352 tbl_off += ret;
1353
1354 if (!encrypt && is_ipsec_esp) {
1355 struct talitos_ptr *tbl_ptr = &edesc->link_tbl[tbl_off];
1356
1357 /* Add an entry to the link table for ICV data */
1358 to_talitos_ptr_ext_set(tbl_ptr - 1, 0, is_sec1);
1359 to_talitos_ptr_ext_set(tbl_ptr, DESC_PTR_LNKTBL_RET, is_sec1);
1360
1361 /* icv data follows link tables */
1362 to_talitos_ptr(tbl_ptr, dma_icv, authsize, is_sec1);
1363 to_talitos_ptr_ext_or(&desc->ptr[5], authsize, is_sec1);
1364 sync_needed = true;
1365 } else if (!encrypt) {
1366 to_talitos_ptr(&desc->ptr[6], dma_icv, authsize, is_sec1);
1367 sync_needed = true;
1368 } else if (!is_ipsec_esp) {
1369 talitos_sg_map(dev, areq->dst, authsize, edesc, &desc->ptr[6],
1370 sg_count, areq->assoclen + cryptlen, tbl_off);
1371 }
1372
1373 /* iv out */
1374 if (is_ipsec_esp)
1375 map_single_talitos_ptr(dev, &desc->ptr[6], ivsize, ctx->iv,
1376 DMA_FROM_DEVICE);
1377
1378 if (sync_needed)
1379 dma_sync_single_for_device(dev, edesc->dma_link_tbl,
1380 edesc->dma_len,
1381 DMA_BIDIRECTIONAL);
1382
1383 ret = talitos_submit(dev, ctx->ch, desc, callback, areq);
1384 if (ret != -EINPROGRESS) {
1385 ipsec_esp_unmap(dev, edesc, areq, encrypt);
1386 kfree(edesc);
1387 }
1388 return ret;
1389}
1390
1391/*
1392 * allocate and map the extended descriptor
1393 */
1394static struct talitos_edesc *talitos_edesc_alloc(struct device *dev,
1395 struct scatterlist *src,
1396 struct scatterlist *dst,
1397 u8 *iv,
1398 unsigned int assoclen,
1399 unsigned int cryptlen,
1400 unsigned int authsize,
1401 unsigned int ivsize,
1402 int icv_stashing,
1403 u32 cryptoflags,
1404 bool encrypt)
1405{
1406 struct talitos_edesc *edesc;
1407 int src_nents, dst_nents, alloc_len, dma_len, src_len, dst_len;
1408 dma_addr_t iv_dma = 0;
1409 gfp_t flags = cryptoflags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL :
1410 GFP_ATOMIC;
1411 struct talitos_private *priv = dev_get_drvdata(dev);
1412 bool is_sec1 = has_ftr_sec1(priv);
1413 int max_len = is_sec1 ? TALITOS1_MAX_DATA_LEN : TALITOS2_MAX_DATA_LEN;
1414
1415 if (cryptlen + authsize > max_len) {
1416 dev_err(dev, "length exceeds h/w max limit\n");
1417 return ERR_PTR(-EINVAL);
1418 }
1419
1420 if (!dst || dst == src) {
1421 src_len = assoclen + cryptlen + authsize;
1422 src_nents = sg_nents_for_len(src, src_len);
1423 if (src_nents < 0) {
1424 dev_err(dev, "Invalid number of src SG.\n");
1425 return ERR_PTR(-EINVAL);
1426 }
1427 src_nents = (src_nents == 1) ? 0 : src_nents;
1428 dst_nents = dst ? src_nents : 0;
1429 dst_len = 0;
1430 } else { /* dst && dst != src*/
1431 src_len = assoclen + cryptlen + (encrypt ? 0 : authsize);
1432 src_nents = sg_nents_for_len(src, src_len);
1433 if (src_nents < 0) {
1434 dev_err(dev, "Invalid number of src SG.\n");
1435 return ERR_PTR(-EINVAL);
1436 }
1437 src_nents = (src_nents == 1) ? 0 : src_nents;
1438 dst_len = assoclen + cryptlen + (encrypt ? authsize : 0);
1439 dst_nents = sg_nents_for_len(dst, dst_len);
1440 if (dst_nents < 0) {
1441 dev_err(dev, "Invalid number of dst SG.\n");
1442 return ERR_PTR(-EINVAL);
1443 }
1444 dst_nents = (dst_nents == 1) ? 0 : dst_nents;
1445 }
1446
1447 /*
1448 * allocate space for base edesc plus the link tables,
1449 * allowing for two separate entries for AD and generated ICV (+ 2),
1450 * and space for two sets of ICVs (stashed and generated)
1451 */
1452 alloc_len = sizeof(struct talitos_edesc);
1453 if (src_nents || dst_nents || !encrypt) {
1454 if (is_sec1)
1455 dma_len = (src_nents ? src_len : 0) +
1456 (dst_nents ? dst_len : 0) + authsize;
1457 else
1458 dma_len = (src_nents + dst_nents + 2) *
1459 sizeof(struct talitos_ptr) + authsize;
1460 alloc_len += dma_len;
1461 } else {
1462 dma_len = 0;
1463 }
1464 alloc_len += icv_stashing ? authsize : 0;
1465 alloc_len += ivsize;
1466
1467 edesc = kmalloc(ALIGN(alloc_len, dma_get_cache_alignment()), flags);
1468 if (!edesc)
1469 return ERR_PTR(-ENOMEM);
1470 if (ivsize) {
1471 iv = memcpy(((u8 *)edesc) + alloc_len - ivsize, iv, ivsize);
1472 iv_dma = dma_map_single(dev, iv, ivsize, DMA_TO_DEVICE);
1473 }
1474 memset(&edesc->desc, 0, sizeof(edesc->desc));
1475
1476 edesc->src_nents = src_nents;
1477 edesc->dst_nents = dst_nents;
1478 edesc->iv_dma = iv_dma;
1479 edesc->dma_len = dma_len;
1480 edesc->next_desc = NULL;
1481 if (dma_len)
1482 edesc->dma_link_tbl = dma_map_single(dev, &edesc->link_tbl[0],
1483 edesc->dma_len,
1484 DMA_BIDIRECTIONAL);
1485
1486 return edesc;
1487}
1488
1489static struct talitos_edesc *aead_edesc_alloc(struct aead_request *areq, u8 *iv,
1490 int icv_stashing, bool encrypt)
1491{
1492 struct crypto_aead *authenc = crypto_aead_reqtfm(areq);
1493 unsigned int authsize = crypto_aead_authsize(authenc);
1494 struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
1495 unsigned int ivsize = crypto_aead_ivsize(authenc);
1496 unsigned int cryptlen = areq->cryptlen - (encrypt ? 0 : authsize);
1497
1498 return talitos_edesc_alloc(ctx->dev, areq->src, areq->dst,
1499 iv, areq->assoclen, cryptlen,
1500 authsize, ivsize, icv_stashing,
1501 areq->base.flags, encrypt);
1502}
1503
1504static int aead_encrypt(struct aead_request *req)
1505{
1506 struct crypto_aead *authenc = crypto_aead_reqtfm(req);
1507 struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
1508 struct talitos_edesc *edesc;
1509
1510 /* allocate extended descriptor */
1511 edesc = aead_edesc_alloc(req, req->iv, 0, true);
1512 if (IS_ERR(edesc))
1513 return PTR_ERR(edesc);
1514
1515 /* set encrypt */
1516 edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_MODE0_ENCRYPT;
1517
1518 return ipsec_esp(edesc, req, true, ipsec_esp_encrypt_done);
1519}
1520
1521static int aead_decrypt(struct aead_request *req)
1522{
1523 struct crypto_aead *authenc = crypto_aead_reqtfm(req);
1524 unsigned int authsize = crypto_aead_authsize(authenc);
1525 struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
1526 struct talitos_private *priv = dev_get_drvdata(ctx->dev);
1527 struct talitos_edesc *edesc;
1528 void *icvdata;
1529
1530 /* allocate extended descriptor */
1531 edesc = aead_edesc_alloc(req, req->iv, 1, false);
1532 if (IS_ERR(edesc))
1533 return PTR_ERR(edesc);
1534
1535 if ((edesc->desc.hdr & DESC_HDR_TYPE_IPSEC_ESP) &&
1536 (priv->features & TALITOS_FTR_HW_AUTH_CHECK) &&
1537 ((!edesc->src_nents && !edesc->dst_nents) ||
1538 priv->features & TALITOS_FTR_SRC_LINK_TBL_LEN_INCLUDES_EXTENT)) {
1539
1540 /* decrypt and check the ICV */
1541 edesc->desc.hdr = ctx->desc_hdr_template |
1542 DESC_HDR_DIR_INBOUND |
1543 DESC_HDR_MODE1_MDEU_CICV;
1544
1545 /* reset integrity check result bits */
1546
1547 return ipsec_esp(edesc, req, false,
1548 ipsec_esp_decrypt_hwauth_done);
1549 }
1550
1551 /* Have to check the ICV with software */
1552 edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_DIR_INBOUND;
1553
1554 /* stash incoming ICV for later cmp with ICV generated by the h/w */
1555 icvdata = edesc->buf + edesc->dma_len;
1556
1557 sg_pcopy_to_buffer(req->src, edesc->src_nents ? : 1, icvdata, authsize,
1558 req->assoclen + req->cryptlen - authsize);
1559
1560 return ipsec_esp(edesc, req, false, ipsec_esp_decrypt_swauth_done);
1561}
1562
1563static int skcipher_setkey(struct crypto_skcipher *cipher,
1564 const u8 *key, unsigned int keylen)
1565{
1566 struct talitos_ctx *ctx = crypto_skcipher_ctx(cipher);
1567 struct device *dev = ctx->dev;
1568
1569 if (ctx->keylen)
1570 dma_unmap_single(dev, ctx->dma_key, ctx->keylen, DMA_TO_DEVICE);
1571
1572 memcpy(&ctx->key, key, keylen);
1573 ctx->keylen = keylen;
1574
1575 ctx->dma_key = dma_map_single(dev, ctx->key, keylen, DMA_TO_DEVICE);
1576
1577 return 0;
1578}
1579
1580static int skcipher_des_setkey(struct crypto_skcipher *cipher,
1581 const u8 *key, unsigned int keylen)
1582{
1583 return verify_skcipher_des_key(cipher, key) ?:
1584 skcipher_setkey(cipher, key, keylen);
1585}
1586
1587static int skcipher_des3_setkey(struct crypto_skcipher *cipher,
1588 const u8 *key, unsigned int keylen)
1589{
1590 return verify_skcipher_des3_key(cipher, key) ?:
1591 skcipher_setkey(cipher, key, keylen);
1592}
1593
1594static int skcipher_aes_setkey(struct crypto_skcipher *cipher,
1595 const u8 *key, unsigned int keylen)
1596{
1597 if (keylen == AES_KEYSIZE_128 || keylen == AES_KEYSIZE_192 ||
1598 keylen == AES_KEYSIZE_256)
1599 return skcipher_setkey(cipher, key, keylen);
1600
1601 return -EINVAL;
1602}
1603
1604static void common_nonsnoop_unmap(struct device *dev,
1605 struct talitos_edesc *edesc,
1606 struct skcipher_request *areq)
1607{
1608 unmap_single_talitos_ptr(dev, &edesc->desc.ptr[5], DMA_FROM_DEVICE);
1609
1610 talitos_sg_unmap(dev, edesc, areq->src, areq->dst, areq->cryptlen, 0);
1611 unmap_single_talitos_ptr(dev, &edesc->desc.ptr[1], DMA_TO_DEVICE);
1612
1613 if (edesc->dma_len)
1614 dma_unmap_single(dev, edesc->dma_link_tbl, edesc->dma_len,
1615 DMA_BIDIRECTIONAL);
1616}
1617
1618static void skcipher_done(struct device *dev,
1619 struct talitos_desc *desc, void *context,
1620 int err)
1621{
1622 struct skcipher_request *areq = context;
1623 struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(areq);
1624 struct talitos_ctx *ctx = crypto_skcipher_ctx(cipher);
1625 unsigned int ivsize = crypto_skcipher_ivsize(cipher);
1626 struct talitos_edesc *edesc;
1627
1628 edesc = container_of(desc, struct talitos_edesc, desc);
1629
1630 common_nonsnoop_unmap(dev, edesc, areq);
1631 memcpy(areq->iv, ctx->iv, ivsize);
1632
1633 kfree(edesc);
1634
1635 skcipher_request_complete(areq, err);
1636}
1637
1638static int common_nonsnoop(struct talitos_edesc *edesc,
1639 struct skcipher_request *areq,
1640 void (*callback) (struct device *dev,
1641 struct talitos_desc *desc,
1642 void *context, int error))
1643{
1644 struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(areq);
1645 struct talitos_ctx *ctx = crypto_skcipher_ctx(cipher);
1646 struct device *dev = ctx->dev;
1647 struct talitos_desc *desc = &edesc->desc;
1648 unsigned int cryptlen = areq->cryptlen;
1649 unsigned int ivsize = crypto_skcipher_ivsize(cipher);
1650 int sg_count, ret;
1651 bool sync_needed = false;
1652 struct talitos_private *priv = dev_get_drvdata(dev);
1653 bool is_sec1 = has_ftr_sec1(priv);
1654 bool is_ctr = (desc->hdr & DESC_HDR_SEL0_MASK) == DESC_HDR_SEL0_AESU &&
1655 (desc->hdr & DESC_HDR_MODE0_AESU_MASK) == DESC_HDR_MODE0_AESU_CTR;
1656
1657 /* first DWORD empty */
1658
1659 /* cipher iv */
1660 to_talitos_ptr(&desc->ptr[1], edesc->iv_dma, ivsize, is_sec1);
1661
1662 /* cipher key */
1663 to_talitos_ptr(&desc->ptr[2], ctx->dma_key, ctx->keylen, is_sec1);
1664
1665 sg_count = edesc->src_nents ?: 1;
1666 if (is_sec1 && sg_count > 1)
1667 sg_copy_to_buffer(areq->src, sg_count, edesc->buf,
1668 cryptlen);
1669 else
1670 sg_count = dma_map_sg(dev, areq->src, sg_count,
1671 (areq->src == areq->dst) ?
1672 DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
1673 /*
1674 * cipher in
1675 */
1676 sg_count = talitos_sg_map_ext(dev, areq->src, cryptlen, edesc, &desc->ptr[3],
1677 sg_count, 0, 0, 0, false, is_ctr ? 16 : 1);
1678 if (sg_count > 1)
1679 sync_needed = true;
1680
1681 /* cipher out */
1682 if (areq->src != areq->dst) {
1683 sg_count = edesc->dst_nents ? : 1;
1684 if (!is_sec1 || sg_count == 1)
1685 dma_map_sg(dev, areq->dst, sg_count, DMA_FROM_DEVICE);
1686 }
1687
1688 ret = talitos_sg_map(dev, areq->dst, cryptlen, edesc, &desc->ptr[4],
1689 sg_count, 0, (edesc->src_nents + 1));
1690 if (ret > 1)
1691 sync_needed = true;
1692
1693 /* iv out */
1694 map_single_talitos_ptr(dev, &desc->ptr[5], ivsize, ctx->iv,
1695 DMA_FROM_DEVICE);
1696
1697 /* last DWORD empty */
1698
1699 if (sync_needed)
1700 dma_sync_single_for_device(dev, edesc->dma_link_tbl,
1701 edesc->dma_len, DMA_BIDIRECTIONAL);
1702
1703 ret = talitos_submit(dev, ctx->ch, desc, callback, areq);
1704 if (ret != -EINPROGRESS) {
1705 common_nonsnoop_unmap(dev, edesc, areq);
1706 kfree(edesc);
1707 }
1708 return ret;
1709}
1710
1711static struct talitos_edesc *skcipher_edesc_alloc(struct skcipher_request *
1712 areq, bool encrypt)
1713{
1714 struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(areq);
1715 struct talitos_ctx *ctx = crypto_skcipher_ctx(cipher);
1716 unsigned int ivsize = crypto_skcipher_ivsize(cipher);
1717
1718 return talitos_edesc_alloc(ctx->dev, areq->src, areq->dst,
1719 areq->iv, 0, areq->cryptlen, 0, ivsize, 0,
1720 areq->base.flags, encrypt);
1721}
1722
1723static int skcipher_encrypt(struct skcipher_request *areq)
1724{
1725 struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(areq);
1726 struct talitos_ctx *ctx = crypto_skcipher_ctx(cipher);
1727 struct talitos_edesc *edesc;
1728 unsigned int blocksize =
1729 crypto_tfm_alg_blocksize(crypto_skcipher_tfm(cipher));
1730
1731 if (!areq->cryptlen)
1732 return 0;
1733
1734 if (areq->cryptlen % blocksize)
1735 return -EINVAL;
1736
1737 /* allocate extended descriptor */
1738 edesc = skcipher_edesc_alloc(areq, true);
1739 if (IS_ERR(edesc))
1740 return PTR_ERR(edesc);
1741
1742 /* set encrypt */
1743 edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_MODE0_ENCRYPT;
1744
1745 return common_nonsnoop(edesc, areq, skcipher_done);
1746}
1747
1748static int skcipher_decrypt(struct skcipher_request *areq)
1749{
1750 struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(areq);
1751 struct talitos_ctx *ctx = crypto_skcipher_ctx(cipher);
1752 struct talitos_edesc *edesc;
1753 unsigned int blocksize =
1754 crypto_tfm_alg_blocksize(crypto_skcipher_tfm(cipher));
1755
1756 if (!areq->cryptlen)
1757 return 0;
1758
1759 if (areq->cryptlen % blocksize)
1760 return -EINVAL;
1761
1762 /* allocate extended descriptor */
1763 edesc = skcipher_edesc_alloc(areq, false);
1764 if (IS_ERR(edesc))
1765 return PTR_ERR(edesc);
1766
1767 edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_DIR_INBOUND;
1768
1769 return common_nonsnoop(edesc, areq, skcipher_done);
1770}
1771
1772static void common_nonsnoop_hash_unmap(struct device *dev,
1773 struct talitos_edesc *edesc,
1774 struct ahash_request *areq)
1775{
1776 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1777 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
1778 struct talitos_private *priv = dev_get_drvdata(dev);
1779 bool is_sec1 = has_ftr_sec1(priv);
1780 struct talitos_desc *desc = &edesc->desc;
1781
1782 unmap_single_talitos_ptr(dev, &desc->ptr[5], DMA_FROM_DEVICE);
1783
1784 if (edesc->last && req_ctx->last_request)
1785 memcpy(areq->result, req_ctx->hw_context,
1786 crypto_ahash_digestsize(tfm));
1787
1788 if (edesc->src)
1789 talitos_sg_unmap(dev, edesc, edesc->src, NULL, 0, 0);
1790
1791 /* When using hashctx-in, must unmap it. */
1792 if (from_talitos_ptr_len(&desc->ptr[1], is_sec1))
1793 unmap_single_talitos_ptr(dev, &desc->ptr[1],
1794 DMA_TO_DEVICE);
1795
1796 if (edesc->dma_len)
1797 dma_unmap_single(dev, edesc->dma_link_tbl, edesc->dma_len,
1798 DMA_BIDIRECTIONAL);
1799}
1800
1801static void free_edesc_list_from(struct ahash_request *areq, struct talitos_edesc *edesc)
1802{
1803 struct talitos_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
1804 struct talitos_edesc *next;
1805
1806 while (edesc) {
1807 next = edesc->next_desc;
1808 common_nonsnoop_hash_unmap(ctx->dev, edesc, areq);
1809 kfree(edesc);
1810 edesc = next;
1811 }
1812}
1813
1814static void ahash_done(struct device *dev,
1815 struct talitos_desc *desc, void *context,
1816 int err)
1817{
1818 struct ahash_request *areq = context;
1819 struct talitos_edesc *edesc =
1820 container_of(desc, struct talitos_edesc, desc);
1821 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1822 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
1823 bool is_sec1 = has_ftr_sec1(dev_get_drvdata(dev));
1824 struct talitos_ctx *ctx = crypto_ahash_ctx(tfm);
1825 struct talitos_edesc *next;
1826
1827 if (is_sec1) {
1828 if (!req_ctx->last_request && req_ctx->to_hash_later) {
1829 /* Position any partial block for next update/final/finup */
1830 req_ctx->buf_idx = (req_ctx->buf_idx + 1) & 1;
1831 req_ctx->nbuf = req_ctx->to_hash_later;
1832 }
1833
1834 free_edesc_list_from(areq, edesc);
1835 ahash_request_complete(areq, err);
1836 } else {
1837 next = edesc->next_desc;
1838
1839 common_nonsnoop_hash_unmap(dev, edesc, areq);
1840 kfree(edesc);
1841
1842 if (err)
1843 goto out;
1844
1845 if (next) {
1846 err = talitos_submit(dev, ctx->ch, &next->desc,
1847 ahash_done, areq);
1848 if (err != -EINPROGRESS)
1849 goto out;
1850 return;
1851 }
1852out:
1853 if (!req_ctx->last_request && req_ctx->to_hash_later) {
1854 /* Position any partial block for next update/final/finup */
1855 req_ctx->buf_idx = (req_ctx->buf_idx + 1) & 1;
1856 req_ctx->nbuf = req_ctx->to_hash_later;
1857 }
1858 if (err && next)
1859 free_edesc_list_from(areq, next);
1860 ahash_request_complete(areq, err);
1861 }
1862}
1863
1864/*
1865 * SEC1 doesn't like hashing of 0 sized message, so we do the padding
1866 * ourself and submit a padded block
1867 */
1868static void talitos_handle_buggy_hash(struct talitos_ctx *ctx,
1869 struct talitos_edesc *edesc,
1870 struct talitos_ptr *ptr)
1871{
1872 static u8 padded_hash[64] = {
1873 0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
1874 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
1875 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
1876 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
1877 };
1878
1879 pr_err_once("Bug in SEC1, padding ourself\n");
1880 edesc->desc.hdr &= ~DESC_HDR_MODE0_MDEU_PAD;
1881 map_single_talitos_ptr(ctx->dev, ptr, sizeof(padded_hash),
1882 (char *)padded_hash, DMA_TO_DEVICE);
1883}
1884
1885static void common_nonsnoop_hash(struct talitos_edesc *edesc,
1886 struct ahash_request *areq,
1887 unsigned int length)
1888{
1889 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
1890 struct talitos_ctx *ctx = crypto_ahash_ctx(tfm);
1891 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1892 struct device *dev = ctx->dev;
1893 struct talitos_desc *desc = &edesc->desc;
1894 bool sync_needed = false;
1895 struct talitos_private *priv = dev_get_drvdata(dev);
1896 bool is_sec1 = has_ftr_sec1(priv);
1897 int sg_count;
1898
1899 /* first DWORD empty */
1900
1901 /* hash context in */
1902 if (!edesc->first || !req_ctx->first_request || req_ctx->swinit) {
1903 map_single_talitos_ptr_nosync(dev, &desc->ptr[1],
1904 req_ctx->hw_context_size,
1905 req_ctx->hw_context,
1906 DMA_TO_DEVICE);
1907 req_ctx->swinit = 0;
1908 }
1909 /* Indicate next op is not the first. */
1910 req_ctx->first_request = 0;
1911
1912 /* HMAC key */
1913 if (ctx->keylen)
1914 to_talitos_ptr(&desc->ptr[2], ctx->dma_key, ctx->keylen,
1915 is_sec1);
1916
1917 sg_count = edesc->src_nents ?: 1;
1918 if (is_sec1 && sg_count > 1)
1919 sg_copy_to_buffer(edesc->src, sg_count, edesc->buf, length);
1920 else if (length)
1921 sg_count = dma_map_sg(dev, edesc->src, sg_count, DMA_TO_DEVICE);
1922
1923 /*
1924 * data in
1925 */
1926 sg_count = talitos_sg_map(dev, edesc->src, length, edesc, &desc->ptr[3],
1927 sg_count, 0, 0);
1928 if (sg_count > 1)
1929 sync_needed = true;
1930
1931 /* fifth DWORD empty */
1932
1933 /* hash/HMAC out -or- hash context out */
1934 if (edesc->last && req_ctx->last_request)
1935 map_single_talitos_ptr(dev, &desc->ptr[5],
1936 crypto_ahash_digestsize(tfm),
1937 req_ctx->hw_context, DMA_FROM_DEVICE);
1938 else
1939 map_single_talitos_ptr_nosync(dev, &desc->ptr[5],
1940 req_ctx->hw_context_size,
1941 req_ctx->hw_context,
1942 DMA_FROM_DEVICE);
1943
1944 /* last DWORD empty */
1945
1946 if (is_sec1 && from_talitos_ptr_len(&desc->ptr[3], true) == 0)
1947 talitos_handle_buggy_hash(ctx, edesc, &desc->ptr[3]);
1948
1949 if (sync_needed)
1950 dma_sync_single_for_device(dev, edesc->dma_link_tbl,
1951 edesc->dma_len, DMA_BIDIRECTIONAL);
1952}
1953
1954static struct talitos_edesc *ahash_edesc_alloc(struct ahash_request *areq,
1955 struct scatterlist *src,
1956 unsigned int nbytes)
1957{
1958 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
1959 struct talitos_ctx *ctx = crypto_ahash_ctx(tfm);
1960
1961 return talitos_edesc_alloc(ctx->dev, src, NULL, NULL, 0,
1962 nbytes, 0, 0, 0, areq->base.flags, false);
1963}
1964
1965static struct talitos_edesc *
1966ahash_process_req_prepare(struct ahash_request *areq, unsigned int nbytes,
1967 unsigned int blocksize, bool is_sec1)
1968{
1969 struct talitos_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
1970 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1971 struct talitos_edesc *first = NULL, *prev_edesc = NULL, *edesc;
1972 size_t desc_max = is_sec1 ? TALITOS1_MAX_DATA_LEN :
1973 TALITOS2_MAX_DATA_LEN;
1974 struct scatterlist tmp[2];
1975 size_t to_hash_this_desc;
1976 struct scatterlist *src;
1977 size_t offset = 0;
1978
1979 do {
1980 src = scatterwalk_ffwd(tmp, req_ctx->psrc, offset);
1981
1982 to_hash_this_desc =
1983 min(nbytes, ALIGN_DOWN(desc_max, blocksize));
1984
1985 /* Allocate extended descriptor */
1986 edesc = ahash_edesc_alloc(areq, src, to_hash_this_desc);
1987 if (IS_ERR(edesc)) {
1988 if (first)
1989 free_edesc_list_from(areq, first);
1990 return edesc;
1991 }
1992
1993 edesc->src =
1994 scatterwalk_ffwd(edesc->bufsl, req_ctx->psrc, offset);
1995 edesc->desc.hdr = ctx->desc_hdr_template;
1996 edesc->first = offset == 0;
1997 edesc->last = nbytes - to_hash_this_desc == 0;
1998
1999 /* On last one, request SEC to pad; otherwise continue */
2000 if (req_ctx->last_request && edesc->last)
2001 edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_PAD;
2002 else
2003 edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_CONT;
2004
2005 /* request SEC to INIT hash. */
2006 if (req_ctx->first_request && edesc->first && !req_ctx->swinit)
2007 edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_INIT;
2008
2009 /*
2010 * When the tfm context has a keylen, it's an HMAC.
2011 * A first or last (ie. not middle) descriptor must request HMAC.
2012 */
2013 if (ctx->keylen && ((req_ctx->first_request && edesc->first) ||
2014 (req_ctx->last_request && edesc->last)))
2015 edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_HMAC;
2016
2017 /* clear the DN bit */
2018 if (is_sec1 && !edesc->last)
2019 edesc->desc.hdr &= ~DESC_HDR_DONE_NOTIFY;
2020
2021 common_nonsnoop_hash(edesc, areq, to_hash_this_desc);
2022
2023 offset += to_hash_this_desc;
2024 nbytes -= to_hash_this_desc;
2025
2026 if (!prev_edesc)
2027 first = edesc;
2028 else
2029 prev_edesc->next_desc = edesc;
2030 prev_edesc = edesc;
2031 } while (nbytes);
2032
2033 return first;
2034}
2035
2036static int ahash_process_req(struct ahash_request *areq, unsigned int nbytes)
2037{
2038 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
2039 struct talitos_ctx *ctx = crypto_ahash_ctx(tfm);
2040 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
2041 struct talitos_edesc *edesc;
2042 unsigned int blocksize =
2043 crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm));
2044 bool is_sec1 = has_ftr_sec1(dev_get_drvdata(ctx->dev));
2045 unsigned int nbytes_to_hash;
2046 unsigned int to_hash_later;
2047 unsigned int nsg;
2048 int nents;
2049 struct device *dev = ctx->dev;
2050 u8 *ctx_buf = req_ctx->buf[req_ctx->buf_idx];
2051 int ret;
2052
2053 if (!req_ctx->last_request && (nbytes + req_ctx->nbuf <= blocksize)) {
2054 /* Buffer up to one whole block */
2055 nents = sg_nents_for_len(areq->src, nbytes);
2056 if (nents < 0) {
2057 dev_err(dev, "Invalid number of src SG.\n");
2058 return nents;
2059 }
2060 sg_copy_to_buffer(areq->src, nents,
2061 ctx_buf + req_ctx->nbuf, nbytes);
2062 req_ctx->nbuf += nbytes;
2063 return 0;
2064 }
2065
2066 /* At least (blocksize + 1) bytes are available to hash */
2067 nbytes_to_hash = nbytes + req_ctx->nbuf;
2068 to_hash_later = nbytes_to_hash & (blocksize - 1);
2069
2070 if (req_ctx->last_request)
2071 to_hash_later = 0;
2072 else if (to_hash_later)
2073 /* There is a partial block. Hash the full block(s) now */
2074 nbytes_to_hash -= to_hash_later;
2075 else {
2076 /* Keep one block buffered */
2077 nbytes_to_hash -= blocksize;
2078 to_hash_later = blocksize;
2079 }
2080
2081 /* Chain in any previously buffered data */
2082 if (req_ctx->nbuf) {
2083 nsg = (req_ctx->nbuf < nbytes_to_hash) ? 2 : 1;
2084 sg_init_table(req_ctx->bufsl, nsg);
2085 sg_set_buf(req_ctx->bufsl, ctx_buf, req_ctx->nbuf);
2086 if (nsg > 1)
2087 sg_chain(req_ctx->bufsl, 2, areq->src);
2088 req_ctx->psrc = req_ctx->bufsl;
2089 } else
2090 req_ctx->psrc = areq->src;
2091
2092 if (to_hash_later) {
2093 nents = sg_nents_for_len(areq->src, nbytes);
2094 if (nents < 0) {
2095 dev_err(dev, "Invalid number of src SG.\n");
2096 return nents;
2097 }
2098 sg_pcopy_to_buffer(areq->src, nents,
2099 req_ctx->buf[(req_ctx->buf_idx + 1) & 1],
2100 to_hash_later,
2101 nbytes - to_hash_later);
2102 }
2103 req_ctx->to_hash_later = to_hash_later;
2104
2105 edesc = ahash_process_req_prepare(areq, nbytes_to_hash, blocksize,
2106 is_sec1);
2107 if (IS_ERR(edesc))
2108 return PTR_ERR(edesc);
2109
2110 ret = talitos_submit(dev, ctx->ch, &edesc->desc, ahash_done, areq);
2111 if (ret != -EINPROGRESS)
2112 free_edesc_list_from(areq, edesc);
2113
2114 return ret;
2115}
2116
2117static int ahash_init(struct ahash_request *areq)
2118{
2119 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
2120 struct talitos_ctx *ctx = crypto_ahash_ctx(tfm);
2121 struct device *dev = ctx->dev;
2122 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
2123 unsigned int size;
2124 dma_addr_t dma;
2125
2126 /* Initialize the context */
2127 req_ctx->buf_idx = 0;
2128 req_ctx->nbuf = 0;
2129 req_ctx->first_request = 1;
2130 req_ctx->swinit = 0; /* assume h/w init of context */
2131 size = (crypto_ahash_digestsize(tfm) <= SHA256_DIGEST_SIZE)
2132 ? TALITOS_MDEU_CONTEXT_SIZE_MD5_SHA1_SHA256
2133 : TALITOS_MDEU_CONTEXT_SIZE_SHA384_SHA512;
2134 req_ctx->hw_context_size = size;
2135 req_ctx->last_request = 0;
2136
2137 dma = dma_map_single(dev, req_ctx->hw_context, req_ctx->hw_context_size,
2138 DMA_TO_DEVICE);
2139 dma_unmap_single(dev, dma, req_ctx->hw_context_size, DMA_TO_DEVICE);
2140
2141 return 0;
2142}
2143
2144/*
2145 * on h/w without explicit sha224 support, we initialize h/w context
2146 * manually with sha224 constants, and tell it to run sha256.
2147 */
2148static int ahash_init_sha224_swinit(struct ahash_request *areq)
2149{
2150 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
2151
2152 req_ctx->hw_context[0] = SHA224_H0;
2153 req_ctx->hw_context[1] = SHA224_H1;
2154 req_ctx->hw_context[2] = SHA224_H2;
2155 req_ctx->hw_context[3] = SHA224_H3;
2156 req_ctx->hw_context[4] = SHA224_H4;
2157 req_ctx->hw_context[5] = SHA224_H5;
2158 req_ctx->hw_context[6] = SHA224_H6;
2159 req_ctx->hw_context[7] = SHA224_H7;
2160
2161 /* init 64-bit count */
2162 req_ctx->hw_context[8] = 0;
2163 req_ctx->hw_context[9] = 0;
2164
2165 ahash_init(areq);
2166 req_ctx->swinit = 1;/* prevent h/w initting context with sha256 values*/
2167
2168 return 0;
2169}
2170
2171static int ahash_update(struct ahash_request *areq)
2172{
2173 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
2174
2175 req_ctx->last_request = 0;
2176
2177 return ahash_process_req(areq, areq->nbytes);
2178}
2179
2180static int ahash_final(struct ahash_request *areq)
2181{
2182 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
2183
2184 req_ctx->last_request = 1;
2185
2186 return ahash_process_req(areq, 0);
2187}
2188
2189static int ahash_finup(struct ahash_request *areq)
2190{
2191 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
2192
2193 req_ctx->last_request = 1;
2194
2195 return ahash_process_req(areq, areq->nbytes);
2196}
2197
2198static int ahash_digest(struct ahash_request *areq)
2199{
2200 ahash_init(areq);
2201 return ahash_finup(areq);
2202}
2203
2204static int ahash_digest_sha224_swinit(struct ahash_request *areq)
2205{
2206 ahash_init_sha224_swinit(areq);
2207 return ahash_finup(areq);
2208}
2209
2210static int ahash_export(struct ahash_request *areq, void *out)
2211{
2212 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
2213 struct talitos_export_state *export = out;
2214 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
2215 struct talitos_ctx *ctx = crypto_ahash_ctx(tfm);
2216 struct device *dev = ctx->dev;
2217 dma_addr_t dma;
2218
2219 dma = dma_map_single(dev, req_ctx->hw_context, req_ctx->hw_context_size,
2220 DMA_FROM_DEVICE);
2221 dma_unmap_single(dev, dma, req_ctx->hw_context_size, DMA_FROM_DEVICE);
2222
2223 memcpy(export->hw_context, req_ctx->hw_context,
2224 req_ctx->hw_context_size);
2225 memcpy(export->buf, req_ctx->buf[req_ctx->buf_idx], req_ctx->nbuf);
2226 export->swinit = req_ctx->swinit;
2227 export->first_request = req_ctx->first_request;
2228 export->last_request = req_ctx->last_request;
2229 export->to_hash_later = req_ctx->to_hash_later;
2230 export->nbuf = req_ctx->nbuf;
2231
2232 return 0;
2233}
2234
2235static int ahash_import(struct ahash_request *areq, const void *in)
2236{
2237 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
2238 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
2239 struct talitos_ctx *ctx = crypto_ahash_ctx(tfm);
2240 struct device *dev = ctx->dev;
2241 const struct talitos_export_state *export = in;
2242 unsigned int size;
2243 dma_addr_t dma;
2244
2245 memset(req_ctx, 0, sizeof(*req_ctx));
2246 size = (crypto_ahash_digestsize(tfm) <= SHA256_DIGEST_SIZE)
2247 ? TALITOS_MDEU_CONTEXT_SIZE_MD5_SHA1_SHA256
2248 : TALITOS_MDEU_CONTEXT_SIZE_SHA384_SHA512;
2249 req_ctx->hw_context_size = size;
2250 memcpy(req_ctx->hw_context, export->hw_context, size);
2251 memcpy(req_ctx->buf[0], export->buf, export->nbuf);
2252 req_ctx->swinit = export->swinit;
2253 req_ctx->first_request = export->first_request;
2254 req_ctx->last_request = export->last_request;
2255 req_ctx->to_hash_later = export->to_hash_later;
2256 req_ctx->nbuf = export->nbuf;
2257
2258 dma = dma_map_single(dev, req_ctx->hw_context, req_ctx->hw_context_size,
2259 DMA_TO_DEVICE);
2260 dma_unmap_single(dev, dma, req_ctx->hw_context_size, DMA_TO_DEVICE);
2261
2262 return 0;
2263}
2264
2265static int keyhash(struct crypto_ahash *tfm, const u8 *key, unsigned int keylen,
2266 u8 *hash)
2267{
2268 struct talitos_ctx *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm));
2269
2270 struct scatterlist sg[1];
2271 struct ahash_request *req;
2272 struct crypto_wait wait;
2273 int ret;
2274
2275 crypto_init_wait(&wait);
2276
2277 req = ahash_request_alloc(tfm, GFP_KERNEL);
2278 if (!req)
2279 return -ENOMEM;
2280
2281 /* Keep tfm keylen == 0 during hash of the long key */
2282 ctx->keylen = 0;
2283 ahash_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
2284 crypto_req_done, &wait);
2285
2286 sg_init_one(&sg[0], key, keylen);
2287
2288 ahash_request_set_crypt(req, sg, hash, keylen);
2289 ret = crypto_wait_req(crypto_ahash_digest(req), &wait);
2290
2291 ahash_request_free(req);
2292
2293 return ret;
2294}
2295
2296static int ahash_setkey(struct crypto_ahash *tfm, const u8 *key,
2297 unsigned int keylen)
2298{
2299 struct talitos_ctx *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm));
2300 struct device *dev = ctx->dev;
2301 unsigned int blocksize =
2302 crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm));
2303 unsigned int digestsize = crypto_ahash_digestsize(tfm);
2304 unsigned int keysize = keylen;
2305 u8 hash[SHA512_DIGEST_SIZE];
2306 int ret;
2307
2308 if (keylen <= blocksize)
2309 memcpy(ctx->key, key, keysize);
2310 else {
2311 /* Must get the hash of the long key */
2312 ret = keyhash(tfm, key, keylen, hash);
2313
2314 if (ret)
2315 return -EINVAL;
2316
2317 keysize = digestsize;
2318 memcpy(ctx->key, hash, digestsize);
2319 }
2320
2321 if (ctx->keylen)
2322 dma_unmap_single(dev, ctx->dma_key, ctx->keylen, DMA_TO_DEVICE);
2323
2324 ctx->keylen = keysize;
2325 ctx->dma_key = dma_map_single(dev, ctx->key, keysize, DMA_TO_DEVICE);
2326
2327 return 0;
2328}
2329
2330
2331struct talitos_alg_template {
2332 u32 type;
2333 u32 priority;
2334 union {
2335 struct skcipher_alg skcipher;
2336 struct ahash_alg hash;
2337 struct aead_alg aead;
2338 } alg;
2339 __be32 desc_hdr_template;
2340};
2341
2342static struct talitos_alg_template driver_algs[] = {
2343 /* AEAD algorithms. These use a single-pass ipsec_esp descriptor */
2344 { .type = CRYPTO_ALG_TYPE_AEAD,
2345 .alg.aead = {
2346 .base = {
2347 .cra_name = "authenc(hmac(sha1),cbc(aes))",
2348 .cra_driver_name = "authenc-hmac-sha1-"
2349 "cbc-aes-talitos",
2350 .cra_blocksize = AES_BLOCK_SIZE,
2351 .cra_flags = CRYPTO_ALG_ASYNC |
2352 CRYPTO_ALG_ALLOCATES_MEMORY,
2353 },
2354 .ivsize = AES_BLOCK_SIZE,
2355 .maxauthsize = SHA1_DIGEST_SIZE,
2356 },
2357 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2358 DESC_HDR_SEL0_AESU |
2359 DESC_HDR_MODE0_AESU_CBC |
2360 DESC_HDR_SEL1_MDEUA |
2361 DESC_HDR_MODE1_MDEU_INIT |
2362 DESC_HDR_MODE1_MDEU_PAD |
2363 DESC_HDR_MODE1_MDEU_SHA1_HMAC,
2364 },
2365 { .type = CRYPTO_ALG_TYPE_AEAD,
2366 .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA,
2367 .alg.aead = {
2368 .base = {
2369 .cra_name = "authenc(hmac(sha1),cbc(aes))",
2370 .cra_driver_name = "authenc-hmac-sha1-"
2371 "cbc-aes-talitos-hsna",
2372 .cra_blocksize = AES_BLOCK_SIZE,
2373 .cra_flags = CRYPTO_ALG_ASYNC |
2374 CRYPTO_ALG_ALLOCATES_MEMORY,
2375 },
2376 .ivsize = AES_BLOCK_SIZE,
2377 .maxauthsize = SHA1_DIGEST_SIZE,
2378 },
2379 .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU |
2380 DESC_HDR_SEL0_AESU |
2381 DESC_HDR_MODE0_AESU_CBC |
2382 DESC_HDR_SEL1_MDEUA |
2383 DESC_HDR_MODE1_MDEU_INIT |
2384 DESC_HDR_MODE1_MDEU_PAD |
2385 DESC_HDR_MODE1_MDEU_SHA1_HMAC,
2386 },
2387 { .type = CRYPTO_ALG_TYPE_AEAD,
2388 .alg.aead = {
2389 .base = {
2390 .cra_name = "authenc(hmac(sha1),"
2391 "cbc(des3_ede))",
2392 .cra_driver_name = "authenc-hmac-sha1-"
2393 "cbc-3des-talitos",
2394 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2395 .cra_flags = CRYPTO_ALG_ASYNC |
2396 CRYPTO_ALG_ALLOCATES_MEMORY,
2397 },
2398 .ivsize = DES3_EDE_BLOCK_SIZE,
2399 .maxauthsize = SHA1_DIGEST_SIZE,
2400 .setkey = aead_des3_setkey,
2401 },
2402 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2403 DESC_HDR_SEL0_DEU |
2404 DESC_HDR_MODE0_DEU_CBC |
2405 DESC_HDR_MODE0_DEU_3DES |
2406 DESC_HDR_SEL1_MDEUA |
2407 DESC_HDR_MODE1_MDEU_INIT |
2408 DESC_HDR_MODE1_MDEU_PAD |
2409 DESC_HDR_MODE1_MDEU_SHA1_HMAC,
2410 },
2411 { .type = CRYPTO_ALG_TYPE_AEAD,
2412 .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA,
2413 .alg.aead = {
2414 .base = {
2415 .cra_name = "authenc(hmac(sha1),"
2416 "cbc(des3_ede))",
2417 .cra_driver_name = "authenc-hmac-sha1-"
2418 "cbc-3des-talitos-hsna",
2419 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2420 .cra_flags = CRYPTO_ALG_ASYNC |
2421 CRYPTO_ALG_ALLOCATES_MEMORY,
2422 },
2423 .ivsize = DES3_EDE_BLOCK_SIZE,
2424 .maxauthsize = SHA1_DIGEST_SIZE,
2425 .setkey = aead_des3_setkey,
2426 },
2427 .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU |
2428 DESC_HDR_SEL0_DEU |
2429 DESC_HDR_MODE0_DEU_CBC |
2430 DESC_HDR_MODE0_DEU_3DES |
2431 DESC_HDR_SEL1_MDEUA |
2432 DESC_HDR_MODE1_MDEU_INIT |
2433 DESC_HDR_MODE1_MDEU_PAD |
2434 DESC_HDR_MODE1_MDEU_SHA1_HMAC,
2435 },
2436 { .type = CRYPTO_ALG_TYPE_AEAD,
2437 .alg.aead = {
2438 .base = {
2439 .cra_name = "authenc(hmac(sha224),cbc(aes))",
2440 .cra_driver_name = "authenc-hmac-sha224-"
2441 "cbc-aes-talitos",
2442 .cra_blocksize = AES_BLOCK_SIZE,
2443 .cra_flags = CRYPTO_ALG_ASYNC |
2444 CRYPTO_ALG_ALLOCATES_MEMORY,
2445 },
2446 .ivsize = AES_BLOCK_SIZE,
2447 .maxauthsize = SHA224_DIGEST_SIZE,
2448 },
2449 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2450 DESC_HDR_SEL0_AESU |
2451 DESC_HDR_MODE0_AESU_CBC |
2452 DESC_HDR_SEL1_MDEUA |
2453 DESC_HDR_MODE1_MDEU_INIT |
2454 DESC_HDR_MODE1_MDEU_PAD |
2455 DESC_HDR_MODE1_MDEU_SHA224_HMAC,
2456 },
2457 { .type = CRYPTO_ALG_TYPE_AEAD,
2458 .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA,
2459 .alg.aead = {
2460 .base = {
2461 .cra_name = "authenc(hmac(sha224),cbc(aes))",
2462 .cra_driver_name = "authenc-hmac-sha224-"
2463 "cbc-aes-talitos-hsna",
2464 .cra_blocksize = AES_BLOCK_SIZE,
2465 .cra_flags = CRYPTO_ALG_ASYNC |
2466 CRYPTO_ALG_ALLOCATES_MEMORY,
2467 },
2468 .ivsize = AES_BLOCK_SIZE,
2469 .maxauthsize = SHA224_DIGEST_SIZE,
2470 },
2471 .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU |
2472 DESC_HDR_SEL0_AESU |
2473 DESC_HDR_MODE0_AESU_CBC |
2474 DESC_HDR_SEL1_MDEUA |
2475 DESC_HDR_MODE1_MDEU_INIT |
2476 DESC_HDR_MODE1_MDEU_PAD |
2477 DESC_HDR_MODE1_MDEU_SHA224_HMAC,
2478 },
2479 { .type = CRYPTO_ALG_TYPE_AEAD,
2480 .alg.aead = {
2481 .base = {
2482 .cra_name = "authenc(hmac(sha224),"
2483 "cbc(des3_ede))",
2484 .cra_driver_name = "authenc-hmac-sha224-"
2485 "cbc-3des-talitos",
2486 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2487 .cra_flags = CRYPTO_ALG_ASYNC |
2488 CRYPTO_ALG_ALLOCATES_MEMORY,
2489 },
2490 .ivsize = DES3_EDE_BLOCK_SIZE,
2491 .maxauthsize = SHA224_DIGEST_SIZE,
2492 .setkey = aead_des3_setkey,
2493 },
2494 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2495 DESC_HDR_SEL0_DEU |
2496 DESC_HDR_MODE0_DEU_CBC |
2497 DESC_HDR_MODE0_DEU_3DES |
2498 DESC_HDR_SEL1_MDEUA |
2499 DESC_HDR_MODE1_MDEU_INIT |
2500 DESC_HDR_MODE1_MDEU_PAD |
2501 DESC_HDR_MODE1_MDEU_SHA224_HMAC,
2502 },
2503 { .type = CRYPTO_ALG_TYPE_AEAD,
2504 .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA,
2505 .alg.aead = {
2506 .base = {
2507 .cra_name = "authenc(hmac(sha224),"
2508 "cbc(des3_ede))",
2509 .cra_driver_name = "authenc-hmac-sha224-"
2510 "cbc-3des-talitos-hsna",
2511 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2512 .cra_flags = CRYPTO_ALG_ASYNC |
2513 CRYPTO_ALG_ALLOCATES_MEMORY,
2514 },
2515 .ivsize = DES3_EDE_BLOCK_SIZE,
2516 .maxauthsize = SHA224_DIGEST_SIZE,
2517 .setkey = aead_des3_setkey,
2518 },
2519 .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU |
2520 DESC_HDR_SEL0_DEU |
2521 DESC_HDR_MODE0_DEU_CBC |
2522 DESC_HDR_MODE0_DEU_3DES |
2523 DESC_HDR_SEL1_MDEUA |
2524 DESC_HDR_MODE1_MDEU_INIT |
2525 DESC_HDR_MODE1_MDEU_PAD |
2526 DESC_HDR_MODE1_MDEU_SHA224_HMAC,
2527 },
2528 { .type = CRYPTO_ALG_TYPE_AEAD,
2529 .alg.aead = {
2530 .base = {
2531 .cra_name = "authenc(hmac(sha256),cbc(aes))",
2532 .cra_driver_name = "authenc-hmac-sha256-"
2533 "cbc-aes-talitos",
2534 .cra_blocksize = AES_BLOCK_SIZE,
2535 .cra_flags = CRYPTO_ALG_ASYNC |
2536 CRYPTO_ALG_ALLOCATES_MEMORY,
2537 },
2538 .ivsize = AES_BLOCK_SIZE,
2539 .maxauthsize = SHA256_DIGEST_SIZE,
2540 },
2541 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2542 DESC_HDR_SEL0_AESU |
2543 DESC_HDR_MODE0_AESU_CBC |
2544 DESC_HDR_SEL1_MDEUA |
2545 DESC_HDR_MODE1_MDEU_INIT |
2546 DESC_HDR_MODE1_MDEU_PAD |
2547 DESC_HDR_MODE1_MDEU_SHA256_HMAC,
2548 },
2549 { .type = CRYPTO_ALG_TYPE_AEAD,
2550 .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA,
2551 .alg.aead = {
2552 .base = {
2553 .cra_name = "authenc(hmac(sha256),cbc(aes))",
2554 .cra_driver_name = "authenc-hmac-sha256-"
2555 "cbc-aes-talitos-hsna",
2556 .cra_blocksize = AES_BLOCK_SIZE,
2557 .cra_flags = CRYPTO_ALG_ASYNC |
2558 CRYPTO_ALG_ALLOCATES_MEMORY,
2559 },
2560 .ivsize = AES_BLOCK_SIZE,
2561 .maxauthsize = SHA256_DIGEST_SIZE,
2562 },
2563 .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU |
2564 DESC_HDR_SEL0_AESU |
2565 DESC_HDR_MODE0_AESU_CBC |
2566 DESC_HDR_SEL1_MDEUA |
2567 DESC_HDR_MODE1_MDEU_INIT |
2568 DESC_HDR_MODE1_MDEU_PAD |
2569 DESC_HDR_MODE1_MDEU_SHA256_HMAC,
2570 },
2571 { .type = CRYPTO_ALG_TYPE_AEAD,
2572 .alg.aead = {
2573 .base = {
2574 .cra_name = "authenc(hmac(sha256),"
2575 "cbc(des3_ede))",
2576 .cra_driver_name = "authenc-hmac-sha256-"
2577 "cbc-3des-talitos",
2578 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2579 .cra_flags = CRYPTO_ALG_ASYNC |
2580 CRYPTO_ALG_ALLOCATES_MEMORY,
2581 },
2582 .ivsize = DES3_EDE_BLOCK_SIZE,
2583 .maxauthsize = SHA256_DIGEST_SIZE,
2584 .setkey = aead_des3_setkey,
2585 },
2586 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2587 DESC_HDR_SEL0_DEU |
2588 DESC_HDR_MODE0_DEU_CBC |
2589 DESC_HDR_MODE0_DEU_3DES |
2590 DESC_HDR_SEL1_MDEUA |
2591 DESC_HDR_MODE1_MDEU_INIT |
2592 DESC_HDR_MODE1_MDEU_PAD |
2593 DESC_HDR_MODE1_MDEU_SHA256_HMAC,
2594 },
2595 { .type = CRYPTO_ALG_TYPE_AEAD,
2596 .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA,
2597 .alg.aead = {
2598 .base = {
2599 .cra_name = "authenc(hmac(sha256),"
2600 "cbc(des3_ede))",
2601 .cra_driver_name = "authenc-hmac-sha256-"
2602 "cbc-3des-talitos-hsna",
2603 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2604 .cra_flags = CRYPTO_ALG_ASYNC |
2605 CRYPTO_ALG_ALLOCATES_MEMORY,
2606 },
2607 .ivsize = DES3_EDE_BLOCK_SIZE,
2608 .maxauthsize = SHA256_DIGEST_SIZE,
2609 .setkey = aead_des3_setkey,
2610 },
2611 .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU |
2612 DESC_HDR_SEL0_DEU |
2613 DESC_HDR_MODE0_DEU_CBC |
2614 DESC_HDR_MODE0_DEU_3DES |
2615 DESC_HDR_SEL1_MDEUA |
2616 DESC_HDR_MODE1_MDEU_INIT |
2617 DESC_HDR_MODE1_MDEU_PAD |
2618 DESC_HDR_MODE1_MDEU_SHA256_HMAC,
2619 },
2620 { .type = CRYPTO_ALG_TYPE_AEAD,
2621 .alg.aead = {
2622 .base = {
2623 .cra_name = "authenc(hmac(sha384),cbc(aes))",
2624 .cra_driver_name = "authenc-hmac-sha384-"
2625 "cbc-aes-talitos",
2626 .cra_blocksize = AES_BLOCK_SIZE,
2627 .cra_flags = CRYPTO_ALG_ASYNC |
2628 CRYPTO_ALG_ALLOCATES_MEMORY,
2629 },
2630 .ivsize = AES_BLOCK_SIZE,
2631 .maxauthsize = SHA384_DIGEST_SIZE,
2632 },
2633 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2634 DESC_HDR_SEL0_AESU |
2635 DESC_HDR_MODE0_AESU_CBC |
2636 DESC_HDR_SEL1_MDEUB |
2637 DESC_HDR_MODE1_MDEU_INIT |
2638 DESC_HDR_MODE1_MDEU_PAD |
2639 DESC_HDR_MODE1_MDEUB_SHA384_HMAC,
2640 },
2641 { .type = CRYPTO_ALG_TYPE_AEAD,
2642 .alg.aead = {
2643 .base = {
2644 .cra_name = "authenc(hmac(sha384),"
2645 "cbc(des3_ede))",
2646 .cra_driver_name = "authenc-hmac-sha384-"
2647 "cbc-3des-talitos",
2648 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2649 .cra_flags = CRYPTO_ALG_ASYNC |
2650 CRYPTO_ALG_ALLOCATES_MEMORY,
2651 },
2652 .ivsize = DES3_EDE_BLOCK_SIZE,
2653 .maxauthsize = SHA384_DIGEST_SIZE,
2654 .setkey = aead_des3_setkey,
2655 },
2656 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2657 DESC_HDR_SEL0_DEU |
2658 DESC_HDR_MODE0_DEU_CBC |
2659 DESC_HDR_MODE0_DEU_3DES |
2660 DESC_HDR_SEL1_MDEUB |
2661 DESC_HDR_MODE1_MDEU_INIT |
2662 DESC_HDR_MODE1_MDEU_PAD |
2663 DESC_HDR_MODE1_MDEUB_SHA384_HMAC,
2664 },
2665 { .type = CRYPTO_ALG_TYPE_AEAD,
2666 .alg.aead = {
2667 .base = {
2668 .cra_name = "authenc(hmac(sha512),cbc(aes))",
2669 .cra_driver_name = "authenc-hmac-sha512-"
2670 "cbc-aes-talitos",
2671 .cra_blocksize = AES_BLOCK_SIZE,
2672 .cra_flags = CRYPTO_ALG_ASYNC |
2673 CRYPTO_ALG_ALLOCATES_MEMORY,
2674 },
2675 .ivsize = AES_BLOCK_SIZE,
2676 .maxauthsize = SHA512_DIGEST_SIZE,
2677 },
2678 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2679 DESC_HDR_SEL0_AESU |
2680 DESC_HDR_MODE0_AESU_CBC |
2681 DESC_HDR_SEL1_MDEUB |
2682 DESC_HDR_MODE1_MDEU_INIT |
2683 DESC_HDR_MODE1_MDEU_PAD |
2684 DESC_HDR_MODE1_MDEUB_SHA512_HMAC,
2685 },
2686 { .type = CRYPTO_ALG_TYPE_AEAD,
2687 .alg.aead = {
2688 .base = {
2689 .cra_name = "authenc(hmac(sha512),"
2690 "cbc(des3_ede))",
2691 .cra_driver_name = "authenc-hmac-sha512-"
2692 "cbc-3des-talitos",
2693 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2694 .cra_flags = CRYPTO_ALG_ASYNC |
2695 CRYPTO_ALG_ALLOCATES_MEMORY,
2696 },
2697 .ivsize = DES3_EDE_BLOCK_SIZE,
2698 .maxauthsize = SHA512_DIGEST_SIZE,
2699 .setkey = aead_des3_setkey,
2700 },
2701 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2702 DESC_HDR_SEL0_DEU |
2703 DESC_HDR_MODE0_DEU_CBC |
2704 DESC_HDR_MODE0_DEU_3DES |
2705 DESC_HDR_SEL1_MDEUB |
2706 DESC_HDR_MODE1_MDEU_INIT |
2707 DESC_HDR_MODE1_MDEU_PAD |
2708 DESC_HDR_MODE1_MDEUB_SHA512_HMAC,
2709 },
2710 { .type = CRYPTO_ALG_TYPE_AEAD,
2711 .alg.aead = {
2712 .base = {
2713 .cra_name = "authenc(hmac(md5),cbc(aes))",
2714 .cra_driver_name = "authenc-hmac-md5-"
2715 "cbc-aes-talitos",
2716 .cra_blocksize = AES_BLOCK_SIZE,
2717 .cra_flags = CRYPTO_ALG_ASYNC |
2718 CRYPTO_ALG_ALLOCATES_MEMORY,
2719 },
2720 .ivsize = AES_BLOCK_SIZE,
2721 .maxauthsize = MD5_DIGEST_SIZE,
2722 },
2723 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2724 DESC_HDR_SEL0_AESU |
2725 DESC_HDR_MODE0_AESU_CBC |
2726 DESC_HDR_SEL1_MDEUA |
2727 DESC_HDR_MODE1_MDEU_INIT |
2728 DESC_HDR_MODE1_MDEU_PAD |
2729 DESC_HDR_MODE1_MDEU_MD5_HMAC,
2730 },
2731 { .type = CRYPTO_ALG_TYPE_AEAD,
2732 .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA,
2733 .alg.aead = {
2734 .base = {
2735 .cra_name = "authenc(hmac(md5),cbc(aes))",
2736 .cra_driver_name = "authenc-hmac-md5-"
2737 "cbc-aes-talitos-hsna",
2738 .cra_blocksize = AES_BLOCK_SIZE,
2739 .cra_flags = CRYPTO_ALG_ASYNC |
2740 CRYPTO_ALG_ALLOCATES_MEMORY,
2741 },
2742 .ivsize = AES_BLOCK_SIZE,
2743 .maxauthsize = MD5_DIGEST_SIZE,
2744 },
2745 .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU |
2746 DESC_HDR_SEL0_AESU |
2747 DESC_HDR_MODE0_AESU_CBC |
2748 DESC_HDR_SEL1_MDEUA |
2749 DESC_HDR_MODE1_MDEU_INIT |
2750 DESC_HDR_MODE1_MDEU_PAD |
2751 DESC_HDR_MODE1_MDEU_MD5_HMAC,
2752 },
2753 { .type = CRYPTO_ALG_TYPE_AEAD,
2754 .alg.aead = {
2755 .base = {
2756 .cra_name = "authenc(hmac(md5),cbc(des3_ede))",
2757 .cra_driver_name = "authenc-hmac-md5-"
2758 "cbc-3des-talitos",
2759 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2760 .cra_flags = CRYPTO_ALG_ASYNC |
2761 CRYPTO_ALG_ALLOCATES_MEMORY,
2762 },
2763 .ivsize = DES3_EDE_BLOCK_SIZE,
2764 .maxauthsize = MD5_DIGEST_SIZE,
2765 .setkey = aead_des3_setkey,
2766 },
2767 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2768 DESC_HDR_SEL0_DEU |
2769 DESC_HDR_MODE0_DEU_CBC |
2770 DESC_HDR_MODE0_DEU_3DES |
2771 DESC_HDR_SEL1_MDEUA |
2772 DESC_HDR_MODE1_MDEU_INIT |
2773 DESC_HDR_MODE1_MDEU_PAD |
2774 DESC_HDR_MODE1_MDEU_MD5_HMAC,
2775 },
2776 { .type = CRYPTO_ALG_TYPE_AEAD,
2777 .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA,
2778 .alg.aead = {
2779 .base = {
2780 .cra_name = "authenc(hmac(md5),cbc(des3_ede))",
2781 .cra_driver_name = "authenc-hmac-md5-"
2782 "cbc-3des-talitos-hsna",
2783 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2784 .cra_flags = CRYPTO_ALG_ASYNC |
2785 CRYPTO_ALG_ALLOCATES_MEMORY,
2786 },
2787 .ivsize = DES3_EDE_BLOCK_SIZE,
2788 .maxauthsize = MD5_DIGEST_SIZE,
2789 .setkey = aead_des3_setkey,
2790 },
2791 .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU |
2792 DESC_HDR_SEL0_DEU |
2793 DESC_HDR_MODE0_DEU_CBC |
2794 DESC_HDR_MODE0_DEU_3DES |
2795 DESC_HDR_SEL1_MDEUA |
2796 DESC_HDR_MODE1_MDEU_INIT |
2797 DESC_HDR_MODE1_MDEU_PAD |
2798 DESC_HDR_MODE1_MDEU_MD5_HMAC,
2799 },
2800 /* SKCIPHER algorithms. */
2801 { .type = CRYPTO_ALG_TYPE_SKCIPHER,
2802 .alg.skcipher = {
2803 .base.cra_name = "ecb(aes)",
2804 .base.cra_driver_name = "ecb-aes-talitos",
2805 .base.cra_blocksize = AES_BLOCK_SIZE,
2806 .base.cra_flags = CRYPTO_ALG_ASYNC |
2807 CRYPTO_ALG_ALLOCATES_MEMORY,
2808 .min_keysize = AES_MIN_KEY_SIZE,
2809 .max_keysize = AES_MAX_KEY_SIZE,
2810 .setkey = skcipher_aes_setkey,
2811 },
2812 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2813 DESC_HDR_SEL0_AESU,
2814 },
2815 { .type = CRYPTO_ALG_TYPE_SKCIPHER,
2816 .alg.skcipher = {
2817 .base.cra_name = "cbc(aes)",
2818 .base.cra_driver_name = "cbc-aes-talitos",
2819 .base.cra_blocksize = AES_BLOCK_SIZE,
2820 .base.cra_flags = CRYPTO_ALG_ASYNC |
2821 CRYPTO_ALG_ALLOCATES_MEMORY,
2822 .min_keysize = AES_MIN_KEY_SIZE,
2823 .max_keysize = AES_MAX_KEY_SIZE,
2824 .ivsize = AES_BLOCK_SIZE,
2825 .setkey = skcipher_aes_setkey,
2826 },
2827 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2828 DESC_HDR_SEL0_AESU |
2829 DESC_HDR_MODE0_AESU_CBC,
2830 },
2831 { .type = CRYPTO_ALG_TYPE_SKCIPHER,
2832 .alg.skcipher = {
2833 .base.cra_name = "ctr(aes)",
2834 .base.cra_driver_name = "ctr-aes-talitos",
2835 .base.cra_blocksize = 1,
2836 .base.cra_flags = CRYPTO_ALG_ASYNC |
2837 CRYPTO_ALG_ALLOCATES_MEMORY,
2838 .min_keysize = AES_MIN_KEY_SIZE,
2839 .max_keysize = AES_MAX_KEY_SIZE,
2840 .ivsize = AES_BLOCK_SIZE,
2841 .setkey = skcipher_aes_setkey,
2842 },
2843 .desc_hdr_template = DESC_HDR_TYPE_AESU_CTR_NONSNOOP |
2844 DESC_HDR_SEL0_AESU |
2845 DESC_HDR_MODE0_AESU_CTR,
2846 },
2847 { .type = CRYPTO_ALG_TYPE_SKCIPHER,
2848 .alg.skcipher = {
2849 .base.cra_name = "ctr(aes)",
2850 .base.cra_driver_name = "ctr-aes-talitos",
2851 .base.cra_blocksize = 1,
2852 .base.cra_flags = CRYPTO_ALG_ASYNC |
2853 CRYPTO_ALG_ALLOCATES_MEMORY,
2854 .min_keysize = AES_MIN_KEY_SIZE,
2855 .max_keysize = AES_MAX_KEY_SIZE,
2856 .ivsize = AES_BLOCK_SIZE,
2857 .setkey = skcipher_aes_setkey,
2858 },
2859 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2860 DESC_HDR_SEL0_AESU |
2861 DESC_HDR_MODE0_AESU_CTR,
2862 },
2863 { .type = CRYPTO_ALG_TYPE_SKCIPHER,
2864 .alg.skcipher = {
2865 .base.cra_name = "ecb(des)",
2866 .base.cra_driver_name = "ecb-des-talitos",
2867 .base.cra_blocksize = DES_BLOCK_SIZE,
2868 .base.cra_flags = CRYPTO_ALG_ASYNC |
2869 CRYPTO_ALG_ALLOCATES_MEMORY,
2870 .min_keysize = DES_KEY_SIZE,
2871 .max_keysize = DES_KEY_SIZE,
2872 .setkey = skcipher_des_setkey,
2873 },
2874 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2875 DESC_HDR_SEL0_DEU,
2876 },
2877 { .type = CRYPTO_ALG_TYPE_SKCIPHER,
2878 .alg.skcipher = {
2879 .base.cra_name = "cbc(des)",
2880 .base.cra_driver_name = "cbc-des-talitos",
2881 .base.cra_blocksize = DES_BLOCK_SIZE,
2882 .base.cra_flags = CRYPTO_ALG_ASYNC |
2883 CRYPTO_ALG_ALLOCATES_MEMORY,
2884 .min_keysize = DES_KEY_SIZE,
2885 .max_keysize = DES_KEY_SIZE,
2886 .ivsize = DES_BLOCK_SIZE,
2887 .setkey = skcipher_des_setkey,
2888 },
2889 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2890 DESC_HDR_SEL0_DEU |
2891 DESC_HDR_MODE0_DEU_CBC,
2892 },
2893 { .type = CRYPTO_ALG_TYPE_SKCIPHER,
2894 .alg.skcipher = {
2895 .base.cra_name = "ecb(des3_ede)",
2896 .base.cra_driver_name = "ecb-3des-talitos",
2897 .base.cra_blocksize = DES3_EDE_BLOCK_SIZE,
2898 .base.cra_flags = CRYPTO_ALG_ASYNC |
2899 CRYPTO_ALG_ALLOCATES_MEMORY,
2900 .min_keysize = DES3_EDE_KEY_SIZE,
2901 .max_keysize = DES3_EDE_KEY_SIZE,
2902 .setkey = skcipher_des3_setkey,
2903 },
2904 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2905 DESC_HDR_SEL0_DEU |
2906 DESC_HDR_MODE0_DEU_3DES,
2907 },
2908 { .type = CRYPTO_ALG_TYPE_SKCIPHER,
2909 .alg.skcipher = {
2910 .base.cra_name = "cbc(des3_ede)",
2911 .base.cra_driver_name = "cbc-3des-talitos",
2912 .base.cra_blocksize = DES3_EDE_BLOCK_SIZE,
2913 .base.cra_flags = CRYPTO_ALG_ASYNC |
2914 CRYPTO_ALG_ALLOCATES_MEMORY,
2915 .min_keysize = DES3_EDE_KEY_SIZE,
2916 .max_keysize = DES3_EDE_KEY_SIZE,
2917 .ivsize = DES3_EDE_BLOCK_SIZE,
2918 .setkey = skcipher_des3_setkey,
2919 },
2920 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2921 DESC_HDR_SEL0_DEU |
2922 DESC_HDR_MODE0_DEU_CBC |
2923 DESC_HDR_MODE0_DEU_3DES,
2924 },
2925 /* AHASH algorithms. */
2926 { .type = CRYPTO_ALG_TYPE_AHASH,
2927 .alg.hash = {
2928 .halg.digestsize = MD5_DIGEST_SIZE,
2929 .halg.statesize = sizeof(struct talitos_export_state),
2930 .halg.base = {
2931 .cra_name = "md5",
2932 .cra_driver_name = "md5-talitos",
2933 .cra_blocksize = MD5_HMAC_BLOCK_SIZE,
2934 .cra_flags = CRYPTO_ALG_ASYNC |
2935 CRYPTO_ALG_ALLOCATES_MEMORY,
2936 }
2937 },
2938 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2939 DESC_HDR_SEL0_MDEUA |
2940 DESC_HDR_MODE0_MDEU_MD5,
2941 },
2942 { .type = CRYPTO_ALG_TYPE_AHASH,
2943 .alg.hash = {
2944 .halg.digestsize = SHA1_DIGEST_SIZE,
2945 .halg.statesize = sizeof(struct talitos_export_state),
2946 .halg.base = {
2947 .cra_name = "sha1",
2948 .cra_driver_name = "sha1-talitos",
2949 .cra_blocksize = SHA1_BLOCK_SIZE,
2950 .cra_flags = CRYPTO_ALG_ASYNC |
2951 CRYPTO_ALG_ALLOCATES_MEMORY,
2952 }
2953 },
2954 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2955 DESC_HDR_SEL0_MDEUA |
2956 DESC_HDR_MODE0_MDEU_SHA1,
2957 },
2958 { .type = CRYPTO_ALG_TYPE_AHASH,
2959 .alg.hash = {
2960 .halg.digestsize = SHA224_DIGEST_SIZE,
2961 .halg.statesize = sizeof(struct talitos_export_state),
2962 .halg.base = {
2963 .cra_name = "sha224",
2964 .cra_driver_name = "sha224-talitos",
2965 .cra_blocksize = SHA224_BLOCK_SIZE,
2966 .cra_flags = CRYPTO_ALG_ASYNC |
2967 CRYPTO_ALG_ALLOCATES_MEMORY,
2968 }
2969 },
2970 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2971 DESC_HDR_SEL0_MDEUA |
2972 DESC_HDR_MODE0_MDEU_SHA224,
2973 },
2974 { .type = CRYPTO_ALG_TYPE_AHASH,
2975 .alg.hash = {
2976 .halg.digestsize = SHA256_DIGEST_SIZE,
2977 .halg.statesize = sizeof(struct talitos_export_state),
2978 .halg.base = {
2979 .cra_name = "sha256",
2980 .cra_driver_name = "sha256-talitos",
2981 .cra_blocksize = SHA256_BLOCK_SIZE,
2982 .cra_flags = CRYPTO_ALG_ASYNC |
2983 CRYPTO_ALG_ALLOCATES_MEMORY,
2984 }
2985 },
2986 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2987 DESC_HDR_SEL0_MDEUA |
2988 DESC_HDR_MODE0_MDEU_SHA256,
2989 },
2990 { .type = CRYPTO_ALG_TYPE_AHASH,
2991 .alg.hash = {
2992 .halg.digestsize = SHA384_DIGEST_SIZE,
2993 .halg.statesize = sizeof(struct talitos_export_state),
2994 .halg.base = {
2995 .cra_name = "sha384",
2996 .cra_driver_name = "sha384-talitos",
2997 .cra_blocksize = SHA384_BLOCK_SIZE,
2998 .cra_flags = CRYPTO_ALG_ASYNC |
2999 CRYPTO_ALG_ALLOCATES_MEMORY,
3000 }
3001 },
3002 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
3003 DESC_HDR_SEL0_MDEUB |
3004 DESC_HDR_MODE0_MDEUB_SHA384,
3005 },
3006 { .type = CRYPTO_ALG_TYPE_AHASH,
3007 .alg.hash = {
3008 .halg.digestsize = SHA512_DIGEST_SIZE,
3009 .halg.statesize = sizeof(struct talitos_export_state),
3010 .halg.base = {
3011 .cra_name = "sha512",
3012 .cra_driver_name = "sha512-talitos",
3013 .cra_blocksize = SHA512_BLOCK_SIZE,
3014 .cra_flags = CRYPTO_ALG_ASYNC |
3015 CRYPTO_ALG_ALLOCATES_MEMORY,
3016 }
3017 },
3018 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
3019 DESC_HDR_SEL0_MDEUB |
3020 DESC_HDR_MODE0_MDEUB_SHA512,
3021 },
3022 { .type = CRYPTO_ALG_TYPE_AHASH,
3023 .alg.hash = {
3024 .halg.digestsize = MD5_DIGEST_SIZE,
3025 .halg.statesize = sizeof(struct talitos_export_state),
3026 .halg.base = {
3027 .cra_name = "hmac(md5)",
3028 .cra_driver_name = "hmac-md5-talitos",
3029 .cra_blocksize = MD5_HMAC_BLOCK_SIZE,
3030 .cra_flags = CRYPTO_ALG_ASYNC |
3031 CRYPTO_ALG_ALLOCATES_MEMORY,
3032 }
3033 },
3034 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
3035 DESC_HDR_SEL0_MDEUA |
3036 DESC_HDR_MODE0_MDEU_MD5,
3037 },
3038 { .type = CRYPTO_ALG_TYPE_AHASH,
3039 .alg.hash = {
3040 .halg.digestsize = SHA1_DIGEST_SIZE,
3041 .halg.statesize = sizeof(struct talitos_export_state),
3042 .halg.base = {
3043 .cra_name = "hmac(sha1)",
3044 .cra_driver_name = "hmac-sha1-talitos",
3045 .cra_blocksize = SHA1_BLOCK_SIZE,
3046 .cra_flags = CRYPTO_ALG_ASYNC |
3047 CRYPTO_ALG_ALLOCATES_MEMORY,
3048 }
3049 },
3050 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
3051 DESC_HDR_SEL0_MDEUA |
3052 DESC_HDR_MODE0_MDEU_SHA1,
3053 },
3054 { .type = CRYPTO_ALG_TYPE_AHASH,
3055 .alg.hash = {
3056 .halg.digestsize = SHA224_DIGEST_SIZE,
3057 .halg.statesize = sizeof(struct talitos_export_state),
3058 .halg.base = {
3059 .cra_name = "hmac(sha224)",
3060 .cra_driver_name = "hmac-sha224-talitos",
3061 .cra_blocksize = SHA224_BLOCK_SIZE,
3062 .cra_flags = CRYPTO_ALG_ASYNC |
3063 CRYPTO_ALG_ALLOCATES_MEMORY,
3064 }
3065 },
3066 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
3067 DESC_HDR_SEL0_MDEUA |
3068 DESC_HDR_MODE0_MDEU_SHA224,
3069 },
3070 { .type = CRYPTO_ALG_TYPE_AHASH,
3071 .alg.hash = {
3072 .halg.digestsize = SHA256_DIGEST_SIZE,
3073 .halg.statesize = sizeof(struct talitos_export_state),
3074 .halg.base = {
3075 .cra_name = "hmac(sha256)",
3076 .cra_driver_name = "hmac-sha256-talitos",
3077 .cra_blocksize = SHA256_BLOCK_SIZE,
3078 .cra_flags = CRYPTO_ALG_ASYNC |
3079 CRYPTO_ALG_ALLOCATES_MEMORY,
3080 }
3081 },
3082 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
3083 DESC_HDR_SEL0_MDEUA |
3084 DESC_HDR_MODE0_MDEU_SHA256,
3085 },
3086 { .type = CRYPTO_ALG_TYPE_AHASH,
3087 .alg.hash = {
3088 .halg.digestsize = SHA384_DIGEST_SIZE,
3089 .halg.statesize = sizeof(struct talitos_export_state),
3090 .halg.base = {
3091 .cra_name = "hmac(sha384)",
3092 .cra_driver_name = "hmac-sha384-talitos",
3093 .cra_blocksize = SHA384_BLOCK_SIZE,
3094 .cra_flags = CRYPTO_ALG_ASYNC |
3095 CRYPTO_ALG_ALLOCATES_MEMORY,
3096 }
3097 },
3098 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
3099 DESC_HDR_SEL0_MDEUB |
3100 DESC_HDR_MODE0_MDEUB_SHA384,
3101 },
3102 { .type = CRYPTO_ALG_TYPE_AHASH,
3103 .alg.hash = {
3104 .halg.digestsize = SHA512_DIGEST_SIZE,
3105 .halg.statesize = sizeof(struct talitos_export_state),
3106 .halg.base = {
3107 .cra_name = "hmac(sha512)",
3108 .cra_driver_name = "hmac-sha512-talitos",
3109 .cra_blocksize = SHA512_BLOCK_SIZE,
3110 .cra_flags = CRYPTO_ALG_ASYNC |
3111 CRYPTO_ALG_ALLOCATES_MEMORY,
3112 }
3113 },
3114 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
3115 DESC_HDR_SEL0_MDEUB |
3116 DESC_HDR_MODE0_MDEUB_SHA512,
3117 }
3118};
3119
3120struct talitos_crypto_alg {
3121 struct list_head entry;
3122 struct device *dev;
3123 struct talitos_alg_template algt;
3124};
3125
3126static int talitos_init_common(struct talitos_ctx *ctx,
3127 struct talitos_crypto_alg *talitos_alg)
3128{
3129 struct talitos_private *priv;
3130
3131 /* update context with ptr to dev */
3132 ctx->dev = talitos_alg->dev;
3133
3134 /* assign SEC channel to tfm in round-robin fashion */
3135 priv = dev_get_drvdata(ctx->dev);
3136 ctx->ch = atomic_inc_return(&priv->last_chan) &
3137 (priv->num_channels - 1);
3138
3139 /* copy descriptor header template value */
3140 ctx->desc_hdr_template = talitos_alg->algt.desc_hdr_template;
3141
3142 /* select done notification */
3143 ctx->desc_hdr_template |= DESC_HDR_DONE_NOTIFY;
3144
3145 return 0;
3146}
3147
3148static int talitos_cra_init_aead(struct crypto_aead *tfm)
3149{
3150 struct aead_alg *alg = crypto_aead_alg(tfm);
3151 struct talitos_crypto_alg *talitos_alg;
3152 struct talitos_ctx *ctx = crypto_aead_ctx(tfm);
3153
3154 talitos_alg = container_of(alg, struct talitos_crypto_alg,
3155 algt.alg.aead);
3156
3157 return talitos_init_common(ctx, talitos_alg);
3158}
3159
3160static int talitos_cra_init_skcipher(struct crypto_skcipher *tfm)
3161{
3162 struct skcipher_alg *alg = crypto_skcipher_alg(tfm);
3163 struct talitos_crypto_alg *talitos_alg;
3164 struct talitos_ctx *ctx = crypto_skcipher_ctx(tfm);
3165
3166 talitos_alg = container_of(alg, struct talitos_crypto_alg,
3167 algt.alg.skcipher);
3168
3169 return talitos_init_common(ctx, talitos_alg);
3170}
3171
3172static int talitos_cra_init_ahash(struct crypto_tfm *tfm)
3173{
3174 struct crypto_alg *alg = tfm->__crt_alg;
3175 struct talitos_crypto_alg *talitos_alg;
3176 struct talitos_ctx *ctx = crypto_tfm_ctx(tfm);
3177
3178 talitos_alg = container_of(__crypto_ahash_alg(alg),
3179 struct talitos_crypto_alg,
3180 algt.alg.hash);
3181
3182 ctx->keylen = 0;
3183 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
3184 sizeof(struct talitos_ahash_req_ctx));
3185
3186 return talitos_init_common(ctx, talitos_alg);
3187}
3188
3189static void talitos_cra_exit(struct crypto_tfm *tfm)
3190{
3191 struct talitos_ctx *ctx = crypto_tfm_ctx(tfm);
3192 struct device *dev = ctx->dev;
3193
3194 if (ctx->keylen)
3195 dma_unmap_single(dev, ctx->dma_key, ctx->keylen, DMA_TO_DEVICE);
3196}
3197
3198/*
3199 * given the alg's descriptor header template, determine whether descriptor
3200 * type and primary/secondary execution units required match the hw
3201 * capabilities description provided in the device tree node.
3202 */
3203static int hw_supports(struct device *dev, __be32 desc_hdr_template)
3204{
3205 struct talitos_private *priv = dev_get_drvdata(dev);
3206 int ret;
3207
3208 ret = (1 << DESC_TYPE(desc_hdr_template) & priv->desc_types) &&
3209 (1 << PRIMARY_EU(desc_hdr_template) & priv->exec_units);
3210
3211 if (SECONDARY_EU(desc_hdr_template))
3212 ret = ret && (1 << SECONDARY_EU(desc_hdr_template)
3213 & priv->exec_units);
3214
3215 return ret;
3216}
3217
3218static void talitos_remove(struct platform_device *ofdev)
3219{
3220 struct device *dev = &ofdev->dev;
3221 struct talitos_private *priv = dev_get_drvdata(dev);
3222 struct talitos_crypto_alg *t_alg, *n;
3223 int i;
3224
3225 list_for_each_entry_safe(t_alg, n, &priv->alg_list, entry) {
3226 switch (t_alg->algt.type) {
3227 case CRYPTO_ALG_TYPE_SKCIPHER:
3228 crypto_unregister_skcipher(&t_alg->algt.alg.skcipher);
3229 break;
3230 case CRYPTO_ALG_TYPE_AEAD:
3231 crypto_unregister_aead(&t_alg->algt.alg.aead);
3232 break;
3233 case CRYPTO_ALG_TYPE_AHASH:
3234 crypto_unregister_ahash(&t_alg->algt.alg.hash);
3235 break;
3236 }
3237 list_del(&t_alg->entry);
3238 }
3239
3240 if (hw_supports(dev, DESC_HDR_SEL0_RNG))
3241 talitos_unregister_rng(dev);
3242
3243 for (i = 0; i < 2; i++)
3244 if (priv->irq[i] > 0)
3245 free_irq(priv->irq[i], dev);
3246
3247 tasklet_kill(&priv->done_task[0]);
3248 if (priv->irq[1] > 0)
3249 tasklet_kill(&priv->done_task[1]);
3250}
3251
3252static struct talitos_crypto_alg *talitos_alg_alloc(struct device *dev,
3253 struct talitos_alg_template
3254 *template)
3255{
3256 struct talitos_private *priv = dev_get_drvdata(dev);
3257 struct talitos_crypto_alg *t_alg;
3258 struct crypto_alg *alg;
3259
3260 t_alg = devm_kzalloc(dev, sizeof(struct talitos_crypto_alg),
3261 GFP_KERNEL);
3262 if (!t_alg)
3263 return ERR_PTR(-ENOMEM);
3264
3265 t_alg->algt = *template;
3266
3267 switch (t_alg->algt.type) {
3268 case CRYPTO_ALG_TYPE_SKCIPHER:
3269 alg = &t_alg->algt.alg.skcipher.base;
3270 alg->cra_exit = talitos_cra_exit;
3271 t_alg->algt.alg.skcipher.init = talitos_cra_init_skcipher;
3272 t_alg->algt.alg.skcipher.setkey =
3273 t_alg->algt.alg.skcipher.setkey ?: skcipher_setkey;
3274 t_alg->algt.alg.skcipher.encrypt = skcipher_encrypt;
3275 t_alg->algt.alg.skcipher.decrypt = skcipher_decrypt;
3276 if (!strcmp(alg->cra_name, "ctr(aes)") && !has_ftr_sec1(priv) &&
3277 DESC_TYPE(t_alg->algt.desc_hdr_template) !=
3278 DESC_TYPE(DESC_HDR_TYPE_AESU_CTR_NONSNOOP)) {
3279 devm_kfree(dev, t_alg);
3280 return ERR_PTR(-ENOTSUPP);
3281 }
3282 break;
3283 case CRYPTO_ALG_TYPE_AEAD:
3284 alg = &t_alg->algt.alg.aead.base;
3285 alg->cra_exit = talitos_cra_exit;
3286 t_alg->algt.alg.aead.init = talitos_cra_init_aead;
3287 t_alg->algt.alg.aead.setkey = t_alg->algt.alg.aead.setkey ?:
3288 aead_setkey;
3289 t_alg->algt.alg.aead.encrypt = aead_encrypt;
3290 t_alg->algt.alg.aead.decrypt = aead_decrypt;
3291 if (!(priv->features & TALITOS_FTR_SHA224_HWINIT) &&
3292 !strncmp(alg->cra_name, "authenc(hmac(sha224)", 20)) {
3293 devm_kfree(dev, t_alg);
3294 return ERR_PTR(-ENOTSUPP);
3295 }
3296 break;
3297 case CRYPTO_ALG_TYPE_AHASH:
3298 alg = &t_alg->algt.alg.hash.halg.base;
3299 alg->cra_init = talitos_cra_init_ahash;
3300 alg->cra_exit = talitos_cra_exit;
3301 t_alg->algt.alg.hash.init = ahash_init;
3302 t_alg->algt.alg.hash.update = ahash_update;
3303 t_alg->algt.alg.hash.final = ahash_final;
3304 t_alg->algt.alg.hash.finup = ahash_finup;
3305 t_alg->algt.alg.hash.digest = ahash_digest;
3306 if (!strncmp(alg->cra_name, "hmac", 4))
3307 t_alg->algt.alg.hash.setkey = ahash_setkey;
3308 t_alg->algt.alg.hash.import = ahash_import;
3309 t_alg->algt.alg.hash.export = ahash_export;
3310
3311 if (!(priv->features & TALITOS_FTR_HMAC_OK) &&
3312 !strncmp(alg->cra_name, "hmac", 4)) {
3313 devm_kfree(dev, t_alg);
3314 return ERR_PTR(-ENOTSUPP);
3315 }
3316 if (!(priv->features & TALITOS_FTR_SHA224_HWINIT) &&
3317 (!strcmp(alg->cra_name, "sha224") ||
3318 !strcmp(alg->cra_name, "hmac(sha224)"))) {
3319 t_alg->algt.alg.hash.init = ahash_init_sha224_swinit;
3320 t_alg->algt.alg.hash.digest =
3321 ahash_digest_sha224_swinit;
3322 t_alg->algt.desc_hdr_template =
3323 DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
3324 DESC_HDR_SEL0_MDEUA |
3325 DESC_HDR_MODE0_MDEU_SHA256;
3326 }
3327 break;
3328 default:
3329 dev_err(dev, "unknown algorithm type %d\n", t_alg->algt.type);
3330 devm_kfree(dev, t_alg);
3331 return ERR_PTR(-EINVAL);
3332 }
3333
3334 alg->cra_module = THIS_MODULE;
3335 if (t_alg->algt.priority)
3336 alg->cra_priority = t_alg->algt.priority;
3337 else
3338 alg->cra_priority = TALITOS_CRA_PRIORITY;
3339 if (has_ftr_sec1(priv) && t_alg->algt.type != CRYPTO_ALG_TYPE_AHASH)
3340 alg->cra_alignmask = 3;
3341 else
3342 alg->cra_alignmask = 0;
3343 alg->cra_ctxsize = sizeof(struct talitos_ctx);
3344 alg->cra_flags |= CRYPTO_ALG_KERN_DRIVER_ONLY;
3345
3346 t_alg->dev = dev;
3347
3348 return t_alg;
3349}
3350
3351static int talitos_probe_irq(struct platform_device *ofdev)
3352{
3353 struct device *dev = &ofdev->dev;
3354 struct talitos_private *priv = dev_get_drvdata(dev);
3355 int err;
3356 bool is_sec1 = has_ftr_sec1(priv);
3357
3358 priv->irq[0] = platform_get_irq(ofdev, 0);
3359 if (priv->irq[0] < 0)
3360 return priv->irq[0];
3361
3362 if (is_sec1) {
3363 err = request_irq(priv->irq[0], talitos1_interrupt_4ch, 0,
3364 dev_driver_string(dev), dev);
3365 goto primary_out;
3366 }
3367
3368 priv->irq[1] = platform_get_irq_optional(ofdev, 1);
3369 if (priv->irq[1] == -EPROBE_DEFER)
3370 return priv->irq[1];
3371
3372 /* get the primary irq line */
3373 if (priv->irq[1] < 0) {
3374 err = request_irq(priv->irq[0], talitos2_interrupt_4ch, 0,
3375 dev_driver_string(dev), dev);
3376 goto primary_out;
3377 }
3378
3379 err = request_irq(priv->irq[0], talitos2_interrupt_ch0_2, 0,
3380 dev_driver_string(dev), dev);
3381 if (err)
3382 goto primary_out;
3383
3384 /* get the secondary irq line */
3385 err = request_irq(priv->irq[1], talitos2_interrupt_ch1_3, 0,
3386 dev_driver_string(dev), dev);
3387 if (err) {
3388 dev_err(dev, "failed to request secondary irq\n");
3389 priv->irq[1] = 0;
3390 }
3391
3392 return err;
3393
3394primary_out:
3395 if (err) {
3396 dev_err(dev, "failed to request primary irq\n");
3397 priv->irq[0] = 0;
3398 }
3399
3400 return err;
3401}
3402
3403static int talitos_probe(struct platform_device *ofdev)
3404{
3405 struct device *dev = &ofdev->dev;
3406 struct device_node *np = ofdev->dev.of_node;
3407 struct talitos_private *priv;
3408 unsigned int num_channels;
3409 int i, err;
3410 int stride;
3411
3412 if (of_property_read_u32(np, "fsl,num-channels", &num_channels))
3413 return -EINVAL;
3414
3415 priv = devm_kzalloc(dev, struct_size(priv, chan, num_channels), GFP_KERNEL);
3416 if (!priv)
3417 return -ENOMEM;
3418
3419 priv->num_channels = num_channels;
3420
3421 INIT_LIST_HEAD(&priv->alg_list);
3422
3423 dev_set_drvdata(dev, priv);
3424
3425 priv->ofdev = ofdev;
3426
3427 spin_lock_init(&priv->reg_lock);
3428
3429 priv->reg = devm_platform_ioremap_resource(ofdev, 0);
3430 if (IS_ERR(priv->reg)) {
3431 dev_err(dev, "failed to of_iomap\n");
3432 err = PTR_ERR(priv->reg);
3433 goto err_out;
3434 }
3435
3436 /* get SEC version capabilities from device tree */
3437 of_property_read_u32(np, "fsl,channel-fifo-len", &priv->chfifo_len);
3438 of_property_read_u32(np, "fsl,exec-units-mask", &priv->exec_units);
3439 of_property_read_u32(np, "fsl,descriptor-types-mask",
3440 &priv->desc_types);
3441
3442 if (!is_power_of_2(priv->num_channels) || !priv->chfifo_len ||
3443 !priv->exec_units || !priv->desc_types) {
3444 dev_err(dev, "invalid property data in device tree node\n");
3445 err = -EINVAL;
3446 goto err_out;
3447 }
3448
3449 if (of_device_is_compatible(np, "fsl,sec3.0"))
3450 priv->features |= TALITOS_FTR_SRC_LINK_TBL_LEN_INCLUDES_EXTENT;
3451
3452 if (of_device_is_compatible(np, "fsl,sec2.1"))
3453 priv->features |= TALITOS_FTR_HW_AUTH_CHECK |
3454 TALITOS_FTR_SHA224_HWINIT |
3455 TALITOS_FTR_HMAC_OK;
3456
3457 if (of_device_is_compatible(np, "fsl,sec1.0"))
3458 priv->features |= TALITOS_FTR_SEC1;
3459
3460 if (of_device_is_compatible(np, "fsl,sec1.2")) {
3461 priv->reg_deu = priv->reg + TALITOS12_DEU;
3462 priv->reg_aesu = priv->reg + TALITOS12_AESU;
3463 priv->reg_mdeu = priv->reg + TALITOS12_MDEU;
3464 stride = TALITOS1_CH_STRIDE;
3465 } else if (of_device_is_compatible(np, "fsl,sec1.0")) {
3466 priv->reg_deu = priv->reg + TALITOS10_DEU;
3467 priv->reg_aesu = priv->reg + TALITOS10_AESU;
3468 priv->reg_mdeu = priv->reg + TALITOS10_MDEU;
3469 priv->reg_afeu = priv->reg + TALITOS10_AFEU;
3470 priv->reg_rngu = priv->reg + TALITOS10_RNGU;
3471 priv->reg_pkeu = priv->reg + TALITOS10_PKEU;
3472 stride = TALITOS1_CH_STRIDE;
3473 } else {
3474 priv->reg_deu = priv->reg + TALITOS2_DEU;
3475 priv->reg_aesu = priv->reg + TALITOS2_AESU;
3476 priv->reg_mdeu = priv->reg + TALITOS2_MDEU;
3477 priv->reg_afeu = priv->reg + TALITOS2_AFEU;
3478 priv->reg_rngu = priv->reg + TALITOS2_RNGU;
3479 priv->reg_pkeu = priv->reg + TALITOS2_PKEU;
3480 priv->reg_keu = priv->reg + TALITOS2_KEU;
3481 priv->reg_crcu = priv->reg + TALITOS2_CRCU;
3482 stride = TALITOS2_CH_STRIDE;
3483 }
3484
3485 err = talitos_probe_irq(ofdev);
3486 if (err)
3487 goto err_out;
3488
3489 if (has_ftr_sec1(priv)) {
3490 if (priv->num_channels == 1)
3491 tasklet_init(&priv->done_task[0], talitos1_done_ch0,
3492 (unsigned long)dev);
3493 else
3494 tasklet_init(&priv->done_task[0], talitos1_done_4ch,
3495 (unsigned long)dev);
3496 } else {
3497 if (priv->irq[1] > 0) {
3498 tasklet_init(&priv->done_task[0], talitos2_done_ch0_2,
3499 (unsigned long)dev);
3500 tasklet_init(&priv->done_task[1], talitos2_done_ch1_3,
3501 (unsigned long)dev);
3502 } else if (priv->num_channels == 1) {
3503 tasklet_init(&priv->done_task[0], talitos2_done_ch0,
3504 (unsigned long)dev);
3505 } else {
3506 tasklet_init(&priv->done_task[0], talitos2_done_4ch,
3507 (unsigned long)dev);
3508 }
3509 }
3510
3511 priv->fifo_len = roundup_pow_of_two(priv->chfifo_len);
3512
3513 for (i = 0; i < priv->num_channels; i++) {
3514 priv->chan[i].reg = priv->reg + stride * (i + 1);
3515 if (priv->irq[1] < 0 || !(i & 1))
3516 priv->chan[i].reg += TALITOS_CH_BASE_OFFSET;
3517
3518 spin_lock_init(&priv->chan[i].head_lock);
3519 spin_lock_init(&priv->chan[i].tail_lock);
3520
3521 priv->chan[i].fifo = devm_kcalloc(dev,
3522 priv->fifo_len,
3523 sizeof(struct talitos_request),
3524 GFP_KERNEL);
3525 if (!priv->chan[i].fifo) {
3526 dev_err(dev, "failed to allocate request fifo %d\n", i);
3527 err = -ENOMEM;
3528 goto err_out;
3529 }
3530
3531 atomic_set(&priv->chan[i].submit_count,
3532 -(priv->chfifo_len - 1));
3533 }
3534
3535 dma_set_mask(dev, DMA_BIT_MASK(36));
3536
3537 /* reset and initialize the h/w */
3538 err = init_device(dev);
3539 if (err) {
3540 dev_err(dev, "failed to initialize device\n");
3541 goto err_out;
3542 }
3543
3544 /* register the RNG, if available */
3545 if (hw_supports(dev, DESC_HDR_SEL0_RNG)) {
3546 err = talitos_register_rng(dev);
3547 if (err) {
3548 dev_err(dev, "failed to register hwrng: %d\n", err);
3549 goto err_out;
3550 } else
3551 dev_info(dev, "hwrng\n");
3552 }
3553
3554 /* register crypto algorithms the device supports */
3555 for (i = 0; i < ARRAY_SIZE(driver_algs); i++) {
3556 if (hw_supports(dev, driver_algs[i].desc_hdr_template)) {
3557 struct talitos_crypto_alg *t_alg;
3558 struct crypto_alg *alg = NULL;
3559
3560 t_alg = talitos_alg_alloc(dev, &driver_algs[i]);
3561 if (IS_ERR(t_alg)) {
3562 err = PTR_ERR(t_alg);
3563 if (err == -ENOTSUPP)
3564 continue;
3565 goto err_out;
3566 }
3567
3568 switch (t_alg->algt.type) {
3569 case CRYPTO_ALG_TYPE_SKCIPHER:
3570 err = crypto_register_skcipher(
3571 &t_alg->algt.alg.skcipher);
3572 alg = &t_alg->algt.alg.skcipher.base;
3573 break;
3574
3575 case CRYPTO_ALG_TYPE_AEAD:
3576 err = crypto_register_aead(
3577 &t_alg->algt.alg.aead);
3578 alg = &t_alg->algt.alg.aead.base;
3579 break;
3580
3581 case CRYPTO_ALG_TYPE_AHASH:
3582 err = crypto_register_ahash(
3583 &t_alg->algt.alg.hash);
3584 alg = &t_alg->algt.alg.hash.halg.base;
3585 break;
3586 }
3587 if (err) {
3588 dev_err(dev, "%s alg registration failed\n",
3589 alg->cra_driver_name);
3590 devm_kfree(dev, t_alg);
3591 } else
3592 list_add_tail(&t_alg->entry, &priv->alg_list);
3593 }
3594 }
3595 if (!list_empty(&priv->alg_list))
3596 dev_info(dev, "%s algorithms registered in /proc/crypto\n",
3597 (char *)of_get_property(np, "compatible", NULL));
3598
3599 return 0;
3600
3601err_out:
3602 talitos_remove(ofdev);
3603
3604 return err;
3605}
3606
3607static const struct of_device_id talitos_match[] = {
3608#ifdef CONFIG_CRYPTO_DEV_TALITOS1
3609 {
3610 .compatible = "fsl,sec1.0",
3611 },
3612#endif
3613#ifdef CONFIG_CRYPTO_DEV_TALITOS2
3614 {
3615 .compatible = "fsl,sec2.0",
3616 },
3617#endif
3618 {},
3619};
3620MODULE_DEVICE_TABLE(of, talitos_match);
3621
3622static struct platform_driver talitos_driver = {
3623 .driver = {
3624 .name = "talitos",
3625 .of_match_table = talitos_match,
3626 },
3627 .probe = talitos_probe,
3628 .remove = talitos_remove,
3629};
3630
3631module_platform_driver(talitos_driver);
3632
3633MODULE_LICENSE("GPL");
3634MODULE_AUTHOR("Kim Phillips <[email protected]>");
3635MODULE_DESCRIPTION("Freescale integrated security engine (SEC) driver");