projects/linux·blob master · refs

153 lines · 3.8 KiB

1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * AES CTR routines supporting VMX instructions on the Power 8
4 *
5 * Copyright (C) 2015 International Business Machines Inc.
6 *
7 * Author: Marcelo Henrique Cerri <[email protected]>
8 */
9
10#include <asm/simd.h>
11#include <asm/switch_to.h>
12#include <crypto/aes.h>
13#include <crypto/internal/simd.h>
14#include <crypto/internal/skcipher.h>
15#include <linux/err.h>
16#include <linux/kernel.h>
17#include <linux/module.h>
18#include <linux/uaccess.h>
19
20#include "aesp8-ppc.h"
21
22struct p8_aes_ctr_ctx {
23 struct crypto_skcipher *fallback;
24 struct p8_aes_key enc_key;
25};
26
27static int p8_aes_ctr_init(struct crypto_skcipher *tfm)
28{
29 struct p8_aes_ctr_ctx *ctx = crypto_skcipher_ctx(tfm);
30 struct crypto_skcipher *fallback;
31
32 fallback = crypto_alloc_skcipher("ctr(aes)", 0,
33 CRYPTO_ALG_NEED_FALLBACK |
34 CRYPTO_ALG_ASYNC);
35 if (IS_ERR(fallback)) {
36 pr_err("Failed to allocate ctr(aes) fallback: %ld\n",
37 PTR_ERR(fallback));
38 return PTR_ERR(fallback);
39 }
40
41 crypto_skcipher_set_reqsize(tfm, sizeof(struct skcipher_request) +
42 crypto_skcipher_reqsize(fallback));
43 ctx->fallback = fallback;
44 return 0;
45}
46
47static void p8_aes_ctr_exit(struct crypto_skcipher *tfm)
48{
49 struct p8_aes_ctr_ctx *ctx = crypto_skcipher_ctx(tfm);
50
51 crypto_free_skcipher(ctx->fallback);
52}
53
54static int p8_aes_ctr_setkey(struct crypto_skcipher *tfm, const u8 *key,
55 unsigned int keylen)
56{
57 struct p8_aes_ctr_ctx *ctx = crypto_skcipher_ctx(tfm);
58 int ret;
59
60 preempt_disable();
61 pagefault_disable();
62 enable_kernel_vsx();
63 ret = aes_p8_set_encrypt_key(key, keylen * 8, &ctx->enc_key);
64 disable_kernel_vsx();
65 pagefault_enable();
66 preempt_enable();
67
68 ret |= crypto_skcipher_setkey(ctx->fallback, key, keylen);
69
70 return ret ? -EINVAL : 0;
71}
72
73static void p8_aes_ctr_final(const struct p8_aes_ctr_ctx *ctx,
74 struct skcipher_walk *walk)
75{
76 const u8 *src = walk->src.virt.addr;
77 u8 *ctrblk = walk->iv;
78 u8 keystream[AES_BLOCK_SIZE];
79 u8 *dst = walk->dst.virt.addr;
80 unsigned int nbytes = walk->nbytes;
81
82 preempt_disable();
83 pagefault_disable();
84 enable_kernel_vsx();
85 aes_p8_encrypt(ctrblk, keystream, &ctx->enc_key);
86 disable_kernel_vsx();
87 pagefault_enable();
88 preempt_enable();
89
90 crypto_xor_cpy(dst, keystream, src, nbytes);
91 crypto_inc(ctrblk, AES_BLOCK_SIZE);
92}
93
94static int p8_aes_ctr_crypt(struct skcipher_request *req)
95{
96 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
97 const struct p8_aes_ctr_ctx *ctx = crypto_skcipher_ctx(tfm);
98 struct skcipher_walk walk;
99 unsigned int nbytes;
100 int ret;
101
102 if (!crypto_simd_usable()) {
103 struct skcipher_request *subreq = skcipher_request_ctx(req);
104
105 *subreq = *req;
106 skcipher_request_set_tfm(subreq, ctx->fallback);
107 return crypto_skcipher_encrypt(subreq);
108 }
109
110 ret = skcipher_walk_virt(&walk, req, false);
111 while ((nbytes = walk.nbytes) >= AES_BLOCK_SIZE) {
112 preempt_disable();
113 pagefault_disable();
114 enable_kernel_vsx();
115 aes_p8_ctr32_encrypt_blocks(walk.src.virt.addr,
116 walk.dst.virt.addr,
117 nbytes / AES_BLOCK_SIZE,
118 &ctx->enc_key, walk.iv);
119 disable_kernel_vsx();
120 pagefault_enable();
121 preempt_enable();
122
123 do {
124 crypto_inc(walk.iv, AES_BLOCK_SIZE);
125 } while ((nbytes -= AES_BLOCK_SIZE) >= AES_BLOCK_SIZE);
126
127 ret = skcipher_walk_done(&walk, nbytes);
128 }
129 if (nbytes) {
130 p8_aes_ctr_final(ctx, &walk);
131 ret = skcipher_walk_done(&walk, 0);
132 }
133 return ret;
134}
135
136struct skcipher_alg p8_aes_ctr_alg = {
137 .base.cra_name = "ctr(aes)",
138 .base.cra_driver_name = "p8_aes_ctr",
139 .base.cra_module = THIS_MODULE,
140 .base.cra_priority = 2000,
141 .base.cra_flags = CRYPTO_ALG_NEED_FALLBACK,
142 .base.cra_blocksize = 1,
143 .base.cra_ctxsize = sizeof(struct p8_aes_ctr_ctx),
144 .setkey = p8_aes_ctr_setkey,
145 .encrypt = p8_aes_ctr_crypt,
146 .decrypt = p8_aes_ctr_crypt,
147 .init = p8_aes_ctr_init,
148 .exit = p8_aes_ctr_exit,
149 .min_keysize = AES_MIN_KEY_SIZE,
150 .max_keysize = AES_MAX_KEY_SIZE,
151 .ivsize = AES_BLOCK_SIZE,
152 .chunksize = AES_BLOCK_SIZE,
153};