projects/linux·blob master · refs

477 lines · 11.9 KiB

1// SPDX-License-Identifier: GPL-2.0-or-later
2/* XTS: as defined in IEEE1619/D16
3 * http://grouper.ieee.org/groups/1619/email/pdf00086.pdf
4 *
5 * Copyright (c) 2007 Rik Snel <[email protected]>
6 *
7 * Based on ecb.c
8 * Copyright (c) 2006 Herbert Xu <[email protected]>
9 */
10#include <crypto/internal/cipher.h>
11#include <crypto/internal/skcipher.h>
12#include <crypto/scatterwalk.h>
13#include <linux/err.h>
14#include <linux/init.h>
15#include <linux/kernel.h>
16#include <linux/module.h>
17#include <linux/scatterlist.h>
18#include <linux/slab.h>
19#include <linux/string.h>
20
21#include <crypto/xts.h>
22#include <crypto/b128ops.h>
23#include <crypto/gf128mul.h>
24
25struct xts_tfm_ctx {
26 struct crypto_skcipher *child;
27 struct crypto_cipher *tweak;
28};
29
30struct xts_instance_ctx {
31 struct crypto_skcipher_spawn spawn;
32 struct crypto_cipher_spawn tweak_spawn;
33};
34
35struct xts_request_ctx {
36 le128 t;
37 struct scatterlist *tail;
38 struct scatterlist sg[2];
39 struct skcipher_request subreq;
40};
41
42static int xts_setkey(struct crypto_skcipher *parent, const u8 *key,
43 unsigned int keylen)
44{
45 struct xts_tfm_ctx *ctx = crypto_skcipher_ctx(parent);
46 struct crypto_skcipher *child;
47 struct crypto_cipher *tweak;
48 int err;
49
50 err = xts_verify_key(parent, key, keylen);
51 if (err)
52 return err;
53
54 keylen /= 2;
55
56 /* we need two cipher instances: one to compute the initial 'tweak'
57 * by encrypting the IV (usually the 'plain' iv) and the other
58 * one to encrypt and decrypt the data */
59
60 /* tweak cipher, uses Key2 i.e. the second half of *key */
61 tweak = ctx->tweak;
62 crypto_cipher_clear_flags(tweak, CRYPTO_TFM_REQ_MASK);
63 crypto_cipher_set_flags(tweak, crypto_skcipher_get_flags(parent) &
64 CRYPTO_TFM_REQ_MASK);
65 err = crypto_cipher_setkey(tweak, key + keylen, keylen);
66 if (err)
67 return err;
68
69 /* data cipher, uses Key1 i.e. the first half of *key */
70 child = ctx->child;
71 crypto_skcipher_clear_flags(child, CRYPTO_TFM_REQ_MASK);
72 crypto_skcipher_set_flags(child, crypto_skcipher_get_flags(parent) &
73 CRYPTO_TFM_REQ_MASK);
74 return crypto_skcipher_setkey(child, key, keylen);
75}
76
77/*
78 * We compute the tweak masks twice (both before and after the ECB encryption or
79 * decryption) to avoid having to allocate a temporary buffer and/or make
80 * multiple calls to the 'ecb(..)' instance, which usually would be slower than
81 * just doing the gf128mul_x_ble() calls again.
82 */
83static int xts_xor_tweak(struct skcipher_request *req, bool second_pass,
84 bool enc)
85{
86 struct xts_request_ctx *rctx = skcipher_request_ctx(req);
87 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
88 const bool cts = (req->cryptlen % XTS_BLOCK_SIZE);
89 const int bs = XTS_BLOCK_SIZE;
90 struct skcipher_walk w;
91 le128 t = rctx->t;
92 int err;
93
94 if (second_pass) {
95 req = &rctx->subreq;
96 /* set to our TFM to enforce correct alignment: */
97 skcipher_request_set_tfm(req, tfm);
98 }
99 err = skcipher_walk_virt(&w, req, false);
100
101 while (w.nbytes) {
102 unsigned int avail = w.nbytes;
103 const le128 *wsrc;
104 le128 *wdst;
105
106 wsrc = w.src.virt.addr;
107 wdst = w.dst.virt.addr;
108
109 do {
110 if (unlikely(cts) &&
111 w.total - w.nbytes + avail < 2 * XTS_BLOCK_SIZE) {
112 if (!enc) {
113 if (second_pass)
114 rctx->t = t;
115 gf128mul_x_ble(&t, &t);
116 }
117 le128_xor(wdst, &t, wsrc);
118 if (enc && second_pass)
119 gf128mul_x_ble(&rctx->t, &t);
120 skcipher_walk_done(&w, avail - bs);
121 return 0;
122 }
123
124 le128_xor(wdst++, &t, wsrc++);
125 gf128mul_x_ble(&t, &t);
126 } while ((avail -= bs) >= bs);
127
128 err = skcipher_walk_done(&w, avail);
129 }
130
131 return err;
132}
133
134static int xts_xor_tweak_pre(struct skcipher_request *req, bool enc)
135{
136 return xts_xor_tweak(req, false, enc);
137}
138
139static int xts_xor_tweak_post(struct skcipher_request *req, bool enc)
140{
141 return xts_xor_tweak(req, true, enc);
142}
143
144static void xts_cts_done(void *data, int err)
145{
146 struct skcipher_request *req = data;
147 le128 b;
148
149 if (!err) {
150 struct xts_request_ctx *rctx = skcipher_request_ctx(req);
151
152 scatterwalk_map_and_copy(&b, rctx->tail, 0, XTS_BLOCK_SIZE, 0);
153 le128_xor(&b, &rctx->t, &b);
154 scatterwalk_map_and_copy(&b, rctx->tail, 0, XTS_BLOCK_SIZE, 1);
155 }
156
157 skcipher_request_complete(req, err);
158}
159
160static int xts_cts_final(struct skcipher_request *req,
161 int (*crypt)(struct skcipher_request *req))
162{
163 const struct xts_tfm_ctx *ctx =
164 crypto_skcipher_ctx(crypto_skcipher_reqtfm(req));
165 int offset = req->cryptlen & ~(XTS_BLOCK_SIZE - 1);
166 struct xts_request_ctx *rctx = skcipher_request_ctx(req);
167 struct skcipher_request *subreq = &rctx->subreq;
168 int tail = req->cryptlen % XTS_BLOCK_SIZE;
169 le128 b[2];
170 int err;
171
172 rctx->tail = scatterwalk_ffwd(rctx->sg, req->dst,
173 offset - XTS_BLOCK_SIZE);
174
175 scatterwalk_map_and_copy(b, rctx->tail, 0, XTS_BLOCK_SIZE, 0);
176 b[1] = b[0];
177 scatterwalk_map_and_copy(b, req->src, offset, tail, 0);
178
179 le128_xor(b, &rctx->t, b);
180
181 scatterwalk_map_and_copy(b, rctx->tail, 0, XTS_BLOCK_SIZE + tail, 1);
182
183 skcipher_request_set_tfm(subreq, ctx->child);
184 skcipher_request_set_callback(subreq, req->base.flags, xts_cts_done,
185 req);
186 skcipher_request_set_crypt(subreq, rctx->tail, rctx->tail,
187 XTS_BLOCK_SIZE, NULL);
188
189 err = crypt(subreq);
190 if (err)
191 return err;
192
193 scatterwalk_map_and_copy(b, rctx->tail, 0, XTS_BLOCK_SIZE, 0);
194 le128_xor(b, &rctx->t, b);
195 scatterwalk_map_and_copy(b, rctx->tail, 0, XTS_BLOCK_SIZE, 1);
196
197 return 0;
198}
199
200static void xts_encrypt_done(void *data, int err)
201{
202 struct skcipher_request *req = data;
203
204 if (!err) {
205 struct xts_request_ctx *rctx = skcipher_request_ctx(req);
206
207 rctx->subreq.base.flags &= CRYPTO_TFM_REQ_MAY_BACKLOG;
208 err = xts_xor_tweak_post(req, true);
209
210 if (!err && unlikely(req->cryptlen % XTS_BLOCK_SIZE)) {
211 err = xts_cts_final(req, crypto_skcipher_encrypt);
212 if (err == -EINPROGRESS || err == -EBUSY)
213 return;
214 }
215 }
216
217 skcipher_request_complete(req, err);
218}
219
220static void xts_decrypt_done(void *data, int err)
221{
222 struct skcipher_request *req = data;
223
224 if (!err) {
225 struct xts_request_ctx *rctx = skcipher_request_ctx(req);
226
227 rctx->subreq.base.flags &= CRYPTO_TFM_REQ_MAY_BACKLOG;
228 err = xts_xor_tweak_post(req, false);
229
230 if (!err && unlikely(req->cryptlen % XTS_BLOCK_SIZE)) {
231 err = xts_cts_final(req, crypto_skcipher_decrypt);
232 if (err == -EINPROGRESS || err == -EBUSY)
233 return;
234 }
235 }
236
237 skcipher_request_complete(req, err);
238}
239
240static int xts_init_crypt(struct skcipher_request *req,
241 crypto_completion_t compl)
242{
243 const struct xts_tfm_ctx *ctx =
244 crypto_skcipher_ctx(crypto_skcipher_reqtfm(req));
245 struct xts_request_ctx *rctx = skcipher_request_ctx(req);
246 struct skcipher_request *subreq = &rctx->subreq;
247
248 if (req->cryptlen < XTS_BLOCK_SIZE)
249 return -EINVAL;
250
251 skcipher_request_set_tfm(subreq, ctx->child);
252 skcipher_request_set_callback(subreq, req->base.flags, compl, req);
253 skcipher_request_set_crypt(subreq, req->dst, req->dst,
254 req->cryptlen & ~(XTS_BLOCK_SIZE - 1), NULL);
255
256 /* calculate first value of T */
257 crypto_cipher_encrypt_one(ctx->tweak, (u8 *)&rctx->t, req->iv);
258
259 return 0;
260}
261
262static int xts_encrypt(struct skcipher_request *req)
263{
264 struct xts_request_ctx *rctx = skcipher_request_ctx(req);
265 struct skcipher_request *subreq = &rctx->subreq;
266 int err;
267
268 err = xts_init_crypt(req, xts_encrypt_done) ?:
269 xts_xor_tweak_pre(req, true) ?:
270 crypto_skcipher_encrypt(subreq) ?:
271 xts_xor_tweak_post(req, true);
272
273 if (err || likely((req->cryptlen % XTS_BLOCK_SIZE) == 0))
274 return err;
275
276 return xts_cts_final(req, crypto_skcipher_encrypt);
277}
278
279static int xts_decrypt(struct skcipher_request *req)
280{
281 struct xts_request_ctx *rctx = skcipher_request_ctx(req);
282 struct skcipher_request *subreq = &rctx->subreq;
283 int err;
284
285 err = xts_init_crypt(req, xts_decrypt_done) ?:
286 xts_xor_tweak_pre(req, false) ?:
287 crypto_skcipher_decrypt(subreq) ?:
288 xts_xor_tweak_post(req, false);
289
290 if (err || likely((req->cryptlen % XTS_BLOCK_SIZE) == 0))
291 return err;
292
293 return xts_cts_final(req, crypto_skcipher_decrypt);
294}
295
296static int xts_init_tfm(struct crypto_skcipher *tfm)
297{
298 struct skcipher_instance *inst = skcipher_alg_instance(tfm);
299 struct xts_instance_ctx *ictx = skcipher_instance_ctx(inst);
300 struct xts_tfm_ctx *ctx = crypto_skcipher_ctx(tfm);
301 struct crypto_skcipher *child;
302 struct crypto_cipher *tweak;
303
304 child = crypto_spawn_skcipher(&ictx->spawn);
305 if (IS_ERR(child))
306 return PTR_ERR(child);
307
308 ctx->child = child;
309
310 tweak = crypto_spawn_cipher(&ictx->tweak_spawn);
311 if (IS_ERR(tweak)) {
312 crypto_free_skcipher(ctx->child);
313 return PTR_ERR(tweak);
314 }
315
316 ctx->tweak = tweak;
317
318 crypto_skcipher_set_reqsize(tfm, crypto_skcipher_reqsize(child) +
319 sizeof(struct xts_request_ctx));
320
321 return 0;
322}
323
324static void xts_exit_tfm(struct crypto_skcipher *tfm)
325{
326 struct xts_tfm_ctx *ctx = crypto_skcipher_ctx(tfm);
327
328 crypto_free_skcipher(ctx->child);
329 crypto_free_cipher(ctx->tweak);
330}
331
332static void xts_free_instance(struct skcipher_instance *inst)
333{
334 struct xts_instance_ctx *ictx = skcipher_instance_ctx(inst);
335
336 crypto_drop_skcipher(&ictx->spawn);
337 crypto_drop_cipher(&ictx->tweak_spawn);
338 kfree(inst);
339}
340
341static int xts_create(struct crypto_template *tmpl, struct rtattr **tb)
342{
343 struct skcipher_alg_common *alg;
344 char name[CRYPTO_MAX_ALG_NAME];
345 struct skcipher_instance *inst;
346 struct xts_instance_ctx *ctx;
347 const char *cipher_name;
348 u32 mask;
349 int err;
350
351 err = crypto_check_attr_type(tb, CRYPTO_ALG_TYPE_SKCIPHER, &mask);
352 if (err)
353 return err;
354
355 cipher_name = crypto_attr_alg_name(tb[1]);
356 if (IS_ERR(cipher_name))
357 return PTR_ERR(cipher_name);
358
359 inst = kzalloc(sizeof(*inst) + sizeof(*ctx), GFP_KERNEL);
360 if (!inst)
361 return -ENOMEM;
362
363 ctx = skcipher_instance_ctx(inst);
364
365 err = crypto_grab_skcipher(&ctx->spawn, skcipher_crypto_instance(inst),
366 cipher_name, 0, mask);
367 if (err == -ENOENT && memcmp(cipher_name, "ecb(", 4)) {
368 err = -ENAMETOOLONG;
369 if (snprintf(name, CRYPTO_MAX_ALG_NAME, "ecb(%s)",
370 cipher_name) >= CRYPTO_MAX_ALG_NAME)
371 goto err_free_inst;
372
373 err = crypto_grab_skcipher(&ctx->spawn,
374 skcipher_crypto_instance(inst),
375 name, 0, mask);
376 }
377
378 if (err)
379 goto err_free_inst;
380
381 alg = crypto_spawn_skcipher_alg_common(&ctx->spawn);
382
383 err = -EINVAL;
384 if (alg->base.cra_blocksize != XTS_BLOCK_SIZE)
385 goto err_free_inst;
386
387 if (alg->ivsize)
388 goto err_free_inst;
389
390 err = crypto_inst_setname(skcipher_crypto_instance(inst), "xts",
391 &alg->base);
392 if (err)
393 goto err_free_inst;
394
395 err = -EINVAL;
396 cipher_name = alg->base.cra_name;
397
398 /* Alas we screwed up the naming so we have to mangle the
399 * cipher name.
400 */
401 if (!memcmp(cipher_name, "ecb(", 4)) {
402 int len;
403
404 len = strscpy(name, cipher_name + 4);
405 if (len < 2)
406 goto err_free_inst;
407
408 if (name[len - 1] != ')')
409 goto err_free_inst;
410
411 name[len - 1] = 0;
412
413 if (snprintf(inst->alg.base.cra_name, CRYPTO_MAX_ALG_NAME,
414 "xts(%s)", name) >= CRYPTO_MAX_ALG_NAME) {
415 err = -ENAMETOOLONG;
416 goto err_free_inst;
417 }
418 } else
419 goto err_free_inst;
420
421 err = crypto_grab_cipher(&ctx->tweak_spawn,
422 skcipher_crypto_instance(inst), name, 0, mask);
423 if (err)
424 goto err_free_inst;
425
426 inst->alg.base.cra_priority = alg->base.cra_priority;
427 inst->alg.base.cra_blocksize = XTS_BLOCK_SIZE;
428 inst->alg.base.cra_alignmask = alg->base.cra_alignmask |
429 (__alignof__(u64) - 1);
430
431 inst->alg.ivsize = XTS_BLOCK_SIZE;
432 inst->alg.min_keysize = alg->min_keysize * 2;
433 inst->alg.max_keysize = alg->max_keysize * 2;
434
435 inst->alg.base.cra_ctxsize = sizeof(struct xts_tfm_ctx);
436
437 inst->alg.init = xts_init_tfm;
438 inst->alg.exit = xts_exit_tfm;
439
440 inst->alg.setkey = xts_setkey;
441 inst->alg.encrypt = xts_encrypt;
442 inst->alg.decrypt = xts_decrypt;
443
444 inst->free = xts_free_instance;
445
446 err = skcipher_register_instance(tmpl, inst);
447 if (err) {
448err_free_inst:
449 xts_free_instance(inst);
450 }
451 return err;
452}
453
454static struct crypto_template xts_tmpl = {
455 .name = "xts",
456 .create = xts_create,
457 .module = THIS_MODULE,
458};
459
460static int __init xts_module_init(void)
461{
462 return crypto_register_template(&xts_tmpl);
463}
464
465static void __exit xts_module_exit(void)
466{
467 crypto_unregister_template(&xts_tmpl);
468}
469
470module_init(xts_module_init);
471module_exit(xts_module_exit);
472
473MODULE_LICENSE("GPL");
474MODULE_DESCRIPTION("XTS block cipher mode");
475MODULE_ALIAS_CRYPTO("xts");
476MODULE_IMPORT_NS("CRYPTO_INTERNAL");
477MODULE_SOFTDEP("pre: ecb");