projects/linux·blob master · refs

480 lines · 13.9 KiB

1// SPDX-License-Identifier: GPL-2.0
2/*
3 * HCTR2 length-preserving encryption mode
4 *
5 * Copyright 2021 Google LLC
6 */
7
8
9/*
10 * HCTR2 is a length-preserving encryption mode that is efficient on
11 * processors with instructions to accelerate AES and carryless
12 * multiplication, e.g. x86 processors with AES-NI and CLMUL, and ARM
13 * processors with the ARMv8 crypto extensions.
14 *
15 * For more details, see the paper: "Length-preserving encryption with HCTR2"
16 * (https://eprint.iacr.org/2021/1441.pdf)
17 */
18
19#include <crypto/gf128hash.h>
20#include <crypto/internal/cipher.h>
21#include <crypto/internal/skcipher.h>
22#include <crypto/scatterwalk.h>
23#include <linux/module.h>
24
25#define BLOCKCIPHER_BLOCK_SIZE 16
26
27/*
28 * The specification allows variable-length tweaks, but Linux's crypto API
29 * currently only allows algorithms to support a single length. The "natural"
30 * tweak length for HCTR2 is 16, since that fits into one POLYVAL block for
31 * the best performance. But longer tweaks are useful for fscrypt, to avoid
32 * needing to derive per-file keys. So instead we use two blocks, or 32 bytes.
33 */
34#define TWEAK_SIZE 32
35
36struct hctr2_instance_ctx {
37 struct crypto_cipher_spawn blockcipher_spawn;
38 struct crypto_skcipher_spawn xctr_spawn;
39};
40
41struct hctr2_tfm_ctx {
42 struct crypto_cipher *blockcipher;
43 struct crypto_skcipher *xctr;
44 struct polyval_key poly_key;
45 struct polyval_elem hashed_tweaklens[2];
46 u8 L[BLOCKCIPHER_BLOCK_SIZE];
47};
48
49struct hctr2_request_ctx {
50 u8 first_block[BLOCKCIPHER_BLOCK_SIZE];
51 u8 xctr_iv[BLOCKCIPHER_BLOCK_SIZE];
52 struct scatterlist *bulk_part_dst;
53 struct scatterlist *bulk_part_src;
54 struct scatterlist sg_src[2];
55 struct scatterlist sg_dst[2];
56 struct polyval_elem hashed_tweak;
57 /*
58 * skcipher sub-request size is unknown at compile-time, so it needs to
59 * go after the members with known sizes.
60 */
61 union {
62 struct polyval_ctx poly_ctx;
63 struct skcipher_request xctr_req;
64 } u;
65};
66
67/*
68 * The input data for each HCTR2 hash step begins with a 16-byte block that
69 * contains the tweak length and a flag that indicates whether the input is evenly
70 * divisible into blocks. Since this implementation only supports one tweak
71 * length, we precompute the two hash states resulting from hashing the two
72 * possible values of this initial block. This reduces by one block the amount of
73 * data that needs to be hashed for each encryption/decryption
74 *
75 * These precomputed hashes are stored in hctr2_tfm_ctx.
76 */
77static void hctr2_hash_tweaklens(struct hctr2_tfm_ctx *tctx)
78{
79 struct polyval_ctx ctx;
80
81 for (int has_remainder = 0; has_remainder < 2; has_remainder++) {
82 const __le64 tweak_length_block[2] = {
83 cpu_to_le64(TWEAK_SIZE * 8 * 2 + 2 + has_remainder),
84 };
85
86 polyval_init(&ctx, &tctx->poly_key);
87 polyval_update(&ctx, (const u8 *)&tweak_length_block,
88 sizeof(tweak_length_block));
89 static_assert(sizeof(tweak_length_block) == POLYVAL_BLOCK_SIZE);
90 polyval_export_blkaligned(
91 &ctx, &tctx->hashed_tweaklens[has_remainder]);
92 }
93 memzero_explicit(&ctx, sizeof(ctx));
94}
95
96static int hctr2_setkey(struct crypto_skcipher *tfm, const u8 *key,
97 unsigned int keylen)
98{
99 struct hctr2_tfm_ctx *tctx = crypto_skcipher_ctx(tfm);
100 u8 hbar[BLOCKCIPHER_BLOCK_SIZE];
101 int err;
102
103 crypto_cipher_clear_flags(tctx->blockcipher, CRYPTO_TFM_REQ_MASK);
104 crypto_cipher_set_flags(tctx->blockcipher,
105 crypto_skcipher_get_flags(tfm) &
106 CRYPTO_TFM_REQ_MASK);
107 err = crypto_cipher_setkey(tctx->blockcipher, key, keylen);
108 if (err)
109 return err;
110
111 crypto_skcipher_clear_flags(tctx->xctr, CRYPTO_TFM_REQ_MASK);
112 crypto_skcipher_set_flags(tctx->xctr,
113 crypto_skcipher_get_flags(tfm) &
114 CRYPTO_TFM_REQ_MASK);
115 err = crypto_skcipher_setkey(tctx->xctr, key, keylen);
116 if (err)
117 return err;
118
119 memset(hbar, 0, sizeof(hbar));
120 crypto_cipher_encrypt_one(tctx->blockcipher, hbar, hbar);
121
122 memset(tctx->L, 0, sizeof(tctx->L));
123 tctx->L[0] = 0x01;
124 crypto_cipher_encrypt_one(tctx->blockcipher, tctx->L, tctx->L);
125
126 static_assert(sizeof(hbar) == POLYVAL_BLOCK_SIZE);
127 polyval_preparekey(&tctx->poly_key, hbar);
128 memzero_explicit(hbar, sizeof(hbar));
129
130 hctr2_hash_tweaklens(tctx);
131 return 0;
132}
133
134static void hctr2_hash_tweak(struct skcipher_request *req)
135{
136 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
137 const struct hctr2_tfm_ctx *tctx = crypto_skcipher_ctx(tfm);
138 struct hctr2_request_ctx *rctx = skcipher_request_ctx(req);
139 struct polyval_ctx *poly_ctx = &rctx->u.poly_ctx;
140 bool has_remainder = req->cryptlen % POLYVAL_BLOCK_SIZE;
141
142 polyval_import_blkaligned(poly_ctx, &tctx->poly_key,
143 &tctx->hashed_tweaklens[has_remainder]);
144 polyval_update(poly_ctx, req->iv, TWEAK_SIZE);
145
146 // Store the hashed tweak, since we need it when computing both
147 // H(T || N) and H(T || V).
148 static_assert(TWEAK_SIZE % POLYVAL_BLOCK_SIZE == 0);
149 polyval_export_blkaligned(poly_ctx, &rctx->hashed_tweak);
150}
151
152static void hctr2_hash_message(struct skcipher_request *req,
153 struct scatterlist *sgl,
154 u8 digest[POLYVAL_DIGEST_SIZE])
155{
156 static const u8 padding = 0x1;
157 struct hctr2_request_ctx *rctx = skcipher_request_ctx(req);
158 struct polyval_ctx *poly_ctx = &rctx->u.poly_ctx;
159 const unsigned int bulk_len = req->cryptlen - BLOCKCIPHER_BLOCK_SIZE;
160 struct sg_mapping_iter miter;
161 int i;
162 int n = 0;
163
164 sg_miter_start(&miter, sgl, sg_nents(sgl),
165 SG_MITER_FROM_SG | SG_MITER_ATOMIC);
166 for (i = 0; i < bulk_len; i += n) {
167 sg_miter_next(&miter);
168 n = min_t(unsigned int, miter.length, bulk_len - i);
169 polyval_update(poly_ctx, miter.addr, n);
170 }
171 sg_miter_stop(&miter);
172
173 if (req->cryptlen % BLOCKCIPHER_BLOCK_SIZE)
174 polyval_update(poly_ctx, &padding, 1);
175 polyval_final(poly_ctx, digest);
176}
177
178static int hctr2_finish(struct skcipher_request *req)
179{
180 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
181 const struct hctr2_tfm_ctx *tctx = crypto_skcipher_ctx(tfm);
182 struct hctr2_request_ctx *rctx = skcipher_request_ctx(req);
183 struct polyval_ctx *poly_ctx = &rctx->u.poly_ctx;
184 u8 digest[POLYVAL_DIGEST_SIZE];
185
186 // U = UU ^ H(T || V)
187 // or M = MM ^ H(T || N)
188 polyval_import_blkaligned(poly_ctx, &tctx->poly_key,
189 &rctx->hashed_tweak);
190 hctr2_hash_message(req, rctx->bulk_part_dst, digest);
191 crypto_xor(rctx->first_block, digest, BLOCKCIPHER_BLOCK_SIZE);
192
193 // Copy U (or M) into dst scatterlist
194 scatterwalk_map_and_copy(rctx->first_block, req->dst,
195 0, BLOCKCIPHER_BLOCK_SIZE, 1);
196 return 0;
197}
198
199static void hctr2_xctr_done(void *data, int err)
200{
201 struct skcipher_request *req = data;
202
203 if (!err)
204 err = hctr2_finish(req);
205
206 skcipher_request_complete(req, err);
207}
208
209static int hctr2_crypt(struct skcipher_request *req, bool enc)
210{
211 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
212 const struct hctr2_tfm_ctx *tctx = crypto_skcipher_ctx(tfm);
213 struct hctr2_request_ctx *rctx = skcipher_request_ctx(req);
214 u8 digest[POLYVAL_DIGEST_SIZE];
215 int bulk_len = req->cryptlen - BLOCKCIPHER_BLOCK_SIZE;
216
217 // Requests must be at least one block
218 if (req->cryptlen < BLOCKCIPHER_BLOCK_SIZE)
219 return -EINVAL;
220
221 // Copy M (or U) into a temporary buffer
222 scatterwalk_map_and_copy(rctx->first_block, req->src,
223 0, BLOCKCIPHER_BLOCK_SIZE, 0);
224
225 // Create scatterlists for N and V
226 rctx->bulk_part_src = scatterwalk_ffwd(rctx->sg_src, req->src,
227 BLOCKCIPHER_BLOCK_SIZE);
228 rctx->bulk_part_dst = scatterwalk_ffwd(rctx->sg_dst, req->dst,
229 BLOCKCIPHER_BLOCK_SIZE);
230
231 // MM = M ^ H(T || N)
232 // or UU = U ^ H(T || V)
233 hctr2_hash_tweak(req);
234 hctr2_hash_message(req, rctx->bulk_part_src, digest);
235 crypto_xor(digest, rctx->first_block, BLOCKCIPHER_BLOCK_SIZE);
236
237 // UU = E(MM)
238 // or MM = D(UU)
239 if (enc)
240 crypto_cipher_encrypt_one(tctx->blockcipher, rctx->first_block,
241 digest);
242 else
243 crypto_cipher_decrypt_one(tctx->blockcipher, rctx->first_block,
244 digest);
245
246 // S = MM ^ UU ^ L
247 crypto_xor(digest, rctx->first_block, BLOCKCIPHER_BLOCK_SIZE);
248 crypto_xor_cpy(rctx->xctr_iv, digest, tctx->L, BLOCKCIPHER_BLOCK_SIZE);
249
250 // V = XCTR(S, N)
251 // or N = XCTR(S, V)
252 skcipher_request_set_tfm(&rctx->u.xctr_req, tctx->xctr);
253 skcipher_request_set_crypt(&rctx->u.xctr_req, rctx->bulk_part_src,
254 rctx->bulk_part_dst, bulk_len,
255 rctx->xctr_iv);
256 skcipher_request_set_callback(&rctx->u.xctr_req,
257 req->base.flags,
258 hctr2_xctr_done, req);
259 return crypto_skcipher_encrypt(&rctx->u.xctr_req) ?:
260 hctr2_finish(req);
261}
262
263static int hctr2_encrypt(struct skcipher_request *req)
264{
265 return hctr2_crypt(req, true);
266}
267
268static int hctr2_decrypt(struct skcipher_request *req)
269{
270 return hctr2_crypt(req, false);
271}
272
273static int hctr2_init_tfm(struct crypto_skcipher *tfm)
274{
275 struct skcipher_instance *inst = skcipher_alg_instance(tfm);
276 struct hctr2_instance_ctx *ictx = skcipher_instance_ctx(inst);
277 struct hctr2_tfm_ctx *tctx = crypto_skcipher_ctx(tfm);
278 struct crypto_skcipher *xctr;
279 struct crypto_cipher *blockcipher;
280 int err;
281
282 xctr = crypto_spawn_skcipher(&ictx->xctr_spawn);
283 if (IS_ERR(xctr))
284 return PTR_ERR(xctr);
285
286 blockcipher = crypto_spawn_cipher(&ictx->blockcipher_spawn);
287 if (IS_ERR(blockcipher)) {
288 err = PTR_ERR(blockcipher);
289 goto err_free_xctr;
290 }
291
292 tctx->xctr = xctr;
293 tctx->blockcipher = blockcipher;
294
295 BUILD_BUG_ON(offsetofend(struct hctr2_request_ctx, u) !=
296 sizeof(struct hctr2_request_ctx));
297 crypto_skcipher_set_reqsize(
298 tfm, max(sizeof(struct hctr2_request_ctx),
299 offsetofend(struct hctr2_request_ctx, u.xctr_req) +
300 crypto_skcipher_reqsize(xctr)));
301 return 0;
302
303err_free_xctr:
304 crypto_free_skcipher(xctr);
305 return err;
306}
307
308static void hctr2_exit_tfm(struct crypto_skcipher *tfm)
309{
310 struct hctr2_tfm_ctx *tctx = crypto_skcipher_ctx(tfm);
311
312 crypto_free_cipher(tctx->blockcipher);
313 crypto_free_skcipher(tctx->xctr);
314}
315
316static void hctr2_free_instance(struct skcipher_instance *inst)
317{
318 struct hctr2_instance_ctx *ictx = skcipher_instance_ctx(inst);
319
320 crypto_drop_cipher(&ictx->blockcipher_spawn);
321 crypto_drop_skcipher(&ictx->xctr_spawn);
322 kfree(inst);
323}
324
325static int hctr2_create_common(struct crypto_template *tmpl, struct rtattr **tb,
326 const char *xctr_name)
327{
328 struct skcipher_alg_common *xctr_alg;
329 u32 mask;
330 struct skcipher_instance *inst;
331 struct hctr2_instance_ctx *ictx;
332 struct crypto_alg *blockcipher_alg;
333 char blockcipher_name[CRYPTO_MAX_ALG_NAME];
334 int len;
335 int err;
336
337 err = crypto_check_attr_type(tb, CRYPTO_ALG_TYPE_SKCIPHER, &mask);
338 if (err)
339 return err;
340
341 inst = kzalloc(sizeof(*inst) + sizeof(*ictx), GFP_KERNEL);
342 if (!inst)
343 return -ENOMEM;
344 ictx = skcipher_instance_ctx(inst);
345
346 /* Stream cipher, xctr(block_cipher) */
347 err = crypto_grab_skcipher(&ictx->xctr_spawn,
348 skcipher_crypto_instance(inst),
349 xctr_name, 0, mask);
350 if (err)
351 goto err_free_inst;
352 xctr_alg = crypto_spawn_skcipher_alg_common(&ictx->xctr_spawn);
353
354 err = -EINVAL;
355 if (strncmp(xctr_alg->base.cra_name, "xctr(", 5))
356 goto err_free_inst;
357 len = strscpy(blockcipher_name, xctr_alg->base.cra_name + 5);
358 if (len < 1)
359 goto err_free_inst;
360 if (blockcipher_name[len - 1] != ')')
361 goto err_free_inst;
362 blockcipher_name[len - 1] = 0;
363
364 /* Block cipher, e.g. "aes" */
365 err = crypto_grab_cipher(&ictx->blockcipher_spawn,
366 skcipher_crypto_instance(inst),
367 blockcipher_name, 0, mask);
368 if (err)
369 goto err_free_inst;
370 blockcipher_alg = crypto_spawn_cipher_alg(&ictx->blockcipher_spawn);
371
372 /* Require blocksize of 16 bytes */
373 err = -EINVAL;
374 if (blockcipher_alg->cra_blocksize != BLOCKCIPHER_BLOCK_SIZE)
375 goto err_free_inst;
376
377 /* Instance fields */
378
379 err = -ENAMETOOLONG;
380 if (snprintf(inst->alg.base.cra_name, CRYPTO_MAX_ALG_NAME, "hctr2(%s)",
381 blockcipher_alg->cra_name) >= CRYPTO_MAX_ALG_NAME)
382 goto err_free_inst;
383 if (snprintf(inst->alg.base.cra_driver_name, CRYPTO_MAX_ALG_NAME,
384 "hctr2_base(%s,polyval-lib)",
385 xctr_alg->base.cra_driver_name) >= CRYPTO_MAX_ALG_NAME)
386 goto err_free_inst;
387
388 inst->alg.base.cra_blocksize = BLOCKCIPHER_BLOCK_SIZE;
389 inst->alg.base.cra_ctxsize = sizeof(struct hctr2_tfm_ctx);
390 inst->alg.base.cra_alignmask = xctr_alg->base.cra_alignmask;
391 inst->alg.base.cra_priority = (2 * xctr_alg->base.cra_priority +
392 blockcipher_alg->cra_priority) /
393 3;
394
395 inst->alg.setkey = hctr2_setkey;
396 inst->alg.encrypt = hctr2_encrypt;
397 inst->alg.decrypt = hctr2_decrypt;
398 inst->alg.init = hctr2_init_tfm;
399 inst->alg.exit = hctr2_exit_tfm;
400 inst->alg.min_keysize = xctr_alg->min_keysize;
401 inst->alg.max_keysize = xctr_alg->max_keysize;
402 inst->alg.ivsize = TWEAK_SIZE;
403
404 inst->free = hctr2_free_instance;
405
406 err = skcipher_register_instance(tmpl, inst);
407 if (err) {
408err_free_inst:
409 hctr2_free_instance(inst);
410 }
411 return err;
412}
413
414static int hctr2_create_base(struct crypto_template *tmpl, struct rtattr **tb)
415{
416 const char *xctr_name;
417 const char *polyval_name;
418
419 xctr_name = crypto_attr_alg_name(tb[1]);
420 if (IS_ERR(xctr_name))
421 return PTR_ERR(xctr_name);
422
423 polyval_name = crypto_attr_alg_name(tb[2]);
424 if (IS_ERR(polyval_name))
425 return PTR_ERR(polyval_name);
426 if (strcmp(polyval_name, "polyval") != 0 &&
427 strcmp(polyval_name, "polyval-lib") != 0)
428 return -ENOENT;
429
430 return hctr2_create_common(tmpl, tb, xctr_name);
431}
432
433static int hctr2_create(struct crypto_template *tmpl, struct rtattr **tb)
434{
435 const char *blockcipher_name;
436 char xctr_name[CRYPTO_MAX_ALG_NAME];
437
438 blockcipher_name = crypto_attr_alg_name(tb[1]);
439 if (IS_ERR(blockcipher_name))
440 return PTR_ERR(blockcipher_name);
441
442 if (snprintf(xctr_name, CRYPTO_MAX_ALG_NAME, "xctr(%s)",
443 blockcipher_name) >= CRYPTO_MAX_ALG_NAME)
444 return -ENAMETOOLONG;
445
446 return hctr2_create_common(tmpl, tb, xctr_name);
447}
448
449static struct crypto_template hctr2_tmpls[] = {
450 {
451 /* hctr2_base(xctr_name, polyval_name) */
452 .name = "hctr2_base",
453 .create = hctr2_create_base,
454 .module = THIS_MODULE,
455 }, {
456 /* hctr2(blockcipher_name) */
457 .name = "hctr2",
458 .create = hctr2_create,
459 .module = THIS_MODULE,
460 }
461};
462
463static int __init hctr2_module_init(void)
464{
465 return crypto_register_templates(hctr2_tmpls, ARRAY_SIZE(hctr2_tmpls));
466}
467
468static void __exit hctr2_module_exit(void)
469{
470 return crypto_unregister_templates(hctr2_tmpls,
471 ARRAY_SIZE(hctr2_tmpls));
472}
473
474module_init(hctr2_module_init);
475module_exit(hctr2_module_exit);
476
477MODULE_DESCRIPTION("HCTR2 length-preserving encryption mode");
478MODULE_LICENSE("GPL v2");
479MODULE_ALIAS_CRYPTO("hctr2");
480MODULE_IMPORT_NS("CRYPTO_INTERNAL");