projects/linux·blob master · refs

598 lines · 16.5 KiB

1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Copyright 2019 Google LLC
4 */
5
6/*
7 * Refer to Documentation/block/inline-encryption.rst for detailed explanation.
8 */
9
10#define pr_fmt(fmt) "blk-crypto: " fmt
11
12#include <linux/bio.h>
13#include <linux/blkdev.h>
14#include <linux/blk-crypto-profile.h>
15#include <linux/module.h>
16#include <linux/ratelimit.h>
17#include <linux/slab.h>
18
19#include "blk-crypto-internal.h"
20
21const struct blk_crypto_mode blk_crypto_modes[] = {
22 [BLK_ENCRYPTION_MODE_AES_256_XTS] = {
23 .name = "AES-256-XTS",
24 .cipher_str = "xts(aes)",
25 .keysize = 64,
26 .security_strength = 32,
27 .ivsize = 16,
28 },
29 [BLK_ENCRYPTION_MODE_AES_128_CBC_ESSIV] = {
30 .name = "AES-128-CBC-ESSIV",
31 .cipher_str = "essiv(cbc(aes),sha256)",
32 .keysize = 16,
33 .security_strength = 16,
34 .ivsize = 16,
35 },
36 [BLK_ENCRYPTION_MODE_ADIANTUM] = {
37 .name = "Adiantum",
38 .cipher_str = "adiantum(xchacha12,aes)",
39 .keysize = 32,
40 .security_strength = 32,
41 .ivsize = 32,
42 },
43 [BLK_ENCRYPTION_MODE_SM4_XTS] = {
44 .name = "SM4-XTS",
45 .cipher_str = "xts(sm4)",
46 .keysize = 32,
47 .security_strength = 16,
48 .ivsize = 16,
49 },
50};
51
52/*
53 * This number needs to be at least (the number of threads doing IO
54 * concurrently) * (maximum recursive depth of a bio), so that we don't
55 * deadlock on crypt_ctx allocations. The default is chosen to be the same
56 * as the default number of post read contexts in both EXT4 and F2FS.
57 */
58static int num_prealloc_crypt_ctxs = 128;
59
60module_param(num_prealloc_crypt_ctxs, int, 0444);
61MODULE_PARM_DESC(num_prealloc_crypt_ctxs,
62 "Number of bio crypto contexts to preallocate");
63
64static struct kmem_cache *bio_crypt_ctx_cache;
65static mempool_t *bio_crypt_ctx_pool;
66
67static int __init bio_crypt_ctx_init(void)
68{
69 size_t i;
70
71 bio_crypt_ctx_cache = KMEM_CACHE(bio_crypt_ctx, 0);
72 if (!bio_crypt_ctx_cache)
73 goto out_no_mem;
74
75 bio_crypt_ctx_pool = mempool_create_slab_pool(num_prealloc_crypt_ctxs,
76 bio_crypt_ctx_cache);
77 if (!bio_crypt_ctx_pool)
78 goto out_no_mem;
79
80 /* This is assumed in various places. */
81 BUILD_BUG_ON(BLK_ENCRYPTION_MODE_INVALID != 0);
82
83 /*
84 * Validate the crypto mode properties. This ideally would be done with
85 * static assertions, but boot-time checks are the next best thing.
86 */
87 for (i = 0; i < BLK_ENCRYPTION_MODE_MAX; i++) {
88 BUG_ON(blk_crypto_modes[i].keysize >
89 BLK_CRYPTO_MAX_RAW_KEY_SIZE);
90 BUG_ON(blk_crypto_modes[i].security_strength >
91 blk_crypto_modes[i].keysize);
92 BUG_ON(blk_crypto_modes[i].ivsize > BLK_CRYPTO_MAX_IV_SIZE);
93 }
94
95 return 0;
96out_no_mem:
97 panic("Failed to allocate mem for bio crypt ctxs\n");
98}
99subsys_initcall(bio_crypt_ctx_init);
100
101void bio_crypt_set_ctx(struct bio *bio, const struct blk_crypto_key *key,
102 const u64 dun[BLK_CRYPTO_DUN_ARRAY_SIZE], gfp_t gfp_mask)
103{
104 struct bio_crypt_ctx *bc;
105
106 /*
107 * The caller must use a gfp_mask that contains __GFP_DIRECT_RECLAIM so
108 * that the mempool_alloc() can't fail.
109 */
110 WARN_ON_ONCE(!(gfp_mask & __GFP_DIRECT_RECLAIM));
111
112 bc = mempool_alloc(bio_crypt_ctx_pool, gfp_mask);
113
114 bc->bc_key = key;
115 memcpy(bc->bc_dun, dun, sizeof(bc->bc_dun));
116
117 bio->bi_crypt_context = bc;
118}
119EXPORT_SYMBOL_GPL(bio_crypt_set_ctx);
120
121void __bio_crypt_free_ctx(struct bio *bio)
122{
123 mempool_free(bio->bi_crypt_context, bio_crypt_ctx_pool);
124 bio->bi_crypt_context = NULL;
125}
126
127int __bio_crypt_clone(struct bio *dst, struct bio *src, gfp_t gfp_mask)
128{
129 dst->bi_crypt_context = mempool_alloc(bio_crypt_ctx_pool, gfp_mask);
130 if (!dst->bi_crypt_context)
131 return -ENOMEM;
132 *dst->bi_crypt_context = *src->bi_crypt_context;
133 return 0;
134}
135
136/* Increments @dun by @inc, treating @dun as a multi-limb integer. */
137void bio_crypt_dun_increment(u64 dun[BLK_CRYPTO_DUN_ARRAY_SIZE],
138 unsigned int inc)
139{
140 int i;
141
142 for (i = 0; inc && i < BLK_CRYPTO_DUN_ARRAY_SIZE; i++) {
143 dun[i] += inc;
144 /*
145 * If the addition in this limb overflowed, then we need to
146 * carry 1 into the next limb. Else the carry is 0.
147 */
148 if (dun[i] < inc)
149 inc = 1;
150 else
151 inc = 0;
152 }
153}
154
155void __bio_crypt_advance(struct bio *bio, unsigned int bytes)
156{
157 struct bio_crypt_ctx *bc = bio->bi_crypt_context;
158
159 bio_crypt_dun_increment(bc->bc_dun,
160 bytes >> bc->bc_key->data_unit_size_bits);
161}
162
163/*
164 * Returns true if @bc->bc_dun plus @bytes converted to data units is equal to
165 * @next_dun, treating the DUNs as multi-limb integers.
166 */
167bool bio_crypt_dun_is_contiguous(const struct bio_crypt_ctx *bc,
168 unsigned int bytes,
169 const u64 next_dun[BLK_CRYPTO_DUN_ARRAY_SIZE])
170{
171 int i;
172 unsigned int carry = bytes >> bc->bc_key->data_unit_size_bits;
173
174 for (i = 0; i < BLK_CRYPTO_DUN_ARRAY_SIZE; i++) {
175 if (bc->bc_dun[i] + carry != next_dun[i])
176 return false;
177 /*
178 * If the addition in this limb overflowed, then we need to
179 * carry 1 into the next limb. Else the carry is 0.
180 */
181 if ((bc->bc_dun[i] + carry) < carry)
182 carry = 1;
183 else
184 carry = 0;
185 }
186
187 /* If the DUN wrapped through 0, don't treat it as contiguous. */
188 return carry == 0;
189}
190
191/*
192 * Checks that two bio crypt contexts are compatible - i.e. that
193 * they are mergeable except for data_unit_num continuity.
194 */
195static bool bio_crypt_ctx_compatible(struct bio_crypt_ctx *bc1,
196 struct bio_crypt_ctx *bc2)
197{
198 if (!bc1)
199 return !bc2;
200
201 return bc2 && bc1->bc_key == bc2->bc_key;
202}
203
204bool bio_crypt_rq_ctx_compatible(struct request *rq, struct bio *bio)
205{
206 return bio_crypt_ctx_compatible(rq->crypt_ctx, bio->bi_crypt_context);
207}
208
209/*
210 * Checks that two bio crypt contexts are compatible, and also
211 * that their data_unit_nums are continuous (and can hence be merged)
212 * in the order @bc1 followed by @bc2.
213 */
214bool bio_crypt_ctx_mergeable(struct bio_crypt_ctx *bc1, unsigned int bc1_bytes,
215 struct bio_crypt_ctx *bc2)
216{
217 if (!bio_crypt_ctx_compatible(bc1, bc2))
218 return false;
219
220 return !bc1 || bio_crypt_dun_is_contiguous(bc1, bc1_bytes, bc2->bc_dun);
221}
222
223blk_status_t __blk_crypto_rq_get_keyslot(struct request *rq)
224{
225 return blk_crypto_get_keyslot(rq->q->crypto_profile,
226 rq->crypt_ctx->bc_key,
227 &rq->crypt_keyslot);
228}
229
230void __blk_crypto_rq_put_keyslot(struct request *rq)
231{
232 blk_crypto_put_keyslot(rq->crypt_keyslot);
233 rq->crypt_keyslot = NULL;
234}
235
236void __blk_crypto_free_request(struct request *rq)
237{
238 /* The keyslot, if one was needed, should have been released earlier. */
239 if (WARN_ON_ONCE(rq->crypt_keyslot))
240 __blk_crypto_rq_put_keyslot(rq);
241
242 mempool_free(rq->crypt_ctx, bio_crypt_ctx_pool);
243 rq->crypt_ctx = NULL;
244}
245
246/*
247 * Process a bio with a crypto context. Returns true if the caller should
248 * submit the passed in bio, false if the bio is consumed.
249 *
250 * See the kerneldoc comment for blk_crypto_submit_bio for further details.
251 */
252bool __blk_crypto_submit_bio(struct bio *bio)
253{
254 const struct blk_crypto_key *bc_key = bio->bi_crypt_context->bc_key;
255 struct block_device *bdev = bio->bi_bdev;
256
257 /* Error if bio has no data. */
258 if (WARN_ON_ONCE(!bio_has_data(bio))) {
259 bio_io_error(bio);
260 return false;
261 }
262
263 /*
264 * If the device does not natively support the encryption context, try to use
265 * the fallback if available.
266 */
267 if (!blk_crypto_config_supported_natively(bdev, &bc_key->crypto_cfg)) {
268 if (!IS_ENABLED(CONFIG_BLK_INLINE_ENCRYPTION_FALLBACK)) {
269 pr_warn_once("%pg: crypto API fallback disabled; failing request.\n",
270 bdev);
271 bio_endio_status(bio, BLK_STS_NOTSUPP);
272 return false;
273 }
274 return blk_crypto_fallback_bio_prep(bio);
275 }
276
277 return true;
278}
279EXPORT_SYMBOL_GPL(__blk_crypto_submit_bio);
280
281int __blk_crypto_rq_bio_prep(struct request *rq, struct bio *bio,
282 gfp_t gfp_mask)
283{
284 if (!rq->crypt_ctx) {
285 rq->crypt_ctx = mempool_alloc(bio_crypt_ctx_pool, gfp_mask);
286 if (!rq->crypt_ctx)
287 return -ENOMEM;
288 }
289 *rq->crypt_ctx = *bio->bi_crypt_context;
290 return 0;
291}
292
293/**
294 * blk_crypto_init_key() - Prepare a key for use with blk-crypto
295 * @blk_key: Pointer to the blk_crypto_key to initialize.
296 * @key_bytes: the bytes of the key
297 * @key_size: size of the key in bytes
298 * @key_type: type of the key -- either raw or hardware-wrapped
299 * @crypto_mode: identifier for the encryption algorithm to use
300 * @dun_bytes: number of bytes that will be used to specify the DUN when this
301 * key is used
302 * @data_unit_size: the data unit size to use for en/decryption
303 * @flags: BLK_CRYPTO_CFG_* flags
304 *
305 * Return: 0 on success, -errno on failure. The caller is responsible for
306 * zeroizing both blk_key and key_bytes when done with them.
307 */
308int blk_crypto_init_key(struct blk_crypto_key *blk_key,
309 const u8 *key_bytes, size_t key_size,
310 enum blk_crypto_key_type key_type,
311 enum blk_crypto_mode_num crypto_mode,
312 unsigned int dun_bytes,
313 unsigned int data_unit_size, int flags)
314{
315 const struct blk_crypto_mode *mode;
316
317 memset(blk_key, 0, sizeof(*blk_key));
318
319 if (crypto_mode >= ARRAY_SIZE(blk_crypto_modes))
320 return -EINVAL;
321
322 if (flags & ~BLK_CRYPTO_CFG_ALLOW_HW)
323 return -EINVAL;
324
325 mode = &blk_crypto_modes[crypto_mode];
326 switch (key_type) {
327 case BLK_CRYPTO_KEY_TYPE_RAW:
328 if (key_size != mode->keysize)
329 return -EINVAL;
330 break;
331 case BLK_CRYPTO_KEY_TYPE_HW_WRAPPED:
332 if (key_size < mode->security_strength ||
333 key_size > BLK_CRYPTO_MAX_HW_WRAPPED_KEY_SIZE)
334 return -EINVAL;
335 if (!(flags & BLK_CRYPTO_CFG_ALLOW_HW))
336 return -EINVAL;
337 break;
338 default:
339 return -EINVAL;
340 }
341
342 if (dun_bytes == 0 || dun_bytes > mode->ivsize)
343 return -EINVAL;
344
345 if (!is_power_of_2(data_unit_size))
346 return -EINVAL;
347
348 blk_key->crypto_cfg.crypto_mode = crypto_mode;
349 blk_key->crypto_cfg.dun_bytes = dun_bytes;
350 blk_key->crypto_cfg.data_unit_size = data_unit_size;
351 blk_key->crypto_cfg.key_type = key_type;
352 blk_key->crypto_cfg.flags = flags;
353 blk_key->data_unit_size_bits = ilog2(data_unit_size);
354 blk_key->size = key_size;
355 memcpy(blk_key->bytes, key_bytes, key_size);
356
357 return 0;
358}
359EXPORT_SYMBOL_GPL(blk_crypto_init_key);
360
361/**
362 * blk_crypto_config_supported_natively() - Check whether a block device
363 * supports hardware inline encryption
364 * with the given configuration.
365 * @bdev: the block device
366 * @cfg: the crypto configuration to check for
367 *
368 * Return: %true if @bdev supports hardware inline encryption with @cfg.
369 */
370bool blk_crypto_config_supported_natively(struct block_device *bdev,
371 const struct blk_crypto_config *cfg)
372{
373 struct blk_crypto_profile *profile =
374 bdev_get_queue(bdev)->crypto_profile;
375
376 if (!profile)
377 return false;
378 if (!(cfg->flags & BLK_CRYPTO_CFG_ALLOW_HW))
379 return false;
380 if (!(profile->modes_supported[cfg->crypto_mode] & cfg->data_unit_size))
381 return false;
382 if (profile->max_dun_bytes_supported < cfg->dun_bytes)
383 return false;
384 if (!(profile->key_types_supported & cfg->key_type))
385 return false;
386 return true;
387}
388
389/**
390 * blk_crypto_start_using_key() - Start using a blk_crypto_key on a device
391 * @bdev: block device to operate on
392 * @key: A key to use on the device
393 *
394 * Upper layers must call this function to ensure that either the hardware
395 * supports the key's crypto settings, or the crypto API fallback has transforms
396 * for the needed mode allocated and ready to go. This function may allocate
397 * an skcipher, and *should not* be called from the data path, since that might
398 * cause a deadlock
399 *
400 * Return: 0 on success; -EOPNOTSUPP if the key is wrapped but the hardware does
401 * not support wrapped keys; -ENOPKG if the key is a raw key but the
402 * hardware does not support raw keys and blk-crypto-fallback is either
403 * disabled or the needed algorithm is disabled in the crypto API; or
404 * another -errno code if something else went wrong.
405 */
406int blk_crypto_start_using_key(struct block_device *bdev,
407 const struct blk_crypto_key *key)
408{
409 if (blk_crypto_config_supported_natively(bdev, &key->crypto_cfg))
410 return 0;
411 if (key->crypto_cfg.key_type != BLK_CRYPTO_KEY_TYPE_RAW) {
412 pr_warn_ratelimited("%pg: no support for wrapped keys\n", bdev);
413 return -EOPNOTSUPP;
414 }
415 return blk_crypto_fallback_start_using_mode(key->crypto_cfg.crypto_mode);
416}
417EXPORT_SYMBOL_GPL(blk_crypto_start_using_key);
418
419/**
420 * blk_crypto_evict_key() - Evict a blk_crypto_key from a block_device
421 * @bdev: a block_device on which I/O using the key may have been done
422 * @key: the key to evict
423 *
424 * For a given block_device, this function removes the given blk_crypto_key from
425 * the keyslot management structures and evicts it from any underlying hardware
426 * keyslot(s) or blk-crypto-fallback keyslot it may have been programmed into.
427 *
428 * Upper layers must call this before freeing the blk_crypto_key. It must be
429 * called for every block_device the key may have been used on. The key must no
430 * longer be in use by any I/O when this function is called.
431 *
432 * Context: May sleep.
433 */
434void blk_crypto_evict_key(struct block_device *bdev,
435 const struct blk_crypto_key *key)
436{
437 struct request_queue *q = bdev_get_queue(bdev);
438 int err;
439
440 if (blk_crypto_config_supported_natively(bdev, &key->crypto_cfg))
441 err = __blk_crypto_evict_key(q->crypto_profile, key);
442 else
443 err = blk_crypto_fallback_evict_key(key);
444 /*
445 * An error can only occur here if the key failed to be evicted from a
446 * keyslot (due to a hardware or driver issue) or is allegedly still in
447 * use by I/O (due to a kernel bug). Even in these cases, the key is
448 * still unlinked from the keyslot management structures, and the caller
449 * is allowed and expected to free it right away. There's nothing
450 * callers can do to handle errors, so just log them and return void.
451 */
452 if (err)
453 pr_warn_ratelimited("%pg: error %d evicting key\n", bdev, err);
454}
455EXPORT_SYMBOL_GPL(blk_crypto_evict_key);
456
457static int blk_crypto_ioctl_import_key(struct blk_crypto_profile *profile,
458 void __user *argp)
459{
460 struct blk_crypto_import_key_arg arg;
461 u8 raw_key[BLK_CRYPTO_MAX_RAW_KEY_SIZE];
462 u8 lt_key[BLK_CRYPTO_MAX_HW_WRAPPED_KEY_SIZE];
463 int ret;
464
465 if (copy_from_user(&arg, argp, sizeof(arg)))
466 return -EFAULT;
467
468 if (memchr_inv(arg.reserved, 0, sizeof(arg.reserved)))
469 return -EINVAL;
470
471 if (arg.raw_key_size < 16 || arg.raw_key_size > sizeof(raw_key))
472 return -EINVAL;
473
474 if (copy_from_user(raw_key, u64_to_user_ptr(arg.raw_key_ptr),
475 arg.raw_key_size)) {
476 ret = -EFAULT;
477 goto out;
478 }
479 ret = blk_crypto_import_key(profile, raw_key, arg.raw_key_size, lt_key);
480 if (ret < 0)
481 goto out;
482 if (ret > arg.lt_key_size) {
483 ret = -EOVERFLOW;
484 goto out;
485 }
486 arg.lt_key_size = ret;
487 if (copy_to_user(u64_to_user_ptr(arg.lt_key_ptr), lt_key,
488 arg.lt_key_size) ||
489 copy_to_user(argp, &arg, sizeof(arg))) {
490 ret = -EFAULT;
491 goto out;
492 }
493 ret = 0;
494
495out:
496 memzero_explicit(raw_key, sizeof(raw_key));
497 memzero_explicit(lt_key, sizeof(lt_key));
498 return ret;
499}
500
501static int blk_crypto_ioctl_generate_key(struct blk_crypto_profile *profile,
502 void __user *argp)
503{
504 struct blk_crypto_generate_key_arg arg;
505 u8 lt_key[BLK_CRYPTO_MAX_HW_WRAPPED_KEY_SIZE];
506 int ret;
507
508 if (copy_from_user(&arg, argp, sizeof(arg)))
509 return -EFAULT;
510
511 if (memchr_inv(arg.reserved, 0, sizeof(arg.reserved)))
512 return -EINVAL;
513
514 ret = blk_crypto_generate_key(profile, lt_key);
515 if (ret < 0)
516 goto out;
517 if (ret > arg.lt_key_size) {
518 ret = -EOVERFLOW;
519 goto out;
520 }
521 arg.lt_key_size = ret;
522 if (copy_to_user(u64_to_user_ptr(arg.lt_key_ptr), lt_key,
523 arg.lt_key_size) ||
524 copy_to_user(argp, &arg, sizeof(arg))) {
525 ret = -EFAULT;
526 goto out;
527 }
528 ret = 0;
529
530out:
531 memzero_explicit(lt_key, sizeof(lt_key));
532 return ret;
533}
534
535static int blk_crypto_ioctl_prepare_key(struct blk_crypto_profile *profile,
536 void __user *argp)
537{
538 struct blk_crypto_prepare_key_arg arg;
539 u8 lt_key[BLK_CRYPTO_MAX_HW_WRAPPED_KEY_SIZE];
540 u8 eph_key[BLK_CRYPTO_MAX_HW_WRAPPED_KEY_SIZE];
541 int ret;
542
543 if (copy_from_user(&arg, argp, sizeof(arg)))
544 return -EFAULT;
545
546 if (memchr_inv(arg.reserved, 0, sizeof(arg.reserved)))
547 return -EINVAL;
548
549 if (arg.lt_key_size > sizeof(lt_key))
550 return -EINVAL;
551
552 if (copy_from_user(lt_key, u64_to_user_ptr(arg.lt_key_ptr),
553 arg.lt_key_size)) {
554 ret = -EFAULT;
555 goto out;
556 }
557 ret = blk_crypto_prepare_key(profile, lt_key, arg.lt_key_size, eph_key);
558 if (ret < 0)
559 goto out;
560 if (ret > arg.eph_key_size) {
561 ret = -EOVERFLOW;
562 goto out;
563 }
564 arg.eph_key_size = ret;
565 if (copy_to_user(u64_to_user_ptr(arg.eph_key_ptr), eph_key,
566 arg.eph_key_size) ||
567 copy_to_user(argp, &arg, sizeof(arg))) {
568 ret = -EFAULT;
569 goto out;
570 }
571 ret = 0;
572
573out:
574 memzero_explicit(lt_key, sizeof(lt_key));
575 memzero_explicit(eph_key, sizeof(eph_key));
576 return ret;
577}
578
579int blk_crypto_ioctl(struct block_device *bdev, unsigned int cmd,
580 void __user *argp)
581{
582 struct blk_crypto_profile *profile =
583 bdev_get_queue(bdev)->crypto_profile;
584
585 if (!profile)
586 return -EOPNOTSUPP;
587
588 switch (cmd) {
589 case BLKCRYPTOIMPORTKEY:
590 return blk_crypto_ioctl_import_key(profile, argp);
591 case BLKCRYPTOGENERATEKEY:
592 return blk_crypto_ioctl_generate_key(profile, argp);
593 case BLKCRYPTOPREPAREKEY:
594 return blk_crypto_ioctl_prepare_key(profile, argp);
595 default:
596 return -ENOTTY;
597 }
598}