projects/linux·blob master · refs

620 lines · 15.7 KiB

1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Copyright 2024 Google LLC
4 */
5
6#include <linux/blk-crypto.h>
7#include <linux/ctype.h>
8#include <linux/device-mapper.h>
9#include <linux/hex.h>
10#include <linux/module.h>
11#include <keys/user-type.h>
12
13#define DM_MSG_PREFIX "inlinecrypt"
14
15static const struct dm_inlinecrypt_cipher {
16 const char *name;
17 enum blk_crypto_mode_num mode_num;
18} dm_inlinecrypt_ciphers[] = {
19 {
20 .name = "aes-xts-plain64",
21 .mode_num = BLK_ENCRYPTION_MODE_AES_256_XTS,
22 },
23};
24
25/**
26 * struct inlinecrypt_ctx - private data of an inlinecrypt target
27 * @dev: the underlying device
28 * @start: starting sector of the range of @dev which this target actually maps.
29 * For this purpose a "sector" is 512 bytes.
30 * @cipher_string: the name of the encryption algorithm being used
31 * @key_size: size of the encryption key in bytes
32 * @iv_offset: starting offset for IVs. IVs are generated as if the target were
33 * preceded by @iv_offset 512-byte sectors.
34 * @sector_size: crypto sector size in bytes (usually 4096)
35 * @sector_bits: log2(sector_size)
36 * @key_type: type of the key -- either raw or hardware-wrapped
37 * @key: the encryption key to use
38 * @max_dun: the maximum DUN that may be used (computed from other params)
39 */
40struct inlinecrypt_ctx {
41 struct dm_dev *dev;
42 sector_t start;
43 const char *cipher_string;
44 unsigned int key_size;
45 u64 iv_offset;
46 unsigned int sector_size;
47 unsigned int sector_bits;
48 enum blk_crypto_key_type key_type;
49 struct blk_crypto_key key;
50 u64 max_dun;
51};
52
53static const struct dm_inlinecrypt_cipher *
54lookup_cipher(const char *cipher_string)
55{
56 int i;
57
58 for (i = 0; i < ARRAY_SIZE(dm_inlinecrypt_ciphers); i++) {
59 if (strcmp(cipher_string, dm_inlinecrypt_ciphers[i].name) == 0)
60 return &dm_inlinecrypt_ciphers[i];
61 }
62 return NULL;
63}
64
65static void inlinecrypt_dtr(struct dm_target *ti)
66{
67 struct inlinecrypt_ctx *ctx = ti->private;
68
69 if (ctx->dev) {
70 if (ctx->key.size)
71 blk_crypto_evict_key(ctx->dev->bdev, &ctx->key);
72 dm_put_device(ti, ctx->dev);
73 }
74 kfree_sensitive(ctx->cipher_string);
75 kfree_sensitive(ctx);
76}
77
78#ifdef CONFIG_KEYS
79
80static bool contains_whitespace(const char *str)
81{
82 while (*str)
83 if (isspace(*str++))
84 return true;
85 return false;
86}
87
88static int set_key_user(struct key *key, char *key_bytes,
89 const unsigned int key_bytes_size)
90{
91 const struct user_key_payload *ukp;
92
93 ukp = user_key_payload_locked(key);
94 if (!ukp)
95 return -EKEYREVOKED;
96
97 if (key_bytes_size != ukp->datalen)
98 return -EINVAL;
99
100 memcpy(key_bytes, ukp->data, key_bytes_size);
101
102 return 0;
103}
104
105static int inlinecrypt_get_keyring_key(const char *key_string, u8 *key_bytes,
106 const unsigned int key_bytes_size)
107{
108 char *key_desc;
109 int ret;
110 struct key_type *type;
111 struct key *key;
112 int (*set_key)(struct key *key, char *key_bytes,
113 const unsigned int key_bytes_size);
114
115 /*
116 * Reject key_string with whitespace. dm core currently lacks code for
117 * proper whitespace escaping in arguments on DM_TABLE_STATUS path.
118 */
119 if (contains_whitespace(key_string)) {
120 DMERR("whitespace chars not allowed in key string");
121 return -EINVAL;
122 }
123
124 /* look for next ':' separating key_type from key_description */
125 key_desc = strchr(key_string, ':');
126 if (!key_desc || key_desc == key_string || !strlen(key_desc + 1))
127 return -EINVAL;
128
129 if (!strncmp(key_string, "logon:", key_desc - key_string + 1)) {
130 type = &key_type_logon;
131 set_key = set_key_user;
132 } else {
133 return -EINVAL;
134 }
135
136 key = request_key(type, key_desc + 1, NULL);
137 if (IS_ERR(key))
138 return PTR_ERR(key);
139
140 down_read(&key->sem);
141
142 ret = set_key(key, (char *)key_bytes, key_bytes_size);
143
144 up_read(&key->sem);
145 key_put(key);
146
147 return ret;
148}
149
150static int get_key_size(char **key_string)
151{
152 char *colon, dummy;
153 int ret;
154
155 if (*key_string[0] != ':') {
156 ret = strlen(*key_string);
157
158 if (ret > 2 * BLK_CRYPTO_MAX_ANY_KEY_SIZE
159 || ret % 2
160 || !ret) {
161 DMERR("Invalid keysize");
162 return -EINVAL;
163 }
164 return ret >> 1;
165 }
166
167 /* look for next ':' in key string */
168 colon = strpbrk(*key_string + 1, ":");
169 if (!colon)
170 return -EINVAL;
171
172 if (sscanf(*key_string + 1, "%u%c", &ret, &dummy) != 2 || dummy != ':')
173 return -EINVAL;
174
175 /* remaining key string should be :<logon|user>:<key_desc> */
176 *key_string = colon;
177
178 return ret;
179}
180
181#else
182
183static int inlinecrypt_get_keyring_key(const char *key_string, u8 *key_bytes,
184 const unsigned int key_bytes_size)
185{
186 return -EINVAL;
187}
188
189static int get_key_size(char **key_string)
190{
191 int key_hex_size = strlen(*key_string);
192
193 if (*key_string[0] == ':')
194 return -EINVAL;
195
196 if (key_hex_size > 2 * BLK_CRYPTO_MAX_ANY_KEY_SIZE
197 || key_hex_size % 2
198 || !key_hex_size) {
199 DMERR("Invalid keysize");
200 return -EINVAL;
201 }
202
203 return key_hex_size >> 1;
204}
205
206#endif /* CONFIG_KEYS */
207
208static int inlinecrypt_get_key(const char *key_string,
209 u8 key[BLK_CRYPTO_MAX_ANY_KEY_SIZE],
210 const unsigned int key_size)
211{
212 int ret = 0;
213
214 if (key_size > BLK_CRYPTO_MAX_ANY_KEY_SIZE) {
215 DMERR("Invalid keysize");
216 return -EINVAL;
217 }
218
219 /* ':' means the key is in kernel keyring, short-circuit normal key processing */
220 if (key_string[0] == ':') {
221 /* key string should be :<logon|user>:<key_desc> */
222 ret = inlinecrypt_get_keyring_key(key_string + 1, key, key_size);
223 goto out;
224 }
225
226 if (hex2bin(key, key_string, key_size) != 0)
227 ret = -EINVAL;
228
229out:
230 return ret;
231}
232
233static int inlinecrypt_ctr_optional(struct dm_target *ti,
234 unsigned int argc, char **argv)
235{
236 struct inlinecrypt_ctx *ctx = ti->private;
237 struct dm_arg_set as;
238 static const struct dm_arg _args[] = {
239 {0, 4, "Invalid number of feature args"},
240 };
241 unsigned int opt_params;
242 const char *opt_string;
243 bool iv_large_sectors = false;
244 char dummy;
245 int err;
246
247 as.argc = argc;
248 as.argv = argv;
249
250 err = dm_read_arg_group(_args, &as, &opt_params, &ti->error);
251 if (err)
252 return err;
253
254 while (opt_params--) {
255 opt_string = dm_shift_arg(&as);
256 if (!opt_string) {
257 ti->error = "Not enough feature arguments";
258 return -EINVAL;
259 }
260 if (str_has_prefix(opt_string, "keytype:")) {
261 const char *val = opt_string + strlen("keytype:");
262
263 if (!*val) {
264 ti->error = "Invalid block key type";
265 return -EINVAL;
266 }
267
268 if (!strcmp(val, "raw")) {
269 ctx->key_type = BLK_CRYPTO_KEY_TYPE_RAW;
270 } else if (!strcmp(val, "hw-wrapped")) {
271 ctx->key_type = BLK_CRYPTO_KEY_TYPE_HW_WRAPPED;
272 } else {
273 ti->error = "Invalid block key type";
274 return -EINVAL;
275 }
276 } else if (!strcmp(opt_string, "allow_discards")) {
277 ti->num_discard_bios = 1;
278 } else if (sscanf(opt_string, "sector_size:%u%c",
279 &ctx->sector_size, &dummy) == 1) {
280 if (ctx->sector_size < SECTOR_SIZE ||
281 ctx->sector_size > 4096 ||
282 !is_power_of_2(ctx->sector_size)) {
283 ti->error = "Invalid sector_size";
284 return -EINVAL;
285 }
286 } else if (!strcmp(opt_string, "iv_large_sectors")) {
287 iv_large_sectors = true;
288 } else {
289 ti->error = "Invalid feature arguments";
290 return -EINVAL;
291 }
292 }
293
294 /* dm-inlinecrypt doesn't implement iv_large_sectors=false. */
295 if (ctx->sector_size != SECTOR_SIZE && !iv_large_sectors) {
296 ti->error = "iv_large_sectors must be specified";
297 return -EINVAL;
298 }
299
300 return 0;
301}
302
303/*
304 * Construct an inlinecrypt mapping:
305 * <cipher> [<key>|:<key_size>:<logon>:<key_description>] <iv_offset> <dev_path> <start>
306 *
307 * This syntax matches dm-crypt's, but the set of supported functionality has
308 * been stripped down.
309 */
310static int inlinecrypt_ctr(struct dm_target *ti, unsigned int argc, char **argv)
311{
312 struct inlinecrypt_ctx *ctx;
313 const struct dm_inlinecrypt_cipher *cipher;
314 u8 key_bytes[BLK_CRYPTO_MAX_ANY_KEY_SIZE];
315 unsigned int dun_bytes;
316 unsigned long long tmpll;
317 char dummy;
318 int err;
319
320 if (argc < 5) {
321 ti->error = "Not enough arguments";
322 return -EINVAL;
323 }
324
325 ctx = kzalloc_obj(*ctx);
326 if (!ctx) {
327 ti->error = "Out of memory";
328 return -ENOMEM;
329 }
330 ti->private = ctx;
331
332 /* <cipher> */
333 ctx->cipher_string = kstrdup(argv[0], GFP_KERNEL);
334 if (!ctx->cipher_string) {
335 ti->error = "Out of memory";
336 err = -ENOMEM;
337 goto bad;
338 }
339 cipher = lookup_cipher(ctx->cipher_string);
340 if (!cipher) {
341 ti->error = "Unsupported cipher";
342 err = -EINVAL;
343 goto bad;
344 }
345
346 /* <key> */
347 err = get_key_size(&argv[1]);
348 if (err < 0) {
349 ti->error = "Cannot parse key size";
350 goto bad;
351 }
352 ctx->key_size = err;
353
354 err = inlinecrypt_get_key(argv[1], key_bytes, ctx->key_size);
355 if (err) {
356 ti->error = "Malformed key string";
357 goto bad;
358 }
359
360 /* <iv_offset> */
361 if (sscanf(argv[2], "%llu%c", &ctx->iv_offset, &dummy) != 1) {
362 ti->error = "Invalid iv_offset sector";
363 err = -EINVAL;
364 goto bad;
365 }
366
367 /* <dev_path> */
368 err = dm_get_device(ti, argv[3], dm_table_get_mode(ti->table),
369 &ctx->dev);
370 if (err) {
371 ti->error = "Device lookup failed";
372 goto bad;
373 }
374
375 /* <start> */
376 if (sscanf(argv[4], "%llu%c", &tmpll, &dummy) != 1 ||
377 tmpll != (sector_t)tmpll) {
378 ti->error = "Invalid start sector";
379 err = -EINVAL;
380 goto bad;
381 }
382 ctx->start = tmpll;
383
384 /* optional arguments */
385 ctx->sector_size = SECTOR_SIZE;
386 ctx->key_type = BLK_CRYPTO_KEY_TYPE_RAW;
387 if (argc > 5) {
388 err = inlinecrypt_ctr_optional(ti, argc - 5, &argv[5]);
389 if (err)
390 goto bad;
391 }
392 ctx->sector_bits = ilog2(ctx->sector_size);
393 if (ti->len & ((ctx->sector_size >> SECTOR_SHIFT) - 1)) {
394 ti->error = "Device size is not a multiple of sector_size";
395 err = -EINVAL;
396 goto bad;
397 }
398 if (ctx->iv_offset & ((ctx->sector_size >> SECTOR_SHIFT) - 1)) {
399 ti->error = "Wrong alignment of iv_offset sector";
400 err = -EINVAL;
401 goto bad;
402 }
403
404 ctx->max_dun = (ctx->iv_offset + ti->len - 1) >>
405 (ctx->sector_bits - SECTOR_SHIFT);
406 dun_bytes = DIV_ROUND_UP(fls64(ctx->max_dun), 8);
407
408 err = blk_crypto_init_key(&ctx->key, key_bytes, ctx->key_size,
409 ctx->key_type, cipher->mode_num,
410 dun_bytes, ctx->sector_size,
411 BLK_CRYPTO_CFG_ALLOW_HW);
412 if (err) {
413 ti->error = "Error initializing blk-crypto key";
414 goto bad;
415 }
416
417 err = blk_crypto_start_using_key(ctx->dev->bdev, &ctx->key);
418 if (err) {
419 ti->error = "Error starting to use blk-crypto";
420 goto bad;
421 }
422
423 ti->num_flush_bios = 1;
424
425 err = 0;
426 goto out;
427
428bad:
429 inlinecrypt_dtr(ti);
430out:
431 memzero_explicit(key_bytes, sizeof(key_bytes));
432 return err;
433}
434
435static int inlinecrypt_map(struct dm_target *ti, struct bio *bio)
436{
437 const struct inlinecrypt_ctx *ctx = ti->private;
438 sector_t sector_in_target;
439 u64 dun[BLK_CRYPTO_DUN_ARRAY_SIZE] = {};
440
441 bio_set_dev(bio, ctx->dev->bdev);
442
443 /*
444 * If the bio is a device-level request which doesn't target a specific
445 * sector, there's nothing more to do.
446 */
447 if (bio_sectors(bio) == 0)
448 return DM_MAPIO_REMAPPED;
449
450 /*
451 * The bio should never have an encryption context already, since
452 * dm-inlinecrypt doesn't pass through any inline encryption
453 * capabilities to the layer above it.
454 */
455 if (WARN_ON_ONCE(bio_has_crypt_ctx(bio)))
456 return DM_MAPIO_KILL;
457
458 /* Map the bio's sector to the underlying device. (512-byte sectors) */
459 sector_in_target = dm_target_offset(ti, bio->bi_iter.bi_sector);
460 bio->bi_iter.bi_sector = ctx->start + sector_in_target;
461 /*
462 * If the bio doesn't have any data (e.g. if it's a DISCARD request),
463 * there's nothing more to do.
464 */
465 if (!bio_has_data(bio))
466 return DM_MAPIO_REMAPPED;
467
468 /* Calculate the DUN and enforce data-unit (crypto sector) alignment. */
469 dun[0] = ctx->iv_offset + sector_in_target; /* 512-byte sectors */
470 if (dun[0] & ((ctx->sector_size >> SECTOR_SHIFT) - 1))
471 return DM_MAPIO_KILL;
472 dun[0] >>= ctx->sector_bits - SECTOR_SHIFT; /* crypto sectors */
473
474 /*
475 * This check isn't necessary as we should have calculated max_dun
476 * correctly, but be safe.
477 */
478 if (WARN_ON_ONCE(dun[0] > ctx->max_dun))
479 return DM_MAPIO_KILL;
480
481 bio_crypt_set_ctx(bio, &ctx->key, dun, GFP_NOIO);
482
483 /*
484 * Since we've added an encryption context to the bio and
485 * blk-crypto-fallback may be needed to process it, it's necessary to
486 * use the fallback-aware bio submission code rather than
487 * unconditionally returning DM_MAPIO_REMAPPED.
488 *
489 * To get the correct accounting for a dm target in the case where
490 * __blk_crypto_submit_bio() doesn't take ownership of the bio (returns
491 * true), call __blk_crypto_submit_bio() directly and return
492 * DM_MAPIO_REMAPPED in that case, rather than relying on
493 * blk_crypto_submit_bio() which calls submit_bio() in that case.
494 *
495 * TODO: blk-crypto fallback write slow-path currently double-accounts
496 * IO in vmstat, as encrypted bios are submitted via submit_bio().
497 * This does not affect data correctness. Consider fixing this if
498 * a cleaner accounting model for derived bios is introduced.
499 */
500 if (__blk_crypto_submit_bio(bio))
501 return DM_MAPIO_REMAPPED;
502 return DM_MAPIO_SUBMITTED;
503}
504
505static void inlinecrypt_status(struct dm_target *ti, status_type_t type,
506 unsigned int status_flags, char *result,
507 unsigned int maxlen)
508{
509 const struct inlinecrypt_ctx *ctx = ti->private;
510 unsigned int sz = 0;
511 int num_feature_args = 0;
512
513 switch (type) {
514 case STATUSTYPE_INFO:
515 case STATUSTYPE_IMA:
516 result[0] = '\0';
517 break;
518
519 case STATUSTYPE_TABLE:
520 /*
521 * Warning: like dm-crypt, dm-inlinecrypt includes the key in
522 * the returned table. Userspace is responsible for redacting
523 * the key when needed.
524 */
525 DMEMIT("%s %*phN %u %llu %s %llu", ctx->cipher_string,
526 ctx->key.size, ctx->key.bytes,
527 ctx->key_type, ctx->iv_offset,
528 ctx->dev->name, ctx->start);
529 num_feature_args += !!ti->num_discard_bios;
530 if (ctx->sector_size != SECTOR_SIZE)
531 num_feature_args += 2;
532 if (num_feature_args != 0) {
533 DMEMIT(" %d", num_feature_args);
534 if (ti->num_discard_bios)
535 DMEMIT(" allow_discards");
536 if (ctx->sector_size != SECTOR_SIZE) {
537 DMEMIT(" sector_size:%u", ctx->sector_size);
538 DMEMIT(" iv_large_sectors");
539 }
540 }
541 break;
542 }
543}
544
545static int inlinecrypt_prepare_ioctl(struct dm_target *ti,
546 struct block_device **bdev, unsigned int cmd,
547 unsigned long arg, bool *forward)
548{
549 const struct inlinecrypt_ctx *ctx = ti->private;
550 const struct dm_dev *dev = ctx->dev;
551
552 *bdev = dev->bdev;
553
554 /* Only pass ioctls through if the device sizes match exactly. */
555 return ctx->start != 0 || ti->len != bdev_nr_sectors(dev->bdev);
556}
557
558static int inlinecrypt_iterate_devices(struct dm_target *ti,
559 iterate_devices_callout_fn fn,
560 void *data)
561{
562 const struct inlinecrypt_ctx *ctx = ti->private;
563
564 return fn(ti, ctx->dev, ctx->start, ti->len, data);
565}
566
567#ifdef CONFIG_BLK_DEV_ZONED
568static int inlinecrypt_report_zones(struct dm_target *ti,
569 struct dm_report_zones_args *args,
570 unsigned int nr_zones)
571{
572 const struct inlinecrypt_ctx *ctx = ti->private;
573
574 return dm_report_zones(ctx->dev->bdev, ctx->start,
575 ctx->start + dm_target_offset(ti, args->next_sector),
576 args, nr_zones);
577}
578#else
579#define inlinecrypt_report_zones NULL
580#endif
581
582static void inlinecrypt_io_hints(struct dm_target *ti,
583 struct queue_limits *limits)
584{
585 const struct inlinecrypt_ctx *ctx = ti->private;
586 const unsigned int sector_size = ctx->sector_size;
587
588 limits->logical_block_size =
589 max_t(unsigned int, limits->logical_block_size, sector_size);
590 limits->physical_block_size =
591 max_t(unsigned int, limits->physical_block_size, sector_size);
592 limits->io_min = max_t(unsigned int, limits->io_min, sector_size);
593 limits->dma_alignment = limits->logical_block_size - 1;
594}
595
596static struct target_type inlinecrypt_target = {
597 .name = "inlinecrypt",
598 .version = {1, 0, 0},
599 /*
600 * Do not set DM_TARGET_PASSES_CRYPTO, since dm-inlinecrypt consumes the
601 * crypto capability itself.
602 */
603 .features = DM_TARGET_ZONED_HM,
604 .module = THIS_MODULE,
605 .ctr = inlinecrypt_ctr,
606 .dtr = inlinecrypt_dtr,
607 .map = inlinecrypt_map,
608 .status = inlinecrypt_status,
609 .prepare_ioctl = inlinecrypt_prepare_ioctl,
610 .iterate_devices = inlinecrypt_iterate_devices,
611 .report_zones = inlinecrypt_report_zones,
612 .io_hints = inlinecrypt_io_hints,
613};
614
615module_dm(inlinecrypt);
616
617MODULE_AUTHOR("Eric Biggers <[email protected]>");
618MODULE_AUTHOR("Linlin Zhang <[email protected]>");
619MODULE_DESCRIPTION(DM_NAME " target for inline encryption");
620MODULE_LICENSE("GPL");