projects/linux·blob master · refs

415 lines · 12.9 KiB

1// SPDX-License-Identifier: GPL-2.0
2/*
3 * File contents en/decryption on block-based filesystems
4 *
5 * Copyright 2019 Google LLC
6 */
7
8/*
9 * This file implements fscrypt's file contents en/decryption using blk-crypto
10 * (Documentation/block/inline-encryption.rst). fscrypt assigns a bio_crypt_ctx
11 * with a key and IV to each bio, and the block layer does the en/decryption.
12 *
13 * This file's exported functions are called only by block-based filesystems.
14 */
15
16#include <linux/blk-crypto.h>
17#include <linux/blkdev.h>
18#include <linux/export.h>
19#include <linux/sched/mm.h>
20#include <linux/slab.h>
21#include <linux/uio.h>
22
23#include "fscrypt_private.h"
24
25static unsigned int
26fscrypt_get_devices(struct super_block *sb,
27 struct block_device *devs[FSCRYPT_MAX_DEVICES])
28{
29 if (sb->s_cop->get_devices)
30 return sb->s_cop->get_devices(sb, devs);
31 devs[0] = sb->s_bdev;
32 return 1;
33}
34
35static unsigned int fscrypt_get_dun_bytes(const struct fscrypt_inode_info *ci)
36{
37 const struct super_block *sb = ci->ci_inode->i_sb;
38 unsigned int flags = fscrypt_policy_flags(&ci->ci_policy);
39 int dun_bits;
40
41 if (flags & FSCRYPT_POLICY_FLAG_DIRECT_KEY)
42 return offsetofend(union fscrypt_iv, nonce);
43
44 if (flags & FSCRYPT_POLICY_FLAG_IV_INO_LBLK_64)
45 return sizeof(__le64);
46
47 if (flags & FSCRYPT_POLICY_FLAG_IV_INO_LBLK_32)
48 return sizeof(__le32);
49
50 /* Default case: IVs are just the file data unit index */
51 dun_bits = fscrypt_max_file_dun_bits(sb, ci->ci_data_unit_bits);
52 return DIV_ROUND_UP(dun_bits, 8);
53}
54
55/*
56 * Log a message when starting to use blk-crypto (native) or blk-crypto-fallback
57 * for an encryption mode for the first time. This is the blk-crypto
58 * counterpart to the message logged when starting to use the crypto API for the
59 * first time. A limitation is that these messages don't convey which specific
60 * filesystems or files are using each implementation. However, *usually*
61 * systems use just one implementation per mode, which makes these messages
62 * helpful for debugging problems where the "wrong" implementation is used.
63 */
64static void fscrypt_log_blk_crypto_impl(struct fscrypt_mode *mode,
65 struct block_device *dev,
66 const struct blk_crypto_key *blk_key)
67{
68 if (blk_crypto_config_supported_natively(dev, &blk_key->crypto_cfg)) {
69 if (!xchg(&mode->logged_blk_crypto_native, 1))
70 pr_info("fscrypt: %s using blk-crypto (native)\n",
71 mode->friendly_name);
72 } else if (!xchg(&mode->logged_blk_crypto_fallback, 1)) {
73 pr_info("fscrypt: %s using blk-crypto-fallback\n",
74 mode->friendly_name);
75 }
76}
77
78int fscrypt_prepare_inline_crypt_key(struct fscrypt_prepared_key *prep_key,
79 const u8 *key_bytes, size_t key_size,
80 bool is_hw_wrapped,
81 const struct fscrypt_inode_info *ci)
82{
83 const struct inode *inode = ci->ci_inode;
84 struct super_block *sb = inode->i_sb;
85 bool inlinecrypt = sb->s_flags & SB_INLINECRYPT;
86 struct fscrypt_mode *mode = ci->ci_mode;
87 enum blk_crypto_key_type key_type = is_hw_wrapped ?
88 BLK_CRYPTO_KEY_TYPE_HW_WRAPPED : BLK_CRYPTO_KEY_TYPE_RAW;
89 struct blk_crypto_key *blk_key;
90 struct block_device *devs[FSCRYPT_MAX_DEVICES];
91 unsigned int num_devs;
92 unsigned int i;
93 int err;
94
95 if (is_hw_wrapped && !inlinecrypt) {
96 /*
97 * blk_crypto_init_key() would catch this anyway, but this
98 * provides a clearer error message.
99 */
100 fscrypt_err(
101 inode,
102 "Hardware-wrapped keys require inline encryption (-o inlinecrypt)");
103 return -EINVAL;
104 }
105
106 blk_key = kmalloc_obj(*blk_key);
107 if (!blk_key)
108 return -ENOMEM;
109
110 err = blk_crypto_init_key(blk_key, key_bytes, key_size, key_type,
111 mode->blk_crypto_mode,
112 fscrypt_get_dun_bytes(ci),
113 1U << ci->ci_data_unit_bits,
114 inlinecrypt ? BLK_CRYPTO_CFG_ALLOW_HW : 0);
115 if (err) {
116 fscrypt_err(inode, "Error %d initializing blk-crypto key", err);
117 goto fail;
118 }
119
120 /* Start using blk-crypto on all the filesystem's block devices. */
121 num_devs = fscrypt_get_devices(sb, devs);
122 for (i = 0; i < num_devs; i++) {
123 err = blk_crypto_start_using_key(devs[i], blk_key);
124 if (err)
125 break;
126 fscrypt_log_blk_crypto_impl(mode, devs[i], blk_key);
127 }
128 if (err) {
129 if (err == -EOPNOTSUPP && is_hw_wrapped)
130 fscrypt_err(
131 inode,
132 "Hardware-wrapped key required, but no suitable inline encryption capabilities are available");
133 else
134 fscrypt_err(inode,
135 "Error %d starting to use blk-crypto", err);
136 goto fail;
137 }
138
139 prep_key->blk_key = blk_key;
140 return 0;
141
142fail:
143 kfree_sensitive(blk_key);
144 return err;
145}
146
147void fscrypt_destroy_inline_crypt_key(struct super_block *sb,
148 struct fscrypt_prepared_key *prep_key)
149{
150 struct blk_crypto_key *blk_key = prep_key->blk_key;
151 struct block_device *devs[FSCRYPT_MAX_DEVICES];
152 unsigned int num_devs;
153 unsigned int i;
154
155 if (!blk_key)
156 return;
157
158 /*
159 * Evict the key from all the filesystem's block devices.
160 * This *must* be done before the key is freed.
161 */
162 num_devs = fscrypt_get_devices(sb, devs);
163 for (i = 0; i < num_devs; i++)
164 blk_crypto_evict_key(devs[i], blk_key);
165
166 kfree_sensitive(blk_key);
167}
168
169/*
170 * Ask the inline encryption hardware to derive the software secret from a
171 * hardware-wrapped key. Returns -EOPNOTSUPP if hardware-wrapped keys aren't
172 * supported on this filesystem or hardware.
173 */
174int fscrypt_derive_sw_secret(struct super_block *sb,
175 const u8 *wrapped_key, size_t wrapped_key_size,
176 u8 sw_secret[BLK_CRYPTO_SW_SECRET_SIZE])
177{
178 int err;
179
180 /* The filesystem must be mounted with -o inlinecrypt. */
181 if (!(sb->s_flags & SB_INLINECRYPT)) {
182 fscrypt_warn(NULL,
183 "%s: filesystem not mounted with inlinecrypt\n",
184 sb->s_id);
185 return -EOPNOTSUPP;
186 }
187
188 err = blk_crypto_derive_sw_secret(sb->s_bdev, wrapped_key,
189 wrapped_key_size, sw_secret);
190 if (err == -EOPNOTSUPP)
191 fscrypt_warn(NULL,
192 "%s: block device doesn't support hardware-wrapped keys\n",
193 sb->s_id);
194 return err;
195}
196
197static void fscrypt_generate_dun(const struct fscrypt_inode_info *ci,
198 loff_t pos, u64 dun[BLK_CRYPTO_DUN_ARRAY_SIZE])
199{
200 union fscrypt_iv iv;
201 int i;
202
203 fscrypt_generate_iv(&iv, pos >> ci->ci_data_unit_bits, ci);
204
205 BUILD_BUG_ON(FSCRYPT_MAX_IV_SIZE > BLK_CRYPTO_MAX_IV_SIZE);
206 memset(dun, 0, BLK_CRYPTO_MAX_IV_SIZE);
207 for (i = 0; i < ci->ci_mode->ivsize/sizeof(dun[0]); i++)
208 dun[i] = le64_to_cpu(iv.dun[i]);
209}
210
211/**
212 * fscrypt_set_bio_crypt_ctx() - prepare a file contents bio for inline crypto
213 * @bio: a bio which will eventually be submitted to the file
214 * @inode: the file's inode
215 * @pos: the first file position (in bytes) in the I/O
216 * @gfp_mask: memory allocation flags - these must be a waiting mask so that
217 * bio_crypt_set_ctx can't fail.
218 *
219 * If the contents of the file should be encrypted (or decrypted), then assign
220 * the appropriate encryption context to the bio.
221 *
222 * Normally the bio should be newly allocated (i.e. no pages added yet), as
223 * otherwise fscrypt_mergeable_bio() won't work as intended.
224 *
225 * The encryption context will be freed automatically when the bio is freed.
226 */
227void fscrypt_set_bio_crypt_ctx(struct bio *bio, const struct inode *inode,
228 loff_t pos, gfp_t gfp_mask)
229{
230 const struct fscrypt_inode_info *ci;
231 u64 dun[BLK_CRYPTO_DUN_ARRAY_SIZE];
232
233 if (!fscrypt_needs_contents_encryption(inode))
234 return;
235 ci = fscrypt_get_inode_info_raw(inode);
236
237 fscrypt_generate_dun(ci, pos, dun);
238 bio_crypt_set_ctx(bio, ci->ci_enc_key.blk_key, dun, gfp_mask);
239}
240EXPORT_SYMBOL_GPL(fscrypt_set_bio_crypt_ctx);
241
242/**
243 * fscrypt_mergeable_bio() - test whether data can be added to a bio
244 * @bio: the bio being built up
245 * @inode: the inode for the next part of the I/O
246 * @pos: the next file position (in bytes) in the I/O
247 *
248 * When building a bio which may contain data which should undergo encryption
249 * (or decryption) via fscrypt, filesystems should call this function to ensure
250 * that the resulting bio contains only contiguous data unit numbers. This will
251 * return false if the next part of the I/O cannot be merged with the bio
252 * because either the encryption key would be different or the encryption data
253 * unit numbers would be discontiguous.
254 *
255 * fscrypt_set_bio_crypt_ctx() must have already been called on the bio.
256 *
257 * This function isn't required in cases where crypto-mergeability is ensured in
258 * another way, such as I/O targeting only a single file (and thus a single key)
259 * combined with fscrypt_limit_io_blocks() to ensure DUN contiguity.
260 *
261 * Return: true iff the I/O is mergeable
262 */
263bool fscrypt_mergeable_bio(struct bio *bio, const struct inode *inode,
264 loff_t pos)
265{
266 const struct bio_crypt_ctx *bc = bio->bi_crypt_context;
267 const struct fscrypt_inode_info *ci;
268 u64 next_dun[BLK_CRYPTO_DUN_ARRAY_SIZE];
269
270 if (!!bc != fscrypt_needs_contents_encryption(inode))
271 return false;
272 if (!bc)
273 return true;
274 ci = fscrypt_get_inode_info_raw(inode);
275
276 /*
277 * Comparing the key pointers is good enough, as all I/O for each key
278 * uses the same pointer. I.e., there's currently no need to support
279 * merging requests where the keys are the same but the pointers differ.
280 */
281 if (bc->bc_key != ci->ci_enc_key.blk_key)
282 return false;
283
284 fscrypt_generate_dun(ci, pos, next_dun);
285 return bio_crypt_dun_is_contiguous(bc, bio->bi_iter.bi_size, next_dun);
286}
287EXPORT_SYMBOL_GPL(fscrypt_mergeable_bio);
288
289/**
290 * fscrypt_limit_io_blocks() - limit I/O blocks to avoid discontiguous DUNs
291 * @inode: the file on which I/O is being done
292 * @lblk: the block at which the I/O is being started from
293 * @nr_blocks: the number of blocks we want to submit starting at @lblk
294 *
295 * Determine the limit to the number of blocks that can be submitted in a bio
296 * targeting @lblk without causing a data unit number (DUN) discontiguity.
297 *
298 * This is normally just @nr_blocks, as normally the DUNs just increment along
299 * with the logical blocks. (Or the file is not encrypted.)
300 *
301 * In rare cases, fscrypt can be using an IV generation method that allows the
302 * DUN to wrap around within logically contiguous blocks, and that wraparound
303 * will occur. If this happens, a value less than @nr_blocks will be returned
304 * so that the wraparound doesn't occur in the middle of a bio, which would
305 * cause encryption/decryption to produce wrong results.
306 *
307 * Return: the actual number of blocks that can be submitted
308 */
309u64 fscrypt_limit_io_blocks(const struct inode *inode, u64 lblk, u64 nr_blocks)
310{
311 const struct fscrypt_inode_info *ci;
312 u32 dun;
313
314 if (!fscrypt_needs_contents_encryption(inode))
315 return nr_blocks;
316
317 if (nr_blocks <= 1)
318 return nr_blocks;
319
320 ci = fscrypt_get_inode_info_raw(inode);
321 if (!(fscrypt_policy_flags(&ci->ci_policy) &
322 FSCRYPT_POLICY_FLAG_IV_INO_LBLK_32))
323 return nr_blocks;
324
325 /* With IV_INO_LBLK_32, the DUN can wrap around from U32_MAX to 0. */
326
327 dun = ci->ci_hashed_ino + lblk;
328
329 return min_t(u64, nr_blocks, (u64)U32_MAX + 1 - dun);
330}
331EXPORT_SYMBOL_GPL(fscrypt_limit_io_blocks);
332
333struct fscrypt_zero_done {
334 atomic_t pending;
335 blk_status_t status;
336 struct completion done;
337};
338
339static void fscrypt_zeroout_range_done(struct fscrypt_zero_done *done)
340{
341 if (atomic_dec_and_test(&done->pending))
342 complete(&done->done);
343}
344
345static void fscrypt_zeroout_range_end_io(struct bio *bio)
346{
347 struct fscrypt_zero_done *done = bio->bi_private;
348
349 if (bio->bi_status)
350 cmpxchg(&done->status, 0, bio->bi_status);
351 fscrypt_zeroout_range_done(done);
352 bio_put(bio);
353}
354
355/**
356 * fscrypt_zeroout_range() - zero out a range of blocks in an encrypted file
357 * @inode: the file's inode
358 * @pos: the first file position (in bytes) to zero out
359 * @sector: the first sector to zero out
360 * @len: bytes to zero out
361 *
362 * Zero out filesystem blocks in an encrypted regular file on-disk, i.e. write
363 * ciphertext blocks which decrypt to the all-zeroes block. The blocks must be
364 * both logically and physically contiguous. It's also assumed that the
365 * filesystem only uses a single block device, ->s_bdev. @len must be a
366 * multiple of the file system logical block size.
367 *
368 * Note that since each block uses a different IV, this involves writing a
369 * different ciphertext to each block; we can't simply reuse the same one.
370 *
371 * Return: 0 on success; -errno on failure.
372 */
373int fscrypt_zeroout_range(const struct inode *inode, loff_t pos,
374 sector_t sector, u64 len)
375{
376 struct fscrypt_zero_done done = {
377 .pending = ATOMIC_INIT(1),
378 .done = COMPLETION_INITIALIZER_ONSTACK(done.done),
379 };
380
381 if (len == 0)
382 return 0;
383
384 do {
385 struct bio *bio;
386 unsigned int n;
387
388 bio = bio_alloc(inode->i_sb->s_bdev, BIO_MAX_VECS, REQ_OP_WRITE,
389 GFP_NOFS);
390 bio->bi_iter.bi_sector = sector;
391 bio->bi_private = &done;
392 bio->bi_end_io = fscrypt_zeroout_range_end_io;
393 fscrypt_set_bio_crypt_ctx(bio, inode, pos, GFP_NOFS);
394
395 for (n = 0; n < BIO_MAX_VECS; n++) {
396 unsigned int bytes_this_page = min(len, PAGE_SIZE);
397
398 __bio_add_page(bio, ZERO_PAGE(0), bytes_this_page, 0);
399 len -= bytes_this_page;
400 pos += bytes_this_page;
401 sector += (bytes_this_page >> SECTOR_SHIFT);
402 if (!len || !fscrypt_mergeable_bio(bio, inode, pos))
403 break;
404 }
405
406 atomic_inc(&done.pending);
407 blk_crypto_submit_bio(bio);
408 } while (len);
409
410 fscrypt_zeroout_range_done(&done);
411
412 wait_for_completion(&done.done);
413 return blk_status_to_errno(done.status);
414}
415EXPORT_SYMBOL(fscrypt_zeroout_range);