diff --git a/crypto/aes.c b/crypto/aes.c
index 6bf23eb..94791f4 100644
--- a/crypto/aes.c
+++ b/crypto/aes.c
@@ -6,12 +6,22 @@
*/
#include <crypto/aes-cbc-macs.h>
+#include <crypto/aes-cbc.h>
+#include <crypto/aes-ccm.h>
+#include <crypto/aes-ctr.h>
+#include <crypto/aes-ecb.h>
+#include <crypto/aes-gcm.h>
+#include <crypto/aes-xts.h>
#include <crypto/aes.h>
#include <crypto/algapi.h>
+#include <crypto/internal/aead.h>
#include <crypto/internal/hash.h>
+#include <crypto/internal/skcipher.h>
+#include <crypto/scatterwalk.h>
#include <linux/module.h>
static_assert(__alignof__(struct aes_key) <= CRYPTO_MINALIGN);
+static_assert(__alignof__(struct aes_enckey) <= CRYPTO_MINALIGN);
static int crypto_aes_setkey(struct crypto_tfm *tfm, const u8 *in_key,
unsigned int key_len)
@@ -85,7 +95,6 @@ static int __maybe_unused crypto_aes_cmac_digest(struct shash_desc *desc,
return 0;
}
-static_assert(__alignof__(struct aes_enckey) <= CRYPTO_MINALIGN);
#define AES_CBCMAC_KEY(tfm) ((struct aes_enckey *)crypto_shash_ctx(tfm))
#define AES_CBCMAC_CTX(desc) ((struct aes_cbcmac_ctx *)shash_desc_ctx(desc))
@@ -200,6 +209,828 @@ static struct shash_alg mac_algs[] = {
#endif
};
+static __maybe_unused int
+crypto_aes_skcipher_setkey(struct crypto_skcipher *tfm, const u8 *in_key,
+ unsigned int key_len)
+{
+ struct aes_key *key = crypto_skcipher_ctx(tfm);
+
+ return aes_preparekey(key, in_key, key_len);
+}
+
+static __maybe_unused int
+crypto_aes_skcipher_setenckey(struct crypto_skcipher *tfm, const u8 *in_key,
+ unsigned int key_len)
+{
+ struct aes_enckey *key = crypto_skcipher_ctx(tfm);
+
+ return aes_prepareenckey(key, in_key, key_len);
+}
+
+/*
+ * Return true if the request uses only a single scatterlist element and high
+ * memory isn't enabled. This assumes that both scatterlists are non-NULL, i.e.
+ * the caller must have handled the cryptlen == 0 case already.
+ */
+static inline bool
+skcipher_request_is_linear_lowmem(const struct skcipher_request *req)
+{
+ return !IS_ENABLED(CONFIG_HIGHMEM) &&
+ req->dst->length >= req->cryptlen &&
+ req->src->length >= req->cryptlen;
+}
+
+/*
+ * Call crypt_func() (a function that operates on simple virtual addresses) zero
+ * or more times to en/decrypt 'cryptlen' bytes of data from the source
+ * scatterlist 'src' and write it into the destination scatterlist 'dst',
+ * starting at 'start_pos' bytes into both.
+ *
+ * This always calls crypt_func() with a length that's a multiple of
+ * AES_BLOCK_SIZE, except the last call which includes any remainder. This is
+ * implemented by using an on-stack bounce buffer when necessary. The current
+ * implementation also tries to prefer passing at least 4 blocks, so e.g.
+ * scatterlist entries [16,16,16,16] result in a single 64-byte call.
+ *
+ * The scatterlists must describe either entirely different memory
+ * (out-of-place) or entirely the same memory (in-place). In the latter case,
+ * crypt_func() is always called with the source and dest pointers the same.
+ */
+#define AES_CRYPT_SG(crypt_func, dst, src, cryptlen, start_pos, ...) \
+ ({ \
+ unsigned int remaining = (cryptlen); \
+ unsigned int spos = (start_pos); \
+ \
+ if (remaining != 0) { \
+ struct scatter_walk dst_walk, src_walk; \
+ u8 tmp[4 * AES_BLOCK_SIZE] __aligned( \
+ __alignof__(long)); \
+ \
+ scatterwalk_start_at_pos(&dst_walk, (dst), spos); \
+ scatterwalk_start_at_pos(&src_walk, (src), spos); \
+ do { \
+ unsigned int dst_avail = scatterwalk_clamp( \
+ &dst_walk, remaining); \
+ unsigned int src_avail = scatterwalk_clamp( \
+ &src_walk, remaining); \
+ unsigned int n = min(dst_avail, src_avail); \
+ u8 *dst_virt; \
+ const u8 *src_virt; \
+ \
+ if (n < remaining) { \
+ if (n < sizeof(tmp)) { \
+ n = min(remaining, \
+ sizeof(tmp)); \
+ memcpy_from_scatterwalk( \
+ tmp, &src_walk, n); \
+ crypt_func(tmp, tmp, n, \
+ ##__VA_ARGS__); \
+ memcpy_to_scatterwalk( \
+ &dst_walk, tmp, n); \
+ remaining -= n; \
+ continue; \
+ } \
+ n = round_down(n, AES_BLOCK_SIZE); \
+ } \
+ \
+ scatterwalk_map(&dst_walk); \
+ dst_virt = dst_walk.addr; \
+ if (IS_ENABLED(CONFIG_HIGHMEM) && \
+ offset_in_page(src_walk.offset) == \
+ offset_in_page(dst_walk.offset) && \
+ sg_page(src_walk.sg) + (src_walk.offset / \
+ PAGE_SIZE) == \
+ sg_page(dst_walk.sg) + \
+ (dst_walk.offset / \
+ PAGE_SIZE)) { \
+ src_virt = dst_virt; \
+ } else { \
+ scatterwalk_map(&src_walk); \
+ src_virt = src_walk.addr; \
+ } \
+ crypt_func(dst_virt, src_virt, n, \
+ ##__VA_ARGS__); \
+ if (src_virt != dst_virt) \
+ scatterwalk_unmap(&src_walk); \
+ scatterwalk_advance(&src_walk, n); \
+ scatterwalk_done_dst(&dst_walk, n); \
+ remaining -= n; \
+ } while (remaining); \
+ memzero_explicit(tmp, sizeof(tmp)); \
+ } \
+ })
+
+/*
+ * Call ad_func() as needed to process the associated data in the first
+ * 'assoclen' bytes of the scatterlist 'src'.
+ */
+#define AES_PROCESS_ASSOC_DATA(ad_func, src, assoclen, ctx) \
+ ({ \
+ unsigned int remaining = (assoclen); \
+ \
+ if (remaining != 0) { \
+ struct scatter_walk walk; \
+ \
+ scatterwalk_start(&walk, (src)); \
+ do { \
+ unsigned int n = \
+ scatterwalk_next(&walk, remaining); \
+ \
+ ad_func((ctx), walk.addr, n); \
+ scatterwalk_done_src(&walk, n); \
+ remaining -= n; \
+ } while (remaining); \
+ } \
+ })
+
+/* AES-ECB */
+
+static __maybe_unused int crypto_aes_ecb_encrypt(struct skcipher_request *req)
+{
+ const struct aes_key *key =
+ crypto_skcipher_ctx(crypto_skcipher_reqtfm(req));
+
+ if (unlikely(req->cryptlen % AES_BLOCK_SIZE))
+ return -EINVAL;
+ AES_CRYPT_SG(aes_ecb_encrypt, req->dst, req->src, req->cryptlen, 0,
+ key);
+ return 0;
+}
+
+static __maybe_unused int crypto_aes_ecb_decrypt(struct skcipher_request *req)
+{
+ const struct aes_key *key =
+ crypto_skcipher_ctx(crypto_skcipher_reqtfm(req));
+
+ if (unlikely(req->cryptlen % AES_BLOCK_SIZE))
+ return -EINVAL;
+ AES_CRYPT_SG(aes_ecb_decrypt, req->dst, req->src, req->cryptlen, 0,
+ key);
+ return 0;
+}
+
+/* AES-CBC */
+
+static void crypto_aes_cbc_encrypt_sg(struct skcipher_request *req,
+ unsigned int cryptlen,
+ const struct aes_key *key)
+{
+ AES_CRYPT_SG(aes_cbc_encrypt, req->dst, req->src, cryptlen, 0, req->iv,
+ key);
+}
+
+static void crypto_aes_cbc_decrypt_sg(struct skcipher_request *req,
+ unsigned int cryptlen,
+ const struct aes_key *key)
+{
+ AES_CRYPT_SG(aes_cbc_decrypt, req->dst, req->src, cryptlen, 0, req->iv,
+ key);
+}
+
+static __maybe_unused int crypto_aes_cbc_encrypt(struct skcipher_request *req)
+{
+ const struct aes_key *key =
+ crypto_skcipher_ctx(crypto_skcipher_reqtfm(req));
+
+ if (unlikely(req->cryptlen % AES_BLOCK_SIZE))
+ return -EINVAL;
+ crypto_aes_cbc_encrypt_sg(req, req->cryptlen, key);
+ return 0;
+}
+
+static __maybe_unused int crypto_aes_cbc_decrypt(struct skcipher_request *req)
+{
+ const struct aes_key *key =
+ crypto_skcipher_ctx(crypto_skcipher_reqtfm(req));
+
+ if (unlikely(req->cryptlen % AES_BLOCK_SIZE))
+ return -EINVAL;
+ crypto_aes_cbc_decrypt_sg(req, req->cryptlen, key);
+ return 0;
+}
+
+/* AES-CBC-CTS */
+
+/*
+ * This handles AES-CBC-CTS en/decryption requests that use a nonlinear
+ * scatterlist layout or where HIGHMEM is enabled. It is explicitly 'noinline'
+ * to keep the temporary buffer out of the stack frame of the fast path.
+ */
+static noinline int
+crypto_aes_cbc_cts_crypt_nonlinear(struct skcipher_request *req, bool enc)
+{
+ const struct aes_key *key =
+ crypto_skcipher_ctx(crypto_skcipher_reqtfm(req));
+ unsigned int main_len = req->cryptlen;
+ unsigned int tail_len;
+ u8 tmp[2 * AES_BLOCK_SIZE] __aligned(__alignof__(long));
+
+ if (main_len == AES_BLOCK_SIZE) {
+ /* Single block is a special case that just does CBC. */
+ if (enc)
+ crypto_aes_cbc_encrypt_sg(req, main_len, key);
+ else
+ crypto_aes_cbc_decrypt_sg(req, main_len, key);
+ return 0;
+ }
+ /* Just do the last two blocks separately. */
+ tail_len = AES_BLOCK_SIZE + ((main_len - 1) % AES_BLOCK_SIZE) + 1;
+ main_len -= tail_len;
+ if (enc)
+ crypto_aes_cbc_encrypt_sg(req, main_len, key);
+ else
+ crypto_aes_cbc_decrypt_sg(req, main_len, key);
+ memcpy_from_sglist(tmp, req->src, main_len, tail_len);
+ if (enc)
+ aes_cbc_cts_encrypt(tmp, tmp, tail_len, req->iv, key);
+ else
+ aes_cbc_cts_decrypt(tmp, tmp, tail_len, req->iv, key);
+ memcpy_to_sglist(req->dst, main_len, tmp, tail_len);
+ memzero_explicit(tmp, sizeof(tmp));
+ return 0;
+}
+
+static __maybe_unused int
+crypto_aes_cbc_cts_encrypt(struct skcipher_request *req)
+{
+ const struct aes_key *key =
+ crypto_skcipher_ctx(crypto_skcipher_reqtfm(req));
+
+ if (unlikely(req->cryptlen < AES_BLOCK_SIZE))
+ return -EINVAL;
+ if (likely(skcipher_request_is_linear_lowmem(req))) {
+ /* Fast path */
+ aes_cbc_cts_encrypt(sg_virt(req->dst), sg_virt(req->src),
+ req->cryptlen, req->iv, key);
+ return 0;
+ }
+ return crypto_aes_cbc_cts_crypt_nonlinear(req, /* enc= */ true);
+}
+
+static __maybe_unused int
+crypto_aes_cbc_cts_decrypt(struct skcipher_request *req)
+{
+ const struct aes_key *key =
+ crypto_skcipher_ctx(crypto_skcipher_reqtfm(req));
+
+ if (unlikely(req->cryptlen < AES_BLOCK_SIZE))
+ return -EINVAL;
+ if (likely(skcipher_request_is_linear_lowmem(req))) {
+ /* Fast path */
+ aes_cbc_cts_decrypt(sg_virt(req->dst), sg_virt(req->src),
+ req->cryptlen, req->iv, key);
+ return 0;
+ }
+ return crypto_aes_cbc_cts_crypt_nonlinear(req, /* enc= */ false);
+}
+
+/* AES-CTR */
+
+static __maybe_unused int crypto_aes_ctr_crypt(struct skcipher_request *req)
+{
+ const struct aes_enckey *key =
+ crypto_skcipher_ctx(crypto_skcipher_reqtfm(req));
+