projects/linux·blob master · refs

471 lines · 12.8 KiB

1/* Copyright (c) 2018, Mellanox Technologies All rights reserved.
2 *
3 * This software is available to you under a choice of one of two
4 * licenses. You may choose to be licensed under the terms of the GNU
5 * General Public License (GPL) Version 2, available from the file
6 * COPYING in the main directory of this source tree, or the
7 * OpenIB.org BSD license below:
8 *
9 * Redistribution and use in source and binary forms, with or
10 * without modification, are permitted provided that the following
11 * conditions are met:
12 *
13 * - Redistributions of source code must retain the above
14 * copyright notice, this list of conditions and the following
15 * disclaimer.
16 *
17 * - Redistributions in binary form must reproduce the above
18 * copyright notice, this list of conditions and the following
19 * disclaimer in the documentation and/or other materials
20 * provided with the distribution.
21 *
22 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
23 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
24 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
25 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
26 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
27 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
28 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
29 * SOFTWARE.
30 */
31
32#include <net/tls.h>
33#include <crypto/aead.h>
34#include <crypto/scatterwalk.h>
35#include <net/ip6_checksum.h>
36#include <linux/skbuff_ref.h>
37
38#include "tls.h"
39
40static int tls_enc_record(struct aead_request *aead_req,
41 struct crypto_aead *aead, char *aad,
42 char *iv, __be64 rcd_sn,
43 struct scatter_walk *in,
44 struct scatter_walk *out, int *in_len,
45 struct tls_prot_info *prot)
46{
47 unsigned char buf[TLS_HEADER_SIZE + TLS_MAX_IV_SIZE];
48 const struct tls_cipher_desc *cipher_desc;
49 struct scatterlist sg_in[3];
50 struct scatterlist sg_out[3];
51 unsigned int buf_size;
52 u16 len;
53 int rc;
54
55 cipher_desc = get_cipher_desc(prot->cipher_type);
56 DEBUG_NET_WARN_ON_ONCE(!cipher_desc || !cipher_desc->offloadable);
57
58 buf_size = TLS_HEADER_SIZE + cipher_desc->iv;
59 len = min_t(int, *in_len, buf_size);
60
61 memcpy_from_scatterwalk(buf, in, len);
62 memcpy_to_scatterwalk(out, buf, len);
63
64 *in_len -= len;
65 if (!*in_len)
66 return 0;
67
68 len = buf[4] | (buf[3] << 8);
69 len -= cipher_desc->iv;
70
71 tls_make_aad(aad, len - cipher_desc->tag, (char *)&rcd_sn, buf[0], prot);
72
73 memcpy(iv + cipher_desc->salt, buf + TLS_HEADER_SIZE, cipher_desc->iv);
74
75 sg_init_table(sg_in, ARRAY_SIZE(sg_in));
76 sg_init_table(sg_out, ARRAY_SIZE(sg_out));
77 sg_set_buf(sg_in, aad, TLS_AAD_SPACE_SIZE);
78 sg_set_buf(sg_out, aad, TLS_AAD_SPACE_SIZE);
79 scatterwalk_get_sglist(in, sg_in + 1);
80 scatterwalk_get_sglist(out, sg_out + 1);
81
82 *in_len -= len;
83 if (*in_len < 0) {
84 *in_len += cipher_desc->tag;
85 /* the input buffer doesn't contain the entire record.
86 * trim len accordingly. The resulting authentication tag
87 * will contain garbage, but we don't care, so we won't
88 * include any of it in the output skb
89 * Note that we assume the output buffer length
90 * is larger then input buffer length + tag size
91 */
92 if (*in_len < 0)
93 len += *in_len;
94
95 *in_len = 0;
96 }
97
98 if (*in_len) {
99 scatterwalk_skip(in, len);
100 scatterwalk_skip(out, len);
101 }
102
103 len -= cipher_desc->tag;
104 aead_request_set_crypt(aead_req, sg_in, sg_out, len, iv);
105
106 rc = crypto_aead_encrypt(aead_req);
107
108 return rc;
109}
110
111static void tls_init_aead_request(struct aead_request *aead_req,
112 struct crypto_aead *aead)
113{
114 aead_request_set_tfm(aead_req, aead);
115 aead_request_set_ad(aead_req, TLS_AAD_SPACE_SIZE);
116}
117
118static struct aead_request *tls_alloc_aead_request(struct crypto_aead *aead,
119 gfp_t flags)
120{
121 unsigned int req_size = sizeof(struct aead_request) +
122 crypto_aead_reqsize(aead);
123 struct aead_request *aead_req;
124
125 aead_req = kzalloc(req_size, flags);
126 if (aead_req)
127 tls_init_aead_request(aead_req, aead);
128 return aead_req;
129}
130
131static int tls_enc_records(struct aead_request *aead_req,
132 struct crypto_aead *aead, struct scatterlist *sg_in,
133 struct scatterlist *sg_out, char *aad, char *iv,
134 u64 rcd_sn, int len, struct tls_prot_info *prot)
135{
136 struct scatter_walk out, in;
137 int rc;
138
139 scatterwalk_start(&in, sg_in);
140 scatterwalk_start(&out, sg_out);
141
142 do {
143 rc = tls_enc_record(aead_req, aead, aad, iv,
144 cpu_to_be64(rcd_sn), &in, &out, &len, prot);
145 rcd_sn++;
146
147 } while (rc == 0 && len);
148
149 return rc;
150}
151
152static void update_chksum(struct sk_buff *skb, int headln)
153{
154 struct tcphdr *th = tcp_hdr(skb);
155 int datalen = skb->len - headln;
156 const struct ipv6hdr *ipv6h;
157 const struct iphdr *iph;
158
159 /* We only changed the payload so if we are using partial we don't
160 * need to update anything.
161 */
162 if (likely(skb->ip_summed == CHECKSUM_PARTIAL))
163 return;
164
165 skb->ip_summed = CHECKSUM_PARTIAL;
166 skb->csum_start = skb_transport_header(skb) - skb->head;
167 skb->csum_offset = offsetof(struct tcphdr, check);
168
169 if (skb->sk->sk_family == AF_INET6) {
170 ipv6h = ipv6_hdr(skb);
171 th->check = ~csum_ipv6_magic(&ipv6h->saddr, &ipv6h->daddr,
172 datalen, IPPROTO_TCP, 0);
173 } else {
174 iph = ip_hdr(skb);
175 th->check = ~csum_tcpudp_magic(iph->saddr, iph->daddr, datalen,
176 IPPROTO_TCP, 0);
177 }
178}
179
180static void complete_skb(struct sk_buff *nskb, struct sk_buff *skb, int headln)
181{
182 struct sock *sk = skb->sk;
183 int delta;
184
185 skb_copy_header(nskb, skb);
186
187 skb_put(nskb, skb->len);
188 memcpy(nskb->data, skb->data, headln);
189
190 nskb->destructor = skb->destructor;
191 nskb->sk = sk;
192 skb->destructor = NULL;
193 skb->sk = NULL;
194
195 update_chksum(nskb, headln);
196
197 /* sock_efree means skb must gone through skb_orphan_partial() */
198 if (nskb->destructor == sock_efree)
199 return;
200
201 delta = nskb->truesize - skb->truesize;
202 if (likely(delta < 0))
203 WARN_ON_ONCE(refcount_sub_and_test(-delta, &sk->sk_wmem_alloc));
204 else if (delta)
205 refcount_add(delta, &sk->sk_wmem_alloc);
206}
207
208/* This function may be called after the user socket is already
209 * closed so make sure we don't use anything freed during
210 * tls_sk_proto_close here
211 */
212
213static int fill_sg_in(struct scatterlist *sg_in,
214 struct sk_buff *skb,
215 struct tls_offload_context_tx *ctx,
216 u64 *rcd_sn,
217 s32 *sync_size,
218 int *resync_sgs)
219{
220 int tcp_payload_offset = skb_tcp_all_headers(skb);
221 int payload_len = skb->len - tcp_payload_offset;
222 u32 tcp_seq = ntohl(tcp_hdr(skb)->seq);
223 struct tls_record_info *record;
224 unsigned long flags;
225 int remaining;
226 int i;
227
228 spin_lock_irqsave(&ctx->lock, flags);
229 record = tls_get_record(ctx, tcp_seq, rcd_sn);
230 if (!record) {
231 spin_unlock_irqrestore(&ctx->lock, flags);
232 return -EINVAL;
233 }
234
235 *sync_size = tcp_seq - tls_record_start_seq(record);
236 if (*sync_size < 0) {
237 int is_start_marker = tls_record_is_start_marker(record);
238
239 spin_unlock_irqrestore(&ctx->lock, flags);
240 /* This should only occur if the relevant record was
241 * already acked. In that case it should be ok
242 * to drop the packet and avoid retransmission.
243 *
244 * There is a corner case where the packet contains
245 * both an acked and a non-acked record.
246 * We currently don't handle that case and rely
247 * on TCP to retransmit a packet that doesn't contain
248 * already acked payload.
249 */
250 if (!is_start_marker)
251 *sync_size = 0;
252 return -EINVAL;
253 }
254
255 remaining = *sync_size;
256 for (i = 0; remaining > 0; i++) {
257 skb_frag_t *frag = &record->frags[i];
258
259 __skb_frag_ref(frag);
260 sg_set_page(sg_in + i, skb_frag_page(frag),
261 skb_frag_size(frag), skb_frag_off(frag));
262
263 remaining -= skb_frag_size(frag);
264
265 if (remaining < 0)
266 sg_in[i].length += remaining;
267 }
268 *resync_sgs = i;
269
270 spin_unlock_irqrestore(&ctx->lock, flags);
271 if (skb_to_sgvec(skb, &sg_in[i], tcp_payload_offset, payload_len) < 0)
272 return -EINVAL;
273
274 return 0;
275}
276
277static void fill_sg_out(struct scatterlist sg_out[3], void *buf,
278 struct tls_context *tls_ctx,
279 struct sk_buff *nskb,
280 int tcp_payload_offset,
281 int payload_len,
282 int sync_size,
283 void *dummy_buf)
284{
285 const struct tls_cipher_desc *cipher_desc =
286 get_cipher_desc(tls_ctx->crypto_send.info.cipher_type);
287
288 sg_set_buf(&sg_out[0], dummy_buf, sync_size);
289 sg_set_buf(&sg_out[1], nskb->data + tcp_payload_offset, payload_len);
290 /* Add room for authentication tag produced by crypto */
291 dummy_buf += sync_size;
292 sg_set_buf(&sg_out[2], dummy_buf, cipher_desc->tag);
293}
294
295static struct sk_buff *tls_enc_skb(struct tls_context *tls_ctx,
296 struct scatterlist sg_out[3],
297 struct scatterlist *sg_in,
298 struct sk_buff *skb,
299 s32 sync_size, u64 rcd_sn)
300{
301 struct tls_offload_context_tx *ctx = tls_offload_ctx_tx(tls_ctx);
302 int tcp_payload_offset = skb_tcp_all_headers(skb);
303 int payload_len = skb->len - tcp_payload_offset;
304 const struct tls_cipher_desc *cipher_desc;
305 void *buf, *iv, *aad, *dummy_buf, *salt;
306 struct aead_request *aead_req;
307 struct sk_buff *nskb = NULL;
308 int buf_len;
309
310 aead_req = tls_alloc_aead_request(ctx->aead_send, GFP_ATOMIC);
311 if (!aead_req)
312 return NULL;
313
314 cipher_desc = get_cipher_desc(tls_ctx->crypto_send.info.cipher_type);
315 DEBUG_NET_WARN_ON_ONCE(!cipher_desc || !cipher_desc->offloadable);
316
317 buf_len = cipher_desc->salt + cipher_desc->iv + TLS_AAD_SPACE_SIZE +
318 sync_size + cipher_desc->tag;
319 buf = kmalloc(buf_len, GFP_ATOMIC);
320 if (!buf)
321 goto free_req;
322
323 iv = buf;
324 salt = crypto_info_salt(&tls_ctx->crypto_send.info, cipher_desc);
325 memcpy(iv, salt, cipher_desc->salt);
326 aad = buf + cipher_desc->salt + cipher_desc->iv;
327 dummy_buf = aad + TLS_AAD_SPACE_SIZE;
328
329 nskb = alloc_skb(skb_headroom(skb) + skb->len, GFP_ATOMIC);
330 if (!nskb)
331 goto free_buf;
332
333 skb_reserve(nskb, skb_headroom(skb));
334
335 fill_sg_out(sg_out, buf, tls_ctx, nskb, tcp_payload_offset,
336 payload_len, sync_size, dummy_buf);
337
338 if (tls_enc_records(aead_req, ctx->aead_send, sg_in, sg_out, aad, iv,
339 rcd_sn, sync_size + payload_len,
340 &tls_ctx->prot_info) < 0)
341 goto free_nskb;
342
343 complete_skb(nskb, skb, tcp_payload_offset);
344
345 /* validate_xmit_skb_list assumes that if the skb wasn't segmented
346 * nskb->prev will point to the skb itself
347 */
348 nskb->prev = nskb;
349
350free_buf:
351 kfree(buf);
352free_req:
353 kfree(aead_req);
354 return nskb;
355free_nskb:
356 kfree_skb(nskb);
357 nskb = NULL;
358 goto free_buf;
359}
360
361static struct sk_buff *tls_sw_fallback(struct sock *sk, struct sk_buff *skb)
362{
363 int tcp_payload_offset = skb_tcp_all_headers(skb);
364 struct tls_context *tls_ctx = tls_get_ctx(sk);
365 struct tls_offload_context_tx *ctx = tls_offload_ctx_tx(tls_ctx);
366 int payload_len = skb->len - tcp_payload_offset;
367 struct scatterlist *sg_in, sg_out[3];
368 struct sk_buff *nskb = NULL;
369 int sg_in_max_elements;
370 int resync_sgs = 0;
371 s32 sync_size = 0;
372 u64 rcd_sn;
373
374 /* worst case is:
375 * MAX_SKB_FRAGS in tls_record_info
376 * MAX_SKB_FRAGS + 1 in SKB head and frags.
377 */
378 sg_in_max_elements = 2 * MAX_SKB_FRAGS + 1;
379
380 if (!payload_len)
381 return skb;
382
383 sg_in = kmalloc_objs(*sg_in, sg_in_max_elements, GFP_ATOMIC);
384 if (!sg_in)
385 goto free_orig;
386
387 sg_init_table(sg_in, sg_in_max_elements);
388 sg_init_table(sg_out, ARRAY_SIZE(sg_out));
389
390 if (fill_sg_in(sg_in, skb, ctx, &rcd_sn, &sync_size, &resync_sgs)) {
391 /* bypass packets before kernel TLS socket option was set */
392 if (sync_size < 0 && payload_len <= -sync_size)
393 nskb = skb_get(skb);
394 goto put_sg;
395 }
396
397 nskb = tls_enc_skb(tls_ctx, sg_out, sg_in, skb, sync_size, rcd_sn);
398
399put_sg:
400 while (resync_sgs)
401 put_page(sg_page(&sg_in[--resync_sgs]));
402 kfree(sg_in);
403free_orig:
404 if (nskb)
405 consume_skb(skb);
406 else
407 kfree_skb(skb);
408 return nskb;
409}
410
411struct sk_buff *tls_validate_xmit_skb(struct sock *sk,
412 struct net_device *dev,
413 struct sk_buff *skb)
414{
415 if (dev == rcu_dereference_bh(tls_get_ctx(sk)->netdev) ||
416 netif_is_bond_master(dev))
417 return skb;
418
419 return tls_sw_fallback(sk, skb);
420}
421EXPORT_SYMBOL_GPL(tls_validate_xmit_skb);
422
423struct sk_buff *tls_validate_xmit_skb_sw(struct sock *sk,
424 struct net_device *dev,
425 struct sk_buff *skb)
426{
427 return tls_sw_fallback(sk, skb);
428}
429
430struct sk_buff *tls_encrypt_skb(struct sk_buff *skb)
431{
432 return tls_sw_fallback(skb->sk, skb);
433}
434EXPORT_SYMBOL_GPL(tls_encrypt_skb);
435
436int tls_sw_fallback_init(struct sock *sk,
437 struct tls_offload_context_tx *offload_ctx,
438 struct tls_crypto_info *crypto_info)
439{
440 const struct tls_cipher_desc *cipher_desc;
441 int rc;
442
443 cipher_desc = get_cipher_desc(crypto_info->cipher_type);
444 if (!cipher_desc || !cipher_desc->offloadable)
445 return -EINVAL;
446
447 offload_ctx->aead_send =
448 crypto_alloc_aead(cipher_desc->cipher_name, 0, CRYPTO_ALG_ASYNC);
449 if (IS_ERR(offload_ctx->aead_send)) {
450 rc = PTR_ERR(offload_ctx->aead_send);
451 pr_err_ratelimited("crypto_alloc_aead failed rc=%d\n", rc);
452 offload_ctx->aead_send = NULL;
453 goto err_out;
454 }
455
456 rc = crypto_aead_setkey(offload_ctx->aead_send,
457 crypto_info_key(crypto_info, cipher_desc),
458 cipher_desc->key);
459 if (rc)
460 goto free_aead;
461
462 rc = crypto_aead_setauthsize(offload_ctx->aead_send, cipher_desc->tag);
463 if (rc)
464 goto free_aead;
465
466 return 0;
467free_aead:
468 crypto_free_aead(offload_ctx->aead_send);
469err_out:
470 return rc;
471}