projects/linux·blob master · refs

1258 lines · 31.4 KiB

1/*
2 * Copyright (c) 2016-2017, Mellanox Technologies. All rights reserved.
3 * Copyright (c) 2016-2017, Dave Watson <[email protected]>. All rights reserved.
4 *
5 * This software is available to you under a choice of one of two
6 * licenses. You may choose to be licensed under the terms of the GNU
7 * General Public License (GPL) Version 2, available from the file
8 * COPYING in the main directory of this source tree, or the
9 * OpenIB.org BSD license below:
10 *
11 * Redistribution and use in source and binary forms, with or
12 * without modification, are permitted provided that the following
13 * conditions are met:
14 *
15 * - Redistributions of source code must retain the above
16 * copyright notice, this list of conditions and the following
17 * disclaimer.
18 *
19 * - Redistributions in binary form must reproduce the above
20 * copyright notice, this list of conditions and the following
21 * disclaimer in the documentation and/or other materials
22 * provided with the distribution.
23 *
24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
31 * SOFTWARE.
32 */
33
34#include <linux/module.h>
35
36#include <net/tcp.h>
37#include <net/inet_common.h>
38#include <linux/highmem.h>
39#include <linux/netdevice.h>
40#include <linux/sched/signal.h>
41#include <linux/inetdevice.h>
42#include <linux/inet_diag.h>
43
44#include <net/snmp.h>
45#include <net/tls.h>
46#include "tls.h"
47
48MODULE_AUTHOR("Mellanox Technologies");
49MODULE_DESCRIPTION("Transport Layer Security Support");
50MODULE_LICENSE("Dual BSD/GPL");
51MODULE_ALIAS_TCP_ULP("tls");
52
53enum {
54 TLSV4,
55 TLSV6,
56 TLS_NUM_PROTS,
57};
58
59#define CHECK_CIPHER_DESC(cipher,ci) \
60 static_assert(cipher ## _IV_SIZE <= TLS_MAX_IV_SIZE); \
61 static_assert(cipher ## _SALT_SIZE <= TLS_MAX_SALT_SIZE); \
62 static_assert(cipher ## _REC_SEQ_SIZE <= TLS_MAX_REC_SEQ_SIZE); \
63 static_assert(cipher ## _TAG_SIZE == TLS_TAG_SIZE); \
64 static_assert(sizeof_field(struct ci, iv) == cipher ## _IV_SIZE); \
65 static_assert(sizeof_field(struct ci, key) == cipher ## _KEY_SIZE); \
66 static_assert(sizeof_field(struct ci, salt) == cipher ## _SALT_SIZE); \
67 static_assert(sizeof_field(struct ci, rec_seq) == cipher ## _REC_SEQ_SIZE);
68
69#define __CIPHER_DESC(ci) \
70 .iv_offset = offsetof(struct ci, iv), \
71 .key_offset = offsetof(struct ci, key), \
72 .salt_offset = offsetof(struct ci, salt), \
73 .rec_seq_offset = offsetof(struct ci, rec_seq), \
74 .crypto_info = sizeof(struct ci)
75
76#define CIPHER_DESC(cipher,ci,algname,_offloadable) [cipher - TLS_CIPHER_MIN] = { \
77 .nonce = cipher ## _IV_SIZE, \
78 .iv = cipher ## _IV_SIZE, \
79 .key = cipher ## _KEY_SIZE, \
80 .salt = cipher ## _SALT_SIZE, \
81 .tag = cipher ## _TAG_SIZE, \
82 .rec_seq = cipher ## _REC_SEQ_SIZE, \
83 .cipher_name = algname, \
84 .offloadable = _offloadable, \
85 __CIPHER_DESC(ci), \
86}
87
88#define CIPHER_DESC_NONCE0(cipher,ci,algname,_offloadable) [cipher - TLS_CIPHER_MIN] = { \
89 .nonce = 0, \
90 .iv = cipher ## _IV_SIZE, \
91 .key = cipher ## _KEY_SIZE, \
92 .salt = cipher ## _SALT_SIZE, \
93 .tag = cipher ## _TAG_SIZE, \
94 .rec_seq = cipher ## _REC_SEQ_SIZE, \
95 .cipher_name = algname, \
96 .offloadable = _offloadable, \
97 __CIPHER_DESC(ci), \
98}
99
100const struct tls_cipher_desc tls_cipher_desc[TLS_CIPHER_MAX + 1 - TLS_CIPHER_MIN] = {
101 CIPHER_DESC(TLS_CIPHER_AES_GCM_128, tls12_crypto_info_aes_gcm_128, "gcm(aes)", true),
102 CIPHER_DESC(TLS_CIPHER_AES_GCM_256, tls12_crypto_info_aes_gcm_256, "gcm(aes)", true),
103 CIPHER_DESC(TLS_CIPHER_AES_CCM_128, tls12_crypto_info_aes_ccm_128, "ccm(aes)", false),
104 CIPHER_DESC_NONCE0(TLS_CIPHER_CHACHA20_POLY1305, tls12_crypto_info_chacha20_poly1305, "rfc7539(chacha20,poly1305)", false),
105 CIPHER_DESC(TLS_CIPHER_SM4_GCM, tls12_crypto_info_sm4_gcm, "gcm(sm4)", false),
106 CIPHER_DESC(TLS_CIPHER_SM4_CCM, tls12_crypto_info_sm4_ccm, "ccm(sm4)", false),
107 CIPHER_DESC(TLS_CIPHER_ARIA_GCM_128, tls12_crypto_info_aria_gcm_128, "gcm(aria)", false),
108 CIPHER_DESC(TLS_CIPHER_ARIA_GCM_256, tls12_crypto_info_aria_gcm_256, "gcm(aria)", false),
109};
110
111CHECK_CIPHER_DESC(TLS_CIPHER_AES_GCM_128, tls12_crypto_info_aes_gcm_128);
112CHECK_CIPHER_DESC(TLS_CIPHER_AES_GCM_256, tls12_crypto_info_aes_gcm_256);
113CHECK_CIPHER_DESC(TLS_CIPHER_AES_CCM_128, tls12_crypto_info_aes_ccm_128);
114CHECK_CIPHER_DESC(TLS_CIPHER_CHACHA20_POLY1305, tls12_crypto_info_chacha20_poly1305);
115CHECK_CIPHER_DESC(TLS_CIPHER_SM4_GCM, tls12_crypto_info_sm4_gcm);
116CHECK_CIPHER_DESC(TLS_CIPHER_SM4_CCM, tls12_crypto_info_sm4_ccm);
117CHECK_CIPHER_DESC(TLS_CIPHER_ARIA_GCM_128, tls12_crypto_info_aria_gcm_128);
118CHECK_CIPHER_DESC(TLS_CIPHER_ARIA_GCM_256, tls12_crypto_info_aria_gcm_256);
119
120static const struct proto *saved_tcpv6_prot;
121static DEFINE_MUTEX(tcpv6_prot_mutex);
122static const struct proto *saved_tcpv4_prot;
123static DEFINE_MUTEX(tcpv4_prot_mutex);
124static struct proto tls_prots[TLS_NUM_PROTS][TLS_NUM_CONFIG][TLS_NUM_CONFIG];
125static struct proto_ops tls_proto_ops[TLS_NUM_PROTS][TLS_NUM_CONFIG][TLS_NUM_CONFIG];
126static void build_protos(struct proto prot[TLS_NUM_CONFIG][TLS_NUM_CONFIG],
127 const struct proto *base);
128
129void update_sk_prot(struct sock *sk, struct tls_context *ctx)
130{
131 int ip_ver = sk->sk_family == AF_INET6 ? TLSV6 : TLSV4;
132
133 WRITE_ONCE(sk->sk_prot,
134 &tls_prots[ip_ver][ctx->tx_conf][ctx->rx_conf]);
135 WRITE_ONCE(sk->sk_socket->ops,
136 &tls_proto_ops[ip_ver][ctx->tx_conf][ctx->rx_conf]);
137}
138
139int wait_on_pending_writer(struct sock *sk, long *timeo)
140{
141 DEFINE_WAIT_FUNC(wait, woken_wake_function);
142 int ret, rc = 0;
143
144 add_wait_queue(sk_sleep(sk), &wait);
145 while (1) {
146 if (!*timeo) {
147 rc = -EAGAIN;
148 break;
149 }
150
151 if (signal_pending(current)) {
152 rc = sock_intr_errno(*timeo);
153 break;
154 }
155
156 ret = sk_wait_event(sk, timeo,
157 !READ_ONCE(sk->sk_write_pending), &wait);
158 if (ret) {
159 if (ret < 0)
160 rc = ret;
161 break;
162 }
163 }
164 remove_wait_queue(sk_sleep(sk), &wait);
165 return rc;
166}
167
168int tls_push_sg(struct sock *sk,
169 struct tls_context *ctx,
170 struct scatterlist *sg,
171 u16 first_offset,
172 int flags)
173{
174 struct bio_vec bvec;
175 struct msghdr msg = {
176 .msg_flags = MSG_SPLICE_PAGES | flags,
177 };
178 int ret = 0;
179 struct page *p;
180 size_t size;
181 int offset = first_offset;
182
183 size = sg->length - offset;
184 offset += sg->offset;
185
186 ctx->splicing_pages = true;
187 while (1) {
188 /* is sending application-limited? */
189 tcp_rate_check_app_limited(sk);
190 p = sg_page(sg);
191retry:
192 bvec_set_page(&bvec, p, size, offset);
193 iov_iter_bvec(&msg.msg_iter, ITER_SOURCE, &bvec, 1, size);
194
195 ret = tcp_sendmsg_locked(sk, &msg, size);
196
197 if (ret != size) {
198 if (ret > 0) {
199 offset += ret;
200 size -= ret;
201 goto retry;
202 }
203
204 offset -= sg->offset;
205 ctx->partially_sent_offset = offset;
206 ctx->partially_sent_record = (void *)sg;
207 ctx->splicing_pages = false;
208 return ret;
209 }
210
211 put_page(p);
212 sk_mem_uncharge(sk, sg->length);
213 sg = sg_next(sg);
214 if (!sg)
215 break;
216
217 offset = sg->offset;
218 size = sg->length;
219 }
220
221 ctx->splicing_pages = false;
222
223 return 0;
224}
225
226static int tls_handle_open_record(struct sock *sk, int flags)
227{
228 struct tls_context *ctx = tls_get_ctx(sk);
229
230 if (tls_is_pending_open_record(ctx))
231 return ctx->push_pending_record(sk, flags);
232
233 return 0;
234}
235
236int tls_process_cmsg(struct sock *sk, struct msghdr *msg,
237 unsigned char *record_type)
238{
239 struct cmsghdr *cmsg;
240 int rc = -EINVAL;
241
242 for_each_cmsghdr(cmsg, msg) {
243 if (!CMSG_OK(msg, cmsg))
244 return -EINVAL;
245 if (cmsg->cmsg_level != SOL_TLS)
246 continue;
247
248 switch (cmsg->cmsg_type) {
249 case TLS_SET_RECORD_TYPE:
250 if (cmsg->cmsg_len < CMSG_LEN(sizeof(*record_type)))
251 return -EINVAL;
252
253 if (msg->msg_flags & MSG_MORE)
254 return -EINVAL;
255
256 *record_type = *(unsigned char *)CMSG_DATA(cmsg);
257
258 rc = tls_handle_open_record(sk, msg->msg_flags);
259 break;
260 default:
261 return -EINVAL;
262 }
263 }
264
265 return rc;
266}
267
268int tls_push_partial_record(struct sock *sk, struct tls_context *ctx,
269 int flags)
270{
271 struct scatterlist *sg;
272 u16 offset;
273
274 sg = ctx->partially_sent_record;
275 offset = ctx->partially_sent_offset;
276
277 ctx->partially_sent_record = NULL;
278 return tls_push_sg(sk, ctx, sg, offset, flags);
279}
280
281void tls_free_partial_record(struct sock *sk, struct tls_context *ctx)
282{
283 struct scatterlist *sg;
284
285 for (sg = ctx->partially_sent_record; sg; sg = sg_next(sg)) {
286 put_page(sg_page(sg));
287 sk_mem_uncharge(sk, sg->length);
288 }
289 ctx->partially_sent_record = NULL;
290}
291
292static void tls_write_space(struct sock *sk)
293{
294 struct tls_context *ctx = tls_get_ctx(sk);
295
296 /* If splicing_pages call lower protocol write space handler
297 * to ensure we wake up any waiting operations there. For example
298 * if splicing pages where to call sk_wait_event.
299 */
300 if (ctx->splicing_pages) {
301 ctx->sk_write_space(sk);
302 return;
303 }
304
305#ifdef CONFIG_TLS_DEVICE
306 if (ctx->tx_conf == TLS_HW)
307 tls_device_write_space(sk, ctx);
308 else
309#endif
310 tls_sw_write_space(sk, ctx);
311
312 ctx->sk_write_space(sk);
313}
314
315/**
316 * tls_ctx_free() - free TLS ULP context
317 * @sk: socket to with @ctx is attached
318 * @ctx: TLS context structure
319 *
320 * Free TLS context. If @sk is %NULL caller guarantees that the socket
321 * to which @ctx was attached has no outstanding references.
322 */
323void tls_ctx_free(struct sock *sk, struct tls_context *ctx)
324{
325 if (!ctx)
326 return;
327
328 memzero_explicit(&ctx->crypto_send, sizeof(ctx->crypto_send));
329 memzero_explicit(&ctx->crypto_recv, sizeof(ctx->crypto_recv));
330 mutex_destroy(&ctx->tx_lock);
331
332 if (sk)
333 kfree_rcu(ctx, rcu);
334 else
335 kfree(ctx);
336}
337
338static void tls_sk_proto_cleanup(struct sock *sk,
339 struct tls_context *ctx, long timeo)
340{
341 if (unlikely(sk->sk_write_pending) &&
342 !wait_on_pending_writer(sk, &timeo))
343 tls_handle_open_record(sk, 0);
344
345 /* We need these for tls_sw_fallback handling of other packets */
346 if (ctx->tx_conf == TLS_SW) {
347 tls_sw_release_resources_tx(sk);
348 TLS_DEC_STATS(sock_net(sk), LINUX_MIB_TLSCURRTXSW);
349 } else if (ctx->tx_conf == TLS_HW) {
350 tls_device_free_resources_tx(sk);
351 TLS_DEC_STATS(sock_net(sk), LINUX_MIB_TLSCURRTXDEVICE);
352 }
353
354 if (ctx->rx_conf == TLS_SW) {
355 tls_sw_release_resources_rx(sk);
356 TLS_DEC_STATS(sock_net(sk), LINUX_MIB_TLSCURRRXSW);
357 } else if (ctx->rx_conf == TLS_HW) {
358 tls_device_offload_cleanup_rx(sk);
359 TLS_DEC_STATS(sock_net(sk), LINUX_MIB_TLSCURRRXDEVICE);
360 }
361}
362
363static void tls_sk_proto_close(struct sock *sk, long timeout)
364{
365 struct inet_connection_sock *icsk = inet_csk(sk);
366 struct tls_context *ctx = tls_get_ctx(sk);
367 long timeo = sock_sndtimeo(sk, 0);
368 bool free_ctx;
369
370 if (ctx->tx_conf == TLS_SW)
371 tls_sw_cancel_work_tx(ctx);
372
373 lock_sock(sk);
374 free_ctx = ctx->tx_conf != TLS_HW && ctx->rx_conf != TLS_HW;
375
376 if (ctx->tx_conf != TLS_BASE || ctx->rx_conf != TLS_BASE)
377 tls_sk_proto_cleanup(sk, ctx, timeo);
378
379 write_lock_bh(&sk->sk_callback_lock);
380 if (free_ctx)
381 rcu_assign_pointer(icsk->icsk_ulp_data, NULL);
382 WRITE_ONCE(sk->sk_prot, ctx->sk_proto);
383 if (sk->sk_write_space == tls_write_space)
384 sk->sk_write_space = ctx->sk_write_space;
385 write_unlock_bh(&sk->sk_callback_lock);
386 release_sock(sk);
387 if (ctx->tx_conf == TLS_SW)
388 tls_sw_free_ctx_tx(ctx);
389 if (ctx->rx_conf == TLS_SW || ctx->rx_conf == TLS_HW)
390 tls_sw_strparser_done(ctx);
391 if (ctx->rx_conf == TLS_SW)
392 tls_sw_free_ctx_rx(ctx);
393 ctx->sk_proto->close(sk, timeout);
394
395 if (free_ctx)
396 tls_ctx_free(sk, ctx);
397}
398
399static __poll_t tls_sk_poll(struct file *file, struct socket *sock,
400 struct poll_table_struct *wait)
401{
402 struct tls_sw_context_rx *ctx;
403 struct tls_context *tls_ctx;
404 struct sock *sk = sock->sk;
405 __poll_t mask = 0;
406 u8 shutdown;
407 int state;
408
409 mask = tcp_poll(file, sock, wait);
410
411 state = inet_sk_state_load(sk);
412 shutdown = READ_ONCE(sk->sk_shutdown);
413 if (unlikely(state != TCP_ESTABLISHED || shutdown & RCV_SHUTDOWN))
414 return mask;
415
416 tls_ctx = tls_get_ctx(sk);
417 ctx = tls_sw_ctx_rx(tls_ctx);
418
419 if ((skb_queue_empty_lockless(&ctx->rx_list) &&
420 !tls_strp_msg_ready(ctx)) ||
421 READ_ONCE(ctx->key_update_pending))
422 mask &= ~(EPOLLIN | EPOLLRDNORM);
423
424 return mask;
425}
426
427static int do_tls_getsockopt_conf(struct sock *sk, sockopt_t *opt, int tx)
428{
429 int rc = 0;
430 const struct tls_cipher_desc *cipher_desc;
431 struct tls_context *ctx = tls_get_ctx(sk);
432 struct tls_crypto_info *crypto_info;
433 struct cipher_context *cctx;
434 int len = opt->optlen;
435
436 if (!opt->iter_out.ubuf || len < sizeof(*crypto_info)) {
437 rc = -EINVAL;
438 goto out;
439 }
440
441 if (!ctx) {
442 rc = -EBUSY;
443 goto out;
444 }
445
446 /* get user crypto info */
447 if (tx) {
448 crypto_info = &ctx->crypto_send.info;
449 cctx = &ctx->tx;
450 } else {
451 crypto_info = &ctx->crypto_recv.info;
452 cctx = &ctx->rx;
453 }
454
455 if (!TLS_CRYPTO_INFO_READY(crypto_info)) {
456 rc = -EBUSY;
457 goto out;
458 }
459
460 if (len == sizeof(*crypto_info)) {
461 if (copy_to_iter(crypto_info, sizeof(*crypto_info),
462 &opt->iter_out) != sizeof(*crypto_info))
463 rc = -EFAULT;
464 goto out;
465 }
466
467 cipher_desc = get_cipher_desc(crypto_info->cipher_type);
468 if (!cipher_desc || len != cipher_desc->crypto_info) {
469 rc = -EINVAL;
470 goto out;
471 }
472
473 memcpy(crypto_info_iv(crypto_info, cipher_desc),
474 cctx->iv + cipher_desc->salt, cipher_desc->iv);
475 memcpy(crypto_info_rec_seq(crypto_info, cipher_desc),
476 cctx->rec_seq, cipher_desc->rec_seq);
477
478 if (copy_to_iter(crypto_info, cipher_desc->crypto_info,
479 &opt->iter_out) != cipher_desc->crypto_info)
480 rc = -EFAULT;
481
482out:
483 return rc;
484}
485
486static int do_tls_getsockopt_tx_zc(struct sock *sk, sockopt_t *opt)
487{
488 struct tls_context *ctx = tls_get_ctx(sk);
489 unsigned int value;
490 int len = opt->optlen;
491
492 if (len != sizeof(value))
493 return -EINVAL;
494
495 value = ctx->zerocopy_sendfile;
496 if (copy_to_iter(&value, sizeof(value), &opt->iter_out) != sizeof(value))
497 return -EFAULT;
498
499 return 0;
500}
501
502static int do_tls_getsockopt_no_pad(struct sock *sk, sockopt_t *opt)
503{
504 struct tls_context *ctx = tls_get_ctx(sk);
505 int value, len = opt->optlen;
506
507 if (ctx->prot_info.version != TLS_1_3_VERSION)
508 return -EINVAL;
509
510 if (len < sizeof(value))
511 return -EINVAL;
512
513 value = -EINVAL;
514 if (ctx->rx_conf == TLS_SW || ctx->rx_conf == TLS_HW)
515 value = ctx->rx_no_pad;
516 if (value < 0)
517 return value;
518
519 opt->optlen = sizeof(value);
520 if (copy_to_iter(&value, sizeof(value), &opt->iter_out) != sizeof(value))
521 return -EFAULT;
522
523 return 0;
524}
525
526static int do_tls_getsockopt_tx_payload_len(struct sock *sk, sockopt_t *opt)
527{
528 struct tls_context *ctx = tls_get_ctx(sk);
529 u16 payload_len = ctx->tx_max_payload_len;
530 int len = opt->optlen;
531
532 if (len < sizeof(payload_len))
533 return -EINVAL;
534
535 opt->optlen = sizeof(payload_len);
536 if (copy_to_iter(&payload_len, sizeof(payload_len),
537 &opt->iter_out) != sizeof(payload_len))
538 return -EFAULT;
539
540 return 0;
541}
542
543static int do_tls_getsockopt(struct sock *sk, int optname, sockopt_t *opt)
544{
545 int rc = 0;
546
547 lock_sock(sk);
548
549 switch (optname) {
550 case TLS_TX:
551 case TLS_RX:
552 rc = do_tls_getsockopt_conf(sk, opt, optname == TLS_TX);
553 break;
554 case TLS_TX_ZEROCOPY_RO:
555 rc = do_tls_getsockopt_tx_zc(sk, opt);
556 break;
557 case TLS_RX_EXPECT_NO_PAD:
558 rc = do_tls_getsockopt_no_pad(sk, opt);
559 break;
560 case TLS_TX_MAX_PAYLOAD_LEN:
561 rc = do_tls_getsockopt_tx_payload_len(sk, opt);
562 break;
563 default:
564 rc = -ENOPROTOOPT;
565 break;
566 }
567
568 release_sock(sk);
569
570 return rc;
571}
572
573static int tls_getsockopt(struct sock *sk, int level, int optname,
574 char __user *optval, int __user *optlen)
575{
576 struct tls_context *ctx = tls_get_ctx(sk);
577 sockopt_t opt;
578 int err;
579
580 if (level != SOL_TLS)
581 return ctx->sk_proto->getsockopt(sk, level,
582 optname, optval, optlen);
583
584 err = sockopt_init_user(&opt, optval, optlen);
585 if (err)
586 return err;
587
588 err = do_tls_getsockopt(sk, optname, &opt);
589 if (err)
590 return err;
591
592 if (put_user(opt.optlen, optlen))
593 return -EFAULT;
594
595 return 0;
596}
597
598static int validate_crypto_info(const struct tls_crypto_info *crypto_info,
599 const struct tls_crypto_info *alt_crypto_info)
600{
601 if (crypto_info->version != TLS_1_2_VERSION &&
602 crypto_info->version != TLS_1_3_VERSION)
603 return -EINVAL;
604
605 switch (crypto_info->cipher_type) {
606 case TLS_CIPHER_ARIA_GCM_128:
607 case TLS_CIPHER_ARIA_GCM_256:
608 if (crypto_info->version != TLS_1_2_VERSION)
609 return -EINVAL;
610 break;
611 }
612
613 /* Ensure that TLS version and ciphers are same in both directions */
614 if (TLS_CRYPTO_INFO_READY(alt_crypto_info)) {
615 if (alt_crypto_info->version != crypto_info->version ||
616 alt_crypto_info->cipher_type != crypto_info->cipher_type)
617 return -EINVAL;
618 }
619
620 return 0;
621}
622
623static int do_tls_setsockopt_conf(struct sock *sk, sockptr_t optval,
624 unsigned int optlen, int tx)
625{
626 struct tls_crypto_info *crypto_info, *alt_crypto_info;
627 struct tls_crypto_info *old_crypto_info = NULL;
628 struct tls_context *ctx = tls_get_ctx(sk);
629 const struct tls_cipher_desc *cipher_desc;
630 union tls_crypto_context *crypto_ctx;
631 union tls_crypto_context tmp = {};
632 bool update = false;
633 int rc = 0;
634 int conf;
635
636 /* TLS and sockmap are mutually exclusive. A socket already in a
637 * sockmap (i.e. with a psock attached) cannot be upgraded to TLS.
638 * sockmap rejects TLS sockets already (see sk_psock_init()).
639 */
640 rcu_read_lock();
641 if (sk_psock(sk)) {
642 rcu_read_unlock();
643 return -EINVAL;
644 }
645 rcu_read_unlock();
646
647 if (sockptr_is_null(optval) || (optlen < sizeof(*crypto_info)))
648 return -EINVAL;
649
650 if (tx) {
651 crypto_ctx = &ctx->crypto_send;
652 alt_crypto_info = &ctx->crypto_recv.info;
653 } else {
654 crypto_ctx = &ctx->crypto_recv;
655 alt_crypto_info = &ctx->crypto_send.info;
656 }
657
658 crypto_info = &crypto_ctx->info;
659
660 if (TLS_CRYPTO_INFO_READY(crypto_info)) {
661 /* Currently we only support setting crypto info more
662 * than one time for TLS 1.3
663 */
664 if (crypto_info->version != TLS_1_3_VERSION) {
665 TLS_INC_STATS(sock_net(sk), tx ? LINUX_MIB_TLSTXREKEYERROR
666 : LINUX_MIB_TLSRXREKEYERROR);
667 return -EBUSY;
668 }
669
670 update = true;
671 old_crypto_info = crypto_info;
672 crypto_info = &tmp.info;
673 crypto_ctx = &tmp;
674 }
675
676 rc = copy_from_sockptr(crypto_info, optval, sizeof(*crypto_info));
677 if (rc) {
678 rc = -EFAULT;
679 goto err_crypto_info;
680 }
681
682 if (update) {
683 /* Ensure that TLS version and ciphers are not modified */
684 if (crypto_info->version != old_crypto_info->version ||
685 crypto_info->cipher_type != old_crypto_info->cipher_type)
686 rc = -EINVAL;
687 } else {
688 rc = validate_crypto_info(crypto_info, alt_crypto_info);
689 }
690 if (rc)
691 goto err_crypto_info;
692
693 cipher_desc = get_cipher_desc(crypto_info->cipher_type);
694 if (!cipher_desc) {
695 rc = -EINVAL;
696 goto err_crypto_info;
697 }
698
699 if (optlen != cipher_desc->crypto_info) {
700 rc = -EINVAL;
701 goto err_crypto_info;
702 }
703
704 rc = copy_from_sockptr_offset(crypto_info + 1, optval,
705 sizeof(*crypto_info),
706 optlen - sizeof(*crypto_info));
707 if (rc) {
708 rc = -EFAULT;
709 goto err_crypto_info;
710 }
711
712 if (tx) {
713 rc = tls_set_device_offload(sk);
714 conf = TLS_HW;
715 if (!rc) {
716 TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSTXDEVICE);
717 TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSCURRTXDEVICE);
718 } else {
719 rc = tls_set_sw_offload(sk, 1,
720 update ? crypto_info : NULL);
721 if (rc)
722 goto err_crypto_info;
723
724 if (update) {
725 TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSTXREKEYOK);
726 } else {
727 TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSTXSW);
728 TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSCURRTXSW);
729 }
730 conf = TLS_SW;
731 }
732 } else {
733 rc = tls_set_device_offload_rx(sk, ctx);
734 conf = TLS_HW;
735 if (!rc) {
736 TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSRXDEVICE);
737 TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSCURRRXDEVICE);
738 } else {
739 rc = tls_set_sw_offload(sk, 0,
740 update ? crypto_info : NULL);
741 if (rc)
742 goto err_crypto_info;
743
744 if (update) {
745 TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSRXREKEYOK);
746 } else {
747 TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSRXSW);
748 TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSCURRRXSW);
749 }
750 conf = TLS_SW;
751 }
752 if (!update)
753 tls_sw_strparser_arm(sk, ctx);
754 }
755
756 if (tx)
757 ctx->tx_conf = conf;
758 else
759 ctx->rx_conf = conf;
760 update_sk_prot(sk, ctx);
761
762 if (update)
763 return 0;
764
765 if (tx) {
766 ctx->sk_write_space = sk->sk_write_space;
767 sk->sk_write_space = tls_write_space;
768 } else {
769 struct tls_sw_context_rx *rx_ctx = tls_sw_ctx_rx(ctx);
770
771 tls_strp_check_rcv(&rx_ctx->strp, true);
772 }
773 return 0;
774
775err_crypto_info:
776 if (update) {
777 TLS_INC_STATS(sock_net(sk), tx ? LINUX_MIB_TLSTXREKEYERROR
778 : LINUX_MIB_TLSRXREKEYERROR);
779 }
780 memzero_explicit(crypto_ctx, sizeof(*crypto_ctx));
781 return rc;
782}
783
784static int do_tls_setsockopt_tx_zc(struct sock *sk, sockptr_t optval,
785 unsigned int optlen)
786{
787 struct tls_context *ctx = tls_get_ctx(sk);
788 unsigned int value;
789
790 if (sockptr_is_null(optval) || optlen != sizeof(value))
791 return -EINVAL;
792
793 if (copy_from_sockptr(&value, optval, sizeof(value)))
794 return -EFAULT;
795
796 if (value > 1)
797 return -EINVAL;
798
799 ctx->zerocopy_sendfile = value;
800
801 return 0;
802}
803
804static int do_tls_setsockopt_no_pad(struct sock *sk, sockptr_t optval,
805 unsigned int optlen)
806{
807 struct tls_context *ctx = tls_get_ctx(sk);
808 u32 val;
809 int rc;
810
811 if (ctx->prot_info.version != TLS_1_3_VERSION ||
812 sockptr_is_null(optval) || optlen < sizeof(val))
813 return -EINVAL;
814
815 rc = copy_from_sockptr(&val, optval, sizeof(val));
816 if (rc)
817 return -EFAULT;
818 if (val > 1)
819 return -EINVAL;
820 rc = check_zeroed_sockptr(optval, sizeof(val), optlen - sizeof(val));
821 if (rc < 1)
822 return rc == 0 ? -EINVAL : rc;
823
824 lock_sock(sk);
825 rc = -EINVAL;
826 if (ctx->rx_conf == TLS_SW || ctx->rx_conf == TLS_HW) {
827 ctx->rx_no_pad = val;
828 tls_update_rx_zc_capable(ctx);
829 rc = 0;
830 }
831 release_sock(sk);
832
833 return rc;
834}
835
836static int do_tls_setsockopt_tx_payload_len(struct sock *sk, sockptr_t optval,
837 unsigned int optlen)
838{
839 struct tls_context *ctx = tls_get_ctx(sk);
840 struct tls_sw_context_tx *sw_ctx = tls_sw_ctx_tx(ctx);
841 u16 value;
842 bool tls_13 = ctx->prot_info.version == TLS_1_3_VERSION;
843
844 if (sw_ctx && sw_ctx->open_rec)
845 return -EBUSY;
846
847 if (sockptr_is_null(optval) || optlen != sizeof(value))
848 return -EINVAL;
849
850 if (copy_from_sockptr(&value, optval, sizeof(value)))
851 return -EFAULT;
852
853 if (value < TLS_MIN_RECORD_SIZE_LIM - (tls_13 ? 1 : 0) ||
854 value > TLS_MAX_PAYLOAD_SIZE)
855 return -EINVAL;
856
857 ctx->tx_max_payload_len = value;
858
859 return 0;
860}
861
862static int do_tls_setsockopt(struct sock *sk, int optname, sockptr_t optval,
863 unsigned int optlen)
864{
865 int rc = 0;
866
867 switch (optname) {
868 case TLS_TX:
869 case TLS_RX:
870 lock_sock(sk);
871 rc = do_tls_setsockopt_conf(sk, optval, optlen,
872 optname == TLS_TX);
873 release_sock(sk);
874 break;
875 case TLS_TX_ZEROCOPY_RO:
876 lock_sock(sk);
877 rc = do_tls_setsockopt_tx_zc(sk, optval, optlen);
878 release_sock(sk);
879 break;
880 case TLS_RX_EXPECT_NO_PAD:
881 rc = do_tls_setsockopt_no_pad(sk, optval, optlen);
882 break;
883 case TLS_TX_MAX_PAYLOAD_LEN:
884 lock_sock(sk);
885 rc = do_tls_setsockopt_tx_payload_len(sk, optval, optlen);
886 release_sock(sk);
887 break;
888 default:
889 rc = -ENOPROTOOPT;
890 break;
891 }
892 return rc;
893}
894
895static int tls_setsockopt(struct sock *sk, int level, int optname,
896 sockptr_t optval, unsigned int optlen)
897{
898 struct tls_context *ctx = tls_get_ctx(sk);
899
900 if (level != SOL_TLS)
901 return ctx->sk_proto->setsockopt(sk, level, optname, optval,
902 optlen);
903
904 return do_tls_setsockopt(sk, optname, optval, optlen);
905}
906
907static int tls_disconnect(struct sock *sk, int flags)
908{
909 return -EOPNOTSUPP;
910}
911
912struct tls_context *tls_ctx_create(struct sock *sk)
913{
914 struct inet_connection_sock *icsk = inet_csk(sk);
915 struct tls_context *ctx;
916
917 ctx = kzalloc_obj(*ctx, GFP_ATOMIC);
918 if (!ctx)
919 return NULL;
920
921 mutex_init(&ctx->tx_lock);
922 ctx->sk_proto = READ_ONCE(sk->sk_prot);
923 ctx->sk = sk;
924 /* Release semantic of rcu_assign_pointer() ensures that
925 * ctx->sk_proto is visible before changing sk->sk_prot in
926 * update_sk_prot(), and prevents reading uninitialized value in
927 * tls_{getsockopt, setsockopt}. Note that we do not need a
928 * read barrier in tls_{getsockopt,setsockopt} as there is an
929 * address dependency between sk->sk_proto->{getsockopt,setsockopt}
930 * and ctx->sk_proto.
931 */
932 rcu_assign_pointer(icsk->icsk_ulp_data, ctx);
933 return ctx;
934}
935
936static void build_proto_ops(struct proto_ops ops[TLS_NUM_CONFIG][TLS_NUM_CONFIG],
937 const struct proto_ops *base)
938{
939 ops[TLS_BASE][TLS_BASE] = *base;
940
941 ops[TLS_SW ][TLS_BASE] = ops[TLS_BASE][TLS_BASE];
942 ops[TLS_SW ][TLS_BASE].splice_eof = tls_sw_splice_eof;
943
944 ops[TLS_BASE][TLS_SW ] = ops[TLS_BASE][TLS_BASE];
945 ops[TLS_BASE][TLS_SW ].splice_read = tls_sw_splice_read;
946 ops[TLS_BASE][TLS_SW ].poll = tls_sk_poll;
947 ops[TLS_BASE][TLS_SW ].read_sock = tls_sw_read_sock;
948
949 ops[TLS_SW ][TLS_SW ] = ops[TLS_SW ][TLS_BASE];
950 ops[TLS_SW ][TLS_SW ].splice_read = tls_sw_splice_read;
951 ops[TLS_SW ][TLS_SW ].poll = tls_sk_poll;
952 ops[TLS_SW ][TLS_SW ].read_sock = tls_sw_read_sock;
953
954#ifdef CONFIG_TLS_DEVICE
955 ops[TLS_HW ][TLS_BASE] = ops[TLS_BASE][TLS_BASE];
956
957 ops[TLS_HW ][TLS_SW ] = ops[TLS_BASE][TLS_SW ];
958
959 ops[TLS_BASE][TLS_HW ] = ops[TLS_BASE][TLS_SW ];
960
961 ops[TLS_SW ][TLS_HW ] = ops[TLS_SW ][TLS_SW ];
962
963 ops[TLS_HW ][TLS_HW ] = ops[TLS_HW ][TLS_SW ];
964#endif
965}
966
967static void tls_build_proto(struct sock *sk)
968{
969 int ip_ver = sk->sk_family == AF_INET6 ? TLSV6 : TLSV4;
970 struct proto *prot = READ_ONCE(sk->sk_prot);
971
972 /* Build IPv6 TLS whenever the address of tcpv6 _prot changes */
973 if (ip_ver == TLSV6 &&
974 unlikely(prot != smp_load_acquire(&saved_tcpv6_prot))) {
975 mutex_lock(&tcpv6_prot_mutex);
976 if (likely(prot != saved_tcpv6_prot)) {
977 build_protos(tls_prots[TLSV6], prot);
978 build_proto_ops(tls_proto_ops[TLSV6],
979 sk->sk_socket->ops);
980 smp_store_release(&saved_tcpv6_prot, prot);
981 }
982 mutex_unlock(&tcpv6_prot_mutex);
983 }
984
985 if (ip_ver == TLSV4 &&
986 unlikely(prot != smp_load_acquire(&saved_tcpv4_prot))) {
987 mutex_lock(&tcpv4_prot_mutex);
988 if (likely(prot != saved_tcpv4_prot)) {
989 build_protos(tls_prots[TLSV4], prot);
990 build_proto_ops(tls_proto_ops[TLSV4],
991 sk->sk_socket->ops);
992 smp_store_release(&saved_tcpv4_prot, prot);
993 }
994 mutex_unlock(&tcpv4_prot_mutex);
995 }
996}
997
998static void build_protos(struct proto prot[TLS_NUM_CONFIG][TLS_NUM_CONFIG],
999 const struct proto *base)
1000{
1001 prot[TLS_BASE][TLS_BASE] = *base;
1002 prot[TLS_BASE][TLS_BASE].setsockopt = tls_setsockopt;
1003 prot[TLS_BASE][TLS_BASE].getsockopt = tls_getsockopt;
1004 prot[TLS_BASE][TLS_BASE].disconnect = tls_disconnect;
1005 prot[TLS_BASE][TLS_BASE].close = tls_sk_proto_close;
1006
1007 prot[TLS_SW][TLS_BASE] = prot[TLS_BASE][TLS_BASE];
1008 prot[TLS_SW][TLS_BASE].sendmsg = tls_sw_sendmsg;
1009 prot[TLS_SW][TLS_BASE].splice_eof = tls_sw_splice_eof;
1010
1011 prot[TLS_BASE][TLS_SW] = prot[TLS_BASE][TLS_BASE];
1012 prot[TLS_BASE][TLS_SW].recvmsg = tls_sw_recvmsg;
1013 prot[TLS_BASE][TLS_SW].sock_is_readable = tls_sw_sock_is_readable;
1014 prot[TLS_BASE][TLS_SW].close = tls_sk_proto_close;
1015
1016 prot[TLS_SW][TLS_SW] = prot[TLS_SW][TLS_BASE];
1017 prot[TLS_SW][TLS_SW].recvmsg = tls_sw_recvmsg;
1018 prot[TLS_SW][TLS_SW].sock_is_readable = tls_sw_sock_is_readable;
1019 prot[TLS_SW][TLS_SW].close = tls_sk_proto_close;
1020
1021#ifdef CONFIG_TLS_DEVICE
1022 prot[TLS_HW][TLS_BASE] = prot[TLS_BASE][TLS_BASE];
1023 prot[TLS_HW][TLS_BASE].sendmsg = tls_device_sendmsg;
1024 prot[TLS_HW][TLS_BASE].splice_eof = tls_device_splice_eof;
1025
1026 prot[TLS_HW][TLS_SW] = prot[TLS_BASE][TLS_SW];
1027 prot[TLS_HW][TLS_SW].sendmsg = tls_device_sendmsg;
1028 prot[TLS_HW][TLS_SW].splice_eof = tls_device_splice_eof;
1029
1030 prot[TLS_BASE][TLS_HW] = prot[TLS_BASE][TLS_SW];
1031
1032 prot[TLS_SW][TLS_HW] = prot[TLS_SW][TLS_SW];
1033
1034 prot[TLS_HW][TLS_HW] = prot[TLS_HW][TLS_SW];
1035#endif
1036}
1037
1038static int tls_init(struct sock *sk)
1039{
1040 struct tls_context *ctx;
1041 int rc = 0;
1042
1043 tls_build_proto(sk);
1044
1045 /* The TLS ulp is currently supported only for TCP sockets
1046 * in ESTABLISHED state.
1047 * Supporting sockets in LISTEN state will require us
1048 * to modify the accept implementation to clone rather then
1049 * share the ulp context.
1050 */
1051 if (sk->sk_state != TCP_ESTABLISHED)
1052 return -ENOTCONN;
1053
1054 /* allocate tls context */
1055 write_lock_bh(&sk->sk_callback_lock);
1056 ctx = tls_ctx_create(sk);
1057 if (!ctx) {
1058 rc = -ENOMEM;
1059 goto out;
1060 }
1061
1062 ctx->tx_conf = TLS_BASE;
1063 ctx->rx_conf = TLS_BASE;
1064 ctx->tx_max_payload_len = TLS_MAX_PAYLOAD_SIZE;
1065 update_sk_prot(sk, ctx);
1066out:
1067 write_unlock_bh(&sk->sk_callback_lock);
1068 return rc;
1069}
1070
1071static void tls_update(struct sock *sk, struct proto *p,
1072 void (*write_space)(struct sock *sk))
1073{
1074 struct tls_context *ctx;
1075
1076 WARN_ON_ONCE(sk->sk_prot == p);
1077
1078 ctx = tls_get_ctx(sk);
1079 if (likely(ctx)) {
1080 ctx->sk_write_space = write_space;
1081 ctx->sk_proto = p;
1082 } else {
1083 /* Pairs with lockless read in sk_clone_lock(). */
1084 WRITE_ONCE(sk->sk_prot, p);
1085 sk->sk_write_space = write_space;
1086 }
1087}
1088
1089static u16 tls_user_config(struct tls_context *ctx, bool tx)
1090{
1091 u16 config = tx ? ctx->tx_conf : ctx->rx_conf;
1092
1093 switch (config) {
1094 case TLS_BASE:
1095 return TLS_CONF_BASE;
1096 case TLS_SW:
1097 return TLS_CONF_SW;
1098 case TLS_HW:
1099 return TLS_CONF_HW;
1100 }
1101 return 0;
1102}
1103
1104static int tls_get_info(struct sock *sk, struct sk_buff *skb, bool net_admin)
1105{
1106 u16 version, cipher_type;
1107 struct tls_context *ctx;
1108 struct nlattr *start;
1109 int err;
1110
1111 start = nla_nest_start_noflag(skb, INET_ULP_INFO_TLS);
1112 if (!start)
1113 return -EMSGSIZE;
1114
1115 rcu_read_lock();
1116 ctx = rcu_dereference(inet_csk(sk)->icsk_ulp_data);
1117 if (!ctx) {
1118 err = 0;
1119 goto nla_failure;
1120 }
1121 version = ctx->prot_info.version;
1122 if (version) {
1123 err = nla_put_u16(skb, TLS_INFO_VERSION, version);
1124 if (err)
1125 goto nla_failure;
1126 }
1127 cipher_type = ctx->prot_info.cipher_type;
1128 if (cipher_type) {
1129 err = nla_put_u16(skb, TLS_INFO_CIPHER, cipher_type);
1130 if (err)
1131 goto nla_failure;
1132 }
1133 err = nla_put_u16(skb, TLS_INFO_TXCONF, tls_user_config(ctx, true));
1134 if (err)
1135 goto nla_failure;
1136
1137 err = nla_put_u16(skb, TLS_INFO_RXCONF, tls_user_config(ctx, false));
1138 if (err)
1139 goto nla_failure;
1140
1141 if (ctx->tx_conf == TLS_HW && ctx->zerocopy_sendfile) {
1142 err = nla_put_flag(skb, TLS_INFO_ZC_RO_TX);
1143 if (err)
1144 goto nla_failure;
1145 }
1146 if (ctx->rx_no_pad) {
1147 err = nla_put_flag(skb, TLS_INFO_RX_NO_PAD);
1148 if (err)
1149 goto nla_failure;
1150 }
1151
1152 err = nla_put_u16(skb, TLS_INFO_TX_MAX_PAYLOAD_LEN,
1153 ctx->tx_max_payload_len);
1154
1155 if (err)
1156 goto nla_failure;
1157
1158 rcu_read_unlock();
1159 nla_nest_end(skb, start);
1160 return 0;
1161
1162nla_failure:
1163 rcu_read_unlock();
1164 nla_nest_cancel(skb, start);
1165 return err;
1166}
1167
1168static size_t tls_get_info_size(const struct sock *sk, bool net_admin)
1169{
1170 size_t size = 0;
1171
1172 size += nla_total_size(0) + /* INET_ULP_INFO_TLS */
1173 nla_total_size(sizeof(u16)) + /* TLS_INFO_VERSION */
1174 nla_total_size(sizeof(u16)) + /* TLS_INFO_CIPHER */
1175 nla_total_size(sizeof(u16)) + /* TLS_INFO_RXCONF */
1176 nla_total_size(sizeof(u16)) + /* TLS_INFO_TXCONF */
1177 nla_total_size(0) + /* TLS_INFO_ZC_RO_TX */
1178 nla_total_size(0) + /* TLS_INFO_RX_NO_PAD */
1179 nla_total_size(sizeof(u16)) + /* TLS_INFO_TX_MAX_PAYLOAD_LEN */
1180 0;
1181
1182 return size;
1183}
1184
1185static int __net_init tls_init_net(struct net *net)
1186{
1187 int err;
1188
1189 net->mib.tls_statistics = alloc_percpu(struct linux_tls_mib);
1190 if (!net->mib.tls_statistics)
1191 return -ENOMEM;
1192
1193 err = tls_proc_init(net);
1194 if (err)
1195 goto err_free_stats;
1196
1197 return 0;
1198err_free_stats:
1199 free_percpu(net->mib.tls_statistics);
1200 return err;
1201}
1202
1203static void __net_exit tls_exit_net(struct net *net)
1204{
1205 tls_proc_fini(net);
1206 free_percpu(net->mib.tls_statistics);
1207}
1208
1209static struct pernet_operations tls_proc_ops = {
1210 .init = tls_init_net,
1211 .exit = tls_exit_net,
1212};
1213
1214static struct tcp_ulp_ops tcp_tls_ulp_ops __read_mostly = {
1215 .name = "tls",
1216 .owner = THIS_MODULE,
1217 .init = tls_init,
1218 .update = tls_update,
1219 .get_info = tls_get_info,
1220 .get_info_size = tls_get_info_size,
1221};
1222
1223static int __init tls_register(void)
1224{
1225 int err;
1226
1227 err = register_pernet_subsys(&tls_proc_ops);
1228 if (err)
1229 return err;
1230
1231 err = tls_strp_dev_init();
1232 if (err)
1233 goto err_pernet;
1234
1235 err = tls_device_init();
1236 if (err)
1237 goto err_strp;
1238
1239 tcp_register_ulp(&tcp_tls_ulp_ops);
1240
1241 return 0;
1242err_strp:
1243 tls_strp_dev_exit();
1244err_pernet:
1245 unregister_pernet_subsys(&tls_proc_ops);
1246 return err;
1247}
1248
1249static void __exit tls_unregister(void)
1250{
1251 tcp_unregister_ulp(&tcp_tls_ulp_ops);
1252 tls_strp_dev_exit();
1253 tls_device_cleanup();
1254 unregister_pernet_subsys(&tls_proc_ops);
1255}
1256
1257module_init(tls_register);
1258module_exit(tls_unregister);