| 1 | |
| 2 | |
| 3 | * INET An implementation of the TCP/IP protocol suite for the LINUX
|
| 4 | * operating system. INET is implemented using the BSD Socket
|
| 5 | * interface as the means of communication with the user level.
|
| 6 | *
|
| 7 | * Implementation of the Transmission Control Protocol(TCP).
|
| 8 | *
|
| 9 | * IPv4 specific functions
|
| 10 | *
|
| 11 | * code split from:
|
| 12 | * linux/ipv4/tcp.c
|
| 13 | * linux/ipv4/tcp_input.c
|
| 14 | * linux/ipv4/tcp_output.c
|
| 15 | *
|
| 16 | * See tcp.c for author information
|
| 17 |
|
| 18 |
|
| 19 | |
| 20 | * Changes:
|
| 21 | * David S. Miller : New socket lookup architecture.
|
| 22 | * This code is dedicated to John Dyson.
|
| 23 | * David S. Miller : Change semantics of established hash,
|
| 24 | * half is devoted to TIME_WAIT sockets
|
| 25 | * and the rest go in the other half.
|
| 26 | * Andi Kleen : Add support for syncookies and fixed
|
| 27 | * some bugs: ip options weren't passed to
|
| 28 | * the TCP layer, missed a check for an
|
| 29 | * ACK bit.
|
| 30 | * Andi Kleen : Implemented fast path mtu discovery.
|
| 31 | * Fixed many serious bugs in the
|
| 32 | * request_sock handling and moved
|
| 33 | * most of it into the af independent code.
|
| 34 | * Added tail drop and some other bugfixes.
|
| 35 | * Added new listen semantics.
|
| 36 | * Mike McLagan : Routing by source
|
| 37 | * Juan Jose Ciarlante: ip_dynaddr bits
|
| 38 | * Andi Kleen: various fixes.
|
| 39 | * Vitaly E. Lavrov : Transparent proxy revived after year
|
| 40 | * coma.
|
| 41 | * Andi Kleen : Fix new listen.
|
| 42 | * Andi Kleen : Fix accept error reporting.
|
| 43 | * YOSHIFUJI Hideaki @USAGI and: Support IPV6_V6ONLY socket option, which
|
| 44 | * Alexey Kuznetsov allow both IPv4 and IPv6 sockets to bind
|
| 45 | * a single port at the same time.
|
| 46 |
|
| 47 |
|
| 48 | #define pr_fmt(fmt) "TCP: " fmt
|
| 49 |
|
| 50 | #include <linux/bottom_half.h>
|
| 51 | #include <linux/types.h>
|
| 52 | #include <linux/fcntl.h>
|
| 53 | #include <linux/module.h>
|
| 54 | #include <linux/random.h>
|
| 55 | #include <linux/cache.h>
|
| 56 | #include <linux/fips.h>
|
| 57 | #include <linux/jhash.h>
|
| 58 | #include <linux/init.h>
|
| 59 | #include <linux/times.h>
|
| 60 | #include <linux/slab.h>
|
| 61 | #include <linux/sched.h>
|
| 62 | #include <linux/sock_diag.h>
|
| 63 |
|
| 64 | #include <net/aligned_data.h>
|
| 65 | #include <net/net_namespace.h>
|
| 66 | #include <net/icmp.h>
|
| 67 | #include <net/inet_hashtables.h>
|
| 68 | #include <net/tcp.h>
|
| 69 | #include <net/tcp_ecn.h>
|
| 70 | #include <net/transp_v6.h>
|
| 71 | #include <net/ipv6.h>
|
| 72 | #include <net/inet_common.h>
|
| 73 | #include <net/inet_ecn.h>
|
| 74 | #include <net/timewait_sock.h>
|
| 75 | #include <net/xfrm.h>
|
| 76 | #include <net/secure_seq.h>
|
| 77 | #include <net/busy_poll.h>
|
| 78 | #include <net/rstreason.h>
|
| 79 | #include <net/psp.h>
|
| 80 |
|
| 81 | #include <linux/inet.h>
|
| 82 | #include <linux/ipv6.h>
|
| 83 | #include <linux/stddef.h>
|
| 84 | #include <linux/proc_fs.h>
|
| 85 | #include <linux/seq_file.h>
|
| 86 | #include <linux/inetdevice.h>
|
| 87 | #include <linux/btf_ids.h>
|
| 88 | #include <linux/skbuff_ref.h>
|
| 89 |
|
| 90 | #include <crypto/md5.h>
|
| 91 | #include <crypto/utils.h>
|
| 92 |
|
| 93 | #include <trace/events/tcp.h>
|
| 94 |
|
| 95 | #ifdef CONFIG_TCP_MD5SIG
|
| 96 | static void tcp_v4_md5_hash_hdr(char *md5_hash, const struct tcp_md5sig_key *key,
|
| 97 | __be32 daddr, __be32 saddr, const struct tcphdr *th);
|
| 98 | #endif
|
| 99 |
|
| 100 | struct inet_hashinfo tcp_hashinfo;
|
| 101 |
|
| 102 | static DEFINE_PER_CPU(struct sock_bh_locked, ipv4_tcp_sk) = {
|
| 103 | .bh_lock = INIT_LOCAL_LOCK(bh_lock),
|
| 104 | };
|
| 105 |
|
| 106 | static DEFINE_MUTEX(tcp_exit_batch_mutex);
|
| 107 |
|
| 108 | INDIRECT_CALLABLE_SCOPE union tcp_seq_and_ts_off
|
| 109 | tcp_v4_init_seq_and_ts_off(const struct net *net, const struct sk_buff *skb)
|
| 110 | {
|
| 111 | return secure_tcp_seq_and_ts_off(net,
|
| 112 | ip_hdr(skb)->daddr,
|
| 113 | ip_hdr(skb)->saddr,
|
| 114 | tcp_hdr(skb)->dest,
|
| 115 | tcp_hdr(skb)->source);
|
| 116 | }
|
| 117 |
|
| 118 | int tcp_twsk_unique(struct sock *sk, struct sock *sktw, void *twp)
|
| 119 | {
|
| 120 | int reuse = READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_tw_reuse);
|
| 121 | const struct inet_timewait_sock *tw = inet_twsk(sktw);
|
| 122 | const struct tcp_timewait_sock *tcptw = tcp_twsk(sktw);
|
| 123 | struct tcp_sock *tp = tcp_sk(sk);
|
| 124 | int ts_recent_stamp;
|
| 125 | u32 reuse_thresh;
|
| 126 |
|
| 127 | if (READ_ONCE(tw->tw_substate) == TCP_FIN_WAIT2)
|
| 128 | reuse = 0;
|
| 129 |
|
| 130 | if (reuse == 2) {
|
| 131 | |
| 132 | * lo, since we require a loopback src or dst address
|
| 133 | * or direct binding to 'lo' interface.
|
| 134 |
|
| 135 | bool loopback = false;
|
| 136 | if (tw->tw_bound_dev_if == LOOPBACK_IFINDEX)
|
| 137 | loopback = true;
|
| 138 | #if IS_ENABLED(CONFIG_IPV6)
|
| 139 | if (tw->tw_family == AF_INET6) {
|
| 140 | if (ipv6_addr_loopback(&tw->tw_v6_daddr) ||
|
| 141 | ipv6_addr_v4mapped_loopback(&tw->tw_v6_daddr) ||
|
| 142 | ipv6_addr_loopback(&tw->tw_v6_rcv_saddr) ||
|
| 143 | ipv6_addr_v4mapped_loopback(&tw->tw_v6_rcv_saddr))
|
| 144 | loopback = true;
|
| 145 | } else
|
| 146 | #endif
|
| 147 | {
|
| 148 | if (ipv4_is_loopback(tw->tw_daddr) ||
|
| 149 | ipv4_is_loopback(tw->tw_rcv_saddr))
|
| 150 | loopback = true;
|
| 151 | }
|
| 152 | if (!loopback)
|
| 153 | reuse = 0;
|
| 154 | }
|
| 155 |
|
| 156 | |
| 157 | of data integrity. Even without PAWS it is safe provided sequence
|
| 158 | spaces do not overlap i.e. at data rates <= 80Mbit/sec.
|
| 159 |
|
| 160 | Actually, the idea is close to VJ's one, only timestamp cache is
|
| 161 | held not per host, but per port pair and TW bucket is used as state
|
| 162 | holder.
|
| 163 |
|
| 164 | If TW bucket has been already destroyed we fall back to VJ's scheme
|
| 165 | and use initial timestamp retrieved from peer table.
|
| 166 |
|
| 167 | ts_recent_stamp = READ_ONCE(tcptw->tw_ts_recent_stamp);
|
| 168 | reuse_thresh = READ_ONCE(tw->tw_entry_stamp) +
|
| 169 | READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_tw_reuse_delay);
|
| 170 | if (ts_recent_stamp &&
|
| 171 | (!twp || (reuse && time_after32(tcp_clock_ms(), reuse_thresh)))) {
|
| 172 | |
| 173 | * and releasing the bucket lock.
|
| 174 |
|
| 175 | if (unlikely(!refcount_inc_not_zero(&sktw->sk_refcnt)))
|
| 176 | return 0;
|
| 177 |
|
| 178 | |
| 179 | * want to be sure that it is safe as above but honor the
|
| 180 | * sequence numbers and time stamps set as part of the repair
|
| 181 | * process.
|
| 182 | *
|
| 183 | * Without this check re-using a TIME-WAIT socket with TCP
|
| 184 | * repair would accumulate a -1 on the repair assigned
|
| 185 | * sequence number. The first time it is reused the sequence
|
| 186 | * is -1, the second time -2, etc. This fixes that issue
|
| 187 | * without appearing to create any others.
|
| 188 |
|
| 189 | if (likely(!tp->repair)) {
|
| 190 | u32 seq = tcptw->tw_snd_nxt + 65535 + 2;
|
| 191 |
|
| 192 | if (!seq)
|
| 193 | seq = 1;
|
| 194 | WRITE_ONCE(tp->write_seq, seq);
|
| 195 | tp->rx_opt.ts_recent = READ_ONCE(tcptw->tw_ts_recent);
|
| 196 | tp->rx_opt.ts_recent_stamp = ts_recent_stamp;
|
| 197 | }
|
| 198 |
|
| 199 | return 1;
|
| 200 | }
|
| 201 |
|
| 202 | return 0;
|
| 203 | }
|
| 204 |
|
| 205 | static int tcp_v4_pre_connect(struct sock *sk, struct sockaddr_unsized *uaddr,
|
| 206 | int addr_len)
|
| 207 | {
|
| 208 | |
| 209 | * prevent BPF program called below from accessing bytes that are out
|
| 210 | * of the bound specified by user in addr_len.
|
| 211 |
|
| 212 | if (addr_len < sizeof(struct sockaddr_in))
|
| 213 | return -EINVAL;
|
| 214 |
|
| 215 | sock_owned_by_me(sk);
|
| 216 |
|
| 217 | return BPF_CGROUP_RUN_PROG_INET4_CONNECT(sk, uaddr, &addr_len);
|
| 218 | }
|
| 219 |
|
| 220 |
|
| 221 | int tcp_v4_connect(struct sock *sk, struct sockaddr_unsized *uaddr, int addr_len)
|
| 222 | {
|
| 223 | struct sockaddr_in *usin = (struct sockaddr_in *)uaddr;
|
| 224 | struct inet_timewait_death_row *tcp_death_row;
|
| 225 | struct inet_sock *inet = inet_sk(sk);
|
| 226 | struct tcp_sock *tp = tcp_sk(sk);
|
| 227 | struct ip_options_rcu *inet_opt;
|
| 228 | struct net *net = sock_net(sk);
|
| 229 | __be16 orig_sport, orig_dport;
|
| 230 | __be32 daddr, nexthop;
|
| 231 | struct flowi4 *fl4;
|
| 232 | struct rtable *rt;
|
| 233 | int err;
|
| 234 |
|
| 235 | if (addr_len < sizeof(struct sockaddr_in))
|
| 236 | return -EINVAL;
|
| 237 |
|
| 238 | if (usin->sin_family != AF_INET)
|
| 239 | return -EAFNOSUPPORT;
|
| 240 |
|
| 241 | nexthop = daddr = usin->sin_addr.s_addr;
|
| 242 | inet_opt = rcu_dereference_protected(inet->inet_opt,
|
| 243 | lockdep_sock_is_held(sk));
|
| 244 | if (inet_opt && inet_opt->opt.srr) {
|
| 245 | if (!daddr)
|
| 246 | return -EINVAL;
|
| 247 | nexthop = inet_opt->opt.faddr;
|
| 248 | }
|
| 249 |
|
| 250 | orig_sport = inet->inet_sport;
|
| 251 | orig_dport = usin->sin_port;
|
| 252 | fl4 = &inet->cork.fl.u.ip4;
|
| 253 | rt = ip_route_connect(fl4, nexthop, inet->inet_saddr,
|
| 254 | sk->sk_bound_dev_if, IPPROTO_TCP, orig_sport,
|
| 255 | orig_dport, sk);
|
| 256 | if (IS_ERR(rt)) {
|
| 257 | err = PTR_ERR(rt);
|
| 258 | if (err == -ENETUNREACH)
|
| 259 | IP_INC_STATS(net, IPSTATS_MIB_OUTNOROUTES);
|
| 260 | return err;
|
| 261 | }
|
| 262 |
|
| 263 | if (rt->rt_flags & (RTCF_MULTICAST | RTCF_BROADCAST)) {
|
| 264 | ip_rt_put(rt);
|
| 265 | return -ENETUNREACH;
|
| 266 | }
|
| 267 |
|
| 268 | if (!inet_opt || !inet_opt->opt.srr)
|
| 269 | daddr = fl4->daddr;
|
| 270 |
|
| 271 | tcp_death_row = &sock_net(sk)->ipv4.tcp_death_row;
|
| 272 |
|
| 273 | if (!inet->inet_saddr) {
|
| 274 | err = inet_bhash2_update_saddr(sk, &fl4->saddr, AF_INET);
|
| 275 | if (err) {
|
| 276 | ip_rt_put(rt);
|
| 277 | return err;
|
| 278 | }
|
| 279 | } else {
|
| 280 | sk_rcv_saddr_set(sk, inet->inet_saddr);
|
| 281 | }
|
| 282 |
|
| 283 | if (tp->rx_opt.ts_recent_stamp && inet->inet_daddr != daddr) {
|
| 284 |
|
| 285 | tp->rx_opt.ts_recent = 0;
|
| 286 | tp->rx_opt.ts_recent_stamp = 0;
|
| 287 | if (likely(!tp->repair))
|
| 288 | WRITE_ONCE(tp->write_seq, 0);
|
| 289 | }
|
| 290 |
|
| 291 | inet->inet_dport = usin->sin_port;
|
| 292 | sk_daddr_set(sk, daddr);
|
| 293 |
|
| 294 | inet_csk(sk)->icsk_ext_hdr_len = psp_sk_overhead(sk);
|
| 295 | if (inet_opt)
|
| 296 | inet_csk(sk)->icsk_ext_hdr_len += inet_opt->opt.optlen;
|
| 297 |
|
| 298 | tp->rx_opt.mss_clamp = TCP_MSS_DEFAULT;
|
| 299 |
|
| 300 | |
| 301 | * However we set state to SYN-SENT and not releasing socket
|
| 302 | * lock select source port, enter ourselves into the hash tables and
|
| 303 | * complete initialization after this.
|
| 304 |
|
| 305 | tcp_set_state(sk, TCP_SYN_SENT);
|
| 306 | err = inet_hash_connect(tcp_death_row, sk);
|
| 307 | if (err)
|
| 308 | goto failure;
|
| 309 |
|
| 310 | sk_set_txhash(sk);
|
| 311 |
|
| 312 | rt = ip_route_newports(fl4, rt, orig_sport, orig_dport,
|
| 313 | inet->inet_sport, inet->inet_dport, sk);
|
| 314 | if (IS_ERR(rt)) {
|
| 315 | err = PTR_ERR(rt);
|
| 316 | rt = NULL;
|
| 317 | goto failure;
|
| 318 | }
|
| 319 | tp->tcp_usec_ts = dst_tcp_usec_ts(&rt->dst);
|
| 320 |
|
| 321 | sk->sk_gso_type = SKB_GSO_TCPV4;
|
| 322 | sk_setup_caps(sk, &rt->dst);
|
| 323 | rt = NULL;
|
| 324 |
|
| 325 | if (likely(!tp->repair)) {
|
| 326 | union tcp_seq_and_ts_off st;
|
| 327 |
|
| 328 | st = secure_tcp_seq_and_ts_off(net,
|
| 329 | inet->inet_saddr,
|
| 330 | inet->inet_daddr,
|
| 331 | inet->inet_sport,
|
| 332 | usin->sin_port);
|
| 333 | if (!tp->write_seq)
|
| 334 | WRITE_ONCE(tp->write_seq, st.seq);
|
| 335 | WRITE_ONCE(tp->tsoffset, st.ts_off);
|
| 336 | }
|
| 337 |
|
| 338 | atomic_set(&inet->inet_id, get_random_u16());
|
| 339 |
|
| 340 | if (tcp_fastopen_defer_connect(sk, &err))
|
| 341 | return err;
|
| 342 | if (err)
|
| 343 | goto failure;
|
| 344 |
|
| 345 | err = tcp_connect(sk);
|
| 346 |
|
| 347 | if (err)
|
| 348 | goto failure;
|
| 349 |
|
| 350 | return 0;
|
| 351 |
|
| 352 | failure:
|
| 353 | |
| 354 | * This unhashes the socket and releases the local port,
|
| 355 | * if necessary.
|
| 356 |
|
| 357 | tcp_set_state(sk, TCP_CLOSE);
|
| 358 | inet_bhash2_reset_saddr(sk);
|
| 359 | ip_rt_put(rt);
|
| 360 | sk->sk_route_caps = 0;
|
| 361 | inet->inet_dport = 0;
|
| 362 | return err;
|
| 363 | }
|
| 364 |
|
| 365 | |
| 366 | * This routine reacts to ICMP_FRAG_NEEDED mtu indications as defined in RFC1191.
|
| 367 | * It can be called through tcp_release_cb() if socket was owned by user
|
| 368 | * at the time tcp_v4_err() was called to handle ICMP message.
|
| 369 |
|
| 370 | void tcp_v4_mtu_reduced(struct sock *sk)
|
| 371 | {
|
| 372 | struct inet_sock *inet = inet_sk(sk);
|
| 373 | struct dst_entry *dst;
|
| 374 | u32 mtu, dmtu;
|
| 375 |
|
| 376 | if ((1 << sk->sk_state) & (TCPF_LISTEN | TCPF_CLOSE))
|
| 377 | return;
|
| 378 | mtu = READ_ONCE(tcp_sk(sk)->mtu_info);
|
| 379 | dst = inet_csk_update_pmtu(sk, mtu);
|
| 380 | if (!dst)
|
| 381 | return;
|
| 382 |
|
| 383 | |
| 384 | * for the case, if this connection will not able to recover.
|
| 385 |
|
| 386 | dmtu = dst4_mtu(dst);
|
| 387 | if (mtu < dmtu && ip_dont_fragment(sk, dst))
|
| 388 | WRITE_ONCE(sk->sk_err_soft, EMSGSIZE);
|
| 389 |
|
| 390 | if (inet->pmtudisc != IP_PMTUDISC_DONT &&
|
| 391 | ip_sk_accept_pmtu(sk) &&
|
| 392 | inet_csk(sk)->icsk_pmtu_cookie > dmtu) {
|
| 393 | tcp_sync_mss(sk, dmtu);
|
| 394 |
|
| 395 | |
| 396 | * clear that the old packet has been
|
| 397 | * dropped. This is the new "fast" path mtu
|
| 398 | * discovery.
|
| 399 |
|
| 400 | tcp_simple_retransmit(sk);
|
| 401 | }
|
| 402 | }
|
| 403 |
|
| 404 | static void do_redirect(struct sk_buff *skb, struct sock *sk)
|
| 405 | {
|
| 406 | struct dst_entry *dst = __sk_dst_check(sk, 0);
|
| 407 |
|
| 408 | if (dst)
|
| 409 | dst->ops->redirect(dst, sk, skb);
|
| 410 | }
|
| 411 |
|
| 412 |
|
| 413 |
|
| 414 | void tcp_req_err(struct sock *sk, u32 seq, bool abort)
|
| 415 | {
|
| 416 | struct request_sock *req = inet_reqsk(sk);
|
| 417 | struct net *net = sock_net(sk);
|
| 418 |
|
| 419 | |
| 420 | * an established socket here.
|
| 421 |
|
| 422 | if (seq != tcp_rsk(req)->snt_isn) {
|
| 423 | __NET_INC_STATS(net, LINUX_MIB_OUTOFWINDOWICMPS);
|
| 424 | } else if (abort) {
|
| 425 | |
| 426 | * Still in SYN_RECV, just remove it silently.
|
| 427 | * There is no good way to pass the error to the newly
|
| 428 | * created socket, and POSIX does not want network
|
| 429 | * errors returned from accept().
|
| 430 |
|
| 431 | inet_csk_reqsk_queue_drop(req->rsk_listener, req);
|
| 432 | tcp_listendrop(req->rsk_listener);
|
| 433 | }
|
| 434 | reqsk_put(req);
|
| 435 | }
|
| 436 |
|
| 437 |
|
| 438 | void tcp_ld_RTO_revert(struct sock *sk, u32 seq)
|
| 439 | {
|
| 440 | struct inet_connection_sock *icsk = inet_csk(sk);
|
| 441 | struct tcp_sock *tp = tcp_sk(sk);
|
| 442 | struct sk_buff *skb;
|
| 443 | s32 remaining;
|
| 444 | u32 delta_us;
|
| 445 |
|
| 446 | if (sock_owned_by_user(sk))
|
| 447 | return;
|
| 448 |
|
| 449 | if (seq != tp->snd_una || !icsk->icsk_retransmits ||
|
| 450 | !icsk->icsk_backoff)
|
| 451 | return;
|
| 452 |
|
| 453 | skb = tcp_rtx_queue_head(sk);
|
| 454 | if (WARN_ON_ONCE(!skb))
|
| 455 | return;
|
| 456 |
|
| 457 | icsk->icsk_backoff--;
|
| 458 | icsk->icsk_rto = tp->srtt_us ? __tcp_set_rto(tp) : TCP_TIMEOUT_INIT;
|
| 459 | icsk->icsk_rto = inet_csk_rto_backoff(icsk, tcp_rto_max(sk));
|
| 460 |
|
| 461 | tcp_mstamp_refresh(tp);
|
| 462 | delta_us = (u32)(tp->tcp_mstamp - tcp_skb_timestamp_us(skb));
|
| 463 | remaining = icsk->icsk_rto - usecs_to_jiffies(delta_us);
|
| 464 |
|
| 465 | if (remaining > 0) {
|
| 466 | tcp_reset_xmit_timer(sk, ICSK_TIME_RETRANS, remaining, false);
|
| 467 | } else {
|
| 468 | |
| 469 | * Will retransmit now.
|
| 470 |
|
| 471 | tcp_retransmit_timer(sk);
|
| 472 | }
|
| 473 | }
|
| 474 |
|
| 475 | |
| 476 | * This routine is called by the ICMP module when it gets some
|
| 477 | * sort of error condition. If err < 0 then the socket should
|
| 478 | * be closed and the error returned to the user. If err > 0
|
| 479 | * it's just the icmp type << 8 | icmp code. After adjustment
|
| 480 | * header points to the first 8 bytes of the tcp header. We need
|
| 481 | * to find the appropriate port.
|
| 482 | *
|
| 483 | * The locking strategy used here is very "optimistic". When
|
| 484 | * someone else accesses the socket the ICMP is just dropped
|
| 485 | * and for some paths there is no check at all.
|
| 486 | * A more general error queue to queue errors for later handling
|
| 487 | * is probably better.
|
| 488 | *
|
| 489 |
|
| 490 |
|
| 491 | int tcp_v4_err(struct sk_buff *skb, u32 info)
|
| 492 | {
|
| 493 | const struct iphdr *iph = (const struct iphdr *)skb->data;
|
| 494 | struct tcphdr *th = (struct tcphdr *)(skb->data + (iph->ihl << 2));
|
| 495 | struct net *net = dev_net_rcu(skb->dev);
|
| 496 | const int type = icmp_hdr(skb)->type;
|
| 497 | const int code = icmp_hdr(skb)->code;
|
| 498 | struct request_sock *fastopen;
|
| 499 | struct tcp_sock *tp;
|
| 500 | u32 seq, snd_una;
|
| 501 | struct sock *sk;
|
| 502 | int err;
|
| 503 |
|
| 504 | sk = __inet_lookup_established(net, iph->daddr, th->dest, iph->saddr,
|
| 505 | ntohs(th->source), inet_iif(skb), 0);
|
| 506 | if (!sk) {
|
| 507 | __ICMP_INC_STATS(net, ICMP_MIB_INERRORS);
|
| 508 | return -ENOENT;
|
| 509 | }
|
| 510 | if (sk->sk_state == TCP_TIME_WAIT) {
|
| 511 |
|
| 512 | tcp_ao_ignore_icmp(sk, AF_INET, type, code);
|
| 513 | inet_twsk_put(inet_twsk(sk));
|
| 514 | return 0;
|
| 515 | }
|
| 516 | seq = ntohl(th->seq);
|
| 517 | if (sk->sk_state == TCP_NEW_SYN_RECV) {
|
| 518 | tcp_req_err(sk, seq, type == ICMP_PARAMETERPROB ||
|
| 519 | type == ICMP_TIME_EXCEEDED ||
|
| 520 | (type == ICMP_DEST_UNREACH &&
|
| 521 | (code == ICMP_NET_UNREACH ||
|
| 522 | code == ICMP_HOST_UNREACH)));
|
| 523 | return 0;
|
| 524 | }
|
| 525 |
|
| 526 | if (tcp_ao_ignore_icmp(sk, AF_INET, type, code)) {
|
| 527 | sock_put(sk);
|
| 528 | return 0;
|
| 529 | }
|
| 530 |
|
| 531 | bh_lock_sock(sk);
|
| 532 | |
| 533 | * servers this needs to be solved differently.
|
| 534 | * We do take care of PMTU discovery (RFC1191) special case :
|
| 535 | * we can receive locally generated ICMP messages while socket is held.
|
| 536 |
|
| 537 | if (sock_owned_by_user(sk)) {
|
| 538 | if (!(type == ICMP_DEST_UNREACH && code == ICMP_FRAG_NEEDED))
|
| 539 | __NET_INC_STATS(net, LINUX_MIB_LOCKDROPPEDICMPS);
|
| 540 | }
|
| 541 | if (sk->sk_state == TCP_CLOSE)
|
| 542 | goto out;
|
| 543 |
|
| 544 | if (static_branch_unlikely(&ip4_min_ttl)) {
|
| 545 |
|
| 546 | if (unlikely(iph->ttl < READ_ONCE(inet_sk(sk)->min_ttl))) {
|
| 547 | __NET_INC_STATS(net, LINUX_MIB_TCPMINTTLDROP);
|
| 548 | goto out;
|
| 549 | }
|
| 550 | }
|
| 551 |
|
| 552 | tp = tcp_sk(sk);
|
| 553 |
|
| 554 | fastopen = rcu_dereference(tp->fastopen_rsk);
|
| 555 | snd_una = fastopen ? tcp_rsk(fastopen)->snt_isn : tp->snd_una;
|
| 556 | if (sk->sk_state != TCP_LISTEN &&
|
| 557 | !between(seq, snd_una, tp->snd_nxt)) {
|
| 558 | __NET_INC_STATS(net, LINUX_MIB_OUTOFWINDOWICMPS);
|
| 559 | goto out;
|
| 560 | }
|
| 561 |
|
| 562 | switch (type) {
|
| 563 | case ICMP_REDIRECT:
|
| 564 | if (!sock_owned_by_user(sk))
|
| 565 | do_redirect(skb, sk);
|
| 566 | goto out;
|
| 567 | case ICMP_SOURCE_QUENCH:
|
| 568 |
|
| 569 | goto out;
|
| 570 | case ICMP_PARAMETERPROB:
|
| 571 | err = EPROTO;
|
| 572 | break;
|
| 573 | case ICMP_DEST_UNREACH:
|
| 574 | if (code > NR_ICMP_UNREACH)
|
| 575 | goto out;
|
| 576 |
|
| 577 | if (code == ICMP_FRAG_NEEDED) {
|
| 578 | |
| 579 | * (SYN-ACKs send out by Linux are always <576bytes so
|
| 580 | * they should go through unfragmented).
|
| 581 |
|
| 582 | if (sk->sk_state == TCP_LISTEN)
|
| 583 | goto out;
|
| 584 |
|
| 585 | WRITE_ONCE(tp->mtu_info, info);
|
| 586 | if (!sock_owned_by_user(sk)) {
|
| 587 | tcp_v4_mtu_reduced(sk);
|
| 588 | } else {
|
| 589 | if (!test_and_set_bit(TCP_MTU_REDUCED_DEFERRED, &sk->sk_tsq_flags))
|
| 590 | sock_hold(sk);
|
| 591 | }
|
| 592 | goto out;
|
| 593 | }
|
| 594 |
|
| 595 | err = icmp_err_convert[code].errno;
|
| 596 | |
| 597 | * (see RFC 6069)
|
| 598 |
|
| 599 | if (!fastopen &&
|
| 600 | (code == ICMP_NET_UNREACH || code == ICMP_HOST_UNREACH))
|
| 601 | tcp_ld_RTO_revert(sk, seq);
|
| 602 | break;
|
| 603 | case ICMP_TIME_EXCEEDED:
|
| 604 | err = EHOSTUNREACH;
|
| 605 | break;
|
| 606 | default:
|
| 607 | goto out;
|
| 608 | }
|
| 609 |
|
| 610 | switch (sk->sk_state) {
|
| 611 | case TCP_SYN_SENT:
|
| 612 | case TCP_SYN_RECV:
|
| 613 | |
| 614 | * already accepted it is treated as a connected one below.
|
| 615 |
|
| 616 | if (fastopen && !fastopen->sk)
|
| 617 | break;
|
| 618 |
|
| 619 | ip_icmp_error(sk, skb, err, th->dest, info, (u8 *)th);
|
| 620 |
|
| 621 | if (!sock_owned_by_user(sk))
|
| 622 | tcp_done_with_error(sk, err);
|
| 623 | else
|
| 624 | WRITE_ONCE(sk->sk_err_soft, err);
|
| 625 | goto out;
|
| 626 | }
|
| 627 |
|
| 628 | |
| 629 | * until we time out, or the user gives up.
|
| 630 | *
|
| 631 | * rfc1122 4.2.3.9 allows to consider as hard errors
|
| 632 | * only PROTO_UNREACH and PORT_UNREACH (well, FRAG_FAILED too,
|
| 633 | * but it is obsoleted by pmtu discovery).
|
| 634 | *
|
| 635 | * Note, that in modern internet, where routing is unreliable
|
| 636 | * and in each dark corner broken firewalls sit, sending random
|
| 637 | * errors ordered by their masters even this two messages finally lose
|
| 638 | * their original sense (even Linux sends invalid PORT_UNREACHs)
|
| 639 | *
|
| 640 | * Now we are in compliance with RFCs.
|
| 641 | * --ANK (980905)
|
| 642 |
|
| 643 |
|
| 644 | if (!sock_owned_by_user(sk) &&
|
| 645 | inet_test_bit(RECVERR, sk)) {
|
| 646 | WRITE_ONCE(sk->sk_err, err);
|
| 647 | sk_error_report(sk);
|
| 648 | } else {
|
| 649 | WRITE_ONCE(sk->sk_err_soft, err);
|
| 650 | }
|
| 651 |
|
| 652 | out:
|
| 653 | bh_unlock_sock(sk);
|
| 654 | sock_put(sk);
|
| 655 | return 0;
|
| 656 | }
|
| 657 |
|
| 658 | #define REPLY_OPTIONS_LEN (MAX_TCP_OPTION_SPACE / sizeof(__be32))
|
| 659 |
|
| 660 | static bool tcp_v4_ao_sign_reset(const struct sock *sk, struct sk_buff *skb,
|
| 661 | const struct tcp_ao_hdr *aoh,
|
| 662 | struct ip_reply_arg *arg, struct tcphdr *reply,
|
| 663 | __be32 reply_options[REPLY_OPTIONS_LEN])
|
| 664 | {
|
| 665 | #ifdef CONFIG_TCP_AO
|
| 666 | int sdif = tcp_v4_sdif(skb);
|
| 667 | int dif = inet_iif(skb);
|
| 668 | int l3index = sdif ? dif : 0;
|
| 669 | bool allocated_traffic_key;
|
| 670 | struct tcp_ao_key *key;
|
| 671 | char *traffic_key;
|
| 672 | bool drop = true;
|
| 673 | u32 ao_sne = 0;
|
| 674 | u8 keyid;
|
| 675 |
|
| 676 | rcu_read_lock();
|
| 677 | if (tcp_ao_prepare_reset(sk, skb, aoh, l3index, ntohl(reply->seq),
|
| 678 | &key, &traffic_key, &allocated_traffic_key,
|
| 679 | &keyid, &ao_sne))
|
| 680 | goto out;
|
| 681 |
|
| 682 | reply_options[0] = htonl((TCPOPT_AO << 24) | (tcp_ao_len(key) << 16) |
|
| 683 | (aoh->rnext_keyid << 8) | keyid);
|
| 684 | arg->iov[0].iov_len += tcp_ao_len_aligned(key);
|
| 685 | reply->doff = arg->iov[0].iov_len / 4;
|
| 686 |
|
| 687 | if (tcp_ao_hash_hdr(AF_INET, (char *)&reply_options[1],
|
| 688 | key, traffic_key,
|
| 689 | (union tcp_ao_addr *)&ip_hdr(skb)->saddr,
|
| 690 | (union tcp_ao_addr *)&ip_hdr(skb)->daddr,
|
| 691 | reply, ao_sne))
|
| 692 | goto out;
|
| 693 | drop = false;
|
| 694 | out:
|
| 695 | rcu_read_unlock();
|
| 696 | if (allocated_traffic_key)
|
| 697 | kfree(traffic_key);
|
| 698 | return drop;
|
| 699 | #else
|
| 700 | return true;
|
| 701 | #endif
|
| 702 | }
|
| 703 |
|
| 704 | |
| 705 | * This routine will send an RST to the other tcp.
|
| 706 | *
|
| 707 | * Someone asks: why I NEVER use socket parameters (TOS, TTL etc.)
|
| 708 | * for reset.
|
| 709 | * Answer: if a packet caused RST, it is not for a socket
|
| 710 | * existing in our system, if it is matched to a socket,
|
| 711 | * it is just duplicate segment or bug in other side's TCP.
|
| 712 | * So that we build reply only basing on parameters
|
| 713 | * arrived with segment.
|
| 714 | * Exception: precedence violation. We do not implement it in any case.
|
| 715 |
|
| 716 |
|
| 717 | static void tcp_v4_send_reset(const struct sock *sk, struct sk_buff *skb,
|
| 718 | enum sk_rst_reason reason)
|
| 719 | {
|
| 720 | const struct tcphdr *th = tcp_hdr(skb);
|
| 721 | struct {
|
| 722 | struct tcphdr th;
|
| 723 | __be32 opt[REPLY_OPTIONS_LEN];
|
| 724 | } rep;
|
| 725 | const __u8 *md5_hash_location = NULL;
|
| 726 | const struct tcp_ao_hdr *aoh;
|
| 727 | struct ip_reply_arg arg;
|
| 728 | #ifdef CONFIG_TCP_MD5SIG
|
| 729 | struct tcp_md5sig_key *key = NULL;
|
| 730 | unsigned char newhash[16];
|
| 731 | struct sock *sk1 = NULL;
|
| 732 | #endif
|
| 733 | u64 transmit_time = 0;
|
| 734 | struct sock *ctl_sk;
|
| 735 | struct net *net;
|
| 736 | u32 txhash = 0;
|
| 737 |
|
| 738 |
|
| 739 | if (th->rst)
|
| 740 | return;
|
| 741 |
|
| 742 | |
| 743 | * route had to be correct. prequeue might have dropped our dst.
|
| 744 |
|
| 745 | if (!sk && skb_rtable(skb)->rt_type != RTN_LOCAL)
|
| 746 | return;
|
| 747 |
|
| 748 |
|
| 749 | memset(&rep, 0, sizeof(rep));
|
| 750 | rep.th.dest = th->source;
|
| 751 | rep.th.source = th->dest;
|
| 752 | rep.th.doff = sizeof(struct tcphdr) / 4;
|
| 753 | rep.th.rst = 1;
|
| 754 |
|
| 755 | if (th->ack) {
|
| 756 | rep.th.seq = th->ack_seq;
|
| 757 | } else {
|
| 758 | rep.th.ack = 1;
|
| 759 | rep.th.ack_seq = htonl(ntohl(th->seq) + th->syn + th->fin +
|
| 760 | skb->len - (th->doff << 2));
|
| 761 | }
|
| 762 |
|
| 763 | memset(&arg, 0, sizeof(arg));
|
| 764 | arg.iov[0].iov_base = (unsigned char *)&rep;
|
| 765 | arg.iov[0].iov_len = sizeof(rep.th);
|
| 766 |
|
| 767 | net = sk ? sock_net(sk) : skb_dst_dev_net_rcu(skb);
|
| 768 |
|
| 769 |
|
| 770 | if (tcp_parse_auth_options(tcp_hdr(skb), &md5_hash_location, &aoh))
|
| 771 | return;
|
| 772 |
|
| 773 | if (aoh && tcp_v4_ao_sign_reset(sk, skb, aoh, &arg, &rep.th, rep.opt))
|
| 774 | return;
|
| 775 |
|
| 776 | #ifdef CONFIG_TCP_MD5SIG
|
| 777 | rcu_read_lock();
|
| 778 | if (sk && sk_fullsock(sk)) {
|
| 779 | const union tcp_md5_addr *addr;
|
| 780 | int l3index;
|
| 781 |
|
| 782 | |
| 783 | * in an L3 domain and inet_iif is set to it.
|
| 784 |
|
| 785 | l3index = tcp_v4_sdif(skb) ? inet_iif(skb) : 0;
|
| 786 | addr = (union tcp_md5_addr *)&ip_hdr(skb)->saddr;
|
| 787 | key = tcp_md5_do_lookup(sk, l3index, addr, AF_INET);
|
| 788 | } else if (md5_hash_location) {
|
| 789 | const union tcp_md5_addr *addr;
|
| 790 | int sdif = tcp_v4_sdif(skb);
|
| 791 | int dif = inet_iif(skb);
|
| 792 | int l3index;
|
| 793 |
|
| 794 | |
| 795 | * active side is lost. Try to find listening socket through
|
| 796 | * source port, and then find md5 key through listening socket.
|
| 797 | * we are not loose security here:
|
| 798 | * Incoming packet is checked with md5 hash with finding key,
|
| 799 | * no RST generated if md5 hash doesn't match.
|
| 800 |
|
| 801 | sk1 = __inet_lookup_listener(net, NULL, 0, ip_hdr(skb)->saddr,
|
| 802 | th->source, ip_hdr(skb)->daddr,
|
| 803 | ntohs(th->source), dif, sdif);
|
| 804 |
|
| 805 | if (!sk1)
|
| 806 | goto out;
|
| 807 |
|
| 808 | |
| 809 | * in an L3 domain and dif is set to it.
|
| 810 |
|
| 811 | l3index = sdif ? dif : 0;
|
| 812 | addr = (union tcp_md5_addr *)&ip_hdr(skb)->saddr;
|
| 813 | key = tcp_md5_do_lookup(sk1, l3index, addr, AF_INET);
|
| 814 | if (!key)
|
| 815 | goto out;
|
| 816 |
|
| 817 | tcp_v4_md5_hash_skb(newhash, key, NULL, skb);
|
| 818 | if (crypto_memneq(md5_hash_location, newhash, 16))
|
| 819 | goto out;
|
| 820 | }
|
| 821 |
|
| 822 | if (key) {
|
| 823 | rep.opt[0] = htonl((TCPOPT_NOP << 24) |
|
| 824 | (TCPOPT_NOP << 16) |
|
| 825 | (TCPOPT_MD5SIG << 8) |
|
| 826 | TCPOLEN_MD5SIG);
|
| 827 |
|
| 828 | arg.iov[0].iov_len += TCPOLEN_MD5SIG_ALIGNED;
|
| 829 | rep.th.doff = arg.iov[0].iov_len / 4;
|
| 830 |
|
| 831 | tcp_v4_md5_hash_hdr((__u8 *) &rep.opt[1],
|
| 832 | key, ip_hdr(skb)->saddr,
|
| 833 | ip_hdr(skb)->daddr, &rep.th);
|
| 834 | }
|
| 835 | #endif
|
| 836 |
|
| 837 | if (rep.opt[0] == 0) {
|
| 838 | __be32 mrst = mptcp_reset_option(skb);
|
| 839 |
|
| 840 | if (mrst) {
|
| 841 | rep.opt[0] = mrst;
|
| 842 | arg.iov[0].iov_len += sizeof(mrst);
|
| 843 | rep.th.doff = arg.iov[0].iov_len / 4;
|
| 844 | }
|
| 845 | }
|
| 846 |
|
| 847 | arg.csum = csum_tcpudp_nofold(ip_hdr(skb)->daddr,
|
| 848 | ip_hdr(skb)->saddr,
|
| 849 | arg.iov[0].iov_len, IPPROTO_TCP, 0);
|
| 850 | arg.csumoffset = offsetof(struct tcphdr, check) / 2;
|
| 851 | arg.flags = (sk && inet_sk_transparent(sk)) ? IP_REPLY_ARG_NOSRCCHECK : 0;
|
| 852 |
|
| 853 | |
| 854 | * routing might fail in this case. No choice here, if we choose to force
|
| 855 | * input interface, we will misroute in case of asymmetric route.
|
| 856 |
|
| 857 | if (sk)
|
| 858 | arg.bound_dev_if = sk->sk_bound_dev_if;
|
| 859 |
|
| 860 | trace_tcp_send_reset(sk, skb, reason);
|
| 861 |
|
| 862 | BUILD_BUG_ON(offsetof(struct sock, sk_bound_dev_if) !=
|
| 863 | offsetof(struct inet_timewait_sock, tw_bound_dev_if));
|
| 864 |
|
| 865 |
|
| 866 | arg.tos = ip_hdr(skb)->tos & ~INET_ECN_MASK;
|
| 867 | arg.uid = sock_net_uid(net, sk && sk_fullsock(sk) ? sk : NULL);
|
| 868 | local_bh_disable();
|
| 869 | local_lock_nested_bh(&ipv4_tcp_sk.bh_lock);
|
| 870 | ctl_sk = this_cpu_read(ipv4_tcp_sk.sock);
|
| 871 |
|
| 872 | sock_net_set(ctl_sk, net);
|
| 873 | if (sk) {
|
| 874 | ctl_sk->sk_mark = (sk->sk_state == TCP_TIME_WAIT) ?
|
| 875 | inet_twsk(sk)->tw_mark : READ_ONCE(sk->sk_mark);
|
| 876 | ctl_sk->sk_priority = (sk->sk_state == TCP_TIME_WAIT) ?
|
| 877 | inet_twsk(sk)->tw_priority : READ_ONCE(sk->sk_priority);
|
| 878 | transmit_time = tcp_transmit_time(sk);
|
| 879 | xfrm_sk_clone_policy(ctl_sk, sk);
|
| 880 | txhash = (sk->sk_state == TCP_TIME_WAIT) ?
|
| 881 | inet_twsk(sk)->tw_txhash : sk->sk_txhash;
|
| 882 | } else {
|
| 883 | ctl_sk->sk_mark = 0;
|
| 884 | ctl_sk->sk_priority = 0;
|
| 885 | }
|
| 886 | ip_send_unicast_reply(ctl_sk, sk,
|
| 887 | skb, &TCP_SKB_CB(skb)->header.h4.opt,
|
| 888 | ip_hdr(skb)->saddr, ip_hdr(skb)->daddr,
|
| 889 | &arg, arg.iov[0].iov_len,
|
| 890 | transmit_time, txhash);
|
| 891 |
|
| 892 | xfrm_sk_free_policy(ctl_sk);
|
| 893 | sock_net_set(ctl_sk, &init_net);
|
| 894 | __TCP_INC_STATS(net, TCP_MIB_OUTSEGS);
|
| 895 | __TCP_INC_STATS(net, TCP_MIB_OUTRSTS);
|
| 896 | local_unlock_nested_bh(&ipv4_tcp_sk.bh_lock);
|
| 897 | local_bh_enable();
|
| 898 |
|
| 899 | #ifdef CONFIG_TCP_MD5SIG
|
| 900 | out:
|
| 901 | rcu_read_unlock();
|
| 902 | #endif
|
| 903 | }
|
| 904 |
|
| 905 | |
| 906 | outside socket context is ugly, certainly. What can I do?
|
| 907 |
|
| 908 |
|
| 909 | static void tcp_v4_send_ack(const struct sock *sk,
|
| 910 | struct sk_buff *skb, u32 seq, u32 ack,
|
| 911 | u32 win, u32 tsval, u32 tsecr, int oif,
|
| 912 | struct tcp_key *key,
|
| 913 | int reply_flags, u8 tos, u32 txhash)
|
| 914 | {
|
| 915 | const struct tcphdr *th = tcp_hdr(skb);
|
| 916 | struct {
|
| 917 | struct tcphdr th;
|
| 918 | __be32 opt[(MAX_TCP_OPTION_SPACE >> 2)];
|
| 919 | } rep;
|
| 920 | struct net *net = sock_net(sk);
|
| 921 | struct ip_reply_arg arg;
|
| 922 | struct sock *ctl_sk;
|
| 923 | u64 transmit_time;
|
| 924 |
|
| 925 | memset(&rep.th, 0, sizeof(struct tcphdr));
|
| 926 | memset(&arg, 0, sizeof(arg));
|
| 927 |
|
| 928 | arg.iov[0].iov_base = (unsigned char *)&rep;
|
| 929 | arg.iov[0].iov_len = sizeof(rep.th);
|
| 930 | if (tsecr) {
|
| 931 | rep.opt[0] = htonl((TCPOPT_NOP << 24) | (TCPOPT_NOP << 16) |
|
| 932 | (TCPOPT_TIMESTAMP << 8) |
|
| 933 | TCPOLEN_TIMESTAMP);
|
| 934 | rep.opt[1] = htonl(tsval);
|
| 935 | rep.opt[2] = htonl(tsecr);
|
| 936 | arg.iov[0].iov_len += TCPOLEN_TSTAMP_ALIGNED;
|
| 937 | }
|
| 938 |
|
| 939 |
|
| 940 | rep.th.dest = th->source;
|
| 941 | rep.th.source = th->dest;
|
| 942 | rep.th.doff = arg.iov[0].iov_len / 4;
|
| 943 | rep.th.seq = htonl(seq);
|
| 944 | rep.th.ack_seq = htonl(ack);
|
| 945 | rep.th.ack = 1;
|
| 946 | rep.th.window = htons(win);
|
| 947 |
|
| 948 | #ifdef CONFIG_TCP_MD5SIG
|
| 949 | if (tcp_key_is_md5(key)) {
|
| 950 | int offset = (tsecr) ? 3 : 0;
|
| 951 |
|
| 952 | rep.opt[offset++] = htonl((TCPOPT_NOP << 24) |
|
| 953 | (TCPOPT_NOP << 16) |
|
| 954 | (TCPOPT_MD5SIG << 8) |
|
| 955 | TCPOLEN_MD5SIG);
|
| 956 | arg.iov[0].iov_len += TCPOLEN_MD5SIG_ALIGNED;
|
| 957 | rep.th.doff = arg.iov[0].iov_len/4;
|
| 958 |
|
| 959 | tcp_v4_md5_hash_hdr((__u8 *) &rep.opt[offset],
|
| 960 | key->md5_key, ip_hdr(skb)->saddr,
|
| 961 | ip_hdr(skb)->daddr, &rep.th);
|
| 962 | }
|
| 963 | #endif
|
| 964 | #ifdef CONFIG_TCP_AO
|
| 965 | if (tcp_key_is_ao(key)) {
|
| 966 | int offset = (tsecr) ? 3 : 0;
|
| 967 |
|
| 968 | rep.opt[offset++] = htonl((TCPOPT_AO << 24) |
|
| 969 | (tcp_ao_len(key->ao_key) << 16) |
|
| 970 | (key->ao_key->sndid << 8) |
|
| 971 | key->rcv_next);
|
| 972 | arg.iov[0].iov_len += tcp_ao_len_aligned(key->ao_key);
|
| 973 | rep.th.doff = arg.iov[0].iov_len / 4;
|
| 974 | memset((u8 *)&rep.opt[offset] + tcp_ao_maclen(key->ao_key),
|
| 975 | TCPOPT_NOP, tcp_ao_len_aligned(key->ao_key) -
|
| 976 | tcp_ao_len(key->ao_key));
|
| 977 |
|
| 978 | tcp_ao_hash_hdr(AF_INET, (char *)&rep.opt[offset],
|
| 979 | key->ao_key, key->traffic_key,
|
| 980 | (union tcp_ao_addr *)&ip_hdr(skb)->saddr,
|
| 981 | (union tcp_ao_addr *)&ip_hdr(skb)->daddr,
|
| 982 | &rep.th, key->sne);
|
| 983 | }
|
| 984 | #endif
|
| 985 | arg.flags = reply_flags;
|
| 986 | arg.csum = csum_tcpudp_nofold(ip_hdr(skb)->daddr,
|
| 987 | ip_hdr(skb)->saddr,
|
| 988 | arg.iov[0].iov_len, IPPROTO_TCP, 0);
|
| 989 | arg.csumoffset = offsetof(struct tcphdr, check) / 2;
|
| 990 | if (oif)
|
| 991 | arg.bound_dev_if = oif;
|
| 992 | arg.tos = tos;
|
| 993 | arg.uid = sock_net_uid(net, sk_fullsock(sk) ? sk : NULL);
|
| 994 | local_bh_disable();
|
| 995 | local_lock_nested_bh(&ipv4_tcp_sk.bh_lock);
|
| 996 | ctl_sk = this_cpu_read(ipv4_tcp_sk.sock);
|
| 997 | sock_net_set(ctl_sk, net);
|
| 998 | ctl_sk->sk_mark = (sk->sk_state == TCP_TIME_WAIT) ?
|
| 999 | inet_twsk(sk)->tw_mark : READ_ONCE(sk->sk_mark);
|
| 1000 | ctl_sk->sk_priority = (sk->sk_state == TCP_TIME_WAIT) ?
|
| 1001 | inet_twsk(sk)->tw_priority : READ_ONCE(sk->sk_priority);
|
| 1002 | transmit_time = tcp_transmit_time(sk);
|
| 1003 | ip_send_unicast_reply(ctl_sk, sk,
|
| 1004 | skb, &TCP_SKB_CB(skb)->header.h4.opt,
|
| 1005 | ip_hdr(skb)->saddr, ip_hdr(skb)->daddr,
|
| 1006 | &arg, arg.iov[0].iov_len,
|
| 1007 | transmit_time, txhash);
|
| 1008 |
|
| 1009 | sock_net_set(ctl_sk, &init_net);
|
| 1010 | __TCP_INC_STATS(net, TCP_MIB_OUTSEGS);
|
| 1011 | local_unlock_nested_bh(&ipv4_tcp_sk.bh_lock);
|
| 1012 | local_bh_enable();
|
| 1013 | }
|
| 1014 |
|
| 1015 | static void tcp_v4_timewait_ack(struct sock *sk, struct sk_buff *skb,
|
| 1016 | enum tcp_tw_status tw_status)
|
| 1017 | {
|
| 1018 | struct inet_timewait_sock *tw = inet_twsk(sk);
|
| 1019 | struct tcp_timewait_sock *tcptw = tcp_twsk(sk);
|
| 1020 | struct tcp_key key = {};
|
| 1021 | u8 tos = tw->tw_tos;
|
| 1022 |
|
| 1023 | |
| 1024 | * while not cleaning ECN bits of other TW ACKs to avoid these ACKs
|
| 1025 | * being placed in a different service queues (Classic rather than L4S)
|
| 1026 |
|
| 1027 | if (tw_status == TCP_TW_ACK_OOW)
|
| 1028 | tos &= ~INET_ECN_MASK;
|
| 1029 |
|
| 1030 | #ifdef CONFIG_TCP_AO
|
| 1031 | struct tcp_ao_info *ao_info;
|
| 1032 |
|
| 1033 | if (static_branch_unlikely(&tcp_ao_needed.key)) {
|
| 1034 |
|
| 1035 | ao_info = rcu_dereference(tcptw->ao_info);
|
| 1036 | if (ao_info) {
|
| 1037 | const struct tcp_ao_hdr *aoh;
|
| 1038 |
|
| 1039 | if (tcp_parse_auth_options(tcp_hdr(skb), NULL, &aoh)) {
|
| 1040 | inet_twsk_put(tw);
|
| 1041 | return;
|
| 1042 | }
|
| 1043 |
|
| 1044 | if (aoh)
|
| 1045 | key.ao_key = tcp_ao_established_key(sk, ao_info,
|
| 1046 | aoh->rnext_keyid, -1);
|
| 1047 | }
|
| 1048 | }
|
| 1049 | if (key.ao_key) {
|
| 1050 | struct tcp_ao_key *rnext_key;
|
| 1051 |
|
| 1052 | key.traffic_key = snd_other_key(key.ao_key);
|
| 1053 | key.sne = READ_ONCE(ao_info->snd_sne);
|
| 1054 | rnext_key = READ_ONCE(ao_info->rnext_key);
|
| 1055 | key.rcv_next = rnext_key->rcvid;
|
| 1056 | key.type = TCP_KEY_AO;
|
| 1057 | #else
|
| 1058 | if (0) {
|
| 1059 | #endif
|
| 1060 | } else if (static_branch_tcp_md5()) {
|
| 1061 | key.md5_key = tcp_twsk_md5_key(tcptw);
|
| 1062 | if (key.md5_key)
|
| 1063 | key.type = TCP_KEY_MD5;
|
| 1064 | }
|
| 1065 |
|
| 1066 | tcp_v4_send_ack(sk, skb,
|
| 1067 | tcptw->tw_snd_nxt, READ_ONCE(tcptw->tw_rcv_nxt),
|
| 1068 | tcptw->tw_rcv_wnd >> tw->tw_rcv_wscale,
|
| 1069 | tcp_tw_tsval(tcptw),
|
| 1070 | READ_ONCE(tcptw->tw_ts_recent),
|
| 1071 | tw->tw_bound_dev_if, &key,
|
| 1072 | tw->tw_transparent ? IP_REPLY_ARG_NOSRCCHECK : 0,
|
| 1073 | tos,
|
| 1074 | tw->tw_txhash);
|
| 1075 |
|
| 1076 | inet_twsk_put(tw);
|
| 1077 | }
|
| 1078 |
|
| 1079 | static void tcp_v4_reqsk_send_ack(const struct sock *sk, struct sk_buff *skb,
|
| 1080 | struct request_sock *req)
|
| 1081 | {
|
| 1082 | struct tcp_key key = {};
|
| 1083 |
|
| 1084 | |
| 1085 | * sk->sk_state == TCP_SYN_RECV -> for Fast Open.
|
| 1086 |
|
| 1087 | u32 seq = (sk->sk_state == TCP_LISTEN) ? tcp_rsk(req)->snt_isn + 1 :
|
| 1088 | tcp_sk(sk)->snd_nxt;
|
| 1089 |
|
| 1090 | #ifdef CONFIG_TCP_AO
|
| 1091 | if (static_branch_unlikely(&tcp_ao_needed.key) &&
|
| 1092 | tcp_rsk_used_ao(req)) {
|
| 1093 | const union tcp_md5_addr *addr;
|
| 1094 | const struct tcp_ao_hdr *aoh;
|
| 1095 | int l3index;
|
| 1096 |
|
| 1097 |
|
| 1098 | if (tcp_parse_auth_options(tcp_hdr(skb), NULL, &aoh))
|
| 1099 | return;
|
| 1100 | if (!aoh)
|
| 1101 | return;
|
| 1102 |
|
| 1103 | addr = (union tcp_md5_addr *)&ip_hdr(skb)->saddr;
|
| 1104 | l3index = tcp_v4_sdif(skb) ? inet_iif(skb) : 0;
|
| 1105 | key.ao_key = tcp_ao_do_lookup(sk, l3index, addr, AF_INET,
|
| 1106 | aoh->rnext_keyid, -1);
|
| 1107 | if (unlikely(!key.ao_key)) {
|
| 1108 |
|
| 1109 | key.ao_key = tcp_ao_do_lookup(sk, l3index, addr, AF_INET, -1, -1);
|
| 1110 | |
| 1111 | * let the handshake timeout.
|
| 1112 |
|
| 1113 | if (!key.ao_key) {
|
| 1114 | net_info_ratelimited("TCP-AO key for (%pI4, %d)->(%pI4, %d) suddenly disappeared, won't ACK new connection\n",
|
| 1115 | addr,
|
| 1116 | ntohs(tcp_hdr(skb)->source),
|
| 1117 | &ip_hdr(skb)->daddr,
|
| 1118 | ntohs(tcp_hdr(skb)->dest));
|
| 1119 | return;
|
| 1120 | }
|
| 1121 | }
|
| 1122 | key.traffic_key = kmalloc(tcp_ao_digest_size(key.ao_key), GFP_ATOMIC);
|
| 1123 | if (!key.traffic_key)
|
| 1124 | return;
|
| 1125 |
|
| 1126 | key.type = TCP_KEY_AO;
|
| 1127 | key.rcv_next = aoh->keyid;
|
| 1128 | tcp_v4_ao_calc_key_rsk(key.ao_key, key.traffic_key, req);
|
| 1129 | #else
|
| 1130 | if (0) {
|
| 1131 | #endif
|
| 1132 | } else if (static_branch_tcp_md5()) {
|
| 1133 | const union tcp_md5_addr *addr;
|
| 1134 | int l3index;
|
| 1135 |
|
| 1136 | addr = (union tcp_md5_addr *)&ip_hdr(skb)->saddr;
|
| 1137 | l3index = tcp_v4_sdif(skb) ? inet_iif(skb) : 0;
|
| 1138 | key.md5_key = tcp_md5_do_lookup(sk, l3index, addr, AF_INET);
|
| 1139 | if (key.md5_key)
|
| 1140 | key.type = TCP_KEY_MD5;
|
| 1141 | }
|
| 1142 |
|
| 1143 |
|
| 1144 | tcp_v4_send_ack(sk, skb, seq,
|
| 1145 | tcp_rsk(req)->rcv_nxt,
|
| 1146 | tcp_synack_window(req) >> inet_rsk(req)->rcv_wscale,
|
| 1147 | tcp_rsk_tsval(tcp_rsk(req)),
|
| 1148 | req->ts_recent,
|
| 1149 | 0, &key,
|
| 1150 | inet_rsk(req)->no_srccheck ? IP_REPLY_ARG_NOSRCCHECK : 0,
|
| 1151 | ip_hdr(skb)->tos & ~INET_ECN_MASK,
|
| 1152 | READ_ONCE(tcp_rsk(req)->txhash));
|
| 1153 | if (tcp_key_is_ao(&key))
|
| 1154 | kfree(key.traffic_key);
|
| 1155 | }
|
| 1156 |
|
| 1157 | |
| 1158 | * Send a SYN-ACK after having received a SYN.
|
| 1159 | * This still operates on a request_sock only, not on a big
|
| 1160 | * socket.
|
| 1161 |
|
| 1162 | static int tcp_v4_send_synack(const struct sock *sk, struct dst_entry *dst,
|
| 1163 | struct flowi *fl,
|
| 1164 | struct request_sock *req,
|
| 1165 | struct tcp_fastopen_cookie *foc,
|
| 1166 | enum tcp_synack_type synack_type,
|
| 1167 | struct sk_buff *syn_skb)
|
| 1168 | {
|
| 1169 | struct inet_request_sock *ireq = inet_rsk(req);
|
| 1170 | struct flowi4 fl4;
|
| 1171 | int err = -1;
|
| 1172 | struct sk_buff *skb;
|
| 1173 | u8 tos;
|
| 1174 |
|
| 1175 |
|
| 1176 | if (!dst && (dst = inet_csk_route_req(sk, &fl4, req)) == NULL)
|
| 1177 | return -1;
|
| 1178 |
|
| 1179 | skb = tcp_make_synack(sk, dst, req, foc, synack_type, syn_skb);
|
| 1180 |
|
| 1181 | if (skb) {
|
| 1182 | tcp_rsk(req)->syn_ect_snt = inet_sk(sk)->tos & INET_ECN_MASK;
|
| 1183 | __tcp_v4_send_check(skb, ireq->ir_loc_addr, ireq->ir_rmt_addr);
|
| 1184 |
|
| 1185 | tos = READ_ONCE(inet_sk(sk)->tos);
|
| 1186 |
|
| 1187 | if (READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_reflect_tos))
|
| 1188 | tos = (tcp_rsk(req)->syn_tos & ~INET_ECN_MASK) |
|
| 1189 | (tos & INET_ECN_MASK);
|
| 1190 |
|
| 1191 | if (!INET_ECN_is_capable(tos) &&
|
| 1192 | tcp_bpf_ca_needs_ecn((struct sock *)req))
|
| 1193 | tos |= INET_ECN_ECT_0;
|
| 1194 |
|
| 1195 | rcu_read_lock();
|
| 1196 | err = ip_build_and_send_pkt(skb, sk, ireq->ir_loc_addr,
|
| 1197 | ireq->ir_rmt_addr,
|
| 1198 | rcu_dereference(ireq->ireq_opt),
|
| 1199 | tos);
|
| 1200 | rcu_read_unlock();
|
| 1201 | err = net_xmit_eval(err);
|
| 1202 | }
|
| 1203 |
|
| 1204 | return err;
|
| 1205 | }
|
| 1206 |
|
| 1207 | |
| 1208 | * IPv4 request_sock destructor.
|
| 1209 |
|
| 1210 | static void tcp_v4_reqsk_destructor(struct request_sock *req)
|
| 1211 | {
|
| 1212 | kfree_rcu(rcu_dereference_protected(inet_rsk(req)->ireq_opt, 1), rcu);
|
| 1213 | }
|
| 1214 |
|
| 1215 | #ifdef CONFIG_TCP_MD5SIG
|
| 1216 | |
| 1217 | * RFC2385 MD5 checksumming requires a mapping of
|
| 1218 | * IP address->MD5 Key.
|
| 1219 | * We need to maintain these in the sk structure.
|
| 1220 |
|
| 1221 |
|
| 1222 | DEFINE_STATIC_KEY_DEFERRED_FALSE(tcp_md5_needed, HZ);
|
| 1223 |
|
| 1224 | static bool better_md5_match(struct tcp_md5sig_key *old, struct tcp_md5sig_key *new)
|
| 1225 | {
|
| 1226 | if (!old)
|
| 1227 | return true;
|
| 1228 |
|
| 1229 |
|
| 1230 | if (old->l3index && new->l3index == 0)
|
| 1231 | return false;
|
| 1232 | if (old->l3index == 0 && new->l3index)
|
| 1233 | return true;
|
| 1234 |
|
| 1235 | return old->prefixlen < new->prefixlen;
|
| 1236 | }
|
| 1237 |
|
| 1238 |
|
| 1239 | struct tcp_md5sig_key *__tcp_md5_do_lookup(const struct sock *sk, int l3index,
|
| 1240 | const union tcp_md5_addr *addr,
|
| 1241 | int family, bool any_l3index)
|
| 1242 | {
|
| 1243 | const struct tcp_sock *tp = tcp_sk(sk);
|
| 1244 | struct tcp_md5sig_key *key;
|
| 1245 | const struct tcp_md5sig_info *md5sig;
|
| 1246 | __be32 mask;
|
| 1247 | struct tcp_md5sig_key *best_match = NULL;
|
| 1248 | bool match;
|
| 1249 |
|
| 1250 |
|
| 1251 | md5sig = rcu_dereference_check(tp->md5sig_info,
|
| 1252 | lockdep_sock_is_held(sk));
|
| 1253 | if (!md5sig)
|
| 1254 | return NULL;
|
| 1255 |
|
| 1256 | hlist_for_each_entry_rcu(key, &md5sig->head, node,
|
| 1257 | lockdep_sock_is_held(sk)) {
|
| 1258 | if (key->family != family)
|
| 1259 | continue;
|
| 1260 | if (!any_l3index && key->flags & TCP_MD5SIG_FLAG_IFINDEX &&
|
| 1261 | key->l3index != l3index)
|
| 1262 | continue;
|
| 1263 | if (family == AF_INET) {
|
| 1264 | mask = inet_make_mask(key->prefixlen);
|
| 1265 | match = (key->addr.a4.s_addr & mask) ==
|
| 1266 | (addr->a4.s_addr & mask);
|
| 1267 | #if IS_ENABLED(CONFIG_IPV6)
|
| 1268 | } else if (family == AF_INET6) {
|
| 1269 | match = ipv6_prefix_equal(&key->addr.a6, &addr->a6,
|
| 1270 | key->prefixlen);
|
| 1271 | #endif
|
| 1272 | } else {
|
| 1273 | match = false;
|
| 1274 | }
|
| 1275 |
|
| 1276 | if (match && better_md5_match(best_match, key))
|
| 1277 | best_match = key;
|
| 1278 | }
|
| 1279 | return best_match;
|
| 1280 | }
|
| 1281 |
|
| 1282 | static struct tcp_md5sig_key *tcp_md5_do_lookup_exact(const struct sock *sk,
|
| 1283 | const union tcp_md5_addr *addr,
|
| 1284 | int family, u8 prefixlen,
|
| 1285 | int l3index, u8 flags)
|
| 1286 | {
|
| 1287 | const struct tcp_sock *tp = tcp_sk(sk);
|
| 1288 | struct tcp_md5sig_key *key;
|
| 1289 | unsigned int size = sizeof(struct in_addr);
|
| 1290 | const struct tcp_md5sig_info *md5sig;
|
| 1291 |
|
| 1292 |
|
| 1293 | md5sig = rcu_dereference_check(tp->md5sig_info,
|
| 1294 | lockdep_sock_is_held(sk));
|
| 1295 | if (!md5sig)
|
| 1296 | return NULL;
|
| 1297 | #if IS_ENABLED(CONFIG_IPV6)
|
| 1298 | if (family == AF_INET6)
|
| 1299 | size = sizeof(struct in6_addr);
|
| 1300 | #endif
|
| 1301 | hlist_for_each_entry_rcu(key, &md5sig->head, node,
|
| 1302 | lockdep_sock_is_held(sk)) {
|
| 1303 | if (key->family != family)
|
| 1304 | continue;
|
| 1305 | if ((key->flags & TCP_MD5SIG_FLAG_IFINDEX) != (flags & TCP_MD5SIG_FLAG_IFINDEX))
|
| 1306 | continue;
|
| 1307 | if (key->l3index != l3index)
|
| 1308 | continue;
|
| 1309 | if (!memcmp(&key->addr, addr, size) &&
|
| 1310 | key->prefixlen == prefixlen)
|
| 1311 | return key;
|
| 1312 | }
|
| 1313 | return NULL;
|
| 1314 | }
|
| 1315 |
|
| 1316 | struct tcp_md5sig_key *tcp_v4_md5_lookup(const struct sock *sk,
|
| 1317 | const struct sock *addr_sk)
|
| 1318 | {
|
| 1319 | const union tcp_md5_addr *addr;
|
| 1320 | int l3index;
|
| 1321 |
|
| 1322 | l3index = l3mdev_master_ifindex_by_index(sock_net(sk),
|
| 1323 | addr_sk->sk_bound_dev_if);
|
| 1324 | addr = (const union tcp_md5_addr *)&addr_sk->sk_daddr;
|
| 1325 | return tcp_md5_do_lookup(sk, l3index, addr, AF_INET);
|
| 1326 | }
|
| 1327 |
|
| 1328 | static int tcp_md5sig_info_add(struct sock *sk, gfp_t gfp)
|
| 1329 | {
|
| 1330 | struct tcp_sock *tp = tcp_sk(sk);
|
| 1331 | struct tcp_md5sig_info *md5sig;
|
| 1332 |
|
| 1333 | md5sig = kmalloc_obj(*md5sig, gfp);
|
| 1334 | if (!md5sig)
|
| 1335 | return -ENOMEM;
|
| 1336 |
|
| 1337 | sk_gso_disable(sk);
|
| 1338 | INIT_HLIST_HEAD(&md5sig->head);
|
| 1339 | rcu_assign_pointer(tp->md5sig_info, md5sig);
|
| 1340 | return 0;
|
| 1341 | }
|
| 1342 |
|
| 1343 |
|
| 1344 | static int __tcp_md5_do_add(struct sock *sk, const union tcp_md5_addr *addr,
|
| 1345 | int family, u8 prefixlen, int l3index, u8 flags,
|
| 1346 | const u8 *newkey, u8 newkeylen, gfp_t gfp)
|
| 1347 | {
|
| 1348 |
|
| 1349 | struct tcp_md5sig_key *key;
|
| 1350 | struct tcp_sock *tp = tcp_sk(sk);
|
| 1351 | struct tcp_md5sig_info *md5sig;
|
| 1352 |
|
| 1353 | key = tcp_md5_do_lookup_exact(sk, addr, family, prefixlen, l3index, flags);
|
| 1354 | if (key) {
|
| 1355 | |
| 1356 | * Note that the key might be used concurrently.
|
| 1357 | * data_race() is telling kcsan that we do not care of
|
| 1358 | * key mismatches, since changing MD5 key on live flows
|
| 1359 | * can lead to packet drops.
|
| 1360 |
|
| 1361 | data_race(memcpy(key->key, newkey, newkeylen));
|
| 1362 |
|
| 1363 | |
| 1364 | * Also note that a reader could catch new key->keylen value
|
| 1365 | * but old key->key[], this is the reason we use __GFP_ZERO
|
| 1366 | * at sock_kmalloc() time below these lines.
|
| 1367 |
|
| 1368 | WRITE_ONCE(key->keylen, newkeylen);
|
| 1369 |
|
| 1370 | return 0;
|
| 1371 | }
|
| 1372 |
|
| 1373 | md5sig = rcu_dereference_protected(tp->md5sig_info,
|
| 1374 | lockdep_sock_is_held(sk));
|
| 1375 |
|
| 1376 | key = sock_kmalloc(sk, sizeof(*key), gfp | __GFP_ZERO);
|
| 1377 | if (!key)
|
| 1378 | return -ENOMEM;
|
| 1379 |
|
| 1380 | memcpy(key->key, newkey, newkeylen);
|
| 1381 | key->keylen = newkeylen;
|
| 1382 | key->family = family;
|
| 1383 | key->prefixlen = prefixlen;
|
| 1384 | key->l3index = l3index;
|
| 1385 | key->flags = flags;
|
| 1386 | memcpy(&key->addr, addr,
|
| 1387 | (IS_ENABLED(CONFIG_IPV6) && family == AF_INET6) ? sizeof(struct in6_addr) :
|
| 1388 | sizeof(struct in_addr));
|
| 1389 | hlist_add_head_rcu(&key->node, &md5sig->head);
|
| 1390 | return 0;
|
| 1391 | }
|
| 1392 |
|
| 1393 | int tcp_md5_do_add(struct sock *sk, const union tcp_md5_addr *addr,
|
| 1394 | int family, u8 prefixlen, int l3index, u8 flags,
|
| 1395 | const u8 *newkey, u8 newkeylen)
|
| 1396 | {
|
| 1397 | struct tcp_sock *tp = tcp_sk(sk);
|
| 1398 |
|
| 1399 | if (!rcu_dereference_protected(tp->md5sig_info, lockdep_sock_is_held(sk))) {
|
| 1400 | if (fips_enabled) {
|
| 1401 | pr_warn_once("TCP-MD5 support is disabled due to FIPS\n");
|
| 1402 | return -EOPNOTSUPP;
|
| 1403 | }
|
| 1404 |
|
| 1405 | if (tcp_md5sig_info_add(sk, GFP_KERNEL))
|
| 1406 | return -ENOMEM;
|
| 1407 |
|
| 1408 | if (!static_branch_inc(&tcp_md5_needed.key)) {
|
| 1409 | struct tcp_md5sig_info *md5sig;
|
| 1410 |
|
| 1411 | md5sig = rcu_dereference_protected(tp->md5sig_info, lockdep_sock_is_held(sk));
|
| 1412 | rcu_assign_pointer(tp->md5sig_info, NULL);
|
| 1413 | kfree_rcu(md5sig, rcu);
|
| 1414 | return -EUSERS;
|
| 1415 | }
|
| 1416 | }
|
| 1417 |
|
| 1418 | return __tcp_md5_do_add(sk, addr, family, prefixlen, l3index, flags,
|
| 1419 | newkey, newkeylen, GFP_KERNEL);
|
| 1420 | }
|
| 1421 |
|
| 1422 | int tcp_md5_key_copy(struct sock *sk, const union tcp_md5_addr *addr,
|
| 1423 | int family, u8 prefixlen, int l3index,
|
| 1424 | struct tcp_md5sig_key *key)
|
| 1425 | {
|
| 1426 | struct tcp_sock *tp = tcp_sk(sk);
|
| 1427 |
|
| 1428 | if (!rcu_dereference_protected(tp->md5sig_info, lockdep_sock_is_held(sk))) {
|
| 1429 |
|
| 1430 | if (tcp_md5sig_info_add(sk, sk_gfp_mask(sk, GFP_ATOMIC)))
|
| 1431 | return -ENOMEM;
|
| 1432 |
|
| 1433 | if (!static_key_fast_inc_not_disabled(&tcp_md5_needed.key.key)) {
|
| 1434 | struct tcp_md5sig_info *md5sig;
|
| 1435 |
|
| 1436 | md5sig = rcu_dereference_protected(tp->md5sig_info, lockdep_sock_is_held(sk));
|
| 1437 | net_warn_ratelimited("Too many TCP-MD5 keys in the system\n");
|
| 1438 | rcu_assign_pointer(tp->md5sig_info, NULL);
|
| 1439 | kfree_rcu(md5sig, rcu);
|
| 1440 | return -EUSERS;
|
| 1441 | }
|
| 1442 | }
|
| 1443 |
|
| 1444 | return __tcp_md5_do_add(sk, addr, family, prefixlen, l3index,
|
| 1445 | key->flags, key->key, key->keylen,
|
| 1446 | sk_gfp_mask(sk, GFP_ATOMIC));
|
| 1447 | }
|
| 1448 |
|
| 1449 | int tcp_md5_do_del(struct sock *sk, const union tcp_md5_addr *addr, int family,
|
| 1450 | u8 prefixlen, int l3index, u8 flags)
|
| 1451 | {
|
| 1452 | struct tcp_md5sig_key *key;
|
| 1453 |
|
| 1454 | key = tcp_md5_do_lookup_exact(sk, addr, family, prefixlen, l3index, flags);
|
| 1455 | if (!key)
|
| 1456 | return -ENOENT;
|
| 1457 | hlist_del_rcu(&key->node);
|
| 1458 | atomic_sub(sizeof(*key), &sk->sk_omem_alloc);
|
| 1459 | kfree_rcu(key, rcu);
|
| 1460 | return 0;
|
| 1461 | }
|
| 1462 |
|
| 1463 | void tcp_clear_md5_list(struct sock *sk)
|
| 1464 | {
|
| 1465 | struct tcp_sock *tp = tcp_sk(sk);
|
| 1466 | struct tcp_md5sig_key *key;
|
| 1467 | struct hlist_node *n;
|
| 1468 | struct tcp_md5sig_info *md5sig;
|
| 1469 |
|
| 1470 | md5sig = rcu_dereference_protected(tp->md5sig_info, 1);
|
| 1471 |
|
| 1472 | hlist_for_each_entry_safe(key, n, &md5sig->head, node) {
|
| 1473 | hlist_del_rcu(&key->node);
|
| 1474 | atomic_sub(sizeof(*key), &sk->sk_omem_alloc);
|
| 1475 | kfree_rcu(key, rcu);
|
| 1476 | }
|
| 1477 | }
|
| 1478 |
|
| 1479 | static int tcp_v4_parse_md5_keys(struct sock *sk, int optname,
|
| 1480 | sockptr_t optval, int optlen)
|
| 1481 | {
|
| 1482 | struct tcp_md5sig cmd;
|
| 1483 | struct sockaddr_in *sin = (struct sockaddr_in *)&cmd.tcpm_addr;
|
| 1484 | const union tcp_md5_addr *addr;
|
| 1485 | u8 prefixlen = 32;
|
| 1486 | int l3index = 0;
|
| 1487 | bool l3flag;
|
| 1488 | u8 flags;
|
| 1489 |
|
| 1490 | if (optlen < sizeof(cmd))
|
| 1491 | return -EINVAL;
|
| 1492 |
|
| 1493 | if (copy_from_sockptr(&cmd, optval, sizeof(cmd)))
|
| 1494 | return -EFAULT;
|
| 1495 |
|
| 1496 | if (sin->sin_family != AF_INET)
|
| 1497 | return -EINVAL;
|
| 1498 |
|
| 1499 | flags = cmd.tcpm_flags & TCP_MD5SIG_FLAG_IFINDEX;
|
| 1500 | l3flag = cmd.tcpm_flags & TCP_MD5SIG_FLAG_IFINDEX;
|
| 1501 |
|
| 1502 | if (optname == TCP_MD5SIG_EXT &&
|
| 1503 | cmd.tcpm_flags & TCP_MD5SIG_FLAG_PREFIX) {
|
| 1504 | prefixlen = cmd.tcpm_prefixlen;
|
| 1505 | if (prefixlen > 32)
|
| 1506 | return -EINVAL;
|
| 1507 | }
|
| 1508 |
|
| 1509 | if (optname == TCP_MD5SIG_EXT && cmd.tcpm_ifindex &&
|
| 1510 | cmd.tcpm_flags & TCP_MD5SIG_FLAG_IFINDEX) {
|
| 1511 | struct net_device *dev;
|
| 1512 |
|
| 1513 | rcu_read_lock();
|
| 1514 | dev = dev_get_by_index_rcu(sock_net(sk), cmd.tcpm_ifindex);
|
| 1515 | if (dev && netif_is_l3_master(dev))
|
| 1516 | l3index = dev->ifindex;
|
| 1517 |
|
| 1518 | rcu_read_unlock();
|
| 1519 |
|
| 1520 | |
| 1521 | * right now device MUST be an L3 master
|
| 1522 |
|
| 1523 | if (!dev || !l3index)
|
| 1524 | return -EINVAL;
|
| 1525 | }
|
| 1526 |
|
| 1527 | addr = (union tcp_md5_addr *)&sin->sin_addr.s_addr;
|
| 1528 |
|
| 1529 | if (!cmd.tcpm_keylen)
|
| 1530 | return tcp_md5_do_del(sk, addr, AF_INET, prefixlen, l3index, flags);
|
| 1531 |
|
| 1532 | if (cmd.tcpm_keylen > TCP_MD5SIG_MAXKEYLEN)
|
| 1533 | return -EINVAL;
|
| 1534 |
|
| 1535 | |
| 1536 | * See the comment in tcp_ao_add_cmd()
|
| 1537 |
|
| 1538 | if (tcp_ao_required(sk, addr, AF_INET, l3flag ? l3index : -1, false))
|
| 1539 | return -EKEYREJECTED;
|
| 1540 |
|
| 1541 | return tcp_md5_do_add(sk, addr, AF_INET, prefixlen, l3index, flags,
|
| 1542 | cmd.tcpm_key, cmd.tcpm_keylen);
|
| 1543 | }
|
| 1544 |
|
| 1545 | static void tcp_v4_md5_hash_headers(struct md5_ctx *ctx,
|
| 1546 | __be32 daddr, __be32 saddr,
|
| 1547 | const struct tcphdr *th, int nbytes)
|
| 1548 | {
|
| 1549 | struct {
|
| 1550 | struct tcp4_pseudohdr ip;
|
| 1551 | struct tcphdr tcp;
|
| 1552 | } h;
|
| 1553 |
|
| 1554 | h.ip.saddr = saddr;
|
| 1555 | h.ip.daddr = daddr;
|
| 1556 | h.ip.pad = 0;
|
| 1557 | h.ip.protocol = IPPROTO_TCP;
|
| 1558 | h.ip.len = cpu_to_be16(nbytes);
|
| 1559 | h.tcp = *th;
|
| 1560 | h.tcp.check = 0;
|
| 1561 | md5_update(ctx, (const u8 *)&h, sizeof(h.ip) + sizeof(h.tcp));
|
| 1562 | }
|
| 1563 |
|
| 1564 | static noinline_for_stack void
|
| 1565 | tcp_v4_md5_hash_hdr(char *md5_hash, const struct tcp_md5sig_key *key,
|
| 1566 | __be32 daddr, __be32 saddr, const struct tcphdr *th)
|
| 1567 | {
|
| 1568 | struct md5_ctx ctx;
|
| 1569 |
|
| 1570 | md5_init(&ctx);
|
| 1571 | tcp_v4_md5_hash_headers(&ctx, daddr, saddr, th, th->doff << 2);
|
| 1572 | tcp_md5_hash_key(&ctx, key);
|
| 1573 | md5_final(&ctx, md5_hash);
|
| 1574 | }
|
| 1575 |
|
| 1576 | noinline_for_stack void
|
| 1577 | tcp_v4_md5_hash_skb(char *md5_hash, const struct tcp_md5sig_key *key,
|
| 1578 | const struct sock *sk, const struct sk_buff *skb)
|
| 1579 | {
|
| 1580 | const struct tcphdr *th = tcp_hdr(skb);
|
| 1581 | __be32 saddr, daddr;
|
| 1582 | struct md5_ctx ctx;
|
| 1583 |
|
| 1584 | if (sk) {
|
| 1585 | saddr = sk->sk_rcv_saddr;
|
| 1586 | daddr = sk->sk_daddr;
|
| 1587 | } else {
|
| 1588 | const struct iphdr *iph = ip_hdr(skb);
|
| 1589 | saddr = iph->saddr;
|
| 1590 | daddr = iph->daddr;
|
| 1591 | }
|
| 1592 |
|
| 1593 | md5_init(&ctx);
|
| 1594 | tcp_v4_md5_hash_headers(&ctx, daddr, saddr, th, skb->len);
|
| 1595 | tcp_md5_hash_skb_data(&ctx, skb, th->doff << 2);
|
| 1596 | tcp_md5_hash_key(&ctx, key);
|
| 1597 | md5_final(&ctx, md5_hash);
|
| 1598 | }
|
| 1599 |
|
| 1600 | #endif
|
| 1601 |
|
| 1602 | static void tcp_v4_init_req(struct request_sock *req,
|
| 1603 | const struct sock *sk_listener,
|
| 1604 | struct sk_buff *skb)
|
| 1605 | {
|
| 1606 | struct inet_request_sock *ireq = inet_rsk(req);
|
| 1607 | struct net *net = sock_net(sk_listener);
|
| 1608 |
|
| 1609 | sk_rcv_saddr_set(req_to_sk(req), ip_hdr(skb)->daddr);
|
| 1610 | sk_daddr_set(req_to_sk(req), ip_hdr(skb)->saddr);
|
| 1611 | RCU_INIT_POINTER(ireq->ireq_opt, tcp_v4_save_options(net, skb));
|
| 1612 | }
|
| 1613 |
|
| 1614 | static struct dst_entry *tcp_v4_route_req(const struct sock *sk,
|
| 1615 | struct sk_buff *skb,
|
| 1616 | struct flowi *fl,
|
| 1617 | struct request_sock *req,
|
| 1618 | u32 tw_isn)
|
| 1619 | {
|
| 1620 | tcp_v4_init_req(req, sk, skb);
|
| 1621 |
|
| 1622 | if (security_inet_conn_request(sk, skb, req))
|
| 1623 | return NULL;
|
| 1624 |
|
| 1625 | return inet_csk_route_req(sk, &fl->u.ip4, req);
|
| 1626 | }
|
| 1627 |
|
| 1628 | struct request_sock_ops tcp_request_sock_ops __read_mostly = {
|
| 1629 | .family = PF_INET,
|
| 1630 | .obj_size = sizeof(struct tcp_request_sock),
|
| 1631 | .send_ack = tcp_v4_reqsk_send_ack,
|
| 1632 | .destructor = tcp_v4_reqsk_destructor,
|
| 1633 | .send_reset = tcp_v4_send_reset,
|
| 1634 | };
|
| 1635 |
|
| 1636 | const struct tcp_request_sock_ops tcp_request_sock_ipv4_ops = {
|
| 1637 | .mss_clamp = TCP_MSS_DEFAULT,
|
| 1638 | #ifdef CONFIG_TCP_MD5SIG
|
| 1639 | .req_md5_lookup = tcp_v4_md5_lookup,
|
| 1640 | .calc_md5_hash = tcp_v4_md5_hash_skb,
|
| 1641 | #endif
|
| 1642 | #ifdef CONFIG_TCP_AO
|
| 1643 | .ao_lookup = tcp_v4_ao_lookup_rsk,
|
| 1644 | .ao_calc_key = tcp_v4_ao_calc_key_rsk,
|
| 1645 | .ao_synack_hash = tcp_v4_ao_synack_hash,
|
| 1646 | #endif
|
| 1647 | #ifdef CONFIG_SYN_COOKIES
|
| 1648 | .cookie_init_seq = cookie_v4_init_sequence,
|
| 1649 | #endif
|
| 1650 | .route_req = tcp_v4_route_req,
|
| 1651 | .init_seq_and_ts_off = tcp_v4_init_seq_and_ts_off,
|
| 1652 | .send_synack = tcp_v4_send_synack,
|
| 1653 | };
|
| 1654 |
|
| 1655 | int tcp_v4_conn_request(struct sock *sk, struct sk_buff *skb)
|
| 1656 | {
|
| 1657 |
|
| 1658 | if (skb_rtable(skb)->rt_flags & (RTCF_BROADCAST | RTCF_MULTICAST))
|
| 1659 | goto drop;
|
| 1660 |
|
| 1661 | return tcp_conn_request(&tcp_request_sock_ops,
|
| 1662 | &tcp_request_sock_ipv4_ops, sk, skb);
|
| 1663 |
|
| 1664 | drop:
|
| 1665 | tcp_listendrop(sk);
|
| 1666 | return 0;
|
| 1667 | }
|
| 1668 |
|
| 1669 |
|
| 1670 | |
| 1671 | * The three way handshake has completed - we got a valid synack -
|
| 1672 | * now create the new socket.
|
| 1673 |
|
| 1674 | struct sock *tcp_v4_syn_recv_sock(const struct sock *sk, struct sk_buff *skb,
|
| 1675 | struct request_sock *req,
|
| 1676 | struct dst_entry *dst,
|
| 1677 | struct request_sock *req_unhash,
|
| 1678 | bool *own_req,
|
| 1679 | void (*opt_child_init)(struct sock *newsk,
|
| 1680 | const struct sock *sk))
|
| 1681 | {
|
| 1682 | struct inet_request_sock *ireq;
|
| 1683 | bool found_dup_sk = false;
|
| 1684 | struct inet_sock *newinet;
|
| 1685 | struct tcp_sock *newtp;
|
| 1686 | struct sock *newsk;
|
| 1687 | #ifdef CONFIG_TCP_MD5SIG
|
| 1688 | const union tcp_md5_addr *addr;
|
| 1689 | struct tcp_md5sig_key *key;
|
| 1690 | int l3index;
|
| 1691 | #endif
|
| 1692 | struct ip_options_rcu *inet_opt;
|
| 1693 |
|
| 1694 | if (sk_acceptq_is_full(sk))
|
| 1695 | goto exit_overflow;
|
| 1696 |
|
| 1697 | newsk = tcp_create_openreq_child(sk, req, skb);
|
| 1698 | if (!newsk)
|
| 1699 | goto exit_nonewsk;
|
| 1700 |
|
| 1701 | newsk->sk_gso_type = SKB_GSO_TCPV4;
|
| 1702 | inet_sk_rx_dst_set(newsk, skb);
|
| 1703 |
|
| 1704 | newtp = tcp_sk(newsk);
|
| 1705 | newinet = inet_sk(newsk);
|
| 1706 | ireq = inet_rsk(req);
|
| 1707 | inet_opt = rcu_dereference(ireq->ireq_opt);
|
| 1708 | RCU_INIT_POINTER(newinet->inet_opt, inet_opt);
|
| 1709 | newinet->mc_index = inet_iif(skb);
|
| 1710 | newinet->mc_ttl = ip_hdr(skb)->ttl;
|
| 1711 | newinet->rcv_tos = ip_hdr(skb)->tos;
|
| 1712 | inet_csk(newsk)->icsk_ext_hdr_len = 0;
|
| 1713 | if (inet_opt)
|
| 1714 | inet_csk(newsk)->icsk_ext_hdr_len = inet_opt->opt.optlen;
|
| 1715 | atomic_set(&newinet->inet_id, get_random_u16());
|
| 1716 |
|
| 1717 | |
| 1718 | * ECT bits are set later in tcp_init_transfer().
|
| 1719 |
|
| 1720 | if (READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_reflect_tos))
|
| 1721 | newinet->tos = tcp_rsk(req)->syn_tos & ~INET_ECN_MASK;
|
| 1722 |
|
| 1723 | if (!dst) {
|
| 1724 | dst = inet_csk_route_child_sock(sk, newsk, req);
|
| 1725 | if (!dst)
|
| 1726 | goto put_and_exit;
|
| 1727 | } else {
|
| 1728 |
|
| 1729 | }
|
| 1730 | sk_setup_caps(newsk, dst);
|
| 1731 |
|
| 1732 | #if IS_ENABLED(CONFIG_IPV6)
|
| 1733 | if (opt_child_init)
|
| 1734 | opt_child_init(newsk, sk);
|
| 1735 | #endif
|
| 1736 | tcp_ca_openreq_child(newsk, dst);
|
| 1737 |
|
| 1738 | tcp_sync_mss(newsk, dst4_mtu(dst));
|
| 1739 | newtp->advmss = tcp_mss_clamp(tcp_sk(sk), tcp_dst_advmss(dst));
|
| 1740 |
|
| 1741 | tcp_initialize_rcv_mss(newsk);
|
| 1742 |
|
| 1743 | #ifdef CONFIG_TCP_MD5SIG
|
| 1744 | l3index = l3mdev_master_ifindex_by_index(sock_net(sk), ireq->ir_iif);
|
| 1745 |
|
| 1746 | addr = (union tcp_md5_addr *)&newinet->inet_daddr;
|
| 1747 | key = tcp_md5_do_lookup(sk, l3index, addr, AF_INET);
|
| 1748 | if (key && !tcp_rsk_used_ao(req)) {
|
| 1749 | if (tcp_md5_key_copy(newsk, addr, AF_INET, 32, l3index, key))
|
| 1750 | goto put_and_exit;
|
| 1751 | sk_gso_disable(newsk);
|
| 1752 | }
|
| 1753 | #endif
|
| 1754 | #ifdef CONFIG_TCP_AO
|
| 1755 | if (tcp_ao_copy_all_matching(sk, newsk, req, skb, AF_INET))
|
| 1756 | goto put_and_exit;
|
| 1757 | #endif
|
| 1758 |
|
| 1759 | if (__inet_inherit_port(sk, newsk) < 0)
|
| 1760 | goto put_and_exit;
|
| 1761 | *own_req = inet_ehash_nolisten(newsk, req_to_sk(req_unhash),
|
| 1762 | &found_dup_sk);
|
| 1763 | if (likely(*own_req)) {
|
| 1764 | tcp_move_syn(newtp, req);
|
| 1765 | ireq->ireq_opt = NULL;
|
| 1766 | } else {
|
| 1767 | newinet->inet_opt = NULL;
|
| 1768 |
|
| 1769 | if (!req_unhash && found_dup_sk) {
|
| 1770 | |
| 1771 | * syncookie case only
|
| 1772 |
|
| 1773 | bh_unlock_sock(newsk);
|
| 1774 | sock_put(newsk);
|
| 1775 | newsk = NULL;
|
| 1776 | }
|
| 1777 | }
|
| 1778 | return newsk;
|
| 1779 |
|
| 1780 | exit_overflow:
|
| 1781 | NET_INC_STATS(sock_net(sk), LINUX_MIB_LISTENOVERFLOWS);
|
| 1782 | exit_nonewsk:
|
| 1783 | dst_release(dst);
|
| 1784 | exit:
|
| 1785 | tcp_listendrop(sk);
|
| 1786 | return NULL;
|
| 1787 | put_and_exit:
|
| 1788 | newinet->inet_opt = NULL;
|
| 1789 | inet_csk_prepare_forced_close(newsk);
|
| 1790 | tcp_done(newsk);
|
| 1791 | goto exit;
|
| 1792 | }
|
| 1793 |
|
| 1794 | static struct sock *tcp_v4_cookie_check(struct sock *sk, struct sk_buff *skb)
|
| 1795 | {
|
| 1796 | #ifdef CONFIG_SYN_COOKIES
|
| 1797 | const struct tcphdr *th = tcp_hdr(skb);
|
| 1798 |
|
| 1799 | if (!th->syn)
|
| 1800 | sk = cookie_v4_check(sk, skb);
|
| 1801 | #endif
|
| 1802 | return sk;
|
| 1803 | }
|
| 1804 |
|
| 1805 | u16 tcp_v4_get_syncookie(struct sock *sk, struct iphdr *iph,
|
| 1806 | struct tcphdr *th, u32 *cookie)
|
| 1807 | {
|
| 1808 | u16 mss = 0;
|
| 1809 | #ifdef CONFIG_SYN_COOKIES
|
| 1810 | mss = tcp_get_syncookie_mss(&tcp_request_sock_ops,
|
| 1811 | &tcp_request_sock_ipv4_ops, sk, th);
|
| 1812 | if (mss) {
|
| 1813 | *cookie = __cookie_v4_init_sequence(iph, th, &mss);
|
| 1814 | tcp_synq_overflow(sk);
|
| 1815 | }
|
| 1816 | #endif
|
| 1817 | return mss;
|
| 1818 | }
|
| 1819 |
|
| 1820 | INDIRECT_CALLABLE_DECLARE(struct dst_entry *ipv4_dst_check(struct dst_entry *,
|
| 1821 | u32));
|
| 1822 | |
| 1823 | * here, unless it is a TCP_LISTEN socket.
|
| 1824 | *
|
| 1825 | * We have a potential double-lock case here, so even when
|
| 1826 | * doing backlog processing we use the BH locking scheme.
|
| 1827 | * This is because we cannot sleep with the original spinlock
|
| 1828 | * held.
|
| 1829 |
|
| 1830 | int tcp_v4_do_rcv(struct sock *sk, struct sk_buff *skb)
|
| 1831 | {
|
| 1832 | enum skb_drop_reason reason;
|
| 1833 |
|
| 1834 | reason = psp_sk_rx_policy_check(sk, skb);
|
| 1835 | if (reason)
|
| 1836 | goto err_discard;
|
| 1837 |
|
| 1838 | if (sk->sk_state == TCP_ESTABLISHED) {
|
| 1839 | struct dst_entry *dst;
|
| 1840 |
|
| 1841 | dst = rcu_dereference_protected(sk->sk_rx_dst,
|
| 1842 | lockdep_sock_is_held(sk));
|
| 1843 |
|
| 1844 | sock_rps_save_rxhash(sk, skb);
|
| 1845 | sk_mark_napi_id(sk, skb);
|
| 1846 | if (dst && unlikely(dst != skb_dst(skb))) {
|
| 1847 | if (sk->sk_rx_dst_ifindex != skb->skb_iif ||
|
| 1848 | !INDIRECT_CALL_1(dst->ops->check, ipv4_dst_check,
|
| 1849 | dst, 0)) {
|
| 1850 | RCU_INIT_POINTER(sk->sk_rx_dst, NULL);
|
| 1851 | dst_release(dst);
|
| 1852 | }
|
| 1853 | }
|
| 1854 | tcp_rcv_established(sk, skb);
|
| 1855 | return 0;
|
| 1856 | }
|
| 1857 |
|
| 1858 | if (tcp_checksum_complete(skb))
|
| 1859 | goto csum_err;
|
| 1860 |
|
| 1861 | if (sk->sk_state == TCP_LISTEN) {
|
| 1862 | struct sock *nsk = tcp_v4_cookie_check(sk, skb);
|
| 1863 |
|
| 1864 | if (!nsk)
|
| 1865 | return 0;
|
| 1866 | if (nsk != sk) {
|
| 1867 | reason = tcp_child_process(sk, nsk, skb);
|
| 1868 | sock_put(nsk);
|
| 1869 | if (reason)
|
| 1870 | goto reset;
|
| 1871 | return 0;
|
| 1872 | }
|
| 1873 | } else
|
| 1874 | sock_rps_save_rxhash(sk, skb);
|
| 1875 |
|
| 1876 | reason = tcp_rcv_state_process(sk, skb);
|
| 1877 | if (reason)
|
| 1878 | goto reset;
|
| 1879 | return 0;
|
| 1880 |
|
| 1881 | reset:
|
| 1882 | tcp_v4_send_reset(sk, skb, sk_rst_convert_drop_reason(reason));
|
| 1883 | discard:
|
| 1884 | sk_skb_reason_drop(sk, skb, reason);
|
| 1885 | |
| 1886 | * gcc suffers from register pressure on the x86, sk (in %ebx)
|
| 1887 | * might be destroyed here. This current version compiles correctly,
|
| 1888 | * but you have been warned.
|
| 1889 |
|
| 1890 | return 0;
|
| 1891 |
|
| 1892 | csum_err:
|
| 1893 | reason = SKB_DROP_REASON_TCP_CSUM;
|
| 1894 | trace_tcp_bad_csum(skb);
|
| 1895 | TCP_INC_STATS(sock_net(sk), TCP_MIB_CSUMERRORS);
|
| 1896 | err_discard:
|
| 1897 | TCP_INC_STATS(sock_net(sk), TCP_MIB_INERRS);
|
| 1898 | goto discard;
|
| 1899 | }
|
| 1900 |
|
| 1901 | enum skb_drop_reason tcp_add_backlog(struct sock *sk, struct sk_buff *skb)
|
| 1902 | {
|
| 1903 | u32 tail_gso_size, tail_gso_segs;
|
| 1904 | struct skb_shared_info *shinfo;
|
| 1905 | const struct tcphdr *th;
|
| 1906 | struct tcphdr *thtail;
|
| 1907 | struct sk_buff *tail;
|
| 1908 | unsigned int hdrlen;
|
| 1909 | bool fragstolen;
|
| 1910 | u32 gso_segs;
|
| 1911 | u32 gso_size;
|
| 1912 | u64 limit;
|
| 1913 | int delta;
|
| 1914 | int err;
|
| 1915 |
|
| 1916 | |
| 1917 | * we can fix skb->truesize to its real value to avoid future drops.
|
| 1918 | * This is valid because skb is not yet charged to the socket.
|
| 1919 | * It has been noticed pure SACK packets were sometimes dropped
|
| 1920 | * (if cooked by drivers without copybreak feature).
|
| 1921 |
|
| 1922 | skb_condense(skb);
|
| 1923 |
|
| 1924 | tcp_cleanup_skb(skb);
|
| 1925 |
|
| 1926 | if (unlikely(tcp_checksum_complete(skb))) {
|
| 1927 | bh_unlock_sock(sk);
|
| 1928 | trace_tcp_bad_csum(skb);
|
| 1929 | __TCP_INC_STATS(sock_net(sk), TCP_MIB_CSUMERRORS);
|
| 1930 | __TCP_INC_STATS(sock_net(sk), TCP_MIB_INERRS);
|
| 1931 | return SKB_DROP_REASON_TCP_CSUM;
|
| 1932 | }
|
| 1933 |
|
| 1934 | |
| 1935 | * above the limits.
|
| 1936 | * This is okay because skb capacity is limited to MAX_SKB_FRAGS.
|
| 1937 |
|
| 1938 | th = (const struct tcphdr *)skb->data;
|
| 1939 | hdrlen = th->doff * 4;
|
| 1940 |
|
| 1941 | tail = sk->sk_backlog.tail;
|
| 1942 | if (!tail)
|
| 1943 | goto no_coalesce;
|
| 1944 | thtail = (struct tcphdr *)tail->data;
|
| 1945 |
|
| 1946 | if (TCP_SKB_CB(tail)->end_seq != TCP_SKB_CB(skb)->seq ||
|
| 1947 | TCP_SKB_CB(tail)->ip_dsfield != TCP_SKB_CB(skb)->ip_dsfield ||
|
| 1948 | ((TCP_SKB_CB(tail)->tcp_flags |
|
| 1949 | TCP_SKB_CB(skb)->tcp_flags) & (TCPHDR_SYN | TCPHDR_RST | TCPHDR_URG)) ||
|
| 1950 | !((TCP_SKB_CB(tail)->tcp_flags &
|
| 1951 | TCP_SKB_CB(skb)->tcp_flags) & TCPHDR_ACK) ||
|
| 1952 | ((TCP_SKB_CB(tail)->tcp_flags ^
|
| 1953 | TCP_SKB_CB(skb)->tcp_flags) &
|
| 1954 | (TCPHDR_ECE | TCPHDR_CWR | TCPHDR_AE)) ||
|
| 1955 | !tcp_skb_can_collapse_rx(tail, skb) ||
|
| 1956 | thtail->doff != th->doff ||
|
| 1957 | memcmp(thtail + 1, th + 1, hdrlen - sizeof(*th)) ||
|
| 1958 |
|
| 1959 | psp_skb_coalesce_diff(tail, skb))
|
| 1960 | goto no_coalesce;
|
| 1961 |
|
| 1962 | __skb_pull(skb, hdrlen);
|
| 1963 |
|
| 1964 | shinfo = skb_shinfo(skb);
|
| 1965 | gso_size = shinfo->gso_size ?: skb->len;
|
| 1966 | gso_segs = shinfo->gso_segs ?: 1;
|
| 1967 |
|
| 1968 | shinfo = skb_shinfo(tail);
|
| 1969 | tail_gso_size = shinfo->gso_size ?: (tail->len - hdrlen);
|
| 1970 | tail_gso_segs = shinfo->gso_segs ?: 1;
|
| 1971 |
|
| 1972 | if (skb_try_coalesce(tail, skb, &fragstolen, &delta)) {
|
| 1973 | TCP_SKB_CB(tail)->end_seq = TCP_SKB_CB(skb)->end_seq;
|
| 1974 |
|
| 1975 | if (likely(!before(TCP_SKB_CB(skb)->ack_seq, TCP_SKB_CB(tail)->ack_seq))) {
|
| 1976 | TCP_SKB_CB(tail)->ack_seq = TCP_SKB_CB(skb)->ack_seq;
|
| 1977 | thtail->window = th->window;
|
| 1978 | }
|
| 1979 |
|
| 1980 | |
| 1981 | * thtail->fin, so that the fast path in tcp_rcv_established()
|
| 1982 | * is not entered if we append a packet with a FIN.
|
| 1983 | * SYN, RST, URG are not present.
|
| 1984 | * ACK is set on both packets.
|
| 1985 | * PSH : we do not really care in TCP stack,
|
| 1986 | * at least for 'GRO' packets.
|
| 1987 |
|
| 1988 | thtail->fin |= th->fin;
|
| 1989 | TCP_SKB_CB(tail)->tcp_flags |= TCP_SKB_CB(skb)->tcp_flags;
|
| 1990 |
|
| 1991 | if (TCP_SKB_CB(skb)->has_rxtstamp) {
|
| 1992 | TCP_SKB_CB(tail)->has_rxtstamp = true;
|
| 1993 | tail->tstamp = skb->tstamp;
|
| 1994 | skb_hwtstamps(tail)->hwtstamp = skb_hwtstamps(skb)->hwtstamp;
|
| 1995 | }
|
| 1996 |
|
| 1997 |
|
| 1998 | shinfo->gso_size = max(gso_size, tail_gso_size);
|
| 1999 | shinfo->gso_segs = min_t(u32, gso_segs + tail_gso_segs, 0xFFFF);
|
| 2000 |
|
| 2001 | sk->sk_backlog.len += delta;
|
| 2002 | __NET_INC_STATS(sock_net(sk),
|
| 2003 | LINUX_MIB_TCPBACKLOGCOALESCE);
|
| 2004 | kfree_skb_partial(skb, fragstolen);
|
| 2005 | return SKB_NOT_DROPPED_YET;
|
| 2006 | }
|
| 2007 | __skb_push(skb, hdrlen);
|
| 2008 |
|
| 2009 | no_coalesce:
|
| 2010 | |
| 2011 | * Both sk->sk_backlog.len and sk->sk_rmem_alloc could reach
|
| 2012 | * sk_rcvbuf in normal conditions.
|
| 2013 |
|
| 2014 | limit = ((u64)READ_ONCE(sk->sk_rcvbuf)) << 1;
|
| 2015 |
|
| 2016 | limit += ((u32)READ_ONCE(sk->sk_sndbuf)) >> 1;
|
| 2017 |
|
| 2018 | |
| 2019 | * to reduce memory overhead, so add a little headroom here.
|
| 2020 | * Few sockets backlog are possibly concurrently non empty.
|
| 2021 |
|
| 2022 | limit += 64 * 1024;
|
| 2023 |
|
| 2024 | limit = min_t(u64, limit, UINT_MAX);
|
| 2025 |
|
| 2026 | err = sk_add_backlog(sk, skb, limit);
|
| 2027 | if (unlikely(err)) {
|
| 2028 | bh_unlock_sock(sk);
|
| 2029 | if (err == -ENOMEM) {
|
| 2030 | __NET_INC_STATS(sock_net(sk), LINUX_MIB_PFMEMALLOCDROP);
|
| 2031 | return SKB_DROP_REASON_PFMEMALLOC;
|
| 2032 | }
|
| 2033 | __NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPBACKLOGDROP);
|
| 2034 | return SKB_DROP_REASON_SOCKET_BACKLOG;
|
| 2035 | }
|
| 2036 | return SKB_NOT_DROPPED_YET;
|
| 2037 | }
|
| 2038 |
|
| 2039 | static void tcp_v4_restore_cb(struct sk_buff *skb)
|
| 2040 | {
|
| 2041 | memmove(IPCB(skb), &TCP_SKB_CB(skb)->header.h4,
|
| 2042 | sizeof(struct inet_skb_parm));
|
| 2043 | }
|
| 2044 |
|
| 2045 | static void tcp_v4_fill_cb(struct sk_buff *skb, const struct iphdr *iph,
|
| 2046 | const struct tcphdr *th)
|
| 2047 | {
|
| 2048 | |
| 2049 | * barrier() makes sure compiler wont play fool^Waliasing games.
|
| 2050 |
|
| 2051 | memmove(&TCP_SKB_CB(skb)->header.h4, IPCB(skb),
|
| 2052 | sizeof(struct inet_skb_parm));
|
| 2053 | barrier();
|
| 2054 |
|
| 2055 | TCP_SKB_CB(skb)->seq = ntohl(th->seq);
|
| 2056 | TCP_SKB_CB(skb)->end_seq = (TCP_SKB_CB(skb)->seq + th->syn + th->fin +
|
| 2057 | skb->len - th->doff * 4);
|
| 2058 | TCP_SKB_CB(skb)->ack_seq = ntohl(th->ack_seq);
|
| 2059 | TCP_SKB_CB(skb)->tcp_flags = tcp_flags_ntohs(th);
|
| 2060 | TCP_SKB_CB(skb)->ip_dsfield = ipv4_get_dsfield(iph);
|
| 2061 | TCP_SKB_CB(skb)->sacked = 0;
|
| 2062 | TCP_SKB_CB(skb)->has_rxtstamp =
|
| 2063 | skb->tstamp || skb_hwtstamps(skb)->hwtstamp;
|
| 2064 | }
|
| 2065 |
|
| 2066 | |
| 2067 | * From tcp_input.c
|
| 2068 |
|
| 2069 |
|
| 2070 | int tcp_v4_rcv(struct sk_buff *skb)
|
| 2071 | {
|
| 2072 | struct net *net = dev_net_rcu(skb->dev);
|
| 2073 | enum skb_drop_reason drop_reason;
|
| 2074 | enum tcp_tw_status tw_status;
|
| 2075 | int sdif = inet_sdif(skb);
|
| 2076 | int dif = inet_iif(skb);
|
| 2077 | const struct iphdr *iph;
|
| 2078 | const struct tcphdr *th;
|
| 2079 | struct sock *sk = NULL;
|
| 2080 | bool refcounted;
|
| 2081 | int ret;
|
| 2082 | u32 isn;
|
| 2083 |
|
| 2084 | drop_reason = SKB_DROP_REASON_NOT_SPECIFIED;
|
| 2085 | if (skb->pkt_type != PACKET_HOST)
|
| 2086 | goto discard_it;
|
| 2087 |
|
| 2088 |
|
| 2089 | __TCP_INC_STATS(net, TCP_MIB_INSEGS);
|
| 2090 |
|
| 2091 | if (!pskb_may_pull(skb, sizeof(struct tcphdr)))
|
| 2092 | goto discard_it;
|
| 2093 |
|
| 2094 | th = (const struct tcphdr *)skb->data;
|
| 2095 |
|
| 2096 | if (unlikely(th->doff < sizeof(struct tcphdr) / 4)) {
|
| 2097 | drop_reason = SKB_DROP_REASON_PKT_TOO_SMALL;
|
| 2098 | goto bad_packet;
|
| 2099 | }
|
| 2100 | if (!pskb_may_pull(skb, th->doff * 4))
|
| 2101 | goto discard_it;
|
| 2102 |
|
| 2103 | |
| 2104 | * Packet length and doff are validated by header prediction,
|
| 2105 | * provided case of th->doff==0 is eliminated.
|
| 2106 | * So, we defer the checks.
|
| 2107 |
|
| 2108 | if (skb_checksum_init(skb, IPPROTO_TCP, inet_compute_pseudo))
|
| 2109 | goto csum_error;
|
| 2110 |
|
| 2111 | th = (const struct tcphdr *)skb->data;
|
| 2112 | iph = ip_hdr(skb);
|
| 2113 | lookup:
|
| 2114 | sk = __inet_lookup_skb(skb, __tcp_hdrlen(th), th->source,
|
| 2115 | th->dest, sdif, &refcounted);
|
| 2116 | if (!sk)
|
| 2117 | goto no_tcp_socket;
|
| 2118 |
|
| 2119 | if (sk->sk_state == TCP_TIME_WAIT)
|
| 2120 | goto do_time_wait;
|
| 2121 |
|
| 2122 | if (sk->sk_state == TCP_NEW_SYN_RECV) {
|
| 2123 | struct request_sock *req = inet_reqsk(sk);
|
| 2124 | bool req_stolen = false;
|
| 2125 | struct sock *nsk;
|
| 2126 |
|
| 2127 | sk = req->rsk_listener;
|
| 2128 | if (!xfrm4_policy_check(sk, XFRM_POLICY_IN, skb))
|
| 2129 | drop_reason = SKB_DROP_REASON_XFRM_POLICY;
|
| 2130 | else
|
| 2131 | drop_reason = tcp_inbound_hash(sk, req, skb,
|
| 2132 | &iph->saddr, &iph->daddr,
|
| 2133 | AF_INET, dif, sdif);
|
| 2134 | if (unlikely(drop_reason)) {
|
| 2135 | sk_drops_skbadd(sk, skb);
|
| 2136 | reqsk_put(req);
|
| 2137 | goto discard_it;
|
| 2138 | }
|
| 2139 | if (tcp_checksum_complete(skb)) {
|
| 2140 | reqsk_put(req);
|
| 2141 | goto csum_error;
|
| 2142 | }
|
| 2143 | if (unlikely(sk->sk_state != TCP_LISTEN)) {
|
| 2144 | nsk = reuseport_migrate_sock(sk, req_to_sk(req), skb);
|
| 2145 | if (!nsk) {
|
| 2146 | inet_csk_reqsk_queue_drop_and_put(sk, req);
|
| 2147 | goto lookup;
|
| 2148 | }
|
| 2149 | sk = nsk;
|
| 2150 | |
| 2151 | * before returning.
|
| 2152 |
|
| 2153 | } else {
|
| 2154 | |
| 2155 | * as we might lose it too soon.
|
| 2156 |
|
| 2157 | sock_hold(sk);
|
| 2158 | }
|
| 2159 | refcounted = true;
|
| 2160 | nsk = NULL;
|
| 2161 | drop_reason = tcp_filter(sk, skb);
|
| 2162 | if (!drop_reason) {
|
| 2163 | th = (const struct tcphdr *)skb->data;
|
| 2164 | iph = ip_hdr(skb);
|
| 2165 | tcp_v4_fill_cb(skb, iph, th);
|
| 2166 | nsk = tcp_check_req(sk, skb, req, false, &req_stolen,
|
| 2167 | &drop_reason);
|
| 2168 | }
|
| 2169 | if (!nsk) {
|
| 2170 | reqsk_put(req);
|
| 2171 | if (req_stolen) {
|
| 2172 | |
| 2173 | * and created a full blown socket.
|
| 2174 | * Try to feed this packet to this socket
|
| 2175 | * instead of discarding it.
|
| 2176 |
|
| 2177 | tcp_v4_restore_cb(skb);
|
| 2178 | sock_put(sk);
|
| 2179 | goto lookup;
|
| 2180 | }
|
| 2181 | goto discard_and_relse;
|
| 2182 | }
|
| 2183 | nf_reset_ct(skb);
|
| 2184 | if (nsk == sk) {
|
| 2185 | reqsk_put(req);
|
| 2186 | tcp_v4_restore_cb(skb);
|
| 2187 | } else {
|
| 2188 | drop_reason = tcp_child_process(sk, nsk, skb);
|
| 2189 | if (drop_reason) {
|
| 2190 | enum sk_rst_reason rst_reason;
|
| 2191 |
|
| 2192 | rst_reason = sk_rst_convert_drop_reason(drop_reason);
|
| 2193 | tcp_v4_send_reset(nsk, skb, rst_reason);
|
| 2194 | sock_put(nsk);
|
| 2195 | goto discard_and_relse;
|
| 2196 | }
|
| 2197 | sock_put(nsk);
|
| 2198 | sock_put(sk);
|
| 2199 | return 0;
|
| 2200 | }
|
| 2201 | }
|
| 2202 |
|
| 2203 | isn = 0;
|
| 2204 | process:
|
| 2205 | if (static_branch_unlikely(&ip4_min_ttl)) {
|
| 2206 |
|
| 2207 | if (unlikely(iph->ttl < READ_ONCE(inet_sk(sk)->min_ttl))) {
|
| 2208 | __NET_INC_STATS(net, LINUX_MIB_TCPMINTTLDROP);
|
| 2209 | drop_reason = SKB_DROP_REASON_TCP_MINTTL;
|
| 2210 | goto discard_and_relse;
|
| 2211 | }
|
| 2212 | }
|
| 2213 |
|
| 2214 | if (!xfrm4_policy_check(sk, XFRM_POLICY_IN, skb)) {
|
| 2215 | drop_reason = SKB_DROP_REASON_XFRM_POLICY;
|
| 2216 | goto discard_and_relse;
|
| 2217 | }
|
| 2218 |
|
| 2219 | drop_reason = tcp_inbound_hash(sk, NULL, skb, &iph->saddr, &iph->daddr,
|
| 2220 | AF_INET, dif, sdif);
|
| 2221 | if (drop_reason)
|
| 2222 | goto discard_and_relse;
|
| 2223 |
|
| 2224 | nf_reset_ct(skb);
|
| 2225 |
|
| 2226 | drop_reason = tcp_filter(sk, skb);
|
| 2227 | if (drop_reason)
|
| 2228 | goto discard_and_relse;
|
| 2229 |
|
| 2230 | th = (const struct tcphdr *)skb->data;
|
| 2231 | iph = ip_hdr(skb);
|
| 2232 | tcp_v4_fill_cb(skb, iph, th);
|
| 2233 | TCP_SKB_CB(skb)->tcp_tw_isn = isn;
|
| 2234 |
|
| 2235 | skb->dev = NULL;
|
| 2236 |
|
| 2237 | if (sk->sk_state == TCP_LISTEN) {
|
| 2238 | ret = tcp_v4_do_rcv(sk, skb);
|
| 2239 | goto put_and_return;
|
| 2240 | }
|
| 2241 |
|
| 2242 | sk_incoming_cpu_update(sk);
|
| 2243 |
|
| 2244 | bh_lock_sock_nested(sk);
|
| 2245 | tcp_segs_in(tcp_sk(sk), skb);
|
| 2246 | ret = 0;
|
| 2247 | if (!sock_owned_by_user(sk)) {
|
| 2248 | ret = tcp_v4_do_rcv(sk, skb);
|
| 2249 | } else {
|
| 2250 | drop_reason = tcp_add_backlog(sk, skb);
|
| 2251 | if (drop_reason)
|
| 2252 | goto discard_and_relse;
|
| 2253 | }
|
| 2254 | bh_unlock_sock(sk);
|
| 2255 |
|
| 2256 | put_and_return:
|
| 2257 | if (refcounted)
|
| 2258 | sock_put(sk);
|
| 2259 |
|
| 2260 | return ret;
|
| 2261 |
|
| 2262 | no_tcp_socket:
|
| 2263 | drop_reason = SKB_DROP_REASON_NO_SOCKET;
|
| 2264 | if (!xfrm4_policy_check(NULL, XFRM_POLICY_IN, skb))
|
| 2265 | goto discard_it;
|
| 2266 |
|
| 2267 | tcp_v4_fill_cb(skb, iph, th);
|
| 2268 |
|
| 2269 | if (tcp_checksum_complete(skb)) {
|
| 2270 | csum_error:
|
| 2271 | drop_reason = SKB_DROP_REASON_TCP_CSUM;
|
| 2272 | trace_tcp_bad_csum(skb);
|
| 2273 | __TCP_INC_STATS(net, TCP_MIB_CSUMERRORS);
|
| 2274 | bad_packet:
|
| 2275 | __TCP_INC_STATS(net, TCP_MIB_INERRS);
|
| 2276 | } else {
|
| 2277 | tcp_v4_send_reset(NULL, skb, sk_rst_convert_drop_reason(drop_reason));
|
| 2278 | }
|
| 2279 |
|
| 2280 | discard_it:
|
| 2281 | SKB_DR_OR(drop_reason, NOT_SPECIFIED);
|
| 2282 |
|
| 2283 | sk_skb_reason_drop(sk, skb, drop_reason);
|
| 2284 | return 0;
|
| 2285 |
|
| 2286 | discard_and_relse:
|
| 2287 | sk_drops_skbadd(sk, skb);
|
| 2288 | if (refcounted)
|
| 2289 | sock_put(sk);
|
| 2290 | goto discard_it;
|
| 2291 |
|
| 2292 | do_time_wait:
|
| 2293 | if (!xfrm4_policy_check(NULL, XFRM_POLICY_IN, skb)) {
|
| 2294 | drop_reason = SKB_DROP_REASON_XFRM_POLICY;
|
| 2295 | inet_twsk_put(inet_twsk(sk));
|
| 2296 | goto discard_it;
|
| 2297 | }
|
| 2298 |
|
| 2299 | tcp_v4_fill_cb(skb, iph, th);
|
| 2300 |
|
| 2301 | if (tcp_checksum_complete(skb)) {
|
| 2302 | inet_twsk_put(inet_twsk(sk));
|
| 2303 | goto csum_error;
|
| 2304 | }
|
| 2305 |
|
| 2306 | tw_status = tcp_timewait_state_process(inet_twsk(sk), skb, th, &isn,
|
| 2307 | &drop_reason);
|
| 2308 | switch (tw_status) {
|
| 2309 | case TCP_TW_SYN: {
|
| 2310 | struct sock *sk2 = inet_lookup_listener(net, skb, __tcp_hdrlen(th),
|
| 2311 | iph->saddr, th->source,
|
| 2312 | iph->daddr, th->dest,
|
| 2313 | inet_iif(skb),
|
| 2314 | sdif);
|
| 2315 | if (sk2) {
|
| 2316 | inet_twsk_deschedule_put(inet_twsk(sk));
|
| 2317 | sk = sk2;
|
| 2318 | tcp_v4_restore_cb(skb);
|
| 2319 | refcounted = false;
|
| 2320 | goto process;
|
| 2321 | }
|
| 2322 |
|
| 2323 | drop_reason = psp_twsk_rx_policy_check(inet_twsk(sk), skb);
|
| 2324 | if (drop_reason) {
|
| 2325 | inet_twsk_put(inet_twsk(sk));
|
| 2326 | goto discard_it;
|
| 2327 | }
|
| 2328 | }
|
| 2329 |
|
| 2330 | fallthrough;
|
| 2331 | case TCP_TW_ACK:
|
| 2332 | case TCP_TW_ACK_OOW:
|
| 2333 | tcp_v4_timewait_ack(sk, skb, tw_status);
|
| 2334 | break;
|
| 2335 | case TCP_TW_RST:
|
| 2336 | tcp_v4_send_reset(sk, skb, SK_RST_REASON_TCP_TIMEWAIT_SOCKET);
|
| 2337 | inet_twsk_deschedule_put(inet_twsk(sk));
|
| 2338 | goto discard_it;
|
| 2339 | case TCP_TW_SUCCESS:;
|
| 2340 | }
|
| 2341 | goto discard_it;
|
| 2342 | }
|
| 2343 |
|
| 2344 | static struct timewait_sock_ops tcp_timewait_sock_ops = {
|
| 2345 | .twsk_obj_size = sizeof(struct tcp_timewait_sock),
|
| 2346 | };
|
| 2347 |
|
| 2348 | void inet_sk_rx_dst_set(struct sock *sk, const struct sk_buff *skb)
|
| 2349 | {
|
| 2350 | struct dst_entry *dst = skb_dst(skb);
|
| 2351 |
|
| 2352 | if (dst && dst_hold_safe(dst)) {
|
| 2353 | rcu_assign_pointer(sk->sk_rx_dst, dst);
|
| 2354 | sk->sk_rx_dst_ifindex = skb->skb_iif;
|
| 2355 | }
|
| 2356 | }
|
| 2357 |
|
| 2358 | const struct inet_connection_sock_af_ops ipv4_specific = {
|
| 2359 | .queue_xmit = ip_queue_xmit,
|
| 2360 | .rebuild_header = inet_sk_rebuild_header,
|
| 2361 | .sk_rx_dst_set = inet_sk_rx_dst_set,
|
| 2362 | .conn_request = tcp_v4_conn_request,
|
| 2363 | .syn_recv_sock = tcp_v4_syn_recv_sock,
|
| 2364 | .net_header_len = sizeof(struct iphdr),
|
| 2365 | .setsockopt = ip_setsockopt,
|
| 2366 | .getsockopt = ip_getsockopt,
|
| 2367 | .mtu_reduced = tcp_v4_mtu_reduced,
|
| 2368 | };
|
| 2369 |
|
| 2370 | #if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO)
|
| 2371 | static const struct tcp_sock_af_ops tcp_sock_ipv4_specific = {
|
| 2372 | #ifdef CONFIG_TCP_MD5SIG
|
| 2373 | .md5_lookup = tcp_v4_md5_lookup,
|
| 2374 | .calc_md5_hash = tcp_v4_md5_hash_skb,
|
| 2375 | .md5_parse = tcp_v4_parse_md5_keys,
|
| 2376 | #endif
|
| 2377 | #ifdef CONFIG_TCP_AO
|
| 2378 | .ao_lookup = tcp_v4_ao_lookup,
|
| 2379 | .calc_ao_hash = tcp_v4_ao_hash_skb,
|
| 2380 | .ao_parse = tcp_v4_parse_ao,
|
| 2381 | .ao_calc_key_sk = tcp_v4_ao_calc_key_sk,
|
| 2382 | #endif
|
| 2383 | };
|
| 2384 |
|
| 2385 | static void tcp4_destruct_sock(struct sock *sk)
|
| 2386 | {
|
| 2387 | tcp_md5_destruct_sock(sk);
|
| 2388 | tcp_ao_destroy_sock(sk, false);
|
| 2389 | inet_sock_destruct(sk);
|
| 2390 | }
|
| 2391 | #endif
|
| 2392 |
|
| 2393 | |
| 2394 | * sk_alloc() so need not be done here.
|
| 2395 |
|
| 2396 | static int tcp_v4_init_sock(struct sock *sk)
|
| 2397 | {
|
| 2398 | struct inet_connection_sock *icsk = inet_csk(sk);
|
| 2399 |
|
| 2400 | tcp_init_sock(sk);
|
| 2401 |
|
| 2402 | icsk->icsk_af_ops = &ipv4_specific;
|
| 2403 |
|
| 2404 | #if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO)
|
| 2405 | tcp_sk(sk)->af_specific = &tcp_sock_ipv4_specific;
|
| 2406 | sk->sk_destruct = tcp4_destruct_sock;
|
| 2407 | #endif
|
| 2408 |
|
| 2409 | return 0;
|
| 2410 | }
|
| 2411 |
|
| 2412 | static void tcp_release_user_frags(struct sock *sk)
|
| 2413 | {
|
| 2414 | #ifdef CONFIG_PAGE_POOL
|
| 2415 | unsigned long index;
|
| 2416 | void *netmem;
|
| 2417 |
|
| 2418 | xa_for_each(&sk->sk_user_frags, index, netmem)
|
| 2419 | WARN_ON_ONCE(!napi_pp_put_page((__force netmem_ref)netmem));
|
| 2420 | #endif
|
| 2421 | }
|
| 2422 |
|
| 2423 | void tcp_v4_destroy_sock(struct sock *sk)
|
| 2424 | {
|
| 2425 | struct tcp_sock *tp = tcp_sk(sk);
|
| 2426 |
|
| 2427 | tcp_release_user_frags(sk);
|
| 2428 |
|
| 2429 | xa_destroy(&sk->sk_user_frags);
|
| 2430 |
|
| 2431 | trace_tcp_destroy_sock(sk);
|
| 2432 |
|
| 2433 | tcp_clear_xmit_timers(sk);
|
| 2434 |
|
| 2435 | tcp_cleanup_congestion_control(sk);
|
| 2436 |
|
| 2437 | tcp_cleanup_ulp(sk);
|
| 2438 |
|
| 2439 |
|
| 2440 | tcp_write_queue_purge(sk);
|
| 2441 |
|
| 2442 |
|
| 2443 | tcp_fastopen_active_disable_ofo_check(sk);
|
| 2444 |
|
| 2445 |
|
| 2446 | skb_rbtree_purge(&tp->out_of_order_queue);
|
| 2447 |
|
| 2448 |
|
| 2449 | if (inet_csk(sk)->icsk_bind_hash)
|
| 2450 | inet_put_port(sk);
|
| 2451 |
|
| 2452 | BUG_ON(rcu_access_pointer(tp->fastopen_rsk));
|
| 2453 |
|
| 2454 |
|
| 2455 | tcp_free_fastopen_req(tp);
|
| 2456 | tcp_fastopen_destroy_cipher(sk);
|
| 2457 | tcp_saved_syn_free(tp);
|
| 2458 |
|
| 2459 | sk_sockets_allocated_dec(sk);
|
| 2460 | }
|
| 2461 |
|
| 2462 | #ifdef CONFIG_PROC_FS
|
| 2463 |
|
| 2464 |
|
| 2465 | static unsigned short seq_file_family(const struct seq_file *seq);
|
| 2466 |
|
| 2467 | static bool seq_sk_match(struct seq_file *seq, const struct sock *sk)
|
| 2468 | {
|
| 2469 | unsigned short family = seq_file_family(seq);
|
| 2470 |
|
| 2471 |
|
| 2472 | return ((family == AF_UNSPEC || family == sk->sk_family) &&
|
| 2473 | net_eq(sock_net(sk), seq_file_net(seq)));
|
| 2474 | }
|
| 2475 |
|
| 2476 | |
| 2477 | * and return the first sk from it.
|
| 2478 |
|
| 2479 | static void *listening_get_first(struct seq_file *seq)
|
| 2480 | {
|
| 2481 | struct inet_hashinfo *hinfo = seq_file_net(seq)->ipv4.tcp_death_row.hashinfo;
|
| 2482 | struct tcp_iter_state *st = seq->private;
|
| 2483 |
|
| 2484 | st->offset = 0;
|
| 2485 | for (; st->bucket <= hinfo->lhash2_mask; st->bucket++) {
|
| 2486 | struct inet_listen_hashbucket *ilb2;
|
| 2487 | struct hlist_nulls_node *node;
|
| 2488 | struct sock *sk;
|
| 2489 |
|
| 2490 | ilb2 = &hinfo->lhash2[st->bucket];
|
| 2491 | if (hlist_nulls_empty(&ilb2->nulls_head))
|
| 2492 | continue;
|
| 2493 |
|
| 2494 | spin_lock(&ilb2->lock);
|
| 2495 | sk_nulls_for_each(sk, node, &ilb2->nulls_head) {
|
| 2496 | if (seq_sk_match(seq, sk))
|
| 2497 | return sk;
|
| 2498 | }
|
| 2499 | spin_unlock(&ilb2->lock);
|
| 2500 | }
|
| 2501 |
|
| 2502 | return NULL;
|
| 2503 | }
|
| 2504 |
|
| 2505 | |
| 2506 | * If "cur" is the last one in the st->bucket,
|
| 2507 | * call listening_get_first() to return the first sk of the next
|
| 2508 | * non empty bucket.
|
| 2509 |
|
| 2510 | static void *listening_get_next(struct seq_file *seq, void *cur)
|
| 2511 | {
|
| 2512 | struct tcp_iter_state *st = seq->private;
|
| 2513 | struct inet_listen_hashbucket *ilb2;
|
| 2514 | struct hlist_nulls_node *node;
|
| 2515 | struct inet_hashinfo *hinfo;
|
| 2516 | struct sock *sk = cur;
|
| 2517 |
|
| 2518 | ++st->num;
|
| 2519 | ++st->offset;
|
| 2520 |
|
| 2521 | sk = sk_nulls_next(sk);
|
| 2522 | sk_nulls_for_each_from(sk, node) {
|
| 2523 | if (seq_sk_match(seq, sk))
|
| 2524 | return sk;
|
| 2525 | }
|
| 2526 |
|
| 2527 | hinfo = seq_file_net(seq)->ipv4.tcp_death_row.hashinfo;
|
| 2528 | ilb2 = &hinfo->lhash2[st->bucket];
|
| 2529 | spin_unlock(&ilb2->lock);
|
| 2530 | ++st->bucket;
|
| 2531 | return listening_get_first(seq);
|
| 2532 | }
|
| 2533 |
|
| 2534 | static void *listening_get_idx(struct seq_file *seq, loff_t *pos)
|
| 2535 | {
|
| 2536 | struct tcp_iter_state *st = seq->private;
|
| 2537 | void *rc;
|
| 2538 |
|
| 2539 | st->bucket = 0;
|
| 2540 | st->offset = 0;
|
| 2541 | rc = listening_get_first(seq);
|
| 2542 |
|
| 2543 | while (rc && *pos) {
|
| 2544 | rc = listening_get_next(seq, rc);
|
| 2545 | --*pos;
|
| 2546 | }
|
| 2547 | return rc;
|
| 2548 | }
|
| 2549 |
|
| 2550 | static inline bool empty_bucket(struct inet_hashinfo *hinfo,
|
| 2551 | const struct tcp_iter_state *st)
|
| 2552 | {
|
| 2553 | return hlist_nulls_empty(&hinfo->ehash[st->bucket].chain);
|
| 2554 | }
|
| 2555 |
|
| 2556 | |
| 2557 | * Get first established socket starting from bucket given in st->bucket.
|
| 2558 | * If st->bucket is zero, the very first socket in the hash is returned.
|
| 2559 |
|
| 2560 | static void *established_get_first(struct seq_file *seq)
|
| 2561 | {
|
| 2562 | struct inet_hashinfo *hinfo = seq_file_net(seq)->ipv4.tcp_death_row.hashinfo;
|
| 2563 | struct tcp_iter_state *st = seq->private;
|
| 2564 |
|
| 2565 | st->offset = 0;
|
| 2566 | for (; st->bucket <= hinfo->ehash_mask; ++st->bucket) {
|
| 2567 | struct sock *sk;
|
| 2568 | struct hlist_nulls_node *node;
|
| 2569 | spinlock_t *lock = inet_ehash_lockp(hinfo, st->bucket);
|
| 2570 |
|
| 2571 | cond_resched();
|
| 2572 |
|
| 2573 |
|
| 2574 | if (empty_bucket(hinfo, st))
|
| 2575 | continue;
|
| 2576 |
|
| 2577 | spin_lock_bh(lock);
|
| 2578 | sk_nulls_for_each(sk, node, &hinfo->ehash[st->bucket].chain) {
|
| 2579 | if (seq_sk_match(seq, sk))
|
| 2580 | return sk;
|
| 2581 | }
|
| 2582 | spin_unlock_bh(lock);
|
| 2583 | }
|
| 2584 |
|
| 2585 | return NULL;
|
| 2586 | }
|
| 2587 |
|
| 2588 | static void *established_get_next(struct seq_file *seq, void *cur)
|
| 2589 | {
|
| 2590 | struct inet_hashinfo *hinfo = seq_file_net(seq)->ipv4.tcp_death_row.hashinfo;
|
| 2591 | struct tcp_iter_state *st = seq->private;
|
| 2592 | struct hlist_nulls_node *node;
|
| 2593 | struct sock *sk = cur;
|
| 2594 |
|
| 2595 | ++st->num;
|
| 2596 | ++st->offset;
|
| 2597 |
|
| 2598 | sk = sk_nulls_next(sk);
|
| 2599 |
|
| 2600 | sk_nulls_for_each_from(sk, node) {
|
| 2601 | if (seq_sk_match(seq, sk))
|
| 2602 | return sk;
|
| 2603 | }
|
| 2604 |
|
| 2605 | spin_unlock_bh(inet_ehash_lockp(hinfo, st->bucket));
|
| 2606 | ++st->bucket;
|
| 2607 | return established_get_first(seq);
|
| 2608 | }
|
| 2609 |
|
| 2610 | static void *established_get_idx(struct seq_file *seq, loff_t pos)
|
| 2611 | {
|
| 2612 | struct tcp_iter_state *st = seq->private;
|
| 2613 | void *rc;
|
| 2614 |
|
| 2615 | st->bucket = 0;
|
| 2616 | rc = established_get_first(seq);
|
| 2617 |
|
| 2618 | while (rc && pos) {
|
| 2619 | rc = established_get_next(seq, rc);
|
| 2620 | --pos;
|
| 2621 | }
|
| 2622 | return rc;
|
| 2623 | }
|
| 2624 |
|
| 2625 | static void *tcp_get_idx(struct seq_file *seq, loff_t pos)
|
| 2626 | {
|
| 2627 | void *rc;
|
| 2628 | struct tcp_iter_state *st = seq->private;
|
| 2629 |
|
| 2630 | st->state = TCP_SEQ_STATE_LISTENING;
|
| 2631 | rc = listening_get_idx(seq, &pos);
|
| 2632 |
|
| 2633 | if (!rc) {
|
| 2634 | st->state = TCP_SEQ_STATE_ESTABLISHED;
|
| 2635 | rc = established_get_idx(seq, pos);
|
| 2636 | }
|
| 2637 |
|
| 2638 | return rc;
|
| 2639 | }
|
| 2640 |
|
| 2641 | static void *tcp_seek_last_pos(struct seq_file *seq)
|
| 2642 | {
|
| 2643 | struct inet_hashinfo *hinfo = seq_file_net(seq)->ipv4.tcp_death_row.hashinfo;
|
| 2644 | struct tcp_iter_state *st = seq->private;
|
| 2645 | int bucket = st->bucket;
|
| 2646 | int offset = st->offset;
|
| 2647 | int orig_num = st->num;
|
| 2648 | void *rc = NULL;
|
| 2649 |
|
| 2650 | switch (st->state) {
|
| 2651 | case TCP_SEQ_STATE_LISTENING:
|
| 2652 | if (st->bucket > hinfo->lhash2_mask)
|
| 2653 | break;
|
| 2654 | rc = listening_get_first(seq);
|
| 2655 | while (offset-- && rc && bucket == st->bucket)
|
| 2656 | rc = listening_get_next(seq, rc);
|
| 2657 | if (rc)
|
| 2658 | break;
|
| 2659 | st->bucket = 0;
|
| 2660 | st->state = TCP_SEQ_STATE_ESTABLISHED;
|
| 2661 | fallthrough;
|
| 2662 | case TCP_SEQ_STATE_ESTABLISHED:
|
| 2663 | if (st->bucket > hinfo->ehash_mask)
|
| 2664 | break;
|
| 2665 | rc = established_get_first(seq);
|
| 2666 | while (offset-- && rc && bucket == st->bucket)
|
| 2667 | rc = established_get_next(seq, rc);
|
| 2668 | }
|
| 2669 |
|
| 2670 | st->num = orig_num;
|
| 2671 |
|
| 2672 | return rc;
|
| 2673 | }
|
| 2674 |
|
| 2675 | void *tcp_seq_start(struct seq_file *seq, loff_t *pos)
|
| 2676 | {
|
| 2677 | struct tcp_iter_state *st = seq->private;
|
| 2678 | void *rc;
|
| 2679 |
|
| 2680 | if (*pos && *pos == st->last_pos) {
|
| 2681 | rc = tcp_seek_last_pos(seq);
|
| 2682 | if (rc)
|
| 2683 | goto out;
|
| 2684 | }
|
| 2685 |
|
| 2686 | st->state = TCP_SEQ_STATE_LISTENING;
|
| 2687 | st->num = 0;
|
| 2688 | st->bucket = 0;
|
| 2689 | st->offset = 0;
|
| 2690 | rc = *pos ? tcp_get_idx(seq, *pos - 1) : SEQ_START_TOKEN;
|
| 2691 |
|
| 2692 | out:
|
| 2693 | st->last_pos = *pos;
|
| 2694 | return rc;
|
| 2695 | }
|
| 2696 |
|
| 2697 | void *tcp_seq_next(struct seq_file *seq, void *v, loff_t *pos)
|
| 2698 | {
|
| 2699 | struct tcp_iter_state *st = seq->private;
|
| 2700 | void *rc = NULL;
|
| 2701 |
|
| 2702 | if (v == SEQ_START_TOKEN) {
|
| 2703 | rc = tcp_get_idx(seq, 0);
|
| 2704 | goto out;
|
| 2705 | }
|
| 2706 |
|
| 2707 | switch (st->state) {
|
| 2708 | case TCP_SEQ_STATE_LISTENING:
|
| 2709 | rc = listening_get_next(seq, v);
|
| 2710 | if (!rc) {
|
| 2711 | st->state = TCP_SEQ_STATE_ESTABLISHED;
|
| 2712 | st->bucket = 0;
|
| 2713 | st->offset = 0;
|
| 2714 | rc = established_get_first(seq);
|
| 2715 | }
|
| 2716 | break;
|
| 2717 | case TCP_SEQ_STATE_ESTABLISHED:
|
| 2718 | rc = established_get_next(seq, v);
|
| 2719 | break;
|
| 2720 | }
|
| 2721 | out:
|
| 2722 | ++*pos;
|
| 2723 | st->last_pos = *pos;
|
| 2724 | return rc;
|
| 2725 | }
|
| 2726 |
|
| 2727 | void tcp_seq_stop(struct seq_file *seq, void *v)
|
| 2728 | {
|
| 2729 | struct inet_hashinfo *hinfo = seq_file_net(seq)->ipv4.tcp_death_row.hashinfo;
|
| 2730 | struct tcp_iter_state *st = seq->private;
|
| 2731 |
|
| 2732 | switch (st->state) {
|
| 2733 | case TCP_SEQ_STATE_LISTENING:
|
| 2734 | if (v != SEQ_START_TOKEN)
|
| 2735 | spin_unlock(&hinfo->lhash2[st->bucket].lock);
|
| 2736 | break;
|
| 2737 | case TCP_SEQ_STATE_ESTABLISHED:
|
| 2738 | if (v)
|
| 2739 | spin_unlock_bh(inet_ehash_lockp(hinfo, st->bucket));
|
| 2740 | break;
|
| 2741 | }
|
| 2742 | }
|
| 2743 |
|
| 2744 | static void get_openreq4(const struct request_sock *req,
|
| 2745 | struct seq_file *f, int i)
|
| 2746 | {
|
| 2747 | const struct inet_request_sock *ireq = inet_rsk(req);
|
| 2748 | long delta = req->rsk_timer.expires - jiffies;
|
| 2749 |
|
| 2750 | seq_printf(f, "%4d: %08X:%04X %08X:%04X"
|
| 2751 | " %02X %08X:%08X %02X:%08lX %08X %5u %8d %u %d %pK",
|
| 2752 | i,
|
| 2753 | ireq->ir_loc_addr,
|
| 2754 | ireq->ir_num,
|
| 2755 | ireq->ir_rmt_addr,
|
| 2756 | ntohs(ireq->ir_rmt_port),
|
| 2757 | TCP_SYN_RECV,
|
| 2758 | 0, 0,
|
| 2759 | 1,
|
| 2760 | jiffies_delta_to_clock_t(delta),
|
| 2761 | req->num_timeout,
|
| 2762 | from_kuid_munged(seq_user_ns(f),
|
| 2763 | sk_uid(req->rsk_listener)),
|
| 2764 | 0,
|
| 2765 | 0,
|
| 2766 | 0,
|
| 2767 | req);
|
| 2768 | }
|
| 2769 |
|
| 2770 | static void get_tcp4_sock(struct sock *sk, struct seq_file *f, int i)
|
| 2771 | {
|
| 2772 | int timer_active;
|
| 2773 | unsigned long timer_expires;
|
| 2774 | const struct tcp_sock *tp = tcp_sk(sk);
|
| 2775 | const struct inet_connection_sock *icsk = inet_csk(sk);
|
| 2776 | const struct inet_sock *inet = inet_sk(sk);
|
| 2777 | const struct fastopen_queue *fastopenq = &icsk->icsk_accept_queue.fastopenq;
|
| 2778 | __be32 dest = inet->inet_daddr;
|
| 2779 | __be32 src = inet->inet_rcv_saddr;
|
| 2780 | __u16 destp = ntohs(inet->inet_dport);
|
| 2781 | __u16 srcp = ntohs(inet->inet_sport);
|
| 2782 | u8 icsk_pending;
|
| 2783 | int rx_queue;
|
| 2784 | int state;
|
| 2785 |
|
| 2786 | icsk_pending = smp_load_acquire(&icsk->icsk_pending);
|
| 2787 | if (icsk_pending == ICSK_TIME_RETRANS ||
|
| 2788 | icsk_pending == ICSK_TIME_REO_TIMEOUT ||
|
| 2789 | icsk_pending == ICSK_TIME_LOSS_PROBE) {
|
| 2790 | timer_active = 1;
|
| 2791 | timer_expires = tcp_timeout_expires(sk);
|
| 2792 | } else if (icsk_pending == ICSK_TIME_PROBE0) {
|
| 2793 | timer_active = 4;
|
| 2794 | timer_expires = tcp_timeout_expires(sk);
|
| 2795 | } else if (timer_pending(&icsk->icsk_keepalive_timer)) {
|
| 2796 | timer_active = 2;
|
| 2797 | timer_expires = icsk->icsk_keepalive_timer.expires;
|
| 2798 | } else {
|
| 2799 | timer_active = 0;
|
| 2800 | timer_expires = jiffies;
|
| 2801 | }
|
| 2802 |
|
| 2803 | state = inet_sk_state_load(sk);
|
| 2804 | if (state == TCP_LISTEN)
|
| 2805 | rx_queue = READ_ONCE(sk->sk_ack_backlog);
|
| 2806 | else
|
| 2807 | |
| 2808 | * we might find a transient negative value.
|
| 2809 |
|
| 2810 | rx_queue = max_t(int, READ_ONCE(tp->rcv_nxt) -
|
| 2811 | READ_ONCE(tp->copied_seq), 0);
|
| 2812 |
|
| 2813 | seq_printf(f, "%4d: %08X:%04X %08X:%04X %02X %08X:%08X %02X:%08lX "
|
| 2814 | "%08X %5u %8d %llu %d %pK %lu %lu %u %u %d",
|
| 2815 | i, src, srcp, dest, destp, state,
|
| 2816 | READ_ONCE(tp->write_seq) - tp->snd_una,
|
| 2817 | rx_queue,
|
| 2818 | timer_active,
|
| 2819 | jiffies_delta_to_clock_t(timer_expires - jiffies),
|
| 2820 | READ_ONCE(icsk->icsk_retransmits),
|
| 2821 | from_kuid_munged(seq_user_ns(f), sk_uid(sk)),
|
| 2822 | READ_ONCE(icsk->icsk_probes_out),
|
| 2823 | sock_i_ino(sk),
|
| 2824 | refcount_read(&sk->sk_refcnt), sk,
|
| 2825 | jiffies_to_clock_t(icsk->icsk_rto),
|
| 2826 | jiffies_to_clock_t(icsk->icsk_ack.ato),
|
| 2827 | (icsk->icsk_ack.quick << 1) | inet_csk_in_pingpong_mode(sk),
|
| 2828 | tcp_snd_cwnd(tp),
|
| 2829 | state == TCP_LISTEN ?
|
| 2830 | fastopenq->max_qlen :
|
| 2831 | (tcp_in_initial_slowstart(tp) ? -1 : tp->snd_ssthresh));
|
| 2832 | }
|
| 2833 |
|
| 2834 | static void get_timewait4_sock(const struct inet_timewait_sock *tw,
|
| 2835 | struct seq_file *f, int i)
|
| 2836 | {
|
| 2837 | long delta = tw->tw_timer.expires - jiffies;
|
| 2838 | __be32 dest, src;
|
| 2839 | __u16 destp, srcp;
|
| 2840 |
|
| 2841 | dest = tw->tw_daddr;
|
| 2842 | src = tw->tw_rcv_saddr;
|
| 2843 | destp = ntohs(tw->tw_dport);
|
| 2844 | srcp = ntohs(tw->tw_sport);
|
| 2845 |
|
| 2846 | seq_printf(f, "%4d: %08X:%04X %08X:%04X"
|
| 2847 | " %02X %08X:%08X %02X:%08lX %08X %5d %8d %d %d %pK",
|
| 2848 | i, src, srcp, dest, destp, READ_ONCE(tw->tw_substate), 0, 0,
|
| 2849 | 3, jiffies_delta_to_clock_t(delta), 0, 0, 0, 0,
|
| 2850 | refcount_read(&tw->tw_refcnt), tw);
|
| 2851 | }
|
| 2852 |
|
| 2853 | #define TMPSZ 150
|
| 2854 |
|
| 2855 | static int tcp4_seq_show(struct seq_file *seq, void *v)
|
| 2856 | {
|
| 2857 | struct tcp_iter_state *st;
|
| 2858 | struct sock *sk = v;
|
| 2859 |
|
| 2860 | seq_setwidth(seq, TMPSZ - 1);
|
| 2861 | if (v == SEQ_START_TOKEN) {
|
| 2862 | seq_puts(seq, " sl local_address rem_address st tx_queue "
|
| 2863 | "rx_queue tr tm->when retrnsmt uid timeout "
|
| 2864 | "inode");
|
| 2865 | goto out;
|
| 2866 | }
|
| 2867 | st = seq->private;
|
| 2868 |
|
| 2869 | if (sk->sk_state == TCP_TIME_WAIT)
|
| 2870 | get_timewait4_sock(v, seq, st->num);
|
| 2871 | else if (sk->sk_state == TCP_NEW_SYN_RECV)
|
| 2872 | get_openreq4(v, seq, st->num);
|
| 2873 | else
|
| 2874 | get_tcp4_sock(v, seq, st->num);
|
| 2875 | out:
|
| 2876 | seq_pad(seq, '\n');
|
| 2877 | return 0;
|
| 2878 | }
|
| 2879 |
|
| 2880 | #ifdef CONFIG_BPF_SYSCALL
|
| 2881 | union bpf_tcp_iter_batch_item {
|
| 2882 | struct sock *sk;
|
| 2883 | __u64 cookie;
|
| 2884 | };
|
| 2885 |
|
| 2886 | struct bpf_tcp_iter_state {
|
| 2887 | struct tcp_iter_state state;
|
| 2888 | unsigned int cur_sk;
|
| 2889 | unsigned int end_sk;
|
| 2890 | unsigned int max_sk;
|
| 2891 | union bpf_tcp_iter_batch_item *batch;
|
| 2892 | };
|
| 2893 |
|
| 2894 | struct bpf_iter__tcp {
|
| 2895 | __bpf_md_ptr(struct bpf_iter_meta *, meta);
|
| 2896 | __bpf_md_ptr(struct sock_common *, sk_common);
|
| 2897 | uid_t uid __aligned(8);
|
| 2898 | };
|
| 2899 |
|
| 2900 | static int tcp_prog_seq_show(struct bpf_prog *prog, struct bpf_iter_meta *meta,
|
| 2901 | struct sock_common *sk_common, uid_t uid)
|
| 2902 | {
|
| 2903 | struct bpf_iter__tcp ctx;
|
| 2904 |
|
| 2905 | meta->seq_num--;
|
| 2906 | ctx.meta = meta;
|
| 2907 | ctx.sk_common = sk_common;
|
| 2908 | ctx.uid = uid;
|
| 2909 | return bpf_iter_run_prog(prog, &ctx);
|
| 2910 | }
|
| 2911 |
|
| 2912 | static void bpf_iter_tcp_put_batch(struct bpf_tcp_iter_state *iter)
|
| 2913 | {
|
| 2914 | union bpf_tcp_iter_batch_item *item;
|
| 2915 | unsigned int cur_sk = iter->cur_sk;
|
| 2916 | __u64 cookie;
|
| 2917 |
|
| 2918 | |
| 2919 | * pick up where we left off next time around.
|
| 2920 |
|
| 2921 | while (cur_sk < iter->end_sk) {
|
| 2922 | item = &iter->batch[cur_sk++];
|
| 2923 | cookie = sock_gen_cookie(item->sk);
|
| 2924 | sock_gen_put(item->sk);
|
| 2925 | item->cookie = cookie;
|
| 2926 | }
|
| 2927 | }
|
| 2928 |
|
| 2929 | static int bpf_iter_tcp_realloc_batch(struct bpf_tcp_iter_state *iter,
|
| 2930 | unsigned int new_batch_sz, gfp_t flags)
|
| 2931 | {
|
| 2932 | union bpf_tcp_iter_batch_item *new_batch;
|
| 2933 |
|
| 2934 | new_batch = kvmalloc_objs(*new_batch, new_batch_sz,
|
| 2935 | flags | __GFP_NOWARN);
|
| 2936 | if (!new_batch)
|
| 2937 | return -ENOMEM;
|
| 2938 |
|
| 2939 | memcpy(new_batch, iter->batch, sizeof(*iter->batch) * iter->end_sk);
|
| 2940 | kvfree(iter->batch);
|
| 2941 | iter->batch = new_batch;
|
| 2942 | iter->max_sk = new_batch_sz;
|
| 2943 |
|
| 2944 | return 0;
|
| 2945 | }
|
| 2946 |
|
| 2947 | static struct sock *bpf_iter_tcp_resume_bucket(struct sock *first_sk,
|
| 2948 | union bpf_tcp_iter_batch_item *cookies,
|
| 2949 | int n_cookies)
|
| 2950 | {
|
| 2951 | struct hlist_nulls_node *node;
|
| 2952 | struct sock *sk;
|
| 2953 | int i;
|
| 2954 |
|
| 2955 | for (i = 0; i < n_cookies; i++) {
|
| 2956 | sk = first_sk;
|
| 2957 | sk_nulls_for_each_from(sk, node)
|
| 2958 | if (cookies[i].cookie == atomic64_read(&sk->sk_cookie))
|
| 2959 | return sk;
|
| 2960 | }
|
| 2961 |
|
| 2962 | return NULL;
|
| 2963 | }
|
| 2964 |
|
| 2965 | static struct sock *bpf_iter_tcp_resume_listening(struct seq_file *seq)
|
| 2966 | {
|
| 2967 | struct inet_hashinfo *hinfo = seq_file_net(seq)->ipv4.tcp_death_row.hashinfo;
|
| 2968 | struct bpf_tcp_iter_state *iter = seq->private;
|
| 2969 | struct tcp_iter_state *st = &iter->state;
|
| 2970 | unsigned int find_cookie = iter->cur_sk;
|
| 2971 | unsigned int end_cookie = iter->end_sk;
|
| 2972 | int resume_bucket = st->bucket;
|
| 2973 | struct sock *sk;
|
| 2974 |
|
| 2975 | if (end_cookie && find_cookie == end_cookie)
|
| 2976 | ++st->bucket;
|
| 2977 |
|
| 2978 | sk = listening_get_first(seq);
|
| 2979 | iter->cur_sk = 0;
|
| 2980 | iter->end_sk = 0;
|
| 2981 |
|
| 2982 | if (sk && st->bucket == resume_bucket && end_cookie) {
|
| 2983 | sk = bpf_iter_tcp_resume_bucket(sk, &iter->batch[find_cookie],
|
| 2984 | end_cookie - find_cookie);
|
| 2985 | if (!sk) {
|
| 2986 | spin_unlock(&hinfo->lhash2[st->bucket].lock);
|
| 2987 | ++st->bucket;
|
| 2988 | sk = listening_get_first(seq);
|
| 2989 | }
|
| 2990 | }
|
| 2991 |
|
| 2992 | return sk;
|
| 2993 | }
|
| 2994 |
|
| 2995 | static struct sock *bpf_iter_tcp_resume_established(struct seq_file *seq)
|
| 2996 | {
|
| 2997 | struct inet_hashinfo *hinfo = seq_file_net(seq)->ipv4.tcp_death_row.hashinfo;
|
| 2998 | struct bpf_tcp_iter_state *iter = seq->private;
|
| 2999 | struct tcp_iter_state *st = &iter->state;
|
| 3000 | unsigned int find_cookie = iter->cur_sk;
|
| 3001 | unsigned int end_cookie = iter->end_sk;
|
| 3002 | int resume_bucket = st->bucket;
|
| 3003 | struct sock *sk;
|
| 3004 |
|
| 3005 | if (end_cookie && find_cookie == end_cookie)
|
| 3006 | ++st->bucket;
|
| 3007 |
|
| 3008 | sk = established_get_first(seq);
|
| 3009 | iter->cur_sk = 0;
|
| 3010 | iter->end_sk = 0;
|
| 3011 |
|
| 3012 | if (sk && st->bucket == resume_bucket && end_cookie) {
|
| 3013 | sk = bpf_iter_tcp_resume_bucket(sk, &iter->batch[find_cookie],
|
| 3014 | end_cookie - find_cookie);
|
| 3015 | if (!sk) {
|
| 3016 | spin_unlock_bh(inet_ehash_lockp(hinfo, st->bucket));
|
| 3017 | ++st->bucket;
|
| 3018 | sk = established_get_first(seq);
|
| 3019 | }
|
| 3020 | }
|
| 3021 |
|
| 3022 | return sk;
|
| 3023 | }
|
| 3024 |
|
| 3025 | static struct sock *bpf_iter_tcp_resume(struct seq_file *seq)
|
| 3026 | {
|
| 3027 | struct bpf_tcp_iter_state *iter = seq->private;
|
| 3028 | struct tcp_iter_state *st = &iter->state;
|
| 3029 | struct sock *sk = NULL;
|
| 3030 |
|
| 3031 | switch (st->state) {
|
| 3032 | case TCP_SEQ_STATE_LISTENING:
|
| 3033 | sk = bpf_iter_tcp_resume_listening(seq);
|
| 3034 | if (sk)
|
| 3035 | break;
|
| 3036 | st->bucket = 0;
|
| 3037 | st->state = TCP_SEQ_STATE_ESTABLISHED;
|
| 3038 | fallthrough;
|
| 3039 | case TCP_SEQ_STATE_ESTABLISHED:
|
| 3040 | sk = bpf_iter_tcp_resume_established(seq);
|
| 3041 | break;
|
| 3042 | }
|
| 3043 |
|
| 3044 | return sk;
|
| 3045 | }
|
| 3046 |
|
| 3047 | static unsigned int bpf_iter_tcp_listening_batch(struct seq_file *seq,
|
| 3048 | struct sock **start_sk)
|
| 3049 | {
|
| 3050 | struct bpf_tcp_iter_state *iter = seq->private;
|
| 3051 | struct hlist_nulls_node *node;
|
| 3052 | unsigned int expected = 1;
|
| 3053 | struct sock *sk;
|
| 3054 |
|
| 3055 | sock_hold(*start_sk);
|
| 3056 | iter->batch[iter->end_sk++].sk = *start_sk;
|
| 3057 |
|
| 3058 | sk = sk_nulls_next(*start_sk);
|
| 3059 | *start_sk = NULL;
|
| 3060 | sk_nulls_for_each_from(sk, node) {
|
| 3061 | if (seq_sk_match(seq, sk)) {
|
| 3062 | if (iter->end_sk < iter->max_sk) {
|
| 3063 | sock_hold(sk);
|
| 3064 | iter->batch[iter->end_sk++].sk = sk;
|
| 3065 | } else if (!*start_sk) {
|
| 3066 |
|
| 3067 | *start_sk = sk;
|
| 3068 | }
|
| 3069 | expected++;
|
| 3070 | }
|
| 3071 | }
|
| 3072 |
|
| 3073 | return expected;
|
| 3074 | }
|
| 3075 |
|
| 3076 | static unsigned int bpf_iter_tcp_established_batch(struct seq_file *seq,
|
| 3077 | struct sock **start_sk)
|
| 3078 | {
|
| 3079 | struct bpf_tcp_iter_state *iter = seq->private;
|
| 3080 | struct hlist_nulls_node *node;
|
| 3081 | struct sock *sk = *start_sk;
|
| 3082 | unsigned int expected = 0;
|
| 3083 |
|
| 3084 | *start_sk = NULL;
|
| 3085 | sk_nulls_for_each_from(sk, node) {
|
| 3086 | if (!seq_sk_match(seq, sk))
|
| 3087 | continue;
|
| 3088 | expected++;
|
| 3089 | if (iter->end_sk < iter->max_sk) {
|
| 3090 | |
| 3091 | * and refcount_set()s it after the bucket lock drops.
|
| 3092 |
|
| 3093 | if (unlikely(!refcount_inc_not_zero(&sk->sk_refcnt)))
|
| 3094 | continue;
|
| 3095 | iter->batch[iter->end_sk++].sk = sk;
|
| 3096 | } else if (!*start_sk) {
|
| 3097 |
|
| 3098 | *start_sk = sk;
|
| 3099 | }
|
| 3100 | }
|
| 3101 |
|
| 3102 | return expected;
|
| 3103 | }
|
| 3104 |
|
| 3105 | static unsigned int bpf_iter_fill_batch(struct seq_file *seq,
|
| 3106 | struct sock **start_sk)
|
| 3107 | {
|
| 3108 | struct bpf_tcp_iter_state *iter = seq->private;
|
| 3109 | struct tcp_iter_state *st = &iter->state;
|
| 3110 |
|
| 3111 | if (st->state == TCP_SEQ_STATE_LISTENING)
|
| 3112 | return bpf_iter_tcp_listening_batch(seq, start_sk);
|
| 3113 | else
|
| 3114 | return bpf_iter_tcp_established_batch(seq, start_sk);
|
| 3115 | }
|
| 3116 |
|
| 3117 | static void bpf_iter_tcp_unlock_bucket(struct seq_file *seq)
|
| 3118 | {
|
| 3119 | struct inet_hashinfo *hinfo = seq_file_net(seq)->ipv4.tcp_death_row.hashinfo;
|
| 3120 | struct bpf_tcp_iter_state *iter = seq->private;
|
| 3121 | struct tcp_iter_state *st = &iter->state;
|
| 3122 |
|
| 3123 | if (st->state == TCP_SEQ_STATE_LISTENING)
|
| 3124 | spin_unlock(&hinfo->lhash2[st->bucket].lock);
|
| 3125 | else
|
| 3126 | spin_unlock_bh(inet_ehash_lockp(hinfo, st->bucket));
|
| 3127 | }
|
| 3128 |
|
| 3129 | static struct sock *bpf_iter_tcp_batch(struct seq_file *seq)
|
| 3130 | {
|
| 3131 | struct bpf_tcp_iter_state *iter = seq->private;
|
| 3132 | unsigned int expected;
|
| 3133 | struct sock *sk;
|
| 3134 | int err;
|
| 3135 |
|
| 3136 | again:
|
| 3137 | sk = bpf_iter_tcp_resume(seq);
|
| 3138 | if (!sk)
|
| 3139 | return NULL;
|
| 3140 |
|
| 3141 | expected = bpf_iter_fill_batch(seq, &sk);
|
| 3142 | if (likely(!sk))
|
| 3143 | goto done;
|
| 3144 |
|
| 3145 |
|
| 3146 | bpf_iter_tcp_unlock_bucket(seq);
|
| 3147 | bpf_iter_tcp_put_batch(iter);
|
| 3148 | err = bpf_iter_tcp_realloc_batch(iter, expected * 3 / 2,
|
| 3149 | GFP_USER);
|
| 3150 | if (err) {
|
| 3151 | iter->cur_sk = 0;
|
| 3152 | iter->end_sk = 0;
|
| 3153 | return ERR_PTR(err);
|
| 3154 | }
|
| 3155 |
|
| 3156 | sk = bpf_iter_tcp_resume(seq);
|
| 3157 | if (!sk)
|
| 3158 | return NULL;
|
| 3159 |
|
| 3160 | expected = bpf_iter_fill_batch(seq, &sk);
|
| 3161 | if (likely(!sk))
|
| 3162 | goto done;
|
| 3163 |
|
| 3164 | |
| 3165 | * again with a larger batch to make sure the current bucket's size
|
| 3166 | * does not change in the meantime.
|
| 3167 |
|
| 3168 | err = bpf_iter_tcp_realloc_batch(iter, expected, GFP_NOWAIT);
|
| 3169 | if (err) {
|
| 3170 | bpf_iter_tcp_unlock_bucket(seq);
|
| 3171 | return ERR_PTR(err);
|
| 3172 | }
|
| 3173 |
|
| 3174 | bpf_iter_fill_batch(seq, &sk);
|
| 3175 | WARN_ON_ONCE(sk);
|
| 3176 | done:
|
| 3177 | bpf_iter_tcp_unlock_bucket(seq);
|
| 3178 | if (unlikely(!iter->end_sk)) {
|
| 3179 | ++iter->state.bucket;
|
| 3180 | goto again;
|
| 3181 | }
|
| 3182 | return iter->batch[0].sk;
|
| 3183 | }
|
| 3184 |
|
| 3185 | static void *bpf_iter_tcp_seq_start(struct seq_file *seq, loff_t *pos)
|
| 3186 | {
|
| 3187 | |
| 3188 | * continue from where it was stop()-ped.
|
| 3189 |
|
| 3190 | if (*pos)
|
| 3191 | return bpf_iter_tcp_batch(seq);
|
| 3192 |
|
| 3193 | return SEQ_START_TOKEN;
|
| 3194 | }
|
| 3195 |
|
| 3196 | static void *bpf_iter_tcp_seq_next(struct seq_file *seq, void *v, loff_t *pos)
|
| 3197 | {
|
| 3198 | struct bpf_tcp_iter_state *iter = seq->private;
|
| 3199 | struct tcp_iter_state *st = &iter->state;
|
| 3200 | struct sock *sk;
|
| 3201 |
|
| 3202 | |
| 3203 | * done with seq_show(), so advance to the next sk in
|
| 3204 | * the batch.
|
| 3205 |
|
| 3206 | if (iter->cur_sk < iter->end_sk) {
|
| 3207 | |
| 3208 | * bpf_iter_tcp does not use st->num.
|
| 3209 | * meta.seq_num is used instead.
|
| 3210 |
|
| 3211 | st->num++;
|
| 3212 | sock_gen_put(iter->batch[iter->cur_sk++].sk);
|
| 3213 | }
|
| 3214 |
|
| 3215 | if (iter->cur_sk < iter->end_sk)
|
| 3216 | sk = iter->batch[iter->cur_sk].sk;
|
| 3217 | else
|
| 3218 | sk = bpf_iter_tcp_batch(seq);
|
| 3219 |
|
| 3220 | ++*pos;
|
| 3221 | |
| 3222 | * bpf iter does not do lseek, so st->last_pos always equals to *pos.
|
| 3223 |
|
| 3224 | st->last_pos = *pos;
|
| 3225 | return sk;
|
| 3226 | }
|
| 3227 |
|
| 3228 | static int bpf_iter_tcp_seq_show(struct seq_file *seq, void *v)
|
| 3229 | {
|
| 3230 | struct bpf_iter_meta meta;
|
| 3231 | struct bpf_prog *prog;
|
| 3232 | struct sock *sk = v;
|
| 3233 | uid_t uid;
|
| 3234 | int ret;
|
| 3235 |
|
| 3236 | if (v == SEQ_START_TOKEN)
|
| 3237 | return 0;
|
| 3238 |
|
| 3239 | if (sk_fullsock(sk))
|
| 3240 | lock_sock(sk);
|
| 3241 |
|
| 3242 | if (unlikely(sk_unhashed(sk))) {
|
| 3243 | ret = SEQ_SKIP;
|
| 3244 | goto unlock;
|
| 3245 | }
|
| 3246 |
|
| 3247 | if (sk->sk_state == TCP_TIME_WAIT) {
|
| 3248 | uid = 0;
|
| 3249 | } else if (sk->sk_state == TCP_NEW_SYN_RECV) {
|
| 3250 | const struct request_sock *req = v;
|
| 3251 |
|
| 3252 | uid = from_kuid_munged(seq_user_ns(seq),
|
| 3253 | sk_uid(req->rsk_listener));
|
| 3254 | } else {
|
| 3255 | uid = from_kuid_munged(seq_user_ns(seq), sk_uid(sk));
|
| 3256 | }
|
| 3257 |
|
| 3258 | meta.seq = seq;
|
| 3259 | prog = bpf_iter_get_info(&meta, false);
|
| 3260 | ret = tcp_prog_seq_show(prog, &meta, v, uid);
|
| 3261 |
|
| 3262 | unlock:
|
| 3263 | if (sk_fullsock(sk))
|
| 3264 | release_sock(sk);
|
| 3265 | return ret;
|
| 3266 |
|
| 3267 | }
|
| 3268 |
|
| 3269 | static void bpf_iter_tcp_seq_stop(struct seq_file *seq, void *v)
|
| 3270 | {
|
| 3271 | struct bpf_tcp_iter_state *iter = seq->private;
|
| 3272 | struct bpf_iter_meta meta;
|
| 3273 | struct bpf_prog *prog;
|
| 3274 |
|
| 3275 | if (!v) {
|
| 3276 | meta.seq = seq;
|
| 3277 | prog = bpf_iter_get_info(&meta, true);
|
| 3278 | if (prog)
|
| 3279 | (void)tcp_prog_seq_show(prog, &meta, v, 0);
|
| 3280 | }
|
| 3281 |
|
| 3282 | if (iter->cur_sk < iter->end_sk)
|
| 3283 | bpf_iter_tcp_put_batch(iter);
|
| 3284 | }
|
| 3285 |
|
| 3286 | static const struct seq_operations bpf_iter_tcp_seq_ops = {
|
| 3287 | .show = bpf_iter_tcp_seq_show,
|
| 3288 | .start = bpf_iter_tcp_seq_start,
|
| 3289 | .next = bpf_iter_tcp_seq_next,
|
| 3290 | .stop = bpf_iter_tcp_seq_stop,
|
| 3291 | };
|
| 3292 | #endif
|
| 3293 | static unsigned short seq_file_family(const struct seq_file *seq)
|
| 3294 | {
|
| 3295 | const struct tcp_seq_afinfo *afinfo;
|
| 3296 |
|
| 3297 | #ifdef CONFIG_BPF_SYSCALL
|
| 3298 |
|
| 3299 | if (seq->op == &bpf_iter_tcp_seq_ops)
|
| 3300 | return AF_UNSPEC;
|
| 3301 | #endif
|
| 3302 |
|
| 3303 |
|
| 3304 | afinfo = pde_data(file_inode(seq->file));
|
| 3305 | return afinfo->family;
|
| 3306 | }
|
| 3307 |
|
| 3308 | static const struct seq_operations tcp4_seq_ops = {
|
| 3309 | .show = tcp4_seq_show,
|
| 3310 | .start = tcp_seq_start,
|
| 3311 | .next = tcp_seq_next,
|
| 3312 | .stop = tcp_seq_stop,
|
| 3313 | };
|
| 3314 |
|
| 3315 | static struct tcp_seq_afinfo tcp4_seq_afinfo = {
|
| 3316 | .family = AF_INET,
|
| 3317 | };
|
| 3318 |
|
| 3319 | static int __net_init tcp4_proc_init_net(struct net *net)
|
| 3320 | {
|
| 3321 | if (!proc_create_net_data("tcp", 0444, net->proc_net, &tcp4_seq_ops,
|
| 3322 | sizeof(struct tcp_iter_state), &tcp4_seq_afinfo))
|
| 3323 | return -ENOMEM;
|
| 3324 | return 0;
|
| 3325 | }
|
| 3326 |
|
| 3327 | static void __net_exit tcp4_proc_exit_net(struct net *net)
|
| 3328 | {
|
| 3329 | remove_proc_entry("tcp", net->proc_net);
|
| 3330 | }
|
| 3331 |
|
| 3332 | static struct pernet_operations tcp4_net_ops = {
|
| 3333 | .init = tcp4_proc_init_net,
|
| 3334 | .exit = tcp4_proc_exit_net,
|
| 3335 | };
|
| 3336 |
|
| 3337 | int __init tcp4_proc_init(void)
|
| 3338 | {
|
| 3339 | return register_pernet_subsys(&tcp4_net_ops);
|
| 3340 | }
|
| 3341 |
|
| 3342 | void tcp4_proc_exit(void)
|
| 3343 | {
|
| 3344 | unregister_pernet_subsys(&tcp4_net_ops);
|
| 3345 | }
|
| 3346 | #endif
|
| 3347 |
|
| 3348 | struct proto tcp_prot = {
|
| 3349 | .name = "TCP",
|
| 3350 | .owner = THIS_MODULE,
|
| 3351 | .close = tcp_close,
|
| 3352 | .pre_connect = tcp_v4_pre_connect,
|
| 3353 | .connect = tcp_v4_connect,
|
| 3354 | .disconnect = tcp_disconnect,
|
| 3355 | .accept = inet_csk_accept,
|
| 3356 | .ioctl = tcp_ioctl,
|
| 3357 | .init = tcp_v4_init_sock,
|
| 3358 | .destroy = tcp_v4_destroy_sock,
|
| 3359 | .shutdown = tcp_shutdown,
|
| 3360 | .setsockopt = tcp_setsockopt,
|
| 3361 | .getsockopt = tcp_getsockopt,
|
| 3362 | .bpf_bypass_getsockopt = tcp_bpf_bypass_getsockopt,
|
| 3363 | .keepalive = tcp_set_keepalive,
|
| 3364 | .recvmsg = tcp_recvmsg,
|
| 3365 | .sendmsg = tcp_sendmsg,
|
| 3366 | .splice_eof = tcp_splice_eof,
|
| 3367 | .backlog_rcv = tcp_v4_do_rcv,
|
| 3368 | .release_cb = tcp_release_cb,
|
| 3369 | .hash = inet_hash,
|
| 3370 | .unhash = inet_unhash,
|
| 3371 | .get_port = inet_csk_get_port,
|
| 3372 | .put_port = inet_put_port,
|
| 3373 | #ifdef CONFIG_BPF_SYSCALL
|
| 3374 | .psock_update_sk_prot = tcp_bpf_update_proto,
|
| 3375 | #endif
|
| 3376 | .enter_memory_pressure = tcp_enter_memory_pressure,
|
| 3377 | .leave_memory_pressure = tcp_leave_memory_pressure,
|
| 3378 | .stream_memory_free = tcp_stream_memory_free,
|
| 3379 | .sockets_allocated = &tcp_sockets_allocated,
|
| 3380 |
|
| 3381 | .memory_allocated = &net_aligned_data.tcp_memory_allocated,
|
| 3382 | .per_cpu_fw_alloc = &tcp_memory_per_cpu_fw_alloc,
|
| 3383 |
|
| 3384 | .memory_pressure = &tcp_memory_pressure,
|
| 3385 | .sysctl_mem = sysctl_tcp_mem,
|
| 3386 | .sysctl_wmem_offset = offsetof(struct net, ipv4.sysctl_tcp_wmem),
|
| 3387 | .sysctl_rmem_offset = offsetof(struct net, ipv4.sysctl_tcp_rmem),
|
| 3388 | .max_header = MAX_TCP_HEADER,
|
| 3389 | .obj_size = sizeof(struct tcp_sock),
|
| 3390 | .freeptr_offset = offsetof(struct tcp_sock,
|
| 3391 | inet_conn.icsk_inet.sk.sk_freeptr),
|
| 3392 | .slab_flags = SLAB_TYPESAFE_BY_RCU,
|
| 3393 | .twsk_prot = &tcp_timewait_sock_ops,
|
| 3394 | .rsk_prot = &tcp_request_sock_ops,
|
| 3395 | .h.hashinfo = NULL,
|
| 3396 | .no_autobind = true,
|
| 3397 | .diag_destroy = tcp_abort,
|
| 3398 | };
|
| 3399 | EXPORT_SYMBOL(tcp_prot);
|
| 3400 |
|
| 3401 | static void __net_exit tcp_sk_exit(struct net *net)
|
| 3402 | {
|
| 3403 | if (net->ipv4.tcp_congestion_control)
|
| 3404 | bpf_module_put(net->ipv4.tcp_congestion_control,
|
| 3405 | net->ipv4.tcp_congestion_control->owner);
|
| 3406 | }
|
| 3407 |
|
| 3408 | static void __net_init tcp_set_hashinfo(struct net *net)
|
| 3409 | {
|
| 3410 | struct inet_hashinfo *hinfo;
|
| 3411 | unsigned int ehash_entries;
|
| 3412 | struct net *old_net;
|
| 3413 |
|
| 3414 | if (net_eq(net, &init_net))
|
| 3415 | goto fallback;
|
| 3416 |
|
| 3417 | old_net = current->nsproxy->net_ns;
|
| 3418 | ehash_entries = READ_ONCE(old_net->ipv4.sysctl_tcp_child_ehash_entries);
|
| 3419 | if (!ehash_entries)
|
| 3420 | goto fallback;
|
| 3421 |
|
| 3422 | ehash_entries = roundup_pow_of_two(ehash_entries);
|
| 3423 | hinfo = inet_pernet_hashinfo_alloc(&tcp_hashinfo, ehash_entries);
|
| 3424 | if (!hinfo) {
|
| 3425 | pr_warn("Failed to allocate TCP ehash (entries: %u) "
|
| 3426 | "for a netns, fallback to the global one\n",
|
| 3427 | ehash_entries);
|
| 3428 | fallback:
|
| 3429 | hinfo = &tcp_hashinfo;
|
| 3430 | ehash_entries = tcp_hashinfo.ehash_mask + 1;
|
| 3431 | }
|
| 3432 |
|
| 3433 | net->ipv4.tcp_death_row.hashinfo = hinfo;
|
| 3434 | net->ipv4.tcp_death_row.sysctl_max_tw_buckets = ehash_entries / 2;
|
| 3435 | net->ipv4.sysctl_max_syn_backlog = max(128U, ehash_entries / 128);
|
| 3436 | }
|
| 3437 |
|
| 3438 | static int __net_init tcp_sk_init(struct net *net)
|
| 3439 | {
|
| 3440 | net->ipv4.sysctl_tcp_ecn = TCP_ECN_IN_ECN_OUT_NOECN;
|
| 3441 | net->ipv4.sysctl_tcp_ecn_option = TCP_ACCECN_OPTION_FULL;
|
| 3442 | net->ipv4.sysctl_tcp_ecn_option_beacon = TCP_ACCECN_OPTION_BEACON;
|
| 3443 | net->ipv4.sysctl_tcp_ecn_fallback = 1;
|
| 3444 |
|
| 3445 | net->ipv4.sysctl_tcp_base_mss = TCP_BASE_MSS;
|
| 3446 | net->ipv4.sysctl_tcp_min_snd_mss = TCP_MIN_SND_MSS;
|
| 3447 | net->ipv4.sysctl_tcp_probe_threshold = TCP_PROBE_THRESHOLD;
|
| 3448 | net->ipv4.sysctl_tcp_probe_interval = TCP_PROBE_INTERVAL;
|
| 3449 | net->ipv4.sysctl_tcp_mtu_probe_floor = TCP_MIN_SND_MSS;
|
| 3450 |
|
| 3451 | net->ipv4.sysctl_tcp_keepalive_time = TCP_KEEPALIVE_TIME;
|
| 3452 | net->ipv4.sysctl_tcp_keepalive_probes = TCP_KEEPALIVE_PROBES;
|
| 3453 | net->ipv4.sysctl_tcp_keepalive_intvl = TCP_KEEPALIVE_INTVL;
|
| 3454 |
|
| 3455 | net->ipv4.sysctl_tcp_syn_retries = TCP_SYN_RETRIES;
|
| 3456 | net->ipv4.sysctl_tcp_synack_retries = TCP_SYNACK_RETRIES;
|
| 3457 | net->ipv4.sysctl_tcp_syncookies = 1;
|
| 3458 | net->ipv4.sysctl_tcp_reordering = TCP_FASTRETRANS_THRESH;
|
| 3459 | net->ipv4.sysctl_tcp_retries1 = TCP_RETR1;
|
| 3460 | net->ipv4.sysctl_tcp_retries2 = TCP_RETR2;
|
| 3461 | net->ipv4.sysctl_tcp_orphan_retries = 0;
|
| 3462 | net->ipv4.sysctl_tcp_fin_timeout = TCP_FIN_TIMEOUT;
|
| 3463 | net->ipv4.sysctl_tcp_notsent_lowat = UINT_MAX;
|
| 3464 | net->ipv4.sysctl_tcp_tw_reuse = 2;
|
| 3465 | net->ipv4.sysctl_tcp_tw_reuse_delay = 1 * MSEC_PER_SEC;
|
| 3466 | net->ipv4.sysctl_tcp_no_ssthresh_metrics_save = 1;
|
| 3467 |
|
| 3468 | refcount_set(&net->ipv4.tcp_death_row.tw_refcount, 1);
|
| 3469 | tcp_set_hashinfo(net);
|
| 3470 |
|
| 3471 | net->ipv4.sysctl_tcp_sack = 1;
|
| 3472 | net->ipv4.sysctl_tcp_window_scaling = 1;
|
| 3473 | net->ipv4.sysctl_tcp_timestamps = 1;
|
| 3474 | net->ipv4.sysctl_tcp_early_retrans = 3;
|
| 3475 | net->ipv4.sysctl_tcp_recovery = TCP_RACK_LOSS_DETECTION;
|
| 3476 | net->ipv4.sysctl_tcp_slow_start_after_idle = 1;
|
| 3477 | net->ipv4.sysctl_tcp_retrans_collapse = 1;
|
| 3478 | net->ipv4.sysctl_tcp_max_reordering = 300;
|
| 3479 | net->ipv4.sysctl_tcp_dsack = 1;
|
| 3480 | net->ipv4.sysctl_tcp_app_win = 31;
|
| 3481 | net->ipv4.sysctl_tcp_adv_win_scale = 1;
|
| 3482 | net->ipv4.sysctl_tcp_frto = 2;
|
| 3483 | net->ipv4.sysctl_tcp_moderate_rcvbuf = 1;
|
| 3484 | net->ipv4.sysctl_tcp_rcvbuf_low_rtt = USEC_PER_MSEC;
|
| 3485 | |
| 3486 | * will allow a single TSO frame to consume. Building TSO frames
|
| 3487 | * which are too large can cause TCP streams to be bursty.
|
| 3488 |
|
| 3489 | net->ipv4.sysctl_tcp_tso_win_divisor = 3;
|
| 3490 |
|
| 3491 | net->ipv4.sysctl_tcp_limit_output_bytes = 4 << 20;
|
| 3492 |
|
| 3493 |
|
| 3494 | net->ipv4.sysctl_tcp_challenge_ack_limit = INT_MAX;
|
| 3495 |
|
| 3496 | net->ipv4.sysctl_tcp_min_tso_segs = 2;
|
| 3497 | net->ipv4.sysctl_tcp_tso_rtt_log = 9;
|
| 3498 | net->ipv4.sysctl_tcp_min_rtt_wlen = 300;
|
| 3499 | net->ipv4.sysctl_tcp_autocorking = 1;
|
| 3500 | net->ipv4.sysctl_tcp_invalid_ratelimit = HZ/2;
|
| 3501 | net->ipv4.sysctl_tcp_pacing_ss_ratio = 200;
|
| 3502 | net->ipv4.sysctl_tcp_pacing_ca_ratio = 120;
|
| 3503 | if (net != &init_net) {
|
| 3504 | memcpy(net->ipv4.sysctl_tcp_rmem,
|
| 3505 | init_net.ipv4.sysctl_tcp_rmem,
|
| 3506 | sizeof(init_net.ipv4.sysctl_tcp_rmem));
|
| 3507 | memcpy(net->ipv4.sysctl_tcp_wmem,
|
| 3508 | init_net.ipv4.sysctl_tcp_wmem,
|
| 3509 | sizeof(init_net.ipv4.sysctl_tcp_wmem));
|
| 3510 | }
|
| 3511 | net->ipv4.sysctl_tcp_comp_sack_delay_ns = NSEC_PER_MSEC;
|
| 3512 | net->ipv4.sysctl_tcp_comp_sack_slack_ns = 10 * NSEC_PER_USEC;
|
| 3513 | net->ipv4.sysctl_tcp_comp_sack_nr = 44;
|
| 3514 | net->ipv4.sysctl_tcp_comp_sack_rtt_percent = 33;
|
| 3515 | net->ipv4.sysctl_tcp_backlog_ack_defer = 1;
|
| 3516 | net->ipv4.sysctl_tcp_fastopen = TFO_CLIENT_ENABLE;
|
| 3517 | net->ipv4.sysctl_tcp_fastopen_blackhole_timeout = 0;
|
| 3518 | atomic_set(&net->ipv4.tfo_active_disable_times, 0);
|
| 3519 |
|
| 3520 |
|
| 3521 | net->ipv4.sysctl_tcp_plb_enabled = 0;
|
| 3522 | net->ipv4.sysctl_tcp_plb_idle_rehash_rounds = 3;
|
| 3523 | net->ipv4.sysctl_tcp_plb_rehash_rounds = 12;
|
| 3524 | net->ipv4.sysctl_tcp_plb_suspend_rto_sec = 60;
|
| 3525 |
|
| 3526 | net->ipv4.sysctl_tcp_plb_cong_thresh = (1 << TCP_PLB_SCALE) / 2;
|
| 3527 |
|
| 3528 |
|
| 3529 | if (!net_eq(net, &init_net) &&
|
| 3530 | bpf_try_module_get(init_net.ipv4.tcp_congestion_control,
|
| 3531 | init_net.ipv4.tcp_congestion_control->owner))
|
| 3532 | net->ipv4.tcp_congestion_control = init_net.ipv4.tcp_congestion_control;
|
| 3533 | else
|
| 3534 | net->ipv4.tcp_congestion_control = &tcp_reno;
|
| 3535 |
|
| 3536 | net->ipv4.sysctl_tcp_syn_linear_timeouts = 4;
|
| 3537 | net->ipv4.sysctl_tcp_shrink_window = 0;
|
| 3538 |
|
| 3539 | net->ipv4.sysctl_tcp_pingpong_thresh = 1;
|
| 3540 | net->ipv4.sysctl_tcp_rto_min_us = jiffies_to_usecs(TCP_RTO_MIN);
|
| 3541 | net->ipv4.sysctl_tcp_rto_max_ms = TCP_RTO_MAX_SEC * MSEC_PER_SEC;
|
| 3542 |
|
| 3543 | return 0;
|
| 3544 | }
|
| 3545 |
|
| 3546 | static void __net_exit tcp_sk_exit_batch(struct list_head *net_exit_list)
|
| 3547 | {
|
| 3548 | struct net *net;
|
| 3549 |
|
| 3550 | |
| 3551 | * and failed setup_net error unwinding path are serialized.
|
| 3552 | *
|
| 3553 | * tcp_twsk_purge() handles twsk in any dead netns, not just those in
|
| 3554 | * net_exit_list, the thread that dismantles a particular twsk must
|
| 3555 | * do so without other thread progressing to refcount_dec_and_test() of
|
| 3556 | * tcp_death_row.tw_refcount.
|
| 3557 |
|
| 3558 | mutex_lock(&tcp_exit_batch_mutex);
|
| 3559 |
|
| 3560 | tcp_twsk_purge(net_exit_list);
|
| 3561 |
|
| 3562 | list_for_each_entry(net, net_exit_list, exit_list) {
|
| 3563 | inet_pernet_hashinfo_free(net->ipv4.tcp_death_row.hashinfo);
|
| 3564 | WARN_ON_ONCE(!refcount_dec_and_test(&net->ipv4.tcp_death_row.tw_refcount));
|
| 3565 | tcp_fastopen_ctx_destroy(net);
|
| 3566 | }
|
| 3567 |
|
| 3568 | mutex_unlock(&tcp_exit_batch_mutex);
|
| 3569 | }
|
| 3570 |
|
| 3571 | static struct pernet_operations __net_initdata tcp_sk_ops = {
|
| 3572 | .init = tcp_sk_init,
|
| 3573 | .exit = tcp_sk_exit,
|
| 3574 | .exit_batch = tcp_sk_exit_batch,
|
| 3575 | };
|
| 3576 |
|
| 3577 | #if defined(CONFIG_BPF_SYSCALL) && defined(CONFIG_PROC_FS)
|
| 3578 | DEFINE_BPF_ITER_FUNC(tcp, struct bpf_iter_meta *meta,
|
| 3579 | struct sock_common *sk_common, uid_t uid)
|
| 3580 |
|
| 3581 | #define INIT_BATCH_SZ 16
|
| 3582 |
|
| 3583 | static int bpf_iter_init_tcp(void *priv_data, struct bpf_iter_aux_info *aux)
|
| 3584 | {
|
| 3585 | struct bpf_tcp_iter_state *iter = priv_data;
|
| 3586 | int err;
|
| 3587 |
|
| 3588 | err = bpf_iter_init_seq_net(priv_data, aux);
|
| 3589 | if (err)
|
| 3590 | return err;
|
| 3591 |
|
| 3592 | err = bpf_iter_tcp_realloc_batch(iter, INIT_BATCH_SZ, GFP_USER);
|
| 3593 | if (err) {
|
| 3594 | bpf_iter_fini_seq_net(priv_data);
|
| 3595 | return err;
|
| 3596 | }
|
| 3597 |
|
| 3598 | return 0;
|
| 3599 | }
|
| 3600 |
|
| 3601 | static void bpf_iter_fini_tcp(void *priv_data)
|
| 3602 | {
|
| 3603 | struct bpf_tcp_iter_state *iter = priv_data;
|
| 3604 |
|
| 3605 | bpf_iter_fini_seq_net(priv_data);
|
| 3606 | kvfree(iter->batch);
|
| 3607 | }
|
| 3608 |
|
| 3609 | static const struct bpf_iter_seq_info tcp_seq_info = {
|
| 3610 | .seq_ops = &bpf_iter_tcp_seq_ops,
|
| 3611 | .init_seq_private = bpf_iter_init_tcp,
|
| 3612 | .fini_seq_private = bpf_iter_fini_tcp,
|
| 3613 | .seq_priv_size = sizeof(struct bpf_tcp_iter_state),
|
| 3614 | };
|
| 3615 |
|
| 3616 | static const struct bpf_func_proto *
|
| 3617 | bpf_iter_tcp_get_func_proto(enum bpf_func_id func_id,
|
| 3618 | const struct bpf_prog *prog)
|
| 3619 | {
|
| 3620 | switch (func_id) {
|
| 3621 | case BPF_FUNC_setsockopt:
|
| 3622 | return &bpf_sk_setsockopt_proto;
|
| 3623 | case BPF_FUNC_getsockopt:
|
| 3624 | return &bpf_sk_getsockopt_proto;
|
| 3625 | default:
|
| 3626 | return NULL;
|
| 3627 | }
|
| 3628 | }
|
| 3629 |
|
| 3630 | static struct bpf_iter_reg tcp_reg_info = {
|
| 3631 | .target = "tcp",
|
| 3632 | .ctx_arg_info_size = 1,
|
| 3633 | .ctx_arg_info = {
|
| 3634 | { offsetof(struct bpf_iter__tcp, sk_common),
|
| 3635 | PTR_TO_BTF_ID_OR_NULL | PTR_TRUSTED },
|
| 3636 | },
|
| 3637 | .get_func_proto = bpf_iter_tcp_get_func_proto,
|
| 3638 | .seq_info = &tcp_seq_info,
|
| 3639 | };
|
| 3640 |
|
| 3641 | static void __init bpf_iter_register(void)
|
| 3642 | {
|
| 3643 | tcp_reg_info.ctx_arg_info[0].btf_id = btf_sock_ids[BTF_SOCK_TYPE_SOCK_COMMON];
|
| 3644 | if (bpf_iter_reg_target(&tcp_reg_info))
|
| 3645 | pr_warn("Warning: could not register bpf iterator tcp\n");
|
| 3646 | }
|
| 3647 |
|
| 3648 | #endif
|
| 3649 |
|
| 3650 | void __init tcp_v4_init(void)
|
| 3651 | {
|
| 3652 | int cpu, res;
|
| 3653 |
|
| 3654 | for_each_possible_cpu(cpu) {
|
| 3655 | struct sock *sk;
|
| 3656 |
|
| 3657 | res = inet_ctl_sock_create(&sk, PF_INET, SOCK_RAW,
|
| 3658 | IPPROTO_TCP, &init_net);
|
| 3659 | if (res)
|
| 3660 | panic("Failed to create the TCP control socket.\n");
|
| 3661 | sock_set_flag(sk, SOCK_USE_WRITE_QUEUE);
|
| 3662 |
|
| 3663 | |
| 3664 | * ACK sent in SYN-RECV and TIME-WAIT state.
|
| 3665 |
|
| 3666 | inet_sk(sk)->pmtudisc = IP_PMTUDISC_DO;
|
| 3667 |
|
| 3668 | sk->sk_clockid = CLOCK_MONOTONIC;
|
| 3669 |
|
| 3670 | per_cpu(ipv4_tcp_sk.sock, cpu) = sk;
|
| 3671 | }
|
| 3672 | if (register_pernet_subsys(&tcp_sk_ops))
|
| 3673 | panic("Failed to create the TCP control socket.\n");
|
| 3674 |
|
| 3675 | #if defined(CONFIG_BPF_SYSCALL) && defined(CONFIG_PROC_FS)
|
| 3676 | bpf_iter_register();
|
| 3677 | #endif
|
| 3678 | }
|