| 1 | |
| 2 | |
| 3 | * INET An implementation of the TCP/IP protocol suite for the LINUX
|
| 4 | * operating system. INET is implemented using the BSD Socket
|
| 5 | * interface as the means of communication with the user level.
|
| 6 | *
|
| 7 | * Implementation of the Transmission Control Protocol(TCP).
|
| 8 | *
|
| 9 | * Authors: Ross Biro
|
| 10 | * Fred N. van Kempen, <[email protected]>
|
| 11 | * Mark Evans, <[email protected]>
|
| 12 | * Corey Minyard <[email protected]>
|
| 13 | * Florian La Roche, <[email protected]>
|
| 14 | * Charles Hedrick, <[email protected]>
|
| 15 | * Linus Torvalds, <[email protected]>
|
| 16 | * Alan Cox, <[email protected]>
|
| 17 | * Matthew Dillon, <[email protected]>
|
| 18 | * Arnt Gulbrandsen, <[email protected]>
|
| 19 | * Jorge Cwik, <[email protected]>
|
| 20 | *
|
| 21 | * Fixes:
|
| 22 | * Alan Cox : Numerous verify_area() calls
|
| 23 | * Alan Cox : Set the ACK bit on a reset
|
| 24 | * Alan Cox : Stopped it crashing if it closed while
|
| 25 | * sk->inuse=1 and was trying to connect
|
| 26 | * (tcp_err()).
|
| 27 | * Alan Cox : All icmp error handling was broken
|
| 28 | * pointers passed where wrong and the
|
| 29 | * socket was looked up backwards. Nobody
|
| 30 | * tested any icmp error code obviously.
|
| 31 | * Alan Cox : tcp_err() now handled properly. It
|
| 32 | * wakes people on errors. poll
|
| 33 | * behaves and the icmp error race
|
| 34 | * has gone by moving it into sock.c
|
| 35 | * Alan Cox : tcp_send_reset() fixed to work for
|
| 36 | * everything not just packets for
|
| 37 | * unknown sockets.
|
| 38 | * Alan Cox : tcp option processing.
|
| 39 | * Alan Cox : Reset tweaked (still not 100%) [Had
|
| 40 | * syn rule wrong]
|
| 41 | * Herp Rosmanith : More reset fixes
|
| 42 | * Alan Cox : No longer acks invalid rst frames.
|
| 43 | * Acking any kind of RST is right out.
|
| 44 | * Alan Cox : Sets an ignore me flag on an rst
|
| 45 | * receive otherwise odd bits of prattle
|
| 46 | * escape still
|
| 47 | * Alan Cox : Fixed another acking RST frame bug.
|
| 48 | * Should stop LAN workplace lockups.
|
| 49 | * Alan Cox : Some tidyups using the new skb list
|
| 50 | * facilities
|
| 51 | * Alan Cox : sk->keepopen now seems to work
|
| 52 | * Alan Cox : Pulls options out correctly on accepts
|
| 53 | * Alan Cox : Fixed assorted sk->rqueue->next errors
|
| 54 | * Alan Cox : PSH doesn't end a TCP read. Switched a
|
| 55 | * bit to skb ops.
|
| 56 | * Alan Cox : Tidied tcp_data to avoid a potential
|
| 57 | * nasty.
|
| 58 | * Alan Cox : Added some better commenting, as the
|
| 59 | * tcp is hard to follow
|
| 60 | * Alan Cox : Removed incorrect check for 20 * psh
|
| 61 | * Michael O'Reilly : ack < copied bug fix.
|
| 62 | * Johannes Stille : Misc tcp fixes (not all in yet).
|
| 63 | * Alan Cox : FIN with no memory -> CRASH
|
| 64 | * Alan Cox : Added socket option proto entries.
|
| 65 | * Also added awareness of them to accept.
|
| 66 | * Alan Cox : Added TCP options (SOL_TCP)
|
| 67 | * Alan Cox : Switched wakeup calls to callbacks,
|
| 68 | * so the kernel can layer network
|
| 69 | * sockets.
|
| 70 | * Alan Cox : Use ip_tos/ip_ttl settings.
|
| 71 | * Alan Cox : Handle FIN (more) properly (we hope).
|
| 72 | * Alan Cox : RST frames sent on unsynchronised
|
| 73 | * state ack error.
|
| 74 | * Alan Cox : Put in missing check for SYN bit.
|
| 75 | * Alan Cox : Added tcp_select_window() aka NET2E
|
| 76 | * window non shrink trick.
|
| 77 | * Alan Cox : Added a couple of small NET2E timer
|
| 78 | * fixes
|
| 79 | * Charles Hedrick : TCP fixes
|
| 80 | * Toomas Tamm : TCP window fixes
|
| 81 | * Alan Cox : Small URG fix to rlogin ^C ack fight
|
| 82 | * Charles Hedrick : Rewrote most of it to actually work
|
| 83 | * Linus : Rewrote tcp_read() and URG handling
|
| 84 | * completely
|
| 85 | * Gerhard Koerting: Fixed some missing timer handling
|
| 86 | * Matthew Dillon : Reworked TCP machine states as per RFC
|
| 87 | * Gerhard Koerting: PC/TCP workarounds
|
| 88 | * Adam Caldwell : Assorted timer/timing errors
|
| 89 | * Matthew Dillon : Fixed another RST bug
|
| 90 | * Alan Cox : Move to kernel side addressing changes.
|
| 91 | * Alan Cox : Beginning work on TCP fastpathing
|
| 92 | * (not yet usable)
|
| 93 | * Arnt Gulbrandsen: Turbocharged tcp_check() routine.
|
| 94 | * Alan Cox : TCP fast path debugging
|
| 95 | * Alan Cox : Window clamping
|
| 96 | * Michael Riepe : Bug in tcp_check()
|
| 97 | * Matt Dillon : More TCP improvements and RST bug fixes
|
| 98 | * Matt Dillon : Yet more small nasties remove from the
|
| 99 | * TCP code (Be very nice to this man if
|
| 100 | * tcp finally works 100%) 8)
|
| 101 | * Alan Cox : BSD accept semantics.
|
| 102 | * Alan Cox : Reset on closedown bug.
|
| 103 | * Peter De Schrijver : ENOTCONN check missing in tcp_sendto().
|
| 104 | * Michael Pall : Handle poll() after URG properly in
|
| 105 | * all cases.
|
| 106 | * Michael Pall : Undo the last fix in tcp_read_urg()
|
| 107 | * (multi URG PUSH broke rlogin).
|
| 108 | * Michael Pall : Fix the multi URG PUSH problem in
|
| 109 | * tcp_readable(), poll() after URG
|
| 110 | * works now.
|
| 111 | * Michael Pall : recv(...,MSG_OOB) never blocks in the
|
| 112 | * BSD api.
|
| 113 | * Alan Cox : Changed the semantics of sk->socket to
|
| 114 | * fix a race and a signal problem with
|
| 115 | * accept() and async I/O.
|
| 116 | * Alan Cox : Relaxed the rules on tcp_sendto().
|
| 117 | * Yury Shevchuk : Really fixed accept() blocking problem.
|
| 118 | * Craig I. Hagan : Allow for BSD compatible TIME_WAIT for
|
| 119 | * clients/servers which listen in on
|
| 120 | * fixed ports.
|
| 121 | * Alan Cox : Cleaned the above up and shrank it to
|
| 122 | * a sensible code size.
|
| 123 | * Alan Cox : Self connect lockup fix.
|
| 124 | * Alan Cox : No connect to multicast.
|
| 125 | * Ross Biro : Close unaccepted children on master
|
| 126 | * socket close.
|
| 127 | * Alan Cox : Reset tracing code.
|
| 128 | * Alan Cox : Spurious resets on shutdown.
|
| 129 | * Alan Cox : Giant 15 minute/60 second timer error
|
| 130 | * Alan Cox : Small whoops in polling before an
|
| 131 | * accept.
|
| 132 | * Alan Cox : Kept the state trace facility since
|
| 133 | * it's handy for debugging.
|
| 134 | * Alan Cox : More reset handler fixes.
|
| 135 | * Alan Cox : Started rewriting the code based on
|
| 136 | * the RFC's for other useful protocol
|
| 137 | * references see: Comer, KA9Q NOS, and
|
| 138 | * for a reference on the difference
|
| 139 | * between specifications and how BSD
|
| 140 | * works see the 4.4lite source.
|
| 141 | * A.N.Kuznetsov : Don't time wait on completion of tidy
|
| 142 | * close.
|
| 143 | * Linus Torvalds : Fin/Shutdown & copied_seq changes.
|
| 144 | * Linus Torvalds : Fixed BSD port reuse to work first syn
|
| 145 | * Alan Cox : Reimplemented timers as per the RFC
|
| 146 | * and using multiple timers for sanity.
|
| 147 | * Alan Cox : Small bug fixes, and a lot of new
|
| 148 | * comments.
|
| 149 | * Alan Cox : Fixed dual reader crash by locking
|
| 150 | * the buffers (much like datagram.c)
|
| 151 | * Alan Cox : Fixed stuck sockets in probe. A probe
|
| 152 | * now gets fed up of retrying without
|
| 153 | * (even a no space) answer.
|
| 154 | * Alan Cox : Extracted closing code better
|
| 155 | * Alan Cox : Fixed the closing state machine to
|
| 156 | * resemble the RFC.
|
| 157 | * Alan Cox : More 'per spec' fixes.
|
| 158 | * Jorge Cwik : Even faster checksumming.
|
| 159 | * Alan Cox : tcp_data() doesn't ack illegal PSH
|
| 160 | * only frames. At least one pc tcp stack
|
| 161 | * generates them.
|
| 162 | * Alan Cox : Cache last socket.
|
| 163 | * Alan Cox : Per route irtt.
|
| 164 | * Matt Day : poll()->select() match BSD precisely on error
|
| 165 | * Alan Cox : New buffers
|
| 166 | * Marc Tamsky : Various sk->prot->retransmits and
|
| 167 | * sk->retransmits misupdating fixed.
|
| 168 | * Fixed tcp_write_timeout: stuck close,
|
| 169 | * and TCP syn retries gets used now.
|
| 170 | * Mark Yarvis : In tcp_read_wakeup(), don't send an
|
| 171 | * ack if state is TCP_CLOSED.
|
| 172 | * Alan Cox : Look up device on a retransmit - routes may
|
| 173 | * change. Doesn't yet cope with MSS shrink right
|
| 174 | * but it's a start!
|
| 175 | * Marc Tamsky : Closing in closing fixes.
|
| 176 | * Mike Shaver : RFC1122 verifications.
|
| 177 | * Alan Cox : rcv_saddr errors.
|
| 178 | * Alan Cox : Block double connect().
|
| 179 | * Alan Cox : Small hooks for enSKIP.
|
| 180 | * Alexey Kuznetsov: Path MTU discovery.
|
| 181 | * Alan Cox : Support soft errors.
|
| 182 | * Alan Cox : Fix MTU discovery pathological case
|
| 183 | * when the remote claims no mtu!
|
| 184 | * Marc Tamsky : TCP_CLOSE fix.
|
| 185 | * Colin (G3TNE) : Send a reset on syn ack replies in
|
| 186 | * window but wrong (fixes NT lpd problems)
|
| 187 | * Pedro Roque : Better TCP window handling, delayed ack.
|
| 188 | * Joerg Reuter : No modification of locked buffers in
|
| 189 | * tcp_do_retransmit()
|
| 190 | * Eric Schenk : Changed receiver side silly window
|
| 191 | * avoidance algorithm to BSD style
|
| 192 | * algorithm. This doubles throughput
|
| 193 | * against machines running Solaris,
|
| 194 | * and seems to result in general
|
| 195 | * improvement.
|
| 196 | * Stefan Magdalinski : adjusted tcp_readable() to fix FIONREAD
|
| 197 | * Willy Konynenberg : Transparent proxying support.
|
| 198 | * Mike McLagan : Routing by source
|
| 199 | * Keith Owens : Do proper merging with partial SKB's in
|
| 200 | * tcp_do_sendmsg to avoid burstiness.
|
| 201 | * Eric Schenk : Fix fast close down bug with
|
| 202 | * shutdown() followed by close().
|
| 203 | * Andi Kleen : Make poll agree with SIGIO
|
| 204 | * Salvatore Sanfilippo : Support SO_LINGER with linger == 1 and
|
| 205 | * lingertime == 0 (RFC 793 ABORT Call)
|
| 206 | * Hirokazu Takahashi : Use copy_from_user() instead of
|
| 207 | * csum_and_copy_from_user() if possible.
|
| 208 | *
|
| 209 | * Description of States:
|
| 210 | *
|
| 211 | * TCP_SYN_SENT sent a connection request, waiting for ack
|
| 212 | *
|
| 213 | * TCP_SYN_RECV received a connection request, sent ack,
|
| 214 | * waiting for final ack in three-way handshake.
|
| 215 | *
|
| 216 | * TCP_ESTABLISHED connection established
|
| 217 | *
|
| 218 | * TCP_FIN_WAIT1 our side has shutdown, waiting to complete
|
| 219 | * transmission of remaining buffered data
|
| 220 | *
|
| 221 | * TCP_FIN_WAIT2 all buffered data sent, waiting for remote
|
| 222 | * to shutdown
|
| 223 | *
|
| 224 | * TCP_CLOSING both sides have shutdown but we still have
|
| 225 | * data we have to finish sending
|
| 226 | *
|
| 227 | * TCP_TIME_WAIT timeout to catch resent junk before entering
|
| 228 | * closed, can only be entered from FIN_WAIT2
|
| 229 | * or CLOSING. Required because the other end
|
| 230 | * may not have gotten our last ACK causing it
|
| 231 | * to retransmit the data packet (which we ignore)
|
| 232 | *
|
| 233 | * TCP_CLOSE_WAIT remote side has shutdown and is waiting for
|
| 234 | * us to finish writing our data and to shutdown
|
| 235 | * (we have to close() to move on to LAST_ACK)
|
| 236 | *
|
| 237 | * TCP_LAST_ACK out side has shutdown after remote has
|
| 238 | * shutdown. There may still be data in our
|
| 239 | * buffer that we have to finish sending
|
| 240 | *
|
| 241 | * TCP_CLOSE socket is finished
|
| 242 |
|
| 243 |
|
| 244 | #define pr_fmt(fmt) "TCP: " fmt
|
| 245 |
|
| 246 | #include <crypto/md5.h>
|
| 247 | #include <crypto/utils.h>
|
| 248 | #include <linux/kernel.h>
|
| 249 | #include <linux/module.h>
|
| 250 | #include <linux/types.h>
|
| 251 | #include <linux/fcntl.h>
|
| 252 | #include <linux/poll.h>
|
| 253 | #include <linux/inet_diag.h>
|
| 254 | #include <linux/init.h>
|
| 255 | #include <linux/fs.h>
|
| 256 | #include <linux/skbuff.h>
|
| 257 | #include <linux/splice.h>
|
| 258 | #include <linux/net.h>
|
| 259 | #include <linux/socket.h>
|
| 260 | #include <linux/random.h>
|
| 261 | #include <linux/memblock.h>
|
| 262 | #include <linux/highmem.h>
|
| 263 | #include <linux/cache.h>
|
| 264 | #include <linux/err.h>
|
| 265 | #include <linux/time.h>
|
| 266 | #include <linux/slab.h>
|
| 267 | #include <linux/errqueue.h>
|
| 268 | #include <linux/static_key.h>
|
| 269 | #include <linux/btf.h>
|
| 270 |
|
| 271 | #include <net/icmp.h>
|
| 272 | #include <net/inet_common.h>
|
| 273 | #include <net/inet_ecn.h>
|
| 274 | #include <net/tcp.h>
|
| 275 | #include <net/tcp_ecn.h>
|
| 276 | #include <net/mptcp.h>
|
| 277 | #include <net/proto_memory.h>
|
| 278 | #include <net/xfrm.h>
|
| 279 | #include <net/ip.h>
|
| 280 | #include <net/psp.h>
|
| 281 | #include <net/sock.h>
|
| 282 | #include <net/rstreason.h>
|
| 283 |
|
| 284 | #include <linux/uaccess.h>
|
| 285 | #include <asm/ioctls.h>
|
| 286 | #include <net/busy_poll.h>
|
| 287 | #include <net/hotdata.h>
|
| 288 | #include <trace/events/tcp.h>
|
| 289 | #include <net/rps.h>
|
| 290 |
|
| 291 | #include "../core/devmem.h"
|
| 292 |
|
| 293 |
|
| 294 | enum {
|
| 295 | TCP_CMSG_INQ = 1,
|
| 296 | TCP_CMSG_TS = 2
|
| 297 | };
|
| 298 |
|
| 299 | DEFINE_PER_CPU(unsigned int, tcp_orphan_count);
|
| 300 |
|
| 301 | long sysctl_tcp_mem[3] __read_mostly;
|
| 302 |
|
| 303 | DEFINE_PER_CPU(int, tcp_memory_per_cpu_fw_alloc);
|
| 304 | EXPORT_PER_CPU_SYMBOL_GPL(tcp_memory_per_cpu_fw_alloc);
|
| 305 |
|
| 306 | #if IS_ENABLED(CONFIG_SMC)
|
| 307 | DEFINE_STATIC_KEY_FALSE(tcp_have_smc);
|
| 308 | EXPORT_SYMBOL(tcp_have_smc);
|
| 309 | #endif
|
| 310 |
|
| 311 | |
| 312 | * Current number of TCP sockets.
|
| 313 |
|
| 314 | struct percpu_counter tcp_sockets_allocated ____cacheline_aligned_in_smp;
|
| 315 |
|
| 316 | |
| 317 | * Pressure flag: try to collapse.
|
| 318 | * Technical note: it is used by multiple contexts non atomically.
|
| 319 | * All the __sk_mem_schedule() is of this nature: accounting
|
| 320 | * is strict, actions are advisory and have some latency.
|
| 321 |
|
| 322 | unsigned long tcp_memory_pressure __read_mostly;
|
| 323 | EXPORT_SYMBOL_GPL(tcp_memory_pressure);
|
| 324 |
|
| 325 | void tcp_enter_memory_pressure(struct sock *sk)
|
| 326 | {
|
| 327 | unsigned long val;
|
| 328 |
|
| 329 | if (READ_ONCE(tcp_memory_pressure))
|
| 330 | return;
|
| 331 | val = jiffies;
|
| 332 |
|
| 333 | if (!val)
|
| 334 | val--;
|
| 335 | if (!cmpxchg(&tcp_memory_pressure, 0, val))
|
| 336 | NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPMEMORYPRESSURES);
|
| 337 | }
|
| 338 |
|
| 339 | void tcp_leave_memory_pressure(struct sock *sk)
|
| 340 | {
|
| 341 | unsigned long val;
|
| 342 |
|
| 343 | if (!READ_ONCE(tcp_memory_pressure))
|
| 344 | return;
|
| 345 | val = xchg(&tcp_memory_pressure, 0);
|
| 346 | if (val)
|
| 347 | NET_ADD_STATS(sock_net(sk), LINUX_MIB_TCPMEMORYPRESSURESCHRONO,
|
| 348 | jiffies_to_msecs(jiffies - val));
|
| 349 | }
|
| 350 |
|
| 351 |
|
| 352 | static u8 secs_to_retrans(int seconds, int timeout, int rto_max)
|
| 353 | {
|
| 354 | u8 res = 0;
|
| 355 |
|
| 356 | if (seconds > 0) {
|
| 357 | int period = timeout;
|
| 358 |
|
| 359 | res = 1;
|
| 360 | while (seconds > period && res < 255) {
|
| 361 | res++;
|
| 362 | timeout <<= 1;
|
| 363 | if (timeout > rto_max)
|
| 364 | timeout = rto_max;
|
| 365 | period += timeout;
|
| 366 | }
|
| 367 | }
|
| 368 | return res;
|
| 369 | }
|
| 370 |
|
| 371 |
|
| 372 | static int retrans_to_secs(u8 retrans, int timeout, int rto_max)
|
| 373 | {
|
| 374 | int period = 0;
|
| 375 |
|
| 376 | if (retrans > 0) {
|
| 377 | period = timeout;
|
| 378 | while (--retrans) {
|
| 379 | timeout <<= 1;
|
| 380 | if (timeout > rto_max)
|
| 381 | timeout = rto_max;
|
| 382 | period += timeout;
|
| 383 | }
|
| 384 | }
|
| 385 | return period;
|
| 386 | }
|
| 387 |
|
| 388 | static u64 tcp_compute_delivery_rate(const struct tcp_sock *tp)
|
| 389 | {
|
| 390 | u32 rate = READ_ONCE(tp->rate_delivered);
|
| 391 | u32 intv = READ_ONCE(tp->rate_interval_us);
|
| 392 | u64 rate64 = 0;
|
| 393 |
|
| 394 | if (rate && intv) {
|
| 395 | rate64 = (u64)rate * tp->mss_cache * USEC_PER_SEC;
|
| 396 | do_div(rate64, intv);
|
| 397 | }
|
| 398 | return rate64;
|
| 399 | }
|
| 400 |
|
| 401 | #ifdef CONFIG_TCP_MD5SIG
|
| 402 | void tcp_md5_destruct_sock(struct sock *sk)
|
| 403 | {
|
| 404 | struct tcp_sock *tp = tcp_sk(sk);
|
| 405 |
|
| 406 | if (tp->md5sig_info) {
|
| 407 |
|
| 408 | tcp_clear_md5_list(sk);
|
| 409 | kfree(rcu_replace_pointer(tp->md5sig_info, NULL, 1));
|
| 410 | static_branch_slow_dec_deferred(&tcp_md5_needed);
|
| 411 | }
|
| 412 | }
|
| 413 | #endif
|
| 414 |
|
| 415 | |
| 416 | *
|
| 417 | * NOTE: A lot of things set to zero explicitly by call to
|
| 418 | * sk_alloc() so need not be done here.
|
| 419 |
|
| 420 | void tcp_init_sock(struct sock *sk)
|
| 421 | {
|
| 422 | struct inet_connection_sock *icsk = inet_csk(sk);
|
| 423 | struct tcp_sock *tp = tcp_sk(sk);
|
| 424 | int rto_min_us, rto_max_ms;
|
| 425 |
|
| 426 | tp->out_of_order_queue = RB_ROOT;
|
| 427 | sk->tcp_rtx_queue = RB_ROOT;
|
| 428 | tcp_init_xmit_timers(sk);
|
| 429 | INIT_LIST_HEAD(&tp->tsq_node);
|
| 430 | INIT_LIST_HEAD(&tp->tsorted_sent_queue);
|
| 431 |
|
| 432 | icsk->icsk_rto = TCP_TIMEOUT_INIT;
|
| 433 |
|
| 434 | rto_max_ms = READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_rto_max_ms);
|
| 435 | icsk->icsk_rto_max = msecs_to_jiffies(rto_max_ms);
|
| 436 |
|
| 437 | rto_min_us = READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_rto_min_us);
|
| 438 | icsk->icsk_rto_min = usecs_to_jiffies(rto_min_us);
|
| 439 | icsk->icsk_delack_max = TCP_DELACK_MAX;
|
| 440 | tp->mdev_us = jiffies_to_usecs(TCP_TIMEOUT_INIT);
|
| 441 | minmax_reset(&tp->rtt_min, tcp_jiffies32, ~0U);
|
| 442 |
|
| 443 | |
| 444 | * initial SYN frame in their delayed-ACK and congestion control
|
| 445 | * algorithms that we must have the following bandaid to talk
|
| 446 | * efficiently to them. -DaveM
|
| 447 |
|
| 448 | tcp_snd_cwnd_set(tp, TCP_INIT_CWND);
|
| 449 |
|
| 450 |
|
| 451 | tp->app_limited = ~0U;
|
| 452 | tp->rate_app_limited = 1;
|
| 453 |
|
| 454 | |
| 455 | * initialization of these values.
|
| 456 |
|
| 457 | tp->snd_ssthresh = TCP_INFINITE_SSTHRESH;
|
| 458 | tp->snd_cwnd_clamp = ~0;
|
| 459 | tp->mss_cache = TCP_MSS_DEFAULT;
|
| 460 |
|
| 461 | tp->reordering = READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_reordering);
|
| 462 | tcp_assign_congestion_control(sk);
|
| 463 |
|
| 464 | tp->tsoffset = 0;
|
| 465 | tp->rack.reo_wnd_steps = 1;
|
| 466 |
|
| 467 | sk->sk_write_space = sk_stream_write_space;
|
| 468 | sock_set_flag(sk, SOCK_USE_WRITE_QUEUE);
|
| 469 |
|
| 470 | icsk->icsk_sync_mss = tcp_sync_mss;
|
| 471 |
|
| 472 | WRITE_ONCE(sk->sk_sndbuf, READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_wmem[1]));
|
| 473 | WRITE_ONCE(sk->sk_rcvbuf, READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_rmem[1]));
|
| 474 | tcp_scaling_ratio_init(sk);
|
| 475 |
|
| 476 | set_bit(SOCK_SUPPORT_ZC, &sk->sk_socket->flags);
|
| 477 | sk_sockets_allocated_inc(sk);
|
| 478 | xa_init_flags(&sk->sk_user_frags, XA_FLAGS_ALLOC1);
|
| 479 | }
|
| 480 |
|
| 481 | static void tcp_tx_timestamp(struct sock *sk, struct sockcm_cookie *sockc)
|
| 482 | {
|
| 483 | struct sk_buff *skb = tcp_write_queue_tail(sk);
|
| 484 | u32 tsflags = sockc->tsflags;
|
| 485 |
|
| 486 | if (unlikely(!skb))
|
| 487 | skb = skb_rb_last(&sk->tcp_rtx_queue);
|
| 488 |
|
| 489 | if (tsflags && skb) {
|
| 490 | struct skb_shared_info *shinfo = skb_shinfo(skb);
|
| 491 | struct tcp_skb_cb *tcb = TCP_SKB_CB(skb);
|
| 492 |
|
| 493 | sock_tx_timestamp(sk, sockc, &shinfo->tx_flags);
|
| 494 | if (tsflags & SOF_TIMESTAMPING_TX_ACK)
|
| 495 | tcb->txstamp_ack |= TSTAMP_ACK_SK;
|
| 496 | if (tsflags & SOF_TIMESTAMPING_TX_RECORD_MASK)
|
| 497 | shinfo->tskey = TCP_SKB_CB(skb)->seq + skb->len - 1;
|
| 498 | }
|
| 499 |
|
| 500 | if (cgroup_bpf_enabled(CGROUP_SOCK_OPS) &&
|
| 501 | SK_BPF_CB_FLAG_TEST(sk, SK_BPF_CB_TX_TIMESTAMPING) && skb)
|
| 502 | bpf_skops_tx_timestamping(sk, skb, BPF_SOCK_OPS_TSTAMP_SENDMSG_CB);
|
| 503 | }
|
| 504 |
|
| 505 | |
| 506 | * This sends EPOLLOUT only if notsent_bytes is half the limit.
|
| 507 | * This mimics the strategy used in sock_def_write_space().
|
| 508 |
|
| 509 | bool tcp_stream_memory_free(const struct sock *sk, int wake)
|
| 510 | {
|
| 511 | const struct tcp_sock *tp = tcp_sk(sk);
|
| 512 | u32 notsent_bytes = READ_ONCE(tp->write_seq) - READ_ONCE(tp->snd_nxt);
|
| 513 |
|
| 514 | return (notsent_bytes << wake) < tcp_notsent_lowat(tp);
|
| 515 | }
|
| 516 | EXPORT_SYMBOL(tcp_stream_memory_free);
|
| 517 |
|
| 518 | static bool tcp_stream_is_readable(struct sock *sk, int target)
|
| 519 | {
|
| 520 | if (tcp_epollin_ready(sk, target))
|
| 521 | return true;
|
| 522 | return sk_is_readable(sk);
|
| 523 | }
|
| 524 |
|
| 525 | |
| 526 | * Wait for a TCP event.
|
| 527 | *
|
| 528 | * Note that we don't need to lock the socket, as the upper poll layers
|
| 529 | * take care of normal races (between the test and the event) and we don't
|
| 530 | * go look at any of the socket buffers directly.
|
| 531 |
|
| 532 | __poll_t tcp_poll(struct file *file, struct socket *sock, poll_table *wait)
|
| 533 | {
|
| 534 | __poll_t mask;
|
| 535 | struct sock *sk = sock->sk;
|
| 536 | const struct tcp_sock *tp = tcp_sk(sk);
|
| 537 | u8 shutdown;
|
| 538 | int state;
|
| 539 |
|
| 540 | sock_poll_wait(file, sock, wait);
|
| 541 |
|
| 542 | state = inet_sk_state_load(sk);
|
| 543 | if (state == TCP_LISTEN)
|
| 544 | return inet_csk_listen_poll(sk);
|
| 545 |
|
| 546 | |
| 547 | * by poll logic and correct handling of state changes
|
| 548 | * made by other threads is impossible in any case.
|
| 549 |
|
| 550 |
|
| 551 | mask = 0;
|
| 552 |
|
| 553 | |
| 554 | * EPOLLHUP is certainly not done right. But poll() doesn't
|
| 555 | * have a notion of HUP in just one direction, and for a
|
| 556 | * socket the read side is more interesting.
|
| 557 | *
|
| 558 | * Some poll() documentation says that EPOLLHUP is incompatible
|
| 559 | * with the EPOLLOUT/POLLWR flags, so somebody should check this
|
| 560 | * all. But careful, it tends to be safer to return too many
|
| 561 | * bits than too few, and you can easily break real applications
|
| 562 | * if you don't tell them that something has hung up!
|
| 563 | *
|
| 564 | * Check-me.
|
| 565 | *
|
| 566 | * Check number 1. EPOLLHUP is _UNMASKABLE_ event (see UNIX98 and
|
| 567 | * our fs/select.c). It means that after we received EOF,
|
| 568 | * poll always returns immediately, making impossible poll() on write()
|
| 569 | * in state CLOSE_WAIT. One solution is evident --- to set EPOLLHUP
|
| 570 | * if and only if shutdown has been made in both directions.
|
| 571 | * Actually, it is interesting to look how Solaris and DUX
|
| 572 | * solve this dilemma. I would prefer, if EPOLLHUP were maskable,
|
| 573 | * then we could set it on SND_SHUTDOWN. BTW examples given
|
| 574 | * in Stevens' books assume exactly this behaviour, it explains
|
| 575 | * why EPOLLHUP is incompatible with EPOLLOUT. --ANK
|
| 576 | *
|
| 577 | * NOTE. Check for TCP_CLOSE is added. The goal is to prevent
|
| 578 | * blocking on fresh not-connected or disconnected socket. --ANK
|
| 579 |
|
| 580 | shutdown = READ_ONCE(sk->sk_shutdown);
|
| 581 | if (shutdown == SHUTDOWN_MASK || state == TCP_CLOSE)
|
| 582 | mask |= EPOLLHUP;
|
| 583 | if (shutdown & RCV_SHUTDOWN)
|
| 584 | mask |= EPOLLIN | EPOLLRDNORM | EPOLLRDHUP;
|
| 585 |
|
| 586 |
|
| 587 | if (state != TCP_SYN_SENT &&
|
| 588 | (state != TCP_SYN_RECV || rcu_access_pointer(tp->fastopen_rsk))) {
|
| 589 | int target = sock_rcvlowat(sk, 0, INT_MAX);
|
| 590 | u16 urg_data = READ_ONCE(tp->urg_data);
|
| 591 |
|
| 592 | if (unlikely(urg_data) &&
|
| 593 | READ_ONCE(tp->urg_seq) == READ_ONCE(tp->copied_seq) &&
|
| 594 | !sock_flag(sk, SOCK_URGINLINE))
|
| 595 | target++;
|
| 596 |
|
| 597 | if (tcp_stream_is_readable(sk, target))
|
| 598 | mask |= EPOLLIN | EPOLLRDNORM;
|
| 599 |
|
| 600 | if (!(shutdown & SEND_SHUTDOWN)) {
|
| 601 | if (__sk_stream_is_writeable(sk, 1)) {
|
| 602 | mask |= EPOLLOUT | EPOLLWRNORM;
|
| 603 | } else {
|
| 604 | sk_set_bit(SOCKWQ_ASYNC_NOSPACE, sk);
|
| 605 | set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
|
| 606 |
|
| 607 | |
| 608 | * wspace test but before the flags are set,
|
| 609 | * IO signal will be lost. Memory barrier
|
| 610 | * pairs with the input side.
|
| 611 |
|
| 612 | smp_mb__after_atomic();
|
| 613 | if (__sk_stream_is_writeable(sk, 1))
|
| 614 | mask |= EPOLLOUT | EPOLLWRNORM;
|
| 615 | }
|
| 616 | } else
|
| 617 | mask |= EPOLLOUT | EPOLLWRNORM;
|
| 618 |
|
| 619 | if (urg_data & TCP_URG_VALID)
|
| 620 | mask |= EPOLLPRI;
|
| 621 | } else if (state == TCP_SYN_SENT &&
|
| 622 | inet_test_bit(DEFER_CONNECT, sk)) {
|
| 623 | |
| 624 | * Return EPOLLOUT so application can call write()
|
| 625 | * in order for kernel to generate SYN+data
|
| 626 |
|
| 627 | mask |= EPOLLOUT | EPOLLWRNORM;
|
| 628 | }
|
| 629 |
|
| 630 | smp_rmb();
|
| 631 | if (READ_ONCE(sk->sk_err) ||
|
| 632 | !skb_queue_empty_lockless(&sk->sk_error_queue))
|
| 633 | mask |= EPOLLERR;
|
| 634 |
|
| 635 | return mask;
|
| 636 | }
|
| 637 | EXPORT_SYMBOL(tcp_poll);
|
| 638 |
|
| 639 | int tcp_ioctl(struct sock *sk, int cmd, int *karg)
|
| 640 | {
|
| 641 | struct tcp_sock *tp = tcp_sk(sk);
|
| 642 | int answ;
|
| 643 | bool slow;
|
| 644 |
|
| 645 | switch (cmd) {
|
| 646 | case SIOCINQ:
|
| 647 | if (sk->sk_state == TCP_LISTEN)
|
| 648 | return -EINVAL;
|
| 649 |
|
| 650 | slow = lock_sock_fast(sk);
|
| 651 | answ = tcp_inq(sk);
|
| 652 | unlock_sock_fast(sk, slow);
|
| 653 | break;
|
| 654 | case SIOCATMARK:
|
| 655 | answ = READ_ONCE(tp->urg_data) &&
|
| 656 | READ_ONCE(tp->urg_seq) == READ_ONCE(tp->copied_seq);
|
| 657 | break;
|
| 658 | case SIOCOUTQ:
|
| 659 | if (sk->sk_state == TCP_LISTEN)
|
| 660 | return -EINVAL;
|
| 661 |
|
| 662 | if ((1 << sk->sk_state) & (TCPF_SYN_SENT | TCPF_SYN_RECV))
|
| 663 | answ = 0;
|
| 664 | else
|
| 665 | answ = READ_ONCE(tp->write_seq) - tp->snd_una;
|
| 666 | break;
|
| 667 | case SIOCOUTQNSD:
|
| 668 | if (sk->sk_state == TCP_LISTEN)
|
| 669 | return -EINVAL;
|
| 670 |
|
| 671 | if ((1 << sk->sk_state) & (TCPF_SYN_SENT | TCPF_SYN_RECV))
|
| 672 | answ = 0;
|
| 673 | else
|
| 674 | answ = READ_ONCE(tp->write_seq) -
|
| 675 | READ_ONCE(tp->snd_nxt);
|
| 676 | break;
|
| 677 | default:
|
| 678 | return -ENOIOCTLCMD;
|
| 679 | }
|
| 680 |
|
| 681 | *karg = answ;
|
| 682 | return 0;
|
| 683 | }
|
| 684 |
|
| 685 | void tcp_mark_push(struct tcp_sock *tp, struct sk_buff *skb)
|
| 686 | {
|
| 687 | TCP_SKB_CB(skb)->tcp_flags |= TCPHDR_PSH;
|
| 688 | tp->pushed_seq = tp->write_seq;
|
| 689 | }
|
| 690 |
|
| 691 | static inline bool forced_push(const struct tcp_sock *tp)
|
| 692 | {
|
| 693 | return after(tp->write_seq, tp->pushed_seq + (tp->max_window >> 1));
|
| 694 | }
|
| 695 |
|
| 696 | void tcp_skb_entail(struct sock *sk, struct sk_buff *skb)
|
| 697 | {
|
| 698 | struct tcp_sock *tp = tcp_sk(sk);
|
| 699 | struct tcp_skb_cb *tcb = TCP_SKB_CB(skb);
|
| 700 |
|
| 701 | tcb->seq = tcb->end_seq = tp->write_seq;
|
| 702 | tcb->tcp_flags = TCPHDR_ACK;
|
| 703 | __skb_header_release(skb);
|
| 704 | psp_enqueue_set_decrypted(sk, skb);
|
| 705 | tcp_add_write_queue_tail(sk, skb);
|
| 706 | sk_wmem_queued_add(sk, skb->truesize);
|
| 707 | sk_mem_charge(sk, skb->truesize);
|
| 708 | if (tp->nonagle & TCP_NAGLE_PUSH)
|
| 709 | tp->nonagle &= ~TCP_NAGLE_PUSH;
|
| 710 |
|
| 711 | tcp_slow_start_after_idle_check(sk);
|
| 712 | }
|
| 713 |
|
| 714 | static inline void tcp_mark_urg(struct tcp_sock *tp, int flags)
|
| 715 | {
|
| 716 | if (flags & MSG_OOB)
|
| 717 | tp->snd_up = tp->write_seq;
|
| 718 | }
|
| 719 |
|
| 720 | |
| 721 | * we have data packets in Qdisc or NIC queues :
|
| 722 | * Because TX completion will happen shortly, it gives a chance
|
| 723 | * to coalesce future sendmsg() payload into this skb, without
|
| 724 | * need for a timer, and with no latency trade off.
|
| 725 | * As packets containing data payload have a bigger truesize
|
| 726 | * than pure acks (dataless) packets, the last checks prevent
|
| 727 | * autocorking if we only have an ACK in Qdisc/NIC queues,
|
| 728 | * or if TX completion was delayed after we processed ACK packet.
|
| 729 |
|
| 730 | static bool tcp_should_autocork(struct sock *sk, struct sk_buff *skb,
|
| 731 | int size_goal)
|
| 732 | {
|
| 733 | return skb->len < size_goal &&
|
| 734 | READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_autocorking) &&
|
| 735 | !tcp_rtx_queue_empty(sk) &&
|
| 736 | refcount_read(&sk->sk_wmem_alloc) > skb->truesize &&
|
| 737 | tcp_skb_can_collapse_to(skb);
|
| 738 | }
|
| 739 |
|
| 740 | void tcp_push(struct sock *sk, int flags, int mss_now,
|
| 741 | int nonagle, int size_goal)
|
| 742 | {
|
| 743 | struct tcp_sock *tp = tcp_sk(sk);
|
| 744 | struct sk_buff *skb;
|
| 745 |
|
| 746 | skb = tcp_write_queue_tail(sk);
|
| 747 | if (!skb)
|
| 748 | return;
|
| 749 | if (!(flags & MSG_MORE) || forced_push(tp))
|
| 750 | tcp_mark_push(tp, skb);
|
| 751 |
|
| 752 | tcp_mark_urg(tp, flags);
|
| 753 |
|
| 754 | if (tcp_should_autocork(sk, skb, size_goal)) {
|
| 755 |
|
| 756 |
|
| 757 | if (!test_bit(TSQ_THROTTLED, &sk->sk_tsq_flags)) {
|
| 758 | NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPAUTOCORKING);
|
| 759 | set_bit(TSQ_THROTTLED, &sk->sk_tsq_flags);
|
| 760 | smp_mb__after_atomic();
|
| 761 | }
|
| 762 | |
| 763 | * before we set TSQ_THROTTLED.
|
| 764 |
|
| 765 | if (refcount_read(&sk->sk_wmem_alloc) > skb->truesize)
|
| 766 | return;
|
| 767 | }
|
| 768 |
|
| 769 | if (flags & MSG_MORE)
|
| 770 | nonagle = TCP_NAGLE_CORK;
|
| 771 |
|
| 772 | __tcp_push_pending_frames(sk, mss_now, nonagle);
|
| 773 | }
|
| 774 |
|
| 775 | int tcp_splice_data_recv(read_descriptor_t *rd_desc, struct sk_buff *skb,
|
| 776 | unsigned int offset, size_t len)
|
| 777 | {
|
| 778 | struct tcp_splice_state *tss = rd_desc->arg.data;
|
| 779 | int ret;
|
| 780 |
|
| 781 | ret = skb_splice_bits(skb, skb->sk, offset, tss->pipe,
|
| 782 | min(rd_desc->count, len), tss->flags);
|
| 783 | if (ret > 0)
|
| 784 | rd_desc->count -= ret;
|
| 785 | return ret;
|
| 786 | }
|
| 787 |
|
| 788 | static int __tcp_splice_read(struct sock *sk, struct tcp_splice_state *tss)
|
| 789 | {
|
| 790 |
|
| 791 | read_descriptor_t rd_desc = {
|
| 792 | .arg.data = tss,
|
| 793 | .count = tss->len,
|
| 794 | };
|
| 795 |
|
| 796 | return tcp_read_sock(sk, &rd_desc, tcp_splice_data_recv);
|
| 797 | }
|
| 798 |
|
| 799 | |
| 800 | * tcp_splice_read - splice data from TCP socket to a pipe
|
| 801 | * @sock: socket to splice from
|
| 802 | * @ppos: position (not valid)
|
| 803 | * @pipe: pipe to splice to
|
| 804 | * @len: number of bytes to splice
|
| 805 | * @flags: splice modifier flags
|
| 806 | *
|
| 807 | * Description:
|
| 808 | * Will read pages from given socket and fill them into a pipe.
|
| 809 | *
|
| 810 | *
|
| 811 | ssize_t tcp_splice_read(struct socket *sock, loff_t *ppos,
|
| 812 | struct pipe_inode_info *pipe, size_t len,
|
| 813 | unsigned int flags)
|
| 814 | {
|
| 815 | struct sock *sk = sock->sk;
|
| 816 | struct tcp_splice_state tss = {
|
| 817 | .pipe = pipe,
|
| 818 | .len = len,
|
| 819 | .flags = flags,
|
| 820 | };
|
| 821 | long timeo;
|
| 822 | ssize_t spliced;
|
| 823 | int ret;
|
| 824 |
|
| 825 | sock_rps_record_flow(sk);
|
| 826 | |
| 827 | * We can't seek on a socket input
|
| 828 |
|
| 829 | if (unlikely(*ppos))
|
| 830 | return -ESPIPE;
|
| 831 |
|
| 832 | ret = spliced = 0;
|
| 833 |
|
| 834 | lock_sock(sk);
|
| 835 |
|
| 836 | timeo = sock_rcvtimeo(sk, sock->file->f_flags & O_NONBLOCK);
|
| 837 | while (tss.len) {
|
| 838 | ret = __tcp_splice_read(sk, &tss);
|
| 839 | if (ret < 0)
|
| 840 | break;
|
| 841 | else if (!ret) {
|
| 842 | if (spliced)
|
| 843 | break;
|
| 844 | if (sock_flag(sk, SOCK_DONE))
|
| 845 | break;
|
| 846 | if (sk->sk_err) {
|
| 847 | ret = sock_error(sk);
|
| 848 | break;
|
| 849 | }
|
| 850 | if (sk->sk_shutdown & RCV_SHUTDOWN)
|
| 851 | break;
|
| 852 | if (sk->sk_state == TCP_CLOSE) {
|
| 853 | |
| 854 | * This occurs when user tries to read
|
| 855 | * from never connected socket.
|
| 856 |
|
| 857 | ret = -ENOTCONN;
|
| 858 | break;
|
| 859 | }
|
| 860 | if (!timeo) {
|
| 861 | ret = -EAGAIN;
|
| 862 | break;
|
| 863 | }
|
| 864 | |
| 865 | * an skb in receive queue, we do not want to loop.
|
| 866 | * This might happen with URG data.
|
| 867 |
|
| 868 | if (!skb_queue_empty(&sk->sk_receive_queue))
|
| 869 | break;
|
| 870 | ret = sk_wait_data(sk, &timeo, NULL);
|
| 871 | if (ret < 0)
|
| 872 | break;
|
| 873 | if (signal_pending(current)) {
|
| 874 | ret = sock_intr_errno(timeo);
|
| 875 | break;
|
| 876 | }
|
| 877 | continue;
|
| 878 | }
|
| 879 | tss.len -= ret;
|
| 880 | spliced += ret;
|
| 881 |
|
| 882 | if (!tss.len || !timeo)
|
| 883 | break;
|
| 884 | release_sock(sk);
|
| 885 | lock_sock(sk);
|
| 886 |
|
| 887 | if (tcp_recv_should_stop(sk))
|
| 888 | break;
|
| 889 | }
|
| 890 |
|
| 891 | release_sock(sk);
|
| 892 |
|
| 893 | if (spliced)
|
| 894 | return spliced;
|
| 895 |
|
| 896 | return ret;
|
| 897 | }
|
| 898 |
|
| 899 | |
| 900 | * connection tear down and (memory) recovery.
|
| 901 | * Otherwise tcp_send_fin() could be tempted to either delay FIN
|
| 902 | * or even be forced to close flow without any FIN.
|
| 903 | * In general, we want to allow one skb per socket to avoid hangs
|
| 904 | * with edge trigger epoll()
|
| 905 |
|
| 906 | void sk_forced_mem_schedule(struct sock *sk, int size)
|
| 907 | {
|
| 908 | int delta, amt;
|
| 909 |
|
| 910 | delta = size - sk->sk_forward_alloc;
|
| 911 | if (delta <= 0)
|
| 912 | return;
|
| 913 |
|
| 914 | amt = sk_mem_pages(delta);
|
| 915 | sk_forward_alloc_add(sk, amt << PAGE_SHIFT);
|
| 916 |
|
| 917 | if (mem_cgroup_sk_enabled(sk))
|
| 918 | mem_cgroup_sk_charge(sk, amt, gfp_memcg_charge() | __GFP_NOFAIL);
|
| 919 |
|
| 920 | if (sk->sk_bypass_prot_mem)
|
| 921 | return;
|
| 922 |
|
| 923 | sk_memory_allocated_add(sk, amt);
|
| 924 | }
|
| 925 |
|
| 926 | struct sk_buff *tcp_stream_alloc_skb(struct sock *sk, gfp_t gfp,
|
| 927 | bool force_schedule)
|
| 928 | {
|
| 929 | struct sk_buff *skb;
|
| 930 |
|
| 931 | skb = alloc_skb_fclone(MAX_TCP_HEADER, gfp);
|
| 932 | if (likely(skb)) {
|
| 933 | bool mem_scheduled;
|
| 934 |
|
| 935 | skb->truesize = SKB_TRUESIZE(skb_end_offset(skb));
|
| 936 | if (force_schedule) {
|
| 937 | mem_scheduled = true;
|
| 938 | sk_forced_mem_schedule(sk, skb->truesize);
|
| 939 | } else {
|
| 940 | mem_scheduled = sk_wmem_schedule(sk, skb->truesize);
|
| 941 | }
|
| 942 | if (likely(mem_scheduled)) {
|
| 943 | skb_reserve(skb, MAX_TCP_HEADER);
|
| 944 | skb->ip_summed = CHECKSUM_PARTIAL;
|
| 945 | INIT_LIST_HEAD(&skb->tcp_tsorted_anchor);
|
| 946 | return skb;
|
| 947 | }
|
| 948 | __kfree_skb(skb);
|
| 949 | } else {
|
| 950 | if (!sk->sk_bypass_prot_mem)
|
| 951 | tcp_enter_memory_pressure(sk);
|
| 952 | sk_stream_moderate_sndbuf(sk);
|
| 953 | }
|
| 954 | return NULL;
|
| 955 | }
|
| 956 |
|
| 957 | static unsigned int tcp_xmit_size_goal(struct sock *sk, u32 mss_now,
|
| 958 | int large_allowed)
|
| 959 | {
|
| 960 | struct tcp_sock *tp = tcp_sk(sk);
|
| 961 | u32 new_size_goal, size_goal;
|
| 962 |
|
| 963 | if (!large_allowed)
|
| 964 | return mss_now;
|
| 965 |
|
| 966 |
|
| 967 | new_size_goal = tcp_bound_to_half_wnd(tp, sk->sk_gso_max_size);
|
| 968 |
|
| 969 |
|
| 970 | size_goal = tp->gso_segs * mss_now;
|
| 971 | if (unlikely(new_size_goal < size_goal ||
|
| 972 | new_size_goal >= size_goal + mss_now)) {
|
| 973 | tp->gso_segs = min_t(u16, new_size_goal / mss_now,
|
| 974 | sk->sk_gso_max_segs);
|
| 975 | size_goal = tp->gso_segs * mss_now;
|
| 976 | }
|
| 977 |
|
| 978 | return max(size_goal, mss_now);
|
| 979 | }
|
| 980 |
|
| 981 | int tcp_send_mss(struct sock *sk, int *size_goal, int flags)
|
| 982 | {
|
| 983 | int mss_now;
|
| 984 |
|
| 985 | mss_now = tcp_current_mss(sk);
|
| 986 | *size_goal = tcp_xmit_size_goal(sk, mss_now, !(flags & MSG_OOB));
|
| 987 |
|
| 988 | return mss_now;
|
| 989 | }
|
| 990 |
|
| 991 | |
| 992 | * but failed adding payload on it. We need to remove it to consume less
|
| 993 | * memory, but more importantly be able to generate EPOLLOUT for Edge Trigger
|
| 994 | * epoll() users. Another reason is that tcp_write_xmit() does not like
|
| 995 | * finding an empty skb in the write queue.
|
| 996 |
|
| 997 | void tcp_remove_empty_skb(struct sock *sk)
|
| 998 | {
|
| 999 | struct sk_buff *skb = tcp_write_queue_tail(sk);
|
| 1000 |
|
| 1001 | if (skb && TCP_SKB_CB(skb)->seq == TCP_SKB_CB(skb)->end_seq) {
|
| 1002 | tcp_unlink_write_queue(skb, sk);
|
| 1003 | if (tcp_write_queue_empty(sk))
|
| 1004 | tcp_chrono_stop(sk, TCP_CHRONO_BUSY);
|
| 1005 | tcp_wmem_free_skb(sk, skb);
|
| 1006 | }
|
| 1007 | }
|
| 1008 |
|
| 1009 |
|
| 1010 | static int tcp_downgrade_zcopy_pure(struct sock *sk, struct sk_buff *skb)
|
| 1011 | {
|
| 1012 | if (unlikely(skb_zcopy_pure(skb))) {
|
| 1013 | u32 extra = skb->truesize -
|
| 1014 | SKB_TRUESIZE(skb_end_offset(skb));
|
| 1015 |
|
| 1016 | if (!sk_wmem_schedule(sk, extra))
|
| 1017 | return -ENOMEM;
|
| 1018 |
|
| 1019 | sk_mem_charge(sk, extra);
|
| 1020 | skb_shinfo(skb)->flags &= ~SKBFL_PURE_ZEROCOPY;
|
| 1021 | }
|
| 1022 | return 0;
|
| 1023 | }
|
| 1024 |
|
| 1025 |
|
| 1026 | int tcp_wmem_schedule(struct sock *sk, int copy)
|
| 1027 | {
|
| 1028 | int left;
|
| 1029 |
|
| 1030 | if (likely(sk_wmem_schedule(sk, copy)))
|
| 1031 | return copy;
|
| 1032 |
|
| 1033 | |
| 1034 | * Use whatever is left in sk->sk_forward_alloc and tcp_wmem[0]
|
| 1035 | * to guarantee some progress.
|
| 1036 |
|
| 1037 | left = READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_wmem[0]) - sk->sk_wmem_queued;
|
| 1038 | if (left > 0)
|
| 1039 | sk_forced_mem_schedule(sk, min(left, copy));
|
| 1040 | return min(copy, sk->sk_forward_alloc);
|
| 1041 | }
|
| 1042 |
|
| 1043 | void tcp_free_fastopen_req(struct tcp_sock *tp)
|
| 1044 | {
|
| 1045 | if (tp->fastopen_req) {
|
| 1046 | kfree(tp->fastopen_req);
|
| 1047 | tp->fastopen_req = NULL;
|
| 1048 | }
|
| 1049 | }
|
| 1050 |
|
| 1051 | int tcp_sendmsg_fastopen(struct sock *sk, struct msghdr *msg, int *copied,
|
| 1052 | size_t size, struct ubuf_info *uarg)
|
| 1053 | {
|
| 1054 | struct tcp_sock *tp = tcp_sk(sk);
|
| 1055 | struct inet_sock *inet = inet_sk(sk);
|
| 1056 | struct sockaddr *uaddr = msg->msg_name;
|
| 1057 | int err, flags;
|
| 1058 |
|
| 1059 | if (!(READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_fastopen) &
|
| 1060 | TFO_CLIENT_ENABLE) ||
|
| 1061 | (uaddr && msg->msg_namelen >= sizeof(uaddr->sa_family) &&
|
| 1062 | uaddr->sa_family == AF_UNSPEC))
|
| 1063 | return -EOPNOTSUPP;
|
| 1064 | if (tp->fastopen_req)
|
| 1065 | return -EALREADY;
|
| 1066 |
|
| 1067 | tp->fastopen_req = kzalloc_obj(struct tcp_fastopen_request,
|
| 1068 | sk->sk_allocation);
|
| 1069 | if (unlikely(!tp->fastopen_req))
|
| 1070 | return -ENOBUFS;
|
| 1071 | tp->fastopen_req->data = msg;
|
| 1072 | tp->fastopen_req->size = size;
|
| 1073 | tp->fastopen_req->uarg = uarg;
|
| 1074 |
|
| 1075 | if (inet_test_bit(DEFER_CONNECT, sk)) {
|
| 1076 | err = tcp_connect(sk);
|
| 1077 |
|
| 1078 | if (err) {
|
| 1079 | tcp_set_state(sk, TCP_CLOSE);
|
| 1080 | inet->inet_dport = 0;
|
| 1081 | sk->sk_route_caps = 0;
|
| 1082 | }
|
| 1083 | }
|
| 1084 | flags = (msg->msg_flags & MSG_DONTWAIT) ? O_NONBLOCK : 0;
|
| 1085 | err = __inet_stream_connect(sk->sk_socket, (struct sockaddr_unsized *)uaddr,
|
| 1086 | msg->msg_namelen, flags, 1);
|
| 1087 | |
| 1088 | * if the connection times out or gets rst
|
| 1089 |
|
| 1090 | if (tp->fastopen_req) {
|
| 1091 | *copied = tp->fastopen_req->copied;
|
| 1092 | tcp_free_fastopen_req(tp);
|
| 1093 | inet_clear_bit(DEFER_CONNECT, sk);
|
| 1094 | }
|
| 1095 | return err;
|
| 1096 | }
|
| 1097 |
|
| 1098 |
|
| 1099 | void tcp_rate_check_app_limited(struct sock *sk)
|
| 1100 | {
|
| 1101 | struct tcp_sock *tp = tcp_sk(sk);
|
| 1102 |
|
| 1103 | if (
|
| 1104 | tp->write_seq - tp->snd_nxt < tp->mss_cache &&
|
| 1105 |
|
| 1106 | sk_wmem_alloc_get(sk) < SKB_TRUESIZE(1) &&
|
| 1107 |
|
| 1108 | tcp_packets_in_flight(tp) < tcp_snd_cwnd(tp) &&
|
| 1109 |
|
| 1110 | tp->lost_out <= tp->retrans_out)
|
| 1111 | tp->app_limited =
|
| 1112 | (tp->delivered + tcp_packets_in_flight(tp)) ? : 1;
|
| 1113 | }
|
| 1114 | EXPORT_SYMBOL_GPL(tcp_rate_check_app_limited);
|
| 1115 |
|
| 1116 | int tcp_sendmsg_locked(struct sock *sk, struct msghdr *msg, size_t size)
|
| 1117 | {
|
| 1118 | struct net_devmem_dmabuf_binding *binding = NULL;
|
| 1119 | struct tcp_sock *tp = tcp_sk(sk);
|
| 1120 | struct ubuf_info *uarg = NULL;
|
| 1121 | struct sk_buff *skb;
|
| 1122 | struct sockcm_cookie sockc;
|
| 1123 | int flags, err, copied = 0;
|
| 1124 | int mss_now = 0, size_goal, copied_syn = 0;
|
| 1125 | int process_backlog = 0;
|
| 1126 | int sockc_err = 0;
|
| 1127 | int zc = 0;
|
| 1128 | long timeo;
|
| 1129 |
|
| 1130 | flags = msg->msg_flags;
|
| 1131 |
|
| 1132 | sockc = (struct sockcm_cookie){ .tsflags = READ_ONCE(sk->sk_tsflags) };
|
| 1133 | if (msg->msg_controllen) {
|
| 1134 | sockc_err = sock_cmsg_send(sk, msg, &sockc);
|
| 1135 | |
| 1136 | * that may succeed even if the cmsg is invalid.
|
| 1137 |
|
| 1138 | }
|
| 1139 |
|
| 1140 | if ((flags & MSG_ZEROCOPY) && size) {
|
| 1141 | if (msg->msg_ubuf) {
|
| 1142 | uarg = msg->msg_ubuf;
|
| 1143 | if (sk->sk_route_caps & NETIF_F_SG)
|
| 1144 | zc = MSG_ZEROCOPY;
|
| 1145 | } else if (sock_flag(sk, SOCK_ZEROCOPY)) {
|
| 1146 | skb = tcp_write_queue_tail(sk);
|
| 1147 | uarg = msg_zerocopy_realloc(sk, size, skb_zcopy(skb),
|
| 1148 | !sockc_err && sockc.dmabuf_id);
|
| 1149 | if (!uarg) {
|
| 1150 | err = -ENOBUFS;
|
| 1151 | goto out_err;
|
| 1152 | }
|
| 1153 | if (sk->sk_route_caps & NETIF_F_SG)
|
| 1154 | zc = MSG_ZEROCOPY;
|
| 1155 | else
|
| 1156 | uarg_to_msgzc(uarg)->zerocopy = 0;
|
| 1157 |
|
| 1158 | if (!sockc_err && sockc.dmabuf_id) {
|
| 1159 | binding = net_devmem_get_binding(sk, sockc.dmabuf_id);
|
| 1160 | if (IS_ERR(binding)) {
|
| 1161 | err = PTR_ERR(binding);
|
| 1162 | binding = NULL;
|
| 1163 | goto out_err;
|
| 1164 | }
|
| 1165 | }
|
| 1166 | }
|
| 1167 | } else if (unlikely(msg->msg_flags & MSG_SPLICE_PAGES) && size) {
|
| 1168 | if (sk->sk_route_caps & NETIF_F_SG)
|
| 1169 | zc = MSG_SPLICE_PAGES;
|
| 1170 | }
|
| 1171 |
|
| 1172 | if (!sockc_err && sockc.dmabuf_id && (zc != MSG_ZEROCOPY || !binding)) {
|
| 1173 | err = -EINVAL;
|
| 1174 | goto out_err;
|
| 1175 | }
|
| 1176 |
|
| 1177 | if (unlikely(flags & MSG_FASTOPEN ||
|
| 1178 | inet_test_bit(DEFER_CONNECT, sk)) &&
|
| 1179 | !tp->repair) {
|
| 1180 | err = tcp_sendmsg_fastopen(sk, msg, &copied_syn, size, uarg);
|
| 1181 | if (err == -EINPROGRESS && copied_syn > 0)
|
| 1182 | goto out;
|
| 1183 | else if (err)
|
| 1184 | goto out_err;
|
| 1185 | }
|
| 1186 |
|
| 1187 | timeo = sock_sndtimeo(sk, flags & MSG_DONTWAIT);
|
| 1188 |
|
| 1189 | tcp_rate_check_app_limited(sk);
|
| 1190 |
|
| 1191 | |
| 1192 | * (passive side) where data is allowed to be sent before a connection
|
| 1193 | * is fully established.
|
| 1194 |
|
| 1195 | if (((1 << sk->sk_state) & ~(TCPF_ESTABLISHED | TCPF_CLOSE_WAIT)) &&
|
| 1196 | !tcp_passive_fastopen(sk)) {
|
| 1197 | err = sk_stream_wait_connect(sk, &timeo);
|
| 1198 | if (err != 0)
|
| 1199 | goto do_error;
|
| 1200 | }
|
| 1201 |
|
| 1202 | if (unlikely(tp->repair)) {
|
| 1203 | if (tp->repair_queue == TCP_RECV_QUEUE) {
|
| 1204 | copied = tcp_send_rcvq(sk, msg, size);
|
| 1205 | goto out_nopush;
|
| 1206 | }
|
| 1207 |
|
| 1208 | err = -EINVAL;
|
| 1209 | if (tp->repair_queue == TCP_NO_QUEUE)
|
| 1210 | goto out_err;
|
| 1211 |
|
| 1212 |
|
| 1213 | }
|
| 1214 |
|
| 1215 | if (sockc_err) {
|
| 1216 | err = sockc_err;
|
| 1217 | goto out_err;
|
| 1218 | }
|
| 1219 |
|
| 1220 |
|
| 1221 | sk_clear_bit(SOCKWQ_ASYNC_NOSPACE, sk);
|
| 1222 |
|
| 1223 |
|
| 1224 | copied = 0;
|
| 1225 |
|
| 1226 | restart:
|
| 1227 | mss_now = tcp_send_mss(sk, &size_goal, flags);
|
| 1228 |
|
| 1229 | err = -EPIPE;
|
| 1230 | if (sk->sk_err || (sk->sk_shutdown & SEND_SHUTDOWN))
|
| 1231 | goto do_error;
|
| 1232 |
|
| 1233 | while (msg_data_left(msg)) {
|
| 1234 | int copy = 0;
|
| 1235 |
|
| 1236 | skb = tcp_write_queue_tail(sk);
|
| 1237 | if (skb)
|
| 1238 | copy = size_goal - skb->len;
|
| 1239 |
|
| 1240 | trace_tcp_sendmsg_locked(sk, msg, skb, size_goal);
|
| 1241 |
|
| 1242 | if (copy <= 0 || !tcp_skb_can_collapse_to(skb) ||
|
| 1243 | unlikely(skb_frags_readable(skb) != !binding)) {
|
| 1244 | bool first_skb;
|
| 1245 |
|
| 1246 | new_segment:
|
| 1247 | if (!sk_stream_memory_free(sk))
|
| 1248 | goto wait_for_space;
|
| 1249 |
|
| 1250 | if (unlikely(process_backlog >= 16)) {
|
| 1251 | process_backlog = 0;
|
| 1252 | if (sk_flush_backlog(sk))
|
| 1253 | goto restart;
|
| 1254 | }
|
| 1255 | first_skb = tcp_rtx_and_write_queues_empty(sk);
|
| 1256 | skb = tcp_stream_alloc_skb(sk, sk->sk_allocation,
|
| 1257 | first_skb);
|
| 1258 | if (!skb)
|
| 1259 | goto wait_for_space;
|
| 1260 |
|
| 1261 | process_backlog++;
|
| 1262 |
|
| 1263 | #ifdef CONFIG_SKB_DECRYPTED
|
| 1264 | skb->decrypted = !!(flags & MSG_SENDPAGE_DECRYPTED);
|
| 1265 | #endif
|
| 1266 | tcp_skb_entail(sk, skb);
|
| 1267 | copy = size_goal;
|
| 1268 |
|
| 1269 | |
| 1270 | * already been sent. skb_mstamp_ns isn't set to
|
| 1271 | * avoid wrong rtt estimation.
|
| 1272 |
|
| 1273 | if (tp->repair)
|
| 1274 | TCP_SKB_CB(skb)->sacked |= TCPCB_REPAIRED;
|
| 1275 | }
|
| 1276 |
|
| 1277 |
|
| 1278 | if (copy > msg_data_left(msg))
|
| 1279 | copy = msg_data_left(msg);
|
| 1280 |
|
| 1281 | if (zc == 0) {
|
| 1282 | bool merge = true;
|
| 1283 | int i = skb_shinfo(skb)->nr_frags;
|
| 1284 | struct page_frag *pfrag = sk_page_frag(sk);
|
| 1285 |
|
| 1286 | if (!sk_page_frag_refill(sk, pfrag))
|
| 1287 | goto wait_for_space;
|
| 1288 |
|
| 1289 | if (!skb_can_coalesce(skb, i, pfrag->page,
|
| 1290 | pfrag->offset)) {
|
| 1291 | if (i >= READ_ONCE(net_hotdata.sysctl_max_skb_frags)) {
|
| 1292 | tcp_mark_push(tp, skb);
|
| 1293 | goto new_segment;
|
| 1294 | }
|
| 1295 | merge = false;
|
| 1296 | }
|
| 1297 |
|
| 1298 | copy = min_t(int, copy, pfrag->size - pfrag->offset);
|
| 1299 |
|
| 1300 | if (unlikely(skb_zcopy_pure(skb) || skb_zcopy_managed(skb))) {
|
| 1301 | if (tcp_downgrade_zcopy_pure(sk, skb))
|
| 1302 | goto wait_for_space;
|
| 1303 | skb_zcopy_downgrade_managed(skb);
|
| 1304 | }
|
| 1305 |
|
| 1306 | copy = tcp_wmem_schedule(sk, copy);
|
| 1307 | if (!copy)
|
| 1308 | goto wait_for_space;
|
| 1309 |
|
| 1310 | err = skb_copy_to_page_nocache(sk, &msg->msg_iter, skb,
|
| 1311 | pfrag->page,
|
| 1312 | pfrag->offset,
|
| 1313 | copy);
|
| 1314 | if (err)
|
| 1315 | goto do_error;
|
| 1316 |
|
| 1317 |
|
| 1318 | if (merge) {
|
| 1319 | skb_frag_size_add(&skb_shinfo(skb)->frags[i - 1], copy);
|
| 1320 | } else {
|
| 1321 | skb_fill_page_desc(skb, i, pfrag->page,
|
| 1322 | pfrag->offset, copy);
|
| 1323 | page_ref_inc(pfrag->page);
|
| 1324 | }
|
| 1325 | pfrag->offset += copy;
|
| 1326 | } else if (zc == MSG_ZEROCOPY) {
|
| 1327 | |
| 1328 | * pure zerocopy skb
|
| 1329 |
|
| 1330 | if (!skb->len)
|
| 1331 | skb_shinfo(skb)->flags |= SKBFL_PURE_ZEROCOPY;
|
| 1332 |
|
| 1333 | if (!skb_zcopy_pure(skb)) {
|
| 1334 | copy = tcp_wmem_schedule(sk, copy);
|
| 1335 | if (!copy)
|
| 1336 | goto wait_for_space;
|
| 1337 | }
|
| 1338 |
|
| 1339 | err = skb_zerocopy_iter_stream(sk, skb, msg, copy, uarg,
|
| 1340 | binding);
|
| 1341 | if (err == -EMSGSIZE || err == -EEXIST) {
|
| 1342 | tcp_mark_push(tp, skb);
|
| 1343 | goto new_segment;
|
| 1344 | }
|
| 1345 | if (err < 0)
|
| 1346 | goto do_error;
|
| 1347 | copy = err;
|
| 1348 | } else if (zc == MSG_SPLICE_PAGES) {
|
| 1349 |
|
| 1350 | if (tcp_downgrade_zcopy_pure(sk, skb))
|
| 1351 | goto wait_for_space;
|
| 1352 | copy = tcp_wmem_schedule(sk, copy);
|
| 1353 | if (!copy)
|
| 1354 | goto wait_for_space;
|
| 1355 |
|
| 1356 | err = skb_splice_from_iter(skb, &msg->msg_iter, copy);
|
| 1357 | if (err < 0) {
|
| 1358 | if (err == -EMSGSIZE) {
|
| 1359 | tcp_mark_push(tp, skb);
|
| 1360 | goto new_segment;
|
| 1361 | }
|
| 1362 | goto do_error;
|
| 1363 | }
|
| 1364 | copy = err;
|
| 1365 |
|
| 1366 | if (!(flags & MSG_NO_SHARED_FRAGS))
|
| 1367 | skb_shinfo(skb)->flags |= SKBFL_SHARED_FRAG;
|
| 1368 |
|
| 1369 | sk_wmem_queued_add(sk, copy);
|
| 1370 | sk_mem_charge(sk, copy);
|
| 1371 | }
|
| 1372 |
|
| 1373 | if (!copied)
|
| 1374 | TCP_SKB_CB(skb)->tcp_flags &= ~TCPHDR_PSH;
|
| 1375 |
|
| 1376 | WRITE_ONCE(tp->write_seq, tp->write_seq + copy);
|
| 1377 | TCP_SKB_CB(skb)->end_seq += copy;
|
| 1378 | tcp_skb_pcount_set(skb, 0);
|
| 1379 |
|
| 1380 | copied += copy;
|
| 1381 | if (!msg_data_left(msg)) {
|
| 1382 | if (unlikely(flags & MSG_EOR))
|
| 1383 | TCP_SKB_CB(skb)->eor = 1;
|
| 1384 | goto out;
|
| 1385 | }
|
| 1386 |
|
| 1387 | if (skb->len < size_goal || (flags & MSG_OOB) || unlikely(tp->repair))
|
| 1388 | continue;
|
| 1389 |
|
| 1390 | if (forced_push(tp)) {
|
| 1391 | tcp_mark_push(tp, skb);
|
| 1392 | __tcp_push_pending_frames(sk, mss_now, TCP_NAGLE_PUSH);
|
| 1393 | } else if (skb == tcp_send_head(sk))
|
| 1394 | tcp_push_one(sk, mss_now);
|
| 1395 | continue;
|
| 1396 |
|
| 1397 | wait_for_space:
|
| 1398 | set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
|
| 1399 | tcp_remove_empty_skb(sk);
|
| 1400 | if (copied)
|
| 1401 | tcp_push(sk, flags & ~MSG_MORE, mss_now,
|
| 1402 | TCP_NAGLE_PUSH, size_goal);
|
| 1403 |
|
| 1404 | err = sk_stream_wait_memory(sk, &timeo);
|
| 1405 | if (err != 0)
|
| 1406 | goto do_error;
|
| 1407 |
|
| 1408 | mss_now = tcp_send_mss(sk, &size_goal, flags);
|
| 1409 | }
|
| 1410 |
|
| 1411 | out:
|
| 1412 | if (copied) {
|
| 1413 | tcp_tx_timestamp(sk, &sockc);
|
| 1414 | tcp_push(sk, flags, mss_now, tp->nonagle, size_goal);
|
| 1415 | }
|
| 1416 | out_nopush:
|
| 1417 |
|
| 1418 | if (uarg && !msg->msg_ubuf)
|
| 1419 | net_zcopy_put(uarg);
|
| 1420 | if (binding)
|
| 1421 | net_devmem_dmabuf_binding_put(binding);
|
| 1422 | return copied + copied_syn;
|
| 1423 |
|
| 1424 | do_error:
|
| 1425 | tcp_remove_empty_skb(sk);
|
| 1426 |
|
| 1427 | if (copied + copied_syn)
|
| 1428 | goto out;
|
| 1429 | out_err:
|
| 1430 |
|
| 1431 | if (uarg && !msg->msg_ubuf)
|
| 1432 | net_zcopy_put_abort(uarg, true);
|
| 1433 | err = sk_stream_error(sk, flags, err);
|
| 1434 |
|
| 1435 | if (unlikely(tcp_rtx_and_write_queues_empty(sk) && err == -EAGAIN)) {
|
| 1436 | READ_ONCE(sk->sk_write_space)(sk);
|
| 1437 | tcp_chrono_stop(sk, TCP_CHRONO_SNDBUF_LIMITED);
|
| 1438 | }
|
| 1439 | if (binding)
|
| 1440 | net_devmem_dmabuf_binding_put(binding);
|
| 1441 |
|
| 1442 | return err;
|
| 1443 | }
|
| 1444 | EXPORT_SYMBOL_GPL(tcp_sendmsg_locked);
|
| 1445 |
|
| 1446 | int tcp_sendmsg(struct sock *sk, struct msghdr *msg, size_t size)
|
| 1447 | {
|
| 1448 | int ret;
|
| 1449 |
|
| 1450 | lock_sock(sk);
|
| 1451 | ret = tcp_sendmsg_locked(sk, msg, size);
|
| 1452 | release_sock(sk);
|
| 1453 |
|
| 1454 | return ret;
|
| 1455 | }
|
| 1456 | EXPORT_SYMBOL(tcp_sendmsg);
|
| 1457 |
|
| 1458 | void tcp_splice_eof(struct socket *sock)
|
| 1459 | {
|
| 1460 | struct sock *sk = sock->sk;
|
| 1461 | struct tcp_sock *tp = tcp_sk(sk);
|
| 1462 | int mss_now, size_goal;
|
| 1463 |
|
| 1464 | if (!tcp_write_queue_tail(sk))
|
| 1465 | return;
|
| 1466 |
|
| 1467 | lock_sock(sk);
|
| 1468 | mss_now = tcp_send_mss(sk, &size_goal, 0);
|
| 1469 | tcp_push(sk, 0, mss_now, tp->nonagle, size_goal);
|
| 1470 | release_sock(sk);
|
| 1471 | }
|
| 1472 |
|
| 1473 | |
| 1474 | * Handle reading urgent data. BSD has very simple semantics for
|
| 1475 | * this, no blocking and very strange errors 8)
|
| 1476 |
|
| 1477 |
|
| 1478 | static int tcp_recv_urg(struct sock *sk, struct msghdr *msg, int len, int flags)
|
| 1479 | {
|
| 1480 | struct tcp_sock *tp = tcp_sk(sk);
|
| 1481 |
|
| 1482 |
|
| 1483 | if (sock_flag(sk, SOCK_URGINLINE) || !tp->urg_data ||
|
| 1484 | tp->urg_data == TCP_URG_READ)
|
| 1485 | return -EINVAL;
|
| 1486 |
|
| 1487 | if (sk->sk_state == TCP_CLOSE && !sock_flag(sk, SOCK_DONE))
|
| 1488 | return -ENOTCONN;
|
| 1489 |
|
| 1490 | if (tp->urg_data & TCP_URG_VALID) {
|
| 1491 | int err = 0;
|
| 1492 | char c = tp->urg_data;
|
| 1493 |
|
| 1494 | if (!(flags & MSG_PEEK))
|
| 1495 | WRITE_ONCE(tp->urg_data, TCP_URG_READ);
|
| 1496 |
|
| 1497 |
|
| 1498 | msg->msg_flags |= MSG_OOB;
|
| 1499 |
|
| 1500 | if (len > 0) {
|
| 1501 | if (!(flags & MSG_TRUNC))
|
| 1502 | err = memcpy_to_msg(msg, &c, 1);
|
| 1503 | len = 1;
|
| 1504 | } else
|
| 1505 | msg->msg_flags |= MSG_TRUNC;
|
| 1506 |
|
| 1507 | return err ? -EFAULT : len;
|
| 1508 | }
|
| 1509 |
|
| 1510 | if (sk->sk_state == TCP_CLOSE || (sk->sk_shutdown & RCV_SHUTDOWN))
|
| 1511 | return 0;
|
| 1512 |
|
| 1513 | |
| 1514 | * the available implementations agree in this case:
|
| 1515 | * this call should never block, independent of the
|
| 1516 | * blocking state of the socket.
|
| 1517 | * Mike <[email protected]>
|
| 1518 |
|
| 1519 | return -EAGAIN;
|
| 1520 | }
|
| 1521 |
|
| 1522 | static int tcp_peek_sndq(struct sock *sk, struct msghdr *msg, int len)
|
| 1523 | {
|
| 1524 | struct sk_buff *skb;
|
| 1525 | int copied = 0, err = 0;
|
| 1526 |
|
| 1527 | skb_rbtree_walk(skb, &sk->tcp_rtx_queue) {
|
| 1528 | err = skb_copy_datagram_msg(skb, 0, msg, skb->len);
|
| 1529 | if (err)
|
| 1530 | return err;
|
| 1531 | copied += skb->len;
|
| 1532 | }
|
| 1533 |
|
| 1534 | skb_queue_walk(&sk->sk_write_queue, skb) {
|
| 1535 | err = skb_copy_datagram_msg(skb, 0, msg, skb->len);
|
| 1536 | if (err)
|
| 1537 | break;
|
| 1538 |
|
| 1539 | copied += skb->len;
|
| 1540 | }
|
| 1541 |
|
| 1542 | return err ?: copied;
|
| 1543 | }
|
| 1544 |
|
| 1545 | |
| 1546 | * then send an ACK if necessary. COPIED is the number of bytes
|
| 1547 | * tcp_recvmsg has given to the user so far, it speeds up the
|
| 1548 | * calculation of whether or not we must ACK for the sake of
|
| 1549 | * a window update.
|
| 1550 |
|
| 1551 | void __tcp_cleanup_rbuf(struct sock *sk, int copied)
|
| 1552 | {
|
| 1553 | struct tcp_sock *tp = tcp_sk(sk);
|
| 1554 | bool time_to_ack = false;
|
| 1555 |
|
| 1556 | if (inet_csk_ack_scheduled(sk)) {
|
| 1557 | const struct inet_connection_sock *icsk = inet_csk(sk);
|
| 1558 |
|
| 1559 | if (
|
| 1560 | tp->rcv_nxt - tp->rcv_wup > icsk->icsk_ack.rcv_mss ||
|
| 1561 | |
| 1562 | * If this read emptied read buffer, we send ACK, if
|
| 1563 | * connection is not bidirectional, user drained
|
| 1564 | * receive buffer and there was a small segment
|
| 1565 | * in queue.
|
| 1566 |
|
| 1567 | (copied > 0 &&
|
| 1568 | ((icsk->icsk_ack.pending & ICSK_ACK_PUSHED2) ||
|
| 1569 | ((icsk->icsk_ack.pending & ICSK_ACK_PUSHED) &&
|
| 1570 | !inet_csk_in_pingpong_mode(sk))) &&
|
| 1571 | !atomic_read(&sk->sk_rmem_alloc)))
|
| 1572 | time_to_ack = true;
|
| 1573 | }
|
| 1574 |
|
| 1575 | |
| 1576 | * which has been raised "significantly".
|
| 1577 | *
|
| 1578 | * Even if window raised up to infinity, do not send window open ACK
|
| 1579 | * in states, where we will not receive more. It is useless.
|
| 1580 |
|
| 1581 | if (copied > 0 && !time_to_ack && !(sk->sk_shutdown & RCV_SHUTDOWN)) {
|
| 1582 | __u32 rcv_window_now = tcp_receive_window(tp);
|
| 1583 |
|
| 1584 |
|
| 1585 | if (2*rcv_window_now <= tp->window_clamp) {
|
| 1586 | __u32 new_window = __tcp_select_window(sk);
|
| 1587 |
|
| 1588 | |
| 1589 | * in our buffer. Certainly, new_window is new window.
|
| 1590 | * We can advertise it now, if it is not less than current one.
|
| 1591 | * "Lots" means "at least twice" here.
|
| 1592 |
|
| 1593 | if (new_window && new_window >= 2 * rcv_window_now)
|
| 1594 | time_to_ack = true;
|
| 1595 | }
|
| 1596 | }
|
| 1597 | if (time_to_ack) {
|
| 1598 | tcp_mstamp_refresh(tp);
|
| 1599 | tcp_send_ack(sk);
|
| 1600 | }
|
| 1601 | }
|
| 1602 |
|
| 1603 | void tcp_cleanup_rbuf(struct sock *sk, int copied)
|
| 1604 | {
|
| 1605 | struct sk_buff *skb = skb_peek(&sk->sk_receive_queue);
|
| 1606 | struct tcp_sock *tp = tcp_sk(sk);
|
| 1607 |
|
| 1608 | WARN(skb && !before(tp->copied_seq, TCP_SKB_CB(skb)->end_seq),
|
| 1609 | "cleanup rbuf bug: copied %X seq %X rcvnxt %X\n",
|
| 1610 | tp->copied_seq, TCP_SKB_CB(skb)->end_seq, tp->rcv_nxt);
|
| 1611 | __tcp_cleanup_rbuf(sk, copied);
|
| 1612 | }
|
| 1613 |
|
| 1614 | static void tcp_eat_recv_skb(struct sock *sk, struct sk_buff *skb)
|
| 1615 | {
|
| 1616 | __skb_unlink(skb, &sk->sk_receive_queue);
|
| 1617 | if (likely(skb->destructor == sock_rfree)) {
|
| 1618 | sock_rfree(skb);
|
| 1619 | skb->destructor = NULL;
|
| 1620 | skb->sk = NULL;
|
| 1621 | return skb_attempt_defer_free(skb);
|
| 1622 | }
|
| 1623 | __kfree_skb(skb);
|
| 1624 | }
|
| 1625 |
|
| 1626 | struct sk_buff *tcp_recv_skb(struct sock *sk, u32 seq, u32 *off)
|
| 1627 | {
|
| 1628 | struct sk_buff *skb;
|
| 1629 | u32 offset;
|
| 1630 |
|
| 1631 | while ((skb = skb_peek(&sk->sk_receive_queue)) != NULL) {
|
| 1632 | offset = seq - TCP_SKB_CB(skb)->seq;
|
| 1633 | if (unlikely(TCP_SKB_CB(skb)->tcp_flags & TCPHDR_SYN)) {
|
| 1634 | pr_err_once("%s: found a SYN, please report !\n", __func__);
|
| 1635 | offset--;
|
| 1636 | }
|
| 1637 | if (offset < skb->len || (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN)) {
|
| 1638 | *off = offset;
|
| 1639 | return skb;
|
| 1640 | }
|
| 1641 | |
| 1642 | * splitted a fat GRO packet, while we released socket lock
|
| 1643 | * in skb_splice_bits()
|
| 1644 |
|
| 1645 | tcp_eat_recv_skb(sk, skb);
|
| 1646 | }
|
| 1647 | return NULL;
|
| 1648 | }
|
| 1649 | EXPORT_SYMBOL(tcp_recv_skb);
|
| 1650 |
|
| 1651 | |
| 1652 | * This routine provides an alternative to tcp_recvmsg() for routines
|
| 1653 | * that would like to handle copying from skbuffs directly in 'sendfile'
|
| 1654 | * fashion.
|
| 1655 | * Note:
|
| 1656 | * - It is assumed that the socket was locked by the caller.
|
| 1657 | * - The routine does not block.
|
| 1658 | * - At present, there is no support for reading OOB data
|
| 1659 | * or for 'peeking' the socket using this routine
|
| 1660 | * (although both would be easy to implement).
|
| 1661 |
|
| 1662 | static int __tcp_read_sock(struct sock *sk, read_descriptor_t *desc,
|
| 1663 | sk_read_actor_t recv_actor, bool noack,
|
| 1664 | u32 *copied_seq)
|
| 1665 | {
|
| 1666 | struct sk_buff *skb;
|
| 1667 | struct tcp_sock *tp = tcp_sk(sk);
|
| 1668 | u32 seq = *copied_seq;
|
| 1669 | u32 offset;
|
| 1670 | int copied = 0;
|
| 1671 |
|
| 1672 | if (sk->sk_state == TCP_LISTEN)
|
| 1673 | return -ENOTCONN;
|
| 1674 | while ((skb = tcp_recv_skb(sk, seq, &offset)) != NULL) {
|
| 1675 | if (offset < skb->len) {
|
| 1676 | int used;
|
| 1677 | size_t len;
|
| 1678 |
|
| 1679 | len = skb->len - offset;
|
| 1680 |
|
| 1681 | if (unlikely(tp->urg_data)) {
|
| 1682 | u32 urg_offset = tp->urg_seq - seq;
|
| 1683 | if (urg_offset < len)
|
| 1684 | len = urg_offset;
|
| 1685 | if (!len)
|
| 1686 | break;
|
| 1687 | }
|
| 1688 | used = recv_actor(desc, skb, offset, len);
|
| 1689 | if (used <= 0) {
|
| 1690 | if (!copied)
|
| 1691 | copied = used;
|
| 1692 | break;
|
| 1693 | }
|
| 1694 | if (WARN_ON_ONCE(used > len))
|
| 1695 | used = len;
|
| 1696 | seq += used;
|
| 1697 | copied += used;
|
| 1698 | offset += used;
|
| 1699 |
|
| 1700 | |
| 1701 | * receive) the skb pointer might be invalid when
|
| 1702 | * getting here: tcp_collapse might have deleted it
|
| 1703 | * while aggregating skbs from the socket queue.
|
| 1704 |
|
| 1705 | skb = tcp_recv_skb(sk, seq - 1, &offset);
|
| 1706 | if (!skb)
|
| 1707 | break;
|
| 1708 | |
| 1709 | * Try to splice more frags
|
| 1710 |
|
| 1711 | if (offset + 1 != skb->len)
|
| 1712 | continue;
|
| 1713 | }
|
| 1714 | if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN) {
|
| 1715 | tcp_eat_recv_skb(sk, skb);
|
| 1716 | ++seq;
|
| 1717 | break;
|
| 1718 | }
|
| 1719 | tcp_eat_recv_skb(sk, skb);
|
| 1720 | if (!desc->count)
|
| 1721 | break;
|
| 1722 | WRITE_ONCE(*copied_seq, seq);
|
| 1723 | }
|
| 1724 | WRITE_ONCE(*copied_seq, seq);
|
| 1725 |
|
| 1726 | if (noack)
|
| 1727 | goto out;
|
| 1728 |
|
| 1729 | tcp_rcv_space_adjust(sk);
|
| 1730 |
|
| 1731 |
|
| 1732 | if (copied > 0) {
|
| 1733 | tcp_recv_skb(sk, seq, &offset);
|
| 1734 | tcp_cleanup_rbuf(sk, copied);
|
| 1735 | }
|
| 1736 | out:
|
| 1737 | return copied;
|
| 1738 | }
|
| 1739 |
|
| 1740 | int tcp_read_sock(struct sock *sk, read_descriptor_t *desc,
|
| 1741 | sk_read_actor_t recv_actor)
|
| 1742 | {
|
| 1743 | return __tcp_read_sock(sk, desc, recv_actor, false,
|
| 1744 | &tcp_sk(sk)->copied_seq);
|
| 1745 | }
|
| 1746 | EXPORT_SYMBOL(tcp_read_sock);
|
| 1747 |
|
| 1748 | int tcp_read_sock_noack(struct sock *sk, read_descriptor_t *desc,
|
| 1749 | sk_read_actor_t recv_actor, bool noack,
|
| 1750 | u32 *copied_seq)
|
| 1751 | {
|
| 1752 | return __tcp_read_sock(sk, desc, recv_actor, noack, copied_seq);
|
| 1753 | }
|
| 1754 |
|
| 1755 | int tcp_read_skb(struct sock *sk, skb_read_actor_t recv_actor)
|
| 1756 | {
|
| 1757 | struct sk_buff *skb;
|
| 1758 | int copied = 0;
|
| 1759 |
|
| 1760 | if (sk->sk_state == TCP_LISTEN)
|
| 1761 | return -ENOTCONN;
|
| 1762 |
|
| 1763 | while ((skb = skb_peek(&sk->sk_receive_queue)) != NULL) {
|
| 1764 | u8 tcp_flags;
|
| 1765 | int used;
|
| 1766 |
|
| 1767 | __skb_unlink(skb, &sk->sk_receive_queue);
|
| 1768 | WARN_ON_ONCE(!skb_set_owner_sk_safe(skb, sk));
|
| 1769 | tcp_flags = TCP_SKB_CB(skb)->tcp_flags;
|
| 1770 | used = recv_actor(sk, skb);
|
| 1771 | if (used < 0) {
|
| 1772 | if (!copied)
|
| 1773 | copied = used;
|
| 1774 | break;
|
| 1775 | }
|
| 1776 | copied += used;
|
| 1777 |
|
| 1778 | if (tcp_flags & TCPHDR_FIN)
|
| 1779 | break;
|
| 1780 | }
|
| 1781 | return copied;
|
| 1782 | }
|
| 1783 |
|
| 1784 | void tcp_read_done(struct sock *sk, size_t len)
|
| 1785 | {
|
| 1786 | struct tcp_sock *tp = tcp_sk(sk);
|
| 1787 | u32 seq = tp->copied_seq;
|
| 1788 | struct sk_buff *skb;
|
| 1789 | size_t left;
|
| 1790 | u32 offset;
|
| 1791 |
|
| 1792 | if (sk->sk_state == TCP_LISTEN)
|
| 1793 | return;
|
| 1794 |
|
| 1795 | left = len;
|
| 1796 | while (left && (skb = tcp_recv_skb(sk, seq, &offset)) != NULL) {
|
| 1797 | int used;
|
| 1798 |
|
| 1799 | used = min_t(size_t, skb->len - offset, left);
|
| 1800 | seq += used;
|
| 1801 | left -= used;
|
| 1802 |
|
| 1803 | if (skb->len > offset + used)
|
| 1804 | break;
|
| 1805 |
|
| 1806 | if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN) {
|
| 1807 | tcp_eat_recv_skb(sk, skb);
|
| 1808 | ++seq;
|
| 1809 | break;
|
| 1810 | }
|
| 1811 | tcp_eat_recv_skb(sk, skb);
|
| 1812 | }
|
| 1813 | WRITE_ONCE(tp->copied_seq, seq);
|
| 1814 |
|
| 1815 | tcp_rcv_space_adjust(sk);
|
| 1816 |
|
| 1817 |
|
| 1818 | if (left != len)
|
| 1819 | tcp_cleanup_rbuf(sk, len - left);
|
| 1820 | }
|
| 1821 | EXPORT_SYMBOL(tcp_read_done);
|
| 1822 |
|
| 1823 | int tcp_peek_len(struct socket *sock)
|
| 1824 | {
|
| 1825 | return tcp_inq(sock->sk);
|
| 1826 | }
|
| 1827 |
|
| 1828 |
|
| 1829 | int tcp_set_rcvlowat(struct sock *sk, int val)
|
| 1830 | {
|
| 1831 | struct tcp_sock *tp = tcp_sk(sk);
|
| 1832 | int space, cap;
|
| 1833 |
|
| 1834 | if (sk->sk_userlocks & SOCK_RCVBUF_LOCK)
|
| 1835 | cap = sk->sk_rcvbuf >> 1;
|
| 1836 | else
|
| 1837 | cap = READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_rmem[2]) >> 1;
|
| 1838 | val = min(val, cap);
|
| 1839 | WRITE_ONCE(sk->sk_rcvlowat, val ? : 1);
|
| 1840 |
|
| 1841 |
|
| 1842 | tcp_data_ready(sk);
|
| 1843 |
|
| 1844 | if (sk->sk_userlocks & SOCK_RCVBUF_LOCK)
|
| 1845 | return 0;
|
| 1846 |
|
| 1847 | space = tcp_space_from_win(sk, val);
|
| 1848 | if (space > sk->sk_rcvbuf) {
|
| 1849 | WRITE_ONCE(sk->sk_rcvbuf, space);
|
| 1850 |
|
| 1851 | if (tp->window_clamp && tp->window_clamp < val)
|
| 1852 | WRITE_ONCE(tp->window_clamp, val);
|
| 1853 | }
|
| 1854 | return 0;
|
| 1855 | }
|
| 1856 |
|
| 1857 | void tcp_set_rcvbuf(struct sock *sk, int val)
|
| 1858 | {
|
| 1859 | tcp_set_window_clamp(sk, tcp_win_from_space(sk, val));
|
| 1860 | }
|
| 1861 |
|
| 1862 | #ifdef CONFIG_MMU
|
| 1863 | static const struct vm_operations_struct tcp_vm_ops = {
|
| 1864 | };
|
| 1865 |
|
| 1866 | int tcp_mmap(struct file *file, struct socket *sock,
|
| 1867 | struct vm_area_struct *vma)
|
| 1868 | {
|
| 1869 | if (vma->vm_flags & (VM_WRITE | VM_EXEC))
|
| 1870 | return -EPERM;
|
| 1871 | vm_flags_clear(vma, VM_MAYWRITE | VM_MAYEXEC);
|
| 1872 |
|
| 1873 |
|
| 1874 | vm_flags_set(vma, VM_MIXEDMAP);
|
| 1875 |
|
| 1876 | vma->vm_ops = &tcp_vm_ops;
|
| 1877 | return 0;
|
| 1878 | }
|
| 1879 |
|
| 1880 | static skb_frag_t *skb_advance_to_frag(struct sk_buff *skb, u32 offset_skb,
|
| 1881 | u32 *offset_frag)
|
| 1882 | {
|
| 1883 | skb_frag_t *frag;
|
| 1884 |
|
| 1885 | if (unlikely(offset_skb >= skb->len))
|
| 1886 | return NULL;
|
| 1887 |
|
| 1888 | offset_skb -= skb_headlen(skb);
|
| 1889 | if ((int)offset_skb < 0 || skb_has_frag_list(skb))
|
| 1890 | return NULL;
|
| 1891 |
|
| 1892 | frag = skb_shinfo(skb)->frags;
|
| 1893 | while (offset_skb) {
|
| 1894 | if (skb_frag_size(frag) > offset_skb) {
|
| 1895 | *offset_frag = offset_skb;
|
| 1896 | return frag;
|
| 1897 | }
|
| 1898 | offset_skb -= skb_frag_size(frag);
|
| 1899 | ++frag;
|
| 1900 | }
|
| 1901 | *offset_frag = 0;
|
| 1902 | return frag;
|
| 1903 | }
|
| 1904 |
|
| 1905 | static bool can_map_frag(const skb_frag_t *frag)
|
| 1906 | {
|
| 1907 | struct page *page;
|
| 1908 |
|
| 1909 | if (skb_frag_size(frag) != PAGE_SIZE || skb_frag_off(frag))
|
| 1910 | return false;
|
| 1911 | if (skb_frag_is_net_iov(frag))
|
| 1912 | return false;
|
| 1913 |
|
| 1914 | page = skb_frag_page(frag);
|
| 1915 |
|
| 1916 | if (PageCompound(page) || page->mapping)
|
| 1917 | return false;
|
| 1918 |
|
| 1919 | return true;
|
| 1920 | }
|
| 1921 |
|
| 1922 | static int find_next_mappable_frag(const skb_frag_t *frag,
|
| 1923 | int remaining_in_skb)
|
| 1924 | {
|
| 1925 | int offset = 0;
|
| 1926 |
|
| 1927 | if (likely(can_map_frag(frag)))
|
| 1928 | return 0;
|
| 1929 |
|
| 1930 | while (offset < remaining_in_skb && !can_map_frag(frag)) {
|
| 1931 | offset += skb_frag_size(frag);
|
| 1932 | ++frag;
|
| 1933 | }
|
| 1934 | return offset;
|
| 1935 | }
|
| 1936 |
|
| 1937 | static void tcp_zerocopy_set_hint_for_skb(struct sock *sk,
|
| 1938 | struct tcp_zerocopy_receive *zc,
|
| 1939 | struct sk_buff *skb, u32 offset)
|
| 1940 | {
|
| 1941 | u32 frag_offset, partial_frag_remainder = 0;
|
| 1942 | int mappable_offset;
|
| 1943 | skb_frag_t *frag;
|
| 1944 |
|
| 1945 |
|
| 1946 | zc->recv_skip_hint = skb->len - offset;
|
| 1947 |
|
| 1948 |
|
| 1949 | frag = skb_advance_to_frag(skb, offset, &frag_offset);
|
| 1950 | if (!frag)
|
| 1951 | return;
|
| 1952 |
|
| 1953 | if (frag_offset) {
|
| 1954 | struct skb_shared_info *info = skb_shinfo(skb);
|
| 1955 |
|
| 1956 |
|
| 1957 | if (frag == &info->frags[info->nr_frags - 1])
|
| 1958 | return;
|
| 1959 |
|
| 1960 |
|
| 1961 | partial_frag_remainder = skb_frag_size(frag) - frag_offset;
|
| 1962 | zc->recv_skip_hint -= partial_frag_remainder;
|
| 1963 | ++frag;
|
| 1964 | }
|
| 1965 |
|
| 1966 | |
| 1967 | * mappable_offset: Bytes till next mappable frag, *not* counting bytes
|
| 1968 | * in partial_frag_remainder.
|
| 1969 |
|
| 1970 | mappable_offset = find_next_mappable_frag(frag, zc->recv_skip_hint);
|
| 1971 | zc->recv_skip_hint = mappable_offset + partial_frag_remainder;
|
| 1972 | }
|
| 1973 |
|
| 1974 | static int tcp_recvmsg_locked(struct sock *sk, struct msghdr *msg, size_t len,
|
| 1975 | int flags, struct scm_timestamping_internal *tss,
|
| 1976 | int *cmsg_flags);
|
| 1977 | static int receive_fallback_to_copy(struct sock *sk,
|
| 1978 | struct tcp_zerocopy_receive *zc, int inq,
|
| 1979 | struct scm_timestamping_internal *tss)
|
| 1980 | {
|
| 1981 | unsigned long copy_address = (unsigned long)zc->copybuf_address;
|
| 1982 | struct msghdr msg = {};
|
| 1983 | int err;
|
| 1984 |
|
| 1985 | zc->length = 0;
|
| 1986 | zc->recv_skip_hint = 0;
|
| 1987 |
|
| 1988 | if (copy_address != zc->copybuf_address)
|
| 1989 | return -EINVAL;
|
| 1990 |
|
| 1991 | err = import_ubuf(ITER_DEST, (void __user *)copy_address, inq,
|
| 1992 | &msg.msg_iter);
|
| 1993 | if (err)
|
| 1994 | return err;
|
| 1995 |
|
| 1996 | err = tcp_recvmsg_locked(sk, &msg, inq, MSG_DONTWAIT,
|
| 1997 | tss, &zc->msg_flags);
|
| 1998 | if (err < 0)
|
| 1999 | return err;
|
| 2000 |
|
| 2001 | zc->copybuf_len = err;
|
| 2002 | if (likely(zc->copybuf_len)) {
|
| 2003 | struct sk_buff *skb;
|
| 2004 | u32 offset;
|
| 2005 |
|
| 2006 | skb = tcp_recv_skb(sk, tcp_sk(sk)->copied_seq, &offset);
|
| 2007 | if (skb)
|
| 2008 | tcp_zerocopy_set_hint_for_skb(sk, zc, skb, offset);
|
| 2009 | }
|
| 2010 | return 0;
|
| 2011 | }
|
| 2012 |
|
| 2013 | static int tcp_copy_straggler_data(struct tcp_zerocopy_receive *zc,
|
| 2014 | struct sk_buff *skb, u32 copylen,
|
| 2015 | u32 *offset, u32 *seq)
|
| 2016 | {
|
| 2017 | unsigned long copy_address = (unsigned long)zc->copybuf_address;
|
| 2018 | struct msghdr msg = {};
|
| 2019 | int err;
|
| 2020 |
|
| 2021 | if (copy_address != zc->copybuf_address)
|
| 2022 | return -EINVAL;
|
| 2023 |
|
| 2024 | err = import_ubuf(ITER_DEST, (void __user *)copy_address, copylen,
|
| 2025 | &msg.msg_iter);
|
| 2026 | if (err)
|
| 2027 | return err;
|
| 2028 | err = skb_copy_datagram_msg(skb, *offset, &msg, copylen);
|
| 2029 | if (err)
|
| 2030 | return err;
|
| 2031 | zc->recv_skip_hint -= copylen;
|
| 2032 | *offset += copylen;
|
| 2033 | *seq += copylen;
|
| 2034 | return (__s32)copylen;
|
| 2035 | }
|
| 2036 |
|
| 2037 | static int tcp_zc_handle_leftover(struct tcp_zerocopy_receive *zc,
|
| 2038 | struct sock *sk,
|
| 2039 | struct sk_buff *skb,
|
| 2040 | u32 *seq,
|
| 2041 | s32 copybuf_len,
|
| 2042 | struct scm_timestamping_internal *tss)
|
| 2043 | {
|
| 2044 | u32 offset, copylen = min_t(u32, copybuf_len, zc->recv_skip_hint);
|
| 2045 |
|
| 2046 | if (!copylen)
|
| 2047 | return 0;
|
| 2048 |
|
| 2049 | if (skb) {
|
| 2050 | offset = *seq - TCP_SKB_CB(skb)->seq;
|
| 2051 | } else {
|
| 2052 | skb = tcp_recv_skb(sk, *seq, &offset);
|
| 2053 | if (TCP_SKB_CB(skb)->has_rxtstamp) {
|
| 2054 | tcp_update_recv_tstamps(skb, tss);
|
| 2055 | zc->msg_flags |= TCP_CMSG_TS;
|
| 2056 | }
|
| 2057 | }
|
| 2058 |
|
| 2059 | zc->copybuf_len = tcp_copy_straggler_data(zc, skb, copylen, &offset,
|
| 2060 | seq);
|
| 2061 | return zc->copybuf_len < 0 ? 0 : copylen;
|
| 2062 | }
|
| 2063 |
|
| 2064 | static int tcp_zerocopy_vm_insert_batch_error(struct vm_area_struct *vma,
|
| 2065 | struct page **pending_pages,
|
| 2066 | unsigned long pages_remaining,
|
| 2067 | unsigned long *address,
|
| 2068 | u32 *length,
|
| 2069 | u32 *seq,
|
| 2070 | struct tcp_zerocopy_receive *zc,
|
| 2071 | u32 total_bytes_to_map,
|
| 2072 | int err)
|
| 2073 | {
|
| 2074 |
|
| 2075 | if (err == -EBUSY &&
|
| 2076 | zc->flags & TCP_RECEIVE_ZEROCOPY_FLAG_TLB_CLEAN_HINT) {
|
| 2077 | u32 maybe_zap_len;
|
| 2078 |
|
| 2079 | maybe_zap_len = total_bytes_to_map -
|
| 2080 | *length +
|
| 2081 | (pages_remaining * PAGE_SIZE);
|
| 2082 | zap_vma_range(vma, *address, maybe_zap_len);
|
| 2083 | err = 0;
|
| 2084 | }
|
| 2085 |
|
| 2086 | if (!err) {
|
| 2087 | unsigned long leftover_pages = pages_remaining;
|
| 2088 | int bytes_mapped;
|
| 2089 |
|
| 2090 |
|
| 2091 | err = vm_insert_pages(vma, *address,
|
| 2092 | pending_pages,
|
| 2093 | &pages_remaining);
|
| 2094 | bytes_mapped = PAGE_SIZE * (leftover_pages - pages_remaining);
|
| 2095 | *seq += bytes_mapped;
|
| 2096 | *address += bytes_mapped;
|
| 2097 | }
|
| 2098 | if (err) {
|
| 2099 | |
| 2100 | * insert, and still had an issue. Either ways, pages_remaining
|
| 2101 | * is the number of pages we were unable to map, and we unroll
|
| 2102 | * some state we speculatively touched before.
|
| 2103 |
|
| 2104 | const int bytes_not_mapped = PAGE_SIZE * pages_remaining;
|
| 2105 |
|
| 2106 | *length -= bytes_not_mapped;
|
| 2107 | zc->recv_skip_hint += bytes_not_mapped;
|
| 2108 | }
|
| 2109 | return err;
|
| 2110 | }
|
| 2111 |
|
| 2112 | static int tcp_zerocopy_vm_insert_batch(struct vm_area_struct *vma,
|
| 2113 | struct page **pages,
|
| 2114 | unsigned int pages_to_map,
|
| 2115 | unsigned long *address,
|
| 2116 | u32 *length,
|
| 2117 | u32 *seq,
|
| 2118 | struct tcp_zerocopy_receive *zc,
|
| 2119 | u32 total_bytes_to_map)
|
| 2120 | {
|
| 2121 | unsigned long pages_remaining = pages_to_map;
|
| 2122 | unsigned int pages_mapped;
|
| 2123 | unsigned int bytes_mapped;
|
| 2124 | int err;
|
| 2125 |
|
| 2126 | err = vm_insert_pages(vma, *address, pages, &pages_remaining);
|
| 2127 | pages_mapped = pages_to_map - (unsigned int)pages_remaining;
|
| 2128 | bytes_mapped = PAGE_SIZE * pages_mapped;
|
| 2129 | |
| 2130 | * mapping (some but not all of the pages).
|
| 2131 |
|
| 2132 | *seq += bytes_mapped;
|
| 2133 | *address += bytes_mapped;
|
| 2134 |
|
| 2135 | if (likely(!err))
|
| 2136 | return 0;
|
| 2137 |
|
| 2138 |
|
| 2139 | return tcp_zerocopy_vm_insert_batch_error(vma, pages + pages_mapped,
|
| 2140 | pages_remaining, address, length, seq, zc, total_bytes_to_map,
|
| 2141 | err);
|
| 2142 | }
|
| 2143 |
|
| 2144 | #define TCP_VALID_ZC_MSG_FLAGS (TCP_CMSG_TS)
|
| 2145 | static void tcp_zc_finalize_rx_tstamp(struct sock *sk,
|
| 2146 | struct tcp_zerocopy_receive *zc,
|
| 2147 | struct scm_timestamping_internal *tss)
|
| 2148 | {
|
| 2149 | unsigned long msg_control_addr;
|
| 2150 | struct msghdr cmsg_dummy;
|
| 2151 |
|
| 2152 | msg_control_addr = (unsigned long)zc->msg_control;
|
| 2153 | cmsg_dummy.msg_control_user = (void __user *)msg_control_addr;
|
| 2154 | cmsg_dummy.msg_controllen =
|
| 2155 | (__kernel_size_t)zc->msg_controllen;
|
| 2156 | cmsg_dummy.msg_flags = in_compat_syscall()
|
| 2157 | ? MSG_CMSG_COMPAT : 0;
|
| 2158 | cmsg_dummy.msg_control_is_user = true;
|
| 2159 | zc->msg_flags = 0;
|
| 2160 | if (zc->msg_control == msg_control_addr &&
|
| 2161 | zc->msg_controllen == cmsg_dummy.msg_controllen) {
|
| 2162 | tcp_recv_timestamp(&cmsg_dummy, sk, tss);
|
| 2163 | zc->msg_control = (__u64)
|
| 2164 | ((uintptr_t)cmsg_dummy.msg_control_user);
|
| 2165 | zc->msg_controllen =
|
| 2166 | (__u64)cmsg_dummy.msg_controllen;
|
| 2167 | zc->msg_flags = (__u32)cmsg_dummy.msg_flags;
|
| 2168 | }
|
| 2169 | }
|
| 2170 |
|
| 2171 | static struct vm_area_struct *find_tcp_vma(struct mm_struct *mm,
|
| 2172 | unsigned long address,
|
| 2173 | bool *mmap_locked)
|
| 2174 | {
|
| 2175 | struct vm_area_struct *vma = lock_vma_under_rcu(mm, address);
|
| 2176 |
|
| 2177 | if (vma) {
|
| 2178 | if (vma->vm_ops != &tcp_vm_ops) {
|
| 2179 | vma_end_read(vma);
|
| 2180 | return NULL;
|
| 2181 | }
|
| 2182 | *mmap_locked = false;
|
| 2183 | return vma;
|
| 2184 | }
|
| 2185 |
|
| 2186 | mmap_read_lock(mm);
|
| 2187 | vma = vma_lookup(mm, address);
|
| 2188 | if (!vma || vma->vm_ops != &tcp_vm_ops) {
|
| 2189 | mmap_read_unlock(mm);
|
| 2190 | return NULL;
|
| 2191 | }
|
| 2192 | *mmap_locked = true;
|
| 2193 | return vma;
|
| 2194 | }
|
| 2195 |
|
| 2196 | #define TCP_ZEROCOPY_PAGE_BATCH_SIZE 32
|
| 2197 | static int tcp_zerocopy_receive(struct sock *sk,
|
| 2198 | struct tcp_zerocopy_receive *zc,
|
| 2199 | struct scm_timestamping_internal *tss)
|
| 2200 | {
|
| 2201 | u32 length = 0, offset, vma_len, avail_len, copylen = 0;
|
| 2202 | unsigned long address = (unsigned long)zc->address;
|
| 2203 | struct page *pages[TCP_ZEROCOPY_PAGE_BATCH_SIZE];
|
| 2204 | s32 copybuf_len = zc->copybuf_len;
|
| 2205 | struct tcp_sock *tp = tcp_sk(sk);
|
| 2206 | const skb_frag_t *frags = NULL;
|
| 2207 | unsigned int pages_to_map = 0;
|
| 2208 | struct vm_area_struct *vma;
|
| 2209 | struct sk_buff *skb = NULL;
|
| 2210 | u32 seq = tp->copied_seq;
|
| 2211 | u32 total_bytes_to_map;
|
| 2212 | int inq = tcp_inq(sk);
|
| 2213 | bool mmap_locked;
|
| 2214 | int ret;
|
| 2215 |
|
| 2216 | zc->copybuf_len = 0;
|
| 2217 | zc->msg_flags = 0;
|
| 2218 |
|
| 2219 | if (address & (PAGE_SIZE - 1) || address != zc->address)
|
| 2220 | return -EINVAL;
|
| 2221 |
|
| 2222 | if (sk->sk_state == TCP_LISTEN)
|
| 2223 | return -ENOTCONN;
|
| 2224 |
|
| 2225 | sock_rps_record_flow(sk);
|
| 2226 |
|
| 2227 | if (inq && inq <= copybuf_len)
|
| 2228 | return receive_fallback_to_copy(sk, zc, inq, tss);
|
| 2229 |
|
| 2230 | if (inq < PAGE_SIZE) {
|
| 2231 | zc->length = 0;
|
| 2232 | zc->recv_skip_hint = inq;
|
| 2233 | if (!inq && sock_flag(sk, SOCK_DONE))
|
| 2234 | return -EIO;
|
| 2235 | return 0;
|
| 2236 | }
|
| 2237 |
|
| 2238 | vma = find_tcp_vma(current->mm, address, &mmap_locked);
|
| 2239 | if (!vma)
|
| 2240 | return -EINVAL;
|
| 2241 |
|
| 2242 | vma_len = min_t(unsigned long, zc->length, vma->vm_end - address);
|
| 2243 | avail_len = min_t(u32, vma_len, inq);
|
| 2244 | total_bytes_to_map = avail_len & ~(PAGE_SIZE - 1);
|
| 2245 | if (total_bytes_to_map) {
|
| 2246 | if (!(zc->flags & TCP_RECEIVE_ZEROCOPY_FLAG_TLB_CLEAN_HINT))
|
| 2247 | zap_vma_range(vma, address, total_bytes_to_map);
|
| 2248 | zc->length = total_bytes_to_map;
|
| 2249 | zc->recv_skip_hint = 0;
|
| 2250 | } else {
|
| 2251 | zc->length = avail_len;
|
| 2252 | zc->recv_skip_hint = avail_len;
|
| 2253 | }
|
| 2254 | ret = 0;
|
| 2255 | while (length + PAGE_SIZE <= zc->length) {
|
| 2256 | int mappable_offset;
|
| 2257 | struct page *page;
|
| 2258 |
|
| 2259 | if (zc->recv_skip_hint < PAGE_SIZE) {
|
| 2260 | u32 offset_frag;
|
| 2261 |
|
| 2262 | if (skb) {
|
| 2263 | if (zc->recv_skip_hint > 0)
|
| 2264 | break;
|
| 2265 | skb = skb->next;
|
| 2266 | offset = seq - TCP_SKB_CB(skb)->seq;
|
| 2267 | } else {
|
| 2268 | skb = tcp_recv_skb(sk, seq, &offset);
|
| 2269 | }
|
| 2270 |
|
| 2271 | if (!skb_frags_readable(skb))
|
| 2272 | break;
|
| 2273 |
|
| 2274 | if (TCP_SKB_CB(skb)->has_rxtstamp) {
|
| 2275 | tcp_update_recv_tstamps(skb, tss);
|
| 2276 | zc->msg_flags |= TCP_CMSG_TS;
|
| 2277 | }
|
| 2278 | zc->recv_skip_hint = skb->len - offset;
|
| 2279 | frags = skb_advance_to_frag(skb, offset, &offset_frag);
|
| 2280 | if (!frags || offset_frag)
|
| 2281 | break;
|
| 2282 | }
|
| 2283 |
|
| 2284 | mappable_offset = find_next_mappable_frag(frags,
|
| 2285 | zc->recv_skip_hint);
|
| 2286 | if (mappable_offset) {
|
| 2287 | zc->recv_skip_hint = mappable_offset;
|
| 2288 | break;
|
| 2289 | }
|
| 2290 | page = skb_frag_page(frags);
|
| 2291 | if (WARN_ON_ONCE(!page))
|
| 2292 | break;
|
| 2293 |
|
| 2294 | prefetchw(page);
|
| 2295 | pages[pages_to_map++] = page;
|
| 2296 | length += PAGE_SIZE;
|
| 2297 | zc->recv_skip_hint -= PAGE_SIZE;
|
| 2298 | frags++;
|
| 2299 | if (pages_to_map == TCP_ZEROCOPY_PAGE_BATCH_SIZE ||
|
| 2300 | zc->recv_skip_hint < PAGE_SIZE) {
|
| 2301 | |
| 2302 | * (and we cannot unroll failed ops across skbs).
|
| 2303 |
|
| 2304 | ret = tcp_zerocopy_vm_insert_batch(vma, pages,
|
| 2305 | pages_to_map,
|
| 2306 | &address, &length,
|
| 2307 | &seq, zc,
|
| 2308 | total_bytes_to_map);
|
| 2309 | if (ret)
|
| 2310 | goto out;
|
| 2311 | pages_to_map = 0;
|
| 2312 | }
|
| 2313 | }
|
| 2314 | if (pages_to_map) {
|
| 2315 | ret = tcp_zerocopy_vm_insert_batch(vma, pages, pages_to_map,
|
| 2316 | &address, &length, &seq,
|
| 2317 | zc, total_bytes_to_map);
|
| 2318 | }
|
| 2319 | out:
|
| 2320 | if (mmap_locked)
|
| 2321 | mmap_read_unlock(current->mm);
|
| 2322 | else
|
| 2323 | vma_end_read(vma);
|
| 2324 |
|
| 2325 | if (!ret)
|
| 2326 | copylen = tcp_zc_handle_leftover(zc, sk, skb, &seq, copybuf_len, tss);
|
| 2327 |
|
| 2328 | if (length + copylen) {
|
| 2329 | WRITE_ONCE(tp->copied_seq, seq);
|
| 2330 | tcp_rcv_space_adjust(sk);
|
| 2331 |
|
| 2332 |
|
| 2333 | tcp_recv_skb(sk, seq, &offset);
|
| 2334 | tcp_cleanup_rbuf(sk, length + copylen);
|
| 2335 | ret = 0;
|
| 2336 | if (length == zc->length)
|
| 2337 | zc->recv_skip_hint = 0;
|
| 2338 | } else {
|
| 2339 | if (!zc->recv_skip_hint && sock_flag(sk, SOCK_DONE))
|
| 2340 | ret = -EIO;
|
| 2341 | }
|
| 2342 | zc->length = length;
|
| 2343 | return ret;
|
| 2344 | }
|
| 2345 | #endif
|
| 2346 |
|
| 2347 |
|
| 2348 | void tcp_recv_timestamp(struct msghdr *msg, const struct sock *sk,
|
| 2349 | struct scm_timestamping_internal *tss)
|
| 2350 | {
|
| 2351 | int new_tstamp = sock_flag(sk, SOCK_TSTAMP_NEW);
|
| 2352 | u32 tsflags = READ_ONCE(sk->sk_tsflags);
|
| 2353 |
|
| 2354 | if (tss->ts[0]) {
|
| 2355 | if (sock_flag(sk, SOCK_RCVTSTAMP)) {
|
| 2356 | struct timespec64 tv = ktime_to_timespec64(tss->ts[0]);
|
| 2357 |
|
| 2358 | if (sock_flag(sk, SOCK_RCVTSTAMPNS)) {
|
| 2359 | if (new_tstamp) {
|
| 2360 | struct __kernel_timespec kts = {
|
| 2361 | .tv_sec = tv.tv_sec,
|
| 2362 | .tv_nsec = tv.tv_nsec,
|
| 2363 | };
|
| 2364 | put_cmsg(msg, SOL_SOCKET, SO_TIMESTAMPNS_NEW,
|
| 2365 | sizeof(kts), &kts);
|
| 2366 | } else {
|
| 2367 | struct __kernel_old_timespec ts_old = {
|
| 2368 | .tv_sec = tv.tv_sec,
|
| 2369 | .tv_nsec = tv.tv_nsec,
|
| 2370 | };
|
| 2371 | put_cmsg(msg, SOL_SOCKET, SO_TIMESTAMPNS_OLD,
|
| 2372 | sizeof(ts_old), &ts_old);
|
| 2373 | }
|
| 2374 | } else {
|
| 2375 | if (new_tstamp) {
|
| 2376 | struct __kernel_sock_timeval stv = {
|
| 2377 | .tv_sec = tv.tv_sec,
|
| 2378 | .tv_usec = tv.tv_nsec / 1000,
|
| 2379 | };
|
| 2380 | put_cmsg(msg, SOL_SOCKET, SO_TIMESTAMP_NEW,
|
| 2381 | sizeof(stv), &stv);
|
| 2382 | } else {
|
| 2383 | struct __kernel_old_timeval otv = {
|
| 2384 | .tv_sec = tv.tv_sec,
|
| 2385 | .tv_usec = tv.tv_nsec / 1000,
|
| 2386 | };
|
| 2387 | put_cmsg(msg, SOL_SOCKET, SO_TIMESTAMP_OLD,
|
| 2388 | sizeof(otv), &otv);
|
| 2389 | }
|
| 2390 | }
|
| 2391 | }
|
| 2392 |
|
| 2393 | if (!(tsflags & SOF_TIMESTAMPING_SOFTWARE &&
|
| 2394 | (tsflags & SOF_TIMESTAMPING_RX_SOFTWARE ||
|
| 2395 | !(tsflags & SOF_TIMESTAMPING_OPT_RX_FILTER))))
|
| 2396 | tss->ts[0] = 0;
|
| 2397 | }
|
| 2398 |
|
| 2399 | if (tss->ts[2]) {
|
| 2400 | if (!(tsflags & SOF_TIMESTAMPING_RAW_HARDWARE &&
|
| 2401 | (tsflags & SOF_TIMESTAMPING_RX_HARDWARE ||
|
| 2402 | !(tsflags & SOF_TIMESTAMPING_OPT_RX_FILTER))))
|
| 2403 | tss->ts[2] = 0;
|
| 2404 | }
|
| 2405 |
|
| 2406 | if (tss->ts[0] | tss->ts[2]) {
|
| 2407 | tss->ts[1] = 0;
|
| 2408 | if (sock_flag(sk, SOCK_TSTAMP_NEW))
|
| 2409 | put_cmsg_scm_timestamping64(msg, tss);
|
| 2410 | else
|
| 2411 | put_cmsg_scm_timestamping(msg, tss);
|
| 2412 | }
|
| 2413 | }
|
| 2414 |
|
| 2415 | static int tcp_inq_hint(struct sock *sk)
|
| 2416 | {
|
| 2417 | const struct tcp_sock *tp = tcp_sk(sk);
|
| 2418 | u32 copied_seq = READ_ONCE(tp->copied_seq);
|
| 2419 | u32 rcv_nxt = READ_ONCE(tp->rcv_nxt);
|
| 2420 | int inq;
|
| 2421 |
|
| 2422 | inq = rcv_nxt - copied_seq;
|
| 2423 | if (unlikely(inq < 0 || copied_seq != READ_ONCE(tp->copied_seq))) {
|
| 2424 | lock_sock(sk);
|
| 2425 | inq = tp->rcv_nxt - tp->copied_seq;
|
| 2426 | release_sock(sk);
|
| 2427 | }
|
| 2428 | |
| 2429 | * by returning a non-zero inq.
|
| 2430 |
|
| 2431 | if (inq == 0 && sock_flag(sk, SOCK_DONE))
|
| 2432 | inq = 1;
|
| 2433 | return inq;
|
| 2434 | }
|
| 2435 |
|
| 2436 |
|
| 2437 | struct tcp_xa_pool {
|
| 2438 | u8 max;
|
| 2439 | u8 idx;
|
| 2440 | __u32 tokens[MAX_SKB_FRAGS];
|
| 2441 | netmem_ref netmems[MAX_SKB_FRAGS];
|
| 2442 | };
|
| 2443 |
|
| 2444 | static void tcp_xa_pool_commit_locked(struct sock *sk, struct tcp_xa_pool *p)
|
| 2445 | {
|
| 2446 | int i;
|
| 2447 |
|
| 2448 |
|
| 2449 | for (i = 0; i < p->idx; i++)
|
| 2450 | __xa_cmpxchg(&sk->sk_user_frags, p->tokens[i], XA_ZERO_ENTRY,
|
| 2451 | (__force void *)p->netmems[i], GFP_KERNEL);
|
| 2452 |
|
| 2453 | for (; i < p->max; i++)
|
| 2454 | __xa_erase(&sk->sk_user_frags, p->tokens[i]);
|
| 2455 |
|
| 2456 | p->max = 0;
|
| 2457 | p->idx = 0;
|
| 2458 | }
|
| 2459 |
|
| 2460 | static void tcp_xa_pool_commit(struct sock *sk, struct tcp_xa_pool *p)
|
| 2461 | {
|
| 2462 | if (!p->max)
|
| 2463 | return;
|
| 2464 |
|
| 2465 | xa_lock_bh(&sk->sk_user_frags);
|
| 2466 |
|
| 2467 | tcp_xa_pool_commit_locked(sk, p);
|
| 2468 |
|
| 2469 | xa_unlock_bh(&sk->sk_user_frags);
|
| 2470 | }
|
| 2471 |
|
| 2472 | static int tcp_xa_pool_refill(struct sock *sk, struct tcp_xa_pool *p,
|
| 2473 | unsigned int max_frags)
|
| 2474 | {
|
| 2475 | int err, k;
|
| 2476 |
|
| 2477 | if (p->idx < p->max)
|
| 2478 | return 0;
|
| 2479 |
|
| 2480 | xa_lock_bh(&sk->sk_user_frags);
|
| 2481 |
|
| 2482 | tcp_xa_pool_commit_locked(sk, p);
|
| 2483 |
|
| 2484 | for (k = 0; k < max_frags; k++) {
|
| 2485 | err = __xa_alloc(&sk->sk_user_frags, &p->tokens[k],
|
| 2486 | XA_ZERO_ENTRY, xa_limit_31b, GFP_KERNEL);
|
| 2487 | if (err)
|
| 2488 | break;
|
| 2489 | }
|
| 2490 |
|
| 2491 | xa_unlock_bh(&sk->sk_user_frags);
|
| 2492 |
|
| 2493 | p->max = k;
|
| 2494 | p->idx = 0;
|
| 2495 | return k ? 0 : err;
|
| 2496 | }
|
| 2497 |
|
| 2498 | |
| 2499 | * user. May not consume all of @remaining_len.
|
| 2500 |
|
| 2501 | static int tcp_recvmsg_dmabuf(struct sock *sk, const struct sk_buff *skb,
|
| 2502 | unsigned int offset, struct msghdr *msg,
|
| 2503 | int remaining_len)
|
| 2504 | {
|
| 2505 | struct dmabuf_cmsg dmabuf_cmsg = { 0 };
|
| 2506 | struct tcp_xa_pool tcp_xa_pool;
|
| 2507 | unsigned int start;
|
| 2508 | int i, copy, n;
|
| 2509 | int sent = 0;
|
| 2510 | int err = 0;
|
| 2511 |
|
| 2512 | tcp_xa_pool.max = 0;
|
| 2513 | tcp_xa_pool.idx = 0;
|
| 2514 | do {
|
| 2515 | start = skb_headlen(skb);
|
| 2516 |
|
| 2517 | if (skb_frags_readable(skb)) {
|
| 2518 | err = -ENODEV;
|
| 2519 | goto out;
|
| 2520 | }
|
| 2521 |
|
| 2522 |
|
| 2523 | copy = start - offset;
|
| 2524 | if (copy > 0) {
|
| 2525 | copy = min(copy, remaining_len);
|
| 2526 |
|
| 2527 | n = copy_to_iter(skb->data + offset, copy,
|
| 2528 | &msg->msg_iter);
|
| 2529 | if (n != copy) {
|
| 2530 | err = -EFAULT;
|
| 2531 | goto out;
|
| 2532 | }
|
| 2533 |
|
| 2534 | offset += copy;
|
| 2535 | remaining_len -= copy;
|
| 2536 |
|
| 2537 | |
| 2538 | * buffer.
|
| 2539 |
|
| 2540 | memset(&dmabuf_cmsg, 0, sizeof(dmabuf_cmsg));
|
| 2541 | dmabuf_cmsg.frag_size = copy;
|
| 2542 | err = put_cmsg_notrunc(msg, SOL_SOCKET,
|
| 2543 | SO_DEVMEM_LINEAR,
|
| 2544 | sizeof(dmabuf_cmsg),
|
| 2545 | &dmabuf_cmsg);
|
| 2546 | if (err)
|
| 2547 | goto out;
|
| 2548 |
|
| 2549 | sent += copy;
|
| 2550 |
|
| 2551 | if (remaining_len == 0)
|
| 2552 | goto out;
|
| 2553 | }
|
| 2554 |
|
| 2555 | |
| 2556 | * sequence of cmsg
|
| 2557 |
|
| 2558 | for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
|
| 2559 | skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
|
| 2560 | struct net_iov *niov;
|
| 2561 | u64 frag_offset;
|
| 2562 | int end;
|
| 2563 |
|
| 2564 | |
| 2565 | * frags in this skb are dmabuf net_iovs. We're checking
|
| 2566 | * for that flag above, but also check individual frags
|
| 2567 | * here. If the tcp stack is not setting
|
| 2568 | * skb_frags_readable() correctly, we still don't want
|
| 2569 | * to crash here.
|
| 2570 |
|
| 2571 | if (!skb_frag_net_iov(frag)) {
|
| 2572 | net_err_ratelimited("Found non-dmabuf skb with net_iov");
|
| 2573 | err = -ENODEV;
|
| 2574 | goto out;
|
| 2575 | }
|
| 2576 |
|
| 2577 | niov = skb_frag_net_iov(frag);
|
| 2578 | if (!net_is_devmem_iov(niov)) {
|
| 2579 | err = -ENODEV;
|
| 2580 | goto out;
|
| 2581 | }
|
| 2582 |
|
| 2583 | end = start + skb_frag_size(frag);
|
| 2584 | copy = end - offset;
|
| 2585 |
|
| 2586 | if (copy > 0) {
|
| 2587 | copy = min(copy, remaining_len);
|
| 2588 |
|
| 2589 | frag_offset = net_iov_virtual_addr(niov) +
|
| 2590 | skb_frag_off(frag) + offset -
|
| 2591 | start;
|
| 2592 | dmabuf_cmsg.frag_offset = frag_offset;
|
| 2593 | dmabuf_cmsg.frag_size = copy;
|
| 2594 | err = tcp_xa_pool_refill(sk, &tcp_xa_pool,
|
| 2595 | skb_shinfo(skb)->nr_frags - i);
|
| 2596 | if (err)
|
| 2597 | goto out;
|
| 2598 |
|
| 2599 |
|
| 2600 | dmabuf_cmsg.frag_token = tcp_xa_pool.tokens[tcp_xa_pool.idx];
|
| 2601 | dmabuf_cmsg.dmabuf_id = net_devmem_iov_binding_id(niov);
|
| 2602 |
|
| 2603 | offset += copy;
|
| 2604 | remaining_len -= copy;
|
| 2605 |
|
| 2606 | err = put_cmsg_notrunc(msg, SOL_SOCKET,
|
| 2607 | SO_DEVMEM_DMABUF,
|
| 2608 | sizeof(dmabuf_cmsg),
|
| 2609 | &dmabuf_cmsg);
|
| 2610 | if (err)
|
| 2611 | goto out;
|
| 2612 |
|
| 2613 | atomic_long_inc(&niov->desc.pp_ref_count);
|
| 2614 | tcp_xa_pool.netmems[tcp_xa_pool.idx++] = skb_frag_netmem(frag);
|
| 2615 |
|
| 2616 | sent += copy;
|
| 2617 |
|
| 2618 | if (remaining_len == 0)
|
| 2619 | goto out;
|
| 2620 | }
|
| 2621 | start = end;
|
| 2622 | }
|
| 2623 |
|
| 2624 | tcp_xa_pool_commit(sk, &tcp_xa_pool);
|
| 2625 | if (!remaining_len)
|
| 2626 | goto out;
|
| 2627 |
|
| 2628 | |
| 2629 | * next frag in the frag_list to satisfy remaining_len.
|
| 2630 |
|
| 2631 | skb = skb_shinfo(skb)->frag_list ?: skb->next;
|
| 2632 |
|
| 2633 | offset = offset - start;
|
| 2634 | } while (skb);
|
| 2635 |
|
| 2636 | if (remaining_len) {
|
| 2637 | err = -EFAULT;
|
| 2638 | goto out;
|
| 2639 | }
|
| 2640 |
|
| 2641 | out:
|
| 2642 | tcp_xa_pool_commit(sk, &tcp_xa_pool);
|
| 2643 | if (!sent)
|
| 2644 | sent = err;
|
| 2645 |
|
| 2646 | return sent;
|
| 2647 | }
|
| 2648 |
|
| 2649 | |
| 2650 | * This routine copies from a sock struct into the user buffer.
|
| 2651 | *
|
| 2652 | * Technical note: in 2.3 we work on _locked_ socket, so that
|
| 2653 | * tricks with *seq access order and skb->users are not required.
|
| 2654 | * Probably, code can be easily improved even more.
|
| 2655 |
|
| 2656 |
|
| 2657 | static int tcp_recvmsg_locked(struct sock *sk, struct msghdr *msg, size_t len,
|
| 2658 | int flags, struct scm_timestamping_internal *tss,
|
| 2659 | int *cmsg_flags)
|
| 2660 | {
|
| 2661 | struct tcp_sock *tp = tcp_sk(sk);
|
| 2662 | int last_copied_dmabuf = -1;
|
| 2663 | int copied = 0;
|
| 2664 | u32 peek_seq;
|
| 2665 | u32 *seq;
|
| 2666 | unsigned long used;
|
| 2667 | int err;
|
| 2668 | int target;
|
| 2669 | long timeo;
|
| 2670 | struct sk_buff *skb, *last;
|
| 2671 | u32 peek_offset = 0;
|
| 2672 | u32 urg_hole = 0;
|
| 2673 |
|
| 2674 | err = -ENOTCONN;
|
| 2675 | if (sk->sk_state == TCP_LISTEN)
|
| 2676 | goto out;
|
| 2677 |
|
| 2678 | if (tp->recvmsg_inq)
|
| 2679 | *cmsg_flags = TCP_CMSG_INQ;
|
| 2680 | timeo = sock_rcvtimeo(sk, flags & MSG_DONTWAIT);
|
| 2681 |
|
| 2682 |
|
| 2683 | if (flags & MSG_OOB)
|
| 2684 | goto recv_urg;
|
| 2685 |
|
| 2686 | if (unlikely(tp->repair)) {
|
| 2687 | err = -EPERM;
|
| 2688 | if (!(flags & MSG_PEEK))
|
| 2689 | goto out;
|
| 2690 |
|
| 2691 | if (tp->repair_queue == TCP_SEND_QUEUE)
|
| 2692 | goto recv_sndq;
|
| 2693 |
|
| 2694 | err = -EINVAL;
|
| 2695 | if (tp->repair_queue == TCP_NO_QUEUE)
|
| 2696 | goto out;
|
| 2697 |
|
| 2698 |
|
| 2699 | }
|
| 2700 |
|
| 2701 | seq = &tp->copied_seq;
|
| 2702 | if (flags & MSG_PEEK) {
|
| 2703 | peek_offset = max(sk_peek_offset(sk, flags), 0);
|
| 2704 | peek_seq = tp->copied_seq + peek_offset;
|
| 2705 | seq = &peek_seq;
|
| 2706 | }
|
| 2707 |
|
| 2708 | target = sock_rcvlowat(sk, flags & MSG_WAITALL, len);
|
| 2709 |
|
| 2710 | do {
|
| 2711 | u32 offset;
|
| 2712 |
|
| 2713 |
|
| 2714 | if (unlikely(tp->urg_data) && tp->urg_seq == *seq) {
|
| 2715 | if (copied)
|
| 2716 | break;
|
| 2717 | if (signal_pending(current)) {
|
| 2718 | copied = timeo ? sock_intr_errno(timeo) : -EAGAIN;
|
| 2719 | break;
|
| 2720 | }
|
| 2721 | }
|
| 2722 |
|
| 2723 |
|
| 2724 |
|
| 2725 | last = skb_peek_tail(&sk->sk_receive_queue);
|
| 2726 | skb_queue_walk(&sk->sk_receive_queue, skb) {
|
| 2727 | last = skb;
|
| 2728 | |
| 2729 | * shouldn't happen.
|
| 2730 |
|
| 2731 | if (WARN(before(*seq, TCP_SKB_CB(skb)->seq),
|
| 2732 | "TCP recvmsg seq # bug: copied %X, seq %X, rcvnxt %X, fl %X\n",
|
| 2733 | *seq, TCP_SKB_CB(skb)->seq, tp->rcv_nxt,
|
| 2734 | flags))
|
| 2735 | break;
|
| 2736 |
|
| 2737 | offset = *seq - TCP_SKB_CB(skb)->seq;
|
| 2738 | if (unlikely(TCP_SKB_CB(skb)->tcp_flags & TCPHDR_SYN)) {
|
| 2739 | pr_err_once("%s: found a SYN, please report !\n", __func__);
|
| 2740 | offset--;
|
| 2741 | }
|
| 2742 | if (offset < skb->len)
|
| 2743 | goto found_ok_skb;
|
| 2744 | if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN)
|
| 2745 | goto found_fin_ok;
|
| 2746 | WARN(!(flags & MSG_PEEK),
|
| 2747 | "TCP recvmsg seq # bug 2: copied %X, seq %X, rcvnxt %X, fl %X\n",
|
| 2748 | *seq, TCP_SKB_CB(skb)->seq, tp->rcv_nxt, flags);
|
| 2749 | }
|
| 2750 |
|
| 2751 |
|
| 2752 |
|
| 2753 | if (copied >= target && !READ_ONCE(sk->sk_backlog.tail))
|
| 2754 | break;
|
| 2755 |
|
| 2756 | if (copied) {
|
| 2757 | if (!timeo ||
|
| 2758 | tcp_recv_should_stop(sk))
|
| 2759 | break;
|
| 2760 | } else {
|
| 2761 | if (sock_flag(sk, SOCK_DONE))
|
| 2762 | break;
|
| 2763 |
|
| 2764 | if (sk->sk_err) {
|
| 2765 | copied = sock_error(sk);
|
| 2766 | break;
|
| 2767 | }
|
| 2768 |
|
| 2769 | if (sk->sk_shutdown & RCV_SHUTDOWN)
|
| 2770 | break;
|
| 2771 |
|
| 2772 | if (sk->sk_state == TCP_CLOSE) {
|
| 2773 | |
| 2774 | * from never connected socket.
|
| 2775 |
|
| 2776 | copied = -ENOTCONN;
|
| 2777 | break;
|
| 2778 | }
|
| 2779 |
|
| 2780 | if (!timeo) {
|
| 2781 | copied = -EAGAIN;
|
| 2782 | break;
|
| 2783 | }
|
| 2784 |
|
| 2785 | if (signal_pending(current)) {
|
| 2786 | copied = sock_intr_errno(timeo);
|
| 2787 | break;
|
| 2788 | }
|
| 2789 | }
|
| 2790 |
|
| 2791 | if (copied >= target) {
|
| 2792 |
|
| 2793 | __sk_flush_backlog(sk);
|
| 2794 | } else {
|
| 2795 | tcp_cleanup_rbuf(sk, copied);
|
| 2796 | err = sk_wait_data(sk, &timeo, last);
|
| 2797 | if (err < 0) {
|
| 2798 | err = copied ? : err;
|
| 2799 | goto out;
|
| 2800 | }
|
| 2801 | }
|
| 2802 |
|
| 2803 | if ((flags & MSG_PEEK) &&
|
| 2804 | (peek_seq - peek_offset - copied - urg_hole != tp->copied_seq)) {
|
| 2805 | net_dbg_ratelimited("TCP(%s:%d): Application bug, race in MSG_PEEK\n",
|
| 2806 | current->comm,
|
| 2807 | task_pid_nr(current));
|
| 2808 | peek_seq = tp->copied_seq + peek_offset;
|
| 2809 | }
|
| 2810 | continue;
|
| 2811 |
|
| 2812 | found_ok_skb:
|
| 2813 |
|
| 2814 | used = skb->len - offset;
|
| 2815 | if (len < used)
|
| 2816 | used = len;
|
| 2817 |
|
| 2818 |
|
| 2819 | if (unlikely(tp->urg_data)) {
|
| 2820 | u32 urg_offset = tp->urg_seq - *seq;
|
| 2821 | if (urg_offset < used) {
|
| 2822 | if (!urg_offset) {
|
| 2823 | if (!sock_flag(sk, SOCK_URGINLINE)) {
|
| 2824 | WRITE_ONCE(*seq, *seq + 1);
|
| 2825 | urg_hole++;
|
| 2826 | offset++;
|
| 2827 | used--;
|
| 2828 | if (!used)
|
| 2829 | goto skip_copy;
|
| 2830 | }
|
| 2831 | } else
|
| 2832 | used = urg_offset;
|
| 2833 | }
|
| 2834 | }
|
| 2835 |
|
| 2836 | if (!(flags & MSG_TRUNC)) {
|
| 2837 | if (last_copied_dmabuf != -1 &&
|
| 2838 | last_copied_dmabuf != !skb_frags_readable(skb))
|
| 2839 | break;
|
| 2840 |
|
| 2841 | if (skb_frags_readable(skb)) {
|
| 2842 | err = skb_copy_datagram_msg(skb, offset, msg,
|
| 2843 | used);
|
| 2844 | if (err) {
|
| 2845 |
|
| 2846 | if (!copied)
|
| 2847 | copied = -EFAULT;
|
| 2848 | break;
|
| 2849 | }
|
| 2850 | } else {
|
| 2851 | if (!(flags & MSG_SOCK_DEVMEM)) {
|
| 2852 | |
| 2853 | * with the MSG_SOCK_DEVMEM flag.
|
| 2854 |
|
| 2855 | if (!copied)
|
| 2856 | copied = -EFAULT;
|
| 2857 |
|
| 2858 | break;
|
| 2859 | }
|
| 2860 |
|
| 2861 | err = tcp_recvmsg_dmabuf(sk, skb, offset, msg,
|
| 2862 | used);
|
| 2863 | if (err < 0) {
|
| 2864 | if (!copied)
|
| 2865 | copied = err;
|
| 2866 |
|
| 2867 | break;
|
| 2868 | }
|
| 2869 | used = err;
|
| 2870 | }
|
| 2871 | }
|
| 2872 |
|
| 2873 | last_copied_dmabuf = !skb_frags_readable(skb);
|
| 2874 |
|
| 2875 | WRITE_ONCE(*seq, *seq + used);
|
| 2876 | copied += used;
|
| 2877 | len -= used;
|
| 2878 | if (flags & MSG_PEEK)
|
| 2879 | sk_peek_offset_fwd(sk, used);
|
| 2880 | else
|
| 2881 | sk_peek_offset_bwd(sk, used);
|
| 2882 | tcp_rcv_space_adjust(sk);
|
| 2883 |
|
| 2884 | skip_copy:
|
| 2885 | if (unlikely(tp->urg_data) && after(tp->copied_seq, tp->urg_seq)) {
|
| 2886 | WRITE_ONCE(tp->urg_data, 0);
|
| 2887 | tcp_fast_path_check(sk);
|
| 2888 | }
|
| 2889 |
|
| 2890 | if (TCP_SKB_CB(skb)->has_rxtstamp) {
|
| 2891 | tcp_update_recv_tstamps(skb, tss);
|
| 2892 | *cmsg_flags |= TCP_CMSG_TS;
|
| 2893 | }
|
| 2894 |
|
| 2895 | if (used + offset < skb->len)
|
| 2896 | continue;
|
| 2897 |
|
| 2898 | if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN)
|
| 2899 | goto found_fin_ok;
|
| 2900 | if (!(flags & MSG_PEEK))
|
| 2901 | tcp_eat_recv_skb(sk, skb);
|
| 2902 | continue;
|
| 2903 |
|
| 2904 | found_fin_ok:
|
| 2905 |
|
| 2906 | WRITE_ONCE(*seq, *seq + 1);
|
| 2907 | if (!(flags & MSG_PEEK))
|
| 2908 | tcp_eat_recv_skb(sk, skb);
|
| 2909 | break;
|
| 2910 | } while (len > 0);
|
| 2911 |
|
| 2912 | |
| 2913 | * on connected socket. I was just happy when found this 8) --ANK
|
| 2914 |
|
| 2915 |
|
| 2916 |
|
| 2917 | tcp_cleanup_rbuf(sk, copied);
|
| 2918 | return copied;
|
| 2919 |
|
| 2920 | out:
|
| 2921 | return err;
|
| 2922 |
|
| 2923 | recv_urg:
|
| 2924 | err = tcp_recv_urg(sk, msg, len, flags);
|
| 2925 | goto out;
|
| 2926 |
|
| 2927 | recv_sndq:
|
| 2928 | err = tcp_peek_sndq(sk, msg, len);
|
| 2929 | goto out;
|
| 2930 | }
|
| 2931 |
|
| 2932 | int tcp_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, int flags)
|
| 2933 | {
|
| 2934 | int cmsg_flags = 0, ret;
|
| 2935 | struct scm_timestamping_internal tss;
|
| 2936 |
|
| 2937 | if (unlikely(flags & MSG_ERRQUEUE))
|
| 2938 | return inet_recv_error(sk, msg, len);
|
| 2939 |
|
| 2940 | if (sk_can_busy_loop(sk) &&
|
| 2941 | skb_queue_empty_lockless(&sk->sk_receive_queue) &&
|
| 2942 | sk->sk_state == TCP_ESTABLISHED)
|
| 2943 | sk_busy_loop(sk, flags & MSG_DONTWAIT);
|
| 2944 |
|
| 2945 | lock_sock(sk);
|
| 2946 | ret = tcp_recvmsg_locked(sk, msg, len, flags, &tss, &cmsg_flags);
|
| 2947 | release_sock(sk);
|
| 2948 |
|
| 2949 | if ((cmsg_flags | msg->msg_get_inq) && ret >= 0) {
|
| 2950 | if (cmsg_flags & TCP_CMSG_TS)
|
| 2951 | tcp_recv_timestamp(msg, sk, &tss);
|
| 2952 | if ((cmsg_flags & TCP_CMSG_INQ) | msg->msg_get_inq) {
|
| 2953 | msg->msg_inq = tcp_inq_hint(sk);
|
| 2954 | if (cmsg_flags & TCP_CMSG_INQ)
|
| 2955 | put_cmsg(msg, SOL_TCP, TCP_CM_INQ,
|
| 2956 | sizeof(msg->msg_inq), &msg->msg_inq);
|
| 2957 | }
|
| 2958 | }
|
| 2959 | return ret;
|
| 2960 | }
|
| 2961 |
|
| 2962 | void tcp_set_state(struct sock *sk, int state)
|
| 2963 | {
|
| 2964 | int oldstate = sk->sk_state;
|
| 2965 |
|
| 2966 | |
| 2967 | * so as not force the internal TCP states to be frozen. The
|
| 2968 | * following checks will detect if an internal state value ever
|
| 2969 | * differs from the BPF value. If this ever happens, then we will
|
| 2970 | * need to remap the internal value to the BPF value before calling
|
| 2971 | * tcp_call_bpf_2arg.
|
| 2972 |
|
| 2973 | BUILD_BUG_ON((int)BPF_TCP_ESTABLISHED != (int)TCP_ESTABLISHED);
|
| 2974 | BUILD_BUG_ON((int)BPF_TCP_SYN_SENT != (int)TCP_SYN_SENT);
|
| 2975 | BUILD_BUG_ON((int)BPF_TCP_SYN_RECV != (int)TCP_SYN_RECV);
|
| 2976 | BUILD_BUG_ON((int)BPF_TCP_FIN_WAIT1 != (int)TCP_FIN_WAIT1);
|
| 2977 | BUILD_BUG_ON((int)BPF_TCP_FIN_WAIT2 != (int)TCP_FIN_WAIT2);
|
| 2978 | BUILD_BUG_ON((int)BPF_TCP_TIME_WAIT != (int)TCP_TIME_WAIT);
|
| 2979 | BUILD_BUG_ON((int)BPF_TCP_CLOSE != (int)TCP_CLOSE);
|
| 2980 | BUILD_BUG_ON((int)BPF_TCP_CLOSE_WAIT != (int)TCP_CLOSE_WAIT);
|
| 2981 | BUILD_BUG_ON((int)BPF_TCP_LAST_ACK != (int)TCP_LAST_ACK);
|
| 2982 | BUILD_BUG_ON((int)BPF_TCP_LISTEN != (int)TCP_LISTEN);
|
| 2983 | BUILD_BUG_ON((int)BPF_TCP_CLOSING != (int)TCP_CLOSING);
|
| 2984 | BUILD_BUG_ON((int)BPF_TCP_NEW_SYN_RECV != (int)TCP_NEW_SYN_RECV);
|
| 2985 | BUILD_BUG_ON((int)BPF_TCP_BOUND_INACTIVE != (int)TCP_BOUND_INACTIVE);
|
| 2986 | BUILD_BUG_ON((int)BPF_TCP_MAX_STATES != (int)TCP_MAX_STATES);
|
| 2987 |
|
| 2988 | |
| 2989 | * BPF_TCP_* used by bpf programs. Currently gcc built vmlinux
|
| 2990 | * is able to emit this enum in DWARF due to the above BUILD_BUG_ON.
|
| 2991 | * But clang built vmlinux does not have this enum in DWARF
|
| 2992 | * since clang removes the above code before generating IR/debuginfo.
|
| 2993 | * Let us explicitly emit the type debuginfo to ensure the
|
| 2994 | * above-mentioned anonymous enum in the vmlinux DWARF and hence BTF
|
| 2995 | * regardless of which compiler is used.
|
| 2996 |
|
| 2997 | BTF_TYPE_EMIT_ENUM(BPF_TCP_ESTABLISHED);
|
| 2998 |
|
| 2999 | if (BPF_SOCK_OPS_TEST_FLAG(tcp_sk(sk), BPF_SOCK_OPS_STATE_CB_FLAG))
|
| 3000 | tcp_call_bpf_2arg(sk, BPF_SOCK_OPS_STATE_CB, oldstate, state);
|
| 3001 |
|
| 3002 | switch (state) {
|
| 3003 | case TCP_ESTABLISHED:
|
| 3004 | if (oldstate != TCP_ESTABLISHED)
|
| 3005 | TCP_INC_STATS(sock_net(sk), TCP_MIB_CURRESTAB);
|
| 3006 | break;
|
| 3007 | case TCP_CLOSE_WAIT:
|
| 3008 | if (oldstate == TCP_SYN_RECV)
|
| 3009 | TCP_INC_STATS(sock_net(sk), TCP_MIB_CURRESTAB);
|
| 3010 | break;
|
| 3011 |
|
| 3012 | case TCP_CLOSE:
|
| 3013 | if (oldstate == TCP_CLOSE_WAIT || oldstate == TCP_ESTABLISHED)
|
| 3014 | TCP_INC_STATS(sock_net(sk), TCP_MIB_ESTABRESETS);
|
| 3015 |
|
| 3016 | sk->sk_prot->unhash(sk);
|
| 3017 | if (inet_csk(sk)->icsk_bind_hash &&
|
| 3018 | !(sk->sk_userlocks & SOCK_BINDPORT_LOCK))
|
| 3019 | inet_put_port(sk);
|
| 3020 | fallthrough;
|
| 3021 | default:
|
| 3022 | if (oldstate == TCP_ESTABLISHED || oldstate == TCP_CLOSE_WAIT)
|
| 3023 | TCP_DEC_STATS(sock_net(sk), TCP_MIB_CURRESTAB);
|
| 3024 | }
|
| 3025 |
|
| 3026 | |
| 3027 | * socket sitting in hash tables.
|
| 3028 |
|
| 3029 | inet_sk_state_store(sk, state);
|
| 3030 | }
|
| 3031 |
|
| 3032 | |
| 3033 | * State processing on a close. This implements the state shift for
|
| 3034 | * sending our FIN frame. Note that we only send a FIN for some
|
| 3035 | * states. A shutdown() may have already sent the FIN, or we may be
|
| 3036 | * closed.
|
| 3037 |
|
| 3038 |
|
| 3039 | static const unsigned char new_state[16] = {
|
| 3040 |
|
| 3041 | [0 ] = TCP_CLOSE,
|
| 3042 | [TCP_ESTABLISHED] = TCP_FIN_WAIT1 | TCP_ACTION_FIN,
|
| 3043 | [TCP_SYN_SENT] = TCP_CLOSE,
|
| 3044 | [TCP_SYN_RECV] = TCP_FIN_WAIT1 | TCP_ACTION_FIN,
|
| 3045 | [TCP_FIN_WAIT1] = TCP_FIN_WAIT1,
|
| 3046 | [TCP_FIN_WAIT2] = TCP_FIN_WAIT2,
|
| 3047 | [TCP_TIME_WAIT] = TCP_CLOSE,
|
| 3048 | [TCP_CLOSE] = TCP_CLOSE,
|
| 3049 | [TCP_CLOSE_WAIT] = TCP_LAST_ACK | TCP_ACTION_FIN,
|
| 3050 | [TCP_LAST_ACK] = TCP_LAST_ACK,
|
| 3051 | [TCP_LISTEN] = TCP_CLOSE,
|
| 3052 | [TCP_CLOSING] = TCP_CLOSING,
|
| 3053 | [TCP_NEW_SYN_RECV] = TCP_CLOSE,
|
| 3054 | };
|
| 3055 |
|
| 3056 | static int tcp_close_state(struct sock *sk)
|
| 3057 | {
|
| 3058 | int next = (int)new_state[sk->sk_state];
|
| 3059 | int ns = next & TCP_STATE_MASK;
|
| 3060 |
|
| 3061 | tcp_set_state(sk, ns);
|
| 3062 |
|
| 3063 | return next & TCP_ACTION_FIN;
|
| 3064 | }
|
| 3065 |
|
| 3066 | |
| 3067 | * Shutdown the sending side of a connection. Much like close except
|
| 3068 | * that we don't receive shut down or sock_set_flag(sk, SOCK_DEAD).
|
| 3069 |
|
| 3070 |
|
| 3071 | void tcp_shutdown(struct sock *sk, int how)
|
| 3072 | {
|
| 3073 | |
| 3074 | * and then put it into the queue to be sent.
|
| 3075 | * Tim MacKenzie([email protected]) 4 Dec '92.
|
| 3076 |
|
| 3077 | if (!(how & SEND_SHUTDOWN))
|
| 3078 | return;
|
| 3079 |
|
| 3080 |
|
| 3081 | if ((1 << sk->sk_state) &
|
| 3082 | (TCPF_ESTABLISHED | TCPF_SYN_SENT |
|
| 3083 | TCPF_CLOSE_WAIT)) {
|
| 3084 |
|
| 3085 | if (tcp_close_state(sk))
|
| 3086 | tcp_send_fin(sk);
|
| 3087 | }
|
| 3088 | }
|
| 3089 |
|
| 3090 | int tcp_orphan_count_sum(void)
|
| 3091 | {
|
| 3092 | int i, total = 0;
|
| 3093 |
|
| 3094 | for_each_possible_cpu(i)
|
| 3095 | total += per_cpu(tcp_orphan_count, i);
|
| 3096 |
|
| 3097 | return max(total, 0);
|
| 3098 | }
|
| 3099 |
|
| 3100 | static int tcp_orphan_cache;
|
| 3101 | static struct timer_list tcp_orphan_timer;
|
| 3102 | #define TCP_ORPHAN_TIMER_PERIOD msecs_to_jiffies(100)
|
| 3103 |
|
| 3104 | static void tcp_orphan_update(struct timer_list *unused)
|
| 3105 | {
|
| 3106 | WRITE_ONCE(tcp_orphan_cache, tcp_orphan_count_sum());
|
| 3107 | mod_timer(&tcp_orphan_timer, jiffies + TCP_ORPHAN_TIMER_PERIOD);
|
| 3108 | }
|
| 3109 |
|
| 3110 | static bool tcp_too_many_orphans(int shift)
|
| 3111 | {
|
| 3112 | return READ_ONCE(tcp_orphan_cache) << shift >
|
| 3113 | READ_ONCE(sysctl_tcp_max_orphans);
|
| 3114 | }
|
| 3115 |
|
| 3116 | static bool tcp_out_of_memory(const struct sock *sk)
|
| 3117 | {
|
| 3118 | if (sk->sk_wmem_queued > SOCK_MIN_SNDBUF &&
|
| 3119 | sk_memory_allocated(sk) > sk_prot_mem_limits(sk, 2))
|
| 3120 | return true;
|
| 3121 | return false;
|
| 3122 | }
|
| 3123 |
|
| 3124 | bool tcp_check_oom(const struct sock *sk, int shift)
|
| 3125 | {
|
| 3126 | bool too_many_orphans, out_of_socket_memory;
|
| 3127 |
|
| 3128 | too_many_orphans = tcp_too_many_orphans(shift);
|
| 3129 | out_of_socket_memory = tcp_out_of_memory(sk);
|
| 3130 |
|
| 3131 | if (too_many_orphans)
|
| 3132 | net_info_ratelimited("too many orphaned sockets\n");
|
| 3133 | if (out_of_socket_memory)
|
| 3134 | net_info_ratelimited("out of memory -- consider tuning tcp_mem\n");
|
| 3135 | return too_many_orphans || out_of_socket_memory;
|
| 3136 | }
|
| 3137 |
|
| 3138 | void __tcp_close(struct sock *sk, long timeout)
|
| 3139 | {
|
| 3140 | bool data_was_unread = false;
|
| 3141 | struct sk_buff *skb;
|
| 3142 | int state;
|
| 3143 |
|
| 3144 | WRITE_ONCE(sk->sk_shutdown, SHUTDOWN_MASK);
|
| 3145 |
|
| 3146 | if (sk->sk_state == TCP_LISTEN) {
|
| 3147 | tcp_set_state(sk, TCP_CLOSE);
|
| 3148 |
|
| 3149 |
|
| 3150 | inet_csk_listen_stop(sk);
|
| 3151 |
|
| 3152 | goto adjudge_to_death;
|
| 3153 | }
|
| 3154 |
|
| 3155 | |
| 3156 | * descriptor close, not protocol-sourced closes, because the
|
| 3157 | * reader process may not have drained the data yet!
|
| 3158 |
|
| 3159 | while ((skb = skb_peek(&sk->sk_receive_queue)) != NULL) {
|
| 3160 | u32 end_seq = TCP_SKB_CB(skb)->end_seq;
|
| 3161 |
|
| 3162 | if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN)
|
| 3163 | end_seq--;
|
| 3164 | if (after(end_seq, tcp_sk(sk)->copied_seq))
|
| 3165 | data_was_unread = true;
|
| 3166 | tcp_eat_recv_skb(sk, skb);
|
| 3167 | }
|
| 3168 |
|
| 3169 |
|
| 3170 | if (sk->sk_state == TCP_CLOSE)
|
| 3171 | goto adjudge_to_death;
|
| 3172 |
|
| 3173 | |
| 3174 | * data was lost. To witness the awful effects of the old behavior of
|
| 3175 | * always doing a FIN, run an older 2.1.x kernel or 2.0.x, start a bulk
|
| 3176 | * GET in an FTP client, suspend the process, wait for the client to
|
| 3177 | * advertise a zero window, then kill -9 the FTP client, wheee...
|
| 3178 | * Note: timeout is always zero in such a case.
|
| 3179 |
|
| 3180 | if (unlikely(tcp_sk(sk)->repair)) {
|
| 3181 | sk->sk_prot->disconnect(sk, 0);
|
| 3182 | } else if (data_was_unread) {
|
| 3183 |
|
| 3184 | NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPABORTONCLOSE);
|
| 3185 | tcp_set_state(sk, TCP_CLOSE);
|
| 3186 | tcp_send_active_reset(sk, SK_RST_REASON_TCP_ABORT_ON_CLOSE);
|
| 3187 | } else if (sock_flag(sk, SOCK_LINGER) && !sk->sk_lingertime) {
|
| 3188 |
|
| 3189 | sk->sk_prot->disconnect(sk, 0);
|
| 3190 | NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPABORTONDATA);
|
| 3191 | } else if (tcp_close_state(sk)) {
|
| 3192 | |
| 3193 | * zapping the connection.
|
| 3194 |
|
| 3195 |
|
| 3196 | |
| 3197 | * machine. State transitions:
|
| 3198 | *
|
| 3199 | * TCP_ESTABLISHED -> TCP_FIN_WAIT1
|
| 3200 | * TCP_SYN_RECV -> TCP_FIN_WAIT1 (it is difficult)
|
| 3201 | * TCP_CLOSE_WAIT -> TCP_LAST_ACK
|
| 3202 | *
|
| 3203 | * are legal only when FIN has been sent (i.e. in window),
|
| 3204 | * rather than queued out of window. Purists blame.
|
| 3205 | *
|
| 3206 | * F.e. "RFC state" is ESTABLISHED,
|
| 3207 | * if Linux state is FIN-WAIT-1, but FIN is still not sent.
|
| 3208 | *
|
| 3209 | * The visible declinations are that sometimes
|
| 3210 | * we enter time-wait state, when it is not required really
|
| 3211 | * (harmless), do not send active resets, when they are
|
| 3212 | * required by specs (TCP_ESTABLISHED, TCP_CLOSE_WAIT, when
|
| 3213 | * they look as CLOSING or LAST_ACK for Linux)
|
| 3214 | * Probably, I missed some more holelets.
|
| 3215 | * --ANK
|
| 3216 | * XXX (TFO) - To start off we don't support SYN+ACK+FIN
|
| 3217 | * in a single packet! (May consider it later but will
|
| 3218 | * probably need API support or TCP_CORK SYN-ACK until
|
| 3219 | * data is written and socket is closed.)
|
| 3220 |
|
| 3221 | tcp_send_fin(sk);
|
| 3222 | }
|
| 3223 |
|
| 3224 | sk_stream_wait_close(sk, timeout);
|
| 3225 |
|
| 3226 | adjudge_to_death:
|
| 3227 | state = sk->sk_state;
|
| 3228 | sock_hold(sk);
|
| 3229 | sock_orphan(sk);
|
| 3230 |
|
| 3231 | local_bh_disable();
|
| 3232 | bh_lock_sock(sk);
|
| 3233 |
|
| 3234 | __release_sock(sk);
|
| 3235 |
|
| 3236 | tcp_orphan_count_inc();
|
| 3237 |
|
| 3238 |
|
| 3239 | if (state != TCP_CLOSE && sk->sk_state == TCP_CLOSE)
|
| 3240 | goto out;
|
| 3241 |
|
| 3242 | |
| 3243 | * problem with TCP as specified in that the other end could
|
| 3244 | * keep a socket open forever with no application left this end.
|
| 3245 | * We use a 1 minute timeout (about the same as BSD) then kill
|
| 3246 | * our end. If they send after that then tough - BUT: long enough
|
| 3247 | * that we won't make the old 4*rto = almost no time - whoops
|
| 3248 | * reset mistake.
|
| 3249 | *
|
| 3250 | * Nope, it was not mistake. It is really desired behaviour
|
| 3251 | * f.e. on http servers, when such sockets are useless, but
|
| 3252 | * consume significant resources. Let's do it with special
|
| 3253 | * linger2 option. --ANK
|
| 3254 |
|
| 3255 |
|
| 3256 | if (sk->sk_state == TCP_FIN_WAIT2) {
|
| 3257 | struct tcp_sock *tp = tcp_sk(sk);
|
| 3258 | if (READ_ONCE(tp->linger2) < 0) {
|
| 3259 | tcp_set_state(sk, TCP_CLOSE);
|
| 3260 | tcp_send_active_reset(sk,
|
| 3261 | SK_RST_REASON_TCP_ABORT_ON_LINGER);
|
| 3262 | __NET_INC_STATS(sock_net(sk),
|
| 3263 | LINUX_MIB_TCPABORTONLINGER);
|
| 3264 | } else {
|
| 3265 | const int tmo = tcp_fin_time(sk);
|
| 3266 |
|
| 3267 | if (tmo > TCP_TIMEWAIT_LEN) {
|
| 3268 | tcp_reset_keepalive_timer(sk,
|
| 3269 | tmo - TCP_TIMEWAIT_LEN);
|
| 3270 | } else {
|
| 3271 | tcp_time_wait(sk, TCP_FIN_WAIT2, tmo);
|
| 3272 | goto out;
|
| 3273 | }
|
| 3274 | }
|
| 3275 | }
|
| 3276 | if (sk->sk_state != TCP_CLOSE) {
|
| 3277 | if (tcp_check_oom(sk, 0)) {
|
| 3278 | tcp_set_state(sk, TCP_CLOSE);
|
| 3279 | tcp_send_active_reset(sk,
|
| 3280 | SK_RST_REASON_TCP_ABORT_ON_MEMORY);
|
| 3281 | __NET_INC_STATS(sock_net(sk),
|
| 3282 | LINUX_MIB_TCPABORTONMEMORY);
|
| 3283 | } else if (!check_net(sock_net(sk))) {
|
| 3284 |
|
| 3285 | tcp_set_state(sk, TCP_CLOSE);
|
| 3286 | }
|
| 3287 | }
|
| 3288 |
|
| 3289 | if (sk->sk_state == TCP_CLOSE) {
|
| 3290 | struct request_sock *req;
|
| 3291 |
|
| 3292 | req = rcu_dereference_protected(tcp_sk(sk)->fastopen_rsk,
|
| 3293 | lockdep_sock_is_held(sk));
|
| 3294 | |
| 3295 | * aborted (e.g., closed with unread data) before 3WHS
|
| 3296 | * finishes.
|
| 3297 |
|
| 3298 | if (req)
|
| 3299 | reqsk_fastopen_remove(sk, req, false);
|
| 3300 | inet_csk_destroy_sock(sk);
|
| 3301 | }
|
| 3302 |
|
| 3303 |
|
| 3304 | out:
|
| 3305 | bh_unlock_sock(sk);
|
| 3306 | local_bh_enable();
|
| 3307 | }
|
| 3308 |
|
| 3309 | void tcp_close(struct sock *sk, long timeout)
|
| 3310 | {
|
| 3311 | lock_sock(sk);
|
| 3312 | __tcp_close(sk, timeout);
|
| 3313 | release_sock(sk);
|
| 3314 | if (!sk->sk_net_refcnt)
|
| 3315 | inet_csk_clear_xmit_timers_sync(sk);
|
| 3316 | sock_put(sk);
|
| 3317 | }
|
| 3318 | EXPORT_SYMBOL(tcp_close);
|
| 3319 |
|
| 3320 |
|
| 3321 |
|
| 3322 | static inline bool tcp_need_reset(int state)
|
| 3323 | {
|
| 3324 | return (1 << state) &
|
| 3325 | (TCPF_ESTABLISHED | TCPF_CLOSE_WAIT | TCPF_FIN_WAIT1 |
|
| 3326 | TCPF_FIN_WAIT2 | TCPF_SYN_RECV);
|
| 3327 | }
|
| 3328 |
|
| 3329 | static void tcp_rtx_queue_purge(struct sock *sk)
|
| 3330 | {
|
| 3331 | struct rb_node *p = rb_first(&sk->tcp_rtx_queue);
|
| 3332 |
|
| 3333 | tcp_sk(sk)->highest_sack = NULL;
|
| 3334 | while (p) {
|
| 3335 | struct sk_buff *skb = rb_to_skb(p);
|
| 3336 |
|
| 3337 | p = rb_next(p);
|
| 3338 | |
| 3339 | * list_del(&skb->tcp_tsorted_anchor)
|
| 3340 |
|
| 3341 | tcp_rtx_queue_unlink(skb, sk);
|
| 3342 | tcp_wmem_free_skb(sk, skb);
|
| 3343 | }
|
| 3344 | }
|
| 3345 |
|
| 3346 | void tcp_write_queue_purge(struct sock *sk)
|
| 3347 | {
|
| 3348 | struct sk_buff *skb;
|
| 3349 |
|
| 3350 | tcp_chrono_stop(sk, TCP_CHRONO_BUSY);
|
| 3351 | while ((skb = __skb_dequeue(&sk->sk_write_queue)) != NULL) {
|
| 3352 | tcp_skb_tsorted_anchor_cleanup(skb);
|
| 3353 | tcp_wmem_free_skb(sk, skb);
|
| 3354 | }
|
| 3355 | tcp_rtx_queue_purge(sk);
|
| 3356 | INIT_LIST_HEAD(&tcp_sk(sk)->tsorted_sent_queue);
|
| 3357 | tcp_clear_all_retrans_hints(tcp_sk(sk));
|
| 3358 | tcp_sk(sk)->packets_out = 0;
|
| 3359 | inet_csk(sk)->icsk_backoff = 0;
|
| 3360 | }
|
| 3361 |
|
| 3362 | int tcp_disconnect(struct sock *sk, int flags)
|
| 3363 | {
|
| 3364 | struct inet_sock *inet = inet_sk(sk);
|
| 3365 | struct inet_connection_sock *icsk = inet_csk(sk);
|
| 3366 | struct tcp_sock *tp = tcp_sk(sk);
|
| 3367 | int old_state = sk->sk_state;
|
| 3368 | struct request_sock *req;
|
| 3369 | u32 seq;
|
| 3370 |
|
| 3371 | if (old_state != TCP_CLOSE)
|
| 3372 | tcp_set_state(sk, TCP_CLOSE);
|
| 3373 |
|
| 3374 |
|
| 3375 | if (old_state == TCP_LISTEN) {
|
| 3376 | inet_csk_listen_stop(sk);
|
| 3377 | } else if (unlikely(tp->repair)) {
|
| 3378 | WRITE_ONCE(sk->sk_err, ECONNABORTED);
|
| 3379 | } else if (tcp_need_reset(old_state)) {
|
| 3380 | tcp_send_active_reset(sk, SK_RST_REASON_TCP_STATE);
|
| 3381 | WRITE_ONCE(sk->sk_err, ECONNRESET);
|
| 3382 | } else if (tp->snd_nxt != tp->write_seq &&
|
| 3383 | (1 << old_state) & (TCPF_CLOSING | TCPF_LAST_ACK)) {
|
| 3384 | |
| 3385 | * states
|
| 3386 |
|
| 3387 | tcp_send_active_reset(sk,
|
| 3388 | SK_RST_REASON_TCP_DISCONNECT_WITH_DATA);
|
| 3389 | WRITE_ONCE(sk->sk_err, ECONNRESET);
|
| 3390 | } else if (old_state == TCP_SYN_SENT)
|
| 3391 | WRITE_ONCE(sk->sk_err, ECONNRESET);
|
| 3392 |
|
| 3393 | tcp_clear_xmit_timers(sk);
|
| 3394 | __skb_queue_purge(&sk->sk_receive_queue);
|
| 3395 | WRITE_ONCE(tp->copied_seq, tp->rcv_nxt);
|
| 3396 | WRITE_ONCE(tp->urg_data, 0);
|
| 3397 | sk_set_peek_off(sk, -1);
|
| 3398 | tcp_write_queue_purge(sk);
|
| 3399 | tcp_fastopen_active_disable_ofo_check(sk);
|
| 3400 | skb_rbtree_purge(&tp->out_of_order_queue);
|
| 3401 |
|
| 3402 | inet->inet_dport = 0;
|
| 3403 |
|
| 3404 | inet_bhash2_reset_saddr(sk);
|
| 3405 |
|
| 3406 | WRITE_ONCE(sk->sk_shutdown, 0);
|
| 3407 | sock_reset_flag(sk, SOCK_DONE);
|
| 3408 | tp->srtt_us = 0;
|
| 3409 | tp->mdev_us = jiffies_to_usecs(TCP_TIMEOUT_INIT);
|
| 3410 | tp->rcv_rtt_last_tsecr = 0;
|
| 3411 |
|
| 3412 | seq = tp->write_seq + tp->max_window + 2;
|
| 3413 | if (!seq)
|
| 3414 | seq = 1;
|
| 3415 | WRITE_ONCE(tp->write_seq, seq);
|
| 3416 |
|
| 3417 | icsk->icsk_backoff = 0;
|
| 3418 | WRITE_ONCE(icsk->icsk_probes_out, 0);
|
| 3419 | icsk->icsk_probes_tstamp = 0;
|
| 3420 | icsk->icsk_rto = TCP_TIMEOUT_INIT;
|
| 3421 | WRITE_ONCE(icsk->icsk_rto_min, TCP_RTO_MIN);
|
| 3422 | WRITE_ONCE(icsk->icsk_delack_max, TCP_DELACK_MAX);
|
| 3423 | WRITE_ONCE(tp->snd_ssthresh, TCP_INFINITE_SSTHRESH);
|
| 3424 | tcp_snd_cwnd_set(tp, TCP_INIT_CWND);
|
| 3425 | tp->snd_cwnd_cnt = 0;
|
| 3426 | tp->is_cwnd_limited = 0;
|
| 3427 | tp->max_packets_out = 0;
|
| 3428 | tp->window_clamp = 0;
|
| 3429 | tp->delivered = 0;
|
| 3430 | tp->delivered_ce = 0;
|
| 3431 | tp->accecn_fail_mode = 0;
|
| 3432 | tp->saw_accecn_opt = TCP_ACCECN_OPT_NOT_SEEN;
|
| 3433 | tcp_accecn_init_counters(tp);
|
| 3434 | tp->prev_ecnfield = 0;
|
| 3435 | tp->accecn_opt_tstamp = 0;
|
| 3436 | tp->pkts_acked_ewma = 0;
|
| 3437 | if (icsk->icsk_ca_initialized && icsk->icsk_ca_ops->release)
|
| 3438 | icsk->icsk_ca_ops->release(sk);
|
| 3439 | memset(icsk->icsk_ca_priv, 0, sizeof(icsk->icsk_ca_priv));
|
| 3440 | icsk->icsk_ca_initialized = 0;
|
| 3441 | tcp_set_ca_state(sk, TCP_CA_Open);
|
| 3442 | tp->is_sack_reneg = 0;
|
| 3443 | tcp_clear_retrans(tp);
|
| 3444 | tp->total_retrans = 0;
|
| 3445 | inet_csk_delack_init(sk);
|
| 3446 | |
| 3447 | * issue in __tcp_select_window()
|
| 3448 |
|
| 3449 | icsk->icsk_ack.rcv_mss = TCP_MIN_MSS;
|
| 3450 | memset(&tp->rx_opt, 0, sizeof(tp->rx_opt));
|
| 3451 | __sk_dst_reset(sk);
|
| 3452 | dst_release(unrcu_pointer(xchg(&sk->sk_rx_dst, NULL)));
|
| 3453 | tcp_saved_syn_free(tp);
|
| 3454 | tp->compressed_ack = 0;
|
| 3455 | tp->segs_in = 0;
|
| 3456 | tp->segs_out = 0;
|
| 3457 | tp->bytes_sent = 0;
|
| 3458 | tp->bytes_acked = 0;
|
| 3459 | tp->bytes_received = 0;
|
| 3460 | tp->bytes_retrans = 0;
|
| 3461 | tp->data_segs_in = 0;
|
| 3462 | tp->data_segs_out = 0;
|
| 3463 | tp->duplicate_sack[0].start_seq = 0;
|
| 3464 | tp->duplicate_sack[0].end_seq = 0;
|
| 3465 | tp->dsack_dups = 0;
|
| 3466 | tp->reord_seen = 0;
|
| 3467 | tp->retrans_out = 0;
|
| 3468 | tp->sacked_out = 0;
|
| 3469 | tp->tlp_high_seq = 0;
|
| 3470 | tp->last_oow_ack_time = 0;
|
| 3471 | tp->plb_rehash = 0;
|
| 3472 |
|
| 3473 | tp->app_limited = ~0U;
|
| 3474 | tp->rate_app_limited = 1;
|
| 3475 | tp->rack.mstamp = 0;
|
| 3476 | tp->rack.advanced = 0;
|
| 3477 | tp->rack.reo_wnd_steps = 1;
|
| 3478 | tp->rack.last_delivered = 0;
|
| 3479 | tp->rack.reo_wnd_persist = 0;
|
| 3480 | tp->rack.dsack_seen = 0;
|
| 3481 | tp->syn_data_acked = 0;
|
| 3482 | tp->syn_fastopen_child = 0;
|
| 3483 | tp->rx_opt.saw_tstamp = 0;
|
| 3484 | tp->rx_opt.dsack = 0;
|
| 3485 | tp->rx_opt.num_sacks = 0;
|
| 3486 | tp->rcv_ooopack = 0;
|
| 3487 |
|
| 3488 |
|
| 3489 |
|
| 3490 | req = rcu_dereference_protected(tp->fastopen_rsk,
|
| 3491 | lockdep_sock_is_held(sk));
|
| 3492 | if (req)
|
| 3493 | reqsk_fastopen_remove(sk, req, false);
|
| 3494 | tcp_free_fastopen_req(tp);
|
| 3495 | inet_clear_bit(DEFER_CONNECT, sk);
|
| 3496 | tp->fastopen_client_fail = 0;
|
| 3497 |
|
| 3498 | WARN_ON(inet->inet_num && !icsk->icsk_bind_hash);
|
| 3499 |
|
| 3500 | if (sk->sk_frag.page) {
|
| 3501 | put_page(sk->sk_frag.page);
|
| 3502 | sk->sk_frag.page = NULL;
|
| 3503 | sk->sk_frag.offset = 0;
|
| 3504 | }
|
| 3505 | sk_error_report(sk);
|
| 3506 | return 0;
|
| 3507 | }
|
| 3508 |
|
| 3509 | static inline bool tcp_can_repair_sock(const struct sock *sk)
|
| 3510 | {
|
| 3511 | return sockopt_ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN) &&
|
| 3512 | (sk->sk_state != TCP_LISTEN);
|
| 3513 | }
|
| 3514 |
|
| 3515 | static int tcp_repair_set_window(struct tcp_sock *tp, sockptr_t optbuf, int len)
|
| 3516 | {
|
| 3517 | struct tcp_repair_window opt;
|
| 3518 |
|
| 3519 | if (!tp->repair)
|
| 3520 | return -EPERM;
|
| 3521 |
|
| 3522 | if (len != sizeof(opt))
|
| 3523 | return -EINVAL;
|
| 3524 |
|
| 3525 | if (copy_from_sockptr(&opt, optbuf, sizeof(opt)))
|
| 3526 | return -EFAULT;
|
| 3527 |
|
| 3528 | if (opt.max_window < opt.snd_wnd)
|
| 3529 | return -EINVAL;
|
| 3530 |
|
| 3531 | if (after(opt.snd_wl1, tp->rcv_nxt + opt.rcv_wnd))
|
| 3532 | return -EINVAL;
|
| 3533 |
|
| 3534 | if (after(opt.rcv_wup, tp->rcv_nxt))
|
| 3535 | return -EINVAL;
|
| 3536 |
|
| 3537 | tp->snd_wl1 = opt.snd_wl1;
|
| 3538 | tp->snd_wnd = opt.snd_wnd;
|
| 3539 | tp->max_window = opt.max_window;
|
| 3540 |
|
| 3541 | tp->rcv_wnd = opt.rcv_wnd;
|
| 3542 | tp->rcv_wup = opt.rcv_wup;
|
| 3543 | tp->rcv_mwnd_seq = opt.rcv_wup + opt.rcv_wnd;
|
| 3544 |
|
| 3545 | return 0;
|
| 3546 | }
|
| 3547 |
|
| 3548 | static int tcp_repair_options_est(struct sock *sk, sockptr_t optbuf,
|
| 3549 | unsigned int len)
|
| 3550 | {
|
| 3551 | struct tcp_sock *tp = tcp_sk(sk);
|
| 3552 | struct tcp_repair_opt opt;
|
| 3553 | size_t offset = 0;
|
| 3554 |
|
| 3555 | while (len >= sizeof(opt)) {
|
| 3556 | if (copy_from_sockptr_offset(&opt, optbuf, offset, sizeof(opt)))
|
| 3557 | return -EFAULT;
|
| 3558 |
|
| 3559 | offset += sizeof(opt);
|
| 3560 | len -= sizeof(opt);
|
| 3561 |
|
| 3562 | switch (opt.opt_code) {
|
| 3563 | case TCPOPT_MSS:
|
| 3564 | tp->rx_opt.mss_clamp = opt.opt_val;
|
| 3565 | tcp_mtup_init(sk);
|
| 3566 | break;
|
| 3567 | case TCPOPT_WINDOW:
|
| 3568 | {
|
| 3569 | u16 snd_wscale = opt.opt_val & 0xFFFF;
|
| 3570 | u16 rcv_wscale = opt.opt_val >> 16;
|
| 3571 |
|
| 3572 | if (snd_wscale > TCP_MAX_WSCALE || rcv_wscale > TCP_MAX_WSCALE)
|
| 3573 | return -EFBIG;
|
| 3574 |
|
| 3575 | tp->rx_opt.snd_wscale = snd_wscale;
|
| 3576 | tp->rx_opt.rcv_wscale = rcv_wscale;
|
| 3577 | tp->rx_opt.wscale_ok = 1;
|
| 3578 | }
|
| 3579 | break;
|
| 3580 | case TCPOPT_SACK_PERM:
|
| 3581 | if (opt.opt_val != 0)
|
| 3582 | return -EINVAL;
|
| 3583 |
|
| 3584 | tp->rx_opt.sack_ok |= TCP_SACK_SEEN;
|
| 3585 | break;
|
| 3586 | case TCPOPT_TIMESTAMP:
|
| 3587 | if (opt.opt_val != 0)
|
| 3588 | return -EINVAL;
|
| 3589 |
|
| 3590 | tp->rx_opt.tstamp_ok = 1;
|
| 3591 | break;
|
| 3592 | }
|
| 3593 | }
|
| 3594 |
|
| 3595 | return 0;
|
| 3596 | }
|
| 3597 |
|
| 3598 | DEFINE_STATIC_KEY_FALSE(tcp_tx_delay_enabled);
|
| 3599 |
|
| 3600 | static void tcp_enable_tx_delay(struct sock *sk, int val)
|
| 3601 | {
|
| 3602 | struct tcp_sock *tp = tcp_sk(sk);
|
| 3603 | s32 delta = (val - tp->tcp_tx_delay) << 3;
|
| 3604 |
|
| 3605 | if (val && !static_branch_unlikely(&tcp_tx_delay_enabled)) {
|
| 3606 | static int __tcp_tx_delay_enabled = 0;
|
| 3607 |
|
| 3608 | if (cmpxchg(&__tcp_tx_delay_enabled, 0, 1) == 0) {
|
| 3609 | static_branch_enable(&tcp_tx_delay_enabled);
|
| 3610 | pr_info("TCP_TX_DELAY enabled\n");
|
| 3611 | }
|
| 3612 | }
|
| 3613 | |
| 3614 | * tp->rtt_min, icsk_rto and sk->sk_pacing_rate.
|
| 3615 | * This is best effort.
|
| 3616 |
|
| 3617 | if (delta && sk->sk_state == TCP_ESTABLISHED) {
|
| 3618 | s64 srtt = (s64)tp->srtt_us + delta;
|
| 3619 |
|
| 3620 | WRITE_ONCE(tp->srtt_us,
|
| 3621 | clamp_t(s64, srtt, 1, ~0U));
|
| 3622 |
|
| 3623 |
|
| 3624 | tcp_set_rto(sk);
|
| 3625 |
|
| 3626 | minmax_reset(&tp->rtt_min, tcp_jiffies32, ~0U);
|
| 3627 |
|
| 3628 | tcp_update_pacing_rate(sk);
|
| 3629 | }
|
| 3630 | }
|
| 3631 |
|
| 3632 | |
| 3633 | * this option and we transmit any pending partial frames in the queue. This is
|
| 3634 | * meant to be used alongside sendfile() to get properly filled frames when the
|
| 3635 | * user (for example) must write out headers with a write() call first and then
|
| 3636 | * use sendfile to send out the data parts.
|
| 3637 | *
|
| 3638 | * TCP_CORK can be set together with TCP_NODELAY and it is stronger than
|
| 3639 | * TCP_NODELAY.
|
| 3640 |
|
| 3641 | void __tcp_sock_set_cork(struct sock *sk, bool on)
|
| 3642 | {
|
| 3643 | struct tcp_sock *tp = tcp_sk(sk);
|
| 3644 |
|
| 3645 | if (on) {
|
| 3646 | tp->nonagle |= TCP_NAGLE_CORK;
|
| 3647 | } else {
|
| 3648 | tp->nonagle &= ~TCP_NAGLE_CORK;
|
| 3649 | if (tp->nonagle & TCP_NAGLE_OFF)
|
| 3650 | tp->nonagle |= TCP_NAGLE_PUSH;
|
| 3651 | tcp_push_pending_frames(sk);
|
| 3652 | }
|
| 3653 | }
|
| 3654 |
|
| 3655 | void tcp_sock_set_cork(struct sock *sk, bool on)
|
| 3656 | {
|
| 3657 | lock_sock(sk);
|
| 3658 | __tcp_sock_set_cork(sk, on);
|
| 3659 | release_sock(sk);
|
| 3660 | }
|
| 3661 | EXPORT_SYMBOL(tcp_sock_set_cork);
|
| 3662 |
|
| 3663 | |
| 3664 | * remembered, but it is not activated until cork is cleared.
|
| 3665 | *
|
| 3666 | * However, when TCP_NODELAY is set we make an explicit push, which overrides
|
| 3667 | * even TCP_CORK for currently queued segments.
|
| 3668 |
|
| 3669 | void __tcp_sock_set_nodelay(struct sock *sk, bool on)
|
| 3670 | {
|
| 3671 | if (on) {
|
| 3672 | tcp_sk(sk)->nonagle |= TCP_NAGLE_OFF|TCP_NAGLE_PUSH;
|
| 3673 | tcp_push_pending_frames(sk);
|
| 3674 | } else {
|
| 3675 | tcp_sk(sk)->nonagle &= ~TCP_NAGLE_OFF;
|
| 3676 | }
|
| 3677 | }
|
| 3678 |
|
| 3679 | void tcp_sock_set_nodelay(struct sock *sk)
|
| 3680 | {
|
| 3681 | lock_sock(sk);
|
| 3682 | __tcp_sock_set_nodelay(sk, true);
|
| 3683 | release_sock(sk);
|
| 3684 | }
|
| 3685 | EXPORT_SYMBOL(tcp_sock_set_nodelay);
|
| 3686 |
|
| 3687 | static void __tcp_sock_set_quickack(struct sock *sk, int val)
|
| 3688 | {
|
| 3689 | if (!val) {
|
| 3690 | inet_csk_enter_pingpong_mode(sk);
|
| 3691 | return;
|
| 3692 | }
|
| 3693 |
|
| 3694 | inet_csk_exit_pingpong_mode(sk);
|
| 3695 | if ((1 << sk->sk_state) & (TCPF_ESTABLISHED | TCPF_CLOSE_WAIT) &&
|
| 3696 | inet_csk_ack_scheduled(sk)) {
|
| 3697 | inet_csk(sk)->icsk_ack.pending |= ICSK_ACK_PUSHED;
|
| 3698 | tcp_cleanup_rbuf(sk, 1);
|
| 3699 | if (!(val & 1))
|
| 3700 | inet_csk_enter_pingpong_mode(sk);
|
| 3701 | }
|
| 3702 | }
|
| 3703 |
|
| 3704 | void tcp_sock_set_quickack(struct sock *sk, int val)
|
| 3705 | {
|
| 3706 | lock_sock(sk);
|
| 3707 | __tcp_sock_set_quickack(sk, val);
|
| 3708 | release_sock(sk);
|
| 3709 | }
|
| 3710 | EXPORT_SYMBOL(tcp_sock_set_quickack);
|
| 3711 |
|
| 3712 | int tcp_sock_set_syncnt(struct sock *sk, int val)
|
| 3713 | {
|
| 3714 | if (val < 1 || val > MAX_TCP_SYNCNT)
|
| 3715 | return -EINVAL;
|
| 3716 |
|
| 3717 | WRITE_ONCE(inet_csk(sk)->icsk_syn_retries, val);
|
| 3718 | return 0;
|
| 3719 | }
|
| 3720 | EXPORT_SYMBOL(tcp_sock_set_syncnt);
|
| 3721 |
|
| 3722 | int tcp_sock_set_user_timeout(struct sock *sk, int val)
|
| 3723 | {
|
| 3724 | |
| 3725 | * before giving up and aborting (ETIMEDOUT) a connection.
|
| 3726 |
|
| 3727 | if (val < 0)
|
| 3728 | return -EINVAL;
|
| 3729 |
|
| 3730 | WRITE_ONCE(inet_csk(sk)->icsk_user_timeout, val);
|
| 3731 | return 0;
|
| 3732 | }
|
| 3733 | EXPORT_SYMBOL(tcp_sock_set_user_timeout);
|
| 3734 |
|
| 3735 | int tcp_sock_set_keepidle_locked(struct sock *sk, int val)
|
| 3736 | {
|
| 3737 | struct tcp_sock *tp = tcp_sk(sk);
|
| 3738 |
|
| 3739 | if (val < 1 || val > MAX_TCP_KEEPIDLE)
|
| 3740 | return -EINVAL;
|
| 3741 |
|
| 3742 |
|
| 3743 | WRITE_ONCE(tp->keepalive_time, val * HZ);
|
| 3744 | if (sock_flag(sk, SOCK_KEEPOPEN) &&
|
| 3745 | !((1 << sk->sk_state) & (TCPF_CLOSE | TCPF_LISTEN))) {
|
| 3746 | u32 elapsed = keepalive_time_elapsed(tp);
|
| 3747 |
|
| 3748 | if (tp->keepalive_time > elapsed)
|
| 3749 | elapsed = tp->keepalive_time - elapsed;
|
| 3750 | else
|
| 3751 | elapsed = 0;
|
| 3752 | tcp_reset_keepalive_timer(sk, elapsed);
|
| 3753 | }
|
| 3754 |
|
| 3755 | return 0;
|
| 3756 | }
|
| 3757 |
|
| 3758 | int tcp_sock_set_keepidle(struct sock *sk, int val)
|
| 3759 | {
|
| 3760 | int err;
|
| 3761 |
|
| 3762 | lock_sock(sk);
|
| 3763 | err = tcp_sock_set_keepidle_locked(sk, val);
|
| 3764 | release_sock(sk);
|
| 3765 | return err;
|
| 3766 | }
|
| 3767 | EXPORT_SYMBOL(tcp_sock_set_keepidle);
|
| 3768 |
|
| 3769 | int tcp_sock_set_keepintvl(struct sock *sk, int val)
|
| 3770 | {
|
| 3771 | if (val < 1 || val > MAX_TCP_KEEPINTVL)
|
| 3772 | return -EINVAL;
|
| 3773 |
|
| 3774 | WRITE_ONCE(tcp_sk(sk)->keepalive_intvl, val * HZ);
|
| 3775 | return 0;
|
| 3776 | }
|
| 3777 | EXPORT_SYMBOL(tcp_sock_set_keepintvl);
|
| 3778 |
|
| 3779 | int tcp_sock_set_keepcnt(struct sock *sk, int val)
|
| 3780 | {
|
| 3781 | if (val < 1 || val > MAX_TCP_KEEPCNT)
|
| 3782 | return -EINVAL;
|
| 3783 |
|
| 3784 |
|
| 3785 | WRITE_ONCE(tcp_sk(sk)->keepalive_probes, val);
|
| 3786 | return 0;
|
| 3787 | }
|
| 3788 | EXPORT_SYMBOL(tcp_sock_set_keepcnt);
|
| 3789 |
|
| 3790 | int tcp_set_window_clamp(struct sock *sk, int val)
|
| 3791 | {
|
| 3792 | u32 old_window_clamp, new_window_clamp, new_rcv_ssthresh;
|
| 3793 | struct tcp_sock *tp = tcp_sk(sk);
|
| 3794 |
|
| 3795 | if (!val) {
|
| 3796 | if (sk->sk_state != TCP_CLOSE)
|
| 3797 | return -EINVAL;
|
| 3798 | WRITE_ONCE(tp->window_clamp, 0);
|
| 3799 | return 0;
|
| 3800 | }
|
| 3801 |
|
| 3802 | old_window_clamp = tp->window_clamp;
|
| 3803 | new_window_clamp = max_t(int, SOCK_MIN_RCVBUF / 2, val);
|
| 3804 |
|
| 3805 | if (new_window_clamp == old_window_clamp)
|
| 3806 | return 0;
|
| 3807 |
|
| 3808 | WRITE_ONCE(tp->window_clamp, new_window_clamp);
|
| 3809 |
|
| 3810 | |
| 3811 | * when shrinking the window clamp.
|
| 3812 |
|
| 3813 | if (new_window_clamp < old_window_clamp) {
|
| 3814 | __tcp_adjust_rcv_ssthresh(sk, new_window_clamp);
|
| 3815 | } else {
|
| 3816 | new_rcv_ssthresh = min(tp->rcv_wnd, new_window_clamp);
|
| 3817 | tp->rcv_ssthresh = max(new_rcv_ssthresh, tp->rcv_ssthresh);
|
| 3818 | }
|
| 3819 | return 0;
|
| 3820 | }
|
| 3821 |
|
| 3822 | int tcp_sock_set_maxseg(struct sock *sk, int val)
|
| 3823 | {
|
| 3824 | |
| 3825 | * at the point when this call is done we typically don't yet
|
| 3826 | * know which interface is going to be used
|
| 3827 |
|
| 3828 | if (val && (val < TCP_MIN_MSS || val > MAX_TCP_WINDOW))
|
| 3829 | return -EINVAL;
|
| 3830 |
|
| 3831 | WRITE_ONCE(tcp_sk(sk)->rx_opt.user_mss, val);
|
| 3832 | return 0;
|
| 3833 | }
|
| 3834 |
|
| 3835 | |
| 3836 | * Socket option code for TCP.
|
| 3837 |
|
| 3838 | int do_tcp_setsockopt(struct sock *sk, int level, int optname,
|
| 3839 | sockptr_t optval, unsigned int optlen)
|
| 3840 | {
|
| 3841 | struct tcp_sock *tp = tcp_sk(sk);
|
| 3842 | struct inet_connection_sock *icsk = inet_csk(sk);
|
| 3843 | struct net *net = sock_net(sk);
|
| 3844 | int val;
|
| 3845 | int err = 0;
|
| 3846 |
|
| 3847 |
|
| 3848 | switch (optname) {
|
| 3849 | case TCP_CONGESTION: {
|
| 3850 | char name[TCP_CA_NAME_MAX];
|
| 3851 |
|
| 3852 | if (optlen < 1)
|
| 3853 | return -EINVAL;
|
| 3854 |
|
| 3855 | val = strncpy_from_sockptr(name, optval,
|
| 3856 | min_t(long, TCP_CA_NAME_MAX-1, optlen));
|
| 3857 | if (val < 0)
|
| 3858 | return -EFAULT;
|
| 3859 | name[val] = 0;
|
| 3860 |
|
| 3861 | sockopt_lock_sock(sk);
|
| 3862 | err = tcp_set_congestion_control(sk, name, !has_current_bpf_ctx(),
|
| 3863 | sockopt_ns_capable(sock_net(sk)->user_ns,
|
| 3864 | CAP_NET_ADMIN));
|
| 3865 | sockopt_release_sock(sk);
|
| 3866 | return err;
|
| 3867 | }
|
| 3868 | case TCP_ULP: {
|
| 3869 | char name[TCP_ULP_NAME_MAX];
|
| 3870 |
|
| 3871 | if (optlen < 1)
|
| 3872 | return -EINVAL;
|
| 3873 |
|
| 3874 | val = strncpy_from_sockptr(name, optval,
|
| 3875 | min_t(long, TCP_ULP_NAME_MAX - 1,
|
| 3876 | optlen));
|
| 3877 | if (val < 0)
|
| 3878 | return -EFAULT;
|
| 3879 | name[val] = 0;
|
| 3880 |
|
| 3881 | sockopt_lock_sock(sk);
|
| 3882 | err = tcp_set_ulp(sk, name);
|
| 3883 | sockopt_release_sock(sk);
|
| 3884 | return err;
|
| 3885 | }
|
| 3886 | case TCP_FASTOPEN_KEY: {
|
| 3887 | __u8 key[TCP_FASTOPEN_KEY_BUF_LENGTH];
|
| 3888 | __u8 *backup_key = NULL;
|
| 3889 |
|
| 3890 | |
| 3891 | * First key is the active one.
|
| 3892 |
|
| 3893 | if (optlen != TCP_FASTOPEN_KEY_LENGTH &&
|
| 3894 | optlen != TCP_FASTOPEN_KEY_BUF_LENGTH)
|
| 3895 | return -EINVAL;
|
| 3896 |
|
| 3897 | if (copy_from_sockptr(key, optval, optlen))
|
| 3898 | return -EFAULT;
|
| 3899 |
|
| 3900 | if (optlen == TCP_FASTOPEN_KEY_BUF_LENGTH)
|
| 3901 | backup_key = key + TCP_FASTOPEN_KEY_LENGTH;
|
| 3902 |
|
| 3903 | return tcp_fastopen_reset_cipher(net, sk, key, backup_key);
|
| 3904 | }
|
| 3905 | default:
|
| 3906 |
|
| 3907 | break;
|
| 3908 | }
|
| 3909 |
|
| 3910 | if (optlen < sizeof(int))
|
| 3911 | return -EINVAL;
|
| 3912 |
|
| 3913 | if (copy_from_sockptr(&val, optval, sizeof(val)))
|
| 3914 | return -EFAULT;
|
| 3915 |
|
| 3916 |
|
| 3917 | switch (optname) {
|
| 3918 | case TCP_SYNCNT:
|
| 3919 | return tcp_sock_set_syncnt(sk, val);
|
| 3920 | case TCP_USER_TIMEOUT:
|
| 3921 | return tcp_sock_set_user_timeout(sk, val);
|
| 3922 | case TCP_KEEPINTVL:
|
| 3923 | return tcp_sock_set_keepintvl(sk, val);
|
| 3924 | case TCP_KEEPCNT:
|
| 3925 | return tcp_sock_set_keepcnt(sk, val);
|
| 3926 | case TCP_LINGER2:
|
| 3927 | if (val < 0)
|
| 3928 | WRITE_ONCE(tp->linger2, -1);
|
| 3929 | else if (val > TCP_FIN_TIMEOUT_MAX / HZ)
|
| 3930 | WRITE_ONCE(tp->linger2, TCP_FIN_TIMEOUT_MAX);
|
| 3931 | else
|
| 3932 | WRITE_ONCE(tp->linger2, val * HZ);
|
| 3933 | return 0;
|
| 3934 | case TCP_DEFER_ACCEPT:
|
| 3935 |
|
| 3936 | WRITE_ONCE(icsk->icsk_accept_queue.rskq_defer_accept,
|
| 3937 | secs_to_retrans(val, TCP_TIMEOUT_INIT / HZ,
|
| 3938 | TCP_RTO_MAX / HZ));
|
| 3939 | return 0;
|
| 3940 | case TCP_RTO_MAX_MS:
|
| 3941 | if (val < MSEC_PER_SEC || val > TCP_RTO_MAX_SEC * MSEC_PER_SEC)
|
| 3942 | return -EINVAL;
|
| 3943 | WRITE_ONCE(inet_csk(sk)->icsk_rto_max, msecs_to_jiffies(val));
|
| 3944 | return 0;
|
| 3945 | case TCP_RTO_MIN_US: {
|
| 3946 | int rto_min = usecs_to_jiffies(val);
|
| 3947 |
|
| 3948 | if (rto_min > TCP_RTO_MIN || rto_min < TCP_TIMEOUT_MIN)
|
| 3949 | return -EINVAL;
|
| 3950 | WRITE_ONCE(inet_csk(sk)->icsk_rto_min, rto_min);
|
| 3951 | return 0;
|
| 3952 | }
|
| 3953 | case TCP_DELACK_MAX_US: {
|
| 3954 | int delack_max = usecs_to_jiffies(val);
|
| 3955 |
|
| 3956 | if (delack_max > TCP_DELACK_MAX || delack_max < TCP_TIMEOUT_MIN)
|
| 3957 | return -EINVAL;
|
| 3958 | WRITE_ONCE(inet_csk(sk)->icsk_delack_max, delack_max);
|
| 3959 | return 0;
|
| 3960 | }
|
| 3961 | case TCP_MAXSEG:
|
| 3962 | return tcp_sock_set_maxseg(sk, val);
|
| 3963 | }
|
| 3964 |
|
| 3965 | sockopt_lock_sock(sk);
|
| 3966 |
|
| 3967 | switch (optname) {
|
| 3968 | case TCP_NODELAY:
|
| 3969 | __tcp_sock_set_nodelay(sk, val);
|
| 3970 | break;
|
| 3971 |
|
| 3972 | case TCP_THIN_LINEAR_TIMEOUTS:
|
| 3973 | if (val < 0 || val > 1)
|
| 3974 | err = -EINVAL;
|
| 3975 | else
|
| 3976 | tp->thin_lto = val;
|
| 3977 | break;
|
| 3978 |
|
| 3979 | case TCP_THIN_DUPACK:
|
| 3980 | if (val < 0 || val > 1)
|
| 3981 | err = -EINVAL;
|
| 3982 | break;
|
| 3983 |
|
| 3984 | case TCP_REPAIR:
|
| 3985 | if (!tcp_can_repair_sock(sk))
|
| 3986 | err = -EPERM;
|
| 3987 | else if (val == TCP_REPAIR_ON) {
|
| 3988 | tp->repair = 1;
|
| 3989 | sk->sk_reuse = SK_FORCE_REUSE;
|
| 3990 | tp->repair_queue = TCP_NO_QUEUE;
|
| 3991 | } else if (val == TCP_REPAIR_OFF) {
|
| 3992 | tp->repair = 0;
|
| 3993 | sk->sk_reuse = SK_NO_REUSE;
|
| 3994 | tcp_send_window_probe(sk);
|
| 3995 | } else if (val == TCP_REPAIR_OFF_NO_WP) {
|
| 3996 | tp->repair = 0;
|
| 3997 | sk->sk_reuse = SK_NO_REUSE;
|
| 3998 | } else
|
| 3999 | err = -EINVAL;
|
| 4000 |
|
| 4001 | break;
|
| 4002 |
|
| 4003 | case TCP_REPAIR_QUEUE:
|
| 4004 | if (!tp->repair)
|
| 4005 | err = -EPERM;
|
| 4006 | else if ((unsigned int)val < TCP_QUEUES_NR)
|
| 4007 | tp->repair_queue = val;
|
| 4008 | else
|
| 4009 | err = -EINVAL;
|
| 4010 | break;
|
| 4011 |
|
| 4012 | case TCP_QUEUE_SEQ:
|
| 4013 | if (sk->sk_state != TCP_CLOSE) {
|
| 4014 | err = -EPERM;
|
| 4015 | } else if (tp->repair_queue == TCP_SEND_QUEUE) {
|
| 4016 | if (!tcp_rtx_queue_empty(sk))
|
| 4017 | err = -EPERM;
|
| 4018 | else
|
| 4019 | WRITE_ONCE(tp->write_seq, val);
|
| 4020 | } else if (tp->repair_queue == TCP_RECV_QUEUE) {
|
| 4021 | if (tp->rcv_nxt != tp->copied_seq) {
|
| 4022 | err = -EPERM;
|
| 4023 | } else {
|
| 4024 | WRITE_ONCE(tp->rcv_nxt, val);
|
| 4025 | WRITE_ONCE(tp->copied_seq, val);
|
| 4026 | }
|
| 4027 | } else {
|
| 4028 | err = -EINVAL;
|
| 4029 | }
|
| 4030 | break;
|
| 4031 |
|
| 4032 | case TCP_REPAIR_OPTIONS:
|
| 4033 | if (!tp->repair)
|
| 4034 | err = -EINVAL;
|
| 4035 | else if (sk->sk_state == TCP_ESTABLISHED && !tp->bytes_sent)
|
| 4036 | err = tcp_repair_options_est(sk, optval, optlen);
|
| 4037 | else
|
| 4038 | err = -EPERM;
|
| 4039 | break;
|
| 4040 |
|
| 4041 | case TCP_CORK:
|
| 4042 | __tcp_sock_set_cork(sk, val);
|
| 4043 | break;
|
| 4044 |
|
| 4045 | case TCP_KEEPIDLE:
|
| 4046 | err = tcp_sock_set_keepidle_locked(sk, val);
|
| 4047 | break;
|
| 4048 | case TCP_SAVE_SYN:
|
| 4049 |
|
| 4050 | if (val < 0 || val > 2)
|
| 4051 | err = -EINVAL;
|
| 4052 | else
|
| 4053 | tp->save_syn = val;
|
| 4054 | break;
|
| 4055 |
|
| 4056 | case TCP_WINDOW_CLAMP:
|
| 4057 | err = tcp_set_window_clamp(sk, val);
|
| 4058 | break;
|
| 4059 |
|
| 4060 | case TCP_QUICKACK:
|
| 4061 | __tcp_sock_set_quickack(sk, val);
|
| 4062 | break;
|
| 4063 |
|
| 4064 | case TCP_AO_REPAIR:
|
| 4065 | if (!tcp_can_repair_sock(sk)) {
|
| 4066 | err = -EPERM;
|
| 4067 | break;
|
| 4068 | }
|
| 4069 | err = tcp_ao_set_repair(sk, optval, optlen);
|
| 4070 | break;
|
| 4071 | #ifdef CONFIG_TCP_AO
|
| 4072 | case TCP_AO_ADD_KEY:
|
| 4073 | case TCP_AO_DEL_KEY:
|
| 4074 | case TCP_AO_INFO: {
|
| 4075 | |
| 4076 | * sk_state has to be LISTEN or CLOSE. Allow TCP_REPAIR
|
| 4077 | * in any state.
|
| 4078 |
|
| 4079 | if ((1 << sk->sk_state) & (TCPF_LISTEN | TCPF_CLOSE))
|
| 4080 | goto ao_parse;
|
| 4081 | if (rcu_dereference_protected(tcp_sk(sk)->ao_info,
|
| 4082 | lockdep_sock_is_held(sk)))
|
| 4083 | goto ao_parse;
|
| 4084 | if (tp->repair)
|
| 4085 | goto ao_parse;
|
| 4086 | err = -EISCONN;
|
| 4087 | break;
|
| 4088 | ao_parse:
|
| 4089 | err = tp->af_specific->ao_parse(sk, optname, optval, optlen);
|
| 4090 | break;
|
| 4091 | }
|
| 4092 | #endif
|
| 4093 | #ifdef CONFIG_TCP_MD5SIG
|
| 4094 | case TCP_MD5SIG:
|
| 4095 | case TCP_MD5SIG_EXT:
|
| 4096 | err = tp->af_specific->md5_parse(sk, optname, optval, optlen);
|
| 4097 | break;
|
| 4098 | #endif
|
| 4099 | case TCP_FASTOPEN:
|
| 4100 | if (val >= 0 && ((1 << sk->sk_state) & (TCPF_CLOSE |
|
| 4101 | TCPF_LISTEN))) {
|
| 4102 | tcp_fastopen_init_key_once(net);
|
| 4103 |
|
| 4104 | fastopen_queue_tune(sk, val);
|
| 4105 | } else {
|
| 4106 | err = -EINVAL;
|
| 4107 | }
|
| 4108 | break;
|
| 4109 | case TCP_FASTOPEN_CONNECT:
|
| 4110 | if (val > 1 || val < 0) {
|
| 4111 | err = -EINVAL;
|
| 4112 | } else if (READ_ONCE(net->ipv4.sysctl_tcp_fastopen) &
|
| 4113 | TFO_CLIENT_ENABLE) {
|
| 4114 | if (sk->sk_state == TCP_CLOSE)
|
| 4115 | tp->fastopen_connect = val;
|
| 4116 | else
|
| 4117 | err = -EINVAL;
|
| 4118 | } else {
|
| 4119 | err = -EOPNOTSUPP;
|
| 4120 | }
|
| 4121 | break;
|
| 4122 | case TCP_FASTOPEN_NO_COOKIE:
|
| 4123 | if (val > 1 || val < 0)
|
| 4124 | err = -EINVAL;
|
| 4125 | else if (!((1 << sk->sk_state) & (TCPF_CLOSE | TCPF_LISTEN)))
|
| 4126 | err = -EINVAL;
|
| 4127 | else
|
| 4128 | tp->fastopen_no_cookie = val;
|
| 4129 | break;
|
| 4130 | case TCP_TIMESTAMP:
|
| 4131 | if (!tp->repair) {
|
| 4132 | err = -EPERM;
|
| 4133 | break;
|
| 4134 | }
|
| 4135 | |
| 4136 | * and low order bit contains usec_ts enable bit.
|
| 4137 | * Its a best effort, and we do not care if user makes an error.
|
| 4138 |
|
| 4139 | tp->tcp_usec_ts = val & 1;
|
| 4140 | WRITE_ONCE(tp->tsoffset, val - tcp_clock_ts(tp->tcp_usec_ts));
|
| 4141 | break;
|
| 4142 | case TCP_REPAIR_WINDOW:
|
| 4143 | err = tcp_repair_set_window(tp, optval, optlen);
|
| 4144 | break;
|
| 4145 | case TCP_NOTSENT_LOWAT:
|
| 4146 | WRITE_ONCE(tp->notsent_lowat, val);
|
| 4147 | READ_ONCE(sk->sk_write_space)(sk);
|
| 4148 | break;
|
| 4149 | case TCP_INQ:
|
| 4150 | if (val > 1 || val < 0)
|
| 4151 | err = -EINVAL;
|
| 4152 | else
|
| 4153 | tp->recvmsg_inq = val;
|
| 4154 | break;
|
| 4155 | case TCP_TX_DELAY:
|
| 4156 |
|
| 4157 | if (val < 0 || val >= (1U << (31 - 3))) {
|
| 4158 | err = -EINVAL;
|
| 4159 | break;
|
| 4160 | }
|
| 4161 | tcp_enable_tx_delay(sk, val);
|
| 4162 | WRITE_ONCE(tp->tcp_tx_delay, val);
|
| 4163 | break;
|
| 4164 | default:
|
| 4165 | err = -ENOPROTOOPT;
|
| 4166 | break;
|
| 4167 | }
|
| 4168 |
|
| 4169 | sockopt_release_sock(sk);
|
| 4170 | return err;
|
| 4171 | }
|
| 4172 |
|
| 4173 | int tcp_setsockopt(struct sock *sk, int level, int optname, sockptr_t optval,
|
| 4174 | unsigned int optlen)
|
| 4175 | {
|
| 4176 | const struct inet_connection_sock *icsk = inet_csk(sk);
|
| 4177 |
|
| 4178 | if (level != SOL_TCP)
|
| 4179 |
|
| 4180 | return READ_ONCE(icsk->icsk_af_ops)->setsockopt(sk, level, optname,
|
| 4181 | optval, optlen);
|
| 4182 | return do_tcp_setsockopt(sk, level, optname, optval, optlen);
|
| 4183 | }
|
| 4184 |
|
| 4185 | static void tcp_get_info_chrono_stats(const struct tcp_sock *tp,
|
| 4186 | struct tcp_info *info)
|
| 4187 | {
|
| 4188 | u64 stats[__TCP_CHRONO_MAX], total = 0;
|
| 4189 | enum tcp_chrono i, cur;
|
| 4190 |
|
| 4191 | |
| 4192 | * This is because socket lock might not be owned by us at this point.
|
| 4193 | * This is best effort, tcp_get_timestamping_opt_stats() can
|
| 4194 | * see wrong values. A real fix would be too costly for TCP fast path.
|
| 4195 |
|
| 4196 | cur = READ_ONCE(tp->chrono_type);
|
| 4197 | for (i = TCP_CHRONO_BUSY; i < __TCP_CHRONO_MAX; ++i) {
|
| 4198 | stats[i] = READ_ONCE(tp->chrono_stat[i - 1]);
|
| 4199 | if (i == cur)
|
| 4200 | stats[i] += tcp_jiffies32 - READ_ONCE(tp->chrono_start);
|
| 4201 | stats[i] *= USEC_PER_SEC / HZ;
|
| 4202 | total += stats[i];
|
| 4203 | }
|
| 4204 |
|
| 4205 | info->tcpi_busy_time = total;
|
| 4206 | info->tcpi_rwnd_limited = stats[TCP_CHRONO_RWND_LIMITED];
|
| 4207 | info->tcpi_sndbuf_limited = stats[TCP_CHRONO_SNDBUF_LIMITED];
|
| 4208 | }
|
| 4209 |
|
| 4210 |
|
| 4211 | void tcp_get_info(struct sock *sk, struct tcp_info *info)
|
| 4212 | {
|
| 4213 | const struct tcp_sock *tp = tcp_sk(sk);
|
| 4214 | const struct inet_connection_sock *icsk = inet_csk(sk);
|
| 4215 | const u8 ect1_idx = INET_ECN_ECT_1 - 1;
|
| 4216 | const u8 ect0_idx = INET_ECN_ECT_0 - 1;
|
| 4217 | const u8 ce_idx = INET_ECN_CE - 1;
|
| 4218 | unsigned long rate;
|
| 4219 | u32 now;
|
| 4220 | u64 rate64;
|
| 4221 | bool slow;
|
| 4222 |
|
| 4223 | memset(info, 0, sizeof(*info));
|
| 4224 | if (sk->sk_type != SOCK_STREAM)
|
| 4225 | return;
|
| 4226 |
|
| 4227 | info->tcpi_state = inet_sk_state_load(sk);
|
| 4228 |
|
| 4229 |
|
| 4230 | rate = READ_ONCE(sk->sk_pacing_rate);
|
| 4231 | rate64 = (rate != ~0UL) ? rate : ~0ULL;
|
| 4232 | info->tcpi_pacing_rate = rate64;
|
| 4233 |
|
| 4234 | rate = READ_ONCE(sk->sk_max_pacing_rate);
|
| 4235 | rate64 = (rate != ~0UL) ? rate : ~0ULL;
|
| 4236 | info->tcpi_max_pacing_rate = rate64;
|
| 4237 |
|
| 4238 | info->tcpi_reordering = tp->reordering;
|
| 4239 | info->tcpi_snd_cwnd = tcp_snd_cwnd(tp);
|
| 4240 |
|
| 4241 | if (info->tcpi_state == TCP_LISTEN) {
|
| 4242 | |
| 4243 | * tcpi_unacked -> Number of children ready for accept()
|
| 4244 | * tcpi_sacked -> max backlog
|
| 4245 |
|
| 4246 | info->tcpi_unacked = READ_ONCE(sk->sk_ack_backlog);
|
| 4247 | info->tcpi_sacked = READ_ONCE(sk->sk_max_ack_backlog);
|
| 4248 | return;
|
| 4249 | }
|
| 4250 |
|
| 4251 | slow = lock_sock_fast(sk);
|
| 4252 |
|
| 4253 | info->tcpi_ca_state = icsk->icsk_ca_state;
|
| 4254 | info->tcpi_retransmits = icsk->icsk_retransmits;
|
| 4255 | info->tcpi_probes = icsk->icsk_probes_out;
|
| 4256 | info->tcpi_backoff = icsk->icsk_backoff;
|
| 4257 |
|
| 4258 | if (tp->rx_opt.tstamp_ok)
|
| 4259 | info->tcpi_options |= TCPI_OPT_TIMESTAMPS;
|
| 4260 | if (tcp_is_sack(tp))
|
| 4261 | info->tcpi_options |= TCPI_OPT_SACK;
|
| 4262 | if (tp->rx_opt.wscale_ok) {
|
| 4263 | info->tcpi_options |= TCPI_OPT_WSCALE;
|
| 4264 | info->tcpi_snd_wscale = tp->rx_opt.snd_wscale;
|
| 4265 | info->tcpi_rcv_wscale = tp->rx_opt.rcv_wscale;
|
| 4266 | }
|
| 4267 |
|
| 4268 | if (tcp_ecn_mode_any(tp))
|
| 4269 | info->tcpi_options |= TCPI_OPT_ECN;
|
| 4270 | if (tp->ecn_flags & TCP_ECN_SEEN)
|
| 4271 | info->tcpi_options |= TCPI_OPT_ECN_SEEN;
|
| 4272 | if (tp->syn_data_acked)
|
| 4273 | info->tcpi_options |= TCPI_OPT_SYN_DATA;
|
| 4274 | if (tp->tcp_usec_ts)
|
| 4275 | info->tcpi_options |= TCPI_OPT_USEC_TS;
|
| 4276 | if (tp->syn_fastopen_child)
|
| 4277 | info->tcpi_options |= TCPI_OPT_TFO_CHILD;
|
| 4278 |
|
| 4279 | info->tcpi_rto = jiffies_to_usecs(icsk->icsk_rto);
|
| 4280 | info->tcpi_ato = jiffies_to_usecs(min_t(u32, icsk->icsk_ack.ato,
|
| 4281 | tcp_delack_max(sk)));
|
| 4282 | info->tcpi_snd_mss = tp->mss_cache;
|
| 4283 | info->tcpi_rcv_mss = icsk->icsk_ack.rcv_mss;
|
| 4284 |
|
| 4285 | info->tcpi_unacked = tp->packets_out;
|
| 4286 | info->tcpi_sacked = tp->sacked_out;
|
| 4287 |
|
| 4288 | info->tcpi_lost = tp->lost_out;
|
| 4289 | info->tcpi_retrans = tp->retrans_out;
|
| 4290 |
|
| 4291 | now = tcp_jiffies32;
|
| 4292 | info->tcpi_last_data_sent = jiffies_to_msecs(now - tp->lsndtime);
|
| 4293 | info->tcpi_last_data_recv = jiffies_to_msecs(now - icsk->icsk_ack.lrcvtime);
|
| 4294 | info->tcpi_last_ack_recv = jiffies_to_msecs(now - tp->rcv_tstamp);
|
| 4295 |
|
| 4296 | info->tcpi_pmtu = icsk->icsk_pmtu_cookie;
|
| 4297 | info->tcpi_rcv_ssthresh = tp->rcv_ssthresh;
|
| 4298 | info->tcpi_rtt = tp->srtt_us >> 3;
|
| 4299 | info->tcpi_rttvar = tp->mdev_us >> 2;
|
| 4300 | info->tcpi_snd_ssthresh = tp->snd_ssthresh;
|
| 4301 | info->tcpi_advmss = tp->advmss;
|
| 4302 |
|
| 4303 | info->tcpi_rcv_rtt = tp->rcv_rtt_est.rtt_us >> 3;
|
| 4304 | info->tcpi_rcv_space = tp->rcvq_space.space;
|
| 4305 |
|
| 4306 | info->tcpi_total_retrans = tp->total_retrans;
|
| 4307 |
|
| 4308 | info->tcpi_bytes_acked = tp->bytes_acked;
|
| 4309 | info->tcpi_bytes_received = tp->bytes_received;
|
| 4310 | info->tcpi_notsent_bytes = max_t(int, 0, tp->write_seq - tp->snd_nxt);
|
| 4311 | tcp_get_info_chrono_stats(tp, info);
|
| 4312 |
|
| 4313 | info->tcpi_segs_out = tp->segs_out;
|
| 4314 |
|
| 4315 |
|
| 4316 | info->tcpi_segs_in = READ_ONCE(tp->segs_in);
|
| 4317 | info->tcpi_data_segs_in = READ_ONCE(tp->data_segs_in);
|
| 4318 |
|
| 4319 | info->tcpi_min_rtt = tcp_min_rtt(tp);
|
| 4320 | info->tcpi_data_segs_out = tp->data_segs_out;
|
| 4321 |
|
| 4322 | info->tcpi_delivery_rate_app_limited = tp->rate_app_limited ? 1 : 0;
|
| 4323 | rate64 = tcp_compute_delivery_rate(tp);
|
| 4324 | if (rate64)
|
| 4325 | info->tcpi_delivery_rate = rate64;
|
| 4326 | info->tcpi_delivered = tp->delivered;
|
| 4327 | info->tcpi_delivered_ce = tp->delivered_ce;
|
| 4328 | info->tcpi_bytes_sent = tp->bytes_sent;
|
| 4329 | info->tcpi_bytes_retrans = tp->bytes_retrans;
|
| 4330 | info->tcpi_dsack_dups = tp->dsack_dups;
|
| 4331 | info->tcpi_reord_seen = tp->reord_seen;
|
| 4332 | info->tcpi_rcv_ooopack = tp->rcv_ooopack;
|
| 4333 | info->tcpi_snd_wnd = tp->snd_wnd;
|
| 4334 | info->tcpi_rcv_wnd = tp->rcv_wnd;
|
| 4335 | info->tcpi_rehash = tp->plb_rehash + tp->timeout_rehash;
|
| 4336 | info->tcpi_fastopen_client_fail = tp->fastopen_client_fail;
|
| 4337 |
|
| 4338 | info->tcpi_total_rto = tp->total_rto;
|
| 4339 | info->tcpi_total_rto_recoveries = tp->total_rto_recoveries;
|
| 4340 | info->tcpi_total_rto_time = tp->total_rto_time;
|
| 4341 | if (tp->rto_stamp)
|
| 4342 | info->tcpi_total_rto_time += tcp_clock_ms() - tp->rto_stamp;
|
| 4343 |
|
| 4344 | if (tcp_ecn_disabled(tp))
|
| 4345 | info->tcpi_ecn_mode = TCPI_ECN_MODE_DISABLED;
|
| 4346 | else if (tcp_ecn_mode_rfc3168(tp))
|
| 4347 | info->tcpi_ecn_mode = TCPI_ECN_MODE_RFC3168;
|
| 4348 | else if (tcp_ecn_mode_accecn(tp))
|
| 4349 | info->tcpi_ecn_mode = TCPI_ECN_MODE_ACCECN;
|
| 4350 | else if (tcp_ecn_mode_pending(tp))
|
| 4351 | info->tcpi_ecn_mode = TCPI_ECN_MODE_PENDING;
|
| 4352 | info->tcpi_accecn_fail_mode = tp->accecn_fail_mode;
|
| 4353 | info->tcpi_accecn_opt_seen = tp->saw_accecn_opt;
|
| 4354 | info->tcpi_received_ce = tp->received_ce;
|
| 4355 | info->tcpi_delivered_e1_bytes = tp->delivered_ecn_bytes[ect1_idx];
|
| 4356 | info->tcpi_delivered_e0_bytes = tp->delivered_ecn_bytes[ect0_idx];
|
| 4357 | info->tcpi_delivered_ce_bytes = tp->delivered_ecn_bytes[ce_idx];
|
| 4358 | info->tcpi_received_e1_bytes = tp->received_ecn_bytes[ect1_idx];
|
| 4359 | info->tcpi_received_e0_bytes = tp->received_ecn_bytes[ect0_idx];
|
| 4360 | info->tcpi_received_ce_bytes = tp->received_ecn_bytes[ce_idx];
|
| 4361 |
|
| 4362 | unlock_sock_fast(sk, slow);
|
| 4363 | }
|
| 4364 | EXPORT_SYMBOL_GPL(tcp_get_info);
|
| 4365 |
|
| 4366 | static size_t tcp_opt_stats_get_size(void)
|
| 4367 | {
|
| 4368 | return
|
| 4369 | nla_total_size_64bit(sizeof(u64)) +
|
| 4370 | nla_total_size_64bit(sizeof(u64)) +
|
| 4371 | nla_total_size_64bit(sizeof(u64)) +
|
| 4372 | nla_total_size_64bit(sizeof(u64)) +
|
| 4373 | nla_total_size_64bit(sizeof(u64)) +
|
| 4374 | nla_total_size_64bit(sizeof(u64)) +
|
| 4375 | nla_total_size_64bit(sizeof(u64)) +
|
| 4376 | nla_total_size(sizeof(u32)) +
|
| 4377 | nla_total_size(sizeof(u32)) +
|
| 4378 | nla_total_size(sizeof(u32)) +
|
| 4379 | nla_total_size(sizeof(u8)) +
|
| 4380 | nla_total_size(sizeof(u8)) +
|
| 4381 | nla_total_size(sizeof(u32)) +
|
| 4382 | nla_total_size(sizeof(u8)) +
|
| 4383 | nla_total_size(sizeof(u32)) +
|
| 4384 | nla_total_size(sizeof(u32)) +
|
| 4385 | nla_total_size(sizeof(u32)) +
|
| 4386 | nla_total_size_64bit(sizeof(u64)) +
|
| 4387 | nla_total_size_64bit(sizeof(u64)) +
|
| 4388 | nla_total_size(sizeof(u32)) +
|
| 4389 | nla_total_size(sizeof(u32)) +
|
| 4390 | nla_total_size(sizeof(u32)) +
|
| 4391 | nla_total_size(sizeof(u16)) +
|
| 4392 | nla_total_size(sizeof(u32)) +
|
| 4393 | nla_total_size_64bit(sizeof(u64)) +
|
| 4394 | nla_total_size(sizeof(u8)) +
|
| 4395 | nla_total_size(sizeof(u32)) +
|
| 4396 | 0;
|
| 4397 | }
|
| 4398 |
|
| 4399 |
|
| 4400 | static u8 tcp_skb_ttl_or_hop_limit(const struct sk_buff *skb)
|
| 4401 | {
|
| 4402 | if (skb->protocol == htons(ETH_P_IP))
|
| 4403 | return ip_hdr(skb)->ttl;
|
| 4404 | else if (skb->protocol == htons(ETH_P_IPV6))
|
| 4405 | return ipv6_hdr(skb)->hop_limit;
|
| 4406 | else
|
| 4407 | return 0;
|
| 4408 | }
|
| 4409 |
|
| 4410 | struct sk_buff *tcp_get_timestamping_opt_stats(const struct sock *sk,
|
| 4411 | const struct sk_buff *orig_skb,
|
| 4412 | const struct sk_buff *ack_skb)
|
| 4413 | {
|
| 4414 | const struct tcp_sock *tp = tcp_sk(sk);
|
| 4415 | struct sk_buff *stats;
|
| 4416 | struct tcp_info info;
|
| 4417 | unsigned long rate;
|
| 4418 | u64 rate64;
|
| 4419 |
|
| 4420 | stats = alloc_skb(tcp_opt_stats_get_size(), GFP_ATOMIC);
|
| 4421 | if (!stats)
|
| 4422 | return NULL;
|
| 4423 |
|
| 4424 | tcp_get_info_chrono_stats(tp, &info);
|
| 4425 | nla_put_u64_64bit(stats, TCP_NLA_BUSY,
|
| 4426 | info.tcpi_busy_time, TCP_NLA_PAD);
|
| 4427 | nla_put_u64_64bit(stats, TCP_NLA_RWND_LIMITED,
|
| 4428 | info.tcpi_rwnd_limited, TCP_NLA_PAD);
|
| 4429 | nla_put_u64_64bit(stats, TCP_NLA_SNDBUF_LIMITED,
|
| 4430 | info.tcpi_sndbuf_limited, TCP_NLA_PAD);
|
| 4431 | nla_put_u64_64bit(stats, TCP_NLA_DATA_SEGS_OUT,
|
| 4432 | READ_ONCE(tp->data_segs_out), TCP_NLA_PAD);
|
| 4433 | nla_put_u64_64bit(stats, TCP_NLA_TOTAL_RETRANS,
|
| 4434 | READ_ONCE(tp->total_retrans), TCP_NLA_PAD);
|
| 4435 |
|
| 4436 | rate = READ_ONCE(sk->sk_pacing_rate);
|
| 4437 | rate64 = (rate != ~0UL) ? rate : ~0ULL;
|
| 4438 | nla_put_u64_64bit(stats, TCP_NLA_PACING_RATE, rate64, TCP_NLA_PAD);
|
| 4439 |
|
| 4440 | rate64 = tcp_compute_delivery_rate(tp);
|
| 4441 | nla_put_u64_64bit(stats, TCP_NLA_DELIVERY_RATE, rate64, TCP_NLA_PAD);
|
| 4442 |
|
| 4443 | nla_put_u32(stats, TCP_NLA_SND_CWND, READ_ONCE(tp->snd_cwnd));
|
| 4444 | nla_put_u32(stats, TCP_NLA_REORDERING, READ_ONCE(tp->reordering));
|
| 4445 | nla_put_u32(stats, TCP_NLA_MIN_RTT, data_race(tcp_min_rtt(tp)));
|
| 4446 |
|
| 4447 | nla_put_u8(stats, TCP_NLA_RECUR_RETRANS,
|
| 4448 | READ_ONCE(inet_csk(sk)->icsk_retransmits));
|
| 4449 | nla_put_u8(stats, TCP_NLA_DELIVERY_RATE_APP_LMT, data_race(!!tp->rate_app_limited));
|
| 4450 | nla_put_u32(stats, TCP_NLA_SND_SSTHRESH, READ_ONCE(tp->snd_ssthresh));
|
| 4451 | nla_put_u32(stats, TCP_NLA_DELIVERED, READ_ONCE(tp->delivered));
|
| 4452 | nla_put_u32(stats, TCP_NLA_DELIVERED_CE, READ_ONCE(tp->delivered_ce));
|
| 4453 |
|
| 4454 | nla_put_u32(stats, TCP_NLA_SNDQ_SIZE,
|
| 4455 | max_t(int, 0,
|
| 4456 | READ_ONCE(tp->write_seq) - READ_ONCE(tp->snd_una)));
|
| 4457 | nla_put_u8(stats, TCP_NLA_CA_STATE, inet_csk(sk)->icsk_ca_state);
|
| 4458 |
|
| 4459 | nla_put_u64_64bit(stats, TCP_NLA_BYTES_SENT, READ_ONCE(tp->bytes_sent),
|
| 4460 | TCP_NLA_PAD);
|
| 4461 | nla_put_u64_64bit(stats, TCP_NLA_BYTES_RETRANS,
|
| 4462 | READ_ONCE(tp->bytes_retrans), TCP_NLA_PAD);
|
| 4463 | nla_put_u32(stats, TCP_NLA_DSACK_DUPS, READ_ONCE(tp->dsack_dups));
|
| 4464 | nla_put_u32(stats, TCP_NLA_REORD_SEEN, READ_ONCE(tp->reord_seen));
|
| 4465 | nla_put_u32(stats, TCP_NLA_SRTT, READ_ONCE(tp->srtt_us) >> 3);
|
| 4466 | nla_put_u16(stats, TCP_NLA_TIMEOUT_REHASH,
|
| 4467 | READ_ONCE(tp->timeout_rehash));
|
| 4468 | nla_put_u32(stats, TCP_NLA_BYTES_NOTSENT,
|
| 4469 | max_t(int, 0,
|
| 4470 | READ_ONCE(tp->write_seq) - READ_ONCE(tp->snd_nxt)));
|
| 4471 | nla_put_u64_64bit(stats, TCP_NLA_EDT, orig_skb->skb_mstamp_ns,
|
| 4472 | TCP_NLA_PAD);
|
| 4473 | if (ack_skb)
|
| 4474 | nla_put_u8(stats, TCP_NLA_TTL,
|
| 4475 | tcp_skb_ttl_or_hop_limit(ack_skb));
|
| 4476 |
|
| 4477 | nla_put_u32(stats, TCP_NLA_REHASH,
|
| 4478 | READ_ONCE(tp->plb_rehash) + READ_ONCE(tp->timeout_rehash));
|
| 4479 | return stats;
|
| 4480 | }
|
| 4481 |
|
| 4482 | int do_tcp_getsockopt(struct sock *sk, int level,
|
| 4483 | int optname, sockptr_t optval, sockptr_t optlen)
|
| 4484 | {
|
| 4485 | struct inet_connection_sock *icsk = inet_csk(sk);
|
| 4486 | struct tcp_sock *tp = tcp_sk(sk);
|
| 4487 | struct net *net = sock_net(sk);
|
| 4488 | int user_mss;
|
| 4489 | int val, len;
|
| 4490 |
|
| 4491 | if (copy_from_sockptr(&len, optlen, sizeof(int)))
|
| 4492 | return -EFAULT;
|
| 4493 |
|
| 4494 | if (len < 0)
|
| 4495 | return -EINVAL;
|
| 4496 |
|
| 4497 | len = min_t(unsigned int, len, sizeof(int));
|
| 4498 |
|
| 4499 | switch (optname) {
|
| 4500 | case TCP_MAXSEG:
|
| 4501 | val = tp->mss_cache;
|
| 4502 | user_mss = READ_ONCE(tp->rx_opt.user_mss);
|
| 4503 | if (user_mss &&
|
| 4504 | ((1 << sk->sk_state) & (TCPF_CLOSE | TCPF_LISTEN)))
|
| 4505 | val = user_mss;
|
| 4506 | if (tp->repair)
|
| 4507 | val = tp->rx_opt.mss_clamp;
|
| 4508 | break;
|
| 4509 | case TCP_NODELAY:
|
| 4510 | val = !!(tp->nonagle&TCP_NAGLE_OFF);
|
| 4511 | break;
|
| 4512 | case TCP_CORK:
|
| 4513 | val = !!(tp->nonagle&TCP_NAGLE_CORK);
|
| 4514 | break;
|
| 4515 | case TCP_KEEPIDLE:
|
| 4516 | val = keepalive_time_when(tp) / HZ;
|
| 4517 | break;
|
| 4518 | case TCP_KEEPINTVL:
|
| 4519 | val = keepalive_intvl_when(tp) / HZ;
|
| 4520 | break;
|
| 4521 | case TCP_KEEPCNT:
|
| 4522 | val = keepalive_probes(tp);
|
| 4523 | break;
|
| 4524 | case TCP_SYNCNT:
|
| 4525 | val = READ_ONCE(icsk->icsk_syn_retries) ? :
|
| 4526 | READ_ONCE(net->ipv4.sysctl_tcp_syn_retries);
|
| 4527 | break;
|
| 4528 | case TCP_LINGER2:
|
| 4529 | val = READ_ONCE(tp->linger2);
|
| 4530 | if (val >= 0)
|
| 4531 | val = (val ? : READ_ONCE(net->ipv4.sysctl_tcp_fin_timeout)) / HZ;
|
| 4532 | break;
|
| 4533 | case TCP_DEFER_ACCEPT:
|
| 4534 | val = READ_ONCE(icsk->icsk_accept_queue.rskq_defer_accept);
|
| 4535 | val = retrans_to_secs(val, TCP_TIMEOUT_INIT / HZ,
|
| 4536 | TCP_RTO_MAX / HZ);
|
| 4537 | break;
|
| 4538 | case TCP_WINDOW_CLAMP:
|
| 4539 | val = READ_ONCE(tp->window_clamp);
|
| 4540 | break;
|
| 4541 | case TCP_INFO: {
|
| 4542 | struct tcp_info info;
|
| 4543 |
|
| 4544 | if (copy_from_sockptr(&len, optlen, sizeof(int)))
|
| 4545 | return -EFAULT;
|
| 4546 |
|
| 4547 | tcp_get_info(sk, &info);
|
| 4548 |
|
| 4549 | len = min_t(unsigned int, len, sizeof(info));
|
| 4550 | if (copy_to_sockptr(optlen, &len, sizeof(int)))
|
| 4551 | return -EFAULT;
|
| 4552 | if (copy_to_sockptr(optval, &info, len))
|
| 4553 | return -EFAULT;
|
| 4554 | return 0;
|
| 4555 | }
|
| 4556 | case TCP_CC_INFO: {
|
| 4557 | const struct tcp_congestion_ops *ca_ops;
|
| 4558 | union tcp_cc_info info;
|
| 4559 | size_t sz = 0;
|
| 4560 | int attr;
|
| 4561 |
|
| 4562 | if (copy_from_sockptr(&len, optlen, sizeof(int)))
|
| 4563 | return -EFAULT;
|
| 4564 |
|
| 4565 | rcu_read_lock();
|
| 4566 | ca_ops = READ_ONCE(icsk->icsk_ca_ops);
|
| 4567 | if (ca_ops && ca_ops->get_info)
|
| 4568 | sz = ca_ops->get_info(sk, ~0U, &attr, &info);
|
| 4569 | rcu_read_unlock();
|
| 4570 |
|
| 4571 | len = min_t(unsigned int, len, sz);
|
| 4572 | if (copy_to_sockptr(optlen, &len, sizeof(int)))
|
| 4573 | return -EFAULT;
|
| 4574 | if (copy_to_sockptr(optval, &info, len))
|
| 4575 | return -EFAULT;
|
| 4576 | return 0;
|
| 4577 | }
|
| 4578 | case TCP_QUICKACK:
|
| 4579 | val = !inet_csk_in_pingpong_mode(sk);
|
| 4580 | break;
|
| 4581 |
|
| 4582 | case TCP_CONGESTION: {
|
| 4583 | char ca_name[TCP_CA_NAME_MAX] = {};
|
| 4584 |
|
| 4585 | if (copy_from_sockptr(&len, optlen, sizeof(int)))
|
| 4586 | return -EFAULT;
|
| 4587 | len = min_t(unsigned int, len, TCP_CA_NAME_MAX);
|
| 4588 | if (copy_to_sockptr(optlen, &len, sizeof(int)))
|
| 4589 | return -EFAULT;
|
| 4590 |
|
| 4591 | rcu_read_lock();
|
| 4592 | memcpy(ca_name, READ_ONCE(icsk->icsk_ca_ops)->name,
|
| 4593 | sizeof(ca_name));
|
| 4594 | rcu_read_unlock();
|
| 4595 |
|
| 4596 | if (copy_to_sockptr(optval, ca_name, len))
|
| 4597 | return -EFAULT;
|
| 4598 | return 0;
|
| 4599 | }
|
| 4600 | case TCP_ULP:
|
| 4601 | if (copy_from_sockptr(&len, optlen, sizeof(int)))
|
| 4602 | return -EFAULT;
|
| 4603 | len = min_t(unsigned int, len, TCP_ULP_NAME_MAX);
|
| 4604 | if (!icsk->icsk_ulp_ops) {
|
| 4605 | len = 0;
|
| 4606 | if (copy_to_sockptr(optlen, &len, sizeof(int)))
|
| 4607 | return -EFAULT;
|
| 4608 | return 0;
|
| 4609 | }
|
| 4610 | if (copy_to_sockptr(optlen, &len, sizeof(int)))
|
| 4611 | return -EFAULT;
|
| 4612 | if (copy_to_sockptr(optval, icsk->icsk_ulp_ops->name, len))
|
| 4613 | return -EFAULT;
|
| 4614 | return 0;
|
| 4615 |
|
| 4616 | case TCP_FASTOPEN_KEY: {
|
| 4617 | u64 key[TCP_FASTOPEN_KEY_BUF_LENGTH / sizeof(u64)];
|
| 4618 | unsigned int key_len;
|
| 4619 |
|
| 4620 | if (copy_from_sockptr(&len, optlen, sizeof(int)))
|
| 4621 | return -EFAULT;
|
| 4622 |
|
| 4623 | key_len = tcp_fastopen_get_cipher(net, icsk, key) *
|
| 4624 | TCP_FASTOPEN_KEY_LENGTH;
|
| 4625 | len = min_t(unsigned int, len, key_len);
|
| 4626 | if (copy_to_sockptr(optlen, &len, sizeof(int)))
|
| 4627 | return -EFAULT;
|
| 4628 | if (copy_to_sockptr(optval, key, len))
|
| 4629 | return -EFAULT;
|
| 4630 | return 0;
|
| 4631 | }
|
| 4632 | case TCP_THIN_LINEAR_TIMEOUTS:
|
| 4633 | val = tp->thin_lto;
|
| 4634 | break;
|
| 4635 |
|
| 4636 | case TCP_THIN_DUPACK:
|
| 4637 | val = 0;
|
| 4638 | break;
|
| 4639 |
|
| 4640 | case TCP_REPAIR:
|
| 4641 | val = tp->repair;
|
| 4642 | break;
|
| 4643 |
|
| 4644 | case TCP_REPAIR_QUEUE:
|
| 4645 | if (tp->repair)
|
| 4646 | val = tp->repair_queue;
|
| 4647 | else
|
| 4648 | return -EINVAL;
|
| 4649 | break;
|
| 4650 |
|
| 4651 | case TCP_REPAIR_WINDOW: {
|
| 4652 | struct tcp_repair_window opt;
|
| 4653 |
|
| 4654 | if (copy_from_sockptr(&len, optlen, sizeof(int)))
|
| 4655 | return -EFAULT;
|
| 4656 |
|
| 4657 | if (len != sizeof(opt))
|
| 4658 | return -EINVAL;
|
| 4659 |
|
| 4660 | if (!tp->repair)
|
| 4661 | return -EPERM;
|
| 4662 |
|
| 4663 | opt.snd_wl1 = tp->snd_wl1;
|
| 4664 | opt.snd_wnd = tp->snd_wnd;
|
| 4665 | opt.max_window = tp->max_window;
|
| 4666 | opt.rcv_wnd = tp->rcv_wnd;
|
| 4667 | opt.rcv_wup = tp->rcv_wup;
|
| 4668 |
|
| 4669 | if (copy_to_sockptr(optval, &opt, len))
|
| 4670 | return -EFAULT;
|
| 4671 | return 0;
|
| 4672 | }
|
| 4673 | case TCP_QUEUE_SEQ:
|
| 4674 | if (tp->repair_queue == TCP_SEND_QUEUE)
|
| 4675 | val = tp->write_seq;
|
| 4676 | else if (tp->repair_queue == TCP_RECV_QUEUE)
|
| 4677 | val = tp->rcv_nxt;
|
| 4678 | else
|
| 4679 | return -EINVAL;
|
| 4680 | break;
|
| 4681 |
|
| 4682 | case TCP_USER_TIMEOUT:
|
| 4683 | val = READ_ONCE(icsk->icsk_user_timeout);
|
| 4684 | break;
|
| 4685 |
|
| 4686 | case TCP_FASTOPEN:
|
| 4687 | val = READ_ONCE(icsk->icsk_accept_queue.fastopenq.max_qlen);
|
| 4688 | break;
|
| 4689 |
|
| 4690 | case TCP_FASTOPEN_CONNECT:
|
| 4691 | val = tp->fastopen_connect;
|
| 4692 | break;
|
| 4693 |
|
| 4694 | case TCP_FASTOPEN_NO_COOKIE:
|
| 4695 | val = tp->fastopen_no_cookie;
|
| 4696 | break;
|
| 4697 |
|
| 4698 | case TCP_TX_DELAY:
|
| 4699 | val = READ_ONCE(tp->tcp_tx_delay);
|
| 4700 | break;
|
| 4701 |
|
| 4702 | case TCP_TIMESTAMP:
|
| 4703 | val = tcp_clock_ts(tp->tcp_usec_ts) + READ_ONCE(tp->tsoffset);
|
| 4704 | if (tp->tcp_usec_ts)
|
| 4705 | val |= 1;
|
| 4706 | else
|
| 4707 | val &= ~1;
|
| 4708 | break;
|
| 4709 | case TCP_NOTSENT_LOWAT:
|
| 4710 | val = READ_ONCE(tp->notsent_lowat);
|
| 4711 | break;
|
| 4712 | case TCP_INQ:
|
| 4713 | val = tp->recvmsg_inq;
|
| 4714 | break;
|
| 4715 | case TCP_SAVE_SYN:
|
| 4716 | val = tp->save_syn;
|
| 4717 | break;
|
| 4718 | case TCP_SAVED_SYN: {
|
| 4719 | if (copy_from_sockptr(&len, optlen, sizeof(int)))
|
| 4720 | return -EFAULT;
|
| 4721 |
|
| 4722 | sockopt_lock_sock(sk);
|
| 4723 | if (tp->saved_syn) {
|
| 4724 | if (len < tcp_saved_syn_len(tp->saved_syn)) {
|
| 4725 | len = tcp_saved_syn_len(tp->saved_syn);
|
| 4726 | if (copy_to_sockptr(optlen, &len, sizeof(int))) {
|
| 4727 | sockopt_release_sock(sk);
|
| 4728 | return -EFAULT;
|
| 4729 | }
|
| 4730 | sockopt_release_sock(sk);
|
| 4731 | return -EINVAL;
|
| 4732 | }
|
| 4733 | len = tcp_saved_syn_len(tp->saved_syn);
|
| 4734 | if (copy_to_sockptr(optlen, &len, sizeof(int))) {
|
| 4735 | sockopt_release_sock(sk);
|
| 4736 | return -EFAULT;
|
| 4737 | }
|
| 4738 | if (copy_to_sockptr(optval, tp->saved_syn->data, len)) {
|
| 4739 | sockopt_release_sock(sk);
|
| 4740 | return -EFAULT;
|
| 4741 | }
|
| 4742 | tcp_saved_syn_free(tp);
|
| 4743 | sockopt_release_sock(sk);
|
| 4744 | } else {
|
| 4745 | sockopt_release_sock(sk);
|
| 4746 | len = 0;
|
| 4747 | if (copy_to_sockptr(optlen, &len, sizeof(int)))
|
| 4748 | return -EFAULT;
|
| 4749 | }
|
| 4750 | return 0;
|
| 4751 | }
|
| 4752 | #ifdef CONFIG_MMU
|
| 4753 | case TCP_ZEROCOPY_RECEIVE: {
|
| 4754 | struct scm_timestamping_internal tss;
|
| 4755 | struct tcp_zerocopy_receive zc = {};
|
| 4756 | int err;
|
| 4757 |
|
| 4758 | if (copy_from_sockptr(&len, optlen, sizeof(int)))
|
| 4759 | return -EFAULT;
|
| 4760 | if (len < 0 ||
|
| 4761 | len < offsetofend(struct tcp_zerocopy_receive, length))
|
| 4762 | return -EINVAL;
|
| 4763 | if (unlikely(len > sizeof(zc))) {
|
| 4764 | err = check_zeroed_sockptr(optval, sizeof(zc),
|
| 4765 | len - sizeof(zc));
|
| 4766 | if (err < 1)
|
| 4767 | return err == 0 ? -EINVAL : err;
|
| 4768 | len = sizeof(zc);
|
| 4769 | if (copy_to_sockptr(optlen, &len, sizeof(int)))
|
| 4770 | return -EFAULT;
|
| 4771 | }
|
| 4772 | if (copy_from_sockptr(&zc, optval, len))
|
| 4773 | return -EFAULT;
|
| 4774 | if (zc.reserved)
|
| 4775 | return -EINVAL;
|
| 4776 | if (zc.msg_flags & ~(TCP_VALID_ZC_MSG_FLAGS))
|
| 4777 | return -EINVAL;
|
| 4778 | sockopt_lock_sock(sk);
|
| 4779 | err = tcp_zerocopy_receive(sk, &zc, &tss);
|
| 4780 | err = BPF_CGROUP_RUN_PROG_GETSOCKOPT_KERN(sk, level, optname,
|
| 4781 | &zc, &len, err);
|
| 4782 | sockopt_release_sock(sk);
|
| 4783 | if (len >= offsetofend(struct tcp_zerocopy_receive, msg_flags))
|
| 4784 | goto zerocopy_rcv_cmsg;
|
| 4785 | switch (len) {
|
| 4786 | case offsetofend(struct tcp_zerocopy_receive, msg_flags):
|
| 4787 | goto zerocopy_rcv_cmsg;
|
| 4788 | case offsetofend(struct tcp_zerocopy_receive, msg_controllen):
|
| 4789 | case offsetofend(struct tcp_zerocopy_receive, msg_control):
|
| 4790 | case offsetofend(struct tcp_zerocopy_receive, flags):
|
| 4791 | case offsetofend(struct tcp_zerocopy_receive, copybuf_len):
|
| 4792 | case offsetofend(struct tcp_zerocopy_receive, copybuf_address):
|
| 4793 | case offsetofend(struct tcp_zerocopy_receive, err):
|
| 4794 | goto zerocopy_rcv_sk_err;
|
| 4795 | case offsetofend(struct tcp_zerocopy_receive, inq):
|
| 4796 | goto zerocopy_rcv_inq;
|
| 4797 | case offsetofend(struct tcp_zerocopy_receive, length):
|
| 4798 | default:
|
| 4799 | goto zerocopy_rcv_out;
|
| 4800 | }
|
| 4801 | zerocopy_rcv_cmsg:
|
| 4802 | if (zc.msg_flags & TCP_CMSG_TS)
|
| 4803 | tcp_zc_finalize_rx_tstamp(sk, &zc, &tss);
|
| 4804 | else
|
| 4805 | zc.msg_flags = 0;
|
| 4806 | zerocopy_rcv_sk_err:
|
| 4807 | if (!err)
|
| 4808 | zc.err = sock_error(sk);
|
| 4809 | zerocopy_rcv_inq:
|
| 4810 | zc.inq = tcp_inq_hint(sk);
|
| 4811 | zerocopy_rcv_out:
|
| 4812 | if (!err && copy_to_sockptr(optval, &zc, len))
|
| 4813 | err = -EFAULT;
|
| 4814 | return err;
|
| 4815 | }
|
| 4816 | #endif
|
| 4817 | case TCP_AO_REPAIR:
|
| 4818 | if (!tcp_can_repair_sock(sk))
|
| 4819 | return -EPERM;
|
| 4820 | return tcp_ao_get_repair(sk, optval, optlen);
|
| 4821 | case TCP_AO_GET_KEYS:
|
| 4822 | case TCP_AO_INFO: {
|
| 4823 | int err;
|
| 4824 |
|
| 4825 | sockopt_lock_sock(sk);
|
| 4826 | if (optname == TCP_AO_GET_KEYS)
|
| 4827 | err = tcp_ao_get_mkts(sk, optval, optlen);
|
| 4828 | else
|
| 4829 | err = tcp_ao_get_sock_info(sk, optval, optlen);
|
| 4830 | sockopt_release_sock(sk);
|
| 4831 |
|
| 4832 | return err;
|
| 4833 | }
|
| 4834 | case TCP_IS_MPTCP:
|
| 4835 | val = 0;
|
| 4836 | break;
|
| 4837 | case TCP_RTO_MAX_MS:
|
| 4838 | val = jiffies_to_msecs(tcp_rto_max(sk));
|
| 4839 | break;
|
| 4840 | case TCP_RTO_MIN_US:
|
| 4841 | val = jiffies_to_usecs(READ_ONCE(inet_csk(sk)->icsk_rto_min));
|
| 4842 | break;
|
| 4843 | case TCP_DELACK_MAX_US:
|
| 4844 | val = jiffies_to_usecs(READ_ONCE(inet_csk(sk)->icsk_delack_max));
|
| 4845 | break;
|
| 4846 | default:
|
| 4847 | return -ENOPROTOOPT;
|
| 4848 | }
|
| 4849 |
|
| 4850 | if (copy_to_sockptr(optlen, &len, sizeof(int)))
|
| 4851 | return -EFAULT;
|
| 4852 | if (copy_to_sockptr(optval, &val, len))
|
| 4853 | return -EFAULT;
|
| 4854 | return 0;
|
| 4855 | }
|
| 4856 |
|
| 4857 | bool tcp_bpf_bypass_getsockopt(int level, int optname)
|
| 4858 | {
|
| 4859 | |
| 4860 | * to avoid extra socket lock for TCP_ZEROCOPY_RECEIVE.
|
| 4861 |
|
| 4862 | if (level == SOL_TCP && optname == TCP_ZEROCOPY_RECEIVE)
|
| 4863 | return true;
|
| 4864 |
|
| 4865 | return false;
|
| 4866 | }
|
| 4867 |
|
| 4868 | int tcp_getsockopt(struct sock *sk, int level, int optname, char __user *optval,
|
| 4869 | int __user *optlen)
|
| 4870 | {
|
| 4871 | struct inet_connection_sock *icsk = inet_csk(sk);
|
| 4872 |
|
| 4873 | if (level != SOL_TCP)
|
| 4874 |
|
| 4875 | return READ_ONCE(icsk->icsk_af_ops)->getsockopt(sk, level, optname,
|
| 4876 | optval, optlen);
|
| 4877 | return do_tcp_getsockopt(sk, level, optname, USER_SOCKPTR(optval),
|
| 4878 | USER_SOCKPTR(optlen));
|
| 4879 | }
|
| 4880 |
|
| 4881 | #ifdef CONFIG_TCP_MD5SIG
|
| 4882 | void tcp_md5_hash_skb_data(struct md5_ctx *ctx, const struct sk_buff *skb,
|
| 4883 | unsigned int header_len)
|
| 4884 | {
|
| 4885 | const unsigned int head_data_len = skb_headlen(skb) > header_len ?
|
| 4886 | skb_headlen(skb) - header_len : 0;
|
| 4887 | const struct skb_shared_info *shi = skb_shinfo(skb);
|
| 4888 | struct sk_buff *frag_iter;
|
| 4889 | unsigned int i;
|
| 4890 |
|
| 4891 | md5_update(ctx, (const u8 *)tcp_hdr(skb) + header_len, head_data_len);
|
| 4892 |
|
| 4893 | for (i = 0; i < shi->nr_frags; ++i) {
|
| 4894 | const skb_frag_t *f = &shi->frags[i];
|
| 4895 | u32 p_off, p_len, copied;
|
| 4896 | const void *vaddr;
|
| 4897 | struct page *p;
|
| 4898 |
|
| 4899 | skb_frag_foreach_page(f, skb_frag_off(f), skb_frag_size(f),
|
| 4900 | p, p_off, p_len, copied) {
|
| 4901 | vaddr = kmap_local_page(p);
|
| 4902 | md5_update(ctx, vaddr + p_off, p_len);
|
| 4903 | kunmap_local(vaddr);
|
| 4904 | }
|
| 4905 | }
|
| 4906 |
|
| 4907 | skb_walk_frags(skb, frag_iter)
|
| 4908 | tcp_md5_hash_skb_data(ctx, frag_iter, 0);
|
| 4909 | }
|
| 4910 |
|
| 4911 | void tcp_md5_hash_key(struct md5_ctx *ctx,
|
| 4912 | const struct tcp_md5sig_key *key)
|
| 4913 | {
|
| 4914 | u8 keylen = READ_ONCE(key->keylen);
|
| 4915 |
|
| 4916 | |
| 4917 | * key->key under us
|
| 4918 |
|
| 4919 | data_race(({ md5_update(ctx, key->key, keylen), 0; }));
|
| 4920 | }
|
| 4921 |
|
| 4922 |
|
| 4923 | static enum skb_drop_reason
|
| 4924 | tcp_inbound_md5_hash(const struct sock *sk, const struct sk_buff *skb,
|
| 4925 | const void *saddr, const void *daddr,
|
| 4926 | int family, int l3index, const __u8 *hash_location)
|
| 4927 | {
|
| 4928 | |
| 4929 | * We have 2 drop cases:
|
| 4930 | * o An MD5 signature is present, but we're not expecting one.
|
| 4931 | * o The MD5 signature is wrong.
|
| 4932 |
|
| 4933 | const struct tcp_sock *tp = tcp_sk(sk);
|
| 4934 | struct tcp_md5sig_key *key;
|
| 4935 | u8 newhash[16];
|
| 4936 |
|
| 4937 | key = tcp_md5_do_lookup(sk, l3index, saddr, family);
|
| 4938 | if (!key) {
|
| 4939 | NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPMD5UNEXPECTED);
|
| 4940 | trace_tcp_hash_md5_unexpected(sk, skb);
|
| 4941 | return SKB_DROP_REASON_TCP_MD5UNEXPECTED;
|
| 4942 | }
|
| 4943 |
|
| 4944 | |
| 4945 | * To support dual stack listeners, we need to handle
|
| 4946 | * IPv4-mapped case.
|
| 4947 |
|
| 4948 | if (family == AF_INET)
|
| 4949 | tcp_v4_md5_hash_skb(newhash, key, NULL, skb);
|
| 4950 | else
|
| 4951 | tp->af_specific->calc_md5_hash(newhash, key, NULL, skb);
|
| 4952 | if (crypto_memneq(hash_location, newhash, 16)) {
|
| 4953 | NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPMD5FAILURE);
|
| 4954 | trace_tcp_hash_md5_mismatch(sk, skb);
|
| 4955 | return SKB_DROP_REASON_TCP_MD5FAILURE;
|
| 4956 | }
|
| 4957 | return SKB_NOT_DROPPED_YET;
|
| 4958 | }
|
| 4959 | #else
|
| 4960 | static inline enum skb_drop_reason
|
| 4961 | tcp_inbound_md5_hash(const struct sock *sk, const struct sk_buff *skb,
|
| 4962 | const void *saddr, const void *daddr,
|
| 4963 | int family, int l3index, const __u8 *hash_location)
|
| 4964 | {
|
| 4965 | return SKB_NOT_DROPPED_YET;
|
| 4966 | }
|
| 4967 |
|
| 4968 | #endif
|
| 4969 |
|
| 4970 | #if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO)
|
| 4971 | |
| 4972 | * Parse Signature options
|
| 4973 |
|
| 4974 | int tcp_do_parse_auth_options(const struct tcphdr *th,
|
| 4975 | const u8 **md5_hash, const u8 **ao_hash)
|
| 4976 | {
|
| 4977 | int length = (th->doff << 2) - sizeof(*th);
|
| 4978 | const u8 *ptr = (const u8 *)(th + 1);
|
| 4979 | unsigned int minlen = TCPOLEN_MD5SIG;
|
| 4980 |
|
| 4981 | if (IS_ENABLED(CONFIG_TCP_AO))
|
| 4982 | minlen = sizeof(struct tcp_ao_hdr) + 1;
|
| 4983 |
|
| 4984 | *md5_hash = NULL;
|
| 4985 | *ao_hash = NULL;
|
| 4986 |
|
| 4987 |
|
| 4988 | while (length >= minlen) {
|
| 4989 | int opcode = *ptr++;
|
| 4990 | int opsize;
|
| 4991 |
|
| 4992 | switch (opcode) {
|
| 4993 | case TCPOPT_EOL:
|
| 4994 | return 0;
|
| 4995 | case TCPOPT_NOP:
|
| 4996 | length--;
|
| 4997 | continue;
|
| 4998 | default:
|
| 4999 | opsize = *ptr++;
|
| 5000 | if (opsize < 2 || opsize > length)
|
| 5001 | return -EINVAL;
|
| 5002 | if (opcode == TCPOPT_MD5SIG) {
|
| 5003 | if (opsize != TCPOLEN_MD5SIG)
|
| 5004 | return -EINVAL;
|
| 5005 | if (unlikely(*md5_hash || *ao_hash))
|
| 5006 | return -EEXIST;
|
| 5007 | *md5_hash = ptr;
|
| 5008 | } else if (opcode == TCPOPT_AO) {
|
| 5009 | if (opsize <= sizeof(struct tcp_ao_hdr))
|
| 5010 | return -EINVAL;
|
| 5011 | if (unlikely(*md5_hash || *ao_hash))
|
| 5012 | return -EEXIST;
|
| 5013 | *ao_hash = ptr;
|
| 5014 | }
|
| 5015 | }
|
| 5016 | ptr += opsize - 2;
|
| 5017 | length -= opsize;
|
| 5018 | }
|
| 5019 | return 0;
|
| 5020 | }
|
| 5021 | #endif
|
| 5022 |
|
| 5023 |
|
| 5024 | enum skb_drop_reason
|
| 5025 | tcp_inbound_hash(struct sock *sk, const struct request_sock *req,
|
| 5026 | const struct sk_buff *skb,
|
| 5027 | const void *saddr, const void *daddr,
|
| 5028 | int family, int dif, int sdif)
|
| 5029 | {
|
| 5030 | const struct tcphdr *th = tcp_hdr(skb);
|
| 5031 | const struct tcp_ao_hdr *aoh;
|
| 5032 | const __u8 *md5_location;
|
| 5033 | int l3index;
|
| 5034 |
|
| 5035 |
|
| 5036 | if (tcp_parse_auth_options(th, &md5_location, &aoh)) {
|
| 5037 | trace_tcp_hash_bad_header(sk, skb);
|
| 5038 | return SKB_DROP_REASON_TCP_AUTH_HDR;
|
| 5039 | }
|
| 5040 |
|
| 5041 | if (req) {
|
| 5042 | if (tcp_rsk_used_ao(req) != !!aoh) {
|
| 5043 | u8 keyid, rnext, maclen;
|
| 5044 |
|
| 5045 | if (aoh) {
|
| 5046 | keyid = aoh->keyid;
|
| 5047 | rnext = aoh->rnext_keyid;
|
| 5048 | maclen = tcp_ao_hdr_maclen(aoh);
|
| 5049 | } else {
|
| 5050 | keyid = rnext = maclen = 0;
|
| 5051 | }
|
| 5052 |
|
| 5053 | NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPAOBAD);
|
| 5054 | trace_tcp_ao_handshake_failure(sk, skb, keyid, rnext, maclen);
|
| 5055 | return SKB_DROP_REASON_TCP_AOFAILURE;
|
| 5056 | }
|
| 5057 | }
|
| 5058 |
|
| 5059 | |
| 5060 | * in an L3 domain and dif is set to the l3mdev
|
| 5061 |
|
| 5062 | l3index = sdif ? dif : 0;
|
| 5063 |
|
| 5064 |
|
| 5065 | if (likely(!md5_location && !aoh)) {
|
| 5066 | |
| 5067 | * for the remote peer. On TCP-AO established connection
|
| 5068 | * the last key is impossible to remove, so there's
|
| 5069 | * always at least one current_key.
|
| 5070 |
|
| 5071 | if (tcp_ao_required(sk, saddr, family, l3index, true)) {
|
| 5072 | trace_tcp_hash_ao_required(sk, skb);
|
| 5073 | return SKB_DROP_REASON_TCP_AONOTFOUND;
|
| 5074 | }
|
| 5075 | if (unlikely(tcp_md5_do_lookup(sk, l3index, saddr, family))) {
|
| 5076 | NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPMD5NOTFOUND);
|
| 5077 | trace_tcp_hash_md5_required(sk, skb);
|
| 5078 | return SKB_DROP_REASON_TCP_MD5NOTFOUND;
|
| 5079 | }
|
| 5080 | return SKB_NOT_DROPPED_YET;
|
| 5081 | }
|
| 5082 |
|
| 5083 | if (aoh)
|
| 5084 | return tcp_inbound_ao_hash(sk, skb, family, req, l3index, aoh);
|
| 5085 |
|
| 5086 | return tcp_inbound_md5_hash(sk, skb, saddr, daddr, family,
|
| 5087 | l3index, md5_location);
|
| 5088 | }
|
| 5089 |
|
| 5090 | void tcp_done(struct sock *sk)
|
| 5091 | {
|
| 5092 | struct request_sock *req;
|
| 5093 |
|
| 5094 | |
| 5095 | * inet_csk_prepare_forced_close() has been called
|
| 5096 | * so we can not use lockdep_sock_is_held(sk)
|
| 5097 |
|
| 5098 | req = rcu_dereference_protected(tcp_sk(sk)->fastopen_rsk, 1);
|
| 5099 |
|
| 5100 | if (sk->sk_state == TCP_SYN_SENT || sk->sk_state == TCP_SYN_RECV)
|
| 5101 | TCP_INC_STATS(sock_net(sk), TCP_MIB_ATTEMPTFAILS);
|
| 5102 |
|
| 5103 | tcp_set_state(sk, TCP_CLOSE);
|
| 5104 | tcp_clear_xmit_timers(sk);
|
| 5105 | if (req)
|
| 5106 | reqsk_fastopen_remove(sk, req, false);
|
| 5107 |
|
| 5108 | WRITE_ONCE(sk->sk_shutdown, SHUTDOWN_MASK);
|
| 5109 |
|
| 5110 | if (!sock_flag(sk, SOCK_DEAD))
|
| 5111 | sk->sk_state_change(sk);
|
| 5112 | else
|
| 5113 | inet_csk_destroy_sock(sk);
|
| 5114 | }
|
| 5115 |
|
| 5116 | int tcp_abort(struct sock *sk, int err)
|
| 5117 | {
|
| 5118 | int state = inet_sk_state_load(sk);
|
| 5119 |
|
| 5120 | if (state == TCP_NEW_SYN_RECV) {
|
| 5121 | struct request_sock *req = inet_reqsk(sk);
|
| 5122 |
|
| 5123 | local_bh_disable();
|
| 5124 | inet_csk_reqsk_queue_drop(req->rsk_listener, req);
|
| 5125 | local_bh_enable();
|
| 5126 | return 0;
|
| 5127 | }
|
| 5128 | if (state == TCP_TIME_WAIT) {
|
| 5129 | struct inet_timewait_sock *tw = inet_twsk(sk);
|
| 5130 |
|
| 5131 | refcount_inc(&tw->tw_refcnt);
|
| 5132 | local_bh_disable();
|
| 5133 | inet_twsk_deschedule_put(tw);
|
| 5134 | local_bh_enable();
|
| 5135 | return 0;
|
| 5136 | }
|
| 5137 |
|
| 5138 |
|
| 5139 | if (!has_current_bpf_ctx())
|
| 5140 |
|
| 5141 | lock_sock(sk);
|
| 5142 |
|
| 5143 |
|
| 5144 | if (sk->sk_state == TCP_CLOSE) {
|
| 5145 | if (!has_current_bpf_ctx())
|
| 5146 | release_sock(sk);
|
| 5147 | return -ENOENT;
|
| 5148 | }
|
| 5149 |
|
| 5150 | if (sk->sk_state == TCP_LISTEN) {
|
| 5151 | tcp_set_state(sk, TCP_CLOSE);
|
| 5152 | inet_csk_listen_stop(sk);
|
| 5153 | }
|
| 5154 |
|
| 5155 |
|
| 5156 | local_bh_disable();
|
| 5157 | bh_lock_sock(sk);
|
| 5158 |
|
| 5159 | if (tcp_need_reset(sk->sk_state))
|
| 5160 | tcp_send_active_reset(sk, SK_RST_REASON_TCP_STATE);
|
| 5161 | tcp_done_with_error(sk, err);
|
| 5162 |
|
| 5163 | bh_unlock_sock(sk);
|
| 5164 | local_bh_enable();
|
| 5165 | if (!has_current_bpf_ctx())
|
| 5166 | release_sock(sk);
|
| 5167 | return 0;
|
| 5168 | }
|
| 5169 | EXPORT_SYMBOL_GPL(tcp_abort);
|
| 5170 |
|
| 5171 | extern struct tcp_congestion_ops tcp_reno;
|
| 5172 |
|
| 5173 | static __initdata unsigned long thash_entries;
|
| 5174 | static int __init set_thash_entries(char *str)
|
| 5175 | {
|
| 5176 | ssize_t ret;
|
| 5177 |
|
| 5178 | if (!str)
|
| 5179 | return 0;
|
| 5180 |
|
| 5181 | ret = kstrtoul(str, 0, &thash_entries);
|
| 5182 | if (ret)
|
| 5183 | return 0;
|
| 5184 |
|
| 5185 | return 1;
|
| 5186 | }
|
| 5187 | __setup("thash_entries=", set_thash_entries);
|
| 5188 |
|
| 5189 | static void __init tcp_init_mem(void)
|
| 5190 | {
|
| 5191 | unsigned long limit = nr_free_buffer_pages() / 16;
|
| 5192 |
|
| 5193 | limit = max(limit, 128UL);
|
| 5194 | sysctl_tcp_mem[0] = limit / 4 * 3;
|
| 5195 | sysctl_tcp_mem[1] = limit;
|
| 5196 | sysctl_tcp_mem[2] = sysctl_tcp_mem[0] * 2;
|
| 5197 | }
|
| 5198 |
|
| 5199 | static void __init tcp_struct_check(void)
|
| 5200 | {
|
| 5201 |
|
| 5202 | CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_tx, max_window);
|
| 5203 | CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_tx, rcv_ssthresh);
|
| 5204 | CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_tx, reordering);
|
| 5205 | CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_tx, notsent_lowat);
|
| 5206 | CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_tx, gso_segs);
|
| 5207 | CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_tx, retransmit_skb_hint);
|
| 5208 | #if IS_ENABLED(CONFIG_TLS_DEVICE)
|
| 5209 | CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_tx, tcp_clean_acked);
|
| 5210 | #endif
|
| 5211 |
|
| 5212 |
|
| 5213 | CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_txrx, tsoffset);
|
| 5214 | CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_txrx, snd_wnd);
|
| 5215 | CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_txrx, mss_cache);
|
| 5216 | CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_txrx, snd_cwnd);
|
| 5217 | CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_txrx, prr_out);
|
| 5218 | CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_txrx, lost_out);
|
| 5219 | CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_txrx, sacked_out);
|
| 5220 | CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_txrx, scaling_ratio);
|
| 5221 |
|
| 5222 |
|
| 5223 | CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_rx, copied_seq);
|
| 5224 | CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_rx, snd_wl1);
|
| 5225 | CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_rx, tlp_high_seq);
|
| 5226 | CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_rx, rttvar_us);
|
| 5227 | CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_rx, retrans_out);
|
| 5228 | CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_rx, advmss);
|
| 5229 | CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_rx, urg_data);
|
| 5230 | CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_rx, lost);
|
| 5231 | CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_rx, rtt_min);
|
| 5232 | CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_rx, out_of_order_queue);
|
| 5233 | CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_rx, snd_ssthresh);
|
| 5234 |
|
| 5235 |
|
| 5236 | CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, data_segs_out);
|
| 5237 | CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, delivered);
|
| 5238 | CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, delivered_ce);
|
| 5239 | CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, bytes_acked);
|
| 5240 | CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, bytes_sent);
|
| 5241 | CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, first_tx_mstamp);
|
| 5242 | CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, delivered_mstamp);
|
| 5243 | CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, snd_sml);
|
| 5244 | CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, chrono_start);
|
| 5245 | CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, chrono_stat);
|
| 5246 | CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, write_seq);
|
| 5247 | CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, pushed_seq);
|
| 5248 | CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, lsndtime);
|
| 5249 | CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, mdev_us);
|
| 5250 | CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, tcp_wstamp_ns);
|
| 5251 | CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, accecn_opt_tstamp);
|
| 5252 | CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, rtt_seq);
|
| 5253 | CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, max_packets_out);
|
| 5254 | CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, cwnd_usage_seq);
|
| 5255 | CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, rate_delivered);
|
| 5256 | CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, rate_interval_us);
|
| 5257 | CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, tsorted_sent_queue);
|
| 5258 | CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, highest_sack);
|
| 5259 | CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, ecn_flags);
|
| 5260 |
|
| 5261 |
|
| 5262 | CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_txrx, pred_flags);
|
| 5263 | CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_txrx, tcp_clock_cache);
|
| 5264 | CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_txrx, tcp_mstamp);
|
| 5265 | CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_txrx, rcv_nxt);
|
| 5266 | CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_txrx, snd_nxt);
|
| 5267 | CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_txrx, snd_una);
|
| 5268 | CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_txrx, window_clamp);
|
| 5269 | CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_txrx, srtt_us);
|
| 5270 | CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_txrx, packets_out);
|
| 5271 | CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_txrx, snd_up);
|
| 5272 | CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_txrx, received_ce);
|
| 5273 | CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_txrx, received_ecn_bytes);
|
| 5274 | CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_txrx, app_limited);
|
| 5275 | CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_txrx, rcv_wnd);
|
| 5276 | CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_txrx, rcv_mwnd_seq);
|
| 5277 | CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_txrx, rcv_tstamp);
|
| 5278 | CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_txrx, rx_opt);
|
| 5279 | CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_txrx, segs_in);
|
| 5280 | CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_txrx, segs_out);
|
| 5281 |
|
| 5282 |
|
| 5283 | CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_rx, bytes_received);
|
| 5284 | CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_rx, data_segs_in);
|
| 5285 | CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_rx, rcv_wup);
|
| 5286 | CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_rx, rcv_rtt_last_tsecr);
|
| 5287 | CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_rx, delivered_ecn_bytes);
|
| 5288 | CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_rx, pkts_acked_ewma);
|
| 5289 | CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_rx, rcv_rtt_est);
|
| 5290 | CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_rx, rcvq_space);
|
| 5291 | }
|
| 5292 |
|
| 5293 | void __init tcp_init(void)
|
| 5294 | {
|
| 5295 | int max_rshare, max_wshare, cnt;
|
| 5296 | unsigned long limit;
|
| 5297 | unsigned int i;
|
| 5298 |
|
| 5299 | BUILD_BUG_ON(TCP_MIN_SND_MSS <= MAX_TCP_OPTION_SPACE);
|
| 5300 | BUILD_BUG_ON(sizeof(struct tcp_skb_cb) >
|
| 5301 | sizeof_field(struct sk_buff, cb));
|
| 5302 |
|
| 5303 | tcp_struct_check();
|
| 5304 |
|
| 5305 | percpu_counter_init(&tcp_sockets_allocated, 0, GFP_KERNEL);
|
| 5306 |
|
| 5307 | timer_setup(&tcp_orphan_timer, tcp_orphan_update, TIMER_DEFERRABLE);
|
| 5308 | mod_timer(&tcp_orphan_timer, jiffies + TCP_ORPHAN_TIMER_PERIOD);
|
| 5309 |
|
| 5310 | inet_hashinfo2_init(&tcp_hashinfo, "tcp_listen_portaddr_hash",
|
| 5311 | thash_entries, 21,
|
| 5312 | 0, 64 * 1024);
|
| 5313 | tcp_hashinfo.bind_bucket_cachep =
|
| 5314 | kmem_cache_create("tcp_bind_bucket",
|
| 5315 | sizeof(struct inet_bind_bucket), 0,
|
| 5316 | SLAB_HWCACHE_ALIGN | SLAB_PANIC |
|
| 5317 | SLAB_ACCOUNT,
|
| 5318 | NULL);
|
| 5319 | tcp_hashinfo.bind2_bucket_cachep =
|
| 5320 | kmem_cache_create("tcp_bind2_bucket",
|
| 5321 | sizeof(struct inet_bind2_bucket), 0,
|
| 5322 | SLAB_HWCACHE_ALIGN | SLAB_PANIC |
|
| 5323 | SLAB_ACCOUNT,
|
| 5324 | NULL);
|
| 5325 |
|
| 5326 | |
| 5327 | * hash tables.
|
| 5328 | *
|
| 5329 | * The methodology is similar to that of the buffer cache.
|
| 5330 |
|
| 5331 | tcp_hashinfo.ehash =
|
| 5332 | alloc_large_system_hash("TCP established",
|
| 5333 | sizeof(struct inet_ehash_bucket),
|
| 5334 | thash_entries,
|
| 5335 | 17,
|
| 5336 | 0,
|
| 5337 | NULL,
|
| 5338 | &tcp_hashinfo.ehash_mask,
|
| 5339 | 0,
|
| 5340 | thash_entries ? 0 : 512 * 1024);
|
| 5341 | for (i = 0; i <= tcp_hashinfo.ehash_mask; i++)
|
| 5342 | INIT_HLIST_NULLS_HEAD(&tcp_hashinfo.ehash[i].chain, i);
|
| 5343 |
|
| 5344 | if (inet_ehash_locks_alloc(&tcp_hashinfo))
|
| 5345 | panic("TCP: failed to alloc ehash_locks");
|
| 5346 | tcp_hashinfo.bhash =
|
| 5347 | alloc_large_system_hash("TCP bind",
|
| 5348 | 2 * sizeof(struct inet_bind_hashbucket),
|
| 5349 | tcp_hashinfo.ehash_mask + 1,
|
| 5350 | 17,
|
| 5351 | 0,
|
| 5352 | &tcp_hashinfo.bhash_size,
|
| 5353 | NULL,
|
| 5354 | 0,
|
| 5355 | 64 * 1024);
|
| 5356 | tcp_hashinfo.bhash_size = 1U << tcp_hashinfo.bhash_size;
|
| 5357 | tcp_hashinfo.bhash2 = tcp_hashinfo.bhash + tcp_hashinfo.bhash_size;
|
| 5358 | for (i = 0; i < tcp_hashinfo.bhash_size; i++) {
|
| 5359 | spin_lock_init(&tcp_hashinfo.bhash[i].lock);
|
| 5360 | INIT_HLIST_HEAD(&tcp_hashinfo.bhash[i].chain);
|
| 5361 | spin_lock_init(&tcp_hashinfo.bhash2[i].lock);
|
| 5362 | INIT_HLIST_HEAD(&tcp_hashinfo.bhash2[i].chain);
|
| 5363 | }
|
| 5364 |
|
| 5365 | tcp_hashinfo.pernet = false;
|
| 5366 |
|
| 5367 | cnt = tcp_hashinfo.ehash_mask + 1;
|
| 5368 | sysctl_tcp_max_orphans = cnt / 2;
|
| 5369 |
|
| 5370 | tcp_init_mem();
|
| 5371 |
|
| 5372 | limit = nr_free_buffer_pages() << (PAGE_SHIFT - 7);
|
| 5373 | max_wshare = min(4UL*1024*1024, limit);
|
| 5374 | max_rshare = min(32UL*1024*1024, limit);
|
| 5375 |
|
| 5376 | init_net.ipv4.sysctl_tcp_wmem[0] = PAGE_SIZE;
|
| 5377 | init_net.ipv4.sysctl_tcp_wmem[1] = 16*1024;
|
| 5378 | init_net.ipv4.sysctl_tcp_wmem[2] = max(64*1024, max_wshare);
|
| 5379 |
|
| 5380 | init_net.ipv4.sysctl_tcp_rmem[0] = PAGE_SIZE;
|
| 5381 | init_net.ipv4.sysctl_tcp_rmem[1] = 131072;
|
| 5382 | init_net.ipv4.sysctl_tcp_rmem[2] = max(131072, max_rshare);
|
| 5383 |
|
| 5384 | pr_info("Hash tables configured (established %u bind %u)\n",
|
| 5385 | tcp_hashinfo.ehash_mask + 1, tcp_hashinfo.bhash_size);
|
| 5386 |
|
| 5387 | tcp_v4_init();
|
| 5388 | tcp_metrics_init();
|
| 5389 | BUG_ON(tcp_register_congestion_control(&tcp_reno) != 0);
|
| 5390 | tcp_tsq_work_init();
|
| 5391 | mptcp_init();
|
| 5392 | }
|