| 1 | |
| 2 | |
| 3 | * INET An implementation of the TCP/IP protocol suite for the LINUX
|
| 4 | * operating system. INET is implemented using the BSD Socket
|
| 5 | * interface as the means of communication with the user level.
|
| 6 | *
|
| 7 | * The User Datagram Protocol (UDP).
|
| 8 | *
|
| 9 | * Authors: Ross Biro
|
| 10 | * Fred N. van Kempen, <[email protected]>
|
| 11 | * Arnt Gulbrandsen, <[email protected]>
|
| 12 | * Alan Cox, <[email protected]>
|
| 13 | * Hirokazu Takahashi, <[email protected]>
|
| 14 | *
|
| 15 | * Fixes:
|
| 16 | * Alan Cox : verify_area() calls
|
| 17 | * Alan Cox : stopped close while in use off icmp
|
| 18 | * messages. Not a fix but a botch that
|
| 19 | * for udp at least is 'valid'.
|
| 20 | * Alan Cox : Fixed icmp handling properly
|
| 21 | * Alan Cox : Correct error for oversized datagrams
|
| 22 | * Alan Cox : Tidied select() semantics.
|
| 23 | * Alan Cox : udp_err() fixed properly, also now
|
| 24 | * select and read wake correctly on errors
|
| 25 | * Alan Cox : udp_send verify_area moved to avoid mem leak
|
| 26 | * Alan Cox : UDP can count its memory
|
| 27 | * Alan Cox : send to an unknown connection causes
|
| 28 | * an ECONNREFUSED off the icmp, but
|
| 29 | * does NOT close.
|
| 30 | * Alan Cox : Switched to new sk_buff handlers. No more backlog!
|
| 31 | * Alan Cox : Using generic datagram code. Even smaller and the PEEK
|
| 32 | * bug no longer crashes it.
|
| 33 | * Fred Van Kempen : Net2e support for sk->broadcast.
|
| 34 | * Alan Cox : Uses skb_free_datagram
|
| 35 | * Alan Cox : Added get/set sockopt support.
|
| 36 | * Alan Cox : Broadcasting without option set returns EACCES.
|
| 37 | * Alan Cox : No wakeup calls. Instead we now use the callbacks.
|
| 38 | * Alan Cox : Use ip_tos and ip_ttl
|
| 39 | * Alan Cox : SNMP Mibs
|
| 40 | * Alan Cox : MSG_DONTROUTE, and 0.0.0.0 support.
|
| 41 | * Matt Dillon : UDP length checks.
|
| 42 | * Alan Cox : Smarter af_inet used properly.
|
| 43 | * Alan Cox : Use new kernel side addressing.
|
| 44 | * Alan Cox : Incorrect return on truncated datagram receive.
|
| 45 | * Arnt Gulbrandsen : New udp_send and stuff
|
| 46 | * Alan Cox : Cache last socket
|
| 47 | * Alan Cox : Route cache
|
| 48 | * Jon Peatfield : Minor efficiency fix to sendto().
|
| 49 | * Mike Shaver : RFC1122 checks.
|
| 50 | * Alan Cox : Nonblocking error fix.
|
| 51 | * Willy Konynenberg : Transparent proxying support.
|
| 52 | * Mike McLagan : Routing by source
|
| 53 | * David S. Miller : New socket lookup architecture.
|
| 54 | * Last socket cache retained as it
|
| 55 | * does have a high hit rate.
|
| 56 | * Olaf Kirch : Don't linearise iovec on sendmsg.
|
| 57 | * Andi Kleen : Some cleanups, cache destination entry
|
| 58 | * for connect.
|
| 59 | * Vitaly E. Lavrov : Transparent proxy revived after year coma.
|
| 60 | * Melvin Smith : Check msg_name not msg_namelen in sendto(),
|
| 61 | * return ENOTCONN for unconnected sockets (POSIX)
|
| 62 | * Janos Farkas : don't deliver multi/broadcasts to a different
|
| 63 | * bound-to-device socket
|
| 64 | * Hirokazu Takahashi : HW checksumming for outgoing UDP
|
| 65 | * datagrams.
|
| 66 | * Hirokazu Takahashi : sendfile() on UDP works now.
|
| 67 | * Arnaldo C. Melo : convert /proc/net/udp to seq_file
|
| 68 | * YOSHIFUJI Hideaki @USAGI and: Support IPV6_V6ONLY socket option, which
|
| 69 | * Alexey Kuznetsov: allow both IPv4 and IPv6 sockets to bind
|
| 70 | * a single port at the same time.
|
| 71 | * Derek Atkins <[email protected]>: Add Encapsulation Support
|
| 72 | * James Chapman : Add L2TP encapsulation type.
|
| 73 |
|
| 74 |
|
| 75 | #define pr_fmt(fmt) "UDP: " fmt
|
| 76 |
|
| 77 | #include <linux/bpf-cgroup.h>
|
| 78 | #include <linux/uaccess.h>
|
| 79 | #include <linux/uio.h>
|
| 80 | #include <asm/ioctls.h>
|
| 81 | #include <linux/memblock.h>
|
| 82 | #include <linux/highmem.h>
|
| 83 | #include <linux/types.h>
|
| 84 | #include <linux/fcntl.h>
|
| 85 | #include <linux/module.h>
|
| 86 | #include <linux/socket.h>
|
| 87 | #include <linux/sockios.h>
|
| 88 | #include <linux/igmp.h>
|
| 89 | #include <linux/inetdevice.h>
|
| 90 | #include <linux/in.h>
|
| 91 | #include <linux/errno.h>
|
| 92 | #include <linux/timer.h>
|
| 93 | #include <linux/mm.h>
|
| 94 | #include <linux/inet.h>
|
| 95 | #include <linux/netdevice.h>
|
| 96 | #include <linux/slab.h>
|
| 97 | #include <linux/sock_diag.h>
|
| 98 | #include <net/tcp_states.h>
|
| 99 | #include <linux/skbuff.h>
|
| 100 | #include <linux/proc_fs.h>
|
| 101 | #include <linux/seq_file.h>
|
| 102 | #include <net/aligned_data.h>
|
| 103 | #include <net/net_namespace.h>
|
| 104 | #include <net/icmp.h>
|
| 105 | #include <net/inet_common.h>
|
| 106 | #include <net/inet_hashtables.h>
|
| 107 | #include <net/ip.h>
|
| 108 | #include <net/ip_tunnels.h>
|
| 109 | #include <net/route.h>
|
| 110 | #include <net/checksum.h>
|
| 111 | #include <net/gso.h>
|
| 112 | #include <net/xfrm.h>
|
| 113 | #include <trace/events/udp.h>
|
| 114 | #include <linux/static_key.h>
|
| 115 | #include <linux/btf_ids.h>
|
| 116 | #include <trace/events/skb.h>
|
| 117 | #include <net/busy_poll.h>
|
| 118 | #include <net/sock_reuseport.h>
|
| 119 | #include <net/addrconf.h>
|
| 120 | #include <net/udp_tunnel.h>
|
| 121 | #include <net/gro.h>
|
| 122 | #include <net/rps.h>
|
| 123 |
|
| 124 | struct udp_table udp_table __read_mostly;
|
| 125 |
|
| 126 | long sysctl_udp_mem[3] __read_mostly;
|
| 127 |
|
| 128 | DEFINE_PER_CPU(int, udp_memory_per_cpu_fw_alloc);
|
| 129 | EXPORT_PER_CPU_SYMBOL_GPL(udp_memory_per_cpu_fw_alloc);
|
| 130 |
|
| 131 | #define MAX_UDP_PORTS 65536
|
| 132 | #define PORTS_PER_CHAIN (MAX_UDP_PORTS / UDP_HTABLE_SIZE_MIN_PERNET)
|
| 133 |
|
| 134 | static int udp_lib_lport_inuse(struct net *net, __u16 num,
|
| 135 | const struct udp_hslot *hslot,
|
| 136 | unsigned long *bitmap,
|
| 137 | struct sock *sk, unsigned int log)
|
| 138 | {
|
| 139 | kuid_t uid = sk_uid(sk);
|
| 140 | struct sock *sk2;
|
| 141 |
|
| 142 | sk_for_each(sk2, &hslot->head) {
|
| 143 | if (net_eq(sock_net(sk2), net) &&
|
| 144 | sk2 != sk &&
|
| 145 | (bitmap || udp_sk(sk2)->udp_port_hash == num) &&
|
| 146 | (!sk2->sk_reuse || !sk->sk_reuse) &&
|
| 147 | (!sk2->sk_bound_dev_if || !sk->sk_bound_dev_if ||
|
| 148 | sk2->sk_bound_dev_if == sk->sk_bound_dev_if) &&
|
| 149 | inet_rcv_saddr_equal(sk, sk2, true)) {
|
| 150 | if (sk2->sk_reuseport && sk->sk_reuseport &&
|
| 151 | !rcu_access_pointer(sk->sk_reuseport_cb) &&
|
| 152 | uid_eq(uid, sk_uid(sk2))) {
|
| 153 | if (!bitmap)
|
| 154 | return 0;
|
| 155 | } else {
|
| 156 | if (!bitmap)
|
| 157 | return 1;
|
| 158 | __set_bit(udp_sk(sk2)->udp_port_hash >> log,
|
| 159 | bitmap);
|
| 160 | }
|
| 161 | }
|
| 162 | }
|
| 163 | return 0;
|
| 164 | }
|
| 165 |
|
| 166 | |
| 167 | * Note: we still hold spinlock of primary hash chain, so no other writer
|
| 168 | * can insert/delete a socket with local_port == num
|
| 169 |
|
| 170 | static int udp_lib_lport_inuse2(struct net *net, __u16 num,
|
| 171 | struct udp_hslot *hslot2,
|
| 172 | struct sock *sk)
|
| 173 | {
|
| 174 | kuid_t uid = sk_uid(sk);
|
| 175 | struct sock *sk2;
|
| 176 | int res = 0;
|
| 177 |
|
| 178 | spin_lock(&hslot2->lock);
|
| 179 | udp_portaddr_for_each_entry(sk2, &hslot2->head) {
|
| 180 | if (net_eq(sock_net(sk2), net) &&
|
| 181 | sk2 != sk &&
|
| 182 | (udp_sk(sk2)->udp_port_hash == num) &&
|
| 183 | (!sk2->sk_reuse || !sk->sk_reuse) &&
|
| 184 | (!sk2->sk_bound_dev_if || !sk->sk_bound_dev_if ||
|
| 185 | sk2->sk_bound_dev_if == sk->sk_bound_dev_if) &&
|
| 186 | inet_rcv_saddr_equal(sk, sk2, true)) {
|
| 187 | if (sk2->sk_reuseport && sk->sk_reuseport &&
|
| 188 | !rcu_access_pointer(sk->sk_reuseport_cb) &&
|
| 189 | uid_eq(uid, sk_uid(sk2))) {
|
| 190 | res = 0;
|
| 191 | } else {
|
| 192 | res = 1;
|
| 193 | }
|
| 194 | break;
|
| 195 | }
|
| 196 | }
|
| 197 | spin_unlock(&hslot2->lock);
|
| 198 | return res;
|
| 199 | }
|
| 200 |
|
| 201 | static int udp_reuseport_add_sock(struct sock *sk, struct udp_hslot *hslot)
|
| 202 | {
|
| 203 | struct net *net = sock_net(sk);
|
| 204 | kuid_t uid = sk_uid(sk);
|
| 205 | struct sock *sk2;
|
| 206 |
|
| 207 | sk_for_each(sk2, &hslot->head) {
|
| 208 | if (net_eq(sock_net(sk2), net) &&
|
| 209 | sk2 != sk &&
|
| 210 | sk2->sk_family == sk->sk_family &&
|
| 211 | ipv6_only_sock(sk2) == ipv6_only_sock(sk) &&
|
| 212 | (udp_sk(sk2)->udp_port_hash == udp_sk(sk)->udp_port_hash) &&
|
| 213 | (sk2->sk_bound_dev_if == sk->sk_bound_dev_if) &&
|
| 214 | sk2->sk_reuseport && uid_eq(uid, sk_uid(sk2)) &&
|
| 215 | inet_rcv_saddr_equal(sk, sk2, false)) {
|
| 216 | return reuseport_add_sock(sk, sk2,
|
| 217 | inet_rcv_saddr_any(sk));
|
| 218 | }
|
| 219 | }
|
| 220 |
|
| 221 | return reuseport_alloc(sk, inet_rcv_saddr_any(sk));
|
| 222 | }
|
| 223 |
|
| 224 | |
| 225 | * udp_lib_get_port - UDP port lookup for IPv4 and IPv6
|
| 226 | *
|
| 227 | * @sk: socket struct in question
|
| 228 | * @snum: port number to look up
|
| 229 | * @hash2_nulladdr: AF-dependent hash value in secondary hash chains,
|
| 230 | * with NULL address
|
| 231 |
|
| 232 | int udp_lib_get_port(struct sock *sk, unsigned short snum,
|
| 233 | unsigned int hash2_nulladdr)
|
| 234 | {
|
| 235 | struct udp_hslot *hslot, *hslot2;
|
| 236 | struct net *net = sock_net(sk);
|
| 237 | struct udp_table *udptable;
|
| 238 | int error = -EADDRINUSE;
|
| 239 |
|
| 240 | udptable = net->ipv4.udp_table;
|
| 241 |
|
| 242 | if (!snum) {
|
| 243 | DECLARE_BITMAP(bitmap, PORTS_PER_CHAIN);
|
| 244 | unsigned short first, last;
|
| 245 | int low, high, remaining;
|
| 246 | unsigned int rand;
|
| 247 |
|
| 248 | inet_sk_get_local_port_range(sk, &low, &high);
|
| 249 | remaining = (high - low) + 1;
|
| 250 |
|
| 251 | rand = get_random_u32();
|
| 252 | first = reciprocal_scale(rand, remaining) + low;
|
| 253 | |
| 254 | * force rand to be an odd multiple of UDP_HTABLE_SIZE
|
| 255 |
|
| 256 | rand = (rand | 1) * (udptable->mask + 1);
|
| 257 | last = first + udptable->mask + 1;
|
| 258 | do {
|
| 259 | hslot = udp_hashslot(udptable, net, first);
|
| 260 | bitmap_zero(bitmap, PORTS_PER_CHAIN);
|
| 261 | spin_lock_bh(&hslot->lock);
|
| 262 | udp_lib_lport_inuse(net, snum, hslot, bitmap, sk,
|
| 263 | udptable->log);
|
| 264 |
|
| 265 | snum = first;
|
| 266 | |
| 267 | * Iterate on all possible values of snum for this hash.
|
| 268 | * Using steps of an odd multiple of UDP_HTABLE_SIZE
|
| 269 | * give us randomization and full range coverage.
|
| 270 |
|
| 271 | do {
|
| 272 | if (low <= snum && snum <= high &&
|
| 273 | !test_bit(snum >> udptable->log, bitmap) &&
|
| 274 | !inet_is_local_reserved_port(net, snum))
|
| 275 | goto found;
|
| 276 | snum += rand;
|
| 277 | } while (snum != first);
|
| 278 | spin_unlock_bh(&hslot->lock);
|
| 279 | cond_resched();
|
| 280 | } while (++first != last);
|
| 281 | goto fail;
|
| 282 | } else {
|
| 283 | hslot = udp_hashslot(udptable, net, snum);
|
| 284 | spin_lock_bh(&hslot->lock);
|
| 285 | if (inet_use_hash2_on_bind(sk) && hslot->count > 10) {
|
| 286 | int exist;
|
| 287 | unsigned int slot2 = udp_sk(sk)->udp_portaddr_hash ^ snum;
|
| 288 |
|
| 289 | slot2 &= udptable->mask;
|
| 290 | hash2_nulladdr &= udptable->mask;
|
| 291 |
|
| 292 | hslot2 = udp_hashslot2(udptable, slot2);
|
| 293 | if (hslot->count < hslot2->count)
|
| 294 | goto scan_primary_hash;
|
| 295 |
|
| 296 | exist = udp_lib_lport_inuse2(net, snum, hslot2, sk);
|
| 297 | if (!exist && (hash2_nulladdr != slot2)) {
|
| 298 | hslot2 = udp_hashslot2(udptable, hash2_nulladdr);
|
| 299 | exist = udp_lib_lport_inuse2(net, snum, hslot2,
|
| 300 | sk);
|
| 301 | }
|
| 302 | if (exist)
|
| 303 | goto fail_unlock;
|
| 304 | else
|
| 305 | goto found;
|
| 306 | }
|
| 307 | scan_primary_hash:
|
| 308 | if (udp_lib_lport_inuse(net, snum, hslot, NULL, sk, 0))
|
| 309 | goto fail_unlock;
|
| 310 | }
|
| 311 | found:
|
| 312 | inet_sk(sk)->inet_num = snum;
|
| 313 | udp_sk(sk)->udp_port_hash = snum;
|
| 314 | udp_sk(sk)->udp_portaddr_hash ^= snum;
|
| 315 | if (sk_unhashed(sk)) {
|
| 316 | if (sk->sk_reuseport &&
|
| 317 | udp_reuseport_add_sock(sk, hslot)) {
|
| 318 | inet_sk(sk)->inet_num = 0;
|
| 319 | udp_sk(sk)->udp_port_hash = 0;
|
| 320 | udp_sk(sk)->udp_portaddr_hash ^= snum;
|
| 321 | goto fail_unlock;
|
| 322 | }
|
| 323 |
|
| 324 | sock_set_flag(sk, SOCK_RCU_FREE);
|
| 325 |
|
| 326 | sk_add_node_rcu(sk, &hslot->head);
|
| 327 | hslot->count++;
|
| 328 | sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1);
|
| 329 |
|
| 330 | hslot2 = udp_hashslot2(udptable, udp_sk(sk)->udp_portaddr_hash);
|
| 331 | spin_lock(&hslot2->lock);
|
| 332 | if (IS_ENABLED(CONFIG_IPV6) && sk->sk_reuseport &&
|
| 333 | sk->sk_family == AF_INET6)
|
| 334 | hlist_add_tail_rcu(&udp_sk(sk)->udp_portaddr_node,
|
| 335 | &hslot2->head);
|
| 336 | else
|
| 337 | hlist_add_head_rcu(&udp_sk(sk)->udp_portaddr_node,
|
| 338 | &hslot2->head);
|
| 339 | hslot2->count++;
|
| 340 | spin_unlock(&hslot2->lock);
|
| 341 | }
|
| 342 |
|
| 343 | error = 0;
|
| 344 | fail_unlock:
|
| 345 | spin_unlock_bh(&hslot->lock);
|
| 346 | fail:
|
| 347 | return error;
|
| 348 | }
|
| 349 |
|
| 350 | static int udp_v4_get_port(struct sock *sk, unsigned short snum)
|
| 351 | {
|
| 352 | unsigned int hash2_nulladdr =
|
| 353 | ipv4_portaddr_hash(sock_net(sk), htonl(INADDR_ANY), snum);
|
| 354 | unsigned int hash2_partial =
|
| 355 | ipv4_portaddr_hash(sock_net(sk), inet_sk(sk)->inet_rcv_saddr, 0);
|
| 356 |
|
| 357 |
|
| 358 | udp_sk(sk)->udp_portaddr_hash = hash2_partial;
|
| 359 | return udp_lib_get_port(sk, snum, hash2_nulladdr);
|
| 360 | }
|
| 361 |
|
| 362 | static __always_inline int
|
| 363 | compute_score(struct sock *sk, const struct net *net,
|
| 364 | __be32 saddr, __be16 sport, __be32 daddr,
|
| 365 | unsigned short hnum, int dif, int sdif)
|
| 366 | {
|
| 367 | int score;
|
| 368 | struct inet_sock *inet;
|
| 369 | bool dev_match;
|
| 370 |
|
| 371 | if (!net_eq(sock_net(sk), net) ||
|
| 372 | udp_sk(sk)->udp_port_hash != hnum ||
|
| 373 | ipv6_only_sock(sk))
|
| 374 | return -1;
|
| 375 |
|
| 376 | if (sk->sk_rcv_saddr != daddr)
|
| 377 | return -1;
|
| 378 |
|
| 379 | score = (sk->sk_family == PF_INET) ? 2 : 1;
|
| 380 |
|
| 381 | inet = inet_sk(sk);
|
| 382 | if (inet->inet_daddr) {
|
| 383 | if (inet->inet_daddr != saddr)
|
| 384 | return -1;
|
| 385 | score += 4;
|
| 386 | }
|
| 387 |
|
| 388 | if (inet->inet_dport) {
|
| 389 | if (inet->inet_dport != sport)
|
| 390 | return -1;
|
| 391 | score += 4;
|
| 392 | }
|
| 393 |
|
| 394 | dev_match = udp_sk_bound_dev_eq(net, sk->sk_bound_dev_if,
|
| 395 | dif, sdif);
|
| 396 | if (!dev_match)
|
| 397 | return -1;
|
| 398 | if (sk->sk_bound_dev_if)
|
| 399 | score += 4;
|
| 400 |
|
| 401 | if (READ_ONCE(sk->sk_incoming_cpu) == raw_smp_processor_id())
|
| 402 | score++;
|
| 403 | return score;
|
| 404 | }
|
| 405 |
|
| 406 | u32 udp_ehashfn(const struct net *net, const __be32 laddr, const __u16 lport,
|
| 407 | const __be32 faddr, const __be16 fport)
|
| 408 | {
|
| 409 | net_get_random_once(&udp_ehash_secret, sizeof(udp_ehash_secret));
|
| 410 |
|
| 411 | return __inet_ehashfn(laddr, lport, faddr, fport,
|
| 412 | udp_ehash_secret + net_hash_mix(net));
|
| 413 | }
|
| 414 |
|
| 415 | |
| 416 | * udp4_lib_lookup1() - Simplified lookup using primary hash (destination port)
|
| 417 | * @net: Network namespace
|
| 418 | * @saddr: Source address, network order
|
| 419 | * @sport: Source port, network order
|
| 420 | * @daddr: Destination address, network order
|
| 421 | * @hnum: Destination port, host order
|
| 422 | * @dif: Destination interface index
|
| 423 | * @sdif: Destination bridge port index, if relevant
|
| 424 | * @udptable: Set of UDP hash tables
|
| 425 | *
|
| 426 | * Simplified lookup to be used as fallback if no sockets are found due to a
|
| 427 | * potential race between (receive) address change, and lookup happening before
|
| 428 | * the rehash operation. This function ignores SO_REUSEPORT groups while scoring
|
| 429 | * result sockets, because if we have one, we don't need the fallback at all.
|
| 430 | *
|
| 431 | * Called under rcu_read_lock().
|
| 432 | *
|
| 433 | * Return: socket with highest matching score if any, NULL if none
|
| 434 |
|
| 435 | static struct sock *udp4_lib_lookup1(const struct net *net,
|
| 436 | __be32 saddr, __be16 sport,
|
| 437 | __be32 daddr, unsigned int hnum,
|
| 438 | int dif, int sdif,
|
| 439 | const struct udp_table *udptable)
|
| 440 | {
|
| 441 | unsigned int slot = udp_hashfn(net, hnum, udptable->mask);
|
| 442 | struct udp_hslot *hslot = &udptable->hash[slot];
|
| 443 | struct sock *sk, *result = NULL;
|
| 444 | int score, badness = 0;
|
| 445 |
|
| 446 | sk_for_each_rcu(sk, &hslot->head) {
|
| 447 | score = compute_score(sk, net,
|
| 448 | saddr, sport, daddr, hnum, dif, sdif);
|
| 449 | if (score > badness) {
|
| 450 | result = sk;
|
| 451 | badness = score;
|
| 452 | }
|
| 453 | }
|
| 454 |
|
| 455 | return result;
|
| 456 | }
|
| 457 |
|
| 458 |
|
| 459 | static struct sock *udp4_lib_lookup2(const struct net *net,
|
| 460 | __be32 saddr, __be16 sport,
|
| 461 | __be32 daddr, unsigned int hnum,
|
| 462 | int dif, int sdif,
|
| 463 | struct udp_hslot *hslot2,
|
| 464 | struct sk_buff *skb)
|
| 465 | {
|
| 466 | struct sock *sk, *result;
|
| 467 | int score, badness;
|
| 468 | bool need_rescore;
|
| 469 |
|
| 470 | result = NULL;
|
| 471 | badness = 0;
|
| 472 | udp_portaddr_for_each_entry_rcu(sk, &hslot2->head) {
|
| 473 | need_rescore = false;
|
| 474 | rescore:
|
| 475 | score = compute_score(need_rescore ? result : sk, net, saddr,
|
| 476 | sport, daddr, hnum, dif, sdif);
|
| 477 | if (score > badness) {
|
| 478 | badness = score;
|
| 479 |
|
| 480 | if (need_rescore)
|
| 481 | continue;
|
| 482 |
|
| 483 | if (sk->sk_state == TCP_ESTABLISHED) {
|
| 484 | result = sk;
|
| 485 | continue;
|
| 486 | }
|
| 487 |
|
| 488 | result = inet_lookup_reuseport(net, sk, skb, sizeof(struct udphdr),
|
| 489 | saddr, sport, daddr, hnum, udp_ehashfn);
|
| 490 | if (!result) {
|
| 491 | result = sk;
|
| 492 | continue;
|
| 493 | }
|
| 494 |
|
| 495 |
|
| 496 | if (!reuseport_has_conns(sk))
|
| 497 | return result;
|
| 498 |
|
| 499 |
|
| 500 | if (IS_ERR(result))
|
| 501 | continue;
|
| 502 |
|
| 503 | |
| 504 | * inlined twice here, and calling it uninlined
|
| 505 | * here yields measurable overhead for some
|
| 506 | * workloads. Work around it by jumping
|
| 507 | * backwards to rescore 'result'.
|
| 508 |
|
| 509 | need_rescore = true;
|
| 510 | goto rescore;
|
| 511 | }
|
| 512 | }
|
| 513 | return result;
|
| 514 | }
|
| 515 |
|
| 516 | #if IS_ENABLED(CONFIG_BASE_SMALL)
|
| 517 | static struct sock *udp4_lib_lookup4(const struct net *net,
|
| 518 | __be32 saddr, __be16 sport,
|
| 519 | __be32 daddr, unsigned int hnum,
|
| 520 | int dif, int sdif,
|
| 521 | struct udp_table *udptable)
|
| 522 | {
|
| 523 | return NULL;
|
| 524 | }
|
| 525 |
|
| 526 | static void udp_rehash4(struct udp_table *udptable, struct sock *sk,
|
| 527 | u16 newhash4)
|
| 528 | {
|
| 529 | }
|
| 530 |
|
| 531 | static void udp_unhash4(struct udp_table *udptable, struct sock *sk)
|
| 532 | {
|
| 533 | }
|
| 534 | #else
|
| 535 | static struct sock *udp4_lib_lookup4(const struct net *net,
|
| 536 | __be32 saddr, __be16 sport,
|
| 537 | __be32 daddr, unsigned int hnum,
|
| 538 | int dif, int sdif,
|
| 539 | struct udp_table *udptable)
|
| 540 | {
|
| 541 | const __portpair ports = INET_COMBINED_PORTS(sport, hnum);
|
| 542 | const struct hlist_nulls_node *node;
|
| 543 | struct udp_hslot *hslot4;
|
| 544 | unsigned int hash4, slot;
|
| 545 | struct udp_sock *up;
|
| 546 | struct sock *sk;
|
| 547 |
|
| 548 | hash4 = udp_ehashfn(net, daddr, hnum, saddr, sport);
|
| 549 | slot = hash4 & udptable->mask;
|
| 550 | hslot4 = &udptable->hash4[slot];
|
| 551 | INET_ADDR_COOKIE(acookie, saddr, daddr);
|
| 552 |
|
| 553 | begin:
|
| 554 |
|
| 555 | udp_lrpa_for_each_entry_rcu(up, node, &hslot4->nulls_head) {
|
| 556 | sk = (struct sock *)up;
|
| 557 | if (inet_match(net, sk, acookie, ports, dif, sdif))
|
| 558 | return sk;
|
| 559 | }
|
| 560 |
|
| 561 | |
| 562 | * expected one, we must restart lookup. We probably met an item that
|
| 563 | * was moved to another chain due to rehash.
|
| 564 |
|
| 565 | if (get_nulls_value(node) != slot)
|
| 566 | goto begin;
|
| 567 |
|
| 568 | return NULL;
|
| 569 | }
|
| 570 |
|
| 571 |
|
| 572 | static void udp_rehash4(struct udp_table *udptable, struct sock *sk,
|
| 573 | u16 newhash4)
|
| 574 | {
|
| 575 | struct udp_hslot *hslot4, *nhslot4;
|
| 576 |
|
| 577 | hslot4 = udp_hashslot4(udptable, udp_sk(sk)->udp_lrpa_hash);
|
| 578 | nhslot4 = udp_hashslot4(udptable, newhash4);
|
| 579 | udp_sk(sk)->udp_lrpa_hash = newhash4;
|
| 580 |
|
| 581 | if (hslot4 != nhslot4) {
|
| 582 | spin_lock_bh(&hslot4->lock);
|
| 583 | hlist_nulls_del_init_rcu(&udp_sk(sk)->udp_lrpa_node);
|
| 584 | hslot4->count--;
|
| 585 | spin_unlock_bh(&hslot4->lock);
|
| 586 |
|
| 587 | spin_lock_bh(&nhslot4->lock);
|
| 588 | hlist_nulls_add_head_rcu(&udp_sk(sk)->udp_lrpa_node,
|
| 589 | &nhslot4->nulls_head);
|
| 590 | nhslot4->count++;
|
| 591 | spin_unlock_bh(&nhslot4->lock);
|
| 592 | }
|
| 593 | }
|
| 594 |
|
| 595 | static void udp_unhash4(struct udp_table *udptable, struct sock *sk)
|
| 596 | {
|
| 597 | struct udp_hslot *hslot2, *hslot4;
|
| 598 |
|
| 599 | if (udp_hashed4(sk)) {
|
| 600 | hslot2 = udp_hashslot2(udptable, udp_sk(sk)->udp_portaddr_hash);
|
| 601 | hslot4 = udp_hashslot4(udptable, udp_sk(sk)->udp_lrpa_hash);
|
| 602 |
|
| 603 | spin_lock(&hslot4->lock);
|
| 604 | hlist_nulls_del_init_rcu(&udp_sk(sk)->udp_lrpa_node);
|
| 605 | hslot4->count--;
|
| 606 | spin_unlock(&hslot4->lock);
|
| 607 |
|
| 608 | spin_lock(&hslot2->lock);
|
| 609 | udp_hash4_dec(hslot2);
|
| 610 | spin_unlock(&hslot2->lock);
|
| 611 | }
|
| 612 | }
|
| 613 |
|
| 614 | void udp_lib_hash4(struct sock *sk, u16 hash)
|
| 615 | {
|
| 616 | struct udp_hslot *hslot, *hslot2, *hslot4;
|
| 617 | struct net *net = sock_net(sk);
|
| 618 | struct udp_table *udptable;
|
| 619 |
|
| 620 | udptable = net->ipv4.udp_table;
|
| 621 | hslot = udp_hashslot(udptable, net, udp_sk(sk)->udp_port_hash);
|
| 622 |
|
| 623 | |
| 624 | * relocates it, but only runs when the local address changes, so a
|
| 625 | * socket bound to a specific address would stay filed under the
|
| 626 | * previous peer's hash. Move it here.
|
| 627 |
|
| 628 | if (udp_hashed4(sk)) {
|
| 629 | if (udp_sk(sk)->udp_lrpa_hash != hash) {
|
| 630 | spin_lock_bh(&hslot->lock);
|
| 631 | udp_rehash4(udptable, sk, hash);
|
| 632 | spin_unlock_bh(&hslot->lock);
|
| 633 | }
|
| 634 | return;
|
| 635 | }
|
| 636 |
|
| 637 | hslot2 = udp_hashslot2(udptable, udp_sk(sk)->udp_portaddr_hash);
|
| 638 | hslot4 = udp_hashslot4(udptable, hash);
|
| 639 | udp_sk(sk)->udp_lrpa_hash = hash;
|
| 640 |
|
| 641 | spin_lock_bh(&hslot->lock);
|
| 642 | if (rcu_access_pointer(sk->sk_reuseport_cb))
|
| 643 | reuseport_detach_sock(sk);
|
| 644 |
|
| 645 | spin_lock(&hslot4->lock);
|
| 646 | hlist_nulls_add_head_rcu(&udp_sk(sk)->udp_lrpa_node,
|
| 647 | &hslot4->nulls_head);
|
| 648 | hslot4->count++;
|
| 649 | spin_unlock(&hslot4->lock);
|
| 650 |
|
| 651 | spin_lock(&hslot2->lock);
|
| 652 | udp_hash4_inc(hslot2);
|
| 653 | spin_unlock(&hslot2->lock);
|
| 654 |
|
| 655 | spin_unlock_bh(&hslot->lock);
|
| 656 | }
|
| 657 |
|
| 658 |
|
| 659 | void udp4_hash4(struct sock *sk)
|
| 660 | {
|
| 661 | struct net *net = sock_net(sk);
|
| 662 | unsigned int hash;
|
| 663 |
|
| 664 | if (sk_unhashed(sk) || sk->sk_rcv_saddr == htonl(INADDR_ANY))
|
| 665 | return;
|
| 666 |
|
| 667 | hash = udp_ehashfn(net, sk->sk_rcv_saddr, sk->sk_num,
|
| 668 | sk->sk_daddr, sk->sk_dport);
|
| 669 |
|
| 670 | udp_lib_hash4(sk, hash);
|
| 671 | }
|
| 672 | #endif
|
| 673 |
|
| 674 | |
| 675 | * harder than this. -DaveM
|
| 676 |
|
| 677 | struct sock *__udp4_lib_lookup(const struct net *net, __be32 saddr,
|
| 678 | __be16 sport, __be32 daddr, __be16 dport,
|
| 679 | int dif, int sdif, struct sk_buff *skb)
|
| 680 | {
|
| 681 | struct udp_table *udptable = net->ipv4.udp_table;
|
| 682 | unsigned short hnum = ntohs(dport);
|
| 683 | struct udp_hslot *hslot2;
|
| 684 | struct sock *result, *sk;
|
| 685 | unsigned int hash2;
|
| 686 |
|
| 687 | hash2 = ipv4_portaddr_hash(net, daddr, hnum);
|
| 688 | hslot2 = udp_hashslot2(udptable, hash2);
|
| 689 |
|
| 690 | if (udp_has_hash4(hslot2)) {
|
| 691 | result = udp4_lib_lookup4(net, saddr, sport, daddr, hnum,
|
| 692 | dif, sdif, udptable);
|
| 693 | if (result)
|
| 694 | return result;
|
| 695 | }
|
| 696 |
|
| 697 |
|
| 698 | result = udp4_lib_lookup2(net, saddr, sport,
|
| 699 | daddr, hnum, dif, sdif,
|
| 700 | hslot2, skb);
|
| 701 | if (!IS_ERR_OR_NULL(result) && result->sk_state == TCP_ESTABLISHED)
|
| 702 | goto done;
|
| 703 |
|
| 704 |
|
| 705 | if (static_branch_unlikely(&bpf_sk_lookup_enabled)) {
|
| 706 | sk = inet_lookup_run_sk_lookup(net, IPPROTO_UDP, skb, sizeof(struct udphdr),
|
| 707 | saddr, sport, daddr, hnum, dif,
|
| 708 | udp_ehashfn);
|
| 709 | if (sk) {
|
| 710 | result = sk;
|
| 711 | goto done;
|
| 712 | }
|
| 713 | }
|
| 714 |
|
| 715 |
|
| 716 | if (result)
|
| 717 | goto done;
|
| 718 |
|
| 719 |
|
| 720 | hash2 = ipv4_portaddr_hash(net, htonl(INADDR_ANY), hnum);
|
| 721 | hslot2 = udp_hashslot2(udptable, hash2);
|
| 722 |
|
| 723 | result = udp4_lib_lookup2(net, saddr, sport,
|
| 724 | htonl(INADDR_ANY), hnum, dif, sdif,
|
| 725 | hslot2, skb);
|
| 726 | if (!IS_ERR_OR_NULL(result))
|
| 727 | goto done;
|
| 728 |
|
| 729 | |
| 730 | * 1. __ip4_datagram_connect() sets sk_rcv_saddr
|
| 731 | * 2. lookup (this function): new sk_rcv_saddr, hashes not updated yet
|
| 732 | * 3. rehash operation updating _secondary and four-tuple_ hashes
|
| 733 | * The primary hash doesn't need an update after 1., so, thanks to this
|
| 734 | * further step, 1. and 3. don't need to be atomic against the lookup.
|
| 735 |
|
| 736 | result = udp4_lib_lookup1(net, saddr, sport, daddr, hnum, dif, sdif,
|
| 737 | udptable);
|
| 738 |
|
| 739 | done:
|
| 740 | if (IS_ERR(result))
|
| 741 | return NULL;
|
| 742 | return result;
|
| 743 | }
|
| 744 | EXPORT_SYMBOL_GPL(__udp4_lib_lookup);
|
| 745 |
|
| 746 | static inline struct sock *__udp4_lib_lookup_skb(struct sk_buff *skb,
|
| 747 | __be16 sport, __be16 dport)
|
| 748 | {
|
| 749 | const struct iphdr *iph = ip_hdr(skb);
|
| 750 |
|
| 751 | return __udp4_lib_lookup(dev_net(skb->dev), iph->saddr, sport,
|
| 752 | iph->daddr, dport, inet_iif(skb),
|
| 753 | inet_sdif(skb), skb);
|
| 754 | }
|
| 755 |
|
| 756 | struct sock *udp4_lib_lookup_skb(const struct sk_buff *skb,
|
| 757 | __be16 sport, __be16 dport)
|
| 758 | {
|
| 759 | const u16 offset = NAPI_GRO_CB(skb)->network_offsets[skb->encapsulation];
|
| 760 | const struct iphdr *iph = (struct iphdr *)(skb->data + offset);
|
| 761 | int iif, sdif;
|
| 762 |
|
| 763 | inet_get_iif_sdif(skb, &iif, &sdif);
|
| 764 |
|
| 765 | return __udp4_lib_lookup(dev_net(skb->dev), iph->saddr, sport,
|
| 766 | iph->daddr, dport, iif, sdif, NULL);
|
| 767 | }
|
| 768 |
|
| 769 | |
| 770 | * Does increment socket refcount.
|
| 771 |
|
| 772 | #if IS_ENABLED(CONFIG_NF_TPROXY_IPV4) || IS_ENABLED(CONFIG_NF_SOCKET_IPV4)
|
| 773 | struct sock *udp4_lib_lookup(const struct net *net, __be32 saddr, __be16 sport,
|
| 774 | __be32 daddr, __be16 dport, int dif)
|
| 775 | {
|
| 776 | struct sock *sk;
|
| 777 |
|
| 778 | sk = __udp4_lib_lookup(net, saddr, sport, daddr, dport, dif, 0, NULL);
|
| 779 | if (sk && !refcount_inc_not_zero(&sk->sk_refcnt))
|
| 780 | sk = NULL;
|
| 781 | return sk;
|
| 782 | }
|
| 783 | EXPORT_SYMBOL_GPL(udp4_lib_lookup);
|
| 784 | #endif
|
| 785 |
|
| 786 | static inline bool __udp_is_mcast_sock(struct net *net, const struct sock *sk,
|
| 787 | __be16 loc_port, __be32 loc_addr,
|
| 788 | __be16 rmt_port, __be32 rmt_addr,
|
| 789 | int dif, int sdif, unsigned short hnum)
|
| 790 | {
|
| 791 | const struct inet_sock *inet = inet_sk(sk);
|
| 792 |
|
| 793 | if (!net_eq(sock_net(sk), net) ||
|
| 794 | udp_sk(sk)->udp_port_hash != hnum ||
|
| 795 | (inet->inet_daddr && inet->inet_daddr != rmt_addr) ||
|
| 796 | (inet->inet_dport != rmt_port && inet->inet_dport) ||
|
| 797 | (inet->inet_rcv_saddr && inet->inet_rcv_saddr != loc_addr) ||
|
| 798 | ipv6_only_sock(sk) ||
|
| 799 | !udp_sk_bound_dev_eq(net, sk->sk_bound_dev_if, dif, sdif))
|
| 800 | return false;
|
| 801 | if (!ip_mc_sf_allow(sk, loc_addr, rmt_addr, dif, sdif))
|
| 802 | return false;
|
| 803 | return true;
|
| 804 | }
|
| 805 |
|
| 806 | DEFINE_STATIC_KEY_FALSE(udp_encap_needed_key);
|
| 807 |
|
| 808 | #if IS_ENABLED(CONFIG_IPV6)
|
| 809 | DEFINE_STATIC_KEY_FALSE(udpv6_encap_needed_key);
|
| 810 | #endif
|
| 811 |
|
| 812 | void udp_encap_enable(void)
|
| 813 | {
|
| 814 | static_branch_inc(&udp_encap_needed_key);
|
| 815 | }
|
| 816 | EXPORT_SYMBOL(udp_encap_enable);
|
| 817 |
|
| 818 | void udp_encap_disable(void)
|
| 819 | {
|
| 820 | static_branch_dec(&udp_encap_needed_key);
|
| 821 | }
|
| 822 | EXPORT_SYMBOL(udp_encap_disable);
|
| 823 |
|
| 824 | |
| 825 | * through error handlers in encapsulations looking for a match.
|
| 826 |
|
| 827 | static int __udp4_lib_err_encap_no_sk(struct sk_buff *skb, u32 info)
|
| 828 | {
|
| 829 | int i;
|
| 830 |
|
| 831 | for (i = 0; i < MAX_IPTUN_ENCAP_OPS; i++) {
|
| 832 | int (*handler)(struct sk_buff *skb, u32 info);
|
| 833 | const struct ip_tunnel_encap_ops *encap;
|
| 834 |
|
| 835 | encap = rcu_dereference(iptun_encaps[i]);
|
| 836 | if (!encap)
|
| 837 | continue;
|
| 838 | handler = encap->err_handler;
|
| 839 | if (handler && !handler(skb, info))
|
| 840 | return 0;
|
| 841 | }
|
| 842 |
|
| 843 | return -ENOENT;
|
| 844 | }
|
| 845 |
|
| 846 | |
| 847 | * reversing source and destination port: this will match tunnels that force the
|
| 848 | * same destination port on both endpoints (e.g. VXLAN, GENEVE). Note that
|
| 849 | * lwtunnels might actually break this assumption by being configured with
|
| 850 | * different destination ports on endpoints, in this case we won't be able to
|
| 851 | * trace ICMP messages back to them.
|
| 852 | *
|
| 853 | * If this doesn't match any socket, probe tunnels with arbitrary destination
|
| 854 | * ports (e.g. FoU, GUE): there, the receiving socket is useless, as the port
|
| 855 | * we've sent packets to won't necessarily match the local destination port.
|
| 856 | *
|
| 857 | * Then ask the tunnel implementation to match the error against a valid
|
| 858 | * association.
|
| 859 | *
|
| 860 | * Return an error if we can't find a match, the socket if we need further
|
| 861 | * processing, zero otherwise.
|
| 862 |
|
| 863 | static struct sock *__udp4_lib_err_encap(struct net *net,
|
| 864 | const struct iphdr *iph,
|
| 865 | struct udphdr *uh,
|
| 866 | struct sock *sk,
|
| 867 | struct sk_buff *skb, u32 info)
|
| 868 | {
|
| 869 | int (*lookup)(struct sock *sk, struct sk_buff *skb);
|
| 870 | int network_offset, transport_offset;
|
| 871 | struct udp_sock *up;
|
| 872 |
|
| 873 | network_offset = skb_network_offset(skb);
|
| 874 | transport_offset = skb_transport_offset(skb);
|
| 875 |
|
| 876 |
|
| 877 | skb_reset_network_header(skb);
|
| 878 |
|
| 879 |
|
| 880 | skb_set_transport_header(skb, iph->ihl << 2);
|
| 881 |
|
| 882 | if (sk) {
|
| 883 | up = udp_sk(sk);
|
| 884 |
|
| 885 | lookup = READ_ONCE(up->encap_err_lookup);
|
| 886 | if (lookup && lookup(sk, skb))
|
| 887 | sk = NULL;
|
| 888 |
|
| 889 | goto out;
|
| 890 | }
|
| 891 |
|
| 892 | sk = __udp4_lib_lookup(net, iph->daddr, uh->source,
|
| 893 | iph->saddr, uh->dest, skb->dev->ifindex, 0, NULL);
|
| 894 | if (sk) {
|
| 895 | up = udp_sk(sk);
|
| 896 |
|
| 897 | lookup = READ_ONCE(up->encap_err_lookup);
|
| 898 | if (!lookup || lookup(sk, skb))
|
| 899 | sk = NULL;
|
| 900 | }
|
| 901 |
|
| 902 | out:
|
| 903 | if (!sk)
|
| 904 | sk = ERR_PTR(__udp4_lib_err_encap_no_sk(skb, info));
|
| 905 |
|
| 906 | skb_set_transport_header(skb, transport_offset);
|
| 907 | skb_set_network_header(skb, network_offset);
|
| 908 |
|
| 909 | return sk;
|
| 910 | }
|
| 911 |
|
| 912 | static void udp_err_update_exception(struct net *net, struct sk_buff *skb,
|
| 913 | int type, int code, u32 info)
|
| 914 | {
|
| 915 | if (type == ICMP_DEST_UNREACH && code == ICMP_FRAG_NEEDED)
|
| 916 | ipv4_update_pmtu(skb, net, info, 0, IPPROTO_UDP);
|
| 917 | else if (type == ICMP_REDIRECT)
|
| 918 | ipv4_redirect(skb, net, 0, IPPROTO_UDP);
|
| 919 | }
|
| 920 |
|
| 921 | |
| 922 | * This routine is called by the ICMP module when it gets some
|
| 923 | * sort of error condition. If err < 0 then the socket should
|
| 924 | * be closed and the error returned to the user. If err > 0
|
| 925 | * it's just the icmp type << 8 | icmp code.
|
| 926 | * Header points to the ip header of the error packet. We move
|
| 927 | * on past this. Then (as it used to claim before adjustment)
|
| 928 | * header points to the first 8 bytes of the udp header. We need
|
| 929 | * to find the appropriate port.
|
| 930 |
|
| 931 | int udp_err(struct sk_buff *skb, u32 info)
|
| 932 | {
|
| 933 | const struct iphdr *iph = (const struct iphdr *)skb->data;
|
| 934 | const int type = icmp_hdr(skb)->type;
|
| 935 | const int code = icmp_hdr(skb)->code;
|
| 936 | struct net *net = dev_net(skb->dev);
|
| 937 | struct inet_sock *inet;
|
| 938 | bool tunnel = false;
|
| 939 | struct udphdr *uh;
|
| 940 | struct sock *sk;
|
| 941 | int harderr;
|
| 942 | int err;
|
| 943 |
|
| 944 | udp_err_update_exception(net, skb, type, code, info);
|
| 945 |
|
| 946 | uh = (struct udphdr *)(skb->data + (iph->ihl << 2));
|
| 947 | sk = __udp4_lib_lookup(net, iph->daddr, uh->dest,
|
| 948 | iph->saddr, uh->source, skb->dev->ifindex,
|
| 949 | inet_sdif(skb), NULL);
|
| 950 |
|
| 951 | if (!sk || READ_ONCE(udp_sk(sk)->encap_type)) {
|
| 952 |
|
| 953 | if (static_branch_unlikely(&udp_encap_needed_key)) {
|
| 954 | sk = __udp4_lib_err_encap(net, iph, uh, sk, skb, info);
|
| 955 | if (!sk)
|
| 956 | return 0;
|
| 957 | } else
|
| 958 | sk = ERR_PTR(-ENOENT);
|
| 959 |
|
| 960 | if (IS_ERR(sk)) {
|
| 961 | __ICMP_INC_STATS(net, ICMP_MIB_INERRORS);
|
| 962 | return PTR_ERR(sk);
|
| 963 | }
|
| 964 |
|
| 965 | tunnel = true;
|
| 966 | }
|
| 967 |
|
| 968 | err = 0;
|
| 969 | harderr = 0;
|
| 970 | inet = inet_sk(sk);
|
| 971 |
|
| 972 | switch (type) {
|
| 973 | default:
|
| 974 | case ICMP_TIME_EXCEEDED:
|
| 975 | err = EHOSTUNREACH;
|
| 976 | break;
|
| 977 | case ICMP_SOURCE_QUENCH:
|
| 978 | goto out;
|
| 979 | case ICMP_PARAMETERPROB:
|
| 980 | err = EPROTO;
|
| 981 | harderr = 1;
|
| 982 | break;
|
| 983 | case ICMP_DEST_UNREACH:
|
| 984 | if (code == ICMP_FRAG_NEEDED) {
|
| 985 | ipv4_sk_update_pmtu(skb, sk, info);
|
| 986 | if (READ_ONCE(inet->pmtudisc) != IP_PMTUDISC_DONT) {
|
| 987 | err = EMSGSIZE;
|
| 988 | harderr = 1;
|
| 989 | break;
|
| 990 | }
|
| 991 | goto out;
|
| 992 | }
|
| 993 | err = EHOSTUNREACH;
|
| 994 | if (code <= NR_ICMP_UNREACH) {
|
| 995 | harderr = icmp_err_convert[code].fatal;
|
| 996 | err = icmp_err_convert[code].errno;
|
| 997 | }
|
| 998 | break;
|
| 999 | case ICMP_REDIRECT:
|
| 1000 | ipv4_sk_redirect(skb, sk);
|
| 1001 | goto out;
|
| 1002 | }
|
| 1003 |
|
| 1004 | |
| 1005 | * RFC1122: OK. Passes ICMP errors back to application, as per
|
| 1006 | * 4.1.3.3.
|
| 1007 |
|
| 1008 | if (tunnel) {
|
| 1009 |
|
| 1010 | if (udp_sk(sk)->encap_err_rcv)
|
| 1011 | udp_sk(sk)->encap_err_rcv(sk, skb, err, uh->dest, info,
|
| 1012 | (u8 *)(uh+1));
|
| 1013 | goto out;
|
| 1014 | }
|
| 1015 | if (!inet_test_bit(RECVERR, sk)) {
|
| 1016 | if (!harderr || sk->sk_state != TCP_ESTABLISHED)
|
| 1017 | goto out;
|
| 1018 | } else
|
| 1019 | ip_icmp_error(sk, skb, err, uh->dest, info, (u8 *)(uh+1));
|
| 1020 |
|
| 1021 | sk->sk_err = err;
|
| 1022 | sk_error_report(sk);
|
| 1023 | out:
|
| 1024 | return 0;
|
| 1025 | }
|
| 1026 |
|
| 1027 | |
| 1028 | * Throw away all pending data and cancel the corking. Socket is locked.
|
| 1029 |
|
| 1030 | void udp_flush_pending_frames(struct sock *sk)
|
| 1031 | {
|
| 1032 | struct udp_sock *up = udp_sk(sk);
|
| 1033 |
|
| 1034 | if (up->pending) {
|
| 1035 | up->len = 0;
|
| 1036 | WRITE_ONCE(up->pending, 0);
|
| 1037 | ip_flush_pending_frames(sk);
|
| 1038 | }
|
| 1039 | }
|
| 1040 |
|
| 1041 | |
| 1042 | * udp4_hwcsum - handle outgoing HW checksumming
|
| 1043 | * @skb: sk_buff containing the filled-in UDP header
|
| 1044 | * (checksum field must be zeroed out)
|
| 1045 | * @src: source IP address
|
| 1046 | * @dst: destination IP address
|
| 1047 |
|
| 1048 | void udp4_hwcsum(struct sk_buff *skb, __be32 src, __be32 dst)
|
| 1049 | {
|
| 1050 | struct udphdr *uh = udp_hdr(skb);
|
| 1051 | int offset = skb_transport_offset(skb);
|
| 1052 | int len = skb->len - offset;
|
| 1053 | int hlen = len;
|
| 1054 | __wsum csum = 0;
|
| 1055 |
|
| 1056 | if (!skb_has_frag_list(skb)) {
|
| 1057 | |
| 1058 | * Only one fragment on the socket.
|
| 1059 |
|
| 1060 | skb->csum_start = skb_transport_header(skb) - skb->head;
|
| 1061 | skb->csum_offset = offsetof(struct udphdr, check);
|
| 1062 | uh->check = ~csum_tcpudp_magic(src, dst, len,
|
| 1063 | IPPROTO_UDP, 0);
|
| 1064 | } else {
|
| 1065 | struct sk_buff *frags;
|
| 1066 |
|
| 1067 | |
| 1068 | * HW-checksum won't work as there are two or more
|
| 1069 | * fragments on the socket so that all csums of sk_buffs
|
| 1070 | * should be together
|
| 1071 |
|
| 1072 | skb_walk_frags(skb, frags) {
|
| 1073 | csum = csum_add(csum, frags->csum);
|
| 1074 | hlen -= frags->len;
|
| 1075 | }
|
| 1076 |
|
| 1077 | csum = skb_checksum(skb, offset, hlen, csum);
|
| 1078 | skb->ip_summed = CHECKSUM_NONE;
|
| 1079 |
|
| 1080 | uh->check = csum_tcpudp_magic(src, dst, len, IPPROTO_UDP, csum);
|
| 1081 | if (uh->check == 0)
|
| 1082 | uh->check = CSUM_MANGLED_0;
|
| 1083 | }
|
| 1084 | }
|
| 1085 | EXPORT_SYMBOL_GPL(udp4_hwcsum);
|
| 1086 |
|
| 1087 | |
| 1088 | * for the simple case like when setting the checksum for a UDP tunnel.
|
| 1089 |
|
| 1090 | void udp_set_csum(bool nocheck, struct sk_buff *skb,
|
| 1091 | __be32 saddr, __be32 daddr, int len)
|
| 1092 | {
|
| 1093 | struct udphdr *uh = udp_hdr(skb);
|
| 1094 |
|
| 1095 | if (nocheck) {
|
| 1096 | uh->check = 0;
|
| 1097 | } else if (skb_is_gso(skb)) {
|
| 1098 | uh->check = ~udp_v4_check(len, saddr, daddr, 0);
|
| 1099 | } else if (skb->ip_summed == CHECKSUM_PARTIAL) {
|
| 1100 | uh->check = 0;
|
| 1101 | uh->check = udp_v4_check(len, saddr, daddr, lco_csum(skb));
|
| 1102 | if (uh->check == 0)
|
| 1103 | uh->check = CSUM_MANGLED_0;
|
| 1104 | } else {
|
| 1105 | skb->ip_summed = CHECKSUM_PARTIAL;
|
| 1106 | skb->csum_start = skb_transport_header(skb) - skb->head;
|
| 1107 | skb->csum_offset = offsetof(struct udphdr, check);
|
| 1108 | uh->check = ~udp_v4_check(len, saddr, daddr, 0);
|
| 1109 | }
|
| 1110 | }
|
| 1111 | EXPORT_SYMBOL(udp_set_csum);
|
| 1112 |
|
| 1113 | static int udp_send_skb(struct sk_buff *skb, struct flowi4 *fl4,
|
| 1114 | struct inet_cork *cork)
|
| 1115 | {
|
| 1116 | struct sock *sk = skb->sk;
|
| 1117 | int offset, len, datalen;
|
| 1118 | struct udphdr *uh;
|
| 1119 | int err;
|
| 1120 |
|
| 1121 | offset = skb_transport_offset(skb);
|
| 1122 | len = skb->len - offset;
|
| 1123 | datalen = len - sizeof(*uh);
|
| 1124 |
|
| 1125 | |
| 1126 | * Create a UDP header
|
| 1127 |
|
| 1128 | uh = udp_hdr(skb);
|
| 1129 | uh->source = inet_sk(sk)->inet_sport;
|
| 1130 | uh->dest = fl4->fl4_dport;
|
| 1131 |
|
| 1132 | udp_set_len_short(uh, len);
|
| 1133 | uh->check = 0;
|
| 1134 |
|
| 1135 | if (cork->gso_size) {
|
| 1136 | const int hlen = skb_network_header_len(skb) +
|
| 1137 | sizeof(struct udphdr);
|
| 1138 |
|
| 1139 | if (hlen + min(datalen, cork->gso_size) > cork->fragsize) {
|
| 1140 | kfree_skb(skb);
|
| 1141 | return -EMSGSIZE;
|
| 1142 | }
|
| 1143 | if (datalen > cork->gso_size * UDP_MAX_SEGMENTS) {
|
| 1144 | kfree_skb(skb);
|
| 1145 | return -EINVAL;
|
| 1146 | }
|
| 1147 | if (sk->sk_no_check_tx) {
|
| 1148 | kfree_skb(skb);
|
| 1149 | return -EINVAL;
|
| 1150 | }
|
| 1151 | if (dst_xfrm(skb_dst(skb))) {
|
| 1152 | kfree_skb(skb);
|
| 1153 | return -EIO;
|
| 1154 | }
|
| 1155 |
|
| 1156 | if (datalen > cork->gso_size) {
|
| 1157 | skb_shinfo(skb)->gso_size = cork->gso_size;
|
| 1158 | skb_shinfo(skb)->gso_type = SKB_GSO_UDP_L4;
|
| 1159 | skb_shinfo(skb)->gso_segs = DIV_ROUND_UP(datalen,
|
| 1160 | cork->gso_size);
|
| 1161 |
|
| 1162 |
|
| 1163 | goto csum_partial;
|
| 1164 | }
|
| 1165 | }
|
| 1166 |
|
| 1167 | if (sk->sk_no_check_tx) {
|
| 1168 | skb->ip_summed = CHECKSUM_NONE;
|
| 1169 | goto send;
|
| 1170 | } else if (skb->ip_summed == CHECKSUM_PARTIAL) {
|
| 1171 | csum_partial:
|
| 1172 | udp4_hwcsum(skb, fl4->saddr, fl4->daddr);
|
| 1173 | goto send;
|
| 1174 | }
|
| 1175 |
|
| 1176 |
|
| 1177 | uh->check = csum_tcpudp_magic(fl4->saddr, fl4->daddr, len,
|
| 1178 | IPPROTO_UDP, udp_csum(skb));
|
| 1179 | if (uh->check == 0)
|
| 1180 | uh->check = CSUM_MANGLED_0;
|
| 1181 |
|
| 1182 | send:
|
| 1183 | err = ip_send_skb(sock_net(sk), skb);
|
| 1184 | if (unlikely(err)) {
|
| 1185 | if (err == -ENOBUFS &&
|
| 1186 | !inet_test_bit(RECVERR, sk)) {
|
| 1187 | UDP_INC_STATS(sock_net(sk), UDP_MIB_SNDBUFERRORS);
|
| 1188 | err = 0;
|
| 1189 | }
|
| 1190 | } else {
|
| 1191 | UDP_INC_STATS(sock_net(sk), UDP_MIB_OUTDATAGRAMS);
|
| 1192 | }
|
| 1193 | return err;
|
| 1194 | }
|
| 1195 |
|
| 1196 | |
| 1197 | * Push out all pending data as one UDP datagram. Socket is locked.
|
| 1198 |
|
| 1199 | int udp_push_pending_frames(struct sock *sk)
|
| 1200 | {
|
| 1201 | struct udp_sock *up = udp_sk(sk);
|
| 1202 | struct inet_sock *inet = inet_sk(sk);
|
| 1203 | struct flowi4 *fl4 = &inet->cork.fl.u.ip4;
|
| 1204 | struct sk_buff *skb;
|
| 1205 | int err = 0;
|
| 1206 |
|
| 1207 | skb = ip_finish_skb(sk, fl4);
|
| 1208 | if (!skb)
|
| 1209 | goto out;
|
| 1210 |
|
| 1211 | err = udp_send_skb(skb, fl4, &inet->cork.base);
|
| 1212 |
|
| 1213 | out:
|
| 1214 | up->len = 0;
|
| 1215 | WRITE_ONCE(up->pending, 0);
|
| 1216 | return err;
|
| 1217 | }
|
| 1218 |
|
| 1219 | static int __udp_cmsg_send(struct cmsghdr *cmsg, u16 *gso_size)
|
| 1220 | {
|
| 1221 | switch (cmsg->cmsg_type) {
|
| 1222 | case UDP_SEGMENT:
|
| 1223 | if (cmsg->cmsg_len != CMSG_LEN(sizeof(__u16)))
|
| 1224 | return -EINVAL;
|
| 1225 | *gso_size = *(__u16 *)CMSG_DATA(cmsg);
|
| 1226 | return 0;
|
| 1227 | default:
|
| 1228 | return -EINVAL;
|
| 1229 | }
|
| 1230 | }
|
| 1231 |
|
| 1232 | int udp_cmsg_send(struct sock *sk, struct msghdr *msg, u16 *gso_size)
|
| 1233 | {
|
| 1234 | struct cmsghdr *cmsg;
|
| 1235 | bool need_ip = false;
|
| 1236 | int err;
|
| 1237 |
|
| 1238 | for_each_cmsghdr(cmsg, msg) {
|
| 1239 | if (!CMSG_OK(msg, cmsg))
|
| 1240 | return -EINVAL;
|
| 1241 |
|
| 1242 | if (cmsg->cmsg_level != SOL_UDP) {
|
| 1243 | need_ip = true;
|
| 1244 | continue;
|
| 1245 | }
|
| 1246 |
|
| 1247 | err = __udp_cmsg_send(cmsg, gso_size);
|
| 1248 | if (err)
|
| 1249 | return err;
|
| 1250 | }
|
| 1251 |
|
| 1252 | return need_ip;
|
| 1253 | }
|
| 1254 |
|
| 1255 | int udp_sendmsg(struct sock *sk, struct msghdr *msg, size_t len)
|
| 1256 | {
|
| 1257 | int corkreq = udp_test_bit(CORK, sk) || msg->msg_flags & MSG_MORE;
|
| 1258 | DEFINE_RAW_FLEX(struct ip_options_rcu, opt_copy, opt.__data,
|
| 1259 | IP_OPTIONS_DATA_FIXED_SIZE);
|
| 1260 | DECLARE_SOCKADDR(struct sockaddr_in *, usin, msg->msg_name);
|
| 1261 | int ulen = len, free = 0, connected = 0;
|
| 1262 | struct inet_sock *inet = inet_sk(sk);
|
| 1263 | struct udp_sock *up = udp_sk(sk);
|
| 1264 | __be32 daddr, faddr, saddr;
|
| 1265 | struct rtable *rt = NULL;
|
| 1266 | struct flowi4 fl4_stack;
|
| 1267 | struct ipcm_cookie ipc;
|
| 1268 | struct sk_buff *skb;
|
| 1269 | struct flowi4 *fl4;
|
| 1270 | __be16 dport;
|
| 1271 | int uc_index;
|
| 1272 | u8 scope;
|
| 1273 | int err;
|
| 1274 |
|
| 1275 | if (len > 0xFFFF)
|
| 1276 | return -EMSGSIZE;
|
| 1277 |
|
| 1278 | |
| 1279 | * Check the flags.
|
| 1280 |
|
| 1281 |
|
| 1282 | if (msg->msg_flags & MSG_OOB)
|
| 1283 | return -EOPNOTSUPP;
|
| 1284 |
|
| 1285 | fl4 = &inet->cork.fl.u.ip4;
|
| 1286 | if (READ_ONCE(up->pending)) {
|
| 1287 | |
| 1288 | * There are pending frames.
|
| 1289 | * The socket lock must be held while it's corked.
|
| 1290 |
|
| 1291 | lock_sock(sk);
|
| 1292 | if (likely(up->pending)) {
|
| 1293 | if (unlikely(up->pending != AF_INET)) {
|
| 1294 | release_sock(sk);
|
| 1295 | return -EINVAL;
|
| 1296 | }
|
| 1297 | goto do_append_data;
|
| 1298 | }
|
| 1299 | release_sock(sk);
|
| 1300 | }
|
| 1301 | ulen += sizeof(struct udphdr);
|
| 1302 |
|
| 1303 | |
| 1304 | * Get and verify the address.
|
| 1305 |
|
| 1306 | if (usin) {
|
| 1307 | if (msg->msg_namelen < sizeof(*usin))
|
| 1308 | return -EINVAL;
|
| 1309 | if (usin->sin_family != AF_INET) {
|
| 1310 | if (usin->sin_family != AF_UNSPEC)
|
| 1311 | return -EAFNOSUPPORT;
|
| 1312 | }
|
| 1313 |
|
| 1314 | daddr = usin->sin_addr.s_addr;
|
| 1315 | dport = usin->sin_port;
|
| 1316 | if (dport == 0)
|
| 1317 | return -EINVAL;
|
| 1318 | } else {
|
| 1319 | if (sk->sk_state != TCP_ESTABLISHED)
|
| 1320 | return -EDESTADDRREQ;
|
| 1321 | daddr = inet->inet_daddr;
|
| 1322 | dport = inet->inet_dport;
|
| 1323 | |
| 1324 | Route will not be used, if at least one option is set.
|
| 1325 |
|
| 1326 | connected = 1;
|
| 1327 | }
|
| 1328 |
|
| 1329 | ipcm_init_sk(&ipc, inet);
|
| 1330 | ipc.gso_size = READ_ONCE(up->gso_size);
|
| 1331 |
|
| 1332 | if (msg->msg_controllen) {
|
| 1333 | err = udp_cmsg_send(sk, msg, &ipc.gso_size);
|
| 1334 | if (err > 0) {
|
| 1335 | err = ip_cmsg_send(sk, msg, &ipc,
|
| 1336 | sk->sk_family == AF_INET6);
|
| 1337 | connected = 0;
|
| 1338 | }
|
| 1339 | if (unlikely(err < 0)) {
|
| 1340 | kfree(ipc.opt);
|
| 1341 | return err;
|
| 1342 | }
|
| 1343 | if (ipc.opt)
|
| 1344 | free = 1;
|
| 1345 | }
|
| 1346 | if (!ipc.opt) {
|
| 1347 | struct ip_options_rcu *inet_opt;
|
| 1348 |
|
| 1349 | rcu_read_lock();
|
| 1350 | inet_opt = rcu_dereference(inet->inet_opt);
|
| 1351 | if (inet_opt) {
|
| 1352 | memcpy(opt_copy, inet_opt,
|
| 1353 | sizeof(*inet_opt) + inet_opt->opt.optlen);
|
| 1354 | ipc.opt = opt_copy;
|
| 1355 | }
|
| 1356 | rcu_read_unlock();
|
| 1357 | }
|
| 1358 |
|
| 1359 | if (cgroup_bpf_enabled(CGROUP_UDP4_SENDMSG) && !connected) {
|
| 1360 | err = BPF_CGROUP_RUN_PROG_UDP4_SENDMSG_LOCK(sk,
|
| 1361 | (struct sockaddr *)usin,
|
| 1362 | &msg->msg_namelen,
|
| 1363 | &ipc.addr);
|
| 1364 | if (err)
|
| 1365 | goto out_free;
|
| 1366 | if (usin) {
|
| 1367 | if (usin->sin_port == 0) {
|
| 1368 |
|
| 1369 | err = -EINVAL;
|
| 1370 | goto out_free;
|
| 1371 | }
|
| 1372 | daddr = usin->sin_addr.s_addr;
|
| 1373 | dport = usin->sin_port;
|
| 1374 | }
|
| 1375 | }
|
| 1376 |
|
| 1377 | saddr = ipc.addr;
|
| 1378 | ipc.addr = faddr = daddr;
|
| 1379 |
|
| 1380 | if (ipc.opt && ipc.opt->opt.srr) {
|
| 1381 | if (!daddr) {
|
| 1382 | err = -EINVAL;
|
| 1383 | goto out_free;
|
| 1384 | }
|
| 1385 | faddr = ipc.opt->opt.faddr;
|
| 1386 | connected = 0;
|
| 1387 | }
|
| 1388 | scope = ip_sendmsg_scope(inet, &ipc, msg);
|
| 1389 | if (scope == RT_SCOPE_LINK)
|
| 1390 | connected = 0;
|
| 1391 |
|
| 1392 | uc_index = READ_ONCE(inet->uc_index);
|
| 1393 | if (ipv4_is_multicast(daddr)) {
|
| 1394 | if (!ipc.oif || netif_index_is_l3_master(sock_net(sk), ipc.oif))
|
| 1395 | ipc.oif = READ_ONCE(inet->mc_index);
|
| 1396 | if (!saddr)
|
| 1397 | saddr = READ_ONCE(inet->mc_addr);
|
| 1398 | connected = 0;
|
| 1399 | } else if (!ipc.oif) {
|
| 1400 | ipc.oif = uc_index;
|
| 1401 | } else if (ipv4_is_lbcast(daddr) && uc_index) {
|
| 1402 | |
| 1403 | * uc_index is set. oif is most likely set
|
| 1404 | * by sk_bound_dev_if. If uc_index != oif check if the
|
| 1405 | * oif is an L3 master and uc_index is an L3 slave.
|
| 1406 | * If so, we want to allow the send using the uc_index.
|
| 1407 |
|
| 1408 | if (ipc.oif != uc_index &&
|
| 1409 | ipc.oif == l3mdev_master_ifindex_by_index(sock_net(sk),
|
| 1410 | uc_index)) {
|
| 1411 | ipc.oif = uc_index;
|
| 1412 | }
|
| 1413 | }
|
| 1414 |
|
| 1415 | if (connected)
|
| 1416 | rt = dst_rtable(sk_dst_check(sk, 0));
|
| 1417 |
|
| 1418 | if (!rt) {
|
| 1419 | struct net *net = sock_net(sk);
|
| 1420 | __u8 flow_flags = inet_sk_flowi_flags(sk);
|
| 1421 |
|
| 1422 | fl4 = &fl4_stack;
|
| 1423 |
|
| 1424 | flowi4_init_output(fl4, ipc.oif, ipc.sockc.mark,
|
| 1425 | ipc.tos & INET_DSCP_MASK, scope,
|
| 1426 | IPPROTO_UDP, flow_flags, faddr, saddr,
|
| 1427 | dport, inet->inet_sport,
|
| 1428 | sk_uid(sk));
|
| 1429 |
|
| 1430 | security_sk_classify_flow(sk, flowi4_to_flowi_common(fl4));
|
| 1431 | rt = ip_route_output_flow(net, fl4, sk);
|
| 1432 | if (IS_ERR(rt)) {
|
| 1433 | err = PTR_ERR(rt);
|
| 1434 | rt = NULL;
|
| 1435 | if (err == -ENETUNREACH)
|
| 1436 | IP_INC_STATS(net, IPSTATS_MIB_OUTNOROUTES);
|
| 1437 | goto out;
|
| 1438 | }
|
| 1439 |
|
| 1440 | err = -EACCES;
|
| 1441 | if ((rt->rt_flags & RTCF_BROADCAST) &&
|
| 1442 | !sock_flag(sk, SOCK_BROADCAST))
|
| 1443 | goto out;
|
| 1444 | if (connected)
|
| 1445 | sk_dst_set(sk, dst_clone(&rt->dst));
|
| 1446 | }
|
| 1447 |
|
| 1448 | if (msg->msg_flags&MSG_CONFIRM)
|
| 1449 | goto do_confirm;
|
| 1450 | back_from_confirm:
|
| 1451 |
|
| 1452 | saddr = fl4->saddr;
|
| 1453 | if (!ipc.addr)
|
| 1454 | daddr = ipc.addr = fl4->daddr;
|
| 1455 |
|
| 1456 |
|
| 1457 | if (!corkreq) {
|
| 1458 | struct inet_cork cork;
|
| 1459 |
|
| 1460 | skb = ip_make_skb(sk, fl4, ip_generic_getfrag, msg, ulen,
|
| 1461 | sizeof(struct udphdr), &ipc, &rt,
|
| 1462 | &cork, msg->msg_flags);
|
| 1463 | err = PTR_ERR(skb);
|
| 1464 | if (!IS_ERR_OR_NULL(skb))
|
| 1465 | err = udp_send_skb(skb, fl4, &cork);
|
| 1466 | goto out;
|
| 1467 | }
|
| 1468 |
|
| 1469 | lock_sock(sk);
|
| 1470 | if (unlikely(up->pending)) {
|
| 1471 |
|
| 1472 |
|
| 1473 | release_sock(sk);
|
| 1474 |
|
| 1475 | net_dbg_ratelimited("socket already corked\n");
|
| 1476 | err = -EINVAL;
|
| 1477 | goto out;
|
| 1478 | }
|
| 1479 | |
| 1480 | * Now cork the socket to pend data.
|
| 1481 |
|
| 1482 | fl4 = &inet->cork.fl.u.ip4;
|
| 1483 | fl4->daddr = daddr;
|
| 1484 | fl4->saddr = saddr;
|
| 1485 | fl4->fl4_dport = dport;
|
| 1486 | fl4->fl4_sport = inet->inet_sport;
|
| 1487 | WRITE_ONCE(up->pending, AF_INET);
|
| 1488 |
|
| 1489 | do_append_data:
|
| 1490 | up->len += ulen;
|
| 1491 | err = ip_append_data(sk, fl4, ip_generic_getfrag, msg, ulen,
|
| 1492 | sizeof(struct udphdr), &ipc, &rt,
|
| 1493 | corkreq ? msg->msg_flags|MSG_MORE : msg->msg_flags);
|
| 1494 | if (err)
|
| 1495 | udp_flush_pending_frames(sk);
|
| 1496 | else if (!corkreq)
|
| 1497 | err = udp_push_pending_frames(sk);
|
| 1498 | else if (unlikely(skb_queue_empty(&sk->sk_write_queue)))
|
| 1499 | WRITE_ONCE(up->pending, 0);
|
| 1500 | release_sock(sk);
|
| 1501 |
|
| 1502 | out:
|
| 1503 | ip_rt_put(rt);
|
| 1504 | out_free:
|
| 1505 | if (free)
|
| 1506 | kfree(ipc.opt);
|
| 1507 | if (!err)
|
| 1508 | return len;
|
| 1509 | |
| 1510 | * ENOBUFS = no kernel mem, SOCK_NOSPACE = no sndbuf space. Reporting
|
| 1511 | * ENOBUFS might not be good (it's not tunable per se), but otherwise
|
| 1512 | * we don't have a good statistic (IpOutDiscards but it can be too many
|
| 1513 | * things). We could add another new stat but at least for now that
|
| 1514 | * seems like overkill.
|
| 1515 |
|
| 1516 | if (err == -ENOBUFS || test_bit(SOCK_NOSPACE, &sk->sk_socket->flags))
|
| 1517 | UDP_INC_STATS(sock_net(sk), UDP_MIB_SNDBUFERRORS);
|
| 1518 |
|
| 1519 | return err;
|
| 1520 |
|
| 1521 | do_confirm:
|
| 1522 | if (msg->msg_flags & MSG_PROBE)
|
| 1523 | dst_confirm_neigh(&rt->dst, &fl4->daddr);
|
| 1524 | if (!(msg->msg_flags&MSG_PROBE) || len)
|
| 1525 | goto back_from_confirm;
|
| 1526 | err = 0;
|
| 1527 | goto out;
|
| 1528 | }
|
| 1529 | EXPORT_SYMBOL(udp_sendmsg);
|
| 1530 |
|
| 1531 | void udp_splice_eof(struct socket *sock)
|
| 1532 | {
|
| 1533 | struct sock *sk = sock->sk;
|
| 1534 | struct udp_sock *up = udp_sk(sk);
|
| 1535 |
|
| 1536 | if (!READ_ONCE(up->pending) || udp_test_bit(CORK, sk))
|
| 1537 | return;
|
| 1538 |
|
| 1539 | lock_sock(sk);
|
| 1540 | if (up->pending && !udp_test_bit(CORK, sk))
|
| 1541 | udp_push_pending_frames(sk);
|
| 1542 | release_sock(sk);
|
| 1543 | }
|
| 1544 |
|
| 1545 | #define UDP_SKB_IS_STATELESS 0x80000000
|
| 1546 |
|
| 1547 | |
| 1548 | * cleared by udp_rcv().
|
| 1549 | *
|
| 1550 | * We need to preserve secpath, if present, to eventually process
|
| 1551 | * IP_CMSG_PASSSEC at recvmsg() time.
|
| 1552 | *
|
| 1553 | * Other extensions can be cleared.
|
| 1554 |
|
| 1555 | static bool udp_try_make_stateless(struct sk_buff *skb)
|
| 1556 | {
|
| 1557 | if (!skb_has_extensions(skb))
|
| 1558 | return true;
|
| 1559 |
|
| 1560 | if (!secpath_exists(skb)) {
|
| 1561 | skb_ext_reset(skb);
|
| 1562 | return true;
|
| 1563 | }
|
| 1564 |
|
| 1565 | return false;
|
| 1566 | }
|
| 1567 |
|
| 1568 | static void udp_set_dev_scratch(struct sk_buff *skb)
|
| 1569 | {
|
| 1570 | struct udp_dev_scratch *scratch = udp_skb_scratch(skb);
|
| 1571 |
|
| 1572 | BUILD_BUG_ON(sizeof(struct udp_dev_scratch) > sizeof(long));
|
| 1573 | scratch->_tsize_state = skb->truesize;
|
| 1574 | #if BITS_PER_LONG == 64
|
| 1575 | scratch->len = skb->len;
|
| 1576 | scratch->csum_unnecessary = !!skb_csum_unnecessary(skb);
|
| 1577 | scratch->is_linear = !skb_is_nonlinear(skb);
|
| 1578 | #endif
|
| 1579 | if (udp_try_make_stateless(skb))
|
| 1580 | scratch->_tsize_state |= UDP_SKB_IS_STATELESS;
|
| 1581 | }
|
| 1582 |
|
| 1583 | static void udp_skb_csum_unnecessary_set(struct sk_buff *skb)
|
| 1584 | {
|
| 1585 | |
| 1586 | * This means that __skb_checksum_complete() might have
|
| 1587 | * set skb->csum_valid to 1.
|
| 1588 | * On 64bit platforms, we can set csum_unnecessary
|
| 1589 | * to true, but only if the skb is not shared.
|
| 1590 |
|
| 1591 | #if BITS_PER_LONG == 64
|
| 1592 | if (!skb_shared(skb))
|
| 1593 | udp_skb_scratch(skb)->csum_unnecessary = true;
|
| 1594 | #endif
|
| 1595 | }
|
| 1596 |
|
| 1597 | static int udp_skb_truesize(struct sk_buff *skb)
|
| 1598 | {
|
| 1599 | return udp_skb_scratch(skb)->_tsize_state & ~UDP_SKB_IS_STATELESS;
|
| 1600 | }
|
| 1601 |
|
| 1602 | static bool udp_skb_has_head_state(struct sk_buff *skb)
|
| 1603 | {
|
| 1604 | return !(udp_skb_scratch(skb)->_tsize_state & UDP_SKB_IS_STATELESS);
|
| 1605 | }
|
| 1606 |
|
| 1607 |
|
| 1608 | static void udp_rmem_release(struct sock *sk, unsigned int size,
|
| 1609 | int partial, bool rx_queue_lock_held)
|
| 1610 | {
|
| 1611 | struct udp_sock *up = udp_sk(sk);
|
| 1612 | struct sk_buff_head *sk_queue;
|
| 1613 | unsigned int amt;
|
| 1614 |
|
| 1615 | if (likely(partial)) {
|
| 1616 | up->forward_deficit += size;
|
| 1617 | size = up->forward_deficit;
|
| 1618 | if (size < READ_ONCE(up->forward_threshold) &&
|
| 1619 | !skb_queue_empty(&up->reader_queue))
|
| 1620 | return;
|
| 1621 | } else {
|
| 1622 | size += up->forward_deficit;
|
| 1623 | }
|
| 1624 | up->forward_deficit = 0;
|
| 1625 |
|
| 1626 | |
| 1627 | * if the called don't held it already
|
| 1628 |
|
| 1629 | sk_queue = &sk->sk_receive_queue;
|
| 1630 | if (!rx_queue_lock_held)
|
| 1631 | spin_lock(&sk_queue->lock);
|
| 1632 |
|
| 1633 | amt = (size + sk->sk_forward_alloc - partial) & ~(PAGE_SIZE - 1);
|
| 1634 | sk_forward_alloc_add(sk, size - amt);
|
| 1635 |
|
| 1636 | if (amt)
|
| 1637 | __sk_mem_reduce_allocated(sk, amt >> PAGE_SHIFT);
|
| 1638 |
|
| 1639 | atomic_sub(size, &sk->sk_rmem_alloc);
|
| 1640 |
|
| 1641 |
|
| 1642 | skb_queue_splice_tail_init(sk_queue, &up->reader_queue);
|
| 1643 |
|
| 1644 | if (!rx_queue_lock_held)
|
| 1645 | spin_unlock(&sk_queue->lock);
|
| 1646 | }
|
| 1647 |
|
| 1648 | |
| 1649 | * Instead of using skb->truesize here, find a copy of it in skb->dev_scratch
|
| 1650 | * This avoids a cache line miss while receive_queue lock is held.
|
| 1651 | * Look at __udp_enqueue_schedule_skb() to find where this copy is done.
|
| 1652 |
|
| 1653 | void udp_skb_destructor(struct sock *sk, struct sk_buff *skb)
|
| 1654 | {
|
| 1655 | prefetch(&skb->data);
|
| 1656 | udp_rmem_release(sk, udp_skb_truesize(skb), 1, false);
|
| 1657 | }
|
| 1658 |
|
| 1659 |
|
| 1660 | static void udp_skb_dtor_locked(struct sock *sk, struct sk_buff *skb)
|
| 1661 | {
|
| 1662 | prefetch(&skb->data);
|
| 1663 | udp_rmem_release(sk, udp_skb_truesize(skb), 1, true);
|
| 1664 | }
|
| 1665 |
|
| 1666 | static int udp_rmem_schedule(struct sock *sk, int size)
|
| 1667 | {
|
| 1668 | int delta;
|
| 1669 |
|
| 1670 | delta = size - sk->sk_forward_alloc;
|
| 1671 | if (delta > 0 && !__sk_mem_schedule(sk, delta, SK_MEM_RECV))
|
| 1672 | return -ENOBUFS;
|
| 1673 |
|
| 1674 | return 0;
|
| 1675 | }
|
| 1676 |
|
| 1677 | int __udp_enqueue_schedule_skb(struct sock *sk, struct sk_buff *skb)
|
| 1678 | {
|
| 1679 | struct sk_buff_head *list = &sk->sk_receive_queue;
|
| 1680 | struct udp_prod_queue *udp_prod_queue;
|
| 1681 | struct sk_buff *next, *to_drop = NULL;
|
| 1682 | struct llist_node *ll_list;
|
| 1683 | unsigned int rmem, rcvbuf;
|
| 1684 | int size, err = -ENOMEM;
|
| 1685 | int total_size = 0;
|
| 1686 | int q_size = 0;
|
| 1687 | int dropcount;
|
| 1688 | int nb = 0;
|
| 1689 |
|
| 1690 | rmem = atomic_read(&sk->sk_rmem_alloc);
|
| 1691 | rcvbuf = READ_ONCE(sk->sk_rcvbuf);
|
| 1692 | size = skb->truesize;
|
| 1693 |
|
| 1694 | udp_prod_queue = &udp_sk(sk)->udp_prod_queue[numa_node_id()];
|
| 1695 |
|
| 1696 | rmem += atomic_read(&udp_prod_queue->rmem_alloc);
|
| 1697 |
|
| 1698 | |
| 1699 | * Cast to unsigned int performs the boundary check for INT_MAX.
|
| 1700 |
|
| 1701 | if (rmem + size > rcvbuf) {
|
| 1702 | if (rcvbuf > INT_MAX >> 1)
|
| 1703 | goto drop;
|
| 1704 |
|
| 1705 |
|
| 1706 | if (rmem)
|
| 1707 | goto drop;
|
| 1708 | }
|
| 1709 |
|
| 1710 | |
| 1711 | * having linear skbs :
|
| 1712 | * - Reduce memory overhead and thus increase receive queue capacity
|
| 1713 | * - Less cache line misses at copyout() time
|
| 1714 | * - Less work at consume_skb() (less alien page frag freeing)
|
| 1715 |
|
| 1716 | if (rmem > (rcvbuf >> 1)) {
|
| 1717 | skb_condense(skb);
|
| 1718 | size = skb->truesize;
|
| 1719 | }
|
| 1720 |
|
| 1721 | udp_set_dev_scratch(skb);
|
| 1722 |
|
| 1723 | atomic_add(size, &udp_prod_queue->rmem_alloc);
|
| 1724 |
|
| 1725 | if (!llist_add(&skb->ll_node, &udp_prod_queue->ll_root))
|
| 1726 | return 0;
|
| 1727 |
|
| 1728 | dropcount = sock_flag(sk, SOCK_RXQ_OVFL) ? sk_drops_read(sk) : 0;
|
| 1729 |
|
| 1730 | spin_lock(&list->lock);
|
| 1731 |
|
| 1732 | ll_list = llist_del_all(&udp_prod_queue->ll_root);
|
| 1733 |
|
| 1734 | ll_list = llist_reverse_order(ll_list);
|
| 1735 |
|
| 1736 | llist_for_each_entry_safe(skb, next, ll_list, ll_node) {
|
| 1737 | size = udp_skb_truesize(skb);
|
| 1738 | total_size += size;
|
| 1739 | err = udp_rmem_schedule(sk, size);
|
| 1740 | if (unlikely(err)) {
|
| 1741 |
|
| 1742 | skb->next = to_drop;
|
| 1743 | to_drop = skb;
|
| 1744 | continue;
|
| 1745 | }
|
| 1746 |
|
| 1747 | q_size += size;
|
| 1748 | sk_forward_alloc_add(sk, -size);
|
| 1749 |
|
| 1750 | |
| 1751 | * forward allocated memory on dequeue
|
| 1752 |
|
| 1753 | SOCK_SKB_CB(skb)->dropcount = dropcount;
|
| 1754 | nb++;
|
| 1755 | __skb_queue_tail(list, skb);
|
| 1756 | }
|
| 1757 |
|
| 1758 | atomic_add(q_size, &sk->sk_rmem_alloc);
|
| 1759 |
|
| 1760 | spin_unlock(&list->lock);
|
| 1761 |
|
| 1762 | if (!sock_flag(sk, SOCK_DEAD)) {
|
| 1763 | |
| 1764 | * using prepare_to_wait_exclusive().
|
| 1765 |
|
| 1766 | while (nb) {
|
| 1767 | INDIRECT_CALL_1(READ_ONCE(sk->sk_data_ready),
|
| 1768 | sock_def_readable, sk);
|
| 1769 | nb--;
|
| 1770 | }
|
| 1771 | }
|
| 1772 |
|
| 1773 | if (unlikely(to_drop)) {
|
| 1774 | int err_ipv4 = 0;
|
| 1775 | int err_ipv6 = 0;
|
| 1776 |
|
| 1777 | for (nb = 0; to_drop != NULL; nb++) {
|
| 1778 | skb = to_drop;
|
| 1779 | if (skb->protocol == htons(ETH_P_IP))
|
| 1780 | err_ipv4++;
|
| 1781 | else
|
| 1782 | err_ipv6++;
|
| 1783 | to_drop = skb->next;
|
| 1784 | skb_mark_not_on_list(skb);
|
| 1785 | sk_skb_reason_drop(sk, skb, SKB_DROP_REASON_PROTO_MEM);
|
| 1786 | }
|
| 1787 | numa_drop_add(&udp_sk(sk)->drop_counters, nb);
|
| 1788 | if (err_ipv4 > 0) {
|
| 1789 | SNMP_ADD_STATS(__UDPX_MIB(sk, true), UDP_MIB_MEMERRORS,
|
| 1790 | err_ipv4);
|
| 1791 | SNMP_ADD_STATS(__UDPX_MIB(sk, true), UDP_MIB_INERRORS,
|
| 1792 | err_ipv4);
|
| 1793 | }
|
| 1794 | if (err_ipv6 > 0) {
|
| 1795 | SNMP_ADD_STATS(__UDPX_MIB(sk, false), UDP_MIB_MEMERRORS,
|
| 1796 | err_ipv6);
|
| 1797 | SNMP_ADD_STATS(__UDPX_MIB(sk, false), UDP_MIB_INERRORS,
|
| 1798 | err_ipv6);
|
| 1799 | }
|
| 1800 | }
|
| 1801 |
|
| 1802 | atomic_sub(total_size, &udp_prod_queue->rmem_alloc);
|
| 1803 |
|
| 1804 | return 0;
|
| 1805 |
|
| 1806 | drop:
|
| 1807 | udp_drops_inc(sk);
|
| 1808 | return err;
|
| 1809 | }
|
| 1810 |
|
| 1811 | void udp_destruct_common(struct sock *sk)
|
| 1812 | {
|
| 1813 |
|
| 1814 | struct udp_sock *up = udp_sk(sk);
|
| 1815 | unsigned int total = 0;
|
| 1816 | struct sk_buff *skb;
|
| 1817 |
|
| 1818 | skb_queue_splice_tail_init(&sk->sk_receive_queue, &up->reader_queue);
|
| 1819 | while ((skb = __skb_dequeue(&up->reader_queue)) != NULL) {
|
| 1820 | total += skb->truesize;
|
| 1821 | kfree_skb(skb);
|
| 1822 | }
|
| 1823 | udp_rmem_release(sk, total, 0, true);
|
| 1824 | kfree(up->udp_prod_queue);
|
| 1825 | }
|
| 1826 |
|
| 1827 | static void udp_destruct_sock(struct sock *sk)
|
| 1828 | {
|
| 1829 | udp_destruct_common(sk);
|
| 1830 | inet_sock_destruct(sk);
|
| 1831 | }
|
| 1832 |
|
| 1833 | static int udp_init_sock(struct sock *sk)
|
| 1834 | {
|
| 1835 | int res = udp_lib_init_sock(sk);
|
| 1836 |
|
| 1837 | sk->sk_destruct = udp_destruct_sock;
|
| 1838 | set_bit(SOCK_SUPPORT_ZC, &sk->sk_socket->flags);
|
| 1839 | return res;
|
| 1840 | }
|
| 1841 |
|
| 1842 | void skb_consume_udp(struct sock *sk, struct sk_buff *skb, int len)
|
| 1843 | {
|
| 1844 | if (unlikely(READ_ONCE(udp_sk(sk)->peeking_with_offset)))
|
| 1845 | sk_peek_offset_bwd(sk, len);
|
| 1846 |
|
| 1847 | if (!skb_shared(skb)) {
|
| 1848 | skb_orphan(skb);
|
| 1849 | skb_attempt_defer_free(skb);
|
| 1850 | return;
|
| 1851 | }
|
| 1852 |
|
| 1853 | if (!skb_unref(skb))
|
| 1854 | return;
|
| 1855 |
|
| 1856 | |
| 1857 | * see __udp_queue_rcv_skb().
|
| 1858 |
|
| 1859 | if (unlikely(udp_skb_has_head_state(skb)))
|
| 1860 | skb_release_head_state(skb);
|
| 1861 | __consume_stateless_skb(skb);
|
| 1862 | }
|
| 1863 |
|
| 1864 | static struct sk_buff *__first_packet_length(struct sock *sk,
|
| 1865 | struct sk_buff_head *rcvq,
|
| 1866 | unsigned int *total)
|
| 1867 | {
|
| 1868 | struct sk_buff *skb;
|
| 1869 |
|
| 1870 | while ((skb = skb_peek(rcvq)) != NULL) {
|
| 1871 | if (udp_lib_checksum_complete(skb)) {
|
| 1872 | struct net *net = sock_net(sk);
|
| 1873 |
|
| 1874 | __UDP_INC_STATS(net, UDP_MIB_CSUMERRORS);
|
| 1875 | __UDP_INC_STATS(net, UDP_MIB_INERRORS);
|
| 1876 | udp_drops_inc(sk);
|
| 1877 | __skb_unlink(skb, rcvq);
|
| 1878 | *total += skb->truesize;
|
| 1879 | kfree_skb_reason(skb, SKB_DROP_REASON_UDP_CSUM);
|
| 1880 | } else {
|
| 1881 | udp_skb_csum_unnecessary_set(skb);
|
| 1882 | break;
|
| 1883 | }
|
| 1884 | }
|
| 1885 | return skb;
|
| 1886 | }
|
| 1887 |
|
| 1888 | |
| 1889 | * first_packet_length - return length of first packet in receive queue
|
| 1890 | * @sk: socket
|
| 1891 | *
|
| 1892 | * Drops all bad checksum frames, until a valid one is found.
|
| 1893 | * Returns the length of found skb, or -1 if none is found.
|
| 1894 |
|
| 1895 | static int first_packet_length(struct sock *sk)
|
| 1896 | {
|
| 1897 | struct sk_buff_head *rcvq = &udp_sk(sk)->reader_queue;
|
| 1898 | struct sk_buff_head *sk_queue = &sk->sk_receive_queue;
|
| 1899 | unsigned int total = 0;
|
| 1900 | struct sk_buff *skb;
|
| 1901 | int res;
|
| 1902 |
|
| 1903 | spin_lock_bh(&rcvq->lock);
|
| 1904 | skb = __first_packet_length(sk, rcvq, &total);
|
| 1905 | if (!skb && !skb_queue_empty_lockless(sk_queue)) {
|
| 1906 | spin_lock(&sk_queue->lock);
|
| 1907 | skb_queue_splice_tail_init(sk_queue, rcvq);
|
| 1908 | spin_unlock(&sk_queue->lock);
|
| 1909 |
|
| 1910 | skb = __first_packet_length(sk, rcvq, &total);
|
| 1911 | }
|
| 1912 | res = skb ? skb->len : -1;
|
| 1913 | if (total)
|
| 1914 | udp_rmem_release(sk, total, 1, false);
|
| 1915 | spin_unlock_bh(&rcvq->lock);
|
| 1916 | return res;
|
| 1917 | }
|
| 1918 |
|
| 1919 | |
| 1920 | * IOCTL requests applicable to the UDP protocol
|
| 1921 |
|
| 1922 |
|
| 1923 | int udp_ioctl(struct sock *sk, int cmd, int *karg)
|
| 1924 | {
|
| 1925 | switch (cmd) {
|
| 1926 | case SIOCOUTQ:
|
| 1927 | {
|
| 1928 | *karg = sk_wmem_alloc_get(sk);
|
| 1929 | return 0;
|
| 1930 | }
|
| 1931 |
|
| 1932 | case SIOCINQ:
|
| 1933 | {
|
| 1934 | *karg = max_t(int, 0, first_packet_length(sk));
|
| 1935 | return 0;
|
| 1936 | }
|
| 1937 |
|
| 1938 | default:
|
| 1939 | return -ENOIOCTLCMD;
|
| 1940 | }
|
| 1941 |
|
| 1942 | return 0;
|
| 1943 | }
|
| 1944 |
|
| 1945 | struct sk_buff *__skb_recv_udp(struct sock *sk, unsigned int flags,
|
| 1946 | int *off, int *err)
|
| 1947 | {
|
| 1948 | struct sk_buff_head *sk_queue = &sk->sk_receive_queue;
|
| 1949 | struct sk_buff_head *queue;
|
| 1950 | struct sk_buff *last;
|
| 1951 | long timeo;
|
| 1952 | int error;
|
| 1953 |
|
| 1954 | queue = &udp_sk(sk)->reader_queue;
|
| 1955 | timeo = sock_rcvtimeo(sk, flags & MSG_DONTWAIT);
|
| 1956 | do {
|
| 1957 | struct sk_buff *skb;
|
| 1958 |
|
| 1959 | error = sock_error(sk);
|
| 1960 | if (error)
|
| 1961 | break;
|
| 1962 |
|
| 1963 | error = -EAGAIN;
|
| 1964 | do {
|
| 1965 | spin_lock_bh(&queue->lock);
|
| 1966 | skb = __skb_try_recv_from_queue(queue, flags, off, err,
|
| 1967 | &last);
|
| 1968 | if (skb) {
|
| 1969 | if (!(flags & MSG_PEEK))
|
| 1970 | udp_skb_destructor(sk, skb);
|
| 1971 | spin_unlock_bh(&queue->lock);
|
| 1972 | return skb;
|
| 1973 | }
|
| 1974 |
|
| 1975 | if (skb_queue_empty_lockless(sk_queue)) {
|
| 1976 | spin_unlock_bh(&queue->lock);
|
| 1977 | goto busy_check;
|
| 1978 | }
|
| 1979 |
|
| 1980 | |
| 1981 | * keep both queues locked to avoid re-acquiring
|
| 1982 | * the sk_receive_queue lock if fwd memory scheduling
|
| 1983 | * is needed.
|
| 1984 |
|
| 1985 | spin_lock(&sk_queue->lock);
|
| 1986 | skb_queue_splice_tail_init(sk_queue, queue);
|
| 1987 |
|
| 1988 | skb = __skb_try_recv_from_queue(queue, flags, off, err,
|
| 1989 | &last);
|
| 1990 | if (skb && !(flags & MSG_PEEK))
|
| 1991 | udp_skb_dtor_locked(sk, skb);
|
| 1992 | spin_unlock(&sk_queue->lock);
|
| 1993 | spin_unlock_bh(&queue->lock);
|
| 1994 | if (skb)
|
| 1995 | return skb;
|
| 1996 |
|
| 1997 | busy_check:
|
| 1998 | if (!sk_can_busy_loop(sk))
|
| 1999 | break;
|
| 2000 |
|
| 2001 | sk_busy_loop(sk, flags & MSG_DONTWAIT);
|
| 2002 | } while (!skb_queue_empty_lockless(sk_queue));
|
| 2003 |
|
| 2004 |
|
| 2005 | } while (timeo &&
|
| 2006 | !__skb_wait_for_more_packets(sk, &sk->sk_receive_queue,
|
| 2007 | &error, &timeo,
|
| 2008 | (struct sk_buff *)sk_queue));
|
| 2009 |
|
| 2010 | *err = error;
|
| 2011 | return NULL;
|
| 2012 | }
|
| 2013 | EXPORT_SYMBOL(__skb_recv_udp);
|
| 2014 |
|
| 2015 | int udp_read_skb(struct sock *sk, skb_read_actor_t recv_actor)
|
| 2016 | {
|
| 2017 | struct sk_buff *skb;
|
| 2018 | int err;
|
| 2019 |
|
| 2020 | try_again:
|
| 2021 | skb = skb_recv_udp(sk, MSG_DONTWAIT, &err);
|
| 2022 | if (!skb)
|
| 2023 | return err;
|
| 2024 |
|
| 2025 | if (udp_lib_checksum_complete(skb)) {
|
| 2026 | struct net *net = sock_net(sk);
|
| 2027 |
|
| 2028 | __UDP_INC_STATS(net, UDP_MIB_CSUMERRORS);
|
| 2029 | __UDP_INC_STATS(net, UDP_MIB_INERRORS);
|
| 2030 | udp_drops_inc(sk);
|
| 2031 | kfree_skb_reason(skb, SKB_DROP_REASON_UDP_CSUM);
|
| 2032 | goto try_again;
|
| 2033 | }
|
| 2034 |
|
| 2035 | WARN_ON_ONCE(!skb_set_owner_sk_safe(skb, sk));
|
| 2036 |
|
| 2037 | |
| 2038 | * skb->dev still aliases the UDP rx dev_scratch (its charge was freed
|
| 2039 | * on dequeue above); a sockmap verdict program may deref it via
|
| 2040 | * bpf_sk_lookup_*(), so clear it -> bpf_skc_lookup() uses skb->sk
|
| 2041 |
|
| 2042 | skb->dev = NULL;
|
| 2043 |
|
| 2044 | return recv_actor(sk, skb);
|
| 2045 | }
|
| 2046 |
|
| 2047 | |
| 2048 | * This should be easy, if there is something there we
|
| 2049 | * return it, otherwise we block.
|
| 2050 |
|
| 2051 |
|
| 2052 | INDIRECT_CALLABLE_SCOPE
|
| 2053 | int udp_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, int flags)
|
| 2054 | {
|
| 2055 | DECLARE_SOCKADDR(struct sockaddr_in *, sin, msg->msg_name);
|
| 2056 | int off, err, peeking = flags & MSG_PEEK;
|
| 2057 | struct inet_sock *inet = inet_sk(sk);
|
| 2058 | struct net *net = sock_net(sk);
|
| 2059 | bool checksum_valid = false;
|
| 2060 | unsigned int ulen, copied;
|
| 2061 | struct sk_buff *skb;
|
| 2062 |
|
| 2063 | if (flags & MSG_ERRQUEUE)
|
| 2064 | return ip_recv_error(sk, msg, len);
|
| 2065 |
|
| 2066 | try_again:
|
| 2067 | off = sk_peek_offset(sk, flags);
|
| 2068 | skb = __skb_recv_udp(sk, flags, &off, &err);
|
| 2069 | if (!skb)
|
| 2070 | return err;
|
| 2071 |
|
| 2072 | ulen = udp_skb_len(skb);
|
| 2073 | copied = len;
|
| 2074 | if (copied > ulen - off)
|
| 2075 | copied = ulen - off;
|
| 2076 | else if (copied < ulen)
|
| 2077 | msg->msg_flags |= MSG_TRUNC;
|
| 2078 |
|
| 2079 | |
| 2080 | * data. If the data is truncated, do it before the copy.
|
| 2081 |
|
| 2082 | if (copied < ulen || peeking) {
|
| 2083 | checksum_valid = udp_skb_csum_unnecessary(skb) ||
|
| 2084 | !__udp_lib_checksum_complete(skb);
|
| 2085 | if (!checksum_valid)
|
| 2086 | goto csum_copy_err;
|
| 2087 | }
|
| 2088 |
|
| 2089 | if (checksum_valid || udp_skb_csum_unnecessary(skb)) {
|
| 2090 | if (udp_skb_is_linear(skb))
|
| 2091 | err = copy_linear_skb(skb, copied, off, &msg->msg_iter);
|
| 2092 | else
|
| 2093 | err = skb_copy_datagram_msg(skb, off, msg, copied);
|
| 2094 | } else {
|
| 2095 | err = skb_copy_and_csum_datagram_msg(skb, off, msg);
|
| 2096 |
|
| 2097 | if (err == -EINVAL)
|
| 2098 | goto csum_copy_err;
|
| 2099 | }
|
| 2100 |
|
| 2101 | if (unlikely(err)) {
|
| 2102 | if (!peeking) {
|
| 2103 | udp_drops_inc(sk);
|
| 2104 | UDP_INC_STATS(net, UDP_MIB_INERRORS);
|
| 2105 | }
|
| 2106 | kfree_skb(skb);
|
| 2107 | return err;
|
| 2108 | }
|
| 2109 |
|
| 2110 | if (!peeking)
|
| 2111 | UDP_INC_STATS(net, UDP_MIB_INDATAGRAMS);
|
| 2112 |
|
| 2113 | sock_recv_cmsgs(msg, sk, skb);
|
| 2114 |
|
| 2115 |
|
| 2116 | if (sin) {
|
| 2117 | sin->sin_family = AF_INET;
|
| 2118 | sin->sin_port = udp_hdr(skb)->source;
|
| 2119 | sin->sin_addr.s_addr = ip_hdr(skb)->saddr;
|
| 2120 | memset(sin->sin_zero, 0, sizeof(sin->sin_zero));
|
| 2121 | msg->msg_namelen = sizeof(*sin);
|
| 2122 |
|
| 2123 | BPF_CGROUP_RUN_PROG_UDP4_RECVMSG_LOCK(sk,
|
| 2124 | (struct sockaddr *)sin,
|
| 2125 | &msg->msg_namelen);
|
| 2126 | }
|
| 2127 |
|
| 2128 | if (udp_test_bit(GRO_ENABLED, sk))
|
| 2129 | udp_cmsg_recv(msg, sk, skb);
|
| 2130 |
|
| 2131 | if (inet_cmsg_flags(inet))
|
| 2132 | ip_cmsg_recv_offset(msg, sk, skb, sizeof(struct udphdr), off);
|
| 2133 |
|
| 2134 | err = copied;
|
| 2135 | if (flags & MSG_TRUNC)
|
| 2136 | err = ulen;
|
| 2137 |
|
| 2138 | skb_consume_udp(sk, skb, peeking ? -err : err);
|
| 2139 | return err;
|
| 2140 |
|
| 2141 | csum_copy_err:
|
| 2142 | if (!__sk_queue_drop_skb(sk, &udp_sk(sk)->reader_queue, skb, flags,
|
| 2143 | udp_skb_destructor)) {
|
| 2144 | UDP_INC_STATS(net, UDP_MIB_CSUMERRORS);
|
| 2145 | UDP_INC_STATS(net, UDP_MIB_INERRORS);
|
| 2146 | }
|
| 2147 | kfree_skb_reason(skb, SKB_DROP_REASON_UDP_CSUM);
|
| 2148 |
|
| 2149 |
|
| 2150 | cond_resched();
|
| 2151 | msg->msg_flags &= ~MSG_TRUNC;
|
| 2152 | goto try_again;
|
| 2153 | }
|
| 2154 |
|
| 2155 | int udp_pre_connect(struct sock *sk, struct sockaddr_unsized *uaddr,
|
| 2156 | int addr_len)
|
| 2157 | {
|
| 2158 | |
| 2159 | * intended to prevent BPF program called below from accessing bytes
|
| 2160 | * that are out of the bound specified by user in addr_len.
|
| 2161 |
|
| 2162 | if (addr_len < sizeof(struct sockaddr_in))
|
| 2163 | return -EINVAL;
|
| 2164 |
|
| 2165 | return BPF_CGROUP_RUN_PROG_INET4_CONNECT_LOCK(sk, uaddr, &addr_len);
|
| 2166 | }
|
| 2167 |
|
| 2168 | static int udp_connect(struct sock *sk, struct sockaddr_unsized *uaddr,
|
| 2169 | int addr_len)
|
| 2170 | {
|
| 2171 | int res;
|
| 2172 |
|
| 2173 | lock_sock(sk);
|
| 2174 | res = __ip4_datagram_connect(sk, uaddr, addr_len);
|
| 2175 | if (!res)
|
| 2176 | udp4_hash4(sk);
|
| 2177 | release_sock(sk);
|
| 2178 | return res;
|
| 2179 | }
|
| 2180 |
|
| 2181 | int __udp_disconnect(struct sock *sk, int flags)
|
| 2182 | {
|
| 2183 | struct inet_sock *inet = inet_sk(sk);
|
| 2184 | |
| 2185 | * 1003.1g - break association.
|
| 2186 |
|
| 2187 |
|
| 2188 | sk->sk_state = TCP_CLOSE;
|
| 2189 | WRITE_ONCE(inet->inet_daddr, 0);
|
| 2190 | inet->inet_dport = 0;
|
| 2191 | sock_rps_reset_rxhash(sk);
|
| 2192 | WRITE_ONCE(sk->sk_bound_dev_if, 0);
|
| 2193 | if (!(sk->sk_userlocks & SOCK_BINDADDR_LOCK)) {
|
| 2194 | inet_reset_saddr(sk);
|
| 2195 | if (sk->sk_prot->rehash &&
|
| 2196 | (sk->sk_userlocks & SOCK_BINDPORT_LOCK))
|
| 2197 | sk->sk_prot->rehash(sk);
|
| 2198 | }
|
| 2199 |
|
| 2200 | if (!(sk->sk_userlocks & SOCK_BINDPORT_LOCK)) {
|
| 2201 | sk->sk_prot->unhash(sk);
|
| 2202 | inet->inet_sport = 0;
|
| 2203 | }
|
| 2204 | sk_dst_reset(sk);
|
| 2205 | return 0;
|
| 2206 | }
|
| 2207 | EXPORT_SYMBOL(__udp_disconnect);
|
| 2208 |
|
| 2209 | |
| 2210 | * ->rehash() or ->unhash(), and neither runs for a socket bound to a
|
| 2211 | * specific address and port. Remove it here, before its peer is cleared.
|
| 2212 |
|
| 2213 | static void udp_unhash4_on_disconnect(struct sock *sk)
|
| 2214 | {
|
| 2215 | struct net *net = sock_net(sk);
|
| 2216 | struct udp_table *udptable;
|
| 2217 | struct udp_hslot *hslot;
|
| 2218 |
|
| 2219 | if (!udp_hashed4(sk))
|
| 2220 | return;
|
| 2221 |
|
| 2222 | udptable = net->ipv4.udp_table;
|
| 2223 | hslot = udp_hashslot(udptable, net, udp_sk(sk)->udp_port_hash);
|
| 2224 |
|
| 2225 | spin_lock_bh(&hslot->lock);
|
| 2226 | udp_unhash4(udptable, sk);
|
| 2227 | spin_unlock_bh(&hslot->lock);
|
| 2228 | }
|
| 2229 |
|
| 2230 | int udp_disconnect(struct sock *sk, int flags)
|
| 2231 | {
|
| 2232 | lock_sock(sk);
|
| 2233 | udp_unhash4_on_disconnect(sk);
|
| 2234 | __udp_disconnect(sk, flags);
|
| 2235 | release_sock(sk);
|
| 2236 | return 0;
|
| 2237 | }
|
| 2238 |
|
| 2239 | void udp_lib_unhash(struct sock *sk)
|
| 2240 | {
|
| 2241 | if (sk_hashed(sk)) {
|
| 2242 | struct udp_hslot *hslot, *hslot2;
|
| 2243 | struct net *net = sock_net(sk);
|
| 2244 | struct udp_table *udptable;
|
| 2245 |
|
| 2246 | sock_rps_delete_flow(sk);
|
| 2247 | udptable = net->ipv4.udp_table;
|
| 2248 | hslot = udp_hashslot(udptable, net, udp_sk(sk)->udp_port_hash);
|
| 2249 | hslot2 = udp_hashslot2(udptable, udp_sk(sk)->udp_portaddr_hash);
|
| 2250 |
|
| 2251 | spin_lock_bh(&hslot->lock);
|
| 2252 | if (rcu_access_pointer(sk->sk_reuseport_cb))
|
| 2253 | reuseport_detach_sock(sk);
|
| 2254 | if (sk_del_node_init_rcu(sk)) {
|
| 2255 | hslot->count--;
|
| 2256 | inet_sk(sk)->inet_num = 0;
|
| 2257 | sock_prot_inuse_add(net, sk->sk_prot, -1);
|
| 2258 |
|
| 2259 | spin_lock(&hslot2->lock);
|
| 2260 | hlist_del_init_rcu(&udp_sk(sk)->udp_portaddr_node);
|
| 2261 | hslot2->count--;
|
| 2262 | spin_unlock(&hslot2->lock);
|
| 2263 |
|
| 2264 | udp_unhash4(udptable, sk);
|
| 2265 | }
|
| 2266 | spin_unlock_bh(&hslot->lock);
|
| 2267 | }
|
| 2268 | }
|
| 2269 |
|
| 2270 | |
| 2271 | * inet_rcv_saddr was changed, we must rehash secondary hash
|
| 2272 |
|
| 2273 | void udp_lib_rehash(struct sock *sk, u16 newhash, u16 newhash4)
|
| 2274 | {
|
| 2275 | if (sk_hashed(sk)) {
|
| 2276 | struct udp_hslot *hslot, *hslot2, *nhslot2;
|
| 2277 | struct net *net = sock_net(sk);
|
| 2278 | struct udp_table *udptable;
|
| 2279 |
|
| 2280 | udptable = net->ipv4.udp_table;
|
| 2281 | hslot = udp_hashslot(udptable, net, udp_sk(sk)->udp_port_hash);
|
| 2282 | hslot2 = udp_hashslot2(udptable, udp_sk(sk)->udp_portaddr_hash);
|
| 2283 | nhslot2 = udp_hashslot2(udptable, newhash);
|
| 2284 |
|
| 2285 | if (hslot2 != nhslot2 ||
|
| 2286 | rcu_access_pointer(sk->sk_reuseport_cb)) {
|
| 2287 |
|
| 2288 | spin_lock_bh(&hslot->lock);
|
| 2289 | if (rcu_access_pointer(sk->sk_reuseport_cb))
|
| 2290 | reuseport_detach_sock(sk);
|
| 2291 |
|
| 2292 | if (hslot2 != nhslot2) {
|
| 2293 | spin_lock(&hslot2->lock);
|
| 2294 | hlist_del_init_rcu(&udp_sk(sk)->udp_portaddr_node);
|
| 2295 | hslot2->count--;
|
| 2296 | spin_unlock(&hslot2->lock);
|
| 2297 |
|
| 2298 | spin_lock(&nhslot2->lock);
|
| 2299 | hlist_add_head_rcu(&udp_sk(sk)->udp_portaddr_node,
|
| 2300 | &nhslot2->head);
|
| 2301 | nhslot2->count++;
|
| 2302 | spin_unlock(&nhslot2->lock);
|
| 2303 | }
|
| 2304 |
|
| 2305 | spin_unlock_bh(&hslot->lock);
|
| 2306 | }
|
| 2307 |
|
| 2308 | |
| 2309 | * (1) update hslot4;
|
| 2310 | * (2) update hslot2->hash4_cnt.
|
| 2311 | * Note that hslot2/hslot4 should be checked separately, as
|
| 2312 | * either of them may change with the other unchanged.
|
| 2313 |
|
| 2314 | if (udp_hashed4(sk)) {
|
| 2315 | spin_lock_bh(&hslot->lock);
|
| 2316 |
|
| 2317 | if (inet_rcv_saddr_any(sk)) {
|
| 2318 | udp_unhash4(udptable, sk);
|
| 2319 | } else {
|
| 2320 | udp_rehash4(udptable, sk, newhash4);
|
| 2321 | if (hslot2 != nhslot2) {
|
| 2322 | spin_lock(&hslot2->lock);
|
| 2323 | udp_hash4_dec(hslot2);
|
| 2324 | spin_unlock(&hslot2->lock);
|
| 2325 |
|
| 2326 | spin_lock(&nhslot2->lock);
|
| 2327 | udp_hash4_inc(nhslot2);
|
| 2328 | spin_unlock(&nhslot2->lock);
|
| 2329 | }
|
| 2330 | }
|
| 2331 |
|
| 2332 | spin_unlock_bh(&hslot->lock);
|
| 2333 | }
|
| 2334 |
|
| 2335 | udp_sk(sk)->udp_portaddr_hash = newhash;
|
| 2336 | }
|
| 2337 | }
|
| 2338 |
|
| 2339 | static void udp_v4_rehash(struct sock *sk)
|
| 2340 | {
|
| 2341 | u16 new_hash = ipv4_portaddr_hash(sock_net(sk),
|
| 2342 | inet_sk(sk)->inet_rcv_saddr,
|
| 2343 | inet_sk(sk)->inet_num);
|
| 2344 | u16 new_hash4 = udp_ehashfn(sock_net(sk),
|
| 2345 | sk->sk_rcv_saddr, sk->sk_num,
|
| 2346 | sk->sk_daddr, sk->sk_dport);
|
| 2347 |
|
| 2348 | udp_lib_rehash(sk, new_hash, new_hash4);
|
| 2349 | }
|
| 2350 |
|
| 2351 | static int __udp_queue_rcv_skb(struct sock *sk, struct sk_buff *skb)
|
| 2352 | {
|
| 2353 | int rc;
|
| 2354 |
|
| 2355 | if (inet_sk(sk)->inet_daddr) {
|
| 2356 | sock_rps_save_rxhash(sk, skb);
|
| 2357 | sk_mark_napi_id(sk, skb);
|
| 2358 | sk_incoming_cpu_update(sk);
|
| 2359 | } else {
|
| 2360 | sk_mark_napi_id_once(sk, skb);
|
| 2361 | }
|
| 2362 |
|
| 2363 | rc = __udp_enqueue_schedule_skb(sk, skb);
|
| 2364 | if (rc < 0) {
|
| 2365 | struct net *net = sock_net(sk);
|
| 2366 | int drop_reason;
|
| 2367 |
|
| 2368 |
|
| 2369 | if (rc == -ENOMEM) {
|
| 2370 | UDP_INC_STATS(net, UDP_MIB_RCVBUFERRORS);
|
| 2371 | drop_reason = SKB_DROP_REASON_SOCKET_RCVBUFF;
|
| 2372 | } else {
|
| 2373 | UDP_INC_STATS(net, UDP_MIB_MEMERRORS);
|
| 2374 | drop_reason = SKB_DROP_REASON_PROTO_MEM;
|
| 2375 | }
|
| 2376 | UDP_INC_STATS(net, UDP_MIB_INERRORS);
|
| 2377 | trace_udp_fail_queue_rcv_skb(rc, sk, skb);
|
| 2378 | sk_skb_reason_drop(sk, skb, drop_reason);
|
| 2379 | return -1;
|
| 2380 | }
|
| 2381 |
|
| 2382 | return 0;
|
| 2383 | }
|
| 2384 |
|
| 2385 | |
| 2386 | * -1: error
|
| 2387 | * 0: success
|
| 2388 | * >0: "udp encap" protocol resubmission
|
| 2389 | *
|
| 2390 | * Note that in the success and error cases, the skb is assumed to
|
| 2391 | * have either been requeued or freed.
|
| 2392 |
|
| 2393 | static int udp_queue_rcv_one_skb(struct sock *sk, struct sk_buff *skb)
|
| 2394 | {
|
| 2395 | enum skb_drop_reason drop_reason = SKB_DROP_REASON_NOT_SPECIFIED;
|
| 2396 | struct udp_sock *up = udp_sk(sk);
|
| 2397 | struct net *net = sock_net(sk);
|
| 2398 |
|
| 2399 | |
| 2400 | * Charge it to the socket, dropping if the queue is full.
|
| 2401 |
|
| 2402 | if (!xfrm4_policy_check(sk, XFRM_POLICY_IN, skb)) {
|
| 2403 | drop_reason = SKB_DROP_REASON_XFRM_POLICY;
|
| 2404 | goto drop;
|
| 2405 | }
|
| 2406 | nf_reset_ct(skb);
|
| 2407 |
|
| 2408 | if (static_branch_unlikely(&udp_encap_needed_key) &&
|
| 2409 | READ_ONCE(up->encap_type)) {
|
| 2410 | int (*encap_rcv)(struct sock *sk, struct sk_buff *skb);
|
| 2411 |
|
| 2412 | |
| 2413 | * This is an encapsulation socket so pass the skb to
|
| 2414 | * the socket's udp_encap_rcv() hook. Otherwise, just
|
| 2415 | * fall through and pass this up the UDP socket.
|
| 2416 | * up->encap_rcv() returns the following value:
|
| 2417 | * =0 if skb was successfully passed to the encap
|
| 2418 | * handler or was discarded by it.
|
| 2419 | * >0 if skb should be passed on to UDP.
|
| 2420 | * <0 if skb should be resubmitted as proto -N
|
| 2421 |
|
| 2422 |
|
| 2423 |
|
| 2424 | encap_rcv = READ_ONCE(up->encap_rcv);
|
| 2425 | if (encap_rcv) {
|
| 2426 | int ret;
|
| 2427 |
|
| 2428 |
|
| 2429 | if (udp_lib_checksum_complete(skb))
|
| 2430 | goto csum_error;
|
| 2431 |
|
| 2432 | ret = encap_rcv(sk, skb);
|
| 2433 | if (ret <= 0) {
|
| 2434 | __UDP_INC_STATS(net, UDP_MIB_INDATAGRAMS);
|
| 2435 | return -ret;
|
| 2436 | }
|
| 2437 | }
|
| 2438 |
|
| 2439 |
|
| 2440 | }
|
| 2441 |
|
| 2442 | prefetch(&sk->sk_rmem_alloc);
|
| 2443 | if (rcu_access_pointer(sk->sk_filter) &&
|
| 2444 | udp_lib_checksum_complete(skb))
|
| 2445 | goto csum_error;
|
| 2446 |
|
| 2447 | drop_reason = sk_filter_trim_cap(sk, skb, sizeof(struct udphdr));
|
| 2448 | if (drop_reason)
|
| 2449 | goto drop;
|
| 2450 |
|
| 2451 | udp_csum_pull_header(skb);
|
| 2452 |
|
| 2453 | ipv4_pktinfo_prepare(sk, skb, true);
|
| 2454 | return __udp_queue_rcv_skb(sk, skb);
|
| 2455 |
|
| 2456 | csum_error:
|
| 2457 | drop_reason = SKB_DROP_REASON_UDP_CSUM;
|
| 2458 | __UDP_INC_STATS(net, UDP_MIB_CSUMERRORS);
|
| 2459 | drop:
|
| 2460 | __UDP_INC_STATS(net, UDP_MIB_INERRORS);
|
| 2461 | udp_drops_inc(sk);
|
| 2462 | sk_skb_reason_drop(sk, skb, drop_reason);
|
| 2463 | return -1;
|
| 2464 | }
|
| 2465 |
|
| 2466 | static int udp_queue_rcv_skb(struct sock *sk, struct sk_buff *skb)
|
| 2467 | {
|
| 2468 | struct sk_buff *next, *segs;
|
| 2469 | int ret;
|
| 2470 |
|
| 2471 | if (likely(!udp_unexpected_gso(sk, skb)))
|
| 2472 | return udp_queue_rcv_one_skb(sk, skb);
|
| 2473 |
|
| 2474 | BUILD_BUG_ON(sizeof(struct udp_skb_cb) > SKB_GSO_CB_OFFSET);
|
| 2475 | __skb_push(skb, -skb_mac_offset(skb));
|
| 2476 | segs = udp_rcv_segment(sk, skb, true);
|
| 2477 | skb_list_walk_safe(segs, skb, next) {
|
| 2478 | __skb_pull(skb, skb_transport_offset(skb));
|
| 2479 |
|
| 2480 | udp_post_segment_fix_csum(skb);
|
| 2481 | ret = udp_queue_rcv_one_skb(sk, skb);
|
| 2482 | if (ret > 0)
|
| 2483 | ip_protocol_deliver_rcu(dev_net(skb->dev), skb, ret);
|
| 2484 | }
|
| 2485 | return 0;
|
| 2486 | }
|
| 2487 |
|
| 2488 | |
| 2489 | * For UDP, we use xchg() to guard against concurrent changes.
|
| 2490 |
|
| 2491 | bool udp_sk_rx_dst_set(struct sock *sk, struct dst_entry *dst)
|
| 2492 | {
|
| 2493 | struct dst_entry *old;
|
| 2494 |
|
| 2495 | if (dst_hold_safe(dst)) {
|
| 2496 | old = unrcu_pointer(xchg(&sk->sk_rx_dst, RCU_INITIALIZER(dst)));
|
| 2497 | dst_release(old);
|
| 2498 | return old != dst;
|
| 2499 | }
|
| 2500 | return false;
|
| 2501 | }
|
| 2502 |
|
| 2503 | |
| 2504 | * Multicasts and broadcasts go to each listener.
|
| 2505 | *
|
| 2506 | * Note: called only from the BH handler context.
|
| 2507 |
|
| 2508 | static int __udp4_lib_mcast_deliver(struct net *net, struct sk_buff *skb,
|
| 2509 | struct udphdr *uh,
|
| 2510 | __be32 saddr, __be32 daddr)
|
| 2511 | {
|
| 2512 | struct udp_table *udptable = net->ipv4.udp_table;
|
| 2513 | unsigned int hash2, hash2_any, offset;
|
| 2514 | unsigned short hnum = ntohs(uh->dest);
|
| 2515 | struct sock *sk, *first = NULL;
|
| 2516 | int dif = skb->dev->ifindex;
|
| 2517 | int sdif = inet_sdif(skb);
|
| 2518 | struct hlist_node *node;
|
| 2519 | struct udp_hslot *hslot;
|
| 2520 | struct sk_buff *nskb;
|
| 2521 | bool use_hash2;
|
| 2522 | int ret;
|
| 2523 |
|
| 2524 | hash2_any = 0;
|
| 2525 | hash2 = 0;
|
| 2526 | hslot = udp_hashslot(udptable, net, hnum);
|
| 2527 | use_hash2 = hslot->count > 10;
|
| 2528 | offset = offsetof(typeof(*sk), sk_node);
|
| 2529 |
|
| 2530 | if (use_hash2) {
|
| 2531 | hash2_any = ipv4_portaddr_hash(net, htonl(INADDR_ANY), hnum) &
|
| 2532 | udptable->mask;
|
| 2533 | hash2 = ipv4_portaddr_hash(net, daddr, hnum) & udptable->mask;
|
| 2534 | start_lookup:
|
| 2535 | hslot = &udptable->hash2[hash2].hslot;
|
| 2536 | offset = offsetof(typeof(*sk), __sk_common.skc_portaddr_node);
|
| 2537 | }
|
| 2538 |
|
| 2539 | sk_for_each_entry_offset_rcu(sk, node, &hslot->head, offset) {
|
| 2540 | if (!__udp_is_mcast_sock(net, sk, uh->dest, daddr,
|
| 2541 | uh->source, saddr, dif, sdif, hnum))
|
| 2542 | continue;
|
| 2543 |
|
| 2544 | if (!first) {
|
| 2545 | first = sk;
|
| 2546 | continue;
|
| 2547 | }
|
| 2548 | nskb = skb_clone(skb, GFP_ATOMIC);
|
| 2549 |
|
| 2550 | if (unlikely(!nskb)) {
|
| 2551 | udp_drops_inc(sk);
|
| 2552 | __UDP_INC_STATS(net, UDP_MIB_RCVBUFERRORS);
|
| 2553 | __UDP_INC_STATS(net, UDP_MIB_INERRORS);
|
| 2554 | continue;
|
| 2555 | }
|
| 2556 | if (udp_queue_rcv_skb(sk, nskb) > 0)
|
| 2557 | consume_skb(nskb);
|
| 2558 | }
|
| 2559 |
|
| 2560 |
|
| 2561 | if (use_hash2 && hash2 != hash2_any) {
|
| 2562 | hash2 = hash2_any;
|
| 2563 | goto start_lookup;
|
| 2564 | }
|
| 2565 |
|
| 2566 | if (first) {
|
| 2567 | ret = udp_queue_rcv_skb(first, skb);
|
| 2568 | if (ret > 0)
|
| 2569 | return -ret;
|
| 2570 | } else {
|
| 2571 | kfree_skb(skb);
|
| 2572 | __UDP_INC_STATS(net, UDP_MIB_IGNOREDMULTI);
|
| 2573 | }
|
| 2574 | return 0;
|
| 2575 | }
|
| 2576 |
|
| 2577 | |
| 2578 | * CHECKSUM_UNNECESSARY means, that no more checks are required.
|
| 2579 | * Otherwise, csum completion requires checksumming packet body,
|
| 2580 | * including udp header and folding it to skb->csum.
|
| 2581 |
|
| 2582 | static inline int udp4_csum_init(struct sk_buff *skb, struct udphdr *uh)
|
| 2583 | {
|
| 2584 | int err;
|
| 2585 |
|
| 2586 | |
| 2587 | * force to int.
|
| 2588 |
|
| 2589 | err = (__force int)skb_checksum_init_zero_check(skb, IPPROTO_UDP, uh->check,
|
| 2590 | inet_compute_pseudo);
|
| 2591 | if (err)
|
| 2592 | return err;
|
| 2593 |
|
| 2594 | if (skb->ip_summed == CHECKSUM_COMPLETE && !skb->csum_valid) {
|
| 2595 |
|
| 2596 | if (skb->csum_complete_sw)
|
| 2597 | return 1;
|
| 2598 |
|
| 2599 | |
| 2600 | * skb->csum is no longer the full packet checksum,
|
| 2601 | * so don't treat it as such.
|
| 2602 |
|
| 2603 | skb_checksum_complete_unset(skb);
|
| 2604 | }
|
| 2605 |
|
| 2606 | return 0;
|
| 2607 | }
|
| 2608 |
|
| 2609 | |
| 2610 | * return code conversion for ip layer consumption
|
| 2611 |
|
| 2612 | static int udp_unicast_rcv_skb(struct sock *sk, struct sk_buff *skb,
|
| 2613 | struct udphdr *uh)
|
| 2614 | {
|
| 2615 | int ret;
|
| 2616 |
|
| 2617 | if (inet_get_convert_csum(sk) && uh->check)
|
| 2618 | skb_checksum_try_convert(skb, IPPROTO_UDP, inet_compute_pseudo);
|
| 2619 |
|
| 2620 | ret = udp_queue_rcv_skb(sk, skb);
|
| 2621 |
|
| 2622 | |
| 2623 | * it wants the return to be -protocol, or 0
|
| 2624 |
|
| 2625 | if (ret > 0)
|
| 2626 | return -ret;
|
| 2627 | return 0;
|
| 2628 | }
|
| 2629 |
|
| 2630 | |
| 2631 | * All we need to do is get the socket, and then do a checksum.
|
| 2632 |
|
| 2633 |
|
| 2634 | int udp_rcv(struct sk_buff *skb)
|
| 2635 | {
|
| 2636 | struct rtable *rt = skb_rtable(skb);
|
| 2637 | struct net *net = dev_net(skb->dev);
|
| 2638 | struct sock *sk = NULL;
|
| 2639 | __be32 saddr, daddr;
|
| 2640 | unsigned int ulen;
|
| 2641 | struct udphdr *uh;
|
| 2642 | bool refcounted;
|
| 2643 | int drop_reason;
|
| 2644 |
|
| 2645 | drop_reason = SKB_DROP_REASON_NOT_SPECIFIED;
|
| 2646 |
|
| 2647 | |
| 2648 | * Validate the packet.
|
| 2649 |
|
| 2650 | if (!pskb_may_pull(skb, sizeof(struct udphdr)))
|
| 2651 | goto drop;
|
| 2652 |
|
| 2653 | uh = udp_hdr(skb);
|
| 2654 | ulen = udp_get_len(skb, uh, 0);
|
| 2655 | saddr = ip_hdr(skb)->saddr;
|
| 2656 | daddr = ip_hdr(skb)->daddr;
|
| 2657 |
|
| 2658 | if (ulen > skb->len)
|
| 2659 | goto short_packet;
|
| 2660 |
|
| 2661 | if (ulen < sizeof(*uh))
|
| 2662 | goto short_packet;
|
| 2663 |
|
| 2664 | if (ulen < skb->len) {
|
| 2665 | if (pskb_trim_rcsum(skb, ulen))
|
| 2666 | goto short_packet;
|
| 2667 |
|
| 2668 | uh = udp_hdr(skb);
|
| 2669 | }
|
| 2670 |
|
| 2671 | if (udp4_csum_init(skb, uh))
|
| 2672 | goto csum_error;
|
| 2673 |
|
| 2674 | sk = inet_steal_sock(net, skb, sizeof(struct udphdr), saddr, uh->source, daddr, uh->dest,
|
| 2675 | &refcounted, udp_ehashfn);
|
| 2676 | if (IS_ERR(sk))
|
| 2677 | goto no_sk;
|
| 2678 |
|
| 2679 | if (sk) {
|
| 2680 | struct dst_entry *dst = skb_dst(skb);
|
| 2681 | int ret;
|
| 2682 |
|
| 2683 | if (unlikely(rcu_dereference(sk->sk_rx_dst) != dst))
|
| 2684 | udp_sk_rx_dst_set(sk, dst);
|
| 2685 |
|
| 2686 | ret = udp_unicast_rcv_skb(sk, skb, uh);
|
| 2687 | if (refcounted)
|
| 2688 | sock_put(sk);
|
| 2689 | return ret;
|
| 2690 | }
|
| 2691 |
|
| 2692 | if (rt->rt_flags & (RTCF_BROADCAST|RTCF_MULTICAST))
|
| 2693 | return __udp4_lib_mcast_deliver(net, skb, uh, saddr, daddr);
|
| 2694 |
|
| 2695 | sk = __udp4_lib_lookup_skb(skb, uh->source, uh->dest);
|
| 2696 | if (sk)
|
| 2697 | return udp_unicast_rcv_skb(sk, skb, uh);
|
| 2698 | no_sk:
|
| 2699 | if (!xfrm4_policy_check(NULL, XFRM_POLICY_IN, skb))
|
| 2700 | goto drop;
|
| 2701 | nf_reset_ct(skb);
|
| 2702 |
|
| 2703 |
|
| 2704 | if (udp_lib_checksum_complete(skb))
|
| 2705 | goto csum_error;
|
| 2706 |
|
| 2707 | drop_reason = SKB_DROP_REASON_NO_SOCKET;
|
| 2708 | __UDP_INC_STATS(net, UDP_MIB_NOPORTS);
|
| 2709 | icmp_send(skb, ICMP_DEST_UNREACH, ICMP_PORT_UNREACH, 0);
|
| 2710 |
|
| 2711 | |
| 2712 | * Hmm. We got an UDP packet to a port to which we
|
| 2713 | * don't wanna listen. Ignore it.
|
| 2714 |
|
| 2715 | sk_skb_reason_drop(sk, skb, drop_reason);
|
| 2716 | return 0;
|
| 2717 |
|
| 2718 | short_packet:
|
| 2719 | drop_reason = SKB_DROP_REASON_PKT_TOO_SMALL;
|
| 2720 | net_dbg_ratelimited("UDP: short packet: From %pI4:%u %d/%d to %pI4:%u\n",
|
| 2721 | &saddr, ntohs(uh->source),
|
| 2722 | ulen, skb->len,
|
| 2723 | &daddr, ntohs(uh->dest));
|
| 2724 | goto drop;
|
| 2725 |
|
| 2726 | csum_error:
|
| 2727 | |
| 2728 | * RFC1122: OK. Discards the bad packet silently (as far as
|
| 2729 | * the network is concerned, anyway) as per 4.1.3.4 (MUST).
|
| 2730 |
|
| 2731 | drop_reason = SKB_DROP_REASON_UDP_CSUM;
|
| 2732 | net_dbg_ratelimited("UDP: bad checksum. From %pI4:%u to %pI4:%u ulen %d\n",
|
| 2733 | &saddr, ntohs(uh->source), &daddr, ntohs(uh->dest),
|
| 2734 | ulen);
|
| 2735 | __UDP_INC_STATS(net, UDP_MIB_CSUMERRORS);
|
| 2736 | drop:
|
| 2737 | __UDP_INC_STATS(net, UDP_MIB_INERRORS);
|
| 2738 | sk_skb_reason_drop(sk, skb, drop_reason);
|
| 2739 | return 0;
|
| 2740 | }
|
| 2741 |
|
| 2742 | |
| 2743 | * If more than one socket found returns NULL
|
| 2744 |
|
| 2745 | static struct sock *__udp4_lib_mcast_demux_lookup(struct net *net,
|
| 2746 | __be16 loc_port, __be32 loc_addr,
|
| 2747 | __be16 rmt_port, __be32 rmt_addr,
|
| 2748 | int dif, int sdif)
|
| 2749 | {
|
| 2750 | struct udp_table *udptable = net->ipv4.udp_table;
|
| 2751 | unsigned short hnum = ntohs(loc_port);
|
| 2752 | struct sock *sk, *result;
|
| 2753 | struct udp_hslot *hslot;
|
| 2754 | unsigned int slot;
|
| 2755 |
|
| 2756 | slot = udp_hashfn(net, hnum, udptable->mask);
|
| 2757 | hslot = &udptable->hash[slot];
|
| 2758 |
|
| 2759 |
|
| 2760 | if (hslot->count > 10)
|
| 2761 | return NULL;
|
| 2762 |
|
| 2763 | result = NULL;
|
| 2764 | sk_for_each_rcu(sk, &hslot->head) {
|
| 2765 | if (__udp_is_mcast_sock(net, sk, loc_port, loc_addr,
|
| 2766 | rmt_port, rmt_addr, dif, sdif, hnum)) {
|
| 2767 | if (result)
|
| 2768 | return NULL;
|
| 2769 | result = sk;
|
| 2770 | }
|
| 2771 | }
|
| 2772 |
|
| 2773 | return result;
|
| 2774 | }
|
| 2775 |
|
| 2776 | |
| 2777 | * break forwarding setups. The chains here can be long so only check
|
| 2778 | * if the first socket is an exact match and if not move on.
|
| 2779 |
|
| 2780 | static struct sock *__udp4_lib_demux_lookup(struct net *net,
|
| 2781 | __be16 loc_port, __be32 loc_addr,
|
| 2782 | __be16 rmt_port, __be32 rmt_addr,
|
| 2783 | int dif, int sdif)
|
| 2784 | {
|
| 2785 | struct udp_table *udptable = net->ipv4.udp_table;
|
| 2786 | INET_ADDR_COOKIE(acookie, rmt_addr, loc_addr);
|
| 2787 | unsigned short hnum = ntohs(loc_port);
|
| 2788 | struct udp_hslot *hslot2;
|
| 2789 | unsigned int hash2;
|
| 2790 | __portpair ports;
|
| 2791 | struct sock *sk;
|
| 2792 |
|
| 2793 | hash2 = ipv4_portaddr_hash(net, loc_addr, hnum);
|
| 2794 | hslot2 = udp_hashslot2(udptable, hash2);
|
| 2795 | ports = INET_COMBINED_PORTS(rmt_port, hnum);
|
| 2796 |
|
| 2797 | udp_portaddr_for_each_entry_rcu(sk, &hslot2->head) {
|
| 2798 | if (inet_match(net, sk, acookie, ports, dif, sdif))
|
| 2799 | return sk;
|
| 2800 |
|
| 2801 | break;
|
| 2802 | }
|
| 2803 | return NULL;
|
| 2804 | }
|
| 2805 |
|
| 2806 | enum skb_drop_reason udp_v4_early_demux(struct sk_buff *skb)
|
| 2807 | {
|
| 2808 | struct net *net = dev_net(skb->dev);
|
| 2809 | struct in_device *in_dev = NULL;
|
| 2810 | const struct iphdr *iph;
|
| 2811 | const struct udphdr *uh;
|
| 2812 | struct sock *sk = NULL;
|
| 2813 | struct dst_entry *dst;
|
| 2814 | int dif = skb->dev->ifindex;
|
| 2815 | int sdif = inet_sdif(skb);
|
| 2816 | int ours;
|
| 2817 |
|
| 2818 |
|
| 2819 | if (!pskb_may_pull(skb, skb_transport_offset(skb) + sizeof(struct udphdr)))
|
| 2820 | return SKB_NOT_DROPPED_YET;
|
| 2821 |
|
| 2822 | iph = ip_hdr(skb);
|
| 2823 | uh = udp_hdr(skb);
|
| 2824 |
|
| 2825 | if (skb->pkt_type == PACKET_MULTICAST) {
|
| 2826 | in_dev = __in_dev_get_rcu(skb->dev);
|
| 2827 |
|
| 2828 | if (!in_dev)
|
| 2829 | return SKB_NOT_DROPPED_YET;
|
| 2830 |
|
| 2831 | ours = ip_check_mc_rcu(in_dev, iph->daddr, iph->saddr,
|
| 2832 | iph->protocol);
|
| 2833 | if (!ours)
|
| 2834 | return SKB_NOT_DROPPED_YET;
|
| 2835 |
|
| 2836 | sk = __udp4_lib_mcast_demux_lookup(net, uh->dest, iph->daddr,
|
| 2837 | uh->source, iph->saddr,
|
| 2838 | dif, sdif);
|
| 2839 | } else if (skb->pkt_type == PACKET_HOST) {
|
| 2840 | sk = __udp4_lib_demux_lookup(net, uh->dest, iph->daddr,
|
| 2841 | uh->source, iph->saddr, dif, sdif);
|
| 2842 | }
|
| 2843 |
|
| 2844 | if (!sk)
|
| 2845 | return SKB_NOT_DROPPED_YET;
|
| 2846 |
|
| 2847 | skb->sk = sk;
|
| 2848 | DEBUG_NET_WARN_ON_ONCE(sk_is_refcounted(sk));
|
| 2849 | skb->destructor = sock_pfree;
|
| 2850 | dst = rcu_dereference(sk->sk_rx_dst);
|
| 2851 |
|
| 2852 | if (dst)
|
| 2853 | dst = dst_check(dst, 0);
|
| 2854 | if (dst) {
|
| 2855 | u32 itag = 0;
|
| 2856 |
|
| 2857 | |
| 2858 | * any place which wants to hold dst has to call
|
| 2859 | * dst_hold_safe()
|
| 2860 |
|
| 2861 | skb_dst_set_noref(skb, dst);
|
| 2862 |
|
| 2863 | |
| 2864 | * the source on each packet
|
| 2865 |
|
| 2866 | if (!inet_sk(sk)->inet_daddr && in_dev)
|
| 2867 | return ip_mc_validate_source(skb, iph->daddr,
|
| 2868 | iph->saddr,
|
| 2869 | ip4h_dscp(iph),
|
| 2870 | skb->dev, in_dev, &itag);
|
| 2871 | }
|
| 2872 | return SKB_NOT_DROPPED_YET;
|
| 2873 | }
|
| 2874 |
|
| 2875 | static void udp_destroy_sock(struct sock *sk)
|
| 2876 | {
|
| 2877 | struct udp_sock *up = udp_sk(sk);
|
| 2878 | bool slow = lock_sock_fast(sk);
|
| 2879 |
|
| 2880 |
|
| 2881 | sock_set_flag(sk, SOCK_DEAD);
|
| 2882 | udp_flush_pending_frames(sk);
|
| 2883 | unlock_sock_fast(sk, slow);
|
| 2884 | if (static_branch_unlikely(&udp_encap_needed_key)) {
|
| 2885 | if (up->encap_type) {
|
| 2886 | void (*encap_destroy)(struct sock *sk);
|
| 2887 | encap_destroy = READ_ONCE(up->encap_destroy);
|
| 2888 | if (encap_destroy)
|
| 2889 | encap_destroy(sk);
|
| 2890 | }
|
| 2891 | if (udp_test_bit(ENCAP_ENABLED, sk)) {
|
| 2892 | static_branch_dec(&udp_encap_needed_key);
|
| 2893 | udp_tunnel_cleanup_gro(sk);
|
| 2894 | }
|
| 2895 | }
|
| 2896 | }
|
| 2897 |
|
| 2898 | typedef struct sk_buff *(*udp_gro_receive_t)(struct sock *sk,
|
| 2899 | struct list_head *head,
|
| 2900 | struct sk_buff *skb);
|
| 2901 |
|
| 2902 | static void set_xfrm_gro_udp_encap_rcv(__u16 encap_type, unsigned short family,
|
| 2903 | struct sock *sk)
|
| 2904 | {
|
| 2905 | #ifdef CONFIG_XFRM
|
| 2906 | udp_gro_receive_t new_gro_receive;
|
| 2907 |
|
| 2908 | if (udp_test_bit(GRO_ENABLED, sk) && encap_type == UDP_ENCAP_ESPINUDP) {
|
| 2909 | if (IS_ENABLED(CONFIG_IPV6) && family == AF_INET6)
|
| 2910 | new_gro_receive = xfrm6_gro_udp_encap_rcv;
|
| 2911 | else
|
| 2912 | new_gro_receive = xfrm4_gro_udp_encap_rcv;
|
| 2913 |
|
| 2914 | if (udp_sk(sk)->gro_receive != new_gro_receive) {
|
| 2915 | |
| 2916 | * With IPV6_ADDRFORM the gro callback could change
|
| 2917 | * after being set, unregister the old one, if valid.
|
| 2918 |
|
| 2919 | if (udp_sk(sk)->gro_receive)
|
| 2920 | udp_tunnel_update_gro_rcv(sk, false);
|
| 2921 |
|
| 2922 | WRITE_ONCE(udp_sk(sk)->gro_receive, new_gro_receive);
|
| 2923 | udp_tunnel_update_gro_rcv(sk, true);
|
| 2924 | }
|
| 2925 | }
|
| 2926 | #endif
|
| 2927 | }
|
| 2928 |
|
| 2929 | |
| 2930 | * Socket option code for UDP
|
| 2931 |
|
| 2932 | int udp_lib_setsockopt(struct sock *sk, int level, int optname,
|
| 2933 | sockptr_t optval, unsigned int optlen,
|
| 2934 | int (*push_pending_frames)(struct sock *))
|
| 2935 | {
|
| 2936 | struct udp_sock *up = udp_sk(sk);
|
| 2937 | int val, valbool;
|
| 2938 | int err = 0;
|
| 2939 |
|
| 2940 | if (level == SOL_SOCKET) {
|
| 2941 | err = sk_setsockopt(sk, level, optname, optval, optlen);
|
| 2942 |
|
| 2943 | if (optname == SO_RCVBUF || optname == SO_RCVBUFFORCE) {
|
| 2944 | sockopt_lock_sock(sk);
|
| 2945 |
|
| 2946 | WRITE_ONCE(up->forward_threshold, sk->sk_rcvbuf >> 2);
|
| 2947 | sockopt_release_sock(sk);
|
| 2948 | }
|
| 2949 | return err;
|
| 2950 | }
|
| 2951 |
|
| 2952 | if (optlen < sizeof(int))
|
| 2953 | return -EINVAL;
|
| 2954 |
|
| 2955 | if (copy_from_sockptr(&val, optval, sizeof(val)))
|
| 2956 | return -EFAULT;
|
| 2957 |
|
| 2958 | valbool = val ? 1 : 0;
|
| 2959 |
|
| 2960 | switch (optname) {
|
| 2961 | case UDP_CORK:
|
| 2962 | if (val != 0) {
|
| 2963 | udp_set_bit(CORK, sk);
|
| 2964 | } else {
|
| 2965 | udp_clear_bit(CORK, sk);
|
| 2966 | lock_sock(sk);
|
| 2967 | push_pending_frames(sk);
|
| 2968 | release_sock(sk);
|
| 2969 | }
|
| 2970 | break;
|
| 2971 |
|
| 2972 | case UDP_ENCAP:
|
| 2973 | sockopt_lock_sock(sk);
|
| 2974 | switch (val) {
|
| 2975 | case 0:
|
| 2976 | #ifdef CONFIG_XFRM
|
| 2977 | case UDP_ENCAP_ESPINUDP:
|
| 2978 | set_xfrm_gro_udp_encap_rcv(val, sk->sk_family, sk);
|
| 2979 | #if IS_ENABLED(CONFIG_IPV6)
|
| 2980 | if (sk->sk_family == AF_INET6)
|
| 2981 | WRITE_ONCE(up->encap_rcv,
|
| 2982 | xfrm6_udp_encap_rcv);
|
| 2983 | else
|
| 2984 | #endif
|
| 2985 | WRITE_ONCE(up->encap_rcv,
|
| 2986 | xfrm4_udp_encap_rcv);
|
| 2987 | #endif
|
| 2988 | fallthrough;
|
| 2989 | case UDP_ENCAP_L2TPINUDP:
|
| 2990 | WRITE_ONCE(up->encap_type, val);
|
| 2991 | udp_tunnel_encap_enable(sk);
|
| 2992 | break;
|
| 2993 | default:
|
| 2994 | err = -ENOPROTOOPT;
|
| 2995 | break;
|
| 2996 | }
|
| 2997 | sockopt_release_sock(sk);
|
| 2998 | break;
|
| 2999 |
|
| 3000 | case UDP_NO_CHECK6_TX:
|
| 3001 | udp_set_no_check6_tx(sk, valbool);
|
| 3002 | break;
|
| 3003 |
|
| 3004 | case UDP_NO_CHECK6_RX:
|
| 3005 | udp_set_no_check6_rx(sk, valbool);
|
| 3006 | break;
|
| 3007 |
|
| 3008 | case UDP_SEGMENT:
|
| 3009 | if (val < 0 || val > USHRT_MAX)
|
| 3010 | return -EINVAL;
|
| 3011 | WRITE_ONCE(up->gso_size, val);
|
| 3012 | break;
|
| 3013 |
|
| 3014 | case UDP_GRO:
|
| 3015 | sockopt_lock_sock(sk);
|
| 3016 |
|
| 3017 | if (valbool)
|
| 3018 | udp_tunnel_encap_enable(sk);
|
| 3019 | udp_assign_bit(GRO_ENABLED, sk, valbool);
|
| 3020 | udp_assign_bit(ACCEPT_L4, sk, valbool);
|
| 3021 | set_xfrm_gro_udp_encap_rcv(up->encap_type, sk->sk_family, sk);
|
| 3022 | sockopt_release_sock(sk);
|
| 3023 | break;
|
| 3024 |
|
| 3025 | default:
|
| 3026 | err = -ENOPROTOOPT;
|
| 3027 | break;
|
| 3028 | }
|
| 3029 |
|
| 3030 | return err;
|
| 3031 | }
|
| 3032 |
|
| 3033 | static int udp_setsockopt(struct sock *sk, int level, int optname, sockptr_t optval,
|
| 3034 | unsigned int optlen)
|
| 3035 | {
|
| 3036 | if (level == SOL_UDP || level == SOL_SOCKET)
|
| 3037 | return udp_lib_setsockopt(sk, level, optname,
|
| 3038 | optval, optlen,
|
| 3039 | udp_push_pending_frames);
|
| 3040 | return ip_setsockopt(sk, level, optname, optval, optlen);
|
| 3041 | }
|
| 3042 |
|
| 3043 | int udp_lib_getsockopt(struct sock *sk, int level, int optname,
|
| 3044 | sockopt_t *opt)
|
| 3045 | {
|
| 3046 | struct udp_sock *up = udp_sk(sk);
|
| 3047 | int val, len;
|
| 3048 |
|
| 3049 | len = opt->optlen;
|
| 3050 |
|
| 3051 | if (len < 0)
|
| 3052 | return -EINVAL;
|
| 3053 |
|
| 3054 | len = min_t(unsigned int, len, sizeof(int));
|
| 3055 |
|
| 3056 | switch (optname) {
|
| 3057 | case UDP_CORK:
|
| 3058 | val = udp_test_bit(CORK, sk);
|
| 3059 | break;
|
| 3060 |
|
| 3061 | case UDP_ENCAP:
|
| 3062 | val = READ_ONCE(up->encap_type);
|
| 3063 | break;
|
| 3064 |
|
| 3065 | case UDP_NO_CHECK6_TX:
|
| 3066 | val = udp_get_no_check6_tx(sk);
|
| 3067 | break;
|
| 3068 |
|
| 3069 | case UDP_NO_CHECK6_RX:
|
| 3070 | val = udp_get_no_check6_rx(sk);
|
| 3071 | break;
|
| 3072 |
|
| 3073 | case UDP_SEGMENT:
|
| 3074 | val = READ_ONCE(up->gso_size);
|
| 3075 | break;
|
| 3076 |
|
| 3077 | case UDP_GRO:
|
| 3078 | val = udp_test_bit(GRO_ENABLED, sk);
|
| 3079 | break;
|
| 3080 |
|
| 3081 | default:
|
| 3082 | return -ENOPROTOOPT;
|
| 3083 | }
|
| 3084 |
|
| 3085 | opt->optlen = len;
|
| 3086 | if (copy_to_iter(&val, len, &opt->iter_out) != len)
|
| 3087 | return -EFAULT;
|
| 3088 | return 0;
|
| 3089 | }
|
| 3090 |
|
| 3091 | static int udp_getsockopt(struct sock *sk, int level, int optname,
|
| 3092 | char __user *optval, int __user *optlen)
|
| 3093 | {
|
| 3094 | sockopt_t opt;
|
| 3095 | int err;
|
| 3096 |
|
| 3097 | |
| 3098 | * keep the old __user pointers, until ip_getsockopt() moves
|
| 3099 | * to sockopt_t
|
| 3100 |
|
| 3101 | if (level != SOL_UDP)
|
| 3102 | return ip_getsockopt(sk, level, optname, optval, optlen);
|
| 3103 |
|
| 3104 | err = sockopt_init_user(&opt, optval, optlen);
|
| 3105 | if (err)
|
| 3106 | return err;
|
| 3107 |
|
| 3108 | err = udp_lib_getsockopt(sk, level, optname, &opt);
|
| 3109 | if (err)
|
| 3110 | return err;
|
| 3111 |
|
| 3112 |
|
| 3113 | if (put_user(opt.optlen, optlen))
|
| 3114 | return -EFAULT;
|
| 3115 |
|
| 3116 | return 0;
|
| 3117 | }
|
| 3118 |
|
| 3119 | |
| 3120 | * udp_poll - wait for a UDP event.
|
| 3121 | * @file: - file struct
|
| 3122 | * @sock: - socket
|
| 3123 | * @wait: - poll table
|
| 3124 | *
|
| 3125 | * This is same as datagram poll, except for the special case of
|
| 3126 | * blocking sockets. If application is using a blocking fd
|
| 3127 | * and a packet with checksum error is in the queue;
|
| 3128 | * then it could get return from select indicating data available
|
| 3129 | * but then block when reading it. Add special case code
|
| 3130 | * to work around these arguably broken applications.
|
| 3131 |
|
| 3132 | __poll_t udp_poll(struct file *file, struct socket *sock, poll_table *wait)
|
| 3133 | {
|
| 3134 | __poll_t mask = datagram_poll(file, sock, wait);
|
| 3135 | struct sock *sk = sock->sk;
|
| 3136 |
|
| 3137 | if (!skb_queue_empty_lockless(&udp_sk(sk)->reader_queue))
|
| 3138 | mask |= EPOLLIN | EPOLLRDNORM;
|
| 3139 |
|
| 3140 |
|
| 3141 | if ((mask & EPOLLRDNORM) && !(file->f_flags & O_NONBLOCK) &&
|
| 3142 | !(sk->sk_shutdown & RCV_SHUTDOWN) && first_packet_length(sk) == -1)
|
| 3143 | mask &= ~(EPOLLIN | EPOLLRDNORM);
|
| 3144 |
|
| 3145 |
|
| 3146 | if (sk_is_readable(sk))
|
| 3147 | mask |= EPOLLIN | EPOLLRDNORM;
|
| 3148 | return mask;
|
| 3149 |
|
| 3150 | }
|
| 3151 |
|
| 3152 | int udp_abort(struct sock *sk, int err)
|
| 3153 | {
|
| 3154 | if (!has_current_bpf_ctx())
|
| 3155 | lock_sock(sk);
|
| 3156 |
|
| 3157 | |
| 3158 | * with close()
|
| 3159 |
|
| 3160 | if (sock_flag(sk, SOCK_DEAD))
|
| 3161 | goto out;
|
| 3162 |
|
| 3163 | sk->sk_err = err;
|
| 3164 | sk_error_report(sk);
|
| 3165 | udp_unhash4_on_disconnect(sk);
|
| 3166 | __udp_disconnect(sk, 0);
|
| 3167 |
|
| 3168 | out:
|
| 3169 | if (!has_current_bpf_ctx())
|
| 3170 | release_sock(sk);
|
| 3171 |
|
| 3172 | return 0;
|
| 3173 | }
|
| 3174 |
|
| 3175 | struct proto udp_prot = {
|
| 3176 | .name = "UDP",
|
| 3177 | .owner = THIS_MODULE,
|
| 3178 | .close = udp_lib_close,
|
| 3179 | .pre_connect = udp_pre_connect,
|
| 3180 | .connect = udp_connect,
|
| 3181 | .disconnect = udp_disconnect,
|
| 3182 | .ioctl = udp_ioctl,
|
| 3183 | .init = udp_init_sock,
|
| 3184 | .destroy = udp_destroy_sock,
|
| 3185 | .setsockopt = udp_setsockopt,
|
| 3186 | .getsockopt = udp_getsockopt,
|
| 3187 | .sendmsg = udp_sendmsg,
|
| 3188 | .recvmsg = udp_recvmsg,
|
| 3189 | .splice_eof = udp_splice_eof,
|
| 3190 | .release_cb = ip4_datagram_release_cb,
|
| 3191 | .hash = udp_lib_hash,
|
| 3192 | .unhash = udp_lib_unhash,
|
| 3193 | .rehash = udp_v4_rehash,
|
| 3194 | .get_port = udp_v4_get_port,
|
| 3195 | .put_port = udp_lib_unhash,
|
| 3196 | #ifdef CONFIG_BPF_SYSCALL
|
| 3197 | .psock_update_sk_prot = udp_bpf_update_proto,
|
| 3198 | #endif
|
| 3199 | .memory_allocated = &net_aligned_data.udp_memory_allocated,
|
| 3200 | .per_cpu_fw_alloc = &udp_memory_per_cpu_fw_alloc,
|
| 3201 |
|
| 3202 | .sysctl_mem = sysctl_udp_mem,
|
| 3203 | .sysctl_wmem_offset = offsetof(struct net, ipv4.sysctl_udp_wmem_min),
|
| 3204 | .sysctl_rmem_offset = offsetof(struct net, ipv4.sysctl_udp_rmem_min),
|
| 3205 | .obj_size = sizeof(struct udp_sock),
|
| 3206 | .diag_destroy = udp_abort,
|
| 3207 | };
|
| 3208 | EXPORT_SYMBOL(udp_prot);
|
| 3209 |
|
| 3210 |
|
| 3211 | #ifdef CONFIG_PROC_FS
|
| 3212 |
|
| 3213 | static unsigned short seq_file_family(const struct seq_file *seq);
|
| 3214 | static bool seq_sk_match(struct seq_file *seq, const struct sock *sk)
|
| 3215 | {
|
| 3216 | unsigned short family = seq_file_family(seq);
|
| 3217 |
|
| 3218 |
|
| 3219 | return ((family == AF_UNSPEC || family == sk->sk_family) &&
|
| 3220 | net_eq(sock_net(sk), seq_file_net(seq)));
|
| 3221 | }
|
| 3222 |
|
| 3223 | #ifdef CONFIG_BPF_SYSCALL
|
| 3224 | static const struct seq_operations bpf_iter_udp_seq_ops;
|
| 3225 | #endif
|
| 3226 |
|
| 3227 | static struct sock *udp_get_first(struct seq_file *seq, int start)
|
| 3228 | {
|
| 3229 | struct udp_iter_state *state = seq->private;
|
| 3230 | struct net *net = seq_file_net(seq);
|
| 3231 | struct udp_table *udptable;
|
| 3232 | struct sock *sk;
|
| 3233 |
|
| 3234 | udptable = net->ipv4.udp_table;
|
| 3235 |
|
| 3236 | for (state->bucket = start; state->bucket <= udptable->mask;
|
| 3237 | ++state->bucket) {
|
| 3238 | struct udp_hslot *hslot = &udptable->hash[state->bucket];
|
| 3239 |
|
| 3240 | if (hlist_empty(&hslot->head))
|
| 3241 | continue;
|
| 3242 |
|
| 3243 | spin_lock_bh(&hslot->lock);
|
| 3244 | sk_for_each(sk, &hslot->head) {
|
| 3245 | if (seq_sk_match(seq, sk))
|
| 3246 | goto found;
|
| 3247 | }
|
| 3248 | spin_unlock_bh(&hslot->lock);
|
| 3249 | }
|
| 3250 | sk = NULL;
|
| 3251 | found:
|
| 3252 | return sk;
|
| 3253 | }
|
| 3254 |
|
| 3255 | static struct sock *udp_get_next(struct seq_file *seq, struct sock *sk)
|
| 3256 | {
|
| 3257 | struct udp_iter_state *state = seq->private;
|
| 3258 | struct net *net = seq_file_net(seq);
|
| 3259 | struct udp_table *udptable;
|
| 3260 |
|
| 3261 | do {
|
| 3262 | sk = sk_next(sk);
|
| 3263 | } while (sk && !seq_sk_match(seq, sk));
|
| 3264 |
|
| 3265 | if (!sk) {
|
| 3266 | udptable = net->ipv4.udp_table;
|
| 3267 |
|
| 3268 | if (state->bucket <= udptable->mask)
|
| 3269 | spin_unlock_bh(&udptable->hash[state->bucket].lock);
|
| 3270 |
|
| 3271 | return udp_get_first(seq, state->bucket + 1);
|
| 3272 | }
|
| 3273 | return sk;
|
| 3274 | }
|
| 3275 |
|
| 3276 | static struct sock *udp_get_idx(struct seq_file *seq, loff_t pos)
|
| 3277 | {
|
| 3278 | struct sock *sk = udp_get_first(seq, 0);
|
| 3279 |
|
| 3280 | if (sk)
|
| 3281 | while (pos && (sk = udp_get_next(seq, sk)) != NULL)
|
| 3282 | --pos;
|
| 3283 | return pos ? NULL : sk;
|
| 3284 | }
|
| 3285 |
|
| 3286 | void *udp_seq_start(struct seq_file *seq, loff_t *pos)
|
| 3287 | {
|
| 3288 | struct udp_iter_state *state = seq->private;
|
| 3289 | state->bucket = MAX_UDP_PORTS;
|
| 3290 |
|
| 3291 | return *pos ? udp_get_idx(seq, *pos-1) : SEQ_START_TOKEN;
|
| 3292 | }
|
| 3293 |
|
| 3294 | void *udp_seq_next(struct seq_file *seq, void *v, loff_t *pos)
|
| 3295 | {
|
| 3296 | struct sock *sk;
|
| 3297 |
|
| 3298 | if (v == SEQ_START_TOKEN)
|
| 3299 | sk = udp_get_idx(seq, 0);
|
| 3300 | else
|
| 3301 | sk = udp_get_next(seq, v);
|
| 3302 |
|
| 3303 | ++*pos;
|
| 3304 | return sk;
|
| 3305 | }
|
| 3306 |
|
| 3307 | void udp_seq_stop(struct seq_file *seq, void *v)
|
| 3308 | {
|
| 3309 | struct udp_iter_state *state = seq->private;
|
| 3310 | struct udp_table *udptable;
|
| 3311 |
|
| 3312 | udptable = seq_file_net(seq)->ipv4.udp_table;
|
| 3313 |
|
| 3314 | if (state->bucket <= udptable->mask)
|
| 3315 | spin_unlock_bh(&udptable->hash[state->bucket].lock);
|
| 3316 | }
|
| 3317 |
|
| 3318 |
|
| 3319 | static void udp4_format_sock(struct sock *sp, struct seq_file *f,
|
| 3320 | int bucket)
|
| 3321 | {
|
| 3322 | struct inet_sock *inet = inet_sk(sp);
|
| 3323 | __be32 dest = inet->inet_daddr;
|
| 3324 | __be32 src = inet->inet_rcv_saddr;
|
| 3325 | __u16 destp = ntohs(inet->inet_dport);
|
| 3326 | __u16 srcp = ntohs(inet->inet_sport);
|
| 3327 |
|
| 3328 | seq_printf(f, "%5d: %08X:%04X %08X:%04X"
|
| 3329 | " %02X %08X:%08X %02X:%08lX %08X %5u %8d %llu %d %pK %u",
|
| 3330 | bucket, src, srcp, dest, destp, sp->sk_state,
|
| 3331 | sk_wmem_alloc_get(sp),
|
| 3332 | udp_rqueue_get(sp),
|
| 3333 | 0, 0L, 0,
|
| 3334 | from_kuid_munged(seq_user_ns(f), sk_uid(sp)),
|
| 3335 | 0, sock_i_ino(sp),
|
| 3336 | refcount_read(&sp->sk_refcnt), sp,
|
| 3337 | sk_drops_read(sp));
|
| 3338 | }
|
| 3339 |
|
| 3340 | static int udp4_seq_show(struct seq_file *seq, void *v)
|
| 3341 | {
|
| 3342 | seq_setwidth(seq, 127);
|
| 3343 | if (v == SEQ_START_TOKEN)
|
| 3344 | seq_puts(seq, " sl local_address rem_address st tx_queue "
|
| 3345 | "rx_queue tr tm->when retrnsmt uid timeout "
|
| 3346 | "inode ref pointer drops");
|
| 3347 | else {
|
| 3348 | struct udp_iter_state *state = seq->private;
|
| 3349 |
|
| 3350 | udp4_format_sock(v, seq, state->bucket);
|
| 3351 | }
|
| 3352 | seq_pad(seq, '\n');
|
| 3353 | return 0;
|
| 3354 | }
|
| 3355 |
|
| 3356 | #ifdef CONFIG_BPF_SYSCALL
|
| 3357 | struct bpf_iter__udp {
|
| 3358 | __bpf_md_ptr(struct bpf_iter_meta *, meta);
|
| 3359 | __bpf_md_ptr(struct udp_sock *, udp_sk);
|
| 3360 | uid_t uid __aligned(8);
|
| 3361 | int bucket __aligned(8);
|
| 3362 | };
|
| 3363 |
|
| 3364 | union bpf_udp_iter_batch_item {
|
| 3365 | struct sock *sk;
|
| 3366 | __u64 cookie;
|
| 3367 | };
|
| 3368 |
|
| 3369 | struct bpf_udp_iter_state {
|
| 3370 | struct udp_iter_state state;
|
| 3371 | unsigned int cur_sk;
|
| 3372 | unsigned int end_sk;
|
| 3373 | unsigned int max_sk;
|
| 3374 | union bpf_udp_iter_batch_item *batch;
|
| 3375 | };
|
| 3376 |
|
| 3377 | static int bpf_iter_udp_realloc_batch(struct bpf_udp_iter_state *iter,
|
| 3378 | unsigned int new_batch_sz, gfp_t flags);
|
| 3379 | static struct sock *bpf_iter_udp_resume(struct sock *first_sk,
|
| 3380 | union bpf_udp_iter_batch_item *cookies,
|
| 3381 | int n_cookies)
|
| 3382 | {
|
| 3383 | struct sock *sk = NULL;
|
| 3384 | int i;
|
| 3385 |
|
| 3386 | for (i = 0; i < n_cookies; i++) {
|
| 3387 | sk = first_sk;
|
| 3388 | udp_portaddr_for_each_entry_from(sk)
|
| 3389 | if (cookies[i].cookie == atomic64_read(&sk->sk_cookie))
|
| 3390 | goto done;
|
| 3391 | }
|
| 3392 | done:
|
| 3393 | return sk;
|
| 3394 | }
|
| 3395 |
|
| 3396 | static struct sock *bpf_iter_udp_batch(struct seq_file *seq)
|
| 3397 | {
|
| 3398 | struct bpf_udp_iter_state *iter = seq->private;
|
| 3399 | struct udp_iter_state *state = &iter->state;
|
| 3400 | unsigned int find_cookie, end_cookie;
|
| 3401 | struct net *net = seq_file_net(seq);
|
| 3402 | struct udp_table *udptable;
|
| 3403 | unsigned int batch_sks = 0;
|
| 3404 | int resume_bucket;
|
| 3405 | int resizes = 0;
|
| 3406 | struct sock *sk;
|
| 3407 | int err = 0;
|
| 3408 |
|
| 3409 | resume_bucket = state->bucket;
|
| 3410 |
|
| 3411 |
|
| 3412 | if (iter->cur_sk == iter->end_sk)
|
| 3413 | state->bucket++;
|
| 3414 |
|
| 3415 | udptable = net->ipv4.udp_table;
|
| 3416 |
|
| 3417 | again:
|
| 3418 | |
| 3419 | * Iterate over the hash table to find a bucket with sockets matching
|
| 3420 | * the iterator attributes, and return the first matching socket from
|
| 3421 | * the bucket. The remaining matched sockets from the bucket are batched
|
| 3422 | * before releasing the bucket lock. This allows BPF programs that are
|
| 3423 | * called in seq_show to acquire the bucket lock if needed.
|
| 3424 |
|
| 3425 | find_cookie = iter->cur_sk;
|
| 3426 | end_cookie = iter->end_sk;
|
| 3427 | iter->cur_sk = 0;
|
| 3428 | iter->end_sk = 0;
|
| 3429 | batch_sks = 0;
|
| 3430 |
|
| 3431 | for (; state->bucket <= udptable->mask; state->bucket++) {
|
| 3432 | struct udp_hslot *hslot2 = &udptable->hash2[state->bucket].hslot;
|
| 3433 |
|
| 3434 | if (hlist_empty(&hslot2->head))
|
| 3435 | goto next_bucket;
|
| 3436 |
|
| 3437 | spin_lock_bh(&hslot2->lock);
|
| 3438 | sk = hlist_entry_safe(hslot2->head.first, struct sock,
|
| 3439 | __sk_common.skc_portaddr_node);
|
| 3440 | |
| 3441 | * the last batch that still exists in resume_bucket. Most of
|
| 3442 | * the time this will just be where the last iteration left off
|
| 3443 | * in resume_bucket unless that socket disappeared between
|
| 3444 | * reads.
|
| 3445 |
|
| 3446 | if (state->bucket == resume_bucket)
|
| 3447 | sk = bpf_iter_udp_resume(sk, &iter->batch[find_cookie],
|
| 3448 | end_cookie - find_cookie);
|
| 3449 | fill_batch:
|
| 3450 | udp_portaddr_for_each_entry_from(sk) {
|
| 3451 | if (seq_sk_match(seq, sk)) {
|
| 3452 | if (iter->end_sk < iter->max_sk) {
|
| 3453 | sock_hold(sk);
|
| 3454 | iter->batch[iter->end_sk++].sk = sk;
|
| 3455 | }
|
| 3456 | batch_sks++;
|
| 3457 | }
|
| 3458 | }
|
| 3459 |
|
| 3460 |
|
| 3461 | if (unlikely(resizes <= 1 && iter->end_sk &&
|
| 3462 | iter->end_sk != batch_sks)) {
|
| 3463 | resizes++;
|
| 3464 |
|
| 3465 | |
| 3466 | * grabbing more memory.
|
| 3467 |
|
| 3468 | if (resizes == 1) {
|
| 3469 | spin_unlock_bh(&hslot2->lock);
|
| 3470 | err = bpf_iter_udp_realloc_batch(iter,
|
| 3471 | batch_sks * 3 / 2,
|
| 3472 | GFP_USER);
|
| 3473 | if (err)
|
| 3474 | return ERR_PTR(err);
|
| 3475 |
|
| 3476 | goto again;
|
| 3477 | }
|
| 3478 |
|
| 3479 | |
| 3480 | * change while we get the rest of the sockets.
|
| 3481 |
|
| 3482 | err = bpf_iter_udp_realloc_batch(iter, batch_sks,
|
| 3483 | GFP_NOWAIT);
|
| 3484 | if (err) {
|
| 3485 | spin_unlock_bh(&hslot2->lock);
|
| 3486 | return ERR_PTR(err);
|
| 3487 | }
|
| 3488 |
|
| 3489 |
|
| 3490 | sk = iter->batch[iter->end_sk - 1].sk;
|
| 3491 | sk = hlist_entry_safe(sk->__sk_common.skc_portaddr_node.next,
|
| 3492 | struct sock,
|
| 3493 | __sk_common.skc_portaddr_node);
|
| 3494 | batch_sks = iter->end_sk;
|
| 3495 | goto fill_batch;
|
| 3496 | }
|
| 3497 |
|
| 3498 | spin_unlock_bh(&hslot2->lock);
|
| 3499 |
|
| 3500 | if (iter->end_sk)
|
| 3501 | break;
|
| 3502 | next_bucket:
|
| 3503 | resizes = 0;
|
| 3504 | }
|
| 3505 |
|
| 3506 | WARN_ON_ONCE(iter->end_sk != batch_sks);
|
| 3507 | return iter->end_sk ? iter->batch[0].sk : NULL;
|
| 3508 | }
|
| 3509 |
|
| 3510 | static void *bpf_iter_udp_seq_next(struct seq_file *seq, void *v, loff_t *pos)
|
| 3511 | {
|
| 3512 | struct bpf_udp_iter_state *iter = seq->private;
|
| 3513 | struct sock *sk;
|
| 3514 |
|
| 3515 | |
| 3516 | * done with seq_show(), so unref the iter->cur_sk.
|
| 3517 |
|
| 3518 | if (iter->cur_sk < iter->end_sk)
|
| 3519 | sock_put(iter->batch[iter->cur_sk++].sk);
|
| 3520 |
|
| 3521 | |
| 3522 | * available in the current bucket batch.
|
| 3523 |
|
| 3524 | if (iter->cur_sk < iter->end_sk)
|
| 3525 | sk = iter->batch[iter->cur_sk].sk;
|
| 3526 | else
|
| 3527 |
|
| 3528 | sk = bpf_iter_udp_batch(seq);
|
| 3529 |
|
| 3530 | ++*pos;
|
| 3531 | return sk;
|
| 3532 | }
|
| 3533 |
|
| 3534 | static void *bpf_iter_udp_seq_start(struct seq_file *seq, loff_t *pos)
|
| 3535 | {
|
| 3536 | |
| 3537 | * continue from where it was stop()-ped.
|
| 3538 |
|
| 3539 | if (*pos)
|
| 3540 | return bpf_iter_udp_batch(seq);
|
| 3541 |
|
| 3542 | return SEQ_START_TOKEN;
|
| 3543 | }
|
| 3544 |
|
| 3545 | static int udp_prog_seq_show(struct bpf_prog *prog, struct bpf_iter_meta *meta,
|
| 3546 | struct udp_sock *udp_sk, uid_t uid, int bucket)
|
| 3547 | {
|
| 3548 | struct bpf_iter__udp ctx;
|
| 3549 |
|
| 3550 | meta->seq_num--;
|
| 3551 | ctx.meta = meta;
|
| 3552 | ctx.udp_sk = udp_sk;
|
| 3553 | ctx.uid = uid;
|
| 3554 | ctx.bucket = bucket;
|
| 3555 | return bpf_iter_run_prog(prog, &ctx);
|
| 3556 | }
|
| 3557 |
|
| 3558 | static int bpf_iter_udp_seq_show(struct seq_file *seq, void *v)
|
| 3559 | {
|
| 3560 | struct udp_iter_state *state = seq->private;
|
| 3561 | struct bpf_iter_meta meta;
|
| 3562 | struct bpf_prog *prog;
|
| 3563 | struct sock *sk = v;
|
| 3564 | uid_t uid;
|
| 3565 | int ret;
|
| 3566 |
|
| 3567 | if (v == SEQ_START_TOKEN)
|
| 3568 | return 0;
|
| 3569 |
|
| 3570 | lock_sock(sk);
|
| 3571 |
|
| 3572 | if (unlikely(sk_unhashed(sk))) {
|
| 3573 | ret = SEQ_SKIP;
|
| 3574 | goto unlock;
|
| 3575 | }
|
| 3576 |
|
| 3577 | uid = from_kuid_munged(seq_user_ns(seq), sk_uid(sk));
|
| 3578 | meta.seq = seq;
|
| 3579 | prog = bpf_iter_get_info(&meta, false);
|
| 3580 | ret = udp_prog_seq_show(prog, &meta, v, uid, state->bucket);
|
| 3581 |
|
| 3582 | unlock:
|
| 3583 | release_sock(sk);
|
| 3584 | return ret;
|
| 3585 | }
|
| 3586 |
|
| 3587 | static void bpf_iter_udp_put_batch(struct bpf_udp_iter_state *iter)
|
| 3588 | {
|
| 3589 | union bpf_udp_iter_batch_item *item;
|
| 3590 | unsigned int cur_sk = iter->cur_sk;
|
| 3591 | __u64 cookie;
|
| 3592 |
|
| 3593 | |
| 3594 | * pick up where we left off next time around.
|
| 3595 |
|
| 3596 | while (cur_sk < iter->end_sk) {
|
| 3597 | item = &iter->batch[cur_sk++];
|
| 3598 | cookie = sock_gen_cookie(item->sk);
|
| 3599 | sock_put(item->sk);
|
| 3600 | item->cookie = cookie;
|
| 3601 | }
|
| 3602 | }
|
| 3603 |
|
| 3604 | static void bpf_iter_udp_seq_stop(struct seq_file *seq, void *v)
|
| 3605 | {
|
| 3606 | struct bpf_udp_iter_state *iter = seq->private;
|
| 3607 | struct bpf_iter_meta meta;
|
| 3608 | struct bpf_prog *prog;
|
| 3609 |
|
| 3610 | if (!v) {
|
| 3611 | meta.seq = seq;
|
| 3612 | prog = bpf_iter_get_info(&meta, true);
|
| 3613 | if (prog)
|
| 3614 | (void)udp_prog_seq_show(prog, &meta, v, 0, 0);
|
| 3615 | }
|
| 3616 |
|
| 3617 | if (iter->cur_sk < iter->end_sk)
|
| 3618 | bpf_iter_udp_put_batch(iter);
|
| 3619 | }
|
| 3620 |
|
| 3621 | static const struct seq_operations bpf_iter_udp_seq_ops = {
|
| 3622 | .start = bpf_iter_udp_seq_start,
|
| 3623 | .next = bpf_iter_udp_seq_next,
|
| 3624 | .stop = bpf_iter_udp_seq_stop,
|
| 3625 | .show = bpf_iter_udp_seq_show,
|
| 3626 | };
|
| 3627 | #endif
|
| 3628 |
|
| 3629 | static unsigned short seq_file_family(const struct seq_file *seq)
|
| 3630 | {
|
| 3631 | const struct udp_seq_afinfo *afinfo;
|
| 3632 |
|
| 3633 | #ifdef CONFIG_BPF_SYSCALL
|
| 3634 |
|
| 3635 | if (seq->op == &bpf_iter_udp_seq_ops)
|
| 3636 | return AF_UNSPEC;
|
| 3637 | #endif
|
| 3638 |
|
| 3639 |
|
| 3640 | afinfo = pde_data(file_inode(seq->file));
|
| 3641 | return afinfo->family;
|
| 3642 | }
|
| 3643 |
|
| 3644 | static const struct seq_operations udp_seq_ops = {
|
| 3645 | .start = udp_seq_start,
|
| 3646 | .next = udp_seq_next,
|
| 3647 | .stop = udp_seq_stop,
|
| 3648 | .show = udp4_seq_show,
|
| 3649 | };
|
| 3650 |
|
| 3651 | static struct udp_seq_afinfo udp4_seq_afinfo = {
|
| 3652 | .family = AF_INET,
|
| 3653 | };
|
| 3654 |
|
| 3655 | static int __net_init udp4_proc_init_net(struct net *net)
|
| 3656 | {
|
| 3657 | if (!proc_create_net_data("udp", 0444, net->proc_net, &udp_seq_ops,
|
| 3658 | sizeof(struct udp_iter_state), &udp4_seq_afinfo))
|
| 3659 | return -ENOMEM;
|
| 3660 | return 0;
|
| 3661 | }
|
| 3662 |
|
| 3663 | static void __net_exit udp4_proc_exit_net(struct net *net)
|
| 3664 | {
|
| 3665 | remove_proc_entry("udp", net->proc_net);
|
| 3666 | }
|
| 3667 |
|
| 3668 | static struct pernet_operations udp4_net_ops = {
|
| 3669 | .init = udp4_proc_init_net,
|
| 3670 | .exit = udp4_proc_exit_net,
|
| 3671 | };
|
| 3672 |
|
| 3673 | int __init udp4_proc_init(void)
|
| 3674 | {
|
| 3675 | return register_pernet_subsys(&udp4_net_ops);
|
| 3676 | }
|
| 3677 |
|
| 3678 | void udp4_proc_exit(void)
|
| 3679 | {
|
| 3680 | unregister_pernet_subsys(&udp4_net_ops);
|
| 3681 | }
|
| 3682 | #endif
|
| 3683 |
|
| 3684 | static __initdata unsigned long uhash_entries;
|
| 3685 | static int __init set_uhash_entries(char *str)
|
| 3686 | {
|
| 3687 | ssize_t ret;
|
| 3688 |
|
| 3689 | if (!str)
|
| 3690 | return 0;
|
| 3691 |
|
| 3692 | ret = kstrtoul(str, 0, &uhash_entries);
|
| 3693 | if (ret)
|
| 3694 | return 0;
|
| 3695 |
|
| 3696 | if (uhash_entries && uhash_entries < UDP_HTABLE_SIZE_MIN)
|
| 3697 | uhash_entries = UDP_HTABLE_SIZE_MIN;
|
| 3698 | return 1;
|
| 3699 | }
|
| 3700 | __setup("uhash_entries=", set_uhash_entries);
|
| 3701 |
|
| 3702 | static void __init udp_table_init(struct udp_table *table, const char *name)
|
| 3703 | {
|
| 3704 | unsigned int i, slot_size;
|
| 3705 |
|
| 3706 | slot_size = sizeof(struct udp_hslot) + sizeof(struct udp_hslot_main) +
|
| 3707 | udp_hash4_slot_size();
|
| 3708 | table->hash = alloc_large_system_hash(name,
|
| 3709 | slot_size,
|
| 3710 | uhash_entries,
|
| 3711 | 21,
|
| 3712 | 0,
|
| 3713 | &table->log,
|
| 3714 | &table->mask,
|
| 3715 | UDP_HTABLE_SIZE_MIN,
|
| 3716 | UDP_HTABLE_SIZE_MAX);
|
| 3717 |
|
| 3718 | table->hash2 = (void *)(table->hash + (table->mask + 1));
|
| 3719 | for (i = 0; i <= table->mask; i++) {
|
| 3720 | INIT_HLIST_HEAD(&table->hash[i].head);
|
| 3721 | table->hash[i].count = 0;
|
| 3722 | spin_lock_init(&table->hash[i].lock);
|
| 3723 | }
|
| 3724 | for (i = 0; i <= table->mask; i++) {
|
| 3725 | INIT_HLIST_HEAD(&table->hash2[i].hslot.head);
|
| 3726 | table->hash2[i].hslot.count = 0;
|
| 3727 | spin_lock_init(&table->hash2[i].hslot.lock);
|
| 3728 | }
|
| 3729 | udp_table_hash4_init(table);
|
| 3730 | }
|
| 3731 |
|
| 3732 | u32 udp_flow_hashrnd(void)
|
| 3733 | {
|
| 3734 | static u32 hashrnd __read_mostly;
|
| 3735 |
|
| 3736 | net_get_random_once(&hashrnd, sizeof(hashrnd));
|
| 3737 |
|
| 3738 | return hashrnd;
|
| 3739 | }
|
| 3740 | EXPORT_SYMBOL(udp_flow_hashrnd);
|
| 3741 |
|
| 3742 | static void __net_init udp_sysctl_init(struct net *net)
|
| 3743 | {
|
| 3744 | net->ipv4.sysctl_udp_rmem_min = PAGE_SIZE;
|
| 3745 | net->ipv4.sysctl_udp_wmem_min = PAGE_SIZE;
|
| 3746 |
|
| 3747 | #ifdef CONFIG_NET_L3_MASTER_DEV
|
| 3748 | net->ipv4.sysctl_udp_l3mdev_accept = 0;
|
| 3749 | #endif
|
| 3750 | }
|
| 3751 |
|
| 3752 | static struct udp_table __net_init *udp_pernet_table_alloc(unsigned int hash_entries)
|
| 3753 | {
|
| 3754 | struct udp_table *udptable;
|
| 3755 | unsigned int slot_size;
|
| 3756 | int i;
|
| 3757 |
|
| 3758 | udptable = kmalloc_obj(*udptable);
|
| 3759 | if (!udptable)
|
| 3760 | goto out;
|
| 3761 |
|
| 3762 | slot_size = sizeof(struct udp_hslot) + sizeof(struct udp_hslot_main) +
|
| 3763 | udp_hash4_slot_size();
|
| 3764 | udptable->hash = vmalloc_huge(hash_entries * slot_size,
|
| 3765 | GFP_KERNEL_ACCOUNT);
|
| 3766 | if (!udptable->hash)
|
| 3767 | goto free_table;
|
| 3768 |
|
| 3769 | udptable->hash2 = (void *)(udptable->hash + hash_entries);
|
| 3770 | udptable->mask = hash_entries - 1;
|
| 3771 | udptable->log = ilog2(hash_entries);
|
| 3772 |
|
| 3773 | for (i = 0; i < hash_entries; i++) {
|
| 3774 | INIT_HLIST_HEAD(&udptable->hash[i].head);
|
| 3775 | udptable->hash[i].count = 0;
|
| 3776 | spin_lock_init(&udptable->hash[i].lock);
|
| 3777 |
|
| 3778 | INIT_HLIST_HEAD(&udptable->hash2[i].hslot.head);
|
| 3779 | udptable->hash2[i].hslot.count = 0;
|
| 3780 | spin_lock_init(&udptable->hash2[i].hslot.lock);
|
| 3781 | }
|
| 3782 | udp_table_hash4_init(udptable);
|
| 3783 |
|
| 3784 | return udptable;
|
| 3785 |
|
| 3786 | free_table:
|
| 3787 | kfree(udptable);
|
| 3788 | out:
|
| 3789 | return NULL;
|
| 3790 | }
|
| 3791 |
|
| 3792 | static void __net_exit udp_pernet_table_free(struct net *net)
|
| 3793 | {
|
| 3794 | struct udp_table *udptable = net->ipv4.udp_table;
|
| 3795 |
|
| 3796 | if (udptable == &udp_table)
|
| 3797 | return;
|
| 3798 |
|
| 3799 | kvfree(udptable->hash);
|
| 3800 | kfree(udptable);
|
| 3801 | }
|
| 3802 |
|
| 3803 | static void __net_init udp_set_table(struct net *net)
|
| 3804 | {
|
| 3805 | struct udp_table *udptable;
|
| 3806 | unsigned int hash_entries;
|
| 3807 | struct net *old_net;
|
| 3808 |
|
| 3809 | if (net_eq(net, &init_net))
|
| 3810 | goto fallback;
|
| 3811 |
|
| 3812 | old_net = current->nsproxy->net_ns;
|
| 3813 | hash_entries = READ_ONCE(old_net->ipv4.sysctl_udp_child_hash_entries);
|
| 3814 | if (!hash_entries)
|
| 3815 | goto fallback;
|
| 3816 |
|
| 3817 |
|
| 3818 | if (hash_entries < UDP_HTABLE_SIZE_MIN_PERNET)
|
| 3819 | hash_entries = UDP_HTABLE_SIZE_MIN_PERNET;
|
| 3820 | else
|
| 3821 | hash_entries = roundup_pow_of_two(hash_entries);
|
| 3822 |
|
| 3823 | udptable = udp_pernet_table_alloc(hash_entries);
|
| 3824 | if (udptable) {
|
| 3825 | net->ipv4.udp_table = udptable;
|
| 3826 | } else {
|
| 3827 | pr_warn("Failed to allocate UDP hash table (entries: %u) "
|
| 3828 | "for a netns, fallback to the global one\n",
|
| 3829 | hash_entries);
|
| 3830 | fallback:
|
| 3831 | net->ipv4.udp_table = &udp_table;
|
| 3832 | }
|
| 3833 | }
|
| 3834 |
|
| 3835 | static int __net_init udp_pernet_init(struct net *net)
|
| 3836 | {
|
| 3837 | #if IS_ENABLED(CONFIG_NET_UDP_TUNNEL)
|
| 3838 | int i;
|
| 3839 |
|
| 3840 |
|
| 3841 | for (i = 0; i < ARRAY_SIZE(net->ipv4.udp_tunnel_gro); ++i) {
|
| 3842 | INIT_HLIST_HEAD(&net->ipv4.udp_tunnel_gro[i].list);
|
| 3843 | RCU_INIT_POINTER(net->ipv4.udp_tunnel_gro[i].sk, NULL);
|
| 3844 | }
|
| 3845 | #endif
|
| 3846 | udp_sysctl_init(net);
|
| 3847 | udp_set_table(net);
|
| 3848 |
|
| 3849 | return 0;
|
| 3850 | }
|
| 3851 |
|
| 3852 | static void __net_exit udp_pernet_exit(struct net *net)
|
| 3853 | {
|
| 3854 | udp_pernet_table_free(net);
|
| 3855 | }
|
| 3856 |
|
| 3857 | static struct pernet_operations __net_initdata udp_sysctl_ops = {
|
| 3858 | .init = udp_pernet_init,
|
| 3859 | .exit = udp_pernet_exit,
|
| 3860 | };
|
| 3861 |
|
| 3862 | #if defined(CONFIG_BPF_SYSCALL) && defined(CONFIG_PROC_FS)
|
| 3863 | DEFINE_BPF_ITER_FUNC(udp, struct bpf_iter_meta *meta,
|
| 3864 | struct udp_sock *udp_sk, uid_t uid, int bucket)
|
| 3865 |
|
| 3866 | static int bpf_iter_udp_realloc_batch(struct bpf_udp_iter_state *iter,
|
| 3867 | unsigned int new_batch_sz, gfp_t flags)
|
| 3868 | {
|
| 3869 | union bpf_udp_iter_batch_item *new_batch;
|
| 3870 |
|
| 3871 | new_batch = kvmalloc_objs(*new_batch, new_batch_sz,
|
| 3872 | flags | __GFP_NOWARN);
|
| 3873 | if (!new_batch)
|
| 3874 | return -ENOMEM;
|
| 3875 |
|
| 3876 | if (flags != GFP_NOWAIT)
|
| 3877 | bpf_iter_udp_put_batch(iter);
|
| 3878 |
|
| 3879 | memcpy(new_batch, iter->batch, sizeof(*iter->batch) * iter->end_sk);
|
| 3880 | kvfree(iter->batch);
|
| 3881 | iter->batch = new_batch;
|
| 3882 | iter->max_sk = new_batch_sz;
|
| 3883 |
|
| 3884 | return 0;
|
| 3885 | }
|
| 3886 |
|
| 3887 | #define INIT_BATCH_SZ 16
|
| 3888 |
|
| 3889 | static int bpf_iter_init_udp(void *priv_data, struct bpf_iter_aux_info *aux)
|
| 3890 | {
|
| 3891 | struct bpf_udp_iter_state *iter = priv_data;
|
| 3892 | int ret;
|
| 3893 |
|
| 3894 | ret = bpf_iter_init_seq_net(priv_data, aux);
|
| 3895 | if (ret)
|
| 3896 | return ret;
|
| 3897 |
|
| 3898 | ret = bpf_iter_udp_realloc_batch(iter, INIT_BATCH_SZ, GFP_USER);
|
| 3899 | if (ret)
|
| 3900 | bpf_iter_fini_seq_net(priv_data);
|
| 3901 |
|
| 3902 | iter->state.bucket = -1;
|
| 3903 |
|
| 3904 | return ret;
|
| 3905 | }
|
| 3906 |
|
| 3907 | static void bpf_iter_fini_udp(void *priv_data)
|
| 3908 | {
|
| 3909 | struct bpf_udp_iter_state *iter = priv_data;
|
| 3910 |
|
| 3911 | bpf_iter_fini_seq_net(priv_data);
|
| 3912 | kvfree(iter->batch);
|
| 3913 | }
|
| 3914 |
|
| 3915 | static const struct bpf_iter_seq_info udp_seq_info = {
|
| 3916 | .seq_ops = &bpf_iter_udp_seq_ops,
|
| 3917 | .init_seq_private = bpf_iter_init_udp,
|
| 3918 | .fini_seq_private = bpf_iter_fini_udp,
|
| 3919 | .seq_priv_size = sizeof(struct bpf_udp_iter_state),
|
| 3920 | };
|
| 3921 |
|
| 3922 | static struct bpf_iter_reg udp_reg_info = {
|
| 3923 | .target = "udp",
|
| 3924 | .ctx_arg_info_size = 1,
|
| 3925 | .ctx_arg_info = {
|
| 3926 | { offsetof(struct bpf_iter__udp, udp_sk),
|
| 3927 | PTR_TO_BTF_ID_OR_NULL | PTR_TRUSTED },
|
| 3928 | },
|
| 3929 | .seq_info = &udp_seq_info,
|
| 3930 | };
|
| 3931 |
|
| 3932 | static void __init bpf_iter_register(void)
|
| 3933 | {
|
| 3934 | udp_reg_info.ctx_arg_info[0].btf_id = btf_sock_ids[BTF_SOCK_TYPE_UDP];
|
| 3935 | if (bpf_iter_reg_target(&udp_reg_info))
|
| 3936 | pr_warn("Warning: could not register bpf iterator udp\n");
|
| 3937 | }
|
| 3938 | #endif
|
| 3939 |
|
| 3940 | void __init udp_init(void)
|
| 3941 | {
|
| 3942 | unsigned long limit;
|
| 3943 |
|
| 3944 | udp_table_init(&udp_table, "UDP");
|
| 3945 | limit = nr_free_buffer_pages() / 8;
|
| 3946 | limit = max(limit, 128UL);
|
| 3947 | sysctl_udp_mem[0] = limit / 4 * 3;
|
| 3948 | sysctl_udp_mem[1] = limit;
|
| 3949 | sysctl_udp_mem[2] = sysctl_udp_mem[0] * 2;
|
| 3950 |
|
| 3951 | if (register_pernet_subsys(&udp_sysctl_ops))
|
| 3952 | panic("UDP: failed to init sysctl parameters.\n");
|
| 3953 |
|
| 3954 | #if defined(CONFIG_BPF_SYSCALL) && defined(CONFIG_PROC_FS)
|
| 3955 | bpf_iter_register();
|
| 3956 | #endif
|
| 3957 | }
|