| 1 | |
| 2 | |
| 3 | * INET An implementation of the TCP/IP protocol suite for the LINUX
|
| 4 | * operating system. INET is implemented using the BSD Socket
|
| 5 | * interface as the means of communication with the user level.
|
| 6 | *
|
| 7 | * PACKET - implements raw packet sockets.
|
| 8 | *
|
| 9 | * Authors: Ross Biro
|
| 10 | * Fred N. van Kempen, <[email protected]>
|
| 11 | * Alan Cox, <[email protected]>
|
| 12 | *
|
| 13 | * Fixes:
|
| 14 | * Alan Cox : verify_area() now used correctly
|
| 15 | * Alan Cox : new skbuff lists, look ma no backlogs!
|
| 16 | * Alan Cox : tidied skbuff lists.
|
| 17 | * Alan Cox : Now uses generic datagram routines I
|
| 18 | * added. Also fixed the peek/read crash
|
| 19 | * from all old Linux datagram code.
|
| 20 | * Alan Cox : Uses the improved datagram code.
|
| 21 | * Alan Cox : Added NULL's for socket options.
|
| 22 | * Alan Cox : Re-commented the code.
|
| 23 | * Alan Cox : Use new kernel side addressing
|
| 24 | * Rob Janssen : Correct MTU usage.
|
| 25 | * Dave Platt : Counter leaks caused by incorrect
|
| 26 | * interrupt locking and some slightly
|
| 27 | * dubious gcc output. Can you read
|
| 28 | * compiler: it said _VOLATILE_
|
| 29 | * Richard Kooijman : Timestamp fixes.
|
| 30 | * Alan Cox : New buffers. Use sk->mac.raw.
|
| 31 | * Alan Cox : sendmsg/recvmsg support.
|
| 32 | * Alan Cox : Protocol setting support
|
| 33 | * Alexey Kuznetsov : Untied from IPv4 stack.
|
| 34 | * Cyrus Durgin : Fixed kerneld for kmod.
|
| 35 | * Michal Ostrowski : Module initialization cleanup.
|
| 36 | * Ulises Alonso : Frame number limit removal and
|
| 37 | * packet_set_ring memory leak.
|
| 38 | * Eric Biederman : Allow for > 8 byte hardware addresses.
|
| 39 | * The convention is that longer addresses
|
| 40 | * will simply extend the hardware address
|
| 41 | * byte arrays at the end of sockaddr_ll
|
| 42 | * and packet_mreq.
|
| 43 | * Johann Baudy : Added TX RING.
|
| 44 | * Chetan Loke : Implemented TPACKET_V3 block abstraction
|
| 45 | * layer.
|
| 46 | * Copyright (C) 2011, <[email protected]>
|
| 47 |
|
| 48 |
|
| 49 | #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
|
| 50 |
|
| 51 | #include <linux/ethtool.h>
|
| 52 | #include <linux/uio.h>
|
| 53 | #include <linux/filter.h>
|
| 54 | #include <linux/types.h>
|
| 55 | #include <linux/mm.h>
|
| 56 | #include <linux/capability.h>
|
| 57 | #include <linux/fcntl.h>
|
| 58 | #include <linux/socket.h>
|
| 59 | #include <linux/in.h>
|
| 60 | #include <linux/inet.h>
|
| 61 | #include <linux/netdevice.h>
|
| 62 | #include <linux/if_packet.h>
|
| 63 | #include <linux/wireless.h>
|
| 64 | #include <linux/kernel.h>
|
| 65 | #include <linux/kmod.h>
|
| 66 | #include <linux/slab.h>
|
| 67 | #include <linux/vmalloc.h>
|
| 68 | #include <net/net_namespace.h>
|
| 69 | #include <net/ip.h>
|
| 70 | #include <net/protocol.h>
|
| 71 | #include <linux/skbuff.h>
|
| 72 | #include <net/sock.h>
|
| 73 | #include <linux/errno.h>
|
| 74 | #include <linux/timer.h>
|
| 75 | #include <linux/uaccess.h>
|
| 76 | #include <asm/ioctls.h>
|
| 77 | #include <asm/page.h>
|
| 78 | #include <asm/cacheflush.h>
|
| 79 | #include <asm/io.h>
|
| 80 | #include <linux/proc_fs.h>
|
| 81 | #include <linux/seq_file.h>
|
| 82 | #include <linux/poll.h>
|
| 83 | #include <linux/module.h>
|
| 84 | #include <linux/init.h>
|
| 85 | #include <linux/mutex.h>
|
| 86 | #include <linux/if_vlan.h>
|
| 87 | #include <linux/virtio_net.h>
|
| 88 | #include <linux/errqueue.h>
|
| 89 | #include <linux/net_tstamp.h>
|
| 90 | #include <linux/percpu.h>
|
| 91 | #include <linux/workqueue.h>
|
| 92 | #ifdef CONFIG_INET
|
| 93 | #include <net/inet_common.h>
|
| 94 | #endif
|
| 95 | #include <linux/bpf.h>
|
| 96 | #include <net/compat.h>
|
| 97 | #include <linux/netfilter_netdev.h>
|
| 98 |
|
| 99 | #include "internal.h"
|
| 100 |
|
| 101 | |
| 102 | Assumptions:
|
| 103 | - If the device has no dev->header_ops->create, there is no LL header
|
| 104 | visible above the device. In this case, its hard_header_len should be 0.
|
| 105 | The device may prepend its own header internally. In this case, its
|
| 106 | needed_headroom should be set to the space needed for it to add its
|
| 107 | internal header.
|
| 108 | For example, a WiFi driver pretending to be an Ethernet driver should
|
| 109 | set its hard_header_len to be the Ethernet header length, and set its
|
| 110 | needed_headroom to be (the real WiFi header length - the fake Ethernet
|
| 111 | header length).
|
| 112 | - packet socket receives packets with pulled ll header,
|
| 113 | so that SOCK_RAW should push it back.
|
| 114 |
|
| 115 | On receive:
|
| 116 | -----------
|
| 117 |
|
| 118 | Incoming, dev_has_header(dev) == true
|
| 119 | mac_header -> ll header
|
| 120 | data -> data
|
| 121 |
|
| 122 | Outgoing, dev_has_header(dev) == true
|
| 123 | mac_header -> ll header
|
| 124 | data -> ll header
|
| 125 |
|
| 126 | Incoming, dev_has_header(dev) == false
|
| 127 | mac_header -> data
|
| 128 | However drivers often make it point to the ll header.
|
| 129 | This is incorrect because the ll header should be invisible to us.
|
| 130 | data -> data
|
| 131 |
|
| 132 | Outgoing, dev_has_header(dev) == false
|
| 133 | mac_header -> data. ll header is invisible to us.
|
| 134 | data -> data
|
| 135 |
|
| 136 | Resume
|
| 137 | If dev_has_header(dev) == false we are unable to restore the ll header,
|
| 138 | because it is invisible to us.
|
| 139 |
|
| 140 |
|
| 141 | On transmit:
|
| 142 | ------------
|
| 143 |
|
| 144 | dev_has_header(dev) == true
|
| 145 | mac_header -> ll header
|
| 146 | data -> ll header
|
| 147 |
|
| 148 | dev_has_header(dev) == false (ll header is invisible to us)
|
| 149 | mac_header -> data
|
| 150 | data -> data
|
| 151 |
|
| 152 | We should set network_header on output to the correct position,
|
| 153 | packet classifier depends on it.
|
| 154 |
|
| 155 |
|
| 156 |
|
| 157 |
|
| 158 | |
| 159 | * a longer address field.
|
| 160 |
|
| 161 | struct packet_mreq_max {
|
| 162 | int mr_ifindex;
|
| 163 | unsigned short mr_type;
|
| 164 | unsigned short mr_alen;
|
| 165 | unsigned char mr_address[MAX_ADDR_LEN];
|
| 166 | };
|
| 167 |
|
| 168 | union tpacket_uhdr {
|
| 169 | struct tpacket_hdr *h1;
|
| 170 | struct tpacket2_hdr *h2;
|
| 171 | struct tpacket3_hdr *h3;
|
| 172 | void *raw;
|
| 173 | };
|
| 174 |
|
| 175 | static int packet_set_ring(struct sock *sk, union tpacket_req_u *req_u,
|
| 176 | int closing, int tx_ring);
|
| 177 |
|
| 178 | #define V3_ALIGNMENT (8)
|
| 179 |
|
| 180 | #define BLK_HDR_LEN (ALIGN(sizeof(struct tpacket_block_desc), V3_ALIGNMENT))
|
| 181 |
|
| 182 | #define BLK_PLUS_PRIV(sz_of_priv) \
|
| 183 | (BLK_HDR_LEN + ALIGN((sz_of_priv), V3_ALIGNMENT))
|
| 184 |
|
| 185 | #define BLOCK_STATUS(x) ((x)->hdr.bh1.block_status)
|
| 186 | #define BLOCK_NUM_PKTS(x) ((x)->hdr.bh1.num_pkts)
|
| 187 | #define BLOCK_O2FP(x) ((x)->hdr.bh1.offset_to_first_pkt)
|
| 188 | #define BLOCK_LEN(x) ((x)->hdr.bh1.blk_len)
|
| 189 | #define BLOCK_SNUM(x) ((x)->hdr.bh1.seq_num)
|
| 190 | #define BLOCK_O2PRIV(x) ((x)->offset_to_priv)
|
| 191 |
|
| 192 | struct packet_sock;
|
| 193 | static int tpacket_rcv(struct sk_buff *skb, struct net_device *dev,
|
| 194 | struct packet_type *pt, struct net_device *orig_dev);
|
| 195 |
|
| 196 | static void *packet_previous_frame(struct packet_sock *po,
|
| 197 | struct packet_ring_buffer *rb,
|
| 198 | int status);
|
| 199 | static void packet_increment_head(struct packet_ring_buffer *buff);
|
| 200 | static int prb_curr_blk_in_use(struct tpacket_block_desc *);
|
| 201 | static void *prb_dispatch_next_block(struct tpacket_kbdq_core *,
|
| 202 | struct packet_sock *);
|
| 203 | static void prb_retire_current_block(struct tpacket_kbdq_core *,
|
| 204 | struct packet_sock *, unsigned int status);
|
| 205 | static int prb_queue_frozen(struct tpacket_kbdq_core *);
|
| 206 | static void prb_open_block(struct tpacket_kbdq_core *,
|
| 207 | struct tpacket_block_desc *);
|
| 208 | static enum hrtimer_restart prb_retire_rx_blk_timer_expired(struct hrtimer *);
|
| 209 | static void prb_fill_rxhash(struct tpacket_kbdq_core *, struct tpacket3_hdr *);
|
| 210 | static void prb_clear_rxhash(struct tpacket_kbdq_core *,
|
| 211 | struct tpacket3_hdr *);
|
| 212 | static void prb_fill_vlan_info(struct tpacket_kbdq_core *,
|
| 213 | struct tpacket3_hdr *);
|
| 214 | static void packet_flush_mclist(struct sock *sk);
|
| 215 | static u16 packet_pick_tx_queue(struct sk_buff *skb);
|
| 216 |
|
| 217 | struct packet_skb_cb {
|
| 218 | union {
|
| 219 | struct sockaddr_pkt pkt;
|
| 220 | union {
|
| 221 | |
| 222 | * ll.sll_family and ll.protocol in order
|
| 223 | * to save room.
|
| 224 |
|
| 225 | unsigned int origlen;
|
| 226 | struct sockaddr_ll ll;
|
| 227 | };
|
| 228 | } sa;
|
| 229 | };
|
| 230 |
|
| 231 | #define vio_le() virtio_legacy_is_little_endian()
|
| 232 |
|
| 233 | #define PACKET_SKB_CB(__skb) ((struct packet_skb_cb *)((__skb)->cb))
|
| 234 |
|
| 235 | #define GET_PBDQC_FROM_RB(x) ((struct tpacket_kbdq_core *)(&(x)->prb_bdqc))
|
| 236 | #define GET_PBLOCK_DESC(x, bid) \
|
| 237 | ((struct tpacket_block_desc *)((x)->pkbdq[(bid)].buffer))
|
| 238 | #define GET_CURR_PBLOCK_DESC_FROM_CORE(x) \
|
| 239 | ((struct tpacket_block_desc *)((x)->pkbdq[(x)->kactive_blk_num].buffer))
|
| 240 | #define GET_NEXT_PRB_BLK_NUM(x) \
|
| 241 | (((x)->kactive_blk_num < ((x)->knum_blocks-1)) ? \
|
| 242 | ((x)->kactive_blk_num+1) : 0)
|
| 243 |
|
| 244 | static void __fanout_unlink(struct sock *sk, struct packet_sock *po);
|
| 245 | static void __fanout_link(struct sock *sk, struct packet_sock *po);
|
| 246 |
|
| 247 | #ifdef CONFIG_NETFILTER_EGRESS
|
| 248 | static noinline struct sk_buff *nf_hook_direct_egress(struct sk_buff *skb)
|
| 249 | {
|
| 250 | struct sk_buff *next, *head = NULL, *tail;
|
| 251 | int rc;
|
| 252 |
|
| 253 | rcu_read_lock();
|
| 254 | for (; skb != NULL; skb = next) {
|
| 255 | next = skb->next;
|
| 256 | skb_mark_not_on_list(skb);
|
| 257 |
|
| 258 | if (!nf_hook_egress(skb, &rc, skb->dev))
|
| 259 | continue;
|
| 260 |
|
| 261 | if (!head)
|
| 262 | head = skb;
|
| 263 | else
|
| 264 | tail->next = skb;
|
| 265 |
|
| 266 | tail = skb;
|
| 267 | }
|
| 268 | rcu_read_unlock();
|
| 269 |
|
| 270 | return head;
|
| 271 | }
|
| 272 | #endif
|
| 273 |
|
| 274 | static int packet_xmit(const struct packet_sock *po, struct sk_buff *skb)
|
| 275 | {
|
| 276 | if (!packet_sock_flag(po, PACKET_SOCK_QDISC_BYPASS))
|
| 277 | return dev_queue_xmit(skb);
|
| 278 |
|
| 279 | #ifdef CONFIG_NETFILTER_EGRESS
|
| 280 | if (nf_hook_egress_active()) {
|
| 281 | skb = nf_hook_direct_egress(skb);
|
| 282 | if (!skb)
|
| 283 | return NET_XMIT_DROP;
|
| 284 | }
|
| 285 | #endif
|
| 286 | return dev_direct_xmit(skb, packet_pick_tx_queue(skb));
|
| 287 | }
|
| 288 |
|
| 289 | static struct net_device *packet_cached_dev_get(struct packet_sock *po)
|
| 290 | {
|
| 291 | struct net_device *dev;
|
| 292 |
|
| 293 | rcu_read_lock();
|
| 294 | dev = rcu_dereference(po->cached_dev);
|
| 295 | dev_hold(dev);
|
| 296 | rcu_read_unlock();
|
| 297 |
|
| 298 | return dev;
|
| 299 | }
|
| 300 |
|
| 301 | static void packet_cached_dev_assign(struct packet_sock *po,
|
| 302 | struct net_device *dev)
|
| 303 | {
|
| 304 | rcu_assign_pointer(po->cached_dev, dev);
|
| 305 | }
|
| 306 |
|
| 307 | static void packet_cached_dev_reset(struct packet_sock *po)
|
| 308 | {
|
| 309 | RCU_INIT_POINTER(po->cached_dev, NULL);
|
| 310 | }
|
| 311 |
|
| 312 | static u16 packet_pick_tx_queue(struct sk_buff *skb)
|
| 313 | {
|
| 314 | struct net_device *dev = skb->dev;
|
| 315 | const struct net_device_ops *ops = dev->netdev_ops;
|
| 316 | int cpu = raw_smp_processor_id();
|
| 317 | u16 queue_index;
|
| 318 |
|
| 319 | #ifdef CONFIG_XPS
|
| 320 | skb->sender_cpu = cpu + 1;
|
| 321 | #endif
|
| 322 | skb_record_rx_queue(skb, cpu % dev->real_num_tx_queues);
|
| 323 | if (ops->ndo_select_queue) {
|
| 324 | queue_index = ops->ndo_select_queue(dev, skb, NULL);
|
| 325 | queue_index = netdev_cap_txqueue(dev, queue_index);
|
| 326 | } else {
|
| 327 | queue_index = netdev_pick_tx(dev, skb, NULL);
|
| 328 | }
|
| 329 |
|
| 330 | return queue_index;
|
| 331 | }
|
| 332 |
|
| 333 | |
| 334 | * or from a context in which asynchronous accesses to the packet
|
| 335 | * socket is not possible (packet_create()).
|
| 336 |
|
| 337 | static void __register_prot_hook(struct sock *sk)
|
| 338 | {
|
| 339 | struct packet_sock *po = pkt_sk(sk);
|
| 340 |
|
| 341 | if (!packet_sock_flag(po, PACKET_SOCK_RUNNING)) {
|
| 342 | if (po->fanout)
|
| 343 | __fanout_link(sk, po);
|
| 344 | else
|
| 345 | dev_add_pack(&po->prot_hook);
|
| 346 |
|
| 347 | sock_hold(sk);
|
| 348 | packet_sock_flag_set(po, PACKET_SOCK_RUNNING, 1);
|
| 349 | }
|
| 350 | }
|
| 351 |
|
| 352 | static void register_prot_hook(struct sock *sk)
|
| 353 | {
|
| 354 | lockdep_assert_held_once(&pkt_sk(sk)->bind_lock);
|
| 355 | __register_prot_hook(sk);
|
| 356 | }
|
| 357 |
|
| 358 | |
| 359 | * the po->bind_lock and do a synchronize_net to make sure no
|
| 360 | * asynchronous packet processing paths still refer to the elements
|
| 361 | * of po->prot_hook. If the sync parameter is false, it is the
|
| 362 | * callers responsibility to take care of this.
|
| 363 |
|
| 364 | static void __unregister_prot_hook(struct sock *sk, bool sync)
|
| 365 | {
|
| 366 | struct packet_sock *po = pkt_sk(sk);
|
| 367 |
|
| 368 | lockdep_assert_held_once(&po->bind_lock);
|
| 369 |
|
| 370 | packet_sock_flag_set(po, PACKET_SOCK_RUNNING, 0);
|
| 371 |
|
| 372 | if (po->fanout)
|
| 373 | __fanout_unlink(sk, po);
|
| 374 | else
|
| 375 | __dev_remove_pack(&po->prot_hook);
|
| 376 |
|
| 377 | __sock_put(sk);
|
| 378 |
|
| 379 | if (sync) {
|
| 380 | spin_unlock(&po->bind_lock);
|
| 381 | synchronize_net();
|
| 382 | spin_lock(&po->bind_lock);
|
| 383 | }
|
| 384 | }
|
| 385 |
|
| 386 | static void unregister_prot_hook(struct sock *sk, bool sync)
|
| 387 | {
|
| 388 | struct packet_sock *po = pkt_sk(sk);
|
| 389 |
|
| 390 | if (packet_sock_flag(po, PACKET_SOCK_RUNNING))
|
| 391 | __unregister_prot_hook(sk, sync);
|
| 392 | }
|
| 393 |
|
| 394 | static inline struct page * __pure pgv_to_page(void *addr)
|
| 395 | {
|
| 396 | if (is_vmalloc_addr(addr))
|
| 397 | return vmalloc_to_page(addr);
|
| 398 | return virt_to_page(addr);
|
| 399 | }
|
| 400 |
|
| 401 | static void __packet_set_status(struct packet_sock *po, void *frame, int status)
|
| 402 | {
|
| 403 | union tpacket_uhdr h;
|
| 404 |
|
| 405 |
|
| 406 |
|
| 407 | h.raw = frame;
|
| 408 | switch (po->tp_version) {
|
| 409 | case TPACKET_V1:
|
| 410 | WRITE_ONCE(h.h1->tp_status, status);
|
| 411 | flush_dcache_page(pgv_to_page(&h.h1->tp_status));
|
| 412 | break;
|
| 413 | case TPACKET_V2:
|
| 414 | WRITE_ONCE(h.h2->tp_status, status);
|
| 415 | flush_dcache_page(pgv_to_page(&h.h2->tp_status));
|
| 416 | break;
|
| 417 | case TPACKET_V3:
|
| 418 | WRITE_ONCE(h.h3->tp_status, status);
|
| 419 | flush_dcache_page(pgv_to_page(&h.h3->tp_status));
|
| 420 | break;
|
| 421 | default:
|
| 422 | WARN(1, "TPACKET version not supported.\n");
|
| 423 | BUG();
|
| 424 | }
|
| 425 |
|
| 426 | smp_wmb();
|
| 427 | }
|
| 428 |
|
| 429 | static int __packet_get_status(const struct packet_sock *po, void *frame)
|
| 430 | {
|
| 431 | union tpacket_uhdr h;
|
| 432 |
|
| 433 | smp_rmb();
|
| 434 |
|
| 435 |
|
| 436 |
|
| 437 | h.raw = frame;
|
| 438 | switch (po->tp_version) {
|
| 439 | case TPACKET_V1:
|
| 440 | flush_dcache_page(pgv_to_page(&h.h1->tp_status));
|
| 441 | return READ_ONCE(h.h1->tp_status);
|
| 442 | case TPACKET_V2:
|
| 443 | flush_dcache_page(pgv_to_page(&h.h2->tp_status));
|
| 444 | return READ_ONCE(h.h2->tp_status);
|
| 445 | case TPACKET_V3:
|
| 446 | flush_dcache_page(pgv_to_page(&h.h3->tp_status));
|
| 447 | return READ_ONCE(h.h3->tp_status);
|
| 448 | default:
|
| 449 | WARN(1, "TPACKET version not supported.\n");
|
| 450 | BUG();
|
| 451 | return 0;
|
| 452 | }
|
| 453 | }
|
| 454 |
|
| 455 | static __u32 tpacket_get_timestamp(struct sk_buff *skb, struct timespec64 *ts,
|
| 456 | unsigned int flags)
|
| 457 | {
|
| 458 | struct skb_shared_hwtstamps *shhwtstamps = skb_hwtstamps(skb);
|
| 459 |
|
| 460 | if (shhwtstamps &&
|
| 461 | (flags & SOF_TIMESTAMPING_RAW_HARDWARE) &&
|
| 462 | ktime_to_timespec64_cond(shhwtstamps->hwtstamp, ts))
|
| 463 | return TP_STATUS_TS_RAW_HARDWARE;
|
| 464 |
|
| 465 | if ((flags & SOF_TIMESTAMPING_SOFTWARE) &&
|
| 466 | ktime_to_timespec64_cond(skb_tstamp(skb), ts))
|
| 467 | return TP_STATUS_TS_SOFTWARE;
|
| 468 |
|
| 469 | return 0;
|
| 470 | }
|
| 471 |
|
| 472 | static __u32 __packet_set_timestamp(struct packet_sock *po, void *frame,
|
| 473 | struct sk_buff *skb)
|
| 474 | {
|
| 475 | union tpacket_uhdr h;
|
| 476 | struct timespec64 ts;
|
| 477 | __u32 ts_status;
|
| 478 |
|
| 479 | if (!(ts_status = tpacket_get_timestamp(skb, &ts, READ_ONCE(po->tp_tstamp))))
|
| 480 | return 0;
|
| 481 |
|
| 482 | h.raw = frame;
|
| 483 | |
| 484 | * versions 1 through 3 overflow the timestamps in y2106, since they
|
| 485 | * all store the seconds in a 32-bit unsigned integer.
|
| 486 | * If we create a version 4, that should have a 64-bit timestamp,
|
| 487 | * either 64-bit seconds + 32-bit nanoseconds, or just 64-bit
|
| 488 | * nanoseconds.
|
| 489 |
|
| 490 | switch (po->tp_version) {
|
| 491 | case TPACKET_V1:
|
| 492 | h.h1->tp_sec = ts.tv_sec;
|
| 493 | h.h1->tp_usec = ts.tv_nsec / NSEC_PER_USEC;
|
| 494 | break;
|
| 495 | case TPACKET_V2:
|
| 496 | h.h2->tp_sec = ts.tv_sec;
|
| 497 | h.h2->tp_nsec = ts.tv_nsec;
|
| 498 | break;
|
| 499 | case TPACKET_V3:
|
| 500 | h.h3->tp_sec = ts.tv_sec;
|
| 501 | h.h3->tp_nsec = ts.tv_nsec;
|
| 502 | break;
|
| 503 | default:
|
| 504 | WARN(1, "TPACKET version not supported.\n");
|
| 505 | BUG();
|
| 506 | }
|
| 507 |
|
| 508 |
|
| 509 | flush_dcache_page(pgv_to_page(&h.h1->tp_sec));
|
| 510 | smp_wmb();
|
| 511 |
|
| 512 | return ts_status;
|
| 513 | }
|
| 514 |
|
| 515 | static void *packet_lookup_frame(const struct packet_sock *po,
|
| 516 | const struct packet_ring_buffer *rb,
|
| 517 | unsigned int position,
|
| 518 | int status)
|
| 519 | {
|
| 520 | unsigned int pg_vec_pos, frame_offset;
|
| 521 | union tpacket_uhdr h;
|
| 522 |
|
| 523 | pg_vec_pos = position / rb->frames_per_block;
|
| 524 | frame_offset = position % rb->frames_per_block;
|
| 525 |
|
| 526 | h.raw = rb->pg_vec[pg_vec_pos].buffer +
|
| 527 | (frame_offset * rb->frame_size);
|
| 528 |
|
| 529 | if (status != __packet_get_status(po, h.raw))
|
| 530 | return NULL;
|
| 531 |
|
| 532 | return h.raw;
|
| 533 | }
|
| 534 |
|
| 535 | static void *packet_current_frame(struct packet_sock *po,
|
| 536 | struct packet_ring_buffer *rb,
|
| 537 | int status)
|
| 538 | {
|
| 539 | return packet_lookup_frame(po, rb, rb->head, status);
|
| 540 | }
|
| 541 |
|
| 542 | static u16 vlan_get_tci(const struct sk_buff *skb, struct net_device *dev)
|
| 543 | {
|
| 544 | struct vlan_hdr vhdr, *vh;
|
| 545 | unsigned int header_len;
|
| 546 |
|
| 547 | if (!dev)
|
| 548 | return 0;
|
| 549 |
|
| 550 | |
| 551 | * protocol, which is after the VLAN headers. The outer VLAN
|
| 552 | * header is at the hard_header_len offset in non-variable
|
| 553 | * length link layer headers. If it's a VLAN device, the
|
| 554 | * min_header_len should be used to exclude the VLAN header
|
| 555 | * size.
|
| 556 |
|
| 557 | if (dev->min_header_len == dev->hard_header_len)
|
| 558 | header_len = dev->hard_header_len;
|
| 559 | else if (is_vlan_dev(dev))
|
| 560 | header_len = dev->min_header_len;
|
| 561 | else
|
| 562 | return 0;
|
| 563 |
|
| 564 | vh = skb_header_pointer(skb, skb_mac_offset(skb) + header_len,
|
| 565 | sizeof(vhdr), &vhdr);
|
| 566 | if (unlikely(!vh))
|
| 567 | return 0;
|
| 568 |
|
| 569 | return ntohs(vh->h_vlan_TCI);
|
| 570 | }
|
| 571 |
|
| 572 | static __be16 vlan_get_protocol_dgram(const struct sk_buff *skb)
|
| 573 | {
|
| 574 | __be16 proto = skb->protocol;
|
| 575 |
|
| 576 | if (unlikely(eth_type_vlan(proto)))
|
| 577 | proto = vlan_get_protocol_offset_inline(skb, proto,
|
| 578 | skb_mac_offset(skb),
|
| 579 | NULL);
|
| 580 |
|
| 581 | return proto;
|
| 582 | }
|
| 583 |
|
| 584 | static void prb_shutdown_retire_blk_timer(struct packet_sock *po,
|
| 585 | struct sk_buff_head *rb_queue)
|
| 586 | {
|
| 587 | struct tpacket_kbdq_core *pkc;
|
| 588 |
|
| 589 | pkc = GET_PBDQC_FROM_RB(&po->rx_ring);
|
| 590 | hrtimer_cancel(&pkc->retire_blk_timer);
|
| 591 | }
|
| 592 |
|
| 593 | static int prb_calc_retire_blk_tmo(struct packet_sock *po,
|
| 594 | int blk_size_in_bytes)
|
| 595 | {
|
| 596 | struct net_device *dev;
|
| 597 | unsigned int mbits, div;
|
| 598 | struct ethtool_link_ksettings ecmd;
|
| 599 | int err;
|
| 600 |
|
| 601 | rtnl_lock();
|
| 602 | dev = __dev_get_by_index(sock_net(&po->sk), po->ifindex);
|
| 603 | if (unlikely(!dev)) {
|
| 604 | rtnl_unlock();
|
| 605 | return DEFAULT_PRB_RETIRE_TOV;
|
| 606 | }
|
| 607 | err = __ethtool_get_link_ksettings(dev, &ecmd);
|
| 608 | rtnl_unlock();
|
| 609 | if (err)
|
| 610 | return DEFAULT_PRB_RETIRE_TOV;
|
| 611 |
|
| 612 | |
| 613 | * need to worry about perf anyways
|
| 614 |
|
| 615 | if (ecmd.base.speed < SPEED_1000 ||
|
| 616 | ecmd.base.speed == SPEED_UNKNOWN)
|
| 617 | return DEFAULT_PRB_RETIRE_TOV;
|
| 618 |
|
| 619 | div = ecmd.base.speed / 1000;
|
| 620 | mbits = (u64)blk_size_in_bytes * 8 / (1024 * 1024);
|
| 621 |
|
| 622 | if (div)
|
| 623 | mbits /= div;
|
| 624 |
|
| 625 | if (div)
|
| 626 | return mbits + 1;
|
| 627 | return mbits;
|
| 628 | }
|
| 629 |
|
| 630 | static void prb_init_ft_ops(struct tpacket_kbdq_core *p1,
|
| 631 | union tpacket_req_u *req_u)
|
| 632 | {
|
| 633 | p1->feature_req_word = req_u->req3.tp_feature_req_word;
|
| 634 | }
|
| 635 |
|
| 636 | static void init_prb_bdqc(struct packet_sock *po,
|
| 637 | struct packet_ring_buffer *rb,
|
| 638 | struct pgv *pg_vec,
|
| 639 | union tpacket_req_u *req_u)
|
| 640 | {
|
| 641 | struct tpacket_kbdq_core *p1 = GET_PBDQC_FROM_RB(rb);
|
| 642 | struct tpacket_block_desc *pbd;
|
| 643 |
|
| 644 | memset(p1, 0x0, sizeof(*p1));
|
| 645 |
|
| 646 | p1->knxt_seq_num = 1;
|
| 647 | p1->pkbdq = pg_vec;
|
| 648 | pbd = (struct tpacket_block_desc *)pg_vec[0].buffer;
|
| 649 | p1->pkblk_start = pg_vec[0].buffer;
|
| 650 | p1->kblk_size = req_u->req3.tp_block_size;
|
| 651 | p1->knum_blocks = req_u->req3.tp_block_nr;
|
| 652 | p1->hdrlen = po->tp_hdrlen;
|
| 653 | p1->version = po->tp_version;
|
| 654 | po->stats.stats3.tp_freeze_q_cnt = 0;
|
| 655 | if (req_u->req3.tp_retire_blk_tov)
|
| 656 | p1->interval_ktime = ms_to_ktime(req_u->req3.tp_retire_blk_tov);
|
| 657 | else
|
| 658 | p1->interval_ktime = ms_to_ktime(prb_calc_retire_blk_tmo(po,
|
| 659 | req_u->req3.tp_block_size));
|
| 660 | p1->blk_sizeof_priv = req_u->req3.tp_sizeof_priv;
|
| 661 | rwlock_init(&p1->blk_fill_in_prog_lock);
|
| 662 |
|
| 663 | p1->max_frame_len = p1->kblk_size - BLK_PLUS_PRIV(p1->blk_sizeof_priv);
|
| 664 | prb_init_ft_ops(p1, req_u);
|
| 665 | hrtimer_setup(&p1->retire_blk_timer, prb_retire_rx_blk_timer_expired,
|
| 666 | CLOCK_MONOTONIC, HRTIMER_MODE_REL_SOFT);
|
| 667 | hrtimer_start(&p1->retire_blk_timer, p1->interval_ktime,
|
| 668 | HRTIMER_MODE_REL_SOFT);
|
| 669 | prb_open_block(p1, pbd);
|
| 670 | }
|
| 671 |
|
| 672 | |
| 673 | * With a 1MB block-size, on a 1Gbps line, it will take
|
| 674 | * i) ~8 ms to fill a block + ii) memcpy etc.
|
| 675 | * In this cut we are not accounting for the memcpy time.
|
| 676 | *
|
| 677 | * Since the tmo granularity is in msecs, it is not too expensive
|
| 678 | * to refresh the timer, lets say every '8' msecs.
|
| 679 | * Either the user can set the 'tmo' or we can derive it based on
|
| 680 | * a) line-speed and b) block-size.
|
| 681 | * prb_calc_retire_blk_tmo() calculates the tmo.
|
| 682 |
|
| 683 | static enum hrtimer_restart prb_retire_rx_blk_timer_expired(struct hrtimer *t)
|
| 684 | {
|
| 685 | struct packet_sock *po =
|
| 686 | timer_container_of(po, t, rx_ring.prb_bdqc.retire_blk_timer);
|
| 687 | struct tpacket_kbdq_core *pkc = GET_PBDQC_FROM_RB(&po->rx_ring);
|
| 688 | unsigned int frozen;
|
| 689 | struct tpacket_block_desc *pbd;
|
| 690 |
|
| 691 | spin_lock(&po->sk.sk_receive_queue.lock);
|
| 692 |
|
| 693 | frozen = prb_queue_frozen(pkc);
|
| 694 | pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
|
| 695 |
|
| 696 | |
| 697 | * tpacket_rcv:
|
| 698 | * lock(); increment BLOCK_NUM_PKTS; unlock()
|
| 699 | * copy_bits() is in progress ...
|
| 700 | * timer fires on other cpu:
|
| 701 | * we can't retire the current block because copy_bits
|
| 702 | * is in progress.
|
| 703 | *
|
| 704 |
|
| 705 | if (BLOCK_NUM_PKTS(pbd)) {
|
| 706 |
|
| 707 | write_lock(&pkc->blk_fill_in_prog_lock);
|
| 708 | write_unlock(&pkc->blk_fill_in_prog_lock);
|
| 709 | }
|
| 710 |
|
| 711 | if (!frozen) {
|
| 712 | if (BLOCK_NUM_PKTS(pbd)) {
|
| 713 |
|
| 714 | prb_retire_current_block(pkc, po, TP_STATUS_BLK_TMO);
|
| 715 | prb_dispatch_next_block(pkc, po);
|
| 716 | }
|
| 717 | } else {
|
| 718 | |
| 719 | * lagging behind.
|
| 720 |
|
| 721 | if (!prb_curr_blk_in_use(pbd)) {
|
| 722 | |
| 723 | * now the link went idle && the timer fired.
|
| 724 | * We don't have a block to close.So we open this
|
| 725 | * block and restart the timer.
|
| 726 | * opening a block thaws the queue,restarts timer
|
| 727 | * Thawing/timer-refresh is a side effect.
|
| 728 |
|
| 729 | prb_open_block(pkc, pbd);
|
| 730 | }
|
| 731 | }
|
| 732 |
|
| 733 | hrtimer_forward_now(&pkc->retire_blk_timer, pkc->interval_ktime);
|
| 734 | spin_unlock(&po->sk.sk_receive_queue.lock);
|
| 735 | return HRTIMER_RESTART;
|
| 736 | }
|
| 737 |
|
| 738 | static void prb_flush_block(struct tpacket_kbdq_core *pkc1,
|
| 739 | struct tpacket_block_desc *pbd1, __u32 status)
|
| 740 | {
|
| 741 |
|
| 742 |
|
| 743 | #if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE == 1
|
| 744 | u8 *start, *end;
|
| 745 |
|
| 746 | start = (u8 *)pbd1;
|
| 747 |
|
| 748 |
|
| 749 | start += PAGE_SIZE;
|
| 750 |
|
| 751 | end = (u8 *)PAGE_ALIGN((unsigned long)pkc1->pkblk_end);
|
| 752 | for (; start < end; start += PAGE_SIZE)
|
| 753 | flush_dcache_page(pgv_to_page(start));
|
| 754 |
|
| 755 | smp_wmb();
|
| 756 | #endif
|
| 757 |
|
| 758 |
|
| 759 |
|
| 760 | BLOCK_STATUS(pbd1) = status;
|
| 761 |
|
| 762 |
|
| 763 |
|
| 764 | #if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE == 1
|
| 765 | start = (u8 *)pbd1;
|
| 766 | flush_dcache_page(pgv_to_page(start));
|
| 767 |
|
| 768 | smp_wmb();
|
| 769 | #endif
|
| 770 | }
|
| 771 |
|
| 772 | |
| 773 | * Side effect:
|
| 774 | *
|
| 775 | * 1) flush the block
|
| 776 | * 2) Increment active_blk_num
|
| 777 | *
|
| 778 | * Note:We DONT refresh the timer on purpose.
|
| 779 | * Because almost always the next block will be opened.
|
| 780 |
|
| 781 | static void prb_close_block(struct tpacket_kbdq_core *pkc1,
|
| 782 | struct tpacket_block_desc *pbd1,
|
| 783 | struct packet_sock *po, unsigned int stat)
|
| 784 | {
|
| 785 | __u32 status = TP_STATUS_USER | stat;
|
| 786 |
|
| 787 | struct tpacket3_hdr *last_pkt;
|
| 788 | struct tpacket_hdr_v1 *h1 = &pbd1->hdr.bh1;
|
| 789 | struct sock *sk = &po->sk;
|
| 790 |
|
| 791 | if (atomic_read(&po->tp_drops))
|
| 792 | status |= TP_STATUS_LOSING;
|
| 793 |
|
| 794 | last_pkt = (struct tpacket3_hdr *)pkc1->prev;
|
| 795 | last_pkt->tp_next_offset = 0;
|
| 796 |
|
| 797 |
|
| 798 | if (BLOCK_NUM_PKTS(pbd1)) {
|
| 799 | h1->ts_last_pkt.ts_sec = last_pkt->tp_sec;
|
| 800 | h1->ts_last_pkt.ts_nsec = last_pkt->tp_nsec;
|
| 801 | } else {
|
| 802 | |
| 803 | *
|
| 804 | * It shouldn't really happen as we don't close empty
|
| 805 | * blocks. See prb_retire_rx_blk_timer_expired().
|
| 806 |
|
| 807 | struct timespec64 ts;
|
| 808 | ktime_get_real_ts64(&ts);
|
| 809 | h1->ts_last_pkt.ts_sec = ts.tv_sec;
|
| 810 | h1->ts_last_pkt.ts_nsec = ts.tv_nsec;
|
| 811 | }
|
| 812 |
|
| 813 | smp_wmb();
|
| 814 |
|
| 815 |
|
| 816 | prb_flush_block(pkc1, pbd1, status);
|
| 817 |
|
| 818 | sk->sk_data_ready(sk);
|
| 819 |
|
| 820 | pkc1->kactive_blk_num = GET_NEXT_PRB_BLK_NUM(pkc1);
|
| 821 | }
|
| 822 |
|
| 823 | static void prb_thaw_queue(struct tpacket_kbdq_core *pkc)
|
| 824 | {
|
| 825 | pkc->reset_pending_on_curr_blk = 0;
|
| 826 | }
|
| 827 |
|
| 828 | |
| 829 | * prb_open_block is called by tpacket_rcv or timer callback.
|
| 830 | *
|
| 831 | * Reasons why NOT update hrtimer in prb_open_block:
|
| 832 | * 1) It will increase complexity to distinguish the two caller scenario.
|
| 833 | * 2) hrtimer_cancel and hrtimer_start need to be called if you want to update
|
| 834 | * TMO of an already enqueued hrtimer, leading to complex shutdown logic.
|
| 835 | *
|
| 836 | * One side effect of NOT update hrtimer when called by tpacket_rcv is that
|
| 837 | * a newly opened block triggered by tpacket_rcv may be retired earlier than
|
| 838 | * expected. On the other hand, if timeout is updated in prb_open_block, the
|
| 839 | * frequent reception of network packets that leads to prb_open_block being
|
| 840 | * called may cause hrtimer to be removed and enqueued repeatedly.
|
| 841 |
|
| 842 | static void prb_open_block(struct tpacket_kbdq_core *pkc1,
|
| 843 | struct tpacket_block_desc *pbd1)
|
| 844 | {
|
| 845 | struct timespec64 ts;
|
| 846 | struct tpacket_hdr_v1 *h1 = &pbd1->hdr.bh1;
|
| 847 |
|
| 848 | smp_rmb();
|
| 849 |
|
| 850 | |
| 851 | * flexibility of making the priv area sticky
|
| 852 |
|
| 853 |
|
| 854 | BLOCK_SNUM(pbd1) = pkc1->knxt_seq_num++;
|
| 855 | BLOCK_NUM_PKTS(pbd1) = 0;
|
| 856 | BLOCK_LEN(pbd1) = BLK_PLUS_PRIV(pkc1->blk_sizeof_priv);
|
| 857 |
|
| 858 | ktime_get_real_ts64(&ts);
|
| 859 |
|
| 860 | h1->ts_first_pkt.ts_sec = ts.tv_sec;
|
| 861 | h1->ts_first_pkt.ts_nsec = ts.tv_nsec;
|
| 862 |
|
| 863 | pkc1->pkblk_start = (char *)pbd1;
|
| 864 | pkc1->nxt_offset = pkc1->pkblk_start + BLK_PLUS_PRIV(pkc1->blk_sizeof_priv);
|
| 865 |
|
| 866 | BLOCK_O2FP(pbd1) = (__u32)BLK_PLUS_PRIV(pkc1->blk_sizeof_priv);
|
| 867 | BLOCK_O2PRIV(pbd1) = BLK_HDR_LEN;
|
| 868 |
|
| 869 | pbd1->version = pkc1->version;
|
| 870 | pkc1->prev = pkc1->nxt_offset;
|
| 871 | pkc1->pkblk_end = pkc1->pkblk_start + pkc1->kblk_size;
|
| 872 |
|
| 873 | prb_thaw_queue(pkc1);
|
| 874 |
|
| 875 | smp_wmb();
|
| 876 | }
|
| 877 |
|
| 878 | |
| 879 | * Queue freeze logic:
|
| 880 | * 1) Assume tp_block_nr = 8 blocks.
|
| 881 | * 2) At time 't0', user opens Rx ring.
|
| 882 | * 3) Some time past 't0', kernel starts filling blocks starting from 0 .. 7
|
| 883 | * 4) user-space is either sleeping or processing block '0'.
|
| 884 | * 5) tpacket_rcv is currently filling block '7', since there is no space left,
|
| 885 | * it will close block-7,loop around and try to fill block '0'.
|
| 886 | * call-flow:
|
| 887 | * __packet_lookup_frame_in_block
|
| 888 | * prb_retire_current_block()
|
| 889 | * prb_dispatch_next_block()
|
| 890 | * |->(BLOCK_STATUS == USER) evaluates to true
|
| 891 | * 5.1) Since block-0 is currently in-use, we just freeze the queue.
|
| 892 | * 6) Now there are two cases:
|
| 893 | * 6.1) Link goes idle right after the queue is frozen.
|
| 894 | * But remember, the last open_block() refreshed the timer.
|
| 895 | * When this timer expires,it will refresh itself so that we can
|
| 896 | * re-open block-0 in near future.
|
| 897 | * 6.2) Link is busy and keeps on receiving packets. This is a simple
|
| 898 | * case and __packet_lookup_frame_in_block will check if block-0
|
| 899 | * is free and can now be re-used.
|
| 900 |
|
| 901 | static void prb_freeze_queue(struct tpacket_kbdq_core *pkc,
|
| 902 | struct packet_sock *po)
|
| 903 | {
|
| 904 | pkc->reset_pending_on_curr_blk = 1;
|
| 905 | po->stats.stats3.tp_freeze_q_cnt++;
|
| 906 | }
|
| 907 |
|
| 908 | #define TOTAL_PKT_LEN_INCL_ALIGN(length) (ALIGN((length), V3_ALIGNMENT))
|
| 909 |
|
| 910 | |
| 911 | * If the next block is free then we will dispatch it
|
| 912 | * and return a good offset.
|
| 913 | * Else, we will freeze the queue.
|
| 914 | * So, caller must check the return value.
|
| 915 |
|
| 916 | static void *prb_dispatch_next_block(struct tpacket_kbdq_core *pkc,
|
| 917 | struct packet_sock *po)
|
| 918 | {
|
| 919 | struct tpacket_block_desc *pbd;
|
| 920 |
|
| 921 | smp_rmb();
|
| 922 |
|
| 923 |
|
| 924 | pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
|
| 925 |
|
| 926 |
|
| 927 | if (TP_STATUS_USER & BLOCK_STATUS(pbd)) {
|
| 928 | prb_freeze_queue(pkc, po);
|
| 929 | return NULL;
|
| 930 | }
|
| 931 |
|
| 932 | |
| 933 | * 3.
|
| 934 | * open this block and return the offset where the first packet
|
| 935 | * needs to get stored.
|
| 936 |
|
| 937 | prb_open_block(pkc, pbd);
|
| 938 | return (void *)pkc->nxt_offset;
|
| 939 | }
|
| 940 |
|
| 941 | static void prb_retire_current_block(struct tpacket_kbdq_core *pkc,
|
| 942 | struct packet_sock *po, unsigned int status)
|
| 943 | {
|
| 944 | struct tpacket_block_desc *pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
|
| 945 |
|
| 946 |
|
| 947 | if (likely(TP_STATUS_KERNEL == BLOCK_STATUS(pbd))) {
|
| 948 | |
| 949 | * Plug the case where copy_bits() is in progress on
|
| 950 | * cpu-0 and tpacket_rcv() got invoked on cpu-1, didn't
|
| 951 | * have space to copy the pkt in the current block and
|
| 952 | * called prb_retire_current_block()
|
| 953 | *
|
| 954 | * We don't need to worry about the TMO case because
|
| 955 | * the timer-handler already handled this case.
|
| 956 |
|
| 957 | if (!(status & TP_STATUS_BLK_TMO)) {
|
| 958 |
|
| 959 | write_lock(&pkc->blk_fill_in_prog_lock);
|
| 960 | write_unlock(&pkc->blk_fill_in_prog_lock);
|
| 961 | }
|
| 962 | prb_close_block(pkc, pbd, po, status);
|
| 963 | return;
|
| 964 | }
|
| 965 | }
|
| 966 |
|
| 967 | static int prb_curr_blk_in_use(struct tpacket_block_desc *pbd)
|
| 968 | {
|
| 969 | return TP_STATUS_USER & BLOCK_STATUS(pbd);
|
| 970 | }
|
| 971 |
|
| 972 | static int prb_queue_frozen(struct tpacket_kbdq_core *pkc)
|
| 973 | {
|
| 974 | return pkc->reset_pending_on_curr_blk;
|
| 975 | }
|
| 976 |
|
| 977 | static void prb_clear_blk_fill_status(struct packet_ring_buffer *rb)
|
| 978 | __releases(&pkc->blk_fill_in_prog_lock)
|
| 979 | {
|
| 980 | struct tpacket_kbdq_core *pkc = GET_PBDQC_FROM_RB(rb);
|
| 981 |
|
| 982 | read_unlock(&pkc->blk_fill_in_prog_lock);
|
| 983 | }
|
| 984 |
|
| 985 | static void prb_fill_rxhash(struct tpacket_kbdq_core *pkc,
|
| 986 | struct tpacket3_hdr *ppd)
|
| 987 | {
|
| 988 | ppd->hv1.tp_rxhash = skb_get_hash(pkc->skb);
|
| 989 | }
|
| 990 |
|
| 991 | static void prb_clear_rxhash(struct tpacket_kbdq_core *pkc,
|
| 992 | struct tpacket3_hdr *ppd)
|
| 993 | {
|
| 994 | ppd->hv1.tp_rxhash = 0;
|
| 995 | }
|
| 996 |
|
| 997 | static void prb_fill_vlan_info(struct tpacket_kbdq_core *pkc,
|
| 998 | struct tpacket3_hdr *ppd)
|
| 999 | {
|
| 1000 | struct packet_sock *po = container_of(pkc, struct packet_sock, rx_ring.prb_bdqc);
|
| 1001 |
|
| 1002 | if (skb_vlan_tag_present(pkc->skb)) {
|
| 1003 | ppd->hv1.tp_vlan_tci = skb_vlan_tag_get(pkc->skb);
|
| 1004 | ppd->hv1.tp_vlan_tpid = ntohs(pkc->skb->vlan_proto);
|
| 1005 | ppd->tp_status = TP_STATUS_VLAN_VALID | TP_STATUS_VLAN_TPID_VALID;
|
| 1006 | } else if (unlikely(po->sk.sk_type == SOCK_DGRAM && eth_type_vlan(pkc->skb->protocol))) {
|
| 1007 | ppd->hv1.tp_vlan_tci = vlan_get_tci(pkc->skb, pkc->skb->dev);
|
| 1008 | ppd->hv1.tp_vlan_tpid = ntohs(pkc->skb->protocol);
|
| 1009 | ppd->tp_status = TP_STATUS_VLAN_VALID | TP_STATUS_VLAN_TPID_VALID;
|
| 1010 | } else {
|
| 1011 | ppd->hv1.tp_vlan_tci = 0;
|
| 1012 | ppd->hv1.tp_vlan_tpid = 0;
|
| 1013 | ppd->tp_status = TP_STATUS_AVAILABLE;
|
| 1014 | }
|
| 1015 | }
|
| 1016 |
|
| 1017 | static void prb_run_all_ft_ops(struct tpacket_kbdq_core *pkc,
|
| 1018 | struct tpacket3_hdr *ppd)
|
| 1019 | {
|
| 1020 | ppd->hv1.tp_padding = 0;
|
| 1021 | prb_fill_vlan_info(pkc, ppd);
|
| 1022 |
|
| 1023 | if (pkc->feature_req_word & TP_FT_REQ_FILL_RXHASH)
|
| 1024 | prb_fill_rxhash(pkc, ppd);
|
| 1025 | else
|
| 1026 | prb_clear_rxhash(pkc, ppd);
|
| 1027 | }
|
| 1028 |
|
| 1029 | static void prb_fill_curr_block(char *curr,
|
| 1030 | struct tpacket_kbdq_core *pkc,
|
| 1031 | struct tpacket_block_desc *pbd,
|
| 1032 | unsigned int len)
|
| 1033 | __acquires(&pkc->blk_fill_in_prog_lock)
|
| 1034 | {
|
| 1035 | struct tpacket3_hdr *ppd;
|
| 1036 |
|
| 1037 | ppd = (struct tpacket3_hdr *)curr;
|
| 1038 | ppd->tp_next_offset = TOTAL_PKT_LEN_INCL_ALIGN(len);
|
| 1039 | pkc->prev = curr;
|
| 1040 | pkc->nxt_offset += TOTAL_PKT_LEN_INCL_ALIGN(len);
|
| 1041 | BLOCK_LEN(pbd) += TOTAL_PKT_LEN_INCL_ALIGN(len);
|
| 1042 | BLOCK_NUM_PKTS(pbd) += 1;
|
| 1043 | read_lock(&pkc->blk_fill_in_prog_lock);
|
| 1044 | prb_run_all_ft_ops(pkc, ppd);
|
| 1045 | }
|
| 1046 |
|
| 1047 |
|
| 1048 | static void *__packet_lookup_frame_in_block(struct packet_sock *po,
|
| 1049 | struct sk_buff *skb,
|
| 1050 | unsigned int len
|
| 1051 | )
|
| 1052 | {
|
| 1053 | struct tpacket_kbdq_core *pkc;
|
| 1054 | struct tpacket_block_desc *pbd;
|
| 1055 | char *curr, *end;
|
| 1056 |
|
| 1057 | pkc = GET_PBDQC_FROM_RB(&po->rx_ring);
|
| 1058 | pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
|
| 1059 |
|
| 1060 |
|
| 1061 | if (prb_queue_frozen(pkc)) {
|
| 1062 | |
| 1063 | * Check if that last block which caused the queue to freeze,
|
| 1064 | * is still in_use by user-space.
|
| 1065 |
|
| 1066 | if (prb_curr_blk_in_use(pbd)) {
|
| 1067 |
|
| 1068 | return NULL;
|
| 1069 | } else {
|
| 1070 | |
| 1071 | * Ok, the block was released by user-space.
|
| 1072 | * Now let's open that block.
|
| 1073 | * opening a block also thaws the queue.
|
| 1074 | * Thawing is a side effect.
|
| 1075 |
|
| 1076 | prb_open_block(pkc, pbd);
|
| 1077 | }
|
| 1078 | }
|
| 1079 |
|
| 1080 | smp_mb();
|
| 1081 | curr = pkc->nxt_offset;
|
| 1082 | pkc->skb = skb;
|
| 1083 | end = (char *)pbd + pkc->kblk_size;
|
| 1084 |
|
| 1085 |
|
| 1086 | if (curr+TOTAL_PKT_LEN_INCL_ALIGN(len) < end) {
|
| 1087 | prb_fill_curr_block(curr, pkc, pbd, len);
|
| 1088 | return (void *)curr;
|
| 1089 | }
|
| 1090 |
|
| 1091 |
|
| 1092 | prb_retire_current_block(pkc, po, 0);
|
| 1093 |
|
| 1094 |
|
| 1095 | curr = (char *)prb_dispatch_next_block(pkc, po);
|
| 1096 | if (curr) {
|
| 1097 | pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
|
| 1098 | prb_fill_curr_block(curr, pkc, pbd, len);
|
| 1099 | return (void *)curr;
|
| 1100 | }
|
| 1101 |
|
| 1102 | |
| 1103 | * No free blocks are available.user_space hasn't caught up yet.
|
| 1104 | * Queue was just frozen and now this packet will get dropped.
|
| 1105 |
|
| 1106 | return NULL;
|
| 1107 | }
|
| 1108 |
|
| 1109 | static void *packet_current_rx_frame(struct packet_sock *po,
|
| 1110 | struct sk_buff *skb,
|
| 1111 | int status, unsigned int len)
|
| 1112 | {
|
| 1113 | char *curr = NULL;
|
| 1114 | switch (po->tp_version) {
|
| 1115 | case TPACKET_V1:
|
| 1116 | case TPACKET_V2:
|
| 1117 | curr = packet_lookup_frame(po, &po->rx_ring,
|
| 1118 | po->rx_ring.head, status);
|
| 1119 | return curr;
|
| 1120 | case TPACKET_V3:
|
| 1121 | return __packet_lookup_frame_in_block(po, skb, len);
|
| 1122 | default:
|
| 1123 | WARN(1, "TPACKET version not supported\n");
|
| 1124 | BUG();
|
| 1125 | return NULL;
|
| 1126 | }
|
| 1127 | }
|
| 1128 |
|
| 1129 | static void *prb_lookup_block(const struct packet_sock *po,
|
| 1130 | const struct packet_ring_buffer *rb,
|
| 1131 | unsigned int idx,
|
| 1132 | int status)
|
| 1133 | {
|
| 1134 | struct tpacket_kbdq_core *pkc = GET_PBDQC_FROM_RB(rb);
|
| 1135 | struct tpacket_block_desc *pbd = GET_PBLOCK_DESC(pkc, idx);
|
| 1136 |
|
| 1137 | if (status != BLOCK_STATUS(pbd))
|
| 1138 | return NULL;
|
| 1139 | return pbd;
|
| 1140 | }
|
| 1141 |
|
| 1142 | static int prb_previous_blk_num(struct packet_ring_buffer *rb)
|
| 1143 | {
|
| 1144 | unsigned int prev;
|
| 1145 | if (rb->prb_bdqc.kactive_blk_num)
|
| 1146 | prev = rb->prb_bdqc.kactive_blk_num-1;
|
| 1147 | else
|
| 1148 | prev = rb->prb_bdqc.knum_blocks-1;
|
| 1149 | return prev;
|
| 1150 | }
|
| 1151 |
|
| 1152 |
|
| 1153 | static void *__prb_previous_block(struct packet_sock *po,
|
| 1154 | struct packet_ring_buffer *rb,
|
| 1155 | int status)
|
| 1156 | {
|
| 1157 | unsigned int previous = prb_previous_blk_num(rb);
|
| 1158 | return prb_lookup_block(po, rb, previous, status);
|
| 1159 | }
|
| 1160 |
|
| 1161 | static void *packet_previous_rx_frame(struct packet_sock *po,
|
| 1162 | struct packet_ring_buffer *rb,
|
| 1163 | int status)
|
| 1164 | {
|
| 1165 | if (po->tp_version <= TPACKET_V2)
|
| 1166 | return packet_previous_frame(po, rb, status);
|
| 1167 |
|
| 1168 | return __prb_previous_block(po, rb, status);
|
| 1169 | }
|
| 1170 |
|
| 1171 | static void packet_increment_rx_head(struct packet_sock *po,
|
| 1172 | struct packet_ring_buffer *rb)
|
| 1173 | {
|
| 1174 | switch (po->tp_version) {
|
| 1175 | case TPACKET_V1:
|
| 1176 | case TPACKET_V2:
|
| 1177 | return packet_increment_head(rb);
|
| 1178 | case TPACKET_V3:
|
| 1179 | default:
|
| 1180 | WARN(1, "TPACKET version not supported.\n");
|
| 1181 | BUG();
|
| 1182 | return;
|
| 1183 | }
|
| 1184 | }
|
| 1185 |
|
| 1186 | static void *packet_previous_frame(struct packet_sock *po,
|
| 1187 | struct packet_ring_buffer *rb,
|
| 1188 | int status)
|
| 1189 | {
|
| 1190 | unsigned int previous = rb->head ? rb->head - 1 : rb->frame_max;
|
| 1191 | return packet_lookup_frame(po, rb, previous, status);
|
| 1192 | }
|
| 1193 |
|
| 1194 | static void packet_increment_head(struct packet_ring_buffer *buff)
|
| 1195 | {
|
| 1196 | buff->head = buff->head != buff->frame_max ? buff->head+1 : 0;
|
| 1197 | }
|
| 1198 |
|
| 1199 | static void packet_inc_pending(struct packet_ring_buffer *rb)
|
| 1200 | {
|
| 1201 | this_cpu_inc(*rb->pending_refcnt);
|
| 1202 | }
|
| 1203 |
|
| 1204 | static void packet_dec_pending(struct packet_ring_buffer *rb)
|
| 1205 | {
|
| 1206 | this_cpu_dec(*rb->pending_refcnt);
|
| 1207 | }
|
| 1208 |
|
| 1209 | static unsigned int packet_read_pending(const struct packet_ring_buffer *rb)
|
| 1210 | {
|
| 1211 | unsigned int refcnt = 0;
|
| 1212 | int cpu;
|
| 1213 |
|
| 1214 |
|
| 1215 | if (rb->pending_refcnt == NULL)
|
| 1216 | return 0;
|
| 1217 |
|
| 1218 | for_each_possible_cpu(cpu)
|
| 1219 | refcnt += *per_cpu_ptr(rb->pending_refcnt, cpu);
|
| 1220 |
|
| 1221 | return refcnt;
|
| 1222 | }
|
| 1223 |
|
| 1224 | static int packet_alloc_pending(struct packet_sock *po)
|
| 1225 | {
|
| 1226 | po->rx_ring.pending_refcnt = NULL;
|
| 1227 |
|
| 1228 | po->tx_ring.pending_refcnt = alloc_percpu(unsigned int);
|
| 1229 | if (unlikely(po->tx_ring.pending_refcnt == NULL))
|
| 1230 | return -ENOBUFS;
|
| 1231 |
|
| 1232 | return 0;
|
| 1233 | }
|
| 1234 |
|
| 1235 | static void packet_free_pending(struct packet_sock *po)
|
| 1236 | {
|
| 1237 | free_percpu(po->tx_ring.pending_refcnt);
|
| 1238 | }
|
| 1239 |
|
| 1240 | #define ROOM_POW_OFF 2
|
| 1241 | #define ROOM_NONE 0x0
|
| 1242 | #define ROOM_LOW 0x1
|
| 1243 | #define ROOM_NORMAL 0x2
|
| 1244 |
|
| 1245 | static bool __tpacket_has_room(const struct packet_sock *po, int pow_off)
|
| 1246 | {
|
| 1247 | int idx, len;
|
| 1248 |
|
| 1249 | len = READ_ONCE(po->rx_ring.frame_max) + 1;
|
| 1250 | idx = READ_ONCE(po->rx_ring.head);
|
| 1251 | if (pow_off)
|
| 1252 | idx += len >> pow_off;
|
| 1253 | if (idx >= len)
|
| 1254 | idx -= len;
|
| 1255 | return packet_lookup_frame(po, &po->rx_ring, idx, TP_STATUS_KERNEL);
|
| 1256 | }
|
| 1257 |
|
| 1258 | static bool __tpacket_v3_has_room(const struct packet_sock *po, int pow_off)
|
| 1259 | {
|
| 1260 | int idx, len;
|
| 1261 |
|
| 1262 | len = READ_ONCE(po->rx_ring.prb_bdqc.knum_blocks);
|
| 1263 | idx = READ_ONCE(po->rx_ring.prb_bdqc.kactive_blk_num);
|
| 1264 | if (pow_off)
|
| 1265 | idx += len >> pow_off;
|
| 1266 | if (idx >= len)
|
| 1267 | idx -= len;
|
| 1268 | return prb_lookup_block(po, &po->rx_ring, idx, TP_STATUS_KERNEL);
|
| 1269 | }
|
| 1270 |
|
| 1271 | static int __packet_rcv_has_room(const struct packet_sock *po,
|
| 1272 | const struct sk_buff *skb)
|
| 1273 | {
|
| 1274 | const struct sock *sk = &po->sk;
|
| 1275 | int ret = ROOM_NONE;
|
| 1276 |
|
| 1277 | if (po->prot_hook.func != tpacket_rcv) {
|
| 1278 | int rcvbuf = READ_ONCE(sk->sk_rcvbuf);
|
| 1279 | int avail = rcvbuf - atomic_read(&sk->sk_rmem_alloc)
|
| 1280 | - (skb ? skb->truesize : 0);
|
| 1281 |
|
| 1282 | if (avail > (rcvbuf >> ROOM_POW_OFF))
|
| 1283 | return ROOM_NORMAL;
|
| 1284 | else if (avail > 0)
|
| 1285 | return ROOM_LOW;
|
| 1286 | else
|
| 1287 | return ROOM_NONE;
|
| 1288 | }
|
| 1289 |
|
| 1290 | if (po->tp_version == TPACKET_V3) {
|
| 1291 | if (__tpacket_v3_has_room(po, ROOM_POW_OFF))
|
| 1292 | ret = ROOM_NORMAL;
|
| 1293 | else if (__tpacket_v3_has_room(po, 0))
|
| 1294 | ret = ROOM_LOW;
|
| 1295 | } else {
|
| 1296 | if (__tpacket_has_room(po, ROOM_POW_OFF))
|
| 1297 | ret = ROOM_NORMAL;
|
| 1298 | else if (__tpacket_has_room(po, 0))
|
| 1299 | ret = ROOM_LOW;
|
| 1300 | }
|
| 1301 |
|
| 1302 | return ret;
|
| 1303 | }
|
| 1304 |
|
| 1305 | static int packet_rcv_has_room(struct packet_sock *po, struct sk_buff *skb)
|
| 1306 | {
|
| 1307 | bool pressure;
|
| 1308 | int ret;
|
| 1309 |
|
| 1310 | ret = __packet_rcv_has_room(po, skb);
|
| 1311 | pressure = ret != ROOM_NORMAL;
|
| 1312 |
|
| 1313 | if (packet_sock_flag(po, PACKET_SOCK_PRESSURE) != pressure)
|
| 1314 | packet_sock_flag_set(po, PACKET_SOCK_PRESSURE, pressure);
|
| 1315 |
|
| 1316 | return ret;
|
| 1317 | }
|
| 1318 |
|
| 1319 | static void __packet_rcv_try_clear_pressure(struct packet_sock *po)
|
| 1320 | {
|
| 1321 | if (packet_sock_flag(po, PACKET_SOCK_PRESSURE) &&
|
| 1322 | __packet_rcv_has_room(po, NULL) == ROOM_NORMAL)
|
| 1323 | packet_sock_flag_set(po, PACKET_SOCK_PRESSURE, false);
|
| 1324 | }
|
| 1325 |
|
| 1326 | static void packet_rcv_try_clear_pressure(struct packet_sock *po)
|
| 1327 | {
|
| 1328 | struct sock *sk = &po->sk;
|
| 1329 |
|
| 1330 | if (!packet_sock_flag(po, PACKET_SOCK_PRESSURE))
|
| 1331 | return;
|
| 1332 |
|
| 1333 | spin_lock_bh(&sk->sk_receive_queue.lock);
|
| 1334 | __packet_rcv_try_clear_pressure(po);
|
| 1335 | spin_unlock_bh(&sk->sk_receive_queue.lock);
|
| 1336 | }
|
| 1337 |
|
| 1338 | static void packet_sock_destruct(struct sock *sk)
|
| 1339 | {
|
| 1340 | skb_queue_purge(&sk->sk_error_queue);
|
| 1341 |
|
| 1342 | WARN_ON(atomic_read(&sk->sk_rmem_alloc));
|
| 1343 | WARN_ON(refcount_read(&sk->sk_wmem_alloc));
|
| 1344 |
|
| 1345 | packet_free_pending(pkt_sk(sk));
|
| 1346 |
|
| 1347 | if (!sock_flag(sk, SOCK_DEAD)) {
|
| 1348 | pr_err("Attempt to release alive packet socket: %p\n", sk);
|
| 1349 | return;
|
| 1350 | }
|
| 1351 | }
|
| 1352 |
|
| 1353 | static bool fanout_flow_is_huge(struct packet_sock *po, struct sk_buff *skb)
|
| 1354 | {
|
| 1355 | u32 *history = po->rollover->history;
|
| 1356 | u32 victim, rxhash;
|
| 1357 | int i, count = 0;
|
| 1358 |
|
| 1359 | rxhash = skb_get_hash(skb);
|
| 1360 | for (i = 0; i < ROLLOVER_HLEN; i++)
|
| 1361 | if (READ_ONCE(history[i]) == rxhash)
|
| 1362 | count++;
|
| 1363 |
|
| 1364 | victim = get_random_u32_below(ROLLOVER_HLEN);
|
| 1365 |
|
| 1366 |
|
| 1367 | if (READ_ONCE(history[victim]) != rxhash)
|
| 1368 | WRITE_ONCE(history[victim], rxhash);
|
| 1369 |
|
| 1370 | return count > (ROLLOVER_HLEN >> 1);
|
| 1371 | }
|
| 1372 |
|
| 1373 | static unsigned int fanout_demux_hash(struct packet_fanout *f,
|
| 1374 | struct sk_buff *skb,
|
| 1375 | unsigned int num)
|
| 1376 | {
|
| 1377 | return reciprocal_scale(__skb_get_hash_symmetric(skb), num);
|
| 1378 | }
|
| 1379 |
|
| 1380 | static unsigned int fanout_demux_lb(struct packet_fanout *f,
|
| 1381 | struct sk_buff *skb,
|
| 1382 | unsigned int num)
|
| 1383 | {
|
| 1384 | unsigned int val = atomic_inc_return(&f->rr_cur);
|
| 1385 |
|
| 1386 | return val % num;
|
| 1387 | }
|
| 1388 |
|
| 1389 | static unsigned int fanout_demux_cpu(struct packet_fanout *f,
|
| 1390 | struct sk_buff *skb,
|
| 1391 | unsigned int num)
|
| 1392 | {
|
| 1393 | return smp_processor_id() % num;
|
| 1394 | }
|
| 1395 |
|
| 1396 | static unsigned int fanout_demux_rnd(struct packet_fanout *f,
|
| 1397 | struct sk_buff *skb,
|
| 1398 | unsigned int num)
|
| 1399 | {
|
| 1400 | return get_random_u32_below(num);
|
| 1401 | }
|
| 1402 |
|
| 1403 | static unsigned int fanout_demux_rollover(struct packet_fanout *f,
|
| 1404 | struct sk_buff *skb,
|
| 1405 | unsigned int idx, bool try_self,
|
| 1406 | unsigned int num)
|
| 1407 | {
|
| 1408 | struct packet_sock *po, *po_next, *po_skip = NULL;
|
| 1409 | unsigned int i, j, room = ROOM_NONE;
|
| 1410 |
|
| 1411 | po = pkt_sk(rcu_dereference(f->arr[idx]));
|
| 1412 |
|
| 1413 | if (try_self) {
|
| 1414 | room = packet_rcv_has_room(po, skb);
|
| 1415 | if (room == ROOM_NORMAL ||
|
| 1416 | (room == ROOM_LOW && !fanout_flow_is_huge(po, skb)))
|
| 1417 | return idx;
|
| 1418 | po_skip = po;
|
| 1419 | }
|
| 1420 |
|
| 1421 | i = j = min_t(int, po->rollover->sock, num - 1);
|
| 1422 | do {
|
| 1423 | po_next = pkt_sk(rcu_dereference(f->arr[i]));
|
| 1424 | if (po_next != po_skip &&
|
| 1425 | !packet_sock_flag(po_next, PACKET_SOCK_PRESSURE) &&
|
| 1426 | packet_rcv_has_room(po_next, skb) == ROOM_NORMAL) {
|
| 1427 | if (i != j)
|
| 1428 | po->rollover->sock = i;
|
| 1429 | atomic_long_inc(&po->rollover->num);
|
| 1430 | if (room == ROOM_LOW)
|
| 1431 | atomic_long_inc(&po->rollover->num_huge);
|
| 1432 | return i;
|
| 1433 | }
|
| 1434 |
|
| 1435 | if (++i == num)
|
| 1436 | i = 0;
|
| 1437 | } while (i != j);
|
| 1438 |
|
| 1439 | atomic_long_inc(&po->rollover->num_failed);
|
| 1440 | return idx;
|
| 1441 | }
|
| 1442 |
|
| 1443 | static unsigned int fanout_demux_qm(struct packet_fanout *f,
|
| 1444 | struct sk_buff *skb,
|
| 1445 | unsigned int num)
|
| 1446 | {
|
| 1447 | return skb_get_queue_mapping(skb) % num;
|
| 1448 | }
|
| 1449 |
|
| 1450 | static unsigned int fanout_demux_bpf(struct packet_fanout *f,
|
| 1451 | struct sk_buff *skb,
|
| 1452 | unsigned int num)
|
| 1453 | {
|
| 1454 | struct bpf_prog *prog;
|
| 1455 | unsigned int ret = 0;
|
| 1456 |
|
| 1457 | rcu_read_lock();
|
| 1458 | prog = rcu_dereference(f->bpf_prog);
|
| 1459 | if (prog)
|
| 1460 | ret = bpf_prog_run_clear_cb(prog, skb) % num;
|
| 1461 | rcu_read_unlock();
|
| 1462 |
|
| 1463 | return ret;
|
| 1464 | }
|
| 1465 |
|
| 1466 | static bool fanout_has_flag(struct packet_fanout *f, u16 flag)
|
| 1467 | {
|
| 1468 | return f->flags & (flag >> 8);
|
| 1469 | }
|
| 1470 |
|
| 1471 | static int packet_rcv_fanout(struct sk_buff *skb, struct net_device *dev,
|
| 1472 | struct packet_type *pt, struct net_device *orig_dev)
|
| 1473 | {
|
| 1474 | struct packet_fanout *f = pt->af_packet_priv;
|
| 1475 | unsigned int num = READ_ONCE(f->num_members);
|
| 1476 | struct net *net = read_pnet(&f->net);
|
| 1477 | struct packet_sock *po;
|
| 1478 | unsigned int idx;
|
| 1479 |
|
| 1480 | if (!net_eq(dev_net(dev), net) || !num) {
|
| 1481 | kfree_skb(skb);
|
| 1482 | return 0;
|
| 1483 | }
|
| 1484 |
|
| 1485 | if (fanout_has_flag(f, PACKET_FANOUT_FLAG_DEFRAG)) {
|
| 1486 | skb = ip_check_defrag(net, skb, IP_DEFRAG_AF_PACKET);
|
| 1487 | if (!skb)
|
| 1488 | return 0;
|
| 1489 | }
|
| 1490 | switch (f->type) {
|
| 1491 | case PACKET_FANOUT_HASH:
|
| 1492 | default:
|
| 1493 | idx = fanout_demux_hash(f, skb, num);
|
| 1494 | break;
|
| 1495 | case PACKET_FANOUT_LB:
|
| 1496 | idx = fanout_demux_lb(f, skb, num);
|
| 1497 | break;
|
| 1498 | case PACKET_FANOUT_CPU:
|
| 1499 | idx = fanout_demux_cpu(f, skb, num);
|
| 1500 | break;
|
| 1501 | case PACKET_FANOUT_RND:
|
| 1502 | idx = fanout_demux_rnd(f, skb, num);
|
| 1503 | break;
|
| 1504 | case PACKET_FANOUT_QM:
|
| 1505 | idx = fanout_demux_qm(f, skb, num);
|
| 1506 | break;
|
| 1507 | case PACKET_FANOUT_ROLLOVER:
|
| 1508 | idx = fanout_demux_rollover(f, skb, 0, false, num);
|
| 1509 | break;
|
| 1510 | case PACKET_FANOUT_CBPF:
|
| 1511 | case PACKET_FANOUT_EBPF:
|
| 1512 | idx = fanout_demux_bpf(f, skb, num);
|
| 1513 | break;
|
| 1514 | }
|
| 1515 |
|
| 1516 | if (fanout_has_flag(f, PACKET_FANOUT_FLAG_ROLLOVER))
|
| 1517 | idx = fanout_demux_rollover(f, skb, idx, true, num);
|
| 1518 |
|
| 1519 | po = pkt_sk(rcu_dereference(f->arr[idx]));
|
| 1520 | return po->prot_hook.func(skb, dev, &po->prot_hook, orig_dev);
|
| 1521 | }
|
| 1522 |
|
| 1523 | DEFINE_MUTEX(fanout_mutex);
|
| 1524 | EXPORT_SYMBOL_GPL(fanout_mutex);
|
| 1525 | static LIST_HEAD(fanout_list);
|
| 1526 | static u16 fanout_next_id;
|
| 1527 |
|
| 1528 | static void __fanout_link(struct sock *sk, struct packet_sock *po)
|
| 1529 | {
|
| 1530 | struct packet_fanout *f = po->fanout;
|
| 1531 |
|
| 1532 | spin_lock(&f->lock);
|
| 1533 | rcu_assign_pointer(f->arr[f->num_members], sk);
|
| 1534 | smp_wmb();
|
| 1535 | f->num_members++;
|
| 1536 | if (f->num_members == 1)
|
| 1537 | dev_add_pack(&f->prot_hook);
|
| 1538 | spin_unlock(&f->lock);
|
| 1539 | }
|
| 1540 |
|
| 1541 | static void __fanout_unlink(struct sock *sk, struct packet_sock *po)
|
| 1542 | {
|
| 1543 | struct packet_fanout *f = po->fanout;
|
| 1544 | int i;
|
| 1545 |
|
| 1546 | spin_lock(&f->lock);
|
| 1547 | for (i = 0; i < f->num_members; i++) {
|
| 1548 | if (rcu_dereference_protected(f->arr[i],
|
| 1549 | lockdep_is_held(&f->lock)) == sk)
|
| 1550 | break;
|
| 1551 | }
|
| 1552 | BUG_ON(i >= f->num_members);
|
| 1553 | rcu_assign_pointer(f->arr[i],
|
| 1554 | rcu_dereference_protected(f->arr[f->num_members - 1],
|
| 1555 | lockdep_is_held(&f->lock)));
|
| 1556 | f->num_members--;
|
| 1557 | if (f->num_members == 0)
|
| 1558 | __dev_remove_pack(&f->prot_hook);
|
| 1559 | spin_unlock(&f->lock);
|
| 1560 | }
|
| 1561 |
|
| 1562 | static bool match_fanout_group(struct packet_type *ptype, struct sock *sk)
|
| 1563 | {
|
| 1564 | if (sk->sk_family != PF_PACKET)
|
| 1565 | return false;
|
| 1566 |
|
| 1567 | return ptype->af_packet_priv == pkt_sk(sk)->fanout;
|
| 1568 | }
|
| 1569 |
|
| 1570 | static void fanout_init_data(struct packet_fanout *f)
|
| 1571 | {
|
| 1572 | switch (f->type) {
|
| 1573 | case PACKET_FANOUT_LB:
|
| 1574 | atomic_set(&f->rr_cur, 0);
|
| 1575 | break;
|
| 1576 | case PACKET_FANOUT_CBPF:
|
| 1577 | case PACKET_FANOUT_EBPF:
|
| 1578 | RCU_INIT_POINTER(f->bpf_prog, NULL);
|
| 1579 | break;
|
| 1580 | }
|
| 1581 | }
|
| 1582 |
|
| 1583 | static void __fanout_set_data_bpf(struct packet_fanout *f, struct bpf_prog *new)
|
| 1584 | {
|
| 1585 | struct bpf_prog *old;
|
| 1586 |
|
| 1587 | spin_lock(&f->lock);
|
| 1588 | old = rcu_dereference_protected(f->bpf_prog, lockdep_is_held(&f->lock));
|
| 1589 | rcu_assign_pointer(f->bpf_prog, new);
|
| 1590 | spin_unlock(&f->lock);
|
| 1591 |
|
| 1592 | if (old) {
|
| 1593 | synchronize_net();
|
| 1594 | bpf_prog_destroy(old);
|
| 1595 | }
|
| 1596 | }
|
| 1597 |
|
| 1598 | static int fanout_set_data_cbpf(struct packet_sock *po, sockptr_t data,
|
| 1599 | unsigned int len)
|
| 1600 | {
|
| 1601 | struct bpf_prog *new;
|
| 1602 | struct sock_fprog fprog;
|
| 1603 | int ret;
|
| 1604 |
|
| 1605 | if (sock_flag(&po->sk, SOCK_FILTER_LOCKED))
|
| 1606 | return -EPERM;
|
| 1607 |
|
| 1608 | ret = copy_bpf_fprog_from_user(&fprog, data, len);
|
| 1609 | if (ret)
|
| 1610 | return ret;
|
| 1611 |
|
| 1612 | ret = bpf_prog_create_from_user(&new, &fprog, NULL, false);
|
| 1613 | if (ret)
|
| 1614 | return ret;
|
| 1615 |
|
| 1616 | __fanout_set_data_bpf(po->fanout, new);
|
| 1617 | return 0;
|
| 1618 | }
|
| 1619 |
|
| 1620 | static int fanout_set_data_ebpf(struct packet_sock *po, sockptr_t data,
|
| 1621 | unsigned int len)
|
| 1622 | {
|
| 1623 | struct bpf_prog *new;
|
| 1624 | u32 fd;
|
| 1625 |
|
| 1626 | if (sock_flag(&po->sk, SOCK_FILTER_LOCKED))
|
| 1627 | return -EPERM;
|
| 1628 | if (len != sizeof(fd))
|
| 1629 | return -EINVAL;
|
| 1630 | if (copy_from_sockptr(&fd, data, len))
|
| 1631 | return -EFAULT;
|
| 1632 |
|
| 1633 | new = bpf_prog_get_type(fd, BPF_PROG_TYPE_SOCKET_FILTER);
|
| 1634 | if (IS_ERR(new))
|
| 1635 | return PTR_ERR(new);
|
| 1636 |
|
| 1637 | __fanout_set_data_bpf(po->fanout, new);
|
| 1638 | return 0;
|
| 1639 | }
|
| 1640 |
|
| 1641 | static int fanout_set_data(struct packet_sock *po, sockptr_t data,
|
| 1642 | unsigned int len)
|
| 1643 | {
|
| 1644 | switch (po->fanout->type) {
|
| 1645 | case PACKET_FANOUT_CBPF:
|
| 1646 | return fanout_set_data_cbpf(po, data, len);
|
| 1647 | case PACKET_FANOUT_EBPF:
|
| 1648 | return fanout_set_data_ebpf(po, data, len);
|
| 1649 | default:
|
| 1650 | return -EINVAL;
|
| 1651 | }
|
| 1652 | }
|
| 1653 |
|
| 1654 | static void fanout_release_data(struct packet_fanout *f)
|
| 1655 | {
|
| 1656 | switch (f->type) {
|
| 1657 | case PACKET_FANOUT_CBPF:
|
| 1658 | case PACKET_FANOUT_EBPF:
|
| 1659 | __fanout_set_data_bpf(f, NULL);
|
| 1660 | }
|
| 1661 | }
|
| 1662 |
|
| 1663 | static bool __fanout_id_is_free(struct sock *sk, u16 candidate_id)
|
| 1664 | {
|
| 1665 | struct packet_fanout *f;
|
| 1666 |
|
| 1667 | list_for_each_entry(f, &fanout_list, list) {
|
| 1668 | if (f->id == candidate_id &&
|
| 1669 | read_pnet(&f->net) == sock_net(sk)) {
|
| 1670 | return false;
|
| 1671 | }
|
| 1672 | }
|
| 1673 | return true;
|
| 1674 | }
|
| 1675 |
|
| 1676 | static bool fanout_find_new_id(struct sock *sk, u16 *new_id)
|
| 1677 | {
|
| 1678 | u16 id = fanout_next_id;
|
| 1679 |
|
| 1680 | do {
|
| 1681 | if (__fanout_id_is_free(sk, id)) {
|
| 1682 | *new_id = id;
|
| 1683 | fanout_next_id = id + 1;
|
| 1684 | return true;
|
| 1685 | }
|
| 1686 |
|
| 1687 | id++;
|
| 1688 | } while (id != fanout_next_id);
|
| 1689 |
|
| 1690 | return false;
|
| 1691 | }
|
| 1692 |
|
| 1693 | static int fanout_add(struct sock *sk, struct fanout_args *args)
|
| 1694 | {
|
| 1695 | struct packet_rollover *rollover = NULL;
|
| 1696 | struct packet_sock *po = pkt_sk(sk);
|
| 1697 | u16 type_flags = args->type_flags;
|
| 1698 | struct packet_fanout *f, *match;
|
| 1699 | u8 type = type_flags & 0xff;
|
| 1700 | u8 flags = type_flags >> 8;
|
| 1701 | u16 id = args->id;
|
| 1702 | int err;
|
| 1703 |
|
| 1704 | switch (type) {
|
| 1705 | case PACKET_FANOUT_ROLLOVER:
|
| 1706 | if (type_flags & PACKET_FANOUT_FLAG_ROLLOVER)
|
| 1707 | return -EINVAL;
|
| 1708 | break;
|
| 1709 | case PACKET_FANOUT_HASH:
|
| 1710 | case PACKET_FANOUT_LB:
|
| 1711 | case PACKET_FANOUT_CPU:
|
| 1712 | case PACKET_FANOUT_RND:
|
| 1713 | case PACKET_FANOUT_QM:
|
| 1714 | case PACKET_FANOUT_CBPF:
|
| 1715 | case PACKET_FANOUT_EBPF:
|
| 1716 | break;
|
| 1717 | default:
|
| 1718 | return -EINVAL;
|
| 1719 | }
|
| 1720 |
|
| 1721 | mutex_lock(&fanout_mutex);
|
| 1722 |
|
| 1723 | err = -EALREADY;
|
| 1724 | if (po->fanout)
|
| 1725 | goto out;
|
| 1726 |
|
| 1727 | if (type == PACKET_FANOUT_ROLLOVER ||
|
| 1728 | (type_flags & PACKET_FANOUT_FLAG_ROLLOVER)) {
|
| 1729 | err = -ENOMEM;
|
| 1730 | rollover = kzalloc_obj(*rollover);
|
| 1731 | if (!rollover)
|
| 1732 | goto out;
|
| 1733 | atomic_long_set(&rollover->num, 0);
|
| 1734 | atomic_long_set(&rollover->num_huge, 0);
|
| 1735 | atomic_long_set(&rollover->num_failed, 0);
|
| 1736 | }
|
| 1737 |
|
| 1738 | if (type_flags & PACKET_FANOUT_FLAG_UNIQUEID) {
|
| 1739 | if (id != 0) {
|
| 1740 | err = -EINVAL;
|
| 1741 | goto out;
|
| 1742 | }
|
| 1743 | if (!fanout_find_new_id(sk, &id)) {
|
| 1744 | err = -ENOMEM;
|
| 1745 | goto out;
|
| 1746 | }
|
| 1747 |
|
| 1748 | flags &= ~(PACKET_FANOUT_FLAG_UNIQUEID >> 8);
|
| 1749 | }
|
| 1750 |
|
| 1751 | match = NULL;
|
| 1752 | list_for_each_entry(f, &fanout_list, list) {
|
| 1753 | if (f->id == id &&
|
| 1754 | read_pnet(&f->net) == sock_net(sk)) {
|
| 1755 | match = f;
|
| 1756 | break;
|
| 1757 | }
|
| 1758 | }
|
| 1759 | err = -EINVAL;
|
| 1760 | if (match) {
|
| 1761 | if (match->flags != flags)
|
| 1762 | goto out;
|
| 1763 | if (args->max_num_members &&
|
| 1764 | args->max_num_members != match->max_num_members)
|
| 1765 | goto out;
|
| 1766 | } else {
|
| 1767 | if (args->max_num_members > PACKET_FANOUT_MAX)
|
| 1768 | goto out;
|
| 1769 | if (!args->max_num_members)
|
| 1770 |
|
| 1771 | args->max_num_members = 256;
|
| 1772 | err = -ENOMEM;
|
| 1773 | match = kvzalloc_flex(*match, arr, args->max_num_members);
|
| 1774 | if (!match)
|
| 1775 | goto out;
|
| 1776 | write_pnet(&match->net, sock_net(sk));
|
| 1777 | match->id = id;
|
| 1778 | match->type = type;
|
| 1779 | match->flags = flags;
|
| 1780 | INIT_LIST_HEAD(&match->list);
|
| 1781 | spin_lock_init(&match->lock);
|
| 1782 | refcount_set(&match->sk_ref, 0);
|
| 1783 | fanout_init_data(match);
|
| 1784 | match->prot_hook.type = po->prot_hook.type;
|
| 1785 | match->prot_hook.dev = po->prot_hook.dev;
|
| 1786 | match->prot_hook.func = packet_rcv_fanout;
|
| 1787 | match->prot_hook.af_packet_priv = match;
|
| 1788 | match->prot_hook.af_packet_net = read_pnet(&match->net);
|
| 1789 | match->prot_hook.id_match = match_fanout_group;
|
| 1790 | match->max_num_members = args->max_num_members;
|
| 1791 | match->prot_hook.ignore_outgoing = type_flags & PACKET_FANOUT_FLAG_IGNORE_OUTGOING;
|
| 1792 | list_add(&match->list, &fanout_list);
|
| 1793 | }
|
| 1794 | err = -EINVAL;
|
| 1795 |
|
| 1796 | spin_lock(&po->bind_lock);
|
| 1797 | if (po->num &&
|
| 1798 | match->type == type &&
|
| 1799 | match->prot_hook.type == po->prot_hook.type &&
|
| 1800 | match->prot_hook.dev == po->prot_hook.dev) {
|
| 1801 | err = -ENOSPC;
|
| 1802 | if (refcount_read(&match->sk_ref) < match->max_num_members) {
|
| 1803 |
|
| 1804 | WRITE_ONCE(po->fanout, match);
|
| 1805 |
|
| 1806 | po->rollover = rollover;
|
| 1807 | rollover = NULL;
|
| 1808 | refcount_set(&match->sk_ref, refcount_read(&match->sk_ref) + 1);
|
| 1809 | if (packet_sock_flag(po, PACKET_SOCK_RUNNING)) {
|
| 1810 | __dev_remove_pack(&po->prot_hook);
|
| 1811 | __fanout_link(sk, po);
|
| 1812 | }
|
| 1813 | err = 0;
|
| 1814 | }
|
| 1815 | }
|
| 1816 | spin_unlock(&po->bind_lock);
|
| 1817 |
|
| 1818 | if (err && !refcount_read(&match->sk_ref)) {
|
| 1819 | list_del(&match->list);
|
| 1820 | kvfree(match);
|
| 1821 | }
|
| 1822 |
|
| 1823 | out:
|
| 1824 | kfree(rollover);
|
| 1825 | mutex_unlock(&fanout_mutex);
|
| 1826 | return err;
|
| 1827 | }
|
| 1828 |
|
| 1829 | |
| 1830 | * pkt_sk(sk)->fanout from fanout_list and returns pkt_sk(sk)->fanout.
|
| 1831 | * It is the responsibility of the caller to call fanout_release_data() and
|
| 1832 | * free the returned packet_fanout (after synchronize_net())
|
| 1833 |
|
| 1834 | static struct packet_fanout *fanout_release(struct sock *sk)
|
| 1835 | {
|
| 1836 | struct packet_sock *po = pkt_sk(sk);
|
| 1837 | struct packet_fanout *f;
|
| 1838 |
|
| 1839 | mutex_lock(&fanout_mutex);
|
| 1840 | f = po->fanout;
|
| 1841 | if (f) {
|
| 1842 | po->fanout = NULL;
|
| 1843 |
|
| 1844 | if (refcount_dec_and_test(&f->sk_ref))
|
| 1845 | list_del(&f->list);
|
| 1846 | else
|
| 1847 | f = NULL;
|
| 1848 | }
|
| 1849 | mutex_unlock(&fanout_mutex);
|
| 1850 |
|
| 1851 | return f;
|
| 1852 | }
|
| 1853 |
|
| 1854 | static bool packet_extra_vlan_len_allowed(const struct net_device *dev,
|
| 1855 | struct sk_buff *skb)
|
| 1856 | {
|
| 1857 | |
| 1858 | * this now that we have the actual packet in hand. We can only
|
| 1859 | * do this check on Ethernet devices.
|
| 1860 |
|
| 1861 | if (unlikely(dev->type != ARPHRD_ETHER))
|
| 1862 | return false;
|
| 1863 |
|
| 1864 | skb_reset_mac_header(skb);
|
| 1865 | return likely(eth_hdr(skb)->h_proto == htons(ETH_P_8021Q));
|
| 1866 | }
|
| 1867 |
|
| 1868 | static const struct proto_ops packet_ops;
|
| 1869 |
|
| 1870 | static const struct proto_ops packet_ops_spkt;
|
| 1871 |
|
| 1872 | static int packet_rcv_spkt(struct sk_buff *skb, struct net_device *dev,
|
| 1873 | struct packet_type *pt, struct net_device *orig_dev)
|
| 1874 | {
|
| 1875 | struct sock *sk;
|
| 1876 | struct sockaddr_pkt *spkt;
|
| 1877 |
|
| 1878 | |
| 1879 | * When we registered the protocol we saved the socket in the data
|
| 1880 | * field for just this event.
|
| 1881 |
|
| 1882 |
|
| 1883 | sk = pt->af_packet_priv;
|
| 1884 |
|
| 1885 | |
| 1886 | * Yank back the headers [hope the device set this
|
| 1887 | * right or kerboom...]
|
| 1888 | *
|
| 1889 | * Incoming packets have ll header pulled,
|
| 1890 | * push it back.
|
| 1891 | *
|
| 1892 | * For outgoing ones skb->data == skb_mac_header(skb)
|
| 1893 | * so that this procedure is noop.
|
| 1894 |
|
| 1895 |
|
| 1896 | if (skb->pkt_type == PACKET_LOOPBACK)
|
| 1897 | goto out;
|
| 1898 |
|
| 1899 | if (!net_eq(dev_net(dev), sock_net(sk)))
|
| 1900 | goto out;
|
| 1901 |
|
| 1902 | skb = skb_share_check(skb, GFP_ATOMIC);
|
| 1903 | if (skb == NULL)
|
| 1904 | goto oom;
|
| 1905 |
|
| 1906 |
|
| 1907 | skb_dst_drop(skb);
|
| 1908 |
|
| 1909 |
|
| 1910 | nf_reset_ct(skb);
|
| 1911 |
|
| 1912 | spkt = &PACKET_SKB_CB(skb)->sa.pkt;
|
| 1913 |
|
| 1914 | if (dev_has_header(dev))
|
| 1915 | skb_push(skb, skb->data - skb_mac_header(skb));
|
| 1916 |
|
| 1917 | |
| 1918 | * The SOCK_PACKET socket receives _all_ frames.
|
| 1919 |
|
| 1920 |
|
| 1921 | spkt->spkt_family = dev->type;
|
| 1922 | strscpy(spkt->spkt_device, dev->name, sizeof(spkt->spkt_device));
|
| 1923 | spkt->spkt_protocol = skb->protocol;
|
| 1924 |
|
| 1925 | |
| 1926 | * Charge the memory to the socket. This is done specifically
|
| 1927 | * to prevent sockets using all the memory up.
|
| 1928 |
|
| 1929 |
|
| 1930 | if (sock_queue_rcv_skb(sk, skb) == 0)
|
| 1931 | return 0;
|
| 1932 |
|
| 1933 | out:
|
| 1934 | kfree_skb(skb);
|
| 1935 | oom:
|
| 1936 | return 0;
|
| 1937 | }
|
| 1938 |
|
| 1939 | static void packet_parse_headers(struct sk_buff *skb, struct socket *sock)
|
| 1940 | {
|
| 1941 | int depth;
|
| 1942 |
|
| 1943 |
|
| 1944 | skb_reset_mac_header(skb);
|
| 1945 |
|
| 1946 | if (!skb->protocol && sock->type == SOCK_RAW)
|
| 1947 | skb->protocol = dev_parse_header_protocol(skb);
|
| 1948 |
|
| 1949 | skb_probe_transport_header(skb);
|
| 1950 |
|
| 1951 |
|
| 1952 | if (likely(skb->dev->type == ARPHRD_ETHER) &&
|
| 1953 | eth_type_vlan(skb->protocol) &&
|
| 1954 | vlan_get_protocol_and_depth(skb, skb->protocol, &depth) != 0)
|
| 1955 | skb_set_network_header(skb, depth);
|
| 1956 | }
|
| 1957 |
|
| 1958 | |
| 1959 | * Output a raw packet to a device layer. This bypasses all the other
|
| 1960 | * protocol layers and you must therefore supply it with a complete frame
|
| 1961 |
|
| 1962 |
|
| 1963 | static int packet_sendmsg_spkt(struct socket *sock, struct msghdr *msg,
|
| 1964 | size_t len)
|
| 1965 | {
|
| 1966 | struct sock *sk = sock->sk;
|
| 1967 | DECLARE_SOCKADDR(struct sockaddr_pkt *, saddr, msg->msg_name);
|
| 1968 | struct sk_buff *skb = NULL;
|
| 1969 | struct net_device *dev;
|
| 1970 | struct sockcm_cookie sockc;
|
| 1971 | __be16 proto = 0;
|
| 1972 | int hard_header_len;
|
| 1973 | int extra_len = 0;
|
| 1974 | int err;
|
| 1975 |
|
| 1976 | |
| 1977 | * Get and verify the address.
|
| 1978 |
|
| 1979 |
|
| 1980 | if (saddr) {
|
| 1981 | if (msg->msg_namelen < sizeof(struct sockaddr))
|
| 1982 | return -EINVAL;
|
| 1983 | if (msg->msg_namelen == sizeof(struct sockaddr_pkt))
|
| 1984 | proto = saddr->spkt_protocol;
|
| 1985 | } else
|
| 1986 | return -ENOTCONN;
|
| 1987 |
|
| 1988 | |
| 1989 | * Find the device first to size check it
|
| 1990 |
|
| 1991 |
|
| 1992 | saddr->spkt_device[sizeof(saddr->spkt_device) - 1] = 0;
|
| 1993 | retry:
|
| 1994 | rcu_read_lock();
|
| 1995 | dev = dev_get_by_name_rcu(sock_net(sk), saddr->spkt_device);
|
| 1996 | err = -ENODEV;
|
| 1997 | if (dev == NULL)
|
| 1998 | goto out_unlock;
|
| 1999 |
|
| 2000 | err = -ENETDOWN;
|
| 2001 | if (!(dev->flags & IFF_UP))
|
| 2002 | goto out_unlock;
|
| 2003 |
|
| 2004 | |
| 2005 | * You may not queue a frame bigger than the mtu. This is the lowest level
|
| 2006 | * raw protocol and you must do your own fragmentation at this level.
|
| 2007 |
|
| 2008 |
|
| 2009 | if (unlikely(sock_flag(sk, SOCK_NOFCS))) {
|
| 2010 | if (!netif_supports_nofcs(dev)) {
|
| 2011 | err = -EPROTONOSUPPORT;
|
| 2012 | goto out_unlock;
|
| 2013 | }
|
| 2014 | extra_len = 4;
|
| 2015 | }
|
| 2016 |
|
| 2017 |
|
| 2018 | if (!skb)
|
| 2019 | hard_header_len = READ_ONCE(dev->hard_header_len);
|
| 2020 |
|
| 2021 | err = -EMSGSIZE;
|
| 2022 | if (len > dev->mtu + hard_header_len + VLAN_HLEN + extra_len)
|
| 2023 | goto out_unlock;
|
| 2024 |
|
| 2025 | if (!skb) {
|
| 2026 | size_t reserved = LL_RESERVED_SPACE_EX(dev, hard_header_len);
|
| 2027 | int tlen = dev->needed_tailroom;
|
| 2028 | unsigned int hhlen = dev->header_ops ? hard_header_len : 0;
|
| 2029 |
|
| 2030 | rcu_read_unlock();
|
| 2031 | skb = sock_wmalloc(sk, len + reserved + tlen, 0, GFP_KERNEL);
|
| 2032 | if (skb == NULL)
|
| 2033 | return -ENOBUFS;
|
| 2034 | |
| 2035 | * header at transmission time by themselves. PPP is the notable
|
| 2036 | * one here. This should really be fixed at the driver level.
|
| 2037 |
|
| 2038 | skb_reserve(skb, reserved);
|
| 2039 | skb_reset_network_header(skb);
|
| 2040 |
|
| 2041 |
|
| 2042 | if (hhlen) {
|
| 2043 | skb->data -= hhlen;
|
| 2044 | skb->tail -= hhlen;
|
| 2045 | if (len < hhlen)
|
| 2046 | skb_reset_network_header(skb);
|
| 2047 | }
|
| 2048 | err = memcpy_from_msg(skb_put(skb, len), msg, len);
|
| 2049 | if (err)
|
| 2050 | goto out_free;
|
| 2051 | goto retry;
|
| 2052 | }
|
| 2053 |
|
| 2054 | if (!dev_validate_header(dev, skb->data, len) || !skb->len) {
|
| 2055 | err = -EINVAL;
|
| 2056 | goto out_unlock;
|
| 2057 | }
|
| 2058 | if (len > (dev->mtu + hard_header_len + extra_len) &&
|
| 2059 | !packet_extra_vlan_len_allowed(dev, skb)) {
|
| 2060 | err = -EMSGSIZE;
|
| 2061 | goto out_unlock;
|
| 2062 | }
|
| 2063 |
|
| 2064 | sockcm_init(&sockc, sk);
|
| 2065 | if (msg->msg_controllen) {
|
| 2066 | err = sock_cmsg_send(sk, msg, &sockc);
|
| 2067 | if (unlikely(err))
|
| 2068 | goto out_unlock;
|
| 2069 | }
|
| 2070 |
|
| 2071 | skb->protocol = proto;
|
| 2072 | skb->dev = dev;
|
| 2073 | skb->priority = sockc.priority;
|
| 2074 | skb->mark = sockc.mark;
|
| 2075 | skb_set_delivery_type_by_clockid(skb, sockc.transmit_time, sk->sk_clockid);
|
| 2076 | skb_setup_tx_timestamp(skb, &sockc);
|
| 2077 |
|
| 2078 | if (unlikely(extra_len == 4))
|
| 2079 | skb->no_fcs = 1;
|
| 2080 |
|
| 2081 | packet_parse_headers(skb, sock);
|
| 2082 |
|
| 2083 | dev_queue_xmit(skb);
|
| 2084 | rcu_read_unlock();
|
| 2085 | return len;
|
| 2086 |
|
| 2087 | out_unlock:
|
| 2088 | rcu_read_unlock();
|
| 2089 | out_free:
|
| 2090 | kfree_skb(skb);
|
| 2091 | return err;
|
| 2092 | }
|
| 2093 |
|
| 2094 | static unsigned int run_filter(struct sk_buff *skb,
|
| 2095 | const struct sock *sk,
|
| 2096 | unsigned int res)
|
| 2097 | {
|
| 2098 | struct sk_filter *filter;
|
| 2099 |
|
| 2100 | rcu_read_lock();
|
| 2101 | filter = rcu_dereference(sk->sk_filter);
|
| 2102 | if (filter != NULL)
|
| 2103 | res = bpf_prog_run_clear_cb(filter->prog, skb);
|
| 2104 | rcu_read_unlock();
|
| 2105 |
|
| 2106 | return res;
|
| 2107 | }
|
| 2108 |
|
| 2109 | static int packet_rcv_vnet(struct msghdr *msg, const struct sk_buff *skb,
|
| 2110 | size_t *len, int vnet_hdr_sz)
|
| 2111 | {
|
| 2112 | struct virtio_net_hdr_mrg_rxbuf vnet_hdr = { .num_buffers = 0 };
|
| 2113 |
|
| 2114 | if (*len < vnet_hdr_sz)
|
| 2115 | return -EINVAL;
|
| 2116 | *len -= vnet_hdr_sz;
|
| 2117 |
|
| 2118 | if (virtio_net_hdr_from_skb(skb, (struct virtio_net_hdr *)&vnet_hdr, vio_le(), true, 0))
|
| 2119 | return -EINVAL;
|
| 2120 |
|
| 2121 | return memcpy_to_msg(msg, (void *)&vnet_hdr, vnet_hdr_sz);
|
| 2122 | }
|
| 2123 |
|
| 2124 | |
| 2125 | * This function makes lazy skb cloning in hope that most of packets
|
| 2126 | * are discarded by BPF.
|
| 2127 | *
|
| 2128 | * Note tricky part: we DO mangle shared skb! skb->data, skb->len
|
| 2129 | * and skb->cb are mangled. It works because (and until) packets
|
| 2130 | * falling here are owned by current CPU. Output packets are cloned
|
| 2131 | * by dev_queue_xmit_nit(), input packets are processed by net_bh
|
| 2132 | * sequentially, so that if we return skb to original state on exit,
|
| 2133 | * we will not harm anyone.
|
| 2134 |
|
| 2135 |
|
| 2136 | static int packet_rcv(struct sk_buff *skb, struct net_device *dev,
|
| 2137 | struct packet_type *pt, struct net_device *orig_dev)
|
| 2138 | {
|
| 2139 | enum skb_drop_reason drop_reason = SKB_CONSUMED;
|
| 2140 | struct sock *sk = NULL;
|
| 2141 | struct sockaddr_ll *sll;
|
| 2142 | struct packet_sock *po;
|
| 2143 | u8 *skb_head = skb->data;
|
| 2144 | int skb_len = skb->len;
|
| 2145 | unsigned int snaplen, res;
|
| 2146 |
|
| 2147 | if (skb->pkt_type == PACKET_LOOPBACK)
|
| 2148 | goto drop;
|
| 2149 |
|
| 2150 | sk = pt->af_packet_priv;
|
| 2151 | po = pkt_sk(sk);
|
| 2152 |
|
| 2153 | if (!net_eq(dev_net(dev), sock_net(sk)))
|
| 2154 | goto drop;
|
| 2155 |
|
| 2156 | skb->dev = dev;
|
| 2157 |
|
| 2158 | if (dev_has_header(dev)) {
|
| 2159 | |
| 2160 | * exported to higher levels.
|
| 2161 | *
|
| 2162 | * Otherwise, the device hides details of its frame
|
| 2163 | * structure, so that corresponding packet head is
|
| 2164 | * never delivered to user.
|
| 2165 |
|
| 2166 | if (sk->sk_type != SOCK_DGRAM)
|
| 2167 | skb_push(skb, skb->data - skb_mac_header(skb));
|
| 2168 | else if (skb->pkt_type == PACKET_OUTGOING) {
|
| 2169 |
|
| 2170 | skb_pull(skb, skb_network_offset(skb));
|
| 2171 | }
|
| 2172 | }
|
| 2173 |
|
| 2174 | snaplen = skb_frags_readable(skb) ? skb->len : skb_headlen(skb);
|
| 2175 |
|
| 2176 | res = run_filter(skb, sk, snaplen);
|
| 2177 | if (!res)
|
| 2178 | goto drop_n_restore;
|
| 2179 | if (snaplen > res)
|
| 2180 | snaplen = res;
|
| 2181 |
|
| 2182 | if (atomic_read(&sk->sk_rmem_alloc) >= sk->sk_rcvbuf)
|
| 2183 | goto drop_n_acct;
|
| 2184 |
|
| 2185 | if (skb_shared(skb)) {
|
| 2186 | struct sk_buff *nskb = skb_clone(skb, GFP_ATOMIC);
|
| 2187 | if (nskb == NULL)
|
| 2188 | goto drop_n_acct;
|
| 2189 |
|
| 2190 | if (skb_head != skb->data) {
|
| 2191 | skb->data = skb_head;
|
| 2192 | skb->len = skb_len;
|
| 2193 | }
|
| 2194 | consume_skb(skb);
|
| 2195 | skb = nskb;
|
| 2196 | }
|
| 2197 |
|
| 2198 | sock_skb_cb_check_size(sizeof(*PACKET_SKB_CB(skb)) + MAX_ADDR_LEN - 8);
|
| 2199 |
|
| 2200 | sll = &PACKET_SKB_CB(skb)->sa.ll;
|
| 2201 | sll->sll_hatype = dev->type;
|
| 2202 | sll->sll_pkttype = skb->pkt_type;
|
| 2203 | if (unlikely(packet_sock_flag(po, PACKET_SOCK_ORIGDEV)))
|
| 2204 | sll->sll_ifindex = orig_dev->ifindex;
|
| 2205 | else
|
| 2206 | sll->sll_ifindex = dev->ifindex;
|
| 2207 |
|
| 2208 | sll->sll_halen = dev_parse_header(skb, sll->sll_addr);
|
| 2209 |
|
| 2210 | |
| 2211 | * Use their space for storing the original skb length.
|
| 2212 |
|
| 2213 | PACKET_SKB_CB(skb)->sa.origlen = skb->len;
|
| 2214 |
|
| 2215 | if (pskb_trim(skb, snaplen))
|
| 2216 | goto drop_n_acct;
|
| 2217 |
|
| 2218 | skb_set_owner_r(skb, sk);
|
| 2219 | skb->dev = NULL;
|
| 2220 | skb_dst_drop(skb);
|
| 2221 |
|
| 2222 |
|
| 2223 | nf_reset_ct(skb);
|
| 2224 |
|
| 2225 | spin_lock(&sk->sk_receive_queue.lock);
|
| 2226 | po->stats.stats1.tp_packets++;
|
| 2227 | sock_skb_set_dropcount(sk, skb);
|
| 2228 | skb_clear_delivery_time(skb);
|
| 2229 | __skb_queue_tail(&sk->sk_receive_queue, skb);
|
| 2230 | spin_unlock(&sk->sk_receive_queue.lock);
|
| 2231 | sk->sk_data_ready(sk);
|
| 2232 | return 0;
|
| 2233 |
|
| 2234 | drop_n_acct:
|
| 2235 | atomic_inc(&po->tp_drops);
|
| 2236 | sk_drops_inc(sk);
|
| 2237 | drop_reason = SKB_DROP_REASON_PACKET_SOCK_ERROR;
|
| 2238 |
|
| 2239 | drop_n_restore:
|
| 2240 | if (skb_head != skb->data && skb_shared(skb)) {
|
| 2241 | skb->data = skb_head;
|
| 2242 | skb->len = skb_len;
|
| 2243 | }
|
| 2244 | drop:
|
| 2245 | sk_skb_reason_drop(sk, skb, drop_reason);
|
| 2246 | return 0;
|
| 2247 | }
|
| 2248 |
|
| 2249 | static int tpacket_rcv(struct sk_buff *skb, struct net_device *dev,
|
| 2250 | struct packet_type *pt, struct net_device *orig_dev)
|
| 2251 | {
|
| 2252 | enum skb_drop_reason drop_reason = SKB_CONSUMED;
|
| 2253 | struct sock *sk = NULL;
|
| 2254 | struct packet_sock *po;
|
| 2255 | struct sockaddr_ll *sll;
|
| 2256 | union tpacket_uhdr h;
|
| 2257 | u8 *skb_head = skb->data;
|
| 2258 | int skb_len = skb->len;
|
| 2259 | unsigned int snaplen, res;
|
| 2260 | unsigned long status = TP_STATUS_USER;
|
| 2261 | unsigned short macoff, hdrlen;
|
| 2262 | unsigned int netoff;
|
| 2263 | struct sk_buff *copy_skb = NULL;
|
| 2264 | struct timespec64 ts;
|
| 2265 | __u32 ts_status;
|
| 2266 | unsigned int slot_id = 0;
|
| 2267 | int vnet_hdr_sz = 0;
|
| 2268 |
|
| 2269 | |
| 2270 | * We may add members to them until current aligned size without forcing
|
| 2271 | * userspace to call getsockopt(..., PACKET_HDRLEN, ...).
|
| 2272 |
|
| 2273 | BUILD_BUG_ON(TPACKET_ALIGN(sizeof(*h.h2)) != 32);
|
| 2274 | BUILD_BUG_ON(TPACKET_ALIGN(sizeof(*h.h3)) != 48);
|
| 2275 |
|
| 2276 | if (skb->pkt_type == PACKET_LOOPBACK)
|
| 2277 | goto drop;
|
| 2278 |
|
| 2279 | sk = pt->af_packet_priv;
|
| 2280 | po = pkt_sk(sk);
|
| 2281 |
|
| 2282 | if (!net_eq(dev_net(dev), sock_net(sk)))
|
| 2283 | goto drop;
|
| 2284 |
|
| 2285 | if (dev_has_header(dev)) {
|
| 2286 | if (sk->sk_type != SOCK_DGRAM)
|
| 2287 | skb_push(skb, skb->data - skb_mac_header(skb));
|
| 2288 | else if (skb->pkt_type == PACKET_OUTGOING) {
|
| 2289 |
|
| 2290 | skb_pull(skb, skb_network_offset(skb));
|
| 2291 | }
|
| 2292 | }
|
| 2293 |
|
| 2294 | snaplen = skb_frags_readable(skb) ? skb->len : skb_headlen(skb);
|
| 2295 |
|
| 2296 | res = run_filter(skb, sk, snaplen);
|
| 2297 | if (!res)
|
| 2298 | goto drop_n_restore;
|
| 2299 |
|
| 2300 |
|
| 2301 | if (__packet_rcv_has_room(po, skb) == ROOM_NONE) {
|
| 2302 | atomic_inc(&po->tp_drops);
|
| 2303 | goto drop_n_restore;
|
| 2304 | }
|
| 2305 |
|
| 2306 | if (skb->ip_summed == CHECKSUM_PARTIAL)
|
| 2307 | status |= TP_STATUS_CSUMNOTREADY;
|
| 2308 | else if (skb->pkt_type != PACKET_OUTGOING &&
|
| 2309 | skb_csum_unnecessary(skb))
|
| 2310 | status |= TP_STATUS_CSUM_VALID;
|
| 2311 | if (skb_is_gso(skb) && skb_is_gso_tcp(skb))
|
| 2312 | status |= TP_STATUS_GSO_TCP;
|
| 2313 |
|
| 2314 | if (snaplen > res)
|
| 2315 | snaplen = res;
|
| 2316 |
|
| 2317 | if (sk->sk_type == SOCK_DGRAM) {
|
| 2318 | macoff = netoff = TPACKET_ALIGN(po->tp_hdrlen) + 16 +
|
| 2319 | po->tp_reserve;
|
| 2320 | } else {
|
| 2321 | unsigned int maclen = skb_network_offset(skb);
|
| 2322 | netoff = TPACKET_ALIGN(po->tp_hdrlen +
|
| 2323 | (maclen < 16 ? 16 : maclen)) +
|
| 2324 | po->tp_reserve;
|
| 2325 | vnet_hdr_sz = READ_ONCE(po->vnet_hdr_sz);
|
| 2326 | if (vnet_hdr_sz)
|
| 2327 | netoff += vnet_hdr_sz;
|
| 2328 | macoff = netoff - maclen;
|
| 2329 | }
|
| 2330 | if (netoff > USHRT_MAX) {
|
| 2331 | atomic_inc(&po->tp_drops);
|
| 2332 | goto drop_n_restore;
|
| 2333 | }
|
| 2334 | if (po->tp_version <= TPACKET_V2) {
|
| 2335 | if (macoff + snaplen > po->rx_ring.frame_size) {
|
| 2336 | if (READ_ONCE(po->copy_thresh) &&
|
| 2337 | atomic_read(&sk->sk_rmem_alloc) < sk->sk_rcvbuf) {
|
| 2338 | if (skb_shared(skb)) {
|
| 2339 | copy_skb = skb_clone(skb, GFP_ATOMIC);
|
| 2340 | } else {
|
| 2341 | copy_skb = skb_get(skb);
|
| 2342 | skb_head = skb->data;
|
| 2343 | }
|
| 2344 | if (copy_skb) {
|
| 2345 | memset(&PACKET_SKB_CB(copy_skb)->sa.ll, 0,
|
| 2346 | sizeof(PACKET_SKB_CB(copy_skb)->sa.ll));
|
| 2347 | skb_set_owner_r(copy_skb, sk);
|
| 2348 | }
|
| 2349 | }
|
| 2350 | snaplen = po->rx_ring.frame_size - macoff;
|
| 2351 | if ((int)snaplen < 0) {
|
| 2352 | snaplen = 0;
|
| 2353 | vnet_hdr_sz = 0;
|
| 2354 | }
|
| 2355 | }
|
| 2356 | } else if (unlikely(macoff + snaplen >
|
| 2357 | GET_PBDQC_FROM_RB(&po->rx_ring)->max_frame_len)) {
|
| 2358 | u32 nval;
|
| 2359 |
|
| 2360 | nval = GET_PBDQC_FROM_RB(&po->rx_ring)->max_frame_len - macoff;
|
| 2361 | pr_err_once("tpacket_rcv: packet too big, clamped from %u to %u. macoff=%u\n",
|
| 2362 | snaplen, nval, macoff);
|
| 2363 | snaplen = nval;
|
| 2364 | if (unlikely((int)snaplen < 0)) {
|
| 2365 | snaplen = 0;
|
| 2366 | macoff = GET_PBDQC_FROM_RB(&po->rx_ring)->max_frame_len;
|
| 2367 | vnet_hdr_sz = 0;
|
| 2368 | }
|
| 2369 | }
|
| 2370 | spin_lock(&sk->sk_receive_queue.lock);
|
| 2371 | h.raw = packet_current_rx_frame(po, skb,
|
| 2372 | TP_STATUS_KERNEL, (macoff+snaplen));
|
| 2373 | if (!h.raw)
|
| 2374 | goto drop_n_account;
|
| 2375 |
|
| 2376 | if (po->tp_version <= TPACKET_V2) {
|
| 2377 | slot_id = po->rx_ring.head;
|
| 2378 | if (test_bit(slot_id, po->rx_ring.rx_owner_map))
|
| 2379 | goto drop_n_account;
|
| 2380 | __set_bit(slot_id, po->rx_ring.rx_owner_map);
|
| 2381 | }
|
| 2382 |
|
| 2383 | if (vnet_hdr_sz &&
|
| 2384 | virtio_net_hdr_from_skb(skb, h.raw + macoff -
|
| 2385 | sizeof(struct virtio_net_hdr),
|
| 2386 | vio_le(), true, 0)) {
|
| 2387 | if (po->tp_version <= TPACKET_V2)
|
| 2388 | __clear_bit(slot_id, po->rx_ring.rx_owner_map);
|
| 2389 | else
|
| 2390 | prb_clear_blk_fill_status(&po->rx_ring);
|
| 2391 | goto drop_n_account;
|
| 2392 | }
|
| 2393 |
|
| 2394 | if (po->tp_version <= TPACKET_V2) {
|
| 2395 | packet_increment_rx_head(po, &po->rx_ring);
|
| 2396 | |
| 2397 | * LOSING will be reported till you read the stats,
|
| 2398 | * because it's COR - Clear On Read.
|
| 2399 | * Anyways, moving it for V1/V2 only as V3 doesn't need this
|
| 2400 | * at packet level.
|
| 2401 |
|
| 2402 | if (atomic_read(&po->tp_drops))
|
| 2403 | status |= TP_STATUS_LOSING;
|
| 2404 | }
|
| 2405 |
|
| 2406 | po->stats.stats1.tp_packets++;
|
| 2407 | if (copy_skb) {
|
| 2408 | status |= TP_STATUS_COPY;
|
| 2409 | skb_clear_delivery_time(copy_skb);
|
| 2410 | __skb_queue_tail(&sk->sk_receive_queue, copy_skb);
|
| 2411 | }
|
| 2412 | spin_unlock(&sk->sk_receive_queue.lock);
|
| 2413 |
|
| 2414 | skb_copy_bits(skb, 0, h.raw + macoff, snaplen);
|
| 2415 |
|
| 2416 | |
| 2417 | * closer to the time of capture.
|
| 2418 |
|
| 2419 | ts_status = tpacket_get_timestamp(skb, &ts,
|
| 2420 | READ_ONCE(po->tp_tstamp) |
|
| 2421 | SOF_TIMESTAMPING_SOFTWARE);
|
| 2422 | if (!ts_status)
|
| 2423 | ktime_get_real_ts64(&ts);
|
| 2424 |
|
| 2425 | status |= ts_status;
|
| 2426 |
|
| 2427 | switch (po->tp_version) {
|
| 2428 | case TPACKET_V1:
|
| 2429 | h.h1->tp_len = skb->len;
|
| 2430 | h.h1->tp_snaplen = snaplen;
|
| 2431 | h.h1->tp_mac = macoff;
|
| 2432 | h.h1->tp_net = netoff;
|
| 2433 | h.h1->tp_sec = ts.tv_sec;
|
| 2434 | h.h1->tp_usec = ts.tv_nsec / NSEC_PER_USEC;
|
| 2435 | hdrlen = sizeof(*h.h1);
|
| 2436 | break;
|
| 2437 | case TPACKET_V2:
|
| 2438 | h.h2->tp_len = skb->len;
|
| 2439 | h.h2->tp_snaplen = snaplen;
|
| 2440 | h.h2->tp_mac = macoff;
|
| 2441 | h.h2->tp_net = netoff;
|
| 2442 | h.h2->tp_sec = ts.tv_sec;
|
| 2443 | h.h2->tp_nsec = ts.tv_nsec;
|
| 2444 | if (skb_vlan_tag_present(skb)) {
|
| 2445 | h.h2->tp_vlan_tci = skb_vlan_tag_get(skb);
|
| 2446 | h.h2->tp_vlan_tpid = ntohs(skb->vlan_proto);
|
| 2447 | status |= TP_STATUS_VLAN_VALID | TP_STATUS_VLAN_TPID_VALID;
|
| 2448 | } else if (unlikely(sk->sk_type == SOCK_DGRAM && eth_type_vlan(skb->protocol))) {
|
| 2449 | h.h2->tp_vlan_tci = vlan_get_tci(skb, skb->dev);
|
| 2450 | h.h2->tp_vlan_tpid = ntohs(skb->protocol);
|
| 2451 | status |= TP_STATUS_VLAN_VALID | TP_STATUS_VLAN_TPID_VALID;
|
| 2452 | } else {
|
| 2453 | h.h2->tp_vlan_tci = 0;
|
| 2454 | h.h2->tp_vlan_tpid = 0;
|
| 2455 | }
|
| 2456 | memset(h.h2->tp_padding, 0, sizeof(h.h2->tp_padding));
|
| 2457 | hdrlen = sizeof(*h.h2);
|
| 2458 | break;
|
| 2459 | case TPACKET_V3:
|
| 2460 | |
| 2461 | * So DONT clear those fields here
|
| 2462 |
|
| 2463 | h.h3->tp_status |= status;
|
| 2464 | h.h3->tp_len = skb->len;
|
| 2465 | h.h3->tp_snaplen = snaplen;
|
| 2466 | h.h3->tp_mac = macoff;
|
| 2467 | h.h3->tp_net = netoff;
|
| 2468 | h.h3->tp_sec = ts.tv_sec;
|
| 2469 | h.h3->tp_nsec = ts.tv_nsec;
|
| 2470 | memset(h.h3->tp_padding, 0, sizeof(h.h3->tp_padding));
|
| 2471 | hdrlen = sizeof(*h.h3);
|
| 2472 | break;
|
| 2473 | default:
|
| 2474 | BUG();
|
| 2475 | }
|
| 2476 |
|
| 2477 | sll = h.raw + TPACKET_ALIGN(hdrlen);
|
| 2478 | sll->sll_halen = dev_parse_header(skb, sll->sll_addr);
|
| 2479 | sll->sll_family = AF_PACKET;
|
| 2480 | sll->sll_hatype = dev->type;
|
| 2481 | sll->sll_protocol = (sk->sk_type == SOCK_DGRAM) ?
|
| 2482 | vlan_get_protocol_dgram(skb) : skb->protocol;
|
| 2483 | sll->sll_pkttype = skb->pkt_type;
|
| 2484 | if (unlikely(packet_sock_flag(po, PACKET_SOCK_ORIGDEV)))
|
| 2485 | sll->sll_ifindex = orig_dev->ifindex;
|
| 2486 | else
|
| 2487 | sll->sll_ifindex = dev->ifindex;
|
| 2488 |
|
| 2489 | smp_mb();
|
| 2490 |
|
| 2491 | #if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE == 1
|
| 2492 | if (po->tp_version <= TPACKET_V2) {
|
| 2493 | u8 *start, *end;
|
| 2494 |
|
| 2495 | end = (u8 *) PAGE_ALIGN((unsigned long) h.raw +
|
| 2496 | macoff + snaplen);
|
| 2497 |
|
| 2498 | for (start = h.raw; start < end; start += PAGE_SIZE)
|
| 2499 | flush_dcache_page(pgv_to_page(start));
|
| 2500 | }
|
| 2501 | smp_wmb();
|
| 2502 | #endif
|
| 2503 |
|
| 2504 | if (po->tp_version <= TPACKET_V2) {
|
| 2505 | spin_lock(&sk->sk_receive_queue.lock);
|
| 2506 | __packet_set_status(po, h.raw, status);
|
| 2507 | __clear_bit(slot_id, po->rx_ring.rx_owner_map);
|
| 2508 | spin_unlock(&sk->sk_receive_queue.lock);
|
| 2509 | sk->sk_data_ready(sk);
|
| 2510 | } else if (po->tp_version == TPACKET_V3) {
|
| 2511 | prb_clear_blk_fill_status(&po->rx_ring);
|
| 2512 | }
|
| 2513 |
|
| 2514 | drop_n_restore:
|
| 2515 | if (skb_head != skb->data && skb_shared(skb)) {
|
| 2516 | skb->data = skb_head;
|
| 2517 | skb->len = skb_len;
|
| 2518 | }
|
| 2519 | drop:
|
| 2520 | sk_skb_reason_drop(sk, skb, drop_reason);
|
| 2521 | return 0;
|
| 2522 |
|
| 2523 | drop_n_account:
|
| 2524 | spin_unlock(&sk->sk_receive_queue.lock);
|
| 2525 | atomic_inc(&po->tp_drops);
|
| 2526 | drop_reason = SKB_DROP_REASON_PACKET_SOCK_ERROR;
|
| 2527 |
|
| 2528 | sk->sk_data_ready(sk);
|
| 2529 | sk_skb_reason_drop(sk, copy_skb, drop_reason);
|
| 2530 | goto drop_n_restore;
|
| 2531 | }
|
| 2532 |
|
| 2533 | static int __packet_snd_vnet_parse(struct virtio_net_hdr *vnet_hdr, size_t len)
|
| 2534 | {
|
| 2535 | if ((vnet_hdr->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) &&
|
| 2536 | (__virtio16_to_cpu(vio_le(), vnet_hdr->csum_start) +
|
| 2537 | __virtio16_to_cpu(vio_le(), vnet_hdr->csum_offset) + 2 >
|
| 2538 | __virtio16_to_cpu(vio_le(), vnet_hdr->hdr_len)))
|
| 2539 | vnet_hdr->hdr_len = __cpu_to_virtio16(vio_le(),
|
| 2540 | __virtio16_to_cpu(vio_le(), vnet_hdr->csum_start) +
|
| 2541 | __virtio16_to_cpu(vio_le(), vnet_hdr->csum_offset) + 2);
|
| 2542 |
|
| 2543 | if (__virtio16_to_cpu(vio_le(), vnet_hdr->hdr_len) > len)
|
| 2544 | return -EINVAL;
|
| 2545 |
|
| 2546 | return 0;
|
| 2547 | }
|
| 2548 |
|
| 2549 | static int packet_snd_vnet_parse(struct msghdr *msg, size_t *len,
|
| 2550 | struct virtio_net_hdr *vnet_hdr, int vnet_hdr_sz)
|
| 2551 | {
|
| 2552 | int ret;
|
| 2553 |
|
| 2554 | if (*len < vnet_hdr_sz)
|
| 2555 | return -EINVAL;
|
| 2556 | *len -= vnet_hdr_sz;
|
| 2557 |
|
| 2558 | if (!copy_from_iter_full(vnet_hdr, sizeof(*vnet_hdr), &msg->msg_iter))
|
| 2559 | return -EFAULT;
|
| 2560 |
|
| 2561 | ret = __packet_snd_vnet_parse(vnet_hdr, *len);
|
| 2562 | if (ret)
|
| 2563 | return ret;
|
| 2564 |
|
| 2565 |
|
| 2566 | if (vnet_hdr_sz != sizeof(struct virtio_net_hdr))
|
| 2567 | iov_iter_advance(&msg->msg_iter, vnet_hdr_sz - sizeof(struct virtio_net_hdr));
|
| 2568 |
|
| 2569 | return 0;
|
| 2570 | }
|
| 2571 |
|
| 2572 | struct tpacket_uarg {
|
| 2573 | struct ubuf_info ubuf;
|
| 2574 | struct packet_sock *po;
|
| 2575 | void *ph;
|
| 2576 | };
|
| 2577 |
|
| 2578 | static void tpacket_ubuf_complete(struct sk_buff *skb, struct ubuf_info *uarg,
|
| 2579 | bool success)
|
| 2580 | {
|
| 2581 | struct tpacket_uarg *tu = container_of(uarg, struct tpacket_uarg, ubuf);
|
| 2582 | struct packet_sock *po = tu->po;
|
| 2583 | void *ph = tu->ph;
|
| 2584 | __u32 ts;
|
| 2585 |
|
| 2586 | DEBUG_NET_WARN_ON_ONCE(!skb);
|
| 2587 |
|
| 2588 | if (!refcount_dec_and_test(&uarg->refcnt))
|
| 2589 | return;
|
| 2590 |
|
| 2591 | ts = __packet_set_timestamp(po, ph, skb);
|
| 2592 | __packet_set_status(po, ph, TP_STATUS_AVAILABLE | ts);
|
| 2593 |
|
| 2594 | packet_dec_pending(&po->tx_ring);
|
| 2595 | complete(&po->skb_completion);
|
| 2596 |
|
| 2597 | kfree(tu);
|
| 2598 | sk_free(&po->sk);
|
| 2599 | }
|
| 2600 |
|
| 2601 | static const struct ubuf_info_ops tpacket_ubuf_ops = {
|
| 2602 | .complete = tpacket_ubuf_complete,
|
| 2603 | };
|
| 2604 |
|
| 2605 | static int tpacket_fill_skb(struct packet_sock *po, struct sk_buff *skb,
|
| 2606 | struct net_device *dev, void *data, int tp_len,
|
| 2607 | __be16 proto, unsigned char *addr, int hlen, int copylen,
|
| 2608 | int hard_header_len,
|
| 2609 | const struct sockcm_cookie *sockc)
|
| 2610 | {
|
| 2611 | int to_write, offset, len, nr_frags, len_max;
|
| 2612 | struct socket *sock = po->sk.sk_socket;
|
| 2613 | struct page *page;
|
| 2614 | int err;
|
| 2615 |
|
| 2616 | if (proto != htons(ETH_P_ALL))
|
| 2617 | skb->protocol = proto;
|
| 2618 | skb->dev = dev;
|
| 2619 | skb->priority = sockc->priority;
|
| 2620 | skb->mark = sockc->mark;
|
| 2621 | skb_set_delivery_type_by_clockid(skb, sockc->transmit_time, po->sk.sk_clockid);
|
| 2622 | skb_setup_tx_timestamp(skb, sockc);
|
| 2623 |
|
| 2624 | skb_reserve(skb, hlen);
|
| 2625 | skb_reset_network_header(skb);
|
| 2626 |
|
| 2627 | to_write = tp_len;
|
| 2628 |
|
| 2629 | if (sock->type == SOCK_DGRAM) {
|
| 2630 | err = dev_hard_header(skb, dev, ntohs(proto), addr,
|
| 2631 | NULL, tp_len);
|
| 2632 | if (unlikely(err < 0))
|
| 2633 | return -EINVAL;
|
| 2634 | } else if (copylen) {
|
| 2635 | int hdrlen = min_t(int, copylen, tp_len);
|
| 2636 |
|
| 2637 | skb_push(skb, hard_header_len);
|
| 2638 | skb_put(skb, copylen - hard_header_len);
|
| 2639 | err = skb_store_bits(skb, 0, data, hdrlen);
|
| 2640 | if (unlikely(err))
|
| 2641 | return err;
|
| 2642 | if (!dev_validate_header(dev, skb->data, hdrlen))
|
| 2643 | return -EINVAL;
|
| 2644 |
|
| 2645 | data += hdrlen;
|
| 2646 | to_write -= hdrlen;
|
| 2647 | }
|
| 2648 |
|
| 2649 | offset = offset_in_page(data);
|
| 2650 | len_max = PAGE_SIZE - offset;
|
| 2651 | len = ((to_write > len_max) ? len_max : to_write);
|
| 2652 |
|
| 2653 | skb->data_len = to_write;
|
| 2654 | skb->len += to_write;
|
| 2655 | skb->truesize += to_write;
|
| 2656 | refcount_add(to_write, &po->sk.sk_wmem_alloc);
|
| 2657 |
|
| 2658 | while (likely(to_write)) {
|
| 2659 | nr_frags = skb_shinfo(skb)->nr_frags;
|
| 2660 |
|
| 2661 | if (unlikely(nr_frags >= MAX_SKB_FRAGS)) {
|
| 2662 | pr_err("Packet exceed the number of skb frags(%u)\n",
|
| 2663 | (unsigned int)MAX_SKB_FRAGS);
|
| 2664 | return -EFAULT;
|
| 2665 | }
|
| 2666 |
|
| 2667 | page = pgv_to_page(data);
|
| 2668 | data += len;
|
| 2669 | flush_dcache_page(page);
|
| 2670 | get_page(page);
|
| 2671 | skb_fill_page_desc(skb, nr_frags, page, offset, len);
|
| 2672 | to_write -= len;
|
| 2673 | offset = 0;
|
| 2674 | len_max = PAGE_SIZE;
|
| 2675 | len = ((to_write > len_max) ? len_max : to_write);
|
| 2676 | }
|
| 2677 |
|
| 2678 | if (unlikely(!skb->len))
|
| 2679 | return -EINVAL;
|
| 2680 |
|
| 2681 | return tp_len;
|
| 2682 | }
|
| 2683 |
|
| 2684 | static int tpacket_parse_header(struct packet_sock *po, void *frame,
|
| 2685 | int size_max, void **data)
|
| 2686 | {
|
| 2687 | union tpacket_uhdr ph;
|
| 2688 | u32 tp_len;
|
| 2689 | int off;
|
| 2690 |
|
| 2691 | ph.raw = frame;
|
| 2692 |
|
| 2693 | switch (po->tp_version) {
|
| 2694 | case TPACKET_V3:
|
| 2695 | if (ph.h3->tp_next_offset != 0) {
|
| 2696 | pr_warn_once("variable sized slot not supported");
|
| 2697 | return -EINVAL;
|
| 2698 | }
|
| 2699 | tp_len = ph.h3->tp_len;
|
| 2700 | break;
|
| 2701 | case TPACKET_V2:
|
| 2702 | tp_len = ph.h2->tp_len;
|
| 2703 | break;
|
| 2704 | default:
|
| 2705 | tp_len = ph.h1->tp_len;
|
| 2706 | break;
|
| 2707 | }
|
| 2708 | if (unlikely(tp_len > size_max)) {
|
| 2709 | pr_err("packet size is too long (%u > %d)\n", tp_len, size_max);
|
| 2710 | return -EMSGSIZE;
|
| 2711 | }
|
| 2712 |
|
| 2713 | if (unlikely(packet_sock_flag(po, PACKET_SOCK_TX_HAS_OFF))) {
|
| 2714 | int off_min, off_max;
|
| 2715 |
|
| 2716 | off_min = po->tp_hdrlen - sizeof(struct sockaddr_ll);
|
| 2717 | off_max = po->tx_ring.frame_size - tp_len;
|
| 2718 | if (po->sk.sk_type == SOCK_DGRAM) {
|
| 2719 | switch (po->tp_version) {
|
| 2720 | case TPACKET_V3:
|
| 2721 | off = ph.h3->tp_net;
|
| 2722 | break;
|
| 2723 | case TPACKET_V2:
|
| 2724 | off = ph.h2->tp_net;
|
| 2725 | break;
|
| 2726 | default:
|
| 2727 | off = ph.h1->tp_net;
|
| 2728 | break;
|
| 2729 | }
|
| 2730 | } else {
|
| 2731 | switch (po->tp_version) {
|
| 2732 | case TPACKET_V3:
|
| 2733 | off = ph.h3->tp_mac;
|
| 2734 | break;
|
| 2735 | case TPACKET_V2:
|
| 2736 | off = ph.h2->tp_mac;
|
| 2737 | break;
|
| 2738 | default:
|
| 2739 | off = ph.h1->tp_mac;
|
| 2740 | break;
|
| 2741 | }
|
| 2742 | }
|
| 2743 | if (unlikely((off < off_min) || (off_max < off)))
|
| 2744 | return -EINVAL;
|
| 2745 | } else {
|
| 2746 | off = po->tp_hdrlen - sizeof(struct sockaddr_ll);
|
| 2747 | }
|
| 2748 |
|
| 2749 | *data = frame + off;
|
| 2750 | return tp_len;
|
| 2751 | }
|
| 2752 |
|
| 2753 | static int tpacket_snd(struct packet_sock *po, struct msghdr *msg)
|
| 2754 | {
|
| 2755 | struct sk_buff *skb = NULL;
|
| 2756 | struct net_device *dev;
|
| 2757 | struct virtio_net_hdr vnet_hdr;
|
| 2758 | bool has_vnet_hdr = false;
|
| 2759 | struct sockcm_cookie sockc;
|
| 2760 | struct tpacket_uarg *uarg;
|
| 2761 | __be16 proto;
|
| 2762 | int err, reserve = 0;
|
| 2763 | void *ph;
|
| 2764 | DECLARE_SOCKADDR(struct sockaddr_ll *, saddr, msg->msg_name);
|
| 2765 | bool need_wait = !(msg->msg_flags & MSG_DONTWAIT);
|
| 2766 | int vnet_hdr_sz = READ_ONCE(po->vnet_hdr_sz);
|
| 2767 | unsigned char *addr = NULL;
|
| 2768 | int tp_len, size_max;
|
| 2769 | void *data;
|
| 2770 | int len_sum = 0;
|
| 2771 | int status = TP_STATUS_AVAILABLE;
|
| 2772 | int hard_header_len, hlen, tlen, copylen = 0;
|
| 2773 | long timeo;
|
| 2774 |
|
| 2775 | mutex_lock(&po->pg_vec_lock);
|
| 2776 |
|
| 2777 | |
| 2778 | * we need to confirm it under protection of pg_vec_lock.
|
| 2779 |
|
| 2780 | if (unlikely(!po->tx_ring.pg_vec)) {
|
| 2781 | err = -EBUSY;
|
| 2782 | goto out;
|
| 2783 | }
|
| 2784 | if (likely(saddr == NULL)) {
|
| 2785 | dev = packet_cached_dev_get(po);
|
| 2786 | proto = READ_ONCE(po->num);
|
| 2787 | } else {
|
| 2788 | err = -EINVAL;
|
| 2789 | if (msg->msg_namelen < sizeof(struct sockaddr_ll))
|
| 2790 | goto out;
|
| 2791 | if (msg->msg_namelen < (saddr->sll_halen
|
| 2792 | + offsetof(struct sockaddr_ll,
|
| 2793 | sll_addr)))
|
| 2794 | goto out;
|
| 2795 | proto = saddr->sll_protocol;
|
| 2796 | dev = dev_get_by_index(sock_net(&po->sk), saddr->sll_ifindex);
|
| 2797 | if (po->sk.sk_socket->type == SOCK_DGRAM) {
|
| 2798 | if (dev && msg->msg_namelen < dev->addr_len +
|
| 2799 | offsetof(struct sockaddr_ll, sll_addr))
|
| 2800 | goto out_put;
|
| 2801 | addr = saddr->sll_addr;
|
| 2802 | }
|
| 2803 | }
|
| 2804 |
|
| 2805 | err = -ENXIO;
|
| 2806 | if (unlikely(dev == NULL))
|
| 2807 | goto out;
|
| 2808 | err = -ENETDOWN;
|
| 2809 | if (unlikely(!(dev->flags & IFF_UP)))
|
| 2810 | goto out_put;
|
| 2811 |
|
| 2812 | sockcm_init(&sockc, &po->sk);
|
| 2813 | if (msg->msg_controllen) {
|
| 2814 | err = sock_cmsg_send(&po->sk, msg, &sockc);
|
| 2815 | if (unlikely(err))
|
| 2816 | goto out_put;
|
| 2817 | }
|
| 2818 |
|
| 2819 | hard_header_len = READ_ONCE(dev->hard_header_len);
|
| 2820 | if (po->sk.sk_socket->type == SOCK_RAW)
|
| 2821 | reserve = hard_header_len;
|
| 2822 | size_max = po->tx_ring.frame_size
|
| 2823 | - (po->tp_hdrlen - sizeof(struct sockaddr_ll));
|
| 2824 |
|
| 2825 | if ((size_max > dev->mtu + reserve + VLAN_HLEN) && !vnet_hdr_sz)
|
| 2826 | size_max = dev->mtu + reserve + VLAN_HLEN;
|
| 2827 |
|
| 2828 | timeo = sock_sndtimeo(&po->sk, msg->msg_flags & MSG_DONTWAIT);
|
| 2829 | reinit_completion(&po->skb_completion);
|
| 2830 |
|
| 2831 | do {
|
| 2832 | ph = packet_current_frame(po, &po->tx_ring,
|
| 2833 | TP_STATUS_SEND_REQUEST);
|
| 2834 | if (unlikely(ph == NULL)) {
|
| 2835 | |
| 2836 | * have to call it as it's per_cpu variable, but in
|
| 2837 | * fast-path we don't have to call it, only when ph
|
| 2838 | * is NULL, we need to check the pending_refcnt.
|
| 2839 |
|
| 2840 | if (need_wait && packet_read_pending(&po->tx_ring)) {
|
| 2841 | timeo = wait_for_completion_interruptible_timeout(&po->skb_completion, timeo);
|
| 2842 | if (timeo <= 0) {
|
| 2843 | err = !timeo ? -ETIMEDOUT : -ERESTARTSYS;
|
| 2844 | goto out_put;
|
| 2845 | }
|
| 2846 |
|
| 2847 | continue;
|
| 2848 | } else
|
| 2849 | break;
|
| 2850 | }
|
| 2851 |
|
| 2852 | skb = NULL;
|
| 2853 | tp_len = tpacket_parse_header(po, ph, size_max, &data);
|
| 2854 | if (tp_len < 0)
|
| 2855 | goto tpacket_error;
|
| 2856 |
|
| 2857 | status = TP_STATUS_SEND_REQUEST;
|
| 2858 | hlen = LL_RESERVED_SPACE_EX(dev, hard_header_len);
|
| 2859 | tlen = dev->needed_tailroom;
|
| 2860 | if (vnet_hdr_sz) {
|
| 2861 | data += vnet_hdr_sz;
|
| 2862 | tp_len -= vnet_hdr_sz;
|
| 2863 | if (tp_len < 0) {
|
| 2864 | tp_len = -EINVAL;
|
| 2865 | goto tpacket_error;
|
| 2866 | }
|
| 2867 | memcpy(&vnet_hdr, data - vnet_hdr_sz, sizeof(vnet_hdr));
|
| 2868 | if (__packet_snd_vnet_parse(&vnet_hdr, tp_len)) {
|
| 2869 | tp_len = -EINVAL;
|
| 2870 | goto tpacket_error;
|
| 2871 | }
|
| 2872 | copylen = __virtio16_to_cpu(vio_le(),
|
| 2873 | vnet_hdr.hdr_len);
|
| 2874 | has_vnet_hdr = true;
|
| 2875 | }
|
| 2876 |
|
| 2877 | if (unlikely(tp_len > INT_MAX - len_sum))
|
| 2878 | break;
|
| 2879 | copylen = max_t(int, copylen, hard_header_len);
|
| 2880 | skb = sock_alloc_send_skb(&po->sk,
|
| 2881 | hlen + tlen + sizeof(struct sockaddr_ll) +
|
| 2882 | (copylen - hard_header_len),
|
| 2883 | !need_wait, &err);
|
| 2884 |
|
| 2885 | if (unlikely(skb == NULL)) {
|
| 2886 |
|
| 2887 | if (likely(len_sum > 0))
|
| 2888 | err = len_sum;
|
| 2889 | goto out_status;
|
| 2890 | }
|
| 2891 | tp_len = tpacket_fill_skb(po, skb, dev, data, tp_len, proto,
|
| 2892 | addr, hlen, copylen, hard_header_len,
|
| 2893 | &sockc);
|
| 2894 | if (likely(tp_len >= 0) &&
|
| 2895 | tp_len > dev->mtu + reserve &&
|
| 2896 | !vnet_hdr_sz &&
|
| 2897 | !packet_extra_vlan_len_allowed(dev, skb))
|
| 2898 | tp_len = -EMSGSIZE;
|
| 2899 |
|
| 2900 | if (unlikely(tp_len < 0)) {
|
| 2901 | tpacket_error:
|
| 2902 | if (packet_sock_flag(po, PACKET_SOCK_TP_LOSS)) {
|
| 2903 | __packet_set_status(po, ph,
|
| 2904 | TP_STATUS_AVAILABLE);
|
| 2905 | packet_increment_head(&po->tx_ring);
|
| 2906 | kfree_skb(skb);
|
| 2907 | continue;
|
| 2908 | } else {
|
| 2909 | status = TP_STATUS_WRONG_FORMAT;
|
| 2910 | err = tp_len;
|
| 2911 | goto out_status;
|
| 2912 | }
|
| 2913 | }
|
| 2914 |
|
| 2915 | if (has_vnet_hdr) {
|
| 2916 | if (virtio_net_hdr_to_skb(skb, &vnet_hdr, vio_le())) {
|
| 2917 | tp_len = -EINVAL;
|
| 2918 | goto tpacket_error;
|
| 2919 | }
|
| 2920 | }
|
| 2921 |
|
| 2922 | packet_parse_headers(skb, po->sk.sk_socket);
|
| 2923 |
|
| 2924 | if (has_vnet_hdr)
|
| 2925 | virtio_net_hdr_set_proto(skb, &vnet_hdr);
|
| 2926 |
|
| 2927 | uarg = kmalloc(sizeof(*uarg), GFP_KERNEL);
|
| 2928 | if (unlikely(!uarg)) {
|
| 2929 | if (likely(len_sum > 0))
|
| 2930 | err = len_sum;
|
| 2931 | else
|
| 2932 | err = -ENOMEM;
|
| 2933 | goto out_status;
|
| 2934 | }
|
| 2935 | uarg->po = po;
|
| 2936 | uarg->ph = ph;
|
| 2937 | uarg->ubuf.ops = &tpacket_ubuf_ops;
|
| 2938 | uarg->ubuf.flags = SKBFL_ZEROCOPY_FRAG;
|
| 2939 | refcount_set(&uarg->ubuf.refcnt, 1);
|
| 2940 |
|
| 2941 |
|
| 2942 | refcount_inc(&po->sk.sk_wmem_alloc);
|
| 2943 | skb_zcopy_init(skb, &uarg->ubuf);
|
| 2944 |
|
| 2945 | __packet_set_status(po, ph, TP_STATUS_SENDING);
|
| 2946 | packet_inc_pending(&po->tx_ring);
|
| 2947 |
|
| 2948 | status = TP_STATUS_SEND_REQUEST;
|
| 2949 | err = packet_xmit(po, skb);
|
| 2950 | if (unlikely(err != 0)) {
|
| 2951 | if (err > 0)
|
| 2952 | err = net_xmit_errno(err);
|
| 2953 | if (err && __packet_get_status(po, ph) ==
|
| 2954 | TP_STATUS_AVAILABLE) {
|
| 2955 |
|
| 2956 | skb = NULL;
|
| 2957 | goto out_status;
|
| 2958 | }
|
| 2959 | |
| 2960 | * skb was dropped but not destructed yet;
|
| 2961 | * let's treat it like congestion or err < 0
|
| 2962 |
|
| 2963 | err = 0;
|
| 2964 | }
|
| 2965 | packet_increment_head(&po->tx_ring);
|
| 2966 | len_sum += tp_len;
|
| 2967 | } while (1);
|
| 2968 |
|
| 2969 | err = len_sum;
|
| 2970 | goto out_put;
|
| 2971 |
|
| 2972 | out_status:
|
| 2973 | __packet_set_status(po, ph, status);
|
| 2974 | kfree_skb(skb);
|
| 2975 | out_put:
|
| 2976 | dev_put(dev);
|
| 2977 | out:
|
| 2978 | mutex_unlock(&po->pg_vec_lock);
|
| 2979 | return err;
|
| 2980 | }
|
| 2981 |
|
| 2982 | static struct sk_buff *packet_alloc_skb(struct sock *sk, size_t prepad,
|
| 2983 | size_t reserve, size_t len,
|
| 2984 | size_t linear, int noblock,
|
| 2985 | int *err)
|
| 2986 | {
|
| 2987 | struct sk_buff *skb;
|
| 2988 |
|
| 2989 |
|
| 2990 | if (prepad + len < PAGE_SIZE || !linear)
|
| 2991 | linear = len;
|
| 2992 |
|
| 2993 | if (len - linear > MAX_SKB_FRAGS * (PAGE_SIZE << PAGE_ALLOC_COSTLY_ORDER))
|
| 2994 | linear = len - MAX_SKB_FRAGS * (PAGE_SIZE << PAGE_ALLOC_COSTLY_ORDER);
|
| 2995 | skb = sock_alloc_send_pskb(sk, prepad + linear, len - linear, noblock,
|
| 2996 | err, PAGE_ALLOC_COSTLY_ORDER);
|
| 2997 | if (!skb)
|
| 2998 | return NULL;
|
| 2999 |
|
| 3000 | skb_reserve(skb, reserve);
|
| 3001 | skb_put(skb, linear);
|
| 3002 | skb->data_len = len - linear;
|
| 3003 | skb->len += len - linear;
|
| 3004 |
|
| 3005 | return skb;
|
| 3006 | }
|
| 3007 |
|
| 3008 | static int packet_snd(struct socket *sock, struct msghdr *msg, size_t len)
|
| 3009 | {
|
| 3010 | struct sock *sk = sock->sk;
|
| 3011 | DECLARE_SOCKADDR(struct sockaddr_ll *, saddr, msg->msg_name);
|
| 3012 | struct sk_buff *skb;
|
| 3013 | struct net_device *dev;
|
| 3014 | __be16 proto;
|
| 3015 | unsigned char *addr = NULL;
|
| 3016 | int err, reserve = 0;
|
| 3017 | struct sockcm_cookie sockc;
|
| 3018 | struct virtio_net_hdr vnet_hdr = { 0 };
|
| 3019 | int offset = 0;
|
| 3020 | struct packet_sock *po = pkt_sk(sk);
|
| 3021 | int vnet_hdr_sz = READ_ONCE(po->vnet_hdr_sz);
|
| 3022 | int hard_header_len, hlen, tlen, linear;
|
| 3023 | int extra_len = 0;
|
| 3024 |
|
| 3025 | |
| 3026 | * Get and verify the address.
|
| 3027 |
|
| 3028 |
|
| 3029 | if (likely(saddr == NULL)) {
|
| 3030 | dev = packet_cached_dev_get(po);
|
| 3031 | proto = READ_ONCE(po->num);
|
| 3032 | } else {
|
| 3033 | err = -EINVAL;
|
| 3034 | if (msg->msg_namelen < sizeof(struct sockaddr_ll))
|
| 3035 | goto out;
|
| 3036 | if (msg->msg_namelen < (saddr->sll_halen + offsetof(struct sockaddr_ll, sll_addr)))
|
| 3037 | goto out;
|
| 3038 | proto = saddr->sll_protocol;
|
| 3039 | dev = dev_get_by_index(sock_net(sk), saddr->sll_ifindex);
|
| 3040 | if (sock->type == SOCK_DGRAM) {
|
| 3041 | if (dev && msg->msg_namelen < dev->addr_len +
|
| 3042 | offsetof(struct sockaddr_ll, sll_addr))
|
| 3043 | goto out_unlock;
|
| 3044 | addr = saddr->sll_addr;
|
| 3045 | }
|
| 3046 | }
|
| 3047 |
|
| 3048 | err = -ENXIO;
|
| 3049 | if (unlikely(dev == NULL))
|
| 3050 | goto out_unlock;
|
| 3051 | err = -ENETDOWN;
|
| 3052 | if (unlikely(!(dev->flags & IFF_UP)))
|
| 3053 | goto out_unlock;
|
| 3054 |
|
| 3055 | sockcm_init(&sockc, sk);
|
| 3056 | if (msg->msg_controllen) {
|
| 3057 | err = sock_cmsg_send(sk, msg, &sockc);
|
| 3058 | if (unlikely(err))
|
| 3059 | goto out_unlock;
|
| 3060 | }
|
| 3061 |
|
| 3062 | hard_header_len = READ_ONCE(dev->hard_header_len);
|
| 3063 | if (sock->type == SOCK_RAW)
|
| 3064 | reserve = hard_header_len;
|
| 3065 | if (vnet_hdr_sz) {
|
| 3066 | err = packet_snd_vnet_parse(msg, &len, &vnet_hdr, vnet_hdr_sz);
|
| 3067 | if (err)
|
| 3068 | goto out_unlock;
|
| 3069 | }
|
| 3070 |
|
| 3071 | if (unlikely(sock_flag(sk, SOCK_NOFCS))) {
|
| 3072 | if (!netif_supports_nofcs(dev)) {
|
| 3073 | err = -EPROTONOSUPPORT;
|
| 3074 | goto out_unlock;
|
| 3075 | }
|
| 3076 | extra_len = 4;
|
| 3077 | }
|
| 3078 |
|
| 3079 | err = -EMSGSIZE;
|
| 3080 | if (!vnet_hdr.gso_type &&
|
| 3081 | (len > dev->mtu + reserve + VLAN_HLEN + extra_len))
|
| 3082 | goto out_unlock;
|
| 3083 |
|
| 3084 | err = -ENOBUFS;
|
| 3085 | hlen = LL_RESERVED_SPACE_EX(dev, hard_header_len);
|
| 3086 | tlen = dev->needed_tailroom;
|
| 3087 | linear = __virtio16_to_cpu(vio_le(), vnet_hdr.hdr_len);
|
| 3088 | linear = max(linear, min_t(int, len, hard_header_len));
|
| 3089 | skb = packet_alloc_skb(sk, hlen + tlen, hlen, len, linear,
|
| 3090 | msg->msg_flags & MSG_DONTWAIT, &err);
|
| 3091 | if (skb == NULL)
|
| 3092 | goto out_unlock;
|
| 3093 |
|
| 3094 | skb_reset_network_header(skb);
|
| 3095 |
|
| 3096 | err = -EINVAL;
|
| 3097 | if (sock->type == SOCK_DGRAM) {
|
| 3098 | offset = dev_hard_header(skb, dev, ntohs(proto), addr, NULL, len);
|
| 3099 | if (unlikely(offset < 0))
|
| 3100 | goto out_free;
|
| 3101 | } else if (reserve) {
|
| 3102 | skb_reserve(skb, -reserve);
|
| 3103 | if (len < reserve + sizeof(struct ipv6hdr) &&
|
| 3104 | dev->min_header_len != hard_header_len)
|
| 3105 | skb_reset_network_header(skb);
|
| 3106 | }
|
| 3107 |
|
| 3108 |
|
| 3109 | err = skb_copy_datagram_from_iter(skb, offset, &msg->msg_iter, len);
|
| 3110 | if (err)
|
| 3111 | goto out_free;
|
| 3112 |
|
| 3113 | if ((sock->type == SOCK_RAW &&
|
| 3114 | !dev_validate_header(dev, skb->data, len)) || !skb->len) {
|
| 3115 | err = -EINVAL;
|
| 3116 | goto out_free;
|
| 3117 | }
|
| 3118 |
|
| 3119 | skb_setup_tx_timestamp(skb, &sockc);
|
| 3120 |
|
| 3121 | if (!vnet_hdr.gso_type && (len > dev->mtu + reserve + extra_len) &&
|
| 3122 | !packet_extra_vlan_len_allowed(dev, skb)) {
|
| 3123 | err = -EMSGSIZE;
|
| 3124 | goto out_free;
|
| 3125 | }
|
| 3126 |
|
| 3127 | if (proto != htons(ETH_P_ALL))
|
| 3128 | skb->protocol = proto;
|
| 3129 | skb->dev = dev;
|
| 3130 | skb->priority = sockc.priority;
|
| 3131 | skb->mark = sockc.mark;
|
| 3132 | skb_set_delivery_type_by_clockid(skb, sockc.transmit_time, sk->sk_clockid);
|
| 3133 |
|
| 3134 | if (unlikely(extra_len == 4))
|
| 3135 | skb->no_fcs = 1;
|
| 3136 |
|
| 3137 | if (vnet_hdr_sz) {
|
| 3138 | err = virtio_net_hdr_to_skb(skb, &vnet_hdr, vio_le());
|
| 3139 | if (err)
|
| 3140 | goto out_free;
|
| 3141 | len += vnet_hdr_sz;
|
| 3142 | }
|
| 3143 |
|
| 3144 | packet_parse_headers(skb, sock);
|
| 3145 |
|
| 3146 | if (vnet_hdr_sz)
|
| 3147 | virtio_net_hdr_set_proto(skb, &vnet_hdr);
|
| 3148 |
|
| 3149 | err = packet_xmit(po, skb);
|
| 3150 |
|
| 3151 | if (unlikely(err != 0)) {
|
| 3152 | if (err > 0)
|
| 3153 | err = net_xmit_errno(err);
|
| 3154 | if (err)
|
| 3155 | goto out_unlock;
|
| 3156 | }
|
| 3157 |
|
| 3158 | dev_put(dev);
|
| 3159 |
|
| 3160 | return len;
|
| 3161 |
|
| 3162 | out_free:
|
| 3163 | kfree_skb(skb);
|
| 3164 | out_unlock:
|
| 3165 | dev_put(dev);
|
| 3166 | out:
|
| 3167 | return err;
|
| 3168 | }
|
| 3169 |
|
| 3170 | static int packet_sendmsg(struct socket *sock, struct msghdr *msg, size_t len)
|
| 3171 | {
|
| 3172 | struct sock *sk = sock->sk;
|
| 3173 | struct packet_sock *po = pkt_sk(sk);
|
| 3174 |
|
| 3175 | |
| 3176 | * tpacket_snd() will redo the check safely.
|
| 3177 |
|
| 3178 | if (data_race(po->tx_ring.pg_vec))
|
| 3179 | return tpacket_snd(po, msg);
|
| 3180 |
|
| 3181 | return packet_snd(sock, msg, len);
|
| 3182 | }
|
| 3183 |
|
| 3184 | |
| 3185 | * Close a PACKET socket. This is fairly simple. We immediately go
|
| 3186 | * to 'closed' state and remove our protocol entry in the device list.
|
| 3187 |
|
| 3188 |
|
| 3189 | static int packet_release(struct socket *sock)
|
| 3190 | {
|
| 3191 | struct sock *sk = sock->sk;
|
| 3192 | struct packet_sock *po;
|
| 3193 | struct packet_fanout *f;
|
| 3194 | struct net *net;
|
| 3195 | union tpacket_req_u req_u;
|
| 3196 |
|
| 3197 | if (!sk)
|
| 3198 | return 0;
|
| 3199 |
|
| 3200 | net = sock_net(sk);
|
| 3201 | po = pkt_sk(sk);
|
| 3202 |
|
| 3203 | mutex_lock(&net->packet.sklist_lock);
|
| 3204 | sk_del_node_init_rcu(sk);
|
| 3205 | mutex_unlock(&net->packet.sklist_lock);
|
| 3206 |
|
| 3207 | sock_prot_inuse_add(net, sk->sk_prot, -1);
|
| 3208 |
|
| 3209 | spin_lock(&po->bind_lock);
|
| 3210 | unregister_prot_hook(sk, false);
|
| 3211 | WRITE_ONCE(po->num, 0);
|
| 3212 | packet_cached_dev_reset(po);
|
| 3213 |
|
| 3214 | if (po->prot_hook.dev) {
|
| 3215 | netdev_put(po->prot_hook.dev, &po->prot_hook.dev_tracker);
|
| 3216 | po->prot_hook.dev = NULL;
|
| 3217 | }
|
| 3218 | spin_unlock(&po->bind_lock);
|
| 3219 |
|
| 3220 | packet_flush_mclist(sk);
|
| 3221 |
|
| 3222 | lock_sock(sk);
|
| 3223 | if (po->rx_ring.pg_vec) {
|
| 3224 | memset(&req_u, 0, sizeof(req_u));
|
| 3225 | packet_set_ring(sk, &req_u, 1, 0);
|
| 3226 | }
|
| 3227 |
|
| 3228 | if (po->tx_ring.pg_vec) {
|
| 3229 | memset(&req_u, 0, sizeof(req_u));
|
| 3230 | packet_set_ring(sk, &req_u, 1, 1);
|
| 3231 | }
|
| 3232 | release_sock(sk);
|
| 3233 |
|
| 3234 | f = fanout_release(sk);
|
| 3235 |
|
| 3236 | synchronize_net();
|
| 3237 |
|
| 3238 | kfree(po->rollover);
|
| 3239 | if (f) {
|
| 3240 | fanout_release_data(f);
|
| 3241 | kvfree(f);
|
| 3242 | }
|
| 3243 | |
| 3244 | * Now the socket is dead. No more input will appear.
|
| 3245 |
|
| 3246 | sock_orphan(sk);
|
| 3247 | sock->sk = NULL;
|
| 3248 |
|
| 3249 |
|
| 3250 |
|
| 3251 | skb_queue_purge(&sk->sk_receive_queue);
|
| 3252 |
|
| 3253 | sock_put(sk);
|
| 3254 | return 0;
|
| 3255 | }
|
| 3256 |
|
| 3257 | |
| 3258 | * Attach a packet hook.
|
| 3259 |
|
| 3260 |
|
| 3261 | static int packet_do_bind(struct sock *sk, const char *name, int ifindex,
|
| 3262 | __be16 proto)
|
| 3263 | {
|
| 3264 | struct packet_sock *po = pkt_sk(sk);
|
| 3265 | struct net_device *dev = NULL;
|
| 3266 | bool unlisted = false;
|
| 3267 | bool need_rehook;
|
| 3268 | int ret = 0;
|
| 3269 |
|
| 3270 | lock_sock(sk);
|
| 3271 | spin_lock(&po->bind_lock);
|
| 3272 | if (!proto)
|
| 3273 | proto = po->num;
|
| 3274 |
|
| 3275 | rcu_read_lock();
|
| 3276 |
|
| 3277 | if (po->fanout) {
|
| 3278 | ret = -EINVAL;
|
| 3279 | goto out_unlock;
|
| 3280 | }
|
| 3281 |
|
| 3282 | if (name) {
|
| 3283 | dev = dev_get_by_name_rcu(sock_net(sk), name);
|
| 3284 | if (!dev) {
|
| 3285 | ret = -ENODEV;
|
| 3286 | goto out_unlock;
|
| 3287 | }
|
| 3288 | } else if (ifindex) {
|
| 3289 | dev = dev_get_by_index_rcu(sock_net(sk), ifindex);
|
| 3290 | if (!dev) {
|
| 3291 | ret = -ENODEV;
|
| 3292 | goto out_unlock;
|
| 3293 | }
|
| 3294 | }
|
| 3295 |
|
| 3296 | need_rehook = po->prot_hook.type != proto || po->prot_hook.dev != dev;
|
| 3297 |
|
| 3298 | if (need_rehook) {
|
| 3299 | dev_hold(dev);
|
| 3300 | if (packet_sock_flag(po, PACKET_SOCK_RUNNING)) {
|
| 3301 | rcu_read_unlock();
|
| 3302 | |
| 3303 | * register_prot_hook()
|
| 3304 |
|
| 3305 | WRITE_ONCE(po->num, 0);
|
| 3306 | __unregister_prot_hook(sk, true);
|
| 3307 | rcu_read_lock();
|
| 3308 | if (dev)
|
| 3309 | unlisted = !dev_get_by_index_rcu(sock_net(sk),
|
| 3310 | dev->ifindex);
|
| 3311 | }
|
| 3312 |
|
| 3313 | BUG_ON(packet_sock_flag(po, PACKET_SOCK_RUNNING));
|
| 3314 | WRITE_ONCE(po->num, proto);
|
| 3315 | po->prot_hook.type = proto;
|
| 3316 |
|
| 3317 | netdev_put(po->prot_hook.dev, &po->prot_hook.dev_tracker);
|
| 3318 |
|
| 3319 | if (unlikely(unlisted)) {
|
| 3320 | po->prot_hook.dev = NULL;
|
| 3321 | WRITE_ONCE(po->ifindex, -1);
|
| 3322 | packet_cached_dev_reset(po);
|
| 3323 | } else {
|
| 3324 | netdev_hold(dev, &po->prot_hook.dev_tracker,
|
| 3325 | GFP_ATOMIC);
|
| 3326 | po->prot_hook.dev = dev;
|
| 3327 | WRITE_ONCE(po->ifindex, dev ? dev->ifindex : 0);
|
| 3328 | packet_cached_dev_assign(po, dev);
|
| 3329 | }
|
| 3330 | dev_put(dev);
|
| 3331 | }
|
| 3332 |
|
| 3333 | if (proto == 0 || !need_rehook)
|
| 3334 | goto out_unlock;
|
| 3335 |
|
| 3336 | if (!unlisted && (!dev || (dev->flags & IFF_UP))) {
|
| 3337 | register_prot_hook(sk);
|
| 3338 | } else {
|
| 3339 | sk->sk_err = ENETDOWN;
|
| 3340 | if (!sock_flag(sk, SOCK_DEAD))
|
| 3341 | sk_error_report(sk);
|
| 3342 | }
|
| 3343 |
|
| 3344 | out_unlock:
|
| 3345 | rcu_read_unlock();
|
| 3346 | spin_unlock(&po->bind_lock);
|
| 3347 | release_sock(sk);
|
| 3348 | return ret;
|
| 3349 | }
|
| 3350 |
|
| 3351 | |
| 3352 | * Bind a packet socket to a device
|
| 3353 |
|
| 3354 |
|
| 3355 | static int packet_bind_spkt(struct socket *sock, struct sockaddr_unsized *uaddr,
|
| 3356 | int addr_len)
|
| 3357 | {
|
| 3358 | struct sock *sk = sock->sk;
|
| 3359 | struct sockaddr *sa = (struct sockaddr *)uaddr;
|
| 3360 | char name[sizeof(sa->sa_data) + 1];
|
| 3361 |
|
| 3362 | |
| 3363 | * Check legality
|
| 3364 |
|
| 3365 |
|
| 3366 | if (addr_len != sizeof(struct sockaddr))
|
| 3367 | return -EINVAL;
|
| 3368 | |
| 3369 | * zero-terminated.
|
| 3370 |
|
| 3371 | memcpy(name, sa->sa_data, sizeof(sa->sa_data));
|
| 3372 | name[sizeof(sa->sa_data)] = 0;
|
| 3373 |
|
| 3374 | return packet_do_bind(sk, name, 0, 0);
|
| 3375 | }
|
| 3376 |
|
| 3377 | static int packet_bind(struct socket *sock, struct sockaddr_unsized *uaddr, int addr_len)
|
| 3378 | {
|
| 3379 | struct sockaddr_ll *sll = (struct sockaddr_ll *)uaddr;
|
| 3380 | struct sock *sk = sock->sk;
|
| 3381 |
|
| 3382 | |
| 3383 | * Check legality
|
| 3384 |
|
| 3385 |
|
| 3386 | if (addr_len < sizeof(struct sockaddr_ll))
|
| 3387 | return -EINVAL;
|
| 3388 | if (sll->sll_family != AF_PACKET)
|
| 3389 | return -EINVAL;
|
| 3390 |
|
| 3391 | return packet_do_bind(sk, NULL, sll->sll_ifindex, sll->sll_protocol);
|
| 3392 | }
|
| 3393 |
|
| 3394 | static struct proto packet_proto = {
|
| 3395 | .name = "PACKET",
|
| 3396 | .owner = THIS_MODULE,
|
| 3397 | .obj_size = sizeof(struct packet_sock),
|
| 3398 | };
|
| 3399 |
|
| 3400 | |
| 3401 | * Create a packet of type SOCK_PACKET.
|
| 3402 |
|
| 3403 |
|
| 3404 | static int packet_create(struct net *net, struct socket *sock, int protocol,
|
| 3405 | int kern)
|
| 3406 | {
|
| 3407 | struct sock *sk;
|
| 3408 | struct packet_sock *po;
|
| 3409 | __be16 proto = (__force __be16)protocol;
|
| 3410 | int err;
|
| 3411 |
|
| 3412 | if (!ns_capable(net->user_ns, CAP_NET_RAW))
|
| 3413 | return -EPERM;
|
| 3414 | if (sock->type != SOCK_DGRAM && sock->type != SOCK_RAW &&
|
| 3415 | sock->type != SOCK_PACKET)
|
| 3416 | return -ESOCKTNOSUPPORT;
|
| 3417 |
|
| 3418 | sock->state = SS_UNCONNECTED;
|
| 3419 |
|
| 3420 | err = -ENOBUFS;
|
| 3421 | sk = sk_alloc(net, PF_PACKET, GFP_KERNEL, &packet_proto, kern);
|
| 3422 | if (sk == NULL)
|
| 3423 | goto out;
|
| 3424 |
|
| 3425 | sock->ops = &packet_ops;
|
| 3426 | if (sock->type == SOCK_PACKET)
|
| 3427 | sock->ops = &packet_ops_spkt;
|
| 3428 |
|
| 3429 | po = pkt_sk(sk);
|
| 3430 | err = packet_alloc_pending(po);
|
| 3431 | if (err)
|
| 3432 | goto out_sk_free;
|
| 3433 |
|
| 3434 | sock_init_data(sock, sk);
|
| 3435 |
|
| 3436 | init_completion(&po->skb_completion);
|
| 3437 | sk->sk_family = PF_PACKET;
|
| 3438 | po->num = proto;
|
| 3439 |
|
| 3440 | packet_cached_dev_reset(po);
|
| 3441 |
|
| 3442 | sk->sk_destruct = packet_sock_destruct;
|
| 3443 |
|
| 3444 | |
| 3445 | * Attach a protocol block
|
| 3446 |
|
| 3447 |
|
| 3448 | spin_lock_init(&po->bind_lock);
|
| 3449 | mutex_init(&po->pg_vec_lock);
|
| 3450 | po->rollover = NULL;
|
| 3451 | po->prot_hook.func = packet_rcv;
|
| 3452 |
|
| 3453 | if (sock->type == SOCK_PACKET)
|
| 3454 | po->prot_hook.func = packet_rcv_spkt;
|
| 3455 |
|
| 3456 | po->prot_hook.af_packet_priv = sk;
|
| 3457 | po->prot_hook.af_packet_net = sock_net(sk);
|
| 3458 |
|
| 3459 | if (proto) {
|
| 3460 | po->prot_hook.type = proto;
|
| 3461 | __register_prot_hook(sk);
|
| 3462 | }
|
| 3463 |
|
| 3464 | mutex_lock(&net->packet.sklist_lock);
|
| 3465 | sk_add_node_tail_rcu(sk, &net->packet.sklist);
|
| 3466 | mutex_unlock(&net->packet.sklist_lock);
|
| 3467 |
|
| 3468 | sock_prot_inuse_add(net, &packet_proto, 1);
|
| 3469 |
|
| 3470 | return 0;
|
| 3471 | out_sk_free:
|
| 3472 | sk_free(sk);
|
| 3473 | out:
|
| 3474 | return err;
|
| 3475 | }
|
| 3476 |
|
| 3477 | |
| 3478 | * Pull a packet from our receive queue and hand it to the user.
|
| 3479 | * If necessary we block.
|
| 3480 |
|
| 3481 |
|
| 3482 | static int packet_recvmsg(struct socket *sock, struct msghdr *msg, size_t len,
|
| 3483 | int flags)
|
| 3484 | {
|
| 3485 | struct sock *sk = sock->sk;
|
| 3486 | struct sk_buff *skb;
|
| 3487 | int copied, err;
|
| 3488 | int vnet_hdr_len = READ_ONCE(pkt_sk(sk)->vnet_hdr_sz);
|
| 3489 | unsigned int origlen = 0;
|
| 3490 |
|
| 3491 | err = -EINVAL;
|
| 3492 | if (flags & ~(MSG_PEEK|MSG_DONTWAIT|MSG_TRUNC|MSG_CMSG_COMPAT|MSG_ERRQUEUE))
|
| 3493 | goto out;
|
| 3494 |
|
| 3495 | #if 0 |
| 3496 | /* What error should we return now? EUNATTACH? */
|
| 3497 | if (pkt_sk(sk)->ifindex < 0)
|
| 3498 | return -ENODEV;
|
| 3499 | #endif
|
| 3500 |
|
| 3501 | if (flags & MSG_ERRQUEUE) {
|
| 3502 | err = sock_recv_errqueue(sk, msg, len,
|
| 3503 | SOL_PACKET, PACKET_TX_TIMESTAMP);
|
| 3504 | goto out;
|
| 3505 | }
|
| 3506 |
|
| 3507 | |
| 3508 | * Call the generic datagram receiver. This handles all sorts
|
| 3509 | * of horrible races and re-entrancy so we can forget about it
|
| 3510 | * in the protocol layers.
|
| 3511 | *
|
| 3512 | * Now it will return ENETDOWN, if device have just gone down,
|
| 3513 | * but then it will block.
|
| 3514 |
|
| 3515 |
|
| 3516 | skb = skb_recv_datagram(sk, flags, &err);
|
| 3517 |
|
| 3518 | |
| 3519 | * An error occurred so return it. Because skb_recv_datagram()
|
| 3520 | * handles the blocking we don't see and worry about blocking
|
| 3521 | * retries.
|
| 3522 |
|
| 3523 |
|
| 3524 | if (skb == NULL)
|
| 3525 | goto out;
|
| 3526 |
|
| 3527 | packet_rcv_try_clear_pressure(pkt_sk(sk));
|
| 3528 |
|
| 3529 | if (vnet_hdr_len) {
|
| 3530 | err = packet_rcv_vnet(msg, skb, &len, vnet_hdr_len);
|
| 3531 | if (err)
|
| 3532 | goto out_free;
|
| 3533 | }
|
| 3534 |
|
| 3535 | |
| 3536 | * a user program they can ask the device for its MTU
|
| 3537 | * anyway.
|
| 3538 |
|
| 3539 | copied = skb->len;
|
| 3540 | if (copied > len) {
|
| 3541 | copied = len;
|
| 3542 | msg->msg_flags |= MSG_TRUNC;
|
| 3543 | }
|
| 3544 |
|
| 3545 | err = skb_copy_datagram_msg(skb, 0, msg, copied);
|
| 3546 | if (err)
|
| 3547 | goto out_free;
|
| 3548 |
|
| 3549 | if (sock->type != SOCK_PACKET) {
|
| 3550 | struct sockaddr_ll *sll = &PACKET_SKB_CB(skb)->sa.ll;
|
| 3551 |
|
| 3552 |
|
| 3553 | origlen = PACKET_SKB_CB(skb)->sa.origlen;
|
| 3554 | sll->sll_family = AF_PACKET;
|
| 3555 | sll->sll_protocol = (sock->type == SOCK_DGRAM) ?
|
| 3556 | vlan_get_protocol_dgram(skb) : skb->protocol;
|
| 3557 | }
|
| 3558 |
|
| 3559 | sock_recv_cmsgs(msg, sk, skb);
|
| 3560 |
|
| 3561 | if (msg->msg_name) {
|
| 3562 | const size_t max_len = min(sizeof(skb->cb),
|
| 3563 | sizeof(struct sockaddr_storage));
|
| 3564 | int copy_len;
|
| 3565 |
|
| 3566 | |
| 3567 | * in, we fill it in now.
|
| 3568 |
|
| 3569 | if (sock->type == SOCK_PACKET) {
|
| 3570 | __sockaddr_check_size(sizeof(struct sockaddr_pkt));
|
| 3571 | msg->msg_namelen = sizeof(struct sockaddr_pkt);
|
| 3572 | copy_len = msg->msg_namelen;
|
| 3573 | } else {
|
| 3574 | struct sockaddr_ll *sll = &PACKET_SKB_CB(skb)->sa.ll;
|
| 3575 |
|
| 3576 | msg->msg_namelen = sll->sll_halen +
|
| 3577 | offsetof(struct sockaddr_ll, sll_addr);
|
| 3578 | copy_len = msg->msg_namelen;
|
| 3579 | if (msg->msg_namelen < sizeof(struct sockaddr_ll)) {
|
| 3580 | memset(msg->msg_name +
|
| 3581 | offsetof(struct sockaddr_ll, sll_addr),
|
| 3582 | 0, sizeof(sll->sll_addr));
|
| 3583 | msg->msg_namelen = sizeof(struct sockaddr_ll);
|
| 3584 | }
|
| 3585 | }
|
| 3586 | if (WARN_ON_ONCE(copy_len > max_len)) {
|
| 3587 | copy_len = max_len;
|
| 3588 | msg->msg_namelen = copy_len;
|
| 3589 | }
|
| 3590 | memcpy(msg->msg_name, &PACKET_SKB_CB(skb)->sa, copy_len);
|
| 3591 | }
|
| 3592 |
|
| 3593 | if (packet_sock_flag(pkt_sk(sk), PACKET_SOCK_AUXDATA)) {
|
| 3594 | struct tpacket_auxdata aux;
|
| 3595 |
|
| 3596 | aux.tp_status = TP_STATUS_USER;
|
| 3597 | if (skb->ip_summed == CHECKSUM_PARTIAL)
|
| 3598 | aux.tp_status |= TP_STATUS_CSUMNOTREADY;
|
| 3599 | else if (skb->pkt_type != PACKET_OUTGOING &&
|
| 3600 | skb_csum_unnecessary(skb))
|
| 3601 | aux.tp_status |= TP_STATUS_CSUM_VALID;
|
| 3602 | if (skb_is_gso(skb) && skb_is_gso_tcp(skb))
|
| 3603 | aux.tp_status |= TP_STATUS_GSO_TCP;
|
| 3604 |
|
| 3605 | aux.tp_len = origlen;
|
| 3606 | aux.tp_snaplen = skb->len;
|
| 3607 | aux.tp_mac = 0;
|
| 3608 | aux.tp_net = skb_network_offset(skb);
|
| 3609 | if (skb_vlan_tag_present(skb)) {
|
| 3610 | aux.tp_vlan_tci = skb_vlan_tag_get(skb);
|
| 3611 | aux.tp_vlan_tpid = ntohs(skb->vlan_proto);
|
| 3612 | aux.tp_status |= TP_STATUS_VLAN_VALID | TP_STATUS_VLAN_TPID_VALID;
|
| 3613 | } else if (unlikely(sock->type == SOCK_DGRAM && eth_type_vlan(skb->protocol))) {
|
| 3614 | struct sockaddr_ll *sll = &PACKET_SKB_CB(skb)->sa.ll;
|
| 3615 | struct net_device *dev;
|
| 3616 |
|
| 3617 | rcu_read_lock();
|
| 3618 | dev = dev_get_by_index_rcu(sock_net(sk), sll->sll_ifindex);
|
| 3619 | if (dev) {
|
| 3620 | aux.tp_vlan_tci = vlan_get_tci(skb, dev);
|
| 3621 | aux.tp_vlan_tpid = ntohs(skb->protocol);
|
| 3622 | aux.tp_status |= TP_STATUS_VLAN_VALID | TP_STATUS_VLAN_TPID_VALID;
|
| 3623 | } else {
|
| 3624 | aux.tp_vlan_tci = 0;
|
| 3625 | aux.tp_vlan_tpid = 0;
|
| 3626 | }
|
| 3627 | rcu_read_unlock();
|
| 3628 | } else {
|
| 3629 | aux.tp_vlan_tci = 0;
|
| 3630 | aux.tp_vlan_tpid = 0;
|
| 3631 | }
|
| 3632 | put_cmsg(msg, SOL_PACKET, PACKET_AUXDATA, sizeof(aux), &aux);
|
| 3633 | }
|
| 3634 |
|
| 3635 | |
| 3636 | * Free or return the buffer as appropriate. Again this
|
| 3637 | * hides all the races and re-entrancy issues from us.
|
| 3638 |
|
| 3639 | err = vnet_hdr_len + ((flags&MSG_TRUNC) ? skb->len : copied);
|
| 3640 |
|
| 3641 | out_free:
|
| 3642 | skb_free_datagram(sk, skb);
|
| 3643 | out:
|
| 3644 | return err;
|
| 3645 | }
|
| 3646 |
|
| 3647 | static int packet_getname_spkt(struct socket *sock, struct sockaddr *uaddr,
|
| 3648 | int peer)
|
| 3649 | {
|
| 3650 | struct net_device *dev;
|
| 3651 | struct sock *sk = sock->sk;
|
| 3652 |
|
| 3653 | if (peer)
|
| 3654 | return -EOPNOTSUPP;
|
| 3655 |
|
| 3656 | uaddr->sa_family = AF_PACKET;
|
| 3657 | memset(uaddr->sa_data, 0, sizeof(uaddr->sa_data));
|
| 3658 | rcu_read_lock();
|
| 3659 | dev = dev_get_by_index_rcu(sock_net(sk), READ_ONCE(pkt_sk(sk)->ifindex));
|
| 3660 | if (dev)
|
| 3661 | strscpy(uaddr->sa_data, dev->name, sizeof(uaddr->sa_data));
|
| 3662 | rcu_read_unlock();
|
| 3663 |
|
| 3664 | return sizeof(*uaddr);
|
| 3665 | }
|
| 3666 |
|
| 3667 | static int packet_getname(struct socket *sock, struct sockaddr *uaddr,
|
| 3668 | int peer)
|
| 3669 | {
|
| 3670 | struct net_device *dev;
|
| 3671 | struct sock *sk = sock->sk;
|
| 3672 | struct packet_sock *po = pkt_sk(sk);
|
| 3673 | DECLARE_SOCKADDR(struct sockaddr_ll *, sll, uaddr);
|
| 3674 | int ifindex;
|
| 3675 |
|
| 3676 | if (peer)
|
| 3677 | return -EOPNOTSUPP;
|
| 3678 |
|
| 3679 | ifindex = READ_ONCE(po->ifindex);
|
| 3680 | sll->sll_family = AF_PACKET;
|
| 3681 | sll->sll_ifindex = ifindex;
|
| 3682 | sll->sll_protocol = READ_ONCE(po->num);
|
| 3683 | sll->sll_pkttype = 0;
|
| 3684 | rcu_read_lock();
|
| 3685 | dev = dev_get_by_index_rcu(sock_net(sk), ifindex);
|
| 3686 | if (dev) {
|
| 3687 | sll->sll_hatype = dev->type;
|
| 3688 | sll->sll_halen = dev->addr_len;
|
| 3689 |
|
| 3690 |
|
| 3691 | memcpy(((struct sockaddr_storage *)sll)->__data +
|
| 3692 | offsetof(struct sockaddr_ll, sll_addr) -
|
| 3693 | offsetofend(struct sockaddr_ll, sll_family),
|
| 3694 | dev->dev_addr, dev->addr_len);
|
| 3695 | } else {
|
| 3696 | sll->sll_hatype = 0;
|
| 3697 | sll->sll_halen = 0;
|
| 3698 | }
|
| 3699 | rcu_read_unlock();
|
| 3700 |
|
| 3701 | return offsetof(struct sockaddr_ll, sll_addr) + sll->sll_halen;
|
| 3702 | }
|
| 3703 |
|
| 3704 | static int packet_dev_mc(struct net_device *dev, struct packet_mclist *i,
|
| 3705 | int what)
|
| 3706 | {
|
| 3707 | switch (i->type) {
|
| 3708 | case PACKET_MR_MULTICAST:
|
| 3709 | if (i->alen != dev->addr_len)
|
| 3710 | return -EINVAL;
|
| 3711 | if (what > 0)
|
| 3712 | return dev_mc_add(dev, i->addr);
|
| 3713 | else
|
| 3714 | return dev_mc_del(dev, i->addr);
|
| 3715 | break;
|
| 3716 | case PACKET_MR_PROMISC:
|
| 3717 | return dev_set_promiscuity(dev, what);
|
| 3718 | case PACKET_MR_ALLMULTI:
|
| 3719 | return dev_set_allmulti(dev, what);
|
| 3720 | case PACKET_MR_UNICAST:
|
| 3721 | if (i->alen != dev->addr_len)
|
| 3722 | return -EINVAL;
|
| 3723 | if (what > 0)
|
| 3724 | return dev_uc_add(dev, i->addr);
|
| 3725 | else
|
| 3726 | return dev_uc_del(dev, i->addr);
|
| 3727 | break;
|
| 3728 | default:
|
| 3729 | break;
|
| 3730 | }
|
| 3731 | return 0;
|
| 3732 | }
|
| 3733 |
|
| 3734 | static void packet_dev_mclist_delete(struct net_device *dev,
|
| 3735 | struct packet_mclist **mlp,
|
| 3736 | struct list_head *list)
|
| 3737 | {
|
| 3738 | struct packet_mclist *ml;
|
| 3739 |
|
| 3740 | while ((ml = *mlp) != NULL) {
|
| 3741 | if (ml->ifindex == dev->ifindex) {
|
| 3742 | list_add(&ml->remove_list, list);
|
| 3743 | *mlp = ml->next;
|
| 3744 | } else
|
| 3745 | mlp = &ml->next;
|
| 3746 | }
|
| 3747 | }
|
| 3748 |
|
| 3749 | static int packet_mc_add(struct sock *sk, struct packet_mreq_max *mreq)
|
| 3750 | {
|
| 3751 | struct packet_sock *po = pkt_sk(sk);
|
| 3752 | struct packet_mclist *ml, *i;
|
| 3753 | struct net_device *dev;
|
| 3754 | int err;
|
| 3755 |
|
| 3756 | rtnl_lock();
|
| 3757 |
|
| 3758 | err = -ENODEV;
|
| 3759 | dev = __dev_get_by_index(sock_net(sk), mreq->mr_ifindex);
|
| 3760 | if (!dev)
|
| 3761 | goto done;
|
| 3762 |
|
| 3763 | err = -EINVAL;
|
| 3764 | if (mreq->mr_alen > dev->addr_len)
|
| 3765 | goto done;
|
| 3766 |
|
| 3767 | err = -ENOBUFS;
|
| 3768 | i = kmalloc_obj(*i);
|
| 3769 | if (i == NULL)
|
| 3770 | goto done;
|
| 3771 |
|
| 3772 | err = 0;
|
| 3773 | for (ml = po->mclist; ml; ml = ml->next) {
|
| 3774 | if (ml->ifindex == mreq->mr_ifindex &&
|
| 3775 | ml->type == mreq->mr_type &&
|
| 3776 | ml->alen == mreq->mr_alen &&
|
| 3777 | memcmp(ml->addr, mreq->mr_address, ml->alen) == 0) {
|
| 3778 | ml->count++;
|
| 3779 |
|
| 3780 | kfree(i);
|
| 3781 | goto done;
|
| 3782 | }
|
| 3783 | }
|
| 3784 |
|
| 3785 | i->type = mreq->mr_type;
|
| 3786 | i->ifindex = mreq->mr_ifindex;
|
| 3787 | i->alen = mreq->mr_alen;
|
| 3788 | memcpy(i->addr, mreq->mr_address, i->alen);
|
| 3789 | memset(i->addr + i->alen, 0, sizeof(i->addr) - i->alen);
|
| 3790 | i->count = 1;
|
| 3791 | INIT_LIST_HEAD(&i->remove_list);
|
| 3792 | i->next = po->mclist;
|
| 3793 | po->mclist = i;
|
| 3794 | err = packet_dev_mc(dev, i, 1);
|
| 3795 | if (err) {
|
| 3796 | po->mclist = i->next;
|
| 3797 | kfree(i);
|
| 3798 | }
|
| 3799 |
|
| 3800 | done:
|
| 3801 | rtnl_unlock();
|
| 3802 | return err;
|
| 3803 | }
|
| 3804 |
|
| 3805 | static int packet_mc_drop(struct sock *sk, struct packet_mreq_max *mreq)
|
| 3806 | {
|
| 3807 | struct packet_mclist *ml, **mlp;
|
| 3808 |
|
| 3809 | rtnl_lock();
|
| 3810 |
|
| 3811 | for (mlp = &pkt_sk(sk)->mclist; (ml = *mlp) != NULL; mlp = &ml->next) {
|
| 3812 | if (ml->ifindex == mreq->mr_ifindex &&
|
| 3813 | ml->type == mreq->mr_type &&
|
| 3814 | ml->alen == mreq->mr_alen &&
|
| 3815 | memcmp(ml->addr, mreq->mr_address, ml->alen) == 0) {
|
| 3816 | if (--ml->count == 0) {
|
| 3817 | struct net_device *dev;
|
| 3818 | *mlp = ml->next;
|
| 3819 | dev = __dev_get_by_index(sock_net(sk), ml->ifindex);
|
| 3820 | if (dev)
|
| 3821 | packet_dev_mc(dev, ml, -1);
|
| 3822 | kfree(ml);
|
| 3823 | }
|
| 3824 | break;
|
| 3825 | }
|
| 3826 | }
|
| 3827 | rtnl_unlock();
|
| 3828 | return 0;
|
| 3829 | }
|
| 3830 |
|
| 3831 | static void packet_flush_mclist(struct sock *sk)
|
| 3832 | {
|
| 3833 | struct packet_sock *po = pkt_sk(sk);
|
| 3834 | struct packet_mclist *ml;
|
| 3835 |
|
| 3836 | if (!po->mclist)
|
| 3837 | return;
|
| 3838 |
|
| 3839 | rtnl_lock();
|
| 3840 | while ((ml = po->mclist) != NULL) {
|
| 3841 | struct net_device *dev;
|
| 3842 |
|
| 3843 | po->mclist = ml->next;
|
| 3844 | dev = __dev_get_by_index(sock_net(sk), ml->ifindex);
|
| 3845 | if (dev != NULL)
|
| 3846 | packet_dev_mc(dev, ml, -1);
|
| 3847 | kfree(ml);
|
| 3848 | }
|
| 3849 | rtnl_unlock();
|
| 3850 | }
|
| 3851 |
|
| 3852 | static int
|
| 3853 | packet_setsockopt(struct socket *sock, int level, int optname, sockptr_t optval,
|
| 3854 | unsigned int optlen)
|
| 3855 | {
|
| 3856 | struct sock *sk = sock->sk;
|
| 3857 | struct packet_sock *po = pkt_sk(sk);
|
| 3858 | int ret;
|
| 3859 |
|
| 3860 | if (level != SOL_PACKET)
|
| 3861 | return -ENOPROTOOPT;
|
| 3862 |
|
| 3863 | switch (optname) {
|
| 3864 | case PACKET_ADD_MEMBERSHIP:
|
| 3865 | case PACKET_DROP_MEMBERSHIP:
|
| 3866 | {
|
| 3867 | struct packet_mreq_max mreq;
|
| 3868 | int len = optlen;
|
| 3869 | memset(&mreq, 0, sizeof(mreq));
|
| 3870 | if (len < sizeof(struct packet_mreq))
|
| 3871 | return -EINVAL;
|
| 3872 | if (len > sizeof(mreq))
|
| 3873 | len = sizeof(mreq);
|
| 3874 | if (copy_from_sockptr(&mreq, optval, len))
|
| 3875 | return -EFAULT;
|
| 3876 | if (len < (mreq.mr_alen + offsetof(struct packet_mreq, mr_address)))
|
| 3877 | return -EINVAL;
|
| 3878 | if (optname == PACKET_ADD_MEMBERSHIP)
|
| 3879 | ret = packet_mc_add(sk, &mreq);
|
| 3880 | else
|
| 3881 | ret = packet_mc_drop(sk, &mreq);
|
| 3882 | return ret;
|
| 3883 | }
|
| 3884 |
|
| 3885 | case PACKET_RX_RING:
|
| 3886 | case PACKET_TX_RING:
|
| 3887 | {
|
| 3888 | union tpacket_req_u req_u;
|
| 3889 |
|
| 3890 | ret = -EINVAL;
|
| 3891 | lock_sock(sk);
|
| 3892 | switch (po->tp_version) {
|
| 3893 | case TPACKET_V1:
|
| 3894 | case TPACKET_V2:
|
| 3895 | if (optlen < sizeof(req_u.req))
|
| 3896 | break;
|
| 3897 | ret = copy_from_sockptr(&req_u.req, optval,
|
| 3898 | sizeof(req_u.req)) ?
|
| 3899 | -EINVAL : 0;
|
| 3900 | break;
|
| 3901 | case TPACKET_V3:
|
| 3902 | default:
|
| 3903 | if (optlen < sizeof(req_u.req3))
|
| 3904 | break;
|
| 3905 | ret = copy_from_sockptr(&req_u.req3, optval,
|
| 3906 | sizeof(req_u.req3)) ?
|
| 3907 | -EINVAL : 0;
|
| 3908 | break;
|
| 3909 | }
|
| 3910 | if (!ret)
|
| 3911 | ret = packet_set_ring(sk, &req_u, 0,
|
| 3912 | optname == PACKET_TX_RING);
|
| 3913 | release_sock(sk);
|
| 3914 | return ret;
|
| 3915 | }
|
| 3916 | case PACKET_COPY_THRESH:
|
| 3917 | {
|
| 3918 | int val;
|
| 3919 |
|
| 3920 | if (optlen != sizeof(val))
|
| 3921 | return -EINVAL;
|
| 3922 | if (copy_from_sockptr(&val, optval, sizeof(val)))
|
| 3923 | return -EFAULT;
|
| 3924 |
|
| 3925 | WRITE_ONCE(pkt_sk(sk)->copy_thresh, val);
|
| 3926 | return 0;
|
| 3927 | }
|
| 3928 | case PACKET_VERSION:
|
| 3929 | {
|
| 3930 | int val;
|
| 3931 |
|
| 3932 | if (optlen != sizeof(val))
|
| 3933 | return -EINVAL;
|
| 3934 | if (copy_from_sockptr(&val, optval, sizeof(val)))
|
| 3935 | return -EFAULT;
|
| 3936 | switch (val) {
|
| 3937 | case TPACKET_V1:
|
| 3938 | case TPACKET_V2:
|
| 3939 | case TPACKET_V3:
|
| 3940 | break;
|
| 3941 | default:
|
| 3942 | return -EINVAL;
|
| 3943 | }
|
| 3944 | lock_sock(sk);
|
| 3945 | if (po->rx_ring.pg_vec || po->tx_ring.pg_vec) {
|
| 3946 | ret = -EBUSY;
|
| 3947 | } else {
|
| 3948 | po->tp_version = val;
|
| 3949 | ret = 0;
|
| 3950 | }
|
| 3951 | release_sock(sk);
|
| 3952 | return ret;
|
| 3953 | }
|
| 3954 | case PACKET_RESERVE:
|
| 3955 | {
|
| 3956 | unsigned int val;
|
| 3957 |
|
| 3958 | if (optlen != sizeof(val))
|
| 3959 | return -EINVAL;
|
| 3960 | if (copy_from_sockptr(&val, optval, sizeof(val)))
|
| 3961 | return -EFAULT;
|
| 3962 | if (val > INT_MAX)
|
| 3963 | return -EINVAL;
|
| 3964 | lock_sock(sk);
|
| 3965 | if (po->rx_ring.pg_vec || po->tx_ring.pg_vec) {
|
| 3966 | ret = -EBUSY;
|
| 3967 | } else {
|
| 3968 | po->tp_reserve = val;
|
| 3969 | ret = 0;
|
| 3970 | }
|
| 3971 | release_sock(sk);
|
| 3972 | return ret;
|
| 3973 | }
|
| 3974 | case PACKET_LOSS:
|
| 3975 | {
|
| 3976 | unsigned int val;
|
| 3977 |
|
| 3978 | if (optlen != sizeof(val))
|
| 3979 | return -EINVAL;
|
| 3980 | if (copy_from_sockptr(&val, optval, sizeof(val)))
|
| 3981 | return -EFAULT;
|
| 3982 |
|
| 3983 | lock_sock(sk);
|
| 3984 | if (po->rx_ring.pg_vec || po->tx_ring.pg_vec) {
|
| 3985 | ret = -EBUSY;
|
| 3986 | } else {
|
| 3987 | packet_sock_flag_set(po, PACKET_SOCK_TP_LOSS, val);
|
| 3988 | ret = 0;
|
| 3989 | }
|
| 3990 | release_sock(sk);
|
| 3991 | return ret;
|
| 3992 | }
|
| 3993 | case PACKET_AUXDATA:
|
| 3994 | {
|
| 3995 | int val;
|
| 3996 |
|
| 3997 | if (optlen < sizeof(val))
|
| 3998 | return -EINVAL;
|
| 3999 | if (copy_from_sockptr(&val, optval, sizeof(val)))
|
| 4000 | return -EFAULT;
|
| 4001 |
|
| 4002 | packet_sock_flag_set(po, PACKET_SOCK_AUXDATA, val);
|
| 4003 | return 0;
|
| 4004 | }
|
| 4005 | case PACKET_ORIGDEV:
|
| 4006 | {
|
| 4007 | int val;
|
| 4008 |
|
| 4009 | if (optlen < sizeof(val))
|
| 4010 | return -EINVAL;
|
| 4011 | if (copy_from_sockptr(&val, optval, sizeof(val)))
|
| 4012 | return -EFAULT;
|
| 4013 |
|
| 4014 | packet_sock_flag_set(po, PACKET_SOCK_ORIGDEV, val);
|
| 4015 | return 0;
|
| 4016 | }
|
| 4017 | case PACKET_VNET_HDR:
|
| 4018 | case PACKET_VNET_HDR_SZ:
|
| 4019 | {
|
| 4020 | int val, hdr_len;
|
| 4021 |
|
| 4022 | if (sock->type != SOCK_RAW)
|
| 4023 | return -EINVAL;
|
| 4024 | if (optlen < sizeof(val))
|
| 4025 | return -EINVAL;
|
| 4026 | if (copy_from_sockptr(&val, optval, sizeof(val)))
|
| 4027 | return -EFAULT;
|
| 4028 |
|
| 4029 | if (optname == PACKET_VNET_HDR_SZ) {
|
| 4030 | if (val && val != sizeof(struct virtio_net_hdr) &&
|
| 4031 | val != sizeof(struct virtio_net_hdr_mrg_rxbuf))
|
| 4032 | return -EINVAL;
|
| 4033 | hdr_len = val;
|
| 4034 | } else {
|
| 4035 | hdr_len = val ? sizeof(struct virtio_net_hdr) : 0;
|
| 4036 | }
|
| 4037 | lock_sock(sk);
|
| 4038 | if (po->rx_ring.pg_vec || po->tx_ring.pg_vec) {
|
| 4039 | ret = -EBUSY;
|
| 4040 | } else {
|
| 4041 | WRITE_ONCE(po->vnet_hdr_sz, hdr_len);
|
| 4042 | ret = 0;
|
| 4043 | }
|
| 4044 | release_sock(sk);
|
| 4045 | return ret;
|
| 4046 | }
|
| 4047 | case PACKET_TIMESTAMP:
|
| 4048 | {
|
| 4049 | int val;
|
| 4050 |
|
| 4051 | if (optlen != sizeof(val))
|
| 4052 | return -EINVAL;
|
| 4053 | if (copy_from_sockptr(&val, optval, sizeof(val)))
|
| 4054 | return -EFAULT;
|
| 4055 |
|
| 4056 | WRITE_ONCE(po->tp_tstamp, val);
|
| 4057 | return 0;
|
| 4058 | }
|
| 4059 | case PACKET_FANOUT:
|
| 4060 | {
|
| 4061 | struct fanout_args args = { 0 };
|
| 4062 |
|
| 4063 | if (optlen != sizeof(int) && optlen != sizeof(args))
|
| 4064 | return -EINVAL;
|
| 4065 | if (copy_from_sockptr(&args, optval, optlen))
|
| 4066 | return -EFAULT;
|
| 4067 |
|
| 4068 | return fanout_add(sk, &args);
|
| 4069 | }
|
| 4070 | case PACKET_FANOUT_DATA:
|
| 4071 | {
|
| 4072 |
|
| 4073 | if (!READ_ONCE(po->fanout))
|
| 4074 | return -EINVAL;
|
| 4075 |
|
| 4076 | return fanout_set_data(po, optval, optlen);
|
| 4077 | }
|
| 4078 | case PACKET_IGNORE_OUTGOING:
|
| 4079 | {
|
| 4080 | int val;
|
| 4081 |
|
| 4082 | if (optlen != sizeof(val))
|
| 4083 | return -EINVAL;
|
| 4084 | if (copy_from_sockptr(&val, optval, sizeof(val)))
|
| 4085 | return -EFAULT;
|
| 4086 | if (val < 0 || val > 1)
|
| 4087 | return -EINVAL;
|
| 4088 |
|
| 4089 | WRITE_ONCE(po->prot_hook.ignore_outgoing, !!val);
|
| 4090 | return 0;
|
| 4091 | }
|
| 4092 | case PACKET_TX_HAS_OFF:
|
| 4093 | {
|
| 4094 | unsigned int val;
|
| 4095 |
|
| 4096 | if (optlen != sizeof(val))
|
| 4097 | return -EINVAL;
|
| 4098 | if (copy_from_sockptr(&val, optval, sizeof(val)))
|
| 4099 | return -EFAULT;
|
| 4100 |
|
| 4101 | lock_sock(sk);
|
| 4102 | if (!po->rx_ring.pg_vec && !po->tx_ring.pg_vec)
|
| 4103 | packet_sock_flag_set(po, PACKET_SOCK_TX_HAS_OFF, val);
|
| 4104 |
|
| 4105 | release_sock(sk);
|
| 4106 | return 0;
|
| 4107 | }
|
| 4108 | case PACKET_QDISC_BYPASS:
|
| 4109 | {
|
| 4110 | int val;
|
| 4111 |
|
| 4112 | if (optlen != sizeof(val))
|
| 4113 | return -EINVAL;
|
| 4114 | if (copy_from_sockptr(&val, optval, sizeof(val)))
|
| 4115 | return -EFAULT;
|
| 4116 |
|
| 4117 | packet_sock_flag_set(po, PACKET_SOCK_QDISC_BYPASS, val);
|
| 4118 | return 0;
|
| 4119 | }
|
| 4120 | default:
|
| 4121 | return -ENOPROTOOPT;
|
| 4122 | }
|
| 4123 | }
|
| 4124 |
|
| 4125 | static int packet_getsockopt(struct socket *sock, int level, int optname,
|
| 4126 | sockopt_t *opt)
|
| 4127 | {
|
| 4128 | int len;
|
| 4129 | int val, lv = sizeof(val);
|
| 4130 | struct sock *sk = sock->sk;
|
| 4131 | struct packet_sock *po = pkt_sk(sk);
|
| 4132 | void *data = &val;
|
| 4133 | union tpacket_stats_u st;
|
| 4134 | struct tpacket_rollover_stats rstats;
|
| 4135 | int drops;
|
| 4136 |
|
| 4137 | if (level != SOL_PACKET)
|
| 4138 | return -ENOPROTOOPT;
|
| 4139 |
|
| 4140 | len = opt->optlen;
|
| 4141 |
|
| 4142 | if (len < 0)
|
| 4143 | return -EINVAL;
|
| 4144 |
|
| 4145 | switch (optname) {
|
| 4146 | case PACKET_STATISTICS:
|
| 4147 | spin_lock_bh(&sk->sk_receive_queue.lock);
|
| 4148 | memcpy(&st, &po->stats, sizeof(st));
|
| 4149 | memset(&po->stats, 0, sizeof(po->stats));
|
| 4150 | spin_unlock_bh(&sk->sk_receive_queue.lock);
|
| 4151 | drops = atomic_xchg(&po->tp_drops, 0);
|
| 4152 |
|
| 4153 | if (po->tp_version == TPACKET_V3) {
|
| 4154 | lv = sizeof(struct tpacket_stats_v3);
|
| 4155 | st.stats3.tp_drops = drops;
|
| 4156 | st.stats3.tp_packets += drops;
|
| 4157 | data = &st.stats3;
|
| 4158 | } else {
|
| 4159 | lv = sizeof(struct tpacket_stats);
|
| 4160 | st.stats1.tp_drops = drops;
|
| 4161 | st.stats1.tp_packets += drops;
|
| 4162 | data = &st.stats1;
|
| 4163 | }
|
| 4164 |
|
| 4165 | break;
|
| 4166 | case PACKET_AUXDATA:
|
| 4167 | val = packet_sock_flag(po, PACKET_SOCK_AUXDATA);
|
| 4168 | break;
|
| 4169 | case PACKET_ORIGDEV:
|
| 4170 | val = packet_sock_flag(po, PACKET_SOCK_ORIGDEV);
|
| 4171 | break;
|
| 4172 | case PACKET_VNET_HDR:
|
| 4173 | val = !!READ_ONCE(po->vnet_hdr_sz);
|
| 4174 | break;
|
| 4175 | case PACKET_VNET_HDR_SZ:
|
| 4176 | val = READ_ONCE(po->vnet_hdr_sz);
|
| 4177 | break;
|
| 4178 | case PACKET_COPY_THRESH:
|
| 4179 | val = READ_ONCE(pkt_sk(sk)->copy_thresh);
|
| 4180 | break;
|
| 4181 | case PACKET_VERSION:
|
| 4182 | val = po->tp_version;
|
| 4183 | break;
|
| 4184 | case PACKET_HDRLEN:
|
| 4185 | if (len > sizeof(int))
|
| 4186 | len = sizeof(int);
|
| 4187 | if (len < sizeof(int))
|
| 4188 | return -EINVAL;
|
| 4189 | if (copy_from_iter(&val, len, &opt->iter_in) != len)
|
| 4190 | return -EFAULT;
|
| 4191 | switch (val) {
|
| 4192 | case TPACKET_V1:
|
| 4193 | val = sizeof(struct tpacket_hdr);
|
| 4194 | break;
|
| 4195 | case TPACKET_V2:
|
| 4196 | val = sizeof(struct tpacket2_hdr);
|
| 4197 | break;
|
| 4198 | case TPACKET_V3:
|
| 4199 | val = sizeof(struct tpacket3_hdr);
|
| 4200 | break;
|
| 4201 | default:
|
| 4202 | return -EINVAL;
|
| 4203 | }
|
| 4204 | break;
|
| 4205 | case PACKET_RESERVE:
|
| 4206 | val = po->tp_reserve;
|
| 4207 | break;
|
| 4208 | case PACKET_LOSS:
|
| 4209 | val = packet_sock_flag(po, PACKET_SOCK_TP_LOSS);
|
| 4210 | break;
|
| 4211 | case PACKET_TIMESTAMP:
|
| 4212 | val = READ_ONCE(po->tp_tstamp);
|
| 4213 | break;
|
| 4214 | case PACKET_FANOUT:
|
| 4215 | val = (po->fanout ?
|
| 4216 | ((u32)po->fanout->id |
|
| 4217 | ((u32)po->fanout->type << 16) |
|
| 4218 | ((u32)po->fanout->flags << 24)) :
|
| 4219 | 0);
|
| 4220 | break;
|
| 4221 | case PACKET_IGNORE_OUTGOING:
|
| 4222 | val = READ_ONCE(po->prot_hook.ignore_outgoing);
|
| 4223 | break;
|
| 4224 | case PACKET_ROLLOVER_STATS:
|
| 4225 | if (!po->rollover)
|
| 4226 | return -EINVAL;
|
| 4227 | rstats.tp_all = atomic_long_read(&po->rollover->num);
|
| 4228 | rstats.tp_huge = atomic_long_read(&po->rollover->num_huge);
|
| 4229 | rstats.tp_failed = atomic_long_read(&po->rollover->num_failed);
|
| 4230 | data = &rstats;
|
| 4231 | lv = sizeof(rstats);
|
| 4232 | break;
|
| 4233 | case PACKET_TX_HAS_OFF:
|
| 4234 | val = packet_sock_flag(po, PACKET_SOCK_TX_HAS_OFF);
|
| 4235 | break;
|
| 4236 | case PACKET_QDISC_BYPASS:
|
| 4237 | val = packet_sock_flag(po, PACKET_SOCK_QDISC_BYPASS);
|
| 4238 | break;
|
| 4239 | default:
|
| 4240 | return -ENOPROTOOPT;
|
| 4241 | }
|
| 4242 |
|
| 4243 | if (len > lv)
|
| 4244 | len = lv;
|
| 4245 | opt->optlen = len;
|
| 4246 | if (copy_to_iter(data, len, &opt->iter_out) != len)
|
| 4247 | return -EFAULT;
|
| 4248 | return 0;
|
| 4249 | }
|
| 4250 |
|
| 4251 | static int packet_notifier(struct notifier_block *this,
|
| 4252 | unsigned long msg, void *ptr)
|
| 4253 | {
|
| 4254 | struct net_device *dev = netdev_notifier_info_to_dev(ptr);
|
| 4255 | struct net *net = dev_net(dev);
|
| 4256 | struct packet_mclist *ml, *tmp;
|
| 4257 | LIST_HEAD(mclist);
|
| 4258 | struct sock *sk;
|
| 4259 |
|
| 4260 | rcu_read_lock();
|
| 4261 | sk_for_each_rcu(sk, &net->packet.sklist) {
|
| 4262 | struct packet_sock *po = pkt_sk(sk);
|
| 4263 |
|
| 4264 | switch (msg) {
|
| 4265 | case NETDEV_UNREGISTER:
|
| 4266 | if (po->mclist)
|
| 4267 | packet_dev_mclist_delete(dev, &po->mclist,
|
| 4268 | &mclist);
|
| 4269 | fallthrough;
|
| 4270 |
|
| 4271 | case NETDEV_DOWN:
|
| 4272 | if (dev->ifindex == po->ifindex) {
|
| 4273 | spin_lock(&po->bind_lock);
|
| 4274 | if (packet_sock_flag(po, PACKET_SOCK_RUNNING)) {
|
| 4275 | __unregister_prot_hook(sk, false);
|
| 4276 | sk->sk_err = ENETDOWN;
|
| 4277 | if (!sock_flag(sk, SOCK_DEAD))
|
| 4278 | sk_error_report(sk);
|
| 4279 | }
|
| 4280 | if (msg == NETDEV_UNREGISTER) {
|
| 4281 | packet_cached_dev_reset(po);
|
| 4282 | WRITE_ONCE(po->ifindex, -1);
|
| 4283 | netdev_put(po->prot_hook.dev,
|
| 4284 | &po->prot_hook.dev_tracker);
|
| 4285 | po->prot_hook.dev = NULL;
|
| 4286 | }
|
| 4287 | spin_unlock(&po->bind_lock);
|
| 4288 | }
|
| 4289 | break;
|
| 4290 | case NETDEV_UP:
|
| 4291 | if (dev->ifindex == po->ifindex) {
|
| 4292 | spin_lock(&po->bind_lock);
|
| 4293 | if (po->num)
|
| 4294 | register_prot_hook(sk);
|
| 4295 | spin_unlock(&po->bind_lock);
|
| 4296 | }
|
| 4297 | break;
|
| 4298 | }
|
| 4299 | }
|
| 4300 | rcu_read_unlock();
|
| 4301 |
|
| 4302 |
|
| 4303 | list_for_each_entry_safe(ml, tmp, &mclist, remove_list) {
|
| 4304 | packet_dev_mc(dev, ml, -1);
|
| 4305 | kfree(ml);
|
| 4306 | }
|
| 4307 |
|
| 4308 | return NOTIFY_DONE;
|
| 4309 | }
|
| 4310 |
|
| 4311 |
|
| 4312 | static int packet_ioctl(struct socket *sock, unsigned int cmd,
|
| 4313 | unsigned long arg)
|
| 4314 | {
|
| 4315 | struct sock *sk = sock->sk;
|
| 4316 |
|
| 4317 | switch (cmd) {
|
| 4318 | case SIOCOUTQ:
|
| 4319 | {
|
| 4320 | int amount = sk_wmem_alloc_get(sk);
|
| 4321 |
|
| 4322 | return put_user(amount, (int __user *)arg);
|
| 4323 | }
|
| 4324 | case SIOCINQ:
|
| 4325 | {
|
| 4326 | struct sk_buff *skb;
|
| 4327 | int amount = 0;
|
| 4328 |
|
| 4329 | spin_lock_bh(&sk->sk_receive_queue.lock);
|
| 4330 | skb = skb_peek(&sk->sk_receive_queue);
|
| 4331 | if (skb)
|
| 4332 | amount = skb->len;
|
| 4333 | spin_unlock_bh(&sk->sk_receive_queue.lock);
|
| 4334 | return put_user(amount, (int __user *)arg);
|
| 4335 | }
|
| 4336 | #ifdef CONFIG_INET
|
| 4337 | case SIOCADDRT:
|
| 4338 | case SIOCDELRT:
|
| 4339 | case SIOCDARP:
|
| 4340 | case SIOCGARP:
|
| 4341 | case SIOCSARP:
|
| 4342 | case SIOCGIFADDR:
|
| 4343 | case SIOCSIFADDR:
|
| 4344 | case SIOCGIFBRDADDR:
|
| 4345 | case SIOCSIFBRDADDR:
|
| 4346 | case SIOCGIFNETMASK:
|
| 4347 | case SIOCSIFNETMASK:
|
| 4348 | case SIOCGIFDSTADDR:
|
| 4349 | case SIOCSIFDSTADDR:
|
| 4350 | case SIOCSIFFLAGS:
|
| 4351 | return inet_dgram_ops.ioctl(sock, cmd, arg);
|
| 4352 | #endif
|
| 4353 |
|
| 4354 | default:
|
| 4355 | return -ENOIOCTLCMD;
|
| 4356 | }
|
| 4357 | return 0;
|
| 4358 | }
|
| 4359 |
|
| 4360 | static __poll_t packet_poll(struct file *file, struct socket *sock,
|
| 4361 | poll_table *wait)
|
| 4362 | {
|
| 4363 | struct sock *sk = sock->sk;
|
| 4364 | struct packet_sock *po = pkt_sk(sk);
|
| 4365 | __poll_t mask = datagram_poll(file, sock, wait);
|
| 4366 |
|
| 4367 | spin_lock_bh(&sk->sk_receive_queue.lock);
|
| 4368 | if (po->rx_ring.pg_vec) {
|
| 4369 | if (!packet_previous_rx_frame(po, &po->rx_ring,
|
| 4370 | TP_STATUS_KERNEL))
|
| 4371 | mask |= EPOLLIN | EPOLLRDNORM;
|
| 4372 | }
|
| 4373 | __packet_rcv_try_clear_pressure(po);
|
| 4374 | spin_unlock_bh(&sk->sk_receive_queue.lock);
|
| 4375 | spin_lock_bh(&sk->sk_write_queue.lock);
|
| 4376 | if (po->tx_ring.pg_vec) {
|
| 4377 | if (packet_current_frame(po, &po->tx_ring, TP_STATUS_AVAILABLE))
|
| 4378 | mask |= EPOLLOUT | EPOLLWRNORM;
|
| 4379 | }
|
| 4380 | spin_unlock_bh(&sk->sk_write_queue.lock);
|
| 4381 | return mask;
|
| 4382 | }
|
| 4383 |
|
| 4384 |
|
| 4385 | |
| 4386 | * for user mmaps.
|
| 4387 |
|
| 4388 |
|
| 4389 | static void packet_mm_open(struct vm_area_struct *vma)
|
| 4390 | {
|
| 4391 | struct file *file = vma->vm_file;
|
| 4392 | struct socket *sock = file->private_data;
|
| 4393 | struct sock *sk = sock->sk;
|
| 4394 |
|
| 4395 | if (sk)
|
| 4396 | atomic_long_inc(&pkt_sk(sk)->mapped);
|
| 4397 | }
|
| 4398 |
|
| 4399 | static void packet_mm_close(struct vm_area_struct *vma)
|
| 4400 | {
|
| 4401 | struct file *file = vma->vm_file;
|
| 4402 | struct socket *sock = file->private_data;
|
| 4403 | struct sock *sk = sock->sk;
|
| 4404 |
|
| 4405 | if (sk)
|
| 4406 | atomic_long_dec(&pkt_sk(sk)->mapped);
|
| 4407 | }
|
| 4408 |
|
| 4409 | static const struct vm_operations_struct packet_mmap_ops = {
|
| 4410 | .open = packet_mm_open,
|
| 4411 | .close = packet_mm_close,
|
| 4412 | };
|
| 4413 |
|
| 4414 | struct packet_pg_vec {
|
| 4415 | struct packet_pg_vec_free *deferred;
|
| 4416 | unsigned int order;
|
| 4417 | unsigned int len;
|
| 4418 | struct pgv pg_vec[] __counted_by(len);
|
| 4419 | };
|
| 4420 |
|
| 4421 | struct packet_pg_vec_free {
|
| 4422 | struct delayed_work work;
|
| 4423 | struct sock *sk;
|
| 4424 | struct packet_pg_vec *vec;
|
| 4425 | };
|
| 4426 |
|
| 4427 | static void free_pg_vec(struct pgv *pg_vec, unsigned int order,
|
| 4428 | unsigned int len)
|
| 4429 | {
|
| 4430 | struct packet_pg_vec *vec;
|
| 4431 | int i;
|
| 4432 |
|
| 4433 | vec = container_of_const(pg_vec, struct packet_pg_vec, pg_vec[0]);
|
| 4434 | for (i = 0; i < len; i++) {
|
| 4435 | if (likely(pg_vec[i].buffer)) {
|
| 4436 | if (is_vmalloc_addr(pg_vec[i].buffer))
|
| 4437 | vfree(pg_vec[i].buffer);
|
| 4438 | else
|
| 4439 | free_pages((unsigned long)pg_vec[i].buffer,
|
| 4440 | order);
|
| 4441 | pg_vec[i].buffer = NULL;
|
| 4442 | }
|
| 4443 | }
|
| 4444 | kfree(vec->deferred);
|
| 4445 | kfree(vec);
|
| 4446 | }
|
| 4447 |
|
| 4448 | static void packet_free_pg_vec_work(struct work_struct *work)
|
| 4449 | {
|
| 4450 | struct packet_pg_vec_free *deferred;
|
| 4451 | struct packet_pg_vec *vec;
|
| 4452 | struct sock *sk;
|
| 4453 |
|
| 4454 | deferred = container_of_const(to_delayed_work(work),
|
| 4455 | struct packet_pg_vec_free, work);
|
| 4456 | vec = deferred->vec;
|
| 4457 | sk = deferred->sk;
|
| 4458 | if (sk_wmem_alloc_get(sk)) {
|
| 4459 | queue_delayed_work(system_long_wq, &deferred->work, 1);
|
| 4460 | return;
|
| 4461 | }
|
| 4462 |
|
| 4463 | free_pg_vec(vec->pg_vec, vec->order, vec->len);
|
| 4464 | sock_put(sk);
|
| 4465 | }
|
| 4466 |
|
| 4467 | static void packet_free_tx_ring(struct sock *sk, struct pgv *pg_vec,
|
| 4468 | unsigned int order, unsigned int len)
|
| 4469 | {
|
| 4470 | struct packet_pg_vec_free *deferred;
|
| 4471 | struct packet_pg_vec *vec;
|
| 4472 |
|
| 4473 | vec = container_of_const(pg_vec, struct packet_pg_vec, pg_vec[0]);
|
| 4474 | deferred = vec->deferred;
|
| 4475 | if (!deferred || !sk_wmem_alloc_get(sk)) {
|
| 4476 | free_pg_vec(pg_vec, order, len);
|
| 4477 | return;
|
| 4478 | }
|
| 4479 |
|
| 4480 |
|
| 4481 | deferred->sk = sk;
|
| 4482 | sock_hold(sk);
|
| 4483 | queue_delayed_work(system_long_wq, &deferred->work, 0);
|
| 4484 | }
|
| 4485 |
|
| 4486 | static char *alloc_one_pg_vec_page(unsigned long order)
|
| 4487 | {
|
| 4488 | char *buffer;
|
| 4489 | gfp_t gfp_flags = GFP_KERNEL | __GFP_COMP |
|
| 4490 | __GFP_ZERO | __GFP_NOWARN | __GFP_NORETRY;
|
| 4491 |
|
| 4492 | buffer = (char *) __get_free_pages(gfp_flags, order);
|
| 4493 | if (buffer)
|
| 4494 | return buffer;
|
| 4495 |
|
| 4496 |
|
| 4497 | buffer = vzalloc(array_size((1 << order), PAGE_SIZE));
|
| 4498 | if (buffer)
|
| 4499 | return buffer;
|
| 4500 |
|
| 4501 |
|
| 4502 | gfp_flags &= ~__GFP_NORETRY;
|
| 4503 | buffer = (char *) __get_free_pages(gfp_flags, order);
|
| 4504 | if (buffer)
|
| 4505 | return buffer;
|
| 4506 |
|
| 4507 |
|
| 4508 | return NULL;
|
| 4509 | }
|
| 4510 |
|
| 4511 | static struct pgv *alloc_pg_vec(struct tpacket_req *req, int order, bool tx_ring)
|
| 4512 | {
|
| 4513 | unsigned int block_nr = req->tp_block_nr;
|
| 4514 | struct packet_pg_vec *vec;
|
| 4515 | struct pgv *pg_vec;
|
| 4516 | int i;
|
| 4517 |
|
| 4518 | vec = kzalloc_flex(*vec, pg_vec, block_nr, GFP_KERNEL | __GFP_NOWARN);
|
| 4519 | if (unlikely(!vec))
|
| 4520 | return NULL;
|
| 4521 | vec->order = order;
|
| 4522 | vec->len = block_nr;
|
| 4523 | pg_vec = vec->pg_vec;
|
| 4524 |
|
| 4525 | if (tx_ring) {
|
| 4526 | vec->deferred = kzalloc_obj(*vec->deferred,
|
| 4527 | GFP_KERNEL | __GFP_NOWARN);
|
| 4528 | if (!vec->deferred)
|
| 4529 | goto out_free_pgvec;
|
| 4530 | vec->deferred->vec = vec;
|
| 4531 | INIT_DELAYED_WORK(&vec->deferred->work,
|
| 4532 | packet_free_pg_vec_work);
|
| 4533 | }
|
| 4534 |
|
| 4535 | for (i = 0; i < block_nr; i++) {
|
| 4536 | pg_vec[i].buffer = alloc_one_pg_vec_page(order);
|
| 4537 | if (unlikely(!pg_vec[i].buffer))
|
| 4538 | goto out_free_pgvec;
|
| 4539 | }
|
| 4540 |
|
| 4541 | out:
|
| 4542 | return pg_vec;
|
| 4543 |
|
| 4544 | out_free_pgvec:
|
| 4545 | free_pg_vec(pg_vec, order, block_nr);
|
| 4546 | pg_vec = NULL;
|
| 4547 | goto out;
|
| 4548 | }
|
| 4549 |
|
| 4550 | static int packet_set_ring(struct sock *sk, union tpacket_req_u *req_u,
|
| 4551 | int closing, int tx_ring)
|
| 4552 | {
|
| 4553 | struct pgv *pg_vec = NULL;
|
| 4554 | struct packet_sock *po = pkt_sk(sk);
|
| 4555 | unsigned long *rx_owner_map = NULL;
|
| 4556 | int was_running, order = 0;
|
| 4557 | struct packet_ring_buffer *rb;
|
| 4558 | struct sk_buff_head *rb_queue;
|
| 4559 | __be16 num;
|
| 4560 | int err;
|
| 4561 |
|
| 4562 | struct tpacket_req *req = &req_u->req;
|
| 4563 |
|
| 4564 | rb = tx_ring ? &po->tx_ring : &po->rx_ring;
|
| 4565 | rb_queue = tx_ring ? &sk->sk_write_queue : &sk->sk_receive_queue;
|
| 4566 |
|
| 4567 | err = -EBUSY;
|
| 4568 | if (!closing) {
|
| 4569 | if (atomic_long_read(&po->mapped))
|
| 4570 | goto out;
|
| 4571 | if (packet_read_pending(rb))
|
| 4572 | goto out;
|
| 4573 | }
|
| 4574 |
|
| 4575 | if (req->tp_block_nr) {
|
| 4576 | unsigned int min_frame_size;
|
| 4577 |
|
| 4578 |
|
| 4579 | err = -EBUSY;
|
| 4580 | if (unlikely(rb->pg_vec))
|
| 4581 | goto out;
|
| 4582 |
|
| 4583 | switch (po->tp_version) {
|
| 4584 | case TPACKET_V1:
|
| 4585 | po->tp_hdrlen = TPACKET_HDRLEN;
|
| 4586 | break;
|
| 4587 | case TPACKET_V2:
|
| 4588 | po->tp_hdrlen = TPACKET2_HDRLEN;
|
| 4589 | break;
|
| 4590 | case TPACKET_V3:
|
| 4591 | po->tp_hdrlen = TPACKET3_HDRLEN;
|
| 4592 | break;
|
| 4593 | }
|
| 4594 |
|
| 4595 | err = -EINVAL;
|
| 4596 | if (unlikely((int)req->tp_block_size <= 0))
|
| 4597 | goto out;
|
| 4598 | if (unlikely(!PAGE_ALIGNED(req->tp_block_size)))
|
| 4599 | goto out;
|
| 4600 | min_frame_size = po->tp_hdrlen + po->tp_reserve;
|
| 4601 | if (po->tp_version >= TPACKET_V3 &&
|
| 4602 | req->tp_block_size <
|
| 4603 | BLK_PLUS_PRIV((u64)req_u->req3.tp_sizeof_priv) + min_frame_size)
|
| 4604 | goto out;
|
| 4605 | if (unlikely(req->tp_frame_size < min_frame_size))
|
| 4606 | goto out;
|
| 4607 | if (unlikely(req->tp_frame_size & (TPACKET_ALIGNMENT - 1)))
|
| 4608 | goto out;
|
| 4609 |
|
| 4610 | rb->frames_per_block = req->tp_block_size / req->tp_frame_size;
|
| 4611 | if (unlikely(rb->frames_per_block == 0))
|
| 4612 | goto out;
|
| 4613 | if (unlikely(rb->frames_per_block > UINT_MAX / req->tp_block_nr))
|
| 4614 | goto out;
|
| 4615 | if (unlikely((rb->frames_per_block * req->tp_block_nr) !=
|
| 4616 | req->tp_frame_nr))
|
| 4617 | goto out;
|
| 4618 |
|
| 4619 | err = -ENOMEM;
|
| 4620 | order = get_order(req->tp_block_size);
|
| 4621 | pg_vec = alloc_pg_vec(req, order, tx_ring);
|
| 4622 | if (unlikely(!pg_vec))
|
| 4623 | goto out;
|
| 4624 | switch (po->tp_version) {
|
| 4625 | case TPACKET_V3:
|
| 4626 |
|
| 4627 | if (!tx_ring) {
|
| 4628 | init_prb_bdqc(po, rb, pg_vec, req_u);
|
| 4629 | } else {
|
| 4630 | struct tpacket_req3 *req3 = &req_u->req3;
|
| 4631 |
|
| 4632 | if (req3->tp_retire_blk_tov ||
|
| 4633 | req3->tp_sizeof_priv ||
|
| 4634 | req3->tp_feature_req_word) {
|
| 4635 | err = -EINVAL;
|
| 4636 | goto out_free_pg_vec;
|
| 4637 | }
|
| 4638 | }
|
| 4639 | break;
|
| 4640 | default:
|
| 4641 | if (!tx_ring) {
|
| 4642 | rx_owner_map = bitmap_alloc(req->tp_frame_nr,
|
| 4643 | GFP_KERNEL | __GFP_NOWARN | __GFP_ZERO);
|
| 4644 | if (!rx_owner_map)
|
| 4645 | goto out_free_pg_vec;
|
| 4646 | }
|
| 4647 | break;
|
| 4648 | }
|
| 4649 | }
|
| 4650 |
|
| 4651 | else {
|
| 4652 | err = -EINVAL;
|
| 4653 | if (unlikely(req->tp_frame_nr))
|
| 4654 | goto out;
|
| 4655 | }
|
| 4656 |
|
| 4657 |
|
| 4658 |
|
| 4659 | spin_lock(&po->bind_lock);
|
| 4660 | was_running = packet_sock_flag(po, PACKET_SOCK_RUNNING);
|
| 4661 | num = po->num;
|
| 4662 | WRITE_ONCE(po->num, 0);
|
| 4663 | if (was_running)
|
| 4664 | __unregister_prot_hook(sk, false);
|
| 4665 |
|
| 4666 | spin_unlock(&po->bind_lock);
|
| 4667 |
|
| 4668 | synchronize_net();
|
| 4669 |
|
| 4670 | err = -EBUSY;
|
| 4671 | mutex_lock(&po->pg_vec_lock);
|
| 4672 | if (closing || atomic_long_read(&po->mapped) == 0) {
|
| 4673 | if (tx_ring && !closing && packet_read_pending(rb))
|
| 4674 | goto out_unlock;
|
| 4675 |
|
| 4676 | err = 0;
|
| 4677 | spin_lock_bh(&rb_queue->lock);
|
| 4678 | swap(rb->pg_vec, pg_vec);
|
| 4679 | if (po->tp_version <= TPACKET_V2)
|
| 4680 | swap(rb->rx_owner_map, rx_owner_map);
|
| 4681 | rb->frame_max = (req->tp_frame_nr - 1);
|
| 4682 | rb->head = 0;
|
| 4683 | rb->frame_size = req->tp_frame_size;
|
| 4684 | po->prot_hook.func = (po->rx_ring.pg_vec) ?
|
| 4685 | tpacket_rcv : packet_rcv;
|
| 4686 | spin_unlock_bh(&rb_queue->lock);
|
| 4687 |
|
| 4688 | swap(rb->pg_vec_order, order);
|
| 4689 | swap(rb->pg_vec_len, req->tp_block_nr);
|
| 4690 |
|
| 4691 | rb->pg_vec_pages = req->tp_block_size/PAGE_SIZE;
|
| 4692 | skb_queue_purge(rb_queue);
|
| 4693 | if (atomic_long_read(&po->mapped))
|
| 4694 | pr_err("packet_mmap: vma is busy: %ld\n",
|
| 4695 | atomic_long_read(&po->mapped));
|
| 4696 | }
|
| 4697 | out_unlock:
|
| 4698 | mutex_unlock(&po->pg_vec_lock);
|
| 4699 |
|
| 4700 | spin_lock(&po->bind_lock);
|
| 4701 | WRITE_ONCE(po->num, num);
|
| 4702 | |
| 4703 | * NETDEV_UNREGISTER may have invalidated the binding while bind_lock
|
| 4704 | * was dropped above. Do not re-add a fanout hook to a dead device.
|
| 4705 |
|
| 4706 | if (was_running && READ_ONCE(po->ifindex) != -1)
|
| 4707 | register_prot_hook(sk);
|
| 4708 |
|
| 4709 | spin_unlock(&po->bind_lock);
|
| 4710 | if (pg_vec && (po->tp_version > TPACKET_V2)) {
|
| 4711 |
|
| 4712 | if (!tx_ring)
|
| 4713 | prb_shutdown_retire_blk_timer(po, rb_queue);
|
| 4714 | }
|
| 4715 |
|
| 4716 | out_free_pg_vec:
|
| 4717 | if (pg_vec) {
|
| 4718 | bitmap_free(rx_owner_map);
|
| 4719 | if (tx_ring && closing)
|
| 4720 | packet_free_tx_ring(sk, pg_vec, order, req->tp_block_nr);
|
| 4721 | else
|
| 4722 | free_pg_vec(pg_vec, order, req->tp_block_nr);
|
| 4723 | }
|
| 4724 | out:
|
| 4725 | return err;
|
| 4726 | }
|
| 4727 |
|
| 4728 | static int packet_mmap(struct file *file, struct socket *sock,
|
| 4729 | struct vm_area_struct *vma)
|
| 4730 | {
|
| 4731 | struct sock *sk = sock->sk;
|
| 4732 | struct packet_sock *po = pkt_sk(sk);
|
| 4733 | unsigned long size, expected_size;
|
| 4734 | struct packet_ring_buffer *rb;
|
| 4735 | unsigned long start;
|
| 4736 | int err = -EINVAL;
|
| 4737 | int i;
|
| 4738 |
|
| 4739 | if (vma->vm_pgoff)
|
| 4740 | return -EINVAL;
|
| 4741 |
|
| 4742 | mutex_lock(&po->pg_vec_lock);
|
| 4743 |
|
| 4744 | expected_size = 0;
|
| 4745 | for (rb = &po->rx_ring; rb <= &po->tx_ring; rb++) {
|
| 4746 | if (rb->pg_vec) {
|
| 4747 | expected_size += rb->pg_vec_len
|
| 4748 | * rb->pg_vec_pages
|
| 4749 | * PAGE_SIZE;
|
| 4750 | }
|
| 4751 | }
|
| 4752 |
|
| 4753 | if (expected_size == 0)
|
| 4754 | goto out;
|
| 4755 |
|
| 4756 | size = vma->vm_end - vma->vm_start;
|
| 4757 | if (size != expected_size)
|
| 4758 | goto out;
|
| 4759 |
|
| 4760 | start = vma->vm_start;
|
| 4761 | for (rb = &po->rx_ring; rb <= &po->tx_ring; rb++) {
|
| 4762 | if (rb->pg_vec == NULL)
|
| 4763 | continue;
|
| 4764 |
|
| 4765 | for (i = 0; i < rb->pg_vec_len; i++) {
|
| 4766 | struct page *page;
|
| 4767 | void *kaddr = rb->pg_vec[i].buffer;
|
| 4768 | int pg_num;
|
| 4769 |
|
| 4770 | for (pg_num = 0; pg_num < rb->pg_vec_pages; pg_num++) {
|
| 4771 | page = pgv_to_page(kaddr);
|
| 4772 | err = vm_insert_page(vma, start, page);
|
| 4773 | if (unlikely(err))
|
| 4774 | goto out;
|
| 4775 | start += PAGE_SIZE;
|
| 4776 | kaddr += PAGE_SIZE;
|
| 4777 | }
|
| 4778 | }
|
| 4779 | }
|
| 4780 |
|
| 4781 | atomic_long_inc(&po->mapped);
|
| 4782 | vma->vm_ops = &packet_mmap_ops;
|
| 4783 | err = 0;
|
| 4784 |
|
| 4785 | out:
|
| 4786 | mutex_unlock(&po->pg_vec_lock);
|
| 4787 | return err;
|
| 4788 | }
|
| 4789 |
|
| 4790 | static const struct proto_ops packet_ops_spkt = {
|
| 4791 | .family = PF_PACKET,
|
| 4792 | .owner = THIS_MODULE,
|
| 4793 | .release = packet_release,
|
| 4794 | .bind = packet_bind_spkt,
|
| 4795 | .connect = sock_no_connect,
|
| 4796 | .socketpair = sock_no_socketpair,
|
| 4797 | .accept = sock_no_accept,
|
| 4798 | .getname = packet_getname_spkt,
|
| 4799 | .poll = datagram_poll,
|
| 4800 | .ioctl = packet_ioctl,
|
| 4801 | .gettstamp = sock_gettstamp,
|
| 4802 | .listen = sock_no_listen,
|
| 4803 | .shutdown = sock_no_shutdown,
|
| 4804 | .sendmsg = packet_sendmsg_spkt,
|
| 4805 | .recvmsg = packet_recvmsg,
|
| 4806 | .mmap = sock_no_mmap,
|
| 4807 | };
|
| 4808 |
|
| 4809 | static const struct proto_ops packet_ops = {
|
| 4810 | .family = PF_PACKET,
|
| 4811 | .owner = THIS_MODULE,
|
| 4812 | .release = packet_release,
|
| 4813 | .bind = packet_bind,
|
| 4814 | .connect = sock_no_connect,
|
| 4815 | .socketpair = sock_no_socketpair,
|
| 4816 | .accept = sock_no_accept,
|
| 4817 | .getname = packet_getname,
|
| 4818 | .poll = packet_poll,
|
| 4819 | .ioctl = packet_ioctl,
|
| 4820 | .gettstamp = sock_gettstamp,
|
| 4821 | .listen = sock_no_listen,
|
| 4822 | .shutdown = sock_no_shutdown,
|
| 4823 | .setsockopt = packet_setsockopt,
|
| 4824 | .getsockopt_iter = packet_getsockopt,
|
| 4825 | .sendmsg = packet_sendmsg,
|
| 4826 | .recvmsg = packet_recvmsg,
|
| 4827 | .mmap = packet_mmap,
|
| 4828 | };
|
| 4829 |
|
| 4830 | static const struct net_proto_family packet_family_ops = {
|
| 4831 | .family = PF_PACKET,
|
| 4832 | .create = packet_create,
|
| 4833 | .owner = THIS_MODULE,
|
| 4834 | };
|
| 4835 |
|
| 4836 | static struct notifier_block packet_netdev_notifier = {
|
| 4837 | .notifier_call = packet_notifier,
|
| 4838 | };
|
| 4839 |
|
| 4840 | #ifdef CONFIG_PROC_FS
|
| 4841 |
|
| 4842 | static void *packet_seq_start(struct seq_file *seq, loff_t *pos)
|
| 4843 | __acquires(RCU)
|
| 4844 | {
|
| 4845 | struct net *net = seq_file_net(seq);
|
| 4846 |
|
| 4847 | rcu_read_lock();
|
| 4848 | return seq_hlist_start_head_rcu(&net->packet.sklist, *pos);
|
| 4849 | }
|
| 4850 |
|
| 4851 | static void *packet_seq_next(struct seq_file *seq, void *v, loff_t *pos)
|
| 4852 | {
|
| 4853 | struct net *net = seq_file_net(seq);
|
| 4854 | return seq_hlist_next_rcu(v, &net->packet.sklist, pos);
|
| 4855 | }
|
| 4856 |
|
| 4857 | static void packet_seq_stop(struct seq_file *seq, void *v)
|
| 4858 | __releases(RCU)
|
| 4859 | {
|
| 4860 | rcu_read_unlock();
|
| 4861 | }
|
| 4862 |
|
| 4863 | static int packet_seq_show(struct seq_file *seq, void *v)
|
| 4864 | {
|
| 4865 | if (v == SEQ_START_TOKEN)
|
| 4866 | seq_printf(seq,
|
| 4867 | "%*sRefCnt Type Proto Iface R Rmem User Inode\n",
|
| 4868 | IS_ENABLED(CONFIG_64BIT) ? -17 : -9, "sk");
|
| 4869 | else {
|
| 4870 | struct sock *s = sk_entry(v);
|
| 4871 | const struct packet_sock *po = pkt_sk(s);
|
| 4872 |
|
| 4873 | seq_printf(seq,
|
| 4874 | "%pK %-6d %-4d %04x %-5d %1d %-6u %-6u %-6llu\n",
|
| 4875 | s,
|
| 4876 | refcount_read(&s->sk_refcnt),
|
| 4877 | s->sk_type,
|
| 4878 | ntohs(READ_ONCE(po->num)),
|
| 4879 | READ_ONCE(po->ifindex),
|
| 4880 | packet_sock_flag(po, PACKET_SOCK_RUNNING),
|
| 4881 | atomic_read(&s->sk_rmem_alloc),
|
| 4882 | from_kuid_munged(seq_user_ns(seq), sk_uid(s)),
|
| 4883 | sock_i_ino(s));
|
| 4884 | }
|
| 4885 |
|
| 4886 | return 0;
|
| 4887 | }
|
| 4888 |
|
| 4889 | static const struct seq_operations packet_seq_ops = {
|
| 4890 | .start = packet_seq_start,
|
| 4891 | .next = packet_seq_next,
|
| 4892 | .stop = packet_seq_stop,
|
| 4893 | .show = packet_seq_show,
|
| 4894 | };
|
| 4895 | #endif
|
| 4896 |
|
| 4897 | static int __net_init packet_net_init(struct net *net)
|
| 4898 | {
|
| 4899 | mutex_init(&net->packet.sklist_lock);
|
| 4900 | INIT_HLIST_HEAD(&net->packet.sklist);
|
| 4901 |
|
| 4902 | #ifdef CONFIG_PROC_FS
|
| 4903 | if (!proc_create_net("packet", 0, net->proc_net, &packet_seq_ops,
|
| 4904 | sizeof(struct seq_net_private)))
|
| 4905 | return -ENOMEM;
|
| 4906 | #endif
|
| 4907 |
|
| 4908 | return 0;
|
| 4909 | }
|
| 4910 |
|
| 4911 | static void __net_exit packet_net_exit(struct net *net)
|
| 4912 | {
|
| 4913 | remove_proc_entry("packet", net->proc_net);
|
| 4914 | WARN_ON_ONCE(!hlist_empty(&net->packet.sklist));
|
| 4915 | }
|
| 4916 |
|
| 4917 | static struct pernet_operations packet_net_ops = {
|
| 4918 | .init = packet_net_init,
|
| 4919 | .exit = packet_net_exit,
|
| 4920 | };
|
| 4921 |
|
| 4922 |
|
| 4923 | static void __exit packet_exit(void)
|
| 4924 | {
|
| 4925 | sock_unregister(PF_PACKET);
|
| 4926 | proto_unregister(&packet_proto);
|
| 4927 | unregister_netdevice_notifier(&packet_netdev_notifier);
|
| 4928 | unregister_pernet_subsys(&packet_net_ops);
|
| 4929 | }
|
| 4930 |
|
| 4931 | static int __init packet_init(void)
|
| 4932 | {
|
| 4933 | int rc;
|
| 4934 |
|
| 4935 | rc = register_pernet_subsys(&packet_net_ops);
|
| 4936 | if (rc)
|
| 4937 | goto out;
|
| 4938 | rc = register_netdevice_notifier(&packet_netdev_notifier);
|
| 4939 | if (rc)
|
| 4940 | goto out_pernet;
|
| 4941 | rc = proto_register(&packet_proto, 0);
|
| 4942 | if (rc)
|
| 4943 | goto out_notifier;
|
| 4944 | rc = sock_register(&packet_family_ops);
|
| 4945 | if (rc)
|
| 4946 | goto out_proto;
|
| 4947 |
|
| 4948 | return 0;
|
| 4949 |
|
| 4950 | out_proto:
|
| 4951 | proto_unregister(&packet_proto);
|
| 4952 | out_notifier:
|
| 4953 | unregister_netdevice_notifier(&packet_netdev_notifier);
|
| 4954 | out_pernet:
|
| 4955 | unregister_pernet_subsys(&packet_net_ops);
|
| 4956 | out:
|
| 4957 | return rc;
|
| 4958 | }
|
| 4959 |
|
| 4960 | module_init(packet_init);
|
| 4961 | module_exit(packet_exit);
|
| 4962 | MODULE_DESCRIPTION("Packet socket support (AF_PACKET)");
|
| 4963 | MODULE_LICENSE("GPL");
|
| 4964 | MODULE_ALIAS_NETPROTO(PF_PACKET);
|