projects/linux·blob master · refs

316 lines · 8.8 KiB

1/* SPDX-License-Identifier: GPL-2.0-only */
2/* Copyright (C) 2024-2025 Intel Corporation */
3
4#ifndef __LIBETH_RX_H
5#define __LIBETH_RX_H
6
7#include <linux/if_vlan.h>
8
9#include <net/page_pool/helpers.h>
10#include <net/xdp.h>
11
12/* Rx buffer management */
13
14/* Space reserved in front of each frame */
15#define LIBETH_SKB_HEADROOM (NET_SKB_PAD + NET_IP_ALIGN)
16#define LIBETH_XDP_HEADROOM (ALIGN(XDP_PACKET_HEADROOM, NET_SKB_PAD) + \
17 NET_IP_ALIGN)
18/* Maximum headroom for worst-case calculations */
19#define LIBETH_MAX_HEADROOM LIBETH_XDP_HEADROOM
20/* Link layer / L2 overhead: Ethernet, 2 VLAN tags (C + S), FCS */
21#define LIBETH_RX_LL_LEN (ETH_HLEN + 2 * VLAN_HLEN + ETH_FCS_LEN)
22/* Maximum supported L2-L4 header length */
23#define LIBETH_MAX_HEAD roundup_pow_of_two(max(MAX_HEADER, 256))
24
25/* Always use order-0 pages */
26#define LIBETH_RX_PAGE_ORDER 0
27/* Pick a sane buffer stride and align to a cacheline boundary */
28#define LIBETH_RX_BUF_STRIDE SKB_DATA_ALIGN(128)
29/* HW-writeable space in one buffer: truesize - headroom/tailroom, aligned */
30#define LIBETH_RX_PAGE_LEN(hr) \
31 ALIGN_DOWN(SKB_MAX_ORDER(hr, LIBETH_RX_PAGE_ORDER), \
32 LIBETH_RX_BUF_STRIDE)
33
34/**
35 * struct libeth_fqe - structure representing an Rx buffer (fill queue element)
36 * @netmem: network memory reference holding the buffer
37 * @offset: offset from the page start (to the headroom)
38 * @truesize: total space occupied by the buffer (w/ headroom and tailroom)
39 *
40 * Depending on the MTU, API switches between one-page-per-frame and shared
41 * page model (to conserve memory on bigger-page platforms). In case of the
42 * former, @offset is always 0 and @truesize is always ```PAGE_SIZE```.
43 */
44struct libeth_fqe {
45 netmem_ref netmem;
46 u32 offset;
47 u32 truesize;
48} __aligned_largest;
49
50/**
51 * enum libeth_fqe_type - enum representing types of Rx buffers
52 * @LIBETH_FQE_MTU: buffer size is determined by MTU
53 * @LIBETH_FQE_SHORT: buffer size is smaller than MTU, for short frames
54 * @LIBETH_FQE_HDR: buffer size is ```LIBETH_MAX_HEAD```-sized, for headers
55 */
56enum libeth_fqe_type {
57 LIBETH_FQE_MTU = 0U,
58 LIBETH_FQE_SHORT,
59 LIBETH_FQE_HDR,
60};
61
62/**
63 * struct libeth_fq - structure representing a buffer (fill) queue
64 * @fp: hotpath part of the structure
65 * @pp: &page_pool for buffer management
66 * @fqes: array of Rx buffers
67 * @truesize: size to allocate per buffer, w/overhead
68 * @count: number of descriptors/buffers the queue has
69 * @type: type of the buffers this queue has
70 * @hsplit: flag whether header split is enabled
71 * @xdp: flag indicating whether XDP is enabled
72 * @no_napi: the queue is not a data queue and does not have NAPI
73 * @buf_len: HW-writeable length per each buffer
74 * @nid: ID of the closest NUMA node with memory
75 */
76struct libeth_fq {
77 struct_group_tagged(libeth_fq_fp, fp,
78 struct page_pool *pp;
79 struct libeth_fqe *fqes;
80
81 u32 truesize;
82 u32 count;
83 );
84
85 /* Cold fields */
86 enum libeth_fqe_type type:2;
87 bool hsplit:1;
88 bool xdp:1;
89 bool no_napi:1;
90
91 u32 buf_len;
92 int nid;
93};
94
95int libeth_rx_fq_create(struct libeth_fq *fq, void *napi_dev);
96void libeth_rx_fq_destroy(struct libeth_fq *fq);
97
98/**
99 * libeth_rx_alloc - allocate a new Rx buffer
100 * @fq: fill queue to allocate for
101 * @i: index of the buffer within the queue
102 *
103 * Return: DMA address to be passed to HW for Rx on successful allocation,
104 * ```DMA_MAPPING_ERROR``` otherwise.
105 */
106static inline dma_addr_t libeth_rx_alloc(const struct libeth_fq_fp *fq, u32 i)
107{
108 struct libeth_fqe *buf = &fq->fqes[i];
109
110 buf->truesize = fq->truesize;
111 buf->netmem = page_pool_dev_alloc_netmem(fq->pp, &buf->offset,
112 &buf->truesize);
113 if (unlikely(!buf->netmem))
114 return DMA_MAPPING_ERROR;
115
116 return page_pool_get_dma_addr_netmem(buf->netmem) + buf->offset +
117 fq->pp->p.offset;
118}
119
120void libeth_rx_recycle_slow(netmem_ref netmem);
121
122/**
123 * libeth_rx_sync_for_cpu - synchronize or recycle buffer post DMA
124 * @fqe: buffer to process
125 * @len: frame length from the descriptor
126 *
127 * Process the buffer after it's written by HW. The regular path is to
128 * synchronize DMA for CPU, but in case of no data it will be immediately
129 * recycled back to its PP.
130 *
131 * Return: true when there's data to process, false otherwise.
132 */
133static inline bool libeth_rx_sync_for_cpu(const struct libeth_fqe *fqe,
134 u32 len)
135{
136 netmem_ref netmem = fqe->netmem;
137
138 /* Very rare, but possible case. The most common reason:
139 * the last fragment contained FCS only, which was then
140 * stripped by the HW.
141 */
142 if (unlikely(!len)) {
143 libeth_rx_recycle_slow(netmem);
144 return false;
145 }
146
147 page_pool_dma_sync_netmem_for_cpu(netmem_get_pp(netmem), netmem,
148 fqe->offset, len);
149
150 return true;
151}
152
153/* Converting abstract packet type numbers into a software structure with
154 * the packet parameters to do O(1) lookup on Rx.
155 */
156
157enum {
158 LIBETH_RX_PT_OUTER_L2 = 0U,
159 LIBETH_RX_PT_OUTER_IPV4,
160 LIBETH_RX_PT_OUTER_IPV6,
161};
162
163enum {
164 LIBETH_RX_PT_NOT_FRAG = 0U,
165 LIBETH_RX_PT_FRAG,
166};
167
168enum {
169 LIBETH_RX_PT_TUNNEL_IP_NONE = 0U,
170 LIBETH_RX_PT_TUNNEL_IP_IP,
171 LIBETH_RX_PT_TUNNEL_IP_GRENAT,
172 LIBETH_RX_PT_TUNNEL_IP_GRENAT_MAC,
173 LIBETH_RX_PT_TUNNEL_IP_GRENAT_MAC_VLAN,
174};
175
176enum {
177 LIBETH_RX_PT_TUNNEL_END_NONE = 0U,
178 LIBETH_RX_PT_TUNNEL_END_IPV4,
179 LIBETH_RX_PT_TUNNEL_END_IPV6,
180};
181
182enum {
183 LIBETH_RX_PT_INNER_NONE = 0U,
184 LIBETH_RX_PT_INNER_UDP,
185 LIBETH_RX_PT_INNER_TCP,
186 LIBETH_RX_PT_INNER_SCTP,
187 LIBETH_RX_PT_INNER_ICMP,
188 LIBETH_RX_PT_INNER_TIMESYNC,
189};
190
191#define LIBETH_RX_PT_PAYLOAD_NONE PKT_HASH_TYPE_NONE
192#define LIBETH_RX_PT_PAYLOAD_L2 PKT_HASH_TYPE_L2
193#define LIBETH_RX_PT_PAYLOAD_L3 PKT_HASH_TYPE_L3
194#define LIBETH_RX_PT_PAYLOAD_L4 PKT_HASH_TYPE_L4
195
196struct libeth_rx_pt {
197 u32 outer_ip:2;
198 u32 outer_frag:1;
199 u32 tunnel_type:3;
200 u32 tunnel_end_prot:2;
201 u32 tunnel_end_frag:1;
202 u32 inner_prot:3;
203 enum pkt_hash_types payload_layer:2;
204
205 u32 pad:2;
206 enum xdp_rss_hash_type hash_type:16;
207};
208
209/**
210 * struct libeth_rx_csum - checksum offload bits decoded from the Rx descriptor
211 * @l3l4p: detectable L3 and L4 integrity check is processed by the hardware
212 * @ipe: IP checksum error
213 * @eipe: external (outermost) IP header (only for tunels)
214 * @eudpe: external (outermost) UDP checksum error (only for tunels)
215 * @ipv6exadd: IPv6 header with extension headers
216 * @l4e: L4 integrity error
217 * @pprs: set for packets that skip checksum calculation in the HW pre parser
218 * @nat: the packet is a UDP tunneled packet
219 * @raw_csum_valid: set if raw checksum is valid
220 * @pad: padding to naturally align raw_csum field
221 * @raw_csum: raw checksum
222 */
223struct libeth_rx_csum {
224 u32 l3l4p:1;
225 u32 ipe:1;
226 u32 eipe:1;
227 u32 eudpe:1;
228 u32 ipv6exadd:1;
229 u32 l4e:1;
230 u32 pprs:1;
231 u32 nat:1;
232
233 u32 raw_csum_valid:1;
234 u32 pad:7;
235 u32 raw_csum:16;
236};
237
238/**
239 * struct libeth_rqe_info - receive queue element info
240 * @len: packet length
241 * @ptype: packet type based on types programmed into the device
242 * @eop: whether it's the last fragment of the packet
243 * @rxe: MAC errors: CRC, Alignment, Oversize, Undersizes, Length error
244 * @vlan: C-VLAN or S-VLAN tag depending on the VLAN offload configuration
245 */
246struct libeth_rqe_info {
247 u32 len;
248
249 u32 ptype:14;
250 u32 eop:1;
251 u32 rxe:1;
252
253 u32 vlan:16;
254};
255
256void libeth_rx_pt_gen_hash_type(struct libeth_rx_pt *pt);
257
258/**
259 * libeth_rx_pt_get_ip_ver - get IP version from a packet type structure
260 * @pt: packet type params
261 *
262 * Wrapper to compile out the IPv6 code from the drivers when not supported
263 * by the kernel.
264 *
265 * Return: @pt.outer_ip or stub for IPv6 when not compiled-in.
266 */
267static inline u32 libeth_rx_pt_get_ip_ver(struct libeth_rx_pt pt)
268{
269#if !IS_ENABLED(CONFIG_IPV6)
270 switch (pt.outer_ip) {
271 case LIBETH_RX_PT_OUTER_IPV4:
272 return LIBETH_RX_PT_OUTER_IPV4;
273 default:
274 return LIBETH_RX_PT_OUTER_L2;
275 }
276#else
277 return pt.outer_ip;
278#endif
279}
280
281/* libeth_has_*() can be used to quickly check whether the HW metadata is
282 * available to avoid further expensive processing such as descriptor reads.
283 * They already check for the corresponding netdev feature to be enabled,
284 * thus can be used as drop-in replacements.
285 */
286
287static inline bool libeth_rx_pt_has_checksum(const struct net_device *dev,
288 struct libeth_rx_pt pt)
289{
290 /* Non-zero _INNER* is only possible when _OUTER_IPV* is set,
291 * it is enough to check only for the L4 type.
292 */
293 return likely(pt.inner_prot > LIBETH_RX_PT_INNER_NONE &&
294 (dev->features & NETIF_F_RXCSUM));
295}
296
297static inline bool libeth_rx_pt_has_hash(const struct net_device *dev,
298 struct libeth_rx_pt pt)
299{
300 return likely(pt.payload_layer > LIBETH_RX_PT_PAYLOAD_NONE &&
301 (dev->features & NETIF_F_RXHASH));
302}
303
304/**
305 * libeth_rx_pt_set_hash - fill in skb hash value basing on the PT
306 * @skb: skb to fill the hash in
307 * @hash: 32-bit hash value from the descriptor
308 * @pt: packet type
309 */
310static inline void libeth_rx_pt_set_hash(struct sk_buff *skb, u32 hash,
311 struct libeth_rx_pt pt)
312{
313 skb_set_hash(skb, hash, pt.payload_layer);
314}
315
316#endif /* __LIBETH_RX_H */