projects/linux·blob master · refs

560 lines · 13.4 KiB

1/* SPDX-License-Identifier: GPL-2.0 */
2/* Interface for implementing AF_XDP zero-copy support in drivers.
3 * Copyright(c) 2020 Intel Corporation.
4 */
5
6#ifndef _LINUX_XDP_SOCK_DRV_H
7#define _LINUX_XDP_SOCK_DRV_H
8
9#include <net/xdp_sock.h>
10#include <net/xsk_buff_pool.h>
11
12#define XDP_UMEM_MIN_CHUNK_SHIFT 11
13#define XDP_UMEM_MIN_CHUNK_SIZE (1 << XDP_UMEM_MIN_CHUNK_SHIFT)
14
15#define NETDEV_XDP_ACT_XSK (NETDEV_XDP_ACT_BASIC | \
16 NETDEV_XDP_ACT_REDIRECT | \
17 NETDEV_XDP_ACT_XSK_ZEROCOPY)
18
19struct xsk_cb_desc {
20 void *src;
21 u8 off;
22 u8 bytes;
23};
24
25#ifdef CONFIG_XDP_SOCKETS
26
27void xsk_tx_completed(struct xsk_buff_pool *pool, u32 nb_entries);
28bool xsk_tx_peek_desc(struct xsk_buff_pool *pool, struct xdp_desc *desc);
29u32 xsk_tx_peek_release_desc_batch(struct xsk_buff_pool *pool, u32 max);
30void xsk_tx_release(struct xsk_buff_pool *pool);
31struct xsk_buff_pool *xsk_get_pool_from_qid(struct net_device *dev,
32 u16 queue_id);
33void xsk_set_rx_need_wakeup(struct xsk_buff_pool *pool);
34void xsk_set_tx_need_wakeup(struct xsk_buff_pool *pool);
35void xsk_clear_rx_need_wakeup(struct xsk_buff_pool *pool);
36void xsk_clear_tx_need_wakeup(struct xsk_buff_pool *pool);
37bool xsk_uses_need_wakeup(struct xsk_buff_pool *pool);
38
39static inline u32 xsk_pool_get_headroom(struct xsk_buff_pool *pool)
40{
41 return XDP_PACKET_HEADROOM + pool->headroom;
42}
43
44static inline u32 xsk_pool_get_tailroom(bool mbuf)
45{
46 return mbuf ? SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) : 0;
47}
48
49static inline u32 xsk_pool_get_chunk_size(struct xsk_buff_pool *pool)
50{
51 return pool->chunk_size;
52}
53
54static inline u32 __xsk_pool_get_rx_frame_size(struct xsk_buff_pool *pool)
55{
56 return xsk_pool_get_chunk_size(pool) - xsk_pool_get_headroom(pool);
57}
58
59static inline u32 xsk_pool_get_rx_frame_size(struct xsk_buff_pool *pool)
60{
61 u32 frame_size = __xsk_pool_get_rx_frame_size(pool);
62 struct xdp_umem *umem = pool->umem;
63 bool mbuf;
64
65 /* Reserve tailroom only for zero-copy pools that opted into
66 * multi-buffer. The reserved area is used for skb_shared_info,
67 * matching the XDP core's xdp_data_hard_end() layout.
68 */
69 mbuf = pool->dev && (umem->flags & XDP_UMEM_SG_FLAG);
70 frame_size -= xsk_pool_get_tailroom(mbuf);
71
72 return ALIGN_DOWN(frame_size, 128);
73}
74
75static inline u32 xsk_pool_get_rx_frag_step(struct xsk_buff_pool *pool)
76{
77 return pool->unaligned ? 0 : xsk_pool_get_chunk_size(pool);
78}
79
80static inline void xsk_pool_set_rxq_info(struct xsk_buff_pool *pool,
81 struct xdp_rxq_info *rxq)
82{
83 xp_set_rxq_info(pool, rxq);
84}
85
86static inline void xsk_pool_fill_cb(struct xsk_buff_pool *pool,
87 struct xsk_cb_desc *desc)
88{
89 xp_fill_cb(pool, desc);
90}
91
92static inline void xsk_pool_dma_unmap(struct xsk_buff_pool *pool,
93 unsigned long attrs)
94{
95 xp_dma_unmap(pool, attrs);
96}
97
98static inline int xsk_pool_dma_map(struct xsk_buff_pool *pool,
99 struct device *dev, unsigned long attrs)
100{
101 struct xdp_umem *umem = pool->umem;
102
103 return xp_dma_map(pool, dev, attrs, umem->pgs, umem->npgs);
104}
105
106static inline dma_addr_t xsk_buff_xdp_get_dma(struct xdp_buff *xdp)
107{
108 struct xdp_buff_xsk *xskb = container_of(xdp, struct xdp_buff_xsk, xdp);
109
110 return xp_get_dma(xskb);
111}
112
113static inline dma_addr_t xsk_buff_xdp_get_frame_dma(struct xdp_buff *xdp)
114{
115 struct xdp_buff_xsk *xskb = container_of(xdp, struct xdp_buff_xsk, xdp);
116
117 return xp_get_frame_dma(xskb);
118}
119
120static inline struct xdp_buff *xsk_buff_alloc(struct xsk_buff_pool *pool)
121{
122 return xp_alloc(pool);
123}
124
125static inline bool xsk_is_eop_desc(const struct xdp_desc *desc)
126{
127 return !xp_mb_desc(desc);
128}
129
130/* Returns as many entries as possible up to max. 0 <= N <= max. */
131static inline u32 xsk_buff_alloc_batch(struct xsk_buff_pool *pool, struct xdp_buff **xdp, u32 max)
132{
133 return xp_alloc_batch(pool, xdp, max);
134}
135
136static inline bool xsk_buff_can_alloc(struct xsk_buff_pool *pool, u32 count)
137{
138 return xp_can_alloc(pool, count);
139}
140
141static inline void xsk_buff_free(struct xdp_buff *xdp)
142{
143 struct xdp_buff_xsk *xskb = container_of(xdp, struct xdp_buff_xsk, xdp);
144 struct list_head *xskb_list = &xskb->pool->xskb_list;
145 struct xdp_buff_xsk *pos, *tmp;
146
147 if (likely(!xdp_buff_has_frags(xdp)))
148 goto out;
149
150 list_for_each_entry_safe(pos, tmp, xskb_list, list_node) {
151 list_del_init(&pos->list_node);
152 xp_free(pos);
153 }
154
155 xdp_get_shared_info_from_buff(xdp)->nr_frags = 0;
156out:
157 xp_free(xskb);
158}
159
160static inline bool xsk_buff_add_frag(struct xdp_buff *head,
161 struct xdp_buff *xdp)
162{
163 const void *data = xdp->data;
164 struct xdp_buff_xsk *frag;
165
166 if (!__xdp_buff_add_frag(head, virt_to_netmem(data),
167 offset_in_page(data), xdp->data_end - data,
168 xdp->frame_sz, false))
169 return false;
170
171 frag = container_of(xdp, struct xdp_buff_xsk, xdp);
172 list_add_tail(&frag->list_node, &frag->pool->xskb_list);
173
174 return true;
175}
176
177static inline struct xdp_buff *xsk_buff_get_frag(const struct xdp_buff *first)
178{
179 struct xdp_buff_xsk *xskb = container_of(first, struct xdp_buff_xsk, xdp);
180 struct xdp_buff *ret = NULL;
181 struct xdp_buff_xsk *frag;
182
183 frag = list_first_entry_or_null(&xskb->pool->xskb_list,
184 struct xdp_buff_xsk, list_node);
185 if (frag) {
186 list_del_init(&frag->list_node);
187 ret = &frag->xdp;
188 }
189
190 return ret;
191}
192
193static inline void xsk_buff_del_frag(struct xdp_buff *xdp)
194{
195 struct xdp_buff_xsk *xskb = container_of(xdp, struct xdp_buff_xsk, xdp);
196
197 list_del_init(&xskb->list_node);
198}
199
200static inline struct xdp_buff *xsk_buff_get_head(struct xdp_buff *first)
201{
202 struct xdp_buff_xsk *xskb = container_of(first, struct xdp_buff_xsk, xdp);
203 struct xdp_buff_xsk *frag;
204
205 frag = list_first_entry(&xskb->pool->xskb_list, struct xdp_buff_xsk,
206 list_node);
207 return &frag->xdp;
208}
209
210static inline struct xdp_buff *xsk_buff_get_tail(struct xdp_buff *first)
211{
212 struct xdp_buff_xsk *xskb = container_of(first, struct xdp_buff_xsk, xdp);
213 struct xdp_buff_xsk *frag;
214
215 frag = list_last_entry(&xskb->pool->xskb_list, struct xdp_buff_xsk,
216 list_node);
217 return &frag->xdp;
218}
219
220static inline void xsk_buff_set_size(struct xdp_buff *xdp, u32 size)
221{
222 xdp->data = xdp->data_hard_start + XDP_PACKET_HEADROOM;
223 xdp->data_meta = xdp->data;
224 xdp->data_end = xdp->data + size;
225 xdp->flags = 0;
226}
227
228static inline dma_addr_t xsk_buff_raw_get_dma(struct xsk_buff_pool *pool,
229 u64 addr)
230{
231 return xp_raw_get_dma(pool, addr);
232}
233
234static inline void *xsk_buff_raw_get_data(struct xsk_buff_pool *pool, u64 addr)
235{
236 return xp_raw_get_data(pool, addr);
237}
238
239/**
240 * xsk_buff_raw_get_ctx - get &xdp_desc context
241 * @pool: XSk buff pool desc address belongs to
242 * @addr: desc address (from userspace)
243 * @options: desc options (from userspace)
244 *
245 * Wrapper for xp_raw_get_ctx() to be used in drivers, see its kdoc for
246 * details.
247 *
248 * Return: new &xdp_desc_ctx struct containing desc's DMA address and metadata
249 * pointer, if it is present (initialized to %NULL otherwise).
250 */
251static inline struct xdp_desc_ctx
252xsk_buff_raw_get_ctx(const struct xsk_buff_pool *pool, u64 addr, u32 options)
253{
254 return xp_raw_get_ctx(pool, addr, options);
255}
256
257#define XDP_TXMD_FLAGS_VALID ( \
258 XDP_TXMD_FLAGS_TIMESTAMP | \
259 XDP_TXMD_FLAGS_CHECKSUM | \
260 XDP_TXMD_FLAGS_LAUNCH_TIME | \
261 0)
262
263static inline bool
264xsk_buff_valid_tx_metadata(const struct xsk_buff_pool *pool,
265 const struct xsk_tx_metadata *meta, u64 *flags)
266{
267 *flags = READ_ONCE(meta->flags);
268 if (*flags & XDP_TXMD_FLAGS_LAUNCH_TIME)
269 if (pool->tx_metadata_len <
270 offsetofend(struct xsk_tx_metadata, request.launch_time))
271 return false;
272 return !(*flags & ~XDP_TXMD_FLAGS_VALID);
273}
274
275/**
276 * xsk_tx_metadata_request - Evaluate AF_XDP TX metadata at submission
277 * and call appropriate xsk_tx_metadata_ops operation.
278 * @pool: pointer to AF_XDP buffer pool, used to validate the metadata
279 * @pmeta: pointer to pointer to AF_XDP metadata area
280 * @ops: pointer to struct xsk_tx_metadata_ops
281 * @priv: pointer to driver-private area
282 *
283 * This function should be called by the networking device when
284 * it prepares AF_XDP egress packet.
285 */
286static inline void
287xsk_tx_metadata_request(const struct xsk_buff_pool *pool,
288 struct xsk_tx_metadata **pmeta,
289 const struct xsk_tx_metadata_ops *ops, void *priv)
290{
291 const struct xsk_tx_metadata *meta = *pmeta;
292 u64 flags;
293
294 if (!meta)
295 return;
296
297 if (unlikely(!xsk_buff_valid_tx_metadata(pool, meta, &flags))) {
298 *pmeta = NULL;
299 return; /* no way to signal the error to the user */
300 }
301
302 if (ops->tmo_request_launch_time)
303 if (flags & XDP_TXMD_FLAGS_LAUNCH_TIME)
304 ops->tmo_request_launch_time(
305 READ_ONCE(meta->request.launch_time), priv);
306
307 if (ops->tmo_request_timestamp)
308 if (flags & XDP_TXMD_FLAGS_TIMESTAMP)
309 ops->tmo_request_timestamp(priv);
310
311 if (ops->tmo_request_checksum)
312 if (flags & XDP_TXMD_FLAGS_CHECKSUM)
313 ops->tmo_request_checksum(
314 READ_ONCE(meta->request.csum_start),
315 READ_ONCE(meta->request.csum_offset), priv);
316
317 if (!(flags & XDP_TXMD_FLAGS_TIMESTAMP))
318 *pmeta = NULL;
319}
320
321static inline struct xsk_tx_metadata *
322__xsk_buff_get_metadata(const struct xsk_buff_pool *pool, void *data,
323 unsigned int options)
324{
325 if (!pool->tx_metadata_len || !(options & XDP_TX_METADATA))
326 return NULL;
327
328 return data - pool->tx_metadata_len;
329}
330
331static inline struct xsk_tx_metadata *
332xsk_buff_get_metadata(struct xsk_buff_pool *pool, u64 addr, u32 options)
333{
334 return __xsk_buff_get_metadata(pool, xp_raw_get_data(pool, addr),
335 options);
336}
337
338static inline void xsk_buff_dma_sync_for_cpu(struct xdp_buff *xdp)
339{
340 struct xdp_buff_xsk *xskb = container_of(xdp, struct xdp_buff_xsk, xdp);
341
342 xp_dma_sync_for_cpu(xskb);
343}
344
345static inline void xsk_buff_raw_dma_sync_for_device(struct xsk_buff_pool *pool,
346 dma_addr_t dma,
347 size_t size)
348{
349 xp_dma_sync_for_device(pool, dma, size);
350}
351
352#else
353
354static inline void xsk_tx_completed(struct xsk_buff_pool *pool, u32 nb_entries)
355{
356}
357
358static inline bool xsk_tx_peek_desc(struct xsk_buff_pool *pool,
359 struct xdp_desc *desc)
360{
361 return false;
362}
363
364static inline u32 xsk_tx_peek_release_desc_batch(struct xsk_buff_pool *pool, u32 max)
365{
366 return 0;
367}
368
369static inline void xsk_tx_release(struct xsk_buff_pool *pool)
370{
371}
372
373static inline struct xsk_buff_pool *
374xsk_get_pool_from_qid(struct net_device *dev, u16 queue_id)
375{
376 return NULL;
377}
378
379static inline void xsk_set_rx_need_wakeup(struct xsk_buff_pool *pool)
380{
381}
382
383static inline void xsk_set_tx_need_wakeup(struct xsk_buff_pool *pool)
384{
385}
386
387static inline void xsk_clear_rx_need_wakeup(struct xsk_buff_pool *pool)
388{
389}
390
391static inline void xsk_clear_tx_need_wakeup(struct xsk_buff_pool *pool)
392{
393}
394
395static inline bool xsk_uses_need_wakeup(struct xsk_buff_pool *pool)
396{
397 return false;
398}
399
400static inline u32 xsk_pool_get_headroom(struct xsk_buff_pool *pool)
401{
402 return 0;
403}
404
405static inline u32 xsk_pool_get_chunk_size(struct xsk_buff_pool *pool)
406{
407 return 0;
408}
409
410static inline u32 xsk_pool_get_rx_frame_size(struct xsk_buff_pool *pool)
411{
412 return 0;
413}
414
415static inline u32 xsk_pool_get_rx_frag_step(struct xsk_buff_pool *pool)
416{
417 return 0;
418}
419
420static inline void xsk_pool_set_rxq_info(struct xsk_buff_pool *pool,
421 struct xdp_rxq_info *rxq)
422{
423}
424
425static inline void xsk_pool_fill_cb(struct xsk_buff_pool *pool,
426 struct xsk_cb_desc *desc)
427{
428}
429
430static inline void xsk_pool_dma_unmap(struct xsk_buff_pool *pool,
431 unsigned long attrs)
432{
433}
434
435static inline int xsk_pool_dma_map(struct xsk_buff_pool *pool,
436 struct device *dev, unsigned long attrs)
437{
438 return 0;
439}
440
441static inline dma_addr_t xsk_buff_xdp_get_dma(struct xdp_buff *xdp)
442{
443 return 0;
444}
445
446static inline dma_addr_t xsk_buff_xdp_get_frame_dma(struct xdp_buff *xdp)
447{
448 return 0;
449}
450
451static inline struct xdp_buff *xsk_buff_alloc(struct xsk_buff_pool *pool)
452{
453 return NULL;
454}
455
456static inline bool xsk_is_eop_desc(const struct xdp_desc *desc)
457{
458 return false;
459}
460
461static inline u32 xsk_buff_alloc_batch(struct xsk_buff_pool *pool, struct xdp_buff **xdp, u32 max)
462{
463 return 0;
464}
465
466static inline bool xsk_buff_can_alloc(struct xsk_buff_pool *pool, u32 count)
467{
468 return false;
469}
470
471static inline void xsk_buff_free(struct xdp_buff *xdp)
472{
473}
474
475static inline bool xsk_buff_add_frag(struct xdp_buff *head,
476 struct xdp_buff *xdp)
477{
478 return false;
479}
480
481static inline struct xdp_buff *xsk_buff_get_frag(const struct xdp_buff *first)
482{
483 return NULL;
484}
485
486static inline void xsk_buff_del_frag(struct xdp_buff *xdp)
487{
488}
489
490static inline struct xdp_buff *xsk_buff_get_head(struct xdp_buff *first)
491{
492 return NULL;
493}
494
495static inline struct xdp_buff *xsk_buff_get_tail(struct xdp_buff *first)
496{
497 return NULL;
498}
499
500static inline void xsk_buff_set_size(struct xdp_buff *xdp, u32 size)
501{
502}
503
504static inline dma_addr_t xsk_buff_raw_get_dma(struct xsk_buff_pool *pool,
505 u64 addr)
506{
507 return 0;
508}
509
510static inline void *xsk_buff_raw_get_data(struct xsk_buff_pool *pool, u64 addr)
511{
512 return NULL;
513}
514
515static inline struct xdp_desc_ctx
516xsk_buff_raw_get_ctx(const struct xsk_buff_pool *pool, u64 addr, u32 options)
517{
518 return (struct xdp_desc_ctx){ };
519}
520
521static inline bool
522xsk_buff_valid_tx_metadata(const struct xsk_buff_pool *pool,
523 const struct xsk_tx_metadata *meta, u64 *flags)
524{
525 return false;
526}
527
528static inline void
529xsk_tx_metadata_request(const struct xsk_buff_pool *pool,
530 struct xsk_tx_metadata **pmeta,
531 const struct xsk_tx_metadata_ops *ops, void *priv)
532{
533}
534
535static inline struct xsk_tx_metadata *
536__xsk_buff_get_metadata(const struct xsk_buff_pool *pool, void *data,
537 unsigned int options)
538{
539 return NULL;
540}
541
542static inline struct xsk_tx_metadata *
543xsk_buff_get_metadata(struct xsk_buff_pool *pool, u64 addr, u32 options)
544{
545 return NULL;
546}
547
548static inline void xsk_buff_dma_sync_for_cpu(struct xdp_buff *xdp)
549{
550}
551
552static inline void xsk_buff_raw_dma_sync_for_device(struct xsk_buff_pool *pool,
553 dma_addr_t dma,
554 size_t size)
555{
556}
557
558#endif /* CONFIG_XDP_SOCKETS */
559
560#endif /* _LINUX_XDP_SOCK_DRV_H */