projects/linux·blob master · refs

525 lines · 16.7 KiB

1/* SPDX-License-Identifier: GPL-2.0
2 *
3 * page_pool/helpers.h
4 * Author: Jesper Dangaard Brouer <[email protected]>
5 * Copyright (C) 2016 Red Hat, Inc.
6 */
7
8/**
9 * DOC: page_pool allocator
10 *
11 * The page_pool allocator is optimized for recycling page or page fragment used
12 * by skb packet and xdp frame.
13 *
14 * Basic use involves replacing any alloc_pages() calls with page_pool_alloc(),
15 * which allocate memory with or without page splitting depending on the
16 * requested memory size.
17 *
18 * If the driver knows that it always requires full pages or its allocations are
19 * always smaller than half a page, it can use one of the more specific API
20 * calls:
21 *
22 * 1. page_pool_alloc_pages(): allocate memory without page splitting when
23 * driver knows that the memory it need is always bigger than half of the page
24 * allocated from page pool. There is no cache line dirtying for 'struct page'
25 * when a page is recycled back to the page pool.
26 *
27 * 2. page_pool_alloc_frag(): allocate memory with page splitting when driver
28 * knows that the memory it need is always smaller than or equal to half of the
29 * page allocated from page pool. Page splitting enables memory saving and thus
30 * avoids TLB/cache miss for data access, but there also is some cost to
31 * implement page splitting, mainly some cache line dirtying/bouncing for
32 * 'struct page' and atomic operation for page->pp_ref_count.
33 *
34 * The API keeps track of in-flight pages, in order to let API users know when
35 * it is safe to free a page_pool object, the API users must call
36 * page_pool_put_page() or page_pool_free_va() to free the page_pool object, or
37 * attach the page_pool object to a page_pool-aware object like skbs marked with
38 * skb_mark_for_recycle().
39 *
40 * page_pool_put_page() may be called multiple times on the same page if a page
41 * is split into multiple fragments. For the last fragment, it will either
42 * recycle the page, or in case of page->_refcount > 1, it will release the DMA
43 * mapping and in-flight state accounting.
44 *
45 * dma_sync_single_range_for_device() is only called for the last fragment when
46 * page_pool is created with PP_FLAG_DMA_SYNC_DEV flag, so it depends on the
47 * last freed fragment to do the sync_for_device operation for all fragments in
48 * the same page when a page is split. The API user must setup pool->p.max_len
49 * and pool->p.offset correctly and ensure that page_pool_put_page() is called
50 * with dma_sync_size being -1 for fragment API.
51 */
52#ifndef _NET_PAGE_POOL_HELPERS_H
53#define _NET_PAGE_POOL_HELPERS_H
54
55#include <linux/dma-mapping.h>
56
57#include <net/page_pool/types.h>
58#include <net/net_debug.h>
59#include <net/netmem.h>
60
61#ifdef CONFIG_PAGE_POOL_STATS
62/* Deprecated driver-facing API, use netlink instead */
63int page_pool_ethtool_stats_get_count(void);
64u8 *page_pool_ethtool_stats_get_strings(u8 *data);
65u64 *page_pool_ethtool_stats_get(u64 *data, const void *stats);
66
67void page_pool_get_stats(const struct page_pool *pool,
68 struct page_pool_stats *stats);
69#else
70static inline int page_pool_ethtool_stats_get_count(void)
71{
72 return 0;
73}
74
75static inline u8 *page_pool_ethtool_stats_get_strings(u8 *data)
76{
77 return data;
78}
79
80static inline u64 *page_pool_ethtool_stats_get(u64 *data, const void *stats)
81{
82 return data;
83}
84#endif
85
86/**
87 * page_pool_dev_alloc_pages() - allocate a page.
88 * @pool: pool from which to allocate
89 *
90 * Get a page from the page allocator or page_pool caches.
91 */
92static inline struct page *page_pool_dev_alloc_pages(struct page_pool *pool)
93{
94 gfp_t gfp = (GFP_ATOMIC | __GFP_NOWARN);
95
96 return page_pool_alloc_pages(pool, gfp);
97}
98
99/**
100 * page_pool_dev_alloc_frag() - allocate a page fragment.
101 * @pool: pool from which to allocate
102 * @offset: offset to the allocated page
103 * @size: requested size
104 *
105 * Get a page fragment from the page allocator or page_pool caches.
106 *
107 * Return: allocated page fragment, otherwise return NULL.
108 */
109static inline struct page *page_pool_dev_alloc_frag(struct page_pool *pool,
110 unsigned int *offset,
111 unsigned int size)
112{
113 gfp_t gfp = (GFP_ATOMIC | __GFP_NOWARN);
114
115 return page_pool_alloc_frag(pool, offset, size, gfp);
116}
117
118static inline netmem_ref page_pool_alloc_netmem(struct page_pool *pool,
119 unsigned int *offset,
120 unsigned int *size, gfp_t gfp)
121{
122 unsigned int max_size = PAGE_SIZE << pool->p.order;
123 netmem_ref netmem;
124
125 if ((*size << 1) > max_size) {
126 *size = max_size;
127 *offset = 0;
128 return page_pool_alloc_netmems(pool, gfp);
129 }
130
131 netmem = page_pool_alloc_frag_netmem(pool, offset, *size, gfp);
132 if (unlikely(!netmem))
133 return 0;
134
135 /* There is very likely not enough space for another fragment, so append
136 * the remaining size to the current fragment to avoid truesize
137 * underestimate problem.
138 */
139 if (pool->frag_offset + *size > max_size) {
140 *size = max_size - *offset;
141 pool->frag_offset = max_size;
142 }
143
144 return netmem;
145}
146
147static inline netmem_ref page_pool_dev_alloc_netmem(struct page_pool *pool,
148 unsigned int *offset,
149 unsigned int *size)
150{
151 gfp_t gfp = GFP_ATOMIC | __GFP_NOWARN;
152
153 return page_pool_alloc_netmem(pool, offset, size, gfp);
154}
155
156static inline netmem_ref page_pool_dev_alloc_netmems(struct page_pool *pool)
157{
158 gfp_t gfp = GFP_ATOMIC | __GFP_NOWARN;
159
160 return page_pool_alloc_netmems(pool, gfp);
161}
162
163static inline struct page *page_pool_alloc(struct page_pool *pool,
164 unsigned int *offset,
165 unsigned int *size, gfp_t gfp)
166{
167 return netmem_to_page(page_pool_alloc_netmem(pool, offset, size, gfp));
168}
169
170/**
171 * page_pool_dev_alloc() - allocate a page or a page fragment.
172 * @pool: pool from which to allocate
173 * @offset: offset to the allocated page
174 * @size: in as the requested size, out as the allocated size
175 *
176 * Get a page or a page fragment from the page allocator or page_pool caches
177 * depending on the requested size in order to allocate memory with least memory
178 * utilization and performance penalty.
179 *
180 * Return: allocated page or page fragment, otherwise return NULL.
181 */
182static inline struct page *page_pool_dev_alloc(struct page_pool *pool,
183 unsigned int *offset,
184 unsigned int *size)
185{
186 gfp_t gfp = (GFP_ATOMIC | __GFP_NOWARN);
187
188 return page_pool_alloc(pool, offset, size, gfp);
189}
190
191static inline void *page_pool_alloc_va(struct page_pool *pool,
192 unsigned int *size, gfp_t gfp)
193{
194 unsigned int offset;
195 struct page *page;
196
197 /* Mask off __GFP_HIGHMEM to ensure we can use page_address() */
198 page = page_pool_alloc(pool, &offset, size, gfp & ~__GFP_HIGHMEM);
199 if (unlikely(!page))
200 return NULL;
201
202 return page_address(page) + offset;
203}
204
205/**
206 * page_pool_dev_alloc_va() - allocate a page or a page fragment and return its
207 * va.
208 * @pool: pool from which to allocate
209 * @size: in as the requested size, out as the allocated size
210 *
211 * This is just a thin wrapper around the page_pool_alloc() API, and
212 * it returns va of the allocated page or page fragment.
213 *
214 * Return: the va for the allocated page or page fragment, otherwise return NULL.
215 */
216static inline void *page_pool_dev_alloc_va(struct page_pool *pool,
217 unsigned int *size)
218{
219 gfp_t gfp = (GFP_ATOMIC | __GFP_NOWARN);
220
221 return page_pool_alloc_va(pool, size, gfp);
222}
223
224/**
225 * page_pool_get_dma_dir() - Retrieve the stored DMA direction.
226 * @pool: pool from which page was allocated
227 *
228 * Get the stored dma direction. A driver might decide to store this locally
229 * and avoid the extra cache line from page_pool to determine the direction.
230 */
231static inline enum dma_data_direction
232page_pool_get_dma_dir(const struct page_pool *pool)
233{
234 return pool->p.dma_dir;
235}
236
237static inline void page_pool_fragment_netmem(netmem_ref netmem, long nr)
238{
239 atomic_long_set(netmem_get_pp_ref_count_ref(netmem), nr);
240}
241
242/**
243 * page_pool_fragment_page() - split a fresh page into fragments
244 * @page: page to split
245 * @nr: references to set
246 *
247 * pp_ref_count represents the number of outstanding references to the page,
248 * which will be freed using page_pool APIs (rather than page allocator APIs
249 * like put_page()). Such references are usually held by page_pool-aware
250 * objects like skbs marked for page pool recycling.
251 *
252 * This helper allows the caller to take (set) multiple references to a
253 * freshly allocated page. The page must be freshly allocated (have a
254 * pp_ref_count of 1). This is commonly done by drivers and
255 * "fragment allocators" to save atomic operations - either when they know
256 * upfront how many references they will need; or to take MAX references and
257 * return the unused ones with a single atomic dec(), instead of performing
258 * multiple atomic inc() operations.
259 */
260static inline void page_pool_fragment_page(struct page *page, long nr)
261{
262 page_pool_fragment_netmem(page_to_netmem(page), nr);
263}
264
265static inline long page_pool_unref_netmem(netmem_ref netmem, long nr)
266{
267 atomic_long_t *pp_ref_count = netmem_get_pp_ref_count_ref(netmem);
268 long ret;
269
270 /* If nr == pp_ref_count then we have cleared all remaining
271 * references to the page:
272 * 1. 'n == 1': no need to actually overwrite it.
273 * 2. 'n != 1': overwrite it with one, which is the rare case
274 * for pp_ref_count draining.
275 *
276 * The main advantage to doing this is that not only we avoid a atomic
277 * update, as an atomic_read is generally a much cheaper operation than
278 * an atomic update, especially when dealing with a page that may be
279 * referenced by only 2 or 3 users; but also unify the pp_ref_count
280 * handling by ensuring all pages have partitioned into only 1 piece
281 * initially, and only overwrite it when the page is partitioned into
282 * more than one piece.
283 */
284 if (atomic_long_read(pp_ref_count) == nr) {
285 /* As we have ensured nr is always one for constant case using
286 * the BUILD_BUG_ON(), only need to handle the non-constant case
287 * here for pp_ref_count draining, which is a rare case.
288 */
289 BUILD_BUG_ON(__builtin_constant_p(nr) && nr != 1);
290 if (!__builtin_constant_p(nr))
291 atomic_long_set(pp_ref_count, 1);
292
293 return 0;
294 }
295
296 ret = atomic_long_sub_return(nr, pp_ref_count);
297 WARN_ON(ret < 0);
298
299 /* We are the last user here too, reset pp_ref_count back to 1 to
300 * ensure all pages have been partitioned into 1 piece initially,
301 * this should be the rare case when the last two fragment users call
302 * page_pool_unref_page() currently.
303 */
304 if (unlikely(!ret))
305 atomic_long_set(pp_ref_count, 1);
306
307 return ret;
308}
309
310static inline long page_pool_unref_page(struct page *page, long nr)
311{
312 return page_pool_unref_netmem(page_to_netmem(page), nr);
313}
314
315static inline void page_pool_ref_netmem(netmem_ref netmem)
316{
317 atomic_long_inc(netmem_get_pp_ref_count_ref(netmem));
318}
319
320static inline void page_pool_ref_page(struct page *page)
321{
322 page_pool_ref_netmem(page_to_netmem(page));
323}
324
325static inline bool page_pool_unref_and_test(netmem_ref netmem)
326{
327 /* If page_pool_unref_page() returns 0, we were the last user */
328 return page_pool_unref_netmem(netmem, 1) == 0;
329}
330
331static inline void page_pool_put_netmem(struct page_pool *pool,
332 netmem_ref netmem,
333 unsigned int dma_sync_size,
334 bool allow_direct)
335{
336 /* When page_pool isn't compiled-in, net/core/xdp.c doesn't
337 * allow registering MEM_TYPE_PAGE_POOL, but shield linker.
338 */
339#ifdef CONFIG_PAGE_POOL
340 if (!page_pool_unref_and_test(netmem))
341 return;
342
343 page_pool_put_unrefed_netmem(pool, netmem, dma_sync_size, allow_direct);
344#endif
345}
346
347/**
348 * page_pool_put_page() - release a reference to a page pool page
349 * @pool: pool from which page was allocated
350 * @page: page to release a reference on
351 * @dma_sync_size: how much of the page may have been touched by the device
352 * @allow_direct: released by the consumer, allow lockless caching
353 *
354 * The outcome of this depends on the page refcnt. If the driver bumps
355 * the refcnt > 1 this will unmap the page. If the page refcnt is 1
356 * the allocator owns the page and will try to recycle it in one of the pool
357 * caches. If PP_FLAG_DMA_SYNC_DEV is set, the page will be synced for_device
358 * using dma_sync_single_range_for_device().
359 */
360static inline void page_pool_put_page(struct page_pool *pool,
361 struct page *page,
362 unsigned int dma_sync_size,
363 bool allow_direct)
364{
365 page_pool_put_netmem(pool, page_to_netmem(page), dma_sync_size,
366 allow_direct);
367}
368
369static inline void page_pool_put_full_netmem(struct page_pool *pool,
370 netmem_ref netmem,
371 bool allow_direct)
372{
373 page_pool_put_netmem(pool, netmem, -1, allow_direct);
374}
375
376/**
377 * page_pool_put_full_page() - release a reference on a page pool page
378 * @pool: pool from which page was allocated
379 * @page: page to release a reference on
380 * @allow_direct: released by the consumer, allow lockless caching
381 *
382 * Similar to page_pool_put_page(), but will DMA sync the entire memory area
383 * as configured in &page_pool_params.max_len.
384 */
385static inline void page_pool_put_full_page(struct page_pool *pool,
386 struct page *page, bool allow_direct)
387{
388 page_pool_put_netmem(pool, page_to_netmem(page), -1, allow_direct);
389}
390
391/**
392 * page_pool_recycle_direct() - release a reference on a page pool page
393 * @pool: pool from which page was allocated
394 * @page: page to release a reference on
395 *
396 * Similar to page_pool_put_full_page() but caller must guarantee safe context
397 * (e.g NAPI), since it will recycle the page directly into the pool fast cache.
398 */
399static inline void page_pool_recycle_direct(struct page_pool *pool,
400 struct page *page)
401{
402 page_pool_put_full_page(pool, page, true);
403}
404
405static inline void page_pool_recycle_direct_netmem(struct page_pool *pool,
406 netmem_ref netmem)
407{
408 page_pool_put_full_netmem(pool, netmem, true);
409}
410
411#define PAGE_POOL_32BIT_ARCH_WITH_64BIT_DMA \
412 (sizeof(dma_addr_t) > sizeof(unsigned long))
413
414/**
415 * page_pool_free_va() - free a va into the page_pool
416 * @pool: pool from which va was allocated
417 * @va: va to be freed
418 * @allow_direct: freed by the consumer, allow lockless caching
419 *
420 * Free a va allocated from page_pool_allo_va().
421 */
422static inline void page_pool_free_va(struct page_pool *pool, void *va,
423 bool allow_direct)
424{
425 page_pool_put_page(pool, virt_to_head_page(va), -1, allow_direct);
426}
427
428static inline dma_addr_t page_pool_get_dma_addr_netmem(netmem_ref netmem)
429{
430 dma_addr_t ret = netmem_get_dma_addr(netmem);
431
432 if (PAGE_POOL_32BIT_ARCH_WITH_64BIT_DMA)
433 ret <<= PAGE_SHIFT;
434
435 return ret;
436}
437
438/**
439 * page_pool_get_dma_addr() - Retrieve the stored DMA address.
440 * @page: page allocated from a page pool
441 *
442 * Fetch the DMA address of the page. The page pool to which the page belongs
443 * must had been created with PP_FLAG_DMA_MAP.
444 */
445static inline dma_addr_t page_pool_get_dma_addr(const struct page *page)
446{
447 return page_pool_get_dma_addr_netmem(page_to_netmem(page));
448}
449
450static inline void __page_pool_dma_sync_for_cpu(const struct page_pool *pool,
451 const dma_addr_t dma_addr,
452 u32 offset, u32 dma_sync_size)
453{
454 dma_sync_single_range_for_cpu(pool->p.dev, dma_addr,
455 offset + pool->p.offset, dma_sync_size,
456 page_pool_get_dma_dir(pool));
457}
458
459/**
460 * page_pool_dma_sync_for_cpu - sync Rx page for CPU after it's written by HW
461 * @pool: &page_pool the @page belongs to
462 * @page: page to sync
463 * @offset: offset from page start to "hard" start if using PP frags
464 * @dma_sync_size: size of the data written to the page
465 *
466 * Can be used as a shorthand to sync Rx pages before accessing them in the
467 * driver. Caller must ensure the pool was created with ``PP_FLAG_DMA_MAP``.
468 * Note that this version performs DMA sync unconditionally, even if the
469 * associated PP doesn't perform sync-for-device.
470 */
471static inline void page_pool_dma_sync_for_cpu(const struct page_pool *pool,
472 const struct page *page,
473 u32 offset, u32 dma_sync_size)
474{
475 __page_pool_dma_sync_for_cpu(pool, page_pool_get_dma_addr(page), offset,
476 dma_sync_size);
477}
478
479static inline void
480page_pool_dma_sync_netmem_for_cpu(const struct page_pool *pool,
481 const netmem_ref netmem, u32 offset,
482 u32 dma_sync_size)
483{
484 if (!pool->dma_sync_for_cpu)
485 return;
486
487 __page_pool_dma_sync_for_cpu(pool,
488 page_pool_get_dma_addr_netmem(netmem),
489 offset, dma_sync_size);
490}
491
492static inline void page_pool_get(struct page_pool *pool)
493{
494 refcount_inc(&pool->user_cnt);
495}
496
497static inline bool page_pool_put(struct page_pool *pool)
498{
499 return refcount_dec_and_test(&pool->user_cnt);
500}
501
502static inline void page_pool_nid_changed(struct page_pool *pool, int new_nid)
503{
504 if (unlikely(pool->p.nid != new_nid))
505 page_pool_update_nid(pool, new_nid);
506}
507
508/**
509 * page_pool_is_unreadable() - will allocated buffers be unreadable for the CPU
510 * @pool: queried page pool
511 *
512 * Check if page pool will return buffers which are unreadable to the CPU /
513 * kernel. This will only be the case if user space bound a memory provider (mp)
514 * which returns unreadable memory to the queue served by the page pool.
515 * If %PP_FLAG_ALLOW_UNREADABLE_NETMEM was set but there is no mp bound
516 * this helper will return false. See also netif_rxq_has_unreadable_mp().
517 *
518 * Return: true if memory allocated by the page pool may be unreadable
519 */
520static inline bool page_pool_is_unreadable(struct page_pool *pool)
521{
522 return !!pool->mp_ops;
523}
524
525#endif /* _NET_PAGE_POOL_HELPERS_H */