projects/linux·blob master · refs

1364 lines · 37.0 KiB

1/* SPDX-License-Identifier: GPL-2.0
2 *
3 * page_pool.c
4 * Author: Jesper Dangaard Brouer <[email protected]>
5 * Copyright (C) 2016 Red Hat, Inc.
6 */
7
8#include <linux/error-injection.h>
9#include <linux/types.h>
10#include <linux/kernel.h>
11#include <linux/slab.h>
12#include <linux/device.h>
13
14#include <net/netdev_lock.h>
15#include <net/netdev_rx_queue.h>
16#include <net/page_pool/helpers.h>
17#include <net/page_pool/memory_provider.h>
18#include <net/xdp.h>
19
20#include <linux/dma-direction.h>
21#include <linux/dma-mapping.h>
22#include <linux/page-flags.h>
23#include <linux/mm.h> /* for put_page() */
24#include <linux/poison.h>
25#include <linux/ethtool.h>
26#include <linux/netdevice.h>
27
28#include <trace/events/page_pool.h>
29
30#include "dev.h"
31#include "mp_dmabuf_devmem.h"
32#include "netmem_priv.h"
33#include "page_pool_priv.h"
34
35DEFINE_STATIC_KEY_FALSE(page_pool_mem_providers);
36
37#define DEFER_TIME (msecs_to_jiffies(1000))
38#define DEFER_WARN_INTERVAL (60 * HZ)
39
40#define BIAS_MAX (LONG_MAX >> 1)
41
42#ifdef CONFIG_PAGE_POOL_STATS
43static DEFINE_PER_CPU(struct page_pool_recycle_stats, pp_system_recycle_stats);
44
45/* alloc_stat_inc is intended to be used in softirq context */
46#define alloc_stat_inc(pool, __stat) (pool->alloc_stats.__stat++)
47/* recycle_stat_inc is safe to use when preemption is possible. */
48#define recycle_stat_inc(pool, __stat) \
49 do { \
50 struct page_pool_recycle_stats __percpu *s = pool->recycle_stats; \
51 this_cpu_inc(s->__stat); \
52 } while (0)
53
54#define recycle_stat_add(pool, __stat, val) \
55 do { \
56 struct page_pool_recycle_stats __percpu *s = pool->recycle_stats; \
57 this_cpu_add(s->__stat, val); \
58 } while (0)
59
60static const char pp_stats[][ETH_GSTRING_LEN] = {
61 "rx_pp_alloc_fast",
62 "rx_pp_alloc_slow",
63 "rx_pp_alloc_slow_ho",
64 "rx_pp_alloc_empty",
65 "rx_pp_alloc_refill",
66 "rx_pp_alloc_waive",
67 "rx_pp_recycle_cached",
68 "rx_pp_recycle_cache_full",
69 "rx_pp_recycle_ring",
70 "rx_pp_recycle_ring_full",
71 "rx_pp_recycle_released_ref",
72};
73
74/**
75 * page_pool_get_stats() - fetch page pool stats
76 * @pool: pool from which page was allocated
77 * @stats: struct page_pool_stats to fill in
78 *
79 * Deprecated driver API for querying stats. Page pool stats can be queried
80 * via netdev Netlink.
81 *
82 * Retrieve statistics about the page_pool. This API is only available
83 * if the kernel has been configured with ``CONFIG_PAGE_POOL_STATS=y``.
84 * A pointer to a caller allocated struct page_pool_stats structure
85 * is passed to this API which is filled in. The caller can then report
86 * those stats to the user (perhaps via ethtool, debugfs, etc.).
87 */
88void page_pool_get_stats(const struct page_pool *pool,
89 struct page_pool_stats *stats)
90{
91 int cpu = 0;
92
93 /* The caller is responsible to initialize stats. */
94 stats->alloc_stats.fast += pool->alloc_stats.fast;
95 stats->alloc_stats.slow += pool->alloc_stats.slow;
96 stats->alloc_stats.slow_high_order += pool->alloc_stats.slow_high_order;
97 stats->alloc_stats.empty += pool->alloc_stats.empty;
98 stats->alloc_stats.refill += pool->alloc_stats.refill;
99 stats->alloc_stats.waive += pool->alloc_stats.waive;
100
101 for_each_possible_cpu(cpu) {
102 const struct page_pool_recycle_stats *pcpu =
103 per_cpu_ptr(pool->recycle_stats, cpu);
104
105 stats->recycle_stats.cached += pcpu->cached;
106 stats->recycle_stats.cache_full += pcpu->cache_full;
107 stats->recycle_stats.ring += pcpu->ring;
108 stats->recycle_stats.ring_full += pcpu->ring_full;
109 stats->recycle_stats.released_refcnt += pcpu->released_refcnt;
110 }
111}
112EXPORT_SYMBOL(page_pool_get_stats);
113
114u8 *page_pool_ethtool_stats_get_strings(u8 *data)
115{
116 int i;
117
118 for (i = 0; i < ARRAY_SIZE(pp_stats); i++) {
119 memcpy(data, pp_stats[i], ETH_GSTRING_LEN);
120 data += ETH_GSTRING_LEN;
121 }
122
123 return data;
124}
125EXPORT_SYMBOL(page_pool_ethtool_stats_get_strings);
126
127int page_pool_ethtool_stats_get_count(void)
128{
129 return ARRAY_SIZE(pp_stats);
130}
131EXPORT_SYMBOL(page_pool_ethtool_stats_get_count);
132
133u64 *page_pool_ethtool_stats_get(u64 *data, const void *stats)
134{
135 const struct page_pool_stats *pool_stats = stats;
136
137 *data++ = pool_stats->alloc_stats.fast;
138 *data++ = pool_stats->alloc_stats.slow;
139 *data++ = pool_stats->alloc_stats.slow_high_order;
140 *data++ = pool_stats->alloc_stats.empty;
141 *data++ = pool_stats->alloc_stats.refill;
142 *data++ = pool_stats->alloc_stats.waive;
143 *data++ = pool_stats->recycle_stats.cached;
144 *data++ = pool_stats->recycle_stats.cache_full;
145 *data++ = pool_stats->recycle_stats.ring;
146 *data++ = pool_stats->recycle_stats.ring_full;
147 *data++ = pool_stats->recycle_stats.released_refcnt;
148
149 return data;
150}
151EXPORT_SYMBOL(page_pool_ethtool_stats_get);
152
153#else
154#define alloc_stat_inc(...) do { } while (0)
155#define recycle_stat_inc(...) do { } while (0)
156#define recycle_stat_add(...) do { } while (0)
157#endif
158
159static bool page_pool_producer_lock(struct page_pool *pool)
160 __acquires(&pool->ring.producer_lock)
161{
162 bool in_softirq = in_softirq();
163
164 if (in_softirq)
165 spin_lock(&pool->ring.producer_lock);
166 else
167 spin_lock_bh(&pool->ring.producer_lock);
168
169 return in_softirq;
170}
171
172static void page_pool_producer_unlock(struct page_pool *pool,
173 bool in_softirq)
174 __releases(&pool->ring.producer_lock)
175{
176 if (in_softirq)
177 spin_unlock(&pool->ring.producer_lock);
178 else
179 spin_unlock_bh(&pool->ring.producer_lock);
180}
181
182static void page_pool_struct_check(void)
183{
184 CACHELINE_ASSERT_GROUP_MEMBER(struct page_pool, frag, frag_users);
185 CACHELINE_ASSERT_GROUP_MEMBER(struct page_pool, frag, frag_page);
186 CACHELINE_ASSERT_GROUP_MEMBER(struct page_pool, frag, frag_offset);
187 CACHELINE_ASSERT_GROUP_SIZE(struct page_pool, frag,
188 PAGE_POOL_FRAG_GROUP_ALIGN);
189}
190
191static int page_pool_init(struct page_pool *pool,
192 const struct page_pool_params *params,
193 int cpuid)
194{
195 unsigned int ring_qsize = 1024; /* Default */
196 struct netdev_rx_queue *rxq;
197 int err;
198
199 page_pool_struct_check();
200
201 memcpy(&pool->p, &params->fast, sizeof(pool->p));
202 memcpy(&pool->slow, &params->slow, sizeof(pool->slow));
203
204 pool->cpuid = cpuid;
205 pool->dma_sync_for_cpu = true;
206
207 /* Validate only known flags were used */
208 if (pool->slow.flags & ~PP_FLAG_ALL)
209 return -EINVAL;
210
211 if (pool->p.pool_size)
212 ring_qsize = min(pool->p.pool_size, 16384);
213
214 /* DMA direction is either DMA_FROM_DEVICE or DMA_BIDIRECTIONAL.
215 * DMA_BIDIRECTIONAL is for allowing page used for DMA sending,
216 * which is the XDP_TX use-case.
217 */
218 if (pool->slow.flags & PP_FLAG_DMA_MAP) {
219 if ((pool->p.dma_dir != DMA_FROM_DEVICE) &&
220 (pool->p.dma_dir != DMA_BIDIRECTIONAL))
221 return -EINVAL;
222
223 pool->dma_map = true;
224 }
225
226 if (pool->slow.flags & PP_FLAG_DMA_SYNC_DEV) {
227 /* In order to request DMA-sync-for-device the page
228 * needs to be mapped
229 */
230 if (!(pool->slow.flags & PP_FLAG_DMA_MAP))
231 return -EINVAL;
232
233 if (!pool->p.max_len)
234 return -EINVAL;
235
236 pool->dma_sync = true;
237
238 /* pool->p.offset has to be set according to the address
239 * offset used by the DMA engine to start copying rx data
240 */
241 }
242
243 pool->has_init_callback = !!pool->slow.init_callback;
244
245#ifdef CONFIG_PAGE_POOL_STATS
246 if (!(pool->slow.flags & PP_FLAG_SYSTEM_POOL)) {
247 pool->recycle_stats = alloc_percpu(struct page_pool_recycle_stats);
248 if (!pool->recycle_stats)
249 return -ENOMEM;
250 } else {
251 /* For system page pool instance we use a singular stats object
252 * instead of allocating a separate percpu variable for each
253 * (also percpu) page pool instance.
254 */
255 pool->recycle_stats = &pp_system_recycle_stats;
256 pool->system = true;
257 }
258#endif
259
260 if (ptr_ring_init(&pool->ring, ring_qsize, GFP_KERNEL) < 0) {
261#ifdef CONFIG_PAGE_POOL_STATS
262 if (!pool->system)
263 free_percpu(pool->recycle_stats);
264#endif
265 return -ENOMEM;
266 }
267
268 atomic_set(&pool->pages_state_release_cnt, 0);
269
270 /* Driver calling page_pool_create() also call page_pool_destroy() */
271 refcount_set(&pool->user_cnt, 1);
272
273 xa_init_flags(&pool->dma_mapped, XA_FLAGS_ALLOC1);
274
275 if (pool->slow.flags & PP_FLAG_ALLOW_UNREADABLE_NETMEM) {
276 netdev_assert_locked(pool->slow.netdev);
277 rxq = __netif_get_rx_queue(pool->slow.netdev,
278 pool->slow.queue_idx);
279 pool->mp_priv = rxq->mp_params.mp_priv;
280 pool->mp_ops = rxq->mp_params.mp_ops;
281 }
282
283 if (pool->mp_ops) {
284 if (!pool->dma_map || !pool->dma_sync) {
285 err = -EOPNOTSUPP;
286 goto free_ptr_ring;
287 }
288
289 if (WARN_ON(!is_kernel_rodata((unsigned long)pool->mp_ops))) {
290 err = -EFAULT;
291 goto free_ptr_ring;
292 }
293
294 err = pool->mp_ops->init(pool);
295 if (err) {
296 pr_warn("%s() mem-provider init failed %d\n", __func__,
297 err);
298 goto free_ptr_ring;
299 }
300
301 static_branch_inc(&page_pool_mem_providers);
302 } else if (pool->p.order > MAX_PAGE_ORDER) {
303 err = -EINVAL;
304 goto free_ptr_ring;
305 }
306
307 return 0;
308
309free_ptr_ring:
310 ptr_ring_cleanup(&pool->ring, NULL);
311 xa_destroy(&pool->dma_mapped);
312#ifdef CONFIG_PAGE_POOL_STATS
313 if (!pool->system)
314 free_percpu(pool->recycle_stats);
315#endif
316 return err;
317}
318
319static void page_pool_uninit(struct page_pool *pool)
320{
321 ptr_ring_cleanup(&pool->ring, NULL);
322 xa_destroy(&pool->dma_mapped);
323
324#ifdef CONFIG_PAGE_POOL_STATS
325 if (!pool->system)
326 free_percpu(pool->recycle_stats);
327#endif
328
329 if (pool->mp_ops) {
330 pool->mp_ops->destroy(pool);
331 static_branch_dec(&page_pool_mem_providers);
332 }
333}
334
335/**
336 * page_pool_create_percpu() - create a page pool for a given cpu.
337 * @params: parameters, see struct page_pool_params
338 * @cpuid: cpu identifier
339 */
340struct page_pool *
341page_pool_create_percpu(const struct page_pool_params *params, int cpuid)
342{
343 struct page_pool *pool;
344 int err;
345
346 pool = kzalloc_node(sizeof(*pool), GFP_KERNEL, params->nid);
347 if (!pool)
348 return ERR_PTR(-ENOMEM);
349
350 err = page_pool_init(pool, params, cpuid);
351 if (err < 0)
352 goto err_free;
353
354 err = page_pool_list(pool);
355 if (err)
356 goto err_uninit;
357
358 return pool;
359
360err_uninit:
361 page_pool_uninit(pool);
362err_free:
363 pr_warn("%s() gave up with errno %d\n", __func__, err);
364 kfree(pool);
365 return ERR_PTR(err);
366}
367EXPORT_SYMBOL(page_pool_create_percpu);
368
369/**
370 * page_pool_create() - create a page pool
371 * @params: parameters, see struct page_pool_params
372 */
373struct page_pool *page_pool_create(const struct page_pool_params *params)
374{
375 return page_pool_create_percpu(params, -1);
376}
377EXPORT_SYMBOL(page_pool_create);
378
379static void page_pool_return_netmem(struct page_pool *pool, netmem_ref netmem);
380
381static noinline netmem_ref page_pool_refill_alloc_cache(struct page_pool *pool)
382{
383 struct ptr_ring *r = &pool->ring;
384 netmem_ref netmem;
385 int pref_nid; /* preferred NUMA node */
386
387 /* Quicker fallback, avoid locks when ring is empty */
388 if (__ptr_ring_empty(r)) {
389 alloc_stat_inc(pool, empty);
390 return 0;
391 }
392
393 /* Softirq guarantee CPU and thus NUMA node is stable. This,
394 * assumes CPU refilling driver RX-ring will also run RX-NAPI.
395 */
396#ifdef CONFIG_NUMA
397 pref_nid = (pool->p.nid == NUMA_NO_NODE) ? numa_mem_id() : pool->p.nid;
398#else
399 /* Ignore pool->p.nid setting if !CONFIG_NUMA, helps compiler */
400 pref_nid = numa_mem_id(); /* will be zero like page_to_nid() */
401#endif
402
403 /* Refill alloc array, but only if NUMA match */
404 do {
405 netmem = (__force netmem_ref)__ptr_ring_consume(r);
406 if (unlikely(!netmem))
407 break;
408
409 if (likely(netmem_is_pref_nid(netmem, pref_nid))) {
410 pool->alloc.cache[pool->alloc.count++] = netmem;
411 } else {
412 /* NUMA mismatch;
413 * (1) release 1 page to page-allocator and
414 * (2) break out to fallthrough to alloc_pages_node.
415 * This limit stress on page buddy alloactor.
416 */
417 page_pool_return_netmem(pool, netmem);
418 alloc_stat_inc(pool, waive);
419 netmem = 0;
420 break;
421 }
422 } while (pool->alloc.count < PP_ALLOC_CACHE_REFILL);
423
424 /* Return last page */
425 if (likely(pool->alloc.count > 0)) {
426 netmem = pool->alloc.cache[--pool->alloc.count];
427 alloc_stat_inc(pool, refill);
428 }
429
430 return netmem;
431}
432
433/* fast path */
434static netmem_ref __page_pool_get_cached(struct page_pool *pool)
435{
436 netmem_ref netmem;
437
438 /* Caller MUST guarantee safe non-concurrent access, e.g. softirq */
439 if (likely(pool->alloc.count)) {
440 /* Fast-path */
441 netmem = pool->alloc.cache[--pool->alloc.count];
442 alloc_stat_inc(pool, fast);
443 } else {
444 netmem = page_pool_refill_alloc_cache(pool);
445 }
446
447 return netmem;
448}
449
450static void __page_pool_dma_sync_for_device(const struct page_pool *pool,
451 netmem_ref netmem,
452 u32 dma_sync_size)
453{
454#if defined(CONFIG_HAS_DMA) && defined(CONFIG_DMA_NEED_SYNC)
455 dma_addr_t dma_addr = page_pool_get_dma_addr_netmem(netmem);
456
457 dma_sync_size = min(dma_sync_size, pool->p.max_len);
458 __dma_sync_single_for_device(pool->p.dev, dma_addr + pool->p.offset,
459 dma_sync_size, pool->p.dma_dir);
460#endif
461}
462
463static __always_inline void
464page_pool_dma_sync_for_device(const struct page_pool *pool,
465 netmem_ref netmem,
466 u32 dma_sync_size)
467{
468 if (pool->dma_sync && dma_dev_need_sync(pool->p.dev)) {
469 rcu_read_lock();
470 /* re-check under rcu_read_lock() to sync with page_pool_scrub() */
471 if (pool->dma_sync)
472 __page_pool_dma_sync_for_device(pool, netmem,
473 dma_sync_size);
474 rcu_read_unlock();
475 }
476}
477
478static int page_pool_register_dma_index(struct page_pool *pool,
479 netmem_ref netmem, gfp_t gfp)
480{
481 int err = 0;
482 u32 id;
483
484 if (unlikely(!PP_DMA_INDEX_BITS))
485 goto out;
486
487 /*
488 * Drivers request GFP flags according to both the current context and
489 * the device constraints, but the XArray entry itself is by no mean
490 * used by the device, so remove zone/policy flags.
491 */
492 gfp &= ~(__GFP_DMA | __GFP_DMA32 | __GFP_HIGHMEM | __GFP_COMP);
493
494 if (in_softirq())
495 err = xa_alloc(&pool->dma_mapped, &id, netmem_to_page(netmem),
496 PP_DMA_INDEX_LIMIT, gfp);
497 else
498 err = xa_alloc_bh(&pool->dma_mapped, &id, netmem_to_page(netmem),
499 PP_DMA_INDEX_LIMIT, gfp);
500 if (err) {
501 WARN_ONCE(err != -ENOMEM, "couldn't track DMA mapping, please report to netdev@");
502 goto out;
503 }
504
505 netmem_set_dma_index(netmem, id);
506out:
507 return err;
508}
509
510static void __page_pool_unmap_netmem_dma(struct page_pool *pool,
511 netmem_ref netmem)
512{
513 struct page *old, *page = netmem_to_page(netmem);
514 unsigned long id;
515 dma_addr_t dma;
516
517 if (!pool->dma_map)
518 return;
519
520 /* Cache dma_addr before xa_cmpxchg. The scrub path holds no page ref;
521 * the unref path calls put_page() regardless of cmpxchg outcome, so
522 * after the cmpxchg we cannot safely touch netmem fields.
523 */
524 dma = page_pool_get_dma_addr_netmem(netmem);
525
526 if (likely(PP_DMA_INDEX_BITS)) {
527 id = netmem_get_dma_index(netmem);
528 if (!id)
529 return;
530
531 if (in_softirq())
532 old = xa_cmpxchg(&pool->dma_mapped,
533 id, page, NULL, 0);
534 else
535 old = xa_cmpxchg_bh(&pool->dma_mapped,
536 id, page, NULL, 0);
537 if (old != page)
538 return;
539 }
540
541 dma_unmap_page_attrs(pool->p.dev, dma,
542 PAGE_SIZE << pool->p.order, pool->p.dma_dir,
543 DMA_ATTR_SKIP_CPU_SYNC | DMA_ATTR_WEAK_ORDERING);
544}
545
546static bool page_pool_dma_map(struct page_pool *pool, netmem_ref netmem, gfp_t gfp)
547{
548 dma_addr_t dma;
549 int err;
550
551 /* Setup DMA mapping: use 'struct page' area for storing DMA-addr
552 * since dma_addr_t can be either 32 or 64 bits and does not always fit
553 * into page private data (i.e 32bit cpu with 64bit DMA caps)
554 * This mapping is kept for lifetime of page, until leaving pool.
555 */
556 dma = dma_map_page_attrs(pool->p.dev, netmem_to_page(netmem), 0,
557 (PAGE_SIZE << pool->p.order), pool->p.dma_dir,
558 DMA_ATTR_SKIP_CPU_SYNC |
559 DMA_ATTR_WEAK_ORDERING);
560 if (dma_mapping_error(pool->p.dev, dma))
561 return false;
562
563 if (page_pool_set_dma_addr_netmem(netmem, dma)) {
564 WARN_ONCE(1, "unexpected DMA address, please report to netdev@");
565 goto unmap_failed;
566 }
567
568 err = page_pool_register_dma_index(pool, netmem, gfp);
569 if (err)
570 goto unset_failed;
571
572 page_pool_dma_sync_for_device(pool, netmem, pool->p.max_len);
573
574 return true;
575
576unset_failed:
577 page_pool_set_dma_addr_netmem(netmem, 0);
578unmap_failed:
579 dma_unmap_page_attrs(pool->p.dev, dma,
580 PAGE_SIZE << pool->p.order, pool->p.dma_dir,
581 DMA_ATTR_SKIP_CPU_SYNC | DMA_ATTR_WEAK_ORDERING);
582 return false;
583}
584
585static struct page *__page_pool_alloc_page_order(struct page_pool *pool,
586 gfp_t gfp)
587{
588 struct page *page;
589
590 gfp |= __GFP_COMP;
591 page = alloc_pages_node(pool->p.nid, gfp, pool->p.order);
592 if (unlikely(!page))
593 return NULL;
594
595 if (pool->dma_map && unlikely(!page_pool_dma_map(pool, page_to_netmem(page), gfp))) {
596 put_page(page);
597 return NULL;
598 }
599
600 alloc_stat_inc(pool, slow_high_order);
601 page_pool_set_pp_info(pool, page_to_netmem(page));
602
603 /* Track how many pages are held 'in-flight' */
604 pool->pages_state_hold_cnt++;
605 trace_page_pool_state_hold(pool, page_to_netmem(page),
606 pool->pages_state_hold_cnt);
607 return page;
608}
609
610/* slow path */
611static noinline netmem_ref __page_pool_alloc_netmems_slow(struct page_pool *pool,
612 gfp_t gfp)
613{
614 const int bulk = PP_ALLOC_CACHE_REFILL;
615 unsigned int pp_order = pool->p.order;
616 bool dma_map = pool->dma_map;
617 netmem_ref netmem;
618 int i, nr_pages;
619
620 /* Unconditionally set NOWARN if allocating from NAPI.
621 * Drivers forget to set it, and OOM reports on packet Rx are useless.
622 */
623 if ((gfp & GFP_ATOMIC) == GFP_ATOMIC)
624 gfp |= __GFP_NOWARN;
625
626 /* Don't support bulk alloc for high-order pages */
627 if (unlikely(pp_order))
628 return page_to_netmem(__page_pool_alloc_page_order(pool, gfp));
629
630 /* Unnecessary as alloc cache is empty, but guarantees zero count */
631 if (unlikely(pool->alloc.count > 0))
632 return pool->alloc.cache[--pool->alloc.count];
633
634 /* Mark empty alloc.cache slots "empty" for alloc_pages_bulk */
635 memset(&pool->alloc.cache, 0, sizeof(void *) * bulk);
636
637 nr_pages = alloc_pages_bulk_node(gfp, pool->p.nid, bulk,
638 (struct page **)pool->alloc.cache);
639 if (unlikely(!nr_pages))
640 return 0;
641
642 /* Pages have been filled into alloc.cache array, but count is zero and
643 * page element have not been (possibly) DMA mapped.
644 */
645 for (i = 0; i < nr_pages; i++) {
646 netmem = pool->alloc.cache[i];
647 if (dma_map && unlikely(!page_pool_dma_map(pool, netmem, gfp))) {
648 put_page(netmem_to_page(netmem));
649 continue;
650 }
651
652 page_pool_set_pp_info(pool, netmem);
653 pool->alloc.cache[pool->alloc.count++] = netmem;
654 /* Track how many pages are held 'in-flight' */
655 pool->pages_state_hold_cnt++;
656 trace_page_pool_state_hold(pool, netmem,
657 pool->pages_state_hold_cnt);
658 }
659
660 /* Return last page */
661 if (likely(pool->alloc.count > 0)) {
662 netmem = pool->alloc.cache[--pool->alloc.count];
663 alloc_stat_inc(pool, slow);
664 } else {
665 netmem = 0;
666 }
667
668 /* When page just alloc'ed is should/must have refcnt 1. */
669 return netmem;
670}
671
672/* For using page_pool replace: alloc_pages() API calls, but provide
673 * synchronization guarantee for allocation side.
674 */
675netmem_ref page_pool_alloc_netmems(struct page_pool *pool, gfp_t gfp)
676{
677 netmem_ref netmem;
678
679 /* Fast-path: Get a page from cache */
680 netmem = __page_pool_get_cached(pool);
681 if (netmem)
682 return netmem;
683
684 /* Slow-path: cache empty, do real allocation */
685 if (static_branch_unlikely(&page_pool_mem_providers) && pool->mp_ops)
686 netmem = pool->mp_ops->alloc_netmems(pool, gfp);
687 else
688 netmem = __page_pool_alloc_netmems_slow(pool, gfp);
689 return netmem;
690}
691EXPORT_SYMBOL(page_pool_alloc_netmems);
692ALLOW_ERROR_INJECTION(page_pool_alloc_netmems, NULL);
693
694struct page *page_pool_alloc_pages(struct page_pool *pool, gfp_t gfp)
695{
696 return netmem_to_page(page_pool_alloc_netmems(pool, gfp));
697}
698EXPORT_SYMBOL(page_pool_alloc_pages);
699
700/* Calculate distance between two u32 values, valid if distance is below 2^(31)
701 * https://en.wikipedia.org/wiki/Serial_number_arithmetic#General_Solution
702 */
703#define _distance(a, b) (s32)((a) - (b))
704
705s32 page_pool_inflight(const struct page_pool *pool, bool strict)
706{
707 u32 release_cnt = atomic_read(&pool->pages_state_release_cnt);
708 u32 hold_cnt = READ_ONCE(pool->pages_state_hold_cnt);
709 s32 inflight;
710
711 inflight = _distance(hold_cnt, release_cnt);
712
713 if (strict) {
714 trace_page_pool_release(pool, inflight, hold_cnt, release_cnt);
715 WARN(inflight < 0, "Negative(%d) inflight packet-pages",
716 inflight);
717 } else {
718 inflight = max(0, inflight);
719 }
720
721 return inflight;
722}
723
724void page_pool_set_pp_info(struct page_pool *pool, netmem_ref netmem)
725{
726 netmem_set_pp(netmem, pool);
727 netmem_or_pp_magic(netmem, PP_SIGNATURE);
728
729 /* Ensuring all pages have been split into one fragment initially:
730 * page_pool_set_pp_info() is only called once for every page when it
731 * is allocated from the page allocator and page_pool_fragment_page()
732 * is dirtying the same cache line as the page->pp_magic above, so
733 * the overhead is negligible.
734 */
735 page_pool_fragment_netmem(netmem, 1);
736 if (pool->has_init_callback)
737 pool->slow.init_callback(netmem, pool->slow.init_arg);
738}
739
740void page_pool_clear_pp_info(netmem_ref netmem)
741{
742 netmem_clear_pp_magic(netmem);
743 netmem_set_pp(netmem, NULL);
744}
745
746static __always_inline void __page_pool_release_netmem_dma(struct page_pool *pool,
747 netmem_ref netmem)
748{
749 /* Caller must hold a page ref: __page_pool_unmap_netmem_dma() is
750 * safe without a ref, but the field clears below require it.
751 */
752 if (!pool->dma_map)
753 return;
754
755 __page_pool_unmap_netmem_dma(pool, netmem);
756 page_pool_set_dma_addr_netmem(netmem, 0);
757 if (likely(PP_DMA_INDEX_BITS))
758 netmem_set_dma_index(netmem, 0);
759}
760
761/* Disconnects a page (from a page_pool). API users can have a need
762 * to disconnect a page (from a page_pool), to allow it to be used as
763 * a regular page (that will eventually be returned to the normal
764 * page-allocator via put_page).
765 */
766static void page_pool_return_netmem(struct page_pool *pool, netmem_ref netmem)
767{
768 int count;
769 bool put;
770
771 put = true;
772 if (static_branch_unlikely(&page_pool_mem_providers) && pool->mp_ops)
773 put = pool->mp_ops->release_netmem(pool, netmem);
774 else
775 __page_pool_release_netmem_dma(pool, netmem);
776
777 /* This may be the last page returned, releasing the pool, so
778 * it is not safe to reference pool afterwards.
779 */
780 count = atomic_inc_return_relaxed(&pool->pages_state_release_cnt);
781 trace_page_pool_state_release(pool, netmem, count);
782
783 if (put) {
784 page_pool_clear_pp_info(netmem);
785 put_page(netmem_to_page(netmem));
786 }
787 /* An optimization would be to call __free_pages(page, pool->p.order)
788 * knowing page is not part of page-cache (thus avoiding a
789 * __page_cache_release() call).
790 */
791}
792
793static bool page_pool_recycle_in_ring(struct page_pool *pool, netmem_ref netmem)
794{
795 bool in_softirq, ret;
796
797 /* BH protection not needed if current is softirq */
798 in_softirq = page_pool_producer_lock(pool);
799 ret = !__ptr_ring_produce(&pool->ring, (__force void *)netmem);
800 if (ret)
801 recycle_stat_inc(pool, ring);
802 page_pool_producer_unlock(pool, in_softirq);
803
804 return ret;
805}
806
807/* Only allow direct recycling in special circumstances, into the
808 * alloc side cache. E.g. during RX-NAPI processing for XDP_DROP use-case.
809 *
810 * Caller must provide appropriate safe context.
811 */
812static bool page_pool_recycle_in_cache(netmem_ref netmem,
813 struct page_pool *pool)
814{
815 if (unlikely(pool->alloc.count == PP_ALLOC_CACHE_SIZE)) {
816 recycle_stat_inc(pool, cache_full);
817 return false;
818 }
819
820 /* Caller MUST have verified/know (page_ref_count(page) == 1) */
821 pool->alloc.cache[pool->alloc.count++] = netmem;
822 recycle_stat_inc(pool, cached);
823 return true;
824}
825
826static bool __page_pool_page_can_be_recycled(netmem_ref netmem)
827{
828 return netmem_is_net_iov(netmem) ||
829 (page_ref_count(netmem_to_page(netmem)) == 1 &&
830 !page_is_pfmemalloc(netmem_to_page(netmem)));
831}
832
833/* If the page refcnt == 1, this will try to recycle the page.
834 * If pool->dma_sync is set, we'll try to sync the DMA area for
835 * the configured size min(dma_sync_size, pool->max_len).
836 * If the page refcnt != 1, then the page will be returned to memory
837 * subsystem.
838 */
839static __always_inline netmem_ref
840__page_pool_put_page(struct page_pool *pool, netmem_ref netmem,
841 unsigned int dma_sync_size, bool allow_direct)
842{
843 lockdep_assert_no_hardirq();
844
845 /* This allocator is optimized for the XDP mode that uses
846 * one-frame-per-page, but have fallbacks that act like the
847 * regular page allocator APIs.
848 *
849 * refcnt == 1 means page_pool owns page, and can recycle it.
850 *
851 * page is NOT reusable when allocated when system is under
852 * some pressure. (page_is_pfmemalloc)
853 */
854 if (likely(__page_pool_page_can_be_recycled(netmem))) {
855 /* Read barrier done in page_ref_count / READ_ONCE */
856
857 page_pool_dma_sync_for_device(pool, netmem, dma_sync_size);
858
859 if (allow_direct && page_pool_recycle_in_cache(netmem, pool))
860 return 0;
861
862 /* Page found as candidate for recycling */
863 return netmem;
864 }
865
866 /* Fallback/non-XDP mode: API user have elevated refcnt.
867 *
868 * Many drivers split up the page into fragments, and some
869 * want to keep doing this to save memory and do refcnt based
870 * recycling. Support this use case too, to ease drivers
871 * switching between XDP/non-XDP.
872 *
873 * In-case page_pool maintains the DMA mapping, API user must
874 * call page_pool_put_page once. In this elevated refcnt
875 * case, the DMA is unmapped/released, as driver is likely
876 * doing refcnt based recycle tricks, meaning another process
877 * will be invoking put_page.
878 */
879 recycle_stat_inc(pool, released_refcnt);
880 page_pool_return_netmem(pool, netmem);
881
882 return 0;
883}
884
885static bool page_pool_napi_local(const struct page_pool *pool)
886{
887 const struct napi_struct *napi;
888 u32 cpuid;
889
890 /* On PREEMPT_RT the softirq can be preempted by the consumer */
891 if (IS_ENABLED(CONFIG_PREEMPT_RT))
892 return false;
893
894 if (unlikely(!in_softirq()))
895 return false;
896
897 /* Allow direct recycle if we have reasons to believe that we are
898 * in the same context as the consumer would run, so there's
899 * no possible race.
900 * __page_pool_put_page() makes sure we're not in hardirq context
901 * and interrupts are enabled prior to accessing the cache.
902 */
903 cpuid = smp_processor_id();
904 if (READ_ONCE(pool->cpuid) == cpuid)
905 return true;
906
907 napi = READ_ONCE(pool->p.napi);
908
909 return napi && READ_ONCE(napi->list_owner) == cpuid;
910}
911
912void page_pool_put_unrefed_netmem(struct page_pool *pool, netmem_ref netmem,
913 unsigned int dma_sync_size, bool allow_direct)
914{
915 if (!allow_direct)
916 allow_direct = page_pool_napi_local(pool);
917
918 netmem = __page_pool_put_page(pool, netmem, dma_sync_size,
919 allow_direct);
920 if (netmem && !page_pool_recycle_in_ring(pool, netmem)) {
921 /* Cache full, fallback to free pages */
922 recycle_stat_inc(pool, ring_full);
923 page_pool_return_netmem(pool, netmem);
924 }
925}
926EXPORT_SYMBOL(page_pool_put_unrefed_netmem);
927
928void page_pool_put_unrefed_page(struct page_pool *pool, struct page *page,
929 unsigned int dma_sync_size, bool allow_direct)
930{
931 page_pool_put_unrefed_netmem(pool, page_to_netmem(page), dma_sync_size,
932 allow_direct);
933}
934EXPORT_SYMBOL(page_pool_put_unrefed_page);
935
936static void page_pool_recycle_ring_bulk(struct page_pool *pool,
937 netmem_ref *bulk,
938 u32 bulk_len)
939{
940 bool in_softirq;
941 u32 i;
942
943 /* Bulk produce into ptr_ring page_pool cache */
944 in_softirq = page_pool_producer_lock(pool);
945
946 for (i = 0; i < bulk_len; i++) {
947 if (__ptr_ring_produce(&pool->ring, (__force void *)bulk[i])) {
948 /* ring full */
949 recycle_stat_inc(pool, ring_full);
950 break;
951 }
952 }
953
954 recycle_stat_add(pool, ring, i);
955 page_pool_producer_unlock(pool, in_softirq);
956
957 /* Hopefully all pages were returned into ptr_ring */
958 if (likely(i == bulk_len))
959 return;
960
961 /*
962 * ptr_ring cache is full, free remaining pages outside producer lock
963 * since put_page() with refcnt == 1 can be an expensive operation.
964 */
965 for (; i < bulk_len; i++)
966 page_pool_return_netmem(pool, bulk[i]);
967}
968
969/**
970 * page_pool_put_netmem_bulk() - release references on multiple netmems
971 * @data: array holding netmem references
972 * @count: number of entries in @data
973 *
974 * Tries to refill a number of netmems into the ptr_ring cache holding ptr_ring
975 * producer lock. If the ptr_ring is full, page_pool_put_netmem_bulk()
976 * will release leftover netmems to the memory provider.
977 * page_pool_put_netmem_bulk() is suitable to be run inside the driver NAPI tx
978 * completion loop for the XDP_REDIRECT use case.
979 *
980 * Please note the caller must not use data area after running
981 * page_pool_put_netmem_bulk(), as this function overwrites it.
982 */
983void page_pool_put_netmem_bulk(netmem_ref *data, u32 count)
984{
985 u32 bulk_len = 0;
986
987 for (u32 i = 0; i < count; i++) {
988 netmem_ref netmem = netmem_compound_head(data[i]);
989
990 if (page_pool_unref_and_test(netmem))
991 data[bulk_len++] = netmem;
992 }
993
994 count = bulk_len;
995 while (count) {
996 netmem_ref bulk[XDP_BULK_QUEUE_SIZE];
997 struct page_pool *pool = NULL;
998 bool allow_direct;
999 u32 foreign = 0;
1000
1001 bulk_len = 0;
1002
1003 for (u32 i = 0; i < count; i++) {
1004 struct page_pool *netmem_pp;
1005 netmem_ref netmem = data[i];
1006
1007 netmem_pp = netmem_get_pp(netmem);
1008 if (unlikely(!pool)) {
1009 pool = netmem_pp;
1010 allow_direct = page_pool_napi_local(pool);
1011 } else if (netmem_pp != pool) {
1012 /*
1013 * If the netmem belongs to a different
1014 * page_pool, save it for another round.
1015 */
1016 data[foreign++] = netmem;
1017 continue;
1018 }
1019
1020 netmem = __page_pool_put_page(pool, netmem, -1,
1021 allow_direct);
1022 /* Approved for bulk recycling in ptr_ring cache */
1023 if (netmem)
1024 bulk[bulk_len++] = netmem;
1025 }
1026
1027 if (bulk_len)
1028 page_pool_recycle_ring_bulk(pool, bulk, bulk_len);
1029
1030 count = foreign;
1031 }
1032}
1033EXPORT_SYMBOL(page_pool_put_netmem_bulk);
1034
1035static netmem_ref page_pool_drain_frag(struct page_pool *pool,
1036 netmem_ref netmem)
1037{
1038 long drain_count = BIAS_MAX - pool->frag_users;
1039
1040 /* Some user is still using the page frag */
1041 if (likely(page_pool_unref_netmem(netmem, drain_count)))
1042 return 0;
1043
1044 if (__page_pool_page_can_be_recycled(netmem)) {
1045 page_pool_dma_sync_for_device(pool, netmem, -1);
1046 return netmem;
1047 }
1048
1049 page_pool_return_netmem(pool, netmem);
1050 return 0;
1051}
1052
1053static void page_pool_free_frag(struct page_pool *pool)
1054{
1055 long drain_count = BIAS_MAX - pool->frag_users;
1056 netmem_ref netmem = pool->frag_page;
1057
1058 pool->frag_page = 0;
1059
1060 if (!netmem || page_pool_unref_netmem(netmem, drain_count))
1061 return;
1062
1063 page_pool_return_netmem(pool, netmem);
1064}
1065
1066netmem_ref page_pool_alloc_frag_netmem(struct page_pool *pool,
1067 unsigned int *offset, unsigned int size,
1068 gfp_t gfp)
1069{
1070 unsigned int max_size = PAGE_SIZE << pool->p.order;
1071 netmem_ref netmem = pool->frag_page;
1072
1073 if (WARN_ON(size > max_size))
1074 return 0;
1075
1076 size = ALIGN(size, max_t(unsigned int, dma_get_cache_alignment(),
1077 __alignof__(struct skb_shared_info)));
1078 *offset = pool->frag_offset;
1079
1080 if (netmem && *offset + size > max_size) {
1081 netmem = page_pool_drain_frag(pool, netmem);
1082 if (netmem) {
1083 recycle_stat_inc(pool, cached);
1084 alloc_stat_inc(pool, fast);
1085 goto frag_reset;
1086 }
1087 }
1088
1089 if (!netmem) {
1090 netmem = page_pool_alloc_netmems(pool, gfp);
1091 if (unlikely(!netmem)) {
1092 pool->frag_page = 0;
1093 return 0;
1094 }
1095
1096 pool->frag_page = netmem;
1097
1098frag_reset:
1099 pool->frag_users = 1;
1100 *offset = 0;
1101 pool->frag_offset = size;
1102 page_pool_fragment_netmem(netmem, BIAS_MAX);
1103 return netmem;
1104 }
1105
1106 pool->frag_users++;
1107 pool->frag_offset = *offset + size;
1108 return netmem;
1109}
1110EXPORT_SYMBOL(page_pool_alloc_frag_netmem);
1111
1112struct page *page_pool_alloc_frag(struct page_pool *pool, unsigned int *offset,
1113 unsigned int size, gfp_t gfp)
1114{
1115 return netmem_to_page(page_pool_alloc_frag_netmem(pool, offset, size,
1116 gfp));
1117}
1118EXPORT_SYMBOL(page_pool_alloc_frag);
1119
1120static void page_pool_empty_ring(struct page_pool *pool)
1121{
1122 netmem_ref netmem;
1123
1124 /* Empty recycle ring */
1125 while ((netmem = (__force netmem_ref)ptr_ring_consume_bh(&pool->ring))) {
1126 /* Verify the refcnt invariant of cached pages */
1127 if (!(netmem_ref_count(netmem) == 1))
1128 pr_crit("%s() page_pool refcnt %d violation\n",
1129 __func__, netmem_ref_count(netmem));
1130
1131 page_pool_return_netmem(pool, netmem);
1132 }
1133}
1134
1135static void __page_pool_destroy(struct page_pool *pool)
1136{
1137 if (pool->disconnect)
1138 pool->disconnect(pool);
1139
1140 page_pool_unlist(pool);
1141 page_pool_uninit(pool);
1142
1143 kfree(pool);
1144}
1145
1146static void page_pool_empty_alloc_cache_once(struct page_pool *pool)
1147{
1148 netmem_ref netmem;
1149
1150 if (pool->destroy_cnt)
1151 return;
1152
1153 /* Empty alloc cache, assume caller made sure this is
1154 * no-longer in use, and page_pool_alloc_pages() cannot be
1155 * call concurrently.
1156 */
1157 while (pool->alloc.count) {
1158 netmem = pool->alloc.cache[--pool->alloc.count];
1159 page_pool_return_netmem(pool, netmem);
1160 }
1161}
1162
1163static void page_pool_scrub(struct page_pool *pool)
1164{
1165 unsigned long id;
1166 void *ptr;
1167
1168 page_pool_empty_alloc_cache_once(pool);
1169 if (!pool->destroy_cnt++ && pool->dma_map) {
1170 if (pool->dma_sync) {
1171 /* Disable page_pool_dma_sync_for_device() */
1172 pool->dma_sync = false;
1173
1174 /* Make sure all concurrent returns that may see the old
1175 * value of dma_sync (and thus perform a sync) have
1176 * finished before doing the unmapping below. Skip the
1177 * wait if the device doesn't actually need syncing, or
1178 * if there are no outstanding mapped pages.
1179 */
1180 if (dma_dev_need_sync(pool->p.dev) &&
1181 !xa_empty(&pool->dma_mapped))
1182 synchronize_net();
1183 }
1184
1185 /* No page ref, dma-unmap only. */
1186 xa_for_each(&pool->dma_mapped, id, ptr)
1187 __page_pool_unmap_netmem_dma(pool, page_to_netmem((struct page *)ptr));
1188 }
1189
1190 /* No more consumers should exist, but producers could still
1191 * be in-flight.
1192 */
1193 page_pool_empty_ring(pool);
1194}
1195
1196static int page_pool_release(struct page_pool *pool)
1197{
1198 bool in_softirq;
1199 int inflight;
1200
1201 page_pool_scrub(pool);
1202 inflight = page_pool_inflight(pool, true);
1203 /* Acquire producer lock to make sure producers have exited. */
1204 in_softirq = page_pool_producer_lock(pool);
1205 page_pool_producer_unlock(pool, in_softirq);
1206 if (!inflight)
1207 __page_pool_destroy(pool);
1208
1209 return inflight;
1210}
1211
1212static void page_pool_release_retry(struct work_struct *wq)
1213{
1214 struct delayed_work *dwq = to_delayed_work(wq);
1215 struct page_pool *pool = container_of(dwq, typeof(*pool), release_dw);
1216 void *netdev;
1217 int inflight;
1218
1219 inflight = page_pool_release(pool);
1220 /* In rare cases, a driver bug may cause inflight to go negative.
1221 * Don't reschedule release if inflight is 0 or negative.
1222 * - If 0, the page_pool has been destroyed
1223 * - if negative, we will never recover
1224 * in both cases no reschedule is necessary.
1225 */
1226 if (inflight <= 0)
1227 return;
1228
1229 /* Periodic warning for page pools the user can't see */
1230 netdev = READ_ONCE(pool->slow.netdev);
1231 if (time_after_eq(jiffies, pool->defer_warn) &&
1232 (!netdev || netdev == NET_PTR_POISON)) {
1233 int sec = (s32)((u32)jiffies - (u32)pool->defer_start) / HZ;
1234
1235 pr_warn("%s() stalled pool shutdown: id %u, %d inflight %d sec\n",
1236 __func__, pool->user.id, inflight, sec);
1237 pool->defer_warn = jiffies + DEFER_WARN_INTERVAL;
1238 }
1239
1240 /* Still not ready to be disconnected, retry later */
1241 schedule_delayed_work(&pool->release_dw, DEFER_TIME);
1242}
1243
1244void page_pool_use_xdp_mem(struct page_pool *pool, void (*disconnect)(void *),
1245 const struct xdp_mem_info *mem)
1246{
1247 refcount_inc(&pool->user_cnt);
1248 pool->disconnect = disconnect;
1249 pool->xdp_mem_id = mem->id;
1250}
1251
1252/**
1253 * page_pool_enable_direct_recycling() - mark page pool as owned by NAPI
1254 * @pool: page pool to modify
1255 * @napi: NAPI instance to associate the page pool with
1256 *
1257 * Associate a page pool with a NAPI instance for lockless page recycling.
1258 * This is useful when a new page pool has to be added to a NAPI instance
1259 * without disabling that NAPI instance, to mark the point at which control
1260 * path "hands over" the page pool to the NAPI instance. In most cases driver
1261 * can simply set the @napi field in struct page_pool_params, and does not
1262 * have to call this helper.
1263 *
1264 * The function is idempotent, but does not implement any refcounting.
1265 * Single page_pool_disable_direct_recycling() will disable recycling,
1266 * no matter how many times enable was called.
1267 */
1268void page_pool_enable_direct_recycling(struct page_pool *pool,
1269 struct napi_struct *napi)
1270{
1271 if (READ_ONCE(pool->p.napi) == napi)
1272 return;
1273 WARN_ON(!napi || pool->p.napi);
1274
1275 mutex_lock(&page_pools_lock);
1276 WRITE_ONCE(pool->p.napi, napi);
1277 mutex_unlock(&page_pools_lock);
1278}
1279EXPORT_SYMBOL(page_pool_enable_direct_recycling);
1280
1281void page_pool_disable_direct_recycling(struct page_pool *pool)
1282{
1283 /* Disable direct recycling based on pool->cpuid.
1284 * Paired with READ_ONCE() in page_pool_napi_local().
1285 */
1286 WRITE_ONCE(pool->cpuid, -1);
1287
1288 if (!pool->p.napi)
1289 return;
1290
1291 napi_assert_will_not_race(pool->p.napi);
1292
1293 mutex_lock(&page_pools_lock);
1294 WRITE_ONCE(pool->p.napi, NULL);
1295 mutex_unlock(&page_pools_lock);
1296}
1297EXPORT_SYMBOL(page_pool_disable_direct_recycling);
1298
1299void page_pool_destroy(struct page_pool *pool)
1300{
1301 if (!pool)
1302 return;
1303
1304 if (!page_pool_put(pool))
1305 return;
1306
1307 page_pool_disable_direct_recycling(pool);
1308 page_pool_free_frag(pool);
1309
1310 if (!page_pool_release(pool))
1311 return;
1312
1313 page_pool_detached(pool);
1314 pool->defer_start = jiffies;
1315 pool->defer_warn = jiffies + DEFER_WARN_INTERVAL;
1316
1317 INIT_DELAYED_WORK(&pool->release_dw, page_pool_release_retry);
1318 schedule_delayed_work(&pool->release_dw, DEFER_TIME);
1319}
1320EXPORT_SYMBOL(page_pool_destroy);
1321
1322/* Caller must provide appropriate safe context, e.g. NAPI. */
1323void page_pool_update_nid(struct page_pool *pool, int new_nid)
1324{
1325 netmem_ref netmem;
1326
1327 trace_page_pool_update_nid(pool, new_nid);
1328 pool->p.nid = new_nid;
1329
1330 /* Flush pool alloc cache, as refill will check NUMA node */
1331 while (pool->alloc.count) {
1332 netmem = pool->alloc.cache[--pool->alloc.count];
1333 page_pool_return_netmem(pool, netmem);
1334 }
1335}
1336EXPORT_SYMBOL(page_pool_update_nid);
1337
1338bool net_mp_niov_set_dma_addr(struct net_iov *niov, dma_addr_t addr)
1339{
1340 return page_pool_set_dma_addr_netmem(net_iov_to_netmem(niov), addr);
1341}
1342
1343/* Associate a niov with a page pool. Should follow with a matching
1344 * net_mp_niov_clear_page_pool()
1345 */
1346void net_mp_niov_set_page_pool(struct page_pool *pool, struct net_iov *niov)
1347{
1348 netmem_ref netmem = net_iov_to_netmem(niov);
1349
1350 page_pool_set_pp_info(pool, netmem);
1351
1352 pool->pages_state_hold_cnt++;
1353 trace_page_pool_state_hold(pool, netmem, pool->pages_state_hold_cnt);
1354}
1355
1356/* Disassociate a niov from a page pool. Should only be used in the
1357 * ->release_netmem() path.
1358 */
1359void net_mp_niov_clear_page_pool(struct net_iov *niov)
1360{
1361 netmem_ref netmem = net_iov_to_netmem(niov);
1362
1363 page_pool_clear_pp_info(netmem);
1364}