projects/linux·blob master · refs

483 lines · 12.3 KiB

1// SPDX-License-Identifier: GPL-2.0
2
3#include <linux/mutex.h>
4#include <linux/netdevice.h>
5#include <linux/xarray.h>
6#include <net/busy_poll.h>
7#include <net/net_debug.h>
8#include <net/netdev_rx_queue.h>
9#include <net/page_pool/helpers.h>
10#include <net/page_pool/types.h>
11#include <net/page_pool/memory_provider.h>
12#include <net/sock.h>
13
14#include "page_pool_priv.h"
15#include "netdev-genl-gen.h"
16
17static DEFINE_XARRAY_FLAGS(page_pools, XA_FLAGS_ALLOC1);
18/* Protects: page_pools, netdevice->page_pools, pool->p.napi, pool->slow.netdev,
19 * pool->user.
20 * Ordering: inside rtnl_lock
21 */
22DEFINE_MUTEX(page_pools_lock);
23
24/* Page pools are only reachable from user space (via netlink) if they are
25 * linked to a netdev at creation time. Following page pool "visibility"
26 * states are possible:
27 * - normal
28 * - user.list: linked to real netdev, netdev: real netdev
29 * - orphaned - real netdev has disappeared
30 * - user.list: linked to lo, netdev: lo
31 * - invisible - either (a) created without netdev linking, (b) unlisted due
32 * to error, or (c) the entire namespace which owned this pool disappeared
33 * - user.list: unhashed, netdev: unknown
34 */
35
36typedef int (*pp_nl_fill_cb)(struct sk_buff *rsp, const struct page_pool *pool,
37 const struct genl_info *info);
38
39static int
40netdev_nl_page_pool_get_do(struct genl_info *info, u32 id, pp_nl_fill_cb fill)
41{
42 struct page_pool *pool;
43 struct sk_buff *rsp;
44 int err;
45
46 mutex_lock(&page_pools_lock);
47 pool = xa_load(&page_pools, id);
48 if (!pool || hlist_unhashed(&pool->user.list) ||
49 !net_eq(dev_net(pool->slow.netdev), genl_info_net(info))) {
50 err = -ENOENT;
51 goto err_unlock;
52 }
53
54 rsp = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL);
55 if (!rsp) {
56 err = -ENOMEM;
57 goto err_unlock;
58 }
59
60 err = fill(rsp, pool, info);
61 if (err)
62 goto err_free_msg;
63
64 mutex_unlock(&page_pools_lock);
65
66 return genlmsg_reply(rsp, info);
67
68err_free_msg:
69 nlmsg_free(rsp);
70err_unlock:
71 mutex_unlock(&page_pools_lock);
72 return err;
73}
74
75struct page_pool_dump_cb {
76 unsigned long ifindex;
77 u32 pp_id;
78};
79
80static int
81netdev_nl_page_pool_get_dump(struct sk_buff *skb, struct netlink_callback *cb,
82 pp_nl_fill_cb fill, struct nlattr *ifindex_attr)
83{
84 struct page_pool_dump_cb *state = (void *)cb->ctx;
85 const struct genl_info *info = genl_info_dump(cb);
86 struct net *net = sock_net(skb->sk);
87 struct net_device *netdev;
88 struct page_pool *pool;
89 int err = 0;
90
91 if (ifindex_attr)
92 state->ifindex = nla_get_u32(ifindex_attr);
93
94 rtnl_lock();
95 mutex_lock(&page_pools_lock);
96 for_each_netdev_dump(net, netdev, state->ifindex) {
97 /* Either the provided ifindex doesn't exist or done dumping */
98 if (ifindex_attr &&
99 netdev->ifindex != nla_get_u32(ifindex_attr))
100 break;
101
102 hlist_for_each_entry(pool, &netdev->page_pools, user.list) {
103 if (state->pp_id && state->pp_id < pool->user.id)
104 continue;
105
106 state->pp_id = pool->user.id;
107 err = fill(skb, pool, info);
108 if (err)
109 goto out;
110 }
111
112 state->pp_id = 0;
113 }
114out:
115 mutex_unlock(&page_pools_lock);
116 rtnl_unlock();
117
118 return err;
119}
120
121static int
122page_pool_nl_stats_fill(struct sk_buff *rsp, const struct page_pool *pool,
123 const struct genl_info *info)
124{
125#ifdef CONFIG_PAGE_POOL_STATS
126 struct page_pool_stats stats = {};
127 struct nlattr *nest;
128 void *hdr;
129
130 page_pool_get_stats(pool, &stats);
131
132 hdr = genlmsg_iput(rsp, info);
133 if (!hdr)
134 return -EMSGSIZE;
135
136 nest = nla_nest_start(rsp, NETDEV_A_PAGE_POOL_STATS_INFO);
137 if (!nest)
138 goto err_cancel_msg;
139
140 if (nla_put_uint(rsp, NETDEV_A_PAGE_POOL_ID, pool->user.id) ||
141 (pool->slow.netdev->ifindex != LOOPBACK_IFINDEX &&
142 nla_put_u32(rsp, NETDEV_A_PAGE_POOL_IFINDEX,
143 pool->slow.netdev->ifindex)))
144 goto err_cancel_nest;
145
146 nla_nest_end(rsp, nest);
147
148 if (nla_put_uint(rsp, NETDEV_A_PAGE_POOL_STATS_ALLOC_FAST,
149 stats.alloc_stats.fast) ||
150 nla_put_uint(rsp, NETDEV_A_PAGE_POOL_STATS_ALLOC_SLOW,
151 stats.alloc_stats.slow) ||
152 nla_put_uint(rsp, NETDEV_A_PAGE_POOL_STATS_ALLOC_SLOW_HIGH_ORDER,
153 stats.alloc_stats.slow_high_order) ||
154 nla_put_uint(rsp, NETDEV_A_PAGE_POOL_STATS_ALLOC_EMPTY,
155 stats.alloc_stats.empty) ||
156 nla_put_uint(rsp, NETDEV_A_PAGE_POOL_STATS_ALLOC_REFILL,
157 stats.alloc_stats.refill) ||
158 nla_put_uint(rsp, NETDEV_A_PAGE_POOL_STATS_ALLOC_WAIVE,
159 stats.alloc_stats.waive) ||
160 nla_put_uint(rsp, NETDEV_A_PAGE_POOL_STATS_RECYCLE_CACHED,
161 stats.recycle_stats.cached) ||
162 nla_put_uint(rsp, NETDEV_A_PAGE_POOL_STATS_RECYCLE_CACHE_FULL,
163 stats.recycle_stats.cache_full) ||
164 nla_put_uint(rsp, NETDEV_A_PAGE_POOL_STATS_RECYCLE_RING,
165 stats.recycle_stats.ring) ||
166 nla_put_uint(rsp, NETDEV_A_PAGE_POOL_STATS_RECYCLE_RING_FULL,
167 stats.recycle_stats.ring_full) ||
168 nla_put_uint(rsp, NETDEV_A_PAGE_POOL_STATS_RECYCLE_RELEASED_REFCNT,
169 stats.recycle_stats.released_refcnt))
170 goto err_cancel_msg;
171
172 genlmsg_end(rsp, hdr);
173
174 return 0;
175err_cancel_nest:
176 nla_nest_cancel(rsp, nest);
177err_cancel_msg:
178 genlmsg_cancel(rsp, hdr);
179 return -EMSGSIZE;
180#else
181 GENL_SET_ERR_MSG(info, "kernel built without CONFIG_PAGE_POOL_STATS");
182 return -EOPNOTSUPP;
183#endif
184}
185
186int netdev_nl_page_pool_stats_get_doit(struct sk_buff *skb,
187 struct genl_info *info)
188{
189 struct nlattr *tb[ARRAY_SIZE(netdev_page_pool_info_nl_policy)];
190 struct nlattr *nest;
191 int err;
192 u32 id;
193
194 if (GENL_REQ_ATTR_CHECK(info, NETDEV_A_PAGE_POOL_STATS_INFO))
195 return -EINVAL;
196
197 nest = info->attrs[NETDEV_A_PAGE_POOL_STATS_INFO];
198 err = nla_parse_nested(tb, ARRAY_SIZE(tb) - 1, nest,
199 netdev_page_pool_info_nl_policy,
200 info->extack);
201 if (err)
202 return err;
203
204 if (NL_REQ_ATTR_CHECK(info->extack, nest, tb, NETDEV_A_PAGE_POOL_ID))
205 return -EINVAL;
206 if (tb[NETDEV_A_PAGE_POOL_IFINDEX]) {
207 NL_SET_ERR_MSG_ATTR(info->extack,
208 tb[NETDEV_A_PAGE_POOL_IFINDEX],
209 "selecting by ifindex not supported");
210 return -EINVAL;
211 }
212
213 id = nla_get_uint(tb[NETDEV_A_PAGE_POOL_ID]);
214
215 return netdev_nl_page_pool_get_do(info, id, page_pool_nl_stats_fill);
216}
217
218static const struct netlink_range_validation page_pool_ifindex_range = {
219 .min = 1ULL,
220 .max = S32_MAX,
221};
222
223static const struct nla_policy
224page_pool_stat_info_policy[NETDEV_A_PAGE_POOL_IFINDEX + 1] = {
225 [NETDEV_A_PAGE_POOL_IFINDEX] =
226 NLA_POLICY_FULL_RANGE(NLA_U32, &page_pool_ifindex_range),
227};
228
229int netdev_nl_page_pool_stats_get_dumpit(struct sk_buff *skb,
230 struct netlink_callback *cb)
231{
232 struct nlattr *tb[ARRAY_SIZE(page_pool_stat_info_policy)];
233 const struct genl_info *info = genl_info_dump(cb);
234 struct nlattr *ifindex_attr = NULL;
235
236 if (info->attrs[NETDEV_A_PAGE_POOL_STATS_INFO]) {
237 struct nlattr *nest;
238 int err;
239
240 nest = info->attrs[NETDEV_A_PAGE_POOL_STATS_INFO];
241 err = nla_parse_nested(tb, ARRAY_SIZE(tb) - 1, nest,
242 page_pool_stat_info_policy,
243 info->extack);
244 if (err)
245 return err;
246
247 ifindex_attr = tb[NETDEV_A_PAGE_POOL_IFINDEX];
248 }
249
250 return netdev_nl_page_pool_get_dump(skb, cb, page_pool_nl_stats_fill,
251 ifindex_attr);
252}
253
254static int
255page_pool_nl_fill(struct sk_buff *rsp, const struct page_pool *pool,
256 const struct genl_info *info)
257{
258 size_t inflight, refsz;
259 unsigned int napi_id;
260 void *hdr;
261
262 hdr = genlmsg_iput(rsp, info);
263 if (!hdr)
264 return -EMSGSIZE;
265
266 if (nla_put_uint(rsp, NETDEV_A_PAGE_POOL_ID, pool->user.id))
267 goto err_cancel;
268
269 if (pool->slow.netdev->ifindex != LOOPBACK_IFINDEX &&
270 nla_put_u32(rsp, NETDEV_A_PAGE_POOL_IFINDEX,
271 pool->slow.netdev->ifindex))
272 goto err_cancel;
273
274 napi_id = pool->p.napi ? READ_ONCE(pool->p.napi->napi_id) : 0;
275 if (napi_id_valid(napi_id) &&
276 nla_put_uint(rsp, NETDEV_A_PAGE_POOL_NAPI_ID, napi_id))
277 goto err_cancel;
278
279 inflight = page_pool_inflight(pool, false);
280 refsz = PAGE_SIZE << pool->p.order;
281 if (nla_put_uint(rsp, NETDEV_A_PAGE_POOL_INFLIGHT, inflight) ||
282 nla_put_uint(rsp, NETDEV_A_PAGE_POOL_INFLIGHT_MEM,
283 inflight * refsz))
284 goto err_cancel;
285 if (pool->user.detach_time &&
286 nla_put_uint(rsp, NETDEV_A_PAGE_POOL_DETACH_TIME,
287 ktime_divns(pool->user.detach_time, NSEC_PER_SEC)))
288 goto err_cancel;
289
290 if (pool->mp_ops && pool->mp_ops->nl_fill(pool->mp_priv, rsp, NULL))
291 goto err_cancel;
292
293 genlmsg_end(rsp, hdr);
294
295 return 0;
296err_cancel:
297 genlmsg_cancel(rsp, hdr);
298 return -EMSGSIZE;
299}
300
301static void netdev_nl_page_pool_event(const struct page_pool *pool, u32 cmd)
302{
303 struct genl_info info;
304 struct sk_buff *ntf;
305 struct net *net;
306
307 lockdep_assert_held(&page_pools_lock);
308
309 /* 'invisible' page pools don't matter */
310 if (hlist_unhashed(&pool->user.list))
311 return;
312 net = dev_net(pool->slow.netdev);
313
314 if (!genl_has_listeners(&netdev_nl_family, net, NETDEV_NLGRP_PAGE_POOL))
315 return;
316
317 genl_info_init_ntf(&info, &netdev_nl_family, cmd);
318
319 ntf = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL);
320 if (!ntf)
321 return;
322
323 if (page_pool_nl_fill(ntf, pool, &info)) {
324 nlmsg_free(ntf);
325 return;
326 }
327
328 genlmsg_multicast_netns(&netdev_nl_family, net, ntf,
329 0, NETDEV_NLGRP_PAGE_POOL, GFP_KERNEL);
330}
331
332int netdev_nl_page_pool_get_doit(struct sk_buff *skb, struct genl_info *info)
333{
334 u32 id;
335
336 if (GENL_REQ_ATTR_CHECK(info, NETDEV_A_PAGE_POOL_ID))
337 return -EINVAL;
338
339 id = nla_get_uint(info->attrs[NETDEV_A_PAGE_POOL_ID]);
340
341 return netdev_nl_page_pool_get_do(info, id, page_pool_nl_fill);
342}
343
344int netdev_nl_page_pool_get_dumpit(struct sk_buff *skb,
345 struct netlink_callback *cb)
346{
347 const struct genl_info *info = genl_info_dump(cb);
348
349 return netdev_nl_page_pool_get_dump(skb, cb, page_pool_nl_fill,
350 info->attrs[NETDEV_A_PAGE_POOL_IFINDEX]);
351}
352
353int page_pool_list(struct page_pool *pool)
354{
355 static u32 id_alloc_next;
356 int err;
357
358 mutex_lock(&page_pools_lock);
359 err = xa_alloc_cyclic(&page_pools, &pool->user.id, pool, xa_limit_32b,
360 &id_alloc_next, GFP_KERNEL);
361 if (err < 0)
362 goto err_unlock;
363
364 INIT_HLIST_NODE(&pool->user.list);
365 if (pool->slow.netdev) {
366 hlist_add_head(&pool->user.list,
367 &pool->slow.netdev->page_pools);
368 netdev_nl_page_pool_event(pool, NETDEV_CMD_PAGE_POOL_ADD_NTF);
369 }
370
371 mutex_unlock(&page_pools_lock);
372 return 0;
373
374err_unlock:
375 mutex_unlock(&page_pools_lock);
376 return err;
377}
378
379void page_pool_detached(struct page_pool *pool)
380{
381 mutex_lock(&page_pools_lock);
382 pool->user.detach_time = ktime_get_boottime();
383 netdev_nl_page_pool_event(pool, NETDEV_CMD_PAGE_POOL_CHANGE_NTF);
384 mutex_unlock(&page_pools_lock);
385}
386
387void page_pool_unlist(struct page_pool *pool)
388{
389 mutex_lock(&page_pools_lock);
390 netdev_nl_page_pool_event(pool, NETDEV_CMD_PAGE_POOL_DEL_NTF);
391 xa_erase(&page_pools, pool->user.id);
392 if (!hlist_unhashed(&pool->user.list))
393 hlist_del(&pool->user.list);
394 mutex_unlock(&page_pools_lock);
395}
396
397int page_pool_check_memory_provider(struct net_device *dev,
398 struct netdev_rx_queue *rxq)
399{
400 void *binding = rxq->mp_params.mp_priv;
401 struct page_pool *pool;
402 struct hlist_node *n;
403
404 if (!binding)
405 return 0;
406
407 mutex_lock(&page_pools_lock);
408 hlist_for_each_entry_safe(pool, n, &dev->page_pools, user.list) {
409 if (pool->mp_priv != binding)
410 continue;
411
412 if (pool->slow.queue_idx == get_netdev_rx_queue_index(rxq)) {
413 mutex_unlock(&page_pools_lock);
414 return 0;
415 }
416 }
417 mutex_unlock(&page_pools_lock);
418 return -ENODATA;
419}
420
421static void page_pool_unreg_netdev_wipe(struct net_device *netdev)
422{
423 struct page_pool *pool;
424 struct hlist_node *n;
425
426 mutex_lock(&page_pools_lock);
427 hlist_for_each_entry_safe(pool, n, &netdev->page_pools, user.list) {
428 hlist_del_init(&pool->user.list);
429 pool->slow.netdev = NET_PTR_POISON;
430 }
431 mutex_unlock(&page_pools_lock);
432}
433
434static void page_pool_unreg_netdev(struct net_device *netdev)
435{
436 struct page_pool *pool, *last;
437 struct net_device *lo;
438
439 lo = dev_net(netdev)->loopback_dev;
440
441 mutex_lock(&page_pools_lock);
442 last = NULL;
443 hlist_for_each_entry(pool, &netdev->page_pools, user.list) {
444 pool->slow.netdev = lo;
445 netdev_nl_page_pool_event(pool,
446 NETDEV_CMD_PAGE_POOL_CHANGE_NTF);
447 last = pool;
448 }
449 if (last)
450 hlist_splice_init(&netdev->page_pools, &last->user.list,
451 &lo->page_pools);
452 mutex_unlock(&page_pools_lock);
453}
454
455static int
456page_pool_netdevice_event(struct notifier_block *nb,
457 unsigned long event, void *ptr)
458{
459 struct net_device *netdev = netdev_notifier_info_to_dev(ptr);
460
461 if (event != NETDEV_UNREGISTER)
462 return NOTIFY_DONE;
463
464 if (hlist_empty(&netdev->page_pools))
465 return NOTIFY_OK;
466
467 if (netdev->ifindex != LOOPBACK_IFINDEX)
468 page_pool_unreg_netdev(netdev);
469 else
470 page_pool_unreg_netdev_wipe(netdev);
471 return NOTIFY_OK;
472}
473
474static struct notifier_block page_pool_netdevice_nb = {
475 .notifier_call = page_pool_netdevice_event,
476};
477
478static int __init page_pool_user_init(void)
479{
480 return register_netdevice_notifier(&page_pool_netdevice_nb);
481}
482
483subsys_initcall(page_pool_user_init);