projects/linux·blob uaccess-cleanups · refs

626 lines · 15.4 KiB

1// SPDX-License-Identifier: GPL-2.0-or-later
2/*
3 * Copyright (c) 2016 Mellanox Technologies. All rights reserved.
4 * Copyright (c) 2016 Jiri Pirko <[email protected]>
5 */
6
7#include <net/genetlink.h>
8#define CREATE_TRACE_POINTS
9#include <trace/events/devlink.h>
10
11#include "devl_internal.h"
12
13EXPORT_TRACEPOINT_SYMBOL_GPL(devlink_hwmsg);
14EXPORT_TRACEPOINT_SYMBOL_GPL(devlink_hwerr);
15EXPORT_TRACEPOINT_SYMBOL_GPL(devlink_trap_report);
16
17DEFINE_XARRAY_FLAGS(devlinks, XA_FLAGS_ALLOC);
18
19static struct devlink *devlinks_xa_get(unsigned long index)
20{
21 struct devlink *devlink;
22
23 rcu_read_lock();
24 devlink = xa_find(&devlinks, &index, index, DEVLINK_REGISTERED);
25 if (!devlink || !devlink_try_get(devlink))
26 devlink = NULL;
27 rcu_read_unlock();
28 return devlink;
29}
30
31/* devlink_rels xarray contains 1:1 relationships between
32 * devlink object and related nested devlink instance.
33 * The xarray index is used to get the nested object from
34 * the nested-in object code.
35 */
36static DEFINE_XARRAY_FLAGS(devlink_rels, XA_FLAGS_ALLOC1);
37
38#define DEVLINK_REL_IN_USE XA_MARK_0
39
40struct devlink_rel {
41 u32 index;
42 refcount_t refcount;
43 u32 devlink_index;
44 struct {
45 u32 devlink_index;
46 u32 obj_index;
47 devlink_rel_notify_cb_t *notify_cb;
48 devlink_rel_cleanup_cb_t *cleanup_cb;
49 struct delayed_work notify_work;
50 } nested_in;
51};
52
53static void devlink_rel_free(struct devlink_rel *rel)
54{
55 xa_erase(&devlink_rels, rel->index);
56 kfree(rel);
57}
58
59static void __devlink_rel_get(struct devlink_rel *rel)
60{
61 refcount_inc(&rel->refcount);
62}
63
64static void __devlink_rel_put(struct devlink_rel *rel)
65{
66 if (refcount_dec_and_test(&rel->refcount))
67 devlink_rel_free(rel);
68}
69
70struct devlink *__must_check devlink_nested_in_get_lock(struct devlink *devlink)
71{
72 devl_assert_locked(devlink);
73 if (!devlink->rel)
74 return NULL;
75 devlink = devlinks_xa_get(devlink->rel->nested_in.devlink_index);
76 if (!devlink)
77 return NULL;
78 devl_lock(devlink);
79 if (devl_is_registered(devlink))
80 return devlink;
81 devl_unlock(devlink);
82 devlink_put(devlink);
83 return NULL;
84}
85
86static void devlink_rel_nested_in_notify_work(struct work_struct *work)
87{
88 struct devlink_rel *rel = container_of(work, struct devlink_rel,
89 nested_in.notify_work.work);
90 struct devlink *devlink;
91
92 devlink = devlinks_xa_get(rel->nested_in.devlink_index);
93 if (!devlink)
94 goto rel_put;
95 if (!devl_trylock(devlink)) {
96 devlink_put(devlink);
97 goto reschedule_work;
98 }
99 if (!devl_is_registered(devlink)) {
100 devl_unlock(devlink);
101 devlink_put(devlink);
102 goto rel_put;
103 }
104 if (!xa_get_mark(&devlink_rels, rel->index, DEVLINK_REL_IN_USE))
105 rel->nested_in.cleanup_cb(devlink, rel->nested_in.obj_index, rel->index);
106 rel->nested_in.notify_cb(devlink, rel->nested_in.obj_index);
107 devl_unlock(devlink);
108 devlink_put(devlink);
109
110rel_put:
111 __devlink_rel_put(rel);
112 return;
113
114reschedule_work:
115 /* The work may have been queued again meanwhile, which took its own
116 * reference. Drop ours in that case.
117 */
118 if (!schedule_delayed_work(&rel->nested_in.notify_work, 1))
119 __devlink_rel_put(rel);
120}
121
122static void devlink_rel_nested_in_notify_work_schedule(struct devlink_rel *rel)
123{
124 __devlink_rel_get(rel);
125 /* The pending work holds a reference already, drop the new one. */
126 if (!schedule_delayed_work(&rel->nested_in.notify_work, 0))
127 __devlink_rel_put(rel);
128}
129
130static struct devlink_rel *devlink_rel_alloc(void)
131{
132 struct devlink_rel *rel;
133 static u32 next;
134 int err;
135
136 rel = kzalloc_obj(*rel);
137 if (!rel)
138 return ERR_PTR(-ENOMEM);
139
140 err = xa_alloc_cyclic(&devlink_rels, &rel->index, rel,
141 xa_limit_32b, &next, GFP_KERNEL);
142 if (err < 0) {
143 kfree(rel);
144 return ERR_PTR(err);
145 }
146
147 refcount_set(&rel->refcount, 1);
148 INIT_DELAYED_WORK(&rel->nested_in.notify_work,
149 &devlink_rel_nested_in_notify_work);
150 return rel;
151}
152
153static void devlink_rel_put(struct devlink *devlink)
154{
155 struct devlink_rel *rel = devlink->rel;
156
157 if (!rel)
158 return;
159 xa_clear_mark(&devlink_rels, rel->index, DEVLINK_REL_IN_USE);
160 devlink_rel_nested_in_notify_work_schedule(rel);
161 __devlink_rel_put(rel);
162 devlink->rel = NULL;
163}
164
165void devlink_rel_nested_in_clear(u32 rel_index)
166{
167 xa_clear_mark(&devlink_rels, rel_index, DEVLINK_REL_IN_USE);
168}
169
170int devlink_rel_nested_in_add(u32 *rel_index, u32 devlink_index,
171 u32 obj_index, devlink_rel_notify_cb_t *notify_cb,
172 devlink_rel_cleanup_cb_t *cleanup_cb,
173 struct devlink *devlink)
174{
175 struct devlink_rel *rel = devlink_rel_alloc();
176
177 ASSERT_DEVLINK_NOT_REGISTERED(devlink);
178
179 if (IS_ERR(rel))
180 return PTR_ERR(rel);
181
182 rel->devlink_index = devlink->index;
183 rel->nested_in.devlink_index = devlink_index;
184 rel->nested_in.obj_index = obj_index;
185 rel->nested_in.notify_cb = notify_cb;
186 rel->nested_in.cleanup_cb = cleanup_cb;
187 *rel_index = rel->index;
188 xa_set_mark(&devlink_rels, rel->index, DEVLINK_REL_IN_USE);
189 devlink->rel = rel;
190 return 0;
191}
192
193/**
194 * devlink_rel_nested_in_notify - Notify the object this devlink
195 * instance is nested in.
196 * @devlink: devlink
197 *
198 * This is called upon network namespace change of devlink instance.
199 * In case this devlink instance is nested in another devlink object,
200 * a notification of a change of this object should be sent
201 * over netlink. The parent devlink instance lock needs to be
202 * taken during the notification preparation.
203 * Since the parent may or may not be locked, 'work' is utilized.
204 */
205void devlink_rel_nested_in_notify(struct devlink *devlink)
206{
207 struct devlink_rel *rel = devlink->rel;
208
209 if (!rel)
210 return;
211 devlink_rel_nested_in_notify_work_schedule(rel);
212}
213
214static struct devlink_rel *devlink_rel_find(unsigned long rel_index)
215{
216 return xa_find(&devlink_rels, &rel_index, rel_index,
217 DEVLINK_REL_IN_USE);
218}
219
220static struct devlink *devlink_rel_devlink_get(u32 rel_index)
221{
222 struct devlink_rel *rel;
223 u32 devlink_index;
224
225 if (!rel_index)
226 return NULL;
227 xa_lock(&devlink_rels);
228 rel = devlink_rel_find(rel_index);
229 if (rel)
230 devlink_index = rel->devlink_index;
231 xa_unlock(&devlink_rels);
232 if (!rel)
233 return NULL;
234 return devlinks_xa_get(devlink_index);
235}
236
237int devlink_rel_devlink_handle_put(struct sk_buff *msg, struct devlink *devlink,
238 u32 rel_index, int attrtype,
239 bool *msg_updated)
240{
241 struct net *net = devlink_net(devlink);
242 struct devlink *rel_devlink;
243 int err;
244
245 rel_devlink = devlink_rel_devlink_get(rel_index);
246 if (!rel_devlink)
247 return 0;
248 err = devlink_nl_put_nested_handle(msg, net, rel_devlink, attrtype);
249 devlink_put(rel_devlink);
250 if (!err && msg_updated)
251 *msg_updated = true;
252 return err;
253}
254
255void *devlink_priv(struct devlink *devlink)
256{
257 return &devlink->priv;
258}
259EXPORT_SYMBOL_GPL(devlink_priv);
260
261struct devlink *priv_to_devlink(void *priv)
262{
263 return container_of(priv, struct devlink, priv);
264}
265EXPORT_SYMBOL_GPL(priv_to_devlink);
266
267struct device *devlink_to_dev(const struct devlink *devlink)
268{
269 return devlink->dev;
270}
271EXPORT_SYMBOL_GPL(devlink_to_dev);
272
273const char *devlink_bus_name(const struct devlink *devlink)
274{
275 return devlink->dev ? devlink->dev->bus->name : DEVLINK_INDEX_BUS_NAME;
276}
277EXPORT_SYMBOL_GPL(devlink_bus_name);
278
279const char *devlink_dev_name(const struct devlink *devlink)
280{
281 return devlink->dev ? dev_name(devlink->dev) : devlink->dev_name_index;
282}
283EXPORT_SYMBOL_GPL(devlink_dev_name);
284
285const char *devlink_dev_driver_name(const struct devlink *devlink)
286{
287 return devlink->dev_driver->name;
288}
289EXPORT_SYMBOL_GPL(devlink_dev_driver_name);
290
291struct net *devlink_net(const struct devlink *devlink)
292{
293 return read_pnet(&devlink->_net);
294}
295EXPORT_SYMBOL_GPL(devlink_net);
296
297void devl_assert_locked(struct devlink *devlink)
298{
299 lockdep_assert_held(&devlink->lock);
300}
301EXPORT_SYMBOL_GPL(devl_assert_locked);
302
303#ifdef CONFIG_LOCKDEP
304/* For use in conjunction with LOCKDEP only e.g. rcu_dereference_protected() */
305bool devl_lock_is_held(struct devlink *devlink)
306{
307 return lockdep_is_held(&devlink->lock);
308}
309EXPORT_SYMBOL_GPL(devl_lock_is_held);
310#endif
311
312void devl_lock(struct devlink *devlink)
313{
314 mutex_lock(&devlink->lock);
315}
316EXPORT_SYMBOL_GPL(devl_lock);
317
318int devl_trylock(struct devlink *devlink)
319{
320 return mutex_trylock(&devlink->lock);
321}
322EXPORT_SYMBOL_GPL(devl_trylock);
323
324void devl_unlock(struct devlink *devlink)
325{
326 mutex_unlock(&devlink->lock);
327}
328EXPORT_SYMBOL_GPL(devl_unlock);
329
330/**
331 * devlink_try_get() - try to obtain a reference on a devlink instance
332 * @devlink: instance to reference
333 *
334 * Obtain a reference on a devlink instance. A reference on a devlink instance
335 * only implies that it's safe to take the instance lock. It does not imply
336 * that the instance is registered, use devl_is_registered() after taking
337 * the instance lock to check registration status.
338 */
339struct devlink *__must_check devlink_try_get(struct devlink *devlink)
340{
341 if (refcount_inc_not_zero(&devlink->refcount))
342 return devlink;
343 return NULL;
344}
345
346static void devlink_release(struct work_struct *work)
347{
348 struct devlink *devlink;
349
350 devlink = container_of(to_rcu_work(work), struct devlink, rwork);
351
352 mutex_destroy(&devlink->lock);
353 lockdep_unregister_key(&devlink->lock_key);
354 if (devlink->dev)
355 put_device(devlink->dev);
356 else
357 kfree(devlink->dev_name_index);
358 kvfree(devlink);
359}
360
361void devlink_put(struct devlink *devlink)
362{
363 if (refcount_dec_and_test(&devlink->refcount))
364 queue_rcu_work(system_percpu_wq, &devlink->rwork);
365}
366
367static struct devlink *__devlinks_xa_find_get(struct net *net,
368 unsigned long *indexp,
369 unsigned long end)
370{
371 struct devlink *devlink = NULL;
372
373 rcu_read_lock();
374retry:
375 devlink = xa_find(&devlinks, indexp, end, DEVLINK_REGISTERED);
376 if (!devlink)
377 goto unlock;
378
379 if (!devlink_try_get(devlink))
380 goto next;
381 if (!net_eq(devlink_net(devlink), net)) {
382 devlink_put(devlink);
383 goto next;
384 }
385unlock:
386 rcu_read_unlock();
387 return devlink;
388
389next:
390 (*indexp)++;
391 goto retry;
392}
393
394struct devlink *devlinks_xa_find_get(struct net *net, unsigned long *indexp)
395{
396 return __devlinks_xa_find_get(net, indexp, ULONG_MAX);
397}
398
399struct devlink *devlinks_xa_lookup_get(struct net *net, unsigned long index)
400{
401 return __devlinks_xa_find_get(net, &index, index);
402}
403
404/**
405 * devl_register - Register devlink instance
406 * @devlink: devlink
407 */
408int devl_register(struct devlink *devlink)
409{
410 ASSERT_DEVLINK_NOT_REGISTERED(devlink);
411 devl_assert_locked(devlink);
412
413 xa_set_mark(&devlinks, devlink->index, DEVLINK_REGISTERED);
414 devlink_notify_register(devlink);
415 devlink_rel_nested_in_notify(devlink);
416
417 return 0;
418}
419EXPORT_SYMBOL_GPL(devl_register);
420
421void devlink_register(struct devlink *devlink)
422{
423 devl_lock(devlink);
424 devl_register(devlink);
425 devl_unlock(devlink);
426}
427EXPORT_SYMBOL_GPL(devlink_register);
428
429/**
430 * devl_unregister - Unregister devlink instance
431 * @devlink: devlink
432 */
433void devl_unregister(struct devlink *devlink)
434{
435 ASSERT_DEVLINK_REGISTERED(devlink);
436 devl_assert_locked(devlink);
437
438 devlink_notify_unregister(devlink);
439 xa_clear_mark(&devlinks, devlink->index, DEVLINK_REGISTERED);
440 devlink_rel_put(devlink);
441}
442EXPORT_SYMBOL_GPL(devl_unregister);
443
444void devlink_unregister(struct devlink *devlink)
445{
446 devl_lock(devlink);
447 devl_unregister(devlink);
448 devl_unlock(devlink);
449}
450EXPORT_SYMBOL_GPL(devlink_unregister);
451
452struct devlink *__devlink_alloc(const struct devlink_ops *ops, size_t priv_size,
453 struct net *net, struct device *dev,
454 const struct device_driver *dev_driver)
455{
456 struct devlink *devlink;
457 static u32 last_id;
458 int ret;
459
460 WARN_ON(!ops || !dev_driver);
461 if (!devlink_reload_actions_valid(ops))
462 return NULL;
463
464 devlink = kvzalloc_flex(*devlink, priv, priv_size);
465 if (!devlink)
466 return NULL;
467
468 ret = xa_alloc_cyclic(&devlinks, &devlink->index, devlink, xa_limit_31b,
469 &last_id, GFP_KERNEL);
470 if (ret < 0)
471 goto err_xa_alloc;
472
473 if (dev) {
474 devlink->dev = get_device(dev);
475 } else {
476 devlink->dev_name_index = kasprintf(GFP_KERNEL, "%u", devlink->index);
477 if (!devlink->dev_name_index)
478 goto err_kasprintf;
479 }
480
481 devlink->ops = ops;
482 devlink->dev_driver = dev_driver;
483 xa_init_flags(&devlink->ports, XA_FLAGS_ALLOC);
484 xa_init_flags(&devlink->params, XA_FLAGS_ALLOC);
485 xa_init_flags(&devlink->snapshot_ids, XA_FLAGS_ALLOC);
486 xa_init_flags(&devlink->nested_rels, XA_FLAGS_ALLOC);
487 write_pnet(&devlink->_net, net);
488 INIT_LIST_HEAD(&devlink->rate_list);
489 INIT_LIST_HEAD(&devlink->linecard_list);
490 INIT_LIST_HEAD(&devlink->sb_list);
491 INIT_LIST_HEAD_RCU(&devlink->dpipe_table_list);
492 INIT_LIST_HEAD(&devlink->resource_list);
493 INIT_LIST_HEAD(&devlink->region_list);
494 INIT_LIST_HEAD(&devlink->reporter_list);
495 INIT_LIST_HEAD(&devlink->trap_list);
496 INIT_LIST_HEAD(&devlink->trap_group_list);
497 INIT_LIST_HEAD(&devlink->trap_policer_list);
498 INIT_RCU_WORK(&devlink->rwork, devlink_release);
499 lockdep_register_key(&devlink->lock_key);
500 mutex_init(&devlink->lock);
501 lockdep_set_class(&devlink->lock, &devlink->lock_key);
502 refcount_set(&devlink->refcount, 1);
503
504 return devlink;
505
506err_kasprintf:
507 xa_erase(&devlinks, devlink->index);
508err_xa_alloc:
509 kvfree(devlink);
510 return NULL;
511}
512
513/**
514 * devlink_alloc_ns - Allocate new devlink instance resources
515 * in specific namespace
516 *
517 * @ops: ops
518 * @priv_size: size of user private data
519 * @net: net namespace
520 * @dev: parent device
521 *
522 * Allocate new devlink instance resources, including devlink index
523 * and name.
524 */
525struct devlink *devlink_alloc_ns(const struct devlink_ops *ops,
526 size_t priv_size, struct net *net,
527 struct device *dev)
528{
529 WARN_ON(!dev);
530 return __devlink_alloc(ops, priv_size, net, dev, dev->driver);
531}
532EXPORT_SYMBOL_GPL(devlink_alloc_ns);
533
534/**
535 * devlink_free - Free devlink instance resources
536 *
537 * @devlink: devlink
538 */
539void devlink_free(struct devlink *devlink)
540{
541 ASSERT_DEVLINK_NOT_REGISTERED(devlink);
542
543 devl_lock(devlink);
544 WARN_ON(devlink_rates_check(devlink, NULL, NULL));
545 devl_unlock(devlink);
546 devlink_rel_put(devlink);
547
548 WARN_ON(!list_empty(&devlink->trap_policer_list));
549 WARN_ON(!list_empty(&devlink->trap_group_list));
550 WARN_ON(!list_empty(&devlink->trap_list));
551 WARN_ON(!list_empty(&devlink->reporter_list));
552 WARN_ON(!list_empty(&devlink->region_list));
553 WARN_ON(!list_empty(&devlink->resource_list));
554 WARN_ON(!list_empty(&devlink->dpipe_table_list));
555 WARN_ON(!list_empty(&devlink->sb_list));
556 WARN_ON(!list_empty(&devlink->linecard_list));
557 WARN_ON(!xa_empty(&devlink->ports));
558
559 xa_destroy(&devlink->nested_rels);
560 xa_destroy(&devlink->snapshot_ids);
561 xa_destroy(&devlink->params);
562 xa_destroy(&devlink->ports);
563
564 xa_erase(&devlinks, devlink->index);
565
566 devlink_put(devlink);
567}
568EXPORT_SYMBOL_GPL(devlink_free);
569
570static void __net_exit devlink_pernet_pre_exit(struct net *net)
571{
572 struct devlink *devlink;
573 u32 actions_performed;
574 unsigned long index;
575 int err;
576
577 /* In case network namespace is getting destroyed, reload
578 * all devlink instances from this namespace into init_net.
579 */
580 devlinks_xa_for_each_registered_get(net, index, devlink) {
581 devl_dev_lock(devlink, true);
582 err = 0;
583 if (devl_is_registered(devlink))
584 err = devlink_reload(devlink, &init_net,
585 DEVLINK_RELOAD_ACTION_DRIVER_REINIT,
586 DEVLINK_RELOAD_LIMIT_UNSPEC,
587 &actions_performed, NULL);
588 devl_dev_unlock(devlink, true);
589 devlink_put(devlink);
590 if (err && err != -EOPNOTSUPP)
591 pr_warn("Failed to reload devlink instance into init_net\n");
592 }
593}
594
595static struct pernet_operations devlink_pernet_ops __net_initdata = {
596 .pre_exit = devlink_pernet_pre_exit,
597};
598
599static struct notifier_block devlink_port_netdevice_nb = {
600 .notifier_call = devlink_port_netdevice_event,
601};
602
603static int __init devlink_init(void)
604{
605 int err;
606
607 err = register_pernet_subsys(&devlink_pernet_ops);
608 if (err)
609 goto out;
610 err = genl_register_family(&devlink_nl_family);
611 if (err)
612 goto out_unreg_pernet_subsys;
613 err = register_netdevice_notifier(&devlink_port_netdevice_nb);
614 if (!err)
615 return 0;
616
617 genl_unregister_family(&devlink_nl_family);
618
619out_unreg_pernet_subsys:
620 unregister_pernet_subsys(&devlink_pernet_ops);
621out:
622 WARN_ON(err);
623 return err;
624}
625
626subsys_initcall(devlink_init);