linux - drivers/infiniband/core/device.c

3346 lines · 89.9 KiB

1	/*
2	* Copyright (c) 2004 Topspin Communications. All rights reserved.
3	* Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved.
4	*
5	* This software is available to you under a choice of one of two
6	* licenses. You may choose to be licensed under the terms of the GNU
7	* General Public License (GPL) Version 2, available from the file
8	* COPYING in the main directory of this source tree, or the
9	* OpenIB.org BSD license below:
10	*
11	* Redistribution and use in source and binary forms, with or
12	* without modification, are permitted provided that the following
13	* conditions are met:
14	*
15	* - Redistributions of source code must retain the above
16	* copyright notice, this list of conditions and the following
17	* disclaimer.
18	*
19	* - Redistributions in binary form must reproduce the above
20	* copyright notice, this list of conditions and the following
21	* disclaimer in the documentation and/or other materials
22	* provided with the distribution.
23	*
24	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
25	* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
26	* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
27	* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
28	* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
29	* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
30	* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
31	* SOFTWARE.
32	*/
33
34	#include <linux/module.h>
35	#include <linux/string.h>
36	#include <linux/errno.h>
37	#include <linux/kernel.h>
38	#include <linux/slab.h>
39	#include <linux/init.h>
40	#include <linux/netdevice.h>
41	#include <net/net_namespace.h>
42	#include <linux/security.h>
43	#include <linux/notifier.h>
44	#include <linux/hashtable.h>
45	#include <linux/cc_platform.h>
46	#include <rdma/rdma_netlink.h>
47	#include <rdma/ib_addr.h>
48	#include <rdma/ib_cache.h>
49	#include <rdma/rdma_counter.h>
50
51	#include "core_priv.h"
52	#include "restrack.h"
53
54	MODULE_AUTHOR("Roland Dreier");
55	MODULE_DESCRIPTION("core kernel InfiniBand API");
56	MODULE_LICENSE("Dual BSD/GPL");
57
58	struct workqueue_struct *ib_comp_wq;
59	struct workqueue_struct *ib_comp_unbound_wq;
60	struct workqueue_struct *ib_wq;
61	EXPORT_SYMBOL_GPL(ib_wq);
62	static struct workqueue_struct *ib_unreg_wq;
63
64	/*
65	* Each of the three rwsem locks (devices, clients, client_data) protects the
66	* xarray of the same name. Specifically it allows the caller to assert that
67	* the MARK will/will not be changing under the lock, and for devices and
68	* clients, that the value in the xarray is still a valid pointer. Change of
69	* the MARK is linked to the object state, so holding the lock and testing the
70	* MARK also asserts that the contained object is in a certain state.
71	*
72	* This is used to build a two stage register/unregister flow where objects
73	* can continue to be in the xarray even though they are still in progress to
74	* register/unregister.
75	*
76	* The xarray itself provides additional locking, and restartable iteration,
77	* which is also relied on.
78	*
79	* Locks should not be nested, with the exception of client_data, which is
80	* allowed to nest under the read side of the other two locks.
81	*
82	* The devices_rwsem also protects the device name list, any change or
83	* assignment of device name must also hold the write side to guarantee unique
84	* names.
85	*/
86
87	/*
88	* devices contains devices that have had their names assigned. The
89	* devices may not be registered. Users that care about the registration
90	* status need to call ib_device_try_get() on the device to ensure it is
91	* registered, and keep it registered, for the required duration.
92	*
93	*/
94	static DEFINE_XARRAY_FLAGS(devices, XA_FLAGS_ALLOC);
95	static DECLARE_RWSEM(devices_rwsem);
96	#define DEVICE_REGISTERED XA_MARK_1
97	#define DEVICE_GID_UPDATES XA_MARK_2
98
99	static u32 highest_client_id;
100	#define CLIENT_REGISTERED XA_MARK_1
101	static DEFINE_XARRAY_FLAGS(clients, XA_FLAGS_ALLOC);
102	static DECLARE_RWSEM(clients_rwsem);
103
104	static void ib_client_put(struct ib_client *client)
105	{
106	if (refcount_dec_and_test(&client->uses))
107	complete(&client->uses_zero);
108	}
109
110	/*
111	* If client_data is registered then the corresponding client must also still
112	* be registered.
113	*/
114	#define CLIENT_DATA_REGISTERED XA_MARK_1
115
116	unsigned int rdma_dev_net_id;
117
118	/*
119	* A list of net namespaces is maintained in an xarray. This is necessary
120	* because we can't get the locking right using the existing net ns list. We
121	* would require a init_net callback after the list is updated.
122	*/
123	static DEFINE_XARRAY_FLAGS(rdma_nets, XA_FLAGS_ALLOC);
124	/*
125	* rwsem to protect accessing the rdma_nets xarray entries.
126	*/
127	static DECLARE_RWSEM(rdma_nets_rwsem);
128
129	bool ib_devices_shared_netns = true;
130	module_param_named(netns_mode, ib_devices_shared_netns, bool, 0444);
131	MODULE_PARM_DESC(netns_mode,
132	"Share device among net namespaces; default=1 (shared). In exclusive mode device names are unique per net namespace");
133	/**
134	* rdma_dev_access_netns() - Return whether an rdma device can be accessed
135	* from a specified net namespace or not.
136	* @dev: Pointer to rdma device which needs to be checked
137	* @net: Pointer to net namesapce for which access to be checked
138	*
139	* When the rdma device is in shared mode, it ignores the net namespace.
140	* When the rdma device is exclusive to a net namespace, rdma device net
141	* namespace is checked against the specified one.
142	*/
143	bool rdma_dev_access_netns(const struct ib_device dev, const struct net net)
144	{
145	return (ib_devices_shared_netns \|\|
146	net_eq(read_pnet(&dev->coredev.rdma_net), net));
147	}
148	EXPORT_SYMBOL(rdma_dev_access_netns);
149
150	/**
151	* rdma_dev_has_raw_cap() - Returns whether a specified rdma device has
152	* CAP_NET_RAW capability or not.
153	*
154	* @dev: Pointer to rdma device whose capability to be checked
155	*
156	* Returns true if a rdma device's owning user namespace has CAP_NET_RAW
157	* capability, otherwise false. When rdma subsystem is in legacy shared network,
158	* namespace mode, the default net namespace is considered.
159	*/
160	bool rdma_dev_has_raw_cap(const struct ib_device *dev)
161	{
162	const struct net *net;
163
164	/* Network namespace is the resource whose user namespace
165	* to be considered. When in shared mode, there is no reliable
166	* network namespace resource, so consider the default net namespace.
167	*/
168	if (ib_devices_shared_netns)
169	net = &init_net;
170	else
171	net = read_pnet(&dev->coredev.rdma_net);
172
173	return ns_capable(net->user_ns, CAP_NET_RAW);
174	}
175	EXPORT_SYMBOL(rdma_dev_has_raw_cap);
176
177	/*
178	* xarray has this behavior where it won't iterate over NULL values stored in
179	* allocated arrays. So we need our own iterator to see all values stored in
180	* the array. This does the same thing as xa_for_each except that it also
181	* returns NULL valued entries if the array is allocating. Simplified to only
182	* work on simple xarrays.
183	*/
184	static void xan_find_marked(struct xarray xa, unsigned long *indexp,
185	xa_mark_t filter)
186	{
187	XA_STATE(xas, xa, *indexp);
188	void *entry;
189
190	rcu_read_lock();
191	do {
192	entry = xas_find_marked(&xas, ULONG_MAX, filter);
193	if (xa_is_zero(entry))
194	break;
195	} while (xas_retry(&xas, entry));
196	rcu_read_unlock();
197
198	if (entry) {
199	*indexp = xas.xa_index;
200	if (xa_is_zero(entry))
201	return NULL;
202	return entry;
203	}
204	return XA_ERROR(-ENOENT);
205	}
206	#define xan_for_each_marked(xa, index, entry, filter) \
207	for (index = 0, entry = xan_find_marked(xa, &(index), filter); \
208	!xa_is_err(entry); \
209	(index)++, entry = xan_find_marked(xa, &(index), filter))
210
211	/* RCU hash table mapping netdevice pointers to struct ib_port_data */
212	static DEFINE_SPINLOCK(ndev_hash_lock);
213	static DECLARE_HASHTABLE(ndev_hash, 5);
214
215	static void free_netdevs(struct ib_device *ib_dev);
216	static void ib_unregister_work(struct work_struct *work);
217	static void __ib_unregister_device(struct ib_device *device);
218	static int ib_security_change(struct notifier_block *nb, unsigned long event,
219	void *lsm_data);
220	static void ib_policy_change_task(struct work_struct *work);
221	static DECLARE_WORK(ib_policy_change_work, ib_policy_change_task);
222
223	static void __ibdev_printk(const char level, const struct ib_device ibdev,
224	struct va_format *vaf)
225	{
226	if (ibdev && ibdev->dev.parent)
227	dev_printk_emit(level[1] - '0',
228	ibdev->dev.parent,
229	"%s %s %s: %pV",
230	dev_driver_string(ibdev->dev.parent),
231	dev_name(ibdev->dev.parent),
232	dev_name(&ibdev->dev),
233	vaf);
234	else if (ibdev)
235	printk("%s%s: %pV",
236	level, dev_name(&ibdev->dev), vaf);
237	else
238	printk("%s(NULL ib_device): %pV", level, vaf);
239	}
240
241	#define define_ibdev_printk_level(func, level) \
242	void func(const struct ib_device ibdev, const char fmt, ...) \
243	{ \
244	struct va_format vaf; \
245	va_list args; \
246	\
247	va_start(args, fmt); \
248	\
249	vaf.fmt = fmt; \
250	vaf.va = &args; \
251	\
252	__ibdev_printk(level, ibdev, &vaf); \
253	\
254	va_end(args); \
255	} \
256	EXPORT_SYMBOL(func);
257
258	define_ibdev_printk_level(ibdev_emerg, KERN_EMERG);
259	define_ibdev_printk_level(ibdev_alert, KERN_ALERT);
260	define_ibdev_printk_level(ibdev_crit, KERN_CRIT);
261	define_ibdev_printk_level(ibdev_err, KERN_ERR);
262	define_ibdev_printk_level(ibdev_warn, KERN_WARNING);
263	define_ibdev_printk_level(ibdev_notice, KERN_NOTICE);
264	define_ibdev_printk_level(ibdev_info, KERN_INFO);
265
266	static struct notifier_block ibdev_lsm_nb = {
267	.notifier_call = ib_security_change,
268	};
269
270	static int rdma_dev_change_netns(struct ib_device device, struct net cur_net,
271	struct net net, const char requested_name,
272	const char *fallback_pattern);
273
274	/* Pointer to the RCU head at the start of the ib_port_data array */
275	struct ib_port_data_rcu {
276	struct rcu_head rcu_head;
277	struct ib_port_data pdata[];
278	};
279
280	static void ib_device_check_mandatory(struct ib_device *device)
281	{
282	#define IB_MANDATORY_FUNC(x) { offsetof(struct ib_device_ops, x), #x }
283	static const struct {
284	size_t offset;
285	char *name;
286	} mandatory_table[] = {
287	IB_MANDATORY_FUNC(query_device),
288	IB_MANDATORY_FUNC(query_port),
289	IB_MANDATORY_FUNC(alloc_pd),
290	IB_MANDATORY_FUNC(dealloc_pd),
291	IB_MANDATORY_FUNC(create_qp),
292	IB_MANDATORY_FUNC(modify_qp),
293	IB_MANDATORY_FUNC(destroy_qp),
294	IB_MANDATORY_FUNC(post_send),
295	IB_MANDATORY_FUNC(post_recv),
296	IB_MANDATORY_FUNC(create_cq),
297	IB_MANDATORY_FUNC(destroy_cq),
298	IB_MANDATORY_FUNC(poll_cq),
299	IB_MANDATORY_FUNC(req_notify_cq),
300	IB_MANDATORY_FUNC(get_dma_mr),
301	IB_MANDATORY_FUNC(reg_user_mr),
302	IB_MANDATORY_FUNC(dereg_mr),
303	IB_MANDATORY_FUNC(get_port_immutable)
304	};
305	int i;
306
307	device->kverbs_provider = true;
308	for (i = 0; i < ARRAY_SIZE(mandatory_table); ++i) {
309	if (!(void ) ((void ) &device->ops +
310	mandatory_table[i].offset)) {
311	device->kverbs_provider = false;
312	break;
313	}
314	}
315	}
316
317	/*
318	* Caller must perform ib_device_put() to return the device reference count
319	* when ib_device_get_by_index() returns valid device pointer.
320	*/
321	struct ib_device ib_device_get_by_index(const struct net net, u32 index)
322	{
323	struct ib_device *device;
324
325	down_read(&devices_rwsem);
326	device = xa_load(&devices, index);
327	if (device) {
328	if (!rdma_dev_access_netns(device, net)) {
329	device = NULL;
330	goto out;
331	}
332
333	if (!ib_device_try_get(device))
334	device = NULL;
335	}
336	out:
337	up_read(&devices_rwsem);
338	return device;
339	}
340
341	/**
342	* ib_device_put - Release IB device reference
343	* @device: device whose reference to be released
344	*
345	* ib_device_put() releases reference to the IB device to allow it to be
346	* unregistered and eventually free.
347	*/
348	void ib_device_put(struct ib_device *device)
349	{
350	if (refcount_dec_and_test(&device->refcount))
351	complete(&device->unreg_completion);
352	}
353	EXPORT_SYMBOL(ib_device_put);
354
355	static struct ib_device __ib_device_get_by_name(const char name,
356	const struct net *net)
357	{
358	struct ib_device *device;
359	unsigned long index;
360
361	xa_for_each (&devices, index, device)
362	if (rdma_dev_access_netns(device, net) &&
363	!strcmp(name, dev_name(&device->dev)))
364	return device;
365
366	return NULL;
367	}
368
369	static int rename_compat_devs(struct ib_device *device)
370	{
371	struct ib_core_device *cdev;
372	unsigned long index;
373	int ret = 0;
374
375	mutex_lock(&device->compat_devs_mutex);
376	xa_for_each (&device->compat_devs, index, cdev) {
377	ret = device_rename(&cdev->dev, dev_name(&device->dev));
378	if (ret) {
379	dev_warn(&cdev->dev,
380	"Fail to rename compatdev to new name %s\n",
381	dev_name(&device->dev));
382	break;
383	}
384	}
385	mutex_unlock(&device->compat_devs_mutex);
386	return ret;
387	}
388
389	int ib_device_rename(struct ib_device ibdev, const char name)
390	{
391	unsigned long index;
392	void *client_data;
393	int ret;
394
395	down_write(&devices_rwsem);
396	if (!strcmp(name, dev_name(&ibdev->dev))) {
397	up_write(&devices_rwsem);
398	return 0;
399	}
400
401	if (__ib_device_get_by_name(name, rdma_dev_net(ibdev))) {
402	up_write(&devices_rwsem);
403	return -EEXIST;
404	}
405
406	ret = device_rename(&ibdev->dev, name);
407	if (ret) {
408	up_write(&devices_rwsem);
409	return ret;
410	}
411
412	strscpy(ibdev->name, name, IB_DEVICE_NAME_MAX);
413	ret = rename_compat_devs(ibdev);
414
415	downgrade_write(&devices_rwsem);
416	down_read(&ibdev->client_data_rwsem);
417	xan_for_each_marked(&ibdev->client_data, index, client_data,
418	CLIENT_DATA_REGISTERED) {
419	struct ib_client *client = xa_load(&clients, index);
420
421	if (!client \|\| !client->rename)
422	continue;
423
424	client->rename(ibdev, client_data);
425	}
426	up_read(&ibdev->client_data_rwsem);
427	rdma_nl_notify_event(ibdev, 0, RDMA_RENAME_EVENT);
428	up_read(&devices_rwsem);
429	return 0;
430	}
431
432	int ib_device_set_dim(struct ib_device *ibdev, u8 use_dim)
433	{
434	if (use_dim > 1)
435	return -EINVAL;
436	ibdev->use_cq_dim = use_dim;
437
438	return 0;
439	}
440
441	/*
442	* Pick a free index for the '%d' style @name pattern within net namespace
443	* @net. Returns the index on success or a negative errno. The caller builds
444	* the final unique device name from the returned index.
445	*/
446	static int __alloc_name_id(struct net net, const char name,
447	const struct ib_device *skip)
448	{
449	struct ib_device *device;
450	unsigned long index;
451	struct ida inuse;
452	int rc;
453	int i;
454
455	lockdep_assert_held_write(&devices_rwsem);
456	ida_init(&inuse);
457	xa_for_each (&devices, index, device) {
458	char buf[IB_DEVICE_NAME_MAX];
459
460	if (device == skip \|\| !rdma_dev_access_netns(device, net))
461	continue;
462	if (sscanf(dev_name(&device->dev), name, &i) != 1)
463	continue;
464	if (i < 0 \|\| i >= INT_MAX)
465	continue;
466	snprintf(buf, sizeof buf, name, i);
467	if (strcmp(buf, dev_name(&device->dev)) != 0)
468	continue;
469
470	rc = ida_alloc_range(&inuse, i, i, GFP_KERNEL);
471	if (rc < 0)
472	goto out;
473	}
474
475	rc = ida_alloc(&inuse, GFP_KERNEL);
476	out:
477	ida_destroy(&inuse);
478	return rc;
479	}
480
481	static int alloc_name_id(struct net net, const char name)
482	{
483	return __alloc_name_id(net, name, NULL);
484	}
485
486	static int alloc_name(struct ib_device ibdev, const char name)
487	{
488	int id;
489
490	id = alloc_name_id(rdma_dev_net(ibdev), name);
491	if (id < 0)
492	return id;
493
494	return dev_set_name(&ibdev->dev, name, id);
495	}
496
497	static void ib_device_release(struct device *device)
498	{
499	struct ib_device *dev = container_of(device, struct ib_device, dev);
500
501	free_netdevs(dev);
502	WARN_ON(refcount_read(&dev->refcount));
503	if (dev->hw_stats_data)
504	ib_device_release_hw_stats(dev->hw_stats_data);
505	if (dev->port_data) {
506	ib_cache_release_one(dev);
507	ib_security_release_port_pkey_list(dev);
508	rdma_counter_release(dev);
509	kfree_rcu(container_of(dev->port_data, struct ib_port_data_rcu,
510	pdata[0]),
511	rcu_head);
512	}
513
514	mutex_destroy(&dev->subdev_lock);
515	mutex_destroy(&dev->unregistration_lock);
516	mutex_destroy(&dev->compat_devs_mutex);
517
518	xa_destroy(&dev->compat_devs);
519	xa_destroy(&dev->client_data);
520	kfree_rcu(dev, rcu_head);
521	}
522
523	static int ib_device_uevent(const struct device *device,
524	struct kobj_uevent_env *env)
525	{
526	if (add_uevent_var(env, "NAME=%s", dev_name(device)))
527	return -ENOMEM;
528
529	/*
530	* It would be nice to pass the node GUID with the event...
531	*/
532
533	return 0;
534	}
535
536	static const struct ns_common net_namespace(const struct device d)
537	{
538	const struct ib_core_device *coredev =
539	container_of(d, struct ib_core_device, dev);
540	struct net *net = read_pnet(&coredev->rdma_net);
541
542	return net ? to_ns_common(net) : NULL;
543	}
544
545	static struct class ib_class = {
546	.name = "infiniband",
547	.dev_release = ib_device_release,
548	.dev_uevent = ib_device_uevent,
549	.ns_type = &net_ns_type_operations,
550	.namespace = net_namespace,
551	};
552
553	static void rdma_init_coredev(struct ib_core_device *coredev,
554	struct ib_device dev, struct net net)
555	{
556	bool is_full_dev = &dev->coredev == coredev;
557
558	/* This BUILD_BUG_ON is intended to catch layout change
559	* of union of ib_core_device and device.
560	* dev must be the first element as ib_core and providers
561	* driver uses it. Adding anything in ib_core_device before
562	* device will break this assumption.
563	*/
564	BUILD_BUG_ON(offsetof(struct ib_device, coredev.dev) !=
565	offsetof(struct ib_device, dev));
566
567	coredev->dev.class = &ib_class;
568	coredev->dev.groups = dev->groups;
569
570	/*
571	* Don't expose hw counters outside of the init namespace.
572	*/
573	if (!is_full_dev && dev->hw_stats_attr_index)
574	coredev->dev.groups[dev->hw_stats_attr_index] = NULL;
575
576	device_initialize(&coredev->dev);
577	coredev->owner = dev;
578	INIT_LIST_HEAD(&coredev->port_list);
579	write_pnet(&coredev->rdma_net, net);
580	}
581
582	/**
583	* _ib_alloc_device - allocate an IB device struct
584	* @size:size of structure to allocate
585	* @net: network namespace device should be located in, namespace
586	* must stay valid until ib_register_device() is completed.
587	*
588	* Low-level drivers should use ib_alloc_device() to allocate &struct
589	* ib_device. @size is the size of the structure to be allocated,
590	* including any private data used by the low-level driver.
591	* ib_dealloc_device() must be used to free structures allocated with
592	* ib_alloc_device().
593	*/
594	struct ib_device _ib_alloc_device(size_t size, struct net net)
595	{
596	struct ib_device *device;
597	unsigned int i;
598
599	if (WARN_ON(size < sizeof(struct ib_device)))
600	return NULL;
601
602	device = kzalloc(size, GFP_KERNEL);
603	if (!device)
604	return NULL;
605
606	if (rdma_restrack_init(device)) {
607	kfree(device);
608	return NULL;
609	}
610
611	/* ib_devices_shared_netns can't change while we have active namespaces
612	* in the system which means either init_net is passed or the user has
613	* no idea what they are doing.
614	*
615	* To avoid breaking backward compatibility, when in shared mode,
616	* force to init the device in the init_net.
617	*/
618	net = ib_devices_shared_netns ? &init_net : net;
619	rdma_init_coredev(&device->coredev, device, net);
620
621	INIT_LIST_HEAD(&device->event_handler_list);
622	spin_lock_init(&device->qp_open_list_lock);
623	init_rwsem(&device->event_handler_rwsem);
624	mutex_init(&device->unregistration_lock);
625	/*
626	* client_data needs to be alloc because we don't want our mark to be
627	* destroyed if the user stores NULL in the client data.
628	*/
629	xa_init_flags(&device->client_data, XA_FLAGS_ALLOC);
630	init_rwsem(&device->client_data_rwsem);
631	xa_init_flags(&device->compat_devs, XA_FLAGS_ALLOC);
632	mutex_init(&device->compat_devs_mutex);
633	init_completion(&device->unreg_completion);
634	INIT_WORK(&device->unregistration_work, ib_unregister_work);
635
636	spin_lock_init(&device->cq_pools_lock);
637	for (i = 0; i < ARRAY_SIZE(device->cq_pools); i++)
638	INIT_LIST_HEAD(&device->cq_pools[i]);
639
640	rwlock_init(&device->cache_lock);
641
642	device->uverbs_cmd_mask =
643	BIT_ULL(IB_USER_VERBS_CMD_ALLOC_MW) \|
644	BIT_ULL(IB_USER_VERBS_CMD_ALLOC_PD) \|
645	BIT_ULL(IB_USER_VERBS_CMD_ATTACH_MCAST) \|
646	BIT_ULL(IB_USER_VERBS_CMD_CLOSE_XRCD) \|
647	BIT_ULL(IB_USER_VERBS_CMD_CREATE_AH) \|
648	BIT_ULL(IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL) \|
649	BIT_ULL(IB_USER_VERBS_CMD_CREATE_CQ) \|
650	BIT_ULL(IB_USER_VERBS_CMD_CREATE_QP) \|
651	BIT_ULL(IB_USER_VERBS_CMD_CREATE_SRQ) \|
652	BIT_ULL(IB_USER_VERBS_CMD_CREATE_XSRQ) \|
653	BIT_ULL(IB_USER_VERBS_CMD_DEALLOC_MW) \|
654	BIT_ULL(IB_USER_VERBS_CMD_DEALLOC_PD) \|
655	BIT_ULL(IB_USER_VERBS_CMD_DEREG_MR) \|
656	BIT_ULL(IB_USER_VERBS_CMD_DESTROY_AH) \|
657	BIT_ULL(IB_USER_VERBS_CMD_DESTROY_CQ) \|
658	BIT_ULL(IB_USER_VERBS_CMD_DESTROY_QP) \|
659	BIT_ULL(IB_USER_VERBS_CMD_DESTROY_SRQ) \|
660	BIT_ULL(IB_USER_VERBS_CMD_DETACH_MCAST) \|
661	BIT_ULL(IB_USER_VERBS_CMD_GET_CONTEXT) \|
662	BIT_ULL(IB_USER_VERBS_CMD_MODIFY_QP) \|
663	BIT_ULL(IB_USER_VERBS_CMD_MODIFY_SRQ) \|
664	BIT_ULL(IB_USER_VERBS_CMD_OPEN_QP) \|
665	BIT_ULL(IB_USER_VERBS_CMD_OPEN_XRCD) \|
666	BIT_ULL(IB_USER_VERBS_CMD_QUERY_DEVICE) \|
667	BIT_ULL(IB_USER_VERBS_CMD_QUERY_PORT) \|
668	BIT_ULL(IB_USER_VERBS_CMD_QUERY_QP) \|
669	BIT_ULL(IB_USER_VERBS_CMD_QUERY_SRQ) \|
670	BIT_ULL(IB_USER_VERBS_CMD_REG_MR) \|
671	BIT_ULL(IB_USER_VERBS_CMD_REREG_MR) \|
672	BIT_ULL(IB_USER_VERBS_CMD_RESIZE_CQ);
673
674	mutex_init(&device->subdev_lock);
675	INIT_LIST_HEAD(&device->subdev_list_head);
676	INIT_LIST_HEAD(&device->subdev_list);
677
678	return device;
679	}
680	EXPORT_SYMBOL(_ib_alloc_device);
681
682	/**
683	* ib_dealloc_device - free an IB device struct
684	* @device:structure to free
685	*
686	* Free a structure allocated with ib_alloc_device().
687	*/
688	void ib_dealloc_device(struct ib_device *device)
689	{
690	if (device->ops.dealloc_driver)
691	device->ops.dealloc_driver(device);
692
693	/*
694	* ib_unregister_driver() requires all devices to remain in the xarray
695	* while their ops are callable. The last op we call is dealloc_driver
696	* above. This is needed to create a fence on op callbacks prior to
697	* allowing the driver module to unload.
698	*/
699	down_write(&devices_rwsem);
700	if (xa_load(&devices, device->index) == device)
701	xa_erase(&devices, device->index);
702	up_write(&devices_rwsem);
703
704	/* Expedite releasing netdev references */
705	free_netdevs(device);
706
707	WARN_ON(!xa_empty(&device->compat_devs));
708	WARN_ON(!xa_empty(&device->client_data));
709	WARN_ON(refcount_read(&device->refcount));
710	rdma_restrack_clean(device);
711	/* Balances with device_initialize */
712	put_device(&device->dev);
713	}
714	EXPORT_SYMBOL(ib_dealloc_device);
715
716	/*
717	* add_client_context() and remove_client_context() must be safe against
718	* parallel calls on the same device - registration/unregistration of both the
719	* device and client can be occurring in parallel.
720	*
721	* The routines need to be a fence, any caller must not return until the add
722	* or remove is fully completed.
723	*/
724	static int add_client_context(struct ib_device *device,
725	struct ib_client *client)
726	{
727	int ret = 0;
728
729	if (!device->kverbs_provider && !client->no_kverbs_req)
730	return 0;
731
732	down_write(&device->client_data_rwsem);
733	/*
734	* So long as the client is registered hold both the client and device
735	* unregistration locks.
736	*/
737	if (!refcount_inc_not_zero(&client->uses))
738	goto out_unlock;
739	refcount_inc(&device->refcount);
740
741	/*
742	* Another caller to add_client_context got here first and has already
743	* completely initialized context.
744	*/
745	if (xa_get_mark(&device->client_data, client->client_id,
746	CLIENT_DATA_REGISTERED))
747	goto out;
748
749	ret = xa_err(xa_store(&device->client_data, client->client_id, NULL,
750	GFP_KERNEL));
751	if (ret)
752	goto out;
753	downgrade_write(&device->client_data_rwsem);
754	if (client->add) {
755	if (client->add(device)) {
756	/*
757	* If a client fails to add then the error code is
758	* ignored, but we won't call any more ops on this
759	* client.
760	*/
761	xa_erase(&device->client_data, client->client_id);
762	up_read(&device->client_data_rwsem);
763	ib_device_put(device);
764	ib_client_put(client);
765	return 0;
766	}
767	}
768
769	/* Readers shall not see a client until add has been completed */
770	xa_set_mark(&device->client_data, client->client_id,
771	CLIENT_DATA_REGISTERED);
772	up_read(&device->client_data_rwsem);
773	return 0;
774
775	out:
776	ib_device_put(device);
777	ib_client_put(client);
778	out_unlock:
779	up_write(&device->client_data_rwsem);
780	return ret;
781	}
782
783	static void remove_client_context(struct ib_device *device,
784	unsigned int client_id)
785	{
786	struct ib_client *client;
787	void *client_data;
788
789	down_write(&device->client_data_rwsem);
790	if (!xa_get_mark(&device->client_data, client_id,
791	CLIENT_DATA_REGISTERED)) {
792	up_write(&device->client_data_rwsem);
793	return;
794	}
795	client_data = xa_load(&device->client_data, client_id);
796	xa_clear_mark(&device->client_data, client_id, CLIENT_DATA_REGISTERED);
797	client = xa_load(&clients, client_id);
798	up_write(&device->client_data_rwsem);
799
800	/*
801	* Notice we cannot be holding any exclusive locks when calling the
802	* remove callback as the remove callback can recurse back into any
803	* public functions in this module and thus try for any locks those
804	* functions take.
805	*
806	* For this reason clients and drivers should not call the
807	* unregistration functions will holdling any locks.
808	*/
809	if (client->remove)
810	client->remove(device, client_data);
811
812	xa_erase(&device->client_data, client_id);
813	ib_device_put(device);
814	ib_client_put(client);
815	}
816
817	static int alloc_port_data(struct ib_device *device)
818	{
819	struct ib_port_data_rcu *pdata_rcu;
820	u32 port;
821
822	if (device->port_data)
823	return 0;
824
825	/* This can only be called once the physical port range is defined */
826	if (WARN_ON(!device->phys_port_cnt))
827	return -EINVAL;
828
829	/* Reserve U32_MAX so the logic to go over all the ports is sane */
830	if (WARN_ON(device->phys_port_cnt == U32_MAX))
831	return -EINVAL;
832
833	/*
834	* device->port_data is indexed directly by the port number to make
835	* access to this data as efficient as possible.
836	*
837	* Therefore port_data is declared as a 1 based array with potential
838	* empty slots at the beginning.
839	*/
840	pdata_rcu = kzalloc_flex(*pdata_rcu, pdata,
841	size_add(rdma_end_port(device), 1));
842	if (!pdata_rcu)
843	return -ENOMEM;
844	/*
845	* The rcu_head is put in front of the port data array and the stored
846	* pointer is adjusted since we never need to see that member until
847	* kfree_rcu.
848	*/
849	device->port_data = pdata_rcu->pdata;
850
851	rdma_for_each_port (device, port) {
852	struct ib_port_data *pdata = &device->port_data[port];
853
854	pdata->ib_dev = device;
855	spin_lock_init(&pdata->pkey_list_lock);
856	INIT_LIST_HEAD(&pdata->pkey_list);
857	spin_lock_init(&pdata->netdev_lock);
858	INIT_HLIST_NODE(&pdata->ndev_hash_link);
859	}
860	return 0;
861	}
862
863	static int verify_immutable(const struct ib_device *dev, u32 port)
864	{
865	return WARN_ON(!rdma_cap_ib_mad(dev, port) &&
866	rdma_max_mad_size(dev, port) != 0);
867	}
868
869	static int setup_port_data(struct ib_device *device)
870	{
871	u32 port;
872	int ret;
873
874	ret = alloc_port_data(device);
875	if (ret)
876	return ret;
877
878	rdma_for_each_port (device, port) {
879	struct ib_port_data *pdata = &device->port_data[port];
880
881	ret = device->ops.get_port_immutable(device, port,
882	&pdata->immutable);
883	if (ret)
884	return ret;
885
886	if (verify_immutable(device, port))
887	return -EINVAL;
888	}
889	return 0;
890	}
891
892	/**
893	* ib_port_immutable_read() - Read rdma port's immutable data
894	* @dev: IB device
895	* @port: port number whose immutable data to read. It starts with index 1 and
896	* valid upto including rdma_end_port().
897	*/
898	const struct ib_port_immutable*
899	ib_port_immutable_read(struct ib_device *dev, unsigned int port)
900	{
901	WARN_ON(!rdma_is_port_valid(dev, port));
902	return &dev->port_data[port].immutable;
903	}
904	EXPORT_SYMBOL(ib_port_immutable_read);
905
906	void ib_get_device_fw_str(struct ib_device dev, char str)
907	{
908	if (dev->ops.get_dev_fw_str)
909	dev->ops.get_dev_fw_str(dev, str);
910	else
911	str[0] = '\0';
912	}
913	EXPORT_SYMBOL(ib_get_device_fw_str);
914
915	static void ib_policy_change_task(struct work_struct *work)
916	{
917	struct ib_device *dev;
918	unsigned long index;
919
920	down_read(&devices_rwsem);
921	xa_for_each_marked (&devices, index, dev, DEVICE_REGISTERED) {
922	unsigned int i;
923
924	rdma_for_each_port (dev, i) {
925	u64 sp;
926	ib_get_cached_subnet_prefix(dev, i, &sp);
927	ib_security_cache_change(dev, i, sp);
928	}
929	}
930	up_read(&devices_rwsem);
931	}
932
933	static int ib_security_change(struct notifier_block *nb, unsigned long event,
934	void *lsm_data)
935	{
936	if (event != LSM_POLICY_CHANGE)
937	return NOTIFY_DONE;
938
939	schedule_work(&ib_policy_change_work);
940	ib_mad_agent_security_change();
941
942	return NOTIFY_OK;
943	}
944
945	static void compatdev_release(struct device *dev)
946	{
947	struct ib_core_device *cdev =
948	container_of(dev, struct ib_core_device, dev);
949
950	kfree(cdev);
951	}
952
953	static int add_one_compat_dev(struct ib_device *device,
954	struct rdma_dev_net *rnet)
955	{
956	struct ib_core_device *cdev;
957	int ret;
958
959	lockdep_assert_held(&rdma_nets_rwsem);
960	if (!ib_devices_shared_netns)
961	return 0;
962
963	/*
964	* Create and add compat device in all namespaces other than where it
965	* is currently bound to.
966	*/
967	if (net_eq(read_pnet(&rnet->net),
968	read_pnet(&device->coredev.rdma_net)))
969	return 0;
970
971	/*
972	* The first of init_net() or ib_register_device() to take the
973	* compat_devs_mutex wins and gets to add the device. Others will wait
974	* for completion here.
975	*/
976	mutex_lock(&device->compat_devs_mutex);
977	cdev = xa_load(&device->compat_devs, rnet->id);
978	if (cdev) {
979	ret = 0;
980	goto done;
981	}
982	ret = xa_reserve(&device->compat_devs, rnet->id, GFP_KERNEL);
983	if (ret)
984	goto done;
985
986	cdev = kzalloc_obj(*cdev);
987	if (!cdev) {
988	ret = -ENOMEM;
989	goto cdev_err;
990	}
991
992	cdev->dev.parent = device->dev.parent;
993	rdma_init_coredev(cdev, device, read_pnet(&rnet->net));
994	cdev->dev.release = compatdev_release;
995	ret = dev_set_name(&cdev->dev, "%s", dev_name(&device->dev));
996	if (ret)
997	goto add_err;
998
999	ret = device_add(&cdev->dev);
1000	if (ret)
1001	goto add_err;
1002	ret = ib_setup_port_attrs(cdev);
1003	if (ret)
1004	goto port_err;
1005
1006	ret = xa_err(xa_store(&device->compat_devs, rnet->id,
1007	cdev, GFP_KERNEL));
1008	if (ret)
1009	goto insert_err;
1010
1011	mutex_unlock(&device->compat_devs_mutex);
1012	return 0;
1013
1014	insert_err:
1015	ib_free_port_attrs(cdev);
1016	port_err:
1017	device_del(&cdev->dev);
1018	add_err:
1019	put_device(&cdev->dev);
1020	cdev_err:
1021	xa_release(&device->compat_devs, rnet->id);
1022	done:
1023	mutex_unlock(&device->compat_devs_mutex);
1024	return ret;
1025	}
1026
1027	static void remove_one_compat_dev(struct ib_device *device, u32 id)
1028	{
1029	struct ib_core_device *cdev;
1030
1031	mutex_lock(&device->compat_devs_mutex);
1032	cdev = xa_erase(&device->compat_devs, id);
1033	mutex_unlock(&device->compat_devs_mutex);
1034	if (cdev) {
1035	ib_free_port_attrs(cdev);
1036	device_del(&cdev->dev);
1037	put_device(&cdev->dev);
1038	}
1039	}
1040
1041	static void remove_compat_devs(struct ib_device *device)
1042	{
1043	struct ib_core_device *cdev;
1044	unsigned long index;
1045
1046	xa_for_each (&device->compat_devs, index, cdev)
1047	remove_one_compat_dev(device, index);
1048	}
1049
1050	static int add_compat_devs(struct ib_device *device)
1051	{
1052	struct rdma_dev_net *rnet;
1053	unsigned long index;
1054	int ret = 0;
1055
1056	lockdep_assert_held(&devices_rwsem);
1057
1058	down_read(&rdma_nets_rwsem);
1059	xa_for_each (&rdma_nets, index, rnet) {
1060	ret = add_one_compat_dev(device, rnet);
1061	if (ret)
1062	break;
1063	}
1064	up_read(&rdma_nets_rwsem);
1065	return ret;
1066	}
1067
1068	static void remove_all_compat_devs(void)
1069	{
1070	struct ib_compat_device *cdev;
1071	struct ib_device *dev;
1072	unsigned long index;
1073
1074	down_read(&devices_rwsem);
1075	xa_for_each (&devices, index, dev) {
1076	unsigned long c_index = 0;
1077
1078	/* Hold nets_rwsem so that any other thread modifying this
1079	* system param can sync with this thread.
1080	*/
1081	down_read(&rdma_nets_rwsem);
1082	xa_for_each (&dev->compat_devs, c_index, cdev)
1083	remove_one_compat_dev(dev, c_index);
1084	up_read(&rdma_nets_rwsem);
1085	}
1086	up_read(&devices_rwsem);
1087	}
1088
1089	static int add_all_compat_devs(void)
1090	{
1091	struct rdma_dev_net *rnet;
1092	struct ib_device *dev;
1093	unsigned long index;
1094	int ret = 0;
1095
1096	down_read(&devices_rwsem);
1097	xa_for_each_marked (&devices, index, dev, DEVICE_REGISTERED) {
1098	unsigned long net_index = 0;
1099
1100	/* Hold nets_rwsem so that any other thread modifying this
1101	* system param can sync with this thread.
1102	*/
1103	down_read(&rdma_nets_rwsem);
1104	xa_for_each (&rdma_nets, net_index, rnet) {
1105	ret = add_one_compat_dev(dev, rnet);
1106	if (ret)
1107	break;
1108	}
1109	up_read(&rdma_nets_rwsem);
1110	}
1111	up_read(&devices_rwsem);
1112	if (ret)
1113	remove_all_compat_devs();
1114	return ret;
1115	}
1116
1117	int rdma_compatdev_set(u8 enable)
1118	{
1119	struct rdma_dev_net *rnet;
1120	unsigned long index;
1121	int ret = 0;
1122
1123	down_write(&rdma_nets_rwsem);
1124	if (ib_devices_shared_netns == enable) {
1125	up_write(&rdma_nets_rwsem);
1126	return 0;
1127	}
1128
1129	/* enable/disable of compat devices is not supported
1130	* when more than default init_net exists.
1131	*/
1132	xa_for_each (&rdma_nets, index, rnet) {
1133	ret++;
1134	break;
1135	}
1136	if (!ret)
1137	ib_devices_shared_netns = enable;
1138	up_write(&rdma_nets_rwsem);
1139	if (ret)
1140	return -EBUSY;
1141
1142	if (enable)
1143	ret = add_all_compat_devs();
1144	else
1145	remove_all_compat_devs();
1146	return ret;
1147	}
1148
1149	static void rdma_dev_exit_net(struct net *net)
1150	{
1151	struct rdma_dev_net *rnet = rdma_net_to_dev_net(net);
1152	struct ib_device *dev;
1153	unsigned long index;
1154	int ret;
1155
1156	down_write(&rdma_nets_rwsem);
1157	/*
1158	* Prevent the ID from being re-used and hide the id from xa_for_each.
1159	*/
1160	ret = xa_err(xa_store(&rdma_nets, rnet->id, NULL, GFP_KERNEL));
1161	WARN_ON(ret);
1162	up_write(&rdma_nets_rwsem);
1163
1164	down_read(&devices_rwsem);
1165	xa_for_each (&devices, index, dev) {
1166	get_device(&dev->dev);
1167	/*
1168	* Release the devices_rwsem so that pontentially blocking
1169	* device_del, doesn't hold the devices_rwsem for too long.
1170	*/
1171	up_read(&devices_rwsem);
1172
1173	remove_one_compat_dev(dev, rnet->id);
1174
1175	/*
1176	* If the real device is in the NS then move it back to init.
1177	* Provide a fallback pattern so a name conflict in init_net
1178	* cannot make the teardown move fail.
1179	*/
1180	if (net_eq(net, read_pnet(&dev->coredev.rdma_net))) {
1181	ret = rdma_dev_change_netns(dev, net, &init_net,
1182	NULL, "ibdev%d");
1183	if (ret && ret != -ENODEV)
1184	WARN(1,
1185	"Failed to move RDMA device %s to init_net on netns exit: %d\n",
1186	dev_name(&dev->dev), ret);
1187	}
1188
1189	put_device(&dev->dev);
1190	down_read(&devices_rwsem);
1191	}
1192	up_read(&devices_rwsem);
1193
1194	rdma_nl_net_exit(rnet);
1195	xa_erase(&rdma_nets, rnet->id);
1196	}
1197
1198	static __net_init int rdma_dev_init_net(struct net *net)
1199	{
1200	struct rdma_dev_net *rnet = rdma_net_to_dev_net(net);
1201	unsigned long index;
1202	struct ib_device *dev;
1203	int ret;
1204
1205	write_pnet(&rnet->net, net);
1206
1207	ret = rdma_nl_net_init(rnet);
1208	if (ret)
1209	return ret;
1210
1211	/* No need to create any compat devices in default init_net. */
1212	if (net_eq(net, &init_net))
1213	return 0;
1214
1215	ret = xa_alloc(&rdma_nets, &rnet->id, rnet, xa_limit_32b, GFP_KERNEL);
1216	if (ret) {
1217	rdma_nl_net_exit(rnet);
1218	return ret;
1219	}
1220
1221	down_read(&devices_rwsem);
1222	xa_for_each_marked (&devices, index, dev, DEVICE_REGISTERED) {
1223	/* Hold nets_rwsem so that netlink command cannot change
1224	* system configuration for device sharing mode.
1225	*/
1226	down_read(&rdma_nets_rwsem);
1227	ret = add_one_compat_dev(dev, rnet);
1228	up_read(&rdma_nets_rwsem);
1229	if (ret)
1230	break;
1231	}
1232	up_read(&devices_rwsem);
1233
1234	if (ret)
1235	rdma_dev_exit_net(net);
1236
1237	return ret;
1238	}
1239
1240	/*
1241	* Assign the unique string device name and the unique device index. The device
1242	* name is unique within the net namespace the device is assigned to. This is
1243	* undone by ib_dealloc_device.
1244	*/
1245	static int assign_name(struct ib_device device, const char name)
1246	{
1247	static u32 last_id;
1248	int ret;
1249
1250	down_write(&devices_rwsem);
1251	/* Assign a unique name to the device */
1252	if (strchr(name, '%'))
1253	ret = alloc_name(device, name);
1254	else
1255	ret = dev_set_name(&device->dev, name);
1256	if (ret)
1257	goto out;
1258
1259	if (__ib_device_get_by_name(dev_name(&device->dev),
1260	rdma_dev_net(device))) {
1261	ret = -ENFILE;
1262	goto out;
1263	}
1264	strscpy(device->name, dev_name(&device->dev), IB_DEVICE_NAME_MAX);
1265
1266	ret = xa_alloc_cyclic(&devices, &device->index, device, xa_limit_31b,
1267	&last_id, GFP_KERNEL);
1268	if (ret > 0)
1269	ret = 0;
1270
1271	out:
1272	up_write(&devices_rwsem);
1273	return ret;
1274	}
1275
1276	/*
1277	* setup_device() allocates memory and sets up data that requires calling the
1278	* device ops, this is the only reason these actions are not done during
1279	* ib_alloc_device. It is undone by ib_dealloc_device().
1280	*/
1281	static int setup_device(struct ib_device *device)
1282	{
1283	int ret;
1284
1285	ib_device_check_mandatory(device);
1286
1287	ret = setup_port_data(device);
1288	if (ret) {
1289	dev_warn(&device->dev, "Couldn't create per-port data\n");
1290	return ret;
1291	}
1292
1293	memset(&device->attrs, 0, sizeof(device->attrs));
1294	ret = device->ops.query_device(device, &device->attrs, NULL);
1295	if (ret) {
1296	dev_warn(&device->dev,
1297	"Couldn't query the device attributes\n");
1298	return ret;
1299	}
1300
1301	return 0;
1302	}
1303
1304	static void disable_device(struct ib_device *device)
1305	{
1306	u32 cid;
1307
1308	WARN_ON(!refcount_read(&device->refcount));
1309
1310	down_write(&devices_rwsem);
1311	xa_clear_mark(&devices, device->index, DEVICE_REGISTERED);
1312	up_write(&devices_rwsem);
1313
1314	/*
1315	* Remove clients in LIFO order, see assign_client_id. This could be
1316	* more efficient if xarray learns to reverse iterate. Since no new
1317	* clients can be added to this ib_device past this point we only need
1318	* the maximum possible client_id value here.
1319	*/
1320	down_read(&clients_rwsem);
1321	cid = highest_client_id;
1322	up_read(&clients_rwsem);
1323	while (cid) {
1324	cid--;
1325	remove_client_context(device, cid);
1326	}
1327
1328	ib_cq_pool_cleanup(device);
1329
1330	/* Pairs with refcount_set in enable_device */
1331	ib_device_put(device);
1332	wait_for_completion(&device->unreg_completion);
1333
1334	/*
1335	* compat devices must be removed after device refcount drops to zero.
1336	* Otherwise init_net() may add more compatdevs after removing compat
1337	* devices and before device is disabled.
1338	*/
1339	remove_compat_devs(device);
1340	}
1341
1342	/*
1343	* An enabled device is visible to all clients and to all the public facing
1344	* APIs that return a device pointer. This always returns with a new get, even
1345	* if it fails.
1346	*/
1347	static int enable_device_and_get(struct ib_device *device)
1348	{
1349	struct ib_client *client;
1350	unsigned long index;
1351	int ret = 0;
1352
1353	/*
1354	* One ref belongs to the xa and the other belongs to this
1355	* thread. This is needed to guard against parallel unregistration.
1356	*/
1357	refcount_set(&device->refcount, 2);
1358	down_write(&devices_rwsem);
1359	xa_set_mark(&devices, device->index, DEVICE_REGISTERED);
1360
1361	/*
1362	* By using downgrade_write() we ensure that no other thread can clear
1363	* DEVICE_REGISTERED while we are completing the client setup.
1364	*/
1365	downgrade_write(&devices_rwsem);
1366
1367	if (device->ops.enable_driver) {
1368	ret = device->ops.enable_driver(device);
1369	if (ret)
1370	goto out;
1371	}
1372
1373	down_read(&clients_rwsem);
1374	xa_for_each_marked (&clients, index, client, CLIENT_REGISTERED) {
1375	ret = add_client_context(device, client);
1376	if (ret)
1377	break;
1378	}
1379	up_read(&clients_rwsem);
1380	if (!ret)
1381	ret = add_compat_devs(device);
1382	out:
1383	up_read(&devices_rwsem);
1384	return ret;
1385	}
1386
1387	static void prevent_dealloc_device(struct ib_device *ib_dev)
1388	{
1389	}
1390
1391	static void ib_device_notify_register(struct ib_device *device)
1392	{
1393	struct net_device *netdev;
1394	u32 port;
1395	int ret;
1396
1397	down_read(&devices_rwsem);
1398
1399	/* Mark for userspace that device is ready */
1400	kobject_uevent(&device->dev.kobj, KOBJ_ADD);
1401
1402	ret = rdma_nl_notify_event(device, 0, RDMA_REGISTER_EVENT);
1403	if (ret)
1404	goto out;
1405
1406	rdma_for_each_port(device, port) {
1407	netdev = ib_device_get_netdev(device, port);
1408	if (!netdev)
1409	continue;
1410
1411	ret = rdma_nl_notify_event(device, port,
1412	RDMA_NETDEV_ATTACH_EVENT);
1413	dev_put(netdev);
1414	if (ret)
1415	goto out;
1416	}
1417
1418	out:
1419	up_read(&devices_rwsem);
1420	}
1421
1422	/**
1423	* ib_register_device - Register an IB device with IB core
1424	* @device: Device to register
1425	* @name: device name, unique within the device's net namespace. This may
1426	* include a '%' which will cause a unique index to be added to the
1427	* passed device name.
1428	* @dma_device: pointer to a DMA-capable device. If %NULL, then the IB
1429	* device will be used. In this case the caller should fully
1430	* setup the ibdev for DMA. This usually means using dma_virt_ops.
1431	*
1432	* Low-level drivers use ib_register_device() to register their
1433	* devices with the IB core. All registered clients will receive a
1434	* callback for each device that is added. @device must be allocated
1435	* with ib_alloc_device().
1436	*
1437	* If the driver uses ops.dealloc_driver and calls any ib_unregister_device()
1438	* asynchronously then the device pointer may become freed as soon as this
1439	* function returns.
1440	*/
1441	int ib_register_device(struct ib_device device, const char name,
1442	struct device *dma_device)
1443	{
1444	int ret;
1445
1446	ret = assign_name(device, name);
1447	if (ret)
1448	return ret;
1449
1450	/*
1451	* If the caller does not provide a DMA capable device then the IB core
1452	* will set up ib_sge and scatterlist structures that stash the kernel
1453	* virtual address into the address field.
1454	*/
1455	WARN_ON(dma_device && !dma_device->dma_parms);
1456	device->dma_device = dma_device;
1457	/*
1458	* In a CoCo guest every device is currently assumed to be untrusted
1459	* (T=0) and therefore subject to DMA bouncing. Once trusted (T=1)
1460	* device detection is wired up, narrow this check to exclude such
1461	* devices.
1462	*/
1463	if (dma_device && cc_platform_has(CC_ATTR_GUEST_MEM_ENCRYPT))
1464	device->cc_dma_bounce = 1;
1465
1466	ret = setup_device(device);
1467	if (ret)
1468	return ret;
1469
1470	ret = ib_cache_setup_one(device);
1471	if (ret) {
1472	dev_warn(&device->dev,
1473	"Couldn't set up InfiniBand P_Key/GID cache\n");
1474	return ret;
1475	}
1476
1477	device->groups[0] = &ib_dev_attr_group;
1478	device->groups[1] = device->ops.device_group;
1479	ret = ib_setup_device_attrs(device);
1480	if (ret)
1481	goto cache_cleanup;
1482
1483	ib_device_register_rdmacg(device);
1484
1485	rdma_counter_init(device);
1486
1487	/*
1488	* Ensure that ADD uevent is not fired because it
1489	* is too early amd device is not initialized yet.
1490	*/
1491	dev_set_uevent_suppress(&device->dev, true);
1492	ret = device_add(&device->dev);
1493	if (ret)
1494	goto cg_cleanup;
1495
1496	ret = ib_setup_port_attrs(&device->coredev);
1497	if (ret) {
1498	dev_warn(&device->dev,
1499	"Couldn't register device with driver model\n");
1500	goto dev_cleanup;
1501	}
1502
1503	ret = enable_device_and_get(device);
1504	if (ret) {
1505	void (dealloc_fn)(struct ib_device );
1506
1507	/*
1508	* If we hit this error flow then we don't want to
1509	* automatically dealloc the device since the caller is
1510	* expected to call ib_dealloc_device() after
1511	* ib_register_device() fails. This is tricky due to the
1512	* possibility for a parallel unregistration along with this
1513	* error flow. Since we have a refcount here we know any
1514	* parallel flow is stopped in disable_device and will see the
1515	* special dealloc_driver pointer, causing the responsibility to
1516	* ib_dealloc_device() to revert back to this thread.
1517	*/
1518	dealloc_fn = device->ops.dealloc_driver;
1519	device->ops.dealloc_driver = prevent_dealloc_device;
1520	ib_device_put(device);
1521	__ib_unregister_device(device);
1522	device->ops.dealloc_driver = dealloc_fn;
1523	dev_set_uevent_suppress(&device->dev, false);
1524	return ret;
1525	}
1526	dev_set_uevent_suppress(&device->dev, false);
1527
1528	ib_device_notify_register(device);
1529
1530	ib_device_put(device);
1531
1532	return 0;
1533
1534	dev_cleanup:
1535	device_del(&device->dev);
1536	cg_cleanup:
1537	dev_set_uevent_suppress(&device->dev, false);
1538	ib_device_unregister_rdmacg(device);
1539	cache_cleanup:
1540	ib_cache_cleanup_one(device);
1541	return ret;
1542	}
1543	EXPORT_SYMBOL(ib_register_device);
1544
1545	/* Callers must hold a get on the device. */
1546	static void __ib_unregister_device(struct ib_device *ib_dev)
1547	{
1548	struct ib_device sub, tmp;
1549
1550	mutex_lock(&ib_dev->subdev_lock);
1551	list_for_each_entry_safe_reverse(sub, tmp,
1552	&ib_dev->subdev_list_head,
1553	subdev_list) {
1554	list_del(&sub->subdev_list);
1555	ib_dev->ops.del_sub_dev(sub);
1556	ib_device_put(ib_dev);
1557	}
1558	mutex_unlock(&ib_dev->subdev_lock);
1559
1560	/*
1561	* We have a registration lock so that all the calls to unregister are
1562	* fully fenced, once any unregister returns the device is truly
1563	* unregistered even if multiple callers are unregistering it at the
1564	* same time. This also interacts with the registration flow and
1565	* provides sane semantics if register and unregister are racing.
1566	*/
1567	mutex_lock(&ib_dev->unregistration_lock);
1568	if (!refcount_read(&ib_dev->refcount))
1569	goto out;
1570
1571	disable_device(ib_dev);
1572	rdma_nl_notify_event(ib_dev, 0, RDMA_UNREGISTER_EVENT);
1573
1574	/* Expedite removing unregistered pointers from the hash table */
1575	free_netdevs(ib_dev);
1576
1577	ib_free_port_attrs(&ib_dev->coredev);
1578	device_del(&ib_dev->dev);
1579	ib_device_unregister_rdmacg(ib_dev);
1580	ib_cache_cleanup_one(ib_dev);
1581
1582	/*
1583	* Drivers using the new flow may not call ib_dealloc_device except
1584	* in error unwind prior to registration success.
1585	*/
1586	if (ib_dev->ops.dealloc_driver &&
1587	ib_dev->ops.dealloc_driver != prevent_dealloc_device) {
1588	WARN_ON(kref_read(&ib_dev->dev.kobj.kref) <= 1);
1589	ib_dealloc_device(ib_dev);
1590	}
1591	out:
1592	mutex_unlock(&ib_dev->unregistration_lock);
1593	}
1594
1595	/**
1596	* ib_unregister_device - Unregister an IB device
1597	* @ib_dev: The device to unregister
1598	*
1599	* Unregister an IB device. All clients will receive a remove callback.
1600	*
1601	* Callers should call this routine only once, and protect against races with
1602	* registration. Typically it should only be called as part of a remove
1603	* callback in an implementation of driver core's struct device_driver and
1604	* related.
1605	*
1606	* If ops.dealloc_driver is used then ib_dev will be freed upon return from
1607	* this function.
1608	*/
1609	void ib_unregister_device(struct ib_device *ib_dev)
1610	{
1611	get_device(&ib_dev->dev);
1612	__ib_unregister_device(ib_dev);
1613	put_device(&ib_dev->dev);
1614	}
1615	EXPORT_SYMBOL(ib_unregister_device);
1616
1617	/**
1618	* ib_unregister_device_and_put - Unregister a device while holding a 'get'
1619	* @ib_dev: The device to unregister
1620	*
1621	* This is the same as ib_unregister_device(), except it includes an internal
1622	* ib_device_put() that should match a 'get' obtained by the caller.
1623	*
1624	* It is safe to call this routine concurrently from multiple threads while
1625	* holding the 'get'. When the function returns the device is fully
1626	* unregistered.
1627	*
1628	* Drivers using this flow MUST use the driver_unregister callback to clean up
1629	* their resources associated with the device and dealloc it.
1630	*/
1631	void ib_unregister_device_and_put(struct ib_device *ib_dev)
1632	{
1633	WARN_ON(!ib_dev->ops.dealloc_driver);
1634	get_device(&ib_dev->dev);
1635	ib_device_put(ib_dev);
1636	__ib_unregister_device(ib_dev);
1637	put_device(&ib_dev->dev);
1638	}
1639	EXPORT_SYMBOL(ib_unregister_device_and_put);
1640
1641	/**
1642	* ib_unregister_driver - Unregister all IB devices for a driver
1643	* @driver_id: The driver to unregister
1644	*
1645	* This implements a fence for device unregistration. It only returns once all
1646	* devices associated with the driver_id have fully completed their
1647	* unregistration and returned from ib_unregister_device*().
1648	*
1649	* If device's are not yet unregistered it goes ahead and starts unregistering
1650	* them.
1651	*
1652	* This does not block creation of new devices with the given driver_id, that
1653	* is the responsibility of the caller.
1654	*/
1655	void ib_unregister_driver(enum rdma_driver_id driver_id)
1656	{
1657	struct ib_device *ib_dev;
1658	unsigned long index;
1659
1660	down_read(&devices_rwsem);
1661	xa_for_each (&devices, index, ib_dev) {
1662	if (ib_dev->ops.driver_id != driver_id)
1663	continue;
1664
1665	get_device(&ib_dev->dev);
1666	up_read(&devices_rwsem);
1667
1668	WARN_ON(!ib_dev->ops.dealloc_driver);
1669	__ib_unregister_device(ib_dev);
1670
1671	put_device(&ib_dev->dev);
1672	down_read(&devices_rwsem);
1673	}
1674	up_read(&devices_rwsem);
1675	}
1676	EXPORT_SYMBOL(ib_unregister_driver);
1677
1678	static void ib_unregister_work(struct work_struct *work)
1679	{
1680	struct ib_device *ib_dev =
1681	container_of(work, struct ib_device, unregistration_work);
1682
1683	__ib_unregister_device(ib_dev);
1684	put_device(&ib_dev->dev);
1685	}
1686
1687	/**
1688	* ib_unregister_device_queued - Unregister a device using a work queue
1689	* @ib_dev: The device to unregister
1690	*
1691	* This schedules an asynchronous unregistration using a WQ for the device. A
1692	* driver should use this to avoid holding locks while doing unregistration,
1693	* such as holding the RTNL lock.
1694	*
1695	* Drivers using this API must use ib_unregister_driver before module unload
1696	* to ensure that all scheduled unregistrations have completed.
1697	*/
1698	void ib_unregister_device_queued(struct ib_device *ib_dev)
1699	{
1700	WARN_ON(!refcount_read(&ib_dev->refcount));
1701	WARN_ON(!ib_dev->ops.dealloc_driver);
1702	get_device(&ib_dev->dev);
1703	if (!queue_work(ib_unreg_wq, &ib_dev->unregistration_work))
1704	put_device(&ib_dev->dev);
1705	}
1706	EXPORT_SYMBOL(ib_unregister_device_queued);
1707
1708	static bool rdma_dev_name_in_netns(struct ib_device skip, struct net net,
1709	const char *name)
1710	{
1711	struct ib_device *device;
1712	unsigned long index;
1713
1714	lockdep_assert_held_write(&devices_rwsem);
1715
1716	xa_for_each(&devices, index, device)
1717	if (device != skip &&
1718	rdma_dev_access_netns(device, net) &&
1719	!strcmp(name, dev_name(&device->dev)))
1720	return true;
1721
1722	return false;
1723	}
1724
1725	/*
1726	* Choose the name @device should use in net namespace @net. @requested_name
1727	* is used as a literal device name when set. Otherwise keep the current name
1728	* when it is free, or use a trusted '%d' @fallback_pattern for teardown. The
1729	* caller must hold the write side of devices_rwsem.
1730	*/
1731	static int rdma_dev_pick_netns_name(struct ib_device device, struct net net,
1732	const char *requested_name,
1733	const char *fallback_pattern,
1734	char *buf, size_t buf_len,
1735	const char **new_name)
1736	{
1737	int id;
1738
1739	lockdep_assert_held_write(&devices_rwsem);
1740
1741	if (requested_name) {
1742	if (!rdma_dev_name_in_netns(device, net, requested_name)) {
1743	*new_name = requested_name;
1744	return 0;
1745	}
1746
1747	return -EEXIST;
1748	}
1749
1750	if (!rdma_dev_name_in_netns(device, net, dev_name(&device->dev))) {
1751	*new_name = dev_name(&device->dev);
1752	return 0;
1753	}
1754
1755	if (!fallback_pattern)
1756	return -EEXIST;
1757
1758	snprintf(buf, buf_len, "ibdev%u", device->index);
1759	if (!rdma_dev_name_in_netns(device, net, buf)) {
1760	*new_name = buf;
1761	return 0;
1762	}
1763
1764	id = __alloc_name_id(net, fallback_pattern, device);
1765	if (id < 0)
1766	return id;
1767	snprintf(buf, buf_len, fallback_pattern, id);
1768	*new_name = buf;
1769	return 0;
1770	}
1771
1772	/*
1773	* The caller must pass in a device that has the kref held and the refcount
1774	* released. If the device is in cur_net and still registered then it is moved
1775	* into net.
1776	*
1777	* Naming rules are handled by rdma_dev_pick_netns_name().
1778	*/
1779	static int rdma_dev_change_netns(struct ib_device device, struct net cur_net,
1780	struct net net, const char requested_name,
1781	const char *fallback_pattern)
1782	{
1783	char buf[IB_DEVICE_NAME_MAX];
1784	const char *new_name;
1785	int ret2 = -EINVAL;
1786	int ret;
1787
1788	mutex_lock(&device->unregistration_lock);
1789
1790	/*
1791	* If a device not under ib_device_get() or if the unregistration_lock
1792	* is not held, the namespace can be changed, or it can be unregistered.
1793	* Check again under the lock.
1794	*/
1795	if (refcount_read(&device->refcount) == 0 \|\|
1796	!net_eq(cur_net, read_pnet(&device->coredev.rdma_net))) {
1797	ret = -ENODEV;
1798	goto out;
1799	}
1800
1801	if (!fallback_pattern) {
1802	/*
1803	* Reject a predictable name conflict before tearing anything
1804	* down, so a doomed user move does not disable a live device.
1805	*/
1806	down_write(&devices_rwsem);
1807	ret = rdma_dev_pick_netns_name(device, net, requested_name,
1808	fallback_pattern, buf,
1809	sizeof(buf), &new_name);
1810	up_write(&devices_rwsem);
1811	if (ret)
1812	goto out;
1813	}
1814
1815	kobject_uevent(&device->dev.kobj, KOBJ_REMOVE);
1816	disable_device(device);
1817
1818	/*
1819	* Recompute the destination name under the write side of devices_rwsem
1820	* now that the device is disabled, closing races with a concurrent
1821	* registration or rename, then publish the new namespace at the sysfs
1822	* level.
1823	*/
1824	down_write(&devices_rwsem);
1825	ret = rdma_dev_pick_netns_name(device, net, requested_name,
1826	fallback_pattern, buf, sizeof(buf),
1827	&new_name);
1828	if (ret) {
1829	if (fallback_pattern) {
1830	WARN(1,
1831	"%s: failed to pick device name during namespace teardown: %d\n",
1832	__func__, ret);
1833	write_pnet(&device->coredev.rdma_net, net);
1834	ret = 0;
1835	}
1836	goto rename_done;
1837	}
1838
1839	write_pnet(&device->coredev.rdma_net, net);
1840	ret = device_rename(&device->dev, new_name);
1841	if (ret) {
1842	if (fallback_pattern) {
1843	WARN(1,
1844	"%s: failed to rename device during namespace teardown: %d\n",
1845	__func__, ret);
1846	ret = 0;
1847	} else {
1848	dev_warn(&device->dev,
1849	"%s: Couldn't rename device after namespace change\n",
1850	__func__);
1851	/* Try and put things back and re-enable the device */
1852	write_pnet(&device->coredev.rdma_net, cur_net);
1853	}
1854	} else {
1855	strscpy(device->name, dev_name(&device->dev),
1856	IB_DEVICE_NAME_MAX);
1857	}
1858	rename_done:
1859	up_write(&devices_rwsem);
1860
1861	ret2 = enable_device_and_get(device);
1862	if (ret2) {
1863	/*
1864	* This shouldn't really happen, but if it does, let the user
1865	* retry at later point. So don't disable the device.
1866	*/
1867	dev_warn(&device->dev,
1868	"%s: Couldn't re-enable device after namespace change\n",
1869	__func__);
1870	}
1871	kobject_uevent(&device->dev.kobj, KOBJ_ADD);
1872
1873	ib_device_put(device);
1874	out:
1875	mutex_unlock(&device->unregistration_lock);
1876	if (ret)
1877	return ret;
1878	return ret2;
1879	}
1880
1881	int ib_device_set_netns_put(struct sk_buff *skb,
1882	struct ib_device dev, u32 ns_fd, const char name,
1883	struct netlink_ext_ack *extack)
1884	{
1885	struct net *net;
1886	int ret;
1887
1888	net = get_net_ns_by_fd(ns_fd);
1889	if (IS_ERR(net)) {
1890	NL_SET_ERR_MSG(extack, "Invalid target net namespace fd");
1891	ret = PTR_ERR(net);
1892	goto net_err;
1893	}
1894
1895	if (!netlink_ns_capable(skb, net->user_ns, CAP_NET_ADMIN)) {
1896	NL_SET_ERR_MSG(extack,
1897	"Missing CAP_NET_ADMIN in the target net namespace");
1898	ret = -EPERM;
1899	goto ns_err;
1900	}
1901
1902	/*
1903	* Moving a device to the namespace it already lives in is a no-op; a
1904	* supplied name still renames it in place.
1905	*/
1906	if (net_eq(net, read_pnet(&dev->coredev.rdma_net))) {
1907	ret = name ? ib_device_rename(dev, name) : 0;
1908
1909	if (ret == -EEXIST)
1910	NL_SET_ERR_MSG(extack,
1911	"Device name already exists in the target net namespace");
1912	else if (ret == -EINVAL && name)
1913	NL_SET_ERR_MSG(extack,
1914	"Unable to use requested device name in the target net namespace");
1915	goto ns_err;
1916	}
1917
1918	/*
1919	* All the ib_clients, including uverbs, are reset when the namespace is
1920	* changed and this cannot be blocked waiting for userspace to do
1921	* something, so disassociation is mandatory.
1922	*/
1923	if (ib_devices_shared_netns) {
1924	NL_SET_ERR_MSG(extack,
1925	"Cannot change net namespace of RDMA device in shared netns mode");
1926	ret = -EOPNOTSUPP;
1927	goto ns_err;
1928	}
1929
1930	if (!dev->ops.disassociate_ucontext) {
1931	NL_SET_ERR_MSG(extack,
1932	"Device does not support namespace changes (no disassociate support)");
1933	ret = -EOPNOTSUPP;
1934	goto ns_err;
1935	}
1936
1937	get_device(&dev->dev);
1938	ib_device_put(dev);
1939	ret = rdma_dev_change_netns(dev, current->nsproxy->net_ns, net, name,
1940	NULL);
1941	put_device(&dev->dev);
1942	if (ret == -EEXIST)
1943	NL_SET_ERR_MSG(extack,
1944	"Device name already exists in the target net namespace");
1945	else if (ret == -EINVAL && name)
1946	NL_SET_ERR_MSG(extack,
1947	"Unable to use requested device name in the target net namespace");
1948
1949	put_net(net);
1950	return ret;
1951
1952	ns_err:
1953	put_net(net);
1954	net_err:
1955	ib_device_put(dev);
1956	return ret;
1957	}
1958
1959	static struct pernet_operations rdma_dev_net_ops = {
1960	.init = rdma_dev_init_net,
1961	.exit = rdma_dev_exit_net,
1962	.id = &rdma_dev_net_id,
1963	.size = sizeof(struct rdma_dev_net),
1964	};
1965
1966	static int assign_client_id(struct ib_client *client)
1967	{
1968	int ret;
1969
1970	lockdep_assert_held(&clients_rwsem);
1971	/*
1972	* The add/remove callbacks must be called in FIFO/LIFO order. To
1973	* achieve this we assign client_ids so they are sorted in
1974	* registration order.
1975	*/
1976	client->client_id = highest_client_id;
1977	ret = xa_insert(&clients, client->client_id, client, GFP_KERNEL);
1978	if (ret)
1979	return ret;
1980
1981	highest_client_id++;
1982	xa_set_mark(&clients, client->client_id, CLIENT_REGISTERED);
1983	return 0;
1984	}
1985
1986	static void remove_client_id(struct ib_client *client)
1987	{
1988	down_write(&clients_rwsem);
1989	xa_erase(&clients, client->client_id);
1990	for (; highest_client_id; highest_client_id--)
1991	if (xa_load(&clients, highest_client_id - 1))
1992	break;
1993	up_write(&clients_rwsem);
1994	}
1995
1996	/**
1997	* ib_register_client - Register an IB client
1998	* @client:Client to register
1999	*
2000	* Upper level users of the IB drivers can use ib_register_client() to
2001	* register callbacks for IB device addition and removal. When an IB
2002	* device is added, each registered client's add method will be called
2003	* (in the order the clients were registered), and when a device is
2004	* removed, each client's remove method will be called (in the reverse
2005	* order that clients were registered). In addition, when
2006	* ib_register_client() is called, the client will receive an add
2007	* callback for all devices already registered.
2008	*/
2009	int ib_register_client(struct ib_client *client)
2010	{
2011	struct ib_device *device;
2012	unsigned long index;
2013	bool need_unreg = false;
2014	int ret;
2015
2016	refcount_set(&client->uses, 1);
2017	init_completion(&client->uses_zero);
2018
2019	/*
2020	* The devices_rwsem is held in write mode to ensure that a racing
2021	* ib_register_device() sees a consisent view of clients and devices.
2022	*/
2023	down_write(&devices_rwsem);
2024	down_write(&clients_rwsem);
2025	ret = assign_client_id(client);
2026	if (ret)
2027	goto out;
2028
2029	need_unreg = true;
2030	xa_for_each_marked (&devices, index, device, DEVICE_REGISTERED) {
2031	ret = add_client_context(device, client);
2032	if (ret)
2033	goto out;
2034	}
2035	ret = 0;
2036	out:
2037	up_write(&clients_rwsem);
2038	up_write(&devices_rwsem);
2039	if (need_unreg && ret)
2040	ib_unregister_client(client);
2041	return ret;
2042	}
2043	EXPORT_SYMBOL(ib_register_client);
2044
2045	/**
2046	* ib_unregister_client - Unregister an IB client
2047	* @client:Client to unregister
2048	*
2049	* Upper level users use ib_unregister_client() to remove their client
2050	* registration. When ib_unregister_client() is called, the client
2051	* will receive a remove callback for each IB device still registered.
2052	*
2053	* This is a full fence, once it returns no client callbacks will be called,
2054	* or are running in another thread.
2055	*/
2056	void ib_unregister_client(struct ib_client *client)
2057	{
2058	struct ib_device *device;
2059	unsigned long index;
2060
2061	down_write(&clients_rwsem);
2062	ib_client_put(client);
2063	xa_clear_mark(&clients, client->client_id, CLIENT_REGISTERED);
2064	up_write(&clients_rwsem);
2065
2066	/* We do not want to have locks while calling client->remove() */
2067	rcu_read_lock();
2068	xa_for_each (&devices, index, device) {
2069	if (!ib_device_try_get(device))
2070	continue;
2071	rcu_read_unlock();
2072
2073	remove_client_context(device, client->client_id);
2074
2075	ib_device_put(device);
2076	rcu_read_lock();
2077	}
2078	rcu_read_unlock();
2079
2080	/*
2081	* remove_client_context() is not a fence, it can return even though a
2082	* removal is ongoing. Wait until all removals are completed.
2083	*/
2084	wait_for_completion(&client->uses_zero);
2085	remove_client_id(client);
2086	}
2087	EXPORT_SYMBOL(ib_unregister_client);
2088
2089	static int __ib_get_global_client_nl_info(const char *client_name,
2090	struct ib_client_nl_info *res)
2091	{
2092	struct ib_client *client;
2093	unsigned long index;
2094	int ret = -ENOENT;
2095
2096	down_read(&clients_rwsem);
2097	xa_for_each_marked (&clients, index, client, CLIENT_REGISTERED) {
2098	if (strcmp(client->name, client_name) != 0)
2099	continue;
2100	if (!client->get_global_nl_info) {
2101	ret = -EOPNOTSUPP;
2102	break;
2103	}
2104	ret = client->get_global_nl_info(res);
2105	if (WARN_ON(ret == -ENOENT))
2106	ret = -EINVAL;
2107	if (!ret && res->cdev)
2108	get_device(res->cdev);
2109	break;
2110	}
2111	up_read(&clients_rwsem);
2112	return ret;
2113	}
2114
2115	static int __ib_get_client_nl_info(struct ib_device *ibdev,
2116	const char *client_name,
2117	struct ib_client_nl_info *res)
2118	{
2119	unsigned long index;
2120	void *client_data;
2121	int ret = -ENOENT;
2122
2123	down_read(&ibdev->client_data_rwsem);
2124	xan_for_each_marked (&ibdev->client_data, index, client_data,
2125	CLIENT_DATA_REGISTERED) {
2126	struct ib_client *client = xa_load(&clients, index);
2127
2128	if (!client \|\| strcmp(client->name, client_name) != 0)
2129	continue;
2130	if (!client->get_nl_info) {
2131	ret = -EOPNOTSUPP;
2132	break;
2133	}
2134	ret = client->get_nl_info(ibdev, client_data, res);
2135	if (WARN_ON(ret == -ENOENT))
2136	ret = -EINVAL;
2137
2138	/*
2139	* The cdev is guaranteed valid as long as we are inside the
2140	* client_data_rwsem as remove_one can't be called. Keep it
2141	* valid for the caller.
2142	*/
2143	if (!ret && res->cdev)
2144	get_device(res->cdev);
2145	break;
2146	}
2147	up_read(&ibdev->client_data_rwsem);
2148
2149	return ret;
2150	}
2151
2152	/**
2153	* ib_get_client_nl_info - Fetch the nl_info from a client
2154	* @ibdev: IB device
2155	* @client_name: Name of the client
2156	* @res: Result of the query
2157	*/
2158	int ib_get_client_nl_info(struct ib_device ibdev, const char client_name,
2159	struct ib_client_nl_info *res)
2160	{
2161	int ret;
2162
2163	if (ibdev)
2164	ret = __ib_get_client_nl_info(ibdev, client_name, res);
2165	else
2166	ret = __ib_get_global_client_nl_info(client_name, res);
2167	#ifdef CONFIG_MODULES
2168	if (ret == -ENOENT) {
2169	request_module("rdma-client-%s", client_name);
2170	if (ibdev)
2171	ret = __ib_get_client_nl_info(ibdev, client_name, res);
2172	else
2173	ret = __ib_get_global_client_nl_info(client_name, res);
2174	}
2175	#endif
2176	if (ret) {
2177	if (ret == -ENOENT)
2178	return -EOPNOTSUPP;
2179	return ret;
2180	}
2181
2182	if (WARN_ON(!res->cdev))
2183	return -EINVAL;
2184	return 0;
2185	}
2186
2187	/**
2188	* ib_set_client_data - Set IB client context
2189	* @device:Device to set context for
2190	* @client:Client to set context for
2191	* @data:Context to set
2192	*
2193	* ib_set_client_data() sets client context data that can be retrieved with
2194	* ib_get_client_data(). This can only be called while the client is
2195	* registered to the device, once the ib_client remove() callback returns this
2196	* cannot be called.
2197	*/
2198	void ib_set_client_data(struct ib_device device, struct ib_client client,
2199	void *data)
2200	{
2201	void *rc;
2202
2203	if (WARN_ON(IS_ERR(data)))
2204	data = NULL;
2205
2206	rc = xa_store(&device->client_data, client->client_id, data,
2207	GFP_KERNEL);
2208	WARN_ON(xa_is_err(rc));
2209	}
2210	EXPORT_SYMBOL(ib_set_client_data);
2211
2212	/**
2213	* ib_register_event_handler - Register an IB event handler
2214	* @event_handler:Handler to register
2215	*
2216	* ib_register_event_handler() registers an event handler that will be
2217	* called back when asynchronous IB events occur (as defined in
2218	* chapter 11 of the InfiniBand Architecture Specification). This
2219	* callback occurs in workqueue context.
2220	*/
2221	void ib_register_event_handler(struct ib_event_handler *event_handler)
2222	{
2223	down_write(&event_handler->device->event_handler_rwsem);
2224	list_add_tail(&event_handler->list,
2225	&event_handler->device->event_handler_list);
2226	up_write(&event_handler->device->event_handler_rwsem);
2227	}
2228	EXPORT_SYMBOL(ib_register_event_handler);
2229
2230	/**
2231	* ib_unregister_event_handler - Unregister an event handler
2232	* @event_handler:Handler to unregister
2233	*
2234	* Unregister an event handler registered with
2235	* ib_register_event_handler().
2236	*/
2237	void ib_unregister_event_handler(struct ib_event_handler *event_handler)
2238	{
2239	down_write(&event_handler->device->event_handler_rwsem);
2240	list_del(&event_handler->list);
2241	up_write(&event_handler->device->event_handler_rwsem);
2242	}
2243	EXPORT_SYMBOL(ib_unregister_event_handler);
2244
2245	void ib_dispatch_event_clients(struct ib_event *event)
2246	{
2247	struct ib_event_handler *handler;
2248
2249	down_read(&event->device->event_handler_rwsem);
2250
2251	list_for_each_entry(handler, &event->device->event_handler_list, list)
2252	handler->handler(handler, event);
2253
2254	up_read(&event->device->event_handler_rwsem);
2255	}
2256
2257	static int iw_query_port(struct ib_device *device,
2258	u32 port_num,
2259	struct ib_port_attr *port_attr)
2260	{
2261	struct in_device *inetdev;
2262	struct net_device *netdev;
2263
2264	memset(port_attr, 0, sizeof(*port_attr));
2265
2266	netdev = ib_device_get_netdev(device, port_num);
2267	if (!netdev)
2268	return -ENODEV;
2269
2270	port_attr->max_mtu = IB_MTU_4096;
2271	port_attr->active_mtu = ib_mtu_int_to_enum(netdev->mtu);
2272
2273	if (!netif_carrier_ok(netdev)) {
2274	port_attr->state = IB_PORT_DOWN;
2275	port_attr->phys_state = IB_PORT_PHYS_STATE_DISABLED;
2276	} else {
2277	rcu_read_lock();
2278	inetdev = __in_dev_get_rcu(netdev);
2279
2280	if (inetdev && inetdev->ifa_list) {
2281	port_attr->state = IB_PORT_ACTIVE;
2282	port_attr->phys_state = IB_PORT_PHYS_STATE_LINK_UP;
2283	} else {
2284	port_attr->state = IB_PORT_INIT;
2285	port_attr->phys_state =
2286	IB_PORT_PHYS_STATE_PORT_CONFIGURATION_TRAINING;
2287	}
2288
2289	rcu_read_unlock();
2290	}
2291
2292	dev_put(netdev);
2293	return device->ops.query_port(device, port_num, port_attr);
2294	}
2295
2296	static int __ib_query_port(struct ib_device *device,
2297	u32 port_num,
2298	struct ib_port_attr *port_attr)
2299	{
2300	int err;
2301
2302	memset(port_attr, 0, sizeof(*port_attr));
2303
2304	err = device->ops.query_port(device, port_num, port_attr);
2305	if (err \|\| port_attr->subnet_prefix)
2306	return err;
2307
2308	if (rdma_port_get_link_layer(device, port_num) !=
2309	IB_LINK_LAYER_INFINIBAND)
2310	return 0;
2311
2312	ib_get_cached_subnet_prefix(device, port_num,
2313	&port_attr->subnet_prefix);
2314	return 0;
2315	}
2316
2317	/**
2318	* ib_query_port - Query IB port attributes
2319	* @device:Device to query
2320	* @port_num:Port number to query
2321	* @port_attr:Port attributes
2322	*
2323	* ib_query_port() returns the attributes of a port through the
2324	* @port_attr pointer.
2325	*/
2326	int ib_query_port(struct ib_device *device,
2327	u32 port_num,
2328	struct ib_port_attr *port_attr)
2329	{
2330	if (!rdma_is_port_valid(device, port_num))
2331	return -EINVAL;
2332
2333	if (rdma_protocol_iwarp(device, port_num))
2334	return iw_query_port(device, port_num, port_attr);
2335	else
2336	return __ib_query_port(device, port_num, port_attr);
2337	}
2338	EXPORT_SYMBOL(ib_query_port);
2339
2340	static void add_ndev_hash(struct ib_port_data *pdata)
2341	{
2342	unsigned long flags;
2343
2344	might_sleep();
2345
2346	spin_lock_irqsave(&ndev_hash_lock, flags);
2347	if (hash_hashed(&pdata->ndev_hash_link)) {
2348	hash_del_rcu(&pdata->ndev_hash_link);
2349	spin_unlock_irqrestore(&ndev_hash_lock, flags);
2350	/*
2351	* We cannot do hash_add_rcu after a hash_del_rcu until the
2352	* grace period
2353	*/
2354	synchronize_rcu();
2355	spin_lock_irqsave(&ndev_hash_lock, flags);
2356	}
2357	if (pdata->netdev)
2358	hash_add_rcu(ndev_hash, &pdata->ndev_hash_link,
2359	(uintptr_t)pdata->netdev);
2360	spin_unlock_irqrestore(&ndev_hash_lock, flags);
2361	}
2362
2363	/**
2364	* ib_device_set_netdev - Associate the ib_dev with an underlying net_device
2365	* @ib_dev: Device to modify
2366	* @ndev: net_device to affiliate, may be NULL
2367	* @port: IB port the net_device is connected to
2368	*
2369	* Drivers should use this to link the ib_device to a netdev so the netdev
2370	* shows up in interfaces like ib_enum_roce_netdev. Only one netdev may be
2371	* affiliated with any port.
2372	*
2373	* The caller must ensure that the given ndev is not unregistered or
2374	* unregistering, and that either the ib_device is unregistered or
2375	* ib_device_set_netdev() is called with NULL when the ndev sends a
2376	* NETDEV_UNREGISTER event.
2377	*/
2378	int ib_device_set_netdev(struct ib_device ib_dev, struct net_device ndev,
2379	u32 port)
2380	{
2381	enum rdma_nl_notify_event_type etype;
2382	struct net_device *old_ndev;
2383	struct ib_port_data *pdata;
2384	unsigned long flags;
2385	int ret;
2386
2387	if (!rdma_is_port_valid(ib_dev, port))
2388	return -EINVAL;
2389
2390	/*
2391	* Drivers wish to call this before ib_register_driver, so we have to
2392	* setup the port data early.
2393	*/
2394	ret = alloc_port_data(ib_dev);
2395	if (ret)
2396	return ret;
2397
2398	pdata = &ib_dev->port_data[port];
2399	spin_lock_irqsave(&pdata->netdev_lock, flags);
2400	old_ndev = rcu_dereference_protected(
2401	pdata->netdev, lockdep_is_held(&pdata->netdev_lock));
2402	if (old_ndev == ndev) {
2403	spin_unlock_irqrestore(&pdata->netdev_lock, flags);
2404	return 0;
2405	}
2406
2407	rcu_assign_pointer(pdata->netdev, ndev);
2408	netdev_put(old_ndev, &pdata->netdev_tracker);
2409	netdev_hold(ndev, &pdata->netdev_tracker, GFP_ATOMIC);
2410	spin_unlock_irqrestore(&pdata->netdev_lock, flags);
2411
2412	add_ndev_hash(pdata);
2413
2414	/* Make sure that the device is registered before we send events */
2415	if (xa_load(&devices, ib_dev->index) != ib_dev)
2416	return 0;
2417
2418	etype = ndev ? RDMA_NETDEV_ATTACH_EVENT : RDMA_NETDEV_DETACH_EVENT;
2419	rdma_nl_notify_event(ib_dev, port, etype);
2420
2421	return 0;
2422	}
2423	EXPORT_SYMBOL(ib_device_set_netdev);
2424
2425	static void free_netdevs(struct ib_device *ib_dev)
2426	{
2427	unsigned long flags;
2428	u32 port;
2429
2430	if (!ib_dev->port_data)
2431	return;
2432
2433	rdma_for_each_port (ib_dev, port) {
2434	struct ib_port_data *pdata = &ib_dev->port_data[port];
2435	struct net_device *ndev;
2436
2437	spin_lock_irqsave(&pdata->netdev_lock, flags);
2438	ndev = rcu_dereference_protected(
2439	pdata->netdev, lockdep_is_held(&pdata->netdev_lock));
2440	if (ndev) {
2441	spin_lock(&ndev_hash_lock);
2442	hash_del_rcu(&pdata->ndev_hash_link);
2443	spin_unlock(&ndev_hash_lock);
2444
2445	/*
2446	* If this is the last dev_put there is still a
2447	* synchronize_rcu before the netdev is kfreed, so we
2448	* can continue to rely on unlocked pointer
2449	* comparisons after the put
2450	*/
2451	rcu_assign_pointer(pdata->netdev, NULL);
2452	netdev_put(ndev, &pdata->netdev_tracker);
2453	}
2454	spin_unlock_irqrestore(&pdata->netdev_lock, flags);
2455	}
2456	}
2457
2458	struct net_device ib_device_get_netdev(struct ib_device ib_dev,
2459	u32 port)
2460	{
2461	struct ib_port_data *pdata;
2462	struct net_device *res;
2463
2464	if (!rdma_is_port_valid(ib_dev, port))
2465	return NULL;
2466
2467	if (!ib_dev->port_data)
2468	return NULL;
2469
2470	pdata = &ib_dev->port_data[port];
2471
2472	/*
2473	* New drivers should use ib_device_set_netdev() not the legacy
2474	* get_netdev().
2475	*/
2476	if (ib_dev->ops.get_netdev)
2477	res = ib_dev->ops.get_netdev(ib_dev, port);
2478	else {
2479	spin_lock(&pdata->netdev_lock);
2480	res = rcu_dereference_protected(
2481	pdata->netdev, lockdep_is_held(&pdata->netdev_lock));
2482	dev_hold(res);
2483	spin_unlock(&pdata->netdev_lock);
2484	}
2485
2486	return res;
2487	}
2488	EXPORT_SYMBOL(ib_device_get_netdev);
2489
2490	/**
2491	* ib_query_netdev_port - Query the port number of a net_device
2492	* associated with an ibdev
2493	* @ibdev: IB device
2494	* @ndev: Network device
2495	* @port: IB port the net_device is connected to
2496	*/
2497	int ib_query_netdev_port(struct ib_device ibdev, struct net_device ndev,
2498	u32 *port)
2499	{
2500	struct net_device *ib_ndev;
2501	u32 port_num;
2502
2503	rdma_for_each_port(ibdev, port_num) {
2504	ib_ndev = ib_device_get_netdev(ibdev, port_num);
2505	if (ndev == ib_ndev) {
2506	*port = port_num;
2507	dev_put(ib_ndev);
2508	return 0;
2509	}
2510	dev_put(ib_ndev);
2511	}
2512
2513	return -ENOENT;
2514	}
2515	EXPORT_SYMBOL(ib_query_netdev_port);
2516
2517	/**
2518	* ib_device_get_by_netdev - Find an IB device associated with a netdev
2519	* @ndev: netdev to locate
2520	* @driver_id: The driver ID that must match (RDMA_DRIVER_UNKNOWN matches all)
2521	*
2522	* Find and hold an ib_device that is associated with a netdev via
2523	* ib_device_set_netdev(). The caller must call ib_device_put() on the
2524	* returned pointer.
2525	*/
2526	struct ib_device ib_device_get_by_netdev(struct net_device ndev,
2527	enum rdma_driver_id driver_id)
2528	{
2529	struct ib_device *res = NULL;
2530	struct ib_port_data *cur;
2531
2532	rcu_read_lock();
2533	hash_for_each_possible_rcu (ndev_hash, cur, ndev_hash_link,
2534	(uintptr_t)ndev) {
2535	if (rcu_access_pointer(cur->netdev) == ndev &&
2536	(driver_id == RDMA_DRIVER_UNKNOWN \|\|
2537	cur->ib_dev->ops.driver_id == driver_id) &&
2538	ib_device_try_get(cur->ib_dev)) {
2539	res = cur->ib_dev;
2540	break;
2541	}
2542	}
2543	rcu_read_unlock();
2544
2545	return res;
2546	}
2547	EXPORT_SYMBOL(ib_device_get_by_netdev);
2548
2549	/**
2550	* ib_enum_roce_netdev - enumerate all RoCE ports
2551	* @ib_dev : IB device we want to query
2552	* @filter: Should we call the callback?
2553	* @filter_cookie: Cookie passed to filter
2554	* @cb: Callback to call for each found RoCE ports
2555	* @cookie: Cookie passed back to the callback
2556	*
2557	* Enumerates all of the physical RoCE ports of ib_dev
2558	* which are related to netdevice and calls callback() on each
2559	* device for which filter() function returns non zero.
2560	*/
2561	void ib_enum_roce_netdev(struct ib_device *ib_dev,
2562	roce_netdev_filter filter,
2563	void *filter_cookie,
2564	roce_netdev_callback cb,
2565	void *cookie)
2566	{
2567	u32 port;
2568
2569	rdma_for_each_port (ib_dev, port)
2570	if (rdma_protocol_roce(ib_dev, port)) {
2571	struct net_device *idev =
2572	ib_device_get_netdev(ib_dev, port);
2573
2574	if (filter(ib_dev, port, idev, filter_cookie))
2575	cb(ib_dev, port, idev, cookie);
2576	dev_put(idev);
2577	}
2578	}
2579
2580	/**
2581	* ib_enum_all_roce_netdevs - enumerate all RoCE devices
2582	* @filter: Should we call the callback?
2583	* @filter_cookie: Cookie passed to filter
2584	* @cb: Callback to call for each found RoCE ports
2585	* @cookie: Cookie passed back to the callback
2586	*
2587	* Enumerates all RoCE devices' physical ports which are related
2588	* to netdevices and calls callback() on each device for which
2589	* filter() function returns non zero.
2590	*/
2591	void ib_enum_all_roce_netdevs(roce_netdev_filter filter,
2592	void *filter_cookie,
2593	roce_netdev_callback cb,
2594	void *cookie)
2595	{
2596	struct ib_device *dev;
2597	unsigned long index;
2598
2599	down_read(&devices_rwsem);
2600	xa_for_each_marked(&devices, index, dev, DEVICE_GID_UPDATES)
2601	ib_enum_roce_netdev(dev, filter, filter_cookie, cb, cookie);
2602	up_read(&devices_rwsem);
2603	}
2604
2605	/**
2606	* ib_device_enable_gid_updates - Mark device as ready for GID cache updates
2607	* @device: Device to mark
2608	*
2609	* Called after GID table is allocated and initialized. After this mark is set,
2610	* netdevice event handlers can update the device's GID cache. This allows
2611	* events that arrive during device registration to be processed, avoiding
2612	* stale GID entries when netdev properties change during the device
2613	* registration process.
2614	*/
2615	void ib_device_enable_gid_updates(struct ib_device *device)
2616	{
2617	down_write(&devices_rwsem);
2618	xa_set_mark(&devices, device->index, DEVICE_GID_UPDATES);
2619	up_write(&devices_rwsem);
2620	}
2621
2622	/**
2623	* ib_device_disable_gid_updates - Clear the GID updates mark
2624	* @device: Device to unmark
2625	*
2626	* Called before GID table cleanup to prevent event handlers from accessing
2627	* the device while it's being torn down.
2628	*/
2629	void ib_device_disable_gid_updates(struct ib_device *device)
2630	{
2631	down_write(&devices_rwsem);
2632	xa_clear_mark(&devices, device->index, DEVICE_GID_UPDATES);
2633	up_write(&devices_rwsem);
2634	}
2635
2636	/*
2637	* ib_enum_all_devs - enumerate all ib_devices
2638	* @cb: Callback to call for each found ib_device
2639	*
2640	* Enumerates all ib_devices and calls callback() on each device.
2641	*/
2642	int ib_enum_all_devs(nldev_callback nldev_cb, struct sk_buff *skb,
2643	struct netlink_callback *cb)
2644	{
2645	unsigned long index;
2646	struct ib_device *dev;
2647	unsigned int idx = 0;
2648	int ret = 0;
2649
2650	down_read(&devices_rwsem);
2651	xa_for_each_marked (&devices, index, dev, DEVICE_REGISTERED) {
2652	if (!rdma_dev_access_netns(dev, sock_net(skb->sk)))
2653	continue;
2654
2655	ret = nldev_cb(dev, skb, cb, idx);
2656	if (ret)
2657	break;
2658	idx++;
2659	}
2660	up_read(&devices_rwsem);
2661	return ret;
2662	}
2663
2664	/**
2665	* ib_query_pkey - Get P_Key table entry
2666	* @device:Device to query
2667	* @port_num:Port number to query
2668	* @index:P_Key table index to query
2669	* @pkey:Returned P_Key
2670	*
2671	* ib_query_pkey() fetches the specified P_Key table entry.
2672	*/
2673	int ib_query_pkey(struct ib_device *device,
2674	u32 port_num, u16 index, u16 *pkey)
2675	{
2676	if (!rdma_is_port_valid(device, port_num))
2677	return -EINVAL;
2678
2679	if (!device->ops.query_pkey)
2680	return -EOPNOTSUPP;
2681
2682	return device->ops.query_pkey(device, port_num, index, pkey);
2683	}
2684	EXPORT_SYMBOL(ib_query_pkey);
2685
2686	/**
2687	* ib_modify_device - Change IB device attributes
2688	* @device:Device to modify
2689	* @device_modify_mask:Mask of attributes to change
2690	* @device_modify:New attribute values
2691	*
2692	* ib_modify_device() changes a device's attributes as specified by
2693	* the @device_modify_mask and @device_modify structure.
2694	*/
2695	int ib_modify_device(struct ib_device *device,
2696	int device_modify_mask,
2697	struct ib_device_modify *device_modify)
2698	{
2699	if (!device->ops.modify_device)
2700	return -EOPNOTSUPP;
2701
2702	return device->ops.modify_device(device, device_modify_mask,
2703	device_modify);
2704	}
2705	EXPORT_SYMBOL(ib_modify_device);
2706
2707	/**
2708	* ib_modify_port - Modifies the attributes for the specified port.
2709	* @device: The device to modify.
2710	* @port_num: The number of the port to modify.
2711	* @port_modify_mask: Mask used to specify which attributes of the port
2712	* to change.
2713	* @port_modify: New attribute values for the port.
2714	*
2715	* ib_modify_port() changes a port's attributes as specified by the
2716	* @port_modify_mask and @port_modify structure.
2717	*/
2718	int ib_modify_port(struct ib_device *device,
2719	u32 port_num, int port_modify_mask,
2720	struct ib_port_modify *port_modify)
2721	{
2722	int rc;
2723
2724	if (!rdma_is_port_valid(device, port_num))
2725	return -EINVAL;
2726
2727	if (device->ops.modify_port)
2728	rc = device->ops.modify_port(device, port_num,
2729	port_modify_mask,
2730	port_modify);
2731	else if (rdma_protocol_roce(device, port_num) &&
2732	((port_modify->set_port_cap_mask & ~IB_PORT_CM_SUP) == 0 \|\|
2733	(port_modify->clr_port_cap_mask & ~IB_PORT_CM_SUP) == 0))
2734	rc = 0;
2735	else
2736	rc = -EOPNOTSUPP;
2737	return rc;
2738	}
2739	EXPORT_SYMBOL(ib_modify_port);
2740
2741	/**
2742	* ib_find_gid - Returns the port number and GID table index where
2743	* a specified GID value occurs. Its searches only for IB link layer.
2744	* @device: The device to query.
2745	* @gid: The GID value to search for.
2746	* @port_num: The port number of the device where the GID value was found.
2747	* @index: The index into the GID table where the GID was found. This
2748	* parameter may be NULL.
2749	*/
2750	int ib_find_gid(struct ib_device device, union ib_gid gid,
2751	u32 port_num, u16 index)
2752	{
2753	union ib_gid tmp_gid;
2754	u32 port;
2755	int ret, i;
2756
2757	rdma_for_each_port (device, port) {
2758	if (!rdma_protocol_ib(device, port))
2759	continue;
2760
2761	for (i = 0; i < device->port_data[port].immutable.gid_tbl_len;
2762	++i) {
2763	ret = rdma_query_gid(device, port, i, &tmp_gid);
2764	if (ret)
2765	continue;
2766
2767	if (!memcmp(&tmp_gid, gid, sizeof *gid)) {
2768	*port_num = port;
2769	if (index)
2770	*index = i;
2771	return 0;
2772	}
2773	}
2774	}
2775
2776	return -ENOENT;
2777	}
2778	EXPORT_SYMBOL(ib_find_gid);
2779
2780	/**
2781	* ib_find_pkey - Returns the PKey table index where a specified
2782	* PKey value occurs.
2783	* @device: The device to query.
2784	* @port_num: The port number of the device to search for the PKey.
2785	* @pkey: The PKey value to search for.
2786	* @index: The index into the PKey table where the PKey was found.
2787	*/
2788	int ib_find_pkey(struct ib_device *device,
2789	u32 port_num, u16 pkey, u16 *index)
2790	{
2791	int ret, i;
2792	u16 tmp_pkey;
2793	int partial_ix = -1;
2794
2795	for (i = 0; i < device->port_data[port_num].immutable.pkey_tbl_len;
2796	++i) {
2797	ret = ib_query_pkey(device, port_num, i, &tmp_pkey);
2798	if (ret)
2799	return ret;
2800	if ((pkey & 0x7fff) == (tmp_pkey & 0x7fff)) {
2801	/* if there is full-member pkey take it.*/
2802	if (tmp_pkey & 0x8000) {
2803	*index = i;
2804	return 0;
2805	}
2806	if (partial_ix < 0)
2807	partial_ix = i;
2808	}
2809	}
2810
2811	/no full-member, if exists take the limited/
2812	if (partial_ix >= 0) {
2813	*index = partial_ix;
2814	return 0;
2815	}
2816	return -ENOENT;
2817	}
2818	EXPORT_SYMBOL(ib_find_pkey);
2819
2820	/**
2821	* ib_get_net_dev_by_params() - Return the appropriate net_dev
2822	* for a received CM request
2823	* @dev: An RDMA device on which the request has been received.
2824	* @port: Port number on the RDMA device.
2825	* @pkey: The Pkey the request came on.
2826	* @gid: A GID that the net_dev uses to communicate.
2827	* @addr: Contains the IP address that the request specified as its
2828	* destination.
2829	*
2830	*/
2831	struct net_device ib_get_net_dev_by_params(struct ib_device dev,
2832	u32 port,
2833	u16 pkey,
2834	const union ib_gid *gid,
2835	const struct sockaddr *addr)
2836	{
2837	struct net_device *net_dev = NULL;
2838	unsigned long index;
2839	void *client_data;
2840
2841	if (!rdma_protocol_ib(dev, port))
2842	return NULL;
2843
2844	/*
2845	* Holding the read side guarantees that the client will not become
2846	* unregistered while we are calling get_net_dev_by_params()
2847	*/
2848	down_read(&dev->client_data_rwsem);
2849	xan_for_each_marked (&dev->client_data, index, client_data,
2850	CLIENT_DATA_REGISTERED) {
2851	struct ib_client *client = xa_load(&clients, index);
2852
2853	if (!client \|\| !client->get_net_dev_by_params)
2854	continue;
2855
2856	net_dev = client->get_net_dev_by_params(dev, port, pkey, gid,
2857	addr, client_data);
2858	if (net_dev)
2859	break;
2860	}
2861	up_read(&dev->client_data_rwsem);
2862
2863	return net_dev;
2864	}
2865	EXPORT_SYMBOL(ib_get_net_dev_by_params);
2866
2867	void ib_set_device_ops(struct ib_device dev, const struct ib_device_ops ops)
2868	{
2869	struct ib_device_ops *dev_ops = &dev->ops;
2870	#define SET_DEVICE_OP(ptr, name) \
2871	do { \
2872	if (ops->name) \
2873	if (!((ptr)->name)) \
2874	(ptr)->name = ops->name; \
2875	} while (0)
2876
2877	#define SET_OBJ_SIZE(ptr, name) SET_DEVICE_OP(ptr, size_##name)
2878
2879	if (ops->driver_id != RDMA_DRIVER_UNKNOWN) {
2880	WARN_ON(dev_ops->driver_id != RDMA_DRIVER_UNKNOWN &&
2881	dev_ops->driver_id != ops->driver_id);
2882	dev_ops->driver_id = ops->driver_id;
2883	}
2884	if (ops->owner) {
2885	WARN_ON(dev_ops->owner && dev_ops->owner != ops->owner);
2886	dev_ops->owner = ops->owner;
2887	}
2888	if (ops->uverbs_abi_ver)
2889	dev_ops->uverbs_abi_ver = ops->uverbs_abi_ver;
2890
2891	dev_ops->uverbs_no_driver_id_binding \|=
2892	ops->uverbs_no_driver_id_binding;
2893	dev_ops->uverbs_robust_udata \|= ops->uverbs_robust_udata;
2894
2895	SET_DEVICE_OP(dev_ops, add_gid);
2896	SET_DEVICE_OP(dev_ops, add_sub_dev);
2897	SET_DEVICE_OP(dev_ops, advise_mr);
2898	SET_DEVICE_OP(dev_ops, alloc_dm);
2899	SET_DEVICE_OP(dev_ops, alloc_dmah);
2900	SET_DEVICE_OP(dev_ops, alloc_hw_device_stats);
2901	SET_DEVICE_OP(dev_ops, alloc_hw_port_stats);
2902	SET_DEVICE_OP(dev_ops, alloc_mr);
2903	SET_DEVICE_OP(dev_ops, alloc_mr_integrity);
2904	SET_DEVICE_OP(dev_ops, alloc_mw);
2905	SET_DEVICE_OP(dev_ops, alloc_pd);
2906	SET_DEVICE_OP(dev_ops, alloc_rdma_netdev);
2907	SET_DEVICE_OP(dev_ops, alloc_ucontext);
2908	SET_DEVICE_OP(dev_ops, alloc_xrcd);
2909	SET_DEVICE_OP(dev_ops, attach_mcast);
2910	SET_DEVICE_OP(dev_ops, check_mr_status);
2911	SET_DEVICE_OP(dev_ops, counter_alloc_stats);
2912	SET_DEVICE_OP(dev_ops, counter_bind_qp);
2913	SET_DEVICE_OP(dev_ops, counter_dealloc);
2914	SET_DEVICE_OP(dev_ops, counter_init);
2915	SET_DEVICE_OP(dev_ops, counter_unbind_qp);
2916	SET_DEVICE_OP(dev_ops, counter_update_stats);
2917	SET_DEVICE_OP(dev_ops, create_ah);
2918	SET_DEVICE_OP(dev_ops, create_counters);
2919	SET_DEVICE_OP(dev_ops, create_cq);
2920	SET_DEVICE_OP(dev_ops, create_comp_cntr);
2921	SET_DEVICE_OP(dev_ops, create_user_cq);
2922	SET_DEVICE_OP(dev_ops, create_flow);
2923	SET_DEVICE_OP(dev_ops, create_qp);
2924	SET_DEVICE_OP(dev_ops, create_rwq_ind_table);
2925	SET_DEVICE_OP(dev_ops, create_srq);
2926	SET_DEVICE_OP(dev_ops, create_user_ah);
2927	SET_DEVICE_OP(dev_ops, create_wq);
2928	SET_DEVICE_OP(dev_ops, dealloc_dm);
2929	SET_DEVICE_OP(dev_ops, dealloc_dmah);
2930	SET_DEVICE_OP(dev_ops, dealloc_driver);
2931	SET_DEVICE_OP(dev_ops, dealloc_mw);
2932	SET_DEVICE_OP(dev_ops, dealloc_pd);
2933	SET_DEVICE_OP(dev_ops, dealloc_ucontext);
2934	SET_DEVICE_OP(dev_ops, dealloc_xrcd);
2935	SET_DEVICE_OP(dev_ops, del_gid);
2936	SET_DEVICE_OP(dev_ops, del_sub_dev);
2937	SET_DEVICE_OP(dev_ops, dereg_mr);
2938	SET_DEVICE_OP(dev_ops, destroy_ah);
2939	SET_DEVICE_OP(dev_ops, destroy_counters);
2940	SET_DEVICE_OP(dev_ops, destroy_cq);
2941	SET_DEVICE_OP(dev_ops, destroy_comp_cntr);
2942	SET_DEVICE_OP(dev_ops, destroy_flow);
2943	SET_DEVICE_OP(dev_ops, destroy_flow_action);
2944	SET_DEVICE_OP(dev_ops, destroy_qp);
2945	SET_DEVICE_OP(dev_ops, destroy_rwq_ind_table);
2946	SET_DEVICE_OP(dev_ops, destroy_srq);
2947	SET_DEVICE_OP(dev_ops, destroy_wq);
2948	SET_DEVICE_OP(dev_ops, device_group);
2949	SET_DEVICE_OP(dev_ops, detach_mcast);
2950	SET_DEVICE_OP(dev_ops, disassociate_ucontext);
2951	SET_DEVICE_OP(dev_ops, drain_rq);
2952	SET_DEVICE_OP(dev_ops, drain_sq);
2953	SET_DEVICE_OP(dev_ops, enable_driver);
2954	SET_DEVICE_OP(dev_ops, fill_res_cm_id_entry);
2955	SET_DEVICE_OP(dev_ops, fill_res_cq_entry);
2956	SET_DEVICE_OP(dev_ops, fill_res_cq_entry_raw);
2957	SET_DEVICE_OP(dev_ops, fill_res_mr_entry);
2958	SET_DEVICE_OP(dev_ops, fill_res_mr_entry_raw);
2959	SET_DEVICE_OP(dev_ops, fill_res_qp_entry);
2960	SET_DEVICE_OP(dev_ops, fill_res_qp_entry_raw);
2961	SET_DEVICE_OP(dev_ops, fill_res_srq_entry);
2962	SET_DEVICE_OP(dev_ops, fill_res_srq_entry_raw);
2963	SET_DEVICE_OP(dev_ops, fill_stat_mr_entry);
2964	SET_DEVICE_OP(dev_ops, get_dev_fw_str);
2965	SET_DEVICE_OP(dev_ops, get_dma_mr);
2966	SET_DEVICE_OP(dev_ops, get_hw_stats);
2967	SET_DEVICE_OP(dev_ops, get_link_layer);
2968	SET_DEVICE_OP(dev_ops, get_netdev);
2969	SET_DEVICE_OP(dev_ops, get_numa_node);
2970	SET_DEVICE_OP(dev_ops, get_port_immutable);
2971	SET_DEVICE_OP(dev_ops, get_vf_config);
2972	SET_DEVICE_OP(dev_ops, get_vf_guid);
2973	SET_DEVICE_OP(dev_ops, get_vf_stats);
2974	SET_DEVICE_OP(dev_ops, iw_accept);
2975	SET_DEVICE_OP(dev_ops, iw_add_ref);
2976	SET_DEVICE_OP(dev_ops, iw_connect);
2977	SET_DEVICE_OP(dev_ops, iw_create_listen);
2978	SET_DEVICE_OP(dev_ops, iw_destroy_listen);
2979	SET_DEVICE_OP(dev_ops, iw_get_qp);
2980	SET_DEVICE_OP(dev_ops, iw_reject);
2981	SET_DEVICE_OP(dev_ops, iw_rem_ref);
2982	SET_DEVICE_OP(dev_ops, map_mr_sg);
2983	SET_DEVICE_OP(dev_ops, map_mr_sg_pi);
2984	SET_DEVICE_OP(dev_ops, mmap);
2985	SET_DEVICE_OP(dev_ops, mmap_get_pfns);
2986	SET_DEVICE_OP(dev_ops, mmap_free);
2987	SET_DEVICE_OP(dev_ops, modify_ah);
2988	SET_DEVICE_OP(dev_ops, modify_cq);
2989	SET_DEVICE_OP(dev_ops, modify_device);
2990	SET_DEVICE_OP(dev_ops, modify_hw_stat);
2991	SET_DEVICE_OP(dev_ops, modify_port);
2992	SET_DEVICE_OP(dev_ops, modify_qp);
2993	SET_DEVICE_OP(dev_ops, qp_attach_comp_cntr);
2994	SET_DEVICE_OP(dev_ops, modify_srq);
2995	SET_DEVICE_OP(dev_ops, modify_wq);
2996	SET_DEVICE_OP(dev_ops, peek_cq);
2997	SET_DEVICE_OP(dev_ops, pgoff_to_mmap_entry);
2998	SET_DEVICE_OP(dev_ops, pre_destroy_cq);
2999	SET_DEVICE_OP(dev_ops, poll_cq);
3000	SET_DEVICE_OP(dev_ops, port_groups);
3001	SET_DEVICE_OP(dev_ops, post_destroy_cq);
3002	SET_DEVICE_OP(dev_ops, post_recv);
3003	SET_DEVICE_OP(dev_ops, post_send);
3004	SET_DEVICE_OP(dev_ops, post_srq_recv);
3005	SET_DEVICE_OP(dev_ops, process_mad);
3006	SET_DEVICE_OP(dev_ops, query_ah);
3007	SET_DEVICE_OP(dev_ops, query_comp_cntr_caps);
3008	SET_DEVICE_OP(dev_ops, query_device);
3009	SET_DEVICE_OP(dev_ops, query_gid);
3010	SET_DEVICE_OP(dev_ops, query_pkey);
3011	SET_DEVICE_OP(dev_ops, query_port);
3012	SET_DEVICE_OP(dev_ops, query_port_speed);
3013	SET_DEVICE_OP(dev_ops, query_qp);
3014	SET_DEVICE_OP(dev_ops, query_srq);
3015	SET_DEVICE_OP(dev_ops, query_ucontext);
3016	SET_DEVICE_OP(dev_ops, rdma_netdev_get_params);
3017	SET_DEVICE_OP(dev_ops, read_counters);
3018	SET_DEVICE_OP(dev_ops, read_comp_cntr);
3019	SET_DEVICE_OP(dev_ops, reg_dm_mr);
3020	SET_DEVICE_OP(dev_ops, reg_user_mr);
3021	SET_DEVICE_OP(dev_ops, reg_user_mr_dmabuf);
3022	SET_DEVICE_OP(dev_ops, req_notify_cq);
3023	SET_DEVICE_OP(dev_ops, rereg_user_mr);
3024	SET_DEVICE_OP(dev_ops, resize_user_cq);
3025	SET_DEVICE_OP(dev_ops, modify_comp_cntr);
3026	SET_DEVICE_OP(dev_ops, set_vf_guid);
3027	SET_DEVICE_OP(dev_ops, set_vf_link_state);
3028	SET_DEVICE_OP(dev_ops, ufile_hw_cleanup);
3029	SET_DEVICE_OP(dev_ops, report_port_event);
3030
3031	SET_OBJ_SIZE(dev_ops, ib_ah);
3032	SET_OBJ_SIZE(dev_ops, ib_counters);
3033	SET_OBJ_SIZE(dev_ops, ib_cq);
3034	SET_OBJ_SIZE(dev_ops, ib_comp_cntr);
3035	SET_OBJ_SIZE(dev_ops, ib_dmah);
3036	SET_OBJ_SIZE(dev_ops, ib_mw);
3037	SET_OBJ_SIZE(dev_ops, ib_pd);
3038	SET_OBJ_SIZE(dev_ops, ib_qp);
3039	SET_OBJ_SIZE(dev_ops, ib_rwq_ind_table);
3040	SET_OBJ_SIZE(dev_ops, ib_srq);
3041	SET_OBJ_SIZE(dev_ops, ib_ucontext);
3042	SET_OBJ_SIZE(dev_ops, ib_xrcd);
3043	SET_OBJ_SIZE(dev_ops, rdma_counter);
3044	}
3045	EXPORT_SYMBOL(ib_set_device_ops);
3046
3047	int ib_add_sub_device(struct ib_device *parent,
3048	enum rdma_nl_dev_type type,
3049	const char *name)
3050	{
3051	struct ib_device *sub;
3052	int ret = 0;
3053
3054	if (!parent->ops.add_sub_dev \|\| !parent->ops.del_sub_dev)
3055	return -EOPNOTSUPP;
3056
3057	if (!ib_device_try_get(parent))
3058	return -EINVAL;
3059
3060	sub = parent->ops.add_sub_dev(parent, type, name);
3061	if (IS_ERR(sub)) {
3062	ib_device_put(parent);
3063	return PTR_ERR(sub);
3064	}
3065
3066	sub->type = type;
3067	sub->parent = parent;
3068
3069	mutex_lock(&parent->subdev_lock);
3070	list_add_tail(&parent->subdev_list_head, &sub->subdev_list);
3071	mutex_unlock(&parent->subdev_lock);
3072
3073	return ret;
3074	}
3075
3076	int ib_del_sub_device_and_put(struct ib_device *sub)
3077	{
3078	struct ib_device *parent = sub->parent;
3079
3080	if (!parent) {
3081	ib_device_put(sub);
3082	return -EOPNOTSUPP;
3083	}
3084
3085	mutex_lock(&parent->subdev_lock);
3086	list_del(&sub->subdev_list);
3087	mutex_unlock(&parent->subdev_lock);
3088
3089	ib_device_put(sub);
3090	parent->ops.del_sub_dev(sub);
3091	ib_device_put(parent);
3092
3093	return 0;
3094	}
3095
3096	#ifdef CONFIG_INFINIBAND_VIRT_DMA
3097	int ib_dma_virt_map_sg(struct ib_device dev, struct scatterlist sg, int nents)
3098	{
3099	struct scatterlist *s;
3100	int i;
3101
3102	for_each_sg(sg, s, nents, i) {
3103	sg_dma_address(s) = (uintptr_t)sg_virt(s);
3104	sg_dma_len(s) = s->length;
3105	}
3106	return nents;
3107	}
3108	EXPORT_SYMBOL(ib_dma_virt_map_sg);
3109	#endif /* CONFIG_INFINIBAND_VIRT_DMA */
3110
3111	static const struct rdma_nl_cbs ibnl_ls_cb_table[RDMA_NL_LS_NUM_OPS] = {
3112	[RDMA_NL_LS_OP_RESOLVE] = {
3113	.doit = ib_nl_handle_resolve_resp,
3114	.flags = RDMA_NL_ADMIN_PERM,
3115	},
3116	[RDMA_NL_LS_OP_SET_TIMEOUT] = {
3117	.doit = ib_nl_handle_set_timeout,
3118	.flags = RDMA_NL_ADMIN_PERM,
3119	},
3120	[RDMA_NL_LS_OP_IP_RESOLVE] = {
3121	.doit = ib_nl_handle_ip_res_resp,
3122	.flags = RDMA_NL_ADMIN_PERM,
3123	},
3124	};
3125
3126	void ib_dispatch_port_state_event(struct ib_device ibdev, struct net_device ndev)
3127	{
3128	enum ib_port_state curr_state;
3129	struct ib_event ibevent = {};
3130	u32 port;
3131
3132	if (ib_query_netdev_port(ibdev, ndev, &port))
3133	return;
3134
3135	curr_state = ib_get_curr_port_state(ndev);
3136
3137	write_lock_irq(&ibdev->cache_lock);
3138	if (ibdev->port_data[port].cache.last_port_state == curr_state) {
3139	write_unlock_irq(&ibdev->cache_lock);
3140	return;
3141	}
3142	ibdev->port_data[port].cache.last_port_state = curr_state;
3143	write_unlock_irq(&ibdev->cache_lock);
3144
3145	ibevent.event = (curr_state == IB_PORT_DOWN) ?
3146	IB_EVENT_PORT_ERR : IB_EVENT_PORT_ACTIVE;
3147	ibevent.device = ibdev;
3148	ibevent.element.port_num = port;
3149	ib_dispatch_event(&ibevent);
3150	}
3151	EXPORT_SYMBOL(ib_dispatch_port_state_event);
3152
3153	static void handle_port_event(struct net_device *ndev, unsigned long event)
3154	{
3155	struct ib_device *ibdev;
3156
3157	/* Currently, link events in bonding scenarios are still
3158	* reported by drivers that support bonding.
3159	*/
3160	if (netif_is_lag_master(ndev) \|\| netif_is_lag_port(ndev))
3161	return;
3162
3163	ibdev = ib_device_get_by_netdev(ndev, RDMA_DRIVER_UNKNOWN);
3164	if (!ibdev)
3165	return;
3166
3167	if (ibdev->ops.report_port_event) {
3168	ibdev->ops.report_port_event(ibdev, ndev, event);
3169	goto put_ibdev;
3170	}
3171
3172	ib_dispatch_port_state_event(ibdev, ndev);
3173
3174	put_ibdev:
3175	ib_device_put(ibdev);
3176	};
3177
3178	static int ib_netdevice_event(struct notifier_block *this,
3179	unsigned long event, void *ptr)
3180	{
3181	struct net_device *ndev = netdev_notifier_info_to_dev(ptr);
3182	struct ib_device *ibdev;
3183	u32 port;
3184
3185	switch (event) {
3186	case NETDEV_CHANGENAME:
3187	ibdev = ib_device_get_by_netdev(ndev, RDMA_DRIVER_UNKNOWN);
3188	if (!ibdev)
3189	return NOTIFY_DONE;
3190
3191	if (ib_query_netdev_port(ibdev, ndev, &port)) {
3192	ib_device_put(ibdev);
3193	break;
3194	}
3195
3196	rdma_nl_notify_event(ibdev, port, RDMA_NETDEV_RENAME_EVENT);
3197	ib_device_put(ibdev);
3198	break;
3199
3200	case NETDEV_UP:
3201	case NETDEV_CHANGE:
3202	case NETDEV_DOWN:
3203	handle_port_event(ndev, event);
3204	break;
3205
3206	default:
3207	break;
3208	}
3209
3210	return NOTIFY_DONE;
3211	}
3212
3213	static struct notifier_block nb_netdevice = {
3214	.notifier_call = ib_netdevice_event,
3215	};
3216
3217	static int __init ib_core_init(void)
3218	{
3219	int ret = -ENOMEM;
3220
3221	ib_wq = alloc_workqueue("infiniband", WQ_PERCPU, 0);
3222	if (!ib_wq)
3223	return -ENOMEM;
3224
3225	ib_unreg_wq = alloc_workqueue("ib-unreg-wq", WQ_UNBOUND,
3226	WQ_UNBOUND_MAX_ACTIVE);
3227	if (!ib_unreg_wq)
3228	goto err;
3229
3230	ib_comp_wq = alloc_workqueue("ib-comp-wq",
3231	WQ_HIGHPRI \| WQ_MEM_RECLAIM \| WQ_SYSFS \| WQ_PERCPU, 0);
3232	if (!ib_comp_wq)
3233	goto err_unbound;
3234
3235	ib_comp_unbound_wq =
3236	alloc_workqueue("ib-comp-unb-wq",
3237	WQ_UNBOUND \| WQ_HIGHPRI \| WQ_MEM_RECLAIM \|
3238	WQ_SYSFS, WQ_UNBOUND_MAX_ACTIVE);
3239	if (!ib_comp_unbound_wq)
3240	goto err_comp;
3241
3242	ret = class_register(&ib_class);
3243	if (ret) {
3244	pr_warn("Couldn't create InfiniBand device class\n");
3245	goto err_comp_unbound;
3246	}
3247
3248	rdma_nl_init();
3249
3250	ret = addr_init();
3251	if (ret) {
3252	pr_warn("Couldn't init IB address resolution\n");
3253	goto err_ibnl;
3254	}
3255
3256	ret = ib_mad_init();
3257	if (ret) {
3258	pr_warn("Couldn't init IB MAD\n");
3259	goto err_addr;
3260	}
3261
3262	ret = ib_sa_init();
3263	if (ret) {
3264	pr_warn("Couldn't init SA\n");
3265	goto err_mad;
3266	}
3267
3268	ret = register_blocking_lsm_notifier(&ibdev_lsm_nb);
3269	if (ret) {
3270	pr_warn("Couldn't register LSM notifier. ret %d\n", ret);
3271	goto err_sa;
3272	}
3273
3274	ret = register_pernet_device(&rdma_dev_net_ops);
3275	if (ret) {
3276	pr_warn("Couldn't init compat dev. ret %d\n", ret);
3277	goto err_compat;
3278	}
3279
3280	nldev_init();
3281	rdma_nl_register(RDMA_NL_LS, ibnl_ls_cb_table);
3282	ret = roce_gid_mgmt_init();
3283	if (ret) {
3284	pr_warn("Couldn't init RoCE GID management\n");
3285	goto err_parent;
3286	}
3287
3288	register_netdevice_notifier(&nb_netdevice);
3289
3290	return 0;
3291
3292	err_parent:
3293	rdma_nl_unregister(RDMA_NL_LS);
3294	nldev_exit();
3295	unregister_pernet_device(&rdma_dev_net_ops);
3296	err_compat:
3297	unregister_blocking_lsm_notifier(&ibdev_lsm_nb);
3298	err_sa:
3299	ib_sa_cleanup();
3300	err_mad:
3301	ib_mad_cleanup();
3302	err_addr:
3303	addr_cleanup();
3304	err_ibnl:
3305	class_unregister(&ib_class);
3306	err_comp_unbound:
3307	destroy_workqueue(ib_comp_unbound_wq);
3308	err_comp:
3309	destroy_workqueue(ib_comp_wq);
3310	err_unbound:
3311	destroy_workqueue(ib_unreg_wq);
3312	err:
3313	destroy_workqueue(ib_wq);
3314	return ret;
3315	}
3316
3317	static void __exit ib_core_cleanup(void)
3318	{
3319	unregister_netdevice_notifier(&nb_netdevice);
3320	roce_gid_mgmt_cleanup();
3321	rdma_nl_unregister(RDMA_NL_LS);
3322	nldev_exit();
3323	unregister_pernet_device(&rdma_dev_net_ops);
3324	unregister_blocking_lsm_notifier(&ibdev_lsm_nb);
3325	ib_sa_cleanup();
3326	ib_mad_cleanup();
3327	addr_cleanup();
3328	rdma_nl_exit();
3329	class_unregister(&ib_class);
3330	destroy_workqueue(ib_comp_unbound_wq);
3331	destroy_workqueue(ib_comp_wq);
3332	/* Make sure that any pending umem accounting work is done. */
3333	destroy_workqueue(ib_wq);
3334	destroy_workqueue(ib_unreg_wq);
3335	rcu_barrier();
3336	WARN_ON(!xa_empty(&clients));
3337	WARN_ON(!xa_empty(&devices));
3338	}
3339
3340	MODULE_ALIAS_RDMA_NETLINK(RDMA_NL_LS, 4);
3341
3342	/* ib core relies on netdev stack to first register net_ns_type_operations
3343	* ns kobject type before ib_core initialization.
3344	*/
3345	fs_initcall(ib_core_init);
3346	module_exit(ib_core_cleanup);