projects/linux·blob master · refs

366 lines · 8.2 KiB

1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Functions for dibs loopback/loopback-ism device.
4 *
5 * Copyright (c) 2024, Alibaba Inc.
6 *
7 * Author: Wen Gu <[email protected]>
8 * Tony Lu <[email protected]>
9 *
10 */
11
12#include <linux/bitops.h>
13#include <linux/device.h>
14#include <linux/dibs.h>
15#include <linux/mm.h>
16#include <linux/slab.h>
17#include <linux/spinlock.h>
18#include <linux/types.h>
19
20#include "dibs_loopback.h"
21
22#define DIBS_LO_SUPPORT_NOCOPY 0x1
23#define DIBS_DMA_ADDR_INVALID (~(dma_addr_t)0)
24
25static const char dibs_lo_dev_name[] = "lo";
26/* global loopback device */
27static struct dibs_lo_dev *lo_dev;
28
29static u16 dibs_lo_get_fabric_id(struct dibs_dev *dibs)
30{
31 return DIBS_LOOPBACK_FABRIC;
32}
33
34static int dibs_lo_query_rgid(struct dibs_dev *dibs, const uuid_t *rgid,
35 u32 vid_valid, u32 vid)
36{
37 /* rgid should be the same as lgid */
38 if (!uuid_equal(rgid, &dibs->gid))
39 return -ENETUNREACH;
40 return 0;
41}
42
43static int dibs_lo_max_dmbs(void)
44{
45 return DIBS_LO_MAX_DMBS;
46}
47
48static int dibs_lo_register_dmb(struct dibs_dev *dibs, struct dibs_dmb *dmb,
49 struct dibs_client *client)
50{
51 struct dibs_lo_dmb_node *dmb_node, *tmp_node;
52 struct dibs_lo_dev *ldev;
53 struct folio *folio;
54 unsigned long flags;
55 int sba_idx, rc;
56
57 ldev = dibs->drv_priv;
58 sba_idx = dmb->idx;
59 /* check space for new dmb */
60 for_each_clear_bit(sba_idx, ldev->sba_idx_mask, DIBS_LO_MAX_DMBS) {
61 if (!test_and_set_bit(sba_idx, ldev->sba_idx_mask))
62 break;
63 }
64 if (sba_idx == DIBS_LO_MAX_DMBS)
65 return -ENOSPC;
66
67 dmb_node = kzalloc_obj(*dmb_node);
68 if (!dmb_node) {
69 rc = -ENOMEM;
70 goto err_bit;
71 }
72
73 dmb_node->sba_idx = sba_idx;
74 dmb_node->len = dmb->dmb_len;
75
76 /* not critical; fail under memory pressure and fallback to TCP */
77 folio = folio_alloc(GFP_KERNEL | __GFP_NOWARN | __GFP_NOMEMALLOC |
78 __GFP_NORETRY | __GFP_ZERO,
79 get_order(dmb_node->len));
80 if (!folio) {
81 rc = -ENOMEM;
82 goto err_node;
83 }
84 dmb_node->cpu_addr = folio_address(folio);
85 dmb_node->dma_addr = DIBS_DMA_ADDR_INVALID;
86 refcount_set(&dmb_node->refcnt, 1);
87
88again:
89 /* add new dmb into hash table */
90 get_random_bytes(&dmb_node->token, sizeof(dmb_node->token));
91 write_lock_bh(&ldev->dmb_ht_lock);
92 hash_for_each_possible(ldev->dmb_ht, tmp_node, list, dmb_node->token) {
93 if (tmp_node->token == dmb_node->token) {
94 write_unlock_bh(&ldev->dmb_ht_lock);
95 goto again;
96 }
97 }
98 hash_add(ldev->dmb_ht, &dmb_node->list, dmb_node->token);
99 write_unlock_bh(&ldev->dmb_ht_lock);
100 atomic_inc(&ldev->dmb_cnt);
101
102 dmb->idx = dmb_node->sba_idx;
103 dmb->dmb_tok = dmb_node->token;
104 dmb->cpu_addr = dmb_node->cpu_addr;
105 dmb->dma_addr = dmb_node->dma_addr;
106 dmb->dmb_len = dmb_node->len;
107
108 spin_lock_irqsave(&dibs->lock, flags);
109 dibs->dmb_clientid_arr[sba_idx] = client->id;
110 spin_unlock_irqrestore(&dibs->lock, flags);
111
112 return 0;
113
114err_node:
115 kfree(dmb_node);
116err_bit:
117 clear_bit(sba_idx, ldev->sba_idx_mask);
118 return rc;
119}
120
121static void dibs_lo_free_dmb(struct dibs_lo_dev *ldev,
122 struct dibs_lo_dmb_node *dmb_node)
123{
124 clear_bit(dmb_node->sba_idx, ldev->sba_idx_mask);
125 folio_put(virt_to_folio(dmb_node->cpu_addr));
126 kfree(dmb_node);
127
128 if (atomic_dec_and_test(&ldev->dmb_cnt))
129 wake_up(&ldev->ldev_release);
130}
131
132static int dibs_lo_unregister_dmb(struct dibs_dev *dibs, struct dibs_dmb *dmb)
133{
134 struct dibs_lo_dmb_node *dmb_node = NULL, *tmp_node;
135 struct dibs_lo_dev *ldev;
136 unsigned long flags;
137 bool last;
138
139 ldev = dibs->drv_priv;
140
141 /* find dmb from hash table */
142 write_lock_bh(&ldev->dmb_ht_lock);
143 hash_for_each_possible(ldev->dmb_ht, tmp_node, list, dmb->dmb_tok) {
144 if (tmp_node->token == dmb->dmb_tok) {
145 dmb_node = tmp_node;
146 break;
147 }
148 }
149 if (!dmb_node) {
150 write_unlock_bh(&ldev->dmb_ht_lock);
151 return -EINVAL;
152 }
153 last = refcount_dec_and_test(&dmb_node->refcnt);
154 if (last)
155 hash_del(&dmb_node->list);
156 write_unlock_bh(&ldev->dmb_ht_lock);
157
158 if (last) {
159 spin_lock_irqsave(&dibs->lock, flags);
160 dibs->dmb_clientid_arr[dmb_node->sba_idx] = NO_DIBS_CLIENT;
161 spin_unlock_irqrestore(&dibs->lock, flags);
162
163 dibs_lo_free_dmb(ldev, dmb_node);
164 }
165 return 0;
166}
167
168static int dibs_lo_support_dmb_nocopy(struct dibs_dev *dibs)
169{
170 return DIBS_LO_SUPPORT_NOCOPY;
171}
172
173static int dibs_lo_attach_dmb(struct dibs_dev *dibs, struct dibs_dmb *dmb)
174{
175 struct dibs_lo_dmb_node *dmb_node = NULL, *tmp_node;
176 struct dibs_lo_dev *ldev;
177
178 ldev = dibs->drv_priv;
179
180 /* find dmb_node according to dmb->dmb_tok */
181 read_lock_bh(&ldev->dmb_ht_lock);
182 hash_for_each_possible(ldev->dmb_ht, tmp_node, list, dmb->dmb_tok) {
183 if (tmp_node->token == dmb->dmb_tok) {
184 dmb_node = tmp_node;
185 break;
186 }
187 }
188 if (!dmb_node) {
189 read_unlock_bh(&ldev->dmb_ht_lock);
190 return -EINVAL;
191 }
192 refcount_inc(&dmb_node->refcnt);
193 read_unlock_bh(&ldev->dmb_ht_lock);
194
195 /* provide dmb information */
196 dmb->idx = dmb_node->sba_idx;
197 dmb->dmb_tok = dmb_node->token;
198 dmb->cpu_addr = dmb_node->cpu_addr;
199 dmb->dma_addr = dmb_node->dma_addr;
200 dmb->dmb_len = dmb_node->len;
201 return 0;
202}
203
204static int dibs_lo_detach_dmb(struct dibs_dev *dibs, u64 token)
205{
206 struct dibs_lo_dmb_node *dmb_node = NULL, *tmp_node;
207 struct dibs_lo_dev *ldev;
208 bool last;
209
210 ldev = dibs->drv_priv;
211
212 /* find dmb_node according to dmb->dmb_tok */
213 write_lock_bh(&ldev->dmb_ht_lock);
214 hash_for_each_possible(ldev->dmb_ht, tmp_node, list, token) {
215 if (tmp_node->token == token) {
216 dmb_node = tmp_node;
217 break;
218 }
219 }
220 if (!dmb_node) {
221 write_unlock_bh(&ldev->dmb_ht_lock);
222 return -EINVAL;
223 }
224 last = refcount_dec_and_test(&dmb_node->refcnt);
225 if (last)
226 hash_del(&dmb_node->list);
227 write_unlock_bh(&ldev->dmb_ht_lock);
228
229 if (last)
230 dibs_lo_free_dmb(ldev, dmb_node);
231
232 return 0;
233}
234
235static int dibs_lo_move_data(struct dibs_dev *dibs, u64 dmb_tok,
236 unsigned int idx, bool sf, unsigned int offset,
237 void *data, unsigned int size)
238{
239 struct dibs_lo_dmb_node *rmb_node = NULL, *tmp_node;
240 struct dibs_lo_dev *ldev;
241 u16 s_mask;
242 u8 client_id;
243 u32 sba_idx;
244
245 ldev = dibs->drv_priv;
246
247 read_lock_bh(&ldev->dmb_ht_lock);
248 hash_for_each_possible(ldev->dmb_ht, tmp_node, list, dmb_tok) {
249 if (tmp_node->token == dmb_tok) {
250 rmb_node = tmp_node;
251 break;
252 }
253 }
254 if (!rmb_node) {
255 read_unlock_bh(&ldev->dmb_ht_lock);
256 return -EINVAL;
257 }
258 if ((u64)offset + size > rmb_node->len) {
259 read_unlock_bh(&ldev->dmb_ht_lock);
260 return -EINVAL;
261 }
262
263 memcpy((char *)rmb_node->cpu_addr + offset, data, size);
264 sba_idx = rmb_node->sba_idx;
265 read_unlock_bh(&ldev->dmb_ht_lock);
266
267 if (!sf)
268 return 0;
269
270 spin_lock(&dibs->lock);
271 client_id = dibs->dmb_clientid_arr[sba_idx];
272 s_mask = ror16(0x1000, idx);
273 if (likely(client_id != NO_DIBS_CLIENT && dibs->subs[client_id]))
274 dibs->subs[client_id]->ops->handle_irq(dibs, sba_idx, s_mask);
275 spin_unlock(&dibs->lock);
276
277 return 0;
278}
279
280static const struct dibs_dev_ops dibs_lo_ops = {
281 .get_fabric_id = dibs_lo_get_fabric_id,
282 .query_remote_gid = dibs_lo_query_rgid,
283 .max_dmbs = dibs_lo_max_dmbs,
284 .register_dmb = dibs_lo_register_dmb,
285 .unregister_dmb = dibs_lo_unregister_dmb,
286 .move_data = dibs_lo_move_data,
287 .support_mmapped_rdmb = dibs_lo_support_dmb_nocopy,
288 .attach_dmb = dibs_lo_attach_dmb,
289 .detach_dmb = dibs_lo_detach_dmb,
290};
291
292static void dibs_lo_dev_init(struct dibs_lo_dev *ldev)
293{
294 rwlock_init(&ldev->dmb_ht_lock);
295 hash_init(ldev->dmb_ht);
296 atomic_set(&ldev->dmb_cnt, 0);
297 init_waitqueue_head(&ldev->ldev_release);
298}
299
300static void dibs_lo_dev_exit(struct dibs_lo_dev *ldev)
301{
302 if (atomic_read(&ldev->dmb_cnt))
303 wait_event(ldev->ldev_release, !atomic_read(&ldev->dmb_cnt));
304}
305
306static int dibs_lo_dev_probe(void)
307{
308 struct dibs_lo_dev *ldev;
309 struct dibs_dev *dibs;
310 int ret;
311
312 ldev = kzalloc_obj(*ldev);
313 if (!ldev)
314 return -ENOMEM;
315
316 dibs = dibs_dev_alloc();
317 if (!dibs) {
318 kfree(ldev);
319 return -ENOMEM;
320 }
321
322 ldev->dibs = dibs;
323 dibs->drv_priv = ldev;
324 dibs_lo_dev_init(ldev);
325 uuid_gen(&dibs->gid);
326 dibs->ops = &dibs_lo_ops;
327
328 dibs->dev.parent = NULL;
329 dev_set_name(&dibs->dev, "%s", dibs_lo_dev_name);
330
331 ret = dibs_dev_add(dibs);
332 if (ret)
333 goto err_reg;
334 lo_dev = ldev;
335 return 0;
336
337err_reg:
338 /* pairs with dibs_dev_alloc() */
339 put_device(&dibs->dev);
340 kfree(ldev);
341
342 return ret;
343}
344
345static void dibs_lo_dev_remove(void)
346{
347 if (!lo_dev)
348 return;
349
350 dibs_dev_del(lo_dev->dibs);
351 dibs_lo_dev_exit(lo_dev);
352 /* pairs with dibs_dev_alloc() */
353 put_device(&lo_dev->dibs->dev);
354 kfree(lo_dev);
355 lo_dev = NULL;
356}
357
358int dibs_loopback_init(void)
359{
360 return dibs_lo_dev_probe();
361}
362
363void dibs_loopback_exit(void)
364{
365 dibs_lo_dev_remove();
366}