projects/linux·blob master · refs

525 lines · 12.4 KiB

1/* SPDX-License-Identifier: GPL-2.0 */
2/* Copyright (C) 2018-2025, Advanced Micro Devices, Inc. */
3
4#ifndef _IONIC_IBDEV_H_
5#define _IONIC_IBDEV_H_
6
7#include <rdma/ib_verbs.h>
8#include <rdma/ib_pack.h>
9#include <rdma/iter.h>
10#include <rdma/uverbs_ioctl.h>
11
12#include <rdma/ionic-abi.h>
13#include <ionic_api.h>
14#include <ionic_regs.h>
15
16#include "ionic_fw.h"
17#include "ionic_queue.h"
18#include "ionic_res.h"
19
20#include "ionic_lif_cfg.h"
21
22/* Config knobs */
23#define IONIC_EQ_DEPTH 511
24#define IONIC_EQ_COUNT 32
25#define IONIC_AQ_DEPTH 63
26#define IONIC_AQ_COUNT 4
27#define IONIC_EQ_ISR_BUDGET 10
28#define IONIC_EQ_WORK_BUDGET 1000
29#define IONIC_MAX_RD_ATOM 16
30#define IONIC_PKEY_TBL_LEN 1
31#define IONIC_GID_TBL_LEN 256
32
33#define IONIC_MAX_QPID 0xffffff
34#define IONIC_SPEC_HIGH 8
35#define IONIC_MAX_PD 1024
36#define IONIC_SQCMB_ORDER 5
37#define IONIC_RQCMB_ORDER 0
38
39#define IONIC_META_LAST ((void *)1ul)
40#define IONIC_META_POSTED ((void *)2ul)
41
42#define IONIC_CQ_GRACE 100
43
44#define IONIC_ROCE_UDP_SPORT 28272
45#define IONIC_DMA_LKEY 0
46#define IONIC_DMA_RKEY IONIC_DMA_LKEY
47
48#define IONIC_CMB_SUPPORTED \
49 (IONIC_CMB_ENABLE | IONIC_CMB_REQUIRE | IONIC_CMB_EXPDB | \
50 IONIC_CMB_WC | IONIC_CMB_UC)
51
52/* resource is not reserved on the device, indicated in tbl_order */
53#define IONIC_RES_INVALID -1
54
55struct ionic_aq;
56struct ionic_cq;
57struct ionic_eq;
58struct ionic_vcq;
59
60enum ionic_admin_state {
61 IONIC_ADMIN_ACTIVE, /* submitting admin commands to queue */
62 IONIC_ADMIN_PAUSED, /* not submitting, but may complete normally */
63 IONIC_ADMIN_KILLED, /* not submitting, locally completed */
64};
65
66enum ionic_admin_flags {
67 IONIC_ADMIN_F_BUSYWAIT = BIT(0), /* Don't sleep */
68 IONIC_ADMIN_F_TEARDOWN = BIT(1), /* In destroy path */
69 IONIC_ADMIN_F_INTERRUPT = BIT(2), /* Interruptible w/timeout */
70};
71
72enum ionic_mmap_flag {
73 IONIC_MMAP_WC = BIT(0),
74 IONIC_MMAP_PHC = BIT(1),
75};
76
77struct ionic_mmap_entry {
78 struct rdma_user_mmap_entry rdma_entry;
79 unsigned long size;
80 unsigned long pfn;
81 u8 mmap_flags;
82};
83
84struct ionic_ibdev {
85 struct ib_device ibdev;
86
87 struct ionic_lif_cfg lif_cfg;
88
89 struct xarray qp_tbl;
90 struct xarray cq_tbl;
91
92 struct ionic_resid_bits inuse_dbid;
93 struct ionic_resid_bits inuse_pdid;
94 struct ionic_resid_bits inuse_ahid;
95 struct ionic_resid_bits inuse_mrid;
96 struct ionic_resid_bits inuse_qpid;
97 struct ionic_resid_bits inuse_cqid;
98
99 u8 half_cqid_udma_shift;
100 u8 half_qpid_udma_shift;
101 u8 next_qpid_udma_idx;
102 u8 next_mrkey;
103
104 struct work_struct reset_work;
105 bool reset_posted;
106 u32 reset_cnt;
107
108 struct delayed_work admin_dwork;
109 struct ionic_aq **aq_vec;
110 atomic_t admin_state;
111
112 struct ionic_eq **eq_vec;
113
114 struct ionic_v1_stat *hw_stats;
115 void *hw_stats_buf;
116 struct rdma_stat_desc *hw_stats_hdrs;
117 struct ionic_counter_stats *counter_stats;
118 int hw_stats_count;
119};
120
121struct ionic_eq {
122 struct ionic_ibdev *dev;
123
124 u32 eqid;
125 u32 intr;
126
127 struct ionic_queue q;
128
129 int armed;
130 bool enable;
131
132 struct work_struct work;
133
134 int irq;
135 char name[32];
136};
137
138struct ionic_admin_wr {
139 struct completion work;
140 struct list_head aq_ent;
141 struct ionic_v1_admin_wqe wqe;
142 struct ionic_v1_cqe cqe;
143 struct ionic_aq *aq;
144 int status;
145};
146
147struct ionic_admin_wr_q {
148 struct ionic_admin_wr *wr;
149 int wqe_strides;
150};
151
152struct ionic_aq {
153 struct ionic_ibdev *dev;
154 struct ionic_vcq *vcq;
155
156 struct work_struct work;
157
158 atomic_t admin_state;
159 unsigned long stamp;
160 bool armed;
161
162 u32 aqid;
163 u32 cqid;
164
165 spinlock_t lock; /* for posting */
166 struct ionic_queue q;
167 struct ionic_admin_wr_q *q_wr;
168 struct list_head wr_prod;
169 struct list_head wr_post;
170};
171
172struct ionic_ctx {
173 struct ib_ucontext ibctx;
174 u32 dbid;
175 struct rdma_user_mmap_entry *mmap_dbell;
176 struct rdma_user_mmap_entry *mmap_phc;
177};
178
179struct ionic_tbl_buf {
180 u32 tbl_limit;
181 u32 tbl_pages;
182 size_t tbl_size;
183 __le64 *tbl_buf;
184 dma_addr_t tbl_dma;
185 u8 page_size_log2;
186};
187
188struct ionic_pd {
189 struct ib_pd ibpd;
190
191 u32 pdid;
192 u32 flags;
193};
194
195struct ionic_cq {
196 struct ionic_vcq *vcq;
197
198 u32 cqid;
199 u32 eqid;
200
201 spinlock_t lock; /* for polling */
202 struct list_head poll_sq;
203 bool flush;
204 struct list_head flush_sq;
205 struct list_head flush_rq;
206 struct list_head ibkill_flush_ent;
207
208 struct ionic_queue q;
209 bool color;
210 int credit;
211 u16 arm_any_prod;
212 u16 arm_sol_prod;
213
214 struct kref cq_kref;
215 struct completion cq_rel_comp;
216
217 /* infrequently accessed, keep at end */
218 struct ib_umem *umem;
219};
220
221struct ionic_vcq {
222 struct ib_cq ibcq;
223 struct ionic_cq cq[2];
224 u8 udma_mask;
225 u8 poll_idx;
226};
227
228struct ionic_sq_meta {
229 u64 wrid;
230 u32 len;
231 u16 seq;
232 u8 ibop;
233 u8 ibsts;
234 u8 remote:1;
235 u8 signal:1;
236 u8 local_comp:1;
237};
238
239struct ionic_rq_meta {
240 struct ionic_rq_meta *next;
241 u64 wrid;
242};
243
244struct ionic_qp {
245 struct ib_qp ibqp;
246 enum ib_qp_state state;
247
248 u32 qpid;
249 u32 ahid;
250 u32 sq_cqid;
251 u32 rq_cqid;
252 u8 udma_idx;
253 u8 has_ah:1;
254 u8 has_sq:1;
255 u8 has_rq:1;
256 u8 sig_all:1;
257
258 struct list_head qp_list_counter;
259
260 struct list_head cq_poll_sq;
261 struct list_head cq_flush_sq;
262 struct list_head cq_flush_rq;
263 struct list_head ibkill_flush_ent;
264
265 spinlock_t sq_lock; /* for posting and polling */
266 struct ionic_queue sq;
267 struct ionic_sq_meta *sq_meta;
268 u16 *sq_msn_idx;
269 int sq_spec;
270 u16 sq_old_prod;
271 u16 sq_msn_prod;
272 u16 sq_msn_cons;
273 u8 sq_cmb;
274 bool sq_flush;
275 bool sq_flush_rcvd;
276
277 spinlock_t rq_lock; /* for posting and polling */
278 struct ionic_queue rq;
279 struct ionic_rq_meta *rq_meta;
280 struct ionic_rq_meta *rq_meta_head;
281 int rq_spec;
282 u16 rq_old_prod;
283 u8 rq_cmb;
284 bool rq_flush;
285
286 struct kref qp_kref;
287 struct completion qp_rel_comp;
288
289 /* infrequently accessed, keep at end */
290 int sgid_index;
291 int sq_cmb_order;
292 u32 sq_cmb_pgid;
293 phys_addr_t sq_cmb_addr;
294 struct rdma_user_mmap_entry *mmap_sq_cmb;
295
296 struct ib_umem *sq_umem;
297
298 int rq_cmb_order;
299 u32 rq_cmb_pgid;
300 phys_addr_t rq_cmb_addr;
301 struct rdma_user_mmap_entry *mmap_rq_cmb;
302
303 struct ib_umem *rq_umem;
304
305 int dcqcn_profile;
306
307 struct ib_ud_header *hdr;
308};
309
310struct ionic_ah {
311 struct ib_ah ibah;
312 u32 ahid;
313 int sgid_index;
314 struct ib_ud_header hdr;
315};
316
317struct ionic_mr {
318 union {
319 struct ib_mr ibmr;
320 struct ib_mw ibmw;
321 };
322
323 u32 mrid;
324 int flags;
325
326 struct ib_umem *umem;
327 struct ionic_tbl_buf buf;
328 bool created;
329};
330
331struct ionic_counter_stats {
332 int queue_stats_count;
333 struct ionic_v1_stat *hdr;
334 struct rdma_stat_desc *stats_hdrs;
335 struct ida counter_ida;
336};
337
338struct ionic_rdma_counter {
339 struct rdma_counter rdma_counter;
340 void *vals;
341 struct list_head qp_list;
342};
343
344static inline struct ionic_rdma_counter *
345to_ionic_rdma_counter(struct rdma_counter *counter)
346{
347 return container_of(counter, struct ionic_rdma_counter, rdma_counter);
348}
349
350static inline struct ionic_ibdev *to_ionic_ibdev(struct ib_device *ibdev)
351{
352 return container_of(ibdev, struct ionic_ibdev, ibdev);
353}
354
355static inline struct ionic_ctx *to_ionic_ctx(struct ib_ucontext *ibctx)
356{
357 return container_of(ibctx, struct ionic_ctx, ibctx);
358}
359
360static inline struct ionic_ctx *to_ionic_ctx_uobj(struct ib_uobject *uobj)
361{
362 if (!uobj)
363 return NULL;
364
365 if (!uobj->context)
366 return NULL;
367
368 return to_ionic_ctx(uobj->context);
369}
370
371static inline struct ionic_pd *to_ionic_pd(struct ib_pd *ibpd)
372{
373 return container_of(ibpd, struct ionic_pd, ibpd);
374}
375
376static inline struct ionic_mr *to_ionic_mr(struct ib_mr *ibmr)
377{
378 return container_of(ibmr, struct ionic_mr, ibmr);
379}
380
381static inline struct ionic_mr *to_ionic_mw(struct ib_mw *ibmw)
382{
383 return container_of(ibmw, struct ionic_mr, ibmw);
384}
385
386static inline struct ionic_vcq *to_ionic_vcq(struct ib_cq *ibcq)
387{
388 return container_of(ibcq, struct ionic_vcq, ibcq);
389}
390
391static inline struct ionic_cq *to_ionic_vcq_cq(struct ib_cq *ibcq,
392 uint8_t udma_idx)
393{
394 return &to_ionic_vcq(ibcq)->cq[udma_idx];
395}
396
397static inline struct ionic_qp *to_ionic_qp(struct ib_qp *ibqp)
398{
399 return container_of(ibqp, struct ionic_qp, ibqp);
400}
401
402static inline struct ionic_ah *to_ionic_ah(struct ib_ah *ibah)
403{
404 return container_of(ibah, struct ionic_ah, ibah);
405}
406
407static inline u32 ionic_ctx_dbid(struct ionic_ibdev *dev,
408 struct ionic_ctx *ctx)
409{
410 if (!ctx)
411 return dev->lif_cfg.dbid;
412
413 return ctx->dbid;
414}
415
416static inline u32 ionic_obj_dbid(struct ionic_ibdev *dev,
417 struct ib_uobject *uobj)
418{
419 return ionic_ctx_dbid(dev, to_ionic_ctx_uobj(uobj));
420}
421
422static inline bool ionic_ibop_is_local(enum ib_wr_opcode op)
423{
424 return op == IB_WR_LOCAL_INV || op == IB_WR_REG_MR;
425}
426
427static inline void ionic_qp_complete(struct kref *kref)
428{
429 struct ionic_qp *qp = container_of(kref, struct ionic_qp, qp_kref);
430
431 complete(&qp->qp_rel_comp);
432}
433
434static inline void ionic_cq_complete(struct kref *kref)
435{
436 struct ionic_cq *cq = container_of(kref, struct ionic_cq, cq_kref);
437
438 complete(&cq->cq_rel_comp);
439}
440
441/* ionic_admin.c */
442extern struct workqueue_struct *ionic_evt_workq;
443void ionic_admin_post(struct ionic_ibdev *dev, struct ionic_admin_wr *wr);
444int ionic_admin_wait(struct ionic_ibdev *dev, struct ionic_admin_wr *wr,
445 enum ionic_admin_flags);
446
447int ionic_rdma_reset_devcmd(struct ionic_ibdev *dev);
448
449int ionic_create_rdma_admin(struct ionic_ibdev *dev);
450void ionic_destroy_rdma_admin(struct ionic_ibdev *dev);
451void ionic_kill_rdma_admin(struct ionic_ibdev *dev, bool fatal_path);
452
453/* ionic_controlpath.c */
454int ionic_create_cq_common(struct ionic_vcq *vcq,
455 struct ionic_tbl_buf *buf,
456 const struct ib_cq_init_attr *attr,
457 struct ionic_ctx *ctx,
458 struct ib_udata *udata,
459 struct ionic_qdesc *req_cq,
460 __u32 *resp_cqid,
461 int udma_idx);
462void ionic_destroy_cq_common(struct ionic_ibdev *dev, struct ionic_cq *cq);
463void ionic_flush_qp(struct ionic_ibdev *dev, struct ionic_qp *qp);
464void ionic_notify_flush_cq(struct ionic_cq *cq);
465
466int ionic_alloc_ucontext(struct ib_ucontext *ibctx, struct ib_udata *udata);
467void ionic_dealloc_ucontext(struct ib_ucontext *ibctx);
468int ionic_mmap(struct ib_ucontext *ibctx, struct vm_area_struct *vma);
469void ionic_mmap_free(struct rdma_user_mmap_entry *rdma_entry);
470int ionic_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata);
471int ionic_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata);
472int ionic_create_ah(struct ib_ah *ibah, struct rdma_ah_init_attr *init_attr,
473 struct ib_udata *udata);
474int ionic_query_ah(struct ib_ah *ibah, struct rdma_ah_attr *ah_attr);
475int ionic_destroy_ah(struct ib_ah *ibah, u32 flags);
476struct ib_mr *ionic_get_dma_mr(struct ib_pd *ibpd, int access);
477struct ib_mr *ionic_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 length,
478 u64 addr, int access, struct ib_dmah *dmah,
479 struct ib_udata *udata);
480struct ib_mr *ionic_reg_user_mr_dmabuf(struct ib_pd *ibpd, u64 offset,
481 u64 length, u64 addr, int fd, int access,
482 struct ib_dmah *dmah,
483 struct uverbs_attr_bundle *attrs);
484int ionic_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata);
485struct ib_mr *ionic_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type type,
486 u32 max_sg);
487int ionic_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents,
488 unsigned int *sg_offset);
489int ionic_alloc_mw(struct ib_mw *ibmw, struct ib_udata *udata);
490int ionic_dealloc_mw(struct ib_mw *ibmw);
491int ionic_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
492 struct uverbs_attr_bundle *attrs);
493int ionic_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata);
494int ionic_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attr,
495 struct ib_udata *udata);
496int ionic_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int mask,
497 struct ib_udata *udata);
498int ionic_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int mask,
499 struct ib_qp_init_attr *init_attr);
500int ionic_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata);
501
502/* ionic_datapath.c */
503int ionic_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr,
504 const struct ib_send_wr **bad);
505int ionic_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr,
506 const struct ib_recv_wr **bad);
507int ionic_poll_cq(struct ib_cq *ibcq, int nwc, struct ib_wc *wc);
508int ionic_req_notify_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags);
509
510/* ionic_hw_stats.c */
511void ionic_stats_init(struct ionic_ibdev *dev);
512void ionic_stats_cleanup(struct ionic_ibdev *dev);
513
514/* ionic_pgtbl.c */
515__le64 ionic_pgtbl_dma(struct ionic_tbl_buf *buf, u64 va);
516__be64 ionic_pgtbl_off(struct ionic_tbl_buf *buf, u64 va);
517int ionic_pgtbl_page(struct ionic_tbl_buf *buf, u64 dma);
518int ionic_pgtbl_init(struct ionic_ibdev *dev,
519 struct ionic_tbl_buf *buf,
520 struct ib_umem *umem,
521 dma_addr_t dma,
522 int limit,
523 u64 page_size);
524void ionic_pgtbl_unbuf(struct ionic_ibdev *dev, struct ionic_tbl_buf *buf);
525#endif /* _IONIC_IBDEV_H_ */