projects/linux·blob master · refs

2768 lines · 64.7 KiB

1// SPDX-License-Identifier: GPL-2.0
2/* Copyright (C) 2018-2025, Advanced Micro Devices, Inc. */
3
4#include <linux/module.h>
5#include <linux/printk.h>
6#include <rdma/ib_addr.h>
7#include <rdma/ib_cache.h>
8#include <rdma/ib_user_verbs.h>
9#include <ionic_api.h>
10
11#include "ionic_fw.h"
12#include "ionic_ibdev.h"
13
14#define ionic_set_ecn(tos) (((tos) | 2u) & ~1u)
15#define ionic_clear_ecn(tos) ((tos) & ~3u)
16
17static int ionic_validate_qdesc(struct ionic_qdesc *q)
18{
19 if (!q->addr || !q->size || !q->mask ||
20 !q->depth_log2 || !q->stride_log2)
21 return -EINVAL;
22
23 if (q->addr & (PAGE_SIZE - 1))
24 return -EINVAL;
25
26 if (q->mask != BIT(q->depth_log2) - 1)
27 return -EINVAL;
28
29 if (q->size < BIT_ULL(q->depth_log2 + q->stride_log2))
30 return -EINVAL;
31
32 return 0;
33}
34
35static u32 ionic_get_eqid(struct ionic_ibdev *dev, u32 comp_vector, u8 udma_idx)
36{
37 /* EQ per vector per udma, and the first eqs reserved for async events.
38 * The rest of the vectors can be requested for completions.
39 */
40 u32 comp_vec_count = dev->lif_cfg.eq_count / dev->lif_cfg.udma_count - 1;
41
42 return (comp_vector % comp_vec_count + 1) * dev->lif_cfg.udma_count + udma_idx;
43}
44
45static int ionic_get_cqid(struct ionic_ibdev *dev, u32 *cqid, u8 udma_idx)
46{
47 unsigned int size, base, bound;
48 int rc;
49
50 size = dev->lif_cfg.cq_count / dev->lif_cfg.udma_count;
51 base = size * udma_idx;
52 bound = base + size;
53
54 rc = ionic_resid_get_shared(&dev->inuse_cqid, base, bound);
55 if (rc >= 0) {
56 /* cq_base is zero or a multiple of two queue groups */
57 *cqid = dev->lif_cfg.cq_base +
58 ionic_bitid_to_qid(rc, dev->lif_cfg.udma_qgrp_shift,
59 dev->half_cqid_udma_shift);
60
61 rc = 0;
62 }
63
64 return rc;
65}
66
67static void ionic_put_cqid(struct ionic_ibdev *dev, u32 cqid)
68{
69 u32 bitid = ionic_qid_to_bitid(cqid - dev->lif_cfg.cq_base,
70 dev->lif_cfg.udma_qgrp_shift,
71 dev->half_cqid_udma_shift);
72
73 ionic_resid_put(&dev->inuse_cqid, bitid);
74}
75
76int ionic_create_cq_common(struct ionic_vcq *vcq,
77 struct ionic_tbl_buf *buf,
78 const struct ib_cq_init_attr *attr,
79 struct ionic_ctx *ctx,
80 struct ib_udata *udata,
81 struct ionic_qdesc *req_cq,
82 __u32 *resp_cqid,
83 int udma_idx)
84{
85 struct ionic_ibdev *dev = to_ionic_ibdev(vcq->ibcq.device);
86 struct ionic_cq *cq = &vcq->cq[udma_idx];
87 void *entry;
88 int rc;
89
90 cq->vcq = vcq;
91
92 if (attr->cqe < 1 || attr->cqe + IONIC_CQ_GRACE > 0xffff) {
93 rc = -EINVAL;
94 goto err_args;
95 }
96
97 rc = ionic_get_cqid(dev, &cq->cqid, udma_idx);
98 if (rc)
99 goto err_args;
100
101 cq->eqid = ionic_get_eqid(dev, attr->comp_vector, udma_idx);
102
103 spin_lock_init(&cq->lock);
104 INIT_LIST_HEAD(&cq->poll_sq);
105 INIT_LIST_HEAD(&cq->flush_sq);
106 INIT_LIST_HEAD(&cq->flush_rq);
107
108 if (udata) {
109 rc = ionic_validate_qdesc(req_cq);
110 if (rc)
111 goto err_qdesc;
112
113 cq->umem = ib_umem_get_va(&dev->ibdev, req_cq->addr,
114 req_cq->size, IB_ACCESS_LOCAL_WRITE);
115 if (IS_ERR(cq->umem)) {
116 rc = PTR_ERR(cq->umem);
117 goto err_qdesc;
118 }
119
120 cq->q.ptr = NULL;
121 cq->q.size = req_cq->size;
122 cq->q.mask = req_cq->mask;
123 cq->q.depth_log2 = req_cq->depth_log2;
124 cq->q.stride_log2 = req_cq->stride_log2;
125
126 *resp_cqid = cq->cqid;
127 } else {
128 rc = ionic_queue_init(&cq->q, dev->lif_cfg.hwdev,
129 attr->cqe + IONIC_CQ_GRACE,
130 sizeof(struct ionic_v1_cqe));
131 if (rc)
132 goto err_q_init;
133
134 ionic_queue_dbell_init(&cq->q, cq->cqid);
135 cq->color = true;
136 cq->credit = cq->q.mask;
137 }
138
139 rc = ionic_pgtbl_init(dev, buf, cq->umem, cq->q.dma, 1, PAGE_SIZE);
140 if (rc)
141 goto err_pgtbl_init;
142
143 init_completion(&cq->cq_rel_comp);
144 kref_init(&cq->cq_kref);
145
146 entry = xa_store_irq(&dev->cq_tbl, cq->cqid, cq, GFP_KERNEL);
147 if (entry) {
148 if (!xa_is_err(entry))
149 rc = -EINVAL;
150 else
151 rc = xa_err(entry);
152
153 goto err_xa;
154 }
155
156 return 0;
157
158err_xa:
159 ionic_pgtbl_unbuf(dev, buf);
160err_pgtbl_init:
161 if (!udata)
162 ionic_queue_destroy(&cq->q, dev->lif_cfg.hwdev);
163err_q_init:
164 if (cq->umem)
165 ib_umem_release(cq->umem);
166err_qdesc:
167 ionic_put_cqid(dev, cq->cqid);
168err_args:
169 cq->vcq = NULL;
170
171 return rc;
172}
173
174void ionic_destroy_cq_common(struct ionic_ibdev *dev, struct ionic_cq *cq)
175{
176 if (!cq->vcq)
177 return;
178
179 xa_erase_irq(&dev->cq_tbl, cq->cqid);
180
181 kref_put(&cq->cq_kref, ionic_cq_complete);
182 wait_for_completion(&cq->cq_rel_comp);
183
184 if (cq->umem)
185 ib_umem_release(cq->umem);
186 else
187 ionic_queue_destroy(&cq->q, dev->lif_cfg.hwdev);
188
189 ionic_put_cqid(dev, cq->cqid);
190
191 cq->vcq = NULL;
192}
193
194static int ionic_validate_qdesc_zero(struct ionic_qdesc *q)
195{
196 if (q->addr || q->size || q->mask || q->depth_log2 || q->stride_log2)
197 return -EINVAL;
198
199 return 0;
200}
201
202static int ionic_get_pdid(struct ionic_ibdev *dev, u32 *pdid)
203{
204 int rc;
205
206 rc = ionic_resid_get(&dev->inuse_pdid);
207 if (rc < 0)
208 return rc;
209
210 *pdid = rc;
211 return 0;
212}
213
214static int ionic_get_ahid(struct ionic_ibdev *dev, u32 *ahid)
215{
216 int rc;
217
218 rc = ionic_resid_get(&dev->inuse_ahid);
219 if (rc < 0)
220 return rc;
221
222 *ahid = rc;
223 return 0;
224}
225
226static int ionic_get_mrid(struct ionic_ibdev *dev, u32 *mrid)
227{
228 int rc;
229
230 /* wrap to 1, skip reserved lkey */
231 rc = ionic_resid_get_shared(&dev->inuse_mrid, 1,
232 dev->inuse_mrid.inuse_size);
233 if (rc < 0)
234 return rc;
235
236 *mrid = ionic_mrid(rc, dev->next_mrkey++);
237 return 0;
238}
239
240static int ionic_get_gsi_qpid(struct ionic_ibdev *dev, u32 *qpid)
241{
242 int rc = 0;
243
244 rc = ionic_resid_get_shared(&dev->inuse_qpid, IB_QPT_GSI, IB_QPT_GSI + 1);
245 if (rc < 0)
246 return rc;
247
248 *qpid = IB_QPT_GSI;
249 return 0;
250}
251
252static int ionic_get_qpid(struct ionic_ibdev *dev, u32 *qpid,
253 u8 *udma_idx, u8 udma_mask)
254{
255 unsigned int size, base, bound;
256 int udma_i, udma_x, udma_ix;
257 int rc = -EINVAL;
258
259 udma_x = dev->next_qpid_udma_idx;
260
261 dev->next_qpid_udma_idx ^= dev->lif_cfg.udma_count - 1;
262
263 for (udma_i = 0; udma_i < dev->lif_cfg.udma_count; ++udma_i) {
264 udma_ix = udma_i ^ udma_x;
265
266 if (!(udma_mask & BIT(udma_ix)))
267 continue;
268
269 size = dev->lif_cfg.qp_count / dev->lif_cfg.udma_count;
270 base = size * udma_ix;
271 bound = base + size;
272
273 /* skip reserved SMI and GSI qpids in group zero */
274 if (!base)
275 base = 2;
276
277 rc = ionic_resid_get_shared(&dev->inuse_qpid, base, bound);
278 if (rc >= 0) {
279 *qpid = ionic_bitid_to_qid(rc,
280 dev->lif_cfg.udma_qgrp_shift,
281 dev->half_qpid_udma_shift);
282 *udma_idx = udma_ix;
283
284 rc = 0;
285 break;
286 }
287 }
288
289 return rc;
290}
291
292static int ionic_get_dbid(struct ionic_ibdev *dev, u32 *dbid, phys_addr_t *addr)
293{
294 int rc, dbpage_num;
295
296 /* wrap to 1, skip kernel reserved */
297 rc = ionic_resid_get_shared(&dev->inuse_dbid, 1,
298 dev->inuse_dbid.inuse_size);
299 if (rc < 0)
300 return rc;
301
302 dbpage_num = (dev->lif_cfg.lif_hw_index * dev->lif_cfg.dbid_count) + rc;
303 *addr = dev->lif_cfg.db_phys + ((phys_addr_t)dbpage_num << PAGE_SHIFT);
304
305 *dbid = rc;
306
307 return 0;
308}
309
310static void ionic_put_pdid(struct ionic_ibdev *dev, u32 pdid)
311{
312 ionic_resid_put(&dev->inuse_pdid, pdid);
313}
314
315static void ionic_put_ahid(struct ionic_ibdev *dev, u32 ahid)
316{
317 ionic_resid_put(&dev->inuse_ahid, ahid);
318}
319
320static void ionic_put_mrid(struct ionic_ibdev *dev, u32 mrid)
321{
322 ionic_resid_put(&dev->inuse_mrid, ionic_mrid_index(mrid));
323}
324
325static void ionic_put_qpid(struct ionic_ibdev *dev, u32 qpid)
326{
327 u32 bitid = ionic_qid_to_bitid(qpid,
328 dev->lif_cfg.udma_qgrp_shift,
329 dev->half_qpid_udma_shift);
330
331 ionic_resid_put(&dev->inuse_qpid, bitid);
332}
333
334static void ionic_put_dbid(struct ionic_ibdev *dev, u32 dbid)
335{
336 ionic_resid_put(&dev->inuse_dbid, dbid);
337}
338
339static struct rdma_user_mmap_entry*
340ionic_mmap_entry_insert(struct ionic_ctx *ctx, unsigned long size,
341 unsigned long pfn, u8 mmap_flags, u64 *offset)
342{
343 struct ionic_mmap_entry *entry;
344 int rc;
345
346 entry = kzalloc_obj(*entry);
347 if (!entry)
348 return NULL;
349
350 entry->size = size;
351 entry->pfn = pfn;
352 entry->mmap_flags = mmap_flags;
353
354 rc = rdma_user_mmap_entry_insert(&ctx->ibctx, &entry->rdma_entry,
355 entry->size);
356 if (rc) {
357 kfree(entry);
358 return NULL;
359 }
360
361 if (offset)
362 *offset = rdma_user_mmap_get_offset(&entry->rdma_entry);
363
364 return &entry->rdma_entry;
365}
366
367int ionic_alloc_ucontext(struct ib_ucontext *ibctx, struct ib_udata *udata)
368{
369 struct ionic_ibdev *dev = to_ionic_ibdev(ibctx->device);
370 struct ionic_ctx *ctx = to_ionic_ctx(ibctx);
371 struct ionic_ctx_resp resp = {};
372 struct ionic_ctx_req req;
373 phys_addr_t db_phys = 0;
374 int rc;
375
376 rc = ib_copy_validate_udata_in(udata, req, rsvd);
377 if (rc)
378 return rc;
379
380 /* try to allocate dbid for user ctx */
381 rc = ionic_get_dbid(dev, &ctx->dbid, &db_phys);
382 if (rc < 0)
383 return rc;
384
385 ibdev_dbg(&dev->ibdev, "user space dbid %u\n", ctx->dbid);
386
387 ctx->mmap_dbell = ionic_mmap_entry_insert(ctx, PAGE_SIZE,
388 PHYS_PFN(db_phys), 0, NULL);
389 if (!ctx->mmap_dbell) {
390 rc = -ENOMEM;
391 goto err_mmap_dbell;
392 }
393
394 if (dev->lif_cfg.phc_state) {
395 ctx->mmap_phc = ionic_mmap_entry_insert(ctx, PAGE_SIZE, 0,
396 IONIC_MMAP_PHC,
397 &resp.phc_offset);
398 if (!ctx->mmap_phc) {
399 rc = -ENOMEM;
400 goto err_mmap_phc;
401 }
402 }
403
404 resp.page_shift = PAGE_SHIFT;
405
406 resp.dbell_offset = db_phys & ~PAGE_MASK;
407
408 resp.version = dev->lif_cfg.rdma_version;
409 resp.qp_opcodes = dev->lif_cfg.qp_opcodes;
410 resp.admin_opcodes = dev->lif_cfg.admin_opcodes;
411
412 resp.sq_qtype = dev->lif_cfg.sq_qtype;
413 resp.rq_qtype = dev->lif_cfg.rq_qtype;
414 resp.cq_qtype = dev->lif_cfg.cq_qtype;
415 resp.admin_qtype = dev->lif_cfg.aq_qtype;
416 resp.max_stride = dev->lif_cfg.max_stride;
417 resp.max_spec = IONIC_SPEC_HIGH;
418
419 resp.udma_count = dev->lif_cfg.udma_count;
420 resp.expdb_mask = dev->lif_cfg.expdb_mask;
421
422 if (dev->lif_cfg.sq_expdb)
423 resp.expdb_qtypes |= IONIC_EXPDB_SQ;
424 if (dev->lif_cfg.rq_expdb)
425 resp.expdb_qtypes |= IONIC_EXPDB_RQ;
426
427 rc = ib_respond_udata(udata, resp);
428 if (rc)
429 goto err_resp;
430
431 return 0;
432
433err_resp:
434 rdma_user_mmap_entry_remove(ctx->mmap_phc);
435err_mmap_phc:
436 rdma_user_mmap_entry_remove(ctx->mmap_dbell);
437err_mmap_dbell:
438 ionic_put_dbid(dev, ctx->dbid);
439
440 return rc;
441}
442
443void ionic_dealloc_ucontext(struct ib_ucontext *ibctx)
444{
445 struct ionic_ibdev *dev = to_ionic_ibdev(ibctx->device);
446 struct ionic_ctx *ctx = to_ionic_ctx(ibctx);
447
448 rdma_user_mmap_entry_remove(ctx->mmap_phc);
449 rdma_user_mmap_entry_remove(ctx->mmap_dbell);
450 ionic_put_dbid(dev, ctx->dbid);
451}
452
453static int ionic_mmap_phc_state(struct ionic_ibdev *dev,
454 struct vm_area_struct *vma)
455{
456 if (!(vma->vm_flags & VM_SHARED))
457 return -EINVAL;
458
459 if (vma->vm_flags & (VM_WRITE | VM_EXEC))
460 return -EPERM;
461
462 vm_flags_clear(vma, VM_MAYWRITE);
463
464 return vm_insert_page(vma, vma->vm_start,
465 virt_to_page(dev->lif_cfg.phc_state));
466}
467
468int ionic_mmap(struct ib_ucontext *ibctx, struct vm_area_struct *vma)
469{
470 struct ionic_ibdev *dev = to_ionic_ibdev(ibctx->device);
471 struct ionic_ctx *ctx = to_ionic_ctx(ibctx);
472 struct rdma_user_mmap_entry *rdma_entry;
473 struct ionic_mmap_entry *ionic_entry;
474 int rc = 0;
475
476 rdma_entry = rdma_user_mmap_entry_get(&ctx->ibctx, vma);
477 if (!rdma_entry) {
478 ibdev_dbg(&dev->ibdev, "not found %#lx\n",
479 vma->vm_pgoff << PAGE_SHIFT);
480 return -EINVAL;
481 }
482
483 ionic_entry = container_of(rdma_entry, struct ionic_mmap_entry,
484 rdma_entry);
485
486 if (ionic_entry->mmap_flags & IONIC_MMAP_PHC) {
487 rc = ionic_mmap_phc_state(dev, vma);
488 rdma_user_mmap_entry_put(rdma_entry);
489 return rc;
490 }
491
492 ibdev_dbg(&dev->ibdev, "writecombine? %d\n",
493 ionic_entry->mmap_flags & IONIC_MMAP_WC);
494 if (ionic_entry->mmap_flags & IONIC_MMAP_WC)
495 vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot);
496 else
497 vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
498
499 ibdev_dbg(&dev->ibdev, "remap st %#lx pf %#lx sz %#lx\n",
500 vma->vm_start, ionic_entry->pfn, ionic_entry->size);
501 rc = rdma_user_mmap_io(&ctx->ibctx, vma, ionic_entry->pfn,
502 ionic_entry->size, vma->vm_page_prot,
503 rdma_entry);
504 if (rc)
505 ibdev_dbg(&dev->ibdev, "remap failed %d\n", rc);
506
507 rdma_user_mmap_entry_put(rdma_entry);
508 return rc;
509}
510
511void ionic_mmap_free(struct rdma_user_mmap_entry *rdma_entry)
512{
513 struct ionic_mmap_entry *ionic_entry;
514
515 ionic_entry = container_of(rdma_entry, struct ionic_mmap_entry,
516 rdma_entry);
517 kfree(ionic_entry);
518}
519
520int ionic_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
521{
522 struct ionic_ibdev *dev = to_ionic_ibdev(ibpd->device);
523 struct ionic_pd *pd = to_ionic_pd(ibpd);
524 int rc;
525
526 rc = ib_no_udata_io(udata);
527 if (rc)
528 return rc;
529
530 return ionic_get_pdid(dev, &pd->pdid);
531}
532
533int ionic_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
534{
535 struct ionic_ibdev *dev = to_ionic_ibdev(ibpd->device);
536 struct ionic_pd *pd = to_ionic_pd(ibpd);
537 int rc;
538
539 rc = ib_no_udata_io(udata);
540 if (rc)
541 return rc;
542
543 ionic_put_pdid(dev, pd->pdid);
544
545 return 0;
546}
547
548static int ionic_build_hdr(struct ionic_ibdev *dev,
549 struct ib_ud_header *hdr,
550 const struct rdma_ah_attr *attr,
551 u16 sport, bool want_ecn)
552{
553 const struct ib_global_route *grh;
554 enum rdma_network_type net;
555 u8 smac[ETH_ALEN];
556 u16 vlan;
557 int rc;
558
559 if (attr->ah_flags != IB_AH_GRH)
560 return -EINVAL;
561 if (attr->type != RDMA_AH_ATTR_TYPE_ROCE)
562 return -EINVAL;
563
564 grh = rdma_ah_read_grh(attr);
565
566 rc = rdma_read_gid_l2_fields(grh->sgid_attr, &vlan, smac);
567 if (rc)
568 return rc;
569
570 net = rdma_gid_attr_network_type(grh->sgid_attr);
571
572 rc = ib_ud_header_init(0, /* no payload */
573 0, /* no lrh */
574 1, /* yes eth */
575 vlan != 0xffff,
576 0, /* no grh */
577 net == RDMA_NETWORK_IPV4 ? 4 : 6,
578 1, /* yes udp */
579 0, /* no imm */
580 hdr);
581 if (rc)
582 return rc;
583
584 ether_addr_copy(hdr->eth.smac_h, smac);
585 ether_addr_copy(hdr->eth.dmac_h, attr->roce.dmac);
586
587 if (net == RDMA_NETWORK_IPV4) {
588 hdr->eth.type = cpu_to_be16(ETH_P_IP);
589 hdr->ip4.frag_off = cpu_to_be16(0x4000); /* don't fragment */
590 hdr->ip4.ttl = grh->hop_limit;
591 hdr->ip4.tot_len = cpu_to_be16(0xffff);
592 hdr->ip4.saddr =
593 *(const __be32 *)(grh->sgid_attr->gid.raw + 12);
594 hdr->ip4.daddr = *(const __be32 *)(grh->dgid.raw + 12);
595
596 if (want_ecn)
597 hdr->ip4.tos = ionic_set_ecn(grh->traffic_class);
598 else
599 hdr->ip4.tos = ionic_clear_ecn(grh->traffic_class);
600 } else {
601 hdr->eth.type = cpu_to_be16(ETH_P_IPV6);
602 hdr->grh.flow_label = cpu_to_be32(grh->flow_label);
603 hdr->grh.hop_limit = grh->hop_limit;
604 hdr->grh.source_gid = grh->sgid_attr->gid;
605 hdr->grh.destination_gid = grh->dgid;
606
607 if (want_ecn)
608 hdr->grh.traffic_class =
609 ionic_set_ecn(grh->traffic_class);
610 else
611 hdr->grh.traffic_class =
612 ionic_clear_ecn(grh->traffic_class);
613 }
614
615 if (vlan != 0xffff) {
616 vlan |= rdma_ah_get_sl(attr) << VLAN_PRIO_SHIFT;
617 hdr->vlan.tag = cpu_to_be16(vlan);
618 hdr->vlan.type = hdr->eth.type;
619 hdr->eth.type = cpu_to_be16(ETH_P_8021Q);
620 }
621
622 hdr->udp.sport = cpu_to_be16(sport);
623 hdr->udp.dport = cpu_to_be16(ROCE_V2_UDP_DPORT);
624
625 return 0;
626}
627
628static void ionic_set_ah_attr(struct ionic_ibdev *dev,
629 struct rdma_ah_attr *ah_attr,
630 struct ib_ud_header *hdr,
631 int sgid_index)
632{
633 u32 flow_label;
634 u16 vlan = 0;
635 u8 tos, ttl;
636
637 if (hdr->vlan_present)
638 vlan = be16_to_cpu(hdr->vlan.tag);
639
640 if (hdr->ipv4_present) {
641 flow_label = 0;
642 ttl = hdr->ip4.ttl;
643 tos = hdr->ip4.tos;
644 *(__be16 *)(hdr->grh.destination_gid.raw + 10) = cpu_to_be16(0xffff);
645 *(__be32 *)(hdr->grh.destination_gid.raw + 12) = hdr->ip4.daddr;
646 } else {
647 flow_label = be32_to_cpu(hdr->grh.flow_label);
648 ttl = hdr->grh.hop_limit;
649 tos = hdr->grh.traffic_class;
650 }
651
652 memset(ah_attr, 0, sizeof(*ah_attr));
653 ah_attr->type = RDMA_AH_ATTR_TYPE_ROCE;
654 if (hdr->eth_present)
655 ether_addr_copy(ah_attr->roce.dmac, hdr->eth.dmac_h);
656 rdma_ah_set_sl(ah_attr, vlan >> VLAN_PRIO_SHIFT);
657 rdma_ah_set_port_num(ah_attr, 1);
658 rdma_ah_set_grh(ah_attr, NULL, flow_label, sgid_index, ttl, tos);
659 rdma_ah_set_dgid_raw(ah_attr, &hdr->grh.destination_gid);
660}
661
662static int ionic_create_ah_cmd(struct ionic_ibdev *dev,
663 struct ionic_ah *ah,
664 struct ionic_pd *pd,
665 struct rdma_ah_attr *attr,
666 u32 flags)
667{
668 struct ionic_admin_wr wr = {
669 .work = COMPLETION_INITIALIZER_ONSTACK(wr.work),
670 .wqe = {
671 .op = IONIC_V1_ADMIN_CREATE_AH,
672 .len = cpu_to_le16(IONIC_ADMIN_CREATE_AH_IN_V1_LEN),
673 .cmd.create_ah = {
674 .pd_id = cpu_to_le32(pd->pdid),
675 .dbid_flags = cpu_to_le16(dev->lif_cfg.dbid),
676 .id_ver = cpu_to_le32(ah->ahid),
677 }
678 }
679 };
680 enum ionic_admin_flags admin_flags = 0;
681 dma_addr_t hdr_dma = 0;
682 void *hdr_buf;
683 gfp_t gfp = GFP_ATOMIC;
684 int rc, hdr_len = 0;
685
686 if (dev->lif_cfg.admin_opcodes <= IONIC_V1_ADMIN_CREATE_AH)
687 return -EBADRQC;
688
689 if (flags & RDMA_CREATE_AH_SLEEPABLE)
690 gfp = GFP_KERNEL;
691 else
692 admin_flags |= IONIC_ADMIN_F_BUSYWAIT;
693
694 rc = ionic_build_hdr(dev, &ah->hdr, attr, IONIC_ROCE_UDP_SPORT, false);
695 if (rc)
696 return rc;
697
698 if (ah->hdr.eth.type == cpu_to_be16(ETH_P_8021Q)) {
699 if (ah->hdr.vlan.type == cpu_to_be16(ETH_P_IP))
700 wr.wqe.cmd.create_ah.csum_profile =
701 IONIC_TFP_CSUM_PROF_ETH_QTAG_IPV4_UDP;
702 else
703 wr.wqe.cmd.create_ah.csum_profile =
704 IONIC_TFP_CSUM_PROF_ETH_QTAG_IPV6_UDP;
705 } else {
706 if (ah->hdr.eth.type == cpu_to_be16(ETH_P_IP))
707 wr.wqe.cmd.create_ah.csum_profile =
708 IONIC_TFP_CSUM_PROF_ETH_IPV4_UDP;
709 else
710 wr.wqe.cmd.create_ah.csum_profile =
711 IONIC_TFP_CSUM_PROF_ETH_IPV6_UDP;
712 }
713
714 ah->sgid_index = rdma_ah_read_grh(attr)->sgid_index;
715
716 hdr_buf = kmalloc(PAGE_SIZE, gfp);
717 if (!hdr_buf)
718 return -ENOMEM;
719
720 hdr_len = ib_ud_header_pack(&ah->hdr, hdr_buf);
721 hdr_len -= IB_BTH_BYTES;
722 hdr_len -= IB_DETH_BYTES;
723 ibdev_dbg(&dev->ibdev, "roce packet header template\n");
724 print_hex_dump_debug("hdr ", DUMP_PREFIX_OFFSET, 16, 1,
725 hdr_buf, hdr_len, true);
726
727 hdr_dma = dma_map_single(dev->lif_cfg.hwdev, hdr_buf, hdr_len,
728 DMA_TO_DEVICE);
729
730 rc = dma_mapping_error(dev->lif_cfg.hwdev, hdr_dma);
731 if (rc)
732 goto err_dma;
733
734 wr.wqe.cmd.create_ah.dma_addr = cpu_to_le64(hdr_dma);
735 wr.wqe.cmd.create_ah.length = cpu_to_le32(hdr_len);
736
737 ionic_admin_post(dev, &wr);
738 rc = ionic_admin_wait(dev, &wr, admin_flags);
739
740 dma_unmap_single(dev->lif_cfg.hwdev, hdr_dma, hdr_len,
741 DMA_TO_DEVICE);
742err_dma:
743 kfree(hdr_buf);
744
745 return rc;
746}
747
748static int ionic_destroy_ah_cmd(struct ionic_ibdev *dev, u32 ahid, u32 flags)
749{
750 struct ionic_admin_wr wr = {
751 .work = COMPLETION_INITIALIZER_ONSTACK(wr.work),
752 .wqe = {
753 .op = IONIC_V1_ADMIN_DESTROY_AH,
754 .len = cpu_to_le16(IONIC_ADMIN_DESTROY_AH_IN_V1_LEN),
755 .cmd.destroy_ah = {
756 .ah_id = cpu_to_le32(ahid),
757 },
758 }
759 };
760 enum ionic_admin_flags admin_flags = IONIC_ADMIN_F_TEARDOWN;
761
762 if (dev->lif_cfg.admin_opcodes <= IONIC_V1_ADMIN_DESTROY_AH)
763 return -EBADRQC;
764
765 if (!(flags & RDMA_CREATE_AH_SLEEPABLE))
766 admin_flags |= IONIC_ADMIN_F_BUSYWAIT;
767
768 ionic_admin_post(dev, &wr);
769 ionic_admin_wait(dev, &wr, admin_flags);
770
771 /* No host-memory resource is associated with ah, so it is ok
772 * to "succeed" and complete this destroy ah on the host.
773 */
774 return 0;
775}
776
777int ionic_create_ah(struct ib_ah *ibah, struct rdma_ah_init_attr *init_attr,
778 struct ib_udata *udata)
779{
780 struct ionic_ibdev *dev = to_ionic_ibdev(ibah->device);
781 struct rdma_ah_attr *attr = init_attr->ah_attr;
782 struct ionic_pd *pd = to_ionic_pd(ibah->pd);
783 struct ionic_ah *ah = to_ionic_ah(ibah);
784 struct ionic_ah_resp resp = {};
785 u32 flags = init_attr->flags;
786 int rc;
787
788 rc = ib_is_udata_in_empty(udata);
789 if (rc)
790 return rc;
791
792 rc = ionic_get_ahid(dev, &ah->ahid);
793 if (rc)
794 return rc;
795
796 rc = ionic_create_ah_cmd(dev, ah, pd, attr, flags);
797 if (rc)
798 goto err_cmd;
799
800 if (udata) {
801 resp.ahid = ah->ahid;
802
803 rc = ib_respond_udata(udata, resp);
804 if (rc)
805 goto err_resp;
806 }
807
808 return 0;
809
810err_resp:
811 ionic_destroy_ah_cmd(dev, ah->ahid, flags);
812err_cmd:
813 ionic_put_ahid(dev, ah->ahid);
814 return rc;
815}
816
817int ionic_query_ah(struct ib_ah *ibah, struct rdma_ah_attr *ah_attr)
818{
819 struct ionic_ibdev *dev = to_ionic_ibdev(ibah->device);
820 struct ionic_ah *ah = to_ionic_ah(ibah);
821
822 ionic_set_ah_attr(dev, ah_attr, &ah->hdr, ah->sgid_index);
823
824 return 0;
825}
826
827int ionic_destroy_ah(struct ib_ah *ibah, u32 flags)
828{
829 struct ionic_ibdev *dev = to_ionic_ibdev(ibah->device);
830 struct ionic_ah *ah = to_ionic_ah(ibah);
831 int rc;
832
833 rc = ionic_destroy_ah_cmd(dev, ah->ahid, flags);
834 if (rc)
835 return rc;
836
837 ionic_put_ahid(dev, ah->ahid);
838
839 return 0;
840}
841
842static int ionic_create_mr_cmd(struct ionic_ibdev *dev,
843 struct ionic_pd *pd,
844 struct ionic_mr *mr,
845 u64 addr,
846 u64 length)
847{
848 struct ionic_admin_wr wr = {
849 .work = COMPLETION_INITIALIZER_ONSTACK(wr.work),
850 .wqe = {
851 .op = IONIC_V1_ADMIN_CREATE_MR,
852 .len = cpu_to_le16(IONIC_ADMIN_CREATE_MR_IN_V1_LEN),
853 .cmd.create_mr = {
854 .va = cpu_to_le64(addr),
855 .length = cpu_to_le64(length),
856 .pd_id = cpu_to_le32(pd->pdid),
857 .page_size_log2 = mr->buf.page_size_log2,
858 .tbl_index = cpu_to_le32(~0),
859 .map_count = cpu_to_le32(mr->buf.tbl_pages),
860 .dma_addr = ionic_pgtbl_dma(&mr->buf, addr),
861 .dbid_flags = cpu_to_le16(mr->flags),
862 .id_ver = cpu_to_le32(mr->mrid),
863 }
864 }
865 };
866 int rc;
867
868 if (dev->lif_cfg.admin_opcodes <= IONIC_V1_ADMIN_CREATE_MR)
869 return -EBADRQC;
870
871 ionic_admin_post(dev, &wr);
872 rc = ionic_admin_wait(dev, &wr, 0);
873 if (!rc)
874 mr->created = true;
875
876 return rc;
877}
878
879static int ionic_destroy_mr_cmd(struct ionic_ibdev *dev, u32 mrid)
880{
881 struct ionic_admin_wr wr = {
882 .work = COMPLETION_INITIALIZER_ONSTACK(wr.work),
883 .wqe = {
884 .op = IONIC_V1_ADMIN_DESTROY_MR,
885 .len = cpu_to_le16(IONIC_ADMIN_DESTROY_MR_IN_V1_LEN),
886 .cmd.destroy_mr = {
887 .mr_id = cpu_to_le32(mrid),
888 },
889 }
890 };
891
892 if (dev->lif_cfg.admin_opcodes <= IONIC_V1_ADMIN_DESTROY_MR)
893 return -EBADRQC;
894
895 ionic_admin_post(dev, &wr);
896
897 return ionic_admin_wait(dev, &wr, IONIC_ADMIN_F_TEARDOWN);
898}
899
900struct ib_mr *ionic_get_dma_mr(struct ib_pd *ibpd, int access)
901{
902 struct ionic_pd *pd = to_ionic_pd(ibpd);
903 struct ionic_mr *mr;
904
905 mr = kzalloc_obj(*mr);
906 if (!mr)
907 return ERR_PTR(-ENOMEM);
908
909 mr->ibmr.lkey = IONIC_DMA_LKEY;
910 mr->ibmr.rkey = IONIC_DMA_RKEY;
911
912 if (pd)
913 pd->flags |= IONIC_QPF_PRIVILEGED;
914
915 return &mr->ibmr;
916}
917
918struct ib_mr *ionic_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 length,
919 u64 addr, int access, struct ib_dmah *dmah,
920 struct ib_udata *udata)
921{
922 struct ionic_ibdev *dev = to_ionic_ibdev(ibpd->device);
923 struct ionic_pd *pd = to_ionic_pd(ibpd);
924 struct ionic_mr *mr;
925 unsigned long pg_sz;
926 int rc;
927
928 rc = ib_no_udata_io(udata);
929 if (rc)
930 return ERR_PTR(rc);
931
932 if (dmah)
933 return ERR_PTR(-EOPNOTSUPP);
934
935 mr = kzalloc_obj(*mr);
936 if (!mr)
937 return ERR_PTR(-ENOMEM);
938
939 rc = ionic_get_mrid(dev, &mr->mrid);
940 if (rc)
941 goto err_mrid;
942
943 mr->ibmr.lkey = mr->mrid;
944 mr->ibmr.rkey = mr->mrid;
945 mr->ibmr.iova = addr;
946 mr->ibmr.length = length;
947
948 mr->flags = IONIC_MRF_USER_MR | to_ionic_mr_flags(access);
949
950 mr->umem = ib_umem_get_va(&dev->ibdev, start, length, access);
951 if (IS_ERR(mr->umem)) {
952 rc = PTR_ERR(mr->umem);
953 goto err_umem;
954 }
955
956 pg_sz = ib_umem_find_best_pgsz(mr->umem,
957 dev->lif_cfg.page_size_supported,
958 addr);
959 if (!pg_sz) {
960 rc = -EINVAL;
961 goto err_pgtbl;
962 }
963
964 rc = ionic_pgtbl_init(dev, &mr->buf, mr->umem, 0, 1, pg_sz);
965 if (rc)
966 goto err_pgtbl;
967
968 rc = ionic_create_mr_cmd(dev, pd, mr, addr, length);
969 if (rc)
970 goto err_cmd;
971
972 ionic_pgtbl_unbuf(dev, &mr->buf);
973
974 return &mr->ibmr;
975
976err_cmd:
977 ionic_pgtbl_unbuf(dev, &mr->buf);
978err_pgtbl:
979 ib_umem_release(mr->umem);
980err_umem:
981 ionic_put_mrid(dev, mr->mrid);
982err_mrid:
983 kfree(mr);
984 return ERR_PTR(rc);
985}
986
987struct ib_mr *ionic_reg_user_mr_dmabuf(struct ib_pd *ibpd, u64 offset,
988 u64 length, u64 addr, int fd, int access,
989 struct ib_dmah *dmah,
990 struct uverbs_attr_bundle *attrs)
991{
992 struct ionic_ibdev *dev = to_ionic_ibdev(ibpd->device);
993 struct ionic_pd *pd = to_ionic_pd(ibpd);
994 struct ib_umem_dmabuf *umem_dmabuf;
995 struct ionic_mr *mr;
996 u64 pg_sz;
997 int rc;
998
999 if (dmah)
1000 return ERR_PTR(-EOPNOTSUPP);
1001
1002 mr = kzalloc_obj(*mr);
1003 if (!mr)
1004 return ERR_PTR(-ENOMEM);
1005
1006 rc = ionic_get_mrid(dev, &mr->mrid);
1007 if (rc)
1008 goto err_mrid;
1009
1010 mr->ibmr.lkey = mr->mrid;
1011 mr->ibmr.rkey = mr->mrid;
1012 mr->ibmr.iova = addr;
1013 mr->ibmr.length = length;
1014
1015 mr->flags = IONIC_MRF_USER_MR | to_ionic_mr_flags(access);
1016
1017 umem_dmabuf = ib_umem_dmabuf_get_pinned(&dev->ibdev, offset, length,
1018 fd, access);
1019 if (IS_ERR(umem_dmabuf)) {
1020 rc = PTR_ERR(umem_dmabuf);
1021 goto err_umem;
1022 }
1023
1024 mr->umem = &umem_dmabuf->umem;
1025
1026 pg_sz = ib_umem_find_best_pgsz(mr->umem,
1027 dev->lif_cfg.page_size_supported,
1028 addr);
1029 if (!pg_sz) {
1030 rc = -EINVAL;
1031 goto err_pgtbl;
1032 }
1033
1034 rc = ionic_pgtbl_init(dev, &mr->buf, mr->umem, 0, 1, pg_sz);
1035 if (rc)
1036 goto err_pgtbl;
1037
1038 rc = ionic_create_mr_cmd(dev, pd, mr, addr, length);
1039 if (rc)
1040 goto err_cmd;
1041
1042 ionic_pgtbl_unbuf(dev, &mr->buf);
1043
1044 return &mr->ibmr;
1045
1046err_cmd:
1047 ionic_pgtbl_unbuf(dev, &mr->buf);
1048err_pgtbl:
1049 ib_umem_release(mr->umem);
1050err_umem:
1051 ionic_put_mrid(dev, mr->mrid);
1052err_mrid:
1053 kfree(mr);
1054 return ERR_PTR(rc);
1055}
1056
1057int ionic_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata)
1058{
1059 struct ionic_ibdev *dev = to_ionic_ibdev(ibmr->device);
1060 struct ionic_mr *mr = to_ionic_mr(ibmr);
1061 int rc;
1062
1063 rc = ib_no_udata_io(udata);
1064 if (rc)
1065 return rc;
1066
1067 if (!mr->ibmr.lkey)
1068 goto out;
1069
1070 if (mr->created) {
1071 rc = ionic_destroy_mr_cmd(dev, mr->mrid);
1072 if (rc)
1073 return rc;
1074 }
1075
1076 ionic_pgtbl_unbuf(dev, &mr->buf);
1077
1078 if (mr->umem)
1079 ib_umem_release(mr->umem);
1080
1081 ionic_put_mrid(dev, mr->mrid);
1082
1083out:
1084 kfree(mr);
1085
1086 return 0;
1087}
1088
1089struct ib_mr *ionic_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type type,
1090 u32 max_sg)
1091{
1092 struct ionic_ibdev *dev = to_ionic_ibdev(ibpd->device);
1093 struct ionic_pd *pd = to_ionic_pd(ibpd);
1094 struct ionic_mr *mr;
1095 int rc;
1096
1097 if (type != IB_MR_TYPE_MEM_REG)
1098 return ERR_PTR(-EINVAL);
1099
1100 mr = kzalloc_obj(*mr);
1101 if (!mr)
1102 return ERR_PTR(-ENOMEM);
1103
1104 rc = ionic_get_mrid(dev, &mr->mrid);
1105 if (rc)
1106 goto err_mrid;
1107
1108 mr->ibmr.lkey = mr->mrid;
1109 mr->ibmr.rkey = mr->mrid;
1110
1111 mr->flags = IONIC_MRF_PHYS_MR;
1112
1113 rc = ionic_pgtbl_init(dev, &mr->buf, mr->umem, 0, max_sg, PAGE_SIZE);
1114 if (rc)
1115 goto err_pgtbl;
1116
1117 mr->buf.tbl_pages = 0;
1118
1119 rc = ionic_create_mr_cmd(dev, pd, mr, 0, 0);
1120 if (rc)
1121 goto err_cmd;
1122
1123 return &mr->ibmr;
1124
1125err_cmd:
1126 ionic_pgtbl_unbuf(dev, &mr->buf);
1127err_pgtbl:
1128 ionic_put_mrid(dev, mr->mrid);
1129err_mrid:
1130 kfree(mr);
1131 return ERR_PTR(rc);
1132}
1133
1134static int ionic_map_mr_page(struct ib_mr *ibmr, u64 dma)
1135{
1136 struct ionic_ibdev *dev = to_ionic_ibdev(ibmr->device);
1137 struct ionic_mr *mr = to_ionic_mr(ibmr);
1138
1139 ibdev_dbg(&dev->ibdev, "dma %p\n", (void *)dma);
1140 return ionic_pgtbl_page(&mr->buf, dma);
1141}
1142
1143int ionic_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents,
1144 unsigned int *sg_offset)
1145{
1146 struct ionic_ibdev *dev = to_ionic_ibdev(ibmr->device);
1147 struct ionic_mr *mr = to_ionic_mr(ibmr);
1148 int rc;
1149
1150 /* mr must be allocated using ib_alloc_mr() */
1151 if (unlikely(!mr->buf.tbl_limit))
1152 return -EINVAL;
1153
1154 mr->buf.tbl_pages = 0;
1155
1156 if (mr->buf.tbl_buf)
1157 dma_sync_single_for_cpu(dev->lif_cfg.hwdev, mr->buf.tbl_dma,
1158 mr->buf.tbl_size, DMA_TO_DEVICE);
1159
1160 ibdev_dbg(&dev->ibdev, "sg %p nent %d\n", sg, sg_nents);
1161 rc = ib_sg_to_pages(ibmr, sg, sg_nents, sg_offset, ionic_map_mr_page);
1162
1163 mr->buf.page_size_log2 = order_base_2(ibmr->page_size);
1164
1165 if (mr->buf.tbl_buf)
1166 dma_sync_single_for_device(dev->lif_cfg.hwdev, mr->buf.tbl_dma,
1167 mr->buf.tbl_size, DMA_TO_DEVICE);
1168
1169 return rc;
1170}
1171
1172int ionic_alloc_mw(struct ib_mw *ibmw, struct ib_udata *udata)
1173{
1174 struct ionic_ibdev *dev = to_ionic_ibdev(ibmw->device);
1175 struct ionic_pd *pd = to_ionic_pd(ibmw->pd);
1176 struct ionic_mr *mr = to_ionic_mw(ibmw);
1177 int rc;
1178
1179 rc = ib_no_udata_io(udata);
1180 if (rc)
1181 return rc;
1182
1183 rc = ionic_get_mrid(dev, &mr->mrid);
1184 if (rc)
1185 return rc;
1186
1187 mr->ibmw.rkey = mr->mrid;
1188
1189 if (mr->ibmw.type == IB_MW_TYPE_1)
1190 mr->flags = IONIC_MRF_MW_1;
1191 else
1192 mr->flags = IONIC_MRF_MW_2;
1193
1194 rc = ionic_create_mr_cmd(dev, pd, mr, 0, 0);
1195 if (rc)
1196 goto err_cmd;
1197
1198 return 0;
1199
1200err_cmd:
1201 ionic_put_mrid(dev, mr->mrid);
1202 return rc;
1203}
1204
1205int ionic_dealloc_mw(struct ib_mw *ibmw)
1206{
1207 struct ionic_ibdev *dev = to_ionic_ibdev(ibmw->device);
1208 struct ionic_mr *mr = to_ionic_mw(ibmw);
1209 int rc;
1210
1211 rc = ionic_destroy_mr_cmd(dev, mr->mrid);
1212 if (rc)
1213 return rc;
1214
1215 ionic_put_mrid(dev, mr->mrid);
1216
1217 return 0;
1218}
1219
1220static int ionic_create_cq_cmd(struct ionic_ibdev *dev,
1221 struct ionic_ctx *ctx,
1222 struct ionic_cq *cq,
1223 struct ionic_tbl_buf *buf)
1224{
1225 const u16 dbid = ionic_ctx_dbid(dev, ctx);
1226 struct ionic_admin_wr wr = {
1227 .work = COMPLETION_INITIALIZER_ONSTACK(wr.work),
1228 .wqe = {
1229 .op = IONIC_V1_ADMIN_CREATE_CQ,
1230 .len = cpu_to_le16(IONIC_ADMIN_CREATE_CQ_IN_V1_LEN),
1231 .cmd.create_cq = {
1232 .eq_id = cpu_to_le32(cq->eqid),
1233 .depth_log2 = cq->q.depth_log2,
1234 .stride_log2 = cq->q.stride_log2,
1235 .page_size_log2 = buf->page_size_log2,
1236 .tbl_index = cpu_to_le32(~0),
1237 .map_count = cpu_to_le32(buf->tbl_pages),
1238 .dma_addr = ionic_pgtbl_dma(buf, 0),
1239 .dbid_flags = cpu_to_le16(dbid),
1240 .id_ver = cpu_to_le32(cq->cqid),
1241 }
1242 }
1243 };
1244
1245 if (dev->lif_cfg.admin_opcodes <= IONIC_V1_ADMIN_CREATE_CQ)
1246 return -EBADRQC;
1247
1248 ionic_admin_post(dev, &wr);
1249
1250 return ionic_admin_wait(dev, &wr, 0);
1251}
1252
1253static int ionic_destroy_cq_cmd(struct ionic_ibdev *dev, u32 cqid)
1254{
1255 struct ionic_admin_wr wr = {
1256 .work = COMPLETION_INITIALIZER_ONSTACK(wr.work),
1257 .wqe = {
1258 .op = IONIC_V1_ADMIN_DESTROY_CQ,
1259 .len = cpu_to_le16(IONIC_ADMIN_DESTROY_CQ_IN_V1_LEN),
1260 .cmd.destroy_cq = {
1261 .cq_id = cpu_to_le32(cqid),
1262 },
1263 }
1264 };
1265
1266 if (dev->lif_cfg.admin_opcodes <= IONIC_V1_ADMIN_DESTROY_CQ)
1267 return -EBADRQC;
1268
1269 ionic_admin_post(dev, &wr);
1270
1271 return ionic_admin_wait(dev, &wr, IONIC_ADMIN_F_TEARDOWN);
1272}
1273
1274int ionic_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
1275 struct uverbs_attr_bundle *attrs)
1276{
1277 struct ionic_ibdev *dev = to_ionic_ibdev(ibcq->device);
1278 struct ib_udata *udata = &attrs->driver_udata;
1279 struct ionic_ctx *ctx =
1280 rdma_udata_to_drv_context(udata, struct ionic_ctx, ibctx);
1281 struct ionic_vcq *vcq = to_ionic_vcq(ibcq);
1282 struct ionic_tbl_buf buf = {};
1283 struct ionic_cq_resp resp = {};
1284 struct ionic_cq_req req;
1285 int udma_idx = 0, rc;
1286
1287 if (udata) {
1288 rc = ib_copy_validate_udata_in(udata, req, rsvd);
1289 if (rc)
1290 return rc;
1291 }
1292
1293 vcq->udma_mask = BIT(dev->lif_cfg.udma_count) - 1;
1294
1295 if (udata)
1296 vcq->udma_mask &= req.udma_mask;
1297
1298 if (!vcq->udma_mask) {
1299 rc = -EINVAL;
1300 goto err_init;
1301 }
1302
1303 for (; udma_idx < dev->lif_cfg.udma_count; ++udma_idx) {
1304 if (!(vcq->udma_mask & BIT(udma_idx)))
1305 continue;
1306
1307 rc = ionic_create_cq_common(vcq, &buf, attr, ctx, udata,
1308 &req.cq[udma_idx],
1309 &resp.cqid[udma_idx],
1310 udma_idx);
1311 if (rc)
1312 goto err_init;
1313
1314 rc = ionic_create_cq_cmd(dev, ctx, &vcq->cq[udma_idx], &buf);
1315 if (rc)
1316 goto err_cmd;
1317
1318 ionic_pgtbl_unbuf(dev, &buf);
1319 }
1320
1321 vcq->ibcq.cqe = attr->cqe;
1322
1323 if (udata) {
1324 resp.udma_mask = vcq->udma_mask;
1325
1326 rc = ib_respond_udata(udata, resp);
1327 if (rc)
1328 goto err_resp;
1329 }
1330
1331 return 0;
1332
1333err_resp:
1334 while (udma_idx) {
1335 --udma_idx;
1336 if (!(vcq->udma_mask & BIT(udma_idx)))
1337 continue;
1338 ionic_destroy_cq_cmd(dev, vcq->cq[udma_idx].cqid);
1339err_cmd:
1340 ionic_pgtbl_unbuf(dev, &buf);
1341 ionic_destroy_cq_common(dev, &vcq->cq[udma_idx]);
1342err_init:
1343 ;
1344 }
1345
1346 return rc;
1347}
1348
1349int ionic_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata)
1350{
1351 struct ionic_ibdev *dev = to_ionic_ibdev(ibcq->device);
1352 struct ionic_vcq *vcq = to_ionic_vcq(ibcq);
1353 int udma_idx, rc_tmp, rc = 0;
1354
1355 rc = ib_no_udata_io(udata);
1356 if (rc)
1357 return rc;
1358
1359 for (udma_idx = dev->lif_cfg.udma_count; udma_idx; ) {
1360 --udma_idx;
1361
1362 if (!(vcq->udma_mask & BIT(udma_idx)))
1363 continue;
1364
1365 rc_tmp = ionic_destroy_cq_cmd(dev, vcq->cq[udma_idx].cqid);
1366 if (rc_tmp) {
1367 if (!rc)
1368 rc = rc_tmp;
1369
1370 continue;
1371 }
1372
1373 ionic_destroy_cq_common(dev, &vcq->cq[udma_idx]);
1374 }
1375
1376 return rc;
1377}
1378
1379static bool pd_remote_privileged(struct ib_pd *pd)
1380{
1381 return pd->flags & IB_PD_UNSAFE_GLOBAL_RKEY;
1382}
1383
1384static int ionic_create_qp_cmd(struct ionic_ibdev *dev,
1385 struct ionic_pd *pd,
1386 struct ionic_cq *send_cq,
1387 struct ionic_cq *recv_cq,
1388 struct ionic_qp *qp,
1389 struct ionic_tbl_buf *sq_buf,
1390 struct ionic_tbl_buf *rq_buf,
1391 struct ib_qp_init_attr *attr)
1392{
1393 const u16 dbid = ionic_obj_dbid(dev, pd->ibpd.uobject);
1394 const u32 flags = to_ionic_qp_flags(0, 0,
1395 qp->sq_cmb & IONIC_CMB_ENABLE,
1396 qp->rq_cmb & IONIC_CMB_ENABLE,
1397 qp->sq_spec, qp->rq_spec,
1398 pd->flags & IONIC_QPF_PRIVILEGED,
1399 pd_remote_privileged(&pd->ibpd));
1400 struct ionic_admin_wr wr = {
1401 .work = COMPLETION_INITIALIZER_ONSTACK(wr.work),
1402 .wqe = {
1403 .op = IONIC_V1_ADMIN_CREATE_QP,
1404 .len = cpu_to_le16(IONIC_ADMIN_CREATE_QP_IN_V1_LEN),
1405 .cmd.create_qp = {
1406 .pd_id = cpu_to_le32(pd->pdid),
1407 .priv_flags = cpu_to_be32(flags),
1408 .type_state = to_ionic_qp_type(attr->qp_type),
1409 .dbid_flags = cpu_to_le16(dbid),
1410 .id_ver = cpu_to_le32(qp->qpid),
1411 }
1412 }
1413 };
1414
1415 if (dev->lif_cfg.admin_opcodes <= IONIC_V1_ADMIN_CREATE_QP)
1416 return -EBADRQC;
1417
1418 if (qp->has_sq) {
1419 wr.wqe.cmd.create_qp.sq_cq_id = cpu_to_le32(send_cq->cqid);
1420 wr.wqe.cmd.create_qp.sq_depth_log2 = qp->sq.depth_log2;
1421 wr.wqe.cmd.create_qp.sq_stride_log2 = qp->sq.stride_log2;
1422 wr.wqe.cmd.create_qp.sq_page_size_log2 = sq_buf->page_size_log2;
1423 wr.wqe.cmd.create_qp.sq_tbl_index_xrcd_id = cpu_to_le32(~0);
1424 wr.wqe.cmd.create_qp.sq_map_count =
1425 cpu_to_le32(sq_buf->tbl_pages);
1426 wr.wqe.cmd.create_qp.sq_dma_addr = ionic_pgtbl_dma(sq_buf, 0);
1427 }
1428
1429 if (qp->has_rq) {
1430 wr.wqe.cmd.create_qp.rq_cq_id = cpu_to_le32(recv_cq->cqid);
1431 wr.wqe.cmd.create_qp.rq_depth_log2 = qp->rq.depth_log2;
1432 wr.wqe.cmd.create_qp.rq_stride_log2 = qp->rq.stride_log2;
1433 wr.wqe.cmd.create_qp.rq_page_size_log2 = rq_buf->page_size_log2;
1434 wr.wqe.cmd.create_qp.rq_tbl_index_srq_id = cpu_to_le32(~0);
1435 wr.wqe.cmd.create_qp.rq_map_count =
1436 cpu_to_le32(rq_buf->tbl_pages);
1437 wr.wqe.cmd.create_qp.rq_dma_addr = ionic_pgtbl_dma(rq_buf, 0);
1438 }
1439
1440 ionic_admin_post(dev, &wr);
1441
1442 return ionic_admin_wait(dev, &wr, 0);
1443}
1444
1445static int ionic_modify_qp_cmd(struct ionic_ibdev *dev,
1446 struct ionic_pd *pd,
1447 struct ionic_qp *qp,
1448 struct ib_qp_attr *attr,
1449 int mask)
1450{
1451 const u32 flags = to_ionic_qp_flags(attr->qp_access_flags,
1452 attr->en_sqd_async_notify,
1453 qp->sq_cmb & IONIC_CMB_ENABLE,
1454 qp->rq_cmb & IONIC_CMB_ENABLE,
1455 qp->sq_spec, qp->rq_spec,
1456 pd->flags & IONIC_QPF_PRIVILEGED,
1457 pd_remote_privileged(qp->ibqp.pd));
1458 const u8 state = to_ionic_qp_modify_state(attr->qp_state,
1459 attr->cur_qp_state);
1460 struct ionic_admin_wr wr = {
1461 .work = COMPLETION_INITIALIZER_ONSTACK(wr.work),
1462 .wqe = {
1463 .op = IONIC_V1_ADMIN_MODIFY_QP,
1464 .len = cpu_to_le16(IONIC_ADMIN_MODIFY_QP_IN_V1_LEN),
1465 .cmd.mod_qp = {
1466 .attr_mask = cpu_to_be32(mask),
1467 .access_flags = cpu_to_be16(flags),
1468 .rq_psn = cpu_to_le32(attr->rq_psn),
1469 .sq_psn = cpu_to_le32(attr->sq_psn),
1470 .rate_limit_kbps =
1471 cpu_to_le32(attr->rate_limit),
1472 .pmtu = (attr->path_mtu + 7),
1473 .retry = (attr->retry_cnt |
1474 (attr->rnr_retry << 4)),
1475 .rnr_timer = attr->min_rnr_timer,
1476 .retry_timeout = attr->timeout,
1477 .type_state = state,
1478 .id_ver = cpu_to_le32(qp->qpid),
1479 }
1480 }
1481 };
1482 const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr);
1483 void *hdr_buf = NULL;
1484 dma_addr_t hdr_dma = 0;
1485 int rc, hdr_len = 0;
1486 u16 sport;
1487
1488 if (dev->lif_cfg.admin_opcodes <= IONIC_V1_ADMIN_MODIFY_QP)
1489 return -EBADRQC;
1490
1491 if ((mask & IB_QP_MAX_DEST_RD_ATOMIC) && attr->max_dest_rd_atomic) {
1492 /* Note, round up/down was already done for allocating
1493 * resources on the device. The allocation order is in cache
1494 * line size. We can't use the order of the resource
1495 * allocation to determine the order wqes here, because for
1496 * queue length <= one cache line it is not distinct.
1497 *
1498 * Therefore, order wqes is computed again here.
1499 *
1500 * Account for hole and round up to the next order.
1501 */
1502 wr.wqe.cmd.mod_qp.rsq_depth =
1503 order_base_2(attr->max_dest_rd_atomic + 1);
1504 wr.wqe.cmd.mod_qp.rsq_index = cpu_to_le32(~0);
1505 }
1506
1507 if ((mask & IB_QP_MAX_QP_RD_ATOMIC) && attr->max_rd_atomic) {
1508 /* Account for hole and round down to the next order */
1509 wr.wqe.cmd.mod_qp.rrq_depth =
1510 order_base_2(attr->max_rd_atomic + 2) - 1;
1511 wr.wqe.cmd.mod_qp.rrq_index = cpu_to_le32(~0);
1512 }
1513
1514 if (qp->ibqp.qp_type == IB_QPT_RC || qp->ibqp.qp_type == IB_QPT_UC)
1515 wr.wqe.cmd.mod_qp.qkey_dest_qpn =
1516 cpu_to_le32(attr->dest_qp_num);
1517 else
1518 wr.wqe.cmd.mod_qp.qkey_dest_qpn = cpu_to_le32(attr->qkey);
1519
1520 if (mask & IB_QP_AV) {
1521 if (!qp->hdr)
1522 return -ENOMEM;
1523
1524 sport = rdma_get_udp_sport(grh->flow_label,
1525 qp->qpid,
1526 attr->dest_qp_num);
1527
1528 rc = ionic_build_hdr(dev, qp->hdr, &attr->ah_attr, sport, true);
1529 if (rc)
1530 return rc;
1531
1532 qp->sgid_index = grh->sgid_index;
1533
1534 hdr_buf = kmalloc(PAGE_SIZE, GFP_KERNEL);
1535 if (!hdr_buf)
1536 return -ENOMEM;
1537
1538 hdr_len = ib_ud_header_pack(qp->hdr, hdr_buf);
1539 hdr_len -= IB_BTH_BYTES;
1540 hdr_len -= IB_DETH_BYTES;
1541 ibdev_dbg(&dev->ibdev, "roce packet header template\n");
1542 print_hex_dump_debug("hdr ", DUMP_PREFIX_OFFSET, 16, 1,
1543 hdr_buf, hdr_len, true);
1544
1545 hdr_dma = dma_map_single(dev->lif_cfg.hwdev, hdr_buf, hdr_len,
1546 DMA_TO_DEVICE);
1547
1548 rc = dma_mapping_error(dev->lif_cfg.hwdev, hdr_dma);
1549 if (rc)
1550 goto err_dma;
1551
1552 if (qp->hdr->ipv4_present) {
1553 wr.wqe.cmd.mod_qp.tfp_csum_profile =
1554 qp->hdr->vlan_present ?
1555 IONIC_TFP_CSUM_PROF_ETH_QTAG_IPV4_UDP :
1556 IONIC_TFP_CSUM_PROF_ETH_IPV4_UDP;
1557 } else {
1558 wr.wqe.cmd.mod_qp.tfp_csum_profile =
1559 qp->hdr->vlan_present ?
1560 IONIC_TFP_CSUM_PROF_ETH_QTAG_IPV6_UDP :
1561 IONIC_TFP_CSUM_PROF_ETH_IPV6_UDP;
1562 }
1563
1564 wr.wqe.cmd.mod_qp.ah_id_len =
1565 cpu_to_le32(qp->ahid | (hdr_len << 24));
1566 wr.wqe.cmd.mod_qp.dma_addr = cpu_to_le64(hdr_dma);
1567
1568 wr.wqe.cmd.mod_qp.en_pcp = attr->ah_attr.sl;
1569 wr.wqe.cmd.mod_qp.ip_dscp = grh->traffic_class >> 2;
1570 }
1571
1572 ionic_admin_post(dev, &wr);
1573
1574 rc = ionic_admin_wait(dev, &wr, 0);
1575
1576 if (mask & IB_QP_AV)
1577 dma_unmap_single(dev->lif_cfg.hwdev, hdr_dma, hdr_len,
1578 DMA_TO_DEVICE);
1579err_dma:
1580 if (mask & IB_QP_AV)
1581 kfree(hdr_buf);
1582
1583 return rc;
1584}
1585
1586static int ionic_query_qp_cmd(struct ionic_ibdev *dev,
1587 struct ionic_qp *qp,
1588 struct ib_qp_attr *attr,
1589 int mask)
1590{
1591 struct ionic_admin_wr wr = {
1592 .work = COMPLETION_INITIALIZER_ONSTACK(wr.work),
1593 .wqe = {
1594 .op = IONIC_V1_ADMIN_QUERY_QP,
1595 .len = cpu_to_le16(IONIC_ADMIN_QUERY_QP_IN_V1_LEN),
1596 .cmd.query_qp = {
1597 .id_ver = cpu_to_le32(qp->qpid),
1598 },
1599 }
1600 };
1601 struct ionic_v1_admin_query_qp_sq *query_sqbuf;
1602 struct ionic_v1_admin_query_qp_rq *query_rqbuf;
1603 dma_addr_t query_sqdma;
1604 dma_addr_t query_rqdma;
1605 dma_addr_t hdr_dma = 0;
1606 void *hdr_buf = NULL;
1607 int flags, rc;
1608
1609 if (dev->lif_cfg.admin_opcodes <= IONIC_V1_ADMIN_QUERY_QP)
1610 return -EBADRQC;
1611
1612 if (qp->has_sq) {
1613 bool expdb = !!(qp->sq_cmb & IONIC_CMB_EXPDB);
1614
1615 attr->cap.max_send_sge =
1616 ionic_v1_send_wqe_max_sge(qp->sq.stride_log2,
1617 qp->sq_spec,
1618 expdb);
1619 attr->cap.max_inline_data =
1620 ionic_v1_send_wqe_max_data(qp->sq.stride_log2, expdb);
1621 }
1622
1623 if (qp->has_rq) {
1624 attr->cap.max_recv_sge =
1625 ionic_v1_recv_wqe_max_sge(qp->rq.stride_log2,
1626 qp->rq_spec,
1627 qp->rq_cmb & IONIC_CMB_EXPDB);
1628 }
1629
1630 query_sqbuf = kzalloc(PAGE_SIZE, GFP_KERNEL);
1631 if (!query_sqbuf)
1632 return -ENOMEM;
1633
1634 query_rqbuf = kzalloc(PAGE_SIZE, GFP_KERNEL);
1635 if (!query_rqbuf) {
1636 rc = -ENOMEM;
1637 goto err_rqbuf;
1638 }
1639
1640 query_sqdma = dma_map_single(dev->lif_cfg.hwdev, query_sqbuf, PAGE_SIZE,
1641 DMA_FROM_DEVICE);
1642 rc = dma_mapping_error(dev->lif_cfg.hwdev, query_sqdma);
1643 if (rc)
1644 goto err_sqdma;
1645
1646 query_rqdma = dma_map_single(dev->lif_cfg.hwdev, query_rqbuf, PAGE_SIZE,
1647 DMA_FROM_DEVICE);
1648 rc = dma_mapping_error(dev->lif_cfg.hwdev, query_rqdma);
1649 if (rc)
1650 goto err_rqdma;
1651
1652 if (mask & IB_QP_AV) {
1653 hdr_buf = kmalloc(PAGE_SIZE, GFP_KERNEL);
1654 if (!hdr_buf) {
1655 rc = -ENOMEM;
1656 goto err_hdrbuf;
1657 }
1658
1659 hdr_dma = dma_map_single(dev->lif_cfg.hwdev, hdr_buf,
1660 PAGE_SIZE, DMA_FROM_DEVICE);
1661 rc = dma_mapping_error(dev->lif_cfg.hwdev, hdr_dma);
1662 if (rc)
1663 goto err_hdrdma;
1664 }
1665
1666 wr.wqe.cmd.query_qp.sq_dma_addr = cpu_to_le64(query_sqdma);
1667 wr.wqe.cmd.query_qp.rq_dma_addr = cpu_to_le64(query_rqdma);
1668 wr.wqe.cmd.query_qp.hdr_dma_addr = cpu_to_le64(hdr_dma);
1669 wr.wqe.cmd.query_qp.ah_id = cpu_to_le32(qp->ahid);
1670
1671 ionic_admin_post(dev, &wr);
1672
1673 rc = ionic_admin_wait(dev, &wr, 0);
1674
1675 if (rc)
1676 goto err_hdrdma;
1677
1678 flags = be16_to_cpu(query_sqbuf->access_perms_flags |
1679 query_rqbuf->access_perms_flags);
1680
1681 print_hex_dump_debug("sqbuf ", DUMP_PREFIX_OFFSET, 16, 1,
1682 query_sqbuf, sizeof(*query_sqbuf), true);
1683 print_hex_dump_debug("rqbuf ", DUMP_PREFIX_OFFSET, 16, 1,
1684 query_rqbuf, sizeof(*query_rqbuf), true);
1685 ibdev_dbg(&dev->ibdev, "query qp %u state_pmtu %#x flags %#x",
1686 qp->qpid, query_rqbuf->state_pmtu, flags);
1687
1688 attr->qp_state = from_ionic_qp_state(query_rqbuf->state_pmtu >> 4);
1689 attr->cur_qp_state = attr->qp_state;
1690 attr->path_mtu = (query_rqbuf->state_pmtu & 0xf) - 7;
1691 attr->path_mig_state = IB_MIG_MIGRATED;
1692 attr->qkey = be32_to_cpu(query_sqbuf->qkey_dest_qpn);
1693 attr->rq_psn = be32_to_cpu(query_sqbuf->rq_psn);
1694 attr->sq_psn = be32_to_cpu(query_rqbuf->sq_psn);
1695 attr->dest_qp_num = attr->qkey;
1696 attr->qp_access_flags = from_ionic_qp_flags(flags);
1697 attr->pkey_index = 0;
1698 attr->alt_pkey_index = 0;
1699 attr->en_sqd_async_notify = !!(flags & IONIC_QPF_SQD_NOTIFY);
1700 attr->sq_draining = !!(flags & IONIC_QPF_SQ_DRAINING);
1701 attr->max_rd_atomic = BIT(query_rqbuf->rrq_depth) - 1;
1702 attr->max_dest_rd_atomic = BIT(query_rqbuf->rsq_depth) - 1;
1703 attr->min_rnr_timer = query_sqbuf->rnr_timer;
1704 attr->port_num = 0;
1705 attr->timeout = query_sqbuf->retry_timeout;
1706 attr->retry_cnt = query_rqbuf->retry_rnrtry & 0xf;
1707 attr->rnr_retry = query_rqbuf->retry_rnrtry >> 4;
1708 attr->alt_port_num = 0;
1709 attr->alt_timeout = 0;
1710 attr->rate_limit = be32_to_cpu(query_sqbuf->rate_limit_kbps);
1711
1712 if (mask & IB_QP_AV)
1713 ionic_set_ah_attr(dev, &attr->ah_attr,
1714 qp->hdr, qp->sgid_index);
1715
1716err_hdrdma:
1717 if (mask & IB_QP_AV) {
1718 dma_unmap_single(dev->lif_cfg.hwdev, hdr_dma,
1719 PAGE_SIZE, DMA_FROM_DEVICE);
1720 kfree(hdr_buf);
1721 }
1722err_hdrbuf:
1723 dma_unmap_single(dev->lif_cfg.hwdev, query_rqdma, sizeof(*query_rqbuf),
1724 DMA_FROM_DEVICE);
1725err_rqdma:
1726 dma_unmap_single(dev->lif_cfg.hwdev, query_sqdma, sizeof(*query_sqbuf),
1727 DMA_FROM_DEVICE);
1728err_sqdma:
1729 kfree(query_rqbuf);
1730err_rqbuf:
1731 kfree(query_sqbuf);
1732
1733 return rc;
1734}
1735
1736static int ionic_destroy_qp_cmd(struct ionic_ibdev *dev, u32 qpid)
1737{
1738 struct ionic_admin_wr wr = {
1739 .work = COMPLETION_INITIALIZER_ONSTACK(wr.work),
1740 .wqe = {
1741 .op = IONIC_V1_ADMIN_DESTROY_QP,
1742 .len = cpu_to_le16(IONIC_ADMIN_DESTROY_QP_IN_V1_LEN),
1743 .cmd.destroy_qp = {
1744 .qp_id = cpu_to_le32(qpid),
1745 },
1746 }
1747 };
1748
1749 if (dev->lif_cfg.admin_opcodes <= IONIC_V1_ADMIN_DESTROY_QP)
1750 return -EBADRQC;
1751
1752 ionic_admin_post(dev, &wr);
1753
1754 return ionic_admin_wait(dev, &wr, IONIC_ADMIN_F_TEARDOWN);
1755}
1756
1757static bool ionic_expdb_wqe_size_supported(struct ionic_ibdev *dev,
1758 uint32_t wqe_size)
1759{
1760 switch (wqe_size) {
1761 case 64: return dev->lif_cfg.expdb_mask & IONIC_EXPDB_64;
1762 case 128: return dev->lif_cfg.expdb_mask & IONIC_EXPDB_128;
1763 case 256: return dev->lif_cfg.expdb_mask & IONIC_EXPDB_256;
1764 case 512: return dev->lif_cfg.expdb_mask & IONIC_EXPDB_512;
1765 }
1766
1767 return false;
1768}
1769
1770static void ionic_qp_sq_init_cmb(struct ionic_ibdev *dev,
1771 struct ionic_qp *qp,
1772 struct ib_udata *udata,
1773 int max_data)
1774{
1775 u8 expdb_stride_log2 = 0;
1776 bool expdb;
1777 int rc;
1778
1779 if (!(qp->sq_cmb & IONIC_CMB_ENABLE))
1780 goto not_in_cmb;
1781
1782 if (qp->sq_cmb & ~IONIC_CMB_SUPPORTED) {
1783 if (qp->sq_cmb & IONIC_CMB_REQUIRE)
1784 goto not_in_cmb;
1785
1786 qp->sq_cmb &= IONIC_CMB_SUPPORTED;
1787 }
1788
1789 if ((qp->sq_cmb & IONIC_CMB_EXPDB) && !dev->lif_cfg.sq_expdb) {
1790 if (qp->sq_cmb & IONIC_CMB_REQUIRE)
1791 goto not_in_cmb;
1792
1793 qp->sq_cmb &= ~IONIC_CMB_EXPDB;
1794 }
1795
1796 qp->sq_cmb_order = order_base_2(qp->sq.size / PAGE_SIZE);
1797
1798 if (qp->sq_cmb_order >= IONIC_SQCMB_ORDER)
1799 goto not_in_cmb;
1800
1801 if (qp->sq_cmb & IONIC_CMB_EXPDB)
1802 expdb_stride_log2 = qp->sq.stride_log2;
1803
1804 rc = ionic_get_cmb(dev->lif_cfg.lif, &qp->sq_cmb_pgid,
1805 &qp->sq_cmb_addr, qp->sq_cmb_order,
1806 expdb_stride_log2, &expdb);
1807 if (rc)
1808 goto not_in_cmb;
1809
1810 if ((qp->sq_cmb & IONIC_CMB_EXPDB) && !expdb) {
1811 if (qp->sq_cmb & IONIC_CMB_REQUIRE)
1812 goto err_map;
1813
1814 qp->sq_cmb &= ~IONIC_CMB_EXPDB;
1815 }
1816
1817 return;
1818
1819err_map:
1820 ionic_put_cmb(dev->lif_cfg.lif, qp->sq_cmb_pgid, qp->sq_cmb_order);
1821not_in_cmb:
1822 if (qp->sq_cmb & IONIC_CMB_REQUIRE)
1823 ibdev_dbg(&dev->ibdev, "could not place sq in cmb as required\n");
1824
1825 qp->sq_cmb = 0;
1826 qp->sq_cmb_order = IONIC_RES_INVALID;
1827 qp->sq_cmb_pgid = 0;
1828 qp->sq_cmb_addr = 0;
1829}
1830
1831static void ionic_qp_sq_destroy_cmb(struct ionic_ibdev *dev,
1832 struct ionic_ctx *ctx,
1833 struct ionic_qp *qp)
1834{
1835 if (!(qp->sq_cmb & IONIC_CMB_ENABLE))
1836 return;
1837
1838 if (ctx)
1839 rdma_user_mmap_entry_remove(qp->mmap_sq_cmb);
1840
1841 ionic_put_cmb(dev->lif_cfg.lif, qp->sq_cmb_pgid, qp->sq_cmb_order);
1842}
1843
1844static int ionic_qp_sq_init(struct ionic_ibdev *dev, struct ionic_ctx *ctx,
1845 struct ionic_qp *qp, struct ionic_qdesc *sq,
1846 struct ionic_tbl_buf *buf, int max_wr, int max_sge,
1847 int max_data, int sq_spec, struct ib_udata *udata)
1848{
1849 u32 wqe_size;
1850 int rc = 0;
1851
1852 qp->sq_msn_prod = 0;
1853 qp->sq_msn_cons = 0;
1854
1855 if (!qp->has_sq) {
1856 if (buf) {
1857 buf->tbl_buf = NULL;
1858 buf->tbl_limit = 0;
1859 buf->tbl_pages = 0;
1860 }
1861 if (udata)
1862 rc = ionic_validate_qdesc_zero(sq);
1863
1864 return rc;
1865 }
1866
1867 rc = -EINVAL;
1868
1869 if (max_wr < 0 || max_wr > 0xffff)
1870 return rc;
1871
1872 if (max_sge < 1)
1873 return rc;
1874
1875 if (max_sge > min(ionic_v1_send_wqe_max_sge(dev->lif_cfg.max_stride, 0,
1876 qp->sq_cmb &
1877 IONIC_CMB_EXPDB),
1878 IONIC_SPEC_HIGH))
1879 return rc;
1880
1881 if (max_data < 0)
1882 return rc;
1883
1884 if (max_data > ionic_v1_send_wqe_max_data(dev->lif_cfg.max_stride,
1885 qp->sq_cmb & IONIC_CMB_EXPDB))
1886 return rc;
1887
1888 if (udata) {
1889 rc = ionic_validate_qdesc(sq);
1890 if (rc)
1891 return rc;
1892
1893 qp->sq_spec = sq_spec;
1894
1895 qp->sq.ptr = NULL;
1896 qp->sq.size = sq->size;
1897 qp->sq.mask = sq->mask;
1898 qp->sq.depth_log2 = sq->depth_log2;
1899 qp->sq.stride_log2 = sq->stride_log2;
1900
1901 qp->sq_meta = NULL;
1902 qp->sq_msn_idx = NULL;
1903
1904 qp->sq_umem = ib_umem_get_va(&dev->ibdev, sq->addr, sq->size, 0);
1905 if (IS_ERR(qp->sq_umem))
1906 return PTR_ERR(qp->sq_umem);
1907 } else {
1908 qp->sq_umem = NULL;
1909
1910 qp->sq_spec = ionic_v1_use_spec_sge(max_sge, sq_spec);
1911 if (sq_spec && !qp->sq_spec)
1912 ibdev_dbg(&dev->ibdev,
1913 "init sq: max_sge %u disables spec\n",
1914 max_sge);
1915
1916 if (qp->sq_cmb & IONIC_CMB_EXPDB) {
1917 wqe_size = ionic_v1_send_wqe_min_size(max_sge, max_data,
1918 qp->sq_spec,
1919 true);
1920
1921 if (!ionic_expdb_wqe_size_supported(dev, wqe_size))
1922 qp->sq_cmb &= ~IONIC_CMB_EXPDB;
1923 }
1924
1925 if (!(qp->sq_cmb & IONIC_CMB_EXPDB))
1926 wqe_size = ionic_v1_send_wqe_min_size(max_sge, max_data,
1927 qp->sq_spec,
1928 false);
1929
1930 rc = ionic_queue_init(&qp->sq, dev->lif_cfg.hwdev,
1931 max_wr, wqe_size);
1932 if (rc)
1933 return rc;
1934
1935 ionic_queue_dbell_init(&qp->sq, qp->qpid);
1936
1937 qp->sq_meta = kmalloc_objs(*qp->sq_meta, (u32)qp->sq.mask + 1);
1938 if (!qp->sq_meta) {
1939 rc = -ENOMEM;
1940 goto err_sq_meta;
1941 }
1942
1943 qp->sq_msn_idx = kmalloc_array((u32)qp->sq.mask + 1,
1944 sizeof(*qp->sq_msn_idx),
1945 GFP_KERNEL);
1946 if (!qp->sq_msn_idx) {
1947 rc = -ENOMEM;
1948 goto err_sq_msn;
1949 }
1950 }
1951
1952 ionic_qp_sq_init_cmb(dev, qp, udata, max_data);
1953
1954 if (qp->sq_cmb & IONIC_CMB_ENABLE)
1955 rc = ionic_pgtbl_init(dev, buf, NULL,
1956 (u64)qp->sq_cmb_pgid << PAGE_SHIFT,
1957 1, PAGE_SIZE);
1958 else
1959 rc = ionic_pgtbl_init(dev, buf,
1960 qp->sq_umem, qp->sq.dma, 1, PAGE_SIZE);
1961 if (rc)
1962 goto err_sq_tbl;
1963
1964 return 0;
1965
1966err_sq_tbl:
1967 ionic_qp_sq_destroy_cmb(dev, ctx, qp);
1968 kfree(qp->sq_msn_idx);
1969err_sq_msn:
1970 kfree(qp->sq_meta);
1971err_sq_meta:
1972 if (qp->sq_umem)
1973 ib_umem_release(qp->sq_umem);
1974 else
1975 ionic_queue_destroy(&qp->sq, dev->lif_cfg.hwdev);
1976 return rc;
1977}
1978
1979static void ionic_qp_sq_destroy(struct ionic_ibdev *dev,
1980 struct ionic_ctx *ctx,
1981 struct ionic_qp *qp)
1982{
1983 if (!qp->has_sq)
1984 return;
1985
1986 ionic_qp_sq_destroy_cmb(dev, ctx, qp);
1987
1988 kfree(qp->sq_msn_idx);
1989 kfree(qp->sq_meta);
1990
1991 if (qp->sq_umem)
1992 ib_umem_release(qp->sq_umem);
1993 else
1994 ionic_queue_destroy(&qp->sq, dev->lif_cfg.hwdev);
1995}
1996
1997static void ionic_qp_rq_init_cmb(struct ionic_ibdev *dev,
1998 struct ionic_qp *qp,
1999 struct ib_udata *udata)
2000{
2001 u8 expdb_stride_log2 = 0;
2002 bool expdb;
2003 int rc;
2004
2005 if (!(qp->rq_cmb & IONIC_CMB_ENABLE))
2006 goto not_in_cmb;
2007
2008 if (qp->rq_cmb & ~IONIC_CMB_SUPPORTED) {
2009 if (qp->rq_cmb & IONIC_CMB_REQUIRE)
2010 goto not_in_cmb;
2011
2012 qp->rq_cmb &= IONIC_CMB_SUPPORTED;
2013 }
2014
2015 if ((qp->rq_cmb & IONIC_CMB_EXPDB) && !dev->lif_cfg.rq_expdb) {
2016 if (qp->rq_cmb & IONIC_CMB_REQUIRE)
2017 goto not_in_cmb;
2018
2019 qp->rq_cmb &= ~IONIC_CMB_EXPDB;
2020 }
2021
2022 qp->rq_cmb_order = order_base_2(qp->rq.size / PAGE_SIZE);
2023
2024 if (qp->rq_cmb_order >= IONIC_RQCMB_ORDER)
2025 goto not_in_cmb;
2026
2027 if (qp->rq_cmb & IONIC_CMB_EXPDB)
2028 expdb_stride_log2 = qp->rq.stride_log2;
2029
2030 rc = ionic_get_cmb(dev->lif_cfg.lif, &qp->rq_cmb_pgid,
2031 &qp->rq_cmb_addr, qp->rq_cmb_order,
2032 expdb_stride_log2, &expdb);
2033 if (rc)
2034 goto not_in_cmb;
2035
2036 if ((qp->rq_cmb & IONIC_CMB_EXPDB) && !expdb) {
2037 if (qp->rq_cmb & IONIC_CMB_REQUIRE)
2038 goto err_map;
2039
2040 qp->rq_cmb &= ~IONIC_CMB_EXPDB;
2041 }
2042
2043 return;
2044
2045err_map:
2046 ionic_put_cmb(dev->lif_cfg.lif, qp->rq_cmb_pgid, qp->rq_cmb_order);
2047not_in_cmb:
2048 if (qp->rq_cmb & IONIC_CMB_REQUIRE)
2049 ibdev_dbg(&dev->ibdev, "could not place rq in cmb as required\n");
2050
2051 qp->rq_cmb = 0;
2052 qp->rq_cmb_order = IONIC_RES_INVALID;
2053 qp->rq_cmb_pgid = 0;
2054 qp->rq_cmb_addr = 0;
2055}
2056
2057static void ionic_qp_rq_destroy_cmb(struct ionic_ibdev *dev,
2058 struct ionic_ctx *ctx,
2059 struct ionic_qp *qp)
2060{
2061 if (!(qp->rq_cmb & IONIC_CMB_ENABLE))
2062 return;
2063
2064 if (ctx)
2065 rdma_user_mmap_entry_remove(qp->mmap_rq_cmb);
2066
2067 ionic_put_cmb(dev->lif_cfg.lif, qp->rq_cmb_pgid, qp->rq_cmb_order);
2068}
2069
2070static int ionic_qp_rq_init(struct ionic_ibdev *dev, struct ionic_ctx *ctx,
2071 struct ionic_qp *qp, struct ionic_qdesc *rq,
2072 struct ionic_tbl_buf *buf, int max_wr, int max_sge,
2073 int rq_spec, struct ib_udata *udata)
2074{
2075 int rc = 0, i;
2076 u32 wqe_size;
2077
2078 if (!qp->has_rq) {
2079 if (buf) {
2080 buf->tbl_buf = NULL;
2081 buf->tbl_limit = 0;
2082 buf->tbl_pages = 0;
2083 }
2084 if (udata)
2085 rc = ionic_validate_qdesc_zero(rq);
2086
2087 return rc;
2088 }
2089
2090 rc = -EINVAL;
2091
2092 if (max_wr < 0 || max_wr > 0xffff)
2093 return rc;
2094
2095 if (max_sge < 1)
2096 return rc;
2097
2098 if (max_sge > min(ionic_v1_recv_wqe_max_sge(dev->lif_cfg.max_stride, 0, false),
2099 IONIC_SPEC_HIGH))
2100 return rc;
2101
2102 if (udata) {
2103 rc = ionic_validate_qdesc(rq);
2104 if (rc)
2105 return rc;
2106
2107 qp->rq_spec = rq_spec;
2108
2109 qp->rq.ptr = NULL;
2110 qp->rq.size = rq->size;
2111 qp->rq.mask = rq->mask;
2112 qp->rq.depth_log2 = rq->depth_log2;
2113 qp->rq.stride_log2 = rq->stride_log2;
2114
2115 qp->rq_meta = NULL;
2116
2117 qp->rq_umem = ib_umem_get_va(&dev->ibdev, rq->addr, rq->size, 0);
2118 if (IS_ERR(qp->rq_umem))
2119 return PTR_ERR(qp->rq_umem);
2120 } else {
2121 qp->rq_umem = NULL;
2122
2123 qp->rq_spec = ionic_v1_use_spec_sge(max_sge, rq_spec);
2124 if (rq_spec && !qp->rq_spec)
2125 ibdev_dbg(&dev->ibdev,
2126 "init rq: max_sge %u disables spec\n",
2127 max_sge);
2128
2129 if (qp->rq_cmb & IONIC_CMB_EXPDB) {
2130 wqe_size = ionic_v1_recv_wqe_min_size(max_sge,
2131 qp->rq_spec,
2132 true);
2133
2134 if (!ionic_expdb_wqe_size_supported(dev, wqe_size))
2135 qp->rq_cmb &= ~IONIC_CMB_EXPDB;
2136 }
2137
2138 if (!(qp->rq_cmb & IONIC_CMB_EXPDB))
2139 wqe_size = ionic_v1_recv_wqe_min_size(max_sge,
2140 qp->rq_spec,
2141 false);
2142
2143 rc = ionic_queue_init(&qp->rq, dev->lif_cfg.hwdev,
2144 max_wr, wqe_size);
2145 if (rc)
2146 return rc;
2147
2148 ionic_queue_dbell_init(&qp->rq, qp->qpid);
2149
2150 qp->rq_meta = kmalloc_objs(*qp->rq_meta, (u32)qp->rq.mask + 1);
2151 if (!qp->rq_meta) {
2152 rc = -ENOMEM;
2153 goto err_rq_meta;
2154 }
2155
2156 for (i = 0; i < qp->rq.mask; ++i)
2157 qp->rq_meta[i].next = &qp->rq_meta[i + 1];
2158 qp->rq_meta[i].next = IONIC_META_LAST;
2159 qp->rq_meta_head = &qp->rq_meta[0];
2160 }
2161
2162 ionic_qp_rq_init_cmb(dev, qp, udata);
2163
2164 if (qp->rq_cmb & IONIC_CMB_ENABLE)
2165 rc = ionic_pgtbl_init(dev, buf, NULL,
2166 (u64)qp->rq_cmb_pgid << PAGE_SHIFT,
2167 1, PAGE_SIZE);
2168 else
2169 rc = ionic_pgtbl_init(dev, buf,
2170 qp->rq_umem, qp->rq.dma, 1, PAGE_SIZE);
2171 if (rc)
2172 goto err_rq_tbl;
2173
2174 return 0;
2175
2176err_rq_tbl:
2177 ionic_qp_rq_destroy_cmb(dev, ctx, qp);
2178 kfree(qp->rq_meta);
2179err_rq_meta:
2180 if (qp->rq_umem)
2181 ib_umem_release(qp->rq_umem);
2182 else
2183 ionic_queue_destroy(&qp->rq, dev->lif_cfg.hwdev);
2184 return rc;
2185}
2186
2187static void ionic_qp_rq_destroy(struct ionic_ibdev *dev,
2188 struct ionic_ctx *ctx,
2189 struct ionic_qp *qp)
2190{
2191 if (!qp->has_rq)
2192 return;
2193
2194 ionic_qp_rq_destroy_cmb(dev, ctx, qp);
2195
2196 kfree(qp->rq_meta);
2197
2198 if (qp->rq_umem)
2199 ib_umem_release(qp->rq_umem);
2200 else
2201 ionic_queue_destroy(&qp->rq, dev->lif_cfg.hwdev);
2202}
2203
2204int ionic_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attr,
2205 struct ib_udata *udata)
2206{
2207 struct ionic_ibdev *dev = to_ionic_ibdev(ibqp->device);
2208 struct ionic_tbl_buf sq_buf = {}, rq_buf = {};
2209 struct ionic_pd *pd = to_ionic_pd(ibqp->pd);
2210 struct ionic_qp *qp = to_ionic_qp(ibqp);
2211 struct ionic_ctx *ctx =
2212 rdma_udata_to_drv_context(udata, struct ionic_ctx, ibctx);
2213 struct ionic_qp_resp resp = {};
2214 struct ionic_qp_req req = {};
2215 struct ionic_cq *cq;
2216 u8 udma_mask;
2217 void *entry;
2218 int rc;
2219
2220 if (udata) {
2221 rc = ib_copy_validate_udata_in(udata, req, rsvd);
2222 if (rc)
2223 return rc;
2224 } else {
2225 req.sq_spec = IONIC_SPEC_HIGH;
2226 req.rq_spec = IONIC_SPEC_HIGH;
2227 }
2228
2229 if (attr->qp_type == IB_QPT_SMI || attr->qp_type > IB_QPT_UD)
2230 return -EOPNOTSUPP;
2231
2232 qp->state = IB_QPS_RESET;
2233
2234 INIT_LIST_HEAD(&qp->cq_poll_sq);
2235 INIT_LIST_HEAD(&qp->cq_flush_sq);
2236 INIT_LIST_HEAD(&qp->cq_flush_rq);
2237
2238 spin_lock_init(&qp->sq_lock);
2239 spin_lock_init(&qp->rq_lock);
2240
2241 qp->has_sq = 1;
2242 qp->has_rq = 1;
2243
2244 if (attr->qp_type == IB_QPT_GSI) {
2245 rc = ionic_get_gsi_qpid(dev, &qp->qpid);
2246 } else {
2247 udma_mask = BIT(dev->lif_cfg.udma_count) - 1;
2248
2249 if (qp->has_sq)
2250 udma_mask &= to_ionic_vcq(attr->send_cq)->udma_mask;
2251
2252 if (qp->has_rq)
2253 udma_mask &= to_ionic_vcq(attr->recv_cq)->udma_mask;
2254
2255 if (udata && req.udma_mask)
2256 udma_mask &= req.udma_mask;
2257
2258 if (!udma_mask)
2259 return -EINVAL;
2260
2261 rc = ionic_get_qpid(dev, &qp->qpid, &qp->udma_idx, udma_mask);
2262 }
2263 if (rc)
2264 return rc;
2265
2266 qp->sig_all = attr->sq_sig_type == IB_SIGNAL_ALL_WR;
2267 qp->has_ah = attr->qp_type == IB_QPT_RC;
2268
2269 if (qp->has_ah) {
2270 qp->hdr = kzalloc_obj(*qp->hdr);
2271 if (!qp->hdr) {
2272 rc = -ENOMEM;
2273 goto err_ah_alloc;
2274 }
2275
2276 rc = ionic_get_ahid(dev, &qp->ahid);
2277 if (rc)
2278 goto err_ahid;
2279 }
2280
2281 if (udata) {
2282 if (req.rq_cmb & IONIC_CMB_ENABLE)
2283 qp->rq_cmb = req.rq_cmb;
2284
2285 if (req.sq_cmb & IONIC_CMB_ENABLE)
2286 qp->sq_cmb = req.sq_cmb;
2287 }
2288
2289 rc = ionic_qp_sq_init(dev, ctx, qp, &req.sq, &sq_buf,
2290 attr->cap.max_send_wr, attr->cap.max_send_sge,
2291 attr->cap.max_inline_data, req.sq_spec, udata);
2292 if (rc)
2293 goto err_sq;
2294
2295 rc = ionic_qp_rq_init(dev, ctx, qp, &req.rq, &rq_buf,
2296 attr->cap.max_recv_wr, attr->cap.max_recv_sge,
2297 req.rq_spec, udata);
2298 if (rc)
2299 goto err_rq;
2300
2301 rc = ionic_create_qp_cmd(dev, pd,
2302 to_ionic_vcq_cq(attr->send_cq, qp->udma_idx),
2303 to_ionic_vcq_cq(attr->recv_cq, qp->udma_idx),
2304 qp, &sq_buf, &rq_buf, attr);
2305 if (rc)
2306 goto err_cmd;
2307
2308 if (udata) {
2309 resp.qpid = qp->qpid;
2310 resp.udma_idx = qp->udma_idx;
2311
2312 if (qp->sq_cmb & IONIC_CMB_ENABLE) {
2313 bool wc;
2314
2315 if ((qp->sq_cmb & (IONIC_CMB_WC | IONIC_CMB_UC)) ==
2316 (IONIC_CMB_WC | IONIC_CMB_UC)) {
2317 ibdev_dbg(&dev->ibdev,
2318 "Both sq_cmb flags IONIC_CMB_WC and IONIC_CMB_UC are set, using default driver mapping\n");
2319 qp->sq_cmb &= ~(IONIC_CMB_WC | IONIC_CMB_UC);
2320 }
2321
2322 wc = (qp->sq_cmb & (IONIC_CMB_WC | IONIC_CMB_UC))
2323 != IONIC_CMB_UC;
2324
2325 /* let userspace know the mapping */
2326 if (wc)
2327 qp->sq_cmb |= IONIC_CMB_WC;
2328 else
2329 qp->sq_cmb |= IONIC_CMB_UC;
2330
2331 qp->mmap_sq_cmb =
2332 ionic_mmap_entry_insert(ctx,
2333 qp->sq.size,
2334 PHYS_PFN(qp->sq_cmb_addr),
2335 wc ? IONIC_MMAP_WC : 0,
2336 &resp.sq_cmb_offset);
2337 if (!qp->mmap_sq_cmb) {
2338 rc = -ENOMEM;
2339 goto err_mmap_sq;
2340 }
2341
2342 resp.sq_cmb = qp->sq_cmb;
2343 }
2344
2345 if (qp->rq_cmb & IONIC_CMB_ENABLE) {
2346 bool wc;
2347
2348 if ((qp->rq_cmb & (IONIC_CMB_WC | IONIC_CMB_UC)) ==
2349 (IONIC_CMB_WC | IONIC_CMB_UC)) {
2350 ibdev_dbg(&dev->ibdev,
2351 "Both rq_cmb flags IONIC_CMB_WC and IONIC_CMB_UC are set, using default driver mapping\n");
2352 qp->rq_cmb &= ~(IONIC_CMB_WC | IONIC_CMB_UC);
2353 }
2354
2355 if (qp->rq_cmb & IONIC_CMB_EXPDB)
2356 wc = (qp->rq_cmb & (IONIC_CMB_WC | IONIC_CMB_UC))
2357 == IONIC_CMB_WC;
2358 else
2359 wc = (qp->rq_cmb & (IONIC_CMB_WC | IONIC_CMB_UC))
2360 != IONIC_CMB_UC;
2361
2362 /* let userspace know the mapping */
2363 if (wc)
2364 qp->rq_cmb |= IONIC_CMB_WC;
2365 else
2366 qp->rq_cmb |= IONIC_CMB_UC;
2367
2368 qp->mmap_rq_cmb =
2369 ionic_mmap_entry_insert(ctx,
2370 qp->rq.size,
2371 PHYS_PFN(qp->rq_cmb_addr),
2372 wc ? IONIC_MMAP_WC : 0,
2373 &resp.rq_cmb_offset);
2374 if (!qp->mmap_rq_cmb) {
2375 rc = -ENOMEM;
2376 goto err_mmap_rq;
2377 }
2378
2379 resp.rq_cmb = qp->rq_cmb;
2380 }
2381
2382 rc = ib_respond_udata(udata, resp);
2383 if (rc)
2384 goto err_resp;
2385 }
2386
2387 ionic_pgtbl_unbuf(dev, &rq_buf);
2388 ionic_pgtbl_unbuf(dev, &sq_buf);
2389
2390 qp->ibqp.qp_num = qp->qpid;
2391
2392 init_completion(&qp->qp_rel_comp);
2393 kref_init(&qp->qp_kref);
2394
2395 entry = xa_store_irq(&dev->qp_tbl, qp->qpid, qp, GFP_KERNEL);
2396 if (entry) {
2397 if (!xa_is_err(entry))
2398 rc = -EINVAL;
2399 else
2400 rc = xa_err(entry);
2401
2402 goto err_resp;
2403 }
2404
2405 if (qp->has_sq) {
2406 cq = to_ionic_vcq_cq(attr->send_cq, qp->udma_idx);
2407
2408 attr->cap.max_send_wr = qp->sq.mask;
2409 attr->cap.max_send_sge =
2410 ionic_v1_send_wqe_max_sge(qp->sq.stride_log2,
2411 qp->sq_spec,
2412 qp->sq_cmb & IONIC_CMB_EXPDB);
2413 attr->cap.max_inline_data =
2414 ionic_v1_send_wqe_max_data(qp->sq.stride_log2,
2415 qp->sq_cmb &
2416 IONIC_CMB_EXPDB);
2417 qp->sq_cqid = cq->cqid;
2418 }
2419
2420 if (qp->has_rq) {
2421 cq = to_ionic_vcq_cq(attr->recv_cq, qp->udma_idx);
2422
2423 attr->cap.max_recv_wr = qp->rq.mask;
2424 attr->cap.max_recv_sge =
2425 ionic_v1_recv_wqe_max_sge(qp->rq.stride_log2,
2426 qp->rq_spec,
2427 qp->rq_cmb & IONIC_CMB_EXPDB);
2428 qp->rq_cqid = cq->cqid;
2429 }
2430
2431 return 0;
2432
2433err_resp:
2434 if (udata && (qp->rq_cmb & IONIC_CMB_ENABLE))
2435 rdma_user_mmap_entry_remove(qp->mmap_rq_cmb);
2436err_mmap_rq:
2437 if (udata && (qp->sq_cmb & IONIC_CMB_ENABLE))
2438 rdma_user_mmap_entry_remove(qp->mmap_sq_cmb);
2439err_mmap_sq:
2440 ionic_destroy_qp_cmd(dev, qp->qpid);
2441err_cmd:
2442 ionic_pgtbl_unbuf(dev, &rq_buf);
2443 ionic_qp_rq_destroy(dev, ctx, qp);
2444err_rq:
2445 ionic_pgtbl_unbuf(dev, &sq_buf);
2446 ionic_qp_sq_destroy(dev, ctx, qp);
2447err_sq:
2448 if (qp->has_ah)
2449 ionic_put_ahid(dev, qp->ahid);
2450err_ahid:
2451 kfree(qp->hdr);
2452err_ah_alloc:
2453 ionic_put_qpid(dev, qp->qpid);
2454 return rc;
2455}
2456
2457void ionic_notify_flush_cq(struct ionic_cq *cq)
2458{
2459 if (cq->flush && cq->vcq->ibcq.comp_handler)
2460 cq->vcq->ibcq.comp_handler(&cq->vcq->ibcq,
2461 cq->vcq->ibcq.cq_context);
2462}
2463
2464static void ionic_notify_qp_cqs(struct ionic_ibdev *dev, struct ionic_qp *qp)
2465{
2466 if (qp->ibqp.send_cq)
2467 ionic_notify_flush_cq(to_ionic_vcq_cq(qp->ibqp.send_cq,
2468 qp->udma_idx));
2469 if (qp->ibqp.recv_cq && qp->ibqp.recv_cq != qp->ibqp.send_cq)
2470 ionic_notify_flush_cq(to_ionic_vcq_cq(qp->ibqp.recv_cq,
2471 qp->udma_idx));
2472}
2473
2474void ionic_flush_qp(struct ionic_ibdev *dev, struct ionic_qp *qp)
2475{
2476 unsigned long irqflags;
2477 struct ionic_cq *cq;
2478
2479 if (qp->ibqp.send_cq) {
2480 cq = to_ionic_vcq_cq(qp->ibqp.send_cq, qp->udma_idx);
2481
2482 /* Hold the CQ lock and QP sq_lock to set up flush */
2483 spin_lock_irqsave(&cq->lock, irqflags);
2484 spin_lock(&qp->sq_lock);
2485 qp->sq_flush = true;
2486 if (!ionic_queue_empty(&qp->sq)) {
2487 cq->flush = true;
2488 list_move_tail(&qp->cq_flush_sq, &cq->flush_sq);
2489 }
2490 spin_unlock(&qp->sq_lock);
2491 spin_unlock_irqrestore(&cq->lock, irqflags);
2492 }
2493
2494 if (qp->ibqp.recv_cq) {
2495 cq = to_ionic_vcq_cq(qp->ibqp.recv_cq, qp->udma_idx);
2496
2497 /* Hold the CQ lock and QP rq_lock to set up flush */
2498 spin_lock_irqsave(&cq->lock, irqflags);
2499 spin_lock(&qp->rq_lock);
2500 qp->rq_flush = true;
2501 if (!ionic_queue_empty(&qp->rq)) {
2502 cq->flush = true;
2503 list_move_tail(&qp->cq_flush_rq, &cq->flush_rq);
2504 }
2505 spin_unlock(&qp->rq_lock);
2506 spin_unlock_irqrestore(&cq->lock, irqflags);
2507 }
2508}
2509
2510static void ionic_clean_cq(struct ionic_cq *cq, u32 qpid)
2511{
2512 struct ionic_v1_cqe *qcqe;
2513 int prod, qtf, qid, type;
2514 bool color;
2515
2516 if (!cq->q.ptr)
2517 return;
2518
2519 color = cq->color;
2520 prod = cq->q.prod;
2521 qcqe = ionic_queue_at(&cq->q, prod);
2522
2523 while (color == ionic_v1_cqe_color(qcqe)) {
2524 qtf = ionic_v1_cqe_qtf(qcqe);
2525 qid = ionic_v1_cqe_qtf_qid(qtf);
2526 type = ionic_v1_cqe_qtf_type(qtf);
2527
2528 if (qid == qpid && type != IONIC_V1_CQE_TYPE_ADMIN)
2529 ionic_v1_cqe_clean(qcqe);
2530
2531 prod = ionic_queue_next(&cq->q, prod);
2532 qcqe = ionic_queue_at(&cq->q, prod);
2533 color = ionic_color_wrap(prod, color);
2534 }
2535}
2536
2537static void ionic_reset_qp(struct ionic_ibdev *dev, struct ionic_qp *qp)
2538{
2539 unsigned long irqflags;
2540 struct ionic_cq *cq;
2541 int i;
2542
2543 local_irq_save(irqflags);
2544
2545 if (qp->ibqp.send_cq) {
2546 cq = to_ionic_vcq_cq(qp->ibqp.send_cq, qp->udma_idx);
2547 spin_lock(&cq->lock);
2548 ionic_clean_cq(cq, qp->qpid);
2549 spin_unlock(&cq->lock);
2550 }
2551
2552 if (qp->ibqp.recv_cq) {
2553 cq = to_ionic_vcq_cq(qp->ibqp.recv_cq, qp->udma_idx);
2554 spin_lock(&cq->lock);
2555 ionic_clean_cq(cq, qp->qpid);
2556 spin_unlock(&cq->lock);
2557 }
2558
2559 if (qp->has_sq) {
2560 spin_lock(&qp->sq_lock);
2561 qp->sq_flush = false;
2562 qp->sq_flush_rcvd = false;
2563 qp->sq_msn_prod = 0;
2564 qp->sq_msn_cons = 0;
2565 qp->sq.prod = 0;
2566 qp->sq.cons = 0;
2567 spin_unlock(&qp->sq_lock);
2568 }
2569
2570 if (qp->has_rq) {
2571 spin_lock(&qp->rq_lock);
2572 qp->rq_flush = false;
2573 qp->rq.prod = 0;
2574 qp->rq.cons = 0;
2575 if (qp->rq_meta) {
2576 for (i = 0; i < qp->rq.mask; ++i)
2577 qp->rq_meta[i].next = &qp->rq_meta[i + 1];
2578 qp->rq_meta[i].next = IONIC_META_LAST;
2579 }
2580 qp->rq_meta_head = &qp->rq_meta[0];
2581 spin_unlock(&qp->rq_lock);
2582 }
2583
2584 local_irq_restore(irqflags);
2585}
2586
2587static bool ionic_qp_cur_state_is_ok(enum ib_qp_state q_state,
2588 enum ib_qp_state attr_state)
2589{
2590 if (q_state == attr_state)
2591 return true;
2592
2593 if (attr_state == IB_QPS_ERR)
2594 return true;
2595
2596 if (attr_state == IB_QPS_SQE)
2597 return q_state == IB_QPS_RTS || q_state == IB_QPS_SQD;
2598
2599 return false;
2600}
2601
2602static bool ionic_is_modify_ok(enum ib_qp_attr_mask ext_mask,
2603 enum ib_qp_type type, enum ib_qp_state cur,
2604 enum ib_qp_state next)
2605{
2606 if (!ext_mask)
2607 return true;
2608
2609 if (ext_mask & ~IB_QP_RATE_LIMIT)
2610 return false;
2611
2612 /* Rate limit is only supported for RC QPs during specific transitions */
2613 return type == IB_QPT_RC &&
2614 ((cur == IB_QPS_INIT && next == IB_QPS_RTR) ||
2615 (cur == IB_QPS_RTR && next == IB_QPS_RTS) ||
2616 (cur == IB_QPS_RTS && next == IB_QPS_RTS));
2617}
2618
2619static int ionic_check_modify_qp(struct ionic_qp *qp, struct ib_qp_attr *attr,
2620 int mask)
2621{
2622 enum ib_qp_state cur_state = (mask & IB_QP_CUR_STATE) ?
2623 attr->cur_qp_state : qp->state;
2624 enum ib_qp_state next_state = (mask & IB_QP_STATE) ?
2625 attr->qp_state : cur_state;
2626
2627 if ((mask & IB_QP_CUR_STATE) &&
2628 !ionic_qp_cur_state_is_ok(qp->state, attr->cur_qp_state))
2629 return -EINVAL;
2630
2631 if (!ib_modify_qp_is_ok(cur_state, next_state, qp->ibqp.qp_type, mask) ||
2632 !ionic_is_modify_ok(mask & ~IB_QP_ATTR_STANDARD_BITS,
2633 qp->ibqp.qp_type, cur_state, next_state))
2634 return -EINVAL;
2635
2636 /* unprivileged qp not allowed privileged qkey */
2637 if ((mask & IB_QP_QKEY) && (attr->qkey & 0x80000000) &&
2638 qp->ibqp.uobject)
2639 return -EPERM;
2640
2641 return 0;
2642}
2643
2644int ionic_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int mask,
2645 struct ib_udata *udata)
2646{
2647 struct ionic_ibdev *dev = to_ionic_ibdev(ibqp->device);
2648 struct ionic_pd *pd = to_ionic_pd(ibqp->pd);
2649 struct ionic_qp *qp = to_ionic_qp(ibqp);
2650 int rc;
2651
2652 rc = ib_no_udata_io(udata);
2653 if (rc)
2654 return rc;
2655
2656 rc = ionic_check_modify_qp(qp, attr, mask);
2657 if (rc)
2658 return rc;
2659
2660 if (mask & IB_QP_CAP)
2661 return -EINVAL;
2662
2663 rc = ionic_modify_qp_cmd(dev, pd, qp, attr, mask);
2664 if (rc)
2665 return rc;
2666
2667 if (mask & IB_QP_STATE) {
2668 qp->state = attr->qp_state;
2669
2670 if (attr->qp_state == IB_QPS_ERR) {
2671 ionic_flush_qp(dev, qp);
2672 ionic_notify_qp_cqs(dev, qp);
2673 } else if (attr->qp_state == IB_QPS_RESET) {
2674 ionic_reset_qp(dev, qp);
2675 }
2676 }
2677
2678 return 0;
2679}
2680
2681int ionic_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
2682 int mask, struct ib_qp_init_attr *init_attr)
2683{
2684 struct ionic_ibdev *dev = to_ionic_ibdev(ibqp->device);
2685 struct ionic_qp *qp = to_ionic_qp(ibqp);
2686 int rc;
2687
2688 memset(attr, 0, sizeof(*attr));
2689 memset(init_attr, 0, sizeof(*init_attr));
2690
2691 rc = ionic_query_qp_cmd(dev, qp, attr, mask);
2692 if (rc)
2693 return rc;
2694
2695 if (qp->has_sq)
2696 attr->cap.max_send_wr = qp->sq.mask;
2697
2698 if (qp->has_rq)
2699 attr->cap.max_recv_wr = qp->rq.mask;
2700
2701 init_attr->event_handler = ibqp->event_handler;
2702 init_attr->qp_context = ibqp->qp_context;
2703 init_attr->send_cq = ibqp->send_cq;
2704 init_attr->recv_cq = ibqp->recv_cq;
2705 init_attr->srq = ibqp->srq;
2706 init_attr->xrcd = ibqp->xrcd;
2707 init_attr->cap = attr->cap;
2708 init_attr->sq_sig_type = qp->sig_all ?
2709 IB_SIGNAL_ALL_WR : IB_SIGNAL_REQ_WR;
2710 init_attr->qp_type = ibqp->qp_type;
2711 init_attr->create_flags = 0;
2712 init_attr->port_num = 0;
2713 init_attr->rwq_ind_tbl = ibqp->rwq_ind_tbl;
2714 init_attr->source_qpn = 0;
2715
2716 return rc;
2717}
2718
2719int ionic_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata)
2720{
2721 struct ionic_ctx *ctx =
2722 rdma_udata_to_drv_context(udata, struct ionic_ctx, ibctx);
2723 struct ionic_ibdev *dev = to_ionic_ibdev(ibqp->device);
2724 struct ionic_qp *qp = to_ionic_qp(ibqp);
2725 unsigned long irqflags;
2726 struct ionic_cq *cq;
2727 int rc;
2728
2729 rc = ib_no_udata_io(udata);
2730 if (rc)
2731 return rc;
2732
2733 rc = ionic_destroy_qp_cmd(dev, qp->qpid);
2734 if (rc)
2735 return rc;
2736
2737 xa_erase_irq(&dev->qp_tbl, qp->qpid);
2738
2739 kref_put(&qp->qp_kref, ionic_qp_complete);
2740 wait_for_completion(&qp->qp_rel_comp);
2741
2742 if (qp->ibqp.send_cq) {
2743 cq = to_ionic_vcq_cq(qp->ibqp.send_cq, qp->udma_idx);
2744 spin_lock_irqsave(&cq->lock, irqflags);
2745 ionic_clean_cq(cq, qp->qpid);
2746 list_del(&qp->cq_poll_sq);
2747 list_del(&qp->cq_flush_sq);
2748 spin_unlock_irqrestore(&cq->lock, irqflags);
2749 }
2750
2751 if (qp->ibqp.recv_cq) {
2752 cq = to_ionic_vcq_cq(qp->ibqp.recv_cq, qp->udma_idx);
2753 spin_lock_irqsave(&cq->lock, irqflags);
2754 ionic_clean_cq(cq, qp->qpid);
2755 list_del(&qp->cq_flush_rq);
2756 spin_unlock_irqrestore(&cq->lock, irqflags);
2757 }
2758
2759 ionic_qp_rq_destroy(dev, ctx, qp);
2760 ionic_qp_sq_destroy(dev, ctx, qp);
2761 if (qp->has_ah) {
2762 ionic_put_ahid(dev, qp->ahid);
2763 kfree(qp->hdr);
2764 }
2765 ionic_put_qpid(dev, qp->qpid);
2766
2767 return 0;
2768}