projects/linux·blob master · refs

1400 lines · 32.3 KiB

1// SPDX-License-Identifier: GPL-2.0
2/* Copyright (C) 2018-2025, Advanced Micro Devices, Inc. */
3
4#include <linux/module.h>
5#include <linux/printk.h>
6#include <rdma/ib_addr.h>
7#include <rdma/ib_user_verbs.h>
8
9#include "ionic_fw.h"
10#include "ionic_ibdev.h"
11
12#define IONIC_OP(version, opname) \
13 ((version) < 2 ? IONIC_V1_OP_##opname : IONIC_V2_OP_##opname)
14
15static bool ionic_next_cqe(struct ionic_ibdev *dev, struct ionic_cq *cq,
16 struct ionic_v1_cqe **cqe)
17{
18 struct ionic_v1_cqe *qcqe = ionic_queue_at_prod(&cq->q);
19
20 if (unlikely(cq->color != ionic_v1_cqe_color(qcqe)))
21 return false;
22
23 /* Prevent out-of-order reads of the CQE */
24 dma_rmb();
25
26 *cqe = qcqe;
27
28 return true;
29}
30
31static int ionic_flush_recv(struct ionic_qp *qp, struct ib_wc *wc)
32{
33 struct ionic_rq_meta *meta;
34 struct ionic_v1_wqe *wqe;
35 u64 wqe_idx;
36
37 if (!qp->rq_flush)
38 return 0;
39
40 if (ionic_queue_empty(&qp->rq))
41 return 0;
42
43 wqe = ionic_queue_at_cons(&qp->rq);
44 wqe_idx = le64_to_cpu(wqe->base.wqe_idx);
45
46 /* wqe_idx must be a valid queue index */
47 if (unlikely(wqe_idx >> qp->rq.depth_log2)) {
48 ibdev_warn(qp->ibqp.device,
49 "flush qp %u recv index %llu invalid\n",
50 qp->qpid, (unsigned long long)wqe_idx);
51 return -EIO;
52 }
53
54 /* wqe_idx must indicate a request that is outstanding */
55 meta = &qp->rq_meta[wqe_idx];
56 if (unlikely(meta->next != IONIC_META_POSTED)) {
57 ibdev_warn(qp->ibqp.device,
58 "flush qp %u recv index %llu not posted\n",
59 qp->qpid, (unsigned long long)wqe_idx);
60 return -EIO;
61 }
62
63 ionic_queue_consume(&qp->rq);
64
65 memset(wc, 0, sizeof(*wc));
66
67 wc->status = IB_WC_WR_FLUSH_ERR;
68 wc->wr_id = meta->wrid;
69 wc->qp = &qp->ibqp;
70
71 meta->next = qp->rq_meta_head;
72 qp->rq_meta_head = meta;
73
74 return 1;
75}
76
77static int ionic_flush_recv_many(struct ionic_qp *qp,
78 struct ib_wc *wc, int nwc)
79{
80 int rc = 0, npolled = 0;
81
82 while (npolled < nwc) {
83 rc = ionic_flush_recv(qp, wc + npolled);
84 if (rc <= 0)
85 break;
86
87 npolled += rc;
88 }
89
90 return npolled ?: rc;
91}
92
93static int ionic_flush_send(struct ionic_qp *qp, struct ib_wc *wc)
94{
95 struct ionic_sq_meta *meta;
96
97 if (!qp->sq_flush)
98 return 0;
99
100 if (ionic_queue_empty(&qp->sq))
101 return 0;
102
103 meta = &qp->sq_meta[qp->sq.cons];
104
105 ionic_queue_consume(&qp->sq);
106
107 memset(wc, 0, sizeof(*wc));
108
109 wc->status = IB_WC_WR_FLUSH_ERR;
110 wc->wr_id = meta->wrid;
111 wc->qp = &qp->ibqp;
112
113 return 1;
114}
115
116static int ionic_flush_send_many(struct ionic_qp *qp,
117 struct ib_wc *wc, int nwc)
118{
119 int rc = 0, npolled = 0;
120
121 while (npolled < nwc) {
122 rc = ionic_flush_send(qp, wc + npolled);
123 if (rc <= 0)
124 break;
125
126 npolled += rc;
127 }
128
129 return npolled ?: rc;
130}
131
132static int ionic_poll_recv(struct ionic_ibdev *dev, struct ionic_cq *cq,
133 struct ionic_qp *cqe_qp, struct ionic_v1_cqe *cqe,
134 struct ib_wc *wc)
135{
136 struct ionic_qp *qp = NULL;
137 struct ionic_rq_meta *meta;
138 u16 vlan_tag, wqe_idx;
139 u32 src_qpn, st_len;
140 u8 op;
141
142 if (cqe_qp->rq_flush)
143 return 0;
144
145 qp = cqe_qp;
146
147 st_len = be32_to_cpu(cqe->status_length);
148
149 /* ignore wqe_idx in case of flush error */
150 if (ionic_v1_cqe_error(cqe) && st_len == IONIC_STS_WQE_FLUSHED_ERR) {
151 cqe_qp->rq_flush = true;
152 cq->flush = true;
153 list_move_tail(&qp->cq_flush_rq, &cq->flush_rq);
154
155 /* posted recvs (if any) flushed by ionic_flush_recv */
156 return 0;
157 }
158
159 /* there had better be something in the recv queue to complete */
160 if (ionic_queue_empty(&qp->rq)) {
161 ibdev_warn(&dev->ibdev, "qp %u is empty\n", qp->qpid);
162 return -EIO;
163 }
164
165 wqe_idx = le64_to_cpu(cqe->recv.wqe_idx_timestamp) & IONIC_V1_CQE_WQE_IDX_MASK;
166 /* wqe_idx must be a valid queue index */
167 if (unlikely(wqe_idx >> qp->rq.depth_log2)) {
168 ibdev_warn(&dev->ibdev,
169 "qp %u recv index %u invalid\n", qp->qpid, wqe_idx);
170 return -EIO;
171 }
172
173 /* wqe_idx must indicate a request that is outstanding */
174 meta = &qp->rq_meta[wqe_idx];
175 if (unlikely(meta->next != IONIC_META_POSTED)) {
176 ibdev_warn(&dev->ibdev,
177 "qp %u recv index %u not posted\n", qp->qpid, wqe_idx);
178 return -EIO;
179 }
180
181 meta->next = qp->rq_meta_head;
182 qp->rq_meta_head = meta;
183
184 memset(wc, 0, sizeof(*wc));
185
186 wc->wr_id = meta->wrid;
187
188 wc->qp = &cqe_qp->ibqp;
189
190 if (ionic_v1_cqe_error(cqe)) {
191 wc->vendor_err = st_len;
192 wc->status = ionic_to_ib_status(st_len);
193
194 cqe_qp->rq_flush = true;
195 cq->flush = true;
196 list_move_tail(&qp->cq_flush_rq, &cq->flush_rq);
197
198 ibdev_warn(&dev->ibdev,
199 "qp %d recv cqe with error\n", qp->qpid);
200 print_hex_dump(KERN_WARNING, "cqe ", DUMP_PREFIX_OFFSET, 16, 1,
201 cqe, BIT(cq->q.stride_log2), true);
202 goto out;
203 }
204
205 wc->vendor_err = 0;
206 wc->status = IB_WC_SUCCESS;
207
208 src_qpn = be32_to_cpu(cqe->recv.src_qpn_op);
209 op = src_qpn >> IONIC_V1_CQE_RECV_OP_SHIFT;
210
211 src_qpn &= IONIC_V1_CQE_RECV_QPN_MASK;
212 op &= IONIC_V1_CQE_RECV_OP_MASK;
213
214 wc->opcode = IB_WC_RECV;
215 switch (op) {
216 case IONIC_V1_CQE_RECV_OP_RDMA_IMM:
217 wc->opcode = IB_WC_RECV_RDMA_WITH_IMM;
218 wc->wc_flags |= IB_WC_WITH_IMM;
219 wc->ex.imm_data = cqe->recv.imm_data_rkey; /* be32 in wc */
220 break;
221 case IONIC_V1_CQE_RECV_OP_SEND_IMM:
222 wc->wc_flags |= IB_WC_WITH_IMM;
223 wc->ex.imm_data = cqe->recv.imm_data_rkey; /* be32 in wc */
224 break;
225 case IONIC_V1_CQE_RECV_OP_SEND_INV:
226 wc->wc_flags |= IB_WC_WITH_INVALIDATE;
227 wc->ex.invalidate_rkey = be32_to_cpu(cqe->recv.imm_data_rkey);
228 break;
229 }
230
231 wc->byte_len = st_len;
232 wc->src_qp = src_qpn;
233
234 if (qp->ibqp.qp_type == IB_QPT_UD ||
235 qp->ibqp.qp_type == IB_QPT_GSI) {
236 wc->wc_flags |= IB_WC_GRH | IB_WC_WITH_SMAC;
237 ether_addr_copy(wc->smac, cqe->recv.src_mac);
238
239 wc->wc_flags |= IB_WC_WITH_NETWORK_HDR_TYPE;
240 if (ionic_v1_cqe_recv_is_ipv4(cqe))
241 wc->network_hdr_type = RDMA_NETWORK_IPV4;
242 else
243 wc->network_hdr_type = RDMA_NETWORK_IPV6;
244
245 if (ionic_v1_cqe_recv_is_vlan(cqe))
246 wc->wc_flags |= IB_WC_WITH_VLAN;
247
248 /* vlan_tag in cqe will be valid from dpath even if no vlan */
249 vlan_tag = be16_to_cpu(cqe->recv.vlan_tag);
250 wc->vlan_id = vlan_tag & 0xfff; /* 802.1q VID */
251 wc->sl = vlan_tag >> VLAN_PRIO_SHIFT; /* 802.1q PCP */
252 }
253
254 wc->pkey_index = 0;
255 wc->port_num = 1;
256
257out:
258 ionic_queue_consume(&qp->rq);
259
260 return 1;
261}
262
263static bool ionic_peek_send(struct ionic_qp *qp)
264{
265 struct ionic_sq_meta *meta;
266
267 if (qp->sq_flush)
268 return false;
269
270 /* completed all send queue requests */
271 if (ionic_queue_empty(&qp->sq))
272 return false;
273
274 meta = &qp->sq_meta[qp->sq.cons];
275
276 /* waiting for remote completion */
277 if (meta->remote && meta->seq == qp->sq_msn_cons)
278 return false;
279
280 /* waiting for local completion */
281 if (!meta->remote && !meta->local_comp)
282 return false;
283
284 return true;
285}
286
287static int ionic_poll_send(struct ionic_ibdev *dev, struct ionic_cq *cq,
288 struct ionic_qp *qp, struct ib_wc *wc)
289{
290 struct ionic_sq_meta *meta;
291
292 if (qp->sq_flush)
293 return 0;
294
295 do {
296 /* completed all send queue requests */
297 if (ionic_queue_empty(&qp->sq))
298 goto out_empty;
299
300 meta = &qp->sq_meta[qp->sq.cons];
301
302 /* waiting for remote completion */
303 if (meta->remote && meta->seq == qp->sq_msn_cons)
304 goto out_empty;
305
306 /* waiting for local completion */
307 if (!meta->remote && !meta->local_comp)
308 goto out_empty;
309
310 ionic_queue_consume(&qp->sq);
311
312 /* produce wc only if signaled or error status */
313 } while (!meta->signal && meta->ibsts == IB_WC_SUCCESS);
314
315 memset(wc, 0, sizeof(*wc));
316
317 wc->status = meta->ibsts;
318 wc->wr_id = meta->wrid;
319 wc->qp = &qp->ibqp;
320
321 if (meta->ibsts == IB_WC_SUCCESS) {
322 wc->byte_len = meta->len;
323 wc->opcode = meta->ibop;
324 } else {
325 wc->vendor_err = meta->len;
326
327 qp->sq_flush = true;
328 cq->flush = true;
329 list_move_tail(&qp->cq_flush_sq, &cq->flush_sq);
330 }
331
332 return 1;
333
334out_empty:
335 if (qp->sq_flush_rcvd) {
336 qp->sq_flush = true;
337 cq->flush = true;
338 list_move_tail(&qp->cq_flush_sq, &cq->flush_sq);
339 }
340 return 0;
341}
342
343static int ionic_poll_send_many(struct ionic_ibdev *dev, struct ionic_cq *cq,
344 struct ionic_qp *qp, struct ib_wc *wc, int nwc)
345{
346 int rc = 0, npolled = 0;
347
348 while (npolled < nwc) {
349 rc = ionic_poll_send(dev, cq, qp, wc + npolled);
350 if (rc <= 0)
351 break;
352
353 npolled += rc;
354 }
355
356 return npolled ?: rc;
357}
358
359static int ionic_validate_cons(u16 prod, u16 cons,
360 u16 comp, u16 mask)
361{
362 if (((prod - cons) & mask) <= ((comp - cons) & mask))
363 return -EIO;
364
365 return 0;
366}
367
368static int ionic_comp_msn(struct ionic_qp *qp, struct ionic_v1_cqe *cqe)
369{
370 struct ionic_sq_meta *meta;
371 u16 cqe_seq, cqe_idx;
372 int rc;
373
374 if (qp->sq_flush)
375 return 0;
376
377 cqe_seq = be32_to_cpu(cqe->send.msg_msn) & qp->sq.mask;
378
379 rc = ionic_validate_cons(qp->sq_msn_prod,
380 qp->sq_msn_cons,
381 cqe_seq - 1,
382 qp->sq.mask);
383 if (rc) {
384 ibdev_warn(qp->ibqp.device,
385 "qp %u bad msn %#x seq %u for prod %u cons %u\n",
386 qp->qpid, be32_to_cpu(cqe->send.msg_msn),
387 cqe_seq, qp->sq_msn_prod, qp->sq_msn_cons);
388 return rc;
389 }
390
391 qp->sq_msn_cons = cqe_seq;
392
393 if (ionic_v1_cqe_error(cqe)) {
394 cqe_idx = qp->sq_msn_idx[(cqe_seq - 1) & qp->sq.mask];
395
396 meta = &qp->sq_meta[cqe_idx];
397 meta->len = be32_to_cpu(cqe->status_length);
398 meta->ibsts = ionic_to_ib_status(meta->len);
399
400 ibdev_warn(qp->ibqp.device,
401 "qp %d msn cqe with error\n", qp->qpid);
402 print_hex_dump(KERN_WARNING, "cqe ", DUMP_PREFIX_OFFSET, 16, 1,
403 cqe, sizeof(*cqe), true);
404 }
405
406 return 0;
407}
408
409static int ionic_comp_npg(struct ionic_qp *qp, struct ionic_v1_cqe *cqe)
410{
411 struct ionic_sq_meta *meta;
412 u16 wqe_idx;
413 u32 st_len;
414
415 if (qp->sq_flush)
416 return 0;
417
418 st_len = be32_to_cpu(cqe->status_length);
419
420 if (ionic_v1_cqe_error(cqe) && st_len == IONIC_STS_WQE_FLUSHED_ERR) {
421 /*
422 * Flush cqe does not consume a wqe on the device, and maybe
423 * no such work request is posted.
424 *
425 * The driver should begin flushing after the last indicated
426 * normal or error completion. Here, only set a hint that the
427 * flush request was indicated. In poll_send, if nothing more
428 * can be polled normally, then begin flushing.
429 */
430 qp->sq_flush_rcvd = true;
431 return 0;
432 }
433
434 wqe_idx = le64_to_cpu(cqe->send.npg_wqe_idx_timestamp) & qp->sq.mask;
435 meta = &qp->sq_meta[wqe_idx];
436 meta->local_comp = true;
437
438 if (ionic_v1_cqe_error(cqe)) {
439 meta->len = st_len;
440 meta->ibsts = ionic_to_ib_status(st_len);
441 meta->remote = false;
442 ibdev_warn(qp->ibqp.device,
443 "qp %d npg cqe with error\n", qp->qpid);
444 print_hex_dump(KERN_WARNING, "cqe ", DUMP_PREFIX_OFFSET, 16, 1,
445 cqe, sizeof(*cqe), true);
446 }
447
448 return 0;
449}
450
451static void ionic_reserve_sync_cq(struct ionic_ibdev *dev, struct ionic_cq *cq)
452{
453 if (!ionic_queue_empty(&cq->q)) {
454 cq->credit += ionic_queue_length(&cq->q);
455 cq->q.cons = cq->q.prod;
456
457 ionic_dbell_ring(dev->lif_cfg.dbpage, dev->lif_cfg.cq_qtype,
458 ionic_queue_dbell_val(&cq->q));
459 }
460}
461
462static void ionic_reserve_cq(struct ionic_ibdev *dev, struct ionic_cq *cq,
463 int spend)
464{
465 cq->credit -= spend;
466
467 if (cq->credit <= 0)
468 ionic_reserve_sync_cq(dev, cq);
469}
470
471static int ionic_poll_vcq_cq(struct ionic_ibdev *dev,
472 struct ionic_cq *cq,
473 int nwc, struct ib_wc *wc)
474{
475 struct ionic_qp *qp, *qp_next;
476 struct ionic_v1_cqe *cqe;
477 int rc = 0, npolled = 0;
478 unsigned long irqflags;
479 u32 qtf, qid;
480 bool peek;
481 u8 type;
482
483 if (nwc < 1)
484 return 0;
485
486 spin_lock_irqsave(&cq->lock, irqflags);
487
488 /* poll already indicated work completions for send queue */
489 list_for_each_entry_safe(qp, qp_next, &cq->poll_sq, cq_poll_sq) {
490 if (npolled == nwc)
491 goto out;
492
493 spin_lock(&qp->sq_lock);
494 rc = ionic_poll_send_many(dev, cq, qp, wc + npolled,
495 nwc - npolled);
496 spin_unlock(&qp->sq_lock);
497
498 if (rc > 0)
499 npolled += rc;
500
501 if (npolled < nwc)
502 list_del_init(&qp->cq_poll_sq);
503 }
504
505 /* poll for more work completions */
506 while (likely(ionic_next_cqe(dev, cq, &cqe))) {
507 if (npolled == nwc)
508 goto out;
509
510 qtf = ionic_v1_cqe_qtf(cqe);
511 qid = ionic_v1_cqe_qtf_qid(qtf);
512 type = ionic_v1_cqe_qtf_type(qtf);
513
514 /*
515 * Safe to access QP without additional reference here as,
516 * 1. We hold cq->lock throughout
517 * 2. ionic_destroy_qp() acquires the same cq->lock before cleanup
518 * 3. QP is removed from qp_tbl before any cleanup begins
519 * This ensures no concurrent access between polling and destruction.
520 */
521 qp = xa_load(&dev->qp_tbl, qid);
522 if (unlikely(!qp)) {
523 ibdev_dbg(&dev->ibdev, "missing qp for qid %u\n", qid);
524 goto cq_next;
525 }
526
527 switch (type) {
528 case IONIC_V1_CQE_TYPE_RECV:
529 spin_lock(&qp->rq_lock);
530 rc = ionic_poll_recv(dev, cq, qp, cqe, wc + npolled);
531 spin_unlock(&qp->rq_lock);
532
533 if (rc < 0)
534 goto out;
535
536 npolled += rc;
537
538 break;
539
540 case IONIC_V1_CQE_TYPE_SEND_MSN:
541 spin_lock(&qp->sq_lock);
542 rc = ionic_comp_msn(qp, cqe);
543 if (!rc) {
544 rc = ionic_poll_send_many(dev, cq, qp,
545 wc + npolled,
546 nwc - npolled);
547 peek = ionic_peek_send(qp);
548 }
549 spin_unlock(&qp->sq_lock);
550
551 if (rc < 0)
552 goto out;
553
554 npolled += rc;
555
556 if (peek)
557 list_move_tail(&qp->cq_poll_sq, &cq->poll_sq);
558 break;
559
560 case IONIC_V1_CQE_TYPE_SEND_NPG:
561 spin_lock(&qp->sq_lock);
562 rc = ionic_comp_npg(qp, cqe);
563 if (!rc) {
564 rc = ionic_poll_send_many(dev, cq, qp,
565 wc + npolled,
566 nwc - npolled);
567 peek = ionic_peek_send(qp);
568 }
569 spin_unlock(&qp->sq_lock);
570
571 if (rc < 0)
572 goto out;
573
574 npolled += rc;
575
576 if (peek)
577 list_move_tail(&qp->cq_poll_sq, &cq->poll_sq);
578 break;
579
580 default:
581 ibdev_warn(&dev->ibdev,
582 "unexpected cqe type %u\n", type);
583 rc = -EIO;
584 goto out;
585 }
586
587cq_next:
588 ionic_queue_produce(&cq->q);
589 cq->color = ionic_color_wrap(cq->q.prod, cq->color);
590 }
591
592 /* lastly, flush send and recv queues */
593 if (likely(!cq->flush))
594 goto out;
595
596 cq->flush = false;
597
598 list_for_each_entry_safe(qp, qp_next, &cq->flush_sq, cq_flush_sq) {
599 if (npolled == nwc)
600 goto out;
601
602 spin_lock(&qp->sq_lock);
603 rc = ionic_flush_send_many(qp, wc + npolled, nwc - npolled);
604 spin_unlock(&qp->sq_lock);
605
606 if (rc > 0)
607 npolled += rc;
608
609 if (npolled < nwc)
610 list_del_init(&qp->cq_flush_sq);
611 else
612 cq->flush = true;
613 }
614
615 list_for_each_entry_safe(qp, qp_next, &cq->flush_rq, cq_flush_rq) {
616 if (npolled == nwc)
617 goto out;
618
619 spin_lock(&qp->rq_lock);
620 rc = ionic_flush_recv_many(qp, wc + npolled, nwc - npolled);
621 spin_unlock(&qp->rq_lock);
622
623 if (rc > 0)
624 npolled += rc;
625
626 if (npolled < nwc)
627 list_del_init(&qp->cq_flush_rq);
628 else
629 cq->flush = true;
630 }
631
632out:
633 /* in case credit was depleted (more work posted than cq depth) */
634 if (cq->credit <= 0)
635 ionic_reserve_sync_cq(dev, cq);
636
637 spin_unlock_irqrestore(&cq->lock, irqflags);
638
639 return npolled ?: rc;
640}
641
642int ionic_poll_cq(struct ib_cq *ibcq, int nwc, struct ib_wc *wc)
643{
644 struct ionic_ibdev *dev = to_ionic_ibdev(ibcq->device);
645 struct ionic_vcq *vcq = to_ionic_vcq(ibcq);
646 int rc_tmp, rc = 0, npolled = 0;
647 int cq_i, cq_x, cq_ix;
648
649 cq_x = vcq->poll_idx;
650 vcq->poll_idx ^= dev->lif_cfg.udma_count - 1;
651
652 for (cq_i = 0; npolled < nwc && cq_i < dev->lif_cfg.udma_count; ++cq_i) {
653 cq_ix = cq_i ^ cq_x;
654
655 if (!(vcq->udma_mask & BIT(cq_ix)))
656 continue;
657
658 rc_tmp = ionic_poll_vcq_cq(dev, &vcq->cq[cq_ix],
659 nwc - npolled,
660 wc + npolled);
661
662 if (rc_tmp >= 0)
663 npolled += rc_tmp;
664 else if (!rc)
665 rc = rc_tmp;
666 }
667
668 return npolled ?: rc;
669}
670
671static int ionic_req_notify_vcq_cq(struct ionic_ibdev *dev, struct ionic_cq *cq,
672 enum ib_cq_notify_flags flags)
673{
674 u64 dbell_val = cq->q.dbell;
675
676 if (flags & IB_CQ_SOLICITED) {
677 cq->arm_sol_prod = ionic_queue_next(&cq->q, cq->arm_sol_prod);
678 dbell_val |= cq->arm_sol_prod | IONIC_CQ_RING_SOL;
679 } else {
680 cq->arm_any_prod = ionic_queue_next(&cq->q, cq->arm_any_prod);
681 dbell_val |= cq->arm_any_prod | IONIC_CQ_RING_ARM;
682 }
683
684 ionic_reserve_sync_cq(dev, cq);
685
686 ionic_dbell_ring(dev->lif_cfg.dbpage, dev->lif_cfg.cq_qtype, dbell_val);
687
688 /*
689 * IB_CQ_REPORT_MISSED_EVENTS:
690 *
691 * The queue index in ring zero guarantees no missed events.
692 *
693 * Here, we check if the color bit in the next cqe is flipped. If it
694 * is flipped, then progress can be made by immediately polling the cq.
695 * Still, the cq will be armed, and an event will be generated. The cq
696 * may be empty when polled after the event, because the next poll
697 * after arming the cq can empty it.
698 */
699 return (flags & IB_CQ_REPORT_MISSED_EVENTS) &&
700 cq->color == ionic_v1_cqe_color(ionic_queue_at_prod(&cq->q));
701}
702
703int ionic_req_notify_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags)
704{
705 struct ionic_ibdev *dev = to_ionic_ibdev(ibcq->device);
706 struct ionic_vcq *vcq = to_ionic_vcq(ibcq);
707 int rc = 0, cq_i;
708
709 for (cq_i = 0; cq_i < dev->lif_cfg.udma_count; ++cq_i) {
710 if (!(vcq->udma_mask & BIT(cq_i)))
711 continue;
712
713 if (ionic_req_notify_vcq_cq(dev, &vcq->cq[cq_i], flags))
714 rc = 1;
715 }
716
717 return rc;
718}
719
720static s64 ionic_prep_inline(void *data, u32 max_data,
721 const struct ib_sge *ib_sgl, int num_sge)
722{
723 static const s64 bit_31 = 1u << 31;
724 s64 len = 0, sg_len;
725 int sg_i;
726
727 for (sg_i = 0; sg_i < num_sge; ++sg_i) {
728 sg_len = ib_sgl[sg_i].length;
729
730 /* sge length zero means 2GB */
731 if (unlikely(sg_len == 0))
732 sg_len = bit_31;
733
734 /* greater than max inline data is invalid */
735 if (unlikely(len + sg_len > max_data))
736 return -EINVAL;
737
738 memcpy(data + len, (void *)ib_sgl[sg_i].addr, sg_len);
739
740 len += sg_len;
741 }
742
743 return len;
744}
745
746static s64 ionic_prep_pld(struct ionic_v1_wqe *wqe,
747 union ionic_v1_pld *pld,
748 int spec, u32 max_sge,
749 const struct ib_sge *ib_sgl,
750 int num_sge)
751{
752 static const s64 bit_31 = 1l << 31;
753 struct ionic_sge *sgl;
754 __be32 *spec32 = NULL;
755 __be16 *spec16 = NULL;
756 s64 len = 0, sg_len;
757 int sg_i = 0;
758
759 if (unlikely(num_sge < 0 || (u32)num_sge > max_sge))
760 return -EINVAL;
761
762 if (spec && num_sge > IONIC_V1_SPEC_FIRST_SGE) {
763 sg_i = IONIC_V1_SPEC_FIRST_SGE;
764
765 if (num_sge > 8) {
766 wqe->base.flags |= cpu_to_be16(IONIC_V1_FLAG_SPEC16);
767 spec16 = pld->spec16;
768 } else {
769 wqe->base.flags |= cpu_to_be16(IONIC_V1_FLAG_SPEC32);
770 spec32 = pld->spec32;
771 }
772 }
773
774 sgl = &pld->sgl[sg_i];
775
776 for (sg_i = 0; sg_i < num_sge; ++sg_i) {
777 sg_len = ib_sgl[sg_i].length;
778
779 /* sge length zero means 2GB */
780 if (unlikely(sg_len == 0))
781 sg_len = bit_31;
782
783 /* greater than 2GB data is invalid */
784 if (unlikely(len + sg_len > bit_31))
785 return -EINVAL;
786
787 sgl[sg_i].va = cpu_to_be64(ib_sgl[sg_i].addr);
788 sgl[sg_i].len = cpu_to_be32(sg_len);
789 sgl[sg_i].lkey = cpu_to_be32(ib_sgl[sg_i].lkey);
790
791 if (spec32) {
792 spec32[sg_i] = sgl[sg_i].len;
793 } else if (spec16) {
794 if (unlikely(sg_len > U16_MAX))
795 return -EINVAL;
796 spec16[sg_i] = cpu_to_be16(sg_len);
797 }
798
799 len += sg_len;
800 }
801
802 return len;
803}
804
805static void ionic_prep_base(struct ionic_qp *qp,
806 const struct ib_send_wr *wr,
807 struct ionic_sq_meta *meta,
808 struct ionic_v1_wqe *wqe)
809{
810 meta->wrid = wr->wr_id;
811 meta->ibsts = IB_WC_SUCCESS;
812 meta->signal = false;
813 meta->local_comp = false;
814
815 wqe->base.wqe_idx = cpu_to_le64(qp->sq.prod);
816
817 if (wr->send_flags & IB_SEND_FENCE)
818 wqe->base.flags |= cpu_to_be16(IONIC_V1_FLAG_FENCE);
819
820 if (wr->send_flags & IB_SEND_SOLICITED)
821 wqe->base.flags |= cpu_to_be16(IONIC_V1_FLAG_SOL);
822
823 if (qp->sig_all || wr->send_flags & IB_SEND_SIGNALED) {
824 wqe->base.flags |= cpu_to_be16(IONIC_V1_FLAG_SIG);
825 meta->signal = true;
826 }
827
828 meta->seq = qp->sq_msn_prod;
829 meta->remote =
830 qp->ibqp.qp_type != IB_QPT_UD &&
831 qp->ibqp.qp_type != IB_QPT_GSI &&
832 !ionic_ibop_is_local(wr->opcode);
833
834 if (meta->remote) {
835 qp->sq_msn_idx[meta->seq] = qp->sq.prod;
836 qp->sq_msn_prod = ionic_queue_next(&qp->sq, qp->sq_msn_prod);
837 }
838
839 ionic_queue_produce(&qp->sq);
840}
841
842static int ionic_prep_common(struct ionic_qp *qp,
843 const struct ib_send_wr *wr,
844 struct ionic_sq_meta *meta,
845 struct ionic_v1_wqe *wqe)
846{
847 s64 signed_len;
848 u32 mval;
849
850 if (wr->send_flags & IB_SEND_INLINE) {
851 wqe->base.num_sge_key = 0;
852 wqe->base.flags |= cpu_to_be16(IONIC_V1_FLAG_INL);
853 mval = ionic_v1_send_wqe_max_data(qp->sq.stride_log2, false);
854 signed_len = ionic_prep_inline(wqe->common.pld.data, mval,
855 wr->sg_list, wr->num_sge);
856 } else {
857 wqe->base.num_sge_key = wr->num_sge;
858 mval = ionic_v1_send_wqe_max_sge(qp->sq.stride_log2,
859 qp->sq_spec,
860 false);
861 signed_len = ionic_prep_pld(wqe, &wqe->common.pld,
862 qp->sq_spec, mval,
863 wr->sg_list, wr->num_sge);
864 }
865
866 if (unlikely(signed_len < 0))
867 return signed_len;
868
869 meta->len = signed_len;
870 wqe->common.length = cpu_to_be32(signed_len);
871
872 ionic_prep_base(qp, wr, meta, wqe);
873
874 return 0;
875}
876
877static void ionic_prep_sq_wqe(struct ionic_qp *qp, void *wqe)
878{
879 memset(wqe, 0, 1u << qp->sq.stride_log2);
880}
881
882static void ionic_prep_rq_wqe(struct ionic_qp *qp, void *wqe)
883{
884 memset(wqe, 0, 1u << qp->rq.stride_log2);
885}
886
887static int ionic_prep_send(struct ionic_qp *qp,
888 const struct ib_send_wr *wr)
889{
890 struct ionic_ibdev *dev = to_ionic_ibdev(qp->ibqp.device);
891 struct ionic_sq_meta *meta;
892 struct ionic_v1_wqe *wqe;
893
894 meta = &qp->sq_meta[qp->sq.prod];
895 wqe = ionic_queue_at_prod(&qp->sq);
896
897 ionic_prep_sq_wqe(qp, wqe);
898
899 meta->ibop = IB_WC_SEND;
900
901 switch (wr->opcode) {
902 case IB_WR_SEND:
903 wqe->base.op = IONIC_OP(dev->lif_cfg.rdma_version, SEND);
904 break;
905 case IB_WR_SEND_WITH_IMM:
906 wqe->base.op = IONIC_OP(dev->lif_cfg.rdma_version, SEND_IMM);
907 wqe->base.imm_data_key = wr->ex.imm_data;
908 break;
909 case IB_WR_SEND_WITH_INV:
910 wqe->base.op = IONIC_OP(dev->lif_cfg.rdma_version, SEND_INV);
911 wqe->base.imm_data_key =
912 cpu_to_be32(wr->ex.invalidate_rkey);
913 break;
914 default:
915 return -EINVAL;
916 }
917
918 return ionic_prep_common(qp, wr, meta, wqe);
919}
920
921static int ionic_prep_send_ud(struct ionic_qp *qp,
922 const struct ib_ud_wr *wr)
923{
924 struct ionic_ibdev *dev = to_ionic_ibdev(qp->ibqp.device);
925 struct ionic_sq_meta *meta;
926 struct ionic_v1_wqe *wqe;
927 struct ionic_ah *ah;
928
929 if (unlikely(!wr->ah))
930 return -EINVAL;
931
932 ah = to_ionic_ah(wr->ah);
933
934 meta = &qp->sq_meta[qp->sq.prod];
935 wqe = ionic_queue_at_prod(&qp->sq);
936
937 ionic_prep_sq_wqe(qp, wqe);
938
939 wqe->common.send.ah_id = cpu_to_be32(ah->ahid);
940 wqe->common.send.dest_qpn = cpu_to_be32(wr->remote_qpn);
941 wqe->common.send.dest_qkey = cpu_to_be32(wr->remote_qkey);
942
943 meta->ibop = IB_WC_SEND;
944
945 switch (wr->wr.opcode) {
946 case IB_WR_SEND:
947 wqe->base.op = IONIC_OP(dev->lif_cfg.rdma_version, SEND);
948 break;
949 case IB_WR_SEND_WITH_IMM:
950 wqe->base.op = IONIC_OP(dev->lif_cfg.rdma_version, SEND_IMM);
951 wqe->base.imm_data_key = wr->wr.ex.imm_data;
952 break;
953 default:
954 return -EINVAL;
955 }
956
957 return ionic_prep_common(qp, &wr->wr, meta, wqe);
958}
959
960static int ionic_prep_rdma(struct ionic_qp *qp,
961 const struct ib_rdma_wr *wr)
962{
963 struct ionic_ibdev *dev = to_ionic_ibdev(qp->ibqp.device);
964 struct ionic_sq_meta *meta;
965 struct ionic_v1_wqe *wqe;
966
967 meta = &qp->sq_meta[qp->sq.prod];
968 wqe = ionic_queue_at_prod(&qp->sq);
969
970 ionic_prep_sq_wqe(qp, wqe);
971
972 meta->ibop = IB_WC_RDMA_WRITE;
973
974 switch (wr->wr.opcode) {
975 case IB_WR_RDMA_READ:
976 if (wr->wr.send_flags & (IB_SEND_SOLICITED | IB_SEND_INLINE))
977 return -EINVAL;
978 meta->ibop = IB_WC_RDMA_READ;
979 wqe->base.op = IONIC_OP(dev->lif_cfg.rdma_version, RDMA_READ);
980 break;
981 case IB_WR_RDMA_WRITE:
982 if (wr->wr.send_flags & IB_SEND_SOLICITED)
983 return -EINVAL;
984 wqe->base.op = IONIC_OP(dev->lif_cfg.rdma_version, RDMA_WRITE);
985 break;
986 case IB_WR_RDMA_WRITE_WITH_IMM:
987 wqe->base.op = IONIC_OP(dev->lif_cfg.rdma_version, RDMA_WRITE_IMM);
988 wqe->base.imm_data_key = wr->wr.ex.imm_data;
989 break;
990 default:
991 return -EINVAL;
992 }
993
994 wqe->common.rdma.remote_va_high = cpu_to_be32(wr->remote_addr >> 32);
995 wqe->common.rdma.remote_va_low = cpu_to_be32(wr->remote_addr);
996 wqe->common.rdma.remote_rkey = cpu_to_be32(wr->rkey);
997
998 return ionic_prep_common(qp, &wr->wr, meta, wqe);
999}
1000
1001static int ionic_prep_atomic(struct ionic_qp *qp,
1002 const struct ib_atomic_wr *wr)
1003{
1004 struct ionic_ibdev *dev = to_ionic_ibdev(qp->ibqp.device);
1005 struct ionic_sq_meta *meta;
1006 struct ionic_v1_wqe *wqe;
1007
1008 if (wr->wr.num_sge != 1 || wr->wr.sg_list[0].length != 8)
1009 return -EINVAL;
1010
1011 if (wr->wr.send_flags & (IB_SEND_SOLICITED | IB_SEND_INLINE))
1012 return -EINVAL;
1013
1014 meta = &qp->sq_meta[qp->sq.prod];
1015 wqe = ionic_queue_at_prod(&qp->sq);
1016
1017 ionic_prep_sq_wqe(qp, wqe);
1018
1019 meta->ibop = IB_WC_RDMA_WRITE;
1020
1021 switch (wr->wr.opcode) {
1022 case IB_WR_ATOMIC_CMP_AND_SWP:
1023 meta->ibop = IB_WC_COMP_SWAP;
1024 wqe->base.op = IONIC_OP(dev->lif_cfg.rdma_version, ATOMIC_CS);
1025 wqe->atomic.swap_add_high = cpu_to_be32(wr->swap >> 32);
1026 wqe->atomic.swap_add_low = cpu_to_be32(wr->swap);
1027 wqe->atomic.compare_high = cpu_to_be32(wr->compare_add >> 32);
1028 wqe->atomic.compare_low = cpu_to_be32(wr->compare_add);
1029 break;
1030 case IB_WR_ATOMIC_FETCH_AND_ADD:
1031 meta->ibop = IB_WC_FETCH_ADD;
1032 wqe->base.op = IONIC_OP(dev->lif_cfg.rdma_version, ATOMIC_FA);
1033 wqe->atomic.swap_add_high = cpu_to_be32(wr->compare_add >> 32);
1034 wqe->atomic.swap_add_low = cpu_to_be32(wr->compare_add);
1035 break;
1036 default:
1037 return -EINVAL;
1038 }
1039
1040 wqe->atomic.remote_va_high = cpu_to_be32(wr->remote_addr >> 32);
1041 wqe->atomic.remote_va_low = cpu_to_be32(wr->remote_addr);
1042 wqe->atomic.remote_rkey = cpu_to_be32(wr->rkey);
1043
1044 wqe->base.num_sge_key = 1;
1045 wqe->atomic.sge.va = cpu_to_be64(wr->wr.sg_list[0].addr);
1046 wqe->atomic.sge.len = cpu_to_be32(8);
1047 wqe->atomic.sge.lkey = cpu_to_be32(wr->wr.sg_list[0].lkey);
1048
1049 return ionic_prep_common(qp, &wr->wr, meta, wqe);
1050}
1051
1052static int ionic_prep_inv(struct ionic_qp *qp,
1053 const struct ib_send_wr *wr)
1054{
1055 struct ionic_ibdev *dev = to_ionic_ibdev(qp->ibqp.device);
1056 struct ionic_sq_meta *meta;
1057 struct ionic_v1_wqe *wqe;
1058
1059 if (wr->send_flags & (IB_SEND_SOLICITED | IB_SEND_INLINE))
1060 return -EINVAL;
1061
1062 meta = &qp->sq_meta[qp->sq.prod];
1063 wqe = ionic_queue_at_prod(&qp->sq);
1064
1065 ionic_prep_sq_wqe(qp, wqe);
1066
1067 wqe->base.op = IONIC_OP(dev->lif_cfg.rdma_version, LOCAL_INV);
1068 wqe->base.imm_data_key = cpu_to_be32(wr->ex.invalidate_rkey);
1069
1070 meta->len = 0;
1071 meta->ibop = IB_WC_LOCAL_INV;
1072
1073 ionic_prep_base(qp, wr, meta, wqe);
1074
1075 return 0;
1076}
1077
1078static int ionic_prep_reg(struct ionic_qp *qp,
1079 const struct ib_reg_wr *wr)
1080{
1081 struct ionic_ibdev *dev = to_ionic_ibdev(qp->ibqp.device);
1082 struct ionic_mr *mr = to_ionic_mr(wr->mr);
1083 struct ionic_sq_meta *meta;
1084 struct ionic_v1_wqe *wqe;
1085 __le64 dma_addr;
1086 int flags;
1087
1088 if (wr->wr.send_flags & (IB_SEND_SOLICITED | IB_SEND_INLINE))
1089 return -EINVAL;
1090
1091 /* must call ib_map_mr_sg before posting reg wr */
1092 if (!mr->buf.tbl_pages)
1093 return -EINVAL;
1094
1095 meta = &qp->sq_meta[qp->sq.prod];
1096 wqe = ionic_queue_at_prod(&qp->sq);
1097
1098 ionic_prep_sq_wqe(qp, wqe);
1099
1100 flags = to_ionic_mr_flags(wr->access);
1101
1102 wqe->base.op = IONIC_OP(dev->lif_cfg.rdma_version, REG_MR);
1103 wqe->base.num_sge_key = wr->key;
1104 wqe->base.imm_data_key = cpu_to_be32(mr->ibmr.lkey);
1105 wqe->reg_mr.va = cpu_to_be64(mr->ibmr.iova);
1106 wqe->reg_mr.length = cpu_to_be64(mr->ibmr.length);
1107 wqe->reg_mr.offset = ionic_pgtbl_off(&mr->buf, mr->ibmr.iova);
1108 dma_addr = ionic_pgtbl_dma(&mr->buf, mr->ibmr.iova);
1109 wqe->reg_mr.dma_addr = cpu_to_be64(le64_to_cpu(dma_addr));
1110
1111 wqe->reg_mr.map_count = cpu_to_be32(mr->buf.tbl_pages);
1112 wqe->reg_mr.flags = cpu_to_be16(flags);
1113 wqe->reg_mr.dir_size_log2 = 0;
1114 wqe->reg_mr.page_size_log2 = order_base_2(mr->ibmr.page_size);
1115
1116 meta->len = 0;
1117 meta->ibop = IB_WC_REG_MR;
1118
1119 ionic_prep_base(qp, &wr->wr, meta, wqe);
1120
1121 return 0;
1122}
1123
1124static int ionic_prep_one_rc(struct ionic_qp *qp,
1125 const struct ib_send_wr *wr)
1126{
1127 struct ionic_ibdev *dev = to_ionic_ibdev(qp->ibqp.device);
1128 int rc = 0;
1129
1130 switch (wr->opcode) {
1131 case IB_WR_SEND:
1132 case IB_WR_SEND_WITH_IMM:
1133 case IB_WR_SEND_WITH_INV:
1134 rc = ionic_prep_send(qp, wr);
1135 break;
1136 case IB_WR_RDMA_READ:
1137 case IB_WR_RDMA_WRITE:
1138 case IB_WR_RDMA_WRITE_WITH_IMM:
1139 rc = ionic_prep_rdma(qp, rdma_wr(wr));
1140 break;
1141 case IB_WR_ATOMIC_CMP_AND_SWP:
1142 case IB_WR_ATOMIC_FETCH_AND_ADD:
1143 rc = ionic_prep_atomic(qp, atomic_wr(wr));
1144 break;
1145 case IB_WR_LOCAL_INV:
1146 rc = ionic_prep_inv(qp, wr);
1147 break;
1148 case IB_WR_REG_MR:
1149 rc = ionic_prep_reg(qp, reg_wr(wr));
1150 break;
1151 default:
1152 ibdev_dbg(&dev->ibdev, "invalid opcode %d\n", wr->opcode);
1153 rc = -EINVAL;
1154 }
1155
1156 return rc;
1157}
1158
1159static int ionic_prep_one_ud(struct ionic_qp *qp,
1160 const struct ib_send_wr *wr)
1161{
1162 struct ionic_ibdev *dev = to_ionic_ibdev(qp->ibqp.device);
1163 int rc = 0;
1164
1165 switch (wr->opcode) {
1166 case IB_WR_SEND:
1167 case IB_WR_SEND_WITH_IMM:
1168 rc = ionic_prep_send_ud(qp, ud_wr(wr));
1169 break;
1170 default:
1171 ibdev_dbg(&dev->ibdev, "invalid opcode %d\n", wr->opcode);
1172 rc = -EINVAL;
1173 }
1174
1175 return rc;
1176}
1177
1178static int ionic_prep_recv(struct ionic_qp *qp,
1179 const struct ib_recv_wr *wr)
1180{
1181 struct ionic_rq_meta *meta;
1182 struct ionic_v1_wqe *wqe;
1183 s64 signed_len;
1184 u32 mval;
1185
1186 wqe = ionic_queue_at_prod(&qp->rq);
1187
1188 /* if wqe is owned by device, caller can try posting again soon */
1189 if (wqe->base.flags & cpu_to_be16(IONIC_V1_FLAG_FENCE))
1190 return -EAGAIN;
1191
1192 meta = qp->rq_meta_head;
1193 if (unlikely(meta == IONIC_META_LAST) ||
1194 unlikely(meta == IONIC_META_POSTED))
1195 return -EIO;
1196
1197 ionic_prep_rq_wqe(qp, wqe);
1198
1199 mval = ionic_v1_recv_wqe_max_sge(qp->rq.stride_log2, qp->rq_spec,
1200 false);
1201 signed_len = ionic_prep_pld(wqe, &wqe->recv.pld,
1202 qp->rq_spec, mval,
1203 wr->sg_list, wr->num_sge);
1204 if (signed_len < 0)
1205 return signed_len;
1206
1207 meta->wrid = wr->wr_id;
1208
1209 wqe->base.wqe_idx = cpu_to_le64(meta - qp->rq_meta);
1210 wqe->base.num_sge_key = wr->num_sge;
1211
1212 /* total length for recv goes in base imm_data_key */
1213 wqe->base.imm_data_key = cpu_to_be32(signed_len);
1214
1215 ionic_queue_produce(&qp->rq);
1216
1217 qp->rq_meta_head = meta->next;
1218 meta->next = IONIC_META_POSTED;
1219
1220 return 0;
1221}
1222
1223static int ionic_post_send_common(struct ionic_ibdev *dev,
1224 struct ionic_vcq *vcq,
1225 struct ionic_cq *cq,
1226 struct ionic_qp *qp,
1227 const struct ib_send_wr *wr,
1228 const struct ib_send_wr **bad)
1229{
1230 unsigned long irqflags;
1231 bool notify = false;
1232 int spend, rc = 0;
1233
1234 if (!bad)
1235 return -EINVAL;
1236
1237 if (!qp->has_sq) {
1238 *bad = wr;
1239 return -EINVAL;
1240 }
1241
1242 if (qp->state < IB_QPS_RTS) {
1243 *bad = wr;
1244 return -EINVAL;
1245 }
1246
1247 spin_lock_irqsave(&qp->sq_lock, irqflags);
1248
1249 while (wr) {
1250 if (ionic_queue_full(&qp->sq)) {
1251 ibdev_dbg(&dev->ibdev, "queue full");
1252 rc = -ENOMEM;
1253 goto out;
1254 }
1255
1256 if (qp->ibqp.qp_type == IB_QPT_UD ||
1257 qp->ibqp.qp_type == IB_QPT_GSI)
1258 rc = ionic_prep_one_ud(qp, wr);
1259 else
1260 rc = ionic_prep_one_rc(qp, wr);
1261 if (rc)
1262 goto out;
1263
1264 wr = wr->next;
1265 }
1266
1267out:
1268 spin_unlock_irqrestore(&qp->sq_lock, irqflags);
1269
1270 spin_lock_irqsave(&cq->lock, irqflags);
1271 spin_lock(&qp->sq_lock);
1272
1273 if (likely(qp->sq.prod != qp->sq_old_prod)) {
1274 /* ring cq doorbell just in time */
1275 spend = (qp->sq.prod - qp->sq_old_prod) & qp->sq.mask;
1276 ionic_reserve_cq(dev, cq, spend);
1277
1278 qp->sq_old_prod = qp->sq.prod;
1279
1280 ionic_dbell_ring(dev->lif_cfg.dbpage, dev->lif_cfg.sq_qtype,
1281 ionic_queue_dbell_val(&qp->sq));
1282 }
1283
1284 if (qp->sq_flush) {
1285 notify = true;
1286 cq->flush = true;
1287 list_move_tail(&qp->cq_flush_sq, &cq->flush_sq);
1288 }
1289
1290 spin_unlock(&qp->sq_lock);
1291 spin_unlock_irqrestore(&cq->lock, irqflags);
1292
1293 if (notify && vcq->ibcq.comp_handler)
1294 vcq->ibcq.comp_handler(&vcq->ibcq, vcq->ibcq.cq_context);
1295
1296 *bad = wr;
1297 return rc;
1298}
1299
1300static int ionic_post_recv_common(struct ionic_ibdev *dev,
1301 struct ionic_vcq *vcq,
1302 struct ionic_cq *cq,
1303 struct ionic_qp *qp,
1304 const struct ib_recv_wr *wr,
1305 const struct ib_recv_wr **bad)
1306{
1307 unsigned long irqflags;
1308 bool notify = false;
1309 int spend, rc = 0;
1310
1311 if (!bad)
1312 return -EINVAL;
1313
1314 if (!qp->has_rq) {
1315 *bad = wr;
1316 return -EINVAL;
1317 }
1318
1319 if (qp->state < IB_QPS_INIT) {
1320 *bad = wr;
1321 return -EINVAL;
1322 }
1323
1324 spin_lock_irqsave(&qp->rq_lock, irqflags);
1325
1326 while (wr) {
1327 if (ionic_queue_full(&qp->rq)) {
1328 ibdev_dbg(&dev->ibdev, "queue full");
1329 rc = -ENOMEM;
1330 goto out;
1331 }
1332
1333 rc = ionic_prep_recv(qp, wr);
1334 if (rc)
1335 goto out;
1336
1337 wr = wr->next;
1338 }
1339
1340out:
1341 if (!cq) {
1342 spin_unlock_irqrestore(&qp->rq_lock, irqflags);
1343 goto out_unlocked;
1344 }
1345 spin_unlock_irqrestore(&qp->rq_lock, irqflags);
1346
1347 spin_lock_irqsave(&cq->lock, irqflags);
1348 spin_lock(&qp->rq_lock);
1349
1350 if (likely(qp->rq.prod != qp->rq_old_prod)) {
1351 /* ring cq doorbell just in time */
1352 spend = (qp->rq.prod - qp->rq_old_prod) & qp->rq.mask;
1353 ionic_reserve_cq(dev, cq, spend);
1354
1355 qp->rq_old_prod = qp->rq.prod;
1356
1357 ionic_dbell_ring(dev->lif_cfg.dbpage, dev->lif_cfg.rq_qtype,
1358 ionic_queue_dbell_val(&qp->rq));
1359 }
1360
1361 if (qp->rq_flush) {
1362 notify = true;
1363 cq->flush = true;
1364 list_move_tail(&qp->cq_flush_rq, &cq->flush_rq);
1365 }
1366
1367 spin_unlock(&qp->rq_lock);
1368 spin_unlock_irqrestore(&cq->lock, irqflags);
1369
1370 if (notify && vcq->ibcq.comp_handler)
1371 vcq->ibcq.comp_handler(&vcq->ibcq, vcq->ibcq.cq_context);
1372
1373out_unlocked:
1374 *bad = wr;
1375 return rc;
1376}
1377
1378int ionic_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr,
1379 const struct ib_send_wr **bad)
1380{
1381 struct ionic_ibdev *dev = to_ionic_ibdev(ibqp->device);
1382 struct ionic_vcq *vcq = to_ionic_vcq(ibqp->send_cq);
1383 struct ionic_qp *qp = to_ionic_qp(ibqp);
1384 struct ionic_cq *cq =
1385 to_ionic_vcq_cq(ibqp->send_cq, qp->udma_idx);
1386
1387 return ionic_post_send_common(dev, vcq, cq, qp, wr, bad);
1388}
1389
1390int ionic_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr,
1391 const struct ib_recv_wr **bad)
1392{
1393 struct ionic_ibdev *dev = to_ionic_ibdev(ibqp->device);
1394 struct ionic_vcq *vcq = to_ionic_vcq(ibqp->recv_cq);
1395 struct ionic_qp *qp = to_ionic_qp(ibqp);
1396 struct ionic_cq *cq =
1397 to_ionic_vcq_cq(ibqp->recv_cq, qp->udma_idx);
1398
1399 return ionic_post_recv_common(dev, vcq, cq, qp, wr, bad);
1400}