projects/linux·blob master · refs

1227 lines · 28.7 KiB

1// SPDX-License-Identifier: GPL-2.0
2/* Copyright (C) 2018-2025, Advanced Micro Devices, Inc. */
3
4#include <linux/interrupt.h>
5#include <linux/module.h>
6#include <linux/printk.h>
7
8#include "ionic_fw.h"
9#include "ionic_ibdev.h"
10
11#define IONIC_EQ_COUNT_MIN 4
12#define IONIC_AQ_COUNT_MIN 1
13
14/* not a valid queue position or negative error status */
15#define IONIC_ADMIN_POSTED 0x10000
16
17/* cpu can be held with irq disabled for COUNT * MS (for create/destroy_ah) */
18#define IONIC_ADMIN_BUSY_RETRY_COUNT 2000
19#define IONIC_ADMIN_BUSY_RETRY_MS 1
20
21/* admin queue will be considered failed if a command takes longer */
22#define IONIC_ADMIN_TIMEOUT (HZ * 2)
23#define IONIC_ADMIN_WARN (HZ / 8)
24
25/* will poll for admin cq to tolerate and report from missed event */
26#define IONIC_ADMIN_DELAY (HZ / 8)
27
28/* work queue for polling the event queue and admin cq */
29struct workqueue_struct *ionic_evt_workq;
30
31static void ionic_admin_timedout(struct ionic_aq *aq)
32{
33 struct ionic_ibdev *dev = aq->dev;
34 unsigned long irqflags;
35 u16 pos;
36
37 spin_lock_irqsave(&aq->lock, irqflags);
38 if (ionic_queue_empty(&aq->q))
39 goto out;
40
41 /* Reset ALL adminq if any one times out */
42 if (atomic_read(&aq->admin_state) < IONIC_ADMIN_KILLED)
43 queue_work(ionic_evt_workq, &dev->reset_work);
44
45 ibdev_err(&dev->ibdev, "admin command timed out, aq %d after: %ums\n",
46 aq->aqid, (u32)jiffies_to_msecs(jiffies - aq->stamp));
47
48 pos = (aq->q.prod - 1) & aq->q.mask;
49 if (pos == aq->q.cons)
50 goto out;
51
52 ibdev_warn(&dev->ibdev, "admin pos %u (last posted)\n", pos);
53 print_hex_dump(KERN_WARNING, "cmd ", DUMP_PREFIX_OFFSET, 16, 1,
54 ionic_queue_at(&aq->q, pos),
55 BIT(aq->q.stride_log2), true);
56
57out:
58 spin_unlock_irqrestore(&aq->lock, irqflags);
59}
60
61static void ionic_admin_reset_dwork(struct ionic_ibdev *dev)
62{
63 if (atomic_read(&dev->admin_state) == IONIC_ADMIN_KILLED)
64 return;
65
66 queue_delayed_work(ionic_evt_workq, &dev->admin_dwork,
67 IONIC_ADMIN_DELAY);
68}
69
70static void ionic_admin_reset_wdog(struct ionic_aq *aq)
71{
72 if (atomic_read(&aq->admin_state) == IONIC_ADMIN_KILLED)
73 return;
74
75 aq->stamp = jiffies;
76 ionic_admin_reset_dwork(aq->dev);
77}
78
79static bool ionic_admin_next_cqe(struct ionic_ibdev *dev, struct ionic_cq *cq,
80 struct ionic_v1_cqe **cqe)
81{
82 struct ionic_v1_cqe *qcqe = ionic_queue_at_prod(&cq->q);
83
84 if (unlikely(cq->color != ionic_v1_cqe_color(qcqe)))
85 return false;
86
87 /* Prevent out-of-order reads of the CQE */
88 dma_rmb();
89 *cqe = qcqe;
90
91 return true;
92}
93
94static void ionic_admin_poll_locked(struct ionic_aq *aq)
95{
96 struct ionic_cq *cq = &aq->vcq->cq[0];
97 struct ionic_admin_wr *wr, *wr_next;
98 struct ionic_ibdev *dev = aq->dev;
99 u32 wr_strides, avlbl_strides;
100 struct ionic_v1_cqe *cqe;
101 u32 qtf, qid;
102 u16 old_prod;
103 u8 type;
104
105 lockdep_assert_held(&aq->lock);
106
107 if (atomic_read(&aq->admin_state) == IONIC_ADMIN_KILLED) {
108 list_for_each_entry_safe(wr, wr_next, &aq->wr_prod, aq_ent) {
109 INIT_LIST_HEAD(&wr->aq_ent);
110 aq->q_wr[wr->status].wr = NULL;
111 wr->status = atomic_read(&aq->admin_state);
112 complete_all(&wr->work);
113 }
114 INIT_LIST_HEAD(&aq->wr_prod);
115
116 list_for_each_entry_safe(wr, wr_next, &aq->wr_post, aq_ent) {
117 INIT_LIST_HEAD(&wr->aq_ent);
118 wr->status = atomic_read(&aq->admin_state);
119 complete_all(&wr->work);
120 }
121 INIT_LIST_HEAD(&aq->wr_post);
122
123 return;
124 }
125
126 old_prod = cq->q.prod;
127
128 while (ionic_admin_next_cqe(dev, cq, &cqe)) {
129 qtf = ionic_v1_cqe_qtf(cqe);
130 qid = ionic_v1_cqe_qtf_qid(qtf);
131 type = ionic_v1_cqe_qtf_type(qtf);
132
133 if (unlikely(type != IONIC_V1_CQE_TYPE_ADMIN)) {
134 ibdev_warn_ratelimited(&dev->ibdev,
135 "bad cqe type %u\n", type);
136 goto cq_next;
137 }
138
139 if (unlikely(qid != aq->aqid)) {
140 ibdev_warn_ratelimited(&dev->ibdev,
141 "bad cqe qid %u\n", qid);
142 goto cq_next;
143 }
144
145 if (unlikely(be16_to_cpu(cqe->admin.cmd_idx) != aq->q.cons)) {
146 ibdev_warn_ratelimited(&dev->ibdev,
147 "bad idx %u cons %u qid %u\n",
148 be16_to_cpu(cqe->admin.cmd_idx),
149 aq->q.cons, qid);
150 goto cq_next;
151 }
152
153 if (unlikely(ionic_queue_empty(&aq->q))) {
154 ibdev_warn_ratelimited(&dev->ibdev,
155 "bad cqe for empty adminq\n");
156 goto cq_next;
157 }
158
159 wr = aq->q_wr[aq->q.cons].wr;
160 if (wr) {
161 aq->q_wr[aq->q.cons].wr = NULL;
162 list_del_init(&wr->aq_ent);
163
164 wr->cqe = *cqe;
165 wr->status = atomic_read(&aq->admin_state);
166 complete_all(&wr->work);
167 }
168
169 ionic_queue_consume_entries(&aq->q,
170 aq->q_wr[aq->q.cons].wqe_strides);
171
172cq_next:
173 ionic_queue_produce(&cq->q);
174 cq->color = ionic_color_wrap(cq->q.prod, cq->color);
175 }
176
177 if (old_prod != cq->q.prod) {
178 ionic_admin_reset_wdog(aq);
179 cq->q.cons = cq->q.prod;
180 ionic_dbell_ring(dev->lif_cfg.dbpage, dev->lif_cfg.cq_qtype,
181 ionic_queue_dbell_val(&cq->q));
182 queue_work(ionic_evt_workq, &aq->work);
183 } else if (!aq->armed) {
184 aq->armed = true;
185 cq->arm_any_prod = ionic_queue_next(&cq->q, cq->arm_any_prod);
186 ionic_dbell_ring(dev->lif_cfg.dbpage, dev->lif_cfg.cq_qtype,
187 cq->q.dbell | IONIC_CQ_RING_ARM |
188 cq->arm_any_prod);
189 queue_work(ionic_evt_workq, &aq->work);
190 }
191
192 if (atomic_read(&aq->admin_state) != IONIC_ADMIN_ACTIVE)
193 return;
194
195 old_prod = aq->q.prod;
196
197 if (ionic_queue_empty(&aq->q) && !list_empty(&aq->wr_post))
198 ionic_admin_reset_wdog(aq);
199
200 if (list_empty(&aq->wr_post))
201 return;
202
203 do {
204 u8 *src;
205 int i, src_len;
206 size_t stride_len;
207
208 wr = list_first_entry(&aq->wr_post, struct ionic_admin_wr,
209 aq_ent);
210 wr_strides = (le16_to_cpu(wr->wqe.len) + ADMIN_WQE_HDR_LEN +
211 (ADMIN_WQE_STRIDE - 1)) >> aq->q.stride_log2;
212 avlbl_strides = ionic_queue_length_remaining(&aq->q);
213
214 if (wr_strides > avlbl_strides)
215 break;
216
217 list_move(&wr->aq_ent, &aq->wr_prod);
218 wr->status = aq->q.prod;
219 aq->q_wr[aq->q.prod].wr = wr;
220 aq->q_wr[aq->q.prod].wqe_strides = wr_strides;
221
222 src_len = le16_to_cpu(wr->wqe.len);
223 src = (uint8_t *)&wr->wqe.cmd;
224
225 /* First stride */
226 memcpy(ionic_queue_at_prod(&aq->q), &wr->wqe,
227 ADMIN_WQE_HDR_LEN);
228 stride_len = ADMIN_WQE_STRIDE - ADMIN_WQE_HDR_LEN;
229 if (stride_len > src_len)
230 stride_len = src_len;
231 memcpy(ionic_queue_at_prod(&aq->q) + ADMIN_WQE_HDR_LEN,
232 src, stride_len);
233 ibdev_dbg(&dev->ibdev, "post admin prod %u (%u strides)\n",
234 aq->q.prod, wr_strides);
235 print_hex_dump_debug("wqe ", DUMP_PREFIX_OFFSET, 16, 1,
236 ionic_queue_at_prod(&aq->q),
237 BIT(aq->q.stride_log2), true);
238 ionic_queue_produce(&aq->q);
239
240 /* Remaining strides */
241 for (i = stride_len; i < src_len; i += stride_len) {
242 stride_len = ADMIN_WQE_STRIDE;
243
244 if (i + stride_len > src_len)
245 stride_len = src_len - i;
246
247 memcpy(ionic_queue_at_prod(&aq->q), src + i,
248 stride_len);
249 print_hex_dump_debug("wqe ", DUMP_PREFIX_OFFSET, 16, 1,
250 ionic_queue_at_prod(&aq->q),
251 BIT(aq->q.stride_log2), true);
252 ionic_queue_produce(&aq->q);
253 }
254 } while (!list_empty(&aq->wr_post));
255
256 if (old_prod != aq->q.prod)
257 ionic_dbell_ring(dev->lif_cfg.dbpage, dev->lif_cfg.aq_qtype,
258 ionic_queue_dbell_val(&aq->q));
259}
260
261static void ionic_admin_dwork(struct work_struct *ws)
262{
263 struct ionic_ibdev *dev =
264 container_of(ws, struct ionic_ibdev, admin_dwork.work);
265 struct ionic_aq *aq, *bad_aq = NULL;
266 bool do_reschedule = false;
267 unsigned long irqflags;
268 bool do_reset = false;
269 u16 pos;
270 int i;
271
272 for (i = 0; i < dev->lif_cfg.aq_count; i++) {
273 aq = dev->aq_vec[i];
274
275 spin_lock_irqsave(&aq->lock, irqflags);
276
277 if (ionic_queue_empty(&aq->q))
278 goto next_aq;
279
280 /* Reschedule if any queue has outstanding work */
281 do_reschedule = true;
282
283 if (time_is_after_eq_jiffies(aq->stamp + IONIC_ADMIN_WARN))
284 /* Warning threshold not met, nothing to do */
285 goto next_aq;
286
287 /* See if polling now makes some progress */
288 pos = aq->q.cons;
289 ionic_admin_poll_locked(aq);
290 if (pos != aq->q.cons) {
291 ibdev_dbg(&dev->ibdev,
292 "missed event for acq %d\n", aq->cqid);
293 goto next_aq;
294 }
295
296 if (time_is_after_eq_jiffies(aq->stamp +
297 IONIC_ADMIN_TIMEOUT)) {
298 /* Timeout threshold not met */
299 ibdev_dbg(&dev->ibdev, "no progress after %ums\n",
300 (u32)jiffies_to_msecs(jiffies - aq->stamp));
301 goto next_aq;
302 }
303
304 /* Queue timed out */
305 bad_aq = aq;
306 do_reset = true;
307next_aq:
308 spin_unlock_irqrestore(&aq->lock, irqflags);
309 }
310
311 if (do_reset)
312 /* Reset RDMA lif on a timeout */
313 ionic_admin_timedout(bad_aq);
314 else if (do_reschedule)
315 /* Try to poll again later */
316 ionic_admin_reset_dwork(dev);
317}
318
319static void ionic_admin_work(struct work_struct *ws)
320{
321 struct ionic_aq *aq = container_of(ws, struct ionic_aq, work);
322 unsigned long irqflags;
323
324 spin_lock_irqsave(&aq->lock, irqflags);
325 ionic_admin_poll_locked(aq);
326 spin_unlock_irqrestore(&aq->lock, irqflags);
327}
328
329static void ionic_admin_post_aq(struct ionic_aq *aq, struct ionic_admin_wr *wr)
330{
331 unsigned long irqflags;
332 bool poll;
333
334 wr->status = IONIC_ADMIN_POSTED;
335 wr->aq = aq;
336
337 spin_lock_irqsave(&aq->lock, irqflags);
338 poll = list_empty(&aq->wr_post);
339 list_add(&wr->aq_ent, &aq->wr_post);
340 if (poll)
341 ionic_admin_poll_locked(aq);
342 spin_unlock_irqrestore(&aq->lock, irqflags);
343}
344
345void ionic_admin_post(struct ionic_ibdev *dev, struct ionic_admin_wr *wr)
346{
347 int aq_idx;
348
349 /* Use cpu id for the adminq selection */
350 aq_idx = raw_smp_processor_id() % dev->lif_cfg.aq_count;
351 ionic_admin_post_aq(dev->aq_vec[aq_idx], wr);
352}
353
354static void ionic_admin_cancel(struct ionic_admin_wr *wr)
355{
356 struct ionic_aq *aq = wr->aq;
357 unsigned long irqflags;
358
359 spin_lock_irqsave(&aq->lock, irqflags);
360
361 if (!list_empty(&wr->aq_ent)) {
362 list_del(&wr->aq_ent);
363 if (wr->status != IONIC_ADMIN_POSTED)
364 aq->q_wr[wr->status].wr = NULL;
365 }
366
367 spin_unlock_irqrestore(&aq->lock, irqflags);
368}
369
370static int ionic_admin_busy_wait(struct ionic_admin_wr *wr)
371{
372 struct ionic_aq *aq = wr->aq;
373 unsigned long irqflags;
374 int try_i;
375
376 for (try_i = 0; try_i < IONIC_ADMIN_BUSY_RETRY_COUNT; ++try_i) {
377 if (completion_done(&wr->work))
378 return 0;
379
380 mdelay(IONIC_ADMIN_BUSY_RETRY_MS);
381
382 spin_lock_irqsave(&aq->lock, irqflags);
383 ionic_admin_poll_locked(aq);
384 spin_unlock_irqrestore(&aq->lock, irqflags);
385 }
386
387 /*
388 * we timed out. Initiate RDMA LIF reset and indicate
389 * error to caller.
390 */
391 ionic_admin_timedout(aq);
392 return -ETIMEDOUT;
393}
394
395int ionic_admin_wait(struct ionic_ibdev *dev, struct ionic_admin_wr *wr,
396 enum ionic_admin_flags flags)
397{
398 int rc, timo;
399
400 if (flags & IONIC_ADMIN_F_BUSYWAIT) {
401 /* Spin */
402 rc = ionic_admin_busy_wait(wr);
403 } else if (flags & IONIC_ADMIN_F_INTERRUPT) {
404 /*
405 * Interruptible sleep, 1s timeout
406 * This is used for commands which are safe for the caller
407 * to clean up without killing and resetting the adminq.
408 */
409 timo = wait_for_completion_interruptible_timeout(&wr->work,
410 HZ);
411 if (timo > 0)
412 rc = 0;
413 else if (timo == 0)
414 rc = -ETIMEDOUT;
415 else
416 rc = timo;
417 } else {
418 /*
419 * Uninterruptible sleep
420 * This is used for commands which are NOT safe for the
421 * caller to clean up. Cleanup must be handled by the
422 * adminq kill and reset process so that host memory is
423 * not corrupted by the device.
424 */
425 wait_for_completion(&wr->work);
426 rc = 0;
427 }
428
429 if (rc) {
430 ibdev_warn(&dev->ibdev, "wait status %d\n", rc);
431 ionic_admin_cancel(wr);
432 } else if (wr->status == IONIC_ADMIN_KILLED) {
433 ibdev_dbg(&dev->ibdev, "admin killed\n");
434
435 /* No error if admin already killed during teardown */
436 rc = (flags & IONIC_ADMIN_F_TEARDOWN) ? 0 : -ENODEV;
437 } else if (ionic_v1_cqe_error(&wr->cqe)) {
438 ibdev_warn(&dev->ibdev, "opcode %u error %u\n",
439 wr->wqe.op,
440 be32_to_cpu(wr->cqe.status_length));
441 rc = -EINVAL;
442 }
443 return rc;
444}
445
446static int ionic_rdma_devcmd(struct ionic_ibdev *dev,
447 struct ionic_admin_ctx *admin)
448{
449 int rc;
450
451 rc = ionic_adminq_post_wait(dev->lif_cfg.lif, admin);
452 if (rc)
453 return rc;
454
455 return ionic_error_to_errno(admin->comp.comp.status);
456}
457
458int ionic_rdma_reset_devcmd(struct ionic_ibdev *dev)
459{
460 struct ionic_admin_ctx admin = {
461 .work = COMPLETION_INITIALIZER_ONSTACK(admin.work),
462 .cmd.rdma_reset = {
463 .opcode = IONIC_CMD_RDMA_RESET_LIF,
464 .lif_index = cpu_to_le16(dev->lif_cfg.lif_index),
465 },
466 };
467
468 return ionic_rdma_devcmd(dev, &admin);
469}
470
471static int ionic_rdma_queue_devcmd(struct ionic_ibdev *dev,
472 struct ionic_queue *q,
473 u32 qid, u32 cid, u16 opcode)
474{
475 struct ionic_admin_ctx admin = {
476 .work = COMPLETION_INITIALIZER_ONSTACK(admin.work),
477 .cmd.rdma_queue = {
478 .opcode = opcode,
479 .lif_index = cpu_to_le16(dev->lif_cfg.lif_index),
480 .qid_ver = cpu_to_le32(qid),
481 .cid = cpu_to_le32(cid),
482 .dbid = cpu_to_le16(dev->lif_cfg.dbid),
483 .depth_log2 = q->depth_log2,
484 .stride_log2 = q->stride_log2,
485 .dma_addr = cpu_to_le64(q->dma),
486 },
487 };
488
489 return ionic_rdma_devcmd(dev, &admin);
490}
491
492static void ionic_rdma_admincq_comp(struct ib_cq *ibcq, void *cq_context)
493{
494 struct ionic_aq *aq = cq_context;
495 unsigned long irqflags;
496
497 spin_lock_irqsave(&aq->lock, irqflags);
498 aq->armed = false;
499 if (atomic_read(&aq->admin_state) < IONIC_ADMIN_KILLED)
500 queue_work(ionic_evt_workq, &aq->work);
501 spin_unlock_irqrestore(&aq->lock, irqflags);
502}
503
504static void ionic_rdma_admincq_event(struct ib_event *event, void *cq_context)
505{
506 struct ionic_aq *aq = cq_context;
507
508 ibdev_err(&aq->dev->ibdev, "admincq event %d\n", event->event);
509}
510
511static struct ionic_vcq *ionic_create_rdma_admincq(struct ionic_ibdev *dev,
512 int comp_vector)
513{
514 struct ib_cq_init_attr attr = {
515 .cqe = IONIC_AQ_DEPTH,
516 .comp_vector = comp_vector,
517 };
518 struct ionic_tbl_buf buf = {};
519 struct ionic_vcq *vcq;
520 struct ionic_cq *cq;
521 int rc;
522
523 vcq = kzalloc_obj(*vcq);
524 if (!vcq)
525 return ERR_PTR(-ENOMEM);
526
527 vcq->ibcq.device = &dev->ibdev;
528 vcq->ibcq.comp_handler = ionic_rdma_admincq_comp;
529 vcq->ibcq.event_handler = ionic_rdma_admincq_event;
530 atomic_set(&vcq->ibcq.usecnt, 0);
531
532 vcq->udma_mask = 1;
533 cq = &vcq->cq[0];
534
535 rc = ionic_create_cq_common(vcq, &buf, &attr, NULL, NULL,
536 NULL, NULL, 0);
537 if (rc)
538 goto err_init;
539
540 rc = ionic_rdma_queue_devcmd(dev, &cq->q, cq->cqid, cq->eqid,
541 IONIC_CMD_RDMA_CREATE_CQ);
542 if (rc)
543 goto err_cmd;
544
545 return vcq;
546
547err_cmd:
548 ionic_destroy_cq_common(dev, cq);
549err_init:
550 kfree(vcq);
551
552 return ERR_PTR(rc);
553}
554
555static struct ionic_aq *__ionic_create_rdma_adminq(struct ionic_ibdev *dev,
556 u32 aqid, u32 cqid)
557{
558 struct ionic_aq *aq;
559 int rc;
560
561 aq = kzalloc_obj(*aq);
562 if (!aq)
563 return ERR_PTR(-ENOMEM);
564
565 atomic_set(&aq->admin_state, IONIC_ADMIN_KILLED);
566 aq->dev = dev;
567 aq->aqid = aqid;
568 aq->cqid = cqid;
569 spin_lock_init(&aq->lock);
570
571 rc = ionic_queue_init(&aq->q, dev->lif_cfg.hwdev, IONIC_EQ_DEPTH,
572 ADMIN_WQE_STRIDE);
573 if (rc)
574 goto err_q;
575
576 ionic_queue_dbell_init(&aq->q, aq->aqid);
577
578 aq->q_wr = kzalloc_objs(*aq->q_wr, (u32)aq->q.mask + 1);
579 if (!aq->q_wr) {
580 rc = -ENOMEM;
581 goto err_wr;
582 }
583
584 INIT_LIST_HEAD(&aq->wr_prod);
585 INIT_LIST_HEAD(&aq->wr_post);
586
587 INIT_WORK(&aq->work, ionic_admin_work);
588 aq->armed = false;
589
590 return aq;
591
592err_wr:
593 ionic_queue_destroy(&aq->q, dev->lif_cfg.hwdev);
594err_q:
595 kfree(aq);
596
597 return ERR_PTR(rc);
598}
599
600static void __ionic_destroy_rdma_adminq(struct ionic_ibdev *dev,
601 struct ionic_aq *aq)
602{
603 kfree(aq->q_wr);
604 ionic_queue_destroy(&aq->q, dev->lif_cfg.hwdev);
605 kfree(aq);
606}
607
608static struct ionic_aq *ionic_create_rdma_adminq(struct ionic_ibdev *dev,
609 u32 aqid, u32 cqid)
610{
611 struct ionic_aq *aq;
612 int rc;
613
614 aq = __ionic_create_rdma_adminq(dev, aqid, cqid);
615 if (IS_ERR(aq))
616 return aq;
617
618 rc = ionic_rdma_queue_devcmd(dev, &aq->q, aq->aqid, aq->cqid,
619 IONIC_CMD_RDMA_CREATE_ADMINQ);
620 if (rc)
621 goto err_cmd;
622
623 return aq;
624
625err_cmd:
626 __ionic_destroy_rdma_adminq(dev, aq);
627
628 return ERR_PTR(rc);
629}
630
631static void ionic_flush_qs(struct ionic_ibdev *dev)
632{
633 struct ionic_qp *qp, *qp_tmp;
634 struct ionic_cq *cq, *cq_tmp;
635 LIST_HEAD(flush_list);
636 unsigned long index;
637
638 WARN_ON(!irqs_disabled());
639
640 /* Flush qp send and recv */
641 xa_lock(&dev->qp_tbl);
642 xa_for_each(&dev->qp_tbl, index, qp) {
643 kref_get(&qp->qp_kref);
644 list_add_tail(&qp->ibkill_flush_ent, &flush_list);
645 }
646 xa_unlock(&dev->qp_tbl);
647
648 list_for_each_entry_safe(qp, qp_tmp, &flush_list, ibkill_flush_ent) {
649 ionic_flush_qp(dev, qp);
650 kref_put(&qp->qp_kref, ionic_qp_complete);
651 list_del(&qp->ibkill_flush_ent);
652 }
653
654 /* Notify completions */
655 xa_lock(&dev->cq_tbl);
656 xa_for_each(&dev->cq_tbl, index, cq) {
657 kref_get(&cq->cq_kref);
658 list_add_tail(&cq->ibkill_flush_ent, &flush_list);
659 }
660 xa_unlock(&dev->cq_tbl);
661
662 list_for_each_entry_safe(cq, cq_tmp, &flush_list, ibkill_flush_ent) {
663 ionic_notify_flush_cq(cq);
664 kref_put(&cq->cq_kref, ionic_cq_complete);
665 list_del(&cq->ibkill_flush_ent);
666 }
667}
668
669static void ionic_kill_ibdev(struct ionic_ibdev *dev, bool fatal_path)
670{
671 unsigned long irqflags;
672 bool do_flush = false;
673 int i;
674
675 /* Mark AQs for drain and flush the QPs while irq is disabled */
676 local_irq_save(irqflags);
677
678 /* Mark the admin queue, flushing at most once */
679 for (i = 0; i < dev->lif_cfg.aq_count; i++) {
680 struct ionic_aq *aq = dev->aq_vec[i];
681
682 spin_lock(&aq->lock);
683 if (atomic_read(&aq->admin_state) != IONIC_ADMIN_KILLED) {
684 atomic_set(&aq->admin_state, IONIC_ADMIN_KILLED);
685 /* Flush incomplete admin commands */
686 ionic_admin_poll_locked(aq);
687 do_flush = true;
688 }
689 spin_unlock(&aq->lock);
690 }
691
692 if (do_flush)
693 ionic_flush_qs(dev);
694
695 local_irq_restore(irqflags);
696
697 /* Post a fatal event if requested */
698 if (fatal_path) {
699 struct ib_event ev;
700
701 ev.device = &dev->ibdev;
702 ev.element.port_num = 1;
703 ev.event = IB_EVENT_DEVICE_FATAL;
704
705 ib_dispatch_event(&ev);
706 }
707
708 atomic_set(&dev->admin_state, IONIC_ADMIN_KILLED);
709}
710
711void ionic_kill_rdma_admin(struct ionic_ibdev *dev, bool fatal_path)
712{
713 enum ionic_admin_state old_state;
714 unsigned long irqflags = 0;
715 int i, rc;
716
717 if (!dev->aq_vec)
718 return;
719
720 /*
721 * Admin queues are transitioned from active to paused to killed state.
722 * When in paused state, no new commands are issued to the device,
723 * nor are any completed locally. After resetting the lif, it will be
724 * safe to resume the rdma admin queues in the killed state. Commands
725 * will not be issued to the device, but will complete locally with status
726 * IONIC_ADMIN_KILLED. Handling completion will ensure that creating or
727 * modifying resources fails, but destroying resources succeeds.
728 * If there was a failure resetting the lif using this strategy,
729 * then the state of the device is unknown.
730 */
731 old_state = atomic_cmpxchg(&dev->admin_state, IONIC_ADMIN_ACTIVE,
732 IONIC_ADMIN_PAUSED);
733 if (old_state != IONIC_ADMIN_ACTIVE)
734 return;
735
736 /* Pause all the AQs */
737 local_irq_save(irqflags);
738 for (i = 0; i < dev->lif_cfg.aq_count; i++) {
739 struct ionic_aq *aq = dev->aq_vec[i];
740
741 spin_lock(&aq->lock);
742 /* pause rdma admin queues to reset lif */
743 if (atomic_read(&aq->admin_state) == IONIC_ADMIN_ACTIVE)
744 atomic_set(&aq->admin_state, IONIC_ADMIN_PAUSED);
745 spin_unlock(&aq->lock);
746 }
747 local_irq_restore(irqflags);
748
749 rc = ionic_rdma_reset_devcmd(dev);
750 if (unlikely(rc)) {
751 ibdev_err(&dev->ibdev, "failed to reset rdma %d\n", rc);
752 ionic_request_rdma_reset(dev->lif_cfg.lif);
753 }
754
755 ionic_kill_ibdev(dev, fatal_path);
756}
757
758static void ionic_reset_work(struct work_struct *ws)
759{
760 struct ionic_ibdev *dev =
761 container_of(ws, struct ionic_ibdev, reset_work);
762
763 ionic_kill_rdma_admin(dev, true);
764}
765
766static bool ionic_next_eqe(struct ionic_eq *eq, struct ionic_v1_eqe *eqe)
767{
768 struct ionic_v1_eqe *qeqe;
769 bool color;
770
771 qeqe = ionic_queue_at_prod(&eq->q);
772 color = ionic_v1_eqe_color(qeqe);
773
774 /* cons is color for eq */
775 if (eq->q.cons != color)
776 return false;
777
778 /* Prevent out-of-order reads of the EQE */
779 dma_rmb();
780
781 ibdev_dbg(&eq->dev->ibdev, "poll eq prod %u\n", eq->q.prod);
782 print_hex_dump_debug("eqe ", DUMP_PREFIX_OFFSET, 16, 1,
783 qeqe, BIT(eq->q.stride_log2), true);
784 *eqe = *qeqe;
785
786 return true;
787}
788
789static void ionic_cq_event(struct ionic_ibdev *dev, u32 cqid, u8 code)
790{
791 unsigned long irqflags;
792 struct ib_event ibev;
793 struct ionic_cq *cq;
794
795 xa_lock_irqsave(&dev->cq_tbl, irqflags);
796 cq = xa_load(&dev->cq_tbl, cqid);
797 if (cq)
798 kref_get(&cq->cq_kref);
799 xa_unlock_irqrestore(&dev->cq_tbl, irqflags);
800
801 if (!cq) {
802 ibdev_dbg(&dev->ibdev,
803 "missing cqid %#x code %u\n", cqid, code);
804 return;
805 }
806
807 switch (code) {
808 case IONIC_V1_EQE_CQ_NOTIFY:
809 if (cq->vcq->ibcq.comp_handler)
810 cq->vcq->ibcq.comp_handler(&cq->vcq->ibcq,
811 cq->vcq->ibcq.cq_context);
812 break;
813
814 case IONIC_V1_EQE_CQ_ERR:
815 if (cq->vcq->ibcq.event_handler) {
816 ibev.event = IB_EVENT_CQ_ERR;
817 ibev.device = &dev->ibdev;
818 ibev.element.cq = &cq->vcq->ibcq;
819
820 cq->vcq->ibcq.event_handler(&ibev,
821 cq->vcq->ibcq.cq_context);
822 }
823 break;
824
825 default:
826 ibdev_dbg(&dev->ibdev,
827 "unrecognized cqid %#x code %u\n", cqid, code);
828 break;
829 }
830
831 kref_put(&cq->cq_kref, ionic_cq_complete);
832}
833
834static void ionic_qp_event(struct ionic_ibdev *dev, u32 qpid, u8 code)
835{
836 unsigned long irqflags;
837 struct ib_event ibev;
838 struct ionic_qp *qp;
839
840 xa_lock_irqsave(&dev->qp_tbl, irqflags);
841 qp = xa_load(&dev->qp_tbl, qpid);
842 if (qp)
843 kref_get(&qp->qp_kref);
844 xa_unlock_irqrestore(&dev->qp_tbl, irqflags);
845
846 if (!qp) {
847 ibdev_dbg(&dev->ibdev,
848 "missing qpid %#x code %u\n", qpid, code);
849 return;
850 }
851
852 ibev.device = &dev->ibdev;
853 ibev.element.qp = &qp->ibqp;
854
855 switch (code) {
856 case IONIC_V1_EQE_SQ_DRAIN:
857 ibev.event = IB_EVENT_SQ_DRAINED;
858 break;
859
860 case IONIC_V1_EQE_QP_COMM_EST:
861 ibev.event = IB_EVENT_COMM_EST;
862 break;
863
864 case IONIC_V1_EQE_QP_LAST_WQE:
865 ibev.event = IB_EVENT_QP_LAST_WQE_REACHED;
866 break;
867
868 case IONIC_V1_EQE_QP_ERR:
869 ibev.event = IB_EVENT_QP_FATAL;
870 break;
871
872 case IONIC_V1_EQE_QP_ERR_REQUEST:
873 ibev.event = IB_EVENT_QP_REQ_ERR;
874 break;
875
876 case IONIC_V1_EQE_QP_ERR_ACCESS:
877 ibev.event = IB_EVENT_QP_ACCESS_ERR;
878 break;
879
880 default:
881 ibdev_dbg(&dev->ibdev,
882 "unrecognized qpid %#x code %u\n", qpid, code);
883 goto out;
884 }
885
886 if (qp->ibqp.event_handler)
887 qp->ibqp.event_handler(&ibev, qp->ibqp.qp_context);
888
889out:
890 kref_put(&qp->qp_kref, ionic_qp_complete);
891}
892
893static u16 ionic_poll_eq(struct ionic_eq *eq, u16 budget)
894{
895 struct ionic_ibdev *dev = eq->dev;
896 struct ionic_v1_eqe eqe;
897 u16 npolled = 0;
898 u8 type, code;
899 u32 evt, qid;
900
901 while (npolled < budget) {
902 if (!ionic_next_eqe(eq, &eqe))
903 break;
904
905 ionic_queue_produce(&eq->q);
906
907 /* cons is color for eq */
908 eq->q.cons = ionic_color_wrap(eq->q.prod, eq->q.cons);
909
910 ++npolled;
911
912 evt = ionic_v1_eqe_evt(&eqe);
913 type = ionic_v1_eqe_evt_type(evt);
914 code = ionic_v1_eqe_evt_code(evt);
915 qid = ionic_v1_eqe_evt_qid(evt);
916
917 switch (type) {
918 case IONIC_V1_EQE_TYPE_CQ:
919 ionic_cq_event(dev, qid, code);
920 break;
921
922 case IONIC_V1_EQE_TYPE_QP:
923 ionic_qp_event(dev, qid, code);
924 break;
925
926 default:
927 ibdev_dbg(&dev->ibdev,
928 "unknown event %#x type %u\n", evt, type);
929 }
930 }
931
932 return npolled;
933}
934
935static void ionic_poll_eq_work(struct work_struct *work)
936{
937 struct ionic_eq *eq = container_of(work, struct ionic_eq, work);
938 u32 npolled;
939
940 if (unlikely(!eq->enable) || WARN_ON(eq->armed))
941 return;
942
943 npolled = ionic_poll_eq(eq, IONIC_EQ_WORK_BUDGET);
944 if (npolled == IONIC_EQ_WORK_BUDGET) {
945 ionic_intr_credits(eq->dev->lif_cfg.intr_ctrl, eq->intr,
946 npolled, 0);
947 queue_work(ionic_evt_workq, &eq->work);
948 } else {
949 xchg(&eq->armed, 1);
950 ionic_intr_credits(eq->dev->lif_cfg.intr_ctrl, eq->intr,
951 0, IONIC_INTR_CRED_UNMASK);
952 }
953}
954
955static irqreturn_t ionic_poll_eq_isr(int irq, void *eqptr)
956{
957 struct ionic_eq *eq = eqptr;
958 int was_armed;
959 u32 npolled;
960
961 was_armed = xchg(&eq->armed, 0);
962
963 if (unlikely(!eq->enable) || !was_armed)
964 return IRQ_HANDLED;
965
966 npolled = ionic_poll_eq(eq, IONIC_EQ_ISR_BUDGET);
967 if (npolled == IONIC_EQ_ISR_BUDGET) {
968 ionic_intr_credits(eq->dev->lif_cfg.intr_ctrl, eq->intr,
969 npolled, 0);
970 queue_work(ionic_evt_workq, &eq->work);
971 } else {
972 xchg(&eq->armed, 1);
973 ionic_intr_credits(eq->dev->lif_cfg.intr_ctrl, eq->intr,
974 0, IONIC_INTR_CRED_UNMASK);
975 }
976
977 return IRQ_HANDLED;
978}
979
980static struct ionic_eq *ionic_create_eq(struct ionic_ibdev *dev, int eqid)
981{
982 struct ionic_intr_info intr_obj = { };
983 struct ionic_eq *eq;
984 int rc;
985
986 eq = kzalloc_obj(*eq);
987 if (!eq)
988 return ERR_PTR(-ENOMEM);
989
990 eq->dev = dev;
991
992 rc = ionic_queue_init(&eq->q, dev->lif_cfg.hwdev, IONIC_EQ_DEPTH,
993 sizeof(struct ionic_v1_eqe));
994 if (rc)
995 goto err_q;
996
997 eq->eqid = eqid;
998
999 eq->armed = true;
1000 eq->enable = false;
1001 INIT_WORK(&eq->work, ionic_poll_eq_work);
1002
1003 rc = ionic_intr_alloc(dev->lif_cfg.lif, &intr_obj);
1004 if (rc < 0)
1005 goto err_intr;
1006
1007 eq->irq = intr_obj.vector;
1008 eq->intr = intr_obj.index;
1009
1010 ionic_queue_dbell_init(&eq->q, eq->eqid);
1011
1012 /* cons is color for eq */
1013 eq->q.cons = true;
1014
1015 snprintf(eq->name, sizeof(eq->name), "%s-%d-%d-eq",
1016 "ionr", dev->lif_cfg.lif_index, eq->eqid);
1017
1018 ionic_intr_mask(dev->lif_cfg.intr_ctrl, eq->intr, IONIC_INTR_MASK_SET);
1019 ionic_intr_mask_assert(dev->lif_cfg.intr_ctrl, eq->intr, IONIC_INTR_MASK_SET);
1020 ionic_intr_coal_init(dev->lif_cfg.intr_ctrl, eq->intr, 0);
1021 ionic_intr_clean(dev->lif_cfg.intr_ctrl, eq->intr);
1022
1023 eq->enable = true;
1024
1025 rc = request_irq(eq->irq, ionic_poll_eq_isr, 0, eq->name, eq);
1026 if (rc)
1027 goto err_irq;
1028
1029 rc = ionic_rdma_queue_devcmd(dev, &eq->q, eq->eqid, eq->intr,
1030 IONIC_CMD_RDMA_CREATE_EQ);
1031 if (rc)
1032 goto err_cmd;
1033
1034 ionic_intr_mask(dev->lif_cfg.intr_ctrl, eq->intr, IONIC_INTR_MASK_CLEAR);
1035
1036 return eq;
1037
1038err_cmd:
1039 eq->enable = false;
1040 free_irq(eq->irq, eq);
1041 flush_work(&eq->work);
1042err_irq:
1043 ionic_intr_free(dev->lif_cfg.lif, eq->intr);
1044err_intr:
1045 ionic_queue_destroy(&eq->q, dev->lif_cfg.hwdev);
1046err_q:
1047 kfree(eq);
1048
1049 return ERR_PTR(rc);
1050}
1051
1052static void ionic_destroy_eq(struct ionic_eq *eq)
1053{
1054 struct ionic_ibdev *dev = eq->dev;
1055
1056 eq->enable = false;
1057 free_irq(eq->irq, eq);
1058 flush_work(&eq->work);
1059
1060 ionic_intr_free(dev->lif_cfg.lif, eq->intr);
1061 ionic_queue_destroy(&eq->q, dev->lif_cfg.hwdev);
1062 kfree(eq);
1063}
1064
1065int ionic_create_rdma_admin(struct ionic_ibdev *dev)
1066{
1067 int eq_i = 0, aq_i = 0, rc = 0;
1068 struct ionic_vcq *vcq;
1069 struct ionic_aq *aq;
1070 struct ionic_eq *eq;
1071
1072 dev->eq_vec = NULL;
1073 dev->aq_vec = NULL;
1074
1075 INIT_WORK(&dev->reset_work, ionic_reset_work);
1076 INIT_DELAYED_WORK(&dev->admin_dwork, ionic_admin_dwork);
1077 atomic_set(&dev->admin_state, IONIC_ADMIN_KILLED);
1078
1079 if (dev->lif_cfg.aq_count > IONIC_AQ_COUNT) {
1080 ibdev_dbg(&dev->ibdev, "limiting adminq count to %d\n",
1081 IONIC_AQ_COUNT);
1082 dev->lif_cfg.aq_count = IONIC_AQ_COUNT;
1083 }
1084
1085 if (dev->lif_cfg.eq_count > IONIC_EQ_COUNT) {
1086 dev_dbg(&dev->ibdev.dev, "limiting eventq count to %d\n",
1087 IONIC_EQ_COUNT);
1088 dev->lif_cfg.eq_count = IONIC_EQ_COUNT;
1089 }
1090
1091 /* need at least two eq and one aq */
1092 if (dev->lif_cfg.eq_count < IONIC_EQ_COUNT_MIN ||
1093 dev->lif_cfg.aq_count < IONIC_AQ_COUNT_MIN) {
1094 rc = -EINVAL;
1095 goto out;
1096 }
1097
1098 dev->eq_vec = kmalloc_objs(*dev->eq_vec, dev->lif_cfg.eq_count);
1099 if (!dev->eq_vec) {
1100 rc = -ENOMEM;
1101 goto out;
1102 }
1103
1104 for (eq_i = 0; eq_i < dev->lif_cfg.eq_count; ++eq_i) {
1105 eq = ionic_create_eq(dev, eq_i + dev->lif_cfg.eq_base);
1106 if (IS_ERR(eq)) {
1107 rc = PTR_ERR(eq);
1108
1109 if (eq_i < IONIC_EQ_COUNT_MIN) {
1110 ibdev_err(&dev->ibdev,
1111 "fail create eq %pe\n", eq);
1112 goto out;
1113 }
1114
1115 /* ok, just fewer eq than device supports */
1116 ibdev_dbg(&dev->ibdev, "eq count %d want %d rc %pe\n",
1117 eq_i, dev->lif_cfg.eq_count, eq);
1118
1119 rc = 0;
1120 break;
1121 }
1122
1123 dev->eq_vec[eq_i] = eq;
1124 }
1125
1126 dev->lif_cfg.eq_count = eq_i;
1127
1128 dev->aq_vec = kmalloc_objs(*dev->aq_vec, dev->lif_cfg.aq_count);
1129 if (!dev->aq_vec) {
1130 rc = -ENOMEM;
1131 goto out;
1132 }
1133
1134 /* Create one CQ per AQ */
1135 for (aq_i = 0; aq_i < dev->lif_cfg.aq_count; ++aq_i) {
1136 vcq = ionic_create_rdma_admincq(dev, aq_i % eq_i);
1137 if (IS_ERR(vcq)) {
1138 rc = PTR_ERR(vcq);
1139
1140 if (!aq_i) {
1141 ibdev_err(&dev->ibdev,
1142 "failed to create acq %pe\n", vcq);
1143 goto out;
1144 }
1145
1146 /* ok, just fewer adminq than device supports */
1147 ibdev_dbg(&dev->ibdev, "acq count %d want %d rc %pe\n",
1148 aq_i, dev->lif_cfg.aq_count, vcq);
1149 break;
1150 }
1151
1152 aq = ionic_create_rdma_adminq(dev, aq_i + dev->lif_cfg.aq_base,
1153 vcq->cq[0].cqid);
1154 if (IS_ERR(aq)) {
1155 /* Clean up the dangling CQ */
1156 ionic_destroy_cq_common(dev, &vcq->cq[0]);
1157 kfree(vcq);
1158
1159 rc = PTR_ERR(aq);
1160
1161 if (!aq_i) {
1162 ibdev_err(&dev->ibdev,
1163 "failed to create aq %pe\n", aq);
1164 goto out;
1165 }
1166
1167 /* ok, just fewer adminq than device supports */
1168 ibdev_dbg(&dev->ibdev, "aq count %d want %d rc %pe\n",
1169 aq_i, dev->lif_cfg.aq_count, aq);
1170 break;
1171 }
1172
1173 vcq->ibcq.cq_context = aq;
1174 aq->vcq = vcq;
1175
1176 atomic_set(&aq->admin_state, IONIC_ADMIN_ACTIVE);
1177 dev->aq_vec[aq_i] = aq;
1178 }
1179
1180 atomic_set(&dev->admin_state, IONIC_ADMIN_ACTIVE);
1181out:
1182 dev->lif_cfg.eq_count = eq_i;
1183 dev->lif_cfg.aq_count = aq_i;
1184
1185 return rc;
1186}
1187
1188void ionic_destroy_rdma_admin(struct ionic_ibdev *dev)
1189{
1190 struct ionic_vcq *vcq;
1191 struct ionic_aq *aq;
1192 struct ionic_eq *eq;
1193
1194 /*
1195 * Killing the admin before destroy makes sure all admin and
1196 * completions are flushed. admin_state = IONIC_ADMIN_KILLED
1197 * stops queueing up further works.
1198 */
1199 cancel_delayed_work_sync(&dev->admin_dwork);
1200 cancel_work_sync(&dev->reset_work);
1201
1202 if (dev->aq_vec) {
1203 while (dev->lif_cfg.aq_count > 0) {
1204 aq = dev->aq_vec[--dev->lif_cfg.aq_count];
1205 vcq = aq->vcq;
1206
1207 cancel_work_sync(&aq->work);
1208
1209 __ionic_destroy_rdma_adminq(dev, aq);
1210 if (vcq) {
1211 ionic_destroy_cq_common(dev, &vcq->cq[0]);
1212 kfree(vcq);
1213 }
1214 }
1215
1216 kfree(dev->aq_vec);
1217 }
1218
1219 if (dev->eq_vec) {
1220 while (dev->lif_cfg.eq_count > 0) {
1221 eq = dev->eq_vec[--dev->lif_cfg.eq_count];
1222 ionic_destroy_eq(eq);
1223 }
1224
1225 kfree(dev->eq_vec);
1226 }
1227}