projects/linux·blob master · refs

723 lines · 18.2 KiB

1// SPDX-License-Identifier: GPL-2.0
2/*
3 * NVMe over Fabrics loopback device.
4 * Copyright (c) 2015-2016 HGST, a Western Digital Company.
5 */
6#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
7#include <linux/scatterlist.h>
8#include <linux/blk-mq.h>
9#include <linux/nvme.h>
10#include <linux/module.h>
11#include <linux/parser.h>
12#include "nvmet.h"
13#include "../host/nvme.h"
14#include "../host/fabrics.h"
15
16#define NVME_LOOP_MAX_SEGMENTS 256
17
18struct nvme_loop_iod {
19 struct nvme_request nvme_req;
20 struct nvme_command cmd;
21 struct nvme_completion cqe;
22 struct nvmet_req req;
23 struct nvme_loop_queue *queue;
24 struct work_struct work;
25 struct sg_table sg_table;
26 struct scatterlist first_sgl[];
27};
28
29struct nvme_loop_ctrl {
30 struct nvme_loop_queue *queues;
31
32 struct blk_mq_tag_set admin_tag_set;
33
34 struct list_head list;
35 struct blk_mq_tag_set tag_set;
36 struct nvme_ctrl ctrl;
37
38 struct nvmet_port *port;
39
40 /* Must be last --ends in a flexible-array member. */
41 struct nvme_loop_iod async_event_iod;
42};
43
44static inline struct nvme_loop_ctrl *to_loop_ctrl(struct nvme_ctrl *ctrl)
45{
46 return container_of(ctrl, struct nvme_loop_ctrl, ctrl);
47}
48
49enum nvme_loop_queue_flags {
50 NVME_LOOP_Q_LIVE = 0,
51};
52
53struct nvme_loop_queue {
54 struct nvmet_cq nvme_cq;
55 struct nvmet_sq nvme_sq;
56 struct nvme_loop_ctrl *ctrl;
57 unsigned long flags;
58};
59
60static LIST_HEAD(nvme_loop_ports);
61static DEFINE_MUTEX(nvme_loop_ports_mutex);
62
63static LIST_HEAD(nvme_loop_ctrl_list);
64static DEFINE_MUTEX(nvme_loop_ctrl_mutex);
65
66static void nvme_loop_queue_response(struct nvmet_req *nvme_req);
67static void nvme_loop_delete_ctrl(struct nvmet_ctrl *ctrl);
68
69static const struct nvmet_fabrics_ops nvme_loop_ops;
70
71static inline int nvme_loop_queue_idx(struct nvme_loop_queue *queue)
72{
73 return queue - queue->ctrl->queues;
74}
75
76static void nvme_loop_complete_rq(struct request *req)
77{
78 struct nvme_loop_iod *iod = blk_mq_rq_to_pdu(req);
79
80 sg_free_table_chained(&iod->sg_table, NVME_INLINE_SG_CNT);
81 nvme_complete_rq(req);
82}
83
84static struct blk_mq_tags *nvme_loop_tagset(struct nvme_loop_queue *queue)
85{
86 u32 queue_idx = nvme_loop_queue_idx(queue);
87
88 if (queue_idx == 0)
89 return queue->ctrl->admin_tag_set.tags[queue_idx];
90 return queue->ctrl->tag_set.tags[queue_idx - 1];
91}
92
93static void nvme_loop_queue_response(struct nvmet_req *req)
94{
95 struct nvme_loop_queue *queue =
96 container_of(req->sq, struct nvme_loop_queue, nvme_sq);
97 struct nvme_completion *cqe = req->cqe;
98
99 /*
100 * AEN requests are special as they don't time out and can
101 * survive any kind of queue freeze and often don't respond to
102 * aborts. We don't even bother to allocate a struct request
103 * for them but rather special case them here.
104 */
105 if (unlikely(nvme_is_aen_req(nvme_loop_queue_idx(queue),
106 cqe->command_id))) {
107 nvme_complete_async_event(&queue->ctrl->ctrl, cqe->status,
108 &cqe->result);
109 } else {
110 struct request *rq;
111
112 rq = nvme_find_rq(nvme_loop_tagset(queue), cqe->command_id);
113 if (!rq) {
114 dev_err(queue->ctrl->ctrl.device,
115 "got bad command_id %#x on queue %d\n",
116 cqe->command_id, nvme_loop_queue_idx(queue));
117 return;
118 }
119
120 if (!nvme_try_complete_req(rq, cqe->status, cqe->result))
121 nvme_loop_complete_rq(rq);
122 }
123}
124
125static void nvme_loop_execute_work(struct work_struct *work)
126{
127 struct nvme_loop_iod *iod =
128 container_of(work, struct nvme_loop_iod, work);
129
130 iod->req.execute(&iod->req);
131}
132
133static blk_status_t nvme_loop_queue_rq(struct blk_mq_hw_ctx *hctx,
134 const struct blk_mq_queue_data *bd)
135{
136 struct nvme_ns *ns = hctx->queue->queuedata;
137 struct nvme_loop_queue *queue = hctx->driver_data;
138 struct request *req = bd->rq;
139 struct nvme_loop_iod *iod = blk_mq_rq_to_pdu(req);
140 bool queue_ready = test_bit(NVME_LOOP_Q_LIVE, &queue->flags);
141 blk_status_t ret;
142
143 if (!nvme_check_ready(&queue->ctrl->ctrl, req, queue_ready))
144 return nvme_fail_nonready_command(&queue->ctrl->ctrl, req);
145
146 ret = nvme_setup_cmd(ns, req);
147 if (ret)
148 return ret;
149
150 nvme_start_request(req);
151 iod->cmd.common.flags |= NVME_CMD_SGL_METABUF;
152 iod->req.port = queue->ctrl->port;
153 if (!nvmet_req_init(&iod->req, &queue->nvme_sq, &nvme_loop_ops))
154 return BLK_STS_OK;
155
156 if (blk_rq_nr_phys_segments(req)) {
157 iod->sg_table.sgl = iod->first_sgl;
158 if (sg_alloc_table_chained(&iod->sg_table,
159 blk_rq_nr_phys_segments(req),
160 iod->sg_table.sgl, NVME_INLINE_SG_CNT)) {
161 nvme_cleanup_cmd(req);
162 return BLK_STS_RESOURCE;
163 }
164
165 iod->req.sg = iod->sg_table.sgl;
166 iod->req.sg_cnt = blk_rq_map_sg(req, iod->sg_table.sgl);
167 iod->req.transfer_len = blk_rq_payload_bytes(req);
168 }
169
170 queue_work(nvmet_wq, &iod->work);
171 return BLK_STS_OK;
172}
173
174static void nvme_loop_submit_async_event(struct nvme_ctrl *arg)
175{
176 struct nvme_loop_ctrl *ctrl = to_loop_ctrl(arg);
177 struct nvme_loop_queue *queue = &ctrl->queues[0];
178 struct nvme_loop_iod *iod = &ctrl->async_event_iod;
179
180 memset(&iod->cmd, 0, sizeof(iod->cmd));
181 iod->cmd.common.opcode = nvme_admin_async_event;
182 iod->cmd.common.command_id = NVME_AQ_BLK_MQ_DEPTH;
183 iod->cmd.common.flags |= NVME_CMD_SGL_METABUF;
184
185 if (!nvmet_req_init(&iod->req, &queue->nvme_sq, &nvme_loop_ops)) {
186 dev_err(ctrl->ctrl.device, "failed async event work\n");
187 return;
188 }
189
190 queue_work(nvmet_wq, &iod->work);
191}
192
193static int nvme_loop_init_iod(struct nvme_loop_ctrl *ctrl,
194 struct nvme_loop_iod *iod, unsigned int queue_idx)
195{
196 iod->req.cmd = &iod->cmd;
197 iod->req.cqe = &iod->cqe;
198 iod->queue = &ctrl->queues[queue_idx];
199 INIT_WORK(&iod->work, nvme_loop_execute_work);
200 return 0;
201}
202
203static int nvme_loop_init_request(struct blk_mq_tag_set *set,
204 struct request *req, unsigned int hctx_idx,
205 int numa_node)
206{
207 struct nvme_loop_ctrl *ctrl = to_loop_ctrl(set->driver_data);
208 struct nvme_loop_iod *iod = blk_mq_rq_to_pdu(req);
209
210 nvme_req(req)->ctrl = &ctrl->ctrl;
211 nvme_req(req)->cmd = &iod->cmd;
212 return nvme_loop_init_iod(ctrl, blk_mq_rq_to_pdu(req),
213 (set == &ctrl->tag_set) ? hctx_idx + 1 : 0);
214}
215
216static struct lock_class_key loop_hctx_fq_lock_key;
217
218static int nvme_loop_init_hctx(struct blk_mq_hw_ctx *hctx, void *data,
219 unsigned int hctx_idx)
220{
221 struct nvme_loop_ctrl *ctrl = to_loop_ctrl(data);
222 struct nvme_loop_queue *queue = &ctrl->queues[hctx_idx + 1];
223
224 BUG_ON(hctx_idx >= ctrl->ctrl.queue_count);
225
226 /*
227 * flush_end_io() can be called recursively for us, so use our own
228 * lock class key for avoiding lockdep possible recursive locking,
229 * then we can remove the dynamically allocated lock class for each
230 * flush queue, that way may cause horrible boot delay.
231 */
232 blk_mq_hctx_set_fq_lock_class(hctx, &loop_hctx_fq_lock_key);
233
234 hctx->driver_data = queue;
235 return 0;
236}
237
238static int nvme_loop_init_admin_hctx(struct blk_mq_hw_ctx *hctx, void *data,
239 unsigned int hctx_idx)
240{
241 struct nvme_loop_ctrl *ctrl = to_loop_ctrl(data);
242 struct nvme_loop_queue *queue = &ctrl->queues[0];
243
244 BUG_ON(hctx_idx != 0);
245
246 hctx->driver_data = queue;
247 return 0;
248}
249
250static const struct blk_mq_ops nvme_loop_mq_ops = {
251 .queue_rq = nvme_loop_queue_rq,
252 .complete = nvme_loop_complete_rq,
253 .init_request = nvme_loop_init_request,
254 .init_hctx = nvme_loop_init_hctx,
255};
256
257static const struct blk_mq_ops nvme_loop_admin_mq_ops = {
258 .queue_rq = nvme_loop_queue_rq,
259 .complete = nvme_loop_complete_rq,
260 .init_request = nvme_loop_init_request,
261 .init_hctx = nvme_loop_init_admin_hctx,
262};
263
264static void nvme_loop_destroy_admin_queue(struct nvme_loop_ctrl *ctrl)
265{
266 if (!test_and_clear_bit(NVME_LOOP_Q_LIVE, &ctrl->queues[0].flags))
267 return;
268 /*
269 * It's possible that some requests might have been added
270 * after admin queue is stopped/quiesced. So now start the
271 * queue to flush these requests to the completion.
272 */
273 nvme_unquiesce_admin_queue(&ctrl->ctrl);
274
275 nvmet_sq_destroy(&ctrl->queues[0].nvme_sq);
276 nvmet_cq_put(&ctrl->queues[0].nvme_cq);
277}
278
279static void nvme_loop_free_ctrl(struct nvme_ctrl *nctrl)
280{
281 struct nvme_loop_ctrl *ctrl = to_loop_ctrl(nctrl);
282
283 if (list_empty(&ctrl->list))
284 goto free_ctrl;
285
286 mutex_lock(&nvme_loop_ctrl_mutex);
287 list_del(&ctrl->list);
288 mutex_unlock(&nvme_loop_ctrl_mutex);
289
290 if (nctrl->tagset)
291 nvme_remove_io_tag_set(nctrl);
292 kfree(ctrl->queues);
293 nvmf_free_options(nctrl->opts);
294free_ctrl:
295 kfree(ctrl);
296}
297
298static void nvme_loop_destroy_io_queues(struct nvme_loop_ctrl *ctrl)
299{
300 int i;
301
302 for (i = 1; i < ctrl->ctrl.queue_count; i++) {
303 clear_bit(NVME_LOOP_Q_LIVE, &ctrl->queues[i].flags);
304 nvmet_sq_destroy(&ctrl->queues[i].nvme_sq);
305 nvmet_cq_put(&ctrl->queues[i].nvme_cq);
306 }
307 ctrl->ctrl.queue_count = 1;
308 /*
309 * It's possible that some requests might have been added
310 * after io queue is stopped/quiesced. So now start the
311 * queue to flush these requests to the completion.
312 */
313 nvme_unquiesce_io_queues(&ctrl->ctrl);
314}
315
316static int nvme_loop_init_io_queues(struct nvme_loop_ctrl *ctrl)
317{
318 struct nvmf_ctrl_options *opts = ctrl->ctrl.opts;
319 unsigned int nr_io_queues;
320 int ret, i;
321
322 nr_io_queues = min(opts->nr_io_queues, num_online_cpus());
323 ret = nvme_set_queue_count(&ctrl->ctrl, &nr_io_queues);
324 if (ret || !nr_io_queues)
325 return ret;
326
327 dev_info(ctrl->ctrl.device, "creating %d I/O queues.\n", nr_io_queues);
328
329 for (i = 1; i <= nr_io_queues; i++) {
330 ctrl->queues[i].ctrl = ctrl;
331 nvmet_cq_init(&ctrl->queues[i].nvme_cq);
332 ret = nvmet_sq_init(&ctrl->queues[i].nvme_sq,
333 &ctrl->queues[i].nvme_cq);
334 if (ret) {
335 nvmet_cq_put(&ctrl->queues[i].nvme_cq);
336 goto out_destroy_queues;
337 }
338
339 ctrl->ctrl.queue_count++;
340 }
341
342 return 0;
343
344out_destroy_queues:
345 nvme_loop_destroy_io_queues(ctrl);
346 return ret;
347}
348
349static int nvme_loop_connect_io_queues(struct nvme_loop_ctrl *ctrl)
350{
351 int i, ret;
352
353 for (i = 1; i < ctrl->ctrl.queue_count; i++) {
354 ret = nvmf_connect_io_queue(&ctrl->ctrl, i);
355 if (ret)
356 return ret;
357 set_bit(NVME_LOOP_Q_LIVE, &ctrl->queues[i].flags);
358 }
359
360 return 0;
361}
362
363static int nvme_loop_configure_admin_queue(struct nvme_loop_ctrl *ctrl)
364{
365 int error;
366
367 ctrl->queues[0].ctrl = ctrl;
368 nvmet_cq_init(&ctrl->queues[0].nvme_cq);
369 error = nvmet_sq_init(&ctrl->queues[0].nvme_sq,
370 &ctrl->queues[0].nvme_cq);
371 if (error) {
372 nvmet_cq_put(&ctrl->queues[0].nvme_cq);
373 return error;
374 }
375 ctrl->ctrl.queue_count = 1;
376
377 /* reset stopped state for the fresh admin queue */
378 clear_bit(NVME_CTRL_ADMIN_Q_STOPPED, &ctrl->ctrl.flags);
379
380 error = nvmf_connect_admin_queue(&ctrl->ctrl);
381 if (error)
382 goto out_free_sq;
383
384 set_bit(NVME_LOOP_Q_LIVE, &ctrl->queues[0].flags);
385
386 error = nvme_enable_ctrl(&ctrl->ctrl);
387 if (error)
388 goto out_free_sq;
389
390 ctrl->ctrl.max_hw_sectors =
391 (NVME_LOOP_MAX_SEGMENTS - 1) << PAGE_SECTORS_SHIFT;
392
393 nvme_unquiesce_admin_queue(&ctrl->ctrl);
394
395 error = nvme_init_ctrl_finish(&ctrl->ctrl, false);
396 if (error)
397 goto out_free_sq;
398
399 return 0;
400
401out_free_sq:
402 clear_bit(NVME_LOOP_Q_LIVE, &ctrl->queues[0].flags);
403 nvmet_sq_destroy(&ctrl->queues[0].nvme_sq);
404 nvmet_cq_put(&ctrl->queues[0].nvme_cq);
405 return error;
406}
407
408static void nvme_loop_shutdown_ctrl(struct nvme_loop_ctrl *ctrl)
409{
410 if (ctrl->ctrl.queue_count > 1) {
411 nvme_quiesce_io_queues(&ctrl->ctrl);
412 nvme_loop_destroy_io_queues(ctrl);
413 }
414
415 nvme_quiesce_admin_queue(&ctrl->ctrl);
416 if (nvme_ctrl_state(&ctrl->ctrl) == NVME_CTRL_LIVE)
417 nvme_disable_ctrl(&ctrl->ctrl, true);
418
419 nvme_loop_destroy_admin_queue(ctrl);
420}
421
422static void nvme_loop_delete_ctrl_host(struct nvme_ctrl *ctrl)
423{
424 nvme_loop_shutdown_ctrl(to_loop_ctrl(ctrl));
425 nvme_remove_admin_tag_set(ctrl);
426}
427
428static void nvme_loop_delete_ctrl(struct nvmet_ctrl *nctrl)
429{
430 struct nvme_loop_ctrl *ctrl;
431
432 mutex_lock(&nvme_loop_ctrl_mutex);
433 list_for_each_entry(ctrl, &nvme_loop_ctrl_list, list) {
434 if (ctrl->ctrl.cntlid == nctrl->cntlid)
435 nvme_delete_ctrl(&ctrl->ctrl);
436 }
437 mutex_unlock(&nvme_loop_ctrl_mutex);
438}
439
440static void nvme_loop_reset_ctrl_work(struct work_struct *work)
441{
442 struct nvme_loop_ctrl *ctrl =
443 container_of(work, struct nvme_loop_ctrl, ctrl.reset_work);
444 int ret;
445
446 nvme_stop_ctrl(&ctrl->ctrl);
447 nvme_loop_shutdown_ctrl(ctrl);
448
449 if (!nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_CONNECTING)) {
450 enum nvme_ctrl_state state = nvme_ctrl_state(&ctrl->ctrl);
451
452 if (state != NVME_CTRL_DELETING &&
453 state != NVME_CTRL_DELETING_NOIO)
454 /* state change failure for non-deleted ctrl? */
455 WARN_ON_ONCE(1);
456 return;
457 }
458
459 ret = nvme_loop_configure_admin_queue(ctrl);
460 if (ret)
461 goto out_disable;
462
463 ret = nvme_loop_init_io_queues(ctrl);
464 if (ret)
465 goto out_destroy_admin;
466
467 ret = nvme_loop_connect_io_queues(ctrl);
468 if (ret)
469 goto out_destroy_io;
470
471 blk_mq_update_nr_hw_queues(&ctrl->tag_set,
472 ctrl->ctrl.queue_count - 1);
473
474 if (!nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_LIVE))
475 WARN_ON_ONCE(1);
476
477 nvme_start_ctrl(&ctrl->ctrl);
478
479 return;
480
481out_destroy_io:
482 nvme_loop_destroy_io_queues(ctrl);
483out_destroy_admin:
484 nvme_quiesce_admin_queue(&ctrl->ctrl);
485 nvme_cancel_admin_tagset(&ctrl->ctrl);
486 nvme_loop_destroy_admin_queue(ctrl);
487out_disable:
488 nvme_remove_admin_tag_set(&ctrl->ctrl);
489 dev_warn(ctrl->ctrl.device, "Removing after reset failure\n");
490 nvme_uninit_ctrl(&ctrl->ctrl);
491}
492
493static const struct nvme_ctrl_ops nvme_loop_ctrl_ops = {
494 .name = "loop",
495 .module = THIS_MODULE,
496 .flags = NVME_F_FABRICS,
497 .reg_read32 = nvmf_reg_read32,
498 .reg_read64 = nvmf_reg_read64,
499 .reg_write32 = nvmf_reg_write32,
500 .free_ctrl = nvme_loop_free_ctrl,
501 .submit_async_event = nvme_loop_submit_async_event,
502 .delete_ctrl = nvme_loop_delete_ctrl_host,
503 .get_address = nvmf_get_address,
504 .get_virt_boundary = nvme_get_virt_boundary,
505};
506
507static int nvme_loop_create_io_queues(struct nvme_loop_ctrl *ctrl)
508{
509 int ret;
510
511 ret = nvme_loop_init_io_queues(ctrl);
512 if (ret)
513 return ret;
514
515 ret = nvme_alloc_io_tag_set(&ctrl->ctrl, &ctrl->tag_set,
516 &nvme_loop_mq_ops, 1,
517 sizeof(struct nvme_loop_iod) +
518 NVME_INLINE_SG_CNT * sizeof(struct scatterlist));
519 if (ret)
520 goto out_destroy_queues;
521
522 ret = nvme_loop_connect_io_queues(ctrl);
523 if (ret)
524 goto out_cleanup_tagset;
525
526 return 0;
527
528out_cleanup_tagset:
529 nvme_remove_io_tag_set(&ctrl->ctrl);
530out_destroy_queues:
531 nvme_loop_destroy_io_queues(ctrl);
532 return ret;
533}
534
535static struct nvmet_port *nvme_loop_find_port(struct nvme_ctrl *ctrl)
536{
537 struct nvmet_port *p, *found = NULL;
538
539 mutex_lock(&nvme_loop_ports_mutex);
540 list_for_each_entry(p, &nvme_loop_ports, entry) {
541 /* if no transport address is specified use the first port */
542 if ((ctrl->opts->mask & NVMF_OPT_TRADDR) &&
543 strcmp(ctrl->opts->traddr, p->disc_addr.traddr))
544 continue;
545 found = p;
546 break;
547 }
548 mutex_unlock(&nvme_loop_ports_mutex);
549 return found;
550}
551
552static struct nvme_ctrl *nvme_loop_create_ctrl(struct device *dev,
553 struct nvmf_ctrl_options *opts)
554{
555 struct nvme_loop_ctrl *ctrl;
556 int ret;
557
558 ctrl = kzalloc_obj(*ctrl);
559 if (!ctrl)
560 return ERR_PTR(-ENOMEM);
561 ctrl->ctrl.opts = opts;
562 INIT_LIST_HEAD(&ctrl->list);
563
564 INIT_WORK(&ctrl->ctrl.reset_work, nvme_loop_reset_ctrl_work);
565
566 ret = nvme_init_ctrl(&ctrl->ctrl, dev, &nvme_loop_ctrl_ops,
567 0 /* no quirks, we're perfect! */);
568 if (ret) {
569 kfree(ctrl);
570 goto out;
571 }
572
573 ret = nvme_add_ctrl(&ctrl->ctrl);
574 if (ret)
575 goto out_put_ctrl;
576
577 if (!nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_CONNECTING))
578 WARN_ON_ONCE(1);
579
580 ret = -ENOMEM;
581
582 ctrl->ctrl.kato = opts->kato;
583 ctrl->port = nvme_loop_find_port(&ctrl->ctrl);
584
585 ctrl->queues = kzalloc_objs(*ctrl->queues, opts->nr_io_queues + 1);
586 if (!ctrl->queues)
587 goto out_uninit_ctrl;
588
589 ret = nvme_alloc_admin_tag_set(&ctrl->ctrl, &ctrl->admin_tag_set,
590 &nvme_loop_admin_mq_ops,
591 sizeof(struct nvme_loop_iod) +
592 NVME_INLINE_SG_CNT * sizeof(struct scatterlist));
593 if (ret)
594 goto out_free_queues;
595
596 ret = nvme_loop_configure_admin_queue(ctrl);
597 if (ret)
598 goto out_remove_admin_tagset;
599
600 if (opts->queue_size > ctrl->ctrl.maxcmd) {
601 /* warn if maxcmd is lower than queue_size */
602 dev_warn(ctrl->ctrl.device,
603 "queue_size %zu > ctrl maxcmd %u, clamping down\n",
604 opts->queue_size, ctrl->ctrl.maxcmd);
605 opts->queue_size = ctrl->ctrl.maxcmd;
606 }
607 ctrl->ctrl.sqsize = opts->queue_size - 1;
608
609 if (opts->nr_io_queues) {
610 ret = nvme_loop_create_io_queues(ctrl);
611 if (ret)
612 goto out_remove_admin_queue;
613 }
614
615 nvme_loop_init_iod(ctrl, &ctrl->async_event_iod, 0);
616
617 dev_info(ctrl->ctrl.device,
618 "new ctrl: \"%s\"\n", ctrl->ctrl.opts->subsysnqn);
619
620 if (!nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_LIVE))
621 WARN_ON_ONCE(1);
622
623 mutex_lock(&nvme_loop_ctrl_mutex);
624 list_add_tail(&ctrl->list, &nvme_loop_ctrl_list);
625 mutex_unlock(&nvme_loop_ctrl_mutex);
626
627 nvme_start_ctrl(&ctrl->ctrl);
628
629 return &ctrl->ctrl;
630
631out_remove_admin_queue:
632 nvme_quiesce_admin_queue(&ctrl->ctrl);
633 nvme_cancel_admin_tagset(&ctrl->ctrl);
634 nvme_loop_destroy_admin_queue(ctrl);
635out_remove_admin_tagset:
636 nvme_remove_admin_tag_set(&ctrl->ctrl);
637out_free_queues:
638 kfree(ctrl->queues);
639out_uninit_ctrl:
640 nvme_uninit_ctrl(&ctrl->ctrl);
641out_put_ctrl:
642 nvme_put_ctrl(&ctrl->ctrl);
643out:
644 if (ret > 0)
645 ret = -EIO;
646 return ERR_PTR(ret);
647}
648
649static int nvme_loop_add_port(struct nvmet_port *port)
650{
651 mutex_lock(&nvme_loop_ports_mutex);
652 list_add_tail(&port->entry, &nvme_loop_ports);
653 mutex_unlock(&nvme_loop_ports_mutex);
654 return 0;
655}
656
657static void nvme_loop_remove_port(struct nvmet_port *port)
658{
659 mutex_lock(&nvme_loop_ports_mutex);
660 list_del_init(&port->entry);
661 mutex_unlock(&nvme_loop_ports_mutex);
662
663 /*
664 * Ensure any ctrls that are in the process of being
665 * deleted are in fact deleted before we return
666 * and free the port. This is to prevent active
667 * ctrls from using a port after it's freed.
668 */
669 flush_workqueue(nvme_delete_wq);
670}
671
672static const struct nvmet_fabrics_ops nvme_loop_ops = {
673 .owner = THIS_MODULE,
674 .type = NVMF_TRTYPE_LOOP,
675 .add_port = nvme_loop_add_port,
676 .remove_port = nvme_loop_remove_port,
677 .queue_response = nvme_loop_queue_response,
678 .delete_ctrl = nvme_loop_delete_ctrl,
679};
680
681static struct nvmf_transport_ops nvme_loop_transport = {
682 .name = "loop",
683 .module = THIS_MODULE,
684 .create_ctrl = nvme_loop_create_ctrl,
685 .allowed_opts = NVMF_OPT_TRADDR,
686};
687
688static int __init nvme_loop_init_module(void)
689{
690 int ret;
691
692 ret = nvmet_register_transport(&nvme_loop_ops);
693 if (ret)
694 return ret;
695
696 ret = nvmf_register_transport(&nvme_loop_transport);
697 if (ret)
698 nvmet_unregister_transport(&nvme_loop_ops);
699
700 return ret;
701}
702
703static void __exit nvme_loop_cleanup_module(void)
704{
705 struct nvme_loop_ctrl *ctrl, *next;
706
707 nvmf_unregister_transport(&nvme_loop_transport);
708 nvmet_unregister_transport(&nvme_loop_ops);
709
710 mutex_lock(&nvme_loop_ctrl_mutex);
711 list_for_each_entry_safe(ctrl, next, &nvme_loop_ctrl_list, list)
712 nvme_delete_ctrl(&ctrl->ctrl);
713 mutex_unlock(&nvme_loop_ctrl_mutex);
714
715 flush_workqueue(nvme_delete_wq);
716}
717
718module_init(nvme_loop_init_module);
719module_exit(nvme_loop_cleanup_module);
720
721MODULE_DESCRIPTION("NVMe target loop transport driver");
722MODULE_LICENSE("GPL v2");
723MODULE_ALIAS("nvmet-transport-254"); /* 254 == NVMF_TRTYPE_LOOP */