linux - drivers/vhost/scsi.c

3034 lines · 76.3 KiB

1	// SPDX-License-Identifier: GPL-2.0+
2	/*******************************************************************************
3	* Vhost kernel TCM fabric driver for virtio SCSI initiators
4	*
5	* (C) Copyright 2010-2013 Datera, Inc.
6	* (C) Copyright 2010-2012 IBM Corp.
7	*
8	* Authors: Nicholas A. Bellinger <[email protected]>
9	* Stefan Hajnoczi <[email protected]>
10	****************************************************************************/
11
12	#include <linux/module.h>
13	#include <linux/moduleparam.h>
14	#include <generated/utsrelease.h>
15	#include <linux/utsname.h>
16	#include <linux/init.h>
17	#include <linux/slab.h>
18	#include <linux/kthread.h>
19	#include <linux/types.h>
20	#include <linux/string.h>
21	#include <linux/configfs.h>
22	#include <linux/ctype.h>
23	#include <linux/compat.h>
24	#include <linux/eventfd.h>
25	#include <linux/fs.h>
26	#include <linux/vmalloc.h>
27	#include <linux/miscdevice.h>
28	#include <linux/blk_types.h>
29	#include <linux/bio.h>
30	#include <linux/unaligned.h>
31	#include <scsi/scsi_common.h>
32	#include <scsi/scsi_proto.h>
33	#include <target/target_core_base.h>
34	#include <target/target_core_fabric.h>
35	#include <linux/vhost.h>
36	#include <linux/virtio_scsi.h>
37	#include <linux/llist.h>
38	#include <linux/bitmap.h>
39
40	#include "vhost.h"
41
42	#define VHOST_SCSI_VERSION "v0.1"
43	#define VHOST_SCSI_NAMELEN 256
44	#define VHOST_SCSI_MAX_CDB_SIZE 32
45	#define VHOST_SCSI_PREALLOC_SGLS 2048
46	#define VHOST_SCSI_PREALLOC_UPAGES 2048
47	#define VHOST_SCSI_PREALLOC_PROT_SGLS 2048
48	/*
49	* For the legacy descriptor case we allocate an iov per byte in the
50	* virtio_scsi_cmd_resp struct.
51	*/
52	#define VHOST_SCSI_MAX_RESP_IOVS sizeof(struct virtio_scsi_cmd_resp)
53
54	static unsigned int vhost_scsi_inline_sg_cnt = VHOST_SCSI_PREALLOC_SGLS;
55
56	#ifdef CONFIG_ARCH_NO_SG_CHAIN
57	static int vhost_scsi_set_inline_sg_cnt(const char *buf,
58	const struct kernel_param *kp)
59	{
60	pr_err("Setting inline_sg_cnt is not supported.\n");
61	return -EOPNOTSUPP;
62	}
63	#else
64	static int vhost_scsi_set_inline_sg_cnt(const char *buf,
65	const struct kernel_param *kp)
66	{
67	unsigned int cnt;
68	int ret;
69
70	ret = kstrtouint(buf, 10, &cnt);
71	if (ret)
72	return ret;
73
74	if (cnt > VHOST_SCSI_PREALLOC_SGLS) {
75	pr_err("Max inline_sg_cnt is %u\n", VHOST_SCSI_PREALLOC_SGLS);
76	return -EINVAL;
77	}
78
79	vhost_scsi_inline_sg_cnt = cnt;
80	return 0;
81	}
82	#endif
83
84	static int vhost_scsi_get_inline_sg_cnt(char *buf,
85	const struct kernel_param *kp)
86	{
87	return sprintf(buf, "%u\n", vhost_scsi_inline_sg_cnt);
88	}
89
90	static const struct kernel_param_ops vhost_scsi_inline_sg_cnt_op = {
91	.get = vhost_scsi_get_inline_sg_cnt,
92	.set = vhost_scsi_set_inline_sg_cnt,
93	};
94
95	module_param_cb(inline_sg_cnt, &vhost_scsi_inline_sg_cnt_op, NULL, 0644);
96	MODULE_PARM_DESC(inline_sg_cnt, "Set the number of scatterlist entries to pre-allocate. The default is 2048.");
97
98	/* Max number of requests before requeueing the job.
99	* Using this limit prevents one virtqueue from starving others with
100	* request.
101	*/
102	#define VHOST_SCSI_WEIGHT 256
103
104	struct vhost_scsi_inflight {
105	/* Wait for the flush operation to finish */
106	struct completion comp;
107	/* Refcount for the inflight reqs */
108	struct kref kref;
109	};
110
111	struct vhost_scsi_cmd {
112	/* Descriptor from vhost_get_vq_desc() for virt_queue segment */
113	int tvc_vq_desc;
114	/* The number of scatterlists associated with this cmd */
115	u32 tvc_sgl_count;
116	u32 tvc_prot_sgl_count;
117	u32 copied_iov:1;
118	const void *read_iov;
119	struct iov_iter *read_iter;
120	struct scatterlist *sgl;
121	struct sg_table table;
122	struct scatterlist *prot_sgl;
123	struct sg_table prot_table;
124	/* Fast path response header iovec used when only one vec is needed */
125	struct iovec tvc_resp_iov;
126	/* Number of iovs for response */
127	unsigned int tvc_resp_iovs_cnt;
128	/* Pointer to response header iovecs if more than one is needed */
129	struct iovec *tvc_resp_iovs;
130	/* Pointer to vhost_virtqueue for the cmd */
131	struct vhost_virtqueue *tvc_vq;
132	/* The TCM I/O descriptor that is accessed via container_of() */
133	struct se_cmd tvc_se_cmd;
134	/* Sense buffer that will be mapped into outgoing status */
135	unsigned char tvc_sense_buf[TRANSPORT_SENSE_BUFFER];
136	/*
137	* Dirty write descriptors of this command.
138	*/
139	struct vhost_log *tvc_log;
140	unsigned int tvc_log_num;
141	/* Completed commands list, serviced from vhost worker thread */
142	struct llist_node tvc_completion_list;
143	/* Used to track inflight cmd */
144	struct vhost_scsi_inflight *inflight;
145	};
146
147	struct vhost_scsi_nexus {
148	/* Pointer to TCM session for I_T Nexus */
149	struct se_session *tvn_se_sess;
150	};
151
152	struct vhost_scsi_tpg {
153	/* Vhost port target portal group tag for TCM */
154	u16 tport_tpgt;
155	/* Used to track number of TPG Port/Lun Links wrt to explicit I_T Nexus shutdown */
156	int tv_tpg_port_count;
157	/* Used for vhost_scsi device reference to tpg_nexus, protected by tv_tpg_mutex */
158	int tv_tpg_vhost_count;
159	/* Used for enabling T10-PI with legacy devices */
160	int tv_fabric_prot_type;
161	/* list for vhost_scsi_list */
162	struct list_head tv_tpg_list;
163	/* Used to protect access for tpg_nexus */
164	struct mutex tv_tpg_mutex;
165	/* Pointer to the TCM VHost I_T Nexus for this TPG endpoint */
166	struct vhost_scsi_nexus *tpg_nexus;
167	/* Pointer back to vhost_scsi_tport */
168	struct vhost_scsi_tport *tport;
169	/* Returned by vhost_scsi_make_tpg() */
170	struct se_portal_group se_tpg;
171	/* Pointer back to vhost_scsi, protected by tv_tpg_mutex */
172	struct vhost_scsi *vhost_scsi;
173	};
174
175	struct vhost_scsi_tport {
176	/* SCSI protocol the tport is providing */
177	u8 tport_proto_id;
178	/* Binary World Wide unique Port Name for Vhost Target port */
179	u64 tport_wwpn;
180	/* ASCII formatted WWPN for Vhost Target port */
181	char tport_name[VHOST_SCSI_NAMELEN];
182	/* Returned by vhost_scsi_make_tport() */
183	struct se_wwn tport_wwn;
184	};
185
186	struct vhost_scsi_evt {
187	/* event to be sent to guest */
188	struct virtio_scsi_event event;
189	/* event list, serviced from vhost worker thread */
190	struct llist_node list;
191	};
192
193	enum {
194	VHOST_SCSI_VQ_CTL = 0,
195	VHOST_SCSI_VQ_EVT = 1,
196	VHOST_SCSI_VQ_IO = 2,
197	};
198
199	/* Note: can't set VIRTIO_F_VERSION_1 yet, since that implies ANY_LAYOUT. */
200	static const int vhost_scsi_bits[] = {
201	VHOST_FEATURES,
202	VIRTIO_SCSI_F_HOTPLUG,
203	VIRTIO_SCSI_F_T10_PI
204	};
205
206	#define VHOST_SCSI_FEATURES VHOST_FEATURES_U64(vhost_scsi_bits, 0)
207
208	#define VHOST_SCSI_MAX_TARGET 256
209	#define VHOST_SCSI_MAX_IO_VQ 1024
210	#define VHOST_SCSI_MAX_EVENT 128
211
212	static unsigned vhost_scsi_max_io_vqs = 128;
213
214	static int vhost_scsi_set_max_io_vqs(const char *val,
215	const struct kernel_param *kp)
216	{
217	unsigned int max_io_vqs;
218	int ret;
219
220	ret = kstrtouint(val, 0, &max_io_vqs);
221	if (ret)
222	return ret;
223
224	if (max_io_vqs > VHOST_SCSI_MAX_IO_VQ) {
225	pr_err("Invalid max_io_vqs of %u. Using %u.\n",
226	max_io_vqs, VHOST_SCSI_MAX_IO_VQ);
227	max_io_vqs = VHOST_SCSI_MAX_IO_VQ;
228	} else if (!max_io_vqs) {
229	pr_err("Invalid max_io_vqs of 0. Using 1.\n");
230	max_io_vqs = 1;
231	}
232
233	WRITE_ONCE(vhost_scsi_max_io_vqs, max_io_vqs);
234	return 0;
235	}
236
237	static const struct kernel_param_ops vhost_scsi_max_io_vqs_op = {
238	.set = vhost_scsi_set_max_io_vqs,
239	.get = param_get_uint,
240	};
241
242	module_param_cb(max_io_vqs, &vhost_scsi_max_io_vqs_op,
243	&vhost_scsi_max_io_vqs, 0644);
244	MODULE_PARM_DESC(max_io_vqs, "Set the max number of IO virtqueues a vhost scsi device can support. The default is 128. The max is 1024.");
245
246	struct vhost_scsi_virtqueue {
247	struct vhost_virtqueue vq;
248	struct vhost_scsi *vs;
249	/*
250	* Reference counting for inflight reqs, used for flush operation. At
251	* each time, one reference tracks new commands submitted, while we
252	* wait for another one to reach 0.
253	*/
254	struct vhost_scsi_inflight inflights[2];
255	/*
256	* Indicate current inflight in use, protected by vq->mutex.
257	* Writers must also take dev mutex and flush under it.
258	*/
259	int inflight_idx;
260	struct vhost_scsi_cmd *scsi_cmds;
261	struct sbitmap scsi_tags;
262	int max_cmds;
263	struct page **upages;
264
265	struct vhost_work completion_work;
266	struct llist_head completion_list;
267	};
268
269	struct vhost_scsi {
270	/* Protected by vhost_scsi->dev.mutex */
271	struct vhost_scsi_tpg **vs_tpg;
272	char vs_vhost_wwpn[TRANSPORT_IQN_LEN];
273
274	struct vhost_dev dev;
275	struct vhost_scsi_virtqueue *vqs;
276	struct vhost_scsi_inflight **old_inflight;
277
278	struct vhost_work vs_event_work; /* evt injection work item */
279	struct llist_head vs_event_list; /* evt injection queue */
280
281	bool vs_events_missed; /* any missed events, protected by vq->mutex */
282	int vs_events_nr; /* num of pending events, protected by vq->mutex */
283
284	unsigned int inline_sg_cnt;
285	};
286
287	struct vhost_scsi_tmf {
288	struct vhost_work vwork;
289	struct work_struct flush_work;
290	struct vhost_scsi *vhost;
291	struct vhost_scsi_virtqueue *svq;
292
293	struct se_cmd se_cmd;
294	u8 scsi_resp;
295	struct vhost_scsi_inflight *inflight;
296	struct iovec resp_iov;
297	int in_iovs;
298	int vq_desc;
299
300	/*
301	* Dirty write descriptors of this command.
302	*/
303	struct vhost_log *tmf_log;
304	unsigned int tmf_log_num;
305	};
306
307	/*
308	* Context for processing request and control queue operations.
309	*/
310	struct vhost_scsi_ctx {
311	int head;
312	unsigned int out, in;
313	size_t req_size, rsp_size;
314	size_t out_size, in_size;
315	u8 target, lunp;
316	void *req;
317	struct iov_iter out_iter;
318	};
319
320	/*
321	* Global mutex to protect vhost_scsi TPG list for vhost IOCTLs and LIO
322	* configfs management operations.
323	*/
324	static DEFINE_MUTEX(vhost_scsi_mutex);
325	static LIST_HEAD(vhost_scsi_list);
326
327	static void vhost_scsi_done_inflight(struct kref *kref)
328	{
329	struct vhost_scsi_inflight *inflight;
330
331	inflight = container_of(kref, struct vhost_scsi_inflight, kref);
332	complete(&inflight->comp);
333	}
334
335	static void vhost_scsi_init_inflight(struct vhost_scsi *vs,
336	struct vhost_scsi_inflight *old_inflight[])
337	{
338	struct vhost_scsi_inflight *new_inflight;
339	struct vhost_virtqueue *vq;
340	int idx, i;
341
342	for (i = 0; i < vs->dev.nvqs; i++) {
343	vq = &vs->vqs[i].vq;
344
345	mutex_lock(&vq->mutex);
346
347	/* store old inflight */
348	idx = vs->vqs[i].inflight_idx;
349	if (old_inflight)
350	old_inflight[i] = &vs->vqs[i].inflights[idx];
351
352	/* setup new inflight */
353	vs->vqs[i].inflight_idx = idx ^ 1;
354	new_inflight = &vs->vqs[i].inflights[idx ^ 1];
355	kref_init(&new_inflight->kref);
356	init_completion(&new_inflight->comp);
357
358	mutex_unlock(&vq->mutex);
359	}
360	}
361
362	static struct vhost_scsi_inflight *
363	vhost_scsi_get_inflight(struct vhost_virtqueue *vq)
364	{
365	struct vhost_scsi_inflight *inflight;
366	struct vhost_scsi_virtqueue *svq;
367
368	svq = container_of(vq, struct vhost_scsi_virtqueue, vq);
369	inflight = &svq->inflights[svq->inflight_idx];
370	kref_get(&inflight->kref);
371
372	return inflight;
373	}
374
375	static void vhost_scsi_put_inflight(struct vhost_scsi_inflight *inflight)
376	{
377	kref_put(&inflight->kref, vhost_scsi_done_inflight);
378	}
379
380	static int vhost_scsi_check_true(struct se_portal_group *se_tpg)
381	{
382	return 1;
383	}
384
385	static char vhost_scsi_get_fabric_wwn(struct se_portal_group se_tpg)
386	{
387	struct vhost_scsi_tpg *tpg = container_of(se_tpg,
388	struct vhost_scsi_tpg, se_tpg);
389	struct vhost_scsi_tport *tport = tpg->tport;
390
391	return &tport->tport_name[0];
392	}
393
394	static u16 vhost_scsi_get_tpgt(struct se_portal_group *se_tpg)
395	{
396	struct vhost_scsi_tpg *tpg = container_of(se_tpg,
397	struct vhost_scsi_tpg, se_tpg);
398	return tpg->tport_tpgt;
399	}
400
401	static int vhost_scsi_check_prot_fabric_only(struct se_portal_group *se_tpg)
402	{
403	struct vhost_scsi_tpg *tpg = container_of(se_tpg,
404	struct vhost_scsi_tpg, se_tpg);
405
406	return tpg->tv_fabric_prot_type;
407	}
408
409	static int vhost_scsi_copy_cmd_log(struct vhost_virtqueue *vq,
410	struct vhost_scsi_cmd *cmd,
411	struct vhost_log *log,
412	unsigned int log_num)
413	{
414	if (!cmd->tvc_log)
415	cmd->tvc_log = kmalloc_objs(*cmd->tvc_log, vq->dev->iov_limit);
416
417	if (unlikely(!cmd->tvc_log)) {
418	vq_err(vq, "Failed to alloc tvc_log\n");
419	return -ENOMEM;
420	}
421
422	memcpy(cmd->tvc_log, log, sizeof(cmd->tvc_log) log_num);
423	cmd->tvc_log_num = log_num;
424
425	return 0;
426	}
427
428	static void vhost_scsi_log_write(struct vhost_virtqueue *vq,
429	struct vhost_log *log,
430	unsigned int log_num)
431	{
432	if (likely(!vhost_has_feature(vq, VHOST_F_LOG_ALL)))
433	return;
434
435	if (likely(!log_num \|\| !log))
436	return;
437
438	/*
439	* vhost-scsi doesn't support VIRTIO_F_ACCESS_PLATFORM.
440	* No requirement for vq->iotlb case.
441	*/
442	WARN_ON_ONCE(unlikely(vq->iotlb));
443	vhost_log_write(vq, log, log_num, U64_MAX, NULL, 0);
444	}
445
446	static void vhost_scsi_release_cmd_res(struct se_cmd *se_cmd)
447	{
448	struct vhost_scsi_cmd *tv_cmd = container_of(se_cmd,
449	struct vhost_scsi_cmd, tvc_se_cmd);
450	struct vhost_scsi_virtqueue *svq = container_of(tv_cmd->tvc_vq,
451	struct vhost_scsi_virtqueue, vq);
452	struct vhost_scsi *vs = svq->vs;
453	struct vhost_scsi_inflight *inflight = tv_cmd->inflight;
454	struct scatterlist *sg;
455	struct page *page;
456	int i;
457
458	if (tv_cmd->tvc_sgl_count) {
459	for_each_sgtable_sg(&tv_cmd->table, sg, i) {
460	page = sg_page(sg);
461	if (!page)
462	continue;
463
464	if (tv_cmd->copied_iov)
465	__free_page(page);
466	else
467	put_page(page);
468	}
469	kfree(tv_cmd->read_iter);
470	kfree(tv_cmd->read_iov);
471	sg_free_table_chained(&tv_cmd->table, vs->inline_sg_cnt);
472	}
473	if (tv_cmd->tvc_prot_sgl_count) {
474	for_each_sgtable_sg(&tv_cmd->prot_table, sg, i) {
475	page = sg_page(sg);
476	if (page)
477	put_page(page);
478	}
479	sg_free_table_chained(&tv_cmd->prot_table, vs->inline_sg_cnt);
480	}
481
482	if (tv_cmd->tvc_resp_iovs != &tv_cmd->tvc_resp_iov)
483	kfree(tv_cmd->tvc_resp_iovs);
484	sbitmap_clear_bit(&svq->scsi_tags, se_cmd->map_tag);
485	vhost_scsi_put_inflight(inflight);
486	}
487
488	static void vhost_scsi_release_tmf_res(struct vhost_scsi_tmf *tmf)
489	{
490	struct vhost_scsi_inflight *inflight = tmf->inflight;
491
492	/*
493	* tmf->tmf_log is default NULL unless VHOST_F_LOG_ALL is set.
494	*/
495	kfree(tmf->tmf_log);
496	kfree(tmf);
497	vhost_scsi_put_inflight(inflight);
498	}
499
500	static void vhost_scsi_drop_cmds(struct vhost_scsi_virtqueue *svq)
501	{
502	struct vhost_scsi_cmd cmd, t;
503	struct llist_node *llnode;
504
505	llnode = llist_del_all(&svq->completion_list);
506	llist_for_each_entry_safe(cmd, t, llnode, tvc_completion_list)
507	vhost_scsi_release_cmd_res(&cmd->tvc_se_cmd);
508	}
509
510	static void vhost_scsi_release_cmd(struct se_cmd *se_cmd)
511	{
512	if (se_cmd->se_cmd_flags & SCF_SCSI_TMR_CDB) {
513	struct vhost_scsi_tmf *tmf = container_of(se_cmd,
514	struct vhost_scsi_tmf, se_cmd);
515
516	schedule_work(&tmf->flush_work);
517	} else {
518	struct vhost_scsi_cmd *cmd = container_of(se_cmd,
519	struct vhost_scsi_cmd, tvc_se_cmd);
520	struct vhost_scsi_virtqueue *svq = container_of(cmd->tvc_vq,
521	struct vhost_scsi_virtqueue, vq);
522
523	llist_add(&cmd->tvc_completion_list, &svq->completion_list);
524	if (!vhost_vq_work_queue(&svq->vq, &svq->completion_work))
525	vhost_scsi_drop_cmds(svq);
526	}
527	}
528
529	static int vhost_scsi_write_pending(struct se_cmd *se_cmd)
530	{
531	/* Go ahead and process the write immediately */
532	target_execute_cmd(se_cmd);
533	return 0;
534	}
535
536	static int vhost_scsi_queue_data_in(struct se_cmd *se_cmd)
537	{
538	transport_generic_free_cmd(se_cmd, 0);
539	return 0;
540	}
541
542	static int vhost_scsi_queue_status(struct se_cmd *se_cmd)
543	{
544	transport_generic_free_cmd(se_cmd, 0);
545	return 0;
546	}
547
548	static void vhost_scsi_queue_tm_rsp(struct se_cmd *se_cmd)
549	{
550	struct vhost_scsi_tmf *tmf = container_of(se_cmd, struct vhost_scsi_tmf,
551	se_cmd);
552
553	tmf->scsi_resp = se_cmd->se_tmr_req->response;
554	transport_generic_free_cmd(&tmf->se_cmd, 0);
555	}
556
557	static void vhost_scsi_aborted_task(struct se_cmd *se_cmd)
558	{
559	return;
560	}
561
562	static void vhost_scsi_free_evt(struct vhost_scsi vs, struct vhost_scsi_evt evt)
563	{
564	vs->vs_events_nr--;
565	kfree(evt);
566	}
567
568	static struct vhost_scsi_evt *
569	vhost_scsi_allocate_evt(struct vhost_scsi *vs,
570	u32 event, u32 reason)
571	{
572	struct vhost_virtqueue *vq = &vs->vqs[VHOST_SCSI_VQ_EVT].vq;
573	struct vhost_scsi_evt *evt;
574
575	if (vs->vs_events_nr > VHOST_SCSI_MAX_EVENT) {
576	vs->vs_events_missed = true;
577	return NULL;
578	}
579
580	evt = kzalloc_obj(*evt);
581	if (!evt) {
582	vq_err(vq, "Failed to allocate vhost_scsi_evt\n");
583	vs->vs_events_missed = true;
584	return NULL;
585	}
586
587	evt->event.event = cpu_to_vhost32(vq, event);
588	evt->event.reason = cpu_to_vhost32(vq, reason);
589	vs->vs_events_nr++;
590
591	return evt;
592	}
593
594	static int vhost_scsi_check_stop_free(struct se_cmd *se_cmd)
595	{
596	return target_put_sess_cmd(se_cmd);
597	}
598
599	static void
600	vhost_scsi_do_evt_work(struct vhost_scsi vs, struct vhost_scsi_evt evt)
601	{
602	struct vhost_virtqueue *vq = &vs->vqs[VHOST_SCSI_VQ_EVT].vq;
603	struct virtio_scsi_event *event = &evt->event;
604	struct virtio_scsi_event __user *eventp;
605	struct vhost_log *vq_log;
606	unsigned int log_num;
607	unsigned out, in;
608	int head, ret;
609
610	if (!vhost_vq_get_backend(vq)) {
611	vs->vs_events_missed = true;
612	return;
613	}
614
615	again:
616	vhost_disable_notify(&vs->dev, vq);
617
618	vq_log = unlikely(vhost_has_feature(vq, VHOST_F_LOG_ALL)) ?
619	vq->log : NULL;
620
621	/*
622	* Reset 'log_num' since vhost_get_vq_desc() may reset it only
623	* after certain condition checks.
624	*/
625	log_num = 0;
626
627	head = vhost_get_vq_desc(vq, vq->iov,
628	ARRAY_SIZE(vq->iov), &out, &in,
629	vq_log, &log_num);
630	if (head < 0) {
631	vs->vs_events_missed = true;
632	return;
633	}
634	if (head == vq->num) {
635	if (vhost_enable_notify(&vs->dev, vq))
636	goto again;
637	vs->vs_events_missed = true;
638	return;
639	}
640
641	if ((vq->iov[out].iov_len != sizeof(struct virtio_scsi_event))) {
642	vq_err(vq, "Expecting virtio_scsi_event, got %zu bytes\n",
643	vq->iov[out].iov_len);
644	vs->vs_events_missed = true;
645	return;
646	}
647
648	if (vs->vs_events_missed) {
649	event->event \|= cpu_to_vhost32(vq, VIRTIO_SCSI_T_EVENTS_MISSED);
650	vs->vs_events_missed = false;
651	}
652
653	eventp = vq->iov[out].iov_base;
654	ret = __copy_to_user(eventp, event, sizeof(*event));
655	if (!ret)
656	vhost_add_used_and_signal(&vs->dev, vq, head, 0);
657	else
658	vq_err(vq, "Faulted on vhost_scsi_send_event\n");
659
660	vhost_scsi_log_write(vq, vq_log, log_num);
661	}
662
663	static void vhost_scsi_complete_events(struct vhost_scsi *vs, bool drop)
664	{
665	struct vhost_virtqueue *vq = &vs->vqs[VHOST_SCSI_VQ_EVT].vq;
666	struct vhost_scsi_evt evt, t;
667	struct llist_node *llnode;
668
669	mutex_lock(&vq->mutex);
670	llnode = llist_del_all(&vs->vs_event_list);
671	llist_for_each_entry_safe(evt, t, llnode, list) {
672	if (!drop)
673	vhost_scsi_do_evt_work(vs, evt);
674	vhost_scsi_free_evt(vs, evt);
675	}
676	mutex_unlock(&vq->mutex);
677	}
678
679	static void vhost_scsi_evt_work(struct vhost_work *work)
680	{
681	struct vhost_scsi *vs = container_of(work, struct vhost_scsi,
682	vs_event_work);
683	vhost_scsi_complete_events(vs, false);
684	}
685
686	static int vhost_scsi_copy_sgl_to_iov(struct vhost_scsi_cmd *cmd)
687	{
688	struct iov_iter *iter = cmd->read_iter;
689	struct scatterlist *sg;
690	struct page *page;
691	size_t len;
692	int i;
693
694	for_each_sgtable_sg(&cmd->table, sg, i) {
695	page = sg_page(sg);
696	if (!page)
697	continue;
698
699	len = sg->length;
700
701	if (copy_page_to_iter(page, 0, len, iter) != len) {
702	pr_err("Could not copy data while handling misaligned cmd. Error %zu\n",
703	len);
704	return -1;
705	}
706	}
707
708	return 0;
709	}
710
711	/* Fill in status and signal that we are done processing this command
712	*
713	* This is scheduled in the vhost work queue so we are called with the owner
714	* process mm and can access the vring.
715	*/
716	static void vhost_scsi_complete_cmd_work(struct vhost_work *work)
717	{
718	struct vhost_scsi_virtqueue *svq = container_of(work,
719	struct vhost_scsi_virtqueue, completion_work);
720	struct virtio_scsi_cmd_resp v_rsp;
721	struct vhost_scsi_cmd cmd, t;
722	struct llist_node *llnode;
723	struct se_cmd *se_cmd;
724	struct iov_iter iov_iter;
725	bool signal = false;
726	int ret;
727
728	llnode = llist_del_all(&svq->completion_list);
729
730	mutex_lock(&svq->vq.mutex);
731
732	llist_for_each_entry_safe(cmd, t, llnode, tvc_completion_list) {
733	se_cmd = &cmd->tvc_se_cmd;
734
735	pr_debug("%s tv_cmd %p resid %u status %#02x\n", __func__,
736	cmd, se_cmd->residual_count, se_cmd->scsi_status);
737	memset(&v_rsp, 0, sizeof(v_rsp));
738
739	if (cmd->read_iter && vhost_scsi_copy_sgl_to_iov(cmd)) {
740	v_rsp.response = VIRTIO_SCSI_S_BAD_TARGET;
741	} else {
742	v_rsp.resid = cpu_to_vhost32(cmd->tvc_vq,
743	se_cmd->residual_count);
744	/* TODO is status_qualifier field needed? */
745	v_rsp.status = se_cmd->scsi_status;
746	v_rsp.sense_len = cpu_to_vhost32(cmd->tvc_vq,
747	se_cmd->scsi_sense_length);
748	memcpy(v_rsp.sense, cmd->tvc_sense_buf,
749	se_cmd->scsi_sense_length);
750	}
751
752	iov_iter_init(&iov_iter, ITER_DEST, cmd->tvc_resp_iovs,
753	cmd->tvc_resp_iovs_cnt, sizeof(v_rsp));
754	ret = copy_to_iter(&v_rsp, sizeof(v_rsp), &iov_iter);
755	if (likely(ret == sizeof(v_rsp))) {
756	signal = true;
757
758	vhost_add_used(cmd->tvc_vq, cmd->tvc_vq_desc, 0);
759	} else
760	pr_err("Faulted on virtio_scsi_cmd_resp\n");
761
762	vhost_scsi_log_write(cmd->tvc_vq, cmd->tvc_log,
763	cmd->tvc_log_num);
764
765	vhost_scsi_release_cmd_res(se_cmd);
766	}
767
768	mutex_unlock(&svq->vq.mutex);
769
770	if (signal)
771	vhost_signal(&svq->vs->dev, &svq->vq);
772	}
773
774	static struct vhost_scsi_cmd *
775	vhost_scsi_get_cmd(struct vhost_virtqueue *vq, u64 scsi_tag)
776	{
777	struct vhost_scsi_virtqueue *svq = container_of(vq,
778	struct vhost_scsi_virtqueue, vq);
779	struct vhost_scsi_cmd *cmd;
780	struct scatterlist sgl, prot_sgl;
781	struct vhost_log *log;
782	int tag;
783
784	tag = sbitmap_get(&svq->scsi_tags);
785	if (tag < 0) {
786	pr_warn_once("Guest sent too many cmds. Returning TASK_SET_FULL.\n");
787	return ERR_PTR(-ENOMEM);
788	}
789
790	cmd = &svq->scsi_cmds[tag];
791	sgl = cmd->sgl;
792	prot_sgl = cmd->prot_sgl;
793	log = cmd->tvc_log;
794	memset(cmd, 0, sizeof(*cmd));
795	cmd->sgl = sgl;
796	cmd->prot_sgl = prot_sgl;
797	cmd->tvc_log = log;
798	cmd->tvc_se_cmd.map_tag = tag;
799	cmd->inflight = vhost_scsi_get_inflight(vq);
800
801	return cmd;
802	}
803
804	static void vhost_scsi_revert_map_iov_to_sgl(struct iov_iter *iter,
805	struct scatterlist *curr,
806	struct scatterlist *end)
807	{
808	size_t revert_bytes = 0;
809	struct page *page;
810
811	while (curr != end) {
812	page = sg_page(curr);
813
814	if (page) {
815	put_page(page);
816	revert_bytes += curr->length;
817	}
818	/* Clear so we can re-use it for the copy path */
819	sg_set_page(curr, NULL, 0, 0);
820	curr = sg_next(curr);
821	}
822	iov_iter_revert(iter, revert_bytes);
823	}
824
825	/*
826	* Map a user memory range into a scatterlist
827	*
828	* Returns the number of scatterlist entries used or -errno on error.
829	*/
830	static int
831	vhost_scsi_map_to_sgl(struct vhost_scsi_cmd *cmd,
832	struct iov_iter *iter,
833	struct sg_table *sg_table,
834	struct scatterlist **sgl,
835	bool is_prot)
836	{
837	struct vhost_scsi_virtqueue *svq = container_of(cmd->tvc_vq,
838	struct vhost_scsi_virtqueue, vq);
839	struct page **pages = svq->upages;
840	struct scatterlist sg = sgl;
841	ssize_t bytes;
842	size_t offset;
843	unsigned int n, npages = 0;
844
845	bytes = iov_iter_get_pages2(iter, pages, LONG_MAX,
846	VHOST_SCSI_PREALLOC_UPAGES, &offset);
847	/* No pages were pinned */
848	if (bytes <= 0)
849	return bytes < 0 ? bytes : -EFAULT;
850
851	while (bytes) {
852	n = min_t(unsigned int, PAGE_SIZE - offset, bytes);
853	/*
854	* The block layer requires bios/requests to be a multiple of
855	* 512 bytes, but Windows can send us vecs that are misaligned.
856	* This can result in bios and later requests with misaligned
857	* sizes if we have to break up a cmd/scatterlist into multiple
858	* bios.
859	*
860	* We currently only break up a command into multiple bios if
861	* we hit the vec/seg limit, so check if our sgl_count is
862	* greater than the max and if a vec in the cmd has a
863	* misaligned offset/size.
864	*/
865	if (!is_prot &&
866	(offset & (SECTOR_SIZE - 1) \|\| n & (SECTOR_SIZE - 1)) &&
867	cmd->tvc_sgl_count > BIO_MAX_VECS) {
868	WARN_ONCE(true,
869	"vhost-scsi detected misaligned IO. Performance may be degraded.");
870	goto revert_iter_get_pages;
871	}
872
873	sg_set_page(sg, pages[npages++], n, offset);
874	sg = sg_next(sg);
875	bytes -= n;
876	offset = 0;
877	}
878
879	*sgl = sg;
880	return npages;
881
882	revert_iter_get_pages:
883	vhost_scsi_revert_map_iov_to_sgl(iter, *sgl, sg);
884
885	iov_iter_revert(iter, bytes);
886	while (bytes) {
887	n = min_t(unsigned int, PAGE_SIZE, bytes);
888
889	put_page(pages[npages++]);
890	bytes -= n;
891	}
892
893	return -EINVAL;
894	}
895
896	static int
897	vhost_scsi_calc_sgls(struct iov_iter *iter, size_t bytes, int max_sgls)
898	{
899	int sgl_count = 0;
900
901	if (!iter \|\| !iter_iov(iter)) {
902	pr_err("%s: iter->iov is NULL, but expected bytes: %zu"
903	" present\n", __func__, bytes);
904	return -EINVAL;
905	}
906
907	sgl_count = iov_iter_npages(iter, 0xffff);
908	if (sgl_count > max_sgls) {
909	pr_err("%s: requested sgl_count: %d exceeds pre-allocated"
910	" max_sgls: %d\n", __func__, sgl_count, max_sgls);
911	return -EINVAL;
912	}
913	return sgl_count;
914	}
915
916	static int
917	vhost_scsi_copy_iov_to_sgl(struct vhost_scsi_cmd cmd, struct iov_iter iter,
918	struct sg_table *sg_table, int sg_count,
919	int data_dir)
920	{
921	size_t len = iov_iter_count(iter);
922	unsigned int nbytes = 0;
923	struct scatterlist *sg;
924	struct page *page;
925	int i, ret;
926
927	if (data_dir == DMA_FROM_DEVICE) {
928	cmd->read_iter = kzalloc_obj(*cmd->read_iter);
929	if (!cmd->read_iter)
930	return -ENOMEM;
931
932	cmd->read_iov = dup_iter(cmd->read_iter, iter, GFP_KERNEL);
933	if (!cmd->read_iov) {
934	ret = -ENOMEM;
935	goto free_iter;
936	}
937	}
938
939	for_each_sgtable_sg(sg_table, sg, i) {
940	page = alloc_page(GFP_KERNEL);
941	if (!page) {
942	ret = -ENOMEM;
943	goto err;
944	}
945
946	nbytes = min_t(unsigned int, PAGE_SIZE, len);
947	sg_set_page(sg, page, nbytes, 0);
948
949	if (data_dir == DMA_TO_DEVICE &&
950	copy_page_from_iter(page, 0, nbytes, iter) != nbytes) {
951	ret = -EFAULT;
952	goto err;
953	}
954
955	len -= nbytes;
956	}
957
958	cmd->copied_iov = 1;
959	return 0;
960
961	err:
962	pr_err("Could not read %u bytes while handling misaligned cmd\n",
963	nbytes);
964
965	for_each_sgtable_sg(sg_table, sg, i) {
966	page = sg_page(sg);
967	if (page)
968	__free_page(page);
969	}
970	kfree(cmd->read_iov);
971	free_iter:
972	kfree(cmd->read_iter);
973	return ret;
974	}
975
976	static int
977	vhost_scsi_map_iov_to_sgl(struct vhost_scsi_cmd cmd, struct iov_iter iter,
978	struct sg_table *sg_table, int sg_count, bool is_prot)
979	{
980	struct scatterlist *sg = sg_table->sgl;
981	int ret;
982
983	while (iov_iter_count(iter)) {
984	ret = vhost_scsi_map_to_sgl(cmd, iter, sg_table, &sg, is_prot);
985	if (ret < 0) {
986	vhost_scsi_revert_map_iov_to_sgl(iter, sg_table->sgl,
987	sg);
988	return ret;
989	}
990	}
991
992	return 0;
993	}
994
995	static int
996	vhost_scsi_mapal(struct vhost_scsi vs, struct vhost_scsi_cmd cmd,
997	size_t prot_bytes, struct iov_iter *prot_iter,
998	size_t data_bytes, struct iov_iter *data_iter, int data_dir)
999	{
1000	int sgl_count, ret;
1001
1002	if (prot_bytes) {
1003	sgl_count = vhost_scsi_calc_sgls(prot_iter, prot_bytes,
1004	VHOST_SCSI_PREALLOC_PROT_SGLS);
1005	if (sgl_count < 0)
1006	return sgl_count;
1007
1008	cmd->prot_table.sgl = cmd->prot_sgl;
1009	ret = sg_alloc_table_chained(&cmd->prot_table, sgl_count,
1010	cmd->prot_table.sgl,
1011	vs->inline_sg_cnt);
1012	if (ret)
1013	return ret;
1014
1015	cmd->tvc_prot_sgl_count = sgl_count;
1016	pr_debug("%s prot_sg %p prot_sgl_count %u\n", __func__,
1017	cmd->prot_table.sgl, cmd->tvc_prot_sgl_count);
1018
1019	ret = vhost_scsi_map_iov_to_sgl(cmd, prot_iter,
1020	&cmd->prot_table,
1021	cmd->tvc_prot_sgl_count, true);
1022	if (ret < 0) {
1023	sg_free_table_chained(&cmd->prot_table,
1024	vs->inline_sg_cnt);
1025	cmd->tvc_prot_sgl_count = 0;
1026	return ret;
1027	}
1028	}
1029	sgl_count = vhost_scsi_calc_sgls(data_iter, data_bytes,
1030	VHOST_SCSI_PREALLOC_SGLS);
1031	if (sgl_count < 0)
1032	return sgl_count;
1033
1034	cmd->table.sgl = cmd->sgl;
1035	ret = sg_alloc_table_chained(&cmd->table, sgl_count, cmd->table.sgl,
1036	vs->inline_sg_cnt);
1037	if (ret)
1038	return ret;
1039
1040	cmd->tvc_sgl_count = sgl_count;
1041	pr_debug("%s data_sg %p data_sgl_count %u\n", __func__,
1042	cmd->table.sgl, cmd->tvc_sgl_count);
1043
1044	ret = vhost_scsi_map_iov_to_sgl(cmd, data_iter, &cmd->table,
1045	cmd->tvc_sgl_count, false);
1046	if (ret == -EINVAL)
1047	ret = vhost_scsi_copy_iov_to_sgl(cmd, data_iter, &cmd->table,
1048	cmd->tvc_sgl_count, data_dir);
1049	if (ret < 0) {
1050	sg_free_table_chained(&cmd->table, vs->inline_sg_cnt);
1051	cmd->tvc_sgl_count = 0;
1052	return ret;
1053	}
1054	return 0;
1055	}
1056
1057	static int vhost_scsi_to_tcm_attr(int attr)
1058	{
1059	switch (attr) {
1060	case VIRTIO_SCSI_S_SIMPLE:
1061	return TCM_SIMPLE_TAG;
1062	case VIRTIO_SCSI_S_ORDERED:
1063	return TCM_ORDERED_TAG;
1064	case VIRTIO_SCSI_S_HEAD:
1065	return TCM_HEAD_TAG;
1066	case VIRTIO_SCSI_S_ACA:
1067	return TCM_ACA_TAG;
1068	default:
1069	break;
1070	}
1071	return TCM_SIMPLE_TAG;
1072	}
1073
1074	static void vhost_scsi_target_queue_cmd(struct vhost_scsi_nexus *nexus,
1075	struct vhost_scsi_cmd *cmd,
1076	unsigned char *cdb, u16 lun,
1077	int task_attr, int data_dir,
1078	u32 exp_data_len)
1079	{
1080	struct se_cmd *se_cmd = &cmd->tvc_se_cmd;
1081	struct scatterlist sg_ptr, sg_prot_ptr = NULL;
1082
1083	/* FIXME: BIDI operation */
1084	if (cmd->tvc_sgl_count) {
1085	sg_ptr = cmd->table.sgl;
1086
1087	if (cmd->tvc_prot_sgl_count)
1088	sg_prot_ptr = cmd->prot_table.sgl;
1089	else
1090	se_cmd->prot_pto = true;
1091	} else {
1092	sg_ptr = NULL;
1093	}
1094
1095	se_cmd->tag = 0;
1096	target_init_cmd(se_cmd, nexus->tvn_se_sess, &cmd->tvc_sense_buf[0],
1097	lun, exp_data_len, vhost_scsi_to_tcm_attr(task_attr),
1098	data_dir, TARGET_SCF_ACK_KREF);
1099
1100	if (target_submit_prep(se_cmd, cdb, sg_ptr,
1101	cmd->tvc_sgl_count, NULL, 0, sg_prot_ptr,
1102	cmd->tvc_prot_sgl_count, GFP_KERNEL))
1103	return;
1104
1105	target_submit(se_cmd);
1106	}
1107
1108	static void
1109	vhost_scsi_send_status(struct vhost_scsi vs, struct vhost_virtqueue vq,
1110	struct vhost_scsi_ctx *vc, u8 status)
1111	{
1112	struct virtio_scsi_cmd_resp rsp;
1113	struct iov_iter iov_iter;
1114	int ret;
1115
1116	memset(&rsp, 0, sizeof(rsp));
1117	rsp.status = status;
1118
1119	iov_iter_init(&iov_iter, ITER_DEST, &vq->iov[vc->out], vc->in,
1120	sizeof(rsp));
1121
1122	ret = copy_to_iter(&rsp, sizeof(rsp), &iov_iter);
1123
1124	if (likely(ret == sizeof(rsp)))
1125	vhost_add_used_and_signal(&vs->dev, vq, vc->head, 0);
1126	else
1127	pr_err("Faulted on virtio_scsi_cmd_resp\n");
1128	}
1129
1130	#define TYPE_IO_CMD 0
1131	#define TYPE_CTRL_TMF 1
1132	#define TYPE_CTRL_AN 2
1133
1134	static void
1135	vhost_scsi_send_bad_target(struct vhost_scsi *vs,
1136	struct vhost_virtqueue *vq,
1137	struct vhost_scsi_ctx *vc, int type)
1138	{
1139	union {
1140	struct virtio_scsi_cmd_resp cmd;
1141	struct virtio_scsi_ctrl_tmf_resp tmf;
1142	struct virtio_scsi_ctrl_an_resp an;
1143	} rsp;
1144	struct iov_iter iov_iter;
1145	size_t rsp_size;
1146	int ret;
1147
1148	memset(&rsp, 0, sizeof(rsp));
1149
1150	if (type == TYPE_IO_CMD) {
1151	rsp_size = sizeof(struct virtio_scsi_cmd_resp);
1152	rsp.cmd.response = VIRTIO_SCSI_S_BAD_TARGET;
1153	} else if (type == TYPE_CTRL_TMF) {
1154	rsp_size = sizeof(struct virtio_scsi_ctrl_tmf_resp);
1155	rsp.tmf.response = VIRTIO_SCSI_S_BAD_TARGET;
1156	} else {
1157	rsp_size = sizeof(struct virtio_scsi_ctrl_an_resp);
1158	rsp.an.response = VIRTIO_SCSI_S_BAD_TARGET;
1159	}
1160
1161	iov_iter_init(&iov_iter, ITER_DEST, &vq->iov[vc->out], vc->in,
1162	rsp_size);
1163
1164	ret = copy_to_iter(&rsp, rsp_size, &iov_iter);
1165
1166	if (likely(ret == rsp_size))
1167	vhost_add_used_and_signal(&vs->dev, vq, vc->head, 0);
1168	else
1169	pr_err("Faulted on virtio scsi type=%d\n", type);
1170	}
1171
1172	static int
1173	vhost_scsi_get_desc(struct vhost_scsi vs, struct vhost_virtqueue vq,
1174	struct vhost_scsi_ctx *vc,
1175	struct vhost_log log, unsigned int log_num)
1176	{
1177	int ret = -ENXIO;
1178
1179	if (likely(log_num))
1180	*log_num = 0;
1181
1182	vc->head = vhost_get_vq_desc(vq, vq->iov,
1183	ARRAY_SIZE(vq->iov), &vc->out, &vc->in,
1184	log, log_num);
1185
1186	pr_debug("vhost_get_vq_desc: head: %d, out: %u in: %u\n",
1187	vc->head, vc->out, vc->in);
1188
1189	/* On error, stop handling until the next kick. */
1190	if (unlikely(vc->head < 0))
1191	goto done;
1192
1193	/* Nothing new? Wait for eventfd to tell us they refilled. */
1194	if (vc->head == vq->num) {
1195	if (unlikely(vhost_enable_notify(&vs->dev, vq))) {
1196	vhost_disable_notify(&vs->dev, vq);
1197	ret = -EAGAIN;
1198	}
1199	goto done;
1200	}
1201
1202	/*
1203	* Get the size of request and response buffers.
1204	* FIXME: Not correct for BIDI operation
1205	*/
1206	vc->out_size = iov_length(vq->iov, vc->out);
1207	vc->in_size = iov_length(&vq->iov[vc->out], vc->in);
1208
1209	/*
1210	* Copy over the virtio-scsi request header, which for a
1211	* ANY_LAYOUT enabled guest may span multiple iovecs, or a
1212	* single iovec may contain both the header + outgoing
1213	* WRITE payloads.
1214	*
1215	* copy_from_iter() will advance out_iter, so that it will
1216	* point at the start of the outgoing WRITE payload, if
1217	* DMA_TO_DEVICE is set.
1218	*/
1219	iov_iter_init(&vc->out_iter, ITER_SOURCE, vq->iov, vc->out, vc->out_size);
1220	ret = 0;
1221
1222	done:
1223	return ret;
1224	}
1225
1226	static int
1227	vhost_scsi_chk_size(struct vhost_virtqueue vq, struct vhost_scsi_ctx vc)
1228	{
1229	if (unlikely(vc->in_size < vc->rsp_size)) {
1230	vq_err(vq,
1231	"Response buf too small, need min %zu bytes got %zu",
1232	vc->rsp_size, vc->in_size);
1233	return -EINVAL;
1234	} else if (unlikely(vc->out_size < vc->req_size)) {
1235	vq_err(vq,
1236	"Request buf too small, need min %zu bytes got %zu",
1237	vc->req_size, vc->out_size);
1238	return -EIO;
1239	}
1240
1241	return 0;
1242	}
1243
1244	static int
1245	vhost_scsi_get_req(struct vhost_virtqueue vq, struct vhost_scsi_ctx vc,
1246	struct vhost_scsi_tpg **tpgp)
1247	{
1248	int ret = -EIO;
1249
1250	if (unlikely(!copy_from_iter_full(vc->req, vc->req_size,
1251	&vc->out_iter))) {
1252	vq_err(vq, "Faulted on copy_from_iter_full\n");
1253	} else if (unlikely(*vc->lunp != 1)) {
1254	/* virtio-scsi spec requires byte 0 of the lun to be 1 */
1255	vq_err(vq, "Illegal virtio-scsi lun: %u\n", *vc->lunp);
1256	} else {
1257	struct vhost_scsi_tpg *vs_tpg, tpg = NULL;
1258
1259	if (vc->target) {
1260	/* validated at handler entry */
1261	vs_tpg = vhost_vq_get_backend(vq);
1262	tpg = READ_ONCE(vs_tpg[*vc->target]);
1263	if (unlikely(!tpg))
1264	goto out;
1265	}
1266
1267	if (tpgp)
1268	*tpgp = tpg;
1269	ret = 0;
1270	}
1271	out:
1272	return ret;
1273	}
1274
1275	static int
1276	vhost_scsi_setup_resp_iovs(struct vhost_scsi_cmd cmd, struct iovec in_iovs,
1277	unsigned int in_iovs_cnt)
1278	{
1279	int i, cnt;
1280
1281	if (!in_iovs_cnt)
1282	return 0;
1283	/*
1284	* Initiators normally just put the virtio_scsi_cmd_resp in the first
1285	* iov, but just in case they wedged in some data with it we check for
1286	* greater than or equal to the response struct.
1287	*/
1288	if (in_iovs[0].iov_len >= sizeof(struct virtio_scsi_cmd_resp)) {
1289	cmd->tvc_resp_iovs = &cmd->tvc_resp_iov;
1290	cmd->tvc_resp_iovs_cnt = 1;
1291	} else {
1292	/*
1293	* Legacy descriptor layouts didn't specify that we must put
1294	* the entire response in one iov. Worst case we have a
1295	* iov per byte.
1296	*/
1297	cnt = min(VHOST_SCSI_MAX_RESP_IOVS, in_iovs_cnt);
1298	cmd->tvc_resp_iovs = kzalloc_objs(struct iovec, cnt);
1299	if (!cmd->tvc_resp_iovs)
1300	return -ENOMEM;
1301
1302	cmd->tvc_resp_iovs_cnt = cnt;
1303	}
1304
1305	for (i = 0; i < cmd->tvc_resp_iovs_cnt; i++)
1306	cmd->tvc_resp_iovs[i] = in_iovs[i];
1307
1308	return 0;
1309	}
1310
1311	static u16 vhost_buf_to_lun(u8 *lun_buf)
1312	{
1313	return ((lun_buf[2] << 8) \| lun_buf[3]) & 0x3FFF;
1314	}
1315
1316	static void
1317	vhost_scsi_handle_vq(struct vhost_scsi vs, struct vhost_virtqueue vq)
1318	{
1319	struct vhost_scsi_tpg *vs_tpg, tpg;
1320	struct virtio_scsi_cmd_req v_req;
1321	struct virtio_scsi_cmd_req_pi v_req_pi;
1322	struct vhost_scsi_nexus *nexus;
1323	struct vhost_scsi_ctx vc;
1324	struct vhost_scsi_cmd *cmd;
1325	struct iov_iter in_iter, prot_iter, data_iter;
1326	u64 tag;
1327	u32 exp_data_len, data_direction;
1328	int ret, prot_bytes, c = 0;
1329	u16 lun;
1330	u8 task_attr;
1331	bool t10_pi = vhost_has_feature(vq, VIRTIO_SCSI_F_T10_PI);
1332	u8 *cdb;
1333	struct vhost_log *vq_log;
1334	unsigned int log_num;
1335
1336	mutex_lock(&vq->mutex);
1337	/*
1338	* We can handle the vq only after the endpoint is setup by calling the
1339	* VHOST_SCSI_SET_ENDPOINT ioctl.
1340	*/
1341	vs_tpg = vhost_vq_get_backend(vq);
1342	if (!vs_tpg)
1343	goto out;
1344
1345	memset(&vc, 0, sizeof(vc));
1346	vc.rsp_size = sizeof(struct virtio_scsi_cmd_resp);
1347
1348	vhost_disable_notify(&vs->dev, vq);
1349
1350	vq_log = unlikely(vhost_has_feature(vq, VHOST_F_LOG_ALL)) ?
1351	vq->log : NULL;
1352
1353	do {
1354	ret = vhost_scsi_get_desc(vs, vq, &vc, vq_log, &log_num);
1355	if (ret)
1356	goto err;
1357
1358	/*
1359	* Setup pointers and values based upon different virtio-scsi
1360	* request header if T10_PI is enabled in KVM guest.
1361	*/
1362	if (t10_pi) {
1363	vc.req = &v_req_pi;
1364	vc.req_size = sizeof(v_req_pi);
1365	vc.lunp = &v_req_pi.lun[0];
1366	vc.target = &v_req_pi.lun[1];
1367	} else {
1368	vc.req = &v_req;
1369	vc.req_size = sizeof(v_req);
1370	vc.lunp = &v_req.lun[0];
1371	vc.target = &v_req.lun[1];
1372	}
1373
1374	/*
1375	* Validate the size of request and response buffers.
1376	* Check for a sane response buffer so we can report
1377	* early errors back to the guest.
1378	*/
1379	ret = vhost_scsi_chk_size(vq, &vc);
1380	if (ret)
1381	goto err;
1382
1383	ret = vhost_scsi_get_req(vq, &vc, &tpg);
1384	if (ret)
1385	goto err;
1386
1387	ret = -EIO; /* bad target on any error from here on */
1388
1389	/*
1390	* Determine data_direction by calculating the total outgoing
1391	* iovec sizes + incoming iovec sizes vs. virtio-scsi request +
1392	* response headers respectively.
1393	*
1394	* For DMA_TO_DEVICE this is out_iter, which is already pointing
1395	* to the right place.
1396	*
1397	* For DMA_FROM_DEVICE, the iovec will be just past the end
1398	* of the virtio-scsi response header in either the same
1399	* or immediately following iovec.
1400	*
1401	* Any associated T10_PI bytes for the outgoing / incoming
1402	* payloads are included in calculation of exp_data_len here.
1403	*/
1404	prot_bytes = 0;
1405
1406	if (vc.out_size > vc.req_size) {
1407	data_direction = DMA_TO_DEVICE;
1408	exp_data_len = vc.out_size - vc.req_size;
1409	data_iter = vc.out_iter;
1410	} else if (vc.in_size > vc.rsp_size) {
1411	data_direction = DMA_FROM_DEVICE;
1412	exp_data_len = vc.in_size - vc.rsp_size;
1413
1414	iov_iter_init(&in_iter, ITER_DEST, &vq->iov[vc.out], vc.in,
1415	vc.rsp_size + exp_data_len);
1416	iov_iter_advance(&in_iter, vc.rsp_size);
1417	data_iter = in_iter;
1418	} else {
1419	data_direction = DMA_NONE;
1420	exp_data_len = 0;
1421	}
1422	/*
1423	* If T10_PI header + payload is present, setup prot_iter values
1424	* and recalculate data_iter for vhost_scsi_mapal() mapping to
1425	* host scatterlists via get_user_pages_fast().
1426	*/
1427	if (t10_pi) {
1428	if (v_req_pi.pi_bytesout) {
1429	if (data_direction != DMA_TO_DEVICE) {
1430	vq_err(vq, "Received non zero pi_bytesout,"
1431	" but wrong data_direction\n");
1432	goto err;
1433	}
1434	prot_bytes = vhost32_to_cpu(vq, v_req_pi.pi_bytesout);
1435	} else if (v_req_pi.pi_bytesin) {
1436	if (data_direction != DMA_FROM_DEVICE) {
1437	vq_err(vq, "Received non zero pi_bytesin,"
1438	" but wrong data_direction\n");
1439	goto err;
1440	}
1441	prot_bytes = vhost32_to_cpu(vq, v_req_pi.pi_bytesin);
1442	}
1443	/*
1444	* Set prot_iter to data_iter and truncate it to
1445	* prot_bytes, and advance data_iter past any
1446	* preceding prot_bytes that may be present.
1447	*
1448	* Also fix up the exp_data_len to reflect only the
1449	* actual data payload length.
1450	*/
1451	if (prot_bytes) {
1452	if (prot_bytes >= exp_data_len) {
1453	vq_err(vq, "Protection data exceeds payload length\n");
1454	goto err;
1455	}
1456
1457	exp_data_len -= prot_bytes;
1458	prot_iter = data_iter;
1459	iov_iter_truncate(&prot_iter, prot_bytes);
1460	iov_iter_advance(&data_iter, prot_bytes);
1461	}
1462	tag = vhost64_to_cpu(vq, v_req_pi.tag);
1463	task_attr = v_req_pi.task_attr;
1464	cdb = &v_req_pi.cdb[0];
1465	lun = vhost_buf_to_lun(v_req_pi.lun);
1466	} else {
1467	tag = vhost64_to_cpu(vq, v_req.tag);
1468	task_attr = v_req.task_attr;
1469	cdb = &v_req.cdb[0];
1470	lun = vhost_buf_to_lun(v_req.lun);
1471	}
1472	/*
1473	* Check that the received CDB size does not exceeded our
1474	* hardcoded max for vhost-scsi, then get a pre-allocated
1475	* cmd descriptor for the new virtio-scsi tag.
1476	*
1477	* TODO what if cdb was too small for varlen cdb header?
1478	*/
1479	if (unlikely(scsi_command_size(cdb) > VHOST_SCSI_MAX_CDB_SIZE)) {
1480	vq_err(vq, "Received SCSI CDB with command_size: %d that"
1481	" exceeds SCSI_MAX_VARLEN_CDB_SIZE: %d\n",
1482	scsi_command_size(cdb), VHOST_SCSI_MAX_CDB_SIZE);
1483	goto err;
1484	}
1485
1486	nexus = tpg->tpg_nexus;
1487	if (!nexus) {
1488	vq_err(vq, "Unable to locate active struct vhost_scsi_nexus\n");
1489	ret = -EIO;
1490	goto err;
1491	}
1492
1493	cmd = vhost_scsi_get_cmd(vq, tag);
1494	if (IS_ERR(cmd)) {
1495	ret = PTR_ERR(cmd);
1496	vq_err(vq, "vhost_scsi_get_tag failed %d\n", ret);
1497	goto err;
1498	}
1499	cmd->tvc_vq = vq;
1500
1501	ret = vhost_scsi_setup_resp_iovs(cmd, &vq->iov[vc.out], vc.in);
1502	if (ret) {
1503	vq_err(vq, "Failed to alloc recv iovs\n");
1504	vhost_scsi_release_cmd_res(&cmd->tvc_se_cmd);
1505	goto err;
1506	}
1507
1508	if (unlikely(vq_log && log_num)) {
1509	ret = vhost_scsi_copy_cmd_log(vq, cmd, vq_log, log_num);
1510	if (unlikely(ret)) {
1511	vhost_scsi_release_cmd_res(&cmd->tvc_se_cmd);
1512	goto err;
1513	}
1514	}
1515
1516	pr_debug("vhost_scsi got command opcode: %#02x, lun: %d\n",
1517	cdb[0], lun);
1518	pr_debug("cmd: %p exp_data_len: %d, prot_bytes: %d data_direction:"
1519	" %d\n", cmd, exp_data_len, prot_bytes, data_direction);
1520
1521	if (data_direction != DMA_NONE) {
1522	ret = vhost_scsi_mapal(vs, cmd, prot_bytes, &prot_iter,
1523	exp_data_len, &data_iter,
1524	data_direction);
1525	if (unlikely(ret)) {
1526	vq_err(vq, "Failed to map iov to sgl\n");
1527	vhost_scsi_release_cmd_res(&cmd->tvc_se_cmd);
1528	goto err;
1529	}
1530	}
1531	/*
1532	* Save the descriptor from vhost_get_vq_desc() to be used to
1533	* complete the virtio-scsi request in TCM callback context via
1534	* vhost_scsi_queue_data_in() and vhost_scsi_queue_status()
1535	*/
1536	cmd->tvc_vq_desc = vc.head;
1537	vhost_scsi_target_queue_cmd(nexus, cmd, cdb, lun, task_attr,
1538	data_direction,
1539	exp_data_len + prot_bytes);
1540	ret = 0;
1541	err:
1542	/*
1543	* ENXIO: No more requests, or read error, wait for next kick
1544	* EINVAL: Invalid response buffer, drop the request
1545	* EIO: Respond with bad target
1546	* EAGAIN: Pending request
1547	* ENOMEM: Could not allocate resources for request
1548	*/
1549	if (ret == -ENXIO)
1550	break;
1551	else if (ret == -EIO) {
1552	vhost_scsi_send_bad_target(vs, vq, &vc, TYPE_IO_CMD);
1553	vhost_scsi_log_write(vq, vq_log, log_num);
1554	} else if (ret == -ENOMEM) {
1555	vhost_scsi_send_status(vs, vq, &vc,
1556	SAM_STAT_TASK_SET_FULL);
1557	vhost_scsi_log_write(vq, vq_log, log_num);
1558	}
1559	} while (likely(!vhost_exceeds_weight(vq, ++c, 0)));
1560	out:
1561	mutex_unlock(&vq->mutex);
1562	}
1563
1564	static void
1565	vhost_scsi_send_tmf_resp(struct vhost_scsi vs, struct vhost_virtqueue vq,
1566	int in_iovs, int vq_desc, struct iovec *resp_iov,
1567	int tmf_resp_code)
1568	{
1569	struct virtio_scsi_ctrl_tmf_resp rsp;
1570	struct iov_iter iov_iter;
1571	int ret;
1572
1573	pr_debug("%s\n", __func__);
1574	memset(&rsp, 0, sizeof(rsp));
1575	rsp.response = tmf_resp_code;
1576
1577	iov_iter_init(&iov_iter, ITER_DEST, resp_iov, in_iovs, sizeof(rsp));
1578
1579	ret = copy_to_iter(&rsp, sizeof(rsp), &iov_iter);
1580	if (likely(ret == sizeof(rsp)))
1581	vhost_add_used_and_signal(&vs->dev, vq, vq_desc, 0);
1582	else
1583	pr_err("Faulted on virtio_scsi_ctrl_tmf_resp\n");
1584	}
1585
1586	static void vhost_scsi_tmf_resp_work(struct vhost_work *work)
1587	{
1588	struct vhost_scsi_tmf *tmf = container_of(work, struct vhost_scsi_tmf,
1589	vwork);
1590	int resp_code;
1591
1592	if (tmf->scsi_resp == TMR_FUNCTION_COMPLETE)
1593	resp_code = VIRTIO_SCSI_S_FUNCTION_SUCCEEDED;
1594	else
1595	resp_code = VIRTIO_SCSI_S_FUNCTION_REJECTED;
1596
1597	mutex_lock(&tmf->svq->vq.mutex);
1598	vhost_scsi_send_tmf_resp(tmf->vhost, &tmf->svq->vq, tmf->in_iovs,
1599	tmf->vq_desc, &tmf->resp_iov, resp_code);
1600	vhost_scsi_log_write(&tmf->svq->vq, tmf->tmf_log,
1601	tmf->tmf_log_num);
1602	mutex_unlock(&tmf->svq->vq.mutex);
1603
1604	vhost_scsi_release_tmf_res(tmf);
1605	}
1606
1607	static void vhost_scsi_tmf_flush_work(struct work_struct *work)
1608	{
1609	struct vhost_scsi_tmf *tmf = container_of(work, struct vhost_scsi_tmf,
1610	flush_work);
1611	struct vhost_virtqueue *vq = &tmf->svq->vq;
1612	/*
1613	* Make sure we have sent responses for other commands before we
1614	* send our response.
1615	*/
1616	vhost_dev_flush(vq->dev);
1617	if (!vhost_vq_work_queue(vq, &tmf->vwork))
1618	vhost_scsi_release_tmf_res(tmf);
1619	}
1620
1621	static void
1622	vhost_scsi_handle_tmf(struct vhost_scsi vs, struct vhost_scsi_tpg tpg,
1623	struct vhost_virtqueue *vq,
1624	struct virtio_scsi_ctrl_tmf_req *vtmf,
1625	struct vhost_scsi_ctx *vc,
1626	struct vhost_log *log, unsigned int log_num)
1627	{
1628	struct vhost_scsi_virtqueue *svq = container_of(vq,
1629	struct vhost_scsi_virtqueue, vq);
1630	struct vhost_scsi_tmf *tmf;
1631
1632	if (vhost32_to_cpu(vq, vtmf->subtype) !=
1633	VIRTIO_SCSI_T_TMF_LOGICAL_UNIT_RESET)
1634	goto send_reject;
1635
1636	if (!tpg->tpg_nexus \|\| !tpg->tpg_nexus->tvn_se_sess) {
1637	pr_err("Unable to locate active struct vhost_scsi_nexus for LUN RESET.\n");
1638	goto send_reject;
1639	}
1640
1641	tmf = kzalloc_obj(*tmf);
1642	if (!tmf)
1643	goto send_reject;
1644
1645	INIT_WORK(&tmf->flush_work, vhost_scsi_tmf_flush_work);
1646	vhost_work_init(&tmf->vwork, vhost_scsi_tmf_resp_work);
1647	tmf->vhost = vs;
1648	tmf->svq = svq;
1649	tmf->resp_iov = vq->iov[vc->out];
1650	tmf->vq_desc = vc->head;
1651	tmf->in_iovs = vc->in;
1652	tmf->inflight = vhost_scsi_get_inflight(vq);
1653
1654	if (unlikely(log && log_num)) {
1655	tmf->tmf_log = kmalloc_objs(*tmf->tmf_log, log_num);
1656	if (tmf->tmf_log) {
1657	memcpy(tmf->tmf_log, log, sizeof(tmf->tmf_log) log_num);
1658	tmf->tmf_log_num = log_num;
1659	} else {
1660	pr_err("vhost_scsi tmf log allocation error\n");
1661	vhost_scsi_release_tmf_res(tmf);
1662	goto send_reject;
1663	}
1664	}
1665
1666	if (target_submit_tmr(&tmf->se_cmd, tpg->tpg_nexus->tvn_se_sess, NULL,
1667	vhost_buf_to_lun(vtmf->lun), NULL,
1668	TMR_LUN_RESET, GFP_KERNEL, 0,
1669	TARGET_SCF_ACK_KREF) < 0) {
1670	vhost_scsi_release_tmf_res(tmf);
1671	goto send_reject;
1672	}
1673
1674	return;
1675
1676	send_reject:
1677	vhost_scsi_send_tmf_resp(vs, vq, vc->in, vc->head, &vq->iov[vc->out],
1678	VIRTIO_SCSI_S_FUNCTION_REJECTED);
1679	vhost_scsi_log_write(vq, log, log_num);
1680	}
1681
1682	static void
1683	vhost_scsi_send_an_resp(struct vhost_scsi *vs,
1684	struct vhost_virtqueue *vq,
1685	struct vhost_scsi_ctx *vc)
1686	{
1687	struct virtio_scsi_ctrl_an_resp rsp;
1688	struct iov_iter iov_iter;
1689	int ret;
1690
1691	pr_debug("%s\n", __func__);
1692	memset(&rsp, 0, sizeof(rsp)); /* event_actual = 0 */
1693	rsp.response = VIRTIO_SCSI_S_OK;
1694
1695	iov_iter_init(&iov_iter, ITER_DEST, &vq->iov[vc->out], vc->in, sizeof(rsp));
1696
1697	ret = copy_to_iter(&rsp, sizeof(rsp), &iov_iter);
1698	if (likely(ret == sizeof(rsp)))
1699	vhost_add_used_and_signal(&vs->dev, vq, vc->head, 0);
1700	else
1701	pr_err("Faulted on virtio_scsi_ctrl_an_resp\n");
1702	}
1703
1704	static void
1705	vhost_scsi_ctl_handle_vq(struct vhost_scsi vs, struct vhost_virtqueue vq)
1706	{
1707	struct vhost_scsi_tpg *tpg;
1708	union {
1709	__virtio32 type;
1710	struct virtio_scsi_ctrl_an_req an;
1711	struct virtio_scsi_ctrl_tmf_req tmf;
1712	} v_req;
1713	struct vhost_scsi_ctx vc;
1714	size_t typ_size;
1715	int ret, c = 0;
1716	struct vhost_log *vq_log;
1717	unsigned int log_num;
1718
1719	mutex_lock(&vq->mutex);
1720	/*
1721	* We can handle the vq only after the endpoint is setup by calling the
1722	* VHOST_SCSI_SET_ENDPOINT ioctl.
1723	*/
1724	if (!vhost_vq_get_backend(vq))
1725	goto out;
1726
1727	memset(&vc, 0, sizeof(vc));
1728
1729	vhost_disable_notify(&vs->dev, vq);
1730
1731	vq_log = unlikely(vhost_has_feature(vq, VHOST_F_LOG_ALL)) ?
1732	vq->log : NULL;
1733
1734	do {
1735	ret = vhost_scsi_get_desc(vs, vq, &vc, vq_log, &log_num);
1736	if (ret)
1737	goto err;
1738
1739	/*
1740	* Get the request type first in order to setup
1741	* other parameters dependent on the type.
1742	*/
1743	vc.req = &v_req.type;
1744	typ_size = sizeof(v_req.type);
1745
1746	if (unlikely(!copy_from_iter_full(vc.req, typ_size,
1747	&vc.out_iter))) {
1748	vq_err(vq, "Faulted on copy_from_iter tmf type\n");
1749	/*
1750	* The size of the response buffer depends on the
1751	* request type and must be validated against it.
1752	* Since the request type is not known, don't send
1753	* a response.
1754	*/
1755	continue;
1756	}
1757
1758	switch (vhost32_to_cpu(vq, v_req.type)) {
1759	case VIRTIO_SCSI_T_TMF:
1760	vc.req = &v_req.tmf;
1761	vc.req_size = sizeof(struct virtio_scsi_ctrl_tmf_req);
1762	vc.rsp_size = sizeof(struct virtio_scsi_ctrl_tmf_resp);
1763	vc.lunp = &v_req.tmf.lun[0];
1764	vc.target = &v_req.tmf.lun[1];
1765	break;
1766	case VIRTIO_SCSI_T_AN_QUERY:
1767	case VIRTIO_SCSI_T_AN_SUBSCRIBE:
1768	vc.req = &v_req.an;
1769	vc.req_size = sizeof(struct virtio_scsi_ctrl_an_req);
1770	vc.rsp_size = sizeof(struct virtio_scsi_ctrl_an_resp);
1771	vc.lunp = &v_req.an.lun[0];
1772	vc.target = NULL;
1773	break;
1774	default:
1775	vq_err(vq, "Unknown control request %d", v_req.type);
1776	continue;
1777	}
1778
1779	/*
1780	* Validate the size of request and response buffers.
1781	* Check for a sane response buffer so we can report
1782	* early errors back to the guest.
1783	*/
1784	ret = vhost_scsi_chk_size(vq, &vc);
1785	if (ret)
1786	goto err;
1787
1788	/*
1789	* Get the rest of the request now that its size is known.
1790	*/
1791	vc.req += typ_size;
1792	vc.req_size -= typ_size;
1793
1794	ret = vhost_scsi_get_req(vq, &vc, &tpg);
1795	if (ret)
1796	goto err;
1797
1798	if (v_req.type == VIRTIO_SCSI_T_TMF)
1799	vhost_scsi_handle_tmf(vs, tpg, vq, &v_req.tmf, &vc,
1800	vq_log, log_num);
1801	else {
1802	vhost_scsi_send_an_resp(vs, vq, &vc);
1803	vhost_scsi_log_write(vq, vq_log, log_num);
1804	}
1805	err:
1806	/*
1807	* ENXIO: No more requests, or read error, wait for next kick
1808	* EINVAL: Invalid response buffer, drop the request
1809	* EIO: Respond with bad target
1810	* EAGAIN: Pending request
1811	*/
1812	if (ret == -ENXIO)
1813	break;
1814	else if (ret == -EIO) {
1815	vhost_scsi_send_bad_target(vs, vq, &vc,
1816	v_req.type == VIRTIO_SCSI_T_TMF ?
1817	TYPE_CTRL_TMF :
1818	TYPE_CTRL_AN);
1819	vhost_scsi_log_write(vq, vq_log, log_num);
1820	}
1821	} while (likely(!vhost_exceeds_weight(vq, ++c, 0)));
1822	out:
1823	mutex_unlock(&vq->mutex);
1824	}
1825
1826	static void vhost_scsi_ctl_handle_kick(struct vhost_work *work)
1827	{
1828	struct vhost_virtqueue *vq = container_of(work, struct vhost_virtqueue,
1829	poll.work);
1830	struct vhost_scsi *vs = container_of(vq->dev, struct vhost_scsi, dev);
1831
1832	pr_debug("%s: The handling func for control queue.\n", __func__);
1833	vhost_scsi_ctl_handle_vq(vs, vq);
1834	}
1835
1836	static void
1837	vhost_scsi_send_evt(struct vhost_scsi vs, struct vhost_virtqueue vq,
1838	struct vhost_scsi_tpg tpg, struct se_lun lun,
1839	u32 event, u32 reason)
1840	{
1841	struct vhost_scsi_evt *evt;
1842
1843	evt = vhost_scsi_allocate_evt(vs, event, reason);
1844	if (!evt)
1845	return;
1846
1847	if (tpg && lun) {
1848	/* TODO: share lun setup code with virtio-scsi.ko */
1849	/*
1850	* Note: evt->event is zeroed when we allocate it and
1851	* lun[4-7] need to be zero according to virtio-scsi spec.
1852	*/
1853	evt->event.lun[0] = 0x01;
1854	evt->event.lun[1] = tpg->tport_tpgt;
1855	if (lun->unpacked_lun >= 256)
1856	evt->event.lun[2] = lun->unpacked_lun >> 8 \| 0x40 ;
1857	evt->event.lun[3] = lun->unpacked_lun & 0xFF;
1858	}
1859
1860	llist_add(&evt->list, &vs->vs_event_list);
1861	if (!vhost_vq_work_queue(vq, &vs->vs_event_work))
1862	vhost_scsi_complete_events(vs, true);
1863	}
1864
1865	static void vhost_scsi_evt_handle_kick(struct vhost_work *work)
1866	{
1867	struct vhost_virtqueue *vq = container_of(work, struct vhost_virtqueue,
1868	poll.work);
1869	struct vhost_scsi *vs = container_of(vq->dev, struct vhost_scsi, dev);
1870
1871	mutex_lock(&vq->mutex);
1872	if (!vhost_vq_get_backend(vq))
1873	goto out;
1874
1875	if (vs->vs_events_missed)
1876	vhost_scsi_send_evt(vs, vq, NULL, NULL, VIRTIO_SCSI_T_NO_EVENT,
1877	0);
1878	out:
1879	mutex_unlock(&vq->mutex);
1880	}
1881
1882	static void vhost_scsi_handle_kick(struct vhost_work *work)
1883	{
1884	struct vhost_virtqueue *vq = container_of(work, struct vhost_virtqueue,
1885	poll.work);
1886	struct vhost_scsi *vs = container_of(vq->dev, struct vhost_scsi, dev);
1887
1888	vhost_scsi_handle_vq(vs, vq);
1889	}
1890
1891	/* Callers must hold dev mutex */
1892	static void vhost_scsi_flush(struct vhost_scsi *vs)
1893	{
1894	int i;
1895
1896	/* Init new inflight and remember the old inflight */
1897	vhost_scsi_init_inflight(vs, vs->old_inflight);
1898
1899	/*
1900	* The inflight->kref was initialized to 1. We decrement it here to
1901	* indicate the start of the flush operation so that it will reach 0
1902	* when all the reqs are finished.
1903	*/
1904	for (i = 0; i < vs->dev.nvqs; i++)
1905	kref_put(&vs->old_inflight[i]->kref, vhost_scsi_done_inflight);
1906
1907	/* Flush both the vhost poll and vhost work */
1908	vhost_dev_flush(&vs->dev);
1909
1910	/* Wait for all reqs issued before the flush to be finished */
1911	for (i = 0; i < vs->dev.nvqs; i++)
1912	wait_for_completion(&vs->old_inflight[i]->comp);
1913	}
1914
1915	static void vhost_scsi_destroy_vq_log(struct vhost_virtqueue *vq)
1916	{
1917	struct vhost_scsi_virtqueue *svq = container_of(vq,
1918	struct vhost_scsi_virtqueue, vq);
1919	struct vhost_scsi_cmd *tv_cmd;
1920	unsigned int i;
1921
1922	if (!svq->scsi_cmds)
1923	return;
1924
1925	for (i = 0; i < svq->max_cmds; i++) {
1926	tv_cmd = &svq->scsi_cmds[i];
1927	kfree(tv_cmd->tvc_log);
1928	tv_cmd->tvc_log = NULL;
1929	tv_cmd->tvc_log_num = 0;
1930	}
1931	}
1932
1933	static void vhost_scsi_destroy_vq_cmds(struct vhost_virtqueue *vq)
1934	{
1935	struct vhost_scsi_virtqueue *svq = container_of(vq,
1936	struct vhost_scsi_virtqueue, vq);
1937	struct vhost_scsi_cmd *tv_cmd;
1938	unsigned int i;
1939
1940	if (!svq->scsi_cmds)
1941	return;
1942
1943	for (i = 0; i < svq->max_cmds; i++) {
1944	tv_cmd = &svq->scsi_cmds[i];
1945
1946	kfree(tv_cmd->sgl);
1947	kfree(tv_cmd->prot_sgl);
1948	}
1949
1950	sbitmap_free(&svq->scsi_tags);
1951	kfree(svq->upages);
1952	vhost_scsi_destroy_vq_log(vq);
1953	kfree(svq->scsi_cmds);
1954	svq->scsi_cmds = NULL;
1955	}
1956
1957	static int vhost_scsi_setup_vq_cmds(struct vhost_virtqueue *vq, int max_cmds)
1958	{
1959	struct vhost_scsi_virtqueue *svq = container_of(vq,
1960	struct vhost_scsi_virtqueue, vq);
1961	struct vhost_scsi *vs = svq->vs;
1962	struct vhost_scsi_cmd *tv_cmd;
1963	unsigned int i;
1964
1965	if (svq->scsi_cmds)
1966	return 0;
1967
1968	if (sbitmap_init_node(&svq->scsi_tags, max_cmds, -1, GFP_KERNEL,
1969	NUMA_NO_NODE, false, true))
1970	return -ENOMEM;
1971	svq->max_cmds = max_cmds;
1972
1973	svq->scsi_cmds = kzalloc_objs(*tv_cmd, max_cmds);
1974	if (!svq->scsi_cmds) {
1975	sbitmap_free(&svq->scsi_tags);
1976	return -ENOMEM;
1977	}
1978
1979	svq->upages = kzalloc_objs(struct page *, VHOST_SCSI_PREALLOC_UPAGES);
1980	if (!svq->upages)
1981	goto out;
1982
1983	for (i = 0; i < max_cmds; i++) {
1984	tv_cmd = &svq->scsi_cmds[i];
1985
1986	if (vs->inline_sg_cnt) {
1987	tv_cmd->sgl = kzalloc_objs(struct scatterlist,
1988	vs->inline_sg_cnt);
1989	if (!tv_cmd->sgl) {
1990	pr_err("Unable to allocate tv_cmd->sgl\n");
1991	goto out;
1992	}
1993	}
1994
1995	if (vhost_has_feature(vq, VIRTIO_SCSI_F_T10_PI) &&
1996	vs->inline_sg_cnt) {
1997	tv_cmd->prot_sgl = kzalloc_objs(struct scatterlist,
1998	vs->inline_sg_cnt);
1999	if (!tv_cmd->prot_sgl) {
2000	pr_err("Unable to allocate tv_cmd->prot_sgl\n");
2001	goto out;
2002	}
2003	}
2004	}
2005	return 0;
2006	out:
2007	vhost_scsi_destroy_vq_cmds(vq);
2008	return -ENOMEM;
2009	}
2010
2011	/*
2012	* Called from vhost_scsi_ioctl() context to walk the list of available
2013	* vhost_scsi_tpg with an active struct vhost_scsi_nexus
2014	*
2015	* The lock nesting rule is:
2016	* vs->dev.mutex -> vhost_scsi_mutex -> tpg->tv_tpg_mutex -> vq->mutex
2017	*/
2018	static int
2019	vhost_scsi_set_endpoint(struct vhost_scsi *vs,
2020	struct vhost_scsi_target *t)
2021	{
2022	struct se_portal_group *se_tpg;
2023	struct vhost_scsi_tport *tv_tport;
2024	struct vhost_scsi_tpg *tpg;
2025	struct vhost_scsi_tpg **vs_tpg;
2026	struct vhost_virtqueue *vq;
2027	int index, ret, i, len;
2028	bool match = false;
2029
2030	mutex_lock(&vs->dev.mutex);
2031
2032	/* Verify that ring has been setup correctly. */
2033	for (index = 0; index < vs->dev.nvqs; ++index) {
2034	/* Verify that ring has been setup correctly. */
2035	if (!vhost_vq_access_ok(&vs->vqs[index].vq)) {
2036	ret = -EFAULT;
2037	goto out;
2038	}
2039	}
2040
2041	if (vs->vs_tpg) {
2042	pr_err("vhost-scsi endpoint already set for %s.\n",
2043	vs->vs_vhost_wwpn);
2044	ret = -EEXIST;
2045	goto out;
2046	}
2047
2048	len = sizeof(vs_tpg[0]) * VHOST_SCSI_MAX_TARGET;
2049	vs_tpg = kzalloc(len, GFP_KERNEL);
2050	if (!vs_tpg) {
2051	ret = -ENOMEM;
2052	goto out;
2053	}
2054
2055	mutex_lock(&vhost_scsi_mutex);
2056	list_for_each_entry(tpg, &vhost_scsi_list, tv_tpg_list) {
2057	mutex_lock(&tpg->tv_tpg_mutex);
2058	if (!tpg->tpg_nexus) {
2059	mutex_unlock(&tpg->tv_tpg_mutex);
2060	continue;
2061	}
2062	if (tpg->tv_tpg_vhost_count != 0) {
2063	mutex_unlock(&tpg->tv_tpg_mutex);
2064	continue;
2065	}
2066	tv_tport = tpg->tport;
2067
2068	if (!strcmp(tv_tport->tport_name, t->vhost_wwpn)) {
2069	/*
2070	* In order to ensure individual vhost-scsi configfs
2071	* groups cannot be removed while in use by vhost ioctl,
2072	* go ahead and take an explicit se_tpg->tpg_group.cg_item
2073	* dependency now.
2074	*/
2075	se_tpg = &tpg->se_tpg;
2076	ret = target_depend_item(&se_tpg->tpg_group.cg_item);
2077	if (ret) {
2078	pr_warn("target_depend_item() failed: %d\n", ret);
2079	mutex_unlock(&tpg->tv_tpg_mutex);
2080	mutex_unlock(&vhost_scsi_mutex);
2081	goto undepend;
2082	}
2083	tpg->tv_tpg_vhost_count++;
2084	tpg->vhost_scsi = vs;
2085	vs_tpg[tpg->tport_tpgt] = tpg;
2086	match = true;
2087	}
2088	mutex_unlock(&tpg->tv_tpg_mutex);
2089	}
2090	mutex_unlock(&vhost_scsi_mutex);
2091
2092	if (match) {
2093	memcpy(vs->vs_vhost_wwpn, t->vhost_wwpn,
2094	sizeof(vs->vs_vhost_wwpn));
2095
2096	for (i = VHOST_SCSI_VQ_IO; i < vs->dev.nvqs; i++) {
2097	vq = &vs->vqs[i].vq;
2098	if (!vhost_vq_is_setup(vq))
2099	continue;
2100
2101	ret = vhost_scsi_setup_vq_cmds(vq, vq->num);
2102	if (ret)
2103	goto destroy_vq_cmds;
2104	}
2105
2106	for (i = 0; i < vs->dev.nvqs; i++) {
2107	vq = &vs->vqs[i].vq;
2108	mutex_lock(&vq->mutex);
2109	vhost_vq_set_backend(vq, vs_tpg);
2110	vhost_vq_init_access(vq);
2111	mutex_unlock(&vq->mutex);
2112	}
2113	ret = 0;
2114	} else {
2115	ret = -ENODEV;
2116	goto free_tpg;
2117	}
2118
2119	/*
2120	* Act as synchronize_rcu to make sure requests after this point
2121	* see a fully setup device.
2122	*/
2123	vhost_scsi_flush(vs);
2124	vs->vs_tpg = vs_tpg;
2125	goto out;
2126
2127	destroy_vq_cmds:
2128	for (i--; i >= VHOST_SCSI_VQ_IO; i--) {
2129	if (!vhost_vq_get_backend(&vs->vqs[i].vq))
2130	vhost_scsi_destroy_vq_cmds(&vs->vqs[i].vq);
2131	}
2132	undepend:
2133	for (i = 0; i < VHOST_SCSI_MAX_TARGET; i++) {
2134	tpg = vs_tpg[i];
2135	if (tpg) {
2136	mutex_lock(&tpg->tv_tpg_mutex);
2137	tpg->vhost_scsi = NULL;
2138	tpg->tv_tpg_vhost_count--;
2139	mutex_unlock(&tpg->tv_tpg_mutex);
2140	target_undepend_item(&tpg->se_tpg.tpg_group.cg_item);
2141	}
2142	}
2143	free_tpg:
2144	kfree(vs_tpg);
2145	out:
2146	mutex_unlock(&vs->dev.mutex);
2147	return ret;
2148	}
2149
2150	static int
2151	vhost_scsi_clear_endpoint(struct vhost_scsi *vs,
2152	struct vhost_scsi_target *t)
2153	{
2154	struct se_portal_group *se_tpg;
2155	struct vhost_scsi_tport *tv_tport;
2156	struct vhost_scsi_tpg *tpg;
2157	struct vhost_virtqueue *vq;
2158	bool match = false;
2159	int index, ret, i;
2160	u8 target;
2161
2162	mutex_lock(&vs->dev.mutex);
2163	/* Verify that ring has been setup correctly. */
2164	for (index = 0; index < vs->dev.nvqs; ++index) {
2165	if (!vhost_vq_access_ok(&vs->vqs[index].vq)) {
2166	ret = -EFAULT;
2167	goto err_dev;
2168	}
2169	}
2170
2171	if (!vs->vs_tpg) {
2172	ret = 0;
2173	goto err_dev;
2174	}
2175
2176	for (i = 0; i < VHOST_SCSI_MAX_TARGET; i++) {
2177	target = i;
2178	tpg = vs->vs_tpg[target];
2179	if (!tpg)
2180	continue;
2181
2182	tv_tport = tpg->tport;
2183	if (!tv_tport) {
2184	ret = -ENODEV;
2185	goto err_dev;
2186	}
2187
2188	if (strcmp(tv_tport->tport_name, t->vhost_wwpn)) {
2189	pr_warn("tv_tport->tport_name: %s, tpg->tport_tpgt: %hu"
2190	" does not match t->vhost_wwpn: %s, t->vhost_tpgt: %hu\n",
2191	tv_tport->tport_name, tpg->tport_tpgt,
2192	t->vhost_wwpn, t->vhost_tpgt);
2193	ret = -EINVAL;
2194	goto err_dev;
2195	}
2196	match = true;
2197	}
2198	if (!match)
2199	goto free_vs_tpg;
2200
2201	/* Prevent new cmds from starting and accessing the tpgs/sessions */
2202	for (i = 0; i < vs->dev.nvqs; i++) {
2203	vq = &vs->vqs[i].vq;
2204	mutex_lock(&vq->mutex);
2205	vhost_vq_set_backend(vq, NULL);
2206	mutex_unlock(&vq->mutex);
2207	}
2208	/* Make sure cmds are not running before tearing them down. */
2209	vhost_scsi_flush(vs);
2210
2211	for (i = 0; i < vs->dev.nvqs; i++) {
2212	vq = &vs->vqs[i].vq;
2213	vhost_scsi_destroy_vq_cmds(vq);
2214	}
2215
2216	/*
2217	* We can now release our hold on the tpg and sessions and userspace
2218	* can free them after this point.
2219	*/
2220	for (i = 0; i < VHOST_SCSI_MAX_TARGET; i++) {
2221	target = i;
2222	tpg = vs->vs_tpg[target];
2223	if (!tpg)
2224	continue;
2225
2226	mutex_lock(&tpg->tv_tpg_mutex);
2227
2228	tpg->tv_tpg_vhost_count--;
2229	tpg->vhost_scsi = NULL;
2230	vs->vs_tpg[target] = NULL;
2231
2232	mutex_unlock(&tpg->tv_tpg_mutex);
2233
2234	se_tpg = &tpg->se_tpg;
2235	target_undepend_item(&se_tpg->tpg_group.cg_item);
2236	}
2237
2238	free_vs_tpg:
2239	/*
2240	* Act as synchronize_rcu to make sure access to
2241	* old vs->vs_tpg is finished.
2242	*/
2243	vhost_scsi_flush(vs);
2244	kfree(vs->vs_tpg);
2245	vs->vs_tpg = NULL;
2246	memset(vs->vs_vhost_wwpn, 0, sizeof(vs->vs_vhost_wwpn));
2247	WARN_ON(vs->vs_events_nr);
2248	mutex_unlock(&vs->dev.mutex);
2249	return 0;
2250
2251	err_dev:
2252	mutex_unlock(&vs->dev.mutex);
2253	return ret;
2254	}
2255
2256	static int vhost_scsi_set_features(struct vhost_scsi *vs, u64 features)
2257	{
2258	struct vhost_virtqueue *vq;
2259	bool is_log, was_log;
2260	u64 old_features;
2261	int i;
2262
2263	if (features & ~VHOST_SCSI_FEATURES)
2264	return -EOPNOTSUPP;
2265
2266	mutex_lock(&vs->dev.mutex);
2267	if ((features & (1 << VHOST_F_LOG_ALL)) &&
2268	!vhost_log_access_ok(&vs->dev)) {
2269	mutex_unlock(&vs->dev.mutex);
2270	return -EFAULT;
2271	}
2272
2273	if (!vs->dev.nvqs)
2274	goto out;
2275
2276	old_features = vs->vqs[0].vq.acked_features;
2277	if (vs->vs_tpg &&
2278	((features ^ old_features) &
2279	~(1ULL << VHOST_F_LOG_ALL))) {
2280	mutex_unlock(&vs->dev.mutex);
2281	return -EBUSY;
2282	}
2283
2284	is_log = features & (1 << VHOST_F_LOG_ALL);
2285	/*
2286	* All VQs should have same feature.
2287	*/
2288	was_log = vhost_has_feature(&vs->vqs[0].vq, VHOST_F_LOG_ALL);
2289
2290	for (i = 0; i < vs->dev.nvqs; i++) {
2291	vq = &vs->vqs[i].vq;
2292	mutex_lock(&vq->mutex);
2293	vq->acked_features = features;
2294	mutex_unlock(&vq->mutex);
2295	}
2296
2297	/*
2298	* If VHOST_F_LOG_ALL is removed, free tvc_log after
2299	* vq->acked_features is committed.
2300	*/
2301	if (!is_log && was_log) {
2302	for (i = VHOST_SCSI_VQ_IO; i < vs->dev.nvqs; i++) {
2303	if (!vs->vqs[i].scsi_cmds)
2304	continue;
2305
2306	vq = &vs->vqs[i].vq;
2307	mutex_lock(&vq->mutex);
2308	vhost_scsi_destroy_vq_log(vq);
2309	mutex_unlock(&vq->mutex);
2310	}
2311	}
2312
2313	out:
2314	mutex_unlock(&vs->dev.mutex);
2315	return 0;
2316	}
2317
2318	static int vhost_scsi_open(struct inode inode, struct file f)
2319	{
2320	struct vhost_scsi_virtqueue *svq;
2321	struct vhost_scsi *vs;
2322	struct vhost_virtqueue **vqs;
2323	int r = -ENOMEM, i, nvqs;
2324
2325	vs = kvzalloc_obj(*vs);
2326	if (!vs)
2327	goto err_vs;
2328	vs->inline_sg_cnt = vhost_scsi_inline_sg_cnt;
2329
2330	nvqs = READ_ONCE(vhost_scsi_max_io_vqs);
2331	nvqs += VHOST_SCSI_VQ_IO;
2332
2333	vs->old_inflight = kmalloc_objs(*vs->old_inflight, nvqs,
2334	GFP_KERNEL \| __GFP_ZERO);
2335	if (!vs->old_inflight)
2336	goto err_inflight;
2337
2338	vs->vqs = kvzalloc_objs(*vs->vqs, nvqs);
2339	if (!vs->vqs)
2340	goto err_vqs;
2341
2342	vqs = kmalloc_objs(*vqs, nvqs);
2343	if (!vqs)
2344	goto err_local_vqs;
2345
2346	vhost_work_init(&vs->vs_event_work, vhost_scsi_evt_work);
2347
2348	vs->vs_events_nr = 0;
2349	vs->vs_events_missed = false;
2350
2351	vqs[VHOST_SCSI_VQ_CTL] = &vs->vqs[VHOST_SCSI_VQ_CTL].vq;
2352	vqs[VHOST_SCSI_VQ_EVT] = &vs->vqs[VHOST_SCSI_VQ_EVT].vq;
2353	vs->vqs[VHOST_SCSI_VQ_CTL].vq.handle_kick = vhost_scsi_ctl_handle_kick;
2354	vs->vqs[VHOST_SCSI_VQ_EVT].vq.handle_kick = vhost_scsi_evt_handle_kick;
2355	for (i = VHOST_SCSI_VQ_IO; i < nvqs; i++) {
2356	svq = &vs->vqs[i];
2357
2358	vqs[i] = &svq->vq;
2359	svq->vs = vs;
2360	init_llist_head(&svq->completion_list);
2361	vhost_work_init(&svq->completion_work,
2362	vhost_scsi_complete_cmd_work);
2363	svq->vq.handle_kick = vhost_scsi_handle_kick;
2364	}
2365	vhost_dev_init(&vs->dev, vqs, nvqs, UIO_MAXIOV,
2366	VHOST_SCSI_WEIGHT, 0, true, NULL);
2367
2368	vhost_scsi_init_inflight(vs, NULL);
2369
2370	f->private_data = vs;
2371	return 0;
2372
2373	err_local_vqs:
2374	kvfree(vs->vqs);
2375	err_vqs:
2376	kfree(vs->old_inflight);
2377	err_inflight:
2378	kvfree(vs);
2379	err_vs:
2380	return r;
2381	}
2382
2383	static int vhost_scsi_release(struct inode inode, struct file f)
2384	{
2385	struct vhost_scsi *vs = f->private_data;
2386	struct vhost_scsi_target t;
2387
2388	mutex_lock(&vs->dev.mutex);
2389	memcpy(t.vhost_wwpn, vs->vs_vhost_wwpn, sizeof(t.vhost_wwpn));
2390	mutex_unlock(&vs->dev.mutex);
2391	vhost_scsi_clear_endpoint(vs, &t);
2392	vhost_dev_stop(&vs->dev);
2393	vhost_dev_cleanup(&vs->dev);
2394	kfree(vs->dev.vqs);
2395	kvfree(vs->vqs);
2396	kfree(vs->old_inflight);
2397	kvfree(vs);
2398	return 0;
2399	}
2400
2401	static long
2402	vhost_scsi_ioctl(struct file *f,
2403	unsigned int ioctl,
2404	unsigned long arg)
2405	{
2406	struct vhost_scsi *vs = f->private_data;
2407	struct vhost_scsi_target backend;
2408	void __user argp = (void __user )arg;
2409	u64 __user *featurep = argp;
2410	u32 __user *eventsp = argp;
2411	u32 events_missed;
2412	u64 features;
2413	int r, abi_version = VHOST_SCSI_ABI_VERSION;
2414	struct vhost_virtqueue *vq = &vs->vqs[VHOST_SCSI_VQ_EVT].vq;
2415
2416	switch (ioctl) {
2417	case VHOST_SCSI_SET_ENDPOINT:
2418	if (copy_from_user(&backend, argp, sizeof backend))
2419	return -EFAULT;
2420	if (backend.reserved != 0)
2421	return -EOPNOTSUPP;
2422
2423	return vhost_scsi_set_endpoint(vs, &backend);
2424	case VHOST_SCSI_CLEAR_ENDPOINT:
2425	if (copy_from_user(&backend, argp, sizeof backend))
2426	return -EFAULT;
2427	if (backend.reserved != 0)
2428	return -EOPNOTSUPP;
2429
2430	return vhost_scsi_clear_endpoint(vs, &backend);
2431	case VHOST_SCSI_GET_ABI_VERSION:
2432	if (copy_to_user(argp, &abi_version, sizeof abi_version))
2433	return -EFAULT;
2434	return 0;
2435	case VHOST_SCSI_SET_EVENTS_MISSED:
2436	if (get_user(events_missed, eventsp))
2437	return -EFAULT;
2438	mutex_lock(&vq->mutex);
2439	vs->vs_events_missed = events_missed;
2440	mutex_unlock(&vq->mutex);
2441	return 0;
2442	case VHOST_SCSI_GET_EVENTS_MISSED:
2443	mutex_lock(&vq->mutex);
2444	events_missed = vs->vs_events_missed;
2445	mutex_unlock(&vq->mutex);
2446	if (put_user(events_missed, eventsp))
2447	return -EFAULT;
2448	return 0;
2449	case VHOST_GET_FEATURES:
2450	features = VHOST_SCSI_FEATURES;
2451	if (copy_to_user(featurep, &features, sizeof features))
2452	return -EFAULT;
2453	return 0;
2454	case VHOST_SET_FEATURES:
2455	if (copy_from_user(&features, featurep, sizeof features))
2456	return -EFAULT;
2457	return vhost_scsi_set_features(vs, features);
2458	case VHOST_NEW_WORKER:
2459	case VHOST_FREE_WORKER:
2460	case VHOST_ATTACH_VRING_WORKER:
2461	case VHOST_GET_VRING_WORKER:
2462	mutex_lock(&vs->dev.mutex);
2463	r = vhost_worker_ioctl(&vs->dev, ioctl, argp);
2464	mutex_unlock(&vs->dev.mutex);
2465	return r;
2466	default:
2467	mutex_lock(&vs->dev.mutex);
2468	r = vhost_dev_ioctl(&vs->dev, ioctl, argp);
2469	if (r == -ENOIOCTLCMD)
2470	r = vhost_vring_ioctl(&vs->dev, ioctl, argp);
2471	else
2472	vhost_scsi_flush(vs);
2473	mutex_unlock(&vs->dev.mutex);
2474	return r;
2475	}
2476	}
2477
2478	static const struct file_operations vhost_scsi_fops = {
2479	.owner = THIS_MODULE,
2480	.release = vhost_scsi_release,
2481	.unlocked_ioctl = vhost_scsi_ioctl,
2482	.compat_ioctl = compat_ptr_ioctl,
2483	.open = vhost_scsi_open,
2484	.llseek = noop_llseek,
2485	};
2486
2487	static struct miscdevice vhost_scsi_misc = {
2488	MISC_DYNAMIC_MINOR,
2489	"vhost-scsi",
2490	&vhost_scsi_fops,
2491	};
2492
2493	static int __init vhost_scsi_register(void)
2494	{
2495	return misc_register(&vhost_scsi_misc);
2496	}
2497
2498	static void vhost_scsi_deregister(void)
2499	{
2500	misc_deregister(&vhost_scsi_misc);
2501	}
2502
2503	static char vhost_scsi_dump_proto_id(struct vhost_scsi_tport tport)
2504	{
2505	switch (tport->tport_proto_id) {
2506	case SCSI_PROTOCOL_SAS:
2507	return "SAS";
2508	case SCSI_PROTOCOL_FCP:
2509	return "FCP";
2510	case SCSI_PROTOCOL_ISCSI:
2511	return "iSCSI";
2512	default:
2513	break;
2514	}
2515
2516	return "Unknown";
2517	}
2518
2519	static void
2520	vhost_scsi_do_plug(struct vhost_scsi_tpg *tpg,
2521	struct se_lun *lun, bool plug)
2522	{
2523
2524	struct vhost_scsi *vs = tpg->vhost_scsi;
2525	struct vhost_virtqueue *vq;
2526	u32 reason;
2527
2528	if (!vs)
2529	return;
2530
2531	if (plug)
2532	reason = VIRTIO_SCSI_EVT_RESET_RESCAN;
2533	else
2534	reason = VIRTIO_SCSI_EVT_RESET_REMOVED;
2535
2536	vq = &vs->vqs[VHOST_SCSI_VQ_EVT].vq;
2537	mutex_lock(&vq->mutex);
2538	/*
2539	* We can't queue events if the backend has been cleared, because
2540	* we could end up queueing an event after the flush.
2541	*/
2542	if (!vhost_vq_get_backend(vq))
2543	goto unlock;
2544
2545	if (vhost_has_feature(vq, VIRTIO_SCSI_F_HOTPLUG))
2546	vhost_scsi_send_evt(vs, vq, tpg, lun,
2547	VIRTIO_SCSI_T_TRANSPORT_RESET, reason);
2548	unlock:
2549	mutex_unlock(&vq->mutex);
2550	}
2551
2552	static void vhost_scsi_hotplug(struct vhost_scsi_tpg tpg, struct se_lun lun)
2553	{
2554	vhost_scsi_do_plug(tpg, lun, true);
2555	}
2556
2557	static void vhost_scsi_hotunplug(struct vhost_scsi_tpg tpg, struct se_lun lun)
2558	{
2559	vhost_scsi_do_plug(tpg, lun, false);
2560	}
2561
2562	static int vhost_scsi_port_link(struct se_portal_group *se_tpg,
2563	struct se_lun *lun)
2564	{
2565	struct vhost_scsi_tpg *tpg = container_of(se_tpg,
2566	struct vhost_scsi_tpg, se_tpg);
2567
2568	mutex_lock(&tpg->tv_tpg_mutex);
2569	tpg->tv_tpg_port_count++;
2570	vhost_scsi_hotplug(tpg, lun);
2571	mutex_unlock(&tpg->tv_tpg_mutex);
2572
2573	return 0;
2574	}
2575
2576	static void vhost_scsi_port_unlink(struct se_portal_group *se_tpg,
2577	struct se_lun *lun)
2578	{
2579	struct vhost_scsi_tpg *tpg = container_of(se_tpg,
2580	struct vhost_scsi_tpg, se_tpg);
2581
2582	mutex_lock(&tpg->tv_tpg_mutex);
2583	tpg->tv_tpg_port_count--;
2584	vhost_scsi_hotunplug(tpg, lun);
2585	mutex_unlock(&tpg->tv_tpg_mutex);
2586	}
2587
2588	static ssize_t vhost_scsi_tpg_attrib_fabric_prot_type_store(
2589	struct config_item item, const char page, size_t count)
2590	{
2591	struct se_portal_group *se_tpg = attrib_to_tpg(item);
2592	struct vhost_scsi_tpg *tpg = container_of(se_tpg,
2593	struct vhost_scsi_tpg, se_tpg);
2594	unsigned long val;
2595	int ret = kstrtoul(page, 0, &val);
2596
2597	if (ret) {
2598	pr_err("kstrtoul() returned %d for fabric_prot_type\n", ret);
2599	return ret;
2600	}
2601	if (val != 0 && val != 1 && val != 3) {
2602	pr_err("Invalid vhost_scsi fabric_prot_type: %lu\n", val);
2603	return -EINVAL;
2604	}
2605	tpg->tv_fabric_prot_type = val;
2606
2607	return count;
2608	}
2609
2610	static ssize_t vhost_scsi_tpg_attrib_fabric_prot_type_show(
2611	struct config_item item, char page)
2612	{
2613	struct se_portal_group *se_tpg = attrib_to_tpg(item);
2614	struct vhost_scsi_tpg *tpg = container_of(se_tpg,
2615	struct vhost_scsi_tpg, se_tpg);
2616
2617	return sysfs_emit(page, "%d\n", tpg->tv_fabric_prot_type);
2618	}
2619
2620	CONFIGFS_ATTR(vhost_scsi_tpg_attrib_, fabric_prot_type);
2621
2622	static struct configfs_attribute *vhost_scsi_tpg_attrib_attrs[] = {
2623	&vhost_scsi_tpg_attrib_attr_fabric_prot_type,
2624	NULL,
2625	};
2626
2627	static int vhost_scsi_make_nexus(struct vhost_scsi_tpg *tpg,
2628	const char *name)
2629	{
2630	struct vhost_scsi_nexus *tv_nexus;
2631
2632	mutex_lock(&tpg->tv_tpg_mutex);
2633	if (tpg->tpg_nexus) {
2634	mutex_unlock(&tpg->tv_tpg_mutex);
2635	pr_debug("tpg->tpg_nexus already exists\n");
2636	return -EEXIST;
2637	}
2638
2639	tv_nexus = kzalloc_obj(*tv_nexus);
2640	if (!tv_nexus) {
2641	mutex_unlock(&tpg->tv_tpg_mutex);
2642	pr_err("Unable to allocate struct vhost_scsi_nexus\n");
2643	return -ENOMEM;
2644	}
2645	/*
2646	* Since we are running in 'demo mode' this call will generate a
2647	* struct se_node_acl for the vhost_scsi struct se_portal_group with
2648	* the SCSI Initiator port name of the passed configfs group 'name'.
2649	*/
2650	tv_nexus->tvn_se_sess = target_setup_session(&tpg->se_tpg, 0, 0,
2651	TARGET_PROT_DIN_PASS \| TARGET_PROT_DOUT_PASS,
2652	(unsigned char *)name, tv_nexus, NULL);
2653	if (IS_ERR(tv_nexus->tvn_se_sess)) {
2654	mutex_unlock(&tpg->tv_tpg_mutex);
2655	kfree(tv_nexus);
2656	return -ENOMEM;
2657	}
2658	tpg->tpg_nexus = tv_nexus;
2659
2660	mutex_unlock(&tpg->tv_tpg_mutex);
2661	return 0;
2662	}
2663
2664	static int vhost_scsi_drop_nexus(struct vhost_scsi_tpg *tpg)
2665	{
2666	struct se_session *se_sess;
2667	struct vhost_scsi_nexus *tv_nexus;
2668
2669	mutex_lock(&tpg->tv_tpg_mutex);
2670	tv_nexus = tpg->tpg_nexus;
2671	if (!tv_nexus) {
2672	mutex_unlock(&tpg->tv_tpg_mutex);
2673	return -ENODEV;
2674	}
2675
2676	se_sess = tv_nexus->tvn_se_sess;
2677	if (!se_sess) {
2678	mutex_unlock(&tpg->tv_tpg_mutex);
2679	return -ENODEV;
2680	}
2681
2682	if (tpg->tv_tpg_port_count != 0) {
2683	mutex_unlock(&tpg->tv_tpg_mutex);
2684	pr_err("Unable to remove TCM_vhost I_T Nexus with"
2685	" active TPG port count: %d\n",
2686	tpg->tv_tpg_port_count);
2687	return -EBUSY;
2688	}
2689
2690	if (tpg->tv_tpg_vhost_count != 0) {
2691	mutex_unlock(&tpg->tv_tpg_mutex);
2692	pr_err("Unable to remove TCM_vhost I_T Nexus with"
2693	" active TPG vhost count: %d\n",
2694	tpg->tv_tpg_vhost_count);
2695	return -EBUSY;
2696	}
2697
2698	pr_debug("TCM_vhost_ConfigFS: Removing I_T Nexus to emulated"
2699	" %s Initiator Port: %s\n", vhost_scsi_dump_proto_id(tpg->tport),
2700	tv_nexus->tvn_se_sess->se_node_acl->initiatorname);
2701
2702	/*
2703	* Release the SCSI I_T Nexus to the emulated vhost Target Port
2704	*/
2705	target_remove_session(se_sess);
2706	tpg->tpg_nexus = NULL;
2707	mutex_unlock(&tpg->tv_tpg_mutex);
2708
2709	kfree(tv_nexus);
2710	return 0;
2711	}
2712
2713	static ssize_t vhost_scsi_tpg_nexus_show(struct config_item item, char page)
2714	{
2715	struct se_portal_group *se_tpg = to_tpg(item);
2716	struct vhost_scsi_tpg *tpg = container_of(se_tpg,
2717	struct vhost_scsi_tpg, se_tpg);
2718	struct vhost_scsi_nexus *tv_nexus;
2719	ssize_t ret;
2720
2721	mutex_lock(&tpg->tv_tpg_mutex);
2722	tv_nexus = tpg->tpg_nexus;
2723	if (!tv_nexus) {
2724	mutex_unlock(&tpg->tv_tpg_mutex);
2725	return -ENODEV;
2726	}
2727	ret = sysfs_emit(page, "%s\n",
2728	tv_nexus->tvn_se_sess->se_node_acl->initiatorname);
2729	mutex_unlock(&tpg->tv_tpg_mutex);
2730
2731	return ret;
2732	}
2733
2734	static ssize_t vhost_scsi_tpg_nexus_store(struct config_item *item,
2735	const char *page, size_t count)
2736	{
2737	struct se_portal_group *se_tpg = to_tpg(item);
2738	struct vhost_scsi_tpg *tpg = container_of(se_tpg,
2739	struct vhost_scsi_tpg, se_tpg);
2740	struct vhost_scsi_tport *tport_wwn = tpg->tport;
2741	unsigned char i_port[VHOST_SCSI_NAMELEN], ptr, port_ptr;
2742	int ret;
2743	/*
2744	* Shutdown the active I_T nexus if 'NULL' is passed..
2745	*/
2746	if (!strncmp(page, "NULL", 4)) {
2747	ret = vhost_scsi_drop_nexus(tpg);
2748	return (!ret) ? count : ret;
2749	}
2750	/*
2751	* Otherwise make sure the passed virtual Initiator port WWN matches
2752	* the fabric protocol_id set in vhost_scsi_make_tport(), and call
2753	* vhost_scsi_make_nexus().
2754	*/
2755	if (strlen(page) >= VHOST_SCSI_NAMELEN) {
2756	pr_err("Emulated NAA Sas Address: %s, exceeds"
2757	" max: %d\n", page, VHOST_SCSI_NAMELEN);
2758	return -EINVAL;
2759	}
2760	snprintf(&i_port[0], VHOST_SCSI_NAMELEN, "%s", page);
2761
2762	ptr = strstr(i_port, "naa.");
2763	if (ptr) {
2764	if (tport_wwn->tport_proto_id != SCSI_PROTOCOL_SAS) {
2765	pr_err("Passed SAS Initiator Port %s does not"
2766	" match target port protoid: %s\n", i_port,
2767	vhost_scsi_dump_proto_id(tport_wwn));
2768	return -EINVAL;
2769	}
2770	port_ptr = &i_port[0];
2771	goto check_newline;
2772	}
2773	ptr = strstr(i_port, "fc.");
2774	if (ptr) {
2775	if (tport_wwn->tport_proto_id != SCSI_PROTOCOL_FCP) {
2776	pr_err("Passed FCP Initiator Port %s does not"
2777	" match target port protoid: %s\n", i_port,
2778	vhost_scsi_dump_proto_id(tport_wwn));
2779	return -EINVAL;
2780	}
2781	port_ptr = &i_port[3]; /* Skip over "fc." */
2782	goto check_newline;
2783	}
2784	ptr = strstr(i_port, "iqn.");
2785	if (ptr) {
2786	if (tport_wwn->tport_proto_id != SCSI_PROTOCOL_ISCSI) {
2787	pr_err("Passed iSCSI Initiator Port %s does not"
2788	" match target port protoid: %s\n", i_port,
2789	vhost_scsi_dump_proto_id(tport_wwn));
2790	return -EINVAL;
2791	}
2792	port_ptr = &i_port[0];
2793	goto check_newline;
2794	}
2795	pr_err("Unable to locate prefix for emulated Initiator Port:"
2796	" %s\n", i_port);
2797	return -EINVAL;
2798	/*
2799	* Clear any trailing newline for the NAA WWN
2800	*/
2801	check_newline:
2802	if (i_port[strlen(i_port)-1] == '\n')
2803	i_port[strlen(i_port)-1] = '\0';
2804
2805	ret = vhost_scsi_make_nexus(tpg, port_ptr);
2806	if (ret < 0)
2807	return ret;
2808
2809	return count;
2810	}
2811
2812	CONFIGFS_ATTR(vhost_scsi_tpg_, nexus);
2813
2814	static struct configfs_attribute *vhost_scsi_tpg_attrs[] = {
2815	&vhost_scsi_tpg_attr_nexus,
2816	NULL,
2817	};
2818
2819	static struct se_portal_group *
2820	vhost_scsi_make_tpg(struct se_wwn wwn, const char name)
2821	{
2822	struct vhost_scsi_tport *tport = container_of(wwn,
2823	struct vhost_scsi_tport, tport_wwn);
2824
2825	struct vhost_scsi_tpg *tpg;
2826	u16 tpgt;
2827	int ret;
2828
2829	if (strstr(name, "tpgt_") != name)
2830	return ERR_PTR(-EINVAL);
2831	if (kstrtou16(name + 5, 10, &tpgt) \|\| tpgt >= VHOST_SCSI_MAX_TARGET)
2832	return ERR_PTR(-EINVAL);
2833
2834	tpg = kzalloc_obj(*tpg);
2835	if (!tpg) {
2836	pr_err("Unable to allocate struct vhost_scsi_tpg");
2837	return ERR_PTR(-ENOMEM);
2838	}
2839	mutex_init(&tpg->tv_tpg_mutex);
2840	INIT_LIST_HEAD(&tpg->tv_tpg_list);
2841	tpg->tport = tport;
2842	tpg->tport_tpgt = tpgt;
2843
2844	ret = core_tpg_register(wwn, &tpg->se_tpg, tport->tport_proto_id);
2845	if (ret < 0) {
2846	kfree(tpg);
2847	return NULL;
2848	}
2849	mutex_lock(&vhost_scsi_mutex);
2850	list_add_tail(&tpg->tv_tpg_list, &vhost_scsi_list);
2851	mutex_unlock(&vhost_scsi_mutex);
2852
2853	return &tpg->se_tpg;
2854	}
2855
2856	static void vhost_scsi_drop_tpg(struct se_portal_group *se_tpg)
2857	{
2858	struct vhost_scsi_tpg *tpg = container_of(se_tpg,
2859	struct vhost_scsi_tpg, se_tpg);
2860
2861	mutex_lock(&vhost_scsi_mutex);
2862	list_del(&tpg->tv_tpg_list);
2863	mutex_unlock(&vhost_scsi_mutex);
2864	/*
2865	* Release the virtual I_T Nexus for this vhost TPG
2866	*/
2867	vhost_scsi_drop_nexus(tpg);
2868	/*
2869	* Deregister the se_tpg from TCM..
2870	*/
2871	core_tpg_deregister(se_tpg);
2872	kfree(tpg);
2873	}
2874
2875	static struct se_wwn *
2876	vhost_scsi_make_tport(struct target_fabric_configfs *tf,
2877	struct config_group *group,
2878	const char *name)
2879	{
2880	struct vhost_scsi_tport *tport;
2881	char *ptr;
2882	u64 wwpn = 0;
2883	int off = 0;
2884
2885	/* if (vhost_scsi_parse_wwn(name, &wwpn, 1) < 0)
2886	return ERR_PTR(-EINVAL); */
2887
2888	tport = kzalloc_obj(*tport);
2889	if (!tport) {
2890	pr_err("Unable to allocate struct vhost_scsi_tport");
2891	return ERR_PTR(-ENOMEM);
2892	}
2893	tport->tport_wwpn = wwpn;
2894	/*
2895	* Determine the emulated Protocol Identifier and Target Port Name
2896	* based on the incoming configfs directory name.
2897	*/
2898	ptr = strstr(name, "naa.");
2899	if (ptr) {
2900	tport->tport_proto_id = SCSI_PROTOCOL_SAS;
2901	goto check_len;
2902	}
2903	ptr = strstr(name, "fc.");
2904	if (ptr) {
2905	tport->tport_proto_id = SCSI_PROTOCOL_FCP;
2906	off = 3; /* Skip over "fc." */
2907	goto check_len;
2908	}
2909	ptr = strstr(name, "iqn.");
2910	if (ptr) {
2911	tport->tport_proto_id = SCSI_PROTOCOL_ISCSI;
2912	goto check_len;
2913	}
2914
2915	pr_err("Unable to locate prefix for emulated Target Port:"
2916	" %s\n", name);
2917	kfree(tport);
2918	return ERR_PTR(-EINVAL);
2919
2920	check_len:
2921	if (strlen(name) >= VHOST_SCSI_NAMELEN) {
2922	pr_err("Emulated %s Address: %s, exceeds"
2923	" max: %d\n", vhost_scsi_dump_proto_id(tport), name,
2924	VHOST_SCSI_NAMELEN);
2925	kfree(tport);
2926	return ERR_PTR(-EINVAL);
2927	}
2928	snprintf(&tport->tport_name[0], VHOST_SCSI_NAMELEN, "%s", &name[off]);
2929
2930	pr_debug("TCM_VHost_ConfigFS: Allocated emulated Target"
2931	" %s Address: %s\n", vhost_scsi_dump_proto_id(tport), name);
2932
2933	return &tport->tport_wwn;
2934	}
2935
2936	static void vhost_scsi_drop_tport(struct se_wwn *wwn)
2937	{
2938	struct vhost_scsi_tport *tport = container_of(wwn,
2939	struct vhost_scsi_tport, tport_wwn);
2940
2941	pr_debug("TCM_VHost_ConfigFS: Deallocating emulated Target"
2942	" %s Address: %s\n", vhost_scsi_dump_proto_id(tport),
2943	tport->tport_name);
2944
2945	kfree(tport);
2946	}
2947
2948	static ssize_t
2949	vhost_scsi_wwn_version_show(struct config_item item, char page)
2950	{
2951	return sysfs_emit(page, "TCM_VHOST fabric module %s on %s/%s"
2952	" on "UTS_RELEASE"\n", VHOST_SCSI_VERSION, utsname()->sysname,
2953	utsname()->machine);
2954	}
2955
2956	CONFIGFS_ATTR_RO(vhost_scsi_wwn_, version);
2957
2958	static struct configfs_attribute *vhost_scsi_wwn_attrs[] = {
2959	&vhost_scsi_wwn_attr_version,
2960	NULL,
2961	};
2962
2963	static const struct target_core_fabric_ops vhost_scsi_ops = {
2964	.module = THIS_MODULE,
2965	.fabric_name = "vhost",
2966	.max_data_sg_nents = VHOST_SCSI_PREALLOC_SGLS,
2967	.tpg_get_wwn = vhost_scsi_get_fabric_wwn,
2968	.tpg_get_tag = vhost_scsi_get_tpgt,
2969	.tpg_check_demo_mode = vhost_scsi_check_true,
2970	.tpg_check_demo_mode_cache = vhost_scsi_check_true,
2971	.tpg_check_prot_fabric_only = vhost_scsi_check_prot_fabric_only,
2972	.release_cmd = vhost_scsi_release_cmd,
2973	.check_stop_free = vhost_scsi_check_stop_free,
2974	.sess_get_initiator_sid = NULL,
2975	.write_pending = vhost_scsi_write_pending,
2976	.queue_data_in = vhost_scsi_queue_data_in,
2977	.queue_status = vhost_scsi_queue_status,
2978	.queue_tm_rsp = vhost_scsi_queue_tm_rsp,
2979	.aborted_task = vhost_scsi_aborted_task,
2980	/*
2981	* Setup callers for generic logic in target_core_fabric_configfs.c
2982	*/
2983	.fabric_make_wwn = vhost_scsi_make_tport,
2984	.fabric_drop_wwn = vhost_scsi_drop_tport,
2985	.fabric_make_tpg = vhost_scsi_make_tpg,
2986	.fabric_drop_tpg = vhost_scsi_drop_tpg,
2987	.fabric_post_link = vhost_scsi_port_link,
2988	.fabric_pre_unlink = vhost_scsi_port_unlink,
2989
2990	.tfc_wwn_attrs = vhost_scsi_wwn_attrs,
2991	.tfc_tpg_base_attrs = vhost_scsi_tpg_attrs,
2992	.tfc_tpg_attrib_attrs = vhost_scsi_tpg_attrib_attrs,
2993
2994	.default_compl_type = TARGET_QUEUE_COMPL,
2995	.direct_compl_supp = 1,
2996	.default_submit_type = TARGET_QUEUE_SUBMIT,
2997	.direct_submit_supp = 1,
2998	};
2999
3000	static int __init vhost_scsi_init(void)
3001	{
3002	int ret = -ENOMEM;
3003
3004	pr_debug("TCM_VHOST fabric module %s on %s/%s"
3005	" on "UTS_RELEASE"\n", VHOST_SCSI_VERSION, utsname()->sysname,
3006	utsname()->machine);
3007
3008	ret = vhost_scsi_register();
3009	if (ret < 0)
3010	goto out;
3011
3012	ret = target_register_template(&vhost_scsi_ops);
3013	if (ret < 0)
3014	goto out_vhost_scsi_deregister;
3015
3016	return 0;
3017
3018	out_vhost_scsi_deregister:
3019	vhost_scsi_deregister();
3020	out:
3021	return ret;
3022	}
3023
3024	static void vhost_scsi_exit(void)
3025	{
3026	target_unregister_template(&vhost_scsi_ops);
3027	vhost_scsi_deregister();
3028	}
3029
3030	MODULE_DESCRIPTION("VHOST_SCSI series fabric driver");
3031	MODULE_ALIAS("tcm_vhost");
3032	MODULE_LICENSE("GPL");
3033	module_init(vhost_scsi_init);
3034	module_exit(vhost_scsi_exit);