linux - drivers/vhost/vhost.c

3433 lines · 81.9 KiB

1	// SPDX-License-Identifier: GPL-2.0-only
2	/* Copyright (C) 2009 Red Hat, Inc.
3	* Copyright (C) 2006 Rusty Russell IBM Corporation
4	*
5	* Author: Michael S. Tsirkin <[email protected]>
6	*
7	* Inspiration, some code, and most witty comments come from
8	* Documentation/virtual/lguest/lguest.c, by Rusty Russell
9	*
10	* Generic code for virtio server in host kernel.
11	*/
12
13	#include <linux/eventfd.h>
14	#include <linux/vhost.h>
15	#include <linux/uio.h>
16	#include <linux/mm.h>
17	#include <linux/miscdevice.h>
18	#include <linux/mutex.h>
19	#include <linux/poll.h>
20	#include <linux/file.h>
21	#include <linux/highmem.h>
22	#include <linux/slab.h>
23	#include <linux/vmalloc.h>
24	#include <linux/kthread.h>
25	#include <linux/cgroup.h>
26	#include <linux/module.h>
27	#include <linux/sort.h>
28	#include <linux/sched/mm.h>
29	#include <linux/sched/signal.h>
30	#include <linux/sched/vhost_task.h>
31	#include <linux/interval_tree_generic.h>
32	#include <linux/nospec.h>
33	#include <linux/kcov.h>
34
35	#include "vhost.h"
36
37	static ushort max_mem_regions = 64;
38	module_param(max_mem_regions, ushort, 0444);
39	MODULE_PARM_DESC(max_mem_regions,
40	"Maximum number of memory regions in memory map. (default: 64)");
41	static int max_iotlb_entries = 2048;
42	module_param(max_iotlb_entries, int, 0444);
43	MODULE_PARM_DESC(max_iotlb_entries,
44	"Maximum number of iotlb entries. (default: 2048)");
45	static bool fork_from_owner_default = VHOST_FORK_OWNER_TASK;
46
47	#ifdef CONFIG_VHOST_ENABLE_FORK_OWNER_CONTROL
48	module_param(fork_from_owner_default, bool, 0444);
49	MODULE_PARM_DESC(fork_from_owner_default,
50	"Set task mode as the default(default: Y)");
51	#endif
52
53	enum {
54	VHOST_MEMORY_F_LOG = 0x1,
55	};
56
57	#define vhost_used_event(vq) ((__virtio16 __user *)&vq->avail->ring[vq->num])
58	#define vhost_avail_event(vq) ((__virtio16 __user *)&vq->used->ring[vq->num])
59
60	#ifdef CONFIG_VHOST_CROSS_ENDIAN_LEGACY
61	static void vhost_disable_cross_endian(struct vhost_virtqueue *vq)
62	{
63	vq->user_be = !virtio_legacy_is_little_endian();
64	}
65
66	static void vhost_enable_cross_endian_big(struct vhost_virtqueue *vq)
67	{
68	vq->user_be = true;
69	}
70
71	static void vhost_enable_cross_endian_little(struct vhost_virtqueue *vq)
72	{
73	vq->user_be = false;
74	}
75
76	static long vhost_set_vring_endian(struct vhost_virtqueue vq, int __user argp)
77	{
78	struct vhost_vring_state s;
79
80	if (vq->private_data)
81	return -EBUSY;
82
83	if (copy_from_user(&s, argp, sizeof(s)))
84	return -EFAULT;
85
86	if (s.num != VHOST_VRING_LITTLE_ENDIAN &&
87	s.num != VHOST_VRING_BIG_ENDIAN)
88	return -EINVAL;
89
90	if (s.num == VHOST_VRING_BIG_ENDIAN)
91	vhost_enable_cross_endian_big(vq);
92	else
93	vhost_enable_cross_endian_little(vq);
94
95	return 0;
96	}
97
98	static long vhost_get_vring_endian(struct vhost_virtqueue *vq, u32 idx,
99	int __user *argp)
100	{
101	struct vhost_vring_state s = {
102	.index = idx,
103	.num = vq->user_be
104	};
105
106	if (copy_to_user(argp, &s, sizeof(s)))
107	return -EFAULT;
108
109	return 0;
110	}
111
112	static void vhost_init_is_le(struct vhost_virtqueue *vq)
113	{
114	/* Note for legacy virtio: user_be is initialized at reset time
115	* according to the host endianness. If userspace does not set an
116	* explicit endianness, the default behavior is native endian, as
117	* expected by legacy virtio.
118	*/
119	vq->is_le = vhost_has_feature(vq, VIRTIO_F_VERSION_1) \|\| !vq->user_be;
120	}
121	#else
122	static void vhost_disable_cross_endian(struct vhost_virtqueue *vq)
123	{
124	}
125
126	static long vhost_set_vring_endian(struct vhost_virtqueue vq, int __user argp)
127	{
128	return -ENOIOCTLCMD;
129	}
130
131	static long vhost_get_vring_endian(struct vhost_virtqueue *vq, u32 idx,
132	int __user *argp)
133	{
134	return -ENOIOCTLCMD;
135	}
136
137	static void vhost_init_is_le(struct vhost_virtqueue *vq)
138	{
139	vq->is_le = vhost_has_feature(vq, VIRTIO_F_VERSION_1)
140	\|\| virtio_legacy_is_little_endian();
141	}
142	#endif /* CONFIG_VHOST_CROSS_ENDIAN_LEGACY */
143
144	static void vhost_reset_is_le(struct vhost_virtqueue *vq)
145	{
146	vhost_init_is_le(vq);
147	}
148
149	struct vhost_flush_struct {
150	struct vhost_work work;
151	struct completion wait_event;
152	};
153
154	static void vhost_flush_work(struct vhost_work *work)
155	{
156	struct vhost_flush_struct *s;
157
158	s = container_of(work, struct vhost_flush_struct, work);
159	complete(&s->wait_event);
160	}
161
162	static void vhost_poll_func(struct file file, wait_queue_head_t wqh,
163	poll_table *pt)
164	{
165	struct vhost_poll *poll;
166
167	poll = container_of(pt, struct vhost_poll, table);
168	poll->wqh = wqh;
169	add_wait_queue(wqh, &poll->wait);
170	}
171
172	static int vhost_poll_wakeup(wait_queue_entry_t *wait, unsigned mode, int sync,
173	void *key)
174	{
175	struct vhost_poll *poll = container_of(wait, struct vhost_poll, wait);
176	struct vhost_work *work = &poll->work;
177
178	if (!(key_to_poll(key) & poll->mask))
179	return 0;
180
181	if (!poll->dev->use_worker)
182	work->fn(work);
183	else
184	vhost_poll_queue(poll);
185
186	return 0;
187	}
188
189	void vhost_work_init(struct vhost_work *work, vhost_work_fn_t fn)
190	{
191	clear_bit(VHOST_WORK_QUEUED, &work->flags);
192	work->fn = fn;
193	}
194	EXPORT_SYMBOL_GPL(vhost_work_init);
195
196	/* Init poll structure */
197	void vhost_poll_init(struct vhost_poll *poll, vhost_work_fn_t fn,
198	__poll_t mask, struct vhost_dev *dev,
199	struct vhost_virtqueue *vq)
200	{
201	init_waitqueue_func_entry(&poll->wait, vhost_poll_wakeup);
202	init_poll_funcptr(&poll->table, vhost_poll_func);
203	poll->mask = mask;
204	poll->dev = dev;
205	poll->wqh = NULL;
206	poll->vq = vq;
207
208	vhost_work_init(&poll->work, fn);
209	}
210	EXPORT_SYMBOL_GPL(vhost_poll_init);
211
212	/* Start polling a file. We add ourselves to file's wait queue. The caller must
213	* keep a reference to a file until after vhost_poll_stop is called. */
214	int vhost_poll_start(struct vhost_poll poll, struct file file)
215	{
216	__poll_t mask;
217
218	if (poll->wqh)
219	return 0;
220
221	mask = vfs_poll(file, &poll->table);
222	if (mask)
223	vhost_poll_wakeup(&poll->wait, 0, 0, poll_to_key(mask));
224	if (mask & EPOLLERR) {
225	vhost_poll_stop(poll);
226	return -EINVAL;
227	}
228
229	return 0;
230	}
231	EXPORT_SYMBOL_GPL(vhost_poll_start);
232
233	/* Stop polling a file. After this function returns, it becomes safe to drop the
234	* file reference. You must also flush afterwards. */
235	void vhost_poll_stop(struct vhost_poll *poll)
236	{
237	if (poll->wqh) {
238	remove_wait_queue(poll->wqh, &poll->wait);
239	poll->wqh = NULL;
240	}
241	}
242	EXPORT_SYMBOL_GPL(vhost_poll_stop);
243
244	static void vhost_worker_queue(struct vhost_worker *worker,
245	struct vhost_work *work)
246	{
247	if (!test_and_set_bit(VHOST_WORK_QUEUED, &work->flags)) {
248	/* We can only add the work to the list after we're
249	* sure it was not in the list.
250	* test_and_set_bit() implies a memory barrier.
251	*/
252	llist_add(&work->node, &worker->work_list);
253	worker->ops->wakeup(worker);
254	}
255	}
256
257	bool vhost_vq_work_queue(struct vhost_virtqueue vq, struct vhost_work work)
258	{
259	struct vhost_worker *worker;
260	bool queued = false;
261
262	rcu_read_lock();
263	worker = rcu_dereference(vq->worker);
264	if (worker) {
265	queued = true;
266	vhost_worker_queue(worker, work);
267	}
268	rcu_read_unlock();
269
270	return queued;
271	}
272	EXPORT_SYMBOL_GPL(vhost_vq_work_queue);
273
274	/**
275	* __vhost_worker_flush - flush a worker
276	* @worker: worker to flush
277	*
278	* The worker's flush_mutex must be held.
279	*/
280	static void __vhost_worker_flush(struct vhost_worker *worker)
281	{
282	struct vhost_flush_struct flush;
283
284	if (!worker->attachment_cnt \|\| worker->killed)
285	return;
286
287	init_completion(&flush.wait_event);
288	vhost_work_init(&flush.work, vhost_flush_work);
289
290	vhost_worker_queue(worker, &flush.work);
291	/*
292	* Drop mutex in case our worker is killed and it needs to take the
293	* mutex to force cleanup.
294	*/
295	mutex_unlock(&worker->mutex);
296	wait_for_completion(&flush.wait_event);
297	mutex_lock(&worker->mutex);
298	}
299
300	static void vhost_worker_flush(struct vhost_worker *worker)
301	{
302	mutex_lock(&worker->mutex);
303	__vhost_worker_flush(worker);
304	mutex_unlock(&worker->mutex);
305	}
306
307	void vhost_dev_flush(struct vhost_dev *dev)
308	{
309	struct vhost_worker *worker;
310	unsigned long i;
311
312	xa_for_each(&dev->worker_xa, i, worker)
313	vhost_worker_flush(worker);
314	}
315	EXPORT_SYMBOL_GPL(vhost_dev_flush);
316
317	/* A lockless hint for busy polling code to exit the loop */
318	bool vhost_vq_has_work(struct vhost_virtqueue *vq)
319	{
320	struct vhost_worker *worker;
321	bool has_work = false;
322
323	rcu_read_lock();
324	worker = rcu_dereference(vq->worker);
325	if (worker && !llist_empty(&worker->work_list))
326	has_work = true;
327	rcu_read_unlock();
328
329	return has_work;
330	}
331	EXPORT_SYMBOL_GPL(vhost_vq_has_work);
332
333	void vhost_poll_queue(struct vhost_poll *poll)
334	{
335	vhost_vq_work_queue(poll->vq, &poll->work);
336	}
337	EXPORT_SYMBOL_GPL(vhost_poll_queue);
338
339	static void __vhost_vq_meta_reset(struct vhost_virtqueue *vq)
340	{
341	int j;
342
343	for (j = 0; j < VHOST_NUM_ADDRS; j++)
344	vq->meta_iotlb[j] = NULL;
345	}
346
347	/* Caller must hold the virtqueue mutex. */
348	static void vhost_vq_invalidate_access(struct vhost_virtqueue *vq)
349	{
350	vq->desc = NULL;
351	vq->avail = NULL;
352	vq->used = NULL;
353	vq->log_used = false;
354	vq->log_addr = -1ull;
355	__vhost_vq_meta_reset(vq);
356	}
357
358	static void vhost_vq_meta_reset(struct vhost_dev *d)
359	{
360	int i;
361
362	for (i = 0; i < d->nvqs; ++i)
363	__vhost_vq_meta_reset(d->vqs[i]);
364	}
365
366	static void vhost_vring_call_reset(struct vhost_vring_call *call_ctx)
367	{
368	call_ctx->ctx = NULL;
369	memset(&call_ctx->producer, 0x0, sizeof(struct irq_bypass_producer));
370	}
371
372	bool vhost_vq_is_setup(struct vhost_virtqueue *vq)
373	{
374	return vq->avail && vq->desc && vq->used && vhost_vq_access_ok(vq);
375	}
376	EXPORT_SYMBOL_GPL(vhost_vq_is_setup);
377
378	static void vhost_vq_reset(struct vhost_dev *dev,
379	struct vhost_virtqueue *vq)
380	{
381	vq->num = 1;
382	vq->desc = NULL;
383	vq->avail = NULL;
384	vq->used = NULL;
385	vq->last_avail_idx = 0;
386	vq->next_avail_head = 0;
387	vq->avail_idx = 0;
388	vq->last_used_idx = 0;
389	vq->signalled_used = 0;
390	vq->signalled_used_valid = false;
391	vq->used_flags = 0;
392	vq->log_used = false;
393	vq->log_addr = -1ull;
394	vq->private_data = NULL;
395	virtio_features_zero(vq->acked_features_array);
396	vq->acked_backend_features = 0;
397	vq->log_base = NULL;
398	vq->error_ctx = NULL;
399	vq->kick = NULL;
400	vq->log_ctx = NULL;
401	vhost_disable_cross_endian(vq);
402	vhost_reset_is_le(vq);
403	vq->busyloop_timeout = 0;
404	vq->umem = NULL;
405	vq->iotlb = NULL;
406	vq->iotlb_miss = NULL;
407	rcu_assign_pointer(vq->worker, NULL);
408	vhost_vring_call_reset(&vq->call_ctx);
409	__vhost_vq_meta_reset(vq);
410	}
411
412	static int vhost_run_work_kthread_list(void *data)
413	{
414	struct vhost_worker *worker = data;
415	struct vhost_work work, work_next;
416	struct vhost_dev *dev = worker->dev;
417	struct llist_node *node;
418
419	kthread_use_mm(dev->mm);
420
421	for (;;) {
422	/* mb paired w/ kthread_stop */
423	set_current_state(TASK_INTERRUPTIBLE);
424
425	if (kthread_should_stop()) {
426	__set_current_state(TASK_RUNNING);
427	break;
428	}
429	node = llist_del_all(&worker->work_list);
430	if (!node)
431	schedule();
432
433	node = llist_reverse_order(node);
434	/* make sure flag is seen after deletion */
435	smp_wmb();
436	llist_for_each_entry_safe(work, work_next, node, node) {
437	clear_bit(VHOST_WORK_QUEUED, &work->flags);
438	__set_current_state(TASK_RUNNING);
439	kcov_remote_start_common(worker->kcov_handle);
440	work->fn(work);
441	kcov_remote_stop();
442	cond_resched();
443	}
444	}
445	kthread_unuse_mm(dev->mm);
446
447	return 0;
448	}
449
450	static bool vhost_run_work_list(void *data)
451	{
452	struct vhost_worker *worker = data;
453	struct vhost_work work, work_next;
454	struct llist_node *node;
455
456	node = llist_del_all(&worker->work_list);
457	if (node) {
458	__set_current_state(TASK_RUNNING);
459
460	node = llist_reverse_order(node);
461	/* make sure flag is seen after deletion */
462	smp_wmb();
463	llist_for_each_entry_safe(work, work_next, node, node) {
464	clear_bit(VHOST_WORK_QUEUED, &work->flags);
465	kcov_remote_start_common(worker->kcov_handle);
466	work->fn(work);
467	kcov_remote_stop();
468	cond_resched();
469	}
470	}
471
472	return !!node;
473	}
474
475	static void vhost_worker_killed(void *data)
476	{
477	struct vhost_worker *worker = data;
478	struct vhost_dev *dev = worker->dev;
479	struct vhost_virtqueue *vq;
480	int i, attach_cnt = 0;
481
482	mutex_lock(&worker->mutex);
483	worker->killed = true;
484
485	for (i = 0; i < dev->nvqs; i++) {
486	vq = dev->vqs[i];
487
488	mutex_lock(&vq->mutex);
489	if (worker ==
490	rcu_dereference_check(vq->worker,
491	lockdep_is_held(&vq->mutex))) {
492	rcu_assign_pointer(vq->worker, NULL);
493	attach_cnt++;
494	}
495	mutex_unlock(&vq->mutex);
496	}
497
498	worker->attachment_cnt -= attach_cnt;
499	if (attach_cnt)
500	synchronize_rcu();
501	/*
502	* Finish vhost_worker_flush calls and any other works that snuck in
503	* before the synchronize_rcu.
504	*/
505	vhost_run_work_list(worker);
506	mutex_unlock(&worker->mutex);
507	}
508
509	static void vhost_vq_free_iovecs(struct vhost_virtqueue *vq)
510	{
511	kfree(vq->indirect);
512	vq->indirect = NULL;
513	kfree(vq->log);
514	vq->log = NULL;
515	kfree(vq->heads);
516	vq->heads = NULL;
517	kfree(vq->nheads);
518	vq->nheads = NULL;
519	}
520
521	/* Helper to allocate iovec buffers for all vqs. */
522	static long vhost_dev_alloc_iovecs(struct vhost_dev *dev)
523	{
524	struct vhost_virtqueue *vq;
525	int i;
526
527	for (i = 0; i < dev->nvqs; ++i) {
528	vq = dev->vqs[i];
529	vq->indirect = kmalloc_objs(*vq->indirect, UIO_MAXIOV);
530	vq->log = kmalloc_objs(*vq->log, dev->iov_limit);
531	vq->heads = kmalloc_objs(*vq->heads, dev->iov_limit);
532	vq->nheads = kmalloc_array(dev->iov_limit, sizeof(*vq->nheads),
533	GFP_KERNEL);
534	if (!vq->indirect \|\| !vq->log \|\| !vq->heads \|\| !vq->nheads)
535	goto err_nomem;
536	}
537	return 0;
538
539	err_nomem:
540	for (; i >= 0; --i)
541	vhost_vq_free_iovecs(dev->vqs[i]);
542	return -ENOMEM;
543	}
544
545	static void vhost_dev_free_iovecs(struct vhost_dev *dev)
546	{
547	int i;
548
549	for (i = 0; i < dev->nvqs; ++i)
550	vhost_vq_free_iovecs(dev->vqs[i]);
551	}
552
553	bool vhost_exceeds_weight(struct vhost_virtqueue *vq,
554	int pkts, int total_len)
555	{
556	struct vhost_dev *dev = vq->dev;
557
558	if ((dev->byte_weight && total_len >= dev->byte_weight) \|\|
559	pkts >= dev->weight) {
560	vhost_poll_queue(&vq->poll);
561	return true;
562	}
563
564	return false;
565	}
566	EXPORT_SYMBOL_GPL(vhost_exceeds_weight);
567
568	static size_t vhost_get_avail_size(struct vhost_virtqueue *vq,
569	unsigned int num)
570	{
571	size_t event __maybe_unused =
572	vhost_has_feature(vq, VIRTIO_RING_F_EVENT_IDX) ? 2 : 0;
573
574	return size_add(struct_size(vq->avail, ring, num), event);
575	}
576
577	static size_t vhost_get_used_size(struct vhost_virtqueue *vq,
578	unsigned int num)
579	{
580	size_t event __maybe_unused =
581	vhost_has_feature(vq, VIRTIO_RING_F_EVENT_IDX) ? 2 : 0;
582
583	return size_add(struct_size(vq->used, ring, num), event);
584	}
585
586	static size_t vhost_get_desc_size(struct vhost_virtqueue *vq,
587	unsigned int num)
588	{
589	return sizeof(vq->desc) num;
590	}
591
592	void vhost_dev_init(struct vhost_dev *dev,
593	struct vhost_virtqueue **vqs, int nvqs,
594	int iov_limit, int weight, int byte_weight,
595	bool use_worker,
596	int (msg_handler)(struct vhost_dev dev, u32 asid,
597	struct vhost_iotlb_msg *msg))
598	{
599	struct vhost_virtqueue *vq;
600	int i;
601
602	dev->vqs = vqs;
603	dev->nvqs = nvqs;
604	mutex_init(&dev->mutex);
605	dev->log_ctx = NULL;
606	dev->umem = NULL;
607	dev->iotlb = NULL;
608	dev->mm = NULL;
609	dev->iov_limit = iov_limit;
610	dev->weight = weight;
611	dev->byte_weight = byte_weight;
612	dev->use_worker = use_worker;
613	dev->msg_handler = msg_handler;
614	dev->fork_owner = fork_from_owner_default;
615	init_waitqueue_head(&dev->wait);
616	INIT_LIST_HEAD(&dev->read_list);
617	INIT_LIST_HEAD(&dev->pending_list);
618	spin_lock_init(&dev->iotlb_lock);
619	xa_init_flags(&dev->worker_xa, XA_FLAGS_ALLOC);
620
621	for (i = 0; i < dev->nvqs; ++i) {
622	vq = dev->vqs[i];
623	vq->log = NULL;
624	vq->indirect = NULL;
625	vq->heads = NULL;
626	vq->nheads = NULL;
627	vq->dev = dev;
628	mutex_init(&vq->mutex);
629	vhost_vq_reset(dev, vq);
630	if (vq->handle_kick)
631	vhost_poll_init(&vq->poll, vq->handle_kick,
632	EPOLLIN, dev, vq);
633	}
634	}
635	EXPORT_SYMBOL_GPL(vhost_dev_init);
636
637	/* Caller should have device mutex */
638	long vhost_dev_check_owner(struct vhost_dev *dev)
639	{
640	/* Are you the owner? If not, I don't think you mean to do that */
641	return dev->mm == current->mm ? 0 : -EPERM;
642	}
643	EXPORT_SYMBOL_GPL(vhost_dev_check_owner);
644
645	struct vhost_attach_cgroups_struct {
646	struct vhost_work work;
647	struct task_struct *owner;
648	int ret;
649	};
650
651	static void vhost_attach_cgroups_work(struct vhost_work *work)
652	{
653	struct vhost_attach_cgroups_struct *s;
654
655	s = container_of(work, struct vhost_attach_cgroups_struct, work);
656	s->ret = cgroup_attach_task_all(s->owner, current);
657	}
658
659	static int vhost_attach_task_to_cgroups(struct vhost_worker *worker)
660	{
661	struct vhost_attach_cgroups_struct attach;
662	int saved_cnt;
663
664	attach.owner = current;
665
666	vhost_work_init(&attach.work, vhost_attach_cgroups_work);
667	vhost_worker_queue(worker, &attach.work);
668
669	mutex_lock(&worker->mutex);
670
671	/*
672	* Bypass attachment_cnt check in __vhost_worker_flush:
673	* Temporarily change it to INT_MAX to bypass the check
674	*/
675	saved_cnt = worker->attachment_cnt;
676	worker->attachment_cnt = INT_MAX;
677	__vhost_worker_flush(worker);
678	worker->attachment_cnt = saved_cnt;
679
680	mutex_unlock(&worker->mutex);
681
682	return attach.ret;
683	}
684
685	/* Caller should have device mutex */
686	bool vhost_dev_has_owner(struct vhost_dev *dev)
687	{
688	return dev->mm;
689	}
690	EXPORT_SYMBOL_GPL(vhost_dev_has_owner);
691
692	static void vhost_attach_mm(struct vhost_dev *dev)
693	{
694	/* No owner, become one */
695	if (dev->use_worker) {
696	dev->mm = get_task_mm(current);
697	} else {
698	/* vDPA device does not use worker thread, so there's
699	* no need to hold the address space for mm. This helps
700	* to avoid deadlock in the case of mmap() which may
701	* hold the refcnt of the file and depends on release
702	* method to remove vma.
703	*/
704	dev->mm = current->mm;
705	mmgrab(dev->mm);
706	}
707	}
708
709	static void vhost_detach_mm(struct vhost_dev *dev)
710	{
711	if (!dev->mm)
712	return;
713
714	if (dev->use_worker)
715	mmput(dev->mm);
716	else
717	mmdrop(dev->mm);
718
719	dev->mm = NULL;
720	}
721
722	static void vhost_worker_destroy(struct vhost_dev *dev,
723	struct vhost_worker *worker)
724	{
725	if (!worker)
726	return;
727
728	WARN_ON(!llist_empty(&worker->work_list));
729	xa_erase(&dev->worker_xa, worker->id);
730	worker->ops->stop(worker);
731	kfree(worker);
732	}
733
734	static void vhost_workers_free(struct vhost_dev *dev)
735	{
736	struct vhost_worker *worker;
737	unsigned long i;
738
739	if (!dev->use_worker)
740	return;
741
742	for (i = 0; i < dev->nvqs; i++)
743	rcu_assign_pointer(dev->vqs[i]->worker, NULL);
744	/*
745	* Free the default worker we created and cleanup workers userspace
746	* created but couldn't clean up (it forgot or crashed).
747	*/
748	xa_for_each(&dev->worker_xa, i, worker)
749	vhost_worker_destroy(dev, worker);
750	xa_destroy(&dev->worker_xa);
751	}
752
753	static void vhost_task_wakeup(struct vhost_worker *worker)
754	{
755	return vhost_task_wake(worker->vtsk);
756	}
757
758	static void vhost_kthread_wakeup(struct vhost_worker *worker)
759	{
760	wake_up_process(worker->kthread_task);
761	}
762
763	static void vhost_task_do_stop(struct vhost_worker *worker)
764	{
765	return vhost_task_stop(worker->vtsk);
766	}
767
768	static void vhost_kthread_do_stop(struct vhost_worker *worker)
769	{
770	kthread_stop(worker->kthread_task);
771	}
772
773	static int vhost_task_worker_create(struct vhost_worker *worker,
774	struct vhost_dev dev, const char name)
775	{
776	struct vhost_task *vtsk;
777	u32 id;
778	int ret;
779
780	vtsk = vhost_task_create(vhost_run_work_list, vhost_worker_killed,
781	worker, name);
782	if (IS_ERR(vtsk))
783	return PTR_ERR(vtsk);
784
785	worker->vtsk = vtsk;
786	vhost_task_start(vtsk);
787	ret = xa_alloc(&dev->worker_xa, &id, worker, xa_limit_32b, GFP_KERNEL);
788	if (ret < 0) {
789	vhost_task_do_stop(worker);
790	return ret;
791	}
792	worker->id = id;
793	return 0;
794	}
795
796	static int vhost_kthread_worker_create(struct vhost_worker *worker,
797	struct vhost_dev dev, const char name)
798	{
799	struct task_struct *task;
800	u32 id;
801	int ret;
802
803	task = kthread_create(vhost_run_work_kthread_list, worker, "%s", name);
804	if (IS_ERR(task))
805	return PTR_ERR(task);
806
807	worker->kthread_task = task;
808	wake_up_process(task);
809	ret = xa_alloc(&dev->worker_xa, &id, worker, xa_limit_32b, GFP_KERNEL);
810	if (ret < 0)
811	goto stop_worker;
812
813	ret = vhost_attach_task_to_cgroups(worker);
814	if (ret)
815	goto free_id;
816
817	worker->id = id;
818	return 0;
819
820	free_id:
821	xa_erase(&dev->worker_xa, id);
822	stop_worker:
823	vhost_kthread_do_stop(worker);
824	return ret;
825	}
826
827	static const struct vhost_worker_ops kthread_ops = {
828	.create = vhost_kthread_worker_create,
829	.stop = vhost_kthread_do_stop,
830	.wakeup = vhost_kthread_wakeup,
831	};
832
833	static const struct vhost_worker_ops vhost_task_ops = {
834	.create = vhost_task_worker_create,
835	.stop = vhost_task_do_stop,
836	.wakeup = vhost_task_wakeup,
837	};
838
839	static struct vhost_worker vhost_worker_create(struct vhost_dev dev)
840	{
841	struct vhost_worker *worker;
842	char name[TASK_COMM_LEN];
843	int ret;
844	const struct vhost_worker_ops *ops = dev->fork_owner ? &vhost_task_ops :
845	&kthread_ops;
846
847	worker = kzalloc_obj(*worker, GFP_KERNEL_ACCOUNT);
848	if (!worker)
849	return NULL;
850
851	worker->dev = dev;
852	worker->ops = ops;
853	snprintf(name, sizeof(name), "vhost-%d", current->pid);
854
855	mutex_init(&worker->mutex);
856	init_llist_head(&worker->work_list);
857	worker->kcov_handle = kcov_common_handle();
858	ret = ops->create(worker, dev, name);
859	if (ret < 0)
860	goto free_worker;
861
862	return worker;
863
864	free_worker:
865	kfree(worker);
866	return NULL;
867	}
868
869	/* Caller must have device mutex */
870	static void __vhost_vq_attach_worker(struct vhost_virtqueue *vq,
871	struct vhost_worker *worker)
872	{
873	struct vhost_worker *old_worker;
874
875	mutex_lock(&worker->mutex);
876	if (worker->killed) {
877	mutex_unlock(&worker->mutex);
878	return;
879	}
880
881	mutex_lock(&vq->mutex);
882
883	old_worker = rcu_dereference_check(vq->worker,
884	lockdep_is_held(&vq->mutex));
885	rcu_assign_pointer(vq->worker, worker);
886	worker->attachment_cnt++;
887
888	if (!old_worker) {
889	mutex_unlock(&vq->mutex);
890	mutex_unlock(&worker->mutex);
891	return;
892	}
893	mutex_unlock(&vq->mutex);
894	mutex_unlock(&worker->mutex);
895
896	/*
897	* Take the worker mutex to make sure we see the work queued from
898	* device wide flushes which doesn't use RCU for execution.
899	*/
900	mutex_lock(&old_worker->mutex);
901	if (old_worker->killed) {
902	mutex_unlock(&old_worker->mutex);
903	return;
904	}
905
906	/*
907	* We don't want to call synchronize_rcu for every vq during setup
908	* because it will slow down VM startup. If we haven't done
909	* VHOST_SET_VRING_KICK and not done the driver specific
910	* SET_ENDPOINT/RUNNING then we can skip the sync since there will
911	* not be any works queued for scsi and net.
912	*/
913	mutex_lock(&vq->mutex);
914	if (!vhost_vq_get_backend(vq) && !vq->kick) {
915	mutex_unlock(&vq->mutex);
916
917	old_worker->attachment_cnt--;
918	mutex_unlock(&old_worker->mutex);
919	/*
920	* vsock can queue anytime after VHOST_VSOCK_SET_GUEST_CID.
921	* Warn if it adds support for multiple workers but forgets to
922	* handle the early queueing case.
923	*/
924	WARN_ON(!old_worker->attachment_cnt &&
925	!llist_empty(&old_worker->work_list));
926	return;
927	}
928	mutex_unlock(&vq->mutex);
929
930	/* Make sure new vq queue/flush/poll calls see the new worker */
931	synchronize_rcu();
932	/* Make sure whatever was queued gets run */
933	__vhost_worker_flush(old_worker);
934	old_worker->attachment_cnt--;
935	mutex_unlock(&old_worker->mutex);
936	}
937
938	/* Caller must have device mutex */
939	static int vhost_vq_attach_worker(struct vhost_virtqueue *vq,
940	struct vhost_vring_worker *info)
941	{
942	unsigned long index = info->worker_id;
943	struct vhost_dev *dev = vq->dev;
944	struct vhost_worker *worker;
945
946	if (!dev->use_worker)
947	return -EINVAL;
948
949	worker = xa_find(&dev->worker_xa, &index, UINT_MAX, XA_PRESENT);
950	if (!worker \|\| worker->id != info->worker_id)
951	return -ENODEV;
952
953	__vhost_vq_attach_worker(vq, worker);
954	return 0;
955	}
956
957	/* Caller must have device mutex */
958	static int vhost_new_worker(struct vhost_dev *dev,
959	struct vhost_worker_state *info)
960	{
961	struct vhost_worker *worker;
962
963	worker = vhost_worker_create(dev);
964	if (!worker)
965	return -ENOMEM;
966
967	info->worker_id = worker->id;
968	return 0;
969	}
970
971	/* Caller must have device mutex */
972	static int vhost_free_worker(struct vhost_dev *dev,
973	struct vhost_worker_state *info)
974	{
975	unsigned long index = info->worker_id;
976	struct vhost_worker *worker;
977
978	worker = xa_find(&dev->worker_xa, &index, UINT_MAX, XA_PRESENT);
979	if (!worker \|\| worker->id != info->worker_id)
980	return -ENODEV;
981
982	mutex_lock(&worker->mutex);
983	if (worker->attachment_cnt \|\| worker->killed) {
984	mutex_unlock(&worker->mutex);
985	return -EBUSY;
986	}
987	/*
988	* A flush might have raced and snuck in before attachment_cnt was set
989	* to zero. Make sure flushes are flushed from the queue before
990	* freeing.
991	*/
992	__vhost_worker_flush(worker);
993	mutex_unlock(&worker->mutex);
994
995	vhost_worker_destroy(dev, worker);
996	return 0;
997	}
998
999	static int vhost_get_vq_from_user(struct vhost_dev dev, void __user argp,
1000	struct vhost_virtqueue *vq, u32 id)
1001	{
1002	u32 __user *idxp = argp;
1003	u32 idx;
1004	long r;
1005
1006	r = get_user(idx, idxp);
1007	if (r < 0)
1008	return r;
1009
1010	if (idx >= dev->nvqs)
1011	return -ENOBUFS;
1012
1013	idx = array_index_nospec(idx, dev->nvqs);
1014
1015	*vq = dev->vqs[idx];
1016	*id = idx;
1017	return 0;
1018	}
1019
1020	/* Caller must have device mutex */
1021	long vhost_worker_ioctl(struct vhost_dev *dev, unsigned int ioctl,
1022	void __user *argp)
1023	{
1024	struct vhost_vring_worker ring_worker;
1025	struct vhost_worker_state state;
1026	struct vhost_worker *worker;
1027	struct vhost_virtqueue *vq;
1028	long ret;
1029	u32 idx;
1030
1031	if (!dev->use_worker)
1032	return -EINVAL;
1033
1034	if (!vhost_dev_has_owner(dev))
1035	return -EINVAL;
1036
1037	ret = vhost_dev_check_owner(dev);
1038	if (ret)
1039	return ret;
1040
1041	switch (ioctl) {
1042	/* dev worker ioctls */
1043	case VHOST_NEW_WORKER:
1044	/*
1045	* vhost_tasks will account for worker threads under the parent's
1046	* NPROC value but kthreads do not. To avoid userspace overflowing
1047	* the system with worker threads fork_owner must be true.
1048	*/
1049	if (!dev->fork_owner)
1050	return -EFAULT;
1051
1052	ret = vhost_new_worker(dev, &state);
1053	if (!ret && copy_to_user(argp, &state, sizeof(state)))
1054	ret = -EFAULT;
1055	return ret;
1056	case VHOST_FREE_WORKER:
1057	if (copy_from_user(&state, argp, sizeof(state)))
1058	return -EFAULT;
1059	return vhost_free_worker(dev, &state);
1060	/* vring worker ioctls */
1061	case VHOST_ATTACH_VRING_WORKER:
1062	case VHOST_GET_VRING_WORKER:
1063	break;
1064	default:
1065	return -ENOIOCTLCMD;
1066	}
1067
1068	ret = vhost_get_vq_from_user(dev, argp, &vq, &idx);
1069	if (ret)
1070	return ret;
1071
1072	switch (ioctl) {
1073	case VHOST_ATTACH_VRING_WORKER:
1074	if (copy_from_user(&ring_worker, argp, sizeof(ring_worker))) {
1075	ret = -EFAULT;
1076	break;
1077	}
1078
1079	ret = vhost_vq_attach_worker(vq, &ring_worker);
1080	break;
1081	case VHOST_GET_VRING_WORKER:
1082	worker = rcu_dereference_check(vq->worker,
1083	lockdep_is_held(&dev->mutex));
1084	if (!worker) {
1085	ret = -EINVAL;
1086	break;
1087	}
1088
1089	ring_worker.index = idx;
1090	ring_worker.worker_id = worker->id;
1091
1092	if (copy_to_user(argp, &ring_worker, sizeof(ring_worker)))
1093	ret = -EFAULT;
1094	break;
1095	default:
1096	ret = -ENOIOCTLCMD;
1097	break;
1098	}
1099
1100	return ret;
1101	}
1102	EXPORT_SYMBOL_GPL(vhost_worker_ioctl);
1103
1104	/* Caller should have device mutex */
1105	long vhost_dev_set_owner(struct vhost_dev *dev)
1106	{
1107	struct vhost_worker *worker;
1108	int err, i;
1109
1110	/* Is there an owner already? */
1111	if (vhost_dev_has_owner(dev)) {
1112	err = -EBUSY;
1113	goto err_mm;
1114	}
1115
1116	vhost_attach_mm(dev);
1117
1118	err = vhost_dev_alloc_iovecs(dev);
1119	if (err)
1120	goto err_iovecs;
1121
1122	if (dev->use_worker) {
1123	/*
1124	* This should be done last, because vsock can queue work
1125	* before VHOST_SET_OWNER so it simplifies the failure path
1126	* below since we don't have to worry about vsock queueing
1127	* while we free the worker.
1128	*/
1129	worker = vhost_worker_create(dev);
1130	if (!worker) {
1131	err = -ENOMEM;
1132	goto err_worker;
1133	}
1134
1135	for (i = 0; i < dev->nvqs; i++)
1136	__vhost_vq_attach_worker(dev->vqs[i], worker);
1137	}
1138
1139	return 0;
1140
1141	err_worker:
1142	vhost_dev_free_iovecs(dev);
1143	err_iovecs:
1144	vhost_detach_mm(dev);
1145	err_mm:
1146	return err;
1147	}
1148	EXPORT_SYMBOL_GPL(vhost_dev_set_owner);
1149
1150	static struct vhost_iotlb *iotlb_alloc(void)
1151	{
1152	if (max_iotlb_entries <= 0)
1153	return NULL;
1154
1155	return vhost_iotlb_alloc(max_iotlb_entries,
1156	VHOST_IOTLB_FLAG_RETIRE);
1157	}
1158
1159	struct vhost_iotlb *vhost_dev_reset_owner_prepare(void)
1160	{
1161	return iotlb_alloc();
1162	}
1163	EXPORT_SYMBOL_GPL(vhost_dev_reset_owner_prepare);
1164
1165	/* Caller should have device mutex */
1166	void vhost_dev_reset_owner(struct vhost_dev dev, struct vhost_iotlb umem)
1167	{
1168	int i;
1169
1170	vhost_dev_cleanup(dev);
1171
1172	dev->fork_owner = fork_from_owner_default;
1173	dev->umem = umem;
1174	/* We don't need VQ locks below since vhost_dev_cleanup makes sure
1175	* VQs aren't running.
1176	*/
1177	for (i = 0; i < dev->nvqs; ++i)
1178	dev->vqs[i]->umem = umem;
1179	}
1180	EXPORT_SYMBOL_GPL(vhost_dev_reset_owner);
1181
1182	void vhost_dev_stop(struct vhost_dev *dev)
1183	{
1184	int i;
1185
1186	for (i = 0; i < dev->nvqs; ++i) {
1187	if (dev->vqs[i]->kick && dev->vqs[i]->handle_kick)
1188	vhost_poll_stop(&dev->vqs[i]->poll);
1189	}
1190
1191	vhost_dev_flush(dev);
1192	}
1193	EXPORT_SYMBOL_GPL(vhost_dev_stop);
1194
1195	static void vhost_free_msg_locked(struct vhost_msg_node *node)
1196	{
1197	if (node->vq->iotlb_miss == node)
1198	node->vq->iotlb_miss = NULL;
1199	kfree(node);
1200	}
1201
1202	static void vhost_free_msg(struct vhost_dev *dev,
1203	struct vhost_msg_node *node)
1204	{
1205	spin_lock(&dev->iotlb_lock);
1206	vhost_free_msg_locked(node);
1207	spin_unlock(&dev->iotlb_lock);
1208	}
1209
1210	void vhost_clear_msg(struct vhost_dev *dev)
1211	{
1212	struct vhost_msg_node node, n;
1213
1214	spin_lock(&dev->iotlb_lock);
1215
1216	list_for_each_entry_safe(node, n, &dev->read_list, node) {
1217	list_del(&node->node);
1218	vhost_free_msg_locked(node);
1219	}
1220
1221	list_for_each_entry_safe(node, n, &dev->pending_list, node) {
1222	list_del(&node->node);
1223	vhost_free_msg_locked(node);
1224	}
1225
1226	spin_unlock(&dev->iotlb_lock);
1227	}
1228	EXPORT_SYMBOL_GPL(vhost_clear_msg);
1229
1230	void vhost_dev_cleanup(struct vhost_dev *dev)
1231	{
1232	int i;
1233
1234	for (i = 0; i < dev->nvqs; ++i) {
1235	if (dev->vqs[i]->error_ctx)
1236	eventfd_ctx_put(dev->vqs[i]->error_ctx);
1237	if (dev->vqs[i]->kick)
1238	fput(dev->vqs[i]->kick);
1239	if (dev->vqs[i]->call_ctx.ctx)
1240	eventfd_ctx_put(dev->vqs[i]->call_ctx.ctx);
1241	vhost_vq_reset(dev, dev->vqs[i]);
1242	}
1243	vhost_dev_free_iovecs(dev);
1244	if (dev->log_ctx)
1245	eventfd_ctx_put(dev->log_ctx);
1246	dev->log_ctx = NULL;
1247	/* No one will access memory at this point */
1248	vhost_iotlb_free(dev->umem);
1249	dev->umem = NULL;
1250	vhost_iotlb_free(dev->iotlb);
1251	dev->iotlb = NULL;
1252	vhost_clear_msg(dev);
1253	wake_up_interruptible_poll(&dev->wait, EPOLLIN \| EPOLLRDNORM);
1254	vhost_workers_free(dev);
1255	vhost_detach_mm(dev);
1256	}
1257	EXPORT_SYMBOL_GPL(vhost_dev_cleanup);
1258
1259	static bool log_access_ok(void __user *log_base, u64 addr, unsigned long sz)
1260	{
1261	u64 a = addr / VHOST_PAGE_SIZE / 8;
1262
1263	/* Make sure 64 bit math will not overflow. */
1264	if (a > ULONG_MAX - (unsigned long)log_base \|\|
1265	a + (unsigned long)log_base > ULONG_MAX)
1266	return false;
1267
1268	return access_ok(log_base + a,
1269	(sz + VHOST_PAGE_SIZE * 8 - 1) / VHOST_PAGE_SIZE / 8);
1270	}
1271
1272	/* Make sure 64 bit math will not overflow. */
1273	static bool vhost_overflow(u64 uaddr, u64 size)
1274	{
1275	if (uaddr > ULONG_MAX \|\| size > ULONG_MAX)
1276	return true;
1277
1278	if (!size)
1279	return false;
1280
1281	return uaddr > ULONG_MAX - size + 1;
1282	}
1283
1284	/* Caller should have vq mutex and device mutex. */
1285	static bool vq_memory_access_ok(void __user log_base, struct vhost_iotlb umem,
1286	int log_all)
1287	{
1288	struct vhost_iotlb_map *map;
1289
1290	if (!umem)
1291	return false;
1292
1293	list_for_each_entry(map, &umem->list, link) {
1294	unsigned long a = map->addr;
1295
1296	if (vhost_overflow(map->addr, map->size))
1297	return false;
1298
1299
1300	if (!access_ok((void __user *)a, map->size))
1301	return false;
1302	else if (log_all && !log_access_ok(log_base,
1303	map->start,
1304	map->size))
1305	return false;
1306	}
1307	return true;
1308	}
1309
1310	static inline void __user vhost_vq_meta_fetch(struct vhost_virtqueue vq,
1311	u64 addr, unsigned int size,
1312	int type)
1313	{
1314	const struct vhost_iotlb_map *map = vq->meta_iotlb[type];
1315
1316	if (!map)
1317	return NULL;
1318
1319	return (void __user *)(uintptr_t)(map->addr + addr - map->start);
1320	}
1321
1322	/* Can we switch to this memory table? */
1323	/* Caller should have device mutex but not vq mutex */
1324	static bool memory_access_ok(struct vhost_dev d, struct vhost_iotlb umem,
1325	int log_all)
1326	{
1327	int i;
1328
1329	for (i = 0; i < d->nvqs; ++i) {
1330	bool ok;
1331	bool log;
1332
1333	mutex_lock(&d->vqs[i]->mutex);
1334	log = log_all \|\| vhost_has_feature(d->vqs[i], VHOST_F_LOG_ALL);
1335	/* If ring is inactive, will check when it's enabled. */
1336	if (d->vqs[i]->private_data)
1337	ok = vq_memory_access_ok(d->vqs[i]->log_base,
1338	umem, log);
1339	else
1340	ok = true;
1341	mutex_unlock(&d->vqs[i]->mutex);
1342	if (!ok)
1343	return false;
1344	}
1345	return true;
1346	}
1347
1348	static int translate_desc(struct vhost_virtqueue *vq, u64 addr, u32 len,
1349	struct iovec iov[], int iov_size, int access);
1350
1351	static int vhost_copy_to_user(struct vhost_virtqueue vq, void __user to,
1352	const void *from, unsigned size)
1353	{
1354	int ret;
1355
1356	if (!vq->iotlb)
1357	return __copy_to_user(to, from, size);
1358	else {
1359	/* This function should be called after iotlb
1360	* prefetch, which means we're sure that all vq
1361	* could be access through iotlb. So -EAGAIN should
1362	* not happen in this case.
1363	*/
1364	struct iov_iter t;
1365	void __user *uaddr = vhost_vq_meta_fetch(vq,
1366	(u64)(uintptr_t)to, size,
1367	VHOST_ADDR_USED);
1368
1369	if (uaddr)
1370	return __copy_to_user(uaddr, from, size);
1371
1372	ret = translate_desc(vq, (u64)(uintptr_t)to, size, vq->iotlb_iov,
1373	ARRAY_SIZE(vq->iotlb_iov),
1374	VHOST_ACCESS_WO);
1375	if (ret < 0)
1376	goto out;
1377	iov_iter_init(&t, ITER_DEST, vq->iotlb_iov, ret, size);
1378	ret = copy_to_iter(from, size, &t);
1379	if (ret == size)
1380	ret = 0;
1381	}
1382	out:
1383	return ret;
1384	}
1385
1386	static int vhost_copy_from_user(struct vhost_virtqueue vq, void to,
1387	void __user *from, unsigned size)
1388	{
1389	int ret;
1390
1391	if (!vq->iotlb)
1392	return __copy_from_user(to, from, size);
1393	else {
1394	/* This function should be called after iotlb
1395	* prefetch, which means we're sure that vq
1396	* could be access through iotlb. So -EAGAIN should
1397	* not happen in this case.
1398	*/
1399	void __user *uaddr = vhost_vq_meta_fetch(vq,
1400	(u64)(uintptr_t)from, size,
1401	VHOST_ADDR_DESC);
1402	struct iov_iter f;
1403
1404	if (uaddr)
1405	return __copy_from_user(to, uaddr, size);
1406
1407	ret = translate_desc(vq, (u64)(uintptr_t)from, size, vq->iotlb_iov,
1408	ARRAY_SIZE(vq->iotlb_iov),
1409	VHOST_ACCESS_RO);
1410	if (ret < 0) {
1411	vq_err(vq, "IOTLB translation failure: uaddr "
1412	"%p size 0x%llx\n", from,
1413	(unsigned long long) size);
1414	goto out;
1415	}
1416	iov_iter_init(&f, ITER_SOURCE, vq->iotlb_iov, ret, size);
1417	ret = copy_from_iter(to, size, &f);
1418	if (ret == size)
1419	ret = 0;
1420	}
1421
1422	out:
1423	return ret;
1424	}
1425
1426	static void __user __vhost_get_user_slow(struct vhost_virtqueue vq,
1427	void __user *addr, unsigned int size,
1428	int type)
1429	{
1430	int ret;
1431
1432	ret = translate_desc(vq, (u64)(uintptr_t)addr, size, vq->iotlb_iov,
1433	ARRAY_SIZE(vq->iotlb_iov),
1434	VHOST_ACCESS_RO);
1435	if (ret < 0) {
1436	vq_err(vq, "IOTLB translation failure: uaddr "
1437	"%p size 0x%llx\n", addr,
1438	(unsigned long long) size);
1439	return NULL;
1440	}
1441
1442	if (ret != 1 \|\| vq->iotlb_iov[0].iov_len != size) {
1443	vq_err(vq, "Non atomic userspace memory access: uaddr "
1444	"%p size 0x%llx\n", addr,
1445	(unsigned long long) size);
1446	return NULL;
1447	}
1448
1449	return vq->iotlb_iov[0].iov_base;
1450	}
1451
1452	/* This function should be called after iotlb
1453	* prefetch, which means we're sure that vq
1454	* could be access through iotlb. So -EAGAIN should
1455	* not happen in this case.
1456	*/
1457	static inline void __user __vhost_get_user(struct vhost_virtqueue vq,
1458	void __user *addr, unsigned int size,
1459	int type)
1460	{
1461	void __user *uaddr = vhost_vq_meta_fetch(vq,
1462	(u64)(uintptr_t)addr, size, type);
1463	if (uaddr)
1464	return uaddr;
1465
1466	return __vhost_get_user_slow(vq, addr, size, type);
1467	}
1468
1469	#define vhost_put_user(vq, x, ptr) \
1470	({ \
1471	int ret; \
1472	if (!vq->iotlb) { \
1473	ret = put_user(x, ptr); \
1474	} else { \
1475	__typeof__(ptr) to = \
1476	(__typeof__(ptr)) __vhost_get_user(vq, ptr, \
1477	sizeof(*ptr), VHOST_ADDR_USED); \
1478	if (to != NULL) \
1479	ret = put_user(x, to); \
1480	else \
1481	ret = -EFAULT; \
1482	} \
1483	ret; \
1484	})
1485
1486	static inline int vhost_put_avail_event(struct vhost_virtqueue *vq)
1487	{
1488	return vhost_put_user(vq, cpu_to_vhost16(vq, vq->avail_idx),
1489	vhost_avail_event(vq));
1490	}
1491
1492	static inline int vhost_put_used(struct vhost_virtqueue *vq,
1493	struct vring_used_elem *head, int idx,
1494	int count)
1495	{
1496	return vhost_copy_to_user(vq, vq->used->ring + idx, head,
1497	count * sizeof(*head));
1498	}
1499
1500	static inline int vhost_put_used_flags(struct vhost_virtqueue *vq)
1501
1502	{
1503	return vhost_put_user(vq, cpu_to_vhost16(vq, vq->used_flags),
1504	&vq->used->flags);
1505	}
1506
1507	static inline int vhost_put_used_idx(struct vhost_virtqueue *vq)
1508
1509	{
1510	return vhost_put_user(vq, cpu_to_vhost16(vq, vq->last_used_idx),
1511	&vq->used->idx);
1512	}
1513
1514	#define vhost_get_user(vq, x, ptr, type) \
1515	({ \
1516	int ret; \
1517	if (!vq->iotlb) { \
1518	ret = get_user(x, ptr); \
1519	} else { \
1520	__typeof__(ptr) from = \
1521	(__typeof__(ptr)) __vhost_get_user(vq, ptr, \
1522	sizeof(*ptr), \
1523	type); \
1524	if (from != NULL) \
1525	ret = get_user(x, from); \
1526	else \
1527	ret = -EFAULT; \
1528	} \
1529	ret; \
1530	})
1531
1532	#define vhost_get_avail(vq, x, ptr) \
1533	vhost_get_user(vq, x, ptr, VHOST_ADDR_AVAIL)
1534
1535	#define vhost_get_used(vq, x, ptr) \
1536	vhost_get_user(vq, x, ptr, VHOST_ADDR_USED)
1537
1538	static void vhost_dev_lock_vqs(struct vhost_dev *d)
1539	{
1540	int i = 0;
1541	for (i = 0; i < d->nvqs; ++i)
1542	mutex_lock_nested(&d->vqs[i]->mutex, i);
1543	}
1544
1545	static void vhost_dev_unlock_vqs(struct vhost_dev *d)
1546	{
1547	int i = 0;
1548	for (i = 0; i < d->nvqs; ++i)
1549	mutex_unlock(&d->vqs[i]->mutex);
1550	}
1551
1552	static inline int vhost_get_avail_idx(struct vhost_virtqueue *vq)
1553	{
1554	__virtio16 idx;
1555	u16 avail_idx;
1556	int r;
1557
1558	r = vhost_get_avail(vq, idx, &vq->avail->idx);
1559	if (unlikely(r < 0)) {
1560	vq_err(vq, "Failed to access available index at %p (%d)\n",
1561	&vq->avail->idx, r);
1562	return r;
1563	}
1564
1565	/* Check it isn't doing very strange thing with available indexes */
1566	avail_idx = vhost16_to_cpu(vq, idx);
1567	if (unlikely((u16)(avail_idx - vq->last_avail_idx) > vq->num)) {
1568	vq_err(vq, "Invalid available index change from %u to %u",
1569	vq->last_avail_idx, avail_idx);
1570	return -EINVAL;
1571	}
1572
1573	/* We're done if there is nothing new */
1574	if (avail_idx == vq->avail_idx)
1575	return 0;
1576
1577	vq->avail_idx = avail_idx;
1578
1579	/*
1580	* We updated vq->avail_idx so we need a memory barrier between
1581	* the index read above and the caller reading avail ring entries.
1582	*/
1583	smp_rmb();
1584	return 1;
1585	}
1586
1587	static inline int vhost_get_avail_head(struct vhost_virtqueue *vq,
1588	__virtio16 *head, int idx)
1589	{
1590	return vhost_get_avail(vq, *head,
1591	&vq->avail->ring[idx & (vq->num - 1)]);
1592	}
1593
1594	static inline int vhost_get_avail_flags(struct vhost_virtqueue *vq,
1595	__virtio16 *flags)
1596	{
1597	return vhost_get_avail(vq, *flags, &vq->avail->flags);
1598	}
1599
1600	static inline int vhost_get_used_event(struct vhost_virtqueue *vq,
1601	__virtio16 *event)
1602	{
1603	return vhost_get_avail(vq, *event, vhost_used_event(vq));
1604	}
1605
1606	static inline int vhost_get_used_idx(struct vhost_virtqueue *vq,
1607	__virtio16 *idx)
1608	{
1609	return vhost_get_used(vq, *idx, &vq->used->idx);
1610	}
1611
1612	static inline int vhost_get_desc(struct vhost_virtqueue *vq,
1613	struct vring_desc *desc, int idx)
1614	{
1615	return vhost_copy_from_user(vq, desc, vq->desc + idx, sizeof(*desc));
1616	}
1617
1618	static void vhost_iotlb_notify_vq(struct vhost_dev *d,
1619	struct vhost_iotlb_msg *msg)
1620	{
1621	struct vhost_msg_node node, n;
1622
1623	spin_lock(&d->iotlb_lock);
1624
1625	list_for_each_entry_safe(node, n, &d->pending_list, node) {
1626	struct vhost_iotlb_msg *vq_msg = &node->msg.iotlb;
1627	if (msg->iova <= vq_msg->iova &&
1628	msg->iova + msg->size - 1 >= vq_msg->iova &&
1629	vq_msg->type == VHOST_IOTLB_MISS) {
1630	vhost_poll_queue(&node->vq->poll);
1631	list_del(&node->node);
1632	vhost_free_msg_locked(node);
1633	}
1634	}
1635
1636	spin_unlock(&d->iotlb_lock);
1637	}
1638
1639	static bool umem_access_ok(u64 uaddr, u64 size, int access)
1640	{
1641	unsigned long a = uaddr;
1642
1643	/* Make sure 64 bit math will not overflow. */
1644	if (vhost_overflow(uaddr, size))
1645	return false;
1646
1647	if ((access & VHOST_ACCESS_RO) &&
1648	!access_ok((void __user *)a, size))
1649	return false;
1650	if ((access & VHOST_ACCESS_WO) &&
1651	!access_ok((void __user *)a, size))
1652	return false;
1653	return true;
1654	}
1655
1656	static int vhost_process_iotlb_msg(struct vhost_dev *dev, u32 asid,
1657	struct vhost_iotlb_msg *msg)
1658	{
1659	int ret = 0;
1660
1661	if (asid != 0)
1662	return -EINVAL;
1663
1664	mutex_lock(&dev->mutex);
1665	vhost_dev_lock_vqs(dev);
1666	switch (msg->type) {
1667	case VHOST_IOTLB_UPDATE:
1668	if (!dev->iotlb) {
1669	ret = -EFAULT;
1670	break;
1671	}
1672	if (!umem_access_ok(msg->uaddr, msg->size, msg->perm)) {
1673	ret = -EFAULT;
1674	break;
1675	}
1676	vhost_vq_meta_reset(dev);
1677	if (vhost_iotlb_add_range(dev->iotlb, msg->iova,
1678	msg->iova + msg->size - 1,
1679	msg->uaddr, msg->perm)) {
1680	ret = -ENOMEM;
1681	break;
1682	}
1683	vhost_iotlb_notify_vq(dev, msg);
1684	break;
1685	case VHOST_IOTLB_INVALIDATE:
1686	if (!dev->iotlb) {
1687	ret = -EFAULT;
1688	break;
1689	}
1690	if (!msg->size) {
1691	ret = -EINVAL;
1692	break;
1693	}
1694	vhost_vq_meta_reset(dev);
1695	vhost_iotlb_del_range(dev->iotlb, msg->iova,
1696	msg->iova + msg->size - 1);
1697	break;
1698	default:
1699	ret = -EINVAL;
1700	break;
1701	}
1702
1703	vhost_dev_unlock_vqs(dev);
1704	mutex_unlock(&dev->mutex);
1705
1706	return ret;
1707	}
1708	ssize_t vhost_chr_write_iter(struct vhost_dev *dev,
1709	struct iov_iter *from)
1710	{
1711	struct vhost_iotlb_msg msg;
1712	size_t offset;
1713	int type, ret;
1714	u32 asid = 0;
1715
1716	ret = copy_from_iter(&type, sizeof(type), from);
1717	if (ret != sizeof(type)) {
1718	ret = -EINVAL;
1719	goto done;
1720	}
1721
1722	switch (type) {
1723	case VHOST_IOTLB_MSG:
1724	/* There maybe a hole after type for V1 message type,
1725	* so skip it here.
1726	*/
1727	offset = offsetof(struct vhost_msg, iotlb) - sizeof(int);
1728	break;
1729	case VHOST_IOTLB_MSG_V2:
1730	if (vhost_backend_has_feature(dev->vqs[0],
1731	VHOST_BACKEND_F_IOTLB_ASID)) {
1732	ret = copy_from_iter(&asid, sizeof(asid), from);
1733	if (ret != sizeof(asid)) {
1734	ret = -EINVAL;
1735	goto done;
1736	}
1737	offset = 0;
1738	} else
1739	offset = sizeof(__u32);
1740	break;
1741	default:
1742	ret = -EINVAL;
1743	goto done;
1744	}
1745
1746	iov_iter_advance(from, offset);
1747	ret = copy_from_iter(&msg, sizeof(msg), from);
1748	if (ret != sizeof(msg)) {
1749	ret = -EINVAL;
1750	goto done;
1751	}
1752
1753	if (msg.type == VHOST_IOTLB_UPDATE && msg.size == 0) {
1754	ret = -EINVAL;
1755	goto done;
1756	}
1757
1758	if (dev->msg_handler)
1759	ret = dev->msg_handler(dev, asid, &msg);
1760	else
1761	ret = vhost_process_iotlb_msg(dev, asid, &msg);
1762	if (ret) {
1763	ret = -EFAULT;
1764	goto done;
1765	}
1766
1767	ret = (type == VHOST_IOTLB_MSG) ? sizeof(struct vhost_msg) :
1768	sizeof(struct vhost_msg_v2);
1769	done:
1770	return ret;
1771	}
1772	EXPORT_SYMBOL(vhost_chr_write_iter);
1773
1774	__poll_t vhost_chr_poll(struct file file, struct vhost_dev dev,
1775	poll_table *wait)
1776	{
1777	__poll_t mask = 0;
1778
1779	poll_wait(file, &dev->wait, wait);
1780
1781	if (!list_empty(&dev->read_list))
1782	mask \|= EPOLLIN \| EPOLLRDNORM;
1783
1784	return mask;
1785	}
1786	EXPORT_SYMBOL(vhost_chr_poll);
1787
1788	ssize_t vhost_chr_read_iter(struct vhost_dev dev, struct iov_iter to,
1789	int noblock)
1790	{
1791	DEFINE_WAIT(wait);
1792	struct vhost_msg_node *node;
1793	ssize_t ret = 0;
1794	unsigned size = sizeof(struct vhost_msg);
1795
1796	if (iov_iter_count(to) < size)
1797	return 0;
1798
1799	while (1) {
1800	if (!noblock)
1801	prepare_to_wait(&dev->wait, &wait,
1802	TASK_INTERRUPTIBLE);
1803
1804	node = vhost_dequeue_msg(dev, &dev->read_list);
1805	if (node)
1806	break;
1807	if (noblock) {
1808	ret = -EAGAIN;
1809	break;
1810	}
1811	if (signal_pending(current)) {
1812	ret = -ERESTARTSYS;
1813	break;
1814	}
1815	if (!dev->iotlb) {
1816	ret = -EBADFD;
1817	break;
1818	}
1819
1820	schedule();
1821	}
1822
1823	if (!noblock)
1824	finish_wait(&dev->wait, &wait);
1825
1826	if (node) {
1827	struct vhost_iotlb_msg *msg;
1828	void *start = &node->msg;
1829
1830	switch (node->msg.type) {
1831	case VHOST_IOTLB_MSG:
1832	size = sizeof(node->msg);
1833	msg = &node->msg.iotlb;
1834	break;
1835	case VHOST_IOTLB_MSG_V2:
1836	size = sizeof(node->msg_v2);
1837	msg = &node->msg_v2.iotlb;
1838	break;
1839	default:
1840	BUG();
1841	break;
1842	}
1843
1844	ret = copy_to_iter(start, size, to);
1845	if (ret != size \|\| msg->type != VHOST_IOTLB_MISS) {
1846	vhost_free_msg(dev, node);
1847	return ret;
1848	}
1849	vhost_enqueue_msg(dev, &dev->pending_list, node);
1850	}
1851
1852	return ret;
1853	}
1854	EXPORT_SYMBOL_GPL(vhost_chr_read_iter);
1855
1856	static int vhost_iotlb_miss(struct vhost_virtqueue *vq, u64 iova, int access)
1857	{
1858	struct vhost_dev *dev = vq->dev;
1859	struct vhost_msg_node *node;
1860	struct vhost_iotlb_msg *msg;
1861	bool v2 = vhost_backend_has_feature(vq, VHOST_BACKEND_F_IOTLB_MSG_V2);
1862
1863	node = vhost_new_msg(vq, v2 ? VHOST_IOTLB_MSG_V2 : VHOST_IOTLB_MSG);
1864	if (!node)
1865	return -ENOMEM;
1866
1867	if (v2) {
1868	node->msg_v2.type = VHOST_IOTLB_MSG_V2;
1869	msg = &node->msg_v2.iotlb;
1870	} else {
1871	msg = &node->msg.iotlb;
1872	}
1873
1874	msg->type = VHOST_IOTLB_MISS;
1875	msg->iova = iova;
1876	msg->perm = access;
1877
1878	spin_lock(&dev->iotlb_lock);
1879	/* VQ processing stops at the first miss until userspace resolves it. */
1880	if (vq->iotlb_miss) {
1881	spin_unlock(&dev->iotlb_lock);
1882	kfree(node);
1883	return 0;
1884	}
1885
1886	vq->iotlb_miss = node;
1887	list_add_tail(&node->node, &dev->read_list);
1888	spin_unlock(&dev->iotlb_lock);
1889
1890	wake_up_interruptible_poll(&dev->wait, EPOLLIN \| EPOLLRDNORM);
1891
1892	return 0;
1893	}
1894
1895	static bool vq_access_ok(struct vhost_virtqueue *vq, unsigned int num,
1896	vring_desc_t __user *desc,
1897	vring_avail_t __user *avail,
1898	vring_used_t __user *used)
1899
1900	{
1901	/* If an IOTLB device is present, the vring addresses are
1902	* GIOVAs. Access validation occurs at prefetch time. */
1903	if (vq->iotlb)
1904	return true;
1905
1906	return access_ok(desc, vhost_get_desc_size(vq, num)) &&
1907	access_ok(avail, vhost_get_avail_size(vq, num)) &&
1908	access_ok(used, vhost_get_used_size(vq, num));
1909	}
1910
1911	static void vhost_vq_meta_update(struct vhost_virtqueue *vq,
1912	const struct vhost_iotlb_map *map,
1913	int type)
1914	{
1915	int access = (type == VHOST_ADDR_USED) ?
1916	VHOST_ACCESS_WO : VHOST_ACCESS_RO;
1917
1918	if (likely(map->perm & access))
1919	vq->meta_iotlb[type] = map;
1920	}
1921
1922	static bool iotlb_access_ok(struct vhost_virtqueue *vq,
1923	int access, u64 addr, u64 len, int type)
1924	{
1925	const struct vhost_iotlb_map *map;
1926	struct vhost_iotlb *umem = vq->iotlb;
1927	u64 s = 0, size, orig_addr = addr, last = addr + len - 1;
1928
1929	if (vhost_vq_meta_fetch(vq, addr, len, type))
1930	return true;
1931
1932	while (len > s) {
1933	map = vhost_iotlb_itree_first(umem, addr, last);
1934	if (map == NULL \|\| map->start > addr) {
1935	vhost_iotlb_miss(vq, addr, access);
1936	return false;
1937	} else if (!(map->perm & access)) {
1938	/* Report the possible access violation by
1939	* request another translation from userspace.
1940	*/
1941	return false;
1942	}
1943
1944	size = map->size - addr + map->start;
1945
1946	if (orig_addr == addr && size >= len)
1947	vhost_vq_meta_update(vq, map, type);
1948
1949	s += size;
1950	addr += size;
1951	}
1952
1953	return true;
1954	}
1955
1956	int vq_meta_prefetch(struct vhost_virtqueue *vq)
1957	{
1958	unsigned int num = vq->num;
1959
1960	/*
1961	* vhost_vq_invalidate_access() clears all three addresses together.
1962	* A single zero address may be a valid GIOVA in IOTLB mode.
1963	*/
1964	if (!vq->desc && !vq->avail && !vq->used)
1965	return 0;
1966
1967	if (!vq->iotlb)
1968	return 1;
1969
1970	return iotlb_access_ok(vq, VHOST_MAP_RO, (u64)(uintptr_t)vq->desc,
1971	vhost_get_desc_size(vq, num), VHOST_ADDR_DESC) &&
1972	iotlb_access_ok(vq, VHOST_MAP_RO, (u64)(uintptr_t)vq->avail,
1973	vhost_get_avail_size(vq, num),
1974	VHOST_ADDR_AVAIL) &&
1975	iotlb_access_ok(vq, VHOST_MAP_WO, (u64)(uintptr_t)vq->used,
1976	vhost_get_used_size(vq, num), VHOST_ADDR_USED);
1977	}
1978	EXPORT_SYMBOL_GPL(vq_meta_prefetch);
1979
1980	/* Can we log writes? */
1981	/* Caller should have device mutex but not vq mutex */
1982	bool vhost_log_access_ok(struct vhost_dev *dev)
1983	{
1984	return memory_access_ok(dev, dev->umem, 1);
1985	}
1986	EXPORT_SYMBOL_GPL(vhost_log_access_ok);
1987
1988	static bool vq_log_used_access_ok(struct vhost_virtqueue *vq,
1989	void __user *log_base,
1990	bool log_used,
1991	u64 log_addr)
1992	{
1993	/* If an IOTLB device is present, log_addr is a GIOVA that
1994	* will never be logged by log_used(). */
1995	if (vq->iotlb)
1996	return true;
1997
1998	return !log_used \|\| log_access_ok(log_base, log_addr,
1999	vhost_get_used_size(vq, vq->num));
2000	}
2001
2002	/* Verify access for write logging. */
2003	/* Caller should have vq mutex and device mutex */
2004	static bool vq_log_access_ok(struct vhost_virtqueue *vq,
2005	void __user *log_base)
2006	{
2007	return vq_memory_access_ok(log_base, vq->umem,
2008	vhost_has_feature(vq, VHOST_F_LOG_ALL)) &&
2009	vq_log_used_access_ok(vq, log_base, vq->log_used, vq->log_addr);
2010	}
2011
2012	/* Can we start vq? */
2013	/* Caller should have vq mutex and device mutex */
2014	bool vhost_vq_access_ok(struct vhost_virtqueue *vq)
2015	{
2016	if (!vq_log_access_ok(vq, vq->log_base))
2017	return false;
2018
2019	return vq_access_ok(vq, vq->num, vq->desc, vq->avail, vq->used);
2020	}
2021	EXPORT_SYMBOL_GPL(vhost_vq_access_ok);
2022
2023	static long vhost_set_memory(struct vhost_dev d, struct vhost_memory __user m)
2024	{
2025	struct vhost_memory mem, *newmem;
2026	struct vhost_memory_region *region;
2027	struct vhost_iotlb newumem, oldumem;
2028	unsigned long size = offsetof(struct vhost_memory, regions);
2029	int i;
2030
2031	if (copy_from_user(&mem, m, size))
2032	return -EFAULT;
2033	if (mem.padding)
2034	return -EOPNOTSUPP;
2035	if (mem.nregions > max_mem_regions)
2036	return -E2BIG;
2037	if (max_iotlb_entries <= 0)
2038	return -EINVAL;
2039	newmem = kvzalloc_flex(*newmem, regions, mem.nregions);
2040	if (!newmem)
2041	return -ENOMEM;
2042
2043	memcpy(newmem, &mem, size);
2044	if (copy_from_user(newmem->regions, m->regions,
2045	flex_array_size(newmem, regions, mem.nregions))) {
2046	kvfree(newmem);
2047	return -EFAULT;
2048	}
2049
2050	newumem = iotlb_alloc();
2051	if (!newumem) {
2052	kvfree(newmem);
2053	return -ENOMEM;
2054	}
2055
2056	for (region = newmem->regions;
2057	region < newmem->regions + mem.nregions;
2058	region++) {
2059	if (vhost_iotlb_add_range(newumem,
2060	region->guest_phys_addr,
2061	region->guest_phys_addr +
2062	region->memory_size - 1,
2063	region->userspace_addr,
2064	VHOST_MAP_RW))
2065	goto err;
2066	}
2067
2068	if (!memory_access_ok(d, newumem, 0))
2069	goto err;
2070
2071	oldumem = d->umem;
2072	d->umem = newumem;
2073
2074	/* All memory accesses are done under some VQ mutex. */
2075	for (i = 0; i < d->nvqs; ++i) {
2076	mutex_lock(&d->vqs[i]->mutex);
2077	d->vqs[i]->umem = newumem;
2078	mutex_unlock(&d->vqs[i]->mutex);
2079	}
2080
2081	kvfree(newmem);
2082	vhost_iotlb_free(oldumem);
2083	return 0;
2084
2085	err:
2086	vhost_iotlb_free(newumem);
2087	kvfree(newmem);
2088	return -EFAULT;
2089	}
2090
2091	static long vhost_vring_set_num(struct vhost_dev *d,
2092	struct vhost_virtqueue *vq,
2093	void __user *argp)
2094	{
2095	struct vhost_vring_state s;
2096
2097	/* Resizing ring with an active backend?
2098	* You don't want to do that. */
2099	if (vq->private_data)
2100	return -EBUSY;
2101
2102	if (copy_from_user(&s, argp, sizeof s))
2103	return -EFAULT;
2104
2105	if (!s.num \|\| s.num > 0xffff \|\| (s.num & (s.num - 1)))
2106	return -EINVAL;
2107	vq->num = s.num;
2108
2109	return 0;
2110	}
2111
2112	static long vhost_vring_set_addr(struct vhost_dev *d,
2113	struct vhost_virtqueue *vq,
2114	void __user *argp)
2115	{
2116	struct vhost_vring_addr a;
2117
2118	if (copy_from_user(&a, argp, sizeof a))
2119	return -EFAULT;
2120	if (a.flags & ~(0x1 << VHOST_VRING_F_LOG))
2121	return -EOPNOTSUPP;
2122
2123	/* For 32bit, verify that the top 32bits of the user
2124	data are set to zero. */
2125	if ((u64)(unsigned long)a.desc_user_addr != a.desc_user_addr \|\|
2126	(u64)(unsigned long)a.used_user_addr != a.used_user_addr \|\|
2127	(u64)(unsigned long)a.avail_user_addr != a.avail_user_addr)
2128	return -EFAULT;
2129
2130	/* Make sure it's safe to cast pointers to vring types. */
2131	BUILD_BUG_ON(__alignof__ *vq->avail > VRING_AVAIL_ALIGN_SIZE);
2132	BUILD_BUG_ON(__alignof__ *vq->used > VRING_USED_ALIGN_SIZE);
2133	if ((a.avail_user_addr & (VRING_AVAIL_ALIGN_SIZE - 1)) \|\|
2134	(a.used_user_addr & (VRING_USED_ALIGN_SIZE - 1)) \|\|
2135	(a.log_guest_addr & (VRING_USED_ALIGN_SIZE - 1)))
2136	return -EINVAL;
2137
2138	/* We only verify access here if backend is configured.
2139	* If it is not, we don't as size might not have been setup.
2140	* We will verify when backend is configured. */
2141	if (vq->private_data) {
2142	if (!vq_access_ok(vq, vq->num,
2143	(void __user *)(unsigned long)a.desc_user_addr,
2144	(void __user *)(unsigned long)a.avail_user_addr,
2145	(void __user *)(unsigned long)a.used_user_addr))
2146	return -EINVAL;
2147
2148	/* Also validate log access for used ring if enabled. */
2149	if (!vq_log_used_access_ok(vq, vq->log_base,
2150	a.flags & (0x1 << VHOST_VRING_F_LOG),
2151	a.log_guest_addr))
2152	return -EINVAL;
2153	}
2154
2155	vq->log_used = !!(a.flags & (0x1 << VHOST_VRING_F_LOG));
2156	vq->desc = (void __user *)(unsigned long)a.desc_user_addr;
2157	vq->avail = (void __user *)(unsigned long)a.avail_user_addr;
2158	vq->log_addr = a.log_guest_addr;
2159	vq->used = (void __user *)(unsigned long)a.used_user_addr;
2160
2161	return 0;
2162	}
2163
2164	static long vhost_vring_set_num_addr(struct vhost_dev *d,
2165	struct vhost_virtqueue *vq,
2166	unsigned int ioctl,
2167	void __user *argp)
2168	{
2169	long r;
2170
2171	mutex_lock(&vq->mutex);
2172
2173	switch (ioctl) {
2174	case VHOST_SET_VRING_NUM:
2175	r = vhost_vring_set_num(d, vq, argp);
2176	break;
2177	case VHOST_SET_VRING_ADDR:
2178	r = vhost_vring_set_addr(d, vq, argp);
2179	break;
2180	default:
2181	BUG();
2182	}
2183
2184	/*
2185	* The metadata cache holds the IOTLB mapping that backed the previous
2186	* desc/avail/used addresses and vring size, both of which are being
2187	* replaced here. iotlb_access_ok() takes a cache hit as proof that the
2188	* region was validated, so the stale entries have to go.
2189	*/
2190	__vhost_vq_meta_reset(vq);
2191
2192	mutex_unlock(&vq->mutex);
2193
2194	return r;
2195	}
2196	long vhost_vring_ioctl(struct vhost_dev d, unsigned int ioctl, void __user argp)
2197	{
2198	struct file eventfp, filep = NULL;
2199	bool pollstart = false, pollstop = false;
2200	struct eventfd_ctx *ctx = NULL;
2201	struct vhost_virtqueue *vq;
2202	struct vhost_vring_state s;
2203	struct vhost_vring_file f;
2204	u32 idx;
2205	long r;
2206
2207	r = vhost_get_vq_from_user(d, argp, &vq, &idx);
2208	if (r < 0)
2209	return r;
2210
2211	if (ioctl == VHOST_SET_VRING_NUM \|\|
2212	ioctl == VHOST_SET_VRING_ADDR) {
2213	return vhost_vring_set_num_addr(d, vq, ioctl, argp);
2214	}
2215
2216	mutex_lock(&vq->mutex);
2217
2218	switch (ioctl) {
2219	case VHOST_SET_VRING_BASE:
2220	/* Moving base with an active backend?
2221	* You don't want to do that. */
2222	if (vq->private_data) {
2223	r = -EBUSY;
2224	break;
2225	}
2226	if (copy_from_user(&s, argp, sizeof s)) {
2227	r = -EFAULT;
2228	break;
2229	}
2230	if (vhost_has_feature(vq, VIRTIO_F_RING_PACKED)) {
2231	vq->next_avail_head = vq->last_avail_idx =
2232	s.num & 0xffff;
2233	vq->last_used_idx = (s.num >> 16) & 0xffff;
2234	} else {
2235	if (s.num > 0xffff) {
2236	r = -EINVAL;
2237	break;
2238	}
2239	vq->next_avail_head = vq->last_avail_idx = s.num;
2240	}
2241	/* Forget the cached index value. */
2242	vq->avail_idx = vq->last_avail_idx;
2243	break;
2244	case VHOST_GET_VRING_BASE:
2245	s.index = idx;
2246	if (vhost_has_feature(vq, VIRTIO_F_RING_PACKED))
2247	s.num = (u32)vq->last_avail_idx \| ((u32)vq->last_used_idx << 16);
2248	else
2249	s.num = vq->last_avail_idx;
2250	if (copy_to_user(argp, &s, sizeof s))
2251	r = -EFAULT;
2252	break;
2253	case VHOST_SET_VRING_KICK:
2254	if (copy_from_user(&f, argp, sizeof f)) {
2255	r = -EFAULT;
2256	break;
2257	}
2258	eventfp = f.fd == VHOST_FILE_UNBIND ? NULL : eventfd_fget(f.fd);
2259	if (IS_ERR(eventfp)) {
2260	r = PTR_ERR(eventfp);
2261	break;
2262	}
2263	if (eventfp != vq->kick) {
2264	pollstop = (filep = vq->kick) != NULL;
2265	pollstart = (vq->kick = eventfp) != NULL;
2266	} else
2267	filep = eventfp;
2268	break;
2269	case VHOST_SET_VRING_CALL:
2270	if (copy_from_user(&f, argp, sizeof f)) {
2271	r = -EFAULT;
2272	break;
2273	}
2274	ctx = f.fd == VHOST_FILE_UNBIND ? NULL : eventfd_ctx_fdget(f.fd);
2275	if (IS_ERR(ctx)) {
2276	r = PTR_ERR(ctx);
2277	break;
2278	}
2279
2280	swap(ctx, vq->call_ctx.ctx);
2281	break;
2282	case VHOST_SET_VRING_ERR:
2283	if (copy_from_user(&f, argp, sizeof f)) {
2284	r = -EFAULT;
2285	break;
2286	}
2287	ctx = f.fd == VHOST_FILE_UNBIND ? NULL : eventfd_ctx_fdget(f.fd);
2288	if (IS_ERR(ctx)) {
2289	r = PTR_ERR(ctx);
2290	break;
2291	}
2292	swap(ctx, vq->error_ctx);
2293	break;
2294	case VHOST_SET_VRING_ENDIAN:
2295	r = vhost_set_vring_endian(vq, argp);
2296	break;
2297	case VHOST_GET_VRING_ENDIAN:
2298	r = vhost_get_vring_endian(vq, idx, argp);
2299	break;
2300	case VHOST_SET_VRING_BUSYLOOP_TIMEOUT:
2301	if (copy_from_user(&s, argp, sizeof(s))) {
2302	r = -EFAULT;
2303	break;
2304	}
2305	vq->busyloop_timeout = s.num;
2306	break;
2307	case VHOST_GET_VRING_BUSYLOOP_TIMEOUT:
2308	s.index = idx;
2309	s.num = vq->busyloop_timeout;
2310	if (copy_to_user(argp, &s, sizeof(s)))
2311	r = -EFAULT;
2312	break;
2313	default:
2314	r = -ENOIOCTLCMD;
2315	}
2316
2317	if (pollstop && vq->handle_kick)
2318	vhost_poll_stop(&vq->poll);
2319
2320	if (!IS_ERR_OR_NULL(ctx))
2321	eventfd_ctx_put(ctx);
2322	if (filep)
2323	fput(filep);
2324
2325	if (pollstart && vq->handle_kick)
2326	r = vhost_poll_start(&vq->poll, vq->kick);
2327
2328	mutex_unlock(&vq->mutex);
2329
2330	if (pollstop && vq->handle_kick)
2331	vhost_dev_flush(vq->poll.dev);
2332	return r;
2333	}
2334	EXPORT_SYMBOL_GPL(vhost_vring_ioctl);
2335
2336	/* Caller must hold the device mutex. */
2337	void vhost_clear_device_iotlb(struct vhost_dev *d)
2338	{
2339	struct vhost_iotlb *iotlb;
2340	int i;
2341
2342	iotlb = d->iotlb;
2343	if (!iotlb)
2344	return;
2345
2346	vhost_dev_lock_vqs(d);
2347
2348	/*
2349	* vhost_dev_lock_vqs() takes all VQ mutexes in index order. Drop the
2350	* device-wide view while they are held, then clear each per-VQ view
2351	* and its cached ring access before releasing the locks. Workers
2352	* cannot observe a mixed address-space state during this handoff.
2353	*/
2354	d->iotlb = NULL;
2355
2356	for (i = 0; i < d->nvqs; ++i) {
2357	struct vhost_virtqueue *vq = d->vqs[i];
2358
2359	vq->iotlb = NULL;
2360	vhost_vq_invalidate_access(vq);
2361	}
2362
2363	vhost_dev_unlock_vqs(d);
2364	vhost_clear_msg(d);
2365	vhost_iotlb_free(iotlb);
2366	wake_up_interruptible_poll(&d->wait, EPOLLIN \| EPOLLRDNORM);
2367	}
2368	EXPORT_SYMBOL_GPL(vhost_clear_device_iotlb);
2369
2370	int vhost_init_device_iotlb(struct vhost_dev *d)
2371	{
2372	struct vhost_iotlb niotlb, oiotlb;
2373	int i;
2374
2375	if (max_iotlb_entries <= 0)
2376	return -EINVAL;
2377
2378	niotlb = iotlb_alloc();
2379	if (!niotlb)
2380	return -ENOMEM;
2381
2382	oiotlb = d->iotlb;
2383	d->iotlb = niotlb;
2384
2385	for (i = 0; i < d->nvqs; ++i) {
2386	struct vhost_virtqueue *vq = d->vqs[i];
2387
2388	mutex_lock(&vq->mutex);
2389	vq->iotlb = niotlb;
2390	if (oiotlb)
2391	__vhost_vq_meta_reset(vq);
2392	else
2393	vhost_vq_invalidate_access(vq);
2394	mutex_unlock(&vq->mutex);
2395	}
2396
2397	vhost_iotlb_free(oiotlb);
2398
2399	return 0;
2400	}
2401	EXPORT_SYMBOL_GPL(vhost_init_device_iotlb);
2402
2403	/* Caller must have device mutex */
2404	long vhost_dev_ioctl(struct vhost_dev d, unsigned int ioctl, void __user argp)
2405	{
2406	struct eventfd_ctx *ctx;
2407	u64 p;
2408	long r;
2409	int i, fd;
2410
2411	/* If you are not the owner, you can become one */
2412	if (ioctl == VHOST_SET_OWNER) {
2413	r = vhost_dev_set_owner(d);
2414	goto done;
2415	}
2416
2417	#ifdef CONFIG_VHOST_ENABLE_FORK_OWNER_CONTROL
2418	if (ioctl == VHOST_SET_FORK_FROM_OWNER) {
2419	/* Only allow modification before owner is set */
2420	if (vhost_dev_has_owner(d)) {
2421	r = -EBUSY;
2422	goto done;
2423	}
2424	u8 fork_owner_val;
2425
2426	if (get_user(fork_owner_val, (u8 __user *)argp)) {
2427	r = -EFAULT;
2428	goto done;
2429	}
2430	if (fork_owner_val != VHOST_FORK_OWNER_TASK &&
2431	fork_owner_val != VHOST_FORK_OWNER_KTHREAD) {
2432	r = -EINVAL;
2433	goto done;
2434	}
2435	d->fork_owner = !!fork_owner_val;
2436	r = 0;
2437	goto done;
2438	}
2439	if (ioctl == VHOST_GET_FORK_FROM_OWNER) {
2440	u8 fork_owner_val = d->fork_owner;
2441
2442	if (fork_owner_val != VHOST_FORK_OWNER_TASK &&
2443	fork_owner_val != VHOST_FORK_OWNER_KTHREAD) {
2444	r = -EINVAL;
2445	goto done;
2446	}
2447	if (put_user(fork_owner_val, (u8 __user *)argp)) {
2448	r = -EFAULT;
2449	goto done;
2450	}
2451	r = 0;
2452	goto done;
2453	}
2454	#endif
2455
2456	/* You must be the owner to do anything else */
2457	r = vhost_dev_check_owner(d);
2458	if (r)
2459	goto done;
2460
2461	switch (ioctl) {
2462	case VHOST_SET_MEM_TABLE:
2463	r = vhost_set_memory(d, argp);
2464	break;
2465	case VHOST_SET_LOG_BASE:
2466	if (copy_from_user(&p, argp, sizeof p)) {
2467	r = -EFAULT;
2468	break;
2469	}
2470	if ((u64)(unsigned long)p != p) {
2471	r = -EFAULT;
2472	break;
2473	}
2474	for (i = 0; i < d->nvqs; ++i) {
2475	struct vhost_virtqueue *vq;
2476	void __user base = (void __user )(unsigned long)p;
2477	vq = d->vqs[i];
2478	mutex_lock(&vq->mutex);
2479	/* If ring is inactive, will check when it's enabled. */
2480	if (vq->private_data && !vq_log_access_ok(vq, base))
2481	r = -EFAULT;
2482	else
2483	vq->log_base = base;
2484	mutex_unlock(&vq->mutex);
2485	}
2486	break;
2487	case VHOST_SET_LOG_FD:
2488	r = get_user(fd, (int __user *)argp);
2489	if (r < 0)
2490	break;
2491	ctx = fd == VHOST_FILE_UNBIND ? NULL : eventfd_ctx_fdget(fd);
2492	if (IS_ERR(ctx)) {
2493	r = PTR_ERR(ctx);
2494	break;
2495	}
2496	swap(ctx, d->log_ctx);
2497	for (i = 0; i < d->nvqs; ++i) {
2498	mutex_lock(&d->vqs[i]->mutex);
2499	d->vqs[i]->log_ctx = d->log_ctx;
2500	mutex_unlock(&d->vqs[i]->mutex);
2501	}
2502	if (ctx)
2503	eventfd_ctx_put(ctx);
2504	break;
2505	default:
2506	r = -ENOIOCTLCMD;
2507	break;
2508	}
2509	done:
2510	return r;
2511	}
2512	EXPORT_SYMBOL_GPL(vhost_dev_ioctl);
2513
2514	/* TODO: This is really inefficient. We need something like get_user()
2515	* (instruction directly accesses the data, with an exception table entry
2516	* returning -EFAULT). See Documentation/arch/x86/exception-tables.rst.
2517	*/
2518	static int set_bit_to_user(int nr, void __user *addr)
2519	{
2520	unsigned long log = (unsigned long)addr;
2521	struct page *page;
2522	void *base;
2523	int bit = nr + (log % PAGE_SIZE) * 8;
2524	int r;
2525
2526	r = pin_user_pages_fast(log, 1, FOLL_WRITE, &page);
2527	if (r < 0)
2528	return r;
2529	BUG_ON(r != 1);
2530	base = kmap_atomic(page);
2531	set_bit(bit, base);
2532	kunmap_atomic(base);
2533	unpin_user_pages_dirty_lock(&page, 1, true);
2534	return 0;
2535	}
2536
2537	static int log_write(void __user *log_base,
2538	u64 write_address, u64 write_length)
2539	{
2540	u64 write_page = write_address / VHOST_PAGE_SIZE;
2541	int r;
2542
2543	if (!write_length)
2544	return 0;
2545	write_length += write_address % VHOST_PAGE_SIZE;
2546	for (;;) {
2547	u64 base = (u64)(unsigned long)log_base;
2548	u64 log = base + write_page / 8;
2549	int bit = write_page % 8;
2550	if ((u64)(unsigned long)log != log)
2551	return -EFAULT;
2552	r = set_bit_to_user(bit, (void __user *)(unsigned long)log);
2553	if (r < 0)
2554	return r;
2555	if (write_length <= VHOST_PAGE_SIZE)
2556	break;
2557	write_length -= VHOST_PAGE_SIZE;
2558	write_page += 1;
2559	}
2560	return r;
2561	}
2562
2563	static int log_write_hva(struct vhost_virtqueue *vq, u64 hva, u64 len)
2564	{
2565	struct vhost_iotlb *umem = vq->umem;
2566	struct vhost_iotlb_map *u;
2567	u64 start, end, l, min;
2568	int r;
2569	bool hit = false;
2570
2571	while (len) {
2572	min = len;
2573	/* More than one GPAs can be mapped into a single HVA. So
2574	* iterate all possible umems here to be safe.
2575	*/
2576	list_for_each_entry(u, &umem->list, link) {
2577	if (u->addr > hva - 1 + len \|\|
2578	u->addr - 1 + u->size < hva)
2579	continue;
2580	start = max(u->addr, hva);
2581	end = min(u->addr - 1 + u->size, hva - 1 + len);
2582	l = end - start + 1;
2583	r = log_write(vq->log_base,
2584	u->start + start - u->addr,
2585	l);
2586	if (r < 0)
2587	return r;
2588	hit = true;
2589	min = min(l, min);
2590	}
2591
2592	if (!hit)
2593	return -EFAULT;
2594
2595	len -= min;
2596	hva += min;
2597	}
2598
2599	return 0;
2600	}
2601
2602	static int log_used(struct vhost_virtqueue *vq, u64 used_offset, u64 len)
2603	{
2604	struct iovec *iov = vq->log_iov;
2605	int i, ret;
2606
2607	if (!vq->iotlb)
2608	return log_write(vq->log_base, vq->log_addr + used_offset, len);
2609
2610	ret = translate_desc(vq, (uintptr_t)vq->used + used_offset,
2611	len, iov, 64, VHOST_ACCESS_WO);
2612	if (ret < 0)
2613	return ret;
2614
2615	for (i = 0; i < ret; i++) {
2616	ret = log_write_hva(vq, (uintptr_t)iov[i].iov_base,
2617	iov[i].iov_len);
2618	if (ret)
2619	return ret;
2620	}
2621
2622	return 0;
2623	}
2624
2625	/*
2626	* vhost_log_write() - Log in dirty page bitmap
2627	* @vq: vhost virtqueue.
2628	* @log: Array of dirty memory in GPA.
2629	* @log_num: Size of vhost_log arrary.
2630	* @len: The total length of memory buffer to log in the dirty bitmap.
2631	* Some drivers may only partially use pages shared via the last
2632	* vring descriptor (i.e. vhost-net RX buffer).
2633	* Use (len == U64_MAX) to indicate the driver would log all
2634	* pages of vring descriptors.
2635	* @iov: Array of dirty memory in HVA.
2636	* @count: Size of iovec array.
2637	*/
2638	int vhost_log_write(struct vhost_virtqueue vq, struct vhost_log log,
2639	unsigned int log_num, u64 len, struct iovec *iov, int count)
2640	{
2641	int i, r;
2642
2643	/* Make sure data written is seen before log. */
2644	smp_wmb();
2645
2646	if (vq->iotlb) {
2647	for (i = 0; i < count; i++) {
2648	r = log_write_hva(vq, (uintptr_t)iov[i].iov_base,
2649	iov[i].iov_len);
2650	if (r < 0)
2651	return r;
2652	}
2653	return 0;
2654	}
2655
2656	for (i = 0; i < log_num; ++i) {
2657	u64 l = min(log[i].len, len);
2658	r = log_write(vq->log_base, log[i].addr, l);
2659	if (r < 0)
2660	return r;
2661
2662	if (len != U64_MAX)
2663	len -= l;
2664	}
2665
2666	if (vq->log_ctx)
2667	eventfd_signal(vq->log_ctx);
2668
2669	return 0;
2670	}
2671	EXPORT_SYMBOL_GPL(vhost_log_write);
2672
2673	static int vhost_update_used_flags(struct vhost_virtqueue *vq)
2674	{
2675	void __user *used;
2676	if (vhost_put_used_flags(vq))
2677	return -EFAULT;
2678	if (unlikely(vq->log_used)) {
2679	/* Make sure the flag is seen before log. */
2680	smp_wmb();
2681	/* Log used flag write. */
2682	used = &vq->used->flags;
2683	log_used(vq, (used - (void __user *)vq->used),
2684	sizeof vq->used->flags);
2685	if (vq->log_ctx)
2686	eventfd_signal(vq->log_ctx);
2687	}
2688	return 0;
2689	}
2690
2691	static int vhost_update_avail_event(struct vhost_virtqueue *vq)
2692	{
2693	if (vhost_put_avail_event(vq))
2694	return -EFAULT;
2695	if (unlikely(vq->log_used)) {
2696	void __user *used;
2697	/* Make sure the event is seen before log. */
2698	smp_wmb();
2699	/* Log avail event write */
2700	used = vhost_avail_event(vq);
2701	log_used(vq, (used - (void __user *)vq->used),
2702	sizeof *vhost_avail_event(vq));
2703	if (vq->log_ctx)
2704	eventfd_signal(vq->log_ctx);
2705	}
2706	return 0;
2707	}
2708
2709	int vhost_vq_init_access(struct vhost_virtqueue *vq)
2710	{
2711	__virtio16 last_used_idx;
2712	int r;
2713	bool is_le = vq->is_le;
2714
2715	if (!vq->private_data)
2716	return 0;
2717
2718	vhost_init_is_le(vq);
2719
2720	r = vhost_update_used_flags(vq);
2721	if (r)
2722	goto err;
2723	vq->signalled_used_valid = false;
2724	if (!vq->iotlb &&
2725	!access_ok(&vq->used->idx, sizeof vq->used->idx)) {
2726	r = -EFAULT;
2727	goto err;
2728	}
2729	r = vhost_get_used_idx(vq, &last_used_idx);
2730	if (r) {
2731	vq_err(vq, "Can't access used idx at %p\n",
2732	&vq->used->idx);
2733	goto err;
2734	}
2735	vq->last_used_idx = vhost16_to_cpu(vq, last_used_idx);
2736	return 0;
2737
2738	err:
2739	vq->is_le = is_le;
2740	return r;
2741	}
2742	EXPORT_SYMBOL_GPL(vhost_vq_init_access);
2743
2744	static int translate_desc(struct vhost_virtqueue *vq, u64 addr, u32 len,
2745	struct iovec iov[], int iov_size, int access)
2746	{
2747	const struct vhost_iotlb_map *map;
2748	struct vhost_dev *dev = vq->dev;
2749	struct vhost_iotlb *umem = dev->iotlb ? dev->iotlb : dev->umem;
2750	struct iovec *_iov;
2751	u64 s = 0, last = addr + len - 1;
2752	int ret = 0;
2753
2754	while ((u64)len > s) {
2755	u64 size;
2756	if (unlikely(ret >= iov_size)) {
2757	ret = -ENOBUFS;
2758	break;
2759	}
2760
2761	map = vhost_iotlb_itree_first(umem, addr, last);
2762	if (map == NULL \|\| map->start > addr) {
2763	if (umem != dev->iotlb) {
2764	ret = -EFAULT;
2765	break;
2766	}
2767	ret = -EAGAIN;
2768	break;
2769	} else if (!(map->perm & access)) {
2770	ret = -EPERM;
2771	break;
2772	}
2773
2774	_iov = iov + ret;
2775	size = map->size - addr + map->start;
2776	_iov->iov_len = min((u64)len - s, size);
2777	_iov->iov_base = (void __user *)(unsigned long)
2778	(map->addr + addr - map->start);
2779	s += size;
2780	addr += size;
2781	++ret;
2782	}
2783
2784	if (ret == -EAGAIN)
2785	vhost_iotlb_miss(vq, addr, access);
2786	return ret;
2787	}
2788
2789	/* Each buffer in the virtqueues is actually a chain of descriptors. This
2790	* function returns the next descriptor in the chain,
2791	* or -1U if we're at the end. */
2792	static unsigned next_desc(struct vhost_virtqueue vq, struct vring_desc desc)
2793	{
2794	unsigned int next;
2795
2796	/* If this descriptor says it doesn't chain, we're done. */
2797	if (!(desc->flags & cpu_to_vhost16(vq, VRING_DESC_F_NEXT)))
2798	return -1U;
2799
2800	/* Check they're not leading us off end of descriptors. */
2801	next = vhost16_to_cpu(vq, READ_ONCE(desc->next));
2802	return next;
2803	}
2804
2805	static int get_indirect(struct vhost_virtqueue *vq,
2806	struct iovec iov[], unsigned int iov_size,
2807	unsigned int out_num, unsigned int in_num,
2808	struct vhost_log log, unsigned int log_num,
2809	struct vring_desc *indirect)
2810	{
2811	struct vring_desc desc;
2812	unsigned int i = 0, count, found = 0;
2813	u32 len = vhost32_to_cpu(vq, indirect->len);
2814	struct iov_iter from;
2815	int ret, access;
2816
2817	/* Sanity check */
2818	if (unlikely(len % sizeof desc)) {
2819	vq_err(vq, "Invalid length in indirect descriptor: "
2820	"len 0x%llx not multiple of 0x%zx\n",
2821	(unsigned long long)len,
2822	sizeof desc);
2823	return -EINVAL;
2824	}
2825
2826	ret = translate_desc(vq, vhost64_to_cpu(vq, indirect->addr), len, vq->indirect,
2827	UIO_MAXIOV, VHOST_ACCESS_RO);
2828	if (unlikely(ret < 0)) {
2829	if (ret != -EAGAIN)
2830	vq_err(vq, "Translation failure %d in indirect.\n", ret);
2831	return ret;
2832	}
2833	iov_iter_init(&from, ITER_SOURCE, vq->indirect, ret, len);
2834	count = len / sizeof desc;
2835	/* Buffers are chained via a 16 bit next field, so
2836	* we can have at most 2^16 of these. */
2837	if (unlikely(count > USHRT_MAX + 1)) {
2838	vq_err(vq, "Indirect buffer length too big: %d\n",
2839	indirect->len);
2840	return -E2BIG;
2841	}
2842
2843	do {
2844	unsigned iov_count = in_num + out_num;
2845	if (unlikely(++found > count)) {
2846	vq_err(vq, "Loop detected: last one at %u "
2847	"indirect size %u\n",
2848	i, count);
2849	return -EINVAL;
2850	}
2851	if (unlikely(!copy_from_iter_full(&desc, sizeof(desc), &from))) {
2852	vq_err(vq, "Failed indirect descriptor: idx %d, %zx\n",
2853	i, (size_t)vhost64_to_cpu(vq, indirect->addr) + i * sizeof desc);
2854	return -EINVAL;
2855	}
2856	if (unlikely(desc.flags & cpu_to_vhost16(vq, VRING_DESC_F_INDIRECT))) {
2857	vq_err(vq, "Nested indirect descriptor: idx %d, %zx\n",
2858	i, (size_t)vhost64_to_cpu(vq, indirect->addr) + i * sizeof desc);
2859	return -EINVAL;
2860	}
2861
2862	if (desc.flags & cpu_to_vhost16(vq, VRING_DESC_F_WRITE))
2863	access = VHOST_ACCESS_WO;
2864	else
2865	access = VHOST_ACCESS_RO;
2866
2867	ret = translate_desc(vq, vhost64_to_cpu(vq, desc.addr),
2868	vhost32_to_cpu(vq, desc.len), iov + iov_count,
2869	iov_size - iov_count, access);
2870	if (unlikely(ret < 0)) {
2871	if (ret != -EAGAIN)
2872	vq_err(vq, "Translation failure %d indirect idx %d\n",
2873	ret, i);
2874	return ret;
2875	}
2876	/* If this is an input descriptor, increment that count. */
2877	if (access == VHOST_ACCESS_WO) {
2878	*in_num += ret;
2879	if (unlikely(log && ret)) {
2880	log[*log_num].addr = vhost64_to_cpu(vq, desc.addr);
2881	log[*log_num].len = vhost32_to_cpu(vq, desc.len);
2882	++*log_num;
2883	}
2884	} else {
2885	/* If it's an output descriptor, they're all supposed
2886	* to come before any input descriptors. */
2887	if (unlikely(*in_num)) {
2888	vq_err(vq, "Indirect descriptor "
2889	"has out after in: idx %d\n", i);
2890	return -EINVAL;
2891	}
2892	*out_num += ret;
2893	}
2894	} while ((i = next_desc(vq, &desc)) != -1);
2895	return 0;
2896	}
2897
2898	/**
2899	* vhost_get_vq_desc_n - Fetch the next available descriptor chain and build iovecs
2900	* @vq: target virtqueue
2901	* @iov: array that receives the scatter/gather segments
2902	* @iov_size: capacity of @iov in elements
2903	* @out_num: the number of output segments
2904	* @in_num: the number of input segments
2905	* @log: optional array to record addr/len for each writable segment; NULL if unused
2906	* @log_num: optional output; number of entries written to @log when provided
2907	* @ndesc: optional output; number of descriptors consumed from the available ring
2908	* (useful for rollback via vhost_discard_vq_desc)
2909	*
2910	* Extracts one available descriptor chain from @vq and translates guest addresses
2911	* into host iovecs.
2912	*
2913	* On success, advances @vq->last_avail_idx by 1 and @vq->next_avail_head by the
2914	* number of descriptors consumed (also stored via @ndesc when non-NULL).
2915	*
2916	* Return:
2917	* - head index in [0, @vq->num) on success;
2918	* - @vq->num if no descriptor is currently available;
2919	* - negative errno on failure
2920	*/
2921	int vhost_get_vq_desc_n(struct vhost_virtqueue *vq,
2922	struct iovec iov[], unsigned int iov_size,
2923	unsigned int out_num, unsigned int in_num,
2924	struct vhost_log log, unsigned int log_num,
2925	unsigned int *ndesc)
2926	{
2927	bool in_order = vhost_has_feature(vq, VIRTIO_F_IN_ORDER);
2928	struct vring_desc desc;
2929	unsigned int i, head, found = 0;
2930	u16 last_avail_idx = vq->last_avail_idx;
2931	__virtio16 ring_head;
2932	int ret, access, c = 0;
2933
2934	if (vq->avail_idx == vq->last_avail_idx) {
2935	ret = vhost_get_avail_idx(vq);
2936	if (unlikely(ret < 0))
2937	return ret;
2938
2939	if (!ret)
2940	return vq->num;
2941	}
2942
2943	if (in_order)
2944	head = vq->next_avail_head & (vq->num - 1);
2945	else {
2946	/* Grab the next descriptor number they're
2947	* advertising, and increment the index we've seen. */
2948	if (unlikely(vhost_get_avail_head(vq, &ring_head,
2949	last_avail_idx))) {
2950	vq_err(vq, "Failed to read head: idx %d address %p\n",
2951	last_avail_idx,
2952	&vq->avail->ring[last_avail_idx % vq->num]);
2953	return -EFAULT;
2954	}
2955	head = vhost16_to_cpu(vq, ring_head);
2956	}
2957
2958	/* If their number is silly, that's an error. */
2959	if (unlikely(head >= vq->num)) {
2960	vq_err(vq, "Guest says index %u > %u is available",
2961	head, vq->num);
2962	return -EINVAL;
2963	}
2964
2965	/* When we start there are none of either input nor output. */
2966	out_num = in_num = 0;
2967	if (unlikely(log))
2968	*log_num = 0;
2969
2970	i = head;
2971	do {
2972	unsigned iov_count = in_num + out_num;
2973	if (unlikely(i >= vq->num)) {
2974	vq_err(vq, "Desc index is %u > %u, head = %u",
2975	i, vq->num, head);
2976	return -EINVAL;
2977	}
2978	if (unlikely(++found > vq->num)) {
2979	vq_err(vq, "Loop detected: last one at %u "
2980	"vq size %u head %u\n",
2981	i, vq->num, head);
2982	return -EINVAL;
2983	}
2984	ret = vhost_get_desc(vq, &desc, i);
2985	if (unlikely(ret)) {
2986	vq_err(vq, "Failed to get descriptor: idx %d addr %p\n",
2987	i, vq->desc + i);
2988	return -EFAULT;
2989	}
2990	if (desc.flags & cpu_to_vhost16(vq, VRING_DESC_F_INDIRECT)) {
2991	ret = get_indirect(vq, iov, iov_size,
2992	out_num, in_num,
2993	log, log_num, &desc);
2994	if (unlikely(ret < 0)) {
2995	if (ret != -EAGAIN)
2996	vq_err(vq, "Failure detected "
2997	"in indirect descriptor at idx %d\n", i);
2998	return ret;
2999	}
3000	++c;
3001	continue;
3002	}
3003
3004	if (desc.flags & cpu_to_vhost16(vq, VRING_DESC_F_WRITE))
3005	access = VHOST_ACCESS_WO;
3006	else
3007	access = VHOST_ACCESS_RO;
3008	ret = translate_desc(vq, vhost64_to_cpu(vq, desc.addr),
3009	vhost32_to_cpu(vq, desc.len), iov + iov_count,
3010	iov_size - iov_count, access);
3011	if (unlikely(ret < 0)) {
3012	if (ret != -EAGAIN)
3013	vq_err(vq, "Translation failure %d descriptor idx %d\n",
3014	ret, i);
3015	return ret;
3016	}
3017	if (access == VHOST_ACCESS_WO) {
3018	/* If this is an input descriptor,
3019	* increment that count. */
3020	*in_num += ret;
3021	if (unlikely(log && ret)) {
3022	log[*log_num].addr = vhost64_to_cpu(vq, desc.addr);
3023	log[*log_num].len = vhost32_to_cpu(vq, desc.len);
3024	++*log_num;
3025	}
3026	} else {
3027	/* If it's an output descriptor, they're all supposed
3028	* to come before any input descriptors. */
3029	if (unlikely(*in_num)) {
3030	vq_err(vq, "Descriptor has out after in: "
3031	"idx %d\n", i);
3032	return -EINVAL;
3033	}
3034	*out_num += ret;
3035	}
3036	++c;
3037	} while ((i = next_desc(vq, &desc)) != -1);
3038
3039	/* On success, increment avail index. */
3040	vq->last_avail_idx++;
3041	vq->next_avail_head += c;
3042
3043	if (ndesc)
3044	*ndesc = c;
3045
3046	/* Assume notifications from guest are disabled at this point,
3047	* if they aren't we would need to update avail_event index. */
3048	BUG_ON(!(vq->used_flags & VRING_USED_F_NO_NOTIFY));
3049	return head;
3050	}
3051	EXPORT_SYMBOL_GPL(vhost_get_vq_desc_n);
3052
3053	/* This looks in the virtqueue and for the first available buffer, and converts
3054	* it to an iovec for convenient access. Since descriptors consist of some
3055	* number of output then some number of input descriptors, it's actually two
3056	* iovecs, but we pack them into one and note how many of each there were.
3057	*
3058	* This function returns the descriptor number found, or vq->num (which is
3059	* never a valid descriptor number) if none was found. A negative code is
3060	* returned on error.
3061	*/
3062	int vhost_get_vq_desc(struct vhost_virtqueue *vq,
3063	struct iovec iov[], unsigned int iov_size,
3064	unsigned int out_num, unsigned int in_num,
3065	struct vhost_log log, unsigned int log_num)
3066	{
3067	return vhost_get_vq_desc_n(vq, iov, iov_size, out_num, in_num,
3068	log, log_num, NULL);
3069	}
3070	EXPORT_SYMBOL_GPL(vhost_get_vq_desc);
3071
3072	/**
3073	* vhost_discard_vq_desc - Reverse the effect of vhost_get_vq_desc_n()
3074	* @vq: target virtqueue
3075	* @nbufs: number of buffers to roll back
3076	* @ndesc: number of descriptors to roll back
3077	*
3078	* Rewinds the internal consumer cursors after a failed attempt to use buffers
3079	* returned by vhost_get_vq_desc_n().
3080	*/
3081	void vhost_discard_vq_desc(struct vhost_virtqueue *vq, int nbufs,
3082	unsigned int ndesc)
3083	{
3084	vq->next_avail_head -= ndesc;
3085	vq->last_avail_idx -= nbufs;
3086	}
3087	EXPORT_SYMBOL_GPL(vhost_discard_vq_desc);
3088
3089	/* After we've used one of their buffers, we tell them about it. We'll then
3090	* want to notify the guest, using eventfd. */
3091	int vhost_add_used(struct vhost_virtqueue *vq, unsigned int head, int len)
3092	{
3093	struct vring_used_elem heads = {
3094	cpu_to_vhost32(vq, head),
3095	cpu_to_vhost32(vq, len)
3096	};
3097	u16 nheads = 1;
3098
3099	return vhost_add_used_n(vq, &heads, &nheads, 1);
3100	}
3101	EXPORT_SYMBOL_GPL(vhost_add_used);
3102
3103	static int __vhost_add_used_n(struct vhost_virtqueue *vq,
3104	struct vring_used_elem *heads,
3105	unsigned count)
3106	{
3107	vring_used_elem_t __user *used;
3108	u16 old, new;
3109	int start;
3110
3111	start = vq->last_used_idx & (vq->num - 1);
3112	used = vq->used->ring + start;
3113	if (vhost_put_used(vq, heads, start, count)) {
3114	vq_err(vq, "Failed to write used");
3115	return -EFAULT;
3116	}
3117	if (unlikely(vq->log_used)) {
3118	/* Make sure data is seen before log. */
3119	smp_wmb();
3120	/* Log used ring entry write. */
3121	log_used(vq, ((void __user )used - (void __user )vq->used),
3122	count * sizeof *used);
3123	}
3124	old = vq->last_used_idx;
3125	new = (vq->last_used_idx += count);
3126	/* If the driver never bothers to signal in a very long while,
3127	* used index might wrap around. If that happens, invalidate
3128	* signalled_used index we stored. TODO: make sure driver
3129	* signals at least once in 2^16 and remove this. */
3130	if (unlikely((u16)(new - vq->signalled_used) < (u16)(new - old)))
3131	vq->signalled_used_valid = false;
3132	return 0;
3133	}
3134
3135	static int vhost_add_used_n_ooo(struct vhost_virtqueue *vq,
3136	struct vring_used_elem *heads,
3137	unsigned count)
3138	{
3139	int start, n, r;
3140
3141	start = vq->last_used_idx & (vq->num - 1);
3142	n = vq->num - start;
3143	if (n < count) {
3144	r = __vhost_add_used_n(vq, heads, n);
3145	if (r < 0)
3146	return r;
3147	heads += n;
3148	count -= n;
3149	}
3150	return __vhost_add_used_n(vq, heads, count);
3151	}
3152
3153	static int vhost_add_used_n_in_order(struct vhost_virtqueue *vq,
3154	struct vring_used_elem *heads,
3155	const u16 *nheads,
3156	unsigned count)
3157	{
3158	vring_used_elem_t __user *used;
3159	u16 old, new = vq->last_used_idx;
3160	int start, i;
3161
3162	if (!nheads)
3163	return -EINVAL;
3164
3165	start = vq->last_used_idx & (vq->num - 1);
3166	used = vq->used->ring + start;
3167
3168	for (i = 0; i < count; i++) {
3169	if (vhost_put_used(vq, &heads[i], start, 1)) {
3170	vq_err(vq, "Failed to write used");
3171	return -EFAULT;
3172	}
3173	start += nheads[i];
3174	new += nheads[i];
3175	if (start >= vq->num)
3176	start -= vq->num;
3177	}
3178
3179	if (unlikely(vq->log_used)) {
3180	/* Make sure data is seen before log. */
3181	smp_wmb();
3182	/* Log used ring entry write. */
3183	log_used(vq, ((void __user )used - (void __user )vq->used),
3184	(vq->num - start) * sizeof *used);
3185	if (start + count > vq->num)
3186	log_used(vq, 0,
3187	(start + count - vq->num) * sizeof *used);
3188	}
3189
3190	old = vq->last_used_idx;
3191	vq->last_used_idx = new;
3192	/* If the driver never bothers to signal in a very long while,
3193	* used index might wrap around. If that happens, invalidate
3194	* signalled_used index we stored. TODO: make sure driver
3195	* signals at least once in 2^16 and remove this. */
3196	if (unlikely((u16)(new - vq->signalled_used) < (u16)(new - old)))
3197	vq->signalled_used_valid = false;
3198	return 0;
3199	}
3200
3201	/* After we've used one of their buffers, we tell them about it. We'll then
3202	* want to notify the guest, using eventfd. */
3203	int vhost_add_used_n(struct vhost_virtqueue vq, struct vring_used_elem heads,
3204	u16 *nheads, unsigned count)
3205	{
3206	bool in_order = vhost_has_feature(vq, VIRTIO_F_IN_ORDER);
3207	int r;
3208
3209	if (!in_order \|\| !nheads)
3210	r = vhost_add_used_n_ooo(vq, heads, count);
3211	else
3212	r = vhost_add_used_n_in_order(vq, heads, nheads, count);
3213
3214	if (r < 0)
3215	return r;
3216
3217	/* Make sure buffer is written before we update index. */
3218	smp_wmb();
3219	if (vhost_put_used_idx(vq)) {
3220	vq_err(vq, "Failed to increment used idx");
3221	return -EFAULT;
3222	}
3223	if (unlikely(vq->log_used)) {
3224	/* Make sure used idx is seen before log. */
3225	smp_wmb();
3226	/* Log used index update. */
3227	log_used(vq, offsetof(struct vring_used, idx),
3228	sizeof vq->used->idx);
3229	if (vq->log_ctx)
3230	eventfd_signal(vq->log_ctx);
3231	}
3232	return r;
3233	}
3234	EXPORT_SYMBOL_GPL(vhost_add_used_n);
3235
3236	static bool vhost_notify(struct vhost_dev dev, struct vhost_virtqueue vq)
3237	{
3238	__u16 old, new;
3239	__virtio16 event;
3240	bool v;
3241	/* Flush out used index updates. This is paired
3242	* with the barrier that the Guest executes when enabling
3243	* interrupts. */
3244	smp_mb();
3245
3246	if (vhost_has_feature(vq, VIRTIO_F_NOTIFY_ON_EMPTY) &&
3247	unlikely(vq->avail_idx == vq->last_avail_idx))
3248	return true;
3249
3250	if (!vhost_has_feature(vq, VIRTIO_RING_F_EVENT_IDX)) {
3251	__virtio16 flags;
3252	if (vhost_get_avail_flags(vq, &flags)) {
3253	vq_err(vq, "Failed to get flags");
3254	return true;
3255	}
3256	return !(flags & cpu_to_vhost16(vq, VRING_AVAIL_F_NO_INTERRUPT));
3257	}
3258	old = vq->signalled_used;
3259	v = vq->signalled_used_valid;
3260	new = vq->signalled_used = vq->last_used_idx;
3261	vq->signalled_used_valid = true;
3262
3263	if (unlikely(!v))
3264	return true;
3265
3266	if (vhost_get_used_event(vq, &event)) {
3267	vq_err(vq, "Failed to get used event idx");
3268	return true;
3269	}
3270	return vring_need_event(vhost16_to_cpu(vq, event), new, old);
3271	}
3272
3273	/* This actually signals the guest, using eventfd. */
3274	void vhost_signal(struct vhost_dev dev, struct vhost_virtqueue vq)
3275	{
3276	/* Signal the Guest tell them we used something up. */
3277	if (vq->call_ctx.ctx && vhost_notify(dev, vq))
3278	eventfd_signal(vq->call_ctx.ctx);
3279	}
3280	EXPORT_SYMBOL_GPL(vhost_signal);
3281
3282	/* And here's the combo meal deal. Supersize me! */
3283	void vhost_add_used_and_signal(struct vhost_dev *dev,
3284	struct vhost_virtqueue *vq,
3285	unsigned int head, int len)
3286	{
3287	vhost_add_used(vq, head, len);
3288	vhost_signal(dev, vq);
3289	}
3290	EXPORT_SYMBOL_GPL(vhost_add_used_and_signal);
3291
3292	/* multi-buffer version of vhost_add_used_and_signal */
3293	void vhost_add_used_and_signal_n(struct vhost_dev *dev,
3294	struct vhost_virtqueue *vq,
3295	struct vring_used_elem *heads,
3296	u16 *nheads,
3297	unsigned count)
3298	{
3299	vhost_add_used_n(vq, heads, nheads, count);
3300	vhost_signal(dev, vq);
3301	}
3302	EXPORT_SYMBOL_GPL(vhost_add_used_and_signal_n);
3303
3304	/* return true if we're sure that available ring is empty */
3305	bool vhost_vq_avail_empty(struct vhost_dev dev, struct vhost_virtqueue vq)
3306	{
3307	int r;
3308
3309	if (vq->avail_idx != vq->last_avail_idx)
3310	return false;
3311
3312	r = vhost_get_avail_idx(vq);
3313
3314	/* Note: we treat error as non-empty here */
3315	return r == 0;
3316	}
3317	EXPORT_SYMBOL_GPL(vhost_vq_avail_empty);
3318
3319	/* OK, now we need to know about added descriptors. */
3320	bool vhost_enable_notify(struct vhost_dev dev, struct vhost_virtqueue vq)
3321	{
3322	int r;
3323
3324	if (!(vq->used_flags & VRING_USED_F_NO_NOTIFY))
3325	return false;
3326	vq->used_flags &= ~VRING_USED_F_NO_NOTIFY;
3327	if (!vhost_has_feature(vq, VIRTIO_RING_F_EVENT_IDX)) {
3328	r = vhost_update_used_flags(vq);
3329	if (r) {
3330	vq_err(vq, "Failed to enable notification at %p: %d\n",
3331	&vq->used->flags, r);
3332	return false;
3333	}
3334	} else {
3335	r = vhost_update_avail_event(vq);
3336	if (r) {
3337	vq_err(vq, "Failed to update avail event index at %p: %d\n",
3338	vhost_avail_event(vq), r);
3339	return false;
3340	}
3341	}
3342	/* They could have slipped one in as we were doing that: make
3343	* sure it's written, then check again. */
3344	smp_mb();
3345
3346	r = vhost_get_avail_idx(vq);
3347	/* Note: we treat error as empty here */
3348	if (unlikely(r < 0))
3349	return false;
3350
3351	return r;
3352	}
3353	EXPORT_SYMBOL_GPL(vhost_enable_notify);
3354
3355	/* We don't need to be notified again. */
3356	void vhost_disable_notify(struct vhost_dev dev, struct vhost_virtqueue vq)
3357	{
3358	int r;
3359
3360	if (vq->used_flags & VRING_USED_F_NO_NOTIFY)
3361	return;
3362	vq->used_flags \|= VRING_USED_F_NO_NOTIFY;
3363	if (!vhost_has_feature(vq, VIRTIO_RING_F_EVENT_IDX)) {
3364	r = vhost_update_used_flags(vq);
3365	if (r)
3366	vq_err(vq, "Failed to disable notification at %p: %d\n",
3367	&vq->used->flags, r);
3368	}
3369	}
3370	EXPORT_SYMBOL_GPL(vhost_disable_notify);
3371
3372	/* Create a new message. */
3373	struct vhost_msg_node vhost_new_msg(struct vhost_virtqueue vq, int type)
3374	{
3375	/* Make sure all padding within the structure is initialized. */
3376	struct vhost_msg_node node = kzalloc_obj(node);
3377	if (!node)
3378	return NULL;
3379
3380	node->vq = vq;
3381	node->msg.type = type;
3382	return node;
3383	}
3384	EXPORT_SYMBOL_GPL(vhost_new_msg);
3385
3386	void vhost_enqueue_msg(struct vhost_dev dev, struct list_head head,
3387	struct vhost_msg_node *node)
3388	{
3389	spin_lock(&dev->iotlb_lock);
3390	list_add_tail(&node->node, head);
3391	spin_unlock(&dev->iotlb_lock);
3392
3393	wake_up_interruptible_poll(&dev->wait, EPOLLIN \| EPOLLRDNORM);
3394	}
3395	EXPORT_SYMBOL_GPL(vhost_enqueue_msg);
3396
3397	struct vhost_msg_node vhost_dequeue_msg(struct vhost_dev dev,
3398	struct list_head *head)
3399	{
3400	struct vhost_msg_node *node = NULL;
3401
3402	spin_lock(&dev->iotlb_lock);
3403	if (!list_empty(head)) {
3404	node = list_first_entry(head, struct vhost_msg_node,
3405	node);
3406	list_del(&node->node);
3407	}
3408	spin_unlock(&dev->iotlb_lock);
3409
3410	return node;
3411	}
3412	EXPORT_SYMBOL_GPL(vhost_dequeue_msg);
3413
3414	void vhost_set_backend_features(struct vhost_dev *dev, u64 features)
3415	{
3416	struct vhost_virtqueue *vq;
3417	int i;
3418
3419	mutex_lock(&dev->mutex);
3420	for (i = 0; i < dev->nvqs; ++i) {
3421	vq = dev->vqs[i];
3422	mutex_lock(&vq->mutex);
3423	vq->acked_backend_features = features;
3424	mutex_unlock(&vq->mutex);
3425	}
3426	mutex_unlock(&dev->mutex);
3427	}
3428	EXPORT_SYMBOL_GPL(vhost_set_backend_features);
3429
3430	MODULE_VERSION("0.0.1");
3431	MODULE_LICENSE("GPL v2");
3432	MODULE_AUTHOR("Michael S. Tsirkin");
3433	MODULE_DESCRIPTION("Host kernel accelerator for virtio");