linux - drivers/virtio/virtio

4021 lines · 104.6 KiB

1	// SPDX-License-Identifier: GPL-2.0-or-later
2	/* Virtio ring implementation.
3	*
4	* Copyright 2007 Rusty Russell IBM Corporation
5	*/
6	#include <linux/virtio.h>
7	#include <linux/virtio_ring.h>
8	#include <linux/virtio_config.h>
9	#include <linux/device.h>
10	#include <linux/slab.h>
11	#include <linux/module.h>
12	#include <linux/hrtimer.h>
13	#include <linux/dma-mapping.h>
14	#include <linux/kmsan.h>
15	#include <linux/spinlock.h>
16	#include <xen/xen.h>
17
18	#ifdef DEBUG
19	/* For development, we want to crash whenever the ring is screwed. */
20	#define BAD_RING(_vq, fmt, args...) \
21	do { \
22	dev_err(&(_vq)->vq.vdev->dev, \
23	"%s:"fmt, (_vq)->vq.name, ##args); \
24	BUG(); \
25	} while (0)
26	/* Caller is supposed to guarantee no reentry. */
27	#define START_USE(_vq) \
28	do { \
29	if ((_vq)->in_use) \
30	panic("%s:in_use = %i\n", \
31	(_vq)->vq.name, (_vq)->in_use); \
32	(_vq)->in_use = __LINE__; \
33	} while (0)
34	#define END_USE(_vq) \
35	do { BUG_ON(!(_vq)->in_use); (_vq)->in_use = 0; } while(0)
36	#define LAST_ADD_TIME_UPDATE(_vq) \
37	do { \
38	ktime_t now = ktime_get(); \
39	\
40	/* No kick or get, with .1 second between? Warn. */ \
41	if ((_vq)->last_add_time_valid) \
42	WARN_ON(ktime_to_ms(ktime_sub(now, \
43	(_vq)->last_add_time)) > 100); \
44	(_vq)->last_add_time = now; \
45	(_vq)->last_add_time_valid = true; \
46	} while (0)
47	#define LAST_ADD_TIME_CHECK(_vq) \
48	do { \
49	if ((_vq)->last_add_time_valid) { \
50	WARN_ON(ktime_to_ms(ktime_sub(ktime_get(), \
51	(_vq)->last_add_time)) > 100); \
52	} \
53	} while (0)
54	#define LAST_ADD_TIME_INVALID(_vq) \
55	((_vq)->last_add_time_valid = false)
56	#else
57	#define BAD_RING(_vq, fmt, args...) \
58	do { \
59	dev_err(&_vq->vq.vdev->dev, \
60	"%s:"fmt, (_vq)->vq.name, ##args); \
61	(_vq)->broken = true; \
62	} while (0)
63	#define START_USE(vq)
64	#define END_USE(vq)
65	#define LAST_ADD_TIME_UPDATE(vq)
66	#define LAST_ADD_TIME_CHECK(vq)
67	#define LAST_ADD_TIME_INVALID(vq)
68	#endif
69
70	enum vq_layout {
71	VQ_LAYOUT_SPLIT = 0,
72	VQ_LAYOUT_PACKED,
73	VQ_LAYOUT_SPLIT_IN_ORDER,
74	VQ_LAYOUT_PACKED_IN_ORDER,
75	};
76
77	struct vring_desc_state_split {
78	void data; / Data for callback. */
79
80	/* Indirect desc table and extra table, if any. These two will be
81	* allocated together. So we won't stress more to the memory allocator.
82	*/
83	struct vring_desc *indir_desc;
84	u32 total_in_len;
85	};
86
87	struct vring_desc_state_packed {
88	void data; / Data for callback. */
89
90	/* Indirect desc table and extra table, if any. These two will be
91	* allocated together. So we won't stress more to the memory allocator.
92	*/
93	struct vring_packed_desc *indir_desc;
94	u16 num; /* Descriptor list length. */
95	u16 last; /* The last desc state in a list. */
96	u32 total_in_len; /* In length for the skipped buffer. */
97	};
98
99	struct vring_desc_extra {
100	dma_addr_t addr; /* Descriptor DMA addr. */
101	u32 len; /* Descriptor length. */
102	u16 flags; /* Descriptor flags. */
103	u16 next; /* The next desc state in a list. */
104	};
105
106	struct vring_virtqueue_split {
107	/* Actual memory layout for this queue. */
108	struct vring vring;
109
110	/* Last written value to avail->flags */
111	u16 avail_flags_shadow;
112
113	/*
114	* Last written value to avail->idx in
115	* guest byte order.
116	*/
117	u16 avail_idx_shadow;
118
119	/* Per-descriptor state. */
120	struct vring_desc_state_split *desc_state;
121	struct vring_desc_extra *desc_extra;
122
123	/* DMA address and size information */
124	dma_addr_t queue_dma_addr;
125	size_t queue_size_in_bytes;
126
127	/*
128	* The parameters for creating vrings are reserved for creating new
129	* vring.
130	*/
131	u32 vring_align;
132	bool may_reduce_num;
133	};
134
135	struct vring_virtqueue_packed {
136	/* Actual memory layout for this queue. */
137	struct {
138	unsigned int num;
139	struct vring_packed_desc *desc;
140	struct vring_packed_desc_event *driver;
141	struct vring_packed_desc_event *device;
142	} vring;
143
144	/* Driver ring wrap counter. */
145	bool avail_wrap_counter;
146
147	/* Avail used flags. */
148	u16 avail_used_flags;
149
150	/* Index of the next avail descriptor. */
151	u16 next_avail_idx;
152
153	/*
154	* Last written value to driver->flags in
155	* guest byte order.
156	*/
157	u16 event_flags_shadow;
158
159	/* Per-descriptor state. */
160	struct vring_desc_state_packed *desc_state;
161	struct vring_desc_extra *desc_extra;
162
163	/* DMA address and size information */
164	dma_addr_t ring_dma_addr;
165	dma_addr_t driver_event_dma_addr;
166	dma_addr_t device_event_dma_addr;
167	size_t ring_size_in_bytes;
168	size_t event_size_in_bytes;
169	};
170
171	struct vring_virtqueue;
172
173	struct virtqueue_ops {
174	int (add)(struct vring_virtqueue vq, struct scatterlist *sgs[],
175	unsigned int total_sg, unsigned int out_sgs,
176	unsigned int in_sgs, void *data,
177	void *ctx, bool premapped, gfp_t gfp,
178	unsigned long attr);
179	void (get)(struct vring_virtqueue vq, unsigned int len, void **ctx);
180	bool (kick_prepare)(struct vring_virtqueue vq);
181	void (disable_cb)(struct vring_virtqueue vq);
182	bool (enable_cb_delayed)(struct vring_virtqueue vq);
183	unsigned int (enable_cb_prepare)(struct vring_virtqueue vq);
184	bool (poll)(const struct vring_virtqueue vq,
185	unsigned int last_used_idx);
186	void (detach_unused_buf)(struct vring_virtqueue *vq);
187	bool (more_used)(const struct vring_virtqueue vq);
188	int (resize)(struct vring_virtqueue vq, u32 num);
189	void (reset)(struct vring_virtqueue vq);
190	};
191
192	struct vring_virtqueue {
193	struct virtqueue vq;
194
195	/* Is DMA API used? */
196	bool use_map_api;
197
198	/* Can we use weak barriers? */
199	bool weak_barriers;
200
201	/* Other side has made a mess, don't try any more. */
202	bool broken;
203
204	/* Host supports indirect buffers */
205	bool indirect;
206
207	/* Host publishes avail event idx */
208	bool event;
209
210	enum vq_layout layout;
211
212	/*
213	* Without IN_ORDER it's the head of free buffer list. With
214	* IN_ORDER and SPLIT, it's the next available buffer
215	* index. With IN_ORDER and PACKED, it's unused.
216	*/
217	unsigned int free_head;
218
219	/*
220	* With IN_ORDER, once we see an in-order batch, this stores
221	* this last entry, and until we return the last buffer.
222	* After this, id is set to UINT_MAX to mark it invalid.
223	* Unused without IN_ORDER.
224	*/
225	struct used_entry {
226	u32 id;
227	u32 len;
228	} batch_last;
229
230	/* Number we've added since last sync. */
231	unsigned int num_added;
232
233	/* Last used index we've seen.
234	* for split ring, it just contains last used index
235	* for packed ring:
236	* bits up to VRING_PACKED_EVENT_F_WRAP_CTR include the last used index.
237	* bits from VRING_PACKED_EVENT_F_WRAP_CTR include the used wrap counter.
238	*/
239	u16 last_used_idx;
240
241	/* With IN_ORDER and SPLIT, last descriptor id we used to
242	* detach buffer.
243	*/
244	u16 last_used;
245
246	/* Hint for event idx: already triggered no need to disable. */
247	bool event_triggered;
248
249	union {
250	/* Available for split ring */
251	struct vring_virtqueue_split split;
252
253	/* Available for packed ring */
254	struct vring_virtqueue_packed packed;
255	};
256
257	/* How to notify other side. FIXME: commonalize hcalls! */
258	bool (notify)(struct virtqueue vq);
259
260	/* DMA, allocation, and size information */
261	bool we_own_ring;
262
263	union virtio_map map;
264
265	#ifdef DEBUG
266	/* They're supposed to lock for us. */
267	unsigned int in_use;
268
269	/* Figure out if their kicks are too delayed. */
270	bool last_add_time_valid;
271	ktime_t last_add_time;
272	#endif
273	};
274
275	/*
276	* Accessors for device-writable fields in virtio rings.
277	* These fields are concurrently written by the device and read by the driver.
278	* Use READ_ONCE() to prevent compiler optimizations, document the
279	* intentional data race and prevent KCSAN warnings.
280	*/
281	static inline u16 vring_read_split_used_idx(const struct vring_virtqueue *vq)
282	{
283	return virtio16_to_cpu(vq->vq.vdev,
284	READ_ONCE(vq->split.vring.used->idx));
285	}
286
287	static inline u32 vring_read_split_used_id(const struct vring_virtqueue *vq,
288	u16 idx)
289	{
290	return virtio32_to_cpu(vq->vq.vdev,
291	READ_ONCE(vq->split.vring.used->ring[idx].id));
292	}
293
294	static inline u32 vring_read_split_used_len(const struct vring_virtqueue *vq, u16 idx)
295	{
296	return virtio32_to_cpu(vq->vq.vdev,
297	READ_ONCE(vq->split.vring.used->ring[idx].len));
298	}
299
300	static inline u16 vring_read_split_avail_event(const struct vring_virtqueue *vq)
301	{
302	return virtio16_to_cpu(vq->vq.vdev,
303	READ_ONCE(vring_avail_event(&vq->split.vring)));
304	}
305
306	static inline u16 vring_read_packed_desc_flags(const struct vring_virtqueue *vq,
307	u16 idx)
308	{
309	return le16_to_cpu(READ_ONCE(vq->packed.vring.desc[idx].flags));
310	}
311
312	static inline u16 vring_read_packed_desc_id(const struct vring_virtqueue *vq,
313	u16 idx)
314	{
315	return le16_to_cpu(READ_ONCE(vq->packed.vring.desc[idx].id));
316	}
317
318	static inline u32 vring_read_packed_desc_len(const struct vring_virtqueue *vq,
319	u16 idx)
320	{
321	return le32_to_cpu(READ_ONCE(vq->packed.vring.desc[idx].len));
322	}
323
324	static struct vring_desc_extra *vring_alloc_desc_extra(unsigned int num);
325	static void vring_free(struct virtqueue *_vq);
326
327	/*
328	* Helpers.
329	*/
330
331	#define to_vvq(_vq) container_of_const(_vq, struct vring_virtqueue, vq)
332
333
334	static inline bool virtqueue_is_packed(const struct vring_virtqueue *vq)
335	{
336	return vq->layout == VQ_LAYOUT_PACKED \|\|
337	vq->layout == VQ_LAYOUT_PACKED_IN_ORDER;
338	}
339
340	static inline bool virtqueue_is_in_order(const struct vring_virtqueue *vq)
341	{
342	return vq->layout == VQ_LAYOUT_SPLIT_IN_ORDER \|\|
343	vq->layout == VQ_LAYOUT_PACKED_IN_ORDER;
344	}
345
346	static bool virtqueue_use_indirect(const struct vring_virtqueue *vq,
347	unsigned int total_sg)
348	{
349	/*
350	* If the host supports indirect descriptor tables, and we have multiple
351	* buffers, then go indirect. FIXME: tune this threshold
352	*/
353	return (vq->indirect && total_sg > 1 && vq->vq.num_free);
354	}
355
356	/*
357	* Modern virtio devices have feature bits to specify whether they need a
358	* quirk and bypass the IOMMU. If not there, just use the DMA API.
359	*
360	* If there, the interaction between virtio and DMA API is messy.
361	*
362	* On most systems with virtio, physical addresses match bus addresses,
363	* and it doesn't particularly matter whether we use the DMA API.
364	*
365	* On some systems, including Xen and any system with a physical device
366	* that speaks virtio behind a physical IOMMU, we must use the DMA API
367	* for virtio DMA to work at all.
368	*
369	* On other systems, including SPARC and PPC64, virtio-pci devices are
370	* enumerated as though they are behind an IOMMU, but the virtio host
371	* ignores the IOMMU, so we must either pretend that the IOMMU isn't
372	* there or somehow map everything as the identity.
373	*
374	* For the time being, we preserve historic behavior and bypass the DMA
375	* API.
376	*
377	* TODO: install a per-device DMA ops structure that does the right thing
378	* taking into account all the above quirks, and use the DMA API
379	* unconditionally on data path.
380	*/
381
382	static bool vring_use_map_api(const struct virtio_device *vdev)
383	{
384	if (!virtio_has_dma_quirk(vdev))
385	return true;
386
387	/* Otherwise, we are left to guess. */
388	/*
389	* In theory, it's possible to have a buggy QEMU-supposed
390	* emulated Q35 IOMMU and Xen enabled at the same time. On
391	* such a configuration, virtio has never worked and will
392	* not work without an even larger kludge. Instead, enable
393	* the DMA API if we're a Xen guest, which at least allows
394	* all of the sensible Xen configurations to work correctly.
395	*/
396	if (xen_domain())
397	return true;
398
399	return false;
400	}
401
402	static bool vring_need_unmap_buffer(const struct vring_virtqueue *vring,
403	const struct vring_desc_extra *extra)
404	{
405	return vring->use_map_api && (extra->addr != DMA_MAPPING_ERROR);
406	}
407
408	size_t virtio_max_dma_size(const struct virtio_device *vdev)
409	{
410	size_t max_segment_size = SIZE_MAX;
411
412	if (vring_use_map_api(vdev)) {
413	if (vdev->map) {
414	max_segment_size =
415	vdev->map->max_mapping_size(vdev->vmap);
416	} else
417	max_segment_size =
418	dma_max_mapping_size(vdev->dev.parent);
419	}
420
421	return max_segment_size;
422	}
423	EXPORT_SYMBOL_GPL(virtio_max_dma_size);
424
425	static void vring_alloc_queue(struct virtio_device vdev, size_t size,
426	dma_addr_t *map_handle, gfp_t flag,
427	union virtio_map map)
428	{
429	if (vring_use_map_api(vdev)) {
430	return virtqueue_map_alloc_coherent(vdev, map, size,
431	map_handle, flag);
432	} else {
433	void *queue = alloc_pages_exact(PAGE_ALIGN(size), flag);
434
435	if (queue) {
436	phys_addr_t phys_addr = virt_to_phys(queue);
437	*map_handle = (dma_addr_t)phys_addr;
438
439	/*
440	* Sanity check: make sure we dind't truncate
441	* the address. The only arches I can find that
442	* have 64-bit phys_addr_t but 32-bit dma_addr_t
443	* are certain non-highmem MIPS and x86
444	* configurations, but these configurations
445	* should never allocate physical pages above 32
446	* bits, so this is fine. Just in case, throw a
447	* warning and abort if we end up with an
448	* unrepresentable address.
449	*/
450	if (WARN_ON_ONCE(*map_handle != phys_addr)) {
451	free_pages_exact(queue, PAGE_ALIGN(size));
452	return NULL;
453	}
454	}
455	return queue;
456	}
457	}
458
459	static void vring_free_queue(struct virtio_device *vdev, size_t size,
460	void *queue, dma_addr_t map_handle,
461	union virtio_map map)
462	{
463	if (vring_use_map_api(vdev))
464	virtqueue_map_free_coherent(vdev, map, size,
465	queue, map_handle);
466	else
467	free_pages_exact(queue, PAGE_ALIGN(size));
468	}
469
470	/*
471	* The DMA ops on various arches are rather gnarly right now, and
472	* making all of the arch DMA ops work on the vring device itself
473	* is a mess.
474	*/
475	static struct device vring_dma_dev(const struct vring_virtqueue vq)
476	{
477	return vq->map.dma_dev;
478	}
479
480	static int vring_mapping_error(const struct vring_virtqueue *vq,
481	dma_addr_t addr)
482	{
483	struct virtio_device *vdev = vq->vq.vdev;
484
485	if (!vq->use_map_api)
486	return 0;
487
488	if (vdev->map)
489	return vdev->map->mapping_error(vq->map, addr);
490	else
491	return dma_mapping_error(vring_dma_dev(vq), addr);
492	}
493
494	/* Map one sg entry. */
495	static int vring_map_one_sg(const struct vring_virtqueue vq, struct scatterlist sg,
496	enum dma_data_direction direction, dma_addr_t *addr,
497	u32 *len, bool premapped, unsigned long attr)
498	{
499	if (premapped) {
500	*addr = sg_dma_address(sg);
501	*len = sg_dma_len(sg);
502	return 0;
503	}
504
505	*len = sg->length;
506
507	if (!vq->use_map_api) {
508	/*
509	* If DMA is not used, KMSAN doesn't know that the scatterlist
510	* is initialized by the hardware. Explicitly check/unpoison it
511	* depending on the direction.
512	*/
513	kmsan_handle_dma(sg_phys(sg), sg->length, direction);
514	*addr = (dma_addr_t)sg_phys(sg);
515	return 0;
516	}
517
518	/*
519	* We can't use dma_map_sg, because we don't use scatterlists in
520	* the way it expects (we don't guarantee that the scatterlist
521	* will exist for the lifetime of the mapping).
522	*/
523	*addr = virtqueue_map_page_attrs(&vq->vq, sg_page(sg),
524	sg->offset, sg->length,
525	direction, attr);
526
527	if (vring_mapping_error(vq, *addr))
528	return -ENOMEM;
529
530	return 0;
531	}
532
533	static dma_addr_t vring_map_single(const struct vring_virtqueue *vq,
534	void *cpu_addr, size_t size,
535	enum dma_data_direction direction)
536	{
537	if (!vq->use_map_api)
538	return (dma_addr_t)virt_to_phys(cpu_addr);
539
540	return virtqueue_map_single_attrs(&vq->vq, cpu_addr,
541	size, direction, 0);
542	}
543
544	static void virtqueue_init(struct vring_virtqueue *vq, u32 num)
545	{
546	vq->vq.num_free = num;
547
548	if (virtqueue_is_packed(vq))
549	vq->last_used_idx = 0 \| (1 << VRING_PACKED_EVENT_F_WRAP_CTR);
550	else
551	vq->last_used_idx = 0;
552
553	vq->last_used = 0;
554
555	vq->event_triggered = false;
556	vq->num_added = 0;
557
558	#ifdef DEBUG
559	vq->in_use = false;
560	vq->last_add_time_valid = false;
561	#endif
562	}
563
564
565	/*
566	* Split ring specific functions - *_split().
567	*/
568
569	static unsigned int vring_unmap_one_split(const struct vring_virtqueue *vq,
570	struct vring_desc_extra *extra)
571	{
572	u16 flags;
573
574	flags = extra->flags;
575
576	if (flags & VRING_DESC_F_INDIRECT) {
577	if (!vq->use_map_api)
578	goto out;
579	} else if (!vring_need_unmap_buffer(vq, extra))
580	goto out;
581
582	virtqueue_unmap_page_attrs(&vq->vq,
583	extra->addr,
584	extra->len,
585	(flags & VRING_DESC_F_WRITE) ?
586	DMA_FROM_DEVICE : DMA_TO_DEVICE,
587	0);
588
589	out:
590	return extra->next;
591	}
592
593	static struct vring_desc alloc_indirect_split(struct vring_virtqueue vq,
594	unsigned int total_sg,
595	gfp_t gfp)
596	{
597	struct vring_desc_extra *extra;
598	struct vring_desc *desc;
599	unsigned int i, size;
600
601	/*
602	* We require lowmem mappings for the descriptors because
603	* otherwise virt_to_phys will give us bogus addresses in the
604	* virtqueue.
605	*/
606	gfp &= ~__GFP_HIGHMEM;
607
608	size = sizeof(desc) total_sg + sizeof(extra) total_sg;
609
610	desc = kmalloc(size, gfp);
611	if (!desc)
612	return NULL;
613
614	extra = (struct vring_desc_extra *)&desc[total_sg];
615
616	for (i = 0; i < total_sg; i++)
617	extra[i].next = i + 1;
618
619	return desc;
620	}
621
622	static inline unsigned int virtqueue_add_desc_split(struct vring_virtqueue *vq,
623	struct vring_desc *desc,
624	struct vring_desc_extra *extra,
625	unsigned int i,
626	dma_addr_t addr,
627	unsigned int len,
628	u16 flags, bool premapped)
629	{
630	struct virtio_device *vdev = vq->vq.vdev;
631	u16 next;
632
633	desc[i].flags = cpu_to_virtio16(vdev, flags);
634	desc[i].addr = cpu_to_virtio64(vdev, addr);
635	desc[i].len = cpu_to_virtio32(vdev, len);
636
637	extra[i].addr = premapped ? DMA_MAPPING_ERROR : addr;
638	extra[i].len = len;
639	extra[i].flags = flags;
640
641	next = extra[i].next;
642
643	desc[i].next = cpu_to_virtio16(vdev, next);
644
645	return next;
646	}
647
648	static inline int virtqueue_add_split(struct vring_virtqueue *vq,
649	struct scatterlist *sgs[],
650	unsigned int total_sg,
651	unsigned int out_sgs,
652	unsigned int in_sgs,
653	void *data,
654	void *ctx,
655	bool premapped,
656	gfp_t gfp,
657	unsigned long attr)
658	{
659	struct vring_desc_extra *extra;
660	struct scatterlist *sg;
661	struct vring_desc *desc;
662	unsigned int i, n, avail, descs_used, err_idx, sg_count = 0;
663	/* Total length for in-order */
664	unsigned int total_in_len = 0;
665	int head;
666	bool indirect;
667
668	START_USE(vq);
669
670	BUG_ON(data == NULL);
671	BUG_ON(ctx && vq->indirect);
672
673	if (unlikely(vq->broken)) {
674	END_USE(vq);
675	return -EIO;
676	}
677
678	LAST_ADD_TIME_UPDATE(vq);
679
680	BUG_ON(total_sg == 0);
681
682	head = vq->free_head;
683
684	if (virtqueue_use_indirect(vq, total_sg))
685	desc = alloc_indirect_split(vq, total_sg, gfp);
686	else {
687	desc = NULL;
688	WARN_ON_ONCE(total_sg > vq->split.vring.num && !vq->indirect);
689	}
690
691	if (desc) {
692	/* Use a single buffer which doesn't continue */
693	indirect = true;
694	/* Set up rest to use this indirect table. */
695	i = 0;
696	descs_used = 1;
697	extra = (struct vring_desc_extra *)&desc[total_sg];
698	} else {
699	indirect = false;
700	desc = vq->split.vring.desc;
701	extra = vq->split.desc_extra;
702	i = head;
703	descs_used = total_sg;
704	}
705
706	if (unlikely(vq->vq.num_free < descs_used)) {
707	pr_debug("Can't add buf len %i - avail = %i\n",
708	descs_used, vq->vq.num_free);
709	/* FIXME: for historical reasons, we force a notify here if
710	* there are outgoing parts to the buffer. Presumably the
711	* host should service the ring ASAP. */
712	if (out_sgs)
713	vq->notify(&vq->vq);
714	if (indirect)
715	kfree(desc);
716	END_USE(vq);
717	return -ENOSPC;
718	}
719
720	for (n = 0; n < out_sgs; n++) {
721	for (sg = sgs[n]; sg; sg = sg_next(sg)) {
722	dma_addr_t addr;
723	u32 len;
724	u16 flags = 0;
725
726	if (++sg_count != total_sg)
727	flags \|= VRING_DESC_F_NEXT;
728
729	if (vring_map_one_sg(vq, sg, DMA_TO_DEVICE, &addr, &len,
730	premapped, attr))
731	goto unmap_release;
732
733	/* Note that we trust indirect descriptor
734	* table since it use stream DMA mapping.
735	*/
736	i = virtqueue_add_desc_split(vq, desc, extra, i, addr,
737	len, flags, premapped);
738	}
739	}
740	for (; n < (out_sgs + in_sgs); n++) {
741	for (sg = sgs[n]; sg; sg = sg_next(sg)) {
742	dma_addr_t addr;
743	u32 len;
744	u16 flags = VRING_DESC_F_WRITE;
745
746	if (++sg_count != total_sg)
747	flags \|= VRING_DESC_F_NEXT;
748
749	if (vring_map_one_sg(vq, sg, DMA_FROM_DEVICE, &addr, &len,
750	premapped, attr))
751	goto unmap_release;
752
753	/* Note that we trust indirect descriptor
754	* table since it use stream DMA mapping.
755	*/
756	i = virtqueue_add_desc_split(vq, desc, extra, i, addr,
757	len, flags, premapped);
758	total_in_len += len;
759	}
760	}
761
762	if (indirect) {
763	/* Now that the indirect table is filled in, map it. */
764	dma_addr_t addr = vring_map_single(
765	vq, desc, total_sg * sizeof(struct vring_desc),
766	DMA_TO_DEVICE);
767	if (vring_mapping_error(vq, addr))
768	goto unmap_release;
769
770	virtqueue_add_desc_split(vq, vq->split.vring.desc,
771	vq->split.desc_extra,
772	head, addr,
773	total_sg * sizeof(struct vring_desc),
774	VRING_DESC_F_INDIRECT, false);
775	}
776
777	/* We're using some buffers from the free list. */
778	vq->vq.num_free -= descs_used;
779
780	/* Update free pointer */
781	if (virtqueue_is_in_order(vq)) {
782	vq->free_head += descs_used;
783	if (vq->free_head >= vq->split.vring.num)
784	vq->free_head -= vq->split.vring.num;
785	vq->split.desc_state[head].total_in_len = total_in_len;
786	} else if (indirect)
787	vq->free_head = vq->split.desc_extra[head].next;
788	else
789	vq->free_head = i;
790
791	/* Store token and indirect buffer state. */
792	vq->split.desc_state[head].data = data;
793	if (indirect)
794	vq->split.desc_state[head].indir_desc = desc;
795	else
796	vq->split.desc_state[head].indir_desc = ctx;
797
798	/* Put entry in available array (but don't update avail->idx until they
799	* do sync). */
800	avail = vq->split.avail_idx_shadow & (vq->split.vring.num - 1);
801	vq->split.vring.avail->ring[avail] = cpu_to_virtio16(vq->vq.vdev, head);
802
803	/* Descriptors and available array need to be set before we expose the
804	* new available array entries. */
805	virtio_wmb(vq->weak_barriers);
806	vq->split.avail_idx_shadow++;
807	vq->split.vring.avail->idx = cpu_to_virtio16(vq->vq.vdev,
808	vq->split.avail_idx_shadow);
809	vq->num_added++;
810
811	pr_debug("Added buffer head %i to %p\n", head, vq);
812	END_USE(vq);
813
814	/* This is very unlikely, but theoretically possible. Kick
815	* just in case. */
816	if (unlikely(vq->num_added == (1 << 16) - 1))
817	virtqueue_kick(&vq->vq);
818
819	return 0;
820
821	unmap_release:
822	err_idx = i;
823
824	if (indirect)
825	i = 0;
826	else
827	i = head;
828
829	for (n = 0; n < total_sg; n++) {
830	if (i == err_idx)
831	break;
832
833	i = vring_unmap_one_split(vq, &extra[i]);
834	}
835
836	if (indirect)
837	kfree(desc);
838
839	END_USE(vq);
840	return -ENOMEM;
841	}
842
843	static bool virtqueue_kick_prepare_split(struct vring_virtqueue *vq)
844	{
845	u16 new, old;
846	bool needs_kick;
847
848	START_USE(vq);
849	/* We need to expose available array entries before checking avail
850	* event. */
851	virtio_mb(vq->weak_barriers);
852
853	old = vq->split.avail_idx_shadow - vq->num_added;
854	new = vq->split.avail_idx_shadow;
855	vq->num_added = 0;
856
857	LAST_ADD_TIME_CHECK(vq);
858	LAST_ADD_TIME_INVALID(vq);
859
860	if (vq->event) {
861	needs_kick = vring_need_event(vring_read_split_avail_event(vq),
862	new, old);
863	} else {
864	needs_kick = !(vq->split.vring.used->flags &
865	cpu_to_virtio16(vq->vq.vdev,
866	VRING_USED_F_NO_NOTIFY));
867	}
868	END_USE(vq);
869	return needs_kick;
870	}
871
872	static void detach_indirect_split(struct vring_virtqueue *vq,
873	unsigned int head)
874	{
875	struct vring_desc_extra *extra = vq->split.desc_extra;
876	struct vring_desc *indir_desc = vq->split.desc_state[head].indir_desc;
877	unsigned int j;
878	u32 len, num;
879
880	/* Free the indirect table, if any, now that it's unmapped. */
881	if (!indir_desc)
882	return;
883	len = vq->split.desc_extra[head].len;
884
885	BUG_ON(!(vq->split.desc_extra[head].flags &
886	VRING_DESC_F_INDIRECT));
887	BUG_ON(len == 0 \|\| len % sizeof(struct vring_desc));
888
889	num = len / sizeof(struct vring_desc);
890
891	extra = (struct vring_desc_extra *)&indir_desc[num];
892
893	if (vq->use_map_api) {
894	for (j = 0; j < num; j++)
895	vring_unmap_one_split(vq, &extra[j]);
896	}
897
898	kfree(indir_desc);
899	vq->split.desc_state[head].indir_desc = NULL;
900	}
901
902	static unsigned detach_buf_split_in_order(struct vring_virtqueue *vq,
903	unsigned int head,
904	void **ctx)
905	{
906	struct vring_desc_extra *extra;
907	unsigned int i;
908	__virtio16 nextflag = cpu_to_virtio16(vq->vq.vdev, VRING_DESC_F_NEXT);
909
910	/* Clear data ptr. */
911	vq->split.desc_state[head].data = NULL;
912
913	extra = vq->split.desc_extra;
914
915	/* Put back on free list: unmap first-level descriptors and find end */
916	i = head;
917
918	while (vq->split.vring.desc[i].flags & nextflag) {
919	i = vring_unmap_one_split(vq, &extra[i]);
920	vq->vq.num_free++;
921	}
922
923	vring_unmap_one_split(vq, &extra[i]);
924
925	/* Plus final descriptor */
926	vq->vq.num_free++;
927
928	if (vq->indirect)
929	detach_indirect_split(vq, head);
930	else if (ctx)
931	*ctx = vq->split.desc_state[head].indir_desc;
932
933	return i;
934	}
935
936	static void detach_buf_split(struct vring_virtqueue *vq, unsigned int head,
937	void **ctx)
938	{
939	unsigned int i = detach_buf_split_in_order(vq, head, ctx);
940
941	vq->split.desc_extra[i].next = vq->free_head;
942	vq->free_head = head;
943	}
944
945	static bool virtqueue_poll_split(const struct vring_virtqueue *vq,
946	unsigned int last_used_idx)
947	{
948	return (u16)last_used_idx != vring_read_split_used_idx(vq);
949	}
950
951	static bool more_used_split(const struct vring_virtqueue *vq)
952	{
953	return virtqueue_poll_split(vq, vq->last_used_idx);
954	}
955
956	static bool more_used_split_in_order(const struct vring_virtqueue *vq)
957	{
958	if (vq->batch_last.id != UINT_MAX)
959	return true;
960
961	return virtqueue_poll_split(vq, vq->last_used_idx);
962	}
963
964	static void virtqueue_get_buf_ctx_split(struct vring_virtqueue vq,
965	unsigned int *len,
966	void **ctx)
967	{
968	void *ret;
969	unsigned int i;
970	u16 last_used;
971
972	START_USE(vq);
973
974	if (unlikely(vq->broken)) {
975	END_USE(vq);
976	return NULL;
977	}
978
979	if (!more_used_split(vq)) {
980	pr_debug("No more buffers in queue\n");
981	END_USE(vq);
982	return NULL;
983	}
984
985	/* Only get used array entries after they have been exposed by host. */
986	virtio_rmb(vq->weak_barriers);
987
988	last_used = (vq->last_used_idx & (vq->split.vring.num - 1));
989	i = vring_read_split_used_id(vq, last_used);
990	*len = vring_read_split_used_len(vq, last_used);
991
992	if (unlikely(i >= vq->split.vring.num)) {
993	BAD_RING(vq, "id %u out of range\n", i);
994	return NULL;
995	}
996	if (unlikely(!vq->split.desc_state[i].data)) {
997	BAD_RING(vq, "id %u is not a head!\n", i);
998	return NULL;
999	}
1000
1001	/* detach_buf_split clears data, so grab it now. */
1002	ret = vq->split.desc_state[i].data;
1003	detach_buf_split(vq, i, ctx);
1004	vq->last_used_idx++;
1005	/* If we expect an interrupt for the next entry, tell host
1006	* by writing event index and flush out the write before
1007	* the read in the next get_buf call. */
1008	if (!(vq->split.avail_flags_shadow & VRING_AVAIL_F_NO_INTERRUPT))
1009	virtio_store_mb(vq->weak_barriers,
1010	&vring_used_event(&vq->split.vring),
1011	cpu_to_virtio16(vq->vq.vdev, vq->last_used_idx));
1012
1013	LAST_ADD_TIME_INVALID(vq);
1014
1015	END_USE(vq);
1016	return ret;
1017	}
1018
1019	static void virtqueue_get_buf_ctx_split_in_order(struct vring_virtqueue vq,
1020	unsigned int *len,
1021	void **ctx)
1022	{
1023	void *ret;
1024	unsigned int num = vq->split.vring.num;
1025	unsigned int num_free = vq->vq.num_free;
1026	u16 last_used, last_used_idx;
1027
1028	START_USE(vq);
1029
1030	if (unlikely(vq->broken)) {
1031	END_USE(vq);
1032	return NULL;
1033	}
1034
1035	last_used = vq->last_used & (num - 1);
1036	last_used_idx = vq->last_used_idx & (num - 1);
1037
1038	if (vq->batch_last.id == UINT_MAX) {
1039	if (!more_used_split_in_order(vq)) {
1040	pr_debug("No more buffers in queue\n");
1041	END_USE(vq);
1042	return NULL;
1043	}
1044
1045	/*
1046	* Only get used array entries after they have been
1047	* exposed by host.
1048	*/
1049	virtio_rmb(vq->weak_barriers);
1050
1051	vq->batch_last.id = vring_read_split_used_id(vq, last_used_idx);
1052	vq->batch_last.len = vring_read_split_used_len(vq, last_used_idx);
1053	}
1054
1055	if (vq->batch_last.id == last_used) {
1056	vq->batch_last.id = UINT_MAX;
1057	*len = vq->batch_last.len;
1058	} else {
1059	*len = vq->split.desc_state[last_used].total_in_len;
1060	}
1061
1062	if (unlikely(!vq->split.desc_state[last_used].data)) {
1063	BAD_RING(vq, "id %u is not a head!\n", last_used);
1064	return NULL;
1065	}
1066
1067	/* detach_buf_split clears data, so grab it now. */
1068	ret = vq->split.desc_state[last_used].data;
1069	detach_buf_split_in_order(vq, last_used, ctx);
1070
1071	vq->last_used_idx++;
1072	vq->last_used += (vq->vq.num_free - num_free);
1073	/* If we expect an interrupt for the next entry, tell host
1074	* by writing event index and flush out the write before
1075	* the read in the next get_buf call. */
1076	if (!(vq->split.avail_flags_shadow & VRING_AVAIL_F_NO_INTERRUPT))
1077	virtio_store_mb(vq->weak_barriers,
1078	&vring_used_event(&vq->split.vring),
1079	cpu_to_virtio16(vq->vq.vdev, vq->last_used_idx));
1080
1081	LAST_ADD_TIME_INVALID(vq);
1082
1083	END_USE(vq);
1084	return ret;
1085	}
1086
1087	static void virtqueue_disable_cb_split(struct vring_virtqueue *vq)
1088	{
1089	if (!(vq->split.avail_flags_shadow & VRING_AVAIL_F_NO_INTERRUPT)) {
1090	vq->split.avail_flags_shadow \|= VRING_AVAIL_F_NO_INTERRUPT;
1091
1092	/*
1093	* If device triggered an event already it won't trigger one again:
1094	* no need to disable.
1095	*/
1096	if (vq->event_triggered)
1097	return;
1098
1099	if (vq->event)
1100	/* TODO: this is a hack. Figure out a cleaner value to write. */
1101	vring_used_event(&vq->split.vring) = 0x0;
1102	else
1103	vq->split.vring.avail->flags =
1104	cpu_to_virtio16(vq->vq.vdev,
1105	vq->split.avail_flags_shadow);
1106	}
1107	}
1108
1109	static unsigned int virtqueue_enable_cb_prepare_split(struct vring_virtqueue *vq)
1110	{
1111	u16 last_used_idx;
1112
1113	START_USE(vq);
1114
1115	/* We optimistically turn back on interrupts, then check if there was
1116	* more to do. */
1117	/* Depending on the VIRTIO_RING_F_EVENT_IDX feature, we need to
1118	* either clear the flags bit or point the event index at the next
1119	* entry. Always do both to keep code simple. */
1120	if (vq->split.avail_flags_shadow & VRING_AVAIL_F_NO_INTERRUPT) {
1121	vq->split.avail_flags_shadow &= ~VRING_AVAIL_F_NO_INTERRUPT;
1122	if (!vq->event)
1123	vq->split.vring.avail->flags =
1124	cpu_to_virtio16(vq->vq.vdev,
1125	vq->split.avail_flags_shadow);
1126	}
1127	vring_used_event(&vq->split.vring) = cpu_to_virtio16(vq->vq.vdev,
1128	last_used_idx = vq->last_used_idx);
1129	END_USE(vq);
1130	return last_used_idx;
1131	}
1132
1133	static bool virtqueue_enable_cb_delayed_split(struct vring_virtqueue *vq)
1134	{
1135	u16 bufs;
1136
1137	START_USE(vq);
1138
1139	/* We optimistically turn back on interrupts, then check if there was
1140	* more to do. */
1141	/* Depending on the VIRTIO_RING_F_USED_EVENT_IDX feature, we need to
1142	* either clear the flags bit or point the event index at the next
1143	* entry. Always update the event index to keep code simple. */
1144	if (vq->split.avail_flags_shadow & VRING_AVAIL_F_NO_INTERRUPT) {
1145	vq->split.avail_flags_shadow &= ~VRING_AVAIL_F_NO_INTERRUPT;
1146	if (!vq->event)
1147	vq->split.vring.avail->flags =
1148	cpu_to_virtio16(vq->vq.vdev,
1149	vq->split.avail_flags_shadow);
1150	}
1151	/* TODO: tune this threshold */
1152	bufs = (u16)(vq->split.avail_idx_shadow - vq->last_used_idx) * 3 / 4;
1153
1154	virtio_store_mb(vq->weak_barriers,
1155	&vring_used_event(&vq->split.vring),
1156	cpu_to_virtio16(vq->vq.vdev, vq->last_used_idx + bufs));
1157
1158	if (unlikely((u16)(vring_read_split_used_idx(vq)
1159	- vq->last_used_idx) > bufs)) {
1160	END_USE(vq);
1161	return false;
1162	}
1163
1164	END_USE(vq);
1165	return true;
1166	}
1167
1168	static void virtqueue_detach_unused_buf_split(struct vring_virtqueue vq)
1169	{
1170	unsigned int i;
1171	void *buf;
1172
1173	START_USE(vq);
1174
1175	for (i = 0; i < vq->split.vring.num; i++) {
1176	if (!vq->split.desc_state[i].data)
1177	continue;
1178	/* detach_buf_split clears data, so grab it now. */
1179	buf = vq->split.desc_state[i].data;
1180	if (virtqueue_is_in_order(vq))
1181	detach_buf_split_in_order(vq, i, NULL);
1182	else
1183	detach_buf_split(vq, i, NULL);
1184	vq->split.avail_idx_shadow--;
1185	vq->split.vring.avail->idx = cpu_to_virtio16(vq->vq.vdev,
1186	vq->split.avail_idx_shadow);
1187	END_USE(vq);
1188	return buf;
1189	}
1190	/* That should have freed everything. */
1191	BUG_ON(vq->vq.num_free != vq->split.vring.num);
1192
1193	END_USE(vq);
1194	return NULL;
1195	}
1196
1197	static void virtqueue_vring_init_split(struct vring_virtqueue_split *vring_split,
1198	struct vring_virtqueue *vq)
1199	{
1200	struct virtio_device *vdev;
1201
1202	vdev = vq->vq.vdev;
1203
1204	vring_split->avail_flags_shadow = 0;
1205	vring_split->avail_idx_shadow = 0;
1206
1207	/* No callback? Tell other side not to bother us. */
1208	if (!vq->vq.callback) {
1209	vring_split->avail_flags_shadow \|= VRING_AVAIL_F_NO_INTERRUPT;
1210	if (!vq->event)
1211	vring_split->vring.avail->flags = cpu_to_virtio16(vdev,
1212	vring_split->avail_flags_shadow);
1213	}
1214	}
1215
1216	static void virtqueue_reset_split(struct vring_virtqueue *vq)
1217	{
1218	int num;
1219
1220	num = vq->split.vring.num;
1221
1222	vq->split.vring.avail->flags = 0;
1223	vq->split.vring.avail->idx = 0;
1224
1225	/* reset avail event */
1226	vq->split.vring.avail->ring[num] = 0;
1227
1228	vq->split.vring.used->flags = 0;
1229	vq->split.vring.used->idx = 0;
1230
1231	/* reset used event */
1232	(__virtio16 )&(vq->split.vring.used->ring[num]) = 0;
1233
1234	virtqueue_init(vq, num);
1235
1236	virtqueue_vring_init_split(&vq->split, vq);
1237	}
1238
1239	static void virtqueue_vring_attach_split(struct vring_virtqueue *vq,
1240	struct vring_virtqueue_split *vring_split)
1241	{
1242	vq->split = *vring_split;
1243
1244	/* Put everything in free lists. */
1245	vq->free_head = 0;
1246	vq->batch_last.id = UINT_MAX;
1247	}
1248
1249	static int vring_alloc_state_extra_split(struct vring_virtqueue_split *vring_split)
1250	{
1251	struct vring_desc_state_split *state;
1252	struct vring_desc_extra *extra;
1253	u32 num = vring_split->vring.num;
1254
1255	state = kmalloc_objs(struct vring_desc_state_split, num);
1256	if (!state)
1257	goto err_state;
1258
1259	extra = vring_alloc_desc_extra(num);
1260	if (!extra)
1261	goto err_extra;
1262
1263	memset(state, 0, num * sizeof(struct vring_desc_state_split));
1264
1265	vring_split->desc_state = state;
1266	vring_split->desc_extra = extra;
1267	return 0;
1268
1269	err_extra:
1270	kfree(state);
1271	err_state:
1272	return -ENOMEM;
1273	}
1274
1275	static void vring_free_split(struct vring_virtqueue_split *vring_split,
1276	struct virtio_device *vdev,
1277	union virtio_map map)
1278	{
1279	vring_free_queue(vdev, vring_split->queue_size_in_bytes,
1280	vring_split->vring.desc,
1281	vring_split->queue_dma_addr,
1282	map);
1283
1284	kfree(vring_split->desc_state);
1285	kfree(vring_split->desc_extra);
1286	}
1287
1288	static int vring_alloc_queue_split(struct vring_virtqueue_split *vring_split,
1289	struct virtio_device *vdev,
1290	u32 num,
1291	unsigned int vring_align,
1292	bool may_reduce_num,
1293	union virtio_map map)
1294	{
1295	void *queue = NULL;
1296	dma_addr_t dma_addr;
1297
1298	/* We assume num is a power of 2. */
1299	if (!is_power_of_2(num)) {
1300	dev_warn(&vdev->dev, "Bad virtqueue length %u\n", num);
1301	return -EINVAL;
1302	}
1303
1304	/* TODO: allocate each queue chunk individually */
1305	for (; num && vring_size(num, vring_align) > PAGE_SIZE; num /= 2) {
1306	queue = vring_alloc_queue(vdev, vring_size(num, vring_align),
1307	&dma_addr,
1308	GFP_KERNEL \| __GFP_NOWARN \| __GFP_ZERO,
1309	map);
1310	if (queue)
1311	break;
1312	if (!may_reduce_num)
1313	return -ENOMEM;
1314	}
1315
1316	if (!num)
1317	return -ENOMEM;
1318
1319	if (!queue) {
1320	/* Try to get a single page. You are my only hope! */
1321	queue = vring_alloc_queue(vdev, vring_size(num, vring_align),
1322	&dma_addr, GFP_KERNEL \| __GFP_ZERO,
1323	map);
1324	}
1325	if (!queue)
1326	return -ENOMEM;
1327
1328	vring_init(&vring_split->vring, num, queue, vring_align);
1329
1330	vring_split->queue_dma_addr = dma_addr;
1331	vring_split->queue_size_in_bytes = vring_size(num, vring_align);
1332
1333	vring_split->vring_align = vring_align;
1334	vring_split->may_reduce_num = may_reduce_num;
1335
1336	return 0;
1337	}
1338
1339	static const struct virtqueue_ops split_ops;
1340
1341	static struct virtqueue *__vring_new_virtqueue_split(unsigned int index,
1342	struct vring_virtqueue_split *vring_split,
1343	struct virtio_device *vdev,
1344	bool weak_barriers,
1345	bool context,
1346	bool (notify)(struct virtqueue ),
1347	void (callback)(struct virtqueue ),
1348	const char *name,
1349	union virtio_map map)
1350	{
1351	struct vring_virtqueue *vq;
1352	int err;
1353
1354	vq = kmalloc_obj(*vq);
1355	if (!vq)
1356	return NULL;
1357
1358	vq->vq.callback = callback;
1359	vq->vq.vdev = vdev;
1360	vq->vq.name = name;
1361	vq->vq.index = index;
1362	vq->vq.reset = false;
1363	vq->we_own_ring = false;
1364	vq->notify = notify;
1365	vq->weak_barriers = weak_barriers;
1366	#ifdef CONFIG_VIRTIO_HARDEN_NOTIFICATION
1367	vq->broken = true;
1368	#else
1369	vq->broken = false;
1370	#endif
1371	vq->map = map;
1372	vq->use_map_api = vring_use_map_api(vdev);
1373
1374	vq->indirect = virtio_has_feature(vdev, VIRTIO_RING_F_INDIRECT_DESC) &&
1375	!context;
1376	vq->event = virtio_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX);
1377	vq->layout = virtio_has_feature(vdev, VIRTIO_F_IN_ORDER) ?
1378	VQ_LAYOUT_SPLIT_IN_ORDER : VQ_LAYOUT_SPLIT;
1379
1380	if (virtio_has_feature(vdev, VIRTIO_F_ORDER_PLATFORM))
1381	vq->weak_barriers = false;
1382
1383	err = vring_alloc_state_extra_split(vring_split);
1384	if (err) {
1385	kfree(vq);
1386	return NULL;
1387	}
1388
1389	virtqueue_vring_init_split(vring_split, vq);
1390
1391	virtqueue_init(vq, vring_split->vring.num);
1392	virtqueue_vring_attach_split(vq, vring_split);
1393
1394	spin_lock(&vdev->vqs_list_lock);
1395	list_add_tail(&vq->vq.list, &vdev->vqs);
1396	spin_unlock(&vdev->vqs_list_lock);
1397	return &vq->vq;
1398	}
1399
1400	static struct virtqueue *vring_create_virtqueue_split(
1401	unsigned int index,
1402	unsigned int num,
1403	unsigned int vring_align,
1404	struct virtio_device *vdev,
1405	bool weak_barriers,
1406	bool may_reduce_num,
1407	bool context,
1408	bool (notify)(struct virtqueue ),
1409	void (callback)(struct virtqueue ),
1410	const char *name,
1411	union virtio_map map)
1412	{
1413	struct vring_virtqueue_split vring_split = {};
1414	struct virtqueue *vq;
1415	int err;
1416
1417	err = vring_alloc_queue_split(&vring_split, vdev, num, vring_align,
1418	may_reduce_num, map);
1419	if (err)
1420	return NULL;
1421
1422	vq = __vring_new_virtqueue_split(index, &vring_split, vdev, weak_barriers,
1423	context, notify, callback, name, map);
1424	if (!vq) {
1425	vring_free_split(&vring_split, vdev, map);
1426	return NULL;
1427	}
1428
1429	to_vvq(vq)->we_own_ring = true;
1430
1431	return vq;
1432	}
1433
1434	static int virtqueue_resize_split(struct vring_virtqueue *vq, u32 num)
1435	{
1436	struct vring_virtqueue_split vring_split = {};
1437	struct virtio_device *vdev = vq->vq.vdev;
1438	int err;
1439
1440	err = vring_alloc_queue_split(&vring_split, vdev, num,
1441	vq->split.vring_align,
1442	vq->split.may_reduce_num,
1443	vq->map);
1444	if (err)
1445	goto err;
1446
1447	err = vring_alloc_state_extra_split(&vring_split);
1448	if (err)
1449	goto err_state_extra;
1450
1451	vring_free(&vq->vq);
1452
1453	virtqueue_vring_init_split(&vring_split, vq);
1454
1455	virtqueue_init(vq, vring_split.vring.num);
1456	virtqueue_vring_attach_split(vq, &vring_split);
1457
1458	return 0;
1459
1460	err_state_extra:
1461	vring_free_split(&vring_split, vdev, vq->map);
1462	err:
1463	virtqueue_reset_split(vq);
1464	return -ENOMEM;
1465	}
1466
1467
1468	/*
1469	* Packed ring specific functions - *_packed().
1470	*/
1471	static bool packed_used_wrap_counter(u16 last_used_idx)
1472	{
1473	return !!(last_used_idx & (1 << VRING_PACKED_EVENT_F_WRAP_CTR));
1474	}
1475
1476	static u16 packed_last_used(u16 last_used_idx)
1477	{
1478	return last_used_idx & ~(-(1 << VRING_PACKED_EVENT_F_WRAP_CTR));
1479	}
1480
1481	static void vring_unmap_extra_packed(const struct vring_virtqueue *vq,
1482	const struct vring_desc_extra *extra)
1483	{
1484	u16 flags;
1485
1486	flags = extra->flags;
1487
1488	if (flags & VRING_DESC_F_INDIRECT) {
1489	if (!vq->use_map_api)
1490	return;
1491	} else if (!vring_need_unmap_buffer(vq, extra))
1492	return;
1493
1494	virtqueue_unmap_page_attrs(&vq->vq,
1495	extra->addr, extra->len,
1496	(flags & VRING_DESC_F_WRITE) ?
1497	DMA_FROM_DEVICE : DMA_TO_DEVICE,
1498	0);
1499	}
1500
1501	static struct vring_packed_desc *alloc_indirect_packed(unsigned int total_sg,
1502	gfp_t gfp)
1503	{
1504	struct vring_desc_extra *extra;
1505	struct vring_packed_desc *desc;
1506	int i, size;
1507
1508	/*
1509	* We require lowmem mappings for the descriptors because
1510	* otherwise virt_to_phys will give us bogus addresses in the
1511	* virtqueue.
1512	*/
1513	gfp &= ~__GFP_HIGHMEM;
1514
1515	size = (sizeof(desc) + sizeof(extra)) * total_sg;
1516
1517	desc = kmalloc(size, gfp);
1518	if (!desc)
1519	return NULL;
1520
1521	extra = (struct vring_desc_extra *)&desc[total_sg];
1522
1523	for (i = 0; i < total_sg; i++)
1524	extra[i].next = i + 1;
1525
1526	return desc;
1527	}
1528
1529	static int virtqueue_add_indirect_packed(struct vring_virtqueue *vq,
1530	struct scatterlist *sgs[],
1531	unsigned int total_sg,
1532	unsigned int out_sgs,
1533	unsigned int in_sgs,
1534	void *data,
1535	bool premapped,
1536	gfp_t gfp,
1537	u16 id,
1538	unsigned long attr)
1539	{
1540	struct vring_desc_extra *extra;
1541	struct vring_packed_desc *desc;
1542	struct scatterlist *sg;
1543	unsigned int i, n, err_idx, len, total_in_len = 0;
1544	u16 head;
1545	dma_addr_t addr;
1546
1547	head = vq->packed.next_avail_idx;
1548	desc = alloc_indirect_packed(total_sg, gfp);
1549	if (!desc)
1550	return -ENOMEM;
1551
1552	extra = (struct vring_desc_extra *)&desc[total_sg];
1553
1554	if (unlikely(vq->vq.num_free < 1)) {
1555	pr_debug("Can't add buf len 1 - avail = 0\n");
1556	kfree(desc);
1557	END_USE(vq);
1558	return -ENOSPC;
1559	}
1560
1561	i = 0;
1562
1563	for (n = 0; n < out_sgs + in_sgs; n++) {
1564	for (sg = sgs[n]; sg; sg = sg_next(sg)) {
1565	if (vring_map_one_sg(vq, sg, n < out_sgs ?
1566	DMA_TO_DEVICE : DMA_FROM_DEVICE,
1567	&addr, &len, premapped, attr))
1568	goto unmap_release;
1569
1570	desc[i].flags = cpu_to_le16(n < out_sgs ?
1571	0 : VRING_DESC_F_WRITE);
1572	desc[i].addr = cpu_to_le64(addr);
1573	desc[i].len = cpu_to_le32(len);
1574
1575	if (unlikely(vq->use_map_api)) {
1576	extra[i].addr = premapped ? DMA_MAPPING_ERROR : addr;
1577	extra[i].len = len;
1578	extra[i].flags = n < out_sgs ? 0 : VRING_DESC_F_WRITE;
1579	}
1580
1581	if (n >= out_sgs)
1582	total_in_len += len;
1583	i++;
1584	}
1585	}
1586
1587	/* Now that the indirect table is filled in, map it. */
1588	addr = vring_map_single(vq, desc,
1589	total_sg * sizeof(struct vring_packed_desc),
1590	DMA_TO_DEVICE);
1591	if (vring_mapping_error(vq, addr))
1592	goto unmap_release;
1593
1594	vq->packed.vring.desc[head].addr = cpu_to_le64(addr);
1595	vq->packed.vring.desc[head].len = cpu_to_le32(total_sg *
1596	sizeof(struct vring_packed_desc));
1597	vq->packed.vring.desc[head].id = cpu_to_le16(id);
1598
1599	if (vq->use_map_api) {
1600	vq->packed.desc_extra[id].addr = addr;
1601	vq->packed.desc_extra[id].len = total_sg *
1602	sizeof(struct vring_packed_desc);
1603	vq->packed.desc_extra[id].flags = VRING_DESC_F_INDIRECT \|
1604	vq->packed.avail_used_flags;
1605	}
1606
1607	/*
1608	* A driver MUST NOT make the first descriptor in the list
1609	* available before all subsequent descriptors comprising
1610	* the list are made available.
1611	*/
1612	virtio_wmb(vq->weak_barriers);
1613	vq->packed.vring.desc[head].flags = cpu_to_le16(VRING_DESC_F_INDIRECT \|
1614	vq->packed.avail_used_flags);
1615
1616	/* We're using some buffers from the free list. */
1617	vq->vq.num_free -= 1;
1618
1619	/* Update free pointer */
1620	n = head + 1;
1621	if (n >= vq->packed.vring.num) {
1622	n = 0;
1623	vq->packed.avail_wrap_counter ^= 1;
1624	vq->packed.avail_used_flags ^=
1625	1 << VRING_PACKED_DESC_F_AVAIL \|
1626	1 << VRING_PACKED_DESC_F_USED;
1627	}
1628	vq->packed.next_avail_idx = n;
1629	if (!virtqueue_is_in_order(vq))
1630	vq->free_head = vq->packed.desc_extra[id].next;
1631
1632	/* Store token and indirect buffer state. */
1633	vq->packed.desc_state[id].num = 1;
1634	vq->packed.desc_state[id].data = data;
1635	vq->packed.desc_state[id].indir_desc = desc;
1636	vq->packed.desc_state[id].last = id;
1637	vq->packed.desc_state[id].total_in_len = total_in_len;
1638
1639	vq->num_added += 1;
1640
1641	pr_debug("Added buffer head %i to %p\n", head, vq);
1642	END_USE(vq);
1643
1644	return 0;
1645
1646	unmap_release:
1647	err_idx = i;
1648
1649	for (i = 0; i < err_idx; i++)
1650	vring_unmap_extra_packed(vq, &extra[i]);
1651
1652	kfree(desc);
1653
1654	END_USE(vq);
1655	return -ENOMEM;
1656	}
1657
1658	static inline int virtqueue_add_packed(struct vring_virtqueue *vq,
1659	struct scatterlist *sgs[],
1660	unsigned int total_sg,
1661	unsigned int out_sgs,
1662	unsigned int in_sgs,
1663	void *data,
1664	void *ctx,
1665	bool premapped,
1666	gfp_t gfp,
1667	unsigned long attr)
1668	{
1669	struct vring_packed_desc *desc;
1670	struct scatterlist *sg;
1671	unsigned int i, n, c, descs_used, err_idx, len;
1672	__le16 head_flags, flags;
1673	u16 head, id, prev, curr, avail_used_flags, unpub_flags;
1674	int err;
1675
1676	START_USE(vq);
1677
1678	BUG_ON(data == NULL);
1679	BUG_ON(ctx && vq->indirect);
1680
1681	if (unlikely(vq->broken)) {
1682	END_USE(vq);
1683	return -EIO;
1684	}
1685
1686	LAST_ADD_TIME_UPDATE(vq);
1687
1688	BUG_ON(total_sg == 0);
1689
1690	if (virtqueue_use_indirect(vq, total_sg)) {
1691	id = vq->free_head;
1692	BUG_ON(id == vq->packed.vring.num);
1693	err = virtqueue_add_indirect_packed(vq, sgs, total_sg, out_sgs,
1694	in_sgs, data, premapped, gfp,
1695	id, attr);
1696	if (err != -ENOMEM) {
1697	END_USE(vq);
1698	return err;
1699	}
1700
1701	/* fall back on direct */
1702	}
1703
1704	head = vq->packed.next_avail_idx;
1705	avail_used_flags = vq->packed.avail_used_flags;
1706
1707	WARN_ON_ONCE(total_sg > vq->packed.vring.num && !vq->indirect);
1708
1709	desc = vq->packed.vring.desc;
1710	i = head;
1711	descs_used = total_sg;
1712
1713	if (unlikely(vq->vq.num_free < descs_used)) {
1714	pr_debug("Can't add buf len %i - avail = %i\n",
1715	descs_used, vq->vq.num_free);
1716	END_USE(vq);
1717	return -ENOSPC;
1718	}
1719
1720	id = vq->free_head;
1721	BUG_ON(id == vq->packed.vring.num);
1722
1723	curr = id;
1724	c = 0;
1725	for (n = 0; n < out_sgs + in_sgs; n++) {
1726	for (sg = sgs[n]; sg; sg = sg_next(sg)) {
1727	dma_addr_t addr;
1728
1729	if (vring_map_one_sg(vq, sg, n < out_sgs ?
1730	DMA_TO_DEVICE : DMA_FROM_DEVICE,
1731	&addr, &len, premapped, attr))
1732	goto unmap_release;
1733
1734	flags = cpu_to_le16(vq->packed.avail_used_flags \|
1735	(++c == total_sg ? 0 : VRING_DESC_F_NEXT) \|
1736	(n < out_sgs ? 0 : VRING_DESC_F_WRITE));
1737	if (i == head)
1738	head_flags = flags;
1739	else
1740	desc[i].flags = flags;
1741
1742	desc[i].addr = cpu_to_le64(addr);
1743	desc[i].len = cpu_to_le32(len);
1744	desc[i].id = cpu_to_le16(id);
1745
1746	if (unlikely(vq->use_map_api)) {
1747	vq->packed.desc_extra[curr].addr = premapped ?
1748	DMA_MAPPING_ERROR : addr;
1749	vq->packed.desc_extra[curr].len = len;
1750	vq->packed.desc_extra[curr].flags =
1751	le16_to_cpu(flags);
1752	}
1753	prev = curr;
1754	curr = vq->packed.desc_extra[curr].next;
1755
1756	if ((unlikely(++i >= vq->packed.vring.num))) {
1757	i = 0;
1758	vq->packed.avail_used_flags ^=
1759	1 << VRING_PACKED_DESC_F_AVAIL \|
1760	1 << VRING_PACKED_DESC_F_USED;
1761	}
1762	}
1763	}
1764
1765	if (i <= head)
1766	vq->packed.avail_wrap_counter ^= 1;
1767
1768	/* We're using some buffers from the free list. */
1769	vq->vq.num_free -= descs_used;
1770
1771	/* Update free pointer */
1772	vq->packed.next_avail_idx = i;
1773	vq->free_head = curr;
1774
1775	/* Store token. */
1776	vq->packed.desc_state[id].num = descs_used;
1777	vq->packed.desc_state[id].data = data;
1778	vq->packed.desc_state[id].indir_desc = ctx;
1779	vq->packed.desc_state[id].last = prev;
1780
1781	/*
1782	* A driver MUST NOT make the first descriptor in the list
1783	* available before all subsequent descriptors comprising
1784	* the list are made available.
1785	*/
1786	virtio_wmb(vq->weak_barriers);
1787	vq->packed.vring.desc[head].flags = head_flags;
1788	vq->num_added += descs_used;
1789
1790	pr_debug("Added buffer head %i to %p\n", head, vq);
1791	END_USE(vq);
1792
1793	return 0;
1794
1795	unmap_release:
1796	err_idx = i;
1797	i = head;
1798	curr = vq->free_head;
1799
1800	vq->packed.avail_used_flags = avail_used_flags;
1801	unpub_flags = avail_used_flags ^ (1 << VRING_PACKED_DESC_F_AVAIL \|
1802	1 << VRING_PACKED_DESC_F_USED);
1803
1804	for (n = 0; n < total_sg; n++) {
1805	if (i == err_idx)
1806	break;
1807	/*
1808	* The mapping loop made every descriptor but the head
1809	* available. Stamp the previous wrap counter's AVAIL and USED
1810	* bits on those, so that a later and shorter chain at this head
1811	* does not leave one of them available beyond its own last
1812	* descriptor. Marking them used instead would hand
1813	* is_used_desc_packed() a completion we never made.
1814	*/
1815	if (i != head)
1816	desc[i].flags = cpu_to_le16(unpub_flags);
1817	vring_unmap_extra_packed(vq, &vq->packed.desc_extra[curr]);
1818	curr = vq->packed.desc_extra[curr].next;
1819	i++;
1820	if (i >= vq->packed.vring.num) {
1821	i = 0;
1822	unpub_flags ^= 1 << VRING_PACKED_DESC_F_AVAIL \|
1823	1 << VRING_PACKED_DESC_F_USED;
1824	}
1825	}
1826
1827	END_USE(vq);
1828	return -EIO;
1829	}
1830
1831	static inline int virtqueue_add_packed_in_order(struct vring_virtqueue *vq,
1832	struct scatterlist *sgs[],
1833	unsigned int total_sg,
1834	unsigned int out_sgs,
1835	unsigned int in_sgs,
1836	void *data,
1837	void *ctx,
1838	bool premapped,
1839	gfp_t gfp,
1840	unsigned long attr)
1841	{
1842	struct vring_packed_desc *desc;
1843	struct scatterlist *sg;
1844	unsigned int i, n, sg_count, err_idx, total_in_len = 0;
1845	__le16 head_flags, flags;
1846	u16 head, avail_used_flags, unpub_flags;
1847	bool avail_wrap_counter;
1848	int err;
1849
1850	START_USE(vq);
1851
1852	BUG_ON(data == NULL);
1853	BUG_ON(ctx && vq->indirect);
1854
1855	if (unlikely(vq->broken)) {
1856	END_USE(vq);
1857	return -EIO;
1858	}
1859
1860	LAST_ADD_TIME_UPDATE(vq);
1861
1862	BUG_ON(total_sg == 0);
1863
1864	if (virtqueue_use_indirect(vq, total_sg)) {
1865	err = virtqueue_add_indirect_packed(vq, sgs, total_sg, out_sgs,
1866	in_sgs, data, premapped, gfp,
1867	vq->packed.next_avail_idx,
1868	attr);
1869	if (err != -ENOMEM) {
1870	END_USE(vq);
1871	return err;
1872	}
1873
1874	/* fall back on direct */
1875	}
1876
1877	head = vq->packed.next_avail_idx;
1878	avail_used_flags = vq->packed.avail_used_flags;
1879	avail_wrap_counter = vq->packed.avail_wrap_counter;
1880
1881	WARN_ON_ONCE(total_sg > vq->packed.vring.num && !vq->indirect);
1882
1883	desc = vq->packed.vring.desc;
1884	i = head;
1885
1886	if (unlikely(vq->vq.num_free < total_sg)) {
1887	pr_debug("Can't add buf len %i - avail = %i\n",
1888	total_sg, vq->vq.num_free);
1889	END_USE(vq);
1890	return -ENOSPC;
1891	}
1892
1893	sg_count = 0;
1894	for (n = 0; n < out_sgs + in_sgs; n++) {
1895	for (sg = sgs[n]; sg; sg = sg_next(sg)) {
1896	dma_addr_t addr;
1897	u32 len;
1898
1899	flags = 0;
1900	if (++sg_count != total_sg)
1901	flags \|= cpu_to_le16(VRING_DESC_F_NEXT);
1902	if (n >= out_sgs)
1903	flags \|= cpu_to_le16(VRING_DESC_F_WRITE);
1904
1905	if (vring_map_one_sg(vq, sg, n < out_sgs ?
1906	DMA_TO_DEVICE : DMA_FROM_DEVICE,
1907	&addr, &len, premapped, attr))
1908	goto unmap_release;
1909
1910	flags \|= cpu_to_le16(vq->packed.avail_used_flags);
1911
1912	if (i == head)
1913	head_flags = flags;
1914	else
1915	desc[i].flags = flags;
1916
1917	desc[i].addr = cpu_to_le64(addr);
1918	desc[i].len = cpu_to_le32(len);
1919	desc[i].id = cpu_to_le16(head);
1920
1921	if (unlikely(vq->use_map_api)) {
1922	vq->packed.desc_extra[i].addr = premapped ?
1923	DMA_MAPPING_ERROR : addr;
1924	vq->packed.desc_extra[i].len = len;
1925	vq->packed.desc_extra[i].flags =
1926	le16_to_cpu(flags);
1927	}
1928
1929	if ((unlikely(++i >= vq->packed.vring.num))) {
1930	i = 0;
1931	vq->packed.avail_used_flags ^=
1932	1 << VRING_PACKED_DESC_F_AVAIL \|
1933	1 << VRING_PACKED_DESC_F_USED;
1934	vq->packed.avail_wrap_counter ^= 1;
1935	}
1936
1937	if (n >= out_sgs)
1938	total_in_len += len;
1939	}
1940	}
1941
1942	/* We're using some buffers from the free list. */
1943	vq->vq.num_free -= total_sg;
1944
1945	/* Update free pointer */
1946	vq->packed.next_avail_idx = i;
1947
1948	/* Store token. */
1949	vq->packed.desc_state[head].num = total_sg;
1950	vq->packed.desc_state[head].data = data;
1951	vq->packed.desc_state[head].indir_desc = ctx;
1952	vq->packed.desc_state[head].total_in_len = total_in_len;
1953
1954	/*
1955	* A driver MUST NOT make the first descriptor in the list
1956	* available before all subsequent descriptors comprising
1957	* the list are made available.
1958	*/
1959	virtio_wmb(vq->weak_barriers);
1960	vq->packed.vring.desc[head].flags = head_flags;
1961	vq->num_added += total_sg;
1962
1963	pr_debug("Added buffer head %i to %p\n", head, vq);
1964	END_USE(vq);
1965
1966	return 0;
1967
1968	unmap_release:
1969	err_idx = i;
1970	i = head;
1971	vq->packed.avail_used_flags = avail_used_flags;
1972	vq->packed.avail_wrap_counter = avail_wrap_counter;
1973	unpub_flags = avail_used_flags ^ (1 << VRING_PACKED_DESC_F_AVAIL \|
1974	1 << VRING_PACKED_DESC_F_USED);
1975
1976	for (n = 0; n < total_sg; n++) {
1977	if (i == err_idx)
1978	break;
1979	/*
1980	* The mapping loop made every descriptor but the head
1981	* available. Stamp the previous wrap counter's AVAIL and USED
1982	* bits on those, so that a later and shorter chain at this head
1983	* does not leave one of them available beyond its own last
1984	* descriptor. Marking them used instead would hand
1985	* is_used_desc_packed() a completion we never made.
1986	*/
1987	if (i != head)
1988	desc[i].flags = cpu_to_le16(unpub_flags);
1989	vring_unmap_extra_packed(vq, &vq->packed.desc_extra[i]);
1990	i++;
1991	if (i >= vq->packed.vring.num) {
1992	i = 0;
1993	unpub_flags ^= 1 << VRING_PACKED_DESC_F_AVAIL \|
1994	1 << VRING_PACKED_DESC_F_USED;
1995	}
1996	}
1997
1998	END_USE(vq);
1999	return -EIO;
2000	}
2001
2002	static bool virtqueue_kick_prepare_packed(struct vring_virtqueue *vq)
2003	{
2004	u16 new, old, off_wrap, flags, wrap_counter, event_idx;
2005	bool needs_kick;
2006	union {
2007	struct {
2008	__le16 off_wrap;
2009	__le16 flags;
2010	};
2011	u32 u32;
2012	} snapshot;
2013
2014	START_USE(vq);
2015
2016	/*
2017	* We need to expose the new flags value before checking notification
2018	* suppressions.
2019	*/
2020	virtio_mb(vq->weak_barriers);
2021
2022	old = vq->packed.next_avail_idx - vq->num_added;
2023	new = vq->packed.next_avail_idx;
2024	vq->num_added = 0;
2025
2026	snapshot.u32 = (u32 )vq->packed.vring.device;
2027	flags = le16_to_cpu(snapshot.flags);
2028
2029	LAST_ADD_TIME_CHECK(vq);
2030	LAST_ADD_TIME_INVALID(vq);
2031
2032	if (flags != VRING_PACKED_EVENT_FLAG_DESC) {
2033	needs_kick = (flags != VRING_PACKED_EVENT_FLAG_DISABLE);
2034	goto out;
2035	}
2036
2037	off_wrap = le16_to_cpu(snapshot.off_wrap);
2038
2039	wrap_counter = off_wrap >> VRING_PACKED_EVENT_F_WRAP_CTR;
2040	event_idx = off_wrap & ~(1 << VRING_PACKED_EVENT_F_WRAP_CTR);
2041	if (wrap_counter != vq->packed.avail_wrap_counter)
2042	event_idx -= vq->packed.vring.num;
2043
2044	needs_kick = vring_need_event(event_idx, new, old);
2045	out:
2046	END_USE(vq);
2047	return needs_kick;
2048	}
2049
2050	static void detach_buf_packed_in_order(struct vring_virtqueue *vq,
2051	unsigned int id, void **ctx)
2052	{
2053	struct vring_desc_state_packed *state = NULL;
2054	struct vring_packed_desc *desc;
2055	unsigned int i, curr;
2056
2057	state = &vq->packed.desc_state[id];
2058
2059	/* Clear data ptr. */
2060	state->data = NULL;
2061
2062	vq->vq.num_free += state->num;
2063
2064	if (unlikely(vq->use_map_api)) {
2065	curr = id;
2066	for (i = 0; i < state->num; i++) {
2067	vring_unmap_extra_packed(vq,
2068	&vq->packed.desc_extra[curr]);
2069	curr = vq->packed.desc_extra[curr].next;
2070	}
2071	}
2072
2073	if (vq->indirect) {
2074	struct vring_desc_extra *extra;
2075	u32 len, num;
2076
2077	/* Free the indirect table, if any, now that it's unmapped. */
2078	desc = state->indir_desc;
2079	if (!desc)
2080	return;
2081
2082	if (vq->use_map_api) {
2083	len = vq->packed.desc_extra[id].len;
2084	num = len / sizeof(struct vring_packed_desc);
2085
2086	extra = (struct vring_desc_extra *)&desc[num];
2087
2088	for (i = 0; i < num; i++)
2089	vring_unmap_extra_packed(vq, &extra[i]);
2090	}
2091	kfree(desc);
2092	state->indir_desc = NULL;
2093	} else if (ctx) {
2094	*ctx = state->indir_desc;
2095	}
2096	}
2097
2098	static void detach_buf_packed(struct vring_virtqueue *vq,
2099	unsigned int id, void **ctx)
2100	{
2101	struct vring_desc_state_packed *state = &vq->packed.desc_state[id];
2102
2103	vq->packed.desc_extra[state->last].next = vq->free_head;
2104	vq->free_head = id;
2105
2106	detach_buf_packed_in_order(vq, id, ctx);
2107	}
2108
2109	static inline bool is_used_desc_packed(const struct vring_virtqueue *vq,
2110	u16 idx, bool used_wrap_counter)
2111	{
2112	u16 flags;
2113	bool avail, used;
2114
2115	flags = vring_read_packed_desc_flags(vq, idx);
2116	avail = !!(flags & (1 << VRING_PACKED_DESC_F_AVAIL));
2117	used = !!(flags & (1 << VRING_PACKED_DESC_F_USED));
2118
2119	return avail == used && used == used_wrap_counter;
2120	}
2121
2122	static bool virtqueue_poll_packed(const struct vring_virtqueue *vq,
2123	unsigned int off_wrap)
2124	{
2125	bool wrap_counter;
2126	u16 used_idx;
2127
2128	wrap_counter = off_wrap >> VRING_PACKED_EVENT_F_WRAP_CTR;
2129	used_idx = off_wrap & ~(1 << VRING_PACKED_EVENT_F_WRAP_CTR);
2130
2131	return is_used_desc_packed(vq, used_idx, wrap_counter);
2132	}
2133
2134	static bool more_used_packed(const struct vring_virtqueue *vq)
2135	{
2136	return virtqueue_poll_packed(vq, READ_ONCE(vq->last_used_idx));
2137	}
2138
2139	static void update_last_used_idx_packed(struct vring_virtqueue *vq,
2140	u16 id, u16 last_used,
2141	u16 used_wrap_counter)
2142	{
2143	last_used += vq->packed.desc_state[id].num;
2144	if (unlikely(last_used >= vq->packed.vring.num)) {
2145	last_used -= vq->packed.vring.num;
2146	used_wrap_counter ^= 1;
2147	}
2148
2149	last_used = (last_used \| (used_wrap_counter << VRING_PACKED_EVENT_F_WRAP_CTR));
2150	WRITE_ONCE(vq->last_used_idx, last_used);
2151
2152	/*
2153	* If we expect an interrupt for the next entry, tell host
2154	* by writing event index and flush out the write before
2155	* the read in the next get_buf call.
2156	*/
2157	if (vq->packed.event_flags_shadow == VRING_PACKED_EVENT_FLAG_DESC)
2158	virtio_store_mb(vq->weak_barriers,
2159	&vq->packed.vring.driver->off_wrap,
2160	cpu_to_le16(vq->last_used_idx));
2161	}
2162
2163	static bool more_used_packed_in_order(const struct vring_virtqueue *vq)
2164	{
2165	if (vq->batch_last.id != UINT_MAX)
2166	return true;
2167
2168	return virtqueue_poll_packed(vq, READ_ONCE(vq->last_used_idx));
2169	}
2170
2171	static void virtqueue_get_buf_ctx_packed_in_order(struct vring_virtqueue vq,
2172	unsigned int *len,
2173	void **ctx)
2174	{
2175	unsigned int num = vq->packed.vring.num;
2176	u16 last_used, last_used_idx;
2177	bool used_wrap_counter;
2178	void *ret;
2179
2180	START_USE(vq);
2181
2182	if (unlikely(vq->broken)) {
2183	END_USE(vq);
2184	return NULL;
2185	}
2186
2187	last_used_idx = vq->last_used_idx;
2188	used_wrap_counter = packed_used_wrap_counter(last_used_idx);
2189	last_used = packed_last_used(last_used_idx);
2190
2191	if (vq->batch_last.id == UINT_MAX) {
2192	if (!more_used_packed_in_order(vq)) {
2193	pr_debug("No more buffers in queue\n");
2194	END_USE(vq);
2195	return NULL;
2196	}
2197	/* Only get used elements after they have been exposed by host. */
2198	virtio_rmb(vq->weak_barriers);
2199	vq->batch_last.id =
2200	le16_to_cpu(vq->packed.vring.desc[last_used].id);
2201	vq->batch_last.len =
2202	le32_to_cpu(vq->packed.vring.desc[last_used].len);
2203	}
2204
2205	if (vq->batch_last.id == last_used) {
2206	vq->batch_last.id = UINT_MAX;
2207	*len = vq->batch_last.len;
2208	} else {
2209	*len = vq->packed.desc_state[last_used].total_in_len;
2210	}
2211
2212	if (unlikely(last_used >= num)) {
2213	BAD_RING(vq, "id %u out of range\n", last_used);
2214	return NULL;
2215	}
2216	if (unlikely(!vq->packed.desc_state[last_used].data)) {
2217	BAD_RING(vq, "id %u is not a head!\n", last_used);
2218	return NULL;
2219	}
2220
2221	/* detach_buf_packed clears data, so grab it now. */
2222	ret = vq->packed.desc_state[last_used].data;
2223	detach_buf_packed_in_order(vq, last_used, ctx);
2224
2225	update_last_used_idx_packed(vq, last_used, last_used,
2226	used_wrap_counter);
2227
2228	LAST_ADD_TIME_INVALID(vq);
2229
2230	END_USE(vq);
2231	return ret;
2232	}
2233
2234	static void virtqueue_get_buf_ctx_packed(struct vring_virtqueue vq,
2235	unsigned int *len,
2236	void **ctx)
2237	{
2238	unsigned int num = vq->packed.vring.num;
2239	u16 last_used, id, last_used_idx;
2240	bool used_wrap_counter;
2241	void *ret;
2242
2243	START_USE(vq);
2244
2245	if (unlikely(vq->broken)) {
2246	END_USE(vq);
2247	return NULL;
2248	}
2249
2250	if (!more_used_packed(vq)) {
2251	pr_debug("No more buffers in queue\n");
2252	END_USE(vq);
2253	return NULL;
2254	}
2255
2256	/* Only get used elements after they have been exposed by host. */
2257	virtio_rmb(vq->weak_barriers);
2258
2259	last_used_idx = READ_ONCE(vq->last_used_idx);
2260	used_wrap_counter = packed_used_wrap_counter(last_used_idx);
2261	last_used = packed_last_used(last_used_idx);
2262	id = vring_read_packed_desc_id(vq, last_used);
2263	*len = vring_read_packed_desc_len(vq, last_used);
2264
2265	if (unlikely(id >= num)) {
2266	BAD_RING(vq, "id %u out of range\n", id);
2267	return NULL;
2268	}
2269	if (unlikely(!vq->packed.desc_state[id].data)) {
2270	BAD_RING(vq, "id %u is not a head!\n", id);
2271	return NULL;
2272	}
2273
2274	/* detach_buf_packed clears data, so grab it now. */
2275	ret = vq->packed.desc_state[id].data;
2276	detach_buf_packed(vq, id, ctx);
2277
2278	update_last_used_idx_packed(vq, id, last_used, used_wrap_counter);
2279
2280	LAST_ADD_TIME_INVALID(vq);
2281
2282	END_USE(vq);
2283	return ret;
2284	}
2285
2286	static void virtqueue_disable_cb_packed(struct vring_virtqueue *vq)
2287	{
2288	if (vq->packed.event_flags_shadow != VRING_PACKED_EVENT_FLAG_DISABLE) {
2289	vq->packed.event_flags_shadow = VRING_PACKED_EVENT_FLAG_DISABLE;
2290
2291	/*
2292	* If device triggered an event already it won't trigger one again:
2293	* no need to disable.
2294	*/
2295	if (vq->event_triggered)
2296	return;
2297
2298	vq->packed.vring.driver->flags =
2299	cpu_to_le16(vq->packed.event_flags_shadow);
2300	}
2301	}
2302
2303	static unsigned int virtqueue_enable_cb_prepare_packed(struct vring_virtqueue *vq)
2304	{
2305	START_USE(vq);
2306
2307	/*
2308	* We optimistically turn back on interrupts, then check if there was
2309	* more to do.
2310	*/
2311
2312	if (vq->event) {
2313	vq->packed.vring.driver->off_wrap =
2314	cpu_to_le16(vq->last_used_idx);
2315	/*
2316	* We need to update event offset and event wrap
2317	* counter first before updating event flags.
2318	*/
2319	virtio_wmb(vq->weak_barriers);
2320	}
2321
2322	if (vq->packed.event_flags_shadow == VRING_PACKED_EVENT_FLAG_DISABLE) {
2323	vq->packed.event_flags_shadow = vq->event ?
2324	VRING_PACKED_EVENT_FLAG_DESC :
2325	VRING_PACKED_EVENT_FLAG_ENABLE;
2326	vq->packed.vring.driver->flags =
2327	cpu_to_le16(vq->packed.event_flags_shadow);
2328	}
2329
2330	END_USE(vq);
2331	return vq->last_used_idx;
2332	}
2333
2334	static bool virtqueue_enable_cb_delayed_packed(struct vring_virtqueue *vq)
2335	{
2336	u16 used_idx, wrap_counter, last_used_idx;
2337	u16 bufs;
2338
2339	START_USE(vq);
2340
2341	/*
2342	* We optimistically turn back on interrupts, then check if there was
2343	* more to do.
2344	*/
2345
2346	if (vq->event) {
2347	/* TODO: tune this threshold */
2348	bufs = (vq->packed.vring.num - vq->vq.num_free) * 3 / 4;
2349	last_used_idx = READ_ONCE(vq->last_used_idx);
2350	wrap_counter = packed_used_wrap_counter(last_used_idx);
2351
2352	used_idx = packed_last_used(last_used_idx) + bufs;
2353	if (used_idx >= vq->packed.vring.num) {
2354	used_idx -= vq->packed.vring.num;
2355	wrap_counter ^= 1;
2356	}
2357
2358	vq->packed.vring.driver->off_wrap = cpu_to_le16(used_idx \|
2359	(wrap_counter << VRING_PACKED_EVENT_F_WRAP_CTR));
2360
2361	/*
2362	* We need to update event offset and event wrap
2363	* counter first before updating event flags.
2364	*/
2365	virtio_wmb(vq->weak_barriers);
2366	}
2367
2368	if (vq->packed.event_flags_shadow == VRING_PACKED_EVENT_FLAG_DISABLE) {
2369	vq->packed.event_flags_shadow = vq->event ?
2370	VRING_PACKED_EVENT_FLAG_DESC :
2371	VRING_PACKED_EVENT_FLAG_ENABLE;
2372	vq->packed.vring.driver->flags =
2373	cpu_to_le16(vq->packed.event_flags_shadow);
2374	}
2375
2376	/*
2377	* We need to update event suppression structure first
2378	* before re-checking for more used buffers.
2379	*/
2380	virtio_mb(vq->weak_barriers);
2381
2382	last_used_idx = READ_ONCE(vq->last_used_idx);
2383	wrap_counter = packed_used_wrap_counter(last_used_idx);
2384	used_idx = packed_last_used(last_used_idx);
2385	if (is_used_desc_packed(vq, used_idx, wrap_counter)) {
2386	END_USE(vq);
2387	return false;
2388	}
2389
2390	END_USE(vq);
2391	return true;
2392	}
2393
2394	static void virtqueue_detach_unused_buf_packed(struct vring_virtqueue vq)
2395	{
2396	unsigned int i;
2397	void *buf;
2398
2399	START_USE(vq);
2400
2401	for (i = 0; i < vq->packed.vring.num; i++) {
2402	if (!vq->packed.desc_state[i].data)
2403	continue;
2404	/* detach_buf clears data, so grab it now. */
2405	buf = vq->packed.desc_state[i].data;
2406	if (virtqueue_is_in_order(vq))
2407	detach_buf_packed_in_order(vq, i, NULL);
2408	else
2409	detach_buf_packed(vq, i, NULL);
2410	END_USE(vq);
2411	return buf;
2412	}
2413	/* That should have freed everything. */
2414	BUG_ON(vq->vq.num_free != vq->packed.vring.num);
2415
2416	END_USE(vq);
2417	return NULL;
2418	}
2419
2420	static struct vring_desc_extra *vring_alloc_desc_extra(unsigned int num)
2421	{
2422	struct vring_desc_extra *desc_extra;
2423	unsigned int i;
2424
2425	desc_extra = kmalloc_objs(struct vring_desc_extra, num);
2426	if (!desc_extra)
2427	return NULL;
2428
2429	memset(desc_extra, 0, num * sizeof(struct vring_desc_extra));
2430
2431	for (i = 0; i < num - 1; i++)
2432	desc_extra[i].next = i + 1;
2433
2434	desc_extra[num - 1].next = 0;
2435
2436	return desc_extra;
2437	}
2438
2439	static void vring_free_packed(struct vring_virtqueue_packed *vring_packed,
2440	struct virtio_device *vdev,
2441	union virtio_map map)
2442	{
2443	if (vring_packed->vring.desc)
2444	vring_free_queue(vdev, vring_packed->ring_size_in_bytes,
2445	vring_packed->vring.desc,
2446	vring_packed->ring_dma_addr,
2447	map);
2448
2449	if (vring_packed->vring.driver)
2450	vring_free_queue(vdev, vring_packed->event_size_in_bytes,
2451	vring_packed->vring.driver,
2452	vring_packed->driver_event_dma_addr,
2453	map);
2454
2455	if (vring_packed->vring.device)
2456	vring_free_queue(vdev, vring_packed->event_size_in_bytes,
2457	vring_packed->vring.device,
2458	vring_packed->device_event_dma_addr,
2459	map);
2460
2461	kfree(vring_packed->desc_state);
2462	kfree(vring_packed->desc_extra);
2463	}
2464
2465	static int vring_alloc_queue_packed(struct vring_virtqueue_packed *vring_packed,
2466	struct virtio_device *vdev,
2467	u32 num, union virtio_map map)
2468	{
2469	struct vring_packed_desc *ring;
2470	struct vring_packed_desc_event driver, device;
2471	dma_addr_t ring_dma_addr, driver_event_dma_addr, device_event_dma_addr;
2472	size_t ring_size_in_bytes, event_size_in_bytes;
2473
2474	ring_size_in_bytes = num * sizeof(struct vring_packed_desc);
2475
2476	ring = vring_alloc_queue(vdev, ring_size_in_bytes,
2477	&ring_dma_addr,
2478	GFP_KERNEL \| __GFP_NOWARN \| __GFP_ZERO,
2479	map);
2480	if (!ring)
2481	goto err;
2482
2483	vring_packed->vring.desc = ring;
2484	vring_packed->ring_dma_addr = ring_dma_addr;
2485	vring_packed->ring_size_in_bytes = ring_size_in_bytes;
2486
2487	event_size_in_bytes = sizeof(struct vring_packed_desc_event);
2488
2489	driver = vring_alloc_queue(vdev, event_size_in_bytes,
2490	&driver_event_dma_addr,
2491	GFP_KERNEL \| __GFP_NOWARN \| __GFP_ZERO,
2492	map);
2493	if (!driver)
2494	goto err;
2495
2496	vring_packed->vring.driver = driver;
2497	vring_packed->event_size_in_bytes = event_size_in_bytes;
2498	vring_packed->driver_event_dma_addr = driver_event_dma_addr;
2499
2500	device = vring_alloc_queue(vdev, event_size_in_bytes,
2501	&device_event_dma_addr,
2502	GFP_KERNEL \| __GFP_NOWARN \| __GFP_ZERO,
2503	map);
2504	if (!device)
2505	goto err;
2506
2507	vring_packed->vring.device = device;
2508	vring_packed->device_event_dma_addr = device_event_dma_addr;
2509
2510	vring_packed->vring.num = num;
2511
2512	return 0;
2513
2514	err:
2515	vring_free_packed(vring_packed, vdev, map);
2516	return -ENOMEM;
2517	}
2518
2519	static int vring_alloc_state_extra_packed(struct vring_virtqueue_packed *vring_packed)
2520	{
2521	struct vring_desc_state_packed *state;
2522	struct vring_desc_extra *extra;
2523	u32 num = vring_packed->vring.num;
2524
2525	state = kmalloc_objs(struct vring_desc_state_packed, num);
2526	if (!state)
2527	goto err_desc_state;
2528
2529	memset(state, 0, num * sizeof(struct vring_desc_state_packed));
2530
2531	extra = vring_alloc_desc_extra(num);
2532	if (!extra)
2533	goto err_desc_extra;
2534
2535	vring_packed->desc_state = state;
2536	vring_packed->desc_extra = extra;
2537
2538	return 0;
2539
2540	err_desc_extra:
2541	kfree(state);
2542	err_desc_state:
2543	return -ENOMEM;
2544	}
2545
2546	static void virtqueue_vring_init_packed(struct vring_virtqueue_packed *vring_packed,
2547	bool callback)
2548	{
2549	vring_packed->next_avail_idx = 0;
2550	vring_packed->avail_wrap_counter = 1;
2551	vring_packed->event_flags_shadow = 0;
2552	vring_packed->avail_used_flags = 1 << VRING_PACKED_DESC_F_AVAIL;
2553
2554	/* No callback? Tell other side not to bother us. */
2555	if (!callback) {
2556	vring_packed->event_flags_shadow = VRING_PACKED_EVENT_FLAG_DISABLE;
2557	vring_packed->vring.driver->flags =
2558	cpu_to_le16(vring_packed->event_flags_shadow);
2559	}
2560	}
2561
2562	static void virtqueue_vring_attach_packed(struct vring_virtqueue *vq,
2563	struct vring_virtqueue_packed *vring_packed)
2564	{
2565	vq->packed = *vring_packed;
2566
2567	if (virtqueue_is_in_order(vq)) {
2568	vq->batch_last.id = UINT_MAX;
2569	} else {
2570	/*
2571	* Put everything in free lists. Note that
2572	* next_avail_idx is sufficient with IN_ORDER so
2573	* free_head is unused.
2574	*/
2575	vq->free_head = 0;
2576	}
2577	}
2578	static void virtqueue_reset_packed(struct vring_virtqueue *vq)
2579	{
2580	memset(vq->packed.vring.device, 0, vq->packed.event_size_in_bytes);
2581	memset(vq->packed.vring.driver, 0, vq->packed.event_size_in_bytes);
2582
2583	/* we need to reset the desc.flags. For more, see is_used_desc_packed() */
2584	memset(vq->packed.vring.desc, 0, vq->packed.ring_size_in_bytes);
2585	virtqueue_init(vq, vq->packed.vring.num);
2586	virtqueue_vring_init_packed(&vq->packed, !!vq->vq.callback);
2587	}
2588
2589	static const struct virtqueue_ops packed_ops;
2590
2591	static struct virtqueue *__vring_new_virtqueue_packed(unsigned int index,
2592	struct vring_virtqueue_packed *vring_packed,
2593	struct virtio_device *vdev,
2594	bool weak_barriers,
2595	bool context,
2596	bool (notify)(struct virtqueue ),
2597	void (callback)(struct virtqueue ),
2598	const char *name,
2599	union virtio_map map)
2600	{
2601	struct vring_virtqueue *vq;
2602	int err;
2603
2604	vq = kmalloc_obj(*vq);
2605	if (!vq)
2606	return NULL;
2607
2608	vq->vq.callback = callback;
2609	vq->vq.vdev = vdev;
2610	vq->vq.name = name;
2611	vq->vq.index = index;
2612	vq->vq.reset = false;
2613	vq->we_own_ring = false;
2614	vq->notify = notify;
2615	vq->weak_barriers = weak_barriers;
2616	#ifdef CONFIG_VIRTIO_HARDEN_NOTIFICATION
2617	vq->broken = true;
2618	#else
2619	vq->broken = false;
2620	#endif
2621	vq->map = map;
2622	vq->use_map_api = vring_use_map_api(vdev);
2623
2624	vq->indirect = virtio_has_feature(vdev, VIRTIO_RING_F_INDIRECT_DESC) &&
2625	!context;
2626	vq->event = virtio_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX);
2627	vq->layout = virtio_has_feature(vdev, VIRTIO_F_IN_ORDER) ?
2628	VQ_LAYOUT_PACKED_IN_ORDER : VQ_LAYOUT_PACKED;
2629
2630	if (virtio_has_feature(vdev, VIRTIO_F_ORDER_PLATFORM))
2631	vq->weak_barriers = false;
2632
2633	err = vring_alloc_state_extra_packed(vring_packed);
2634	if (err) {
2635	kfree(vq);
2636	return NULL;
2637	}
2638
2639	virtqueue_vring_init_packed(vring_packed, !!callback);
2640
2641	virtqueue_init(vq, vring_packed->vring.num);
2642	virtqueue_vring_attach_packed(vq, vring_packed);
2643
2644	spin_lock(&vdev->vqs_list_lock);
2645	list_add_tail(&vq->vq.list, &vdev->vqs);
2646	spin_unlock(&vdev->vqs_list_lock);
2647	return &vq->vq;
2648	}
2649
2650	static struct virtqueue *vring_create_virtqueue_packed(
2651	unsigned int index,
2652	unsigned int num,
2653	unsigned int vring_align,
2654	struct virtio_device *vdev,
2655	bool weak_barriers,
2656	bool may_reduce_num,
2657	bool context,
2658	bool (notify)(struct virtqueue ),
2659	void (callback)(struct virtqueue ),
2660	const char *name,
2661	union virtio_map map)
2662	{
2663	struct vring_virtqueue_packed vring_packed = {};
2664	struct virtqueue *vq;
2665
2666	if (vring_alloc_queue_packed(&vring_packed, vdev, num, map))
2667	return NULL;
2668
2669	vq = __vring_new_virtqueue_packed(index, &vring_packed, vdev, weak_barriers,
2670	context, notify, callback, name, map);
2671	if (!vq) {
2672	vring_free_packed(&vring_packed, vdev, map);
2673	return NULL;
2674	}
2675
2676	to_vvq(vq)->we_own_ring = true;
2677
2678	return vq;
2679	}
2680
2681	static int virtqueue_resize_packed(struct vring_virtqueue *vq, u32 num)
2682	{
2683	struct vring_virtqueue_packed vring_packed = {};
2684	struct virtio_device *vdev = vq->vq.vdev;
2685	int err;
2686
2687	if (vring_alloc_queue_packed(&vring_packed, vdev, num, vq->map))
2688	goto err_ring;
2689
2690	err = vring_alloc_state_extra_packed(&vring_packed);
2691	if (err)
2692	goto err_state_extra;
2693
2694	vring_free(&vq->vq);
2695
2696	virtqueue_vring_init_packed(&vring_packed, !!vq->vq.callback);
2697
2698	virtqueue_init(vq, vring_packed.vring.num);
2699	virtqueue_vring_attach_packed(vq, &vring_packed);
2700
2701	return 0;
2702
2703	err_state_extra:
2704	vring_free_packed(&vring_packed, vdev, vq->map);
2705	err_ring:
2706	virtqueue_reset_packed(vq);
2707	return -ENOMEM;
2708	}
2709
2710	static const struct virtqueue_ops split_ops = {
2711	.add = virtqueue_add_split,
2712	.get = virtqueue_get_buf_ctx_split,
2713	.kick_prepare = virtqueue_kick_prepare_split,
2714	.disable_cb = virtqueue_disable_cb_split,
2715	.enable_cb_delayed = virtqueue_enable_cb_delayed_split,
2716	.enable_cb_prepare = virtqueue_enable_cb_prepare_split,
2717	.poll = virtqueue_poll_split,
2718	.detach_unused_buf = virtqueue_detach_unused_buf_split,
2719	.more_used = more_used_split,
2720	.resize = virtqueue_resize_split,
2721	.reset = virtqueue_reset_split,
2722	};
2723
2724	static const struct virtqueue_ops packed_ops = {
2725	.add = virtqueue_add_packed,
2726	.get = virtqueue_get_buf_ctx_packed,
2727	.kick_prepare = virtqueue_kick_prepare_packed,
2728	.disable_cb = virtqueue_disable_cb_packed,
2729	.enable_cb_delayed = virtqueue_enable_cb_delayed_packed,
2730	.enable_cb_prepare = virtqueue_enable_cb_prepare_packed,
2731	.poll = virtqueue_poll_packed,
2732	.detach_unused_buf = virtqueue_detach_unused_buf_packed,
2733	.more_used = more_used_packed,
2734	.resize = virtqueue_resize_packed,
2735	.reset = virtqueue_reset_packed,
2736	};
2737
2738	static const struct virtqueue_ops split_in_order_ops = {
2739	.add = virtqueue_add_split,
2740	.get = virtqueue_get_buf_ctx_split_in_order,
2741	.kick_prepare = virtqueue_kick_prepare_split,
2742	.disable_cb = virtqueue_disable_cb_split,
2743	.enable_cb_delayed = virtqueue_enable_cb_delayed_split,
2744	.enable_cb_prepare = virtqueue_enable_cb_prepare_split,
2745	.poll = virtqueue_poll_split,
2746	.detach_unused_buf = virtqueue_detach_unused_buf_split,
2747	.more_used = more_used_split_in_order,
2748	.resize = virtqueue_resize_split,
2749	.reset = virtqueue_reset_split,
2750	};
2751
2752	static const struct virtqueue_ops packed_in_order_ops = {
2753	.add = virtqueue_add_packed_in_order,
2754	.get = virtqueue_get_buf_ctx_packed_in_order,
2755	.kick_prepare = virtqueue_kick_prepare_packed,
2756	.disable_cb = virtqueue_disable_cb_packed,
2757	.enable_cb_delayed = virtqueue_enable_cb_delayed_packed,
2758	.enable_cb_prepare = virtqueue_enable_cb_prepare_packed,
2759	.poll = virtqueue_poll_packed,
2760	.detach_unused_buf = virtqueue_detach_unused_buf_packed,
2761	.more_used = more_used_packed_in_order,
2762	.resize = virtqueue_resize_packed,
2763	.reset = virtqueue_reset_packed,
2764	};
2765
2766	static int virtqueue_disable_and_recycle(struct virtqueue *_vq,
2767	void (recycle)(struct virtqueue vq, void *buf))
2768	{
2769	struct vring_virtqueue *vq = to_vvq(_vq);
2770	struct virtio_device *vdev = vq->vq.vdev;
2771	void *buf;
2772	int err;
2773
2774	if (!vq->we_own_ring)
2775	return -EPERM;
2776
2777	if (!vdev->config->disable_vq_and_reset)
2778	return -ENOENT;
2779
2780	if (!vdev->config->enable_vq_after_reset)
2781	return -ENOENT;
2782
2783	err = vdev->config->disable_vq_and_reset(_vq);
2784	if (err)
2785	return err;
2786
2787	while ((buf = virtqueue_detach_unused_buf(_vq)) != NULL)
2788	recycle(_vq, buf);
2789
2790	return 0;
2791	}
2792
2793	static int virtqueue_enable_after_reset(struct virtqueue *_vq)
2794	{
2795	struct vring_virtqueue *vq = to_vvq(_vq);
2796	struct virtio_device *vdev = vq->vq.vdev;
2797
2798	if (vdev->config->enable_vq_after_reset(_vq))
2799	return -EBUSY;
2800
2801	return 0;
2802	}
2803
2804	/*
2805	* Generic functions and exported symbols.
2806	*/
2807
2808	#define VIRTQUEUE_CALL(vq, op, ...) \
2809	({ \
2810	typeof(vq) __VIRTQUEUE_CALL_vq = (vq); \
2811	typeof(split_ops.op(__VIRTQUEUE_CALL_vq, ##__VA_ARGS__)) ret; \
2812	\
2813	switch (__VIRTQUEUE_CALL_vq->layout) { \
2814	case VQ_LAYOUT_SPLIT: \
2815	ret = split_ops.op(__VIRTQUEUE_CALL_vq, ##__VA_ARGS__); \
2816	break; \
2817	case VQ_LAYOUT_PACKED: \
2818	ret = packed_ops.op(__VIRTQUEUE_CALL_vq, ##__VA_ARGS__);\
2819	break; \
2820	case VQ_LAYOUT_SPLIT_IN_ORDER: \
2821	ret = split_in_order_ops.op(vq, ##__VA_ARGS__); \
2822	break; \
2823	case VQ_LAYOUT_PACKED_IN_ORDER: \
2824	ret = packed_in_order_ops.op(vq, ##__VA_ARGS__); \
2825	break; \
2826	default: \
2827	BUG(); \
2828	break; \
2829	} \
2830	ret; \
2831	})
2832
2833	#define VOID_VIRTQUEUE_CALL(vq, op, ...) \
2834	({ \
2835	typeof(vq) __VIRTQUEUE_CALL_vq = (vq); \
2836	\
2837	switch (__VIRTQUEUE_CALL_vq->layout) { \
2838	case VQ_LAYOUT_SPLIT: \
2839	split_ops.op(__VIRTQUEUE_CALL_vq, ##__VA_ARGS__); \
2840	break; \
2841	case VQ_LAYOUT_PACKED: \
2842	packed_ops.op(__VIRTQUEUE_CALL_vq, ##__VA_ARGS__); \
2843	break; \
2844	case VQ_LAYOUT_SPLIT_IN_ORDER: \
2845	split_in_order_ops.op(vq, ##__VA_ARGS__); \
2846	break; \
2847	case VQ_LAYOUT_PACKED_IN_ORDER: \
2848	packed_in_order_ops.op(vq, ##__VA_ARGS__); \
2849	break; \
2850	default: \
2851	BUG(); \
2852	break; \
2853	} \
2854	})
2855
2856	static inline int virtqueue_add(struct virtqueue *_vq,
2857	struct scatterlist *sgs[],
2858	unsigned int total_sg,
2859	unsigned int out_sgs,
2860	unsigned int in_sgs,
2861	void *data,
2862	void *ctx,
2863	bool premapped,
2864	gfp_t gfp,
2865	unsigned long attr)
2866	{
2867	struct vring_virtqueue *vq = to_vvq(_vq);
2868
2869	return VIRTQUEUE_CALL(vq, add, sgs, total_sg,
2870	out_sgs, in_sgs, data,
2871	ctx, premapped, gfp, attr);
2872	}
2873
2874	/**
2875	* virtqueue_add_sgs - expose buffers to other end
2876	* @_vq: the struct virtqueue we're talking about.
2877	* @sgs: array of terminated scatterlists.
2878	* @out_sgs: the number of scatterlists readable by other side
2879	* @in_sgs: the number of scatterlists which are writable (after readable ones)
2880	* @data: the token identifying the buffer.
2881	* @gfp: how to do memory allocations (if necessary).
2882	*
2883	* Caller must ensure we don't call this with other virtqueue operations
2884	* at the same time (except where noted).
2885	*
2886	* Returns zero or a negative error (ie. ENOSPC, ENOMEM, EIO).
2887	*
2888	* NB: ENOSPC is a special code that is only returned on an attempt to add a
2889	* buffer to a full VQ. It indicates that some buffers are outstanding and that
2890	* the operation can be retried after some buffers have been used.
2891	*/
2892	int virtqueue_add_sgs(struct virtqueue *_vq,
2893	struct scatterlist *sgs[],
2894	unsigned int out_sgs,
2895	unsigned int in_sgs,
2896	void *data,
2897	gfp_t gfp)
2898	{
2899	unsigned int i, total_sg = 0;
2900
2901	/* Count them first. */
2902	for (i = 0; i < out_sgs + in_sgs; i++) {
2903	struct scatterlist *sg;
2904
2905	for (sg = sgs[i]; sg; sg = sg_next(sg))
2906	total_sg++;
2907	}
2908	return virtqueue_add(_vq, sgs, total_sg, out_sgs, in_sgs,
2909	data, NULL, false, gfp, 0);
2910	}
2911	EXPORT_SYMBOL_GPL(virtqueue_add_sgs);
2912
2913	/**
2914	* virtqueue_add_outbuf - expose output buffers to other end
2915	* @vq: the struct virtqueue we're talking about.
2916	* @sg: scatterlist (must be well-formed and terminated!)
2917	* @num: the number of entries in @sg readable by other side
2918	* @data: the token identifying the buffer.
2919	* @gfp: how to do memory allocations (if necessary).
2920	*
2921	* Caller must ensure we don't call this with other virtqueue operations
2922	* at the same time (except where noted).
2923	*
2924	* Returns zero or a negative error (ie. ENOSPC, ENOMEM, EIO).
2925	*/
2926	int virtqueue_add_outbuf(struct virtqueue *vq,
2927	struct scatterlist *sg, unsigned int num,
2928	void *data,
2929	gfp_t gfp)
2930	{
2931	return virtqueue_add(vq, &sg, num, 1, 0, data, NULL, false, gfp, 0);
2932	}
2933	EXPORT_SYMBOL_GPL(virtqueue_add_outbuf);
2934
2935	/**
2936	* virtqueue_add_outbuf_premapped - expose output buffers to other end
2937	* @vq: the struct virtqueue we're talking about.
2938	* @sg: scatterlist (must be well-formed and terminated!)
2939	* @num: the number of entries in @sg readable by other side
2940	* @data: the token identifying the buffer.
2941	* @gfp: how to do memory allocations (if necessary).
2942	*
2943	* Caller must ensure we don't call this with other virtqueue operations
2944	* at the same time (except where noted).
2945	*
2946	* Return:
2947	* Returns zero or a negative error (ie. ENOSPC, ENOMEM, EIO).
2948	*/
2949	int virtqueue_add_outbuf_premapped(struct virtqueue *vq,
2950	struct scatterlist *sg, unsigned int num,
2951	void *data,
2952	gfp_t gfp)
2953	{
2954	return virtqueue_add(vq, &sg, num, 1, 0, data, NULL, true, gfp, 0);
2955	}
2956	EXPORT_SYMBOL_GPL(virtqueue_add_outbuf_premapped);
2957
2958	/**
2959	* virtqueue_add_inbuf - expose input buffers to other end
2960	* @vq: the struct virtqueue we're talking about.
2961	* @sg: scatterlist (must be well-formed and terminated!)
2962	* @num: the number of entries in @sg writable by other side
2963	* @data: the token identifying the buffer.
2964	* @gfp: how to do memory allocations (if necessary).
2965	*
2966	* Caller must ensure we don't call this with other virtqueue operations
2967	* at the same time (except where noted).
2968	*
2969	* Returns zero or a negative error (ie. ENOSPC, ENOMEM, EIO).
2970	*/
2971	int virtqueue_add_inbuf(struct virtqueue *vq,
2972	struct scatterlist *sg, unsigned int num,
2973	void *data,
2974	gfp_t gfp)
2975	{
2976	return virtqueue_add(vq, &sg, num, 0, 1, data, NULL, false, gfp, 0);
2977	}
2978	EXPORT_SYMBOL_GPL(virtqueue_add_inbuf);
2979
2980	/**
2981	* virtqueue_add_inbuf_cache_clean - expose input buffers with cache clean
2982	* @vq: the struct virtqueue we're talking about.
2983	* @sg: scatterlist (must be well-formed and terminated!)
2984	* @num: the number of entries in @sg writable by other side
2985	* @data: the token identifying the buffer.
2986	* @gfp: how to do memory allocations (if necessary).
2987	*
2988	* Same as virtqueue_add_inbuf but passes DMA_ATTR_DEBUGGING_IGNORE_CACHELINES
2989	* to indicate that the CPU will not dirty any cacheline overlapping this buffer
2990	* while it is available, and to suppress overlapping cacheline warnings in DMA
2991	* debug builds.
2992	*
2993	* Caller must ensure we don't call this with other virtqueue operations
2994	* at the same time (except where noted).
2995	*
2996	* Returns zero or a negative error (ie. ENOSPC, ENOMEM, EIO).
2997	*/
2998	int virtqueue_add_inbuf_cache_clean(struct virtqueue *vq,
2999	struct scatterlist *sg, unsigned int num,
3000	void *data,
3001	gfp_t gfp)
3002	{
3003	return virtqueue_add(vq, &sg, num, 0, 1, data, NULL, false, gfp,
3004	DMA_ATTR_DEBUGGING_IGNORE_CACHELINES);
3005	}
3006	EXPORT_SYMBOL_GPL(virtqueue_add_inbuf_cache_clean);
3007
3008	/**
3009	* virtqueue_add_inbuf_ctx - expose input buffers to other end
3010	* @vq: the struct virtqueue we're talking about.
3011	* @sg: scatterlist (must be well-formed and terminated!)
3012	* @num: the number of entries in @sg writable by other side
3013	* @data: the token identifying the buffer.
3014	* @ctx: extra context for the token
3015	* @gfp: how to do memory allocations (if necessary).
3016	*
3017	* Caller must ensure we don't call this with other virtqueue operations
3018	* at the same time (except where noted).
3019	*
3020	* Returns zero or a negative error (ie. ENOSPC, ENOMEM, EIO).
3021	*/
3022	int virtqueue_add_inbuf_ctx(struct virtqueue *vq,
3023	struct scatterlist *sg, unsigned int num,
3024	void *data,
3025	void *ctx,
3026	gfp_t gfp)
3027	{
3028	return virtqueue_add(vq, &sg, num, 0, 1, data, ctx, false, gfp, 0);
3029	}
3030	EXPORT_SYMBOL_GPL(virtqueue_add_inbuf_ctx);
3031
3032	/**
3033	* virtqueue_add_inbuf_premapped - expose input buffers to other end
3034	* @vq: the struct virtqueue we're talking about.
3035	* @sg: scatterlist (must be well-formed and terminated!)
3036	* @num: the number of entries in @sg writable by other side
3037	* @data: the token identifying the buffer.
3038	* @ctx: extra context for the token
3039	* @gfp: how to do memory allocations (if necessary).
3040	*
3041	* Caller must ensure we don't call this with other virtqueue operations
3042	* at the same time (except where noted).
3043	*
3044	* Return:
3045	* Returns zero or a negative error (ie. ENOSPC, ENOMEM, EIO).
3046	*/
3047	int virtqueue_add_inbuf_premapped(struct virtqueue *vq,
3048	struct scatterlist *sg, unsigned int num,
3049	void *data,
3050	void *ctx,
3051	gfp_t gfp)
3052	{
3053	return virtqueue_add(vq, &sg, num, 0, 1, data, ctx, true, gfp, 0);
3054	}
3055	EXPORT_SYMBOL_GPL(virtqueue_add_inbuf_premapped);
3056
3057	/**
3058	* virtqueue_dma_dev - get the dma dev
3059	* @_vq: the struct virtqueue we're talking about.
3060	*
3061	* Returns the dma dev. That can been used for dma api.
3062	*/
3063	struct device virtqueue_dma_dev(struct virtqueue _vq)
3064	{
3065	struct vring_virtqueue *vq = to_vvq(_vq);
3066
3067	if (vq->use_map_api && !_vq->vdev->map)
3068	return vq->map.dma_dev;
3069	else
3070	return NULL;
3071	}
3072	EXPORT_SYMBOL_GPL(virtqueue_dma_dev);
3073
3074	/**
3075	* virtqueue_kick_prepare - first half of split virtqueue_kick call.
3076	* @_vq: the struct virtqueue
3077	*
3078	* Instead of virtqueue_kick(), you can do:
3079	* if (virtqueue_kick_prepare(vq))
3080	* virtqueue_notify(vq);
3081	*
3082	* This is sometimes useful because the virtqueue_kick_prepare() needs
3083	* to be serialized, but the actual virtqueue_notify() call does not.
3084	*/
3085	bool virtqueue_kick_prepare(struct virtqueue *_vq)
3086	{
3087	struct vring_virtqueue *vq = to_vvq(_vq);
3088
3089	return VIRTQUEUE_CALL(vq, kick_prepare);
3090	}
3091	EXPORT_SYMBOL_GPL(virtqueue_kick_prepare);
3092
3093	/**
3094	* virtqueue_notify - second half of split virtqueue_kick call.
3095	* @_vq: the struct virtqueue
3096	*
3097	* This does not need to be serialized.
3098	*
3099	* Returns false if host notify failed or queue is broken, otherwise true.
3100	*/
3101	bool virtqueue_notify(struct virtqueue *_vq)
3102	{
3103	struct vring_virtqueue *vq = to_vvq(_vq);
3104
3105	if (unlikely(vq->broken))
3106	return false;
3107
3108	/* Prod other side to tell it about changes. */
3109	if (!vq->notify(_vq)) {
3110	vq->broken = true;
3111	return false;
3112	}
3113	return true;
3114	}
3115	EXPORT_SYMBOL_GPL(virtqueue_notify);
3116
3117	/**
3118	* virtqueue_kick - update after add_buf
3119	* @vq: the struct virtqueue
3120	*
3121	* After one or more virtqueue_add_* calls, invoke this to kick
3122	* the other side.
3123	*
3124	* Caller must ensure we don't call this with other virtqueue
3125	* operations at the same time (except where noted).
3126	*
3127	* Returns false if kick failed, otherwise true.
3128	*/
3129	bool virtqueue_kick(struct virtqueue *vq)
3130	{
3131	if (virtqueue_kick_prepare(vq))
3132	return virtqueue_notify(vq);
3133	return true;
3134	}
3135	EXPORT_SYMBOL_GPL(virtqueue_kick);
3136
3137	/**
3138	* virtqueue_get_buf_ctx - get the next used buffer
3139	* @_vq: the struct virtqueue we're talking about.
3140	* @len: the length written into the buffer
3141	* @ctx: extra context for the token
3142	*
3143	* If the device wrote data into the buffer, @len will be set to the
3144	* amount written. This means you don't need to clear the buffer
3145	* beforehand to ensure there's no data leakage in the case of short
3146	* writes.
3147	*
3148	* Caller must ensure we don't call this with other virtqueue
3149	* operations at the same time (except where noted).
3150	*
3151	* Returns NULL if there are no used buffers, or the "data" token
3152	* handed to virtqueue_add_*().
3153	*/
3154	void virtqueue_get_buf_ctx(struct virtqueue _vq, unsigned int *len,
3155	void **ctx)
3156	{
3157	struct vring_virtqueue *vq = to_vvq(_vq);
3158
3159	return VIRTQUEUE_CALL(vq, get, len, ctx);
3160	}
3161	EXPORT_SYMBOL_GPL(virtqueue_get_buf_ctx);
3162
3163	void virtqueue_get_buf(struct virtqueue _vq, unsigned int *len)
3164	{
3165	return virtqueue_get_buf_ctx(_vq, len, NULL);
3166	}
3167	EXPORT_SYMBOL_GPL(virtqueue_get_buf);
3168	/**
3169	* virtqueue_disable_cb - disable callbacks
3170	* @_vq: the struct virtqueue we're talking about.
3171	*
3172	* Note that this is not necessarily synchronous, hence unreliable and only
3173	* useful as an optimization.
3174	*
3175	* Unlike other operations, this need not be serialized.
3176	*/
3177	void virtqueue_disable_cb(struct virtqueue *_vq)
3178	{
3179	struct vring_virtqueue *vq = to_vvq(_vq);
3180
3181	VOID_VIRTQUEUE_CALL(vq, disable_cb);
3182	}
3183	EXPORT_SYMBOL_GPL(virtqueue_disable_cb);
3184
3185	/**
3186	* virtqueue_enable_cb_prepare - restart callbacks after disable_cb
3187	* @_vq: the struct virtqueue we're talking about.
3188	*
3189	* This re-enables callbacks; it returns current queue state
3190	* in an opaque unsigned value. This value should be later tested by
3191	* virtqueue_poll, to detect a possible race between the driver checking for
3192	* more work, and enabling callbacks.
3193	*
3194	* Caller must ensure we don't call this with other virtqueue
3195	* operations at the same time (except where noted).
3196	*/
3197	unsigned int virtqueue_enable_cb_prepare(struct virtqueue *_vq)
3198	{
3199	struct vring_virtqueue *vq = to_vvq(_vq);
3200
3201	if (vq->event_triggered)
3202	vq->event_triggered = false;
3203
3204	return VIRTQUEUE_CALL(vq, enable_cb_prepare);
3205	}
3206	EXPORT_SYMBOL_GPL(virtqueue_enable_cb_prepare);
3207
3208	/**
3209	* virtqueue_poll - query pending used buffers
3210	* @_vq: the struct virtqueue we're talking about.
3211	* @last_used_idx: virtqueue state (from call to virtqueue_enable_cb_prepare).
3212	*
3213	* Returns "true" if there are pending used buffers in the queue.
3214	*
3215	* This does not need to be serialized.
3216	*/
3217	bool virtqueue_poll(struct virtqueue *_vq, unsigned int last_used_idx)
3218	{
3219	struct vring_virtqueue *vq = to_vvq(_vq);
3220
3221	if (unlikely(vq->broken))
3222	return false;
3223
3224	virtio_mb(vq->weak_barriers);
3225
3226	return VIRTQUEUE_CALL(vq, poll, last_used_idx);
3227	}
3228	EXPORT_SYMBOL_GPL(virtqueue_poll);
3229
3230	/**
3231	* virtqueue_enable_cb - restart callbacks after disable_cb.
3232	* @_vq: the struct virtqueue we're talking about.
3233	*
3234	* This re-enables callbacks; it returns "false" if there are pending
3235	* buffers in the queue, to detect a possible race between the driver
3236	* checking for more work, and enabling callbacks.
3237	*
3238	* Caller must ensure we don't call this with other virtqueue
3239	* operations at the same time (except where noted).
3240	*/
3241	bool virtqueue_enable_cb(struct virtqueue *_vq)
3242	{
3243	unsigned int last_used_idx = virtqueue_enable_cb_prepare(_vq);
3244
3245	return !virtqueue_poll(_vq, last_used_idx);
3246	}
3247	EXPORT_SYMBOL_GPL(virtqueue_enable_cb);
3248
3249	/**
3250	* virtqueue_enable_cb_delayed - restart callbacks after disable_cb.
3251	* @_vq: the struct virtqueue we're talking about.
3252	*
3253	* This re-enables callbacks but hints to the other side to delay
3254	* interrupts until most of the available buffers have been processed;
3255	* it returns "false" if there are many pending buffers in the queue,
3256	* to detect a possible race between the driver checking for more work,
3257	* and enabling callbacks.
3258	*
3259	* Caller must ensure we don't call this with other virtqueue
3260	* operations at the same time (except where noted).
3261	*/
3262	bool virtqueue_enable_cb_delayed(struct virtqueue *_vq)
3263	{
3264	struct vring_virtqueue *vq = to_vvq(_vq);
3265
3266	/*
3267	* When the device is broken there is no point in polling used->idx,
3268	* the backend will never update it. Return true to let callers
3269	* exit their cleanup loops instead of spinning forever.
3270	*/
3271	if (unlikely(vq->broken))
3272	return true;
3273
3274	if (vq->event_triggered)
3275	data_race(vq->event_triggered = false);
3276
3277	return VIRTQUEUE_CALL(vq, enable_cb_delayed);
3278	}
3279	EXPORT_SYMBOL_GPL(virtqueue_enable_cb_delayed);
3280
3281	/**
3282	* virtqueue_detach_unused_buf - detach first unused buffer
3283	* @_vq: the struct virtqueue we're talking about.
3284	*
3285	* Returns NULL or the "data" token handed to virtqueue_add_*().
3286	* This is not valid on an active queue; it is useful for device
3287	* shutdown or the reset queue.
3288	*/
3289	void virtqueue_detach_unused_buf(struct virtqueue _vq)
3290	{
3291	struct vring_virtqueue *vq = to_vvq(_vq);
3292
3293	return VIRTQUEUE_CALL(vq, detach_unused_buf);
3294	}
3295	EXPORT_SYMBOL_GPL(virtqueue_detach_unused_buf);
3296
3297	static inline bool more_used(const struct vring_virtqueue *vq)
3298	{
3299	return VIRTQUEUE_CALL(vq, more_used);
3300	}
3301
3302	/**
3303	* vring_interrupt - notify a virtqueue on an interrupt
3304	* @irq: the IRQ number (ignored)
3305	* @_vq: the struct virtqueue to notify
3306	*
3307	* Calls the callback function of @_vq to process the virtqueue
3308	* notification.
3309	*/
3310	irqreturn_t vring_interrupt(int irq, void *_vq)
3311	{
3312	struct vring_virtqueue *vq = to_vvq(_vq);
3313
3314	if (!more_used(vq)) {
3315	pr_debug("virtqueue interrupt with no work for %p\n", vq);
3316	return IRQ_NONE;
3317	}
3318
3319	if (unlikely(vq->broken)) {
3320	#ifdef CONFIG_VIRTIO_HARDEN_NOTIFICATION
3321	dev_warn_once(&vq->vq.vdev->dev,
3322	"virtio vring IRQ raised before DRIVER_OK");
3323	return IRQ_NONE;
3324	#else
3325	return IRQ_HANDLED;
3326	#endif
3327	}
3328
3329	/* Just a hint for performance: so it's ok that this can be racy! */
3330	if (vq->event)
3331	data_race(vq->event_triggered = true);
3332
3333	pr_debug("virtqueue callback for %p (%p)\n", vq, vq->vq.callback);
3334	if (vq->vq.callback)
3335	vq->vq.callback(&vq->vq);
3336
3337	return IRQ_HANDLED;
3338	}
3339	EXPORT_SYMBOL_GPL(vring_interrupt);
3340
3341	struct virtqueue *vring_create_virtqueue(
3342	unsigned int index,
3343	unsigned int num,
3344	unsigned int vring_align,
3345	struct virtio_device *vdev,
3346	bool weak_barriers,
3347	bool may_reduce_num,
3348	bool context,
3349	bool (notify)(struct virtqueue ),
3350	void (callback)(struct virtqueue ),
3351	const char *name)
3352	{
3353	union virtio_map map = {.dma_dev = vdev->dev.parent};
3354
3355	if (virtio_has_feature(vdev, VIRTIO_F_RING_PACKED))
3356	return vring_create_virtqueue_packed(index, num, vring_align,
3357	vdev, weak_barriers, may_reduce_num,
3358	context, notify, callback, name, map);
3359
3360	return vring_create_virtqueue_split(index, num, vring_align,
3361	vdev, weak_barriers, may_reduce_num,
3362	context, notify, callback, name, map);
3363	}
3364	EXPORT_SYMBOL_GPL(vring_create_virtqueue);
3365
3366	struct virtqueue *vring_create_virtqueue_map(
3367	unsigned int index,
3368	unsigned int num,
3369	unsigned int vring_align,
3370	struct virtio_device *vdev,
3371	bool weak_barriers,
3372	bool may_reduce_num,
3373	bool context,
3374	bool (notify)(struct virtqueue ),
3375	void (callback)(struct virtqueue ),
3376	const char *name,
3377	union virtio_map map)
3378	{
3379
3380	if (virtio_has_feature(vdev, VIRTIO_F_RING_PACKED))
3381	return vring_create_virtqueue_packed(index, num, vring_align,
3382	vdev, weak_barriers, may_reduce_num,
3383	context, notify, callback, name, map);
3384
3385	return vring_create_virtqueue_split(index, num, vring_align,
3386	vdev, weak_barriers, may_reduce_num,
3387	context, notify, callback, name, map);
3388	}
3389	EXPORT_SYMBOL_GPL(vring_create_virtqueue_map);
3390
3391	/**
3392	* virtqueue_resize - resize the vring of vq
3393	* @_vq: the struct virtqueue we're talking about.
3394	* @num: new ring num
3395	* @recycle: callback to recycle unused buffers
3396	* @recycle_done: callback to be invoked when recycle for all unused buffers done
3397	*
3398	* When it is really necessary to create a new vring, it will set the current vq
3399	* into the reset state. Then call the passed callback to recycle the buffer
3400	* that is no longer used. Only after the new vring is successfully created, the
3401	* old vring will be released.
3402	*
3403	* Caller must ensure we don't call this with other virtqueue operations
3404	* at the same time (except where noted).
3405	*
3406	* Returns zero or a negative error.
3407	* 0: success.
3408	* -ENOMEM: Failed to allocate a new ring, fall back to the original ring size.
3409	* vq can still work normally
3410	* -EBUSY: Failed to sync with device, vq may not work properly
3411	* -ENOENT: Transport or device not supported
3412	* -E2BIG/-EINVAL: num error
3413	* -EPERM: Operation not permitted
3414	*
3415	*/
3416	int virtqueue_resize(struct virtqueue *_vq, u32 num,
3417	void (recycle)(struct virtqueue vq, void *buf),
3418	void (recycle_done)(struct virtqueue vq))
3419	{
3420	struct vring_virtqueue *vq = to_vvq(_vq);
3421	int err, err_reset;
3422
3423	if (num > vq->vq.num_max)
3424	return -E2BIG;
3425
3426	if (!num)
3427	return -EINVAL;
3428
3429	if (virtqueue_get_vring_size(_vq) == num)
3430	return 0;
3431
3432	err = virtqueue_disable_and_recycle(_vq, recycle);
3433	if (err)
3434	return err;
3435	if (recycle_done)
3436	recycle_done(_vq);
3437
3438	err = VIRTQUEUE_CALL(vq, resize, num);
3439
3440	err_reset = virtqueue_enable_after_reset(_vq);
3441	if (err_reset)
3442	return err_reset;
3443
3444	return err;
3445	}
3446	EXPORT_SYMBOL_GPL(virtqueue_resize);
3447
3448	/**
3449	* virtqueue_reset - detach and recycle all unused buffers
3450	* @_vq: the struct virtqueue we're talking about.
3451	* @recycle: callback to recycle unused buffers
3452	* @recycle_done: callback to be invoked when recycle for all unused buffers done
3453	*
3454	* Caller must ensure we don't call this with other virtqueue operations
3455	* at the same time (except where noted).
3456	*
3457	* Returns zero or a negative error.
3458	* 0: success.
3459	* -EBUSY: Failed to sync with device, vq may not work properly
3460	* -ENOENT: Transport or device not supported
3461	* -EPERM: Operation not permitted
3462	*/
3463	int virtqueue_reset(struct virtqueue *_vq,
3464	void (recycle)(struct virtqueue vq, void *buf),
3465	void (recycle_done)(struct virtqueue vq))
3466	{
3467	struct vring_virtqueue *vq = to_vvq(_vq);
3468	int err;
3469
3470	err = virtqueue_disable_and_recycle(_vq, recycle);
3471	if (err)
3472	return err;
3473	if (recycle_done)
3474	recycle_done(_vq);
3475
3476	VOID_VIRTQUEUE_CALL(vq, reset);
3477
3478	return virtqueue_enable_after_reset(_vq);
3479	}
3480	EXPORT_SYMBOL_GPL(virtqueue_reset);
3481
3482	struct virtqueue *vring_new_virtqueue(unsigned int index,
3483	unsigned int num,
3484	unsigned int vring_align,
3485	struct virtio_device *vdev,
3486	bool weak_barriers,
3487	bool context,
3488	void *pages,
3489	bool (notify)(struct virtqueue vq),
3490	void (callback)(struct virtqueue vq),
3491	const char *name)
3492	{
3493	struct vring_virtqueue_split vring_split = {};
3494	union virtio_map map = {.dma_dev = vdev->dev.parent};
3495
3496	if (virtio_has_feature(vdev, VIRTIO_F_RING_PACKED)) {
3497	struct vring_virtqueue_packed vring_packed = {};
3498
3499	vring_packed.vring.num = num;
3500	vring_packed.vring.desc = pages;
3501	return __vring_new_virtqueue_packed(index, &vring_packed,
3502	vdev, weak_barriers,
3503	context, notify, callback,
3504	name, map);
3505	}
3506
3507	vring_init(&vring_split.vring, num, pages, vring_align);
3508	return __vring_new_virtqueue_split(index, &vring_split, vdev, weak_barriers,
3509	context, notify, callback, name,
3510	map);
3511	}
3512	EXPORT_SYMBOL_GPL(vring_new_virtqueue);
3513
3514	static void vring_free(struct virtqueue *_vq)
3515	{
3516	struct vring_virtqueue *vq = to_vvq(_vq);
3517
3518	if (vq->we_own_ring) {
3519	if (virtqueue_is_packed(vq)) {
3520	vring_free_queue(vq->vq.vdev,
3521	vq->packed.ring_size_in_bytes,
3522	vq->packed.vring.desc,
3523	vq->packed.ring_dma_addr,
3524	vq->map);
3525
3526	vring_free_queue(vq->vq.vdev,
3527	vq->packed.event_size_in_bytes,
3528	vq->packed.vring.driver,
3529	vq->packed.driver_event_dma_addr,
3530	vq->map);
3531
3532	vring_free_queue(vq->vq.vdev,
3533	vq->packed.event_size_in_bytes,
3534	vq->packed.vring.device,
3535	vq->packed.device_event_dma_addr,
3536	vq->map);
3537
3538	kfree(vq->packed.desc_state);
3539	kfree(vq->packed.desc_extra);
3540	} else {
3541	vring_free_queue(vq->vq.vdev,
3542	vq->split.queue_size_in_bytes,
3543	vq->split.vring.desc,
3544	vq->split.queue_dma_addr,
3545	vq->map);
3546	}
3547	}
3548	if (!virtqueue_is_packed(vq)) {
3549	kfree(vq->split.desc_state);
3550	kfree(vq->split.desc_extra);
3551	}
3552	}
3553
3554	void vring_del_virtqueue(struct virtqueue *_vq)
3555	{
3556	struct vring_virtqueue *vq = to_vvq(_vq);
3557
3558	spin_lock(&vq->vq.vdev->vqs_list_lock);
3559	list_del(&_vq->list);
3560	spin_unlock(&vq->vq.vdev->vqs_list_lock);
3561
3562	vring_free(_vq);
3563
3564	kfree(vq);
3565	}
3566	EXPORT_SYMBOL_GPL(vring_del_virtqueue);
3567
3568	u32 vring_notification_data(struct virtqueue *_vq)
3569	{
3570	struct vring_virtqueue *vq = to_vvq(_vq);
3571	u16 next;
3572
3573	if (virtqueue_is_packed(vq))
3574	next = (vq->packed.next_avail_idx &
3575	~(-(1 << VRING_PACKED_EVENT_F_WRAP_CTR))) \|
3576	vq->packed.avail_wrap_counter <<
3577	VRING_PACKED_EVENT_F_WRAP_CTR;
3578	else
3579	next = vq->split.avail_idx_shadow;
3580
3581	return next << 16 \| _vq->index;
3582	}
3583	EXPORT_SYMBOL_GPL(vring_notification_data);
3584
3585	/* Manipulates transport-specific feature bits. */
3586	void vring_transport_features(struct virtio_device *vdev)
3587	{
3588	unsigned int i;
3589
3590	for (i = VIRTIO_TRANSPORT_F_START; i < VIRTIO_TRANSPORT_F_END; i++) {
3591	switch (i) {
3592	case VIRTIO_RING_F_INDIRECT_DESC:
3593	break;
3594	case VIRTIO_RING_F_EVENT_IDX:
3595	break;
3596	case VIRTIO_F_VERSION_1:
3597	break;
3598	case VIRTIO_F_ACCESS_PLATFORM:
3599	break;
3600	case VIRTIO_F_RING_PACKED:
3601	break;
3602	case VIRTIO_F_ORDER_PLATFORM:
3603	break;
3604	case VIRTIO_F_NOTIFICATION_DATA:
3605	break;
3606	case VIRTIO_F_IN_ORDER:
3607	break;
3608	default:
3609	/* We don't understand this bit. */
3610	__virtio_clear_bit(vdev, i);
3611	}
3612	}
3613	}
3614	EXPORT_SYMBOL_GPL(vring_transport_features);
3615
3616	/**
3617	* virtqueue_get_vring_size - return the size of the virtqueue's vring
3618	* @_vq: the struct virtqueue containing the vring of interest.
3619	*
3620	* Returns the size of the vring. This is mainly used for boasting to
3621	* userspace. Unlike other operations, this need not be serialized.
3622	*/
3623	unsigned int virtqueue_get_vring_size(const struct virtqueue *_vq)
3624	{
3625
3626	const struct vring_virtqueue *vq = to_vvq(_vq);
3627
3628	return virtqueue_is_packed(vq) ? vq->packed.vring.num :
3629	vq->split.vring.num;
3630	}
3631	EXPORT_SYMBOL_GPL(virtqueue_get_vring_size);
3632
3633	/*
3634	* This function should only be called by the core, not directly by the driver.
3635	*/
3636	void __virtqueue_break(struct virtqueue *_vq)
3637	{
3638	struct vring_virtqueue *vq = to_vvq(_vq);
3639
3640	/* Pairs with READ_ONCE() in virtqueue_is_broken(). */
3641	WRITE_ONCE(vq->broken, true);
3642	}
3643	EXPORT_SYMBOL_GPL(__virtqueue_break);
3644
3645	/*
3646	* This function should only be called by the core, not directly by the driver.
3647	*/
3648	void __virtqueue_unbreak(struct virtqueue *_vq)
3649	{
3650	struct vring_virtqueue *vq = to_vvq(_vq);
3651
3652	/* Pairs with READ_ONCE() in virtqueue_is_broken(). */
3653	WRITE_ONCE(vq->broken, false);
3654	}
3655	EXPORT_SYMBOL_GPL(__virtqueue_unbreak);
3656
3657	bool virtqueue_is_broken(const struct virtqueue *_vq)
3658	{
3659	const struct vring_virtqueue *vq = to_vvq(_vq);
3660
3661	return READ_ONCE(vq->broken);
3662	}
3663	EXPORT_SYMBOL_GPL(virtqueue_is_broken);
3664
3665	/*
3666	* This should prevent the device from being used, allowing drivers to
3667	* recover. You may need to grab appropriate locks to flush.
3668	*/
3669	void virtio_break_device(struct virtio_device *dev)
3670	{
3671	struct virtqueue *_vq;
3672
3673	spin_lock(&dev->vqs_list_lock);
3674	list_for_each_entry(_vq, &dev->vqs, list) {
3675	struct vring_virtqueue *vq = to_vvq(_vq);
3676
3677	/* Pairs with READ_ONCE() in virtqueue_is_broken(). */
3678	WRITE_ONCE(vq->broken, true);
3679	}
3680	spin_unlock(&dev->vqs_list_lock);
3681	}
3682	EXPORT_SYMBOL_GPL(virtio_break_device);
3683
3684	/*
3685	* This should allow the device to be used by the driver. You may
3686	* need to grab appropriate locks to flush the write to
3687	* vq->broken. This should only be used in some specific case e.g
3688	* (probing and restoring). This function should only be called by the
3689	* core, not directly by the driver.
3690	*/
3691	void __virtio_unbreak_device(struct virtio_device *dev)
3692	{
3693	struct virtqueue *_vq;
3694
3695	spin_lock(&dev->vqs_list_lock);
3696	list_for_each_entry(_vq, &dev->vqs, list) {
3697	struct vring_virtqueue *vq = to_vvq(_vq);
3698
3699	/* Pairs with READ_ONCE() in virtqueue_is_broken(). */
3700	WRITE_ONCE(vq->broken, false);
3701	}
3702	spin_unlock(&dev->vqs_list_lock);
3703	}
3704	EXPORT_SYMBOL_GPL(__virtio_unbreak_device);
3705
3706	dma_addr_t virtqueue_get_desc_addr(const struct virtqueue *_vq)
3707	{
3708	const struct vring_virtqueue *vq = to_vvq(_vq);
3709
3710	BUG_ON(!vq->we_own_ring);
3711
3712	if (virtqueue_is_packed(vq))
3713	return vq->packed.ring_dma_addr;
3714
3715	return vq->split.queue_dma_addr;
3716	}
3717	EXPORT_SYMBOL_GPL(virtqueue_get_desc_addr);
3718
3719	dma_addr_t virtqueue_get_avail_addr(const struct virtqueue *_vq)
3720	{
3721	const struct vring_virtqueue *vq = to_vvq(_vq);
3722
3723	BUG_ON(!vq->we_own_ring);
3724
3725	if (virtqueue_is_packed(vq))
3726	return vq->packed.driver_event_dma_addr;
3727
3728	return vq->split.queue_dma_addr +
3729	((char )vq->split.vring.avail - (char )vq->split.vring.desc);
3730	}
3731	EXPORT_SYMBOL_GPL(virtqueue_get_avail_addr);
3732
3733	dma_addr_t virtqueue_get_used_addr(const struct virtqueue *_vq)
3734	{
3735	const struct vring_virtqueue *vq = to_vvq(_vq);
3736
3737	BUG_ON(!vq->we_own_ring);
3738
3739	if (virtqueue_is_packed(vq))
3740	return vq->packed.device_event_dma_addr;
3741
3742	return vq->split.queue_dma_addr +
3743	((char )vq->split.vring.used - (char )vq->split.vring.desc);
3744	}
3745	EXPORT_SYMBOL_GPL(virtqueue_get_used_addr);
3746
3747	/* Only available for split ring */
3748	const struct vring virtqueue_get_vring(const struct virtqueue vq)
3749	{
3750	return &to_vvq(vq)->split.vring;
3751	}
3752	EXPORT_SYMBOL_GPL(virtqueue_get_vring);
3753
3754	/**
3755	* virtqueue_map_alloc_coherent - alloc coherent mapping
3756	* @vdev: the virtio device we are talking to
3757	* @map: metadata for performing mapping
3758	* @size: the size of the buffer
3759	* @map_handle: the pointer to the mapped address
3760	* @gfp: allocation flag (GFP_XXX)
3761	*
3762	* return virtual address or NULL on error
3763	*/
3764	void virtqueue_map_alloc_coherent(struct virtio_device vdev,
3765	union virtio_map map,
3766	size_t size, dma_addr_t *map_handle,
3767	gfp_t gfp)
3768	{
3769	if (vdev->map)
3770	return vdev->map->alloc(map, size,
3771	map_handle, gfp);
3772	else
3773	return dma_alloc_coherent(map.dma_dev, size,
3774	map_handle, gfp);
3775	}
3776	EXPORT_SYMBOL_GPL(virtqueue_map_alloc_coherent);
3777
3778	/**
3779	* virtqueue_map_free_coherent - free coherent mapping
3780	* @vdev: the virtio device we are talking to
3781	* @map: metadata for performing mapping
3782	* @size: the size of the buffer
3783	* @vaddr: the virtual address that needs to be freed
3784	* @map_handle: the mapped address that needs to be freed
3785	*
3786	*/
3787	void virtqueue_map_free_coherent(struct virtio_device *vdev,
3788	union virtio_map map, size_t size, void *vaddr,
3789	dma_addr_t map_handle)
3790	{
3791	if (vdev->map)
3792	vdev->map->free(map, size, vaddr,
3793	map_handle, 0);
3794	else
3795	dma_free_coherent(map.dma_dev, size, vaddr, map_handle);
3796	}
3797	EXPORT_SYMBOL_GPL(virtqueue_map_free_coherent);
3798
3799	/**
3800	* virtqueue_map_page_attrs - map a page to the device
3801	* @_vq: the virtqueue we are talking to
3802	* @page: the page that will be mapped by the device
3803	* @offset: the offset in the page for a buffer
3804	* @size: the buffer size
3805	* @dir: mapping direction
3806	* @attrs: mapping attributes
3807	*
3808	* Returns mapped address. Caller should check that by virtqueue_map_mapping_error().
3809	*/
3810	dma_addr_t virtqueue_map_page_attrs(const struct virtqueue *_vq,
3811	struct page *page,
3812	unsigned long offset,
3813	size_t size,
3814	enum dma_data_direction dir,
3815	unsigned long attrs)
3816	{
3817	const struct vring_virtqueue *vq = to_vvq(_vq);
3818	struct virtio_device *vdev = _vq->vdev;
3819
3820	if (vdev->map)
3821	return vdev->map->map_page(vq->map,
3822	page, offset, size,
3823	dir, attrs);
3824
3825	return dma_map_page_attrs(vring_dma_dev(vq),
3826	page, offset, size,
3827	dir, attrs);
3828	}
3829	EXPORT_SYMBOL_GPL(virtqueue_map_page_attrs);
3830
3831	/**
3832	* virtqueue_unmap_page_attrs - map a page to the device
3833	* @_vq: the virtqueue we are talking to
3834	* @map_handle: the mapped address
3835	* @size: the buffer size
3836	* @dir: mapping direction
3837	* @attrs: unmapping attributes
3838	*/
3839	void virtqueue_unmap_page_attrs(const struct virtqueue *_vq,
3840	dma_addr_t map_handle,
3841	size_t size, enum dma_data_direction dir,
3842	unsigned long attrs)
3843	{
3844	const struct vring_virtqueue *vq = to_vvq(_vq);
3845	struct virtio_device *vdev = _vq->vdev;
3846
3847	if (vdev->map)
3848	vdev->map->unmap_page(vq->map,
3849	map_handle, size, dir, attrs);
3850	else
3851	dma_unmap_page_attrs(vring_dma_dev(vq), map_handle,
3852	size, dir, attrs);
3853	}
3854	EXPORT_SYMBOL_GPL(virtqueue_unmap_page_attrs);
3855
3856	/**
3857	* virtqueue_map_single_attrs - map DMA for _vq
3858	* @_vq: the struct virtqueue we're talking about.
3859	* @ptr: the pointer of the buffer to do dma
3860	* @size: the size of the buffer to do dma
3861	* @dir: DMA direction
3862	* @attrs: DMA Attrs
3863	*
3864	* The caller calls this to do dma mapping in advance. The DMA address can be
3865	* passed to this _vq when it is in pre-mapped mode.
3866	*
3867	* return mapped address. Caller should check that by virtqueue_map_mapping_error().
3868	*/
3869	dma_addr_t virtqueue_map_single_attrs(const struct virtqueue _vq, void ptr,
3870	size_t size,
3871	enum dma_data_direction dir,
3872	unsigned long attrs)
3873	{
3874	const struct vring_virtqueue *vq = to_vvq(_vq);
3875
3876	if (!vq->use_map_api) {
3877	kmsan_handle_dma(virt_to_phys(ptr), size, dir);
3878	return (dma_addr_t)virt_to_phys(ptr);
3879	}
3880
3881	/* DMA must never operate on areas that might be remapped. */
3882	if (dev_WARN_ONCE(&_vq->vdev->dev, is_vmalloc_addr(ptr),
3883	"rejecting DMA map of vmalloc memory\n"))
3884	return DMA_MAPPING_ERROR;
3885
3886	return virtqueue_map_page_attrs(&vq->vq, virt_to_page(ptr),
3887	offset_in_page(ptr), size, dir, attrs);
3888	}
3889	EXPORT_SYMBOL_GPL(virtqueue_map_single_attrs);
3890
3891	/**
3892	* virtqueue_unmap_single_attrs - unmap map for _vq
3893	* @_vq: the struct virtqueue we're talking about.
3894	* @addr: the dma address to unmap
3895	* @size: the size of the buffer
3896	* @dir: DMA direction
3897	* @attrs: DMA Attrs
3898	*
3899	* Unmap the address that is mapped by the virtqueue_map_* APIs.
3900	*
3901	*/
3902	void virtqueue_unmap_single_attrs(const struct virtqueue *_vq,
3903	dma_addr_t addr,
3904	size_t size, enum dma_data_direction dir,
3905	unsigned long attrs)
3906	{
3907	const struct vring_virtqueue *vq = to_vvq(_vq);
3908
3909	if (!vq->use_map_api)
3910	return;
3911
3912	virtqueue_unmap_page_attrs(_vq, addr, size, dir, attrs);
3913	}
3914	EXPORT_SYMBOL_GPL(virtqueue_unmap_single_attrs);
3915
3916	/**
3917	* virtqueue_map_mapping_error - check dma address
3918	* @_vq: the struct virtqueue we're talking about.
3919	* @addr: DMA address
3920	*
3921	* Returns 0 means dma valid. Other means invalid dma address.
3922	*/
3923	int virtqueue_map_mapping_error(const struct virtqueue *_vq, dma_addr_t addr)
3924	{
3925	const struct vring_virtqueue *vq = to_vvq(_vq);
3926
3927	return vring_mapping_error(vq, addr);
3928	}
3929	EXPORT_SYMBOL_GPL(virtqueue_map_mapping_error);
3930
3931	/**
3932	* virtqueue_map_need_sync - check a dma address needs sync
3933	* @_vq: the struct virtqueue we're talking about.
3934	* @addr: DMA address
3935	*
3936	* Check if the dma address mapped by the virtqueue_map_* APIs needs to be
3937	* synchronized
3938	*
3939	* return bool
3940	*/
3941	bool virtqueue_map_need_sync(const struct virtqueue *_vq, dma_addr_t addr)
3942	{
3943	const struct vring_virtqueue *vq = to_vvq(_vq);
3944	struct virtio_device *vdev = _vq->vdev;
3945
3946	if (!vq->use_map_api)
3947	return false;
3948
3949	if (vdev->map)
3950	return vdev->map->need_sync(vq->map, addr);
3951	else
3952	return dma_need_sync(vring_dma_dev(vq), addr);
3953	}
3954	EXPORT_SYMBOL_GPL(virtqueue_map_need_sync);
3955
3956	/**
3957	* virtqueue_map_sync_single_range_for_cpu - map sync for cpu
3958	* @_vq: the struct virtqueue we're talking about.
3959	* @addr: DMA address
3960	* @offset: DMA address offset
3961	* @size: buf size for sync
3962	* @dir: DMA direction
3963	*
3964	* Before calling this function, use virtqueue_map_need_sync() to confirm that
3965	* the DMA address really needs to be synchronized
3966	*
3967	*/
3968	void virtqueue_map_sync_single_range_for_cpu(const struct virtqueue *_vq,
3969	dma_addr_t addr,
3970	unsigned long offset, size_t size,
3971	enum dma_data_direction dir)
3972	{
3973	const struct vring_virtqueue *vq = to_vvq(_vq);
3974	struct virtio_device *vdev = _vq->vdev;
3975
3976	if (!vq->use_map_api)
3977	return;
3978
3979	if (vdev->map)
3980	vdev->map->sync_single_for_cpu(vq->map,
3981	addr + offset, size, dir);
3982	else
3983	dma_sync_single_range_for_cpu(vring_dma_dev(vq),
3984	addr, offset, size, dir);
3985	}
3986	EXPORT_SYMBOL_GPL(virtqueue_map_sync_single_range_for_cpu);
3987
3988	/**
3989	* virtqueue_map_sync_single_range_for_device - map sync for device
3990	* @_vq: the struct virtqueue we're talking about.
3991	* @addr: DMA address
3992	* @offset: DMA address offset
3993	* @size: buf size for sync
3994	* @dir: DMA direction
3995	*
3996	* Before calling this function, use virtqueue_map_need_sync() to confirm that
3997	* the DMA address really needs to be synchronized
3998	*/
3999	void virtqueue_map_sync_single_range_for_device(const struct virtqueue *_vq,
4000	dma_addr_t addr,
4001	unsigned long offset, size_t size,
4002	enum dma_data_direction dir)
4003	{
4004	const struct vring_virtqueue *vq = to_vvq(_vq);
4005	struct virtio_device *vdev = _vq->vdev;
4006
4007	if (!vq->use_map_api)
4008	return;
4009
4010	if (vdev->map)
4011	vdev->map->sync_single_for_device(vq->map,
4012	addr + offset,
4013	size, dir);
4014	else
4015	dma_sync_single_range_for_device(vring_dma_dev(vq), addr,
4016	offset, size, dir);
4017	}
4018	EXPORT_SYMBOL_GPL(virtqueue_map_sync_single_range_for_device);
4019
4020	MODULE_DESCRIPTION("Virtio ring implementation");
4021	MODULE_LICENSE("GPL");