projects/linux·blob master · refs

1499 lines · 38.6 KiB

1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Helpers for the host side of a virtio ring.
4 *
5 * Since these may be in userspace, we use (inline) accessors.
6 */
7#include <linux/compiler.h>
8#include <linux/module.h>
9#include <linux/vringh.h>
10#include <linux/virtio_ring.h>
11#include <linux/kernel.h>
12#include <linux/ratelimit.h>
13#include <linux/uaccess.h>
14#include <linux/slab.h>
15#include <linux/export.h>
16#if IS_REACHABLE(CONFIG_VHOST_IOTLB)
17#include <linux/bvec.h>
18#include <linux/highmem.h>
19#include <linux/vhost_iotlb.h>
20#endif
21#include <uapi/linux/virtio_config.h>
22
23static __printf(1,2) __cold void vringh_bad(const char *fmt, ...)
24{
25 static DEFINE_RATELIMIT_STATE(vringh_rs,
26 DEFAULT_RATELIMIT_INTERVAL,
27 DEFAULT_RATELIMIT_BURST);
28 if (__ratelimit(&vringh_rs)) {
29 va_list ap;
30 va_start(ap, fmt);
31 printk(KERN_NOTICE "vringh:");
32 vprintk(fmt, ap);
33 va_end(ap);
34 }
35}
36
37/* Returns vring->num if empty, -ve on error. */
38static inline int __vringh_get_head(const struct vringh *vrh,
39 int (*getu16)(const struct vringh *vrh,
40 u16 *val, const __virtio16 *p),
41 u16 *last_avail_idx)
42{
43 u16 avail_idx, i, head;
44 int err;
45
46 err = getu16(vrh, &avail_idx, &vrh->vring.avail->idx);
47 if (err) {
48 vringh_bad("Failed to access avail idx at %p",
49 &vrh->vring.avail->idx);
50 return err;
51 }
52
53 if (*last_avail_idx == avail_idx)
54 return vrh->vring.num;
55
56 /* Only get avail ring entries after they have been exposed by guest. */
57 virtio_rmb(vrh->weak_barriers);
58
59 i = *last_avail_idx & (vrh->vring.num - 1);
60
61 err = getu16(vrh, &head, &vrh->vring.avail->ring[i]);
62 if (err) {
63 vringh_bad("Failed to read head: idx %d address %p",
64 *last_avail_idx, &vrh->vring.avail->ring[i]);
65 return err;
66 }
67
68 if (head >= vrh->vring.num) {
69 vringh_bad("Guest says index %u > %u is available",
70 head, vrh->vring.num);
71 return -EINVAL;
72 }
73
74 (*last_avail_idx)++;
75 return head;
76}
77
78/**
79 * vringh_kiov_advance - skip bytes from vring_kiov
80 * @iov: an iov passed to vringh_getdesc_*() (updated as we consume)
81 * @len: the maximum length to advance
82 */
83void vringh_kiov_advance(struct vringh_kiov *iov, size_t len)
84{
85 while (len && iov->i < iov->used) {
86 size_t partlen = min(iov->iov[iov->i].iov_len, len);
87
88 iov->consumed += partlen;
89 iov->iov[iov->i].iov_len -= partlen;
90 iov->iov[iov->i].iov_base += partlen;
91
92 if (!iov->iov[iov->i].iov_len) {
93 /* Fix up old iov element then increment. */
94 iov->iov[iov->i].iov_len = iov->consumed;
95 iov->iov[iov->i].iov_base -= iov->consumed;
96
97 iov->consumed = 0;
98 iov->i++;
99 }
100
101 len -= partlen;
102 }
103}
104EXPORT_SYMBOL(vringh_kiov_advance);
105
106/* Copy some bytes to/from the iovec. Returns num copied. */
107static inline ssize_t vringh_iov_xfer(struct vringh *vrh,
108 struct vringh_kiov *iov,
109 void *ptr, size_t len,
110 int (*xfer)(const struct vringh *vrh,
111 void *addr, void *ptr,
112 size_t len))
113{
114 int err, done = 0;
115
116 while (len && iov->i < iov->used) {
117 size_t partlen;
118
119 partlen = min(iov->iov[iov->i].iov_len, len);
120 err = xfer(vrh, iov->iov[iov->i].iov_base, ptr, partlen);
121 if (err)
122 return err;
123 done += partlen;
124 len -= partlen;
125 ptr += partlen;
126 iov->consumed += partlen;
127 iov->iov[iov->i].iov_len -= partlen;
128 iov->iov[iov->i].iov_base += partlen;
129
130 if (!iov->iov[iov->i].iov_len) {
131 /* Fix up old iov element then increment. */
132 iov->iov[iov->i].iov_len = iov->consumed;
133 iov->iov[iov->i].iov_base -= iov->consumed;
134
135 iov->consumed = 0;
136 iov->i++;
137 }
138 }
139 return done;
140}
141
142/* May reduce *len if range is shorter. */
143static inline bool range_check(struct vringh *vrh, u64 addr, size_t *len,
144 struct vringh_range *range,
145 bool (*getrange)(struct vringh *,
146 u64, struct vringh_range *))
147{
148 if (addr < range->start || addr > range->end_incl) {
149 if (!getrange(vrh, addr, range))
150 return false;
151 }
152 BUG_ON(addr < range->start || addr > range->end_incl);
153
154 /* To end of memory? */
155 if (unlikely(addr + *len == 0)) {
156 if (range->end_incl == -1ULL)
157 return true;
158 goto truncate;
159 }
160
161 /* Otherwise, don't wrap. */
162 if (addr + *len < addr) {
163 vringh_bad("Wrapping descriptor %zu@0x%llx",
164 *len, (unsigned long long)addr);
165 return false;
166 }
167
168 if (unlikely(addr + *len - 1 > range->end_incl))
169 goto truncate;
170 return true;
171
172truncate:
173 *len = range->end_incl + 1 - addr;
174 return true;
175}
176
177static inline bool no_range_check(struct vringh *vrh, u64 addr, size_t *len,
178 struct vringh_range *range,
179 bool (*getrange)(struct vringh *,
180 u64, struct vringh_range *))
181{
182 return true;
183}
184
185/* No reason for this code to be inline. */
186static int move_to_indirect(const struct vringh *vrh,
187 int *up_next, u16 *i, void *addr,
188 const struct vring_desc *desc,
189 struct vring_desc **descs, int *desc_max)
190{
191 u32 len;
192
193 /* Indirect tables can't have indirect. */
194 if (*up_next != -1) {
195 vringh_bad("Multilevel indirect %u->%u", *up_next, *i);
196 return -EINVAL;
197 }
198
199 len = vringh32_to_cpu(vrh, desc->len);
200 if (unlikely(len % sizeof(struct vring_desc))) {
201 vringh_bad("Strange indirect len %u", desc->len);
202 return -EINVAL;
203 }
204
205 /* We will check this when we follow it! */
206 if (desc->flags & cpu_to_vringh16(vrh, VRING_DESC_F_NEXT))
207 *up_next = vringh16_to_cpu(vrh, desc->next);
208 else
209 *up_next = -2;
210 *descs = addr;
211 *desc_max = len / sizeof(struct vring_desc);
212
213 /* Now, start at the first indirect. */
214 *i = 0;
215 return 0;
216}
217
218static int resize_iovec(struct vringh_kiov *iov, gfp_t gfp)
219{
220 struct kvec *new;
221 unsigned int flag, new_num = (iov->max_num & ~VRINGH_IOV_ALLOCATED) * 2;
222
223 if (new_num < 8)
224 new_num = 8;
225
226 flag = (iov->max_num & VRINGH_IOV_ALLOCATED);
227 if (flag)
228 new = krealloc_array(iov->iov, new_num, sizeof(*new), gfp);
229 else {
230 new = kmalloc_objs(*new, new_num, gfp);
231 if (new) {
232 memcpy(new, iov->iov,
233 iov->max_num * sizeof(struct iovec));
234 flag = VRINGH_IOV_ALLOCATED;
235 }
236 }
237 if (!new)
238 return -ENOMEM;
239 iov->iov = new;
240 iov->max_num = (new_num | flag);
241 return 0;
242}
243
244static u16 __cold return_from_indirect(const struct vringh *vrh, int *up_next,
245 struct vring_desc **descs, int *desc_max)
246{
247 u16 i = *up_next;
248
249 *up_next = -1;
250 *descs = vrh->vring.desc;
251 *desc_max = vrh->vring.num;
252 return i;
253}
254
255static int slow_copy(struct vringh *vrh, void *dst, const void *src,
256 bool (*rcheck)(struct vringh *vrh, u64 addr, size_t *len,
257 struct vringh_range *range,
258 bool (*getrange)(struct vringh *vrh,
259 u64,
260 struct vringh_range *)),
261 bool (*getrange)(struct vringh *vrh,
262 u64 addr,
263 struct vringh_range *r),
264 struct vringh_range *range,
265 int (*copy)(const struct vringh *vrh,
266 void *dst, const void *src, size_t len))
267{
268 size_t part, len = sizeof(struct vring_desc);
269
270 do {
271 u64 addr;
272 int err;
273
274 part = len;
275 addr = (u64)(unsigned long)src - range->offset;
276
277 if (!rcheck(vrh, addr, &part, range, getrange))
278 return -EINVAL;
279
280 err = copy(vrh, dst, src, part);
281 if (err)
282 return err;
283
284 dst += part;
285 src += part;
286 len -= part;
287 } while (len);
288 return 0;
289}
290
291static inline int
292__vringh_iov(struct vringh *vrh, u16 i,
293 struct vringh_kiov *riov,
294 struct vringh_kiov *wiov,
295 bool (*rcheck)(struct vringh *vrh, u64 addr, size_t *len,
296 struct vringh_range *range,
297 bool (*getrange)(struct vringh *, u64,
298 struct vringh_range *)),
299 bool (*getrange)(struct vringh *, u64, struct vringh_range *),
300 gfp_t gfp,
301 int (*copy)(const struct vringh *vrh,
302 void *dst, const void *src, size_t len))
303{
304 int err, count = 0, indirect_count = 0, up_next, desc_max;
305 struct vring_desc desc, *descs;
306 struct vringh_range range = { -1ULL, 0 }, slowrange;
307 bool slow = false;
308
309 /* We start traversing vring's descriptor table. */
310 descs = vrh->vring.desc;
311 desc_max = vrh->vring.num;
312 up_next = -1;
313
314 /* You must want something! */
315 if (WARN_ON(!riov && !wiov))
316 return -EINVAL;
317
318 if (riov)
319 riov->i = riov->used = riov->consumed = 0;
320 if (wiov)
321 wiov->i = wiov->used = wiov->consumed = 0;
322
323 for (;;) {
324 void *addr;
325 struct vringh_kiov *iov;
326 size_t len;
327
328 if (unlikely(slow))
329 err = slow_copy(vrh, &desc, &descs[i], rcheck, getrange,
330 &slowrange, copy);
331 else
332 err = copy(vrh, &desc, &descs[i], sizeof(desc));
333 if (unlikely(err))
334 goto fail;
335
336 if (unlikely(desc.flags &
337 cpu_to_vringh16(vrh, VRING_DESC_F_INDIRECT))) {
338 u64 a = vringh64_to_cpu(vrh, desc.addr);
339
340 /* Make sure it's OK, and get offset. */
341 len = vringh32_to_cpu(vrh, desc.len);
342 if (!rcheck(vrh, a, &len, &range, getrange)) {
343 err = -EINVAL;
344 goto fail;
345 }
346
347 if (unlikely(len != vringh32_to_cpu(vrh, desc.len))) {
348 slow = true;
349 /* We need to save this range to use offset */
350 slowrange = range;
351 }
352
353 addr = (void *)(long)(a + range.offset);
354 err = move_to_indirect(vrh, &up_next, &i, addr, &desc,
355 &descs, &desc_max);
356 if (err)
357 goto fail;
358 continue;
359 }
360
361 if (up_next == -1)
362 count++;
363 else
364 indirect_count++;
365
366 if (count > vrh->vring.num || indirect_count > desc_max) {
367 vringh_bad("Descriptor loop in %p", descs);
368 err = -ELOOP;
369 goto fail;
370 }
371
372 if (desc.flags & cpu_to_vringh16(vrh, VRING_DESC_F_WRITE))
373 iov = wiov;
374 else {
375 iov = riov;
376 if (unlikely(wiov && wiov->used)) {
377 vringh_bad("Readable desc %p after writable",
378 &descs[i]);
379 err = -EINVAL;
380 goto fail;
381 }
382 }
383
384 if (!iov) {
385 vringh_bad("Unexpected %s desc",
386 !wiov ? "writable" : "readable");
387 err = -EPROTO;
388 goto fail;
389 }
390
391 again:
392 /* Make sure it's OK, and get offset. */
393 len = vringh32_to_cpu(vrh, desc.len);
394 if (!rcheck(vrh, vringh64_to_cpu(vrh, desc.addr), &len, &range,
395 getrange)) {
396 err = -EINVAL;
397 goto fail;
398 }
399 addr = (void *)(unsigned long)(vringh64_to_cpu(vrh, desc.addr) +
400 range.offset);
401
402 if (unlikely(iov->used == (iov->max_num & ~VRINGH_IOV_ALLOCATED))) {
403 err = resize_iovec(iov, gfp);
404 if (err)
405 goto fail;
406 }
407
408 iov->iov[iov->used].iov_base = addr;
409 iov->iov[iov->used].iov_len = len;
410 iov->used++;
411
412 if (unlikely(len != vringh32_to_cpu(vrh, desc.len))) {
413 desc.len = cpu_to_vringh32(vrh,
414 vringh32_to_cpu(vrh, desc.len) - len);
415 desc.addr = cpu_to_vringh64(vrh,
416 vringh64_to_cpu(vrh, desc.addr) + len);
417 goto again;
418 }
419
420 if (desc.flags & cpu_to_vringh16(vrh, VRING_DESC_F_NEXT)) {
421 i = vringh16_to_cpu(vrh, desc.next);
422 } else {
423 /* Just in case we need to finish traversing above. */
424 if (unlikely(up_next > 0)) {
425 i = return_from_indirect(vrh, &up_next,
426 &descs, &desc_max);
427 slow = false;
428 indirect_count = 0;
429 } else
430 break;
431 }
432
433 if (i >= desc_max) {
434 vringh_bad("Chained index %u > %u", i, desc_max);
435 err = -EINVAL;
436 goto fail;
437 }
438 }
439
440 return 0;
441
442fail:
443 return err;
444}
445
446static inline int __vringh_complete(struct vringh *vrh,
447 const struct vring_used_elem *used,
448 unsigned int num_used,
449 int (*putu16)(const struct vringh *vrh,
450 __virtio16 *p, u16 val),
451 int (*putused)(const struct vringh *vrh,
452 struct vring_used_elem *dst,
453 const struct vring_used_elem
454 *src, unsigned num))
455{
456 struct vring_used *used_ring;
457 int err;
458 u16 used_idx, off;
459
460 used_ring = vrh->vring.used;
461 used_idx = vrh->last_used_idx + vrh->completed;
462
463 off = used_idx % vrh->vring.num;
464
465 /* Compiler knows num_used == 1 sometimes, hence extra check */
466 if (num_used > 1 && unlikely(off + num_used >= vrh->vring.num)) {
467 u16 part = vrh->vring.num - off;
468 err = putused(vrh, &used_ring->ring[off], used, part);
469 if (!err)
470 err = putused(vrh, &used_ring->ring[0], used + part,
471 num_used - part);
472 } else
473 err = putused(vrh, &used_ring->ring[off], used, num_used);
474
475 if (err) {
476 vringh_bad("Failed to write %u used entries %u at %p",
477 num_used, off, &used_ring->ring[off]);
478 return err;
479 }
480
481 /* Make sure buffer is written before we update index. */
482 virtio_wmb(vrh->weak_barriers);
483
484 err = putu16(vrh, &vrh->vring.used->idx, used_idx + num_used);
485 if (err) {
486 vringh_bad("Failed to update used index at %p",
487 &vrh->vring.used->idx);
488 return err;
489 }
490
491 vrh->completed += num_used;
492 return 0;
493}
494
495
496static inline int __vringh_need_notify(struct vringh *vrh,
497 int (*getu16)(const struct vringh *vrh,
498 u16 *val,
499 const __virtio16 *p))
500{
501 bool notify;
502 u16 used_event;
503 int err;
504
505 /* Flush out used index update. This is paired with the
506 * barrier that the Guest executes when enabling
507 * interrupts. */
508 virtio_mb(vrh->weak_barriers);
509
510 /* Old-style, without event indices. */
511 if (!vrh->event_indices) {
512 u16 flags;
513 err = getu16(vrh, &flags, &vrh->vring.avail->flags);
514 if (err) {
515 vringh_bad("Failed to get flags at %p",
516 &vrh->vring.avail->flags);
517 return err;
518 }
519 return (!(flags & VRING_AVAIL_F_NO_INTERRUPT));
520 }
521
522 /* Modern: we know when other side wants to know. */
523 err = getu16(vrh, &used_event, &vring_used_event(&vrh->vring));
524 if (err) {
525 vringh_bad("Failed to get used event idx at %p",
526 &vring_used_event(&vrh->vring));
527 return err;
528 }
529
530 /* Just in case we added so many that we wrap. */
531 if (unlikely(vrh->completed > 0xffff))
532 notify = true;
533 else
534 notify = vring_need_event(used_event,
535 vrh->last_used_idx + vrh->completed,
536 vrh->last_used_idx);
537
538 vrh->last_used_idx += vrh->completed;
539 vrh->completed = 0;
540 return notify;
541}
542
543static inline bool __vringh_notify_enable(struct vringh *vrh,
544 int (*getu16)(const struct vringh *vrh,
545 u16 *val, const __virtio16 *p),
546 int (*putu16)(const struct vringh *vrh,
547 __virtio16 *p, u16 val))
548{
549 u16 avail;
550
551 if (!vrh->event_indices) {
552 /* Old-school; update flags. */
553 if (putu16(vrh, &vrh->vring.used->flags, 0) != 0) {
554 vringh_bad("Clearing used flags %p",
555 &vrh->vring.used->flags);
556 return true;
557 }
558 } else {
559 if (putu16(vrh, &vring_avail_event(&vrh->vring),
560 vrh->last_avail_idx) != 0) {
561 vringh_bad("Updating avail event index %p",
562 &vring_avail_event(&vrh->vring));
563 return true;
564 }
565 }
566
567 /* They could have slipped one in as we were doing that: make
568 * sure it's written, then check again. */
569 virtio_mb(vrh->weak_barriers);
570
571 if (getu16(vrh, &avail, &vrh->vring.avail->idx) != 0) {
572 vringh_bad("Failed to check avail idx at %p",
573 &vrh->vring.avail->idx);
574 return true;
575 }
576
577 /* This is unlikely, so we just leave notifications enabled
578 * (if we're using event_indices, we'll only get one
579 * notification anyway). */
580 return avail == vrh->last_avail_idx;
581}
582
583static inline void __vringh_notify_disable(struct vringh *vrh,
584 int (*putu16)(const struct vringh *vrh,
585 __virtio16 *p, u16 val))
586{
587 if (!vrh->event_indices) {
588 /* Old-school; update flags. */
589 if (putu16(vrh, &vrh->vring.used->flags,
590 VRING_USED_F_NO_NOTIFY)) {
591 vringh_bad("Setting used flags %p",
592 &vrh->vring.used->flags);
593 }
594 }
595}
596
597/* Userspace access helpers: in this case, addresses are really userspace. */
598static inline int getu16_user(const struct vringh *vrh, u16 *val, const __virtio16 *p)
599{
600 __virtio16 v = 0;
601 int rc = get_user(v, (__force __virtio16 __user *)p);
602 *val = vringh16_to_cpu(vrh, v);
603 return rc;
604}
605
606static inline int putu16_user(const struct vringh *vrh, __virtio16 *p, u16 val)
607{
608 __virtio16 v = cpu_to_vringh16(vrh, val);
609 return put_user(v, (__force __virtio16 __user *)p);
610}
611
612static inline int copydesc_user(const struct vringh *vrh,
613 void *dst, const void *src, size_t len)
614{
615 return copy_from_user(dst, (__force void __user *)src, len) ?
616 -EFAULT : 0;
617}
618
619static inline int putused_user(const struct vringh *vrh,
620 struct vring_used_elem *dst,
621 const struct vring_used_elem *src,
622 unsigned int num)
623{
624 return copy_to_user((__force void __user *)dst, src,
625 sizeof(*dst) * num) ? -EFAULT : 0;
626}
627
628static inline int xfer_from_user(const struct vringh *vrh, void *src,
629 void *dst, size_t len)
630{
631 return copy_from_user(dst, (__force void __user *)src, len) ?
632 -EFAULT : 0;
633}
634
635static inline int xfer_to_user(const struct vringh *vrh,
636 void *dst, void *src, size_t len)
637{
638 return copy_to_user((__force void __user *)dst, src, len) ?
639 -EFAULT : 0;
640}
641
642/**
643 * vringh_init_user - initialize a vringh for a userspace vring.
644 * @vrh: the vringh to initialize.
645 * @features: the feature bits for this ring.
646 * @num: the number of elements.
647 * @weak_barriers: true if we only need memory barriers, not I/O.
648 * @desc: the userspace descriptor pointer.
649 * @avail: the userspace avail pointer.
650 * @used: the userspace used pointer.
651 *
652 * Returns an error if num is invalid: you should check pointers
653 * yourself!
654 */
655int vringh_init_user(struct vringh *vrh, u64 features,
656 unsigned int num, bool weak_barriers,
657 vring_desc_t __user *desc,
658 vring_avail_t __user *avail,
659 vring_used_t __user *used)
660{
661 /* Sane power of 2 please! */
662 if (!num || num > 0xffff || (num & (num - 1))) {
663 vringh_bad("Bad ring size %u", num);
664 return -EINVAL;
665 }
666
667 vrh->little_endian = (features & (1ULL << VIRTIO_F_VERSION_1));
668 vrh->event_indices = (features & (1 << VIRTIO_RING_F_EVENT_IDX));
669 vrh->weak_barriers = weak_barriers;
670 vrh->completed = 0;
671 vrh->last_avail_idx = 0;
672 vrh->last_used_idx = 0;
673 vrh->vring.num = num;
674 /* vring expects kernel addresses, but only used via accessors. */
675 vrh->vring.desc = (__force struct vring_desc *)desc;
676 vrh->vring.avail = (__force struct vring_avail *)avail;
677 vrh->vring.used = (__force struct vring_used *)used;
678 return 0;
679}
680EXPORT_SYMBOL(vringh_init_user);
681
682/**
683 * vringh_getdesc_user - get next available descriptor from userspace ring.
684 * @vrh: the userspace vring.
685 * @riov: where to put the readable descriptors (or NULL)
686 * @wiov: where to put the writable descriptors (or NULL)
687 * @getrange: function to call to check ranges.
688 * @head: head index we received, for passing to vringh_complete_user().
689 *
690 * Returns 0 if there was no descriptor, 1 if there was, or -errno.
691 *
692 * Note that on error return, you can tell the difference between an
693 * invalid ring and a single invalid descriptor: in the former case,
694 * *head will be vrh->vring.num. You may be able to ignore an invalid
695 * descriptor, but there's not much you can do with an invalid ring.
696 *
697 * Note that you can reuse riov and wiov with subsequent calls. Content is
698 * overwritten and memory reallocated if more space is needed.
699 * When you don't have to use riov and wiov anymore, you should clean up them
700 * calling vringh_iov_cleanup() to release the memory, even on error!
701 */
702int vringh_getdesc_user(struct vringh *vrh,
703 struct vringh_iov *riov,
704 struct vringh_iov *wiov,
705 bool (*getrange)(struct vringh *vrh,
706 u64 addr, struct vringh_range *r),
707 u16 *head)
708{
709 int err;
710
711 *head = vrh->vring.num;
712 err = __vringh_get_head(vrh, getu16_user, &vrh->last_avail_idx);
713 if (err < 0)
714 return err;
715
716 /* Empty... */
717 if (err == vrh->vring.num)
718 return 0;
719
720 /* We need the layouts to be the identical for this to work */
721 BUILD_BUG_ON(sizeof(struct vringh_kiov) != sizeof(struct vringh_iov));
722 BUILD_BUG_ON(offsetof(struct vringh_kiov, iov) !=
723 offsetof(struct vringh_iov, iov));
724 BUILD_BUG_ON(offsetof(struct vringh_kiov, i) !=
725 offsetof(struct vringh_iov, i));
726 BUILD_BUG_ON(offsetof(struct vringh_kiov, used) !=
727 offsetof(struct vringh_iov, used));
728 BUILD_BUG_ON(offsetof(struct vringh_kiov, max_num) !=
729 offsetof(struct vringh_iov, max_num));
730 BUILD_BUG_ON(sizeof(struct iovec) != sizeof(struct kvec));
731 BUILD_BUG_ON(offsetof(struct iovec, iov_base) !=
732 offsetof(struct kvec, iov_base));
733 BUILD_BUG_ON(offsetof(struct iovec, iov_len) !=
734 offsetof(struct kvec, iov_len));
735 BUILD_BUG_ON(sizeof(((struct iovec *)NULL)->iov_base)
736 != sizeof(((struct kvec *)NULL)->iov_base));
737 BUILD_BUG_ON(sizeof(((struct iovec *)NULL)->iov_len)
738 != sizeof(((struct kvec *)NULL)->iov_len));
739
740 *head = err;
741 err = __vringh_iov(vrh, *head, (struct vringh_kiov *)riov,
742 (struct vringh_kiov *)wiov,
743 range_check, getrange, GFP_KERNEL, copydesc_user);
744 if (err)
745 return err;
746
747 return 1;
748}
749EXPORT_SYMBOL(vringh_getdesc_user);
750
751/**
752 * vringh_iov_pull_user - copy bytes from vring_iov.
753 * @riov: the riov as passed to vringh_getdesc_user() (updated as we consume)
754 * @dst: the place to copy.
755 * @len: the maximum length to copy.
756 *
757 * Returns the bytes copied <= len or a negative errno.
758 */
759ssize_t vringh_iov_pull_user(struct vringh_iov *riov, void *dst, size_t len)
760{
761 return vringh_iov_xfer(NULL, (struct vringh_kiov *)riov,
762 dst, len, xfer_from_user);
763}
764EXPORT_SYMBOL(vringh_iov_pull_user);
765
766/**
767 * vringh_iov_push_user - copy bytes into vring_iov.
768 * @wiov: the wiov as passed to vringh_getdesc_user() (updated as we consume)
769 * @src: the place to copy from.
770 * @len: the maximum length to copy.
771 *
772 * Returns the bytes copied <= len or a negative errno.
773 */
774ssize_t vringh_iov_push_user(struct vringh_iov *wiov,
775 const void *src, size_t len)
776{
777 return vringh_iov_xfer(NULL, (struct vringh_kiov *)wiov,
778 (void *)src, len, xfer_to_user);
779}
780EXPORT_SYMBOL(vringh_iov_push_user);
781
782/**
783 * vringh_complete_user - we've finished with descriptor, publish it.
784 * @vrh: the vring.
785 * @head: the head as filled in by vringh_getdesc_user.
786 * @len: the length of data we have written.
787 *
788 * You should check vringh_need_notify_user() after one or more calls
789 * to this function.
790 */
791int vringh_complete_user(struct vringh *vrh, u16 head, u32 len)
792{
793 struct vring_used_elem used;
794
795 used.id = cpu_to_vringh32(vrh, head);
796 used.len = cpu_to_vringh32(vrh, len);
797 return __vringh_complete(vrh, &used, 1, putu16_user, putused_user);
798}
799EXPORT_SYMBOL(vringh_complete_user);
800
801/**
802 * vringh_complete_multi_user - we've finished with many descriptors.
803 * @vrh: the vring.
804 * @used: the head, length pairs.
805 * @num_used: the number of used elements.
806 *
807 * You should check vringh_need_notify_user() after one or more calls
808 * to this function.
809 */
810int vringh_complete_multi_user(struct vringh *vrh,
811 const struct vring_used_elem used[],
812 unsigned num_used)
813{
814 return __vringh_complete(vrh, used, num_used,
815 putu16_user, putused_user);
816}
817EXPORT_SYMBOL(vringh_complete_multi_user);
818
819/**
820 * vringh_notify_enable_user - we want to know if something changes.
821 * @vrh: the vring.
822 *
823 * This always enables notifications, but returns false if there are
824 * now more buffers available in the vring.
825 */
826bool vringh_notify_enable_user(struct vringh *vrh)
827{
828 return __vringh_notify_enable(vrh, getu16_user, putu16_user);
829}
830EXPORT_SYMBOL(vringh_notify_enable_user);
831
832/**
833 * vringh_notify_disable_user - don't tell us if something changes.
834 * @vrh: the vring.
835 *
836 * This is our normal running state: we disable and then only enable when
837 * we're going to sleep.
838 */
839void vringh_notify_disable_user(struct vringh *vrh)
840{
841 __vringh_notify_disable(vrh, putu16_user);
842}
843EXPORT_SYMBOL(vringh_notify_disable_user);
844
845/**
846 * vringh_need_notify_user - must we tell the other side about used buffers?
847 * @vrh: the vring we've called vringh_complete_user() on.
848 *
849 * Returns -errno or 0 if we don't need to tell the other side, 1 if we do.
850 */
851int vringh_need_notify_user(struct vringh *vrh)
852{
853 return __vringh_need_notify(vrh, getu16_user);
854}
855EXPORT_SYMBOL(vringh_need_notify_user);
856
857/* Kernelspace access helpers. */
858static inline int getu16_kern(const struct vringh *vrh,
859 u16 *val, const __virtio16 *p)
860{
861 *val = vringh16_to_cpu(vrh, READ_ONCE(*p));
862 return 0;
863}
864
865static inline int putu16_kern(const struct vringh *vrh, __virtio16 *p, u16 val)
866{
867 WRITE_ONCE(*p, cpu_to_vringh16(vrh, val));
868 return 0;
869}
870
871static inline int copydesc_kern(const struct vringh *vrh,
872 void *dst, const void *src, size_t len)
873{
874 memcpy(dst, src, len);
875 return 0;
876}
877
878static inline int putused_kern(const struct vringh *vrh,
879 struct vring_used_elem *dst,
880 const struct vring_used_elem *src,
881 unsigned int num)
882{
883 memcpy(dst, src, num * sizeof(*dst));
884 return 0;
885}
886
887/**
888 * vringh_init_kern - initialize a vringh for a kernelspace vring.
889 * @vrh: the vringh to initialize.
890 * @features: the feature bits for this ring.
891 * @num: the number of elements.
892 * @weak_barriers: true if we only need memory barriers, not I/O.
893 * @desc: the userspace descriptor pointer.
894 * @avail: the userspace avail pointer.
895 * @used: the userspace used pointer.
896 *
897 * Returns an error if num is invalid.
898 */
899int vringh_init_kern(struct vringh *vrh, u64 features,
900 unsigned int num, bool weak_barriers,
901 struct vring_desc *desc,
902 struct vring_avail *avail,
903 struct vring_used *used)
904{
905 /* Sane power of 2 please! */
906 if (!num || num > 0xffff || (num & (num - 1))) {
907 vringh_bad("Bad ring size %u", num);
908 return -EINVAL;
909 }
910
911 vrh->little_endian = (features & (1ULL << VIRTIO_F_VERSION_1));
912 vrh->event_indices = (features & (1 << VIRTIO_RING_F_EVENT_IDX));
913 vrh->weak_barriers = weak_barriers;
914 vrh->completed = 0;
915 vrh->last_avail_idx = 0;
916 vrh->last_used_idx = 0;
917 vrh->vring.num = num;
918 vrh->vring.desc = desc;
919 vrh->vring.avail = avail;
920 vrh->vring.used = used;
921 return 0;
922}
923EXPORT_SYMBOL(vringh_init_kern);
924
925/**
926 * vringh_getdesc_kern - get next available descriptor from kernelspace ring.
927 * @vrh: the kernelspace vring.
928 * @riov: where to put the readable descriptors (or NULL)
929 * @wiov: where to put the writable descriptors (or NULL)
930 * @head: head index we received, for passing to vringh_complete_kern().
931 * @gfp: flags for allocating larger riov/wiov.
932 *
933 * Returns 0 if there was no descriptor, 1 if there was, or -errno.
934 *
935 * Note that on error return, you can tell the difference between an
936 * invalid ring and a single invalid descriptor: in the former case,
937 * *head will be vrh->vring.num. You may be able to ignore an invalid
938 * descriptor, but there's not much you can do with an invalid ring.
939 *
940 * Note that you can reuse riov and wiov with subsequent calls. Content is
941 * overwritten and memory reallocated if more space is needed.
942 * When you don't have to use riov and wiov anymore, you should clean up them
943 * calling vringh_kiov_cleanup() to release the memory, even on error!
944 */
945int vringh_getdesc_kern(struct vringh *vrh,
946 struct vringh_kiov *riov,
947 struct vringh_kiov *wiov,
948 u16 *head,
949 gfp_t gfp)
950{
951 int err;
952
953 err = __vringh_get_head(vrh, getu16_kern, &vrh->last_avail_idx);
954 if (err < 0)
955 return err;
956
957 /* Empty... */
958 if (err == vrh->vring.num)
959 return 0;
960
961 *head = err;
962 err = __vringh_iov(vrh, *head, riov, wiov, no_range_check, NULL,
963 gfp, copydesc_kern);
964 if (err)
965 return err;
966
967 return 1;
968}
969EXPORT_SYMBOL(vringh_getdesc_kern);
970
971/**
972 * vringh_complete_kern - we've finished with descriptor, publish it.
973 * @vrh: the vring.
974 * @head: the head as filled in by vringh_getdesc_kern.
975 * @len: the length of data we have written.
976 *
977 * You should check vringh_need_notify_kern() after one or more calls
978 * to this function.
979 */
980int vringh_complete_kern(struct vringh *vrh, u16 head, u32 len)
981{
982 struct vring_used_elem used;
983
984 used.id = cpu_to_vringh32(vrh, head);
985 used.len = cpu_to_vringh32(vrh, len);
986
987 return __vringh_complete(vrh, &used, 1, putu16_kern, putused_kern);
988}
989EXPORT_SYMBOL(vringh_complete_kern);
990
991/**
992 * vringh_notify_enable_kern - we want to know if something changes.
993 * @vrh: the vring.
994 *
995 * This always enables notifications, but returns false if there are
996 * now more buffers available in the vring.
997 */
998bool vringh_notify_enable_kern(struct vringh *vrh)
999{
1000 return __vringh_notify_enable(vrh, getu16_kern, putu16_kern);
1001}
1002EXPORT_SYMBOL(vringh_notify_enable_kern);
1003
1004/**
1005 * vringh_notify_disable_kern - don't tell us if something changes.
1006 * @vrh: the vring.
1007 *
1008 * This is our normal running state: we disable and then only enable when
1009 * we're going to sleep.
1010 */
1011void vringh_notify_disable_kern(struct vringh *vrh)
1012{
1013 __vringh_notify_disable(vrh, putu16_kern);
1014}
1015EXPORT_SYMBOL(vringh_notify_disable_kern);
1016
1017/**
1018 * vringh_need_notify_kern - must we tell the other side about used buffers?
1019 * @vrh: the vring we've called vringh_complete_kern() on.
1020 *
1021 * Returns -errno or 0 if we don't need to tell the other side, 1 if we do.
1022 */
1023int vringh_need_notify_kern(struct vringh *vrh)
1024{
1025 return __vringh_need_notify(vrh, getu16_kern);
1026}
1027EXPORT_SYMBOL(vringh_need_notify_kern);
1028
1029#if IS_REACHABLE(CONFIG_VHOST_IOTLB)
1030
1031struct iotlb_vec {
1032 union {
1033 struct iovec *iovec;
1034 struct bio_vec *bvec;
1035 } iov;
1036 size_t count;
1037};
1038
1039static int iotlb_translate(const struct vringh *vrh,
1040 u64 addr, u64 len, u64 *translated,
1041 struct iotlb_vec *ivec, u32 perm)
1042{
1043 struct vhost_iotlb_map *map;
1044 struct vhost_iotlb *iotlb = vrh->iotlb;
1045 int ret = 0;
1046 u64 s = 0, last = addr + len - 1;
1047
1048 spin_lock(vrh->iotlb_lock);
1049
1050 while (len > s) {
1051 uintptr_t io_addr;
1052 size_t io_len;
1053 u64 size;
1054
1055 if (unlikely(ret >= ivec->count)) {
1056 ret = -ENOBUFS;
1057 break;
1058 }
1059
1060 map = vhost_iotlb_itree_first(iotlb, addr, last);
1061 if (!map || map->start > addr) {
1062 ret = -EINVAL;
1063 break;
1064 } else if (!(map->perm & perm)) {
1065 ret = -EPERM;
1066 break;
1067 }
1068
1069 size = map->size - addr + map->start;
1070 io_len = min(len - s, size);
1071 io_addr = map->addr - map->start + addr;
1072
1073 if (vrh->use_va) {
1074 struct iovec *iovec = ivec->iov.iovec;
1075
1076 iovec[ret].iov_len = io_len;
1077 iovec[ret].iov_base = (void __user *)io_addr;
1078 } else {
1079 u64 pfn = io_addr >> PAGE_SHIFT;
1080 struct bio_vec *bvec = ivec->iov.bvec;
1081
1082 bvec_set_page(&bvec[ret], pfn_to_page(pfn), io_len,
1083 io_addr & (PAGE_SIZE - 1));
1084 }
1085
1086 s += size;
1087 addr += size;
1088 ++ret;
1089 }
1090
1091 spin_unlock(vrh->iotlb_lock);
1092
1093 if (translated)
1094 *translated = min(len, s);
1095
1096 return ret;
1097}
1098
1099#define IOTLB_IOV_STRIDE 16
1100
1101static inline int copy_from_iotlb(const struct vringh *vrh, void *dst,
1102 void *src, size_t len)
1103{
1104 struct iotlb_vec ivec;
1105 union {
1106 struct iovec iovec[IOTLB_IOV_STRIDE];
1107 struct bio_vec bvec[IOTLB_IOV_STRIDE];
1108 } iov;
1109 u64 total_translated = 0;
1110
1111 ivec.iov.iovec = iov.iovec;
1112 ivec.count = IOTLB_IOV_STRIDE;
1113
1114 while (total_translated < len) {
1115 struct iov_iter iter;
1116 u64 translated;
1117 int ret;
1118 size_t size;
1119
1120 ret = iotlb_translate(vrh, (u64)(uintptr_t)src,
1121 len - total_translated, &translated,
1122 &ivec, VHOST_MAP_RO);
1123 if (ret == -ENOBUFS)
1124 ret = IOTLB_IOV_STRIDE;
1125 else if (ret < 0)
1126 return ret;
1127
1128 if (vrh->use_va) {
1129 iov_iter_init(&iter, ITER_SOURCE, ivec.iov.iovec, ret,
1130 translated);
1131 } else {
1132 iov_iter_bvec(&iter, ITER_SOURCE, ivec.iov.bvec, ret,
1133 translated);
1134 }
1135
1136 size = copy_from_iter(dst, translated, &iter);
1137 if (size != translated)
1138 return -EFAULT;
1139
1140 src += translated;
1141 dst += translated;
1142 total_translated += translated;
1143 }
1144
1145 return total_translated;
1146}
1147
1148static inline int copy_to_iotlb(const struct vringh *vrh, void *dst,
1149 void *src, size_t len)
1150{
1151 struct iotlb_vec ivec;
1152 union {
1153 struct iovec iovec[IOTLB_IOV_STRIDE];
1154 struct bio_vec bvec[IOTLB_IOV_STRIDE];
1155 } iov;
1156 u64 total_translated = 0;
1157
1158 ivec.iov.iovec = iov.iovec;
1159 ivec.count = IOTLB_IOV_STRIDE;
1160
1161 while (total_translated < len) {
1162 struct iov_iter iter;
1163 u64 translated;
1164 int ret;
1165 size_t size;
1166
1167 ret = iotlb_translate(vrh, (u64)(uintptr_t)dst,
1168 len - total_translated, &translated,
1169 &ivec, VHOST_MAP_WO);
1170 if (ret == -ENOBUFS)
1171 ret = IOTLB_IOV_STRIDE;
1172 else if (ret < 0)
1173 return ret;
1174
1175 if (vrh->use_va) {
1176 iov_iter_init(&iter, ITER_DEST, ivec.iov.iovec, ret,
1177 translated);
1178 } else {
1179 iov_iter_bvec(&iter, ITER_DEST, ivec.iov.bvec, ret,
1180 translated);
1181 }
1182
1183 size = copy_to_iter(src, translated, &iter);
1184 if (size != translated)
1185 return -EFAULT;
1186
1187 src += translated;
1188 dst += translated;
1189 total_translated += translated;
1190 }
1191
1192 return total_translated;
1193}
1194
1195static inline int getu16_iotlb(const struct vringh *vrh,
1196 u16 *val, const __virtio16 *p)
1197{
1198 struct iotlb_vec ivec;
1199 union {
1200 struct iovec iovec[1];
1201 struct bio_vec bvec[1];
1202 } iov;
1203 __virtio16 tmp;
1204 int ret;
1205
1206 ivec.iov.iovec = iov.iovec;
1207 ivec.count = 1;
1208
1209 /* Atomic read is needed for getu16 */
1210 ret = iotlb_translate(vrh, (u64)(uintptr_t)p, sizeof(*p),
1211 NULL, &ivec, VHOST_MAP_RO);
1212 if (ret < 0)
1213 return ret;
1214
1215 if (vrh->use_va) {
1216 ret = __get_user(tmp, (__virtio16 __user *)ivec.iov.iovec[0].iov_base);
1217 if (ret)
1218 return ret;
1219 } else {
1220 __virtio16 *from = bvec_kmap_local(&ivec.iov.bvec[0]);
1221
1222 tmp = READ_ONCE(*from);
1223 kunmap_local(from);
1224 }
1225
1226 *val = vringh16_to_cpu(vrh, tmp);
1227
1228 return 0;
1229}
1230
1231static inline int putu16_iotlb(const struct vringh *vrh,
1232 __virtio16 *p, u16 val)
1233{
1234 struct iotlb_vec ivec;
1235 union {
1236 struct iovec iovec;
1237 struct bio_vec bvec;
1238 } iov;
1239 __virtio16 tmp;
1240 int ret;
1241
1242 ivec.iov.iovec = &iov.iovec;
1243 ivec.count = 1;
1244
1245 /* Atomic write is needed for putu16 */
1246 ret = iotlb_translate(vrh, (u64)(uintptr_t)p, sizeof(*p),
1247 NULL, &ivec, VHOST_MAP_RO);
1248 if (ret < 0)
1249 return ret;
1250
1251 tmp = cpu_to_vringh16(vrh, val);
1252
1253 if (vrh->use_va) {
1254 ret = __put_user(tmp, (__virtio16 __user *)ivec.iov.iovec[0].iov_base);
1255 if (ret)
1256 return ret;
1257 } else {
1258 __virtio16 *to = bvec_kmap_local(&ivec.iov.bvec[0]);
1259
1260 WRITE_ONCE(*to, tmp);
1261 kunmap_local(to);
1262 }
1263
1264 return 0;
1265}
1266
1267static inline int copydesc_iotlb(const struct vringh *vrh,
1268 void *dst, const void *src, size_t len)
1269{
1270 int ret;
1271
1272 ret = copy_from_iotlb(vrh, dst, (void *)src, len);
1273 if (ret != len)
1274 return -EFAULT;
1275
1276 return 0;
1277}
1278
1279static inline int xfer_from_iotlb(const struct vringh *vrh, void *src,
1280 void *dst, size_t len)
1281{
1282 int ret;
1283
1284 ret = copy_from_iotlb(vrh, dst, src, len);
1285 if (ret != len)
1286 return -EFAULT;
1287
1288 return 0;
1289}
1290
1291static inline int xfer_to_iotlb(const struct vringh *vrh,
1292 void *dst, void *src, size_t len)
1293{
1294 int ret;
1295
1296 ret = copy_to_iotlb(vrh, dst, src, len);
1297 if (ret != len)
1298 return -EFAULT;
1299
1300 return 0;
1301}
1302
1303static inline int putused_iotlb(const struct vringh *vrh,
1304 struct vring_used_elem *dst,
1305 const struct vring_used_elem *src,
1306 unsigned int num)
1307{
1308 int size = num * sizeof(*dst);
1309 int ret;
1310
1311 ret = copy_to_iotlb(vrh, dst, (void *)src, num * sizeof(*dst));
1312 if (ret != size)
1313 return -EFAULT;
1314
1315 return 0;
1316}
1317
1318/**
1319 * vringh_init_iotlb - initialize a vringh for a ring with IOTLB.
1320 * @vrh: the vringh to initialize.
1321 * @features: the feature bits for this ring.
1322 * @num: the number of elements.
1323 * @weak_barriers: true if we only need memory barriers, not I/O.
1324 * @desc: the userspace descriptor pointer.
1325 * @avail: the userspace avail pointer.
1326 * @used: the userspace used pointer.
1327 *
1328 * Returns an error if num is invalid.
1329 */
1330int vringh_init_iotlb(struct vringh *vrh, u64 features,
1331 unsigned int num, bool weak_barriers,
1332 struct vring_desc *desc,
1333 struct vring_avail *avail,
1334 struct vring_used *used)
1335{
1336 vrh->use_va = false;
1337
1338 return vringh_init_kern(vrh, features, num, weak_barriers,
1339 desc, avail, used);
1340}
1341EXPORT_SYMBOL(vringh_init_iotlb);
1342
1343/**
1344 * vringh_init_iotlb_va - initialize a vringh for a ring with IOTLB containing
1345 * user VA.
1346 * @vrh: the vringh to initialize.
1347 * @features: the feature bits for this ring.
1348 * @num: the number of elements.
1349 * @weak_barriers: true if we only need memory barriers, not I/O.
1350 * @desc: the userspace descriptor pointer.
1351 * @avail: the userspace avail pointer.
1352 * @used: the userspace used pointer.
1353 *
1354 * Returns an error if num is invalid.
1355 */
1356int vringh_init_iotlb_va(struct vringh *vrh, u64 features,
1357 unsigned int num, bool weak_barriers,
1358 struct vring_desc *desc,
1359 struct vring_avail *avail,
1360 struct vring_used *used)
1361{
1362 vrh->use_va = true;
1363
1364 return vringh_init_kern(vrh, features, num, weak_barriers,
1365 desc, avail, used);
1366}
1367EXPORT_SYMBOL(vringh_init_iotlb_va);
1368
1369/**
1370 * vringh_set_iotlb - initialize a vringh for a ring with IOTLB.
1371 * @vrh: the vring
1372 * @iotlb: iotlb associated with this vring
1373 * @iotlb_lock: spinlock to synchronize the iotlb accesses
1374 */
1375void vringh_set_iotlb(struct vringh *vrh, struct vhost_iotlb *iotlb,
1376 spinlock_t *iotlb_lock)
1377{
1378 vrh->iotlb = iotlb;
1379 vrh->iotlb_lock = iotlb_lock;
1380}
1381EXPORT_SYMBOL(vringh_set_iotlb);
1382
1383/**
1384 * vringh_getdesc_iotlb - get next available descriptor from ring with
1385 * IOTLB.
1386 * @vrh: the kernelspace vring.
1387 * @riov: where to put the readable descriptors (or NULL)
1388 * @wiov: where to put the writable descriptors (or NULL)
1389 * @head: head index we received, for passing to vringh_complete_iotlb().
1390 * @gfp: flags for allocating larger riov/wiov.
1391 *
1392 * Returns 0 if there was no descriptor, 1 if there was, or -errno.
1393 *
1394 * Note that on error return, you can tell the difference between an
1395 * invalid ring and a single invalid descriptor: in the former case,
1396 * *head will be vrh->vring.num. You may be able to ignore an invalid
1397 * descriptor, but there's not much you can do with an invalid ring.
1398 *
1399 * Note that you can reuse riov and wiov with subsequent calls. Content is
1400 * overwritten and memory reallocated if more space is needed.
1401 * When you don't have to use riov and wiov anymore, you should clean up them
1402 * calling vringh_kiov_cleanup() to release the memory, even on error!
1403 */
1404int vringh_getdesc_iotlb(struct vringh *vrh,
1405 struct vringh_kiov *riov,
1406 struct vringh_kiov *wiov,
1407 u16 *head,
1408 gfp_t gfp)
1409{
1410 int err;
1411
1412 err = __vringh_get_head(vrh, getu16_iotlb, &vrh->last_avail_idx);
1413 if (err < 0)
1414 return err;
1415
1416 /* Empty... */
1417 if (err == vrh->vring.num)
1418 return 0;
1419
1420 *head = err;
1421 err = __vringh_iov(vrh, *head, riov, wiov, no_range_check, NULL,
1422 gfp, copydesc_iotlb);
1423 if (err)
1424 return err;
1425
1426 return 1;
1427}
1428EXPORT_SYMBOL(vringh_getdesc_iotlb);
1429
1430/**
1431 * vringh_iov_pull_iotlb - copy bytes from vring_iov.
1432 * @vrh: the vring.
1433 * @riov: the riov as passed to vringh_getdesc_iotlb() (updated as we consume)
1434 * @dst: the place to copy.
1435 * @len: the maximum length to copy.
1436 *
1437 * Returns the bytes copied <= len or a negative errno.
1438 */
1439ssize_t vringh_iov_pull_iotlb(struct vringh *vrh,
1440 struct vringh_kiov *riov,
1441 void *dst, size_t len)
1442{
1443 return vringh_iov_xfer(vrh, riov, dst, len, xfer_from_iotlb);
1444}
1445EXPORT_SYMBOL(vringh_iov_pull_iotlb);
1446
1447/**
1448 * vringh_iov_push_iotlb - copy bytes into vring_iov.
1449 * @vrh: the vring.
1450 * @wiov: the wiov as passed to vringh_getdesc_iotlb() (updated as we consume)
1451 * @src: the place to copy from.
1452 * @len: the maximum length to copy.
1453 *
1454 * Returns the bytes copied <= len or a negative errno.
1455 */
1456ssize_t vringh_iov_push_iotlb(struct vringh *vrh,
1457 struct vringh_kiov *wiov,
1458 const void *src, size_t len)
1459{
1460 return vringh_iov_xfer(vrh, wiov, (void *)src, len, xfer_to_iotlb);
1461}
1462EXPORT_SYMBOL(vringh_iov_push_iotlb);
1463
1464/**
1465 * vringh_complete_iotlb - we've finished with descriptor, publish it.
1466 * @vrh: the vring.
1467 * @head: the head as filled in by vringh_getdesc_iotlb.
1468 * @len: the length of data we have written.
1469 *
1470 * You should check vringh_need_notify_iotlb() after one or more calls
1471 * to this function.
1472 */
1473int vringh_complete_iotlb(struct vringh *vrh, u16 head, u32 len)
1474{
1475 struct vring_used_elem used;
1476
1477 used.id = cpu_to_vringh32(vrh, head);
1478 used.len = cpu_to_vringh32(vrh, len);
1479
1480 return __vringh_complete(vrh, &used, 1, putu16_iotlb, putused_iotlb);
1481}
1482EXPORT_SYMBOL(vringh_complete_iotlb);
1483
1484/**
1485 * vringh_need_notify_iotlb - must we tell the other side about used buffers?
1486 * @vrh: the vring we've called vringh_complete_iotlb() on.
1487 *
1488 * Returns -errno or 0 if we don't need to tell the other side, 1 if we do.
1489 */
1490int vringh_need_notify_iotlb(struct vringh *vrh)
1491{
1492 return __vringh_need_notify(vrh, getu16_iotlb);
1493}
1494EXPORT_SYMBOL(vringh_need_notify_iotlb);
1495
1496#endif
1497
1498MODULE_DESCRIPTION("host side of a virtio ring");
1499MODULE_LICENSE("GPL");