projects/linux·blob master · refs

1756 lines · 40.7 KiB

1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Copyright (C) 2018-2020 Intel Corporation.
4 * Copyright (C) 2020 Red Hat, Inc.
5 *
6 * Author: Tiwei Bie <[email protected]>
7 * Jason Wang <[email protected]>
8 *
9 * Thanks Michael S. Tsirkin for the valuable comments and
10 * suggestions. And thanks to Cunming Liang and Zhihong Wang for all
11 * their supports.
12 */
13
14#include <linux/kernel.h>
15#include <linux/module.h>
16#include <linux/cdev.h>
17#include <linux/device.h>
18#include <linux/mm.h>
19#include <linux/slab.h>
20#include <linux/iommu.h>
21#include <linux/uuid.h>
22#include <linux/vdpa.h>
23#include <linux/nospec.h>
24#include <linux/vhost.h>
25
26#include "vhost.h"
27
28enum {
29 VHOST_VDPA_BACKEND_FEATURES =
30 (1ULL << VHOST_BACKEND_F_IOTLB_MSG_V2) |
31 (1ULL << VHOST_BACKEND_F_IOTLB_BATCH) |
32 (1ULL << VHOST_BACKEND_F_IOTLB_ASID),
33};
34
35#define VHOST_VDPA_DEV_MAX (1U << MINORBITS)
36
37static int max_iotlb_entries = 2048;
38module_param(max_iotlb_entries, int, 0444);
39MODULE_PARM_DESC(max_iotlb_entries,
40 "Maximum number of iotlb entries. (default: 2048)");
41
42#define VHOST_VDPA_IOTLB_BUCKETS 16
43
44struct vhost_vdpa_as {
45 struct hlist_node hash_link;
46 struct vhost_iotlb iotlb;
47 u32 id;
48};
49
50struct vhost_vdpa {
51 struct vhost_dev vdev;
52 struct iommu_domain *domain;
53 struct vhost_virtqueue *vqs;
54 struct completion completion;
55 struct vdpa_device *vdpa;
56 struct hlist_head as[VHOST_VDPA_IOTLB_BUCKETS];
57 struct device dev;
58 struct cdev cdev;
59 atomic_t opened;
60 u32 nvqs;
61 u16 vq_num_max;
62 int virtio_id;
63 int minor;
64 struct eventfd_ctx *config_ctx;
65 /* Serialises vhost_vdpa_config_cb() against config_ctx being replaced. */
66 spinlock_t config_lock;
67 int in_batch;
68 struct vdpa_iova_range range;
69 u32 batch_asid;
70 bool suspended;
71};
72
73static DEFINE_IDA(vhost_vdpa_ida);
74
75static dev_t vhost_vdpa_major;
76
77static void vhost_vdpa_iotlb_unmap(struct vhost_vdpa *v,
78 struct vhost_iotlb *iotlb, u64 start,
79 u64 last, u32 asid);
80
81static inline u32 iotlb_to_asid(struct vhost_iotlb *iotlb)
82{
83 struct vhost_vdpa_as *as = container_of(iotlb, struct
84 vhost_vdpa_as, iotlb);
85 return as->id;
86}
87
88static struct vhost_vdpa_as *asid_to_as(struct vhost_vdpa *v, u32 asid)
89{
90 struct hlist_head *head = &v->as[asid % VHOST_VDPA_IOTLB_BUCKETS];
91 struct vhost_vdpa_as *as;
92
93 hlist_for_each_entry(as, head, hash_link)
94 if (as->id == asid)
95 return as;
96
97 return NULL;
98}
99
100static struct vhost_iotlb *asid_to_iotlb(struct vhost_vdpa *v, u32 asid)
101{
102 struct vhost_vdpa_as *as = asid_to_as(v, asid);
103
104 if (!as)
105 return NULL;
106
107 return &as->iotlb;
108}
109
110static struct vhost_vdpa_as *vhost_vdpa_alloc_as(struct vhost_vdpa *v, u32 asid)
111{
112 struct hlist_head *head = &v->as[asid % VHOST_VDPA_IOTLB_BUCKETS];
113 struct vhost_vdpa_as *as;
114
115 if (asid_to_as(v, asid))
116 return NULL;
117
118 if (asid >= v->vdpa->nas)
119 return NULL;
120 if (max_iotlb_entries <= 0)
121 return NULL;
122
123 as = kmalloc_obj(*as);
124 if (!as)
125 return NULL;
126
127 vhost_iotlb_init(&as->iotlb, max_iotlb_entries, 0);
128 as->id = asid;
129 hlist_add_head(&as->hash_link, head);
130
131 return as;
132}
133
134static struct vhost_vdpa_as *vhost_vdpa_find_alloc_as(struct vhost_vdpa *v,
135 u32 asid)
136{
137 struct vhost_vdpa_as *as = asid_to_as(v, asid);
138
139 if (as)
140 return as;
141
142 return vhost_vdpa_alloc_as(v, asid);
143}
144
145static void vhost_vdpa_reset_map(struct vhost_vdpa *v, u32 asid)
146{
147 struct vdpa_device *vdpa = v->vdpa;
148 const struct vdpa_config_ops *ops = vdpa->config;
149
150 if (ops->reset_map)
151 ops->reset_map(vdpa, asid);
152}
153
154static int vhost_vdpa_remove_as(struct vhost_vdpa *v, u32 asid)
155{
156 struct vhost_vdpa_as *as = asid_to_as(v, asid);
157
158 if (!as)
159 return -EINVAL;
160
161 hlist_del(&as->hash_link);
162 vhost_vdpa_iotlb_unmap(v, &as->iotlb, 0ULL, 0ULL - 1, asid);
163 /*
164 * Devices with vendor specific IOMMU may need to restore
165 * iotlb to the initial or default state, which cannot be
166 * cleaned up in the all range unmap call above. Give them
167 * a chance to clean up or reset the map to the desired
168 * state.
169 */
170 vhost_vdpa_reset_map(v, asid);
171 kfree(as);
172
173 return 0;
174}
175
176static void handle_vq_kick(struct vhost_work *work)
177{
178 struct vhost_virtqueue *vq = container_of(work, struct vhost_virtqueue,
179 poll.work);
180 struct vhost_vdpa *v = container_of(vq->dev, struct vhost_vdpa, vdev);
181 const struct vdpa_config_ops *ops = v->vdpa->config;
182
183 ops->kick_vq(v->vdpa, vq - v->vqs);
184}
185
186static irqreturn_t vhost_vdpa_virtqueue_cb(void *private)
187{
188 struct vhost_virtqueue *vq = private;
189 struct eventfd_ctx *call_ctx = vq->call_ctx.ctx;
190
191 if (call_ctx)
192 eventfd_signal(call_ctx);
193
194 return IRQ_HANDLED;
195}
196
197static irqreturn_t vhost_vdpa_config_cb(void *private)
198{
199 struct vhost_vdpa *v = private;
200 unsigned long flags;
201
202 spin_lock_irqsave(&v->config_lock, flags);
203 if (v->config_ctx)
204 eventfd_signal(v->config_ctx);
205 spin_unlock_irqrestore(&v->config_lock, flags);
206
207 return IRQ_HANDLED;
208}
209
210static void vhost_vdpa_setup_vq_irq(struct vhost_vdpa *v, u16 qid)
211{
212 struct vhost_virtqueue *vq = &v->vqs[qid];
213 const struct vdpa_config_ops *ops = v->vdpa->config;
214 struct vdpa_device *vdpa = v->vdpa;
215 int ret, irq;
216
217 if (!ops->get_vq_irq)
218 return;
219
220 irq = ops->get_vq_irq(vdpa, qid);
221 if (irq < 0)
222 return;
223
224 if (!vq->call_ctx.ctx)
225 return;
226
227 ret = irq_bypass_register_producer(&vq->call_ctx.producer,
228 vq->call_ctx.ctx, irq);
229 if (unlikely(ret))
230 dev_info(&v->dev, "vq %u, irq bypass producer (eventfd %p) registration fails, ret = %d\n",
231 qid, vq->call_ctx.ctx, ret);
232}
233
234static void vhost_vdpa_unsetup_vq_irq(struct vhost_vdpa *v, u16 qid)
235{
236 struct vhost_virtqueue *vq = &v->vqs[qid];
237
238 irq_bypass_unregister_producer(&vq->call_ctx.producer);
239}
240
241static int _compat_vdpa_reset(struct vhost_vdpa *v)
242{
243 struct vdpa_device *vdpa = v->vdpa;
244 const struct vdpa_config_ops *ops = vdpa->config;
245 u32 flags = 0;
246 int ret;
247
248 v->suspended = false;
249
250 if (v->vdev.vqs) {
251 flags |= !vhost_backend_has_feature(v->vdev.vqs[0],
252 VHOST_BACKEND_F_IOTLB_PERSIST) ?
253 VDPA_RESET_F_CLEAN_MAP : 0;
254 }
255
256 v->vq_num_max = 0;
257 ret = vdpa_reset(vdpa, flags);
258 if (!ret) {
259 /* Some backends derive the max from mutable queue state. */
260 v->vq_num_max = ops->get_vq_num_max(vdpa);
261 }
262
263 return ret;
264}
265
266static int vhost_vdpa_reset(struct vhost_vdpa *v)
267{
268 v->in_batch = 0;
269 return _compat_vdpa_reset(v);
270}
271
272static long vhost_vdpa_bind_mm(struct vhost_vdpa *v)
273{
274 struct vdpa_device *vdpa = v->vdpa;
275 const struct vdpa_config_ops *ops = vdpa->config;
276
277 if (!vdpa->use_va || !ops->bind_mm)
278 return 0;
279
280 return ops->bind_mm(vdpa, v->vdev.mm);
281}
282
283static void vhost_vdpa_unbind_mm(struct vhost_vdpa *v)
284{
285 struct vdpa_device *vdpa = v->vdpa;
286 const struct vdpa_config_ops *ops = vdpa->config;
287
288 if (!vdpa->use_va || !ops->unbind_mm)
289 return;
290
291 ops->unbind_mm(vdpa);
292}
293
294static long vhost_vdpa_get_device_id(struct vhost_vdpa *v, u8 __user *argp)
295{
296 struct vdpa_device *vdpa = v->vdpa;
297 const struct vdpa_config_ops *ops = vdpa->config;
298 u32 device_id;
299
300 device_id = ops->get_device_id(vdpa);
301
302 if (copy_to_user(argp, &device_id, sizeof(device_id)))
303 return -EFAULT;
304
305 return 0;
306}
307
308static long vhost_vdpa_get_status(struct vhost_vdpa *v, u8 __user *statusp)
309{
310 struct vdpa_device *vdpa = v->vdpa;
311 const struct vdpa_config_ops *ops = vdpa->config;
312 u8 status;
313
314 status = ops->get_status(vdpa);
315
316 if (copy_to_user(statusp, &status, sizeof(status)))
317 return -EFAULT;
318
319 return 0;
320}
321
322static long vhost_vdpa_set_status(struct vhost_vdpa *v, u8 __user *statusp)
323{
324 struct vdpa_device *vdpa = v->vdpa;
325 const struct vdpa_config_ops *ops = vdpa->config;
326 u8 status, status_old;
327 u32 nvqs = v->nvqs;
328 int ret;
329 u16 i;
330
331 if (copy_from_user(&status, statusp, sizeof(status)))
332 return -EFAULT;
333
334 status_old = ops->get_status(vdpa);
335
336 /*
337 * Userspace shouldn't remove status bits unless reset the
338 * status to 0.
339 */
340 if (status != 0 && (status_old & ~status) != 0)
341 return -EINVAL;
342
343 if ((status_old & VIRTIO_CONFIG_S_DRIVER_OK) && !(status & VIRTIO_CONFIG_S_DRIVER_OK))
344 for (i = 0; i < nvqs; i++)
345 vhost_vdpa_unsetup_vq_irq(v, i);
346
347 if (status == 0) {
348 ret = _compat_vdpa_reset(v);
349 if (ret)
350 return ret;
351 } else
352 vdpa_set_status(vdpa, status);
353
354 if ((status & VIRTIO_CONFIG_S_DRIVER_OK) && !(status_old & VIRTIO_CONFIG_S_DRIVER_OK))
355 for (i = 0; i < nvqs; i++)
356 vhost_vdpa_setup_vq_irq(v, i);
357
358 return 0;
359}
360
361static int vhost_vdpa_config_validate(struct vhost_vdpa *v,
362 struct vhost_vdpa_config *c)
363{
364 struct vdpa_device *vdpa = v->vdpa;
365 size_t size = vdpa->config->get_config_size(vdpa);
366
367 if (c->len == 0 || c->off > size)
368 return -EINVAL;
369
370 if (c->len > size - c->off)
371 return -E2BIG;
372
373 return 0;
374}
375
376static long vhost_vdpa_get_config(struct vhost_vdpa *v,
377 struct vhost_vdpa_config __user *c)
378{
379 struct vdpa_device *vdpa = v->vdpa;
380 struct vhost_vdpa_config config;
381 unsigned long size = offsetof(struct vhost_vdpa_config, buf);
382 u8 *buf;
383
384 if (copy_from_user(&config, c, size))
385 return -EFAULT;
386 if (vhost_vdpa_config_validate(v, &config))
387 return -EINVAL;
388 buf = kvzalloc(config.len, GFP_KERNEL);
389 if (!buf)
390 return -ENOMEM;
391
392 vdpa_get_config(vdpa, config.off, buf, config.len);
393
394 if (copy_to_user(c->buf, buf, config.len)) {
395 kvfree(buf);
396 return -EFAULT;
397 }
398
399 kvfree(buf);
400 return 0;
401}
402
403static long vhost_vdpa_set_config(struct vhost_vdpa *v,
404 struct vhost_vdpa_config __user *c)
405{
406 struct vdpa_device *vdpa = v->vdpa;
407 struct vhost_vdpa_config config;
408 unsigned long size = offsetof(struct vhost_vdpa_config, buf);
409 u8 *buf;
410
411 if (copy_from_user(&config, c, size))
412 return -EFAULT;
413 if (vhost_vdpa_config_validate(v, &config))
414 return -EINVAL;
415
416 buf = vmemdup_user(c->buf, config.len);
417 if (IS_ERR(buf))
418 return PTR_ERR(buf);
419
420 vdpa_set_config(vdpa, config.off, buf, config.len);
421
422 kvfree(buf);
423 return 0;
424}
425
426static bool vhost_vdpa_can_suspend(const struct vhost_vdpa *v)
427{
428 struct vdpa_device *vdpa = v->vdpa;
429 const struct vdpa_config_ops *ops = vdpa->config;
430
431 return ops->suspend;
432}
433
434static bool vhost_vdpa_can_resume(const struct vhost_vdpa *v)
435{
436 struct vdpa_device *vdpa = v->vdpa;
437 const struct vdpa_config_ops *ops = vdpa->config;
438
439 return ops->resume;
440}
441
442static bool vhost_vdpa_has_desc_group(const struct vhost_vdpa *v)
443{
444 struct vdpa_device *vdpa = v->vdpa;
445 const struct vdpa_config_ops *ops = vdpa->config;
446
447 return ops->get_vq_desc_group;
448}
449
450static long vhost_vdpa_get_features(struct vhost_vdpa *v, u64 __user *featurep)
451{
452 struct vdpa_device *vdpa = v->vdpa;
453 const struct vdpa_config_ops *ops = vdpa->config;
454 u64 features;
455
456 features = ops->get_device_features(vdpa);
457
458 if (copy_to_user(featurep, &features, sizeof(features)))
459 return -EFAULT;
460
461 return 0;
462}
463
464static u64 vhost_vdpa_get_backend_features(const struct vhost_vdpa *v)
465{
466 struct vdpa_device *vdpa = v->vdpa;
467 const struct vdpa_config_ops *ops = vdpa->config;
468
469 if (!ops->get_backend_features)
470 return 0;
471 else
472 return ops->get_backend_features(vdpa);
473}
474
475static bool vhost_vdpa_has_persistent_map(const struct vhost_vdpa *v)
476{
477 struct vdpa_device *vdpa = v->vdpa;
478 const struct vdpa_config_ops *ops = vdpa->config;
479
480 return (!ops->set_map && !ops->dma_map) || ops->reset_map ||
481 vhost_vdpa_get_backend_features(v) & BIT_ULL(VHOST_BACKEND_F_IOTLB_PERSIST);
482}
483
484static long vhost_vdpa_set_features(struct vhost_vdpa *v, u64 __user *featurep)
485{
486 struct vdpa_device *vdpa = v->vdpa;
487 const struct vdpa_config_ops *ops = vdpa->config;
488 struct vhost_dev *d = &v->vdev;
489 u64 actual_features;
490 u64 features;
491 int i;
492
493 /*
494 * It's not allowed to change the features after they have
495 * been negotiated.
496 */
497 if (ops->get_status(vdpa) & VIRTIO_CONFIG_S_FEATURES_OK)
498 return -EBUSY;
499
500 if (copy_from_user(&features, featurep, sizeof(features)))
501 return -EFAULT;
502
503 if (vdpa_set_features(vdpa, features))
504 return -EINVAL;
505
506 /* let the vqs know what has been configured */
507 actual_features = ops->get_driver_features(vdpa);
508 for (i = 0; i < d->nvqs; ++i) {
509 struct vhost_virtqueue *vq = d->vqs[i];
510
511 mutex_lock(&vq->mutex);
512 vq->acked_features = actual_features;
513 mutex_unlock(&vq->mutex);
514 }
515
516 return 0;
517}
518
519static long vhost_vdpa_get_vring_num(struct vhost_vdpa *v, u16 __user *argp)
520{
521 struct vdpa_device *vdpa = v->vdpa;
522 const struct vdpa_config_ops *ops = vdpa->config;
523 u16 num;
524
525 num = ops->get_vq_num_max(vdpa);
526
527 if (copy_to_user(argp, &num, sizeof(num)))
528 return -EFAULT;
529
530 return 0;
531}
532
533static void vhost_vdpa_config_put(struct vhost_vdpa *v)
534{
535 struct eventfd_ctx *ctx;
536 unsigned long flags;
537
538 spin_lock_irqsave(&v->config_lock, flags);
539 ctx = v->config_ctx;
540 v->config_ctx = NULL;
541 spin_unlock_irqrestore(&v->config_lock, flags);
542
543 if (ctx)
544 eventfd_ctx_put(ctx);
545}
546
547static long vhost_vdpa_set_config_call(struct vhost_vdpa *v, u32 __user *argp)
548{
549 struct vdpa_callback cb;
550 unsigned long flags;
551 int fd;
552 struct eventfd_ctx *ctx;
553
554 cb.callback = vhost_vdpa_config_cb;
555 cb.private = v;
556 if (copy_from_user(&fd, argp, sizeof(fd)))
557 return -EFAULT;
558
559 ctx = fd == VHOST_FILE_UNBIND ? NULL : eventfd_ctx_fdget(fd);
560 if (IS_ERR(ctx))
561 return PTR_ERR(ctx);
562
563 spin_lock_irqsave(&v->config_lock, flags);
564 swap(ctx, v->config_ctx);
565 spin_unlock_irqrestore(&v->config_lock, flags);
566
567 /*
568 * The callback can no longer reach the old context, so this is the
569 * last reference to it.
570 */
571 if (ctx)
572 eventfd_ctx_put(ctx);
573
574 v->vdpa->config->set_config_cb(v->vdpa, &cb);
575
576 return 0;
577}
578
579static long vhost_vdpa_get_iova_range(struct vhost_vdpa *v, u32 __user *argp)
580{
581 struct vhost_vdpa_iova_range range = {
582 .first = v->range.first,
583 .last = v->range.last,
584 };
585
586 if (copy_to_user(argp, &range, sizeof(range)))
587 return -EFAULT;
588 return 0;
589}
590
591static long vhost_vdpa_get_config_size(struct vhost_vdpa *v, u32 __user *argp)
592{
593 struct vdpa_device *vdpa = v->vdpa;
594 const struct vdpa_config_ops *ops = vdpa->config;
595 u32 size;
596
597 size = ops->get_config_size(vdpa);
598
599 if (copy_to_user(argp, &size, sizeof(size)))
600 return -EFAULT;
601
602 return 0;
603}
604
605static long vhost_vdpa_get_vqs_count(struct vhost_vdpa *v, u32 __user *argp)
606{
607 struct vdpa_device *vdpa = v->vdpa;
608
609 if (copy_to_user(argp, &vdpa->nvqs, sizeof(vdpa->nvqs)))
610 return -EFAULT;
611
612 return 0;
613}
614
615/* After a successful return of ioctl the device must not process more
616 * virtqueue descriptors. The device can answer to read or writes of config
617 * fields as if it were not suspended. In particular, writing to "queue_enable"
618 * with a value of 1 will not make the device start processing buffers.
619 */
620static long vhost_vdpa_suspend(struct vhost_vdpa *v)
621{
622 struct vdpa_device *vdpa = v->vdpa;
623 const struct vdpa_config_ops *ops = vdpa->config;
624 int ret;
625
626 if (!(ops->get_status(vdpa) & VIRTIO_CONFIG_S_DRIVER_OK))
627 return 0;
628
629 if (!ops->suspend)
630 return -EOPNOTSUPP;
631
632 ret = ops->suspend(vdpa);
633 if (!ret)
634 v->suspended = true;
635
636 return ret;
637}
638
639/* After a successful return of this ioctl the device resumes processing
640 * virtqueue descriptors. The device becomes fully operational the same way it
641 * was before it was suspended.
642 */
643static long vhost_vdpa_resume(struct vhost_vdpa *v)
644{
645 struct vdpa_device *vdpa = v->vdpa;
646 const struct vdpa_config_ops *ops = vdpa->config;
647 int ret;
648
649 if (!(ops->get_status(vdpa) & VIRTIO_CONFIG_S_DRIVER_OK))
650 return 0;
651
652 if (!ops->resume)
653 return -EOPNOTSUPP;
654
655 ret = ops->resume(vdpa);
656 if (!ret)
657 v->suspended = false;
658
659 return ret;
660}
661
662static long vhost_vdpa_vring_ioctl(struct vhost_vdpa *v, unsigned int cmd,
663 void __user *argp)
664{
665 struct vdpa_device *vdpa = v->vdpa;
666 const struct vdpa_config_ops *ops = vdpa->config;
667 struct vdpa_vq_state vq_state;
668 struct vdpa_callback cb;
669 struct vhost_virtqueue *vq;
670 struct vhost_vring_state s;
671 u32 idx;
672 long r;
673
674 if (cmd == VHOST_SET_VRING_NUM) {
675 if (copy_from_user(&s, argp, sizeof(s)))
676 return -EFAULT;
677 idx = s.index;
678 } else {
679 r = get_user(idx, (u32 __user *)argp);
680 if (r < 0)
681 return r;
682 }
683
684 if (idx >= v->nvqs)
685 return -ENOBUFS;
686
687 idx = array_index_nospec(idx, v->nvqs);
688 vq = &v->vqs[idx];
689
690 switch (cmd) {
691 case VHOST_SET_VRING_NUM:
692 mutex_lock(&vq->mutex);
693 if (vq->private_data) {
694 r = -EBUSY;
695 } else if (!s.num || s.num > 0xffff ||
696 s.num > v->vq_num_max ||
697 (s.num & (s.num - 1))) {
698 r = -EINVAL;
699 } else {
700 vq->num = s.num;
701 r = 0;
702 }
703 mutex_unlock(&vq->mutex);
704 if (r)
705 return r;
706 ops->set_vq_num(vdpa, idx, s.num);
707 return 0;
708 case VHOST_VDPA_SET_VRING_ENABLE:
709 if (copy_from_user(&s, argp, sizeof(s)))
710 return -EFAULT;
711 ops->set_vq_ready(vdpa, idx, s.num);
712 return 0;
713 case VHOST_VDPA_GET_VRING_GROUP:
714 if (!ops->get_vq_group)
715 return -EOPNOTSUPP;
716 s.index = idx;
717 s.num = ops->get_vq_group(vdpa, idx);
718 if (s.num >= vdpa->ngroups)
719 return -EIO;
720 else if (copy_to_user(argp, &s, sizeof(s)))
721 return -EFAULT;
722 return 0;
723 case VHOST_VDPA_GET_VRING_DESC_GROUP:
724 if (!vhost_vdpa_has_desc_group(v))
725 return -EOPNOTSUPP;
726 s.index = idx;
727 s.num = ops->get_vq_desc_group(vdpa, idx);
728 if (s.num >= vdpa->ngroups)
729 return -EIO;
730 else if (copy_to_user(argp, &s, sizeof(s)))
731 return -EFAULT;
732 return 0;
733 case VHOST_VDPA_SET_GROUP_ASID:
734 if (copy_from_user(&s, argp, sizeof(s)))
735 return -EFAULT;
736 if (idx >= vdpa->ngroups || s.num >= vdpa->nas)
737 return -EINVAL;
738 if (ops->get_status(vdpa) & VIRTIO_CONFIG_S_DRIVER_OK)
739 return -EBUSY;
740 if (!ops->set_group_asid)
741 return -EOPNOTSUPP;
742 return ops->set_group_asid(vdpa, idx, s.num);
743 case VHOST_VDPA_GET_VRING_SIZE:
744 if (!ops->get_vq_size)
745 return -EOPNOTSUPP;
746 s.index = idx;
747 s.num = ops->get_vq_size(vdpa, idx);
748 if (copy_to_user(argp, &s, sizeof(s)))
749 return -EFAULT;
750 return 0;
751 case VHOST_GET_VRING_BASE:
752 r = ops->get_vq_state(v->vdpa, idx, &vq_state);
753 if (r)
754 return r;
755
756 if (vhost_has_feature(vq, VIRTIO_F_RING_PACKED)) {
757 vq->last_avail_idx = vq_state.packed.last_avail_idx |
758 (vq_state.packed.last_avail_counter << 15);
759 vq->last_used_idx = vq_state.packed.last_used_idx |
760 (vq_state.packed.last_used_counter << 15);
761 } else {
762 vq->last_avail_idx = vq_state.split.avail_index;
763 }
764 break;
765 case VHOST_SET_VRING_CALL:
766 if (vq->call_ctx.ctx) {
767 if (ops->get_status(vdpa) &
768 VIRTIO_CONFIG_S_DRIVER_OK)
769 vhost_vdpa_unsetup_vq_irq(v, idx);
770 }
771 break;
772 }
773
774 r = vhost_vring_ioctl(&v->vdev, cmd, argp);
775 if (r)
776 return r;
777
778 switch (cmd) {
779 case VHOST_SET_VRING_ADDR:
780 if ((ops->get_status(vdpa) & VIRTIO_CONFIG_S_DRIVER_OK) && !v->suspended)
781 return -EINVAL;
782
783 if (ops->set_vq_address(vdpa, idx,
784 (u64)(uintptr_t)vq->desc,
785 (u64)(uintptr_t)vq->avail,
786 (u64)(uintptr_t)vq->used))
787 r = -EINVAL;
788 break;
789
790 case VHOST_SET_VRING_BASE:
791 if ((ops->get_status(vdpa) & VIRTIO_CONFIG_S_DRIVER_OK) && !v->suspended)
792 return -EINVAL;
793
794 if (vhost_has_feature(vq, VIRTIO_F_RING_PACKED)) {
795 vq_state.packed.last_avail_idx = vq->last_avail_idx & 0x7fff;
796 vq_state.packed.last_avail_counter = !!(vq->last_avail_idx & 0x8000);
797 vq_state.packed.last_used_idx = vq->last_used_idx & 0x7fff;
798 vq_state.packed.last_used_counter = !!(vq->last_used_idx & 0x8000);
799 } else {
800 vq_state.split.avail_index = vq->last_avail_idx;
801 }
802 r = ops->set_vq_state(vdpa, idx, &vq_state);
803 break;
804
805 case VHOST_SET_VRING_CALL:
806 if (vq->call_ctx.ctx) {
807 cb.callback = vhost_vdpa_virtqueue_cb;
808 cb.private = vq;
809 cb.trigger = vq->call_ctx.ctx;
810 if (ops->get_status(vdpa) &
811 VIRTIO_CONFIG_S_DRIVER_OK)
812 vhost_vdpa_setup_vq_irq(v, idx);
813 } else {
814 cb.callback = NULL;
815 cb.private = NULL;
816 cb.trigger = NULL;
817 }
818 ops->set_vq_cb(vdpa, idx, &cb);
819 break;
820
821 }
822
823 return r;
824}
825
826static long vhost_vdpa_unlocked_ioctl(struct file *filep,
827 unsigned int cmd, unsigned long arg)
828{
829 struct vhost_vdpa *v = filep->private_data;
830 struct vhost_dev *d = &v->vdev;
831 void __user *argp = (void __user *)arg;
832 u64 __user *featurep = argp;
833 u64 features;
834 long r = 0;
835
836 if (cmd == VHOST_SET_BACKEND_FEATURES) {
837 if (copy_from_user(&features, featurep, sizeof(features)))
838 return -EFAULT;
839 if (features & ~(VHOST_VDPA_BACKEND_FEATURES |
840 BIT_ULL(VHOST_BACKEND_F_DESC_ASID) |
841 BIT_ULL(VHOST_BACKEND_F_IOTLB_PERSIST) |
842 BIT_ULL(VHOST_BACKEND_F_SUSPEND) |
843 BIT_ULL(VHOST_BACKEND_F_RESUME) |
844 BIT_ULL(VHOST_BACKEND_F_ENABLE_AFTER_DRIVER_OK)))
845 return -EOPNOTSUPP;
846 if ((features & BIT_ULL(VHOST_BACKEND_F_SUSPEND)) &&
847 !vhost_vdpa_can_suspend(v))
848 return -EOPNOTSUPP;
849 if ((features & BIT_ULL(VHOST_BACKEND_F_RESUME)) &&
850 !vhost_vdpa_can_resume(v))
851 return -EOPNOTSUPP;
852 if ((features & BIT_ULL(VHOST_BACKEND_F_DESC_ASID)) &&
853 !(features & BIT_ULL(VHOST_BACKEND_F_IOTLB_ASID)))
854 return -EINVAL;
855 if ((features & BIT_ULL(VHOST_BACKEND_F_DESC_ASID)) &&
856 !vhost_vdpa_has_desc_group(v))
857 return -EOPNOTSUPP;
858 if ((features & BIT_ULL(VHOST_BACKEND_F_IOTLB_PERSIST)) &&
859 !vhost_vdpa_has_persistent_map(v))
860 return -EOPNOTSUPP;
861 vhost_set_backend_features(&v->vdev, features);
862 return 0;
863 }
864
865 mutex_lock(&d->mutex);
866
867 switch (cmd) {
868 case VHOST_VDPA_GET_DEVICE_ID:
869 r = vhost_vdpa_get_device_id(v, argp);
870 break;
871 case VHOST_VDPA_GET_STATUS:
872 r = vhost_vdpa_get_status(v, argp);
873 break;
874 case VHOST_VDPA_SET_STATUS:
875 r = vhost_vdpa_set_status(v, argp);
876 break;
877 case VHOST_VDPA_GET_CONFIG:
878 r = vhost_vdpa_get_config(v, argp);
879 break;
880 case VHOST_VDPA_SET_CONFIG:
881 r = vhost_vdpa_set_config(v, argp);
882 break;
883 case VHOST_GET_FEATURES:
884 r = vhost_vdpa_get_features(v, argp);
885 break;
886 case VHOST_SET_FEATURES:
887 r = vhost_vdpa_set_features(v, argp);
888 break;
889 case VHOST_VDPA_GET_VRING_NUM:
890 r = vhost_vdpa_get_vring_num(v, argp);
891 break;
892 case VHOST_VDPA_GET_GROUP_NUM:
893 if (copy_to_user(argp, &v->vdpa->ngroups,
894 sizeof(v->vdpa->ngroups)))
895 r = -EFAULT;
896 break;
897 case VHOST_VDPA_GET_AS_NUM:
898 if (copy_to_user(argp, &v->vdpa->nas, sizeof(v->vdpa->nas)))
899 r = -EFAULT;
900 break;
901 case VHOST_SET_LOG_BASE:
902 case VHOST_SET_LOG_FD:
903 r = -ENOIOCTLCMD;
904 break;
905 case VHOST_VDPA_SET_CONFIG_CALL:
906 r = vhost_vdpa_set_config_call(v, argp);
907 break;
908 case VHOST_GET_BACKEND_FEATURES:
909 features = VHOST_VDPA_BACKEND_FEATURES;
910 if (vhost_vdpa_can_suspend(v))
911 features |= BIT_ULL(VHOST_BACKEND_F_SUSPEND);
912 if (vhost_vdpa_can_resume(v))
913 features |= BIT_ULL(VHOST_BACKEND_F_RESUME);
914 if (vhost_vdpa_has_desc_group(v))
915 features |= BIT_ULL(VHOST_BACKEND_F_DESC_ASID);
916 if (vhost_vdpa_has_persistent_map(v))
917 features |= BIT_ULL(VHOST_BACKEND_F_IOTLB_PERSIST);
918 features |= vhost_vdpa_get_backend_features(v);
919 if (copy_to_user(featurep, &features, sizeof(features)))
920 r = -EFAULT;
921 break;
922 case VHOST_VDPA_GET_IOVA_RANGE:
923 r = vhost_vdpa_get_iova_range(v, argp);
924 break;
925 case VHOST_VDPA_GET_CONFIG_SIZE:
926 r = vhost_vdpa_get_config_size(v, argp);
927 break;
928 case VHOST_VDPA_GET_VQS_COUNT:
929 r = vhost_vdpa_get_vqs_count(v, argp);
930 break;
931 case VHOST_VDPA_SUSPEND:
932 r = vhost_vdpa_suspend(v);
933 break;
934 case VHOST_VDPA_RESUME:
935 r = vhost_vdpa_resume(v);
936 break;
937 default:
938 r = vhost_dev_ioctl(&v->vdev, cmd, argp);
939 if (r == -ENOIOCTLCMD)
940 r = vhost_vdpa_vring_ioctl(v, cmd, argp);
941 break;
942 }
943
944 if (r)
945 goto out;
946
947 switch (cmd) {
948 case VHOST_SET_OWNER:
949 r = vhost_vdpa_bind_mm(v);
950 if (r)
951 vhost_dev_reset_owner(d, NULL);
952 break;
953 }
954out:
955 mutex_unlock(&d->mutex);
956 return r;
957}
958static void vhost_vdpa_general_unmap(struct vhost_vdpa *v,
959 struct vhost_iotlb_map *map, u32 asid)
960{
961 struct vdpa_device *vdpa = v->vdpa;
962 const struct vdpa_config_ops *ops = vdpa->config;
963 if (ops->dma_map) {
964 ops->dma_unmap(vdpa, asid, map->start, map->size);
965 } else if (ops->set_map == NULL) {
966 iommu_unmap(v->domain, map->start, map->size);
967 }
968}
969
970static void vhost_vdpa_pa_unmap(struct vhost_vdpa *v, struct vhost_iotlb *iotlb,
971 u64 start, u64 last, u32 asid)
972{
973 struct vhost_dev *dev = &v->vdev;
974 struct vhost_iotlb_map *map;
975 struct page *page;
976 unsigned long pfn, pinned;
977
978 while ((map = vhost_iotlb_itree_first(iotlb, start, last)) != NULL) {
979 pinned = PFN_DOWN(map->size);
980 for (pfn = PFN_DOWN(map->addr);
981 pinned > 0; pfn++, pinned--) {
982 page = pfn_to_page(pfn);
983 if (map->perm & VHOST_ACCESS_WO)
984 set_page_dirty_lock(page);
985 unpin_user_page(page);
986 }
987 atomic64_sub(PFN_DOWN(map->size), &dev->mm->pinned_vm);
988 vhost_vdpa_general_unmap(v, map, asid);
989 vhost_iotlb_map_free(iotlb, map);
990 }
991}
992
993static void vhost_vdpa_va_unmap(struct vhost_vdpa *v, struct vhost_iotlb *iotlb,
994 u64 start, u64 last, u32 asid)
995{
996 struct vhost_iotlb_map *map;
997 struct vdpa_map_file *map_file;
998
999 while ((map = vhost_iotlb_itree_first(iotlb, start, last)) != NULL) {
1000 map_file = (struct vdpa_map_file *)map->opaque;
1001 fput(map_file->file);
1002 kfree(map_file);
1003 vhost_vdpa_general_unmap(v, map, asid);
1004 vhost_iotlb_map_free(iotlb, map);
1005 }
1006}
1007
1008static void vhost_vdpa_iotlb_unmap(struct vhost_vdpa *v,
1009 struct vhost_iotlb *iotlb, u64 start,
1010 u64 last, u32 asid)
1011{
1012 struct vdpa_device *vdpa = v->vdpa;
1013
1014 if (vdpa->use_va)
1015 return vhost_vdpa_va_unmap(v, iotlb, start, last, asid);
1016
1017 return vhost_vdpa_pa_unmap(v, iotlb, start, last, asid);
1018}
1019
1020static int perm_to_iommu_flags(u32 perm)
1021{
1022 int flags = 0;
1023
1024 switch (perm) {
1025 case VHOST_ACCESS_WO:
1026 flags |= IOMMU_WRITE;
1027 break;
1028 case VHOST_ACCESS_RO:
1029 flags |= IOMMU_READ;
1030 break;
1031 case VHOST_ACCESS_RW:
1032 flags |= (IOMMU_WRITE | IOMMU_READ);
1033 break;
1034 default:
1035 WARN(1, "invalidate vhost IOTLB permission\n");
1036 break;
1037 }
1038
1039 return flags | IOMMU_CACHE;
1040}
1041
1042static int vhost_vdpa_map(struct vhost_vdpa *v, struct vhost_iotlb *iotlb,
1043 u64 iova, u64 size, u64 pa, u32 perm, void *opaque)
1044{
1045 struct vhost_dev *dev = &v->vdev;
1046 struct vdpa_device *vdpa = v->vdpa;
1047 const struct vdpa_config_ops *ops = vdpa->config;
1048 u32 asid = iotlb_to_asid(iotlb);
1049 int r = 0;
1050
1051 r = vhost_iotlb_add_range_ctx(iotlb, iova, iova + size - 1,
1052 pa, perm, opaque);
1053 if (r)
1054 return r;
1055
1056 if (ops->dma_map) {
1057 r = ops->dma_map(vdpa, asid, iova, size, pa, perm, opaque);
1058 } else if (ops->set_map) {
1059 if (!v->in_batch)
1060 r = ops->set_map(vdpa, asid, iotlb);
1061 } else {
1062 r = iommu_map(v->domain, iova, pa, size,
1063 perm_to_iommu_flags(perm),
1064 GFP_KERNEL_ACCOUNT);
1065 }
1066 if (r) {
1067 vhost_iotlb_del_range(iotlb, iova, iova + size - 1);
1068 return r;
1069 }
1070
1071 if (!vdpa->use_va)
1072 atomic64_add(PFN_DOWN(size), &dev->mm->pinned_vm);
1073
1074 return 0;
1075}
1076
1077static void vhost_vdpa_unmap(struct vhost_vdpa *v,
1078 struct vhost_iotlb *iotlb,
1079 u64 iova, u64 size)
1080{
1081 struct vdpa_device *vdpa = v->vdpa;
1082 const struct vdpa_config_ops *ops = vdpa->config;
1083 u32 asid = iotlb_to_asid(iotlb);
1084
1085 vhost_vdpa_iotlb_unmap(v, iotlb, iova, iova + size - 1, asid);
1086
1087 if (ops->set_map) {
1088 if (!v->in_batch)
1089 ops->set_map(vdpa, asid, iotlb);
1090 }
1091
1092}
1093
1094static int vhost_vdpa_va_map(struct vhost_vdpa *v,
1095 struct vhost_iotlb *iotlb,
1096 u64 iova, u64 size, u64 uaddr, u32 perm)
1097{
1098 struct vhost_dev *dev = &v->vdev;
1099 u64 offset, map_size, map_iova = iova;
1100 struct vdpa_map_file *map_file;
1101 struct vm_area_struct *vma;
1102 int ret = 0;
1103
1104 mmap_read_lock(dev->mm);
1105
1106 while (size) {
1107 vma = find_vma(dev->mm, uaddr);
1108 if (!vma) {
1109 ret = -EINVAL;
1110 break;
1111 }
1112 map_size = min(size, vma->vm_end - uaddr);
1113 if (!(vma->vm_file && (vma->vm_flags & VM_SHARED) &&
1114 !(vma->vm_flags & (VM_IO | VM_PFNMAP))))
1115 goto next;
1116
1117 map_file = kzalloc_obj(*map_file);
1118 if (!map_file) {
1119 ret = -ENOMEM;
1120 break;
1121 }
1122 offset = (vma->vm_pgoff << PAGE_SHIFT) + uaddr - vma->vm_start;
1123 map_file->offset = offset;
1124 map_file->file = get_file(vma->vm_file);
1125 ret = vhost_vdpa_map(v, iotlb, map_iova, map_size, uaddr,
1126 perm, map_file);
1127 if (ret) {
1128 fput(map_file->file);
1129 kfree(map_file);
1130 break;
1131 }
1132next:
1133 size -= map_size;
1134 uaddr += map_size;
1135 map_iova += map_size;
1136 }
1137 if (ret)
1138 vhost_vdpa_unmap(v, iotlb, iova, map_iova - iova);
1139
1140 mmap_read_unlock(dev->mm);
1141
1142 return ret;
1143}
1144
1145static int vhost_vdpa_pa_map(struct vhost_vdpa *v,
1146 struct vhost_iotlb *iotlb,
1147 u64 iova, u64 size, u64 uaddr, u32 perm)
1148{
1149 struct vhost_dev *dev = &v->vdev;
1150 struct page **page_list;
1151 unsigned long list_size = PAGE_SIZE / sizeof(struct page *);
1152 unsigned int gup_flags = FOLL_LONGTERM;
1153 unsigned long npages, cur_base, map_pfn, last_pfn = 0;
1154 unsigned long lock_limit, sz2pin, nchunks, i;
1155 unsigned long page_offset;
1156 u64 start = iova;
1157 long pinned;
1158 int ret = 0;
1159
1160 /* Limit the use of memory for bookkeeping */
1161 page_list = (struct page **) __get_free_page(GFP_KERNEL);
1162 if (!page_list)
1163 return -ENOMEM;
1164
1165 if (perm & VHOST_ACCESS_WO)
1166 gup_flags |= FOLL_WRITE;
1167
1168 page_offset = iova & ~PAGE_MASK;
1169 if (size > ULONG_MAX - page_offset) {
1170 ret = -EINVAL;
1171 goto free;
1172 }
1173
1174 npages = PFN_UP(size + page_offset);
1175 if (!npages) {
1176 ret = -EINVAL;
1177 goto free;
1178 }
1179
1180 mmap_read_lock(dev->mm);
1181
1182 lock_limit = PFN_DOWN(rlimit(RLIMIT_MEMLOCK));
1183 if (npages + atomic64_read(&dev->mm->pinned_vm) > lock_limit) {
1184 ret = -ENOMEM;
1185 goto unlock;
1186 }
1187
1188 cur_base = uaddr & PAGE_MASK;
1189 iova &= PAGE_MASK;
1190 nchunks = 0;
1191
1192 while (npages) {
1193 sz2pin = min_t(unsigned long, npages, list_size);
1194 pinned = pin_user_pages(cur_base, sz2pin,
1195 gup_flags, page_list);
1196 if (sz2pin != pinned) {
1197 if (pinned < 0) {
1198 ret = pinned;
1199 } else {
1200 unpin_user_pages(page_list, pinned);
1201 ret = -ENOMEM;
1202 }
1203 goto out;
1204 }
1205 nchunks++;
1206
1207 if (!last_pfn)
1208 map_pfn = page_to_pfn(page_list[0]);
1209
1210 for (i = 0; i < pinned; i++) {
1211 unsigned long this_pfn = page_to_pfn(page_list[i]);
1212 u64 csize;
1213
1214 if (last_pfn && (this_pfn != last_pfn + 1)) {
1215 /* Pin a contiguous chunk of memory */
1216 csize = PFN_PHYS(last_pfn - map_pfn + 1);
1217 ret = vhost_vdpa_map(v, iotlb, iova, csize,
1218 PFN_PHYS(map_pfn),
1219 perm, NULL);
1220 if (ret) {
1221 /*
1222 * Unpin the pages that are left unmapped
1223 * from this point on in the current
1224 * page_list. The remaining outstanding
1225 * ones which may stride across several
1226 * chunks will be covered in the common
1227 * error path subsequently.
1228 */
1229 unpin_user_pages(&page_list[i],
1230 pinned - i);
1231 goto out;
1232 }
1233
1234 map_pfn = this_pfn;
1235 iova += csize;
1236 nchunks = 0;
1237 }
1238
1239 last_pfn = this_pfn;
1240 }
1241
1242 cur_base += PFN_PHYS(pinned);
1243 npages -= pinned;
1244 }
1245
1246 /* Pin the rest chunk */
1247 ret = vhost_vdpa_map(v, iotlb, iova, PFN_PHYS(last_pfn - map_pfn + 1),
1248 PFN_PHYS(map_pfn), perm, NULL);
1249out:
1250 if (ret) {
1251 if (nchunks) {
1252 unsigned long pfn;
1253
1254 /*
1255 * Unpin the outstanding pages which are yet to be
1256 * mapped but haven't due to vdpa_map() or
1257 * pin_user_pages() failure.
1258 *
1259 * Mapped pages are accounted in vdpa_map(), hence
1260 * the corresponding unpinning will be handled by
1261 * vdpa_unmap().
1262 */
1263 WARN_ON(!last_pfn);
1264 for (pfn = map_pfn; pfn <= last_pfn; pfn++)
1265 unpin_user_page(pfn_to_page(pfn));
1266 }
1267 vhost_vdpa_unmap(v, iotlb, start, size);
1268 }
1269unlock:
1270 mmap_read_unlock(dev->mm);
1271free:
1272 free_page((unsigned long)page_list);
1273 return ret;
1274
1275}
1276
1277static int vhost_vdpa_process_iotlb_update(struct vhost_vdpa *v,
1278 struct vhost_iotlb *iotlb,
1279 struct vhost_iotlb_msg *msg)
1280{
1281 struct vdpa_device *vdpa = v->vdpa;
1282
1283 if (msg->iova < v->range.first || !msg->size ||
1284 msg->iova > U64_MAX - msg->size + 1 ||
1285 msg->iova + msg->size - 1 > v->range.last)
1286 return -EINVAL;
1287
1288 if (vhost_iotlb_itree_first(iotlb, msg->iova,
1289 msg->iova + msg->size - 1))
1290 return -EEXIST;
1291
1292 if (vdpa->use_va)
1293 return vhost_vdpa_va_map(v, iotlb, msg->iova, msg->size,
1294 msg->uaddr, msg->perm);
1295
1296 return vhost_vdpa_pa_map(v, iotlb, msg->iova, msg->size, msg->uaddr,
1297 msg->perm);
1298}
1299
1300static int vhost_vdpa_process_iotlb_msg(struct vhost_dev *dev, u32 asid,
1301 struct vhost_iotlb_msg *msg)
1302{
1303 struct vhost_vdpa *v = container_of(dev, struct vhost_vdpa, vdev);
1304 struct vdpa_device *vdpa = v->vdpa;
1305 const struct vdpa_config_ops *ops = vdpa->config;
1306 struct vhost_iotlb *iotlb = NULL;
1307 struct vhost_vdpa_as *as = NULL;
1308 int r = 0;
1309
1310 mutex_lock(&dev->mutex);
1311
1312 r = vhost_dev_check_owner(dev);
1313 if (r)
1314 goto unlock;
1315
1316 if (msg->type == VHOST_IOTLB_UPDATE ||
1317 msg->type == VHOST_IOTLB_BATCH_BEGIN) {
1318 as = vhost_vdpa_find_alloc_as(v, asid);
1319 if (!as) {
1320 dev_err(&v->dev, "can't find and alloc asid %d\n",
1321 asid);
1322 r = -EINVAL;
1323 goto unlock;
1324 }
1325 iotlb = &as->iotlb;
1326 } else
1327 iotlb = asid_to_iotlb(v, asid);
1328
1329 if ((v->in_batch && v->batch_asid != asid) || !iotlb) {
1330 if (v->in_batch && v->batch_asid != asid) {
1331 dev_info(&v->dev, "batch id %d asid %d\n",
1332 v->batch_asid, asid);
1333 }
1334 if (!iotlb)
1335 dev_err(&v->dev, "no iotlb for asid %d\n", asid);
1336 r = -EINVAL;
1337 goto unlock;
1338 }
1339
1340 switch (msg->type) {
1341 case VHOST_IOTLB_UPDATE:
1342 r = vhost_vdpa_process_iotlb_update(v, iotlb, msg);
1343 break;
1344 case VHOST_IOTLB_INVALIDATE:
1345 vhost_vdpa_unmap(v, iotlb, msg->iova, msg->size);
1346 break;
1347 case VHOST_IOTLB_BATCH_BEGIN:
1348 v->batch_asid = asid;
1349 v->in_batch = true;
1350 break;
1351 case VHOST_IOTLB_BATCH_END:
1352 if (v->in_batch && ops->set_map)
1353 ops->set_map(vdpa, asid, iotlb);
1354 v->in_batch = false;
1355 break;
1356 default:
1357 r = -EINVAL;
1358 break;
1359 }
1360unlock:
1361 mutex_unlock(&dev->mutex);
1362
1363 return r;
1364}
1365
1366static ssize_t vhost_vdpa_chr_write_iter(struct kiocb *iocb,
1367 struct iov_iter *from)
1368{
1369 struct file *file = iocb->ki_filp;
1370 struct vhost_vdpa *v = file->private_data;
1371 struct vhost_dev *dev = &v->vdev;
1372
1373 return vhost_chr_write_iter(dev, from);
1374}
1375
1376static int vhost_vdpa_alloc_domain(struct vhost_vdpa *v)
1377{
1378 struct vdpa_device *vdpa = v->vdpa;
1379 const struct vdpa_config_ops *ops = vdpa->config;
1380 union virtio_map map = vdpa_get_map(vdpa);
1381 struct device *dma_dev = map.dma_dev;
1382 int ret;
1383
1384 /* Device want to do DMA by itself */
1385 if (ops->set_map || ops->dma_map)
1386 return 0;
1387
1388 if (!device_iommu_capable(dma_dev, IOMMU_CAP_CACHE_COHERENCY)) {
1389 dev_warn_once(&v->dev,
1390 "Failed to allocate domain, device is not IOMMU cache coherent capable\n");
1391 return -ENOTSUPP;
1392 }
1393
1394 v->domain = iommu_paging_domain_alloc(dma_dev);
1395 if (IS_ERR(v->domain)) {
1396 ret = PTR_ERR(v->domain);
1397 v->domain = NULL;
1398 return ret;
1399 }
1400
1401 ret = iommu_attach_device(v->domain, dma_dev);
1402 if (ret)
1403 goto err_attach;
1404
1405 return 0;
1406
1407err_attach:
1408 iommu_domain_free(v->domain);
1409 v->domain = NULL;
1410 return ret;
1411}
1412
1413static void vhost_vdpa_free_domain(struct vhost_vdpa *v)
1414{
1415 struct vdpa_device *vdpa = v->vdpa;
1416 union virtio_map map = vdpa_get_map(vdpa);
1417 struct device *dma_dev = map.dma_dev;
1418
1419 if (v->domain) {
1420 iommu_detach_device(v->domain, dma_dev);
1421 iommu_domain_free(v->domain);
1422 }
1423
1424 v->domain = NULL;
1425}
1426
1427static void vhost_vdpa_set_iova_range(struct vhost_vdpa *v)
1428{
1429 struct vdpa_iova_range *range = &v->range;
1430 struct vdpa_device *vdpa = v->vdpa;
1431 const struct vdpa_config_ops *ops = vdpa->config;
1432
1433 if (ops->get_iova_range) {
1434 *range = ops->get_iova_range(vdpa);
1435 } else if (v->domain && v->domain->geometry.force_aperture) {
1436 range->first = v->domain->geometry.aperture_start;
1437 range->last = v->domain->geometry.aperture_end;
1438 } else {
1439 range->first = 0;
1440 range->last = ULLONG_MAX;
1441 }
1442}
1443
1444static void vhost_vdpa_cleanup(struct vhost_vdpa *v)
1445{
1446 struct vhost_vdpa_as *as;
1447 u32 asid;
1448
1449 for (asid = 0; asid < v->vdpa->nas; asid++) {
1450 as = asid_to_as(v, asid);
1451 if (as)
1452 vhost_vdpa_remove_as(v, asid);
1453 }
1454
1455 vhost_vdpa_free_domain(v);
1456 vhost_dev_cleanup(&v->vdev);
1457 kfree(v->vdev.vqs);
1458 v->vdev.vqs = NULL;
1459}
1460
1461static int vhost_vdpa_open(struct inode *inode, struct file *filep)
1462{
1463 struct vhost_vdpa *v;
1464 struct vhost_dev *dev;
1465 struct vhost_virtqueue **vqs;
1466 int r, opened;
1467 u32 i, nvqs;
1468
1469 v = container_of(inode->i_cdev, struct vhost_vdpa, cdev);
1470
1471 opened = atomic_cmpxchg(&v->opened, 0, 1);
1472 if (opened)
1473 return -EBUSY;
1474
1475 nvqs = v->nvqs;
1476 r = vhost_vdpa_reset(v);
1477 if (r)
1478 goto err;
1479
1480 vqs = kmalloc_objs(*vqs, nvqs);
1481 if (!vqs) {
1482 r = -ENOMEM;
1483 goto err;
1484 }
1485
1486 dev = &v->vdev;
1487 for (i = 0; i < nvqs; i++) {
1488 vqs[i] = &v->vqs[i];
1489 vqs[i]->handle_kick = handle_vq_kick;
1490 vqs[i]->call_ctx.ctx = NULL;
1491 }
1492 vhost_dev_init(dev, vqs, nvqs, 0, 0, 0, false,
1493 vhost_vdpa_process_iotlb_msg);
1494
1495 r = vhost_vdpa_alloc_domain(v);
1496 if (r)
1497 goto err_alloc_domain;
1498
1499 vhost_vdpa_set_iova_range(v);
1500
1501 filep->private_data = v;
1502
1503 return 0;
1504
1505err_alloc_domain:
1506 vhost_vdpa_cleanup(v);
1507err:
1508 atomic_dec(&v->opened);
1509 return r;
1510}
1511
1512static void vhost_vdpa_clean_irq(struct vhost_vdpa *v)
1513{
1514 u32 i;
1515
1516 for (i = 0; i < v->nvqs; i++)
1517 vhost_vdpa_unsetup_vq_irq(v, i);
1518}
1519
1520static int vhost_vdpa_release(struct inode *inode, struct file *filep)
1521{
1522 struct vhost_vdpa *v = filep->private_data;
1523 struct vhost_dev *d = &v->vdev;
1524
1525 mutex_lock(&d->mutex);
1526 filep->private_data = NULL;
1527 vhost_vdpa_clean_irq(v);
1528 vhost_vdpa_reset(v);
1529 vhost_dev_stop(&v->vdev);
1530 vhost_vdpa_unbind_mm(v);
1531 vhost_vdpa_config_put(v);
1532 vhost_vdpa_cleanup(v);
1533 mutex_unlock(&d->mutex);
1534
1535 atomic_dec(&v->opened);
1536 complete(&v->completion);
1537
1538 return 0;
1539}
1540
1541#ifdef CONFIG_MMU
1542static int
1543vhost_vdpa_get_vq_notification(struct vhost_vdpa *v, unsigned long index,
1544 struct vdpa_notification_area *notify)
1545{
1546 struct vdpa_device *vdpa = v->vdpa;
1547 const struct vdpa_config_ops *ops = vdpa->config;
1548
1549 if (index > 65535 || index >= v->nvqs)
1550 return -EINVAL;
1551
1552 index = array_index_nospec(index, v->nvqs);
1553
1554 *notify = ops->get_vq_notification(vdpa, index);
1555
1556 return 0;
1557}
1558
1559static vm_fault_t vhost_vdpa_fault(struct vm_fault *vmf)
1560{
1561 struct vhost_vdpa *v = vmf->vma->vm_file->private_data;
1562 struct vdpa_notification_area notify;
1563 struct vm_area_struct *vma = vmf->vma;
1564 unsigned long index = vma->vm_pgoff;
1565
1566 if (vhost_vdpa_get_vq_notification(v, index, &notify))
1567 return VM_FAULT_SIGBUS;
1568
1569 return vmf_insert_pfn(vma, vmf->address & PAGE_MASK, PFN_DOWN(notify.addr));
1570}
1571
1572static const struct vm_operations_struct vhost_vdpa_vm_ops = {
1573 .fault = vhost_vdpa_fault,
1574};
1575
1576static int vhost_vdpa_mmap(struct file *file, struct vm_area_struct *vma)
1577{
1578 struct vhost_vdpa *v = vma->vm_file->private_data;
1579 struct vdpa_device *vdpa = v->vdpa;
1580 const struct vdpa_config_ops *ops = vdpa->config;
1581 struct vdpa_notification_area notify;
1582 unsigned long index = vma->vm_pgoff;
1583
1584 if (vma->vm_end - vma->vm_start != PAGE_SIZE)
1585 return -EINVAL;
1586 if ((vma->vm_flags & VM_SHARED) == 0)
1587 return -EINVAL;
1588 if (vma->vm_flags & VM_READ)
1589 return -EINVAL;
1590 if (!ops->get_vq_notification)
1591 return -ENOTSUPP;
1592
1593 /* To be safe and easily modelled by userspace, We only
1594 * support the doorbell which sits on the page boundary and
1595 * does not share the page with other registers.
1596 */
1597 if (vhost_vdpa_get_vq_notification(v, index, &notify))
1598 return -EINVAL;
1599 if (notify.addr & (PAGE_SIZE - 1))
1600 return -EINVAL;
1601 if (vma->vm_end - vma->vm_start != notify.size)
1602 return -ENOTSUPP;
1603
1604 vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
1605 vm_flags_set(vma, VM_IO | VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP);
1606 vma->vm_ops = &vhost_vdpa_vm_ops;
1607 return 0;
1608}
1609#endif /* CONFIG_MMU */
1610
1611static const struct file_operations vhost_vdpa_fops = {
1612 .owner = THIS_MODULE,
1613 .open = vhost_vdpa_open,
1614 .release = vhost_vdpa_release,
1615 .write_iter = vhost_vdpa_chr_write_iter,
1616 .unlocked_ioctl = vhost_vdpa_unlocked_ioctl,
1617#ifdef CONFIG_MMU
1618 .mmap = vhost_vdpa_mmap,
1619#endif /* CONFIG_MMU */
1620 .compat_ioctl = compat_ptr_ioctl,
1621};
1622
1623static void vhost_vdpa_release_dev(struct device *device)
1624{
1625 struct vhost_vdpa *v =
1626 container_of(device, struct vhost_vdpa, dev);
1627
1628 ida_free(&vhost_vdpa_ida, v->minor);
1629 kfree(v->vqs);
1630 kfree(v);
1631}
1632
1633static int vhost_vdpa_probe(struct vdpa_device *vdpa)
1634{
1635 const struct vdpa_config_ops *ops = vdpa->config;
1636 struct vhost_vdpa *v;
1637 int minor;
1638 int i, r;
1639
1640 /* We can't support platform IOMMU device with more than 1
1641 * group or as
1642 */
1643 if (!ops->set_map && !ops->dma_map &&
1644 (vdpa->ngroups > 1 || vdpa->nas > 1))
1645 return -EOPNOTSUPP;
1646
1647 v = kzalloc_obj(*v, GFP_KERNEL | __GFP_RETRY_MAYFAIL);
1648 if (!v)
1649 return -ENOMEM;
1650
1651 minor = ida_alloc_max(&vhost_vdpa_ida, VHOST_VDPA_DEV_MAX - 1,
1652 GFP_KERNEL);
1653 if (minor < 0) {
1654 kfree(v);
1655 return minor;
1656 }
1657
1658 atomic_set(&v->opened, 0);
1659 spin_lock_init(&v->config_lock);
1660 v->minor = minor;
1661 v->vdpa = vdpa;
1662 v->nvqs = vdpa->nvqs;
1663 v->virtio_id = ops->get_device_id(vdpa);
1664
1665 device_initialize(&v->dev);
1666 v->dev.release = vhost_vdpa_release_dev;
1667 v->dev.parent = &vdpa->dev;
1668 v->dev.devt = MKDEV(MAJOR(vhost_vdpa_major), minor);
1669 v->vqs = kmalloc_objs(struct vhost_virtqueue, v->nvqs);
1670 if (!v->vqs) {
1671 r = -ENOMEM;
1672 goto err;
1673 }
1674
1675 r = dev_set_name(&v->dev, "vhost-vdpa-%u", minor);
1676 if (r)
1677 goto err;
1678
1679 cdev_init(&v->cdev, &vhost_vdpa_fops);
1680 v->cdev.owner = THIS_MODULE;
1681
1682 r = cdev_device_add(&v->cdev, &v->dev);
1683 if (r)
1684 goto err;
1685
1686 init_completion(&v->completion);
1687 vdpa_set_drvdata(vdpa, v);
1688
1689 for (i = 0; i < VHOST_VDPA_IOTLB_BUCKETS; i++)
1690 INIT_HLIST_HEAD(&v->as[i]);
1691
1692 return 0;
1693
1694err:
1695 put_device(&v->dev);
1696 return r;
1697}
1698
1699static void vhost_vdpa_remove(struct vdpa_device *vdpa)
1700{
1701 struct vhost_vdpa *v = vdpa_get_drvdata(vdpa);
1702 int opened;
1703
1704 cdev_device_del(&v->cdev, &v->dev);
1705
1706 do {
1707 opened = atomic_cmpxchg(&v->opened, 0, 1);
1708 if (!opened)
1709 break;
1710 wait_for_completion(&v->completion);
1711 } while (1);
1712
1713 put_device(&v->dev);
1714}
1715
1716static struct vdpa_driver vhost_vdpa_driver = {
1717 .driver = {
1718 .name = "vhost_vdpa",
1719 },
1720 .probe = vhost_vdpa_probe,
1721 .remove = vhost_vdpa_remove,
1722};
1723
1724static int __init vhost_vdpa_init(void)
1725{
1726 int r;
1727
1728 r = alloc_chrdev_region(&vhost_vdpa_major, 0, VHOST_VDPA_DEV_MAX,
1729 "vhost-vdpa");
1730 if (r)
1731 goto err_alloc_chrdev;
1732
1733 r = vdpa_register_driver(&vhost_vdpa_driver);
1734 if (r)
1735 goto err_vdpa_register_driver;
1736
1737 return 0;
1738
1739err_vdpa_register_driver:
1740 unregister_chrdev_region(vhost_vdpa_major, VHOST_VDPA_DEV_MAX);
1741err_alloc_chrdev:
1742 return r;
1743}
1744module_init(vhost_vdpa_init);
1745
1746static void __exit vhost_vdpa_exit(void)
1747{
1748 vdpa_unregister_driver(&vhost_vdpa_driver);
1749 unregister_chrdev_region(vhost_vdpa_major, VHOST_VDPA_DEV_MAX);
1750}
1751module_exit(vhost_vdpa_exit);
1752
1753MODULE_VERSION("0.0.1");
1754MODULE_LICENSE("GPL v2");
1755MODULE_AUTHOR("Intel Corporation");
1756MODULE_DESCRIPTION("vDPA-based vhost backend for virtio");