projects/linux·blob master · refs

1022 lines · 25.9 KiB

1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * vhost transport for vsock
4 *
5 * Copyright (C) 2013-2015 Red Hat, Inc.
6 * Author: Asias He <[email protected]>
7 * Stefan Hajnoczi <[email protected]>
8 */
9#include <linux/miscdevice.h>
10#include <linux/atomic.h>
11#include <linux/module.h>
12#include <linux/mutex.h>
13#include <linux/vmalloc.h>
14#include <net/sock.h>
15#include <linux/virtio_vsock.h>
16#include <linux/vhost.h>
17#include <linux/hashtable.h>
18
19#include <net/af_vsock.h>
20#include "vhost.h"
21
22#define VHOST_VSOCK_DEFAULT_HOST_CID 2
23/* Max number of bytes transferred before requeueing the job.
24 * Using this limit prevents one virtqueue from starving others. */
25#define VHOST_VSOCK_WEIGHT 0x80000
26/* Max number of packets transferred before requeueing the job.
27 * Using this limit prevents one virtqueue from starving others with
28 * small pkts.
29 */
30#define VHOST_VSOCK_PKT_WEIGHT 256
31
32static const int vhost_vsock_bits[] = {
33 VHOST_FEATURES,
34 VIRTIO_F_ACCESS_PLATFORM,
35 VIRTIO_VSOCK_F_SEQPACKET
36};
37
38#define VHOST_VSOCK_FEATURES VHOST_FEATURES_U64(vhost_vsock_bits, 0)
39
40enum {
41 VHOST_VSOCK_BACKEND_FEATURES = (1ULL << VHOST_BACKEND_F_IOTLB_MSG_V2)
42};
43
44/* Used to track all the vhost_vsock instances on the system. */
45static DEFINE_MUTEX(vhost_vsock_mutex);
46static DEFINE_READ_MOSTLY_HASHTABLE(vhost_vsock_hash, 8);
47
48struct vhost_vsock {
49 struct vhost_dev dev;
50 struct vhost_virtqueue vqs[2];
51 struct net *net;
52 netns_tracker ns_tracker;
53
54 /* Link to global vhost_vsock_hash, writes use vhost_vsock_mutex */
55 struct hlist_node hash;
56
57 struct vhost_work send_pkt_work;
58 struct sk_buff_head send_pkt_queue; /* host->guest pending packets */
59
60 atomic_t queued_replies;
61
62 u32 guest_cid;
63 bool seqpacket_allow;
64};
65
66static u32 vhost_transport_get_local_cid(void)
67{
68 return VHOST_VSOCK_DEFAULT_HOST_CID;
69}
70
71/* Callers must be in an RCU read section or hold the vhost_vsock_mutex.
72 * The return value can only be dereferenced while within the section.
73 */
74static struct vhost_vsock *vhost_vsock_get(u32 guest_cid, struct net *net)
75{
76 struct vhost_vsock *vsock;
77
78 hash_for_each_possible_rcu(vhost_vsock_hash, vsock, hash, guest_cid,
79 lockdep_is_held(&vhost_vsock_mutex)) {
80 u32 other_cid = vsock->guest_cid;
81
82 /* Skip instances that have no CID yet */
83 if (other_cid == 0)
84 continue;
85
86 if (other_cid == guest_cid &&
87 vsock_net_check_mode(net, vsock->net))
88 return vsock;
89 }
90
91 return NULL;
92}
93
94static bool vhost_transport_has_remote_cid(struct vsock_sock *vsk, u32 cid)
95{
96 struct sock *sk = sk_vsock(vsk);
97 struct net *net = sock_net(sk);
98 bool found;
99
100 rcu_read_lock();
101 found = !!vhost_vsock_get(cid, net);
102 rcu_read_unlock();
103 return found;
104}
105
106static void
107vhost_transport_do_send_pkt(struct vhost_vsock *vsock,
108 struct vhost_virtqueue *vq)
109{
110 struct vhost_virtqueue *tx_vq = &vsock->vqs[VSOCK_VQ_TX];
111 int pkts = 0, total_len = 0;
112 bool added = false;
113 bool restart_tx = false;
114
115 mutex_lock(&vq->mutex);
116
117 if (!vhost_vq_get_backend(vq))
118 goto out;
119
120 if (!vq_meta_prefetch(vq))
121 goto out;
122
123 /* Avoid further vmexits, we're already processing the virtqueue */
124 vhost_disable_notify(&vsock->dev, vq);
125
126 do {
127 struct virtio_vsock_hdr *hdr;
128 size_t iov_len, payload_len;
129 struct iov_iter iov_iter;
130 u32 flags_to_restore = 0;
131 struct sk_buff *skb;
132 unsigned out, in;
133 size_t nbytes;
134 u32 offset;
135 int head;
136
137 skb = virtio_vsock_skb_dequeue(&vsock->send_pkt_queue);
138
139 if (!skb) {
140 vhost_enable_notify(&vsock->dev, vq);
141 break;
142 }
143
144 head = vhost_get_vq_desc(vq, vq->iov, ARRAY_SIZE(vq->iov),
145 &out, &in, NULL, NULL);
146 if (head < 0) {
147 virtio_vsock_skb_queue_head(&vsock->send_pkt_queue, skb);
148 break;
149 }
150
151 if (head == vq->num) {
152 virtio_vsock_skb_queue_head(&vsock->send_pkt_queue, skb);
153 /* We cannot finish yet if more buffers snuck in while
154 * re-enabling notify.
155 */
156 if (unlikely(vhost_enable_notify(&vsock->dev, vq))) {
157 vhost_disable_notify(&vsock->dev, vq);
158 continue;
159 }
160 break;
161 }
162
163 if (out) {
164 kfree_skb(skb);
165 vq_err(vq, "Expected 0 output buffers, got %u\n", out);
166 break;
167 }
168
169 iov_len = iov_length(&vq->iov[out], in);
170 if (iov_len < sizeof(*hdr)) {
171 kfree_skb(skb);
172 vq_err(vq, "Buffer len [%zu] too small\n", iov_len);
173 break;
174 }
175
176 iov_iter_init(&iov_iter, ITER_DEST, &vq->iov[out], in, iov_len);
177 offset = VIRTIO_VSOCK_SKB_CB(skb)->offset;
178 payload_len = skb->len - offset;
179 hdr = virtio_vsock_hdr(skb);
180
181 /* If the packet is greater than the space available in the
182 * buffer, we split it using multiple buffers.
183 */
184 if (payload_len > iov_len - sizeof(*hdr)) {
185 payload_len = iov_len - sizeof(*hdr);
186
187 /* As we are copying pieces of large packet's buffer to
188 * small rx buffers, headers of packets in rx queue are
189 * created dynamically and are initialized with header
190 * of current packet(except length). But in case of
191 * SOCK_SEQPACKET, we also must clear message delimeter
192 * bit (VIRTIO_VSOCK_SEQ_EOM) and MSG_EOR bit
193 * (VIRTIO_VSOCK_SEQ_EOR) if set. Otherwise,
194 * there will be sequence of packets with these
195 * bits set. After initialized header will be copied to
196 * rx buffer, these required bits will be restored.
197 */
198 if (le32_to_cpu(hdr->flags) & VIRTIO_VSOCK_SEQ_EOM) {
199 hdr->flags &= ~cpu_to_le32(VIRTIO_VSOCK_SEQ_EOM);
200 flags_to_restore |= VIRTIO_VSOCK_SEQ_EOM;
201
202 if (le32_to_cpu(hdr->flags) & VIRTIO_VSOCK_SEQ_EOR) {
203 hdr->flags &= ~cpu_to_le32(VIRTIO_VSOCK_SEQ_EOR);
204 flags_to_restore |= VIRTIO_VSOCK_SEQ_EOR;
205 }
206 }
207 }
208
209 /* Set the correct length in the header */
210 hdr->len = cpu_to_le32(payload_len);
211
212 nbytes = copy_to_iter(hdr, sizeof(*hdr), &iov_iter);
213 if (nbytes != sizeof(*hdr)) {
214 kfree_skb(skb);
215 vq_err(vq, "Faulted on copying pkt hdr\n");
216 break;
217 }
218
219 if (skb_copy_datagram_iter(skb,
220 offset,
221 &iov_iter,
222 payload_len)) {
223 kfree_skb(skb);
224 vq_err(vq, "Faulted on copying pkt buf\n");
225 break;
226 }
227
228 /* Deliver to monitoring devices all packets that we
229 * will transmit.
230 */
231 virtio_transport_deliver_tap_pkt(skb);
232
233 vhost_add_used(vq, head, sizeof(*hdr) + payload_len);
234 added = true;
235
236 VIRTIO_VSOCK_SKB_CB(skb)->offset += payload_len;
237 total_len += payload_len;
238
239 /* If we didn't send all the payload we can requeue the packet
240 * to send it with the next available buffer.
241 */
242 if (VIRTIO_VSOCK_SKB_CB(skb)->offset < skb->len) {
243 hdr->flags |= cpu_to_le32(flags_to_restore);
244
245 /* We are queueing the same skb to handle
246 * the remaining bytes, and we want to deliver it
247 * to monitoring devices in the next iteration.
248 */
249 virtio_vsock_skb_clear_tap_delivered(skb);
250 virtio_vsock_skb_queue_head(&vsock->send_pkt_queue, skb);
251 } else {
252 if (virtio_vsock_skb_reply(skb)) {
253 int val;
254
255 val = atomic_dec_return(&vsock->queued_replies);
256
257 /* Do we have resources to resume tx
258 * processing?
259 */
260 if (val + 1 == tx_vq->num)
261 restart_tx = true;
262 }
263
264 virtio_transport_consume_skb_sent(skb, true);
265 }
266 } while(likely(!vhost_exceeds_weight(vq, ++pkts, total_len)));
267 if (added)
268 vhost_signal(&vsock->dev, vq);
269
270out:
271 mutex_unlock(&vq->mutex);
272
273 if (restart_tx)
274 vhost_poll_queue(&tx_vq->poll);
275}
276
277static void vhost_transport_send_pkt_work(struct vhost_work *work)
278{
279 struct vhost_virtqueue *vq;
280 struct vhost_vsock *vsock;
281
282 vsock = container_of(work, struct vhost_vsock, send_pkt_work);
283 vq = &vsock->vqs[VSOCK_VQ_RX];
284
285 vhost_transport_do_send_pkt(vsock, vq);
286}
287
288static int
289vhost_transport_send_pkt(struct sk_buff *skb, struct net *net)
290{
291 struct virtio_vsock_hdr *hdr = virtio_vsock_hdr(skb);
292 struct vhost_vsock *vsock;
293 int len = skb->len;
294
295 rcu_read_lock();
296
297 /* Find the vhost_vsock according to guest context id */
298 vsock = vhost_vsock_get(le64_to_cpu(hdr->dst_cid), net);
299 if (!vsock) {
300 rcu_read_unlock();
301 kfree_skb(skb);
302 return -ENODEV;
303 }
304
305 /* Fast-fail if the guest hasn't enabled the RX vq yet. Queuing the packet
306 * and making the caller wait is pointless: even if the guest manages to init
307 * within the timeout, it'll immediately reply with RST, because there's no
308 * listener on the port yet.
309 *
310 * vhost_vq_get_backend() without vq->mutex is acceptable here: locking
311 * the mutex would be too expensive in this hot path, and we already have
312 * all the outcomes covered: if the backend becomes NULL right after the check,
313 * vhost_transport_do_send_pkt() will check it under the mutex anyway.
314 */
315 if (unlikely(!data_race(vhost_vq_get_backend(&vsock->vqs[VSOCK_VQ_RX])))) {
316 rcu_read_unlock();
317 kfree_skb(skb);
318 return -EHOSTUNREACH;
319 }
320
321 if (virtio_vsock_skb_reply(skb))
322 atomic_inc(&vsock->queued_replies);
323
324 virtio_vsock_skb_queue_tail(&vsock->send_pkt_queue, skb);
325 vhost_vq_work_queue(&vsock->vqs[VSOCK_VQ_RX], &vsock->send_pkt_work);
326
327 rcu_read_unlock();
328 return len;
329}
330
331static int
332vhost_transport_cancel_pkt(struct vsock_sock *vsk)
333{
334 struct vhost_vsock *vsock;
335 int cnt = 0;
336 int ret = -ENODEV;
337
338 rcu_read_lock();
339
340 /* Find the vhost_vsock according to guest context id */
341 vsock = vhost_vsock_get(vsk->remote_addr.svm_cid,
342 sock_net(sk_vsock(vsk)));
343 if (!vsock)
344 goto out;
345
346 cnt = virtio_transport_purge_skbs(vsk, &vsock->send_pkt_queue);
347
348 if (cnt) {
349 struct vhost_virtqueue *tx_vq = &vsock->vqs[VSOCK_VQ_TX];
350 int new_cnt;
351
352 new_cnt = atomic_sub_return(cnt, &vsock->queued_replies);
353 if (new_cnt + cnt >= tx_vq->num && new_cnt < tx_vq->num)
354 vhost_poll_queue(&tx_vq->poll);
355 }
356
357 ret = 0;
358out:
359 rcu_read_unlock();
360 return ret;
361}
362
363static struct sk_buff *
364vhost_vsock_alloc_skb(struct vhost_virtqueue *vq,
365 unsigned int out, unsigned int in)
366{
367 struct virtio_vsock_hdr *hdr;
368 struct iov_iter iov_iter;
369 struct sk_buff *skb;
370 size_t payload_len;
371 size_t nbytes;
372 size_t len;
373
374 if (in != 0) {
375 vq_err(vq, "Expected 0 input buffers, got %u\n", in);
376 return NULL;
377 }
378
379 len = iov_length(vq->iov, out);
380
381 if (len < VIRTIO_VSOCK_SKB_HEADROOM ||
382 len > VIRTIO_VSOCK_MAX_PKT_BUF_SIZE + VIRTIO_VSOCK_SKB_HEADROOM)
383 return NULL;
384
385 /* len contains both payload and hdr */
386 skb = virtio_vsock_alloc_skb(len, GFP_KERNEL);
387 if (!skb)
388 return NULL;
389
390 iov_iter_init(&iov_iter, ITER_SOURCE, vq->iov, out, len);
391
392 hdr = virtio_vsock_hdr(skb);
393 nbytes = copy_from_iter(hdr, sizeof(*hdr), &iov_iter);
394 if (nbytes != sizeof(*hdr)) {
395 vq_err(vq, "Expected %zu bytes for pkt->hdr, got %zu bytes\n",
396 sizeof(*hdr), nbytes);
397 kfree_skb(skb);
398 return NULL;
399 }
400
401 payload_len = le32_to_cpu(hdr->len);
402
403 /* No payload */
404 if (!payload_len)
405 return skb;
406
407 /* The pkt is too big or the length in the header is invalid */
408 if (payload_len + sizeof(*hdr) > len) {
409 kfree_skb(skb);
410 return NULL;
411 }
412
413 virtio_vsock_skb_put(skb, payload_len);
414
415 if (skb_copy_datagram_from_iter(skb, 0, &iov_iter, payload_len)) {
416 vq_err(vq, "Failed to copy %zu byte payload\n", payload_len);
417 kfree_skb(skb);
418 return NULL;
419 }
420
421 return skb;
422}
423
424/* Is there space left for replies to rx packets? */
425static bool vhost_vsock_more_replies(struct vhost_vsock *vsock)
426{
427 struct vhost_virtqueue *vq = &vsock->vqs[VSOCK_VQ_TX];
428 int val;
429
430 smp_rmb(); /* paired with atomic_inc() and atomic_dec_return() */
431 val = atomic_read(&vsock->queued_replies);
432
433 return val < vq->num;
434}
435
436static bool vhost_transport_msgzerocopy_allow(void)
437{
438 return true;
439}
440
441static bool vhost_transport_seqpacket_allow(struct vsock_sock *vsk,
442 u32 remote_cid);
443
444static bool
445vhost_transport_stream_allow(struct vsock_sock *vsk, u32 cid, u32 port)
446{
447 return true;
448}
449
450static struct virtio_transport vhost_transport = {
451 .transport = {
452 .module = THIS_MODULE,
453
454 .get_local_cid = vhost_transport_get_local_cid,
455 .has_remote_cid = vhost_transport_has_remote_cid,
456
457 .init = virtio_transport_do_socket_init,
458 .destruct = virtio_transport_destruct,
459 .release = virtio_transport_release,
460 .connect = virtio_transport_connect,
461 .shutdown = virtio_transport_shutdown,
462 .cancel_pkt = vhost_transport_cancel_pkt,
463
464 .dgram_enqueue = virtio_transport_dgram_enqueue,
465 .dgram_dequeue = virtio_transport_dgram_dequeue,
466 .dgram_bind = virtio_transport_dgram_bind,
467 .dgram_allow = virtio_transport_dgram_allow,
468
469 .stream_enqueue = virtio_transport_stream_enqueue,
470 .stream_dequeue = virtio_transport_stream_dequeue,
471 .stream_has_data = virtio_transport_stream_has_data,
472 .stream_has_space = virtio_transport_stream_has_space,
473 .stream_rcvhiwat = virtio_transport_stream_rcvhiwat,
474 .stream_is_active = virtio_transport_stream_is_active,
475 .stream_allow = vhost_transport_stream_allow,
476
477 .seqpacket_dequeue = virtio_transport_seqpacket_dequeue,
478 .seqpacket_enqueue = virtio_transport_seqpacket_enqueue,
479 .seqpacket_allow = vhost_transport_seqpacket_allow,
480 .seqpacket_has_data = virtio_transport_seqpacket_has_data,
481
482 .msgzerocopy_allow = vhost_transport_msgzerocopy_allow,
483
484 .notify_poll_in = virtio_transport_notify_poll_in,
485 .notify_poll_out = virtio_transport_notify_poll_out,
486 .notify_recv_init = virtio_transport_notify_recv_init,
487 .notify_recv_pre_block = virtio_transport_notify_recv_pre_block,
488 .notify_recv_pre_dequeue = virtio_transport_notify_recv_pre_dequeue,
489 .notify_recv_post_dequeue = virtio_transport_notify_recv_post_dequeue,
490 .notify_send_init = virtio_transport_notify_send_init,
491 .notify_send_pre_block = virtio_transport_notify_send_pre_block,
492 .notify_send_pre_enqueue = virtio_transport_notify_send_pre_enqueue,
493 .notify_send_post_enqueue = virtio_transport_notify_send_post_enqueue,
494 .notify_buffer_size = virtio_transport_notify_buffer_size,
495 .notify_set_rcvlowat = virtio_transport_notify_set_rcvlowat,
496
497 .unsent_bytes = virtio_transport_unsent_bytes,
498
499 .read_skb = virtio_transport_read_skb,
500 },
501
502 .send_pkt = vhost_transport_send_pkt,
503};
504
505static bool vhost_transport_seqpacket_allow(struct vsock_sock *vsk,
506 u32 remote_cid)
507{
508 struct net *net = sock_net(sk_vsock(vsk));
509 struct vhost_vsock *vsock;
510 bool seqpacket_allow = false;
511
512 rcu_read_lock();
513 vsock = vhost_vsock_get(remote_cid, net);
514
515 if (vsock)
516 seqpacket_allow = vsock->seqpacket_allow;
517
518 rcu_read_unlock();
519
520 return seqpacket_allow;
521}
522
523static void vhost_vsock_handle_tx_kick(struct vhost_work *work)
524{
525 struct vhost_virtqueue *vq = container_of(work, struct vhost_virtqueue,
526 poll.work);
527 struct vhost_vsock *vsock = container_of(vq->dev, struct vhost_vsock,
528 dev);
529 int head, pkts = 0, total_len = 0;
530 unsigned int out, in;
531 struct sk_buff *skb;
532 bool added = false;
533
534 mutex_lock(&vq->mutex);
535
536 if (!vhost_vq_get_backend(vq))
537 goto out;
538
539 if (!vq_meta_prefetch(vq))
540 goto out;
541
542 vhost_disable_notify(&vsock->dev, vq);
543 do {
544 struct virtio_vsock_hdr *hdr;
545
546 if (!vhost_vsock_more_replies(vsock)) {
547 /* Stop tx until the device processes already
548 * pending replies. Leave tx virtqueue
549 * callbacks disabled.
550 */
551 goto no_more_replies;
552 }
553
554 head = vhost_get_vq_desc(vq, vq->iov, ARRAY_SIZE(vq->iov),
555 &out, &in, NULL, NULL);
556 if (head < 0)
557 break;
558
559 if (head == vq->num) {
560 if (unlikely(vhost_enable_notify(&vsock->dev, vq))) {
561 vhost_disable_notify(&vsock->dev, vq);
562 continue;
563 }
564 break;
565 }
566
567 skb = vhost_vsock_alloc_skb(vq, out, in);
568 if (!skb) {
569 vq_err(vq, "Faulted on pkt\n");
570 continue;
571 }
572
573 total_len += sizeof(*hdr) + skb->len;
574
575 /* Deliver to monitoring devices all received packets */
576 virtio_transport_deliver_tap_pkt(skb);
577
578 hdr = virtio_vsock_hdr(skb);
579
580 /* Only accept correctly addressed packets */
581 if (le64_to_cpu(hdr->src_cid) == vsock->guest_cid &&
582 le64_to_cpu(hdr->dst_cid) ==
583 vhost_transport_get_local_cid())
584 virtio_transport_recv_pkt(&vhost_transport, skb,
585 vsock->net);
586 else
587 kfree_skb(skb);
588
589 vhost_add_used(vq, head, 0);
590 added = true;
591 } while(likely(!vhost_exceeds_weight(vq, ++pkts, total_len)));
592
593no_more_replies:
594 if (added)
595 vhost_signal(&vsock->dev, vq);
596
597out:
598 mutex_unlock(&vq->mutex);
599}
600
601static void vhost_vsock_handle_rx_kick(struct vhost_work *work)
602{
603 struct vhost_virtqueue *vq = container_of(work, struct vhost_virtqueue,
604 poll.work);
605 struct vhost_vsock *vsock = container_of(vq->dev, struct vhost_vsock,
606 dev);
607
608 vhost_transport_do_send_pkt(vsock, vq);
609}
610
611static int vhost_vsock_start(struct vhost_vsock *vsock)
612{
613 struct vhost_virtqueue *vq;
614 size_t i;
615 int ret;
616
617 mutex_lock(&vsock->dev.mutex);
618
619 ret = vhost_dev_check_owner(&vsock->dev);
620 if (ret)
621 goto err;
622
623 for (i = 0; i < ARRAY_SIZE(vsock->vqs); i++) {
624 vq = &vsock->vqs[i];
625
626 mutex_lock(&vq->mutex);
627
628 if (!vhost_vq_access_ok(vq)) {
629 ret = -EFAULT;
630 goto err_vq;
631 }
632
633 if (!vhost_vq_get_backend(vq)) {
634 vhost_vq_set_backend(vq, vsock);
635 ret = vhost_vq_init_access(vq);
636 if (ret)
637 goto err_vq;
638 }
639
640 mutex_unlock(&vq->mutex);
641 }
642
643 /* Some packets may have been queued before the device was started,
644 * let's kick the send worker to send them.
645 */
646 vhost_vq_work_queue(&vsock->vqs[VSOCK_VQ_RX], &vsock->send_pkt_work);
647
648 mutex_unlock(&vsock->dev.mutex);
649 return 0;
650
651err_vq:
652 vhost_vq_set_backend(vq, NULL);
653 mutex_unlock(&vq->mutex);
654
655 for (i = 0; i < ARRAY_SIZE(vsock->vqs); i++) {
656 vq = &vsock->vqs[i];
657
658 mutex_lock(&vq->mutex);
659 vhost_vq_set_backend(vq, NULL);
660 mutex_unlock(&vq->mutex);
661 }
662err:
663 mutex_unlock(&vsock->dev.mutex);
664 return ret;
665}
666
667static int vhost_vsock_stop(struct vhost_vsock *vsock, bool check_owner)
668{
669 size_t i;
670 int ret = 0;
671
672 mutex_lock(&vsock->dev.mutex);
673
674 if (check_owner) {
675 ret = vhost_dev_check_owner(&vsock->dev);
676 if (ret)
677 goto err;
678 }
679
680 for (i = 0; i < ARRAY_SIZE(vsock->vqs); i++) {
681 struct vhost_virtqueue *vq = &vsock->vqs[i];
682
683 mutex_lock(&vq->mutex);
684 vhost_vq_set_backend(vq, NULL);
685 mutex_unlock(&vq->mutex);
686 }
687
688err:
689 mutex_unlock(&vsock->dev.mutex);
690 return ret;
691}
692
693static void vhost_vsock_free(struct vhost_vsock *vsock)
694{
695 kvfree(vsock);
696}
697
698static int vhost_vsock_dev_open(struct inode *inode, struct file *file)
699{
700 struct vhost_virtqueue **vqs;
701 struct vhost_vsock *vsock;
702 struct net *net;
703 int ret;
704
705 /* This struct is large and allocation could fail, fall back to vmalloc
706 * if there is no other way.
707 */
708 vsock = kvmalloc_obj(*vsock, GFP_KERNEL | __GFP_RETRY_MAYFAIL);
709 if (!vsock)
710 return -ENOMEM;
711
712 vqs = kmalloc_objs(*vqs, ARRAY_SIZE(vsock->vqs));
713 if (!vqs) {
714 ret = -ENOMEM;
715 goto out;
716 }
717
718 net = current->nsproxy->net_ns;
719 vsock->net = get_net_track(net, &vsock->ns_tracker, GFP_KERNEL);
720
721 vsock->guest_cid = 0; /* no CID assigned yet */
722 vsock->seqpacket_allow = false;
723
724 atomic_set(&vsock->queued_replies, 0);
725
726 vqs[VSOCK_VQ_TX] = &vsock->vqs[VSOCK_VQ_TX];
727 vqs[VSOCK_VQ_RX] = &vsock->vqs[VSOCK_VQ_RX];
728 vsock->vqs[VSOCK_VQ_TX].handle_kick = vhost_vsock_handle_tx_kick;
729 vsock->vqs[VSOCK_VQ_RX].handle_kick = vhost_vsock_handle_rx_kick;
730
731 vhost_dev_init(&vsock->dev, vqs, ARRAY_SIZE(vsock->vqs),
732 UIO_MAXIOV, VHOST_VSOCK_PKT_WEIGHT,
733 VHOST_VSOCK_WEIGHT, true, NULL);
734
735 file->private_data = vsock;
736 skb_queue_head_init(&vsock->send_pkt_queue);
737 vhost_work_init(&vsock->send_pkt_work, vhost_transport_send_pkt_work);
738 return 0;
739
740out:
741 vhost_vsock_free(vsock);
742 return ret;
743}
744
745static void vhost_vsock_flush(struct vhost_vsock *vsock)
746{
747 vhost_dev_flush(&vsock->dev);
748}
749
750static void vhost_vsock_reset_orphans(struct sock *sk)
751{
752 struct vsock_sock *vsk = vsock_sk(sk);
753
754 /* vmci_transport.c doesn't take sk_lock here either. At least we're
755 * under vsock_table_lock so the sock cannot disappear while we're
756 * executing.
757 */
758
759 rcu_read_lock();
760
761 /* If the peer is still valid, no need to reset connection */
762 if (vhost_vsock_get(vsk->remote_addr.svm_cid, sock_net(sk))) {
763 rcu_read_unlock();
764 return;
765 }
766
767 rcu_read_unlock();
768
769 /* If the close timeout is pending, let it expire. This avoids races
770 * with the timeout callback.
771 */
772 if (vsk->close_work_scheduled)
773 return;
774
775 sock_set_flag(sk, SOCK_DONE);
776 vsk->peer_shutdown = SHUTDOWN_MASK;
777 sk->sk_state = SS_UNCONNECTED;
778 sk->sk_err = ECONNRESET;
779 sk_error_report(sk);
780}
781
782static int vhost_vsock_dev_release(struct inode *inode, struct file *file)
783{
784 struct vhost_vsock *vsock = file->private_data;
785
786 mutex_lock(&vhost_vsock_mutex);
787 if (vsock->guest_cid)
788 hash_del_rcu(&vsock->hash);
789 mutex_unlock(&vhost_vsock_mutex);
790
791 /* Wait for other CPUs to finish using vsock */
792 synchronize_rcu();
793
794 /* Iterating over all connections for all CIDs to find orphans is
795 * inefficient. Room for improvement here. */
796 vsock_for_each_connected_socket(&vhost_transport.transport,
797 vhost_vsock_reset_orphans);
798
799 /* Don't check the owner, because we are in the release path, so we
800 * need to stop the vsock device in any case.
801 * vhost_vsock_stop() can not fail in this case, so we don't need to
802 * check the return code.
803 */
804 vhost_vsock_stop(vsock, false);
805 vhost_vsock_flush(vsock);
806 vhost_dev_stop(&vsock->dev);
807
808 virtio_vsock_skb_queue_purge(&vsock->send_pkt_queue);
809
810 vhost_dev_cleanup(&vsock->dev);
811 put_net_track(vsock->net, &vsock->ns_tracker);
812 kfree(vsock->dev.vqs);
813 vhost_vsock_free(vsock);
814 return 0;
815}
816
817static int vhost_vsock_set_cid(struct vhost_vsock *vsock, u64 guest_cid)
818{
819 struct vhost_vsock *other;
820
821 /* Refuse reserved CIDs */
822 if (guest_cid <= VMADDR_CID_HOST ||
823 guest_cid == U32_MAX)
824 return -EINVAL;
825
826 /* 64-bit CIDs are not yet supported */
827 if (guest_cid > U32_MAX)
828 return -EINVAL;
829
830 /* Refuse if CID is assigned to the guest->host transport (i.e. nested
831 * VM), to make the loopback work.
832 */
833 if (vsock_find_cid(guest_cid))
834 return -EADDRINUSE;
835
836 /* Refuse if CID is already in use */
837 mutex_lock(&vhost_vsock_mutex);
838 other = vhost_vsock_get(guest_cid, vsock->net);
839 if (other && other != vsock) {
840 mutex_unlock(&vhost_vsock_mutex);
841 return -EADDRINUSE;
842 }
843
844 if (vsock->guest_cid)
845 hash_del_rcu(&vsock->hash);
846
847 vsock->guest_cid = guest_cid;
848 hash_add_rcu(vhost_vsock_hash, &vsock->hash, vsock->guest_cid);
849 mutex_unlock(&vhost_vsock_mutex);
850
851 return 0;
852}
853
854static int vhost_vsock_set_features(struct vhost_vsock *vsock, u64 features)
855{
856 struct vhost_virtqueue *vq;
857 int i;
858
859 if (features & ~VHOST_VSOCK_FEATURES)
860 return -EOPNOTSUPP;
861
862 mutex_lock(&vsock->dev.mutex);
863 if ((features & (1 << VHOST_F_LOG_ALL)) &&
864 !vhost_log_access_ok(&vsock->dev)) {
865 goto err;
866 }
867
868 if ((features & (1ULL << VIRTIO_F_ACCESS_PLATFORM))) {
869 if (vhost_init_device_iotlb(&vsock->dev))
870 goto err;
871 } else {
872 vhost_clear_device_iotlb(&vsock->dev);
873 }
874
875 vsock->seqpacket_allow = features & (1ULL << VIRTIO_VSOCK_F_SEQPACKET);
876
877 for (i = 0; i < ARRAY_SIZE(vsock->vqs); i++) {
878 vq = &vsock->vqs[i];
879 mutex_lock(&vq->mutex);
880 vq->acked_features = features;
881 mutex_unlock(&vq->mutex);
882 }
883 mutex_unlock(&vsock->dev.mutex);
884 return 0;
885
886err:
887 mutex_unlock(&vsock->dev.mutex);
888 return -EFAULT;
889}
890
891static long vhost_vsock_dev_ioctl(struct file *f, unsigned int ioctl,
892 unsigned long arg)
893{
894 struct vhost_vsock *vsock = f->private_data;
895 void __user *argp = (void __user *)arg;
896 u64 guest_cid;
897 u64 features;
898 int start;
899 int r;
900
901 switch (ioctl) {
902 case VHOST_VSOCK_SET_GUEST_CID:
903 if (copy_from_user(&guest_cid, argp, sizeof(guest_cid)))
904 return -EFAULT;
905 return vhost_vsock_set_cid(vsock, guest_cid);
906 case VHOST_VSOCK_SET_RUNNING:
907 if (copy_from_user(&start, argp, sizeof(start)))
908 return -EFAULT;
909 if (start)
910 return vhost_vsock_start(vsock);
911 else
912 return vhost_vsock_stop(vsock, true);
913 case VHOST_GET_FEATURES:
914 features = VHOST_VSOCK_FEATURES;
915 if (copy_to_user(argp, &features, sizeof(features)))
916 return -EFAULT;
917 return 0;
918 case VHOST_SET_FEATURES:
919 if (copy_from_user(&features, argp, sizeof(features)))
920 return -EFAULT;
921 return vhost_vsock_set_features(vsock, features);
922 case VHOST_GET_BACKEND_FEATURES:
923 features = VHOST_VSOCK_BACKEND_FEATURES;
924 if (copy_to_user(argp, &features, sizeof(features)))
925 return -EFAULT;
926 return 0;
927 case VHOST_SET_BACKEND_FEATURES:
928 if (copy_from_user(&features, argp, sizeof(features)))
929 return -EFAULT;
930 if (features & ~VHOST_VSOCK_BACKEND_FEATURES)
931 return -EOPNOTSUPP;
932 vhost_set_backend_features(&vsock->dev, features);
933 return 0;
934 default:
935 mutex_lock(&vsock->dev.mutex);
936 r = vhost_dev_ioctl(&vsock->dev, ioctl, argp);
937 if (r == -ENOIOCTLCMD)
938 r = vhost_vring_ioctl(&vsock->dev, ioctl, argp);
939 else
940 vhost_vsock_flush(vsock);
941 mutex_unlock(&vsock->dev.mutex);
942 return r;
943 }
944}
945
946static ssize_t vhost_vsock_chr_read_iter(struct kiocb *iocb, struct iov_iter *to)
947{
948 struct file *file = iocb->ki_filp;
949 struct vhost_vsock *vsock = file->private_data;
950 struct vhost_dev *dev = &vsock->dev;
951 int noblock = file->f_flags & O_NONBLOCK;
952
953 return vhost_chr_read_iter(dev, to, noblock);
954}
955
956static ssize_t vhost_vsock_chr_write_iter(struct kiocb *iocb,
957 struct iov_iter *from)
958{
959 struct file *file = iocb->ki_filp;
960 struct vhost_vsock *vsock = file->private_data;
961 struct vhost_dev *dev = &vsock->dev;
962
963 return vhost_chr_write_iter(dev, from);
964}
965
966static __poll_t vhost_vsock_chr_poll(struct file *file, poll_table *wait)
967{
968 struct vhost_vsock *vsock = file->private_data;
969 struct vhost_dev *dev = &vsock->dev;
970
971 return vhost_chr_poll(file, dev, wait);
972}
973
974static const struct file_operations vhost_vsock_fops = {
975 .owner = THIS_MODULE,
976 .open = vhost_vsock_dev_open,
977 .release = vhost_vsock_dev_release,
978 .llseek = noop_llseek,
979 .unlocked_ioctl = vhost_vsock_dev_ioctl,
980 .compat_ioctl = compat_ptr_ioctl,
981 .read_iter = vhost_vsock_chr_read_iter,
982 .write_iter = vhost_vsock_chr_write_iter,
983 .poll = vhost_vsock_chr_poll,
984};
985
986static struct miscdevice vhost_vsock_misc = {
987 .minor = VHOST_VSOCK_MINOR,
988 .name = "vhost-vsock",
989 .fops = &vhost_vsock_fops,
990};
991
992static int __init vhost_vsock_init(void)
993{
994 int ret;
995
996 ret = vsock_core_register(&vhost_transport.transport,
997 VSOCK_TRANSPORT_F_H2G);
998 if (ret < 0)
999 return ret;
1000
1001 ret = misc_register(&vhost_vsock_misc);
1002 if (ret) {
1003 vsock_core_unregister(&vhost_transport.transport);
1004 return ret;
1005 }
1006
1007 return 0;
1008};
1009
1010static void __exit vhost_vsock_exit(void)
1011{
1012 misc_deregister(&vhost_vsock_misc);
1013 vsock_core_unregister(&vhost_transport.transport);
1014};
1015
1016module_init(vhost_vsock_init);
1017module_exit(vhost_vsock_exit);
1018MODULE_LICENSE("GPL v2");
1019MODULE_AUTHOR("Asias He");
1020MODULE_DESCRIPTION("vhost transport for vsock ");
1021MODULE_ALIAS_MISCDEV(VHOST_VSOCK_MINOR);
1022MODULE_ALIAS("devname:vhost-vsock");