projects/linux·blob master · refs

3167 lines · 78.1 KiB

1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * VMware vSockets Driver
4 *
5 * Copyright (C) 2007-2013 VMware, Inc. All rights reserved.
6 */
7
8/* Implementation notes:
9 *
10 * - There are two kinds of sockets: those created by user action (such as
11 * calling socket(2)) and those created by incoming connection request packets.
12 *
13 * - There are two "global" tables, one for bound sockets (sockets that have
14 * specified an address that they are responsible for) and one for connected
15 * sockets (sockets that have established a connection with another socket).
16 * These tables are "global" in that all sockets on the system are placed
17 * within them. - Note, though, that the bound table contains an extra entry
18 * for a list of unbound sockets and SOCK_DGRAM sockets will always remain in
19 * that list. The bound table is used solely for lookup of sockets when packets
20 * are received and that's not necessary for SOCK_DGRAM sockets since we create
21 * a datagram handle for each and need not perform a lookup. Keeping SOCK_DGRAM
22 * sockets out of the bound hash buckets will reduce the chance of collisions
23 * when looking for SOCK_STREAM sockets and prevents us from having to check the
24 * socket type in the hash table lookups.
25 *
26 * - Sockets created by user action will either be "client" sockets that
27 * initiate a connection or "server" sockets that listen for connections; we do
28 * not support simultaneous connects (two "client" sockets connecting).
29 *
30 * - "Server" sockets are referred to as listener sockets throughout this
31 * implementation because they are in the TCP_LISTEN state. When a
32 * connection request is received (the second kind of socket mentioned above),
33 * we create a new socket and refer to it as a pending socket. These pending
34 * sockets are placed on the pending connection list of the listener socket.
35 * When future packets are received for the address the listener socket is
36 * bound to, we check if the source of the packet is from one that has an
37 * existing pending connection. If it does, we process the packet for the
38 * pending socket. When that socket reaches the connected state, it is removed
39 * from the listener socket's pending list and enqueued in the listener
40 * socket's accept queue. Callers of accept(2) will accept connected sockets
41 * from the listener socket's accept queue. Once the connection is accepted,
42 * it is owned by the user process and the responsibility for cleanup falls
43 * with that user process.
44 *
45 * - It is possible that these pending sockets will never reach the connected
46 * state; in fact, we may never receive another packet after the connection
47 * request. Because of this, we must schedule a cleanup function to run in the
48 * future, after some amount of time passes where a connection should have been
49 * established. This function ensures that the socket is off all lists so it
50 * cannot be retrieved, then drops all references to the socket so it is cleaned
51 * up (sock_put() -> sk_free() -> our sk_destruct implementation).
52 *
53 * - Lock ordering for pending or accept queue sockets is:
54 *
55 * lock_sock(listener);
56 * lock_sock_nested(pending, SINGLE_DEPTH_NESTING);
57 *
58 * Using explicit nested locking keeps lockdep happy since normally only one
59 * lock of a given class may be taken at a time.
60 *
61 * - Sockets created by user action will be cleaned up when the user process
62 * calls close(2), causing our release implementation to be called. Our release
63 * implementation will perform some cleanup then drop the last reference so our
64 * sk_destruct implementation is invoked. Our sk_destruct implementation will
65 * perform additional cleanup that's common for both types of sockets.
66 *
67 * - A socket's reference count is what ensures that the structure won't be
68 * freed. Each entry in a list (such as the "global" bound and connected tables
69 * and the listener socket's pending list and connected queue) ensures a
70 * reference. When we defer work until process context and pass a socket as our
71 * argument, we must ensure the reference count is increased to ensure the
72 * socket isn't freed before the function is run; the deferred function will
73 * then drop the reference.
74 *
75 * - sk->sk_state uses the TCP state constants because they are widely used by
76 * other address families and exposed to userspace tools like ss(8):
77 *
78 * TCP_CLOSE - unconnected
79 * TCP_SYN_SENT - connecting
80 * TCP_ESTABLISHED - connected
81 * TCP_CLOSING - disconnecting
82 * TCP_LISTEN - listening
83 *
84 * - Namespaces in vsock support two different modes: "local" and "global".
85 * Each mode defines how the namespace interacts with CIDs.
86 * Each namespace exposes two sysctl files:
87 *
88 * - /proc/sys/net/vsock/ns_mode (read-only) reports the current namespace's
89 * mode, which is set at namespace creation and immutable thereafter.
90 * - /proc/sys/net/vsock/child_ns_mode (write-once) controls what mode future
91 * child namespaces will inherit when created. The initial value matches
92 * the namespace's own ns_mode.
93 *
94 * Changing child_ns_mode only affects newly created namespaces, not the
95 * current namespace or existing children. A "local" namespace cannot set
96 * child_ns_mode to "global". child_ns_mode is write-once, so that it may be
97 * configured and locked down by a namespace manager. Writing a different
98 * value after the first write returns -EBUSY. At namespace creation, ns_mode
99 * is inherited from the parent's child_ns_mode.
100 *
101 * The init_net mode is "global" and cannot be modified. The init_net
102 * child_ns_mode is also write-once, so an init process (e.g. systemd) can
103 * set it to "local" to ensure all new namespaces inherit local mode.
104 *
105 * The modes affect the allocation and accessibility of CIDs as follows:
106 *
107 * - global - access and allocation are all system-wide
108 * - all CID allocation from global namespaces draw from the same
109 * system-wide pool.
110 * - if one global namespace has already allocated some CID, another
111 * global namespace will not be able to allocate the same CID.
112 * - global mode AF_VSOCK sockets can reach any VM or socket in any global
113 * namespace, they are not contained to only their own namespace.
114 * - AF_VSOCK sockets in a global mode namespace cannot reach VMs or
115 * sockets in any local mode namespace.
116 * - local - access and allocation are contained within the namespace
117 * - CID allocation draws only from a private pool local only to the
118 * namespace, and does not affect the CIDs available for allocation in any
119 * other namespace (global or local).
120 * - VMs in a local namespace do not collide with CIDs in any other local
121 * namespace or any global namespace. For example, if a VM in a local mode
122 * namespace is given CID 10, then CID 10 is still available for
123 * allocation in any other namespace, but not in the same namespace.
124 * - AF_VSOCK sockets in a local mode namespace can connect only to VMs or
125 * other sockets within their own namespace.
126 * - sockets bound to VMADDR_CID_ANY in local namespaces will never resolve
127 * to any transport that is not compatible with local mode. There is no
128 * error that propagates to the user (as there is for connection attempts)
129 * because it is possible for some packet to reach this socket from
130 * a different transport that *does* support local mode. For
131 * example, virtio-vsock may not support local mode, but the socket
132 * may still accept a connection from vhost-vsock which does.
133 */
134
135#include <linux/compat.h>
136#include <linux/types.h>
137#include <linux/bitops.h>
138#include <linux/cred.h>
139#include <linux/errqueue.h>
140#include <linux/init.h>
141#include <linux/io.h>
142#include <linux/kernel.h>
143#include <linux/sched/signal.h>
144#include <linux/kmod.h>
145#include <linux/list.h>
146#include <linux/miscdevice.h>
147#include <linux/module.h>
148#include <linux/mutex.h>
149#include <linux/net.h>
150#include <linux/proc_fs.h>
151#include <linux/poll.h>
152#include <linux/random.h>
153#include <linux/skbuff.h>
154#include <linux/smp.h>
155#include <linux/uio.h>
156#include <linux/socket.h>
157#include <linux/stddef.h>
158#include <linux/sysctl.h>
159#include <linux/unistd.h>
160#include <linux/wait.h>
161#include <linux/workqueue.h>
162#include <net/sock.h>
163#include <net/af_vsock.h>
164#include <net/netns/vsock.h>
165#include <uapi/linux/vm_sockets.h>
166#include <uapi/asm-generic/ioctls.h>
167
168#define VSOCK_NET_MODE_STR_GLOBAL "global"
169#define VSOCK_NET_MODE_STR_LOCAL "local"
170
171/* 6 chars for "global", 1 for null-terminator, and 1 more for '\n'.
172 * The newline is added by proc_dostring() for read operations.
173 */
174#define VSOCK_NET_MODE_STR_MAX 8
175
176static int __vsock_bind(struct sock *sk, struct sockaddr_vm *addr);
177static void vsock_sk_destruct(struct sock *sk);
178static int vsock_queue_rcv_skb(struct sock *sk, struct sk_buff *skb);
179static void vsock_close(struct sock *sk, long timeout);
180
181/* Protocol family. */
182struct proto vsock_proto = {
183 .name = "AF_VSOCK",
184 .owner = THIS_MODULE,
185 .obj_size = sizeof(struct vsock_sock),
186 .close = vsock_close,
187#ifdef CONFIG_BPF_SYSCALL
188 .psock_update_sk_prot = vsock_bpf_update_proto,
189#endif
190};
191
192/* The default peer timeout indicates how long we will wait for a peer response
193 * to a control message.
194 */
195#define VSOCK_DEFAULT_CONNECT_TIMEOUT (2 * HZ)
196
197#define VSOCK_DEFAULT_BUFFER_SIZE (1024 * 256)
198#define VSOCK_DEFAULT_BUFFER_MAX_SIZE (1024 * 256)
199#define VSOCK_DEFAULT_BUFFER_MIN_SIZE 128
200
201/* Transport used for host->guest communication */
202static const struct vsock_transport *transport_h2g;
203/* Transport used for guest->host communication */
204static const struct vsock_transport *transport_g2h;
205/* Transport used for DGRAM communication */
206static const struct vsock_transport *transport_dgram;
207/* Transport used for local communication */
208static const struct vsock_transport *transport_local;
209static DEFINE_MUTEX(vsock_register_mutex);
210
211/**** UTILS ****/
212
213/* Each bound VSocket is stored in the bind hash table and each connected
214 * VSocket is stored in the connected hash table.
215 *
216 * Unbound sockets are all put on the same list attached to the end of the hash
217 * table (vsock_unbound_sockets). Bound sockets are added to the hash table in
218 * the bucket that their local address hashes to (vsock_bound_sockets(addr)
219 * represents the list that addr hashes to).
220 *
221 * Specifically, we initialize the vsock_bind_table array to a size of
222 * VSOCK_HASH_SIZE + 1 so that vsock_bind_table[0] through
223 * vsock_bind_table[VSOCK_HASH_SIZE - 1] are for bound sockets and
224 * vsock_bind_table[VSOCK_HASH_SIZE] is for unbound sockets. The hash function
225 * mods with VSOCK_HASH_SIZE to ensure this.
226 */
227#define MAX_PORT_RETRIES 24
228
229#define VSOCK_HASH(addr) ((addr)->svm_port % VSOCK_HASH_SIZE)
230#define vsock_bound_sockets(addr) (&vsock_bind_table[VSOCK_HASH(addr)])
231#define vsock_unbound_sockets (&vsock_bind_table[VSOCK_HASH_SIZE])
232
233/* XXX This can probably be implemented in a better way. */
234#define VSOCK_CONN_HASH(src, dst) \
235 (((src)->svm_cid ^ (dst)->svm_port) % VSOCK_HASH_SIZE)
236#define vsock_connected_sockets(src, dst) \
237 (&vsock_connected_table[VSOCK_CONN_HASH(src, dst)])
238#define vsock_connected_sockets_vsk(vsk) \
239 vsock_connected_sockets(&(vsk)->remote_addr, &(vsk)->local_addr)
240
241struct list_head vsock_bind_table[VSOCK_HASH_SIZE + 1];
242EXPORT_SYMBOL_GPL(vsock_bind_table);
243struct list_head vsock_connected_table[VSOCK_HASH_SIZE];
244EXPORT_SYMBOL_GPL(vsock_connected_table);
245DEFINE_SPINLOCK(vsock_table_lock);
246EXPORT_SYMBOL_GPL(vsock_table_lock);
247
248/* Autobind this socket to the local address if necessary. */
249static int vsock_auto_bind(struct vsock_sock *vsk)
250{
251 struct sock *sk = sk_vsock(vsk);
252 struct sockaddr_vm local_addr;
253
254 if (vsock_addr_bound(&vsk->local_addr))
255 return 0;
256 vsock_addr_init(&local_addr, VMADDR_CID_ANY, VMADDR_PORT_ANY);
257 return __vsock_bind(sk, &local_addr);
258}
259
260static void vsock_init_tables(void)
261{
262 int i;
263
264 for (i = 0; i < ARRAY_SIZE(vsock_bind_table); i++)
265 INIT_LIST_HEAD(&vsock_bind_table[i]);
266
267 for (i = 0; i < ARRAY_SIZE(vsock_connected_table); i++)
268 INIT_LIST_HEAD(&vsock_connected_table[i]);
269}
270
271static void __vsock_insert_bound(struct list_head *list,
272 struct vsock_sock *vsk)
273{
274 sock_hold(&vsk->sk);
275 list_add(&vsk->bound_table, list);
276}
277
278static void __vsock_insert_connected(struct list_head *list,
279 struct vsock_sock *vsk)
280{
281 sock_hold(&vsk->sk);
282 list_add(&vsk->connected_table, list);
283}
284
285static void __vsock_remove_bound(struct vsock_sock *vsk)
286{
287 list_del_init(&vsk->bound_table);
288 sock_put(&vsk->sk);
289}
290
291static void __vsock_remove_connected(struct vsock_sock *vsk)
292{
293 list_del_init(&vsk->connected_table);
294 sock_put(&vsk->sk);
295}
296
297static struct sock *__vsock_find_bound_socket_net(struct sockaddr_vm *addr,
298 struct net *net)
299{
300 struct vsock_sock *vsk;
301
302 list_for_each_entry(vsk, vsock_bound_sockets(addr), bound_table) {
303 struct sock *sk = sk_vsock(vsk);
304
305 if (vsock_addr_equals_addr(addr, &vsk->local_addr) &&
306 vsock_net_check_mode(sock_net(sk), net))
307 return sk;
308
309 if (addr->svm_port == vsk->local_addr.svm_port &&
310 (vsk->local_addr.svm_cid == VMADDR_CID_ANY ||
311 addr->svm_cid == VMADDR_CID_ANY) &&
312 vsock_net_check_mode(sock_net(sk), net))
313 return sk;
314 }
315
316 return NULL;
317}
318
319static struct sock *
320__vsock_find_connected_socket_net(struct sockaddr_vm *src,
321 struct sockaddr_vm *dst, struct net *net)
322{
323 struct vsock_sock *vsk;
324
325 list_for_each_entry(vsk, vsock_connected_sockets(src, dst),
326 connected_table) {
327 struct sock *sk = sk_vsock(vsk);
328
329 if (vsock_addr_equals_addr(src, &vsk->remote_addr) &&
330 dst->svm_port == vsk->local_addr.svm_port &&
331 vsock_net_check_mode(sock_net(sk), net)) {
332 return sk;
333 }
334 }
335
336 return NULL;
337}
338
339static void vsock_insert_unbound(struct vsock_sock *vsk)
340{
341 spin_lock_bh(&vsock_table_lock);
342 __vsock_insert_bound(vsock_unbound_sockets, vsk);
343 spin_unlock_bh(&vsock_table_lock);
344}
345
346void vsock_insert_connected(struct vsock_sock *vsk)
347{
348 struct list_head *list = vsock_connected_sockets(
349 &vsk->remote_addr, &vsk->local_addr);
350
351 spin_lock_bh(&vsock_table_lock);
352 __vsock_insert_connected(list, vsk);
353 spin_unlock_bh(&vsock_table_lock);
354}
355EXPORT_SYMBOL_GPL(vsock_insert_connected);
356
357void vsock_remove_bound(struct vsock_sock *vsk)
358{
359 spin_lock_bh(&vsock_table_lock);
360 if (__vsock_in_bound_table(vsk))
361 __vsock_remove_bound(vsk);
362 spin_unlock_bh(&vsock_table_lock);
363}
364EXPORT_SYMBOL_GPL(vsock_remove_bound);
365
366void vsock_remove_connected(struct vsock_sock *vsk)
367{
368 spin_lock_bh(&vsock_table_lock);
369 if (__vsock_in_connected_table(vsk))
370 __vsock_remove_connected(vsk);
371 spin_unlock_bh(&vsock_table_lock);
372}
373EXPORT_SYMBOL_GPL(vsock_remove_connected);
374
375/* Find a bound socket, filtering by namespace and namespace mode.
376 *
377 * Use this in transports that are namespace-aware and can provide the
378 * network namespace context.
379 */
380struct sock *vsock_find_bound_socket_net(struct sockaddr_vm *addr,
381 struct net *net)
382{
383 struct sock *sk;
384
385 spin_lock_bh(&vsock_table_lock);
386 sk = __vsock_find_bound_socket_net(addr, net);
387 if (sk)
388 sock_hold(sk);
389
390 spin_unlock_bh(&vsock_table_lock);
391
392 return sk;
393}
394EXPORT_SYMBOL_GPL(vsock_find_bound_socket_net);
395
396/* Find a bound socket without namespace filtering.
397 *
398 * Use this in transports that lack namespace context. All sockets are
399 * treated as if in global mode.
400 */
401struct sock *vsock_find_bound_socket(struct sockaddr_vm *addr)
402{
403 return vsock_find_bound_socket_net(addr, NULL);
404}
405EXPORT_SYMBOL_GPL(vsock_find_bound_socket);
406
407/* Find a connected socket, filtering by namespace and namespace mode.
408 *
409 * Use this in transports that are namespace-aware and can provide the
410 * network namespace context.
411 */
412struct sock *vsock_find_connected_socket_net(struct sockaddr_vm *src,
413 struct sockaddr_vm *dst,
414 struct net *net)
415{
416 struct sock *sk;
417
418 spin_lock_bh(&vsock_table_lock);
419 sk = __vsock_find_connected_socket_net(src, dst, net);
420 if (sk)
421 sock_hold(sk);
422
423 spin_unlock_bh(&vsock_table_lock);
424
425 return sk;
426}
427EXPORT_SYMBOL_GPL(vsock_find_connected_socket_net);
428
429/* Find a connected socket without namespace filtering.
430 *
431 * Use this in transports that lack namespace context. All sockets are
432 * treated as if in global mode.
433 */
434struct sock *vsock_find_connected_socket(struct sockaddr_vm *src,
435 struct sockaddr_vm *dst)
436{
437 return vsock_find_connected_socket_net(src, dst, NULL);
438}
439EXPORT_SYMBOL_GPL(vsock_find_connected_socket);
440
441/**
442 * vsock_check_source - validate a packet source against a socket peer
443 * @vsk: socket receiving the packet
444 * @transport: transport receiving the packet
445 * @src: source address from the packet
446 *
447 * Return: true if the packet arrived on the socket's assigned transport and
448 * its source matches the stored peer. Loopback packets are generated
449 * internally and always use the local CID as their source, including
450 * connections using a valid CID alias.
451 *
452 * The caller must hold the socket lock and must not call this for listening
453 * sockets, which accept packets from any source and have no assigned
454 * transport.
455 */
456bool vsock_check_source(const struct vsock_sock *vsk,
457 const struct vsock_transport *transport,
458 const struct sockaddr_vm *src)
459{
460 if (vsk->transport != transport)
461 return false;
462
463 if (src->svm_port != vsk->remote_addr.svm_port)
464 return false;
465
466 if (src->svm_cid == vsk->remote_addr.svm_cid)
467 return true;
468
469 return transport->get_local_cid() == VMADDR_CID_LOCAL;
470}
471EXPORT_SYMBOL_GPL(vsock_check_source);
472
473void vsock_remove_sock(struct vsock_sock *vsk)
474{
475 /* Transport reassignment must not remove the binding. */
476 if (sock_flag(sk_vsock(vsk), SOCK_DEAD))
477 vsock_remove_bound(vsk);
478
479 vsock_remove_connected(vsk);
480}
481EXPORT_SYMBOL_GPL(vsock_remove_sock);
482
483void vsock_for_each_connected_socket(struct vsock_transport *transport,
484 void (*fn)(struct sock *sk))
485{
486 int i;
487
488 spin_lock_bh(&vsock_table_lock);
489
490 for (i = 0; i < ARRAY_SIZE(vsock_connected_table); i++) {
491 struct vsock_sock *vsk;
492 list_for_each_entry(vsk, &vsock_connected_table[i],
493 connected_table) {
494 if (vsk->transport != transport)
495 continue;
496
497 fn(sk_vsock(vsk));
498 }
499 }
500
501 spin_unlock_bh(&vsock_table_lock);
502}
503EXPORT_SYMBOL_GPL(vsock_for_each_connected_socket);
504
505void vsock_add_pending(struct sock *listener, struct sock *pending)
506{
507 struct vsock_sock *vlistener;
508 struct vsock_sock *vpending;
509
510 vlistener = vsock_sk(listener);
511 vpending = vsock_sk(pending);
512
513 sock_hold(pending);
514 sock_hold(listener);
515 list_add_tail(&vpending->pending_links, &vlistener->pending_links);
516 sk_acceptq_added(listener);
517}
518EXPORT_SYMBOL_GPL(vsock_add_pending);
519
520void vsock_remove_pending(struct sock *listener, struct sock *pending)
521{
522 struct vsock_sock *vpending = vsock_sk(pending);
523
524 list_del_init(&vpending->pending_links);
525 sock_put(listener);
526 sock_put(pending);
527 sk_acceptq_removed(listener);
528}
529EXPORT_SYMBOL_GPL(vsock_remove_pending);
530
531void vsock_pending_to_accept(struct sock *listener, struct sock *pending)
532{
533 struct vsock_sock *vpending = vsock_sk(pending);
534 struct vsock_sock *vlistener = vsock_sk(listener);
535
536 list_del_init(&vpending->pending_links);
537 list_add_tail(&vpending->accept_queue, &vlistener->accept_queue);
538}
539EXPORT_SYMBOL_GPL(vsock_pending_to_accept);
540
541void vsock_enqueue_accept(struct sock *listener, struct sock *connected)
542{
543 struct vsock_sock *vlistener;
544 struct vsock_sock *vconnected;
545
546 vlistener = vsock_sk(listener);
547 vconnected = vsock_sk(connected);
548
549 sock_hold(connected);
550 sock_hold(listener);
551 list_add_tail(&vconnected->accept_queue, &vlistener->accept_queue);
552 sk_acceptq_added(listener);
553}
554EXPORT_SYMBOL_GPL(vsock_enqueue_accept);
555
556static bool vsock_use_local_transport(unsigned int remote_cid)
557{
558 lockdep_assert_held(&vsock_register_mutex);
559
560 if (!transport_local)
561 return false;
562
563 if (remote_cid == VMADDR_CID_LOCAL)
564 return true;
565
566 if (transport_g2h) {
567 return remote_cid == transport_g2h->get_local_cid();
568 } else {
569 return remote_cid == VMADDR_CID_HOST;
570 }
571}
572
573static void vsock_deassign_transport(struct vsock_sock *vsk)
574{
575 if (!vsk->transport)
576 return;
577
578 vsk->transport->destruct(vsk);
579 module_put(vsk->transport->module);
580 vsk->transport = NULL;
581}
582
583/* Assign a transport to a socket and call the .init transport callback.
584 *
585 * Note: for connection oriented socket this must be called when vsk->remote_addr
586 * is set (e.g. during the connect() or when a connection request on a listener
587 * socket is received).
588 * The vsk->remote_addr is used to decide which transport to use:
589 * - remote CID == VMADDR_CID_LOCAL or g2h->local_cid or VMADDR_CID_HOST if
590 * g2h is not loaded, will use local transport;
591 * - remote CID <= VMADDR_CID_HOST or remote flags field includes
592 * VMADDR_FLAG_TO_HOST, will use guest->host transport;
593 * - remote CID > VMADDR_CID_HOST and h2g is loaded and h2g claims that CID,
594 * will use host->guest transport;
595 * - h2g not loaded or h2g does not claim that CID and g2h claims the CID via
596 * has_remote_cid, will use guest->host transport (when g2h_fallback=1)
597 * - anything else goes to h2g or returns -ENODEV if no h2g is available
598 */
599int vsock_assign_transport(struct vsock_sock *vsk, struct vsock_sock *psk)
600{
601 const struct vsock_transport *new_transport;
602 struct sock *sk = sk_vsock(vsk);
603 unsigned int remote_cid = vsk->remote_addr.svm_cid;
604 __u8 remote_flags;
605 int ret;
606
607 /* If the packet is coming with the source and destination CIDs higher
608 * than VMADDR_CID_HOST, then a vsock channel where all the packets are
609 * forwarded to the host should be established. Then the host will
610 * need to forward the packets to the guest.
611 *
612 * The flag is set on the (listen) receive path (psk is not NULL). On
613 * the connect path the flag can be set by the user space application.
614 */
615 if (psk && vsk->local_addr.svm_cid > VMADDR_CID_HOST &&
616 vsk->remote_addr.svm_cid > VMADDR_CID_HOST)
617 vsk->remote_addr.svm_flags |= VMADDR_FLAG_TO_HOST;
618
619 remote_flags = vsk->remote_addr.svm_flags;
620
621 mutex_lock(&vsock_register_mutex);
622
623 switch (sk->sk_type) {
624 case SOCK_DGRAM:
625 new_transport = transport_dgram;
626 break;
627 case SOCK_STREAM:
628 case SOCK_SEQPACKET:
629 if (vsock_use_local_transport(remote_cid))
630 new_transport = transport_local;
631 else if (remote_cid <= VMADDR_CID_HOST ||
632 (remote_flags & VMADDR_FLAG_TO_HOST))
633 new_transport = transport_g2h;
634 else if (transport_h2g &&
635 (!transport_h2g->has_remote_cid ||
636 transport_h2g->has_remote_cid(vsk, remote_cid)))
637 new_transport = transport_h2g;
638 else if (sock_net(sk)->vsock.g2h_fallback &&
639 transport_g2h && transport_g2h->has_remote_cid &&
640 transport_g2h->has_remote_cid(vsk, remote_cid)) {
641 vsk->remote_addr.svm_flags |= VMADDR_FLAG_TO_HOST;
642 new_transport = transport_g2h;
643 } else {
644 new_transport = transport_h2g;
645 }
646 break;
647 default:
648 ret = -ESOCKTNOSUPPORT;
649 goto err;
650 }
651
652 if (vsk->transport && vsk->transport == new_transport) {
653 ret = 0;
654 goto err;
655 }
656
657 /* We increase the module refcnt to prevent the transport unloading
658 * while there are open sockets assigned to it.
659 */
660 if (!new_transport || !try_module_get(new_transport->module)) {
661 ret = -ENODEV;
662 goto err;
663 }
664
665 /* It's safe to release the mutex after a successful try_module_get().
666 * Whichever transport `new_transport` points at, it won't go away until
667 * the last module_put() below or in vsock_deassign_transport().
668 */
669 mutex_unlock(&vsock_register_mutex);
670
671 if (vsk->transport) {
672 /* transport->release() must be called with sock lock acquired.
673 * This path can only be taken during vsock_connect(), where we
674 * have already held the sock lock. In the other cases, this
675 * function is called on a new socket which is not assigned to
676 * any transport.
677 */
678 vsk->transport->release(vsk);
679 vsock_deassign_transport(vsk);
680
681 /* transport's release() and destruct() can touch some socket
682 * state, since we are reassigning the socket to a new transport
683 * during vsock_connect(), let's reset these fields to have a
684 * clean state.
685 */
686 sock_reset_flag(sk, SOCK_DONE);
687 sk->sk_state = TCP_CLOSE;
688 WRITE_ONCE(vsk->peer_shutdown, 0);
689 }
690
691 if (sk->sk_type == SOCK_SEQPACKET) {
692 if (!new_transport->seqpacket_allow ||
693 !new_transport->seqpacket_allow(vsk, remote_cid)) {
694 module_put(new_transport->module);
695 return -ESOCKTNOSUPPORT;
696 }
697 }
698
699 ret = new_transport->init(vsk, psk);
700 if (ret) {
701 module_put(new_transport->module);
702 return ret;
703 }
704
705 vsk->transport = new_transport;
706
707 return 0;
708err:
709 mutex_unlock(&vsock_register_mutex);
710 return ret;
711}
712EXPORT_SYMBOL_GPL(vsock_assign_transport);
713
714/*
715 * Provide safe access to static transport_{h2g,g2h,dgram,local} callbacks.
716 * Otherwise we may race with module removal. Do not use on `vsk->transport`.
717 */
718static u32 vsock_registered_transport_cid(const struct vsock_transport **transport)
719{
720 u32 cid = VMADDR_CID_ANY;
721
722 mutex_lock(&vsock_register_mutex);
723 if (*transport)
724 cid = (*transport)->get_local_cid();
725 mutex_unlock(&vsock_register_mutex);
726
727 return cid;
728}
729
730bool vsock_find_cid(unsigned int cid)
731{
732 if (cid == vsock_registered_transport_cid(&transport_g2h))
733 return true;
734
735 if (transport_h2g && cid == VMADDR_CID_HOST)
736 return true;
737
738 if (transport_local && cid == VMADDR_CID_LOCAL)
739 return true;
740
741 return false;
742}
743EXPORT_SYMBOL_GPL(vsock_find_cid);
744
745static struct sock *vsock_dequeue_accept(struct sock *listener)
746{
747 struct vsock_sock *vlistener;
748 struct vsock_sock *vconnected;
749
750 vlistener = vsock_sk(listener);
751
752 if (list_empty(&vlistener->accept_queue))
753 return NULL;
754
755 vconnected = list_entry(vlistener->accept_queue.next,
756 struct vsock_sock, accept_queue);
757
758 list_del_init(&vconnected->accept_queue);
759 sock_put(listener);
760 /* The caller will need a reference on the connected socket so we let
761 * it call sock_put().
762 */
763
764 return sk_vsock(vconnected);
765}
766
767static bool vsock_is_accept_queue_empty(struct sock *sk)
768{
769 struct vsock_sock *vsk = vsock_sk(sk);
770 return list_empty(&vsk->accept_queue);
771}
772
773static bool vsock_is_pending(struct sock *sk)
774{
775 struct vsock_sock *vsk = vsock_sk(sk);
776 return !list_empty(&vsk->pending_links);
777}
778
779static int vsock_send_shutdown(struct sock *sk, int mode)
780{
781 struct vsock_sock *vsk = vsock_sk(sk);
782
783 if (!vsk->transport)
784 return -ENODEV;
785
786 return vsk->transport->shutdown(vsk, mode);
787}
788
789static void vsock_pending_work(struct work_struct *work)
790{
791 struct sock *sk;
792 struct sock *listener;
793 struct vsock_sock *vsk;
794 bool cleanup;
795
796 vsk = container_of(work, struct vsock_sock, pending_work.work);
797 sk = sk_vsock(vsk);
798 listener = vsk->listener;
799 cleanup = true;
800
801 lock_sock(listener);
802 lock_sock_nested(sk, SINGLE_DEPTH_NESTING);
803
804 if (vsock_is_pending(sk)) {
805 vsock_remove_pending(listener, sk);
806 } else {
807 /* We are not on the pending list so we must have been accepted
808 * by our user process. We just need to drop our references to
809 * the sockets and be on our way.
810 */
811 cleanup = false;
812 goto out;
813 }
814
815 /* We need to remove ourself from the global connected sockets list so
816 * incoming packets can't find this socket, and to reduce the reference
817 * count.
818 */
819 vsock_remove_connected(vsk);
820
821 sk->sk_state = TCP_CLOSE;
822
823out:
824 release_sock(sk);
825 release_sock(listener);
826 if (cleanup)
827 sock_put(sk);
828
829 sock_put(sk);
830 sock_put(listener);
831}
832
833/**** SOCKET OPERATIONS ****/
834
835static int __vsock_bind_connectible(struct vsock_sock *vsk,
836 struct sockaddr_vm *addr)
837{
838 struct net *net = sock_net(sk_vsock(vsk));
839 struct sockaddr_vm new_addr;
840
841 if (!net->vsock.port)
842 net->vsock.port = get_random_u32_above(LAST_RESERVED_PORT);
843
844 vsock_addr_init(&new_addr, addr->svm_cid, addr->svm_port);
845
846 if (addr->svm_port == VMADDR_PORT_ANY) {
847 bool found = false;
848 unsigned int i;
849
850 for (i = 0; i < MAX_PORT_RETRIES; i++) {
851 if (net->vsock.port == VMADDR_PORT_ANY ||
852 net->vsock.port <= LAST_RESERVED_PORT)
853 net->vsock.port = LAST_RESERVED_PORT + 1;
854
855 new_addr.svm_port = net->vsock.port++;
856
857 if (!__vsock_find_bound_socket_net(&new_addr, net)) {
858 found = true;
859 break;
860 }
861 }
862
863 if (!found)
864 return -EADDRNOTAVAIL;
865 } else {
866 /* If port is in reserved range, ensure caller
867 * has necessary privileges.
868 */
869 if (addr->svm_port <= LAST_RESERVED_PORT &&
870 !capable(CAP_NET_BIND_SERVICE)) {
871 return -EACCES;
872 }
873
874 if (__vsock_find_bound_socket_net(&new_addr, net))
875 return -EADDRINUSE;
876 }
877
878 vsock_addr_init(&vsk->local_addr, new_addr.svm_cid, new_addr.svm_port);
879
880 /* Remove connection oriented sockets from the unbound list and add them
881 * to the hash table for easy lookup by its address. The unbound list
882 * is simply an extra entry at the end of the hash table, a trick used
883 * by AF_UNIX.
884 */
885 __vsock_remove_bound(vsk);
886 __vsock_insert_bound(vsock_bound_sockets(&vsk->local_addr), vsk);
887
888 return 0;
889}
890
891static int __vsock_bind_dgram(struct vsock_sock *vsk,
892 struct sockaddr_vm *addr)
893{
894 return vsk->transport->dgram_bind(vsk, addr);
895}
896
897static int __vsock_bind(struct sock *sk, struct sockaddr_vm *addr)
898{
899 struct vsock_sock *vsk = vsock_sk(sk);
900 int retval;
901
902 /* First ensure this socket isn't already bound. */
903 if (vsock_addr_bound(&vsk->local_addr))
904 return -EINVAL;
905
906 /* Now bind to the provided address or select appropriate values if
907 * none are provided (VMADDR_CID_ANY and VMADDR_PORT_ANY). Note that
908 * like AF_INET prevents binding to a non-local IP address (in most
909 * cases), we only allow binding to a local CID.
910 */
911 if (addr->svm_cid != VMADDR_CID_ANY && !vsock_find_cid(addr->svm_cid))
912 return -EADDRNOTAVAIL;
913
914 switch (sk->sk_socket->type) {
915 case SOCK_STREAM:
916 case SOCK_SEQPACKET:
917 spin_lock_bh(&vsock_table_lock);
918 retval = __vsock_bind_connectible(vsk, addr);
919 spin_unlock_bh(&vsock_table_lock);
920 break;
921
922 case SOCK_DGRAM:
923 retval = __vsock_bind_dgram(vsk, addr);
924 break;
925
926 default:
927 retval = -EINVAL;
928 break;
929 }
930
931 return retval;
932}
933
934static void vsock_connect_timeout(struct work_struct *work);
935
936static struct sock *__vsock_create(struct net *net,
937 struct socket *sock,
938 struct sock *parent,
939 gfp_t priority,
940 unsigned short type,
941 int kern)
942{
943 struct sock *sk;
944 struct vsock_sock *psk;
945 struct vsock_sock *vsk;
946
947 sk = sk_alloc(net, AF_VSOCK, priority, &vsock_proto, kern);
948 if (!sk)
949 return NULL;
950
951 sock_init_data(sock, sk);
952
953 /* sk->sk_type is normally set in sock_init_data, but only if sock is
954 * non-NULL. We make sure that our sockets always have a type by
955 * setting it here if needed.
956 */
957 if (!sock)
958 sk->sk_type = type;
959
960 vsk = vsock_sk(sk);
961 vsock_addr_init(&vsk->local_addr, VMADDR_CID_ANY, VMADDR_PORT_ANY);
962 vsock_addr_init(&vsk->remote_addr, VMADDR_CID_ANY, VMADDR_PORT_ANY);
963
964 sk->sk_destruct = vsock_sk_destruct;
965 sk->sk_backlog_rcv = vsock_queue_rcv_skb;
966 sock_reset_flag(sk, SOCK_DONE);
967
968 INIT_LIST_HEAD(&vsk->bound_table);
969 INIT_LIST_HEAD(&vsk->connected_table);
970 vsk->listener = NULL;
971 INIT_LIST_HEAD(&vsk->pending_links);
972 INIT_LIST_HEAD(&vsk->accept_queue);
973 vsk->sent_request = false;
974 vsk->ignore_connecting_rst = false;
975 WRITE_ONCE(vsk->peer_shutdown, 0);
976 INIT_DELAYED_WORK(&vsk->connect_work, vsock_connect_timeout);
977 INIT_DELAYED_WORK(&vsk->pending_work, vsock_pending_work);
978
979 psk = parent ? vsock_sk(parent) : NULL;
980 if (parent) {
981 vsk->trusted = psk->trusted;
982 vsk->owner = get_cred(psk->owner);
983 vsk->connect_timeout = psk->connect_timeout;
984 vsk->buffer_size = psk->buffer_size;
985 vsk->buffer_min_size = psk->buffer_min_size;
986 vsk->buffer_max_size = psk->buffer_max_size;
987 security_sk_clone(parent, sk);
988 } else {
989 vsk->trusted = ns_capable_noaudit(&init_user_ns, CAP_NET_ADMIN);
990 vsk->owner = get_current_cred();
991 vsk->connect_timeout = VSOCK_DEFAULT_CONNECT_TIMEOUT;
992 vsk->buffer_size = VSOCK_DEFAULT_BUFFER_SIZE;
993 vsk->buffer_min_size = VSOCK_DEFAULT_BUFFER_MIN_SIZE;
994 vsk->buffer_max_size = VSOCK_DEFAULT_BUFFER_MAX_SIZE;
995 }
996
997 return sk;
998}
999
1000static bool sock_type_connectible(u16 type)
1001{
1002 return (type == SOCK_STREAM) || (type == SOCK_SEQPACKET);
1003}
1004
1005static void __vsock_release(struct sock *sk, int level)
1006{
1007 struct vsock_sock *vsk;
1008 struct sock *pending;
1009
1010 vsk = vsock_sk(sk);
1011 pending = NULL; /* Compiler warning. */
1012
1013 /* When "level" is SINGLE_DEPTH_NESTING, use the nested
1014 * version to avoid the warning "possible recursive locking
1015 * detected". When "level" is 0, lock_sock_nested(sk, level)
1016 * is the same as lock_sock(sk).
1017 */
1018 lock_sock_nested(sk, level);
1019
1020 /* Indicate to vsock_remove_sock() that the socket is being released and
1021 * can be removed from the bound_table. Unlike transport reassignment
1022 * case, where the socket must remain bound despite vsock_remove_sock()
1023 * being called from the transport release() callback.
1024 */
1025 sock_set_flag(sk, SOCK_DEAD);
1026
1027 if (vsk->transport)
1028 vsk->transport->release(vsk);
1029 else if (sock_type_connectible(sk->sk_type))
1030 vsock_remove_sock(vsk);
1031
1032 sock_orphan(sk);
1033 sk->sk_shutdown = SHUTDOWN_MASK;
1034
1035 skb_queue_purge(&sk->sk_receive_queue);
1036
1037 /* Clean up any sockets that never were accepted. */
1038 while ((pending = vsock_dequeue_accept(sk)) != NULL) {
1039 __vsock_release(pending, SINGLE_DEPTH_NESTING);
1040 sock_put(pending);
1041 }
1042
1043 release_sock(sk);
1044 sock_put(sk);
1045}
1046
1047static void vsock_sk_destruct(struct sock *sk)
1048{
1049 struct vsock_sock *vsk = vsock_sk(sk);
1050
1051 /* Socket buffers may remain in the queue in VMCI datagram mode */
1052 __skb_queue_purge(&sk->sk_receive_queue);
1053 /* Flush MSG_ZEROCOPY leftovers. */
1054 __skb_queue_purge(&sk->sk_error_queue);
1055
1056 vsock_deassign_transport(vsk);
1057
1058 /* When clearing these addresses, there's no need to set the family and
1059 * possibly register the address family with the kernel.
1060 */
1061 vsock_addr_init(&vsk->local_addr, VMADDR_CID_ANY, VMADDR_PORT_ANY);
1062 vsock_addr_init(&vsk->remote_addr, VMADDR_CID_ANY, VMADDR_PORT_ANY);
1063
1064 put_cred(vsk->owner);
1065}
1066
1067static int vsock_queue_rcv_skb(struct sock *sk, struct sk_buff *skb)
1068{
1069 int err;
1070
1071 err = sock_queue_rcv_skb(sk, skb);
1072 if (err)
1073 kfree_skb(skb);
1074
1075 return err;
1076}
1077
1078struct sock *vsock_create_connected(struct sock *parent)
1079{
1080 return __vsock_create(sock_net(parent), NULL, parent, GFP_KERNEL,
1081 parent->sk_type, 0);
1082}
1083EXPORT_SYMBOL_GPL(vsock_create_connected);
1084
1085s64 vsock_stream_has_data(struct vsock_sock *vsk)
1086{
1087 if (WARN_ON(!vsk->transport))
1088 return 0;
1089
1090 return vsk->transport->stream_has_data(vsk);
1091}
1092EXPORT_SYMBOL_GPL(vsock_stream_has_data);
1093
1094s64 vsock_connectible_has_data(struct vsock_sock *vsk)
1095{
1096 struct sock *sk = sk_vsock(vsk);
1097
1098 if (WARN_ON(!vsk->transport))
1099 return 0;
1100
1101 if (sk->sk_type == SOCK_SEQPACKET)
1102 return vsk->transport->seqpacket_has_data(vsk);
1103 else
1104 return vsock_stream_has_data(vsk);
1105}
1106EXPORT_SYMBOL_GPL(vsock_connectible_has_data);
1107
1108s64 vsock_stream_has_space(struct vsock_sock *vsk)
1109{
1110 if (WARN_ON(!vsk->transport))
1111 return 0;
1112
1113 return vsk->transport->stream_has_space(vsk);
1114}
1115EXPORT_SYMBOL_GPL(vsock_stream_has_space);
1116
1117void vsock_data_ready(struct sock *sk)
1118{
1119 struct vsock_sock *vsk = vsock_sk(sk);
1120
1121 if (vsock_stream_has_data(vsk) >= sk->sk_rcvlowat ||
1122 sock_flag(sk, SOCK_DONE))
1123 sk->sk_data_ready(sk);
1124}
1125EXPORT_SYMBOL_GPL(vsock_data_ready);
1126
1127/* Dummy callback required by sockmap.
1128 * See unconditional call of saved_close() in sock_map_close().
1129 */
1130static void vsock_close(struct sock *sk, long timeout)
1131{
1132}
1133
1134static int vsock_release(struct socket *sock)
1135{
1136 struct sock *sk = sock->sk;
1137
1138 if (!sk)
1139 return 0;
1140
1141 sk->sk_prot->close(sk, 0);
1142 __vsock_release(sk, 0);
1143 sock->sk = NULL;
1144 sock->state = SS_FREE;
1145
1146 return 0;
1147}
1148
1149static int
1150vsock_bind(struct socket *sock, struct sockaddr_unsized *addr, int addr_len)
1151{
1152 int err;
1153 struct sock *sk;
1154 struct sockaddr_vm *vm_addr;
1155
1156 sk = sock->sk;
1157
1158 if (vsock_addr_cast(addr, addr_len, &vm_addr) != 0)
1159 return -EINVAL;
1160
1161 lock_sock(sk);
1162 err = __vsock_bind(sk, vm_addr);
1163 release_sock(sk);
1164
1165 return err;
1166}
1167
1168static int vsock_getname(struct socket *sock,
1169 struct sockaddr *addr, int peer)
1170{
1171 int err;
1172 struct sock *sk;
1173 struct vsock_sock *vsk;
1174 struct sockaddr_vm *vm_addr;
1175
1176 sk = sock->sk;
1177 vsk = vsock_sk(sk);
1178 err = 0;
1179
1180 lock_sock(sk);
1181
1182 if (peer) {
1183 if (sock->state != SS_CONNECTED) {
1184 err = -ENOTCONN;
1185 goto out;
1186 }
1187 vm_addr = &vsk->remote_addr;
1188 } else {
1189 vm_addr = &vsk->local_addr;
1190 }
1191
1192 BUILD_BUG_ON(sizeof(*vm_addr) > sizeof(struct sockaddr_storage));
1193 memcpy(addr, vm_addr, sizeof(*vm_addr));
1194 err = sizeof(*vm_addr);
1195
1196out:
1197 release_sock(sk);
1198 return err;
1199}
1200
1201void vsock_linger(struct sock *sk)
1202{
1203 DEFINE_WAIT_FUNC(wait, woken_wake_function);
1204 ssize_t (*unsent)(struct vsock_sock *vsk);
1205 struct vsock_sock *vsk = vsock_sk(sk);
1206 long timeout;
1207
1208 if (!sock_flag(sk, SOCK_LINGER))
1209 return;
1210
1211 timeout = sk->sk_lingertime;
1212 if (!timeout)
1213 return;
1214
1215 /* Transports must implement `unsent_bytes` if they want to support
1216 * SOCK_LINGER through `vsock_linger()` since we use it to check when
1217 * the socket can be closed.
1218 */
1219 unsent = vsk->transport->unsent_bytes;
1220 if (!unsent)
1221 return;
1222
1223 add_wait_queue(sk_sleep(sk), &wait);
1224
1225 do {
1226 if (sk_wait_event(sk, &timeout, unsent(vsk) == 0, &wait))
1227 break;
1228 } while (!signal_pending(current) && timeout);
1229
1230 remove_wait_queue(sk_sleep(sk), &wait);
1231}
1232EXPORT_SYMBOL_GPL(vsock_linger);
1233
1234static int vsock_shutdown(struct socket *sock, int mode)
1235{
1236 int err;
1237 struct sock *sk;
1238
1239 /* User level uses SHUT_RD (0) and SHUT_WR (1), but the kernel uses
1240 * RCV_SHUTDOWN (1) and SEND_SHUTDOWN (2), so we must increment mode
1241 * here like the other address families do. Note also that the
1242 * increment makes SHUT_RDWR (2) into RCV_SHUTDOWN | SEND_SHUTDOWN (3),
1243 * which is what we want.
1244 */
1245 mode++;
1246
1247 if ((mode & ~SHUTDOWN_MASK) || !mode)
1248 return -EINVAL;
1249
1250 /* If this is a connection oriented socket and it is not connected then
1251 * bail out immediately. If it is a DGRAM socket then we must first
1252 * kick the socket so that it wakes up from any sleeping calls, for
1253 * example recv(), and then afterwards return the error.
1254 */
1255
1256 sk = sock->sk;
1257
1258 lock_sock(sk);
1259 if (sock->state == SS_UNCONNECTED) {
1260 err = -ENOTCONN;
1261 if (sock_type_connectible(sk->sk_type))
1262 goto out;
1263 } else {
1264 sock->state = SS_DISCONNECTING;
1265 err = 0;
1266 }
1267
1268 /* Receive and send shutdowns are treated alike. */
1269 mode = mode & (RCV_SHUTDOWN | SEND_SHUTDOWN);
1270 if (mode) {
1271 sk->sk_shutdown |= mode;
1272 sk->sk_state_change(sk);
1273
1274 if (sock_type_connectible(sk->sk_type)) {
1275 sock_reset_flag(sk, SOCK_DONE);
1276 vsock_send_shutdown(sk, mode);
1277 }
1278 }
1279
1280out:
1281 release_sock(sk);
1282 return err;
1283}
1284
1285static __poll_t vsock_poll_shutdown(struct sock *sk, u32 peer_shutdown)
1286{
1287 __poll_t mask = 0;
1288
1289 /* INET sockets treat local write shutdown and peer write shutdown as a
1290 * case of EPOLLHUP set.
1291 */
1292 if (sk->sk_shutdown == SHUTDOWN_MASK ||
1293 ((sk->sk_shutdown & SEND_SHUTDOWN) &&
1294 (peer_shutdown & SEND_SHUTDOWN)))
1295 mask |= EPOLLHUP;
1296
1297 if (sk->sk_shutdown & RCV_SHUTDOWN ||
1298 peer_shutdown & SEND_SHUTDOWN)
1299 mask |= EPOLLRDHUP;
1300
1301 return mask;
1302}
1303
1304static __poll_t vsock_poll(struct file *file, struct socket *sock,
1305 poll_table *wait)
1306{
1307 struct sock *sk;
1308 __poll_t mask;
1309 struct vsock_sock *vsk;
1310
1311 sk = sock->sk;
1312 vsk = vsock_sk(sk);
1313
1314 poll_wait(file, sk_sleep(sk), wait);
1315 mask = 0;
1316
1317 if (sk->sk_err || !skb_queue_empty_lockless(&sk->sk_error_queue))
1318 /* Signify that there has been an error on this socket. */
1319 mask |= EPOLLERR;
1320
1321 if (sk_is_readable(sk))
1322 mask |= EPOLLIN | EPOLLRDNORM;
1323
1324 if (sock->type == SOCK_DGRAM) {
1325 u32 peer_shutdown = READ_ONCE(vsk->peer_shutdown);
1326
1327 /* DGRAM sockets do not take lock_sock() in poll(), so use one
1328 * lockless snapshot for all shutdown-derived mask bits.
1329 */
1330 mask |= vsock_poll_shutdown(sk, peer_shutdown);
1331
1332 /* For datagram sockets we can read if there is something in
1333 * the queue and write as long as the socket isn't shutdown for
1334 * sending.
1335 */
1336 if (!skb_queue_empty_lockless(&sk->sk_receive_queue) ||
1337 (sk->sk_shutdown & RCV_SHUTDOWN)) {
1338 mask |= EPOLLIN | EPOLLRDNORM;
1339 }
1340
1341 if (!(sk->sk_shutdown & SEND_SHUTDOWN))
1342 mask |= EPOLLOUT | EPOLLWRNORM | EPOLLWRBAND;
1343
1344 } else if (sock_type_connectible(sk->sk_type)) {
1345 const struct vsock_transport *transport;
1346 u32 peer_shutdown;
1347
1348 lock_sock(sk);
1349
1350 transport = vsk->transport;
1351
1352 /* Listening sockets that have connections in their accept
1353 * queue can be read.
1354 */
1355 if (sk->sk_state == TCP_LISTEN
1356 && !vsock_is_accept_queue_empty(sk))
1357 mask |= EPOLLIN | EPOLLRDNORM;
1358
1359 /* If there is something in the queue then we can read. */
1360 if (transport && transport->stream_is_active(vsk) &&
1361 !(sk->sk_shutdown & RCV_SHUTDOWN)) {
1362 bool data_ready_now = false;
1363 int target = sock_rcvlowat(sk, 0, INT_MAX);
1364 int ret = transport->notify_poll_in(
1365 vsk, target, &data_ready_now);
1366 if (ret < 0) {
1367 mask |= EPOLLERR;
1368 } else {
1369 if (data_ready_now)
1370 mask |= EPOLLIN | EPOLLRDNORM;
1371
1372 }
1373 }
1374
1375 /* Sockets whose connections have been closed, reset, or
1376 * terminated should also be considered read, and we check the
1377 * shutdown flag for that.
1378 */
1379 peer_shutdown = READ_ONCE(vsk->peer_shutdown);
1380 mask |= vsock_poll_shutdown(sk, peer_shutdown);
1381 if (sk->sk_shutdown & RCV_SHUTDOWN ||
1382 peer_shutdown & SEND_SHUTDOWN) {
1383 mask |= EPOLLIN | EPOLLRDNORM;
1384 }
1385
1386 /* Connected sockets that can produce data can be written. */
1387 if (transport && sk->sk_state == TCP_ESTABLISHED) {
1388 if (!(sk->sk_shutdown & SEND_SHUTDOWN)) {
1389 bool space_avail_now = false;
1390 int ret = transport->notify_poll_out(
1391 vsk, 1, &space_avail_now);
1392 if (ret < 0) {
1393 mask |= EPOLLERR;
1394 } else {
1395 if (space_avail_now)
1396 /* Remove EPOLLWRBAND since INET
1397 * sockets are not setting it.
1398 */
1399 mask |= EPOLLOUT | EPOLLWRNORM;
1400
1401 }
1402 }
1403 }
1404
1405 /* Simulate INET socket poll behaviors, which sets
1406 * EPOLLOUT|EPOLLWRNORM when peer is closed and nothing to read,
1407 * but local send is not shutdown.
1408 */
1409 if (sk->sk_state == TCP_CLOSE || sk->sk_state == TCP_CLOSING) {
1410 if (!(sk->sk_shutdown & SEND_SHUTDOWN))
1411 mask |= EPOLLOUT | EPOLLWRNORM;
1412
1413 }
1414
1415 release_sock(sk);
1416 }
1417
1418 return mask;
1419}
1420
1421static int vsock_read_skb(struct sock *sk, skb_read_actor_t read_actor)
1422{
1423 struct vsock_sock *vsk = vsock_sk(sk);
1424
1425 if (WARN_ON_ONCE(!vsk->transport))
1426 return -ENODEV;
1427
1428 return vsk->transport->read_skb(vsk, read_actor);
1429}
1430
1431static int vsock_dgram_sendmsg(struct socket *sock, struct msghdr *msg,
1432 size_t len)
1433{
1434 int err;
1435 struct sock *sk;
1436 struct vsock_sock *vsk;
1437 struct sockaddr_vm *remote_addr;
1438 const struct vsock_transport *transport;
1439
1440 if (msg->msg_flags & MSG_OOB)
1441 return -EOPNOTSUPP;
1442
1443 /* For now, MSG_DONTWAIT is always assumed... */
1444 err = 0;
1445 sk = sock->sk;
1446 vsk = vsock_sk(sk);
1447
1448 lock_sock(sk);
1449
1450 transport = vsk->transport;
1451
1452 err = vsock_auto_bind(vsk);
1453 if (err)
1454 goto out;
1455
1456
1457 /* If the provided message contains an address, use that. Otherwise
1458 * fall back on the socket's remote handle (if it has been connected).
1459 */
1460 if (msg->msg_name &&
1461 vsock_addr_cast(msg->msg_name, msg->msg_namelen,
1462 &remote_addr) == 0) {
1463 /* Ensure this address is of the right type and is a valid
1464 * destination.
1465 */
1466
1467 if (remote_addr->svm_cid == VMADDR_CID_ANY)
1468 remote_addr->svm_cid = transport->get_local_cid();
1469
1470 if (!vsock_addr_bound(remote_addr)) {
1471 err = -EINVAL;
1472 goto out;
1473 }
1474 } else if (sock->state == SS_CONNECTED) {
1475 remote_addr = &vsk->remote_addr;
1476
1477 if (remote_addr->svm_cid == VMADDR_CID_ANY)
1478 remote_addr->svm_cid = transport->get_local_cid();
1479
1480 /* XXX Should connect() or this function ensure remote_addr is
1481 * bound?
1482 */
1483 if (!vsock_addr_bound(&vsk->remote_addr)) {
1484 err = -EINVAL;
1485 goto out;
1486 }
1487 } else {
1488 err = -EINVAL;
1489 goto out;
1490 }
1491
1492 if (!transport->dgram_allow(vsk, remote_addr->svm_cid,
1493 remote_addr->svm_port)) {
1494 err = -EINVAL;
1495 goto out;
1496 }
1497
1498 err = transport->dgram_enqueue(vsk, remote_addr, msg, len);
1499
1500out:
1501 release_sock(sk);
1502 return err;
1503}
1504
1505static int vsock_dgram_connect(struct socket *sock,
1506 struct sockaddr_unsized *addr, int addr_len, int flags)
1507{
1508 int err;
1509 struct sock *sk;
1510 struct vsock_sock *vsk;
1511 struct sockaddr_vm *remote_addr;
1512
1513 sk = sock->sk;
1514 vsk = vsock_sk(sk);
1515
1516 err = vsock_addr_cast(addr, addr_len, &remote_addr);
1517 if (err == -EAFNOSUPPORT && remote_addr->svm_family == AF_UNSPEC) {
1518 lock_sock(sk);
1519 vsock_addr_init(&vsk->remote_addr, VMADDR_CID_ANY,
1520 VMADDR_PORT_ANY);
1521 sock->state = SS_UNCONNECTED;
1522 release_sock(sk);
1523 return 0;
1524 } else if (err != 0)
1525 return -EINVAL;
1526
1527 lock_sock(sk);
1528
1529 err = vsock_auto_bind(vsk);
1530 if (err)
1531 goto out;
1532
1533 if (!vsk->transport->dgram_allow(vsk, remote_addr->svm_cid,
1534 remote_addr->svm_port)) {
1535 err = -EINVAL;
1536 goto out;
1537 }
1538
1539 memcpy(&vsk->remote_addr, remote_addr, sizeof(vsk->remote_addr));
1540 sock->state = SS_CONNECTED;
1541
1542 /* sock map disallows redirection of non-TCP sockets with sk_state !=
1543 * TCP_ESTABLISHED (see sock_map_redirect_allowed()), so we set
1544 * TCP_ESTABLISHED here to allow redirection of connected vsock dgrams.
1545 *
1546 * This doesn't seem to be abnormal state for datagram sockets, as the
1547 * same approach can be see in other datagram socket types as well
1548 * (such as unix sockets).
1549 */
1550 sk->sk_state = TCP_ESTABLISHED;
1551
1552out:
1553 release_sock(sk);
1554 return err;
1555}
1556
1557int __vsock_dgram_recvmsg(struct socket *sock, struct msghdr *msg,
1558 size_t len, int flags)
1559{
1560 struct sock *sk = sock->sk;
1561 struct vsock_sock *vsk = vsock_sk(sk);
1562
1563 return vsk->transport->dgram_dequeue(vsk, msg, len, flags);
1564}
1565
1566int vsock_dgram_recvmsg(struct socket *sock, struct msghdr *msg,
1567 size_t len, int flags)
1568{
1569#ifdef CONFIG_BPF_SYSCALL
1570 struct sock *sk = sock->sk;
1571 const struct proto *prot;
1572
1573 prot = READ_ONCE(sk->sk_prot);
1574 if (prot != &vsock_proto)
1575 return prot->recvmsg(sk, msg, len, flags);
1576#endif
1577
1578 return __vsock_dgram_recvmsg(sock, msg, len, flags);
1579}
1580EXPORT_SYMBOL_GPL(vsock_dgram_recvmsg);
1581
1582static int vsock_do_ioctl(struct socket *sock, unsigned int cmd,
1583 int __user *arg)
1584{
1585 struct sock *sk = sock->sk;
1586 struct vsock_sock *vsk;
1587 int ret;
1588
1589 vsk = vsock_sk(sk);
1590
1591 switch (cmd) {
1592 case SIOCINQ: {
1593 ssize_t n_bytes;
1594
1595 if (!vsk->transport) {
1596 ret = -EOPNOTSUPP;
1597 break;
1598 }
1599
1600 if (sock_type_connectible(sk->sk_type) &&
1601 sk->sk_state == TCP_LISTEN) {
1602 ret = -EINVAL;
1603 break;
1604 }
1605
1606 n_bytes = vsock_stream_has_data(vsk);
1607 if (n_bytes < 0) {
1608 ret = n_bytes;
1609 break;
1610 }
1611 ret = put_user(n_bytes, arg);
1612 break;
1613 }
1614 case SIOCOUTQ: {
1615 ssize_t n_bytes;
1616
1617 if (!vsk->transport || !vsk->transport->unsent_bytes) {
1618 ret = -EOPNOTSUPP;
1619 break;
1620 }
1621
1622 if (sock_type_connectible(sk->sk_type) && sk->sk_state == TCP_LISTEN) {
1623 ret = -EINVAL;
1624 break;
1625 }
1626
1627 n_bytes = vsk->transport->unsent_bytes(vsk);
1628 if (n_bytes < 0) {
1629 ret = n_bytes;
1630 break;
1631 }
1632
1633 ret = put_user(n_bytes, arg);
1634 break;
1635 }
1636 default:
1637 ret = -ENOIOCTLCMD;
1638 }
1639
1640 return ret;
1641}
1642
1643static int vsock_ioctl(struct socket *sock, unsigned int cmd,
1644 unsigned long arg)
1645{
1646 int ret;
1647
1648 lock_sock(sock->sk);
1649 ret = vsock_do_ioctl(sock, cmd, (int __user *)arg);
1650 release_sock(sock->sk);
1651
1652 return ret;
1653}
1654
1655static const struct proto_ops vsock_dgram_ops = {
1656 .family = PF_VSOCK,
1657 .owner = THIS_MODULE,
1658 .release = vsock_release,
1659 .bind = vsock_bind,
1660 .connect = vsock_dgram_connect,
1661 .socketpair = sock_no_socketpair,
1662 .accept = sock_no_accept,
1663 .getname = vsock_getname,
1664 .poll = vsock_poll,
1665 .ioctl = vsock_ioctl,
1666 .listen = sock_no_listen,
1667 .shutdown = vsock_shutdown,
1668 .sendmsg = vsock_dgram_sendmsg,
1669 .recvmsg = vsock_dgram_recvmsg,
1670 .mmap = sock_no_mmap,
1671 .read_skb = vsock_read_skb,
1672};
1673
1674static int vsock_transport_cancel_pkt(struct vsock_sock *vsk)
1675{
1676 const struct vsock_transport *transport = vsk->transport;
1677
1678 if (!transport || !transport->cancel_pkt)
1679 return -EOPNOTSUPP;
1680
1681 return transport->cancel_pkt(vsk);
1682}
1683
1684static void vsock_connect_timeout(struct work_struct *work)
1685{
1686 struct sock *sk;
1687 struct vsock_sock *vsk;
1688
1689 vsk = container_of(work, struct vsock_sock, connect_work.work);
1690 sk = sk_vsock(vsk);
1691
1692 lock_sock(sk);
1693 if (sk->sk_state == TCP_SYN_SENT &&
1694 (sk->sk_shutdown != SHUTDOWN_MASK)) {
1695 sk->sk_state = TCP_CLOSE;
1696 sk->sk_socket->state = SS_UNCONNECTED;
1697 sk->sk_err = ETIMEDOUT;
1698 sk_error_report(sk);
1699 vsock_transport_cancel_pkt(vsk);
1700 }
1701 release_sock(sk);
1702
1703 sock_put(sk);
1704}
1705
1706static int vsock_connect(struct socket *sock, struct sockaddr_unsized *addr,
1707 int addr_len, int flags)
1708{
1709 int err;
1710 struct sock *sk;
1711 struct vsock_sock *vsk;
1712 const struct vsock_transport *transport;
1713 struct sockaddr_vm *remote_addr;
1714 long timeout;
1715 DEFINE_WAIT(wait);
1716
1717 err = 0;
1718 sk = sock->sk;
1719 vsk = vsock_sk(sk);
1720
1721 lock_sock(sk);
1722
1723 /* XXX AF_UNSPEC should make us disconnect like AF_INET. */
1724 switch (sock->state) {
1725 case SS_CONNECTED:
1726 err = -EISCONN;
1727 goto out;
1728 case SS_DISCONNECTING:
1729 err = -EINVAL;
1730 goto out;
1731 case SS_CONNECTING:
1732 /* This continues on so we can move sock into the SS_CONNECTED
1733 * state once the connection has completed (at which point err
1734 * will be set to zero also). Otherwise, we will either wait
1735 * for the connection or return -EALREADY should this be a
1736 * non-blocking call.
1737 */
1738 err = -EALREADY;
1739 if (flags & O_NONBLOCK)
1740 goto out;
1741 break;
1742 default:
1743 if ((sk->sk_state == TCP_LISTEN) ||
1744 vsock_addr_cast(addr, addr_len, &remote_addr) != 0) {
1745 err = -EINVAL;
1746 goto out;
1747 }
1748
1749 /* Set the remote address that we are connecting to. */
1750 memcpy(&vsk->remote_addr, remote_addr,
1751 sizeof(vsk->remote_addr));
1752
1753 err = vsock_assign_transport(vsk, NULL);
1754 if (err)
1755 goto out;
1756
1757 transport = vsk->transport;
1758
1759 /* The hypervisor and well-known contexts do not have socket
1760 * endpoints.
1761 */
1762 if (!transport ||
1763 !transport->stream_allow(vsk, remote_addr->svm_cid,
1764 remote_addr->svm_port)) {
1765 err = -ENETUNREACH;
1766 goto out;
1767 }
1768
1769 if (vsock_msgzerocopy_allow(transport)) {
1770 set_bit(SOCK_SUPPORT_ZC, &sk->sk_socket->flags);
1771 } else if (sock_flag(sk, SOCK_ZEROCOPY)) {
1772 /* If this option was set before 'connect()',
1773 * when transport was unknown, check that this
1774 * feature is supported here.
1775 */
1776 err = -EOPNOTSUPP;
1777 goto out;
1778 }
1779
1780 err = vsock_auto_bind(vsk);
1781 if (err)
1782 goto out;
1783
1784 sk->sk_state = TCP_SYN_SENT;
1785
1786 err = transport->connect(vsk);
1787 if (err < 0)
1788 goto out;
1789
1790 /* sk_err might have been set as a result of an earlier
1791 * (failed) connect attempt.
1792 */
1793 sk->sk_err = 0;
1794
1795 /* Mark sock as connecting and set the error code to in
1796 * progress in case this is a non-blocking connect.
1797 */
1798 sock->state = SS_CONNECTING;
1799 err = -EINPROGRESS;
1800 }
1801
1802 /* The receive path will handle all communication until we are able to
1803 * enter the connected state. Here we wait for the connection to be
1804 * completed or a notification of an error.
1805 */
1806 timeout = vsk->connect_timeout;
1807 prepare_to_wait(sk_sleep(sk), &wait, TASK_INTERRUPTIBLE);
1808
1809 /* If the socket is already closing or it is in an error state, there
1810 * is no point in waiting.
1811 */
1812 while (sk->sk_state != TCP_ESTABLISHED &&
1813 sk->sk_state != TCP_CLOSING && sk->sk_err == 0) {
1814 if (flags & O_NONBLOCK) {
1815 /* If we're not going to block, we schedule a timeout
1816 * function to generate a timeout on the connection
1817 * attempt, in case the peer doesn't respond in a
1818 * timely manner. We hold on to the socket until the
1819 * timeout fires.
1820 */
1821 sock_hold(sk);
1822
1823 /* If the timeout function is already scheduled,
1824 * reschedule it, then ungrab the socket refcount to
1825 * keep it balanced.
1826 */
1827 if (mod_delayed_work(system_percpu_wq, &vsk->connect_work,
1828 timeout))
1829 sock_put(sk);
1830
1831 /* Skip ahead to preserve error code set above. */
1832 goto out_wait;
1833 }
1834
1835 release_sock(sk);
1836 timeout = schedule_timeout(timeout);
1837 lock_sock(sk);
1838
1839 /* Connection was established. Whatever happens to socket once
1840 * we release it, that's not connect()'s concern. No need to go
1841 * into signal and timeout handling. Call it a day.
1842 *
1843 * Note that allowing to "reset" an already established socket
1844 * here is racy and insecure.
1845 */
1846 if (sk->sk_state == TCP_ESTABLISHED ||
1847 sk->sk_state == TCP_CLOSING) {
1848 err = 0;
1849 goto out_wait;
1850 }
1851
1852 /* If connection was _not_ established and a signal/timeout came
1853 * to be, we want the socket's state reset. User space may want
1854 * to retry, so we keep the binding and the transport assigned.
1855 */
1856 if (signal_pending(current) || timeout == 0) {
1857 err = timeout == 0 ? -ETIMEDOUT : sock_intr_errno(timeout);
1858
1859 /* Listener might have already responded with
1860 * VIRTIO_VSOCK_OP_RESPONSE. Its handling expects our
1861 * sk_state == TCP_SYN_SENT, which hereby we break.
1862 * In such case VIRTIO_VSOCK_OP_RST will follow.
1863 */
1864 sk->sk_state = TCP_CLOSE;
1865 sock->state = SS_UNCONNECTED;
1866
1867 /* Try to cancel VIRTIO_VSOCK_OP_REQUEST skb sent out by
1868 * transport->connect().
1869 */
1870 vsock_transport_cancel_pkt(vsk);
1871
1872 goto out_wait;
1873 }
1874
1875 prepare_to_wait(sk_sleep(sk), &wait, TASK_INTERRUPTIBLE);
1876 }
1877
1878 err = sock_error(sk);
1879 if (err) {
1880 sk->sk_state = TCP_CLOSE;
1881 sock->state = SS_UNCONNECTED;
1882 }
1883
1884out_wait:
1885 finish_wait(sk_sleep(sk), &wait);
1886out:
1887 release_sock(sk);
1888 return err;
1889}
1890
1891static int vsock_accept(struct socket *sock, struct socket *newsock,
1892 struct proto_accept_arg *arg)
1893{
1894 struct sock *listener;
1895 int err;
1896 struct sock *connected;
1897 struct vsock_sock *vconnected;
1898 long timeout;
1899 DEFINE_WAIT(wait);
1900
1901 err = 0;
1902 listener = sock->sk;
1903
1904 lock_sock(listener);
1905
1906 if (!sock_type_connectible(sock->type)) {
1907 err = -EOPNOTSUPP;
1908 goto out;
1909 }
1910
1911 if (listener->sk_state != TCP_LISTEN) {
1912 err = -EINVAL;
1913 goto out;
1914 }
1915
1916 /* Wait for children sockets to appear; these are the new sockets
1917 * created upon connection establishment.
1918 */
1919 timeout = sock_rcvtimeo(listener, arg->flags & O_NONBLOCK);
1920
1921 while ((connected = vsock_dequeue_accept(listener)) == NULL &&
1922 timeout != 0) {
1923 prepare_to_wait(sk_sleep(listener), &wait, TASK_INTERRUPTIBLE);
1924 release_sock(listener);
1925 timeout = schedule_timeout(timeout);
1926 finish_wait(sk_sleep(listener), &wait);
1927 lock_sock(listener);
1928
1929 if (signal_pending(current)) {
1930 err = sock_intr_errno(timeout);
1931 goto out;
1932 }
1933 }
1934
1935 if (!connected) {
1936 err = -EAGAIN;
1937 } else {
1938 sk_acceptq_removed(listener);
1939
1940 lock_sock_nested(connected, SINGLE_DEPTH_NESTING);
1941 vconnected = vsock_sk(connected);
1942
1943 newsock->state = SS_CONNECTED;
1944 sock_graft(connected, newsock);
1945
1946 set_bit(SOCK_CUSTOM_SOCKOPT,
1947 &connected->sk_socket->flags);
1948
1949 if (vsock_msgzerocopy_allow(vconnected->transport))
1950 set_bit(SOCK_SUPPORT_ZC,
1951 &connected->sk_socket->flags);
1952
1953 release_sock(connected);
1954 sock_put(connected);
1955 }
1956
1957out:
1958 release_sock(listener);
1959 return err;
1960}
1961
1962static int vsock_listen(struct socket *sock, int backlog)
1963{
1964 int err;
1965 struct sock *sk;
1966 struct vsock_sock *vsk;
1967
1968 sk = sock->sk;
1969
1970 lock_sock(sk);
1971
1972 if (!sock_type_connectible(sk->sk_type)) {
1973 err = -EOPNOTSUPP;
1974 goto out;
1975 }
1976
1977 if (sock->state != SS_UNCONNECTED) {
1978 err = -EINVAL;
1979 goto out;
1980 }
1981
1982 vsk = vsock_sk(sk);
1983
1984 if (!vsock_addr_bound(&vsk->local_addr)) {
1985 err = -EINVAL;
1986 goto out;
1987 }
1988
1989 sk->sk_max_ack_backlog = backlog;
1990 sk->sk_state = TCP_LISTEN;
1991
1992 err = 0;
1993
1994out:
1995 release_sock(sk);
1996 return err;
1997}
1998
1999static void vsock_update_buffer_size(struct vsock_sock *vsk,
2000 const struct vsock_transport *transport,
2001 u64 val)
2002{
2003 if (val < vsk->buffer_min_size)
2004 val = vsk->buffer_min_size;
2005
2006 if (val > vsk->buffer_max_size)
2007 val = vsk->buffer_max_size;
2008
2009 if (val != vsk->buffer_size &&
2010 transport && transport->notify_buffer_size)
2011 transport->notify_buffer_size(vsk, &val);
2012
2013 vsk->buffer_size = val;
2014}
2015
2016static int vsock_connectible_setsockopt(struct socket *sock,
2017 int level,
2018 int optname,
2019 sockptr_t optval,
2020 unsigned int optlen)
2021{
2022 int err;
2023 struct sock *sk;
2024 struct vsock_sock *vsk;
2025 const struct vsock_transport *transport;
2026 u64 val;
2027
2028 if (level != AF_VSOCK && level != SOL_SOCKET)
2029 return -ENOPROTOOPT;
2030
2031#define COPY_IN(_v) \
2032 do { \
2033 if (optlen < sizeof(_v)) { \
2034 err = -EINVAL; \
2035 goto exit; \
2036 } \
2037 if (copy_from_sockptr(&_v, optval, sizeof(_v)) != 0) { \
2038 err = -EFAULT; \
2039 goto exit; \
2040 } \
2041 } while (0)
2042
2043 err = 0;
2044 sk = sock->sk;
2045 vsk = vsock_sk(sk);
2046
2047 lock_sock(sk);
2048
2049 transport = vsk->transport;
2050
2051 if (level == SOL_SOCKET) {
2052 int zerocopy;
2053
2054 if (optname != SO_ZEROCOPY) {
2055 release_sock(sk);
2056 return sock_setsockopt(sock, level, optname, optval, optlen);
2057 }
2058
2059 /* Use 'int' type here, because variable to
2060 * set this option usually has this type.
2061 */
2062 COPY_IN(zerocopy);
2063
2064 if (zerocopy < 0 || zerocopy > 1) {
2065 err = -EINVAL;
2066 goto exit;
2067 }
2068
2069 if (transport && !vsock_msgzerocopy_allow(transport)) {
2070 err = -EOPNOTSUPP;
2071 goto exit;
2072 }
2073
2074 sock_valbool_flag(sk, SOCK_ZEROCOPY, zerocopy);
2075 goto exit;
2076 }
2077
2078 switch (optname) {
2079 case SO_VM_SOCKETS_BUFFER_SIZE:
2080 COPY_IN(val);
2081 vsock_update_buffer_size(vsk, transport, val);
2082 break;
2083
2084 case SO_VM_SOCKETS_BUFFER_MAX_SIZE:
2085 COPY_IN(val);
2086 vsk->buffer_max_size = val;
2087 vsock_update_buffer_size(vsk, transport, vsk->buffer_size);
2088 break;
2089
2090 case SO_VM_SOCKETS_BUFFER_MIN_SIZE:
2091 COPY_IN(val);
2092 vsk->buffer_min_size = val;
2093 vsock_update_buffer_size(vsk, transport, vsk->buffer_size);
2094 break;
2095
2096 case SO_VM_SOCKETS_CONNECT_TIMEOUT_NEW:
2097 case SO_VM_SOCKETS_CONNECT_TIMEOUT_OLD: {
2098 struct __kernel_sock_timeval tv;
2099
2100 err = sock_copy_user_timeval(&tv, optval, optlen,
2101 optname == SO_VM_SOCKETS_CONNECT_TIMEOUT_OLD);
2102 if (err)
2103 break;
2104 if (tv.tv_sec >= 0 && tv.tv_usec < USEC_PER_SEC &&
2105 tv.tv_sec < (MAX_SCHEDULE_TIMEOUT / HZ - 1)) {
2106 vsk->connect_timeout = tv.tv_sec * HZ +
2107 DIV_ROUND_UP((unsigned long)tv.tv_usec, (USEC_PER_SEC / HZ));
2108 if (vsk->connect_timeout == 0)
2109 vsk->connect_timeout =
2110 VSOCK_DEFAULT_CONNECT_TIMEOUT;
2111
2112 } else {
2113 err = -ERANGE;
2114 }
2115 break;
2116 }
2117
2118 default:
2119 err = -ENOPROTOOPT;
2120 break;
2121 }
2122
2123#undef COPY_IN
2124
2125exit:
2126 release_sock(sk);
2127 return err;
2128}
2129
2130static int vsock_connectible_getsockopt(struct socket *sock,
2131 int level, int optname,
2132 sockopt_t *opt)
2133{
2134 struct sock *sk = sock->sk;
2135 struct vsock_sock *vsk = vsock_sk(sk);
2136
2137 union {
2138 u64 val64;
2139 struct old_timeval32 tm32;
2140 struct __kernel_old_timeval tm;
2141 struct __kernel_sock_timeval stm;
2142 } v;
2143
2144 int lv = sizeof(v.val64);
2145 int len;
2146
2147 if (level != AF_VSOCK)
2148 return -ENOPROTOOPT;
2149
2150 len = opt->optlen;
2151
2152 memset(&v, 0, sizeof(v));
2153
2154 switch (optname) {
2155 case SO_VM_SOCKETS_BUFFER_SIZE:
2156 v.val64 = vsk->buffer_size;
2157 break;
2158
2159 case SO_VM_SOCKETS_BUFFER_MAX_SIZE:
2160 v.val64 = vsk->buffer_max_size;
2161 break;
2162
2163 case SO_VM_SOCKETS_BUFFER_MIN_SIZE:
2164 v.val64 = vsk->buffer_min_size;
2165 break;
2166
2167 case SO_VM_SOCKETS_CONNECT_TIMEOUT_NEW:
2168 case SO_VM_SOCKETS_CONNECT_TIMEOUT_OLD:
2169 lv = sock_get_timeout(vsk->connect_timeout, &v,
2170 optname == SO_VM_SOCKETS_CONNECT_TIMEOUT_OLD);
2171 break;
2172
2173 default:
2174 return -ENOPROTOOPT;
2175 }
2176
2177 if (len < lv)
2178 return -EINVAL;
2179 if (len > lv)
2180 len = lv;
2181 if (copy_to_iter(&v, len, &opt->iter_out) != len)
2182 return -EFAULT;
2183
2184 opt->optlen = len;
2185
2186 return 0;
2187}
2188
2189static int vsock_connectible_sendmsg(struct socket *sock, struct msghdr *msg,
2190 size_t len)
2191{
2192 struct sock *sk;
2193 struct vsock_sock *vsk;
2194 const struct vsock_transport *transport;
2195 ssize_t total_written;
2196 long timeout;
2197 int err;
2198 struct vsock_transport_send_notify_data send_data;
2199 DEFINE_WAIT_FUNC(wait, woken_wake_function);
2200
2201 sk = sock->sk;
2202 vsk = vsock_sk(sk);
2203 total_written = 0;
2204 err = 0;
2205
2206 if (msg->msg_flags & MSG_OOB)
2207 return -EOPNOTSUPP;
2208
2209 lock_sock(sk);
2210
2211 transport = vsk->transport;
2212
2213 /* Callers should not provide a destination with connection oriented
2214 * sockets.
2215 */
2216 if (msg->msg_namelen) {
2217 err = sk->sk_state == TCP_ESTABLISHED ? -EISCONN : -EOPNOTSUPP;
2218 goto out;
2219 }
2220
2221 /* Send data only if both sides are not shutdown in the direction. */
2222 if (sk->sk_shutdown & SEND_SHUTDOWN ||
2223 vsk->peer_shutdown & RCV_SHUTDOWN) {
2224 err = -EPIPE;
2225 goto out;
2226 }
2227
2228 if (!transport || sk->sk_state != TCP_ESTABLISHED ||
2229 !vsock_addr_bound(&vsk->local_addr)) {
2230 err = -ENOTCONN;
2231 goto out;
2232 }
2233
2234 if (!vsock_addr_bound(&vsk->remote_addr)) {
2235 err = -EDESTADDRREQ;
2236 goto out;
2237 }
2238
2239 if (msg->msg_flags & MSG_ZEROCOPY &&
2240 !vsock_msgzerocopy_allow(transport)) {
2241 err = -EOPNOTSUPP;
2242 goto out;
2243 }
2244
2245 /* Wait for room in the produce queue to enqueue our user's data. */
2246 timeout = sock_sndtimeo(sk, msg->msg_flags & MSG_DONTWAIT);
2247
2248 err = transport->notify_send_init(vsk, &send_data);
2249 if (err < 0)
2250 goto out;
2251
2252 while (total_written < len) {
2253 ssize_t written;
2254
2255 add_wait_queue(sk_sleep(sk), &wait);
2256 while (vsock_stream_has_space(vsk) == 0 &&
2257 sk->sk_err == 0 &&
2258 !(sk->sk_shutdown & SEND_SHUTDOWN) &&
2259 !(vsk->peer_shutdown & RCV_SHUTDOWN)) {
2260
2261 /* Don't wait for non-blocking sockets. */
2262 if (timeout == 0) {
2263 err = -EAGAIN;
2264 remove_wait_queue(sk_sleep(sk), &wait);
2265 goto out_err;
2266 }
2267
2268 err = transport->notify_send_pre_block(vsk, &send_data);
2269 if (err < 0) {
2270 remove_wait_queue(sk_sleep(sk), &wait);
2271 goto out_err;
2272 }
2273
2274 release_sock(sk);
2275 timeout = wait_woken(&wait, TASK_INTERRUPTIBLE, timeout);
2276 lock_sock(sk);
2277 if (signal_pending(current)) {
2278 err = sock_intr_errno(timeout);
2279 remove_wait_queue(sk_sleep(sk), &wait);
2280 goto out_err;
2281 } else if (timeout == 0) {
2282 err = -EAGAIN;
2283 remove_wait_queue(sk_sleep(sk), &wait);
2284 goto out_err;
2285 }
2286 }
2287 remove_wait_queue(sk_sleep(sk), &wait);
2288
2289 /* These checks occur both as part of and after the loop
2290 * conditional since we need to check before and after
2291 * sleeping.
2292 */
2293 if (sk->sk_err) {
2294 err = -sk->sk_err;
2295 goto out_err;
2296 } else if ((sk->sk_shutdown & SEND_SHUTDOWN) ||
2297 (vsk->peer_shutdown & RCV_SHUTDOWN)) {
2298 err = -EPIPE;
2299 goto out_err;
2300 }
2301
2302 err = transport->notify_send_pre_enqueue(vsk, &send_data);
2303 if (err < 0)
2304 goto out_err;
2305
2306 /* Note that enqueue will only write as many bytes as are free
2307 * in the produce queue, so we don't need to ensure len is
2308 * smaller than the queue size. It is the caller's
2309 * responsibility to check how many bytes we were able to send.
2310 */
2311
2312 if (sk->sk_type == SOCK_SEQPACKET) {
2313 written = transport->seqpacket_enqueue(vsk,
2314 msg, len - total_written);
2315 } else {
2316 written = transport->stream_enqueue(vsk,
2317 msg, len - total_written);
2318 }
2319
2320 if (written < 0) {
2321 err = written;
2322 goto out_err;
2323 }
2324
2325 total_written += written;
2326
2327 err = transport->notify_send_post_enqueue(
2328 vsk, written, &send_data);
2329 if (err < 0)
2330 goto out_err;
2331
2332 }
2333
2334out_err:
2335 if (total_written > 0) {
2336 /* Return number of written bytes only if:
2337 * 1) SOCK_STREAM socket.
2338 * 2) SOCK_SEQPACKET socket when whole buffer is sent.
2339 */
2340 if (sk->sk_type == SOCK_STREAM || total_written == len)
2341 err = total_written;
2342 }
2343out:
2344 if (sk->sk_type == SOCK_STREAM)
2345 err = sk_stream_error(sk, msg->msg_flags, err);
2346
2347 release_sock(sk);
2348 return err;
2349}
2350
2351static int vsock_connectible_wait_data(struct sock *sk,
2352 struct wait_queue_entry *wait,
2353 long timeout,
2354 struct vsock_transport_recv_notify_data *recv_data,
2355 size_t target)
2356{
2357 const struct vsock_transport *transport;
2358 struct vsock_sock *vsk;
2359 s64 data;
2360 int err;
2361
2362 vsk = vsock_sk(sk);
2363 err = 0;
2364 transport = vsk->transport;
2365
2366 while (1) {
2367 prepare_to_wait(sk_sleep(sk), wait, TASK_INTERRUPTIBLE);
2368 data = vsock_connectible_has_data(vsk);
2369 if (data != 0)
2370 break;
2371
2372 if (sk->sk_err != 0 ||
2373 (sk->sk_shutdown & RCV_SHUTDOWN) ||
2374 (vsk->peer_shutdown & SEND_SHUTDOWN)) {
2375 break;
2376 }
2377
2378 /* Don't wait for non-blocking sockets. */
2379 if (timeout == 0) {
2380 err = -EAGAIN;
2381 break;
2382 }
2383
2384 if (recv_data) {
2385 err = transport->notify_recv_pre_block(vsk, target, recv_data);
2386 if (err < 0)
2387 break;
2388 }
2389
2390 release_sock(sk);
2391 timeout = schedule_timeout(timeout);
2392 lock_sock(sk);
2393
2394 if (signal_pending(current)) {
2395 err = sock_intr_errno(timeout);
2396 break;
2397 } else if (timeout == 0) {
2398 err = -EAGAIN;
2399 break;
2400 }
2401 }
2402
2403 finish_wait(sk_sleep(sk), wait);
2404
2405 if (err)
2406 return err;
2407
2408 /* Internal transport error when checking for available
2409 * data. XXX This should be changed to a connection
2410 * reset in a later change.
2411 */
2412 if (data < 0)
2413 return -ENOMEM;
2414
2415 return data;
2416}
2417
2418static int __vsock_stream_recvmsg(struct sock *sk, struct msghdr *msg,
2419 size_t len, int flags)
2420{
2421 struct vsock_transport_recv_notify_data recv_data;
2422 const struct vsock_transport *transport;
2423 struct vsock_sock *vsk;
2424 ssize_t copied;
2425 size_t target;
2426 long timeout;
2427 int err;
2428
2429 DEFINE_WAIT(wait);
2430
2431 vsk = vsock_sk(sk);
2432 transport = vsk->transport;
2433
2434 /* We must not copy less than target bytes into the user's buffer
2435 * before returning successfully, so we wait for the consume queue to
2436 * have that much data to consume before dequeueing. Note that this
2437 * makes it impossible to handle cases where target is greater than the
2438 * queue size.
2439 */
2440 target = sock_rcvlowat(sk, flags & MSG_WAITALL, len);
2441 if (target >= transport->stream_rcvhiwat(vsk)) {
2442 err = -ENOMEM;
2443 goto out;
2444 }
2445 timeout = sock_rcvtimeo(sk, flags & MSG_DONTWAIT);
2446 copied = 0;
2447
2448 err = transport->notify_recv_init(vsk, target, &recv_data);
2449 if (err < 0)
2450 goto out;
2451
2452
2453 while (1) {
2454 ssize_t read;
2455
2456 err = vsock_connectible_wait_data(sk, &wait, timeout,
2457 &recv_data, target);
2458 if (err <= 0)
2459 break;
2460
2461 err = transport->notify_recv_pre_dequeue(vsk, target,
2462 &recv_data);
2463 if (err < 0)
2464 break;
2465
2466 read = transport->stream_dequeue(vsk, msg, len - copied, flags);
2467 if (read < 0) {
2468 err = read;
2469 break;
2470 }
2471
2472 copied += read;
2473
2474 err = transport->notify_recv_post_dequeue(vsk, target, read,
2475 !(flags & MSG_PEEK), &recv_data);
2476 if (err < 0)
2477 goto out;
2478
2479 if (read >= target || flags & MSG_PEEK)
2480 break;
2481
2482 target -= read;
2483 }
2484
2485 if (sk->sk_err)
2486 err = -sk->sk_err;
2487 else if (sk->sk_shutdown & RCV_SHUTDOWN)
2488 err = 0;
2489
2490 if (copied > 0)
2491 err = copied;
2492
2493out:
2494 return err;
2495}
2496
2497static int __vsock_seqpacket_recvmsg(struct sock *sk, struct msghdr *msg,
2498 size_t len, int flags)
2499{
2500 const struct vsock_transport *transport;
2501 struct vsock_sock *vsk;
2502 ssize_t msg_len;
2503 long timeout;
2504 int err = 0;
2505 DEFINE_WAIT(wait);
2506
2507 vsk = vsock_sk(sk);
2508 transport = vsk->transport;
2509
2510 timeout = sock_rcvtimeo(sk, flags & MSG_DONTWAIT);
2511
2512 err = vsock_connectible_wait_data(sk, &wait, timeout, NULL, 0);
2513 if (err <= 0)
2514 goto out;
2515
2516 msg_len = transport->seqpacket_dequeue(vsk, msg, flags);
2517
2518 if (msg_len < 0) {
2519 err = msg_len;
2520 goto out;
2521 }
2522
2523 if (sk->sk_err) {
2524 err = -sk->sk_err;
2525 } else if (sk->sk_shutdown & RCV_SHUTDOWN) {
2526 err = 0;
2527 } else {
2528 /* User sets MSG_TRUNC, so return real length of
2529 * packet.
2530 */
2531 if (flags & MSG_TRUNC)
2532 err = msg_len;
2533 else
2534 err = len - msg_data_left(msg);
2535
2536 /* Always set MSG_TRUNC if real length of packet is
2537 * bigger than user's buffer.
2538 */
2539 if (msg_len > len)
2540 msg->msg_flags |= MSG_TRUNC;
2541 }
2542
2543out:
2544 return err;
2545}
2546
2547int
2548__vsock_connectible_recvmsg(struct socket *sock, struct msghdr *msg, size_t len,
2549 int flags)
2550{
2551 struct sock *sk;
2552 struct vsock_sock *vsk;
2553 const struct vsock_transport *transport;
2554 int err;
2555
2556 sk = sock->sk;
2557
2558 if (unlikely(flags & MSG_ERRQUEUE))
2559 return sock_recv_errqueue(sk, msg, len, SOL_VSOCK, VSOCK_RECVERR);
2560
2561 vsk = vsock_sk(sk);
2562 err = 0;
2563
2564 lock_sock(sk);
2565
2566 transport = vsk->transport;
2567
2568 if (!transport || sk->sk_state != TCP_ESTABLISHED) {
2569 /* Recvmsg is supposed to return 0 if a peer performs an
2570 * orderly shutdown. Differentiate between that case and when a
2571 * peer has not connected or a local shutdown occurred with the
2572 * SOCK_DONE flag.
2573 */
2574 if (sock_flag(sk, SOCK_DONE))
2575 err = 0;
2576 else
2577 err = -ENOTCONN;
2578
2579 goto out;
2580 }
2581
2582 if (flags & MSG_OOB) {
2583 err = -EOPNOTSUPP;
2584 goto out;
2585 }
2586
2587 /* We don't check peer_shutdown flag here since peer may actually shut
2588 * down, but there can be data in the queue that a local socket can
2589 * receive.
2590 */
2591 if (sk->sk_shutdown & RCV_SHUTDOWN) {
2592 err = 0;
2593 goto out;
2594 }
2595
2596 /* It is valid on Linux to pass in a zero-length receive buffer. This
2597 * is not an error. We may as well bail out now.
2598 */
2599 if (!len) {
2600 err = 0;
2601 goto out;
2602 }
2603
2604 if (sk->sk_type == SOCK_STREAM)
2605 err = __vsock_stream_recvmsg(sk, msg, len, flags);
2606 else
2607 err = __vsock_seqpacket_recvmsg(sk, msg, len, flags);
2608
2609out:
2610 release_sock(sk);
2611 return err;
2612}
2613
2614int
2615vsock_connectible_recvmsg(struct socket *sock, struct msghdr *msg, size_t len,
2616 int flags)
2617{
2618#ifdef CONFIG_BPF_SYSCALL
2619 struct sock *sk = sock->sk;
2620 const struct proto *prot;
2621
2622 prot = READ_ONCE(sk->sk_prot);
2623 if (prot != &vsock_proto)
2624 return prot->recvmsg(sk, msg, len, flags);
2625#endif
2626
2627 return __vsock_connectible_recvmsg(sock, msg, len, flags);
2628}
2629EXPORT_SYMBOL_GPL(vsock_connectible_recvmsg);
2630
2631static int vsock_set_rcvlowat(struct sock *sk, int val)
2632{
2633 const struct vsock_transport *transport;
2634 struct vsock_sock *vsk;
2635
2636 vsk = vsock_sk(sk);
2637
2638 if (val > vsk->buffer_size)
2639 return -EINVAL;
2640
2641 transport = vsk->transport;
2642
2643 if (transport && transport->notify_set_rcvlowat) {
2644 int err;
2645
2646 err = transport->notify_set_rcvlowat(vsk, val);
2647 if (err)
2648 return err;
2649 }
2650
2651 WRITE_ONCE(sk->sk_rcvlowat, val ? : 1);
2652 return 0;
2653}
2654
2655static const struct proto_ops vsock_stream_ops = {
2656 .family = PF_VSOCK,
2657 .owner = THIS_MODULE,
2658 .release = vsock_release,
2659 .bind = vsock_bind,
2660 .connect = vsock_connect,
2661 .socketpair = sock_no_socketpair,
2662 .accept = vsock_accept,
2663 .getname = vsock_getname,
2664 .poll = vsock_poll,
2665 .ioctl = vsock_ioctl,
2666 .listen = vsock_listen,
2667 .shutdown = vsock_shutdown,
2668 .setsockopt = vsock_connectible_setsockopt,
2669 .getsockopt_iter = vsock_connectible_getsockopt,
2670 .sendmsg = vsock_connectible_sendmsg,
2671 .recvmsg = vsock_connectible_recvmsg,
2672 .mmap = sock_no_mmap,
2673 .set_rcvlowat = vsock_set_rcvlowat,
2674 .read_skb = vsock_read_skb,
2675};
2676
2677static const struct proto_ops vsock_seqpacket_ops = {
2678 .family = PF_VSOCK,
2679 .owner = THIS_MODULE,
2680 .release = vsock_release,
2681 .bind = vsock_bind,
2682 .connect = vsock_connect,
2683 .socketpair = sock_no_socketpair,
2684 .accept = vsock_accept,
2685 .getname = vsock_getname,
2686 .poll = vsock_poll,
2687 .ioctl = vsock_ioctl,
2688 .listen = vsock_listen,
2689 .shutdown = vsock_shutdown,
2690 .setsockopt = vsock_connectible_setsockopt,
2691 .getsockopt_iter = vsock_connectible_getsockopt,
2692 .sendmsg = vsock_connectible_sendmsg,
2693 .recvmsg = vsock_connectible_recvmsg,
2694 .mmap = sock_no_mmap,
2695 .read_skb = vsock_read_skb,
2696};
2697
2698static int vsock_create(struct net *net, struct socket *sock,
2699 int protocol, int kern)
2700{
2701 struct vsock_sock *vsk;
2702 struct sock *sk;
2703 int ret;
2704
2705 if (!sock)
2706 return -EINVAL;
2707
2708 if (protocol && protocol != PF_VSOCK)
2709 return -EPROTONOSUPPORT;
2710
2711 switch (sock->type) {
2712 case SOCK_DGRAM:
2713 sock->ops = &vsock_dgram_ops;
2714 break;
2715 case SOCK_STREAM:
2716 sock->ops = &vsock_stream_ops;
2717 break;
2718 case SOCK_SEQPACKET:
2719 sock->ops = &vsock_seqpacket_ops;
2720 break;
2721 default:
2722 return -ESOCKTNOSUPPORT;
2723 }
2724
2725 sock->state = SS_UNCONNECTED;
2726
2727 sk = __vsock_create(net, sock, NULL, GFP_KERNEL, 0, kern);
2728 if (!sk)
2729 return -ENOMEM;
2730
2731 vsk = vsock_sk(sk);
2732
2733 if (sock->type == SOCK_DGRAM) {
2734 ret = vsock_assign_transport(vsk, NULL);
2735 if (ret < 0) {
2736 sock->sk = NULL;
2737 sock_put(sk);
2738 return ret;
2739 }
2740 }
2741
2742 /* SOCK_DGRAM doesn't have 'setsockopt' callback set in its
2743 * proto_ops, so there is no handler for custom logic.
2744 */
2745 if (sock_type_connectible(sock->type))
2746 set_bit(SOCK_CUSTOM_SOCKOPT, &sk->sk_socket->flags);
2747
2748 vsock_insert_unbound(vsk);
2749
2750 return 0;
2751}
2752
2753static const struct net_proto_family vsock_family_ops = {
2754 .family = AF_VSOCK,
2755 .create = vsock_create,
2756 .owner = THIS_MODULE,
2757};
2758
2759static long vsock_dev_do_ioctl(struct file *filp,
2760 unsigned int cmd, void __user *ptr)
2761{
2762 u32 __user *p = ptr;
2763 int retval = 0;
2764 u32 cid;
2765
2766 switch (cmd) {
2767 case IOCTL_VM_SOCKETS_GET_LOCAL_CID:
2768 /* To be compatible with the VMCI behavior, we prioritize the
2769 * guest CID instead of well-know host CID (VMADDR_CID_HOST).
2770 */
2771 cid = vsock_registered_transport_cid(&transport_g2h);
2772 if (cid == VMADDR_CID_ANY)
2773 cid = vsock_registered_transport_cid(&transport_h2g);
2774 if (cid == VMADDR_CID_ANY)
2775 cid = vsock_registered_transport_cid(&transport_local);
2776
2777 if (put_user(cid, p) != 0)
2778 retval = -EFAULT;
2779 break;
2780
2781 default:
2782 retval = -ENOIOCTLCMD;
2783 }
2784
2785 return retval;
2786}
2787
2788static long vsock_dev_ioctl(struct file *filp,
2789 unsigned int cmd, unsigned long arg)
2790{
2791 return vsock_dev_do_ioctl(filp, cmd, (void __user *)arg);
2792}
2793
2794#ifdef CONFIG_COMPAT
2795static long vsock_dev_compat_ioctl(struct file *filp,
2796 unsigned int cmd, unsigned long arg)
2797{
2798 return vsock_dev_do_ioctl(filp, cmd, compat_ptr(arg));
2799}
2800#endif
2801
2802static const struct file_operations vsock_device_ops = {
2803 .owner = THIS_MODULE,
2804 .unlocked_ioctl = vsock_dev_ioctl,
2805#ifdef CONFIG_COMPAT
2806 .compat_ioctl = vsock_dev_compat_ioctl,
2807#endif
2808 .open = nonseekable_open,
2809};
2810
2811static struct miscdevice vsock_device = {
2812 .name = "vsock",
2813 .fops = &vsock_device_ops,
2814};
2815
2816static int __vsock_net_mode_string(const struct ctl_table *table, int write,
2817 void *buffer, size_t *lenp, loff_t *ppos,
2818 enum vsock_net_mode mode,
2819 enum vsock_net_mode *new_mode)
2820{
2821 char data[VSOCK_NET_MODE_STR_MAX] = {0};
2822 struct ctl_table tmp;
2823 int ret;
2824
2825 if (!table->data || !table->maxlen || !*lenp) {
2826 *lenp = 0;
2827 return 0;
2828 }
2829
2830 tmp = *table;
2831 tmp.data = data;
2832
2833 if (!write) {
2834 const char *p;
2835
2836 switch (mode) {
2837 case VSOCK_NET_MODE_GLOBAL:
2838 p = VSOCK_NET_MODE_STR_GLOBAL;
2839 break;
2840 case VSOCK_NET_MODE_LOCAL:
2841 p = VSOCK_NET_MODE_STR_LOCAL;
2842 break;
2843 default:
2844 WARN_ONCE(true, "netns has invalid vsock mode");
2845 *lenp = 0;
2846 return 0;
2847 }
2848
2849 strscpy(data, p, sizeof(data));
2850 tmp.maxlen = strlen(p);
2851 }
2852
2853 ret = proc_dostring(&tmp, write, buffer, lenp, ppos);
2854 if (ret || !write)
2855 return ret;
2856
2857 if (*lenp >= sizeof(data))
2858 return -EINVAL;
2859
2860 if (!strncmp(data, VSOCK_NET_MODE_STR_GLOBAL, sizeof(data)))
2861 *new_mode = VSOCK_NET_MODE_GLOBAL;
2862 else if (!strncmp(data, VSOCK_NET_MODE_STR_LOCAL, sizeof(data)))
2863 *new_mode = VSOCK_NET_MODE_LOCAL;
2864 else
2865 return -EINVAL;
2866
2867 return 0;
2868}
2869
2870static int vsock_net_mode_string(const struct ctl_table *table, int write,
2871 void *buffer, size_t *lenp, loff_t *ppos)
2872{
2873 struct net *net;
2874
2875 if (write)
2876 return -EPERM;
2877
2878 net = container_of(table->data, struct net, vsock.mode);
2879
2880 return __vsock_net_mode_string(table, write, buffer, lenp, ppos,
2881 vsock_net_mode(net), NULL);
2882}
2883
2884static int vsock_net_child_mode_string(const struct ctl_table *table, int write,
2885 void *buffer, size_t *lenp, loff_t *ppos)
2886{
2887 enum vsock_net_mode new_mode;
2888 struct net *net;
2889 int ret;
2890
2891 net = container_of(table->data, struct net, vsock.child_ns_mode);
2892
2893 if (!*lenp)
2894 return 0;
2895
2896 ret = __vsock_net_mode_string(table, write, buffer, lenp, ppos,
2897 vsock_net_child_mode(net), &new_mode);
2898 if (ret)
2899 return ret;
2900
2901 if (write) {
2902 /* Prevent a "local" namespace from escalating to "global",
2903 * which would give nested namespaces access to global CIDs.
2904 */
2905 if (vsock_net_mode(net) == VSOCK_NET_MODE_LOCAL &&
2906 new_mode == VSOCK_NET_MODE_GLOBAL)
2907 return -EPERM;
2908
2909 if (!vsock_net_set_child_mode(net, new_mode))
2910 return -EBUSY;
2911 }
2912
2913 return 0;
2914}
2915
2916static const struct ctl_table vsock_table[] = {
2917 {
2918 .procname = "ns_mode",
2919 .data = &init_net.vsock.mode,
2920 .maxlen = VSOCK_NET_MODE_STR_MAX,
2921 .mode = 0444,
2922 .proc_handler = vsock_net_mode_string
2923 },
2924 {
2925 .procname = "child_ns_mode",
2926 .data = &init_net.vsock.child_ns_mode,
2927 .maxlen = VSOCK_NET_MODE_STR_MAX,
2928 .mode = 0644,
2929 .proc_handler = vsock_net_child_mode_string
2930 },
2931 {
2932 .procname = "g2h_fallback",
2933 .data = &init_net.vsock.g2h_fallback,
2934 .maxlen = sizeof(int),
2935 .mode = 0644,
2936 .proc_handler = proc_dointvec_minmax,
2937 .extra1 = SYSCTL_ZERO,
2938 .extra2 = SYSCTL_ONE,
2939 },
2940};
2941
2942static const struct ctl_table *vsock_table_dup(struct net *net)
2943{
2944 struct ctl_table *table;
2945
2946 table = kmemdup(vsock_table, sizeof(vsock_table), GFP_KERNEL);
2947 if (!table)
2948 return NULL;
2949
2950 table[0].data = &net->vsock.mode;
2951 table[1].data = &net->vsock.child_ns_mode;
2952 table[2].data = &net->vsock.g2h_fallback;
2953
2954 return table;
2955}
2956
2957static int __net_init vsock_sysctl_register(struct net *net)
2958{
2959 const struct ctl_table *table;
2960
2961 if (net_eq(net, &init_net)) {
2962 table = vsock_table;
2963 } else {
2964 table = vsock_table_dup(net);
2965 if (!table)
2966 goto err_alloc;
2967 }
2968
2969 net->vsock.sysctl_hdr = register_net_sysctl_sz(net, "net/vsock", table,
2970 ARRAY_SIZE(vsock_table));
2971 if (!net->vsock.sysctl_hdr)
2972 goto err_reg;
2973
2974 return 0;
2975
2976err_reg:
2977 if (!net_eq(net, &init_net))
2978 kfree(table);
2979err_alloc:
2980 return -ENOMEM;
2981}
2982
2983static void vsock_sysctl_unregister(struct net *net)
2984{
2985 const struct ctl_table *table;
2986
2987 table = net->vsock.sysctl_hdr->ctl_table_arg;
2988 unregister_net_sysctl_table(net->vsock.sysctl_hdr);
2989 if (!net_eq(net, &init_net))
2990 kfree(table);
2991}
2992
2993static void vsock_net_init(struct net *net)
2994{
2995 if (net_eq(net, &init_net))
2996 net->vsock.mode = VSOCK_NET_MODE_GLOBAL;
2997 else
2998 net->vsock.mode = vsock_net_child_mode(current->nsproxy->net_ns);
2999
3000 net->vsock.child_ns_mode = net->vsock.mode;
3001 net->vsock.child_ns_mode_locked = 0;
3002 net->vsock.g2h_fallback = 1;
3003}
3004
3005static __net_init int vsock_sysctl_init_net(struct net *net)
3006{
3007 vsock_net_init(net);
3008
3009 if (vsock_sysctl_register(net))
3010 return -ENOMEM;
3011
3012 return 0;
3013}
3014
3015static __net_exit void vsock_sysctl_exit_net(struct net *net)
3016{
3017 vsock_sysctl_unregister(net);
3018}
3019
3020static struct pernet_operations vsock_sysctl_ops = {
3021 .init = vsock_sysctl_init_net,
3022 .exit = vsock_sysctl_exit_net,
3023};
3024
3025static int __init vsock_init(void)
3026{
3027 int err = 0;
3028
3029 vsock_init_tables();
3030
3031 vsock_proto.owner = THIS_MODULE;
3032 vsock_device.minor = MISC_DYNAMIC_MINOR;
3033 err = misc_register(&vsock_device);
3034 if (err) {
3035 pr_err("Failed to register misc device\n");
3036 goto err_reset_transport;
3037 }
3038
3039 err = proto_register(&vsock_proto, 1); /* we want our slab */
3040 if (err) {
3041 pr_err("Cannot register vsock protocol\n");
3042 goto err_deregister_misc;
3043 }
3044
3045 err = sock_register(&vsock_family_ops);
3046 if (err) {
3047 pr_err("could not register af_vsock (%d) address family: %d\n",
3048 AF_VSOCK, err);
3049 goto err_unregister_proto;
3050 }
3051
3052 if (register_pernet_subsys(&vsock_sysctl_ops)) {
3053 err = -ENOMEM;
3054 goto err_unregister_sock;
3055 }
3056
3057 vsock_bpf_build_proto();
3058
3059 return 0;
3060
3061err_unregister_sock:
3062 sock_unregister(AF_VSOCK);
3063err_unregister_proto:
3064 proto_unregister(&vsock_proto);
3065err_deregister_misc:
3066 misc_deregister(&vsock_device);
3067err_reset_transport:
3068 return err;
3069}
3070
3071static void __exit vsock_exit(void)
3072{
3073 misc_deregister(&vsock_device);
3074 sock_unregister(AF_VSOCK);
3075 proto_unregister(&vsock_proto);
3076 unregister_pernet_subsys(&vsock_sysctl_ops);
3077}
3078
3079const struct vsock_transport *vsock_core_get_transport(struct vsock_sock *vsk)
3080{
3081 return vsk->transport;
3082}
3083EXPORT_SYMBOL_GPL(vsock_core_get_transport);
3084
3085int vsock_core_register(const struct vsock_transport *t, int features)
3086{
3087 const struct vsock_transport *t_h2g, *t_g2h, *t_dgram, *t_local;
3088 int err = mutex_lock_interruptible(&vsock_register_mutex);
3089
3090 if (err)
3091 return err;
3092
3093 t_h2g = transport_h2g;
3094 t_g2h = transport_g2h;
3095 t_dgram = transport_dgram;
3096 t_local = transport_local;
3097
3098 if (features & VSOCK_TRANSPORT_F_H2G) {
3099 if (t_h2g) {
3100 err = -EBUSY;
3101 goto err_busy;
3102 }
3103 t_h2g = t;
3104 }
3105
3106 if (features & VSOCK_TRANSPORT_F_G2H) {
3107 if (t_g2h) {
3108 err = -EBUSY;
3109 goto err_busy;
3110 }
3111 t_g2h = t;
3112 }
3113
3114 if (features & VSOCK_TRANSPORT_F_DGRAM) {
3115 if (t_dgram) {
3116 err = -EBUSY;
3117 goto err_busy;
3118 }
3119 t_dgram = t;
3120 }
3121
3122 if (features & VSOCK_TRANSPORT_F_LOCAL) {
3123 if (t_local) {
3124 err = -EBUSY;
3125 goto err_busy;
3126 }
3127 t_local = t;
3128 }
3129
3130 transport_h2g = t_h2g;
3131 transport_g2h = t_g2h;
3132 transport_dgram = t_dgram;
3133 transport_local = t_local;
3134
3135err_busy:
3136 mutex_unlock(&vsock_register_mutex);
3137 return err;
3138}
3139EXPORT_SYMBOL_GPL(vsock_core_register);
3140
3141void vsock_core_unregister(const struct vsock_transport *t)
3142{
3143 mutex_lock(&vsock_register_mutex);
3144
3145 if (transport_h2g == t)
3146 transport_h2g = NULL;
3147
3148 if (transport_g2h == t)
3149 transport_g2h = NULL;
3150
3151 if (transport_dgram == t)
3152 transport_dgram = NULL;
3153
3154 if (transport_local == t)
3155 transport_local = NULL;
3156
3157 mutex_unlock(&vsock_register_mutex);
3158}
3159EXPORT_SYMBOL_GPL(vsock_core_unregister);
3160
3161module_init(vsock_init);
3162module_exit(vsock_exit);
3163
3164MODULE_AUTHOR("VMware, Inc.");
3165MODULE_DESCRIPTION("VMware Virtual Socket Family");
3166MODULE_VERSION("1.0.2.0-k");
3167MODULE_LICENSE("GPL v2");