projects/linux·blob master · refs

2364 lines · 57.2 KiB

1// SPDX-License-Identifier: GPL-2.0-or-later
2/*
3 * Copyright (C) 2006, 2007, 2009 Rusty Russell, IBM Corporation
4 * Copyright (C) 2009, 2010, 2011 Red Hat, Inc.
5 * Copyright (C) 2009, 2010, 2011 Amit Shah <[email protected]>
6 */
7#include <linux/cdev.h>
8#include <linux/debugfs.h>
9#include <linux/completion.h>
10#include <linux/device.h>
11#include <linux/err.h>
12#include <linux/freezer.h>
13#include <linux/fs.h>
14#include <linux/splice.h>
15#include <linux/pagemap.h>
16#include <linux/idr.h>
17#include <linux/init.h>
18#include <linux/list.h>
19#include <linux/poll.h>
20#include <linux/sched.h>
21#include <linux/slab.h>
22#include <linux/spinlock.h>
23#include <linux/virtio.h>
24#include <linux/virtio_console.h>
25#include <linux/wait.h>
26#include <linux/workqueue.h>
27#include <linux/module.h>
28#include <linux/dma-mapping.h>
29#include <linux/string_choices.h>
30#include "../tty/hvc/hvc_console.h"
31
32#define is_rproc_enabled IS_ENABLED(CONFIG_REMOTEPROC)
33#define VIRTCONS_MAX_PORTS 0x8000
34
35/*
36 * This is a global struct for storing common data for all the devices
37 * this driver handles.
38 *
39 * Mainly, it has a linked list for all the consoles in one place so
40 * that callbacks from hvc for get_chars(), put_chars() work properly
41 * across multiple devices and multiple ports per device.
42 */
43struct ports_driver_data {
44 /* Used for exporting per-port information to debugfs */
45 struct dentry *debugfs_dir;
46
47 /* List of all the devices we're handling */
48 struct list_head portdevs;
49
50 /* All the console devices handled by this driver */
51 struct list_head consoles;
52};
53
54static struct ports_driver_data pdrvdata;
55
56static const struct class port_class = {
57 .name = "virtio-ports",
58};
59
60static DEFINE_SPINLOCK(pdrvdata_lock);
61static DECLARE_COMPLETION(early_console_added);
62
63/* This struct holds information that's relevant only for console ports */
64struct console {
65 /* We'll place all consoles in a list in the pdrvdata struct */
66 struct list_head list;
67
68 /* The hvc device associated with this console port */
69 struct hvc_struct *hvc;
70
71 /* The size of the console */
72 struct winsize ws;
73
74 /*
75 * This number identifies the number that we used to register
76 * with hvc in hvc_instantiate() and hvc_alloc(); this is the
77 * number passed on by the hvc callbacks to us to
78 * differentiate between the other console ports handled by
79 * this driver
80 */
81 u32 vtermno;
82};
83
84static DEFINE_IDA(vtermno_ida);
85
86struct port_buffer {
87 char *buf;
88
89 /* size of the buffer in *buf above */
90 size_t size;
91
92 /* used length of the buffer */
93 size_t len;
94 /* offset in the buf from which to consume data */
95 size_t offset;
96
97 /* DMA address of buffer */
98 dma_addr_t dma;
99
100 /* Device we got DMA memory from */
101 struct device *dev;
102
103 /* List of pending dma buffers to free */
104 struct list_head list;
105
106 /* If sgpages == 0 then buf is used */
107 unsigned int sgpages;
108
109 /* sg is used if spages > 0. sg must be the last in is struct */
110 struct scatterlist sg[] __counted_by(sgpages);
111};
112
113/*
114 * This is a per-device struct that stores data common to all the
115 * ports for that device (vdev->priv).
116 */
117struct ports_device {
118 /* Next portdev in the list, head is in the pdrvdata struct */
119 struct list_head list;
120
121 /*
122 * Workqueue handlers where we process deferred work after
123 * notification
124 */
125 struct work_struct control_work;
126 struct work_struct config_work;
127
128 struct list_head ports;
129
130 /* To protect the list of ports */
131 spinlock_t ports_lock;
132
133 /* To protect the vq operations for the control channel */
134 spinlock_t c_ivq_lock;
135 spinlock_t c_ovq_lock;
136
137 /* max. number of ports this device can hold */
138 u32 max_nr_ports;
139
140 /* The virtio device we're associated with */
141 struct virtio_device *vdev;
142
143 /*
144 * A couple of virtqueues for the control channel: one for
145 * guest->host transfers, one for host->guest transfers
146 */
147 struct virtqueue *c_ivq, *c_ovq;
148
149 /*
150 * A control packet buffer for guest->host requests, protected
151 * by c_ovq_lock.
152 */
153 struct virtio_console_control cpkt;
154
155 /* Array of per-port IO virtqueues */
156 struct virtqueue **in_vqs, **out_vqs;
157
158 /* Major number for this device. Ports will be created as minors. */
159 int chr_major;
160
161 /*
162 * Set to true during PM freeze to block TX paths that may race
163 * with virtqueue teardown (e.g. hvc put_chars with no_console_suspend).
164 */
165 bool pm_freezing;
166};
167
168struct port_stats {
169 unsigned long bytes_sent, bytes_received, bytes_discarded;
170};
171
172/* This struct holds the per-port data */
173struct port {
174 /* Next port in the list, head is in the ports_device */
175 struct list_head list;
176
177 /* Pointer to the parent virtio_console device */
178 struct ports_device *portdev;
179
180 /* The current buffer from which data has to be fed to readers */
181 struct port_buffer *inbuf;
182
183 /*
184 * To protect the operations on the in_vq associated with this
185 * port. Has to be a spinlock because it can be called from
186 * interrupt context (get_char()).
187 */
188 spinlock_t inbuf_lock;
189
190 /* Protect the operations on the out_vq. */
191 spinlock_t outvq_lock;
192
193 /* The IO vqs for this port */
194 struct virtqueue *in_vq, *out_vq;
195
196 /* File in the debugfs directory that exposes this port's information */
197 struct dentry *debugfs_file;
198
199 /*
200 * Keep count of the bytes sent, received and discarded for
201 * this port for accounting and debugging purposes. These
202 * counts are not reset across port open / close events.
203 */
204 struct port_stats stats;
205
206 /*
207 * The entries in this struct will be valid if this port is
208 * hooked up to an hvc console
209 */
210 struct console cons;
211
212 /* Each port associates with a separate char device */
213 struct cdev *cdev;
214 struct device *dev;
215
216 /* Reference-counting to handle port hot-unplugs and file operations */
217 struct kref kref;
218
219 /* A waitqueue for poll() or blocking read operations */
220 wait_queue_head_t waitqueue;
221
222 /* The 'name' of the port that we expose via sysfs properties */
223 char *name;
224
225 /* We can notify apps of host connect / disconnect events via SIGIO */
226 struct fasync_struct *async_queue;
227
228 /* The 'id' to identify the port with the Host */
229 u32 id;
230
231 bool outvq_full;
232
233 /* Is the host device open */
234 bool host_connected;
235
236 /* We should allow only one process to open a port */
237 bool guest_connected;
238};
239
240static struct port *find_port_by_vtermno(u32 vtermno)
241{
242 struct port *port;
243 struct console *cons;
244 unsigned long flags;
245
246 spin_lock_irqsave(&pdrvdata_lock, flags);
247 list_for_each_entry(cons, &pdrvdata.consoles, list) {
248 if (cons->vtermno == vtermno) {
249 port = container_of(cons, struct port, cons);
250 goto out;
251 }
252 }
253 port = NULL;
254out:
255 spin_unlock_irqrestore(&pdrvdata_lock, flags);
256 return port;
257}
258
259static struct port *find_port_by_devt_in_portdev(struct ports_device *portdev,
260 dev_t dev)
261{
262 struct port *port;
263 unsigned long flags;
264
265 spin_lock_irqsave(&portdev->ports_lock, flags);
266 list_for_each_entry(port, &portdev->ports, list) {
267 if (port->cdev->dev == dev) {
268 kref_get(&port->kref);
269 goto out;
270 }
271 }
272 port = NULL;
273out:
274 spin_unlock_irqrestore(&portdev->ports_lock, flags);
275
276 return port;
277}
278
279static struct port *find_port_by_devt(dev_t dev)
280{
281 struct ports_device *portdev;
282 struct port *port;
283 unsigned long flags;
284
285 spin_lock_irqsave(&pdrvdata_lock, flags);
286 list_for_each_entry(portdev, &pdrvdata.portdevs, list) {
287 port = find_port_by_devt_in_portdev(portdev, dev);
288 if (port)
289 goto out;
290 }
291 port = NULL;
292out:
293 spin_unlock_irqrestore(&pdrvdata_lock, flags);
294 return port;
295}
296
297static struct port *find_port_by_id(struct ports_device *portdev, u32 id)
298{
299 struct port *port;
300 unsigned long flags;
301
302 spin_lock_irqsave(&portdev->ports_lock, flags);
303 list_for_each_entry(port, &portdev->ports, list)
304 if (port->id == id)
305 goto out;
306 port = NULL;
307out:
308 spin_unlock_irqrestore(&portdev->ports_lock, flags);
309
310 return port;
311}
312
313/*
314 * Finds a port by the virtqueue and returns a pointer to struct port
315 * with the reference count incremented.
316 *
317 * Callers MUST decrement it when finished.
318 */
319static struct port *find_port_by_vq(struct ports_device *portdev,
320 struct virtqueue *vq)
321{
322 struct port *port;
323 unsigned long flags;
324
325 spin_lock_irqsave(&portdev->ports_lock, flags);
326 list_for_each_entry(port, &portdev->ports, list)
327 if (port->in_vq == vq || port->out_vq == vq) {
328 kref_get(&port->kref);
329 goto out;
330 }
331 port = NULL;
332out:
333 spin_unlock_irqrestore(&portdev->ports_lock, flags);
334 return port;
335}
336
337static bool is_console_port(struct port *port)
338{
339 if (port->cons.hvc)
340 return true;
341 return false;
342}
343
344static bool is_rproc_serial(const struct virtio_device *vdev)
345{
346 return is_rproc_enabled && vdev->id.device == VIRTIO_ID_RPROC_SERIAL;
347}
348
349static inline bool use_multiport(struct ports_device *portdev)
350{
351 /*
352 * This condition can be true when put_chars is called from
353 * early_init
354 */
355 if (!portdev->vdev)
356 return false;
357 return __virtio_test_bit(portdev->vdev, VIRTIO_CONSOLE_F_MULTIPORT);
358}
359
360static DEFINE_SPINLOCK(dma_bufs_lock);
361static LIST_HEAD(pending_free_dma_bufs);
362
363static void free_buf(struct port_buffer *buf, bool can_sleep)
364{
365 unsigned int i;
366
367 for (i = 0; i < buf->sgpages; i++) {
368 struct page *page = sg_page(&buf->sg[i]);
369 if (!page)
370 break;
371 put_page(page);
372 }
373
374 if (!buf->dev) {
375 kfree(buf->buf);
376 } else if (is_rproc_enabled) {
377 unsigned long flags;
378
379 /* dma_free_coherent requires interrupts to be enabled. */
380 if (!can_sleep) {
381 /* queue up dma-buffers to be freed later */
382 spin_lock_irqsave(&dma_bufs_lock, flags);
383 list_add_tail(&buf->list, &pending_free_dma_bufs);
384 spin_unlock_irqrestore(&dma_bufs_lock, flags);
385 return;
386 }
387 dma_free_coherent(buf->dev, buf->size, buf->buf, buf->dma);
388
389 /* Release device refcnt and allow it to be freed */
390 put_device(buf->dev);
391 }
392
393 kfree(buf);
394}
395
396static void reclaim_dma_bufs(void)
397{
398 unsigned long flags;
399 struct port_buffer *buf, *tmp;
400 LIST_HEAD(tmp_list);
401
402 if (list_empty(&pending_free_dma_bufs))
403 return;
404
405 /* Create a copy of the pending_free_dma_bufs while holding the lock */
406 spin_lock_irqsave(&dma_bufs_lock, flags);
407 list_cut_position(&tmp_list, &pending_free_dma_bufs,
408 pending_free_dma_bufs.prev);
409 spin_unlock_irqrestore(&dma_bufs_lock, flags);
410
411 /* Release the dma buffers, without irqs enabled */
412 list_for_each_entry_safe(buf, tmp, &tmp_list, list) {
413 list_del(&buf->list);
414 free_buf(buf, true);
415 }
416}
417
418static struct port_buffer *alloc_buf(struct virtio_device *vdev, size_t buf_size,
419 int pages, gfp_t gfp)
420{
421 struct port_buffer *buf;
422
423 reclaim_dma_bufs();
424
425 /*
426 * Allocate buffer and the sg list. The sg list array is allocated
427 * directly after the port_buffer struct.
428 */
429 buf = kmalloc_flex(*buf, sg, pages, gfp);
430 if (!buf)
431 goto fail;
432
433 buf->sgpages = pages;
434 if (pages > 0) {
435 buf->dev = NULL;
436 buf->buf = NULL;
437 return buf;
438 }
439
440 if (is_rproc_serial(vdev)) {
441 /*
442 * Allocate DMA memory from ancestor. When a virtio
443 * device is created by remoteproc, the DMA memory is
444 * associated with the parent device:
445 * virtioY => remoteprocX#vdevYbuffer.
446 */
447 buf->dev = vdev->dev.parent;
448 if (!buf->dev)
449 goto free_buf;
450
451 /* Increase device refcnt to avoid freeing it */
452 get_device(buf->dev);
453 buf->buf = dma_alloc_coherent(buf->dev, buf_size, &buf->dma, gfp);
454 } else {
455 buf->dev = NULL;
456 buf->buf = kmalloc(buf_size, gfp);
457 }
458
459 if (!buf->buf)
460 goto free_buf;
461 buf->len = 0;
462 buf->offset = 0;
463 buf->size = buf_size;
464 return buf;
465
466free_buf:
467 kfree(buf);
468fail:
469 return NULL;
470}
471
472/* Callers should take appropriate locks */
473static struct port_buffer *get_inbuf(struct port *port)
474{
475 struct port_buffer *buf;
476 unsigned int len;
477
478 if (port->inbuf)
479 return port->inbuf;
480
481 buf = virtqueue_get_buf(port->in_vq, &len);
482 if (buf) {
483 buf->len = min_t(size_t, len, buf->size);
484 buf->offset = 0;
485 port->stats.bytes_received += len;
486 }
487 return buf;
488}
489
490/*
491 * Create a scatter-gather list representing our input buffer and put
492 * it in the queue.
493 *
494 * Callers should take appropriate locks.
495 */
496static int add_inbuf(struct virtqueue *vq, struct port_buffer *buf)
497{
498 struct scatterlist sg[1];
499 int ret;
500
501 sg_init_one(sg, buf->buf, buf->size);
502
503 ret = virtqueue_add_inbuf(vq, sg, 1, buf, GFP_ATOMIC);
504 virtqueue_kick(vq);
505 if (!ret)
506 ret = vq->num_free;
507 return ret;
508}
509
510/* Discard any unread data this port has. Callers lockers. */
511static void discard_port_data(struct port *port)
512{
513 struct port_buffer *buf;
514 unsigned int err;
515
516 if (!port->portdev) {
517 /* Device has been unplugged. vqs are already gone. */
518 return;
519 }
520 buf = get_inbuf(port);
521
522 err = 0;
523 while (buf) {
524 port->stats.bytes_discarded += buf->len - buf->offset;
525 if (add_inbuf(port->in_vq, buf) < 0) {
526 err++;
527 free_buf(buf, false);
528 }
529 port->inbuf = NULL;
530 buf = get_inbuf(port);
531 }
532 if (err)
533 dev_warn(port->dev, "Errors adding %d buffers back to vq\n",
534 err);
535}
536
537static bool port_has_data(struct port *port)
538{
539 unsigned long flags;
540 bool ret;
541
542 ret = false;
543 spin_lock_irqsave(&port->inbuf_lock, flags);
544 port->inbuf = get_inbuf(port);
545 if (port->inbuf)
546 ret = true;
547
548 spin_unlock_irqrestore(&port->inbuf_lock, flags);
549 return ret;
550}
551
552static ssize_t __send_control_msg(struct ports_device *portdev, u32 port_id,
553 unsigned int event, unsigned int value)
554{
555 struct scatterlist sg[1];
556 struct virtqueue *vq;
557 unsigned int len;
558
559 if (!use_multiport(portdev))
560 return 0;
561
562 vq = portdev->c_ovq;
563
564 spin_lock(&portdev->c_ovq_lock);
565
566 portdev->cpkt.id = cpu_to_virtio32(portdev->vdev, port_id);
567 portdev->cpkt.event = cpu_to_virtio16(portdev->vdev, event);
568 portdev->cpkt.value = cpu_to_virtio16(portdev->vdev, value);
569
570 sg_init_one(sg, &portdev->cpkt, sizeof(struct virtio_console_control));
571
572 if (virtqueue_add_outbuf(vq, sg, 1, &portdev->cpkt, GFP_ATOMIC) == 0) {
573 virtqueue_kick(vq);
574 while (!virtqueue_get_buf(vq, &len)
575 && !virtqueue_is_broken(vq))
576 cpu_relax();
577 }
578
579 spin_unlock(&portdev->c_ovq_lock);
580 return 0;
581}
582
583static ssize_t send_control_msg(struct port *port, unsigned int event,
584 unsigned int value)
585{
586 /* Did the port get unplugged before userspace closed it? */
587 if (port->portdev)
588 return __send_control_msg(port->portdev, port->id, event, value);
589 return 0;
590}
591
592
593/* Callers must take the port->outvq_lock */
594static void reclaim_consumed_buffers(struct port *port)
595{
596 struct port_buffer *buf;
597 unsigned int len;
598
599 if (!port->portdev) {
600 /* Device has been unplugged. vqs are already gone. */
601 return;
602 }
603 while ((buf = virtqueue_get_buf(port->out_vq, &len))) {
604 free_buf(buf, false);
605 port->outvq_full = false;
606 }
607}
608
609static ssize_t __send_to_port(struct port *port, struct scatterlist *sg,
610 int nents, size_t in_count,
611 struct port_buffer *buf, bool nonblock)
612{
613 struct virtqueue *out_vq;
614 int err;
615 unsigned long flags;
616 unsigned int len;
617 struct ports_device *portdev;
618
619 spin_lock_irqsave(&port->outvq_lock, flags);
620
621 portdev = READ_ONCE(port->portdev);
622
623 if (!portdev) {
624 in_count = 0;
625 goto free_and_done;
626 }
627
628 /*
629 * Check freeze flag under the lock so that the flag check and
630 * virtqueue_add_outbuf() are atomic with respect to
631 * remove_port_data() which also takes outvq_lock. This
632 * guarantees that once remove_port_data() returns, no new
633 * buffers can be added before remove_vqs() tears down the vq.
634 * Pairs with smp_store_release() in virtcons_freeze/restore.
635 */
636 if (smp_load_acquire(&portdev->pm_freezing)) /* pairs with freeze/restore */
637 goto free_and_done;
638
639 out_vq = port->out_vq;
640
641 reclaim_consumed_buffers(port);
642
643 err = virtqueue_add_outbuf(out_vq, sg, nents, buf, GFP_ATOMIC);
644
645 /* Tell Host to go! */
646 virtqueue_kick(out_vq);
647
648 if (err) {
649 in_count = 0;
650 goto free_and_done;
651 }
652
653 if (out_vq->num_free == 0)
654 port->outvq_full = true;
655
656 if (nonblock)
657 goto done;
658
659 /*
660 * Wait till the host acknowledges it pushed out the data we
661 * sent. This is done for data from the hvc_console; the tty
662 * operations are performed with spinlocks held so we can't
663 * sleep here. An alternative would be to copy the data to a
664 * buffer and relax the spinning requirement. The downside is
665 * we need to kmalloc a GFP_ATOMIC buffer each time the
666 * console driver writes something out.
667 *
668 * Spin until host returns the buffer.
669 * Capture the returned buf so we can free it.
670 * If broken, buf == NULL and buf stays in the vq;
671 * remove_vqs() will call virtqueue_detach_unused_buf() -> free_buf().
672 */
673 while (!(buf = virtqueue_get_buf(out_vq, &len))
674 && !virtqueue_is_broken(out_vq))
675 cpu_relax();
676
677free_and_done:
678 if (buf)
679 free_buf(buf, false);
680done:
681 spin_unlock_irqrestore(&port->outvq_lock, flags);
682
683 port->stats.bytes_sent += in_count;
684 /*
685 * We're expected to return the amount of data we wrote -- all
686 * of it
687 */
688 return in_count;
689}
690
691/*
692 * Give out the data that's requested from the buffer that we have
693 * queued up.
694 */
695static ssize_t fill_readbuf(struct port *port, u8 __user *out_buf,
696 size_t out_count, bool to_user)
697{
698 struct port_buffer *buf;
699 unsigned long flags;
700
701 if (!out_count || !port_has_data(port))
702 return 0;
703
704 buf = port->inbuf;
705 out_count = min(out_count, buf->len - buf->offset);
706
707 if (to_user) {
708 ssize_t ret;
709
710 ret = copy_to_user(out_buf, buf->buf + buf->offset, out_count);
711 if (ret)
712 return -EFAULT;
713 } else {
714 memcpy((__force u8 *)out_buf, buf->buf + buf->offset,
715 out_count);
716 }
717
718 buf->offset += out_count;
719
720 if (buf->offset == buf->len) {
721 /*
722 * We're done using all the data in this buffer.
723 * Re-queue so that the Host can send us more data.
724 */
725 spin_lock_irqsave(&port->inbuf_lock, flags);
726 port->inbuf = NULL;
727
728 if (add_inbuf(port->in_vq, buf) < 0)
729 dev_warn(port->dev, "failed add_buf\n");
730
731 spin_unlock_irqrestore(&port->inbuf_lock, flags);
732 }
733 /* Return the number of bytes actually copied */
734 return out_count;
735}
736
737/* The condition that must be true for polling to end */
738static bool will_read_block(struct port *port)
739{
740 if (!port->guest_connected) {
741 /* Port got hot-unplugged. Let's exit. */
742 return false;
743 }
744 return !port_has_data(port) && port->host_connected;
745}
746
747static bool will_write_block(struct port *port)
748{
749 bool ret;
750
751 if (!port->guest_connected) {
752 /* Port got hot-unplugged. Let's exit. */
753 return false;
754 }
755 if (!port->host_connected)
756 return true;
757
758 spin_lock_irq(&port->outvq_lock);
759 /*
760 * Check if the Host has consumed any buffers since we last
761 * sent data (this is only applicable for nonblocking ports).
762 */
763 reclaim_consumed_buffers(port);
764 ret = port->outvq_full;
765 spin_unlock_irq(&port->outvq_lock);
766
767 return ret;
768}
769
770static ssize_t port_fops_read(struct file *filp, char __user *ubuf,
771 size_t count, loff_t *offp)
772{
773 struct port *port;
774 ssize_t ret;
775
776 port = filp->private_data;
777
778 /* Port is hot-unplugged. */
779 if (!port->guest_connected)
780 return -ENODEV;
781
782 if (!port_has_data(port)) {
783 /*
784 * If nothing's connected on the host just return 0 in
785 * case of list_empty; this tells the userspace app
786 * that there's no connection
787 */
788 if (!port->host_connected)
789 return 0;
790 if (filp->f_flags & O_NONBLOCK)
791 return -EAGAIN;
792
793 ret = wait_event_freezable(port->waitqueue,
794 !will_read_block(port));
795 if (ret < 0)
796 return ret;
797 }
798 /* Port got hot-unplugged while we were waiting above. */
799 if (!port->guest_connected)
800 return -ENODEV;
801 /*
802 * We could've received a disconnection message while we were
803 * waiting for more data.
804 *
805 * This check is not clubbed in the if() statement above as we
806 * might receive some data as well as the host could get
807 * disconnected after we got woken up from our wait. So we
808 * really want to give off whatever data we have and only then
809 * check for host_connected.
810 */
811 if (!port_has_data(port) && !port->host_connected)
812 return 0;
813
814 return fill_readbuf(port, ubuf, count, true);
815}
816
817static int wait_port_writable(struct port *port, bool nonblock)
818{
819 int ret;
820
821 if (will_write_block(port)) {
822 if (nonblock)
823 return -EAGAIN;
824
825 ret = wait_event_freezable(port->waitqueue,
826 !will_write_block(port));
827 if (ret < 0)
828 return ret;
829 }
830 /* Port got hot-unplugged. */
831 if (!port->guest_connected)
832 return -ENODEV;
833
834 return 0;
835}
836
837static ssize_t port_fops_write(struct file *filp, const char __user *ubuf,
838 size_t count, loff_t *offp)
839{
840 struct port *port;
841 struct port_buffer *buf;
842 ssize_t ret;
843 bool nonblock;
844 struct scatterlist sg[1];
845
846 /* Userspace could be out to fool us */
847 if (!count)
848 return 0;
849
850 port = filp->private_data;
851
852 nonblock = filp->f_flags & O_NONBLOCK;
853
854 ret = wait_port_writable(port, nonblock);
855 if (ret < 0)
856 return ret;
857
858 count = min((size_t)(32 * 1024), count);
859
860 buf = alloc_buf(port->portdev->vdev, count, 0, GFP_KERNEL);
861 if (!buf)
862 return -ENOMEM;
863
864 ret = copy_from_user(buf->buf, ubuf, count);
865 if (ret) {
866 free_buf(buf, true);
867 return -EFAULT;
868 }
869
870 /*
871 * We now ask send_buf() to not spin for generic ports -- we
872 * can re-use the same code path that non-blocking file
873 * descriptors take for blocking file descriptors since the
874 * wait is already done and we're certain the write will go
875 * through to the host.
876 */
877 nonblock = true;
878 sg_init_one(sg, buf->buf, count);
879 return __send_to_port(port, sg, 1, count, buf, nonblock);
880}
881
882struct sg_list {
883 unsigned int n;
884 unsigned int size;
885 size_t len;
886 struct scatterlist *sg;
887};
888
889static int pipe_to_sg(struct pipe_inode_info *pipe, struct pipe_buffer *buf,
890 struct splice_desc *sd)
891{
892 struct sg_list *sgl = sd->u.data;
893 unsigned int offset, len;
894
895 if (sgl->n == sgl->size)
896 return 0;
897
898 /* Try lock this page */
899 if (pipe_buf_try_steal(pipe, buf)) {
900 /* Get reference and unlock page for moving */
901 get_page(buf->page);
902 unlock_page(buf->page);
903
904 len = min(buf->len, sd->len);
905 sg_set_page(&(sgl->sg[sgl->n]), buf->page, len, buf->offset);
906 } else {
907 /* Failback to copying a page */
908 struct page *page = alloc_page(GFP_KERNEL);
909 char *src;
910
911 if (!page)
912 return -ENOMEM;
913
914 offset = sd->pos & ~PAGE_MASK;
915
916 len = sd->len;
917 if (len + offset > PAGE_SIZE)
918 len = PAGE_SIZE - offset;
919
920 src = kmap_local_page(buf->page);
921 memcpy(page_address(page) + offset, src + buf->offset, len);
922 kunmap_local(src);
923
924 sg_set_page(&(sgl->sg[sgl->n]), page, len, offset);
925 }
926 sgl->n++;
927 sgl->len += len;
928
929 return len;
930}
931
932/* Faster zero-copy write by splicing */
933static ssize_t port_fops_splice_write(struct pipe_inode_info *pipe,
934 struct file *filp, loff_t *ppos,
935 size_t len, unsigned int flags)
936{
937 struct port *port = filp->private_data;
938 struct sg_list sgl;
939 ssize_t ret;
940 struct port_buffer *buf;
941 struct splice_desc sd = {
942 .total_len = len,
943 .flags = flags,
944 .pos = *ppos,
945 .u.data = &sgl,
946 };
947 unsigned int occupancy;
948
949 /*
950 * Rproc_serial does not yet support splice. To support splice
951 * pipe_to_sg() must allocate dma-buffers and copy content from
952 * regular pages to dma pages. And alloc_buf and free_buf must
953 * support allocating and freeing such a list of dma-buffers.
954 */
955 if (is_rproc_serial(port->out_vq->vdev))
956 return -EINVAL;
957
958 pipe_lock(pipe);
959 ret = 0;
960 if (pipe_is_empty(pipe))
961 goto error_out;
962
963 ret = wait_port_writable(port, filp->f_flags & O_NONBLOCK);
964 if (ret < 0)
965 goto error_out;
966
967 occupancy = pipe_buf_usage(pipe);
968 buf = alloc_buf(port->portdev->vdev, 0, occupancy, GFP_KERNEL);
969
970 if (!buf) {
971 ret = -ENOMEM;
972 goto error_out;
973 }
974
975 sgl.n = 0;
976 sgl.len = 0;
977 sgl.size = occupancy;
978 sgl.sg = buf->sg;
979 sg_init_table(sgl.sg, sgl.size);
980 ret = __splice_from_pipe(pipe, &sd, pipe_to_sg);
981 pipe_unlock(pipe);
982
983 if (likely(ret > 0))
984 ret = __send_to_port(port, buf->sg, sgl.n, sgl.len, buf, true);
985 else
986 free_buf(buf, true);
987
988 return ret;
989
990error_out:
991 pipe_unlock(pipe);
992 return ret;
993}
994
995static __poll_t port_fops_poll(struct file *filp, poll_table *wait)
996{
997 struct port *port;
998 __poll_t ret;
999
1000 port = filp->private_data;
1001 poll_wait(filp, &port->waitqueue, wait);
1002
1003 if (!port->guest_connected) {
1004 /* Port got unplugged */
1005 return EPOLLHUP;
1006 }
1007 ret = 0;
1008 if (!will_read_block(port))
1009 ret |= EPOLLIN | EPOLLRDNORM;
1010 if (!will_write_block(port))
1011 ret |= EPOLLOUT;
1012 if (!port->host_connected)
1013 ret |= EPOLLHUP;
1014
1015 return ret;
1016}
1017
1018static void remove_port(struct kref *kref);
1019
1020static int port_fops_release(struct inode *inode, struct file *filp)
1021{
1022 struct port *port;
1023
1024 port = filp->private_data;
1025
1026 /* Notify host of port being closed */
1027 send_control_msg(port, VIRTIO_CONSOLE_PORT_OPEN, 0);
1028
1029 spin_lock_irq(&port->inbuf_lock);
1030 port->guest_connected = false;
1031
1032 discard_port_data(port);
1033
1034 spin_unlock_irq(&port->inbuf_lock);
1035
1036 spin_lock_irq(&port->outvq_lock);
1037 reclaim_consumed_buffers(port);
1038 spin_unlock_irq(&port->outvq_lock);
1039
1040 reclaim_dma_bufs();
1041 /*
1042 * Locks aren't necessary here as a port can't be opened after
1043 * unplug, and if a port isn't unplugged, a kref would already
1044 * exist for the port. Plus, taking ports_lock here would
1045 * create a dependency on other locks taken by functions
1046 * inside remove_port if we're the last holder of the port,
1047 * creating many problems.
1048 */
1049 kref_put(&port->kref, remove_port);
1050
1051 return 0;
1052}
1053
1054static int port_fops_open(struct inode *inode, struct file *filp)
1055{
1056 struct cdev *cdev = inode->i_cdev;
1057 struct port *port;
1058 int ret;
1059
1060 /* We get the port with a kref here */
1061 port = find_port_by_devt(cdev->dev);
1062 if (!port) {
1063 /* Port was unplugged before we could proceed */
1064 return -ENXIO;
1065 }
1066 filp->private_data = port;
1067
1068 /*
1069 * Don't allow opening of console port devices -- that's done
1070 * via /dev/hvc
1071 */
1072 if (is_console_port(port)) {
1073 ret = -ENXIO;
1074 goto out;
1075 }
1076
1077 /* Allow only one process to open a particular port at a time */
1078 spin_lock_irq(&port->inbuf_lock);
1079 if (port->guest_connected) {
1080 spin_unlock_irq(&port->inbuf_lock);
1081 ret = -EBUSY;
1082 goto out;
1083 }
1084
1085 port->guest_connected = true;
1086 spin_unlock_irq(&port->inbuf_lock);
1087
1088 spin_lock_irq(&port->outvq_lock);
1089 /*
1090 * There might be a chance that we missed reclaiming a few
1091 * buffers in the window of the port getting previously closed
1092 * and opening now.
1093 */
1094 reclaim_consumed_buffers(port);
1095 spin_unlock_irq(&port->outvq_lock);
1096
1097 nonseekable_open(inode, filp);
1098
1099 /* Notify host of port being opened */
1100 send_control_msg(filp->private_data, VIRTIO_CONSOLE_PORT_OPEN, 1);
1101
1102 return 0;
1103out:
1104 kref_put(&port->kref, remove_port);
1105 return ret;
1106}
1107
1108static int port_fops_fasync(int fd, struct file *filp, int mode)
1109{
1110 struct port *port;
1111
1112 port = filp->private_data;
1113 return fasync_helper(fd, filp, mode, &port->async_queue);
1114}
1115
1116/*
1117 * The file operations that we support: programs in the guest can open
1118 * a console device, read from it, write to it, poll for data and
1119 * close it. The devices are at
1120 * /dev/vport<device number>p<port number>
1121 */
1122static const struct file_operations port_fops = {
1123 .owner = THIS_MODULE,
1124 .open = port_fops_open,
1125 .read = port_fops_read,
1126 .write = port_fops_write,
1127 .splice_write = port_fops_splice_write,
1128 .poll = port_fops_poll,
1129 .release = port_fops_release,
1130 .fasync = port_fops_fasync,
1131};
1132
1133/*
1134 * The put_chars() callback is pretty straightforward.
1135 *
1136 * We turn the characters into a scatter-gather list, add it to the
1137 * output queue and then kick the Host. Then we sit here waiting for
1138 * it to finish: inefficient in theory, but in practice
1139 * implementations will do it immediately.
1140 */
1141static ssize_t put_chars(u32 vtermno, const u8 *buf, size_t count)
1142{
1143 struct port *port;
1144 struct scatterlist sg[1];
1145 struct port_buffer *pbuf;
1146 struct ports_device *portdev;
1147
1148 port = find_port_by_vtermno(vtermno);
1149 if (!port)
1150 return -EPIPE;
1151
1152 /*
1153 * Silently drop output in two cases, both by returning count so
1154 * that the hvc layer does not spin-retry:
1155 *
1156 * 1. Device hot-unplug (!portdev): portdev was NULLed by
1157 * unplug_port() after hvc_remove() was already called, so
1158 * the hvc layer will stop invoking put_chars() very soon.
1159 * Returning count avoids a pointless retry loop in the
1160 * interim.
1161 *
1162 * 2. PM freeze (pm_freezing): the hvc console stays active
1163 * under no_console_suspend but virtqueues are being torn
1164 * down. Drop the output silently so the hvc layer does not
1165 * stall suspend.
1166 *
1167 * This early check avoids a pointless GFP_ATOMIC allocation;
1168 * __send_to_port() rechecks under outvq_lock for correctness.
1169 * Pairs with smp_store_release() in virtcons_freeze/restore.
1170 */
1171 portdev = READ_ONCE(port->portdev);
1172 if (!portdev ||
1173 smp_load_acquire(&portdev->pm_freezing)) /* pairs with freeze/restore */
1174 return count;
1175
1176 pbuf = alloc_buf(portdev->vdev, count, 0, GFP_ATOMIC);
1177 if (!pbuf)
1178 return -ENOMEM;
1179
1180 memcpy(pbuf->buf, buf, count);
1181 pbuf->len = count;
1182 sg_init_one(sg, pbuf->buf, count);
1183
1184 /*
1185 * Ownership of pbuf is transferred to __send_to_port().
1186 * Do not touch or free pbuf after this call.
1187 */
1188 return __send_to_port(port, sg, 1, count, pbuf, false);
1189}
1190
1191/*
1192 * get_chars() is the callback from the hvc_console infrastructure
1193 * when an interrupt is received.
1194 *
1195 * We call out to fill_readbuf that gets us the required data from the
1196 * buffers that are queued up.
1197 */
1198static ssize_t get_chars(u32 vtermno, u8 *buf, size_t count)
1199{
1200 struct port *port;
1201
1202 port = find_port_by_vtermno(vtermno);
1203 if (!port)
1204 return -EPIPE;
1205
1206 /* If we don't have an input queue yet, we can't get input. */
1207 BUG_ON(!port->in_vq);
1208
1209 return fill_readbuf(port, (__force u8 __user *)buf, count, false);
1210}
1211
1212static void resize_console(struct port *port)
1213{
1214 struct virtio_device *vdev;
1215
1216 /* The port could have been hot-unplugged */
1217 if (!port || !is_console_port(port))
1218 return;
1219
1220 vdev = port->portdev->vdev;
1221
1222 /* Don't test F_SIZE at all if we're rproc: not a valid feature! */
1223 if (!is_rproc_serial(vdev) &&
1224 virtio_has_feature(vdev, VIRTIO_CONSOLE_F_SIZE))
1225 hvc_resize(port->cons.hvc, port->cons.ws);
1226}
1227
1228/* We set the configuration at this point, since we now have a tty */
1229static int notifier_add_vio(struct hvc_struct *hp, int data)
1230{
1231 struct port *port;
1232
1233 port = find_port_by_vtermno(hp->vtermno);
1234 if (!port)
1235 return -EINVAL;
1236
1237 hp->irq_requested = 1;
1238 resize_console(port);
1239
1240 return 0;
1241}
1242
1243static void notifier_del_vio(struct hvc_struct *hp, int data)
1244{
1245 hp->irq_requested = 0;
1246}
1247
1248/* The operations for console ports. */
1249static const struct hv_ops hv_ops = {
1250 .get_chars = get_chars,
1251 .put_chars = put_chars,
1252 .notifier_add = notifier_add_vio,
1253 .notifier_del = notifier_del_vio,
1254 .notifier_hangup = notifier_del_vio,
1255};
1256
1257static int init_port_console(struct port *port)
1258{
1259 int ret;
1260
1261 /*
1262 * The Host's telling us this port is a console port. Hook it
1263 * up with an hvc console.
1264 *
1265 * To set up and manage our virtual console, we call
1266 * hvc_alloc().
1267 *
1268 * The first argument of hvc_alloc() is the virtual console
1269 * number. The second argument is the parameter for the
1270 * notification mechanism (like irq number). We currently
1271 * leave this as zero, virtqueues have implicit notifications.
1272 *
1273 * The third argument is a "struct hv_ops" containing the
1274 * put_chars() get_chars(), notifier_add() and notifier_del()
1275 * pointers. The final argument is the output buffer size: we
1276 * can do any size, so we put PAGE_SIZE here.
1277 */
1278 ret = ida_alloc_min(&vtermno_ida, 1, GFP_KERNEL);
1279 if (ret < 0)
1280 return ret;
1281
1282 port->cons.vtermno = ret;
1283 port->cons.hvc = hvc_alloc(port->cons.vtermno, 0, &hv_ops, PAGE_SIZE);
1284 if (IS_ERR(port->cons.hvc)) {
1285 ret = PTR_ERR(port->cons.hvc);
1286 dev_err(port->dev,
1287 "error %d allocating hvc for port\n", ret);
1288 port->cons.hvc = NULL;
1289 ida_free(&vtermno_ida, port->cons.vtermno);
1290 return ret;
1291 }
1292 spin_lock_irq(&pdrvdata_lock);
1293 list_add_tail(&port->cons.list, &pdrvdata.consoles);
1294 spin_unlock_irq(&pdrvdata_lock);
1295 port->guest_connected = true;
1296
1297 /* Notify host of port being opened */
1298 send_control_msg(port, VIRTIO_CONSOLE_PORT_OPEN, 1);
1299
1300 return 0;
1301}
1302
1303static ssize_t show_port_name(struct device *dev,
1304 struct device_attribute *attr, char *buffer)
1305{
1306 struct port *port;
1307
1308 port = dev_get_drvdata(dev);
1309
1310 return sprintf(buffer, "%s\n", port->name);
1311}
1312
1313static DEVICE_ATTR(name, S_IRUGO, show_port_name, NULL);
1314
1315static struct attribute *port_sysfs_entries[] = {
1316 &dev_attr_name.attr,
1317 NULL
1318};
1319
1320static const struct attribute_group port_attribute_group = {
1321 .name = NULL, /* put in device directory */
1322 .attrs = port_sysfs_entries,
1323};
1324
1325static int port_debugfs_show(struct seq_file *s, void *data)
1326{
1327 struct port *port = s->private;
1328
1329 seq_printf(s, "name: %s\n", port->name ? port->name : "");
1330 seq_printf(s, "guest_connected: %d\n", port->guest_connected);
1331 seq_printf(s, "host_connected: %d\n", port->host_connected);
1332 seq_printf(s, "outvq_full: %d\n", port->outvq_full);
1333 seq_printf(s, "bytes_sent: %lu\n", port->stats.bytes_sent);
1334 seq_printf(s, "bytes_received: %lu\n", port->stats.bytes_received);
1335 seq_printf(s, "bytes_discarded: %lu\n", port->stats.bytes_discarded);
1336 seq_printf(s, "is_console: %s\n", str_yes_no(is_console_port(port)));
1337 seq_printf(s, "console_vtermno: %u\n", port->cons.vtermno);
1338
1339 return 0;
1340}
1341
1342DEFINE_SHOW_ATTRIBUTE(port_debugfs);
1343
1344static void set_console_size(struct port *port, u16 rows, u16 cols)
1345{
1346 if (!port || !is_console_port(port))
1347 return;
1348
1349 port->cons.ws.ws_row = rows;
1350 port->cons.ws.ws_col = cols;
1351}
1352
1353static int fill_queue(struct virtqueue *vq, spinlock_t *lock)
1354{
1355 struct port_buffer *buf;
1356 int nr_added_bufs;
1357 int ret;
1358
1359 nr_added_bufs = 0;
1360 do {
1361 buf = alloc_buf(vq->vdev, PAGE_SIZE, 0, GFP_KERNEL);
1362 if (!buf)
1363 return -ENOMEM;
1364
1365 spin_lock_irq(lock);
1366 ret = add_inbuf(vq, buf);
1367 if (ret < 0) {
1368 spin_unlock_irq(lock);
1369 free_buf(buf, true);
1370 return ret;
1371 }
1372 nr_added_bufs++;
1373 spin_unlock_irq(lock);
1374 } while (ret > 0);
1375
1376 return nr_added_bufs;
1377}
1378
1379static void send_sigio_to_port(struct port *port)
1380{
1381 if (port->async_queue && port->guest_connected)
1382 kill_fasync(&port->async_queue, SIGIO, POLL_OUT);
1383}
1384
1385static int add_port(struct ports_device *portdev, u32 id)
1386{
1387 struct port *port;
1388 dev_t devt;
1389 int err;
1390
1391 port = kmalloc_obj(*port);
1392 if (!port) {
1393 err = -ENOMEM;
1394 goto fail;
1395 }
1396 kref_init(&port->kref);
1397
1398 port->portdev = portdev;
1399 port->id = id;
1400
1401 port->name = NULL;
1402 port->inbuf = NULL;
1403 port->cons.hvc = NULL;
1404 port->async_queue = NULL;
1405
1406 port->cons.ws.ws_row = port->cons.ws.ws_col = 0;
1407 port->cons.vtermno = 0;
1408
1409 port->host_connected = port->guest_connected = false;
1410 port->stats = (struct port_stats) { 0 };
1411
1412 port->outvq_full = false;
1413
1414 port->in_vq = portdev->in_vqs[port->id];
1415 port->out_vq = portdev->out_vqs[port->id];
1416
1417 port->cdev = cdev_alloc();
1418 if (!port->cdev) {
1419 dev_err(&port->portdev->vdev->dev, "Error allocating cdev\n");
1420 err = -ENOMEM;
1421 goto free_port;
1422 }
1423 port->cdev->ops = &port_fops;
1424
1425 devt = MKDEV(portdev->chr_major, id);
1426 err = cdev_add(port->cdev, devt, 1);
1427 if (err < 0) {
1428 dev_err(&port->portdev->vdev->dev,
1429 "Error %d adding cdev for port %u\n", err, id);
1430 goto free_cdev;
1431 }
1432 port->dev = device_create(&port_class, &port->portdev->vdev->dev,
1433 devt, port, "vport%up%u",
1434 port->portdev->vdev->index, id);
1435 if (IS_ERR(port->dev)) {
1436 err = PTR_ERR(port->dev);
1437 dev_err(&port->portdev->vdev->dev,
1438 "Error %d creating device for port %u\n",
1439 err, id);
1440 goto free_cdev;
1441 }
1442
1443 spin_lock_init(&port->inbuf_lock);
1444 spin_lock_init(&port->outvq_lock);
1445 init_waitqueue_head(&port->waitqueue);
1446
1447 /* We can safely ignore ENOSPC because it means
1448 * the queue already has buffers. Buffers are removed
1449 * only by virtcons_remove(), not by unplug_port()
1450 */
1451 err = fill_queue(port->in_vq, &port->inbuf_lock);
1452 if (err < 0 && err != -ENOSPC) {
1453 dev_err(port->dev, "Error allocating inbufs\n");
1454 goto free_device;
1455 }
1456
1457 if (is_rproc_serial(port->portdev->vdev))
1458 /*
1459 * For rproc_serial assume remote processor is connected.
1460 * rproc_serial does not want the console port, only
1461 * the generic port implementation.
1462 */
1463 port->host_connected = true;
1464 else if (!use_multiport(port->portdev)) {
1465 /*
1466 * If we're not using multiport support,
1467 * this has to be a console port.
1468 */
1469 err = init_port_console(port);
1470 if (err)
1471 goto free_inbufs;
1472 }
1473
1474 spin_lock_irq(&portdev->ports_lock);
1475 list_add_tail(&port->list, &port->portdev->ports);
1476 spin_unlock_irq(&portdev->ports_lock);
1477
1478 /*
1479 * Tell the Host we're set so that it can send us various
1480 * configuration parameters for this port (eg, port name,
1481 * caching, whether this is a console port, etc.)
1482 */
1483 send_control_msg(port, VIRTIO_CONSOLE_PORT_READY, 1);
1484
1485 /*
1486 * Finally, create the debugfs file that we can use to
1487 * inspect a port's state at any time
1488 */
1489 port->debugfs_file = debugfs_create_file(dev_name(port->dev), 0444,
1490 pdrvdata.debugfs_dir,
1491 port, &port_debugfs_fops);
1492 return 0;
1493
1494free_inbufs:
1495free_device:
1496 device_destroy(&port_class, port->dev->devt);
1497free_cdev:
1498 cdev_del(port->cdev);
1499free_port:
1500 kfree(port);
1501fail:
1502 /* The host might want to notify management sw about port add failure */
1503 __send_control_msg(portdev, id, VIRTIO_CONSOLE_PORT_READY, 0);
1504 return err;
1505}
1506
1507/* No users remain, remove all port-specific data. */
1508static void remove_port(struct kref *kref)
1509{
1510 struct port *port;
1511
1512 port = container_of(kref, struct port, kref);
1513
1514 kfree(port);
1515}
1516
1517static void remove_port_data(struct port *port)
1518{
1519 spin_lock_irq(&port->inbuf_lock);
1520 /* Remove unused data this port might have received. */
1521 discard_port_data(port);
1522 spin_unlock_irq(&port->inbuf_lock);
1523
1524 spin_lock_irq(&port->outvq_lock);
1525 reclaim_consumed_buffers(port);
1526 spin_unlock_irq(&port->outvq_lock);
1527}
1528
1529/*
1530 * Port got unplugged. Remove port from portdev's list and drop the
1531 * kref reference. If no userspace has this port opened, it will
1532 * result in immediate removal the port.
1533 */
1534static void unplug_port(struct port *port)
1535{
1536 spin_lock_irq(&port->portdev->ports_lock);
1537 list_del(&port->list);
1538 spin_unlock_irq(&port->portdev->ports_lock);
1539
1540 spin_lock_irq(&port->inbuf_lock);
1541 if (port->guest_connected) {
1542 /* Let the app know the port is going down. */
1543 send_sigio_to_port(port);
1544
1545 /* Do this after sigio is actually sent */
1546 port->guest_connected = false;
1547 port->host_connected = false;
1548
1549 wake_up_interruptible(&port->waitqueue);
1550 }
1551 spin_unlock_irq(&port->inbuf_lock);
1552
1553 if (is_console_port(port)) {
1554 spin_lock_irq(&pdrvdata_lock);
1555 list_del(&port->cons.list);
1556 spin_unlock_irq(&pdrvdata_lock);
1557 hvc_remove(port->cons.hvc);
1558 ida_free(&vtermno_ida, port->cons.vtermno);
1559 }
1560
1561 remove_port_data(port);
1562
1563 /*
1564 * Null out portdev under outvq_lock so that __send_to_port()
1565 * cannot race: it checks port->portdev inside the same lock
1566 * and bails out if NULL, preventing any buffer from being
1567 * enqueued to an already torn-down virtqueue. Also prevents
1568 * a close on an open port later from sending a stale control
1569 * message.
1570 */
1571 spin_lock_irq(&port->outvq_lock);
1572 port->portdev = NULL;
1573 spin_unlock_irq(&port->outvq_lock);
1574
1575 sysfs_remove_group(&port->dev->kobj, &port_attribute_group);
1576 device_destroy(&port_class, port->dev->devt);
1577 cdev_del(port->cdev);
1578
1579 debugfs_remove(port->debugfs_file);
1580 kfree(port->name);
1581
1582 /*
1583 * Locks around here are not necessary - a port can't be
1584 * opened after we removed the port struct from ports_list
1585 * above.
1586 */
1587 kref_put(&port->kref, remove_port);
1588}
1589
1590/* Any private messages that the Host and Guest want to share */
1591static void handle_control_message(struct virtio_device *vdev,
1592 struct ports_device *portdev,
1593 struct port_buffer *buf)
1594{
1595 struct virtio_console_control *cpkt;
1596 struct port *port;
1597 size_t name_size;
1598 int err;
1599
1600 cpkt = (struct virtio_console_control *)(buf->buf + buf->offset);
1601
1602 port = find_port_by_id(portdev, virtio32_to_cpu(vdev, cpkt->id));
1603 if (!port &&
1604 cpkt->event != cpu_to_virtio16(vdev, VIRTIO_CONSOLE_PORT_ADD)) {
1605 /* No valid header at start of buffer. Drop it. */
1606 dev_dbg(&portdev->vdev->dev,
1607 "Invalid index %u in control packet\n",
1608 virtio32_to_cpu(vdev, cpkt->id));
1609 return;
1610 }
1611
1612 switch (virtio16_to_cpu(vdev, cpkt->event)) {
1613 case VIRTIO_CONSOLE_PORT_ADD:
1614 if (port) {
1615 dev_dbg(&portdev->vdev->dev,
1616 "Port %u already added\n", port->id);
1617 send_control_msg(port, VIRTIO_CONSOLE_PORT_READY, 1);
1618 break;
1619 }
1620 if (virtio32_to_cpu(vdev, cpkt->id) >=
1621 portdev->max_nr_ports) {
1622 dev_warn(&portdev->vdev->dev,
1623 "Request for adding port with "
1624 "out-of-bound id %u, max. supported id: %u\n",
1625 virtio32_to_cpu(vdev, cpkt->id),
1626 portdev->max_nr_ports - 1);
1627 break;
1628 }
1629 add_port(portdev, virtio32_to_cpu(vdev, cpkt->id));
1630 break;
1631 case VIRTIO_CONSOLE_PORT_REMOVE:
1632 unplug_port(port);
1633 break;
1634 case VIRTIO_CONSOLE_CONSOLE_PORT:
1635 if (!cpkt->value)
1636 break;
1637 if (is_console_port(port))
1638 break;
1639
1640 init_port_console(port);
1641 complete(&early_console_added);
1642 /*
1643 * Could remove the port here in case init fails - but
1644 * have to notify the host first.
1645 */
1646 break;
1647 case VIRTIO_CONSOLE_RESIZE: {
1648 struct {
1649 __virtio16 cols;
1650 __virtio16 rows;
1651 } size;
1652
1653 if (!is_console_port(port))
1654 break;
1655
1656 memcpy(&size, buf->buf + buf->offset + sizeof(*cpkt),
1657 sizeof(size));
1658 set_console_size(port, virtio16_to_cpu(vdev, size.rows),
1659 virtio16_to_cpu(vdev, size.cols));
1660
1661 port->cons.hvc->irq_requested = 1;
1662 resize_console(port);
1663 break;
1664 }
1665 case VIRTIO_CONSOLE_PORT_OPEN:
1666 port->host_connected = virtio16_to_cpu(vdev, cpkt->value);
1667 wake_up_interruptible(&port->waitqueue);
1668 /*
1669 * If the host port got closed and the host had any
1670 * unconsumed buffers, we'll be able to reclaim them
1671 * now.
1672 */
1673 spin_lock_irq(&port->outvq_lock);
1674 reclaim_consumed_buffers(port);
1675 spin_unlock_irq(&port->outvq_lock);
1676
1677 /*
1678 * If the guest is connected, it'll be interested in
1679 * knowing the host connection state changed.
1680 */
1681 spin_lock_irq(&port->inbuf_lock);
1682 send_sigio_to_port(port);
1683 spin_unlock_irq(&port->inbuf_lock);
1684 break;
1685 case VIRTIO_CONSOLE_PORT_NAME:
1686 /*
1687 * If we woke up after hibernation, we can get this
1688 * again. Skip it in that case.
1689 */
1690 if (port->name)
1691 break;
1692
1693 /*
1694 * Skip the size of the header and the cpkt to get the size
1695 * of the name that was sent
1696 */
1697 name_size = buf->len - buf->offset - sizeof(*cpkt) + 1;
1698
1699 port->name = kmalloc(name_size, GFP_KERNEL);
1700 if (!port->name) {
1701 dev_err(port->dev,
1702 "Not enough space to store port name\n");
1703 break;
1704 }
1705 strscpy(port->name, buf->buf + buf->offset + sizeof(*cpkt),
1706 name_size);
1707
1708 /*
1709 * Since we only have one sysfs attribute, 'name',
1710 * create it only if we have a name for the port.
1711 */
1712 err = sysfs_create_group(&port->dev->kobj,
1713 &port_attribute_group);
1714 if (err) {
1715 dev_err(port->dev,
1716 "Error %d creating sysfs device attributes\n",
1717 err);
1718 } else {
1719 /*
1720 * Generate a udev event so that appropriate
1721 * symlinks can be created based on udev
1722 * rules.
1723 */
1724 kobject_uevent(&port->dev->kobj, KOBJ_CHANGE);
1725 }
1726 break;
1727 }
1728}
1729
1730static void control_work_handler(struct work_struct *work)
1731{
1732 struct ports_device *portdev;
1733 struct virtqueue *vq;
1734 struct port_buffer *buf;
1735 unsigned int len;
1736
1737 portdev = container_of(work, struct ports_device, control_work);
1738 vq = portdev->c_ivq;
1739
1740 spin_lock(&portdev->c_ivq_lock);
1741 while ((buf = virtqueue_get_buf(vq, &len))) {
1742 spin_unlock(&portdev->c_ivq_lock);
1743
1744 buf->len = min_t(size_t, len, buf->size);
1745 buf->offset = 0;
1746
1747 handle_control_message(vq->vdev, portdev, buf);
1748
1749 spin_lock(&portdev->c_ivq_lock);
1750 if (add_inbuf(portdev->c_ivq, buf) < 0) {
1751 dev_warn(&portdev->vdev->dev,
1752 "Error adding buffer to queue\n");
1753 free_buf(buf, false);
1754 }
1755 }
1756 spin_unlock(&portdev->c_ivq_lock);
1757}
1758
1759static void flush_bufs(struct virtqueue *vq, bool can_sleep)
1760{
1761 struct port_buffer *buf;
1762 unsigned int len;
1763
1764 while ((buf = virtqueue_get_buf(vq, &len)))
1765 free_buf(buf, can_sleep);
1766}
1767
1768static void out_intr(struct virtqueue *vq)
1769{
1770 struct port *port;
1771
1772 port = find_port_by_vq(vq->vdev->priv, vq);
1773 if (!port) {
1774 flush_bufs(vq, false);
1775 return;
1776 }
1777
1778 wake_up_interruptible(&port->waitqueue);
1779 kref_put(&port->kref, remove_port);
1780}
1781
1782static void in_intr(struct virtqueue *vq)
1783{
1784 struct port *port;
1785 unsigned long flags;
1786
1787 port = find_port_by_vq(vq->vdev->priv, vq);
1788 if (!port) {
1789 flush_bufs(vq, false);
1790 return;
1791 }
1792
1793 spin_lock_irqsave(&port->inbuf_lock, flags);
1794 port->inbuf = get_inbuf(port);
1795
1796 /*
1797 * Normally the port should not accept data when the port is
1798 * closed. For generic serial ports, the host won't (shouldn't)
1799 * send data till the guest is connected. But this condition
1800 * can be reached when a console port is not yet connected (no
1801 * tty is spawned) and the other side sends out data over the
1802 * vring, or when a remote devices start sending data before
1803 * the ports are opened.
1804 *
1805 * A generic serial port will discard data if not connected,
1806 * while console ports and rproc-serial ports accepts data at
1807 * any time. rproc-serial is initiated with guest_connected to
1808 * false because port_fops_open expects this. Console ports are
1809 * hooked up with an HVC console and is initialized with
1810 * guest_connected to true.
1811 */
1812
1813 if (!port->guest_connected && !is_rproc_serial(port->portdev->vdev))
1814 discard_port_data(port);
1815
1816 /* Send a SIGIO indicating new data in case the process asked for it */
1817 send_sigio_to_port(port);
1818
1819 spin_unlock_irqrestore(&port->inbuf_lock, flags);
1820
1821 wake_up_interruptible(&port->waitqueue);
1822
1823 if (is_console_port(port) && hvc_poll(port->cons.hvc))
1824 hvc_kick();
1825
1826 kref_put(&port->kref, remove_port);
1827}
1828
1829static void control_intr(struct virtqueue *vq)
1830{
1831 struct ports_device *portdev;
1832
1833 portdev = vq->vdev->priv;
1834 schedule_work(&portdev->control_work);
1835}
1836
1837static void config_intr(struct virtio_device *vdev)
1838{
1839 struct ports_device *portdev;
1840
1841 portdev = vdev->priv;
1842
1843 if (!use_multiport(portdev))
1844 schedule_work(&portdev->config_work);
1845}
1846
1847static void update_size_from_config(struct ports_device *portdev)
1848{
1849 struct virtio_device *vdev;
1850 struct port *port;
1851 u16 rows, cols;
1852
1853 vdev = portdev->vdev;
1854
1855 /*
1856 * We'll use this way of resizing only for legacy support.
1857 * For multiport devices, use control messages to indicate
1858 * console size changes so that it can be done per-port.
1859 *
1860 * Don't test F_SIZE at all if we're rproc: not a valid feature.
1861 */
1862 if (is_rproc_serial(vdev) ||
1863 use_multiport(portdev) ||
1864 !virtio_has_feature(vdev, VIRTIO_CONSOLE_F_SIZE))
1865 return;
1866
1867 virtio_cread(vdev, struct virtio_console_config, cols, &cols);
1868 virtio_cread(vdev, struct virtio_console_config, rows, &rows);
1869
1870 port = find_port_by_id(portdev, 0);
1871 set_console_size(port, rows, cols);
1872 resize_console(port);
1873}
1874
1875static void config_work_handler(struct work_struct *work)
1876{
1877 struct ports_device *portdev;
1878
1879 portdev = container_of(work, struct ports_device, config_work);
1880 update_size_from_config(portdev);
1881}
1882
1883static int init_vqs(struct ports_device *portdev)
1884{
1885 struct virtqueue_info *vqs_info;
1886 struct virtqueue **vqs;
1887 u32 i, j, nr_ports, nr_queues;
1888 int err;
1889
1890 nr_ports = portdev->max_nr_ports;
1891 nr_queues = use_multiport(portdev) ? (nr_ports + 1) * 2 : 2;
1892
1893 vqs = kmalloc_objs(struct virtqueue *, nr_queues);
1894 vqs_info = kzalloc_objs(*vqs_info, nr_queues);
1895 portdev->in_vqs = kmalloc_objs(struct virtqueue *, nr_ports);
1896 portdev->out_vqs = kmalloc_objs(struct virtqueue *, nr_ports);
1897 if (!vqs || !vqs_info || !portdev->in_vqs || !portdev->out_vqs) {
1898 err = -ENOMEM;
1899 goto free;
1900 }
1901
1902 /*
1903 * For backward compat (newer host but older guest), the host
1904 * spawns a console port first and also inits the vqs for port
1905 * 0 before others.
1906 */
1907 j = 0;
1908 vqs_info[j].callback = in_intr;
1909 vqs_info[j + 1].callback = out_intr;
1910 vqs_info[j].name = "input";
1911 vqs_info[j + 1].name = "output";
1912 j += 2;
1913
1914 if (use_multiport(portdev)) {
1915 vqs_info[j].callback = control_intr;
1916 vqs_info[j].name = "control-i";
1917 vqs_info[j + 1].name = "control-o";
1918
1919 for (i = 1; i < nr_ports; i++) {
1920 j += 2;
1921 vqs_info[j].callback = in_intr;
1922 vqs_info[j + 1].callback = out_intr;
1923 vqs_info[j].name = "input";
1924 vqs_info[j + 1].name = "output";
1925 }
1926 }
1927 /* Find the queues. */
1928 err = virtio_find_vqs(portdev->vdev, nr_queues, vqs, vqs_info, NULL);
1929 if (err)
1930 goto free;
1931
1932 j = 0;
1933 portdev->in_vqs[0] = vqs[0];
1934 portdev->out_vqs[0] = vqs[1];
1935 j += 2;
1936 if (use_multiport(portdev)) {
1937 portdev->c_ivq = vqs[j];
1938 portdev->c_ovq = vqs[j + 1];
1939
1940 for (i = 1; i < nr_ports; i++) {
1941 j += 2;
1942 portdev->in_vqs[i] = vqs[j];
1943 portdev->out_vqs[i] = vqs[j + 1];
1944 }
1945 }
1946 kfree(vqs_info);
1947 kfree(vqs);
1948
1949 return 0;
1950
1951free:
1952 kfree(portdev->out_vqs);
1953 kfree(portdev->in_vqs);
1954 kfree(vqs_info);
1955 kfree(vqs);
1956
1957 return err;
1958}
1959
1960static const struct file_operations portdev_fops = {
1961 .owner = THIS_MODULE,
1962};
1963
1964static void remove_vqs(struct ports_device *portdev)
1965{
1966 struct virtqueue *vq;
1967 bool multiport = use_multiport(portdev);
1968
1969 virtio_device_for_each_vq(portdev->vdev, vq) {
1970 struct port_buffer *buf;
1971 unsigned int len;
1972
1973 /*
1974 * c_ovq cookies are &portdev->cpkt, not port_buffer.
1975 * Detach them but do not free_buf().
1976 */
1977 if (multiport && vq == portdev->c_ovq) {
1978 spin_lock(&portdev->c_ovq_lock);
1979 while (virtqueue_get_buf(vq, &len))
1980 ;
1981 while (virtqueue_detach_unused_buf(vq))
1982 ;
1983 spin_unlock(&portdev->c_ovq_lock);
1984 } else {
1985 flush_bufs(vq, true);
1986 while ((buf = virtqueue_detach_unused_buf(vq)))
1987 free_buf(buf, true);
1988 }
1989 cond_resched();
1990 }
1991 portdev->vdev->config->del_vqs(portdev->vdev);
1992 kfree(portdev->in_vqs);
1993 kfree(portdev->out_vqs);
1994}
1995
1996static void virtcons_remove(struct virtio_device *vdev)
1997{
1998 struct ports_device *portdev;
1999 struct port *port, *port2;
2000
2001 portdev = vdev->priv;
2002
2003 spin_lock_irq(&pdrvdata_lock);
2004 list_del(&portdev->list);
2005 spin_unlock_irq(&pdrvdata_lock);
2006
2007 /* Device is going away, exit any polling for buffers */
2008 virtio_break_device(vdev);
2009 if (use_multiport(portdev))
2010 flush_work(&portdev->control_work);
2011 else
2012 flush_work(&portdev->config_work);
2013
2014 /* Disable interrupts for vqs */
2015 virtio_reset_device(vdev);
2016 /* Finish up work that's lined up */
2017 if (use_multiport(portdev))
2018 cancel_work_sync(&portdev->control_work);
2019 else
2020 cancel_work_sync(&portdev->config_work);
2021
2022 list_for_each_entry_safe(port, port2, &portdev->ports, list)
2023 unplug_port(port);
2024
2025 unregister_chrdev(portdev->chr_major, "virtio-portsdev");
2026
2027 /*
2028 * When yanking out a device, we immediately lose the
2029 * (device-side) queues. So there's no point in keeping the
2030 * guest side around till we drop our final reference. This
2031 * also means that any ports which are in an open state will
2032 * have to just stop using the port, as the vqs are going
2033 * away.
2034 */
2035 remove_vqs(portdev);
2036 kfree(portdev);
2037}
2038
2039/*
2040 * Once we're further in boot, we get probed like any other virtio
2041 * device.
2042 *
2043 * If the host also supports multiple console ports, we check the
2044 * config space to see how many ports the host has spawned. We
2045 * initialize each port found.
2046 */
2047static int virtcons_probe(struct virtio_device *vdev)
2048{
2049 struct ports_device *portdev;
2050 int err;
2051 bool multiport;
2052
2053 /* We only need a config space if features are offered */
2054 if (!vdev->config->get &&
2055 (virtio_has_feature(vdev, VIRTIO_CONSOLE_F_SIZE)
2056 || virtio_has_feature(vdev, VIRTIO_CONSOLE_F_MULTIPORT))) {
2057 dev_err(&vdev->dev, "%s failure: config access disabled\n",
2058 __func__);
2059 return -EINVAL;
2060 }
2061
2062 portdev = kmalloc_obj(*portdev);
2063 if (!portdev) {
2064 err = -ENOMEM;
2065 goto fail;
2066 }
2067
2068 /* Attach this portdev to this virtio_device, and vice-versa. */
2069 portdev->vdev = vdev;
2070 vdev->priv = portdev;
2071 portdev->pm_freezing = false;
2072
2073 portdev->chr_major = register_chrdev(0, "virtio-portsdev",
2074 &portdev_fops);
2075 if (portdev->chr_major < 0) {
2076 dev_err(&vdev->dev,
2077 "Error %d registering chrdev for device %u\n",
2078 portdev->chr_major, vdev->index);
2079 err = portdev->chr_major;
2080 goto free;
2081 }
2082
2083 multiport = false;
2084 portdev->max_nr_ports = 1;
2085
2086 /* Don't test MULTIPORT at all if we're rproc: not a valid feature! */
2087 if (!is_rproc_serial(vdev) &&
2088 virtio_cread_feature(vdev, VIRTIO_CONSOLE_F_MULTIPORT,
2089 struct virtio_console_config, max_nr_ports,
2090 &portdev->max_nr_ports) == 0) {
2091 if (portdev->max_nr_ports == 0 ||
2092 portdev->max_nr_ports > VIRTCONS_MAX_PORTS) {
2093 dev_err(&vdev->dev,
2094 "Invalidate max_nr_ports %d",
2095 portdev->max_nr_ports);
2096 err = -EINVAL;
2097 goto free;
2098 }
2099 multiport = true;
2100 }
2101
2102 spin_lock_init(&portdev->ports_lock);
2103 INIT_LIST_HEAD(&portdev->ports);
2104 INIT_LIST_HEAD(&portdev->list);
2105
2106 INIT_WORK(&portdev->config_work, &config_work_handler);
2107 INIT_WORK(&portdev->control_work, &control_work_handler);
2108
2109 if (multiport) {
2110 spin_lock_init(&portdev->c_ivq_lock);
2111 spin_lock_init(&portdev->c_ovq_lock);
2112 }
2113
2114 err = init_vqs(portdev);
2115 if (err < 0) {
2116 dev_err(&vdev->dev, "Error %d initializing vqs\n", err);
2117 goto free_chrdev;
2118 }
2119
2120 virtio_device_ready(portdev->vdev);
2121
2122 if (multiport) {
2123 err = fill_queue(portdev->c_ivq, &portdev->c_ivq_lock);
2124 if (err < 0) {
2125 dev_err(&vdev->dev,
2126 "Error allocating buffers for control queue\n");
2127 /*
2128 * The host might want to notify mgmt sw about device
2129 * add failure.
2130 */
2131 __send_control_msg(portdev, VIRTIO_CONSOLE_BAD_ID,
2132 VIRTIO_CONSOLE_DEVICE_READY, 0);
2133 /* Device was functional: we need full cleanup. */
2134 virtcons_remove(vdev);
2135 return err;
2136 }
2137 } else {
2138 /*
2139 * For backward compatibility: Create a console port
2140 * if we're running on older host.
2141 */
2142 add_port(portdev, 0);
2143 }
2144
2145 spin_lock_irq(&pdrvdata_lock);
2146 list_add_tail(&portdev->list, &pdrvdata.portdevs);
2147 spin_unlock_irq(&pdrvdata_lock);
2148
2149 __send_control_msg(portdev, VIRTIO_CONSOLE_BAD_ID,
2150 VIRTIO_CONSOLE_DEVICE_READY, 1);
2151
2152 update_size_from_config(portdev);
2153
2154 return 0;
2155
2156free_chrdev:
2157 unregister_chrdev(portdev->chr_major, "virtio-portsdev");
2158free:
2159 kfree(portdev);
2160fail:
2161 return err;
2162}
2163
2164static const struct virtio_device_id id_table[] = {
2165 { VIRTIO_ID_CONSOLE, VIRTIO_DEV_ANY_ID },
2166 { 0 },
2167};
2168MODULE_DEVICE_TABLE(virtio, id_table);
2169
2170static const unsigned int features[] = {
2171 VIRTIO_CONSOLE_F_SIZE,
2172 VIRTIO_CONSOLE_F_MULTIPORT,
2173};
2174
2175static const struct virtio_device_id rproc_serial_id_table[] = {
2176#if IS_ENABLED(CONFIG_REMOTEPROC)
2177 { VIRTIO_ID_RPROC_SERIAL, VIRTIO_DEV_ANY_ID },
2178#endif
2179 { 0 },
2180};
2181MODULE_DEVICE_TABLE(virtio, rproc_serial_id_table);
2182
2183static const unsigned int rproc_serial_features[] = {
2184};
2185
2186#ifdef CONFIG_PM_SLEEP
2187static int virtcons_freeze(struct virtio_device *vdev)
2188{
2189 struct ports_device *portdev;
2190 struct port *port;
2191 unsigned long flags;
2192
2193 portdev = vdev->priv;
2194
2195 /*
2196 * Block TX paths (put_chars, __send_to_port) before resetting the
2197 * device and tearing down virtqueues. This prevents races with
2198 * hvc console writes that remain active under no_console_suspend.
2199 */
2200 smp_store_release(&portdev->pm_freezing, true);
2201
2202 /*
2203 * Synchronize with any concurrent __send_to_port() that may have
2204 * passed the pm_freezing check. By acquiring and releasing the
2205 * outvq_lock for each port, we ensure all active TX paths have
2206 * completed before we reset the device.
2207 */
2208 spin_lock_irqsave(&portdev->ports_lock, flags);
2209 list_for_each_entry(port, &portdev->ports, list) {
2210 spin_lock(&port->outvq_lock);
2211 spin_unlock(&port->outvq_lock);
2212 }
2213 spin_unlock_irqrestore(&portdev->ports_lock, flags);
2214
2215 virtio_reset_device(vdev);
2216
2217 if (use_multiport(portdev))
2218 virtqueue_disable_cb(portdev->c_ivq);
2219 cancel_work_sync(&portdev->control_work);
2220 cancel_work_sync(&portdev->config_work);
2221 /*
2222 * Once more: if control_work_handler() was running, it would
2223 * enable the cb as the last step.
2224 */
2225 if (use_multiport(portdev))
2226 virtqueue_disable_cb(portdev->c_ivq);
2227
2228 list_for_each_entry(port, &portdev->ports, list) {
2229 virtqueue_disable_cb(port->in_vq);
2230 virtqueue_disable_cb(port->out_vq);
2231 /*
2232 * We'll ask the host later if the new invocation has
2233 * the port opened or closed.
2234 */
2235 port->host_connected = false;
2236 remove_port_data(port);
2237 }
2238 remove_vqs(portdev);
2239
2240 return 0;
2241}
2242
2243static int virtcons_restore(struct virtio_device *vdev)
2244{
2245 struct ports_device *portdev;
2246 struct port *port;
2247 int ret;
2248
2249 portdev = vdev->priv;
2250
2251 ret = init_vqs(portdev);
2252 if (ret)
2253 return ret;
2254
2255 virtio_device_ready(portdev->vdev);
2256
2257 list_for_each_entry(port, &portdev->ports, list) {
2258 port->in_vq = portdev->in_vqs[port->id];
2259 port->out_vq = portdev->out_vqs[port->id];
2260
2261 fill_queue(port->in_vq, &port->inbuf_lock);
2262
2263 /* Get port open/close status on the host */
2264 send_control_msg(port, VIRTIO_CONSOLE_PORT_READY, 1);
2265
2266 /*
2267 * If a port was open at the time of suspending, we
2268 * have to let the host know that it's still open.
2269 */
2270 if (port->guest_connected)
2271 send_control_msg(port, VIRTIO_CONSOLE_PORT_OPEN, 1);
2272 }
2273
2274 /*
2275 * Populate the control receive queue only after the list iteration
2276 * is complete. If we fill this queue before iterating, the host could
2277 * immediately deliver a VIRTIO_CONSOLE_PORT_REMOVE message.
2278 * This would trigger the control workqueue, which modifies the
2279 * portdev->ports list concurrently with the unprotected loop above,
2280 * leading to a Use-After-Free and list corruption.
2281 */
2282 if (use_multiport(portdev))
2283 fill_queue(portdev->c_ivq, &portdev->c_ivq_lock);
2284
2285 /*
2286 * Allow TX paths only after all port->out_vq pointers have
2287 * been reassigned to the newly allocated virtqueues.
2288 */
2289 smp_store_release(&portdev->pm_freezing, false);
2290
2291 return 0;
2292}
2293#endif
2294
2295static struct virtio_driver virtio_console = {
2296 .feature_table = features,
2297 .feature_table_size = ARRAY_SIZE(features),
2298 .driver.name = KBUILD_MODNAME,
2299 .id_table = id_table,
2300 .probe = virtcons_probe,
2301 .remove = virtcons_remove,
2302 .config_changed = config_intr,
2303#ifdef CONFIG_PM_SLEEP
2304 .freeze = virtcons_freeze,
2305 .restore = virtcons_restore,
2306#endif
2307};
2308
2309static struct virtio_driver virtio_rproc_serial = {
2310 .feature_table = rproc_serial_features,
2311 .feature_table_size = ARRAY_SIZE(rproc_serial_features),
2312 .driver.name = "virtio_rproc_serial",
2313 .id_table = rproc_serial_id_table,
2314 .probe = virtcons_probe,
2315 .remove = virtcons_remove,
2316};
2317
2318static int __init virtio_console_init(void)
2319{
2320 int err;
2321
2322 err = class_register(&port_class);
2323 if (err)
2324 return err;
2325
2326 pdrvdata.debugfs_dir = debugfs_create_dir("virtio-ports", NULL);
2327 INIT_LIST_HEAD(&pdrvdata.consoles);
2328 INIT_LIST_HEAD(&pdrvdata.portdevs);
2329
2330 err = register_virtio_driver(&virtio_console);
2331 if (err < 0) {
2332 pr_err("Error %d registering virtio driver\n", err);
2333 goto free;
2334 }
2335 err = register_virtio_driver(&virtio_rproc_serial);
2336 if (err < 0) {
2337 pr_err("Error %d registering virtio rproc serial driver\n",
2338 err);
2339 goto unregister;
2340 }
2341 return 0;
2342unregister:
2343 unregister_virtio_driver(&virtio_console);
2344free:
2345 debugfs_remove_recursive(pdrvdata.debugfs_dir);
2346 class_unregister(&port_class);
2347 return err;
2348}
2349
2350static void __exit virtio_console_fini(void)
2351{
2352 reclaim_dma_bufs();
2353
2354 unregister_virtio_driver(&virtio_console);
2355 unregister_virtio_driver(&virtio_rproc_serial);
2356
2357 class_unregister(&port_class);
2358 debugfs_remove_recursive(pdrvdata.debugfs_dir);
2359}
2360module_init(virtio_console_init);
2361module_exit(virtio_console_fini);
2362
2363MODULE_DESCRIPTION("Virtio console driver");
2364MODULE_LICENSE("GPL");