projects/linux·blob master · refs

1853 lines · 44.9 KiB

1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Stream data over Thunderbolt/USB4 cable
4 *
5 * Copyright (C) 2026, Intel Corporation
6 * Authors: Alan Borzeszkowski <[email protected]>
7 * Mika Westerberg <[email protected]>
8 */
9
10#define pr_fmt(fmt) "tbstream: " fmt
11
12#include <linux/delay.h>
13#include <linux/configfs.h>
14#include <linux/file.h>
15#include <linux/fs.h>
16#include <linux/idr.h>
17#include <linux/ktime.h>
18#include <linux/miscdevice.h>
19#include <linux/module.h>
20#include <linux/mutex.h>
21#include <linux/poll.h>
22#include <linux/sizes.h>
23#include <linux/thunderbolt.h>
24#include <linux/uaccess.h>
25#include <linux/uio.h>
26#include <linux/uuid.h>
27#include <linux/wait.h>
28
29/*
30 * USB4STREAM - Stream data directly over Thunderbolt/USB4 cable
31 *
32 * HopIDs are configured by the user. In Linux this is done through
33 * ConfigFS. Once that is done paths are be established the first time
34 * the stream is opened. Typically the read side is opened first to make
35 * sure all the data will be received.
36 *
37 * End-to-end flow control is mandatory on both sides.
38 *
39 * Data is sent to the other side as tunneled DATA packets. All the data
40 * is owned by the user and passed as-is from the writer to the reader.
41 *
42 * Once the stream device is closed, a CLOSE packet is sent to the peer
43 * so it can take the necessary action. On Linux this typically results
44 * in EOF being returned to the reader.
45 *
46 * Tunneled packet types:
47 *
48 * +-------+---------+------------------+
49 * | PDF | Type | Payload size |
50 * +-------+---------+------------------+
51 * | 2 | DATA | up to 4 KiB |
52 * | 3 | CLOSE | up to 256 bytes |
53 * +-------+---------+------------------+
54 *
55 * Each stream can optionally publish configuration values under its own
56 * XDomain property directory. The name of the directory is the name of
57 * the stream in question and the UUID is up to the stream. For example
58 * if the stream exposes video output then the directory name could be
59 * "video".
60 *
61 * Below values are reserved and can be used by the stream:
62 *
63 * +----------+-----------+-------------------------+
64 * | Key | Type | Contents |
65 * +----------+-----------+-------------------------+
66 * | inhopid | IMMEDIATE | Configured input HopID |
67 * | outhopid | IMMEDIATE | Configured output HopID |
68 * +----------+-----------+-------------------------+
69 *
70 * It is allowed to add more stream specific properties as well if the
71 * above are not enough.
72 */
73
74#define TBSTREAM_DEV_RING_SIZE 256
75#define TBSTREAM_DEV_MIN_RING_SIZE 32
76#define TBSTREAM_DEV_MAX_RING_SIZE 4096
77#define TBSTREAM_DEV_THROTTLING 8192
78#define TBSTREAM_DEV_MAX_THROTTLING 16776960
79
80/**
81 * enum tbstream_frame_pdf - PDF numbers for tunneled frames
82 * @TBSTREAM_FRAME_START: PDF of the start of the frame
83 * @TBSTREAM_DATA: PDF of the DATA frame
84 * @TBSTREAM_CLOSE: PDF of the CLOSE frame
85 */
86enum tbstream_frame_pdf {
87 TBSTREAM_FRAME_START = 1,
88 TBSTREAM_DATA,
89 TBSTREAM_CLOSE,
90};
91
92/**
93 * struct tbstream_frame - Frame submitted to/from the rings
94 * @sdev: Pointer to the stream device
95 * @page: Page holding the packet
96 * @offset: Offset inside @page if partial read is done
97 * @completed: %true if the RX frame is completed
98 * @frame: Underlying frame structure
99 */
100struct tbstream_frame {
101 struct tbstream_dev *sdev;
102 struct page *page;
103 unsigned int offset;
104 bool completed;
105 struct ring_frame frame;
106};
107
108/**
109 * struct tbstream_ring - Stream RX/TX ring structure
110 * @ring: Pointer to the API ring
111 * @prod: Current value of producer
112 * @cons: Current value of consumer
113 * @frames: Holds the ring frames
114 */
115struct tbstream_ring {
116 struct tb_ring *ring;
117 unsigned long prod;
118 unsigned long cons;
119 struct tbstream_frame *frames;
120};
121
122/**
123 * struct tbstream_dev - Stream character device
124 * @group: ConfigFS group for this device
125 * @stream: Pointer to the stream if it is attached (%NULL otherwise)
126 * @misc: Character device used for tunneling
127 * @kref: Reference count
128 * @index: Unique identifier for the character device
129 * @in_hopid: In HopID
130 * @out_hopid: Out HopID
131 * @ring_size: Size of the rings
132 * @throttling: Interrupt throttling rate in ns
133 * @busy_poll: Instead of interrupts, busy poll the rings
134 * @users: Number of times @cdev has been opened
135 * @closed: CLOSE packet was received
136 * @removed: Userspace removed the ConfigFS group underneath.
137 * @wait: Waitqueue for open, read and write
138 * @lock: Lock protecting this structure
139 * @tx_ring: Transmit ring
140 * @rx_ring: Receive ring
141 * @list: Stream devices are linked through this
142 */
143struct tbstream_dev {
144 struct config_group group;
145 struct tbstream *stream;
146 struct miscdevice misc;
147 struct kref kref;
148 int index;
149 int in_hopid;
150 int out_hopid;
151 unsigned int ring_size;
152 unsigned int throttling;
153 bool busy_poll;
154 int users;
155 bool closed;
156 bool removed;
157 wait_queue_head_t wait;
158 struct mutex lock;
159 struct tbstream_ring tx_ring;
160 struct tbstream_ring rx_ring;
161 struct list_head list;
162};
163
164/**
165 * struct tbstream_group - Config group for stream
166 * @group: ConfigFS group for @stream
167 * @stream: Stream the ConfigFS group is attached to. %NULL if there is
168 * no stream attached.
169 * @lock: Lock protecting this structure
170 * @dev_list: List of stream devices
171 *
172 * This is the ConfigFS directory for one connection to another host.
173 * There can be several &struct stream_dev linked through @dev_list of
174 * this structure. Reference count managed through @group.
175 */
176struct tbstream_group {
177 struct config_group group;
178 struct tbstream *stream;
179 struct mutex lock;
180 struct list_head dev_list;
181};
182
183/**
184 * struct tbstream - Stream service private data
185 * @kref: Reference count
186 * @svc: Pointer to the service device
187 * @list: Streams are linked through this in @stream_list
188 *
189 * This represents the actual physical connection between two hosts.
190 */
191struct tbstream {
192 struct kref kref;
193 struct tb_service *svc;
194 struct list_head list;
195};
196
197static DEFINE_IDA(tbstream_indices);
198
199/* Protects tbstream_list */
200static DEFINE_MUTEX(tbstream_lock);
201static LIST_HEAD(tbstream_list);
202
203/* Serializes tbstream_get()/put() */
204static DEFINE_MUTEX(tbstream_kref_lock);
205
206/* Serializes tbstream_dev_get()/put() */
207static DEFINE_MUTEX(tbstream_dev_kref_lock);
208
209/* Stream property directory UUID: 3a1cb984-c4d9-4469-a277-ce2fdfd11f0d */
210static const uuid_t tbstream_dir_uuid =
211 UUID_INIT(0x3a1cb984, 0xc4d9, 0x4469,
212 0xa2, 0x77, 0xce, 0x2f, 0xdf, 0xd1, 0x1f, 0x0d);
213
214static struct tb_property_dir *tbstream_dir;
215
216static void tbstream_release(struct kref *kref)
217{
218 struct tbstream *stream = container_of(kref, typeof(*stream), kref);
219
220 tb_service_put(stream->svc);
221 kfree(stream);
222}
223
224static void tbstream_put(struct tbstream *stream)
225{
226 if (stream) {
227 guard(mutex)(&tbstream_kref_lock);
228 kref_put(&stream->kref, tbstream_release);
229 }
230}
231
232static struct tbstream *tbstream_get(struct tbstream *stream)
233{
234 if (stream) {
235 guard(mutex)(&tbstream_kref_lock);
236 kref_get(&stream->kref);
237 }
238 return stream;
239}
240
241static inline bool tbstream_valid(const struct tbstream *stream)
242{
243 if (stream)
244 return !tb_service_parent(stream->svc)->is_unplugged;
245 return false;
246}
247
248static void tbstream_ring_free(struct tbstream_ring *ring)
249{
250 struct device *dma_dev = tb_ring_dma_device(ring->ring);
251 enum dma_data_direction dir;
252 int i;
253
254 if (ring->ring->is_tx)
255 dir = DMA_TO_DEVICE;
256 else
257 dir = DMA_FROM_DEVICE;
258
259 for (i = 0; i < tb_ring_size(ring->ring); i++) {
260 struct tbstream_frame *sf = &ring->frames[i];
261
262 if (sf->frame.buffer_phy)
263 dma_unmap_page(dma_dev, sf->frame.buffer_phy,
264 TB_MAX_FRAME_SIZE, dir);
265 sf->frame.buffer_phy = 0;
266 if (sf->page)
267 __free_page(sf->page);
268 sf->page = NULL;
269 }
270
271 ring->prod = 0;
272 ring->cons = 0;
273 kfree(ring->frames);
274}
275
276static inline bool tbstream_ring_available(const struct tbstream_ring *ring)
277{
278 return ring->prod > ring->cons;
279}
280
281static inline struct tb_xdomain *tbstream_dev_xdomain(struct tbstream_dev *sdev)
282{
283 if (sdev->stream)
284 return tb_service_parent(sdev->stream->svc);
285 return NULL;
286}
287
288static void tbstream_dev_release(struct kref *kref)
289{
290 struct tbstream_dev *sdev = container_of(kref, struct tbstream_dev, kref);
291
292 if (sdev->stream) {
293 struct tb_xdomain *xd = tbstream_dev_xdomain(sdev);
294
295 if (sdev->out_hopid > 0)
296 tb_xdomain_release_out_hopid(xd, sdev->out_hopid);
297 if (sdev->in_hopid > 0)
298 tb_xdomain_release_in_hopid(xd, sdev->in_hopid);
299
300 tbstream_put(sdev->stream);
301 }
302 ida_free(&tbstream_indices, sdev->index);
303 kfree(sdev->misc.name);
304 kfree(sdev);
305}
306
307static inline void tbstream_dev_put(struct tbstream_dev *sdev)
308{
309 guard(mutex)(&tbstream_dev_kref_lock);
310 kref_put(&sdev->kref, tbstream_dev_release);
311}
312
313static inline struct tbstream_dev *tbstream_dev_get(struct tbstream_dev *sdev)
314{
315 guard(mutex)(&tbstream_dev_kref_lock);
316 kref_get(&sdev->kref);
317 return sdev;
318}
319
320static inline struct tbstream_dev *to_tbstream_dev(struct miscdevice *misc)
321{
322 return container_of(misc, struct tbstream_dev, misc);
323}
324
325static inline int tbstream_dev_valid(const struct tbstream_dev *sdev)
326{
327 const struct tbstream *stream = sdev->stream;
328
329 if (!tbstream_valid(stream))
330 return -ENXIO;
331 if (sdev->in_hopid <= 0 || sdev->out_hopid <= 0)
332 return -EINVAL;
333 return 0;
334}
335
336static inline bool tbstream_dev_removed(const struct tbstream_dev *sdev)
337{
338 return sdev->removed;
339}
340
341static inline bool tbstream_dev_closed(const struct tbstream_dev *sdev)
342{
343 return sdev->closed;
344}
345
346static void
347tbstream_dev_rx_callback(struct tb_ring *ring, struct ring_frame *frame,
348 bool canceled)
349{
350 struct tbstream_frame *sf = container_of(frame, typeof(*sf), frame);
351 struct tbstream_dev *sdev = sf->sdev;
352
353 if (canceled)
354 return;
355
356 sf->completed = true;
357 sdev->rx_ring.prod++;
358
359 if (sf->frame.flags & RING_DESC_CRC_ERROR)
360 pr_warn("RX CRC error\n");
361 else if (sf->frame.flags & RING_DESC_BUFFER_OVERRUN)
362 pr_warn("RX buffer overrun\n");
363 else
364 wake_up_interruptible_poll(&sdev->wait, EPOLLIN | EPOLLRDNORM);
365}
366
367static struct tbstream_frame *
368tbstream_dev_completed_rx(struct tbstream_dev *sdev)
369{
370 struct device *dma_dev = tb_ring_dma_device(sdev->rx_ring.ring);
371 struct tbstream_frame *sf;
372 int index;
373
374 index = sdev->rx_ring.cons % tb_ring_size(sdev->rx_ring.ring);
375 sf = &sdev->rx_ring.frames[index];
376 if (!sf->completed)
377 return NULL;
378
379 dma_sync_single_for_cpu(dma_dev, sf->frame.buffer_phy,
380 tb_ring_frame_size(&sf->frame),
381 DMA_FROM_DEVICE);
382 return sf;
383}
384
385static int tbstream_dev_consume_rx(struct tbstream_dev *sdev)
386{
387 struct device *dma_dev = tb_ring_dma_device(sdev->rx_ring.ring);
388 struct tbstream_frame *sf;
389 int index;
390
391 index = sdev->rx_ring.cons % tb_ring_size(sdev->rx_ring.ring);
392 sdev->rx_ring.cons++;
393
394 sf = &sdev->rx_ring.frames[index];
395 sf->completed = false;
396 sf->offset = 0;
397 sf->frame.size = 0;
398
399 dma_sync_single_for_device(dma_dev, sf->frame.buffer_phy,
400 tb_ring_frame_size(&sf->frame),
401 DMA_FROM_DEVICE);
402
403 return tb_ring_rx(sdev->rx_ring.ring, &sf->frame);
404}
405
406static int tbstream_dev_alloc_rx_buffers(struct tbstream_dev *sdev)
407{
408 size_t ring_size = tb_ring_size(sdev->rx_ring.ring);
409 int i;
410
411 sdev->rx_ring.frames = kzalloc_objs(struct tbstream_frame, ring_size);
412 if (!sdev->rx_ring.frames)
413 return -ENOMEM;
414
415 for (i = 0; i < ring_size; i++) {
416 struct device *dma_dev = tb_ring_dma_device(sdev->rx_ring.ring);
417 struct tbstream_frame *sf = &sdev->rx_ring.frames[i];
418 dma_addr_t dma_addr;
419
420 sf->page = alloc_page(GFP_KERNEL);
421 if (!sf->page)
422 return -ENOMEM;
423
424 dma_addr = dma_map_page(dma_dev, sf->page, 0, TB_MAX_FRAME_SIZE,
425 DMA_FROM_DEVICE);
426 if (dma_mapping_error(dma_dev, dma_addr)) {
427 __free_page(sf->page);
428 sf->page = NULL;
429 return -ENOMEM;
430 }
431
432 sf->sdev = sdev;
433 sf->frame.callback = tbstream_dev_rx_callback;
434 sf->frame.buffer_phy = dma_addr;
435
436 tb_ring_rx(sdev->rx_ring.ring, &sf->frame);
437 }
438
439 sdev->rx_ring.cons = 0;
440 sdev->rx_ring.prod = 0;
441 return 0;
442}
443
444static void
445tbstream_dev_tx_callback(struct tb_ring *ring, struct ring_frame *frame,
446 bool canceled)
447{
448 struct tbstream_frame *sf = container_of(frame, typeof(*sf), frame);
449 struct tbstream_dev *sdev = sf->sdev;
450
451 if (canceled)
452 return;
453
454 sdev->tx_ring.prod++;
455 if (sf->frame.eof == TBSTREAM_DATA)
456 wake_up_interruptible_poll(&sdev->wait, EPOLLOUT | EPOLLWRNORM);
457}
458
459static int tbstream_dev_alloc_tx_buffers(struct tbstream_dev *sdev)
460{
461 struct device *dma_dev = tb_ring_dma_device(sdev->tx_ring.ring);
462 size_t ring_size = tb_ring_size(sdev->tx_ring.ring);
463 int i;
464
465 sdev->tx_ring.frames = kzalloc_objs(struct tbstream_frame, ring_size);
466 if (!sdev->tx_ring.frames)
467 return -ENOMEM;
468
469 for (i = 0; i < ring_size; i++) {
470 struct tbstream_frame *sf = &sdev->tx_ring.frames[i];
471 dma_addr_t dma_addr;
472
473 sf->page = alloc_page(GFP_KERNEL);
474 if (!sf->page)
475 return -ENOMEM;
476
477 dma_addr = dma_map_page(dma_dev, sf->page, 0, TB_MAX_FRAME_SIZE,
478 DMA_TO_DEVICE);
479 if (dma_mapping_error(dma_dev, dma_addr)) {
480 __free_page(sf->page);
481 sf->page = NULL;
482 return -ENOMEM;
483 }
484
485 sf->sdev = sdev;
486 sf->frame.callback = tbstream_dev_tx_callback;
487 sf->frame.buffer_phy = dma_addr;
488 sf->frame.sof = TBSTREAM_FRAME_START;
489 }
490
491 sdev->tx_ring.cons = 0;
492 sdev->tx_ring.prod = ring_size - 1;
493 return 0;
494}
495
496static struct tbstream_frame *
497tbstream_dev_alloc_tx(struct tbstream_dev *sdev, enum tbstream_frame_pdf pdf,
498 struct iov_iter *from, size_t size)
499{
500 struct device *dma_dev = tb_ring_dma_device(sdev->tx_ring.ring);
501 struct tbstream_frame *sf;
502 int index;
503
504 if (!tbstream_ring_available(&sdev->tx_ring))
505 return ERR_PTR(-ENOBUFS);
506
507 index = sdev->tx_ring.cons % tb_ring_size(sdev->tx_ring.ring);
508 sdev->tx_ring.cons++;
509
510 sf = &sdev->tx_ring.frames[index];
511 sf->frame.size = size < TB_MAX_FRAME_SIZE ? size : 0;
512 sf->frame.eof = pdf;
513
514 dma_sync_single_for_cpu(dma_dev, sf->frame.buffer_phy, size,
515 DMA_TO_DEVICE);
516 if (pdf == TBSTREAM_DATA) {
517 if (copy_page_from_iter(sf->page, 0, size, from) != size) {
518 sdev->tx_ring.cons--;
519 return ERR_PTR(-EFAULT);
520 }
521 } else {
522 memset(page_address(sf->page), 0, size);
523 }
524 dma_sync_single_for_device(dma_dev, sf->frame.buffer_phy, size,
525 DMA_TO_DEVICE);
526 return sf;
527}
528
529static int
530tbstream_dev_send_data(struct tbstream_dev *sdev, struct iov_iter *from,
531 size_t size)
532{
533 struct tbstream_frame *sf;
534
535 sf = tbstream_dev_alloc_tx(sdev, TBSTREAM_DATA, from, size);
536 if (IS_ERR(sf))
537 return PTR_ERR(sf);
538 return tb_ring_tx(sdev->tx_ring.ring, &sf->frame);
539}
540
541static void
542tbstream_dev_poll_ring(struct tbstream_dev *sdev, struct tbstream_ring *ring)
543{
544 struct ring_frame *frame;
545
546 if (!sdev->busy_poll)
547 return;
548
549 while ((frame = tb_ring_poll(ring->ring)))
550 frame->callback(ring->ring, frame, false);
551}
552
553static int tbstream_dev_send_close(struct tbstream_dev *sdev)
554{
555 struct tbstream_frame *sf;
556
557 if (sdev->busy_poll) {
558 /*
559 * When busy polling it's the write(2) path that
560 * advances the completions so it is possible that the
561 * ring is full at this point. Advance the ring here so
562 * that there is room for the CLOSE packet to be sent.
563 */
564 ktime_t timeout = ktime_add_ms(ktime_get(), 500);
565
566 do {
567 if (tbstream_ring_available(&sdev->tx_ring))
568 break;
569 tbstream_dev_poll_ring(sdev, &sdev->tx_ring);
570 fsleep(15);
571 } while (ktime_before(ktime_get(), timeout));
572 }
573
574 sf = tbstream_dev_alloc_tx(sdev, TBSTREAM_CLOSE, NULL, SZ_256);
575 if (IS_ERR(sf))
576 return PTR_ERR(sf);
577 return tb_ring_tx(sdev->tx_ring.ring, &sf->frame);
578}
579
580static int tbstream_dev_start(struct tbstream_dev *sdev)
581{
582 struct tb_xdomain *xd = tbstream_dev_xdomain(sdev);
583 unsigned int flags = RING_FLAG_FRAME | RING_FLAG_E2E;
584 u16 sof_mask, eof_mask;
585 struct tb_ring *ring;
586 int ret, e2e_tx_hop;
587
588 if (sdev->busy_poll)
589 flags |= RING_FLAG_NO_INTERRUPT;
590
591 ring = tb_ring_alloc_tx(xd->tb->nhi, -1, sdev->ring_size, flags);
592 if (!ring)
593 return -ENOMEM;
594 sdev->tx_ring.ring = ring;
595
596 ret = tbstream_dev_alloc_tx_buffers(sdev);
597 if (ret)
598 goto err_free_tx;
599
600 e2e_tx_hop = ring->hop;
601 sof_mask = BIT(TBSTREAM_FRAME_START);
602 eof_mask = BIT(TBSTREAM_DATA) | BIT(TBSTREAM_CLOSE);
603
604 ring = tb_ring_alloc_rx(xd->tb->nhi, -1, sdev->ring_size, flags,
605 e2e_tx_hop, sof_mask, eof_mask, NULL, NULL);
606 if (!ring) {
607 ret = -ENOMEM;
608 goto err_free_tx_buffers;
609 }
610 sdev->rx_ring.ring = ring;
611
612 ret = tb_xdomain_enable_paths(xd, sdev->out_hopid,
613 sdev->tx_ring.ring->hop,
614 sdev->in_hopid,
615 sdev->rx_ring.ring->hop);
616 if (ret)
617 goto err_free_rx;
618
619 tb_ring_throttling(sdev->tx_ring.ring, sdev->throttling);
620 tb_ring_throttling(sdev->rx_ring.ring, sdev->throttling);
621
622 tb_ring_start(sdev->tx_ring.ring);
623 tb_ring_start(sdev->rx_ring.ring);
624
625 ret = tbstream_dev_alloc_rx_buffers(sdev);
626 if (ret)
627 goto err_stop;
628 return 0;
629
630err_stop:
631 tb_ring_stop(sdev->rx_ring.ring);
632 tb_ring_stop(sdev->tx_ring.ring);
633err_free_rx:
634 tb_ring_free(sdev->rx_ring.ring);
635err_free_tx_buffers:
636 tbstream_ring_free(&sdev->tx_ring);
637err_free_tx:
638 tb_ring_free(sdev->tx_ring.ring);
639
640 return ret;
641}
642
643static bool tbstream_dev_tx_drained(const struct tbstream_dev *sdev)
644{
645 const struct tbstream_ring *ring = &sdev->tx_ring;
646
647 /*
648 * Everything is completed when number of free TX slots is back
649 * to the maximum.
650 */
651 return ring->prod - ring->cons == tb_ring_size(ring->ring) - 1;
652}
653
654static void tbstream_dev_stop(struct tbstream_dev *sdev)
655{
656 struct tb_xdomain *xd;
657
658 if (sdev->busy_poll) {
659 /*
660 * When busy polling we must advance the ring ourselves
661 * to push all outstanding frames on the wire.
662 */
663 ktime_t timeout = ktime_add_ms(ktime_get(), 500);
664
665 do {
666 if (tbstream_dev_tx_drained(sdev))
667 break;
668 tbstream_dev_poll_ring(sdev, &sdev->tx_ring);
669 fsleep(15);
670 } while (ktime_before(ktime_get(), timeout));
671
672 tb_ring_stop(sdev->tx_ring.ring);
673 tb_ring_stop(sdev->rx_ring.ring);
674 } else {
675 tb_ring_flush(sdev->tx_ring.ring, 500);
676 tb_ring_stop(sdev->tx_ring.ring);
677 tb_ring_flush(sdev->rx_ring.ring, 500);
678 tb_ring_stop(sdev->rx_ring.ring);
679 }
680
681 xd = tbstream_dev_xdomain(sdev);
682 if (xd) {
683 tb_xdomain_disable_paths(xd, sdev->out_hopid,
684 sdev->tx_ring.ring->hop,
685 sdev->in_hopid,
686 sdev->rx_ring.ring->hop);
687 }
688
689 tbstream_ring_free(&sdev->rx_ring);
690 tb_ring_free(sdev->rx_ring.ring);
691 sdev->rx_ring.ring = NULL;
692 tbstream_ring_free(&sdev->tx_ring);
693 tb_ring_free(sdev->tx_ring.ring);
694 sdev->tx_ring.ring = NULL;
695}
696
697/* Use only with read_iter/write_iter() to handle nowait */
698static int tbstream_dev_lock(struct tbstream_dev *sdev, bool nowait)
699{
700 if (nowait) {
701 if (!mutex_trylock(&sdev->lock))
702 return -EAGAIN;
703 } else {
704 if (mutex_lock_interruptible(&sdev->lock))
705 return -ERESTARTSYS;
706 }
707 return 0;
708}
709
710static ssize_t
711tbstream_dev_fops_read_iter(struct kiocb *kiocb, struct iov_iter *to)
712{
713 struct file *file = kiocb->ki_filp;
714 bool nowait = file->f_flags & O_NONBLOCK || kiocb->ki_flags & IOCB_NOWAIT;
715 struct tbstream_dev *sdev = to_tbstream_dev(file->private_data);
716 size_t nbytes;
717 int ret;
718
719 ret = tbstream_dev_valid(sdev);
720 if (ret)
721 return ret;
722
723 ret = tbstream_dev_lock(sdev, nowait);
724 if (ret)
725 return ret;
726
727 for (;;) {
728 /* When busy polling, advance any completions manually */
729 tbstream_dev_poll_ring(sdev, &sdev->rx_ring);
730
731 ret = tbstream_dev_valid(sdev);
732 if (ret) {
733 mutex_unlock(&sdev->lock);
734 return ret;
735 }
736
737 if (tbstream_dev_closed(sdev) || tbstream_dev_removed(sdev)) {
738 mutex_unlock(&sdev->lock);
739 return 0;
740 }
741
742 if (tbstream_ring_available(&sdev->rx_ring))
743 break;
744
745 mutex_unlock(&sdev->lock);
746
747 if (nowait)
748 return -EAGAIN;
749
750 if (sdev->busy_poll) {
751 if (signal_pending(current))
752 return -ERESTARTSYS;
753 cond_resched();
754 } else {
755 ret = wait_event_interruptible(sdev->wait,
756 tbstream_ring_available(&sdev->rx_ring) ||
757 tbstream_dev_valid(sdev) != 0 ||
758 tbstream_dev_closed(sdev) ||
759 tbstream_dev_removed(sdev));
760 if (ret)
761 return ret;
762 }
763
764 ret = tbstream_dev_lock(sdev, nowait);
765 if (ret)
766 return ret;
767 }
768
769 nbytes = 0;
770 while (iov_iter_count(to)) {
771 struct tbstream_frame *sf;
772 size_t size, sf_size;
773
774 sf = tbstream_dev_completed_rx(sdev);
775 if (!sf)
776 break;
777 /*
778 * CLOSE tunneled packet. If userspace already read
779 * something then we stop processing now and return
780 * those bytes. Next time the first frame will be CLOSE
781 * in which case we return EOF to the user.
782 */
783 if (sf->frame.eof == TBSTREAM_CLOSE) {
784 if (!nbytes) {
785 tbstream_dev_consume_rx(sdev);
786 sdev->closed = true;
787 }
788 break;
789 }
790
791 sf_size = tb_ring_frame_size(&sf->frame);
792 size = min(iov_iter_count(to), sf_size);
793
794 if (copy_page_to_iter(sf->page, sf->offset, size, to) != size) {
795 ret = -EFAULT;
796 break;
797 }
798
799 /*
800 * If not all data from the frame is read so leave it in
801 * place and update the offset accordingly so next read
802 * gets the rest.
803 */
804 if (size < sf_size) {
805 sf->offset += size;
806 sf->frame.size = sf_size - size;
807 } else {
808 ret = tbstream_dev_consume_rx(sdev);
809 if (ret)
810 break;
811 }
812
813 nbytes += size;
814 }
815
816 mutex_unlock(&sdev->lock);
817 if (ret)
818 return ret;
819 return nbytes;
820}
821
822static ssize_t
823tbstream_dev_fops_write_iter(struct kiocb *kiocb, struct iov_iter *from)
824{
825 struct file *file = kiocb->ki_filp;
826 bool nowait = file->f_flags & O_NONBLOCK || kiocb->ki_flags & IOCB_NOWAIT;
827 struct tbstream_dev *sdev = to_tbstream_dev(file->private_data);
828 size_t nbytes;
829 int ret;
830
831 ret = tbstream_dev_valid(sdev);
832 if (ret)
833 return ret;
834
835 ret = tbstream_dev_lock(sdev, nowait);
836 if (ret)
837 return ret;
838
839 for (;;) {
840 tbstream_dev_poll_ring(sdev, &sdev->tx_ring);
841
842 ret = tbstream_dev_valid(sdev);
843 if (ret) {
844 mutex_unlock(&sdev->lock);
845 return ret;
846 }
847
848 if (tbstream_dev_closed(sdev) || tbstream_dev_removed(sdev)) {
849 mutex_unlock(&sdev->lock);
850 return -ENXIO;
851 }
852
853 if (tbstream_ring_available(&sdev->tx_ring))
854 break;
855
856 mutex_unlock(&sdev->lock);
857
858 if (nowait)
859 return -EAGAIN;
860
861 if (sdev->busy_poll) {
862 if (signal_pending(current))
863 return -ERESTARTSYS;
864 cond_resched();
865 } else {
866 ret = wait_event_interruptible(sdev->wait,
867 tbstream_ring_available(&sdev->tx_ring) ||
868 tbstream_dev_valid(sdev) != 0 ||
869 tbstream_dev_closed(sdev) ||
870 tbstream_dev_removed(sdev));
871 if (ret)
872 return ret;
873 }
874
875 ret = tbstream_dev_lock(sdev, nowait);
876 if (ret)
877 return ret;
878 }
879
880 nbytes = 0;
881 while (iov_iter_count(from)) {
882 size_t size;
883
884 size = min(iov_iter_count(from), TB_MAX_FRAME_SIZE);
885 ret = tbstream_dev_send_data(sdev, from, size);
886 if (ret) {
887 /*
888 * If there are no more buffers we are done for
889 * this write.
890 */
891 if (ret == -ENOBUFS)
892 ret = 0;
893 break;
894 }
895
896 nbytes += size;
897 }
898
899 mutex_unlock(&sdev->lock);
900 if (ret)
901 return ret;
902 return nbytes;
903}
904
905static __poll_t
906tbstream_dev_fops_poll(struct file *file, struct poll_table_struct *wait)
907{
908 struct tbstream_dev *sdev = to_tbstream_dev(file->private_data);
909 __poll_t mask = 0;
910
911 /*
912 * Without interrupts there is nothing that can wake us up so
913 * return failure instead.
914 */
915 if (sdev->busy_poll)
916 return EPOLLERR;
917
918 poll_wait(file, &sdev->wait, wait);
919 guard(mutex)(&sdev->lock);
920 if (tbstream_dev_valid(sdev) != 0) {
921 mask |= EPOLLHUP | EPOLLERR;
922 } else {
923 if (tbstream_ring_available(&sdev->tx_ring))
924 mask |= EPOLLOUT | EPOLLWRNORM;
925 if (tbstream_ring_available(&sdev->rx_ring))
926 mask |= EPOLLIN | EPOLLRDNORM;
927 }
928 return mask;
929}
930
931static int tbstream_dev_fops_open(struct inode *inode, struct file *file)
932{
933 struct tbstream_dev *sdev = to_tbstream_dev(file->private_data);
934 int ret;
935
936 tbstream_dev_get(sdev);
937
938 if (mutex_lock_interruptible(&sdev->lock)) {
939 tbstream_dev_put(sdev);
940 return -ERESTARTSYS;
941 }
942
943 /*
944 * If there is no stream attached yet, block until it appears
945 * unless this is opened in non-blocking mode.
946 */
947 while ((ret = tbstream_dev_valid(sdev))) {
948 mutex_unlock(&sdev->lock);
949
950 if (ret != -ENXIO || (file->f_flags & O_NONBLOCK))
951 goto err_put;
952
953 ret = wait_event_interruptible(sdev->wait,
954 tbstream_dev_valid(sdev) == 0 ||
955 tbstream_dev_removed(sdev));
956 if (ret)
957 goto err_put;
958
959 if (tbstream_dev_removed(sdev)) {
960 ret = -ENXIO;
961 goto err_put;
962 }
963
964 if (mutex_lock_interruptible(&sdev->lock)) {
965 ret = -ERESTARTSYS;
966 goto err_put;
967 }
968 }
969
970 /* Only on first open we allocate rings and enable paths */
971 if (!sdev->users++) {
972 ret = tbstream_dev_start(sdev);
973 if (ret) {
974 sdev->users--;
975 goto err_unlock;
976 }
977 sdev->closed = false;
978 }
979
980 mutex_unlock(&sdev->lock);
981
982 /* Stream handles IOCB_NOWAIT just fine */
983 file->f_mode |= FMODE_NOWAIT;
984 return 0;
985
986err_unlock:
987 mutex_unlock(&sdev->lock);
988err_put:
989 tbstream_dev_put(sdev);
990
991 return ret;
992}
993
994static int tbstream_dev_fops_release(struct inode *inode, struct file *file)
995{
996 struct tbstream_dev *sdev = to_tbstream_dev(file->private_data);
997
998 mutex_lock(&sdev->lock);
999 if (--sdev->users == 0) {
1000 /*
1001 * Send CLOSE tunneled packet to notify the other end
1002 * that we are closing the file. We do this twice if the
1003 * first one fails.
1004 */
1005 tbstream_dev_send_close(sdev);
1006 tbstream_dev_stop(sdev);
1007 }
1008 mutex_unlock(&sdev->lock);
1009
1010 tbstream_dev_put(sdev);
1011 return 0;
1012}
1013
1014static const struct file_operations tbstream_dev_fops = {
1015 .owner = THIS_MODULE,
1016 .llseek = noop_llseek,
1017 .read_iter = tbstream_dev_fops_read_iter,
1018 .write_iter = tbstream_dev_fops_write_iter,
1019 .poll = tbstream_dev_fops_poll,
1020 .open = tbstream_dev_fops_open,
1021 .release = tbstream_dev_fops_release,
1022};
1023
1024static inline struct tbstream_dev *
1025tbstream_dev_from_group(struct config_group *group)
1026{
1027 return container_of(group, struct tbstream_dev, group);
1028}
1029
1030static ssize_t tbstream_dev_busy_poll_show(struct config_item *item, char *buf)
1031{
1032 struct config_group *group = to_config_group(item);
1033 struct tbstream_dev *sdev = tbstream_dev_from_group(group);
1034
1035 return sysfs_emit(buf, "%u\n", sdev->busy_poll);
1036}
1037
1038static ssize_t
1039tbstream_dev_busy_poll_store(struct config_item *item, const char *buf,
1040 size_t count)
1041{
1042 struct config_group *group = to_config_group(item);
1043 struct tbstream_dev *sdev = tbstream_dev_from_group(group);
1044 bool busy_poll;
1045 int ret;
1046
1047 ret = kstrtobool(buf, &busy_poll);
1048 if (ret)
1049 return ret;
1050
1051 guard(mutex)(&sdev->lock);
1052 if (sdev->users)
1053 return -EBUSY;
1054 sdev->busy_poll = busy_poll;
1055 return count;
1056}
1057CONFIGFS_ATTR(tbstream_dev_, busy_poll);
1058
1059static ssize_t tbstream_dev_index_show(struct config_item *item, char *buf)
1060{
1061 struct config_group *group = to_config_group(item);
1062 struct tbstream_dev *sdev = tbstream_dev_from_group(group);
1063
1064 return sysfs_emit(buf, "%d\n", sdev->index);
1065}
1066CONFIGFS_ATTR_RO(tbstream_dev_, index);
1067
1068static ssize_t tbstream_dev_in_hopid_show(struct config_item *item, char *buf)
1069{
1070 struct config_group *group = to_config_group(item);
1071 struct tbstream_dev *sdev = tbstream_dev_from_group(group);
1072
1073 return sysfs_emit(buf, "%d\n", sdev->in_hopid);
1074}
1075
1076/* svc->lock must be held */
1077static void service_remove_properties(struct tb_service *svc, const char *name)
1078{
1079 struct tb_property *p;
1080
1081 if (!svc->local_properties)
1082 return;
1083
1084 p = tb_property_find(svc->local_properties, name,
1085 TB_PROPERTY_TYPE_DIRECTORY);
1086 if (p) {
1087 tb_property_free_dir(p->value.dir);
1088 tb_property_remove(p);
1089
1090 dev_dbg(&svc->dev, "removed local directory %s\n", name);
1091
1092 /*
1093 * Is the service directory empty already? If it is then
1094 * we can release it as well.
1095 */
1096 tb_property_for_each(svc->local_properties, p) {
1097 if (p->type == TB_PROPERTY_TYPE_DIRECTORY)
1098 return;
1099 }
1100
1101 tb_property_free_dir(svc->local_properties);
1102 svc->local_properties = NULL;
1103 }
1104}
1105
1106static int service_update_properties(struct tb_service *svc, const char *name,
1107 int in_hopid, int out_hopid)
1108{
1109 struct tb_property_dir *dir;
1110 struct tb_property *p;
1111
1112 guard(mutex)(&svc->lock);
1113
1114 if (in_hopid < 8 || out_hopid < 8) {
1115 service_remove_properties(svc, name);
1116 return 0;
1117 }
1118
1119 if (!svc->local_properties) {
1120 /*
1121 * Add the service directory first time we
1122 * populate the entries.
1123 */
1124 svc->local_properties = tb_property_copy_dir(tbstream_dir);
1125 if (!svc->local_properties)
1126 return -ENOMEM;
1127 }
1128
1129 p = tb_property_find(svc->local_properties, name,
1130 TB_PROPERTY_TYPE_DIRECTORY);
1131 if (p) {
1132 dir = p->value.dir;
1133
1134 p = tb_property_find(dir, "inhopid", TB_PROPERTY_TYPE_VALUE);
1135 if (p && p->value.immediate != in_hopid)
1136 p->value.immediate = in_hopid;
1137 p = tb_property_find(dir, "outhopid", TB_PROPERTY_TYPE_VALUE);
1138 if (p && p->value.immediate != out_hopid)
1139 p->value.immediate = out_hopid;
1140
1141 dev_dbg(&svc->dev,
1142 "updated local directory %s: in HopID %d, out HopID %d\n",
1143 name, in_hopid, out_hopid);
1144 } else {
1145 uuid_t uuid;
1146 int ret;
1147
1148 uuid_gen(&uuid);
1149 dir = tb_property_create_dir(&uuid);
1150 if (!dir)
1151 return -ENOMEM;
1152
1153 tb_property_add_immediate(dir, "inhopid", in_hopid);
1154 tb_property_add_immediate(dir, "outhopid", out_hopid);
1155
1156 ret = tb_property_add_dir(svc->local_properties, name, dir);
1157 if (ret) {
1158 tb_property_free_dir(dir);
1159 return ret;
1160 }
1161
1162 dev_dbg(&svc->dev,
1163 "added local directory %s: in HopID %d, out HopID %d\n",
1164 name, in_hopid, out_hopid);
1165 }
1166
1167 return 0;
1168}
1169
1170static int tbstream_dev_update_properties(struct tbstream_dev *sdev)
1171{
1172 struct tbstream *stream;
1173 int ret;
1174
1175 stream = tbstream_get(sdev->stream);
1176 if (!stream)
1177 return 0;
1178
1179 ret = service_update_properties(stream->svc,
1180 config_item_name(&sdev->group.cg_item),
1181 sdev->in_hopid, sdev->out_hopid);
1182 if (!ret)
1183 tb_service_properties_changed(stream->svc);
1184
1185 tbstream_put(stream);
1186 return ret;
1187}
1188
1189static int tbstream_dev_alloc_in_hopid(struct tbstream_dev *sdev, int hopid)
1190{
1191 struct tb_xdomain *xd = tbstream_dev_xdomain(sdev);
1192 int ret;
1193
1194 if (sdev->in_hopid > 0 && sdev->in_hopid != hopid)
1195 tb_xdomain_release_in_hopid(xd, sdev->in_hopid);
1196 if (!hopid) {
1197 sdev->in_hopid = hopid;
1198 return 0;
1199 }
1200 ret = tb_xdomain_alloc_in_hopid(xd, hopid);
1201 if (ret < 0)
1202 return ret;
1203 /*
1204 * If specific HopID was asked by the user and we did not get
1205 * that one then release and return error instead.
1206 */
1207 if (hopid > 0 && hopid != ret) {
1208 tb_xdomain_release_in_hopid(xd, ret);
1209 return -EBUSY;
1210 }
1211 sdev->in_hopid = ret;
1212 return 0;
1213}
1214
1215static int tbstream_dev_alloc_out_hopid(struct tbstream_dev *sdev, int hopid)
1216{
1217 struct tb_xdomain *xd = tbstream_dev_xdomain(sdev);
1218 int ret;
1219
1220 if (sdev->out_hopid > 0 && sdev->out_hopid != hopid)
1221 tb_xdomain_release_out_hopid(xd, sdev->out_hopid);
1222 if (!hopid) {
1223 sdev->out_hopid = hopid;
1224 return 0;
1225 }
1226 ret = tb_xdomain_alloc_out_hopid(xd, hopid);
1227 if (ret < 0)
1228 return ret;
1229 if (hopid > 0 && hopid != ret) {
1230 tb_xdomain_release_out_hopid(xd, ret);
1231 return -EBUSY;
1232 }
1233 sdev->out_hopid = ret;
1234 return 0;
1235}
1236
1237static ssize_t
1238tbstream_dev_in_hopid_store(struct config_item *item, const char *buf,
1239 size_t count)
1240{
1241 struct config_group *group = to_config_group(item);
1242 struct tbstream_dev *sdev = tbstream_dev_from_group(group);
1243 int ret, in_hopid;
1244
1245 ret = kstrtoint(buf, 0, &in_hopid);
1246 if (ret)
1247 return ret;
1248
1249 guard(mutex)(&sdev->lock);
1250 if (sdev->users)
1251 return -EBUSY;
1252 if (sdev->stream) {
1253 ret = tbstream_dev_alloc_in_hopid(sdev, in_hopid);
1254 if (ret)
1255 return ret;
1256 ret = tbstream_dev_update_properties(sdev);
1257 } else {
1258 sdev->in_hopid = in_hopid;
1259 }
1260 return ret ? ret : count;
1261}
1262CONFIGFS_ATTR(tbstream_dev_, in_hopid);
1263
1264static ssize_t tbstream_dev_out_hopid_show(struct config_item *item, char *buf)
1265{
1266 struct config_group *group = to_config_group(item);
1267 struct tbstream_dev *sdev = tbstream_dev_from_group(group);
1268
1269 return sysfs_emit(buf, "%d\n", sdev->out_hopid);
1270}
1271
1272static ssize_t
1273tbstream_dev_out_hopid_store(struct config_item *item, const char *buf,
1274 size_t count)
1275{
1276 struct config_group *group = to_config_group(item);
1277 struct tbstream_dev *sdev = tbstream_dev_from_group(group);
1278 int ret, out_hopid;
1279
1280 ret = kstrtoint(buf, 0, &out_hopid);
1281 if (ret)
1282 return ret;
1283
1284 guard(mutex)(&sdev->lock);
1285 if (sdev->users)
1286 return -EBUSY;
1287 if (sdev->stream) {
1288 ret = tbstream_dev_alloc_out_hopid(sdev, out_hopid);
1289 if (ret)
1290 return ret;
1291 ret = tbstream_dev_update_properties(sdev);
1292 } else {
1293 sdev->out_hopid = out_hopid;
1294 }
1295 return ret ? ret : count;
1296}
1297CONFIGFS_ATTR(tbstream_dev_, out_hopid);
1298
1299static ssize_t tbstream_dev_ring_size_show(struct config_item *item, char *buf)
1300{
1301 struct config_group *group = to_config_group(item);
1302 struct tbstream_dev *sdev = tbstream_dev_from_group(group);
1303
1304 return sysfs_emit(buf, "%u\n", sdev->ring_size);
1305}
1306
1307static ssize_t
1308tbstream_dev_ring_size_store(struct config_item *item, const char *buf,
1309 size_t count)
1310{
1311 struct config_group *group = to_config_group(item);
1312 struct tbstream_dev *sdev = tbstream_dev_from_group(group);
1313 unsigned int ring_size;
1314 int ret;
1315
1316 ret = kstrtouint(buf, 0, &ring_size);
1317 if (ret)
1318 return ret;
1319
1320 if (ring_size < TBSTREAM_DEV_MIN_RING_SIZE ||
1321 ring_size > TBSTREAM_DEV_MAX_RING_SIZE)
1322 return -EINVAL;
1323
1324 guard(mutex)(&sdev->lock);
1325 if (sdev->users)
1326 return -EBUSY;
1327 sdev->ring_size = ring_size;
1328 return count;
1329}
1330CONFIGFS_ATTR(tbstream_dev_, ring_size);
1331
1332static ssize_t tbstream_dev_throttling_show(struct config_item *item, char *buf)
1333{
1334 struct config_group *group = to_config_group(item);
1335 struct tbstream_dev *sdev = tbstream_dev_from_group(group);
1336
1337 return sysfs_emit(buf, "%u\n", sdev->throttling);
1338}
1339
1340static ssize_t
1341tbstream_dev_throttling_store(struct config_item *item, const char *buf,
1342 size_t count)
1343{
1344 struct config_group *group = to_config_group(item);
1345 struct tbstream_dev *sdev = tbstream_dev_from_group(group);
1346 unsigned int throttling;
1347 int ret;
1348
1349 ret = kstrtouint(buf, 0, &throttling);
1350 if (ret)
1351 return ret;
1352
1353 if (throttling > TBSTREAM_DEV_MAX_THROTTLING)
1354 return -EINVAL;
1355
1356 guard(mutex)(&sdev->lock);
1357 if (sdev->users)
1358 return -EBUSY;
1359 sdev->throttling = throttling;
1360 return count;
1361}
1362CONFIGFS_ATTR(tbstream_dev_, throttling);
1363
1364static struct configfs_attribute *tbstream_dev_attrs[] = {
1365 &tbstream_dev_attr_busy_poll,
1366 &tbstream_dev_attr_index,
1367 &tbstream_dev_attr_in_hopid,
1368 &tbstream_dev_attr_out_hopid,
1369 &tbstream_dev_attr_ring_size,
1370 &tbstream_dev_attr_throttling,
1371 NULL,
1372};
1373
1374static void tbstream_dev_item_release(struct config_item *item)
1375{
1376 struct config_group *group = to_config_group(item);
1377 struct tbstream_dev *sdev = tbstream_dev_from_group(group);
1378
1379 misc_deregister(&sdev->misc);
1380 tbstream_dev_put(sdev);
1381}
1382
1383static struct configfs_item_operations tbstream_dev_item_ops = {
1384 .release = tbstream_dev_item_release,
1385};
1386
1387static const struct config_item_type tbstream_dev_type = {
1388 .ct_owner = THIS_MODULE,
1389 .ct_item_ops = &tbstream_dev_item_ops,
1390 .ct_attrs = tbstream_dev_attrs,
1391};
1392
1393static void service_get_hopids(struct tb_service *svc, const char *name,
1394 int *in_hopid, int *out_hopid)
1395{
1396 struct tb_property_dir *dir;
1397 struct tb_property *p;
1398
1399 guard(mutex)(&svc->lock);
1400
1401 /* See if we have directory entry with the matching name */
1402 p = tb_property_find(svc->remote_properties, name,
1403 TB_PROPERTY_TYPE_DIRECTORY);
1404 if (!p)
1405 return;
1406
1407 dir = p->value.dir;
1408
1409 /*
1410 * We need to reverse the HopIDs on our end so that in becomes
1411 * out and vice versa.
1412 */
1413 p = tb_property_find(dir, "inhopid", TB_PROPERTY_TYPE_VALUE);
1414 if (p && p->value.immediate >= 8)
1415 *out_hopid = p->value.immediate;
1416 p = tb_property_find(dir, "outhopid", TB_PROPERTY_TYPE_VALUE);
1417 if (p && p->value.immediate >= 8)
1418 *in_hopid = p->value.immediate;
1419}
1420
1421static void
1422tbstream_dev_attach_stream(struct tbstream_dev *sdev, struct tbstream_group *sg)
1423{
1424 const char *name = config_item_name(&sdev->group.cg_item);
1425 struct tbstream *stream;
1426
1427 stream = tbstream_get(sg->stream);
1428 if (!stream)
1429 return;
1430
1431 scoped_guard(mutex, &sdev->lock) {
1432 sdev->stream = stream;
1433 /*
1434 * If there is no existing configuration (or automatic
1435 * configuration is being used) check if the other side
1436 * has configuration for this and use it.
1437 */
1438 if (sdev->in_hopid <= 0 && sdev->out_hopid <= 0)
1439 service_get_hopids(stream->svc, name, &sdev->in_hopid,
1440 &sdev->out_hopid);
1441 if (sdev->in_hopid)
1442 tbstream_dev_alloc_in_hopid(sdev, sdev->in_hopid);
1443 if (sdev->out_hopid)
1444 tbstream_dev_alloc_out_hopid(sdev, sdev->out_hopid);
1445 }
1446
1447 service_update_properties(stream->svc, name, sdev->in_hopid,
1448 sdev->out_hopid);
1449 tb_service_properties_changed(stream->svc);
1450
1451 /* Notify any openerers that the stream is now attached */
1452 wake_up_interruptible(&sdev->wait);
1453}
1454
1455static void tbstream_dev_detach_stream(struct tbstream_dev *sdev)
1456{
1457 const char *name = config_item_name(&sdev->group.cg_item);
1458 struct tbstream *stream;
1459 struct tb_xdomain *xd;
1460
1461 scoped_guard(mutex, &sdev->lock) {
1462 stream = sdev->stream;
1463 if (!stream)
1464 return;
1465 sdev->stream = NULL;
1466 xd = tb_service_parent(stream->svc);
1467 if (sdev->out_hopid > 0)
1468 tb_xdomain_release_out_hopid(xd, sdev->out_hopid);
1469 if (sdev->in_hopid > 0)
1470 tb_xdomain_release_in_hopid(xd, sdev->in_hopid);
1471 }
1472
1473 service_update_properties(stream->svc, name, 0, 0);
1474 tb_service_properties_changed(stream->svc);
1475
1476 tbstream_put(stream);
1477
1478 /* Notify any task that the stream is not valid anymore */
1479 wake_up_interruptible_poll(&sdev->wait, EPOLLHUP | EPOLLERR);
1480}
1481
1482static inline struct tbstream_group *
1483to_tbstream_group(struct config_group *group)
1484{
1485 return container_of(group, struct tbstream_group, group);
1486}
1487
1488static struct config_group *
1489tbstream_dev_make_group(struct config_group *group, const char *name)
1490{
1491 struct tbstream_group *sg = to_tbstream_group(group);
1492 struct tbstream_dev *sdev;
1493 int ret, index;
1494
1495 /*
1496 * We want the names to be suitable for passing as property
1497 * directory names.
1498 */
1499 if (strlen(name) > TB_PROPERTY_KEY_SIZE)
1500 return ERR_PTR(-ENAMETOOLONG);
1501
1502 sdev = kzalloc_obj(*sdev);
1503 if (!sdev)
1504 return ERR_PTR(-ENOMEM);
1505
1506 index = ida_alloc(&tbstream_indices, GFP_KERNEL);
1507 if (index < 0) {
1508 kfree(sdev);
1509 return ERR_PTR(index);
1510 }
1511
1512 sdev->index = index;
1513 sdev->ring_size = TBSTREAM_DEV_RING_SIZE;
1514 sdev->throttling = TBSTREAM_DEV_THROTTLING;
1515 mutex_init(&sdev->lock);
1516 init_waitqueue_head(&sdev->wait);
1517 INIT_LIST_HEAD(&sdev->list);
1518 /* This point forward tbstream_dev_put() must be used to release sdev */
1519 kref_init(&sdev->kref);
1520
1521 config_group_init_type_name(&sdev->group, name, &tbstream_dev_type);
1522
1523 scoped_guard(mutex, &sg->lock)
1524 list_add_tail(&sdev->list, &sg->dev_list);
1525
1526 tbstream_dev_attach_stream(sdev, sg);
1527
1528 sdev->misc.name = kasprintf(GFP_KERNEL, "tbstream%d", index);
1529 sdev->misc.minor = MISC_DYNAMIC_MINOR;
1530 sdev->misc.fops = &tbstream_dev_fops;
1531
1532 ret = misc_register(&sdev->misc);
1533 if (ret) {
1534 tbstream_dev_detach_stream(sdev);
1535 scoped_guard(mutex, &sg->lock)
1536 list_del(&sdev->list);
1537 /* Calls tbstream_dev_put() */
1538 config_group_put(&sdev->group);
1539 return ERR_PTR(ret);
1540 }
1541
1542 return &sdev->group;
1543}
1544
1545static void
1546tbstream_dev_drop_item(struct config_group *group, struct config_item *item)
1547{
1548 struct config_group *sdev_group = to_config_group(item);
1549 struct tbstream_dev *sdev = tbstream_dev_from_group(sdev_group);
1550 struct tbstream_group *sg = to_tbstream_group(group);
1551
1552 scoped_guard(mutex, &sg->lock)
1553 list_del(&sdev->list);
1554 /* Notify any task that the underlying group was removed */
1555 sdev->removed = true;
1556 wake_up_interruptible_poll(&sdev->wait, EPOLLHUP | EPOLLERR);
1557 config_item_put(item);
1558}
1559
1560static struct configfs_group_operations tbstream_dev_group_ops = {
1561 .make_group = tbstream_dev_make_group,
1562 .drop_item = tbstream_dev_drop_item,
1563};
1564
1565static void tbstream_item_release(struct config_item *item)
1566{
1567 struct config_group *group = to_config_group(item);
1568 struct tbstream_group *sg = to_tbstream_group(group);
1569
1570 tbstream_put(sg->stream);
1571 kfree(sg);
1572}
1573
1574static struct configfs_item_operations tbstream_item_ops = {
1575 .release = tbstream_item_release,
1576};
1577
1578static const struct config_item_type tbstream_dev_group_type = {
1579 .ct_owner = THIS_MODULE,
1580 .ct_group_ops = &tbstream_dev_group_ops,
1581 .ct_item_ops = &tbstream_item_ops,
1582};
1583
1584static struct config_group *
1585tbstream_make_group(struct config_group *group, const char *name)
1586{
1587 struct tbstream_group *sg;
1588 struct tbstream *stream;
1589 int domain, index;
1590 u64 route;
1591
1592 /* Make sure the format is correct */
1593 if (sscanf(name, "%u-%llx.%u", &domain, &route, &index) != 3)
1594 return ERR_PTR(-EINVAL);
1595
1596 sg = kzalloc_obj(*sg);
1597 if (!sg)
1598 return ERR_PTR(-ENOMEM);
1599
1600 mutex_init(&sg->lock);
1601 INIT_LIST_HEAD(&sg->dev_list);
1602
1603 guard(mutex)(&tbstream_lock);
1604 list_for_each_entry(stream, &tbstream_list, list) {
1605 tbstream_get(stream);
1606 if (sysfs_streq(name, dev_name(&stream->svc->dev))) {
1607 sg->stream = stream;
1608 break;
1609 }
1610 tbstream_put(stream);
1611 }
1612
1613 config_group_init_type_name(&sg->group, name, &tbstream_dev_group_type);
1614 return &sg->group;
1615}
1616
1617static struct configfs_group_operations tbstream_group_ops = {
1618 .make_group = tbstream_make_group,
1619};
1620
1621static const struct config_item_type tbstream_group_type = {
1622 .ct_owner = THIS_MODULE,
1623 .ct_group_ops = &tbstream_group_ops,
1624};
1625
1626static struct config_group tbstream_group = {
1627 .cg_item = {
1628 .ci_namebuf = "stream",
1629 .ci_type = &tbstream_group_type,
1630 },
1631};
1632
1633/* Returns reference count increased */
1634static struct tbstream_group *tbstream_group_find(struct tbstream *stream)
1635{
1636 const char *name = dev_name(&stream->svc->dev);
1637 struct config_item *item;
1638
1639 guard(mutex)(&tbstream_group.cg_subsys->su_mutex);
1640 item = config_group_find_item(&tbstream_group, name);
1641 if (!item)
1642 return NULL;
1643 return to_tbstream_group(to_config_group(item));
1644}
1645
1646static void tbstream_group_attach_stream(struct tbstream *stream)
1647{
1648 struct tbstream_group *sg;
1649 struct tbstream_dev *sdev;
1650
1651 sg = tbstream_group_find(stream);
1652 if (!sg)
1653 return;
1654
1655 guard(mutex)(&sg->lock);
1656 if (WARN_ON(sg->stream)) {
1657 config_group_put(&sg->group);
1658 return;
1659 }
1660 sg->stream = tbstream_get(stream);
1661 /*
1662 * If there are existing stream devices, attach the stream to
1663 * them now.
1664 */
1665 list_for_each_entry(sdev, &sg->dev_list, list) {
1666 tbstream_dev_get(sdev);
1667 tbstream_dev_attach_stream(sdev, sg);
1668 tbstream_dev_put(sdev);
1669 }
1670
1671 config_group_put(&sg->group);
1672}
1673
1674static void tbstream_group_detach_stream(struct tbstream *stream)
1675{
1676 struct tbstream_group *sg;
1677 struct tbstream_dev *sdev;
1678
1679 sg = tbstream_group_find(stream);
1680 if (!sg)
1681 return;
1682
1683 guard(mutex)(&sg->lock);
1684 if (sg->stream) {
1685 /* Detach this stream from the stream devices */
1686 list_for_each_entry_reverse(sdev, &sg->dev_list, list) {
1687 tbstream_dev_get(sdev);
1688 tbstream_dev_detach_stream(sdev);
1689 tbstream_dev_put(sdev);
1690 }
1691 tbstream_put(sg->stream);
1692 sg->stream = NULL;
1693 }
1694
1695 config_group_put(&sg->group);
1696}
1697
1698static int tbstream_probe(struct tb_service *svc)
1699{
1700 struct tbstream *stream;
1701
1702 stream = kzalloc_obj(*stream);
1703 if (!stream)
1704 return -ENOMEM;
1705
1706 /* After this point, release stream by calling tbstream_put() */
1707 kref_init(&stream->kref);
1708 stream->svc = tb_service_get(svc);
1709 INIT_LIST_HEAD(&stream->list);
1710
1711 scoped_guard(mutex, &tbstream_lock)
1712 list_add_tail(&stream->list, &tbstream_list);
1713
1714 tbstream_group_attach_stream(stream);
1715 tb_service_set_drvdata(svc, stream);
1716 return 0;
1717}
1718
1719static void tbstream_remove(struct tb_service *svc)
1720{
1721 struct tbstream *stream = tb_service_get_drvdata(svc);
1722
1723 tbstream_group_detach_stream(stream);
1724 scoped_guard(mutex, &tbstream_lock)
1725 list_del(&stream->list);
1726 tbstream_put(stream);
1727}
1728
1729static int __maybe_unused tbstream_suspend(struct device *dev)
1730{
1731 struct tb_service *svc = tb_to_service(dev);
1732 struct tbstream *stream = tb_service_get_drvdata(svc);
1733 struct tbstream_group *sg;
1734 struct tbstream_dev *sdev;
1735
1736 sg = tbstream_group_find(stream);
1737 if (!sg)
1738 return 0;
1739
1740 list_for_each_entry_reverse(sdev, &sg->dev_list, list) {
1741 tbstream_dev_get(sdev);
1742 /* Stop the stream (if it was open) */
1743 if (sdev->users)
1744 tbstream_dev_stop(sdev);
1745 tbstream_dev_put(sdev);
1746 }
1747
1748 config_group_put(&sg->group);
1749 return 0;
1750}
1751
1752static int __maybe_unused tbstream_resume(struct device *dev)
1753{
1754 struct tb_service *svc = tb_to_service(dev);
1755 struct tbstream *stream = tb_service_get_drvdata(svc);
1756 struct tbstream_group *sg;
1757 struct tbstream_dev *sdev;
1758
1759 sg = tbstream_group_find(stream);
1760 if (!sg)
1761 return 0;
1762
1763 list_for_each_entry(sdev, &sg->dev_list, list) {
1764 tbstream_dev_get(sdev);
1765 if (sdev->users) {
1766 int ret;
1767
1768 ret = tbstream_dev_start(sdev);
1769 if (ret) {
1770 tbstream_dev_put(sdev);
1771 config_group_put(&sg->group);
1772 return ret;
1773 }
1774 }
1775 tbstream_dev_put(sdev);
1776 }
1777
1778 config_group_put(&sg->group);
1779 return 0;
1780}
1781
1782static const struct dev_pm_ops tbstream_pm_ops = {
1783 SET_SYSTEM_SLEEP_PM_OPS(tbstream_suspend, tbstream_resume)
1784};
1785
1786static const struct tb_service_id tbstream_ids[] = {
1787 { TB_SERVICE("stream", 1) },
1788 { }
1789};
1790MODULE_DEVICE_TABLE(tbsvc, tbstream_ids);
1791
1792static struct tb_service_driver tbstream_driver = {
1793 .driver = {
1794 .owner = THIS_MODULE,
1795 .name = "thunderbolt_stream",
1796 .pm = &tbstream_pm_ops,
1797 },
1798 .probe = tbstream_probe,
1799 .remove = tbstream_remove,
1800 .id_table = tbstream_ids,
1801};
1802
1803static int __init tbstream_init(void)
1804{
1805 int ret;
1806
1807 tbstream_dir = tb_property_create_dir(&tbstream_dir_uuid);
1808 if (!tbstream_dir)
1809 return -ENOMEM;
1810
1811 tb_property_add_immediate(tbstream_dir, "prtcid", 1);
1812 tb_property_add_immediate(tbstream_dir, "prtcvers", 1);
1813 tb_property_add_immediate(tbstream_dir, "prtcrevs", 0);
1814 tb_property_add_immediate(tbstream_dir, "prtcstns", 0);
1815
1816 ret = tb_register_property_dir("stream", tbstream_dir);
1817 if (ret)
1818 goto err_free_dir;
1819
1820 config_group_init(&tbstream_group);
1821 ret = tb_configfs_register_group(&tbstream_group);
1822 if (ret)
1823 goto err_unregister_dir;
1824
1825 ret = tb_register_service_driver(&tbstream_driver);
1826 if (ret)
1827 goto err_unregister_group;
1828 return 0;
1829
1830err_unregister_group:
1831 tb_configfs_unregister_group(&tbstream_group);
1832err_unregister_dir:
1833 tb_unregister_property_dir("stream", tbstream_dir);
1834err_free_dir:
1835 tb_property_free_dir(tbstream_dir);
1836 return ret;
1837}
1838module_init(tbstream_init);
1839
1840static void __exit tbstream_exit(void)
1841{
1842 tb_unregister_service_driver(&tbstream_driver);
1843 tb_configfs_unregister_group(&tbstream_group);
1844 tb_unregister_property_dir("stream", tbstream_dir);
1845 tb_property_free_dir(tbstream_dir);
1846 ida_destroy(&tbstream_indices);
1847}
1848module_exit(tbstream_exit);
1849
1850MODULE_AUTHOR("Alan Borzeszkowski <[email protected]>");
1851MODULE_AUTHOR("Mika Westerberg <[email protected]>");
1852MODULE_DESCRIPTION("Stream data over Thunderbolt/USB4 cable");
1853MODULE_LICENSE("GPL");