projects/linux·blob 2c455ddc · refs

1012 lines · 23.9 KiB

1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Public API and common code for kernel->userspace relay file support.
4 *
5 * See Documentation/filesystems/relay.rst for an overview.
6 *
7 * Copyright (C) 2002-2005 - Tom Zanussi ([email protected]), IBM Corp
8 * Copyright (C) 1999-2005 - Karim Yaghmour ([email protected])
9 *
10 * Moved to kernel/relay.c by Paul Mundt, 2006.
11 * November 2006 - CPU hotplug support by Mathieu Desnoyers
12 * ([email protected])
13 */
14#include <linux/errno.h>
15#include <linux/stddef.h>
16#include <linux/slab.h>
17#include <linux/export.h>
18#include <linux/string.h>
19#include <linux/relay.h>
20#include <linux/vmalloc.h>
21#include <linux/mm.h>
22#include <linux/cpu.h>
23#include <linux/splice.h>
24
25/* list of open channels, for cpu hotplug */
26static DEFINE_MUTEX(relay_channels_mutex);
27static LIST_HEAD(relay_channels);
28
29/*
30 * fault() vm_op implementation for relay file mapping.
31 */
32static vm_fault_t relay_buf_fault(struct vm_fault *vmf)
33{
34 struct page *page;
35 struct rchan_buf *buf = vmf->vma->vm_private_data;
36 pgoff_t pgoff = vmf->pgoff;
37
38 if (!buf)
39 return VM_FAULT_OOM;
40
41 page = vmalloc_to_page(buf->start + (pgoff << PAGE_SHIFT));
42 if (!page)
43 return VM_FAULT_SIGBUS;
44 get_page(page);
45 vmf->page = page;
46
47 return 0;
48}
49
50/*
51 * vm_ops for relay file mappings.
52 */
53static const struct vm_operations_struct relay_file_mmap_ops = {
54 .fault = relay_buf_fault,
55};
56
57/*
58 * allocate an array of pointers of struct page
59 */
60static struct page **relay_alloc_page_array(unsigned int n_pages)
61{
62 return kvzalloc_objs(struct page *, n_pages);
63}
64
65/*
66 * free an array of pointers of struct page
67 */
68static void relay_free_page_array(struct page **array)
69{
70 kvfree(array);
71}
72
73/**
74 * relay_mmap_prepare_buf: - mmap channel buffer to process address space
75 * @buf: the relay channel buffer
76 * @desc: describing what to map
77 *
78 * Returns 0 if ok, negative on error
79 *
80 * Caller should already have grabbed mmap_lock.
81 */
82static int relay_mmap_prepare_buf(struct rchan_buf *buf,
83 struct vm_area_desc *desc)
84{
85 unsigned long length = vma_desc_size(desc);
86
87 if (!buf)
88 return -EBADF;
89
90 if (length != (unsigned long)buf->chan->alloc_size)
91 return -EINVAL;
92
93 desc->vm_ops = &relay_file_mmap_ops;
94 vma_desc_set_flags(desc, VMA_DONTEXPAND_BIT);
95 desc->private_data = buf;
96
97 return 0;
98}
99
100/**
101 * relay_alloc_buf - allocate a channel buffer
102 * @buf: the buffer struct
103 * @size: total size of the buffer
104 *
105 * Returns a pointer to the resulting buffer, %NULL if unsuccessful. The
106 * passed in size will get page aligned, if it isn't already.
107 */
108static void *relay_alloc_buf(struct rchan_buf *buf, size_t *size)
109{
110 void *mem;
111 unsigned int i, j, n_pages;
112
113 *size = PAGE_ALIGN(*size);
114 n_pages = *size >> PAGE_SHIFT;
115
116 buf->page_array = relay_alloc_page_array(n_pages);
117 if (!buf->page_array)
118 return NULL;
119
120 for (i = 0; i < n_pages; i++) {
121 buf->page_array[i] = alloc_page(GFP_KERNEL | __GFP_ZERO);
122 if (unlikely(!buf->page_array[i]))
123 goto depopulate;
124 set_page_private(buf->page_array[i], (unsigned long)buf);
125 }
126 mem = vmap(buf->page_array, n_pages, VM_MAP, PAGE_KERNEL);
127 if (!mem)
128 goto depopulate;
129
130 buf->page_count = n_pages;
131 return mem;
132
133depopulate:
134 for (j = 0; j < i; j++)
135 __free_page(buf->page_array[j]);
136 relay_free_page_array(buf->page_array);
137 return NULL;
138}
139
140/**
141 * relay_create_buf - allocate and initialize a channel buffer
142 * @chan: the relay channel
143 *
144 * Returns channel buffer if successful, %NULL otherwise.
145 */
146static struct rchan_buf *relay_create_buf(struct rchan *chan)
147{
148 struct rchan_buf *buf;
149
150 if (chan->n_subbufs > KMALLOC_MAX_SIZE / sizeof(size_t))
151 return NULL;
152
153 buf = kzalloc_obj(struct rchan_buf);
154 if (!buf)
155 return NULL;
156 buf->padding = kmalloc_objs(size_t, chan->n_subbufs);
157 if (!buf->padding)
158 goto free_buf;
159
160 buf->start = relay_alloc_buf(buf, &chan->alloc_size);
161 if (!buf->start)
162 goto free_buf;
163
164 buf->chan = chan;
165 kref_get(&buf->chan->kref);
166 return buf;
167
168free_buf:
169 kfree(buf->padding);
170 kfree(buf);
171 return NULL;
172}
173
174/**
175 * relay_destroy_channel - free the channel struct
176 * @kref: target kernel reference that contains the relay channel
177 *
178 * Should only be called from kref_put().
179 */
180static void relay_destroy_channel(struct kref *kref)
181{
182 struct rchan *chan = container_of(kref, struct rchan, kref);
183 free_percpu(chan->buf);
184 kfree(chan);
185}
186
187/**
188 * relay_destroy_buf - destroy an rchan_buf struct and associated buffer
189 * @buf: the buffer struct
190 */
191static void relay_destroy_buf(struct rchan_buf *buf)
192{
193 struct rchan *chan = buf->chan;
194 unsigned int i;
195
196 if (likely(buf->start)) {
197 vunmap(buf->start);
198 for (i = 0; i < buf->page_count; i++)
199 __free_page(buf->page_array[i]);
200 relay_free_page_array(buf->page_array);
201 }
202 *per_cpu_ptr(chan->buf, buf->cpu) = NULL;
203 kfree(buf->padding);
204 kfree(buf);
205 kref_put(&chan->kref, relay_destroy_channel);
206}
207
208/**
209 * relay_remove_buf - remove a channel buffer
210 * @kref: target kernel reference that contains the relay buffer
211 *
212 * Removes the file from the filesystem, which also frees the
213 * rchan_buf_struct and the channel buffer. Should only be called from
214 * kref_put().
215 */
216static void relay_remove_buf(struct kref *kref)
217{
218 struct rchan_buf *buf = container_of(kref, struct rchan_buf, kref);
219 relay_destroy_buf(buf);
220}
221
222/**
223 * relay_buf_empty - boolean, is the channel buffer empty?
224 * @buf: channel buffer
225 *
226 * Returns 1 if the buffer is empty, 0 otherwise.
227 */
228static int relay_buf_empty(struct rchan_buf *buf)
229{
230 return (buf->subbufs_produced - buf->subbufs_consumed) ? 0 : 1;
231}
232
233/**
234 * relay_buf_full - boolean, is the channel buffer full?
235 * @buf: channel buffer
236 *
237 * Returns 1 if the buffer is full, 0 otherwise.
238 */
239int relay_buf_full(struct rchan_buf *buf)
240{
241 size_t ready = buf->subbufs_produced - buf->subbufs_consumed;
242 return (ready >= buf->chan->n_subbufs) ? 1 : 0;
243}
244EXPORT_SYMBOL_GPL(relay_buf_full);
245
246/*
247 * High-level relay kernel API and associated functions.
248 */
249
250static int relay_subbuf_start(struct rchan_buf *buf, void *subbuf,
251 void *prev_subbuf)
252{
253 int full = relay_buf_full(buf);
254
255 if (full)
256 buf->stats.full_count++;
257
258 if (!buf->chan->cb->subbuf_start)
259 return !full;
260
261 return buf->chan->cb->subbuf_start(buf, subbuf,
262 prev_subbuf);
263}
264
265/**
266 * wakeup_readers - wake up readers waiting on a channel
267 * @work: contains the channel buffer
268 *
269 * This is the function used to defer reader waking
270 */
271static void wakeup_readers(struct irq_work *work)
272{
273 struct rchan_buf *buf;
274
275 buf = container_of(work, struct rchan_buf, wakeup_work);
276 wake_up_interruptible(&buf->read_wait);
277}
278
279/**
280 * __relay_reset - reset a channel buffer
281 * @buf: the channel buffer
282 * @init: 1 if this is a first-time initialization
283 *
284 * See relay_reset() for description of effect.
285 */
286static void __relay_reset(struct rchan_buf *buf, unsigned int init)
287{
288 size_t i;
289
290 if (init) {
291 init_waitqueue_head(&buf->read_wait);
292 kref_init(&buf->kref);
293 init_irq_work(&buf->wakeup_work, wakeup_readers);
294 } else {
295 irq_work_sync(&buf->wakeup_work);
296 }
297
298 buf->subbufs_produced = 0;
299 buf->subbufs_consumed = 0;
300 buf->bytes_consumed = 0;
301 buf->finalized = 0;
302 buf->data = buf->start;
303 buf->offset = 0;
304 buf->stats.full_count = 0;
305 buf->stats.big_count = 0;
306
307 for (i = 0; i < buf->chan->n_subbufs; i++)
308 buf->padding[i] = 0;
309
310 relay_subbuf_start(buf, buf->data, NULL);
311}
312
313/**
314 * relay_reset - reset the channel
315 * @chan: the channel
316 *
317 * This has the effect of erasing all data from all channel buffers
318 * and restarting the channel in its initial state. The buffers
319 * are not freed, so any mappings are still in effect.
320 *
321 * NOTE. Care should be taken that the channel isn't actually
322 * being used by anything when this call is made.
323 */
324void relay_reset(struct rchan *chan)
325{
326 struct rchan_buf *buf;
327 unsigned int i;
328
329 if (!chan)
330 return;
331
332 if (chan->is_global && (buf = *per_cpu_ptr(chan->buf, 0))) {
333 __relay_reset(buf, 0);
334 return;
335 }
336
337 mutex_lock(&relay_channels_mutex);
338 for_each_possible_cpu(i)
339 if ((buf = *per_cpu_ptr(chan->buf, i)))
340 __relay_reset(buf, 0);
341 mutex_unlock(&relay_channels_mutex);
342}
343EXPORT_SYMBOL_GPL(relay_reset);
344
345static inline void relay_set_buf_dentry(struct rchan_buf *buf,
346 struct dentry *dentry)
347{
348 buf->dentry = dentry;
349 d_inode(buf->dentry)->i_size = buf->early_bytes;
350}
351
352static struct dentry *relay_create_buf_file(struct rchan *chan,
353 struct rchan_buf *buf,
354 unsigned int cpu)
355{
356 struct dentry *dentry;
357 char *tmpname;
358
359 tmpname = kasprintf(GFP_KERNEL, "%s%d", chan->base_filename, cpu);
360 if (!tmpname)
361 return NULL;
362
363 /* Create file in fs */
364 dentry = chan->cb->create_buf_file(tmpname, chan->parent,
365 S_IRUSR, buf,
366 &chan->is_global);
367 if (IS_ERR(dentry))
368 dentry = NULL;
369
370 kfree(tmpname);
371
372 return dentry;
373}
374
375/*
376 * relay_open_buf - create a new relay channel buffer
377 *
378 * used by relay_open() and CPU hotplug.
379 */
380static struct rchan_buf *relay_open_buf(struct rchan *chan, unsigned int cpu)
381{
382 struct rchan_buf *buf;
383 struct dentry *dentry;
384
385 if (chan->is_global)
386 return *per_cpu_ptr(chan->buf, 0);
387
388 buf = relay_create_buf(chan);
389 if (!buf)
390 return NULL;
391
392 if (chan->has_base_filename) {
393 dentry = relay_create_buf_file(chan, buf, cpu);
394 if (!dentry)
395 goto free_buf;
396 relay_set_buf_dentry(buf, dentry);
397 } else {
398 /* Only retrieve global info, nothing more, nothing less */
399 dentry = chan->cb->create_buf_file(NULL, NULL,
400 S_IRUSR, buf,
401 &chan->is_global);
402 if (IS_ERR_OR_NULL(dentry))
403 goto free_buf;
404 }
405
406 buf->cpu = cpu;
407 __relay_reset(buf, 1);
408
409 if(chan->is_global) {
410 *per_cpu_ptr(chan->buf, 0) = buf;
411 buf->cpu = 0;
412 }
413
414 return buf;
415
416free_buf:
417 relay_destroy_buf(buf);
418 return NULL;
419}
420
421/**
422 * relay_close_buf - close a channel buffer
423 * @buf: channel buffer
424 *
425 * Marks the buffer finalized and restores the default callbacks.
426 * The channel buffer and channel buffer data structure are then freed
427 * automatically when the last reference is given up.
428 */
429static void relay_close_buf(struct rchan_buf *buf)
430{
431 buf->finalized = 1;
432 irq_work_sync(&buf->wakeup_work);
433 buf->chan->cb->remove_buf_file(buf->dentry);
434 kref_put(&buf->kref, relay_remove_buf);
435}
436
437int relay_prepare_cpu(unsigned int cpu)
438{
439 struct rchan *chan;
440 struct rchan_buf *buf;
441
442 mutex_lock(&relay_channels_mutex);
443 list_for_each_entry(chan, &relay_channels, list) {
444 if (*per_cpu_ptr(chan->buf, cpu))
445 continue;
446 buf = relay_open_buf(chan, cpu);
447 if (!buf) {
448 pr_err("relay: cpu %d buffer creation failed\n", cpu);
449 mutex_unlock(&relay_channels_mutex);
450 return -ENOMEM;
451 }
452 *per_cpu_ptr(chan->buf, cpu) = buf;
453 }
454 mutex_unlock(&relay_channels_mutex);
455 return 0;
456}
457
458/**
459 * relay_open - create a new relay channel
460 * @base_filename: base name of files to create
461 * @parent: dentry of parent directory, %NULL for root directory or buffer
462 * @subbuf_size: size of sub-buffers
463 * @n_subbufs: number of sub-buffers
464 * @cb: client callback functions
465 * @private_data: user-defined data
466 *
467 * Returns channel pointer if successful, %NULL otherwise.
468 *
469 * Creates a channel buffer for each cpu using the sizes and
470 * attributes specified. The created channel buffer files
471 * will be named base_filename0...base_filenameN-1. File
472 * permissions will be %S_IRUSR.
473 */
474struct rchan *relay_open(const char *base_filename,
475 struct dentry *parent,
476 size_t subbuf_size,
477 size_t n_subbufs,
478 const struct rchan_callbacks *cb,
479 void *private_data)
480{
481 unsigned int i;
482 struct rchan *chan;
483 struct rchan_buf *buf;
484
485 if (!(subbuf_size && n_subbufs))
486 return NULL;
487 if (subbuf_size > UINT_MAX / n_subbufs)
488 return NULL;
489 if (!cb || !cb->create_buf_file || !cb->remove_buf_file)
490 return NULL;
491
492 chan = kzalloc_obj(struct rchan);
493 if (!chan)
494 return NULL;
495
496 chan->buf = alloc_percpu(struct rchan_buf *);
497 if (!chan->buf) {
498 kfree(chan);
499 return NULL;
500 }
501
502 chan->version = RELAYFS_CHANNEL_VERSION;
503 chan->n_subbufs = n_subbufs;
504 chan->subbuf_size = subbuf_size;
505 chan->alloc_size = PAGE_ALIGN(subbuf_size * n_subbufs);
506 chan->parent = parent;
507 chan->private_data = private_data;
508 if (base_filename) {
509 chan->has_base_filename = 1;
510 strscpy(chan->base_filename, base_filename, NAME_MAX);
511 }
512 chan->cb = cb;
513 kref_init(&chan->kref);
514
515 mutex_lock(&relay_channels_mutex);
516 for_each_online_cpu(i) {
517 buf = relay_open_buf(chan, i);
518 if (!buf)
519 goto free_bufs;
520 *per_cpu_ptr(chan->buf, i) = buf;
521 }
522 list_add(&chan->list, &relay_channels);
523 mutex_unlock(&relay_channels_mutex);
524
525 return chan;
526
527free_bufs:
528 for_each_possible_cpu(i) {
529 if ((buf = *per_cpu_ptr(chan->buf, i)))
530 relay_close_buf(buf);
531 }
532
533 kref_put(&chan->kref, relay_destroy_channel);
534 mutex_unlock(&relay_channels_mutex);
535 return NULL;
536}
537EXPORT_SYMBOL_GPL(relay_open);
538
539struct rchan_percpu_buf_dispatcher {
540 struct rchan_buf *buf;
541 struct dentry *dentry;
542};
543
544/**
545 * relay_switch_subbuf - switch to a new sub-buffer
546 * @buf: channel buffer
547 * @length: size of current event
548 *
549 * Returns either the length passed in or 0 if full.
550 *
551 * Performs sub-buffer-switch tasks such as invoking callbacks,
552 * updating padding counts, waking up readers, etc.
553 */
554size_t relay_switch_subbuf(struct rchan_buf *buf, size_t length)
555{
556 void *old, *new;
557 size_t old_subbuf, new_subbuf;
558
559 if (unlikely(length > buf->chan->subbuf_size))
560 goto toobig;
561
562 if (buf->offset != buf->chan->subbuf_size + 1) {
563 size_t prev_padding;
564
565 prev_padding = buf->chan->subbuf_size - buf->offset;
566 old_subbuf = buf->subbufs_produced % buf->chan->n_subbufs;
567 buf->padding[old_subbuf] = prev_padding;
568 buf->subbufs_produced++;
569 if (buf->dentry)
570 d_inode(buf->dentry)->i_size +=
571 buf->chan->subbuf_size -
572 buf->padding[old_subbuf];
573 else
574 buf->early_bytes += buf->chan->subbuf_size -
575 buf->padding[old_subbuf];
576 smp_mb();
577 if (waitqueue_active(&buf->read_wait)) {
578 /*
579 * Calling wake_up_interruptible() from here
580 * will deadlock if we happen to be logging
581 * from the scheduler (trying to re-grab
582 * rq->lock), so defer it.
583 */
584 irq_work_queue(&buf->wakeup_work);
585 }
586 }
587
588 old = buf->data;
589 new_subbuf = buf->subbufs_produced % buf->chan->n_subbufs;
590 new = buf->start + new_subbuf * buf->chan->subbuf_size;
591 buf->offset = 0;
592 if (!relay_subbuf_start(buf, new, old)) {
593 buf->offset = buf->chan->subbuf_size + 1;
594 return 0;
595 }
596 buf->data = new;
597 buf->padding[new_subbuf] = 0;
598
599 if (unlikely(length + buf->offset > buf->chan->subbuf_size))
600 goto toobig;
601
602 return length;
603
604toobig:
605 buf->stats.big_count++;
606 return 0;
607}
608EXPORT_SYMBOL_GPL(relay_switch_subbuf);
609
610/**
611 * relay_subbufs_consumed - update the buffer's sub-buffers-consumed count
612 * @chan: the channel
613 * @cpu: the cpu associated with the channel buffer to update
614 * @subbufs_consumed: number of sub-buffers to add to current buf's count
615 *
616 * Adds to the channel buffer's consumed sub-buffer count.
617 * subbufs_consumed should be the number of sub-buffers newly consumed,
618 * not the total consumed.
619 *
620 * NOTE. Kernel clients don't need to call this function if the channel
621 * mode is 'overwrite'.
622 */
623void relay_subbufs_consumed(struct rchan *chan,
624 unsigned int cpu,
625 size_t subbufs_consumed)
626{
627 struct rchan_buf *buf;
628
629 if (!chan || cpu >= NR_CPUS)
630 return;
631
632 buf = *per_cpu_ptr(chan->buf, cpu);
633 if (!buf || subbufs_consumed > chan->n_subbufs)
634 return;
635
636 if (subbufs_consumed > buf->subbufs_produced - buf->subbufs_consumed)
637 buf->subbufs_consumed = buf->subbufs_produced;
638 else
639 buf->subbufs_consumed += subbufs_consumed;
640}
641EXPORT_SYMBOL_GPL(relay_subbufs_consumed);
642
643/**
644 * relay_close - close the channel
645 * @chan: the channel
646 *
647 * Closes all channel buffers and frees the channel.
648 */
649void relay_close(struct rchan *chan)
650{
651 struct rchan_buf *buf;
652 unsigned int i;
653
654 if (!chan)
655 return;
656
657 mutex_lock(&relay_channels_mutex);
658 if (chan->is_global && (buf = *per_cpu_ptr(chan->buf, 0)))
659 relay_close_buf(buf);
660 else
661 for_each_possible_cpu(i)
662 if ((buf = *per_cpu_ptr(chan->buf, i)))
663 relay_close_buf(buf);
664
665 list_del(&chan->list);
666 kref_put(&chan->kref, relay_destroy_channel);
667 mutex_unlock(&relay_channels_mutex);
668}
669EXPORT_SYMBOL_GPL(relay_close);
670
671/**
672 * relay_flush - close the channel
673 * @chan: the channel
674 *
675 * Flushes all channel buffers, i.e. forces buffer switch.
676 */
677void relay_flush(struct rchan *chan)
678{
679 struct rchan_buf *buf;
680 unsigned int i;
681
682 if (!chan)
683 return;
684
685 if (chan->is_global && (buf = *per_cpu_ptr(chan->buf, 0))) {
686 relay_switch_subbuf(buf, 0);
687 return;
688 }
689
690 mutex_lock(&relay_channels_mutex);
691 for_each_possible_cpu(i)
692 if ((buf = *per_cpu_ptr(chan->buf, i)))
693 relay_switch_subbuf(buf, 0);
694 mutex_unlock(&relay_channels_mutex);
695}
696EXPORT_SYMBOL_GPL(relay_flush);
697
698/**
699 * relay_stats - get channel buffer statistics
700 * @chan: the channel
701 * @flags: select particular information to get
702 *
703 * Returns the count of certain field that caller specifies.
704 */
705size_t relay_stats(struct rchan *chan, int flags)
706{
707 unsigned int i, count = 0;
708 struct rchan_buf *rbuf;
709
710 if (!chan || flags > RELAY_STATS_LAST)
711 return 0;
712
713 if (chan->is_global) {
714 rbuf = *per_cpu_ptr(chan->buf, 0);
715 if (flags & RELAY_STATS_BUF_FULL)
716 count = rbuf->stats.full_count;
717 else if (flags & RELAY_STATS_WRT_BIG)
718 count = rbuf->stats.big_count;
719 } else {
720 for_each_online_cpu(i) {
721 rbuf = *per_cpu_ptr(chan->buf, i);
722 if (rbuf) {
723 if (flags & RELAY_STATS_BUF_FULL)
724 count += rbuf->stats.full_count;
725 else if (flags & RELAY_STATS_WRT_BIG)
726 count += rbuf->stats.big_count;
727 }
728 }
729 }
730
731 return count;
732}
733
734/**
735 * relay_file_open - open file op for relay files
736 * @inode: the inode
737 * @filp: the file
738 *
739 * Increments the channel buffer refcount.
740 */
741static int relay_file_open(struct inode *inode, struct file *filp)
742{
743 struct rchan_buf *buf = inode->i_private;
744 kref_get(&buf->kref);
745 filp->private_data = buf;
746
747 return nonseekable_open(inode, filp);
748}
749
750/**
751 * relay_file_mmap_prepare - mmap file op for relay files
752 * @desc: describing what to map
753 *
754 * Calls upon relay_mmap_prepare_buf() to map the file into user space.
755 */
756static int relay_file_mmap_prepare(struct vm_area_desc *desc)
757{
758 struct rchan_buf *buf = desc->file->private_data;
759
760 return relay_mmap_prepare_buf(buf, desc);
761}
762
763/**
764 * relay_file_poll - poll file op for relay files
765 * @filp: the file
766 * @wait: poll table
767 *
768 * Poll implemention.
769 */
770static __poll_t relay_file_poll(struct file *filp, poll_table *wait)
771{
772 __poll_t mask = 0;
773 struct rchan_buf *buf = filp->private_data;
774
775 if (buf->finalized)
776 return EPOLLERR;
777
778 if (filp->f_mode & FMODE_READ) {
779 poll_wait(filp, &buf->read_wait, wait);
780 if (!relay_buf_empty(buf))
781 mask |= EPOLLIN | EPOLLRDNORM;
782 }
783
784 return mask;
785}
786
787/**
788 * relay_file_release - release file op for relay files
789 * @inode: the inode
790 * @filp: the file
791 *
792 * Decrements the channel refcount, as the filesystem is
793 * no longer using it.
794 */
795static int relay_file_release(struct inode *inode, struct file *filp)
796{
797 struct rchan_buf *buf = filp->private_data;
798 kref_put(&buf->kref, relay_remove_buf);
799
800 return 0;
801}
802
803/*
804 * relay_file_read_consume - update the consumed count for the buffer
805 */
806static void relay_file_read_consume(struct rchan_buf *buf,
807 size_t read_pos,
808 size_t bytes_consumed)
809{
810 size_t subbuf_size = buf->chan->subbuf_size;
811 size_t n_subbufs = buf->chan->n_subbufs;
812 size_t read_subbuf;
813
814 if (buf->subbufs_produced == buf->subbufs_consumed &&
815 buf->offset == buf->bytes_consumed)
816 return;
817
818 if (buf->bytes_consumed + bytes_consumed > subbuf_size) {
819 relay_subbufs_consumed(buf->chan, buf->cpu, 1);
820 buf->bytes_consumed = 0;
821 }
822
823 buf->bytes_consumed += bytes_consumed;
824 if (!read_pos)
825 read_subbuf = buf->subbufs_consumed % n_subbufs;
826 else
827 read_subbuf = read_pos / buf->chan->subbuf_size;
828 if (buf->bytes_consumed + buf->padding[read_subbuf] == subbuf_size) {
829 if ((read_subbuf == buf->subbufs_produced % n_subbufs) &&
830 (buf->offset == subbuf_size))
831 return;
832 relay_subbufs_consumed(buf->chan, buf->cpu, 1);
833 buf->bytes_consumed = 0;
834 }
835}
836
837/*
838 * relay_file_read_avail - boolean, are there unconsumed bytes available?
839 */
840static int relay_file_read_avail(struct rchan_buf *buf)
841{
842 size_t subbuf_size = buf->chan->subbuf_size;
843 size_t n_subbufs = buf->chan->n_subbufs;
844 size_t produced = buf->subbufs_produced;
845 size_t consumed;
846
847 relay_file_read_consume(buf, 0, 0);
848
849 consumed = buf->subbufs_consumed;
850
851 if (unlikely(buf->offset > subbuf_size)) {
852 if (produced == consumed)
853 return 0;
854 return 1;
855 }
856
857 if (unlikely(produced - consumed >= n_subbufs)) {
858 consumed = produced - n_subbufs + 1;
859 buf->subbufs_consumed = consumed;
860 buf->bytes_consumed = 0;
861 }
862
863 produced = (produced % n_subbufs) * subbuf_size + buf->offset;
864 consumed = (consumed % n_subbufs) * subbuf_size + buf->bytes_consumed;
865
866 if (consumed > produced)
867 produced += n_subbufs * subbuf_size;
868
869 if (consumed == produced) {
870 if (buf->offset == subbuf_size &&
871 buf->subbufs_produced > buf->subbufs_consumed)
872 return 1;
873 return 0;
874 }
875
876 return 1;
877}
878
879/**
880 * relay_file_read_subbuf_avail - return bytes available in sub-buffer
881 * @read_pos: file read position
882 * @buf: relay channel buffer
883 */
884static size_t relay_file_read_subbuf_avail(size_t read_pos,
885 struct rchan_buf *buf)
886{
887 size_t padding, avail = 0;
888 size_t read_subbuf, read_offset, write_subbuf, write_offset;
889 size_t subbuf_size = buf->chan->subbuf_size;
890
891 write_subbuf = (buf->data - buf->start) / subbuf_size;
892 write_offset = buf->offset > subbuf_size ? subbuf_size : buf->offset;
893 read_subbuf = read_pos / subbuf_size;
894 read_offset = read_pos % subbuf_size;
895 padding = buf->padding[read_subbuf];
896
897 if (read_subbuf == write_subbuf) {
898 if (read_offset + padding < write_offset)
899 avail = write_offset - (read_offset + padding);
900 } else
901 avail = (subbuf_size - padding) - read_offset;
902
903 return avail;
904}
905
906/**
907 * relay_file_read_start_pos - find the first available byte to read
908 * @buf: relay channel buffer
909 *
910 * If the read_pos is in the middle of padding, return the
911 * position of the first actually available byte, otherwise
912 * return the original value.
913 */
914static size_t relay_file_read_start_pos(struct rchan_buf *buf)
915{
916 size_t read_subbuf, padding, padding_start, padding_end;
917 size_t subbuf_size = buf->chan->subbuf_size;
918 size_t n_subbufs = buf->chan->n_subbufs;
919 size_t consumed = buf->subbufs_consumed % n_subbufs;
920 size_t read_pos = (consumed * subbuf_size + buf->bytes_consumed)
921 % (n_subbufs * subbuf_size);
922
923 read_subbuf = read_pos / subbuf_size;
924 padding = buf->padding[read_subbuf];
925 padding_start = (read_subbuf + 1) * subbuf_size - padding;
926 padding_end = (read_subbuf + 1) * subbuf_size;
927 if (read_pos >= padding_start && read_pos < padding_end) {
928 read_subbuf = (read_subbuf + 1) % n_subbufs;
929 read_pos = read_subbuf * subbuf_size;
930 }
931
932 return read_pos;
933}
934
935/**
936 * relay_file_read_end_pos - return the new read position
937 * @read_pos: file read position
938 * @buf: relay channel buffer
939 * @count: number of bytes to be read
940 */
941static size_t relay_file_read_end_pos(struct rchan_buf *buf,
942 size_t read_pos,
943 size_t count)
944{
945 size_t read_subbuf, padding, end_pos;
946 size_t subbuf_size = buf->chan->subbuf_size;
947 size_t n_subbufs = buf->chan->n_subbufs;
948
949 read_subbuf = read_pos / subbuf_size;
950 padding = buf->padding[read_subbuf];
951 if (read_pos % subbuf_size + count + padding == subbuf_size)
952 end_pos = (read_subbuf + 1) * subbuf_size;
953 else
954 end_pos = read_pos + count;
955 if (end_pos >= subbuf_size * n_subbufs)
956 end_pos = 0;
957
958 return end_pos;
959}
960
961static ssize_t relay_file_read(struct file *filp,
962 char __user *buffer,
963 size_t count,
964 loff_t *ppos)
965{
966 struct rchan_buf *buf = filp->private_data;
967 size_t read_start, avail;
968 size_t written = 0;
969 int ret;
970
971 if (!count)
972 return 0;
973
974 inode_lock(file_inode(filp));
975 do {
976 void *from;
977
978 if (!relay_file_read_avail(buf))
979 break;
980
981 read_start = relay_file_read_start_pos(buf);
982 avail = relay_file_read_subbuf_avail(read_start, buf);
983 if (!avail)
984 break;
985
986 avail = min(count, avail);
987 from = buf->start + read_start;
988 ret = avail;
989 if (copy_to_user(buffer, from, avail))
990 break;
991
992 buffer += ret;
993 written += ret;
994 count -= ret;
995
996 relay_file_read_consume(buf, read_start, ret);
997 *ppos = relay_file_read_end_pos(buf, read_start, ret);
998 } while (count);
999 inode_unlock(file_inode(filp));
1000
1001 return written;
1002}
1003
1004
1005const struct file_operations relay_file_operations = {
1006 .open = relay_file_open,
1007 .poll = relay_file_poll,
1008 .mmap_prepare = relay_file_mmap_prepare,
1009 .read = relay_file_read,
1010 .release = relay_file_release,
1011};
1012EXPORT_SYMBOL_GPL(relay_file_operations);