2 * Public API and common code for kernel->userspace relay file support.
4 * See Documentation/filesystems/relay.txt for an overview.
6 * Copyright (C) 2002-2005 - Tom Zanussi (zanussi@us.ibm.com), IBM Corp
7 * Copyright (C) 1999-2005 - Karim Yaghmour (karim@opersys.com)
9 * Moved to kernel/relay.c by Paul Mundt, 2006.
10 * November 2006 - CPU hotplug support by Mathieu Desnoyers
11 * (mathieu.desnoyers@polymtl.ca)
13 * This file is released under the GPL.
15 #include <linux/errno.h>
16 #include <linux/stddef.h>
17 #include <linux/slab.h>
18 #include <linux/export.h>
19 #include <linux/string.h>
20 #include <linux/relay.h>
21 #include <linux/vmalloc.h>
23 #include <linux/cpu.h>
24 #include <linux/splice.h>
26 /* list of open channels, for cpu hotplug */
27 static DEFINE_MUTEX(relay_channels_mutex
);
28 static LIST_HEAD(relay_channels
);
31 * close() vm_op implementation for relay file mapping.
33 static void relay_file_mmap_close(struct vm_area_struct
*vma
)
35 struct rchan_buf
*buf
= vma
->vm_private_data
;
36 buf
->chan
->cb
->buf_unmapped(buf
, vma
->vm_file
);
40 * fault() vm_op implementation for relay file mapping.
42 static int relay_buf_fault(struct vm_area_struct
*vma
, struct vm_fault
*vmf
)
45 struct rchan_buf
*buf
= vma
->vm_private_data
;
46 pgoff_t pgoff
= vmf
->pgoff
;
51 page
= vmalloc_to_page(buf
->start
+ (pgoff
<< PAGE_SHIFT
));
53 return VM_FAULT_SIGBUS
;
61 * vm_ops for relay file mappings.
63 static const struct vm_operations_struct relay_file_mmap_ops
= {
64 .fault
= relay_buf_fault
,
65 .close
= relay_file_mmap_close
,
69 * allocate an array of pointers of struct page
71 static struct page
**relay_alloc_page_array(unsigned int n_pages
)
73 const size_t pa_size
= n_pages
* sizeof(struct page
*);
74 if (pa_size
> PAGE_SIZE
)
75 return vzalloc(pa_size
);
76 return kzalloc(pa_size
, GFP_KERNEL
);
80 * free an array of pointers of struct page
82 static void relay_free_page_array(struct page
**array
)
88 * relay_mmap_buf: - mmap channel buffer to process address space
89 * @buf: relay channel buffer
90 * @vma: vm_area_struct describing memory to be mapped
92 * Returns 0 if ok, negative on error
94 * Caller should already have grabbed mmap_sem.
96 static int relay_mmap_buf(struct rchan_buf
*buf
, struct vm_area_struct
*vma
)
98 unsigned long length
= vma
->vm_end
- vma
->vm_start
;
99 struct file
*filp
= vma
->vm_file
;
104 if (length
!= (unsigned long)buf
->chan
->alloc_size
)
107 vma
->vm_ops
= &relay_file_mmap_ops
;
108 vma
->vm_flags
|= VM_DONTEXPAND
;
109 vma
->vm_private_data
= buf
;
110 buf
->chan
->cb
->buf_mapped(buf
, filp
);
116 * relay_alloc_buf - allocate a channel buffer
117 * @buf: the buffer struct
118 * @size: total size of the buffer
120 * Returns a pointer to the resulting buffer, %NULL if unsuccessful. The
121 * passed in size will get page aligned, if it isn't already.
123 static void *relay_alloc_buf(struct rchan_buf
*buf
, size_t *size
)
126 unsigned int i
, j
, n_pages
;
128 *size
= PAGE_ALIGN(*size
);
129 n_pages
= *size
>> PAGE_SHIFT
;
131 buf
->page_array
= relay_alloc_page_array(n_pages
);
132 if (!buf
->page_array
)
135 for (i
= 0; i
< n_pages
; i
++) {
136 buf
->page_array
[i
] = alloc_page(GFP_KERNEL
);
137 if (unlikely(!buf
->page_array
[i
]))
139 set_page_private(buf
->page_array
[i
], (unsigned long)buf
);
141 mem
= vmap(buf
->page_array
, n_pages
, VM_MAP
, PAGE_KERNEL
);
145 memset(mem
, 0, *size
);
146 buf
->page_count
= n_pages
;
150 for (j
= 0; j
< i
; j
++)
151 __free_page(buf
->page_array
[j
]);
152 relay_free_page_array(buf
->page_array
);
157 * relay_create_buf - allocate and initialize a channel buffer
158 * @chan: the relay channel
160 * Returns channel buffer if successful, %NULL otherwise.
162 static struct rchan_buf
*relay_create_buf(struct rchan
*chan
)
164 struct rchan_buf
*buf
;
166 if (chan
->n_subbufs
> UINT_MAX
/ sizeof(size_t *))
169 buf
= kzalloc(sizeof(struct rchan_buf
), GFP_KERNEL
);
172 buf
->padding
= kmalloc(chan
->n_subbufs
* sizeof(size_t *), GFP_KERNEL
);
176 buf
->start
= relay_alloc_buf(buf
, &chan
->alloc_size
);
181 kref_get(&buf
->chan
->kref
);
191 * relay_destroy_channel - free the channel struct
192 * @kref: target kernel reference that contains the relay channel
194 * Should only be called from kref_put().
196 static void relay_destroy_channel(struct kref
*kref
)
198 struct rchan
*chan
= container_of(kref
, struct rchan
, kref
);
203 * relay_destroy_buf - destroy an rchan_buf struct and associated buffer
204 * @buf: the buffer struct
206 static void relay_destroy_buf(struct rchan_buf
*buf
)
208 struct rchan
*chan
= buf
->chan
;
211 if (likely(buf
->start
)) {
213 for (i
= 0; i
< buf
->page_count
; i
++)
214 __free_page(buf
->page_array
[i
]);
215 relay_free_page_array(buf
->page_array
);
217 chan
->buf
[buf
->cpu
] = NULL
;
220 kref_put(&chan
->kref
, relay_destroy_channel
);
224 * relay_remove_buf - remove a channel buffer
225 * @kref: target kernel reference that contains the relay buffer
227 * Removes the file from the filesystem, which also frees the
228 * rchan_buf_struct and the channel buffer. Should only be called from
231 static void relay_remove_buf(struct kref
*kref
)
233 struct rchan_buf
*buf
= container_of(kref
, struct rchan_buf
, kref
);
234 relay_destroy_buf(buf
);
238 * relay_buf_empty - boolean, is the channel buffer empty?
239 * @buf: channel buffer
241 * Returns 1 if the buffer is empty, 0 otherwise.
243 static int relay_buf_empty(struct rchan_buf
*buf
)
245 return (buf
->subbufs_produced
- buf
->subbufs_consumed
) ? 0 : 1;
249 * relay_buf_full - boolean, is the channel buffer full?
250 * @buf: channel buffer
252 * Returns 1 if the buffer is full, 0 otherwise.
254 int relay_buf_full(struct rchan_buf
*buf
)
256 size_t ready
= buf
->subbufs_produced
- buf
->subbufs_consumed
;
257 return (ready
>= buf
->chan
->n_subbufs
) ? 1 : 0;
259 EXPORT_SYMBOL_GPL(relay_buf_full
);
262 * High-level relay kernel API and associated functions.
266 * rchan_callback implementations defining default channel behavior. Used
267 * in place of corresponding NULL values in client callback struct.
271 * subbuf_start() default callback. Does nothing.
273 static int subbuf_start_default_callback (struct rchan_buf
*buf
,
278 if (relay_buf_full(buf
))
285 * buf_mapped() default callback. Does nothing.
287 static void buf_mapped_default_callback(struct rchan_buf
*buf
,
293 * buf_unmapped() default callback. Does nothing.
295 static void buf_unmapped_default_callback(struct rchan_buf
*buf
,
301 * create_buf_file_create() default callback. Does nothing.
303 static struct dentry
*create_buf_file_default_callback(const char *filename
,
304 struct dentry
*parent
,
306 struct rchan_buf
*buf
,
313 * remove_buf_file() default callback. Does nothing.
315 static int remove_buf_file_default_callback(struct dentry
*dentry
)
320 /* relay channel default callbacks */
321 static struct rchan_callbacks default_channel_callbacks
= {
322 .subbuf_start
= subbuf_start_default_callback
,
323 .buf_mapped
= buf_mapped_default_callback
,
324 .buf_unmapped
= buf_unmapped_default_callback
,
325 .create_buf_file
= create_buf_file_default_callback
,
326 .remove_buf_file
= remove_buf_file_default_callback
,
330 * wakeup_readers - wake up readers waiting on a channel
331 * @work: contains the channel buffer
333 * This is the function used to defer reader waking
335 static void wakeup_readers(struct irq_work
*work
)
337 struct rchan_buf
*buf
;
339 buf
= container_of(work
, struct rchan_buf
, wakeup_work
);
340 wake_up_interruptible(&buf
->read_wait
);
344 * __relay_reset - reset a channel buffer
345 * @buf: the channel buffer
346 * @init: 1 if this is a first-time initialization
348 * See relay_reset() for description of effect.
350 static void __relay_reset(struct rchan_buf
*buf
, unsigned int init
)
355 init_waitqueue_head(&buf
->read_wait
);
356 kref_init(&buf
->kref
);
357 init_irq_work(&buf
->wakeup_work
, wakeup_readers
);
359 irq_work_sync(&buf
->wakeup_work
);
362 buf
->subbufs_produced
= 0;
363 buf
->subbufs_consumed
= 0;
364 buf
->bytes_consumed
= 0;
366 buf
->data
= buf
->start
;
369 for (i
= 0; i
< buf
->chan
->n_subbufs
; i
++)
372 buf
->chan
->cb
->subbuf_start(buf
, buf
->data
, NULL
, 0);
376 * relay_reset - reset the channel
379 * This has the effect of erasing all data from all channel buffers
380 * and restarting the channel in its initial state. The buffers
381 * are not freed, so any mappings are still in effect.
383 * NOTE. Care should be taken that the channel isn't actually
384 * being used by anything when this call is made.
386 void relay_reset(struct rchan
*chan
)
393 if (chan
->is_global
&& chan
->buf
[0]) {
394 __relay_reset(chan
->buf
[0], 0);
398 mutex_lock(&relay_channels_mutex
);
399 for_each_possible_cpu(i
)
401 __relay_reset(chan
->buf
[i
], 0);
402 mutex_unlock(&relay_channels_mutex
);
404 EXPORT_SYMBOL_GPL(relay_reset
);
406 static inline void relay_set_buf_dentry(struct rchan_buf
*buf
,
407 struct dentry
*dentry
)
409 buf
->dentry
= dentry
;
410 d_inode(buf
->dentry
)->i_size
= buf
->early_bytes
;
413 static struct dentry
*relay_create_buf_file(struct rchan
*chan
,
414 struct rchan_buf
*buf
,
417 struct dentry
*dentry
;
420 tmpname
= kzalloc(NAME_MAX
+ 1, GFP_KERNEL
);
423 snprintf(tmpname
, NAME_MAX
, "%s%d", chan
->base_filename
, cpu
);
425 /* Create file in fs */
426 dentry
= chan
->cb
->create_buf_file(tmpname
, chan
->parent
,
436 * relay_open_buf - create a new relay channel buffer
438 * used by relay_open() and CPU hotplug.
440 static struct rchan_buf
*relay_open_buf(struct rchan
*chan
, unsigned int cpu
)
442 struct rchan_buf
*buf
= NULL
;
443 struct dentry
*dentry
;
448 buf
= relay_create_buf(chan
);
452 if (chan
->has_base_filename
) {
453 dentry
= relay_create_buf_file(chan
, buf
, cpu
);
456 relay_set_buf_dentry(buf
, dentry
);
458 /* Only retrieve global info, nothing more, nothing less */
459 dentry
= chan
->cb
->create_buf_file(NULL
, NULL
,
467 __relay_reset(buf
, 1);
469 if(chan
->is_global
) {
477 relay_destroy_buf(buf
);
482 * relay_close_buf - close a channel buffer
483 * @buf: channel buffer
485 * Marks the buffer finalized and restores the default callbacks.
486 * The channel buffer and channel buffer data structure are then freed
487 * automatically when the last reference is given up.
489 static void relay_close_buf(struct rchan_buf
*buf
)
492 irq_work_sync(&buf
->wakeup_work
);
493 buf
->chan
->cb
->remove_buf_file(buf
->dentry
);
494 kref_put(&buf
->kref
, relay_remove_buf
);
497 static void setup_callbacks(struct rchan
*chan
,
498 struct rchan_callbacks
*cb
)
501 chan
->cb
= &default_channel_callbacks
;
505 if (!cb
->subbuf_start
)
506 cb
->subbuf_start
= subbuf_start_default_callback
;
508 cb
->buf_mapped
= buf_mapped_default_callback
;
509 if (!cb
->buf_unmapped
)
510 cb
->buf_unmapped
= buf_unmapped_default_callback
;
511 if (!cb
->create_buf_file
)
512 cb
->create_buf_file
= create_buf_file_default_callback
;
513 if (!cb
->remove_buf_file
)
514 cb
->remove_buf_file
= remove_buf_file_default_callback
;
519 * relay_hotcpu_callback - CPU hotplug callback
520 * @nb: notifier block
521 * @action: hotplug action to take
524 * Returns the success/failure of the operation. (%NOTIFY_OK, %NOTIFY_BAD)
526 static int relay_hotcpu_callback(struct notifier_block
*nb
,
527 unsigned long action
,
530 unsigned int hotcpu
= (unsigned long)hcpu
;
535 case CPU_UP_PREPARE_FROZEN
:
536 mutex_lock(&relay_channels_mutex
);
537 list_for_each_entry(chan
, &relay_channels
, list
) {
538 if (chan
->buf
[hotcpu
])
540 chan
->buf
[hotcpu
] = relay_open_buf(chan
, hotcpu
);
541 if(!chan
->buf
[hotcpu
]) {
543 "relay_hotcpu_callback: cpu %d buffer "
544 "creation failed\n", hotcpu
);
545 mutex_unlock(&relay_channels_mutex
);
546 return notifier_from_errno(-ENOMEM
);
549 mutex_unlock(&relay_channels_mutex
);
552 case CPU_DEAD_FROZEN
:
553 /* No need to flush the cpu : will be flushed upon
554 * final relay_flush() call. */
561 * relay_open - create a new relay channel
562 * @base_filename: base name of files to create, %NULL for buffering only
563 * @parent: dentry of parent directory, %NULL for root directory or buffer
564 * @subbuf_size: size of sub-buffers
565 * @n_subbufs: number of sub-buffers
566 * @cb: client callback functions
567 * @private_data: user-defined data
569 * Returns channel pointer if successful, %NULL otherwise.
571 * Creates a channel buffer for each cpu using the sizes and
572 * attributes specified. The created channel buffer files
573 * will be named base_filename0...base_filenameN-1. File
574 * permissions will be %S_IRUSR.
576 * If opening a buffer (@parent = NULL) that you later wish to register
577 * in a filesystem, call relay_late_setup_files() once the @parent dentry
580 struct rchan
*relay_open(const char *base_filename
,
581 struct dentry
*parent
,
584 struct rchan_callbacks
*cb
,
590 if (!(subbuf_size
&& n_subbufs
))
592 if (subbuf_size
> UINT_MAX
/ n_subbufs
)
595 chan
= kzalloc(sizeof(struct rchan
), GFP_KERNEL
);
599 chan
->version
= RELAYFS_CHANNEL_VERSION
;
600 chan
->n_subbufs
= n_subbufs
;
601 chan
->subbuf_size
= subbuf_size
;
602 chan
->alloc_size
= PAGE_ALIGN(subbuf_size
* n_subbufs
);
603 chan
->parent
= parent
;
604 chan
->private_data
= private_data
;
606 chan
->has_base_filename
= 1;
607 strlcpy(chan
->base_filename
, base_filename
, NAME_MAX
);
609 setup_callbacks(chan
, cb
);
610 kref_init(&chan
->kref
);
612 mutex_lock(&relay_channels_mutex
);
613 for_each_online_cpu(i
) {
614 chan
->buf
[i
] = relay_open_buf(chan
, i
);
618 list_add(&chan
->list
, &relay_channels
);
619 mutex_unlock(&relay_channels_mutex
);
624 for_each_possible_cpu(i
) {
626 relay_close_buf(chan
->buf
[i
]);
629 kref_put(&chan
->kref
, relay_destroy_channel
);
630 mutex_unlock(&relay_channels_mutex
);
634 EXPORT_SYMBOL_GPL(relay_open
);
636 struct rchan_percpu_buf_dispatcher
{
637 struct rchan_buf
*buf
;
638 struct dentry
*dentry
;
641 /* Called in atomic context. */
642 static void __relay_set_buf_dentry(void *info
)
644 struct rchan_percpu_buf_dispatcher
*p
= info
;
646 relay_set_buf_dentry(p
->buf
, p
->dentry
);
650 * relay_late_setup_files - triggers file creation
651 * @chan: channel to operate on
652 * @base_filename: base name of files to create
653 * @parent: dentry of parent directory, %NULL for root directory
655 * Returns 0 if successful, non-zero otherwise.
657 * Use to setup files for a previously buffer-only channel created
658 * by relay_open() with a NULL parent dentry.
660 * For example, this is useful for perfomring early tracing in kernel,
661 * before VFS is up and then exposing the early results once the dentry
664 int relay_late_setup_files(struct rchan
*chan
,
665 const char *base_filename
,
666 struct dentry
*parent
)
669 unsigned int i
, curr_cpu
;
671 struct dentry
*dentry
;
672 struct rchan_percpu_buf_dispatcher disp
;
674 if (!chan
|| !base_filename
)
677 strlcpy(chan
->base_filename
, base_filename
, NAME_MAX
);
679 mutex_lock(&relay_channels_mutex
);
680 /* Is chan already set up? */
681 if (unlikely(chan
->has_base_filename
)) {
682 mutex_unlock(&relay_channels_mutex
);
685 chan
->has_base_filename
= 1;
686 chan
->parent
= parent
;
688 if (chan
->is_global
) {
690 if (!WARN_ON_ONCE(!chan
->buf
[0])) {
691 dentry
= relay_create_buf_file(chan
, chan
->buf
[0], 0);
692 if (dentry
&& !WARN_ON_ONCE(!chan
->is_global
)) {
693 relay_set_buf_dentry(chan
->buf
[0], dentry
);
697 mutex_unlock(&relay_channels_mutex
);
701 curr_cpu
= get_cpu();
703 * The CPU hotplug notifier ran before us and created buffers with
704 * no files associated. So it's safe to call relay_setup_buf_file()
705 * on all currently online CPUs.
707 for_each_online_cpu(i
) {
708 if (unlikely(!chan
->buf
[i
])) {
709 WARN_ONCE(1, KERN_ERR
"CPU has no buffer!\n");
714 dentry
= relay_create_buf_file(chan
, chan
->buf
[i
], i
);
715 if (unlikely(!dentry
)) {
721 local_irq_save(flags
);
722 relay_set_buf_dentry(chan
->buf
[i
], dentry
);
723 local_irq_restore(flags
);
725 disp
.buf
= chan
->buf
[i
];
726 disp
.dentry
= dentry
;
728 /* relay_channels_mutex must be held, so wait. */
729 err
= smp_call_function_single(i
,
730 __relay_set_buf_dentry
,
737 mutex_unlock(&relay_channels_mutex
);
741 EXPORT_SYMBOL_GPL(relay_late_setup_files
);
744 * relay_switch_subbuf - switch to a new sub-buffer
745 * @buf: channel buffer
746 * @length: size of current event
748 * Returns either the length passed in or 0 if full.
750 * Performs sub-buffer-switch tasks such as invoking callbacks,
751 * updating padding counts, waking up readers, etc.
753 size_t relay_switch_subbuf(struct rchan_buf
*buf
, size_t length
)
756 size_t old_subbuf
, new_subbuf
;
758 if (unlikely(length
> buf
->chan
->subbuf_size
))
761 if (buf
->offset
!= buf
->chan
->subbuf_size
+ 1) {
762 buf
->prev_padding
= buf
->chan
->subbuf_size
- buf
->offset
;
763 old_subbuf
= buf
->subbufs_produced
% buf
->chan
->n_subbufs
;
764 buf
->padding
[old_subbuf
] = buf
->prev_padding
;
765 buf
->subbufs_produced
++;
767 d_inode(buf
->dentry
)->i_size
+=
768 buf
->chan
->subbuf_size
-
769 buf
->padding
[old_subbuf
];
771 buf
->early_bytes
+= buf
->chan
->subbuf_size
-
772 buf
->padding
[old_subbuf
];
774 if (waitqueue_active(&buf
->read_wait
)) {
776 * Calling wake_up_interruptible() from here
777 * will deadlock if we happen to be logging
778 * from the scheduler (trying to re-grab
779 * rq->lock), so defer it.
781 irq_work_queue(&buf
->wakeup_work
);
786 new_subbuf
= buf
->subbufs_produced
% buf
->chan
->n_subbufs
;
787 new = buf
->start
+ new_subbuf
* buf
->chan
->subbuf_size
;
789 if (!buf
->chan
->cb
->subbuf_start(buf
, new, old
, buf
->prev_padding
)) {
790 buf
->offset
= buf
->chan
->subbuf_size
+ 1;
794 buf
->padding
[new_subbuf
] = 0;
796 if (unlikely(length
+ buf
->offset
> buf
->chan
->subbuf_size
))
802 buf
->chan
->last_toobig
= length
;
805 EXPORT_SYMBOL_GPL(relay_switch_subbuf
);
808 * relay_subbufs_consumed - update the buffer's sub-buffers-consumed count
810 * @cpu: the cpu associated with the channel buffer to update
811 * @subbufs_consumed: number of sub-buffers to add to current buf's count
813 * Adds to the channel buffer's consumed sub-buffer count.
814 * subbufs_consumed should be the number of sub-buffers newly consumed,
815 * not the total consumed.
817 * NOTE. Kernel clients don't need to call this function if the channel
818 * mode is 'overwrite'.
820 void relay_subbufs_consumed(struct rchan
*chan
,
822 size_t subbufs_consumed
)
824 struct rchan_buf
*buf
;
829 if (cpu
>= NR_CPUS
|| !chan
->buf
[cpu
] ||
830 subbufs_consumed
> chan
->n_subbufs
)
833 buf
= chan
->buf
[cpu
];
834 if (subbufs_consumed
> buf
->subbufs_produced
- buf
->subbufs_consumed
)
835 buf
->subbufs_consumed
= buf
->subbufs_produced
;
837 buf
->subbufs_consumed
+= subbufs_consumed
;
839 EXPORT_SYMBOL_GPL(relay_subbufs_consumed
);
842 * relay_close - close the channel
845 * Closes all channel buffers and frees the channel.
847 void relay_close(struct rchan
*chan
)
854 mutex_lock(&relay_channels_mutex
);
855 if (chan
->is_global
&& chan
->buf
[0])
856 relay_close_buf(chan
->buf
[0]);
858 for_each_possible_cpu(i
)
860 relay_close_buf(chan
->buf
[i
]);
862 if (chan
->last_toobig
)
863 printk(KERN_WARNING
"relay: one or more items not logged "
864 "[item size (%Zd) > sub-buffer size (%Zd)]\n",
865 chan
->last_toobig
, chan
->subbuf_size
);
867 list_del(&chan
->list
);
868 kref_put(&chan
->kref
, relay_destroy_channel
);
869 mutex_unlock(&relay_channels_mutex
);
871 EXPORT_SYMBOL_GPL(relay_close
);
874 * relay_flush - close the channel
877 * Flushes all channel buffers, i.e. forces buffer switch.
879 void relay_flush(struct rchan
*chan
)
886 if (chan
->is_global
&& chan
->buf
[0]) {
887 relay_switch_subbuf(chan
->buf
[0], 0);
891 mutex_lock(&relay_channels_mutex
);
892 for_each_possible_cpu(i
)
894 relay_switch_subbuf(chan
->buf
[i
], 0);
895 mutex_unlock(&relay_channels_mutex
);
897 EXPORT_SYMBOL_GPL(relay_flush
);
900 * relay_file_open - open file op for relay files
904 * Increments the channel buffer refcount.
906 static int relay_file_open(struct inode
*inode
, struct file
*filp
)
908 struct rchan_buf
*buf
= inode
->i_private
;
909 kref_get(&buf
->kref
);
910 filp
->private_data
= buf
;
912 return nonseekable_open(inode
, filp
);
916 * relay_file_mmap - mmap file op for relay files
918 * @vma: the vma describing what to map
920 * Calls upon relay_mmap_buf() to map the file into user space.
922 static int relay_file_mmap(struct file
*filp
, struct vm_area_struct
*vma
)
924 struct rchan_buf
*buf
= filp
->private_data
;
925 return relay_mmap_buf(buf
, vma
);
929 * relay_file_poll - poll file op for relay files
935 static unsigned int relay_file_poll(struct file
*filp
, poll_table
*wait
)
937 unsigned int mask
= 0;
938 struct rchan_buf
*buf
= filp
->private_data
;
943 if (filp
->f_mode
& FMODE_READ
) {
944 poll_wait(filp
, &buf
->read_wait
, wait
);
945 if (!relay_buf_empty(buf
))
946 mask
|= POLLIN
| POLLRDNORM
;
953 * relay_file_release - release file op for relay files
957 * Decrements the channel refcount, as the filesystem is
958 * no longer using it.
960 static int relay_file_release(struct inode
*inode
, struct file
*filp
)
962 struct rchan_buf
*buf
= filp
->private_data
;
963 kref_put(&buf
->kref
, relay_remove_buf
);
969 * relay_file_read_consume - update the consumed count for the buffer
971 static void relay_file_read_consume(struct rchan_buf
*buf
,
973 size_t bytes_consumed
)
975 size_t subbuf_size
= buf
->chan
->subbuf_size
;
976 size_t n_subbufs
= buf
->chan
->n_subbufs
;
979 if (buf
->subbufs_produced
== buf
->subbufs_consumed
&&
980 buf
->offset
== buf
->bytes_consumed
)
983 if (buf
->bytes_consumed
+ bytes_consumed
> subbuf_size
) {
984 relay_subbufs_consumed(buf
->chan
, buf
->cpu
, 1);
985 buf
->bytes_consumed
= 0;
988 buf
->bytes_consumed
+= bytes_consumed
;
990 read_subbuf
= buf
->subbufs_consumed
% n_subbufs
;
992 read_subbuf
= read_pos
/ buf
->chan
->subbuf_size
;
993 if (buf
->bytes_consumed
+ buf
->padding
[read_subbuf
] == subbuf_size
) {
994 if ((read_subbuf
== buf
->subbufs_produced
% n_subbufs
) &&
995 (buf
->offset
== subbuf_size
))
997 relay_subbufs_consumed(buf
->chan
, buf
->cpu
, 1);
998 buf
->bytes_consumed
= 0;
1003 * relay_file_read_avail - boolean, are there unconsumed bytes available?
1005 static int relay_file_read_avail(struct rchan_buf
*buf
, size_t read_pos
)
1007 size_t subbuf_size
= buf
->chan
->subbuf_size
;
1008 size_t n_subbufs
= buf
->chan
->n_subbufs
;
1009 size_t produced
= buf
->subbufs_produced
;
1010 size_t consumed
= buf
->subbufs_consumed
;
1012 relay_file_read_consume(buf
, read_pos
, 0);
1014 consumed
= buf
->subbufs_consumed
;
1016 if (unlikely(buf
->offset
> subbuf_size
)) {
1017 if (produced
== consumed
)
1022 if (unlikely(produced
- consumed
>= n_subbufs
)) {
1023 consumed
= produced
- n_subbufs
+ 1;
1024 buf
->subbufs_consumed
= consumed
;
1025 buf
->bytes_consumed
= 0;
1028 produced
= (produced
% n_subbufs
) * subbuf_size
+ buf
->offset
;
1029 consumed
= (consumed
% n_subbufs
) * subbuf_size
+ buf
->bytes_consumed
;
1031 if (consumed
> produced
)
1032 produced
+= n_subbufs
* subbuf_size
;
1034 if (consumed
== produced
) {
1035 if (buf
->offset
== subbuf_size
&&
1036 buf
->subbufs_produced
> buf
->subbufs_consumed
)
1045 * relay_file_read_subbuf_avail - return bytes available in sub-buffer
1046 * @read_pos: file read position
1047 * @buf: relay channel buffer
1049 static size_t relay_file_read_subbuf_avail(size_t read_pos
,
1050 struct rchan_buf
*buf
)
1052 size_t padding
, avail
= 0;
1053 size_t read_subbuf
, read_offset
, write_subbuf
, write_offset
;
1054 size_t subbuf_size
= buf
->chan
->subbuf_size
;
1056 write_subbuf
= (buf
->data
- buf
->start
) / subbuf_size
;
1057 write_offset
= buf
->offset
> subbuf_size
? subbuf_size
: buf
->offset
;
1058 read_subbuf
= read_pos
/ subbuf_size
;
1059 read_offset
= read_pos
% subbuf_size
;
1060 padding
= buf
->padding
[read_subbuf
];
1062 if (read_subbuf
== write_subbuf
) {
1063 if (read_offset
+ padding
< write_offset
)
1064 avail
= write_offset
- (read_offset
+ padding
);
1066 avail
= (subbuf_size
- padding
) - read_offset
;
1072 * relay_file_read_start_pos - find the first available byte to read
1073 * @read_pos: file read position
1074 * @buf: relay channel buffer
1076 * If the @read_pos is in the middle of padding, return the
1077 * position of the first actually available byte, otherwise
1078 * return the original value.
1080 static size_t relay_file_read_start_pos(size_t read_pos
,
1081 struct rchan_buf
*buf
)
1083 size_t read_subbuf
, padding
, padding_start
, padding_end
;
1084 size_t subbuf_size
= buf
->chan
->subbuf_size
;
1085 size_t n_subbufs
= buf
->chan
->n_subbufs
;
1086 size_t consumed
= buf
->subbufs_consumed
% n_subbufs
;
1089 read_pos
= consumed
* subbuf_size
+ buf
->bytes_consumed
;
1090 read_subbuf
= read_pos
/ subbuf_size
;
1091 padding
= buf
->padding
[read_subbuf
];
1092 padding_start
= (read_subbuf
+ 1) * subbuf_size
- padding
;
1093 padding_end
= (read_subbuf
+ 1) * subbuf_size
;
1094 if (read_pos
>= padding_start
&& read_pos
< padding_end
) {
1095 read_subbuf
= (read_subbuf
+ 1) % n_subbufs
;
1096 read_pos
= read_subbuf
* subbuf_size
;
1103 * relay_file_read_end_pos - return the new read position
1104 * @read_pos: file read position
1105 * @buf: relay channel buffer
1106 * @count: number of bytes to be read
1108 static size_t relay_file_read_end_pos(struct rchan_buf
*buf
,
1112 size_t read_subbuf
, padding
, end_pos
;
1113 size_t subbuf_size
= buf
->chan
->subbuf_size
;
1114 size_t n_subbufs
= buf
->chan
->n_subbufs
;
1116 read_subbuf
= read_pos
/ subbuf_size
;
1117 padding
= buf
->padding
[read_subbuf
];
1118 if (read_pos
% subbuf_size
+ count
+ padding
== subbuf_size
)
1119 end_pos
= (read_subbuf
+ 1) * subbuf_size
;
1121 end_pos
= read_pos
+ count
;
1122 if (end_pos
>= subbuf_size
* n_subbufs
)
1129 * subbuf_read_actor - read up to one subbuf's worth of data
1131 static int subbuf_read_actor(size_t read_start
,
1132 struct rchan_buf
*buf
,
1134 read_descriptor_t
*desc
)
1139 from
= buf
->start
+ read_start
;
1141 if (copy_to_user(desc
->arg
.buf
, from
, avail
)) {
1142 desc
->error
= -EFAULT
;
1145 desc
->arg
.data
+= ret
;
1146 desc
->written
+= ret
;
1152 typedef int (*subbuf_actor_t
) (size_t read_start
,
1153 struct rchan_buf
*buf
,
1155 read_descriptor_t
*desc
);
1158 * relay_file_read_subbufs - read count bytes, bridging subbuf boundaries
1160 static ssize_t
relay_file_read_subbufs(struct file
*filp
, loff_t
*ppos
,
1161 subbuf_actor_t subbuf_actor
,
1162 read_descriptor_t
*desc
)
1164 struct rchan_buf
*buf
= filp
->private_data
;
1165 size_t read_start
, avail
;
1171 inode_lock(file_inode(filp
));
1173 if (!relay_file_read_avail(buf
, *ppos
))
1176 read_start
= relay_file_read_start_pos(*ppos
, buf
);
1177 avail
= relay_file_read_subbuf_avail(read_start
, buf
);
1181 avail
= min(desc
->count
, avail
);
1182 ret
= subbuf_actor(read_start
, buf
, avail
, desc
);
1183 if (desc
->error
< 0)
1187 relay_file_read_consume(buf
, read_start
, ret
);
1188 *ppos
= relay_file_read_end_pos(buf
, read_start
, ret
);
1190 } while (desc
->count
&& ret
);
1191 inode_unlock(file_inode(filp
));
1193 return desc
->written
;
1196 static ssize_t
relay_file_read(struct file
*filp
,
1197 char __user
*buffer
,
1201 read_descriptor_t desc
;
1204 desc
.arg
.buf
= buffer
;
1206 return relay_file_read_subbufs(filp
, ppos
, subbuf_read_actor
, &desc
);
1209 static void relay_consume_bytes(struct rchan_buf
*rbuf
, int bytes_consumed
)
1211 rbuf
->bytes_consumed
+= bytes_consumed
;
1213 if (rbuf
->bytes_consumed
>= rbuf
->chan
->subbuf_size
) {
1214 relay_subbufs_consumed(rbuf
->chan
, rbuf
->cpu
, 1);
1215 rbuf
->bytes_consumed
%= rbuf
->chan
->subbuf_size
;
1219 static void relay_pipe_buf_release(struct pipe_inode_info
*pipe
,
1220 struct pipe_buffer
*buf
)
1222 struct rchan_buf
*rbuf
;
1224 rbuf
= (struct rchan_buf
*)page_private(buf
->page
);
1225 relay_consume_bytes(rbuf
, buf
->private);
1228 static const struct pipe_buf_operations relay_pipe_buf_ops
= {
1230 .confirm
= generic_pipe_buf_confirm
,
1231 .release
= relay_pipe_buf_release
,
1232 .steal
= generic_pipe_buf_steal
,
1233 .get
= generic_pipe_buf_get
,
1236 static void relay_page_release(struct splice_pipe_desc
*spd
, unsigned int i
)
1241 * subbuf_splice_actor - splice up to one subbuf's worth of data
1243 static ssize_t
subbuf_splice_actor(struct file
*in
,
1245 struct pipe_inode_info
*pipe
,
1250 unsigned int pidx
, poff
, total_len
, subbuf_pages
, nr_pages
;
1251 struct rchan_buf
*rbuf
= in
->private_data
;
1252 unsigned int subbuf_size
= rbuf
->chan
->subbuf_size
;
1253 uint64_t pos
= (uint64_t) *ppos
;
1254 uint32_t alloc_size
= (uint32_t) rbuf
->chan
->alloc_size
;
1255 size_t read_start
= (size_t) do_div(pos
, alloc_size
);
1256 size_t read_subbuf
= read_start
/ subbuf_size
;
1257 size_t padding
= rbuf
->padding
[read_subbuf
];
1258 size_t nonpad_end
= read_subbuf
* subbuf_size
+ subbuf_size
- padding
;
1259 struct page
*pages
[PIPE_DEF_BUFFERS
];
1260 struct partial_page partial
[PIPE_DEF_BUFFERS
];
1261 struct splice_pipe_desc spd
= {
1264 .nr_pages_max
= PIPE_DEF_BUFFERS
,
1267 .ops
= &relay_pipe_buf_ops
,
1268 .spd_release
= relay_page_release
,
1272 if (rbuf
->subbufs_produced
== rbuf
->subbufs_consumed
)
1274 if (splice_grow_spd(pipe
, &spd
))
1278 * Adjust read len, if longer than what is available
1280 if (len
> (subbuf_size
- read_start
% subbuf_size
))
1281 len
= subbuf_size
- read_start
% subbuf_size
;
1283 subbuf_pages
= rbuf
->chan
->alloc_size
>> PAGE_SHIFT
;
1284 pidx
= (read_start
/ PAGE_SIZE
) % subbuf_pages
;
1285 poff
= read_start
& ~PAGE_MASK
;
1286 nr_pages
= min_t(unsigned int, subbuf_pages
, spd
.nr_pages_max
);
1288 for (total_len
= 0; spd
.nr_pages
< nr_pages
; spd
.nr_pages
++) {
1289 unsigned int this_len
, this_end
, private;
1290 unsigned int cur_pos
= read_start
+ total_len
;
1295 this_len
= min_t(unsigned long, len
, PAGE_SIZE
- poff
);
1298 spd
.pages
[spd
.nr_pages
] = rbuf
->page_array
[pidx
];
1299 spd
.partial
[spd
.nr_pages
].offset
= poff
;
1301 this_end
= cur_pos
+ this_len
;
1302 if (this_end
>= nonpad_end
) {
1303 this_len
= nonpad_end
- cur_pos
;
1304 private = this_len
+ padding
;
1306 spd
.partial
[spd
.nr_pages
].len
= this_len
;
1307 spd
.partial
[spd
.nr_pages
].private = private;
1310 total_len
+= this_len
;
1312 pidx
= (pidx
+ 1) % subbuf_pages
;
1314 if (this_end
>= nonpad_end
) {
1324 ret
= *nonpad_ret
= splice_to_pipe(pipe
, &spd
);
1325 if (ret
< 0 || ret
< total_len
)
1328 if (read_start
+ ret
== nonpad_end
)
1332 splice_shrink_spd(&spd
);
1336 static ssize_t
relay_file_splice_read(struct file
*in
,
1338 struct pipe_inode_info
*pipe
,
1349 while (len
&& !spliced
) {
1350 ret
= subbuf_splice_actor(in
, ppos
, pipe
, len
, flags
, &nonpad_ret
);
1354 if (flags
& SPLICE_F_NONBLOCK
)
1364 spliced
+= nonpad_ret
;
1374 const struct file_operations relay_file_operations
= {
1375 .open
= relay_file_open
,
1376 .poll
= relay_file_poll
,
1377 .mmap
= relay_file_mmap
,
1378 .read
= relay_file_read
,
1379 .llseek
= no_llseek
,
1380 .release
= relay_file_release
,
1381 .splice_read
= relay_file_splice_read
,
1383 EXPORT_SYMBOL_GPL(relay_file_operations
);
1385 static __init
int relay_init(void)
1388 hotcpu_notifier(relay_hotcpu_callback
, 0);
1392 early_initcall(relay_init
);