2 * Public API and common code for kernel->userspace relay file support.
4 * See Documentation/filesystems/relay.txt for an overview.
6 * Copyright (C) 2002-2005 - Tom Zanussi (zanussi@us.ibm.com), IBM Corp
7 * Copyright (C) 1999-2005 - Karim Yaghmour (karim@opersys.com)
9 * Moved to kernel/relay.c by Paul Mundt, 2006.
10 * November 2006 - CPU hotplug support by Mathieu Desnoyers
11 * (mathieu.desnoyers@polymtl.ca)
13 * This file is released under the GPL.
15 #include <linux/errno.h>
16 #include <linux/stddef.h>
17 #include <linux/slab.h>
18 #include <linux/export.h>
19 #include <linux/string.h>
20 #include <linux/relay.h>
21 #include <linux/vmalloc.h>
23 #include <linux/cpu.h>
24 #include <linux/splice.h>
26 /* list of open channels, for cpu hotplug */
27 static DEFINE_MUTEX(relay_channels_mutex
);
28 static LIST_HEAD(relay_channels
);
31 * close() vm_op implementation for relay file mapping.
33 static void relay_file_mmap_close(struct vm_area_struct
*vma
)
35 struct rchan_buf
*buf
= vma
->vm_private_data
;
36 buf
->chan
->cb
->buf_unmapped(buf
, vma
->vm_file
);
40 * fault() vm_op implementation for relay file mapping.
42 static int relay_buf_fault(struct vm_area_struct
*vma
, struct vm_fault
*vmf
)
45 struct rchan_buf
*buf
= vma
->vm_private_data
;
46 pgoff_t pgoff
= vmf
->pgoff
;
51 page
= vmalloc_to_page(buf
->start
+ (pgoff
<< PAGE_SHIFT
));
53 return VM_FAULT_SIGBUS
;
61 * vm_ops for relay file mappings.
63 static const struct vm_operations_struct relay_file_mmap_ops
= {
64 .fault
= relay_buf_fault
,
65 .close
= relay_file_mmap_close
,
69 * allocate an array of pointers of struct page
71 static struct page
**relay_alloc_page_array(unsigned int n_pages
)
73 const size_t pa_size
= n_pages
* sizeof(struct page
*);
74 if (pa_size
> PAGE_SIZE
)
75 return vzalloc(pa_size
);
76 return kzalloc(pa_size
, GFP_KERNEL
);
80 * free an array of pointers of struct page
82 static void relay_free_page_array(struct page
**array
)
88 * relay_mmap_buf: - mmap channel buffer to process address space
89 * @buf: relay channel buffer
90 * @vma: vm_area_struct describing memory to be mapped
92 * Returns 0 if ok, negative on error
94 * Caller should already have grabbed mmap_sem.
96 static int relay_mmap_buf(struct rchan_buf
*buf
, struct vm_area_struct
*vma
)
98 unsigned long length
= vma
->vm_end
- vma
->vm_start
;
99 struct file
*filp
= vma
->vm_file
;
104 if (length
!= (unsigned long)buf
->chan
->alloc_size
)
107 vma
->vm_ops
= &relay_file_mmap_ops
;
108 vma
->vm_flags
|= VM_DONTEXPAND
;
109 vma
->vm_private_data
= buf
;
110 buf
->chan
->cb
->buf_mapped(buf
, filp
);
116 * relay_alloc_buf - allocate a channel buffer
117 * @buf: the buffer struct
118 * @size: total size of the buffer
120 * Returns a pointer to the resulting buffer, %NULL if unsuccessful. The
121 * passed in size will get page aligned, if it isn't already.
123 static void *relay_alloc_buf(struct rchan_buf
*buf
, size_t *size
)
126 unsigned int i
, j
, n_pages
;
128 *size
= PAGE_ALIGN(*size
);
129 n_pages
= *size
>> PAGE_SHIFT
;
131 buf
->page_array
= relay_alloc_page_array(n_pages
);
132 if (!buf
->page_array
)
135 for (i
= 0; i
< n_pages
; i
++) {
136 buf
->page_array
[i
] = alloc_page(GFP_KERNEL
);
137 if (unlikely(!buf
->page_array
[i
]))
139 set_page_private(buf
->page_array
[i
], (unsigned long)buf
);
141 mem
= vmap(buf
->page_array
, n_pages
, VM_MAP
, PAGE_KERNEL
);
145 memset(mem
, 0, *size
);
146 buf
->page_count
= n_pages
;
150 for (j
= 0; j
< i
; j
++)
151 __free_page(buf
->page_array
[j
]);
152 relay_free_page_array(buf
->page_array
);
157 * relay_create_buf - allocate and initialize a channel buffer
158 * @chan: the relay channel
160 * Returns channel buffer if successful, %NULL otherwise.
162 static struct rchan_buf
*relay_create_buf(struct rchan
*chan
)
164 struct rchan_buf
*buf
;
166 if (chan
->n_subbufs
> UINT_MAX
/ sizeof(size_t *))
169 buf
= kzalloc(sizeof(struct rchan_buf
), GFP_KERNEL
);
172 buf
->padding
= kmalloc(chan
->n_subbufs
* sizeof(size_t *), GFP_KERNEL
);
176 buf
->start
= relay_alloc_buf(buf
, &chan
->alloc_size
);
181 kref_get(&buf
->chan
->kref
);
191 * relay_destroy_channel - free the channel struct
192 * @kref: target kernel reference that contains the relay channel
194 * Should only be called from kref_put().
196 static void relay_destroy_channel(struct kref
*kref
)
198 struct rchan
*chan
= container_of(kref
, struct rchan
, kref
);
203 * relay_destroy_buf - destroy an rchan_buf struct and associated buffer
204 * @buf: the buffer struct
206 static void relay_destroy_buf(struct rchan_buf
*buf
)
208 struct rchan
*chan
= buf
->chan
;
211 if (likely(buf
->start
)) {
213 for (i
= 0; i
< buf
->page_count
; i
++)
214 __free_page(buf
->page_array
[i
]);
215 relay_free_page_array(buf
->page_array
);
217 chan
->buf
[buf
->cpu
] = NULL
;
220 kref_put(&chan
->kref
, relay_destroy_channel
);
224 * relay_remove_buf - remove a channel buffer
225 * @kref: target kernel reference that contains the relay buffer
227 * Removes the file from the filesystem, which also frees the
228 * rchan_buf_struct and the channel buffer. Should only be called from
231 static void relay_remove_buf(struct kref
*kref
)
233 struct rchan_buf
*buf
= container_of(kref
, struct rchan_buf
, kref
);
234 relay_destroy_buf(buf
);
238 * relay_buf_empty - boolean, is the channel buffer empty?
239 * @buf: channel buffer
241 * Returns 1 if the buffer is empty, 0 otherwise.
243 static int relay_buf_empty(struct rchan_buf
*buf
)
245 return (buf
->subbufs_produced
- buf
->subbufs_consumed
) ? 0 : 1;
249 * relay_buf_full - boolean, is the channel buffer full?
250 * @buf: channel buffer
252 * Returns 1 if the buffer is full, 0 otherwise.
254 int relay_buf_full(struct rchan_buf
*buf
)
256 size_t ready
= buf
->subbufs_produced
- buf
->subbufs_consumed
;
257 return (ready
>= buf
->chan
->n_subbufs
) ? 1 : 0;
259 EXPORT_SYMBOL_GPL(relay_buf_full
);
262 * High-level relay kernel API and associated functions.
266 * rchan_callback implementations defining default channel behavior. Used
267 * in place of corresponding NULL values in client callback struct.
271 * subbuf_start() default callback. Does nothing.
273 static int subbuf_start_default_callback (struct rchan_buf
*buf
,
278 if (relay_buf_full(buf
))
285 * buf_mapped() default callback. Does nothing.
287 static void buf_mapped_default_callback(struct rchan_buf
*buf
,
293 * buf_unmapped() default callback. Does nothing.
295 static void buf_unmapped_default_callback(struct rchan_buf
*buf
,
301 * create_buf_file_create() default callback. Does nothing.
303 static struct dentry
*create_buf_file_default_callback(const char *filename
,
304 struct dentry
*parent
,
306 struct rchan_buf
*buf
,
313 * remove_buf_file() default callback. Does nothing.
315 static int remove_buf_file_default_callback(struct dentry
*dentry
)
320 /* relay channel default callbacks */
321 static struct rchan_callbacks default_channel_callbacks
= {
322 .subbuf_start
= subbuf_start_default_callback
,
323 .buf_mapped
= buf_mapped_default_callback
,
324 .buf_unmapped
= buf_unmapped_default_callback
,
325 .create_buf_file
= create_buf_file_default_callback
,
326 .remove_buf_file
= remove_buf_file_default_callback
,
330 * wakeup_readers - wake up readers waiting on a channel
331 * @work: contains the channel buffer
333 * This is the function used to defer reader waking
335 static void wakeup_readers(struct irq_work
*work
)
337 struct rchan_buf
*buf
= container_of(work
, struct rchan_buf
, wakeup_work
);
338 wake_up_interruptible(&buf
->read_wait
);
342 * __relay_reset - reset a channel buffer
343 * @buf: the channel buffer
344 * @init: 1 if this is a first-time initialization
346 * See relay_reset() for description of effect.
348 static void __relay_reset(struct rchan_buf
*buf
, unsigned int init
)
353 init_waitqueue_head(&buf
->read_wait
);
354 kref_init(&buf
->kref
);
355 init_irq_work(&buf
->wakeup_work
, wakeup_readers
);
357 irq_work_sync(&buf
->wakeup_work
);
360 buf
->subbufs_produced
= 0;
361 buf
->subbufs_consumed
= 0;
362 buf
->bytes_consumed
= 0;
364 buf
->data
= buf
->start
;
367 for (i
= 0; i
< buf
->chan
->n_subbufs
; i
++)
370 buf
->chan
->cb
->subbuf_start(buf
, buf
->data
, NULL
, 0);
374 * relay_reset - reset the channel
377 * This has the effect of erasing all data from all channel buffers
378 * and restarting the channel in its initial state. The buffers
379 * are not freed, so any mappings are still in effect.
381 * NOTE. Care should be taken that the channel isn't actually
382 * being used by anything when this call is made.
384 void relay_reset(struct rchan
*chan
)
391 if (chan
->is_global
&& chan
->buf
[0]) {
392 __relay_reset(chan
->buf
[0], 0);
396 mutex_lock(&relay_channels_mutex
);
397 for_each_possible_cpu(i
)
399 __relay_reset(chan
->buf
[i
], 0);
400 mutex_unlock(&relay_channels_mutex
);
402 EXPORT_SYMBOL_GPL(relay_reset
);
404 static inline void relay_set_buf_dentry(struct rchan_buf
*buf
,
405 struct dentry
*dentry
)
407 buf
->dentry
= dentry
;
408 d_inode(buf
->dentry
)->i_size
= buf
->early_bytes
;
411 static struct dentry
*relay_create_buf_file(struct rchan
*chan
,
412 struct rchan_buf
*buf
,
415 struct dentry
*dentry
;
418 tmpname
= kzalloc(NAME_MAX
+ 1, GFP_KERNEL
);
421 snprintf(tmpname
, NAME_MAX
, "%s%d", chan
->base_filename
, cpu
);
423 /* Create file in fs */
424 dentry
= chan
->cb
->create_buf_file(tmpname
, chan
->parent
,
434 * relay_open_buf - create a new relay channel buffer
436 * used by relay_open() and CPU hotplug.
438 static struct rchan_buf
*relay_open_buf(struct rchan
*chan
, unsigned int cpu
)
440 struct rchan_buf
*buf
= NULL
;
441 struct dentry
*dentry
;
446 buf
= relay_create_buf(chan
);
450 if (chan
->has_base_filename
) {
451 dentry
= relay_create_buf_file(chan
, buf
, cpu
);
454 relay_set_buf_dentry(buf
, dentry
);
456 /* Only retrieve global info, nothing more, nothing less */
457 dentry
= chan
->cb
->create_buf_file(NULL
, NULL
,
465 __relay_reset(buf
, 1);
467 if(chan
->is_global
) {
475 relay_destroy_buf(buf
);
480 * relay_close_buf - close a channel buffer
481 * @buf: channel buffer
483 * Marks the buffer finalized and restores the default callbacks.
484 * The channel buffer and channel buffer data structure are then freed
485 * automatically when the last reference is given up.
487 static void relay_close_buf(struct rchan_buf
*buf
)
490 irq_work_sync(&buf
->wakeup_work
);
491 buf
->chan
->cb
->remove_buf_file(buf
->dentry
);
492 kref_put(&buf
->kref
, relay_remove_buf
);
495 static void setup_callbacks(struct rchan
*chan
,
496 struct rchan_callbacks
*cb
)
499 chan
->cb
= &default_channel_callbacks
;
503 if (!cb
->subbuf_start
)
504 cb
->subbuf_start
= subbuf_start_default_callback
;
506 cb
->buf_mapped
= buf_mapped_default_callback
;
507 if (!cb
->buf_unmapped
)
508 cb
->buf_unmapped
= buf_unmapped_default_callback
;
509 if (!cb
->create_buf_file
)
510 cb
->create_buf_file
= create_buf_file_default_callback
;
511 if (!cb
->remove_buf_file
)
512 cb
->remove_buf_file
= remove_buf_file_default_callback
;
517 * relay_hotcpu_callback - CPU hotplug callback
518 * @nb: notifier block
519 * @action: hotplug action to take
522 * Returns the success/failure of the operation. (%NOTIFY_OK, %NOTIFY_BAD)
524 static int relay_hotcpu_callback(struct notifier_block
*nb
,
525 unsigned long action
,
528 unsigned int hotcpu
= (unsigned long)hcpu
;
533 case CPU_UP_PREPARE_FROZEN
:
534 mutex_lock(&relay_channels_mutex
);
535 list_for_each_entry(chan
, &relay_channels
, list
) {
536 if (chan
->buf
[hotcpu
])
538 chan
->buf
[hotcpu
] = relay_open_buf(chan
, hotcpu
);
539 if(!chan
->buf
[hotcpu
]) {
541 "relay_hotcpu_callback: cpu %d buffer "
542 "creation failed\n", hotcpu
);
543 mutex_unlock(&relay_channels_mutex
);
544 return notifier_from_errno(-ENOMEM
);
547 mutex_unlock(&relay_channels_mutex
);
550 case CPU_DEAD_FROZEN
:
551 /* No need to flush the cpu : will be flushed upon
552 * final relay_flush() call. */
559 * relay_open - create a new relay channel
560 * @base_filename: base name of files to create, %NULL for buffering only
561 * @parent: dentry of parent directory, %NULL for root directory or buffer
562 * @subbuf_size: size of sub-buffers
563 * @n_subbufs: number of sub-buffers
564 * @cb: client callback functions
565 * @private_data: user-defined data
567 * Returns channel pointer if successful, %NULL otherwise.
569 * Creates a channel buffer for each cpu using the sizes and
570 * attributes specified. The created channel buffer files
571 * will be named base_filename0...base_filenameN-1. File
572 * permissions will be %S_IRUSR.
574 * If opening a buffer (@parent = NULL) that you later wish to register
575 * in a filesystem, call relay_late_setup_files() once the @parent dentry
578 struct rchan
*relay_open(const char *base_filename
,
579 struct dentry
*parent
,
582 struct rchan_callbacks
*cb
,
588 if (!(subbuf_size
&& n_subbufs
))
590 if (subbuf_size
> UINT_MAX
/ n_subbufs
)
593 chan
= kzalloc(sizeof(struct rchan
), GFP_KERNEL
);
597 chan
->version
= RELAYFS_CHANNEL_VERSION
;
598 chan
->n_subbufs
= n_subbufs
;
599 chan
->subbuf_size
= subbuf_size
;
600 chan
->alloc_size
= PAGE_ALIGN(subbuf_size
* n_subbufs
);
601 chan
->parent
= parent
;
602 chan
->private_data
= private_data
;
604 chan
->has_base_filename
= 1;
605 strlcpy(chan
->base_filename
, base_filename
, NAME_MAX
);
607 setup_callbacks(chan
, cb
);
608 kref_init(&chan
->kref
);
610 mutex_lock(&relay_channels_mutex
);
611 for_each_online_cpu(i
) {
612 chan
->buf
[i
] = relay_open_buf(chan
, i
);
616 list_add(&chan
->list
, &relay_channels
);
617 mutex_unlock(&relay_channels_mutex
);
622 for_each_possible_cpu(i
) {
624 relay_close_buf(chan
->buf
[i
]);
627 kref_put(&chan
->kref
, relay_destroy_channel
);
628 mutex_unlock(&relay_channels_mutex
);
632 EXPORT_SYMBOL_GPL(relay_open
);
634 struct rchan_percpu_buf_dispatcher
{
635 struct rchan_buf
*buf
;
636 struct dentry
*dentry
;
639 /* Called in atomic context. */
640 static void __relay_set_buf_dentry(void *info
)
642 struct rchan_percpu_buf_dispatcher
*p
= info
;
644 relay_set_buf_dentry(p
->buf
, p
->dentry
);
648 * relay_late_setup_files - triggers file creation
649 * @chan: channel to operate on
650 * @base_filename: base name of files to create
651 * @parent: dentry of parent directory, %NULL for root directory
653 * Returns 0 if successful, non-zero otherwise.
655 * Use to setup files for a previously buffer-only channel created
656 * by relay_open() with a NULL parent dentry.
658 * For example, this is useful for perfomring early tracing in kernel,
659 * before VFS is up and then exposing the early results once the dentry
662 int relay_late_setup_files(struct rchan
*chan
,
663 const char *base_filename
,
664 struct dentry
*parent
)
667 unsigned int i
, curr_cpu
;
669 struct dentry
*dentry
;
670 struct rchan_percpu_buf_dispatcher disp
;
672 if (!chan
|| !base_filename
)
675 strlcpy(chan
->base_filename
, base_filename
, NAME_MAX
);
677 mutex_lock(&relay_channels_mutex
);
678 /* Is chan already set up? */
679 if (unlikely(chan
->has_base_filename
)) {
680 mutex_unlock(&relay_channels_mutex
);
683 chan
->has_base_filename
= 1;
684 chan
->parent
= parent
;
686 if (chan
->is_global
) {
688 if (!WARN_ON_ONCE(!chan
->buf
[0])) {
689 dentry
= relay_create_buf_file(chan
, chan
->buf
[0], 0);
690 if (dentry
&& !WARN_ON_ONCE(!chan
->is_global
)) {
691 relay_set_buf_dentry(chan
->buf
[0], dentry
);
695 mutex_unlock(&relay_channels_mutex
);
699 curr_cpu
= get_cpu();
701 * The CPU hotplug notifier ran before us and created buffers with
702 * no files associated. So it's safe to call relay_setup_buf_file()
703 * on all currently online CPUs.
705 for_each_online_cpu(i
) {
706 if (unlikely(!chan
->buf
[i
])) {
707 WARN_ONCE(1, KERN_ERR
"CPU has no buffer!\n");
712 dentry
= relay_create_buf_file(chan
, chan
->buf
[i
], i
);
713 if (unlikely(!dentry
)) {
719 local_irq_save(flags
);
720 relay_set_buf_dentry(chan
->buf
[i
], dentry
);
721 local_irq_restore(flags
);
723 disp
.buf
= chan
->buf
[i
];
724 disp
.dentry
= dentry
;
726 /* relay_channels_mutex must be held, so wait. */
727 err
= smp_call_function_single(i
,
728 __relay_set_buf_dentry
,
735 mutex_unlock(&relay_channels_mutex
);
739 EXPORT_SYMBOL_GPL(relay_late_setup_files
);
742 * relay_switch_subbuf - switch to a new sub-buffer
743 * @buf: channel buffer
744 * @length: size of current event
746 * Returns either the length passed in or 0 if full.
748 * Performs sub-buffer-switch tasks such as invoking callbacks,
749 * updating padding counts, waking up readers, etc.
751 size_t relay_switch_subbuf(struct rchan_buf
*buf
, size_t length
)
754 size_t old_subbuf
, new_subbuf
;
756 if (unlikely(length
> buf
->chan
->subbuf_size
))
759 if (buf
->offset
!= buf
->chan
->subbuf_size
+ 1) {
760 buf
->prev_padding
= buf
->chan
->subbuf_size
- buf
->offset
;
761 old_subbuf
= buf
->subbufs_produced
% buf
->chan
->n_subbufs
;
762 buf
->padding
[old_subbuf
] = buf
->prev_padding
;
763 buf
->subbufs_produced
++;
765 d_inode(buf
->dentry
)->i_size
+=
766 buf
->chan
->subbuf_size
-
767 buf
->padding
[old_subbuf
];
769 buf
->early_bytes
+= buf
->chan
->subbuf_size
-
770 buf
->padding
[old_subbuf
];
772 if (waitqueue_active(&buf
->read_wait
)) {
774 * Calling wake_up_interruptible() from here
775 * will deadlock if we happen to be logging
776 * from the scheduler (trying to re-grab
777 * rq->lock), so defer it.
779 irq_work_queue(&buf
->wakeup_work
);
784 new_subbuf
= buf
->subbufs_produced
% buf
->chan
->n_subbufs
;
785 new = buf
->start
+ new_subbuf
* buf
->chan
->subbuf_size
;
787 if (!buf
->chan
->cb
->subbuf_start(buf
, new, old
, buf
->prev_padding
)) {
788 buf
->offset
= buf
->chan
->subbuf_size
+ 1;
792 buf
->padding
[new_subbuf
] = 0;
794 if (unlikely(length
+ buf
->offset
> buf
->chan
->subbuf_size
))
800 buf
->chan
->last_toobig
= length
;
803 EXPORT_SYMBOL_GPL(relay_switch_subbuf
);
806 * relay_subbufs_consumed - update the buffer's sub-buffers-consumed count
808 * @cpu: the cpu associated with the channel buffer to update
809 * @subbufs_consumed: number of sub-buffers to add to current buf's count
811 * Adds to the channel buffer's consumed sub-buffer count.
812 * subbufs_consumed should be the number of sub-buffers newly consumed,
813 * not the total consumed.
815 * NOTE. Kernel clients don't need to call this function if the channel
816 * mode is 'overwrite'.
818 void relay_subbufs_consumed(struct rchan
*chan
,
820 size_t subbufs_consumed
)
822 struct rchan_buf
*buf
;
827 if (cpu
>= NR_CPUS
|| !chan
->buf
[cpu
] ||
828 subbufs_consumed
> chan
->n_subbufs
)
831 buf
= chan
->buf
[cpu
];
832 if (subbufs_consumed
> buf
->subbufs_produced
- buf
->subbufs_consumed
)
833 buf
->subbufs_consumed
= buf
->subbufs_produced
;
835 buf
->subbufs_consumed
+= subbufs_consumed
;
837 EXPORT_SYMBOL_GPL(relay_subbufs_consumed
);
840 * relay_close - close the channel
843 * Closes all channel buffers and frees the channel.
845 void relay_close(struct rchan
*chan
)
852 mutex_lock(&relay_channels_mutex
);
853 if (chan
->is_global
&& chan
->buf
[0])
854 relay_close_buf(chan
->buf
[0]);
856 for_each_possible_cpu(i
)
858 relay_close_buf(chan
->buf
[i
]);
860 if (chan
->last_toobig
)
861 printk(KERN_WARNING
"relay: one or more items not logged "
862 "[item size (%Zd) > sub-buffer size (%Zd)]\n",
863 chan
->last_toobig
, chan
->subbuf_size
);
865 list_del(&chan
->list
);
866 kref_put(&chan
->kref
, relay_destroy_channel
);
867 mutex_unlock(&relay_channels_mutex
);
869 EXPORT_SYMBOL_GPL(relay_close
);
872 * relay_flush - close the channel
875 * Flushes all channel buffers, i.e. forces buffer switch.
877 void relay_flush(struct rchan
*chan
)
884 if (chan
->is_global
&& chan
->buf
[0]) {
885 relay_switch_subbuf(chan
->buf
[0], 0);
889 mutex_lock(&relay_channels_mutex
);
890 for_each_possible_cpu(i
)
892 relay_switch_subbuf(chan
->buf
[i
], 0);
893 mutex_unlock(&relay_channels_mutex
);
895 EXPORT_SYMBOL_GPL(relay_flush
);
898 * relay_file_open - open file op for relay files
902 * Increments the channel buffer refcount.
904 static int relay_file_open(struct inode
*inode
, struct file
*filp
)
906 struct rchan_buf
*buf
= inode
->i_private
;
907 kref_get(&buf
->kref
);
908 filp
->private_data
= buf
;
910 return nonseekable_open(inode
, filp
);
914 * relay_file_mmap - mmap file op for relay files
916 * @vma: the vma describing what to map
918 * Calls upon relay_mmap_buf() to map the file into user space.
920 static int relay_file_mmap(struct file
*filp
, struct vm_area_struct
*vma
)
922 struct rchan_buf
*buf
= filp
->private_data
;
923 return relay_mmap_buf(buf
, vma
);
927 * relay_file_poll - poll file op for relay files
933 static unsigned int relay_file_poll(struct file
*filp
, poll_table
*wait
)
935 unsigned int mask
= 0;
936 struct rchan_buf
*buf
= filp
->private_data
;
941 if (filp
->f_mode
& FMODE_READ
) {
942 poll_wait(filp
, &buf
->read_wait
, wait
);
943 if (!relay_buf_empty(buf
))
944 mask
|= POLLIN
| POLLRDNORM
;
951 * relay_file_release - release file op for relay files
955 * Decrements the channel refcount, as the filesystem is
956 * no longer using it.
958 static int relay_file_release(struct inode
*inode
, struct file
*filp
)
960 struct rchan_buf
*buf
= filp
->private_data
;
961 kref_put(&buf
->kref
, relay_remove_buf
);
967 * relay_file_read_consume - update the consumed count for the buffer
969 static void relay_file_read_consume(struct rchan_buf
*buf
,
971 size_t bytes_consumed
)
973 size_t subbuf_size
= buf
->chan
->subbuf_size
;
974 size_t n_subbufs
= buf
->chan
->n_subbufs
;
977 if (buf
->subbufs_produced
== buf
->subbufs_consumed
&&
978 buf
->offset
== buf
->bytes_consumed
)
981 if (buf
->bytes_consumed
+ bytes_consumed
> subbuf_size
) {
982 relay_subbufs_consumed(buf
->chan
, buf
->cpu
, 1);
983 buf
->bytes_consumed
= 0;
986 buf
->bytes_consumed
+= bytes_consumed
;
988 read_subbuf
= buf
->subbufs_consumed
% n_subbufs
;
990 read_subbuf
= read_pos
/ buf
->chan
->subbuf_size
;
991 if (buf
->bytes_consumed
+ buf
->padding
[read_subbuf
] == subbuf_size
) {
992 if ((read_subbuf
== buf
->subbufs_produced
% n_subbufs
) &&
993 (buf
->offset
== subbuf_size
))
995 relay_subbufs_consumed(buf
->chan
, buf
->cpu
, 1);
996 buf
->bytes_consumed
= 0;
1001 * relay_file_read_avail - boolean, are there unconsumed bytes available?
1003 static int relay_file_read_avail(struct rchan_buf
*buf
, size_t read_pos
)
1005 size_t subbuf_size
= buf
->chan
->subbuf_size
;
1006 size_t n_subbufs
= buf
->chan
->n_subbufs
;
1007 size_t produced
= buf
->subbufs_produced
;
1008 size_t consumed
= buf
->subbufs_consumed
;
1010 relay_file_read_consume(buf
, read_pos
, 0);
1012 consumed
= buf
->subbufs_consumed
;
1014 if (unlikely(buf
->offset
> subbuf_size
)) {
1015 if (produced
== consumed
)
1020 if (unlikely(produced
- consumed
>= n_subbufs
)) {
1021 consumed
= produced
- n_subbufs
+ 1;
1022 buf
->subbufs_consumed
= consumed
;
1023 buf
->bytes_consumed
= 0;
1026 produced
= (produced
% n_subbufs
) * subbuf_size
+ buf
->offset
;
1027 consumed
= (consumed
% n_subbufs
) * subbuf_size
+ buf
->bytes_consumed
;
1029 if (consumed
> produced
)
1030 produced
+= n_subbufs
* subbuf_size
;
1032 if (consumed
== produced
) {
1033 if (buf
->offset
== subbuf_size
&&
1034 buf
->subbufs_produced
> buf
->subbufs_consumed
)
1043 * relay_file_read_subbuf_avail - return bytes available in sub-buffer
1044 * @read_pos: file read position
1045 * @buf: relay channel buffer
1047 static size_t relay_file_read_subbuf_avail(size_t read_pos
,
1048 struct rchan_buf
*buf
)
1050 size_t padding
, avail
= 0;
1051 size_t read_subbuf
, read_offset
, write_subbuf
, write_offset
;
1052 size_t subbuf_size
= buf
->chan
->subbuf_size
;
1054 write_subbuf
= (buf
->data
- buf
->start
) / subbuf_size
;
1055 write_offset
= buf
->offset
> subbuf_size
? subbuf_size
: buf
->offset
;
1056 read_subbuf
= read_pos
/ subbuf_size
;
1057 read_offset
= read_pos
% subbuf_size
;
1058 padding
= buf
->padding
[read_subbuf
];
1060 if (read_subbuf
== write_subbuf
) {
1061 if (read_offset
+ padding
< write_offset
)
1062 avail
= write_offset
- (read_offset
+ padding
);
1064 avail
= (subbuf_size
- padding
) - read_offset
;
1070 * relay_file_read_start_pos - find the first available byte to read
1071 * @read_pos: file read position
1072 * @buf: relay channel buffer
1074 * If the @read_pos is in the middle of padding, return the
1075 * position of the first actually available byte, otherwise
1076 * return the original value.
1078 static size_t relay_file_read_start_pos(size_t read_pos
,
1079 struct rchan_buf
*buf
)
1081 size_t read_subbuf
, padding
, padding_start
, padding_end
;
1082 size_t subbuf_size
= buf
->chan
->subbuf_size
;
1083 size_t n_subbufs
= buf
->chan
->n_subbufs
;
1084 size_t consumed
= buf
->subbufs_consumed
% n_subbufs
;
1087 read_pos
= consumed
* subbuf_size
+ buf
->bytes_consumed
;
1088 read_subbuf
= read_pos
/ subbuf_size
;
1089 padding
= buf
->padding
[read_subbuf
];
1090 padding_start
= (read_subbuf
+ 1) * subbuf_size
- padding
;
1091 padding_end
= (read_subbuf
+ 1) * subbuf_size
;
1092 if (read_pos
>= padding_start
&& read_pos
< padding_end
) {
1093 read_subbuf
= (read_subbuf
+ 1) % n_subbufs
;
1094 read_pos
= read_subbuf
* subbuf_size
;
1101 * relay_file_read_end_pos - return the new read position
1102 * @read_pos: file read position
1103 * @buf: relay channel buffer
1104 * @count: number of bytes to be read
1106 static size_t relay_file_read_end_pos(struct rchan_buf
*buf
,
1110 size_t read_subbuf
, padding
, end_pos
;
1111 size_t subbuf_size
= buf
->chan
->subbuf_size
;
1112 size_t n_subbufs
= buf
->chan
->n_subbufs
;
1114 read_subbuf
= read_pos
/ subbuf_size
;
1115 padding
= buf
->padding
[read_subbuf
];
1116 if (read_pos
% subbuf_size
+ count
+ padding
== subbuf_size
)
1117 end_pos
= (read_subbuf
+ 1) * subbuf_size
;
1119 end_pos
= read_pos
+ count
;
1120 if (end_pos
>= subbuf_size
* n_subbufs
)
1127 * subbuf_read_actor - read up to one subbuf's worth of data
1129 static int subbuf_read_actor(size_t read_start
,
1130 struct rchan_buf
*buf
,
1132 read_descriptor_t
*desc
)
1137 from
= buf
->start
+ read_start
;
1139 if (copy_to_user(desc
->arg
.buf
, from
, avail
)) {
1140 desc
->error
= -EFAULT
;
1143 desc
->arg
.data
+= ret
;
1144 desc
->written
+= ret
;
1150 typedef int (*subbuf_actor_t
) (size_t read_start
,
1151 struct rchan_buf
*buf
,
1153 read_descriptor_t
*desc
);
1156 * relay_file_read_subbufs - read count bytes, bridging subbuf boundaries
1158 static ssize_t
relay_file_read_subbufs(struct file
*filp
, loff_t
*ppos
,
1159 subbuf_actor_t subbuf_actor
,
1160 read_descriptor_t
*desc
)
1162 struct rchan_buf
*buf
= filp
->private_data
;
1163 size_t read_start
, avail
;
1169 inode_lock(file_inode(filp
));
1171 if (!relay_file_read_avail(buf
, *ppos
))
1174 read_start
= relay_file_read_start_pos(*ppos
, buf
);
1175 avail
= relay_file_read_subbuf_avail(read_start
, buf
);
1179 avail
= min(desc
->count
, avail
);
1180 ret
= subbuf_actor(read_start
, buf
, avail
, desc
);
1181 if (desc
->error
< 0)
1185 relay_file_read_consume(buf
, read_start
, ret
);
1186 *ppos
= relay_file_read_end_pos(buf
, read_start
, ret
);
1188 } while (desc
->count
&& ret
);
1189 inode_unlock(file_inode(filp
));
1191 return desc
->written
;
1194 static ssize_t
relay_file_read(struct file
*filp
,
1195 char __user
*buffer
,
1199 read_descriptor_t desc
;
1202 desc
.arg
.buf
= buffer
;
1204 return relay_file_read_subbufs(filp
, ppos
, subbuf_read_actor
, &desc
);
1207 static void relay_consume_bytes(struct rchan_buf
*rbuf
, int bytes_consumed
)
1209 rbuf
->bytes_consumed
+= bytes_consumed
;
1211 if (rbuf
->bytes_consumed
>= rbuf
->chan
->subbuf_size
) {
1212 relay_subbufs_consumed(rbuf
->chan
, rbuf
->cpu
, 1);
1213 rbuf
->bytes_consumed
%= rbuf
->chan
->subbuf_size
;
1217 static void relay_pipe_buf_release(struct pipe_inode_info
*pipe
,
1218 struct pipe_buffer
*buf
)
1220 struct rchan_buf
*rbuf
;
1222 rbuf
= (struct rchan_buf
*)page_private(buf
->page
);
1223 relay_consume_bytes(rbuf
, buf
->private);
1226 static const struct pipe_buf_operations relay_pipe_buf_ops
= {
1228 .confirm
= generic_pipe_buf_confirm
,
1229 .release
= relay_pipe_buf_release
,
1230 .steal
= generic_pipe_buf_steal
,
1231 .get
= generic_pipe_buf_get
,
1234 static void relay_page_release(struct splice_pipe_desc
*spd
, unsigned int i
)
1239 * subbuf_splice_actor - splice up to one subbuf's worth of data
1241 static ssize_t
subbuf_splice_actor(struct file
*in
,
1243 struct pipe_inode_info
*pipe
,
1248 unsigned int pidx
, poff
, total_len
, subbuf_pages
, nr_pages
;
1249 struct rchan_buf
*rbuf
= in
->private_data
;
1250 unsigned int subbuf_size
= rbuf
->chan
->subbuf_size
;
1251 uint64_t pos
= (uint64_t) *ppos
;
1252 uint32_t alloc_size
= (uint32_t) rbuf
->chan
->alloc_size
;
1253 size_t read_start
= (size_t) do_div(pos
, alloc_size
);
1254 size_t read_subbuf
= read_start
/ subbuf_size
;
1255 size_t padding
= rbuf
->padding
[read_subbuf
];
1256 size_t nonpad_end
= read_subbuf
* subbuf_size
+ subbuf_size
- padding
;
1257 struct page
*pages
[PIPE_DEF_BUFFERS
];
1258 struct partial_page partial
[PIPE_DEF_BUFFERS
];
1259 struct splice_pipe_desc spd
= {
1262 .nr_pages_max
= PIPE_DEF_BUFFERS
,
1265 .ops
= &relay_pipe_buf_ops
,
1266 .spd_release
= relay_page_release
,
1270 if (rbuf
->subbufs_produced
== rbuf
->subbufs_consumed
)
1272 if (splice_grow_spd(pipe
, &spd
))
1276 * Adjust read len, if longer than what is available
1278 if (len
> (subbuf_size
- read_start
% subbuf_size
))
1279 len
= subbuf_size
- read_start
% subbuf_size
;
1281 subbuf_pages
= rbuf
->chan
->alloc_size
>> PAGE_SHIFT
;
1282 pidx
= (read_start
/ PAGE_SIZE
) % subbuf_pages
;
1283 poff
= read_start
& ~PAGE_MASK
;
1284 nr_pages
= min_t(unsigned int, subbuf_pages
, spd
.nr_pages_max
);
1286 for (total_len
= 0; spd
.nr_pages
< nr_pages
; spd
.nr_pages
++) {
1287 unsigned int this_len
, this_end
, private;
1288 unsigned int cur_pos
= read_start
+ total_len
;
1293 this_len
= min_t(unsigned long, len
, PAGE_SIZE
- poff
);
1296 spd
.pages
[spd
.nr_pages
] = rbuf
->page_array
[pidx
];
1297 spd
.partial
[spd
.nr_pages
].offset
= poff
;
1299 this_end
= cur_pos
+ this_len
;
1300 if (this_end
>= nonpad_end
) {
1301 this_len
= nonpad_end
- cur_pos
;
1302 private = this_len
+ padding
;
1304 spd
.partial
[spd
.nr_pages
].len
= this_len
;
1305 spd
.partial
[spd
.nr_pages
].private = private;
1308 total_len
+= this_len
;
1310 pidx
= (pidx
+ 1) % subbuf_pages
;
1312 if (this_end
>= nonpad_end
) {
1322 ret
= *nonpad_ret
= splice_to_pipe(pipe
, &spd
);
1323 if (ret
< 0 || ret
< total_len
)
1326 if (read_start
+ ret
== nonpad_end
)
1330 splice_shrink_spd(&spd
);
1334 static ssize_t
relay_file_splice_read(struct file
*in
,
1336 struct pipe_inode_info
*pipe
,
1347 while (len
&& !spliced
) {
1348 ret
= subbuf_splice_actor(in
, ppos
, pipe
, len
, flags
, &nonpad_ret
);
1352 if (flags
& SPLICE_F_NONBLOCK
)
1362 spliced
+= nonpad_ret
;
1372 const struct file_operations relay_file_operations
= {
1373 .open
= relay_file_open
,
1374 .poll
= relay_file_poll
,
1375 .mmap
= relay_file_mmap
,
1376 .read
= relay_file_read
,
1377 .llseek
= no_llseek
,
1378 .release
= relay_file_release
,
1379 .splice_read
= relay_file_splice_read
,
1381 EXPORT_SYMBOL_GPL(relay_file_operations
);
1383 static __init
int relay_init(void)
1386 hotcpu_notifier(relay_hotcpu_callback
, 0);
1390 early_initcall(relay_init
);