2 * Copyright (C) 2006 Jens Axboe <axboe@kernel.dk>
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as
6 * published by the Free Software Foundation.
8 * This program is distributed in the hope that it will be useful,
9 * but WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 * GNU General Public License for more details.
13 * You should have received a copy of the GNU General Public License
14 * along with this program; if not, write to the Free Software
15 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
18 #include <linux/kernel.h>
19 #include <linux/blkdev.h>
20 #include <linux/blktrace_api.h>
21 #include <linux/percpu.h>
22 #include <linux/init.h>
23 #include <linux/mutex.h>
24 #include <linux/debugfs.h>
25 #include <linux/time.h>
26 #include <asm/uaccess.h>
28 static unsigned int blktrace_seq __read_mostly
= 1;
31 * Send out a notify message.
33 static void trace_note(struct blk_trace
*bt
, pid_t pid
, int action
,
34 const void *data
, size_t len
)
36 struct blk_io_trace
*t
;
38 t
= relay_reserve(bt
->rchan
, sizeof(*t
) + len
);
40 const int cpu
= smp_processor_id();
42 t
->magic
= BLK_IO_TRACE_MAGIC
| BLK_IO_TRACE_VERSION
;
43 t
->time
= ktime_to_ns(ktime_get());
49 memcpy((void *) t
+ sizeof(*t
), data
, len
);
54 * Send out a notify for this process, if we haven't done so since a trace
57 static void trace_note_tsk(struct blk_trace
*bt
, struct task_struct
*tsk
)
59 tsk
->btrace_seq
= blktrace_seq
;
60 trace_note(bt
, tsk
->pid
, BLK_TN_PROCESS
, tsk
->comm
, sizeof(tsk
->comm
));
63 static void trace_note_time(struct blk_trace
*bt
)
70 words
[0] = now
.tv_sec
;
71 words
[1] = now
.tv_nsec
;
73 local_irq_save(flags
);
74 trace_note(bt
, 0, BLK_TN_TIMESTAMP
, words
, sizeof(words
));
75 local_irq_restore(flags
);
78 void __trace_note_message(struct blk_trace
*bt
, const char *fmt
, ...)
82 static char bt_msg_buf
[BLK_TN_MAX_MSG
];
85 n
= vscnprintf(bt_msg_buf
, BLK_TN_MAX_MSG
, fmt
, args
);
88 trace_note(bt
, 0, BLK_TN_MESSAGE
, bt_msg_buf
, n
);
90 EXPORT_SYMBOL_GPL(__trace_note_message
);
92 static int act_log_check(struct blk_trace
*bt
, u32 what
, sector_t sector
,
95 if (((bt
->act_mask
<< BLK_TC_SHIFT
) & what
) == 0)
97 if (sector
< bt
->start_lba
|| sector
> bt
->end_lba
)
99 if (bt
->pid
&& pid
!= bt
->pid
)
106 * Data direction bit lookup
108 static u32 ddir_act
[2] __read_mostly
= { BLK_TC_ACT(BLK_TC_READ
), BLK_TC_ACT(BLK_TC_WRITE
) };
111 * Bio action bits of interest
113 static u32 bio_act
[9] __read_mostly
= { 0, BLK_TC_ACT(BLK_TC_BARRIER
), BLK_TC_ACT(BLK_TC_SYNC
), 0, BLK_TC_ACT(BLK_TC_AHEAD
), 0, 0, 0, BLK_TC_ACT(BLK_TC_META
) };
116 * More could be added as needed, taking care to increment the decrementer
117 * to get correct indexing
119 #define trace_barrier_bit(rw) \
120 (((rw) & (1 << BIO_RW_BARRIER)) >> (BIO_RW_BARRIER - 0))
121 #define trace_sync_bit(rw) \
122 (((rw) & (1 << BIO_RW_SYNC)) >> (BIO_RW_SYNC - 1))
123 #define trace_ahead_bit(rw) \
124 (((rw) & (1 << BIO_RW_AHEAD)) << (2 - BIO_RW_AHEAD))
125 #define trace_meta_bit(rw) \
126 (((rw) & (1 << BIO_RW_META)) >> (BIO_RW_META - 3))
129 * The worker for the various blk_add_trace*() types. Fills out a
130 * blk_io_trace structure and places it in a per-cpu subbuffer.
132 void __blk_add_trace(struct blk_trace
*bt
, sector_t sector
, int bytes
,
133 int rw
, u32 what
, int error
, int pdu_len
, void *pdu_data
)
135 struct task_struct
*tsk
= current
;
136 struct blk_io_trace
*t
;
138 unsigned long *sequence
;
142 if (unlikely(bt
->trace_state
!= Blktrace_running
))
145 what
|= ddir_act
[rw
& WRITE
];
146 what
|= bio_act
[trace_barrier_bit(rw
)];
147 what
|= bio_act
[trace_sync_bit(rw
)];
148 what
|= bio_act
[trace_ahead_bit(rw
)];
149 what
|= bio_act
[trace_meta_bit(rw
)];
152 if (unlikely(act_log_check(bt
, what
, sector
, pid
)))
156 * A word about the locking here - we disable interrupts to reserve
157 * some space in the relay per-cpu buffer, to prevent an irq
158 * from coming in and stepping on our toes. Once reserved, it's
159 * enough to get preemption disabled to prevent read of this data
160 * before we are through filling it. get_cpu()/put_cpu() does this
163 local_irq_save(flags
);
165 if (unlikely(tsk
->btrace_seq
!= blktrace_seq
))
166 trace_note_tsk(bt
, tsk
);
168 t
= relay_reserve(bt
->rchan
, sizeof(*t
) + pdu_len
);
170 cpu
= smp_processor_id();
171 sequence
= per_cpu_ptr(bt
->sequence
, cpu
);
173 t
->magic
= BLK_IO_TRACE_MAGIC
| BLK_IO_TRACE_VERSION
;
174 t
->sequence
= ++(*sequence
);
175 t
->time
= ktime_to_ns(ktime_get());
183 t
->pdu_len
= pdu_len
;
186 memcpy((void *) t
+ sizeof(*t
), pdu_data
, pdu_len
);
189 local_irq_restore(flags
);
192 EXPORT_SYMBOL_GPL(__blk_add_trace
);
194 static struct dentry
*blk_tree_root
;
195 static DEFINE_MUTEX(blk_tree_mutex
);
196 static unsigned int root_users
;
198 static inline void blk_remove_root(void)
201 debugfs_remove(blk_tree_root
);
202 blk_tree_root
= NULL
;
206 static void blk_remove_tree(struct dentry
*dir
)
208 mutex_lock(&blk_tree_mutex
);
210 if (--root_users
== 0)
212 mutex_unlock(&blk_tree_mutex
);
215 static struct dentry
*blk_create_tree(const char *blk_name
)
217 struct dentry
*dir
= NULL
;
220 mutex_lock(&blk_tree_mutex
);
222 if (!blk_tree_root
) {
223 blk_tree_root
= debugfs_create_dir("block", NULL
);
229 dir
= debugfs_create_dir(blk_name
, blk_tree_root
);
233 /* Delete root only if we created it */
239 mutex_unlock(&blk_tree_mutex
);
243 static void blk_trace_cleanup(struct blk_trace
*bt
)
245 relay_close(bt
->rchan
);
246 debugfs_remove(bt
->dropped_file
);
247 blk_remove_tree(bt
->dir
);
248 free_percpu(bt
->sequence
);
252 int blk_trace_remove(struct request_queue
*q
)
254 struct blk_trace
*bt
;
256 bt
= xchg(&q
->blk_trace
, NULL
);
260 if (bt
->trace_state
== Blktrace_setup
||
261 bt
->trace_state
== Blktrace_stopped
)
262 blk_trace_cleanup(bt
);
266 EXPORT_SYMBOL_GPL(blk_trace_remove
);
268 static int blk_dropped_open(struct inode
*inode
, struct file
*filp
)
270 filp
->private_data
= inode
->i_private
;
275 static ssize_t
blk_dropped_read(struct file
*filp
, char __user
*buffer
,
276 size_t count
, loff_t
*ppos
)
278 struct blk_trace
*bt
= filp
->private_data
;
281 snprintf(buf
, sizeof(buf
), "%u\n", atomic_read(&bt
->dropped
));
283 return simple_read_from_buffer(buffer
, count
, ppos
, buf
, strlen(buf
));
286 static const struct file_operations blk_dropped_fops
= {
287 .owner
= THIS_MODULE
,
288 .open
= blk_dropped_open
,
289 .read
= blk_dropped_read
,
293 * Keep track of how many times we encountered a full subbuffer, to aid
294 * the user space app in telling how many lost events there were.
296 static int blk_subbuf_start_callback(struct rchan_buf
*buf
, void *subbuf
,
297 void *prev_subbuf
, size_t prev_padding
)
299 struct blk_trace
*bt
;
301 if (!relay_buf_full(buf
))
304 bt
= buf
->chan
->private_data
;
305 atomic_inc(&bt
->dropped
);
309 static int blk_remove_buf_file_callback(struct dentry
*dentry
)
311 debugfs_remove(dentry
);
315 static struct dentry
*blk_create_buf_file_callback(const char *filename
,
316 struct dentry
*parent
,
318 struct rchan_buf
*buf
,
321 return debugfs_create_file(filename
, mode
, parent
, buf
,
322 &relay_file_operations
);
325 static struct rchan_callbacks blk_relay_callbacks
= {
326 .subbuf_start
= blk_subbuf_start_callback
,
327 .create_buf_file
= blk_create_buf_file_callback
,
328 .remove_buf_file
= blk_remove_buf_file_callback
,
332 * Setup everything required to start tracing
334 int do_blk_trace_setup(struct request_queue
*q
, char *name
, dev_t dev
,
335 struct blk_user_trace_setup
*buts
)
337 struct blk_trace
*old_bt
, *bt
= NULL
;
338 struct dentry
*dir
= NULL
;
341 if (!buts
->buf_size
|| !buts
->buf_nr
)
344 strcpy(buts
->name
, name
);
347 * some device names have larger paths - convert the slashes
348 * to underscores for this to work as expected
350 for (i
= 0; i
< strlen(buts
->name
); i
++)
351 if (buts
->name
[i
] == '/')
355 bt
= kzalloc(sizeof(*bt
), GFP_KERNEL
);
359 bt
->sequence
= alloc_percpu(unsigned long);
364 dir
= blk_create_tree(buts
->name
);
370 atomic_set(&bt
->dropped
, 0);
373 bt
->dropped_file
= debugfs_create_file("dropped", 0444, dir
, bt
, &blk_dropped_fops
);
374 if (!bt
->dropped_file
)
377 bt
->rchan
= relay_open("trace", dir
, buts
->buf_size
,
378 buts
->buf_nr
, &blk_relay_callbacks
, bt
);
382 bt
->act_mask
= buts
->act_mask
;
384 bt
->act_mask
= (u16
) -1;
386 bt
->start_lba
= buts
->start_lba
;
387 bt
->end_lba
= buts
->end_lba
;
392 bt
->trace_state
= Blktrace_setup
;
395 old_bt
= xchg(&q
->blk_trace
, bt
);
397 (void) xchg(&q
->blk_trace
, old_bt
);
404 blk_remove_tree(dir
);
406 if (bt
->dropped_file
)
407 debugfs_remove(bt
->dropped_file
);
408 free_percpu(bt
->sequence
);
410 relay_close(bt
->rchan
);
416 int blk_trace_setup(struct request_queue
*q
, char *name
, dev_t dev
,
419 struct blk_user_trace_setup buts
;
422 ret
= copy_from_user(&buts
, arg
, sizeof(buts
));
426 ret
= do_blk_trace_setup(q
, name
, dev
, &buts
);
430 if (copy_to_user(arg
, &buts
, sizeof(buts
)))
435 EXPORT_SYMBOL_GPL(blk_trace_setup
);
437 int blk_trace_startstop(struct request_queue
*q
, int start
)
439 struct blk_trace
*bt
;
442 if ((bt
= q
->blk_trace
) == NULL
)
446 * For starting a trace, we can transition from a setup or stopped
447 * trace. For stopping a trace, the state must be running
451 if (bt
->trace_state
== Blktrace_setup
||
452 bt
->trace_state
== Blktrace_stopped
) {
455 bt
->trace_state
= Blktrace_running
;
461 if (bt
->trace_state
== Blktrace_running
) {
462 bt
->trace_state
= Blktrace_stopped
;
463 relay_flush(bt
->rchan
);
470 EXPORT_SYMBOL_GPL(blk_trace_startstop
);
473 * blk_trace_ioctl: - handle the ioctls associated with tracing
474 * @bdev: the block device
475 * @cmd: the ioctl cmd
476 * @arg: the argument data, if any
479 int blk_trace_ioctl(struct block_device
*bdev
, unsigned cmd
, char __user
*arg
)
481 struct request_queue
*q
;
483 char b
[BDEVNAME_SIZE
];
485 q
= bdev_get_queue(bdev
);
489 mutex_lock(&bdev
->bd_mutex
);
494 ret
= blk_trace_setup(q
, b
, bdev
->bd_dev
, arg
);
499 ret
= blk_trace_startstop(q
, start
);
501 case BLKTRACETEARDOWN
:
502 ret
= blk_trace_remove(q
);
509 mutex_unlock(&bdev
->bd_mutex
);
514 * blk_trace_shutdown: - stop and cleanup trace structures
515 * @q: the request queue associated with the device
518 void blk_trace_shutdown(struct request_queue
*q
)
521 blk_trace_startstop(q
, 0);