2 * Kernel Probes (KProbes)
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation; either version 2 of the License, or
8 * (at your option) any later version.
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
15 * You should have received a copy of the GNU General Public License
16 * along with this program; if not, write to the Free Software
17 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
19 * Copyright (C) IBM Corporation, 2002, 2004
21 * 2002-Oct Created by Vamsi Krishna S <vamsi_krishna@in.ibm.com> Kernel
22 * Probes initial implementation (includes suggestions from
24 * 2004-Aug Updated by Prasanna S Panchamukhi <prasanna@in.ibm.com> with
25 * hlists and exceptions notifier as suggested by Andi Kleen.
26 * 2004-July Suparna Bhattacharya <suparna@in.ibm.com> added jumper probes
27 * interface to access function arguments.
28 * 2004-Sep Prasanna S Panchamukhi <prasanna@in.ibm.com> Changed Kprobes
29 * exceptions notifier to be first on the priority list.
30 * 2005-May Hien Nguyen <hien@us.ibm.com>, Jim Keniston
31 * <jkenisto@us.ibm.com> and Prasanna S Panchamukhi
32 * <prasanna@in.ibm.com> added function-return probes.
34 #include <linux/kprobes.h>
35 #include <linux/spinlock.h>
36 #include <linux/hash.h>
37 #include <linux/init.h>
38 #include <linux/module.h>
39 #include <asm/cacheflush.h>
40 #include <asm/errno.h>
41 #include <asm/kdebug.h>
43 #define KPROBE_HASH_BITS 6
44 #define KPROBE_TABLE_SIZE (1 << KPROBE_HASH_BITS)
46 static struct hlist_head kprobe_table
[KPROBE_TABLE_SIZE
];
47 static struct hlist_head kretprobe_inst_table
[KPROBE_TABLE_SIZE
];
49 unsigned int kprobe_cpu
= NR_CPUS
;
50 static DEFINE_SPINLOCK(kprobe_lock
);
51 static struct kprobe
*curr_kprobe
;
53 /* Locks kprobe: irqs must be disabled */
54 void lock_kprobes(void)
56 spin_lock(&kprobe_lock
);
57 kprobe_cpu
= smp_processor_id();
60 void unlock_kprobes(void)
63 spin_unlock(&kprobe_lock
);
66 /* You have to be holding the kprobe_lock */
67 struct kprobe
*get_kprobe(void *addr
)
69 struct hlist_head
*head
;
70 struct hlist_node
*node
;
72 head
= &kprobe_table
[hash_ptr(addr
, KPROBE_HASH_BITS
)];
73 hlist_for_each(node
, head
) {
74 struct kprobe
*p
= hlist_entry(node
, struct kprobe
, hlist
);
82 * Aggregate handlers for multiple kprobes support - these handlers
83 * take care of invoking the individual kprobe handlers on p->list
85 static int aggr_pre_handler(struct kprobe
*p
, struct pt_regs
*regs
)
89 list_for_each_entry(kp
, &p
->list
, list
) {
90 if (kp
->pre_handler
) {
92 kp
->pre_handler(kp
, regs
);
99 static void aggr_post_handler(struct kprobe
*p
, struct pt_regs
*regs
,
104 list_for_each_entry(kp
, &p
->list
, list
) {
105 if (kp
->post_handler
) {
107 kp
->post_handler(kp
, regs
, flags
);
114 static int aggr_fault_handler(struct kprobe
*p
, struct pt_regs
*regs
,
118 * if we faulted "during" the execution of a user specified
119 * probe handler, invoke just that probe's fault handler
121 if (curr_kprobe
&& curr_kprobe
->fault_handler
) {
122 if (curr_kprobe
->fault_handler(curr_kprobe
, regs
, trapnr
))
128 struct kprobe trampoline_p
= {
129 .addr
= (kprobe_opcode_t
*) &kretprobe_trampoline
,
130 .pre_handler
= trampoline_probe_handler
,
131 .post_handler
= trampoline_post_handler
134 struct kretprobe_instance
*get_free_rp_inst(struct kretprobe
*rp
)
136 struct hlist_node
*node
;
137 struct kretprobe_instance
*ri
;
138 hlist_for_each_entry(ri
, node
, &rp
->free_instances
, uflist
)
143 static struct kretprobe_instance
*get_used_rp_inst(struct kretprobe
*rp
)
145 struct hlist_node
*node
;
146 struct kretprobe_instance
*ri
;
147 hlist_for_each_entry(ri
, node
, &rp
->used_instances
, uflist
)
152 struct kretprobe_instance
*get_rp_inst(void *sara
)
154 struct hlist_head
*head
;
155 struct hlist_node
*node
;
156 struct task_struct
*tsk
;
157 struct kretprobe_instance
*ri
;
159 tsk
= arch_get_kprobe_task(sara
);
160 head
= &kretprobe_inst_table
[hash_ptr(tsk
, KPROBE_HASH_BITS
)];
161 hlist_for_each_entry(ri
, node
, head
, hlist
) {
162 if (ri
->stack_addr
== sara
)
168 void add_rp_inst(struct kretprobe_instance
*ri
)
170 struct task_struct
*tsk
;
172 * Remove rp inst off the free list -
173 * Add it back when probed function returns
175 hlist_del(&ri
->uflist
);
176 tsk
= arch_get_kprobe_task(ri
->stack_addr
);
177 /* Add rp inst onto table */
178 INIT_HLIST_NODE(&ri
->hlist
);
179 hlist_add_head(&ri
->hlist
,
180 &kretprobe_inst_table
[hash_ptr(tsk
, KPROBE_HASH_BITS
)]);
182 /* Also add this rp inst to the used list. */
183 INIT_HLIST_NODE(&ri
->uflist
);
184 hlist_add_head(&ri
->uflist
, &ri
->rp
->used_instances
);
187 void recycle_rp_inst(struct kretprobe_instance
*ri
)
189 /* remove rp inst off the rprobe_inst_table */
190 hlist_del(&ri
->hlist
);
192 /* remove rp inst off the used list */
193 hlist_del(&ri
->uflist
);
194 /* put rp inst back onto the free list */
195 INIT_HLIST_NODE(&ri
->uflist
);
196 hlist_add_head(&ri
->uflist
, &ri
->rp
->free_instances
);
202 struct hlist_head
* kretprobe_inst_table_head(struct task_struct
*tsk
)
204 return &kretprobe_inst_table
[hash_ptr(tsk
, KPROBE_HASH_BITS
)];
207 struct kretprobe_instance
*get_rp_inst_tsk(struct task_struct
*tk
)
209 struct task_struct
*tsk
;
210 struct hlist_head
*head
;
211 struct hlist_node
*node
;
212 struct kretprobe_instance
*ri
;
214 head
= &kretprobe_inst_table
[hash_ptr(tk
, KPROBE_HASH_BITS
)];
216 hlist_for_each_entry(ri
, node
, head
, hlist
) {
217 tsk
= arch_get_kprobe_task(ri
->stack_addr
);
225 * This function is called from do_exit or do_execv when task tk's stack is
226 * about to be recycled. Recycle any function-return probe instances
227 * associated with this task. These represent probed functions that have
228 * been called but may never return.
230 void kprobe_flush_task(struct task_struct
*tk
)
232 unsigned long flags
= 0;
233 spin_lock_irqsave(&kprobe_lock
, flags
);
234 arch_kprobe_flush_task(tk
);
235 spin_unlock_irqrestore(&kprobe_lock
, flags
);
239 * This kprobe pre_handler is registered with every kretprobe. When probe
240 * hits it will set up the return probe.
242 static int pre_handler_kretprobe(struct kprobe
*p
, struct pt_regs
*regs
)
244 struct kretprobe
*rp
= container_of(p
, struct kretprobe
, kp
);
246 /*TODO: consider to only swap the RA after the last pre_handler fired */
247 arch_prepare_kretprobe(rp
, regs
);
251 static inline void free_rp_inst(struct kretprobe
*rp
)
253 struct kretprobe_instance
*ri
;
254 while ((ri
= get_free_rp_inst(rp
)) != NULL
) {
255 hlist_del(&ri
->uflist
);
261 * Fill in the required fields of the "manager kprobe". Replace the
262 * earlier kprobe in the hlist with the manager kprobe
264 static inline void add_aggr_kprobe(struct kprobe
*ap
, struct kprobe
*p
)
267 memcpy(&ap
->opcode
, &p
->opcode
, sizeof(kprobe_opcode_t
));
268 memcpy(&ap
->ainsn
, &p
->ainsn
, sizeof(struct arch_specific_insn
));
270 ap
->pre_handler
= aggr_pre_handler
;
271 ap
->post_handler
= aggr_post_handler
;
272 ap
->fault_handler
= aggr_fault_handler
;
274 INIT_LIST_HEAD(&ap
->list
);
275 list_add(&p
->list
, &ap
->list
);
277 INIT_HLIST_NODE(&ap
->hlist
);
278 hlist_del(&p
->hlist
);
279 hlist_add_head(&ap
->hlist
,
280 &kprobe_table
[hash_ptr(ap
->addr
, KPROBE_HASH_BITS
)]);
284 * This is the second or subsequent kprobe at the address - handle
286 * TODO: Move kcalloc outside the spinlock
288 static int register_aggr_kprobe(struct kprobe
*old_p
, struct kprobe
*p
)
293 if (old_p
->break_handler
|| p
->break_handler
) {
294 ret
= -EEXIST
; /* kprobe and jprobe can't (yet) coexist */
295 } else if (old_p
->pre_handler
== aggr_pre_handler
) {
296 list_add(&p
->list
, &old_p
->list
);
298 ap
= kcalloc(1, sizeof(struct kprobe
), GFP_ATOMIC
);
301 add_aggr_kprobe(ap
, old_p
);
302 list_add(&p
->list
, &ap
->list
);
307 /* kprobe removal house-keeping routines */
308 static inline void cleanup_kprobe(struct kprobe
*p
, unsigned long flags
)
310 arch_disarm_kprobe(p
);
311 hlist_del(&p
->hlist
);
312 spin_unlock_irqrestore(&kprobe_lock
, flags
);
313 arch_remove_kprobe(p
);
316 static inline void cleanup_aggr_kprobe(struct kprobe
*old_p
,
317 struct kprobe
*p
, unsigned long flags
)
320 if (list_empty(&old_p
->list
)) {
321 cleanup_kprobe(old_p
, flags
);
324 spin_unlock_irqrestore(&kprobe_lock
, flags
);
327 int register_kprobe(struct kprobe
*p
)
330 unsigned long flags
= 0;
331 struct kprobe
*old_p
;
333 if ((ret
= arch_prepare_kprobe(p
)) != 0) {
336 spin_lock_irqsave(&kprobe_lock
, flags
);
337 old_p
= get_kprobe(p
->addr
);
340 ret
= register_aggr_kprobe(old_p
, p
);
345 INIT_HLIST_NODE(&p
->hlist
);
346 hlist_add_head(&p
->hlist
,
347 &kprobe_table
[hash_ptr(p
->addr
, KPROBE_HASH_BITS
)]);
352 spin_unlock_irqrestore(&kprobe_lock
, flags
);
355 arch_remove_kprobe(p
);
359 void unregister_kprobe(struct kprobe
*p
)
362 struct kprobe
*old_p
;
364 spin_lock_irqsave(&kprobe_lock
, flags
);
365 old_p
= get_kprobe(p
->addr
);
367 if (old_p
->pre_handler
== aggr_pre_handler
)
368 cleanup_aggr_kprobe(old_p
, p
, flags
);
370 cleanup_kprobe(p
, flags
);
372 spin_unlock_irqrestore(&kprobe_lock
, flags
);
375 static struct notifier_block kprobe_exceptions_nb
= {
376 .notifier_call
= kprobe_exceptions_notify
,
377 .priority
= 0x7fffffff /* we need to notified first */
380 int register_jprobe(struct jprobe
*jp
)
382 /* Todo: Verify probepoint is a function entry point */
383 jp
->kp
.pre_handler
= setjmp_pre_handler
;
384 jp
->kp
.break_handler
= longjmp_break_handler
;
386 return register_kprobe(&jp
->kp
);
389 void unregister_jprobe(struct jprobe
*jp
)
391 unregister_kprobe(&jp
->kp
);
394 #ifdef ARCH_SUPPORTS_KRETPROBES
396 int register_kretprobe(struct kretprobe
*rp
)
399 struct kretprobe_instance
*inst
;
402 rp
->kp
.pre_handler
= pre_handler_kretprobe
;
404 /* Pre-allocate memory for max kretprobe instances */
405 if (rp
->maxactive
<= 0) {
406 #ifdef CONFIG_PREEMPT
407 rp
->maxactive
= max(10, 2 * NR_CPUS
);
409 rp
->maxactive
= NR_CPUS
;
412 INIT_HLIST_HEAD(&rp
->used_instances
);
413 INIT_HLIST_HEAD(&rp
->free_instances
);
414 for (i
= 0; i
< rp
->maxactive
; i
++) {
415 inst
= kmalloc(sizeof(struct kretprobe_instance
), GFP_KERNEL
);
420 INIT_HLIST_NODE(&inst
->uflist
);
421 hlist_add_head(&inst
->uflist
, &rp
->free_instances
);
425 /* Establish function entry probe point */
426 if ((ret
= register_kprobe(&rp
->kp
)) != 0)
431 #else /* ARCH_SUPPORTS_KRETPROBES */
433 int register_kretprobe(struct kretprobe
*rp
)
438 #endif /* ARCH_SUPPORTS_KRETPROBES */
440 void unregister_kretprobe(struct kretprobe
*rp
)
443 struct kretprobe_instance
*ri
;
445 unregister_kprobe(&rp
->kp
);
447 spin_lock_irqsave(&kprobe_lock
, flags
);
449 while ((ri
= get_used_rp_inst(rp
)) != NULL
) {
451 hlist_del(&ri
->uflist
);
453 spin_unlock_irqrestore(&kprobe_lock
, flags
);
456 static int __init
init_kprobes(void)
460 /* FIXME allocate the probe table, currently defined statically */
461 /* initialize all list heads */
462 for (i
= 0; i
< KPROBE_TABLE_SIZE
; i
++) {
463 INIT_HLIST_HEAD(&kprobe_table
[i
]);
464 INIT_HLIST_HEAD(&kretprobe_inst_table
[i
]);
467 err
= register_die_notifier(&kprobe_exceptions_nb
);
468 /* Register the trampoline probe for return probe */
469 register_kprobe(&trampoline_p
);
473 __initcall(init_kprobes
);
475 EXPORT_SYMBOL_GPL(register_kprobe
);
476 EXPORT_SYMBOL_GPL(unregister_kprobe
);
477 EXPORT_SYMBOL_GPL(register_jprobe
);
478 EXPORT_SYMBOL_GPL(unregister_jprobe
);
479 EXPORT_SYMBOL_GPL(jprobe_return
);
480 EXPORT_SYMBOL_GPL(register_kretprobe
);
481 EXPORT_SYMBOL_GPL(unregister_kretprobe
);