Merge remote-tracking branch 'keys/keys-next'
[deliverable/linux.git] / arch / arm64 / kernel / probes / kprobes.c
1 /*
2 * arch/arm64/kernel/probes/kprobes.c
3 *
4 * Kprobes support for ARM64
5 *
6 * Copyright (C) 2013 Linaro Limited.
7 * Author: Sandeepa Prabhu <sandeepa.prabhu@linaro.org>
8 *
9 * This program is free software; you can redistribute it and/or modify
10 * it under the terms of the GNU General Public License version 2 as
11 * published by the Free Software Foundation.
12 *
13 * This program is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 * General Public License for more details.
17 *
18 */
19 #include <linux/kasan.h>
20 #include <linux/kernel.h>
21 #include <linux/kprobes.h>
22 #include <linux/module.h>
23 #include <linux/slab.h>
24 #include <linux/stop_machine.h>
25 #include <linux/stringify.h>
26 #include <asm/traps.h>
27 #include <asm/ptrace.h>
28 #include <asm/cacheflush.h>
29 #include <asm/debug-monitors.h>
30 #include <asm/system_misc.h>
31 #include <asm/insn.h>
32 #include <asm/uaccess.h>
33 #include <asm/irq.h>
34 #include <asm/sections.h>
35
36 #include "decode-insn.h"
37
38 DEFINE_PER_CPU(struct kprobe *, current_kprobe) = NULL;
39 DEFINE_PER_CPU(struct kprobe_ctlblk, kprobe_ctlblk);
40
41 static void __kprobes
42 post_kprobe_handler(struct kprobe_ctlblk *, struct pt_regs *);
43
44 static void __kprobes arch_prepare_ss_slot(struct kprobe *p)
45 {
46 /* prepare insn slot */
47 p->ainsn.insn[0] = cpu_to_le32(p->opcode);
48
49 flush_icache_range((uintptr_t) (p->ainsn.insn),
50 (uintptr_t) (p->ainsn.insn) +
51 MAX_INSN_SIZE * sizeof(kprobe_opcode_t));
52
53 /*
54 * Needs restoring of return address after stepping xol.
55 */
56 p->ainsn.restore = (unsigned long) p->addr +
57 sizeof(kprobe_opcode_t);
58 }
59
60 static void __kprobes arch_prepare_simulate(struct kprobe *p)
61 {
62 /* This instructions is not executed xol. No need to adjust the PC */
63 p->ainsn.restore = 0;
64 }
65
66 static void __kprobes arch_simulate_insn(struct kprobe *p, struct pt_regs *regs)
67 {
68 struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
69
70 if (p->ainsn.handler)
71 p->ainsn.handler((u32)p->opcode, (long)p->addr, regs);
72
73 /* single step simulated, now go for post processing */
74 post_kprobe_handler(kcb, regs);
75 }
76
77 int __kprobes arch_prepare_kprobe(struct kprobe *p)
78 {
79 unsigned long probe_addr = (unsigned long)p->addr;
80 extern char __start_rodata[];
81 extern char __end_rodata[];
82
83 if (probe_addr & 0x3)
84 return -EINVAL;
85
86 /* copy instruction */
87 p->opcode = le32_to_cpu(*p->addr);
88
89 if (in_exception_text(probe_addr))
90 return -EINVAL;
91 if (probe_addr >= (unsigned long) __start_rodata &&
92 probe_addr <= (unsigned long) __end_rodata)
93 return -EINVAL;
94
95 /* decode instruction */
96 switch (arm_kprobe_decode_insn(p->addr, &p->ainsn)) {
97 case INSN_REJECTED: /* insn not supported */
98 return -EINVAL;
99
100 case INSN_GOOD_NO_SLOT: /* insn need simulation */
101 p->ainsn.insn = NULL;
102 break;
103
104 case INSN_GOOD: /* instruction uses slot */
105 p->ainsn.insn = get_insn_slot();
106 if (!p->ainsn.insn)
107 return -ENOMEM;
108 break;
109 };
110
111 /* prepare the instruction */
112 if (p->ainsn.insn)
113 arch_prepare_ss_slot(p);
114 else
115 arch_prepare_simulate(p);
116
117 return 0;
118 }
119
120 static int __kprobes patch_text(kprobe_opcode_t *addr, u32 opcode)
121 {
122 void *addrs[1];
123 u32 insns[1];
124
125 addrs[0] = (void *)addr;
126 insns[0] = (u32)opcode;
127
128 return aarch64_insn_patch_text(addrs, insns, 1);
129 }
130
131 /* arm kprobe: install breakpoint in text */
132 void __kprobes arch_arm_kprobe(struct kprobe *p)
133 {
134 patch_text(p->addr, BRK64_OPCODE_KPROBES);
135 }
136
137 /* disarm kprobe: remove breakpoint from text */
138 void __kprobes arch_disarm_kprobe(struct kprobe *p)
139 {
140 patch_text(p->addr, p->opcode);
141 }
142
143 void __kprobes arch_remove_kprobe(struct kprobe *p)
144 {
145 if (p->ainsn.insn) {
146 free_insn_slot(p->ainsn.insn, 0);
147 p->ainsn.insn = NULL;
148 }
149 }
150
151 static void __kprobes save_previous_kprobe(struct kprobe_ctlblk *kcb)
152 {
153 kcb->prev_kprobe.kp = kprobe_running();
154 kcb->prev_kprobe.status = kcb->kprobe_status;
155 }
156
157 static void __kprobes restore_previous_kprobe(struct kprobe_ctlblk *kcb)
158 {
159 __this_cpu_write(current_kprobe, kcb->prev_kprobe.kp);
160 kcb->kprobe_status = kcb->prev_kprobe.status;
161 }
162
163 static void __kprobes set_current_kprobe(struct kprobe *p)
164 {
165 __this_cpu_write(current_kprobe, p);
166 }
167
168 /*
169 * When PSTATE.D is set (masked), then software step exceptions can not be
170 * generated.
171 * SPSR's D bit shows the value of PSTATE.D immediately before the
172 * exception was taken. PSTATE.D is set while entering into any exception
173 * mode, however software clears it for any normal (none-debug-exception)
174 * mode in the exception entry. Therefore, when we are entering into kprobe
175 * breakpoint handler from any normal mode then SPSR.D bit is already
176 * cleared, however it is set when we are entering from any debug exception
177 * mode.
178 * Since we always need to generate single step exception after a kprobe
179 * breakpoint exception therefore we need to clear it unconditionally, when
180 * we become sure that the current breakpoint exception is for kprobe.
181 */
182 static void __kprobes
183 spsr_set_debug_flag(struct pt_regs *regs, int mask)
184 {
185 unsigned long spsr = regs->pstate;
186
187 if (mask)
188 spsr |= PSR_D_BIT;
189 else
190 spsr &= ~PSR_D_BIT;
191
192 regs->pstate = spsr;
193 }
194
195 /*
196 * Interrupts need to be disabled before single-step mode is set, and not
197 * reenabled until after single-step mode ends.
198 * Without disabling interrupt on local CPU, there is a chance of
199 * interrupt occurrence in the period of exception return and start of
200 * out-of-line single-step, that result in wrongly single stepping
201 * into the interrupt handler.
202 */
203 static void __kprobes kprobes_save_local_irqflag(struct kprobe_ctlblk *kcb,
204 struct pt_regs *regs)
205 {
206 kcb->saved_irqflag = regs->pstate;
207 regs->pstate |= PSR_I_BIT;
208 }
209
210 static void __kprobes kprobes_restore_local_irqflag(struct kprobe_ctlblk *kcb,
211 struct pt_regs *regs)
212 {
213 if (kcb->saved_irqflag & PSR_I_BIT)
214 regs->pstate |= PSR_I_BIT;
215 else
216 regs->pstate &= ~PSR_I_BIT;
217 }
218
219 static void __kprobes
220 set_ss_context(struct kprobe_ctlblk *kcb, unsigned long addr)
221 {
222 kcb->ss_ctx.ss_pending = true;
223 kcb->ss_ctx.match_addr = addr + sizeof(kprobe_opcode_t);
224 }
225
226 static void __kprobes clear_ss_context(struct kprobe_ctlblk *kcb)
227 {
228 kcb->ss_ctx.ss_pending = false;
229 kcb->ss_ctx.match_addr = 0;
230 }
231
232 static void __kprobes setup_singlestep(struct kprobe *p,
233 struct pt_regs *regs,
234 struct kprobe_ctlblk *kcb, int reenter)
235 {
236 unsigned long slot;
237
238 if (reenter) {
239 save_previous_kprobe(kcb);
240 set_current_kprobe(p);
241 kcb->kprobe_status = KPROBE_REENTER;
242 } else {
243 kcb->kprobe_status = KPROBE_HIT_SS;
244 }
245
246
247 if (p->ainsn.insn) {
248 /* prepare for single stepping */
249 slot = (unsigned long)p->ainsn.insn;
250
251 set_ss_context(kcb, slot); /* mark pending ss */
252
253 spsr_set_debug_flag(regs, 0);
254
255 /* IRQs and single stepping do not mix well. */
256 kprobes_save_local_irqflag(kcb, regs);
257 kernel_enable_single_step(regs);
258 instruction_pointer_set(regs, slot);
259 } else {
260 /* insn simulation */
261 arch_simulate_insn(p, regs);
262 }
263 }
264
265 static int __kprobes reenter_kprobe(struct kprobe *p,
266 struct pt_regs *regs,
267 struct kprobe_ctlblk *kcb)
268 {
269 switch (kcb->kprobe_status) {
270 case KPROBE_HIT_SSDONE:
271 case KPROBE_HIT_ACTIVE:
272 kprobes_inc_nmissed_count(p);
273 setup_singlestep(p, regs, kcb, 1);
274 break;
275 case KPROBE_HIT_SS:
276 case KPROBE_REENTER:
277 pr_warn("Unrecoverable kprobe detected at %p.\n", p->addr);
278 dump_kprobe(p);
279 BUG();
280 break;
281 default:
282 WARN_ON(1);
283 return 0;
284 }
285
286 return 1;
287 }
288
289 static void __kprobes
290 post_kprobe_handler(struct kprobe_ctlblk *kcb, struct pt_regs *regs)
291 {
292 struct kprobe *cur = kprobe_running();
293
294 if (!cur)
295 return;
296
297 /* return addr restore if non-branching insn */
298 if (cur->ainsn.restore != 0)
299 instruction_pointer_set(regs, cur->ainsn.restore);
300
301 /* restore back original saved kprobe variables and continue */
302 if (kcb->kprobe_status == KPROBE_REENTER) {
303 restore_previous_kprobe(kcb);
304 return;
305 }
306 /* call post handler */
307 kcb->kprobe_status = KPROBE_HIT_SSDONE;
308 if (cur->post_handler) {
309 /* post_handler can hit breakpoint and single step
310 * again, so we enable D-flag for recursive exception.
311 */
312 cur->post_handler(cur, regs, 0);
313 }
314
315 reset_current_kprobe();
316 }
317
318 int __kprobes kprobe_fault_handler(struct pt_regs *regs, unsigned int fsr)
319 {
320 struct kprobe *cur = kprobe_running();
321 struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
322
323 switch (kcb->kprobe_status) {
324 case KPROBE_HIT_SS:
325 case KPROBE_REENTER:
326 /*
327 * We are here because the instruction being single
328 * stepped caused a page fault. We reset the current
329 * kprobe and the ip points back to the probe address
330 * and allow the page fault handler to continue as a
331 * normal page fault.
332 */
333 instruction_pointer_set(regs, (unsigned long) cur->addr);
334 if (!instruction_pointer(regs))
335 BUG();
336
337 kernel_disable_single_step();
338
339 if (kcb->kprobe_status == KPROBE_REENTER)
340 restore_previous_kprobe(kcb);
341 else
342 reset_current_kprobe();
343
344 break;
345 case KPROBE_HIT_ACTIVE:
346 case KPROBE_HIT_SSDONE:
347 /*
348 * We increment the nmissed count for accounting,
349 * we can also use npre/npostfault count for accounting
350 * these specific fault cases.
351 */
352 kprobes_inc_nmissed_count(cur);
353
354 /*
355 * We come here because instructions in the pre/post
356 * handler caused the page_fault, this could happen
357 * if handler tries to access user space by
358 * copy_from_user(), get_user() etc. Let the
359 * user-specified handler try to fix it first.
360 */
361 if (cur->fault_handler && cur->fault_handler(cur, regs, fsr))
362 return 1;
363
364 /*
365 * In case the user-specified fault handler returned
366 * zero, try to fix up.
367 */
368 if (fixup_exception(regs))
369 return 1;
370 }
371 return 0;
372 }
373
374 int __kprobes kprobe_exceptions_notify(struct notifier_block *self,
375 unsigned long val, void *data)
376 {
377 return NOTIFY_DONE;
378 }
379
380 static void __kprobes kprobe_handler(struct pt_regs *regs)
381 {
382 struct kprobe *p, *cur_kprobe;
383 struct kprobe_ctlblk *kcb;
384 unsigned long addr = instruction_pointer(regs);
385
386 kcb = get_kprobe_ctlblk();
387 cur_kprobe = kprobe_running();
388
389 p = get_kprobe((kprobe_opcode_t *) addr);
390
391 if (p) {
392 if (cur_kprobe) {
393 if (reenter_kprobe(p, regs, kcb))
394 return;
395 } else {
396 /* Probe hit */
397 set_current_kprobe(p);
398 kcb->kprobe_status = KPROBE_HIT_ACTIVE;
399
400 /*
401 * If we have no pre-handler or it returned 0, we
402 * continue with normal processing. If we have a
403 * pre-handler and it returned non-zero, it prepped
404 * for calling the break_handler below on re-entry,
405 * so get out doing nothing more here.
406 *
407 * pre_handler can hit a breakpoint and can step thru
408 * before return, keep PSTATE D-flag enabled until
409 * pre_handler return back.
410 */
411 if (!p->pre_handler || !p->pre_handler(p, regs)) {
412 setup_singlestep(p, regs, kcb, 0);
413 return;
414 }
415 }
416 } else if ((le32_to_cpu(*(kprobe_opcode_t *) addr) ==
417 BRK64_OPCODE_KPROBES) && cur_kprobe) {
418 /* We probably hit a jprobe. Call its break handler. */
419 if (cur_kprobe->break_handler &&
420 cur_kprobe->break_handler(cur_kprobe, regs)) {
421 setup_singlestep(cur_kprobe, regs, kcb, 0);
422 return;
423 }
424 }
425 /*
426 * The breakpoint instruction was removed right
427 * after we hit it. Another cpu has removed
428 * either a probepoint or a debugger breakpoint
429 * at this address. In either case, no further
430 * handling of this interrupt is appropriate.
431 * Return back to original instruction, and continue.
432 */
433 }
434
435 static int __kprobes
436 kprobe_ss_hit(struct kprobe_ctlblk *kcb, unsigned long addr)
437 {
438 if ((kcb->ss_ctx.ss_pending)
439 && (kcb->ss_ctx.match_addr == addr)) {
440 clear_ss_context(kcb); /* clear pending ss */
441 return DBG_HOOK_HANDLED;
442 }
443 /* not ours, kprobes should ignore it */
444 return DBG_HOOK_ERROR;
445 }
446
447 int __kprobes
448 kprobe_single_step_handler(struct pt_regs *regs, unsigned int esr)
449 {
450 struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
451 int retval;
452
453 /* return error if this is not our step */
454 retval = kprobe_ss_hit(kcb, instruction_pointer(regs));
455
456 if (retval == DBG_HOOK_HANDLED) {
457 kprobes_restore_local_irqflag(kcb, regs);
458 kernel_disable_single_step();
459
460 post_kprobe_handler(kcb, regs);
461 }
462
463 return retval;
464 }
465
466 int __kprobes
467 kprobe_breakpoint_handler(struct pt_regs *regs, unsigned int esr)
468 {
469 kprobe_handler(regs);
470 return DBG_HOOK_HANDLED;
471 }
472
473 int __kprobes setjmp_pre_handler(struct kprobe *p, struct pt_regs *regs)
474 {
475 struct jprobe *jp = container_of(p, struct jprobe, kp);
476 struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
477
478 kcb->jprobe_saved_regs = *regs;
479 /*
480 * Since we can't be sure where in the stack frame "stacked"
481 * pass-by-value arguments are stored we just don't try to
482 * duplicate any of the stack. Do not use jprobes on functions that
483 * use more than 64 bytes (after padding each to an 8 byte boundary)
484 * of arguments, or pass individual arguments larger than 16 bytes.
485 */
486
487 instruction_pointer_set(regs, (unsigned long) jp->entry);
488 preempt_disable();
489 pause_graph_tracing();
490 return 1;
491 }
492
493 void __kprobes jprobe_return(void)
494 {
495 struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
496
497 /*
498 * Jprobe handler return by entering break exception,
499 * encoded same as kprobe, but with following conditions
500 * -a special PC to identify it from the other kprobes.
501 * -restore stack addr to original saved pt_regs
502 */
503 asm volatile(" mov sp, %0 \n"
504 "jprobe_return_break: brk %1 \n"
505 :
506 : "r" (kcb->jprobe_saved_regs.sp),
507 "I" (BRK64_ESR_KPROBES)
508 : "memory");
509
510 unreachable();
511 }
512
513 int __kprobes longjmp_break_handler(struct kprobe *p, struct pt_regs *regs)
514 {
515 struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
516 long stack_addr = kcb->jprobe_saved_regs.sp;
517 long orig_sp = kernel_stack_pointer(regs);
518 struct jprobe *jp = container_of(p, struct jprobe, kp);
519 extern const char jprobe_return_break[];
520
521 if (instruction_pointer(regs) != (u64) jprobe_return_break)
522 return 0;
523
524 if (orig_sp != stack_addr) {
525 struct pt_regs *saved_regs =
526 (struct pt_regs *)kcb->jprobe_saved_regs.sp;
527 pr_err("current sp %lx does not match saved sp %lx\n",
528 orig_sp, stack_addr);
529 pr_err("Saved registers for jprobe %p\n", jp);
530 show_regs(saved_regs);
531 pr_err("Current registers\n");
532 show_regs(regs);
533 BUG();
534 }
535 unpause_graph_tracing();
536 *regs = kcb->jprobe_saved_regs;
537 preempt_enable_no_resched();
538 return 1;
539 }
540
541 bool arch_within_kprobe_blacklist(unsigned long addr)
542 {
543 if ((addr >= (unsigned long)__kprobes_text_start &&
544 addr < (unsigned long)__kprobes_text_end) ||
545 (addr >= (unsigned long)__entry_text_start &&
546 addr < (unsigned long)__entry_text_end) ||
547 (addr >= (unsigned long)__idmap_text_start &&
548 addr < (unsigned long)__idmap_text_end) ||
549 !!search_exception_tables(addr))
550 return true;
551
552 if (!is_kernel_in_hyp_mode()) {
553 if ((addr >= (unsigned long)__hyp_text_start &&
554 addr < (unsigned long)__hyp_text_end) ||
555 (addr >= (unsigned long)__hyp_idmap_text_start &&
556 addr < (unsigned long)__hyp_idmap_text_end))
557 return true;
558 }
559
560 return false;
561 }
562
563 void __kprobes __used *trampoline_probe_handler(struct pt_regs *regs)
564 {
565 struct kretprobe_instance *ri = NULL;
566 struct hlist_head *head, empty_rp;
567 struct hlist_node *tmp;
568 unsigned long flags, orig_ret_address = 0;
569 unsigned long trampoline_address =
570 (unsigned long)&kretprobe_trampoline;
571 kprobe_opcode_t *correct_ret_addr = NULL;
572
573 INIT_HLIST_HEAD(&empty_rp);
574 kretprobe_hash_lock(current, &head, &flags);
575
576 /*
577 * It is possible to have multiple instances associated with a given
578 * task either because multiple functions in the call path have
579 * return probes installed on them, and/or more than one
580 * return probe was registered for a target function.
581 *
582 * We can handle this because:
583 * - instances are always pushed into the head of the list
584 * - when multiple return probes are registered for the same
585 * function, the (chronologically) first instance's ret_addr
586 * will be the real return address, and all the rest will
587 * point to kretprobe_trampoline.
588 */
589 hlist_for_each_entry_safe(ri, tmp, head, hlist) {
590 if (ri->task != current)
591 /* another task is sharing our hash bucket */
592 continue;
593
594 orig_ret_address = (unsigned long)ri->ret_addr;
595
596 if (orig_ret_address != trampoline_address)
597 /*
598 * This is the real return address. Any other
599 * instances associated with this task are for
600 * other calls deeper on the call stack
601 */
602 break;
603 }
604
605 kretprobe_assert(ri, orig_ret_address, trampoline_address);
606
607 correct_ret_addr = ri->ret_addr;
608 hlist_for_each_entry_safe(ri, tmp, head, hlist) {
609 if (ri->task != current)
610 /* another task is sharing our hash bucket */
611 continue;
612
613 orig_ret_address = (unsigned long)ri->ret_addr;
614 if (ri->rp && ri->rp->handler) {
615 __this_cpu_write(current_kprobe, &ri->rp->kp);
616 get_kprobe_ctlblk()->kprobe_status = KPROBE_HIT_ACTIVE;
617 ri->ret_addr = correct_ret_addr;
618 ri->rp->handler(ri, regs);
619 __this_cpu_write(current_kprobe, NULL);
620 }
621
622 recycle_rp_inst(ri, &empty_rp);
623
624 if (orig_ret_address != trampoline_address)
625 /*
626 * This is the real return address. Any other
627 * instances associated with this task are for
628 * other calls deeper on the call stack
629 */
630 break;
631 }
632
633 kretprobe_hash_unlock(current, &flags);
634
635 hlist_for_each_entry_safe(ri, tmp, &empty_rp, hlist) {
636 hlist_del(&ri->hlist);
637 kfree(ri);
638 }
639 return (void *)orig_ret_address;
640 }
641
642 void __kprobes arch_prepare_kretprobe(struct kretprobe_instance *ri,
643 struct pt_regs *regs)
644 {
645 ri->ret_addr = (kprobe_opcode_t *)regs->regs[30];
646
647 /* replace return addr (x30) with trampoline */
648 regs->regs[30] = (long)&kretprobe_trampoline;
649 }
650
651 int __kprobes arch_trampoline_kprobe(struct kprobe *p)
652 {
653 return 0;
654 }
655
656 int __init arch_init_kprobes(void)
657 {
658 return 0;
659 }
This page took 0.046982 seconds and 5 git commands to generate.