| 1 | /* |
| 2 | * Dynamic function tracing support. |
| 3 | * |
| 4 | * Copyright (C) 2008 Abhishek Sagar <sagar.abhishek@gmail.com> |
| 5 | * Copyright (C) 2010 Rabin Vincent <rabin@rab.in> |
| 6 | * |
| 7 | * For licencing details, see COPYING. |
| 8 | * |
| 9 | * Defines low-level handling of mcount calls when the kernel |
| 10 | * is compiled with the -pg flag. When using dynamic ftrace, the |
| 11 | * mcount call-sites get patched with NOP till they are enabled. |
| 12 | * All code mutation routines here are called under stop_machine(). |
| 13 | */ |
| 14 | |
| 15 | #include <linux/ftrace.h> |
| 16 | #include <linux/uaccess.h> |
| 17 | |
| 18 | #include <asm/cacheflush.h> |
| 19 | #include <asm/ftrace.h> |
| 20 | |
| 21 | #ifdef CONFIG_THUMB2_KERNEL |
| 22 | #define NOP 0xeb04f85d /* pop.w {lr} */ |
| 23 | #else |
| 24 | #define NOP 0xe8bd4000 /* pop {lr} */ |
| 25 | #endif |
| 26 | |
| 27 | #ifdef CONFIG_DYNAMIC_FTRACE |
| 28 | #ifdef CONFIG_OLD_MCOUNT |
| 29 | #define OLD_MCOUNT_ADDR ((unsigned long) mcount) |
| 30 | #define OLD_FTRACE_ADDR ((unsigned long) ftrace_caller_old) |
| 31 | |
| 32 | #define OLD_NOP 0xe1a00000 /* mov r0, r0 */ |
| 33 | |
| 34 | static unsigned long ftrace_nop_replace(struct dyn_ftrace *rec) |
| 35 | { |
| 36 | return rec->arch.old_mcount ? OLD_NOP : NOP; |
| 37 | } |
| 38 | |
| 39 | static unsigned long adjust_address(struct dyn_ftrace *rec, unsigned long addr) |
| 40 | { |
| 41 | if (!rec->arch.old_mcount) |
| 42 | return addr; |
| 43 | |
| 44 | if (addr == MCOUNT_ADDR) |
| 45 | addr = OLD_MCOUNT_ADDR; |
| 46 | else if (addr == FTRACE_ADDR) |
| 47 | addr = OLD_FTRACE_ADDR; |
| 48 | |
| 49 | return addr; |
| 50 | } |
| 51 | #else |
| 52 | static unsigned long ftrace_nop_replace(struct dyn_ftrace *rec) |
| 53 | { |
| 54 | return NOP; |
| 55 | } |
| 56 | |
| 57 | static unsigned long adjust_address(struct dyn_ftrace *rec, unsigned long addr) |
| 58 | { |
| 59 | return addr; |
| 60 | } |
| 61 | #endif |
| 62 | |
| 63 | #ifdef CONFIG_THUMB2_KERNEL |
| 64 | static unsigned long ftrace_gen_branch(unsigned long pc, unsigned long addr, |
| 65 | bool link) |
| 66 | { |
| 67 | unsigned long s, j1, j2, i1, i2, imm10, imm11; |
| 68 | unsigned long first, second; |
| 69 | long offset; |
| 70 | |
| 71 | offset = (long)addr - (long)(pc + 4); |
| 72 | if (offset < -16777216 || offset > 16777214) { |
| 73 | WARN_ON_ONCE(1); |
| 74 | return 0; |
| 75 | } |
| 76 | |
| 77 | s = (offset >> 24) & 0x1; |
| 78 | i1 = (offset >> 23) & 0x1; |
| 79 | i2 = (offset >> 22) & 0x1; |
| 80 | imm10 = (offset >> 12) & 0x3ff; |
| 81 | imm11 = (offset >> 1) & 0x7ff; |
| 82 | |
| 83 | j1 = (!i1) ^ s; |
| 84 | j2 = (!i2) ^ s; |
| 85 | |
| 86 | first = 0xf000 | (s << 10) | imm10; |
| 87 | second = 0x9000 | (j1 << 13) | (j2 << 11) | imm11; |
| 88 | if (link) |
| 89 | second |= 1 << 14; |
| 90 | |
| 91 | return (second << 16) | first; |
| 92 | } |
| 93 | #else |
| 94 | static unsigned long ftrace_gen_branch(unsigned long pc, unsigned long addr, |
| 95 | bool link) |
| 96 | { |
| 97 | unsigned long opcode = 0xea000000; |
| 98 | long offset; |
| 99 | |
| 100 | if (link) |
| 101 | opcode |= 1 << 24; |
| 102 | |
| 103 | offset = (long)addr - (long)(pc + 8); |
| 104 | if (unlikely(offset < -33554432 || offset > 33554428)) { |
| 105 | /* Can't generate branches that far (from ARM ARM). Ftrace |
| 106 | * doesn't generate branches outside of kernel text. |
| 107 | */ |
| 108 | WARN_ON_ONCE(1); |
| 109 | return 0; |
| 110 | } |
| 111 | |
| 112 | offset = (offset >> 2) & 0x00ffffff; |
| 113 | |
| 114 | return opcode | offset; |
| 115 | } |
| 116 | #endif |
| 117 | |
| 118 | static unsigned long ftrace_call_replace(unsigned long pc, unsigned long addr) |
| 119 | { |
| 120 | return ftrace_gen_branch(pc, addr, true); |
| 121 | } |
| 122 | |
| 123 | static int ftrace_modify_code(unsigned long pc, unsigned long old, |
| 124 | unsigned long new) |
| 125 | { |
| 126 | unsigned long replaced; |
| 127 | |
| 128 | if (probe_kernel_read(&replaced, (void *)pc, MCOUNT_INSN_SIZE)) |
| 129 | return -EFAULT; |
| 130 | |
| 131 | if (replaced != old) |
| 132 | return -EINVAL; |
| 133 | |
| 134 | if (probe_kernel_write((void *)pc, &new, MCOUNT_INSN_SIZE)) |
| 135 | return -EPERM; |
| 136 | |
| 137 | flush_icache_range(pc, pc + MCOUNT_INSN_SIZE); |
| 138 | |
| 139 | return 0; |
| 140 | } |
| 141 | |
| 142 | int ftrace_update_ftrace_func(ftrace_func_t func) |
| 143 | { |
| 144 | unsigned long pc, old; |
| 145 | unsigned long new; |
| 146 | int ret; |
| 147 | |
| 148 | pc = (unsigned long)&ftrace_call; |
| 149 | memcpy(&old, &ftrace_call, MCOUNT_INSN_SIZE); |
| 150 | new = ftrace_call_replace(pc, (unsigned long)func); |
| 151 | |
| 152 | ret = ftrace_modify_code(pc, old, new); |
| 153 | |
| 154 | #ifdef CONFIG_OLD_MCOUNT |
| 155 | if (!ret) { |
| 156 | pc = (unsigned long)&ftrace_call_old; |
| 157 | memcpy(&old, &ftrace_call_old, MCOUNT_INSN_SIZE); |
| 158 | new = ftrace_call_replace(pc, (unsigned long)func); |
| 159 | |
| 160 | ret = ftrace_modify_code(pc, old, new); |
| 161 | } |
| 162 | #endif |
| 163 | |
| 164 | return ret; |
| 165 | } |
| 166 | |
| 167 | int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr) |
| 168 | { |
| 169 | unsigned long new, old; |
| 170 | unsigned long ip = rec->ip; |
| 171 | |
| 172 | old = ftrace_nop_replace(rec); |
| 173 | new = ftrace_call_replace(ip, adjust_address(rec, addr)); |
| 174 | |
| 175 | return ftrace_modify_code(rec->ip, old, new); |
| 176 | } |
| 177 | |
| 178 | int ftrace_make_nop(struct module *mod, |
| 179 | struct dyn_ftrace *rec, unsigned long addr) |
| 180 | { |
| 181 | unsigned long ip = rec->ip; |
| 182 | unsigned long old; |
| 183 | unsigned long new; |
| 184 | int ret; |
| 185 | |
| 186 | old = ftrace_call_replace(ip, adjust_address(rec, addr)); |
| 187 | new = ftrace_nop_replace(rec); |
| 188 | ret = ftrace_modify_code(ip, old, new); |
| 189 | |
| 190 | #ifdef CONFIG_OLD_MCOUNT |
| 191 | if (ret == -EINVAL && addr == MCOUNT_ADDR) { |
| 192 | rec->arch.old_mcount = true; |
| 193 | |
| 194 | old = ftrace_call_replace(ip, adjust_address(rec, addr)); |
| 195 | new = ftrace_nop_replace(rec); |
| 196 | ret = ftrace_modify_code(ip, old, new); |
| 197 | } |
| 198 | #endif |
| 199 | |
| 200 | return ret; |
| 201 | } |
| 202 | |
| 203 | int __init ftrace_dyn_arch_init(void *data) |
| 204 | { |
| 205 | *(unsigned long *)data = 0; |
| 206 | |
| 207 | return 0; |
| 208 | } |
| 209 | #endif /* CONFIG_DYNAMIC_FTRACE */ |
| 210 | |
| 211 | #ifdef CONFIG_FUNCTION_GRAPH_TRACER |
| 212 | void prepare_ftrace_return(unsigned long *parent, unsigned long self_addr, |
| 213 | unsigned long frame_pointer) |
| 214 | { |
| 215 | unsigned long return_hooker = (unsigned long) &return_to_handler; |
| 216 | struct ftrace_graph_ent trace; |
| 217 | unsigned long old; |
| 218 | int err; |
| 219 | |
| 220 | if (unlikely(atomic_read(¤t->tracing_graph_pause))) |
| 221 | return; |
| 222 | |
| 223 | old = *parent; |
| 224 | *parent = return_hooker; |
| 225 | |
| 226 | err = ftrace_push_return_trace(old, self_addr, &trace.depth, |
| 227 | frame_pointer); |
| 228 | if (err == -EBUSY) { |
| 229 | *parent = old; |
| 230 | return; |
| 231 | } |
| 232 | |
| 233 | trace.func = self_addr; |
| 234 | |
| 235 | /* Only trace if the calling function expects to */ |
| 236 | if (!ftrace_graph_entry(&trace)) { |
| 237 | current->curr_ret_stack--; |
| 238 | *parent = old; |
| 239 | } |
| 240 | } |
| 241 | |
| 242 | #ifdef CONFIG_DYNAMIC_FTRACE |
| 243 | extern unsigned long ftrace_graph_call; |
| 244 | extern unsigned long ftrace_graph_call_old; |
| 245 | extern void ftrace_graph_caller_old(void); |
| 246 | |
| 247 | static int __ftrace_modify_caller(unsigned long *callsite, |
| 248 | void (*func) (void), bool enable) |
| 249 | { |
| 250 | unsigned long caller_fn = (unsigned long) func; |
| 251 | unsigned long pc = (unsigned long) callsite; |
| 252 | unsigned long branch = ftrace_gen_branch(pc, caller_fn, false); |
| 253 | unsigned long nop = 0xe1a00000; /* mov r0, r0 */ |
| 254 | unsigned long old = enable ? nop : branch; |
| 255 | unsigned long new = enable ? branch : nop; |
| 256 | |
| 257 | return ftrace_modify_code(pc, old, new); |
| 258 | } |
| 259 | |
| 260 | static int ftrace_modify_graph_caller(bool enable) |
| 261 | { |
| 262 | int ret; |
| 263 | |
| 264 | ret = __ftrace_modify_caller(&ftrace_graph_call, |
| 265 | ftrace_graph_caller, |
| 266 | enable); |
| 267 | |
| 268 | #ifdef CONFIG_OLD_MCOUNT |
| 269 | if (!ret) |
| 270 | ret = __ftrace_modify_caller(&ftrace_graph_call_old, |
| 271 | ftrace_graph_caller_old, |
| 272 | enable); |
| 273 | #endif |
| 274 | |
| 275 | return ret; |
| 276 | } |
| 277 | |
| 278 | int ftrace_enable_ftrace_graph_caller(void) |
| 279 | { |
| 280 | return ftrace_modify_graph_caller(true); |
| 281 | } |
| 282 | |
| 283 | int ftrace_disable_ftrace_graph_caller(void) |
| 284 | { |
| 285 | return ftrace_modify_graph_caller(false); |
| 286 | } |
| 287 | #endif /* CONFIG_DYNAMIC_FTRACE */ |
| 288 | #endif /* CONFIG_FUNCTION_GRAPH_TRACER */ |