| 1 | /* |
| 2 | * RT-Mutexes: blocking mutual exclusion locks with PI support |
| 3 | * |
| 4 | * started by Ingo Molnar and Thomas Gleixner: |
| 5 | * |
| 6 | * Copyright (C) 2004-2006 Red Hat, Inc., Ingo Molnar <mingo@redhat.com> |
| 7 | * Copyright (C) 2006 Timesys Corp., Thomas Gleixner <tglx@timesys.com> |
| 8 | * |
| 9 | * This code is based on the rt.c implementation in the preempt-rt tree. |
| 10 | * Portions of said code are |
| 11 | * |
| 12 | * Copyright (C) 2004 LynuxWorks, Inc., Igor Manyilov, Bill Huey |
| 13 | * Copyright (C) 2006 Esben Nielsen |
| 14 | * Copyright (C) 2006 Kihon Technologies Inc., |
| 15 | * Steven Rostedt <rostedt@goodmis.org> |
| 16 | * |
| 17 | * See rt.c in preempt-rt for proper credits and further information |
| 18 | */ |
| 19 | #include <linux/config.h> |
| 20 | #include <linux/sched.h> |
| 21 | #include <linux/delay.h> |
| 22 | #include <linux/module.h> |
| 23 | #include <linux/spinlock.h> |
| 24 | #include <linux/kallsyms.h> |
| 25 | #include <linux/syscalls.h> |
| 26 | #include <linux/interrupt.h> |
| 27 | #include <linux/plist.h> |
| 28 | #include <linux/fs.h> |
| 29 | |
| 30 | #include "rtmutex_common.h" |
| 31 | |
| 32 | #ifdef CONFIG_DEBUG_RT_MUTEXES |
| 33 | # include "rtmutex-debug.h" |
| 34 | #else |
| 35 | # include "rtmutex.h" |
| 36 | #endif |
| 37 | |
| 38 | # define TRACE_WARN_ON(x) WARN_ON(x) |
| 39 | # define TRACE_BUG_ON(x) BUG_ON(x) |
| 40 | |
| 41 | # define TRACE_OFF() \ |
| 42 | do { \ |
| 43 | if (rt_trace_on) { \ |
| 44 | rt_trace_on = 0; \ |
| 45 | console_verbose(); \ |
| 46 | if (spin_is_locked(¤t->pi_lock)) \ |
| 47 | spin_unlock(¤t->pi_lock); \ |
| 48 | if (spin_is_locked(¤t->held_list_lock)) \ |
| 49 | spin_unlock(¤t->held_list_lock); \ |
| 50 | } \ |
| 51 | } while (0) |
| 52 | |
| 53 | # define TRACE_OFF_NOLOCK() \ |
| 54 | do { \ |
| 55 | if (rt_trace_on) { \ |
| 56 | rt_trace_on = 0; \ |
| 57 | console_verbose(); \ |
| 58 | } \ |
| 59 | } while (0) |
| 60 | |
| 61 | # define TRACE_BUG_LOCKED() \ |
| 62 | do { \ |
| 63 | TRACE_OFF(); \ |
| 64 | BUG(); \ |
| 65 | } while (0) |
| 66 | |
| 67 | # define TRACE_WARN_ON_LOCKED(c) \ |
| 68 | do { \ |
| 69 | if (unlikely(c)) { \ |
| 70 | TRACE_OFF(); \ |
| 71 | WARN_ON(1); \ |
| 72 | } \ |
| 73 | } while (0) |
| 74 | |
| 75 | # define TRACE_BUG_ON_LOCKED(c) \ |
| 76 | do { \ |
| 77 | if (unlikely(c)) \ |
| 78 | TRACE_BUG_LOCKED(); \ |
| 79 | } while (0) |
| 80 | |
| 81 | #ifdef CONFIG_SMP |
| 82 | # define SMP_TRACE_BUG_ON_LOCKED(c) TRACE_BUG_ON_LOCKED(c) |
| 83 | #else |
| 84 | # define SMP_TRACE_BUG_ON_LOCKED(c) do { } while (0) |
| 85 | #endif |
| 86 | |
| 87 | /* |
| 88 | * deadlock detection flag. We turn it off when we detect |
| 89 | * the first problem because we dont want to recurse back |
| 90 | * into the tracing code when doing error printk or |
| 91 | * executing a BUG(): |
| 92 | */ |
| 93 | int rt_trace_on = 1; |
| 94 | |
| 95 | void deadlock_trace_off(void) |
| 96 | { |
| 97 | rt_trace_on = 0; |
| 98 | } |
| 99 | |
| 100 | static void printk_task(task_t *p) |
| 101 | { |
| 102 | if (p) |
| 103 | printk("%16s:%5d [%p, %3d]", p->comm, p->pid, p, p->prio); |
| 104 | else |
| 105 | printk("<none>"); |
| 106 | } |
| 107 | |
| 108 | static void printk_task_short(task_t *p) |
| 109 | { |
| 110 | if (p) |
| 111 | printk("%s/%d [%p, %3d]", p->comm, p->pid, p, p->prio); |
| 112 | else |
| 113 | printk("<none>"); |
| 114 | } |
| 115 | |
| 116 | static void printk_lock(struct rt_mutex *lock, int print_owner) |
| 117 | { |
| 118 | if (lock->name) |
| 119 | printk(" [%p] {%s}\n", |
| 120 | lock, lock->name); |
| 121 | else |
| 122 | printk(" [%p] {%s:%d}\n", |
| 123 | lock, lock->file, lock->line); |
| 124 | |
| 125 | if (print_owner && rt_mutex_owner(lock)) { |
| 126 | printk(".. ->owner: %p\n", lock->owner); |
| 127 | printk(".. held by: "); |
| 128 | printk_task(rt_mutex_owner(lock)); |
| 129 | printk("\n"); |
| 130 | } |
| 131 | if (rt_mutex_owner(lock)) { |
| 132 | printk("... acquired at: "); |
| 133 | print_symbol("%s\n", lock->acquire_ip); |
| 134 | } |
| 135 | } |
| 136 | |
| 137 | static void printk_waiter(struct rt_mutex_waiter *w) |
| 138 | { |
| 139 | printk("-------------------------\n"); |
| 140 | printk("| waiter struct %p:\n", w); |
| 141 | printk("| w->list_entry: [DP:%p/%p|SP:%p/%p|PRI:%d]\n", |
| 142 | w->list_entry.plist.prio_list.prev, w->list_entry.plist.prio_list.next, |
| 143 | w->list_entry.plist.node_list.prev, w->list_entry.plist.node_list.next, |
| 144 | w->list_entry.prio); |
| 145 | printk("| w->pi_list_entry: [DP:%p/%p|SP:%p/%p|PRI:%d]\n", |
| 146 | w->pi_list_entry.plist.prio_list.prev, w->pi_list_entry.plist.prio_list.next, |
| 147 | w->pi_list_entry.plist.node_list.prev, w->pi_list_entry.plist.node_list.next, |
| 148 | w->pi_list_entry.prio); |
| 149 | printk("\n| lock:\n"); |
| 150 | printk_lock(w->lock, 1); |
| 151 | printk("| w->ti->task:\n"); |
| 152 | printk_task(w->task); |
| 153 | printk("| blocked at: "); |
| 154 | print_symbol("%s\n", w->ip); |
| 155 | printk("-------------------------\n"); |
| 156 | } |
| 157 | |
| 158 | static void show_task_locks(task_t *p) |
| 159 | { |
| 160 | switch (p->state) { |
| 161 | case TASK_RUNNING: printk("R"); break; |
| 162 | case TASK_INTERRUPTIBLE: printk("S"); break; |
| 163 | case TASK_UNINTERRUPTIBLE: printk("D"); break; |
| 164 | case TASK_STOPPED: printk("T"); break; |
| 165 | case EXIT_ZOMBIE: printk("Z"); break; |
| 166 | case EXIT_DEAD: printk("X"); break; |
| 167 | default: printk("?"); break; |
| 168 | } |
| 169 | printk_task(p); |
| 170 | if (p->pi_blocked_on) { |
| 171 | struct rt_mutex *lock = p->pi_blocked_on->lock; |
| 172 | |
| 173 | printk(" blocked on:"); |
| 174 | printk_lock(lock, 1); |
| 175 | } else |
| 176 | printk(" (not blocked)\n"); |
| 177 | } |
| 178 | |
| 179 | void rt_mutex_show_held_locks(task_t *task, int verbose) |
| 180 | { |
| 181 | struct list_head *curr, *cursor = NULL; |
| 182 | struct rt_mutex *lock; |
| 183 | task_t *t; |
| 184 | unsigned long flags; |
| 185 | int count = 0; |
| 186 | |
| 187 | if (!rt_trace_on) |
| 188 | return; |
| 189 | |
| 190 | if (verbose) { |
| 191 | printk("------------------------------\n"); |
| 192 | printk("| showing all locks held by: | ("); |
| 193 | printk_task_short(task); |
| 194 | printk("):\n"); |
| 195 | printk("------------------------------\n"); |
| 196 | } |
| 197 | |
| 198 | next: |
| 199 | spin_lock_irqsave(&task->held_list_lock, flags); |
| 200 | list_for_each(curr, &task->held_list_head) { |
| 201 | if (cursor && curr != cursor) |
| 202 | continue; |
| 203 | lock = list_entry(curr, struct rt_mutex, held_list_entry); |
| 204 | t = rt_mutex_owner(lock); |
| 205 | WARN_ON(t != task); |
| 206 | count++; |
| 207 | cursor = curr->next; |
| 208 | spin_unlock_irqrestore(&task->held_list_lock, flags); |
| 209 | |
| 210 | printk("\n#%03d: ", count); |
| 211 | printk_lock(lock, 0); |
| 212 | goto next; |
| 213 | } |
| 214 | spin_unlock_irqrestore(&task->held_list_lock, flags); |
| 215 | |
| 216 | printk("\n"); |
| 217 | } |
| 218 | |
| 219 | void rt_mutex_show_all_locks(void) |
| 220 | { |
| 221 | task_t *g, *p; |
| 222 | int count = 10; |
| 223 | int unlock = 1; |
| 224 | |
| 225 | printk("\n"); |
| 226 | printk("----------------------\n"); |
| 227 | printk("| showing all tasks: |\n"); |
| 228 | printk("----------------------\n"); |
| 229 | |
| 230 | /* |
| 231 | * Here we try to get the tasklist_lock as hard as possible, |
| 232 | * if not successful after 2 seconds we ignore it (but keep |
| 233 | * trying). This is to enable a debug printout even if a |
| 234 | * tasklist_lock-holding task deadlocks or crashes. |
| 235 | */ |
| 236 | retry: |
| 237 | if (!read_trylock(&tasklist_lock)) { |
| 238 | if (count == 10) |
| 239 | printk("hm, tasklist_lock locked, retrying... "); |
| 240 | if (count) { |
| 241 | count--; |
| 242 | printk(" #%d", 10-count); |
| 243 | mdelay(200); |
| 244 | goto retry; |
| 245 | } |
| 246 | printk(" ignoring it.\n"); |
| 247 | unlock = 0; |
| 248 | } |
| 249 | if (count != 10) |
| 250 | printk(" locked it.\n"); |
| 251 | |
| 252 | do_each_thread(g, p) { |
| 253 | show_task_locks(p); |
| 254 | if (!unlock) |
| 255 | if (read_trylock(&tasklist_lock)) |
| 256 | unlock = 1; |
| 257 | } while_each_thread(g, p); |
| 258 | |
| 259 | printk("\n"); |
| 260 | |
| 261 | printk("-----------------------------------------\n"); |
| 262 | printk("| showing all locks held in the system: |\n"); |
| 263 | printk("-----------------------------------------\n"); |
| 264 | |
| 265 | do_each_thread(g, p) { |
| 266 | rt_mutex_show_held_locks(p, 0); |
| 267 | if (!unlock) |
| 268 | if (read_trylock(&tasklist_lock)) |
| 269 | unlock = 1; |
| 270 | } while_each_thread(g, p); |
| 271 | |
| 272 | |
| 273 | printk("=============================================\n\n"); |
| 274 | |
| 275 | if (unlock) |
| 276 | read_unlock(&tasklist_lock); |
| 277 | } |
| 278 | |
| 279 | void rt_mutex_debug_check_no_locks_held(task_t *task) |
| 280 | { |
| 281 | struct rt_mutex_waiter *w; |
| 282 | struct list_head *curr; |
| 283 | struct rt_mutex *lock; |
| 284 | |
| 285 | if (!rt_trace_on) |
| 286 | return; |
| 287 | if (!rt_prio(task->normal_prio) && rt_prio(task->prio)) { |
| 288 | printk("BUG: PI priority boost leaked!\n"); |
| 289 | printk_task(task); |
| 290 | printk("\n"); |
| 291 | } |
| 292 | if (list_empty(&task->held_list_head)) |
| 293 | return; |
| 294 | |
| 295 | spin_lock(&task->pi_lock); |
| 296 | plist_for_each_entry(w, &task->pi_waiters, pi_list_entry) { |
| 297 | TRACE_OFF(); |
| 298 | |
| 299 | printk("hm, PI interest held at exit time? Task:\n"); |
| 300 | printk_task(task); |
| 301 | printk_waiter(w); |
| 302 | return; |
| 303 | } |
| 304 | spin_unlock(&task->pi_lock); |
| 305 | |
| 306 | list_for_each(curr, &task->held_list_head) { |
| 307 | lock = list_entry(curr, struct rt_mutex, held_list_entry); |
| 308 | |
| 309 | printk("BUG: %s/%d, lock held at task exit time!\n", |
| 310 | task->comm, task->pid); |
| 311 | printk_lock(lock, 1); |
| 312 | if (rt_mutex_owner(lock) != task) |
| 313 | printk("exiting task is not even the owner??\n"); |
| 314 | } |
| 315 | } |
| 316 | |
| 317 | int rt_mutex_debug_check_no_locks_freed(const void *from, unsigned long len) |
| 318 | { |
| 319 | const void *to = from + len; |
| 320 | struct list_head *curr; |
| 321 | struct rt_mutex *lock; |
| 322 | unsigned long flags; |
| 323 | void *lock_addr; |
| 324 | |
| 325 | if (!rt_trace_on) |
| 326 | return 0; |
| 327 | |
| 328 | spin_lock_irqsave(¤t->held_list_lock, flags); |
| 329 | list_for_each(curr, ¤t->held_list_head) { |
| 330 | lock = list_entry(curr, struct rt_mutex, held_list_entry); |
| 331 | lock_addr = lock; |
| 332 | if (lock_addr < from || lock_addr >= to) |
| 333 | continue; |
| 334 | TRACE_OFF(); |
| 335 | |
| 336 | printk("BUG: %s/%d, active lock [%p(%p-%p)] freed!\n", |
| 337 | current->comm, current->pid, lock, from, to); |
| 338 | dump_stack(); |
| 339 | printk_lock(lock, 1); |
| 340 | if (rt_mutex_owner(lock) != current) |
| 341 | printk("freeing task is not even the owner??\n"); |
| 342 | return 1; |
| 343 | } |
| 344 | spin_unlock_irqrestore(¤t->held_list_lock, flags); |
| 345 | |
| 346 | return 0; |
| 347 | } |
| 348 | |
| 349 | void rt_mutex_debug_task_free(struct task_struct *task) |
| 350 | { |
| 351 | WARN_ON(!plist_head_empty(&task->pi_waiters)); |
| 352 | WARN_ON(task->pi_blocked_on); |
| 353 | } |
| 354 | |
| 355 | /* |
| 356 | * We fill out the fields in the waiter to store the information about |
| 357 | * the deadlock. We print when we return. act_waiter can be NULL in |
| 358 | * case of a remove waiter operation. |
| 359 | */ |
| 360 | void debug_rt_mutex_deadlock(int detect, struct rt_mutex_waiter *act_waiter, |
| 361 | struct rt_mutex *lock) |
| 362 | { |
| 363 | struct task_struct *task; |
| 364 | |
| 365 | if (!rt_trace_on || detect || !act_waiter) |
| 366 | return; |
| 367 | |
| 368 | task = rt_mutex_owner(act_waiter->lock); |
| 369 | if (task && task != current) { |
| 370 | act_waiter->deadlock_task_pid = task->pid; |
| 371 | act_waiter->deadlock_lock = lock; |
| 372 | } |
| 373 | } |
| 374 | |
| 375 | void debug_rt_mutex_print_deadlock(struct rt_mutex_waiter *waiter) |
| 376 | { |
| 377 | struct task_struct *task; |
| 378 | |
| 379 | if (!waiter->deadlock_lock || !rt_trace_on) |
| 380 | return; |
| 381 | |
| 382 | task = find_task_by_pid(waiter->deadlock_task_pid); |
| 383 | if (!task) |
| 384 | return; |
| 385 | |
| 386 | TRACE_OFF_NOLOCK(); |
| 387 | |
| 388 | printk("\n============================================\n"); |
| 389 | printk( "[ BUG: circular locking deadlock detected! ]\n"); |
| 390 | printk( "--------------------------------------------\n"); |
| 391 | printk("%s/%d is deadlocking current task %s/%d\n\n", |
| 392 | task->comm, task->pid, current->comm, current->pid); |
| 393 | |
| 394 | printk("\n1) %s/%d is trying to acquire this lock:\n", |
| 395 | current->comm, current->pid); |
| 396 | printk_lock(waiter->lock, 1); |
| 397 | |
| 398 | printk("... trying at: "); |
| 399 | print_symbol("%s\n", waiter->ip); |
| 400 | |
| 401 | printk("\n2) %s/%d is blocked on this lock:\n", task->comm, task->pid); |
| 402 | printk_lock(waiter->deadlock_lock, 1); |
| 403 | |
| 404 | rt_mutex_show_held_locks(current, 1); |
| 405 | rt_mutex_show_held_locks(task, 1); |
| 406 | |
| 407 | printk("\n%s/%d's [blocked] stackdump:\n\n", task->comm, task->pid); |
| 408 | show_stack(task, NULL); |
| 409 | printk("\n%s/%d's [current] stackdump:\n\n", |
| 410 | current->comm, current->pid); |
| 411 | dump_stack(); |
| 412 | rt_mutex_show_all_locks(); |
| 413 | printk("[ turning off deadlock detection." |
| 414 | "Please report this trace. ]\n\n"); |
| 415 | local_irq_disable(); |
| 416 | } |
| 417 | |
| 418 | void debug_rt_mutex_lock(struct rt_mutex *lock __IP_DECL__) |
| 419 | { |
| 420 | unsigned long flags; |
| 421 | |
| 422 | if (rt_trace_on) { |
| 423 | TRACE_WARN_ON_LOCKED(!list_empty(&lock->held_list_entry)); |
| 424 | |
| 425 | spin_lock_irqsave(¤t->held_list_lock, flags); |
| 426 | list_add_tail(&lock->held_list_entry, ¤t->held_list_head); |
| 427 | spin_unlock_irqrestore(¤t->held_list_lock, flags); |
| 428 | |
| 429 | lock->acquire_ip = ip; |
| 430 | } |
| 431 | } |
| 432 | |
| 433 | void debug_rt_mutex_unlock(struct rt_mutex *lock) |
| 434 | { |
| 435 | unsigned long flags; |
| 436 | |
| 437 | if (rt_trace_on) { |
| 438 | TRACE_WARN_ON_LOCKED(rt_mutex_owner(lock) != current); |
| 439 | TRACE_WARN_ON_LOCKED(list_empty(&lock->held_list_entry)); |
| 440 | |
| 441 | spin_lock_irqsave(¤t->held_list_lock, flags); |
| 442 | list_del_init(&lock->held_list_entry); |
| 443 | spin_unlock_irqrestore(¤t->held_list_lock, flags); |
| 444 | } |
| 445 | } |
| 446 | |
| 447 | void debug_rt_mutex_proxy_lock(struct rt_mutex *lock, |
| 448 | struct task_struct *powner __IP_DECL__) |
| 449 | { |
| 450 | unsigned long flags; |
| 451 | |
| 452 | if (rt_trace_on) { |
| 453 | TRACE_WARN_ON_LOCKED(!list_empty(&lock->held_list_entry)); |
| 454 | |
| 455 | spin_lock_irqsave(&powner->held_list_lock, flags); |
| 456 | list_add_tail(&lock->held_list_entry, &powner->held_list_head); |
| 457 | spin_unlock_irqrestore(&powner->held_list_lock, flags); |
| 458 | |
| 459 | lock->acquire_ip = ip; |
| 460 | } |
| 461 | } |
| 462 | |
| 463 | void debug_rt_mutex_proxy_unlock(struct rt_mutex *lock) |
| 464 | { |
| 465 | unsigned long flags; |
| 466 | |
| 467 | if (rt_trace_on) { |
| 468 | struct task_struct *owner = rt_mutex_owner(lock); |
| 469 | |
| 470 | TRACE_WARN_ON_LOCKED(!owner); |
| 471 | TRACE_WARN_ON_LOCKED(list_empty(&lock->held_list_entry)); |
| 472 | |
| 473 | spin_lock_irqsave(&owner->held_list_lock, flags); |
| 474 | list_del_init(&lock->held_list_entry); |
| 475 | spin_unlock_irqrestore(&owner->held_list_lock, flags); |
| 476 | } |
| 477 | } |
| 478 | |
| 479 | void debug_rt_mutex_init_waiter(struct rt_mutex_waiter *waiter) |
| 480 | { |
| 481 | memset(waiter, 0x11, sizeof(*waiter)); |
| 482 | plist_node_init(&waiter->list_entry, MAX_PRIO); |
| 483 | plist_node_init(&waiter->pi_list_entry, MAX_PRIO); |
| 484 | } |
| 485 | |
| 486 | void debug_rt_mutex_free_waiter(struct rt_mutex_waiter *waiter) |
| 487 | { |
| 488 | TRACE_WARN_ON(!plist_node_empty(&waiter->list_entry)); |
| 489 | TRACE_WARN_ON(!plist_node_empty(&waiter->pi_list_entry)); |
| 490 | TRACE_WARN_ON(waiter->task); |
| 491 | memset(waiter, 0x22, sizeof(*waiter)); |
| 492 | } |
| 493 | |
| 494 | void debug_rt_mutex_init(struct rt_mutex *lock, const char *name) |
| 495 | { |
| 496 | void *addr = lock; |
| 497 | |
| 498 | if (rt_trace_on) { |
| 499 | rt_mutex_debug_check_no_locks_freed(addr, |
| 500 | sizeof(struct rt_mutex)); |
| 501 | INIT_LIST_HEAD(&lock->held_list_entry); |
| 502 | lock->name = name; |
| 503 | } |
| 504 | } |
| 505 | |
| 506 | void rt_mutex_deadlock_account_lock(struct rt_mutex *lock, task_t *task) |
| 507 | { |
| 508 | } |
| 509 | |
| 510 | void rt_mutex_deadlock_account_unlock(struct task_struct *task) |
| 511 | { |
| 512 | } |
| 513 | |