arch/x86_64/mm/fault.c

   1 /*
   2  *  linux/arch/x86-64/mm/fault.c
   3  *
   4  *  Copyright (C) 1995  Linus Torvalds
   5  *  Copyright (C) 2001,2002 Andi Kleen, SuSE Labs.
   6  */
   7
   8 #include <linux/config.h>
   9 #include <linux/signal.h>
  10 #include <linux/sched.h>
  11 #include <linux/kernel.h>
  12 #include <linux/errno.h>
  13 #include <linux/string.h>
  14 #include <linux/types.h>
  15 #include <linux/ptrace.h>
  16 #include <linux/mman.h>
  17 #include <linux/mm.h>
  18 #include <linux/smp.h>
  19 #include <linux/smp_lock.h>
  20 #include <linux/interrupt.h>
  21 #include <linux/init.h>
  22 #include <linux/tty.h>
  23 #include <linux/vt_kern.h>              /* For unblank_screen() */
  24 #include <linux/compiler.h>
  25 #include <linux/module.h>
  26 #include <linux/kprobes.h>
  27
  28 #include <asm/system.h>
  29 #include <asm/uaccess.h>
  30 #include <asm/pgalloc.h>
  31 #include <asm/smp.h>
  32 #include <asm/tlbflush.h>
  33 #include <asm/proto.h>
  34 #include <asm/kdebug.h>
  35 #include <asm-generic/sections.h>
  36 #include <asm/kdebug.h>
  37
  38 void bust_spinlocks(int yes)
  39 {
  40         int loglevel_save = console_loglevel;
  41         if (yes) {
  42                 oops_in_progress = 1;
  43         } else {
  44 #ifdef CONFIG_VT
  45                 unblank_screen();
  46 #endif
  47                 oops_in_progress = 0;
  48                 /*
  49                  * OK, the message is on the console.  Now we call printk()
  50                  * without oops_in_progress set so that printk will give klogd
  51                  * a poke.  Hold onto your hats...
  52                  */
  53                 console_loglevel = 15;          /* NMI oopser may have shut the console up */
  54                 printk(" ");
  55                 console_loglevel = loglevel_save;
  56         }
  57 }
  58
  59 /* Sometimes the CPU reports invalid exceptions on prefetch.
  60    Check that here and ignore.
  61    Opcode checker based on code by Richard Brunner */
  62 static noinline int is_prefetch(struct pt_regs *regs, unsigned long addr,
  63                                 unsigned long error_code)
  64 {
  65         unsigned char *instr;
  66         int scan_more = 1;
  67         int prefetch = 0;
  68         unsigned char *max_instr;
  69
  70         /* If it was a exec fault ignore */
  71         if (error_code & (1<<4))
  72                 return 0;
  73
  74         instr = (unsigned char *)convert_rip_to_linear(current, regs);
  75         max_instr = instr + 15;
  76
  77         if (user_mode(regs) && instr >= (unsigned char *)TASK_SIZE)
  78                 return 0;
  79
  80         while (scan_more && instr < max_instr) {
  81                 unsigned char opcode;
  82                 unsigned char instr_hi;
  83                 unsigned char instr_lo;
  84
  85                 if (__get_user(opcode, instr))
  86                         break;
  87
  88                 instr_hi = opcode & 0xf0;
  89                 instr_lo = opcode & 0x0f;
  90                 instr++;
  91
  92                 switch (instr_hi) {
  93                 case 0x20:
  94                 case 0x30:
  95                         /* Values 0x26,0x2E,0x36,0x3E are valid x86
  96                            prefixes.  In long mode, the CPU will signal
  97                            invalid opcode if some of these prefixes are
  98                            present so we will never get here anyway */
  99                         scan_more = ((instr_lo & 7) == 0x6);
 100                         break;
 101
 102                 case 0x40:
 103                         /* In AMD64 long mode, 0x40 to 0x4F are valid REX prefixes
 104                            Need to figure out under what instruction mode the
 105                            instruction was issued ... */
 106                         /* Could check the LDT for lm, but for now it's good
 107                            enough to assume that long mode only uses well known
 108                            segments or kernel. */
 109                         scan_more = (!user_mode(regs)) || (regs->cs == __USER_CS);
 110                         break;
 111
 112                 case 0x60:
 113                         /* 0x64 thru 0x67 are valid prefixes in all modes. */
 114                         scan_more = (instr_lo & 0xC) == 0x4;
 115                         break;
 116                 case 0xF0:
 117                         /* 0xF0, 0xF2, and 0xF3 are valid prefixes in all modes. */
 118                         scan_more = !instr_lo || (instr_lo>>1) == 1;
 119                         break;
 120                 case 0x00:
 121                         /* Prefetch instruction is 0x0F0D or 0x0F18 */
 122                         scan_more = 0;
 123                         if (__get_user(opcode, instr))
 124                                 break;
 125                         prefetch = (instr_lo == 0xF) &&
 126                                 (opcode == 0x0D || opcode == 0x18);
 127                         break;
 128                 default:
 129                         scan_more = 0;
 130                         break;
 131                 }
 132         }
 133         return prefetch;
 134 }
 135
 136 static int bad_address(void *p)
 137 {
 138         unsigned long dummy;
 139         return __get_user(dummy, (unsigned long *)p);
 140 }
 141
 142 void dump_pagetable(unsigned long address)
 143 {
 144         pgd_t *pgd;
 145         pud_t *pud;
 146         pmd_t *pmd;
 147         pte_t *pte;
 148
 149         asm("movq %%cr3,%0" : "=r" (pgd));
 150
 151         pgd = __va((unsigned long)pgd & PHYSICAL_PAGE_MASK);
 152         pgd += pgd_index(address);
 153         printk("PGD %lx ", pgd_val(*pgd));
 154         if (bad_address(pgd)) goto bad;
 155         if (!pgd_present(*pgd)) goto ret;
 156
 157         pud = __pud_offset_k((pud_t *)pgd_page(*pgd), address);
 158         if (bad_address(pud)) goto bad;
 159         printk("PUD %lx ", pud_val(*pud));
 160         if (!pud_present(*pud)) goto ret;
 161
 162         pmd = pmd_offset(pud, address);
 163         if (bad_address(pmd)) goto bad;
 164         printk("PMD %lx ", pmd_val(*pmd));
 165         if (!pmd_present(*pmd)) goto ret;
 166
 167         pte = pte_offset_kernel(pmd, address);
 168         if (bad_address(pte)) goto bad;
 169         printk("PTE %lx", pte_val(*pte));
 170 ret:
 171         printk("\n");
 172         return;
 173 bad:
 174         printk("BAD\n");
 175 }
 176
 177 static const char errata93_warning[] =
 178 KERN_ERR "******* Your BIOS seems to not contain a fix for K8 errata #93\n"
 179 KERN_ERR "******* Working around it, but it may cause SEGVs or burn power.\n"
 180 KERN_ERR "******* Please consider a BIOS update.\n"
 181 KERN_ERR "******* Disabling USB legacy in the BIOS may also help.\n";
 182
 183 /* Workaround for K8 erratum #93 & buggy BIOS.
 184    BIOS SMM functions are required to use a specific workaround
 185    to avoid corruption of the 64bit RIP register on C stepping K8.
 186    A lot of BIOS that didn't get tested properly miss this.
 187    The OS sees this as a page fault with the upper 32bits of RIP cleared.
 188    Try to work around it here.
 189    Note we only handle faults in kernel here. */
 190
 191 static int is_errata93(struct pt_regs *regs, unsigned long address)
 192 {
 193         static int warned;
 194         if (address != regs->rip)
 195                 return 0;
 196         if ((address >> 32) != 0)
 197                 return 0;
 198         address |= 0xffffffffUL << 32;
 199         if ((address >= (u64)_stext && address <= (u64)_etext) ||
 200             (address >= MODULES_VADDR && address <= MODULES_END)) {
 201                 if (!warned) {
 202                         printk(errata93_warning);
 203                         warned = 1;
 204                 }
 205                 regs->rip = address;
 206                 return 1;
 207         }
 208         return 0;
 209 }
 210
 211 int unhandled_signal(struct task_struct *tsk, int sig)
 212 {
 213         if (tsk->pid == 1)
 214                 return 1;
 215         /* Warn for strace, but not for gdb */
 216         if (!test_ti_thread_flag(tsk->thread_info, TIF_SYSCALL_TRACE) &&
 217             (tsk->ptrace & PT_PTRACED))
 218                 return 0;
 219         return (tsk->sighand->action[sig-1].sa.sa_handler == SIG_IGN) ||
 220                 (tsk->sighand->action[sig-1].sa.sa_handler == SIG_DFL);
 221 }
 222
 223 static noinline void pgtable_bad(unsigned long address, struct pt_regs *regs,
 224                                  unsigned long error_code)
 225 {
 226         oops_begin();
 227         printk(KERN_ALERT "%s: Corrupted page table at address %lx\n",
 228                current->comm, address);
 229         dump_pagetable(address);
 230         __die("Bad pagetable", regs, error_code);
 231         oops_end();
 232         do_exit(SIGKILL);
 233 }
 234
 235 /*
 236  * Handle a fault on the vmalloc or module mapping area
 237  *
 238  * This assumes no large pages in there.
 239  */
 240 static int vmalloc_fault(unsigned long address)
 241 {
 242         pgd_t *pgd, *pgd_ref;
 243         pud_t *pud, *pud_ref;
 244         pmd_t *pmd, *pmd_ref;
 245         pte_t *pte, *pte_ref;
 246
 247         /* Copy kernel mappings over when needed. This can also
 248            happen within a race in page table update. In the later
 249            case just flush. */
 250
 251         pgd = pgd_offset(current->mm ?: &init_mm, address);
 252         pgd_ref = pgd_offset_k(address);
 253         if (pgd_none(*pgd_ref))
 254                 return -1;
 255         if (pgd_none(*pgd))
 256                 set_pgd(pgd, *pgd_ref);
 257
 258         /* Below here mismatches are bugs because these lower tables
 259            are shared */
 260
 261         pud = pud_offset(pgd, address);
 262         pud_ref = pud_offset(pgd_ref, address);
 263         if (pud_none(*pud_ref))
 264                 return -1;
 265         if (pud_none(*pud) || pud_page(*pud) != pud_page(*pud_ref))
 266                 BUG();
 267         pmd = pmd_offset(pud, address);
 268         pmd_ref = pmd_offset(pud_ref, address);
 269         if (pmd_none(*pmd_ref))
 270                 return -1;
 271         if (pmd_none(*pmd) || pmd_page(*pmd) != pmd_page(*pmd_ref))
 272                 BUG();
 273         pte_ref = pte_offset_kernel(pmd_ref, address);
 274         if (!pte_present(*pte_ref))
 275                 return -1;
 276         pte = pte_offset_kernel(pmd, address);
 277         /* Don't use pte_page here, because the mappings can point
 278            outside mem_map, and the NUMA hash lookup cannot handle
 279            that. */
 280         if (!pte_present(*pte) || pte_pfn(*pte) != pte_pfn(*pte_ref))
 281                 BUG();
 282         __flush_tlb_all();
 283         return 0;
 284 }
 285
 286 int page_fault_trace = 0;
 287 int exception_trace = 1;
 288
 289 /*
 290  * This routine handles page faults.  It determines the address,
 291  * and the problem, and then passes it off to one of the appropriate
 292  * routines.
 293  *
 294  * error_code:
 295  *      bit 0 == 0 means no page found, 1 means protection fault
 296  *      bit 1 == 0 means read, 1 means write
 297  *      bit 2 == 0 means kernel, 1 means user-mode
 298  *      bit 3 == 1 means fault was an instruction fetch
 299  */
 300 asmlinkage void do_page_fault(struct pt_regs *regs, unsigned long error_code)
 301 {
 302         struct task_struct *tsk;
 303         struct mm_struct *mm;
 304         struct vm_area_struct * vma;
 305         unsigned long address;
 306         const struct exception_table_entry *fixup;
 307         int write;
 308         siginfo_t info;
 309
 310 #ifdef CONFIG_CHECKING
 311         {
 312                 unsigned long gs;
 313                 struct x8664_pda *pda = cpu_pda + stack_smp_processor_id();
 314                 rdmsrl(MSR_GS_BASE, gs);
 315                 if (gs != (unsigned long)pda) {
 316                         wrmsrl(MSR_GS_BASE, pda);
 317                         printk("page_fault: wrong gs %lx expected %p\n", gs, pda);
 318                 }
 319         }
 320 #endif
 321
 322         /* get the address */
 323         __asm__("movq %%cr2,%0":"=r" (address));
 324         if (notify_die(DIE_PAGE_FAULT, "page fault", regs, error_code, 14,
 325                                         SIGSEGV) == NOTIFY_STOP)
 326                 return;
 327
 328         if (likely(regs->eflags & X86_EFLAGS_IF))
 329                 local_irq_enable();
 330
 331         if (unlikely(page_fault_trace))
 332                 printk("pagefault rip:%lx rsp:%lx cs:%lu ss:%lu address %lx error %lx\n",
 333                        regs->rip,regs->rsp,regs->cs,regs->ss,address,error_code);
 334
 335         tsk = current;
 336         mm = tsk->mm;
 337         info.si_code = SEGV_MAPERR;
 338
 339
 340         /*
 341          * We fault-in kernel-space virtual memory on-demand. The
 342          * 'reference' page table is init_mm.pgd.
 343          *
 344          * NOTE! We MUST NOT take any locks for this case. We may
 345          * be in an interrupt or a critical region, and should
 346          * only copy the information from the master page table,
 347          * nothing more.
 348          *
 349          * This verifies that the fault happens in kernel space
 350          * (error_code & 4) == 0, and that the fault was not a
 351          * protection error (error_code & 1) == 0.
 352          */
 353         if (unlikely(address >= TASK_SIZE64)) {
 354                 if (!(error_code & 5) &&
 355                       ((address >= VMALLOC_START && address < VMALLOC_END) ||
 356                        (address >= MODULES_VADDR && address < MODULES_END))) {
 357                         if (vmalloc_fault(address) < 0)
 358                                 goto bad_area_nosemaphore;
 359                         return;
 360                 }
 361                 /*
 362                  * Don't take the mm semaphore here. If we fixup a prefetch
 363                  * fault we could otherwise deadlock.
 364                  */
 365                 goto bad_area_nosemaphore;
 366         }
 367
 368         if (unlikely(error_code & (1 << 3)))
 369                 pgtable_bad(address, regs, error_code);
 370
 371         /*
 372          * If we're in an interrupt or have no user
 373          * context, we must not take the fault..
 374          */
 375         if (unlikely(in_atomic() || !mm))
 376                 goto bad_area_nosemaphore;
 377
 378  again:
 379         /* When running in the kernel we expect faults to occur only to
 380          * addresses in user space.  All other faults represent errors in the
 381          * kernel and should generate an OOPS.  Unfortunatly, in the case of an
 382          * erroneous fault occuring in a code path which already holds mmap_sem
 383          * we will deadlock attempting to validate the fault against the
 384          * address space.  Luckily the kernel only validly references user
 385          * space from well defined areas of code, which are listed in the
 386          * exceptions table.
 387          *
 388          * As the vast majority of faults will be valid we will only perform
 389          * the source reference check when there is a possibilty of a deadlock.
 390          * Attempt to lock the address space, if we cannot we then validate the
 391          * source.  If this is invalid we can skip the address space check,
 392          * thus avoiding the deadlock.
 393          */
 394         if (!down_read_trylock(&mm->mmap_sem)) {
 395                 if ((error_code & 4) == 0 &&
 396                     !search_exception_tables(regs->rip))
 397                         goto bad_area_nosemaphore;
 398                 down_read(&mm->mmap_sem);
 399         }
 400
 401         vma = find_vma(mm, address);
 402         if (!vma)
 403                 goto bad_area;
 404         if (likely(vma->vm_start <= address))
 405                 goto good_area;
 406         if (!(vma->vm_flags & VM_GROWSDOWN))
 407                 goto bad_area;
 408         if (error_code & 4) {
 409                 // XXX: align red zone size with ABI
 410                 if (address + 128 < regs->rsp)
 411                         goto bad_area;
 412         }
 413         if (expand_stack(vma, address))
 414                 goto bad_area;
 415 /*
 416  * Ok, we have a good vm_area for this memory access, so
 417  * we can handle it..
 418  */
 419 good_area:
 420         info.si_code = SEGV_ACCERR;
 421         write = 0;
 422         switch (error_code & 3) {
 423                 default:        /* 3: write, present */
 424                         /* fall through */
 425                 case 2:         /* write, not present */
 426                         if (!(vma->vm_flags & VM_WRITE))
 427                                 goto bad_area;
 428                         write++;
 429                         break;
 430                 case 1:         /* read, present */
 431                         goto bad_area;
 432                 case 0:         /* read, not present */
 433                         if (!(vma->vm_flags & (VM_READ | VM_EXEC)))
 434                                 goto bad_area;
 435         }
 436
 437         /*
 438          * If for any reason at all we couldn't handle the fault,
 439          * make sure we exit gracefully rather than endlessly redo
 440          * the fault.
 441          */
 442         switch (handle_mm_fault(mm, vma, address, write)) {
 443         case 1:
 444                 tsk->min_flt++;
 445                 break;
 446         case 2:
 447                 tsk->maj_flt++;
 448                 break;
 449         case 0:
 450                 goto do_sigbus;
 451         default:
 452                 goto out_of_memory;
 453         }
 454
 455         up_read(&mm->mmap_sem);
 456         return;
 457
 458 /*
 459  * Something tried to access memory that isn't in our memory map..
 460  * Fix it, but check if it's kernel or user first..
 461  */
 462 bad_area:
 463         up_read(&mm->mmap_sem);
 464
 465 bad_area_nosemaphore:
 466         /* User mode accesses just cause a SIGSEGV */
 467         if (error_code & 4) {
 468                 if (is_prefetch(regs, address, error_code))
 469                         return;
 470
 471                 /* Work around K8 erratum #100 K8 in compat mode
 472                    occasionally jumps to illegal addresses >4GB.  We
 473                    catch this here in the page fault handler because
 474                    these addresses are not reachable. Just detect this
 475                    case and return.  Any code segment in LDT is
 476                    compatibility mode. */
 477                 if ((regs->cs == __USER32_CS || (regs->cs & (1<<2))) &&
 478                     (address >> 32))
 479                         return;
 480
 481                 if (exception_trace && unhandled_signal(tsk, SIGSEGV)) {
 482                         printk(
 483                        "%s%s[%d]: segfault at %016lx rip %016lx rsp %016lx error %lx\n",
 484                                         tsk->pid > 1 ? KERN_INFO : KERN_EMERG,
 485                                         tsk->comm, tsk->pid, address, regs->rip,
 486                                         regs->rsp, error_code);
 487                 }
 488
 489                 tsk->thread.cr2 = address;
 490                 /* Kernel addresses are always protection faults */
 491                 tsk->thread.error_code = error_code | (address >= TASK_SIZE);
 492                 tsk->thread.trap_no = 14;
 493                 info.si_signo = SIGSEGV;
 494                 info.si_errno = 0;
 495                 /* info.si_code has been set above */
 496                 info.si_addr = (void __user *)address;
 497                 force_sig_info(SIGSEGV, &info, tsk);
 498                 return;
 499         }
 500
 501 no_context:
 502
 503         /* Are we prepared to handle this kernel fault?  */
 504         fixup = search_exception_tables(regs->rip);
 505         if (fixup) {
 506                 regs->rip = fixup->fixup;
 507                 return;
 508         }
 509
 510         /*
 511          * Hall of shame of CPU/BIOS bugs.
 512          */
 513
 514         if (is_prefetch(regs, address, error_code))
 515                 return;
 516
 517         if (is_errata93(regs, address))
 518                 return;
 519
 520 /*
 521  * Oops. The kernel tried to access some bad page. We'll have to
 522  * terminate things with extreme prejudice.
 523  */
 524
 525         oops_begin();
 526
 527         if (address < PAGE_SIZE)
 528                 printk(KERN_ALERT "Unable to handle kernel NULL pointer dereference");
 529         else
 530                 printk(KERN_ALERT "Unable to handle kernel paging request");
 531         printk(" at %016lx RIP: \n" KERN_ALERT,address);
 532         printk_address(regs->rip);
 533         printk("\n");
 534         dump_pagetable(address);
 535         __die("Oops", regs, error_code);
 536         /* Executive summary in case the body of the oops scrolled away */
 537         printk(KERN_EMERG "CR2: %016lx\n", address);
 538         oops_end();
 539         do_exit(SIGKILL);
 540
 541 /*
 542  * We ran out of memory, or some other thing happened to us that made
 543  * us unable to handle the page fault gracefully.
 544  */
 545 out_of_memory:
 546         up_read(&mm->mmap_sem);
 547         if (current->pid == 1) {
 548                 yield();
 549                 goto again;
 550         }
 551         printk("VM: killing process %s\n", tsk->comm);
 552         if (error_code & 4)
 553                 do_exit(SIGKILL);
 554         goto no_context;
 555
 556 do_sigbus:
 557         up_read(&mm->mmap_sem);
 558
 559         /* Kernel mode? Handle exceptions or die */
 560         if (!(error_code & 4))
 561                 goto no_context;
 562
 563         tsk->thread.cr2 = address;
 564         tsk->thread.error_code = error_code;
 565         tsk->thread.trap_no = 14;
 566         info.si_signo = SIGBUS;
 567         info.si_errno = 0;
 568         info.si_code = BUS_ADRERR;
 569         info.si_addr = (void __user *)address;
 570         force_sig_info(SIGBUS, &info, tsk);
 571         return;
 572 }