arch/x86_64/mm/fault.c

   1 /*
   2  *  linux/arch/x86-64/mm/fault.c
   3  *
   4  *  Copyright (C) 1995  Linus Torvalds
   5  *  Copyright (C) 2001,2002 Andi Kleen, SuSE Labs.
   6  */
   7
   8 #include <linux/config.h>
   9 #include <linux/signal.h>
  10 #include <linux/sched.h>
  11 #include <linux/kernel.h>
  12 #include <linux/errno.h>
  13 #include <linux/string.h>
  14 #include <linux/types.h>
  15 #include <linux/ptrace.h>
  16 #include <linux/mman.h>
  17 #include <linux/mm.h>
  18 #include <linux/smp.h>
  19 #include <linux/smp_lock.h>
  20 #include <linux/interrupt.h>
  21 #include <linux/init.h>
  22 #include <linux/tty.h>
  23 #include <linux/vt_kern.h>              /* For unblank_screen() */
  24 #include <linux/compiler.h>
  25 #include <linux/module.h>
  26 #include <linux/kprobes.h>
  27
  28 #include <asm/system.h>
  29 #include <asm/uaccess.h>
  30 #include <asm/pgalloc.h>
  31 #include <asm/smp.h>
  32 #include <asm/tlbflush.h>
  33 #include <asm/proto.h>
  34 #include <asm/kdebug.h>
  35 #include <asm-generic/sections.h>
  36 #include <asm/kdebug.h>
  37
  38 /* Page fault error code bits */
  39 #define PF_PROT (1<<0)          /* or no page found */
  40 #define PF_WRITE        (1<<1)
  41 #define PF_USER (1<<2)
  42 #define PF_RSVD (1<<3)
  43 #define PF_INSTR        (1<<4)
  44
  45 void bust_spinlocks(int yes)
  46 {
  47         int loglevel_save = console_loglevel;
  48         if (yes) {
  49                 oops_in_progress = 1;
  50         } else {
  51 #ifdef CONFIG_VT
  52                 unblank_screen();
  53 #endif
  54                 oops_in_progress = 0;
  55                 /*
  56                  * OK, the message is on the console.  Now we call printk()
  57                  * without oops_in_progress set so that printk will give klogd
  58                  * a poke.  Hold onto your hats...
  59                  */
  60                 console_loglevel = 15;          /* NMI oopser may have shut the console up */
  61                 printk(" ");
  62                 console_loglevel = loglevel_save;
  63         }
  64 }
  65
  66 /* Sometimes the CPU reports invalid exceptions on prefetch.
  67    Check that here and ignore.
  68    Opcode checker based on code by Richard Brunner */
  69 static noinline int is_prefetch(struct pt_regs *regs, unsigned long addr,
  70                                 unsigned long error_code)
  71 {
  72         unsigned char *instr;
  73         int scan_more = 1;
  74         int prefetch = 0;
  75         unsigned char *max_instr;
  76
  77         /* If it was a exec fault ignore */
  78         if (error_code & PF_INSTR)
  79                 return 0;
  80
  81         instr = (unsigned char *)convert_rip_to_linear(current, regs);
  82         max_instr = instr + 15;
  83
  84         if (user_mode(regs) && instr >= (unsigned char *)TASK_SIZE)
  85                 return 0;
  86
  87         while (scan_more && instr < max_instr) {
  88                 unsigned char opcode;
  89                 unsigned char instr_hi;
  90                 unsigned char instr_lo;
  91
  92                 if (__get_user(opcode, instr))
  93                         break;
  94
  95                 instr_hi = opcode & 0xf0;
  96                 instr_lo = opcode & 0x0f;
  97                 instr++;
  98
  99                 switch (instr_hi) {
 100                 case 0x20:
 101                 case 0x30:
 102                         /* Values 0x26,0x2E,0x36,0x3E are valid x86
 103                            prefixes.  In long mode, the CPU will signal
 104                            invalid opcode if some of these prefixes are
 105                            present so we will never get here anyway */
 106                         scan_more = ((instr_lo & 7) == 0x6);
 107                         break;
 108
 109                 case 0x40:
 110                         /* In AMD64 long mode, 0x40 to 0x4F are valid REX prefixes
 111                            Need to figure out under what instruction mode the
 112                            instruction was issued ... */
 113                         /* Could check the LDT for lm, but for now it's good
 114                            enough to assume that long mode only uses well known
 115                            segments or kernel. */
 116                         scan_more = (!user_mode(regs)) || (regs->cs == __USER_CS);
 117                         break;
 118
 119                 case 0x60:
 120                         /* 0x64 thru 0x67 are valid prefixes in all modes. */
 121                         scan_more = (instr_lo & 0xC) == 0x4;
 122                         break;
 123                 case 0xF0:
 124                         /* 0xF0, 0xF2, and 0xF3 are valid prefixes in all modes. */
 125                         scan_more = !instr_lo || (instr_lo>>1) == 1;
 126                         break;
 127                 case 0x00:
 128                         /* Prefetch instruction is 0x0F0D or 0x0F18 */
 129                         scan_more = 0;
 130                         if (__get_user(opcode, instr))
 131                                 break;
 132                         prefetch = (instr_lo == 0xF) &&
 133                                 (opcode == 0x0D || opcode == 0x18);
 134                         break;
 135                 default:
 136                         scan_more = 0;
 137                         break;
 138                 }
 139         }
 140         return prefetch;
 141 }
 142
 143 static int bad_address(void *p)
 144 {
 145         unsigned long dummy;
 146         return __get_user(dummy, (unsigned long *)p);
 147 }
 148
 149 void dump_pagetable(unsigned long address)
 150 {
 151         pgd_t *pgd;
 152         pud_t *pud;
 153         pmd_t *pmd;
 154         pte_t *pte;
 155
 156         asm("movq %%cr3,%0" : "=r" (pgd));
 157
 158         pgd = __va((unsigned long)pgd & PHYSICAL_PAGE_MASK);
 159         pgd += pgd_index(address);
 160         printk("PGD %lx ", pgd_val(*pgd));
 161         if (bad_address(pgd)) goto bad;
 162         if (!pgd_present(*pgd)) goto ret;
 163
 164         pud = __pud_offset_k((pud_t *)pgd_page(*pgd), address);
 165         if (bad_address(pud)) goto bad;
 166         printk("PUD %lx ", pud_val(*pud));
 167         if (!pud_present(*pud)) goto ret;
 168
 169         pmd = pmd_offset(pud, address);
 170         if (bad_address(pmd)) goto bad;
 171         printk("PMD %lx ", pmd_val(*pmd));
 172         if (!pmd_present(*pmd)) goto ret;
 173
 174         pte = pte_offset_kernel(pmd, address);
 175         if (bad_address(pte)) goto bad;
 176         printk("PTE %lx", pte_val(*pte));
 177 ret:
 178         printk("\n");
 179         return;
 180 bad:
 181         printk("BAD\n");
 182 }
 183
 184 static const char errata93_warning[] =
 185 KERN_ERR "******* Your BIOS seems to not contain a fix for K8 errata #93\n"
 186 KERN_ERR "******* Working around it, but it may cause SEGVs or burn power.\n"
 187 KERN_ERR "******* Please consider a BIOS update.\n"
 188 KERN_ERR "******* Disabling USB legacy in the BIOS may also help.\n";
 189
 190 /* Workaround for K8 erratum #93 & buggy BIOS.
 191    BIOS SMM functions are required to use a specific workaround
 192    to avoid corruption of the 64bit RIP register on C stepping K8.
 193    A lot of BIOS that didn't get tested properly miss this.
 194    The OS sees this as a page fault with the upper 32bits of RIP cleared.
 195    Try to work around it here.
 196    Note we only handle faults in kernel here. */
 197
 198 static int is_errata93(struct pt_regs *regs, unsigned long address)
 199 {
 200         static int warned;
 201         if (address != regs->rip)
 202                 return 0;
 203         if ((address >> 32) != 0)
 204                 return 0;
 205         address |= 0xffffffffUL << 32;
 206         if ((address >= (u64)_stext && address <= (u64)_etext) ||
 207             (address >= MODULES_VADDR && address <= MODULES_END)) {
 208                 if (!warned) {
 209                         printk(errata93_warning);
 210                         warned = 1;
 211                 }
 212                 regs->rip = address;
 213                 return 1;
 214         }
 215         return 0;
 216 }
 217
 218 int unhandled_signal(struct task_struct *tsk, int sig)
 219 {
 220         if (tsk->pid == 1)
 221                 return 1;
 222         if (tsk->ptrace & PT_PTRACED)
 223                 return 0;
 224         return (tsk->sighand->action[sig-1].sa.sa_handler == SIG_IGN) ||
 225                 (tsk->sighand->action[sig-1].sa.sa_handler == SIG_DFL);
 226 }
 227
 228 static noinline void pgtable_bad(unsigned long address, struct pt_regs *regs,
 229                                  unsigned long error_code)
 230 {
 231         unsigned long flags = oops_begin();
 232         struct task_struct *tsk;
 233
 234         printk(KERN_ALERT "%s: Corrupted page table at address %lx\n",
 235                current->comm, address);
 236         dump_pagetable(address);
 237         tsk = current;
 238         tsk->thread.cr2 = address;
 239         tsk->thread.trap_no = 14;
 240         tsk->thread.error_code = error_code;
 241         __die("Bad pagetable", regs, error_code);
 242         oops_end(flags);
 243         do_exit(SIGKILL);
 244 }
 245
 246 /*
 247  * Handle a fault on the vmalloc area
 248  *
 249  * This assumes no large pages in there.
 250  */
 251 static int vmalloc_fault(unsigned long address)
 252 {
 253         pgd_t *pgd, *pgd_ref;
 254         pud_t *pud, *pud_ref;
 255         pmd_t *pmd, *pmd_ref;
 256         pte_t *pte, *pte_ref;
 257
 258         /* Copy kernel mappings over when needed. This can also
 259            happen within a race in page table update. In the later
 260            case just flush. */
 261
 262         pgd = pgd_offset(current->mm ?: &init_mm, address);
 263         pgd_ref = pgd_offset_k(address);
 264         if (pgd_none(*pgd_ref))
 265                 return -1;
 266         if (pgd_none(*pgd))
 267                 set_pgd(pgd, *pgd_ref);
 268
 269         /* Below here mismatches are bugs because these lower tables
 270            are shared */
 271
 272         pud = pud_offset(pgd, address);
 273         pud_ref = pud_offset(pgd_ref, address);
 274         if (pud_none(*pud_ref))
 275                 return -1;
 276         if (pud_none(*pud) || pud_page(*pud) != pud_page(*pud_ref))
 277                 BUG();
 278         pmd = pmd_offset(pud, address);
 279         pmd_ref = pmd_offset(pud_ref, address);
 280         if (pmd_none(*pmd_ref))
 281                 return -1;
 282         if (pmd_none(*pmd) || pmd_page(*pmd) != pmd_page(*pmd_ref))
 283                 BUG();
 284         pte_ref = pte_offset_kernel(pmd_ref, address);
 285         if (!pte_present(*pte_ref))
 286                 return -1;
 287         pte = pte_offset_kernel(pmd, address);
 288         /* Don't use pte_page here, because the mappings can point
 289            outside mem_map, and the NUMA hash lookup cannot handle
 290            that. */
 291         if (!pte_present(*pte) || pte_pfn(*pte) != pte_pfn(*pte_ref))
 292                 BUG();
 293         return 0;
 294 }
 295
 296 int page_fault_trace = 0;
 297 int exception_trace = 1;
 298
 299 /*
 300  * This routine handles page faults.  It determines the address,
 301  * and the problem, and then passes it off to one of the appropriate
 302  * routines.
 303  */
 304 asmlinkage void __kprobes do_page_fault(struct pt_regs *regs,
 305                                         unsigned long error_code)
 306 {
 307         struct task_struct *tsk;
 308         struct mm_struct *mm;
 309         struct vm_area_struct * vma;
 310         unsigned long address;
 311         const struct exception_table_entry *fixup;
 312         int write;
 313         unsigned long flags;
 314         siginfo_t info;
 315
 316         /* get the address */
 317         __asm__("movq %%cr2,%0":"=r" (address));
 318         if (notify_die(DIE_PAGE_FAULT, "page fault", regs, error_code, 14,
 319                                         SIGSEGV) == NOTIFY_STOP)
 320                 return;
 321
 322         if (likely(regs->eflags & X86_EFLAGS_IF))
 323                 local_irq_enable();
 324
 325         if (unlikely(page_fault_trace))
 326                 printk("pagefault rip:%lx rsp:%lx cs:%lu ss:%lu address %lx error %lx\n",
 327                        regs->rip,regs->rsp,regs->cs,regs->ss,address,error_code);
 328
 329         tsk = current;
 330         mm = tsk->mm;
 331         info.si_code = SEGV_MAPERR;
 332
 333
 334         /*
 335          * We fault-in kernel-space virtual memory on-demand. The
 336          * 'reference' page table is init_mm.pgd.
 337          *
 338          * NOTE! We MUST NOT take any locks for this case. We may
 339          * be in an interrupt or a critical region, and should
 340          * only copy the information from the master page table,
 341          * nothing more.
 342          *
 343          * This verifies that the fault happens in kernel space
 344          * (error_code & 4) == 0, and that the fault was not a
 345          * protection error (error_code & 9) == 0.
 346          */
 347         if (unlikely(address >= TASK_SIZE64)) {
 348                 /*
 349                  * Don't check for the module range here: its PML4
 350                  * is always initialized because it's shared with the main
 351                  * kernel text. Only vmalloc may need PML4 syncups.
 352                  */
 353                 if (!(error_code & (PF_RSVD|PF_USER|PF_PROT)) &&
 354                       ((address >= VMALLOC_START && address < VMALLOC_END))) {
 355                         if (vmalloc_fault(address) < 0)
 356                                 goto bad_area_nosemaphore;
 357                         return;
 358                 }
 359                 /*
 360                  * Don't take the mm semaphore here. If we fixup a prefetch
 361                  * fault we could otherwise deadlock.
 362                  */
 363                 goto bad_area_nosemaphore;
 364         }
 365
 366         if (unlikely(error_code & PF_RSVD))
 367                 pgtable_bad(address, regs, error_code);
 368
 369         /*
 370          * If we're in an interrupt or have no user
 371          * context, we must not take the fault..
 372          */
 373         if (unlikely(in_atomic() || !mm))
 374                 goto bad_area_nosemaphore;
 375
 376  again:
 377         /* When running in the kernel we expect faults to occur only to
 378          * addresses in user space.  All other faults represent errors in the
 379          * kernel and should generate an OOPS.  Unfortunatly, in the case of an
 380          * erroneous fault occuring in a code path which already holds mmap_sem
 381          * we will deadlock attempting to validate the fault against the
 382          * address space.  Luckily the kernel only validly references user
 383          * space from well defined areas of code, which are listed in the
 384          * exceptions table.
 385          *
 386          * As the vast majority of faults will be valid we will only perform
 387          * the source reference check when there is a possibilty of a deadlock.
 388          * Attempt to lock the address space, if we cannot we then validate the
 389          * source.  If this is invalid we can skip the address space check,
 390          * thus avoiding the deadlock.
 391          */
 392         if (!down_read_trylock(&mm->mmap_sem)) {
 393                 if ((error_code & PF_USER) == 0 &&
 394                     !search_exception_tables(regs->rip))
 395                         goto bad_area_nosemaphore;
 396                 down_read(&mm->mmap_sem);
 397         }
 398
 399         vma = find_vma(mm, address);
 400         if (!vma)
 401                 goto bad_area;
 402         if (likely(vma->vm_start <= address))
 403                 goto good_area;
 404         if (!(vma->vm_flags & VM_GROWSDOWN))
 405                 goto bad_area;
 406         if (error_code & 4) {
 407                 // XXX: align red zone size with ABI
 408                 if (address + 128 < regs->rsp)
 409                         goto bad_area;
 410         }
 411         if (expand_stack(vma, address))
 412                 goto bad_area;
 413 /*
 414  * Ok, we have a good vm_area for this memory access, so
 415  * we can handle it..
 416  */
 417 good_area:
 418         info.si_code = SEGV_ACCERR;
 419         write = 0;
 420         switch (error_code & (PF_PROT|PF_WRITE)) {
 421                 default:        /* 3: write, present */
 422                         /* fall through */
 423                 case PF_WRITE:          /* write, not present */
 424                         if (!(vma->vm_flags & VM_WRITE))
 425                                 goto bad_area;
 426                         write++;
 427                         break;
 428                 case PF_PROT:           /* read, present */
 429                         goto bad_area;
 430                 case 0:                 /* read, not present */
 431                         if (!(vma->vm_flags & (VM_READ | VM_EXEC)))
 432                                 goto bad_area;
 433         }
 434
 435         /*
 436          * If for any reason at all we couldn't handle the fault,
 437          * make sure we exit gracefully rather than endlessly redo
 438          * the fault.
 439          */
 440         switch (handle_mm_fault(mm, vma, address, write)) {
 441         case VM_FAULT_MINOR:
 442                 tsk->min_flt++;
 443                 break;
 444         case VM_FAULT_MAJOR:
 445                 tsk->maj_flt++;
 446                 break;
 447         case VM_FAULT_SIGBUS:
 448                 goto do_sigbus;
 449         default:
 450                 goto out_of_memory;
 451         }
 452
 453         up_read(&mm->mmap_sem);
 454         return;
 455
 456 /*
 457  * Something tried to access memory that isn't in our memory map..
 458  * Fix it, but check if it's kernel or user first..
 459  */
 460 bad_area:
 461         up_read(&mm->mmap_sem);
 462
 463 bad_area_nosemaphore:
 464         /* User mode accesses just cause a SIGSEGV */
 465         if (error_code & PF_USER) {
 466                 if (is_prefetch(regs, address, error_code))
 467                         return;
 468
 469                 /* Work around K8 erratum #100 K8 in compat mode
 470                    occasionally jumps to illegal addresses >4GB.  We
 471                    catch this here in the page fault handler because
 472                    these addresses are not reachable. Just detect this
 473                    case and return.  Any code segment in LDT is
 474                    compatibility mode. */
 475                 if ((regs->cs == __USER32_CS || (regs->cs & (1<<2))) &&
 476                     (address >> 32))
 477                         return;
 478
 479                 if (exception_trace && unhandled_signal(tsk, SIGSEGV)) {
 480                         printk(
 481                        "%s%s[%d]: segfault at %016lx rip %016lx rsp %016lx error %lx\n",
 482                                         tsk->pid > 1 ? KERN_INFO : KERN_EMERG,
 483                                         tsk->comm, tsk->pid, address, regs->rip,
 484                                         regs->rsp, error_code);
 485                 }
 486
 487                 tsk->thread.cr2 = address;
 488                 /* Kernel addresses are always protection faults */
 489                 tsk->thread.error_code = error_code | (address >= TASK_SIZE);
 490                 tsk->thread.trap_no = 14;
 491                 info.si_signo = SIGSEGV;
 492                 info.si_errno = 0;
 493                 /* info.si_code has been set above */
 494                 info.si_addr = (void __user *)address;
 495                 force_sig_info(SIGSEGV, &info, tsk);
 496                 return;
 497         }
 498
 499 no_context:
 500
 501         /* Are we prepared to handle this kernel fault?  */
 502         fixup = search_exception_tables(regs->rip);
 503         if (fixup) {
 504                 regs->rip = fixup->fixup;
 505                 return;
 506         }
 507
 508         /*
 509          * Hall of shame of CPU/BIOS bugs.
 510          */
 511
 512         if (is_prefetch(regs, address, error_code))
 513                 return;
 514
 515         if (is_errata93(regs, address))
 516                 return;
 517
 518 /*
 519  * Oops. The kernel tried to access some bad page. We'll have to
 520  * terminate things with extreme prejudice.
 521  */
 522
 523         flags = oops_begin();
 524
 525         if (address < PAGE_SIZE)
 526                 printk(KERN_ALERT "Unable to handle kernel NULL pointer dereference");
 527         else
 528                 printk(KERN_ALERT "Unable to handle kernel paging request");
 529         printk(" at %016lx RIP: \n" KERN_ALERT,address);
 530         printk_address(regs->rip);
 531         printk("\n");
 532         dump_pagetable(address);
 533         tsk->thread.cr2 = address;
 534         tsk->thread.trap_no = 14;
 535         tsk->thread.error_code = error_code;
 536         __die("Oops", regs, error_code);
 537         /* Executive summary in case the body of the oops scrolled away */
 538         printk(KERN_EMERG "CR2: %016lx\n", address);
 539         oops_end(flags);
 540         do_exit(SIGKILL);
 541
 542 /*
 543  * We ran out of memory, or some other thing happened to us that made
 544  * us unable to handle the page fault gracefully.
 545  */
 546 out_of_memory:
 547         up_read(&mm->mmap_sem);
 548         if (current->pid == 1) {
 549                 yield();
 550                 goto again;
 551         }
 552         printk("VM: killing process %s\n", tsk->comm);
 553         if (error_code & 4)
 554                 do_exit(SIGKILL);
 555         goto no_context;
 556
 557 do_sigbus:
 558         up_read(&mm->mmap_sem);
 559
 560         /* Kernel mode? Handle exceptions or die */
 561         if (!(error_code & PF_USER))
 562                 goto no_context;
 563
 564         tsk->thread.cr2 = address;
 565         tsk->thread.error_code = error_code;
 566         tsk->thread.trap_no = 14;
 567         info.si_signo = SIGBUS;
 568         info.si_errno = 0;
 569         info.si_code = BUS_ADRERR;
 570         info.si_addr = (void __user *)address;
 571         force_sig_info(SIGBUS, &info, tsk);
 572         return;
 573 }
 574
 575 static int __init enable_pagefaulttrace(char *str)
 576 {
 577         page_fault_trace = 1;
 578         return 0;
 579 }
 580 __setup("pagefaulttrace", enable_pagefaulttrace);