2 * linux/arch/x86-64/mm/fault.c
4 * Copyright (C) 1995 Linus Torvalds
5 * Copyright (C) 2001,2002 Andi Kleen, SuSE Labs.
8 #include <linux/config.h>
9 #include <linux/signal.h>
10 #include <linux/sched.h>
11 #include <linux/kernel.h>
12 #include <linux/errno.h>
13 #include <linux/string.h>
14 #include <linux/types.h>
15 #include <linux/ptrace.h>
16 #include <linux/mman.h>
18 #include <linux/smp.h>
19 #include <linux/smp_lock.h>
20 #include <linux/interrupt.h>
21 #include <linux/init.h>
22 #include <linux/tty.h>
23 #include <linux/vt_kern.h> /* For unblank_screen() */
24 #include <linux/compiler.h>
25 #include <linux/module.h>
27 #include <asm/system.h>
28 #include <asm/uaccess.h>
29 #include <asm/pgalloc.h>
31 #include <asm/tlbflush.h>
32 #include <asm/proto.h>
33 #include <asm/kdebug.h>
34 #include <asm-generic/sections.h>
35 #include <asm/kdebug.h>
37 void bust_spinlocks(int yes)
39 int loglevel_save = console_loglevel;
48 * OK, the message is on the console. Now we call printk()
49 * without oops_in_progress set so that printk will give klogd
50 * a poke. Hold onto your hats...
52 console_loglevel = 15; /* NMI oopser may have shut the console up */
54 console_loglevel = loglevel_save;
58 /* Sometimes the CPU reports invalid exceptions on prefetch.
59 Check that here and ignore.
60 Opcode checker based on code by Richard Brunner */
61 static noinline int is_prefetch(struct pt_regs *regs, unsigned long addr,
62 unsigned long error_code)
67 unsigned char *max_instr;
69 /* If it was a exec fault ignore */
70 if (error_code & (1<<4))
73 instr = (unsigned char *)convert_rip_to_linear(current, regs);
74 max_instr = instr + 15;
76 if (user_mode(regs) && instr >= (unsigned char *)TASK_SIZE)
79 while (scan_more && instr < max_instr) {
81 unsigned char instr_hi;
82 unsigned char instr_lo;
84 if (__get_user(opcode, instr))
87 instr_hi = opcode & 0xf0;
88 instr_lo = opcode & 0x0f;
94 /* Values 0x26,0x2E,0x36,0x3E are valid x86
95 prefixes. In long mode, the CPU will signal
96 invalid opcode if some of these prefixes are
97 present so we will never get here anyway */
98 scan_more = ((instr_lo & 7) == 0x6);
102 /* In AMD64 long mode, 0x40 to 0x4F are valid REX prefixes
103 Need to figure out under what instruction mode the
104 instruction was issued ... */
105 /* Could check the LDT for lm, but for now it's good
106 enough to assume that long mode only uses well known
107 segments or kernel. */
108 scan_more = (!user_mode(regs)) || (regs->cs == __USER_CS);
112 /* 0x64 thru 0x67 are valid prefixes in all modes. */
113 scan_more = (instr_lo & 0xC) == 0x4;
116 /* 0xF0, 0xF2, and 0xF3 are valid prefixes in all modes. */
117 scan_more = !instr_lo || (instr_lo>>1) == 1;
120 /* Prefetch instruction is 0x0F0D or 0x0F18 */
122 if (__get_user(opcode, instr))
124 prefetch = (instr_lo == 0xF) &&
125 (opcode == 0x0D || opcode == 0x18);
135 static int bad_address(void *p)
138 return __get_user(dummy, (unsigned long *)p);
141 void dump_pagetable(unsigned long address)
148 asm("movq %%cr3,%0" : "=r" (pgd));
150 pgd = __va((unsigned long)pgd & PHYSICAL_PAGE_MASK);
151 pgd += pgd_index(address);
152 printk("PGD %lx ", pgd_val(*pgd));
153 if (bad_address(pgd)) goto bad;
154 if (!pgd_present(*pgd)) goto ret;
156 pud = __pud_offset_k((pud_t *)pgd_page(*pgd), address);
157 if (bad_address(pud)) goto bad;
158 printk("PUD %lx ", pud_val(*pud));
159 if (!pud_present(*pud)) goto ret;
161 pmd = pmd_offset(pud, address);
162 if (bad_address(pmd)) goto bad;
163 printk("PMD %lx ", pmd_val(*pmd));
164 if (!pmd_present(*pmd)) goto ret;
166 pte = pte_offset_kernel(pmd, address);
167 if (bad_address(pte)) goto bad;
168 printk("PTE %lx", pte_val(*pte));
176 static const char errata93_warning[] =
177 KERN_ERR "******* Your BIOS seems to not contain a fix for K8 errata #93\n"
178 KERN_ERR "******* Working around it, but it may cause SEGVs or burn power.\n"
179 KERN_ERR "******* Please consider a BIOS update.\n"
180 KERN_ERR "******* Disabling USB legacy in the BIOS may also help.\n";
182 /* Workaround for K8 erratum #93 & buggy BIOS.
183 BIOS SMM functions are required to use a specific workaround
184 to avoid corruption of the 64bit RIP register on C stepping K8.
185 A lot of BIOS that didn't get tested properly miss this.
186 The OS sees this as a page fault with the upper 32bits of RIP cleared.
187 Try to work around it here.
188 Note we only handle faults in kernel here. */
190 static int is_errata93(struct pt_regs *regs, unsigned long address)
193 if (address != regs->rip)
195 if ((address >> 32) != 0)
197 address |= 0xffffffffUL << 32;
198 if ((address >= (u64)_stext && address <= (u64)_etext) ||
199 (address >= MODULES_VADDR && address <= MODULES_END)) {
201 printk(errata93_warning);
210 int unhandled_signal(struct task_struct *tsk, int sig)
214 /* Warn for strace, but not for gdb */
215 if (!test_ti_thread_flag(tsk->thread_info, TIF_SYSCALL_TRACE) &&
216 (tsk->ptrace & PT_PTRACED))
218 return (tsk->sighand->action[sig-1].sa.sa_handler == SIG_IGN) ||
219 (tsk->sighand->action[sig-1].sa.sa_handler == SIG_DFL);
222 static noinline void pgtable_bad(unsigned long address, struct pt_regs *regs,
223 unsigned long error_code)
226 printk(KERN_ALERT "%s: Corrupted page table at address %lx\n",
227 current->comm, address);
228 dump_pagetable(address);
229 __die("Bad pagetable", regs, error_code);
235 * Handle a fault on the vmalloc or module mapping area
237 * This assumes no large pages in there.
239 static int vmalloc_fault(unsigned long address)
241 pgd_t *pgd, *pgd_ref;
242 pud_t *pud, *pud_ref;
243 pmd_t *pmd, *pmd_ref;
244 pte_t *pte, *pte_ref;
246 /* Copy kernel mappings over when needed. This can also
247 happen within a race in page table update. In the later
250 pgd = pgd_offset(current->mm ?: &init_mm, address);
251 pgd_ref = pgd_offset_k(address);
252 if (pgd_none(*pgd_ref))
255 set_pgd(pgd, *pgd_ref);
257 /* Below here mismatches are bugs because these lower tables
260 pud = pud_offset(pgd, address);
261 pud_ref = pud_offset(pgd_ref, address);
262 if (pud_none(*pud_ref))
264 if (pud_none(*pud) || pud_page(*pud) != pud_page(*pud_ref))
266 pmd = pmd_offset(pud, address);
267 pmd_ref = pmd_offset(pud_ref, address);
268 if (pmd_none(*pmd_ref))
270 if (pmd_none(*pmd) || pmd_page(*pmd) != pmd_page(*pmd_ref))
272 pte_ref = pte_offset_kernel(pmd_ref, address);
273 if (!pte_present(*pte_ref))
275 pte = pte_offset_kernel(pmd, address);
276 /* Don't use pte_page here, because the mappings can point
277 outside mem_map, and the NUMA hash lookup cannot handle
279 if (!pte_present(*pte) || pte_pfn(*pte) != pte_pfn(*pte_ref))
285 int page_fault_trace = 0;
286 int exception_trace = 1;
289 * This routine handles page faults. It determines the address,
290 * and the problem, and then passes it off to one of the appropriate
294 * bit 0 == 0 means no page found, 1 means protection fault
295 * bit 1 == 0 means read, 1 means write
296 * bit 2 == 0 means kernel, 1 means user-mode
297 * bit 3 == 1 means fault was an instruction fetch
299 asmlinkage void do_page_fault(struct pt_regs *regs, unsigned long error_code)
301 struct task_struct *tsk;
302 struct mm_struct *mm;
303 struct vm_area_struct * vma;
304 unsigned long address;
305 const struct exception_table_entry *fixup;
309 #ifdef CONFIG_CHECKING
312 struct x8664_pda *pda = cpu_pda + stack_smp_processor_id();
313 rdmsrl(MSR_GS_BASE, gs);
314 if (gs != (unsigned long)pda) {
315 wrmsrl(MSR_GS_BASE, pda);
316 printk("page_fault: wrong gs %lx expected %p\n", gs, pda);
321 /* get the address */
322 __asm__("movq %%cr2,%0":"=r" (address));
323 if (notify_die(DIE_PAGE_FAULT, "page fault", regs, error_code, 14,
324 SIGSEGV) == NOTIFY_STOP)
327 if (likely(regs->eflags & X86_EFLAGS_IF))
330 if (unlikely(page_fault_trace))
331 printk("pagefault rip:%lx rsp:%lx cs:%lu ss:%lu address %lx error %lx\n",
332 regs->rip,regs->rsp,regs->cs,regs->ss,address,error_code);
336 info.si_code = SEGV_MAPERR;
340 * We fault-in kernel-space virtual memory on-demand. The
341 * 'reference' page table is init_mm.pgd.
343 * NOTE! We MUST NOT take any locks for this case. We may
344 * be in an interrupt or a critical region, and should
345 * only copy the information from the master page table,
348 * This verifies that the fault happens in kernel space
349 * (error_code & 4) == 0, and that the fault was not a
350 * protection error (error_code & 1) == 0.
352 if (unlikely(address >= TASK_SIZE64)) {
353 if (!(error_code & 5) &&
354 ((address >= VMALLOC_START && address < VMALLOC_END) ||
355 (address >= MODULES_VADDR && address < MODULES_END))) {
356 if (vmalloc_fault(address) < 0)
357 goto bad_area_nosemaphore;
361 * Don't take the mm semaphore here. If we fixup a prefetch
362 * fault we could otherwise deadlock.
364 goto bad_area_nosemaphore;
367 if (unlikely(error_code & (1 << 3)))
368 pgtable_bad(address, regs, error_code);
371 * If we're in an interrupt or have no user
372 * context, we must not take the fault..
374 if (unlikely(in_atomic() || !mm))
375 goto bad_area_nosemaphore;
378 /* When running in the kernel we expect faults to occur only to
379 * addresses in user space. All other faults represent errors in the
380 * kernel and should generate an OOPS. Unfortunatly, in the case of an
381 * erroneous fault occuring in a code path which already holds mmap_sem
382 * we will deadlock attempting to validate the fault against the
383 * address space. Luckily the kernel only validly references user
384 * space from well defined areas of code, which are listed in the
387 * As the vast majority of faults will be valid we will only perform
388 * the source reference check when there is a possibilty of a deadlock.
389 * Attempt to lock the address space, if we cannot we then validate the
390 * source. If this is invalid we can skip the address space check,
391 * thus avoiding the deadlock.
393 if (!down_read_trylock(&mm->mmap_sem)) {
394 if ((error_code & 4) == 0 &&
395 !search_exception_tables(regs->rip))
396 goto bad_area_nosemaphore;
397 down_read(&mm->mmap_sem);
400 vma = find_vma(mm, address);
403 if (likely(vma->vm_start <= address))
405 if (!(vma->vm_flags & VM_GROWSDOWN))
407 if (error_code & 4) {
408 // XXX: align red zone size with ABI
409 if (address + 128 < regs->rsp)
412 if (expand_stack(vma, address))
415 * Ok, we have a good vm_area for this memory access, so
419 info.si_code = SEGV_ACCERR;
421 switch (error_code & 3) {
422 default: /* 3: write, present */
424 case 2: /* write, not present */
425 if (!(vma->vm_flags & VM_WRITE))
429 case 1: /* read, present */
431 case 0: /* read, not present */
432 if (!(vma->vm_flags & (VM_READ | VM_EXEC)))
437 * If for any reason at all we couldn't handle the fault,
438 * make sure we exit gracefully rather than endlessly redo
441 switch (handle_mm_fault(mm, vma, address, write)) {
448 case VM_FAULT_SIGBUS:
454 up_read(&mm->mmap_sem);
458 * Something tried to access memory that isn't in our memory map..
459 * Fix it, but check if it's kernel or user first..
462 up_read(&mm->mmap_sem);
464 bad_area_nosemaphore:
465 /* User mode accesses just cause a SIGSEGV */
466 if (error_code & 4) {
467 if (is_prefetch(regs, address, error_code))
470 /* Work around K8 erratum #100 K8 in compat mode
471 occasionally jumps to illegal addresses >4GB. We
472 catch this here in the page fault handler because
473 these addresses are not reachable. Just detect this
474 case and return. Any code segment in LDT is
475 compatibility mode. */
476 if ((regs->cs == __USER32_CS || (regs->cs & (1<<2))) &&
480 if (exception_trace && unhandled_signal(tsk, SIGSEGV)) {
482 "%s%s[%d]: segfault at %016lx rip %016lx rsp %016lx error %lx\n",
483 tsk->pid > 1 ? KERN_INFO : KERN_EMERG,
484 tsk->comm, tsk->pid, address, regs->rip,
485 regs->rsp, error_code);
488 tsk->thread.cr2 = address;
489 /* Kernel addresses are always protection faults */
490 tsk->thread.error_code = error_code | (address >= TASK_SIZE);
491 tsk->thread.trap_no = 14;
492 info.si_signo = SIGSEGV;
494 /* info.si_code has been set above */
495 info.si_addr = (void __user *)address;
496 force_sig_info(SIGSEGV, &info, tsk);
502 /* Are we prepared to handle this kernel fault? */
503 fixup = search_exception_tables(regs->rip);
505 regs->rip = fixup->fixup;
510 * Hall of shame of CPU/BIOS bugs.
513 if (is_prefetch(regs, address, error_code))
516 if (is_errata93(regs, address))
520 * Oops. The kernel tried to access some bad page. We'll have to
521 * terminate things with extreme prejudice.
526 if (address < PAGE_SIZE)
527 printk(KERN_ALERT "Unable to handle kernel NULL pointer dereference");
529 printk(KERN_ALERT "Unable to handle kernel paging request");
530 printk(" at %016lx RIP: \n" KERN_ALERT,address);
531 printk_address(regs->rip);
533 dump_pagetable(address);
534 __die("Oops", regs, error_code);
535 /* Executive summary in case the body of the oops scrolled away */
536 printk(KERN_EMERG "CR2: %016lx\n", address);
541 * We ran out of memory, or some other thing happened to us that made
542 * us unable to handle the page fault gracefully.
545 up_read(&mm->mmap_sem);
546 if (current->pid == 1) {
550 printk("VM: killing process %s\n", tsk->comm);
556 up_read(&mm->mmap_sem);
558 /* Kernel mode? Handle exceptions or die */
559 if (!(error_code & 4))
562 tsk->thread.cr2 = address;
563 tsk->thread.error_code = error_code;
564 tsk->thread.trap_no = 14;
565 info.si_signo = SIGBUS;
567 info.si_code = BUS_ADRERR;
568 info.si_addr = (void __user *)address;
569 force_sig_info(SIGBUS, &info, tsk);