2 * handle transition of Linux booting another kernel
3 * Copyright (C) 2002-2005 Eric Biederman <ebiederm@xmission.com>
5 * This source code is licensed under the GNU General Public License,
6 * Version 2. See the file COPYING for more details.
9 #define pr_fmt(fmt) "kexec: " fmt
12 #include <linux/kexec.h>
13 #include <linux/string.h>
14 #include <linux/gfp.h>
15 #include <linux/reboot.h>
16 #include <linux/numa.h>
17 #include <linux/ftrace.h>
19 #include <linux/suspend.h>
22 #include <asm/pgtable.h>
23 #include <asm/tlbflush.h>
24 #include <asm/mmu_context.h>
25 #include <asm/debugreg.h>
26 #include <asm/kexec-bzimage64.h>
28 static struct kexec_file_ops
*kexec_file_loaders
[] = {
32 static void free_transition_pgtable(struct kimage
*image
)
34 free_page((unsigned long)image
->arch
.pud
);
35 free_page((unsigned long)image
->arch
.pmd
);
36 free_page((unsigned long)image
->arch
.pte
);
39 static int init_transition_pgtable(struct kimage
*image
, pgd_t
*pgd
)
44 unsigned long vaddr
, paddr
;
47 vaddr
= (unsigned long)relocate_kernel
;
48 paddr
= __pa(page_address(image
->control_code_page
)+PAGE_SIZE
);
49 pgd
+= pgd_index(vaddr
);
50 if (!pgd_present(*pgd
)) {
51 pud
= (pud_t
*)get_zeroed_page(GFP_KERNEL
);
54 image
->arch
.pud
= pud
;
55 set_pgd(pgd
, __pgd(__pa(pud
) | _KERNPG_TABLE
));
57 pud
= pud_offset(pgd
, vaddr
);
58 if (!pud_present(*pud
)) {
59 pmd
= (pmd_t
*)get_zeroed_page(GFP_KERNEL
);
62 image
->arch
.pmd
= pmd
;
63 set_pud(pud
, __pud(__pa(pmd
) | _KERNPG_TABLE
));
65 pmd
= pmd_offset(pud
, vaddr
);
66 if (!pmd_present(*pmd
)) {
67 pte
= (pte_t
*)get_zeroed_page(GFP_KERNEL
);
70 image
->arch
.pte
= pte
;
71 set_pmd(pmd
, __pmd(__pa(pte
) | _KERNPG_TABLE
));
73 pte
= pte_offset_kernel(pmd
, vaddr
);
74 set_pte(pte
, pfn_pte(paddr
>> PAGE_SHIFT
, PAGE_KERNEL_EXEC
));
77 free_transition_pgtable(image
);
81 static void *alloc_pgt_page(void *data
)
83 struct kimage
*image
= (struct kimage
*)data
;
87 page
= kimage_alloc_control_pages(image
, 0);
89 p
= page_address(page
);
96 static int init_pgtable(struct kimage
*image
, unsigned long start_pgtable
)
98 struct x86_mapping_info info
= {
99 .alloc_pgt_page
= alloc_pgt_page
,
101 .pmd_flag
= __PAGE_KERNEL_LARGE_EXEC
,
103 unsigned long mstart
, mend
;
108 level4p
= (pgd_t
*)__va(start_pgtable
);
110 for (i
= 0; i
< nr_pfn_mapped
; i
++) {
111 mstart
= pfn_mapped
[i
].start
<< PAGE_SHIFT
;
112 mend
= pfn_mapped
[i
].end
<< PAGE_SHIFT
;
114 result
= kernel_ident_mapping_init(&info
,
115 level4p
, mstart
, mend
);
121 * segments's mem ranges could be outside 0 ~ max_pfn,
122 * for example when jump back to original kernel from kexeced kernel.
123 * or first kernel is booted with user mem map, and second kernel
124 * could be loaded out of that range.
126 for (i
= 0; i
< image
->nr_segments
; i
++) {
127 mstart
= image
->segment
[i
].mem
;
128 mend
= mstart
+ image
->segment
[i
].memsz
;
130 result
= kernel_ident_mapping_init(&info
,
131 level4p
, mstart
, mend
);
137 return init_transition_pgtable(image
, level4p
);
140 static void set_idt(void *newidt
, u16 limit
)
142 struct desc_ptr curidt
;
144 /* x86-64 supports unaliged loads & stores */
146 curidt
.address
= (unsigned long)newidt
;
148 __asm__
__volatile__ (
155 static void set_gdt(void *newgdt
, u16 limit
)
157 struct desc_ptr curgdt
;
159 /* x86-64 supports unaligned loads & stores */
161 curgdt
.address
= (unsigned long)newgdt
;
163 __asm__
__volatile__ (
169 static void load_segments(void)
171 __asm__
__volatile__ (
177 : : "a" (__KERNEL_DS
) : "memory"
181 int machine_kexec_prepare(struct kimage
*image
)
183 unsigned long start_pgtable
;
186 /* Calculate the offsets */
187 start_pgtable
= page_to_pfn(image
->control_code_page
) << PAGE_SHIFT
;
189 /* Setup the identity mapped 64bit page table */
190 result
= init_pgtable(image
, start_pgtable
);
197 void machine_kexec_cleanup(struct kimage
*image
)
199 free_transition_pgtable(image
);
203 * Do not allocate memory (or fail in any way) in machine_kexec().
204 * We are past the point of no return, committed to rebooting now.
206 void machine_kexec(struct kimage
*image
)
208 unsigned long page_list
[PAGES_NR
];
210 int save_ftrace_enabled
;
212 #ifdef CONFIG_KEXEC_JUMP
213 if (image
->preserve_context
)
214 save_processor_state();
217 save_ftrace_enabled
= __ftrace_enabled_save();
219 /* Interrupts aren't acceptable while we reboot */
221 hw_breakpoint_disable();
223 if (image
->preserve_context
) {
224 #ifdef CONFIG_X86_IO_APIC
226 * We need to put APICs in legacy mode so that we can
227 * get timer interrupts in second kernel. kexec/kdump
228 * paths already have calls to disable_IO_APIC() in
229 * one form or other. kexec jump path also need
236 control_page
= page_address(image
->control_code_page
) + PAGE_SIZE
;
237 memcpy(control_page
, relocate_kernel
, KEXEC_CONTROL_CODE_MAX_SIZE
);
239 page_list
[PA_CONTROL_PAGE
] = virt_to_phys(control_page
);
240 page_list
[VA_CONTROL_PAGE
] = (unsigned long)control_page
;
241 page_list
[PA_TABLE_PAGE
] =
242 (unsigned long)__pa(page_address(image
->control_code_page
));
244 if (image
->type
== KEXEC_TYPE_DEFAULT
)
245 page_list
[PA_SWAP_PAGE
] = (page_to_pfn(image
->swap_page
)
249 * The segment registers are funny things, they have both a
250 * visible and an invisible part. Whenever the visible part is
251 * set to a specific selector, the invisible part is loaded
252 * with from a table in memory. At no other time is the
253 * descriptor table in memory accessed.
255 * I take advantage of this here by force loading the
256 * segments, before I zap the gdt with an invalid value.
260 * The gdt & idt are now invalid.
261 * If you want to load them you must set up your own idt & gdt.
263 set_gdt(phys_to_virt(0), 0);
264 set_idt(phys_to_virt(0), 0);
267 image
->start
= relocate_kernel((unsigned long)image
->head
,
268 (unsigned long)page_list
,
270 image
->preserve_context
);
272 #ifdef CONFIG_KEXEC_JUMP
273 if (image
->preserve_context
)
274 restore_processor_state();
277 __ftrace_enabled_restore(save_ftrace_enabled
);
280 void arch_crash_save_vmcoreinfo(void)
282 VMCOREINFO_SYMBOL(phys_base
);
283 VMCOREINFO_SYMBOL(init_level4_pgt
);
286 VMCOREINFO_SYMBOL(node_data
);
287 VMCOREINFO_LENGTH(node_data
, MAX_NUMNODES
);
289 vmcoreinfo_append_str("KERNELOFFSET=%lx\n",
290 (unsigned long)&_text
- __START_KERNEL
);
293 /* arch-dependent functionality related to kexec file-based syscall */
295 int arch_kexec_kernel_image_probe(struct kimage
*image
, void *buf
,
296 unsigned long buf_len
)
298 int i
, ret
= -ENOEXEC
;
299 struct kexec_file_ops
*fops
;
301 for (i
= 0; i
< ARRAY_SIZE(kexec_file_loaders
); i
++) {
302 fops
= kexec_file_loaders
[i
];
303 if (!fops
|| !fops
->probe
)
306 ret
= fops
->probe(buf
, buf_len
);
316 void *arch_kexec_kernel_image_load(struct kimage
*image
)
318 if (!image
->fops
|| !image
->fops
->load
)
319 return ERR_PTR(-ENOEXEC
);
321 return image
->fops
->load(image
, image
->kernel_buf
,
322 image
->kernel_buf_len
, image
->initrd_buf
,
323 image
->initrd_buf_len
, image
->cmdline_buf
,
324 image
->cmdline_buf_len
);
327 int arch_kimage_file_post_load_cleanup(struct kimage
*image
)
329 if (!image
->fops
|| !image
->fops
->cleanup
)
332 return image
->fops
->cleanup(image
->image_loader_data
);
336 * Apply purgatory relocations.
338 * ehdr: Pointer to elf headers
339 * sechdrs: Pointer to section headers.
340 * relsec: section index of SHT_RELA section.
342 * TODO: Some of the code belongs to generic code. Move that in kexec.c.
344 int arch_kexec_apply_relocations_add(const Elf64_Ehdr
*ehdr
,
345 Elf64_Shdr
*sechdrs
, unsigned int relsec
)
351 Elf64_Shdr
*section
, *symtabsec
;
352 unsigned long address
, sec_base
, value
;
353 const char *strtab
, *name
, *shstrtab
;
356 * ->sh_offset has been modified to keep the pointer to section
359 rel
= (void *)sechdrs
[relsec
].sh_offset
;
361 /* Section to which relocations apply */
362 section
= &sechdrs
[sechdrs
[relsec
].sh_info
];
364 pr_debug("Applying relocate section %u to %u\n", relsec
,
365 sechdrs
[relsec
].sh_info
);
367 /* Associated symbol table */
368 symtabsec
= &sechdrs
[sechdrs
[relsec
].sh_link
];
371 if (symtabsec
->sh_link
>= ehdr
->e_shnum
) {
372 /* Invalid strtab section number */
373 pr_err("Invalid string table section index %d\n",
378 strtab
= (char *)sechdrs
[symtabsec
->sh_link
].sh_offset
;
380 /* section header string table */
381 shstrtab
= (char *)sechdrs
[ehdr
->e_shstrndx
].sh_offset
;
383 for (i
= 0; i
< sechdrs
[relsec
].sh_size
/ sizeof(*rel
); i
++) {
386 * rel[i].r_offset contains byte offset from beginning
387 * of section to the storage unit affected.
389 * This is location to update (->sh_offset). This is temporary
390 * buffer where section is currently loaded. This will finally
391 * be loaded to a different address later, pointed to by
392 * ->sh_addr. kexec takes care of moving it
393 * (kexec_load_segment()).
395 location
= (void *)(section
->sh_offset
+ rel
[i
].r_offset
);
397 /* Final address of the location */
398 address
= section
->sh_addr
+ rel
[i
].r_offset
;
401 * rel[i].r_info contains information about symbol table index
402 * w.r.t which relocation must be made and type of relocation
403 * to apply. ELF64_R_SYM() and ELF64_R_TYPE() macros get
404 * these respectively.
406 sym
= (Elf64_Sym
*)symtabsec
->sh_offset
+
407 ELF64_R_SYM(rel
[i
].r_info
);
410 name
= strtab
+ sym
->st_name
;
412 name
= shstrtab
+ sechdrs
[sym
->st_shndx
].sh_name
;
414 pr_debug("Symbol: %s info: %02x shndx: %02x value=%llx size: %llx\n",
415 name
, sym
->st_info
, sym
->st_shndx
, sym
->st_value
,
418 if (sym
->st_shndx
== SHN_UNDEF
) {
419 pr_err("Undefined symbol: %s\n", name
);
423 if (sym
->st_shndx
== SHN_COMMON
) {
424 pr_err("symbol '%s' in common section\n", name
);
428 if (sym
->st_shndx
== SHN_ABS
)
430 else if (sym
->st_shndx
>= ehdr
->e_shnum
) {
431 pr_err("Invalid section %d for symbol %s\n",
432 sym
->st_shndx
, name
);
435 sec_base
= sechdrs
[sym
->st_shndx
].sh_addr
;
437 value
= sym
->st_value
;
439 value
+= rel
[i
].r_addend
;
441 switch (ELF64_R_TYPE(rel
[i
].r_info
)) {
445 *(u64
*)location
= value
;
448 *(u32
*)location
= value
;
449 if (value
!= *(u32
*)location
)
453 *(s32
*)location
= value
;
454 if ((s64
)value
!= *(s32
*)location
)
458 value
-= (u64
)address
;
459 *(u32
*)location
= value
;
462 pr_err("Unknown rela relocation: %llu\n",
463 ELF64_R_TYPE(rel
[i
].r_info
));
470 pr_err("Overflow in relocation type %d value 0x%lx\n",
471 (int)ELF64_R_TYPE(rel
[i
].r_info
), value
);