arch/x86/kernel/relocate_kernel_64.S

   1 /*
   2  * relocate_kernel.S - put the kernel image in place to boot
   3  * Copyright (C) 2002-2005 Eric Biederman  <ebiederm@xmission.com>
   4  *
   5  * This source code is licensed under the GNU General Public License,
   6  * Version 2.  See the file COPYING for more details.
   7  */
   8
   9 #include <linux/linkage.h>
  10 #include <asm/page.h>
  11 #include <asm/kexec.h>
  12
  13 /*
  14  * Must be relocatable PIC code callable as a C function
  15  */
  16
  17 #define PTR(x) (x << 3)
  18 #define PAGE_ALIGNED (1 << PAGE_SHIFT)
  19 #define PAGE_ATTR 0x63 /* _PAGE_PRESENT|_PAGE_RW|_PAGE_ACCESSED|_PAGE_DIRTY */
  20
  21         .text
  22         .align PAGE_ALIGNED
  23         .code64
  24         .globl relocate_kernel
  25 relocate_kernel:
  26         /* %rdi indirection_page
  27          * %rsi page_list
  28          * %rdx start address
  29          */
  30
  31         /* map the control page at its virtual address */
  32
  33         movq    $0x0000ff8000000000, %r10        /* mask */
  34         mov     $(39 - 3), %cl                   /* bits to shift */
  35         movq    PTR(VA_CONTROL_PAGE)(%rsi), %r11 /* address to map */
  36
  37         movq    %r11, %r9
  38         andq    %r10, %r9
  39         shrq    %cl, %r9
  40
  41         movq    PTR(VA_PGD)(%rsi), %r8
  42         addq    %r8, %r9
  43         movq    PTR(PA_PUD_0)(%rsi), %r8
  44         orq     $PAGE_ATTR, %r8
  45         movq    %r8, (%r9)
  46
  47         shrq    $9, %r10
  48         sub     $9, %cl
  49
  50         movq    %r11, %r9
  51         andq    %r10, %r9
  52         shrq    %cl, %r9
  53
  54         movq    PTR(VA_PUD_0)(%rsi), %r8
  55         addq    %r8, %r9
  56         movq    PTR(PA_PMD_0)(%rsi), %r8
  57         orq     $PAGE_ATTR, %r8
  58         movq    %r8, (%r9)
  59
  60         shrq    $9, %r10
  61         sub     $9, %cl
  62
  63         movq    %r11, %r9
  64         andq    %r10, %r9
  65         shrq    %cl, %r9
  66
  67         movq    PTR(VA_PMD_0)(%rsi), %r8
  68         addq    %r8, %r9
  69         movq    PTR(PA_PTE_0)(%rsi), %r8
  70         orq     $PAGE_ATTR, %r8
  71         movq    %r8, (%r9)
  72
  73         shrq    $9, %r10
  74         sub     $9, %cl
  75
  76         movq    %r11, %r9
  77         andq    %r10, %r9
  78         shrq    %cl, %r9
  79
  80         movq    PTR(VA_PTE_0)(%rsi), %r8
  81         addq    %r8, %r9
  82         movq    PTR(PA_CONTROL_PAGE)(%rsi), %r8
  83         orq     $PAGE_ATTR, %r8
  84         movq    %r8, (%r9)
  85
  86         /* identity map the control page at its physical address */
  87
  88         movq    $0x0000ff8000000000, %r10        /* mask */
  89         mov     $(39 - 3), %cl                   /* bits to shift */
  90         movq    PTR(PA_CONTROL_PAGE)(%rsi), %r11 /* address to map */
  91
  92         movq    %r11, %r9
  93         andq    %r10, %r9
  94         shrq    %cl, %r9
  95
  96         movq    PTR(VA_PGD)(%rsi), %r8
  97         addq    %r8, %r9
  98         movq    PTR(PA_PUD_1)(%rsi), %r8
  99         orq     $PAGE_ATTR, %r8
 100         movq    %r8, (%r9)
 101
 102         shrq    $9, %r10
 103         sub     $9, %cl
 104
 105         movq    %r11, %r9
 106         andq    %r10, %r9
 107         shrq    %cl, %r9
 108
 109         movq    PTR(VA_PUD_1)(%rsi), %r8
 110         addq    %r8, %r9
 111         movq    PTR(PA_PMD_1)(%rsi), %r8
 112         orq     $PAGE_ATTR, %r8
 113         movq    %r8, (%r9)
 114
 115         shrq    $9, %r10
 116         sub     $9, %cl
 117
 118         movq    %r11, %r9
 119         andq    %r10, %r9
 120         shrq    %cl, %r9
 121
 122         movq    PTR(VA_PMD_1)(%rsi), %r8
 123         addq    %r8, %r9
 124         movq    PTR(PA_PTE_1)(%rsi), %r8
 125         orq     $PAGE_ATTR, %r8
 126         movq    %r8, (%r9)
 127
 128         shrq    $9, %r10
 129         sub     $9, %cl
 130
 131         movq    %r11, %r9
 132         andq    %r10, %r9
 133         shrq    %cl, %r9
 134
 135         movq    PTR(VA_PTE_1)(%rsi), %r8
 136         addq    %r8, %r9
 137         movq    PTR(PA_CONTROL_PAGE)(%rsi), %r8
 138         orq     $PAGE_ATTR, %r8
 139         movq    %r8, (%r9)
 140
 141 relocate_new_kernel:
 142         /* %rdi indirection_page
 143          * %rsi page_list
 144          * %rdx start address
 145          */
 146
 147         /* zero out flags, and disable interrupts */
 148         pushq $0
 149         popfq
 150
 151         /* get physical address of control page now */
 152         /* this is impossible after page table switch */
 153         movq    PTR(PA_CONTROL_PAGE)(%rsi), %r8
 154
 155         /* get physical address of page table now too */
 156         movq    PTR(PA_TABLE_PAGE)(%rsi), %rcx
 157
 158         /* switch to new set of page tables */
 159         movq    PTR(PA_PGD)(%rsi), %r9
 160         movq    %r9, %cr3
 161
 162         /* setup a new stack at the end of the physical control page */
 163         lea     4096(%r8), %rsp
 164
 165         /* jump to identity mapped page */
 166         addq    $(identity_mapped - relocate_kernel), %r8
 167         pushq   %r8
 168         ret
 169
 170 identity_mapped:
 171         /* store the start address on the stack */
 172         pushq   %rdx
 173
 174         /* Set cr0 to a known state:
 175          * 31 1 == Paging enabled
 176          * 18 0 == Alignment check disabled
 177          * 16 0 == Write protect disabled
 178          * 3  0 == No task switch
 179          * 2  0 == Don't do FP software emulation.
 180          * 0  1 == Proctected mode enabled
 181          */
 182         movq    %cr0, %rax
 183         andq    $~((1<<18)|(1<<16)|(1<<3)|(1<<2)), %rax
 184         orl     $((1<<31)|(1<<0)), %eax
 185         movq    %rax, %cr0
 186
 187         /* Set cr4 to a known state:
 188          * 10 0 == xmm exceptions disabled
 189          * 9  0 == xmm registers instructions disabled
 190          * 8  0 == performance monitoring counter disabled
 191          * 7  0 == page global disabled
 192          * 6  0 == machine check exceptions disabled
 193          * 5  1 == physical address extension enabled
 194          * 4  0 == page size extensions disabled
 195          * 3  0 == Debug extensions disabled
 196          * 2  0 == Time stamp disable (disabled)
 197          * 1  0 == Protected mode virtual interrupts disabled
 198          * 0  0 == VME disabled
 199          */
 200
 201         movq    $((1<<5)), %rax
 202         movq    %rax, %cr4
 203
 204         jmp 1f
 205 1:
 206
 207         /* Switch to the identity mapped page tables,
 208          * and flush the TLB.
 209         */
 210         movq    %rcx, %cr3
 211
 212         /* Do the copies */
 213         movq    %rdi, %rcx      /* Put the page_list in %rcx */
 214         xorq    %rdi, %rdi
 215         xorq    %rsi, %rsi
 216         jmp     1f
 217
 218 0:      /* top, read another word for the indirection page */
 219
 220         movq    (%rbx), %rcx
 221         addq    $8,     %rbx
 222 1:
 223         testq   $0x1,   %rcx  /* is it a destination page? */
 224         jz      2f
 225         movq    %rcx,   %rdi
 226         andq    $0xfffffffffffff000, %rdi
 227         jmp     0b
 228 2:
 229         testq   $0x2,   %rcx  /* is it an indirection page? */
 230         jz      2f
 231         movq    %rcx,   %rbx
 232         andq    $0xfffffffffffff000, %rbx
 233         jmp     0b
 234 2:
 235         testq   $0x4,   %rcx  /* is it the done indicator? */
 236         jz      2f
 237         jmp     3f
 238 2:
 239         testq   $0x8,   %rcx  /* is it the source indicator? */
 240         jz      0b            /* Ignore it otherwise */
 241         movq    %rcx,   %rsi  /* For ever source page do a copy */
 242         andq    $0xfffffffffffff000, %rsi
 243
 244         movq    $512,   %rcx
 245         rep ; movsq
 246         jmp     0b
 247 3:
 248
 249         /* To be certain of avoiding problems with self-modifying code
 250          * I need to execute a serializing instruction here.
 251          * So I flush the TLB by reloading %cr3 here, it's handy,
 252          * and not processor dependent.
 253          */
 254         movq    %cr3, %rax
 255         movq    %rax, %cr3
 256
 257         /* set all of the registers to known values */
 258         /* leave %rsp alone */
 259
 260         xorq    %rax, %rax
 261         xorq    %rbx, %rbx
 262         xorq    %rcx, %rcx
 263         xorq    %rdx, %rdx
 264         xorq    %rsi, %rsi
 265         xorq    %rdi, %rdi
 266         xorq    %rbp, %rbp
 267         xorq    %r8,  %r8
 268         xorq    %r9,  %r9
 269         xorq    %r10, %r9
 270         xorq    %r11, %r11
 271         xorq    %r12, %r12
 272         xorq    %r13, %r13
 273         xorq    %r14, %r14
 274         xorq    %r15, %r15
 275
 276         ret