x86/efi: Hoist page table switching code into efi_call_virt()

This change is a prerequisite for pending patches that switch to
a dedicated EFI page table, instead of using 'trampoline_pgd'
which shares PGD entries with 'swapper_pg_dir'. The pending
patches make it impossible to dereference the runtime service
function pointer without first switching %cr3.

It's true that we now have duplicated switching code in
efi_call_virt() and efi_call_phys_{prolog,epilog}() but we are
sacrificing code duplication for a little more clarity and the
ease of writing the page table switching code in C instead of
asm.

Signed-off-by: Matt Fleming <matt@codeblueprint.co.uk>
Reviewed-by: Borislav Petkov <bp@suse.de>
Acked-by: Borislav Petkov <bp@suse.de>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: Andy Lutomirski <luto@amacapital.net>
Cc: Andy Lutomirski <luto@kernel.org>
Cc: Ard Biesheuvel <ard.biesheuvel@linaro.org>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Brian Gerst <brgerst@gmail.com>
Cc: Dave Jones <davej@codemonkey.org.uk>
Cc: Denys Vlasenko <dvlasenk@redhat.com>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Sai Praneeth Prakhya <sai.praneeth.prakhya@intel.com>
Cc: Stephen Smalley <sds@tycho.nsa.gov>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Toshi Kani <toshi.kani@hp.com>
Cc: linux-efi@vger.kernel.org
Link: http://lkml.kernel.org/r/1448658575-17029-5-git-send-email-matt@codeblueprint.co.uk
Signed-off-by: Ingo Molnar <mingo@kernel.org>

diff --git a/arch/x86/include/asm/efi.h b/arch/x86/include/asm/efi.h
index 0010c78c4998cf0702299ea2f8a9229e09bb6438..347eeacb06a815f42f3a7a971b4370934cc6c499 100644 (file)
--- a/arch/x86/include/asm/efi.h
+++ b/arch/x86/include/asm/efi.h
@@ -3,6 +3,7 @@
 
 #include <asm/fpu/api.h>
 #include <asm/pgtable.h>
+#include <asm/tlb.h>
 
 /*
  * We map the EFI regions needed for runtime services non-contiguously,
@@ -64,6 +65,17 @@ extern u64 asmlinkage efi_call(void *fp, ...);
 
 #define efi_call_phys(f, args...)              efi_call((f), args)
 
+/*
+ * Scratch space used for switching the pagetable in the EFI stub
+ */
+struct efi_scratch {
+       u64     r15;
+       u64     prev_cr3;
+       pgd_t   *efi_pgt;
+       bool    use_pgd;
+       u64     phys_stack;
+} __packed;
+
 #define efi_call_virt(f, ...)                                          \
 ({                                                                     \
        efi_status_t __s;                                               \
@@ -71,7 +83,20 @@ extern u64 asmlinkage efi_call(void *fp, ...);
        efi_sync_low_kernel_mappings();                                 \
        preempt_disable();                                              \
        __kernel_fpu_begin();                                           \
+                                                                       \
+       if (efi_scratch.use_pgd) {                                      \
+               efi_scratch.prev_cr3 = read_cr3();                      \
+               write_cr3((unsigned long)efi_scratch.efi_pgt);          \
+               __flush_tlb_all();                                      \
+       }                                                               \
+                                                                       \
        __s = efi_call((void *)efi.systab->runtime->f, __VA_ARGS__);    \
+                                                                       \
+       if (efi_scratch.use_pgd) {                                      \
+               write_cr3(efi_scratch.prev_cr3);                        \
+               __flush_tlb_all();                                      \
+       }                                                               \
+                                                                       \
        __kernel_fpu_end();                                             \
        preempt_enable();                                               \
        __s;                                                            \

diff --git a/arch/x86/platform/efi/efi_64.c b/arch/x86/platform/efi/efi_64.c
index 102976dda8c42043c0571a0f2a6e64d3a2f18fc4..b19cdac959b2de86de8ffc40d8d58ac0e2821deb 100644 (file)
--- a/arch/x86/platform/efi/efi_64.c
+++ b/arch/x86/platform/efi/efi_64.c
@@ -47,16 +47,7 @@
  */
 static u64 efi_va = EFI_VA_START;
 
-/*
- * Scratch space used for switching the pagetable in the EFI stub
- */
-struct efi_scratch {
-       u64 r15;
-       u64 prev_cr3;
-       pgd_t *efi_pgt;
-       bool use_pgd;
-       u64 phys_stack;
-} __packed;
+struct efi_scratch efi_scratch;
 
 static void __init early_code_mapping_set_exec(int executable)
 {
@@ -83,8 +74,11 @@ pgd_t * __init efi_call_phys_prolog(void)
        int pgd;
        int n_pgds;
 
-       if (!efi_enabled(EFI_OLD_MEMMAP))
-               return NULL;
+       if (!efi_enabled(EFI_OLD_MEMMAP)) {
+               save_pgd = (pgd_t *)read_cr3();
+               write_cr3((unsigned long)efi_scratch.efi_pgt);
+               goto out;
+       }
 
        early_code_mapping_set_exec(1);
 
@@ -96,6 +90,7 @@ pgd_t * __init efi_call_phys_prolog(void)
                vaddress = (unsigned long)__va(pgd * PGDIR_SIZE);
                set_pgd(pgd_offset_k(pgd * PGDIR_SIZE), *pgd_offset_k(vaddress));
        }
+out:
        __flush_tlb_all();
 
        return save_pgd;
@@ -109,8 +104,11 @@ void __init efi_call_phys_epilog(pgd_t *save_pgd)
        int pgd_idx;
        int nr_pgds;
 
-       if (!save_pgd)
+       if (!efi_enabled(EFI_OLD_MEMMAP)) {
+               write_cr3((unsigned long)save_pgd);
+               __flush_tlb_all();
                return;
+       }
 
        nr_pgds = DIV_ROUND_UP((max_pfn << PAGE_SHIFT) , PGDIR_SIZE);
 

diff --git a/arch/x86/platform/efi/efi_stub_64.S b/arch/x86/platform/efi/efi_stub_64.S
index 86d0f9e08dd95eb1023d5ac7ec4fb006aafb72c9..32020cb8bb08ce0e12ca0fbf6428cf81344e9054 100644 (file)
--- a/arch/x86/platform/efi/efi_stub_64.S
+++ b/arch/x86/platform/efi/efi_stub_64.S
@@ -38,41 +38,6 @@
        mov %rsi, %cr0;                 \
        mov (%rsp), %rsp
 
-       /* stolen from gcc */
-       .macro FLUSH_TLB_ALL
-       movq %r15, efi_scratch(%rip)
-       movq %r14, efi_scratch+8(%rip)
-       movq %cr4, %r15
-       movq %r15, %r14
-       andb $0x7f, %r14b
-       movq %r14, %cr4
-       movq %r15, %cr4
-       movq efi_scratch+8(%rip), %r14
-       movq efi_scratch(%rip), %r15
-       .endm
-
-       .macro SWITCH_PGT
-       cmpb $0, efi_scratch+24(%rip)
-       je 1f
-       movq %r15, efi_scratch(%rip)            # r15
-       # save previous CR3
-       movq %cr3, %r15
-       movq %r15, efi_scratch+8(%rip)          # prev_cr3
-       movq efi_scratch+16(%rip), %r15         # EFI pgt
-       movq %r15, %cr3
-       1:
-       .endm
-
-       .macro RESTORE_PGT
-       cmpb $0, efi_scratch+24(%rip)
-       je 2f
-       movq efi_scratch+8(%rip), %r15
-       movq %r15, %cr3
-       movq efi_scratch(%rip), %r15
-       FLUSH_TLB_ALL
-       2:
-       .endm
-
 ENTRY(efi_call)
        SAVE_XMM
        mov (%rsp), %rax
@@ -83,16 +48,8 @@ ENTRY(efi_call)
        mov %r8, %r9
        mov %rcx, %r8
        mov %rsi, %rcx
-       SWITCH_PGT
        call *%rdi
-       RESTORE_PGT
        addq $48, %rsp
        RESTORE_XMM
        ret
 ENDPROC(efi_call)
-
-       .data
-ENTRY(efi_scratch)
-       .fill 3,8,0
-       .byte 0
-       .quad 0