arch/x86/platform/efi/efi.c

   1 /*
   2  * Common EFI (Extensible Firmware Interface) support functions
   3  * Based on Extensible Firmware Interface Specification version 1.0
   4  *
   5  * Copyright (C) 1999 VA Linux Systems
   6  * Copyright (C) 1999 Walt Drummond <drummond@valinux.com>
   7  * Copyright (C) 1999-2002 Hewlett-Packard Co.
   8  *      David Mosberger-Tang <davidm@hpl.hp.com>
   9  *      Stephane Eranian <eranian@hpl.hp.com>
  10  * Copyright (C) 2005-2008 Intel Co.
  11  *      Fenghua Yu <fenghua.yu@intel.com>
  12  *      Bibo Mao <bibo.mao@intel.com>
  13  *      Chandramouli Narayanan <mouli@linux.intel.com>
  14  *      Huang Ying <ying.huang@intel.com>
  15  * Copyright (C) 2013 SuSE Labs
  16  *      Borislav Petkov <bp@suse.de> - runtime services VA mapping
  17  *
  18  * Copied from efi_32.c to eliminate the duplicated code between EFI
  19  * 32/64 support code. --ying 2007-10-26
  20  *
  21  * All EFI Runtime Services are not implemented yet as EFI only
  22  * supports physical mode addressing on SoftSDV. This is to be fixed
  23  * in a future version.  --drummond 1999-07-20
  24  *
  25  * Implemented EFI runtime services and virtual mode calls.  --davidm
  26  *
  27  * Goutham Rao: <goutham.rao@intel.com>
  28  *      Skip non-WB memory and ignore empty memory ranges.
  29  */
  30
  31 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  32
  33 #include <linux/kernel.h>
  34 #include <linux/init.h>
  35 #include <linux/efi.h>
  36 #include <linux/efi-bgrt.h>
  37 #include <linux/export.h>
  38 #include <linux/bootmem.h>
  39 #include <linux/slab.h>
  40 #include <linux/memblock.h>
  41 #include <linux/spinlock.h>
  42 #include <linux/uaccess.h>
  43 #include <linux/time.h>
  44 #include <linux/io.h>
  45 #include <linux/reboot.h>
  46 #include <linux/bcd.h>
  47
  48 #include <asm/setup.h>
  49 #include <asm/efi.h>
  50 #include <asm/time.h>
  51 #include <asm/cacheflush.h>
  52 #include <asm/tlbflush.h>
  53 #include <asm/x86_init.h>
  54 #include <asm/rtc.h>
  55 #include <asm/uv/uv.h>
  56
  57 #define EFI_DEBUG
  58
  59 struct efi_memory_map memmap;
  60
  61 static struct efi efi_phys __initdata;
  62 static efi_system_table_t efi_systab __initdata;
  63
  64 static efi_config_table_type_t arch_tables[] __initdata = {
  65 #ifdef CONFIG_X86_UV
  66         {UV_SYSTEM_TABLE_GUID, "UVsystab", &efi.uv_systab},
  67 #endif
  68         {NULL_GUID, NULL, NULL},
  69 };
  70
  71 u64 efi_setup;          /* efi setup_data physical address */
  72
  73 int add_efi_memmap;
  74 EXPORT_SYMBOL(add_efi_memmap);
  75
  76 static int __init setup_add_efi_memmap(char *arg)
  77 {
  78         add_efi_memmap = 1;
  79         return 0;
  80 }
  81 early_param("add_efi_memmap", setup_add_efi_memmap);
  82
  83 static efi_status_t __init phys_efi_set_virtual_address_map(
  84         unsigned long memory_map_size,
  85         unsigned long descriptor_size,
  86         u32 descriptor_version,
  87         efi_memory_desc_t *virtual_map)
  88 {
  89         efi_status_t status;
  90
  91         efi_call_phys_prelog();
  92         status = efi_call_phys(efi_phys.set_virtual_address_map,
  93                                memory_map_size, descriptor_size,
  94                                descriptor_version, virtual_map);
  95         efi_call_phys_epilog();
  96         return status;
  97 }
  98
  99 void efi_get_time(struct timespec *now)
 100 {
 101         efi_status_t status;
 102         efi_time_t eft;
 103         efi_time_cap_t cap;
 104
 105         status = efi.get_time(&eft, &cap);
 106         if (status != EFI_SUCCESS)
 107                 pr_err("Oops: efitime: can't read time!\n");
 108
 109         now->tv_sec = mktime(eft.year, eft.month, eft.day, eft.hour,
 110                              eft.minute, eft.second);
 111         now->tv_nsec = 0;
 112 }
 113
 114 /*
 115  * Tell the kernel about the EFI memory map.  This might include
 116  * more than the max 128 entries that can fit in the e820 legacy
 117  * (zeropage) memory map.
 118  */
 119
 120 static void __init do_add_efi_memmap(void)
 121 {
 122         void *p;
 123
 124         for (p = memmap.map; p < memmap.map_end; p += memmap.desc_size) {
 125                 efi_memory_desc_t *md = p;
 126                 unsigned long long start = md->phys_addr;
 127                 unsigned long long size = md->num_pages << EFI_PAGE_SHIFT;
 128                 int e820_type;
 129
 130                 switch (md->type) {
 131                 case EFI_LOADER_CODE:
 132                 case EFI_LOADER_DATA:
 133                 case EFI_BOOT_SERVICES_CODE:
 134                 case EFI_BOOT_SERVICES_DATA:
 135                 case EFI_CONVENTIONAL_MEMORY:
 136                         if (md->attribute & EFI_MEMORY_WB)
 137                                 e820_type = E820_RAM;
 138                         else
 139                                 e820_type = E820_RESERVED;
 140                         break;
 141                 case EFI_ACPI_RECLAIM_MEMORY:
 142                         e820_type = E820_ACPI;
 143                         break;
 144                 case EFI_ACPI_MEMORY_NVS:
 145                         e820_type = E820_NVS;
 146                         break;
 147                 case EFI_UNUSABLE_MEMORY:
 148                         e820_type = E820_UNUSABLE;
 149                         break;
 150                 default:
 151                         /*
 152                          * EFI_RESERVED_TYPE EFI_RUNTIME_SERVICES_CODE
 153                          * EFI_RUNTIME_SERVICES_DATA EFI_MEMORY_MAPPED_IO
 154                          * EFI_MEMORY_MAPPED_IO_PORT_SPACE EFI_PAL_CODE
 155                          */
 156                         e820_type = E820_RESERVED;
 157                         break;
 158                 }
 159                 e820_add_region(start, size, e820_type);
 160         }
 161         sanitize_e820_map(e820.map, ARRAY_SIZE(e820.map), &e820.nr_map);
 162 }
 163
 164 int __init efi_memblock_x86_reserve_range(void)
 165 {
 166         struct efi_info *e = &boot_params.efi_info;
 167         unsigned long pmap;
 168
 169         if (efi_enabled(EFI_PARAVIRT))
 170                 return 0;
 171
 172 #ifdef CONFIG_X86_32
 173         /* Can't handle data above 4GB at this time */
 174         if (e->efi_memmap_hi) {
 175                 pr_err("Memory map is above 4GB, disabling EFI.\n");
 176                 return -EINVAL;
 177         }
 178         pmap =  e->efi_memmap;
 179 #else
 180         pmap = (e->efi_memmap | ((__u64)e->efi_memmap_hi << 32));
 181 #endif
 182         memmap.phys_map         = (void *)pmap;
 183         memmap.nr_map           = e->efi_memmap_size /
 184                                   e->efi_memdesc_size;
 185         memmap.desc_size        = e->efi_memdesc_size;
 186         memmap.desc_version     = e->efi_memdesc_version;
 187
 188         memblock_reserve(pmap, memmap.nr_map * memmap.desc_size);
 189
 190         efi.memmap = &memmap;
 191
 192         return 0;
 193 }
 194
 195 static void __init print_efi_memmap(void)
 196 {
 197 #ifdef EFI_DEBUG
 198         efi_memory_desc_t *md;
 199         void *p;
 200         int i;
 201
 202         for (p = memmap.map, i = 0;
 203              p < memmap.map_end;
 204              p += memmap.desc_size, i++) {
 205                 md = p;
 206                 pr_info("mem%02u: type=%u, attr=0x%llx, range=[0x%016llx-0x%016llx) (%lluMB)\n",
 207                         i, md->type, md->attribute, md->phys_addr,
 208                         md->phys_addr + (md->num_pages << EFI_PAGE_SHIFT),
 209                         (md->num_pages >> (20 - EFI_PAGE_SHIFT)));
 210         }
 211 #endif  /*  EFI_DEBUG  */
 212 }
 213
 214 void __init efi_unmap_memmap(void)
 215 {
 216         clear_bit(EFI_MEMMAP, &efi.flags);
 217         if (memmap.map) {
 218                 early_memunmap(memmap.map, memmap.nr_map * memmap.desc_size);
 219                 memmap.map = NULL;
 220         }
 221 }
 222
 223 static int __init efi_systab_init(void *phys)
 224 {
 225         if (efi_enabled(EFI_64BIT)) {
 226                 efi_system_table_64_t *systab64;
 227                 struct efi_setup_data *data = NULL;
 228                 u64 tmp = 0;
 229
 230                 if (efi_setup) {
 231                         data = early_memremap(efi_setup, sizeof(*data));
 232                         if (!data)
 233                                 return -ENOMEM;
 234                 }
 235                 systab64 = early_memremap((unsigned long)phys,
 236                                          sizeof(*systab64));
 237                 if (systab64 == NULL) {
 238                         pr_err("Couldn't map the system table!\n");
 239                         if (data)
 240                                 early_memunmap(data, sizeof(*data));
 241                         return -ENOMEM;
 242                 }
 243
 244                 efi_systab.hdr = systab64->hdr;
 245                 efi_systab.fw_vendor = data ? (unsigned long)data->fw_vendor :
 246                                               systab64->fw_vendor;
 247                 tmp |= data ? data->fw_vendor : systab64->fw_vendor;
 248                 efi_systab.fw_revision = systab64->fw_revision;
 249                 efi_systab.con_in_handle = systab64->con_in_handle;
 250                 tmp |= systab64->con_in_handle;
 251                 efi_systab.con_in = systab64->con_in;
 252                 tmp |= systab64->con_in;
 253                 efi_systab.con_out_handle = systab64->con_out_handle;
 254                 tmp |= systab64->con_out_handle;
 255                 efi_systab.con_out = systab64->con_out;
 256                 tmp |= systab64->con_out;
 257                 efi_systab.stderr_handle = systab64->stderr_handle;
 258                 tmp |= systab64->stderr_handle;
 259                 efi_systab.stderr = systab64->stderr;
 260                 tmp |= systab64->stderr;
 261                 efi_systab.runtime = data ?
 262                                      (void *)(unsigned long)data->runtime :
 263                                      (void *)(unsigned long)systab64->runtime;
 264                 tmp |= data ? data->runtime : systab64->runtime;
 265                 efi_systab.boottime = (void *)(unsigned long)systab64->boottime;
 266                 tmp |= systab64->boottime;
 267                 efi_systab.nr_tables = systab64->nr_tables;
 268                 efi_systab.tables = data ? (unsigned long)data->tables :
 269                                            systab64->tables;
 270                 tmp |= data ? data->tables : systab64->tables;
 271
 272                 early_memunmap(systab64, sizeof(*systab64));
 273                 if (data)
 274                         early_memunmap(data, sizeof(*data));
 275 #ifdef CONFIG_X86_32
 276                 if (tmp >> 32) {
 277                         pr_err("EFI data located above 4GB, disabling EFI.\n");
 278                         return -EINVAL;
 279                 }
 280 #endif
 281         } else {
 282                 efi_system_table_32_t *systab32;
 283
 284                 systab32 = early_memremap((unsigned long)phys,
 285                                          sizeof(*systab32));
 286                 if (systab32 == NULL) {
 287                         pr_err("Couldn't map the system table!\n");
 288                         return -ENOMEM;
 289                 }
 290
 291                 efi_systab.hdr = systab32->hdr;
 292                 efi_systab.fw_vendor = systab32->fw_vendor;
 293                 efi_systab.fw_revision = systab32->fw_revision;
 294                 efi_systab.con_in_handle = systab32->con_in_handle;
 295                 efi_systab.con_in = systab32->con_in;
 296                 efi_systab.con_out_handle = systab32->con_out_handle;
 297                 efi_systab.con_out = systab32->con_out;
 298                 efi_systab.stderr_handle = systab32->stderr_handle;
 299                 efi_systab.stderr = systab32->stderr;
 300                 efi_systab.runtime = (void *)(unsigned long)systab32->runtime;
 301                 efi_systab.boottime = (void *)(unsigned long)systab32->boottime;
 302                 efi_systab.nr_tables = systab32->nr_tables;
 303                 efi_systab.tables = systab32->tables;
 304
 305                 early_memunmap(systab32, sizeof(*systab32));
 306         }
 307
 308         efi.systab = &efi_systab;
 309
 310         /*
 311          * Verify the EFI Table
 312          */
 313         if (efi.systab->hdr.signature != EFI_SYSTEM_TABLE_SIGNATURE) {
 314                 pr_err("System table signature incorrect!\n");
 315                 return -EINVAL;
 316         }
 317         if ((efi.systab->hdr.revision >> 16) == 0)
 318                 pr_err("Warning: System table version %d.%02d, expected 1.00 or greater!\n",
 319                        efi.systab->hdr.revision >> 16,
 320                        efi.systab->hdr.revision & 0xffff);
 321
 322         set_bit(EFI_SYSTEM_TABLES, &efi.flags);
 323
 324         return 0;
 325 }
 326
 327 static int __init efi_runtime_init32(void)
 328 {
 329         efi_runtime_services_32_t *runtime;
 330
 331         runtime = early_memremap((unsigned long)efi.systab->runtime,
 332                         sizeof(efi_runtime_services_32_t));
 333         if (!runtime) {
 334                 pr_err("Could not map the runtime service table!\n");
 335                 return -ENOMEM;
 336         }
 337
 338         /*
 339          * We will only need *early* access to the following two
 340          * EFI runtime services before set_virtual_address_map
 341          * is invoked.
 342          */
 343         efi_phys.set_virtual_address_map =
 344                         (efi_set_virtual_address_map_t *)
 345                         (unsigned long)runtime->set_virtual_address_map;
 346         early_memunmap(runtime, sizeof(efi_runtime_services_32_t));
 347
 348         return 0;
 349 }
 350
 351 static int __init efi_runtime_init64(void)
 352 {
 353         efi_runtime_services_64_t *runtime;
 354
 355         runtime = early_memremap((unsigned long)efi.systab->runtime,
 356                         sizeof(efi_runtime_services_64_t));
 357         if (!runtime) {
 358                 pr_err("Could not map the runtime service table!\n");
 359                 return -ENOMEM;
 360         }
 361
 362         /*
 363          * We will only need *early* access to the following two
 364          * EFI runtime services before set_virtual_address_map
 365          * is invoked.
 366          */
 367         efi_phys.set_virtual_address_map =
 368                         (efi_set_virtual_address_map_t *)
 369                         (unsigned long)runtime->set_virtual_address_map;
 370         early_memunmap(runtime, sizeof(efi_runtime_services_64_t));
 371
 372         return 0;
 373 }
 374
 375 static int __init efi_runtime_init(void)
 376 {
 377         int rv;
 378
 379         /*
 380          * Check out the runtime services table. We need to map
 381          * the runtime services table so that we can grab the physical
 382          * address of several of the EFI runtime functions, needed to
 383          * set the firmware into virtual mode.
 384          *
 385          * When EFI_PARAVIRT is in force then we could not map runtime
 386          * service memory region because we do not have direct access to it.
 387          * However, runtime services are available through proxy functions
 388          * (e.g. in case of Xen dom0 EFI implementation they call special
 389          * hypercall which executes relevant EFI functions) and that is why
 390          * they are always enabled.
 391          */
 392
 393         if (!efi_enabled(EFI_PARAVIRT)) {
 394                 if (efi_enabled(EFI_64BIT))
 395                         rv = efi_runtime_init64();
 396                 else
 397                         rv = efi_runtime_init32();
 398
 399                 if (rv)
 400                         return rv;
 401         }
 402
 403         set_bit(EFI_RUNTIME_SERVICES, &efi.flags);
 404
 405         return 0;
 406 }
 407
 408 static int __init efi_memmap_init(void)
 409 {
 410         if (efi_enabled(EFI_PARAVIRT))
 411                 return 0;
 412
 413         /* Map the EFI memory map */
 414         memmap.map = early_memremap((unsigned long)memmap.phys_map,
 415                                    memmap.nr_map * memmap.desc_size);
 416         if (memmap.map == NULL) {
 417                 pr_err("Could not map the memory map!\n");
 418                 return -ENOMEM;
 419         }
 420         memmap.map_end = memmap.map + (memmap.nr_map * memmap.desc_size);
 421
 422         if (add_efi_memmap)
 423                 do_add_efi_memmap();
 424
 425         set_bit(EFI_MEMMAP, &efi.flags);
 426
 427         return 0;
 428 }
 429
 430 void __init efi_init(void)
 431 {
 432         efi_char16_t *c16;
 433         char vendor[100] = "unknown";
 434         int i = 0;
 435         void *tmp;
 436
 437 #ifdef CONFIG_X86_32
 438         if (boot_params.efi_info.efi_systab_hi ||
 439             boot_params.efi_info.efi_memmap_hi) {
 440                 pr_info("Table located above 4GB, disabling EFI.\n");
 441                 return;
 442         }
 443         efi_phys.systab = (efi_system_table_t *)boot_params.efi_info.efi_systab;
 444 #else
 445         efi_phys.systab = (efi_system_table_t *)
 446                           (boot_params.efi_info.efi_systab |
 447                           ((__u64)boot_params.efi_info.efi_systab_hi<<32));
 448 #endif
 449
 450         if (efi_systab_init(efi_phys.systab))
 451                 return;
 452
 453         efi.config_table = (unsigned long)efi.systab->tables;
 454         efi.fw_vendor    = (unsigned long)efi.systab->fw_vendor;
 455         efi.runtime      = (unsigned long)efi.systab->runtime;
 456
 457         /*
 458          * Show what we know for posterity
 459          */
 460         c16 = tmp = early_memremap(efi.systab->fw_vendor, 2);
 461         if (c16) {
 462                 for (i = 0; i < sizeof(vendor) - 1 && *c16; ++i)
 463                         vendor[i] = *c16++;
 464                 vendor[i] = '\0';
 465         } else
 466                 pr_err("Could not map the firmware vendor!\n");
 467         early_memunmap(tmp, 2);
 468
 469         pr_info("EFI v%u.%.02u by %s\n",
 470                 efi.systab->hdr.revision >> 16,
 471                 efi.systab->hdr.revision & 0xffff, vendor);
 472
 473         if (efi_reuse_config(efi.systab->tables, efi.systab->nr_tables))
 474                 return;
 475
 476         if (efi_config_init(arch_tables))
 477                 return;
 478
 479         /*
 480          * Note: We currently don't support runtime services on an EFI
 481          * that doesn't match the kernel 32/64-bit mode.
 482          */
 483
 484         if (!efi_runtime_supported())
 485                 pr_info("No EFI runtime due to 32/64-bit mismatch with kernel\n");
 486         else {
 487                 if (efi_runtime_disabled() || efi_runtime_init())
 488                         return;
 489         }
 490         if (efi_memmap_init())
 491                 return;
 492
 493         print_efi_memmap();
 494 }
 495
 496 void __init efi_late_init(void)
 497 {
 498         efi_bgrt_init();
 499 }
 500
 501 void __init efi_set_executable(efi_memory_desc_t *md, bool executable)
 502 {
 503         u64 addr, npages;
 504
 505         addr = md->virt_addr;
 506         npages = md->num_pages;
 507
 508         memrange_efi_to_native(&addr, &npages);
 509
 510         if (executable)
 511                 set_memory_x(addr, npages);
 512         else
 513                 set_memory_nx(addr, npages);
 514 }
 515
 516 void __init runtime_code_page_mkexec(void)
 517 {
 518         efi_memory_desc_t *md;
 519         void *p;
 520
 521         /* Make EFI runtime service code area executable */
 522         for (p = memmap.map; p < memmap.map_end; p += memmap.desc_size) {
 523                 md = p;
 524
 525                 if (md->type != EFI_RUNTIME_SERVICES_CODE)
 526                         continue;
 527
 528                 efi_set_executable(md, true);
 529         }
 530 }
 531
 532 void efi_memory_uc(u64 addr, unsigned long size)
 533 {
 534         unsigned long page_shift = 1UL << EFI_PAGE_SHIFT;
 535         u64 npages;
 536
 537         npages = round_up(size, page_shift) / page_shift;
 538         memrange_efi_to_native(&addr, &npages);
 539         set_memory_uc(addr, npages);
 540 }
 541
 542 void __init old_map_region(efi_memory_desc_t *md)
 543 {
 544         u64 start_pfn, end_pfn, end;
 545         unsigned long size;
 546         void *va;
 547
 548         start_pfn = PFN_DOWN(md->phys_addr);
 549         size      = md->num_pages << PAGE_SHIFT;
 550         end       = md->phys_addr + size;
 551         end_pfn   = PFN_UP(end);
 552
 553         if (pfn_range_is_mapped(start_pfn, end_pfn)) {
 554                 va = __va(md->phys_addr);
 555
 556                 if (!(md->attribute & EFI_MEMORY_WB))
 557                         efi_memory_uc((u64)(unsigned long)va, size);
 558         } else
 559                 va = efi_ioremap(md->phys_addr, size,
 560                                  md->type, md->attribute);
 561
 562         md->virt_addr = (u64) (unsigned long) va;
 563         if (!va)
 564                 pr_err("ioremap of 0x%llX failed!\n",
 565                        (unsigned long long)md->phys_addr);
 566 }
 567
 568 /* Merge contiguous regions of the same type and attribute */
 569 static void __init efi_merge_regions(void)
 570 {
 571         void *p;
 572         efi_memory_desc_t *md, *prev_md = NULL;
 573
 574         for (p = memmap.map; p < memmap.map_end; p += memmap.desc_size) {
 575                 u64 prev_size;
 576                 md = p;
 577
 578                 if (!prev_md) {
 579                         prev_md = md;
 580                         continue;
 581                 }
 582
 583                 if (prev_md->type != md->type ||
 584                     prev_md->attribute != md->attribute) {
 585                         prev_md = md;
 586                         continue;
 587                 }
 588
 589                 prev_size = prev_md->num_pages << EFI_PAGE_SHIFT;
 590
 591                 if (md->phys_addr == (prev_md->phys_addr + prev_size)) {
 592                         prev_md->num_pages += md->num_pages;
 593                         md->type = EFI_RESERVED_TYPE;
 594                         md->attribute = 0;
 595                         continue;
 596                 }
 597                 prev_md = md;
 598         }
 599 }
 600
 601 static void __init get_systab_virt_addr(efi_memory_desc_t *md)
 602 {
 603         unsigned long size;
 604         u64 end, systab;
 605
 606         size = md->num_pages << EFI_PAGE_SHIFT;
 607         end = md->phys_addr + size;
 608         systab = (u64)(unsigned long)efi_phys.systab;
 609         if (md->phys_addr <= systab && systab < end) {
 610                 systab += md->virt_addr - md->phys_addr;
 611                 efi.systab = (efi_system_table_t *)(unsigned long)systab;
 612         }
 613 }
 614
 615 static void __init save_runtime_map(void)
 616 {
 617 #ifdef CONFIG_KEXEC
 618         efi_memory_desc_t *md;
 619         void *tmp, *p, *q = NULL;
 620         int count = 0;
 621
 622         if (efi_enabled(EFI_OLD_MEMMAP))
 623                 return;
 624
 625         for (p = memmap.map; p < memmap.map_end; p += memmap.desc_size) {
 626                 md = p;
 627
 628                 if (!(md->attribute & EFI_MEMORY_RUNTIME) ||
 629                     (md->type == EFI_BOOT_SERVICES_CODE) ||
 630                     (md->type == EFI_BOOT_SERVICES_DATA))
 631                         continue;
 632                 tmp = krealloc(q, (count + 1) * memmap.desc_size, GFP_KERNEL);
 633                 if (!tmp)
 634                         goto out;
 635                 q = tmp;
 636
 637                 memcpy(q + count * memmap.desc_size, md, memmap.desc_size);
 638                 count++;
 639         }
 640
 641         efi_runtime_map_setup(q, count, memmap.desc_size);
 642         return;
 643
 644 out:
 645         kfree(q);
 646         pr_err("Error saving runtime map, efi runtime on kexec non-functional!!\n");
 647 #endif
 648 }
 649
 650 static void *realloc_pages(void *old_memmap, int old_shift)
 651 {
 652         void *ret;
 653
 654         ret = (void *)__get_free_pages(GFP_KERNEL, old_shift + 1);
 655         if (!ret)
 656                 goto out;
 657
 658         /*
 659          * A first-time allocation doesn't have anything to copy.
 660          */
 661         if (!old_memmap)
 662                 return ret;
 663
 664         memcpy(ret, old_memmap, PAGE_SIZE << old_shift);
 665
 666 out:
 667         free_pages((unsigned long)old_memmap, old_shift);
 668         return ret;
 669 }
 670
 671 /*
 672  * Map the efi memory ranges of the runtime services and update new_mmap with
 673  * virtual addresses.
 674  */
 675 static void * __init efi_map_regions(int *count, int *pg_shift)
 676 {
 677         void *p, *new_memmap = NULL;
 678         unsigned long left = 0;
 679         efi_memory_desc_t *md;
 680
 681         for (p = memmap.map; p < memmap.map_end; p += memmap.desc_size) {
 682                 md = p;
 683                 if (!(md->attribute & EFI_MEMORY_RUNTIME)) {
 684 #ifdef CONFIG_X86_64
 685                         if (md->type != EFI_BOOT_SERVICES_CODE &&
 686                             md->type != EFI_BOOT_SERVICES_DATA)
 687 #endif
 688                                 continue;
 689                 }
 690
 691                 efi_map_region(md);
 692                 get_systab_virt_addr(md);
 693
 694                 if (left < memmap.desc_size) {
 695                         new_memmap = realloc_pages(new_memmap, *pg_shift);
 696                         if (!new_memmap)
 697                                 return NULL;
 698
 699                         left += PAGE_SIZE << *pg_shift;
 700                         (*pg_shift)++;
 701                 }
 702
 703                 memcpy(new_memmap + (*count * memmap.desc_size), md,
 704                        memmap.desc_size);
 705
 706                 left -= memmap.desc_size;
 707                 (*count)++;
 708         }
 709
 710         return new_memmap;
 711 }
 712
 713 static void __init kexec_enter_virtual_mode(void)
 714 {
 715 #ifdef CONFIG_KEXEC
 716         efi_memory_desc_t *md;
 717         void *p;
 718
 719         efi.systab = NULL;
 720
 721         /*
 722          * We don't do virtual mode, since we don't do runtime services, on
 723          * non-native EFI
 724          */
 725         if (!efi_is_native()) {
 726                 efi_unmap_memmap();
 727                 return;
 728         }
 729
 730         /*
 731         * Map efi regions which were passed via setup_data. The virt_addr is a
 732         * fixed addr which was used in first kernel of a kexec boot.
 733         */
 734         for (p = memmap.map; p < memmap.map_end; p += memmap.desc_size) {
 735                 md = p;
 736                 efi_map_region_fixed(md); /* FIXME: add error handling */
 737                 get_systab_virt_addr(md);
 738         }
 739
 740         save_runtime_map();
 741
 742         BUG_ON(!efi.systab);
 743
 744         efi_sync_low_kernel_mappings();
 745
 746         /*
 747          * Now that EFI is in virtual mode, update the function
 748          * pointers in the runtime service table to the new virtual addresses.
 749          *
 750          * Call EFI services through wrapper functions.
 751          */
 752         efi.runtime_version = efi_systab.hdr.revision;
 753
 754         efi_native_runtime_setup();
 755
 756         efi.set_virtual_address_map = NULL;
 757
 758         if (efi_enabled(EFI_OLD_MEMMAP) && (__supported_pte_mask & _PAGE_NX))
 759                 runtime_code_page_mkexec();
 760
 761         /* clean DUMMY object */
 762         efi_delete_dummy_variable();
 763 #endif
 764 }
 765
 766 /*
 767  * This function will switch the EFI runtime services to virtual mode.
 768  * Essentially, we look through the EFI memmap and map every region that
 769  * has the runtime attribute bit set in its memory descriptor into the
 770  * ->trampoline_pgd page table using a top-down VA allocation scheme.
 771  *
 772  * The old method which used to update that memory descriptor with the
 773  * virtual address obtained from ioremap() is still supported when the
 774  * kernel is booted with efi=old_map on its command line. Same old
 775  * method enabled the runtime services to be called without having to
 776  * thunk back into physical mode for every invocation.
 777  *
 778  * The new method does a pagetable switch in a preemption-safe manner
 779  * so that we're in a different address space when calling a runtime
 780  * function. For function arguments passing we do copy the PGDs of the
 781  * kernel page table into ->trampoline_pgd prior to each call.
 782  *
 783  * Specially for kexec boot, efi runtime maps in previous kernel should
 784  * be passed in via setup_data. In that case runtime ranges will be mapped
 785  * to the same virtual addresses as the first kernel, see
 786  * kexec_enter_virtual_mode().
 787  */
 788 static void __init __efi_enter_virtual_mode(void)
 789 {
 790         int count = 0, pg_shift = 0;
 791         void *new_memmap = NULL;
 792         efi_status_t status;
 793
 794         efi.systab = NULL;
 795
 796         efi_merge_regions();
 797         new_memmap = efi_map_regions(&count, &pg_shift);
 798         if (!new_memmap) {
 799                 pr_err("Error reallocating memory, EFI runtime non-functional!\n");
 800                 return;
 801         }
 802
 803         save_runtime_map();
 804
 805         BUG_ON(!efi.systab);
 806
 807         if (efi_setup_page_tables(__pa(new_memmap), 1 << pg_shift))
 808                 return;
 809
 810         efi_sync_low_kernel_mappings();
 811         efi_dump_pagetable();
 812
 813         if (efi_is_native()) {
 814                 status = phys_efi_set_virtual_address_map(
 815                                 memmap.desc_size * count,
 816                                 memmap.desc_size,
 817                                 memmap.desc_version,
 818                                 (efi_memory_desc_t *)__pa(new_memmap));
 819         } else {
 820                 status = efi_thunk_set_virtual_address_map(
 821                                 efi_phys.set_virtual_address_map,
 822                                 memmap.desc_size * count,
 823                                 memmap.desc_size,
 824                                 memmap.desc_version,
 825                                 (efi_memory_desc_t *)__pa(new_memmap));
 826         }
 827
 828         if (status != EFI_SUCCESS) {
 829                 pr_alert("Unable to switch EFI into virtual mode (status=%lx)!\n",
 830                          status);
 831                 panic("EFI call to SetVirtualAddressMap() failed!");
 832         }
 833
 834         /*
 835          * Now that EFI is in virtual mode, update the function
 836          * pointers in the runtime service table to the new virtual addresses.
 837          *
 838          * Call EFI services through wrapper functions.
 839          */
 840         efi.runtime_version = efi_systab.hdr.revision;
 841
 842         if (efi_is_native())
 843                 efi_native_runtime_setup();
 844         else
 845                 efi_thunk_runtime_setup();
 846
 847         efi.set_virtual_address_map = NULL;
 848
 849         efi_runtime_mkexec();
 850
 851         /*
 852          * We mapped the descriptor array into the EFI pagetable above but we're
 853          * not unmapping it here. Here's why:
 854          *
 855          * We're copying select PGDs from the kernel page table to the EFI page
 856          * table and when we do so and make changes to those PGDs like unmapping
 857          * stuff from them, those changes appear in the kernel page table and we
 858          * go boom.
 859          *
 860          * From setup_real_mode():
 861          *
 862          * ...
 863          * trampoline_pgd[0] = init_level4_pgt[pgd_index(__PAGE_OFFSET)].pgd;
 864          *
 865          * In this particular case, our allocation is in PGD 0 of the EFI page
 866          * table but we've copied that PGD from PGD[272] of the EFI page table:
 867          *
 868          *      pgd_index(__PAGE_OFFSET = 0xffff880000000000) = 272
 869          *
 870          * where the direct memory mapping in kernel space is.
 871          *
 872          * new_memmap's VA comes from that direct mapping and thus clearing it,
 873          * it would get cleared in the kernel page table too.
 874          *
 875          * efi_cleanup_page_tables(__pa(new_memmap), 1 << pg_shift);
 876          */
 877         free_pages((unsigned long)new_memmap, pg_shift);
 878
 879         /* clean DUMMY object */
 880         efi_delete_dummy_variable();
 881 }
 882
 883 void __init efi_enter_virtual_mode(void)
 884 {
 885         if (efi_enabled(EFI_PARAVIRT))
 886                 return;
 887
 888         if (efi_setup)
 889                 kexec_enter_virtual_mode();
 890         else
 891                 __efi_enter_virtual_mode();
 892 }
 893
 894 /*
 895  * Convenience functions to obtain memory types and attributes
 896  */
 897 u32 efi_mem_type(unsigned long phys_addr)
 898 {
 899         efi_memory_desc_t *md;
 900         void *p;
 901
 902         if (!efi_enabled(EFI_MEMMAP))
 903                 return 0;
 904
 905         for (p = memmap.map; p < memmap.map_end; p += memmap.desc_size) {
 906                 md = p;
 907                 if ((md->phys_addr <= phys_addr) &&
 908                     (phys_addr < (md->phys_addr +
 909                                   (md->num_pages << EFI_PAGE_SHIFT))))
 910                         return md->type;
 911         }
 912         return 0;
 913 }
 914
 915 u64 efi_mem_attributes(unsigned long phys_addr)
 916 {
 917         efi_memory_desc_t *md;
 918         void *p;
 919
 920         if (!efi_enabled(EFI_MEMMAP))
 921                 return 0;
 922
 923         for (p = memmap.map; p < memmap.map_end; p += memmap.desc_size) {
 924                 md = p;
 925                 if ((md->phys_addr <= phys_addr) &&
 926                     (phys_addr < (md->phys_addr +
 927                                   (md->num_pages << EFI_PAGE_SHIFT))))
 928                         return md->attribute;
 929         }
 930         return 0;
 931 }
 932
 933 static int __init arch_parse_efi_cmdline(char *str)
 934 {
 935         if (parse_option_str(str, "old_map"))
 936                 set_bit(EFI_OLD_MEMMAP, &efi.flags);
 937
 938         return 0;
 939 }
 940 early_param("efi", arch_parse_efi_cmdline);