arch/x86/platform/efi/efi.c

   1 /*
   2  * Common EFI (Extensible Firmware Interface) support functions
   3  * Based on Extensible Firmware Interface Specification version 1.0
   4  *
   5  * Copyright (C) 1999 VA Linux Systems
   6  * Copyright (C) 1999 Walt Drummond <drummond@valinux.com>
   7  * Copyright (C) 1999-2002 Hewlett-Packard Co.
   8  *      David Mosberger-Tang <davidm@hpl.hp.com>
   9  *      Stephane Eranian <eranian@hpl.hp.com>
  10  * Copyright (C) 2005-2008 Intel Co.
  11  *      Fenghua Yu <fenghua.yu@intel.com>
  12  *      Bibo Mao <bibo.mao@intel.com>
  13  *      Chandramouli Narayanan <mouli@linux.intel.com>
  14  *      Huang Ying <ying.huang@intel.com>
  15  * Copyright (C) 2013 SuSE Labs
  16  *      Borislav Petkov <bp@suse.de> - runtime services VA mapping
  17  *
  18  * Copied from efi_32.c to eliminate the duplicated code between EFI
  19  * 32/64 support code. --ying 2007-10-26
  20  *
  21  * All EFI Runtime Services are not implemented yet as EFI only
  22  * supports physical mode addressing on SoftSDV. This is to be fixed
  23  * in a future version.  --drummond 1999-07-20
  24  *
  25  * Implemented EFI runtime services and virtual mode calls.  --davidm
  26  *
  27  * Goutham Rao: <goutham.rao@intel.com>
  28  *      Skip non-WB memory and ignore empty memory ranges.
  29  */
  30
  31 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  32
  33 #include <linux/kernel.h>
  34 #include <linux/init.h>
  35 #include <linux/efi.h>
  36 #include <linux/efi-bgrt.h>
  37 #include <linux/export.h>
  38 #include <linux/bootmem.h>
  39 #include <linux/slab.h>
  40 #include <linux/memblock.h>
  41 #include <linux/spinlock.h>
  42 #include <linux/uaccess.h>
  43 #include <linux/time.h>
  44 #include <linux/io.h>
  45 #include <linux/reboot.h>
  46 #include <linux/bcd.h>
  47
  48 #include <asm/setup.h>
  49 #include <asm/efi.h>
  50 #include <asm/time.h>
  51 #include <asm/cacheflush.h>
  52 #include <asm/tlbflush.h>
  53 #include <asm/x86_init.h>
  54 #include <asm/rtc.h>
  55 #include <asm/uv/uv.h>
  56
  57 #define EFI_DEBUG
  58
  59 struct efi_memory_map memmap;
  60
  61 static struct efi efi_phys __initdata;
  62 static efi_system_table_t efi_systab __initdata;
  63
  64 static efi_config_table_type_t arch_tables[] __initdata = {
  65 #ifdef CONFIG_X86_UV
  66         {UV_SYSTEM_TABLE_GUID, "UVsystab", &efi.uv_systab},
  67 #endif
  68         {NULL_GUID, NULL, NULL},
  69 };
  70
  71 u64 efi_setup;          /* efi setup_data physical address */
  72
  73 static bool disable_runtime __initdata = false;
  74 static int __init setup_noefi(char *arg)
  75 {
  76         disable_runtime = true;
  77         return 0;
  78 }
  79 early_param("noefi", setup_noefi);
  80
  81 int add_efi_memmap;
  82 EXPORT_SYMBOL(add_efi_memmap);
  83
  84 static int __init setup_add_efi_memmap(char *arg)
  85 {
  86         add_efi_memmap = 1;
  87         return 0;
  88 }
  89 early_param("add_efi_memmap", setup_add_efi_memmap);
  90
  91 static efi_status_t __init phys_efi_set_virtual_address_map(
  92         unsigned long memory_map_size,
  93         unsigned long descriptor_size,
  94         u32 descriptor_version,
  95         efi_memory_desc_t *virtual_map)
  96 {
  97         efi_status_t status;
  98
  99         efi_call_phys_prelog();
 100         status = efi_call_phys(efi_phys.set_virtual_address_map,
 101                                memory_map_size, descriptor_size,
 102                                descriptor_version, virtual_map);
 103         efi_call_phys_epilog();
 104         return status;
 105 }
 106
 107 void efi_get_time(struct timespec *now)
 108 {
 109         efi_status_t status;
 110         efi_time_t eft;
 111         efi_time_cap_t cap;
 112
 113         status = efi.get_time(&eft, &cap);
 114         if (status != EFI_SUCCESS)
 115                 pr_err("Oops: efitime: can't read time!\n");
 116
 117         now->tv_sec = mktime(eft.year, eft.month, eft.day, eft.hour,
 118                              eft.minute, eft.second);
 119         now->tv_nsec = 0;
 120 }
 121
 122 /*
 123  * Tell the kernel about the EFI memory map.  This might include
 124  * more than the max 128 entries that can fit in the e820 legacy
 125  * (zeropage) memory map.
 126  */
 127
 128 static void __init do_add_efi_memmap(void)
 129 {
 130         void *p;
 131
 132         for (p = memmap.map; p < memmap.map_end; p += memmap.desc_size) {
 133                 efi_memory_desc_t *md = p;
 134                 unsigned long long start = md->phys_addr;
 135                 unsigned long long size = md->num_pages << EFI_PAGE_SHIFT;
 136                 int e820_type;
 137
 138                 switch (md->type) {
 139                 case EFI_LOADER_CODE:
 140                 case EFI_LOADER_DATA:
 141                 case EFI_BOOT_SERVICES_CODE:
 142                 case EFI_BOOT_SERVICES_DATA:
 143                 case EFI_CONVENTIONAL_MEMORY:
 144                         if (md->attribute & EFI_MEMORY_WB)
 145                                 e820_type = E820_RAM;
 146                         else
 147                                 e820_type = E820_RESERVED;
 148                         break;
 149                 case EFI_ACPI_RECLAIM_MEMORY:
 150                         e820_type = E820_ACPI;
 151                         break;
 152                 case EFI_ACPI_MEMORY_NVS:
 153                         e820_type = E820_NVS;
 154                         break;
 155                 case EFI_UNUSABLE_MEMORY:
 156                         e820_type = E820_UNUSABLE;
 157                         break;
 158                 default:
 159                         /*
 160                          * EFI_RESERVED_TYPE EFI_RUNTIME_SERVICES_CODE
 161                          * EFI_RUNTIME_SERVICES_DATA EFI_MEMORY_MAPPED_IO
 162                          * EFI_MEMORY_MAPPED_IO_PORT_SPACE EFI_PAL_CODE
 163                          */
 164                         e820_type = E820_RESERVED;
 165                         break;
 166                 }
 167                 e820_add_region(start, size, e820_type);
 168         }
 169         sanitize_e820_map(e820.map, ARRAY_SIZE(e820.map), &e820.nr_map);
 170 }
 171
 172 int __init efi_memblock_x86_reserve_range(void)
 173 {
 174         struct efi_info *e = &boot_params.efi_info;
 175         unsigned long pmap;
 176
 177         if (efi_enabled(EFI_PARAVIRT))
 178                 return 0;
 179
 180 #ifdef CONFIG_X86_32
 181         /* Can't handle data above 4GB at this time */
 182         if (e->efi_memmap_hi) {
 183                 pr_err("Memory map is above 4GB, disabling EFI.\n");
 184                 return -EINVAL;
 185         }
 186         pmap =  e->efi_memmap;
 187 #else
 188         pmap = (e->efi_memmap | ((__u64)e->efi_memmap_hi << 32));
 189 #endif
 190         memmap.phys_map         = (void *)pmap;
 191         memmap.nr_map           = e->efi_memmap_size /
 192                                   e->efi_memdesc_size;
 193         memmap.desc_size        = e->efi_memdesc_size;
 194         memmap.desc_version     = e->efi_memdesc_version;
 195
 196         memblock_reserve(pmap, memmap.nr_map * memmap.desc_size);
 197
 198         efi.memmap = &memmap;
 199
 200         return 0;
 201 }
 202
 203 static void __init print_efi_memmap(void)
 204 {
 205 #ifdef EFI_DEBUG
 206         efi_memory_desc_t *md;
 207         void *p;
 208         int i;
 209
 210         for (p = memmap.map, i = 0;
 211              p < memmap.map_end;
 212              p += memmap.desc_size, i++) {
 213                 md = p;
 214                 pr_info("mem%02u: type=%u, attr=0x%llx, range=[0x%016llx-0x%016llx) (%lluMB)\n",
 215                         i, md->type, md->attribute, md->phys_addr,
 216                         md->phys_addr + (md->num_pages << EFI_PAGE_SHIFT),
 217                         (md->num_pages >> (20 - EFI_PAGE_SHIFT)));
 218         }
 219 #endif  /*  EFI_DEBUG  */
 220 }
 221
 222 void __init efi_unmap_memmap(void)
 223 {
 224         clear_bit(EFI_MEMMAP, &efi.flags);
 225         if (memmap.map) {
 226                 early_memunmap(memmap.map, memmap.nr_map * memmap.desc_size);
 227                 memmap.map = NULL;
 228         }
 229 }
 230
 231 static int __init efi_systab_init(void *phys)
 232 {
 233         if (efi_enabled(EFI_64BIT)) {
 234                 efi_system_table_64_t *systab64;
 235                 struct efi_setup_data *data = NULL;
 236                 u64 tmp = 0;
 237
 238                 if (efi_setup) {
 239                         data = early_memremap(efi_setup, sizeof(*data));
 240                         if (!data)
 241                                 return -ENOMEM;
 242                 }
 243                 systab64 = early_memremap((unsigned long)phys,
 244                                          sizeof(*systab64));
 245                 if (systab64 == NULL) {
 246                         pr_err("Couldn't map the system table!\n");
 247                         if (data)
 248                                 early_memunmap(data, sizeof(*data));
 249                         return -ENOMEM;
 250                 }
 251
 252                 efi_systab.hdr = systab64->hdr;
 253                 efi_systab.fw_vendor = data ? (unsigned long)data->fw_vendor :
 254                                               systab64->fw_vendor;
 255                 tmp |= data ? data->fw_vendor : systab64->fw_vendor;
 256                 efi_systab.fw_revision = systab64->fw_revision;
 257                 efi_systab.con_in_handle = systab64->con_in_handle;
 258                 tmp |= systab64->con_in_handle;
 259                 efi_systab.con_in = systab64->con_in;
 260                 tmp |= systab64->con_in;
 261                 efi_systab.con_out_handle = systab64->con_out_handle;
 262                 tmp |= systab64->con_out_handle;
 263                 efi_systab.con_out = systab64->con_out;
 264                 tmp |= systab64->con_out;
 265                 efi_systab.stderr_handle = systab64->stderr_handle;
 266                 tmp |= systab64->stderr_handle;
 267                 efi_systab.stderr = systab64->stderr;
 268                 tmp |= systab64->stderr;
 269                 efi_systab.runtime = data ?
 270                                      (void *)(unsigned long)data->runtime :
 271                                      (void *)(unsigned long)systab64->runtime;
 272                 tmp |= data ? data->runtime : systab64->runtime;
 273                 efi_systab.boottime = (void *)(unsigned long)systab64->boottime;
 274                 tmp |= systab64->boottime;
 275                 efi_systab.nr_tables = systab64->nr_tables;
 276                 efi_systab.tables = data ? (unsigned long)data->tables :
 277                                            systab64->tables;
 278                 tmp |= data ? data->tables : systab64->tables;
 279
 280                 early_memunmap(systab64, sizeof(*systab64));
 281                 if (data)
 282                         early_memunmap(data, sizeof(*data));
 283 #ifdef CONFIG_X86_32
 284                 if (tmp >> 32) {
 285                         pr_err("EFI data located above 4GB, disabling EFI.\n");
 286                         return -EINVAL;
 287                 }
 288 #endif
 289         } else {
 290                 efi_system_table_32_t *systab32;
 291
 292                 systab32 = early_memremap((unsigned long)phys,
 293                                          sizeof(*systab32));
 294                 if (systab32 == NULL) {
 295                         pr_err("Couldn't map the system table!\n");
 296                         return -ENOMEM;
 297                 }
 298
 299                 efi_systab.hdr = systab32->hdr;
 300                 efi_systab.fw_vendor = systab32->fw_vendor;
 301                 efi_systab.fw_revision = systab32->fw_revision;
 302                 efi_systab.con_in_handle = systab32->con_in_handle;
 303                 efi_systab.con_in = systab32->con_in;
 304                 efi_systab.con_out_handle = systab32->con_out_handle;
 305                 efi_systab.con_out = systab32->con_out;
 306                 efi_systab.stderr_handle = systab32->stderr_handle;
 307                 efi_systab.stderr = systab32->stderr;
 308                 efi_systab.runtime = (void *)(unsigned long)systab32->runtime;
 309                 efi_systab.boottime = (void *)(unsigned long)systab32->boottime;
 310                 efi_systab.nr_tables = systab32->nr_tables;
 311                 efi_systab.tables = systab32->tables;
 312
 313                 early_memunmap(systab32, sizeof(*systab32));
 314         }
 315
 316         efi.systab = &efi_systab;
 317
 318         /*
 319          * Verify the EFI Table
 320          */
 321         if (efi.systab->hdr.signature != EFI_SYSTEM_TABLE_SIGNATURE) {
 322                 pr_err("System table signature incorrect!\n");
 323                 return -EINVAL;
 324         }
 325         if ((efi.systab->hdr.revision >> 16) == 0)
 326                 pr_err("Warning: System table version %d.%02d, expected 1.00 or greater!\n",
 327                        efi.systab->hdr.revision >> 16,
 328                        efi.systab->hdr.revision & 0xffff);
 329
 330         set_bit(EFI_SYSTEM_TABLES, &efi.flags);
 331
 332         return 0;
 333 }
 334
 335 static int __init efi_runtime_init32(void)
 336 {
 337         efi_runtime_services_32_t *runtime;
 338
 339         runtime = early_memremap((unsigned long)efi.systab->runtime,
 340                         sizeof(efi_runtime_services_32_t));
 341         if (!runtime) {
 342                 pr_err("Could not map the runtime service table!\n");
 343                 return -ENOMEM;
 344         }
 345
 346         /*
 347          * We will only need *early* access to the following two
 348          * EFI runtime services before set_virtual_address_map
 349          * is invoked.
 350          */
 351         efi_phys.set_virtual_address_map =
 352                         (efi_set_virtual_address_map_t *)
 353                         (unsigned long)runtime->set_virtual_address_map;
 354         early_memunmap(runtime, sizeof(efi_runtime_services_32_t));
 355
 356         return 0;
 357 }
 358
 359 static int __init efi_runtime_init64(void)
 360 {
 361         efi_runtime_services_64_t *runtime;
 362
 363         runtime = early_memremap((unsigned long)efi.systab->runtime,
 364                         sizeof(efi_runtime_services_64_t));
 365         if (!runtime) {
 366                 pr_err("Could not map the runtime service table!\n");
 367                 return -ENOMEM;
 368         }
 369
 370         /*
 371          * We will only need *early* access to the following two
 372          * EFI runtime services before set_virtual_address_map
 373          * is invoked.
 374          */
 375         efi_phys.set_virtual_address_map =
 376                         (efi_set_virtual_address_map_t *)
 377                         (unsigned long)runtime->set_virtual_address_map;
 378         early_memunmap(runtime, sizeof(efi_runtime_services_64_t));
 379
 380         return 0;
 381 }
 382
 383 static int __init efi_runtime_init(void)
 384 {
 385         int rv;
 386
 387         /*
 388          * Check out the runtime services table. We need to map
 389          * the runtime services table so that we can grab the physical
 390          * address of several of the EFI runtime functions, needed to
 391          * set the firmware into virtual mode.
 392          *
 393          * When EFI_PARAVIRT is in force then we could not map runtime
 394          * service memory region because we do not have direct access to it.
 395          * However, runtime services are available through proxy functions
 396          * (e.g. in case of Xen dom0 EFI implementation they call special
 397          * hypercall which executes relevant EFI functions) and that is why
 398          * they are always enabled.
 399          */
 400
 401         if (!efi_enabled(EFI_PARAVIRT)) {
 402                 if (efi_enabled(EFI_64BIT))
 403                         rv = efi_runtime_init64();
 404                 else
 405                         rv = efi_runtime_init32();
 406
 407                 if (rv)
 408                         return rv;
 409         }
 410
 411         set_bit(EFI_RUNTIME_SERVICES, &efi.flags);
 412
 413         return 0;
 414 }
 415
 416 static int __init efi_memmap_init(void)
 417 {
 418         if (efi_enabled(EFI_PARAVIRT))
 419                 return 0;
 420
 421         /* Map the EFI memory map */
 422         memmap.map = early_memremap((unsigned long)memmap.phys_map,
 423                                    memmap.nr_map * memmap.desc_size);
 424         if (memmap.map == NULL) {
 425                 pr_err("Could not map the memory map!\n");
 426                 return -ENOMEM;
 427         }
 428         memmap.map_end = memmap.map + (memmap.nr_map * memmap.desc_size);
 429
 430         if (add_efi_memmap)
 431                 do_add_efi_memmap();
 432
 433         set_bit(EFI_MEMMAP, &efi.flags);
 434
 435         return 0;
 436 }
 437
 438 void __init efi_init(void)
 439 {
 440         efi_char16_t *c16;
 441         char vendor[100] = "unknown";
 442         int i = 0;
 443         void *tmp;
 444
 445 #ifdef CONFIG_X86_32
 446         if (boot_params.efi_info.efi_systab_hi ||
 447             boot_params.efi_info.efi_memmap_hi) {
 448                 pr_info("Table located above 4GB, disabling EFI.\n");
 449                 return;
 450         }
 451         efi_phys.systab = (efi_system_table_t *)boot_params.efi_info.efi_systab;
 452 #else
 453         efi_phys.systab = (efi_system_table_t *)
 454                           (boot_params.efi_info.efi_systab |
 455                           ((__u64)boot_params.efi_info.efi_systab_hi<<32));
 456 #endif
 457
 458         if (efi_systab_init(efi_phys.systab))
 459                 return;
 460
 461         efi.config_table = (unsigned long)efi.systab->tables;
 462         efi.fw_vendor    = (unsigned long)efi.systab->fw_vendor;
 463         efi.runtime      = (unsigned long)efi.systab->runtime;
 464
 465         /*
 466          * Show what we know for posterity
 467          */
 468         c16 = tmp = early_memremap(efi.systab->fw_vendor, 2);
 469         if (c16) {
 470                 for (i = 0; i < sizeof(vendor) - 1 && *c16; ++i)
 471                         vendor[i] = *c16++;
 472                 vendor[i] = '\0';
 473         } else
 474                 pr_err("Could not map the firmware vendor!\n");
 475         early_memunmap(tmp, 2);
 476
 477         pr_info("EFI v%u.%.02u by %s\n",
 478                 efi.systab->hdr.revision >> 16,
 479                 efi.systab->hdr.revision & 0xffff, vendor);
 480
 481         if (efi_reuse_config(efi.systab->tables, efi.systab->nr_tables))
 482                 return;
 483
 484         if (efi_config_init(arch_tables))
 485                 return;
 486
 487         /*
 488          * Note: We currently don't support runtime services on an EFI
 489          * that doesn't match the kernel 32/64-bit mode.
 490          */
 491
 492         if (!efi_runtime_supported())
 493                 pr_info("No EFI runtime due to 32/64-bit mismatch with kernel\n");
 494         else {
 495                 if (disable_runtime || efi_runtime_init())
 496                         return;
 497         }
 498         if (efi_memmap_init())
 499                 return;
 500
 501         print_efi_memmap();
 502 }
 503
 504 void __init efi_late_init(void)
 505 {
 506         efi_bgrt_init();
 507 }
 508
 509 void __init efi_set_executable(efi_memory_desc_t *md, bool executable)
 510 {
 511         u64 addr, npages;
 512
 513         addr = md->virt_addr;
 514         npages = md->num_pages;
 515
 516         memrange_efi_to_native(&addr, &npages);
 517
 518         if (executable)
 519                 set_memory_x(addr, npages);
 520         else
 521                 set_memory_nx(addr, npages);
 522 }
 523
 524 void __init runtime_code_page_mkexec(void)
 525 {
 526         efi_memory_desc_t *md;
 527         void *p;
 528
 529         /* Make EFI runtime service code area executable */
 530         for (p = memmap.map; p < memmap.map_end; p += memmap.desc_size) {
 531                 md = p;
 532
 533                 if (md->type != EFI_RUNTIME_SERVICES_CODE)
 534                         continue;
 535
 536                 efi_set_executable(md, true);
 537         }
 538 }
 539
 540 void efi_memory_uc(u64 addr, unsigned long size)
 541 {
 542         unsigned long page_shift = 1UL << EFI_PAGE_SHIFT;
 543         u64 npages;
 544
 545         npages = round_up(size, page_shift) / page_shift;
 546         memrange_efi_to_native(&addr, &npages);
 547         set_memory_uc(addr, npages);
 548 }
 549
 550 void __init old_map_region(efi_memory_desc_t *md)
 551 {
 552         u64 start_pfn, end_pfn, end;
 553         unsigned long size;
 554         void *va;
 555
 556         start_pfn = PFN_DOWN(md->phys_addr);
 557         size      = md->num_pages << PAGE_SHIFT;
 558         end       = md->phys_addr + size;
 559         end_pfn   = PFN_UP(end);
 560
 561         if (pfn_range_is_mapped(start_pfn, end_pfn)) {
 562                 va = __va(md->phys_addr);
 563
 564                 if (!(md->attribute & EFI_MEMORY_WB))
 565                         efi_memory_uc((u64)(unsigned long)va, size);
 566         } else
 567                 va = efi_ioremap(md->phys_addr, size,
 568                                  md->type, md->attribute);
 569
 570         md->virt_addr = (u64) (unsigned long) va;
 571         if (!va)
 572                 pr_err("ioremap of 0x%llX failed!\n",
 573                        (unsigned long long)md->phys_addr);
 574 }
 575
 576 /* Merge contiguous regions of the same type and attribute */
 577 static void __init efi_merge_regions(void)
 578 {
 579         void *p;
 580         efi_memory_desc_t *md, *prev_md = NULL;
 581
 582         for (p = memmap.map; p < memmap.map_end; p += memmap.desc_size) {
 583                 u64 prev_size;
 584                 md = p;
 585
 586                 if (!prev_md) {
 587                         prev_md = md;
 588                         continue;
 589                 }
 590
 591                 if (prev_md->type != md->type ||
 592                     prev_md->attribute != md->attribute) {
 593                         prev_md = md;
 594                         continue;
 595                 }
 596
 597                 prev_size = prev_md->num_pages << EFI_PAGE_SHIFT;
 598
 599                 if (md->phys_addr == (prev_md->phys_addr + prev_size)) {
 600                         prev_md->num_pages += md->num_pages;
 601                         md->type = EFI_RESERVED_TYPE;
 602                         md->attribute = 0;
 603                         continue;
 604                 }
 605                 prev_md = md;
 606         }
 607 }
 608
 609 static void __init get_systab_virt_addr(efi_memory_desc_t *md)
 610 {
 611         unsigned long size;
 612         u64 end, systab;
 613
 614         size = md->num_pages << EFI_PAGE_SHIFT;
 615         end = md->phys_addr + size;
 616         systab = (u64)(unsigned long)efi_phys.systab;
 617         if (md->phys_addr <= systab && systab < end) {
 618                 systab += md->virt_addr - md->phys_addr;
 619                 efi.systab = (efi_system_table_t *)(unsigned long)systab;
 620         }
 621 }
 622
 623 static void __init save_runtime_map(void)
 624 {
 625 #ifdef CONFIG_KEXEC
 626         efi_memory_desc_t *md;
 627         void *tmp, *p, *q = NULL;
 628         int count = 0;
 629
 630         if (efi_enabled(EFI_OLD_MEMMAP))
 631                 return;
 632
 633         for (p = memmap.map; p < memmap.map_end; p += memmap.desc_size) {
 634                 md = p;
 635
 636                 if (!(md->attribute & EFI_MEMORY_RUNTIME) ||
 637                     (md->type == EFI_BOOT_SERVICES_CODE) ||
 638                     (md->type == EFI_BOOT_SERVICES_DATA))
 639                         continue;
 640                 tmp = krealloc(q, (count + 1) * memmap.desc_size, GFP_KERNEL);
 641                 if (!tmp)
 642                         goto out;
 643                 q = tmp;
 644
 645                 memcpy(q + count * memmap.desc_size, md, memmap.desc_size);
 646                 count++;
 647         }
 648
 649         efi_runtime_map_setup(q, count, memmap.desc_size);
 650         return;
 651
 652 out:
 653         kfree(q);
 654         pr_err("Error saving runtime map, efi runtime on kexec non-functional!!\n");
 655 #endif
 656 }
 657
 658 static void *realloc_pages(void *old_memmap, int old_shift)
 659 {
 660         void *ret;
 661
 662         ret = (void *)__get_free_pages(GFP_KERNEL, old_shift + 1);
 663         if (!ret)
 664                 goto out;
 665
 666         /*
 667          * A first-time allocation doesn't have anything to copy.
 668          */
 669         if (!old_memmap)
 670                 return ret;
 671
 672         memcpy(ret, old_memmap, PAGE_SIZE << old_shift);
 673
 674 out:
 675         free_pages((unsigned long)old_memmap, old_shift);
 676         return ret;
 677 }
 678
 679 /*
 680  * Map the efi memory ranges of the runtime services and update new_mmap with
 681  * virtual addresses.
 682  */
 683 static void * __init efi_map_regions(int *count, int *pg_shift)
 684 {
 685         void *p, *new_memmap = NULL;
 686         unsigned long left = 0;
 687         efi_memory_desc_t *md;
 688
 689         for (p = memmap.map; p < memmap.map_end; p += memmap.desc_size) {
 690                 md = p;
 691                 if (!(md->attribute & EFI_MEMORY_RUNTIME)) {
 692 #ifdef CONFIG_X86_64
 693                         if (md->type != EFI_BOOT_SERVICES_CODE &&
 694                             md->type != EFI_BOOT_SERVICES_DATA)
 695 #endif
 696                                 continue;
 697                 }
 698
 699                 efi_map_region(md);
 700                 get_systab_virt_addr(md);
 701
 702                 if (left < memmap.desc_size) {
 703                         new_memmap = realloc_pages(new_memmap, *pg_shift);
 704                         if (!new_memmap)
 705                                 return NULL;
 706
 707                         left += PAGE_SIZE << *pg_shift;
 708                         (*pg_shift)++;
 709                 }
 710
 711                 memcpy(new_memmap + (*count * memmap.desc_size), md,
 712                        memmap.desc_size);
 713
 714                 left -= memmap.desc_size;
 715                 (*count)++;
 716         }
 717
 718         return new_memmap;
 719 }
 720
 721 static void __init kexec_enter_virtual_mode(void)
 722 {
 723 #ifdef CONFIG_KEXEC
 724         efi_memory_desc_t *md;
 725         void *p;
 726
 727         efi.systab = NULL;
 728
 729         /*
 730          * We don't do virtual mode, since we don't do runtime services, on
 731          * non-native EFI
 732          */
 733         if (!efi_is_native()) {
 734                 efi_unmap_memmap();
 735                 return;
 736         }
 737
 738         /*
 739         * Map efi regions which were passed via setup_data. The virt_addr is a
 740         * fixed addr which was used in first kernel of a kexec boot.
 741         */
 742         for (p = memmap.map; p < memmap.map_end; p += memmap.desc_size) {
 743                 md = p;
 744                 efi_map_region_fixed(md); /* FIXME: add error handling */
 745                 get_systab_virt_addr(md);
 746         }
 747
 748         save_runtime_map();
 749
 750         BUG_ON(!efi.systab);
 751
 752         efi_sync_low_kernel_mappings();
 753
 754         /*
 755          * Now that EFI is in virtual mode, update the function
 756          * pointers in the runtime service table to the new virtual addresses.
 757          *
 758          * Call EFI services through wrapper functions.
 759          */
 760         efi.runtime_version = efi_systab.hdr.revision;
 761
 762         efi_native_runtime_setup();
 763
 764         efi.set_virtual_address_map = NULL;
 765
 766         if (efi_enabled(EFI_OLD_MEMMAP) && (__supported_pte_mask & _PAGE_NX))
 767                 runtime_code_page_mkexec();
 768
 769         /* clean DUMMY object */
 770         efi_delete_dummy_variable();
 771 #endif
 772 }
 773
 774 /*
 775  * This function will switch the EFI runtime services to virtual mode.
 776  * Essentially, we look through the EFI memmap and map every region that
 777  * has the runtime attribute bit set in its memory descriptor into the
 778  * ->trampoline_pgd page table using a top-down VA allocation scheme.
 779  *
 780  * The old method which used to update that memory descriptor with the
 781  * virtual address obtained from ioremap() is still supported when the
 782  * kernel is booted with efi=old_map on its command line. Same old
 783  * method enabled the runtime services to be called without having to
 784  * thunk back into physical mode for every invocation.
 785  *
 786  * The new method does a pagetable switch in a preemption-safe manner
 787  * so that we're in a different address space when calling a runtime
 788  * function. For function arguments passing we do copy the PGDs of the
 789  * kernel page table into ->trampoline_pgd prior to each call.
 790  *
 791  * Specially for kexec boot, efi runtime maps in previous kernel should
 792  * be passed in via setup_data. In that case runtime ranges will be mapped
 793  * to the same virtual addresses as the first kernel, see
 794  * kexec_enter_virtual_mode().
 795  */
 796 static void __init __efi_enter_virtual_mode(void)
 797 {
 798         int count = 0, pg_shift = 0;
 799         void *new_memmap = NULL;
 800         efi_status_t status;
 801
 802         efi.systab = NULL;
 803
 804         efi_merge_regions();
 805         new_memmap = efi_map_regions(&count, &pg_shift);
 806         if (!new_memmap) {
 807                 pr_err("Error reallocating memory, EFI runtime non-functional!\n");
 808                 return;
 809         }
 810
 811         save_runtime_map();
 812
 813         BUG_ON(!efi.systab);
 814
 815         if (efi_setup_page_tables(__pa(new_memmap), 1 << pg_shift))
 816                 return;
 817
 818         efi_sync_low_kernel_mappings();
 819         efi_dump_pagetable();
 820
 821         if (efi_is_native()) {
 822                 status = phys_efi_set_virtual_address_map(
 823                                 memmap.desc_size * count,
 824                                 memmap.desc_size,
 825                                 memmap.desc_version,
 826                                 (efi_memory_desc_t *)__pa(new_memmap));
 827         } else {
 828                 status = efi_thunk_set_virtual_address_map(
 829                                 efi_phys.set_virtual_address_map,
 830                                 memmap.desc_size * count,
 831                                 memmap.desc_size,
 832                                 memmap.desc_version,
 833                                 (efi_memory_desc_t *)__pa(new_memmap));
 834         }
 835
 836         if (status != EFI_SUCCESS) {
 837                 pr_alert("Unable to switch EFI into virtual mode (status=%lx)!\n",
 838                          status);
 839                 panic("EFI call to SetVirtualAddressMap() failed!");
 840         }
 841
 842         /*
 843          * Now that EFI is in virtual mode, update the function
 844          * pointers in the runtime service table to the new virtual addresses.
 845          *
 846          * Call EFI services through wrapper functions.
 847          */
 848         efi.runtime_version = efi_systab.hdr.revision;
 849
 850         if (efi_is_native())
 851                 efi_native_runtime_setup();
 852         else
 853                 efi_thunk_runtime_setup();
 854
 855         efi.set_virtual_address_map = NULL;
 856
 857         efi_runtime_mkexec();
 858
 859         /*
 860          * We mapped the descriptor array into the EFI pagetable above but we're
 861          * not unmapping it here. Here's why:
 862          *
 863          * We're copying select PGDs from the kernel page table to the EFI page
 864          * table and when we do so and make changes to those PGDs like unmapping
 865          * stuff from them, those changes appear in the kernel page table and we
 866          * go boom.
 867          *
 868          * From setup_real_mode():
 869          *
 870          * ...
 871          * trampoline_pgd[0] = init_level4_pgt[pgd_index(__PAGE_OFFSET)].pgd;
 872          *
 873          * In this particular case, our allocation is in PGD 0 of the EFI page
 874          * table but we've copied that PGD from PGD[272] of the EFI page table:
 875          *
 876          *      pgd_index(__PAGE_OFFSET = 0xffff880000000000) = 272
 877          *
 878          * where the direct memory mapping in kernel space is.
 879          *
 880          * new_memmap's VA comes from that direct mapping and thus clearing it,
 881          * it would get cleared in the kernel page table too.
 882          *
 883          * efi_cleanup_page_tables(__pa(new_memmap), 1 << pg_shift);
 884          */
 885         free_pages((unsigned long)new_memmap, pg_shift);
 886
 887         /* clean DUMMY object */
 888         efi_delete_dummy_variable();
 889 }
 890
 891 void __init efi_enter_virtual_mode(void)
 892 {
 893         if (efi_enabled(EFI_PARAVIRT))
 894                 return;
 895
 896         if (efi_setup)
 897                 kexec_enter_virtual_mode();
 898         else
 899                 __efi_enter_virtual_mode();
 900 }
 901
 902 /*
 903  * Convenience functions to obtain memory types and attributes
 904  */
 905 u32 efi_mem_type(unsigned long phys_addr)
 906 {
 907         efi_memory_desc_t *md;
 908         void *p;
 909
 910         if (!efi_enabled(EFI_MEMMAP))
 911                 return 0;
 912
 913         for (p = memmap.map; p < memmap.map_end; p += memmap.desc_size) {
 914                 md = p;
 915                 if ((md->phys_addr <= phys_addr) &&
 916                     (phys_addr < (md->phys_addr +
 917                                   (md->num_pages << EFI_PAGE_SHIFT))))
 918                         return md->type;
 919         }
 920         return 0;
 921 }
 922
 923 u64 efi_mem_attributes(unsigned long phys_addr)
 924 {
 925         efi_memory_desc_t *md;
 926         void *p;
 927
 928         if (!efi_enabled(EFI_MEMMAP))
 929                 return 0;
 930
 931         for (p = memmap.map; p < memmap.map_end; p += memmap.desc_size) {
 932                 md = p;
 933                 if ((md->phys_addr <= phys_addr) &&
 934                     (phys_addr < (md->phys_addr +
 935                                   (md->num_pages << EFI_PAGE_SHIFT))))
 936                         return md->attribute;
 937         }
 938         return 0;
 939 }
 940
 941 static int __init parse_efi_cmdline(char *str)
 942 {
 943         if (*str == '=')
 944                 str++;
 945
 946         if (!strncmp(str, "old_map", 7))
 947                 set_bit(EFI_OLD_MEMMAP, &efi.flags);
 948
 949         return 0;
 950 }
 951 early_param("efi", parse_efi_cmdline);