arch/x86/platform/efi/efi.c

   1 /*
   2  * Common EFI (Extensible Firmware Interface) support functions
   3  * Based on Extensible Firmware Interface Specification version 1.0
   4  *
   5  * Copyright (C) 1999 VA Linux Systems
   6  * Copyright (C) 1999 Walt Drummond <drummond@valinux.com>
   7  * Copyright (C) 1999-2002 Hewlett-Packard Co.
   8  *      David Mosberger-Tang <davidm@hpl.hp.com>
   9  *      Stephane Eranian <eranian@hpl.hp.com>
  10  * Copyright (C) 2005-2008 Intel Co.
  11  *      Fenghua Yu <fenghua.yu@intel.com>
  12  *      Bibo Mao <bibo.mao@intel.com>
  13  *      Chandramouli Narayanan <mouli@linux.intel.com>
  14  *      Huang Ying <ying.huang@intel.com>
  15  * Copyright (C) 2013 SuSE Labs
  16  *      Borislav Petkov <bp@suse.de> - runtime services VA mapping
  17  *
  18  * Copied from efi_32.c to eliminate the duplicated code between EFI
  19  * 32/64 support code. --ying 2007-10-26
  20  *
  21  * All EFI Runtime Services are not implemented yet as EFI only
  22  * supports physical mode addressing on SoftSDV. This is to be fixed
  23  * in a future version.  --drummond 1999-07-20
  24  *
  25  * Implemented EFI runtime services and virtual mode calls.  --davidm
  26  *
  27  * Goutham Rao: <goutham.rao@intel.com>
  28  *      Skip non-WB memory and ignore empty memory ranges.
  29  */
  30
  31 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  32
  33 #include <linux/kernel.h>
  34 #include <linux/init.h>
  35 #include <linux/efi.h>
  36 #include <linux/efi-bgrt.h>
  37 #include <linux/export.h>
  38 #include <linux/bootmem.h>
  39 #include <linux/slab.h>
  40 #include <linux/memblock.h>
  41 #include <linux/spinlock.h>
  42 #include <linux/uaccess.h>
  43 #include <linux/time.h>
  44 #include <linux/io.h>
  45 #include <linux/reboot.h>
  46 #include <linux/bcd.h>
  47
  48 #include <asm/setup.h>
  49 #include <asm/efi.h>
  50 #include <asm/time.h>
  51 #include <asm/cacheflush.h>
  52 #include <asm/tlbflush.h>
  53 #include <asm/x86_init.h>
  54 #include <asm/rtc.h>
  55 #include <asm/uv/uv.h>
  56
  57 #define EFI_DEBUG
  58
  59 struct efi_memory_map memmap;
  60
  61 static struct efi efi_phys __initdata;
  62 static efi_system_table_t efi_systab __initdata;
  63
  64 static efi_config_table_type_t arch_tables[] __initdata = {
  65 #ifdef CONFIG_X86_UV
  66         {UV_SYSTEM_TABLE_GUID, "UVsystab", &efi.uv_systab},
  67 #endif
  68         {NULL_GUID, NULL, NULL},
  69 };
  70
  71 u64 efi_setup;          /* efi setup_data physical address */
  72
  73 int add_efi_memmap;
  74 EXPORT_SYMBOL(add_efi_memmap);
  75
  76 static int __init setup_add_efi_memmap(char *arg)
  77 {
  78         add_efi_memmap = 1;
  79         return 0;
  80 }
  81 early_param("add_efi_memmap", setup_add_efi_memmap);
  82
  83 static efi_status_t __init phys_efi_set_virtual_address_map(
  84         unsigned long memory_map_size,
  85         unsigned long descriptor_size,
  86         u32 descriptor_version,
  87         efi_memory_desc_t *virtual_map)
  88 {
  89         efi_status_t status;
  90
  91         efi_call_phys_prelog();
  92         status = efi_call_phys(efi_phys.set_virtual_address_map,
  93                                memory_map_size, descriptor_size,
  94                                descriptor_version, virtual_map);
  95         efi_call_phys_epilog();
  96         return status;
  97 }
  98
  99 void efi_get_time(struct timespec *now)
 100 {
 101         efi_status_t status;
 102         efi_time_t eft;
 103         efi_time_cap_t cap;
 104
 105         status = efi.get_time(&eft, &cap);
 106         if (status != EFI_SUCCESS)
 107                 pr_err("Oops: efitime: can't read time!\n");
 108
 109         now->tv_sec = mktime(eft.year, eft.month, eft.day, eft.hour,
 110                              eft.minute, eft.second);
 111         now->tv_nsec = 0;
 112 }
 113
 114 /*
 115  * Tell the kernel about the EFI memory map.  This might include
 116  * more than the max 128 entries that can fit in the e820 legacy
 117  * (zeropage) memory map.
 118  */
 119
 120 static void __init do_add_efi_memmap(void)
 121 {
 122         void *p;
 123
 124         for (p = memmap.map; p < memmap.map_end; p += memmap.desc_size) {
 125                 efi_memory_desc_t *md = p;
 126                 unsigned long long start = md->phys_addr;
 127                 unsigned long long size = md->num_pages << EFI_PAGE_SHIFT;
 128                 int e820_type;
 129
 130                 switch (md->type) {
 131                 case EFI_LOADER_CODE:
 132                 case EFI_LOADER_DATA:
 133                 case EFI_BOOT_SERVICES_CODE:
 134                 case EFI_BOOT_SERVICES_DATA:
 135                 case EFI_CONVENTIONAL_MEMORY:
 136                         if (md->attribute & EFI_MEMORY_WB)
 137                                 e820_type = E820_RAM;
 138                         else
 139                                 e820_type = E820_RESERVED;
 140                         break;
 141                 case EFI_ACPI_RECLAIM_MEMORY:
 142                         e820_type = E820_ACPI;
 143                         break;
 144                 case EFI_ACPI_MEMORY_NVS:
 145                         e820_type = E820_NVS;
 146                         break;
 147                 case EFI_UNUSABLE_MEMORY:
 148                         e820_type = E820_UNUSABLE;
 149                         break;
 150                 default:
 151                         /*
 152                          * EFI_RESERVED_TYPE EFI_RUNTIME_SERVICES_CODE
 153                          * EFI_RUNTIME_SERVICES_DATA EFI_MEMORY_MAPPED_IO
 154                          * EFI_MEMORY_MAPPED_IO_PORT_SPACE EFI_PAL_CODE
 155                          */
 156                         e820_type = E820_RESERVED;
 157                         break;
 158                 }
 159                 e820_add_region(start, size, e820_type);
 160         }
 161         sanitize_e820_map(e820.map, ARRAY_SIZE(e820.map), &e820.nr_map);
 162 }
 163
 164 int __init efi_memblock_x86_reserve_range(void)
 165 {
 166         struct efi_info *e = &boot_params.efi_info;
 167         unsigned long pmap;
 168
 169         if (efi_enabled(EFI_PARAVIRT))
 170                 return 0;
 171
 172 #ifdef CONFIG_X86_32
 173         /* Can't handle data above 4GB at this time */
 174         if (e->efi_memmap_hi) {
 175                 pr_err("Memory map is above 4GB, disabling EFI.\n");
 176                 return -EINVAL;
 177         }
 178         pmap =  e->efi_memmap;
 179 #else
 180         pmap = (e->efi_memmap | ((__u64)e->efi_memmap_hi << 32));
 181 #endif
 182         memmap.phys_map         = (void *)pmap;
 183         memmap.nr_map           = e->efi_memmap_size /
 184                                   e->efi_memdesc_size;
 185         memmap.desc_size        = e->efi_memdesc_size;
 186         memmap.desc_version     = e->efi_memdesc_version;
 187
 188         memblock_reserve(pmap, memmap.nr_map * memmap.desc_size);
 189
 190         efi.memmap = &memmap;
 191
 192         return 0;
 193 }
 194
 195 static void __init print_efi_memmap(void)
 196 {
 197 #ifdef EFI_DEBUG
 198         efi_memory_desc_t *md;
 199         void *p;
 200         int i;
 201
 202         for (p = memmap.map, i = 0;
 203              p < memmap.map_end;
 204              p += memmap.desc_size, i++) {
 205                 char buf[64];
 206
 207                 md = p;
 208                 pr_info("mem%02u: %s range=[0x%016llx-0x%016llx) (%lluMB)\n",
 209                         i, efi_md_typeattr_format(buf, sizeof(buf), md),
 210                         md->phys_addr,
 211                         md->phys_addr + (md->num_pages << EFI_PAGE_SHIFT),
 212                         (md->num_pages >> (20 - EFI_PAGE_SHIFT)));
 213         }
 214 #endif  /*  EFI_DEBUG  */
 215 }
 216
 217 void __init efi_unmap_memmap(void)
 218 {
 219         clear_bit(EFI_MEMMAP, &efi.flags);
 220         if (memmap.map) {
 221                 early_memunmap(memmap.map, memmap.nr_map * memmap.desc_size);
 222                 memmap.map = NULL;
 223         }
 224 }
 225
 226 static int __init efi_systab_init(void *phys)
 227 {
 228         if (efi_enabled(EFI_64BIT)) {
 229                 efi_system_table_64_t *systab64;
 230                 struct efi_setup_data *data = NULL;
 231                 u64 tmp = 0;
 232
 233                 if (efi_setup) {
 234                         data = early_memremap(efi_setup, sizeof(*data));
 235                         if (!data)
 236                                 return -ENOMEM;
 237                 }
 238                 systab64 = early_memremap((unsigned long)phys,
 239                                          sizeof(*systab64));
 240                 if (systab64 == NULL) {
 241                         pr_err("Couldn't map the system table!\n");
 242                         if (data)
 243                                 early_memunmap(data, sizeof(*data));
 244                         return -ENOMEM;
 245                 }
 246
 247                 efi_systab.hdr = systab64->hdr;
 248                 efi_systab.fw_vendor = data ? (unsigned long)data->fw_vendor :
 249                                               systab64->fw_vendor;
 250                 tmp |= data ? data->fw_vendor : systab64->fw_vendor;
 251                 efi_systab.fw_revision = systab64->fw_revision;
 252                 efi_systab.con_in_handle = systab64->con_in_handle;
 253                 tmp |= systab64->con_in_handle;
 254                 efi_systab.con_in = systab64->con_in;
 255                 tmp |= systab64->con_in;
 256                 efi_systab.con_out_handle = systab64->con_out_handle;
 257                 tmp |= systab64->con_out_handle;
 258                 efi_systab.con_out = systab64->con_out;
 259                 tmp |= systab64->con_out;
 260                 efi_systab.stderr_handle = systab64->stderr_handle;
 261                 tmp |= systab64->stderr_handle;
 262                 efi_systab.stderr = systab64->stderr;
 263                 tmp |= systab64->stderr;
 264                 efi_systab.runtime = data ?
 265                                      (void *)(unsigned long)data->runtime :
 266                                      (void *)(unsigned long)systab64->runtime;
 267                 tmp |= data ? data->runtime : systab64->runtime;
 268                 efi_systab.boottime = (void *)(unsigned long)systab64->boottime;
 269                 tmp |= systab64->boottime;
 270                 efi_systab.nr_tables = systab64->nr_tables;
 271                 efi_systab.tables = data ? (unsigned long)data->tables :
 272                                            systab64->tables;
 273                 tmp |= data ? data->tables : systab64->tables;
 274
 275                 early_memunmap(systab64, sizeof(*systab64));
 276                 if (data)
 277                         early_memunmap(data, sizeof(*data));
 278 #ifdef CONFIG_X86_32
 279                 if (tmp >> 32) {
 280                         pr_err("EFI data located above 4GB, disabling EFI.\n");
 281                         return -EINVAL;
 282                 }
 283 #endif
 284         } else {
 285                 efi_system_table_32_t *systab32;
 286
 287                 systab32 = early_memremap((unsigned long)phys,
 288                                          sizeof(*systab32));
 289                 if (systab32 == NULL) {
 290                         pr_err("Couldn't map the system table!\n");
 291                         return -ENOMEM;
 292                 }
 293
 294                 efi_systab.hdr = systab32->hdr;
 295                 efi_systab.fw_vendor = systab32->fw_vendor;
 296                 efi_systab.fw_revision = systab32->fw_revision;
 297                 efi_systab.con_in_handle = systab32->con_in_handle;
 298                 efi_systab.con_in = systab32->con_in;
 299                 efi_systab.con_out_handle = systab32->con_out_handle;
 300                 efi_systab.con_out = systab32->con_out;
 301                 efi_systab.stderr_handle = systab32->stderr_handle;
 302                 efi_systab.stderr = systab32->stderr;
 303                 efi_systab.runtime = (void *)(unsigned long)systab32->runtime;
 304                 efi_systab.boottime = (void *)(unsigned long)systab32->boottime;
 305                 efi_systab.nr_tables = systab32->nr_tables;
 306                 efi_systab.tables = systab32->tables;
 307
 308                 early_memunmap(systab32, sizeof(*systab32));
 309         }
 310
 311         efi.systab = &efi_systab;
 312
 313         /*
 314          * Verify the EFI Table
 315          */
 316         if (efi.systab->hdr.signature != EFI_SYSTEM_TABLE_SIGNATURE) {
 317                 pr_err("System table signature incorrect!\n");
 318                 return -EINVAL;
 319         }
 320         if ((efi.systab->hdr.revision >> 16) == 0)
 321                 pr_err("Warning: System table version %d.%02d, expected 1.00 or greater!\n",
 322                        efi.systab->hdr.revision >> 16,
 323                        efi.systab->hdr.revision & 0xffff);
 324
 325         set_bit(EFI_SYSTEM_TABLES, &efi.flags);
 326
 327         return 0;
 328 }
 329
 330 static int __init efi_runtime_init32(void)
 331 {
 332         efi_runtime_services_32_t *runtime;
 333
 334         runtime = early_memremap((unsigned long)efi.systab->runtime,
 335                         sizeof(efi_runtime_services_32_t));
 336         if (!runtime) {
 337                 pr_err("Could not map the runtime service table!\n");
 338                 return -ENOMEM;
 339         }
 340
 341         /*
 342          * We will only need *early* access to the following two
 343          * EFI runtime services before set_virtual_address_map
 344          * is invoked.
 345          */
 346         efi_phys.set_virtual_address_map =
 347                         (efi_set_virtual_address_map_t *)
 348                         (unsigned long)runtime->set_virtual_address_map;
 349         early_memunmap(runtime, sizeof(efi_runtime_services_32_t));
 350
 351         return 0;
 352 }
 353
 354 static int __init efi_runtime_init64(void)
 355 {
 356         efi_runtime_services_64_t *runtime;
 357
 358         runtime = early_memremap((unsigned long)efi.systab->runtime,
 359                         sizeof(efi_runtime_services_64_t));
 360         if (!runtime) {
 361                 pr_err("Could not map the runtime service table!\n");
 362                 return -ENOMEM;
 363         }
 364
 365         /*
 366          * We will only need *early* access to the following two
 367          * EFI runtime services before set_virtual_address_map
 368          * is invoked.
 369          */
 370         efi_phys.set_virtual_address_map =
 371                         (efi_set_virtual_address_map_t *)
 372                         (unsigned long)runtime->set_virtual_address_map;
 373         early_memunmap(runtime, sizeof(efi_runtime_services_64_t));
 374
 375         return 0;
 376 }
 377
 378 static int __init efi_runtime_init(void)
 379 {
 380         int rv;
 381
 382         /*
 383          * Check out the runtime services table. We need to map
 384          * the runtime services table so that we can grab the physical
 385          * address of several of the EFI runtime functions, needed to
 386          * set the firmware into virtual mode.
 387          *
 388          * When EFI_PARAVIRT is in force then we could not map runtime
 389          * service memory region because we do not have direct access to it.
 390          * However, runtime services are available through proxy functions
 391          * (e.g. in case of Xen dom0 EFI implementation they call special
 392          * hypercall which executes relevant EFI functions) and that is why
 393          * they are always enabled.
 394          */
 395
 396         if (!efi_enabled(EFI_PARAVIRT)) {
 397                 if (efi_enabled(EFI_64BIT))
 398                         rv = efi_runtime_init64();
 399                 else
 400                         rv = efi_runtime_init32();
 401
 402                 if (rv)
 403                         return rv;
 404         }
 405
 406         set_bit(EFI_RUNTIME_SERVICES, &efi.flags);
 407
 408         return 0;
 409 }
 410
 411 static int __init efi_memmap_init(void)
 412 {
 413         if (efi_enabled(EFI_PARAVIRT))
 414                 return 0;
 415
 416         /* Map the EFI memory map */
 417         memmap.map = early_memremap((unsigned long)memmap.phys_map,
 418                                    memmap.nr_map * memmap.desc_size);
 419         if (memmap.map == NULL) {
 420                 pr_err("Could not map the memory map!\n");
 421                 return -ENOMEM;
 422         }
 423         memmap.map_end = memmap.map + (memmap.nr_map * memmap.desc_size);
 424
 425         if (add_efi_memmap)
 426                 do_add_efi_memmap();
 427
 428         set_bit(EFI_MEMMAP, &efi.flags);
 429
 430         return 0;
 431 }
 432
 433 void __init efi_init(void)
 434 {
 435         efi_char16_t *c16;
 436         char vendor[100] = "unknown";
 437         int i = 0;
 438         void *tmp;
 439
 440 #ifdef CONFIG_X86_32
 441         if (boot_params.efi_info.efi_systab_hi ||
 442             boot_params.efi_info.efi_memmap_hi) {
 443                 pr_info("Table located above 4GB, disabling EFI.\n");
 444                 return;
 445         }
 446         efi_phys.systab = (efi_system_table_t *)boot_params.efi_info.efi_systab;
 447 #else
 448         efi_phys.systab = (efi_system_table_t *)
 449                           (boot_params.efi_info.efi_systab |
 450                           ((__u64)boot_params.efi_info.efi_systab_hi<<32));
 451 #endif
 452
 453         if (efi_systab_init(efi_phys.systab))
 454                 return;
 455
 456         efi.config_table = (unsigned long)efi.systab->tables;
 457         efi.fw_vendor    = (unsigned long)efi.systab->fw_vendor;
 458         efi.runtime      = (unsigned long)efi.systab->runtime;
 459
 460         /*
 461          * Show what we know for posterity
 462          */
 463         c16 = tmp = early_memremap(efi.systab->fw_vendor, 2);
 464         if (c16) {
 465                 for (i = 0; i < sizeof(vendor) - 1 && *c16; ++i)
 466                         vendor[i] = *c16++;
 467                 vendor[i] = '\0';
 468         } else
 469                 pr_err("Could not map the firmware vendor!\n");
 470         early_memunmap(tmp, 2);
 471
 472         pr_info("EFI v%u.%.02u by %s\n",
 473                 efi.systab->hdr.revision >> 16,
 474                 efi.systab->hdr.revision & 0xffff, vendor);
 475
 476         if (efi_reuse_config(efi.systab->tables, efi.systab->nr_tables))
 477                 return;
 478
 479         if (efi_config_init(arch_tables))
 480                 return;
 481
 482         /*
 483          * Note: We currently don't support runtime services on an EFI
 484          * that doesn't match the kernel 32/64-bit mode.
 485          */
 486
 487         if (!efi_runtime_supported())
 488                 pr_info("No EFI runtime due to 32/64-bit mismatch with kernel\n");
 489         else {
 490                 if (efi_runtime_disabled() || efi_runtime_init())
 491                         return;
 492         }
 493         if (efi_memmap_init())
 494                 return;
 495
 496         print_efi_memmap();
 497 }
 498
 499 void __init efi_late_init(void)
 500 {
 501         efi_bgrt_init();
 502 }
 503
 504 void __init efi_set_executable(efi_memory_desc_t *md, bool executable)
 505 {
 506         u64 addr, npages;
 507
 508         addr = md->virt_addr;
 509         npages = md->num_pages;
 510
 511         memrange_efi_to_native(&addr, &npages);
 512
 513         if (executable)
 514                 set_memory_x(addr, npages);
 515         else
 516                 set_memory_nx(addr, npages);
 517 }
 518
 519 void __init runtime_code_page_mkexec(void)
 520 {
 521         efi_memory_desc_t *md;
 522         void *p;
 523
 524         /* Make EFI runtime service code area executable */
 525         for (p = memmap.map; p < memmap.map_end; p += memmap.desc_size) {
 526                 md = p;
 527
 528                 if (md->type != EFI_RUNTIME_SERVICES_CODE)
 529                         continue;
 530
 531                 efi_set_executable(md, true);
 532         }
 533 }
 534
 535 void efi_memory_uc(u64 addr, unsigned long size)
 536 {
 537         unsigned long page_shift = 1UL << EFI_PAGE_SHIFT;
 538         u64 npages;
 539
 540         npages = round_up(size, page_shift) / page_shift;
 541         memrange_efi_to_native(&addr, &npages);
 542         set_memory_uc(addr, npages);
 543 }
 544
 545 void __init old_map_region(efi_memory_desc_t *md)
 546 {
 547         u64 start_pfn, end_pfn, end;
 548         unsigned long size;
 549         void *va;
 550
 551         start_pfn = PFN_DOWN(md->phys_addr);
 552         size      = md->num_pages << PAGE_SHIFT;
 553         end       = md->phys_addr + size;
 554         end_pfn   = PFN_UP(end);
 555
 556         if (pfn_range_is_mapped(start_pfn, end_pfn)) {
 557                 va = __va(md->phys_addr);
 558
 559                 if (!(md->attribute & EFI_MEMORY_WB))
 560                         efi_memory_uc((u64)(unsigned long)va, size);
 561         } else
 562                 va = efi_ioremap(md->phys_addr, size,
 563                                  md->type, md->attribute);
 564
 565         md->virt_addr = (u64) (unsigned long) va;
 566         if (!va)
 567                 pr_err("ioremap of 0x%llX failed!\n",
 568                        (unsigned long long)md->phys_addr);
 569 }
 570
 571 /* Merge contiguous regions of the same type and attribute */
 572 static void __init efi_merge_regions(void)
 573 {
 574         void *p;
 575         efi_memory_desc_t *md, *prev_md = NULL;
 576
 577         for (p = memmap.map; p < memmap.map_end; p += memmap.desc_size) {
 578                 u64 prev_size;
 579                 md = p;
 580
 581                 if (!prev_md) {
 582                         prev_md = md;
 583                         continue;
 584                 }
 585
 586                 if (prev_md->type != md->type ||
 587                     prev_md->attribute != md->attribute) {
 588                         prev_md = md;
 589                         continue;
 590                 }
 591
 592                 prev_size = prev_md->num_pages << EFI_PAGE_SHIFT;
 593
 594                 if (md->phys_addr == (prev_md->phys_addr + prev_size)) {
 595                         prev_md->num_pages += md->num_pages;
 596                         md->type = EFI_RESERVED_TYPE;
 597                         md->attribute = 0;
 598                         continue;
 599                 }
 600                 prev_md = md;
 601         }
 602 }
 603
 604 static void __init get_systab_virt_addr(efi_memory_desc_t *md)
 605 {
 606         unsigned long size;
 607         u64 end, systab;
 608
 609         size = md->num_pages << EFI_PAGE_SHIFT;
 610         end = md->phys_addr + size;
 611         systab = (u64)(unsigned long)efi_phys.systab;
 612         if (md->phys_addr <= systab && systab < end) {
 613                 systab += md->virt_addr - md->phys_addr;
 614                 efi.systab = (efi_system_table_t *)(unsigned long)systab;
 615         }
 616 }
 617
 618 static void __init save_runtime_map(void)
 619 {
 620 #ifdef CONFIG_KEXEC
 621         efi_memory_desc_t *md;
 622         void *tmp, *p, *q = NULL;
 623         int count = 0;
 624
 625         if (efi_enabled(EFI_OLD_MEMMAP))
 626                 return;
 627
 628         for (p = memmap.map; p < memmap.map_end; p += memmap.desc_size) {
 629                 md = p;
 630
 631                 if (!(md->attribute & EFI_MEMORY_RUNTIME) ||
 632                     (md->type == EFI_BOOT_SERVICES_CODE) ||
 633                     (md->type == EFI_BOOT_SERVICES_DATA))
 634                         continue;
 635                 tmp = krealloc(q, (count + 1) * memmap.desc_size, GFP_KERNEL);
 636                 if (!tmp)
 637                         goto out;
 638                 q = tmp;
 639
 640                 memcpy(q + count * memmap.desc_size, md, memmap.desc_size);
 641                 count++;
 642         }
 643
 644         efi_runtime_map_setup(q, count, memmap.desc_size);
 645         return;
 646
 647 out:
 648         kfree(q);
 649         pr_err("Error saving runtime map, efi runtime on kexec non-functional!!\n");
 650 #endif
 651 }
 652
 653 static void *realloc_pages(void *old_memmap, int old_shift)
 654 {
 655         void *ret;
 656
 657         ret = (void *)__get_free_pages(GFP_KERNEL, old_shift + 1);
 658         if (!ret)
 659                 goto out;
 660
 661         /*
 662          * A first-time allocation doesn't have anything to copy.
 663          */
 664         if (!old_memmap)
 665                 return ret;
 666
 667         memcpy(ret, old_memmap, PAGE_SIZE << old_shift);
 668
 669 out:
 670         free_pages((unsigned long)old_memmap, old_shift);
 671         return ret;
 672 }
 673
 674 /*
 675  * Map the efi memory ranges of the runtime services and update new_mmap with
 676  * virtual addresses.
 677  */
 678 static void * __init efi_map_regions(int *count, int *pg_shift)
 679 {
 680         void *p, *new_memmap = NULL;
 681         unsigned long left = 0;
 682         efi_memory_desc_t *md;
 683
 684         for (p = memmap.map; p < memmap.map_end; p += memmap.desc_size) {
 685                 md = p;
 686                 if (!(md->attribute & EFI_MEMORY_RUNTIME)) {
 687 #ifdef CONFIG_X86_64
 688                         if (md->type != EFI_BOOT_SERVICES_CODE &&
 689                             md->type != EFI_BOOT_SERVICES_DATA)
 690 #endif
 691                                 continue;
 692                 }
 693
 694                 efi_map_region(md);
 695                 get_systab_virt_addr(md);
 696
 697                 if (left < memmap.desc_size) {
 698                         new_memmap = realloc_pages(new_memmap, *pg_shift);
 699                         if (!new_memmap)
 700                                 return NULL;
 701
 702                         left += PAGE_SIZE << *pg_shift;
 703                         (*pg_shift)++;
 704                 }
 705
 706                 memcpy(new_memmap + (*count * memmap.desc_size), md,
 707                        memmap.desc_size);
 708
 709                 left -= memmap.desc_size;
 710                 (*count)++;
 711         }
 712
 713         return new_memmap;
 714 }
 715
 716 static void __init kexec_enter_virtual_mode(void)
 717 {
 718 #ifdef CONFIG_KEXEC
 719         efi_memory_desc_t *md;
 720         void *p;
 721
 722         efi.systab = NULL;
 723
 724         /*
 725          * We don't do virtual mode, since we don't do runtime services, on
 726          * non-native EFI
 727          */
 728         if (!efi_is_native()) {
 729                 efi_unmap_memmap();
 730                 clear_bit(EFI_RUNTIME_SERVICES, &efi.flags);
 731                 return;
 732         }
 733
 734         /*
 735         * Map efi regions which were passed via setup_data. The virt_addr is a
 736         * fixed addr which was used in first kernel of a kexec boot.
 737         */
 738         for (p = memmap.map; p < memmap.map_end; p += memmap.desc_size) {
 739                 md = p;
 740                 efi_map_region_fixed(md); /* FIXME: add error handling */
 741                 get_systab_virt_addr(md);
 742         }
 743
 744         save_runtime_map();
 745
 746         BUG_ON(!efi.systab);
 747
 748         efi_sync_low_kernel_mappings();
 749
 750         /*
 751          * Now that EFI is in virtual mode, update the function
 752          * pointers in the runtime service table to the new virtual addresses.
 753          *
 754          * Call EFI services through wrapper functions.
 755          */
 756         efi.runtime_version = efi_systab.hdr.revision;
 757
 758         efi_native_runtime_setup();
 759
 760         efi.set_virtual_address_map = NULL;
 761
 762         if (efi_enabled(EFI_OLD_MEMMAP) && (__supported_pte_mask & _PAGE_NX))
 763                 runtime_code_page_mkexec();
 764
 765         /* clean DUMMY object */
 766         efi_delete_dummy_variable();
 767 #endif
 768 }
 769
 770 /*
 771  * This function will switch the EFI runtime services to virtual mode.
 772  * Essentially, we look through the EFI memmap and map every region that
 773  * has the runtime attribute bit set in its memory descriptor into the
 774  * ->trampoline_pgd page table using a top-down VA allocation scheme.
 775  *
 776  * The old method which used to update that memory descriptor with the
 777  * virtual address obtained from ioremap() is still supported when the
 778  * kernel is booted with efi=old_map on its command line. Same old
 779  * method enabled the runtime services to be called without having to
 780  * thunk back into physical mode for every invocation.
 781  *
 782  * The new method does a pagetable switch in a preemption-safe manner
 783  * so that we're in a different address space when calling a runtime
 784  * function. For function arguments passing we do copy the PGDs of the
 785  * kernel page table into ->trampoline_pgd prior to each call.
 786  *
 787  * Specially for kexec boot, efi runtime maps in previous kernel should
 788  * be passed in via setup_data. In that case runtime ranges will be mapped
 789  * to the same virtual addresses as the first kernel, see
 790  * kexec_enter_virtual_mode().
 791  */
 792 static void __init __efi_enter_virtual_mode(void)
 793 {
 794         int count = 0, pg_shift = 0;
 795         void *new_memmap = NULL;
 796         efi_status_t status;
 797
 798         efi.systab = NULL;
 799
 800         efi_merge_regions();
 801         new_memmap = efi_map_regions(&count, &pg_shift);
 802         if (!new_memmap) {
 803                 pr_err("Error reallocating memory, EFI runtime non-functional!\n");
 804                 clear_bit(EFI_RUNTIME_SERVICES, &efi.flags);
 805                 return;
 806         }
 807
 808         save_runtime_map();
 809
 810         BUG_ON(!efi.systab);
 811
 812         if (efi_setup_page_tables(__pa(new_memmap), 1 << pg_shift)) {
 813                 clear_bit(EFI_RUNTIME_SERVICES, &efi.flags);
 814                 return;
 815         }
 816
 817         efi_sync_low_kernel_mappings();
 818         efi_dump_pagetable();
 819
 820         if (efi_is_native()) {
 821                 status = phys_efi_set_virtual_address_map(
 822                                 memmap.desc_size * count,
 823                                 memmap.desc_size,
 824                                 memmap.desc_version,
 825                                 (efi_memory_desc_t *)__pa(new_memmap));
 826         } else {
 827                 status = efi_thunk_set_virtual_address_map(
 828                                 efi_phys.set_virtual_address_map,
 829                                 memmap.desc_size * count,
 830                                 memmap.desc_size,
 831                                 memmap.desc_version,
 832                                 (efi_memory_desc_t *)__pa(new_memmap));
 833         }
 834
 835         if (status != EFI_SUCCESS) {
 836                 pr_alert("Unable to switch EFI into virtual mode (status=%lx)!\n",
 837                          status);
 838                 panic("EFI call to SetVirtualAddressMap() failed!");
 839         }
 840
 841         /*
 842          * Now that EFI is in virtual mode, update the function
 843          * pointers in the runtime service table to the new virtual addresses.
 844          *
 845          * Call EFI services through wrapper functions.
 846          */
 847         efi.runtime_version = efi_systab.hdr.revision;
 848
 849         if (efi_is_native())
 850                 efi_native_runtime_setup();
 851         else
 852                 efi_thunk_runtime_setup();
 853
 854         efi.set_virtual_address_map = NULL;
 855
 856         efi_runtime_mkexec();
 857
 858         /*
 859          * We mapped the descriptor array into the EFI pagetable above but we're
 860          * not unmapping it here. Here's why:
 861          *
 862          * We're copying select PGDs from the kernel page table to the EFI page
 863          * table and when we do so and make changes to those PGDs like unmapping
 864          * stuff from them, those changes appear in the kernel page table and we
 865          * go boom.
 866          *
 867          * From setup_real_mode():
 868          *
 869          * ...
 870          * trampoline_pgd[0] = init_level4_pgt[pgd_index(__PAGE_OFFSET)].pgd;
 871          *
 872          * In this particular case, our allocation is in PGD 0 of the EFI page
 873          * table but we've copied that PGD from PGD[272] of the EFI page table:
 874          *
 875          *      pgd_index(__PAGE_OFFSET = 0xffff880000000000) = 272
 876          *
 877          * where the direct memory mapping in kernel space is.
 878          *
 879          * new_memmap's VA comes from that direct mapping and thus clearing it,
 880          * it would get cleared in the kernel page table too.
 881          *
 882          * efi_cleanup_page_tables(__pa(new_memmap), 1 << pg_shift);
 883          */
 884         free_pages((unsigned long)new_memmap, pg_shift);
 885
 886         /* clean DUMMY object */
 887         efi_delete_dummy_variable();
 888 }
 889
 890 void __init efi_enter_virtual_mode(void)
 891 {
 892         if (efi_enabled(EFI_PARAVIRT))
 893                 return;
 894
 895         if (efi_setup)
 896                 kexec_enter_virtual_mode();
 897         else
 898                 __efi_enter_virtual_mode();
 899 }
 900
 901 /*
 902  * Convenience functions to obtain memory types and attributes
 903  */
 904 u32 efi_mem_type(unsigned long phys_addr)
 905 {
 906         efi_memory_desc_t *md;
 907         void *p;
 908
 909         if (!efi_enabled(EFI_MEMMAP))
 910                 return 0;
 911
 912         for (p = memmap.map; p < memmap.map_end; p += memmap.desc_size) {
 913                 md = p;
 914                 if ((md->phys_addr <= phys_addr) &&
 915                     (phys_addr < (md->phys_addr +
 916                                   (md->num_pages << EFI_PAGE_SHIFT))))
 917                         return md->type;
 918         }
 919         return 0;
 920 }
 921
 922 u64 efi_mem_attributes(unsigned long phys_addr)
 923 {
 924         efi_memory_desc_t *md;
 925         void *p;
 926
 927         if (!efi_enabled(EFI_MEMMAP))
 928                 return 0;
 929
 930         for (p = memmap.map; p < memmap.map_end; p += memmap.desc_size) {
 931                 md = p;
 932                 if ((md->phys_addr <= phys_addr) &&
 933                     (phys_addr < (md->phys_addr +
 934                                   (md->num_pages << EFI_PAGE_SHIFT))))
 935                         return md->attribute;
 936         }
 937         return 0;
 938 }
 939
 940 static int __init arch_parse_efi_cmdline(char *str)
 941 {
 942         if (parse_option_str(str, "old_map"))
 943                 set_bit(EFI_OLD_MEMMAP, &efi.flags);
 944
 945         return 0;
 946 }
 947 early_param("efi", arch_parse_efi_cmdline);