arch/i386/kernel/cpu/mtrr/generic.c

   1 /* This only handles 32bit MTRR on 32bit hosts. This is strictly wrong
   2    because MTRRs can span upto 40 bits (36bits on most modern x86) */
   3 #include <linux/init.h>
   4 #include <linux/slab.h>
   5 #include <linux/mm.h>
   6 #include <linux/module.h>
   7 #include <asm/io.h>
   8 #include <asm/mtrr.h>
   9 #include <asm/msr.h>
  10 #include <asm/system.h>
  11 #include <asm/cpufeature.h>
  12 #include <asm/tlbflush.h>
  13 #include "mtrr.h"
  14
  15 struct mtrr_state {
  16         struct mtrr_var_range *var_ranges;
  17         mtrr_type fixed_ranges[NUM_FIXED_RANGES];
  18         unsigned char enabled;
  19         unsigned char have_fixed;
  20         mtrr_type def_type;
  21 };
  22
  23 struct fixed_range_block {
  24         int base_msr; /* start address of an MTRR block */
  25         int ranges;   /* number of MTRRs in this block  */
  26 };
  27
  28 static struct fixed_range_block fixed_range_blocks[] = {
  29         { MTRRfix64K_00000_MSR, 1 }, /* one  64k MTRR  */
  30         { MTRRfix16K_80000_MSR, 2 }, /* two  16k MTRRs */
  31         { MTRRfix4K_C0000_MSR,  8 }, /* eight 4k MTRRs */
  32         {}
  33 };
  34
  35 static unsigned long smp_changes_mask;
  36 static struct mtrr_state mtrr_state = {};
  37
  38 #undef MODULE_PARAM_PREFIX
  39 #define MODULE_PARAM_PREFIX "mtrr."
  40
  41 static int mtrr_show;
  42 module_param_named(show, mtrr_show, bool, 0);
  43
  44 /*  Get the MSR pair relating to a var range  */
  45 static void
  46 get_mtrr_var_range(unsigned int index, struct mtrr_var_range *vr)
  47 {
  48         rdmsr(MTRRphysBase_MSR(index), vr->base_lo, vr->base_hi);
  49         rdmsr(MTRRphysMask_MSR(index), vr->mask_lo, vr->mask_hi);
  50 }
  51
  52 static void
  53 get_fixed_ranges(mtrr_type * frs)
  54 {
  55         unsigned int *p = (unsigned int *) frs;
  56         int i;
  57
  58         rdmsr(MTRRfix64K_00000_MSR, p[0], p[1]);
  59
  60         for (i = 0; i < 2; i++)
  61                 rdmsr(MTRRfix16K_80000_MSR + i, p[2 + i * 2], p[3 + i * 2]);
  62         for (i = 0; i < 8; i++)
  63                 rdmsr(MTRRfix4K_C0000_MSR + i, p[6 + i * 2], p[7 + i * 2]);
  64 }
  65
  66 void mtrr_save_fixed_ranges(void *info)
  67 {
  68         get_fixed_ranges(mtrr_state.fixed_ranges);
  69 }
  70
  71 static void print_fixed(unsigned base, unsigned step, const mtrr_type*types)
  72 {
  73         unsigned i;
  74
  75         for (i = 0; i < 8; ++i, ++types, base += step)
  76                 printk(KERN_INFO "MTRR %05X-%05X %s\n",
  77                         base, base + step - 1, mtrr_attrib_to_str(*types));
  78 }
  79
  80 /*  Grab all of the MTRR state for this CPU into *state  */
  81 void get_mtrr_state(void)
  82 {
  83         unsigned int i;
  84         struct mtrr_var_range *vrs;
  85         unsigned lo, dummy;
  86
  87         if (!mtrr_state.var_ranges) {
  88                 mtrr_state.var_ranges = kmalloc(num_var_ranges * sizeof (struct mtrr_var_range),
  89                                                 GFP_KERNEL);
  90                 if (!mtrr_state.var_ranges)
  91                         return;
  92         }
  93         vrs = mtrr_state.var_ranges;
  94
  95         rdmsr(MTRRcap_MSR, lo, dummy);
  96         mtrr_state.have_fixed = (lo >> 8) & 1;
  97
  98         for (i = 0; i < num_var_ranges; i++)
  99                 get_mtrr_var_range(i, &vrs[i]);
 100         if (mtrr_state.have_fixed)
 101                 get_fixed_ranges(mtrr_state.fixed_ranges);
 102
 103         rdmsr(MTRRdefType_MSR, lo, dummy);
 104         mtrr_state.def_type = (lo & 0xff);
 105         mtrr_state.enabled = (lo & 0xc00) >> 10;
 106
 107         if (mtrr_show) {
 108                 int high_width;
 109
 110                 printk(KERN_INFO "MTRR default type: %s\n", mtrr_attrib_to_str(mtrr_state.def_type));
 111                 if (mtrr_state.have_fixed) {
 112                         printk(KERN_INFO "MTRR fixed ranges %sabled:\n",
 113                                mtrr_state.enabled & 1 ? "en" : "dis");
 114                         print_fixed(0x00000, 0x10000, mtrr_state.fixed_ranges + 0);
 115                         for (i = 0; i < 2; ++i)
 116                                 print_fixed(0x80000 + i * 0x20000, 0x04000, mtrr_state.fixed_ranges + (i + 1) * 8);
 117                         for (i = 0; i < 8; ++i)
 118                                 print_fixed(0xC0000 + i * 0x08000, 0x01000, mtrr_state.fixed_ranges + (i + 3) * 8);
 119                 }
 120                 printk(KERN_INFO "MTRR variable ranges %sabled:\n",
 121                        mtrr_state.enabled & 2 ? "en" : "dis");
 122                 high_width = ((size_or_mask ? ffs(size_or_mask) - 1 : 32) - (32 - PAGE_SHIFT) + 3) / 4;
 123                 for (i = 0; i < num_var_ranges; ++i) {
 124                         if (mtrr_state.var_ranges[i].mask_lo & (1 << 11))
 125                                 printk(KERN_INFO "MTRR %u base %0*X%05X000 mask %0*X%05X000 %s\n",
 126                                        i,
 127                                        high_width,
 128                                        mtrr_state.var_ranges[i].base_hi,
 129                                        mtrr_state.var_ranges[i].base_lo >> 12,
 130                                        high_width,
 131                                        mtrr_state.var_ranges[i].mask_hi,
 132                                        mtrr_state.var_ranges[i].mask_lo >> 12,
 133                                        mtrr_attrib_to_str(mtrr_state.var_ranges[i].base_lo & 0xff));
 134                         else
 135                                 printk(KERN_INFO "MTRR %u disabled\n", i);
 136                 }
 137         }
 138 }
 139
 140 /*  Some BIOS's are fucked and don't set all MTRRs the same!  */
 141 void __init mtrr_state_warn(void)
 142 {
 143         unsigned long mask = smp_changes_mask;
 144
 145         if (!mask)
 146                 return;
 147         if (mask & MTRR_CHANGE_MASK_FIXED)
 148                 printk(KERN_WARNING "mtrr: your CPUs had inconsistent fixed MTRR settings\n");
 149         if (mask & MTRR_CHANGE_MASK_VARIABLE)
 150                 printk(KERN_WARNING "mtrr: your CPUs had inconsistent variable MTRR settings\n");
 151         if (mask & MTRR_CHANGE_MASK_DEFTYPE)
 152                 printk(KERN_WARNING "mtrr: your CPUs had inconsistent MTRRdefType settings\n");
 153         printk(KERN_INFO "mtrr: probably your BIOS does not setup all CPUs.\n");
 154         printk(KERN_INFO "mtrr: corrected configuration.\n");
 155 }
 156
 157 /* Doesn't attempt to pass an error out to MTRR users
 158    because it's quite complicated in some cases and probably not
 159    worth it because the best error handling is to ignore it. */
 160 void mtrr_wrmsr(unsigned msr, unsigned a, unsigned b)
 161 {
 162         if (wrmsr_safe(msr, a, b) < 0)
 163                 printk(KERN_ERR
 164                         "MTRR: CPU %u: Writing MSR %x to %x:%x failed\n",
 165                         smp_processor_id(), msr, a, b);
 166 }
 167
 168 /**
 169  * Enable and allow read/write of extended fixed-range MTRR bits on K8 CPUs
 170  * see AMD publication no. 24593, chapter 3.2.1 for more information
 171  */
 172 static inline void k8_enable_fixed_iorrs(void)
 173 {
 174         unsigned lo, hi;
 175
 176         rdmsr(MSR_K8_SYSCFG, lo, hi);
 177         mtrr_wrmsr(MSR_K8_SYSCFG, lo
 178                                 | K8_MTRRFIXRANGE_DRAM_ENABLE
 179                                 | K8_MTRRFIXRANGE_DRAM_MODIFY, hi);
 180 }
 181
 182 /**
 183  * Checks and updates an fixed-range MTRR if it differs from the value it
 184  * should have. If K8 extenstions are wanted, update the K8 SYSCFG MSR also.
 185  * see AMD publication no. 24593, chapter 7.8.1, page 233 for more information
 186  * \param msr MSR address of the MTTR which should be checked and updated
 187  * \param changed pointer which indicates whether the MTRR needed to be changed
 188  * \param msrwords pointer to the MSR values which the MSR should have
 189  */
 190 static void set_fixed_range(int msr, int * changed, unsigned int * msrwords)
 191 {
 192         unsigned lo, hi;
 193
 194         rdmsr(msr, lo, hi);
 195
 196         if (lo != msrwords[0] || hi != msrwords[1]) {
 197                 if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD &&
 198                     boot_cpu_data.x86 == 15 &&
 199                     ((msrwords[0] | msrwords[1]) & K8_MTRR_RDMEM_WRMEM_MASK))
 200                         k8_enable_fixed_iorrs();
 201                 mtrr_wrmsr(msr, msrwords[0], msrwords[1]);
 202                 *changed = TRUE;
 203         }
 204 }
 205
 206 int generic_get_free_region(unsigned long base, unsigned long size, int replace_reg)
 207 /*  [SUMMARY] Get a free MTRR.
 208     <base> The starting (base) address of the region.
 209     <size> The size (in bytes) of the region.
 210     [RETURNS] The index of the region on success, else -1 on error.
 211 */
 212 {
 213         int i, max;
 214         mtrr_type ltype;
 215         unsigned long lbase, lsize;
 216
 217         max = num_var_ranges;
 218         if (replace_reg >= 0 && replace_reg < max)
 219                 return replace_reg;
 220         for (i = 0; i < max; ++i) {
 221                 mtrr_if->get(i, &lbase, &lsize, &ltype);
 222                 if (lsize == 0)
 223                         return i;
 224         }
 225         return -ENOSPC;
 226 }
 227
 228 static void generic_get_mtrr(unsigned int reg, unsigned long *base,
 229                              unsigned long *size, mtrr_type *type)
 230 {
 231         unsigned int mask_lo, mask_hi, base_lo, base_hi;
 232
 233         rdmsr(MTRRphysMask_MSR(reg), mask_lo, mask_hi);
 234         if ((mask_lo & 0x800) == 0) {
 235                 /*  Invalid (i.e. free) range  */
 236                 *base = 0;
 237                 *size = 0;
 238                 *type = 0;
 239                 return;
 240         }
 241
 242         rdmsr(MTRRphysBase_MSR(reg), base_lo, base_hi);
 243
 244         /* Work out the shifted address mask. */
 245         mask_lo = size_or_mask | mask_hi << (32 - PAGE_SHIFT)
 246             | mask_lo >> PAGE_SHIFT;
 247
 248         /* This works correctly if size is a power of two, i.e. a
 249            contiguous range. */
 250         *size = -mask_lo;
 251         *base = base_hi << (32 - PAGE_SHIFT) | base_lo >> PAGE_SHIFT;
 252         *type = base_lo & 0xff;
 253 }
 254
 255 /**
 256  * Checks and updates the fixed-range MTRRs if they differ from the saved set
 257  * \param frs pointer to fixed-range MTRR values, saved by get_fixed_ranges()
 258  */
 259 static int set_fixed_ranges(mtrr_type * frs)
 260 {
 261         unsigned long long *saved = (unsigned long long *) frs;
 262         int changed = FALSE;
 263         int block=-1, range;
 264
 265         while (fixed_range_blocks[++block].ranges)
 266             for (range=0; range < fixed_range_blocks[block].ranges; range++)
 267                 set_fixed_range(fixed_range_blocks[block].base_msr + range,
 268                     &changed, (unsigned int *) saved++);
 269
 270         return changed;
 271 }
 272
 273 /*  Set the MSR pair relating to a var range. Returns TRUE if
 274     changes are made  */
 275 static int set_mtrr_var_ranges(unsigned int index, struct mtrr_var_range *vr)
 276 {
 277         unsigned int lo, hi;
 278         int changed = FALSE;
 279
 280         rdmsr(MTRRphysBase_MSR(index), lo, hi);
 281         if ((vr->base_lo & 0xfffff0ffUL) != (lo & 0xfffff0ffUL)
 282             || (vr->base_hi & (size_and_mask >> (32 - PAGE_SHIFT))) !=
 283                 (hi & (size_and_mask >> (32 - PAGE_SHIFT)))) {
 284                 mtrr_wrmsr(MTRRphysBase_MSR(index), vr->base_lo, vr->base_hi);
 285                 changed = TRUE;
 286         }
 287
 288         rdmsr(MTRRphysMask_MSR(index), lo, hi);
 289
 290         if ((vr->mask_lo & 0xfffff800UL) != (lo & 0xfffff800UL)
 291             || (vr->mask_hi & (size_and_mask >> (32 - PAGE_SHIFT))) !=
 292                 (hi & (size_and_mask >> (32 - PAGE_SHIFT)))) {
 293                 mtrr_wrmsr(MTRRphysMask_MSR(index), vr->mask_lo, vr->mask_hi);
 294                 changed = TRUE;
 295         }
 296         return changed;
 297 }
 298
 299 static u32 deftype_lo, deftype_hi;
 300
 301 static unsigned long set_mtrr_state(void)
 302 /*  [SUMMARY] Set the MTRR state for this CPU.
 303     <state> The MTRR state information to read.
 304     <ctxt> Some relevant CPU context.
 305     [NOTE] The CPU must already be in a safe state for MTRR changes.
 306     [RETURNS] 0 if no changes made, else a mask indication what was changed.
 307 */
 308 {
 309         unsigned int i;
 310         unsigned long change_mask = 0;
 311
 312         for (i = 0; i < num_var_ranges; i++)
 313                 if (set_mtrr_var_ranges(i, &mtrr_state.var_ranges[i]))
 314                         change_mask |= MTRR_CHANGE_MASK_VARIABLE;
 315
 316         if (mtrr_state.have_fixed && set_fixed_ranges(mtrr_state.fixed_ranges))
 317                 change_mask |= MTRR_CHANGE_MASK_FIXED;
 318
 319         /*  Set_mtrr_restore restores the old value of MTRRdefType,
 320            so to set it we fiddle with the saved value  */
 321         if ((deftype_lo & 0xff) != mtrr_state.def_type
 322             || ((deftype_lo & 0xc00) >> 10) != mtrr_state.enabled) {
 323                 deftype_lo = (deftype_lo & ~0xcff) | mtrr_state.def_type | (mtrr_state.enabled << 10);
 324                 change_mask |= MTRR_CHANGE_MASK_DEFTYPE;
 325         }
 326
 327         return change_mask;
 328 }
 329
 330
 331 static unsigned long cr4 = 0;
 332 static DEFINE_SPINLOCK(set_atomicity_lock);
 333
 334 /*
 335  * Since we are disabling the cache don't allow any interrupts - they
 336  * would run extremely slow and would only increase the pain.  The caller must
 337  * ensure that local interrupts are disabled and are reenabled after post_set()
 338  * has been called.
 339  */
 340
 341 static void prepare_set(void) __acquires(set_atomicity_lock)
 342 {
 343         unsigned long cr0;
 344
 345         /*  Note that this is not ideal, since the cache is only flushed/disabled
 346            for this CPU while the MTRRs are changed, but changing this requires
 347            more invasive changes to the way the kernel boots  */
 348
 349         spin_lock(&set_atomicity_lock);
 350
 351         /*  Enter the no-fill (CD=1, NW=0) cache mode and flush caches. */
 352         cr0 = read_cr0() | 0x40000000;  /* set CD flag */
 353         write_cr0(cr0);
 354         wbinvd();
 355
 356         /*  Save value of CR4 and clear Page Global Enable (bit 7)  */
 357         if ( cpu_has_pge ) {
 358                 cr4 = read_cr4();
 359                 write_cr4(cr4 & ~X86_CR4_PGE);
 360         }
 361
 362         /* Flush all TLBs via a mov %cr3, %reg; mov %reg, %cr3 */
 363         __flush_tlb();
 364
 365         /*  Save MTRR state */
 366         rdmsr(MTRRdefType_MSR, deftype_lo, deftype_hi);
 367
 368         /*  Disable MTRRs, and set the default type to uncached  */
 369         mtrr_wrmsr(MTRRdefType_MSR, deftype_lo & ~0xcff, deftype_hi);
 370 }
 371
 372 static void post_set(void) __releases(set_atomicity_lock)
 373 {
 374         /*  Flush TLBs (no need to flush caches - they are disabled)  */
 375         __flush_tlb();
 376
 377         /* Intel (P6) standard MTRRs */
 378         mtrr_wrmsr(MTRRdefType_MSR, deftype_lo, deftype_hi);
 379
 380         /*  Enable caches  */
 381         write_cr0(read_cr0() & 0xbfffffff);
 382
 383         /*  Restore value of CR4  */
 384         if ( cpu_has_pge )
 385                 write_cr4(cr4);
 386         spin_unlock(&set_atomicity_lock);
 387 }
 388
 389 static void generic_set_all(void)
 390 {
 391         unsigned long mask, count;
 392         unsigned long flags;
 393
 394         local_irq_save(flags);
 395         prepare_set();
 396
 397         /* Actually set the state */
 398         mask = set_mtrr_state();
 399
 400         post_set();
 401         local_irq_restore(flags);
 402
 403         /*  Use the atomic bitops to update the global mask  */
 404         for (count = 0; count < sizeof mask * 8; ++count) {
 405                 if (mask & 0x01)
 406                         set_bit(count, &smp_changes_mask);
 407                 mask >>= 1;
 408         }
 409
 410 }
 411
 412 static void generic_set_mtrr(unsigned int reg, unsigned long base,
 413                              unsigned long size, mtrr_type type)
 414 /*  [SUMMARY] Set variable MTRR register on the local CPU.
 415     <reg> The register to set.
 416     <base> The base address of the region.
 417     <size> The size of the region. If this is 0 the region is disabled.
 418     <type> The type of the region.
 419     <do_safe> If TRUE, do the change safely. If FALSE, safety measures should
 420     be done externally.
 421     [RETURNS] Nothing.
 422 */
 423 {
 424         unsigned long flags;
 425         struct mtrr_var_range *vr;
 426
 427         vr = &mtrr_state.var_ranges[reg];
 428
 429         local_irq_save(flags);
 430         prepare_set();
 431
 432         if (size == 0) {
 433                 /* The invalid bit is kept in the mask, so we simply clear the
 434                    relevant mask register to disable a range. */
 435                 mtrr_wrmsr(MTRRphysMask_MSR(reg), 0, 0);
 436                 memset(vr, 0, sizeof(struct mtrr_var_range));
 437         } else {
 438                 vr->base_lo = base << PAGE_SHIFT | type;
 439                 vr->base_hi = (base & size_and_mask) >> (32 - PAGE_SHIFT);
 440                 vr->mask_lo = -size << PAGE_SHIFT | 0x800;
 441                 vr->mask_hi = (-size & size_and_mask) >> (32 - PAGE_SHIFT);
 442
 443                 mtrr_wrmsr(MTRRphysBase_MSR(reg), vr->base_lo, vr->base_hi);
 444                 mtrr_wrmsr(MTRRphysMask_MSR(reg), vr->mask_lo, vr->mask_hi);
 445         }
 446
 447         post_set();
 448         local_irq_restore(flags);
 449 }
 450
 451 int generic_validate_add_page(unsigned long base, unsigned long size, unsigned int type)
 452 {
 453         unsigned long lbase, last;
 454
 455         /*  For Intel PPro stepping <= 7, must be 4 MiB aligned
 456             and not touch 0x70000000->0x7003FFFF */
 457         if (is_cpu(INTEL) && boot_cpu_data.x86 == 6 &&
 458             boot_cpu_data.x86_model == 1 &&
 459             boot_cpu_data.x86_mask <= 7) {
 460                 if (base & ((1 << (22 - PAGE_SHIFT)) - 1)) {
 461                         printk(KERN_WARNING "mtrr: base(0x%lx000) is not 4 MiB aligned\n", base);
 462                         return -EINVAL;
 463                 }
 464                 if (!(base + size < 0x70000 || base > 0x7003F) &&
 465                     (type == MTRR_TYPE_WRCOMB
 466                      || type == MTRR_TYPE_WRBACK)) {
 467                         printk(KERN_WARNING "mtrr: writable mtrr between 0x70000000 and 0x7003FFFF may hang the CPU.\n");
 468                         return -EINVAL;
 469                 }
 470         }
 471
 472         if (base < 0x100) {
 473                 printk(KERN_WARNING "mtrr: cannot set region below 1 MiB (0x%lx000,0x%lx000)\n",
 474                        base, size);
 475                 return -EINVAL;
 476         }
 477         /*  Check upper bits of base and last are equal and lower bits are 0
 478             for base and 1 for last  */
 479         last = base + size - 1;
 480         for (lbase = base; !(lbase & 1) && (last & 1);
 481              lbase = lbase >> 1, last = last >> 1) ;
 482         if (lbase != last) {
 483                 printk(KERN_WARNING "mtrr: base(0x%lx000) is not aligned on a size(0x%lx000) boundary\n",
 484                        base, size);
 485                 return -EINVAL;
 486         }
 487         return 0;
 488 }
 489
 490
 491 static int generic_have_wrcomb(void)
 492 {
 493         unsigned long config, dummy;
 494         rdmsr(MTRRcap_MSR, config, dummy);
 495         return (config & (1 << 10));
 496 }
 497
 498 int positive_have_wrcomb(void)
 499 {
 500         return 1;
 501 }
 502
 503 /* generic structure...
 504  */
 505 struct mtrr_ops generic_mtrr_ops = {
 506         .use_intel_if      = 1,
 507         .set_all           = generic_set_all,
 508         .get               = generic_get_mtrr,
 509         .get_free_region   = generic_get_free_region,
 510         .set               = generic_set_mtrr,
 511         .validate_add_page = generic_validate_add_page,
 512         .have_wrcomb       = generic_have_wrcomb,
 513 };