1 #include "amd64_edac.h"
2 #include <asm/amd_nb.h>
4 static struct edac_pci_ctl_info
*amd64_ctl_pci
;
6 static int report_gart_errors
;
7 module_param(report_gart_errors
, int, 0644);
10 * Set by command line parameter. If BIOS has enabled the ECC, this override is
11 * cleared to prevent re-enabling the hardware by this driver.
13 static int ecc_enable_override
;
14 module_param(ecc_enable_override
, int, 0644);
16 static struct msr __percpu
*msrs
;
19 * count successfully initialized driver instances for setup_pci_device()
21 static atomic_t drv_instances
= ATOMIC_INIT(0);
23 /* Per-node driver instances */
24 static struct mem_ctl_info
**mcis
;
25 static struct ecc_settings
**ecc_stngs
;
28 * Valid scrub rates for the K8 hardware memory scrubber. We map the scrubbing
29 * bandwidth to a valid bit pattern. The 'set' operation finds the 'matching-
32 *FIXME: Produce a better mapping/linearisation.
35 u32 scrubval
; /* bit pattern for scrub rate */
36 u32 bandwidth
; /* bandwidth consumed (bytes/sec) */
38 { 0x01, 1600000000UL},
60 { 0x00, 0UL}, /* scrubbing off */
63 static int __amd64_read_pci_cfg_dword(struct pci_dev
*pdev
, int offset
,
64 u32
*val
, const char *func
)
68 err
= pci_read_config_dword(pdev
, offset
, val
);
70 amd64_warn("%s: error reading F%dx%03x.\n",
71 func
, PCI_FUNC(pdev
->devfn
), offset
);
76 int __amd64_write_pci_cfg_dword(struct pci_dev
*pdev
, int offset
,
77 u32 val
, const char *func
)
81 err
= pci_write_config_dword(pdev
, offset
, val
);
83 amd64_warn("%s: error writing to F%dx%03x.\n",
84 func
, PCI_FUNC(pdev
->devfn
), offset
);
91 * Depending on the family, F2 DCT reads need special handling:
93 * K8: has a single DCT only
95 * F10h: each DCT has its own set of regs
99 * F15h: we select which DCT we access using F1x10C[DctCfgSel]
102 static int k8_read_dct_pci_cfg(struct amd64_pvt
*pvt
, int addr
, u32
*val
,
108 return __amd64_read_pci_cfg_dword(pvt
->F2
, addr
, val
, func
);
111 static int f10_read_dct_pci_cfg(struct amd64_pvt
*pvt
, int addr
, u32
*val
,
114 return __amd64_read_pci_cfg_dword(pvt
->F2
, addr
, val
, func
);
117 static int f15_read_dct_pci_cfg(struct amd64_pvt
*pvt
, int addr
, u32
*val
,
123 if (addr
>= 0x140 && addr
<= 0x1a0) {
128 amd64_read_pci_cfg(pvt
->F1
, DCT_CFG_SEL
, ®
);
131 amd64_write_pci_cfg(pvt
->F1
, DCT_CFG_SEL
, reg
);
133 return __amd64_read_pci_cfg_dword(pvt
->F2
, addr
, val
, func
);
137 * Memory scrubber control interface. For K8, memory scrubbing is handled by
138 * hardware and can involve L2 cache, dcache as well as the main memory. With
139 * F10, this is extended to L3 cache scrubbing on CPU models sporting that
142 * This causes the "units" for the scrubbing speed to vary from 64 byte blocks
143 * (dram) over to cache lines. This is nasty, so we will use bandwidth in
144 * bytes/sec for the setting.
146 * Currently, we only do dram scrubbing. If the scrubbing is done in software on
147 * other archs, we might not have access to the caches directly.
151 * scan the scrub rate mapping table for a close or matching bandwidth value to
152 * issue. If requested is too big, then use last maximum value found.
154 static int __amd64_set_scrub_rate(struct pci_dev
*ctl
, u32 new_bw
, u32 min_rate
)
160 * map the configured rate (new_bw) to a value specific to the AMD64
161 * memory controller and apply to register. Search for the first
162 * bandwidth entry that is greater or equal than the setting requested
163 * and program that. If at last entry, turn off DRAM scrubbing.
165 for (i
= 0; i
< ARRAY_SIZE(scrubrates
); i
++) {
167 * skip scrub rates which aren't recommended
168 * (see F10 BKDG, F3x58)
170 if (scrubrates
[i
].scrubval
< min_rate
)
173 if (scrubrates
[i
].bandwidth
<= new_bw
)
177 * if no suitable bandwidth found, turn off DRAM scrubbing
178 * entirely by falling back to the last element in the
183 scrubval
= scrubrates
[i
].scrubval
;
185 pci_write_bits32(ctl
, SCRCTRL
, scrubval
, 0x001F);
188 return scrubrates
[i
].bandwidth
;
193 static int amd64_set_scrub_rate(struct mem_ctl_info
*mci
, u32 bw
)
195 struct amd64_pvt
*pvt
= mci
->pvt_info
;
196 u32 min_scrubrate
= 0x5;
198 if (boot_cpu_data
.x86
== 0xf)
201 return __amd64_set_scrub_rate(pvt
->F3
, bw
, min_scrubrate
);
204 static int amd64_get_scrub_rate(struct mem_ctl_info
*mci
)
206 struct amd64_pvt
*pvt
= mci
->pvt_info
;
208 int i
, retval
= -EINVAL
;
210 amd64_read_pci_cfg(pvt
->F3
, SCRCTRL
, &scrubval
);
212 scrubval
= scrubval
& 0x001F;
214 for (i
= 0; i
< ARRAY_SIZE(scrubrates
); i
++) {
215 if (scrubrates
[i
].scrubval
== scrubval
) {
216 retval
= scrubrates
[i
].bandwidth
;
224 * returns true if the SysAddr given by sys_addr matches the
225 * DRAM base/limit associated with node_id
227 static bool amd64_base_limit_match(struct amd64_pvt
*pvt
, u64 sys_addr
,
232 /* The K8 treats this as a 40-bit value. However, bits 63-40 will be
233 * all ones if the most significant implemented address bit is 1.
234 * Here we discard bits 63-40. See section 3.4.2 of AMD publication
235 * 24592: AMD x86-64 Architecture Programmer's Manual Volume 1
236 * Application Programming.
238 addr
= sys_addr
& 0x000000ffffffffffull
;
240 return ((addr
>= get_dram_base(pvt
, nid
)) &&
241 (addr
<= get_dram_limit(pvt
, nid
)));
245 * Attempt to map a SysAddr to a node. On success, return a pointer to the
246 * mem_ctl_info structure for the node that the SysAddr maps to.
248 * On failure, return NULL.
250 static struct mem_ctl_info
*find_mc_by_sys_addr(struct mem_ctl_info
*mci
,
253 struct amd64_pvt
*pvt
;
258 * Here we use the DRAM Base (section 3.4.4.1) and DRAM Limit (section
259 * 3.4.4.2) registers to map the SysAddr to a node ID.
264 * The value of this field should be the same for all DRAM Base
265 * registers. Therefore we arbitrarily choose to read it from the
266 * register for node 0.
268 intlv_en
= dram_intlv_en(pvt
, 0);
271 for (node_id
= 0; node_id
< DRAM_RANGES
; node_id
++) {
272 if (amd64_base_limit_match(pvt
, sys_addr
, node_id
))
278 if (unlikely((intlv_en
!= 0x01) &&
279 (intlv_en
!= 0x03) &&
280 (intlv_en
!= 0x07))) {
281 amd64_warn("DRAM Base[IntlvEn] junk value: 0x%x, BIOS bug?\n", intlv_en
);
285 bits
= (((u32
) sys_addr
) >> 12) & intlv_en
;
287 for (node_id
= 0; ; ) {
288 if ((dram_intlv_sel(pvt
, node_id
) & intlv_en
) == bits
)
289 break; /* intlv_sel field matches */
291 if (++node_id
>= DRAM_RANGES
)
295 /* sanity test for sys_addr */
296 if (unlikely(!amd64_base_limit_match(pvt
, sys_addr
, node_id
))) {
297 amd64_warn("%s: sys_addr 0x%llx falls outside base/limit address"
298 "range for node %d with node interleaving enabled.\n",
299 __func__
, sys_addr
, node_id
);
304 return edac_mc_find((int)node_id
);
307 debugf2("sys_addr 0x%lx doesn't match any node\n",
308 (unsigned long)sys_addr
);
314 * compute the CS base address of the @csrow on the DRAM controller @dct.
315 * For details see F2x[5C:40] in the processor's BKDG
317 static void get_cs_base_and_mask(struct amd64_pvt
*pvt
, int csrow
, u8 dct
,
318 u64
*base
, u64
*mask
)
320 u64 csbase
, csmask
, base_bits
, mask_bits
;
323 if (boot_cpu_data
.x86
== 0xf && pvt
->ext_model
< K8_REV_F
) {
324 csbase
= pvt
->csels
[dct
].csbases
[csrow
];
325 csmask
= pvt
->csels
[dct
].csmasks
[csrow
];
326 base_bits
= GENMASK(21, 31) | GENMASK(9, 15);
327 mask_bits
= GENMASK(21, 29) | GENMASK(9, 15);
330 csbase
= pvt
->csels
[dct
].csbases
[csrow
];
331 csmask
= pvt
->csels
[dct
].csmasks
[csrow
>> 1];
334 if (boot_cpu_data
.x86
== 0x15)
335 base_bits
= mask_bits
= GENMASK(19,30) | GENMASK(5,13);
337 base_bits
= mask_bits
= GENMASK(19,28) | GENMASK(5,13);
340 *base
= (csbase
& base_bits
) << addr_shift
;
343 /* poke holes for the csmask */
344 *mask
&= ~(mask_bits
<< addr_shift
);
346 *mask
|= (csmask
& mask_bits
) << addr_shift
;
349 #define for_each_chip_select(i, dct, pvt) \
350 for (i = 0; i < pvt->csels[dct].b_cnt; i++)
352 #define chip_select_base(i, dct, pvt) \
353 pvt->csels[dct].csbases[i]
355 #define for_each_chip_select_mask(i, dct, pvt) \
356 for (i = 0; i < pvt->csels[dct].m_cnt; i++)
359 * @input_addr is an InputAddr associated with the node given by mci. Return the
360 * csrow that input_addr maps to, or -1 on failure (no csrow claims input_addr).
362 static int input_addr_to_csrow(struct mem_ctl_info
*mci
, u64 input_addr
)
364 struct amd64_pvt
*pvt
;
370 for_each_chip_select(csrow
, 0, pvt
) {
371 if (!csrow_enabled(csrow
, 0, pvt
))
374 get_cs_base_and_mask(pvt
, csrow
, 0, &base
, &mask
);
378 if ((input_addr
& mask
) == (base
& mask
)) {
379 debugf2("InputAddr 0x%lx matches csrow %d (node %d)\n",
380 (unsigned long)input_addr
, csrow
,
386 debugf2("no matching csrow for InputAddr 0x%lx (MC node %d)\n",
387 (unsigned long)input_addr
, pvt
->mc_node_id
);
393 * Obtain info from the DRAM Hole Address Register (section 3.4.8, pub #26094)
394 * for the node represented by mci. Info is passed back in *hole_base,
395 * *hole_offset, and *hole_size. Function returns 0 if info is valid or 1 if
396 * info is invalid. Info may be invalid for either of the following reasons:
398 * - The revision of the node is not E or greater. In this case, the DRAM Hole
399 * Address Register does not exist.
401 * - The DramHoleValid bit is cleared in the DRAM Hole Address Register,
402 * indicating that its contents are not valid.
404 * The values passed back in *hole_base, *hole_offset, and *hole_size are
405 * complete 32-bit values despite the fact that the bitfields in the DHAR
406 * only represent bits 31-24 of the base and offset values.
408 int amd64_get_dram_hole_info(struct mem_ctl_info
*mci
, u64
*hole_base
,
409 u64
*hole_offset
, u64
*hole_size
)
411 struct amd64_pvt
*pvt
= mci
->pvt_info
;
414 /* only revE and later have the DRAM Hole Address Register */
415 if (boot_cpu_data
.x86
== 0xf && pvt
->ext_model
< K8_REV_E
) {
416 debugf1(" revision %d for node %d does not support DHAR\n",
417 pvt
->ext_model
, pvt
->mc_node_id
);
421 /* valid for Fam10h and above */
422 if (boot_cpu_data
.x86
>= 0x10 && !dhar_mem_hoist_valid(pvt
)) {
423 debugf1(" Dram Memory Hoisting is DISABLED on this system\n");
427 if (!dhar_valid(pvt
)) {
428 debugf1(" Dram Memory Hoisting is DISABLED on this node %d\n",
433 /* This node has Memory Hoisting */
435 /* +------------------+--------------------+--------------------+-----
436 * | memory | DRAM hole | relocated |
437 * | [0, (x - 1)] | [x, 0xffffffff] | addresses from |
439 * | | | [0x100000000, |
440 * | | | (0x100000000+ |
441 * | | | (0xffffffff-x))] |
442 * +------------------+--------------------+--------------------+-----
444 * Above is a diagram of physical memory showing the DRAM hole and the
445 * relocated addresses from the DRAM hole. As shown, the DRAM hole
446 * starts at address x (the base address) and extends through address
447 * 0xffffffff. The DRAM Hole Address Register (DHAR) relocates the
448 * addresses in the hole so that they start at 0x100000000.
451 base
= dhar_base(pvt
);
454 *hole_size
= (0x1ull
<< 32) - base
;
456 if (boot_cpu_data
.x86
> 0xf)
457 *hole_offset
= f10_dhar_offset(pvt
);
459 *hole_offset
= k8_dhar_offset(pvt
);
461 debugf1(" DHAR info for node %d base 0x%lx offset 0x%lx size 0x%lx\n",
462 pvt
->mc_node_id
, (unsigned long)*hole_base
,
463 (unsigned long)*hole_offset
, (unsigned long)*hole_size
);
467 EXPORT_SYMBOL_GPL(amd64_get_dram_hole_info
);
470 * Return the DramAddr that the SysAddr given by @sys_addr maps to. It is
471 * assumed that sys_addr maps to the node given by mci.
473 * The first part of section 3.4.4 (p. 70) shows how the DRAM Base (section
474 * 3.4.4.1) and DRAM Limit (section 3.4.4.2) registers are used to translate a
475 * SysAddr to a DramAddr. If the DRAM Hole Address Register (DHAR) is enabled,
476 * then it is also involved in translating a SysAddr to a DramAddr. Sections
477 * 3.4.8 and 3.5.8.2 describe the DHAR and how it is used for memory hoisting.
478 * These parts of the documentation are unclear. I interpret them as follows:
480 * When node n receives a SysAddr, it processes the SysAddr as follows:
482 * 1. It extracts the DRAMBase and DRAMLimit values from the DRAM Base and DRAM
483 * Limit registers for node n. If the SysAddr is not within the range
484 * specified by the base and limit values, then node n ignores the Sysaddr
485 * (since it does not map to node n). Otherwise continue to step 2 below.
487 * 2. If the DramHoleValid bit of the DHAR for node n is clear, the DHAR is
488 * disabled so skip to step 3 below. Otherwise see if the SysAddr is within
489 * the range of relocated addresses (starting at 0x100000000) from the DRAM
490 * hole. If not, skip to step 3 below. Else get the value of the
491 * DramHoleOffset field from the DHAR. To obtain the DramAddr, subtract the
492 * offset defined by this value from the SysAddr.
494 * 3. Obtain the base address for node n from the DRAMBase field of the DRAM
495 * Base register for node n. To obtain the DramAddr, subtract the base
496 * address from the SysAddr, as shown near the start of section 3.4.4 (p.70).
498 static u64
sys_addr_to_dram_addr(struct mem_ctl_info
*mci
, u64 sys_addr
)
500 struct amd64_pvt
*pvt
= mci
->pvt_info
;
501 u64 dram_base
, hole_base
, hole_offset
, hole_size
, dram_addr
;
504 dram_base
= get_dram_base(pvt
, pvt
->mc_node_id
);
506 ret
= amd64_get_dram_hole_info(mci
, &hole_base
, &hole_offset
,
509 if ((sys_addr
>= (1ull << 32)) &&
510 (sys_addr
< ((1ull << 32) + hole_size
))) {
511 /* use DHAR to translate SysAddr to DramAddr */
512 dram_addr
= sys_addr
- hole_offset
;
514 debugf2("using DHAR to translate SysAddr 0x%lx to "
516 (unsigned long)sys_addr
,
517 (unsigned long)dram_addr
);
524 * Translate the SysAddr to a DramAddr as shown near the start of
525 * section 3.4.4 (p. 70). Although sys_addr is a 64-bit value, the k8
526 * only deals with 40-bit values. Therefore we discard bits 63-40 of
527 * sys_addr below. If bit 39 of sys_addr is 1 then the bits we
528 * discard are all 1s. Otherwise the bits we discard are all 0s. See
529 * section 3.4.2 of AMD publication 24592: AMD x86-64 Architecture
530 * Programmer's Manual Volume 1 Application Programming.
532 dram_addr
= (sys_addr
& GENMASK(0, 39)) - dram_base
;
534 debugf2("using DRAM Base register to translate SysAddr 0x%lx to "
535 "DramAddr 0x%lx\n", (unsigned long)sys_addr
,
536 (unsigned long)dram_addr
);
541 * @intlv_en is the value of the IntlvEn field from a DRAM Base register
542 * (section 3.4.4.1). Return the number of bits from a SysAddr that are used
543 * for node interleaving.
545 static int num_node_interleave_bits(unsigned intlv_en
)
547 static const int intlv_shift_table
[] = { 0, 1, 0, 2, 0, 0, 0, 3 };
550 BUG_ON(intlv_en
> 7);
551 n
= intlv_shift_table
[intlv_en
];
555 /* Translate the DramAddr given by @dram_addr to an InputAddr. */
556 static u64
dram_addr_to_input_addr(struct mem_ctl_info
*mci
, u64 dram_addr
)
558 struct amd64_pvt
*pvt
;
565 * See the start of section 3.4.4 (p. 70, BKDG #26094, K8, revA-E)
566 * concerning translating a DramAddr to an InputAddr.
568 intlv_shift
= num_node_interleave_bits(dram_intlv_en(pvt
, 0));
569 input_addr
= ((dram_addr
>> intlv_shift
) & GENMASK(12, 35)) +
572 debugf2(" Intlv Shift=%d DramAddr=0x%lx maps to InputAddr=0x%lx\n",
573 intlv_shift
, (unsigned long)dram_addr
,
574 (unsigned long)input_addr
);
580 * Translate the SysAddr represented by @sys_addr to an InputAddr. It is
581 * assumed that @sys_addr maps to the node given by mci.
583 static u64
sys_addr_to_input_addr(struct mem_ctl_info
*mci
, u64 sys_addr
)
588 dram_addr_to_input_addr(mci
, sys_addr_to_dram_addr(mci
, sys_addr
));
590 debugf2("SysAdddr 0x%lx translates to InputAddr 0x%lx\n",
591 (unsigned long)sys_addr
, (unsigned long)input_addr
);
598 * @input_addr is an InputAddr associated with the node represented by mci.
599 * Translate @input_addr to a DramAddr and return the result.
601 static u64
input_addr_to_dram_addr(struct mem_ctl_info
*mci
, u64 input_addr
)
603 struct amd64_pvt
*pvt
;
604 unsigned node_id
, intlv_shift
;
609 * Near the start of section 3.4.4 (p. 70, BKDG #26094, K8, revA-E)
610 * shows how to translate a DramAddr to an InputAddr. Here we reverse
611 * this procedure. When translating from a DramAddr to an InputAddr, the
612 * bits used for node interleaving are discarded. Here we recover these
613 * bits from the IntlvSel field of the DRAM Limit register (section
614 * 3.4.4.2) for the node that input_addr is associated with.
617 node_id
= pvt
->mc_node_id
;
621 intlv_shift
= num_node_interleave_bits(dram_intlv_en(pvt
, 0));
622 if (intlv_shift
== 0) {
623 debugf1(" InputAddr 0x%lx translates to DramAddr of "
624 "same value\n", (unsigned long)input_addr
);
629 bits
= ((input_addr
& GENMASK(12, 35)) << intlv_shift
) +
630 (input_addr
& 0xfff);
632 intlv_sel
= dram_intlv_sel(pvt
, node_id
) & ((1 << intlv_shift
) - 1);
633 dram_addr
= bits
+ (intlv_sel
<< 12);
635 debugf1("InputAddr 0x%lx translates to DramAddr 0x%lx "
636 "(%d node interleave bits)\n", (unsigned long)input_addr
,
637 (unsigned long)dram_addr
, intlv_shift
);
643 * @dram_addr is a DramAddr that maps to the node represented by mci. Convert
644 * @dram_addr to a SysAddr.
646 static u64
dram_addr_to_sys_addr(struct mem_ctl_info
*mci
, u64 dram_addr
)
648 struct amd64_pvt
*pvt
= mci
->pvt_info
;
649 u64 hole_base
, hole_offset
, hole_size
, base
, sys_addr
;
652 ret
= amd64_get_dram_hole_info(mci
, &hole_base
, &hole_offset
,
655 if ((dram_addr
>= hole_base
) &&
656 (dram_addr
< (hole_base
+ hole_size
))) {
657 sys_addr
= dram_addr
+ hole_offset
;
659 debugf1("using DHAR to translate DramAddr 0x%lx to "
660 "SysAddr 0x%lx\n", (unsigned long)dram_addr
,
661 (unsigned long)sys_addr
);
667 base
= get_dram_base(pvt
, pvt
->mc_node_id
);
668 sys_addr
= dram_addr
+ base
;
671 * The sys_addr we have computed up to this point is a 40-bit value
672 * because the k8 deals with 40-bit values. However, the value we are
673 * supposed to return is a full 64-bit physical address. The AMD
674 * x86-64 architecture specifies that the most significant implemented
675 * address bit through bit 63 of a physical address must be either all
676 * 0s or all 1s. Therefore we sign-extend the 40-bit sys_addr to a
677 * 64-bit value below. See section 3.4.2 of AMD publication 24592:
678 * AMD x86-64 Architecture Programmer's Manual Volume 1 Application
681 sys_addr
|= ~((sys_addr
& (1ull << 39)) - 1);
683 debugf1(" Node %d, DramAddr 0x%lx to SysAddr 0x%lx\n",
684 pvt
->mc_node_id
, (unsigned long)dram_addr
,
685 (unsigned long)sys_addr
);
691 * @input_addr is an InputAddr associated with the node given by mci. Translate
692 * @input_addr to a SysAddr.
694 static inline u64
input_addr_to_sys_addr(struct mem_ctl_info
*mci
,
697 return dram_addr_to_sys_addr(mci
,
698 input_addr_to_dram_addr(mci
, input_addr
));
702 * Find the minimum and maximum InputAddr values that map to the given @csrow.
703 * Pass back these values in *input_addr_min and *input_addr_max.
705 static void find_csrow_limits(struct mem_ctl_info
*mci
, int csrow
,
706 u64
*input_addr_min
, u64
*input_addr_max
)
708 struct amd64_pvt
*pvt
;
712 BUG_ON((csrow
< 0) || (csrow
>= pvt
->csels
[0].b_cnt
));
714 get_cs_base_and_mask(pvt
, csrow
, 0, &base
, &mask
);
716 *input_addr_min
= base
& ~mask
;
717 *input_addr_max
= base
| mask
;
720 /* Map the Error address to a PAGE and PAGE OFFSET. */
721 static inline void error_address_to_page_and_offset(u64 error_address
,
722 u32
*page
, u32
*offset
)
724 *page
= (u32
) (error_address
>> PAGE_SHIFT
);
725 *offset
= ((u32
) error_address
) & ~PAGE_MASK
;
729 * @sys_addr is an error address (a SysAddr) extracted from the MCA NB Address
730 * Low (section 3.6.4.5) and MCA NB Address High (section 3.6.4.6) registers
731 * of a node that detected an ECC memory error. mci represents the node that
732 * the error address maps to (possibly different from the node that detected
733 * the error). Return the number of the csrow that sys_addr maps to, or -1 on
736 static int sys_addr_to_csrow(struct mem_ctl_info
*mci
, u64 sys_addr
)
740 csrow
= input_addr_to_csrow(mci
, sys_addr_to_input_addr(mci
, sys_addr
));
743 amd64_mc_err(mci
, "Failed to translate InputAddr to csrow for "
744 "address 0x%lx\n", (unsigned long)sys_addr
);
748 static int get_channel_from_ecc_syndrome(struct mem_ctl_info
*, u16
);
751 * Determine if the DIMMs have ECC enabled. ECC is enabled ONLY if all the DIMMs
754 static enum edac_type
amd64_determine_edac_cap(struct amd64_pvt
*pvt
)
757 enum dev_type edac_cap
= EDAC_FLAG_NONE
;
759 bit
= (boot_cpu_data
.x86
> 0xf || pvt
->ext_model
>= K8_REV_F
)
763 if (pvt
->dclr0
& BIT(bit
))
764 edac_cap
= EDAC_FLAG_SECDED
;
769 static void amd64_debug_display_dimm_sizes(struct amd64_pvt
*, u8
);
771 static void amd64_dump_dramcfg_low(u32 dclr
, int chan
)
773 debugf1("F2x%d90 (DRAM Cfg Low): 0x%08x\n", chan
, dclr
);
775 debugf1(" DIMM type: %sbuffered; all DIMMs support ECC: %s\n",
776 (dclr
& BIT(16)) ? "un" : "",
777 (dclr
& BIT(19)) ? "yes" : "no");
779 debugf1(" PAR/ERR parity: %s\n",
780 (dclr
& BIT(8)) ? "enabled" : "disabled");
782 if (boot_cpu_data
.x86
== 0x10)
783 debugf1(" DCT 128bit mode width: %s\n",
784 (dclr
& BIT(11)) ? "128b" : "64b");
786 debugf1(" x4 logical DIMMs present: L0: %s L1: %s L2: %s L3: %s\n",
787 (dclr
& BIT(12)) ? "yes" : "no",
788 (dclr
& BIT(13)) ? "yes" : "no",
789 (dclr
& BIT(14)) ? "yes" : "no",
790 (dclr
& BIT(15)) ? "yes" : "no");
793 /* Display and decode various NB registers for debug purposes. */
794 static void dump_misc_regs(struct amd64_pvt
*pvt
)
796 debugf1("F3xE8 (NB Cap): 0x%08x\n", pvt
->nbcap
);
798 debugf1(" NB two channel DRAM capable: %s\n",
799 (pvt
->nbcap
& NBCAP_DCT_DUAL
) ? "yes" : "no");
801 debugf1(" ECC capable: %s, ChipKill ECC capable: %s\n",
802 (pvt
->nbcap
& NBCAP_SECDED
) ? "yes" : "no",
803 (pvt
->nbcap
& NBCAP_CHIPKILL
) ? "yes" : "no");
805 amd64_dump_dramcfg_low(pvt
->dclr0
, 0);
807 debugf1("F3xB0 (Online Spare): 0x%08x\n", pvt
->online_spare
);
809 debugf1("F1xF0 (DRAM Hole Address): 0x%08x, base: 0x%08x, "
811 pvt
->dhar
, dhar_base(pvt
),
812 (boot_cpu_data
.x86
== 0xf) ? k8_dhar_offset(pvt
)
813 : f10_dhar_offset(pvt
));
815 debugf1(" DramHoleValid: %s\n", dhar_valid(pvt
) ? "yes" : "no");
817 amd64_debug_display_dimm_sizes(pvt
, 0);
819 /* everything below this point is Fam10h and above */
820 if (boot_cpu_data
.x86
== 0xf)
823 amd64_debug_display_dimm_sizes(pvt
, 1);
825 amd64_info("using %s syndromes.\n", ((pvt
->ecc_sym_sz
== 8) ? "x8" : "x4"));
827 /* Only if NOT ganged does dclr1 have valid info */
828 if (!dct_ganging_enabled(pvt
))
829 amd64_dump_dramcfg_low(pvt
->dclr1
, 1);
833 * see BKDG, F2x[1,0][5C:40], F2[1,0][6C:60]
835 static void prep_chip_selects(struct amd64_pvt
*pvt
)
837 if (boot_cpu_data
.x86
== 0xf && pvt
->ext_model
< K8_REV_F
) {
838 pvt
->csels
[0].b_cnt
= pvt
->csels
[1].b_cnt
= 8;
839 pvt
->csels
[0].m_cnt
= pvt
->csels
[1].m_cnt
= 8;
841 pvt
->csels
[0].b_cnt
= pvt
->csels
[1].b_cnt
= 8;
842 pvt
->csels
[0].m_cnt
= pvt
->csels
[1].m_cnt
= 4;
847 * Function 2 Offset F10_DCSB0; read in the DCS Base and DCS Mask registers
849 static void read_dct_base_mask(struct amd64_pvt
*pvt
)
853 prep_chip_selects(pvt
);
855 for_each_chip_select(cs
, 0, pvt
) {
856 int reg0
= DCSB0
+ (cs
* 4);
857 int reg1
= DCSB1
+ (cs
* 4);
858 u32
*base0
= &pvt
->csels
[0].csbases
[cs
];
859 u32
*base1
= &pvt
->csels
[1].csbases
[cs
];
861 if (!amd64_read_dct_pci_cfg(pvt
, reg0
, base0
))
862 debugf0(" DCSB0[%d]=0x%08x reg: F2x%x\n",
865 if (boot_cpu_data
.x86
== 0xf || dct_ganging_enabled(pvt
))
868 if (!amd64_read_dct_pci_cfg(pvt
, reg1
, base1
))
869 debugf0(" DCSB1[%d]=0x%08x reg: F2x%x\n",
873 for_each_chip_select_mask(cs
, 0, pvt
) {
874 int reg0
= DCSM0
+ (cs
* 4);
875 int reg1
= DCSM1
+ (cs
* 4);
876 u32
*mask0
= &pvt
->csels
[0].csmasks
[cs
];
877 u32
*mask1
= &pvt
->csels
[1].csmasks
[cs
];
879 if (!amd64_read_dct_pci_cfg(pvt
, reg0
, mask0
))
880 debugf0(" DCSM0[%d]=0x%08x reg: F2x%x\n",
883 if (boot_cpu_data
.x86
== 0xf || dct_ganging_enabled(pvt
))
886 if (!amd64_read_dct_pci_cfg(pvt
, reg1
, mask1
))
887 debugf0(" DCSM1[%d]=0x%08x reg: F2x%x\n",
892 static enum mem_type
amd64_determine_memory_type(struct amd64_pvt
*pvt
, int cs
)
896 /* F15h supports only DDR3 */
897 if (boot_cpu_data
.x86
>= 0x15)
898 type
= (pvt
->dclr0
& BIT(16)) ? MEM_DDR3
: MEM_RDDR3
;
899 else if (boot_cpu_data
.x86
== 0x10 || pvt
->ext_model
>= K8_REV_F
) {
900 if (pvt
->dchr0
& DDR3_MODE
)
901 type
= (pvt
->dclr0
& BIT(16)) ? MEM_DDR3
: MEM_RDDR3
;
903 type
= (pvt
->dclr0
& BIT(16)) ? MEM_DDR2
: MEM_RDDR2
;
905 type
= (pvt
->dclr0
& BIT(18)) ? MEM_DDR
: MEM_RDDR
;
908 amd64_info("CS%d: %s\n", cs
, edac_mem_types
[type
]);
913 /* Get the number of DCT channels the memory controller is using. */
914 static int k8_early_channel_count(struct amd64_pvt
*pvt
)
918 if (pvt
->ext_model
>= K8_REV_F
)
919 /* RevF (NPT) and later */
920 flag
= pvt
->dclr0
& WIDTH_128
;
922 /* RevE and earlier */
923 flag
= pvt
->dclr0
& REVE_WIDTH_128
;
928 return (flag
) ? 2 : 1;
931 /* On F10h and later ErrAddr is MC4_ADDR[47:1] */
932 static u64
get_error_address(struct mce
*m
)
937 if (boot_cpu_data
.x86
== 0xf) {
942 return m
->addr
& GENMASK(start_bit
, end_bit
);
945 static void read_dram_base_limit_regs(struct amd64_pvt
*pvt
, unsigned range
)
947 int off
= range
<< 3;
949 amd64_read_pci_cfg(pvt
->F1
, DRAM_BASE_LO
+ off
, &pvt
->ranges
[range
].base
.lo
);
950 amd64_read_pci_cfg(pvt
->F1
, DRAM_LIMIT_LO
+ off
, &pvt
->ranges
[range
].lim
.lo
);
952 if (boot_cpu_data
.x86
== 0xf)
955 if (!dram_rw(pvt
, range
))
958 amd64_read_pci_cfg(pvt
->F1
, DRAM_BASE_HI
+ off
, &pvt
->ranges
[range
].base
.hi
);
959 amd64_read_pci_cfg(pvt
->F1
, DRAM_LIMIT_HI
+ off
, &pvt
->ranges
[range
].lim
.hi
);
962 static void k8_map_sysaddr_to_csrow(struct mem_ctl_info
*mci
, u64 sys_addr
,
965 struct mem_ctl_info
*src_mci
;
966 struct amd64_pvt
*pvt
= mci
->pvt_info
;
970 /* CHIPKILL enabled */
971 if (pvt
->nbcfg
& NBCFG_CHIPKILL
) {
972 channel
= get_channel_from_ecc_syndrome(mci
, syndrome
);
975 * Syndrome didn't map, so we don't know which of the
976 * 2 DIMMs is in error. So we need to ID 'both' of them
979 amd64_mc_warn(mci
, "unknown syndrome 0x%04x - possible "
980 "error reporting race\n", syndrome
);
981 edac_mc_handle_ce_no_info(mci
, EDAC_MOD_STR
);
986 * non-chipkill ecc mode
988 * The k8 documentation is unclear about how to determine the
989 * channel number when using non-chipkill memory. This method
990 * was obtained from email communication with someone at AMD.
991 * (Wish the email was placed in this comment - norsk)
993 channel
= ((sys_addr
& BIT(3)) != 0);
997 * Find out which node the error address belongs to. This may be
998 * different from the node that detected the error.
1000 src_mci
= find_mc_by_sys_addr(mci
, sys_addr
);
1002 amd64_mc_err(mci
, "failed to map error addr 0x%lx to a node\n",
1003 (unsigned long)sys_addr
);
1004 edac_mc_handle_ce_no_info(mci
, EDAC_MOD_STR
);
1008 /* Now map the sys_addr to a CSROW */
1009 csrow
= sys_addr_to_csrow(src_mci
, sys_addr
);
1011 edac_mc_handle_ce_no_info(src_mci
, EDAC_MOD_STR
);
1013 error_address_to_page_and_offset(sys_addr
, &page
, &offset
);
1015 edac_mc_handle_ce(src_mci
, page
, offset
, syndrome
, csrow
,
1016 channel
, EDAC_MOD_STR
);
1020 static int ddr2_cs_size(unsigned i
, bool dct_width
)
1026 else if (!(i
& 0x1))
1029 shift
= (i
+ 1) >> 1;
1031 return 128 << (shift
+ !!dct_width
);
1034 static int k8_dbam_to_chip_select(struct amd64_pvt
*pvt
, u8 dct
,
1037 u32 dclr
= dct
? pvt
->dclr1
: pvt
->dclr0
;
1039 if (pvt
->ext_model
>= K8_REV_F
) {
1040 WARN_ON(cs_mode
> 11);
1041 return ddr2_cs_size(cs_mode
, dclr
& WIDTH_128
);
1043 else if (pvt
->ext_model
>= K8_REV_D
) {
1044 WARN_ON(cs_mode
> 10);
1046 if (cs_mode
== 3 || cs_mode
== 8)
1047 return 32 << (cs_mode
- 1);
1049 return 32 << cs_mode
;
1052 WARN_ON(cs_mode
> 6);
1053 return 32 << cs_mode
;
1058 * Get the number of DCT channels in use.
1061 * number of Memory Channels in operation
1063 * contents of the DCL0_LOW register
1065 static int f1x_early_channel_count(struct amd64_pvt
*pvt
)
1067 int i
, j
, channels
= 0;
1069 /* On F10h, if we are in 128 bit mode, then we are using 2 channels */
1070 if (boot_cpu_data
.x86
== 0x10 && (pvt
->dclr0
& WIDTH_128
))
1074 * Need to check if in unganged mode: In such, there are 2 channels,
1075 * but they are not in 128 bit mode and thus the above 'dclr0' status
1078 * Need to check DCT0[0] and DCT1[0] to see if only one of them has
1079 * their CSEnable bit on. If so, then SINGLE DIMM case.
1081 debugf0("Data width is not 128 bits - need more decoding\n");
1084 * Check DRAM Bank Address Mapping values for each DIMM to see if there
1085 * is more than just one DIMM present in unganged mode. Need to check
1086 * both controllers since DIMMs can be placed in either one.
1088 for (i
= 0; i
< 2; i
++) {
1089 u32 dbam
= (i
? pvt
->dbam1
: pvt
->dbam0
);
1091 for (j
= 0; j
< 4; j
++) {
1092 if (DBAM_DIMM(j
, dbam
) > 0) {
1102 amd64_info("MCT channel count: %d\n", channels
);
1107 static int ddr3_cs_size(unsigned i
, bool dct_width
)
1112 if (i
== 0 || i
== 3 || i
== 4)
1118 else if (!(i
& 0x1))
1121 shift
= (i
+ 1) >> 1;
1124 cs_size
= (128 * (1 << !!dct_width
)) << shift
;
1129 static int f10_dbam_to_chip_select(struct amd64_pvt
*pvt
, u8 dct
,
1132 u32 dclr
= dct
? pvt
->dclr1
: pvt
->dclr0
;
1134 WARN_ON(cs_mode
> 11);
1136 if (pvt
->dchr0
& DDR3_MODE
|| pvt
->dchr1
& DDR3_MODE
)
1137 return ddr3_cs_size(cs_mode
, dclr
& WIDTH_128
);
1139 return ddr2_cs_size(cs_mode
, dclr
& WIDTH_128
);
1143 * F15h supports only 64bit DCT interfaces
1145 static int f15_dbam_to_chip_select(struct amd64_pvt
*pvt
, u8 dct
,
1148 WARN_ON(cs_mode
> 12);
1150 return ddr3_cs_size(cs_mode
, false);
1153 static void read_dram_ctl_register(struct amd64_pvt
*pvt
)
1156 if (boot_cpu_data
.x86
== 0xf)
1159 if (!amd64_read_dct_pci_cfg(pvt
, DCT_SEL_LO
, &pvt
->dct_sel_lo
)) {
1160 debugf0("F2x110 (DCTSelLow): 0x%08x, High range addrs at: 0x%x\n",
1161 pvt
->dct_sel_lo
, dct_sel_baseaddr(pvt
));
1163 debugf0(" DCTs operate in %s mode.\n",
1164 (dct_ganging_enabled(pvt
) ? "ganged" : "unganged"));
1166 if (!dct_ganging_enabled(pvt
))
1167 debugf0(" Address range split per DCT: %s\n",
1168 (dct_high_range_enabled(pvt
) ? "yes" : "no"));
1170 debugf0(" data interleave for ECC: %s, "
1171 "DRAM cleared since last warm reset: %s\n",
1172 (dct_data_intlv_enabled(pvt
) ? "enabled" : "disabled"),
1173 (dct_memory_cleared(pvt
) ? "yes" : "no"));
1175 debugf0(" channel interleave: %s, "
1176 "interleave bits selector: 0x%x\n",
1177 (dct_interleave_enabled(pvt
) ? "enabled" : "disabled"),
1178 dct_sel_interleave_addr(pvt
));
1181 amd64_read_dct_pci_cfg(pvt
, DCT_SEL_HI
, &pvt
->dct_sel_hi
);
1185 * Determine channel (DCT) based on the interleaving mode: F10h BKDG, 2.8.9 Memory
1186 * Interleaving Modes.
1188 static u8
f1x_determine_channel(struct amd64_pvt
*pvt
, u64 sys_addr
,
1189 bool hi_range_sel
, u8 intlv_en
)
1191 u8 dct_sel_high
= (pvt
->dct_sel_lo
>> 1) & 1;
1193 if (dct_ganging_enabled(pvt
))
1197 return dct_sel_high
;
1200 * see F2x110[DctSelIntLvAddr] - channel interleave mode
1202 if (dct_interleave_enabled(pvt
)) {
1203 u8 intlv_addr
= dct_sel_interleave_addr(pvt
);
1205 /* return DCT select function: 0=DCT0, 1=DCT1 */
1207 return sys_addr
>> 6 & 1;
1209 if (intlv_addr
& 0x2) {
1210 u8 shift
= intlv_addr
& 0x1 ? 9 : 6;
1211 u32 temp
= hweight_long((u32
) ((sys_addr
>> 16) & 0x1F)) % 2;
1213 return ((sys_addr
>> shift
) & 1) ^ temp
;
1216 return (sys_addr
>> (12 + hweight8(intlv_en
))) & 1;
1219 if (dct_high_range_enabled(pvt
))
1220 return ~dct_sel_high
& 1;
1225 /* Convert the sys_addr to the normalized DCT address */
1226 static u64
f1x_get_norm_dct_addr(struct amd64_pvt
*pvt
, unsigned range
,
1227 u64 sys_addr
, bool hi_rng
,
1228 u32 dct_sel_base_addr
)
1231 u64 dram_base
= get_dram_base(pvt
, range
);
1232 u64 hole_off
= f10_dhar_offset(pvt
);
1233 u64 dct_sel_base_off
= (pvt
->dct_sel_hi
& 0xFFFFFC00) << 16;
1238 * base address of high range is below 4Gb
1239 * (bits [47:27] at [31:11])
1240 * DRAM address space on this DCT is hoisted above 4Gb &&
1243 * remove hole offset from sys_addr
1245 * remove high range offset from sys_addr
1247 if ((!(dct_sel_base_addr
>> 16) ||
1248 dct_sel_base_addr
< dhar_base(pvt
)) &&
1250 (sys_addr
>= BIT_64(32)))
1251 chan_off
= hole_off
;
1253 chan_off
= dct_sel_base_off
;
1257 * we have a valid hole &&
1262 * remove dram base to normalize to DCT address
1264 if (dhar_valid(pvt
) && (sys_addr
>= BIT_64(32)))
1265 chan_off
= hole_off
;
1267 chan_off
= dram_base
;
1270 return (sys_addr
& GENMASK(6,47)) - (chan_off
& GENMASK(23,47));
1274 * checks if the csrow passed in is marked as SPARED, if so returns the new
1277 static int f10_process_possible_spare(struct amd64_pvt
*pvt
, u8 dct
, int csrow
)
1281 if (online_spare_swap_done(pvt
, dct
) &&
1282 csrow
== online_spare_bad_dramcs(pvt
, dct
)) {
1284 for_each_chip_select(tmp_cs
, dct
, pvt
) {
1285 if (chip_select_base(tmp_cs
, dct
, pvt
) & 0x2) {
1295 * Iterate over the DRAM DCT "base" and "mask" registers looking for a
1296 * SystemAddr match on the specified 'ChannelSelect' and 'NodeID'
1299 * -EINVAL: NOT FOUND
1300 * 0..csrow = Chip-Select Row
1302 static int f1x_lookup_addr_in_dct(u64 in_addr
, u32 nid
, u8 dct
)
1304 struct mem_ctl_info
*mci
;
1305 struct amd64_pvt
*pvt
;
1306 u64 cs_base
, cs_mask
;
1307 int cs_found
= -EINVAL
;
1314 pvt
= mci
->pvt_info
;
1316 debugf1("input addr: 0x%llx, DCT: %d\n", in_addr
, dct
);
1318 for_each_chip_select(csrow
, dct
, pvt
) {
1319 if (!csrow_enabled(csrow
, dct
, pvt
))
1322 get_cs_base_and_mask(pvt
, csrow
, dct
, &cs_base
, &cs_mask
);
1324 debugf1(" CSROW=%d CSBase=0x%llx CSMask=0x%llx\n",
1325 csrow
, cs_base
, cs_mask
);
1329 debugf1(" (InputAddr & ~CSMask)=0x%llx "
1330 "(CSBase & ~CSMask)=0x%llx\n",
1331 (in_addr
& cs_mask
), (cs_base
& cs_mask
));
1333 if ((in_addr
& cs_mask
) == (cs_base
& cs_mask
)) {
1334 cs_found
= f10_process_possible_spare(pvt
, dct
, csrow
);
1336 debugf1(" MATCH csrow=%d\n", cs_found
);
1344 * See F2x10C. Non-interleaved graphics framebuffer memory under the 16G is
1345 * swapped with a region located at the bottom of memory so that the GPU can use
1346 * the interleaved region and thus two channels.
1348 static u64
f1x_swap_interleaved_region(struct amd64_pvt
*pvt
, u64 sys_addr
)
1350 u32 swap_reg
, swap_base
, swap_limit
, rgn_size
, tmp_addr
;
1352 if (boot_cpu_data
.x86
== 0x10) {
1353 /* only revC3 and revE have that feature */
1354 if (boot_cpu_data
.x86_model
< 4 ||
1355 (boot_cpu_data
.x86_model
< 0xa &&
1356 boot_cpu_data
.x86_mask
< 3))
1360 amd64_read_dct_pci_cfg(pvt
, SWAP_INTLV_REG
, &swap_reg
);
1362 if (!(swap_reg
& 0x1))
1365 swap_base
= (swap_reg
>> 3) & 0x7f;
1366 swap_limit
= (swap_reg
>> 11) & 0x7f;
1367 rgn_size
= (swap_reg
>> 20) & 0x7f;
1368 tmp_addr
= sys_addr
>> 27;
1370 if (!(sys_addr
>> 34) &&
1371 (((tmp_addr
>= swap_base
) &&
1372 (tmp_addr
<= swap_limit
)) ||
1373 (tmp_addr
< rgn_size
)))
1374 return sys_addr
^ (u64
)swap_base
<< 27;
1379 /* For a given @dram_range, check if @sys_addr falls within it. */
1380 static int f1x_match_to_this_node(struct amd64_pvt
*pvt
, unsigned range
,
1381 u64 sys_addr
, int *nid
, int *chan_sel
)
1383 int cs_found
= -EINVAL
;
1387 bool high_range
= false;
1389 u8 node_id
= dram_dst_node(pvt
, range
);
1390 u8 intlv_en
= dram_intlv_en(pvt
, range
);
1391 u32 intlv_sel
= dram_intlv_sel(pvt
, range
);
1393 debugf1("(range %d) SystemAddr= 0x%llx Limit=0x%llx\n",
1394 range
, sys_addr
, get_dram_limit(pvt
, range
));
1396 if (dhar_valid(pvt
) &&
1397 dhar_base(pvt
) <= sys_addr
&&
1398 sys_addr
< BIT_64(32)) {
1399 amd64_warn("Huh? Address is in the MMIO hole: 0x%016llx\n",
1405 (intlv_sel
!= ((sys_addr
>> 12) & intlv_en
))) {
1406 amd64_warn("Botched intlv bits, en: 0x%x, sel: 0x%x\n",
1407 intlv_en
, intlv_sel
);
1411 sys_addr
= f1x_swap_interleaved_region(pvt
, sys_addr
);
1413 dct_sel_base
= dct_sel_baseaddr(pvt
);
1416 * check whether addresses >= DctSelBaseAddr[47:27] are to be used to
1417 * select between DCT0 and DCT1.
1419 if (dct_high_range_enabled(pvt
) &&
1420 !dct_ganging_enabled(pvt
) &&
1421 ((sys_addr
>> 27) >= (dct_sel_base
>> 11)))
1424 channel
= f1x_determine_channel(pvt
, sys_addr
, high_range
, intlv_en
);
1426 chan_addr
= f1x_get_norm_dct_addr(pvt
, range
, sys_addr
,
1427 high_range
, dct_sel_base
);
1429 /* Remove node interleaving, see F1x120 */
1431 chan_addr
= ((chan_addr
>> (12 + hweight8(intlv_en
))) << 12) |
1432 (chan_addr
& 0xfff);
1434 /* remove channel interleave */
1435 if (dct_interleave_enabled(pvt
) &&
1436 !dct_high_range_enabled(pvt
) &&
1437 !dct_ganging_enabled(pvt
)) {
1439 if (dct_sel_interleave_addr(pvt
) != 1) {
1440 if (dct_sel_interleave_addr(pvt
) == 0x3)
1442 chan_addr
= ((chan_addr
>> 10) << 9) |
1443 (chan_addr
& 0x1ff);
1445 /* A[6] or hash 6 */
1446 chan_addr
= ((chan_addr
>> 7) << 6) |
1450 chan_addr
= ((chan_addr
>> 13) << 12) |
1451 (chan_addr
& 0xfff);
1454 debugf1(" Normalized DCT addr: 0x%llx\n", chan_addr
);
1456 cs_found
= f1x_lookup_addr_in_dct(chan_addr
, node_id
, channel
);
1458 if (cs_found
>= 0) {
1460 *chan_sel
= channel
;
1465 static int f1x_translate_sysaddr_to_cs(struct amd64_pvt
*pvt
, u64 sys_addr
,
1466 int *node
, int *chan_sel
)
1468 int cs_found
= -EINVAL
;
1471 for (range
= 0; range
< DRAM_RANGES
; range
++) {
1473 if (!dram_rw(pvt
, range
))
1476 if ((get_dram_base(pvt
, range
) <= sys_addr
) &&
1477 (get_dram_limit(pvt
, range
) >= sys_addr
)) {
1479 cs_found
= f1x_match_to_this_node(pvt
, range
,
1490 * For reference see "2.8.5 Routing DRAM Requests" in F10 BKDG. This code maps
1491 * a @sys_addr to NodeID, DCT (channel) and chip select (CSROW).
1493 * The @sys_addr is usually an error address received from the hardware
1496 static void f1x_map_sysaddr_to_csrow(struct mem_ctl_info
*mci
, u64 sys_addr
,
1499 struct amd64_pvt
*pvt
= mci
->pvt_info
;
1501 int nid
, csrow
, chan
= 0;
1503 csrow
= f1x_translate_sysaddr_to_cs(pvt
, sys_addr
, &nid
, &chan
);
1506 edac_mc_handle_ce_no_info(mci
, EDAC_MOD_STR
);
1510 error_address_to_page_and_offset(sys_addr
, &page
, &offset
);
1513 * We need the syndromes for channel detection only when we're
1514 * ganged. Otherwise @chan should already contain the channel at
1517 if (dct_ganging_enabled(pvt
))
1518 chan
= get_channel_from_ecc_syndrome(mci
, syndrome
);
1521 edac_mc_handle_ce(mci
, page
, offset
, syndrome
, csrow
, chan
,
1525 * Channel unknown, report all channels on this CSROW as failed.
1527 for (chan
= 0; chan
< mci
->csrows
[csrow
].nr_channels
; chan
++)
1528 edac_mc_handle_ce(mci
, page
, offset
, syndrome
,
1529 csrow
, chan
, EDAC_MOD_STR
);
1533 * debug routine to display the memory sizes of all logical DIMMs and its
1536 static void amd64_debug_display_dimm_sizes(struct amd64_pvt
*pvt
, u8 ctrl
)
1538 int dimm
, size0
, size1
, factor
= 0;
1539 u32
*dcsb
= ctrl
? pvt
->csels
[1].csbases
: pvt
->csels
[0].csbases
;
1540 u32 dbam
= ctrl
? pvt
->dbam1
: pvt
->dbam0
;
1542 if (boot_cpu_data
.x86
== 0xf) {
1543 if (pvt
->dclr0
& WIDTH_128
)
1546 /* K8 families < revF not supported yet */
1547 if (pvt
->ext_model
< K8_REV_F
)
1553 dbam
= (ctrl
&& !dct_ganging_enabled(pvt
)) ? pvt
->dbam1
: pvt
->dbam0
;
1554 dcsb
= (ctrl
&& !dct_ganging_enabled(pvt
)) ? pvt
->csels
[1].csbases
1555 : pvt
->csels
[0].csbases
;
1557 debugf1("F2x%d80 (DRAM Bank Address Mapping): 0x%08x\n", ctrl
, dbam
);
1559 edac_printk(KERN_DEBUG
, EDAC_MC
, "DCT%d chip selects:\n", ctrl
);
1561 /* Dump memory sizes for DIMM and its CSROWs */
1562 for (dimm
= 0; dimm
< 4; dimm
++) {
1565 if (dcsb
[dimm
*2] & DCSB_CS_ENABLE
)
1566 size0
= pvt
->ops
->dbam_to_cs(pvt
, ctrl
,
1567 DBAM_DIMM(dimm
, dbam
));
1570 if (dcsb
[dimm
*2 + 1] & DCSB_CS_ENABLE
)
1571 size1
= pvt
->ops
->dbam_to_cs(pvt
, ctrl
,
1572 DBAM_DIMM(dimm
, dbam
));
1574 amd64_info(EDAC_MC
": %d: %5dMB %d: %5dMB\n",
1575 dimm
* 2, size0
<< factor
,
1576 dimm
* 2 + 1, size1
<< factor
);
1580 static struct amd64_family_type amd64_family_types
[] = {
1583 .f1_id
= PCI_DEVICE_ID_AMD_K8_NB_ADDRMAP
,
1584 .f3_id
= PCI_DEVICE_ID_AMD_K8_NB_MISC
,
1586 .early_channel_count
= k8_early_channel_count
,
1587 .map_sysaddr_to_csrow
= k8_map_sysaddr_to_csrow
,
1588 .dbam_to_cs
= k8_dbam_to_chip_select
,
1589 .read_dct_pci_cfg
= k8_read_dct_pci_cfg
,
1594 .f1_id
= PCI_DEVICE_ID_AMD_10H_NB_MAP
,
1595 .f3_id
= PCI_DEVICE_ID_AMD_10H_NB_MISC
,
1597 .early_channel_count
= f1x_early_channel_count
,
1598 .map_sysaddr_to_csrow
= f1x_map_sysaddr_to_csrow
,
1599 .dbam_to_cs
= f10_dbam_to_chip_select
,
1600 .read_dct_pci_cfg
= f10_read_dct_pci_cfg
,
1605 .f1_id
= PCI_DEVICE_ID_AMD_15H_NB_F1
,
1606 .f3_id
= PCI_DEVICE_ID_AMD_15H_NB_F3
,
1608 .early_channel_count
= f1x_early_channel_count
,
1609 .map_sysaddr_to_csrow
= f1x_map_sysaddr_to_csrow
,
1610 .dbam_to_cs
= f15_dbam_to_chip_select
,
1611 .read_dct_pci_cfg
= f15_read_dct_pci_cfg
,
1616 static struct pci_dev
*pci_get_related_function(unsigned int vendor
,
1617 unsigned int device
,
1618 struct pci_dev
*related
)
1620 struct pci_dev
*dev
= NULL
;
1622 dev
= pci_get_device(vendor
, device
, dev
);
1624 if ((dev
->bus
->number
== related
->bus
->number
) &&
1625 (PCI_SLOT(dev
->devfn
) == PCI_SLOT(related
->devfn
)))
1627 dev
= pci_get_device(vendor
, device
, dev
);
1634 * These are tables of eigenvectors (one per line) which can be used for the
1635 * construction of the syndrome tables. The modified syndrome search algorithm
1636 * uses those to find the symbol in error and thus the DIMM.
1638 * Algorithm courtesy of Ross LaFetra from AMD.
1640 static u16 x4_vectors
[] = {
1641 0x2f57, 0x1afe, 0x66cc, 0xdd88,
1642 0x11eb, 0x3396, 0x7f4c, 0xeac8,
1643 0x0001, 0x0002, 0x0004, 0x0008,
1644 0x1013, 0x3032, 0x4044, 0x8088,
1645 0x106b, 0x30d6, 0x70fc, 0xe0a8,
1646 0x4857, 0xc4fe, 0x13cc, 0x3288,
1647 0x1ac5, 0x2f4a, 0x5394, 0xa1e8,
1648 0x1f39, 0x251e, 0xbd6c, 0x6bd8,
1649 0x15c1, 0x2a42, 0x89ac, 0x4758,
1650 0x2b03, 0x1602, 0x4f0c, 0xca08,
1651 0x1f07, 0x3a0e, 0x6b04, 0xbd08,
1652 0x8ba7, 0x465e, 0x244c, 0x1cc8,
1653 0x2b87, 0x164e, 0x642c, 0xdc18,
1654 0x40b9, 0x80de, 0x1094, 0x20e8,
1655 0x27db, 0x1eb6, 0x9dac, 0x7b58,
1656 0x11c1, 0x2242, 0x84ac, 0x4c58,
1657 0x1be5, 0x2d7a, 0x5e34, 0xa718,
1658 0x4b39, 0x8d1e, 0x14b4, 0x28d8,
1659 0x4c97, 0xc87e, 0x11fc, 0x33a8,
1660 0x8e97, 0x497e, 0x2ffc, 0x1aa8,
1661 0x16b3, 0x3d62, 0x4f34, 0x8518,
1662 0x1e2f, 0x391a, 0x5cac, 0xf858,
1663 0x1d9f, 0x3b7a, 0x572c, 0xfe18,
1664 0x15f5, 0x2a5a, 0x5264, 0xa3b8,
1665 0x1dbb, 0x3b66, 0x715c, 0xe3f8,
1666 0x4397, 0xc27e, 0x17fc, 0x3ea8,
1667 0x1617, 0x3d3e, 0x6464, 0xb8b8,
1668 0x23ff, 0x12aa, 0xab6c, 0x56d8,
1669 0x2dfb, 0x1ba6, 0x913c, 0x7328,
1670 0x185d, 0x2ca6, 0x7914, 0x9e28,
1671 0x171b, 0x3e36, 0x7d7c, 0xebe8,
1672 0x4199, 0x82ee, 0x19f4, 0x2e58,
1673 0x4807, 0xc40e, 0x130c, 0x3208,
1674 0x1905, 0x2e0a, 0x5804, 0xac08,
1675 0x213f, 0x132a, 0xadfc, 0x5ba8,
1676 0x19a9, 0x2efe, 0xb5cc, 0x6f88,
1679 static u16 x8_vectors
[] = {
1680 0x0145, 0x028a, 0x2374, 0x43c8, 0xa1f0, 0x0520, 0x0a40, 0x1480,
1681 0x0211, 0x0422, 0x0844, 0x1088, 0x01b0, 0x44e0, 0x23c0, 0xed80,
1682 0x1011, 0x0116, 0x022c, 0x0458, 0x08b0, 0x8c60, 0x2740, 0x4e80,
1683 0x0411, 0x0822, 0x1044, 0x0158, 0x02b0, 0x2360, 0x46c0, 0xab80,
1684 0x0811, 0x1022, 0x012c, 0x0258, 0x04b0, 0x4660, 0x8cc0, 0x2780,
1685 0x2071, 0x40e2, 0xa0c4, 0x0108, 0x0210, 0x0420, 0x0840, 0x1080,
1686 0x4071, 0x80e2, 0x0104, 0x0208, 0x0410, 0x0820, 0x1040, 0x2080,
1687 0x8071, 0x0102, 0x0204, 0x0408, 0x0810, 0x1020, 0x2040, 0x4080,
1688 0x019d, 0x03d6, 0x136c, 0x2198, 0x50b0, 0xb2e0, 0x0740, 0x0e80,
1689 0x0189, 0x03ea, 0x072c, 0x0e58, 0x1cb0, 0x56e0, 0x37c0, 0xf580,
1690 0x01fd, 0x0376, 0x06ec, 0x0bb8, 0x1110, 0x2220, 0x4440, 0x8880,
1691 0x0163, 0x02c6, 0x1104, 0x0758, 0x0eb0, 0x2be0, 0x6140, 0xc280,
1692 0x02fd, 0x01c6, 0x0b5c, 0x1108, 0x07b0, 0x25a0, 0x8840, 0x6180,
1693 0x0801, 0x012e, 0x025c, 0x04b8, 0x1370, 0x26e0, 0x57c0, 0xb580,
1694 0x0401, 0x0802, 0x015c, 0x02b8, 0x22b0, 0x13e0, 0x7140, 0xe280,
1695 0x0201, 0x0402, 0x0804, 0x01b8, 0x11b0, 0x31a0, 0x8040, 0x7180,
1696 0x0101, 0x0202, 0x0404, 0x0808, 0x1010, 0x2020, 0x4040, 0x8080,
1697 0x0001, 0x0002, 0x0004, 0x0008, 0x0010, 0x0020, 0x0040, 0x0080,
1698 0x0100, 0x0200, 0x0400, 0x0800, 0x1000, 0x2000, 0x4000, 0x8000,
1701 static int decode_syndrome(u16 syndrome
, u16
*vectors
, unsigned num_vecs
,
1704 unsigned int i
, err_sym
;
1706 for (err_sym
= 0; err_sym
< num_vecs
/ v_dim
; err_sym
++) {
1708 unsigned v_idx
= err_sym
* v_dim
;
1709 unsigned v_end
= (err_sym
+ 1) * v_dim
;
1711 /* walk over all 16 bits of the syndrome */
1712 for (i
= 1; i
< (1U << 16); i
<<= 1) {
1714 /* if bit is set in that eigenvector... */
1715 if (v_idx
< v_end
&& vectors
[v_idx
] & i
) {
1716 u16 ev_comp
= vectors
[v_idx
++];
1718 /* ... and bit set in the modified syndrome, */
1728 /* can't get to zero, move to next symbol */
1733 debugf0("syndrome(%x) not found\n", syndrome
);
1737 static int map_err_sym_to_channel(int err_sym
, int sym_size
)
1750 return err_sym
>> 4;
1756 /* imaginary bits not in a DIMM */
1758 WARN(1, KERN_ERR
"Invalid error symbol: 0x%x\n",
1770 return err_sym
>> 3;
1776 static int get_channel_from_ecc_syndrome(struct mem_ctl_info
*mci
, u16 syndrome
)
1778 struct amd64_pvt
*pvt
= mci
->pvt_info
;
1781 if (pvt
->ecc_sym_sz
== 8)
1782 err_sym
= decode_syndrome(syndrome
, x8_vectors
,
1783 ARRAY_SIZE(x8_vectors
),
1785 else if (pvt
->ecc_sym_sz
== 4)
1786 err_sym
= decode_syndrome(syndrome
, x4_vectors
,
1787 ARRAY_SIZE(x4_vectors
),
1790 amd64_warn("Illegal syndrome type: %u\n", pvt
->ecc_sym_sz
);
1794 return map_err_sym_to_channel(err_sym
, pvt
->ecc_sym_sz
);
1798 * Handle any Correctable Errors (CEs) that have occurred. Check for valid ERROR
1799 * ADDRESS and process.
1801 static void amd64_handle_ce(struct mem_ctl_info
*mci
, struct mce
*m
)
1803 struct amd64_pvt
*pvt
= mci
->pvt_info
;
1807 /* Ensure that the Error Address is VALID */
1808 if (!(m
->status
& MCI_STATUS_ADDRV
)) {
1809 amd64_mc_err(mci
, "HW has no ERROR_ADDRESS available\n");
1810 edac_mc_handle_ce_no_info(mci
, EDAC_MOD_STR
);
1814 sys_addr
= get_error_address(m
);
1815 syndrome
= extract_syndrome(m
->status
);
1817 amd64_mc_err(mci
, "CE ERROR_ADDRESS= 0x%llx\n", sys_addr
);
1819 pvt
->ops
->map_sysaddr_to_csrow(mci
, sys_addr
, syndrome
);
1822 /* Handle any Un-correctable Errors (UEs) */
1823 static void amd64_handle_ue(struct mem_ctl_info
*mci
, struct mce
*m
)
1825 struct mem_ctl_info
*log_mci
, *src_mci
= NULL
;
1832 if (!(m
->status
& MCI_STATUS_ADDRV
)) {
1833 amd64_mc_err(mci
, "HW has no ERROR_ADDRESS available\n");
1834 edac_mc_handle_ue_no_info(log_mci
, EDAC_MOD_STR
);
1838 sys_addr
= get_error_address(m
);
1841 * Find out which node the error address belongs to. This may be
1842 * different from the node that detected the error.
1844 src_mci
= find_mc_by_sys_addr(mci
, sys_addr
);
1846 amd64_mc_err(mci
, "ERROR ADDRESS (0x%lx) NOT mapped to a MC\n",
1847 (unsigned long)sys_addr
);
1848 edac_mc_handle_ue_no_info(log_mci
, EDAC_MOD_STR
);
1854 csrow
= sys_addr_to_csrow(log_mci
, sys_addr
);
1856 amd64_mc_err(mci
, "ERROR_ADDRESS (0x%lx) NOT mapped to CS\n",
1857 (unsigned long)sys_addr
);
1858 edac_mc_handle_ue_no_info(log_mci
, EDAC_MOD_STR
);
1860 error_address_to_page_and_offset(sys_addr
, &page
, &offset
);
1861 edac_mc_handle_ue(log_mci
, page
, offset
, csrow
, EDAC_MOD_STR
);
1865 static inline void __amd64_decode_bus_error(struct mem_ctl_info
*mci
,
1868 u16 ec
= EC(m
->status
);
1869 u8 xec
= XEC(m
->status
, 0x1f);
1870 u8 ecc_type
= (m
->status
>> 45) & 0x3;
1872 /* Bail early out if this was an 'observed' error */
1873 if (PP(ec
) == NBSL_PP_OBS
)
1876 /* Do only ECC errors */
1877 if (xec
&& xec
!= F10_NBSL_EXT_ERR_ECC
)
1881 amd64_handle_ce(mci
, m
);
1882 else if (ecc_type
== 1)
1883 amd64_handle_ue(mci
, m
);
1886 void amd64_decode_bus_error(int node_id
, struct mce
*m
, u32 nbcfg
)
1888 struct mem_ctl_info
*mci
= mcis
[node_id
];
1890 __amd64_decode_bus_error(mci
, m
);
1894 * Use pvt->F2 which contains the F2 CPU PCI device to get the related
1895 * F1 (AddrMap) and F3 (Misc) devices. Return negative value on error.
1897 static int reserve_mc_sibling_devs(struct amd64_pvt
*pvt
, u16 f1_id
, u16 f3_id
)
1899 /* Reserve the ADDRESS MAP Device */
1900 pvt
->F1
= pci_get_related_function(pvt
->F2
->vendor
, f1_id
, pvt
->F2
);
1902 amd64_err("error address map device not found: "
1903 "vendor %x device 0x%x (broken BIOS?)\n",
1904 PCI_VENDOR_ID_AMD
, f1_id
);
1908 /* Reserve the MISC Device */
1909 pvt
->F3
= pci_get_related_function(pvt
->F2
->vendor
, f3_id
, pvt
->F2
);
1911 pci_dev_put(pvt
->F1
);
1914 amd64_err("error F3 device not found: "
1915 "vendor %x device 0x%x (broken BIOS?)\n",
1916 PCI_VENDOR_ID_AMD
, f3_id
);
1920 debugf1("F1: %s\n", pci_name(pvt
->F1
));
1921 debugf1("F2: %s\n", pci_name(pvt
->F2
));
1922 debugf1("F3: %s\n", pci_name(pvt
->F3
));
1927 static void free_mc_sibling_devs(struct amd64_pvt
*pvt
)
1929 pci_dev_put(pvt
->F1
);
1930 pci_dev_put(pvt
->F3
);
1934 * Retrieve the hardware registers of the memory controller (this includes the
1935 * 'Address Map' and 'Misc' device regs)
1937 static void read_mc_regs(struct amd64_pvt
*pvt
)
1939 struct cpuinfo_x86
*c
= &boot_cpu_data
;
1945 * Retrieve TOP_MEM and TOP_MEM2; no masking off of reserved bits since
1946 * those are Read-As-Zero
1948 rdmsrl(MSR_K8_TOP_MEM1
, pvt
->top_mem
);
1949 debugf0(" TOP_MEM: 0x%016llx\n", pvt
->top_mem
);
1951 /* check first whether TOP_MEM2 is enabled */
1952 rdmsrl(MSR_K8_SYSCFG
, msr_val
);
1953 if (msr_val
& (1U << 21)) {
1954 rdmsrl(MSR_K8_TOP_MEM2
, pvt
->top_mem2
);
1955 debugf0(" TOP_MEM2: 0x%016llx\n", pvt
->top_mem2
);
1957 debugf0(" TOP_MEM2 disabled.\n");
1959 amd64_read_pci_cfg(pvt
->F3
, NBCAP
, &pvt
->nbcap
);
1961 read_dram_ctl_register(pvt
);
1963 for (range
= 0; range
< DRAM_RANGES
; range
++) {
1966 /* read settings for this DRAM range */
1967 read_dram_base_limit_regs(pvt
, range
);
1969 rw
= dram_rw(pvt
, range
);
1973 debugf1(" DRAM range[%d], base: 0x%016llx; limit: 0x%016llx\n",
1975 get_dram_base(pvt
, range
),
1976 get_dram_limit(pvt
, range
));
1978 debugf1(" IntlvEn=%s; Range access: %s%s IntlvSel=%d DstNode=%d\n",
1979 dram_intlv_en(pvt
, range
) ? "Enabled" : "Disabled",
1980 (rw
& 0x1) ? "R" : "-",
1981 (rw
& 0x2) ? "W" : "-",
1982 dram_intlv_sel(pvt
, range
),
1983 dram_dst_node(pvt
, range
));
1986 read_dct_base_mask(pvt
);
1988 amd64_read_pci_cfg(pvt
->F1
, DHAR
, &pvt
->dhar
);
1989 amd64_read_dct_pci_cfg(pvt
, DBAM0
, &pvt
->dbam0
);
1991 amd64_read_pci_cfg(pvt
->F3
, F10_ONLINE_SPARE
, &pvt
->online_spare
);
1993 amd64_read_dct_pci_cfg(pvt
, DCLR0
, &pvt
->dclr0
);
1994 amd64_read_dct_pci_cfg(pvt
, DCHR0
, &pvt
->dchr0
);
1996 if (!dct_ganging_enabled(pvt
)) {
1997 amd64_read_dct_pci_cfg(pvt
, DCLR1
, &pvt
->dclr1
);
1998 amd64_read_dct_pci_cfg(pvt
, DCHR1
, &pvt
->dchr1
);
2001 pvt
->ecc_sym_sz
= 4;
2003 if (c
->x86
>= 0x10) {
2004 amd64_read_pci_cfg(pvt
->F3
, EXT_NB_MCA_CFG
, &tmp
);
2005 amd64_read_dct_pci_cfg(pvt
, DBAM1
, &pvt
->dbam1
);
2007 /* F10h, revD and later can do x8 ECC too */
2008 if ((c
->x86
> 0x10 || c
->x86_model
> 7) && tmp
& BIT(25))
2009 pvt
->ecc_sym_sz
= 8;
2011 dump_misc_regs(pvt
);
2015 * NOTE: CPU Revision Dependent code
2018 * @csrow_nr ChipSelect Row Number (0..NUM_CHIPSELECTS-1)
2019 * k8 private pointer to -->
2020 * DRAM Bank Address mapping register
2022 * DCL register where dual_channel_active is
2024 * The DBAM register consists of 4 sets of 4 bits each definitions:
2027 * 0-3 CSROWs 0 and 1
2028 * 4-7 CSROWs 2 and 3
2029 * 8-11 CSROWs 4 and 5
2030 * 12-15 CSROWs 6 and 7
2032 * Values range from: 0 to 15
2033 * The meaning of the values depends on CPU revision and dual-channel state,
2034 * see relevant BKDG more info.
2036 * The memory controller provides for total of only 8 CSROWs in its current
2037 * architecture. Each "pair" of CSROWs normally represents just one DIMM in
2038 * single channel or two (2) DIMMs in dual channel mode.
2040 * The following code logic collapses the various tables for CSROW based on CPU
2044 * The number of PAGE_SIZE pages on the specified CSROW number it
2048 static u32
amd64_csrow_nr_pages(struct amd64_pvt
*pvt
, u8 dct
, int csrow_nr
)
2050 u32 cs_mode
, nr_pages
;
2053 * The math on this doesn't look right on the surface because x/2*4 can
2054 * be simplified to x*2 but this expression makes use of the fact that
2055 * it is integral math where 1/2=0. This intermediate value becomes the
2056 * number of bits to shift the DBAM register to extract the proper CSROW
2059 cs_mode
= (pvt
->dbam0
>> ((csrow_nr
/ 2) * 4)) & 0xF;
2061 nr_pages
= pvt
->ops
->dbam_to_cs(pvt
, dct
, cs_mode
) << (20 - PAGE_SHIFT
);
2064 * If dual channel then double the memory size of single channel.
2065 * Channel count is 1 or 2
2067 nr_pages
<<= (pvt
->channel_count
- 1);
2069 debugf0(" (csrow=%d) DBAM map index= %d\n", csrow_nr
, cs_mode
);
2070 debugf0(" nr_pages= %u channel-count = %d\n",
2071 nr_pages
, pvt
->channel_count
);
2077 * Initialize the array of csrow attribute instances, based on the values
2078 * from pci config hardware registers.
2080 static int init_csrows(struct mem_ctl_info
*mci
)
2082 struct csrow_info
*csrow
;
2083 struct amd64_pvt
*pvt
= mci
->pvt_info
;
2084 u64 input_addr_min
, input_addr_max
, sys_addr
, base
, mask
;
2088 amd64_read_pci_cfg(pvt
->F3
, NBCFG
, &val
);
2092 debugf0("node %d, NBCFG=0x%08x[ChipKillEccCap: %d|DramEccEn: %d]\n",
2093 pvt
->mc_node_id
, val
,
2094 !!(val
& NBCFG_CHIPKILL
), !!(val
& NBCFG_ECC_ENABLE
));
2096 for_each_chip_select(i
, 0, pvt
) {
2097 csrow
= &mci
->csrows
[i
];
2099 if (!csrow_enabled(i
, 0, pvt
)) {
2100 debugf1("----CSROW %d EMPTY for node %d\n", i
,
2105 debugf1("----CSROW %d VALID for MC node %d\n",
2106 i
, pvt
->mc_node_id
);
2109 csrow
->nr_pages
= amd64_csrow_nr_pages(pvt
, 0, i
);
2110 find_csrow_limits(mci
, i
, &input_addr_min
, &input_addr_max
);
2111 sys_addr
= input_addr_to_sys_addr(mci
, input_addr_min
);
2112 csrow
->first_page
= (u32
) (sys_addr
>> PAGE_SHIFT
);
2113 sys_addr
= input_addr_to_sys_addr(mci
, input_addr_max
);
2114 csrow
->last_page
= (u32
) (sys_addr
>> PAGE_SHIFT
);
2116 get_cs_base_and_mask(pvt
, i
, 0, &base
, &mask
);
2117 csrow
->page_mask
= ~mask
;
2118 /* 8 bytes of resolution */
2120 csrow
->mtype
= amd64_determine_memory_type(pvt
, i
);
2122 debugf1(" for MC node %d csrow %d:\n", pvt
->mc_node_id
, i
);
2123 debugf1(" input_addr_min: 0x%lx input_addr_max: 0x%lx\n",
2124 (unsigned long)input_addr_min
,
2125 (unsigned long)input_addr_max
);
2126 debugf1(" sys_addr: 0x%lx page_mask: 0x%lx\n",
2127 (unsigned long)sys_addr
, csrow
->page_mask
);
2128 debugf1(" nr_pages: %u first_page: 0x%lx "
2129 "last_page: 0x%lx\n",
2130 (unsigned)csrow
->nr_pages
,
2131 csrow
->first_page
, csrow
->last_page
);
2134 * determine whether CHIPKILL or JUST ECC or NO ECC is operating
2136 if (pvt
->nbcfg
& NBCFG_ECC_ENABLE
)
2138 (pvt
->nbcfg
& NBCFG_CHIPKILL
) ?
2139 EDAC_S4ECD4ED
: EDAC_SECDED
;
2141 csrow
->edac_mode
= EDAC_NONE
;
2147 /* get all cores on this DCT */
2148 static void get_cpus_on_this_dct_cpumask(struct cpumask
*mask
, unsigned nid
)
2152 for_each_online_cpu(cpu
)
2153 if (amd_get_nb_id(cpu
) == nid
)
2154 cpumask_set_cpu(cpu
, mask
);
2157 /* check MCG_CTL on all the cpus on this node */
2158 static bool amd64_nb_mce_bank_enabled_on_node(unsigned nid
)
2164 if (!zalloc_cpumask_var(&mask
, GFP_KERNEL
)) {
2165 amd64_warn("%s: Error allocating mask\n", __func__
);
2169 get_cpus_on_this_dct_cpumask(mask
, nid
);
2171 rdmsr_on_cpus(mask
, MSR_IA32_MCG_CTL
, msrs
);
2173 for_each_cpu(cpu
, mask
) {
2174 struct msr
*reg
= per_cpu_ptr(msrs
, cpu
);
2175 nbe
= reg
->l
& MSR_MCGCTL_NBE
;
2177 debugf0("core: %u, MCG_CTL: 0x%llx, NB MSR is %s\n",
2179 (nbe
? "enabled" : "disabled"));
2187 free_cpumask_var(mask
);
2191 static int toggle_ecc_err_reporting(struct ecc_settings
*s
, u8 nid
, bool on
)
2193 cpumask_var_t cmask
;
2196 if (!zalloc_cpumask_var(&cmask
, GFP_KERNEL
)) {
2197 amd64_warn("%s: error allocating mask\n", __func__
);
2201 get_cpus_on_this_dct_cpumask(cmask
, nid
);
2203 rdmsr_on_cpus(cmask
, MSR_IA32_MCG_CTL
, msrs
);
2205 for_each_cpu(cpu
, cmask
) {
2207 struct msr
*reg
= per_cpu_ptr(msrs
, cpu
);
2210 if (reg
->l
& MSR_MCGCTL_NBE
)
2211 s
->flags
.nb_mce_enable
= 1;
2213 reg
->l
|= MSR_MCGCTL_NBE
;
2216 * Turn off NB MCE reporting only when it was off before
2218 if (!s
->flags
.nb_mce_enable
)
2219 reg
->l
&= ~MSR_MCGCTL_NBE
;
2222 wrmsr_on_cpus(cmask
, MSR_IA32_MCG_CTL
, msrs
);
2224 free_cpumask_var(cmask
);
2229 static bool enable_ecc_error_reporting(struct ecc_settings
*s
, u8 nid
,
2233 u32 value
, mask
= 0x3; /* UECC/CECC enable */
2235 if (toggle_ecc_err_reporting(s
, nid
, ON
)) {
2236 amd64_warn("Error enabling ECC reporting over MCGCTL!\n");
2240 amd64_read_pci_cfg(F3
, NBCTL
, &value
);
2242 s
->old_nbctl
= value
& mask
;
2243 s
->nbctl_valid
= true;
2246 amd64_write_pci_cfg(F3
, NBCTL
, value
);
2248 amd64_read_pci_cfg(F3
, NBCFG
, &value
);
2250 debugf0("1: node %d, NBCFG=0x%08x[DramEccEn: %d]\n",
2251 nid
, value
, !!(value
& NBCFG_ECC_ENABLE
));
2253 if (!(value
& NBCFG_ECC_ENABLE
)) {
2254 amd64_warn("DRAM ECC disabled on this node, enabling...\n");
2256 s
->flags
.nb_ecc_prev
= 0;
2258 /* Attempt to turn on DRAM ECC Enable */
2259 value
|= NBCFG_ECC_ENABLE
;
2260 amd64_write_pci_cfg(F3
, NBCFG
, value
);
2262 amd64_read_pci_cfg(F3
, NBCFG
, &value
);
2264 if (!(value
& NBCFG_ECC_ENABLE
)) {
2265 amd64_warn("Hardware rejected DRAM ECC enable,"
2266 "check memory DIMM configuration.\n");
2269 amd64_info("Hardware accepted DRAM ECC Enable\n");
2272 s
->flags
.nb_ecc_prev
= 1;
2275 debugf0("2: node %d, NBCFG=0x%08x[DramEccEn: %d]\n",
2276 nid
, value
, !!(value
& NBCFG_ECC_ENABLE
));
2281 static void restore_ecc_error_reporting(struct ecc_settings
*s
, u8 nid
,
2284 u32 value
, mask
= 0x3; /* UECC/CECC enable */
2287 if (!s
->nbctl_valid
)
2290 amd64_read_pci_cfg(F3
, NBCTL
, &value
);
2292 value
|= s
->old_nbctl
;
2294 amd64_write_pci_cfg(F3
, NBCTL
, value
);
2296 /* restore previous BIOS DRAM ECC "off" setting we force-enabled */
2297 if (!s
->flags
.nb_ecc_prev
) {
2298 amd64_read_pci_cfg(F3
, NBCFG
, &value
);
2299 value
&= ~NBCFG_ECC_ENABLE
;
2300 amd64_write_pci_cfg(F3
, NBCFG
, value
);
2303 /* restore the NB Enable MCGCTL bit */
2304 if (toggle_ecc_err_reporting(s
, nid
, OFF
))
2305 amd64_warn("Error restoring NB MCGCTL settings!\n");
2309 * EDAC requires that the BIOS have ECC enabled before
2310 * taking over the processing of ECC errors. A command line
2311 * option allows to force-enable hardware ECC later in
2312 * enable_ecc_error_reporting().
2314 static const char *ecc_msg
=
2315 "ECC disabled in the BIOS or no ECC capability, module will not load.\n"
2316 " Either enable ECC checking or force module loading by setting "
2317 "'ecc_enable_override'.\n"
2318 " (Note that use of the override may cause unknown side effects.)\n";
2320 static bool ecc_enabled(struct pci_dev
*F3
, u8 nid
)
2324 bool nb_mce_en
= false;
2326 amd64_read_pci_cfg(F3
, NBCFG
, &value
);
2328 ecc_en
= !!(value
& NBCFG_ECC_ENABLE
);
2329 amd64_info("DRAM ECC %s.\n", (ecc_en
? "enabled" : "disabled"));
2331 nb_mce_en
= amd64_nb_mce_bank_enabled_on_node(nid
);
2333 amd64_notice("NB MCE bank disabled, set MSR "
2334 "0x%08x[4] on node %d to enable.\n",
2335 MSR_IA32_MCG_CTL
, nid
);
2337 if (!ecc_en
|| !nb_mce_en
) {
2338 amd64_notice("%s", ecc_msg
);
2344 struct mcidev_sysfs_attribute sysfs_attrs
[ARRAY_SIZE(amd64_dbg_attrs
) +
2345 ARRAY_SIZE(amd64_inj_attrs
) +
2348 struct mcidev_sysfs_attribute terminator
= { .attr
= { .name
= NULL
} };
2350 static void set_mc_sysfs_attrs(struct mem_ctl_info
*mci
)
2352 unsigned int i
= 0, j
= 0;
2354 for (; i
< ARRAY_SIZE(amd64_dbg_attrs
); i
++)
2355 sysfs_attrs
[i
] = amd64_dbg_attrs
[i
];
2357 if (boot_cpu_data
.x86
>= 0x10)
2358 for (j
= 0; j
< ARRAY_SIZE(amd64_inj_attrs
); j
++, i
++)
2359 sysfs_attrs
[i
] = amd64_inj_attrs
[j
];
2361 sysfs_attrs
[i
] = terminator
;
2363 mci
->mc_driver_sysfs_attributes
= sysfs_attrs
;
2366 static void setup_mci_misc_attrs(struct mem_ctl_info
*mci
,
2367 struct amd64_family_type
*fam
)
2369 struct amd64_pvt
*pvt
= mci
->pvt_info
;
2371 mci
->mtype_cap
= MEM_FLAG_DDR2
| MEM_FLAG_RDDR2
;
2372 mci
->edac_ctl_cap
= EDAC_FLAG_NONE
;
2374 if (pvt
->nbcap
& NBCAP_SECDED
)
2375 mci
->edac_ctl_cap
|= EDAC_FLAG_SECDED
;
2377 if (pvt
->nbcap
& NBCAP_CHIPKILL
)
2378 mci
->edac_ctl_cap
|= EDAC_FLAG_S4ECD4ED
;
2380 mci
->edac_cap
= amd64_determine_edac_cap(pvt
);
2381 mci
->mod_name
= EDAC_MOD_STR
;
2382 mci
->mod_ver
= EDAC_AMD64_VERSION
;
2383 mci
->ctl_name
= fam
->ctl_name
;
2384 mci
->dev_name
= pci_name(pvt
->F2
);
2385 mci
->ctl_page_to_phys
= NULL
;
2387 /* memory scrubber interface */
2388 mci
->set_sdram_scrub_rate
= amd64_set_scrub_rate
;
2389 mci
->get_sdram_scrub_rate
= amd64_get_scrub_rate
;
2393 * returns a pointer to the family descriptor on success, NULL otherwise.
2395 static struct amd64_family_type
*amd64_per_family_init(struct amd64_pvt
*pvt
)
2397 u8 fam
= boot_cpu_data
.x86
;
2398 struct amd64_family_type
*fam_type
= NULL
;
2402 fam_type
= &amd64_family_types
[K8_CPUS
];
2403 pvt
->ops
= &amd64_family_types
[K8_CPUS
].ops
;
2407 fam_type
= &amd64_family_types
[F10_CPUS
];
2408 pvt
->ops
= &amd64_family_types
[F10_CPUS
].ops
;
2412 fam_type
= &amd64_family_types
[F15_CPUS
];
2413 pvt
->ops
= &amd64_family_types
[F15_CPUS
].ops
;
2417 amd64_err("Unsupported family!\n");
2421 pvt
->ext_model
= boot_cpu_data
.x86_model
>> 4;
2423 amd64_info("%s %sdetected (node %d).\n", fam_type
->ctl_name
,
2425 (pvt
->ext_model
>= K8_REV_F
? "revF or later "
2426 : "revE or earlier ")
2427 : ""), pvt
->mc_node_id
);
2431 static int amd64_init_one_instance(struct pci_dev
*F2
)
2433 struct amd64_pvt
*pvt
= NULL
;
2434 struct amd64_family_type
*fam_type
= NULL
;
2435 struct mem_ctl_info
*mci
= NULL
;
2437 u8 nid
= get_node_id(F2
);
2440 pvt
= kzalloc(sizeof(struct amd64_pvt
), GFP_KERNEL
);
2444 pvt
->mc_node_id
= nid
;
2448 fam_type
= amd64_per_family_init(pvt
);
2453 err
= reserve_mc_sibling_devs(pvt
, fam_type
->f1_id
, fam_type
->f3_id
);
2460 * We need to determine how many memory channels there are. Then use
2461 * that information for calculating the size of the dynamic instance
2462 * tables in the 'mci' structure.
2465 pvt
->channel_count
= pvt
->ops
->early_channel_count(pvt
);
2466 if (pvt
->channel_count
< 0)
2470 mci
= edac_mc_alloc(0, pvt
->csels
[0].b_cnt
, pvt
->channel_count
, nid
);
2474 mci
->pvt_info
= pvt
;
2475 mci
->dev
= &pvt
->F2
->dev
;
2477 setup_mci_misc_attrs(mci
, fam_type
);
2479 if (init_csrows(mci
))
2480 mci
->edac_cap
= EDAC_FLAG_NONE
;
2482 set_mc_sysfs_attrs(mci
);
2485 if (edac_mc_add_mc(mci
)) {
2486 debugf1("failed edac_mc_add_mc()\n");
2490 /* register stuff with EDAC MCE */
2491 if (report_gart_errors
)
2492 amd_report_gart_errors(true);
2494 amd_register_ecc_decoder(amd64_decode_bus_error
);
2498 atomic_inc(&drv_instances
);
2506 free_mc_sibling_devs(pvt
);
2515 static int __devinit
amd64_probe_one_instance(struct pci_dev
*pdev
,
2516 const struct pci_device_id
*mc_type
)
2518 u8 nid
= get_node_id(pdev
);
2519 struct pci_dev
*F3
= node_to_amd_nb(nid
)->misc
;
2520 struct ecc_settings
*s
;
2523 ret
= pci_enable_device(pdev
);
2525 debugf0("ret=%d\n", ret
);
2530 s
= kzalloc(sizeof(struct ecc_settings
), GFP_KERNEL
);
2536 if (!ecc_enabled(F3
, nid
)) {
2539 if (!ecc_enable_override
)
2542 amd64_warn("Forcing ECC on!\n");
2544 if (!enable_ecc_error_reporting(s
, nid
, F3
))
2548 ret
= amd64_init_one_instance(pdev
);
2550 amd64_err("Error probing instance: %d\n", nid
);
2551 restore_ecc_error_reporting(s
, nid
, F3
);
2558 ecc_stngs
[nid
] = NULL
;
2564 static void __devexit
amd64_remove_one_instance(struct pci_dev
*pdev
)
2566 struct mem_ctl_info
*mci
;
2567 struct amd64_pvt
*pvt
;
2568 u8 nid
= get_node_id(pdev
);
2569 struct pci_dev
*F3
= node_to_amd_nb(nid
)->misc
;
2570 struct ecc_settings
*s
= ecc_stngs
[nid
];
2572 /* Remove from EDAC CORE tracking list */
2573 mci
= edac_mc_del_mc(&pdev
->dev
);
2577 pvt
= mci
->pvt_info
;
2579 restore_ecc_error_reporting(s
, nid
, F3
);
2581 free_mc_sibling_devs(pvt
);
2583 /* unregister from EDAC MCE */
2584 amd_report_gart_errors(false);
2585 amd_unregister_ecc_decoder(amd64_decode_bus_error
);
2587 kfree(ecc_stngs
[nid
]);
2588 ecc_stngs
[nid
] = NULL
;
2590 /* Free the EDAC CORE resources */
2591 mci
->pvt_info
= NULL
;
2599 * This table is part of the interface for loading drivers for PCI devices. The
2600 * PCI core identifies what devices are on a system during boot, and then
2601 * inquiry this table to see if this driver is for a given device found.
2603 static const struct pci_device_id amd64_pci_table
[] __devinitdata
= {
2605 .vendor
= PCI_VENDOR_ID_AMD
,
2606 .device
= PCI_DEVICE_ID_AMD_K8_NB_MEMCTL
,
2607 .subvendor
= PCI_ANY_ID
,
2608 .subdevice
= PCI_ANY_ID
,
2613 .vendor
= PCI_VENDOR_ID_AMD
,
2614 .device
= PCI_DEVICE_ID_AMD_10H_NB_DRAM
,
2615 .subvendor
= PCI_ANY_ID
,
2616 .subdevice
= PCI_ANY_ID
,
2621 .vendor
= PCI_VENDOR_ID_AMD
,
2622 .device
= PCI_DEVICE_ID_AMD_15H_NB_F2
,
2623 .subvendor
= PCI_ANY_ID
,
2624 .subdevice
= PCI_ANY_ID
,
2631 MODULE_DEVICE_TABLE(pci
, amd64_pci_table
);
2633 static struct pci_driver amd64_pci_driver
= {
2634 .name
= EDAC_MOD_STR
,
2635 .probe
= amd64_probe_one_instance
,
2636 .remove
= __devexit_p(amd64_remove_one_instance
),
2637 .id_table
= amd64_pci_table
,
2640 static void setup_pci_device(void)
2642 struct mem_ctl_info
*mci
;
2643 struct amd64_pvt
*pvt
;
2651 pvt
= mci
->pvt_info
;
2653 edac_pci_create_generic_ctl(&pvt
->F2
->dev
, EDAC_MOD_STR
);
2655 if (!amd64_ctl_pci
) {
2656 pr_warning("%s(): Unable to create PCI control\n",
2659 pr_warning("%s(): PCI error report via EDAC not set\n",
2665 static int __init
amd64_edac_init(void)
2669 printk(KERN_INFO
"AMD64 EDAC driver v%s\n", EDAC_AMD64_VERSION
);
2673 if (amd_cache_northbridges() < 0)
2677 mcis
= kzalloc(amd_nb_num() * sizeof(mcis
[0]), GFP_KERNEL
);
2678 ecc_stngs
= kzalloc(amd_nb_num() * sizeof(ecc_stngs
[0]), GFP_KERNEL
);
2679 if (!(mcis
&& ecc_stngs
))
2682 msrs
= msrs_alloc();
2686 err
= pci_register_driver(&amd64_pci_driver
);
2691 if (!atomic_read(&drv_instances
))
2692 goto err_no_instances
;
2698 pci_unregister_driver(&amd64_pci_driver
);
2715 static void __exit
amd64_edac_exit(void)
2718 edac_pci_release_generic_ctl(amd64_ctl_pci
);
2720 pci_unregister_driver(&amd64_pci_driver
);
2732 module_init(amd64_edac_init
);
2733 module_exit(amd64_edac_exit
);
2735 MODULE_LICENSE("GPL");
2736 MODULE_AUTHOR("SoftwareBitMaker: Doug Thompson, "
2737 "Dave Peterson, Thayne Harbaugh");
2738 MODULE_DESCRIPTION("MC support for AMD64 memory controllers - "
2739 EDAC_AMD64_VERSION
);
2741 module_param(edac_op_state
, int, 0444);
2742 MODULE_PARM_DESC(edac_op_state
, "EDAC Error Reporting state: 0=Poll,1=NMI");