1 #include "amd64_edac.h"
2 #include <asm/amd_nb.h>
4 static struct edac_pci_ctl_info
*amd64_ctl_pci
;
6 static int report_gart_errors
;
7 module_param(report_gart_errors
, int, 0644);
10 * Set by command line parameter. If BIOS has enabled the ECC, this override is
11 * cleared to prevent re-enabling the hardware by this driver.
13 static int ecc_enable_override
;
14 module_param(ecc_enable_override
, int, 0644);
16 static struct msr __percpu
*msrs
;
19 * count successfully initialized driver instances for setup_pci_device()
21 static atomic_t drv_instances
= ATOMIC_INIT(0);
23 /* Per-node driver instances */
24 static struct mem_ctl_info
**mcis
;
25 static struct ecc_settings
**ecc_stngs
;
28 * Valid scrub rates for the K8 hardware memory scrubber. We map the scrubbing
29 * bandwidth to a valid bit pattern. The 'set' operation finds the 'matching-
32 *FIXME: Produce a better mapping/linearisation.
35 u32 scrubval
; /* bit pattern for scrub rate */
36 u32 bandwidth
; /* bandwidth consumed (bytes/sec) */
38 { 0x01, 1600000000UL},
60 { 0x00, 0UL}, /* scrubbing off */
63 static int __amd64_read_pci_cfg_dword(struct pci_dev
*pdev
, int offset
,
64 u32
*val
, const char *func
)
68 err
= pci_read_config_dword(pdev
, offset
, val
);
70 amd64_warn("%s: error reading F%dx%03x.\n",
71 func
, PCI_FUNC(pdev
->devfn
), offset
);
76 int __amd64_write_pci_cfg_dword(struct pci_dev
*pdev
, int offset
,
77 u32 val
, const char *func
)
81 err
= pci_write_config_dword(pdev
, offset
, val
);
83 amd64_warn("%s: error writing to F%dx%03x.\n",
84 func
, PCI_FUNC(pdev
->devfn
), offset
);
91 * Depending on the family, F2 DCT reads need special handling:
93 * K8: has a single DCT only
95 * F10h: each DCT has its own set of regs
99 * F15h: we select which DCT we access using F1x10C[DctCfgSel]
102 static int k8_read_dct_pci_cfg(struct amd64_pvt
*pvt
, int addr
, u32
*val
,
108 return __amd64_read_pci_cfg_dword(pvt
->F2
, addr
, val
, func
);
111 static int f10_read_dct_pci_cfg(struct amd64_pvt
*pvt
, int addr
, u32
*val
,
114 return __amd64_read_pci_cfg_dword(pvt
->F2
, addr
, val
, func
);
117 static int f15_read_dct_pci_cfg(struct amd64_pvt
*pvt
, int addr
, u32
*val
,
123 if (addr
>= 0x140 && addr
<= 0x1a0) {
128 amd64_read_pci_cfg(pvt
->F1
, DCT_CFG_SEL
, ®
);
131 amd64_write_pci_cfg(pvt
->F1
, DCT_CFG_SEL
, reg
);
133 return __amd64_read_pci_cfg_dword(pvt
->F2
, addr
, val
, func
);
137 * Memory scrubber control interface. For K8, memory scrubbing is handled by
138 * hardware and can involve L2 cache, dcache as well as the main memory. With
139 * F10, this is extended to L3 cache scrubbing on CPU models sporting that
142 * This causes the "units" for the scrubbing speed to vary from 64 byte blocks
143 * (dram) over to cache lines. This is nasty, so we will use bandwidth in
144 * bytes/sec for the setting.
146 * Currently, we only do dram scrubbing. If the scrubbing is done in software on
147 * other archs, we might not have access to the caches directly.
151 * scan the scrub rate mapping table for a close or matching bandwidth value to
152 * issue. If requested is too big, then use last maximum value found.
154 static int __amd64_set_scrub_rate(struct pci_dev
*ctl
, u32 new_bw
, u32 min_rate
)
160 * map the configured rate (new_bw) to a value specific to the AMD64
161 * memory controller and apply to register. Search for the first
162 * bandwidth entry that is greater or equal than the setting requested
163 * and program that. If at last entry, turn off DRAM scrubbing.
165 for (i
= 0; i
< ARRAY_SIZE(scrubrates
); i
++) {
167 * skip scrub rates which aren't recommended
168 * (see F10 BKDG, F3x58)
170 if (scrubrates
[i
].scrubval
< min_rate
)
173 if (scrubrates
[i
].bandwidth
<= new_bw
)
177 * if no suitable bandwidth found, turn off DRAM scrubbing
178 * entirely by falling back to the last element in the
183 scrubval
= scrubrates
[i
].scrubval
;
185 pci_write_bits32(ctl
, SCRCTRL
, scrubval
, 0x001F);
188 return scrubrates
[i
].bandwidth
;
193 static int amd64_set_scrub_rate(struct mem_ctl_info
*mci
, u32 bw
)
195 struct amd64_pvt
*pvt
= mci
->pvt_info
;
196 u32 min_scrubrate
= 0x5;
198 if (boot_cpu_data
.x86
== 0xf)
201 return __amd64_set_scrub_rate(pvt
->F3
, bw
, min_scrubrate
);
204 static int amd64_get_scrub_rate(struct mem_ctl_info
*mci
)
206 struct amd64_pvt
*pvt
= mci
->pvt_info
;
208 int i
, retval
= -EINVAL
;
210 amd64_read_pci_cfg(pvt
->F3
, SCRCTRL
, &scrubval
);
212 scrubval
= scrubval
& 0x001F;
214 for (i
= 0; i
< ARRAY_SIZE(scrubrates
); i
++) {
215 if (scrubrates
[i
].scrubval
== scrubval
) {
216 retval
= scrubrates
[i
].bandwidth
;
224 * returns true if the SysAddr given by sys_addr matches the
225 * DRAM base/limit associated with node_id
227 static bool amd64_base_limit_match(struct amd64_pvt
*pvt
, u64 sys_addr
,
232 /* The K8 treats this as a 40-bit value. However, bits 63-40 will be
233 * all ones if the most significant implemented address bit is 1.
234 * Here we discard bits 63-40. See section 3.4.2 of AMD publication
235 * 24592: AMD x86-64 Architecture Programmer's Manual Volume 1
236 * Application Programming.
238 addr
= sys_addr
& 0x000000ffffffffffull
;
240 return ((addr
>= get_dram_base(pvt
, nid
)) &&
241 (addr
<= get_dram_limit(pvt
, nid
)));
245 * Attempt to map a SysAddr to a node. On success, return a pointer to the
246 * mem_ctl_info structure for the node that the SysAddr maps to.
248 * On failure, return NULL.
250 static struct mem_ctl_info
*find_mc_by_sys_addr(struct mem_ctl_info
*mci
,
253 struct amd64_pvt
*pvt
;
258 * Here we use the DRAM Base (section 3.4.4.1) and DRAM Limit (section
259 * 3.4.4.2) registers to map the SysAddr to a node ID.
264 * The value of this field should be the same for all DRAM Base
265 * registers. Therefore we arbitrarily choose to read it from the
266 * register for node 0.
268 intlv_en
= dram_intlv_en(pvt
, 0);
271 for (node_id
= 0; node_id
< DRAM_RANGES
; node_id
++) {
272 if (amd64_base_limit_match(pvt
, sys_addr
, node_id
))
278 if (unlikely((intlv_en
!= 0x01) &&
279 (intlv_en
!= 0x03) &&
280 (intlv_en
!= 0x07))) {
281 amd64_warn("DRAM Base[IntlvEn] junk value: 0x%x, BIOS bug?\n", intlv_en
);
285 bits
= (((u32
) sys_addr
) >> 12) & intlv_en
;
287 for (node_id
= 0; ; ) {
288 if ((dram_intlv_sel(pvt
, node_id
) & intlv_en
) == bits
)
289 break; /* intlv_sel field matches */
291 if (++node_id
>= DRAM_RANGES
)
295 /* sanity test for sys_addr */
296 if (unlikely(!amd64_base_limit_match(pvt
, sys_addr
, node_id
))) {
297 amd64_warn("%s: sys_addr 0x%llx falls outside base/limit address"
298 "range for node %d with node interleaving enabled.\n",
299 __func__
, sys_addr
, node_id
);
304 return edac_mc_find((int)node_id
);
307 debugf2("sys_addr 0x%lx doesn't match any node\n",
308 (unsigned long)sys_addr
);
314 * compute the CS base address of the @csrow on the DRAM controller @dct.
315 * For details see F2x[5C:40] in the processor's BKDG
317 static void get_cs_base_and_mask(struct amd64_pvt
*pvt
, int csrow
, u8 dct
,
318 u64
*base
, u64
*mask
)
320 u64 csbase
, csmask
, base_bits
, mask_bits
;
323 if (boot_cpu_data
.x86
== 0xf && pvt
->ext_model
< K8_REV_F
) {
324 csbase
= pvt
->csels
[dct
].csbases
[csrow
];
325 csmask
= pvt
->csels
[dct
].csmasks
[csrow
];
326 base_bits
= GENMASK(21, 31) | GENMASK(9, 15);
327 mask_bits
= GENMASK(21, 29) | GENMASK(9, 15);
330 csbase
= pvt
->csels
[dct
].csbases
[csrow
];
331 csmask
= pvt
->csels
[dct
].csmasks
[csrow
>> 1];
334 if (boot_cpu_data
.x86
== 0x15)
335 base_bits
= mask_bits
= GENMASK(19,30) | GENMASK(5,13);
337 base_bits
= mask_bits
= GENMASK(19,28) | GENMASK(5,13);
340 *base
= (csbase
& base_bits
) << addr_shift
;
343 /* poke holes for the csmask */
344 *mask
&= ~(mask_bits
<< addr_shift
);
346 *mask
|= (csmask
& mask_bits
) << addr_shift
;
349 #define for_each_chip_select(i, dct, pvt) \
350 for (i = 0; i < pvt->csels[dct].b_cnt; i++)
352 #define chip_select_base(i, dct, pvt) \
353 pvt->csels[dct].csbases[i]
355 #define for_each_chip_select_mask(i, dct, pvt) \
356 for (i = 0; i < pvt->csels[dct].m_cnt; i++)
359 * @input_addr is an InputAddr associated with the node given by mci. Return the
360 * csrow that input_addr maps to, or -1 on failure (no csrow claims input_addr).
362 static int input_addr_to_csrow(struct mem_ctl_info
*mci
, u64 input_addr
)
364 struct amd64_pvt
*pvt
;
370 for_each_chip_select(csrow
, 0, pvt
) {
371 if (!csrow_enabled(csrow
, 0, pvt
))
374 get_cs_base_and_mask(pvt
, csrow
, 0, &base
, &mask
);
378 if ((input_addr
& mask
) == (base
& mask
)) {
379 debugf2("InputAddr 0x%lx matches csrow %d (node %d)\n",
380 (unsigned long)input_addr
, csrow
,
386 debugf2("no matching csrow for InputAddr 0x%lx (MC node %d)\n",
387 (unsigned long)input_addr
, pvt
->mc_node_id
);
393 * Obtain info from the DRAM Hole Address Register (section 3.4.8, pub #26094)
394 * for the node represented by mci. Info is passed back in *hole_base,
395 * *hole_offset, and *hole_size. Function returns 0 if info is valid or 1 if
396 * info is invalid. Info may be invalid for either of the following reasons:
398 * - The revision of the node is not E or greater. In this case, the DRAM Hole
399 * Address Register does not exist.
401 * - The DramHoleValid bit is cleared in the DRAM Hole Address Register,
402 * indicating that its contents are not valid.
404 * The values passed back in *hole_base, *hole_offset, and *hole_size are
405 * complete 32-bit values despite the fact that the bitfields in the DHAR
406 * only represent bits 31-24 of the base and offset values.
408 int amd64_get_dram_hole_info(struct mem_ctl_info
*mci
, u64
*hole_base
,
409 u64
*hole_offset
, u64
*hole_size
)
411 struct amd64_pvt
*pvt
= mci
->pvt_info
;
414 /* only revE and later have the DRAM Hole Address Register */
415 if (boot_cpu_data
.x86
== 0xf && pvt
->ext_model
< K8_REV_E
) {
416 debugf1(" revision %d for node %d does not support DHAR\n",
417 pvt
->ext_model
, pvt
->mc_node_id
);
421 /* valid for Fam10h and above */
422 if (boot_cpu_data
.x86
>= 0x10 && !dhar_mem_hoist_valid(pvt
)) {
423 debugf1(" Dram Memory Hoisting is DISABLED on this system\n");
427 if (!dhar_valid(pvt
)) {
428 debugf1(" Dram Memory Hoisting is DISABLED on this node %d\n",
433 /* This node has Memory Hoisting */
435 /* +------------------+--------------------+--------------------+-----
436 * | memory | DRAM hole | relocated |
437 * | [0, (x - 1)] | [x, 0xffffffff] | addresses from |
439 * | | | [0x100000000, |
440 * | | | (0x100000000+ |
441 * | | | (0xffffffff-x))] |
442 * +------------------+--------------------+--------------------+-----
444 * Above is a diagram of physical memory showing the DRAM hole and the
445 * relocated addresses from the DRAM hole. As shown, the DRAM hole
446 * starts at address x (the base address) and extends through address
447 * 0xffffffff. The DRAM Hole Address Register (DHAR) relocates the
448 * addresses in the hole so that they start at 0x100000000.
451 base
= dhar_base(pvt
);
454 *hole_size
= (0x1ull
<< 32) - base
;
456 if (boot_cpu_data
.x86
> 0xf)
457 *hole_offset
= f10_dhar_offset(pvt
);
459 *hole_offset
= k8_dhar_offset(pvt
);
461 debugf1(" DHAR info for node %d base 0x%lx offset 0x%lx size 0x%lx\n",
462 pvt
->mc_node_id
, (unsigned long)*hole_base
,
463 (unsigned long)*hole_offset
, (unsigned long)*hole_size
);
467 EXPORT_SYMBOL_GPL(amd64_get_dram_hole_info
);
470 * Return the DramAddr that the SysAddr given by @sys_addr maps to. It is
471 * assumed that sys_addr maps to the node given by mci.
473 * The first part of section 3.4.4 (p. 70) shows how the DRAM Base (section
474 * 3.4.4.1) and DRAM Limit (section 3.4.4.2) registers are used to translate a
475 * SysAddr to a DramAddr. If the DRAM Hole Address Register (DHAR) is enabled,
476 * then it is also involved in translating a SysAddr to a DramAddr. Sections
477 * 3.4.8 and 3.5.8.2 describe the DHAR and how it is used for memory hoisting.
478 * These parts of the documentation are unclear. I interpret them as follows:
480 * When node n receives a SysAddr, it processes the SysAddr as follows:
482 * 1. It extracts the DRAMBase and DRAMLimit values from the DRAM Base and DRAM
483 * Limit registers for node n. If the SysAddr is not within the range
484 * specified by the base and limit values, then node n ignores the Sysaddr
485 * (since it does not map to node n). Otherwise continue to step 2 below.
487 * 2. If the DramHoleValid bit of the DHAR for node n is clear, the DHAR is
488 * disabled so skip to step 3 below. Otherwise see if the SysAddr is within
489 * the range of relocated addresses (starting at 0x100000000) from the DRAM
490 * hole. If not, skip to step 3 below. Else get the value of the
491 * DramHoleOffset field from the DHAR. To obtain the DramAddr, subtract the
492 * offset defined by this value from the SysAddr.
494 * 3. Obtain the base address for node n from the DRAMBase field of the DRAM
495 * Base register for node n. To obtain the DramAddr, subtract the base
496 * address from the SysAddr, as shown near the start of section 3.4.4 (p.70).
498 static u64
sys_addr_to_dram_addr(struct mem_ctl_info
*mci
, u64 sys_addr
)
500 struct amd64_pvt
*pvt
= mci
->pvt_info
;
501 u64 dram_base
, hole_base
, hole_offset
, hole_size
, dram_addr
;
504 dram_base
= get_dram_base(pvt
, pvt
->mc_node_id
);
506 ret
= amd64_get_dram_hole_info(mci
, &hole_base
, &hole_offset
,
509 if ((sys_addr
>= (1ull << 32)) &&
510 (sys_addr
< ((1ull << 32) + hole_size
))) {
511 /* use DHAR to translate SysAddr to DramAddr */
512 dram_addr
= sys_addr
- hole_offset
;
514 debugf2("using DHAR to translate SysAddr 0x%lx to "
516 (unsigned long)sys_addr
,
517 (unsigned long)dram_addr
);
524 * Translate the SysAddr to a DramAddr as shown near the start of
525 * section 3.4.4 (p. 70). Although sys_addr is a 64-bit value, the k8
526 * only deals with 40-bit values. Therefore we discard bits 63-40 of
527 * sys_addr below. If bit 39 of sys_addr is 1 then the bits we
528 * discard are all 1s. Otherwise the bits we discard are all 0s. See
529 * section 3.4.2 of AMD publication 24592: AMD x86-64 Architecture
530 * Programmer's Manual Volume 1 Application Programming.
532 dram_addr
= (sys_addr
& GENMASK(0, 39)) - dram_base
;
534 debugf2("using DRAM Base register to translate SysAddr 0x%lx to "
535 "DramAddr 0x%lx\n", (unsigned long)sys_addr
,
536 (unsigned long)dram_addr
);
541 * @intlv_en is the value of the IntlvEn field from a DRAM Base register
542 * (section 3.4.4.1). Return the number of bits from a SysAddr that are used
543 * for node interleaving.
545 static int num_node_interleave_bits(unsigned intlv_en
)
547 static const int intlv_shift_table
[] = { 0, 1, 0, 2, 0, 0, 0, 3 };
550 BUG_ON(intlv_en
> 7);
551 n
= intlv_shift_table
[intlv_en
];
555 /* Translate the DramAddr given by @dram_addr to an InputAddr. */
556 static u64
dram_addr_to_input_addr(struct mem_ctl_info
*mci
, u64 dram_addr
)
558 struct amd64_pvt
*pvt
;
565 * See the start of section 3.4.4 (p. 70, BKDG #26094, K8, revA-E)
566 * concerning translating a DramAddr to an InputAddr.
568 intlv_shift
= num_node_interleave_bits(dram_intlv_en(pvt
, 0));
569 input_addr
= ((dram_addr
>> intlv_shift
) & GENMASK(12, 35)) +
572 debugf2(" Intlv Shift=%d DramAddr=0x%lx maps to InputAddr=0x%lx\n",
573 intlv_shift
, (unsigned long)dram_addr
,
574 (unsigned long)input_addr
);
580 * Translate the SysAddr represented by @sys_addr to an InputAddr. It is
581 * assumed that @sys_addr maps to the node given by mci.
583 static u64
sys_addr_to_input_addr(struct mem_ctl_info
*mci
, u64 sys_addr
)
588 dram_addr_to_input_addr(mci
, sys_addr_to_dram_addr(mci
, sys_addr
));
590 debugf2("SysAdddr 0x%lx translates to InputAddr 0x%lx\n",
591 (unsigned long)sys_addr
, (unsigned long)input_addr
);
598 * @input_addr is an InputAddr associated with the node represented by mci.
599 * Translate @input_addr to a DramAddr and return the result.
601 static u64
input_addr_to_dram_addr(struct mem_ctl_info
*mci
, u64 input_addr
)
603 struct amd64_pvt
*pvt
;
604 unsigned node_id
, intlv_shift
;
609 * Near the start of section 3.4.4 (p. 70, BKDG #26094, K8, revA-E)
610 * shows how to translate a DramAddr to an InputAddr. Here we reverse
611 * this procedure. When translating from a DramAddr to an InputAddr, the
612 * bits used for node interleaving are discarded. Here we recover these
613 * bits from the IntlvSel field of the DRAM Limit register (section
614 * 3.4.4.2) for the node that input_addr is associated with.
617 node_id
= pvt
->mc_node_id
;
621 intlv_shift
= num_node_interleave_bits(dram_intlv_en(pvt
, 0));
622 if (intlv_shift
== 0) {
623 debugf1(" InputAddr 0x%lx translates to DramAddr of "
624 "same value\n", (unsigned long)input_addr
);
629 bits
= ((input_addr
& GENMASK(12, 35)) << intlv_shift
) +
630 (input_addr
& 0xfff);
632 intlv_sel
= dram_intlv_sel(pvt
, node_id
) & ((1 << intlv_shift
) - 1);
633 dram_addr
= bits
+ (intlv_sel
<< 12);
635 debugf1("InputAddr 0x%lx translates to DramAddr 0x%lx "
636 "(%d node interleave bits)\n", (unsigned long)input_addr
,
637 (unsigned long)dram_addr
, intlv_shift
);
643 * @dram_addr is a DramAddr that maps to the node represented by mci. Convert
644 * @dram_addr to a SysAddr.
646 static u64
dram_addr_to_sys_addr(struct mem_ctl_info
*mci
, u64 dram_addr
)
648 struct amd64_pvt
*pvt
= mci
->pvt_info
;
649 u64 hole_base
, hole_offset
, hole_size
, base
, sys_addr
;
652 ret
= amd64_get_dram_hole_info(mci
, &hole_base
, &hole_offset
,
655 if ((dram_addr
>= hole_base
) &&
656 (dram_addr
< (hole_base
+ hole_size
))) {
657 sys_addr
= dram_addr
+ hole_offset
;
659 debugf1("using DHAR to translate DramAddr 0x%lx to "
660 "SysAddr 0x%lx\n", (unsigned long)dram_addr
,
661 (unsigned long)sys_addr
);
667 base
= get_dram_base(pvt
, pvt
->mc_node_id
);
668 sys_addr
= dram_addr
+ base
;
671 * The sys_addr we have computed up to this point is a 40-bit value
672 * because the k8 deals with 40-bit values. However, the value we are
673 * supposed to return is a full 64-bit physical address. The AMD
674 * x86-64 architecture specifies that the most significant implemented
675 * address bit through bit 63 of a physical address must be either all
676 * 0s or all 1s. Therefore we sign-extend the 40-bit sys_addr to a
677 * 64-bit value below. See section 3.4.2 of AMD publication 24592:
678 * AMD x86-64 Architecture Programmer's Manual Volume 1 Application
681 sys_addr
|= ~((sys_addr
& (1ull << 39)) - 1);
683 debugf1(" Node %d, DramAddr 0x%lx to SysAddr 0x%lx\n",
684 pvt
->mc_node_id
, (unsigned long)dram_addr
,
685 (unsigned long)sys_addr
);
691 * @input_addr is an InputAddr associated with the node given by mci. Translate
692 * @input_addr to a SysAddr.
694 static inline u64
input_addr_to_sys_addr(struct mem_ctl_info
*mci
,
697 return dram_addr_to_sys_addr(mci
,
698 input_addr_to_dram_addr(mci
, input_addr
));
702 * Find the minimum and maximum InputAddr values that map to the given @csrow.
703 * Pass back these values in *input_addr_min and *input_addr_max.
705 static void find_csrow_limits(struct mem_ctl_info
*mci
, int csrow
,
706 u64
*input_addr_min
, u64
*input_addr_max
)
708 struct amd64_pvt
*pvt
;
712 BUG_ON((csrow
< 0) || (csrow
>= pvt
->csels
[0].b_cnt
));
714 get_cs_base_and_mask(pvt
, csrow
, 0, &base
, &mask
);
716 *input_addr_min
= base
& ~mask
;
717 *input_addr_max
= base
| mask
;
720 /* Map the Error address to a PAGE and PAGE OFFSET. */
721 static inline void error_address_to_page_and_offset(u64 error_address
,
722 u32
*page
, u32
*offset
)
724 *page
= (u32
) (error_address
>> PAGE_SHIFT
);
725 *offset
= ((u32
) error_address
) & ~PAGE_MASK
;
729 * @sys_addr is an error address (a SysAddr) extracted from the MCA NB Address
730 * Low (section 3.6.4.5) and MCA NB Address High (section 3.6.4.6) registers
731 * of a node that detected an ECC memory error. mci represents the node that
732 * the error address maps to (possibly different from the node that detected
733 * the error). Return the number of the csrow that sys_addr maps to, or -1 on
736 static int sys_addr_to_csrow(struct mem_ctl_info
*mci
, u64 sys_addr
)
740 csrow
= input_addr_to_csrow(mci
, sys_addr_to_input_addr(mci
, sys_addr
));
743 amd64_mc_err(mci
, "Failed to translate InputAddr to csrow for "
744 "address 0x%lx\n", (unsigned long)sys_addr
);
748 static int get_channel_from_ecc_syndrome(struct mem_ctl_info
*, u16
);
751 * Determine if the DIMMs have ECC enabled. ECC is enabled ONLY if all the DIMMs
754 static enum edac_type
amd64_determine_edac_cap(struct amd64_pvt
*pvt
)
757 enum dev_type edac_cap
= EDAC_FLAG_NONE
;
759 bit
= (boot_cpu_data
.x86
> 0xf || pvt
->ext_model
>= K8_REV_F
)
763 if (pvt
->dclr0
& BIT(bit
))
764 edac_cap
= EDAC_FLAG_SECDED
;
769 static void amd64_debug_display_dimm_sizes(struct amd64_pvt
*, u8
);
771 static void amd64_dump_dramcfg_low(u32 dclr
, int chan
)
773 debugf1("F2x%d90 (DRAM Cfg Low): 0x%08x\n", chan
, dclr
);
775 debugf1(" DIMM type: %sbuffered; all DIMMs support ECC: %s\n",
776 (dclr
& BIT(16)) ? "un" : "",
777 (dclr
& BIT(19)) ? "yes" : "no");
779 debugf1(" PAR/ERR parity: %s\n",
780 (dclr
& BIT(8)) ? "enabled" : "disabled");
782 if (boot_cpu_data
.x86
== 0x10)
783 debugf1(" DCT 128bit mode width: %s\n",
784 (dclr
& BIT(11)) ? "128b" : "64b");
786 debugf1(" x4 logical DIMMs present: L0: %s L1: %s L2: %s L3: %s\n",
787 (dclr
& BIT(12)) ? "yes" : "no",
788 (dclr
& BIT(13)) ? "yes" : "no",
789 (dclr
& BIT(14)) ? "yes" : "no",
790 (dclr
& BIT(15)) ? "yes" : "no");
793 /* Display and decode various NB registers for debug purposes. */
794 static void dump_misc_regs(struct amd64_pvt
*pvt
)
796 debugf1("F3xE8 (NB Cap): 0x%08x\n", pvt
->nbcap
);
798 debugf1(" NB two channel DRAM capable: %s\n",
799 (pvt
->nbcap
& NBCAP_DCT_DUAL
) ? "yes" : "no");
801 debugf1(" ECC capable: %s, ChipKill ECC capable: %s\n",
802 (pvt
->nbcap
& NBCAP_SECDED
) ? "yes" : "no",
803 (pvt
->nbcap
& NBCAP_CHIPKILL
) ? "yes" : "no");
805 amd64_dump_dramcfg_low(pvt
->dclr0
, 0);
807 debugf1("F3xB0 (Online Spare): 0x%08x\n", pvt
->online_spare
);
809 debugf1("F1xF0 (DRAM Hole Address): 0x%08x, base: 0x%08x, "
811 pvt
->dhar
, dhar_base(pvt
),
812 (boot_cpu_data
.x86
== 0xf) ? k8_dhar_offset(pvt
)
813 : f10_dhar_offset(pvt
));
815 debugf1(" DramHoleValid: %s\n", dhar_valid(pvt
) ? "yes" : "no");
817 amd64_debug_display_dimm_sizes(pvt
, 0);
819 /* everything below this point is Fam10h and above */
820 if (boot_cpu_data
.x86
== 0xf)
823 amd64_debug_display_dimm_sizes(pvt
, 1);
825 amd64_info("using %s syndromes.\n", ((pvt
->ecc_sym_sz
== 8) ? "x8" : "x4"));
827 /* Only if NOT ganged does dclr1 have valid info */
828 if (!dct_ganging_enabled(pvt
))
829 amd64_dump_dramcfg_low(pvt
->dclr1
, 1);
833 * see BKDG, F2x[1,0][5C:40], F2[1,0][6C:60]
835 static void prep_chip_selects(struct amd64_pvt
*pvt
)
837 if (boot_cpu_data
.x86
== 0xf && pvt
->ext_model
< K8_REV_F
) {
838 pvt
->csels
[0].b_cnt
= pvt
->csels
[1].b_cnt
= 8;
839 pvt
->csels
[0].m_cnt
= pvt
->csels
[1].m_cnt
= 8;
841 pvt
->csels
[0].b_cnt
= pvt
->csels
[1].b_cnt
= 8;
842 pvt
->csels
[0].m_cnt
= pvt
->csels
[1].m_cnt
= 4;
847 * Function 2 Offset F10_DCSB0; read in the DCS Base and DCS Mask registers
849 static void read_dct_base_mask(struct amd64_pvt
*pvt
)
853 prep_chip_selects(pvt
);
855 for_each_chip_select(cs
, 0, pvt
) {
856 int reg0
= DCSB0
+ (cs
* 4);
857 int reg1
= DCSB1
+ (cs
* 4);
858 u32
*base0
= &pvt
->csels
[0].csbases
[cs
];
859 u32
*base1
= &pvt
->csels
[1].csbases
[cs
];
861 if (!amd64_read_dct_pci_cfg(pvt
, reg0
, base0
))
862 debugf0(" DCSB0[%d]=0x%08x reg: F2x%x\n",
865 if (boot_cpu_data
.x86
== 0xf || dct_ganging_enabled(pvt
))
868 if (!amd64_read_dct_pci_cfg(pvt
, reg1
, base1
))
869 debugf0(" DCSB1[%d]=0x%08x reg: F2x%x\n",
873 for_each_chip_select_mask(cs
, 0, pvt
) {
874 int reg0
= DCSM0
+ (cs
* 4);
875 int reg1
= DCSM1
+ (cs
* 4);
876 u32
*mask0
= &pvt
->csels
[0].csmasks
[cs
];
877 u32
*mask1
= &pvt
->csels
[1].csmasks
[cs
];
879 if (!amd64_read_dct_pci_cfg(pvt
, reg0
, mask0
))
880 debugf0(" DCSM0[%d]=0x%08x reg: F2x%x\n",
883 if (boot_cpu_data
.x86
== 0xf || dct_ganging_enabled(pvt
))
886 if (!amd64_read_dct_pci_cfg(pvt
, reg1
, mask1
))
887 debugf0(" DCSM1[%d]=0x%08x reg: F2x%x\n",
892 static enum mem_type
amd64_determine_memory_type(struct amd64_pvt
*pvt
, int cs
)
896 /* F15h supports only DDR3 */
897 if (boot_cpu_data
.x86
>= 0x15)
898 type
= (pvt
->dclr0
& BIT(16)) ? MEM_DDR3
: MEM_RDDR3
;
899 else if (boot_cpu_data
.x86
== 0x10 || pvt
->ext_model
>= K8_REV_F
) {
900 if (pvt
->dchr0
& DDR3_MODE
)
901 type
= (pvt
->dclr0
& BIT(16)) ? MEM_DDR3
: MEM_RDDR3
;
903 type
= (pvt
->dclr0
& BIT(16)) ? MEM_DDR2
: MEM_RDDR2
;
905 type
= (pvt
->dclr0
& BIT(18)) ? MEM_DDR
: MEM_RDDR
;
908 amd64_info("CS%d: %s\n", cs
, edac_mem_types
[type
]);
913 /* Get the number of DCT channels the memory controller is using. */
914 static int k8_early_channel_count(struct amd64_pvt
*pvt
)
918 if (pvt
->ext_model
>= K8_REV_F
)
919 /* RevF (NPT) and later */
920 flag
= pvt
->dclr0
& WIDTH_128
;
922 /* RevE and earlier */
923 flag
= pvt
->dclr0
& REVE_WIDTH_128
;
928 return (flag
) ? 2 : 1;
931 /* On F10h and later ErrAddr is MC4_ADDR[47:1] */
932 static u64
get_error_address(struct mce
*m
)
937 if (boot_cpu_data
.x86
== 0xf) {
942 return m
->addr
& GENMASK(start_bit
, end_bit
);
945 static void read_dram_base_limit_regs(struct amd64_pvt
*pvt
, unsigned range
)
947 struct cpuinfo_x86
*c
= &boot_cpu_data
;
948 int off
= range
<< 3;
950 amd64_read_pci_cfg(pvt
->F1
, DRAM_BASE_LO
+ off
, &pvt
->ranges
[range
].base
.lo
);
951 amd64_read_pci_cfg(pvt
->F1
, DRAM_LIMIT_LO
+ off
, &pvt
->ranges
[range
].lim
.lo
);
956 if (!dram_rw(pvt
, range
))
959 amd64_read_pci_cfg(pvt
->F1
, DRAM_BASE_HI
+ off
, &pvt
->ranges
[range
].base
.hi
);
960 amd64_read_pci_cfg(pvt
->F1
, DRAM_LIMIT_HI
+ off
, &pvt
->ranges
[range
].lim
.hi
);
962 /* Factor in CC6 save area by reading dst node's limit reg */
963 if (c
->x86
== 0x15) {
964 struct pci_dev
*f1
= NULL
;
965 u8 nid
= dram_dst_node(pvt
, range
);
968 f1
= pci_get_domain_bus_and_slot(0, 0, PCI_DEVFN(0x18 + nid
, 1));
972 amd64_read_pci_cfg(f1
, DRAM_LOCAL_NODE_LIM
, &llim
);
974 pvt
->ranges
[range
].lim
.lo
&= GENMASK(0, 15);
977 pvt
->ranges
[range
].lim
.lo
|= ((llim
& 0x1fff) << 3 | 0x7) << 16;
979 pvt
->ranges
[range
].lim
.hi
&= GENMASK(0, 7);
982 pvt
->ranges
[range
].lim
.hi
|= llim
>> 13;
988 static void k8_map_sysaddr_to_csrow(struct mem_ctl_info
*mci
, u64 sys_addr
,
991 struct mem_ctl_info
*src_mci
;
992 struct amd64_pvt
*pvt
= mci
->pvt_info
;
996 /* CHIPKILL enabled */
997 if (pvt
->nbcfg
& NBCFG_CHIPKILL
) {
998 channel
= get_channel_from_ecc_syndrome(mci
, syndrome
);
1001 * Syndrome didn't map, so we don't know which of the
1002 * 2 DIMMs is in error. So we need to ID 'both' of them
1005 amd64_mc_warn(mci
, "unknown syndrome 0x%04x - possible "
1006 "error reporting race\n", syndrome
);
1007 edac_mc_handle_ce_no_info(mci
, EDAC_MOD_STR
);
1012 * non-chipkill ecc mode
1014 * The k8 documentation is unclear about how to determine the
1015 * channel number when using non-chipkill memory. This method
1016 * was obtained from email communication with someone at AMD.
1017 * (Wish the email was placed in this comment - norsk)
1019 channel
= ((sys_addr
& BIT(3)) != 0);
1023 * Find out which node the error address belongs to. This may be
1024 * different from the node that detected the error.
1026 src_mci
= find_mc_by_sys_addr(mci
, sys_addr
);
1028 amd64_mc_err(mci
, "failed to map error addr 0x%lx to a node\n",
1029 (unsigned long)sys_addr
);
1030 edac_mc_handle_ce_no_info(mci
, EDAC_MOD_STR
);
1034 /* Now map the sys_addr to a CSROW */
1035 csrow
= sys_addr_to_csrow(src_mci
, sys_addr
);
1037 edac_mc_handle_ce_no_info(src_mci
, EDAC_MOD_STR
);
1039 error_address_to_page_and_offset(sys_addr
, &page
, &offset
);
1041 edac_mc_handle_ce(src_mci
, page
, offset
, syndrome
, csrow
,
1042 channel
, EDAC_MOD_STR
);
1046 static int ddr2_cs_size(unsigned i
, bool dct_width
)
1052 else if (!(i
& 0x1))
1055 shift
= (i
+ 1) >> 1;
1057 return 128 << (shift
+ !!dct_width
);
1060 static int k8_dbam_to_chip_select(struct amd64_pvt
*pvt
, u8 dct
,
1063 u32 dclr
= dct
? pvt
->dclr1
: pvt
->dclr0
;
1065 if (pvt
->ext_model
>= K8_REV_F
) {
1066 WARN_ON(cs_mode
> 11);
1067 return ddr2_cs_size(cs_mode
, dclr
& WIDTH_128
);
1069 else if (pvt
->ext_model
>= K8_REV_D
) {
1070 WARN_ON(cs_mode
> 10);
1072 if (cs_mode
== 3 || cs_mode
== 8)
1073 return 32 << (cs_mode
- 1);
1075 return 32 << cs_mode
;
1078 WARN_ON(cs_mode
> 6);
1079 return 32 << cs_mode
;
1084 * Get the number of DCT channels in use.
1087 * number of Memory Channels in operation
1089 * contents of the DCL0_LOW register
1091 static int f1x_early_channel_count(struct amd64_pvt
*pvt
)
1093 int i
, j
, channels
= 0;
1095 /* On F10h, if we are in 128 bit mode, then we are using 2 channels */
1096 if (boot_cpu_data
.x86
== 0x10 && (pvt
->dclr0
& WIDTH_128
))
1100 * Need to check if in unganged mode: In such, there are 2 channels,
1101 * but they are not in 128 bit mode and thus the above 'dclr0' status
1104 * Need to check DCT0[0] and DCT1[0] to see if only one of them has
1105 * their CSEnable bit on. If so, then SINGLE DIMM case.
1107 debugf0("Data width is not 128 bits - need more decoding\n");
1110 * Check DRAM Bank Address Mapping values for each DIMM to see if there
1111 * is more than just one DIMM present in unganged mode. Need to check
1112 * both controllers since DIMMs can be placed in either one.
1114 for (i
= 0; i
< 2; i
++) {
1115 u32 dbam
= (i
? pvt
->dbam1
: pvt
->dbam0
);
1117 for (j
= 0; j
< 4; j
++) {
1118 if (DBAM_DIMM(j
, dbam
) > 0) {
1128 amd64_info("MCT channel count: %d\n", channels
);
1133 static int ddr3_cs_size(unsigned i
, bool dct_width
)
1138 if (i
== 0 || i
== 3 || i
== 4)
1144 else if (!(i
& 0x1))
1147 shift
= (i
+ 1) >> 1;
1150 cs_size
= (128 * (1 << !!dct_width
)) << shift
;
1155 static int f10_dbam_to_chip_select(struct amd64_pvt
*pvt
, u8 dct
,
1158 u32 dclr
= dct
? pvt
->dclr1
: pvt
->dclr0
;
1160 WARN_ON(cs_mode
> 11);
1162 if (pvt
->dchr0
& DDR3_MODE
|| pvt
->dchr1
& DDR3_MODE
)
1163 return ddr3_cs_size(cs_mode
, dclr
& WIDTH_128
);
1165 return ddr2_cs_size(cs_mode
, dclr
& WIDTH_128
);
1169 * F15h supports only 64bit DCT interfaces
1171 static int f15_dbam_to_chip_select(struct amd64_pvt
*pvt
, u8 dct
,
1174 WARN_ON(cs_mode
> 12);
1176 return ddr3_cs_size(cs_mode
, false);
1179 static void read_dram_ctl_register(struct amd64_pvt
*pvt
)
1182 if (boot_cpu_data
.x86
== 0xf)
1185 if (!amd64_read_dct_pci_cfg(pvt
, DCT_SEL_LO
, &pvt
->dct_sel_lo
)) {
1186 debugf0("F2x110 (DCTSelLow): 0x%08x, High range addrs at: 0x%x\n",
1187 pvt
->dct_sel_lo
, dct_sel_baseaddr(pvt
));
1189 debugf0(" DCTs operate in %s mode.\n",
1190 (dct_ganging_enabled(pvt
) ? "ganged" : "unganged"));
1192 if (!dct_ganging_enabled(pvt
))
1193 debugf0(" Address range split per DCT: %s\n",
1194 (dct_high_range_enabled(pvt
) ? "yes" : "no"));
1196 debugf0(" data interleave for ECC: %s, "
1197 "DRAM cleared since last warm reset: %s\n",
1198 (dct_data_intlv_enabled(pvt
) ? "enabled" : "disabled"),
1199 (dct_memory_cleared(pvt
) ? "yes" : "no"));
1201 debugf0(" channel interleave: %s, "
1202 "interleave bits selector: 0x%x\n",
1203 (dct_interleave_enabled(pvt
) ? "enabled" : "disabled"),
1204 dct_sel_interleave_addr(pvt
));
1207 amd64_read_dct_pci_cfg(pvt
, DCT_SEL_HI
, &pvt
->dct_sel_hi
);
1211 * Determine channel (DCT) based on the interleaving mode: F10h BKDG, 2.8.9 Memory
1212 * Interleaving Modes.
1214 static u8
f1x_determine_channel(struct amd64_pvt
*pvt
, u64 sys_addr
,
1215 bool hi_range_sel
, u8 intlv_en
)
1217 u8 dct_sel_high
= (pvt
->dct_sel_lo
>> 1) & 1;
1219 if (dct_ganging_enabled(pvt
))
1223 return dct_sel_high
;
1226 * see F2x110[DctSelIntLvAddr] - channel interleave mode
1228 if (dct_interleave_enabled(pvt
)) {
1229 u8 intlv_addr
= dct_sel_interleave_addr(pvt
);
1231 /* return DCT select function: 0=DCT0, 1=DCT1 */
1233 return sys_addr
>> 6 & 1;
1235 if (intlv_addr
& 0x2) {
1236 u8 shift
= intlv_addr
& 0x1 ? 9 : 6;
1237 u32 temp
= hweight_long((u32
) ((sys_addr
>> 16) & 0x1F)) % 2;
1239 return ((sys_addr
>> shift
) & 1) ^ temp
;
1242 return (sys_addr
>> (12 + hweight8(intlv_en
))) & 1;
1245 if (dct_high_range_enabled(pvt
))
1246 return ~dct_sel_high
& 1;
1251 /* Convert the sys_addr to the normalized DCT address */
1252 static u64
f1x_get_norm_dct_addr(struct amd64_pvt
*pvt
, unsigned range
,
1253 u64 sys_addr
, bool hi_rng
,
1254 u32 dct_sel_base_addr
)
1257 u64 dram_base
= get_dram_base(pvt
, range
);
1258 u64 hole_off
= f10_dhar_offset(pvt
);
1259 u64 dct_sel_base_off
= (pvt
->dct_sel_hi
& 0xFFFFFC00) << 16;
1264 * base address of high range is below 4Gb
1265 * (bits [47:27] at [31:11])
1266 * DRAM address space on this DCT is hoisted above 4Gb &&
1269 * remove hole offset from sys_addr
1271 * remove high range offset from sys_addr
1273 if ((!(dct_sel_base_addr
>> 16) ||
1274 dct_sel_base_addr
< dhar_base(pvt
)) &&
1276 (sys_addr
>= BIT_64(32)))
1277 chan_off
= hole_off
;
1279 chan_off
= dct_sel_base_off
;
1283 * we have a valid hole &&
1288 * remove dram base to normalize to DCT address
1290 if (dhar_valid(pvt
) && (sys_addr
>= BIT_64(32)))
1291 chan_off
= hole_off
;
1293 chan_off
= dram_base
;
1296 return (sys_addr
& GENMASK(6,47)) - (chan_off
& GENMASK(23,47));
1300 * checks if the csrow passed in is marked as SPARED, if so returns the new
1303 static int f10_process_possible_spare(struct amd64_pvt
*pvt
, u8 dct
, int csrow
)
1307 if (online_spare_swap_done(pvt
, dct
) &&
1308 csrow
== online_spare_bad_dramcs(pvt
, dct
)) {
1310 for_each_chip_select(tmp_cs
, dct
, pvt
) {
1311 if (chip_select_base(tmp_cs
, dct
, pvt
) & 0x2) {
1321 * Iterate over the DRAM DCT "base" and "mask" registers looking for a
1322 * SystemAddr match on the specified 'ChannelSelect' and 'NodeID'
1325 * -EINVAL: NOT FOUND
1326 * 0..csrow = Chip-Select Row
1328 static int f1x_lookup_addr_in_dct(u64 in_addr
, u32 nid
, u8 dct
)
1330 struct mem_ctl_info
*mci
;
1331 struct amd64_pvt
*pvt
;
1332 u64 cs_base
, cs_mask
;
1333 int cs_found
= -EINVAL
;
1340 pvt
= mci
->pvt_info
;
1342 debugf1("input addr: 0x%llx, DCT: %d\n", in_addr
, dct
);
1344 for_each_chip_select(csrow
, dct
, pvt
) {
1345 if (!csrow_enabled(csrow
, dct
, pvt
))
1348 get_cs_base_and_mask(pvt
, csrow
, dct
, &cs_base
, &cs_mask
);
1350 debugf1(" CSROW=%d CSBase=0x%llx CSMask=0x%llx\n",
1351 csrow
, cs_base
, cs_mask
);
1355 debugf1(" (InputAddr & ~CSMask)=0x%llx "
1356 "(CSBase & ~CSMask)=0x%llx\n",
1357 (in_addr
& cs_mask
), (cs_base
& cs_mask
));
1359 if ((in_addr
& cs_mask
) == (cs_base
& cs_mask
)) {
1360 cs_found
= f10_process_possible_spare(pvt
, dct
, csrow
);
1362 debugf1(" MATCH csrow=%d\n", cs_found
);
1370 * See F2x10C. Non-interleaved graphics framebuffer memory under the 16G is
1371 * swapped with a region located at the bottom of memory so that the GPU can use
1372 * the interleaved region and thus two channels.
1374 static u64
f1x_swap_interleaved_region(struct amd64_pvt
*pvt
, u64 sys_addr
)
1376 u32 swap_reg
, swap_base
, swap_limit
, rgn_size
, tmp_addr
;
1378 if (boot_cpu_data
.x86
== 0x10) {
1379 /* only revC3 and revE have that feature */
1380 if (boot_cpu_data
.x86_model
< 4 ||
1381 (boot_cpu_data
.x86_model
< 0xa &&
1382 boot_cpu_data
.x86_mask
< 3))
1386 amd64_read_dct_pci_cfg(pvt
, SWAP_INTLV_REG
, &swap_reg
);
1388 if (!(swap_reg
& 0x1))
1391 swap_base
= (swap_reg
>> 3) & 0x7f;
1392 swap_limit
= (swap_reg
>> 11) & 0x7f;
1393 rgn_size
= (swap_reg
>> 20) & 0x7f;
1394 tmp_addr
= sys_addr
>> 27;
1396 if (!(sys_addr
>> 34) &&
1397 (((tmp_addr
>= swap_base
) &&
1398 (tmp_addr
<= swap_limit
)) ||
1399 (tmp_addr
< rgn_size
)))
1400 return sys_addr
^ (u64
)swap_base
<< 27;
1405 /* For a given @dram_range, check if @sys_addr falls within it. */
1406 static int f1x_match_to_this_node(struct amd64_pvt
*pvt
, unsigned range
,
1407 u64 sys_addr
, int *nid
, int *chan_sel
)
1409 int cs_found
= -EINVAL
;
1413 bool high_range
= false;
1415 u8 node_id
= dram_dst_node(pvt
, range
);
1416 u8 intlv_en
= dram_intlv_en(pvt
, range
);
1417 u32 intlv_sel
= dram_intlv_sel(pvt
, range
);
1419 debugf1("(range %d) SystemAddr= 0x%llx Limit=0x%llx\n",
1420 range
, sys_addr
, get_dram_limit(pvt
, range
));
1422 if (dhar_valid(pvt
) &&
1423 dhar_base(pvt
) <= sys_addr
&&
1424 sys_addr
< BIT_64(32)) {
1425 amd64_warn("Huh? Address is in the MMIO hole: 0x%016llx\n",
1430 if (intlv_en
&& (intlv_sel
!= ((sys_addr
>> 12) & intlv_en
)))
1433 sys_addr
= f1x_swap_interleaved_region(pvt
, sys_addr
);
1435 dct_sel_base
= dct_sel_baseaddr(pvt
);
1438 * check whether addresses >= DctSelBaseAddr[47:27] are to be used to
1439 * select between DCT0 and DCT1.
1441 if (dct_high_range_enabled(pvt
) &&
1442 !dct_ganging_enabled(pvt
) &&
1443 ((sys_addr
>> 27) >= (dct_sel_base
>> 11)))
1446 channel
= f1x_determine_channel(pvt
, sys_addr
, high_range
, intlv_en
);
1448 chan_addr
= f1x_get_norm_dct_addr(pvt
, range
, sys_addr
,
1449 high_range
, dct_sel_base
);
1451 /* Remove node interleaving, see F1x120 */
1453 chan_addr
= ((chan_addr
>> (12 + hweight8(intlv_en
))) << 12) |
1454 (chan_addr
& 0xfff);
1456 /* remove channel interleave */
1457 if (dct_interleave_enabled(pvt
) &&
1458 !dct_high_range_enabled(pvt
) &&
1459 !dct_ganging_enabled(pvt
)) {
1461 if (dct_sel_interleave_addr(pvt
) != 1) {
1462 if (dct_sel_interleave_addr(pvt
) == 0x3)
1464 chan_addr
= ((chan_addr
>> 10) << 9) |
1465 (chan_addr
& 0x1ff);
1467 /* A[6] or hash 6 */
1468 chan_addr
= ((chan_addr
>> 7) << 6) |
1472 chan_addr
= ((chan_addr
>> 13) << 12) |
1473 (chan_addr
& 0xfff);
1476 debugf1(" Normalized DCT addr: 0x%llx\n", chan_addr
);
1478 cs_found
= f1x_lookup_addr_in_dct(chan_addr
, node_id
, channel
);
1480 if (cs_found
>= 0) {
1482 *chan_sel
= channel
;
1487 static int f1x_translate_sysaddr_to_cs(struct amd64_pvt
*pvt
, u64 sys_addr
,
1488 int *node
, int *chan_sel
)
1490 int cs_found
= -EINVAL
;
1493 for (range
= 0; range
< DRAM_RANGES
; range
++) {
1495 if (!dram_rw(pvt
, range
))
1498 if ((get_dram_base(pvt
, range
) <= sys_addr
) &&
1499 (get_dram_limit(pvt
, range
) >= sys_addr
)) {
1501 cs_found
= f1x_match_to_this_node(pvt
, range
,
1512 * For reference see "2.8.5 Routing DRAM Requests" in F10 BKDG. This code maps
1513 * a @sys_addr to NodeID, DCT (channel) and chip select (CSROW).
1515 * The @sys_addr is usually an error address received from the hardware
1518 static void f1x_map_sysaddr_to_csrow(struct mem_ctl_info
*mci
, u64 sys_addr
,
1521 struct amd64_pvt
*pvt
= mci
->pvt_info
;
1523 int nid
, csrow
, chan
= 0;
1525 csrow
= f1x_translate_sysaddr_to_cs(pvt
, sys_addr
, &nid
, &chan
);
1528 edac_mc_handle_ce_no_info(mci
, EDAC_MOD_STR
);
1532 error_address_to_page_and_offset(sys_addr
, &page
, &offset
);
1535 * We need the syndromes for channel detection only when we're
1536 * ganged. Otherwise @chan should already contain the channel at
1539 if (dct_ganging_enabled(pvt
))
1540 chan
= get_channel_from_ecc_syndrome(mci
, syndrome
);
1543 edac_mc_handle_ce(mci
, page
, offset
, syndrome
, csrow
, chan
,
1547 * Channel unknown, report all channels on this CSROW as failed.
1549 for (chan
= 0; chan
< mci
->csrows
[csrow
].nr_channels
; chan
++)
1550 edac_mc_handle_ce(mci
, page
, offset
, syndrome
,
1551 csrow
, chan
, EDAC_MOD_STR
);
1555 * debug routine to display the memory sizes of all logical DIMMs and its
1558 static void amd64_debug_display_dimm_sizes(struct amd64_pvt
*pvt
, u8 ctrl
)
1560 int dimm
, size0
, size1
, factor
= 0;
1561 u32
*dcsb
= ctrl
? pvt
->csels
[1].csbases
: pvt
->csels
[0].csbases
;
1562 u32 dbam
= ctrl
? pvt
->dbam1
: pvt
->dbam0
;
1564 if (boot_cpu_data
.x86
== 0xf) {
1565 if (pvt
->dclr0
& WIDTH_128
)
1568 /* K8 families < revF not supported yet */
1569 if (pvt
->ext_model
< K8_REV_F
)
1575 dbam
= (ctrl
&& !dct_ganging_enabled(pvt
)) ? pvt
->dbam1
: pvt
->dbam0
;
1576 dcsb
= (ctrl
&& !dct_ganging_enabled(pvt
)) ? pvt
->csels
[1].csbases
1577 : pvt
->csels
[0].csbases
;
1579 debugf1("F2x%d80 (DRAM Bank Address Mapping): 0x%08x\n", ctrl
, dbam
);
1581 edac_printk(KERN_DEBUG
, EDAC_MC
, "DCT%d chip selects:\n", ctrl
);
1583 /* Dump memory sizes for DIMM and its CSROWs */
1584 for (dimm
= 0; dimm
< 4; dimm
++) {
1587 if (dcsb
[dimm
*2] & DCSB_CS_ENABLE
)
1588 size0
= pvt
->ops
->dbam_to_cs(pvt
, ctrl
,
1589 DBAM_DIMM(dimm
, dbam
));
1592 if (dcsb
[dimm
*2 + 1] & DCSB_CS_ENABLE
)
1593 size1
= pvt
->ops
->dbam_to_cs(pvt
, ctrl
,
1594 DBAM_DIMM(dimm
, dbam
));
1596 amd64_info(EDAC_MC
": %d: %5dMB %d: %5dMB\n",
1597 dimm
* 2, size0
<< factor
,
1598 dimm
* 2 + 1, size1
<< factor
);
1602 static struct amd64_family_type amd64_family_types
[] = {
1605 .f1_id
= PCI_DEVICE_ID_AMD_K8_NB_ADDRMAP
,
1606 .f3_id
= PCI_DEVICE_ID_AMD_K8_NB_MISC
,
1608 .early_channel_count
= k8_early_channel_count
,
1609 .map_sysaddr_to_csrow
= k8_map_sysaddr_to_csrow
,
1610 .dbam_to_cs
= k8_dbam_to_chip_select
,
1611 .read_dct_pci_cfg
= k8_read_dct_pci_cfg
,
1616 .f1_id
= PCI_DEVICE_ID_AMD_10H_NB_MAP
,
1617 .f3_id
= PCI_DEVICE_ID_AMD_10H_NB_MISC
,
1619 .early_channel_count
= f1x_early_channel_count
,
1620 .map_sysaddr_to_csrow
= f1x_map_sysaddr_to_csrow
,
1621 .dbam_to_cs
= f10_dbam_to_chip_select
,
1622 .read_dct_pci_cfg
= f10_read_dct_pci_cfg
,
1627 .f1_id
= PCI_DEVICE_ID_AMD_15H_NB_F1
,
1628 .f3_id
= PCI_DEVICE_ID_AMD_15H_NB_F3
,
1630 .early_channel_count
= f1x_early_channel_count
,
1631 .map_sysaddr_to_csrow
= f1x_map_sysaddr_to_csrow
,
1632 .dbam_to_cs
= f15_dbam_to_chip_select
,
1633 .read_dct_pci_cfg
= f15_read_dct_pci_cfg
,
1638 static struct pci_dev
*pci_get_related_function(unsigned int vendor
,
1639 unsigned int device
,
1640 struct pci_dev
*related
)
1642 struct pci_dev
*dev
= NULL
;
1644 dev
= pci_get_device(vendor
, device
, dev
);
1646 if ((dev
->bus
->number
== related
->bus
->number
) &&
1647 (PCI_SLOT(dev
->devfn
) == PCI_SLOT(related
->devfn
)))
1649 dev
= pci_get_device(vendor
, device
, dev
);
1656 * These are tables of eigenvectors (one per line) which can be used for the
1657 * construction of the syndrome tables. The modified syndrome search algorithm
1658 * uses those to find the symbol in error and thus the DIMM.
1660 * Algorithm courtesy of Ross LaFetra from AMD.
1662 static u16 x4_vectors
[] = {
1663 0x2f57, 0x1afe, 0x66cc, 0xdd88,
1664 0x11eb, 0x3396, 0x7f4c, 0xeac8,
1665 0x0001, 0x0002, 0x0004, 0x0008,
1666 0x1013, 0x3032, 0x4044, 0x8088,
1667 0x106b, 0x30d6, 0x70fc, 0xe0a8,
1668 0x4857, 0xc4fe, 0x13cc, 0x3288,
1669 0x1ac5, 0x2f4a, 0x5394, 0xa1e8,
1670 0x1f39, 0x251e, 0xbd6c, 0x6bd8,
1671 0x15c1, 0x2a42, 0x89ac, 0x4758,
1672 0x2b03, 0x1602, 0x4f0c, 0xca08,
1673 0x1f07, 0x3a0e, 0x6b04, 0xbd08,
1674 0x8ba7, 0x465e, 0x244c, 0x1cc8,
1675 0x2b87, 0x164e, 0x642c, 0xdc18,
1676 0x40b9, 0x80de, 0x1094, 0x20e8,
1677 0x27db, 0x1eb6, 0x9dac, 0x7b58,
1678 0x11c1, 0x2242, 0x84ac, 0x4c58,
1679 0x1be5, 0x2d7a, 0x5e34, 0xa718,
1680 0x4b39, 0x8d1e, 0x14b4, 0x28d8,
1681 0x4c97, 0xc87e, 0x11fc, 0x33a8,
1682 0x8e97, 0x497e, 0x2ffc, 0x1aa8,
1683 0x16b3, 0x3d62, 0x4f34, 0x8518,
1684 0x1e2f, 0x391a, 0x5cac, 0xf858,
1685 0x1d9f, 0x3b7a, 0x572c, 0xfe18,
1686 0x15f5, 0x2a5a, 0x5264, 0xa3b8,
1687 0x1dbb, 0x3b66, 0x715c, 0xe3f8,
1688 0x4397, 0xc27e, 0x17fc, 0x3ea8,
1689 0x1617, 0x3d3e, 0x6464, 0xb8b8,
1690 0x23ff, 0x12aa, 0xab6c, 0x56d8,
1691 0x2dfb, 0x1ba6, 0x913c, 0x7328,
1692 0x185d, 0x2ca6, 0x7914, 0x9e28,
1693 0x171b, 0x3e36, 0x7d7c, 0xebe8,
1694 0x4199, 0x82ee, 0x19f4, 0x2e58,
1695 0x4807, 0xc40e, 0x130c, 0x3208,
1696 0x1905, 0x2e0a, 0x5804, 0xac08,
1697 0x213f, 0x132a, 0xadfc, 0x5ba8,
1698 0x19a9, 0x2efe, 0xb5cc, 0x6f88,
1701 static u16 x8_vectors
[] = {
1702 0x0145, 0x028a, 0x2374, 0x43c8, 0xa1f0, 0x0520, 0x0a40, 0x1480,
1703 0x0211, 0x0422, 0x0844, 0x1088, 0x01b0, 0x44e0, 0x23c0, 0xed80,
1704 0x1011, 0x0116, 0x022c, 0x0458, 0x08b0, 0x8c60, 0x2740, 0x4e80,
1705 0x0411, 0x0822, 0x1044, 0x0158, 0x02b0, 0x2360, 0x46c0, 0xab80,
1706 0x0811, 0x1022, 0x012c, 0x0258, 0x04b0, 0x4660, 0x8cc0, 0x2780,
1707 0x2071, 0x40e2, 0xa0c4, 0x0108, 0x0210, 0x0420, 0x0840, 0x1080,
1708 0x4071, 0x80e2, 0x0104, 0x0208, 0x0410, 0x0820, 0x1040, 0x2080,
1709 0x8071, 0x0102, 0x0204, 0x0408, 0x0810, 0x1020, 0x2040, 0x4080,
1710 0x019d, 0x03d6, 0x136c, 0x2198, 0x50b0, 0xb2e0, 0x0740, 0x0e80,
1711 0x0189, 0x03ea, 0x072c, 0x0e58, 0x1cb0, 0x56e0, 0x37c0, 0xf580,
1712 0x01fd, 0x0376, 0x06ec, 0x0bb8, 0x1110, 0x2220, 0x4440, 0x8880,
1713 0x0163, 0x02c6, 0x1104, 0x0758, 0x0eb0, 0x2be0, 0x6140, 0xc280,
1714 0x02fd, 0x01c6, 0x0b5c, 0x1108, 0x07b0, 0x25a0, 0x8840, 0x6180,
1715 0x0801, 0x012e, 0x025c, 0x04b8, 0x1370, 0x26e0, 0x57c0, 0xb580,
1716 0x0401, 0x0802, 0x015c, 0x02b8, 0x22b0, 0x13e0, 0x7140, 0xe280,
1717 0x0201, 0x0402, 0x0804, 0x01b8, 0x11b0, 0x31a0, 0x8040, 0x7180,
1718 0x0101, 0x0202, 0x0404, 0x0808, 0x1010, 0x2020, 0x4040, 0x8080,
1719 0x0001, 0x0002, 0x0004, 0x0008, 0x0010, 0x0020, 0x0040, 0x0080,
1720 0x0100, 0x0200, 0x0400, 0x0800, 0x1000, 0x2000, 0x4000, 0x8000,
1723 static int decode_syndrome(u16 syndrome
, u16
*vectors
, unsigned num_vecs
,
1726 unsigned int i
, err_sym
;
1728 for (err_sym
= 0; err_sym
< num_vecs
/ v_dim
; err_sym
++) {
1730 unsigned v_idx
= err_sym
* v_dim
;
1731 unsigned v_end
= (err_sym
+ 1) * v_dim
;
1733 /* walk over all 16 bits of the syndrome */
1734 for (i
= 1; i
< (1U << 16); i
<<= 1) {
1736 /* if bit is set in that eigenvector... */
1737 if (v_idx
< v_end
&& vectors
[v_idx
] & i
) {
1738 u16 ev_comp
= vectors
[v_idx
++];
1740 /* ... and bit set in the modified syndrome, */
1750 /* can't get to zero, move to next symbol */
1755 debugf0("syndrome(%x) not found\n", syndrome
);
1759 static int map_err_sym_to_channel(int err_sym
, int sym_size
)
1772 return err_sym
>> 4;
1778 /* imaginary bits not in a DIMM */
1780 WARN(1, KERN_ERR
"Invalid error symbol: 0x%x\n",
1792 return err_sym
>> 3;
1798 static int get_channel_from_ecc_syndrome(struct mem_ctl_info
*mci
, u16 syndrome
)
1800 struct amd64_pvt
*pvt
= mci
->pvt_info
;
1803 if (pvt
->ecc_sym_sz
== 8)
1804 err_sym
= decode_syndrome(syndrome
, x8_vectors
,
1805 ARRAY_SIZE(x8_vectors
),
1807 else if (pvt
->ecc_sym_sz
== 4)
1808 err_sym
= decode_syndrome(syndrome
, x4_vectors
,
1809 ARRAY_SIZE(x4_vectors
),
1812 amd64_warn("Illegal syndrome type: %u\n", pvt
->ecc_sym_sz
);
1816 return map_err_sym_to_channel(err_sym
, pvt
->ecc_sym_sz
);
1820 * Handle any Correctable Errors (CEs) that have occurred. Check for valid ERROR
1821 * ADDRESS and process.
1823 static void amd64_handle_ce(struct mem_ctl_info
*mci
, struct mce
*m
)
1825 struct amd64_pvt
*pvt
= mci
->pvt_info
;
1829 /* Ensure that the Error Address is VALID */
1830 if (!(m
->status
& MCI_STATUS_ADDRV
)) {
1831 amd64_mc_err(mci
, "HW has no ERROR_ADDRESS available\n");
1832 edac_mc_handle_ce_no_info(mci
, EDAC_MOD_STR
);
1836 sys_addr
= get_error_address(m
);
1837 syndrome
= extract_syndrome(m
->status
);
1839 amd64_mc_err(mci
, "CE ERROR_ADDRESS= 0x%llx\n", sys_addr
);
1841 pvt
->ops
->map_sysaddr_to_csrow(mci
, sys_addr
, syndrome
);
1844 /* Handle any Un-correctable Errors (UEs) */
1845 static void amd64_handle_ue(struct mem_ctl_info
*mci
, struct mce
*m
)
1847 struct mem_ctl_info
*log_mci
, *src_mci
= NULL
;
1854 if (!(m
->status
& MCI_STATUS_ADDRV
)) {
1855 amd64_mc_err(mci
, "HW has no ERROR_ADDRESS available\n");
1856 edac_mc_handle_ue_no_info(log_mci
, EDAC_MOD_STR
);
1860 sys_addr
= get_error_address(m
);
1863 * Find out which node the error address belongs to. This may be
1864 * different from the node that detected the error.
1866 src_mci
= find_mc_by_sys_addr(mci
, sys_addr
);
1868 amd64_mc_err(mci
, "ERROR ADDRESS (0x%lx) NOT mapped to a MC\n",
1869 (unsigned long)sys_addr
);
1870 edac_mc_handle_ue_no_info(log_mci
, EDAC_MOD_STR
);
1876 csrow
= sys_addr_to_csrow(log_mci
, sys_addr
);
1878 amd64_mc_err(mci
, "ERROR_ADDRESS (0x%lx) NOT mapped to CS\n",
1879 (unsigned long)sys_addr
);
1880 edac_mc_handle_ue_no_info(log_mci
, EDAC_MOD_STR
);
1882 error_address_to_page_and_offset(sys_addr
, &page
, &offset
);
1883 edac_mc_handle_ue(log_mci
, page
, offset
, csrow
, EDAC_MOD_STR
);
1887 static inline void __amd64_decode_bus_error(struct mem_ctl_info
*mci
,
1890 u16 ec
= EC(m
->status
);
1891 u8 xec
= XEC(m
->status
, 0x1f);
1892 u8 ecc_type
= (m
->status
>> 45) & 0x3;
1894 /* Bail early out if this was an 'observed' error */
1895 if (PP(ec
) == NBSL_PP_OBS
)
1898 /* Do only ECC errors */
1899 if (xec
&& xec
!= F10_NBSL_EXT_ERR_ECC
)
1903 amd64_handle_ce(mci
, m
);
1904 else if (ecc_type
== 1)
1905 amd64_handle_ue(mci
, m
);
1908 void amd64_decode_bus_error(int node_id
, struct mce
*m
, u32 nbcfg
)
1910 struct mem_ctl_info
*mci
= mcis
[node_id
];
1912 __amd64_decode_bus_error(mci
, m
);
1916 * Use pvt->F2 which contains the F2 CPU PCI device to get the related
1917 * F1 (AddrMap) and F3 (Misc) devices. Return negative value on error.
1919 static int reserve_mc_sibling_devs(struct amd64_pvt
*pvt
, u16 f1_id
, u16 f3_id
)
1921 /* Reserve the ADDRESS MAP Device */
1922 pvt
->F1
= pci_get_related_function(pvt
->F2
->vendor
, f1_id
, pvt
->F2
);
1924 amd64_err("error address map device not found: "
1925 "vendor %x device 0x%x (broken BIOS?)\n",
1926 PCI_VENDOR_ID_AMD
, f1_id
);
1930 /* Reserve the MISC Device */
1931 pvt
->F3
= pci_get_related_function(pvt
->F2
->vendor
, f3_id
, pvt
->F2
);
1933 pci_dev_put(pvt
->F1
);
1936 amd64_err("error F3 device not found: "
1937 "vendor %x device 0x%x (broken BIOS?)\n",
1938 PCI_VENDOR_ID_AMD
, f3_id
);
1942 debugf1("F1: %s\n", pci_name(pvt
->F1
));
1943 debugf1("F2: %s\n", pci_name(pvt
->F2
));
1944 debugf1("F3: %s\n", pci_name(pvt
->F3
));
1949 static void free_mc_sibling_devs(struct amd64_pvt
*pvt
)
1951 pci_dev_put(pvt
->F1
);
1952 pci_dev_put(pvt
->F3
);
1956 * Retrieve the hardware registers of the memory controller (this includes the
1957 * 'Address Map' and 'Misc' device regs)
1959 static void read_mc_regs(struct amd64_pvt
*pvt
)
1961 struct cpuinfo_x86
*c
= &boot_cpu_data
;
1967 * Retrieve TOP_MEM and TOP_MEM2; no masking off of reserved bits since
1968 * those are Read-As-Zero
1970 rdmsrl(MSR_K8_TOP_MEM1
, pvt
->top_mem
);
1971 debugf0(" TOP_MEM: 0x%016llx\n", pvt
->top_mem
);
1973 /* check first whether TOP_MEM2 is enabled */
1974 rdmsrl(MSR_K8_SYSCFG
, msr_val
);
1975 if (msr_val
& (1U << 21)) {
1976 rdmsrl(MSR_K8_TOP_MEM2
, pvt
->top_mem2
);
1977 debugf0(" TOP_MEM2: 0x%016llx\n", pvt
->top_mem2
);
1979 debugf0(" TOP_MEM2 disabled.\n");
1981 amd64_read_pci_cfg(pvt
->F3
, NBCAP
, &pvt
->nbcap
);
1983 read_dram_ctl_register(pvt
);
1985 for (range
= 0; range
< DRAM_RANGES
; range
++) {
1988 /* read settings for this DRAM range */
1989 read_dram_base_limit_regs(pvt
, range
);
1991 rw
= dram_rw(pvt
, range
);
1995 debugf1(" DRAM range[%d], base: 0x%016llx; limit: 0x%016llx\n",
1997 get_dram_base(pvt
, range
),
1998 get_dram_limit(pvt
, range
));
2000 debugf1(" IntlvEn=%s; Range access: %s%s IntlvSel=%d DstNode=%d\n",
2001 dram_intlv_en(pvt
, range
) ? "Enabled" : "Disabled",
2002 (rw
& 0x1) ? "R" : "-",
2003 (rw
& 0x2) ? "W" : "-",
2004 dram_intlv_sel(pvt
, range
),
2005 dram_dst_node(pvt
, range
));
2008 read_dct_base_mask(pvt
);
2010 amd64_read_pci_cfg(pvt
->F1
, DHAR
, &pvt
->dhar
);
2011 amd64_read_dct_pci_cfg(pvt
, DBAM0
, &pvt
->dbam0
);
2013 amd64_read_pci_cfg(pvt
->F3
, F10_ONLINE_SPARE
, &pvt
->online_spare
);
2015 amd64_read_dct_pci_cfg(pvt
, DCLR0
, &pvt
->dclr0
);
2016 amd64_read_dct_pci_cfg(pvt
, DCHR0
, &pvt
->dchr0
);
2018 if (!dct_ganging_enabled(pvt
)) {
2019 amd64_read_dct_pci_cfg(pvt
, DCLR1
, &pvt
->dclr1
);
2020 amd64_read_dct_pci_cfg(pvt
, DCHR1
, &pvt
->dchr1
);
2023 pvt
->ecc_sym_sz
= 4;
2025 if (c
->x86
>= 0x10) {
2026 amd64_read_pci_cfg(pvt
->F3
, EXT_NB_MCA_CFG
, &tmp
);
2027 amd64_read_dct_pci_cfg(pvt
, DBAM1
, &pvt
->dbam1
);
2029 /* F10h, revD and later can do x8 ECC too */
2030 if ((c
->x86
> 0x10 || c
->x86_model
> 7) && tmp
& BIT(25))
2031 pvt
->ecc_sym_sz
= 8;
2033 dump_misc_regs(pvt
);
2037 * NOTE: CPU Revision Dependent code
2040 * @csrow_nr ChipSelect Row Number (0..NUM_CHIPSELECTS-1)
2041 * k8 private pointer to -->
2042 * DRAM Bank Address mapping register
2044 * DCL register where dual_channel_active is
2046 * The DBAM register consists of 4 sets of 4 bits each definitions:
2049 * 0-3 CSROWs 0 and 1
2050 * 4-7 CSROWs 2 and 3
2051 * 8-11 CSROWs 4 and 5
2052 * 12-15 CSROWs 6 and 7
2054 * Values range from: 0 to 15
2055 * The meaning of the values depends on CPU revision and dual-channel state,
2056 * see relevant BKDG more info.
2058 * The memory controller provides for total of only 8 CSROWs in its current
2059 * architecture. Each "pair" of CSROWs normally represents just one DIMM in
2060 * single channel or two (2) DIMMs in dual channel mode.
2062 * The following code logic collapses the various tables for CSROW based on CPU
2066 * The number of PAGE_SIZE pages on the specified CSROW number it
2070 static u32
amd64_csrow_nr_pages(struct amd64_pvt
*pvt
, u8 dct
, int csrow_nr
)
2072 u32 cs_mode
, nr_pages
;
2075 * The math on this doesn't look right on the surface because x/2*4 can
2076 * be simplified to x*2 but this expression makes use of the fact that
2077 * it is integral math where 1/2=0. This intermediate value becomes the
2078 * number of bits to shift the DBAM register to extract the proper CSROW
2081 cs_mode
= (pvt
->dbam0
>> ((csrow_nr
/ 2) * 4)) & 0xF;
2083 nr_pages
= pvt
->ops
->dbam_to_cs(pvt
, dct
, cs_mode
) << (20 - PAGE_SHIFT
);
2086 * If dual channel then double the memory size of single channel.
2087 * Channel count is 1 or 2
2089 nr_pages
<<= (pvt
->channel_count
- 1);
2091 debugf0(" (csrow=%d) DBAM map index= %d\n", csrow_nr
, cs_mode
);
2092 debugf0(" nr_pages= %u channel-count = %d\n",
2093 nr_pages
, pvt
->channel_count
);
2099 * Initialize the array of csrow attribute instances, based on the values
2100 * from pci config hardware registers.
2102 static int init_csrows(struct mem_ctl_info
*mci
)
2104 struct csrow_info
*csrow
;
2105 struct amd64_pvt
*pvt
= mci
->pvt_info
;
2106 u64 input_addr_min
, input_addr_max
, sys_addr
, base
, mask
;
2110 amd64_read_pci_cfg(pvt
->F3
, NBCFG
, &val
);
2114 debugf0("node %d, NBCFG=0x%08x[ChipKillEccCap: %d|DramEccEn: %d]\n",
2115 pvt
->mc_node_id
, val
,
2116 !!(val
& NBCFG_CHIPKILL
), !!(val
& NBCFG_ECC_ENABLE
));
2118 for_each_chip_select(i
, 0, pvt
) {
2119 csrow
= &mci
->csrows
[i
];
2121 if (!csrow_enabled(i
, 0, pvt
)) {
2122 debugf1("----CSROW %d EMPTY for node %d\n", i
,
2127 debugf1("----CSROW %d VALID for MC node %d\n",
2128 i
, pvt
->mc_node_id
);
2131 csrow
->nr_pages
= amd64_csrow_nr_pages(pvt
, 0, i
);
2132 find_csrow_limits(mci
, i
, &input_addr_min
, &input_addr_max
);
2133 sys_addr
= input_addr_to_sys_addr(mci
, input_addr_min
);
2134 csrow
->first_page
= (u32
) (sys_addr
>> PAGE_SHIFT
);
2135 sys_addr
= input_addr_to_sys_addr(mci
, input_addr_max
);
2136 csrow
->last_page
= (u32
) (sys_addr
>> PAGE_SHIFT
);
2138 get_cs_base_and_mask(pvt
, i
, 0, &base
, &mask
);
2139 csrow
->page_mask
= ~mask
;
2140 /* 8 bytes of resolution */
2142 csrow
->mtype
= amd64_determine_memory_type(pvt
, i
);
2144 debugf1(" for MC node %d csrow %d:\n", pvt
->mc_node_id
, i
);
2145 debugf1(" input_addr_min: 0x%lx input_addr_max: 0x%lx\n",
2146 (unsigned long)input_addr_min
,
2147 (unsigned long)input_addr_max
);
2148 debugf1(" sys_addr: 0x%lx page_mask: 0x%lx\n",
2149 (unsigned long)sys_addr
, csrow
->page_mask
);
2150 debugf1(" nr_pages: %u first_page: 0x%lx "
2151 "last_page: 0x%lx\n",
2152 (unsigned)csrow
->nr_pages
,
2153 csrow
->first_page
, csrow
->last_page
);
2156 * determine whether CHIPKILL or JUST ECC or NO ECC is operating
2158 if (pvt
->nbcfg
& NBCFG_ECC_ENABLE
)
2160 (pvt
->nbcfg
& NBCFG_CHIPKILL
) ?
2161 EDAC_S4ECD4ED
: EDAC_SECDED
;
2163 csrow
->edac_mode
= EDAC_NONE
;
2169 /* get all cores on this DCT */
2170 static void get_cpus_on_this_dct_cpumask(struct cpumask
*mask
, unsigned nid
)
2174 for_each_online_cpu(cpu
)
2175 if (amd_get_nb_id(cpu
) == nid
)
2176 cpumask_set_cpu(cpu
, mask
);
2179 /* check MCG_CTL on all the cpus on this node */
2180 static bool amd64_nb_mce_bank_enabled_on_node(unsigned nid
)
2186 if (!zalloc_cpumask_var(&mask
, GFP_KERNEL
)) {
2187 amd64_warn("%s: Error allocating mask\n", __func__
);
2191 get_cpus_on_this_dct_cpumask(mask
, nid
);
2193 rdmsr_on_cpus(mask
, MSR_IA32_MCG_CTL
, msrs
);
2195 for_each_cpu(cpu
, mask
) {
2196 struct msr
*reg
= per_cpu_ptr(msrs
, cpu
);
2197 nbe
= reg
->l
& MSR_MCGCTL_NBE
;
2199 debugf0("core: %u, MCG_CTL: 0x%llx, NB MSR is %s\n",
2201 (nbe
? "enabled" : "disabled"));
2209 free_cpumask_var(mask
);
2213 static int toggle_ecc_err_reporting(struct ecc_settings
*s
, u8 nid
, bool on
)
2215 cpumask_var_t cmask
;
2218 if (!zalloc_cpumask_var(&cmask
, GFP_KERNEL
)) {
2219 amd64_warn("%s: error allocating mask\n", __func__
);
2223 get_cpus_on_this_dct_cpumask(cmask
, nid
);
2225 rdmsr_on_cpus(cmask
, MSR_IA32_MCG_CTL
, msrs
);
2227 for_each_cpu(cpu
, cmask
) {
2229 struct msr
*reg
= per_cpu_ptr(msrs
, cpu
);
2232 if (reg
->l
& MSR_MCGCTL_NBE
)
2233 s
->flags
.nb_mce_enable
= 1;
2235 reg
->l
|= MSR_MCGCTL_NBE
;
2238 * Turn off NB MCE reporting only when it was off before
2240 if (!s
->flags
.nb_mce_enable
)
2241 reg
->l
&= ~MSR_MCGCTL_NBE
;
2244 wrmsr_on_cpus(cmask
, MSR_IA32_MCG_CTL
, msrs
);
2246 free_cpumask_var(cmask
);
2251 static bool enable_ecc_error_reporting(struct ecc_settings
*s
, u8 nid
,
2255 u32 value
, mask
= 0x3; /* UECC/CECC enable */
2257 if (toggle_ecc_err_reporting(s
, nid
, ON
)) {
2258 amd64_warn("Error enabling ECC reporting over MCGCTL!\n");
2262 amd64_read_pci_cfg(F3
, NBCTL
, &value
);
2264 s
->old_nbctl
= value
& mask
;
2265 s
->nbctl_valid
= true;
2268 amd64_write_pci_cfg(F3
, NBCTL
, value
);
2270 amd64_read_pci_cfg(F3
, NBCFG
, &value
);
2272 debugf0("1: node %d, NBCFG=0x%08x[DramEccEn: %d]\n",
2273 nid
, value
, !!(value
& NBCFG_ECC_ENABLE
));
2275 if (!(value
& NBCFG_ECC_ENABLE
)) {
2276 amd64_warn("DRAM ECC disabled on this node, enabling...\n");
2278 s
->flags
.nb_ecc_prev
= 0;
2280 /* Attempt to turn on DRAM ECC Enable */
2281 value
|= NBCFG_ECC_ENABLE
;
2282 amd64_write_pci_cfg(F3
, NBCFG
, value
);
2284 amd64_read_pci_cfg(F3
, NBCFG
, &value
);
2286 if (!(value
& NBCFG_ECC_ENABLE
)) {
2287 amd64_warn("Hardware rejected DRAM ECC enable,"
2288 "check memory DIMM configuration.\n");
2291 amd64_info("Hardware accepted DRAM ECC Enable\n");
2294 s
->flags
.nb_ecc_prev
= 1;
2297 debugf0("2: node %d, NBCFG=0x%08x[DramEccEn: %d]\n",
2298 nid
, value
, !!(value
& NBCFG_ECC_ENABLE
));
2303 static void restore_ecc_error_reporting(struct ecc_settings
*s
, u8 nid
,
2306 u32 value
, mask
= 0x3; /* UECC/CECC enable */
2309 if (!s
->nbctl_valid
)
2312 amd64_read_pci_cfg(F3
, NBCTL
, &value
);
2314 value
|= s
->old_nbctl
;
2316 amd64_write_pci_cfg(F3
, NBCTL
, value
);
2318 /* restore previous BIOS DRAM ECC "off" setting we force-enabled */
2319 if (!s
->flags
.nb_ecc_prev
) {
2320 amd64_read_pci_cfg(F3
, NBCFG
, &value
);
2321 value
&= ~NBCFG_ECC_ENABLE
;
2322 amd64_write_pci_cfg(F3
, NBCFG
, value
);
2325 /* restore the NB Enable MCGCTL bit */
2326 if (toggle_ecc_err_reporting(s
, nid
, OFF
))
2327 amd64_warn("Error restoring NB MCGCTL settings!\n");
2331 * EDAC requires that the BIOS have ECC enabled before
2332 * taking over the processing of ECC errors. A command line
2333 * option allows to force-enable hardware ECC later in
2334 * enable_ecc_error_reporting().
2336 static const char *ecc_msg
=
2337 "ECC disabled in the BIOS or no ECC capability, module will not load.\n"
2338 " Either enable ECC checking or force module loading by setting "
2339 "'ecc_enable_override'.\n"
2340 " (Note that use of the override may cause unknown side effects.)\n";
2342 static bool ecc_enabled(struct pci_dev
*F3
, u8 nid
)
2346 bool nb_mce_en
= false;
2348 amd64_read_pci_cfg(F3
, NBCFG
, &value
);
2350 ecc_en
= !!(value
& NBCFG_ECC_ENABLE
);
2351 amd64_info("DRAM ECC %s.\n", (ecc_en
? "enabled" : "disabled"));
2353 nb_mce_en
= amd64_nb_mce_bank_enabled_on_node(nid
);
2355 amd64_notice("NB MCE bank disabled, set MSR "
2356 "0x%08x[4] on node %d to enable.\n",
2357 MSR_IA32_MCG_CTL
, nid
);
2359 if (!ecc_en
|| !nb_mce_en
) {
2360 amd64_notice("%s", ecc_msg
);
2366 struct mcidev_sysfs_attribute sysfs_attrs
[ARRAY_SIZE(amd64_dbg_attrs
) +
2367 ARRAY_SIZE(amd64_inj_attrs
) +
2370 struct mcidev_sysfs_attribute terminator
= { .attr
= { .name
= NULL
} };
2372 static void set_mc_sysfs_attrs(struct mem_ctl_info
*mci
)
2374 unsigned int i
= 0, j
= 0;
2376 for (; i
< ARRAY_SIZE(amd64_dbg_attrs
); i
++)
2377 sysfs_attrs
[i
] = amd64_dbg_attrs
[i
];
2379 if (boot_cpu_data
.x86
>= 0x10)
2380 for (j
= 0; j
< ARRAY_SIZE(amd64_inj_attrs
); j
++, i
++)
2381 sysfs_attrs
[i
] = amd64_inj_attrs
[j
];
2383 sysfs_attrs
[i
] = terminator
;
2385 mci
->mc_driver_sysfs_attributes
= sysfs_attrs
;
2388 static void setup_mci_misc_attrs(struct mem_ctl_info
*mci
,
2389 struct amd64_family_type
*fam
)
2391 struct amd64_pvt
*pvt
= mci
->pvt_info
;
2393 mci
->mtype_cap
= MEM_FLAG_DDR2
| MEM_FLAG_RDDR2
;
2394 mci
->edac_ctl_cap
= EDAC_FLAG_NONE
;
2396 if (pvt
->nbcap
& NBCAP_SECDED
)
2397 mci
->edac_ctl_cap
|= EDAC_FLAG_SECDED
;
2399 if (pvt
->nbcap
& NBCAP_CHIPKILL
)
2400 mci
->edac_ctl_cap
|= EDAC_FLAG_S4ECD4ED
;
2402 mci
->edac_cap
= amd64_determine_edac_cap(pvt
);
2403 mci
->mod_name
= EDAC_MOD_STR
;
2404 mci
->mod_ver
= EDAC_AMD64_VERSION
;
2405 mci
->ctl_name
= fam
->ctl_name
;
2406 mci
->dev_name
= pci_name(pvt
->F2
);
2407 mci
->ctl_page_to_phys
= NULL
;
2409 /* memory scrubber interface */
2410 mci
->set_sdram_scrub_rate
= amd64_set_scrub_rate
;
2411 mci
->get_sdram_scrub_rate
= amd64_get_scrub_rate
;
2415 * returns a pointer to the family descriptor on success, NULL otherwise.
2417 static struct amd64_family_type
*amd64_per_family_init(struct amd64_pvt
*pvt
)
2419 u8 fam
= boot_cpu_data
.x86
;
2420 struct amd64_family_type
*fam_type
= NULL
;
2424 fam_type
= &amd64_family_types
[K8_CPUS
];
2425 pvt
->ops
= &amd64_family_types
[K8_CPUS
].ops
;
2429 fam_type
= &amd64_family_types
[F10_CPUS
];
2430 pvt
->ops
= &amd64_family_types
[F10_CPUS
].ops
;
2434 fam_type
= &amd64_family_types
[F15_CPUS
];
2435 pvt
->ops
= &amd64_family_types
[F15_CPUS
].ops
;
2439 amd64_err("Unsupported family!\n");
2443 pvt
->ext_model
= boot_cpu_data
.x86_model
>> 4;
2445 amd64_info("%s %sdetected (node %d).\n", fam_type
->ctl_name
,
2447 (pvt
->ext_model
>= K8_REV_F
? "revF or later "
2448 : "revE or earlier ")
2449 : ""), pvt
->mc_node_id
);
2453 static int amd64_init_one_instance(struct pci_dev
*F2
)
2455 struct amd64_pvt
*pvt
= NULL
;
2456 struct amd64_family_type
*fam_type
= NULL
;
2457 struct mem_ctl_info
*mci
= NULL
;
2459 u8 nid
= get_node_id(F2
);
2462 pvt
= kzalloc(sizeof(struct amd64_pvt
), GFP_KERNEL
);
2466 pvt
->mc_node_id
= nid
;
2470 fam_type
= amd64_per_family_init(pvt
);
2475 err
= reserve_mc_sibling_devs(pvt
, fam_type
->f1_id
, fam_type
->f3_id
);
2482 * We need to determine how many memory channels there are. Then use
2483 * that information for calculating the size of the dynamic instance
2484 * tables in the 'mci' structure.
2487 pvt
->channel_count
= pvt
->ops
->early_channel_count(pvt
);
2488 if (pvt
->channel_count
< 0)
2492 mci
= edac_mc_alloc(0, pvt
->csels
[0].b_cnt
, pvt
->channel_count
, nid
);
2496 mci
->pvt_info
= pvt
;
2497 mci
->dev
= &pvt
->F2
->dev
;
2499 setup_mci_misc_attrs(mci
, fam_type
);
2501 if (init_csrows(mci
))
2502 mci
->edac_cap
= EDAC_FLAG_NONE
;
2504 set_mc_sysfs_attrs(mci
);
2507 if (edac_mc_add_mc(mci
)) {
2508 debugf1("failed edac_mc_add_mc()\n");
2512 /* register stuff with EDAC MCE */
2513 if (report_gart_errors
)
2514 amd_report_gart_errors(true);
2516 amd_register_ecc_decoder(amd64_decode_bus_error
);
2520 atomic_inc(&drv_instances
);
2528 free_mc_sibling_devs(pvt
);
2537 static int __devinit
amd64_probe_one_instance(struct pci_dev
*pdev
,
2538 const struct pci_device_id
*mc_type
)
2540 u8 nid
= get_node_id(pdev
);
2541 struct pci_dev
*F3
= node_to_amd_nb(nid
)->misc
;
2542 struct ecc_settings
*s
;
2545 ret
= pci_enable_device(pdev
);
2547 debugf0("ret=%d\n", ret
);
2552 s
= kzalloc(sizeof(struct ecc_settings
), GFP_KERNEL
);
2558 if (!ecc_enabled(F3
, nid
)) {
2561 if (!ecc_enable_override
)
2564 amd64_warn("Forcing ECC on!\n");
2566 if (!enable_ecc_error_reporting(s
, nid
, F3
))
2570 ret
= amd64_init_one_instance(pdev
);
2572 amd64_err("Error probing instance: %d\n", nid
);
2573 restore_ecc_error_reporting(s
, nid
, F3
);
2580 ecc_stngs
[nid
] = NULL
;
2586 static void __devexit
amd64_remove_one_instance(struct pci_dev
*pdev
)
2588 struct mem_ctl_info
*mci
;
2589 struct amd64_pvt
*pvt
;
2590 u8 nid
= get_node_id(pdev
);
2591 struct pci_dev
*F3
= node_to_amd_nb(nid
)->misc
;
2592 struct ecc_settings
*s
= ecc_stngs
[nid
];
2594 /* Remove from EDAC CORE tracking list */
2595 mci
= edac_mc_del_mc(&pdev
->dev
);
2599 pvt
= mci
->pvt_info
;
2601 restore_ecc_error_reporting(s
, nid
, F3
);
2603 free_mc_sibling_devs(pvt
);
2605 /* unregister from EDAC MCE */
2606 amd_report_gart_errors(false);
2607 amd_unregister_ecc_decoder(amd64_decode_bus_error
);
2609 kfree(ecc_stngs
[nid
]);
2610 ecc_stngs
[nid
] = NULL
;
2612 /* Free the EDAC CORE resources */
2613 mci
->pvt_info
= NULL
;
2621 * This table is part of the interface for loading drivers for PCI devices. The
2622 * PCI core identifies what devices are on a system during boot, and then
2623 * inquiry this table to see if this driver is for a given device found.
2625 static const struct pci_device_id amd64_pci_table
[] __devinitdata
= {
2627 .vendor
= PCI_VENDOR_ID_AMD
,
2628 .device
= PCI_DEVICE_ID_AMD_K8_NB_MEMCTL
,
2629 .subvendor
= PCI_ANY_ID
,
2630 .subdevice
= PCI_ANY_ID
,
2635 .vendor
= PCI_VENDOR_ID_AMD
,
2636 .device
= PCI_DEVICE_ID_AMD_10H_NB_DRAM
,
2637 .subvendor
= PCI_ANY_ID
,
2638 .subdevice
= PCI_ANY_ID
,
2643 .vendor
= PCI_VENDOR_ID_AMD
,
2644 .device
= PCI_DEVICE_ID_AMD_15H_NB_F2
,
2645 .subvendor
= PCI_ANY_ID
,
2646 .subdevice
= PCI_ANY_ID
,
2653 MODULE_DEVICE_TABLE(pci
, amd64_pci_table
);
2655 static struct pci_driver amd64_pci_driver
= {
2656 .name
= EDAC_MOD_STR
,
2657 .probe
= amd64_probe_one_instance
,
2658 .remove
= __devexit_p(amd64_remove_one_instance
),
2659 .id_table
= amd64_pci_table
,
2662 static void setup_pci_device(void)
2664 struct mem_ctl_info
*mci
;
2665 struct amd64_pvt
*pvt
;
2673 pvt
= mci
->pvt_info
;
2675 edac_pci_create_generic_ctl(&pvt
->F2
->dev
, EDAC_MOD_STR
);
2677 if (!amd64_ctl_pci
) {
2678 pr_warning("%s(): Unable to create PCI control\n",
2681 pr_warning("%s(): PCI error report via EDAC not set\n",
2687 static int __init
amd64_edac_init(void)
2691 printk(KERN_INFO
"AMD64 EDAC driver v%s\n", EDAC_AMD64_VERSION
);
2695 if (amd_cache_northbridges() < 0)
2699 mcis
= kzalloc(amd_nb_num() * sizeof(mcis
[0]), GFP_KERNEL
);
2700 ecc_stngs
= kzalloc(amd_nb_num() * sizeof(ecc_stngs
[0]), GFP_KERNEL
);
2701 if (!(mcis
&& ecc_stngs
))
2704 msrs
= msrs_alloc();
2708 err
= pci_register_driver(&amd64_pci_driver
);
2713 if (!atomic_read(&drv_instances
))
2714 goto err_no_instances
;
2720 pci_unregister_driver(&amd64_pci_driver
);
2737 static void __exit
amd64_edac_exit(void)
2740 edac_pci_release_generic_ctl(amd64_ctl_pci
);
2742 pci_unregister_driver(&amd64_pci_driver
);
2754 module_init(amd64_edac_init
);
2755 module_exit(amd64_edac_exit
);
2757 MODULE_LICENSE("GPL");
2758 MODULE_AUTHOR("SoftwareBitMaker: Doug Thompson, "
2759 "Dave Peterson, Thayne Harbaugh");
2760 MODULE_DESCRIPTION("MC support for AMD64 memory controllers - "
2761 EDAC_AMD64_VERSION
);
2763 module_param(edac_op_state
, int, 0444);
2764 MODULE_PARM_DESC(edac_op_state
, "EDAC Error Reporting state: 0=Poll,1=NMI");