1 #include "amd64_edac.h"
2 #include <asm/amd_nb.h>
4 static struct edac_pci_ctl_info *amd64_ctl_pci;
6 static int report_gart_errors;
7 module_param(report_gart_errors, int, 0644);
10 * Set by command line parameter. If BIOS has enabled the ECC, this override is
11 * cleared to prevent re-enabling the hardware by this driver.
13 static int ecc_enable_override;
14 module_param(ecc_enable_override, int, 0644);
16 static struct msr __percpu *msrs;
19 * count successfully initialized driver instances for setup_pci_device()
21 static atomic_t drv_instances = ATOMIC_INIT(0);
23 /* Per-node driver instances */
24 static struct mem_ctl_info **mcis;
25 static struct ecc_settings **ecc_stngs;
28 * Valid scrub rates for the K8 hardware memory scrubber. We map the scrubbing
29 * bandwidth to a valid bit pattern. The 'set' operation finds the 'matching-
32 *FIXME: Produce a better mapping/linearisation.
34 static const struct scrubrate {
35 u32 scrubval; /* bit pattern for scrub rate */
36 u32 bandwidth; /* bandwidth consumed (bytes/sec) */
38 { 0x01, 1600000000UL},
60 { 0x00, 0UL}, /* scrubbing off */
63 int __amd64_read_pci_cfg_dword(struct pci_dev *pdev, int offset,
64 u32 *val, const char *func)
68 err = pci_read_config_dword(pdev, offset, val);
70 amd64_warn("%s: error reading F%dx%03x.\n",
71 func, PCI_FUNC(pdev->devfn), offset);
76 int __amd64_write_pci_cfg_dword(struct pci_dev *pdev, int offset,
77 u32 val, const char *func)
81 err = pci_write_config_dword(pdev, offset, val);
83 amd64_warn("%s: error writing to F%dx%03x.\n",
84 func, PCI_FUNC(pdev->devfn), offset);
91 * Depending on the family, F2 DCT reads need special handling:
93 * K8: has a single DCT only
95 * F10h: each DCT has its own set of regs
99 * F15h: we select which DCT we access using F1x10C[DctCfgSel]
101 * F16h: has only 1 DCT
103 static int k8_read_dct_pci_cfg(struct amd64_pvt *pvt, int addr, u32 *val,
109 return __amd64_read_pci_cfg_dword(pvt->F2, addr, val, func);
112 static int f10_read_dct_pci_cfg(struct amd64_pvt *pvt, int addr, u32 *val,
115 return __amd64_read_pci_cfg_dword(pvt->F2, addr, val, func);
119 * Select DCT to which PCI cfg accesses are routed
121 static void f15h_select_dct(struct amd64_pvt *pvt, u8 dct)
125 amd64_read_pci_cfg(pvt->F1, DCT_CFG_SEL, ®);
126 reg &= (pvt->model >= 0x30) ? ~3 : ~1;
128 amd64_write_pci_cfg(pvt->F1, DCT_CFG_SEL, reg);
131 static int f15_read_dct_pci_cfg(struct amd64_pvt *pvt, int addr, u32 *val,
136 /* For F15 M30h, the second dct is DCT 3, refer to BKDG Section 2.10 */
137 if (addr >= 0x140 && addr <= 0x1a0) {
138 dct = (pvt->model >= 0x30) ? 3 : 1;
142 f15h_select_dct(pvt, dct);
144 return __amd64_read_pci_cfg_dword(pvt->F2, addr, val, func);
148 * Memory scrubber control interface. For K8, memory scrubbing is handled by
149 * hardware and can involve L2 cache, dcache as well as the main memory. With
150 * F10, this is extended to L3 cache scrubbing on CPU models sporting that
153 * This causes the "units" for the scrubbing speed to vary from 64 byte blocks
154 * (dram) over to cache lines. This is nasty, so we will use bandwidth in
155 * bytes/sec for the setting.
157 * Currently, we only do dram scrubbing. If the scrubbing is done in software on
158 * other archs, we might not have access to the caches directly.
162 * scan the scrub rate mapping table for a close or matching bandwidth value to
163 * issue. If requested is too big, then use last maximum value found.
165 static int __amd64_set_scrub_rate(struct pci_dev *ctl, u32 new_bw, u32 min_rate)
171 * map the configured rate (new_bw) to a value specific to the AMD64
172 * memory controller and apply to register. Search for the first
173 * bandwidth entry that is greater or equal than the setting requested
174 * and program that. If at last entry, turn off DRAM scrubbing.
176 * If no suitable bandwidth is found, turn off DRAM scrubbing entirely
177 * by falling back to the last element in scrubrates[].
179 for (i = 0; i < ARRAY_SIZE(scrubrates) - 1; i++) {
181 * skip scrub rates which aren't recommended
182 * (see F10 BKDG, F3x58)
184 if (scrubrates[i].scrubval < min_rate)
187 if (scrubrates[i].bandwidth <= new_bw)
191 scrubval = scrubrates[i].scrubval;
193 pci_write_bits32(ctl, SCRCTRL, scrubval, 0x001F);
196 return scrubrates[i].bandwidth;
201 static int amd64_set_scrub_rate(struct mem_ctl_info *mci, u32 bw)
203 struct amd64_pvt *pvt = mci->pvt_info;
204 u32 min_scrubrate = 0x5;
206 if (boot_cpu_data.x86 == 0xf)
209 /* Erratum #505 for F15h Model 0x00 - Model 0x01, Stepping 0 */
210 if (boot_cpu_data.x86 == 0x15 &&
211 boot_cpu_data.x86_model <= 0x01 &&
212 boot_cpu_data.x86_mask < 0x1)
213 f15h_select_dct(pvt, 0);
215 return __amd64_set_scrub_rate(pvt->F3, bw, min_scrubrate);
218 static int amd64_get_scrub_rate(struct mem_ctl_info *mci)
220 struct amd64_pvt *pvt = mci->pvt_info;
222 int i, retval = -EINVAL;
224 /* Erratum #505 for F15h Model 0x00 - Model 0x01, Stepping 0 */
225 if (boot_cpu_data.x86 == 0x15 &&
226 boot_cpu_data.x86_model <= 0x01 &&
227 boot_cpu_data.x86_mask < 0x1)
228 f15h_select_dct(pvt, 0);
230 amd64_read_pci_cfg(pvt->F3, SCRCTRL, &scrubval);
232 scrubval = scrubval & 0x001F;
234 for (i = 0; i < ARRAY_SIZE(scrubrates); i++) {
235 if (scrubrates[i].scrubval == scrubval) {
236 retval = scrubrates[i].bandwidth;
244 * returns true if the SysAddr given by sys_addr matches the
245 * DRAM base/limit associated with node_id
247 static bool amd64_base_limit_match(struct amd64_pvt *pvt, u64 sys_addr,
252 /* The K8 treats this as a 40-bit value. However, bits 63-40 will be
253 * all ones if the most significant implemented address bit is 1.
254 * Here we discard bits 63-40. See section 3.4.2 of AMD publication
255 * 24592: AMD x86-64 Architecture Programmer's Manual Volume 1
256 * Application Programming.
258 addr = sys_addr & 0x000000ffffffffffull;
260 return ((addr >= get_dram_base(pvt, nid)) &&
261 (addr <= get_dram_limit(pvt, nid)));
265 * Attempt to map a SysAddr to a node. On success, return a pointer to the
266 * mem_ctl_info structure for the node that the SysAddr maps to.
268 * On failure, return NULL.
270 static struct mem_ctl_info *find_mc_by_sys_addr(struct mem_ctl_info *mci,
273 struct amd64_pvt *pvt;
278 * Here we use the DRAM Base (section 3.4.4.1) and DRAM Limit (section
279 * 3.4.4.2) registers to map the SysAddr to a node ID.
284 * The value of this field should be the same for all DRAM Base
285 * registers. Therefore we arbitrarily choose to read it from the
286 * register for node 0.
288 intlv_en = dram_intlv_en(pvt, 0);
291 for (node_id = 0; node_id < DRAM_RANGES; node_id++) {
292 if (amd64_base_limit_match(pvt, sys_addr, node_id))
298 if (unlikely((intlv_en != 0x01) &&
299 (intlv_en != 0x03) &&
300 (intlv_en != 0x07))) {
301 amd64_warn("DRAM Base[IntlvEn] junk value: 0x%x, BIOS bug?\n", intlv_en);
305 bits = (((u32) sys_addr) >> 12) & intlv_en;
307 for (node_id = 0; ; ) {
308 if ((dram_intlv_sel(pvt, node_id) & intlv_en) == bits)
309 break; /* intlv_sel field matches */
311 if (++node_id >= DRAM_RANGES)
315 /* sanity test for sys_addr */
316 if (unlikely(!amd64_base_limit_match(pvt, sys_addr, node_id))) {
317 amd64_warn("%s: sys_addr 0x%llx falls outside base/limit address"
318 "range for node %d with node interleaving enabled.\n",
319 __func__, sys_addr, node_id);
324 return edac_mc_find((int)node_id);
327 edac_dbg(2, "sys_addr 0x%lx doesn't match any node\n",
328 (unsigned long)sys_addr);
334 * compute the CS base address of the @csrow on the DRAM controller @dct.
335 * For details see F2x[5C:40] in the processor's BKDG
337 static void get_cs_base_and_mask(struct amd64_pvt *pvt, int csrow, u8 dct,
338 u64 *base, u64 *mask)
340 u64 csbase, csmask, base_bits, mask_bits;
343 if (pvt->fam == 0xf && pvt->ext_model < K8_REV_F) {
344 csbase = pvt->csels[dct].csbases[csrow];
345 csmask = pvt->csels[dct].csmasks[csrow];
346 base_bits = GENMASK(21, 31) | GENMASK(9, 15);
347 mask_bits = GENMASK(21, 29) | GENMASK(9, 15);
351 * F16h and F15h, models 30h and later need two addr_shift values:
352 * 8 for high and 6 for low (cf. F16h BKDG).
354 } else if (pvt->fam == 0x16 ||
355 (pvt->fam == 0x15 && pvt->model >= 0x30)) {
356 csbase = pvt->csels[dct].csbases[csrow];
357 csmask = pvt->csels[dct].csmasks[csrow >> 1];
359 *base = (csbase & GENMASK(5, 15)) << 6;
360 *base |= (csbase & GENMASK(19, 30)) << 8;
363 /* poke holes for the csmask */
364 *mask &= ~((GENMASK(5, 15) << 6) |
365 (GENMASK(19, 30) << 8));
367 *mask |= (csmask & GENMASK(5, 15)) << 6;
368 *mask |= (csmask & GENMASK(19, 30)) << 8;
372 csbase = pvt->csels[dct].csbases[csrow];
373 csmask = pvt->csels[dct].csmasks[csrow >> 1];
376 if (boot_cpu_data.x86 == 0x15)
377 base_bits = mask_bits = GENMASK(19,30) | GENMASK(5,13);
379 base_bits = mask_bits = GENMASK(19,28) | GENMASK(5,13);
382 *base = (csbase & base_bits) << addr_shift;
385 /* poke holes for the csmask */
386 *mask &= ~(mask_bits << addr_shift);
388 *mask |= (csmask & mask_bits) << addr_shift;
391 #define for_each_chip_select(i, dct, pvt) \
392 for (i = 0; i < pvt->csels[dct].b_cnt; i++)
394 #define chip_select_base(i, dct, pvt) \
395 pvt->csels[dct].csbases[i]
397 #define for_each_chip_select_mask(i, dct, pvt) \
398 for (i = 0; i < pvt->csels[dct].m_cnt; i++)
401 * @input_addr is an InputAddr associated with the node given by mci. Return the
402 * csrow that input_addr maps to, or -1 on failure (no csrow claims input_addr).
404 static int input_addr_to_csrow(struct mem_ctl_info *mci, u64 input_addr)
406 struct amd64_pvt *pvt;
412 for_each_chip_select(csrow, 0, pvt) {
413 if (!csrow_enabled(csrow, 0, pvt))
416 get_cs_base_and_mask(pvt, csrow, 0, &base, &mask);
420 if ((input_addr & mask) == (base & mask)) {
421 edac_dbg(2, "InputAddr 0x%lx matches csrow %d (node %d)\n",
422 (unsigned long)input_addr, csrow,
428 edac_dbg(2, "no matching csrow for InputAddr 0x%lx (MC node %d)\n",
429 (unsigned long)input_addr, pvt->mc_node_id);
435 * Obtain info from the DRAM Hole Address Register (section 3.4.8, pub #26094)
436 * for the node represented by mci. Info is passed back in *hole_base,
437 * *hole_offset, and *hole_size. Function returns 0 if info is valid or 1 if
438 * info is invalid. Info may be invalid for either of the following reasons:
440 * - The revision of the node is not E or greater. In this case, the DRAM Hole
441 * Address Register does not exist.
443 * - The DramHoleValid bit is cleared in the DRAM Hole Address Register,
444 * indicating that its contents are not valid.
446 * The values passed back in *hole_base, *hole_offset, and *hole_size are
447 * complete 32-bit values despite the fact that the bitfields in the DHAR
448 * only represent bits 31-24 of the base and offset values.
450 int amd64_get_dram_hole_info(struct mem_ctl_info *mci, u64 *hole_base,
451 u64 *hole_offset, u64 *hole_size)
453 struct amd64_pvt *pvt = mci->pvt_info;
455 /* only revE and later have the DRAM Hole Address Register */
456 if (boot_cpu_data.x86 == 0xf && pvt->ext_model < K8_REV_E) {
457 edac_dbg(1, " revision %d for node %d does not support DHAR\n",
458 pvt->ext_model, pvt->mc_node_id);
462 /* valid for Fam10h and above */
463 if (boot_cpu_data.x86 >= 0x10 && !dhar_mem_hoist_valid(pvt)) {
464 edac_dbg(1, " Dram Memory Hoisting is DISABLED on this system\n");
468 if (!dhar_valid(pvt)) {
469 edac_dbg(1, " Dram Memory Hoisting is DISABLED on this node %d\n",
474 /* This node has Memory Hoisting */
476 /* +------------------+--------------------+--------------------+-----
477 * | memory | DRAM hole | relocated |
478 * | [0, (x - 1)] | [x, 0xffffffff] | addresses from |
480 * | | | [0x100000000, |
481 * | | | (0x100000000+ |
482 * | | | (0xffffffff-x))] |
483 * +------------------+--------------------+--------------------+-----
485 * Above is a diagram of physical memory showing the DRAM hole and the
486 * relocated addresses from the DRAM hole. As shown, the DRAM hole
487 * starts at address x (the base address) and extends through address
488 * 0xffffffff. The DRAM Hole Address Register (DHAR) relocates the
489 * addresses in the hole so that they start at 0x100000000.
492 *hole_base = dhar_base(pvt);
493 *hole_size = (1ULL << 32) - *hole_base;
495 if (boot_cpu_data.x86 > 0xf)
496 *hole_offset = f10_dhar_offset(pvt);
498 *hole_offset = k8_dhar_offset(pvt);
500 edac_dbg(1, " DHAR info for node %d base 0x%lx offset 0x%lx size 0x%lx\n",
501 pvt->mc_node_id, (unsigned long)*hole_base,
502 (unsigned long)*hole_offset, (unsigned long)*hole_size);
506 EXPORT_SYMBOL_GPL(amd64_get_dram_hole_info);
509 * Return the DramAddr that the SysAddr given by @sys_addr maps to. It is
510 * assumed that sys_addr maps to the node given by mci.
512 * The first part of section 3.4.4 (p. 70) shows how the DRAM Base (section
513 * 3.4.4.1) and DRAM Limit (section 3.4.4.2) registers are used to translate a
514 * SysAddr to a DramAddr. If the DRAM Hole Address Register (DHAR) is enabled,
515 * then it is also involved in translating a SysAddr to a DramAddr. Sections
516 * 3.4.8 and 3.5.8.2 describe the DHAR and how it is used for memory hoisting.
517 * These parts of the documentation are unclear. I interpret them as follows:
519 * When node n receives a SysAddr, it processes the SysAddr as follows:
521 * 1. It extracts the DRAMBase and DRAMLimit values from the DRAM Base and DRAM
522 * Limit registers for node n. If the SysAddr is not within the range
523 * specified by the base and limit values, then node n ignores the Sysaddr
524 * (since it does not map to node n). Otherwise continue to step 2 below.
526 * 2. If the DramHoleValid bit of the DHAR for node n is clear, the DHAR is
527 * disabled so skip to step 3 below. Otherwise see if the SysAddr is within
528 * the range of relocated addresses (starting at 0x100000000) from the DRAM
529 * hole. If not, skip to step 3 below. Else get the value of the
530 * DramHoleOffset field from the DHAR. To obtain the DramAddr, subtract the
531 * offset defined by this value from the SysAddr.
533 * 3. Obtain the base address for node n from the DRAMBase field of the DRAM
534 * Base register for node n. To obtain the DramAddr, subtract the base
535 * address from the SysAddr, as shown near the start of section 3.4.4 (p.70).
537 static u64 sys_addr_to_dram_addr(struct mem_ctl_info *mci, u64 sys_addr)
539 struct amd64_pvt *pvt = mci->pvt_info;
540 u64 dram_base, hole_base, hole_offset, hole_size, dram_addr;
543 dram_base = get_dram_base(pvt, pvt->mc_node_id);
545 ret = amd64_get_dram_hole_info(mci, &hole_base, &hole_offset,
548 if ((sys_addr >= (1ULL << 32)) &&
549 (sys_addr < ((1ULL << 32) + hole_size))) {
550 /* use DHAR to translate SysAddr to DramAddr */
551 dram_addr = sys_addr - hole_offset;
553 edac_dbg(2, "using DHAR to translate SysAddr 0x%lx to DramAddr 0x%lx\n",
554 (unsigned long)sys_addr,
555 (unsigned long)dram_addr);
562 * Translate the SysAddr to a DramAddr as shown near the start of
563 * section 3.4.4 (p. 70). Although sys_addr is a 64-bit value, the k8
564 * only deals with 40-bit values. Therefore we discard bits 63-40 of
565 * sys_addr below. If bit 39 of sys_addr is 1 then the bits we
566 * discard are all 1s. Otherwise the bits we discard are all 0s. See
567 * section 3.4.2 of AMD publication 24592: AMD x86-64 Architecture
568 * Programmer's Manual Volume 1 Application Programming.
570 dram_addr = (sys_addr & GENMASK(0, 39)) - dram_base;
572 edac_dbg(2, "using DRAM Base register to translate SysAddr 0x%lx to DramAddr 0x%lx\n",
573 (unsigned long)sys_addr, (unsigned long)dram_addr);
578 * @intlv_en is the value of the IntlvEn field from a DRAM Base register
579 * (section 3.4.4.1). Return the number of bits from a SysAddr that are used
580 * for node interleaving.
582 static int num_node_interleave_bits(unsigned intlv_en)
584 static const int intlv_shift_table[] = { 0, 1, 0, 2, 0, 0, 0, 3 };
587 BUG_ON(intlv_en > 7);
588 n = intlv_shift_table[intlv_en];
592 /* Translate the DramAddr given by @dram_addr to an InputAddr. */
593 static u64 dram_addr_to_input_addr(struct mem_ctl_info *mci, u64 dram_addr)
595 struct amd64_pvt *pvt;
602 * See the start of section 3.4.4 (p. 70, BKDG #26094, K8, revA-E)
603 * concerning translating a DramAddr to an InputAddr.
605 intlv_shift = num_node_interleave_bits(dram_intlv_en(pvt, 0));
606 input_addr = ((dram_addr >> intlv_shift) & GENMASK(12, 35)) +
609 edac_dbg(2, " Intlv Shift=%d DramAddr=0x%lx maps to InputAddr=0x%lx\n",
610 intlv_shift, (unsigned long)dram_addr,
611 (unsigned long)input_addr);
617 * Translate the SysAddr represented by @sys_addr to an InputAddr. It is
618 * assumed that @sys_addr maps to the node given by mci.
620 static u64 sys_addr_to_input_addr(struct mem_ctl_info *mci, u64 sys_addr)
625 dram_addr_to_input_addr(mci, sys_addr_to_dram_addr(mci, sys_addr));
627 edac_dbg(2, "SysAdddr 0x%lx translates to InputAddr 0x%lx\n",
628 (unsigned long)sys_addr, (unsigned long)input_addr);
633 /* Map the Error address to a PAGE and PAGE OFFSET. */
634 static inline void error_address_to_page_and_offset(u64 error_address,
635 struct err_info *err)
637 err->page = (u32) (error_address >> PAGE_SHIFT);
638 err->offset = ((u32) error_address) & ~PAGE_MASK;
642 * @sys_addr is an error address (a SysAddr) extracted from the MCA NB Address
643 * Low (section 3.6.4.5) and MCA NB Address High (section 3.6.4.6) registers
644 * of a node that detected an ECC memory error. mci represents the node that
645 * the error address maps to (possibly different from the node that detected
646 * the error). Return the number of the csrow that sys_addr maps to, or -1 on
649 static int sys_addr_to_csrow(struct mem_ctl_info *mci, u64 sys_addr)
653 csrow = input_addr_to_csrow(mci, sys_addr_to_input_addr(mci, sys_addr));
656 amd64_mc_err(mci, "Failed to translate InputAddr to csrow for "
657 "address 0x%lx\n", (unsigned long)sys_addr);
661 static int get_channel_from_ecc_syndrome(struct mem_ctl_info *, u16);
664 * Determine if the DIMMs have ECC enabled. ECC is enabled ONLY if all the DIMMs
667 static unsigned long amd64_determine_edac_cap(struct amd64_pvt *pvt)
670 unsigned long edac_cap = EDAC_FLAG_NONE;
672 bit = (boot_cpu_data.x86 > 0xf || pvt->ext_model >= K8_REV_F)
676 if (pvt->dclr0 & BIT(bit))
677 edac_cap = EDAC_FLAG_SECDED;
682 static void amd64_debug_display_dimm_sizes(struct amd64_pvt *, u8);
684 static void amd64_dump_dramcfg_low(u32 dclr, int chan)
686 edac_dbg(1, "F2x%d90 (DRAM Cfg Low): 0x%08x\n", chan, dclr);
688 edac_dbg(1, " DIMM type: %sbuffered; all DIMMs support ECC: %s\n",
689 (dclr & BIT(16)) ? "un" : "",
690 (dclr & BIT(19)) ? "yes" : "no");
692 edac_dbg(1, " PAR/ERR parity: %s\n",
693 (dclr & BIT(8)) ? "enabled" : "disabled");
695 if (boot_cpu_data.x86 == 0x10)
696 edac_dbg(1, " DCT 128bit mode width: %s\n",
697 (dclr & BIT(11)) ? "128b" : "64b");
699 edac_dbg(1, " x4 logical DIMMs present: L0: %s L1: %s L2: %s L3: %s\n",
700 (dclr & BIT(12)) ? "yes" : "no",
701 (dclr & BIT(13)) ? "yes" : "no",
702 (dclr & BIT(14)) ? "yes" : "no",
703 (dclr & BIT(15)) ? "yes" : "no");
706 /* Display and decode various NB registers for debug purposes. */
707 static void dump_misc_regs(struct amd64_pvt *pvt)
709 edac_dbg(1, "F3xE8 (NB Cap): 0x%08x\n", pvt->nbcap);
711 edac_dbg(1, " NB two channel DRAM capable: %s\n",
712 (pvt->nbcap & NBCAP_DCT_DUAL) ? "yes" : "no");
714 edac_dbg(1, " ECC capable: %s, ChipKill ECC capable: %s\n",
715 (pvt->nbcap & NBCAP_SECDED) ? "yes" : "no",
716 (pvt->nbcap & NBCAP_CHIPKILL) ? "yes" : "no");
718 amd64_dump_dramcfg_low(pvt->dclr0, 0);
720 edac_dbg(1, "F3xB0 (Online Spare): 0x%08x\n", pvt->online_spare);
722 edac_dbg(1, "F1xF0 (DRAM Hole Address): 0x%08x, base: 0x%08x, offset: 0x%08x\n",
723 pvt->dhar, dhar_base(pvt),
724 (boot_cpu_data.x86 == 0xf) ? k8_dhar_offset(pvt)
725 : f10_dhar_offset(pvt));
727 edac_dbg(1, " DramHoleValid: %s\n", dhar_valid(pvt) ? "yes" : "no");
729 amd64_debug_display_dimm_sizes(pvt, 0);
731 /* everything below this point is Fam10h and above */
732 if (boot_cpu_data.x86 == 0xf)
735 amd64_debug_display_dimm_sizes(pvt, 1);
737 amd64_info("using %s syndromes.\n", ((pvt->ecc_sym_sz == 8) ? "x8" : "x4"));
739 /* Only if NOT ganged does dclr1 have valid info */
740 if (!dct_ganging_enabled(pvt))
741 amd64_dump_dramcfg_low(pvt->dclr1, 1);
745 * See BKDG, F2x[1,0][5C:40], F2[1,0][6C:60]
747 static void prep_chip_selects(struct amd64_pvt *pvt)
749 if (pvt->fam == 0xf && pvt->ext_model < K8_REV_F) {
750 pvt->csels[0].b_cnt = pvt->csels[1].b_cnt = 8;
751 pvt->csels[0].m_cnt = pvt->csels[1].m_cnt = 8;
752 } else if (pvt->fam == 0x15 && pvt->model >= 0x30) {
753 pvt->csels[0].b_cnt = pvt->csels[1].b_cnt = 4;
754 pvt->csels[0].m_cnt = pvt->csels[1].m_cnt = 2;
756 pvt->csels[0].b_cnt = pvt->csels[1].b_cnt = 8;
757 pvt->csels[0].m_cnt = pvt->csels[1].m_cnt = 4;
762 * Function 2 Offset F10_DCSB0; read in the DCS Base and DCS Mask registers
764 static void read_dct_base_mask(struct amd64_pvt *pvt)
768 prep_chip_selects(pvt);
770 for_each_chip_select(cs, 0, pvt) {
771 int reg0 = DCSB0 + (cs * 4);
772 int reg1 = DCSB1 + (cs * 4);
773 u32 *base0 = &pvt->csels[0].csbases[cs];
774 u32 *base1 = &pvt->csels[1].csbases[cs];
776 if (!amd64_read_dct_pci_cfg(pvt, reg0, base0))
777 edac_dbg(0, " DCSB0[%d]=0x%08x reg: F2x%x\n",
780 if (boot_cpu_data.x86 == 0xf || dct_ganging_enabled(pvt))
783 if (!amd64_read_dct_pci_cfg(pvt, reg1, base1))
784 edac_dbg(0, " DCSB1[%d]=0x%08x reg: F2x%x\n",
788 for_each_chip_select_mask(cs, 0, pvt) {
789 int reg0 = DCSM0 + (cs * 4);
790 int reg1 = DCSM1 + (cs * 4);
791 u32 *mask0 = &pvt->csels[0].csmasks[cs];
792 u32 *mask1 = &pvt->csels[1].csmasks[cs];
794 if (!amd64_read_dct_pci_cfg(pvt, reg0, mask0))
795 edac_dbg(0, " DCSM0[%d]=0x%08x reg: F2x%x\n",
798 if (boot_cpu_data.x86 == 0xf || dct_ganging_enabled(pvt))
801 if (!amd64_read_dct_pci_cfg(pvt, reg1, mask1))
802 edac_dbg(0, " DCSM1[%d]=0x%08x reg: F2x%x\n",
807 static enum mem_type amd64_determine_memory_type(struct amd64_pvt *pvt, int cs)
811 /* F15h supports only DDR3 */
812 if (boot_cpu_data.x86 >= 0x15)
813 type = (pvt->dclr0 & BIT(16)) ? MEM_DDR3 : MEM_RDDR3;
814 else if (boot_cpu_data.x86 == 0x10 || pvt->ext_model >= K8_REV_F) {
815 if (pvt->dchr0 & DDR3_MODE)
816 type = (pvt->dclr0 & BIT(16)) ? MEM_DDR3 : MEM_RDDR3;
818 type = (pvt->dclr0 & BIT(16)) ? MEM_DDR2 : MEM_RDDR2;
820 type = (pvt->dclr0 & BIT(18)) ? MEM_DDR : MEM_RDDR;
823 amd64_info("CS%d: %s\n", cs, edac_mem_types[type]);
828 /* Get the number of DCT channels the memory controller is using. */
829 static int k8_early_channel_count(struct amd64_pvt *pvt)
833 if (pvt->ext_model >= K8_REV_F)
834 /* RevF (NPT) and later */
835 flag = pvt->dclr0 & WIDTH_128;
837 /* RevE and earlier */
838 flag = pvt->dclr0 & REVE_WIDTH_128;
843 return (flag) ? 2 : 1;
846 /* On F10h and later ErrAddr is MC4_ADDR[47:1] */
847 static u64 get_error_address(struct mce *m)
849 struct cpuinfo_x86 *c = &boot_cpu_data;
859 addr = m->addr & GENMASK(start_bit, end_bit);
862 * Erratum 637 workaround
864 if (c->x86 == 0x15) {
865 struct amd64_pvt *pvt;
866 u64 cc6_base, tmp_addr;
871 if ((addr & GENMASK(24, 47)) >> 24 != 0x00fdf7)
874 mce_nid = amd_get_nb_id(m->extcpu);
875 pvt = mcis[mce_nid]->pvt_info;
877 amd64_read_pci_cfg(pvt->F1, DRAM_LOCAL_NODE_LIM, &tmp);
878 intlv_en = tmp >> 21 & 0x7;
880 /* add [47:27] + 3 trailing bits */
881 cc6_base = (tmp & GENMASK(0, 20)) << 3;
883 /* reverse and add DramIntlvEn */
884 cc6_base |= intlv_en ^ 0x7;
890 return cc6_base | (addr & GENMASK(0, 23));
892 amd64_read_pci_cfg(pvt->F1, DRAM_LOCAL_NODE_BASE, &tmp);
895 tmp_addr = (addr & GENMASK(12, 23)) << __fls(intlv_en + 1);
897 /* OR DramIntlvSel into bits [14:12] */
898 tmp_addr |= (tmp & GENMASK(21, 23)) >> 9;
900 /* add remaining [11:0] bits from original MC4_ADDR */
901 tmp_addr |= addr & GENMASK(0, 11);
903 return cc6_base | tmp_addr;
909 static struct pci_dev *pci_get_related_function(unsigned int vendor,
911 struct pci_dev *related)
913 struct pci_dev *dev = NULL;
915 while ((dev = pci_get_device(vendor, device, dev))) {
916 if (pci_domain_nr(dev->bus) == pci_domain_nr(related->bus) &&
917 (dev->bus->number == related->bus->number) &&
918 (PCI_SLOT(dev->devfn) == PCI_SLOT(related->devfn)))
925 static void read_dram_base_limit_regs(struct amd64_pvt *pvt, unsigned range)
927 struct amd_northbridge *nb;
928 struct pci_dev *f1 = NULL;
929 unsigned int pci_func;
930 int off = range << 3;
933 amd64_read_pci_cfg(pvt->F1, DRAM_BASE_LO + off, &pvt->ranges[range].base.lo);
934 amd64_read_pci_cfg(pvt->F1, DRAM_LIMIT_LO + off, &pvt->ranges[range].lim.lo);
939 if (!dram_rw(pvt, range))
942 amd64_read_pci_cfg(pvt->F1, DRAM_BASE_HI + off, &pvt->ranges[range].base.hi);
943 amd64_read_pci_cfg(pvt->F1, DRAM_LIMIT_HI + off, &pvt->ranges[range].lim.hi);
945 /* F15h: factor in CC6 save area by reading dst node's limit reg */
946 if (pvt->fam != 0x15)
949 nb = node_to_amd_nb(dram_dst_node(pvt, range));
953 pci_func = (pvt->model == 0x30) ? PCI_DEVICE_ID_AMD_15H_M30H_NB_F1
954 : PCI_DEVICE_ID_AMD_15H_NB_F1;
956 f1 = pci_get_related_function(nb->misc->vendor, pci_func, nb->misc);
960 amd64_read_pci_cfg(f1, DRAM_LOCAL_NODE_LIM, &llim);
962 pvt->ranges[range].lim.lo &= GENMASK(0, 15);
965 pvt->ranges[range].lim.lo |= ((llim & 0x1fff) << 3 | 0x7) << 16;
967 pvt->ranges[range].lim.hi &= GENMASK(0, 7);
970 pvt->ranges[range].lim.hi |= llim >> 13;
975 static void k8_map_sysaddr_to_csrow(struct mem_ctl_info *mci, u64 sys_addr,
976 struct err_info *err)
978 struct amd64_pvt *pvt = mci->pvt_info;
980 error_address_to_page_and_offset(sys_addr, err);
983 * Find out which node the error address belongs to. This may be
984 * different from the node that detected the error.
986 err->src_mci = find_mc_by_sys_addr(mci, sys_addr);
988 amd64_mc_err(mci, "failed to map error addr 0x%lx to a node\n",
989 (unsigned long)sys_addr);
990 err->err_code = ERR_NODE;
994 /* Now map the sys_addr to a CSROW */
995 err->csrow = sys_addr_to_csrow(err->src_mci, sys_addr);
996 if (err->csrow < 0) {
997 err->err_code = ERR_CSROW;
1001 /* CHIPKILL enabled */
1002 if (pvt->nbcfg & NBCFG_CHIPKILL) {
1003 err->channel = get_channel_from_ecc_syndrome(mci, err->syndrome);
1004 if (err->channel < 0) {
1006 * Syndrome didn't map, so we don't know which of the
1007 * 2 DIMMs is in error. So we need to ID 'both' of them
1010 amd64_mc_warn(err->src_mci, "unknown syndrome 0x%04x - "
1011 "possible error reporting race\n",
1013 err->err_code = ERR_CHANNEL;
1018 * non-chipkill ecc mode
1020 * The k8 documentation is unclear about how to determine the
1021 * channel number when using non-chipkill memory. This method
1022 * was obtained from email communication with someone at AMD.
1023 * (Wish the email was placed in this comment - norsk)
1025 err->channel = ((sys_addr & BIT(3)) != 0);
1029 static int ddr2_cs_size(unsigned i, bool dct_width)
1035 else if (!(i & 0x1))
1038 shift = (i + 1) >> 1;
1040 return 128 << (shift + !!dct_width);
1043 static int k8_dbam_to_chip_select(struct amd64_pvt *pvt, u8 dct,
1046 u32 dclr = dct ? pvt->dclr1 : pvt->dclr0;
1048 if (pvt->ext_model >= K8_REV_F) {
1049 WARN_ON(cs_mode > 11);
1050 return ddr2_cs_size(cs_mode, dclr & WIDTH_128);
1052 else if (pvt->ext_model >= K8_REV_D) {
1054 WARN_ON(cs_mode > 10);
1057 * the below calculation, besides trying to win an obfuscated C
1058 * contest, maps cs_mode values to DIMM chip select sizes. The
1061 * cs_mode CS size (mb)
1062 * ======= ============
1075 * Basically, it calculates a value with which to shift the
1076 * smallest CS size of 32MB.
1078 * ddr[23]_cs_size have a similar purpose.
1080 diff = cs_mode/3 + (unsigned)(cs_mode > 5);
1082 return 32 << (cs_mode - diff);
1085 WARN_ON(cs_mode > 6);
1086 return 32 << cs_mode;
1091 * Get the number of DCT channels in use.
1094 * number of Memory Channels in operation
1096 * contents of the DCL0_LOW register
1098 static int f1x_early_channel_count(struct amd64_pvt *pvt)
1100 int i, j, channels = 0;
1102 /* On F10h, if we are in 128 bit mode, then we are using 2 channels */
1103 if (boot_cpu_data.x86 == 0x10 && (pvt->dclr0 & WIDTH_128))
1107 * Need to check if in unganged mode: In such, there are 2 channels,
1108 * but they are not in 128 bit mode and thus the above 'dclr0' status
1111 * Need to check DCT0[0] and DCT1[0] to see if only one of them has
1112 * their CSEnable bit on. If so, then SINGLE DIMM case.
1114 edac_dbg(0, "Data width is not 128 bits - need more decoding\n");
1117 * Check DRAM Bank Address Mapping values for each DIMM to see if there
1118 * is more than just one DIMM present in unganged mode. Need to check
1119 * both controllers since DIMMs can be placed in either one.
1121 for (i = 0; i < 2; i++) {
1122 u32 dbam = (i ? pvt->dbam1 : pvt->dbam0);
1124 for (j = 0; j < 4; j++) {
1125 if (DBAM_DIMM(j, dbam) > 0) {
1135 amd64_info("MCT channel count: %d\n", channels);
1140 static int ddr3_cs_size(unsigned i, bool dct_width)
1145 if (i == 0 || i == 3 || i == 4)
1151 else if (!(i & 0x1))
1154 shift = (i + 1) >> 1;
1157 cs_size = (128 * (1 << !!dct_width)) << shift;
1162 static int f10_dbam_to_chip_select(struct amd64_pvt *pvt, u8 dct,
1165 u32 dclr = dct ? pvt->dclr1 : pvt->dclr0;
1167 WARN_ON(cs_mode > 11);
1169 if (pvt->dchr0 & DDR3_MODE || pvt->dchr1 & DDR3_MODE)
1170 return ddr3_cs_size(cs_mode, dclr & WIDTH_128);
1172 return ddr2_cs_size(cs_mode, dclr & WIDTH_128);
1176 * F15h supports only 64bit DCT interfaces
1178 static int f15_dbam_to_chip_select(struct amd64_pvt *pvt, u8 dct,
1181 WARN_ON(cs_mode > 12);
1183 return ddr3_cs_size(cs_mode, false);
1187 * F16h and F15h model 30h have only limited cs_modes.
1189 static int f16_dbam_to_chip_select(struct amd64_pvt *pvt, u8 dct,
1192 WARN_ON(cs_mode > 12);
1194 if (cs_mode == 6 || cs_mode == 8 ||
1195 cs_mode == 9 || cs_mode == 12)
1198 return ddr3_cs_size(cs_mode, false);
1201 static void read_dram_ctl_register(struct amd64_pvt *pvt)
1204 if (boot_cpu_data.x86 == 0xf)
1207 if (!amd64_read_dct_pci_cfg(pvt, DCT_SEL_LO, &pvt->dct_sel_lo)) {
1208 edac_dbg(0, "F2x110 (DCTSelLow): 0x%08x, High range addrs at: 0x%x\n",
1209 pvt->dct_sel_lo, dct_sel_baseaddr(pvt));
1211 edac_dbg(0, " DCTs operate in %s mode\n",
1212 (dct_ganging_enabled(pvt) ? "ganged" : "unganged"));
1214 if (!dct_ganging_enabled(pvt))
1215 edac_dbg(0, " Address range split per DCT: %s\n",
1216 (dct_high_range_enabled(pvt) ? "yes" : "no"));
1218 edac_dbg(0, " data interleave for ECC: %s, DRAM cleared since last warm reset: %s\n",
1219 (dct_data_intlv_enabled(pvt) ? "enabled" : "disabled"),
1220 (dct_memory_cleared(pvt) ? "yes" : "no"));
1222 edac_dbg(0, " channel interleave: %s, "
1223 "interleave bits selector: 0x%x\n",
1224 (dct_interleave_enabled(pvt) ? "enabled" : "disabled"),
1225 dct_sel_interleave_addr(pvt));
1228 amd64_read_dct_pci_cfg(pvt, DCT_SEL_HI, &pvt->dct_sel_hi);
1232 * Determine channel (DCT) based on the interleaving mode (see F15h M30h BKDG,
1233 * 2.10.12 Memory Interleaving Modes).
1235 static u8 f15_m30h_determine_channel(struct amd64_pvt *pvt, u64 sys_addr,
1236 u8 intlv_en, int num_dcts_intlv,
1243 return (u8)(dct_sel);
1245 if (num_dcts_intlv == 2) {
1246 select = (sys_addr >> 8) & 0x3;
1247 channel = select ? 0x3 : 0;
1248 } else if (num_dcts_intlv == 4)
1249 channel = (sys_addr >> 8) & 0x7;
1255 * Determine channel (DCT) based on the interleaving mode: F10h BKDG, 2.8.9 Memory
1256 * Interleaving Modes.
1258 static u8 f1x_determine_channel(struct amd64_pvt *pvt, u64 sys_addr,
1259 bool hi_range_sel, u8 intlv_en)
1261 u8 dct_sel_high = (pvt->dct_sel_lo >> 1) & 1;
1263 if (dct_ganging_enabled(pvt))
1267 return dct_sel_high;
1270 * see F2x110[DctSelIntLvAddr] - channel interleave mode
1272 if (dct_interleave_enabled(pvt)) {
1273 u8 intlv_addr = dct_sel_interleave_addr(pvt);
1275 /* return DCT select function: 0=DCT0, 1=DCT1 */
1277 return sys_addr >> 6 & 1;
1279 if (intlv_addr & 0x2) {
1280 u8 shift = intlv_addr & 0x1 ? 9 : 6;
1281 u32 temp = hweight_long((u32) ((sys_addr >> 16) & 0x1F)) % 2;
1283 return ((sys_addr >> shift) & 1) ^ temp;
1286 return (sys_addr >> (12 + hweight8(intlv_en))) & 1;
1289 if (dct_high_range_enabled(pvt))
1290 return ~dct_sel_high & 1;
1295 /* Convert the sys_addr to the normalized DCT address */
1296 static u64 f1x_get_norm_dct_addr(struct amd64_pvt *pvt, u8 range,
1297 u64 sys_addr, bool hi_rng,
1298 u32 dct_sel_base_addr)
1301 u64 dram_base = get_dram_base(pvt, range);
1302 u64 hole_off = f10_dhar_offset(pvt);
1303 u64 dct_sel_base_off = (pvt->dct_sel_hi & 0xFFFFFC00) << 16;
1308 * base address of high range is below 4Gb
1309 * (bits [47:27] at [31:11])
1310 * DRAM address space on this DCT is hoisted above 4Gb &&
1313 * remove hole offset from sys_addr
1315 * remove high range offset from sys_addr
1317 if ((!(dct_sel_base_addr >> 16) ||
1318 dct_sel_base_addr < dhar_base(pvt)) &&
1320 (sys_addr >= BIT_64(32)))
1321 chan_off = hole_off;
1323 chan_off = dct_sel_base_off;
1327 * we have a valid hole &&
1332 * remove dram base to normalize to DCT address
1334 if (dhar_valid(pvt) && (sys_addr >= BIT_64(32)))
1335 chan_off = hole_off;
1337 chan_off = dram_base;
1340 return (sys_addr & GENMASK(6,47)) - (chan_off & GENMASK(23,47));
1344 * checks if the csrow passed in is marked as SPARED, if so returns the new
1347 static int f10_process_possible_spare(struct amd64_pvt *pvt, u8 dct, int csrow)
1351 if (online_spare_swap_done(pvt, dct) &&
1352 csrow == online_spare_bad_dramcs(pvt, dct)) {
1354 for_each_chip_select(tmp_cs, dct, pvt) {
1355 if (chip_select_base(tmp_cs, dct, pvt) & 0x2) {
1365 * Iterate over the DRAM DCT "base" and "mask" registers looking for a
1366 * SystemAddr match on the specified 'ChannelSelect' and 'NodeID'
1369 * -EINVAL: NOT FOUND
1370 * 0..csrow = Chip-Select Row
1372 static int f1x_lookup_addr_in_dct(u64 in_addr, u8 nid, u8 dct)
1374 struct mem_ctl_info *mci;
1375 struct amd64_pvt *pvt;
1376 u64 cs_base, cs_mask;
1377 int cs_found = -EINVAL;
1384 pvt = mci->pvt_info;
1386 edac_dbg(1, "input addr: 0x%llx, DCT: %d\n", in_addr, dct);
1388 for_each_chip_select(csrow, dct, pvt) {
1389 if (!csrow_enabled(csrow, dct, pvt))
1392 get_cs_base_and_mask(pvt, csrow, dct, &cs_base, &cs_mask);
1394 edac_dbg(1, " CSROW=%d CSBase=0x%llx CSMask=0x%llx\n",
1395 csrow, cs_base, cs_mask);
1399 edac_dbg(1, " (InputAddr & ~CSMask)=0x%llx (CSBase & ~CSMask)=0x%llx\n",
1400 (in_addr & cs_mask), (cs_base & cs_mask));
1402 if ((in_addr & cs_mask) == (cs_base & cs_mask)) {
1403 if (pvt->fam == 0x15 && pvt->model >= 0x30) {
1407 cs_found = f10_process_possible_spare(pvt, dct, csrow);
1409 edac_dbg(1, " MATCH csrow=%d\n", cs_found);
1417 * See F2x10C. Non-interleaved graphics framebuffer memory under the 16G is
1418 * swapped with a region located at the bottom of memory so that the GPU can use
1419 * the interleaved region and thus two channels.
1421 static u64 f1x_swap_interleaved_region(struct amd64_pvt *pvt, u64 sys_addr)
1423 u32 swap_reg, swap_base, swap_limit, rgn_size, tmp_addr;
1425 if (boot_cpu_data.x86 == 0x10) {
1426 /* only revC3 and revE have that feature */
1427 if (boot_cpu_data.x86_model < 4 ||
1428 (boot_cpu_data.x86_model < 0xa &&
1429 boot_cpu_data.x86_mask < 3))
1433 amd64_read_dct_pci_cfg(pvt, SWAP_INTLV_REG, &swap_reg);
1435 if (!(swap_reg & 0x1))
1438 swap_base = (swap_reg >> 3) & 0x7f;
1439 swap_limit = (swap_reg >> 11) & 0x7f;
1440 rgn_size = (swap_reg >> 20) & 0x7f;
1441 tmp_addr = sys_addr >> 27;
1443 if (!(sys_addr >> 34) &&
1444 (((tmp_addr >= swap_base) &&
1445 (tmp_addr <= swap_limit)) ||
1446 (tmp_addr < rgn_size)))
1447 return sys_addr ^ (u64)swap_base << 27;
1452 /* For a given @dram_range, check if @sys_addr falls within it. */
1453 static int f1x_match_to_this_node(struct amd64_pvt *pvt, unsigned range,
1454 u64 sys_addr, int *chan_sel)
1456 int cs_found = -EINVAL;
1460 bool high_range = false;
1462 u8 node_id = dram_dst_node(pvt, range);
1463 u8 intlv_en = dram_intlv_en(pvt, range);
1464 u32 intlv_sel = dram_intlv_sel(pvt, range);
1466 edac_dbg(1, "(range %d) SystemAddr= 0x%llx Limit=0x%llx\n",
1467 range, sys_addr, get_dram_limit(pvt, range));
1469 if (dhar_valid(pvt) &&
1470 dhar_base(pvt) <= sys_addr &&
1471 sys_addr < BIT_64(32)) {
1472 amd64_warn("Huh? Address is in the MMIO hole: 0x%016llx\n",
1477 if (intlv_en && (intlv_sel != ((sys_addr >> 12) & intlv_en)))
1480 sys_addr = f1x_swap_interleaved_region(pvt, sys_addr);
1482 dct_sel_base = dct_sel_baseaddr(pvt);
1485 * check whether addresses >= DctSelBaseAddr[47:27] are to be used to
1486 * select between DCT0 and DCT1.
1488 if (dct_high_range_enabled(pvt) &&
1489 !dct_ganging_enabled(pvt) &&
1490 ((sys_addr >> 27) >= (dct_sel_base >> 11)))
1493 channel = f1x_determine_channel(pvt, sys_addr, high_range, intlv_en);
1495 chan_addr = f1x_get_norm_dct_addr(pvt, range, sys_addr,
1496 high_range, dct_sel_base);
1498 /* Remove node interleaving, see F1x120 */
1500 chan_addr = ((chan_addr >> (12 + hweight8(intlv_en))) << 12) |
1501 (chan_addr & 0xfff);
1503 /* remove channel interleave */
1504 if (dct_interleave_enabled(pvt) &&
1505 !dct_high_range_enabled(pvt) &&
1506 !dct_ganging_enabled(pvt)) {
1508 if (dct_sel_interleave_addr(pvt) != 1) {
1509 if (dct_sel_interleave_addr(pvt) == 0x3)
1511 chan_addr = ((chan_addr >> 10) << 9) |
1512 (chan_addr & 0x1ff);
1514 /* A[6] or hash 6 */
1515 chan_addr = ((chan_addr >> 7) << 6) |
1519 chan_addr = ((chan_addr >> 13) << 12) |
1520 (chan_addr & 0xfff);
1523 edac_dbg(1, " Normalized DCT addr: 0x%llx\n", chan_addr);
1525 cs_found = f1x_lookup_addr_in_dct(chan_addr, node_id, channel);
1528 *chan_sel = channel;
1533 static int f15_m30h_match_to_this_node(struct amd64_pvt *pvt, unsigned range,
1534 u64 sys_addr, int *chan_sel)
1536 int cs_found = -EINVAL;
1537 int num_dcts_intlv = 0;
1538 u64 chan_addr, chan_offset;
1539 u64 dct_base, dct_limit;
1540 u32 dct_cont_base_reg, dct_cont_limit_reg, tmp;
1541 u8 channel, alias_channel, leg_mmio_hole, dct_sel, dct_offset_en;
1543 u64 dhar_offset = f10_dhar_offset(pvt);
1544 u8 intlv_addr = dct_sel_interleave_addr(pvt);
1545 u8 node_id = dram_dst_node(pvt, range);
1546 u8 intlv_en = dram_intlv_en(pvt, range);
1548 amd64_read_pci_cfg(pvt->F1, DRAM_CONT_BASE, &dct_cont_base_reg);
1549 amd64_read_pci_cfg(pvt->F1, DRAM_CONT_LIMIT, &dct_cont_limit_reg);
1551 dct_offset_en = (u8) ((dct_cont_base_reg >> 3) & BIT(0));
1552 dct_sel = (u8) ((dct_cont_base_reg >> 4) & 0x7);
1554 edac_dbg(1, "(range %d) SystemAddr= 0x%llx Limit=0x%llx\n",
1555 range, sys_addr, get_dram_limit(pvt, range));
1557 if (!(get_dram_base(pvt, range) <= sys_addr) &&
1558 !(get_dram_limit(pvt, range) >= sys_addr))
1561 if (dhar_valid(pvt) &&
1562 dhar_base(pvt) <= sys_addr &&
1563 sys_addr < BIT_64(32)) {
1564 amd64_warn("Huh? Address is in the MMIO hole: 0x%016llx\n",
1569 /* Verify sys_addr is within DCT Range. */
1570 dct_base = (dct_sel_baseaddr(pvt) << 27);
1571 dct_limit = (((dct_cont_limit_reg >> 11) & 0x1FFF) << 27) | 0x7FFFFFF;
1573 if (!(dct_cont_base_reg & BIT(0)) &&
1574 !(dct_base <= sys_addr && dct_limit >= sys_addr))
1577 /* Verify number of dct's that participate in channel interleaving. */
1578 num_dcts_intlv = (int) hweight8(intlv_en);
1580 if (!(num_dcts_intlv % 2 == 0) || (num_dcts_intlv > 4))
1583 channel = f15_m30h_determine_channel(pvt, sys_addr, intlv_en,
1584 num_dcts_intlv, dct_sel);
1586 /* Verify we stay within the MAX number of channels allowed */
1587 if (channel > 4 || channel < 0)
1590 leg_mmio_hole = (u8) (dct_cont_base_reg >> 1 & BIT(0));
1592 /* Get normalized DCT addr */
1593 if (leg_mmio_hole && (sys_addr >= BIT_64(32)))
1594 chan_offset = dhar_offset;
1596 chan_offset = dct_base;
1598 chan_addr = sys_addr - chan_offset;
1600 /* remove channel interleave */
1601 if (num_dcts_intlv == 2) {
1602 if (intlv_addr == 0x4)
1603 chan_addr = ((chan_addr >> 9) << 8) |
1605 else if (intlv_addr == 0x5)
1606 chan_addr = ((chan_addr >> 10) << 9) |
1607 (chan_addr & 0x1ff);
1611 } else if (num_dcts_intlv == 4) {
1612 if (intlv_addr == 0x4)
1613 chan_addr = ((chan_addr >> 10) << 8) |
1615 else if (intlv_addr == 0x5)
1616 chan_addr = ((chan_addr >> 11) << 9) |
1617 (chan_addr & 0x1ff);
1622 if (dct_offset_en) {
1623 amd64_read_pci_cfg(pvt->F1,
1624 DRAM_CONT_HIGH_OFF + (int) channel * 4,
1626 chan_addr += ((tmp >> 11) & 0xfff) << 27;
1629 f15h_select_dct(pvt, channel);
1631 edac_dbg(1, " Normalized DCT addr: 0x%llx\n", chan_addr);
1635 * if channel = 3, then alias it to 1. This is because, in F15 M30h,
1636 * there is support for 4 DCT's, but only 2 are currently functional.
1637 * They are DCT0 and DCT3. But we have read all registers of DCT3 into
1638 * pvt->csels[1]. So we need to use '1' here to get correct info.
1639 * Refer F15 M30h BKDG Section 2.10 and 2.10.3 for clarifications.
1641 alias_channel = (channel == 3) ? 1 : channel;
1643 cs_found = f1x_lookup_addr_in_dct(chan_addr, node_id, alias_channel);
1646 *chan_sel = alias_channel;
1651 static int f1x_translate_sysaddr_to_cs(struct amd64_pvt *pvt,
1655 int cs_found = -EINVAL;
1658 for (range = 0; range < DRAM_RANGES; range++) {
1659 if (!dram_rw(pvt, range))
1662 if (pvt->fam == 0x15 && pvt->model >= 0x30)
1663 cs_found = f15_m30h_match_to_this_node(pvt, range,
1667 else if ((get_dram_base(pvt, range) <= sys_addr) &&
1668 (get_dram_limit(pvt, range) >= sys_addr)) {
1669 cs_found = f1x_match_to_this_node(pvt, range,
1670 sys_addr, chan_sel);
1679 * For reference see "2.8.5 Routing DRAM Requests" in F10 BKDG. This code maps
1680 * a @sys_addr to NodeID, DCT (channel) and chip select (CSROW).
1682 * The @sys_addr is usually an error address received from the hardware
1685 static void f1x_map_sysaddr_to_csrow(struct mem_ctl_info *mci, u64 sys_addr,
1686 struct err_info *err)
1688 struct amd64_pvt *pvt = mci->pvt_info;
1690 error_address_to_page_and_offset(sys_addr, err);
1692 err->csrow = f1x_translate_sysaddr_to_cs(pvt, sys_addr, &err->channel);
1693 if (err->csrow < 0) {
1694 err->err_code = ERR_CSROW;
1699 * We need the syndromes for channel detection only when we're
1700 * ganged. Otherwise @chan should already contain the channel at
1703 if (dct_ganging_enabled(pvt))
1704 err->channel = get_channel_from_ecc_syndrome(mci, err->syndrome);
1708 * debug routine to display the memory sizes of all logical DIMMs and its
1711 static void amd64_debug_display_dimm_sizes(struct amd64_pvt *pvt, u8 ctrl)
1713 int dimm, size0, size1;
1714 u32 *dcsb = ctrl ? pvt->csels[1].csbases : pvt->csels[0].csbases;
1715 u32 dbam = ctrl ? pvt->dbam1 : pvt->dbam0;
1717 if (boot_cpu_data.x86 == 0xf) {
1718 /* K8 families < revF not supported yet */
1719 if (pvt->ext_model < K8_REV_F)
1725 dbam = (ctrl && !dct_ganging_enabled(pvt)) ? pvt->dbam1 : pvt->dbam0;
1726 dcsb = (ctrl && !dct_ganging_enabled(pvt)) ? pvt->csels[1].csbases
1727 : pvt->csels[0].csbases;
1729 edac_dbg(1, "F2x%d80 (DRAM Bank Address Mapping): 0x%08x\n",
1732 edac_printk(KERN_DEBUG, EDAC_MC, "DCT%d chip selects:\n", ctrl);
1734 /* Dump memory sizes for DIMM and its CSROWs */
1735 for (dimm = 0; dimm < 4; dimm++) {
1738 if (dcsb[dimm*2] & DCSB_CS_ENABLE)
1739 size0 = pvt->ops->dbam_to_cs(pvt, ctrl,
1740 DBAM_DIMM(dimm, dbam));
1743 if (dcsb[dimm*2 + 1] & DCSB_CS_ENABLE)
1744 size1 = pvt->ops->dbam_to_cs(pvt, ctrl,
1745 DBAM_DIMM(dimm, dbam));
1747 amd64_info(EDAC_MC ": %d: %5dMB %d: %5dMB\n",
1749 dimm * 2 + 1, size1);
1753 static struct amd64_family_type amd64_family_types[] = {
1756 .f1_id = PCI_DEVICE_ID_AMD_K8_NB_ADDRMAP,
1757 .f3_id = PCI_DEVICE_ID_AMD_K8_NB_MISC,
1759 .early_channel_count = k8_early_channel_count,
1760 .map_sysaddr_to_csrow = k8_map_sysaddr_to_csrow,
1761 .dbam_to_cs = k8_dbam_to_chip_select,
1762 .read_dct_pci_cfg = k8_read_dct_pci_cfg,
1767 .f1_id = PCI_DEVICE_ID_AMD_10H_NB_MAP,
1768 .f3_id = PCI_DEVICE_ID_AMD_10H_NB_MISC,
1770 .early_channel_count = f1x_early_channel_count,
1771 .map_sysaddr_to_csrow = f1x_map_sysaddr_to_csrow,
1772 .dbam_to_cs = f10_dbam_to_chip_select,
1773 .read_dct_pci_cfg = f10_read_dct_pci_cfg,
1778 .f1_id = PCI_DEVICE_ID_AMD_15H_NB_F1,
1779 .f3_id = PCI_DEVICE_ID_AMD_15H_NB_F3,
1781 .early_channel_count = f1x_early_channel_count,
1782 .map_sysaddr_to_csrow = f1x_map_sysaddr_to_csrow,
1783 .dbam_to_cs = f15_dbam_to_chip_select,
1784 .read_dct_pci_cfg = f15_read_dct_pci_cfg,
1788 .ctl_name = "F15h_M30h",
1789 .f1_id = PCI_DEVICE_ID_AMD_15H_M30H_NB_F1,
1790 .f3_id = PCI_DEVICE_ID_AMD_15H_M30H_NB_F3,
1792 .early_channel_count = f1x_early_channel_count,
1793 .map_sysaddr_to_csrow = f1x_map_sysaddr_to_csrow,
1794 .dbam_to_cs = f16_dbam_to_chip_select,
1795 .read_dct_pci_cfg = f15_read_dct_pci_cfg,
1800 .f1_id = PCI_DEVICE_ID_AMD_16H_NB_F1,
1801 .f3_id = PCI_DEVICE_ID_AMD_16H_NB_F3,
1803 .early_channel_count = f1x_early_channel_count,
1804 .map_sysaddr_to_csrow = f1x_map_sysaddr_to_csrow,
1805 .dbam_to_cs = f16_dbam_to_chip_select,
1806 .read_dct_pci_cfg = f10_read_dct_pci_cfg,
1812 * These are tables of eigenvectors (one per line) which can be used for the
1813 * construction of the syndrome tables. The modified syndrome search algorithm
1814 * uses those to find the symbol in error and thus the DIMM.
1816 * Algorithm courtesy of Ross LaFetra from AMD.
1818 static const u16 x4_vectors[] = {
1819 0x2f57, 0x1afe, 0x66cc, 0xdd88,
1820 0x11eb, 0x3396, 0x7f4c, 0xeac8,
1821 0x0001, 0x0002, 0x0004, 0x0008,
1822 0x1013, 0x3032, 0x4044, 0x8088,
1823 0x106b, 0x30d6, 0x70fc, 0xe0a8,
1824 0x4857, 0xc4fe, 0x13cc, 0x3288,
1825 0x1ac5, 0x2f4a, 0x5394, 0xa1e8,
1826 0x1f39, 0x251e, 0xbd6c, 0x6bd8,
1827 0x15c1, 0x2a42, 0x89ac, 0x4758,
1828 0x2b03, 0x1602, 0x4f0c, 0xca08,
1829 0x1f07, 0x3a0e, 0x6b04, 0xbd08,
1830 0x8ba7, 0x465e, 0x244c, 0x1cc8,
1831 0x2b87, 0x164e, 0x642c, 0xdc18,
1832 0x40b9, 0x80de, 0x1094, 0x20e8,
1833 0x27db, 0x1eb6, 0x9dac, 0x7b58,
1834 0x11c1, 0x2242, 0x84ac, 0x4c58,
1835 0x1be5, 0x2d7a, 0x5e34, 0xa718,
1836 0x4b39, 0x8d1e, 0x14b4, 0x28d8,
1837 0x4c97, 0xc87e, 0x11fc, 0x33a8,
1838 0x8e97, 0x497e, 0x2ffc, 0x1aa8,
1839 0x16b3, 0x3d62, 0x4f34, 0x8518,
1840 0x1e2f, 0x391a, 0x5cac, 0xf858,
1841 0x1d9f, 0x3b7a, 0x572c, 0xfe18,
1842 0x15f5, 0x2a5a, 0x5264, 0xa3b8,
1843 0x1dbb, 0x3b66, 0x715c, 0xe3f8,
1844 0x4397, 0xc27e, 0x17fc, 0x3ea8,
1845 0x1617, 0x3d3e, 0x6464, 0xb8b8,
1846 0x23ff, 0x12aa, 0xab6c, 0x56d8,
1847 0x2dfb, 0x1ba6, 0x913c, 0x7328,
1848 0x185d, 0x2ca6, 0x7914, 0x9e28,
1849 0x171b, 0x3e36, 0x7d7c, 0xebe8,
1850 0x4199, 0x82ee, 0x19f4, 0x2e58,
1851 0x4807, 0xc40e, 0x130c, 0x3208,
1852 0x1905, 0x2e0a, 0x5804, 0xac08,
1853 0x213f, 0x132a, 0xadfc, 0x5ba8,
1854 0x19a9, 0x2efe, 0xb5cc, 0x6f88,
1857 static const u16 x8_vectors[] = {
1858 0x0145, 0x028a, 0x2374, 0x43c8, 0xa1f0, 0x0520, 0x0a40, 0x1480,
1859 0x0211, 0x0422, 0x0844, 0x1088, 0x01b0, 0x44e0, 0x23c0, 0xed80,
1860 0x1011, 0x0116, 0x022c, 0x0458, 0x08b0, 0x8c60, 0x2740, 0x4e80,
1861 0x0411, 0x0822, 0x1044, 0x0158, 0x02b0, 0x2360, 0x46c0, 0xab80,
1862 0x0811, 0x1022, 0x012c, 0x0258, 0x04b0, 0x4660, 0x8cc0, 0x2780,
1863 0x2071, 0x40e2, 0xa0c4, 0x0108, 0x0210, 0x0420, 0x0840, 0x1080,
1864 0x4071, 0x80e2, 0x0104, 0x0208, 0x0410, 0x0820, 0x1040, 0x2080,
1865 0x8071, 0x0102, 0x0204, 0x0408, 0x0810, 0x1020, 0x2040, 0x4080,
1866 0x019d, 0x03d6, 0x136c, 0x2198, 0x50b0, 0xb2e0, 0x0740, 0x0e80,
1867 0x0189, 0x03ea, 0x072c, 0x0e58, 0x1cb0, 0x56e0, 0x37c0, 0xf580,
1868 0x01fd, 0x0376, 0x06ec, 0x0bb8, 0x1110, 0x2220, 0x4440, 0x8880,
1869 0x0163, 0x02c6, 0x1104, 0x0758, 0x0eb0, 0x2be0, 0x6140, 0xc280,
1870 0x02fd, 0x01c6, 0x0b5c, 0x1108, 0x07b0, 0x25a0, 0x8840, 0x6180,
1871 0x0801, 0x012e, 0x025c, 0x04b8, 0x1370, 0x26e0, 0x57c0, 0xb580,
1872 0x0401, 0x0802, 0x015c, 0x02b8, 0x22b0, 0x13e0, 0x7140, 0xe280,
1873 0x0201, 0x0402, 0x0804, 0x01b8, 0x11b0, 0x31a0, 0x8040, 0x7180,
1874 0x0101, 0x0202, 0x0404, 0x0808, 0x1010, 0x2020, 0x4040, 0x8080,
1875 0x0001, 0x0002, 0x0004, 0x0008, 0x0010, 0x0020, 0x0040, 0x0080,
1876 0x0100, 0x0200, 0x0400, 0x0800, 0x1000, 0x2000, 0x4000, 0x8000,
1879 static int decode_syndrome(u16 syndrome, const u16 *vectors, unsigned num_vecs,
1882 unsigned int i, err_sym;
1884 for (err_sym = 0; err_sym < num_vecs / v_dim; err_sym++) {
1886 unsigned v_idx = err_sym * v_dim;
1887 unsigned v_end = (err_sym + 1) * v_dim;
1889 /* walk over all 16 bits of the syndrome */
1890 for (i = 1; i < (1U << 16); i <<= 1) {
1892 /* if bit is set in that eigenvector... */
1893 if (v_idx < v_end && vectors[v_idx] & i) {
1894 u16 ev_comp = vectors[v_idx++];
1896 /* ... and bit set in the modified syndrome, */
1906 /* can't get to zero, move to next symbol */
1911 edac_dbg(0, "syndrome(%x) not found\n", syndrome);
1915 static int map_err_sym_to_channel(int err_sym, int sym_size)
1928 return err_sym >> 4;
1934 /* imaginary bits not in a DIMM */
1936 WARN(1, KERN_ERR "Invalid error symbol: 0x%x\n",
1948 return err_sym >> 3;
1954 static int get_channel_from_ecc_syndrome(struct mem_ctl_info *mci, u16 syndrome)
1956 struct amd64_pvt *pvt = mci->pvt_info;
1959 if (pvt->ecc_sym_sz == 8)
1960 err_sym = decode_syndrome(syndrome, x8_vectors,
1961 ARRAY_SIZE(x8_vectors),
1963 else if (pvt->ecc_sym_sz == 4)
1964 err_sym = decode_syndrome(syndrome, x4_vectors,
1965 ARRAY_SIZE(x4_vectors),
1968 amd64_warn("Illegal syndrome type: %u\n", pvt->ecc_sym_sz);
1972 return map_err_sym_to_channel(err_sym, pvt->ecc_sym_sz);
1975 static void __log_bus_error(struct mem_ctl_info *mci, struct err_info *err,
1978 enum hw_event_mc_err_type err_type;
1982 err_type = HW_EVENT_ERR_CORRECTED;
1983 else if (ecc_type == 1)
1984 err_type = HW_EVENT_ERR_UNCORRECTED;
1986 WARN(1, "Something is rotten in the state of Denmark.\n");
1990 switch (err->err_code) {
1995 string = "Failed to map error addr to a node";
1998 string = "Failed to map error addr to a csrow";
2001 string = "unknown syndrome - possible error reporting race";
2004 string = "WTF error";
2008 edac_mc_handle_error(err_type, mci, 1,
2009 err->page, err->offset, err->syndrome,
2010 err->csrow, err->channel, -1,
2014 static inline void __amd64_decode_bus_error(struct mem_ctl_info *mci,
2017 struct amd64_pvt *pvt = mci->pvt_info;
2018 u8 ecc_type = (m->status >> 45) & 0x3;
2019 u8 xec = XEC(m->status, 0x1f);
2020 u16 ec = EC(m->status);
2022 struct err_info err;
2024 /* Bail out early if this was an 'observed' error */
2025 if (PP(ec) == NBSL_PP_OBS)
2028 /* Do only ECC errors */
2029 if (xec && xec != F10_NBSL_EXT_ERR_ECC)
2032 memset(&err, 0, sizeof(err));
2034 sys_addr = get_error_address(m);
2037 err.syndrome = extract_syndrome(m->status);
2039 pvt->ops->map_sysaddr_to_csrow(mci, sys_addr, &err);
2041 __log_bus_error(mci, &err, ecc_type);
2044 void amd64_decode_bus_error(int node_id, struct mce *m)
2046 __amd64_decode_bus_error(mcis[node_id], m);
2050 * Use pvt->F2 which contains the F2 CPU PCI device to get the related
2051 * F1 (AddrMap) and F3 (Misc) devices. Return negative value on error.
2053 static int reserve_mc_sibling_devs(struct amd64_pvt *pvt, u16 f1_id, u16 f3_id)
2055 /* Reserve the ADDRESS MAP Device */
2056 pvt->F1 = pci_get_related_function(pvt->F2->vendor, f1_id, pvt->F2);
2058 amd64_err("error address map device not found: "
2059 "vendor %x device 0x%x (broken BIOS?)\n",
2060 PCI_VENDOR_ID_AMD, f1_id);
2064 /* Reserve the MISC Device */
2065 pvt->F3 = pci_get_related_function(pvt->F2->vendor, f3_id, pvt->F2);
2067 pci_dev_put(pvt->F1);
2070 amd64_err("error F3 device not found: "
2071 "vendor %x device 0x%x (broken BIOS?)\n",
2072 PCI_VENDOR_ID_AMD, f3_id);
2076 edac_dbg(1, "F1: %s\n", pci_name(pvt->F1));
2077 edac_dbg(1, "F2: %s\n", pci_name(pvt->F2));
2078 edac_dbg(1, "F3: %s\n", pci_name(pvt->F3));
2083 static void free_mc_sibling_devs(struct amd64_pvt *pvt)
2085 pci_dev_put(pvt->F1);
2086 pci_dev_put(pvt->F3);
2090 * Retrieve the hardware registers of the memory controller (this includes the
2091 * 'Address Map' and 'Misc' device regs)
2093 static void read_mc_regs(struct amd64_pvt *pvt)
2095 struct cpuinfo_x86 *c = &boot_cpu_data;
2101 * Retrieve TOP_MEM and TOP_MEM2; no masking off of reserved bits since
2102 * those are Read-As-Zero
2104 rdmsrl(MSR_K8_TOP_MEM1, pvt->top_mem);
2105 edac_dbg(0, " TOP_MEM: 0x%016llx\n", pvt->top_mem);
2107 /* check first whether TOP_MEM2 is enabled */
2108 rdmsrl(MSR_K8_SYSCFG, msr_val);
2109 if (msr_val & (1U << 21)) {
2110 rdmsrl(MSR_K8_TOP_MEM2, pvt->top_mem2);
2111 edac_dbg(0, " TOP_MEM2: 0x%016llx\n", pvt->top_mem2);
2113 edac_dbg(0, " TOP_MEM2 disabled\n");
2115 amd64_read_pci_cfg(pvt->F3, NBCAP, &pvt->nbcap);
2117 read_dram_ctl_register(pvt);
2119 for (range = 0; range < DRAM_RANGES; range++) {
2122 /* read settings for this DRAM range */
2123 read_dram_base_limit_regs(pvt, range);
2125 rw = dram_rw(pvt, range);
2129 edac_dbg(1, " DRAM range[%d], base: 0x%016llx; limit: 0x%016llx\n",
2131 get_dram_base(pvt, range),
2132 get_dram_limit(pvt, range));
2134 edac_dbg(1, " IntlvEn=%s; Range access: %s%s IntlvSel=%d DstNode=%d\n",
2135 dram_intlv_en(pvt, range) ? "Enabled" : "Disabled",
2136 (rw & 0x1) ? "R" : "-",
2137 (rw & 0x2) ? "W" : "-",
2138 dram_intlv_sel(pvt, range),
2139 dram_dst_node(pvt, range));
2142 read_dct_base_mask(pvt);
2144 amd64_read_pci_cfg(pvt->F1, DHAR, &pvt->dhar);
2145 amd64_read_dct_pci_cfg(pvt, DBAM0, &pvt->dbam0);
2147 amd64_read_pci_cfg(pvt->F3, F10_ONLINE_SPARE, &pvt->online_spare);
2149 amd64_read_dct_pci_cfg(pvt, DCLR0, &pvt->dclr0);
2150 amd64_read_dct_pci_cfg(pvt, DCHR0, &pvt->dchr0);
2152 if (!dct_ganging_enabled(pvt)) {
2153 amd64_read_dct_pci_cfg(pvt, DCLR1, &pvt->dclr1);
2154 amd64_read_dct_pci_cfg(pvt, DCHR1, &pvt->dchr1);
2157 pvt->ecc_sym_sz = 4;
2159 if (c->x86 >= 0x10) {
2160 amd64_read_pci_cfg(pvt->F3, EXT_NB_MCA_CFG, &tmp);
2162 /* F16h has only DCT0 */
2163 amd64_read_dct_pci_cfg(pvt, DBAM1, &pvt->dbam1);
2165 /* F10h, revD and later can do x8 ECC too */
2166 if ((c->x86 > 0x10 || c->x86_model > 7) && tmp & BIT(25))
2167 pvt->ecc_sym_sz = 8;
2169 dump_misc_regs(pvt);
2173 * NOTE: CPU Revision Dependent code
2176 * @csrow_nr ChipSelect Row Number (0..NUM_CHIPSELECTS-1)
2177 * k8 private pointer to -->
2178 * DRAM Bank Address mapping register
2180 * DCL register where dual_channel_active is
2182 * The DBAM register consists of 4 sets of 4 bits each definitions:
2185 * 0-3 CSROWs 0 and 1
2186 * 4-7 CSROWs 2 and 3
2187 * 8-11 CSROWs 4 and 5
2188 * 12-15 CSROWs 6 and 7
2190 * Values range from: 0 to 15
2191 * The meaning of the values depends on CPU revision and dual-channel state,
2192 * see relevant BKDG more info.
2194 * The memory controller provides for total of only 8 CSROWs in its current
2195 * architecture. Each "pair" of CSROWs normally represents just one DIMM in
2196 * single channel or two (2) DIMMs in dual channel mode.
2198 * The following code logic collapses the various tables for CSROW based on CPU
2202 * The number of PAGE_SIZE pages on the specified CSROW number it
2206 static u32 amd64_csrow_nr_pages(struct amd64_pvt *pvt, u8 dct, int csrow_nr)
2208 u32 cs_mode, nr_pages;
2209 u32 dbam = dct ? pvt->dbam1 : pvt->dbam0;
2213 * The math on this doesn't look right on the surface because x/2*4 can
2214 * be simplified to x*2 but this expression makes use of the fact that
2215 * it is integral math where 1/2=0. This intermediate value becomes the
2216 * number of bits to shift the DBAM register to extract the proper CSROW
2219 cs_mode = DBAM_DIMM(csrow_nr / 2, dbam);
2221 nr_pages = pvt->ops->dbam_to_cs(pvt, dct, cs_mode) << (20 - PAGE_SHIFT);
2223 edac_dbg(0, "csrow: %d, channel: %d, DBAM idx: %d\n",
2224 csrow_nr, dct, cs_mode);
2225 edac_dbg(0, "nr_pages/channel: %u\n", nr_pages);
2231 * Initialize the array of csrow attribute instances, based on the values
2232 * from pci config hardware registers.
2234 static int init_csrows(struct mem_ctl_info *mci)
2236 struct amd64_pvt *pvt = mci->pvt_info;
2237 struct csrow_info *csrow;
2238 struct dimm_info *dimm;
2239 enum edac_type edac_mode;
2240 enum mem_type mtype;
2241 int i, j, empty = 1;
2245 amd64_read_pci_cfg(pvt->F3, NBCFG, &val);
2249 edac_dbg(0, "node %d, NBCFG=0x%08x[ChipKillEccCap: %d|DramEccEn: %d]\n",
2250 pvt->mc_node_id, val,
2251 !!(val & NBCFG_CHIPKILL), !!(val & NBCFG_ECC_ENABLE));
2254 * We iterate over DCT0 here but we look at DCT1 in parallel, if needed.
2256 for_each_chip_select(i, 0, pvt) {
2257 bool row_dct0 = !!csrow_enabled(i, 0, pvt);
2258 bool row_dct1 = false;
2260 if (boot_cpu_data.x86 != 0xf)
2261 row_dct1 = !!csrow_enabled(i, 1, pvt);
2263 if (!row_dct0 && !row_dct1)
2266 csrow = mci->csrows[i];
2269 edac_dbg(1, "MC node: %d, csrow: %d\n",
2270 pvt->mc_node_id, i);
2273 nr_pages = amd64_csrow_nr_pages(pvt, 0, i);
2274 csrow->channels[0]->dimm->nr_pages = nr_pages;
2277 /* K8 has only one DCT */
2278 if (boot_cpu_data.x86 != 0xf && row_dct1) {
2279 int row_dct1_pages = amd64_csrow_nr_pages(pvt, 1, i);
2281 csrow->channels[1]->dimm->nr_pages = row_dct1_pages;
2282 nr_pages += row_dct1_pages;
2285 mtype = amd64_determine_memory_type(pvt, i);
2287 edac_dbg(1, "Total csrow%d pages: %u\n", i, nr_pages);
2290 * determine whether CHIPKILL or JUST ECC or NO ECC is operating
2292 if (pvt->nbcfg & NBCFG_ECC_ENABLE)
2293 edac_mode = (pvt->nbcfg & NBCFG_CHIPKILL) ?
2294 EDAC_S4ECD4ED : EDAC_SECDED;
2296 edac_mode = EDAC_NONE;
2298 for (j = 0; j < pvt->channel_count; j++) {
2299 dimm = csrow->channels[j]->dimm;
2300 dimm->mtype = mtype;
2301 dimm->edac_mode = edac_mode;
2308 /* get all cores on this DCT */
2309 static void get_cpus_on_this_dct_cpumask(struct cpumask *mask, u16 nid)
2313 for_each_online_cpu(cpu)
2314 if (amd_get_nb_id(cpu) == nid)
2315 cpumask_set_cpu(cpu, mask);
2318 /* check MCG_CTL on all the cpus on this node */
2319 static bool amd64_nb_mce_bank_enabled_on_node(u16 nid)
2325 if (!zalloc_cpumask_var(&mask, GFP_KERNEL)) {
2326 amd64_warn("%s: Error allocating mask\n", __func__);
2330 get_cpus_on_this_dct_cpumask(mask, nid);
2332 rdmsr_on_cpus(mask, MSR_IA32_MCG_CTL, msrs);
2334 for_each_cpu(cpu, mask) {
2335 struct msr *reg = per_cpu_ptr(msrs, cpu);
2336 nbe = reg->l & MSR_MCGCTL_NBE;
2338 edac_dbg(0, "core: %u, MCG_CTL: 0x%llx, NB MSR is %s\n",
2340 (nbe ? "enabled" : "disabled"));
2348 free_cpumask_var(mask);
2352 static int toggle_ecc_err_reporting(struct ecc_settings *s, u16 nid, bool on)
2354 cpumask_var_t cmask;
2357 if (!zalloc_cpumask_var(&cmask, GFP_KERNEL)) {
2358 amd64_warn("%s: error allocating mask\n", __func__);
2362 get_cpus_on_this_dct_cpumask(cmask, nid);
2364 rdmsr_on_cpus(cmask, MSR_IA32_MCG_CTL, msrs);
2366 for_each_cpu(cpu, cmask) {
2368 struct msr *reg = per_cpu_ptr(msrs, cpu);
2371 if (reg->l & MSR_MCGCTL_NBE)
2372 s->flags.nb_mce_enable = 1;
2374 reg->l |= MSR_MCGCTL_NBE;
2377 * Turn off NB MCE reporting only when it was off before
2379 if (!s->flags.nb_mce_enable)
2380 reg->l &= ~MSR_MCGCTL_NBE;
2383 wrmsr_on_cpus(cmask, MSR_IA32_MCG_CTL, msrs);
2385 free_cpumask_var(cmask);
2390 static bool enable_ecc_error_reporting(struct ecc_settings *s, u16 nid,
2394 u32 value, mask = 0x3; /* UECC/CECC enable */
2396 if (toggle_ecc_err_reporting(s, nid, ON)) {
2397 amd64_warn("Error enabling ECC reporting over MCGCTL!\n");
2401 amd64_read_pci_cfg(F3, NBCTL, &value);
2403 s->old_nbctl = value & mask;
2404 s->nbctl_valid = true;
2407 amd64_write_pci_cfg(F3, NBCTL, value);
2409 amd64_read_pci_cfg(F3, NBCFG, &value);
2411 edac_dbg(0, "1: node %d, NBCFG=0x%08x[DramEccEn: %d]\n",
2412 nid, value, !!(value & NBCFG_ECC_ENABLE));
2414 if (!(value & NBCFG_ECC_ENABLE)) {
2415 amd64_warn("DRAM ECC disabled on this node, enabling...\n");
2417 s->flags.nb_ecc_prev = 0;
2419 /* Attempt to turn on DRAM ECC Enable */
2420 value |= NBCFG_ECC_ENABLE;
2421 amd64_write_pci_cfg(F3, NBCFG, value);
2423 amd64_read_pci_cfg(F3, NBCFG, &value);
2425 if (!(value & NBCFG_ECC_ENABLE)) {
2426 amd64_warn("Hardware rejected DRAM ECC enable,"
2427 "check memory DIMM configuration.\n");
2430 amd64_info("Hardware accepted DRAM ECC Enable\n");
2433 s->flags.nb_ecc_prev = 1;
2436 edac_dbg(0, "2: node %d, NBCFG=0x%08x[DramEccEn: %d]\n",
2437 nid, value, !!(value & NBCFG_ECC_ENABLE));
2442 static void restore_ecc_error_reporting(struct ecc_settings *s, u16 nid,
2445 u32 value, mask = 0x3; /* UECC/CECC enable */
2448 if (!s->nbctl_valid)
2451 amd64_read_pci_cfg(F3, NBCTL, &value);
2453 value |= s->old_nbctl;
2455 amd64_write_pci_cfg(F3, NBCTL, value);
2457 /* restore previous BIOS DRAM ECC "off" setting we force-enabled */
2458 if (!s->flags.nb_ecc_prev) {
2459 amd64_read_pci_cfg(F3, NBCFG, &value);
2460 value &= ~NBCFG_ECC_ENABLE;
2461 amd64_write_pci_cfg(F3, NBCFG, value);
2464 /* restore the NB Enable MCGCTL bit */
2465 if (toggle_ecc_err_reporting(s, nid, OFF))
2466 amd64_warn("Error restoring NB MCGCTL settings!\n");
2470 * EDAC requires that the BIOS have ECC enabled before
2471 * taking over the processing of ECC errors. A command line
2472 * option allows to force-enable hardware ECC later in
2473 * enable_ecc_error_reporting().
2475 static const char *ecc_msg =
2476 "ECC disabled in the BIOS or no ECC capability, module will not load.\n"
2477 " Either enable ECC checking or force module loading by setting "
2478 "'ecc_enable_override'.\n"
2479 " (Note that use of the override may cause unknown side effects.)\n";
2481 static bool ecc_enabled(struct pci_dev *F3, u16 nid)
2485 bool nb_mce_en = false;
2487 amd64_read_pci_cfg(F3, NBCFG, &value);
2489 ecc_en = !!(value & NBCFG_ECC_ENABLE);
2490 amd64_info("DRAM ECC %s.\n", (ecc_en ? "enabled" : "disabled"));
2492 nb_mce_en = amd64_nb_mce_bank_enabled_on_node(nid);
2494 amd64_notice("NB MCE bank disabled, set MSR "
2495 "0x%08x[4] on node %d to enable.\n",
2496 MSR_IA32_MCG_CTL, nid);
2498 if (!ecc_en || !nb_mce_en) {
2499 amd64_notice("%s", ecc_msg);
2505 static int set_mc_sysfs_attrs(struct mem_ctl_info *mci)
2509 rc = amd64_create_sysfs_dbg_files(mci);
2513 if (boot_cpu_data.x86 >= 0x10) {
2514 rc = amd64_create_sysfs_inject_files(mci);
2522 static void del_mc_sysfs_attrs(struct mem_ctl_info *mci)
2524 amd64_remove_sysfs_dbg_files(mci);
2526 if (boot_cpu_data.x86 >= 0x10)
2527 amd64_remove_sysfs_inject_files(mci);
2530 static void setup_mci_misc_attrs(struct mem_ctl_info *mci,
2531 struct amd64_family_type *fam)
2533 struct amd64_pvt *pvt = mci->pvt_info;
2535 mci->mtype_cap = MEM_FLAG_DDR2 | MEM_FLAG_RDDR2;
2536 mci->edac_ctl_cap = EDAC_FLAG_NONE;
2538 if (pvt->nbcap & NBCAP_SECDED)
2539 mci->edac_ctl_cap |= EDAC_FLAG_SECDED;
2541 if (pvt->nbcap & NBCAP_CHIPKILL)
2542 mci->edac_ctl_cap |= EDAC_FLAG_S4ECD4ED;
2544 mci->edac_cap = amd64_determine_edac_cap(pvt);
2545 mci->mod_name = EDAC_MOD_STR;
2546 mci->mod_ver = EDAC_AMD64_VERSION;
2547 mci->ctl_name = fam->ctl_name;
2548 mci->dev_name = pci_name(pvt->F2);
2549 mci->ctl_page_to_phys = NULL;
2551 /* memory scrubber interface */
2552 mci->set_sdram_scrub_rate = amd64_set_scrub_rate;
2553 mci->get_sdram_scrub_rate = amd64_get_scrub_rate;
2557 * returns a pointer to the family descriptor on success, NULL otherwise.
2559 static struct amd64_family_type *amd64_per_family_init(struct amd64_pvt *pvt)
2561 struct amd64_family_type *fam_type = NULL;
2563 pvt->ext_model = boot_cpu_data.x86_model >> 4;
2564 pvt->model = boot_cpu_data.x86_model;
2565 pvt->fam = boot_cpu_data.x86;
2569 fam_type = &amd64_family_types[K8_CPUS];
2570 pvt->ops = &amd64_family_types[K8_CPUS].ops;
2574 fam_type = &amd64_family_types[F10_CPUS];
2575 pvt->ops = &amd64_family_types[F10_CPUS].ops;
2579 if (pvt->model == 0x30) {
2580 fam_type = &amd64_family_types[F15_M30H_CPUS];
2581 pvt->ops = &amd64_family_types[F15_M30H_CPUS].ops;
2585 fam_type = &amd64_family_types[F15_CPUS];
2586 pvt->ops = &amd64_family_types[F15_CPUS].ops;
2590 fam_type = &amd64_family_types[F16_CPUS];
2591 pvt->ops = &amd64_family_types[F16_CPUS].ops;
2595 amd64_err("Unsupported family!\n");
2599 amd64_info("%s %sdetected (node %d).\n", fam_type->ctl_name,
2601 (pvt->ext_model >= K8_REV_F ? "revF or later "
2602 : "revE or earlier ")
2603 : ""), pvt->mc_node_id);
2607 static int amd64_init_one_instance(struct pci_dev *F2)
2609 struct amd64_pvt *pvt = NULL;
2610 struct amd64_family_type *fam_type = NULL;
2611 struct mem_ctl_info *mci = NULL;
2612 struct edac_mc_layer layers[2];
2614 u16 nid = amd_get_node_id(F2);
2617 pvt = kzalloc(sizeof(struct amd64_pvt), GFP_KERNEL);
2621 pvt->mc_node_id = nid;
2625 fam_type = amd64_per_family_init(pvt);
2630 err = reserve_mc_sibling_devs(pvt, fam_type->f1_id, fam_type->f3_id);
2637 * We need to determine how many memory channels there are. Then use
2638 * that information for calculating the size of the dynamic instance
2639 * tables in the 'mci' structure.
2642 pvt->channel_count = pvt->ops->early_channel_count(pvt);
2643 if (pvt->channel_count < 0)
2647 layers[0].type = EDAC_MC_LAYER_CHIP_SELECT;
2648 layers[0].size = pvt->csels[0].b_cnt;
2649 layers[0].is_virt_csrow = true;
2650 layers[1].type = EDAC_MC_LAYER_CHANNEL;
2651 layers[1].size = pvt->channel_count;
2652 layers[1].is_virt_csrow = false;
2653 mci = edac_mc_alloc(nid, ARRAY_SIZE(layers), layers, 0);
2657 mci->pvt_info = pvt;
2658 mci->pdev = &pvt->F2->dev;
2660 setup_mci_misc_attrs(mci, fam_type);
2662 if (init_csrows(mci))
2663 mci->edac_cap = EDAC_FLAG_NONE;
2666 if (edac_mc_add_mc(mci)) {
2667 edac_dbg(1, "failed edac_mc_add_mc()\n");
2670 if (set_mc_sysfs_attrs(mci)) {
2671 edac_dbg(1, "failed edac_mc_add_mc()\n");
2675 /* register stuff with EDAC MCE */
2676 if (report_gart_errors)
2677 amd_report_gart_errors(true);
2679 amd_register_ecc_decoder(amd64_decode_bus_error);
2683 atomic_inc(&drv_instances);
2688 edac_mc_del_mc(mci->pdev);
2693 free_mc_sibling_devs(pvt);
2702 static int amd64_probe_one_instance(struct pci_dev *pdev,
2703 const struct pci_device_id *mc_type)
2705 u16 nid = amd_get_node_id(pdev);
2706 struct pci_dev *F3 = node_to_amd_nb(nid)->misc;
2707 struct ecc_settings *s;
2710 ret = pci_enable_device(pdev);
2712 edac_dbg(0, "ret=%d\n", ret);
2717 s = kzalloc(sizeof(struct ecc_settings), GFP_KERNEL);
2723 if (!ecc_enabled(F3, nid)) {
2726 if (!ecc_enable_override)
2729 amd64_warn("Forcing ECC on!\n");
2731 if (!enable_ecc_error_reporting(s, nid, F3))
2735 ret = amd64_init_one_instance(pdev);
2737 amd64_err("Error probing instance: %d\n", nid);
2738 restore_ecc_error_reporting(s, nid, F3);
2745 ecc_stngs[nid] = NULL;
2751 static void amd64_remove_one_instance(struct pci_dev *pdev)
2753 struct mem_ctl_info *mci;
2754 struct amd64_pvt *pvt;
2755 u16 nid = amd_get_node_id(pdev);
2756 struct pci_dev *F3 = node_to_amd_nb(nid)->misc;
2757 struct ecc_settings *s = ecc_stngs[nid];
2759 mci = find_mci_by_dev(&pdev->dev);
2760 del_mc_sysfs_attrs(mci);
2761 /* Remove from EDAC CORE tracking list */
2762 mci = edac_mc_del_mc(&pdev->dev);
2766 pvt = mci->pvt_info;
2768 restore_ecc_error_reporting(s, nid, F3);
2770 free_mc_sibling_devs(pvt);
2772 /* unregister from EDAC MCE */
2773 amd_report_gart_errors(false);
2774 amd_unregister_ecc_decoder(amd64_decode_bus_error);
2776 kfree(ecc_stngs[nid]);
2777 ecc_stngs[nid] = NULL;
2779 /* Free the EDAC CORE resources */
2780 mci->pvt_info = NULL;
2788 * This table is part of the interface for loading drivers for PCI devices. The
2789 * PCI core identifies what devices are on a system during boot, and then
2790 * inquiry this table to see if this driver is for a given device found.
2792 static DEFINE_PCI_DEVICE_TABLE(amd64_pci_table) = {
2794 .vendor = PCI_VENDOR_ID_AMD,
2795 .device = PCI_DEVICE_ID_AMD_K8_NB_MEMCTL,
2796 .subvendor = PCI_ANY_ID,
2797 .subdevice = PCI_ANY_ID,
2802 .vendor = PCI_VENDOR_ID_AMD,
2803 .device = PCI_DEVICE_ID_AMD_10H_NB_DRAM,
2804 .subvendor = PCI_ANY_ID,
2805 .subdevice = PCI_ANY_ID,
2810 .vendor = PCI_VENDOR_ID_AMD,
2811 .device = PCI_DEVICE_ID_AMD_15H_NB_F2,
2812 .subvendor = PCI_ANY_ID,
2813 .subdevice = PCI_ANY_ID,
2818 .vendor = PCI_VENDOR_ID_AMD,
2819 .device = PCI_DEVICE_ID_AMD_15H_M30H_NB_F2,
2820 .subvendor = PCI_ANY_ID,
2821 .subdevice = PCI_ANY_ID,
2826 .vendor = PCI_VENDOR_ID_AMD,
2827 .device = PCI_DEVICE_ID_AMD_16H_NB_F2,
2828 .subvendor = PCI_ANY_ID,
2829 .subdevice = PCI_ANY_ID,
2836 MODULE_DEVICE_TABLE(pci, amd64_pci_table);
2838 static struct pci_driver amd64_pci_driver = {
2839 .name = EDAC_MOD_STR,
2840 .probe = amd64_probe_one_instance,
2841 .remove = amd64_remove_one_instance,
2842 .id_table = amd64_pci_table,
2845 static void setup_pci_device(void)
2847 struct mem_ctl_info *mci;
2848 struct amd64_pvt *pvt;
2856 pvt = mci->pvt_info;
2858 edac_pci_create_generic_ctl(&pvt->F2->dev, EDAC_MOD_STR);
2860 if (!amd64_ctl_pci) {
2861 pr_warning("%s(): Unable to create PCI control\n",
2864 pr_warning("%s(): PCI error report via EDAC not set\n",
2870 static int __init amd64_edac_init(void)
2874 printk(KERN_INFO "AMD64 EDAC driver v%s\n", EDAC_AMD64_VERSION);
2878 if (amd_cache_northbridges() < 0)
2882 mcis = kzalloc(amd_nb_num() * sizeof(mcis[0]), GFP_KERNEL);
2883 ecc_stngs = kzalloc(amd_nb_num() * sizeof(ecc_stngs[0]), GFP_KERNEL);
2884 if (!(mcis && ecc_stngs))
2887 msrs = msrs_alloc();
2891 err = pci_register_driver(&amd64_pci_driver);
2896 if (!atomic_read(&drv_instances))
2897 goto err_no_instances;
2903 pci_unregister_driver(&amd64_pci_driver);
2920 static void __exit amd64_edac_exit(void)
2923 edac_pci_release_generic_ctl(amd64_ctl_pci);
2925 pci_unregister_driver(&amd64_pci_driver);
2937 module_init(amd64_edac_init);
2938 module_exit(amd64_edac_exit);
2940 MODULE_LICENSE("GPL");
2941 MODULE_AUTHOR("SoftwareBitMaker: Doug Thompson, "
2942 "Dave Peterson, Thayne Harbaugh");
2943 MODULE_DESCRIPTION("MC support for AMD64 memory controllers - "
2944 EDAC_AMD64_VERSION);
2946 module_param(edac_op_state, int, 0444);
2947 MODULE_PARM_DESC(edac_op_state, "EDAC Error Reporting state: 0=Poll,1=NMI");