projects/linux·blob master · refs

631 lines · 15.3 KiB

1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Driver for Intel(R) servers with Integrated Memory/IO Hub-based memory controller.
4 * Copyright (c) 2025, Intel Corporation.
5 */
6
7#include <linux/kernel.h>
8#include <linux/io.h>
9#include <asm/cpu_device_id.h>
10#include <asm/intel-family.h>
11#include <asm/mce.h>
12#include <asm/cpu.h>
13#include "edac_module.h"
14#include "skx_common.h"
15
16#define IMH_REVISION "v0.0.1"
17#define EDAC_MOD_STR "imh_edac"
18
19/* Debug macros */
20#define imh_printk(level, fmt, arg...) \
21 edac_printk(level, "imh", fmt, ##arg)
22
23/* Configuration Agent(Ubox) */
24#define MMIO_BASE_H(reg) (((u64)GET_BITFIELD(reg, 0, 29)) << 23)
25#define SOCKET_ID(reg) GET_BITFIELD(reg, 0, 3)
26
27/* PUNIT */
28#define DDR_IMC_BITMAP(reg) GET_BITFIELD(reg, 23, 30)
29
30/* Memory Controller */
31#define ECC_ENABLED(reg) GET_BITFIELD(reg, 2, 2)
32#define DIMM_POPULATED(reg) GET_BITFIELD(reg, 15, 15)
33
34/* System Cache Agent(SCA) */
35#define TOLM(reg) (((u64)GET_BITFIELD(reg, 16, 31)) << 16)
36#define TOHM(reg) (((u64)GET_BITFIELD(reg, 16, 51)) << 16)
37
38/* Home Agent (HA) */
39#define NMCACHING(reg) GET_BITFIELD(reg, 8, 8)
40
41/**
42 * struct local_reg - A register as described in the local package view.
43 *
44 * @pkg: (input) The package where the register is located.
45 * @pbase: (input) The IP MMIO base physical address in the local package view.
46 * @size: (input) The IP MMIO size.
47 * @offset: (input) The register offset from the IP MMIO base @pbase.
48 * @width: (input) The register width in byte.
49 * @vbase: (internal) The IP MMIO base virtual address.
50 * @val: (output) The register value.
51 */
52struct local_reg {
53 int pkg;
54 u64 pbase;
55 u32 size;
56 u32 offset;
57 u8 width;
58 void __iomem *vbase;
59 u64 val;
60};
61
62#define DEFINE_LOCAL_REG(name, cfg, package, north, ip_name, ip_idx, reg_name) \
63 struct local_reg name = { \
64 .pkg = package, \
65 .pbase = (north ? (cfg)->mmio_base_l_north : \
66 (cfg)->mmio_base_l_south) + \
67 (cfg)->ip_name##_base + \
68 (cfg)->ip_name##_size * (ip_idx), \
69 .size = (cfg)->ip_name##_size, \
70 .offset = (cfg)->ip_name##_reg_##reg_name##_offset, \
71 .width = (cfg)->ip_name##_reg_##reg_name##_width, \
72 }
73
74static struct res_config *res_cfg;
75static int retry_rd_err_log;
76
77#define REG_RRL_DEFINE(a0, a1, a2, a3, a4, a5, a6, b0, b1, b2, b3) \
78 { \
79 .set_num = 4, \
80 .reg_num = 7, \
81 .sources = {RRL_SRC_FRE_SCRUB, RRL_SRC_FRE_DEMAND, RRL_SRC_LRE_SCRUB, RRL_SRC_LRE_DEMAND}, \
82 .offsets = { \
83 {a0, a1, a2, a3, a4, a5, a6}, \
84 {a0 + 4, a1 + 4, a2 + 8, a3 + 4, a4 + 4, a5 + 8, a6 + 8}, \
85 {a0 + 8, a1 + 8, a2 + 16, a3 + 8, a4 + 8, a5 + 16, a6 + 16}, \
86 {a0 + 12, a1 + 12, a2 + 24, a3 + 12, a4 + 12, a5 + 24, a6 + 24}, \
87 }, \
88 .widths = {4, 4, 8, 4, 4, 8, 8}, \
89 .v_mask = BIT(0), \
90 .uc_mask = BIT(1), \
91 .over_mask = BIT(2), \
92 .en_mask = BIT(12), \
93 .en_patspr_mask = BIT(14), \
94 .noover_mask = BIT(15), \
95 .cecnt_num = 4, \
96 .cecnt_offsets = {b0, b1, b2, b3}, \
97 .cecnt_widths = {8, 8, 8, 8}, \
98}
99
100static struct reg_rrl dmr_reg_rrl_ddr_subch0 = REG_RRL_DEFINE(
101 0x2dc0, 0x2dd0, 0x2de0, 0x2e00, 0x2e10, 0x2f70, 0x0200,
102 0x2c10, 0x2c18, 0x2c20, 0x2c28);
103static struct reg_rrl dmr_reg_rrl_ddr_subch1 = REG_RRL_DEFINE(
104 0x6dc0, 0x6dd0, 0x6de0, 0x6e00, 0x6e10, 0x6f70, 0x4200,
105 0x6c10, 0x6c18, 0x6c20, 0x6c28);
106
107static void __read_local_reg(void *reg)
108{
109 struct local_reg *r = (struct local_reg *)reg;
110
111 r->val = skx_readx(r->vbase + r->offset, r->width);
112}
113
114/* Read a local-view register. */
115static bool read_local_reg(struct local_reg *reg)
116{
117 int cpu;
118
119 /* Get the target CPU in the package @reg->pkg. */
120 for_each_online_cpu(cpu) {
121 if (reg->pkg == topology_physical_package_id(cpu))
122 break;
123 }
124
125 if (cpu >= nr_cpu_ids)
126 return false;
127
128 reg->vbase = ioremap(reg->pbase, reg->size);
129 if (!reg->vbase) {
130 imh_printk(KERN_ERR, "Failed to ioremap 0x%llx\n", reg->pbase);
131 return false;
132 }
133
134 /* Get the target CPU to read the register. */
135 smp_call_function_single(cpu, __read_local_reg, reg, 1);
136 iounmap(reg->vbase);
137
138 return true;
139}
140
141/* Get the bitmap of memory controller instances in package @pkg. */
142static u32 get_imc_bitmap(struct res_config *cfg, int pkg, bool north)
143{
144 DEFINE_LOCAL_REG(reg, cfg, pkg, north, pcu, 0, capid3);
145
146 if (!read_local_reg(&reg))
147 return 0;
148
149 edac_dbg(2, "Pkg%d %s mc instances bitmap 0x%llx (reg 0x%llx)\n",
150 pkg, north ? "north" : "south",
151 DDR_IMC_BITMAP(reg.val), reg.val);
152
153 return DDR_IMC_BITMAP(reg.val);
154}
155
156static void imc_release(struct device *dev)
157{
158 edac_dbg(2, "imc device %s released\n", dev_name(dev));
159 kfree(dev);
160}
161
162static int __get_ddr_munits(struct res_config *cfg, struct skx_dev *d,
163 bool north, int lmc)
164{
165 unsigned long size = cfg->ddr_chan_mmio_sz * cfg->ddr_chan_num;
166 unsigned long bitmap = get_imc_bitmap(cfg, d->pkg, north);
167 void __iomem *mbase;
168 struct device *dev;
169 int i, rc, pmc;
170 u64 base;
171
172 for_each_set_bit(i, &bitmap, sizeof(bitmap) * 8) {
173 base = north ? d->mmio_base_h_north : d->mmio_base_h_south;
174 base += cfg->ddr_imc_base + size * i;
175
176 edac_dbg(2, "Pkg%d mc%d mmio base 0x%llx size 0x%lx\n",
177 d->pkg, lmc, base, size);
178
179 /* Set up the imc MMIO. */
180 mbase = ioremap(base, size);
181 if (!mbase) {
182 imh_printk(KERN_ERR, "Failed to ioremap 0x%llx\n", base);
183 return -ENOMEM;
184 }
185
186 d->imc[lmc].mbase = mbase;
187 d->imc[lmc].lmc = lmc;
188
189 /* Create the imc device instance. */
190 dev = kzalloc_obj(*dev);
191 if (!dev)
192 return -ENOMEM;
193
194 dev->release = imc_release;
195 device_initialize(dev);
196 rc = dev_set_name(dev, "0x%llx", base);
197 if (rc) {
198 imh_printk(KERN_ERR, "Failed to set dev name\n");
199 put_device(dev);
200 return rc;
201 }
202
203 d->imc[lmc].dev = dev;
204
205 /* Set up the imc index mapping. */
206 pmc = north ? i : 8 + i;
207 skx_set_mc_mapping(d, pmc, lmc);
208
209 lmc++;
210 }
211
212 return lmc;
213}
214
215static bool get_ddr_munits(struct res_config *cfg, struct skx_dev *d)
216{
217 int lmc = __get_ddr_munits(cfg, d, true, 0);
218
219 if (lmc < 0)
220 return false;
221
222 lmc = __get_ddr_munits(cfg, d, false, lmc);
223 if (lmc <= 0)
224 return false;
225
226 return true;
227}
228
229static bool get_socket_id(struct res_config *cfg, struct skx_dev *d)
230{
231 DEFINE_LOCAL_REG(reg, cfg, d->pkg, true, ubox, 0, socket_id);
232 u8 src_id;
233 int i;
234
235 if (!read_local_reg(&reg))
236 return false;
237
238 src_id = SOCKET_ID(reg.val);
239 edac_dbg(2, "socket id 0x%x (reg 0x%llx)\n", src_id, reg.val);
240
241 for (i = 0; i < cfg->ddr_imc_num; i++)
242 d->imc[i].src_id = src_id;
243
244 return true;
245}
246
247/* Get TOLM (Top Of Low Memory) and TOHM (Top Of High Memory) parameters. */
248static bool imh_get_tolm_tohm(struct res_config *cfg, u64 *tolm, u64 *tohm)
249{
250 DEFINE_LOCAL_REG(reg, cfg, 0, true, sca, 0, tolm);
251
252 if (!read_local_reg(&reg))
253 return false;
254
255 *tolm = TOLM(reg.val);
256 edac_dbg(2, "tolm 0x%llx (reg 0x%llx)\n", *tolm, reg.val);
257
258 DEFINE_LOCAL_REG(reg2, cfg, 0, true, sca, 0, tohm);
259
260 if (!read_local_reg(&reg2))
261 return false;
262
263 *tohm = TOHM(reg2.val);
264 edac_dbg(2, "tohm 0x%llx (reg 0x%llx)\n", *tohm, reg2.val);
265
266 return true;
267}
268
269/* Get the system-view MMIO_BASE_H for {north,south}-IMH. */
270static int imh_get_all_mmio_base_h(struct res_config *cfg, struct list_head *edac_list)
271{
272 int i, n = topology_max_packages(), imc_num = cfg->ddr_imc_num + cfg->hbm_imc_num;
273 struct skx_dev *d;
274
275 for (i = 0; i < n; i++) {
276 d = kzalloc_flex(*d, imc, imc_num);
277 if (!d)
278 return -ENOMEM;
279
280 DEFINE_LOCAL_REG(reg, cfg, i, true, ubox, 0, mmio_base);
281
282 /* Get MMIO_BASE_H for the north-IMH. */
283 if (!read_local_reg(&reg) || !reg.val) {
284 kfree(d);
285 imh_printk(KERN_ERR, "Pkg%d has no north mmio_base_h\n", i);
286 return -ENODEV;
287 }
288
289 d->mmio_base_h_north = MMIO_BASE_H(reg.val);
290 edac_dbg(2, "Pkg%d north mmio_base_h 0x%llx (reg 0x%llx)\n",
291 i, d->mmio_base_h_north, reg.val);
292
293 /* Get MMIO_BASE_H for the south-IMH (optional). */
294 DEFINE_LOCAL_REG(reg2, cfg, i, false, ubox, 0, mmio_base);
295
296 if (read_local_reg(&reg2)) {
297 d->mmio_base_h_south = MMIO_BASE_H(reg2.val);
298 edac_dbg(2, "Pkg%d south mmio_base_h 0x%llx (reg 0x%llx)\n",
299 i, d->mmio_base_h_south, reg2.val);
300 }
301
302 d->pkg = i;
303 d->num_imc = imc_num;
304 skx_init_mc_mapping(d);
305 list_add_tail(&d->list, edac_list);
306 }
307
308 return 0;
309}
310
311/* Get the number of per-package memory controllers. */
312static int imh_get_imc_num(struct res_config *cfg)
313{
314 int imc_num = hweight32(get_imc_bitmap(cfg, 0, true)) +
315 hweight32(get_imc_bitmap(cfg, 0, false));
316
317 if (!imc_num) {
318 imh_printk(KERN_ERR, "Invalid mc number\n");
319 return -ENODEV;
320 }
321
322 if (cfg->ddr_imc_num != imc_num) {
323 /*
324 * Update the configuration data to reflect the number of
325 * present DDR memory controllers.
326 */
327 cfg->ddr_imc_num = imc_num;
328 edac_dbg(2, "Set ddr mc number %d\n", imc_num);
329 }
330
331 return 0;
332}
333
334/* Get all memory controllers' parameters. */
335static int imh_get_munits(struct res_config *cfg, struct list_head *edac_list)
336{
337 struct skx_imc *imc;
338 struct skx_dev *d;
339 u8 mc = 0;
340 int i;
341
342 list_for_each_entry(d, edac_list, list) {
343 if (!get_ddr_munits(cfg, d)) {
344 imh_printk(KERN_ERR, "No mc found\n");
345 return -ENODEV;
346 }
347
348 if (!get_socket_id(cfg, d)) {
349 imh_printk(KERN_ERR, "Failed to get socket id\n");
350 return -ENODEV;
351 }
352
353 for (i = 0; i < cfg->ddr_imc_num; i++) {
354 imc = &d->imc[i];
355 if (!imc->mbase)
356 continue;
357
358 imc->chan_mmio_sz = cfg->ddr_chan_mmio_sz;
359 imc->num_channels = cfg->ddr_chan_num;
360 imc->num_dimms = cfg->ddr_dimm_num;
361 imc->mc = mc++;
362 }
363 }
364
365 return 0;
366}
367
368static bool check_2lm_enabled(struct res_config *cfg, struct skx_dev *d, int ha_idx)
369{
370 DEFINE_LOCAL_REG(reg, cfg, d->pkg, true, ha, ha_idx, mode);
371
372 if (!read_local_reg(&reg))
373 return false;
374
375 if (!NMCACHING(reg.val))
376 return false;
377
378 edac_dbg(2, "2-level memory configuration (reg 0x%llx, ha idx %d)\n", reg.val, ha_idx);
379 return true;
380}
381
382/* Check whether the system has a 2-level memory configuration. */
383static bool imh_2lm_enabled(struct res_config *cfg, struct list_head *head)
384{
385 struct skx_dev *d;
386 int i;
387
388 list_for_each_entry(d, head, list) {
389 for (i = 0; i < cfg->ddr_imc_num; i++)
390 if (check_2lm_enabled(cfg, d, i))
391 return true;
392 }
393
394 return false;
395}
396
397static u32 read_imc_mcmtr(struct res_config *cfg, struct skx_imc *imc, int chan)
398{
399 return (u32)skx_read_imc_reg(imc, chan, cfg->ddr_reg_mcmtr_offset, cfg->ddr_reg_mcmtr_width);
400}
401
402static u32 read_imc_dimmmtr(struct res_config *cfg, struct skx_imc *imc, int chan, int dimm)
403{
404 return (u32)skx_read_imc_reg(imc, chan, cfg->ddr_reg_dimmmtr_offset +
405 cfg->ddr_reg_dimmmtr_width * dimm,
406 cfg->ddr_reg_dimmmtr_width);
407}
408
409static bool ecc_enabled(u32 mcmtr)
410{
411 return (bool)ECC_ENABLED(mcmtr);
412}
413
414static bool dimm_populated(u32 dimmmtr)
415{
416 return (bool)DIMM_POPULATED(dimmmtr);
417}
418
419/* Get each DIMM's configurations of the memory controller @mci. */
420static int imh_get_dimm_config(struct mem_ctl_info *mci, struct res_config *cfg)
421{
422 struct skx_pvt *pvt = mci->pvt_info;
423 struct skx_imc *imc = pvt->imc;
424 struct dimm_info *dimm;
425 u32 mcmtr, dimmmtr;
426 int i, j, ndimms;
427
428 for (i = 0; i < imc->num_channels; i++) {
429 if (!imc->mbase)
430 continue;
431
432 mcmtr = read_imc_mcmtr(cfg, imc, i);
433
434 for (ndimms = 0, j = 0; j < imc->num_dimms; j++) {
435 dimmmtr = read_imc_dimmmtr(cfg, imc, i, j);
436 edac_dbg(1, "mcmtr 0x%x dimmmtr 0x%x (mc%d ch%d dimm%d)\n",
437 mcmtr, dimmmtr, imc->mc, i, j);
438
439 if (!dimm_populated(dimmmtr))
440 continue;
441
442 dimm = edac_get_dimm(mci, i, j, 0);
443 ndimms += skx_get_dimm_info(dimmmtr, 0, 0, dimm,
444 imc, i, j, cfg);
445 }
446
447 if (ndimms && !ecc_enabled(mcmtr)) {
448 imh_printk(KERN_ERR, "ECC is disabled on mc%d ch%d\n",
449 imc->mc, i);
450 return -ENODEV;
451 }
452 }
453
454 return 0;
455}
456
457/* Register all memory controllers to the EDAC core. */
458static int imh_register_mci(struct res_config *cfg, struct list_head *edac_list)
459{
460 struct skx_imc *imc;
461 struct skx_dev *d;
462 int i, rc;
463
464 list_for_each_entry(d, edac_list, list) {
465 for (i = 0; i < cfg->ddr_imc_num; i++) {
466 imc = &d->imc[i];
467 if (!imc->mbase)
468 continue;
469
470 rc = skx_register_mci(imc, imc->dev,
471 dev_name(imc->dev),
472 "Intel IMH-based Socket",
473 EDAC_MOD_STR,
474 imh_get_dimm_config, cfg);
475 if (rc)
476 return rc;
477 }
478 }
479
480 return 0;
481}
482
483static struct res_config dmr_cfg = {
484 .type = DMR,
485 .support_ddr5 = true,
486 .mmio_base_l_north = 0xf6800000,
487 .mmio_base_l_south = 0xf6000000,
488 .ddr_chan_num = 1,
489 .ddr_dimm_num = 2,
490 .ddr_imc_base = 0x39b000,
491 .ddr_chan_mmio_sz = 0x8000,
492 .ddr_reg_mcmtr_offset = 0x360,
493 .ddr_reg_mcmtr_width = 4,
494 .ddr_reg_dimmmtr_offset = 0x370,
495 .ddr_reg_dimmmtr_width = 4,
496 .ubox_base = 0x0,
497 .ubox_size = 0x2000,
498 .ubox_reg_mmio_base_offset = 0x580,
499 .ubox_reg_mmio_base_width = 4,
500 .ubox_reg_socket_id_offset = 0x1080,
501 .ubox_reg_socket_id_width = 4,
502 .pcu_base = 0x3000,
503 .pcu_size = 0x10000,
504 .pcu_reg_capid3_offset = 0x290,
505 .pcu_reg_capid3_width = 4,
506 .sca_base = 0x24c000,
507 .sca_size = 0x2500,
508 .sca_reg_tolm_offset = 0x2100,
509 .sca_reg_tolm_width = 8,
510 .sca_reg_tohm_offset = 0x2108,
511 .sca_reg_tohm_width = 8,
512 .ha_base = 0x3eb000,
513 .ha_size = 0x1000,
514 .ha_reg_mode_offset = 0x4a0,
515 .ha_reg_mode_width = 4,
516 .reg_rrl_ddr[0] = &dmr_reg_rrl_ddr_subch0,
517 .reg_rrl_ddr[1] = &dmr_reg_rrl_ddr_subch1,
518};
519
520static const struct x86_cpu_id imh_cpuids[] = {
521 X86_MATCH_VFM(INTEL_DIAMONDRAPIDS_X, &dmr_cfg),
522 {}
523};
524MODULE_DEVICE_TABLE(x86cpu, imh_cpuids);
525
526static struct notifier_block imh_mce_dec = {
527 .notifier_call = skx_mce_check_error,
528 .priority = MCE_PRIO_EDAC,
529};
530
531static int __init imh_init(void)
532{
533 const struct x86_cpu_id *id;
534 struct list_head *edac_list;
535 struct res_config *cfg;
536 const char *owner;
537 u64 tolm, tohm;
538 int rc;
539
540 edac_dbg(2, "\n");
541
542 if (ghes_get_devices())
543 return -EBUSY;
544
545 owner = edac_get_owner();
546 if (owner && strncmp(owner, EDAC_MOD_STR, sizeof(EDAC_MOD_STR)))
547 return -EBUSY;
548
549 if (cpu_feature_enabled(X86_FEATURE_HYPERVISOR))
550 return -ENODEV;
551
552 id = x86_match_cpu(imh_cpuids);
553 if (!id)
554 return -ENODEV;
555 cfg = (struct res_config *)id->driver_data;
556 skx_set_res_cfg(cfg);
557 res_cfg = cfg;
558
559 if (!imh_get_tolm_tohm(cfg, &tolm, &tohm))
560 return -ENODEV;
561
562 skx_set_hi_lo(tolm, tohm);
563
564 rc = imh_get_imc_num(cfg);
565 if (rc < 0)
566 goto fail;
567
568 edac_list = skx_get_edac_list();
569
570 rc = imh_get_all_mmio_base_h(cfg, edac_list);
571 if (rc)
572 goto fail;
573
574 rc = imh_get_munits(cfg, edac_list);
575 if (rc)
576 goto fail;
577
578 skx_set_mem_cfg(imh_2lm_enabled(cfg, edac_list));
579
580 rc = imh_register_mci(cfg, edac_list);
581 if (rc)
582 goto fail;
583
584 rc = skx_adxl_get();
585 if (rc)
586 goto fail;
587
588 opstate_init();
589 mce_register_decode_chain(&imh_mce_dec);
590 skx_setup_debug("imh_test");
591
592 cfg->rrl_ctrl_mode = retry_rd_err_log;
593 if (retry_rd_err_log && cfg->reg_rrl_ddr[0]) {
594 skx_set_show_rrl(skx_show_rrl);
595 if (retry_rd_err_log == RRL_CTRL_LINUX)
596 skx_enable_rrl(true);
597 }
598
599 imh_printk(KERN_INFO, "%s\n", IMH_REVISION);
600
601 return 0;
602fail:
603 skx_remove();
604 return rc;
605}
606
607static void __exit imh_exit(void)
608{
609 edac_dbg(2, "\n");
610
611 if (retry_rd_err_log && res_cfg->reg_rrl_ddr[0]) {
612 if (retry_rd_err_log == RRL_CTRL_LINUX)
613 skx_enable_rrl(false);
614 skx_set_show_rrl(NULL);
615 }
616
617 skx_teardown_debug();
618 mce_unregister_decode_chain(&imh_mce_dec);
619 skx_adxl_put();
620 skx_remove();
621}
622
623module_init(imh_init);
624module_exit(imh_exit);
625
626module_param(retry_rd_err_log, int, 0444);
627MODULE_PARM_DESC(retry_rd_err_log, "retry_rd_err_log: 0=off(default), 1=bios(Linux doesn't reset any control bits, but just reports values.), 2=linux(Linux tries to take control and resets mode bits, clear valid/UC bits after reading.)");
628
629MODULE_LICENSE("GPL");
630MODULE_AUTHOR("Qiuxu Zhuo");
631MODULE_DESCRIPTION("MC Driver for Intel servers using IMH-based memory controller");