projects/linux·blob master · refs

1904 lines · 48.8 KiB

1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Common code for Intel Running Average Power Limit (RAPL) support.
4 * Copyright (c) 2019, Intel Corporation.
5 */
6#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
7
8#include <linux/bitmap.h>
9#include <linux/cleanup.h>
10#include <linux/cpu.h>
11#include <linux/delay.h>
12#include <linux/device.h>
13#include <linux/intel_rapl.h>
14#include <linux/kernel.h>
15#include <linux/list.h>
16#include <linux/log2.h>
17#include <linux/module.h>
18#include <linux/nospec.h>
19#include <linux/perf_event.h>
20#include <linux/platform_device.h>
21#include <linux/powercap.h>
22#include <linux/processor.h>
23#include <linux/slab.h>
24#include <linux/suspend.h>
25#include <linux/sysfs.h>
26#include <linux/types.h>
27#include <linux/units.h>
28
29#include <asm/cpu_device_id.h>
30#include <asm/intel-family.h>
31#include <asm/msr.h>
32
33#define ENERGY_STATUS_MASK GENMASK(31, 0)
34
35/* Width of the RAPL energy counters, see the *_ENERGY_STATUS_MASK defines */
36#define RAPL_CNTR_WIDTH 32
37
38#define POWER_UNIT_OFFSET 0x00
39#define POWER_UNIT_MASK GENMASK(3, 0)
40
41#define ENERGY_UNIT_OFFSET 0x08
42#define ENERGY_UNIT_MASK GENMASK(12, 8)
43
44#define TIME_UNIT_OFFSET 0x10
45#define TIME_UNIT_MASK GENMASK(19, 16)
46
47/* Non HW constants */
48#define RAPL_PRIMITIVE_DUMMY BIT(2)
49
50#define ENERGY_UNIT_SCALE 1000 /* scale from driver unit to powercap unit */
51
52/* per domain data, some are optional */
53#define NR_RAW_PRIMITIVES (NR_RAPL_PRIMITIVES - 2)
54
55#define PACKAGE_PLN_INT_SAVED BIT(0)
56
57#define RAPL_EVENT_MASK GENMASK(7, 0)
58
59static const char *pl_names[NR_POWER_LIMITS] = {
60 [POWER_LIMIT1] = "long_term",
61 [POWER_LIMIT2] = "short_term",
62 [POWER_LIMIT4] = "peak_power",
63};
64
65enum pl_prims {
66 PL_ENABLE,
67 PL_CLAMP,
68 PL_LIMIT,
69 PL_TIME_WINDOW,
70 PL_MAX_POWER,
71 PL_LOCK,
72};
73
74static bool is_pl_valid(struct rapl_domain *rd, int pl)
75{
76 if (pl < POWER_LIMIT1 || pl > POWER_LIMIT4)
77 return false;
78 return rd->rpl[pl].name ? true : false;
79}
80
81static int get_pl_lock_prim(struct rapl_domain *rd, int pl)
82{
83 if (rd->rp->priv->type == RAPL_IF_TPMI) {
84 if (pl == POWER_LIMIT1)
85 return PL1_LOCK;
86 if (pl == POWER_LIMIT2)
87 return PL2_LOCK;
88 if (pl == POWER_LIMIT4)
89 return PL4_LOCK;
90 }
91
92 /* MSR/MMIO Interface doesn't have Lock bit for PL4 */
93 if (pl == POWER_LIMIT4)
94 return -EINVAL;
95
96 /*
97 * Power Limit register that supports two power limits has a different
98 * bit position for the Lock bit.
99 */
100 if (rd->rp->priv->limits[rd->id] & BIT(POWER_LIMIT2))
101 return FW_HIGH_LOCK;
102 return FW_LOCK;
103}
104
105static int get_pl_prim(struct rapl_domain *rd, int pl, enum pl_prims prim)
106{
107 switch (pl) {
108 case POWER_LIMIT1:
109 if (prim == PL_ENABLE)
110 return PL1_ENABLE;
111 if (prim == PL_CLAMP && rd->rp->priv->type != RAPL_IF_TPMI)
112 return PL1_CLAMP;
113 if (prim == PL_LIMIT)
114 return POWER_LIMIT1;
115 if (prim == PL_TIME_WINDOW)
116 return TIME_WINDOW1;
117 if (prim == PL_MAX_POWER)
118 return THERMAL_SPEC_POWER;
119 if (prim == PL_LOCK)
120 return get_pl_lock_prim(rd, pl);
121 return -EINVAL;
122 case POWER_LIMIT2:
123 if (prim == PL_ENABLE)
124 return PL2_ENABLE;
125 if (prim == PL_CLAMP && rd->rp->priv->type != RAPL_IF_TPMI)
126 return PL2_CLAMP;
127 if (prim == PL_LIMIT)
128 return POWER_LIMIT2;
129 if (prim == PL_TIME_WINDOW)
130 return TIME_WINDOW2;
131 if (prim == PL_MAX_POWER)
132 return MAX_POWER;
133 if (prim == PL_LOCK)
134 return get_pl_lock_prim(rd, pl);
135 return -EINVAL;
136 case POWER_LIMIT4:
137 if (prim == PL_LIMIT)
138 return POWER_LIMIT4;
139 if (prim == PL_ENABLE)
140 return PL4_ENABLE;
141 /* PL4 would be around two times PL2, use same prim as PL2. */
142 if (prim == PL_MAX_POWER)
143 return MAX_POWER;
144 if (prim == PL_LOCK)
145 return get_pl_lock_prim(rd, pl);
146 return -EINVAL;
147 default:
148 return -EINVAL;
149 }
150}
151
152#define power_zone_to_rapl_domain(_zone) \
153 container_of(_zone, struct rapl_domain, power_zone)
154
155static const struct rapl_defaults *get_defaults(struct rapl_package *rp)
156{
157 return rp->priv->defaults;
158}
159
160static void rapl_init_domains(struct rapl_package *rp);
161static int rapl_read_data_raw(struct rapl_domain *rd,
162 enum rapl_primitives prim,
163 bool xlate, u64 *data,
164 bool pmu_ctx);
165static int rapl_write_data_raw(struct rapl_domain *rd,
166 enum rapl_primitives prim,
167 unsigned long long value);
168static int rapl_read_pl_data(struct rapl_domain *rd, int pl,
169 enum pl_prims pl_prim,
170 bool xlate, u64 *data);
171static int rapl_write_pl_data(struct rapl_domain *rd, int pl,
172 enum pl_prims pl_prim,
173 unsigned long long value);
174static u64 rapl_unit_xlate(struct rapl_domain *rd,
175 enum unit_type type, u64 value, int to_raw);
176static void package_power_limit_irq_save(struct rapl_package *rp);
177
178static LIST_HEAD(rapl_packages); /* guarded by CPU hotplug lock */
179
180static const char *const rapl_domain_names[] = {
181 "package",
182 "core",
183 "uncore",
184 "dram",
185 "psys",
186};
187
188static int get_energy_counter(struct powercap_zone *power_zone,
189 u64 *energy_raw)
190{
191 struct rapl_domain *rd;
192 u64 energy_now;
193
194 /* prevent CPU hotplug, make sure the RAPL domain does not go
195 * away while reading the counter.
196 */
197 cpus_read_lock();
198 rd = power_zone_to_rapl_domain(power_zone);
199
200 if (!rapl_read_data_raw(rd, ENERGY_COUNTER, true, &energy_now, false)) {
201 *energy_raw = energy_now;
202 cpus_read_unlock();
203
204 return 0;
205 }
206 cpus_read_unlock();
207
208 return -EIO;
209}
210
211static int get_max_energy_counter(struct powercap_zone *pcd_dev, u64 *energy)
212{
213 struct rapl_domain *rd = power_zone_to_rapl_domain(pcd_dev);
214
215 *energy = rapl_unit_xlate(rd, ENERGY_UNIT, ENERGY_STATUS_MASK, 0);
216 return 0;
217}
218
219static int release_zone(struct powercap_zone *power_zone)
220{
221 struct rapl_domain *rd = power_zone_to_rapl_domain(power_zone);
222 struct rapl_package *rp = rd->rp;
223
224 /* package zone is the last zone of a package, we can free
225 * memory here since all children has been unregistered.
226 */
227 if (rd->id == RAPL_DOMAIN_PACKAGE) {
228 kfree(rd);
229 rp->domains = NULL;
230 }
231
232 return 0;
233
234}
235
236static int find_nr_power_limit(struct rapl_domain *rd)
237{
238 int i, nr_pl = 0;
239
240 for (i = 0; i < NR_POWER_LIMITS; i++) {
241 if (is_pl_valid(rd, i))
242 nr_pl++;
243 }
244
245 return nr_pl;
246}
247
248static int set_domain_enable(struct powercap_zone *power_zone, bool mode)
249{
250 struct rapl_domain *rd = power_zone_to_rapl_domain(power_zone);
251 const struct rapl_defaults *defaults = get_defaults(rd->rp);
252 u64 val;
253 int ret;
254
255 cpus_read_lock();
256 ret = rapl_write_pl_data(rd, POWER_LIMIT1, PL_ENABLE, mode);
257 if (ret)
258 goto end;
259
260 ret = rapl_read_pl_data(rd, POWER_LIMIT1, PL_ENABLE, false, &val);
261 if (ret)
262 goto end;
263
264 if (mode != val) {
265 pr_debug("%s cannot be %s\n", power_zone->name,
266 str_enabled_disabled(mode));
267 goto end;
268 }
269
270 if (defaults->set_floor_freq)
271 defaults->set_floor_freq(rd, mode);
272
273end:
274 cpus_read_unlock();
275
276 return ret;
277}
278
279static int get_domain_enable(struct powercap_zone *power_zone, bool *mode)
280{
281 struct rapl_domain *rd = power_zone_to_rapl_domain(power_zone);
282 u64 val;
283 int ret;
284
285 if (rd->rpl[POWER_LIMIT1].locked) {
286 *mode = false;
287 return 0;
288 }
289 cpus_read_lock();
290 ret = rapl_read_pl_data(rd, POWER_LIMIT1, PL_ENABLE, true, &val);
291 if (!ret)
292 *mode = val;
293 cpus_read_unlock();
294
295 return ret;
296}
297
298/* per RAPL domain ops, in the order of rapl_domain_type */
299static const struct powercap_zone_ops zone_ops[] = {
300 /* RAPL_DOMAIN_PACKAGE */
301 {
302 .get_energy_uj = get_energy_counter,
303 .get_max_energy_range_uj = get_max_energy_counter,
304 .release = release_zone,
305 .set_enable = set_domain_enable,
306 .get_enable = get_domain_enable,
307 },
308 /* RAPL_DOMAIN_PP0 */
309 {
310 .get_energy_uj = get_energy_counter,
311 .get_max_energy_range_uj = get_max_energy_counter,
312 .release = release_zone,
313 .set_enable = set_domain_enable,
314 .get_enable = get_domain_enable,
315 },
316 /* RAPL_DOMAIN_PP1 */
317 {
318 .get_energy_uj = get_energy_counter,
319 .get_max_energy_range_uj = get_max_energy_counter,
320 .release = release_zone,
321 .set_enable = set_domain_enable,
322 .get_enable = get_domain_enable,
323 },
324 /* RAPL_DOMAIN_DRAM */
325 {
326 .get_energy_uj = get_energy_counter,
327 .get_max_energy_range_uj = get_max_energy_counter,
328 .release = release_zone,
329 .set_enable = set_domain_enable,
330 .get_enable = get_domain_enable,
331 },
332 /* RAPL_DOMAIN_PLATFORM */
333 {
334 .get_energy_uj = get_energy_counter,
335 .get_max_energy_range_uj = get_max_energy_counter,
336 .release = release_zone,
337 .set_enable = set_domain_enable,
338 .get_enable = get_domain_enable,
339 },
340};
341
342/*
343 * Constraint index used by powercap can be different than power limit (PL)
344 * index in that some PLs maybe missing due to non-existent MSRs. So we
345 * need to convert here by finding the valid PLs only (name populated).
346 */
347static int contraint_to_pl(struct rapl_domain *rd, int cid)
348{
349 int i, j;
350
351 for (i = POWER_LIMIT1, j = 0; i < NR_POWER_LIMITS; i++) {
352 if (is_pl_valid(rd, i) && j++ == cid) {
353 pr_debug("%s: index %d\n", __func__, i);
354 return i;
355 }
356 }
357 pr_err("Cannot find matching power limit for constraint %d\n", cid);
358
359 return -EINVAL;
360}
361
362static int set_power_limit(struct powercap_zone *power_zone, int cid,
363 u64 power_limit)
364{
365 struct rapl_domain *rd;
366 struct rapl_package *rp;
367 int ret = 0;
368 int id;
369
370 cpus_read_lock();
371 rd = power_zone_to_rapl_domain(power_zone);
372 id = contraint_to_pl(rd, cid);
373 rp = rd->rp;
374
375 ret = rapl_write_pl_data(rd, id, PL_LIMIT, power_limit);
376 if (!ret)
377 package_power_limit_irq_save(rp);
378 cpus_read_unlock();
379 return ret;
380}
381
382static int get_current_power_limit(struct powercap_zone *power_zone, int cid,
383 u64 *data)
384{
385 struct rapl_domain *rd;
386 u64 val;
387 int ret = 0;
388 int id;
389
390 cpus_read_lock();
391 rd = power_zone_to_rapl_domain(power_zone);
392 id = contraint_to_pl(rd, cid);
393
394 ret = rapl_read_pl_data(rd, id, PL_LIMIT, true, &val);
395 if (!ret)
396 *data = val;
397
398 cpus_read_unlock();
399
400 return ret;
401}
402
403static int set_time_window(struct powercap_zone *power_zone, int cid,
404 u64 window)
405{
406 struct rapl_domain *rd;
407 int ret = 0;
408 int id;
409
410 cpus_read_lock();
411 rd = power_zone_to_rapl_domain(power_zone);
412 id = contraint_to_pl(rd, cid);
413
414 ret = rapl_write_pl_data(rd, id, PL_TIME_WINDOW, window);
415
416 cpus_read_unlock();
417 return ret;
418}
419
420static int get_time_window(struct powercap_zone *power_zone, int cid,
421 u64 *data)
422{
423 struct rapl_domain *rd;
424 u64 val;
425 int ret = 0;
426 int id;
427
428 cpus_read_lock();
429 rd = power_zone_to_rapl_domain(power_zone);
430 id = contraint_to_pl(rd, cid);
431
432 ret = rapl_read_pl_data(rd, id, PL_TIME_WINDOW, true, &val);
433 if (!ret)
434 *data = val;
435
436 cpus_read_unlock();
437
438 return ret;
439}
440
441static const char *get_constraint_name(struct powercap_zone *power_zone,
442 int cid)
443{
444 struct rapl_domain *rd;
445 int id;
446
447 rd = power_zone_to_rapl_domain(power_zone);
448 id = contraint_to_pl(rd, cid);
449 if (id >= 0)
450 return rd->rpl[id].name;
451
452 return NULL;
453}
454
455static int get_max_power(struct powercap_zone *power_zone, int cid, u64 *data)
456{
457 struct rapl_domain *rd;
458 u64 val;
459 int ret = 0;
460 int id;
461
462 cpus_read_lock();
463 rd = power_zone_to_rapl_domain(power_zone);
464 id = contraint_to_pl(rd, cid);
465
466 ret = rapl_read_pl_data(rd, id, PL_MAX_POWER, true, &val);
467 if (!ret)
468 *data = val;
469
470 /* As a generalization rule, PL4 would be around two times PL2. */
471 if (id == POWER_LIMIT4)
472 *data = *data * 2;
473
474 cpus_read_unlock();
475
476 return ret;
477}
478
479static const struct powercap_zone_constraint_ops constraint_ops = {
480 .set_power_limit_uw = set_power_limit,
481 .get_power_limit_uw = get_current_power_limit,
482 .set_time_window_us = set_time_window,
483 .get_time_window_us = get_time_window,
484 .get_max_power_uw = get_max_power,
485 .get_name = get_constraint_name,
486};
487
488/* Return the id used for read_raw/write_raw callback */
489static int get_rid(struct rapl_package *rp)
490{
491 return rp->lead_cpu >= 0 ? rp->lead_cpu : rp->id;
492}
493
494/* called after domain detection and package level data are set */
495static void rapl_init_domains(struct rapl_package *rp)
496{
497 enum rapl_domain_type i;
498 enum rapl_domain_reg_id j;
499 struct rapl_domain *rd = rp->domains;
500
501 for (i = 0; i < RAPL_DOMAIN_MAX; i++) {
502 unsigned int mask = rp->domain_map & (1 << i);
503 int t;
504
505 if (!mask)
506 continue;
507
508 rd->rp = rp;
509
510 if (i == RAPL_DOMAIN_PLATFORM && rp->id > 0) {
511 snprintf(rd->name, RAPL_DOMAIN_NAME_LENGTH, "psys-%d",
512 rp->lead_cpu >= 0 ? topology_physical_package_id(rp->lead_cpu) :
513 rp->id);
514 } else {
515 snprintf(rd->name, RAPL_DOMAIN_NAME_LENGTH, "%s",
516 rapl_domain_names[i]);
517 }
518
519 rd->id = i;
520
521 /* PL1 is supported by default */
522 rp->priv->limits[i] |= BIT(POWER_LIMIT1);
523
524 for (t = POWER_LIMIT1; t < NR_POWER_LIMITS; t++) {
525 if (rp->priv->limits[i] & BIT(t))
526 rd->rpl[t].name = pl_names[t];
527 }
528
529 for (j = 0; j < RAPL_DOMAIN_REG_MAX; j++)
530 rd->regs[j] = rp->priv->regs[i][j];
531
532 rd++;
533 }
534}
535
536static u64 rapl_unit_xlate(struct rapl_domain *rd, enum unit_type type,
537 u64 value, int to_raw)
538{
539 u64 units = 1;
540 const struct rapl_defaults *defaults = get_defaults(rd->rp);
541 u64 scale = 1;
542
543 switch (type) {
544 case POWER_UNIT:
545 units = rd->power_unit;
546 break;
547 case ENERGY_UNIT:
548 scale = ENERGY_UNIT_SCALE;
549 units = rd->energy_unit;
550 break;
551 case TIME_UNIT:
552 return defaults->compute_time_window(rd, value, to_raw);
553 case ARBITRARY_UNIT:
554 default:
555 return value;
556 }
557
558 if (to_raw)
559 return div64_u64(value, units) * scale;
560
561 value *= units;
562
563 return div64_u64(value, scale);
564}
565
566static struct rapl_primitive_info *get_rpi(struct rapl_package *rp, int prim)
567{
568 struct rapl_primitive_info *rpi = rp->priv->rpi;
569
570 if (prim < 0 || prim >= NR_RAPL_PRIMITIVES || !rpi)
571 return NULL;
572
573 return &rpi[prim];
574}
575
576static int rapl_config(struct rapl_package *rp)
577{
578 /* defaults_msr can be NULL on unsupported platforms */
579 if (!rp->priv->defaults || !rp->priv->rpi)
580 return -ENODEV;
581
582 return 0;
583}
584
585static enum rapl_primitives
586prim_fixups(struct rapl_domain *rd, enum rapl_primitives prim)
587{
588 const struct rapl_defaults *defaults = get_defaults(rd->rp);
589
590 if (!defaults->spr_psys_bits)
591 return prim;
592
593 if (rd->id != RAPL_DOMAIN_PLATFORM)
594 return prim;
595
596 switch (prim) {
597 case POWER_LIMIT1:
598 return PSYS_POWER_LIMIT1;
599 case POWER_LIMIT2:
600 return PSYS_POWER_LIMIT2;
601 case PL1_ENABLE:
602 return PSYS_PL1_ENABLE;
603 case PL2_ENABLE:
604 return PSYS_PL2_ENABLE;
605 case TIME_WINDOW1:
606 return PSYS_TIME_WINDOW1;
607 case TIME_WINDOW2:
608 return PSYS_TIME_WINDOW2;
609 default:
610 return prim;
611 }
612}
613
614/* Read primitive data based on its related struct rapl_primitive_info.
615 * if xlate flag is set, return translated data based on data units, i.e.
616 * time, energy, and power.
617 * RAPL MSRs are non-architectual and are laid out not consistently across
618 * domains. Here we use primitive info to allow writing consolidated access
619 * functions.
620 * For a given primitive, it is processed by MSR mask and shift. Unit conversion
621 * is pre-assigned based on RAPL unit MSRs read at init time.
622 * 63-------------------------- 31--------------------------- 0
623 * | xxxxx (mask) |
624 * | |<- shift ----------------|
625 * 63-------------------------- 31--------------------------- 0
626 */
627static int rapl_read_data_raw(struct rapl_domain *rd,
628 enum rapl_primitives prim, bool xlate, u64 *data,
629 bool pmu_ctx)
630{
631 u64 value;
632 enum rapl_primitives prim_fixed = prim_fixups(rd, prim);
633 struct rapl_primitive_info *rpi = get_rpi(rd->rp, prim_fixed);
634 struct reg_action ra;
635
636 if (!rpi || !rpi->name || rpi->flag & RAPL_PRIMITIVE_DUMMY)
637 return -EINVAL;
638
639 ra.reg = rd->regs[rpi->id];
640 if (!ra.reg.val)
641 return -EINVAL;
642
643 ra.mask = rpi->mask;
644
645 if (rd->rp->priv->read_raw(get_rid(rd->rp), &ra, pmu_ctx)) {
646 pr_debug("failed to read reg 0x%llx for %s:%s\n", ra.reg.val, rd->rp->name, rd->name);
647 return -EIO;
648 }
649
650 value = ra.value >> rpi->shift;
651
652 if (xlate)
653 *data = rapl_unit_xlate(rd, rpi->unit, value, 0);
654 else
655 *data = value;
656
657 return 0;
658}
659
660/* Similar use of primitive info in the read counterpart */
661static int rapl_write_data_raw(struct rapl_domain *rd,
662 enum rapl_primitives prim,
663 unsigned long long value)
664{
665 enum rapl_primitives prim_fixed = prim_fixups(rd, prim);
666 struct rapl_primitive_info *rpi = get_rpi(rd->rp, prim_fixed);
667 u64 bits;
668 struct reg_action ra;
669 int ret;
670
671 if (!rpi || !rpi->name || rpi->flag & RAPL_PRIMITIVE_DUMMY)
672 return -EINVAL;
673
674 bits = rapl_unit_xlate(rd, rpi->unit, value, 1);
675 bits <<= rpi->shift;
676 bits &= rpi->mask;
677
678 memset(&ra, 0, sizeof(ra));
679
680 ra.reg = rd->regs[rpi->id];
681 ra.mask = rpi->mask;
682 ra.value = bits;
683
684 ret = rd->rp->priv->write_raw(get_rid(rd->rp), &ra);
685
686 return ret;
687}
688
689static int rapl_read_pl_data(struct rapl_domain *rd, int pl,
690 enum pl_prims pl_prim, bool xlate, u64 *data)
691{
692 enum rapl_primitives prim = get_pl_prim(rd, pl, pl_prim);
693
694 if (!is_pl_valid(rd, pl))
695 return -EINVAL;
696
697 return rapl_read_data_raw(rd, prim, xlate, data, false);
698}
699
700static int rapl_write_pl_data(struct rapl_domain *rd, int pl,
701 enum pl_prims pl_prim,
702 unsigned long long value)
703{
704 enum rapl_primitives prim = get_pl_prim(rd, pl, pl_prim);
705
706 if (!is_pl_valid(rd, pl))
707 return -EINVAL;
708
709 if (rd->rpl[pl].locked) {
710 pr_debug("%s:%s:%s locked by BIOS\n", rd->rp->name, rd->name, pl_names[pl]);
711 return -EACCES;
712 }
713
714 return rapl_write_data_raw(rd, prim, value);
715}
716/*
717 * Raw RAPL data stored in MSRs are in certain scales. We need to
718 * convert them into standard units based on the units reported in
719 * the RAPL unit MSRs. This is specific to CPUs as the method to
720 * calculate units differ on different CPUs.
721 * We convert the units to below format based on CPUs.
722 * i.e.
723 * energy unit: picoJoules : Represented in picoJoules by default
724 * power unit : microWatts : Represented in milliWatts by default
725 * time unit : microseconds: Represented in seconds by default
726 */
727int rapl_default_check_unit(struct rapl_domain *rd)
728{
729 struct reg_action ra;
730 u32 value;
731
732 ra.reg = rd->regs[RAPL_DOMAIN_REG_UNIT];
733 ra.mask = ~0;
734 if (rd->rp->priv->read_raw(get_rid(rd->rp), &ra, false)) {
735 pr_err("Failed to read power unit REG 0x%llx on %s:%s, exit.\n",
736 ra.reg.val, rd->rp->name, rd->name);
737 return -ENODEV;
738 }
739
740 value = (ra.value & ENERGY_UNIT_MASK) >> ENERGY_UNIT_OFFSET;
741 rd->energy_unit = (ENERGY_UNIT_SCALE * MICROJOULE_PER_JOULE) >> value;
742
743 value = (ra.value & POWER_UNIT_MASK) >> POWER_UNIT_OFFSET;
744 rd->power_unit = MICROWATT_PER_WATT >> value;
745
746 value = (ra.value & TIME_UNIT_MASK) >> TIME_UNIT_OFFSET;
747 rd->time_unit = USEC_PER_SEC >> value;
748
749 pr_debug("Core CPU %s:%s energy=%dpJ, time=%dus, power=%duW\n",
750 rd->rp->name, rd->name, rd->energy_unit, rd->time_unit, rd->power_unit);
751
752 return 0;
753}
754EXPORT_SYMBOL_NS_GPL(rapl_default_check_unit, "INTEL_RAPL");
755
756static void power_limit_irq_save_cpu(void *info)
757{
758 struct msr val;
759 struct rapl_package *rp = (struct rapl_package *)info;
760
761 /* save the state of PLN irq mask bit before disabling it */
762 rdmsrq_safe(MSR_IA32_PACKAGE_THERM_INTERRUPT, &val.q);
763 if (!(rp->power_limit_irq & PACKAGE_PLN_INT_SAVED)) {
764 rp->power_limit_irq = val.l & PACKAGE_THERM_INT_PLN_ENABLE;
765 rp->power_limit_irq |= PACKAGE_PLN_INT_SAVED;
766 }
767 val.l &= ~PACKAGE_THERM_INT_PLN_ENABLE;
768 wrmsrq_safe(MSR_IA32_PACKAGE_THERM_INTERRUPT, val.q);
769}
770
771/* REVISIT:
772 * When package power limit is set artificially low by RAPL, LVT
773 * thermal interrupt for package power limit should be ignored
774 * since we are not really exceeding the real limit. The intention
775 * is to avoid excessive interrupts while we are trying to save power.
776 * A useful feature might be routing the package_power_limit interrupt
777 * to userspace via eventfd. once we have a usecase, this is simple
778 * to do by adding an atomic notifier.
779 */
780
781static void package_power_limit_irq_save(struct rapl_package *rp)
782{
783 if (rp->lead_cpu < 0)
784 return;
785
786 if (!boot_cpu_has(X86_FEATURE_PTS) || !boot_cpu_has(X86_FEATURE_PLN))
787 return;
788
789 smp_call_function_single(rp->lead_cpu, power_limit_irq_save_cpu, rp, 1);
790}
791
792/*
793 * Restore per package power limit interrupt enable state. Called from cpu
794 * hotplug code on package removal.
795 */
796static void package_power_limit_irq_restore(struct rapl_package *rp)
797{
798 struct msr val;
799
800 if (rp->lead_cpu < 0)
801 return;
802
803 if (!boot_cpu_has(X86_FEATURE_PTS) || !boot_cpu_has(X86_FEATURE_PLN))
804 return;
805
806 /* irq enable state not saved, nothing to restore */
807 if (!(rp->power_limit_irq & PACKAGE_PLN_INT_SAVED))
808 return;
809
810 rdmsrq_safe(MSR_IA32_PACKAGE_THERM_INTERRUPT, &val.q);
811
812 if (rp->power_limit_irq & PACKAGE_THERM_INT_PLN_ENABLE)
813 val.l |= PACKAGE_THERM_INT_PLN_ENABLE;
814 else
815 val.l &= ~PACKAGE_THERM_INT_PLN_ENABLE;
816
817 wrmsrq_safe(MSR_IA32_PACKAGE_THERM_INTERRUPT, val.q);
818}
819
820void rapl_default_set_floor_freq(struct rapl_domain *rd, bool mode)
821{
822 int i;
823
824 /* always enable clamp such that p-state can go below OS requested
825 * range. power capping priority over guranteed frequency.
826 */
827 rapl_write_pl_data(rd, POWER_LIMIT1, PL_CLAMP, mode);
828
829 for (i = POWER_LIMIT2; i < NR_POWER_LIMITS; i++) {
830 rapl_write_pl_data(rd, i, PL_ENABLE, mode);
831 rapl_write_pl_data(rd, i, PL_CLAMP, mode);
832 }
833}
834EXPORT_SYMBOL_NS_GPL(rapl_default_set_floor_freq, "INTEL_RAPL");
835
836u64 rapl_default_compute_time_window(struct rapl_domain *rd, u64 value, bool to_raw)
837{
838 u64 f, y; /* fraction and exp. used for time unit */
839
840 /*
841 * Special processing based on 2^Y*(1+F/4), refer
842 * to Intel Software Developer's manual Vol.3B: CH 14.9.3.
843 */
844 if (!to_raw) {
845 f = (value & 0x60) >> 5;
846 y = value & 0x1f;
847 value = (1ULL << y) * (4 + f) * rd->time_unit / 4;
848 } else {
849 if (value < rd->time_unit)
850 return 0;
851
852 do_div(value, rd->time_unit);
853 y = ilog2(value);
854
855 /*
856 * The target hardware field is 7 bits wide, so return all ones
857 * if the exponent is too large.
858 */
859 if (y > 0x1f)
860 return 0x7f;
861
862 f = div64_u64(4 * (value - BIT_ULL(y)), BIT_ULL(y));
863 value = (y & 0x1f) | ((f & 0x3) << 5);
864 }
865 return value;
866}
867EXPORT_SYMBOL_NS_GPL(rapl_default_compute_time_window, "INTEL_RAPL");
868
869/* Read once for all raw primitive data for domains */
870static void rapl_update_domain_data(struct rapl_package *rp)
871{
872 int dmn, prim;
873 u64 val;
874
875 for (dmn = 0; dmn < rp->nr_domains; dmn++) {
876 pr_debug("update %s domain %s data\n", rp->name,
877 rp->domains[dmn].name);
878 /* exclude non-raw primitives */
879 for (prim = 0; prim < NR_RAW_PRIMITIVES; prim++) {
880 struct rapl_primitive_info *rpi = get_rpi(rp, prim);
881
882 if (!rapl_read_data_raw(&rp->domains[dmn], prim,
883 rpi->unit, &val, false))
884 rp->domains[dmn].rdd.primitives[prim] = val;
885 }
886 }
887
888}
889
890static int rapl_package_register_powercap(struct rapl_package *rp)
891{
892 struct rapl_domain *rd;
893 struct powercap_zone *power_zone = NULL;
894 int nr_pl, ret;
895
896 /* Update the domain data of the new package */
897 rapl_update_domain_data(rp);
898
899 /* first we register package domain as the parent zone */
900 for (rd = rp->domains; rd < rp->domains + rp->nr_domains; rd++) {
901 if (rd->id == RAPL_DOMAIN_PACKAGE) {
902 nr_pl = find_nr_power_limit(rd);
903 pr_debug("register package domain %s\n", rp->name);
904 power_zone = powercap_register_zone(&rd->power_zone,
905 rp->priv->control_type, rp->name,
906 NULL, &zone_ops[rd->id], nr_pl,
907 &constraint_ops);
908 if (IS_ERR(power_zone)) {
909 pr_debug("failed to register power zone %s\n",
910 rp->name);
911 return PTR_ERR(power_zone);
912 }
913 /* track parent zone in per package/socket data */
914 rp->power_zone = power_zone;
915 /* done, only one package domain per socket */
916 break;
917 }
918 }
919 if (!power_zone) {
920 pr_err("no package domain found, unknown topology!\n");
921 return -ENODEV;
922 }
923 /* now register domains as children of the socket/package */
924 for (rd = rp->domains; rd < rp->domains + rp->nr_domains; rd++) {
925 struct powercap_zone *parent = rp->power_zone;
926
927 if (rd->id == RAPL_DOMAIN_PACKAGE)
928 continue;
929 if (rd->id == RAPL_DOMAIN_PLATFORM)
930 parent = NULL;
931 /* number of power limits per domain varies */
932 nr_pl = find_nr_power_limit(rd);
933 power_zone = powercap_register_zone(&rd->power_zone,
934 rp->priv->control_type,
935 rd->name, parent,
936 &zone_ops[rd->id], nr_pl,
937 &constraint_ops);
938
939 if (IS_ERR(power_zone)) {
940 pr_debug("failed to register power_zone, %s:%s\n",
941 rp->name, rd->name);
942 ret = PTR_ERR(power_zone);
943 goto err_cleanup;
944 }
945 }
946 return 0;
947
948err_cleanup:
949 /*
950 * Clean up previously initialized domains within the package if we
951 * failed after the first domain setup.
952 */
953 while (--rd >= rp->domains) {
954 pr_debug("unregister %s domain %s\n", rp->name, rd->name);
955 powercap_unregister_zone(rp->priv->control_type,
956 &rd->power_zone);
957 }
958
959 return ret;
960}
961
962static int rapl_check_domain(int domain, struct rapl_package *rp)
963{
964 struct reg_action ra;
965
966 switch (domain) {
967 case RAPL_DOMAIN_PACKAGE:
968 case RAPL_DOMAIN_PP0:
969 case RAPL_DOMAIN_PP1:
970 case RAPL_DOMAIN_DRAM:
971 case RAPL_DOMAIN_PLATFORM:
972 ra.reg = rp->priv->regs[domain][RAPL_DOMAIN_REG_STATUS];
973 break;
974 default:
975 pr_err("invalid domain id %d\n", domain);
976 return -EINVAL;
977 }
978 /* make sure domain counters are available and contains non-zero
979 * values, otherwise skip it.
980 */
981
982 ra.mask = ENERGY_STATUS_MASK;
983 if (rp->priv->read_raw(get_rid(rp), &ra, false) || !ra.value)
984 return -ENODEV;
985
986 return 0;
987}
988
989/*
990 * Get per domain energy/power/time unit.
991 * RAPL Interfaces without per domain unit register will use the package
992 * scope unit register to set per domain units.
993 */
994static int rapl_get_domain_unit(struct rapl_domain *rd)
995{
996 const struct rapl_defaults *defaults = get_defaults(rd->rp);
997 int ret;
998
999 if (!rd->regs[RAPL_DOMAIN_REG_UNIT].val) {
1000 if (!rd->rp->priv->reg_unit.val) {
1001 pr_err("No valid Unit register found\n");
1002 return -ENODEV;
1003 }
1004 rd->regs[RAPL_DOMAIN_REG_UNIT] = rd->rp->priv->reg_unit;
1005 }
1006
1007 if (!defaults->check_unit) {
1008 pr_err("missing .check_unit() callback\n");
1009 return -ENODEV;
1010 }
1011
1012 ret = defaults->check_unit(rd);
1013 if (ret)
1014 return ret;
1015
1016 if (rd->id == RAPL_DOMAIN_DRAM && defaults->dram_domain_energy_unit)
1017 rd->energy_unit = defaults->dram_domain_energy_unit;
1018 if (rd->id == RAPL_DOMAIN_PLATFORM && defaults->psys_domain_energy_unit)
1019 rd->energy_unit = defaults->psys_domain_energy_unit;
1020 return 0;
1021}
1022
1023/*
1024 * Check if power limits are available. Two cases when they are not available:
1025 * 1. Locked by BIOS, in this case we still provide read-only access so that
1026 * users can see what limit is set by the BIOS.
1027 * 2. Some CPUs make some domains monitoring only which means PLx MSRs may not
1028 * exist at all. In this case, we do not show the constraints in powercap.
1029 *
1030 * Called after domains are detected and initialized.
1031 */
1032static void rapl_detect_powerlimit(struct rapl_domain *rd)
1033{
1034 u64 val64;
1035 int i;
1036
1037 for (i = POWER_LIMIT1; i < NR_POWER_LIMITS; i++) {
1038 if (!rapl_read_pl_data(rd, i, PL_LOCK, false, &val64)) {
1039 if (val64) {
1040 rd->rpl[i].locked = true;
1041 pr_info("%s:%s:%s locked by BIOS\n",
1042 rd->rp->name, rd->name, pl_names[i]);
1043 }
1044 }
1045
1046 if (rapl_read_pl_data(rd, i, PL_LIMIT, false, &val64))
1047 rd->rpl[i].name = NULL;
1048 }
1049}
1050
1051/* Detect active and valid domains for the given CPU, caller must
1052 * ensure the CPU belongs to the targeted package and CPU hotlug is disabled.
1053 */
1054static int rapl_detect_domains(struct rapl_package *rp)
1055{
1056 struct rapl_domain *rd;
1057 int i;
1058
1059 for (i = 0; i < RAPL_DOMAIN_MAX; i++) {
1060 /* use physical package id to read counters */
1061 if (!rapl_check_domain(i, rp)) {
1062 rp->domain_map |= 1 << i;
1063 pr_info("Found RAPL domain %s\n", rapl_domain_names[i]);
1064 }
1065 }
1066 rp->nr_domains = bitmap_weight(&rp->domain_map, RAPL_DOMAIN_MAX);
1067 if (!rp->nr_domains) {
1068 pr_debug("no valid rapl domains found in %s\n", rp->name);
1069 return -ENODEV;
1070 }
1071 pr_debug("found %d domains on %s\n", rp->nr_domains, rp->name);
1072
1073 rp->domains = kzalloc_objs(struct rapl_domain, rp->nr_domains);
1074 if (!rp->domains)
1075 return -ENOMEM;
1076
1077 rapl_init_domains(rp);
1078
1079 for (rd = rp->domains; rd < rp->domains + rp->nr_domains; rd++) {
1080 rapl_get_domain_unit(rd);
1081 rapl_detect_powerlimit(rd);
1082 }
1083
1084 return 0;
1085}
1086
1087#ifdef CONFIG_PERF_EVENTS
1088
1089/*
1090 * Support for RAPL PMU
1091 *
1092 * Register a PMU if any of the registered RAPL Packages have the requirement
1093 * of exposing its energy counters via Perf PMU.
1094 *
1095 * PMU Name:
1096 * power
1097 *
1098 * Events:
1099 * Name Event id RAPL Domain
1100 * energy_cores 0x01 RAPL_DOMAIN_PP0
1101 * energy_pkg 0x02 RAPL_DOMAIN_PACKAGE
1102 * energy_ram 0x03 RAPL_DOMAIN_DRAM
1103 * energy_gpu 0x04 RAPL_DOMAIN_PP1
1104 * energy_psys 0x05 RAPL_DOMAIN_PLATFORM
1105 *
1106 * Unit:
1107 * Joules
1108 *
1109 * Scale:
1110 * 2.3283064365386962890625e-10
1111 * The same RAPL domain in different RAPL Packages may have different
1112 * energy units. Use 2.3283064365386962890625e-10 (2^-32) Joules as
1113 * the fixed unit for all energy counters, and covert each hardware
1114 * counter increase to N times of PMU event counter increases.
1115 *
1116 * This is fully compatible with the current MSR RAPL PMU. This means that
1117 * userspace programs like turbostat can use the same code to handle RAPL Perf
1118 * PMU, no matter what RAPL Interface driver (MSR/TPMI, etc) is running
1119 * underlying on the platform.
1120 *
1121 * Note that RAPL Packages can be probed/removed dynamically, and the events
1122 * supported by each TPMI RAPL device can be different. Thus the RAPL PMU
1123 * support is done on demand, which means
1124 * 1. PMU is registered only if it is needed by a RAPL Package. PMU events for
1125 * unsupported counters are not exposed.
1126 * 2. PMU is unregistered and registered when a new RAPL Package is probed and
1127 * supports new counters that are not supported by current PMU.
1128 * 3. PMU is unregistered when all registered RAPL Packages don't need PMU.
1129 */
1130
1131struct rapl_pmu {
1132 struct pmu pmu; /* Perf PMU structure */
1133 u64 timer_ms; /* Maximum expiration time to avoid counter overflow */
1134 unsigned long domain_map; /* Events supported by current registered PMU */
1135 bool registered; /* Whether the PMU has been registered or not */
1136};
1137
1138static struct rapl_pmu rapl_pmu;
1139
1140/* PMU helpers */
1141
1142static void set_pmu_cpumask(struct rapl_package *rp, cpumask_var_t mask)
1143{
1144 int cpu;
1145
1146 if (!rp->has_pmu)
1147 return;
1148
1149 /* Only TPMI & MSR RAPL are supported for now */
1150 if (rp->priv->type != RAPL_IF_TPMI && rp->priv->type != RAPL_IF_MSR)
1151 return;
1152
1153 /* TPMI/MSR RAPL uses any CPU in the package for PMU */
1154 for_each_online_cpu(cpu)
1155 if (topology_physical_package_id(cpu) == rp->id)
1156 cpumask_set_cpu(cpu, mask);
1157}
1158
1159static bool is_rp_pmu_cpu(struct rapl_package *rp, int cpu)
1160{
1161 if (!rp->has_pmu)
1162 return false;
1163
1164 /* Only TPMI & MSR RAPL are supported for now */
1165 if (rp->priv->type != RAPL_IF_TPMI && rp->priv->type != RAPL_IF_MSR)
1166 return false;
1167
1168 /* TPMI/MSR RAPL uses any CPU in the package for PMU */
1169 return topology_physical_package_id(cpu) == rp->id;
1170}
1171
1172static struct rapl_package_pmu_data *event_to_pmu_data(struct perf_event *event)
1173{
1174 struct rapl_package *rp = event->pmu_private;
1175
1176 return &rp->pmu_data;
1177}
1178
1179/* PMU event callbacks */
1180
1181static u64 event_read_counter(struct perf_event *event)
1182{
1183 struct rapl_package *rp = event->pmu_private;
1184 u64 val;
1185 int ret;
1186
1187 /* Return 0 for unsupported events */
1188 if (event->hw.idx < 0)
1189 return 0;
1190
1191 ret = rapl_read_data_raw(&rp->domains[event->hw.idx], ENERGY_COUNTER, false, &val, true);
1192
1193 /* Return 0 for failed read */
1194 if (ret)
1195 return 0;
1196
1197 return val;
1198}
1199
1200static void __rapl_pmu_event_start(struct perf_event *event)
1201{
1202 struct rapl_package_pmu_data *data = event_to_pmu_data(event);
1203
1204 if (WARN_ON_ONCE(!(event->hw.state & PERF_HES_STOPPED)))
1205 return;
1206
1207 event->hw.state = 0;
1208
1209 list_add_tail(&event->active_entry, &data->active_list);
1210
1211 local64_set(&event->hw.prev_count, event_read_counter(event));
1212 if (++data->n_active == 1)
1213 hrtimer_start(&data->hrtimer, data->timer_interval,
1214 HRTIMER_MODE_REL_PINNED);
1215}
1216
1217static void rapl_pmu_event_start(struct perf_event *event, int mode)
1218{
1219 struct rapl_package_pmu_data *data = event_to_pmu_data(event);
1220 unsigned long flags;
1221
1222 raw_spin_lock_irqsave(&data->lock, flags);
1223 __rapl_pmu_event_start(event);
1224 raw_spin_unlock_irqrestore(&data->lock, flags);
1225}
1226
1227static u64 rapl_event_update(struct perf_event *event)
1228{
1229 struct hw_perf_event *hwc = &event->hw;
1230 struct rapl_package_pmu_data *data = event_to_pmu_data(event);
1231 u64 prev_raw_count, new_raw_count;
1232 s64 delta, sdelta;
1233 int shift = 64 - RAPL_CNTR_WIDTH;
1234
1235 /*
1236 * Follow the generic code to drain hwc->prev_count.
1237 * The loop is not expected to run for multiple times.
1238 */
1239 prev_raw_count = local64_read(&hwc->prev_count);
1240 do {
1241 new_raw_count = event_read_counter(event);
1242 } while (!local64_try_cmpxchg(&hwc->prev_count,
1243 &prev_raw_count, new_raw_count));
1244
1245
1246 /*
1247 * Now we have the new raw value and have updated the prev
1248 * timestamp already. We can now calculate the elapsed delta
1249 * (event-)time and add that to the generic event.
1250 *
1251 * Careful, the counter is narrower than u64 and is not
1252 * sign-extended above its physical width. Shift both values up
1253 * so that the subtraction wraps, then shift the result back down.
1254 */
1255 delta = (new_raw_count << shift) - (prev_raw_count << shift);
1256 delta >>= shift;
1257
1258 /*
1259 * Scale delta to smallest unit (2^-32)
1260 * users must then scale back: count * 1/(1e9*2^32) to get Joules
1261 * or use ldexp(count, -32).
1262 * Watts = Joules/Time delta
1263 */
1264 sdelta = delta * data->scale[event->hw.flags];
1265
1266 local64_add(sdelta, &event->count);
1267
1268 return new_raw_count;
1269}
1270
1271static void rapl_pmu_event_stop(struct perf_event *event, int mode)
1272{
1273 struct rapl_package_pmu_data *data = event_to_pmu_data(event);
1274 struct hw_perf_event *hwc = &event->hw;
1275 unsigned long flags;
1276
1277 raw_spin_lock_irqsave(&data->lock, flags);
1278
1279 /* Mark event as deactivated and stopped */
1280 if (!(hwc->state & PERF_HES_STOPPED)) {
1281 WARN_ON_ONCE(data->n_active <= 0);
1282 if (--data->n_active == 0)
1283 hrtimer_cancel(&data->hrtimer);
1284
1285 list_del(&event->active_entry);
1286
1287 WARN_ON_ONCE(hwc->state & PERF_HES_STOPPED);
1288 hwc->state |= PERF_HES_STOPPED;
1289 }
1290
1291 /* Check if update of sw counter is necessary */
1292 if ((mode & PERF_EF_UPDATE) && !(hwc->state & PERF_HES_UPTODATE)) {
1293 /*
1294 * Drain the remaining delta count out of a event
1295 * that we are disabling:
1296 */
1297 rapl_event_update(event);
1298 hwc->state |= PERF_HES_UPTODATE;
1299 }
1300
1301 raw_spin_unlock_irqrestore(&data->lock, flags);
1302}
1303
1304static int rapl_pmu_event_add(struct perf_event *event, int mode)
1305{
1306 struct rapl_package_pmu_data *data = event_to_pmu_data(event);
1307 struct hw_perf_event *hwc = &event->hw;
1308 unsigned long flags;
1309
1310 raw_spin_lock_irqsave(&data->lock, flags);
1311
1312 hwc->state = PERF_HES_UPTODATE | PERF_HES_STOPPED;
1313
1314 if (mode & PERF_EF_START)
1315 __rapl_pmu_event_start(event);
1316
1317 raw_spin_unlock_irqrestore(&data->lock, flags);
1318
1319 return 0;
1320}
1321
1322static void rapl_pmu_event_del(struct perf_event *event, int flags)
1323{
1324 rapl_pmu_event_stop(event, PERF_EF_UPDATE);
1325}
1326
1327/* RAPL PMU event ids, same as shown in sysfs */
1328enum perf_rapl_events {
1329 PERF_RAPL_PP0 = 1, /* all cores */
1330 PERF_RAPL_PKG, /* entire package */
1331 PERF_RAPL_RAM, /* DRAM */
1332 PERF_RAPL_PP1, /* gpu */
1333 PERF_RAPL_PSYS, /* psys */
1334 PERF_RAPL_MAX
1335};
1336
1337static const int event_to_domain[PERF_RAPL_MAX] = {
1338 [PERF_RAPL_PP0] = RAPL_DOMAIN_PP0,
1339 [PERF_RAPL_PKG] = RAPL_DOMAIN_PACKAGE,
1340 [PERF_RAPL_RAM] = RAPL_DOMAIN_DRAM,
1341 [PERF_RAPL_PP1] = RAPL_DOMAIN_PP1,
1342 [PERF_RAPL_PSYS] = RAPL_DOMAIN_PLATFORM,
1343};
1344
1345static int rapl_pmu_event_init(struct perf_event *event)
1346{
1347 struct rapl_package *pos, *rp = NULL;
1348 u64 cfg = event->attr.config & RAPL_EVENT_MASK;
1349 int domain, idx;
1350
1351 /* Only look at RAPL events */
1352 if (event->attr.type != event->pmu->type)
1353 return -ENOENT;
1354
1355 /* Check for supported events only */
1356 if (!cfg || cfg >= PERF_RAPL_MAX)
1357 return -EINVAL;
1358
1359 if (event->cpu < 0)
1360 return -EINVAL;
1361
1362 /* Find out which Package the event belongs to */
1363 list_for_each_entry(pos, &rapl_packages, plist) {
1364 if (is_rp_pmu_cpu(pos, event->cpu)) {
1365 rp = pos;
1366 break;
1367 }
1368 }
1369 if (!rp)
1370 return -ENODEV;
1371
1372 /* Find out which RAPL Domain the event belongs to */
1373 domain = event_to_domain[cfg];
1374
1375 event->event_caps |= PERF_EV_CAP_READ_ACTIVE_PKG;
1376 event->pmu_private = rp; /* Which package */
1377 event->hw.flags = domain; /* Which domain */
1378
1379 event->hw.idx = -1;
1380 /* Find out the index in rp->domains[] to get domain pointer */
1381 for (idx = 0; idx < rp->nr_domains; idx++) {
1382 if (rp->domains[idx].id == domain) {
1383 event->hw.idx = idx;
1384 break;
1385 }
1386 }
1387
1388 return 0;
1389}
1390
1391static void rapl_pmu_event_read(struct perf_event *event)
1392{
1393 rapl_event_update(event);
1394}
1395
1396static enum hrtimer_restart rapl_hrtimer_handle(struct hrtimer *hrtimer)
1397{
1398 struct rapl_package_pmu_data *data =
1399 container_of(hrtimer, struct rapl_package_pmu_data, hrtimer);
1400 struct perf_event *event;
1401 unsigned long flags;
1402
1403 if (!data->n_active)
1404 return HRTIMER_NORESTART;
1405
1406 raw_spin_lock_irqsave(&data->lock, flags);
1407
1408 list_for_each_entry(event, &data->active_list, active_entry)
1409 rapl_event_update(event);
1410
1411 raw_spin_unlock_irqrestore(&data->lock, flags);
1412
1413 hrtimer_forward_now(hrtimer, data->timer_interval);
1414
1415 return HRTIMER_RESTART;
1416}
1417
1418/* PMU sysfs attributes */
1419
1420/*
1421 * There are no default events, but we need to create "events" group (with
1422 * empty attrs) before updating it with detected events.
1423 */
1424static struct attribute *attrs_empty[] = {
1425 NULL,
1426};
1427
1428static struct attribute_group pmu_events_group = {
1429 .name = "events",
1430 .attrs = attrs_empty,
1431};
1432
1433static ssize_t cpumask_show(struct device *dev,
1434 struct device_attribute *attr, char *buf)
1435{
1436 struct rapl_package *rp;
1437 cpumask_var_t cpu_mask;
1438 int ret;
1439
1440 if (!alloc_cpumask_var(&cpu_mask, GFP_KERNEL))
1441 return -ENOMEM;
1442
1443 cpus_read_lock();
1444
1445 cpumask_clear(cpu_mask);
1446
1447 /* Choose a cpu for each RAPL Package */
1448 list_for_each_entry(rp, &rapl_packages, plist) {
1449 set_pmu_cpumask(rp, cpu_mask);
1450 }
1451 cpus_read_unlock();
1452
1453 ret = sysfs_emit(buf, "%*pbl\n", cpumask_pr_args(cpu_mask));
1454
1455 free_cpumask_var(cpu_mask);
1456
1457 return ret;
1458}
1459
1460static DEVICE_ATTR_RO(cpumask);
1461
1462static struct attribute *pmu_cpumask_attrs[] = {
1463 &dev_attr_cpumask.attr,
1464 NULL
1465};
1466
1467static struct attribute_group pmu_cpumask_group = {
1468 .attrs = pmu_cpumask_attrs,
1469};
1470
1471PMU_FORMAT_ATTR(event, "config:0-7");
1472static struct attribute *pmu_format_attr[] = {
1473 &format_attr_event.attr,
1474 NULL
1475};
1476
1477static struct attribute_group pmu_format_group = {
1478 .name = "format",
1479 .attrs = pmu_format_attr,
1480};
1481
1482static const struct attribute_group *pmu_attr_groups[] = {
1483 &pmu_events_group,
1484 &pmu_cpumask_group,
1485 &pmu_format_group,
1486 NULL
1487};
1488
1489#define RAPL_EVENT_ATTR_STR(_name, v, str) \
1490static struct perf_pmu_events_attr event_attr_##v = { \
1491 .attr = __ATTR(_name, 0444, perf_event_sysfs_show, NULL), \
1492 .event_str = str, \
1493}
1494
1495RAPL_EVENT_ATTR_STR(energy-cores, rapl_cores, "event=0x01");
1496RAPL_EVENT_ATTR_STR(energy-pkg, rapl_pkg, "event=0x02");
1497RAPL_EVENT_ATTR_STR(energy-ram, rapl_ram, "event=0x03");
1498RAPL_EVENT_ATTR_STR(energy-gpu, rapl_gpu, "event=0x04");
1499RAPL_EVENT_ATTR_STR(energy-psys, rapl_psys, "event=0x05");
1500
1501RAPL_EVENT_ATTR_STR(energy-cores.unit, rapl_unit_cores, "Joules");
1502RAPL_EVENT_ATTR_STR(energy-pkg.unit, rapl_unit_pkg, "Joules");
1503RAPL_EVENT_ATTR_STR(energy-ram.unit, rapl_unit_ram, "Joules");
1504RAPL_EVENT_ATTR_STR(energy-gpu.unit, rapl_unit_gpu, "Joules");
1505RAPL_EVENT_ATTR_STR(energy-psys.unit, rapl_unit_psys, "Joules");
1506
1507RAPL_EVENT_ATTR_STR(energy-cores.scale, rapl_scale_cores, "2.3283064365386962890625e-10");
1508RAPL_EVENT_ATTR_STR(energy-pkg.scale, rapl_scale_pkg, "2.3283064365386962890625e-10");
1509RAPL_EVENT_ATTR_STR(energy-ram.scale, rapl_scale_ram, "2.3283064365386962890625e-10");
1510RAPL_EVENT_ATTR_STR(energy-gpu.scale, rapl_scale_gpu, "2.3283064365386962890625e-10");
1511RAPL_EVENT_ATTR_STR(energy-psys.scale, rapl_scale_psys, "2.3283064365386962890625e-10");
1512
1513#define RAPL_EVENT_GROUP(_name, domain) \
1514static struct attribute *pmu_attr_##_name[] = { \
1515 &event_attr_rapl_##_name.attr.attr, \
1516 &event_attr_rapl_unit_##_name.attr.attr, \
1517 &event_attr_rapl_scale_##_name.attr.attr, \
1518 NULL \
1519}; \
1520static umode_t is_visible_##_name(struct kobject *kobj, struct attribute *attr, int event) \
1521{ \
1522 return rapl_pmu.domain_map & BIT(domain) ? attr->mode : 0; \
1523} \
1524static struct attribute_group pmu_group_##_name = { \
1525 .name = "events", \
1526 .attrs = pmu_attr_##_name, \
1527 .is_visible = is_visible_##_name, \
1528}
1529
1530RAPL_EVENT_GROUP(cores, RAPL_DOMAIN_PP0);
1531RAPL_EVENT_GROUP(pkg, RAPL_DOMAIN_PACKAGE);
1532RAPL_EVENT_GROUP(ram, RAPL_DOMAIN_DRAM);
1533RAPL_EVENT_GROUP(gpu, RAPL_DOMAIN_PP1);
1534RAPL_EVENT_GROUP(psys, RAPL_DOMAIN_PLATFORM);
1535
1536static const struct attribute_group *pmu_attr_update[] = {
1537 &pmu_group_cores,
1538 &pmu_group_pkg,
1539 &pmu_group_ram,
1540 &pmu_group_gpu,
1541 &pmu_group_psys,
1542 NULL
1543};
1544
1545static int rapl_pmu_update(struct rapl_package *rp)
1546{
1547 int ret = 0;
1548
1549 /* Return if PMU already covers all events supported by current RAPL Package */
1550 if (rapl_pmu.registered && !(rp->domain_map & (~rapl_pmu.domain_map)))
1551 goto end;
1552
1553 /* Unregister previous registered PMU */
1554 if (rapl_pmu.registered)
1555 perf_pmu_unregister(&rapl_pmu.pmu);
1556
1557 rapl_pmu.registered = false;
1558 rapl_pmu.domain_map |= rp->domain_map;
1559
1560 memset(&rapl_pmu.pmu, 0, sizeof(struct pmu));
1561 rapl_pmu.pmu.attr_groups = pmu_attr_groups;
1562 rapl_pmu.pmu.attr_update = pmu_attr_update;
1563 rapl_pmu.pmu.task_ctx_nr = perf_invalid_context;
1564 rapl_pmu.pmu.event_init = rapl_pmu_event_init;
1565 rapl_pmu.pmu.add = rapl_pmu_event_add;
1566 rapl_pmu.pmu.del = rapl_pmu_event_del;
1567 rapl_pmu.pmu.start = rapl_pmu_event_start;
1568 rapl_pmu.pmu.stop = rapl_pmu_event_stop;
1569 rapl_pmu.pmu.read = rapl_pmu_event_read;
1570 rapl_pmu.pmu.module = THIS_MODULE;
1571 rapl_pmu.pmu.capabilities = PERF_PMU_CAP_NO_EXCLUDE | PERF_PMU_CAP_NO_INTERRUPT;
1572 ret = perf_pmu_register(&rapl_pmu.pmu, "power", -1);
1573 if (ret) {
1574 pr_info("Failed to register PMU\n");
1575 return ret;
1576 }
1577
1578 rapl_pmu.registered = true;
1579end:
1580 rp->has_pmu = true;
1581 return ret;
1582}
1583
1584int rapl_package_add_pmu_locked(struct rapl_package *rp)
1585{
1586 struct rapl_package_pmu_data *data = &rp->pmu_data;
1587 int idx;
1588
1589 if (rp->has_pmu)
1590 return -EEXIST;
1591
1592 for (idx = 0; idx < rp->nr_domains; idx++) {
1593 struct rapl_domain *rd = &rp->domains[idx];
1594 int domain = rd->id;
1595 u64 val;
1596
1597 if (!test_bit(domain, &rp->domain_map))
1598 continue;
1599
1600 /*
1601 * The RAPL PMU granularity is 2^-32 Joules
1602 * data->scale[]: times of 2^-32 Joules for each ENERGY COUNTER increase
1603 */
1604 val = rd->energy_unit * (1ULL << 32);
1605 do_div(val, ENERGY_UNIT_SCALE * 1000000);
1606 data->scale[domain] = val;
1607
1608 if (!rapl_pmu.timer_ms) {
1609 struct rapl_primitive_info *rpi = get_rpi(rp, ENERGY_COUNTER);
1610
1611 /*
1612 * Calculate the timer rate:
1613 * Use reference of 200W for scaling the timeout to avoid counter
1614 * overflows.
1615 *
1616 * max_count = rpi->mask >> rpi->shift + 1
1617 * max_energy_pj = max_count * rd->energy_unit
1618 * max_time_sec = (max_energy_pj / 1000000000) / 200w
1619 *
1620 * rapl_pmu.timer_ms = max_time_sec * 1000 / 2
1621 */
1622 val = (rpi->mask >> rpi->shift) + 1;
1623 val *= rd->energy_unit;
1624 do_div(val, 1000000 * 200 * 2);
1625 rapl_pmu.timer_ms = val;
1626
1627 pr_debug("%llu ms overflow timer\n", rapl_pmu.timer_ms);
1628 }
1629
1630 pr_debug("Domain %s: hw unit %lld * 2^-32 Joules\n", rd->name, data->scale[domain]);
1631 }
1632
1633 /* Initialize per package PMU data */
1634 raw_spin_lock_init(&data->lock);
1635 INIT_LIST_HEAD(&data->active_list);
1636 data->timer_interval = ms_to_ktime(rapl_pmu.timer_ms);
1637 hrtimer_setup(&data->hrtimer, rapl_hrtimer_handle, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
1638
1639 return rapl_pmu_update(rp);
1640}
1641EXPORT_SYMBOL_NS_GPL(rapl_package_add_pmu_locked, "INTEL_RAPL");
1642
1643int rapl_package_add_pmu(struct rapl_package *rp)
1644{
1645 guard(cpus_read_lock)();
1646
1647 return rapl_package_add_pmu_locked(rp);
1648}
1649EXPORT_SYMBOL_NS_GPL(rapl_package_add_pmu, "INTEL_RAPL");
1650
1651void rapl_package_remove_pmu_locked(struct rapl_package *rp)
1652{
1653 struct rapl_package *pos;
1654
1655 if (!rp->has_pmu)
1656 return;
1657
1658 list_for_each_entry(pos, &rapl_packages, plist) {
1659 /* PMU is still needed */
1660 if (pos->has_pmu && pos != rp)
1661 return;
1662 }
1663
1664 if (rapl_pmu.registered)
1665 perf_pmu_unregister(&rapl_pmu.pmu);
1666
1667 memset(&rapl_pmu, 0, sizeof(struct rapl_pmu));
1668}
1669EXPORT_SYMBOL_NS_GPL(rapl_package_remove_pmu_locked, "INTEL_RAPL");
1670
1671void rapl_package_remove_pmu(struct rapl_package *rp)
1672{
1673 guard(cpus_read_lock)();
1674
1675 rapl_package_remove_pmu_locked(rp);
1676}
1677EXPORT_SYMBOL_NS_GPL(rapl_package_remove_pmu, "INTEL_RAPL");
1678#endif
1679
1680/* called from CPU hotplug notifier, hotplug lock held */
1681void rapl_remove_package_cpuslocked(struct rapl_package *rp)
1682{
1683 struct rapl_domain *rd, *rd_package = NULL;
1684
1685 package_power_limit_irq_restore(rp);
1686
1687 for (rd = rp->domains; rd < rp->domains + rp->nr_domains; rd++) {
1688 int i;
1689
1690 for (i = POWER_LIMIT1; i < NR_POWER_LIMITS; i++) {
1691 rapl_write_pl_data(rd, i, PL_ENABLE, 0);
1692 rapl_write_pl_data(rd, i, PL_CLAMP, 0);
1693 }
1694
1695 if (rd->id == RAPL_DOMAIN_PACKAGE) {
1696 rd_package = rd;
1697 continue;
1698 }
1699 pr_debug("remove package, undo power limit on %s: %s\n",
1700 rp->name, rd->name);
1701 powercap_unregister_zone(rp->priv->control_type,
1702 &rd->power_zone);
1703 }
1704 /* do parent zone last */
1705 powercap_unregister_zone(rp->priv->control_type,
1706 &rd_package->power_zone);
1707 list_del(&rp->plist);
1708 kfree(rp);
1709}
1710EXPORT_SYMBOL_NS_GPL(rapl_remove_package_cpuslocked, "INTEL_RAPL");
1711
1712void rapl_remove_package(struct rapl_package *rp)
1713{
1714 guard(cpus_read_lock)();
1715 rapl_remove_package_cpuslocked(rp);
1716}
1717EXPORT_SYMBOL_NS_GPL(rapl_remove_package, "INTEL_RAPL");
1718
1719/*
1720 * RAPL Package energy counter scope:
1721 * 1. AMD/HYGON platforms use per-PKG package energy counter
1722 * 2. For Intel platforms
1723 * 2.1 CLX-AP platform has per-DIE package energy counter
1724 * 2.2 Other platforms that uses MSR RAPL are single die systems so the
1725 * package energy counter can be considered as per-PKG/per-DIE,
1726 * here it is considered as per-DIE.
1727 * 2.3 New platforms that use TPMI RAPL doesn't care about the
1728 * scope because they are not MSR/CPU based.
1729 */
1730#define rapl_msrs_are_pkg_scope() \
1731 (boot_cpu_data.x86_vendor == X86_VENDOR_AMD || \
1732 boot_cpu_data.x86_vendor == X86_VENDOR_HYGON)
1733
1734/* caller to ensure CPU hotplug lock is held */
1735struct rapl_package *rapl_find_package_domain_cpuslocked(int id, struct rapl_if_priv *priv,
1736 bool id_is_cpu)
1737{
1738 struct rapl_package *rp;
1739 int uid;
1740
1741 if (id_is_cpu) {
1742 uid = rapl_msrs_are_pkg_scope() ?
1743 topology_physical_package_id(id) : topology_logical_die_id(id);
1744 if (uid < 0) {
1745 pr_err("topology_logical_(package/die)_id() returned a negative value");
1746 return NULL;
1747 }
1748 }
1749 else
1750 uid = id;
1751
1752 list_for_each_entry(rp, &rapl_packages, plist) {
1753 if (rp->id == uid
1754 && rp->priv->control_type == priv->control_type)
1755 return rp;
1756 }
1757
1758 return NULL;
1759}
1760EXPORT_SYMBOL_NS_GPL(rapl_find_package_domain_cpuslocked, "INTEL_RAPL");
1761
1762struct rapl_package *rapl_find_package_domain(int id, struct rapl_if_priv *priv, bool id_is_cpu)
1763{
1764 guard(cpus_read_lock)();
1765 return rapl_find_package_domain_cpuslocked(id, priv, id_is_cpu);
1766}
1767EXPORT_SYMBOL_NS_GPL(rapl_find_package_domain, "INTEL_RAPL");
1768
1769/* called from CPU hotplug notifier, hotplug lock held */
1770struct rapl_package *rapl_add_package_cpuslocked(int id, struct rapl_if_priv *priv, bool id_is_cpu)
1771{
1772 struct rapl_package *rp;
1773 int ret;
1774
1775 rp = kzalloc_obj(struct rapl_package);
1776 if (!rp)
1777 return ERR_PTR(-ENOMEM);
1778
1779 if (id_is_cpu) {
1780 rp->id = rapl_msrs_are_pkg_scope() ?
1781 topology_physical_package_id(id) : topology_logical_die_id(id);
1782 if ((int)(rp->id) < 0) {
1783 pr_err("topology_logical_(package/die)_id() returned a negative value");
1784 ret = -EINVAL;
1785 goto err_free_package;
1786 }
1787 rp->lead_cpu = id;
1788 if (!rapl_msrs_are_pkg_scope() && topology_max_dies_per_package() > 1)
1789 snprintf(rp->name, PACKAGE_DOMAIN_NAME_LENGTH, "package-%d-die-%d",
1790 topology_physical_package_id(id), topology_die_id(id));
1791 else
1792 snprintf(rp->name, PACKAGE_DOMAIN_NAME_LENGTH, "package-%d",
1793 topology_physical_package_id(id));
1794 } else {
1795 rp->id = id;
1796 rp->lead_cpu = -1;
1797 snprintf(rp->name, PACKAGE_DOMAIN_NAME_LENGTH, "package-%d", id);
1798 }
1799
1800 rp->priv = priv;
1801 ret = rapl_config(rp);
1802 if (ret)
1803 goto err_free_package;
1804
1805 /* check if the package contains valid domains */
1806 if (rapl_detect_domains(rp)) {
1807 ret = -ENODEV;
1808 goto err_free_package;
1809 }
1810 ret = rapl_package_register_powercap(rp);
1811 if (!ret) {
1812 INIT_LIST_HEAD(&rp->plist);
1813 list_add(&rp->plist, &rapl_packages);
1814 return rp;
1815 }
1816
1817err_free_package:
1818 kfree(rp->domains);
1819 kfree(rp);
1820 return ERR_PTR(ret);
1821}
1822EXPORT_SYMBOL_NS_GPL(rapl_add_package_cpuslocked, "INTEL_RAPL");
1823
1824struct rapl_package *rapl_add_package(int id, struct rapl_if_priv *priv, bool id_is_cpu)
1825{
1826 guard(cpus_read_lock)();
1827 return rapl_add_package_cpuslocked(id, priv, id_is_cpu);
1828}
1829EXPORT_SYMBOL_NS_GPL(rapl_add_package, "INTEL_RAPL");
1830
1831static void power_limit_state_save(void)
1832{
1833 struct rapl_package *rp;
1834 struct rapl_domain *rd;
1835 int ret, i;
1836
1837 cpus_read_lock();
1838 list_for_each_entry(rp, &rapl_packages, plist) {
1839 if (!rp->power_zone)
1840 continue;
1841 rd = power_zone_to_rapl_domain(rp->power_zone);
1842 for (i = POWER_LIMIT1; i < NR_POWER_LIMITS; i++) {
1843 ret = rapl_read_pl_data(rd, i, PL_LIMIT, true,
1844 &rd->rpl[i].last_power_limit);
1845 if (ret)
1846 rd->rpl[i].last_power_limit = 0;
1847 }
1848 }
1849 cpus_read_unlock();
1850}
1851
1852static void power_limit_state_restore(void)
1853{
1854 struct rapl_package *rp;
1855 struct rapl_domain *rd;
1856 int i;
1857
1858 cpus_read_lock();
1859 list_for_each_entry(rp, &rapl_packages, plist) {
1860 if (!rp->power_zone)
1861 continue;
1862 rd = power_zone_to_rapl_domain(rp->power_zone);
1863 for (i = POWER_LIMIT1; i < NR_POWER_LIMITS; i++)
1864 if (rd->rpl[i].last_power_limit)
1865 rapl_write_pl_data(rd, i, PL_LIMIT,
1866 rd->rpl[i].last_power_limit);
1867 }
1868 cpus_read_unlock();
1869}
1870
1871static int rapl_pm_callback(struct notifier_block *nb,
1872 unsigned long mode, void *_unused)
1873{
1874 switch (mode) {
1875 case PM_SUSPEND_PREPARE:
1876 power_limit_state_save();
1877 break;
1878 case PM_POST_SUSPEND:
1879 power_limit_state_restore();
1880 break;
1881 }
1882 return NOTIFY_OK;
1883}
1884
1885static struct notifier_block rapl_pm_notifier = {
1886 .notifier_call = rapl_pm_callback,
1887};
1888
1889static int __init rapl_init(void)
1890{
1891 return register_pm_notifier(&rapl_pm_notifier);
1892}
1893
1894static void __exit rapl_exit(void)
1895{
1896 unregister_pm_notifier(&rapl_pm_notifier);
1897}
1898
1899fs_initcall(rapl_init);
1900module_exit(rapl_exit);
1901
1902MODULE_DESCRIPTION("Intel Runtime Average Power Limit (RAPL) common code");
1903MODULE_AUTHOR("Jacob Pan <[email protected]>");
1904MODULE_LICENSE("GPL v2");