| 1 | |
| 2 | |
| 3 | * intel_pstate.c: Native P state management for Intel processors
|
| 4 | *
|
| 5 | * (C) Copyright 2012 Intel Corporation
|
| 6 | * Author: Dirk Brandewie <[email protected]>
|
| 7 |
|
| 8 |
|
| 9 | #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
|
| 10 |
|
| 11 | #include <linux/kernel.h>
|
| 12 | #include <linux/kernel_stat.h>
|
| 13 | #include <linux/module.h>
|
| 14 | #include <linux/ktime.h>
|
| 15 | #include <linux/hrtimer.h>
|
| 16 | #include <linux/tick.h>
|
| 17 | #include <linux/slab.h>
|
| 18 | #include <linux/sched/cpufreq.h>
|
| 19 | #include <linux/sched/smt.h>
|
| 20 | #include <linux/list.h>
|
| 21 | #include <linux/cpu.h>
|
| 22 | #include <linux/cpufreq.h>
|
| 23 | #include <linux/sysfs.h>
|
| 24 | #include <linux/types.h>
|
| 25 | #include <linux/fs.h>
|
| 26 | #include <linux/acpi.h>
|
| 27 | #include <linux/vmalloc.h>
|
| 28 | #include <linux/pm_qos.h>
|
| 29 | #include <linux/bitfield.h>
|
| 30 | #include <trace/events/power.h>
|
| 31 | #include <linux/units.h>
|
| 32 |
|
| 33 | #include <asm/cpu.h>
|
| 34 | #include <asm/div64.h>
|
| 35 | #include <asm/msr.h>
|
| 36 | #include <asm/cpu_device_id.h>
|
| 37 | #include <asm/cpufeature.h>
|
| 38 | #include <asm/intel-family.h>
|
| 39 | #include "../drivers/thermal/intel/thermal_interrupt.h"
|
| 40 |
|
| 41 | #define INTEL_PSTATE_SAMPLING_INTERVAL (10 * NSEC_PER_MSEC)
|
| 42 |
|
| 43 | #define INTEL_CPUFREQ_TRANSITION_LATENCY 20000
|
| 44 | #define INTEL_CPUFREQ_TRANSITION_DELAY_HWP 5000
|
| 45 | #define INTEL_CPUFREQ_TRANSITION_DELAY 500
|
| 46 |
|
| 47 | #ifdef CONFIG_ACPI
|
| 48 | #include <acpi/processor.h>
|
| 49 | #include <acpi/cppc_acpi.h>
|
| 50 | #endif
|
| 51 |
|
| 52 | #define FRAC_BITS 8
|
| 53 | #define int_tofp(X) ((int64_t)(X) << FRAC_BITS)
|
| 54 | #define fp_toint(X) ((X) >> FRAC_BITS)
|
| 55 |
|
| 56 | #define ONE_EIGHTH_FP ((int64_t)1 << (FRAC_BITS - 3))
|
| 57 |
|
| 58 | #define EXT_BITS 6
|
| 59 | #define EXT_FRAC_BITS (EXT_BITS + FRAC_BITS)
|
| 60 | #define fp_ext_toint(X) ((X) >> EXT_FRAC_BITS)
|
| 61 | #define int_ext_tofp(X) ((int64_t)(X) << EXT_FRAC_BITS)
|
| 62 |
|
| 63 | static inline int32_t mul_fp(int32_t x, int32_t y)
|
| 64 | {
|
| 65 | return ((int64_t)x * (int64_t)y) >> FRAC_BITS;
|
| 66 | }
|
| 67 |
|
| 68 | static inline int32_t div_fp(s64 x, s64 y)
|
| 69 | {
|
| 70 | return div64_s64((int64_t)x << FRAC_BITS, y);
|
| 71 | }
|
| 72 |
|
| 73 | static inline int ceiling_fp(int32_t x)
|
| 74 | {
|
| 75 | int mask, ret;
|
| 76 |
|
| 77 | ret = fp_toint(x);
|
| 78 | mask = (1 << FRAC_BITS) - 1;
|
| 79 | if (x & mask)
|
| 80 | ret += 1;
|
| 81 | return ret;
|
| 82 | }
|
| 83 |
|
| 84 | static inline u64 mul_ext_fp(u64 x, u64 y)
|
| 85 | {
|
| 86 | return (x * y) >> EXT_FRAC_BITS;
|
| 87 | }
|
| 88 |
|
| 89 | static inline u64 div_ext_fp(u64 x, u64 y)
|
| 90 | {
|
| 91 | return div64_u64(x << EXT_FRAC_BITS, y);
|
| 92 | }
|
| 93 |
|
| 94 | |
| 95 | * struct sample - Store performance sample
|
| 96 | * @core_avg_perf: Ratio of APERF/MPERF which is the actual average
|
| 97 | * performance during last sample period
|
| 98 | * @busy_scaled: Scaled busy value which is used to calculate next
|
| 99 | * P state. This can be different than core_avg_perf
|
| 100 | * to account for cpu idle period
|
| 101 | * @aperf: Difference of actual performance frequency clock count
|
| 102 | * read from APERF MSR between last and current sample
|
| 103 | * @mperf: Difference of maximum performance frequency clock count
|
| 104 | * read from MPERF MSR between last and current sample
|
| 105 | * @tsc: Difference of time stamp counter between last and
|
| 106 | * current sample
|
| 107 | * @time: Current time from scheduler
|
| 108 | *
|
| 109 | * This structure is used in the cpudata structure to store performance sample
|
| 110 | * data for choosing next P State.
|
| 111 |
|
| 112 | struct sample {
|
| 113 | int32_t core_avg_perf;
|
| 114 | int32_t busy_scaled;
|
| 115 | u64 aperf;
|
| 116 | u64 mperf;
|
| 117 | u64 tsc;
|
| 118 | u64 time;
|
| 119 | };
|
| 120 |
|
| 121 | |
| 122 | * struct pstate_data - Store P state data
|
| 123 | * @current_pstate: Current requested P state
|
| 124 | * @min_pstate: Min P state possible for this platform
|
| 125 | * @max_pstate: Max P state possible for this platform
|
| 126 | * @max_pstate_physical:This is physical Max P state for a processor
|
| 127 | * This can be higher than the max_pstate which can
|
| 128 | * be limited by platform thermal design power limits
|
| 129 | * @perf_ctl_scaling: PERF_CTL P-state to frequency scaling factor
|
| 130 | * @scaling: Scaling factor between performance and frequency
|
| 131 | * @turbo_pstate: Max Turbo P state possible for this platform
|
| 132 | * @min_freq: @min_pstate frequency in cpufreq units
|
| 133 | * @max_freq: @max_pstate frequency in cpufreq units
|
| 134 | * @turbo_freq: @turbo_pstate frequency in cpufreq units
|
| 135 | *
|
| 136 | * Stores the per cpu model P state limits and current P state.
|
| 137 |
|
| 138 | struct pstate_data {
|
| 139 | int current_pstate;
|
| 140 | int min_pstate;
|
| 141 | int max_pstate;
|
| 142 | int max_pstate_physical;
|
| 143 | int perf_ctl_scaling;
|
| 144 | int scaling;
|
| 145 | int turbo_pstate;
|
| 146 | unsigned int min_freq;
|
| 147 | unsigned int max_freq;
|
| 148 | unsigned int turbo_freq;
|
| 149 | };
|
| 150 |
|
| 151 | |
| 152 | * struct vid_data - Stores voltage information data
|
| 153 | * @min: VID data for this platform corresponding to
|
| 154 | * the lowest P state
|
| 155 | * @max: VID data corresponding to the highest P State.
|
| 156 | * @turbo: VID data for turbo P state
|
| 157 | * @ratio: Ratio of (vid max - vid min) /
|
| 158 | * (max P state - Min P State)
|
| 159 | *
|
| 160 | * Stores the voltage data for DVFS (Dynamic Voltage and Frequency Scaling)
|
| 161 | * This data is used in Atom platforms, where in addition to target P state,
|
| 162 | * the voltage data needs to be specified to select next P State.
|
| 163 |
|
| 164 | struct vid_data {
|
| 165 | int min;
|
| 166 | int max;
|
| 167 | int turbo;
|
| 168 | int32_t ratio;
|
| 169 | };
|
| 170 |
|
| 171 | |
| 172 | * struct global_params - Global parameters, mostly tunable via sysfs.
|
| 173 | * @no_turbo: Whether or not to use turbo P-states.
|
| 174 | * @turbo_disabled: Whether or not turbo P-states are available at all,
|
| 175 | * based on the MSR_IA32_MISC_ENABLE value and whether or
|
| 176 | * not the maximum reported turbo P-state is different from
|
| 177 | * the maximum reported non-turbo one.
|
| 178 | * @min_perf_pct: Minimum capacity limit in percent of the maximum turbo
|
| 179 | * P-state capacity.
|
| 180 | * @max_perf_pct: Maximum capacity limit in percent of the maximum turbo
|
| 181 | * P-state capacity.
|
| 182 |
|
| 183 | struct global_params {
|
| 184 | bool no_turbo;
|
| 185 | bool turbo_disabled;
|
| 186 | int max_perf_pct;
|
| 187 | int min_perf_pct;
|
| 188 | };
|
| 189 |
|
| 190 | |
| 191 | * struct cpudata - Per CPU instance data storage
|
| 192 | * @cpu: CPU number for this instance data
|
| 193 | * @policy: CPUFreq policy value
|
| 194 | * @update_util: CPUFreq utility callback information
|
| 195 | * @update_util_set: CPUFreq utility callback is set
|
| 196 | * @iowait_boost: iowait-related boost fraction
|
| 197 | * @last_update: Time of the last update.
|
| 198 | * @pstate: Stores P state limits for this CPU
|
| 199 | * @vid: Stores VID limits for this CPU
|
| 200 | * @last_sample_time: Last Sample time
|
| 201 | * @aperf_mperf_shift: APERF vs MPERF counting frequency difference
|
| 202 | * @prev_aperf: Last APERF value read from APERF MSR
|
| 203 | * @prev_mperf: Last MPERF value read from MPERF MSR
|
| 204 | * @prev_tsc: Last timestamp counter (TSC) value
|
| 205 | * @sample: Storage for storing last Sample data
|
| 206 | * @min_perf_ratio: Minimum capacity in terms of PERF or HWP ratios
|
| 207 | * @max_perf_ratio: Maximum capacity in terms of PERF or HWP ratios
|
| 208 | * @acpi_perf_data: Stores ACPI perf information read from _PSS
|
| 209 | * @valid_pss_table: Set to true for valid ACPI _PSS entries found
|
| 210 | * @epp_powersave: Last saved HWP energy performance preference
|
| 211 | * (EPP) or energy performance bias (EPB),
|
| 212 | * when policy switched to performance
|
| 213 | * @epp_policy: Last saved policy used to set EPP/EPB
|
| 214 | * @epp_default: Power on default HWP energy performance
|
| 215 | * preference/bias
|
| 216 | * @epp_cached: Cached HWP energy-performance preference value
|
| 217 | * @hwp_req_cached: Cached value of the last HWP Request MSR
|
| 218 | * @hwp_cap_cached: Cached value of the last HWP Capabilities MSR
|
| 219 | * @last_io_update: Last time when IO wake flag was set
|
| 220 | * @capacity_perf: Highest perf used for scale invariance
|
| 221 | * @sched_flags: Store scheduler flags for possible cross CPU update
|
| 222 | * @hwp_boost_min: Last HWP boosted min performance
|
| 223 | * @suspended: Whether or not the driver has been suspended.
|
| 224 | * @pd_registered: Set when a perf domain is registered for this CPU.
|
| 225 | * @hwp_notify_work: workqueue for HWP notifications.
|
| 226 | *
|
| 227 | * This structure stores per CPU instance data for all CPUs.
|
| 228 |
|
| 229 | struct cpudata {
|
| 230 | int cpu;
|
| 231 |
|
| 232 | unsigned int policy;
|
| 233 | struct update_util_data update_util;
|
| 234 | bool update_util_set;
|
| 235 |
|
| 236 | struct pstate_data pstate;
|
| 237 | struct vid_data vid;
|
| 238 |
|
| 239 | u64 last_update;
|
| 240 | u64 last_sample_time;
|
| 241 | u64 aperf_mperf_shift;
|
| 242 | u64 prev_aperf;
|
| 243 | u64 prev_mperf;
|
| 244 | u64 prev_tsc;
|
| 245 | struct sample sample;
|
| 246 | int32_t min_perf_ratio;
|
| 247 | int32_t max_perf_ratio;
|
| 248 | #ifdef CONFIG_ACPI
|
| 249 | struct acpi_processor_performance acpi_perf_data;
|
| 250 | bool valid_pss_table;
|
| 251 | #endif
|
| 252 | unsigned int iowait_boost;
|
| 253 | s16 epp_powersave;
|
| 254 | s16 epp_policy;
|
| 255 | s16 epp_default;
|
| 256 | s16 epp_cached;
|
| 257 | u64 hwp_req_cached;
|
| 258 | u64 hwp_cap_cached;
|
| 259 | u64 last_io_update;
|
| 260 | unsigned int capacity_perf;
|
| 261 | unsigned int sched_flags;
|
| 262 | u32 hwp_boost_min;
|
| 263 | bool suspended;
|
| 264 | #ifdef CONFIG_ENERGY_MODEL
|
| 265 | bool pd_registered;
|
| 266 | #endif
|
| 267 | struct delayed_work hwp_notify_work;
|
| 268 | };
|
| 269 |
|
| 270 | static struct cpudata **all_cpu_data;
|
| 271 |
|
| 272 | |
| 273 | * struct pstate_funcs - Per CPU model specific callbacks
|
| 274 | * @get_max: Callback to get maximum non turbo effective P state
|
| 275 | * @get_max_physical: Callback to get maximum non turbo physical P state
|
| 276 | * @get_min: Callback to get minimum P state
|
| 277 | * @get_turbo: Callback to get turbo P state
|
| 278 | * @get_scaling: Callback to get frequency scaling factor
|
| 279 | * @get_cpu_scaling: Get frequency scaling factor for a given cpu
|
| 280 | * @get_aperf_mperf_shift: Callback to get the APERF vs MPERF frequency difference
|
| 281 | * @get_val: Callback to convert P state to actual MSR write value
|
| 282 | * @get_vid: Callback to get VID data for Atom platforms
|
| 283 | *
|
| 284 | * Core and Atom CPU models have different way to get P State limits. This
|
| 285 | * structure is used to store those callbacks.
|
| 286 |
|
| 287 | struct pstate_funcs {
|
| 288 | int (*get_max)(int cpu);
|
| 289 | int (*get_max_physical)(int cpu);
|
| 290 | int (*get_min)(int cpu);
|
| 291 | int (*get_turbo)(int cpu);
|
| 292 | int (*get_scaling)(void);
|
| 293 | int (*get_cpu_scaling)(int cpu);
|
| 294 | int (*get_aperf_mperf_shift)(void);
|
| 295 | u64 (*get_val)(struct cpudata*, int pstate);
|
| 296 | void (*get_vid)(struct cpudata *);
|
| 297 | };
|
| 298 |
|
| 299 | static struct pstate_funcs pstate_funcs __read_mostly;
|
| 300 |
|
| 301 | static bool hwp_active __ro_after_init;
|
| 302 | static bool per_cpu_limits __ro_after_init;
|
| 303 | static bool hwp_forced __ro_after_init;
|
| 304 | static bool hwp_boost __read_mostly;
|
| 305 | static bool hwp_is_hybrid;
|
| 306 | static u32 hwp_desired_mask __read_mostly = ~0U;
|
| 307 |
|
| 308 | static struct cpufreq_driver *intel_pstate_driver __read_mostly;
|
| 309 |
|
| 310 | #define INTEL_PSTATE_CORE_SCALING 100000
|
| 311 | #define HYBRID_SCALING_FACTOR_ADL 78741
|
| 312 | #define HYBRID_SCALING_FACTOR_MTL 80000
|
| 313 | #define HYBRID_SCALING_FACTOR_LNL 86957
|
| 314 |
|
| 315 | static int hybrid_scaling_factor;
|
| 316 |
|
| 317 | static inline int core_get_scaling(void)
|
| 318 | {
|
| 319 | return INTEL_PSTATE_CORE_SCALING;
|
| 320 | }
|
| 321 |
|
| 322 | #ifdef CONFIG_ACPI
|
| 323 | static bool acpi_ppc;
|
| 324 | #endif
|
| 325 |
|
| 326 | static struct global_params global;
|
| 327 |
|
| 328 | static DEFINE_MUTEX(intel_pstate_driver_lock);
|
| 329 | static DEFINE_MUTEX(intel_pstate_limits_lock);
|
| 330 |
|
| 331 | #ifdef CONFIG_ACPI
|
| 332 |
|
| 333 | static bool intel_pstate_acpi_pm_profile_server(void)
|
| 334 | {
|
| 335 | if (acpi_gbl_FADT.preferred_profile == PM_ENTERPRISE_SERVER ||
|
| 336 | acpi_gbl_FADT.preferred_profile == PM_PERFORMANCE_SERVER)
|
| 337 | return true;
|
| 338 |
|
| 339 | return false;
|
| 340 | }
|
| 341 |
|
| 342 | static bool intel_pstate_get_ppc_enable_status(void)
|
| 343 | {
|
| 344 | if (intel_pstate_acpi_pm_profile_server())
|
| 345 | return true;
|
| 346 |
|
| 347 | return acpi_ppc;
|
| 348 | }
|
| 349 |
|
| 350 | #ifdef CONFIG_ACPI_CPPC_LIB
|
| 351 |
|
| 352 |
|
| 353 | static void intel_pstste_sched_itmt_work_fn(struct work_struct *work)
|
| 354 | {
|
| 355 | sched_set_itmt_support();
|
| 356 | }
|
| 357 |
|
| 358 | static DECLARE_WORK(sched_itmt_work, intel_pstste_sched_itmt_work_fn);
|
| 359 |
|
| 360 | #define CPPC_MAX_PERF U8_MAX
|
| 361 |
|
| 362 | static void intel_pstate_set_itmt_prio(int cpu)
|
| 363 | {
|
| 364 | struct cppc_perf_caps cppc_perf;
|
| 365 | static u32 max_highest_perf = 0, min_highest_perf = U32_MAX;
|
| 366 | int ret;
|
| 367 |
|
| 368 | ret = cppc_get_perf_caps(cpu, &cppc_perf);
|
| 369 | |
| 370 | * If CPPC is not available, fall back to MSR_HWP_CAPABILITIES bits [8:0].
|
| 371 | *
|
| 372 | * Also, on some systems with overclocking enabled, CPPC.highest_perf is
|
| 373 | * hardcoded to 0xff, so CPPC.highest_perf cannot be used to enable ITMT.
|
| 374 | * Fall back to MSR_HWP_CAPABILITIES then too.
|
| 375 |
|
| 376 | if (ret || cppc_perf.highest_perf == CPPC_MAX_PERF)
|
| 377 | cppc_perf.highest_perf = HWP_HIGHEST_PERF(READ_ONCE(all_cpu_data[cpu]->hwp_cap_cached));
|
| 378 |
|
| 379 | |
| 380 | * The priorities can be set regardless of whether or not
|
| 381 | * sched_set_itmt_support(true) has been called and it is valid to
|
| 382 | * update them at any time after it has been called.
|
| 383 |
|
| 384 | sched_set_itmt_core_prio(cppc_perf.highest_perf, cpu);
|
| 385 |
|
| 386 | if (max_highest_perf <= min_highest_perf) {
|
| 387 | if (cppc_perf.highest_perf > max_highest_perf)
|
| 388 | max_highest_perf = cppc_perf.highest_perf;
|
| 389 |
|
| 390 | if (cppc_perf.highest_perf < min_highest_perf)
|
| 391 | min_highest_perf = cppc_perf.highest_perf;
|
| 392 |
|
| 393 | if (max_highest_perf > min_highest_perf) {
|
| 394 | |
| 395 | * This code can be run during CPU online under the
|
| 396 | * CPU hotplug locks, so sched_set_itmt_support()
|
| 397 | * cannot be called from here. Queue up a work item
|
| 398 | * to invoke it.
|
| 399 |
|
| 400 | schedule_work(&sched_itmt_work);
|
| 401 | }
|
| 402 | }
|
| 403 | }
|
| 404 |
|
| 405 | static int intel_pstate_get_cppc_guaranteed(int cpu)
|
| 406 | {
|
| 407 | struct cppc_perf_caps cppc_perf;
|
| 408 | int ret;
|
| 409 |
|
| 410 | ret = cppc_get_perf_caps(cpu, &cppc_perf);
|
| 411 | if (ret)
|
| 412 | return ret;
|
| 413 |
|
| 414 | if (cppc_perf.guaranteed_perf)
|
| 415 | return cppc_perf.guaranteed_perf;
|
| 416 |
|
| 417 | return cppc_perf.nominal_perf;
|
| 418 | }
|
| 419 |
|
| 420 | static int intel_pstate_cppc_get_scaling(int cpu)
|
| 421 | {
|
| 422 | struct cppc_perf_caps cppc_perf;
|
| 423 |
|
| 424 | |
| 425 | * Compute the perf-to-frequency scaling factor for the given CPU if
|
| 426 | * possible, unless it would be 0.
|
| 427 |
|
| 428 | if (!cppc_get_perf_caps(cpu, &cppc_perf) &&
|
| 429 | cppc_perf.nominal_perf && cppc_perf.nominal_freq)
|
| 430 | return div_u64(cppc_perf.nominal_freq * KHZ_PER_MHZ,
|
| 431 | cppc_perf.nominal_perf);
|
| 432 |
|
| 433 | return core_get_scaling();
|
| 434 | }
|
| 435 |
|
| 436 | #else
|
| 437 | static inline void intel_pstate_set_itmt_prio(int cpu)
|
| 438 | {
|
| 439 | }
|
| 440 | #endif
|
| 441 |
|
| 442 | static void intel_pstate_init_acpi_perf_limits(struct cpufreq_policy *policy)
|
| 443 | {
|
| 444 | struct cpudata *cpu;
|
| 445 | int ret;
|
| 446 | int i;
|
| 447 |
|
| 448 | if (hwp_active) {
|
| 449 | intel_pstate_set_itmt_prio(policy->cpu);
|
| 450 | return;
|
| 451 | }
|
| 452 |
|
| 453 | if (!intel_pstate_get_ppc_enable_status())
|
| 454 | return;
|
| 455 |
|
| 456 | cpu = all_cpu_data[policy->cpu];
|
| 457 |
|
| 458 | ret = acpi_processor_register_performance(&cpu->acpi_perf_data,
|
| 459 | policy->cpu);
|
| 460 | if (ret)
|
| 461 | return;
|
| 462 |
|
| 463 | |
| 464 | * Check if the control value in _PSS is for PERF_CTL MSR, which should
|
| 465 | * guarantee that the states returned by it map to the states in our
|
| 466 | * list directly.
|
| 467 |
|
| 468 | if (cpu->acpi_perf_data.control_register.space_id !=
|
| 469 | ACPI_ADR_SPACE_FIXED_HARDWARE)
|
| 470 | goto err;
|
| 471 |
|
| 472 | |
| 473 | * If there is only one entry _PSS, simply ignore _PSS and continue as
|
| 474 | * usual without taking _PSS into account
|
| 475 |
|
| 476 | if (cpu->acpi_perf_data.state_count < 2)
|
| 477 | goto err;
|
| 478 |
|
| 479 | pr_debug("CPU%u - ACPI _PSS perf data\n", policy->cpu);
|
| 480 | for (i = 0; i < cpu->acpi_perf_data.state_count; i++) {
|
| 481 | pr_debug(" %cP%d: %u MHz, %u mW, 0x%x\n",
|
| 482 | (i == cpu->acpi_perf_data.state ? '*' : ' '), i,
|
| 483 | (u32) cpu->acpi_perf_data.states[i].core_frequency,
|
| 484 | (u32) cpu->acpi_perf_data.states[i].power,
|
| 485 | (u32) cpu->acpi_perf_data.states[i].control);
|
| 486 | }
|
| 487 |
|
| 488 | cpu->valid_pss_table = true;
|
| 489 | pr_debug("_PPC limits will be enforced\n");
|
| 490 |
|
| 491 | return;
|
| 492 |
|
| 493 | err:
|
| 494 | cpu->valid_pss_table = false;
|
| 495 | acpi_processor_unregister_performance(policy->cpu);
|
| 496 | }
|
| 497 |
|
| 498 | static void intel_pstate_exit_perf_limits(struct cpufreq_policy *policy)
|
| 499 | {
|
| 500 | struct cpudata *cpu;
|
| 501 |
|
| 502 | cpu = all_cpu_data[policy->cpu];
|
| 503 | if (!cpu->valid_pss_table)
|
| 504 | return;
|
| 505 |
|
| 506 | acpi_processor_unregister_performance(policy->cpu);
|
| 507 | }
|
| 508 | #else
|
| 509 | static inline void intel_pstate_init_acpi_perf_limits(struct cpufreq_policy *policy)
|
| 510 | {
|
| 511 | }
|
| 512 |
|
| 513 | static inline void intel_pstate_exit_perf_limits(struct cpufreq_policy *policy)
|
| 514 | {
|
| 515 | }
|
| 516 |
|
| 517 | static inline bool intel_pstate_acpi_pm_profile_server(void)
|
| 518 | {
|
| 519 | return false;
|
| 520 | }
|
| 521 | #endif
|
| 522 |
|
| 523 | #ifndef CONFIG_ACPI_CPPC_LIB
|
| 524 | static inline int intel_pstate_get_cppc_guaranteed(int cpu)
|
| 525 | {
|
| 526 | return -ENOTSUPP;
|
| 527 | }
|
| 528 |
|
| 529 | static int intel_pstate_cppc_get_scaling(int cpu)
|
| 530 | {
|
| 531 | return core_get_scaling();
|
| 532 | }
|
| 533 | #endif
|
| 534 |
|
| 535 | static int intel_pstate_freq_to_hwp_rel(struct cpudata *cpu, int freq,
|
| 536 | unsigned int relation)
|
| 537 | {
|
| 538 | if (freq == cpu->pstate.turbo_freq)
|
| 539 | return cpu->pstate.turbo_pstate;
|
| 540 |
|
| 541 | if (freq == cpu->pstate.max_freq)
|
| 542 | return cpu->pstate.max_pstate;
|
| 543 |
|
| 544 | switch (relation) {
|
| 545 | case CPUFREQ_RELATION_H:
|
| 546 | return freq / cpu->pstate.scaling;
|
| 547 | case CPUFREQ_RELATION_C:
|
| 548 | return DIV_ROUND_CLOSEST(freq, cpu->pstate.scaling);
|
| 549 | }
|
| 550 |
|
| 551 | return DIV_ROUND_UP(freq, cpu->pstate.scaling);
|
| 552 | }
|
| 553 |
|
| 554 | static int intel_pstate_freq_to_hwp(struct cpudata *cpu, int freq)
|
| 555 | {
|
| 556 | return intel_pstate_freq_to_hwp_rel(cpu, freq, CPUFREQ_RELATION_L);
|
| 557 | }
|
| 558 |
|
| 559 | static bool turbo_is_disabled(void)
|
| 560 | {
|
| 561 | u64 misc_en;
|
| 562 |
|
| 563 | rdmsrq(MSR_IA32_MISC_ENABLE, misc_en);
|
| 564 |
|
| 565 | return !!(misc_en & MSR_IA32_MISC_ENABLE_TURBO_DISABLE);
|
| 566 | }
|
| 567 |
|
| 568 | static int min_perf_pct_min(void)
|
| 569 | {
|
| 570 | struct cpudata *cpu = all_cpu_data[0];
|
| 571 | int turbo_pstate = cpu->pstate.turbo_pstate;
|
| 572 |
|
| 573 | return turbo_pstate ?
|
| 574 | (cpu->pstate.min_pstate * 100 / turbo_pstate) : 0;
|
| 575 | }
|
| 576 |
|
| 577 | static s16 intel_pstate_get_epp(struct cpudata *cpu_data, u64 hwp_req_data)
|
| 578 | {
|
| 579 | s16 epp = -EOPNOTSUPP;
|
| 580 |
|
| 581 | if (boot_cpu_has(X86_FEATURE_HWP_EPP)) {
|
| 582 | |
| 583 | * When hwp_req_data is 0, means that caller didn't read
|
| 584 | * MSR_HWP_REQUEST, so need to read and get EPP.
|
| 585 |
|
| 586 | if (!hwp_req_data) {
|
| 587 | epp = rdmsrq_on_cpu(cpu_data->cpu, MSR_HWP_REQUEST,
|
| 588 | &hwp_req_data);
|
| 589 | if (epp)
|
| 590 | return epp;
|
| 591 | }
|
| 592 | epp = (hwp_req_data >> 24) & 0xff;
|
| 593 | }
|
| 594 |
|
| 595 | return epp;
|
| 596 | }
|
| 597 |
|
| 598 | |
| 599 | * EPP display strings corresponding to EPP index in the
|
| 600 | * energy_perf_strings[]
|
| 601 | * index String
|
| 602 | *-------------------------------------
|
| 603 | * 0 default
|
| 604 | * 1 performance
|
| 605 | * 2 balance_performance
|
| 606 | * 3 balance_power
|
| 607 | * 4 power
|
| 608 |
|
| 609 |
|
| 610 | enum energy_perf_value_index {
|
| 611 | EPP_INDEX_DEFAULT = 0,
|
| 612 | EPP_INDEX_PERFORMANCE,
|
| 613 | EPP_INDEX_BALANCE_PERFORMANCE,
|
| 614 | EPP_INDEX_BALANCE_POWERSAVE,
|
| 615 | EPP_INDEX_POWERSAVE,
|
| 616 | };
|
| 617 |
|
| 618 | static const char * const energy_perf_strings[] = {
|
| 619 | [EPP_INDEX_DEFAULT] = "default",
|
| 620 | [EPP_INDEX_PERFORMANCE] = "performance",
|
| 621 | [EPP_INDEX_BALANCE_PERFORMANCE] = "balance_performance",
|
| 622 | [EPP_INDEX_BALANCE_POWERSAVE] = "balance_power",
|
| 623 | [EPP_INDEX_POWERSAVE] = "power",
|
| 624 | NULL
|
| 625 | };
|
| 626 | static unsigned int epp_values[] = {
|
| 627 | [EPP_INDEX_DEFAULT] = 0,
|
| 628 | [EPP_INDEX_PERFORMANCE] = HWP_EPP_PERFORMANCE,
|
| 629 | [EPP_INDEX_BALANCE_PERFORMANCE] = HWP_EPP_BALANCE_PERFORMANCE,
|
| 630 | [EPP_INDEX_BALANCE_POWERSAVE] = HWP_EPP_BALANCE_POWERSAVE,
|
| 631 | [EPP_INDEX_POWERSAVE] = HWP_EPP_POWERSAVE,
|
| 632 | };
|
| 633 |
|
| 634 | static int intel_pstate_get_energy_pref_index(struct cpudata *cpu_data, int *raw_epp)
|
| 635 | {
|
| 636 | s16 epp;
|
| 637 | int index = -EINVAL;
|
| 638 |
|
| 639 | *raw_epp = 0;
|
| 640 | epp = intel_pstate_get_epp(cpu_data, 0);
|
| 641 | if (epp < 0)
|
| 642 | return epp;
|
| 643 |
|
| 644 | if (boot_cpu_has(X86_FEATURE_HWP_EPP)) {
|
| 645 | if (epp == epp_values[EPP_INDEX_PERFORMANCE])
|
| 646 | return EPP_INDEX_PERFORMANCE;
|
| 647 | if (epp == epp_values[EPP_INDEX_BALANCE_PERFORMANCE])
|
| 648 | return EPP_INDEX_BALANCE_PERFORMANCE;
|
| 649 | if (epp == epp_values[EPP_INDEX_BALANCE_POWERSAVE])
|
| 650 | return EPP_INDEX_BALANCE_POWERSAVE;
|
| 651 | if (epp == epp_values[EPP_INDEX_POWERSAVE])
|
| 652 | return EPP_INDEX_POWERSAVE;
|
| 653 | *raw_epp = epp;
|
| 654 | return 0;
|
| 655 | } else if (boot_cpu_has(X86_FEATURE_EPB)) {
|
| 656 | |
| 657 | * Range:
|
| 658 | * 0x00-0x03 : Performance
|
| 659 | * 0x04-0x07 : Balance performance
|
| 660 | * 0x08-0x0B : Balance power
|
| 661 | * 0x0C-0x0F : Power
|
| 662 | * The EPB is a 4 bit value, but our ranges restrict the
|
| 663 | * value which can be set. Here only using top two bits
|
| 664 | * effectively.
|
| 665 |
|
| 666 | index = (epp >> 2) + 1;
|
| 667 | }
|
| 668 |
|
| 669 | return index;
|
| 670 | }
|
| 671 |
|
| 672 | static int intel_pstate_set_epp(struct cpudata *cpu, u32 epp)
|
| 673 | {
|
| 674 | int ret;
|
| 675 |
|
| 676 | |
| 677 | * Use the cached HWP Request MSR value, because in the active mode the
|
| 678 | * register itself may be updated by intel_pstate_hwp_boost_up() or
|
| 679 | * intel_pstate_hwp_boost_down() at any time.
|
| 680 |
|
| 681 | u64 value = READ_ONCE(cpu->hwp_req_cached);
|
| 682 |
|
| 683 | value &= ~GENMASK_ULL(31, 24);
|
| 684 | value |= (u64)epp << 24;
|
| 685 | |
| 686 | * The only other updater of hwp_req_cached in the active mode,
|
| 687 | * intel_pstate_hwp_set(), is called under the same lock as this
|
| 688 | * function, so it cannot run in parallel with the update below.
|
| 689 |
|
| 690 | WRITE_ONCE(cpu->hwp_req_cached, value);
|
| 691 | ret = wrmsrq_on_cpu(cpu->cpu, MSR_HWP_REQUEST, value);
|
| 692 | if (!ret)
|
| 693 | cpu->epp_cached = epp;
|
| 694 |
|
| 695 | return ret;
|
| 696 | }
|
| 697 |
|
| 698 | static int intel_pstate_set_energy_pref_index(struct cpudata *cpu_data,
|
| 699 | int pref_index, bool use_raw,
|
| 700 | u32 raw_epp)
|
| 701 | {
|
| 702 | int epp = -EINVAL;
|
| 703 | int ret = -EOPNOTSUPP;
|
| 704 |
|
| 705 | if (!pref_index)
|
| 706 | epp = cpu_data->epp_default;
|
| 707 |
|
| 708 | if (boot_cpu_has(X86_FEATURE_HWP_EPP)) {
|
| 709 | if (use_raw)
|
| 710 | epp = raw_epp;
|
| 711 | else if (epp == -EINVAL)
|
| 712 | epp = epp_values[pref_index];
|
| 713 |
|
| 714 | |
| 715 | * To avoid confusion, refuse to set EPP to any values different
|
| 716 | * from 0 (performance) if the current policy is "performance",
|
| 717 | * because those values would be overridden.
|
| 718 |
|
| 719 | if (epp > 0 && cpu_data->policy == CPUFREQ_POLICY_PERFORMANCE)
|
| 720 | return -EBUSY;
|
| 721 |
|
| 722 | ret = intel_pstate_set_epp(cpu_data, epp);
|
| 723 | }
|
| 724 |
|
| 725 | return ret;
|
| 726 | }
|
| 727 |
|
| 728 | static ssize_t show_energy_performance_available_preferences(
|
| 729 | struct cpufreq_policy *policy, char *buf)
|
| 730 | {
|
| 731 | int i = 0;
|
| 732 | int ret = 0;
|
| 733 |
|
| 734 | while (energy_perf_strings[i] != NULL)
|
| 735 | ret += sprintf(&buf[ret], "%s ", energy_perf_strings[i++]);
|
| 736 |
|
| 737 | ret += sprintf(&buf[ret], "\n");
|
| 738 |
|
| 739 | return ret;
|
| 740 | }
|
| 741 |
|
| 742 | cpufreq_freq_attr_ro(energy_performance_available_preferences);
|
| 743 |
|
| 744 | static struct cpufreq_driver intel_pstate;
|
| 745 |
|
| 746 | static ssize_t store_energy_performance_preference(
|
| 747 | struct cpufreq_policy *policy, const char *buf, size_t count)
|
| 748 | {
|
| 749 | struct cpudata *cpu = all_cpu_data[policy->cpu];
|
| 750 | char str_preference[21];
|
| 751 | bool raw = false;
|
| 752 | ssize_t ret;
|
| 753 | u32 epp = 0;
|
| 754 |
|
| 755 | ret = sscanf(buf, "%20s", str_preference);
|
| 756 | if (ret != 1)
|
| 757 | return -EINVAL;
|
| 758 |
|
| 759 | ret = match_string(energy_perf_strings, -1, str_preference);
|
| 760 | if (ret < 0) {
|
| 761 | if (!boot_cpu_has(X86_FEATURE_HWP_EPP))
|
| 762 | return ret;
|
| 763 |
|
| 764 | ret = kstrtouint(buf, 10, &epp);
|
| 765 | if (ret)
|
| 766 | return ret;
|
| 767 |
|
| 768 | if (epp > 255)
|
| 769 | return -EINVAL;
|
| 770 |
|
| 771 | raw = true;
|
| 772 | }
|
| 773 |
|
| 774 | |
| 775 | * This function runs with the policy R/W semaphore held, which
|
| 776 | * guarantees that the driver pointer will not change while it is
|
| 777 | * running.
|
| 778 |
|
| 779 | if (!intel_pstate_driver)
|
| 780 | return -EAGAIN;
|
| 781 |
|
| 782 | mutex_lock(&intel_pstate_limits_lock);
|
| 783 |
|
| 784 | if (intel_pstate_driver == &intel_pstate) {
|
| 785 | ret = intel_pstate_set_energy_pref_index(cpu, ret, raw, epp);
|
| 786 | } else {
|
| 787 | |
| 788 | * In the passive mode the governor needs to be stopped on the
|
| 789 | * target CPU before the EPP update and restarted after it,
|
| 790 | * which is super-heavy-weight, so make sure it is worth doing
|
| 791 | * upfront.
|
| 792 |
|
| 793 | if (!raw)
|
| 794 | epp = ret ? epp_values[ret] : cpu->epp_default;
|
| 795 |
|
| 796 | if (cpu->epp_cached != epp) {
|
| 797 | int err;
|
| 798 |
|
| 799 | cpufreq_stop_governor(policy);
|
| 800 | ret = intel_pstate_set_epp(cpu, epp);
|
| 801 | err = cpufreq_start_governor(policy);
|
| 802 | if (!ret)
|
| 803 | ret = err;
|
| 804 | } else {
|
| 805 | ret = 0;
|
| 806 | }
|
| 807 | }
|
| 808 |
|
| 809 | mutex_unlock(&intel_pstate_limits_lock);
|
| 810 |
|
| 811 | return ret ?: count;
|
| 812 | }
|
| 813 |
|
| 814 | static ssize_t show_energy_performance_preference(
|
| 815 | struct cpufreq_policy *policy, char *buf)
|
| 816 | {
|
| 817 | struct cpudata *cpu_data = all_cpu_data[policy->cpu];
|
| 818 | int preference, raw_epp;
|
| 819 |
|
| 820 | preference = intel_pstate_get_energy_pref_index(cpu_data, &raw_epp);
|
| 821 | if (preference < 0)
|
| 822 | return preference;
|
| 823 |
|
| 824 | if (raw_epp)
|
| 825 | return sprintf(buf, "%d\n", raw_epp);
|
| 826 | else
|
| 827 | return sprintf(buf, "%s\n", energy_perf_strings[preference]);
|
| 828 | }
|
| 829 |
|
| 830 | cpufreq_freq_attr_rw(energy_performance_preference);
|
| 831 |
|
| 832 | static ssize_t show_base_frequency(struct cpufreq_policy *policy, char *buf)
|
| 833 | {
|
| 834 | struct cpudata *cpu = all_cpu_data[policy->cpu];
|
| 835 | int ratio, freq;
|
| 836 |
|
| 837 | ratio = intel_pstate_get_cppc_guaranteed(policy->cpu);
|
| 838 | if (ratio <= 0) {
|
| 839 | u64 cap;
|
| 840 |
|
| 841 | rdmsrq_on_cpu(policy->cpu, MSR_HWP_CAPABILITIES, &cap);
|
| 842 | ratio = HWP_GUARANTEED_PERF(cap);
|
| 843 | }
|
| 844 |
|
| 845 | freq = ratio * cpu->pstate.scaling;
|
| 846 | if (cpu->pstate.scaling != cpu->pstate.perf_ctl_scaling)
|
| 847 | freq = rounddown(freq, cpu->pstate.perf_ctl_scaling);
|
| 848 |
|
| 849 | return sprintf(buf, "%d\n", freq);
|
| 850 | }
|
| 851 |
|
| 852 | cpufreq_freq_attr_ro(base_frequency);
|
| 853 |
|
| 854 | enum hwp_cpufreq_attr_index {
|
| 855 | HWP_BASE_FREQUENCY_INDEX = 0,
|
| 856 | HWP_PERFORMANCE_PREFERENCE_INDEX,
|
| 857 | HWP_PERFORMANCE_AVAILABLE_PREFERENCES_INDEX,
|
| 858 | HWP_CPUFREQ_ATTR_COUNT,
|
| 859 | };
|
| 860 |
|
| 861 | static struct freq_attr *hwp_cpufreq_attrs[] = {
|
| 862 | [HWP_BASE_FREQUENCY_INDEX] = &base_frequency,
|
| 863 | [HWP_PERFORMANCE_PREFERENCE_INDEX] = &energy_performance_preference,
|
| 864 | [HWP_PERFORMANCE_AVAILABLE_PREFERENCES_INDEX] =
|
| 865 | &energy_performance_available_preferences,
|
| 866 | [HWP_CPUFREQ_ATTR_COUNT] = NULL,
|
| 867 | };
|
| 868 |
|
| 869 | static u8 hybrid_get_cpu_type(unsigned int cpu)
|
| 870 | {
|
| 871 | return cpu_data(cpu).topo.intel_type;
|
| 872 | }
|
| 873 |
|
| 874 | static bool no_cas __ro_after_init;
|
| 875 |
|
| 876 | static struct cpudata *hybrid_max_perf_cpu __read_mostly;
|
| 877 | |
| 878 | * Protects hybrid_max_perf_cpu, the capacity_perf fields in struct cpudata,
|
| 879 | * and the x86 arch scale-invariance information from concurrent updates.
|
| 880 |
|
| 881 | static DEFINE_MUTEX(hybrid_capacity_lock);
|
| 882 |
|
| 883 | #ifdef CONFIG_ENERGY_MODEL
|
| 884 | #define HYBRID_EM_STATE_COUNT 4
|
| 885 |
|
| 886 | static int hybrid_active_power(struct device *dev, unsigned long *power,
|
| 887 | unsigned long *freq)
|
| 888 | {
|
| 889 | |
| 890 | * Create four "states" corresponding to 40%, 60%, 80%, and 100% of the
|
| 891 | * full capacity.
|
| 892 | *
|
| 893 | * For this purpose, return the "frequency" of 2 for the first
|
| 894 | * performance level and otherwise leave the value set by the caller.
|
| 895 |
|
| 896 | if (!*freq)
|
| 897 | *freq = 2;
|
| 898 |
|
| 899 |
|
| 900 | *power = EM_MAX_POWER;
|
| 901 |
|
| 902 | return 0;
|
| 903 | }
|
| 904 |
|
| 905 | static bool hybrid_has_l3(unsigned int cpu)
|
| 906 | {
|
| 907 | struct cpu_cacheinfo *cacheinfo = get_cpu_cacheinfo(cpu);
|
| 908 | unsigned int i;
|
| 909 |
|
| 910 | if (!cacheinfo)
|
| 911 | return false;
|
| 912 |
|
| 913 | for (i = 0; i < cacheinfo->num_leaves; i++) {
|
| 914 | if (cacheinfo->info_list[i].level == 3)
|
| 915 | return true;
|
| 916 | }
|
| 917 |
|
| 918 | return false;
|
| 919 | }
|
| 920 |
|
| 921 | static int hybrid_get_cost(struct device *dev, unsigned long freq,
|
| 922 | unsigned long *cost)
|
| 923 | {
|
| 924 |
|
| 925 | *cost = freq;
|
| 926 | |
| 927 | * Adjust the cost depending on CPU type.
|
| 928 | *
|
| 929 | * The idea is to start loading up LPE-cores before E-cores and start
|
| 930 | * to populate E-cores when LPE-cores are utilized above 60% of the
|
| 931 | * capacity. Similarly, P-cores start to be populated when E-cores are
|
| 932 | * utilized above 60% of the capacity.
|
| 933 |
|
| 934 | if (hybrid_get_cpu_type(dev->id) == INTEL_CPU_TYPE_CORE)
|
| 935 | *cost += 2;
|
| 936 | else if (hybrid_has_l3(dev->id))
|
| 937 | *cost += 1;
|
| 938 |
|
| 939 | return 0;
|
| 940 | }
|
| 941 |
|
| 942 | static bool hybrid_register_perf_domain(unsigned int cpu)
|
| 943 | {
|
| 944 | static const struct em_data_callback cb
|
| 945 | = EM_ADV_DATA_CB(hybrid_active_power, hybrid_get_cost);
|
| 946 | struct cpudata *cpudata = all_cpu_data[cpu];
|
| 947 | struct device *cpu_dev;
|
| 948 |
|
| 949 | |
| 950 | * Registering EM perf domains without enabling asymmetric CPU capacity
|
| 951 | * support is not really useful and one domain should not be registered
|
| 952 | * more than once.
|
| 953 |
|
| 954 | if (!hybrid_max_perf_cpu || cpudata->pd_registered)
|
| 955 | return false;
|
| 956 |
|
| 957 | cpu_dev = get_cpu_device(cpu);
|
| 958 | if (!cpu_dev)
|
| 959 | return false;
|
| 960 |
|
| 961 | if (em_dev_register_pd_no_update(cpu_dev, HYBRID_EM_STATE_COUNT, &cb,
|
| 962 | cpumask_of(cpu), false))
|
| 963 | return false;
|
| 964 |
|
| 965 | cpudata->pd_registered = true;
|
| 966 |
|
| 967 | return true;
|
| 968 | }
|
| 969 |
|
| 970 | static void hybrid_register_all_perf_domains(void)
|
| 971 | {
|
| 972 | unsigned int cpu;
|
| 973 |
|
| 974 | for_each_online_cpu(cpu)
|
| 975 | hybrid_register_perf_domain(cpu);
|
| 976 | }
|
| 977 |
|
| 978 | static void hybrid_update_perf_domain(struct cpudata *cpu)
|
| 979 | {
|
| 980 | if (cpu->pd_registered)
|
| 981 | em_adjust_cpu_capacity(cpu->cpu);
|
| 982 | }
|
| 983 | #else
|
| 984 | static inline bool hybrid_register_perf_domain(unsigned int cpu) { return false; }
|
| 985 | static inline void hybrid_register_all_perf_domains(void) {}
|
| 986 | static inline void hybrid_update_perf_domain(struct cpudata *cpu) {}
|
| 987 | #endif
|
| 988 |
|
| 989 | static void hybrid_set_cpu_capacity(struct cpudata *cpu)
|
| 990 | {
|
| 991 | arch_set_cpu_capacity(cpu->cpu, cpu->capacity_perf,
|
| 992 | hybrid_max_perf_cpu->capacity_perf,
|
| 993 | cpu->capacity_perf,
|
| 994 | cpu->pstate.max_pstate_physical);
|
| 995 | hybrid_update_perf_domain(cpu);
|
| 996 |
|
| 997 | topology_set_cpu_scale(cpu->cpu, arch_scale_cpu_capacity(cpu->cpu));
|
| 998 |
|
| 999 | pr_debug("CPU%d: capacity perf = %u, base perf = %u, sys max perf = %u\n",
|
| 1000 | cpu->cpu, cpu->capacity_perf, cpu->pstate.max_pstate_physical,
|
| 1001 | hybrid_max_perf_cpu->capacity_perf);
|
| 1002 | }
|
| 1003 |
|
| 1004 | static void hybrid_clear_cpu_capacity(unsigned int cpunum)
|
| 1005 | {
|
| 1006 | arch_set_cpu_capacity(cpunum, 1, 1, 1, 1);
|
| 1007 | }
|
| 1008 |
|
| 1009 | static void hybrid_get_capacity_perf(struct cpudata *cpu)
|
| 1010 | {
|
| 1011 | u64 hwp_cap = READ_ONCE(cpu->hwp_cap_cached);
|
| 1012 |
|
| 1013 | if (READ_ONCE(global.no_turbo)) {
|
| 1014 | cpu->capacity_perf = HWP_GUARANTEED_PERF(hwp_cap);
|
| 1015 | return;
|
| 1016 | }
|
| 1017 |
|
| 1018 | cpu->capacity_perf = HWP_HIGHEST_PERF(hwp_cap);
|
| 1019 | }
|
| 1020 |
|
| 1021 | static void hybrid_set_capacity_of_cpus(void)
|
| 1022 | {
|
| 1023 | int cpunum;
|
| 1024 |
|
| 1025 | for_each_online_cpu(cpunum) {
|
| 1026 | struct cpudata *cpu = all_cpu_data[cpunum];
|
| 1027 |
|
| 1028 | if (cpu)
|
| 1029 | hybrid_set_cpu_capacity(cpu);
|
| 1030 | }
|
| 1031 | }
|
| 1032 |
|
| 1033 | static void hybrid_update_cpu_capacity_scaling(void)
|
| 1034 | {
|
| 1035 | struct cpudata *max_perf_cpu = NULL;
|
| 1036 | unsigned int max_cap_perf = 0;
|
| 1037 | int cpunum;
|
| 1038 |
|
| 1039 | for_each_online_cpu(cpunum) {
|
| 1040 | struct cpudata *cpu = all_cpu_data[cpunum];
|
| 1041 |
|
| 1042 | if (!cpu)
|
| 1043 | continue;
|
| 1044 |
|
| 1045 | |
| 1046 | * During initialization, CPU performance at full capacity needs
|
| 1047 | * to be determined.
|
| 1048 |
|
| 1049 | if (!hybrid_max_perf_cpu)
|
| 1050 | hybrid_get_capacity_perf(cpu);
|
| 1051 |
|
| 1052 | |
| 1053 | * If hybrid_max_perf_cpu is not NULL at this point, it is
|
| 1054 | * being replaced, so don't take it into account when looking
|
| 1055 | * for the new one.
|
| 1056 |
|
| 1057 | if (cpu == hybrid_max_perf_cpu)
|
| 1058 | continue;
|
| 1059 |
|
| 1060 | if (cpu->capacity_perf > max_cap_perf) {
|
| 1061 | max_cap_perf = cpu->capacity_perf;
|
| 1062 | max_perf_cpu = cpu;
|
| 1063 | }
|
| 1064 | }
|
| 1065 |
|
| 1066 | if (max_perf_cpu) {
|
| 1067 | hybrid_max_perf_cpu = max_perf_cpu;
|
| 1068 | hybrid_set_capacity_of_cpus();
|
| 1069 | } else {
|
| 1070 | pr_info("Found no CPUs with nonzero maximum performance\n");
|
| 1071 |
|
| 1072 | for_each_online_cpu(cpunum)
|
| 1073 | hybrid_clear_cpu_capacity(cpunum);
|
| 1074 | }
|
| 1075 | }
|
| 1076 |
|
| 1077 | static void __hybrid_refresh_cpu_capacity_scaling(void)
|
| 1078 | {
|
| 1079 | hybrid_max_perf_cpu = NULL;
|
| 1080 | hybrid_update_cpu_capacity_scaling();
|
| 1081 | }
|
| 1082 |
|
| 1083 | static void hybrid_refresh_cpu_capacity_scaling(void)
|
| 1084 | {
|
| 1085 | guard(mutex)(&hybrid_capacity_lock);
|
| 1086 |
|
| 1087 | __hybrid_refresh_cpu_capacity_scaling();
|
| 1088 | |
| 1089 | * Perf domains are not registered before setting hybrid_max_perf_cpu,
|
| 1090 | * so register them all after setting up CPU capacity scaling.
|
| 1091 |
|
| 1092 | hybrid_register_all_perf_domains();
|
| 1093 | }
|
| 1094 |
|
| 1095 | static void hybrid_init_cpu_capacity_scaling(bool refresh)
|
| 1096 | {
|
| 1097 |
|
| 1098 | if (no_cas)
|
| 1099 | return;
|
| 1100 |
|
| 1101 | |
| 1102 | * If hybrid_max_perf_cpu is set at this point, the hybrid CPU capacity
|
| 1103 | * scaling has been enabled already and the driver is just changing the
|
| 1104 | * operation mode.
|
| 1105 |
|
| 1106 | if (refresh) {
|
| 1107 | hybrid_refresh_cpu_capacity_scaling();
|
| 1108 | return;
|
| 1109 | }
|
| 1110 |
|
| 1111 | |
| 1112 | * On hybrid systems, use asym capacity instead of ITMT, but because
|
| 1113 | * the capacity of SMT threads is not deterministic even approximately,
|
| 1114 | * do not do that when SMT is in use.
|
| 1115 |
|
| 1116 | if (hwp_is_hybrid && !cpu_smt_possible() && arch_enable_hybrid_capacity_scale()) {
|
| 1117 | hybrid_refresh_cpu_capacity_scaling();
|
| 1118 | |
| 1119 | * Disabling ITMT causes sched domains to be rebuilt to disable asym
|
| 1120 | * packing and enable asym capacity and EAS.
|
| 1121 |
|
| 1122 | sched_clear_itmt_support();
|
| 1123 | }
|
| 1124 | }
|
| 1125 |
|
| 1126 | static bool hybrid_clear_max_perf_cpu(void)
|
| 1127 | {
|
| 1128 | bool ret;
|
| 1129 |
|
| 1130 | guard(mutex)(&hybrid_capacity_lock);
|
| 1131 |
|
| 1132 | ret = !!hybrid_max_perf_cpu;
|
| 1133 | hybrid_max_perf_cpu = NULL;
|
| 1134 |
|
| 1135 | return ret;
|
| 1136 | }
|
| 1137 |
|
| 1138 | static unsigned int intel_pstate_scale_freq_ref(struct cpufreq_policy *policy)
|
| 1139 | {
|
| 1140 | if (READ_ONCE(all_cpu_data[policy->cpu]->capacity_perf))
|
| 1141 | return policy->cpuinfo.max_freq;
|
| 1142 |
|
| 1143 | return 0;
|
| 1144 | }
|
| 1145 |
|
| 1146 | static void intel_pstate_update_freq_limits(struct cpudata *cpu)
|
| 1147 | {
|
| 1148 | int scaling = cpu->pstate.scaling;
|
| 1149 | unsigned int turbo_freq = cpu->pstate.turbo_pstate * scaling;
|
| 1150 | unsigned int max_freq = cpu->pstate.max_pstate * scaling;
|
| 1151 | int perf_ctl_scaling = cpu->pstate.perf_ctl_scaling;
|
| 1152 |
|
| 1153 | if (scaling != perf_ctl_scaling) {
|
| 1154 | turbo_freq = rounddown(turbo_freq, perf_ctl_scaling);
|
| 1155 | max_freq = rounddown(max_freq, perf_ctl_scaling);
|
| 1156 | }
|
| 1157 |
|
| 1158 | cpu->pstate.turbo_freq = turbo_freq;
|
| 1159 | cpu->pstate.max_freq = max_freq;
|
| 1160 | }
|
| 1161 |
|
| 1162 | static void __intel_pstate_get_hwp_cap(struct cpudata *cpu)
|
| 1163 | {
|
| 1164 | u64 cap;
|
| 1165 |
|
| 1166 | rdmsrq_on_cpu(cpu->cpu, MSR_HWP_CAPABILITIES, &cap);
|
| 1167 | WRITE_ONCE(cpu->hwp_cap_cached, cap);
|
| 1168 | cpu->pstate.max_pstate = HWP_GUARANTEED_PERF(cap);
|
| 1169 | cpu->pstate.turbo_pstate = HWP_HIGHEST_PERF(cap);
|
| 1170 | }
|
| 1171 |
|
| 1172 | static void intel_pstate_get_hwp_cap(struct cpudata *cpu)
|
| 1173 | {
|
| 1174 | __intel_pstate_get_hwp_cap(cpu);
|
| 1175 | intel_pstate_update_freq_limits(cpu);
|
| 1176 | }
|
| 1177 |
|
| 1178 | static void hybrid_update_capacity(struct cpudata *cpu)
|
| 1179 | {
|
| 1180 | unsigned int max_cap_perf;
|
| 1181 |
|
| 1182 | mutex_lock(&hybrid_capacity_lock);
|
| 1183 |
|
| 1184 | if (!hybrid_max_perf_cpu)
|
| 1185 | goto unlock;
|
| 1186 |
|
| 1187 | |
| 1188 | * The maximum performance of the CPU may have changed, but assume
|
| 1189 | * that the performance of the other CPUs has not changed.
|
| 1190 |
|
| 1191 | max_cap_perf = hybrid_max_perf_cpu->capacity_perf;
|
| 1192 |
|
| 1193 | intel_pstate_get_hwp_cap(cpu);
|
| 1194 |
|
| 1195 | hybrid_get_capacity_perf(cpu);
|
| 1196 |
|
| 1197 | if (cpu->capacity_perf > max_cap_perf) {
|
| 1198 | hybrid_max_perf_cpu = cpu;
|
| 1199 | hybrid_set_capacity_of_cpus();
|
| 1200 | goto unlock;
|
| 1201 | }
|
| 1202 |
|
| 1203 |
|
| 1204 | if (cpu == hybrid_max_perf_cpu && cpu->capacity_perf < max_cap_perf) {
|
| 1205 | hybrid_update_cpu_capacity_scaling();
|
| 1206 | goto unlock;
|
| 1207 | }
|
| 1208 |
|
| 1209 | hybrid_set_cpu_capacity(cpu);
|
| 1210 | |
| 1211 | * If the CPU was offline to start with and it is going online for the
|
| 1212 | * first time, a perf domain needs to be registered for it if hybrid
|
| 1213 | * capacity scaling has been enabled already. In that case, sched
|
| 1214 | * domains need to be rebuilt to take the new perf domain into account.
|
| 1215 |
|
| 1216 | if (hybrid_register_perf_domain(cpu->cpu))
|
| 1217 | em_rebuild_sched_domains();
|
| 1218 |
|
| 1219 | unlock:
|
| 1220 | mutex_unlock(&hybrid_capacity_lock);
|
| 1221 | }
|
| 1222 |
|
| 1223 | static void intel_pstate_hwp_set(unsigned int cpu)
|
| 1224 | {
|
| 1225 | struct cpudata *cpu_data = all_cpu_data[cpu];
|
| 1226 | int max, min;
|
| 1227 | u64 value;
|
| 1228 | s16 epp;
|
| 1229 |
|
| 1230 | max = cpu_data->max_perf_ratio;
|
| 1231 | min = cpu_data->min_perf_ratio;
|
| 1232 |
|
| 1233 | if (cpu_data->policy == CPUFREQ_POLICY_PERFORMANCE)
|
| 1234 | min = max;
|
| 1235 |
|
| 1236 | rdmsrq_on_cpu(cpu, MSR_HWP_REQUEST, &value);
|
| 1237 |
|
| 1238 | value &= ~HWP_MIN_PERF(~0L);
|
| 1239 | value |= HWP_MIN_PERF(min);
|
| 1240 |
|
| 1241 | value &= ~HWP_MAX_PERF(~0L);
|
| 1242 | value |= HWP_MAX_PERF(max);
|
| 1243 |
|
| 1244 | if (cpu_data->epp_policy == cpu_data->policy)
|
| 1245 | goto skip_epp;
|
| 1246 |
|
| 1247 | cpu_data->epp_policy = cpu_data->policy;
|
| 1248 |
|
| 1249 | if (cpu_data->policy == CPUFREQ_POLICY_PERFORMANCE) {
|
| 1250 | epp = intel_pstate_get_epp(cpu_data, value);
|
| 1251 | cpu_data->epp_powersave = epp;
|
| 1252 |
|
| 1253 | if (epp < 0)
|
| 1254 | goto skip_epp;
|
| 1255 |
|
| 1256 | epp = 0;
|
| 1257 | } else {
|
| 1258 |
|
| 1259 | if (cpu_data->epp_powersave < 0)
|
| 1260 | goto skip_epp;
|
| 1261 |
|
| 1262 | |
| 1263 | * No need to restore EPP when it is not zero. This
|
| 1264 | * means:
|
| 1265 | * - Policy is not changed
|
| 1266 | * - user has manually changed
|
| 1267 | * - Error reading EPB
|
| 1268 |
|
| 1269 | epp = intel_pstate_get_epp(cpu_data, value);
|
| 1270 | if (epp)
|
| 1271 | goto skip_epp;
|
| 1272 |
|
| 1273 | epp = cpu_data->epp_powersave;
|
| 1274 | }
|
| 1275 | if (boot_cpu_has(X86_FEATURE_HWP_EPP)) {
|
| 1276 | value &= ~GENMASK_ULL(31, 24);
|
| 1277 | value |= (u64)epp << 24;
|
| 1278 | }
|
| 1279 |
|
| 1280 | skip_epp:
|
| 1281 | WRITE_ONCE(cpu_data->hwp_req_cached, value);
|
| 1282 | wrmsrq_on_cpu(cpu, MSR_HWP_REQUEST, value);
|
| 1283 | }
|
| 1284 |
|
| 1285 | static void intel_pstate_disable_hwp_interrupt(struct cpudata *cpudata);
|
| 1286 |
|
| 1287 | static void intel_pstate_hwp_offline(struct cpudata *cpu)
|
| 1288 | {
|
| 1289 | u64 value = READ_ONCE(cpu->hwp_req_cached);
|
| 1290 | int min_perf;
|
| 1291 |
|
| 1292 | intel_pstate_disable_hwp_interrupt(cpu);
|
| 1293 |
|
| 1294 | if (boot_cpu_has(X86_FEATURE_HWP_EPP)) {
|
| 1295 | |
| 1296 | * In case the EPP has been set to "performance" by the
|
| 1297 | * active mode "performance" scaling algorithm, replace that
|
| 1298 | * temporary value with the cached EPP one.
|
| 1299 |
|
| 1300 | value &= ~GENMASK_ULL(31, 24);
|
| 1301 | value |= HWP_ENERGY_PERF_PREFERENCE(cpu->epp_cached);
|
| 1302 | |
| 1303 | * However, make sure that EPP will be set to "performance" when
|
| 1304 | * the CPU is brought back online again and the "performance"
|
| 1305 | * scaling algorithm is still in effect.
|
| 1306 |
|
| 1307 | cpu->epp_policy = CPUFREQ_POLICY_UNKNOWN;
|
| 1308 | }
|
| 1309 |
|
| 1310 | |
| 1311 | * Clear the desired perf field in the cached HWP request value to
|
| 1312 | * prevent nonzero desired values from being leaked into the active
|
| 1313 | * mode.
|
| 1314 |
|
| 1315 | value &= ~HWP_DESIRED_PERF(~0L);
|
| 1316 | WRITE_ONCE(cpu->hwp_req_cached, value);
|
| 1317 |
|
| 1318 | value &= ~GENMASK_ULL(31, 0);
|
| 1319 | min_perf = HWP_LOWEST_PERF(READ_ONCE(cpu->hwp_cap_cached));
|
| 1320 |
|
| 1321 |
|
| 1322 | value |= HWP_MAX_PERF(min_perf);
|
| 1323 | value |= HWP_MIN_PERF(min_perf);
|
| 1324 |
|
| 1325 |
|
| 1326 | if (boot_cpu_has(X86_FEATURE_HWP_EPP))
|
| 1327 | value |= HWP_ENERGY_PERF_PREFERENCE(HWP_EPP_POWERSAVE);
|
| 1328 |
|
| 1329 | wrmsrq_on_cpu(cpu->cpu, MSR_HWP_REQUEST, value);
|
| 1330 |
|
| 1331 | mutex_lock(&hybrid_capacity_lock);
|
| 1332 |
|
| 1333 | if (!hybrid_max_perf_cpu) {
|
| 1334 | mutex_unlock(&hybrid_capacity_lock);
|
| 1335 |
|
| 1336 | return;
|
| 1337 | }
|
| 1338 |
|
| 1339 | if (hybrid_max_perf_cpu == cpu)
|
| 1340 | hybrid_update_cpu_capacity_scaling();
|
| 1341 |
|
| 1342 | mutex_unlock(&hybrid_capacity_lock);
|
| 1343 |
|
| 1344 |
|
| 1345 | hybrid_clear_cpu_capacity(cpu->cpu);
|
| 1346 | }
|
| 1347 |
|
| 1348 | #define POWER_CTL_EE_ENABLE 1
|
| 1349 | #define POWER_CTL_EE_DISABLE 2
|
| 1350 |
|
| 1351 |
|
| 1352 | #define POWER_CTL_DEC_ENABLE 27
|
| 1353 |
|
| 1354 | static int power_ctl_ee_state;
|
| 1355 |
|
| 1356 | static void set_power_ctl_ee_state(bool input)
|
| 1357 | {
|
| 1358 | u64 power_ctl;
|
| 1359 |
|
| 1360 | guard(mutex)(&intel_pstate_driver_lock);
|
| 1361 |
|
| 1362 | rdmsrq(MSR_IA32_POWER_CTL, power_ctl);
|
| 1363 | if (input) {
|
| 1364 | power_ctl &= ~BIT(MSR_IA32_POWER_CTL_BIT_EE);
|
| 1365 | power_ctl_ee_state = POWER_CTL_EE_ENABLE;
|
| 1366 | } else {
|
| 1367 | power_ctl |= BIT(MSR_IA32_POWER_CTL_BIT_EE);
|
| 1368 | power_ctl_ee_state = POWER_CTL_EE_DISABLE;
|
| 1369 | }
|
| 1370 | wrmsrq(MSR_IA32_POWER_CTL, power_ctl);
|
| 1371 | }
|
| 1372 |
|
| 1373 | static void intel_pstate_hwp_enable(struct cpudata *cpudata);
|
| 1374 |
|
| 1375 | static void intel_pstate_hwp_reenable(struct cpudata *cpu)
|
| 1376 | {
|
| 1377 | intel_pstate_hwp_enable(cpu);
|
| 1378 | wrmsrq_on_cpu(cpu->cpu, MSR_HWP_REQUEST, READ_ONCE(cpu->hwp_req_cached));
|
| 1379 | }
|
| 1380 |
|
| 1381 | static int intel_pstate_suspend(struct cpufreq_policy *policy)
|
| 1382 | {
|
| 1383 | struct cpudata *cpu = all_cpu_data[policy->cpu];
|
| 1384 |
|
| 1385 | pr_debug("CPU %d suspending\n", cpu->cpu);
|
| 1386 |
|
| 1387 | cpu->suspended = true;
|
| 1388 |
|
| 1389 |
|
| 1390 | intel_pstate_disable_hwp_interrupt(cpu);
|
| 1391 |
|
| 1392 | return 0;
|
| 1393 | }
|
| 1394 |
|
| 1395 | static int intel_pstate_resume(struct cpufreq_policy *policy)
|
| 1396 | {
|
| 1397 | struct cpudata *cpu = all_cpu_data[policy->cpu];
|
| 1398 |
|
| 1399 | pr_debug("CPU %d resuming\n", cpu->cpu);
|
| 1400 |
|
| 1401 |
|
| 1402 | if (power_ctl_ee_state == POWER_CTL_EE_ENABLE)
|
| 1403 | set_power_ctl_ee_state(true);
|
| 1404 | else if (power_ctl_ee_state == POWER_CTL_EE_DISABLE)
|
| 1405 | set_power_ctl_ee_state(false);
|
| 1406 |
|
| 1407 | if (cpu->suspended && hwp_active) {
|
| 1408 | mutex_lock(&intel_pstate_limits_lock);
|
| 1409 |
|
| 1410 |
|
| 1411 | intel_pstate_hwp_reenable(cpu);
|
| 1412 |
|
| 1413 | mutex_unlock(&intel_pstate_limits_lock);
|
| 1414 | }
|
| 1415 |
|
| 1416 | cpu->suspended = false;
|
| 1417 |
|
| 1418 | return 0;
|
| 1419 | }
|
| 1420 |
|
| 1421 | static void intel_pstate_update_policies(void)
|
| 1422 | {
|
| 1423 | int cpu;
|
| 1424 |
|
| 1425 | for_each_possible_cpu(cpu)
|
| 1426 | cpufreq_update_policy(cpu);
|
| 1427 | }
|
| 1428 |
|
| 1429 | static void __intel_pstate_update_max_freq(struct cpufreq_policy *policy,
|
| 1430 | struct cpudata *cpudata)
|
| 1431 | {
|
| 1432 | guard(cpufreq_policy_write)(policy);
|
| 1433 |
|
| 1434 | if (hwp_active)
|
| 1435 | intel_pstate_get_hwp_cap(cpudata);
|
| 1436 |
|
| 1437 | policy->cpuinfo.max_freq = READ_ONCE(global.no_turbo) ?
|
| 1438 | cpudata->pstate.max_freq : cpudata->pstate.turbo_freq;
|
| 1439 |
|
| 1440 | refresh_frequency_limits(policy);
|
| 1441 | }
|
| 1442 |
|
| 1443 | static bool intel_pstate_update_max_freq(int cpu)
|
| 1444 | {
|
| 1445 | struct cpufreq_policy *policy __free(put_cpufreq_policy) = cpufreq_cpu_get(cpu);
|
| 1446 | if (!policy)
|
| 1447 | return false;
|
| 1448 |
|
| 1449 | __intel_pstate_update_max_freq(policy, all_cpu_data[cpu]);
|
| 1450 |
|
| 1451 | return true;
|
| 1452 | }
|
| 1453 |
|
| 1454 | static void intel_pstate_update_limits(struct cpufreq_policy *policy)
|
| 1455 | {
|
| 1456 | struct cpudata *cpudata = all_cpu_data[policy->cpu];
|
| 1457 |
|
| 1458 | __intel_pstate_update_max_freq(policy, cpudata);
|
| 1459 |
|
| 1460 | hybrid_update_capacity(cpudata);
|
| 1461 | }
|
| 1462 |
|
| 1463 | static void intel_pstate_update_limits_for_all(void)
|
| 1464 | {
|
| 1465 | int cpu;
|
| 1466 |
|
| 1467 | for_each_possible_cpu(cpu)
|
| 1468 | intel_pstate_update_max_freq(cpu);
|
| 1469 |
|
| 1470 | mutex_lock(&hybrid_capacity_lock);
|
| 1471 |
|
| 1472 | if (hybrid_max_perf_cpu)
|
| 1473 | __hybrid_refresh_cpu_capacity_scaling();
|
| 1474 |
|
| 1475 | mutex_unlock(&hybrid_capacity_lock);
|
| 1476 | }
|
| 1477 |
|
| 1478 |
|
| 1479 | #define show_one(file_name, object) \
|
| 1480 | static ssize_t show_##file_name \
|
| 1481 | (struct kobject *kobj, struct kobj_attribute *attr, char *buf) \
|
| 1482 | { \
|
| 1483 | return sprintf(buf, "%u\n", global.object); \
|
| 1484 | }
|
| 1485 |
|
| 1486 | static ssize_t intel_pstate_show_status(char *buf);
|
| 1487 | static int intel_pstate_update_status(const char *buf, size_t size);
|
| 1488 |
|
| 1489 | static ssize_t show_status(struct kobject *kobj,
|
| 1490 | struct kobj_attribute *attr, char *buf)
|
| 1491 | {
|
| 1492 | guard(mutex)(&intel_pstate_driver_lock);
|
| 1493 |
|
| 1494 | return intel_pstate_show_status(buf);
|
| 1495 | }
|
| 1496 |
|
| 1497 | static ssize_t store_status(struct kobject *a, struct kobj_attribute *b,
|
| 1498 | const char *buf, size_t count)
|
| 1499 | {
|
| 1500 | char *p = memchr(buf, '\n', count);
|
| 1501 | int ret;
|
| 1502 |
|
| 1503 | guard(mutex)(&intel_pstate_driver_lock);
|
| 1504 |
|
| 1505 | ret = intel_pstate_update_status(buf, p ? p - buf : count);
|
| 1506 | if (ret < 0)
|
| 1507 | return ret;
|
| 1508 |
|
| 1509 | return count;
|
| 1510 | }
|
| 1511 |
|
| 1512 | static ssize_t show_turbo_pct(struct kobject *kobj,
|
| 1513 | struct kobj_attribute *attr, char *buf)
|
| 1514 | {
|
| 1515 | struct cpudata *cpu;
|
| 1516 | int total, no_turbo, turbo_pct;
|
| 1517 | uint32_t turbo_fp;
|
| 1518 |
|
| 1519 | guard(mutex)(&intel_pstate_driver_lock);
|
| 1520 |
|
| 1521 | if (!intel_pstate_driver)
|
| 1522 | return -EAGAIN;
|
| 1523 |
|
| 1524 | cpu = all_cpu_data[0];
|
| 1525 |
|
| 1526 | total = cpu->pstate.turbo_pstate - cpu->pstate.min_pstate + 1;
|
| 1527 | no_turbo = cpu->pstate.max_pstate - cpu->pstate.min_pstate + 1;
|
| 1528 | turbo_fp = div_fp(no_turbo, total);
|
| 1529 | turbo_pct = 100 - fp_toint(mul_fp(turbo_fp, int_tofp(100)));
|
| 1530 |
|
| 1531 | return sprintf(buf, "%u\n", turbo_pct);
|
| 1532 | }
|
| 1533 |
|
| 1534 | static ssize_t show_num_pstates(struct kobject *kobj,
|
| 1535 | struct kobj_attribute *attr, char *buf)
|
| 1536 | {
|
| 1537 | struct cpudata *cpu;
|
| 1538 | int total;
|
| 1539 |
|
| 1540 | guard(mutex)(&intel_pstate_driver_lock);
|
| 1541 |
|
| 1542 | if (!intel_pstate_driver)
|
| 1543 | return -EAGAIN;
|
| 1544 |
|
| 1545 | cpu = all_cpu_data[0];
|
| 1546 | total = cpu->pstate.turbo_pstate - cpu->pstate.min_pstate + 1;
|
| 1547 |
|
| 1548 | return sprintf(buf, "%u\n", total);
|
| 1549 | }
|
| 1550 |
|
| 1551 | static ssize_t show_no_turbo(struct kobject *kobj,
|
| 1552 | struct kobj_attribute *attr, char *buf)
|
| 1553 | {
|
| 1554 | guard(mutex)(&intel_pstate_driver_lock);
|
| 1555 |
|
| 1556 | if (!intel_pstate_driver)
|
| 1557 | return -EAGAIN;
|
| 1558 |
|
| 1559 | return sprintf(buf, "%u\n", global.no_turbo);
|
| 1560 | }
|
| 1561 |
|
| 1562 | static ssize_t store_no_turbo(struct kobject *a, struct kobj_attribute *b,
|
| 1563 | const char *buf, size_t count)
|
| 1564 | {
|
| 1565 | unsigned int input;
|
| 1566 | bool no_turbo;
|
| 1567 |
|
| 1568 | if (sscanf(buf, "%u", &input) != 1)
|
| 1569 | return -EINVAL;
|
| 1570 |
|
| 1571 | guard(mutex)(&intel_pstate_driver_lock);
|
| 1572 |
|
| 1573 | if (!intel_pstate_driver)
|
| 1574 | return -EAGAIN;
|
| 1575 |
|
| 1576 | no_turbo = !!clamp_t(int, input, 0, 1);
|
| 1577 |
|
| 1578 | WRITE_ONCE(global.turbo_disabled, turbo_is_disabled());
|
| 1579 | if (global.turbo_disabled && !no_turbo) {
|
| 1580 | pr_notice("Turbo disabled by BIOS or unavailable on processor\n");
|
| 1581 | if (global.no_turbo)
|
| 1582 | return -EPERM;
|
| 1583 |
|
| 1584 | no_turbo = 1;
|
| 1585 | }
|
| 1586 |
|
| 1587 | if (no_turbo == global.no_turbo)
|
| 1588 | return count;
|
| 1589 |
|
| 1590 | WRITE_ONCE(global.no_turbo, no_turbo);
|
| 1591 |
|
| 1592 | mutex_lock(&intel_pstate_limits_lock);
|
| 1593 |
|
| 1594 | if (no_turbo) {
|
| 1595 | struct cpudata *cpu = all_cpu_data[0];
|
| 1596 | int pct = cpu->pstate.max_pstate * 100 / cpu->pstate.turbo_pstate;
|
| 1597 |
|
| 1598 |
|
| 1599 | if (global.min_perf_pct > pct)
|
| 1600 | global.min_perf_pct = pct;
|
| 1601 | }
|
| 1602 |
|
| 1603 | mutex_unlock(&intel_pstate_limits_lock);
|
| 1604 |
|
| 1605 | intel_pstate_update_limits_for_all();
|
| 1606 | arch_set_max_freq_ratio(no_turbo);
|
| 1607 |
|
| 1608 | return count;
|
| 1609 | }
|
| 1610 |
|
| 1611 | static void update_cpu_qos_request(int cpu, enum freq_qos_req_type type)
|
| 1612 | {
|
| 1613 | struct cpudata *cpudata = all_cpu_data[cpu];
|
| 1614 | struct freq_qos_request *req;
|
| 1615 | unsigned int freq;
|
| 1616 |
|
| 1617 | struct cpufreq_policy *policy __free(put_cpufreq_policy) = cpufreq_cpu_get(cpu);
|
| 1618 | if (!policy)
|
| 1619 | return;
|
| 1620 |
|
| 1621 | req = policy->driver_data;
|
| 1622 | if (!req)
|
| 1623 | return;
|
| 1624 |
|
| 1625 | if (hwp_active)
|
| 1626 | intel_pstate_get_hwp_cap(cpudata);
|
| 1627 |
|
| 1628 | freq = cpudata->pstate.turbo_freq;
|
| 1629 |
|
| 1630 | if (type == FREQ_QOS_MIN) {
|
| 1631 | freq = DIV_ROUND_UP(freq * global.min_perf_pct, 100);
|
| 1632 | } else {
|
| 1633 | req++;
|
| 1634 | freq = (freq * global.max_perf_pct) / 100;
|
| 1635 | }
|
| 1636 |
|
| 1637 | if (freq_qos_update_request(req, freq) < 0)
|
| 1638 | pr_warn("Failed to update freq constraint: CPU%d\n", cpu);
|
| 1639 | }
|
| 1640 |
|
| 1641 | static void update_qos_requests(enum freq_qos_req_type type)
|
| 1642 | {
|
| 1643 | int i;
|
| 1644 |
|
| 1645 | for_each_possible_cpu(i)
|
| 1646 | update_cpu_qos_request(i, type);
|
| 1647 | }
|
| 1648 |
|
| 1649 | static ssize_t store_max_perf_pct(struct kobject *a, struct kobj_attribute *b,
|
| 1650 | const char *buf, size_t count)
|
| 1651 | {
|
| 1652 | unsigned int input;
|
| 1653 | int ret;
|
| 1654 |
|
| 1655 | ret = sscanf(buf, "%u", &input);
|
| 1656 | if (ret != 1)
|
| 1657 | return -EINVAL;
|
| 1658 |
|
| 1659 | guard(mutex)(&intel_pstate_driver_lock);
|
| 1660 |
|
| 1661 | if (!intel_pstate_driver)
|
| 1662 | return -EAGAIN;
|
| 1663 |
|
| 1664 | mutex_lock(&intel_pstate_limits_lock);
|
| 1665 |
|
| 1666 | global.max_perf_pct = clamp_t(int, input, global.min_perf_pct, 100);
|
| 1667 |
|
| 1668 | mutex_unlock(&intel_pstate_limits_lock);
|
| 1669 |
|
| 1670 | if (intel_pstate_driver == &intel_pstate)
|
| 1671 | intel_pstate_update_policies();
|
| 1672 | else
|
| 1673 | update_qos_requests(FREQ_QOS_MAX);
|
| 1674 |
|
| 1675 | return count;
|
| 1676 | }
|
| 1677 |
|
| 1678 | static ssize_t store_min_perf_pct(struct kobject *a, struct kobj_attribute *b,
|
| 1679 | const char *buf, size_t count)
|
| 1680 | {
|
| 1681 | unsigned int input;
|
| 1682 | int ret;
|
| 1683 |
|
| 1684 | ret = sscanf(buf, "%u", &input);
|
| 1685 | if (ret != 1)
|
| 1686 | return -EINVAL;
|
| 1687 |
|
| 1688 | guard(mutex)(&intel_pstate_driver_lock);
|
| 1689 |
|
| 1690 | if (!intel_pstate_driver)
|
| 1691 | return -EAGAIN;
|
| 1692 |
|
| 1693 | mutex_lock(&intel_pstate_limits_lock);
|
| 1694 |
|
| 1695 | global.min_perf_pct = clamp_t(int, input,
|
| 1696 | min_perf_pct_min(), global.max_perf_pct);
|
| 1697 |
|
| 1698 | mutex_unlock(&intel_pstate_limits_lock);
|
| 1699 |
|
| 1700 | if (intel_pstate_driver == &intel_pstate)
|
| 1701 | intel_pstate_update_policies();
|
| 1702 | else
|
| 1703 | update_qos_requests(FREQ_QOS_MIN);
|
| 1704 |
|
| 1705 | return count;
|
| 1706 | }
|
| 1707 |
|
| 1708 | static ssize_t show_hwp_dynamic_boost(struct kobject *kobj,
|
| 1709 | struct kobj_attribute *attr, char *buf)
|
| 1710 | {
|
| 1711 | return sprintf(buf, "%u\n", hwp_boost);
|
| 1712 | }
|
| 1713 |
|
| 1714 | static ssize_t store_hwp_dynamic_boost(struct kobject *a,
|
| 1715 | struct kobj_attribute *b,
|
| 1716 | const char *buf, size_t count)
|
| 1717 | {
|
| 1718 | unsigned int input;
|
| 1719 | int ret;
|
| 1720 |
|
| 1721 | ret = kstrtouint(buf, 10, &input);
|
| 1722 | if (ret)
|
| 1723 | return ret;
|
| 1724 |
|
| 1725 | guard(mutex)(&intel_pstate_driver_lock);
|
| 1726 |
|
| 1727 | hwp_boost = !!input;
|
| 1728 | intel_pstate_update_policies();
|
| 1729 |
|
| 1730 | return count;
|
| 1731 | }
|
| 1732 |
|
| 1733 | static ssize_t show_energy_efficiency(struct kobject *kobj, struct kobj_attribute *attr,
|
| 1734 | char *buf)
|
| 1735 | {
|
| 1736 | u64 power_ctl;
|
| 1737 | int enable;
|
| 1738 |
|
| 1739 | rdmsrq(MSR_IA32_POWER_CTL, power_ctl);
|
| 1740 | enable = !!(power_ctl & BIT(MSR_IA32_POWER_CTL_BIT_EE));
|
| 1741 | return sprintf(buf, "%d\n", !enable);
|
| 1742 | }
|
| 1743 |
|
| 1744 | static ssize_t store_energy_efficiency(struct kobject *a, struct kobj_attribute *b,
|
| 1745 | const char *buf, size_t count)
|
| 1746 | {
|
| 1747 | bool input;
|
| 1748 | int ret;
|
| 1749 |
|
| 1750 | ret = kstrtobool(buf, &input);
|
| 1751 | if (ret)
|
| 1752 | return ret;
|
| 1753 |
|
| 1754 | set_power_ctl_ee_state(input);
|
| 1755 |
|
| 1756 | return count;
|
| 1757 | }
|
| 1758 |
|
| 1759 | show_one(max_perf_pct, max_perf_pct);
|
| 1760 | show_one(min_perf_pct, min_perf_pct);
|
| 1761 |
|
| 1762 | define_one_global_rw(status);
|
| 1763 | define_one_global_rw(no_turbo);
|
| 1764 | define_one_global_rw(max_perf_pct);
|
| 1765 | define_one_global_rw(min_perf_pct);
|
| 1766 | define_one_global_ro(turbo_pct);
|
| 1767 | define_one_global_ro(num_pstates);
|
| 1768 | define_one_global_rw(hwp_dynamic_boost);
|
| 1769 | define_one_global_rw(energy_efficiency);
|
| 1770 |
|
| 1771 | static struct attribute *intel_pstate_attributes[] = {
|
| 1772 | &status.attr,
|
| 1773 | &no_turbo.attr,
|
| 1774 | NULL
|
| 1775 | };
|
| 1776 |
|
| 1777 | static const struct attribute_group intel_pstate_attr_group = {
|
| 1778 | .attrs = intel_pstate_attributes,
|
| 1779 | };
|
| 1780 |
|
| 1781 | static const struct x86_cpu_id intel_pstate_cpu_ee_disable_ids[];
|
| 1782 |
|
| 1783 | static struct kobject *intel_pstate_kobject;
|
| 1784 |
|
| 1785 | static void __init intel_pstate_sysfs_expose_params(void)
|
| 1786 | {
|
| 1787 | struct device *dev_root = bus_get_dev_root(&cpu_subsys);
|
| 1788 | int rc;
|
| 1789 |
|
| 1790 | if (dev_root) {
|
| 1791 | intel_pstate_kobject = kobject_create_and_add("intel_pstate", &dev_root->kobj);
|
| 1792 | put_device(dev_root);
|
| 1793 | }
|
| 1794 | if (WARN_ON(!intel_pstate_kobject))
|
| 1795 | return;
|
| 1796 |
|
| 1797 | rc = sysfs_create_group(intel_pstate_kobject, &intel_pstate_attr_group);
|
| 1798 | if (WARN_ON(rc))
|
| 1799 | return;
|
| 1800 |
|
| 1801 | if (!boot_cpu_has(X86_FEATURE_HYBRID_CPU)) {
|
| 1802 | rc = sysfs_create_file(intel_pstate_kobject, &turbo_pct.attr);
|
| 1803 | WARN_ON(rc);
|
| 1804 |
|
| 1805 | rc = sysfs_create_file(intel_pstate_kobject, &num_pstates.attr);
|
| 1806 | WARN_ON(rc);
|
| 1807 | }
|
| 1808 |
|
| 1809 | |
| 1810 | * If per cpu limits are enforced there are no global limits, so
|
| 1811 | * return without creating max/min_perf_pct attributes
|
| 1812 |
|
| 1813 | if (per_cpu_limits)
|
| 1814 | return;
|
| 1815 |
|
| 1816 | rc = sysfs_create_file(intel_pstate_kobject, &max_perf_pct.attr);
|
| 1817 | WARN_ON(rc);
|
| 1818 |
|
| 1819 | rc = sysfs_create_file(intel_pstate_kobject, &min_perf_pct.attr);
|
| 1820 | WARN_ON(rc);
|
| 1821 |
|
| 1822 | if (x86_match_cpu(intel_pstate_cpu_ee_disable_ids)) {
|
| 1823 | rc = sysfs_create_file(intel_pstate_kobject, &energy_efficiency.attr);
|
| 1824 | WARN_ON(rc);
|
| 1825 | }
|
| 1826 | }
|
| 1827 |
|
| 1828 | static void __init intel_pstate_sysfs_remove(void)
|
| 1829 | {
|
| 1830 | if (!intel_pstate_kobject)
|
| 1831 | return;
|
| 1832 |
|
| 1833 | sysfs_remove_group(intel_pstate_kobject, &intel_pstate_attr_group);
|
| 1834 |
|
| 1835 | if (!boot_cpu_has(X86_FEATURE_HYBRID_CPU)) {
|
| 1836 | sysfs_remove_file(intel_pstate_kobject, &num_pstates.attr);
|
| 1837 | sysfs_remove_file(intel_pstate_kobject, &turbo_pct.attr);
|
| 1838 | }
|
| 1839 |
|
| 1840 | if (!per_cpu_limits) {
|
| 1841 | sysfs_remove_file(intel_pstate_kobject, &max_perf_pct.attr);
|
| 1842 | sysfs_remove_file(intel_pstate_kobject, &min_perf_pct.attr);
|
| 1843 |
|
| 1844 | if (x86_match_cpu(intel_pstate_cpu_ee_disable_ids))
|
| 1845 | sysfs_remove_file(intel_pstate_kobject, &energy_efficiency.attr);
|
| 1846 | }
|
| 1847 |
|
| 1848 | kobject_put(intel_pstate_kobject);
|
| 1849 | }
|
| 1850 |
|
| 1851 | static void intel_pstate_sysfs_expose_hwp_dynamic_boost(void)
|
| 1852 | {
|
| 1853 | int rc;
|
| 1854 |
|
| 1855 | if (!hwp_active)
|
| 1856 | return;
|
| 1857 |
|
| 1858 | rc = sysfs_create_file(intel_pstate_kobject, &hwp_dynamic_boost.attr);
|
| 1859 | WARN_ON_ONCE(rc);
|
| 1860 | }
|
| 1861 |
|
| 1862 | static void intel_pstate_sysfs_hide_hwp_dynamic_boost(void)
|
| 1863 | {
|
| 1864 | if (!hwp_active)
|
| 1865 | return;
|
| 1866 |
|
| 1867 | sysfs_remove_file(intel_pstate_kobject, &hwp_dynamic_boost.attr);
|
| 1868 | }
|
| 1869 |
|
| 1870 |
|
| 1871 |
|
| 1872 | static void intel_pstate_notify_work(struct work_struct *work)
|
| 1873 | {
|
| 1874 | struct cpudata *cpudata =
|
| 1875 | container_of(to_delayed_work(work), struct cpudata, hwp_notify_work);
|
| 1876 |
|
| 1877 | if (intel_pstate_update_max_freq(cpudata->cpu)) {
|
| 1878 | |
| 1879 | * The driver will not be unregistered while this function is
|
| 1880 | * running, so update the capacity without acquiring the driver
|
| 1881 | * lock.
|
| 1882 |
|
| 1883 | hybrid_update_capacity(cpudata);
|
| 1884 | }
|
| 1885 |
|
| 1886 | wrmsrq_on_cpu(cpudata->cpu, MSR_HWP_STATUS, 0);
|
| 1887 | }
|
| 1888 |
|
| 1889 | static DEFINE_RAW_SPINLOCK(hwp_notify_lock);
|
| 1890 | static cpumask_t hwp_intr_enable_mask;
|
| 1891 |
|
| 1892 | #define HWP_GUARANTEED_PERF_CHANGE_STATUS BIT(0)
|
| 1893 | #define HWP_HIGHEST_PERF_CHANGE_STATUS BIT(3)
|
| 1894 |
|
| 1895 | void notify_hwp_interrupt(void)
|
| 1896 | {
|
| 1897 | unsigned int this_cpu = smp_processor_id();
|
| 1898 | u64 value, status_mask;
|
| 1899 | unsigned long flags;
|
| 1900 |
|
| 1901 | if (!hwp_active || !cpu_feature_enabled(X86_FEATURE_HWP_NOTIFY))
|
| 1902 | return;
|
| 1903 |
|
| 1904 | status_mask = HWP_GUARANTEED_PERF_CHANGE_STATUS;
|
| 1905 | if (cpu_feature_enabled(X86_FEATURE_HWP_HIGHEST_PERF_CHANGE))
|
| 1906 | status_mask |= HWP_HIGHEST_PERF_CHANGE_STATUS;
|
| 1907 |
|
| 1908 | rdmsrq_safe(MSR_HWP_STATUS, &value);
|
| 1909 | if (!(value & status_mask))
|
| 1910 | return;
|
| 1911 |
|
| 1912 | raw_spin_lock_irqsave(&hwp_notify_lock, flags);
|
| 1913 |
|
| 1914 | if (!cpumask_test_cpu(this_cpu, &hwp_intr_enable_mask))
|
| 1915 | goto ack_intr;
|
| 1916 |
|
| 1917 | schedule_delayed_work(&all_cpu_data[this_cpu]->hwp_notify_work,
|
| 1918 | msecs_to_jiffies(10));
|
| 1919 |
|
| 1920 | raw_spin_unlock_irqrestore(&hwp_notify_lock, flags);
|
| 1921 |
|
| 1922 | return;
|
| 1923 |
|
| 1924 | ack_intr:
|
| 1925 | wrmsrq_safe(MSR_HWP_STATUS, 0);
|
| 1926 | raw_spin_unlock_irqrestore(&hwp_notify_lock, flags);
|
| 1927 | }
|
| 1928 |
|
| 1929 | static void intel_pstate_disable_hwp_interrupt(struct cpudata *cpudata)
|
| 1930 | {
|
| 1931 | bool cancel_work;
|
| 1932 |
|
| 1933 | if (!cpu_feature_enabled(X86_FEATURE_HWP_NOTIFY))
|
| 1934 | return;
|
| 1935 |
|
| 1936 |
|
| 1937 | wrmsrq_on_cpu(cpudata->cpu, MSR_HWP_INTERRUPT, 0x00);
|
| 1938 |
|
| 1939 | raw_spin_lock_irq(&hwp_notify_lock);
|
| 1940 | cancel_work = cpumask_test_and_clear_cpu(cpudata->cpu, &hwp_intr_enable_mask);
|
| 1941 | raw_spin_unlock_irq(&hwp_notify_lock);
|
| 1942 |
|
| 1943 | if (cancel_work)
|
| 1944 | cancel_delayed_work_sync(&cpudata->hwp_notify_work);
|
| 1945 | }
|
| 1946 |
|
| 1947 | #define HWP_GUARANTEED_PERF_CHANGE_REQ BIT(0)
|
| 1948 | #define HWP_HIGHEST_PERF_CHANGE_REQ BIT(2)
|
| 1949 |
|
| 1950 | static void intel_pstate_enable_hwp_interrupt(struct cpudata *cpudata)
|
| 1951 | {
|
| 1952 |
|
| 1953 | if (boot_cpu_has(X86_FEATURE_HWP_NOTIFY)) {
|
| 1954 | u64 interrupt_mask = HWP_GUARANTEED_PERF_CHANGE_REQ;
|
| 1955 |
|
| 1956 | raw_spin_lock_irq(&hwp_notify_lock);
|
| 1957 | INIT_DELAYED_WORK(&cpudata->hwp_notify_work, intel_pstate_notify_work);
|
| 1958 | cpumask_set_cpu(cpudata->cpu, &hwp_intr_enable_mask);
|
| 1959 | raw_spin_unlock_irq(&hwp_notify_lock);
|
| 1960 |
|
| 1961 | if (cpu_feature_enabled(X86_FEATURE_HWP_HIGHEST_PERF_CHANGE))
|
| 1962 | interrupt_mask |= HWP_HIGHEST_PERF_CHANGE_REQ;
|
| 1963 |
|
| 1964 |
|
| 1965 | wrmsrq_on_cpu(cpudata->cpu, MSR_HWP_INTERRUPT, interrupt_mask);
|
| 1966 | wrmsrq_on_cpu(cpudata->cpu, MSR_HWP_STATUS, 0);
|
| 1967 | }
|
| 1968 | }
|
| 1969 |
|
| 1970 | static void intel_pstate_update_epp_defaults(struct cpudata *cpudata)
|
| 1971 | {
|
| 1972 | cpudata->epp_default = intel_pstate_get_epp(cpudata, 0);
|
| 1973 |
|
| 1974 | |
| 1975 | * If the EPP is set by firmware, which means that firmware enabled HWP
|
| 1976 | * - Is equal or less than 0x80 (default balance_perf EPP)
|
| 1977 | * - But less performance oriented than performance EPP
|
| 1978 | * then use this as new balance_perf EPP.
|
| 1979 |
|
| 1980 | if (hwp_forced && cpudata->epp_default <= HWP_EPP_BALANCE_PERFORMANCE &&
|
| 1981 | cpudata->epp_default > HWP_EPP_PERFORMANCE) {
|
| 1982 | epp_values[EPP_INDEX_BALANCE_PERFORMANCE] = cpudata->epp_default;
|
| 1983 | return;
|
| 1984 | }
|
| 1985 |
|
| 1986 | |
| 1987 | * If this CPU gen doesn't call for change in balance_perf
|
| 1988 | * EPP return.
|
| 1989 |
|
| 1990 | if (epp_values[EPP_INDEX_BALANCE_PERFORMANCE] == HWP_EPP_BALANCE_PERFORMANCE)
|
| 1991 | return;
|
| 1992 |
|
| 1993 | |
| 1994 | * Use hard coded value per gen to update the balance_perf
|
| 1995 | * and default EPP.
|
| 1996 |
|
| 1997 | cpudata->epp_default = epp_values[EPP_INDEX_BALANCE_PERFORMANCE];
|
| 1998 | intel_pstate_set_epp(cpudata, cpudata->epp_default);
|
| 1999 | }
|
| 2000 |
|
| 2001 | static void intel_pstate_hwp_enable(struct cpudata *cpudata)
|
| 2002 | {
|
| 2003 |
|
| 2004 | if (boot_cpu_has(X86_FEATURE_HWP_NOTIFY))
|
| 2005 | wrmsrq_on_cpu(cpudata->cpu, MSR_HWP_INTERRUPT, 0x00);
|
| 2006 |
|
| 2007 | wrmsrq_on_cpu(cpudata->cpu, MSR_PM_ENABLE, 0x1);
|
| 2008 |
|
| 2009 | intel_pstate_enable_hwp_interrupt(cpudata);
|
| 2010 |
|
| 2011 | if (cpudata->epp_default >= 0)
|
| 2012 | return;
|
| 2013 |
|
| 2014 | intel_pstate_update_epp_defaults(cpudata);
|
| 2015 | }
|
| 2016 |
|
| 2017 | static u64 get_perf_ctl_val(int pstate)
|
| 2018 | {
|
| 2019 | u64 val;
|
| 2020 |
|
| 2021 | val = (u64)pstate << 8;
|
| 2022 | if (READ_ONCE(global.no_turbo) && !READ_ONCE(global.turbo_disabled) &&
|
| 2023 | cpu_feature_enabled(X86_FEATURE_IDA))
|
| 2024 | val |= (u64)1 << 32;
|
| 2025 |
|
| 2026 | return val;
|
| 2027 | }
|
| 2028 |
|
| 2029 | static int atom_get_min_pstate(int not_used)
|
| 2030 | {
|
| 2031 | u64 value;
|
| 2032 |
|
| 2033 | rdmsrq(MSR_ATOM_CORE_RATIOS, value);
|
| 2034 | return (value >> 8) & 0x7F;
|
| 2035 | }
|
| 2036 |
|
| 2037 | static int atom_get_max_pstate(int not_used)
|
| 2038 | {
|
| 2039 | u64 value;
|
| 2040 |
|
| 2041 | rdmsrq(MSR_ATOM_CORE_RATIOS, value);
|
| 2042 | return (value >> 16) & 0x7F;
|
| 2043 | }
|
| 2044 |
|
| 2045 | static int atom_get_turbo_pstate(int not_used)
|
| 2046 | {
|
| 2047 | u64 value;
|
| 2048 |
|
| 2049 | rdmsrq(MSR_ATOM_CORE_TURBO_RATIOS, value);
|
| 2050 | return value & 0x7F;
|
| 2051 | }
|
| 2052 |
|
| 2053 | static u64 atom_get_val(struct cpudata *cpudata, int pstate)
|
| 2054 | {
|
| 2055 | u64 val = get_perf_ctl_val(pstate);
|
| 2056 | int32_t vid_fp;
|
| 2057 | u32 vid;
|
| 2058 |
|
| 2059 | vid_fp = cpudata->vid.min + mul_fp(
|
| 2060 | int_tofp(pstate - cpudata->pstate.min_pstate),
|
| 2061 | cpudata->vid.ratio);
|
| 2062 |
|
| 2063 | vid_fp = clamp_t(int32_t, vid_fp, cpudata->vid.min, cpudata->vid.max);
|
| 2064 | vid = ceiling_fp(vid_fp);
|
| 2065 |
|
| 2066 | if (pstate > cpudata->pstate.max_pstate)
|
| 2067 | vid = cpudata->vid.turbo;
|
| 2068 |
|
| 2069 | return val | vid;
|
| 2070 | }
|
| 2071 |
|
| 2072 | static int silvermont_get_scaling(void)
|
| 2073 | {
|
| 2074 | u64 value;
|
| 2075 | int i;
|
| 2076 |
|
| 2077 | static int silvermont_freq_table[] = {
|
| 2078 | 83300, 100000, 133300, 116700, 80000};
|
| 2079 |
|
| 2080 | rdmsrq(MSR_FSB_FREQ, value);
|
| 2081 | i = value & 0x7;
|
| 2082 | WARN_ON(i > 4);
|
| 2083 |
|
| 2084 | return silvermont_freq_table[i];
|
| 2085 | }
|
| 2086 |
|
| 2087 | static int airmont_get_scaling(void)
|
| 2088 | {
|
| 2089 | u64 value;
|
| 2090 | int i;
|
| 2091 |
|
| 2092 | static int airmont_freq_table[] = {
|
| 2093 | 83300, 100000, 133300, 116700, 80000,
|
| 2094 | 93300, 90000, 88900, 87500};
|
| 2095 |
|
| 2096 | rdmsrq(MSR_FSB_FREQ, value);
|
| 2097 | i = value & 0xF;
|
| 2098 | WARN_ON(i > 8);
|
| 2099 |
|
| 2100 | return airmont_freq_table[i];
|
| 2101 | }
|
| 2102 |
|
| 2103 | static void atom_get_vid(struct cpudata *cpudata)
|
| 2104 | {
|
| 2105 | u64 value;
|
| 2106 |
|
| 2107 | rdmsrq(MSR_ATOM_CORE_VIDS, value);
|
| 2108 | cpudata->vid.min = int_tofp((value >> 8) & 0x7f);
|
| 2109 | cpudata->vid.max = int_tofp((value >> 16) & 0x7f);
|
| 2110 | cpudata->vid.ratio = div_fp(
|
| 2111 | cpudata->vid.max - cpudata->vid.min,
|
| 2112 | int_tofp(cpudata->pstate.max_pstate -
|
| 2113 | cpudata->pstate.min_pstate));
|
| 2114 |
|
| 2115 | rdmsrq(MSR_ATOM_CORE_TURBO_VIDS, value);
|
| 2116 | cpudata->vid.turbo = value & 0x7f;
|
| 2117 | }
|
| 2118 |
|
| 2119 | static int core_get_min_pstate(int cpu)
|
| 2120 | {
|
| 2121 | u64 value;
|
| 2122 |
|
| 2123 | rdmsrq_on_cpu(cpu, MSR_PLATFORM_INFO, &value);
|
| 2124 | return (value >> 40) & 0xFF;
|
| 2125 | }
|
| 2126 |
|
| 2127 | static int core_get_max_pstate_physical(int cpu)
|
| 2128 | {
|
| 2129 | u64 value;
|
| 2130 |
|
| 2131 | rdmsrq_on_cpu(cpu, MSR_PLATFORM_INFO, &value);
|
| 2132 | return (value >> 8) & 0xFF;
|
| 2133 | }
|
| 2134 |
|
| 2135 | static int core_get_tdp_ratio(int cpu, u64 plat_info)
|
| 2136 | {
|
| 2137 |
|
| 2138 | if (plat_info & 0x600000000) {
|
| 2139 | u64 tdp_ctrl;
|
| 2140 | u64 tdp_ratio;
|
| 2141 | int tdp_msr;
|
| 2142 | int err;
|
| 2143 |
|
| 2144 |
|
| 2145 | err = rdmsrq_safe_on_cpu(cpu, MSR_CONFIG_TDP_CONTROL, &tdp_ctrl);
|
| 2146 | if (err)
|
| 2147 | return err;
|
| 2148 |
|
| 2149 |
|
| 2150 | tdp_msr = MSR_CONFIG_TDP_NOMINAL + (tdp_ctrl & 0x03);
|
| 2151 | err = rdmsrq_safe_on_cpu(cpu, tdp_msr, &tdp_ratio);
|
| 2152 | if (err)
|
| 2153 | return err;
|
| 2154 |
|
| 2155 |
|
| 2156 | if (tdp_ctrl & 0x03)
|
| 2157 | tdp_ratio >>= 16;
|
| 2158 |
|
| 2159 | tdp_ratio &= 0xff;
|
| 2160 | pr_debug("tdp_ratio %x\n", (int)tdp_ratio);
|
| 2161 |
|
| 2162 | return (int)tdp_ratio;
|
| 2163 | }
|
| 2164 |
|
| 2165 | return -ENXIO;
|
| 2166 | }
|
| 2167 |
|
| 2168 | static int core_get_max_pstate(int cpu)
|
| 2169 | {
|
| 2170 | u64 tar;
|
| 2171 | u64 plat_info;
|
| 2172 | int max_pstate;
|
| 2173 | int tdp_ratio;
|
| 2174 | int err;
|
| 2175 |
|
| 2176 | rdmsrq_on_cpu(cpu, MSR_PLATFORM_INFO, &plat_info);
|
| 2177 | max_pstate = (plat_info >> 8) & 0xFF;
|
| 2178 |
|
| 2179 | tdp_ratio = core_get_tdp_ratio(cpu, plat_info);
|
| 2180 | if (tdp_ratio <= 0)
|
| 2181 | return max_pstate;
|
| 2182 |
|
| 2183 | if (hwp_active) {
|
| 2184 |
|
| 2185 | return tdp_ratio;
|
| 2186 | }
|
| 2187 |
|
| 2188 | err = rdmsrq_safe_on_cpu(cpu, MSR_TURBO_ACTIVATION_RATIO, &tar);
|
| 2189 | if (!err) {
|
| 2190 | int tar_levels;
|
| 2191 |
|
| 2192 |
|
| 2193 | tar_levels = tar & 0xff;
|
| 2194 | if (tdp_ratio - 1 == tar_levels) {
|
| 2195 | max_pstate = tar_levels;
|
| 2196 | pr_debug("max_pstate=TAC %x\n", max_pstate);
|
| 2197 | }
|
| 2198 | }
|
| 2199 |
|
| 2200 | return max_pstate;
|
| 2201 | }
|
| 2202 |
|
| 2203 | static int core_get_turbo_pstate(int cpu)
|
| 2204 | {
|
| 2205 | u64 value;
|
| 2206 | int nont, ret;
|
| 2207 |
|
| 2208 | rdmsrq_on_cpu(cpu, MSR_TURBO_RATIO_LIMIT, &value);
|
| 2209 | nont = core_get_max_pstate(cpu);
|
| 2210 | ret = (value) & 255;
|
| 2211 | if (ret <= nont)
|
| 2212 | ret = nont;
|
| 2213 | return ret;
|
| 2214 | }
|
| 2215 |
|
| 2216 | static u64 core_get_val(struct cpudata *cpudata, int pstate)
|
| 2217 | {
|
| 2218 | return get_perf_ctl_val(pstate);
|
| 2219 | }
|
| 2220 |
|
| 2221 | static int knl_get_aperf_mperf_shift(void)
|
| 2222 | {
|
| 2223 | return 10;
|
| 2224 | }
|
| 2225 |
|
| 2226 | static int knl_get_turbo_pstate(int cpu)
|
| 2227 | {
|
| 2228 | u64 value;
|
| 2229 | int nont, ret;
|
| 2230 |
|
| 2231 | rdmsrq_on_cpu(cpu, MSR_TURBO_RATIO_LIMIT, &value);
|
| 2232 | nont = core_get_max_pstate(cpu);
|
| 2233 | ret = (((value) >> 8) & 0xFF);
|
| 2234 | if (ret <= nont)
|
| 2235 | ret = nont;
|
| 2236 | return ret;
|
| 2237 | }
|
| 2238 |
|
| 2239 | static int hwp_get_cpu_scaling(int cpu)
|
| 2240 | {
|
| 2241 | if (hybrid_scaling_factor) {
|
| 2242 | |
| 2243 | * Return the hybrid scaling factor for P-cores and use the
|
| 2244 | * default core scaling for E-cores.
|
| 2245 |
|
| 2246 | if (hybrid_get_cpu_type(cpu) != INTEL_CPU_TYPE_ATOM)
|
| 2247 | return hybrid_scaling_factor;
|
| 2248 |
|
| 2249 | return core_get_scaling();
|
| 2250 | }
|
| 2251 |
|
| 2252 |
|
| 2253 | if (!cpu_feature_enabled(X86_FEATURE_HYBRID_CPU))
|
| 2254 | return core_get_scaling();
|
| 2255 |
|
| 2256 | |
| 2257 | * The system is hybrid, but the hybrid scaling factor is not known or
|
| 2258 | * the CPU type is not one of the above, so use CPPC to compute the
|
| 2259 | * scaling factor for this CPU.
|
| 2260 |
|
| 2261 | return intel_pstate_cppc_get_scaling(cpu);
|
| 2262 | }
|
| 2263 |
|
| 2264 | static void intel_pstate_get_hwp_pstates(struct cpudata *cpu)
|
| 2265 | {
|
| 2266 | int perf_ctl_max_phys = cpu->pstate.max_pstate_physical;
|
| 2267 | int perf_ctl_scaling = cpu->pstate.perf_ctl_scaling;
|
| 2268 | int perf_ctl_turbo = cpu->pstate.turbo_pstate;
|
| 2269 | int cpuid = cpu->cpu;
|
| 2270 |
|
| 2271 | __intel_pstate_get_hwp_cap(cpu);
|
| 2272 |
|
| 2273 | if (!pstate_funcs.get_cpu_scaling)
|
| 2274 | return;
|
| 2275 |
|
| 2276 | pr_debug("CPU%d: PERF_CTL max_phys = %d\n", cpuid, perf_ctl_max_phys);
|
| 2277 | pr_debug("CPU%d: PERF_CTL turbo = %d\n", cpuid, perf_ctl_turbo);
|
| 2278 | pr_debug("CPU%d: PERF_CTL scaling = %d\n", cpuid, perf_ctl_scaling);
|
| 2279 | pr_debug("CPU%d: PERF_CTL min = %d\n", cpuid, cpu->pstate.min_pstate);
|
| 2280 | pr_debug("CPU%d: HWP_CAP guaranteed = %d\n", cpuid, cpu->pstate.max_pstate);
|
| 2281 | pr_debug("CPU%d: HWP_CAP highest = %d\n", cpuid, cpu->pstate.turbo_pstate);
|
| 2282 |
|
| 2283 | cpu->pstate.scaling = pstate_funcs.get_cpu_scaling(cpuid);
|
| 2284 |
|
| 2285 | pr_debug("CPU%d: HWP-to-frequency scaling = %d\n", cpuid, cpu->pstate.scaling);
|
| 2286 |
|
| 2287 | |
| 2288 | * On hybrid processors, HWP may expose more performance levels than
|
| 2289 | * there are P-states accessible through the PERF_CTL interface. If
|
| 2290 | * that happens, the scaling between HWP performance levels and CPU
|
| 2291 | * frequency will be less than the scaling between P-state values and
|
| 2292 | * CPU frequency. In that case, update the maximum physical non-turbo
|
| 2293 | * performance level accordingly.
|
| 2294 |
|
| 2295 | if (cpu->pstate.scaling != perf_ctl_scaling) {
|
| 2296 | int freq;
|
| 2297 |
|
| 2298 | freq = perf_ctl_max_phys * perf_ctl_scaling;
|
| 2299 | cpu->pstate.max_pstate_physical = intel_pstate_freq_to_hwp(cpu, freq);
|
| 2300 |
|
| 2301 | freq = cpu->pstate.min_freq;
|
| 2302 | cpu->pstate.min_pstate = intel_pstate_freq_to_hwp(cpu, freq);
|
| 2303 |
|
| 2304 | hwp_is_hybrid = true;
|
| 2305 | }
|
| 2306 | |
| 2307 | * If the CPU is going online for the first time and it was offline
|
| 2308 | * initially, asym capacity scaling may need to be updated.
|
| 2309 |
|
| 2310 | hybrid_update_capacity(cpu);
|
| 2311 | }
|
| 2312 |
|
| 2313 | static void intel_pstate_get_cpu_pstates(struct cpudata *cpu)
|
| 2314 | {
|
| 2315 | int perf_ctl_scaling = pstate_funcs.get_scaling();
|
| 2316 |
|
| 2317 | cpu->pstate.max_pstate_physical = pstate_funcs.get_max_physical(cpu->cpu);
|
| 2318 | cpu->pstate.turbo_pstate = pstate_funcs.get_turbo(cpu->cpu);
|
| 2319 | cpu->pstate.min_pstate = pstate_funcs.get_min(cpu->cpu);
|
| 2320 | cpu->pstate.min_freq = cpu->pstate.min_pstate * perf_ctl_scaling;
|
| 2321 | cpu->pstate.perf_ctl_scaling = perf_ctl_scaling;
|
| 2322 | cpu->pstate.scaling = perf_ctl_scaling;
|
| 2323 |
|
| 2324 | if (hwp_active)
|
| 2325 | intel_pstate_get_hwp_pstates(cpu);
|
| 2326 | else
|
| 2327 | cpu->pstate.max_pstate = pstate_funcs.get_max(cpu->cpu);
|
| 2328 |
|
| 2329 | intel_pstate_update_freq_limits(cpu);
|
| 2330 |
|
| 2331 | if (pstate_funcs.get_aperf_mperf_shift)
|
| 2332 | cpu->aperf_mperf_shift = pstate_funcs.get_aperf_mperf_shift();
|
| 2333 |
|
| 2334 | if (pstate_funcs.get_vid)
|
| 2335 | pstate_funcs.get_vid(cpu);
|
| 2336 | }
|
| 2337 |
|
| 2338 | |
| 2339 | * Long hold time will keep high perf limits for long time,
|
| 2340 | * which negatively impacts perf/watt for some workloads,
|
| 2341 | * like specpower. 3ms is based on experiements on some
|
| 2342 | * workoads.
|
| 2343 |
|
| 2344 | static int hwp_boost_hold_time_ns = 3 * NSEC_PER_MSEC;
|
| 2345 |
|
| 2346 | static inline void intel_pstate_hwp_boost_up(struct cpudata *cpu)
|
| 2347 | {
|
| 2348 | u64 hwp_req = READ_ONCE(cpu->hwp_req_cached);
|
| 2349 | u64 hwp_cap = READ_ONCE(cpu->hwp_cap_cached);
|
| 2350 | u32 max_limit = (hwp_req & 0xff00) >> 8;
|
| 2351 | u32 min_limit = (hwp_req & 0xff);
|
| 2352 | u32 boost_level1;
|
| 2353 |
|
| 2354 | |
| 2355 | * Cases to consider (User changes via sysfs or boot time):
|
| 2356 | * If, P0 (Turbo max) = P1 (Guaranteed max) = min:
|
| 2357 | * No boost, return.
|
| 2358 | * If, P0 (Turbo max) > P1 (Guaranteed max) = min:
|
| 2359 | * Should result in one level boost only for P0.
|
| 2360 | * If, P0 (Turbo max) = P1 (Guaranteed max) > min:
|
| 2361 | * Should result in two level boost:
|
| 2362 | * (min + p1)/2 and P1.
|
| 2363 | * If, P0 (Turbo max) > P1 (Guaranteed max) > min:
|
| 2364 | * Should result in three level boost:
|
| 2365 | * (min + p1)/2, P1 and P0.
|
| 2366 |
|
| 2367 |
|
| 2368 |
|
| 2369 | if (max_limit == min_limit || cpu->hwp_boost_min >= max_limit)
|
| 2370 | return;
|
| 2371 |
|
| 2372 | if (!cpu->hwp_boost_min)
|
| 2373 | cpu->hwp_boost_min = min_limit;
|
| 2374 |
|
| 2375 |
|
| 2376 | boost_level1 = (HWP_GUARANTEED_PERF(hwp_cap) + min_limit) >> 1;
|
| 2377 |
|
| 2378 | if (cpu->hwp_boost_min < boost_level1)
|
| 2379 | cpu->hwp_boost_min = boost_level1;
|
| 2380 | else if (cpu->hwp_boost_min < HWP_GUARANTEED_PERF(hwp_cap))
|
| 2381 | cpu->hwp_boost_min = HWP_GUARANTEED_PERF(hwp_cap);
|
| 2382 | else if (cpu->hwp_boost_min == HWP_GUARANTEED_PERF(hwp_cap) &&
|
| 2383 | max_limit != HWP_GUARANTEED_PERF(hwp_cap))
|
| 2384 | cpu->hwp_boost_min = max_limit;
|
| 2385 | else
|
| 2386 | return;
|
| 2387 |
|
| 2388 | hwp_req = (hwp_req & ~GENMASK_ULL(7, 0)) | cpu->hwp_boost_min;
|
| 2389 | wrmsrq(MSR_HWP_REQUEST, hwp_req);
|
| 2390 | cpu->last_update = cpu->sample.time;
|
| 2391 | }
|
| 2392 |
|
| 2393 | static inline void intel_pstate_hwp_boost_down(struct cpudata *cpu)
|
| 2394 | {
|
| 2395 | if (cpu->hwp_boost_min) {
|
| 2396 | bool expired;
|
| 2397 |
|
| 2398 |
|
| 2399 | expired = time_after64(cpu->sample.time, cpu->last_update +
|
| 2400 | hwp_boost_hold_time_ns);
|
| 2401 | if (expired) {
|
| 2402 | wrmsrq(MSR_HWP_REQUEST, cpu->hwp_req_cached);
|
| 2403 | cpu->hwp_boost_min = 0;
|
| 2404 | }
|
| 2405 | }
|
| 2406 | cpu->last_update = cpu->sample.time;
|
| 2407 | }
|
| 2408 |
|
| 2409 | static inline void intel_pstate_update_util_hwp_local(struct cpudata *cpu,
|
| 2410 | u64 time)
|
| 2411 | {
|
| 2412 | cpu->sample.time = time;
|
| 2413 |
|
| 2414 | if (cpu->sched_flags & SCHED_CPUFREQ_IOWAIT) {
|
| 2415 | bool do_io = false;
|
| 2416 |
|
| 2417 | cpu->sched_flags = 0;
|
| 2418 | |
| 2419 | * Set iowait_boost flag and update time. Since IO WAIT flag
|
| 2420 | * is set all the time, we can't just conclude that there is
|
| 2421 | * some IO bound activity is scheduled on this CPU with just
|
| 2422 | * one occurrence. If we receive at least two in two
|
| 2423 | * consecutive ticks, then we treat as boost candidate.
|
| 2424 |
|
| 2425 | if (time_before64(time, cpu->last_io_update + 2 * TICK_NSEC))
|
| 2426 | do_io = true;
|
| 2427 |
|
| 2428 | cpu->last_io_update = time;
|
| 2429 |
|
| 2430 | if (do_io)
|
| 2431 | intel_pstate_hwp_boost_up(cpu);
|
| 2432 |
|
| 2433 | } else {
|
| 2434 | intel_pstate_hwp_boost_down(cpu);
|
| 2435 | }
|
| 2436 | }
|
| 2437 |
|
| 2438 | static inline void intel_pstate_update_util_hwp(struct update_util_data *data,
|
| 2439 | u64 time, unsigned int flags)
|
| 2440 | {
|
| 2441 | struct cpudata *cpu = container_of(data, struct cpudata, update_util);
|
| 2442 |
|
| 2443 | cpu->sched_flags |= flags;
|
| 2444 |
|
| 2445 | if (smp_processor_id() == cpu->cpu)
|
| 2446 | intel_pstate_update_util_hwp_local(cpu, time);
|
| 2447 | }
|
| 2448 |
|
| 2449 | static inline void intel_pstate_calc_avg_perf(struct cpudata *cpu)
|
| 2450 | {
|
| 2451 | struct sample *sample = &cpu->sample;
|
| 2452 |
|
| 2453 | sample->core_avg_perf = div_ext_fp(sample->aperf, sample->mperf);
|
| 2454 | }
|
| 2455 |
|
| 2456 | static inline bool intel_pstate_sample(struct cpudata *cpu, u64 time)
|
| 2457 | {
|
| 2458 | u64 aperf, mperf;
|
| 2459 | unsigned long flags;
|
| 2460 | u64 tsc;
|
| 2461 |
|
| 2462 | local_irq_save(flags);
|
| 2463 | rdmsrq(MSR_IA32_APERF, aperf);
|
| 2464 | rdmsrq(MSR_IA32_MPERF, mperf);
|
| 2465 | tsc = rdtsc();
|
| 2466 | if (cpu->prev_mperf == mperf || cpu->prev_tsc == tsc) {
|
| 2467 | local_irq_restore(flags);
|
| 2468 | return false;
|
| 2469 | }
|
| 2470 | local_irq_restore(flags);
|
| 2471 |
|
| 2472 | cpu->last_sample_time = cpu->sample.time;
|
| 2473 | cpu->sample.time = time;
|
| 2474 | cpu->sample.aperf = aperf;
|
| 2475 | cpu->sample.mperf = mperf;
|
| 2476 | cpu->sample.tsc = tsc;
|
| 2477 | cpu->sample.aperf -= cpu->prev_aperf;
|
| 2478 | cpu->sample.mperf -= cpu->prev_mperf;
|
| 2479 | cpu->sample.tsc -= cpu->prev_tsc;
|
| 2480 |
|
| 2481 | cpu->prev_aperf = aperf;
|
| 2482 | cpu->prev_mperf = mperf;
|
| 2483 | cpu->prev_tsc = tsc;
|
| 2484 | |
| 2485 | * First time this function is invoked in a given cycle, all of the
|
| 2486 | * previous sample data fields are equal to zero or stale and they must
|
| 2487 | * be populated with meaningful numbers for things to work, so assume
|
| 2488 | * that sample.time will always be reset before setting the utilization
|
| 2489 | * update hook and make the caller skip the sample then.
|
| 2490 |
|
| 2491 | if (likely(cpu->last_sample_time)) {
|
| 2492 | intel_pstate_calc_avg_perf(cpu);
|
| 2493 | return true;
|
| 2494 | }
|
| 2495 | return false;
|
| 2496 | }
|
| 2497 |
|
| 2498 | static inline int32_t get_avg_frequency(struct cpudata *cpu)
|
| 2499 | {
|
| 2500 | return mul_ext_fp(cpu->sample.core_avg_perf, cpu_khz);
|
| 2501 | }
|
| 2502 |
|
| 2503 | static inline int32_t get_avg_pstate(struct cpudata *cpu)
|
| 2504 | {
|
| 2505 | return mul_ext_fp(cpu->pstate.max_pstate_physical,
|
| 2506 | cpu->sample.core_avg_perf);
|
| 2507 | }
|
| 2508 |
|
| 2509 | static inline int32_t get_target_pstate(struct cpudata *cpu)
|
| 2510 | {
|
| 2511 | struct sample *sample = &cpu->sample;
|
| 2512 | int32_t busy_frac;
|
| 2513 | int target, avg_pstate;
|
| 2514 |
|
| 2515 | busy_frac = div_fp(sample->mperf << cpu->aperf_mperf_shift,
|
| 2516 | sample->tsc);
|
| 2517 |
|
| 2518 | if (busy_frac < cpu->iowait_boost)
|
| 2519 | busy_frac = cpu->iowait_boost;
|
| 2520 |
|
| 2521 | sample->busy_scaled = busy_frac * 100;
|
| 2522 |
|
| 2523 | target = READ_ONCE(global.no_turbo) ?
|
| 2524 | cpu->pstate.max_pstate : cpu->pstate.turbo_pstate;
|
| 2525 | target += target >> 2;
|
| 2526 | target = mul_fp(target, busy_frac);
|
| 2527 | if (target < cpu->pstate.min_pstate)
|
| 2528 | target = cpu->pstate.min_pstate;
|
| 2529 |
|
| 2530 | |
| 2531 | * If the average P-state during the previous cycle was higher than the
|
| 2532 | * current target, add 50% of the difference to the target to reduce
|
| 2533 | * possible performance oscillations and offset possible performance
|
| 2534 | * loss related to moving the workload from one CPU to another within
|
| 2535 | * a package/module.
|
| 2536 |
|
| 2537 | avg_pstate = get_avg_pstate(cpu);
|
| 2538 | if (avg_pstate > target)
|
| 2539 | target += (avg_pstate - target) >> 1;
|
| 2540 |
|
| 2541 | return target;
|
| 2542 | }
|
| 2543 |
|
| 2544 | static int intel_pstate_prepare_request(struct cpudata *cpu, int pstate)
|
| 2545 | {
|
| 2546 | int min_pstate = max(cpu->pstate.min_pstate, cpu->min_perf_ratio);
|
| 2547 | int max_pstate = max(min_pstate, cpu->max_perf_ratio);
|
| 2548 |
|
| 2549 | return clamp_t(int, pstate, min_pstate, max_pstate);
|
| 2550 | }
|
| 2551 |
|
| 2552 | static void intel_pstate_update_pstate(struct cpudata *cpu, int pstate)
|
| 2553 | {
|
| 2554 | if (pstate == cpu->pstate.current_pstate)
|
| 2555 | return;
|
| 2556 |
|
| 2557 | cpu->pstate.current_pstate = pstate;
|
| 2558 | wrmsrq(MSR_IA32_PERF_CTL, pstate_funcs.get_val(cpu, pstate));
|
| 2559 | }
|
| 2560 |
|
| 2561 | static void intel_pstate_adjust_pstate(struct cpudata *cpu)
|
| 2562 | {
|
| 2563 | int from = cpu->pstate.current_pstate;
|
| 2564 | struct sample *sample;
|
| 2565 | int target_pstate;
|
| 2566 |
|
| 2567 | target_pstate = get_target_pstate(cpu);
|
| 2568 | target_pstate = intel_pstate_prepare_request(cpu, target_pstate);
|
| 2569 | trace_cpu_frequency(target_pstate * cpu->pstate.scaling, cpu->cpu);
|
| 2570 | intel_pstate_update_pstate(cpu, target_pstate);
|
| 2571 |
|
| 2572 | sample = &cpu->sample;
|
| 2573 | trace_pstate_sample(mul_ext_fp(100, sample->core_avg_perf),
|
| 2574 | fp_toint(sample->busy_scaled),
|
| 2575 | from,
|
| 2576 | cpu->pstate.current_pstate,
|
| 2577 | sample->mperf,
|
| 2578 | sample->aperf,
|
| 2579 | sample->tsc,
|
| 2580 | get_avg_frequency(cpu),
|
| 2581 | fp_toint(cpu->iowait_boost * 100));
|
| 2582 | }
|
| 2583 |
|
| 2584 | static void intel_pstate_update_util(struct update_util_data *data, u64 time,
|
| 2585 | unsigned int flags)
|
| 2586 | {
|
| 2587 | struct cpudata *cpu = container_of(data, struct cpudata, update_util);
|
| 2588 | u64 delta_ns;
|
| 2589 |
|
| 2590 |
|
| 2591 | if (smp_processor_id() != cpu->cpu)
|
| 2592 | return;
|
| 2593 |
|
| 2594 | delta_ns = time - cpu->last_update;
|
| 2595 | if (flags & SCHED_CPUFREQ_IOWAIT) {
|
| 2596 |
|
| 2597 | if (delta_ns > TICK_NSEC) {
|
| 2598 | cpu->iowait_boost = ONE_EIGHTH_FP;
|
| 2599 | } else if (cpu->iowait_boost >= ONE_EIGHTH_FP) {
|
| 2600 | cpu->iowait_boost <<= 1;
|
| 2601 | if (cpu->iowait_boost > int_tofp(1))
|
| 2602 | cpu->iowait_boost = int_tofp(1);
|
| 2603 | } else {
|
| 2604 | cpu->iowait_boost = ONE_EIGHTH_FP;
|
| 2605 | }
|
| 2606 | } else if (cpu->iowait_boost) {
|
| 2607 |
|
| 2608 | if (delta_ns > TICK_NSEC)
|
| 2609 | cpu->iowait_boost = 0;
|
| 2610 | else
|
| 2611 | cpu->iowait_boost >>= 1;
|
| 2612 | }
|
| 2613 | cpu->last_update = time;
|
| 2614 | delta_ns = time - cpu->sample.time;
|
| 2615 | if ((s64)delta_ns < INTEL_PSTATE_SAMPLING_INTERVAL)
|
| 2616 | return;
|
| 2617 |
|
| 2618 | if (intel_pstate_sample(cpu, time))
|
| 2619 | intel_pstate_adjust_pstate(cpu);
|
| 2620 | }
|
| 2621 |
|
| 2622 | static struct pstate_funcs core_funcs = {
|
| 2623 | .get_max = core_get_max_pstate,
|
| 2624 | .get_max_physical = core_get_max_pstate_physical,
|
| 2625 | .get_min = core_get_min_pstate,
|
| 2626 | .get_turbo = core_get_turbo_pstate,
|
| 2627 | .get_scaling = core_get_scaling,
|
| 2628 | .get_val = core_get_val,
|
| 2629 | };
|
| 2630 |
|
| 2631 | static const struct pstate_funcs silvermont_funcs = {
|
| 2632 | .get_max = atom_get_max_pstate,
|
| 2633 | .get_max_physical = atom_get_max_pstate,
|
| 2634 | .get_min = atom_get_min_pstate,
|
| 2635 | .get_turbo = atom_get_turbo_pstate,
|
| 2636 | .get_val = atom_get_val,
|
| 2637 | .get_scaling = silvermont_get_scaling,
|
| 2638 | .get_vid = atom_get_vid,
|
| 2639 | };
|
| 2640 |
|
| 2641 | static const struct pstate_funcs airmont_funcs = {
|
| 2642 | .get_max = atom_get_max_pstate,
|
| 2643 | .get_max_physical = atom_get_max_pstate,
|
| 2644 | .get_min = atom_get_min_pstate,
|
| 2645 | .get_turbo = atom_get_turbo_pstate,
|
| 2646 | .get_val = atom_get_val,
|
| 2647 | .get_scaling = airmont_get_scaling,
|
| 2648 | .get_vid = atom_get_vid,
|
| 2649 | };
|
| 2650 |
|
| 2651 | static const struct pstate_funcs knl_funcs = {
|
| 2652 | .get_max = core_get_max_pstate,
|
| 2653 | .get_max_physical = core_get_max_pstate_physical,
|
| 2654 | .get_min = core_get_min_pstate,
|
| 2655 | .get_turbo = knl_get_turbo_pstate,
|
| 2656 | .get_aperf_mperf_shift = knl_get_aperf_mperf_shift,
|
| 2657 | .get_scaling = core_get_scaling,
|
| 2658 | .get_val = core_get_val,
|
| 2659 | };
|
| 2660 |
|
| 2661 | #define X86_MATCH(vfm, policy) \
|
| 2662 | X86_MATCH_VFM_FEATURE(vfm, X86_FEATURE_APERFMPERF, &policy)
|
| 2663 |
|
| 2664 | static const struct x86_cpu_id intel_pstate_cpu_ids[] = {
|
| 2665 | X86_MATCH(INTEL_SANDYBRIDGE, core_funcs),
|
| 2666 | X86_MATCH(INTEL_SANDYBRIDGE_X, core_funcs),
|
| 2667 | X86_MATCH(INTEL_ATOM_SILVERMONT, silvermont_funcs),
|
| 2668 | X86_MATCH(INTEL_IVYBRIDGE, core_funcs),
|
| 2669 | X86_MATCH(INTEL_HASWELL, core_funcs),
|
| 2670 | X86_MATCH(INTEL_BROADWELL, core_funcs),
|
| 2671 | X86_MATCH(INTEL_IVYBRIDGE_X, core_funcs),
|
| 2672 | X86_MATCH(INTEL_HASWELL_X, core_funcs),
|
| 2673 | X86_MATCH(INTEL_HASWELL_L, core_funcs),
|
| 2674 | X86_MATCH(INTEL_HASWELL_G, core_funcs),
|
| 2675 | X86_MATCH(INTEL_BROADWELL_G, core_funcs),
|
| 2676 | X86_MATCH(INTEL_ATOM_AIRMONT, airmont_funcs),
|
| 2677 | X86_MATCH(INTEL_SKYLAKE_L, core_funcs),
|
| 2678 | X86_MATCH(INTEL_BROADWELL_X, core_funcs),
|
| 2679 | X86_MATCH(INTEL_SKYLAKE, core_funcs),
|
| 2680 | X86_MATCH(INTEL_BROADWELL_D, core_funcs),
|
| 2681 | X86_MATCH(INTEL_XEON_PHI_KNL, knl_funcs),
|
| 2682 | X86_MATCH(INTEL_XEON_PHI_KNM, knl_funcs),
|
| 2683 | X86_MATCH(INTEL_ATOM_GOLDMONT, core_funcs),
|
| 2684 | X86_MATCH(INTEL_ATOM_GOLDMONT_PLUS, core_funcs),
|
| 2685 | X86_MATCH(INTEL_SKYLAKE_X, core_funcs),
|
| 2686 | X86_MATCH(INTEL_COMETLAKE, core_funcs),
|
| 2687 | X86_MATCH(INTEL_ICELAKE_X, core_funcs),
|
| 2688 | X86_MATCH(INTEL_TIGERLAKE, core_funcs),
|
| 2689 | X86_MATCH(INTEL_SAPPHIRERAPIDS_X, core_funcs),
|
| 2690 | X86_MATCH(INTEL_EMERALDRAPIDS_X, core_funcs),
|
| 2691 | X86_MATCH(INTEL_GRANITERAPIDS_D, core_funcs),
|
| 2692 | X86_MATCH(INTEL_GRANITERAPIDS_X, core_funcs),
|
| 2693 | {}
|
| 2694 | };
|
| 2695 | MODULE_DEVICE_TABLE(x86cpu, intel_pstate_cpu_ids);
|
| 2696 |
|
| 2697 | #ifdef CONFIG_ACPI
|
| 2698 | static const struct x86_cpu_id intel_pstate_cpu_oob_ids[] __initconst = {
|
| 2699 | X86_MATCH(INTEL_BROADWELL_D, core_funcs),
|
| 2700 | X86_MATCH(INTEL_BROADWELL_X, core_funcs),
|
| 2701 | X86_MATCH(INTEL_SKYLAKE_X, core_funcs),
|
| 2702 | X86_MATCH(INTEL_ICELAKE_X, core_funcs),
|
| 2703 | X86_MATCH(INTEL_SAPPHIRERAPIDS_X, core_funcs),
|
| 2704 | X86_MATCH(INTEL_EMERALDRAPIDS_X, core_funcs),
|
| 2705 | X86_MATCH(INTEL_GRANITERAPIDS_D, core_funcs),
|
| 2706 | X86_MATCH(INTEL_GRANITERAPIDS_X, core_funcs),
|
| 2707 | X86_MATCH(INTEL_ATOM_CRESTMONT, core_funcs),
|
| 2708 | X86_MATCH(INTEL_ATOM_CRESTMONT_X, core_funcs),
|
| 2709 | X86_MATCH(INTEL_ATOM_DARKMONT_X, core_funcs),
|
| 2710 | X86_MATCH(INTEL_DIAMONDRAPIDS_X, core_funcs),
|
| 2711 | {}
|
| 2712 | };
|
| 2713 | #endif
|
| 2714 |
|
| 2715 | static const struct x86_cpu_id intel_pstate_cpu_ee_disable_ids[] = {
|
| 2716 | X86_MATCH(INTEL_KABYLAKE, core_funcs),
|
| 2717 | {}
|
| 2718 | };
|
| 2719 |
|
| 2720 | static int intel_pstate_init_cpu(unsigned int cpunum)
|
| 2721 | {
|
| 2722 | struct cpudata *cpu;
|
| 2723 |
|
| 2724 | cpu = all_cpu_data[cpunum];
|
| 2725 |
|
| 2726 | if (!cpu) {
|
| 2727 | cpu = kzalloc_obj(*cpu);
|
| 2728 | if (!cpu)
|
| 2729 | return -ENOMEM;
|
| 2730 |
|
| 2731 | WRITE_ONCE(all_cpu_data[cpunum], cpu);
|
| 2732 |
|
| 2733 | cpu->cpu = cpunum;
|
| 2734 |
|
| 2735 | cpu->epp_default = -EINVAL;
|
| 2736 |
|
| 2737 | if (hwp_active) {
|
| 2738 | intel_pstate_hwp_enable(cpu);
|
| 2739 |
|
| 2740 | if (intel_pstate_acpi_pm_profile_server())
|
| 2741 | hwp_boost = true;
|
| 2742 | }
|
| 2743 | } else if (hwp_active) {
|
| 2744 | |
| 2745 | * Re-enable HWP in case this happens after a resume from ACPI
|
| 2746 | * S3 if the CPU was offline during the whole system/resume
|
| 2747 | * cycle.
|
| 2748 |
|
| 2749 | intel_pstate_hwp_reenable(cpu);
|
| 2750 | }
|
| 2751 |
|
| 2752 | cpu->epp_powersave = -EINVAL;
|
| 2753 | cpu->epp_policy = CPUFREQ_POLICY_UNKNOWN;
|
| 2754 |
|
| 2755 | intel_pstate_get_cpu_pstates(cpu);
|
| 2756 |
|
| 2757 | pr_debug("controlling: cpu %d\n", cpunum);
|
| 2758 |
|
| 2759 | return 0;
|
| 2760 | }
|
| 2761 |
|
| 2762 | static void intel_pstate_set_update_util_hook(unsigned int cpu_num)
|
| 2763 | {
|
| 2764 | struct cpudata *cpu = all_cpu_data[cpu_num];
|
| 2765 |
|
| 2766 | if (hwp_active && !hwp_boost)
|
| 2767 | return;
|
| 2768 |
|
| 2769 | if (cpu->update_util_set)
|
| 2770 | return;
|
| 2771 |
|
| 2772 |
|
| 2773 | cpu->sample.time = 0;
|
| 2774 | cpufreq_add_update_util_hook(cpu_num, &cpu->update_util,
|
| 2775 | (hwp_active ?
|
| 2776 | intel_pstate_update_util_hwp :
|
| 2777 | intel_pstate_update_util));
|
| 2778 | cpu->update_util_set = true;
|
| 2779 | }
|
| 2780 |
|
| 2781 | static void intel_pstate_clear_update_util_hook(unsigned int cpu)
|
| 2782 | {
|
| 2783 | struct cpudata *cpu_data = all_cpu_data[cpu];
|
| 2784 |
|
| 2785 | if (!cpu_data->update_util_set)
|
| 2786 | return;
|
| 2787 |
|
| 2788 | cpufreq_remove_update_util_hook(cpu);
|
| 2789 | cpu_data->update_util_set = false;
|
| 2790 | synchronize_rcu();
|
| 2791 | }
|
| 2792 |
|
| 2793 | static int intel_pstate_get_max_freq(struct cpudata *cpu)
|
| 2794 | {
|
| 2795 | return READ_ONCE(global.no_turbo) ?
|
| 2796 | cpu->pstate.max_freq : cpu->pstate.turbo_freq;
|
| 2797 | }
|
| 2798 |
|
| 2799 | static void intel_pstate_update_perf_limits(struct cpudata *cpu,
|
| 2800 | unsigned int policy_min,
|
| 2801 | unsigned int policy_max)
|
| 2802 | {
|
| 2803 | int perf_ctl_scaling = cpu->pstate.perf_ctl_scaling;
|
| 2804 | int32_t max_policy_perf, min_policy_perf;
|
| 2805 |
|
| 2806 | max_policy_perf = policy_max / perf_ctl_scaling;
|
| 2807 | if (policy_max == policy_min) {
|
| 2808 | min_policy_perf = max_policy_perf;
|
| 2809 | } else {
|
| 2810 | min_policy_perf = policy_min / perf_ctl_scaling;
|
| 2811 | min_policy_perf = clamp_t(int32_t, min_policy_perf,
|
| 2812 | 0, max_policy_perf);
|
| 2813 | }
|
| 2814 |
|
| 2815 | |
| 2816 | * HWP needs some special consideration, because HWP_REQUEST uses
|
| 2817 | * abstract values to represent performance rather than pure ratios.
|
| 2818 |
|
| 2819 | if (hwp_active && cpu->pstate.scaling != perf_ctl_scaling) {
|
| 2820 | int freq;
|
| 2821 |
|
| 2822 | freq = max_policy_perf * perf_ctl_scaling;
|
| 2823 | max_policy_perf = intel_pstate_freq_to_hwp(cpu, freq);
|
| 2824 | freq = min_policy_perf * perf_ctl_scaling;
|
| 2825 | min_policy_perf = intel_pstate_freq_to_hwp(cpu, freq);
|
| 2826 | }
|
| 2827 |
|
| 2828 | pr_debug("cpu:%d min_policy_perf:%d max_policy_perf:%d\n",
|
| 2829 | cpu->cpu, min_policy_perf, max_policy_perf);
|
| 2830 |
|
| 2831 |
|
| 2832 | if (per_cpu_limits) {
|
| 2833 | cpu->min_perf_ratio = min_policy_perf;
|
| 2834 | cpu->max_perf_ratio = max_policy_perf;
|
| 2835 | } else {
|
| 2836 | int turbo_max = cpu->pstate.turbo_pstate;
|
| 2837 | int32_t global_min, global_max;
|
| 2838 |
|
| 2839 |
|
| 2840 | global_max = DIV_ROUND_UP(turbo_max * global.max_perf_pct, 100);
|
| 2841 | global_min = DIV_ROUND_UP(turbo_max * global.min_perf_pct, 100);
|
| 2842 | global_min = clamp_t(int32_t, global_min, 0, global_max);
|
| 2843 |
|
| 2844 | pr_debug("cpu:%d global_min:%d global_max:%d\n", cpu->cpu,
|
| 2845 | global_min, global_max);
|
| 2846 |
|
| 2847 | cpu->min_perf_ratio = max(min_policy_perf, global_min);
|
| 2848 | cpu->min_perf_ratio = min(cpu->min_perf_ratio, max_policy_perf);
|
| 2849 | cpu->max_perf_ratio = min(max_policy_perf, global_max);
|
| 2850 | cpu->max_perf_ratio = max(min_policy_perf, cpu->max_perf_ratio);
|
| 2851 |
|
| 2852 |
|
| 2853 | cpu->min_perf_ratio = min(cpu->min_perf_ratio,
|
| 2854 | cpu->max_perf_ratio);
|
| 2855 |
|
| 2856 | }
|
| 2857 | pr_debug("cpu:%d max_perf_ratio:%d min_perf_ratio:%d\n", cpu->cpu,
|
| 2858 | cpu->max_perf_ratio,
|
| 2859 | cpu->min_perf_ratio);
|
| 2860 | }
|
| 2861 |
|
| 2862 | static void intel_pstate_set_pstate(struct cpudata *cpu, int pstate)
|
| 2863 | {
|
| 2864 | trace_cpu_frequency(pstate * cpu->pstate.scaling, cpu->cpu);
|
| 2865 | cpu->pstate.current_pstate = pstate;
|
| 2866 | |
| 2867 | * Generally, there is no guarantee that this code will always run on
|
| 2868 | * the CPU being updated, so force the register update to run on the
|
| 2869 | * right CPU.
|
| 2870 |
|
| 2871 | wrmsrq_on_cpu(cpu->cpu, MSR_IA32_PERF_CTL,
|
| 2872 | pstate_funcs.get_val(cpu, pstate));
|
| 2873 | }
|
| 2874 |
|
| 2875 | static int intel_pstate_set_policy(struct cpufreq_policy *policy)
|
| 2876 | {
|
| 2877 | unsigned int freq = policy->min;
|
| 2878 | struct cpudata *cpu;
|
| 2879 |
|
| 2880 | if (!policy->cpuinfo.max_freq)
|
| 2881 | return -ENODEV;
|
| 2882 |
|
| 2883 | pr_debug("set_policy cpuinfo.max %u policy->max %u\n",
|
| 2884 | policy->cpuinfo.max_freq, policy->max);
|
| 2885 |
|
| 2886 | cpu = all_cpu_data[policy->cpu];
|
| 2887 | cpu->policy = policy->policy;
|
| 2888 |
|
| 2889 | mutex_lock(&intel_pstate_limits_lock);
|
| 2890 |
|
| 2891 | intel_pstate_update_perf_limits(cpu, policy->min, policy->max);
|
| 2892 |
|
| 2893 | if (hwp_active) {
|
| 2894 | |
| 2895 | * The active mode only requires an update util hook if HWP
|
| 2896 | * boost is used and the policy is not "performance".
|
| 2897 |
|
| 2898 | if (hwp_boost && cpu->policy != CPUFREQ_POLICY_PERFORMANCE) {
|
| 2899 | intel_pstate_set_update_util_hook(policy->cpu);
|
| 2900 | } else {
|
| 2901 | intel_pstate_clear_update_util_hook(policy->cpu);
|
| 2902 | if (cpu->policy == CPUFREQ_POLICY_PERFORMANCE) {
|
| 2903 | freq = cpu->max_perf_ratio * cpu->pstate.scaling;
|
| 2904 | if (cpu->pstate.scaling != cpu->pstate.perf_ctl_scaling)
|
| 2905 | freq = rounddown(freq, cpu->pstate.perf_ctl_scaling);
|
| 2906 | }
|
| 2907 | }
|
| 2908 | intel_pstate_hwp_set(policy->cpu);
|
| 2909 | } else if (cpu->policy == CPUFREQ_POLICY_PERFORMANCE) {
|
| 2910 | int pstate = max(cpu->pstate.min_pstate, cpu->max_perf_ratio);
|
| 2911 |
|
| 2912 | |
| 2913 | * NOHZ_FULL CPUs need this as the governor callback may not
|
| 2914 | * be invoked on them.
|
| 2915 |
|
| 2916 | intel_pstate_clear_update_util_hook(policy->cpu);
|
| 2917 | intel_pstate_set_pstate(cpu, pstate);
|
| 2918 | freq = pstate * cpu->pstate.scaling;
|
| 2919 | } else {
|
| 2920 | intel_pstate_set_update_util_hook(policy->cpu);
|
| 2921 | }
|
| 2922 | |
| 2923 | * policy->cur is never updated in the intel_pstate driver, but it is
|
| 2924 | * used as a stale frequency value, so set it to reflect the actual
|
| 2925 | * requested P-state in the "performance" policy case and to the min
|
| 2926 | * otherwise.
|
| 2927 |
|
| 2928 | policy->cur = freq;
|
| 2929 |
|
| 2930 | mutex_unlock(&intel_pstate_limits_lock);
|
| 2931 |
|
| 2932 | return 0;
|
| 2933 | }
|
| 2934 |
|
| 2935 | static void intel_pstate_adjust_policy_max(struct cpudata *cpu,
|
| 2936 | struct cpufreq_policy_data *policy)
|
| 2937 | {
|
| 2938 | if (!hwp_active &&
|
| 2939 | cpu->pstate.max_pstate_physical > cpu->pstate.max_pstate &&
|
| 2940 | policy->max < policy->cpuinfo.max_freq &&
|
| 2941 | policy->max > cpu->pstate.max_freq) {
|
| 2942 | pr_debug("policy->max > max non turbo frequency\n");
|
| 2943 | policy->max = policy->cpuinfo.max_freq;
|
| 2944 | }
|
| 2945 | }
|
| 2946 |
|
| 2947 | static void intel_pstate_verify_cpu_policy(struct cpudata *cpu,
|
| 2948 | struct cpufreq_policy_data *policy)
|
| 2949 | {
|
| 2950 | int max_freq;
|
| 2951 |
|
| 2952 | if (hwp_active) {
|
| 2953 | intel_pstate_get_hwp_cap(cpu);
|
| 2954 | max_freq = READ_ONCE(global.no_turbo) ?
|
| 2955 | cpu->pstate.max_freq : cpu->pstate.turbo_freq;
|
| 2956 | } else {
|
| 2957 | max_freq = intel_pstate_get_max_freq(cpu);
|
| 2958 | }
|
| 2959 | cpufreq_verify_within_limits(policy, policy->cpuinfo.min_freq, max_freq);
|
| 2960 |
|
| 2961 | intel_pstate_adjust_policy_max(cpu, policy);
|
| 2962 | }
|
| 2963 |
|
| 2964 | static int intel_pstate_verify_policy(struct cpufreq_policy_data *policy)
|
| 2965 | {
|
| 2966 | intel_pstate_verify_cpu_policy(all_cpu_data[policy->cpu], policy);
|
| 2967 |
|
| 2968 | return 0;
|
| 2969 | }
|
| 2970 |
|
| 2971 | static void intel_pstate_set_min_pstate(struct cpudata *cpu)
|
| 2972 | {
|
| 2973 | intel_pstate_set_pstate(cpu, cpu->pstate.min_pstate);
|
| 2974 | }
|
| 2975 |
|
| 2976 | static int intel_cpufreq_cpu_offline(struct cpufreq_policy *policy)
|
| 2977 | {
|
| 2978 | struct cpudata *cpu = all_cpu_data[policy->cpu];
|
| 2979 |
|
| 2980 | pr_debug("CPU %d going offline\n", cpu->cpu);
|
| 2981 |
|
| 2982 | if (cpu->suspended)
|
| 2983 | return 0;
|
| 2984 |
|
| 2985 | |
| 2986 | * If the CPU is an SMT thread and it goes offline with the performance
|
| 2987 | * settings different from the minimum, it will prevent its sibling
|
| 2988 | * from getting to lower performance levels, so force the minimum
|
| 2989 | * performance on CPU offline to prevent that from happening.
|
| 2990 |
|
| 2991 | if (hwp_active) {
|
| 2992 | intel_pstate_hwp_offline(cpu);
|
| 2993 | } else {
|
| 2994 | intel_pstate_set_min_pstate(cpu);
|
| 2995 | policy->cur = cpu->pstate.min_freq;
|
| 2996 | }
|
| 2997 |
|
| 2998 | intel_pstate_exit_perf_limits(policy);
|
| 2999 |
|
| 3000 | return 0;
|
| 3001 | }
|
| 3002 |
|
| 3003 | static int intel_pstate_cpu_online(struct cpufreq_policy *policy)
|
| 3004 | {
|
| 3005 | struct cpudata *cpu = all_cpu_data[policy->cpu];
|
| 3006 |
|
| 3007 | pr_debug("CPU %d going online\n", cpu->cpu);
|
| 3008 |
|
| 3009 | intel_pstate_init_acpi_perf_limits(policy);
|
| 3010 |
|
| 3011 | if (hwp_active) {
|
| 3012 | |
| 3013 | * Re-enable HWP and clear the "suspended" flag to let "resume"
|
| 3014 | * know that it need not do that.
|
| 3015 |
|
| 3016 | intel_pstate_hwp_reenable(cpu);
|
| 3017 | cpu->suspended = false;
|
| 3018 |
|
| 3019 | hybrid_update_capacity(cpu);
|
| 3020 | }
|
| 3021 |
|
| 3022 | return 0;
|
| 3023 | }
|
| 3024 |
|
| 3025 | static int intel_pstate_cpu_offline(struct cpufreq_policy *policy)
|
| 3026 | {
|
| 3027 | intel_pstate_clear_update_util_hook(policy->cpu);
|
| 3028 |
|
| 3029 | return intel_cpufreq_cpu_offline(policy);
|
| 3030 | }
|
| 3031 |
|
| 3032 | static void intel_pstate_cpu_exit(struct cpufreq_policy *policy)
|
| 3033 | {
|
| 3034 | pr_debug("CPU %d exiting\n", policy->cpu);
|
| 3035 |
|
| 3036 | policy->fast_switch_possible = false;
|
| 3037 | }
|
| 3038 |
|
| 3039 | static int __intel_pstate_cpu_init(struct cpufreq_policy *policy)
|
| 3040 | {
|
| 3041 | struct cpudata *cpu;
|
| 3042 | int rc;
|
| 3043 |
|
| 3044 | rc = intel_pstate_init_cpu(policy->cpu);
|
| 3045 | if (rc)
|
| 3046 | return rc;
|
| 3047 |
|
| 3048 | cpu = all_cpu_data[policy->cpu];
|
| 3049 |
|
| 3050 | cpu->max_perf_ratio = 0xFF;
|
| 3051 | cpu->min_perf_ratio = 0;
|
| 3052 |
|
| 3053 |
|
| 3054 | policy->cpuinfo.min_freq = cpu->pstate.min_freq;
|
| 3055 | policy->cpuinfo.max_freq = READ_ONCE(global.no_turbo) ?
|
| 3056 | cpu->pstate.max_freq : cpu->pstate.turbo_freq;
|
| 3057 |
|
| 3058 | intel_pstate_init_acpi_perf_limits(policy);
|
| 3059 |
|
| 3060 | policy->fast_switch_possible = true;
|
| 3061 |
|
| 3062 | return 0;
|
| 3063 | }
|
| 3064 |
|
| 3065 | static int intel_pstate_cpu_init(struct cpufreq_policy *policy)
|
| 3066 | {
|
| 3067 | int ret = __intel_pstate_cpu_init(policy);
|
| 3068 | struct cpudata *cpu;
|
| 3069 |
|
| 3070 | if (ret)
|
| 3071 | return ret;
|
| 3072 |
|
| 3073 | |
| 3074 | * Set the policy to powersave to provide a valid fallback value in case
|
| 3075 | * the default cpufreq governor is neither powersave nor performance.
|
| 3076 |
|
| 3077 | policy->policy = CPUFREQ_POLICY_POWERSAVE;
|
| 3078 |
|
| 3079 | cpu = all_cpu_data[policy->cpu];
|
| 3080 | if (hwp_active)
|
| 3081 | cpu->epp_cached = intel_pstate_get_epp(cpu, 0);
|
| 3082 | else
|
| 3083 | intel_pstate_set_min_pstate(cpu);
|
| 3084 |
|
| 3085 | return 0;
|
| 3086 | }
|
| 3087 |
|
| 3088 | static struct cpufreq_driver intel_pstate = {
|
| 3089 | .flags = CPUFREQ_CONST_LOOPS,
|
| 3090 | .verify = intel_pstate_verify_policy,
|
| 3091 | .setpolicy = intel_pstate_set_policy,
|
| 3092 | .suspend = intel_pstate_suspend,
|
| 3093 | .resume = intel_pstate_resume,
|
| 3094 | .init = intel_pstate_cpu_init,
|
| 3095 | .exit = intel_pstate_cpu_exit,
|
| 3096 | .offline = intel_pstate_cpu_offline,
|
| 3097 | .online = intel_pstate_cpu_online,
|
| 3098 | .update_limits = intel_pstate_update_limits,
|
| 3099 | .scale_freq_ref = intel_pstate_scale_freq_ref,
|
| 3100 | .name = "intel_pstate",
|
| 3101 | };
|
| 3102 |
|
| 3103 | static int intel_cpufreq_verify_policy(struct cpufreq_policy_data *policy)
|
| 3104 | {
|
| 3105 | struct cpudata *cpu = all_cpu_data[policy->cpu];
|
| 3106 |
|
| 3107 | intel_pstate_verify_cpu_policy(cpu, policy);
|
| 3108 | intel_pstate_update_perf_limits(cpu, policy->min, policy->max);
|
| 3109 |
|
| 3110 | return 0;
|
| 3111 | }
|
| 3112 |
|
| 3113 | |
| 3114 | *
|
| 3115 | * In passive mode the trace core_busy field (also known as the
|
| 3116 | * performance field, and lablelled as such on the graphs; also known as
|
| 3117 | * core_avg_perf) is not needed and so is re-assigned to indicate if the
|
| 3118 | * driver call was via the normal or fast switch path. Various graphs
|
| 3119 | * output from the intel_pstate_tracer.py utility that include core_busy
|
| 3120 | * (or performance or core_avg_perf) have a fixed y-axis from 0 to 100%,
|
| 3121 | * so we use 10 to indicate the normal path through the driver, and
|
| 3122 | * 90 to indicate the fast switch path through the driver.
|
| 3123 | * The scaled_busy field is not used, and is set to 0.
|
| 3124 |
|
| 3125 |
|
| 3126 | #define INTEL_PSTATE_TRACE_TARGET 10
|
| 3127 | #define INTEL_PSTATE_TRACE_FAST_SWITCH 90
|
| 3128 |
|
| 3129 | static void intel_cpufreq_trace(struct cpudata *cpu, unsigned int trace_type, int old_pstate)
|
| 3130 | {
|
| 3131 | struct sample *sample;
|
| 3132 |
|
| 3133 | if (!trace_pstate_sample_enabled())
|
| 3134 | return;
|
| 3135 |
|
| 3136 | if (!intel_pstate_sample(cpu, ktime_get()))
|
| 3137 | return;
|
| 3138 |
|
| 3139 | sample = &cpu->sample;
|
| 3140 | trace_call__pstate_sample(trace_type,
|
| 3141 | 0,
|
| 3142 | old_pstate,
|
| 3143 | cpu->pstate.current_pstate,
|
| 3144 | sample->mperf,
|
| 3145 | sample->aperf,
|
| 3146 | sample->tsc,
|
| 3147 | get_avg_frequency(cpu),
|
| 3148 | fp_toint(cpu->iowait_boost * 100));
|
| 3149 | }
|
| 3150 |
|
| 3151 | static void intel_cpufreq_hwp_update(struct cpudata *cpu, u32 min, u32 max,
|
| 3152 | u32 desired, bool fast_switch)
|
| 3153 | {
|
| 3154 | u64 prev = READ_ONCE(cpu->hwp_req_cached), value = prev;
|
| 3155 |
|
| 3156 | value &= ~HWP_MIN_PERF(~0L);
|
| 3157 | value |= HWP_MIN_PERF(min);
|
| 3158 |
|
| 3159 | value &= ~HWP_MAX_PERF(~0L);
|
| 3160 | value |= HWP_MAX_PERF(max);
|
| 3161 |
|
| 3162 | value &= ~HWP_DESIRED_PERF(~0L);
|
| 3163 | value |= HWP_DESIRED_PERF(desired & hwp_desired_mask);
|
| 3164 |
|
| 3165 | if (value == prev)
|
| 3166 | return;
|
| 3167 |
|
| 3168 | WRITE_ONCE(cpu->hwp_req_cached, value);
|
| 3169 | if (fast_switch)
|
| 3170 | wrmsrq(MSR_HWP_REQUEST, value);
|
| 3171 | else
|
| 3172 | wrmsrq_on_cpu(cpu->cpu, MSR_HWP_REQUEST, value);
|
| 3173 | }
|
| 3174 |
|
| 3175 | static void intel_cpufreq_perf_ctl_update(struct cpudata *cpu,
|
| 3176 | u32 target_pstate, bool fast_switch)
|
| 3177 | {
|
| 3178 | if (fast_switch)
|
| 3179 | wrmsrq(MSR_IA32_PERF_CTL,
|
| 3180 | pstate_funcs.get_val(cpu, target_pstate));
|
| 3181 | else
|
| 3182 | wrmsrq_on_cpu(cpu->cpu, MSR_IA32_PERF_CTL,
|
| 3183 | pstate_funcs.get_val(cpu, target_pstate));
|
| 3184 | }
|
| 3185 |
|
| 3186 | static int intel_cpufreq_update_pstate(struct cpufreq_policy *policy,
|
| 3187 | int target_pstate, bool fast_switch)
|
| 3188 | {
|
| 3189 | struct cpudata *cpu = all_cpu_data[policy->cpu];
|
| 3190 | int old_pstate = cpu->pstate.current_pstate;
|
| 3191 |
|
| 3192 | target_pstate = intel_pstate_prepare_request(cpu, target_pstate);
|
| 3193 | if (hwp_active) {
|
| 3194 | int max_pstate = policy->strict_target ?
|
| 3195 | target_pstate : cpu->max_perf_ratio;
|
| 3196 |
|
| 3197 | intel_cpufreq_hwp_update(cpu, target_pstate, max_pstate,
|
| 3198 | target_pstate, fast_switch);
|
| 3199 | } else if (target_pstate != old_pstate) {
|
| 3200 | intel_cpufreq_perf_ctl_update(cpu, target_pstate, fast_switch);
|
| 3201 | }
|
| 3202 |
|
| 3203 | cpu->pstate.current_pstate = target_pstate;
|
| 3204 |
|
| 3205 | intel_cpufreq_trace(cpu, fast_switch ? INTEL_PSTATE_TRACE_FAST_SWITCH :
|
| 3206 | INTEL_PSTATE_TRACE_TARGET, old_pstate);
|
| 3207 |
|
| 3208 | return target_pstate;
|
| 3209 | }
|
| 3210 |
|
| 3211 | static int intel_cpufreq_target(struct cpufreq_policy *policy,
|
| 3212 | unsigned int target_freq,
|
| 3213 | unsigned int relation)
|
| 3214 | {
|
| 3215 | struct cpudata *cpu = all_cpu_data[policy->cpu];
|
| 3216 | struct cpufreq_freqs freqs;
|
| 3217 | int target_pstate;
|
| 3218 |
|
| 3219 | freqs.old = policy->cur;
|
| 3220 | freqs.new = target_freq;
|
| 3221 |
|
| 3222 | cpufreq_freq_transition_begin(policy, &freqs);
|
| 3223 |
|
| 3224 | target_pstate = intel_pstate_freq_to_hwp_rel(cpu, freqs.new, relation);
|
| 3225 | target_pstate = intel_cpufreq_update_pstate(policy, target_pstate, false);
|
| 3226 |
|
| 3227 | freqs.new = target_pstate * cpu->pstate.scaling;
|
| 3228 |
|
| 3229 | cpufreq_freq_transition_end(policy, &freqs, false);
|
| 3230 |
|
| 3231 | return 0;
|
| 3232 | }
|
| 3233 |
|
| 3234 | static unsigned int intel_cpufreq_fast_switch(struct cpufreq_policy *policy,
|
| 3235 | unsigned int target_freq)
|
| 3236 | {
|
| 3237 | struct cpudata *cpu = all_cpu_data[policy->cpu];
|
| 3238 | int target_pstate;
|
| 3239 |
|
| 3240 | target_pstate = intel_pstate_freq_to_hwp(cpu, target_freq);
|
| 3241 |
|
| 3242 | target_pstate = intel_cpufreq_update_pstate(policy, target_pstate, true);
|
| 3243 |
|
| 3244 | return target_pstate * cpu->pstate.scaling;
|
| 3245 | }
|
| 3246 |
|
| 3247 | static void intel_cpufreq_adjust_perf(struct cpufreq_policy *policy,
|
| 3248 | unsigned long min_perf,
|
| 3249 | unsigned long target_perf,
|
| 3250 | unsigned long max_perf,
|
| 3251 | unsigned long capacity)
|
| 3252 | {
|
| 3253 | struct cpudata *cpu = all_cpu_data[policy->cpu];
|
| 3254 | u64 hwp_cap = READ_ONCE(cpu->hwp_cap_cached);
|
| 3255 | int old_pstate = cpu->pstate.current_pstate;
|
| 3256 | int cap_pstate, min_pstate, max_pstate, target_pstate;
|
| 3257 |
|
| 3258 | cap_pstate = READ_ONCE(global.no_turbo) ?
|
| 3259 | HWP_GUARANTEED_PERF(hwp_cap) :
|
| 3260 | HWP_HIGHEST_PERF(hwp_cap);
|
| 3261 |
|
| 3262 |
|
| 3263 |
|
| 3264 | target_pstate = cap_pstate;
|
| 3265 | if (target_perf < capacity)
|
| 3266 | target_pstate = DIV_ROUND_UP(cap_pstate * target_perf, capacity);
|
| 3267 |
|
| 3268 | min_pstate = cap_pstate;
|
| 3269 | if (min_perf < capacity)
|
| 3270 | min_pstate = DIV_ROUND_UP(cap_pstate * min_perf, capacity);
|
| 3271 |
|
| 3272 | if (min_pstate < cpu->pstate.min_pstate)
|
| 3273 | min_pstate = cpu->pstate.min_pstate;
|
| 3274 |
|
| 3275 | if (min_pstate < cpu->min_perf_ratio)
|
| 3276 | min_pstate = cpu->min_perf_ratio;
|
| 3277 |
|
| 3278 | if (min_pstate > cpu->max_perf_ratio)
|
| 3279 | min_pstate = cpu->max_perf_ratio;
|
| 3280 |
|
| 3281 | max_pstate = cap_pstate;
|
| 3282 | if (max_perf < capacity)
|
| 3283 | max_pstate = DIV_ROUND_UP(cap_pstate * max_perf, capacity);
|
| 3284 |
|
| 3285 | if (max_pstate > cpu->max_perf_ratio)
|
| 3286 | max_pstate = cpu->max_perf_ratio;
|
| 3287 |
|
| 3288 | if (max_pstate < min_pstate)
|
| 3289 | max_pstate = min_pstate;
|
| 3290 |
|
| 3291 | target_pstate = clamp_t(int, target_pstate, min_pstate, max_pstate);
|
| 3292 |
|
| 3293 | intel_cpufreq_hwp_update(cpu, min_pstate, max_pstate, target_pstate, true);
|
| 3294 |
|
| 3295 | cpu->pstate.current_pstate = target_pstate;
|
| 3296 | intel_cpufreq_trace(cpu, INTEL_PSTATE_TRACE_FAST_SWITCH, old_pstate);
|
| 3297 | }
|
| 3298 |
|
| 3299 | static int intel_cpufreq_cpu_init(struct cpufreq_policy *policy)
|
| 3300 | {
|
| 3301 | struct freq_qos_request *req;
|
| 3302 | struct cpudata *cpu;
|
| 3303 | struct device *dev;
|
| 3304 | int ret, freq;
|
| 3305 |
|
| 3306 | dev = get_cpu_device(policy->cpu);
|
| 3307 | if (!dev)
|
| 3308 | return -ENODEV;
|
| 3309 |
|
| 3310 | ret = __intel_pstate_cpu_init(policy);
|
| 3311 | if (ret)
|
| 3312 | return ret;
|
| 3313 |
|
| 3314 | policy->cpuinfo.transition_latency = INTEL_CPUFREQ_TRANSITION_LATENCY;
|
| 3315 |
|
| 3316 | req = kzalloc_objs(*req, 2);
|
| 3317 | if (!req) {
|
| 3318 | ret = -ENOMEM;
|
| 3319 | goto pstate_exit;
|
| 3320 | }
|
| 3321 |
|
| 3322 | cpu = all_cpu_data[policy->cpu];
|
| 3323 |
|
| 3324 | if (hwp_active) {
|
| 3325 | u64 value;
|
| 3326 |
|
| 3327 | policy->transition_delay_us = INTEL_CPUFREQ_TRANSITION_DELAY_HWP;
|
| 3328 |
|
| 3329 | intel_pstate_get_hwp_cap(cpu);
|
| 3330 |
|
| 3331 | rdmsrq_on_cpu(cpu->cpu, MSR_HWP_REQUEST, &value);
|
| 3332 | WRITE_ONCE(cpu->hwp_req_cached, value);
|
| 3333 |
|
| 3334 | cpu->epp_cached = intel_pstate_get_epp(cpu, value);
|
| 3335 |
|
| 3336 | intel_cpufreq_hwp_update(cpu, cpu->pstate.min_pstate,
|
| 3337 | cpu->pstate.max_pstate,
|
| 3338 | cpu->pstate.min_pstate, false);
|
| 3339 | } else {
|
| 3340 | policy->transition_delay_us = INTEL_CPUFREQ_TRANSITION_DELAY;
|
| 3341 | intel_pstate_set_min_pstate(cpu);
|
| 3342 | }
|
| 3343 | policy->cur = policy->cpuinfo.min_freq;
|
| 3344 |
|
| 3345 | freq = DIV_ROUND_UP(cpu->pstate.turbo_freq * global.min_perf_pct, 100);
|
| 3346 |
|
| 3347 | ret = freq_qos_add_request(&policy->constraints, req, FREQ_QOS_MIN,
|
| 3348 | freq);
|
| 3349 | if (ret < 0) {
|
| 3350 | dev_err(dev, "Failed to add min-freq constraint (%d)\n", ret);
|
| 3351 | goto free_req;
|
| 3352 | }
|
| 3353 |
|
| 3354 | freq = DIV_ROUND_UP(cpu->pstate.turbo_freq * global.max_perf_pct, 100);
|
| 3355 |
|
| 3356 | ret = freq_qos_add_request(&policy->constraints, req + 1, FREQ_QOS_MAX,
|
| 3357 | freq);
|
| 3358 | if (ret < 0) {
|
| 3359 | dev_err(dev, "Failed to add max-freq constraint (%d)\n", ret);
|
| 3360 | goto remove_min_req;
|
| 3361 | }
|
| 3362 |
|
| 3363 | policy->driver_data = req;
|
| 3364 |
|
| 3365 | return 0;
|
| 3366 |
|
| 3367 | remove_min_req:
|
| 3368 | freq_qos_remove_request(req);
|
| 3369 | free_req:
|
| 3370 | kfree(req);
|
| 3371 | pstate_exit:
|
| 3372 | intel_pstate_exit_perf_limits(policy);
|
| 3373 |
|
| 3374 | return ret;
|
| 3375 | }
|
| 3376 |
|
| 3377 | static void intel_cpufreq_cpu_exit(struct cpufreq_policy *policy)
|
| 3378 | {
|
| 3379 | struct freq_qos_request *req;
|
| 3380 |
|
| 3381 | req = policy->driver_data;
|
| 3382 |
|
| 3383 | freq_qos_remove_request(req + 1);
|
| 3384 | freq_qos_remove_request(req);
|
| 3385 | kfree(req);
|
| 3386 |
|
| 3387 | intel_pstate_cpu_exit(policy);
|
| 3388 | }
|
| 3389 |
|
| 3390 | static int intel_cpufreq_suspend(struct cpufreq_policy *policy)
|
| 3391 | {
|
| 3392 | intel_pstate_suspend(policy);
|
| 3393 |
|
| 3394 | if (hwp_active) {
|
| 3395 | struct cpudata *cpu = all_cpu_data[policy->cpu];
|
| 3396 | u64 value = READ_ONCE(cpu->hwp_req_cached);
|
| 3397 |
|
| 3398 | |
| 3399 | * Clear the desired perf field in MSR_HWP_REQUEST in case
|
| 3400 | * intel_cpufreq_adjust_perf() is in use and the last value
|
| 3401 | * written by it may not be suitable.
|
| 3402 |
|
| 3403 | value &= ~HWP_DESIRED_PERF(~0L);
|
| 3404 | wrmsrq_on_cpu(cpu->cpu, MSR_HWP_REQUEST, value);
|
| 3405 | WRITE_ONCE(cpu->hwp_req_cached, value);
|
| 3406 | }
|
| 3407 |
|
| 3408 | return 0;
|
| 3409 | }
|
| 3410 |
|
| 3411 | static struct cpufreq_driver intel_cpufreq = {
|
| 3412 | .flags = CPUFREQ_CONST_LOOPS,
|
| 3413 | .verify = intel_cpufreq_verify_policy,
|
| 3414 | .target = intel_cpufreq_target,
|
| 3415 | .fast_switch = intel_cpufreq_fast_switch,
|
| 3416 | .init = intel_cpufreq_cpu_init,
|
| 3417 | .exit = intel_cpufreq_cpu_exit,
|
| 3418 | .offline = intel_cpufreq_cpu_offline,
|
| 3419 | .online = intel_pstate_cpu_online,
|
| 3420 | .suspend = intel_cpufreq_suspend,
|
| 3421 | .resume = intel_pstate_resume,
|
| 3422 | .update_limits = intel_pstate_update_limits,
|
| 3423 | .scale_freq_ref = intel_pstate_scale_freq_ref,
|
| 3424 | .name = "intel_cpufreq",
|
| 3425 | };
|
| 3426 |
|
| 3427 | static struct cpufreq_driver *default_driver;
|
| 3428 |
|
| 3429 | static void intel_pstate_driver_cleanup(void)
|
| 3430 | {
|
| 3431 | unsigned int cpu;
|
| 3432 |
|
| 3433 | cpus_read_lock();
|
| 3434 | for_each_online_cpu(cpu) {
|
| 3435 | if (all_cpu_data[cpu]) {
|
| 3436 | if (intel_pstate_driver == &intel_pstate)
|
| 3437 | intel_pstate_clear_update_util_hook(cpu);
|
| 3438 |
|
| 3439 | kfree(all_cpu_data[cpu]);
|
| 3440 | WRITE_ONCE(all_cpu_data[cpu], NULL);
|
| 3441 | }
|
| 3442 | }
|
| 3443 | cpus_read_unlock();
|
| 3444 |
|
| 3445 | intel_pstate_driver = NULL;
|
| 3446 | }
|
| 3447 |
|
| 3448 | static int intel_pstate_register_driver(struct cpufreq_driver *driver)
|
| 3449 | {
|
| 3450 | bool refresh_cpu_cap_scaling;
|
| 3451 | int ret;
|
| 3452 |
|
| 3453 | if (driver == &intel_pstate)
|
| 3454 | intel_pstate_sysfs_expose_hwp_dynamic_boost();
|
| 3455 |
|
| 3456 | memset(&global, 0, sizeof(global));
|
| 3457 | global.max_perf_pct = 100;
|
| 3458 | global.turbo_disabled = turbo_is_disabled();
|
| 3459 | global.no_turbo = global.turbo_disabled;
|
| 3460 |
|
| 3461 | arch_set_max_freq_ratio(global.turbo_disabled);
|
| 3462 |
|
| 3463 | refresh_cpu_cap_scaling = hybrid_clear_max_perf_cpu();
|
| 3464 |
|
| 3465 | intel_pstate_driver = driver;
|
| 3466 | ret = cpufreq_register_driver(intel_pstate_driver);
|
| 3467 | if (ret) {
|
| 3468 | intel_pstate_driver_cleanup();
|
| 3469 | return ret;
|
| 3470 | }
|
| 3471 |
|
| 3472 | global.min_perf_pct = min_perf_pct_min();
|
| 3473 |
|
| 3474 | hybrid_init_cpu_capacity_scaling(refresh_cpu_cap_scaling);
|
| 3475 |
|
| 3476 | return 0;
|
| 3477 | }
|
| 3478 |
|
| 3479 | static ssize_t intel_pstate_show_status(char *buf)
|
| 3480 | {
|
| 3481 | if (!intel_pstate_driver)
|
| 3482 | return sprintf(buf, "off\n");
|
| 3483 |
|
| 3484 | return sprintf(buf, "%s\n", intel_pstate_driver == &intel_pstate ?
|
| 3485 | "active" : "passive");
|
| 3486 | }
|
| 3487 |
|
| 3488 | static int intel_pstate_update_status(const char *buf, size_t size)
|
| 3489 | {
|
| 3490 | if (size == 3 && !strncmp(buf, "off", size)) {
|
| 3491 | if (!intel_pstate_driver)
|
| 3492 | return 0;
|
| 3493 |
|
| 3494 | if (hwp_active)
|
| 3495 | return -EBUSY;
|
| 3496 |
|
| 3497 | cpufreq_unregister_driver(intel_pstate_driver);
|
| 3498 | intel_pstate_driver_cleanup();
|
| 3499 | return 0;
|
| 3500 | }
|
| 3501 |
|
| 3502 | if (size == 6 && !strncmp(buf, "active", size)) {
|
| 3503 | if (intel_pstate_driver) {
|
| 3504 | if (intel_pstate_driver == &intel_pstate)
|
| 3505 | return 0;
|
| 3506 |
|
| 3507 | cpufreq_unregister_driver(intel_pstate_driver);
|
| 3508 | }
|
| 3509 |
|
| 3510 | return intel_pstate_register_driver(&intel_pstate);
|
| 3511 | }
|
| 3512 |
|
| 3513 | if (size == 7 && !strncmp(buf, "passive", size)) {
|
| 3514 | if (intel_pstate_driver) {
|
| 3515 | if (intel_pstate_driver == &intel_cpufreq)
|
| 3516 | return 0;
|
| 3517 |
|
| 3518 | cpufreq_unregister_driver(intel_pstate_driver);
|
| 3519 | intel_pstate_sysfs_hide_hwp_dynamic_boost();
|
| 3520 | }
|
| 3521 |
|
| 3522 | return intel_pstate_register_driver(&intel_cpufreq);
|
| 3523 | }
|
| 3524 |
|
| 3525 | return -EINVAL;
|
| 3526 | }
|
| 3527 |
|
| 3528 | static int no_load __initdata;
|
| 3529 | static int no_hwp __initdata;
|
| 3530 | static int hwp_only __initdata;
|
| 3531 | static unsigned int force_load __initdata;
|
| 3532 |
|
| 3533 | static int __init intel_pstate_msrs_not_valid(void)
|
| 3534 | {
|
| 3535 | if (!pstate_funcs.get_max(0) ||
|
| 3536 | !pstate_funcs.get_min(0) ||
|
| 3537 | !pstate_funcs.get_turbo(0))
|
| 3538 | return -ENODEV;
|
| 3539 |
|
| 3540 | return 0;
|
| 3541 | }
|
| 3542 |
|
| 3543 | static void __init copy_cpu_funcs(struct pstate_funcs *funcs)
|
| 3544 | {
|
| 3545 | pstate_funcs.get_max = funcs->get_max;
|
| 3546 | pstate_funcs.get_max_physical = funcs->get_max_physical;
|
| 3547 | pstate_funcs.get_min = funcs->get_min;
|
| 3548 | pstate_funcs.get_turbo = funcs->get_turbo;
|
| 3549 | pstate_funcs.get_scaling = funcs->get_scaling;
|
| 3550 | pstate_funcs.get_val = funcs->get_val;
|
| 3551 | pstate_funcs.get_vid = funcs->get_vid;
|
| 3552 | pstate_funcs.get_aperf_mperf_shift = funcs->get_aperf_mperf_shift;
|
| 3553 | }
|
| 3554 |
|
| 3555 | #ifdef CONFIG_ACPI
|
| 3556 |
|
| 3557 | static bool __init intel_pstate_no_acpi_pss(void)
|
| 3558 | {
|
| 3559 | int i;
|
| 3560 |
|
| 3561 | for_each_possible_cpu(i) {
|
| 3562 | acpi_status status;
|
| 3563 | union acpi_object *pss;
|
| 3564 | struct acpi_buffer buffer = { ACPI_ALLOCATE_BUFFER, NULL };
|
| 3565 | struct acpi_processor *pr = per_cpu(processors, i);
|
| 3566 |
|
| 3567 | if (!pr)
|
| 3568 | continue;
|
| 3569 |
|
| 3570 | status = acpi_evaluate_object(pr->handle, "_PSS", NULL, &buffer);
|
| 3571 | if (ACPI_FAILURE(status))
|
| 3572 | continue;
|
| 3573 |
|
| 3574 | pss = buffer.pointer;
|
| 3575 | if (pss && pss->type == ACPI_TYPE_PACKAGE) {
|
| 3576 | kfree(pss);
|
| 3577 | return false;
|
| 3578 | }
|
| 3579 |
|
| 3580 | kfree(pss);
|
| 3581 | }
|
| 3582 |
|
| 3583 | pr_debug("ACPI _PSS not found\n");
|
| 3584 | return true;
|
| 3585 | }
|
| 3586 |
|
| 3587 | static bool __init intel_pstate_no_acpi_pcch(void)
|
| 3588 | {
|
| 3589 | acpi_status status;
|
| 3590 | acpi_handle handle;
|
| 3591 |
|
| 3592 | status = acpi_get_handle(NULL, "\\_SB", &handle);
|
| 3593 | if (ACPI_FAILURE(status))
|
| 3594 | goto not_found;
|
| 3595 |
|
| 3596 | if (acpi_has_method(handle, "PCCH"))
|
| 3597 | return false;
|
| 3598 |
|
| 3599 | not_found:
|
| 3600 | pr_debug("ACPI PCCH not found\n");
|
| 3601 | return true;
|
| 3602 | }
|
| 3603 |
|
| 3604 | static bool __init intel_pstate_has_acpi_ppc(void)
|
| 3605 | {
|
| 3606 | int i;
|
| 3607 |
|
| 3608 | for_each_possible_cpu(i) {
|
| 3609 | struct acpi_processor *pr = per_cpu(processors, i);
|
| 3610 |
|
| 3611 | if (!pr)
|
| 3612 | continue;
|
| 3613 | if (acpi_has_method(pr->handle, "_PPC"))
|
| 3614 | return true;
|
| 3615 | }
|
| 3616 | pr_debug("ACPI _PPC not found\n");
|
| 3617 | return false;
|
| 3618 | }
|
| 3619 |
|
| 3620 | enum {
|
| 3621 | PSS,
|
| 3622 | PPC,
|
| 3623 | };
|
| 3624 |
|
| 3625 |
|
| 3626 | static struct acpi_platform_list plat_info[] __initdata = {
|
| 3627 | {"HP ", "ProLiant", 0, ACPI_SIG_FADT, all_versions, NULL, PSS},
|
| 3628 | {"ORACLE", "X4-2 ", 0, ACPI_SIG_FADT, all_versions, NULL, PPC},
|
| 3629 | {"ORACLE", "X4-2L ", 0, ACPI_SIG_FADT, all_versions, NULL, PPC},
|
| 3630 | {"ORACLE", "X4-2B ", 0, ACPI_SIG_FADT, all_versions, NULL, PPC},
|
| 3631 | {"ORACLE", "X3-2 ", 0, ACPI_SIG_FADT, all_versions, NULL, PPC},
|
| 3632 | {"ORACLE", "X3-2L ", 0, ACPI_SIG_FADT, all_versions, NULL, PPC},
|
| 3633 | {"ORACLE", "X3-2B ", 0, ACPI_SIG_FADT, all_versions, NULL, PPC},
|
| 3634 | {"ORACLE", "X4470M2 ", 0, ACPI_SIG_FADT, all_versions, NULL, PPC},
|
| 3635 | {"ORACLE", "X4270M3 ", 0, ACPI_SIG_FADT, all_versions, NULL, PPC},
|
| 3636 | {"ORACLE", "X4270M2 ", 0, ACPI_SIG_FADT, all_versions, NULL, PPC},
|
| 3637 | {"ORACLE", "X4170M2 ", 0, ACPI_SIG_FADT, all_versions, NULL, PPC},
|
| 3638 | {"ORACLE", "X4170 M3", 0, ACPI_SIG_FADT, all_versions, NULL, PPC},
|
| 3639 | {"ORACLE", "X4275 M3", 0, ACPI_SIG_FADT, all_versions, NULL, PPC},
|
| 3640 | {"ORACLE", "X6-2 ", 0, ACPI_SIG_FADT, all_versions, NULL, PPC},
|
| 3641 | {"ORACLE", "Sudbury ", 0, ACPI_SIG_FADT, all_versions, NULL, PPC},
|
| 3642 | { }
|
| 3643 | };
|
| 3644 |
|
| 3645 | #define BITMASK_OOB (BIT(8) | BIT(18))
|
| 3646 |
|
| 3647 | static bool __init intel_pstate_platform_pwr_mgmt_exists(void)
|
| 3648 | {
|
| 3649 | const struct x86_cpu_id *id;
|
| 3650 | u64 misc_pwr;
|
| 3651 | int idx;
|
| 3652 |
|
| 3653 | id = x86_match_cpu(intel_pstate_cpu_oob_ids);
|
| 3654 | if (id) {
|
| 3655 | rdmsrq(MSR_MISC_PWR_MGMT, misc_pwr);
|
| 3656 | if (misc_pwr & BITMASK_OOB) {
|
| 3657 | pr_debug("Bit 8 or 18 in the MISC_PWR_MGMT MSR set\n");
|
| 3658 | pr_debug("P states are controlled in Out of Band mode by the firmware/hardware\n");
|
| 3659 | return true;
|
| 3660 | }
|
| 3661 | }
|
| 3662 |
|
| 3663 | idx = acpi_match_platform_list(plat_info);
|
| 3664 | if (idx < 0)
|
| 3665 | return false;
|
| 3666 |
|
| 3667 | switch (plat_info[idx].data) {
|
| 3668 | case PSS:
|
| 3669 | if (!intel_pstate_no_acpi_pss())
|
| 3670 | return false;
|
| 3671 |
|
| 3672 | return intel_pstate_no_acpi_pcch();
|
| 3673 | case PPC:
|
| 3674 | return intel_pstate_has_acpi_ppc() && !force_load;
|
| 3675 | }
|
| 3676 |
|
| 3677 | return false;
|
| 3678 | }
|
| 3679 |
|
| 3680 | static void intel_pstate_request_control_from_smm(void)
|
| 3681 | {
|
| 3682 | |
| 3683 | * It may be unsafe to request P-states control from SMM if _PPC support
|
| 3684 | * has not been enabled.
|
| 3685 |
|
| 3686 | if (acpi_ppc)
|
| 3687 | acpi_processor_pstate_control();
|
| 3688 | }
|
| 3689 | #else
|
| 3690 | static inline bool intel_pstate_platform_pwr_mgmt_exists(void) { return false; }
|
| 3691 | static inline bool intel_pstate_has_acpi_ppc(void) { return false; }
|
| 3692 | static inline void intel_pstate_request_control_from_smm(void) {}
|
| 3693 | #endif
|
| 3694 |
|
| 3695 | #define INTEL_PSTATE_HWP_NOT_HYBRID 0x01
|
| 3696 |
|
| 3697 | #define X86_MATCH_HWP(vfm, hwp_mode) \
|
| 3698 | X86_MATCH_VFM_FEATURE(vfm, X86_FEATURE_HWP, hwp_mode)
|
| 3699 |
|
| 3700 | static const struct x86_cpu_id hwp_support_ids[] __initconst = {
|
| 3701 | X86_MATCH_HWP(INTEL_BROADWELL_X, INTEL_PSTATE_HWP_NOT_HYBRID),
|
| 3702 | X86_MATCH_HWP(INTEL_BROADWELL_D, INTEL_PSTATE_HWP_NOT_HYBRID),
|
| 3703 | X86_MATCH_HWP(INTEL_ANY, 0),
|
| 3704 | {}
|
| 3705 | };
|
| 3706 |
|
| 3707 | static bool intel_pstate_hwp_is_enabled(void)
|
| 3708 | {
|
| 3709 | u64 value;
|
| 3710 |
|
| 3711 | rdmsrq(MSR_PM_ENABLE, value);
|
| 3712 | return !!(value & 0x1);
|
| 3713 | }
|
| 3714 |
|
| 3715 | #define POWERSAVE_MASK GENMASK(7, 0)
|
| 3716 | #define BALANCE_POWER_MASK GENMASK(15, 8)
|
| 3717 | #define BALANCE_PERFORMANCE_MASK GENMASK(23, 16)
|
| 3718 | #define PERFORMANCE_MASK GENMASK(31, 24)
|
| 3719 |
|
| 3720 | #define HWP_SET_EPP_VALUES(powersave, balance_power, balance_perf, performance) \
|
| 3721 | (FIELD_PREP_CONST(POWERSAVE_MASK, powersave) |\
|
| 3722 | FIELD_PREP_CONST(BALANCE_POWER_MASK, balance_power) |\
|
| 3723 | FIELD_PREP_CONST(BALANCE_PERFORMANCE_MASK, balance_perf) |\
|
| 3724 | FIELD_PREP_CONST(PERFORMANCE_MASK, performance))
|
| 3725 |
|
| 3726 | #define HWP_SET_DEF_BALANCE_PERF_EPP(balance_perf) \
|
| 3727 | (HWP_SET_EPP_VALUES(HWP_EPP_POWERSAVE, HWP_EPP_BALANCE_POWERSAVE,\
|
| 3728 | balance_perf, HWP_EPP_PERFORMANCE))
|
| 3729 |
|
| 3730 | static const struct x86_cpu_id intel_epp_default[] = {
|
| 3731 | |
| 3732 | * Set EPP value as 102, this is the max suggested EPP
|
| 3733 | * which can result in one core turbo frequency for
|
| 3734 | * AlderLake Mobile CPUs.
|
| 3735 |
|
| 3736 | X86_MATCH_VFM(INTEL_ALDERLAKE_L, HWP_SET_DEF_BALANCE_PERF_EPP(102)),
|
| 3737 | X86_MATCH_VFM(INTEL_SAPPHIRERAPIDS_X, HWP_SET_DEF_BALANCE_PERF_EPP(32)),
|
| 3738 | X86_MATCH_VFM(INTEL_EMERALDRAPIDS_X, HWP_SET_DEF_BALANCE_PERF_EPP(32)),
|
| 3739 | X86_MATCH_VFM(INTEL_GRANITERAPIDS_X, HWP_SET_DEF_BALANCE_PERF_EPP(32)),
|
| 3740 | X86_MATCH_VFM(INTEL_GRANITERAPIDS_D, HWP_SET_DEF_BALANCE_PERF_EPP(32)),
|
| 3741 | X86_MATCH_VFM(INTEL_METEORLAKE_L, HWP_SET_EPP_VALUES(HWP_EPP_POWERSAVE,
|
| 3742 | 179, 64, 16)),
|
| 3743 | X86_MATCH_VFM(INTEL_ARROWLAKE, HWP_SET_EPP_VALUES(HWP_EPP_POWERSAVE,
|
| 3744 | 179, 64, 16)),
|
| 3745 | {}
|
| 3746 | };
|
| 3747 |
|
| 3748 | static const struct x86_cpu_id intel_hybrid_scaling_factor[] = {
|
| 3749 | X86_MATCH_VFM(INTEL_ALDERLAKE, HYBRID_SCALING_FACTOR_ADL),
|
| 3750 | X86_MATCH_VFM(INTEL_ALDERLAKE_L, HYBRID_SCALING_FACTOR_ADL),
|
| 3751 | X86_MATCH_VFM(INTEL_RAPTORLAKE, HYBRID_SCALING_FACTOR_ADL),
|
| 3752 | X86_MATCH_VFM(INTEL_RAPTORLAKE_P, HYBRID_SCALING_FACTOR_ADL),
|
| 3753 | X86_MATCH_VFM(INTEL_RAPTORLAKE_S, HYBRID_SCALING_FACTOR_ADL),
|
| 3754 | X86_MATCH_VFM(INTEL_BARTLETTLAKE, HYBRID_SCALING_FACTOR_ADL),
|
| 3755 | X86_MATCH_VFM(INTEL_METEORLAKE_L, HYBRID_SCALING_FACTOR_MTL),
|
| 3756 | X86_MATCH_VFM(INTEL_LUNARLAKE_M, HYBRID_SCALING_FACTOR_LNL),
|
| 3757 | {}
|
| 3758 | };
|
| 3759 |
|
| 3760 | static bool hwp_check_epp(void)
|
| 3761 | {
|
| 3762 | if (boot_cpu_has(X86_FEATURE_HWP_EPP))
|
| 3763 | return true;
|
| 3764 |
|
| 3765 |
|
| 3766 | hwp_cpufreq_attrs[HWP_PERFORMANCE_PREFERENCE_INDEX] = NULL;
|
| 3767 | hwp_cpufreq_attrs[HWP_PERFORMANCE_AVAILABLE_PREFERENCES_INDEX] = NULL;
|
| 3768 |
|
| 3769 | return false;
|
| 3770 | }
|
| 3771 |
|
| 3772 | static bool hwp_check_dec(void)
|
| 3773 | {
|
| 3774 | u64 power_ctl = 0;
|
| 3775 |
|
| 3776 | rdmsrq_safe(MSR_IA32_POWER_CTL, &power_ctl);
|
| 3777 | return !!(power_ctl & BIT(POWER_CTL_DEC_ENABLE));
|
| 3778 | }
|
| 3779 |
|
| 3780 | static int __init intel_pstate_init(void)
|
| 3781 | {
|
| 3782 | static struct cpudata **_all_cpu_data;
|
| 3783 | const struct x86_cpu_id *id;
|
| 3784 | int rc;
|
| 3785 |
|
| 3786 | if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL)
|
| 3787 | return -ENODEV;
|
| 3788 |
|
| 3789 | |
| 3790 | * The Intel pstate driver will be ignored if the platform
|
| 3791 | * firmware has its own power management modes.
|
| 3792 |
|
| 3793 | if (intel_pstate_platform_pwr_mgmt_exists()) {
|
| 3794 | pr_info("P-states controlled by the platform\n");
|
| 3795 | return -ENODEV;
|
| 3796 | }
|
| 3797 |
|
| 3798 | id = x86_match_cpu(hwp_support_ids);
|
| 3799 | if (id) {
|
| 3800 | bool epp_present = hwp_check_epp();
|
| 3801 | bool dec_present = hwp_check_dec();
|
| 3802 |
|
| 3803 | |
| 3804 | * If HWP is enabled already, there is no choice but to deal
|
| 3805 | * with it.
|
| 3806 |
|
| 3807 | hwp_forced = intel_pstate_hwp_is_enabled();
|
| 3808 | if (hwp_forced) {
|
| 3809 | pr_info("HWP enabled by BIOS\n");
|
| 3810 | no_hwp = 0;
|
| 3811 | } else if (no_load) {
|
| 3812 | return -ENODEV;
|
| 3813 | } else if (!epp_present && !dec_present) {
|
| 3814 | |
| 3815 | * Avoid enabling HWP for processors without EPP support
|
| 3816 | * unless the Dynamic Efficiency Control (DEC) enable
|
| 3817 | * bit (MSR_IA32_POWER_CTL, bit 27) is set because that
|
| 3818 | * means incomplete HWP implementation which is a corner
|
| 3819 | * case and supporting it is generally problematic.
|
| 3820 |
|
| 3821 | no_hwp = 1;
|
| 3822 | }
|
| 3823 |
|
| 3824 | copy_cpu_funcs(&core_funcs);
|
| 3825 |
|
| 3826 | if (!no_hwp) {
|
| 3827 | hwp_active = true;
|
| 3828 | intel_pstate.attr = hwp_cpufreq_attrs;
|
| 3829 | intel_cpufreq.attr = hwp_cpufreq_attrs;
|
| 3830 | intel_cpufreq.flags |= CPUFREQ_NEED_UPDATE_LIMITS;
|
| 3831 | intel_cpufreq.adjust_perf = intel_cpufreq_adjust_perf;
|
| 3832 | if (!default_driver)
|
| 3833 | default_driver = &intel_pstate;
|
| 3834 |
|
| 3835 | if (dec_present)
|
| 3836 | hwp_desired_mask = 0;
|
| 3837 |
|
| 3838 | if (!id->driver_data)
|
| 3839 | pstate_funcs.get_cpu_scaling = hwp_get_cpu_scaling;
|
| 3840 |
|
| 3841 | goto hwp_cpu_matched;
|
| 3842 | }
|
| 3843 | pr_info("HWP not enabled\n");
|
| 3844 | } else {
|
| 3845 | if (no_load)
|
| 3846 | return -ENODEV;
|
| 3847 |
|
| 3848 | id = x86_match_cpu(intel_hybrid_scaling_factor);
|
| 3849 | if (id) {
|
| 3850 | pr_info("HWP-disabled hybrid CPU is not supported\n");
|
| 3851 | return -ENODEV;
|
| 3852 | }
|
| 3853 |
|
| 3854 | id = x86_match_cpu(intel_pstate_cpu_ids);
|
| 3855 | if (!id) {
|
| 3856 | pr_info("CPU model not supported\n");
|
| 3857 | return -ENODEV;
|
| 3858 | }
|
| 3859 |
|
| 3860 | copy_cpu_funcs((struct pstate_funcs *)id->driver_data);
|
| 3861 | }
|
| 3862 |
|
| 3863 | if (intel_pstate_msrs_not_valid()) {
|
| 3864 | pr_info("Invalid MSRs\n");
|
| 3865 | return -ENODEV;
|
| 3866 | }
|
| 3867 |
|
| 3868 | if (!default_driver)
|
| 3869 | default_driver = &intel_cpufreq;
|
| 3870 |
|
| 3871 | hwp_cpu_matched:
|
| 3872 | if (!hwp_active && hwp_only)
|
| 3873 | return -ENOTSUPP;
|
| 3874 |
|
| 3875 | pr_info("Intel P-state driver initializing\n");
|
| 3876 |
|
| 3877 | _all_cpu_data = vzalloc(array_size(sizeof(void *), num_possible_cpus()));
|
| 3878 | if (!_all_cpu_data)
|
| 3879 | return -ENOMEM;
|
| 3880 |
|
| 3881 | WRITE_ONCE(all_cpu_data, _all_cpu_data);
|
| 3882 |
|
| 3883 | intel_pstate_request_control_from_smm();
|
| 3884 |
|
| 3885 | intel_pstate_sysfs_expose_params();
|
| 3886 |
|
| 3887 | if (hwp_active) {
|
| 3888 | const struct x86_cpu_id *id = x86_match_cpu(intel_epp_default);
|
| 3889 | const struct x86_cpu_id *hybrid_id = x86_match_cpu(intel_hybrid_scaling_factor);
|
| 3890 |
|
| 3891 | if (id) {
|
| 3892 | epp_values[EPP_INDEX_POWERSAVE] =
|
| 3893 | FIELD_GET(POWERSAVE_MASK, id->driver_data);
|
| 3894 | epp_values[EPP_INDEX_BALANCE_POWERSAVE] =
|
| 3895 | FIELD_GET(BALANCE_POWER_MASK, id->driver_data);
|
| 3896 | epp_values[EPP_INDEX_BALANCE_PERFORMANCE] =
|
| 3897 | FIELD_GET(BALANCE_PERFORMANCE_MASK, id->driver_data);
|
| 3898 | epp_values[EPP_INDEX_PERFORMANCE] =
|
| 3899 | FIELD_GET(PERFORMANCE_MASK, id->driver_data);
|
| 3900 | pr_debug("Updated EPPs powersave:%x balanced power:%x balanced perf:%x performance:%x\n",
|
| 3901 | epp_values[EPP_INDEX_POWERSAVE],
|
| 3902 | epp_values[EPP_INDEX_BALANCE_POWERSAVE],
|
| 3903 | epp_values[EPP_INDEX_BALANCE_PERFORMANCE],
|
| 3904 | epp_values[EPP_INDEX_PERFORMANCE]);
|
| 3905 | }
|
| 3906 |
|
| 3907 | if (hybrid_id) {
|
| 3908 | hybrid_scaling_factor = hybrid_id->driver_data;
|
| 3909 | pr_debug("hybrid scaling factor: %d\n", hybrid_scaling_factor);
|
| 3910 | }
|
| 3911 |
|
| 3912 | }
|
| 3913 |
|
| 3914 | scoped_guard(mutex, &intel_pstate_driver_lock) {
|
| 3915 | rc = intel_pstate_register_driver(default_driver);
|
| 3916 | }
|
| 3917 | if (rc) {
|
| 3918 | intel_pstate_sysfs_remove();
|
| 3919 | return rc;
|
| 3920 | }
|
| 3921 |
|
| 3922 | if (hwp_active) {
|
| 3923 | const struct x86_cpu_id *id;
|
| 3924 |
|
| 3925 | id = x86_match_cpu(intel_pstate_cpu_ee_disable_ids);
|
| 3926 | if (id) {
|
| 3927 | set_power_ctl_ee_state(false);
|
| 3928 | pr_info("Disabling energy efficiency optimization\n");
|
| 3929 | }
|
| 3930 |
|
| 3931 | pr_info("HWP enabled\n");
|
| 3932 | } else if (boot_cpu_has(X86_FEATURE_HYBRID_CPU)) {
|
| 3933 | pr_warn("Problematic setup: Hybrid processor with disabled HWP\n");
|
| 3934 | }
|
| 3935 |
|
| 3936 | return 0;
|
| 3937 | }
|
| 3938 | device_initcall(intel_pstate_init);
|
| 3939 |
|
| 3940 | static int __init intel_pstate_setup(char *str)
|
| 3941 | {
|
| 3942 | if (!str)
|
| 3943 | return -EINVAL;
|
| 3944 |
|
| 3945 | if (!strcmp(str, "disable"))
|
| 3946 | no_load = 1;
|
| 3947 | else if (!strcmp(str, "active"))
|
| 3948 | default_driver = &intel_pstate;
|
| 3949 | else if (!strcmp(str, "passive"))
|
| 3950 | default_driver = &intel_cpufreq;
|
| 3951 |
|
| 3952 | if (!strcmp(str, "no_hwp"))
|
| 3953 | no_hwp = 1;
|
| 3954 |
|
| 3955 | if (!strcmp(str, "no_cas"))
|
| 3956 | no_cas = true;
|
| 3957 |
|
| 3958 | if (!strcmp(str, "force"))
|
| 3959 | force_load = 1;
|
| 3960 | if (!strcmp(str, "hwp_only"))
|
| 3961 | hwp_only = 1;
|
| 3962 | if (!strcmp(str, "per_cpu_perf_limits"))
|
| 3963 | per_cpu_limits = true;
|
| 3964 |
|
| 3965 | #ifdef CONFIG_ACPI
|
| 3966 | if (!strcmp(str, "support_acpi_ppc"))
|
| 3967 | acpi_ppc = true;
|
| 3968 | #endif
|
| 3969 |
|
| 3970 | return 0;
|
| 3971 | }
|
| 3972 | early_param("intel_pstate", intel_pstate_setup);
|
| 3973 |
|
| 3974 | MODULE_AUTHOR("Dirk Brandewie <[email protected]>");
|
| 3975 | MODULE_DESCRIPTION("'intel_pstate' - P state driver Intel Core processors");
|