Lines Matching +full:fine +full:- +full:tune
1 // SPDX-License-Identifier: GPL-2.0-or-later
3 * amd-pstate.c - AMD Processor P-state Frequency Driver
9 * AMD P-State introduces a new CPU performance scaling design for AMD
12 * frequency control range. It is to replace the legacy ACPI P-States control,
13 * allows a flexible, low-latency interface for the Linux kernel to directly
16 * AMD P-State is supported on recent AMD Zen base CPU series include some of
18 * P-State supported system. And there are two types of hardware implementations
19 * for AMD P-State: 1) Full MSR Solution and 2) Shared Memory Solution.
39 #include <linux/amd-pstate.h>
48 #include "amd-pstate-trace.h"
54 * TODO: We need more time to fine tune processors with shared memory solution
58 * Suse. We are co-working with them to fine tune the shared memory solution. So
59 * we disable it by default to go acpi-cpufreq on these processors and add a
77 *-------------------------------------
119 return -EINVAL; in get_mode_idx_from_str()
132 epp = rdmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, in amd_pstate_get_epp()
139 ret = cppc_get_epp_perf(cpudata->cpu, &epp); in amd_pstate_get_epp()
142 return -EIO; in amd_pstate_get_epp()
152 int index = -EINVAL; in amd_pstate_get_energy_pref_index()
184 u64 value = READ_ONCE(cpudata->cppc_req_cached); in amd_pstate_set_epp()
188 WRITE_ONCE(cpudata->cppc_req_cached, value); in amd_pstate_set_epp()
190 ret = wrmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, value); in amd_pstate_set_epp()
192 cpudata->epp_cached = epp; in amd_pstate_set_epp()
195 ret = cppc_set_epp_perf(cpudata->cpu, &perf_ctrls, 1); in amd_pstate_set_epp()
200 cpudata->epp_cached = epp; in amd_pstate_set_epp()
209 int epp = -EINVAL; in amd_pstate_set_energy_pref_index()
214 return -EINVAL; in amd_pstate_set_energy_pref_index()
217 if (epp == -EINVAL) in amd_pstate_set_energy_pref_index()
220 if (epp > 0 && cpudata->policy == CPUFREQ_POLICY_PERFORMANCE) { in amd_pstate_set_energy_pref_index()
222 return -EBUSY; in amd_pstate_set_energy_pref_index()
295 int ret = rdmsrl_safe_on_cpu(cpudata->cpu, MSR_AMD_CPPC_CAP1, in pstate_init_perf()
309 WRITE_ONCE(cpudata->highest_perf, highest_perf); in pstate_init_perf()
310 WRITE_ONCE(cpudata->max_limit_perf, highest_perf); in pstate_init_perf()
311 WRITE_ONCE(cpudata->nominal_perf, AMD_CPPC_NOMINAL_PERF(cap1)); in pstate_init_perf()
312 WRITE_ONCE(cpudata->lowest_nonlinear_perf, AMD_CPPC_LOWNONLIN_PERF(cap1)); in pstate_init_perf()
313 WRITE_ONCE(cpudata->lowest_perf, AMD_CPPC_LOWEST_PERF(cap1)); in pstate_init_perf()
314 WRITE_ONCE(cpudata->min_limit_perf, AMD_CPPC_LOWEST_PERF(cap1)); in pstate_init_perf()
323 int ret = cppc_get_perf_caps(cpudata->cpu, &cppc_perf); in cppc_init_perf()
331 WRITE_ONCE(cpudata->highest_perf, highest_perf); in cppc_init_perf()
332 WRITE_ONCE(cpudata->max_limit_perf, highest_perf); in cppc_init_perf()
333 WRITE_ONCE(cpudata->nominal_perf, cppc_perf.nominal_perf); in cppc_init_perf()
334 WRITE_ONCE(cpudata->lowest_nonlinear_perf, in cppc_init_perf()
336 WRITE_ONCE(cpudata->lowest_perf, cppc_perf.lowest_perf); in cppc_init_perf()
337 WRITE_ONCE(cpudata->min_limit_perf, cppc_perf.lowest_perf); in cppc_init_perf()
342 ret = cppc_get_auto_sel_caps(cpudata->cpu, &cppc_perf); in cppc_init_perf()
348 ret = cppc_set_auto_sel(cpudata->cpu, in cppc_init_perf()
368 wrmsrl(MSR_AMD_CPPC_REQ, READ_ONCE(cpudata->cppc_req_cached)); in pstate_update_perf()
370 wrmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, in pstate_update_perf()
371 READ_ONCE(cpudata->cppc_req_cached)); in pstate_update_perf()
384 cppc_set_perf(cpudata->cpu, &perf_ctrls); in cppc_update_perf()
407 if (cpudata->prev.mperf == mperf || cpudata->prev.tsc == tsc) { in amd_pstate_sample()
414 cpudata->cur.aperf = aperf; in amd_pstate_sample()
415 cpudata->cur.mperf = mperf; in amd_pstate_sample()
416 cpudata->cur.tsc = tsc; in amd_pstate_sample()
417 cpudata->cur.aperf -= cpudata->prev.aperf; in amd_pstate_sample()
418 cpudata->cur.mperf -= cpudata->prev.mperf; in amd_pstate_sample()
419 cpudata->cur.tsc -= cpudata->prev.tsc; in amd_pstate_sample()
421 cpudata->prev.aperf = aperf; in amd_pstate_sample()
422 cpudata->prev.mperf = mperf; in amd_pstate_sample()
423 cpudata->prev.tsc = tsc; in amd_pstate_sample()
425 cpudata->freq = div64_u64((cpudata->cur.aperf * cpu_khz), cpudata->cur.mperf); in amd_pstate_sample()
433 u64 prev = READ_ONCE(cpudata->cppc_req_cached); in amd_pstate_update()
436 min_perf = clamp_t(unsigned long, min_perf, cpudata->min_limit_perf, in amd_pstate_update()
437 cpudata->max_limit_perf); in amd_pstate_update()
438 max_perf = clamp_t(unsigned long, max_perf, cpudata->min_limit_perf, in amd_pstate_update()
439 cpudata->max_limit_perf); in amd_pstate_update()
457 trace_amd_pstate_perf(min_perf, des_perf, max_perf, cpudata->freq, in amd_pstate_update()
458 cpudata->cur.mperf, cpudata->cur.aperf, cpudata->cur.tsc, in amd_pstate_update()
459 cpudata->cpu, (value != prev), fast_switch); in amd_pstate_update()
465 WRITE_ONCE(cpudata->cppc_req_cached, value); in amd_pstate_update()
481 struct amd_cpudata *cpudata = policy->driver_data; in amd_pstate_update_min_max_limit()
483 max_limit_perf = div_u64(policy->max * cpudata->highest_perf, cpudata->max_freq); in amd_pstate_update_min_max_limit()
484 min_limit_perf = div_u64(policy->min * cpudata->highest_perf, cpudata->max_freq); in amd_pstate_update_min_max_limit()
486 WRITE_ONCE(cpudata->max_limit_perf, max_limit_perf); in amd_pstate_update_min_max_limit()
487 WRITE_ONCE(cpudata->min_limit_perf, min_limit_perf); in amd_pstate_update_min_max_limit()
488 WRITE_ONCE(cpudata->max_limit_freq, policy->max); in amd_pstate_update_min_max_limit()
489 WRITE_ONCE(cpudata->min_limit_freq, policy->min); in amd_pstate_update_min_max_limit()
498 struct amd_cpudata *cpudata = policy->driver_data; in amd_pstate_update_freq()
501 if (!cpudata->max_freq) in amd_pstate_update_freq()
502 return -ENODEV; in amd_pstate_update_freq()
504 if (policy->min != cpudata->min_limit_freq || policy->max != cpudata->max_limit_freq) in amd_pstate_update_freq()
507 cap_perf = READ_ONCE(cpudata->highest_perf); in amd_pstate_update_freq()
508 min_perf = READ_ONCE(cpudata->lowest_perf); in amd_pstate_update_freq()
511 freqs.old = policy->cur; in amd_pstate_update_freq()
515 cpudata->max_freq); in amd_pstate_update_freq()
517 WARN_ON(fast_switch && !policy->fast_switch_enabled); in amd_pstate_update_freq()
527 max_perf, fast_switch, policy->governor->flags); in amd_pstate_update_freq()
547 return policy->cur; in amd_pstate_fast_switch()
558 struct amd_cpudata *cpudata = policy->driver_data; in amd_pstate_adjust_perf()
561 if (policy->min != cpudata->min_limit_freq || policy->max != cpudata->max_limit_freq) in amd_pstate_adjust_perf()
565 cap_perf = READ_ONCE(cpudata->highest_perf); in amd_pstate_adjust_perf()
566 lowest_nonlinear_perf = READ_ONCE(cpudata->lowest_nonlinear_perf); in amd_pstate_adjust_perf()
567 max_freq = READ_ONCE(cpudata->max_freq); in amd_pstate_adjust_perf()
573 min_perf = READ_ONCE(cpudata->highest_perf); in amd_pstate_adjust_perf()
586 policy->cur = target_freq; in amd_pstate_adjust_perf()
589 policy->governor->flags); in amd_pstate_adjust_perf()
597 int ret = cppc_get_perf_caps(cpudata->cpu, &cppc_perf); in amd_get_min_freq()
611 int ret = cppc_get_perf_caps(cpudata->cpu, &cppc_perf); in amd_get_max_freq()
616 nominal_perf = READ_ONCE(cpudata->nominal_perf); in amd_get_max_freq()
617 max_perf = READ_ONCE(cpudata->highest_perf); in amd_get_max_freq()
632 int ret = cppc_get_perf_caps(cpudata->cpu, &cppc_perf); in amd_get_nominal_freq()
647 int ret = cppc_get_perf_caps(cpudata->cpu, &cppc_perf); in amd_get_lowest_nonlinear_freq()
652 nominal_perf = READ_ONCE(cpudata->nominal_perf); in amd_get_lowest_nonlinear_freq()
667 struct amd_cpudata *cpudata = policy->driver_data; in amd_pstate_set_boost()
670 if (!cpudata->boost_supported) { in amd_pstate_set_boost()
672 return -EINVAL; in amd_pstate_set_boost()
676 policy->cpuinfo.max_freq = cpudata->max_freq; in amd_pstate_set_boost()
678 policy->cpuinfo.max_freq = cpudata->nominal_freq; in amd_pstate_set_boost()
680 policy->max = policy->cpuinfo.max_freq; in amd_pstate_set_boost()
682 ret = freq_qos_update_request(&cpudata->req[1], in amd_pstate_set_boost()
683 policy->cpuinfo.max_freq); in amd_pstate_set_boost()
694 highest_perf = READ_ONCE(cpudata->highest_perf); in amd_pstate_boost_init()
695 nominal_perf = READ_ONCE(cpudata->nominal_perf); in amd_pstate_boost_init()
700 cpudata->boost_supported = true; in amd_pstate_boost_init()
701 current_pstate_driver->boost_enabled = true; in amd_pstate_boost_init()
719 amd_perf_ctl_reset(policy->cpu); in amd_pstate_cpu_init()
720 dev = get_cpu_device(policy->cpu); in amd_pstate_cpu_init()
722 return -ENODEV; in amd_pstate_cpu_init()
726 return -ENOMEM; in amd_pstate_cpu_init()
728 cpudata->cpu = policy->cpu; in amd_pstate_cpu_init()
742 ret = -EINVAL; in amd_pstate_cpu_init()
746 policy->cpuinfo.transition_latency = AMD_PSTATE_TRANSITION_LATENCY; in amd_pstate_cpu_init()
747 policy->transition_delay_us = AMD_PSTATE_TRANSITION_DELAY; in amd_pstate_cpu_init()
749 policy->min = min_freq; in amd_pstate_cpu_init()
750 policy->max = max_freq; in amd_pstate_cpu_init()
752 policy->cpuinfo.min_freq = min_freq; in amd_pstate_cpu_init()
753 policy->cpuinfo.max_freq = max_freq; in amd_pstate_cpu_init()
756 policy->cur = policy->cpuinfo.min_freq; in amd_pstate_cpu_init()
759 policy->fast_switch_possible = true; in amd_pstate_cpu_init()
761 ret = freq_qos_add_request(&policy->constraints, &cpudata->req[0], in amd_pstate_cpu_init()
762 FREQ_QOS_MIN, policy->cpuinfo.min_freq); in amd_pstate_cpu_init()
764 dev_err(dev, "Failed to add min-freq constraint (%d)\n", ret); in amd_pstate_cpu_init()
768 ret = freq_qos_add_request(&policy->constraints, &cpudata->req[1], in amd_pstate_cpu_init()
769 FREQ_QOS_MAX, policy->cpuinfo.max_freq); in amd_pstate_cpu_init()
771 dev_err(dev, "Failed to add max-freq constraint (%d)\n", ret); in amd_pstate_cpu_init()
776 cpudata->max_freq = max_freq; in amd_pstate_cpu_init()
777 cpudata->min_freq = min_freq; in amd_pstate_cpu_init()
778 cpudata->max_limit_freq = max_freq; in amd_pstate_cpu_init()
779 cpudata->min_limit_freq = min_freq; in amd_pstate_cpu_init()
780 cpudata->nominal_freq = nominal_freq; in amd_pstate_cpu_init()
781 cpudata->lowest_nonlinear_freq = lowest_nonlinear_freq; in amd_pstate_cpu_init()
783 policy->driver_data = cpudata; in amd_pstate_cpu_init()
786 if (!current_pstate_driver->adjust_perf) in amd_pstate_cpu_init()
787 current_pstate_driver->adjust_perf = amd_pstate_adjust_perf; in amd_pstate_cpu_init()
792 freq_qos_remove_request(&cpudata->req[0]); in amd_pstate_cpu_init()
800 struct amd_cpudata *cpudata = policy->driver_data; in amd_pstate_cpu_exit()
802 freq_qos_remove_request(&cpudata->req[1]); in amd_pstate_cpu_exit()
803 freq_qos_remove_request(&cpudata->req[0]); in amd_pstate_cpu_exit()
804 policy->fast_switch_possible = false; in amd_pstate_cpu_exit()
816 pr_err("failed to enable amd-pstate during resume, return %d\n", ret); in amd_pstate_cpu_resume()
827 pr_err("failed to disable amd-pstate during suspend, return %d\n", ret); in amd_pstate_cpu_suspend()
843 struct amd_cpudata *cpudata = policy->driver_data; in show_amd_pstate_max_freq()
856 struct amd_cpudata *cpudata = policy->driver_data; in show_amd_pstate_lowest_nonlinear_freq()
873 struct amd_cpudata *cpudata = policy->driver_data; in show_amd_pstate_highest_perf()
875 perf = READ_ONCE(cpudata->highest_perf); in show_amd_pstate_highest_perf()
885 struct amd_cpudata *cpudata = policy->driver_data; in show_energy_performance_available_preferences()
887 if (cpudata->policy == CPUFREQ_POLICY_PERFORMANCE) in show_energy_performance_available_preferences()
902 struct amd_cpudata *cpudata = policy->driver_data; in store_energy_performance_preference()
908 return -EINVAL; in store_energy_performance_preference()
910 ret = match_string(energy_perf_strings, -1, str_preference); in store_energy_performance_preference()
912 return -EINVAL; in store_energy_performance_preference()
924 struct amd_cpudata *cpudata = policy->driver_data; in show_energy_performance_preference()
950 return -EINVAL; in amd_pstate_register_driver()
1039 return -EINVAL; in amd_pstate_update_status()
1044 return -EINVAL; in amd_pstate_update_status()
1071 ret = amd_pstate_update_status(buf, p ? p - buf : count); in status_store()
1142 amd_perf_ctl_reset(policy->cpu); in amd_pstate_epp_cpu_init()
1143 dev = get_cpu_device(policy->cpu); in amd_pstate_epp_cpu_init()
1145 return -ENODEV; in amd_pstate_epp_cpu_init()
1149 return -ENOMEM; in amd_pstate_epp_cpu_init()
1151 cpudata->cpu = policy->cpu; in amd_pstate_epp_cpu_init()
1152 cpudata->epp_policy = 0; in amd_pstate_epp_cpu_init()
1165 ret = -EINVAL; in amd_pstate_epp_cpu_init()
1169 policy->cpuinfo.min_freq = min_freq; in amd_pstate_epp_cpu_init()
1170 policy->cpuinfo.max_freq = max_freq; in amd_pstate_epp_cpu_init()
1172 policy->cur = policy->cpuinfo.min_freq; in amd_pstate_epp_cpu_init()
1175 cpudata->max_freq = max_freq; in amd_pstate_epp_cpu_init()
1176 cpudata->min_freq = min_freq; in amd_pstate_epp_cpu_init()
1177 cpudata->nominal_freq = nominal_freq; in amd_pstate_epp_cpu_init()
1178 cpudata->lowest_nonlinear_freq = lowest_nonlinear_freq; in amd_pstate_epp_cpu_init()
1180 policy->driver_data = cpudata; in amd_pstate_epp_cpu_init()
1182 cpudata->epp_cached = amd_pstate_get_epp(cpudata, 0); in amd_pstate_epp_cpu_init()
1184 policy->min = policy->cpuinfo.min_freq; in amd_pstate_epp_cpu_init()
1185 policy->max = policy->cpuinfo.max_freq; in amd_pstate_epp_cpu_init()
1193 policy->policy = CPUFREQ_POLICY_PERFORMANCE; in amd_pstate_epp_cpu_init()
1195 policy->policy = CPUFREQ_POLICY_POWERSAVE; in amd_pstate_epp_cpu_init()
1198 ret = rdmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, &value); in amd_pstate_epp_cpu_init()
1201 WRITE_ONCE(cpudata->cppc_req_cached, value); in amd_pstate_epp_cpu_init()
1203 ret = rdmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_CAP1, &value); in amd_pstate_epp_cpu_init()
1206 WRITE_ONCE(cpudata->cppc_cap1_cached, value); in amd_pstate_epp_cpu_init()
1219 pr_debug("CPU %d exiting\n", policy->cpu); in amd_pstate_epp_cpu_exit()
1225 struct amd_cpudata *cpudata = policy->driver_data; in amd_pstate_epp_update_limit()
1230 max_perf = READ_ONCE(cpudata->highest_perf); in amd_pstate_epp_update_limit()
1231 min_perf = READ_ONCE(cpudata->lowest_perf); in amd_pstate_epp_update_limit()
1232 max_limit_perf = div_u64(policy->max * cpudata->highest_perf, cpudata->max_freq); in amd_pstate_epp_update_limit()
1233 min_limit_perf = div_u64(policy->min * cpudata->highest_perf, cpudata->max_freq); in amd_pstate_epp_update_limit()
1235 WRITE_ONCE(cpudata->max_limit_perf, max_limit_perf); in amd_pstate_epp_update_limit()
1236 WRITE_ONCE(cpudata->min_limit_perf, min_limit_perf); in amd_pstate_epp_update_limit()
1238 max_perf = clamp_t(unsigned long, max_perf, cpudata->min_limit_perf, in amd_pstate_epp_update_limit()
1239 cpudata->max_limit_perf); in amd_pstate_epp_update_limit()
1240 min_perf = clamp_t(unsigned long, min_perf, cpudata->min_limit_perf, in amd_pstate_epp_update_limit()
1241 cpudata->max_limit_perf); in amd_pstate_epp_update_limit()
1242 value = READ_ONCE(cpudata->cppc_req_cached); in amd_pstate_epp_update_limit()
1244 if (cpudata->policy == CPUFREQ_POLICY_PERFORMANCE) in amd_pstate_epp_update_limit()
1258 cpudata->epp_policy = cpudata->policy; in amd_pstate_epp_update_limit()
1260 /* Get BIOS pre-defined epp value */ in amd_pstate_epp_update_limit()
1270 if (cpudata->policy == CPUFREQ_POLICY_PERFORMANCE) in amd_pstate_epp_update_limit()
1279 WRITE_ONCE(cpudata->cppc_req_cached, value); in amd_pstate_epp_update_limit()
1285 struct amd_cpudata *cpudata = policy->driver_data; in amd_pstate_epp_set_policy()
1287 if (!policy->cpuinfo.max_freq) in amd_pstate_epp_set_policy()
1288 return -ENODEV; in amd_pstate_epp_set_policy()
1290 pr_debug("set_policy: cpuinfo.max %u policy->max %u\n", in amd_pstate_epp_set_policy()
1291 policy->cpuinfo.max_freq, policy->max); in amd_pstate_epp_set_policy()
1293 cpudata->policy = policy->policy; in amd_pstate_epp_set_policy()
1310 value = READ_ONCE(cpudata->cppc_req_cached); in amd_pstate_epp_reenable()
1311 max_perf = READ_ONCE(cpudata->highest_perf); in amd_pstate_epp_reenable()
1314 wrmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, value); in amd_pstate_epp_reenable()
1317 perf_ctrls.energy_perf = AMD_CPPC_ENERGY_PERF_PREF(cpudata->epp_cached); in amd_pstate_epp_reenable()
1318 cppc_set_perf(cpudata->cpu, &perf_ctrls); in amd_pstate_epp_reenable()
1324 struct amd_cpudata *cpudata = policy->driver_data; in amd_pstate_epp_cpu_online()
1326 pr_debug("AMD CPU Core %d going online\n", cpudata->cpu); in amd_pstate_epp_cpu_online()
1330 cpudata->suspended = false; in amd_pstate_epp_cpu_online()
1338 struct amd_cpudata *cpudata = policy->driver_data; in amd_pstate_epp_offline()
1343 min_perf = READ_ONCE(cpudata->lowest_perf); in amd_pstate_epp_offline()
1344 value = READ_ONCE(cpudata->cppc_req_cached); in amd_pstate_epp_offline()
1348 cpudata->epp_policy = CPUFREQ_POLICY_UNKNOWN; in amd_pstate_epp_offline()
1355 wrmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, value); in amd_pstate_epp_offline()
1360 cppc_set_perf(cpudata->cpu, &perf_ctrls); in amd_pstate_epp_offline()
1367 struct amd_cpudata *cpudata = policy->driver_data; in amd_pstate_epp_cpu_offline()
1369 pr_debug("AMD CPU Core %d going offline\n", cpudata->cpu); in amd_pstate_epp_cpu_offline()
1371 if (cpudata->suspended) in amd_pstate_epp_cpu_offline()
1383 pr_debug("policy_max =%d, policy_min=%d\n", policy->max, policy->min); in amd_pstate_epp_verify_policy()
1389 struct amd_cpudata *cpudata = policy->driver_data; in amd_pstate_epp_suspend()
1397 cpudata->suspended = true; in amd_pstate_epp_suspend()
1409 struct amd_cpudata *cpudata = policy->driver_data; in amd_pstate_epp_resume()
1411 if (cpudata->suspended) { in amd_pstate_epp_resume()
1419 cpudata->suspended = false; in amd_pstate_epp_resume()
1435 .name = "amd-pstate",
1449 .name = "amd-pstate-epp",
1469 return -EINVAL; in amd_pstate_set_driver()
1478 return -ENODEV; in amd_pstate_init()
1482 return -ENODEV; in amd_pstate_init()
1487 return -EEXIST; in amd_pstate_init()
1500 return -ENODEV; in amd_pstate_init()
1507 return -ENODEV; in amd_pstate_init()
1513 return -EINVAL; in amd_pstate_init()
1520 current_pstate_driver->adjust_perf = amd_pstate_adjust_perf; in amd_pstate_init()
1541 ret = sysfs_create_group(&dev_root->kobj, &amd_pstate_global_attr_group); in amd_pstate_init()
1563 return -EINVAL; in amd_pstate_param()
1573 MODULE_DESCRIPTION("AMD Processor P-state Frequency Driver");