230 static int msr_update_perf(struct cpufreq_policy *policy, u8 min_perf,  in msr_update_perf()  argument
233 	struct amd_cpudata *cpudata = policy->driver_data;  in msr_update_perf()
253 					  policy->boost_enabled,  in msr_update_perf()
277 static inline int amd_pstate_update_perf(struct cpufreq_policy *policy,  in amd_pstate_update_perf()  argument
282 	return static_call(amd_pstate_update_perf)(policy, min_perf, des_perf,  in amd_pstate_update_perf()
286 static int msr_set_epp(struct cpufreq_policy *policy, u8 epp)  in msr_set_epp()  argument
288 	struct amd_cpudata *cpudata = policy->driver_data;  in msr_set_epp()
305 					  policy->boost_enabled,  in msr_set_epp()
326 static inline int amd_pstate_set_epp(struct cpufreq_policy *policy, u8 epp)  in amd_pstate_set_epp()  argument
328 	return static_call(amd_pstate_set_epp)(policy, epp);  in amd_pstate_set_epp()
331 static int shmem_set_epp(struct cpufreq_policy *policy, u8 epp)  in shmem_set_epp()  argument
333 	struct amd_cpudata *cpudata = policy->driver_data;  in shmem_set_epp()
350 					  policy->boost_enabled,  in shmem_set_epp()
372 static inline int msr_cppc_enable(struct cpufreq_policy *policy)  in msr_cppc_enable()  argument
374 	return wrmsrl_safe_on_cpu(policy->cpu, MSR_AMD_CPPC_ENABLE, 1);  in msr_cppc_enable()
377 static int shmem_cppc_enable(struct cpufreq_policy *policy)  in shmem_cppc_enable()  argument
379 	return cppc_set_enable(policy->cpu, 1);  in shmem_cppc_enable()
384 static inline int amd_pstate_cppc_enable(struct cpufreq_policy *policy)  in amd_pstate_cppc_enable()  argument
386 	return static_call(amd_pstate_cppc_enable)(policy);  in amd_pstate_cppc_enable()
463 static int shmem_update_perf(struct cpufreq_policy *policy, u8 min_perf,  in shmem_update_perf()  argument
466 	struct amd_cpudata *cpudata = policy->driver_data;  in shmem_update_perf()
472 		int ret = shmem_set_epp(policy, epp);  in shmem_update_perf()
495 					  policy->boost_enabled,  in shmem_update_perf()
551 	struct cpufreq_policy *policy __free(put_cpufreq_policy) = cpufreq_cpu_get(cpudata->cpu);  in amd_pstate_update()
554 	if (!policy)  in amd_pstate_update()
559 	policy->cur = perf_to_freq(perf, cpudata->nominal_freq, des_perf);  in amd_pstate_update()
576 	amd_pstate_update_perf(policy, min_perf, des_perf, max_perf, 0, fast_switch);  in amd_pstate_update()
582 	 * Initialize lower frequency limit (i.e.policy->min) with  in amd_pstate_verify()
588 		struct cpufreq_policy *policy __free(put_cpufreq_policy) =  in amd_pstate_verify()
592 		if (!policy)  in amd_pstate_verify()
595 		cpudata = policy->driver_data;  in amd_pstate_verify()
604 static void amd_pstate_update_min_max_limit(struct cpufreq_policy *policy)  in amd_pstate_update_min_max_limit()  argument
606 	struct amd_cpudata *cpudata = policy->driver_data;  in amd_pstate_update_min_max_limit()
609 	perf.max_limit_perf = freq_to_perf(perf, cpudata->nominal_freq, policy->max);  in amd_pstate_update_min_max_limit()
610 	WRITE_ONCE(cpudata->max_limit_freq, policy->max);  in amd_pstate_update_min_max_limit()
612 	if (cpudata->policy == CPUFREQ_POLICY_PERFORMANCE) {  in amd_pstate_update_min_max_limit()
616 		perf.min_limit_perf = freq_to_perf(perf, cpudata->nominal_freq, policy->min);  in amd_pstate_update_min_max_limit()
617 		WRITE_ONCE(cpudata->min_limit_freq, policy->min);  in amd_pstate_update_min_max_limit()
623 static int amd_pstate_update_freq(struct cpufreq_policy *policy,  in amd_pstate_update_freq()  argument
631 	cpudata = policy->driver_data;  in amd_pstate_update_freq()
633 	if (policy->min != cpudata->min_limit_freq || policy->max != cpudata->max_limit_freq)  in amd_pstate_update_freq()
634 		amd_pstate_update_min_max_limit(policy);  in amd_pstate_update_freq()
638 	freqs.old = policy->cur;  in amd_pstate_update_freq()
643 	WARN_ON(fast_switch && !policy->fast_switch_enabled);  in amd_pstate_update_freq()
650 		cpufreq_freq_transition_begin(policy, &freqs);  in amd_pstate_update_freq()
654 			  policy->governor->flags);  in amd_pstate_update_freq()
657 		cpufreq_freq_transition_end(policy, &freqs, false);  in amd_pstate_update_freq()
662 static int amd_pstate_target(struct cpufreq_policy *policy,  in amd_pstate_target()  argument
666 	return amd_pstate_update_freq(policy, target_freq, false);  in amd_pstate_target()
669 static unsigned int amd_pstate_fast_switch(struct cpufreq_policy *policy,  in amd_pstate_fast_switch()  argument
672 	if (!amd_pstate_update_freq(policy, target_freq, true))  in amd_pstate_fast_switch()
674 	return policy->cur;  in amd_pstate_fast_switch()
683 	struct cpufreq_policy *policy __free(put_cpufreq_policy) = cpufreq_cpu_get(cpu);  in amd_pstate_adjust_perf()
687 	if (!policy)  in amd_pstate_adjust_perf()
690 	cpudata = policy->driver_data;  in amd_pstate_adjust_perf()
692 	if (policy->min != cpudata->min_limit_freq || policy->max != cpudata->max_limit_freq)  in amd_pstate_adjust_perf()
693 		amd_pstate_update_min_max_limit(policy);  in amd_pstate_adjust_perf()
715 			policy->governor->flags);  in amd_pstate_adjust_perf()
718 static int amd_pstate_cpu_boost_update(struct cpufreq_policy *policy, bool on)  in amd_pstate_cpu_boost_update()  argument
720 	struct amd_cpudata *cpudata = policy->driver_data;  in amd_pstate_cpu_boost_update()
729 		policy->cpuinfo.max_freq = max_freq;  in amd_pstate_cpu_boost_update()
730 	else if (policy->cpuinfo.max_freq > nominal_freq)  in amd_pstate_cpu_boost_update()
731 		policy->cpuinfo.max_freq = nominal_freq;  in amd_pstate_cpu_boost_update()
733 	policy->max = policy->cpuinfo.max_freq;  in amd_pstate_cpu_boost_update()
736 		ret = freq_qos_update_request(&cpudata->req[1], policy->cpuinfo.max_freq);  in amd_pstate_cpu_boost_update()
744 static int amd_pstate_set_boost(struct cpufreq_policy *policy, int state)  in amd_pstate_set_boost()  argument
746 	struct amd_cpudata *cpudata = policy->driver_data;  in amd_pstate_set_boost()
754 	ret = amd_pstate_cpu_boost_update(policy, state);  in amd_pstate_set_boost()
755 	refresh_frequency_limits(policy);  in amd_pstate_set_boost()
813 	struct cpufreq_policy *policy __free(put_cpufreq_policy) = cpufreq_cpu_get(cpu);  in amd_pstate_update_limits()
821 	if (!policy)  in amd_pstate_update_limits()
827 	cpudata = policy->driver_data;  in amd_pstate_update_limits()
935 static int amd_pstate_cpu_init(struct cpufreq_policy *policy)  in amd_pstate_cpu_init()  argument
946 	amd_perf_ctl_reset(policy->cpu);  in amd_pstate_cpu_init()
947 	dev = get_cpu_device(policy->cpu);  in amd_pstate_cpu_init()
955 	cpudata->cpu = policy->cpu;  in amd_pstate_cpu_init()
971 	policy->cpuinfo.transition_latency = amd_pstate_get_transition_latency(policy->cpu);  in amd_pstate_cpu_init()
972 	policy->transition_delay_us = amd_pstate_get_transition_delay_us(policy->cpu);  in amd_pstate_cpu_init()
976 	policy->cpuinfo.min_freq = policy->min = perf_to_freq(perf,  in amd_pstate_cpu_init()
979 	policy->cpuinfo.max_freq = policy->max = perf_to_freq(perf,  in amd_pstate_cpu_init()
983 	ret = amd_pstate_cppc_enable(policy);  in amd_pstate_cpu_init()
987 	policy->boost_supported = READ_ONCE(cpudata->boost_supported);  in amd_pstate_cpu_init()
990 	policy->cur = policy->cpuinfo.min_freq;  in amd_pstate_cpu_init()
993 		policy->fast_switch_possible = true;  in amd_pstate_cpu_init()
995 	ret = freq_qos_add_request(&policy->constraints, &cpudata->req[0],  in amd_pstate_cpu_init()
1002 	ret = freq_qos_add_request(&policy->constraints, &cpudata->req[1],  in amd_pstate_cpu_init()
1003 				   FREQ_QOS_MAX, policy->cpuinfo.max_freq);  in amd_pstate_cpu_init()
1009 	policy->driver_data = cpudata;  in amd_pstate_cpu_init()
1019 	pr_warn("Failed to initialize CPU %d: %d\n", policy->cpu, ret);  in amd_pstate_cpu_init()
1024 static void amd_pstate_cpu_exit(struct cpufreq_policy *policy)  in amd_pstate_cpu_exit()  argument
1026 	struct amd_cpudata *cpudata = policy->driver_data;  in amd_pstate_cpu_exit()
1030 	policy->fast_switch_possible = false;  in amd_pstate_cpu_exit()
1041 static ssize_t show_amd_pstate_max_freq(struct cpufreq_policy *policy,  in show_amd_pstate_max_freq()  argument
1047 	cpudata = policy->driver_data;  in show_amd_pstate_max_freq()
1054 static ssize_t show_amd_pstate_lowest_nonlinear_freq(struct cpufreq_policy *policy,  in show_amd_pstate_lowest_nonlinear_freq()  argument
1060 	cpudata = policy->driver_data;  in show_amd_pstate_lowest_nonlinear_freq()
1071 static ssize_t show_amd_pstate_highest_perf(struct cpufreq_policy *policy,  in show_amd_pstate_highest_perf()  argument
1076 	cpudata = policy->driver_data;  in show_amd_pstate_highest_perf()
1081 static ssize_t show_amd_pstate_prefcore_ranking(struct cpufreq_policy *policy,  in show_amd_pstate_prefcore_ranking()  argument
1085 	struct amd_cpudata *cpudata = policy->driver_data;  in show_amd_pstate_prefcore_ranking()
1092 static ssize_t show_amd_pstate_hw_prefcore(struct cpufreq_policy *policy,  in show_amd_pstate_hw_prefcore()  argument
1096 	struct amd_cpudata *cpudata = policy->driver_data;  in show_amd_pstate_hw_prefcore()
1104 				struct cpufreq_policy *policy, char *buf)  in show_energy_performance_available_preferences()  argument
1108 	struct amd_cpudata *cpudata = policy->driver_data;  in show_energy_performance_available_preferences()
1110 	if (cpudata->policy == CPUFREQ_POLICY_PERFORMANCE)  in show_energy_performance_available_preferences()
1123 		struct cpufreq_policy *policy, const char *buf, size_t count)  in store_energy_performance_preference()  argument
1125 	struct amd_cpudata *cpudata = policy->driver_data;  in store_energy_performance_preference()
1143 	if (epp > 0 && policy->policy == CPUFREQ_POLICY_PERFORMANCE) {  in store_energy_performance_preference()
1144 		pr_debug("EPP cannot be set under performance policy\n");  in store_energy_performance_preference()
1148 	ret = amd_pstate_set_epp(policy, epp);  in store_energy_performance_preference()
1154 				struct cpufreq_policy *policy, char *buf)  in show_energy_performance_preference()  argument
1156 	struct amd_cpudata *cpudata = policy->driver_data;  in show_energy_performance_preference()
1417 static int amd_pstate_epp_cpu_init(struct cpufreq_policy *policy)  in amd_pstate_epp_cpu_init()  argument
1429 	amd_perf_ctl_reset(policy->cpu);  in amd_pstate_epp_cpu_init()
1430 	dev = get_cpu_device(policy->cpu);  in amd_pstate_epp_cpu_init()
1438 	cpudata->cpu = policy->cpu;  in amd_pstate_epp_cpu_init()
1456 	policy->cpuinfo.min_freq = policy->min = perf_to_freq(perf,  in amd_pstate_epp_cpu_init()
1459 	policy->cpuinfo.max_freq = policy->max = perf_to_freq(perf,  in amd_pstate_epp_cpu_init()
1462 	policy->driver_data = cpudata;  in amd_pstate_epp_cpu_init()
1464 	ret = amd_pstate_cppc_enable(policy);  in amd_pstate_epp_cpu_init()
1469 	policy->cur = policy->cpuinfo.min_freq;  in amd_pstate_epp_cpu_init()
1472 	policy->boost_supported = READ_ONCE(cpudata->boost_supported);  in amd_pstate_epp_cpu_init()
1475 	 * Set the policy to provide a valid fallback value in case  in amd_pstate_epp_cpu_init()
1480 		policy->policy = CPUFREQ_POLICY_PERFORMANCE;  in amd_pstate_epp_cpu_init()
1483 		policy->policy = CPUFREQ_POLICY_POWERSAVE;  in amd_pstate_epp_cpu_init()
1493 	ret = amd_pstate_set_epp(policy, cpudata->epp_default);  in amd_pstate_epp_cpu_init()
1502 	pr_warn("Failed to initialize CPU %d: %d\n", policy->cpu, ret);  in amd_pstate_epp_cpu_init()
1507 static void amd_pstate_epp_cpu_exit(struct cpufreq_policy *policy)  in amd_pstate_epp_cpu_exit()  argument
1509 	struct amd_cpudata *cpudata = policy->driver_data;  in amd_pstate_epp_cpu_exit()
1513 		policy->driver_data = NULL;  in amd_pstate_epp_cpu_exit()
1516 	pr_debug("CPU %d exiting\n", policy->cpu);  in amd_pstate_epp_cpu_exit()
1519 static int amd_pstate_epp_update_limit(struct cpufreq_policy *policy)  in amd_pstate_epp_update_limit()  argument
1521 	struct amd_cpudata *cpudata = policy->driver_data;  in amd_pstate_epp_update_limit()
1525 	if (policy->min != cpudata->min_limit_freq || policy->max != cpudata->max_limit_freq)  in amd_pstate_epp_update_limit()
1526 		amd_pstate_update_min_max_limit(policy);  in amd_pstate_epp_update_limit()
1528 	if (cpudata->policy == CPUFREQ_POLICY_PERFORMANCE)  in amd_pstate_epp_update_limit()
1535 	return amd_pstate_update_perf(policy, perf.min_limit_perf, 0U,  in amd_pstate_epp_update_limit()
1539 static int amd_pstate_epp_set_policy(struct cpufreq_policy *policy)  in amd_pstate_epp_set_policy()  argument
1541 	struct amd_cpudata *cpudata = policy->driver_data;  in amd_pstate_epp_set_policy()
1544 	if (!policy->cpuinfo.max_freq)  in amd_pstate_epp_set_policy()
1547 	cpudata->policy = policy->policy;  in amd_pstate_epp_set_policy()
1549 	ret = amd_pstate_epp_update_limit(policy);  in amd_pstate_epp_set_policy()
1554 	 * policy->cur is never updated with the amd_pstate_epp driver, but it  in amd_pstate_epp_set_policy()
1557 	policy->cur = policy->min;  in amd_pstate_epp_set_policy()
1562 static int amd_pstate_epp_cpu_online(struct cpufreq_policy *policy)  in amd_pstate_epp_cpu_online()  argument
1564 	pr_debug("AMD CPU Core %d going online\n", policy->cpu);  in amd_pstate_epp_cpu_online()
1566 	return amd_pstate_cppc_enable(policy);  in amd_pstate_epp_cpu_online()
1569 static int amd_pstate_epp_cpu_offline(struct cpufreq_policy *policy)  in amd_pstate_epp_cpu_offline()  argument
1574 static int amd_pstate_epp_suspend(struct cpufreq_policy *policy)  in amd_pstate_epp_suspend()  argument
1576 	struct amd_cpudata *cpudata = policy->driver_data;  in amd_pstate_epp_suspend()
1587 static int amd_pstate_epp_resume(struct cpufreq_policy *policy)  in amd_pstate_epp_resume()  argument
1589 	struct amd_cpudata *cpudata = policy->driver_data;  in amd_pstate_epp_resume()
1595 		ret = amd_pstate_epp_update_limit(policy);  in amd_pstate_epp_resume()