1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Intel Uncore Frequency Setting
4 * Copyright (c) 2022, Intel Corporation.
5 * All rights reserved.
6 *
7 * Provide interface to set MSR 620 at a granularity of per die. On CPU online,
8 * one control CPU is identified per die to read/write limit. This control CPU
9 * is changed, if the CPU state is changed to offline. When the last CPU is
10 * offline in a die then remove the sysfs object for that die.
11 * The majority of actual code is related to sysfs create and read/write
12 * attributes.
13 *
14 * Author: Srinivas Pandruvada <srinivas.pandruvada@linux.intel.com>
15 */
16
17 #include <linux/bitfield.h>
18 #include <linux/cpu.h>
19 #include <linux/module.h>
20 #include <linux/slab.h>
21 #include <linux/suspend.h>
22 #include <asm/cpu_device_id.h>
23 #include <asm/intel-family.h>
24
25 #include "uncore-frequency-common.h"
26
27 /* Max instances for uncore data, one for each die */
28 static int uncore_max_entries __read_mostly;
29 /* Storage for uncore data for all instances */
30 static struct uncore_data *uncore_instances;
31 /* Stores the CPU mask of the target CPUs to use during uncore read/write */
32 static cpumask_t uncore_cpu_mask;
33 /* CPU online callback register instance */
34 static enum cpuhp_state uncore_hp_state __read_mostly;
35
36 #define MSR_UNCORE_RATIO_LIMIT 0x620
37 #define MSR_UNCORE_PERF_STATUS 0x621
38 #define UNCORE_FREQ_KHZ_MULTIPLIER 100000
39
40 #define UNCORE_MAX_RATIO_MASK GENMASK_ULL(6, 0)
41 #define UNCORE_MIN_RATIO_MASK GENMASK_ULL(14, 8)
42
43 #define UNCORE_CURRENT_RATIO_MASK GENMASK_ULL(6, 0)
44
uncore_read_control_freq(struct uncore_data * data,unsigned int * value,enum uncore_index index)45 static int uncore_read_control_freq(struct uncore_data *data, unsigned int *value,
46 enum uncore_index index)
47 {
48 u64 cap;
49 int ret;
50
51 if (data->control_cpu < 0)
52 return -ENXIO;
53
54 ret = rdmsrl_on_cpu(data->control_cpu, MSR_UNCORE_RATIO_LIMIT, &cap);
55 if (ret)
56 return ret;
57
58 if (index == UNCORE_INDEX_MAX_FREQ)
59 *value = FIELD_GET(UNCORE_MAX_RATIO_MASK, cap) * UNCORE_FREQ_KHZ_MULTIPLIER;
60 else
61 *value = FIELD_GET(UNCORE_MIN_RATIO_MASK, cap) * UNCORE_FREQ_KHZ_MULTIPLIER;
62
63 return 0;
64 }
65
uncore_write_control_freq(struct uncore_data * data,unsigned int input,enum uncore_index index)66 static int uncore_write_control_freq(struct uncore_data *data, unsigned int input,
67 enum uncore_index index)
68 {
69 int ret;
70 u64 cap;
71
72 input /= UNCORE_FREQ_KHZ_MULTIPLIER;
73 if (!input || input > FIELD_MAX(UNCORE_MAX_RATIO_MASK))
74 return -EINVAL;
75
76 if (data->control_cpu < 0)
77 return -ENXIO;
78
79 ret = rdmsrl_on_cpu(data->control_cpu, MSR_UNCORE_RATIO_LIMIT, &cap);
80 if (ret)
81 return ret;
82
83 if (index == UNCORE_INDEX_MAX_FREQ) {
84 cap &= ~UNCORE_MAX_RATIO_MASK;
85 cap |= FIELD_PREP(UNCORE_MAX_RATIO_MASK, input);
86 } else {
87 cap &= ~UNCORE_MIN_RATIO_MASK;
88 cap |= FIELD_PREP(UNCORE_MIN_RATIO_MASK, input);
89 }
90
91 ret = wrmsrl_on_cpu(data->control_cpu, MSR_UNCORE_RATIO_LIMIT, cap);
92 if (ret)
93 return ret;
94
95 data->stored_uncore_data = cap;
96
97 return 0;
98 }
99
uncore_read_freq(struct uncore_data * data,unsigned int * freq)100 static int uncore_read_freq(struct uncore_data *data, unsigned int *freq)
101 {
102 u64 ratio;
103 int ret;
104
105 if (data->control_cpu < 0)
106 return -ENXIO;
107
108 ret = rdmsrl_on_cpu(data->control_cpu, MSR_UNCORE_PERF_STATUS, &ratio);
109 if (ret)
110 return ret;
111
112 *freq = FIELD_GET(UNCORE_CURRENT_RATIO_MASK, ratio) * UNCORE_FREQ_KHZ_MULTIPLIER;
113
114 return 0;
115 }
116
uncore_read(struct uncore_data * data,unsigned int * value,enum uncore_index index)117 static int uncore_read(struct uncore_data *data, unsigned int *value, enum uncore_index index)
118 {
119 switch (index) {
120 case UNCORE_INDEX_MIN_FREQ:
121 case UNCORE_INDEX_MAX_FREQ:
122 return uncore_read_control_freq(data, value, index);
123
124 case UNCORE_INDEX_CURRENT_FREQ:
125 return uncore_read_freq(data, value);
126
127 default:
128 break;
129 }
130
131 return -EOPNOTSUPP;
132 }
133
134 /* Caller provides protection */
uncore_get_instance(unsigned int cpu)135 static struct uncore_data *uncore_get_instance(unsigned int cpu)
136 {
137 int id = topology_logical_die_id(cpu);
138
139 if (id >= 0 && id < uncore_max_entries)
140 return &uncore_instances[id];
141
142 return NULL;
143 }
144
uncore_event_cpu_online(unsigned int cpu)145 static int uncore_event_cpu_online(unsigned int cpu)
146 {
147 struct uncore_data *data;
148 int target;
149 int ret;
150
151 /* Check if there is an online cpu in the package for uncore MSR */
152 target = cpumask_any_and(&uncore_cpu_mask, topology_die_cpumask(cpu));
153 if (target < nr_cpu_ids)
154 return 0;
155
156 data = uncore_get_instance(cpu);
157 if (!data)
158 return 0;
159
160 data->package_id = topology_physical_package_id(cpu);
161 data->die_id = topology_die_id(cpu);
162 data->domain_id = UNCORE_DOMAIN_ID_INVALID;
163
164 ret = uncore_freq_add_entry(data, cpu);
165 if (ret)
166 return ret;
167
168 /* Use this CPU on this die as a control CPU */
169 cpumask_set_cpu(cpu, &uncore_cpu_mask);
170
171 return 0;
172 }
173
uncore_event_cpu_offline(unsigned int cpu)174 static int uncore_event_cpu_offline(unsigned int cpu)
175 {
176 struct uncore_data *data;
177 int target;
178
179 data = uncore_get_instance(cpu);
180 if (!data)
181 return 0;
182
183 /* Check if existing cpu is used for uncore MSRs */
184 if (!cpumask_test_and_clear_cpu(cpu, &uncore_cpu_mask))
185 return 0;
186
187 /* Find a new cpu to set uncore MSR */
188 target = cpumask_any_but(topology_die_cpumask(cpu), cpu);
189
190 if (target < nr_cpu_ids) {
191 cpumask_set_cpu(target, &uncore_cpu_mask);
192 uncore_freq_add_entry(data, target);
193 } else {
194 uncore_freq_remove_die_entry(data);
195 }
196
197 return 0;
198 }
199
uncore_pm_notify(struct notifier_block * nb,unsigned long mode,void * _unused)200 static int uncore_pm_notify(struct notifier_block *nb, unsigned long mode,
201 void *_unused)
202 {
203 int i;
204
205 switch (mode) {
206 case PM_POST_HIBERNATION:
207 case PM_POST_RESTORE:
208 case PM_POST_SUSPEND:
209 for (i = 0; i < uncore_max_entries; ++i) {
210 struct uncore_data *data = &uncore_instances[i];
211
212 if (!data || !data->valid || !data->stored_uncore_data)
213 return 0;
214
215 wrmsrl_on_cpu(data->control_cpu, MSR_UNCORE_RATIO_LIMIT,
216 data->stored_uncore_data);
217 }
218 break;
219 default:
220 break;
221 }
222 return 0;
223 }
224
225 static struct notifier_block uncore_pm_nb = {
226 .notifier_call = uncore_pm_notify,
227 };
228
229 static const struct x86_cpu_id intel_uncore_cpu_ids[] = {
230 X86_MATCH_VFM(INTEL_BROADWELL_G, NULL),
231 X86_MATCH_VFM(INTEL_BROADWELL_X, NULL),
232 X86_MATCH_VFM(INTEL_BROADWELL_D, NULL),
233 X86_MATCH_VFM(INTEL_SKYLAKE_X, NULL),
234 X86_MATCH_VFM(INTEL_ICELAKE_X, NULL),
235 X86_MATCH_VFM(INTEL_ICELAKE_D, NULL),
236 X86_MATCH_VFM(INTEL_SAPPHIRERAPIDS_X, NULL),
237 X86_MATCH_VFM(INTEL_EMERALDRAPIDS_X, NULL),
238 X86_MATCH_VFM(INTEL_KABYLAKE, NULL),
239 X86_MATCH_VFM(INTEL_KABYLAKE_L, NULL),
240 X86_MATCH_VFM(INTEL_COMETLAKE, NULL),
241 X86_MATCH_VFM(INTEL_COMETLAKE_L, NULL),
242 X86_MATCH_VFM(INTEL_CANNONLAKE_L, NULL),
243 X86_MATCH_VFM(INTEL_ICELAKE, NULL),
244 X86_MATCH_VFM(INTEL_ICELAKE_L, NULL),
245 X86_MATCH_VFM(INTEL_ROCKETLAKE, NULL),
246 X86_MATCH_VFM(INTEL_TIGERLAKE, NULL),
247 X86_MATCH_VFM(INTEL_TIGERLAKE_L, NULL),
248 X86_MATCH_VFM(INTEL_ALDERLAKE, NULL),
249 X86_MATCH_VFM(INTEL_ALDERLAKE_L, NULL),
250 X86_MATCH_VFM(INTEL_RAPTORLAKE, NULL),
251 X86_MATCH_VFM(INTEL_RAPTORLAKE_P, NULL),
252 X86_MATCH_VFM(INTEL_RAPTORLAKE_S, NULL),
253 X86_MATCH_VFM(INTEL_METEORLAKE, NULL),
254 X86_MATCH_VFM(INTEL_METEORLAKE_L, NULL),
255 X86_MATCH_VFM(INTEL_ARROWLAKE, NULL),
256 X86_MATCH_VFM(INTEL_ARROWLAKE_H, NULL),
257 X86_MATCH_VFM(INTEL_LUNARLAKE_M, NULL),
258 {}
259 };
260 MODULE_DEVICE_TABLE(x86cpu, intel_uncore_cpu_ids);
261
intel_uncore_init(void)262 static int __init intel_uncore_init(void)
263 {
264 const struct x86_cpu_id *id;
265 int ret;
266
267 if (cpu_feature_enabled(X86_FEATURE_HYPERVISOR))
268 return -ENODEV;
269
270 id = x86_match_cpu(intel_uncore_cpu_ids);
271 if (!id)
272 return -ENODEV;
273
274 uncore_max_entries = topology_max_packages() *
275 topology_max_dies_per_package();
276 uncore_instances = kcalloc(uncore_max_entries,
277 sizeof(*uncore_instances), GFP_KERNEL);
278 if (!uncore_instances)
279 return -ENOMEM;
280
281 ret = uncore_freq_common_init(uncore_read, uncore_write_control_freq);
282 if (ret)
283 goto err_free;
284
285 ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN,
286 "platform/x86/uncore-freq:online",
287 uncore_event_cpu_online,
288 uncore_event_cpu_offline);
289 if (ret < 0)
290 goto err_rem_kobj;
291
292 uncore_hp_state = ret;
293
294 ret = register_pm_notifier(&uncore_pm_nb);
295 if (ret)
296 goto err_rem_state;
297
298 return 0;
299
300 err_rem_state:
301 cpuhp_remove_state(uncore_hp_state);
302 err_rem_kobj:
303 uncore_freq_common_exit();
304 err_free:
305 kfree(uncore_instances);
306
307 return ret;
308 }
module_init(intel_uncore_init)309 module_init(intel_uncore_init)
310
311 static void __exit intel_uncore_exit(void)
312 {
313 int i;
314
315 unregister_pm_notifier(&uncore_pm_nb);
316 cpuhp_remove_state(uncore_hp_state);
317 for (i = 0; i < uncore_max_entries; ++i)
318 uncore_freq_remove_die_entry(&uncore_instances[i]);
319 uncore_freq_common_exit();
320 kfree(uncore_instances);
321 }
322 module_exit(intel_uncore_exit)
323
324 MODULE_IMPORT_NS("INTEL_UNCORE_FREQUENCY");
325 MODULE_LICENSE("GPL v2");
326 MODULE_DESCRIPTION("Intel Uncore Frequency Limits Driver");
327