1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Intel Uncore Frequency Setting 4 * Copyright (c) 2022, Intel Corporation. 5 * All rights reserved. 6 * 7 * Provide interface to set MSR 620 at a granularity of per die. On CPU online, 8 * one control CPU is identified per die to read/write limit. This control CPU 9 * is changed, if the CPU state is changed to offline. When the last CPU is 10 * offline in a die then remove the sysfs object for that die. 11 * The majority of actual code is related to sysfs create and read/write 12 * attributes. 13 * 14 * Author: Srinivas Pandruvada <srinivas.pandruvada@linux.intel.com> 15 */ 16 17 #include <linux/bitfield.h> 18 #include <linux/cpu.h> 19 #include <linux/module.h> 20 #include <linux/slab.h> 21 #include <linux/suspend.h> 22 #include <asm/cpu_device_id.h> 23 #include <asm/intel-family.h> 24 #include <asm/msr.h> 25 26 #include "uncore-frequency-common.h" 27 28 /* Max instances for uncore data, one for each die */ 29 static int uncore_max_entries __read_mostly; 30 /* Storage for uncore data for all instances */ 31 static struct uncore_data *uncore_instances; 32 /* Stores the CPU mask of the target CPUs to use during uncore read/write */ 33 static cpumask_t uncore_cpu_mask; 34 /* CPU online callback register instance */ 35 static enum cpuhp_state uncore_hp_state __read_mostly; 36 37 #define MSR_UNCORE_RATIO_LIMIT 0x620 38 #define MSR_UNCORE_PERF_STATUS 0x621 39 #define UNCORE_FREQ_KHZ_MULTIPLIER 100000 40 41 #define UNCORE_MAX_RATIO_MASK GENMASK_ULL(6, 0) 42 #define UNCORE_MIN_RATIO_MASK GENMASK_ULL(14, 8) 43 44 #define UNCORE_CURRENT_RATIO_MASK GENMASK_ULL(6, 0) 45 46 static int uncore_read_control_freq(struct uncore_data *data, unsigned int *value, 47 enum uncore_index index) 48 { 49 u64 cap; 50 int ret; 51 52 if (data->control_cpu < 0) 53 return -ENXIO; 54 55 ret = rdmsrq_on_cpu(data->control_cpu, MSR_UNCORE_RATIO_LIMIT, &cap); 56 if (ret) 57 return ret; 58 59 if (index == UNCORE_INDEX_MAX_FREQ) 60 *value = FIELD_GET(UNCORE_MAX_RATIO_MASK, cap) * UNCORE_FREQ_KHZ_MULTIPLIER; 61 else 62 *value = FIELD_GET(UNCORE_MIN_RATIO_MASK, cap) * UNCORE_FREQ_KHZ_MULTIPLIER; 63 64 return 0; 65 } 66 67 static int uncore_write_control_freq(struct uncore_data *data, unsigned int input, 68 enum uncore_index index) 69 { 70 int ret; 71 u64 cap; 72 73 input /= UNCORE_FREQ_KHZ_MULTIPLIER; 74 if (!input || input > FIELD_MAX(UNCORE_MAX_RATIO_MASK)) 75 return -EINVAL; 76 77 if (data->control_cpu < 0) 78 return -ENXIO; 79 80 ret = rdmsrq_on_cpu(data->control_cpu, MSR_UNCORE_RATIO_LIMIT, &cap); 81 if (ret) 82 return ret; 83 84 if (index == UNCORE_INDEX_MAX_FREQ) { 85 cap &= ~UNCORE_MAX_RATIO_MASK; 86 cap |= FIELD_PREP(UNCORE_MAX_RATIO_MASK, input); 87 } else { 88 cap &= ~UNCORE_MIN_RATIO_MASK; 89 cap |= FIELD_PREP(UNCORE_MIN_RATIO_MASK, input); 90 } 91 92 ret = wrmsrq_on_cpu(data->control_cpu, MSR_UNCORE_RATIO_LIMIT, cap); 93 if (ret) 94 return ret; 95 96 data->stored_uncore_data = cap; 97 98 return 0; 99 } 100 101 static int uncore_read_freq(struct uncore_data *data, unsigned int *freq) 102 { 103 u64 ratio; 104 int ret; 105 106 if (data->control_cpu < 0) 107 return -ENXIO; 108 109 ret = rdmsrq_on_cpu(data->control_cpu, MSR_UNCORE_PERF_STATUS, &ratio); 110 if (ret) 111 return ret; 112 113 *freq = FIELD_GET(UNCORE_CURRENT_RATIO_MASK, ratio) * UNCORE_FREQ_KHZ_MULTIPLIER; 114 115 return 0; 116 } 117 118 static int uncore_read(struct uncore_data *data, unsigned int *value, enum uncore_index index) 119 { 120 switch (index) { 121 case UNCORE_INDEX_MIN_FREQ: 122 case UNCORE_INDEX_MAX_FREQ: 123 return uncore_read_control_freq(data, value, index); 124 125 case UNCORE_INDEX_CURRENT_FREQ: 126 return uncore_read_freq(data, value); 127 128 default: 129 break; 130 } 131 132 return -EOPNOTSUPP; 133 } 134 135 /* Caller provides protection */ 136 static struct uncore_data *uncore_get_instance(unsigned int cpu) 137 { 138 int id = topology_logical_die_id(cpu); 139 140 if (id >= 0 && id < uncore_max_entries) 141 return &uncore_instances[id]; 142 143 return NULL; 144 } 145 146 static int uncore_event_cpu_online(unsigned int cpu) 147 { 148 struct uncore_data *data; 149 int target; 150 int ret; 151 152 /* Check if there is an online cpu in the package for uncore MSR */ 153 target = cpumask_any_and(&uncore_cpu_mask, topology_die_cpumask(cpu)); 154 if (target < nr_cpu_ids) 155 return 0; 156 157 data = uncore_get_instance(cpu); 158 if (!data) 159 return 0; 160 161 data->package_id = topology_physical_package_id(cpu); 162 data->die_id = topology_die_id(cpu); 163 data->domain_id = UNCORE_DOMAIN_ID_INVALID; 164 165 ret = uncore_freq_add_entry(data, cpu); 166 if (ret) 167 return ret; 168 169 /* Use this CPU on this die as a control CPU */ 170 cpumask_set_cpu(cpu, &uncore_cpu_mask); 171 172 return 0; 173 } 174 175 static int uncore_event_cpu_offline(unsigned int cpu) 176 { 177 struct uncore_data *data; 178 int target; 179 180 data = uncore_get_instance(cpu); 181 if (!data) 182 return 0; 183 184 /* Check if existing cpu is used for uncore MSRs */ 185 if (!cpumask_test_and_clear_cpu(cpu, &uncore_cpu_mask)) 186 return 0; 187 188 /* Find a new cpu to set uncore MSR */ 189 target = cpumask_any_but(topology_die_cpumask(cpu), cpu); 190 191 if (target < nr_cpu_ids) { 192 cpumask_set_cpu(target, &uncore_cpu_mask); 193 uncore_freq_add_entry(data, target); 194 } else { 195 uncore_freq_remove_die_entry(data); 196 } 197 198 return 0; 199 } 200 201 static int uncore_pm_notify(struct notifier_block *nb, unsigned long mode, 202 void *_unused) 203 { 204 int i; 205 206 switch (mode) { 207 case PM_POST_HIBERNATION: 208 case PM_POST_RESTORE: 209 case PM_POST_SUSPEND: 210 for (i = 0; i < uncore_max_entries; ++i) { 211 struct uncore_data *data = &uncore_instances[i]; 212 213 if (!data || !data->valid || !data->stored_uncore_data) 214 return 0; 215 216 wrmsrq_on_cpu(data->control_cpu, MSR_UNCORE_RATIO_LIMIT, 217 data->stored_uncore_data); 218 } 219 break; 220 default: 221 break; 222 } 223 return 0; 224 } 225 226 static struct notifier_block uncore_pm_nb = { 227 .notifier_call = uncore_pm_notify, 228 }; 229 230 static const struct x86_cpu_id intel_uncore_cpu_ids[] = { 231 X86_MATCH_VFM(INTEL_BROADWELL_G, NULL), 232 X86_MATCH_VFM(INTEL_BROADWELL_X, NULL), 233 X86_MATCH_VFM(INTEL_BROADWELL_D, NULL), 234 X86_MATCH_VFM(INTEL_SKYLAKE_X, NULL), 235 X86_MATCH_VFM(INTEL_ICELAKE_X, NULL), 236 X86_MATCH_VFM(INTEL_ICELAKE_D, NULL), 237 X86_MATCH_VFM(INTEL_SAPPHIRERAPIDS_X, NULL), 238 X86_MATCH_VFM(INTEL_EMERALDRAPIDS_X, NULL), 239 X86_MATCH_VFM(INTEL_KABYLAKE, NULL), 240 X86_MATCH_VFM(INTEL_KABYLAKE_L, NULL), 241 X86_MATCH_VFM(INTEL_COMETLAKE, NULL), 242 X86_MATCH_VFM(INTEL_COMETLAKE_L, NULL), 243 X86_MATCH_VFM(INTEL_CANNONLAKE_L, NULL), 244 X86_MATCH_VFM(INTEL_ICELAKE, NULL), 245 X86_MATCH_VFM(INTEL_ICELAKE_L, NULL), 246 X86_MATCH_VFM(INTEL_ROCKETLAKE, NULL), 247 X86_MATCH_VFM(INTEL_TIGERLAKE, NULL), 248 X86_MATCH_VFM(INTEL_TIGERLAKE_L, NULL), 249 X86_MATCH_VFM(INTEL_ALDERLAKE, NULL), 250 X86_MATCH_VFM(INTEL_ALDERLAKE_L, NULL), 251 X86_MATCH_VFM(INTEL_RAPTORLAKE, NULL), 252 X86_MATCH_VFM(INTEL_RAPTORLAKE_P, NULL), 253 X86_MATCH_VFM(INTEL_RAPTORLAKE_S, NULL), 254 X86_MATCH_VFM(INTEL_METEORLAKE, NULL), 255 X86_MATCH_VFM(INTEL_METEORLAKE_L, NULL), 256 X86_MATCH_VFM(INTEL_ARROWLAKE, NULL), 257 X86_MATCH_VFM(INTEL_ARROWLAKE_H, NULL), 258 X86_MATCH_VFM(INTEL_LUNARLAKE_M, NULL), 259 {} 260 }; 261 MODULE_DEVICE_TABLE(x86cpu, intel_uncore_cpu_ids); 262 263 static int __init intel_uncore_init(void) 264 { 265 const struct x86_cpu_id *id; 266 int ret; 267 268 if (cpu_feature_enabled(X86_FEATURE_HYPERVISOR)) 269 return -ENODEV; 270 271 id = x86_match_cpu(intel_uncore_cpu_ids); 272 if (!id) 273 return -ENODEV; 274 275 uncore_max_entries = topology_max_packages() * 276 topology_max_dies_per_package(); 277 uncore_instances = kcalloc(uncore_max_entries, 278 sizeof(*uncore_instances), GFP_KERNEL); 279 if (!uncore_instances) 280 return -ENOMEM; 281 282 ret = uncore_freq_common_init(uncore_read, uncore_write_control_freq); 283 if (ret) 284 goto err_free; 285 286 ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, 287 "platform/x86/uncore-freq:online", 288 uncore_event_cpu_online, 289 uncore_event_cpu_offline); 290 if (ret < 0) 291 goto err_rem_kobj; 292 293 uncore_hp_state = ret; 294 295 ret = register_pm_notifier(&uncore_pm_nb); 296 if (ret) 297 goto err_rem_state; 298 299 return 0; 300 301 err_rem_state: 302 cpuhp_remove_state(uncore_hp_state); 303 err_rem_kobj: 304 uncore_freq_common_exit(); 305 err_free: 306 kfree(uncore_instances); 307 308 return ret; 309 } 310 module_init(intel_uncore_init) 311 312 static void __exit intel_uncore_exit(void) 313 { 314 int i; 315 316 unregister_pm_notifier(&uncore_pm_nb); 317 cpuhp_remove_state(uncore_hp_state); 318 for (i = 0; i < uncore_max_entries; ++i) 319 uncore_freq_remove_die_entry(&uncore_instances[i]); 320 uncore_freq_common_exit(); 321 kfree(uncore_instances); 322 } 323 module_exit(intel_uncore_exit) 324 325 MODULE_IMPORT_NS("INTEL_UNCORE_FREQUENCY"); 326 MODULE_LICENSE("GPL v2"); 327 MODULE_DESCRIPTION("Intel Uncore Frequency Limits Driver"); 328