1 /* 2 * arch/arm64/kernel/topology.c 3 * 4 * Copyright (C) 2011,2013,2014 Linaro Limited. 5 * 6 * Based on the arm32 version written by Vincent Guittot in turn based on 7 * arch/sh/kernel/topology.c 8 * 9 * This file is subject to the terms and conditions of the GNU General Public 10 * License. See the file "COPYING" in the main directory of this archive 11 * for more details. 12 */ 13 14 #include <linux/acpi.h> 15 #include <linux/arch_topology.h> 16 #include <linux/cacheinfo.h> 17 #include <linux/cpufreq.h> 18 #include <linux/cpu_smt.h> 19 #include <linux/init.h> 20 #include <linux/percpu.h> 21 #include <linux/sched/isolation.h> 22 #include <linux/xarray.h> 23 24 #include <asm/cpu.h> 25 #include <asm/cputype.h> 26 #include <asm/topology.h> 27 28 #ifdef CONFIG_ACPI 29 static bool __init acpi_cpu_is_threaded(int cpu) 30 { 31 int is_threaded = acpi_pptt_cpu_is_thread(cpu); 32 33 /* 34 * if the PPTT doesn't have thread information, assume a homogeneous 35 * machine and return the current CPU's thread state. 36 */ 37 if (is_threaded < 0) 38 is_threaded = read_cpuid_mpidr() & MPIDR_MT_BITMASK; 39 40 return !!is_threaded; 41 } 42 43 struct cpu_smt_info { 44 unsigned int thread_num; 45 int core_id; 46 }; 47 48 /* 49 * Propagate the topology information of the processor_topology_node tree to the 50 * cpu_topology array. 51 */ 52 int __init parse_acpi_topology(void) 53 { 54 unsigned int max_smt_thread_num = 1; 55 struct cpu_smt_info *entry; 56 struct xarray hetero_cpu; 57 unsigned long hetero_id; 58 int cpu, topology_id; 59 60 if (acpi_disabled) 61 return 0; 62 63 xa_init(&hetero_cpu); 64 65 for_each_possible_cpu(cpu) { 66 topology_id = find_acpi_cpu_topology(cpu, 0); 67 if (topology_id < 0) 68 return topology_id; 69 70 if (acpi_cpu_is_threaded(cpu)) { 71 cpu_topology[cpu].thread_id = topology_id; 72 topology_id = find_acpi_cpu_topology(cpu, 1); 73 cpu_topology[cpu].core_id = topology_id; 74 75 /* 76 * In the PPTT, CPUs below a node with the 'identical 77 * implementation' flag have the same number of threads. 78 * Count the number of threads for only one CPU (i.e. 79 * one core_id) among those with the same hetero_id. 80 * See the comment of find_acpi_cpu_topology_hetero_id() 81 * for more details. 82 * 83 * One entry is created for each node having: 84 * - the 'identical implementation' flag 85 * - its parent not having the flag 86 */ 87 hetero_id = find_acpi_cpu_topology_hetero_id(cpu); 88 entry = xa_load(&hetero_cpu, hetero_id); 89 if (!entry) { 90 entry = kzalloc(sizeof(*entry), GFP_KERNEL); 91 WARN_ON_ONCE(!entry); 92 93 if (entry) { 94 entry->core_id = topology_id; 95 entry->thread_num = 1; 96 xa_store(&hetero_cpu, hetero_id, 97 entry, GFP_KERNEL); 98 } 99 } else if (entry->core_id == topology_id) { 100 entry->thread_num++; 101 } 102 } else { 103 cpu_topology[cpu].thread_id = -1; 104 cpu_topology[cpu].core_id = topology_id; 105 } 106 topology_id = find_acpi_cpu_topology_cluster(cpu); 107 cpu_topology[cpu].cluster_id = topology_id; 108 topology_id = find_acpi_cpu_topology_package(cpu); 109 cpu_topology[cpu].package_id = topology_id; 110 } 111 112 /* 113 * This is a short loop since the number of XArray elements is the 114 * number of heterogeneous CPU clusters. On a homogeneous system 115 * there's only one entry in the XArray. 116 */ 117 xa_for_each(&hetero_cpu, hetero_id, entry) { 118 max_smt_thread_num = max(max_smt_thread_num, entry->thread_num); 119 xa_erase(&hetero_cpu, hetero_id); 120 kfree(entry); 121 } 122 123 cpu_smt_set_num_threads(max_smt_thread_num, max_smt_thread_num); 124 xa_destroy(&hetero_cpu); 125 return 0; 126 } 127 #endif 128 129 #ifdef CONFIG_ARM64_AMU_EXTN 130 #define read_corecnt() read_sysreg_s(SYS_AMEVCNTR0_CORE_EL0) 131 #define read_constcnt() read_sysreg_s(SYS_AMEVCNTR0_CONST_EL0) 132 #else 133 #define read_corecnt() (0UL) 134 #define read_constcnt() (0UL) 135 #endif 136 137 #undef pr_fmt 138 #define pr_fmt(fmt) "AMU: " fmt 139 140 /* 141 * Ensure that amu_scale_freq_tick() will return SCHED_CAPACITY_SCALE until 142 * the CPU capacity and its associated frequency have been correctly 143 * initialized. 144 */ 145 static DEFINE_PER_CPU_READ_MOSTLY(unsigned long, arch_max_freq_scale) = 1UL << (2 * SCHED_CAPACITY_SHIFT); 146 static cpumask_var_t amu_fie_cpus; 147 148 struct amu_cntr_sample { 149 u64 arch_const_cycles_prev; 150 u64 arch_core_cycles_prev; 151 unsigned long last_scale_update; 152 }; 153 154 static DEFINE_PER_CPU_SHARED_ALIGNED(struct amu_cntr_sample, cpu_amu_samples); 155 156 void update_freq_counters_refs(void) 157 { 158 struct amu_cntr_sample *amu_sample = this_cpu_ptr(&cpu_amu_samples); 159 160 amu_sample->arch_core_cycles_prev = read_corecnt(); 161 amu_sample->arch_const_cycles_prev = read_constcnt(); 162 } 163 164 static inline bool freq_counters_valid(int cpu) 165 { 166 struct amu_cntr_sample *amu_sample = per_cpu_ptr(&cpu_amu_samples, cpu); 167 168 if ((cpu >= nr_cpu_ids) || !cpumask_test_cpu(cpu, cpu_present_mask)) 169 return false; 170 171 if (!cpu_has_amu_feat(cpu)) { 172 pr_debug("CPU%d: counters are not supported.\n", cpu); 173 return false; 174 } 175 176 if (unlikely(!amu_sample->arch_const_cycles_prev || 177 !amu_sample->arch_core_cycles_prev)) { 178 pr_debug("CPU%d: cycle counters are not enabled.\n", cpu); 179 return false; 180 } 181 182 return true; 183 } 184 185 void freq_inv_set_max_ratio(int cpu, u64 max_rate) 186 { 187 u64 ratio, ref_rate = arch_timer_get_rate(); 188 189 if (unlikely(!max_rate || !ref_rate)) { 190 WARN_ONCE(1, "CPU%d: invalid maximum or reference frequency.\n", 191 cpu); 192 return; 193 } 194 195 /* 196 * Pre-compute the fixed ratio between the frequency of the constant 197 * reference counter and the maximum frequency of the CPU. 198 * 199 * ref_rate 200 * arch_max_freq_scale = ---------- * SCHED_CAPACITY_SCALE² 201 * max_rate 202 * 203 * We use a factor of 2 * SCHED_CAPACITY_SHIFT -> SCHED_CAPACITY_SCALE² 204 * in order to ensure a good resolution for arch_max_freq_scale for 205 * very low reference frequencies (down to the KHz range which should 206 * be unlikely). 207 */ 208 ratio = ref_rate << (2 * SCHED_CAPACITY_SHIFT); 209 ratio = div64_u64(ratio, max_rate); 210 if (!ratio) { 211 WARN_ONCE(1, "Reference frequency too low.\n"); 212 return; 213 } 214 215 WRITE_ONCE(per_cpu(arch_max_freq_scale, cpu), (unsigned long)ratio); 216 } 217 218 static void amu_scale_freq_tick(void) 219 { 220 struct amu_cntr_sample *amu_sample = this_cpu_ptr(&cpu_amu_samples); 221 u64 prev_core_cnt, prev_const_cnt; 222 u64 core_cnt, const_cnt, scale; 223 224 prev_const_cnt = amu_sample->arch_const_cycles_prev; 225 prev_core_cnt = amu_sample->arch_core_cycles_prev; 226 227 update_freq_counters_refs(); 228 229 const_cnt = amu_sample->arch_const_cycles_prev; 230 core_cnt = amu_sample->arch_core_cycles_prev; 231 232 /* 233 * This should not happen unless the AMUs have been reset and the 234 * counter values have not been restored - unlikely 235 */ 236 if (unlikely(core_cnt <= prev_core_cnt || 237 const_cnt <= prev_const_cnt)) 238 return; 239 240 /* 241 * /\core arch_max_freq_scale 242 * scale = ------- * -------------------- 243 * /\const SCHED_CAPACITY_SCALE 244 * 245 * See validate_cpu_freq_invariance_counters() for details on 246 * arch_max_freq_scale and the use of SCHED_CAPACITY_SHIFT. 247 */ 248 scale = core_cnt - prev_core_cnt; 249 scale *= this_cpu_read(arch_max_freq_scale); 250 scale = div64_u64(scale >> SCHED_CAPACITY_SHIFT, 251 const_cnt - prev_const_cnt); 252 253 scale = min_t(unsigned long, scale, SCHED_CAPACITY_SCALE); 254 this_cpu_write(arch_freq_scale, (unsigned long)scale); 255 256 amu_sample->last_scale_update = jiffies; 257 } 258 259 static struct scale_freq_data amu_sfd = { 260 .source = SCALE_FREQ_SOURCE_ARCH, 261 .set_freq_scale = amu_scale_freq_tick, 262 }; 263 264 static __always_inline bool amu_fie_cpu_supported(unsigned int cpu) 265 { 266 return cpumask_available(amu_fie_cpus) && 267 cpumask_test_cpu(cpu, amu_fie_cpus); 268 } 269 270 void arch_cpu_idle_enter(void) 271 { 272 unsigned int cpu = smp_processor_id(); 273 274 if (!amu_fie_cpu_supported(cpu)) 275 return; 276 277 /* Kick in AMU update but only if one has not happened already */ 278 if (housekeeping_cpu(cpu, HK_TYPE_TICK) && 279 time_is_before_jiffies(per_cpu(cpu_amu_samples.last_scale_update, cpu))) 280 amu_scale_freq_tick(); 281 } 282 283 #define AMU_SAMPLE_EXP_MS 20 284 285 int arch_freq_get_on_cpu(int cpu) 286 { 287 struct amu_cntr_sample *amu_sample; 288 unsigned int start_cpu = cpu; 289 unsigned long last_update; 290 unsigned int freq = 0; 291 u64 scale; 292 293 if (!amu_fie_cpu_supported(cpu) || !arch_scale_freq_ref(cpu)) 294 return -EOPNOTSUPP; 295 296 while (1) { 297 298 amu_sample = per_cpu_ptr(&cpu_amu_samples, cpu); 299 300 last_update = amu_sample->last_scale_update; 301 302 /* 303 * For those CPUs that are in full dynticks mode, or those that have 304 * not seen tick for a while, try an alternative source for the counters 305 * (and thus freq scale), if available, for given policy: this boils 306 * down to identifying an active cpu within the same freq domain, if any. 307 */ 308 if (!housekeeping_cpu(cpu, HK_TYPE_TICK) || 309 time_is_before_jiffies(last_update + msecs_to_jiffies(AMU_SAMPLE_EXP_MS))) { 310 struct cpufreq_policy *policy = cpufreq_cpu_get(cpu); 311 int ref_cpu; 312 313 if (!policy) 314 return -EINVAL; 315 316 if (!cpumask_intersects(policy->related_cpus, 317 housekeeping_cpumask(HK_TYPE_TICK))) { 318 cpufreq_cpu_put(policy); 319 return -EOPNOTSUPP; 320 } 321 322 for_each_cpu_wrap(ref_cpu, policy->cpus, cpu + 1) { 323 if (ref_cpu == start_cpu) { 324 /* Prevent verifying same CPU twice */ 325 ref_cpu = nr_cpu_ids; 326 break; 327 } 328 if (!idle_cpu(ref_cpu)) 329 break; 330 } 331 332 cpufreq_cpu_put(policy); 333 334 if (ref_cpu >= nr_cpu_ids) 335 /* No alternative to pull info from */ 336 return -EAGAIN; 337 338 cpu = ref_cpu; 339 } else { 340 break; 341 } 342 } 343 /* 344 * Reversed computation to the one used to determine 345 * the arch_freq_scale value 346 * (see amu_scale_freq_tick for details) 347 */ 348 scale = arch_scale_freq_capacity(cpu); 349 freq = scale * arch_scale_freq_ref(cpu); 350 freq >>= SCHED_CAPACITY_SHIFT; 351 return freq; 352 } 353 354 static void amu_fie_setup(const struct cpumask *cpus) 355 { 356 int cpu; 357 358 /* We are already set since the last insmod of cpufreq driver */ 359 if (cpumask_available(amu_fie_cpus) && 360 unlikely(cpumask_subset(cpus, amu_fie_cpus))) 361 return; 362 363 for_each_cpu(cpu, cpus) 364 if (!freq_counters_valid(cpu)) 365 return; 366 367 if (!cpumask_available(amu_fie_cpus) && 368 !zalloc_cpumask_var(&amu_fie_cpus, GFP_KERNEL)) { 369 WARN_ONCE(1, "Failed to allocate FIE cpumask for CPUs[%*pbl]\n", 370 cpumask_pr_args(cpus)); 371 return; 372 } 373 374 cpumask_or(amu_fie_cpus, amu_fie_cpus, cpus); 375 376 topology_set_scale_freq_source(&amu_sfd, amu_fie_cpus); 377 378 pr_debug("CPUs[%*pbl]: counters will be used for FIE.", 379 cpumask_pr_args(cpus)); 380 } 381 382 static int init_amu_fie_callback(struct notifier_block *nb, unsigned long val, 383 void *data) 384 { 385 struct cpufreq_policy *policy = data; 386 387 if (val == CPUFREQ_CREATE_POLICY) 388 amu_fie_setup(policy->related_cpus); 389 390 /* 391 * We don't need to handle CPUFREQ_REMOVE_POLICY event as the AMU 392 * counters don't have any dependency on cpufreq driver once we have 393 * initialized AMU support and enabled invariance. The AMU counters will 394 * keep on working just fine in the absence of the cpufreq driver, and 395 * for the CPUs for which there are no counters available, the last set 396 * value of arch_freq_scale will remain valid as that is the frequency 397 * those CPUs are running at. 398 */ 399 400 return 0; 401 } 402 403 static struct notifier_block init_amu_fie_notifier = { 404 .notifier_call = init_amu_fie_callback, 405 }; 406 407 static int __init init_amu_fie(void) 408 { 409 return cpufreq_register_notifier(&init_amu_fie_notifier, 410 CPUFREQ_POLICY_NOTIFIER); 411 } 412 core_initcall(init_amu_fie); 413 414 #ifdef CONFIG_ACPI_CPPC_LIB 415 #include <acpi/cppc_acpi.h> 416 417 static void cpu_read_corecnt(void *val) 418 { 419 /* 420 * A value of 0 can be returned if the current CPU does not support AMUs 421 * or if the counter is disabled for this CPU. A return value of 0 at 422 * counter read is properly handled as an error case by the users of the 423 * counter. 424 */ 425 *(u64 *)val = read_corecnt(); 426 } 427 428 static void cpu_read_constcnt(void *val) 429 { 430 /* 431 * Return 0 if the current CPU is affected by erratum 2457168. A value 432 * of 0 is also returned if the current CPU does not support AMUs or if 433 * the counter is disabled. A return value of 0 at counter read is 434 * properly handled as an error case by the users of the counter. 435 */ 436 *(u64 *)val = this_cpu_has_cap(ARM64_WORKAROUND_2457168) ? 437 0UL : read_constcnt(); 438 } 439 440 static inline 441 int counters_read_on_cpu(int cpu, smp_call_func_t func, u64 *val) 442 { 443 /* 444 * Abort call on counterless CPU or when interrupts are 445 * disabled - can lead to deadlock in smp sync call. 446 */ 447 if (!cpu_has_amu_feat(cpu)) 448 return -EOPNOTSUPP; 449 450 if (WARN_ON_ONCE(irqs_disabled())) 451 return -EPERM; 452 453 smp_call_function_single(cpu, func, val, 1); 454 455 return 0; 456 } 457 458 /* 459 * Refer to drivers/acpi/cppc_acpi.c for the description of the functions 460 * below. 461 */ 462 bool cpc_ffh_supported(void) 463 { 464 int cpu = get_cpu_with_amu_feat(); 465 466 /* 467 * FFH is considered supported if there is at least one present CPU that 468 * supports AMUs. Using FFH to read core and reference counters for CPUs 469 * that do not support AMUs, have counters disabled or that are affected 470 * by errata, will result in a return value of 0. 471 * 472 * This is done to allow any enabled and valid counters to be read 473 * through FFH, knowing that potentially returning 0 as counter value is 474 * properly handled by the users of these counters. 475 */ 476 if ((cpu >= nr_cpu_ids) || !cpumask_test_cpu(cpu, cpu_present_mask)) 477 return false; 478 479 return true; 480 } 481 482 int cpc_read_ffh(int cpu, struct cpc_reg *reg, u64 *val) 483 { 484 int ret = -EOPNOTSUPP; 485 486 switch ((u64)reg->address) { 487 case 0x0: 488 ret = counters_read_on_cpu(cpu, cpu_read_corecnt, val); 489 break; 490 case 0x1: 491 ret = counters_read_on_cpu(cpu, cpu_read_constcnt, val); 492 break; 493 } 494 495 if (!ret) { 496 *val &= GENMASK_ULL(reg->bit_offset + reg->bit_width - 1, 497 reg->bit_offset); 498 *val >>= reg->bit_offset; 499 } 500 501 return ret; 502 } 503 504 int cpc_write_ffh(int cpunum, struct cpc_reg *reg, u64 val) 505 { 506 return -EOPNOTSUPP; 507 } 508 #endif /* CONFIG_ACPI_CPPC_LIB */ 509