1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * PSCI CPU idle driver. 4 * 5 * Copyright (C) 2019 ARM Ltd. 6 * Author: Lorenzo Pieralisi <lorenzo.pieralisi@arm.com> 7 */ 8 9 #define pr_fmt(fmt) "CPUidle PSCI: " fmt 10 11 #include <linux/cpuhotplug.h> 12 #include <linux/cpu_cooling.h> 13 #include <linux/cpuidle.h> 14 #include <linux/cpumask.h> 15 #include <linux/cpu_pm.h> 16 #include <linux/kernel.h> 17 #include <linux/module.h> 18 #include <linux/of.h> 19 #include <linux/platform_device.h> 20 #include <linux/psci.h> 21 #include <linux/pm_domain.h> 22 #include <linux/pm_runtime.h> 23 #include <linux/slab.h> 24 #include <linux/string.h> 25 #include <linux/syscore_ops.h> 26 27 #include <asm/cpuidle.h> 28 #include <trace/events/power.h> 29 30 #include "cpuidle-psci.h" 31 #include "dt_idle_states.h" 32 #include "dt_idle_genpd.h" 33 34 struct psci_cpuidle_data { 35 u32 *psci_states; 36 struct device *dev; 37 }; 38 39 static DEFINE_PER_CPU_READ_MOSTLY(struct psci_cpuidle_data, psci_cpuidle_data); 40 static DEFINE_PER_CPU(u32, domain_state); 41 static bool psci_cpuidle_use_syscore; 42 static bool psci_cpuidle_use_cpuhp; 43 44 void psci_set_domain_state(u32 state) 45 { 46 __this_cpu_write(domain_state, state); 47 } 48 49 static inline u32 psci_get_domain_state(void) 50 { 51 return __this_cpu_read(domain_state); 52 } 53 54 static __cpuidle int __psci_enter_domain_idle_state(struct cpuidle_device *dev, 55 struct cpuidle_driver *drv, int idx, 56 bool s2idle) 57 { 58 struct psci_cpuidle_data *data = this_cpu_ptr(&psci_cpuidle_data); 59 u32 *states = data->psci_states; 60 struct device *pd_dev = data->dev; 61 u32 state; 62 int ret; 63 64 ret = cpu_pm_enter(); 65 if (ret) 66 return -1; 67 68 /* Do runtime PM to manage a hierarchical CPU toplogy. */ 69 if (s2idle) 70 dev_pm_genpd_suspend(pd_dev); 71 else 72 pm_runtime_put_sync_suspend(pd_dev); 73 74 state = psci_get_domain_state(); 75 if (!state) 76 state = states[idx]; 77 78 trace_psci_domain_idle_enter(dev->cpu, state, s2idle); 79 ret = psci_cpu_suspend_enter(state) ? -1 : idx; 80 trace_psci_domain_idle_exit(dev->cpu, state, s2idle); 81 82 if (s2idle) 83 dev_pm_genpd_resume(pd_dev); 84 else 85 pm_runtime_get_sync(pd_dev); 86 87 cpu_pm_exit(); 88 89 /* Clear the domain state to start fresh when back from idle. */ 90 psci_set_domain_state(0); 91 return ret; 92 } 93 94 static int psci_enter_domain_idle_state(struct cpuidle_device *dev, 95 struct cpuidle_driver *drv, int idx) 96 { 97 return __psci_enter_domain_idle_state(dev, drv, idx, false); 98 } 99 100 static int psci_enter_s2idle_domain_idle_state(struct cpuidle_device *dev, 101 struct cpuidle_driver *drv, 102 int idx) 103 { 104 return __psci_enter_domain_idle_state(dev, drv, idx, true); 105 } 106 107 static int psci_idle_cpuhp_up(unsigned int cpu) 108 { 109 struct device *pd_dev = __this_cpu_read(psci_cpuidle_data.dev); 110 111 if (pd_dev) 112 pm_runtime_get_sync(pd_dev); 113 114 return 0; 115 } 116 117 static int psci_idle_cpuhp_down(unsigned int cpu) 118 { 119 struct device *pd_dev = __this_cpu_read(psci_cpuidle_data.dev); 120 121 if (pd_dev) { 122 pm_runtime_put_sync(pd_dev); 123 /* Clear domain state to start fresh at next online. */ 124 psci_set_domain_state(0); 125 } 126 127 return 0; 128 } 129 130 static void psci_idle_syscore_switch(bool suspend) 131 { 132 bool cleared = false; 133 struct device *dev; 134 int cpu; 135 136 for_each_possible_cpu(cpu) { 137 dev = per_cpu_ptr(&psci_cpuidle_data, cpu)->dev; 138 139 if (dev && suspend) { 140 dev_pm_genpd_suspend(dev); 141 } else if (dev) { 142 dev_pm_genpd_resume(dev); 143 144 /* Account for userspace having offlined a CPU. */ 145 if (pm_runtime_status_suspended(dev)) 146 pm_runtime_set_active(dev); 147 148 /* Clear domain state to re-start fresh. */ 149 if (!cleared) { 150 psci_set_domain_state(0); 151 cleared = true; 152 } 153 } 154 } 155 } 156 157 static int psci_idle_syscore_suspend(void) 158 { 159 psci_idle_syscore_switch(true); 160 return 0; 161 } 162 163 static void psci_idle_syscore_resume(void) 164 { 165 psci_idle_syscore_switch(false); 166 } 167 168 static struct syscore_ops psci_idle_syscore_ops = { 169 .suspend = psci_idle_syscore_suspend, 170 .resume = psci_idle_syscore_resume, 171 }; 172 173 static void psci_idle_init_syscore(void) 174 { 175 if (psci_cpuidle_use_syscore) 176 register_syscore_ops(&psci_idle_syscore_ops); 177 } 178 179 static void psci_idle_init_cpuhp(void) 180 { 181 int err; 182 183 if (!psci_cpuidle_use_cpuhp) 184 return; 185 186 err = cpuhp_setup_state_nocalls(CPUHP_AP_CPU_PM_STARTING, 187 "cpuidle/psci:online", 188 psci_idle_cpuhp_up, 189 psci_idle_cpuhp_down); 190 if (err) 191 pr_warn("Failed %d while setup cpuhp state\n", err); 192 } 193 194 static __cpuidle int psci_enter_idle_state(struct cpuidle_device *dev, 195 struct cpuidle_driver *drv, int idx) 196 { 197 u32 *state = __this_cpu_read(psci_cpuidle_data.psci_states); 198 199 return CPU_PM_CPU_IDLE_ENTER_PARAM_RCU(psci_cpu_suspend_enter, idx, state[idx]); 200 } 201 202 static const struct of_device_id psci_idle_state_match[] = { 203 { .compatible = "arm,idle-state", 204 .data = psci_enter_idle_state }, 205 { }, 206 }; 207 208 int psci_dt_parse_state_node(struct device_node *np, u32 *state) 209 { 210 int err = of_property_read_u32(np, "arm,psci-suspend-param", state); 211 212 if (err) { 213 pr_warn("%pOF missing arm,psci-suspend-param property\n", np); 214 return err; 215 } 216 217 if (!psci_power_state_is_valid(*state)) { 218 pr_warn("Invalid PSCI power state %#x\n", *state); 219 return -EINVAL; 220 } 221 222 return 0; 223 } 224 225 static int psci_dt_cpu_init_topology(struct cpuidle_driver *drv, 226 struct psci_cpuidle_data *data, 227 unsigned int state_count, int cpu) 228 { 229 /* Currently limit the hierarchical topology to be used in OSI mode. */ 230 if (!psci_has_osi_support()) 231 return 0; 232 233 data->dev = dt_idle_attach_cpu(cpu, "psci"); 234 if (IS_ERR_OR_NULL(data->dev)) 235 return PTR_ERR_OR_ZERO(data->dev); 236 237 psci_cpuidle_use_syscore = true; 238 239 /* 240 * Using the deepest state for the CPU to trigger a potential selection 241 * of a shared state for the domain, assumes the domain states are all 242 * deeper states. On PREEMPT_RT the hierarchical topology is limited to 243 * s2ram and s2idle. 244 */ 245 drv->states[state_count - 1].enter_s2idle = psci_enter_s2idle_domain_idle_state; 246 if (!IS_ENABLED(CONFIG_PREEMPT_RT)) { 247 drv->states[state_count - 1].enter = psci_enter_domain_idle_state; 248 psci_cpuidle_use_cpuhp = true; 249 } 250 251 return 0; 252 } 253 254 static int psci_dt_cpu_init_idle(struct device *dev, struct cpuidle_driver *drv, 255 struct device_node *cpu_node, 256 unsigned int state_count, int cpu) 257 { 258 int i, ret = 0; 259 u32 *psci_states; 260 struct device_node *state_node; 261 struct psci_cpuidle_data *data = per_cpu_ptr(&psci_cpuidle_data, cpu); 262 263 state_count++; /* Add WFI state too */ 264 psci_states = devm_kcalloc(dev, state_count, sizeof(*psci_states), 265 GFP_KERNEL); 266 if (!psci_states) 267 return -ENOMEM; 268 269 for (i = 1; i < state_count; i++) { 270 state_node = of_get_cpu_state_node(cpu_node, i - 1); 271 if (!state_node) 272 break; 273 274 ret = psci_dt_parse_state_node(state_node, &psci_states[i]); 275 of_node_put(state_node); 276 277 if (ret) 278 return ret; 279 280 pr_debug("psci-power-state %#x index %d\n", psci_states[i], i); 281 } 282 283 if (i != state_count) 284 return -ENODEV; 285 286 /* Initialize optional data, used for the hierarchical topology. */ 287 ret = psci_dt_cpu_init_topology(drv, data, state_count, cpu); 288 if (ret < 0) 289 return ret; 290 291 /* Idle states parsed correctly, store them in the per-cpu struct. */ 292 data->psci_states = psci_states; 293 return 0; 294 } 295 296 static int psci_cpu_init_idle(struct device *dev, struct cpuidle_driver *drv, 297 unsigned int cpu, unsigned int state_count) 298 { 299 struct device_node *cpu_node; 300 int ret; 301 302 /* 303 * If the PSCI cpu_suspend function hook has not been initialized 304 * idle states must not be enabled, so bail out 305 */ 306 if (!psci_ops.cpu_suspend) 307 return -EOPNOTSUPP; 308 309 cpu_node = of_cpu_device_node_get(cpu); 310 if (!cpu_node) 311 return -ENODEV; 312 313 ret = psci_dt_cpu_init_idle(dev, drv, cpu_node, state_count, cpu); 314 315 of_node_put(cpu_node); 316 317 return ret; 318 } 319 320 static void psci_cpu_deinit_idle(int cpu) 321 { 322 struct psci_cpuidle_data *data = per_cpu_ptr(&psci_cpuidle_data, cpu); 323 324 dt_idle_detach_cpu(data->dev); 325 psci_cpuidle_use_syscore = false; 326 psci_cpuidle_use_cpuhp = false; 327 } 328 329 static int psci_idle_init_cpu(struct device *dev, int cpu) 330 { 331 struct cpuidle_driver *drv; 332 struct device_node *cpu_node; 333 const char *enable_method; 334 int ret = 0; 335 336 cpu_node = of_cpu_device_node_get(cpu); 337 if (!cpu_node) 338 return -ENODEV; 339 340 /* 341 * Check whether the enable-method for the cpu is PSCI, fail 342 * if it is not. 343 */ 344 enable_method = of_get_property(cpu_node, "enable-method", NULL); 345 if (!enable_method || (strcmp(enable_method, "psci"))) 346 ret = -ENODEV; 347 348 of_node_put(cpu_node); 349 if (ret) 350 return ret; 351 352 drv = devm_kzalloc(dev, sizeof(*drv), GFP_KERNEL); 353 if (!drv) 354 return -ENOMEM; 355 356 drv->name = "psci_idle"; 357 drv->owner = THIS_MODULE; 358 drv->cpumask = (struct cpumask *)cpumask_of(cpu); 359 360 /* 361 * PSCI idle states relies on architectural WFI to be represented as 362 * state index 0. 363 */ 364 drv->states[0].enter = psci_enter_idle_state; 365 drv->states[0].exit_latency = 1; 366 drv->states[0].target_residency = 1; 367 drv->states[0].power_usage = UINT_MAX; 368 strcpy(drv->states[0].name, "WFI"); 369 strcpy(drv->states[0].desc, "ARM WFI"); 370 371 /* 372 * If no DT idle states are detected (ret == 0) let the driver 373 * initialization fail accordingly since there is no reason to 374 * initialize the idle driver if only wfi is supported, the 375 * default archictectural back-end already executes wfi 376 * on idle entry. 377 */ 378 ret = dt_init_idle_driver(drv, psci_idle_state_match, 1); 379 if (ret <= 0) 380 return ret ? : -ENODEV; 381 382 /* 383 * Initialize PSCI idle states. 384 */ 385 ret = psci_cpu_init_idle(dev, drv, cpu, ret); 386 if (ret) { 387 pr_err("CPU %d failed to PSCI idle\n", cpu); 388 return ret; 389 } 390 391 ret = cpuidle_register(drv, NULL); 392 if (ret) 393 goto deinit; 394 395 cpuidle_cooling_register(drv); 396 397 return 0; 398 deinit: 399 psci_cpu_deinit_idle(cpu); 400 return ret; 401 } 402 403 /* 404 * psci_idle_probe - Initializes PSCI cpuidle driver 405 * 406 * Initializes PSCI cpuidle driver for all present CPUs, if any CPU fails 407 * to register cpuidle driver then rollback to cancel all CPUs 408 * registration. 409 */ 410 static int psci_cpuidle_probe(struct platform_device *pdev) 411 { 412 int cpu, ret; 413 struct cpuidle_driver *drv; 414 struct cpuidle_device *dev; 415 416 for_each_present_cpu(cpu) { 417 ret = psci_idle_init_cpu(&pdev->dev, cpu); 418 if (ret) 419 goto out_fail; 420 } 421 422 psci_idle_init_syscore(); 423 psci_idle_init_cpuhp(); 424 return 0; 425 426 out_fail: 427 while (--cpu >= 0) { 428 dev = per_cpu(cpuidle_devices, cpu); 429 drv = cpuidle_get_cpu_driver(dev); 430 cpuidle_unregister(drv); 431 psci_cpu_deinit_idle(cpu); 432 } 433 434 return ret; 435 } 436 437 static struct platform_driver psci_cpuidle_driver = { 438 .probe = psci_cpuidle_probe, 439 .driver = { 440 .name = "psci-cpuidle", 441 }, 442 }; 443 444 static int __init psci_idle_init(void) 445 { 446 struct platform_device *pdev; 447 int ret; 448 449 ret = platform_driver_register(&psci_cpuidle_driver); 450 if (ret) 451 return ret; 452 453 pdev = platform_device_register_simple("psci-cpuidle", -1, NULL, 0); 454 if (IS_ERR(pdev)) { 455 platform_driver_unregister(&psci_cpuidle_driver); 456 return PTR_ERR(pdev); 457 } 458 459 return 0; 460 } 461 device_initcall(psci_idle_init); 462