Lines Matching +full:idle +full:- +full:state +full:- +full:name
1 // SPDX-License-Identifier: GPL-2.0-only
3 * intel_idle.c - native hardware idle loop for modern Intel processors
5 * Copyright (c) 2013 - 2020, Intel Corporation.
20 * All CPUs have same idle states as boot CPU
23 * for preventing entry into deep C-states
25 * CPU will flush caches as needed when entering a C-state via MWAIT
33 * ACPI has a .suspend hack to turn off deep c-statees during suspend
39 /* un-comment DEBUG to enable pr_debug() statements */
56 #include <asm/intel-family.h>
58 #include <asm/spec-ctrl.h>
66 .name = "intel_idle",
70 static int max_cstate = CPUIDLE_STATE_MAX - 1;
90 * Hardware C-state auto-demotion may not always be optimal.
104 * Enable interrupts before entering the C-state. On some platforms and for
105 * some C-states, this may measurably decrease interrupt latency.
110 * Enable this state by default even if the ACPI _CST does not list it.
115 * Disable IBRS across idle (when KERNEL_IBRS), is exclusive vs IRQ_ENABLE
121 * Initialize large xstate for the C6-state entrance.
126 * Ignore the sub-state when matching mwait hints between the ACPI _CST and
132 * MWAIT takes an 8-bit "hint" in EAX "suggesting"
133 * the C-state (top nibble) and sub-state (bottom nibble)
136 * We store the hint at the top of our "flags" for each state.
145 struct cpuidle_state *state = &drv->states[index]; in __intel_idle() local
146 unsigned long eax = flg2MWAIT(state->flags); in __intel_idle()
155 * intel_idle - Ask the processor to enter the given idle state.
158 * @index: Target idle state index.
161 * @dev is idle and it can try to enter the idle state corresponding to @index.
163 * If the local APIC timer is not known to be reliable in the target idle state,
164 * enable one-shot tick broadcasting for the target CPU before executing MWAIT.
206 * intel_idle_s2idle - Ask the processor to enter the given idle state.
209 * @index: Target idle state index.
212 * @dev is idle and it can try to enter the idle state corresponding to @index.
214 * Invoked as a suspend-to-idle callback routine with frozen user space, frozen
221 struct cpuidle_state *state = &drv->states[index]; in intel_idle_s2idle() local
222 unsigned long eax = flg2MWAIT(state->flags); in intel_idle_s2idle()
224 if (state->flags & CPUIDLE_FLAG_INIT_XSTATE) in intel_idle_s2idle()
235 struct cpuidle_state *state = &drv->states[index]; in intel_idle_enter_dead() local
236 unsigned long eax = flg2MWAIT(state->flags); in intel_idle_enter_dead()
248 .name = "C1",
256 .name = "C1E",
264 .name = "C3",
272 .name = "C6",
285 .name = "C1",
293 .name = "C1E",
301 .name = "C3",
309 .name = "C6",
317 .name = "C7",
330 .name = "C1",
338 .name = "C6N",
346 .name = "C6S",
354 .name = "C7",
362 .name = "C7S",
375 .name = "C1",
383 .name = "C6N",
391 .name = "C6S",
399 .name = "C7",
407 .name = "C7S",
420 .name = "C1",
428 .name = "C1E",
436 .name = "C3",
444 .name = "C6",
452 .name = "C7",
465 .name = "C1",
473 .name = "C1E",
481 .name = "C3",
489 .name = "C6",
502 .name = "C1",
510 .name = "C1E",
518 .name = "C3",
526 .name = "C6",
539 .name = "C1",
547 .name = "C1E",
555 .name = "C3",
563 .name = "C6",
576 .name = "C1",
584 .name = "C1E",
592 .name = "C3",
600 .name = "C6",
608 .name = "C7s",
616 .name = "C8",
624 .name = "C9",
632 .name = "C10",
644 .name = "C1",
652 .name = "C1E",
660 .name = "C3",
668 .name = "C6",
676 .name = "C7s",
684 .name = "C8",
692 .name = "C9",
700 .name = "C10",
713 .name = "C1",
721 .name = "C1E",
729 .name = "C3",
737 .name = "C6",
745 .name = "C7s",
753 .name = "C8",
761 .name = "C9",
769 .name = "C10",
782 .name = "C1",
790 .name = "C1E",
798 .name = "C6",
811 .name = "C1",
819 .name = "C1E",
827 .name = "C6",
850 .name = "C1",
858 .name = "C1E",
866 .name = "C6",
874 .name = "C8",
882 .name = "C10",
895 .name = "C1",
903 .name = "C1E",
911 .name = "C6",
919 .name = "C8",
927 .name = "C10",
940 .name = "C1E",
948 .name = "C6",
956 .name = "C10",
969 .name = "C1",
977 .name = "C1E",
985 .name = "C6",
993 .name = "C8",
1001 .name = "C10",
1014 .name = "C1",
1022 .name = "C1E",
1030 .name = "C6",
1044 .name = "C1",
1052 .name = "C1E",
1060 .name = "C6",
1070 .name = "C6P",
1085 .name = "C1",
1093 .name = "C1E",
1101 .name = "C6",
1111 .name = "C6P",
1126 .name = "C1E",
1134 .name = "C2",
1142 .name = "C4",
1150 .name = "C6",
1162 .name = "C1",
1170 .name = "C4",
1178 .name = "C6",
1186 .name = "C7",
1194 .name = "C9",
1206 .name = "C1",
1214 .name = "C6",
1226 .name = "C1",
1234 .name = "C6",
1247 .name = "C1",
1255 .name = "C1E",
1263 .name = "C6",
1271 .name = "C7s",
1279 .name = "C8",
1287 .name = "C9",
1295 .name = "C10",
1308 .name = "C1",
1316 .name = "C1E",
1324 .name = "C6",
1341 .name = "C1",
1349 .name = "C1E",
1357 .name = "C6",
1370 .name = "C1",
1378 .name = "C1E",
1386 .name = "C6S",
1399 .name = "C1",
1407 .name = "C1E",
1415 .name = "C6S",
1424 .name = "C6SP",
1681 static bool __init intel_idle_state_needs_timer_stop(struct cpuidle_state *state) in intel_idle_state_needs_timer_stop() argument
1683 unsigned long eax = flg2MWAIT(state->flags); in intel_idle_state_needs_timer_stop()
1689 * Switch over to one-shot tick broadcast if the target C-state in intel_idle_state_needs_timer_stop()
1700 MODULE_PARM_DESC(no_acpi, "Do not use ACPI _CST for building the idle states list");
1704 MODULE_PARM_DESC(use_acpi, "Use ACPI _CST for building the idle states list");
1708 MODULE_PARM_DESC(no_native, "Ignore cpu specific (native) idle states in lieu of ACPI idle states");
1713 * intel_idle_cst_usable - Check if the _CST information can be used.
1715 * Check if all of the C-states listed by _CST in the max_cstate range are
1728 if (cx->entry_method != ACPI_CSTATE_FFH) in intel_idle_cst_usable()
1750 if (acpi_processor_evaluate_cst(pr->handle, cpu, &acpi_state_table)) in intel_idle_acpi_cst_extract()
1779 struct cpuidle_state *state; in intel_idle_init_cstates_acpi() local
1781 if (intel_idle_max_cstate_reached(cstate - 1)) in intel_idle_init_cstates_acpi()
1786 state = &drv->states[drv->state_count++]; in intel_idle_init_cstates_acpi()
1788 snprintf(state->name, CPUIDLE_NAME_LEN, "C%d_ACPI", cstate); in intel_idle_init_cstates_acpi()
1789 strscpy(state->desc, cx->desc, CPUIDLE_DESC_LEN); in intel_idle_init_cstates_acpi()
1790 state->exit_latency = cx->latency; in intel_idle_init_cstates_acpi()
1792 * For C1-type C-states use the same number for both the exit in intel_idle_init_cstates_acpi()
1794 * C1 in the majority of the static C-states tables above. in intel_idle_init_cstates_acpi()
1795 * For the other types of C-states, however, set the target in intel_idle_init_cstates_acpi()
1797 * a reasonable balance between energy-efficiency and in intel_idle_init_cstates_acpi()
1800 state->target_residency = cx->latency; in intel_idle_init_cstates_acpi()
1801 if (cx->type > ACPI_STATE_C1) in intel_idle_init_cstates_acpi()
1802 state->target_residency *= 3; in intel_idle_init_cstates_acpi()
1804 state->flags = MWAIT2flg(cx->address); in intel_idle_init_cstates_acpi()
1805 if (cx->type > ACPI_STATE_C2) in intel_idle_init_cstates_acpi()
1806 state->flags |= CPUIDLE_FLAG_TLB_FLUSHED; in intel_idle_init_cstates_acpi()
1809 state->flags |= CPUIDLE_FLAG_OFF; in intel_idle_init_cstates_acpi()
1811 if (intel_idle_state_needs_timer_stop(state)) in intel_idle_init_cstates_acpi()
1812 state->flags |= CPUIDLE_FLAG_TIMER_STOP; in intel_idle_init_cstates_acpi()
1814 if (cx->type > ACPI_STATE_C1 && !boot_cpu_has(X86_FEATURE_NONSTOP_TSC)) in intel_idle_init_cstates_acpi()
1815 mark_tsc_unstable("TSC halts in idle"); in intel_idle_init_cstates_acpi()
1817 state->enter = intel_idle; in intel_idle_init_cstates_acpi()
1818 state->enter_dead = intel_idle_enter_dead; in intel_idle_init_cstates_acpi()
1819 state->enter_s2idle = intel_idle_s2idle; in intel_idle_init_cstates_acpi()
1828 * If there are no _CST C-states, do not disable any C-states by in intel_idle_off_by_default()
1871 * ivt_idle_state_table_update - Tune the idle states table for Ivy Town.
1873 * Tune IVT multi-socket targets.
1878 /* IVT uses a different table for 1-2, 3-4, and > 4 sockets */ in ivt_idle_state_table_update()
1900 * irtl_2_usec - IRTL to microseconds conversion.
1921 * bxt_idle_state_table_update - Fix up the Broxton idle states table.
1969 * sklh_idle_state_table_update - Fix up the Sky Lake idle states table.
1971 * On SKL-H (model 0x5e) skip C8 and C9 if C10 is enabled and SGX disabled.
1989 /* PC10 is not enabled in PKG C-state limit */ in sklh_idle_state_table_update()
2006 skl_cstates[5].flags |= CPUIDLE_FLAG_UNUSABLE; /* C8-SKL */ in sklh_idle_state_table_update()
2007 skl_cstates[6].flags |= CPUIDLE_FLAG_UNUSABLE; /* C9-SKL */ in sklh_idle_state_table_update()
2011 * skx_idle_state_table_update - Adjust the Sky Lake/Cascade Lake
2012 * idle states table.
2021 * 000b: C0/C1 (no package C-state support) in skx_idle_state_table_update()
2023 * 010b: C6 (non-retention) in skx_idle_state_table_update()
2025 * 111b: No Package C state limits. in skx_idle_state_table_update()
2041 * adl_idle_state_table_update - Adjust AlderLake idle states table.
2060 * spr_idle_state_table_update - Adjust Sapphire Rapids idle states table.
2067 * By default, the C6 state assumes the worst-case scenario of package in spr_idle_state_table_update()
2081 * byt_cht_auto_demotion_disable - Disable Bay/Cherry Trail auto-demotion.
2096 /* Ignore the C-state if there are NO sub-states in CPUID for it. */ in intel_idle_verify_cstate()
2101 mark_tsc_unstable("TSC halts in idle states deeper than C2"); in intel_idle_verify_cstate()
2106 static void state_update_enter_method(struct cpuidle_state *state, int cstate) in state_update_enter_method() argument
2108 if (state->flags & CPUIDLE_FLAG_INIT_XSTATE) { in state_update_enter_method()
2113 WARN_ON_ONCE(state->flags & CPUIDLE_FLAG_IBRS); in state_update_enter_method()
2114 WARN_ON_ONCE(state->flags & CPUIDLE_FLAG_IRQ_ENABLE); in state_update_enter_method()
2115 state->enter = intel_idle_xstate; in state_update_enter_method()
2120 ((state->flags & CPUIDLE_FLAG_IBRS) || ibrs_off)) { in state_update_enter_method()
2122 * IBRS mitigation requires that C-states are entered in state_update_enter_method()
2125 if (ibrs_off && (state->flags & CPUIDLE_FLAG_IRQ_ENABLE)) in state_update_enter_method()
2126 state->flags &= ~CPUIDLE_FLAG_IRQ_ENABLE; in state_update_enter_method()
2127 WARN_ON_ONCE(state->flags & CPUIDLE_FLAG_IRQ_ENABLE); in state_update_enter_method()
2128 state->enter = intel_idle_ibrs; in state_update_enter_method()
2132 if (state->flags & CPUIDLE_FLAG_IRQ_ENABLE) { in state_update_enter_method()
2133 state->enter = intel_idle_irq; in state_update_enter_method()
2138 pr_info("forced intel_idle_irq for state %d\n", cstate); in state_update_enter_method()
2139 state->enter = intel_idle_irq; in state_update_enter_method()
2177 struct cpuidle_state *state; in intel_idle_init_cstates_icpu() local
2190 /* If marked as unusable, skip this state. */ in intel_idle_init_cstates_icpu()
2192 pr_debug("state %s is disabled\n", in intel_idle_init_cstates_icpu()
2193 cpuidle_state_table[cstate].name); in intel_idle_init_cstates_icpu()
2202 drv->states[drv->state_count] = cpuidle_state_table[cstate]; in intel_idle_init_cstates_icpu()
2203 state = &drv->states[drv->state_count]; in intel_idle_init_cstates_icpu()
2205 state_update_enter_method(state, cstate); in intel_idle_init_cstates_icpu()
2208 if ((disabled_states_mask & BIT(drv->state_count)) || in intel_idle_init_cstates_icpu()
2209 ((icpu->use_acpi || force_use_acpi) && in intel_idle_init_cstates_icpu()
2210 intel_idle_off_by_default(state->flags, mwait_hint) && in intel_idle_init_cstates_icpu()
2211 !(state->flags & CPUIDLE_FLAG_ALWAYS_ENABLE))) in intel_idle_init_cstates_icpu()
2212 state->flags |= CPUIDLE_FLAG_OFF; in intel_idle_init_cstates_icpu()
2214 if (intel_idle_state_needs_timer_stop(state)) in intel_idle_init_cstates_icpu()
2215 state->flags |= CPUIDLE_FLAG_TIMER_STOP; in intel_idle_init_cstates_icpu()
2217 drv->state_count++; in intel_idle_init_cstates_icpu()
2222 * intel_idle_cpuidle_driver_init - Create the list of available idle states.
2230 drv->states[0].flags |= CPUIDLE_FLAG_OFF; in intel_idle_cpuidle_driver_init()
2232 drv->state_count = 1; in intel_idle_cpuidle_driver_init()
2234 if (icpu && icpu->state_table) in intel_idle_cpuidle_driver_init()
2268 * intel_idle_cpu_init - Register the target CPU with the cpuidle core.
2279 dev->cpu = cpu; in intel_idle_cpu_init()
2283 return -EIO; in intel_idle_cpu_init()
2310 if (!dev->registered) in intel_idle_cpu_online()
2317 * intel_idle_cpuidle_devices_uninit - Unregister all cpuidle devices.
2333 /* Do not load intel_idle at all for now if idle= is passed */ in intel_idle_init()
2335 return -ENODEV; in intel_idle_init()
2339 return -EPERM; in intel_idle_init()
2346 return -ENODEV; in intel_idle_init()
2351 return -ENODEV; in intel_idle_init()
2359 return -ENODEV; in intel_idle_init()
2363 icpu = (const struct idle_cpu *)id->driver_data; in intel_idle_init()
2365 pr_debug("ignoring native CPU idle states\n"); in intel_idle_init()
2369 if (icpu->state_table) in intel_idle_init()
2370 cpuidle_state_table = icpu->state_table; in intel_idle_init()
2372 return -ENODEV; in intel_idle_init()
2374 auto_demotion_disable_flags = icpu->auto_demotion_disable_flags; in intel_idle_init()
2375 if (icpu->disable_promotion_to_c1e) in intel_idle_init()
2377 if (icpu->use_acpi || force_use_acpi) in intel_idle_init()
2380 return -ENODEV; in intel_idle_init()
2388 return -ENOMEM; in intel_idle_init()
2396 drv ? drv->name : "none"); in intel_idle_init()
2400 retval = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "idle/intel:online", in intel_idle_init()
2406 boot_cpu_has(X86_FEATURE_ARAT) ? "all C-states" : "C1"); in intel_idle_init()
2422 * support "intel_idle.max_cstate=..." at boot and also a read-only export of
2423 * it at /sys/module/intel_idle/parameters/max_cstate -- so using module_param
2429 * idle states to be disabled by default (as reflected by the names of the
2430 * corresponding idle state directories in sysfs, "state0", "state1" ...
2431 * "state<i>" ..., where <i> is the index of the given state).
2434 MODULE_PARM_DESC(states_off, "Mask of disabled idle states");
2436 * Some platforms come with mutually exclusive C-states, so that if one is
2437 * enabled, the other C-states must not be used. Example: C1 and C1E on
2439 * preferred C-states among the groups of mutually exclusive C-states - the
2440 * selected C-states will be registered, the other C-states from the mutually
2442 * exclusive C-states, this parameter has no effect.
2445 MODULE_PARM_DESC(preferred_cstates, "Mask of preferred idle states");
2447 * Debugging option that forces the driver to enter all C-states with
2448 * interrupts enabled. Does not apply to C-states with
2457 MODULE_PARM_DESC(ibrs_off, "Disable IBRS when idle");