Lines Matching +full:switching +full:- +full:freq
1 // SPDX-License-Identifier: GPL-2.0
3 * CPUFreq governor based on scheduler-provided CPU utilization data.
59 /* The field below is for single-CPU policies only: */
74 * Since cpufreq_update_util() is called with rq->lock held for in sugov_should_update_freq()
75 * the @target_cpu, our per-CPU data is fully serialized. in sugov_should_update_freq()
77 * However, drivers cannot in general deal with cross-CPU in sugov_should_update_freq()
79 * sugov_update_commit() call may not for the fast switching platforms. in sugov_should_update_freq()
85 * This is needed on the slow switching platforms too to prevent CPUs in sugov_should_update_freq()
88 if (!cpufreq_this_cpu_can_update(sg_policy->policy)) in sugov_should_update_freq()
91 if (unlikely(sg_policy->limits_changed)) { in sugov_should_update_freq()
92 sg_policy->limits_changed = false; in sugov_should_update_freq()
93 sg_policy->need_freq_update = true; in sugov_should_update_freq()
97 delta_ns = time - sg_policy->last_freq_update_time; in sugov_should_update_freq()
99 return delta_ns >= sg_policy->freq_update_delay_ns; in sugov_should_update_freq()
105 if (!sg_policy->need_freq_update) { in sugov_update_next_freq()
106 if (sg_policy->next_freq == next_freq) in sugov_update_next_freq()
109 sg_policy->need_freq_update = cpufreq_driver_test_flags(CPUFREQ_NEED_UPDATE_LIMITS); in sugov_update_next_freq()
112 sg_policy->next_freq = next_freq; in sugov_update_next_freq()
113 sg_policy->last_freq_update_time = time; in sugov_update_next_freq()
122 cpufreq_driver_fast_switch(sg_policy->policy, next_freq); in sugov_fast_switch()
131 if (!sg_policy->work_in_progress) { in sugov_deferred_update()
132 sg_policy->work_in_progress = true; in sugov_deferred_update()
133 irq_work_queue(&sg_policy->irq_work); in sugov_deferred_update()
138 * get_next_freq - Compute a new frequency for a given cpufreq policy.
143 * If the utilization is frequency-invariant, choose the new frequency to be
148 * Otherwise, approximate the would-be frequency-invariant utilization by
155 * The lowest driver-supported frequency which is equal or greater than the raw
162 struct cpufreq_policy *policy = sg_policy->policy; in get_next_freq()
163 unsigned int freq = arch_scale_freq_invariant() ? in get_next_freq() local
164 policy->cpuinfo.max_freq : policy->cur; in get_next_freq()
166 freq = map_util_freq(util, freq, max); in get_next_freq()
168 if (freq == sg_policy->cached_raw_freq && !sg_policy->need_freq_update) in get_next_freq()
169 return sg_policy->next_freq; in get_next_freq()
171 sg_policy->cached_raw_freq = freq; in get_next_freq()
172 return cpufreq_driver_resolve_freq(policy, freq); in get_next_freq()
187 * The cfs,rt,dl utilization are the running times measured with rq->clock_task
188 * which excludes things like IRQ and steal-time. These latter are then accrued
203 type == FREQUENCY_UTIL && rt_rq_is_runnable(&rq->rt)) { in schedutil_cpu_util()
209 * because of inaccuracies in how we track these -- see in schedutil_cpu_util()
241 * saturation when we should -- something for later. in schedutil_cpu_util()
258 * max - irq in schedutil_cpu_util()
259 * U' = irq + --------- * U in schedutil_cpu_util()
271 * Ideally we would like to set bw_dl as min/guaranteed freq and util + in schedutil_cpu_util()
272 * bw_dl as requested freq. However, cpufreq is not yet ready for such in schedutil_cpu_util()
283 struct rq *rq = cpu_rq(sg_cpu->cpu); in sugov_get_util()
285 unsigned long max = arch_scale_cpu_capacity(sg_cpu->cpu); in sugov_get_util()
287 sg_cpu->max = max; in sugov_get_util()
288 sg_cpu->bw_dl = cpu_bw_dl(rq); in sugov_get_util()
290 return schedutil_cpu_util(sg_cpu->cpu, util, max, FREQUENCY_UTIL, NULL); in sugov_get_util()
294 * sugov_iowait_reset() - Reset the IO boost status of a CPU.
307 s64 delta_ns = time - sg_cpu->last_update; in sugov_iowait_reset()
313 sg_cpu->iowait_boost = set_iowait_boost ? IOWAIT_BOOST_MIN : 0; in sugov_iowait_reset()
314 sg_cpu->iowait_boost_pending = set_iowait_boost; in sugov_iowait_reset()
320 * sugov_iowait_boost() - Updates the IO boost status of a CPU.
339 if (sg_cpu->iowait_boost && in sugov_iowait_boost()
348 if (sg_cpu->iowait_boost_pending) in sugov_iowait_boost()
350 sg_cpu->iowait_boost_pending = true; in sugov_iowait_boost()
353 if (sg_cpu->iowait_boost) { in sugov_iowait_boost()
354 sg_cpu->iowait_boost = in sugov_iowait_boost()
355 min_t(unsigned int, sg_cpu->iowait_boost << 1, SCHED_CAPACITY_SCALE); in sugov_iowait_boost()
360 sg_cpu->iowait_boost = IOWAIT_BOOST_MIN; in sugov_iowait_boost()
364 * sugov_iowait_apply() - Apply the IO boost to a CPU.
388 if (!sg_cpu->iowait_boost) in sugov_iowait_apply()
395 if (!sg_cpu->iowait_boost_pending) { in sugov_iowait_apply()
399 sg_cpu->iowait_boost >>= 1; in sugov_iowait_apply()
400 if (sg_cpu->iowait_boost < IOWAIT_BOOST_MIN) { in sugov_iowait_apply()
401 sg_cpu->iowait_boost = 0; in sugov_iowait_apply()
406 sg_cpu->iowait_boost_pending = false; in sugov_iowait_apply()
412 boost = (sg_cpu->iowait_boost * max) >> SCHED_CAPACITY_SHIFT; in sugov_iowait_apply()
419 unsigned long idle_calls = tick_nohz_get_idle_calls_cpu(sg_cpu->cpu); in sugov_cpu_is_busy()
420 bool ret = idle_calls == sg_cpu->saved_idle_calls; in sugov_cpu_is_busy()
422 sg_cpu->saved_idle_calls = idle_calls; in sugov_cpu_is_busy()
435 if (cpu_bw_dl(cpu_rq(sg_cpu->cpu)) > sg_cpu->bw_dl) in ignore_dl_rate_limit()
436 sg_policy->limits_changed = true; in ignore_dl_rate_limit()
443 struct sugov_policy *sg_policy = sg_cpu->sg_policy; in sugov_update_single()
446 unsigned int cached_freq = sg_policy->cached_raw_freq; in sugov_update_single()
449 sg_cpu->last_update = time; in sugov_update_single()
457 max = sg_cpu->max; in sugov_update_single()
464 if (sugov_cpu_is_busy(sg_cpu) && next_f < sg_policy->next_freq) { in sugov_update_single()
465 next_f = sg_policy->next_freq; in sugov_update_single()
467 /* Restore cached freq as next_freq has changed */ in sugov_update_single()
468 sg_policy->cached_raw_freq = cached_freq; in sugov_update_single()
472 * This code runs under rq->lock for the target CPU, so it won't run in sugov_update_single()
476 if (sg_policy->policy->fast_switch_enabled) { in sugov_update_single()
479 raw_spin_lock(&sg_policy->update_lock); in sugov_update_single()
481 raw_spin_unlock(&sg_policy->update_lock); in sugov_update_single()
487 struct sugov_policy *sg_policy = sg_cpu->sg_policy; in sugov_next_freq_shared()
488 struct cpufreq_policy *policy = sg_policy->policy; in sugov_next_freq_shared()
492 for_each_cpu(j, policy->cpus) { in sugov_next_freq_shared()
497 j_max = j_sg_cpu->max; in sugov_next_freq_shared()
513 struct sugov_policy *sg_policy = sg_cpu->sg_policy; in sugov_update_shared()
516 raw_spin_lock(&sg_policy->update_lock); in sugov_update_shared()
519 sg_cpu->last_update = time; in sugov_update_shared()
526 if (sg_policy->policy->fast_switch_enabled) in sugov_update_shared()
532 raw_spin_unlock(&sg_policy->update_lock); in sugov_update_shared()
538 unsigned int freq; in sugov_work() local
542 * Hold sg_policy->update_lock shortly to handle the case where: in sugov_work()
543 * incase sg_policy->next_freq is read here, and then updated by in sugov_work()
551 raw_spin_lock_irqsave(&sg_policy->update_lock, flags); in sugov_work()
552 freq = sg_policy->next_freq; in sugov_work()
553 sg_policy->work_in_progress = false; in sugov_work()
554 raw_spin_unlock_irqrestore(&sg_policy->update_lock, flags); in sugov_work()
556 mutex_lock(&sg_policy->work_lock); in sugov_work()
557 __cpufreq_driver_target(sg_policy->policy, freq, CPUFREQ_RELATION_L); in sugov_work()
558 mutex_unlock(&sg_policy->work_lock); in sugov_work()
567 kthread_queue_work(&sg_policy->worker, &sg_policy->work); in sugov_irq_work()
584 return sprintf(buf, "%u\n", tunables->rate_limit_us); in rate_limit_us_show()
595 return -EINVAL; in rate_limit_us_store()
597 tunables->rate_limit_us = rate_limit_us; in rate_limit_us_store()
599 list_for_each_entry(sg_policy, &attr_set->policy_list, tunables_hook) in rate_limit_us_store()
600 sg_policy->freq_update_delay_ns = rate_limit_us * NSEC_PER_USEC; in rate_limit_us_store()
630 sg_policy->policy = policy; in sugov_policy_alloc()
631 raw_spin_lock_init(&sg_policy->update_lock); in sugov_policy_alloc()
657 struct cpufreq_policy *policy = sg_policy->policy; in sugov_kthread_create()
661 if (policy->fast_switch_enabled) in sugov_kthread_create()
664 kthread_init_work(&sg_policy->work, sugov_work); in sugov_kthread_create()
665 kthread_init_worker(&sg_policy->worker); in sugov_kthread_create()
666 thread = kthread_create(kthread_worker_fn, &sg_policy->worker, in sugov_kthread_create()
668 cpumask_first(policy->related_cpus)); in sugov_kthread_create()
681 sg_policy->thread = thread; in sugov_kthread_create()
682 kthread_bind_mask(thread, policy->related_cpus); in sugov_kthread_create()
683 init_irq_work(&sg_policy->irq_work, sugov_irq_work); in sugov_kthread_create()
684 mutex_init(&sg_policy->work_lock); in sugov_kthread_create()
694 if (sg_policy->policy->fast_switch_enabled) in sugov_kthread_stop()
697 kthread_flush_worker(&sg_policy->worker); in sugov_kthread_stop()
698 kthread_stop(sg_policy->thread); in sugov_kthread_stop()
699 mutex_destroy(&sg_policy->work_lock); in sugov_kthread_stop()
708 gov_attr_set_init(&tunables->attr_set, &sg_policy->tunables_hook); in sugov_tunables_alloc()
730 if (policy->governor_data) in sugov_init()
731 return -EBUSY; in sugov_init()
737 ret = -ENOMEM; in sugov_init()
749 ret = -EINVAL; in sugov_init()
752 policy->governor_data = sg_policy; in sugov_init()
753 sg_policy->tunables = global_tunables; in sugov_init()
755 gov_attr_set_get(&global_tunables->attr_set, &sg_policy->tunables_hook); in sugov_init()
761 ret = -ENOMEM; in sugov_init()
765 tunables->rate_limit_us = cpufreq_policy_transition_delay_us(policy); in sugov_init()
767 policy->governor_data = sg_policy; in sugov_init()
768 sg_policy->tunables = tunables; in sugov_init()
770 ret = kobject_init_and_add(&tunables->attr_set.kobj, &sugov_tunables_ktype, in sugov_init()
781 kobject_put(&tunables->attr_set.kobj); in sugov_init()
782 policy->governor_data = NULL; in sugov_init()
801 struct sugov_policy *sg_policy = policy->governor_data; in sugov_exit()
802 struct sugov_tunables *tunables = sg_policy->tunables; in sugov_exit()
807 count = gov_attr_set_put(&tunables->attr_set, &sg_policy->tunables_hook); in sugov_exit()
808 policy->governor_data = NULL; in sugov_exit()
821 struct sugov_policy *sg_policy = policy->governor_data; in sugov_start()
824 sg_policy->freq_update_delay_ns = sg_policy->tunables->rate_limit_us * NSEC_PER_USEC; in sugov_start()
825 sg_policy->last_freq_update_time = 0; in sugov_start()
826 sg_policy->next_freq = 0; in sugov_start()
827 sg_policy->work_in_progress = false; in sugov_start()
828 sg_policy->limits_changed = false; in sugov_start()
829 sg_policy->cached_raw_freq = 0; in sugov_start()
831 sg_policy->need_freq_update = cpufreq_driver_test_flags(CPUFREQ_NEED_UPDATE_LIMITS); in sugov_start()
833 for_each_cpu(cpu, policy->cpus) { in sugov_start()
837 sg_cpu->cpu = cpu; in sugov_start()
838 sg_cpu->sg_policy = sg_policy; in sugov_start()
841 for_each_cpu(cpu, policy->cpus) { in sugov_start()
844 cpufreq_add_update_util_hook(cpu, &sg_cpu->update_util, in sugov_start()
854 struct sugov_policy *sg_policy = policy->governor_data; in sugov_stop()
857 for_each_cpu(cpu, policy->cpus) in sugov_stop()
862 if (!policy->fast_switch_enabled) { in sugov_stop()
863 irq_work_sync(&sg_policy->irq_work); in sugov_stop()
864 kthread_cancel_work_sync(&sg_policy->work); in sugov_stop()
870 struct sugov_policy *sg_policy = policy->governor_data; in sugov_limits()
872 if (!policy->fast_switch_enabled) { in sugov_limits()
873 mutex_lock(&sg_policy->work_lock); in sugov_limits()
875 mutex_unlock(&sg_policy->work_lock); in sugov_limits()
878 sg_policy->limits_changed = true; in sugov_limits()
922 if (old_gov == &schedutil_gov || policy->governor == &schedutil_gov) { in sched_cpufreq_governor_change()