Lines Matching +full:max +full:- +full:freq
1 // SPDX-License-Identifier: GPL-2.0
3 * CPUFreq governor based on scheduler-provided CPU utilization data.
52 /* The field below is for single-CPU policies only: */
67 * Since cpufreq_update_util() is called with rq->lock held for in sugov_should_update_freq()
68 * the @target_cpu, our per-CPU data is fully serialized. in sugov_should_update_freq()
70 * However, drivers cannot in general deal with cross-CPU in sugov_should_update_freq()
81 if (!cpufreq_this_cpu_can_update(sg_policy->policy)) in sugov_should_update_freq()
84 if (unlikely(READ_ONCE(sg_policy->limits_changed))) { in sugov_should_update_freq()
85 WRITE_ONCE(sg_policy->limits_changed, false); in sugov_should_update_freq()
86 sg_policy->need_freq_update = true; in sugov_should_update_freq()
99 } else if (sg_policy->need_freq_update) { in sugov_should_update_freq()
104 delta_ns = time - sg_policy->last_freq_update_time; in sugov_should_update_freq()
106 return delta_ns >= sg_policy->freq_update_delay_ns; in sugov_should_update_freq()
112 if (sg_policy->need_freq_update) { in sugov_update_next_freq()
113 sg_policy->need_freq_update = false; in sugov_update_next_freq()
122 if (sg_policy->next_freq == next_freq && in sugov_update_next_freq()
125 } else if (sg_policy->next_freq == next_freq) { in sugov_update_next_freq()
129 sg_policy->next_freq = next_freq; in sugov_update_next_freq()
130 sg_policy->last_freq_update_time = time; in sugov_update_next_freq()
137 if (!sg_policy->work_in_progress) { in sugov_deferred_update()
138 sg_policy->work_in_progress = true; in sugov_deferred_update()
139 irq_work_queue(&sg_policy->irq_work); in sugov_deferred_update()
144 * get_capacity_ref_freq - get the reference frequency that has been used to
154 unsigned int freq = arch_scale_freq_ref(policy->cpu); in get_capacity_ref_freq() local
156 if (freq) in get_capacity_ref_freq()
157 return freq; in get_capacity_ref_freq()
160 return policy->cpuinfo.max_freq; in get_capacity_ref_freq()
166 return policy->cur + (policy->cur >> 2); in get_capacity_ref_freq()
170 * get_next_freq - Compute a new frequency for a given cpufreq policy.
173 * @max: CPU capacity.
175 * If the utilization is frequency-invariant, choose the new frequency to be
178 * next_freq = C * max_freq * util / max
180 * Otherwise, approximate the would-be frequency-invariant utilization by
183 * next_freq = C * curr_freq * util_raw / max
185 * Take C = 1.25 for the frequency tipping point at (util / max) = 0.8.
187 * The lowest driver-supported frequency which is equal or greater than the raw
188 * next_freq (as calculated above) is returned, subject to policy min/max and
192 unsigned long util, unsigned long max) in get_next_freq() argument
194 struct cpufreq_policy *policy = sg_policy->policy; in get_next_freq()
195 unsigned int freq; in get_next_freq() local
197 freq = get_capacity_ref_freq(policy); in get_next_freq()
198 freq = map_util_freq(util, freq, max); in get_next_freq()
200 if (freq == sg_policy->cached_raw_freq && !sg_policy->need_freq_update) in get_next_freq()
201 return sg_policy->next_freq; in get_next_freq()
203 sg_policy->cached_raw_freq = freq; in get_next_freq()
204 return cpufreq_driver_resolve_freq(policy, freq); in get_next_freq()
209 unsigned long max) in sugov_effective_cpu_perf() argument
213 /* Actually we don't need to target the max performance */ in sugov_effective_cpu_perf()
214 if (actual < max) in sugov_effective_cpu_perf()
215 max = actual; in sugov_effective_cpu_perf()
221 return max(min, max); in sugov_effective_cpu_perf()
226 unsigned long min, max, util = scx_cpuperf_target(sg_cpu->cpu); in sugov_get_util() local
229 util += cpu_util_cfs_boost(sg_cpu->cpu); in sugov_get_util()
230 util = effective_cpu_util(sg_cpu->cpu, util, &min, &max); in sugov_get_util()
231 util = max(util, boost); in sugov_get_util()
232 sg_cpu->bw_min = min; in sugov_get_util()
233 sg_cpu->util = sugov_effective_cpu_perf(sg_cpu->cpu, util, min, max); in sugov_get_util()
237 * sugov_iowait_reset() - Reset the IO boost status of a CPU.
250 s64 delta_ns = time - sg_cpu->last_update; in sugov_iowait_reset()
256 sg_cpu->iowait_boost = set_iowait_boost ? IOWAIT_BOOST_MIN : 0; in sugov_iowait_reset()
257 sg_cpu->iowait_boost_pending = set_iowait_boost; in sugov_iowait_reset()
263 * sugov_iowait_boost() - Updates the IO boost status of a CPU.
282 if (sg_cpu->iowait_boost && in sugov_iowait_boost()
291 if (sg_cpu->iowait_boost_pending) in sugov_iowait_boost()
293 sg_cpu->iowait_boost_pending = true; in sugov_iowait_boost()
296 if (sg_cpu->iowait_boost) { in sugov_iowait_boost()
297 sg_cpu->iowait_boost = in sugov_iowait_boost()
298 min_t(unsigned int, sg_cpu->iowait_boost << 1, SCHED_CAPACITY_SCALE); in sugov_iowait_boost()
303 sg_cpu->iowait_boost = IOWAIT_BOOST_MIN; in sugov_iowait_boost()
307 * sugov_iowait_apply() - Apply the IO boost to a CPU.
310 * @max_cap: the max CPU capacity
328 if (!sg_cpu->iowait_boost) in sugov_iowait_apply()
335 if (!sg_cpu->iowait_boost_pending) { in sugov_iowait_apply()
339 sg_cpu->iowait_boost >>= 1; in sugov_iowait_apply()
340 if (sg_cpu->iowait_boost < IOWAIT_BOOST_MIN) { in sugov_iowait_apply()
341 sg_cpu->iowait_boost = 0; in sugov_iowait_apply()
346 sg_cpu->iowait_boost_pending = false; in sugov_iowait_apply()
349 * sg_cpu->util is already in capacity scale; convert iowait_boost in sugov_iowait_apply()
352 return (sg_cpu->iowait_boost * max_cap) >> SCHED_CAPACITY_SHIFT; in sugov_iowait_apply()
370 if (uclamp_rq_is_capped(cpu_rq(sg_cpu->cpu))) in sugov_hold_freq()
377 idle_calls = tick_nohz_get_idle_calls_cpu(sg_cpu->cpu); in sugov_hold_freq()
378 ret = idle_calls == sg_cpu->saved_idle_calls; in sugov_hold_freq()
380 sg_cpu->saved_idle_calls = idle_calls; in sugov_hold_freq()
393 if (cpu_bw_dl(cpu_rq(sg_cpu->cpu)) > sg_cpu->bw_min) in ignore_dl_rate_limit()
394 sg_cpu->sg_policy->need_freq_update = true; in ignore_dl_rate_limit()
404 sg_cpu->last_update = time; in sugov_update_single_common()
408 if (!sugov_should_update_freq(sg_cpu->sg_policy, time)) in sugov_update_single_common()
421 struct sugov_policy *sg_policy = sg_cpu->sg_policy; in sugov_update_single_freq()
422 unsigned int cached_freq = sg_policy->cached_raw_freq; in sugov_update_single_freq()
426 max_cap = arch_scale_cpu_capacity(sg_cpu->cpu); in sugov_update_single_freq()
431 next_f = get_next_freq(sg_policy, sg_cpu->util, max_cap); in sugov_update_single_freq()
433 if (sugov_hold_freq(sg_cpu) && next_f < sg_policy->next_freq && in sugov_update_single_freq()
434 !sg_policy->need_freq_update) { in sugov_update_single_freq()
435 next_f = sg_policy->next_freq; in sugov_update_single_freq()
437 /* Restore cached freq as next_freq has changed */ in sugov_update_single_freq()
438 sg_policy->cached_raw_freq = cached_freq; in sugov_update_single_freq()
445 * This code runs under rq->lock for the target CPU, so it won't run in sugov_update_single_freq()
449 if (sg_policy->policy->fast_switch_enabled) { in sugov_update_single_freq()
450 cpufreq_driver_fast_switch(sg_policy->policy, next_f); in sugov_update_single_freq()
452 raw_spin_lock(&sg_policy->update_lock); in sugov_update_single_freq()
454 raw_spin_unlock(&sg_policy->update_lock); in sugov_update_single_freq()
462 unsigned long prev_util = sg_cpu->util; in sugov_update_single_perf()
475 max_cap = arch_scale_cpu_capacity(sg_cpu->cpu); in sugov_update_single_perf()
480 if (sugov_hold_freq(sg_cpu) && sg_cpu->util < prev_util) in sugov_update_single_perf()
481 sg_cpu->util = prev_util; in sugov_update_single_perf()
483 cpufreq_driver_adjust_perf(sg_cpu->cpu, sg_cpu->bw_min, in sugov_update_single_perf()
484 sg_cpu->util, max_cap); in sugov_update_single_perf()
486 sg_cpu->sg_policy->last_freq_update_time = time; in sugov_update_single_perf()
491 struct sugov_policy *sg_policy = sg_cpu->sg_policy; in sugov_next_freq_shared()
492 struct cpufreq_policy *policy = sg_policy->policy; in sugov_next_freq_shared()
496 max_cap = arch_scale_cpu_capacity(sg_cpu->cpu); in sugov_next_freq_shared()
498 for_each_cpu(j, policy->cpus) { in sugov_next_freq_shared()
505 util = max(j_sg_cpu->util, util); in sugov_next_freq_shared()
515 struct sugov_policy *sg_policy = sg_cpu->sg_policy; in sugov_update_shared()
518 raw_spin_lock(&sg_policy->update_lock); in sugov_update_shared()
521 sg_cpu->last_update = time; in sugov_update_shared()
531 if (sg_policy->policy->fast_switch_enabled) in sugov_update_shared()
532 cpufreq_driver_fast_switch(sg_policy->policy, next_f); in sugov_update_shared()
537 raw_spin_unlock(&sg_policy->update_lock); in sugov_update_shared()
543 unsigned int freq; in sugov_work() local
547 * Hold sg_policy->update_lock shortly to handle the case where: in sugov_work()
548 * in case sg_policy->next_freq is read here, and then updated by in sugov_work()
556 raw_spin_lock_irqsave(&sg_policy->update_lock, flags); in sugov_work()
557 freq = sg_policy->next_freq; in sugov_work()
558 sg_policy->work_in_progress = false; in sugov_work()
559 raw_spin_unlock_irqrestore(&sg_policy->update_lock, flags); in sugov_work()
561 mutex_lock(&sg_policy->work_lock); in sugov_work()
562 __cpufreq_driver_target(sg_policy->policy, freq, CPUFREQ_RELATION_L); in sugov_work()
563 mutex_unlock(&sg_policy->work_lock); in sugov_work()
572 kthread_queue_work(&sg_policy->worker, &sg_policy->work); in sugov_irq_work()
589 return sprintf(buf, "%u\n", tunables->rate_limit_us); in rate_limit_us_show()
600 return -EINVAL; in rate_limit_us_store()
602 tunables->rate_limit_us = rate_limit_us; in rate_limit_us_store()
604 list_for_each_entry(sg_policy, &attr_set->policy_list, tunables_hook) in rate_limit_us_store()
605 sg_policy->freq_update_delay_ns = rate_limit_us * NSEC_PER_USEC; in rate_limit_us_store()
643 sg_policy->policy = policy; in sugov_policy_alloc()
644 raw_spin_lock_init(&sg_policy->update_lock); in sugov_policy_alloc()
670 struct cpufreq_policy *policy = sg_policy->policy; in sugov_kthread_create()
674 if (policy->fast_switch_enabled) in sugov_kthread_create()
677 kthread_init_work(&sg_policy->work, sugov_work); in sugov_kthread_create()
678 kthread_init_worker(&sg_policy->worker); in sugov_kthread_create()
679 thread = kthread_create(kthread_worker_fn, &sg_policy->worker, in sugov_kthread_create()
681 cpumask_first(policy->related_cpus)); in sugov_kthread_create()
694 sg_policy->thread = thread; in sugov_kthread_create()
695 if (policy->dvfs_possible_from_any_cpu) in sugov_kthread_create()
696 set_cpus_allowed_ptr(thread, policy->related_cpus); in sugov_kthread_create()
698 kthread_bind_mask(thread, policy->related_cpus); in sugov_kthread_create()
700 init_irq_work(&sg_policy->irq_work, sugov_irq_work); in sugov_kthread_create()
701 mutex_init(&sg_policy->work_lock); in sugov_kthread_create()
711 if (sg_policy->policy->fast_switch_enabled) in sugov_kthread_stop()
714 kthread_flush_worker(&sg_policy->worker); in sugov_kthread_stop()
715 kthread_stop(sg_policy->thread); in sugov_kthread_stop()
716 mutex_destroy(&sg_policy->work_lock); in sugov_kthread_stop()
725 gov_attr_set_init(&tunables->attr_set, &sg_policy->tunables_hook); in sugov_tunables_alloc()
745 if (policy->governor_data) in sugov_init()
746 return -EBUSY; in sugov_init()
752 ret = -ENOMEM; in sugov_init()
764 ret = -EINVAL; in sugov_init()
767 policy->governor_data = sg_policy; in sugov_init()
768 sg_policy->tunables = global_tunables; in sugov_init()
770 gov_attr_set_get(&global_tunables->attr_set, &sg_policy->tunables_hook); in sugov_init()
776 ret = -ENOMEM; in sugov_init()
780 tunables->rate_limit_us = cpufreq_policy_transition_delay_us(policy); in sugov_init()
782 policy->governor_data = sg_policy; in sugov_init()
783 sg_policy->tunables = tunables; in sugov_init()
785 ret = kobject_init_and_add(&tunables->attr_set.kobj, &sugov_tunables_ktype, in sugov_init()
801 kobject_put(&tunables->attr_set.kobj); in sugov_init()
802 policy->governor_data = NULL; in sugov_init()
821 struct sugov_policy *sg_policy = policy->governor_data; in sugov_exit()
822 struct sugov_tunables *tunables = sg_policy->tunables; in sugov_exit()
827 count = gov_attr_set_put(&tunables->attr_set, &sg_policy->tunables_hook); in sugov_exit()
828 policy->governor_data = NULL; in sugov_exit()
843 struct sugov_policy *sg_policy = policy->governor_data; in sugov_start()
847 sg_policy->freq_update_delay_ns = sg_policy->tunables->rate_limit_us * NSEC_PER_USEC; in sugov_start()
848 sg_policy->last_freq_update_time = 0; in sugov_start()
849 sg_policy->next_freq = 0; in sugov_start()
850 sg_policy->work_in_progress = false; in sugov_start()
851 sg_policy->limits_changed = false; in sugov_start()
852 sg_policy->cached_raw_freq = 0; in sugov_start()
854 sg_policy->need_freq_update = cpufreq_driver_test_flags(CPUFREQ_NEED_UPDATE_LIMITS); in sugov_start()
858 else if (policy->fast_switch_enabled && cpufreq_driver_has_adjust_perf()) in sugov_start()
863 for_each_cpu(cpu, policy->cpus) { in sugov_start()
867 sg_cpu->cpu = cpu; in sugov_start()
868 sg_cpu->sg_policy = sg_policy; in sugov_start()
869 cpufreq_add_update_util_hook(cpu, &sg_cpu->update_util, uu); in sugov_start()
876 struct sugov_policy *sg_policy = policy->governor_data; in sugov_stop()
879 for_each_cpu(cpu, policy->cpus) in sugov_stop()
884 if (!policy->fast_switch_enabled) { in sugov_stop()
885 irq_work_sync(&sg_policy->irq_work); in sugov_stop()
886 kthread_cancel_work_sync(&sg_policy->work); in sugov_stop()
892 struct sugov_policy *sg_policy = policy->governor_data; in sugov_limits()
894 if (!policy->fast_switch_enabled) { in sugov_limits()
895 mutex_lock(&sg_policy->work_lock); in sugov_limits()
897 mutex_unlock(&sg_policy->work_lock); in sugov_limits()
909 WRITE_ONCE(sg_policy->limits_changed, true); in sugov_limits()