Lines Matching full:group
182 static void group_init(struct psi_group *group) in group_init() argument
187 seqcount_init(&per_cpu_ptr(group->pcpu, cpu)->seq); in group_init()
188 group->avg_last_update = sched_clock(); in group_init()
189 group->avg_next_update = group->avg_last_update + psi_period; in group_init()
190 INIT_DELAYED_WORK(&group->avgs_work, psi_avgs_work); in group_init()
191 mutex_init(&group->avgs_lock); in group_init()
193 mutex_init(&group->trigger_lock); in group_init()
194 INIT_LIST_HEAD(&group->triggers); in group_init()
195 memset(group->nr_triggers, 0, sizeof(group->nr_triggers)); in group_init()
196 group->poll_states = 0; in group_init()
197 group->poll_min_period = U32_MAX; in group_init()
198 memset(group->polling_total, 0, sizeof(group->polling_total)); in group_init()
199 group->polling_next_update = ULLONG_MAX; in group_init()
200 group->polling_until = 0; in group_init()
201 rcu_assign_pointer(group->poll_task, NULL); in group_init()
236 static void get_recent_times(struct psi_group *group, int cpu, in get_recent_times() argument
240 struct psi_group_cpu *groupc = per_cpu_ptr(group->pcpu, cpu); in get_recent_times()
301 static void collect_percpu_times(struct psi_group *group, in collect_percpu_times() argument
324 get_recent_times(group, cpu, aggregator, times, in collect_percpu_times()
349 group->total[aggregator][s] += in collect_percpu_times()
356 static u64 update_averages(struct psi_group *group, u64 now) in update_averages() argument
364 expires = group->avg_next_update; in update_averages()
376 period = now - (group->avg_last_update + (missed_periods * psi_period)); in update_averages()
377 group->avg_last_update = now; in update_averages()
382 sample = group->total[PSI_AVGS][s] - group->avg_total[s]; in update_averages()
402 group->avg_total[s] += sample; in update_averages()
403 calc_avgs(group->avg[s], missed_periods, sample, period); in update_averages()
412 struct psi_group *group; in psi_avgs_work() local
418 group = container_of(dwork, struct psi_group, avgs_work); in psi_avgs_work()
420 mutex_lock(&group->avgs_lock); in psi_avgs_work()
424 collect_percpu_times(group, PSI_AVGS, &changed_states); in psi_avgs_work()
433 if (now >= group->avg_next_update) in psi_avgs_work()
434 group->avg_next_update = update_averages(group, now); in psi_avgs_work()
438 group->avg_next_update - now) + 1); in psi_avgs_work()
441 mutex_unlock(&group->avgs_lock); in psi_avgs_work()
490 static void init_triggers(struct psi_group *group, u64 now) in init_triggers() argument
494 list_for_each_entry(t, &group->triggers, node) in init_triggers()
496 group->total[PSI_POLL][t->state], 0); in init_triggers()
497 memcpy(group->polling_total, group->total[PSI_POLL], in init_triggers()
498 sizeof(group->polling_total)); in init_triggers()
499 group->polling_next_update = now + group->poll_min_period; in init_triggers()
502 static u64 update_triggers(struct psi_group *group, u64 now) in update_triggers() argument
506 u64 *total = group->total[PSI_POLL]; in update_triggers()
512 list_for_each_entry(t, &group->triggers, node) { in update_triggers()
516 if (group->polling_total[t->state] == total[t->state]) in update_triggers()
521 * remember to update group->polling_total[] once we've in update_triggers()
543 memcpy(group->polling_total, total, in update_triggers()
544 sizeof(group->polling_total)); in update_triggers()
546 return now + group->poll_min_period; in update_triggers()
550 static void psi_schedule_poll_work(struct psi_group *group, unsigned long delay) in psi_schedule_poll_work() argument
557 * mod_timer below can be tolerated because group->polling_next_update in psi_schedule_poll_work()
560 if (timer_pending(&group->poll_timer)) in psi_schedule_poll_work()
565 task = rcu_dereference(group->poll_task); in psi_schedule_poll_work()
571 mod_timer(&group->poll_timer, jiffies + delay); in psi_schedule_poll_work()
576 static void psi_poll_work(struct psi_group *group) in psi_poll_work() argument
581 mutex_lock(&group->trigger_lock); in psi_poll_work()
585 collect_percpu_times(group, PSI_POLL, &changed_states); in psi_poll_work()
587 if (changed_states & group->poll_states) { in psi_poll_work()
589 if (now > group->polling_until) in psi_poll_work()
590 init_triggers(group, now); in psi_poll_work()
597 group->polling_until = now + in psi_poll_work()
598 group->poll_min_period * UPDATES_PER_WINDOW; in psi_poll_work()
601 if (now > group->polling_until) { in psi_poll_work()
602 group->polling_next_update = ULLONG_MAX; in psi_poll_work()
606 if (now >= group->polling_next_update) in psi_poll_work()
607 group->polling_next_update = update_triggers(group, now); in psi_poll_work()
609 psi_schedule_poll_work(group, in psi_poll_work()
610 nsecs_to_jiffies(group->polling_next_update - now) + 1); in psi_poll_work()
613 mutex_unlock(&group->trigger_lock); in psi_poll_work()
618 struct psi_group *group = (struct psi_group *)data; in psi_poll_worker() local
623 wait_event_interruptible(group->poll_wait, in psi_poll_worker()
624 atomic_cmpxchg(&group->poll_wakeup, 1, 0) || in psi_poll_worker()
629 psi_poll_work(group); in psi_poll_worker()
636 struct psi_group *group = from_timer(group, t, poll_timer); in poll_timer_fn() local
638 atomic_set(&group->poll_wakeup, 1); in poll_timer_fn()
639 wake_up_interruptible(&group->poll_wait); in poll_timer_fn()
688 static void psi_group_change(struct psi_group *group, int cpu, in psi_group_change() argument
697 groupc = per_cpu_ptr(group->pcpu, cpu); in psi_group_change()
737 if (state_mask & group->poll_states) in psi_group_change()
738 psi_schedule_poll_work(group, 1); in psi_group_change()
740 if (wake_clock && !delayed_work_pending(&group->avgs_work)) in psi_group_change()
741 schedule_delayed_work(&group->avgs_work, PSI_FREQ); in psi_group_change()
786 struct psi_group *group; in psi_task_change() local
806 while ((group = iterate_groups(task, &iter))) in psi_task_change()
807 psi_group_change(group, cpu, clear, set, wake_clock); in psi_task_change()
813 struct psi_group *group, *common = NULL; in psi_task_switch() local
820 * When moving state between tasks, the group that in psi_task_switch()
827 while ((group = iterate_groups(next, &iter))) { in psi_task_switch()
828 if (per_cpu_ptr(group->pcpu, cpu)->tasks[NR_ONCPU]) { in psi_task_switch()
829 common = group; in psi_task_switch()
833 psi_group_change(group, cpu, 0, TSK_ONCPU, true); in psi_task_switch()
849 while ((group = iterate_groups(prev, &iter)) && group != common) in psi_task_switch()
850 psi_group_change(group, cpu, TSK_ONCPU, 0, true); in psi_task_switch()
856 struct psi_group *group; in psi_memstall_tick() local
859 while ((group = iterate_groups(task, &iter))) { in psi_memstall_tick()
862 groupc = per_cpu_ptr(group->pcpu, cpu); in psi_memstall_tick()
1005 int psi_show(struct seq_file *m, struct psi_group *group, enum psi_res res) in psi_show() argument
1014 mutex_lock(&group->avgs_lock); in psi_show()
1016 collect_percpu_times(group, PSI_AVGS, NULL); in psi_show()
1017 if (now >= group->avg_next_update) in psi_show()
1018 group->avg_next_update = update_averages(group, now); in psi_show()
1019 mutex_unlock(&group->avgs_lock); in psi_show()
1027 avg[w] = group->avg[res * 2 + full][w]; in psi_show()
1028 total = div_u64(group->total[PSI_AVGS][res * 2 + full], in psi_show()
1072 struct psi_trigger *psi_trigger_create(struct psi_group *group, in psi_trigger_create() argument
1105 t->group = group; in psi_trigger_create()
1116 mutex_lock(&group->trigger_lock); in psi_trigger_create()
1118 if (!rcu_access_pointer(group->poll_task)) { in psi_trigger_create()
1121 task = kthread_create(psi_poll_worker, group, "psimon"); in psi_trigger_create()
1124 mutex_unlock(&group->trigger_lock); in psi_trigger_create()
1127 atomic_set(&group->poll_wakeup, 0); in psi_trigger_create()
1128 init_waitqueue_head(&group->poll_wait); in psi_trigger_create()
1130 timer_setup(&group->poll_timer, poll_timer_fn, 0); in psi_trigger_create()
1131 rcu_assign_pointer(group->poll_task, task); in psi_trigger_create()
1134 list_add(&t->node, &group->triggers); in psi_trigger_create()
1135 group->poll_min_period = min(group->poll_min_period, in psi_trigger_create()
1137 group->nr_triggers[t->state]++; in psi_trigger_create()
1138 group->poll_states |= (1 << t->state); in psi_trigger_create()
1140 mutex_unlock(&group->trigger_lock); in psi_trigger_create()
1148 struct psi_group *group = t->group; in psi_trigger_destroy() local
1160 mutex_lock(&group->trigger_lock); in psi_trigger_destroy()
1167 group->nr_triggers[t->state]--; in psi_trigger_destroy()
1168 if (!group->nr_triggers[t->state]) in psi_trigger_destroy()
1169 group->poll_states &= ~(1 << t->state); in psi_trigger_destroy()
1171 list_for_each_entry(tmp, &group->triggers, node) in psi_trigger_destroy()
1174 group->poll_min_period = period; in psi_trigger_destroy()
1176 if (group->poll_states == 0) { in psi_trigger_destroy()
1177 group->polling_until = 0; in psi_trigger_destroy()
1179 group->poll_task, in psi_trigger_destroy()
1180 lockdep_is_held(&group->trigger_lock)); in psi_trigger_destroy()
1181 rcu_assign_pointer(group->poll_task, NULL); in psi_trigger_destroy()
1185 mutex_unlock(&group->trigger_lock); in psi_trigger_destroy()
1200 * can no longer be found through group->poll_task. in psi_trigger_destroy()
1204 del_timer_sync(&group->poll_timer); in psi_trigger_destroy()