477 static bool tmigr_check_migrator(struct tmigr_group *group, u8 childmask)  in tmigr_check_migrator()  argument
481 	s.state = atomic_read(&group->migr_state);  in tmigr_check_migrator()
489 static bool tmigr_check_migrator_and_lonely(struct tmigr_group *group, u8 childmask)  in tmigr_check_migrator_and_lonely()  argument
495 	s.state = atomic_read(&group->migr_state);  in tmigr_check_migrator_and_lonely()
506 static bool tmigr_check_lonely(struct tmigr_group *group)  in tmigr_check_lonely()  argument
511 	s.state = atomic_read(&group->migr_state);  in tmigr_check_lonely()
560 			       struct tmigr_group *child, struct tmigr_group *group)  in __walk_groups_from()  argument
563 		WARN_ON_ONCE(group->level >= tmigr_hierarchy_levels);  in __walk_groups_from()
565 		if (up(group, child, data))  in __walk_groups_from()
568 		child = group;  in __walk_groups_from()
573 		group = READ_ONCE(group->parent);  in __walk_groups_from()
576 	} while (group);  in __walk_groups_from()
598 static struct tmigr_event *tmigr_next_groupevt(struct tmigr_group *group)  in tmigr_next_groupevt()  argument
603 	lockdep_assert_held(&group->lock);  in tmigr_next_groupevt()
605 	WRITE_ONCE(group->next_expiry, KTIME_MAX);  in tmigr_next_groupevt()
607 	while ((node = timerqueue_getnext(&group->events))) {  in tmigr_next_groupevt()
611 			WRITE_ONCE(group->next_expiry, evt->nextevt.expires);  in tmigr_next_groupevt()
619 		if (!timerqueue_del(&group->events, node))  in tmigr_next_groupevt()
631 static struct tmigr_event *tmigr_next_expired_groupevt(struct tmigr_group *group,  in tmigr_next_expired_groupevt()  argument
634 	struct tmigr_event *evt = tmigr_next_groupevt(group);  in tmigr_next_expired_groupevt()
642 	timerqueue_del(&group->events, &evt->nextevt);  in tmigr_next_expired_groupevt()
643 	tmigr_next_groupevt(group);  in tmigr_next_expired_groupevt()
648 static u64 tmigr_next_groupevt_expires(struct tmigr_group *group)  in tmigr_next_groupevt_expires()  argument
652 	evt = tmigr_next_groupevt(group);  in tmigr_next_groupevt_expires()
660 static bool tmigr_active_up(struct tmigr_group *group,  in tmigr_active_up()  argument
674 	curstate.state = atomic_read(&group->migr_state);  in tmigr_active_up()
690 	} while (!atomic_try_cmpxchg(&group->migr_state, &curstate.state, newstate.state));  in tmigr_active_up()
692 	trace_tmigr_group_set_cpu_active(group, newstate, childmask);  in tmigr_active_up()
706 	WRITE_ONCE(group->groupevt.ignore, true);  in tmigr_active_up()
760 bool tmigr_update_events(struct tmigr_group *group, struct tmigr_group *child,  in tmigr_update_events()  argument
772 		raw_spin_lock_nested(&group->lock, SINGLE_DEPTH_NESTING);  in tmigr_update_events()
775 		groupstate.state = atomic_read(&group->migr_state);  in tmigr_update_events()
824 		if (ignore && !remote && group->parent)  in tmigr_update_events()
827 		raw_spin_lock(&group->lock);  in tmigr_update_events()
830 		groupstate.state = atomic_read(&group->migr_state);  in tmigr_update_events()
844 		if (!timerqueue_del(&group->events, &evt->nextevt))  in tmigr_update_events()
845 			WRITE_ONCE(group->next_expiry, KTIME_MAX);  in tmigr_update_events()
868 		if (timerqueue_add(&group->events, &evt->nextevt))  in tmigr_update_events()
869 			WRITE_ONCE(group->next_expiry, nextexp);  in tmigr_update_events()
873 	if (!group->parent && (groupstate.migrator == TMIGR_NONE)) {  in tmigr_update_events()
892 		data->firstexp = tmigr_next_groupevt_expires(group);  in tmigr_update_events()
895 	trace_tmigr_update_events(child, group, childstate, groupstate,  in tmigr_update_events()
899 	raw_spin_unlock(&group->lock);  in tmigr_update_events()
907 static bool tmigr_new_timer_up(struct tmigr_group *group,  in tmigr_new_timer_up()  argument
911 	return tmigr_update_events(group, child, data);  in tmigr_new_timer_up()
1040 static bool tmigr_handle_remote_up(struct tmigr_group *group,  in tmigr_handle_remote_up()  argument
1054 	trace_tmigr_handle_remote(group);  in tmigr_handle_remote_up()
1061 	if (!tmigr_check_migrator(group, childmask))  in tmigr_handle_remote_up()
1064 	raw_spin_lock_irq(&group->lock);  in tmigr_handle_remote_up()
1066 	evt = tmigr_next_expired_groupevt(group, now);  in tmigr_handle_remote_up()
1071 		raw_spin_unlock_irq(&group->lock);  in tmigr_handle_remote_up()
1084 	data->firstexp = group->next_expiry;  in tmigr_handle_remote_up()
1086 	raw_spin_unlock_irq(&group->lock);  in tmigr_handle_remote_up()
1139 static bool tmigr_requires_handle_remote_up(struct tmigr_group *group,  in tmigr_requires_handle_remote_up()  argument
1152 	if (!tmigr_check_migrator(group, childmask))  in tmigr_requires_handle_remote_up()
1161 		data->firstexp = READ_ONCE(group->next_expiry);  in tmigr_requires_handle_remote_up()
1167 		raw_spin_lock(&group->lock);  in tmigr_requires_handle_remote_up()
1168 		data->firstexp = group->next_expiry;  in tmigr_requires_handle_remote_up()
1169 		if (data->now >= group->next_expiry) {  in tmigr_requires_handle_remote_up()
1171 			raw_spin_unlock(&group->lock);  in tmigr_requires_handle_remote_up()
1174 		raw_spin_unlock(&group->lock);  in tmigr_requires_handle_remote_up()
1272 static bool tmigr_inactive_up(struct tmigr_group *group,  in tmigr_inactive_up()  argument
1289 	curstate.state = atomic_read_acquire(&group->migr_state);  in tmigr_inactive_up()
1327 		if (atomic_try_cmpxchg(&group->migr_state, &curstate.state, newstate.state)) {  in tmigr_inactive_up()
1328 			trace_tmigr_group_set_cpu_inactive(group, newstate, childmask);  in tmigr_inactive_up()
1344 	tmigr_update_events(group, child, data);  in tmigr_inactive_up()
1424 	struct tmigr_group *group = tmc->tmgroup;  in tmigr_quick_check()  local
1436 		if (!tmigr_check_lonely(group))  in tmigr_quick_check()
1445 		nextevt = min_t(u64, nextevt, READ_ONCE(group->next_expiry));  in tmigr_quick_check()
1446 		group = group->parent;  in tmigr_quick_check()
1447 	} while (group);  in tmigr_quick_check()
1632 static void tmigr_init_group(struct tmigr_group *group, unsigned int lvl,  in tmigr_init_group()  argument
1637 	raw_spin_lock_init(&group->lock);  in tmigr_init_group()
1639 	group->level = lvl;  in tmigr_init_group()
1640 	group->numa_node = lvl < tmigr_crossnode_level ? node : NUMA_NO_NODE;  in tmigr_init_group()
1642 	group->num_children = 0;  in tmigr_init_group()
1647 	atomic_set(&group->migr_state, s.state);  in tmigr_init_group()
1649 	timerqueue_init_head(&group->events);  in tmigr_init_group()
1650 	timerqueue_init(&group->groupevt.nextevt);  in tmigr_init_group()
1651 	group->groupevt.nextevt.expires = KTIME_MAX;  in tmigr_init_group()
1652 	WRITE_ONCE(group->next_expiry, KTIME_MAX);  in tmigr_init_group()
1653 	group->groupevt.ignore = true;  in tmigr_init_group()
1658 	struct tmigr_group *tmp, *group = NULL;  in tmigr_get_group()  local
1682 		group = tmp;  in tmigr_get_group()
1686 	if (group)  in tmigr_get_group()
1687 		return group;  in tmigr_get_group()
1690 	group = kzalloc_node(sizeof(*group), GFP_KERNEL, node);  in tmigr_get_group()
1691 	if (!group)  in tmigr_get_group()
1694 	tmigr_init_group(group, lvl, node);  in tmigr_get_group()
1697 	list_add(&group->list, &tmigr_level_list[lvl]);  in tmigr_get_group()
1698 	trace_tmigr_group_set(group);  in tmigr_get_group()
1699 	return group;  in tmigr_get_group()
1702 static bool tmigr_init_root(struct tmigr_group *group, bool activate)  in tmigr_init_root()  argument
1704 	if (!group->parent && group != tmigr_root) {  in tmigr_init_root()
1710 		group->groupmask = BIT(0);  in tmigr_init_root()
1763 	struct tmigr_group *group, *child, **stack;  in tmigr_setup_groups()  local
1780 		group = tmigr_get_group(node, i);  in tmigr_setup_groups()
1781 		if (IS_ERR(group)) {  in tmigr_setup_groups()
1782 			err = PTR_ERR(group);  in tmigr_setup_groups()
1788 		stack[i] = group;  in tmigr_setup_groups()
1799 		if (group->parent)  in tmigr_setup_groups()
1811 		group = stack[i];  in tmigr_setup_groups()
1814 			list_del(&group->list);  in tmigr_setup_groups()
1815 			kfree(group);  in tmigr_setup_groups()
1819 		WARN_ON_ONCE(i != group->level);  in tmigr_setup_groups()
1827 			tmc->tmgroup = group;  in tmigr_setup_groups()
1828 			tmc->groupmask = BIT(group->num_children++);  in tmigr_setup_groups()
1830 			tmigr_init_root(group, activate);  in tmigr_setup_groups()
1838 			tmigr_connect_child_parent(child, group, activate);  in tmigr_setup_groups()
1880 		group = list_first_entry(&tmigr_level_list[top],  in tmigr_setup_groups()
1881 					 typeof(*group), list);  in tmigr_setup_groups()
1882 		WARN_ON_ONCE(group->parent);  in tmigr_setup_groups()
1887 		tmigr_root = group;  in tmigr_setup_groups()