23 static inline bool css_uses_rstat(struct cgroup_subsys_state *css)  in css_uses_rstat()  argument
25 	return css_is_self(css) || css->ss->css_rstat_flush != NULL;  in css_uses_rstat()
29 		struct cgroup_subsys_state *css, int cpu)  in css_rstat_cpu()  argument
31 	return per_cpu_ptr(css->rstat_cpu, cpu);  in css_rstat_cpu()
70 __bpf_kfunc void css_rstat_updated(struct cgroup_subsys_state *css, int cpu)  in css_rstat_updated()  argument
80 	if (!css_uses_rstat(css))  in css_rstat_updated()
93 	rstatc = css_rstat_cpu(css, cpu);  in css_rstat_updated()
124 	lhead = ss_lhead_cpu(css->ss, cpu);  in css_rstat_updated()
128 static void __css_process_update_tree(struct cgroup_subsys_state *css, int cpu)  in __css_process_update_tree()  argument
132 		struct css_rstat_cpu *rstatc = css_rstat_cpu(css, cpu);  in __css_process_update_tree()
133 		struct cgroup_subsys_state *parent = css->parent;  in __css_process_update_tree()
145 			rstatc->updated_next = css;  in __css_process_update_tree()
151 		prstatc->updated_children = css;  in __css_process_update_tree()
153 		css = parent;  in __css_process_update_tree()
358 static inline void __css_rstat_lock(struct cgroup_subsys_state *css,  in __css_rstat_lock()  argument
360 	__acquires(ss_rstat_lock(css->ss))  in __css_rstat_lock()
362 	struct cgroup *cgrp = css->cgroup;  in __css_rstat_lock()
366 	lock = ss_rstat_lock(css->ss);  in __css_rstat_lock()
375 static inline void __css_rstat_unlock(struct cgroup_subsys_state *css,  in __css_rstat_unlock()  argument
377 	__releases(ss_rstat_lock(css->ss))  in __css_rstat_unlock()
379 	struct cgroup *cgrp = css->cgroup;  in __css_rstat_unlock()
382 	lock = ss_rstat_lock(css->ss);  in __css_rstat_unlock()
400 __bpf_kfunc void css_rstat_flush(struct cgroup_subsys_state *css)  in css_rstat_flush()  argument
403 	bool is_self = css_is_self(css);  in css_rstat_flush()
409 	if (!css_uses_rstat(css))  in css_rstat_flush()
417 		__css_rstat_lock(css, cpu);  in css_rstat_flush()
418 		pos = css_rstat_updated_list(css, cpu);  in css_rstat_flush()
427 		__css_rstat_unlock(css, cpu);  in css_rstat_flush()
433 int css_rstat_init(struct cgroup_subsys_state *css)  in css_rstat_init()  argument
435 	struct cgroup *cgrp = css->cgroup;  in css_rstat_init()
437 	bool is_self = css_is_self(css);  in css_rstat_init()
446 	} else if (css->ss->css_rstat_flush == NULL)  in css_rstat_init()
450 	if (!css->rstat_cpu) {  in css_rstat_init()
451 		css->rstat_cpu = alloc_percpu(struct css_rstat_cpu);  in css_rstat_init()
452 		if (!css->rstat_cpu) {  in css_rstat_init()
462 		struct css_rstat_cpu *rstatc = css_rstat_cpu(css, cpu);  in css_rstat_init()
464 		rstatc->owner = rstatc->updated_children = css;  in css_rstat_init()
478 void css_rstat_exit(struct cgroup_subsys_state *css)  in css_rstat_exit()  argument
482 	if (!css_uses_rstat(css))  in css_rstat_exit()
485 	if (!css->rstat_cpu)  in css_rstat_exit()
488 	css_rstat_flush(css);  in css_rstat_exit()
492 		struct css_rstat_cpu *rstatc = css_rstat_cpu(css, cpu);  in css_rstat_exit()
494 		if (WARN_ON_ONCE(rstatc->updated_children != css) ||  in css_rstat_exit()
499 	if (css_is_self(css)) {  in css_rstat_exit()
500 		struct cgroup *cgrp = css->cgroup;  in css_rstat_exit()
506 	free_percpu(css->rstat_cpu);  in css_rstat_exit()
507 	css->rstat_cpu = NULL;  in css_rstat_exit()