1 // SPDX-License-Identifier: GPL-2.0-or-later
3 #include "cgroup-internal.h"
4 #include "cpuset-internal.h"
11 	struct cpuset *cs;  member
15  * Frequency meter - How fast is some event occurring?
19  *   fmeter_init() - initialize a frequency meter.
20  *   fmeter_markevent() - called each time the event happens.
21  *   fmeter_getrate() - returns the recent rate of such events.
22  *   fmeter_update() - internal routine used to update fmeter.
29  * The filter is single-pole low-pass recursive (IIR).  The time unit
30  * is 1 second.  Arithmetic is done using 32-bit integers scaled to
34  * has a half-life of 10 seconds, meaning that if the events quit
49  * per msec it maxes out at values just under 1,000,000.  At constant
59 #define FM_COEF 933		/* coefficient for half-life of 10 secs */
67 	fmp->cnt = 0;  in fmeter_init()
68 	fmp->val = 0;  in fmeter_init()
69 	fmp->time = 0;  in fmeter_init()
70 	spin_lock_init(&fmp->lock);  in fmeter_init()
73 /* Internal meter update - process cnt events and update value */
80 	ticks = now - fmp->time;  in fmeter_update()
86 	while (ticks-- > 0)  in fmeter_update()
87 		fmp->val = (FM_COEF * fmp->val) / FM_SCALE;  in fmeter_update()
88 	fmp->time = now;  in fmeter_update()
90 	fmp->val += ((FM_SCALE - FM_COEF) * fmp->cnt) / FM_SCALE;  in fmeter_update()
91 	fmp->cnt = 0;  in fmeter_update()
97 	spin_lock(&fmp->lock);  in fmeter_markevent()
99 	fmp->cnt = min(FM_MAXCNT, fmp->cnt + FM_SCALE);  in fmeter_markevent()
100 	spin_unlock(&fmp->lock);  in fmeter_markevent()
108 	spin_lock(&fmp->lock);  in fmeter_getrate()
110 	val = fmp->val;  in fmeter_getrate()
111 	spin_unlock(&fmp->lock);  in fmeter_getrate()
124  * __cpuset_memory_pressure_bump - keep stats of per-cpuset reclaims.
135  * Display to user space in the per-cpuset read-only file
144 	fmeter_markevent(&task_cs(current)->fmeter);  in __cpuset_memory_pressure_bump()
148 static int update_relax_domain_level(struct cpuset *cs, s64 val)  in update_relax_domain_level()  argument
151 	if (val < -1 || val > sched_domain_level_max + 1)  in update_relax_domain_level()
152 		return -EINVAL;  in update_relax_domain_level()
155 	if (val != cs->relax_domain_level) {  in update_relax_domain_level()
156 		cs->relax_domain_level = val;  in update_relax_domain_level()
157 		if (!cpumask_empty(cs->cpus_allowed) &&  in update_relax_domain_level()
158 		    is_sched_load_balance(cs))  in update_relax_domain_level()
168 	struct cpuset *cs = css_cs(css);  in cpuset_write_s64()  local
169 	cpuset_filetype_t type = cft->private;  in cpuset_write_s64()
170 	int retval = -ENODEV;  in cpuset_write_s64()
174 	if (!is_cpuset_online(cs))  in cpuset_write_s64()
179 		pr_info_once("cpuset.%s is deprecated\n", cft->name);  in cpuset_write_s64()
180 		retval = update_relax_domain_level(cs, val);  in cpuset_write_s64()
183 		retval = -EINVAL;  in cpuset_write_s64()
194 	struct cpuset *cs = css_cs(css);  in cpuset_read_s64()  local
195 	cpuset_filetype_t type = cft->private;  in cpuset_read_s64()
199 		return cs->relax_domain_level;  in cpuset_read_s64()
214 void cpuset1_update_task_spread_flags(struct cpuset *cs,  in cpuset1_update_task_spread_flags()  argument
220 	if (is_spread_page(cs))  in cpuset1_update_task_spread_flags()
225 	if (is_spread_slab(cs))  in cpuset1_update_task_spread_flags()
232  * cpuset1_update_tasks_flags - update the spread flags of tasks in the cpuset.
233  * @cs: the cpuset in which each task's spread flags needs to be changed
235  * Iterate through each task of @cs updating its spread flags.  As this
239 void cpuset1_update_tasks_flags(struct cpuset *cs)  in cpuset1_update_tasks_flags()  argument
244 	css_task_iter_start(&cs->css, 0, &it);  in cpuset1_update_tasks_flags()
246 		cpuset1_update_task_spread_flags(cs, task);  in cpuset1_update_tasks_flags()
255  * cpuset to its next-highest non-empty parent.
257 static void remove_tasks_in_empty_cpuset(struct cpuset *cs)  in remove_tasks_in_empty_cpuset()  argument
262 	 * Find its next-highest non-empty parent, (top cpuset  in remove_tasks_in_empty_cpuset()
265 	parent = parent_cs(cs);  in remove_tasks_in_empty_cpuset()
266 	while (cpumask_empty(parent->cpus_allowed) ||  in remove_tasks_in_empty_cpuset()
267 			nodes_empty(parent->mems_allowed))  in remove_tasks_in_empty_cpuset()
270 	if (cgroup_transfer_tasks(parent->css.cgroup, cs->css.cgroup)) {  in remove_tasks_in_empty_cpuset()
271 		pr_err("cpuset: failed to transfer tasks out of empty cpuset ");  in remove_tasks_in_empty_cpuset()
272 		pr_cont_cgroup_name(cs->css.cgroup);  in remove_tasks_in_empty_cpuset()
282 	remove_tasks_in_empty_cpuset(s->cs);  in cpuset_migrate_tasks_workfn()
283 	css_put(&s->cs->css);  in cpuset_migrate_tasks_workfn()
287 void cpuset1_hotplug_update_tasks(struct cpuset *cs,  in cpuset1_hotplug_update_tasks()  argument
294 	cpumask_copy(cs->cpus_allowed, new_cpus);  in cpuset1_hotplug_update_tasks()
295 	cpumask_copy(cs->effective_cpus, new_cpus);  in cpuset1_hotplug_update_tasks()
296 	cs->mems_allowed = *new_mems;  in cpuset1_hotplug_update_tasks()
297 	cs->effective_mems = *new_mems;  in cpuset1_hotplug_update_tasks()
304 	if (cpus_updated && !cpumask_empty(cs->cpus_allowed))  in cpuset1_hotplug_update_tasks()
305 		cpuset_update_tasks_cpumask(cs, new_cpus);  in cpuset1_hotplug_update_tasks()
306 	if (mems_updated && !nodes_empty(cs->mems_allowed))  in cpuset1_hotplug_update_tasks()
307 		cpuset_update_tasks_nodemask(cs);  in cpuset1_hotplug_update_tasks()
309 	is_empty = cpumask_empty(cs->cpus_allowed) ||  in cpuset1_hotplug_update_tasks()
310 		   nodes_empty(cs->mems_allowed);  in cpuset1_hotplug_update_tasks()
317 	if (is_empty && cs->css.cgroup->nr_populated_csets &&  in cpuset1_hotplug_update_tasks()
318 	    css_tryget_online(&cs->css)) {  in cpuset1_hotplug_update_tasks()
323 			css_put(&cs->css);  in cpuset1_hotplug_update_tasks()
327 		s->cs = cs;  in cpuset1_hotplug_update_tasks()
328 		INIT_WORK(&s->work, cpuset_migrate_tasks_workfn);  in cpuset1_hotplug_update_tasks()
329 		schedule_work(&s->work);  in cpuset1_hotplug_update_tasks()
334  * is_cpuset_subset(p, q) - Is cpuset p a subset of cpuset q?
343 	return	cpumask_subset(p->cpus_allowed, q->cpus_allowed) &&  in is_cpuset_subset()
344 		nodes_subset(p->mems_allowed, q->mems_allowed) &&  in is_cpuset_subset()
350  * cpuset1_validate_change() - Validate conditions specific to legacy (v1)
362 	ret = -EBUSY;  in cpuset1_validate_change()
365 			goto out;  in cpuset1_validate_change()
368 	ret = -EACCES;  in cpuset1_validate_change()
371 		goto out;  in cpuset1_validate_change()
374 out:  in cpuset1_validate_change()
381  *  - Print tasks cpuset path into seq_file.
382  *  - Used for /proc/<pid>/cpuset.
391 	retval = -ENOMEM;  in proc_cpuset_show()
394 		goto out;  in proc_cpuset_show()
399 	retval = cgroup_path_ns_locked(css->cgroup, buf, PATH_MAX,  in proc_cpuset_show()
400 				       current->nsproxy->cgroup_ns);  in proc_cpuset_show()
404 	if (retval == -E2BIG)  in proc_cpuset_show()
405 		retval = -ENAMETOOLONG;  in proc_cpuset_show()
413 out:  in proc_cpuset_show()
420 	struct cpuset *cs = css_cs(css);  in cpuset_read_u64()  local
421 	cpuset_filetype_t type = cft->private;  in cpuset_read_u64()
425 		return is_cpu_exclusive(cs);  in cpuset_read_u64()
427 		return is_mem_exclusive(cs);  in cpuset_read_u64()
429 		return is_mem_hardwall(cs);  in cpuset_read_u64()
431 		return is_sched_load_balance(cs);  in cpuset_read_u64()
433 		return is_memory_migrate(cs);  in cpuset_read_u64()
437 		return fmeter_getrate(&cs->fmeter);  in cpuset_read_u64()
439 		return is_spread_page(cs);  in cpuset_read_u64()
441 		return is_spread_slab(cs);  in cpuset_read_u64()
453 	struct cpuset *cs = css_cs(css);  in cpuset_write_u64()  local
454 	cpuset_filetype_t type = cft->private;  in cpuset_write_u64()
459 	if (!is_cpuset_online(cs)) {  in cpuset_write_u64()
460 		retval = -ENODEV;  in cpuset_write_u64()
466 		retval = cpuset_update_flag(CS_CPU_EXCLUSIVE, cs, val);  in cpuset_write_u64()
469 		pr_info_once("cpuset.%s is deprecated\n", cft->name);  in cpuset_write_u64()
470 		retval = cpuset_update_flag(CS_MEM_EXCLUSIVE, cs, val);  in cpuset_write_u64()
473 		pr_info_once("cpuset.%s is deprecated\n", cft->name);  in cpuset_write_u64()
474 		retval = cpuset_update_flag(CS_MEM_HARDWALL, cs, val);  in cpuset_write_u64()
477 		pr_info_once("cpuset.%s is deprecated, use cpuset.cpus.partition instead\n", cft->name);  in cpuset_write_u64()
478 		retval = cpuset_update_flag(CS_SCHED_LOAD_BALANCE, cs, val);  in cpuset_write_u64()
481 		pr_info_once("cpuset.%s is deprecated\n", cft->name);  in cpuset_write_u64()
482 		retval = cpuset_update_flag(CS_MEMORY_MIGRATE, cs, val);  in cpuset_write_u64()
485 		pr_info_once("cpuset.%s is deprecated, use memory.pressure with CONFIG_PSI instead\n", cft->name);  in cpuset_write_u64()
489 		pr_info_once("cpuset.%s is deprecated\n", cft->name);  in cpuset_write_u64()
490 		retval = cpuset_update_flag(CS_SPREAD_PAGE, cs, val);  in cpuset_write_u64()
493 		pr_warn_once("cpuset.%s is deprecated\n", cft->name);  in cpuset_write_u64()
494 		retval = cpuset_update_flag(CS_SPREAD_SLAB, cs, val);  in cpuset_write_u64()
497 		retval = -EINVAL;  in cpuset_write_u64()