1 /* SPDX-License-Identifier: GPL-2.0 */
7  * set of CPUs in a system, one bit position per CPU number.  In general,
20  * cpumask_pr_args - printf args to output a cpumask
27 #if (NR_CPUS == 1) || defined(CONFIG_FORCE_NR_CPUS)
35 #if (NR_CPUS == 1) || defined(CONFIG_FORCE_NR_CPUS)  in set_nr_cpu_ids()
47  * optimized routines that work for the single-word case, but only when
57  * they set bits or just don't have any faster fixed-sized versions. We
63  * optimization comes from being able to potentially use a compile-time
64  * constant instead of a run-time generated exact number of CPUs.
82  *     cpu_possible_mask- has bit 'cpu' set iff cpu is populatable
83  *     cpu_present_mask - has bit 'cpu' set iff cpu is populated
84  *     cpu_enabled_mask - has bit 'cpu' set iff cpu can be brought online
85  *     cpu_online_mask  - has bit 'cpu' set iff cpu available to scheduler
86  *     cpu_active_mask  - has bit 'cpu' set iff cpu available to migration
90  *  The cpu_possible_mask is fixed at boot time, as the set of CPU IDs
105  * 1) UP ARCHes (NR_CPUS == 1, CONFIG_SMP not defined) hardcode
106  *    assumption that their single CPU is online.  The UP
110  *    optimization - don't waste any instructions or memory references
112  *    only one CPU.
132 static __always_inline void cpu_max_bits_warn(unsigned int cpu, unsigned int bits)  in cpu_max_bits_warn()  argument
135 	WARN_ON_ONCE(cpu >= bits);  in cpu_max_bits_warn()
139 /* verify cpu argument to cpumask_* operators */
140 static __always_inline unsigned int cpumask_check(unsigned int cpu)  in cpumask_check()  argument
142 	cpu_max_bits_warn(cpu, small_cpumask_bits);  in cpumask_check()
143 	return cpu;  in cpumask_check()
147  * cpumask_first - get the first cpu in a cpumask
158  * cpumask_first_zero - get the first unset cpu in a cpumask
169  * cpumask_first_and - return the first cpu from *srcp1 & *srcp2
182  * cpumask_first_and_and - return the first cpu from *srcp1 & *srcp2 & *srcp3
199  * cpumask_last - get the last CPU in a cpumask
200  * @srcp:	- the cpumask pointer
210  * cpumask_next - get the next cpu in a cpumask
211  * @n: the cpu prior to the place to search (i.e. return will be > @n)
219 	/* -1 is a legal arg here. */  in cpumask_next()
220 	if (n != -1)  in cpumask_next()
222 	return find_next_bit(cpumask_bits(srcp), small_cpumask_bits, n + 1);  in cpumask_next()
226  * cpumask_next_zero - get the next unset cpu in a cpumask
227  * @n: the cpu prior to the place to search (i.e. return will be > @n)
235 	/* -1 is a legal arg here. */  in cpumask_next_zero()
236 	if (n != -1)  in cpumask_next_zero()
238 	return find_next_zero_bit(cpumask_bits(srcp), small_cpumask_bits, n+1);  in cpumask_next_zero()
241 #if NR_CPUS == 1
242 /* Uniprocessor: there is only one valid CPU */
269  * cpumask_next_and - get the next cpu in *src1p & *src2p
270  * @n: the cpu prior to the place to search (i.e. return will be > @n)
280 	/* -1 is a legal arg here. */  in cpumask_next_and()
281 	if (n != -1)  in cpumask_next_and()
284 		small_cpumask_bits, n + 1);  in cpumask_next_and()
288  * cpumask_next_and_wrap - get the next cpu in *src1p & *src2p, starting from
289  *			   @n+1. If nothing found, wrap around and start from
291  * @n: the cpu prior to the place to search (i.e. search starts from @n+1)
301 	/* -1 is a legal arg here. */  in cpumask_next_and_wrap()
302 	if (n != -1)  in cpumask_next_and_wrap()
305 		small_cpumask_bits, n + 1);  in cpumask_next_and_wrap()
309  * cpumask_next_wrap - get the next cpu in *src, starting from @n+1. If nothing
311  * @n: the cpu prior to the place to search (i.e. search starts from @n+1)
319 	/* -1 is a legal arg here. */  in cpumask_next_wrap()
320 	if (n != -1)  in cpumask_next_wrap()
322 	return find_next_bit_wrap(cpumask_bits(src), small_cpumask_bits, n + 1);  in cpumask_next_wrap()
326  * for_each_cpu - iterate over every cpu in a mask
327  * @cpu: the (optionally unsigned) integer iterator
330  * After the loop, cpu is >= nr_cpu_ids.
332 #define for_each_cpu(cpu, mask)				\  argument
333 	for_each_set_bit(cpu, cpumask_bits(mask), small_cpumask_bits)
336  * for_each_cpu_wrap - iterate over every cpu in a mask, starting at a specified location
337  * @cpu: the (optionally unsigned) integer iterator
343  * After the loop, cpu is >= nr_cpu_ids.
345 #define for_each_cpu_wrap(cpu, mask, start)				\  argument
346 	for_each_set_bit_wrap(cpu, cpumask_bits(mask), small_cpumask_bits, start)
349  * for_each_cpu_and - iterate over every cpu in both masks
350  * @cpu: the (optionally unsigned) integer iterator
354  * This saves a temporary CPU mask in many places.  It is equivalent to:
357  *	for_each_cpu(cpu, &tmp)
360  * After the loop, cpu is >= nr_cpu_ids.
362 #define for_each_cpu_and(cpu, mask1, mask2)				\  argument
363 	for_each_and_bit(cpu, cpumask_bits(mask1), cpumask_bits(mask2), small_cpumask_bits)
366  * for_each_cpu_andnot - iterate over every cpu present in one mask, excluding
368  * @cpu: the (optionally unsigned) integer iterator
372  * This saves a temporary CPU mask in many places.  It is equivalent to:
375  *	for_each_cpu(cpu, &tmp)
378  * After the loop, cpu is >= nr_cpu_ids.
380 #define for_each_cpu_andnot(cpu, mask1, mask2)				\  argument
381 	for_each_andnot_bit(cpu, cpumask_bits(mask1), cpumask_bits(mask2), small_cpumask_bits)
384  * for_each_cpu_or - iterate over every cpu present in either mask
385  * @cpu: the (optionally unsigned) integer iterator
389  * This saves a temporary CPU mask in many places.  It is equivalent to:
392  *	for_each_cpu(cpu, &tmp)
395  * After the loop, cpu is >= nr_cpu_ids.
397 #define for_each_cpu_or(cpu, mask1, mask2)				\  argument
398 	for_each_or_bit(cpu, cpumask_bits(mask1), cpumask_bits(mask2), small_cpumask_bits)
401  * for_each_cpu_from - iterate over CPUs present in @mask, from @cpu to the end of @mask.
402  * @cpu: the (optionally unsigned) integer iterator
405  * After the loop, cpu is >= nr_cpu_ids.
407 #define for_each_cpu_from(cpu, mask)				\  argument
408 	for_each_set_bit_from(cpu, cpumask_bits(mask), small_cpumask_bits)
411  * cpumask_any_but - return an arbitrary cpu in a cpumask, but not this one.
413  * @cpu: the cpu to ignore.
415  * Often used to find any cpu but smp_processor_id() in a mask.
419 unsigned int cpumask_any_but(const struct cpumask *mask, unsigned int cpu)  in cpumask_any_but()  argument
423 	cpumask_check(cpu);  in cpumask_any_but()
425 		if (i != cpu)  in cpumask_any_but()
431  * cpumask_any_and_but - pick an arbitrary cpu from *mask1 & *mask2, but not this one.
434  * @cpu: the cpu to ignore
441 				 unsigned int cpu)  in cpumask_any_and_but()
445 	cpumask_check(cpu);  in cpumask_any_and_but()
447 	if (i != cpu)  in cpumask_any_and_but()
450 	return cpumask_next_and(cpu, mask1, mask2);  in cpumask_any_and_but()
454  * cpumask_nth - get the Nth cpu in a cpumask
456  * @cpu: the Nth cpu to find, starting from 0
458  * Return: >= nr_cpu_ids if such cpu doesn't exist.
461 unsigned int cpumask_nth(unsigned int cpu, const struct cpumask *srcp)  in cpumask_nth()  argument
463 	return find_nth_bit(cpumask_bits(srcp), small_cpumask_bits, cpumask_check(cpu));  in cpumask_nth()
467  * cpumask_nth_and - get the Nth cpu in 2 cpumasks
470  * @cpu: the Nth cpu to find, starting from 0
472  * Return: >= nr_cpu_ids if such cpu doesn't exist.
475 unsigned int cpumask_nth_and(unsigned int cpu, const struct cpumask *srcp1,  in cpumask_nth_and()  argument
479 				small_cpumask_bits, cpumask_check(cpu));  in cpumask_nth_and()
483  * cpumask_nth_andnot - get the Nth cpu set in 1st cpumask, and clear in 2nd.
486  * @cpu: the Nth cpu to find, starting from 0
488  * Return: >= nr_cpu_ids if such cpu doesn't exist.
491 unsigned int cpumask_nth_andnot(unsigned int cpu, const struct cpumask *srcp1,  in cpumask_nth_andnot()  argument
495 				small_cpumask_bits, cpumask_check(cpu));  in cpumask_nth_andnot()
499  * cpumask_nth_and_andnot - get the Nth cpu set in 1st and 2nd cpumask, and clear in 3rd.
503  * @cpu: the Nth cpu to find, starting from 0
505  * Return: >= nr_cpu_ids if such cpu doesn't exist.
508 unsigned int cpumask_nth_and_andnot(unsigned int cpu, const struct cpumask *srcp1,  in cpumask_nth_and_andnot()  argument
515 					small_cpumask_bits, cpumask_check(cpu));  in cpumask_nth_and_andnot()
520 	[0 ... BITS_TO_LONGS(NR_CPUS)-1] = 0UL			\
525 	[0] =  1UL						\
529  * cpumask_set_cpu - set a cpu in a cpumask
530  * @cpu: cpu number (< nr_cpu_ids)
534 void cpumask_set_cpu(unsigned int cpu, struct cpumask *dstp)  in cpumask_set_cpu()  argument
536 	set_bit(cpumask_check(cpu), cpumask_bits(dstp));  in cpumask_set_cpu()
540 void __cpumask_set_cpu(unsigned int cpu, struct cpumask *dstp)  in __cpumask_set_cpu()  argument
542 	__set_bit(cpumask_check(cpu), cpumask_bits(dstp));  in __cpumask_set_cpu()
547  * cpumask_clear_cpu - clear a cpu in a cpumask
548  * @cpu: cpu number (< nr_cpu_ids)
551 static __always_inline void cpumask_clear_cpu(int cpu, struct cpumask *dstp)  in cpumask_clear_cpu()  argument
553 	clear_bit(cpumask_check(cpu), cpumask_bits(dstp));  in cpumask_clear_cpu()
556 static __always_inline void __cpumask_clear_cpu(int cpu, struct cpumask *dstp)  in __cpumask_clear_cpu()  argument
558 	__clear_bit(cpumask_check(cpu), cpumask_bits(dstp));  in __cpumask_clear_cpu()
562  * cpumask_assign_cpu - assign a cpu in a cpumask
563  * @cpu: cpu number (< nr_cpu_ids)
567 static __always_inline void cpumask_assign_cpu(int cpu, struct cpumask *dstp, bool value)  in cpumask_assign_cpu()  argument
569 	assign_bit(cpumask_check(cpu), cpumask_bits(dstp), value);  in cpumask_assign_cpu()
572 static __always_inline void __cpumask_assign_cpu(int cpu, struct cpumask *dstp, bool value)  in __cpumask_assign_cpu()  argument
574 	__assign_bit(cpumask_check(cpu), cpumask_bits(dstp), value);  in __cpumask_assign_cpu()
578  * cpumask_test_cpu - test for a cpu in a cpumask
579  * @cpu: cpu number (< nr_cpu_ids)
582  * Return: true if @cpu is set in @cpumask, else returns false
585 bool cpumask_test_cpu(int cpu, const struct cpumask *cpumask)  in cpumask_test_cpu()  argument
587 	return test_bit(cpumask_check(cpu), cpumask_bits((cpumask)));  in cpumask_test_cpu()
591  * cpumask_test_and_set_cpu - atomically test and set a cpu in a cpumask
592  * @cpu: cpu number (< nr_cpu_ids)
597  * Return: true if @cpu is set in old bitmap of @cpumask, else returns false
600 bool cpumask_test_and_set_cpu(int cpu, struct cpumask *cpumask)  in cpumask_test_and_set_cpu()  argument
602 	return test_and_set_bit(cpumask_check(cpu), cpumask_bits(cpumask));  in cpumask_test_and_set_cpu()
606  * cpumask_test_and_clear_cpu - atomically test and clear a cpu in a cpumask
607  * @cpu: cpu number (< nr_cpu_ids)
612  * Return: true if @cpu is set in old bitmap of @cpumask, else returns false
615 bool cpumask_test_and_clear_cpu(int cpu, struct cpumask *cpumask)  in cpumask_test_and_clear_cpu()  argument
617 	return test_and_clear_bit(cpumask_check(cpu), cpumask_bits(cpumask));  in cpumask_test_and_clear_cpu()
621  * cpumask_setall - set all cpus (< nr_cpu_ids) in a cpumask
634  * cpumask_clear - clear all cpus (< nr_cpu_ids) in a cpumask
643  * cpumask_and - *dstp = *src1p & *src2p
659  * cpumask_or - *dstp = *src1p | *src2p
673  * cpumask_xor - *dstp = *src1p ^ *src2p
687  * cpumask_andnot - *dstp = *src1p & ~*src2p
703  * cpumask_equal - *src1p == *src2p
717  * cpumask_or_equal - *src1p | *src2p == *src3p
734  * cpumask_intersects - (*src1p & *src2p) != 0
738  * Return: true if first cpumask ANDed with second cpumask is non-empty,
749  * cpumask_subset - (*src1p & ~*src2p) == 0
763  * cpumask_empty - *srcp == 0
774  * cpumask_full - *srcp == 0xFFFFFFFF...
785  * cpumask_weight - Count of bits in *srcp
796  * cpumask_weight_and - Count of bits in (*srcp1 & *srcp2)
809  * cpumask_weight_andnot - Count of bits in (*srcp1 & ~*srcp2)
823  * cpumask_shift_right - *dstp = *srcp >> n
836  * cpumask_shift_left - *dstp = *srcp << n
849  * cpumask_copy - *dstp = *srcp
860  * cpumask_any - pick an arbitrary cpu from *srcp
868  * cpumask_any_and - pick an arbitrary cpu from *mask1 & *mask2
877  * cpumask_of - the cpumask containing just a given cpu
878  * @cpu: the cpu (<= nr_cpu_ids)
880 #define cpumask_of(cpu) (get_cpu_mask(cpu))  argument
883  * cpumask_parse_user - extract a cpumask from a user string
888  * Return: -errno, or 0 for success.
897  * cpumask_parselist_user - extract a cpumask from a user string
902  * Return: -errno, or 0 for success.
912  * cpumask_parse - extract a cpumask from a string
916  * Return: -errno, or 0 for success.
924  * cpulist_parse - extract a cpumask from a user string of ranges
928  * Return: -errno, or 0 for success.
936  * cpumask_size - calculate size to allocate for a 'struct cpumask' in bytes
959  * alloc_cpumask_var - allocate a struct cpumask
964  * a nop returning a constant 1 (in <linux/cpumask.h>).
1048 #if NR_CPUS == 1
1049 /* Uniprocessor: the possible/online/present masks are always "1" */
1050 #define for_each_possible_cpu(cpu)	for ((cpu) = 0; (cpu) < 1; (cpu)++)  argument
1051 #define for_each_online_cpu(cpu)	for ((cpu) = 0; (cpu) < 1; (cpu)++)  argument
1052 #define for_each_present_cpu(cpu)	for ((cpu) = 0; (cpu) < 1; (cpu)++)  argument
1054 #define for_each_possible_cpu_wrap(cpu, start)	\  argument
1055 	for ((void)(start), (cpu) = 0; (cpu) < 1; (cpu)++)
1056 #define for_each_online_cpu_wrap(cpu, start)	\  argument
1057 	for ((void)(start), (cpu) = 0; (cpu) < 1; (cpu)++)
1059 #define for_each_possible_cpu(cpu) for_each_cpu((cpu), cpu_possible_mask)  argument
1060 #define for_each_online_cpu(cpu)   for_each_cpu((cpu), cpu_online_mask)  argument
1061 #define for_each_enabled_cpu(cpu)   for_each_cpu((cpu), cpu_enabled_mask)  argument
1062 #define for_each_present_cpu(cpu)  for_each_cpu((cpu), cpu_present_mask)  argument
1064 #define for_each_possible_cpu_wrap(cpu, start)	\  argument
1065 	for_each_cpu_wrap((cpu), cpu_possible_mask, (start))
1066 #define for_each_online_cpu_wrap(cpu, start)	\  argument
1067 	for_each_cpu_wrap((cpu), cpu_online_mask, (start))
1070 /* Wrappers for arch boot code to manipulate normally-constant masks */
1074 #define assign_cpu(cpu, mask, val)	\  argument
1075 	assign_bit(cpumask_check(cpu), cpumask_bits(mask), (val))
1077 #define set_cpu_possible(cpu, possible)	assign_cpu((cpu), &__cpu_possible_mask, (possible))  argument
1078 #define set_cpu_enabled(cpu, enabled)	assign_cpu((cpu), &__cpu_enabled_mask, (enabled))  argument
1079 #define set_cpu_present(cpu, present)	assign_cpu((cpu), &__cpu_present_mask, (present))  argument
1080 #define set_cpu_active(cpu, active)	assign_cpu((cpu), &__cpu_active_mask, (active))  argument
1081 #define set_cpu_dying(cpu, dying)	assign_cpu((cpu), &__cpu_dying_mask, (dying))  argument
1083 void set_cpu_online(unsigned int cpu, bool online);
1086  * to_cpumask - convert a NR_CPUS bitmap to a struct cpumask *
1096 	((struct cpumask *)(1 ? (bitmap)				\
1101 	return 1;  in __check_is_bitmap()
1105  * Special-case data structure for "single bit set only" constant CPU masks.
1107  * We pre-generate all the 64 (or 32) possible bit positions, with enough
1112 	cpu_bit_bitmap[BITS_PER_LONG+1][BITS_TO_LONGS(NR_CPUS)];
1114 static __always_inline const struct cpumask *get_cpu_mask(unsigned int cpu)  in get_cpu_mask()  argument
1116 	const unsigned long *p = cpu_bit_bitmap[1 + cpu % BITS_PER_LONG];  in get_cpu_mask()
1117 	p -= cpu / BITS_PER_LONG;  in get_cpu_mask()
1121 #if NR_CPUS > 1
1123  * num_online_cpus() - Read the number of online CPUs
1127  * concurrent CPU hotplug operations unless invoked from a cpuhp_lock held
1141 static __always_inline bool cpu_online(unsigned int cpu)  in cpu_online()  argument
1143 	return cpumask_test_cpu(cpu, cpu_online_mask);  in cpu_online()
1146 static __always_inline bool cpu_enabled(unsigned int cpu)  in cpu_enabled()  argument
1148 	return cpumask_test_cpu(cpu, cpu_enabled_mask);  in cpu_enabled()
1151 static __always_inline bool cpu_possible(unsigned int cpu)  in cpu_possible()  argument
1153 	return cpumask_test_cpu(cpu, cpu_possible_mask);  in cpu_possible()
1156 static __always_inline bool cpu_present(unsigned int cpu)  in cpu_present()  argument
1158 	return cpumask_test_cpu(cpu, cpu_present_mask);  in cpu_present()
1161 static __always_inline bool cpu_active(unsigned int cpu)  in cpu_active()  argument
1163 	return cpumask_test_cpu(cpu, cpu_active_mask);  in cpu_active()
1166 static __always_inline bool cpu_dying(unsigned int cpu)  in cpu_dying()  argument
1168 	return cpumask_test_cpu(cpu, cpu_dying_mask);  in cpu_dying()
1173 #define num_online_cpus()	1U
1174 #define num_possible_cpus()	1U
1175 #define num_enabled_cpus()	1U
1176 #define num_present_cpus()	1U
1177 #define num_active_cpus()	1U
1179 static __always_inline bool cpu_online(unsigned int cpu)  in cpu_online()  argument
1181 	return cpu == 0;  in cpu_online()
1184 static __always_inline bool cpu_possible(unsigned int cpu)  in cpu_possible()  argument
1186 	return cpu == 0;  in cpu_possible()
1189 static __always_inline bool cpu_enabled(unsigned int cpu)  in cpu_enabled()  argument
1191 	return cpu == 0;  in cpu_enabled()
1194 static __always_inline bool cpu_present(unsigned int cpu)  in cpu_present()  argument
1196 	return cpu == 0;  in cpu_present()
1199 static __always_inline bool cpu_active(unsigned int cpu)  in cpu_active()  argument
1201 	return cpu == 0;  in cpu_active()
1204 static __always_inline bool cpu_dying(unsigned int cpu)  in cpu_dying()  argument
1209 #endif /* NR_CPUS > 1 */
1211 #define cpu_is_offline(cpu)	unlikely(!cpu_online(cpu))  argument
1216 	[BITS_TO_LONGS(NR_CPUS)-1] = BITMAP_LAST_WORD_MASK(NR_CPUS)	\
1223 	[0 ... BITS_TO_LONGS(NR_CPUS)-2] = ~0UL,		\
1224 	[BITS_TO_LONGS(NR_CPUS)-1] = BITMAP_LAST_WORD_MASK(NR_CPUS)	\
1229  * cpumap_print_to_pagebuf  - copies the cpumask into the buffer either
1230  *	as comma-separated list of cpus or hex values of cpumask
1235  * Return: the length of the (null-terminated) @buf string, zero if
1246  * cpumap_print_bitmask_to_buf  - copies the cpumask into the buffer as
1266 				   nr_cpu_ids, off, count) - 1;  in cpumap_print_bitmask_to_buf()
1270  * cpumap_print_list_to_buf  - copies the cpumask into the buffer as
1271  *	comma-separated list of cpus
1288 				   nr_cpu_ids, off, count) - 1;  in cpumap_print_list_to_buf()
1294 	[BITS_TO_LONGS(NR_CPUS)-1] = BITMAP_LAST_WORD_MASK(NR_CPUS)	\
1299 	[0 ... BITS_TO_LONGS(NR_CPUS)-2] = ~0UL,			\
1300 	[BITS_TO_LONGS(NR_CPUS)-1] = BITMAP_LAST_WORD_MASK(NR_CPUS)	\
1306 	[0 ... BITS_TO_LONGS(NR_CPUS)-1] =  0UL				\
1311 	[0] =  1UL							\
1319  * for cpumap NR_CPUS * 9/32 - 1 should be an exact length.
1321  * For cpulist 7 is (ceil(log10(NR_CPUS)) + 1) allowing for NR_CPUS to be up
1323  * cover a worst-case of every other cpu being on one of two nodes for a
1327  *  unsigned comparison to -1.
1330 					? (NR_CPUS * 9)/32 - 1 : PAGE_SIZE)