15  * interleave     Allocate memory interleaved over a set of nodes,
22  * bind           Only allocate memory on a specific set of nodes,
26  *                the allocation to memory nodes instead
34  * preferred many Try a set of nodes first before normal fallback. This is
186 	int (*create)(struct mempolicy *pol, const nodemask_t *nodes);
187 	void (*rebind)(struct mempolicy *pol, const nodemask_t *nodes);
203 static int mpol_new_nodemask(struct mempolicy *pol, const nodemask_t *nodes)  in mpol_new_nodemask()  argument
205 	if (nodes_empty(*nodes))  in mpol_new_nodemask()
207 	pol->nodes = *nodes;  in mpol_new_nodemask()
211 static int mpol_new_preferred(struct mempolicy *pol, const nodemask_t *nodes)  in mpol_new_preferred()  argument
213 	if (nodes_empty(*nodes))  in mpol_new_preferred()
216 	nodes_clear(pol->nodes);  in mpol_new_preferred()
217 	node_set(first_node(*nodes), pol->nodes);  in mpol_new_preferred()
223  * any, for the new policy.  mpol_new() has already validated the nodes
230 		     const nodemask_t *nodes, struct nodemask_scratch *nsc)  in mpol_set_nodemask()  argument
246 	VM_BUG_ON(!nodes);  in mpol_set_nodemask()
249 		mpol_relative_nodemask(&nsc->mask2, nodes, &nsc->mask1);  in mpol_set_nodemask()
251 		nodes_and(nsc->mask2, *nodes, nsc->mask1);  in mpol_set_nodemask()
254 		pol->w.user_nodemask = *nodes;  in mpol_set_nodemask()
264  * initialization. You must invoke mpol_set_nodemask() to set nodes.
267 				  nodemask_t *nodes)  in mpol_new()  argument
272 		if (nodes && !nodes_empty(*nodes))  in mpol_new()
276 	VM_BUG_ON(!nodes);  in mpol_new()
284 		if (nodes_empty(*nodes)) {  in mpol_new()
292 		if (!nodes_empty(*nodes) ||  in mpol_new()
296 	} else if (nodes_empty(*nodes))  in mpol_new()
318 static void mpol_rebind_default(struct mempolicy *pol, const nodemask_t *nodes)  in mpol_rebind_default()  argument
322 static void mpol_rebind_nodemask(struct mempolicy *pol, const nodemask_t *nodes)  in mpol_rebind_nodemask()  argument
327 		nodes_and(tmp, pol->w.user_nodemask, *nodes);  in mpol_rebind_nodemask()
329 		mpol_relative_nodemask(&tmp, &pol->w.user_nodemask, nodes);  in mpol_rebind_nodemask()
331 		nodes_remap(tmp, pol->nodes, pol->w.cpuset_mems_allowed,  in mpol_rebind_nodemask()
332 								*nodes);  in mpol_rebind_nodemask()
333 		pol->w.cpuset_mems_allowed = *nodes;  in mpol_rebind_nodemask()
337 		tmp = *nodes;  in mpol_rebind_nodemask()
339 	pol->nodes = tmp;  in mpol_rebind_nodemask()
343 						const nodemask_t *nodes)  in mpol_rebind_preferred()  argument
345 	pol->w.cpuset_mems_allowed = *nodes;  in mpol_rebind_preferred()
349  * mpol_rebind_policy - Migrate a policy to a different set of nodes
689 	 * Check page nodes, and queue pages to move, in the current vma.  in queue_pages_test_walk()
714  * If pages found in a given range are not on the required set of @nodes,
727 		nodemask_t *nodes, unsigned long flags,  in queue_pages_range()  argument
734 		.nmask = nodes,  in queue_pages_range()
814 			     nodemask_t *nodes)  in do_set_mempolicy()  argument
823 	new = mpol_new(mode, flags, nodes);  in do_set_mempolicy()
830 	ret = mpol_set_nodemask(new, nodes, scratch);  in do_set_mempolicy()
854 static void get_policy_nodemask(struct mempolicy *pol, nodemask_t *nodes)  in get_policy_nodemask()  argument
856 	nodes_clear(*nodes);  in get_policy_nodemask()
865 		*nodes = pol->nodes;  in get_policy_nodemask()
948 			*policy = next_node_in(current->il_prev, pol->nodes);  in do_get_mempolicy()
1091 	 * This lets us pick a pair of nodes to migrate between, such that  in do_migrate_pages()
1120 			 * However if the number of source nodes is not equal to  in do_migrate_pages()
1121 			 * the number of destination nodes we can not preserve  in do_migrate_pages()
1141 			/* dest not in remaining from nodes? */  in do_migrate_pages()
1309 		 * targeted nodes, for the first VMA to be migrated; for later  in do_mbind()
1310 		 * VMAs, the nodes will still be interleaved from the targeted  in do_mbind()
1388 static int get_nodes(nodemask_t *nodes, const unsigned long __user *nmask,  in get_nodes()  argument
1392 	nodes_clear(*nodes);  in get_nodes()
1399 	 * When the user specified more nodes than supported just check  in get_nodes()
1420 	return get_bitmap(nodes_addr(*nodes), nmask, maxnode);  in get_nodes()
1425 			      nodemask_t *nodes)  in copy_nodes_to_user()  argument
1445 					 nodes_addr(*nodes), maxnode);  in copy_nodes_to_user()
1447 	return copy_to_user(mask, nodes_addr(*nodes), copy) ? -EFAULT : 0;  in copy_nodes_to_user()
1473 	nodemask_t nodes;  in kernel_mbind()  local
1482 	err = get_nodes(&nodes, nmask, maxnode);  in kernel_mbind()
1486 	return do_mbind(start, len, lmode, mode_flags, &nodes, flags);  in kernel_mbind()
1568 	nodemask_t nodes;  in kernel_set_mempolicy()  local
1576 	err = get_nodes(&nodes, nmask, maxnode);  in kernel_set_mempolicy()
1580 	return do_set_mempolicy(lmode, mode_flags, &nodes);  in kernel_set_mempolicy()
1639 	/* Is the user allowed to access the target nodes? */  in kernel_migrate_pages()
1692 	nodemask_t nodes;  in kernel_get_mempolicy()  local
1699 	err = do_get_mempolicy(&pval, &nodes, addr, flags);  in kernel_get_mempolicy()
1708 		err = copy_nodes_to_user(nmask, maxnode, &nodes);  in kernel_get_mempolicy()
1815 	 * if policy->nodes has movable memory only,  in apply_policy_zone()
1818 	 * policy->nodes is intersect with node_states[N_MEMORY].  in apply_policy_zone()
1820 	 * policy->nodes has movable memory only.  in apply_policy_zone()
1822 	if (!nodes_intersects(policy->nodes, node_states[N_HIGH_MEMORY]))  in apply_policy_zone()
1833 	nid = next_node_in(current->il_prev, policy->nodes);  in interleave_nodes()
1857 		return first_node(policy->nodes);  in mempolicy_slab_node()
1875 							&policy->nodes);  in mempolicy_slab_node()
1888  * node in pol->nodes (starting from ilx=0), wrapping around if ilx
1889  * exceeds the number of present nodes.
1893 	nodemask_t nodemask = pol->nodes;  in interleave_nid()
1901 	 * Between first_node() and next_node(), pol->nodes could be changed  in interleave_nid()
1902 	 * by other threads. So we put pol->nodes in a local stack.  in interleave_nid()
1917  * Return a nodemask representing a mempolicy for filtering nodes for
1928 		*nid = first_node(pol->nodes);  in policy_nodemask()
1931 		nodemask = &pol->nodes;  in policy_nodemask()
1938 		    cpuset_nodemask_valid_mems_allowed(&pol->nodes))  in policy_nodemask()
1939 			nodemask = &pol->nodes;  in policy_nodemask()
2015 		*mask = mempolicy->nodes;  in init_nodemask_of_mempolicy()
2037  * memory allocated from all nodes in system.
2053 		ret = nodes_intersects(mempolicy->nodes, *mask);  in mempolicy_in_oom_domain()
2067 	 * preferred nodes but skip the direct reclaim and allow the  in alloc_pages_preferred_many()
2069 	 * nodes in system.  in alloc_pages_preferred_many()
2108 		 * node and don't fall back to other nodes, as the cost of  in alloc_pages_mpol()
2219 	int nodes;  in alloc_pages_bulk_array_interleave()  local
2226 	nodes = nodes_weight(pol->nodes);  in alloc_pages_bulk_array_interleave()
2227 	nr_pages_per_node = nr_pages / nodes;  in alloc_pages_bulk_array_interleave()
2228 	delta = nr_pages - nodes * nr_pages_per_node;  in alloc_pages_bulk_array_interleave()
2230 	for (i = 0; i < nodes; i++) {  in alloc_pages_bulk_array_interleave()
2260 	nr_allocated  = __alloc_pages_bulk(preferred_gfp, nid, &pol->nodes,  in alloc_pages_bulk_array_preferred_many()
2365 		return !!nodes_equal(a->nodes, b->nodes);  in __mpol_equal()
2502 		if (node_isset(curnid, pol->nodes))  in mpol_misplaced()
2504 		polnid = first_node(pol->nodes);  in mpol_misplaced()
2512 		/* Optimize placement among multiple nodes via NUMA balancing */  in mpol_misplaced()
2514 			if (node_isset(thisnid, pol->nodes))  in mpol_misplaced()
2524 		 * If no allowed nodes, use current [!misplaced].  in mpol_misplaced()
2526 		if (node_isset(curnid, pol->nodes))  in mpol_misplaced()
2531 				&pol->nodes);  in mpol_misplaced()
2826 			.nodes = nodemask_of_node(nid),  in numa_policy_init()
2832 	 * enabled across suitably sized nodes (default is >= 16MB), or  in numa_policy_init()
2894 	nodemask_t nodes;  in mpol_parse_str()  local
2905 		if (nodelist_parse(nodelist, nodes))  in mpol_parse_str()
2907 		if (!nodes_subset(nodes, node_states[N_MEMORY]))  in mpol_parse_str()
2910 		nodes_clear(nodes);  in mpol_parse_str()
2920 		 * we use first_node(nodes) to grab a single node, so here  in mpol_parse_str()
2921 		 * nodelist (or nodes) cannot be empty.  in mpol_parse_str()
2929 			if (nodes_empty(nodes))  in mpol_parse_str()
2935 		 * Default to online nodes with memory if no nodelist  in mpol_parse_str()
2938 			nodes = node_states[N_MEMORY];  in mpol_parse_str()
2977 	new = mpol_new(mode, mode_flags, &nodes);  in mpol_parse_str()
2982 	 * Save nodes for mpol_to_str() to show the tmpfs mount options  in mpol_parse_str()
2986 		new->nodes = nodes;  in mpol_parse_str()
2988 		nodes_clear(new->nodes);  in mpol_parse_str()
2989 		node_set(first_node(nodes), new->nodes);  in mpol_parse_str()
2995 	 * Save nodes for contextualization: this will be used to "clone"  in mpol_parse_str()
2998 	new->w.user_nodemask = nodes;  in mpol_parse_str()
3027 	nodemask_t nodes = NODE_MASK_NONE;  in mpol_to_str()  local
3044 		nodes = pol->nodes;  in mpol_to_str()
3066 	if (!nodes_empty(nodes))  in mpol_to_str()
3068 			       nodemask_pr_args(&nodes));  in mpol_to_str()