Lines Matching +full:current +full:- +full:mode
1 // SPDX-License-Identifier: GPL-2.0-only
120 * run-time system-wide default policy => local allocation
124 .mode = MPOL_PREFERRED,
131 * numa_map_to_online_node - Find closest online node
158 struct mempolicy *pol = p->mempolicy; in get_task_policy()
168 if (pol->mode) in get_task_policy()
182 return pol->flags & MPOL_MODE_FLAGS; in mpol_store_user_nodemask()
196 return -EINVAL; in mpol_new_interleave()
197 pol->v.nodes = *nodes; in mpol_new_interleave()
204 pol->flags |= MPOL_F_LOCAL; /* local allocation */ in mpol_new_preferred()
206 return -EINVAL; /* no allowed nodes */ in mpol_new_preferred()
208 pol->v.preferred_node = first_node(*nodes); in mpol_new_preferred()
215 return -EINVAL; in mpol_new_bind()
216 pol->v.nodes = *nodes; in mpol_new_bind()
223 * parameter with respect to the policy mode and flags. But, we need to
234 /* if mode is MPOL_DEFAULT, pol is NULL. This is right. */ in mpol_set_nodemask()
238 nodes_and(nsc->mask1, in mpol_set_nodemask()
242 if (pol->mode == MPOL_PREFERRED && nodes_empty(*nodes)) in mpol_set_nodemask()
245 if (pol->flags & MPOL_F_RELATIVE_NODES) in mpol_set_nodemask()
246 mpol_relative_nodemask(&nsc->mask2, nodes, &nsc->mask1); in mpol_set_nodemask()
248 nodes_and(nsc->mask2, *nodes, nsc->mask1); in mpol_set_nodemask()
251 pol->w.user_nodemask = *nodes; in mpol_set_nodemask()
253 pol->w.cpuset_mems_allowed = in mpol_set_nodemask()
258 ret = mpol_ops[pol->mode].create(pol, &nsc->mask2); in mpol_set_nodemask()
260 ret = mpol_ops[pol->mode].create(pol, NULL); in mpol_set_nodemask()
268 static struct mempolicy *mpol_new(unsigned short mode, unsigned short flags, in mpol_new() argument
273 pr_debug("setting mode %d flags %d nodes[0] %lx\n", in mpol_new()
274 mode, flags, nodes ? nodes_addr(*nodes)[0] : NUMA_NO_NODE); in mpol_new()
276 if (mode == MPOL_DEFAULT) { in mpol_new()
278 return ERR_PTR(-EINVAL); in mpol_new()
286 * All other modes require a valid pointer to a non-empty nodemask. in mpol_new()
288 if (mode == MPOL_PREFERRED) { in mpol_new()
292 return ERR_PTR(-EINVAL); in mpol_new()
294 } else if (mode == MPOL_LOCAL) { in mpol_new()
298 return ERR_PTR(-EINVAL); in mpol_new()
299 mode = MPOL_PREFERRED; in mpol_new()
301 return ERR_PTR(-EINVAL); in mpol_new()
304 return ERR_PTR(-ENOMEM); in mpol_new()
305 atomic_set(&policy->refcnt, 1); in mpol_new()
306 policy->mode = mode; in mpol_new()
307 policy->flags = flags; in mpol_new()
315 if (!atomic_dec_and_test(&p->refcnt)) in __mpol_put()
328 if (pol->flags & MPOL_F_STATIC_NODES) in mpol_rebind_nodemask()
329 nodes_and(tmp, pol->w.user_nodemask, *nodes); in mpol_rebind_nodemask()
330 else if (pol->flags & MPOL_F_RELATIVE_NODES) in mpol_rebind_nodemask()
331 mpol_relative_nodemask(&tmp, &pol->w.user_nodemask, nodes); in mpol_rebind_nodemask()
333 nodes_remap(tmp, pol->v.nodes,pol->w.cpuset_mems_allowed, in mpol_rebind_nodemask()
335 pol->w.cpuset_mems_allowed = *nodes; in mpol_rebind_nodemask()
341 pol->v.nodes = tmp; in mpol_rebind_nodemask()
349 if (pol->flags & MPOL_F_STATIC_NODES) { in mpol_rebind_preferred()
350 int node = first_node(pol->w.user_nodemask); in mpol_rebind_preferred()
353 pol->v.preferred_node = node; in mpol_rebind_preferred()
354 pol->flags &= ~MPOL_F_LOCAL; in mpol_rebind_preferred()
356 pol->flags |= MPOL_F_LOCAL; in mpol_rebind_preferred()
357 } else if (pol->flags & MPOL_F_RELATIVE_NODES) { in mpol_rebind_preferred()
358 mpol_relative_nodemask(&tmp, &pol->w.user_nodemask, nodes); in mpol_rebind_preferred()
359 pol->v.preferred_node = first_node(tmp); in mpol_rebind_preferred()
360 } else if (!(pol->flags & MPOL_F_LOCAL)) { in mpol_rebind_preferred()
361 pol->v.preferred_node = node_remap(pol->v.preferred_node, in mpol_rebind_preferred()
362 pol->w.cpuset_mems_allowed, in mpol_rebind_preferred()
364 pol->w.cpuset_mems_allowed = *nodes; in mpol_rebind_preferred()
369 * mpol_rebind_policy - Migrate a policy to a different set of nodes
371 * Per-vma policies are protected by mmap_lock. Allocations using per-task
372 * policies are protected by task->mems_allowed_seq to prevent a premature
379 if (!mpol_store_user_nodemask(pol) && !(pol->flags & MPOL_F_LOCAL) && in mpol_rebind_policy()
380 nodes_equal(pol->w.cpuset_mems_allowed, *newmask)) in mpol_rebind_policy()
383 mpol_ops[pol->mode].rebind(pol, newmask); in mpol_rebind_policy()
395 mpol_rebind_policy(tsk->mempolicy, new); in mpol_rebind_task()
401 * Call holding a reference to mm. Takes mm->mmap_lock during call.
409 for (vma = mm->mmap; vma; vma = vma->vm_next) in mpol_rebind_mm()
410 mpol_rebind_policy(vma->vm_policy, new); in mpol_rebind_mm()
445 * Check if the page's nid is in qp->nmask.
447 * If MPOL_MF_INVERT is set in qp->flags, check if the nid is
448 * in the invert of qp->nmask.
454 unsigned long flags = qp->flags; in queue_pages_required()
456 return node_isset(nid, *qp->nmask) == !(flags & MPOL_MF_INVERT); in queue_pages_required()
461 * 0 - pages are placed on the right node or queued successfully.
462 * 1 - there is unmovable page, and MPOL_MF_MOVE* & MPOL_MF_STRICT were
464 * 2 - THP was split.
465 * -EIO - is migration entry or only MPOL_MF_STRICT was specified and an
475 struct queue_pages *qp = walk->private; in queue_pages_pmd()
479 ret = -EIO; in queue_pages_pmd()
485 __split_huge_pmd(walk->vma, pmd, addr, false, NULL); in queue_pages_pmd()
492 flags = qp->flags; in queue_pages_pmd()
495 if (!vma_migratable(walk->vma) || in queue_pages_pmd()
496 migrate_page_add(page, qp->pagelist, flags)) { in queue_pages_pmd()
501 ret = -EIO; in queue_pages_pmd()
513 * 0 - pages are placed on the right node or queued successfully.
514 * 1 - there is unmovable page, and MPOL_MF_MOVE* & MPOL_MF_STRICT were
516 * -EIO - only MPOL_MF_STRICT was specified and an existing page was already
522 struct vm_area_struct *vma = walk->vma; in queue_pages_pte_range()
524 struct queue_pages *qp = walk->private; in queue_pages_pte_range()
525 unsigned long flags = qp->flags; in queue_pages_pte_range()
542 mapped_pte = pte = pte_offset_map_lock(walk->mm, pmd, addr, &ptl); in queue_pages_pte_range()
569 if (migrate_page_add(page, qp->pagelist, flags)) in queue_pages_pte_range()
580 return addr != end ? -EIO : 0; in queue_pages_pte_range()
589 struct queue_pages *qp = walk->private; in queue_pages_hugetlb()
590 unsigned long flags = (qp->flags & MPOL_MF_VALID); in queue_pages_hugetlb()
595 ptl = huge_pte_lock(hstate_vma(walk->vma), walk->mm, pte); in queue_pages_hugetlb()
608 ret = -EIO; in queue_pages_hugetlb()
612 if (!vma_migratable(walk->vma)) { in queue_pages_hugetlb()
615 * stopped walking current vma. in queue_pages_hugetlb()
626 if (!isolate_huge_page(page, qp->pagelist) && in queue_pages_hugetlb()
674 struct vm_area_struct *vma = walk->vma; in queue_pages_test_walk()
675 struct queue_pages *qp = walk->private; in queue_pages_test_walk()
676 unsigned long endvma = vma->vm_end; in queue_pages_test_walk()
677 unsigned long flags = qp->flags; in queue_pages_test_walk()
680 VM_BUG_ON_VMA((vma->vm_start > start) || (vma->vm_end < end), vma); in queue_pages_test_walk()
682 if (!qp->first) { in queue_pages_test_walk()
683 qp->first = vma; in queue_pages_test_walk()
685 (qp->start < vma->vm_start)) in queue_pages_test_walk()
687 return -EFAULT; in queue_pages_test_walk()
690 ((vma->vm_end < qp->end) && in queue_pages_test_walk()
691 (!vma->vm_next || vma->vm_end < vma->vm_next->vm_start))) in queue_pages_test_walk()
693 return -EFAULT; in queue_pages_test_walk()
696 * Need check MPOL_MF_STRICT to return -EIO if possible in queue_pages_test_walk()
709 !(vma->vm_flags & VM_MIXEDMAP)) in queue_pages_test_walk()
714 /* queue pages from current vma */ in queue_pages_test_walk()
734 * 1 - there is unmovable page, but MPOL_MF_MOVE* & MPOL_MF_STRICT were
736 * 0 - queue pages successfully or no misplaced page.
737 * errno - i.e. misplaced pages with MPOL_MF_STRICT specified (-EIO) or
739 * your accessible address space (-EFAULT)
760 err = -EFAULT; in queue_pages_range()
776 pr_debug("vma %lx-%lx/%lx vm_ops %p vm_file %p set_policy %p\n", in vma_replace_policy()
777 vma->vm_start, vma->vm_end, vma->vm_pgoff, in vma_replace_policy()
778 vma->vm_ops, vma->vm_file, in vma_replace_policy()
779 vma->vm_ops ? vma->vm_ops->set_policy : NULL); in vma_replace_policy()
785 if (vma->vm_ops && vma->vm_ops->set_policy) { in vma_replace_policy()
786 err = vma->vm_ops->set_policy(vma, new); in vma_replace_policy()
791 old = vma->vm_policy; in vma_replace_policy()
792 vma->vm_policy = new; /* protected by mmap_lock */ in vma_replace_policy()
816 prev = vma->vm_prev; in mbind_range()
817 if (start > vma->vm_start) in mbind_range()
820 for (; vma && vma->vm_start < end; prev = vma, vma = next) { in mbind_range()
821 next = vma->vm_next; in mbind_range()
822 vmstart = max(start, vma->vm_start); in mbind_range()
823 vmend = min(end, vma->vm_end); in mbind_range()
828 pgoff = vma->vm_pgoff + in mbind_range()
829 ((vmstart - vma->vm_start) >> PAGE_SHIFT); in mbind_range()
830 prev = vma_merge(mm, prev, vmstart, vmend, vma->vm_flags, in mbind_range()
831 vma->anon_vma, vma->vm_file, pgoff, in mbind_range()
832 new_pol, vma->vm_userfaultfd_ctx); in mbind_range()
835 next = vma->vm_next; in mbind_range()
838 /* vma_merge() joined vma && vma->next, case 8 */ in mbind_range()
841 if (vma->vm_start != vmstart) { in mbind_range()
842 err = split_vma(vma->vm_mm, vma, vmstart, 1); in mbind_range()
846 if (vma->vm_end != vmend) { in mbind_range()
847 err = split_vma(vma->vm_mm, vma, vmend, 0); in mbind_range()
862 static long do_set_mempolicy(unsigned short mode, unsigned short flags, in do_set_mempolicy() argument
870 return -ENOMEM; in do_set_mempolicy()
872 new = mpol_new(mode, flags, nodes); in do_set_mempolicy()
883 task_lock(current); in do_set_mempolicy()
884 old = current->mempolicy; in do_set_mempolicy()
885 current->mempolicy = new; in do_set_mempolicy()
886 if (new && new->mode == MPOL_INTERLEAVE) in do_set_mempolicy()
887 current->il_prev = MAX_NUMNODES-1; in do_set_mempolicy()
888 task_unlock(current); in do_set_mempolicy()
907 switch (p->mode) { in get_policy_nodemask()
910 *nodes = p->v.nodes; in get_policy_nodemask()
913 if (!(p->flags & MPOL_F_LOCAL)) in get_policy_nodemask()
914 node_set(p->v.preferred_node, *nodes); in get_policy_nodemask()
943 struct mm_struct *mm = current->mm; in do_get_mempolicy()
945 struct mempolicy *pol = current->mempolicy, *pol_refcount = NULL; in do_get_mempolicy()
949 return -EINVAL; in do_get_mempolicy()
953 return -EINVAL; in do_get_mempolicy()
955 task_lock(current); in do_get_mempolicy()
957 task_unlock(current); in do_get_mempolicy()
971 return -EFAULT; in do_get_mempolicy()
973 if (vma->vm_ops && vma->vm_ops->get_policy) in do_get_mempolicy()
974 pol = vma->vm_ops->get_policy(vma, addr); in do_get_mempolicy()
976 pol = vma->vm_policy; in do_get_mempolicy()
978 return -EINVAL; in do_get_mempolicy()
998 } else if (pol == current->mempolicy && in do_get_mempolicy()
999 pol->mode == MPOL_INTERLEAVE) { in do_get_mempolicy()
1000 *policy = next_node_in(current->il_prev, pol->v.nodes); in do_get_mempolicy()
1002 err = -EINVAL; in do_get_mempolicy()
1007 pol->mode; in do_get_mempolicy()
1012 *policy |= (pol->flags & MPOL_MODE_FLAGS); in do_get_mempolicy()
1018 *nmask = pol->w.user_nodemask; in do_get_mempolicy()
1020 task_lock(current); in do_get_mempolicy()
1022 task_unlock(current); in do_get_mempolicy()
1048 list_add_tail(&head->lru, pagelist); in migrate_page_add()
1054 * Non-movable page may reach here. And, there may be in migrate_page_add()
1055 * temporary off LRU pages or non-LRU movable pages. in migrate_page_add()
1058 * should return -EIO for this case too. in migrate_page_add()
1060 return -EIO; in migrate_page_add()
1091 queue_pages_range(mm, mm->mmap->vm_start, mm->task_size, &nmask, in migrate_to_node()
1173 * Example: [2,3,4] -> [3,4,5] moves everything. in do_migrate_pages()
1174 * [0-7] - > [3,4,5] moves only 0,1,2,6,7. in do_migrate_pages()
1213 * list of pages handed to migrate_pages()--which is how we get here--
1221 vma = find_vma(current->mm, start); in new_page()
1224 if (address != -EFAULT) in new_page()
1226 vma = vma->vm_next; in new_page()
1253 return -EIO; in migrate_page_add()
1259 return -ENOSYS; in do_migrate_pages()
1269 unsigned short mode, unsigned short mode_flags, in do_mbind() argument
1272 struct mm_struct *mm = current->mm; in do_mbind()
1280 return -EINVAL; in do_mbind()
1282 return -EPERM; in do_mbind()
1285 return -EINVAL; in do_mbind()
1287 if (mode == MPOL_DEFAULT) in do_mbind()
1290 len = (len + PAGE_SIZE - 1) & PAGE_MASK; in do_mbind()
1294 return -EINVAL; in do_mbind()
1298 new = mpol_new(mode, mode_flags, nmask); in do_mbind()
1303 new->flags |= MPOL_F_MOF; in do_mbind()
1312 pr_debug("mbind %lx-%lx mode:%d flags:%d nodes:%lx\n", in do_mbind()
1313 start, start + len, mode, mode_flags, in do_mbind()
1330 err = -ENOMEM; in do_mbind()
1358 err = -EIO; in do_mbind()
1384 --maxnode; in get_nodes()
1389 return -EINVAL; in get_nodes()
1395 endmask = (1UL << (maxnode % BITS_PER_LONG)) - 1; in get_nodes()
1409 return -EFAULT; in get_nodes()
1410 if (k == nlongs - 1) { in get_nodes()
1412 return -EINVAL; in get_nodes()
1414 return -EINVAL; in get_nodes()
1423 valid_mask &= ~((1UL << (MAX_NUMNODES % BITS_PER_LONG)) - 1); in get_nodes()
1424 if (get_user(t, nmask + nlongs - 1)) in get_nodes()
1425 return -EFAULT; in get_nodes()
1427 return -EINVAL; in get_nodes()
1431 return -EFAULT; in get_nodes()
1432 nodes_addr(*nodes)[nlongs-1] &= endmask; in get_nodes()
1440 unsigned long copy = ALIGN(maxnode-1, 64) / 8; in copy_nodes_to_user()
1445 return -EINVAL; in copy_nodes_to_user()
1446 if (clear_user((char __user *)mask + nbytes, copy - nbytes)) in copy_nodes_to_user()
1447 return -EFAULT; in copy_nodes_to_user()
1450 return copy_to_user(mask, nodes_addr(*nodes), copy) ? -EFAULT : 0; in copy_nodes_to_user()
1454 unsigned long mode, const unsigned long __user *nmask, in kernel_mbind() argument
1462 mode_flags = mode & MPOL_MODE_FLAGS; in kernel_mbind()
1463 mode &= ~MPOL_MODE_FLAGS; in kernel_mbind()
1464 if (mode >= MPOL_MAX) in kernel_mbind()
1465 return -EINVAL; in kernel_mbind()
1468 return -EINVAL; in kernel_mbind()
1472 return do_mbind(start, len, mode, mode_flags, &nodes, flags); in kernel_mbind()
1476 unsigned long, mode, const unsigned long __user *, nmask, in SYSCALL_DEFINE6() argument
1479 return kernel_mbind(start, len, mode, nmask, maxnode, flags); in SYSCALL_DEFINE6()
1483 static long kernel_set_mempolicy(int mode, const unsigned long __user *nmask, in kernel_set_mempolicy() argument
1490 flags = mode & MPOL_MODE_FLAGS; in kernel_set_mempolicy()
1491 mode &= ~MPOL_MODE_FLAGS; in kernel_set_mempolicy()
1492 if ((unsigned int)mode >= MPOL_MAX) in kernel_set_mempolicy()
1493 return -EINVAL; in kernel_set_mempolicy()
1495 return -EINVAL; in kernel_set_mempolicy()
1499 return do_set_mempolicy(mode, flags, &nodes); in kernel_set_mempolicy()
1502 SYSCALL_DEFINE3(set_mempolicy, int, mode, const unsigned long __user *, nmask, in SYSCALL_DEFINE3() argument
1505 return kernel_set_mempolicy(mode, nmask, maxnode); in SYSCALL_DEFINE3()
1521 return -ENOMEM; in kernel_migrate_pages()
1523 old = &scratch->mask1; in kernel_migrate_pages()
1524 new = &scratch->mask2; in kernel_migrate_pages()
1536 task = pid ? find_task_by_vpid(pid) : current; in kernel_migrate_pages()
1539 err = -ESRCH; in kernel_migrate_pages()
1544 err = -EINVAL; in kernel_migrate_pages()
1552 err = -EPERM; in kernel_migrate_pages()
1560 err = -EPERM; in kernel_migrate_pages()
1564 task_nodes = cpuset_mems_allowed(current); in kernel_migrate_pages()
1577 err = -EINVAL; in kernel_migrate_pages()
1616 return -EINVAL; in kernel_get_mempolicy()
1626 return -EFAULT; in kernel_get_mempolicy()
1653 nr_bits = min_t(unsigned long, maxnode-1, nr_node_ids); in COMPAT_SYSCALL_DEFINE5()
1666 err |= clear_user(nmask, ALIGN(maxnode-1, 8) / 8); in COMPAT_SYSCALL_DEFINE5()
1673 COMPAT_SYSCALL_DEFINE3(set_mempolicy, int, mode, compat_ulong_t __user *, nmask, in COMPAT_SYSCALL_DEFINE3() argument
1680 nr_bits = min_t(unsigned long, maxnode-1, MAX_NUMNODES); in COMPAT_SYSCALL_DEFINE3()
1685 return -EFAULT; in COMPAT_SYSCALL_DEFINE3()
1688 return -EFAULT; in COMPAT_SYSCALL_DEFINE3()
1691 return kernel_set_mempolicy(mode, nm, nr_bits+1); in COMPAT_SYSCALL_DEFINE3()
1695 compat_ulong_t, mode, compat_ulong_t __user *, nmask, in COMPAT_SYSCALL_DEFINE6() argument
1702 nr_bits = min_t(unsigned long, maxnode-1, MAX_NUMNODES); in COMPAT_SYSCALL_DEFINE6()
1707 return -EFAULT; in COMPAT_SYSCALL_DEFINE6()
1710 return -EFAULT; in COMPAT_SYSCALL_DEFINE6()
1713 return kernel_mbind(start, len, mode, nm, nr_bits+1, flags); in COMPAT_SYSCALL_DEFINE6()
1727 nr_bits = min_t(unsigned long, maxnode - 1, MAX_NUMNODES); in COMPAT_SYSCALL_DEFINE4()
1731 return -EFAULT; in COMPAT_SYSCALL_DEFINE4()
1736 return -EFAULT; in COMPAT_SYSCALL_DEFINE4()
1740 return -EFAULT; in COMPAT_SYSCALL_DEFINE4()
1744 return -EFAULT; in COMPAT_SYSCALL_DEFINE4()
1753 if (vma->vm_flags & (VM_IO | VM_PFNMAP)) in vma_migratable()
1772 if (vma->vm_file && in vma_migratable()
1773 gfp_zone(mapping_gfp_mask(vma->vm_file->f_mapping)) in vma_migratable()
1785 if (vma->vm_ops && vma->vm_ops->get_policy) { in __get_vma_policy()
1786 pol = vma->vm_ops->get_policy(vma, addr); in __get_vma_policy()
1787 } else if (vma->vm_policy) { in __get_vma_policy()
1788 pol = vma->vm_policy; in __get_vma_policy()
1792 * a pseudo vma whose vma->vm_ops=NULL. Take a reference in __get_vma_policy()
1810 * Falls back to current->mempolicy or system default policy, as necessary.
1812 * count--added by the get_policy() vm_op, as appropriate--to protect against
1822 pol = get_task_policy(current); in get_vma_policy()
1831 if (vma->vm_ops && vma->vm_ops->get_policy) { in vma_policy_mof()
1834 pol = vma->vm_ops->get_policy(vma, vma->vm_start); in vma_policy_mof()
1835 if (pol && (pol->flags & MPOL_F_MOF)) in vma_policy_mof()
1842 pol = vma->vm_policy; in vma_policy_mof()
1844 pol = get_task_policy(current); in vma_policy_mof()
1846 return pol->flags & MPOL_F_MOF; in vma_policy_mof()
1856 * if policy->v.nodes has movable memory only, in apply_policy_zone()
1859 * policy->v.nodes is intersect with node_states[N_MEMORY]. in apply_policy_zone()
1861 * policy->v.nodes has movable memory only. in apply_policy_zone()
1863 if (!nodes_intersects(policy->v.nodes, node_states[N_HIGH_MEMORY])) in apply_policy_zone()
1876 if (unlikely(policy->mode == MPOL_BIND) && in policy_nodemask()
1878 cpuset_nodemask_valid_mems_allowed(&policy->v.nodes)) in policy_nodemask()
1879 return &policy->v.nodes; in policy_nodemask()
1887 if (policy->mode == MPOL_PREFERRED && !(policy->flags & MPOL_F_LOCAL)) in policy_node()
1888 nd = policy->v.preferred_node; in policy_node()
1895 WARN_ON_ONCE(policy->mode == MPOL_BIND && (gfp & __GFP_THISNODE)); in policy_node()
1905 struct task_struct *me = current; in interleave_nodes()
1907 next = next_node_in(me->il_prev, policy->v.nodes); in interleave_nodes()
1909 me->il_prev = next; in interleave_nodes()
1925 policy = current->mempolicy; in mempolicy_slab_node()
1926 if (!policy || policy->flags & MPOL_F_LOCAL) in mempolicy_slab_node()
1929 switch (policy->mode) { in mempolicy_slab_node()
1934 return policy->v.preferred_node; in mempolicy_slab_node()
1948 zonelist = &NODE_DATA(node)->node_zonelists[ZONELIST_FALLBACK]; in mempolicy_slab_node()
1950 &policy->v.nodes); in mempolicy_slab_node()
1951 return z->zone ? zone_to_nid(z->zone) : node; in mempolicy_slab_node()
1961 * node in pol->v.nodes (starting from n=0), wrapping around if n exceeds the
1966 unsigned nnodes = nodes_weight(pol->v.nodes); in offset_il_node()
1974 nid = first_node(pol->v.nodes); in offset_il_node()
1976 nid = next_node(nid, pol->v.nodes); in offset_il_node()
1989 * shift and PAGE_SHIFT, so the bit-shift is safe. in interleave_nid()
1995 off = vma->vm_pgoff >> (shift - PAGE_SHIFT); in interleave_nid()
1996 off += (addr - vma->vm_start) >> shift; in interleave_nid()
2026 if (unlikely((*mpol)->mode == MPOL_INTERLEAVE)) { in huge_node()
2031 if ((*mpol)->mode == MPOL_BIND) in huge_node()
2032 *nodemask = &(*mpol)->v.nodes; in huge_node()
2040 * If the current task's mempolicy is "default" [NULL], return 'false'
2045 * of non-default mempolicy.
2048 * because the current task is examining it's own mempolicy and a task's
2058 if (!(mask && current->mempolicy)) in init_nodemask_of_mempolicy()
2061 task_lock(current); in init_nodemask_of_mempolicy()
2062 mempolicy = current->mempolicy; in init_nodemask_of_mempolicy()
2063 switch (mempolicy->mode) { in init_nodemask_of_mempolicy()
2065 if (mempolicy->flags & MPOL_F_LOCAL) in init_nodemask_of_mempolicy()
2068 nid = mempolicy->v.preferred_node; in init_nodemask_of_mempolicy()
2074 *mask = mempolicy->v.nodes; in init_nodemask_of_mempolicy()
2080 task_unlock(current); in init_nodemask_of_mempolicy()
2105 mempolicy = tsk->mempolicy; in mempolicy_nodemask_intersects()
2109 switch (mempolicy->mode) { in mempolicy_nodemask_intersects()
2120 ret = nodes_intersects(mempolicy->v.nodes, *mask); in mempolicy_nodemask_intersects()
2150 * alloc_pages_vma - Allocate a page for a VMA.
2166 * a NUMA policy associated with the VMA or the current process.
2167 * When VMA is not NULL caller must read-lock the mmap_lock of the
2183 if (pol->mode == MPOL_INTERLEAVE) { in alloc_pages_vma()
2196 * For hugepage allocation and non-interleave policy which in alloc_pages_vma()
2197 * allows the current node (or other explicitly preferred in alloc_pages_vma()
2198 * node) we only try to allocate from the current/preferred in alloc_pages_vma()
2202 * If the policy is interleave, or does not allow the current in alloc_pages_vma()
2205 if (pol->mode == MPOL_PREFERRED && !(pol->flags & MPOL_F_LOCAL)) in alloc_pages_vma()
2206 hpage_node = pol->v.preferred_node; in alloc_pages_vma()
2242 * alloc_pages_current - Allocate pages.
2253 * interrupt context and apply the current process NUMA policy.
2262 pol = get_task_policy(current); in alloc_pages_current()
2265 * No reference counting needed for current->mempolicy in alloc_pages_current()
2268 if (pol->mode == MPOL_INTERLEAVE) in alloc_pages_current()
2285 dst->vm_policy = pol; in vma_dup_policy()
2290 * If mpol_dup() sees current->cpuset == cpuset_being_rebound, then it
2296 * current's mempolicy may be rebinded by the other task(the task that changes
2297 * cpuset's mems), so we needn't do rebind work for current task.
2306 return ERR_PTR(-ENOMEM); in __mpol_dup()
2309 if (old == current->mempolicy) { in __mpol_dup()
2310 task_lock(current); in __mpol_dup()
2312 task_unlock(current); in __mpol_dup()
2317 nodemask_t mems = cpuset_mems_allowed(current); in __mpol_dup()
2320 atomic_set(&new->refcnt, 1); in __mpol_dup()
2329 if (a->mode != b->mode) in __mpol_equal()
2331 if (a->flags != b->flags) in __mpol_equal()
2334 if (!nodes_equal(a->w.user_nodemask, b->w.user_nodemask)) in __mpol_equal()
2337 switch (a->mode) { in __mpol_equal()
2340 return !!nodes_equal(a->v.nodes, b->v.nodes); in __mpol_equal()
2342 /* a's ->flags is the same as b's */ in __mpol_equal()
2343 if (a->flags & MPOL_F_LOCAL) in __mpol_equal()
2345 return a->v.preferred_node == b->v.preferred_node; in __mpol_equal()
2356 * The policies are kept in Red-Black tree linked from the inode.
2357 * They are protected by the sp->lock rwlock, which should be held
2362 * lookup first element intersecting start-end. Caller holds sp->lock for
2368 struct rb_node *n = sp->root.rb_node; in sp_lookup()
2373 if (start >= p->end) in sp_lookup()
2374 n = n->rb_right; in sp_lookup()
2375 else if (end <= p->start) in sp_lookup()
2376 n = n->rb_left; in sp_lookup()
2388 if (w->end <= start) in sp_lookup()
2396 * Insert a new shared policy into the list. Caller holds sp->lock for
2401 struct rb_node **p = &sp->root.rb_node; in sp_insert()
2408 if (new->start < nd->start) in sp_insert()
2409 p = &(*p)->rb_left; in sp_insert()
2410 else if (new->end > nd->end) in sp_insert()
2411 p = &(*p)->rb_right; in sp_insert()
2415 rb_link_node(&new->nd, parent, p); in sp_insert()
2416 rb_insert_color(&new->nd, &sp->root); in sp_insert()
2417 pr_debug("inserting %lx-%lx: %d\n", new->start, new->end, in sp_insert()
2418 new->policy ? new->policy->mode : 0); in sp_insert()
2428 if (!sp->root.rb_node) in mpol_shared_policy_lookup()
2430 read_lock(&sp->lock); in mpol_shared_policy_lookup()
2433 mpol_get(sn->policy); in mpol_shared_policy_lookup()
2434 pol = sn->policy; in mpol_shared_policy_lookup()
2436 read_unlock(&sp->lock); in mpol_shared_policy_lookup()
2442 mpol_put(n->policy); in sp_free()
2447 * mpol_misplaced - check whether current page node is valid in policy
2453 * Lookup current policy node id for vma,addr and "compare to" page's
2457 * -1 - not misplaced, page is in the right node
2458 * node - node id where the page should be
2472 int ret = -1; in mpol_misplaced()
2475 if (!(pol->flags & MPOL_F_MOF)) in mpol_misplaced()
2478 switch (pol->mode) { in mpol_misplaced()
2480 pgoff = vma->vm_pgoff; in mpol_misplaced()
2481 pgoff += (addr - vma->vm_start) >> PAGE_SHIFT; in mpol_misplaced()
2486 if (pol->flags & MPOL_F_LOCAL) in mpol_misplaced()
2489 polnid = pol->v.preferred_node; in mpol_misplaced()
2496 * use current page if in policy nodemask, in mpol_misplaced()
2498 * If no allowed nodes, use current [!misplaced]. in mpol_misplaced()
2500 if (node_isset(curnid, pol->v.nodes)) in mpol_misplaced()
2505 &pol->v.nodes); in mpol_misplaced()
2506 polnid = zone_to_nid(z->zone); in mpol_misplaced()
2514 if (pol->flags & MPOL_F_MORON) { in mpol_misplaced()
2517 if (!should_numa_migrate_memory(current, page, curnid, thiscpu)) in mpol_misplaced()
2530 * Drop the (possibly final) reference to task->mempolicy. It needs to be
2531 * dropped after task->mempolicy is set to NULL so that any allocation done as
2540 pol = task->mempolicy; in mpol_put_task_policy()
2541 task->mempolicy = NULL; in mpol_put_task_policy()
2548 pr_debug("deleting %lx-l%lx\n", n->start, n->end); in sp_delete()
2549 rb_erase(&n->nd, &sp->root); in sp_delete()
2556 node->start = start; in sp_node_init()
2557 node->end = end; in sp_node_init()
2558 node->policy = pol; in sp_node_init()
2576 newpol->flags |= MPOL_F_SHARED; in sp_alloc()
2592 write_lock(&sp->lock); in shared_policy_replace()
2595 while (n && n->start < end) { in shared_policy_replace()
2596 struct rb_node *next = rb_next(&n->nd); in shared_policy_replace()
2597 if (n->start >= start) { in shared_policy_replace()
2598 if (n->end <= end) in shared_policy_replace()
2601 n->start = end; in shared_policy_replace()
2604 if (n->end > end) { in shared_policy_replace()
2608 *mpol_new = *n->policy; in shared_policy_replace()
2609 atomic_set(&mpol_new->refcnt, 1); in shared_policy_replace()
2610 sp_node_init(n_new, end, n->end, mpol_new); in shared_policy_replace()
2611 n->end = start; in shared_policy_replace()
2617 n->end = start; in shared_policy_replace()
2625 write_unlock(&sp->lock); in shared_policy_replace()
2637 write_unlock(&sp->lock); in shared_policy_replace()
2638 ret = -ENOMEM; in shared_policy_replace()
2649 * mpol_shared_policy_init - initialize shared policy for inode
2653 * Install non-NULL @mpol in inode's shared policy rb-tree.
2654 * On entry, the current task has a reference on a non-NULL @mpol.
2662 sp->root = RB_ROOT; /* empty tree == default mempolicy */ in mpol_shared_policy_init()
2663 rwlock_init(&sp->lock); in mpol_shared_policy_init()
2673 new = mpol_new(mpol->mode, mpol->flags, &mpol->w.user_nodemask); in mpol_shared_policy_init()
2677 task_lock(current); in mpol_shared_policy_init()
2678 ret = mpol_set_nodemask(new, &mpol->w.user_nodemask, scratch); in mpol_shared_policy_init()
2679 task_unlock(current); in mpol_shared_policy_init()
2683 /* Create pseudo-vma that contains just the policy */ in mpol_shared_policy_init()
2705 vma->vm_pgoff, in mpol_set_shared_policy()
2706 sz, npol ? npol->mode : -1, in mpol_set_shared_policy()
2707 npol ? npol->flags : -1, in mpol_set_shared_policy()
2708 npol ? nodes_addr(npol->v.nodes)[0] : NUMA_NO_NODE); in mpol_set_shared_policy()
2711 new = sp_alloc(vma->vm_pgoff, vma->vm_pgoff + sz, npol); in mpol_set_shared_policy()
2713 return -ENOMEM; in mpol_set_shared_policy()
2715 err = shared_policy_replace(info, vma->vm_pgoff, vma->vm_pgoff+sz, new); in mpol_set_shared_policy()
2727 if (!p->root.rb_node) in mpol_free_shared_policy()
2729 write_lock(&p->lock); in mpol_free_shared_policy()
2730 next = rb_first(&p->root); in mpol_free_shared_policy()
2733 next = rb_next(&n->nd); in mpol_free_shared_policy()
2736 write_unlock(&p->lock); in mpol_free_shared_policy()
2749 /* Parsed by setup_numabalancing. override == 1 enables, -1 disables */ in check_numabalancing_enable()
2770 numabalancing_override = -1; in setup_numabalancing()
2804 .mode = MPOL_PREFERRED, in numa_policy_init()
2840 /* Reset policy of current process to default */
2865 * mpol_parse_str - parse string to mempolicy, for tmpfs mpol mount option.
2870 * <mode>[=<flags>][:<nodelist>]
2881 int err = 1, mode; in mpol_parse_str() local
2884 *flags++ = '\0'; /* terminate mode string */ in mpol_parse_str()
2887 /* NUL-terminate mode or flags string */ in mpol_parse_str()
2896 mode = match_string(policy_modes, MPOL_MAX, str); in mpol_parse_str()
2897 if (mode < 0) in mpol_parse_str()
2900 switch (mode) { in mpol_parse_str()
2930 mode = MPOL_PREFERRED; in mpol_parse_str()
2951 * mode flags. in mpol_parse_str()
2961 new = mpol_new(mode, mode_flags, &nodes); in mpol_parse_str()
2969 if (mode != MPOL_PREFERRED) in mpol_parse_str()
2970 new->v.nodes = nodes; in mpol_parse_str()
2972 new->v.preferred_node = first_node(nodes); in mpol_parse_str()
2974 new->flags |= MPOL_F_LOCAL; in mpol_parse_str()
2980 new->w.user_nodemask = nodes; in mpol_parse_str()
2987 *--nodelist = ':'; in mpol_parse_str()
2989 *--flags = '='; in mpol_parse_str()
2997 * mpol_to_str - format a mempolicy structure for printing
3003 * Recommend a @maxlen of at least 32 for the longest mode, "interleave", the
3010 unsigned short mode = MPOL_DEFAULT; in mpol_to_str() local
3013 if (pol && pol != &default_policy && !(pol->flags & MPOL_F_MORON)) { in mpol_to_str()
3014 mode = pol->mode; in mpol_to_str()
3015 flags = pol->flags; in mpol_to_str()
3018 switch (mode) { in mpol_to_str()
3023 mode = MPOL_LOCAL; in mpol_to_str()
3025 node_set(pol->v.preferred_node, nodes); in mpol_to_str()
3029 nodes = pol->v.nodes; in mpol_to_str()
3037 p += snprintf(p, maxlen, "%s", policy_modes[mode]); in mpol_to_str()
3040 p += snprintf(p, buffer + maxlen - p, "="); in mpol_to_str()
3046 p += snprintf(p, buffer + maxlen - p, "static"); in mpol_to_str()
3048 p += snprintf(p, buffer + maxlen - p, "relative"); in mpol_to_str()
3052 p += scnprintf(p, buffer + maxlen - p, ":%*pbl", in mpol_to_str()