Lines Matching +full:scrubber +full:- +full:done
1 // SPDX-License-Identifier: GPL-2.0-only
7 * hardware as being corrupted usually due to a multi-bit ECC memory or cache
11 * not-yet-corrupted-by-suspicious pages without killing anything.
23 * - You know how to test it.
24 * - You have a test that can be added to mce-test
25 * https://git.kernel.org/cgit/utils/cpu/mce/mce-test.git/
26 * - The case actually shows up as a frequent (top 10) page state in
27 * tools/mm/page-types when running a real workload.
41 #include <linux/page-flags.h>
50 #include <linux/backing-dev.h>
86 if (pfn != -1UL) in num_poisoned_pages_sub()
91 * MF_ATTR_RO - Create sysfs entry for each memory failure statistics.
100 &NODE_DATA(dev->id)->mf_stats; \
101 return sprintf(buf, "%lu\n", mf_stats->_name); \
171 * returns 0 for non-hugetlb pages as well. in page_handle_poison()
177 * acceptable because soft-offlined page is not broken in page_handle_poison()
216 if (mapping == NULL || mapping->host == NULL) in hwpoison_filter_dev()
217 return -EINVAL; in hwpoison_filter_dev()
219 dev = mapping->host->i_sb->s_dev; in hwpoison_filter_dev()
222 return -EINVAL; in hwpoison_filter_dev()
225 return -EINVAL; in hwpoison_filter_dev()
239 return -EINVAL; in hwpoison_filter_flags()
261 return -EINVAL; in hwpoison_filter_task()
275 return -EINVAL; in hwpoison_filter()
278 return -EINVAL; in hwpoison_filter()
281 return -EINVAL; in hwpoison_filter()
305 * from the VMAs. So do a brute-force search over all
330 struct task_struct *t = tk->tsk; in kill_proc()
331 short addr_lsb = tk->size_shift; in kill_proc()
335 pfn, t->comm, t->pid); in kill_proc()
339 (void __user *)tk->addr, addr_lsb); in kill_proc()
349 ret = send_sig_mceerr(BUS_MCEERR_AO, (void __user *)tk->addr, in kill_proc()
353 t->comm, t->pid, ret); in kill_proc()
366 * TODO: Could shrink slab caches here if a lightweight range-based in shake_page()
387 VM_BUG_ON_VMA(address == -EFAULT, vma); in dev_pagemap_mapping_shift()
388 pgd = pgd_offset(vma->vm_mm, address); in dev_pagemap_mapping_shift()
427 * memory_failure event. In all other cases, page->index and
428 * page->mapping are sufficient for mapping the page back to its
443 tk->addr = ksm_addr ? ksm_addr : page_address_in_vma(p, vma); in __add_to_kill()
446 tk->addr = vma_pgoff_address(fsdax_pgoff, 1, vma); in __add_to_kill()
447 tk->size_shift = dev_pagemap_mapping_shift(vma, tk->addr); in __add_to_kill()
449 tk->size_shift = page_shift(compound_head(p)); in __add_to_kill()
452 * Send SIGKILL if "tk->addr == -EFAULT". Also, as in __add_to_kill()
453 * "tk->size_shift" is always non-zero for !is_zone_device_page(), in __add_to_kill()
454 * so "tk->size_shift == 0" effectively checks no mapping on in __add_to_kill()
461 if (tk->addr == -EFAULT) { in __add_to_kill()
463 page_to_pfn(p), tsk->comm); in __add_to_kill()
464 } else if (tk->size_shift == 0) { in __add_to_kill()
470 tk->tsk = tsk; in __add_to_kill()
471 list_add_tail(&tk->nd, to_kill); in __add_to_kill()
488 if (tk->tsk == tsk) in task_in_to_kill_list()
522 if (fail || tk->addr == -EFAULT) { in kill_procs()
524 pfn, tk->tsk->comm, tk->tsk->pid); in kill_procs()
526 tk->tsk, PIDTYPE_PID); in kill_procs()
531 * something else on the address in-between. We could in kill_procs()
537 pfn, tk->tsk->comm, tk->tsk->pid); in kill_procs()
539 list_del(&tk->nd); in kill_procs()
540 put_task_struct(tk->tsk); in kill_procs()
558 if (t->flags & PF_MCE_PROCESS) { in find_early_kill_thread()
559 if (t->flags & PF_MCE_EARLY) in find_early_kill_thread()
583 if (!tsk->mm) in task_early_kill()
586 * Comparing ->mm here because current task might represent in task_early_kill()
589 if (force_early && tsk->mm == current->mm) in task_early_kill()
618 anon_vma_interval_tree_foreach(vmac, &av->rb_root, in collect_procs_anon()
620 vma = vmac->vma; in collect_procs_anon()
621 if (vma->vm_mm != t->mm) in collect_procs_anon()
640 struct address_space *mapping = folio->mapping; in collect_procs_file()
651 vma_interval_tree_foreach(vma, &mapping->i_mmap, pgoff, in collect_procs_file()
660 if (vma->vm_mm == t->mm) in collect_procs_file()
700 vma_interval_tree_foreach(vma, &mapping->i_mmap, pgoff, pgoff) { in collect_procs_fsdax()
701 if (vma->vm_mm == t->mm) in collect_procs_fsdax()
716 if (!folio->mapping) in collect_procs()
734 tk->addr = addr; in set_to_kill()
735 tk->size_shift = shift; in set_to_kill()
770 if (pfn <= hwp->pfn && hwp->pfn < pfn + HPAGE_PMD_NR) { in check_hwpoisoned_pmd_entry()
771 hwpoison_vaddr = addr + ((hwp->pfn - pfn) << PAGE_SHIFT); in check_hwpoisoned_pmd_entry()
772 set_to_kill(&hwp->tk, hwpoison_vaddr, PAGE_SHIFT); in check_hwpoisoned_pmd_entry()
788 struct hwpoison_walk *hwp = walk->private; in hwpoison_pte_range()
793 ptl = pmd_trans_huge_lock(pmdp, walk->vma); in hwpoison_pte_range()
800 mapped_pte = ptep = pte_offset_map_lock(walk->vma->vm_mm, pmdp, in hwpoison_pte_range()
807 hwp->pfn, &hwp->tk); in hwpoison_pte_range()
822 struct hwpoison_walk *hwp = walk->private; in hwpoison_hugetlb_range()
824 struct hstate *h = hstate_vma(walk->vma); in hwpoison_hugetlb_range()
827 hwp->pfn, &hwp->tk); in hwpoison_hugetlb_range()
861 if (!p->mm) in kill_accessing_process()
862 return -EFAULT; in kill_accessing_process()
864 mmap_read_lock(p->mm); in kill_accessing_process()
865 ret = walk_page_range(p->mm, 0, TASK_SIZE, &hwpoison_walk_ops, in kill_accessing_process()
871 mmap_read_unlock(p->mm); in kill_accessing_process()
872 return ret > 0 ? -EHWPOISON : -EFAULT; in kill_accessing_process()
884 [MF_MSG_KERNEL_HIGH_ORDER] = "high-order kernel page",
916 * complain when the folio is unpoison-and-freed. in delete_from_lru_cache()
933 return -EIO; in delete_from_lru_cache()
941 if (mapping->a_ops->error_remove_folio) { in truncate_error_folio()
942 int err = mapping->a_ops->error_remove_folio(mapping, folio); in truncate_error_folio()
969 /* Callback ->action() has to unlock the relevant page inside it. */
982 int count = page_count(p) - 1; in has_extra_refcount()
985 count -= folio_nr_pages(page_folio(p)); in has_extra_refcount()
989 page_to_pfn(p), action_page_types[ps->type], count); in has_extra_refcount()
1054 * so is expected to have an extra refcount after error-handling. in me_pagecache_clean()
1119 mapping_set_error(mapping, -EIO); in me_pagecache_dirty()
1134 * - clear dirty bit to prevent IO
1135 * - remove from LRU
1136 * - but keep in the swap cache, so that when we return to it on
1185 * - Error on hugepage is contained in hugepage unit (not in raw page unit.)
1226 * A page state is defined by its current page->flags bits.
1300 mf_stats = &NODE_DATA(nid)->mf_stats; in update_per_node_mf_stats()
1303 ++mf_stats->ignored; in update_per_node_mf_stats()
1306 ++mf_stats->failed; in update_per_node_mf_stats()
1309 ++mf_stats->delayed; in update_per_node_mf_stats()
1312 ++mf_stats->recovered; in update_per_node_mf_stats()
1318 ++mf_stats->total; in update_per_node_mf_stats()
1337 return (result == MF_RECOVERED || result == MF_DELAYED) ? 0 : -EBUSY; in action_result()
1345 /* page p should be unlocked after returning from ps->action(). */ in page_action()
1346 result = ps->action(ps, p); in page_action()
1353 return action_result(pfn, ps->type, result); in page_action()
1383 /* Soft offline could migrate non-LRU movable pages */ in HWPoisonHandlable()
1412 if (!HWPoisonHandlable(&folio->page, flags)) in __get_hwpoison_page()
1413 return -EBUSY; in __get_hwpoison_page()
1442 ret = -EBUSY; in get_any_page()
1447 ret = -EIO; in get_any_page()
1450 } else if (ret == -EBUSY) { in get_any_page()
1459 ret = -EIO; in get_any_page()
1478 ret = -EIO; in get_any_page()
1481 if (ret == -EIO) in get_any_page()
1508 return -EHWPOISON; in __get_unpoison_page()
1514 * get_hwpoison_page() - Get refcount for memory error handling
1519 * error on it, after checking that the error page is in a well-defined state
1520 * (defined as a page-type we can successfully handle the memory error on it,
1526 * extra care for the error page's state (as done in __get_hwpoison_page()),
1534 * 1 on success for in-use pages in a well-defined state,
1535 * -EIO for pages on which we can not handle memory errors,
1536 * -EBUSY when get_hwpoison_page() has raced with page lifecycle
1538 * -EHWPOISON when the page is hwpoisoned and taken off from buddy.
1570 * Here we are interested only in user-mapped pages, so skip any in hwpoison_user_mappings()
1610 * mapped in dirty form. This has to be done before try_to_unmap, in hwpoison_user_mappings()
1647 * struct page and all unmaps done we can decide if in hwpoison_user_mappings()
1652 * use a more force-full uncatchable kill to prevent in hwpoison_user_mappings()
1673 if ((p->flags & ps->mask) == ps->res) in identify_page_state()
1676 page_flags |= (p->flags & (1UL << PG_dirty)); in identify_page_state()
1678 if (!ps->mask) in identify_page_state()
1680 if ((page_flags & ps->mask) == ps->res) in identify_page_state()
1706 if (tk->size_shift) in unmap_and_kill()
1707 size = max(size, 1UL << tk->size_shift); in unmap_and_kill()
1711 * Unmap the largest mapping to avoid breaking up device-dax in unmap_and_kill()
1716 loff_t start = ((loff_t)index << PAGE_SHIFT) & ~(size - 1); in unmap_and_kill()
1750 return -EBUSY; in mf_generic_kill_procs()
1752 if (hwpoison_filter(&folio->page)) { in mf_generic_kill_procs()
1753 rc = -EOPNOTSUPP; in mf_generic_kill_procs()
1757 switch (pgmap->type) { in mf_generic_kill_procs()
1762 * with device-side memory. in mf_generic_kill_procs()
1764 rc = -ENXIO; in mf_generic_kill_procs()
1774 SetPageHWPoison(&folio->page); in mf_generic_kill_procs()
1777 * Unlike System-RAM there is no possibility to swap in a in mf_generic_kill_procs()
1783 collect_procs(folio, &folio->page, &to_kill, true); in mf_generic_kill_procs()
1785 unmap_and_kill(&to_kill, pfn, folio->mapping, folio->index, flags); in mf_generic_kill_procs()
1793 * mf_dax_kill_procs - Collect and kill processes who are using this file range
1814 return -EBUSY; in mf_dax_kill_procs()
1840 * constructing singly linked list from ->_hugetlb_hwpoison field of folio.
1849 return (struct llist_head *)&folio->_hugetlb_hwpoison; in raw_hwp_list_head()
1875 llist_for_each_entry(p, raw_hwp_head->first, node) { in is_raw_hwpoison_page_in_hugepage()
1876 if (page == p->page) { in is_raw_hwpoison_page_in_hugepage()
1896 SetPageHWPoison(p->page); in __folio_free_raw_hwp()
1898 num_poisoned_pages_sub(page_to_pfn(p->page), 1); in __folio_free_raw_hwp()
1910 int ret = folio_test_set_hwpoison(folio) ? -EHWPOISON : 0; in folio_set_hugetlb_hwpoison()
1918 return -EHWPOISON; in folio_set_hugetlb_hwpoison()
1920 llist_for_each_entry_safe(p, next, head->first, node) { in folio_set_hugetlb_hwpoison()
1921 if (p->page == page) in folio_set_hugetlb_hwpoison()
1922 return -EHWPOISON; in folio_set_hugetlb_hwpoison()
1927 raw_hwp->page = page; in folio_set_hugetlb_hwpoison()
1928 llist_add(&raw_hwp->node, head); in folio_set_hugetlb_hwpoison()
1981 * 0 - free hugepage
1982 * 1 - in-use hugepage
1983 * 2 - not a hugepage
1984 * -EBUSY - the hugepage is busy (try to retry)
1985 * -EHWPOISON - the hugepage is already hwpoisoned
2008 ret = -EBUSY; in __get_huge_page_for_hwpoison()
2014 ret = -EHWPOISON; in __get_huge_page_for_hwpoison()
2038 * PageHWPoison) should be done in single hugetlb_lock range.
2054 } else if (res == -EHWPOISON) { in try_memory_failure_hugetlb()
2061 } else if (res == -EBUSY) { in try_memory_failure_hugetlb()
2079 return -EOPNOTSUPP; in try_memory_failure_hugetlb()
2097 page_flags = folio->flags; in try_memory_failure_hugetlb()
2099 if (!hwpoison_user_mappings(p, pfn, flags, &folio->page)) { in try_memory_failure_hugetlb()
2135 int rc = -ENXIO; in memory_failure_dev_pagemap()
2146 rc = pgmap->ops->memory_failure(pgmap, pfn, 1, flags); in memory_failure_dev_pagemap()
2151 if (rc != -EOPNOTSUPP) in memory_failure_dev_pagemap()
2159 if (rc != -EOPNOTSUPP) in memory_failure_dev_pagemap()
2165 * memory_failure - Handle memory failure of a page.
2176 * detected by a background scrubber)
2182 * -EOPNOTSUPP for hwpoison_filter() filtered the error event,
2183 * < 0(except -EOPNOTSUPP) on failure.
2219 res = -ENXIO; in memory_failure()
2230 res = -EHWPOISON; in memory_failure()
2242 * 2) it's part of a non-compound high order page. in memory_failure()
2300 * We ignore non-LRU pages for good reasons. in memory_failure()
2301 * - PG_locked is only well defined for LRU pages and a few others in memory_failure()
2302 * - to avoid races with __SetPageLocked() in memory_failure()
2303 * - to avoid races with __SetPageSlab*() (and more non-atomic ops) in memory_failure()
2312 * We're only intended to deal with the non-Compound page here. in memory_failure()
2337 page_flags = p->flags; in memory_failure()
2343 res = -EOPNOTSUPP; in memory_failure()
2373 if (PageLRU(p) && !PageSwapCache(p) && p->mapping == NULL) { in memory_failure()
2408 * memory_failure_queue - Schedule handling memory failure of a page.
2419 * detected by a background scrubber)
2433 spin_lock_irqsave(&mf_cpu->lock, proc_flags); in memory_failure_queue()
2434 if (kfifo_put(&mf_cpu->fifo, entry)) in memory_failure_queue()
2435 schedule_work_on(smp_processor_id(), &mf_cpu->work); in memory_failure_queue()
2439 spin_unlock_irqrestore(&mf_cpu->lock, proc_flags); in memory_failure_queue()
2453 spin_lock_irqsave(&mf_cpu->lock, proc_flags); in memory_failure_work_func()
2454 gotten = kfifo_get(&mf_cpu->fifo, &entry); in memory_failure_work_func()
2455 spin_unlock_irqrestore(&mf_cpu->lock, proc_flags); in memory_failure_work_func()
2467 * Used to avoid return-to-userspace racing with the memory_failure workqueue.
2474 cancel_work_sync(&mf_cpu->work); in memory_failure_queue_kick()
2475 memory_failure_work_func(&mf_cpu->work); in memory_failure_queue_kick()
2485 spin_lock_init(&mf_cpu->lock); in memory_failure_init()
2486 INIT_KFIFO(mf_cpu->fifo); in memory_failure_init()
2487 INIT_WORK(&mf_cpu->work, memory_failure_work_func); in memory_failure_init()
2505 * unpoison_memory - Unpoison a previously poisoned page
2508 * Software-unpoison a page that has been poisoned by
2511 * This is only done on the software-level, so it only works
2514 * Returns 0 for success, otherwise -errno.
2520 int ret = -EBUSY, ghp; in unpoison_memory()
2527 return -ENXIO; in unpoison_memory()
2537 ret = -EOPNOTSUPP; in unpoison_memory()
2553 if (folio_test_slab(folio) || PageTable(&folio->page) || in unpoison_memory()
2554 folio_test_reserved(folio) || PageOffline(&folio->page)) in unpoison_memory()
2558 * Note that folio->_mapcount is overloaded in SLAB, so the simple test in unpoison_memory()
2559 * in folio_mapped() has to be done after folio_test_slab() is checked. in unpoison_memory()
2568 unpoison_pr_info("Unpoison: the hwpoison page has non-NULL mapping %#lx\n", in unpoison_memory()
2581 ret = folio_test_clear_hwpoison(folio) ? 0 : -EBUSY; in unpoison_memory()
2583 if (ghp == -EHWPOISON) { in unpoison_memory()
2584 ret = put_page_back_buddy(p) ? 0 : -EBUSY; in unpoison_memory()
2612 unpoison_pr_info("Unpoison: Software-unpoisoned page %#lx\n", in unpoison_memory()
2631 isolated = isolate_movable_page(&folio->page, in mf_isolate_folio()
2635 list_add(&folio->lru, pagelist); in mf_isolate_folio()
2654 * soft_offline_in_use_page handles hugetlb-pages and non-hugetlb pages.
2655 * If the page is a non-dirty unmapped page-cache page, it simply invalidates.
2674 return -EBUSY; in soft_offline_in_use_page()
2710 ret = -EBUSY; in soft_offline_in_use_page()
2716 pfn, msg_page[huge], ret, &page->flags); in soft_offline_in_use_page()
2718 ret = -EBUSY; in soft_offline_in_use_page()
2722 pfn, msg_page[huge], page_count(page), &page->flags); in soft_offline_in_use_page()
2723 ret = -EBUSY; in soft_offline_in_use_page()
2729 * soft_offline_page - Soft offline a page.
2730 * @pfn: pfn to soft-offline
2734 * -EOPNOTSUPP for hwpoison_filter() filtered the error event
2760 return -ENXIO; in soft_offline_page()
2763 /* Only online pages can be soft-offlined (esp., not ZONE_DEVICE). */ in soft_offline_page()
2767 return -EIO; in soft_offline_page()
2789 return -EOPNOTSUPP; in soft_offline_page()
2801 ret = -EBUSY; in soft_offline_page()