590 static inline void free_stable_node(struct ksm_stable_node *stable_node)  in free_stable_node()  argument
592 	VM_BUG_ON(stable_node->rmap_hlist_len &&  in free_stable_node()
593 		  !is_stable_node_chain(stable_node));  in free_stable_node()
594 	kmem_cache_free(stable_node_cache, stable_node);  in free_stable_node()
879 static void remove_node_from_stable_tree(struct ksm_stable_node *stable_node)  in remove_node_from_stable_tree()  argument
884 	BUG_ON(stable_node->rmap_hlist_len < 0);  in remove_node_from_stable_tree()
886 	hlist_for_each_entry(rmap_item, &stable_node->hlist, hlist) {  in remove_node_from_stable_tree()
889 			trace_ksm_remove_rmap_item(stable_node->kpfn, rmap_item, rmap_item->mm);  in remove_node_from_stable_tree()
896 		VM_BUG_ON(stable_node->rmap_hlist_len <= 0);  in remove_node_from_stable_tree()
897 		stable_node->rmap_hlist_len--;  in remove_node_from_stable_tree()
913 	trace_ksm_remove_ksm_page(stable_node->kpfn);  in remove_node_from_stable_tree()
914 	if (stable_node->head == &migrate_nodes)  in remove_node_from_stable_tree()
915 		list_del(&stable_node->list);  in remove_node_from_stable_tree()
917 		stable_node_dup_del(stable_node);  in remove_node_from_stable_tree()
918 	free_stable_node(stable_node);  in remove_node_from_stable_tree()
946 static struct folio *ksm_get_folio(struct ksm_stable_node *stable_node,  in ksm_get_folio()  argument
953 	expected_mapping = (void *)((unsigned long)stable_node |  in ksm_get_folio()
956 	kpfn = READ_ONCE(stable_node->kpfn); /* Address dependency. */  in ksm_get_folio()
1016 	if (READ_ONCE(stable_node->kpfn) != kpfn)  in ksm_get_folio()
1018 	remove_node_from_stable_tree(stable_node);  in ksm_get_folio()
1029 		struct ksm_stable_node *stable_node;  in remove_rmap_item_from_tree()  local
1032 		stable_node = rmap_item->head;  in remove_rmap_item_from_tree()
1033 		folio = ksm_get_folio(stable_node, KSM_GET_FOLIO_LOCK);  in remove_rmap_item_from_tree()
1041 		if (!hlist_empty(&stable_node->hlist))  in remove_rmap_item_from_tree()
1048 		VM_BUG_ON(stable_node->rmap_hlist_len <= 0);  in remove_rmap_item_from_tree()
1049 		stable_node->rmap_hlist_len--;  in remove_rmap_item_from_tree()
1093 					 struct ksm_stable_node *stable_node)  in folio_set_stable_node()  argument
1096 	folio->mapping = (void *)((unsigned long)stable_node | FOLIO_MAPPING_KSM);  in folio_set_stable_node()
1103 static int remove_stable_node(struct ksm_stable_node *stable_node)  in remove_stable_node()  argument
1108 	folio = ksm_get_folio(stable_node, KSM_GET_FOLIO_LOCK);  in remove_stable_node()
1132 		remove_node_from_stable_tree(stable_node);  in remove_stable_node()
1141 static int remove_stable_node_chain(struct ksm_stable_node *stable_node,  in remove_stable_node_chain()  argument
1147 	if (!is_stable_node_chain(stable_node)) {  in remove_stable_node_chain()
1148 		VM_BUG_ON(is_stable_node_dup(stable_node));  in remove_stable_node_chain()
1149 		if (remove_stable_node(stable_node))  in remove_stable_node_chain()
1156 				  &stable_node->hlist, hlist_dup) {  in remove_stable_node_chain()
1161 	BUG_ON(!hlist_empty(&stable_node->hlist));  in remove_stable_node_chain()
1162 	free_stable_node_chain(stable_node, root);  in remove_stable_node_chain()
1168 	struct ksm_stable_node *stable_node, *next;  in remove_all_stable_nodes()  local
1174 			stable_node = rb_entry(root_stable_tree[nid].rb_node,  in remove_all_stable_nodes()
1176 			if (remove_stable_node_chain(stable_node,  in remove_all_stable_nodes()
1184 	list_for_each_entry_safe(stable_node, next, &migrate_nodes, list) {  in remove_all_stable_nodes()
1185 		if (remove_stable_node(stable_node))  in remove_all_stable_nodes()
1640 bool __is_page_sharing_candidate(struct ksm_stable_node *stable_node, int offset)  in __is_page_sharing_candidate()  argument
1642 	VM_BUG_ON(stable_node->rmap_hlist_len < 0);  in __is_page_sharing_candidate()
1649 	return stable_node->rmap_hlist_len &&  in __is_page_sharing_candidate()
1650 		stable_node->rmap_hlist_len + offset < ksm_max_page_sharing;  in __is_page_sharing_candidate()
1654 bool is_page_sharing_candidate(struct ksm_stable_node *stable_node)  in is_page_sharing_candidate()  argument
1656 	return __is_page_sharing_candidate(stable_node, 0);  in is_page_sharing_candidate()
1664 	struct ksm_stable_node *dup, *found = NULL, *stable_node = *_stable_node;  in stable_node_dup()  local
1670 	    time_before(jiffies, stable_node->chain_prune_time +  in stable_node_dup()
1675 		stable_node->chain_prune_time = jiffies;  in stable_node_dup()
1678 				  &stable_node->hlist, hlist_dup) {  in stable_node_dup()
1712 		if (hlist_is_singular_node(&found->hlist_dup, &stable_node->hlist)) {  in stable_node_dup()
1719 			BUG_ON(stable_node->hlist.first->next);  in stable_node_dup()
1725 			rb_replace_node(&stable_node->node, &found->node,  in stable_node_dup()
1727 			free_stable_node(stable_node);  in stable_node_dup()
1742 			stable_node = NULL;  in stable_node_dup()
1743 		} else if (stable_node->hlist.first != &found->hlist_dup &&  in stable_node_dup()
1762 				       &stable_node->hlist);  in stable_node_dup()
1766 		free_stable_node_chain(stable_node, root);  in stable_node_dup()
1792 	struct ksm_stable_node *stable_node = *_stable_node;  in __stable_node_chain()  local
1794 	if (!is_stable_node_chain(stable_node)) {  in __stable_node_chain()
1795 		*_stable_node_dup = stable_node;  in __stable_node_chain()
1796 		return ksm_get_folio(stable_node, KSM_GET_FOLIO_NOLOCK);  in __stable_node_chain()
1831 	struct ksm_stable_node *stable_node, *stable_node_dup;  in stable_tree_search()  local
1854 		stable_node = rb_entry(*new, struct ksm_stable_node, node);  in stable_tree_search()
1855 		tree_folio = chain_prune(&stable_node_dup, &stable_node, root);  in stable_tree_search()
1962 	if (stable_node_dup == stable_node) {  in stable_tree_search()
1982 		VM_BUG_ON(!is_stable_node_chain(stable_node));  in stable_tree_search()
1988 			stable_node_chain_add_dup(page_node, stable_node);  in stable_tree_search()
2010 	if (stable_node_dup == stable_node) {  in stable_tree_search()
2013 		stable_node = alloc_stable_node_chain(stable_node_dup,  in stable_tree_search()
2015 		if (!stable_node)  in stable_tree_search()
2028 	stable_node_chain_add_dup(page_node, stable_node);  in stable_tree_search()
2046 	struct ksm_stable_node *stable_node, *stable_node_dup;  in stable_tree_insert()  local
2061 		stable_node = rb_entry(*new, struct ksm_stable_node, node);  in stable_tree_insert()
2062 		tree_folio = chain(&stable_node_dup, &stable_node, root);  in stable_tree_insert()
2102 		if (!is_stable_node_chain(stable_node)) {  in stable_tree_insert()
2103 			struct ksm_stable_node *orig = stable_node;  in stable_tree_insert()
2105 			stable_node = alloc_stable_node_chain(orig, root);  in stable_tree_insert()
2106 			if (!stable_node) {  in stable_tree_insert()
2111 		stable_node_chain_add_dup(stable_node_dup, stable_node);  in stable_tree_insert()
2206 			       struct ksm_stable_node *stable_node,  in stable_tree_append()  argument
2219 	BUG_ON(stable_node->rmap_hlist_len < 0);  in stable_tree_append()
2221 	stable_node->rmap_hlist_len++;  in stable_tree_append()
2224 		WARN_ON_ONCE(stable_node->rmap_hlist_len >  in stable_tree_append()
2227 	rmap_item->head = stable_node;  in stable_tree_append()
2229 	hlist_add_head(&rmap_item->hlist, &stable_node->hlist);  in stable_tree_append()
2253 	struct ksm_stable_node *stable_node;  in cmp_and_merge_page()  local
2259 	stable_node = folio_stable_node(folio);  in cmp_and_merge_page()
2260 	if (stable_node) {  in cmp_and_merge_page()
2261 		if (stable_node->head != &migrate_nodes &&  in cmp_and_merge_page()
2262 		    get_kpfn_nid(READ_ONCE(stable_node->kpfn)) !=  in cmp_and_merge_page()
2263 		    NUMA(stable_node->nid)) {  in cmp_and_merge_page()
2264 			stable_node_dup_del(stable_node);  in cmp_and_merge_page()
2265 			stable_node->head = &migrate_nodes;  in cmp_and_merge_page()
2266 			list_add(&stable_node->list, stable_node->head);  in cmp_and_merge_page()
2268 		if (stable_node->head != &migrate_nodes &&  in cmp_and_merge_page()
2269 		    rmap_item->head == stable_node)  in cmp_and_merge_page()
2275 		if (!is_page_sharing_candidate(stable_node))  in cmp_and_merge_page()
2298 	if (kfolio == folio && rmap_item->head == stable_node) {  in cmp_and_merge_page()
2350 			stable_node = stable_tree_insert(kfolio);  in cmp_and_merge_page()
2351 			if (stable_node) {  in cmp_and_merge_page()
2352 				stable_tree_append(tree_rmap_item, stable_node,  in cmp_and_merge_page()
2354 				stable_tree_append(rmap_item, stable_node,  in cmp_and_merge_page()
2365 			if (!stable_node) {  in cmp_and_merge_page()
2611 			struct ksm_stable_node *stable_node, *next;  in scan_get_next_rmap_item()  local
2614 			list_for_each_entry_safe(stable_node, next,  in scan_get_next_rmap_item()
2616 				folio = ksm_get_folio(stable_node,  in scan_get_next_rmap_item()
3154 	struct ksm_stable_node *stable_node;  in rmap_walk_ksm()  local
3166 	stable_node = folio_stable_node(folio);  in rmap_walk_ksm()
3167 	if (!stable_node)  in rmap_walk_ksm()
3170 	hlist_for_each_entry(rmap_item, &stable_node->hlist, hlist) {  in rmap_walk_ksm()
3229 	struct ksm_stable_node *stable_node;  in collect_procs_ksm()  local
3234 	stable_node = folio_stable_node(folio);  in collect_procs_ksm()
3235 	if (!stable_node)  in collect_procs_ksm()
3237 	hlist_for_each_entry(rmap_item, &stable_node->hlist, hlist) {  in collect_procs_ksm()
3269 	struct ksm_stable_node *stable_node;  in folio_migrate_ksm()  local
3275 	stable_node = folio_stable_node(folio);  in folio_migrate_ksm()
3276 	if (stable_node) {  in folio_migrate_ksm()
3277 		VM_BUG_ON_FOLIO(stable_node->kpfn != folio_pfn(folio), folio);  in folio_migrate_ksm()
3278 		stable_node->kpfn = folio_pfn(newfolio);  in folio_migrate_ksm()
3302 static bool stable_node_dup_remove_range(struct ksm_stable_node *stable_node,  in stable_node_dup_remove_range()  argument
3306 	if (stable_node->kpfn >= start_pfn &&  in stable_node_dup_remove_range()
3307 	    stable_node->kpfn < end_pfn) {  in stable_node_dup_remove_range()
3312 		remove_node_from_stable_tree(stable_node);  in stable_node_dup_remove_range()
3318 static bool stable_node_chain_remove_range(struct ksm_stable_node *stable_node,  in stable_node_chain_remove_range()  argument
3326 	if (!is_stable_node_chain(stable_node)) {  in stable_node_chain_remove_range()
3327 		VM_BUG_ON(is_stable_node_dup(stable_node));  in stable_node_chain_remove_range()
3328 		return stable_node_dup_remove_range(stable_node, start_pfn,  in stable_node_chain_remove_range()
3333 				  &stable_node->hlist, hlist_dup) {  in stable_node_chain_remove_range()
3337 	if (hlist_empty(&stable_node->hlist)) {  in stable_node_chain_remove_range()
3338 		free_stable_node_chain(stable_node, root);  in stable_node_chain_remove_range()
3347 	struct ksm_stable_node *stable_node, *next;  in ksm_check_stable_tree()  local
3354 			stable_node = rb_entry(node, struct ksm_stable_node, node);  in ksm_check_stable_tree()
3355 			if (stable_node_chain_remove_range(stable_node,  in ksm_check_stable_tree()
3365 	list_for_each_entry_safe(stable_node, next, &migrate_nodes, list) {  in ksm_check_stable_tree()
3366 		if (stable_node->kpfn >= start_pfn &&  in ksm_check_stable_tree()
3367 		    stable_node->kpfn < end_pfn)  in ksm_check_stable_tree()
3368 			remove_node_from_stable_tree(stable_node);  in ksm_check_stable_tree()