Lines Matching +full:- +full:- +full:target +full:- +full:list
1 // SPDX-License-Identifier: GPL-2.0
7 #include <linux/memory-tiers.h>
15 struct list_head list; member
16 /* list of all memory types part of this tier */
41 * The list is used to store all memory types that are not created
56 * folio_use_access_time - check if a folio reuses cpupid for page access time
88 * memory_tiers0 = 0-1
89 * memory_tiers1 = 2-3
98 * Node 0 & 1 are CPU + DRAM nodes, node 2 is memory-only DRAM node.
106 * memory_tiers0 = 0-2
153 list_for_each_entry(memtype, &memtier->memory_types, tier_sibling) in get_memtier_nodemask()
154 nodes_or(nodes, nodes, memtype->nodes); in get_memtier_nodemask()
202 int adistance = memtype->adistance; in find_create_memory_tier()
212 if (!list_empty(&memtype->tier_sibling)) { in find_create_memory_tier()
213 list_for_each_entry(memtier, &memory_tiers, list) { in find_create_memory_tier()
214 if (adistance == memtier->adistance_start) in find_create_memory_tier()
218 return ERR_PTR(-EINVAL); in find_create_memory_tier()
221 list_for_each_entry(memtier, &memory_tiers, list) { in find_create_memory_tier()
222 if (adistance == memtier->adistance_start) { in find_create_memory_tier()
224 } else if (adistance < memtier->adistance_start) { in find_create_memory_tier()
232 return ERR_PTR(-ENOMEM); in find_create_memory_tier()
234 new_memtier->adistance_start = adistance; in find_create_memory_tier()
235 INIT_LIST_HEAD(&new_memtier->list); in find_create_memory_tier()
236 INIT_LIST_HEAD(&new_memtier->memory_types); in find_create_memory_tier()
238 list_add_tail(&new_memtier->list, &memtier->list); in find_create_memory_tier()
240 list_add_tail(&new_memtier->list, &memory_tiers); in find_create_memory_tier()
242 new_memtier->dev.id = adistance >> MEMTIER_CHUNK_BITS; in find_create_memory_tier()
243 new_memtier->dev.bus = &memory_tier_subsys; in find_create_memory_tier()
244 new_memtier->dev.release = memory_tier_device_release; in find_create_memory_tier()
245 new_memtier->dev.groups = memtier_dev_groups; in find_create_memory_tier()
247 ret = device_register(&new_memtier->dev); in find_create_memory_tier()
249 list_del(&new_memtier->list); in find_create_memory_tier()
250 put_device(&new_memtier->dev); in find_create_memory_tier()
256 list_add(&memtype->tier_sibling, &memtier->memory_types); in find_create_memory_tier()
272 return rcu_dereference_check(pgdat->memtier, in __node_get_memory_tier()
288 memtier = rcu_dereference(pgdat->memtier); in node_is_toptier()
293 if (memtier->adistance_start <= top_tier_adistance) in node_is_toptier()
312 memtier = rcu_dereference(pgdat->memtier); in node_get_allowed_targets()
314 *targets = memtier->lower_tier_mask; in node_get_allowed_targets()
321 * next_demotion_node() - Get the next node in the demotion path
327 * target.
332 int target; in next_demotion_node() local
348 * If there are multiple target nodes, just select one in next_demotion_node()
349 * target node randomly. in next_demotion_node()
351 * In addition, we can also use round-robin to select in next_demotion_node()
352 * target node, but we should introduce another variable in next_demotion_node()
353 * for node_demotion[] to record last selected target node, in next_demotion_node()
354 * that may cause cache ping-pong due to the changing of in next_demotion_node()
355 * last target node. Or introducing per-cpu data to avoid in next_demotion_node()
357 * target node randomly seems better until now. in next_demotion_node()
359 target = node_random(&nd->preferred); in next_demotion_node()
362 return target; in next_demotion_node()
374 * to access pgda->memtier. in disable_all_demotion_targets()
378 memtier->lower_tier_mask = NODE_MASK_NONE; in disable_all_demotion_targets()
405 nodemask_pr_args(&memtier->lower_tier_mask)); in dump_demotion_targets()
410 * Find an automatic demotion target for all memory
418 int target = NUMA_NO_NODE, node; in establish_demotion_targets() local
430 best_distance = -1; in establish_demotion_targets()
434 if (!memtier || list_is_last(&memtier->list, &memory_tiers)) in establish_demotion_targets()
437 * Get the lower memtier to find the demotion node list. in establish_demotion_targets()
439 memtier = list_next_entry(memtier, list); in establish_demotion_targets()
442 * find_next_best_node, use 'used' nodemask as a skip list. in establish_demotion_targets()
444 * nodelist to skip list so that we find the best node from the in establish_demotion_targets()
450 * Find all the nodes in the memory tier node list of same best distance. in establish_demotion_targets()
455 target = find_next_best_node(node, &tier_nodes); in establish_demotion_targets()
456 if (target == NUMA_NO_NODE) in establish_demotion_targets()
459 distance = node_distance(node, target); in establish_demotion_targets()
460 if (distance == best_distance || best_distance == -1) { in establish_demotion_targets()
462 node_set(target, nd->preferred); in establish_demotion_targets()
476 list_for_each_entry_reverse(memtier, &memory_tiers, list) { in establish_demotion_targets()
484 top_tier_adistance = memtier->adistance_start + in establish_demotion_targets()
485 MEMTIER_CHUNK_SIZE - 1; in establish_demotion_targets()
496 list_for_each_entry(memtier, &memory_tiers, list) { in establish_demotion_targets()
504 memtier->lower_tier_mask = lower_tier; in establish_demotion_targets()
528 kref_get(&memtype->kref); in __init_node_memory_type()
543 return ERR_PTR(-EINVAL); in set_node_memory_tier()
557 node_set(node, memtype->nodes); in set_node_memory_tier()
560 rcu_assign_pointer(pgdat->memtier, memtier); in set_node_memory_tier()
566 list_del(&memtier->list); in destroy_memory_tier()
567 device_unregister(&memtier->dev); in destroy_memory_tier()
592 rcu_assign_pointer(pgdat->memtier, NULL); in clear_node_memory_tier()
595 node_clear(node, memtype->nodes); in clear_node_memory_tier()
596 if (nodes_empty(memtype->nodes)) { in clear_node_memory_tier()
597 list_del_init(&memtype->tier_sibling); in clear_node_memory_tier()
598 if (list_empty(&memtier->memory_types)) in clear_node_memory_tier()
620 return ERR_PTR(-ENOMEM); in alloc_memory_type()
622 memtype->adistance = adistance; in alloc_memory_type()
623 INIT_LIST_HEAD(&memtype->tier_sibling); in alloc_memory_type()
624 memtype->nodes = NODE_MASK_NONE; in alloc_memory_type()
625 kref_init(&memtype->kref); in alloc_memory_type()
632 kref_put(&memtype->kref, release_memtype); in put_memory_type()
649 node_memory_types[node].map_count--; in clear_node_memory_type()
667 list_for_each_entry(mtype, memory_types, list) in mt_find_alloc_memory_type()
668 if (mtype->adistance == adist) in mt_find_alloc_memory_type()
675 list_add(&mtype->list, memory_types); in mt_find_alloc_memory_type()
685 list_for_each_entry_safe(mtype, mtn, memory_types, list) { in mt_put_memory_types()
686 list_del(&mtype->list); in mt_put_memory_types()
732 prefix, coord->read_latency, coord->write_latency, in dump_hmem_attrs()
733 coord->read_bandwidth, coord->write_bandwidth); in dump_hmem_attrs()
741 return -EIO; in mt_set_default_dram_perf()
743 if (perf->read_latency + perf->write_latency == 0 || in mt_set_default_dram_perf()
744 perf->read_bandwidth + perf->write_bandwidth == 0) in mt_set_default_dram_perf()
745 return -EINVAL; in mt_set_default_dram_perf()
760 if (abs(perf->read_latency - default_dram_perf.read_latency) * 10 > in mt_set_default_dram_perf()
762 abs(perf->write_latency - default_dram_perf.write_latency) * 10 > in mt_set_default_dram_perf()
764 abs(perf->read_bandwidth - default_dram_perf.read_bandwidth) * 10 > in mt_set_default_dram_perf()
766 abs(perf->write_bandwidth - default_dram_perf.write_bandwidth) * 10 > in mt_set_default_dram_perf()
769 "memory-tiers: the performance of DRAM node %d mismatches that of the reference\n" in mt_set_default_dram_perf()
779 return -EINVAL; in mt_set_default_dram_perf()
789 return -EIO; in mt_perf_to_adistance()
791 if (perf->read_latency + perf->write_latency == 0 || in mt_perf_to_adistance()
792 perf->read_bandwidth + perf->write_bandwidth == 0) in mt_perf_to_adistance()
793 return -EINVAL; in mt_perf_to_adistance()
796 return -ENOENT; in mt_perf_to_adistance()
806 (perf->read_latency + perf->write_latency) / in mt_perf_to_adistance()
809 (perf->read_bandwidth + perf->write_bandwidth); in mt_perf_to_adistance()
816 * register_mt_adistance_algorithm() - Register memory tiering abstract distance algorithm
844 * unregister_mt_adistance_algorithm() - Unregister memory tiering abstract distance algorithm
856 * mt_calc_adistance() - Calculate abstract distance with registered algorithms
879 * changing status, like online->offline. in memtier_hotplug_callback()
881 if (arg->status_change_nid < 0) in memtier_hotplug_callback()
887 if (clear_node_memory_tier(arg->status_change_nid)) in memtier_hotplug_callback()
893 memtier = set_node_memory_tier(arg->status_change_nid); in memtier_hotplug_callback()
980 return -ENOMEM; in numa_init_sysfs()