Lines Matching +full:memory +full:- +full:to +full:- +full:memory
1 // SPDX-License-Identifier: GPL-2.0
3 * Memory subsystem support
8 * This file provides the necessary infrastructure to represent
9 * a SPARSEMEM-memory-model system's physical memory in /sysfs.
10 * All arch-independent code that assumes MEMORY_HOTPLUG requires
19 #include <linux/memory.h>
29 #define MEMORY_CLASS_NAME "memory"
46 return -EINVAL; in mhp_online_type_from_str()
79 * Memory blocks are cached in a local radix tree to avoid
86 * Memory groups, indexed by memory group id (mgid).
109 WARN_ON(mem->altmap); in memory_block_release()
119 /* Show the memory block ID, relative to the memory block size */
125 return sysfs_emit(buf, "%08lx\n", memory_block_id(mem->start_section_nr)); in phys_index_show()
130 * with CONFIG_MEMORY_HOTREMOVE - bad heuristic.
149 * so that they're not open-coded in state_show()
151 switch (mem->state) { in state_show()
159 output = "going-offline"; in state_show()
163 return sysfs_emit(buf, "ERROR-UNKNOWN-%ld\n", mem->state); in state_show()
188 unsigned long start_pfn = section_nr_to_pfn(mem->start_section_nr); in memory_block_online()
196 return -EHWPOISON; in memory_block_online()
198 zone = zone_for_pfn_range(mem->online_type, mem->nid, mem->group, in memory_block_online()
203 * they describe (they remain until the memory is unplugged), doing in memory_block_online()
204 * their initialization and accounting at memory onlining/offlining in memory_block_online()
205 * stage helps to keep accounting easier to follow - e.g vmemmaps in memory_block_online()
206 * belong to the same zone as the memory they backed. in memory_block_online()
208 if (mem->altmap) in memory_block_online()
209 nr_vmemmap_pages = mem->altmap->free; in memory_block_online()
214 arg.nr_pages = nr_pages - nr_vmemmap_pages; in memory_block_online()
223 zone, mem->altmap->inaccessible); in memory_block_online()
229 nr_pages - nr_vmemmap_pages, zone, mem->group); in memory_block_online()
241 adjust_present_page_count(pfn_to_page(start_pfn), mem->group, in memory_block_online()
244 mem->zone = zone; in memory_block_online()
259 unsigned long start_pfn = section_nr_to_pfn(mem->start_section_nr); in memory_block_offline()
265 if (!mem->zone) in memory_block_offline()
266 return -EINVAL; in memory_block_offline()
272 if (mem->altmap) in memory_block_offline()
273 nr_vmemmap_pages = mem->altmap->free; in memory_block_offline()
277 adjust_present_page_count(pfn_to_page(start_pfn), mem->group, in memory_block_offline()
278 -nr_vmemmap_pages); in memory_block_offline()
281 nr_pages - nr_vmemmap_pages, mem->zone, mem->group); in memory_block_offline()
286 mem->group, nr_vmemmap_pages); in memory_block_offline()
293 mem->zone = NULL; in memory_block_offline()
297 arg.nr_pages = nr_pages - nr_vmemmap_pages; in memory_block_offline()
306 * OK to have direct references to sparsemem variables in here.
322 "%ld\n", __func__, mem->start_section_nr, action, action); in memory_block_action()
323 ret = -EINVAL; in memory_block_action()
334 if (mem->state != from_state_req) in memory_block_change_state()
335 return -EINVAL; in memory_block_change_state()
338 mem->state = MEM_GOING_OFFLINE; in memory_block_change_state()
341 mem->state = ret ? from_state_req : to_state; in memory_block_change_state()
352 if (mem->state == MEM_ONLINE) in memory_subsys_online()
357 * we want to default to MMOP_ONLINE. in memory_subsys_online()
359 if (mem->online_type == MMOP_OFFLINE) in memory_subsys_online()
360 mem->online_type = MMOP_ONLINE; in memory_subsys_online()
363 mem->online_type = MMOP_OFFLINE; in memory_subsys_online()
372 if (mem->state == MEM_OFFLINE) in memory_subsys_offline()
386 return -EINVAL; in state_store()
396 /* mem->online_type is protected by device_hotplug_lock */ in state_store()
397 mem->online_type = online_type; in state_store()
398 ret = device_online(&mem->dev); in state_store()
401 ret = device_offline(&mem->dev); in state_store()
404 ret = -EINVAL; /* should never happen */ in state_store()
412 return -EINVAL; in state_store()
419 * covered by a memory block, allowing for identifying which memory blocks
420 * comprise a storage increment. Since a memory block spans complete
428 unsigned long start_pfn = section_nr_to_pfn(mem->start_section_nr); in phys_device_show()
446 return sysfs_emit_at(buf, len, " %s", zone->name); in print_allowed_zone()
453 unsigned long start_pfn = section_nr_to_pfn(mem->start_section_nr); in valid_zones_show()
455 struct memory_group *group = mem->group; in valid_zones_show()
457 int nid = mem->nid; in valid_zones_show()
464 if (mem->state == MEM_ONLINE) { in valid_zones_show()
466 * If !mem->zone, the memory block spans multiple zones and in valid_zones_show()
470 mem->zone ? mem->zone->name : "none"); in valid_zones_show()
476 len = sysfs_emit(buf, "%s", default_zone->name); in valid_zones_show()
493 * Show the memory block size (shared by all memory blocks).
504 * Memory auto online policy.
521 return -EINVAL; in auto_online_blocks_store()
540 * Some architectures will have custom drivers to do this, and
541 * will not need to do it from userspace. The fake hot-add code
557 if (phys_addr & ((pages_per_block << PAGE_SHIFT) - 1)) in probe_store()
558 return -EINVAL; in probe_store()
583 * Support for offlining pages of memory
594 return -EPERM; in soft_offline_page_store()
596 return -EINVAL; in soft_offline_page_store()
610 return -EPERM; in hard_offline_page_store()
612 return -EINVAL; in hard_offline_page_store()
615 if (ret == -EOPNOTSUPP) in hard_offline_page_store()
631 * A reference for the returned memory block device is acquired.
641 get_device(&mem->dev); in find_memory_block_by_id()
675 static int __add_memory_block(struct memory_block *memory) in __add_memory_block() argument
679 memory->dev.bus = &memory_subsys; in __add_memory_block()
680 memory->dev.id = memory->start_section_nr / sections_per_block; in __add_memory_block()
681 memory->dev.release = memory_block_release; in __add_memory_block()
682 memory->dev.groups = memory_memblk_attr_groups; in __add_memory_block()
683 memory->dev.offline = memory->state == MEM_OFFLINE; in __add_memory_block()
685 ret = device_register(&memory->dev); in __add_memory_block()
687 put_device(&memory->dev); in __add_memory_block()
690 ret = xa_err(xa_store(&memory_blocks, memory->dev.id, memory, in __add_memory_block()
693 device_unregister(&memory->dev); in __add_memory_block()
701 const unsigned long start_pfn = section_nr_to_pfn(mem->start_section_nr); in early_node_zone_for_memory_block()
708 * This logic only works for early memory, when the applicable zones in early_node_zone_for_memory_block()
709 * already span the memory block. We don't expect overlapping zones on in early_node_zone_for_memory_block()
710 * a single node for early memory. So if we're told that some PFNs in early_node_zone_for_memory_block()
711 * of a node fall into this memory block, we can assume that all node in early_node_zone_for_memory_block()
712 * zones that intersect with the memory block are actually applicable. in early_node_zone_for_memory_block()
713 * No need to look at the memmap. in early_node_zone_for_memory_block()
716 zone = pgdat->node_zones + i; in early_node_zone_for_memory_block()
734 * memory_block_add_nid() - Indicate that system RAM falling into this memory
735 * block device (partially) belongs to the given node.
736 * @mem: The memory block device.
738 * @context: The memory initialization context.
740 * Indicate that system RAM falling into this memory block (partially) belongs
741 * to the given node. If the context indicates ("early") that we are adding the
743 * set/adjust mem->zone based on the zone ranges of the given node.
748 if (context == MEMINIT_EARLY && mem->nid != nid) { in memory_block_add_nid()
750 * For early memory we have to determine the zone when setting in memory_block_add_nid()
752 * memory block by indicate via zone == NULL that we're not in memory_block_add_nid()
755 * setting the node id a second time to a different node, in memory_block_add_nid()
758 if (mem->nid == NUMA_NO_NODE) in memory_block_add_nid()
759 mem->zone = early_node_zone_for_memory_block(mem, nid); in memory_block_add_nid()
761 mem->zone = NULL; in memory_block_add_nid()
765 * If this memory block spans multiple nodes, we only indicate in memory_block_add_nid()
767 * to hotplugged memory), zone == NULL will prohibit memory offlining in memory_block_add_nid()
770 mem->nid = nid; in memory_block_add_nid()
783 put_device(&mem->dev); in add_memory_block()
784 return -EEXIST; in add_memory_block()
788 return -ENOMEM; in add_memory_block()
790 mem->start_section_nr = block_id * sections_per_block; in add_memory_block()
791 mem->state = state; in add_memory_block()
792 mem->nid = NUMA_NO_NODE; in add_memory_block()
793 mem->altmap = altmap; in add_memory_block()
794 INIT_LIST_HEAD(&mem->group_next); in add_memory_block()
799 * MEM_ONLINE at this point implies early memory. With NUMA, in add_memory_block()
801 * memory_block_add_nid(). Memory hotplug updated the zone in add_memory_block()
802 * manually when memory onlining/offlining succeeds. in add_memory_block()
804 mem->zone = early_node_zone_for_memory_block(mem, NUMA_NO_NODE); in add_memory_block()
812 mem->group = group; in add_memory_block()
813 list_add(&mem->group_next, &group->memory_blocks); in add_memory_block()
826 static void remove_memory_block(struct memory_block *memory) in remove_memory_block() argument
828 if (WARN_ON_ONCE(memory->dev.bus != &memory_subsys)) in remove_memory_block()
831 WARN_ON(xa_erase(&memory_blocks, memory->dev.id) == NULL); in remove_memory_block()
833 if (memory->group) { in remove_memory_block()
834 list_del(&memory->group_next); in remove_memory_block()
835 memory->group = NULL; in remove_memory_block()
839 put_device(&memory->dev); in remove_memory_block()
840 device_unregister(&memory->dev); in remove_memory_block()
844 * Create memory block devices for the given memory area. Start and size
845 * have to be aligned to memory block granularity. Memory block devices
862 return -EINVAL; in create_memory_block_devices()
883 * Remove memory block devices for the given memory area. Start and size
884 * have to be aligned to memory block granularity. Memory block devices
885 * have to be offline.
904 num_poisoned_pages_sub(-1UL, memblk_nr_poison(mem)); in remove_memory_block_devices()
938 * Initialize the sysfs support for memory devices. At the time this function
939 * is called, we cannot have concurrent creation/deletion of memory block
947 /* Validate the configured memory block size */ in memory_dev_init()
950 panic("Memory block size not suitable: 0x%lx\n", block_sz); in memory_dev_init()
955 panic("%s() failed to register subsystem: %d\n", __func__, ret); in memory_dev_init()
958 * Create entries for memory sections that were found during boot in memory_dev_init()
959 * and have been initialized. Use @block_id to track the last in memory_dev_init()
960 * handled block and initialize it to an invalid value (ULONG_MAX) in memory_dev_init()
961 * to bypass the block ID matching check for the first present in memory_dev_init()
972 panic("%s() failed to add memory block: %d\n", in memory_dev_init()
979 * walk_memory_blocks - walk through all present memory blocks overlapped
982 * @start: start address of the memory range
983 * @size: size of the memory range
984 * @arg: argument passed to func
985 * @func: callback for each memory section walked
987 * This function walks through all present memory blocks overlapped by the
988 * range [start, start + size), calling func on each memory block.
999 const unsigned long end_block_id = phys_to_block_id(start + size - 1); in walk_memory_blocks()
1013 put_device(&mem->dev); in walk_memory_blocks()
1030 return cb_data->func(mem, cb_data->arg); in for_each_memory_block_cb()
1034 * for_each_memory_block - walk through all present memory blocks
1036 * @arg: argument passed to func
1037 * @func: callback for each memory block walked
1039 * This function walks through all present memory blocks, calling func on
1040 * each memory block.
1057 * This is an internal helper to unify allocation and initialization of
1058 * memory groups. Note that the passed memory group will be copied to a
1059 * dynamically allocated memory group. After this call, the passed
1060 * memory group should no longer be used.
1069 return -EINVAL; in memory_group_register()
1073 return -ENOMEM; in memory_group_register()
1075 INIT_LIST_HEAD(&new_group->memory_blocks); in memory_group_register()
1089 * memory_group_register_static() - Register a static memory group.
1091 * @max_pages: The maximum number of pages we'll have in this static memory
1094 * Register a new static memory group and return the memory group id.
1095 * All memory in the group belongs to a single unit, such as a DIMM. All
1096 * memory belonging to a static memory group is added in one go to be removed
1097 * in one go -- it's static.
1099 * Returns an error if out of memory, if the node id is invalid, if no new
1100 * memory groups can be registered, or if max_pages is invalid (0). Otherwise,
1101 * returns the new memory group id.
1113 return -EINVAL; in memory_group_register_static()
1119 * memory_group_register_dynamic() - Register a dynamic memory group.
1121 * @unit_pages: Unit in pages in which is memory added/removed in this dynamic
1122 * memory group.
1124 * Register a new dynamic memory group and return the memory group id.
1125 * Memory within a dynamic memory group is added/removed dynamically
1128 * Returns an error if out of memory, if the node id is invalid, if no new
1129 * memory groups can be registered, or if unit_pages is invalid (0, not a
1130 * power of two, smaller than a single memory block). Otherwise, returns the
1131 * new memory group id.
1145 return -EINVAL; in memory_group_register_dynamic()
1151 * memory_group_unregister() - Unregister a memory group.
1152 * @mgid: the memory group id
1154 * Unregister a memory group. If any memory block still belongs to this
1155 * memory group, unregistering will fail.
1157 * Returns -EINVAL if the memory group id is invalid, returns -EBUSY if some
1158 * memory blocks still belong to this memory group and returns 0 if
1166 return -EINVAL; in memory_group_unregister()
1170 return -EINVAL; in memory_group_unregister()
1171 if (!list_empty(&group->memory_blocks)) in memory_group_unregister()
1172 return -EBUSY; in memory_group_unregister()
1180 * This is an internal helper only to be used in core memory hotplug code to
1181 * lookup a memory group. We don't care about locking, as we don't expect a
1182 * memory group to get unregistered while adding memory to it -- because
1183 * the group and the memory is managed by the same driver.
1191 * This is an internal helper only to be used in core memory hotplug code to
1192 * walk all dynamic memory groups excluding a given memory group, either
1193 * belonging to a specific node, or belonging to any node.
1207 if (nid != NUMA_NO_NODE && group->nid != nid) in walk_dynamic_memory_groups()
1224 atomic_long_inc(&mem->nr_hwpoison); in memblk_nr_poison_inc()
1233 atomic_long_sub(i, &mem->nr_hwpoison); in memblk_nr_poison_sub()
1238 return atomic_long_read(&mem->nr_hwpoison); in memblk_nr_poison()