332 	struct vm_struct *area;  in ioremap_page_range()  local
334 	area = find_vm_area((void *)addr);  in ioremap_page_range()
335 	if (!area || !(area->flags & VM_IOREMAP)) {  in ioremap_page_range()
339 	if (addr != (unsigned long)area->addr ||  in ioremap_page_range()
340 	    (void *)end != area->addr + get_vm_area_size(area)) {  in ioremap_page_range()
342 			  addr, end, (long)area->addr,  in ioremap_page_range()
343 			  (long)area->addr + get_vm_area_size(area));  in ioremap_page_range()
472  * @addr: start of the VM area to unmap
473  * @end: end of the VM area to unmap (non-inclusive)
646  * @addr: start of the VM area to map
647  * @end: end of the VM area to map (non-inclusive)
666 static int check_sparse_vm_area(struct vm_struct *area, unsigned long start,  in check_sparse_vm_area()  argument
670 	if (WARN_ON_ONCE(area->flags & VM_FLUSH_RESET_PERMS))  in check_sparse_vm_area()
672 	if (WARN_ON_ONCE(area->flags & VM_NO_GUARD))  in check_sparse_vm_area()
674 	if (WARN_ON_ONCE(!(area->flags & VM_SPARSE)))  in check_sparse_vm_area()
678 	if (start < (unsigned long)area->addr ||  in check_sparse_vm_area()
679 	    (void *)end > area->addr + get_vm_area_size(area))  in check_sparse_vm_area()
686  * @area: vm_area
691 int vm_area_map_pages(struct vm_struct *area, unsigned long start,  in vm_area_map_pages()  argument
696 	err = check_sparse_vm_area(area, start, end);  in vm_area_map_pages()
705  * @area: vm_area
709 void vm_area_unmap_pages(struct vm_struct *area, unsigned long start,  in vm_area_unmap_pages()  argument
712 	if (check_sparse_vm_area(area, start, end))  in vm_area_unmap_pages()
836  * find a lowest match of free area.
1355  * free area is inserted. If VA has been merged, it is
1403 			/* Point to the new merged area. */  in __merge_or_add_vmap_area()
1434 			/* Point to the new merged area. */  in __merge_or_add_vmap_area()
1755  * Returns a start address of the newly allocated area, if success.
1823 	 * when fit type of free area is NE_FIT_TYPE. It guarantees that  in preload_this_cpu_lock()
2343  * Free a vmap area, caller ensuring that the area has been unmapped,
2380  * Free and unmap a vmap area
2491 #define VMAP_RAM		0x1 /* indicates vm_map_ram area*/
3041  * vm_area_add_early - add vmap area early during boot
3044  * This function is used to add fixed kernel vm area to vmlist before
3067  * vm_area_register_early - register vmap area early during boot
3071  * This function is used to register kernel vm area before
3115 	struct vm_struct *area;  in __get_vm_area_node()  local
3127 	area = kzalloc_node(sizeof(*area), gfp_mask & GFP_RECLAIM_MASK, node);  in __get_vm_area_node()
3128 	if (unlikely(!area))  in __get_vm_area_node()
3134 	area->flags = flags;  in __get_vm_area_node()
3135 	area->caller = caller;  in __get_vm_area_node()
3136 	area->requested_size = requested_size;  in __get_vm_area_node()
3138 	va = alloc_vmap_area(size, align, start, end, node, gfp_mask, 0, area);  in __get_vm_area_node()
3140 		kfree(area);  in __get_vm_area_node()
3153 		area->addr = kasan_unpoison_vmalloc(area->addr, requested_size,  in __get_vm_area_node()
3156 	return area;  in __get_vm_area_node()
3168  * get_vm_area - reserve a contiguous kernel virtual area
3169  * @size:	 size of the area
3172  * Search an area of @size in the kernel virtual mapping area,
3173  * and reserved it for out purposes.  Returns the area descriptor
3176  * Return: the area descriptor on success or %NULL on failure.
3195  * find_vm_area - find a continuous kernel virtual area
3198  * Search for the kernel VM area starting at @addr, and return it.
3202  * Return: the area descriptor on success or %NULL on failure.
3216  * remove_vm_area - find and remove a continuous kernel virtual area
3219  * Search for the kernel VM area starting at @addr, and remove it.
3220  * This function returns the found VM area, but using it is NOT safe
3223  * Return: the area descriptor on success or %NULL on failure.
3250 static inline void set_area_direct_map(const struct vm_struct *area,  in set_area_direct_map()  argument
3256 	for (i = 0; i < area->nr_pages; i++)  in set_area_direct_map()
3257 		if (page_address(area->pages[i]))  in set_area_direct_map()
3258 			set_direct_map(area->pages[i]);  in set_area_direct_map()
3264 static void vm_reset_perms(struct vm_struct *area)  in vm_reset_perms()  argument
3267 	unsigned int page_order = vm_area_page_order(area);  in vm_reset_perms()
3275 	for (i = 0; i < area->nr_pages; i += 1U << page_order) {  in vm_reset_perms()
3276 		unsigned long addr = (unsigned long)page_address(area->pages[i]);  in vm_reset_perms()
3293 	set_area_direct_map(area, set_direct_map_invalid_noflush);  in vm_reset_perms()
3295 	set_area_direct_map(area, set_direct_map_default_noflush);  in vm_reset_perms()
3335  * Free the virtually continuous memory area starting at @addr, as obtained
3367 		WARN(1, KERN_ERR "Trying to vfree() nonexistent vm area (%p)\n",  in vfree()
3398  * Free the virtually contiguous memory area starting at @addr,
3414 		WARN(1, KERN_ERR "Trying to vunmap() nonexistent vm area (%p)\n",  in vunmap()
3435  * Return: the address of the area or %NULL on failure
3440 	struct vm_struct *area;  in vmap()  local
3460 	area = get_vm_area_caller(size, flags, __builtin_return_address(0));  in vmap()
3461 	if (!area)  in vmap()
3464 	addr = (unsigned long)area->addr;  in vmap()
3467 		vunmap(area->addr);  in vmap()
3472 		area->pages = pages;  in vmap()
3473 		area->nr_pages = count;  in vmap()
3475 	return area->addr;  in vmap()
3514 	struct vm_struct *area;  in vmap_pfn()  local
3516 	area = get_vm_area_caller(count * PAGE_SIZE, VM_IOREMAP,  in vmap_pfn()
3518 	if (!area)  in vmap_pfn()
3520 	if (apply_to_page_range(&init_mm, (unsigned long)area->addr,  in vmap_pfn()
3522 		free_vm_area(area);  in vmap_pfn()
3526 	flush_cache_vmap((unsigned long)area->addr,  in vmap_pfn()
3527 			 (unsigned long)area->addr + count * PAGE_SIZE);  in vmap_pfn()
3529 	return area->addr;  in vmap_pfn()
3624 static void *__vmalloc_area_node(struct vm_struct *area, gfp_t gfp_mask,  in __vmalloc_area_node()  argument
3630 	unsigned long addr = (unsigned long)area->addr;  in __vmalloc_area_node()
3631 	unsigned long size = get_vm_area_size(area);  in __vmalloc_area_node()
3645 		area->pages = __vmalloc_node_noprof(array_size, 1, nested_gfp, node,  in __vmalloc_area_node()
3646 					area->caller);  in __vmalloc_area_node()
3648 		area->pages = kmalloc_node_noprof(array_size, nested_gfp, node);  in __vmalloc_area_node()
3651 	if (!area->pages) {  in __vmalloc_area_node()
3655 		free_vm_area(area);  in __vmalloc_area_node()
3659 	set_vm_area_page_order(area, page_shift - PAGE_SHIFT);  in __vmalloc_area_node()
3660 	page_order = vm_area_page_order(area);  in __vmalloc_area_node()
3670 	area->nr_pages = vm_area_alloc_pages((page_order ?  in __vmalloc_area_node()
3672 		node, page_order, nr_small_pages, area->pages);  in __vmalloc_area_node()
3674 	atomic_long_add(area->nr_pages, &nr_vmalloc_pages);  in __vmalloc_area_node()
3678 		for (i = 0; i < area->nr_pages; i++)  in __vmalloc_area_node()
3679 			mod_memcg_page_state(area->pages[i], MEMCG_VMALLOC, 1);  in __vmalloc_area_node()
3686 	if (area->nr_pages != nr_small_pages) {  in __vmalloc_area_node()
3700 				area->nr_pages * PAGE_SIZE);  in __vmalloc_area_node()
3714 		ret = vmap_pages_range(addr, addr + size, prot, area->pages,  in __vmalloc_area_node()
3728 			area->nr_pages * PAGE_SIZE);  in __vmalloc_area_node()
3732 	return area->addr;  in __vmalloc_area_node()
3735 	vfree(area->addr);  in __vmalloc_area_node()
3743  * @start:		  vm area range start
3744  * @end:		  vm area range end
3747  * @vm_flags:		  additional vm area flags (e.g. %VM_NO_GUARD)
3765  * Return: the address of the area or %NULL on failure
3772 	struct vm_struct *area;  in __vmalloc_node_range_noprof()  local
3805 	area = __get_vm_area_node(size, align, shift, VM_ALLOC |  in __vmalloc_node_range_noprof()
3808 	if (!area) {  in __vmalloc_node_range_noprof()
3845 	ret = __vmalloc_area_node(area, gfp_mask, prot, shift, node);  in __vmalloc_node_range_noprof()
3862 	area->addr = kasan_unpoison_vmalloc(area->addr, size, kasan_flags);  in __vmalloc_node_range_noprof()
3869 	clear_vm_uninitialized_flag(area);  in __vmalloc_node_range_noprof()
3872 		kmemleak_vmalloc(area, PAGE_ALIGN(size), gfp_mask);  in __vmalloc_node_range_noprof()
3874 	return area->addr;  in __vmalloc_node_range_noprof()
3990  * The resulting memory area is zeroed so it can be mapped to userspace
4080 			WARN(1, "Trying to vrealloc() nonexistent vm area (%p)\n", p);  in vrealloc_noprof()
4087 			 "vrealloc() has mismatched area vs requested sizes (%p)\n", p))  in vrealloc_noprof()
4164  * The resulting memory area is 32bit addressable and zeroed so it can be
4223 		 * To do safe access to this _mapped_ area, we need lock. But  in aligned_vread_iter()
4262 	 * If it's area created by vm_map_ram() interface directly, but  in vmap_ram_vread_iter()
4272 	 * Area is split into regions and tracked with vmap_block, read out  in vmap_ram_vread_iter()
4332  * vread_iter() - read vmalloc area in a safe way to an iterator.
4337  * This function checks that addr is a valid vmalloc'ed area, and
4338  * copy data from that area to a given buffer. If the given memory range
4340  * proper area of @buf. If there are memory holes, they'll be zero-filled.
4341  * IOREMAP area is treated as memory hole and no copy is done.
4344  * vm_struct area, returns 0. @buf should be kernel's buffer.
4347  * should know vmalloc() area is valid and can use memcpy().
4348  * This is for routines which have to access vmalloc area without
4353  * include any intersection with valid vmalloc area
4389 		 * VMAP_BLOCK indicates a sub-type of vm_map_ram area, need  in vread_iter()
4428 		else /* IOREMAP | SPARSE area is treated as memory hole */  in vread_iter()
4462  * @size:		size of map area
4466  * This function checks that @kaddr is a valid vmalloc'ed area,
4477 	struct vm_struct *area;  in remap_vmalloc_range_partial()  local
4489 	area = find_vm_area(kaddr);  in remap_vmalloc_range_partial()
4490 	if (!area)  in remap_vmalloc_range_partial()
4493 	if (!(area->flags & (VM_USERMAP | VM_DMA_COHERENT)))  in remap_vmalloc_range_partial()
4497 	    end_index > get_vm_area_size(area))  in remap_vmalloc_range_partial()
4527  * This function checks that addr is a valid vmalloc'ed area, and
4542 void free_vm_area(struct vm_struct *area)  in free_vm_area()  argument
4545 	ret = remove_vm_area(area->addr);  in free_vm_area()
4546 	BUG_ON(ret != area);  in free_vm_area()
4547 	kfree(area);  in free_vm_area()
4561  * Returns: vmap_area if it is found. If there is no such area
4621  * @offsets: array containing offset of each area
4622  * @sizes: array containing size of each area
4639  * base address is pulled down to fit the area. Scanning is repeated till
4651 	int area, area2, last_area, term_area;  in pcpu_get_vm_areas()  local
4657 	for (last_area = 0, area = 0; area < nr_vms; area++) {  in pcpu_get_vm_areas()
4658 		start = offsets[area];  in pcpu_get_vm_areas()
4659 		end = start + sizes[area];  in pcpu_get_vm_areas()
4662 		BUG_ON(!IS_ALIGNED(offsets[area], align));  in pcpu_get_vm_areas()
4663 		BUG_ON(!IS_ALIGNED(sizes[area], align));  in pcpu_get_vm_areas()
4665 		/* detect the area with the highest address */  in pcpu_get_vm_areas()
4667 			last_area = area;  in pcpu_get_vm_areas()
4669 		for (area2 = area + 1; area2 < nr_vms; area2++) {  in pcpu_get_vm_areas()
4688 	for (area = 0; area < nr_vms; area++) {  in pcpu_get_vm_areas()
4689 		vas[area] = kmem_cache_zalloc(vmap_area_cachep, GFP_KERNEL);  in pcpu_get_vm_areas()
4690 		vms[area] = kzalloc(sizeof(struct vm_struct), GFP_KERNEL);  in pcpu_get_vm_areas()
4691 		if (!vas[area] || !vms[area])  in pcpu_get_vm_areas()
4697 	/* start scanning - we scan from the top, begin with the last area */  in pcpu_get_vm_areas()
4698 	area = term_area = last_area;  in pcpu_get_vm_areas()
4699 	start = offsets[area];  in pcpu_get_vm_areas()
4700 	end = start + sizes[area];  in pcpu_get_vm_areas()
4725 			term_area = area;  in pcpu_get_vm_areas()
4735 			term_area = area;  in pcpu_get_vm_areas()
4740 		 * This area fits, move on to the previous one.  If  in pcpu_get_vm_areas()
4743 		area = (area + nr_vms - 1) % nr_vms;  in pcpu_get_vm_areas()
4744 		if (area == term_area)  in pcpu_get_vm_areas()
4747 		start = offsets[area];  in pcpu_get_vm_areas()
4748 		end = start + sizes[area];  in pcpu_get_vm_areas()
4753 	for (area = 0; area < nr_vms; area++) {  in pcpu_get_vm_areas()
4756 		start = base + offsets[area];  in pcpu_get_vm_areas()
4757 		size = sizes[area];  in pcpu_get_vm_areas()
4770 		/* Allocated area. */  in pcpu_get_vm_areas()
4771 		va = vas[area];  in pcpu_get_vm_areas()
4779 	for (area = 0; area < nr_vms; area++) {  in pcpu_get_vm_areas()
4780 		if (kasan_populate_vmalloc(vas[area]->va_start, sizes[area]))  in pcpu_get_vm_areas()
4785 	for (area = 0; area < nr_vms; area++) {  in pcpu_get_vm_areas()
4786 		struct vmap_node *vn = addr_to_node(vas[area]->va_start);  in pcpu_get_vm_areas()
4789 		insert_vmap_area(vas[area], &vn->busy.root, &vn->busy.head);  in pcpu_get_vm_areas()
4790 		setup_vmalloc_vm(vms[area], vas[area], VM_ALLOC,  in pcpu_get_vm_areas()
4801 	for (area = 0; area < nr_vms; area++)  in pcpu_get_vm_areas()
4802 		vms[area]->addr = kasan_unpoison_vmalloc(vms[area]->addr,  in pcpu_get_vm_areas()
4803 				vms[area]->size, KASAN_VMALLOC_PROT_NORMAL);  in pcpu_get_vm_areas()
4815 	while (area--) {  in pcpu_get_vm_areas()
4816 		orig_start = vas[area]->va_start;  in pcpu_get_vm_areas()
4817 		orig_end = vas[area]->va_end;  in pcpu_get_vm_areas()
4818 		va = merge_or_add_vmap_area_augment(vas[area], &free_vmap_area_root,  in pcpu_get_vm_areas()
4824 		vas[area] = NULL;  in pcpu_get_vm_areas()
4834 		for (area = 0; area < nr_vms; area++) {  in pcpu_get_vm_areas()
4835 			if (vas[area])  in pcpu_get_vm_areas()
4838 			vas[area] = kmem_cache_zalloc(  in pcpu_get_vm_areas()
4840 			if (!vas[area])  in pcpu_get_vm_areas()
4848 	for (area = 0; area < nr_vms; area++) {  in pcpu_get_vm_areas()
4849 		if (vas[area])  in pcpu_get_vm_areas()
4850 			kmem_cache_free(vmap_area_cachep, vas[area]);  in pcpu_get_vm_areas()
4852 		kfree(vms[area]);  in pcpu_get_vm_areas()
4866 	for (area = 0; area < nr_vms; area++) {  in pcpu_get_vm_areas()
4867 		orig_start = vas[area]->va_start;  in pcpu_get_vm_areas()
4868 		orig_end = vas[area]->va_end;  in pcpu_get_vm_areas()
4869 		va = merge_or_add_vmap_area_augment(vas[area], &free_vmap_area_root,  in pcpu_get_vm_areas()
4875 		vas[area] = NULL;  in pcpu_get_vm_areas()
4876 		kfree(vms[area]);  in pcpu_get_vm_areas()