1 // SPDX-License-Identifier: GPL-2.0-or-later
2 
3 /*
4  * Functions for initialisaing, allocating, freeing and duplicating VMAs. Shared
5  * between CONFIG_MMU and non-CONFIG_MMU kernel configurations.
6  */
7 
8 #include "vma_internal.h"
9 #include "vma.h"
10 
11 /* SLAB cache for vm_area_struct structures */
12 static struct kmem_cache *vm_area_cachep;
13 
14 void __init vma_state_init(void)
15 {
16 	struct kmem_cache_args args = {
17 		.use_freeptr_offset = true,
18 		.freeptr_offset = offsetof(struct vm_area_struct, vm_freeptr),
19 	};
20 
21 	vm_area_cachep = kmem_cache_create("vm_area_struct",
22 			sizeof(struct vm_area_struct), &args,
23 			SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_TYPESAFE_BY_RCU|
24 			SLAB_ACCOUNT);
25 }
26 
27 struct vm_area_struct *vm_area_alloc(struct mm_struct *mm)
28 {
29 	struct vm_area_struct *vma;
30 
31 	vma = kmem_cache_alloc(vm_area_cachep, GFP_KERNEL);
32 	if (!vma)
33 		return NULL;
34 
35 	vma_init(vma, mm);
36 
37 	return vma;
38 }
39 
40 static void vm_area_init_from(const struct vm_area_struct *src,
41 			      struct vm_area_struct *dest)
42 {
43 	dest->vm_mm = src->vm_mm;
44 	dest->vm_ops = src->vm_ops;
45 	dest->vm_start = src->vm_start;
46 	dest->vm_end = src->vm_end;
47 	dest->anon_vma = src->anon_vma;
48 	dest->vm_pgoff = src->vm_pgoff;
49 	dest->vm_file = src->vm_file;
50 	dest->vm_private_data = src->vm_private_data;
51 	vm_flags_init(dest, src->vm_flags);
52 	memcpy(&dest->vm_page_prot, &src->vm_page_prot,
53 	       sizeof(dest->vm_page_prot));
54 	/*
55 	 * src->shared.rb may be modified concurrently when called from
56 	 * dup_mmap(), but the clone will reinitialize it.
57 	 */
58 	data_race(memcpy(&dest->shared, &src->shared, sizeof(dest->shared)));
59 	memcpy(&dest->vm_userfaultfd_ctx, &src->vm_userfaultfd_ctx,
60 	       sizeof(dest->vm_userfaultfd_ctx));
61 #ifdef CONFIG_ANON_VMA_NAME
62 	dest->anon_name = src->anon_name;
63 #endif
64 #ifdef CONFIG_SWAP
65 	memcpy(&dest->swap_readahead_info, &src->swap_readahead_info,
66 	       sizeof(dest->swap_readahead_info));
67 #endif
68 #ifndef CONFIG_MMU
69 	dest->vm_region = src->vm_region;
70 #endif
71 #ifdef CONFIG_NUMA
72 	dest->vm_policy = src->vm_policy;
73 #endif
74 #ifdef __HAVE_PFNMAP_TRACKING
75 	dest->pfnmap_track_ctx = NULL;
76 #endif
77 }
78 
79 #ifdef __HAVE_PFNMAP_TRACKING
80 static inline int vma_pfnmap_track_ctx_dup(struct vm_area_struct *orig,
81 		struct vm_area_struct *new)
82 {
83 	struct pfnmap_track_ctx *ctx = orig->pfnmap_track_ctx;
84 
85 	if (likely(!ctx))
86 		return 0;
87 
88 	/*
89 	 * We don't expect to ever hit this. If ever required, we would have
90 	 * to duplicate the tracking.
91 	 */
92 	if (unlikely(kref_read(&ctx->kref) >= REFCOUNT_MAX))
93 		return -ENOMEM;
94 	kref_get(&ctx->kref);
95 	new->pfnmap_track_ctx = ctx;
96 	return 0;
97 }
98 
99 static inline void vma_pfnmap_track_ctx_release(struct vm_area_struct *vma)
100 {
101 	struct pfnmap_track_ctx *ctx = vma->pfnmap_track_ctx;
102 
103 	if (likely(!ctx))
104 		return;
105 
106 	kref_put(&ctx->kref, pfnmap_track_ctx_release);
107 	vma->pfnmap_track_ctx = NULL;
108 }
109 #else
110 static inline int vma_pfnmap_track_ctx_dup(struct vm_area_struct *orig,
111 		struct vm_area_struct *new)
112 {
113 	return 0;
114 }
115 static inline void vma_pfnmap_track_ctx_release(struct vm_area_struct *vma)
116 {
117 }
118 #endif
119 
120 struct vm_area_struct *vm_area_dup(struct vm_area_struct *orig)
121 {
122 	struct vm_area_struct *new = kmem_cache_alloc(vm_area_cachep, GFP_KERNEL);
123 
124 	if (!new)
125 		return NULL;
126 
127 	ASSERT_EXCLUSIVE_WRITER(orig->vm_flags);
128 	ASSERT_EXCLUSIVE_WRITER(orig->vm_file);
129 	vm_area_init_from(orig, new);
130 
131 	if (vma_pfnmap_track_ctx_dup(orig, new)) {
132 		kmem_cache_free(vm_area_cachep, new);
133 		return NULL;
134 	}
135 	vma_lock_init(new, true);
136 	INIT_LIST_HEAD(&new->anon_vma_chain);
137 	vma_numab_state_init(new);
138 	dup_anon_vma_name(orig, new);
139 
140 	return new;
141 }
142 
143 void vm_area_free(struct vm_area_struct *vma)
144 {
145 	/* The vma should be detached while being destroyed. */
146 	vma_assert_detached(vma);
147 	vma_numab_state_free(vma);
148 	free_anon_vma_name(vma);
149 	vma_pfnmap_track_ctx_release(vma);
150 	kmem_cache_free(vm_area_cachep, vma);
151 }
152