1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2011 NetApp, Inc.
5 * All rights reserved.
6 */
7
8 #include <sys/types.h>
9 #include <sys/lock.h>
10 #include <sys/malloc.h>
11 #include <sys/sx.h>
12 #include <sys/systm.h>
13
14 #include <machine/vmm.h>
15
16 #include <vm/vm.h>
17 #include <vm/vm_param.h>
18 #include <vm/vm_extern.h>
19 #include <vm/pmap.h>
20 #include <vm/vm_map.h>
21 #include <vm/vm_object.h>
22 #include <vm/vm_page.h>
23
24 #include <dev/vmm/vmm_dev.h>
25 #include <dev/vmm/vmm_mem.h>
26 #include <dev/vmm/vmm_vm.h>
27
28 static void vm_free_memmap(struct vm *vm, int ident);
29
30 int
vm_mem_init(struct vm_mem * mem,vm_offset_t lo,vm_offset_t hi)31 vm_mem_init(struct vm_mem *mem, vm_offset_t lo, vm_offset_t hi)
32 {
33 mem->mem_vmspace = vmmops_vmspace_alloc(lo, hi);
34 if (mem->mem_vmspace == NULL)
35 return (ENOMEM);
36 sx_init(&mem->mem_segs_lock, "vm_mem_segs");
37 return (0);
38 }
39
40 static bool
sysmem_mapping(struct vm_mem * mem,int idx)41 sysmem_mapping(struct vm_mem *mem, int idx)
42 {
43 if (mem->mem_maps[idx].len != 0 &&
44 mem->mem_segs[mem->mem_maps[idx].segid].sysmem)
45 return (true);
46 else
47 return (false);
48 }
49
50 bool
vm_memseg_sysmem(struct vm * vm,int ident)51 vm_memseg_sysmem(struct vm *vm, int ident)
52 {
53 struct vm_mem *mem;
54
55 mem = vm_mem(vm);
56 vm_assert_memseg_locked(vm);
57
58 if (ident < 0 || ident >= VM_MAX_MEMSEGS)
59 return (false);
60
61 return (mem->mem_segs[ident].sysmem);
62 }
63
64 void
vm_mem_cleanup(struct vm * vm)65 vm_mem_cleanup(struct vm *vm)
66 {
67 struct vm_mem *mem;
68
69 mem = vm_mem(vm);
70
71 /*
72 * System memory is removed from the guest address space only when
73 * the VM is destroyed. This is because the mapping remains the same
74 * across VM reset.
75 *
76 * Device memory can be relocated by the guest (e.g. using PCI BARs)
77 * so those mappings are removed on a VM reset.
78 */
79 for (int i = 0; i < VM_MAX_MEMMAPS; i++) {
80 if (!sysmem_mapping(mem, i))
81 vm_free_memmap(vm, i);
82 }
83 }
84
85 void
vm_mem_destroy(struct vm * vm)86 vm_mem_destroy(struct vm *vm)
87 {
88 struct vm_mem *mem;
89
90 mem = vm_mem(vm);
91 vm_assert_memseg_xlocked(vm);
92
93 for (int i = 0; i < VM_MAX_MEMMAPS; i++) {
94 if (sysmem_mapping(mem, i))
95 vm_free_memmap(vm, i);
96 }
97
98 for (int i = 0; i < VM_MAX_MEMSEGS; i++)
99 vm_free_memseg(vm, i);
100
101 vmmops_vmspace_free(mem->mem_vmspace);
102
103 sx_xunlock(&mem->mem_segs_lock);
104 sx_destroy(&mem->mem_segs_lock);
105 }
106
107 struct vmspace *
vm_vmspace(struct vm * vm)108 vm_vmspace(struct vm *vm)
109 {
110 struct vm_mem *mem;
111
112 mem = vm_mem(vm);
113 return (mem->mem_vmspace);
114 }
115
116 void
vm_slock_memsegs(struct vm * vm)117 vm_slock_memsegs(struct vm *vm)
118 {
119 sx_slock(&vm_mem(vm)->mem_segs_lock);
120 }
121
122 void
vm_xlock_memsegs(struct vm * vm)123 vm_xlock_memsegs(struct vm *vm)
124 {
125 sx_xlock(&vm_mem(vm)->mem_segs_lock);
126 }
127
128 void
vm_unlock_memsegs(struct vm * vm)129 vm_unlock_memsegs(struct vm *vm)
130 {
131 sx_unlock(&vm_mem(vm)->mem_segs_lock);
132 }
133
134 void
vm_assert_memseg_locked(struct vm * vm)135 vm_assert_memseg_locked(struct vm *vm)
136 {
137 sx_assert(&vm_mem(vm)->mem_segs_lock, SX_LOCKED);
138 }
139
140 void
vm_assert_memseg_xlocked(struct vm * vm)141 vm_assert_memseg_xlocked(struct vm *vm)
142 {
143 sx_assert(&vm_mem(vm)->mem_segs_lock, SX_XLOCKED);
144 }
145
146 /*
147 * Return 'true' if 'gpa' is allocated in the guest address space.
148 *
149 * This function is called in the context of a running vcpu which acts as
150 * an implicit lock on 'vm->mem_maps[]'.
151 */
152 bool
vm_mem_allocated(struct vcpu * vcpu,vm_paddr_t gpa)153 vm_mem_allocated(struct vcpu *vcpu, vm_paddr_t gpa)
154 {
155 struct vm *vm = vcpu_vm(vcpu);
156 struct vm_mem_map *mm;
157 int i;
158
159 #ifdef INVARIANTS
160 int hostcpu, state;
161 state = vcpu_get_state(vcpu, &hostcpu);
162 KASSERT(state == VCPU_RUNNING && hostcpu == curcpu,
163 ("%s: invalid vcpu state %d/%d", __func__, state, hostcpu));
164 #endif
165
166 for (i = 0; i < VM_MAX_MEMMAPS; i++) {
167 mm = &vm_mem(vm)->mem_maps[i];
168 if (mm->len != 0 && gpa >= mm->gpa && gpa < mm->gpa + mm->len)
169 return (true); /* 'gpa' is sysmem or devmem */
170 }
171
172 return (false);
173 }
174
175 int
vm_alloc_memseg(struct vm * vm,int ident,size_t len,bool sysmem,struct domainset * obj_domainset)176 vm_alloc_memseg(struct vm *vm, int ident, size_t len, bool sysmem,
177 struct domainset *obj_domainset)
178 {
179 struct vm_mem_seg *seg;
180 struct vm_mem *mem;
181 vm_object_t obj;
182
183 mem = vm_mem(vm);
184 vm_assert_memseg_xlocked(vm);
185
186 if (ident < 0 || ident >= VM_MAX_MEMSEGS)
187 return (EINVAL);
188
189 if (len == 0 || (len & PAGE_MASK))
190 return (EINVAL);
191
192 seg = &mem->mem_segs[ident];
193 if (seg->object != NULL) {
194 if (seg->len == len && seg->sysmem == sysmem)
195 return (EEXIST);
196 else
197 return (EINVAL);
198 }
199
200 /*
201 * When given an impossible policy, signal an
202 * error to the user.
203 */
204 if (obj_domainset != NULL && domainset_empty_vm(obj_domainset))
205 return (EINVAL);
206 obj = vm_object_allocate(OBJT_SWAP, len >> PAGE_SHIFT);
207 if (obj == NULL)
208 return (ENOMEM);
209
210 seg->len = len;
211 seg->object = obj;
212 if (obj_domainset != NULL)
213 seg->object->domain.dr_policy = obj_domainset;
214 seg->sysmem = sysmem;
215
216 return (0);
217 }
218
219 int
vm_get_memseg(struct vm * vm,int ident,size_t * len,bool * sysmem,vm_object_t * objptr)220 vm_get_memseg(struct vm *vm, int ident, size_t *len, bool *sysmem,
221 vm_object_t *objptr)
222 {
223 struct vm_mem *mem;
224 struct vm_mem_seg *seg;
225
226 mem = vm_mem(vm);
227
228 vm_assert_memseg_locked(vm);
229
230 if (ident < 0 || ident >= VM_MAX_MEMSEGS)
231 return (EINVAL);
232
233 seg = &mem->mem_segs[ident];
234 if (len)
235 *len = seg->len;
236 if (sysmem)
237 *sysmem = seg->sysmem;
238 if (objptr)
239 *objptr = seg->object;
240 return (0);
241 }
242
243 void
vm_free_memseg(struct vm * vm,int ident)244 vm_free_memseg(struct vm *vm, int ident)
245 {
246 struct vm_mem_seg *seg;
247
248 KASSERT(ident >= 0 && ident < VM_MAX_MEMSEGS,
249 ("%s: invalid memseg ident %d", __func__, ident));
250
251 seg = &vm_mem(vm)->mem_segs[ident];
252 if (seg->object != NULL) {
253 vm_object_deallocate(seg->object);
254 bzero(seg, sizeof(struct vm_mem_seg));
255 }
256 }
257
258 int
vm_mmap_memseg(struct vm * vm,vm_paddr_t gpa,int segid,vm_ooffset_t first,size_t len,int prot,int flags)259 vm_mmap_memseg(struct vm *vm, vm_paddr_t gpa, int segid, vm_ooffset_t first,
260 size_t len, int prot, int flags)
261 {
262 struct vm_mem *mem;
263 struct vm_mem_seg *seg;
264 struct vm_mem_map *m, *map;
265 struct vm_map *vmmap;
266 vm_ooffset_t last;
267 int i, error;
268
269 if (prot == 0 || (prot & ~(VM_PROT_ALL)) != 0)
270 return (EINVAL);
271
272 if (flags & ~VM_MEMMAP_F_WIRED)
273 return (EINVAL);
274
275 if (segid < 0 || segid >= VM_MAX_MEMSEGS)
276 return (EINVAL);
277
278 mem = vm_mem(vm);
279 seg = &mem->mem_segs[segid];
280 if (seg->object == NULL)
281 return (EINVAL);
282
283 if (first + len < first || gpa + len < gpa)
284 return (EINVAL);
285 last = first + len;
286 if (first >= last || last > seg->len)
287 return (EINVAL);
288
289 if ((gpa | first | last) & PAGE_MASK)
290 return (EINVAL);
291
292 map = NULL;
293 for (i = 0; i < VM_MAX_MEMMAPS; i++) {
294 m = &mem->mem_maps[i];
295 if (m->len == 0) {
296 map = m;
297 break;
298 }
299 }
300 if (map == NULL)
301 return (ENOSPC);
302
303 vmmap = &mem->mem_vmspace->vm_map;
304 vm_map_lock(vmmap);
305 error = vm_map_insert(vmmap, seg->object, first, gpa, gpa + len,
306 prot, prot, 0);
307 vm_map_unlock(vmmap);
308 if (error != KERN_SUCCESS)
309 return (vm_mmap_to_errno(error));
310 vm_object_reference(seg->object);
311
312 if (flags & VM_MEMMAP_F_WIRED) {
313 error = vm_map_wire(vmmap, gpa, gpa + len,
314 VM_MAP_WIRE_USER | VM_MAP_WIRE_NOHOLES);
315 if (error != KERN_SUCCESS) {
316 vm_map_remove(vmmap, gpa, gpa + len);
317 return (error == KERN_RESOURCE_SHORTAGE ? ENOMEM :
318 EFAULT);
319 }
320 }
321
322 map->gpa = gpa;
323 map->len = len;
324 map->segoff = first;
325 map->segid = segid;
326 map->prot = prot;
327 map->flags = flags;
328 return (0);
329 }
330
331 int
vm_munmap_memseg(struct vm * vm,vm_paddr_t gpa,size_t len)332 vm_munmap_memseg(struct vm *vm, vm_paddr_t gpa, size_t len)
333 {
334 struct vm_mem *mem;
335 struct vm_mem_map *m;
336 int i;
337
338 mem = vm_mem(vm);
339 for (i = 0; i < VM_MAX_MEMMAPS; i++) {
340 m = &mem->mem_maps[i];
341 #ifdef VM_MEMMAP_F_IOMMU
342 if ((m->flags & VM_MEMMAP_F_IOMMU) != 0)
343 continue;
344 #endif
345 if (m->gpa == gpa && m->len == len) {
346 vm_free_memmap(vm, i);
347 return (0);
348 }
349 }
350
351 return (EINVAL);
352 }
353
354 int
vm_mmap_getnext(struct vm * vm,vm_paddr_t * gpa,int * segid,vm_ooffset_t * segoff,size_t * len,int * prot,int * flags)355 vm_mmap_getnext(struct vm *vm, vm_paddr_t *gpa, int *segid,
356 vm_ooffset_t *segoff, size_t *len, int *prot, int *flags)
357 {
358 struct vm_mem *mem;
359 struct vm_mem_map *mm, *mmnext;
360 int i;
361
362 mem = vm_mem(vm);
363
364 mmnext = NULL;
365 for (i = 0; i < VM_MAX_MEMMAPS; i++) {
366 mm = &mem->mem_maps[i];
367 if (mm->len == 0 || mm->gpa < *gpa)
368 continue;
369 if (mmnext == NULL || mm->gpa < mmnext->gpa)
370 mmnext = mm;
371 }
372
373 if (mmnext != NULL) {
374 *gpa = mmnext->gpa;
375 if (segid)
376 *segid = mmnext->segid;
377 if (segoff)
378 *segoff = mmnext->segoff;
379 if (len)
380 *len = mmnext->len;
381 if (prot)
382 *prot = mmnext->prot;
383 if (flags)
384 *flags = mmnext->flags;
385 return (0);
386 } else {
387 return (ENOENT);
388 }
389 }
390
391 static void
vm_free_memmap(struct vm * vm,int ident)392 vm_free_memmap(struct vm *vm, int ident)
393 {
394 struct vm_mem_map *mm;
395 int error __diagused;
396
397 mm = &vm_mem(vm)->mem_maps[ident];
398 if (mm->len) {
399 error = vm_map_remove(&vm_vmspace(vm)->vm_map, mm->gpa,
400 mm->gpa + mm->len);
401 KASSERT(error == KERN_SUCCESS, ("%s: vm_map_remove error %d",
402 __func__, error));
403 bzero(mm, sizeof(struct vm_mem_map));
404 }
405 }
406
407 vm_paddr_t
vmm_sysmem_maxaddr(struct vm * vm)408 vmm_sysmem_maxaddr(struct vm *vm)
409 {
410 struct vm_mem *mem;
411 struct vm_mem_map *mm;
412 vm_paddr_t maxaddr;
413 int i;
414
415 mem = vm_mem(vm);
416 maxaddr = 0;
417 for (i = 0; i < VM_MAX_MEMMAPS; i++) {
418 mm = &mem->mem_maps[i];
419 if (sysmem_mapping(mem, i)) {
420 if (maxaddr < mm->gpa + mm->len)
421 maxaddr = mm->gpa + mm->len;
422 }
423 }
424 return (maxaddr);
425 }
426
427 static void *
_vm_gpa_hold(struct vm * vm,vm_paddr_t gpa,size_t len,int reqprot,void ** cookie)428 _vm_gpa_hold(struct vm *vm, vm_paddr_t gpa, size_t len, int reqprot,
429 void **cookie)
430 {
431 struct vm_mem_map *mm;
432 vm_page_t m;
433 int i, count, pageoff;
434
435 pageoff = gpa & PAGE_MASK;
436 if (len > PAGE_SIZE - pageoff)
437 panic("vm_gpa_hold: invalid gpa/len: 0x%016lx/%lu", gpa, len);
438
439 count = 0;
440 for (i = 0; i < VM_MAX_MEMMAPS; i++) {
441 mm = &vm_mem(vm)->mem_maps[i];
442 if (gpa >= mm->gpa && gpa < mm->gpa + mm->len) {
443 count = vm_fault_quick_hold_pages(
444 &vm_vmspace(vm)->vm_map, trunc_page(gpa),
445 PAGE_SIZE, reqprot, &m, 1);
446 break;
447 }
448 }
449
450 if (count == 1) {
451 *cookie = m;
452 return ((void *)(PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m)) + pageoff));
453 } else {
454 *cookie = NULL;
455 return (NULL);
456 }
457 }
458
459 void *
vm_gpa_hold(struct vcpu * vcpu,vm_paddr_t gpa,size_t len,int reqprot,void ** cookie)460 vm_gpa_hold(struct vcpu *vcpu, vm_paddr_t gpa, size_t len, int reqprot,
461 void **cookie)
462 {
463 #ifdef INVARIANTS
464 /*
465 * The current vcpu should be frozen to ensure 'vm_memmap[]'
466 * stability.
467 */
468 int state = vcpu_get_state(vcpu, NULL);
469 KASSERT(state == VCPU_FROZEN, ("%s: invalid vcpu state %d",
470 __func__, state));
471 #endif
472 return (_vm_gpa_hold(vcpu_vm(vcpu), gpa, len, reqprot, cookie));
473 }
474
475 void *
vm_gpa_hold_global(struct vm * vm,vm_paddr_t gpa,size_t len,int reqprot,void ** cookie)476 vm_gpa_hold_global(struct vm *vm, vm_paddr_t gpa, size_t len, int reqprot,
477 void **cookie)
478 {
479 vm_assert_memseg_locked(vm);
480 return (_vm_gpa_hold(vm, gpa, len, reqprot, cookie));
481 }
482
483 void
vm_gpa_release(void * cookie)484 vm_gpa_release(void *cookie)
485 {
486 vm_page_t m = cookie;
487
488 vm_page_unwire(m, PQ_ACTIVE);
489 }
490