1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2025 Ruslan Bukin <br@bsdpad.com>
5 *
6 * This software was developed by the University of Cambridge Computer
7 * Laboratory (Department of Computer Science and Technology) under Innovate
8 * UK project 105694, "Digital Security by Design (DSbD) Technology Platform
9 * Prototype".
10 *
11 * Redistribution and use in source and binary forms, with or without
12 * modification, are permitted provided that the following conditions
13 * are met:
14 * 1. Redistributions of source code must retain the above copyright
15 * notice, this list of conditions and the following disclaimer.
16 * 2. Redistributions in binary form must reproduce the above copyright
17 * notice, this list of conditions and the following disclaimer in the
18 * documentation and/or other materials provided with the distribution.
19 *
20 * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND
21 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
22 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
23 * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
24 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
25 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
26 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
27 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
28 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
29 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
30 * SUCH DAMAGE.
31 */
32
33 #include <sys/param.h>
34 #include <sys/systm.h>
35 #include <sys/smp.h>
36 #include <sys/kernel.h>
37 #include <sys/malloc.h>
38 #include <sys/lock.h>
39 #include <sys/mutex.h>
40 #include <sys/bus.h>
41
42 #include <machine/vmm.h>
43
44 #include <dev/vmm/vmm_vm.h>
45
46 #include "riscv.h"
47 #include "vmm_fence.h"
48
49 static bool
vmm_fence_dequeue(struct hypctx * hypctx,struct vmm_fence * new_fence)50 vmm_fence_dequeue(struct hypctx *hypctx, struct vmm_fence *new_fence)
51 {
52 struct vmm_fence *queue;
53 struct vmm_fence *fence;
54
55 mtx_lock_spin(&hypctx->fence_queue_mtx);
56 queue = hypctx->fence_queue;
57 fence = &queue[hypctx->fence_queue_head];
58 if (fence->type != VMM_RISCV_FENCE_INVALID) {
59 *new_fence = *fence;
60 fence->type = VMM_RISCV_FENCE_INVALID;
61 hypctx->fence_queue_head =
62 (hypctx->fence_queue_head + 1) % VMM_FENCE_QUEUE_SIZE;
63 } else {
64 mtx_unlock_spin(&hypctx->fence_queue_mtx);
65 return (false);
66 }
67 mtx_unlock_spin(&hypctx->fence_queue_mtx);
68
69 return (true);
70 }
71
72 static bool
vmm_fence_enqueue(struct hypctx * hypctx,struct vmm_fence * new_fence)73 vmm_fence_enqueue(struct hypctx *hypctx, struct vmm_fence *new_fence)
74 {
75 struct vmm_fence *queue;
76 struct vmm_fence *fence;
77
78 mtx_lock_spin(&hypctx->fence_queue_mtx);
79 queue = hypctx->fence_queue;
80 fence = &queue[hypctx->fence_queue_tail];
81 if (fence->type == VMM_RISCV_FENCE_INVALID) {
82 *fence = *new_fence;
83 hypctx->fence_queue_tail =
84 (hypctx->fence_queue_tail + 1) % VMM_FENCE_QUEUE_SIZE;
85 } else {
86 mtx_unlock_spin(&hypctx->fence_queue_mtx);
87 return (false);
88 }
89 mtx_unlock_spin(&hypctx->fence_queue_mtx);
90
91 return (true);
92 }
93
94 static void
vmm_fence_process_one(struct vmm_fence * fence)95 vmm_fence_process_one(struct vmm_fence *fence)
96 {
97 uint64_t va;
98
99 KASSERT(fence->type == VMM_RISCV_FENCE_VMA ||
100 fence->type == VMM_RISCV_FENCE_VMA_ASID,
101 ("%s: wrong fence type %d", __func__, fence->type));
102
103 switch (fence->type) {
104 case VMM_RISCV_FENCE_VMA:
105 for (va = fence->start; va < fence->start + fence->size;
106 va += PAGE_SIZE)
107 sfence_vma_page(va);
108 break;
109 case VMM_RISCV_FENCE_VMA_ASID:
110 if ((fence->start == 0 && fence->size == 0) ||
111 fence->size == -1)
112 sfence_vma_asid(fence->asid);
113 else
114 for (va = fence->start; va < fence->start + fence->size;
115 va += PAGE_SIZE)
116 sfence_vma_asid_page(fence->asid, va);
117 break;
118 default:
119 break;
120 }
121 }
122
123 void
vmm_fence_process(struct hypctx * hypctx)124 vmm_fence_process(struct hypctx *hypctx)
125 {
126 struct vmm_fence fence;
127 int pending;
128
129 pending = atomic_readandclear_32(&hypctx->fence_req);
130
131 KASSERT((pending & ~(FENCE_REQ_I | FENCE_REQ_VMA)) == 0,
132 ("wrong fence bit mask"));
133
134 if (pending & FENCE_REQ_I)
135 fence_i();
136
137 if (pending & FENCE_REQ_VMA)
138 sfence_vma();
139
140 while (vmm_fence_dequeue(hypctx, &fence) == true)
141 vmm_fence_process_one(&fence);
142 }
143
144 void
vmm_fence_add(struct vm * vm,cpuset_t * cpus,struct vmm_fence * fence)145 vmm_fence_add(struct vm *vm, cpuset_t *cpus, struct vmm_fence *fence)
146 {
147 struct hypctx *hypctx;
148 cpuset_t running_cpus;
149 struct vcpu *vcpu;
150 uint16_t maxcpus;
151 int hostcpu;
152 bool enq;
153 int i;
154
155 CPU_ZERO(&running_cpus);
156
157 maxcpus = vm_get_maxcpus(vm);
158 for (i = 0; i < maxcpus; i++) {
159 if (!CPU_ISSET(i, cpus))
160 continue;
161 vcpu = vm_vcpu(vm, i);
162 hypctx = vcpu_get_cookie(vcpu);
163
164 enq = false;
165
166 /* No need to enqueue fences i and vma global. */
167 switch (fence->type) {
168 case VMM_RISCV_FENCE_I:
169 atomic_set_32(&hypctx->fence_req, FENCE_REQ_I);
170 break;
171 case VMM_RISCV_FENCE_VMA:
172 if ((fence->start == 0 && fence->size == 0) ||
173 fence->size == -1)
174 atomic_set_32(&hypctx->fence_req,
175 FENCE_REQ_VMA);
176 else
177 enq = true;
178 break;
179 case VMM_RISCV_FENCE_VMA_ASID:
180 enq = true;
181 break;
182 default:
183 KASSERT(0, ("%s: wrong fence type %d", __func__,
184 fence->type));
185 break;
186 }
187
188 /*
189 * Try to enqueue. In case of failure use more conservative
190 * request.
191 */
192 if (enq)
193 if (vmm_fence_enqueue(hypctx, fence) == false)
194 atomic_set_32(&hypctx->fence_req,
195 FENCE_REQ_VMA);
196
197 mb();
198
199 if (vcpu_is_running(vcpu, &hostcpu))
200 CPU_SET(hostcpu, &running_cpus);
201 }
202
203 /*
204 * Interrupt other cores. On reception of IPI they will leave guest.
205 * On entry back to the guest they will process fence request.
206 *
207 * If vcpu migrates to another cpu right here, it should process
208 * all fences on entry to the guest as well.
209 */
210 if (!CPU_EMPTY(&running_cpus))
211 smp_rendezvous_cpus(running_cpus, NULL, NULL, NULL, NULL);
212 }
213