1 #include "kvm/kvm.h"
2 #include "kvm/kvm-cpu.h"
3 #include "kvm/rbtree-interval.h"
4 #include "kvm/mutex.h"
5
6 #include <stdio.h>
7 #include <stdlib.h>
8
9 #include <sys/ioctl.h>
10 #include <linux/kvm.h>
11 #include <linux/types.h>
12 #include <linux/rbtree.h>
13 #include <linux/err.h>
14 #include <errno.h>
15
16 #define mmio_node(n) rb_entry(n, struct mmio_mapping, node)
17
18 static DEFINE_MUTEX(mmio_lock);
19
20 struct mmio_mapping {
21 struct rb_int_node node;
22 mmio_handler_fn mmio_fn;
23 void *ptr;
24 u32 refcount;
25 bool remove;
26 };
27
28 static struct rb_root mmio_tree = RB_ROOT;
29 static struct rb_root pio_tree = RB_ROOT;
30
mmio_search(struct rb_root * root,u64 addr,u64 len)31 static struct mmio_mapping *mmio_search(struct rb_root *root, u64 addr, u64 len)
32 {
33 struct rb_int_node *node;
34
35 /* If len is zero or if there's an overflow, the MMIO op is invalid. */
36 if (addr + len <= addr)
37 return NULL;
38
39 node = rb_int_search_range(root, addr, addr + len);
40 if (node == NULL)
41 return NULL;
42
43 return mmio_node(node);
44 }
45
46 /* Find lowest match, Check for overlap */
mmio_search_single(struct rb_root * root,u64 addr)47 static struct mmio_mapping *mmio_search_single(struct rb_root *root, u64 addr)
48 {
49 struct rb_int_node *node;
50
51 node = rb_int_search_single(root, addr);
52 if (node == NULL)
53 return NULL;
54
55 return mmio_node(node);
56 }
57
mmio_insert(struct rb_root * root,struct mmio_mapping * data)58 static int mmio_insert(struct rb_root *root, struct mmio_mapping *data)
59 {
60 return rb_int_insert(root, &data->node);
61 }
62
mmio_remove(struct rb_root * root,struct mmio_mapping * data)63 static void mmio_remove(struct rb_root *root, struct mmio_mapping *data)
64 {
65 rb_int_erase(root, &data->node);
66 }
67
to_direction(u8 is_write)68 static const char *to_direction(u8 is_write)
69 {
70 if (is_write)
71 return "write";
72
73 return "read";
74 }
75
mmio_get(struct rb_root * root,u64 phys_addr,u32 len)76 static struct mmio_mapping *mmio_get(struct rb_root *root, u64 phys_addr, u32 len)
77 {
78 struct mmio_mapping *mmio;
79
80 mutex_lock(&mmio_lock);
81 mmio = mmio_search(root, phys_addr, len);
82 if (mmio)
83 mmio->refcount++;
84 mutex_unlock(&mmio_lock);
85
86 return mmio;
87 }
88
89 /* Called with mmio_lock held. */
mmio_deregister(struct kvm * kvm,struct rb_root * root,struct mmio_mapping * mmio)90 static void mmio_deregister(struct kvm *kvm, struct rb_root *root, struct mmio_mapping *mmio)
91 {
92 struct kvm_coalesced_mmio_zone zone = (struct kvm_coalesced_mmio_zone) {
93 .addr = rb_int_start(&mmio->node),
94 .size = 1,
95 };
96 ioctl(kvm->vm_fd, KVM_UNREGISTER_COALESCED_MMIO, &zone);
97
98 mmio_remove(root, mmio);
99 free(mmio);
100 }
101
mmio_put(struct kvm * kvm,struct rb_root * root,struct mmio_mapping * mmio)102 static void mmio_put(struct kvm *kvm, struct rb_root *root, struct mmio_mapping *mmio)
103 {
104 mutex_lock(&mmio_lock);
105 mmio->refcount--;
106 if (mmio->remove && mmio->refcount == 0)
107 mmio_deregister(kvm, root, mmio);
108 mutex_unlock(&mmio_lock);
109 }
110
trap_is_mmio(unsigned int flags)111 static bool trap_is_mmio(unsigned int flags)
112 {
113 return (flags & IOTRAP_BUS_MASK) == DEVICE_BUS_MMIO;
114 }
115
kvm__register_iotrap(struct kvm * kvm,u64 phys_addr,u64 phys_addr_len,mmio_handler_fn mmio_fn,void * ptr,unsigned int flags)116 int kvm__register_iotrap(struct kvm *kvm, u64 phys_addr, u64 phys_addr_len,
117 mmio_handler_fn mmio_fn, void *ptr,
118 unsigned int flags)
119 {
120 struct mmio_mapping *mmio;
121 struct kvm_coalesced_mmio_zone zone;
122 int ret;
123
124 mmio = malloc(sizeof(*mmio));
125 if (mmio == NULL)
126 return -ENOMEM;
127
128 *mmio = (struct mmio_mapping) {
129 .node = RB_INT_INIT(phys_addr, phys_addr + phys_addr_len),
130 .mmio_fn = mmio_fn,
131 .ptr = ptr,
132 /*
133 * Start from 0 because kvm__deregister_mmio() doesn't decrement
134 * the reference count.
135 */
136 .refcount = 0,
137 .remove = false,
138 };
139
140 if (trap_is_mmio(flags) && (flags & IOTRAP_COALESCE)) {
141 zone = (struct kvm_coalesced_mmio_zone) {
142 .addr = phys_addr,
143 .size = phys_addr_len,
144 };
145 ret = ioctl(kvm->vm_fd, KVM_REGISTER_COALESCED_MMIO, &zone);
146 if (ret < 0) {
147 free(mmio);
148 return -errno;
149 }
150 }
151
152 mutex_lock(&mmio_lock);
153 if (trap_is_mmio(flags))
154 ret = mmio_insert(&mmio_tree, mmio);
155 else
156 ret = mmio_insert(&pio_tree, mmio);
157 mutex_unlock(&mmio_lock);
158
159 return ret;
160 }
161
kvm__deregister_iotrap(struct kvm * kvm,u64 phys_addr,unsigned int flags)162 bool kvm__deregister_iotrap(struct kvm *kvm, u64 phys_addr, unsigned int flags)
163 {
164 struct mmio_mapping *mmio;
165 struct rb_root *tree;
166
167 if (trap_is_mmio(flags))
168 tree = &mmio_tree;
169 else
170 tree = &pio_tree;
171
172 mutex_lock(&mmio_lock);
173 mmio = mmio_search_single(tree, phys_addr);
174 if (mmio == NULL) {
175 mutex_unlock(&mmio_lock);
176 return false;
177 }
178 /*
179 * The PCI emulation code calls this function when memory access is
180 * disabled for a device, or when a BAR has a new address assigned. PCI
181 * emulation doesn't use any locks and as a result we can end up in a
182 * situation where we have called mmio_get() to do emulation on one VCPU
183 * thread (let's call it VCPU0), and several other VCPU threads have
184 * called kvm__deregister_mmio(). In this case, if we decrement refcount
185 * kvm__deregister_mmio() (either directly, or by calling mmio_put()),
186 * refcount will reach 0 and we will free the mmio node before VCPU0 has
187 * called mmio_put(). This will trigger use-after-free errors on VCPU0.
188 */
189 if (mmio->refcount == 0)
190 mmio_deregister(kvm, tree, mmio);
191 else
192 mmio->remove = true;
193 mutex_unlock(&mmio_lock);
194
195 return true;
196 }
197
kvm__emulate_mmio(struct kvm_cpu * vcpu,u64 phys_addr,u8 * data,u32 len,u8 is_write)198 bool kvm__emulate_mmio(struct kvm_cpu *vcpu, u64 phys_addr, u8 *data,
199 u32 len, u8 is_write)
200 {
201 struct mmio_mapping *mmio;
202
203 mmio = mmio_get(&mmio_tree, phys_addr, len);
204 if (!mmio) {
205 if (vcpu->kvm->cfg.mmio_debug)
206 fprintf(stderr, "MMIO warning: Ignoring MMIO %s at %016llx (length %u)\n",
207 to_direction(is_write),
208 (unsigned long long)phys_addr, len);
209 goto out;
210 }
211
212 mmio->mmio_fn(vcpu, phys_addr, data, len, is_write, mmio->ptr);
213 mmio_put(vcpu->kvm, &mmio_tree, mmio);
214
215 out:
216 return true;
217 }
218
kvm__emulate_io(struct kvm_cpu * vcpu,u16 port,void * data,int direction,int size,u32 count)219 bool kvm__emulate_io(struct kvm_cpu *vcpu, u16 port, void *data,
220 int direction, int size, u32 count)
221 {
222 struct mmio_mapping *mmio;
223 bool is_write = direction == KVM_EXIT_IO_OUT;
224
225 mmio = mmio_get(&pio_tree, port, size);
226 if (!mmio) {
227 if (vcpu->kvm->cfg.ioport_debug) {
228 fprintf(stderr, "IO error: %s port=%x, size=%d, count=%u\n",
229 to_direction(direction), port, size, count);
230
231 return false;
232 }
233 return true;
234 }
235
236 while (count--) {
237 mmio->mmio_fn(vcpu, port, data, size, is_write, mmio->ptr);
238
239 data += size;
240 }
241
242 mmio_put(vcpu->kvm, &pio_tree, mmio);
243
244 return true;
245 }
246