1 #include "kvm/fdt.h"
2 #include "kvm/irq.h"
3 #include "kvm/kvm.h"
4 #include "kvm/virtio.h"
5
6 #include "arm-common/gic.h"
7
8 #include <linux/byteorder.h>
9 #include <linux/kernel.h>
10 #include <linux/kvm.h>
11 #include <linux/sizes.h>
12
13 #define IRQCHIP_GIC 0
14
15 static int gic_fd = -1;
16 static u64 gic_redists_base;
17 static u64 gic_redists_size;
18 static u64 gic_msi_base;
19 static u64 gic_msi_size = 0;
20 static bool vgic_is_init = false;
21
22 struct kvm_irqfd_line {
23 unsigned int gsi;
24 int trigger_fd;
25 int resample_fd;
26 struct list_head list;
27 };
28
29 static LIST_HEAD(irqfd_lines);
30
irqchip_parser(const struct option * opt,const char * arg,int unset)31 int irqchip_parser(const struct option *opt, const char *arg, int unset)
32 {
33 enum irqchip_type *type = opt->value;
34
35 if (!strcmp(arg, "gicv2")) {
36 *type = IRQCHIP_GICV2;
37 } else if (!strcmp(arg, "gicv2m")) {
38 *type = IRQCHIP_GICV2M;
39 } else if (!strcmp(arg, "gicv3")) {
40 *type = IRQCHIP_GICV3;
41 } else if (!strcmp(arg, "gicv3-its")) {
42 *type = IRQCHIP_GICV3_ITS;
43 } else {
44 pr_err("irqchip: unknown type \"%s\"\n", arg);
45 return -1;
46 }
47
48 return 0;
49 }
50
irq__setup_irqfd_lines(struct kvm * kvm)51 static int irq__setup_irqfd_lines(struct kvm *kvm)
52 {
53 int ret;
54 struct kvm_irqfd_line *line, *tmp;
55
56 list_for_each_entry_safe(line, tmp, &irqfd_lines, list) {
57 ret = irq__common_add_irqfd(kvm, line->gsi, line->trigger_fd,
58 line->resample_fd);
59 if (ret < 0) {
60 pr_err("Failed to register IRQFD");
61 return ret;
62 }
63
64 list_del(&line->list);
65 free(line);
66 }
67
68 return 0;
69 }
70
irq__routing_init(struct kvm * kvm)71 static int irq__routing_init(struct kvm *kvm)
72 {
73 int r;
74 int irqlines = ALIGN(irq__get_nr_allocated_lines(), 32);
75
76 /*
77 * This describes the default routing that the kernel uses without
78 * any routing explicitly set up via KVM_SET_GSI_ROUTING. So we
79 * don't need to commit these setting right now. The first actual
80 * user (MSI routing) will engage these mappings then.
81 */
82 for (next_gsi = 0; next_gsi < irqlines; next_gsi++) {
83 r = irq__allocate_routing_entry();
84 if (r)
85 return r;
86
87 irq_routing->entries[irq_routing->nr++] =
88 (struct kvm_irq_routing_entry) {
89 .gsi = next_gsi,
90 .type = KVM_IRQ_ROUTING_IRQCHIP,
91 .u.irqchip.irqchip = IRQCHIP_GIC,
92 .u.irqchip.pin = next_gsi,
93 };
94 }
95
96 return 0;
97 }
98
gic__create_its_frame(struct kvm * kvm,u64 its_frame_addr)99 static int gic__create_its_frame(struct kvm *kvm, u64 its_frame_addr)
100 {
101 struct kvm_create_device its_device = {
102 .type = KVM_DEV_TYPE_ARM_VGIC_ITS,
103 .flags = 0,
104 };
105 struct kvm_device_attr its_attr = {
106 .group = KVM_DEV_ARM_VGIC_GRP_ADDR,
107 .attr = KVM_VGIC_ITS_ADDR_TYPE,
108 .addr = (u64)(unsigned long)&its_frame_addr,
109 };
110 struct kvm_device_attr its_init_attr = {
111 .group = KVM_DEV_ARM_VGIC_GRP_CTRL,
112 .attr = KVM_DEV_ARM_VGIC_CTRL_INIT,
113 };
114 int err;
115
116 err = ioctl(kvm->vm_fd, KVM_CREATE_DEVICE, &its_device);
117 if (err) {
118 pr_err("GICv3 ITS requested, but kernel does not support it.");
119 pr_err("Try --irqchip=gicv3 instead");
120 return err;
121 }
122
123 err = ioctl(its_device.fd, KVM_HAS_DEVICE_ATTR, &its_attr);
124 if (err) {
125 close(its_device.fd);
126 its_device.fd = -1;
127 return err;
128 }
129
130 err = ioctl(its_device.fd, KVM_SET_DEVICE_ATTR, &its_attr);
131 if (err)
132 return err;
133
134 return ioctl(its_device.fd, KVM_SET_DEVICE_ATTR, &its_init_attr);
135 }
136
gic__create_msi_frame(struct kvm * kvm,enum irqchip_type type,u64 msi_frame_addr)137 static int gic__create_msi_frame(struct kvm *kvm, enum irqchip_type type,
138 u64 msi_frame_addr)
139 {
140 switch (type) {
141 case IRQCHIP_GICV2M:
142 return gic__create_gicv2m_frame(kvm, msi_frame_addr);
143 case IRQCHIP_GICV3_ITS:
144 return gic__create_its_frame(kvm, msi_frame_addr);
145 default: /* No MSI frame needed */
146 return 0;
147 }
148 }
149
gic__create_device(struct kvm * kvm,enum irqchip_type type)150 static int gic__create_device(struct kvm *kvm, enum irqchip_type type)
151 {
152 int err;
153 u64 cpu_if_addr = ARM_GIC_CPUI_BASE;
154 u64 dist_addr = ARM_GIC_DIST_BASE;
155 struct kvm_create_device gic_device = {
156 .flags = 0,
157 };
158 struct kvm_device_attr cpu_if_attr = {
159 .group = KVM_DEV_ARM_VGIC_GRP_ADDR,
160 .attr = KVM_VGIC_V2_ADDR_TYPE_CPU,
161 .addr = (u64)(unsigned long)&cpu_if_addr,
162 };
163 struct kvm_device_attr dist_attr = {
164 .group = KVM_DEV_ARM_VGIC_GRP_ADDR,
165 .addr = (u64)(unsigned long)&dist_addr,
166 };
167 struct kvm_device_attr redist_attr = {
168 .group = KVM_DEV_ARM_VGIC_GRP_ADDR,
169 .attr = KVM_VGIC_V3_ADDR_TYPE_REDIST,
170 .addr = (u64)(unsigned long)&gic_redists_base,
171 };
172
173 switch (type) {
174 case IRQCHIP_GICV2M:
175 case IRQCHIP_GICV2:
176 gic_device.type = KVM_DEV_TYPE_ARM_VGIC_V2;
177 dist_attr.attr = KVM_VGIC_V2_ADDR_TYPE_DIST;
178 break;
179 case IRQCHIP_GICV3:
180 case IRQCHIP_GICV3_ITS:
181 gic_device.type = KVM_DEV_TYPE_ARM_VGIC_V3;
182 dist_attr.attr = KVM_VGIC_V3_ADDR_TYPE_DIST;
183 break;
184 case IRQCHIP_AUTO:
185 return -ENODEV;
186 }
187
188 err = ioctl(kvm->vm_fd, KVM_CREATE_DEVICE, &gic_device);
189 if (err)
190 return err;
191
192 gic_fd = gic_device.fd;
193
194 switch (type) {
195 case IRQCHIP_GICV2M:
196 case IRQCHIP_GICV2:
197 err = ioctl(gic_fd, KVM_SET_DEVICE_ATTR, &cpu_if_attr);
198 break;
199 case IRQCHIP_GICV3_ITS:
200 case IRQCHIP_GICV3:
201 err = ioctl(gic_fd, KVM_SET_DEVICE_ATTR, &redist_attr);
202 break;
203 case IRQCHIP_AUTO:
204 return -ENODEV;
205 }
206 if (err)
207 goto out_err;
208
209 err = ioctl(gic_fd, KVM_SET_DEVICE_ATTR, &dist_attr);
210 if (err)
211 goto out_err;
212
213 err = gic__create_msi_frame(kvm, type, gic_msi_base);
214 if (err)
215 goto out_err;
216
217 return 0;
218
219 out_err:
220 close(gic_fd);
221 gic_fd = -1;
222 return err;
223 }
224
gic__create_irqchip(struct kvm * kvm)225 static int gic__create_irqchip(struct kvm *kvm)
226 {
227 int err;
228 struct kvm_arm_device_addr gic_addr[] = {
229 [0] = {
230 .id = KVM_VGIC_V2_ADDR_TYPE_DIST |
231 (KVM_ARM_DEVICE_VGIC_V2 << KVM_ARM_DEVICE_ID_SHIFT),
232 .addr = ARM_GIC_DIST_BASE,
233 },
234 [1] = {
235 .id = KVM_VGIC_V2_ADDR_TYPE_CPU |
236 (KVM_ARM_DEVICE_VGIC_V2 << KVM_ARM_DEVICE_ID_SHIFT),
237 .addr = ARM_GIC_CPUI_BASE,
238 }
239 };
240
241 err = ioctl(kvm->vm_fd, KVM_CREATE_IRQCHIP);
242 if (err)
243 return err;
244
245 err = ioctl(kvm->vm_fd, KVM_ARM_SET_DEVICE_ADDR, &gic_addr[0]);
246 if (err)
247 return err;
248
249 err = ioctl(kvm->vm_fd, KVM_ARM_SET_DEVICE_ADDR, &gic_addr[1]);
250 return err;
251 }
252
gic__create(struct kvm * kvm,enum irqchip_type type)253 int gic__create(struct kvm *kvm, enum irqchip_type type)
254 {
255 enum irqchip_type try;
256 int err;
257
258 switch (type) {
259 case IRQCHIP_AUTO:
260 for (try = IRQCHIP_GICV3_ITS; try >= IRQCHIP_GICV2; try--) {
261 err = gic__create(kvm, try);
262 if (!err)
263 break;
264 }
265 if (err)
266 return err;
267
268 kvm->cfg.arch.irqchip = try;
269 return 0;
270 case IRQCHIP_GICV2M:
271 gic_msi_size = KVM_VGIC_V2M_SIZE;
272 gic_msi_base = ARM_GIC_CPUI_BASE - gic_msi_size;
273 break;
274 case IRQCHIP_GICV2:
275 break;
276 case IRQCHIP_GICV3_ITS:
277 /* The 64K page with the doorbell is included. */
278 gic_msi_size = KVM_VGIC_V3_ITS_SIZE;
279 /* fall through */
280 case IRQCHIP_GICV3:
281 gic_redists_size = kvm->cfg.nrcpus * ARM_GIC_REDIST_SIZE;
282 gic_redists_base = ARM_GIC_DIST_BASE - gic_redists_size;
283 gic_msi_base = gic_redists_base - gic_msi_size;
284 break;
285 default:
286 return -ENODEV;
287 }
288
289 /* Try the new way first, and fallback on legacy method otherwise */
290 err = gic__create_device(kvm, type);
291 if (err && type == IRQCHIP_GICV2)
292 err = gic__create_irqchip(kvm);
293
294 return err;
295 }
296
297 /*
298 * Sets the number of used interrupts and finalizes the GIC init explicitly.
299 */
gic__init_gic(struct kvm * kvm)300 static int gic__init_gic(struct kvm *kvm)
301 {
302 int ret;
303
304 int lines = irq__get_nr_allocated_lines();
305 u32 nr_irqs = ALIGN(lines, 32) + GIC_SPI_IRQ_BASE;
306 struct kvm_device_attr nr_irqs_attr = {
307 .group = KVM_DEV_ARM_VGIC_GRP_NR_IRQS,
308 .addr = (u64)(unsigned long)&nr_irqs,
309 };
310 struct kvm_device_attr vgic_init_attr = {
311 .group = KVM_DEV_ARM_VGIC_GRP_CTRL,
312 .attr = KVM_DEV_ARM_VGIC_CTRL_INIT,
313 };
314
315 /*
316 * If we didn't use the KVM_CREATE_DEVICE method, KVM will
317 * give us some default number of interrupts. The GIC initialization
318 * will be done automatically in this case.
319 */
320 if (gic_fd < 0)
321 return 0;
322
323 if (!ioctl(gic_fd, KVM_HAS_DEVICE_ATTR, &nr_irqs_attr)) {
324 ret = ioctl(gic_fd, KVM_SET_DEVICE_ATTR, &nr_irqs_attr);
325 if (ret)
326 return ret;
327 }
328
329 irq__routing_init(kvm);
330
331 if (!ioctl(gic_fd, KVM_HAS_DEVICE_ATTR, &vgic_init_attr)) {
332 ret = ioctl(gic_fd, KVM_SET_DEVICE_ATTR, &vgic_init_attr);
333 if (ret)
334 return ret;
335 }
336
337 kvm->msix_needs_devid = kvm__supports_vm_extension(kvm,
338 KVM_CAP_MSI_DEVID);
339
340 vgic_is_init = true;
341
342 return irq__setup_irqfd_lines(kvm);
343 }
late_init(gic__init_gic)344 late_init(gic__init_gic)
345
346 void gic__generate_fdt_nodes(void *fdt, enum irqchip_type type)
347 {
348 const char *compatible, *msi_compatible = NULL;
349 u64 msi_prop[2];
350 u64 reg_prop[] = {
351 cpu_to_fdt64(ARM_GIC_DIST_BASE), cpu_to_fdt64(ARM_GIC_DIST_SIZE),
352 0, 0, /* to be filled */
353 };
354
355 switch (type) {
356 case IRQCHIP_GICV2M:
357 msi_compatible = "arm,gic-v2m-frame";
358 /* fall-through */
359 case IRQCHIP_GICV2:
360 compatible = "arm,cortex-a15-gic";
361 reg_prop[2] = cpu_to_fdt64(ARM_GIC_CPUI_BASE);
362 reg_prop[3] = cpu_to_fdt64(ARM_GIC_CPUI_SIZE);
363 break;
364 case IRQCHIP_GICV3_ITS:
365 msi_compatible = "arm,gic-v3-its";
366 /* fall-through */
367 case IRQCHIP_GICV3:
368 compatible = "arm,gic-v3";
369 reg_prop[2] = cpu_to_fdt64(gic_redists_base);
370 reg_prop[3] = cpu_to_fdt64(gic_redists_size);
371 break;
372 default:
373 return;
374 }
375
376 _FDT(fdt_begin_node(fdt, "intc"));
377 _FDT(fdt_property_string(fdt, "compatible", compatible));
378 _FDT(fdt_property_cell(fdt, "#interrupt-cells", GIC_FDT_IRQ_NUM_CELLS));
379 _FDT(fdt_property(fdt, "interrupt-controller", NULL, 0));
380 _FDT(fdt_property(fdt, "reg", reg_prop, sizeof(reg_prop)));
381 _FDT(fdt_property_cell(fdt, "phandle", PHANDLE_GIC));
382 _FDT(fdt_property_cell(fdt, "#address-cells", 2));
383 _FDT(fdt_property_cell(fdt, "#size-cells", 2));
384
385 if (msi_compatible) {
386 _FDT(fdt_property(fdt, "ranges", NULL, 0));
387
388 _FDT(fdt_begin_node(fdt, "msic"));
389 _FDT(fdt_property_string(fdt, "compatible", msi_compatible));
390 _FDT(fdt_property(fdt, "msi-controller", NULL, 0));
391 _FDT(fdt_property_cell(fdt, "phandle", PHANDLE_MSI));
392 msi_prop[0] = cpu_to_fdt64(gic_msi_base);
393 msi_prop[1] = cpu_to_fdt64(gic_msi_size);
394 _FDT(fdt_property(fdt, "reg", msi_prop, sizeof(msi_prop)));
395 _FDT(fdt_end_node(fdt));
396 }
397
398 _FDT(fdt_end_node(fdt));
399 }
400
gic__get_fdt_irq_cpumask(struct kvm * kvm)401 u32 gic__get_fdt_irq_cpumask(struct kvm *kvm)
402 {
403 /* Only for GICv2 */
404 if (kvm->cfg.arch.irqchip == IRQCHIP_GICV3 ||
405 kvm->cfg.arch.irqchip == IRQCHIP_GICV3_ITS)
406 return 0;
407
408 if (kvm->nrcpus > 8)
409 return GIC_FDT_IRQ_PPI_CPU_MASK;
410
411 return ((1U << kvm->nrcpus) - 1) << GIC_FDT_IRQ_PPI_CPU_SHIFT;
412 }
413
414 #define KVM_IRQCHIP_IRQ(x) (KVM_ARM_IRQ_TYPE_SPI << KVM_ARM_IRQ_TYPE_SHIFT) |\
415 ((x) & KVM_ARM_IRQ_NUM_MASK)
416
kvm__irq_line(struct kvm * kvm,int irq,int level)417 void kvm__irq_line(struct kvm *kvm, int irq, int level)
418 {
419 struct kvm_irq_level irq_level = {
420 .irq = KVM_IRQCHIP_IRQ(irq),
421 .level = !!level,
422 };
423
424 if (irq < GIC_SPI_IRQ_BASE || irq > GIC_MAX_IRQ)
425 pr_warning("Ignoring invalid GIC IRQ %d", irq);
426 else if (ioctl(kvm->vm_fd, KVM_IRQ_LINE, &irq_level) < 0)
427 pr_warning("Could not KVM_IRQ_LINE for irq %d", irq);
428 }
429
kvm__irq_trigger(struct kvm * kvm,int irq)430 void kvm__irq_trigger(struct kvm *kvm, int irq)
431 {
432 kvm__irq_line(kvm, irq, VIRTIO_IRQ_HIGH);
433 kvm__irq_line(kvm, irq, VIRTIO_IRQ_LOW);
434 }
435
gic__add_irqfd(struct kvm * kvm,unsigned int gsi,int trigger_fd,int resample_fd)436 int gic__add_irqfd(struct kvm *kvm, unsigned int gsi, int trigger_fd,
437 int resample_fd)
438 {
439 struct kvm_irqfd_line *line;
440
441 if (vgic_is_init)
442 return irq__common_add_irqfd(kvm, gsi, trigger_fd, resample_fd);
443
444 /* Postpone the routing setup until we have a distributor */
445 line = malloc(sizeof(*line));
446 if (!line)
447 return -ENOMEM;
448
449 *line = (struct kvm_irqfd_line) {
450 .gsi = gsi,
451 .trigger_fd = trigger_fd,
452 .resample_fd = resample_fd,
453 };
454 list_add(&line->list, &irqfd_lines);
455
456 return 0;
457 }
458
gic__del_irqfd(struct kvm * kvm,unsigned int gsi,int trigger_fd)459 void gic__del_irqfd(struct kvm *kvm, unsigned int gsi, int trigger_fd)
460 {
461 struct kvm_irqfd_line *line;
462
463 if (vgic_is_init) {
464 irq__common_del_irqfd(kvm, gsi, trigger_fd);
465 return;
466 }
467
468 list_for_each_entry(line, &irqfd_lines, list) {
469 if (line->gsi != gsi)
470 continue;
471
472 list_del(&line->list);
473 free(line);
474 break;
475 }
476 }
477