1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Copyright (C) 2021 Western Digital Corporation or its affiliates.
4 * Copyright (C) 2022 Ventana Micro Systems Inc.
5 *
6 * Authors:
7 * Anup Patel <apatel@ventanamicro.com>
8 */
9
10 #include <linux/kernel.h>
11 #include <linux/bitops.h>
12 #include <linux/irq.h>
13 #include <linux/irqchip/riscv-imsic.h>
14 #include <linux/irqdomain.h>
15 #include <linux/kvm_host.h>
16 #include <linux/percpu.h>
17 #include <linux/spinlock.h>
18 #include <asm/cpufeature.h>
19 #include <asm/kvm_nacl.h>
20
21 struct aia_hgei_control {
22 raw_spinlock_t lock;
23 unsigned long free_bitmap;
24 struct kvm_vcpu *owners[BITS_PER_LONG];
25 };
26 static DEFINE_PER_CPU(struct aia_hgei_control, aia_hgei);
27 static int hgei_parent_irq;
28
29 unsigned int kvm_riscv_aia_nr_hgei;
30 unsigned int kvm_riscv_aia_max_ids;
31 DEFINE_STATIC_KEY_FALSE(kvm_riscv_aia_available);
32
aia_hvictl_value(bool ext_irq_pending)33 static inline unsigned long aia_hvictl_value(bool ext_irq_pending)
34 {
35 unsigned long hvictl;
36
37 /*
38 * HVICTL.IID == 9 and HVICTL.IPRIO == 0 represents
39 * no interrupt in HVICTL.
40 */
41
42 hvictl = (IRQ_S_EXT << HVICTL_IID_SHIFT) & HVICTL_IID;
43 hvictl |= ext_irq_pending;
44 return hvictl;
45 }
46
47 #ifdef CONFIG_32BIT
kvm_riscv_vcpu_aia_flush_interrupts(struct kvm_vcpu * vcpu)48 void kvm_riscv_vcpu_aia_flush_interrupts(struct kvm_vcpu *vcpu)
49 {
50 struct kvm_vcpu_aia_csr *csr = &vcpu->arch.aia_context.guest_csr;
51 unsigned long mask, val;
52
53 if (!kvm_riscv_aia_available())
54 return;
55
56 if (READ_ONCE(vcpu->arch.irqs_pending_mask[1])) {
57 mask = xchg_acquire(&vcpu->arch.irqs_pending_mask[1], 0);
58 val = READ_ONCE(vcpu->arch.irqs_pending[1]) & mask;
59
60 csr->hviph &= ~mask;
61 csr->hviph |= val;
62 }
63 }
64
kvm_riscv_vcpu_aia_sync_interrupts(struct kvm_vcpu * vcpu)65 void kvm_riscv_vcpu_aia_sync_interrupts(struct kvm_vcpu *vcpu)
66 {
67 struct kvm_vcpu_aia_csr *csr = &vcpu->arch.aia_context.guest_csr;
68
69 if (kvm_riscv_aia_available())
70 csr->vsieh = ncsr_read(CSR_VSIEH);
71 }
72 #endif
73
kvm_riscv_vcpu_aia_has_interrupts(struct kvm_vcpu * vcpu,u64 mask)74 bool kvm_riscv_vcpu_aia_has_interrupts(struct kvm_vcpu *vcpu, u64 mask)
75 {
76 unsigned long seip;
77
78 if (!kvm_riscv_aia_available())
79 return false;
80
81 #ifdef CONFIG_32BIT
82 if (READ_ONCE(vcpu->arch.irqs_pending[1]) &
83 (vcpu->arch.aia_context.guest_csr.vsieh & upper_32_bits(mask)))
84 return true;
85 #endif
86
87 seip = vcpu->arch.guest_csr.vsie;
88 seip &= (unsigned long)mask;
89 seip &= BIT(IRQ_S_EXT);
90
91 if (!kvm_riscv_aia_initialized(vcpu->kvm) || !seip)
92 return false;
93
94 return kvm_riscv_vcpu_aia_imsic_has_interrupt(vcpu);
95 }
96
kvm_riscv_vcpu_aia_update_hvip(struct kvm_vcpu * vcpu)97 void kvm_riscv_vcpu_aia_update_hvip(struct kvm_vcpu *vcpu)
98 {
99 struct kvm_vcpu_csr *csr = &vcpu->arch.guest_csr;
100
101 if (!kvm_riscv_aia_available())
102 return;
103
104 #ifdef CONFIG_32BIT
105 ncsr_write(CSR_HVIPH, vcpu->arch.aia_context.guest_csr.hviph);
106 #endif
107 ncsr_write(CSR_HVICTL, aia_hvictl_value(!!(csr->hvip & BIT(IRQ_VS_EXT))));
108 }
109
kvm_riscv_vcpu_aia_load(struct kvm_vcpu * vcpu,int cpu)110 void kvm_riscv_vcpu_aia_load(struct kvm_vcpu *vcpu, int cpu)
111 {
112 struct kvm_vcpu_aia_csr *csr = &vcpu->arch.aia_context.guest_csr;
113 void *nsh;
114
115 if (!kvm_riscv_aia_available())
116 return;
117
118 if (kvm_riscv_nacl_sync_csr_available()) {
119 nsh = nacl_shmem();
120 nacl_csr_write(nsh, CSR_VSISELECT, csr->vsiselect);
121 nacl_csr_write(nsh, CSR_HVIPRIO1, csr->hviprio1);
122 nacl_csr_write(nsh, CSR_HVIPRIO2, csr->hviprio2);
123 #ifdef CONFIG_32BIT
124 nacl_csr_write(nsh, CSR_VSIEH, csr->vsieh);
125 nacl_csr_write(nsh, CSR_HVIPH, csr->hviph);
126 nacl_csr_write(nsh, CSR_HVIPRIO1H, csr->hviprio1h);
127 nacl_csr_write(nsh, CSR_HVIPRIO2H, csr->hviprio2h);
128 #endif
129 } else {
130 csr_write(CSR_VSISELECT, csr->vsiselect);
131 csr_write(CSR_HVIPRIO1, csr->hviprio1);
132 csr_write(CSR_HVIPRIO2, csr->hviprio2);
133 #ifdef CONFIG_32BIT
134 csr_write(CSR_VSIEH, csr->vsieh);
135 csr_write(CSR_HVIPH, csr->hviph);
136 csr_write(CSR_HVIPRIO1H, csr->hviprio1h);
137 csr_write(CSR_HVIPRIO2H, csr->hviprio2h);
138 #endif
139 }
140
141 if (kvm_riscv_aia_initialized(vcpu->kvm))
142 kvm_riscv_vcpu_aia_imsic_load(vcpu, cpu);
143 }
144
kvm_riscv_vcpu_aia_put(struct kvm_vcpu * vcpu)145 void kvm_riscv_vcpu_aia_put(struct kvm_vcpu *vcpu)
146 {
147 struct kvm_vcpu_aia_csr *csr = &vcpu->arch.aia_context.guest_csr;
148 void *nsh;
149
150 if (!kvm_riscv_aia_available())
151 return;
152
153 if (kvm_riscv_aia_initialized(vcpu->kvm))
154 kvm_riscv_vcpu_aia_imsic_put(vcpu);
155
156 if (kvm_riscv_nacl_available()) {
157 nsh = nacl_shmem();
158 csr->vsiselect = nacl_csr_read(nsh, CSR_VSISELECT);
159 csr->hviprio1 = nacl_csr_read(nsh, CSR_HVIPRIO1);
160 csr->hviprio2 = nacl_csr_read(nsh, CSR_HVIPRIO2);
161 #ifdef CONFIG_32BIT
162 csr->vsieh = nacl_csr_read(nsh, CSR_VSIEH);
163 csr->hviph = nacl_csr_read(nsh, CSR_HVIPH);
164 csr->hviprio1h = nacl_csr_read(nsh, CSR_HVIPRIO1H);
165 csr->hviprio2h = nacl_csr_read(nsh, CSR_HVIPRIO2H);
166 #endif
167 } else {
168 csr->vsiselect = csr_read(CSR_VSISELECT);
169 csr->hviprio1 = csr_read(CSR_HVIPRIO1);
170 csr->hviprio2 = csr_read(CSR_HVIPRIO2);
171 #ifdef CONFIG_32BIT
172 csr->vsieh = csr_read(CSR_VSIEH);
173 csr->hviph = csr_read(CSR_HVIPH);
174 csr->hviprio1h = csr_read(CSR_HVIPRIO1H);
175 csr->hviprio2h = csr_read(CSR_HVIPRIO2H);
176 #endif
177 }
178 }
179
kvm_riscv_vcpu_aia_get_csr(struct kvm_vcpu * vcpu,unsigned long reg_num,unsigned long * out_val)180 int kvm_riscv_vcpu_aia_get_csr(struct kvm_vcpu *vcpu,
181 unsigned long reg_num,
182 unsigned long *out_val)
183 {
184 struct kvm_vcpu_aia_csr *csr = &vcpu->arch.aia_context.guest_csr;
185
186 if (reg_num >= sizeof(struct kvm_riscv_aia_csr) / sizeof(unsigned long))
187 return -ENOENT;
188
189 *out_val = 0;
190 if (kvm_riscv_aia_available())
191 *out_val = ((unsigned long *)csr)[reg_num];
192
193 return 0;
194 }
195
kvm_riscv_vcpu_aia_set_csr(struct kvm_vcpu * vcpu,unsigned long reg_num,unsigned long val)196 int kvm_riscv_vcpu_aia_set_csr(struct kvm_vcpu *vcpu,
197 unsigned long reg_num,
198 unsigned long val)
199 {
200 struct kvm_vcpu_aia_csr *csr = &vcpu->arch.aia_context.guest_csr;
201
202 if (reg_num >= sizeof(struct kvm_riscv_aia_csr) / sizeof(unsigned long))
203 return -ENOENT;
204
205 if (kvm_riscv_aia_available()) {
206 ((unsigned long *)csr)[reg_num] = val;
207
208 #ifdef CONFIG_32BIT
209 if (reg_num == KVM_REG_RISCV_CSR_AIA_REG(siph))
210 WRITE_ONCE(vcpu->arch.irqs_pending_mask[1], 0);
211 #endif
212 }
213
214 return 0;
215 }
216
kvm_riscv_vcpu_aia_rmw_topei(struct kvm_vcpu * vcpu,unsigned int csr_num,unsigned long * val,unsigned long new_val,unsigned long wr_mask)217 int kvm_riscv_vcpu_aia_rmw_topei(struct kvm_vcpu *vcpu,
218 unsigned int csr_num,
219 unsigned long *val,
220 unsigned long new_val,
221 unsigned long wr_mask)
222 {
223 /* If AIA not available then redirect trap */
224 if (!kvm_riscv_aia_available())
225 return KVM_INSN_ILLEGAL_TRAP;
226
227 /* If AIA not initialized then forward to user space */
228 if (!kvm_riscv_aia_initialized(vcpu->kvm))
229 return KVM_INSN_EXIT_TO_USER_SPACE;
230
231 return kvm_riscv_vcpu_aia_imsic_rmw(vcpu, KVM_RISCV_AIA_IMSIC_TOPEI,
232 val, new_val, wr_mask);
233 }
234
235 /*
236 * External IRQ priority always read-only zero. This means default
237 * priority order is always preferred for external IRQs unless
238 * HVICTL.IID == 9 and HVICTL.IPRIO != 0
239 */
240 static int aia_irq2bitpos[] = {
241 0, 8, -1, -1, 16, 24, -1, -1, /* 0 - 7 */
242 32, -1, -1, -1, -1, 40, 48, 56, /* 8 - 15 */
243 64, 72, 80, 88, 96, 104, 112, 120, /* 16 - 23 */
244 -1, -1, -1, -1, -1, -1, -1, -1, /* 24 - 31 */
245 -1, -1, -1, -1, -1, -1, -1, -1, /* 32 - 39 */
246 -1, -1, -1, -1, -1, -1, -1, -1, /* 40 - 47 */
247 -1, -1, -1, -1, -1, -1, -1, -1, /* 48 - 55 */
248 -1, -1, -1, -1, -1, -1, -1, -1, /* 56 - 63 */
249 };
250
aia_get_iprio8(struct kvm_vcpu * vcpu,unsigned int irq)251 static u8 aia_get_iprio8(struct kvm_vcpu *vcpu, unsigned int irq)
252 {
253 unsigned long hviprio;
254 int bitpos = aia_irq2bitpos[irq];
255
256 if (bitpos < 0)
257 return 0;
258
259 switch (bitpos / BITS_PER_LONG) {
260 case 0:
261 hviprio = ncsr_read(CSR_HVIPRIO1);
262 break;
263 case 1:
264 #ifndef CONFIG_32BIT
265 hviprio = ncsr_read(CSR_HVIPRIO2);
266 break;
267 #else
268 hviprio = ncsr_read(CSR_HVIPRIO1H);
269 break;
270 case 2:
271 hviprio = ncsr_read(CSR_HVIPRIO2);
272 break;
273 case 3:
274 hviprio = ncsr_read(CSR_HVIPRIO2H);
275 break;
276 #endif
277 default:
278 return 0;
279 }
280
281 return (hviprio >> (bitpos % BITS_PER_LONG)) & TOPI_IPRIO_MASK;
282 }
283
aia_set_iprio8(struct kvm_vcpu * vcpu,unsigned int irq,u8 prio)284 static void aia_set_iprio8(struct kvm_vcpu *vcpu, unsigned int irq, u8 prio)
285 {
286 unsigned long hviprio;
287 int bitpos = aia_irq2bitpos[irq];
288
289 if (bitpos < 0)
290 return;
291
292 switch (bitpos / BITS_PER_LONG) {
293 case 0:
294 hviprio = ncsr_read(CSR_HVIPRIO1);
295 break;
296 case 1:
297 #ifndef CONFIG_32BIT
298 hviprio = ncsr_read(CSR_HVIPRIO2);
299 break;
300 #else
301 hviprio = ncsr_read(CSR_HVIPRIO1H);
302 break;
303 case 2:
304 hviprio = ncsr_read(CSR_HVIPRIO2);
305 break;
306 case 3:
307 hviprio = ncsr_read(CSR_HVIPRIO2H);
308 break;
309 #endif
310 default:
311 return;
312 }
313
314 hviprio &= ~(TOPI_IPRIO_MASK << (bitpos % BITS_PER_LONG));
315 hviprio |= (unsigned long)prio << (bitpos % BITS_PER_LONG);
316
317 switch (bitpos / BITS_PER_LONG) {
318 case 0:
319 ncsr_write(CSR_HVIPRIO1, hviprio);
320 break;
321 case 1:
322 #ifndef CONFIG_32BIT
323 ncsr_write(CSR_HVIPRIO2, hviprio);
324 break;
325 #else
326 ncsr_write(CSR_HVIPRIO1H, hviprio);
327 break;
328 case 2:
329 ncsr_write(CSR_HVIPRIO2, hviprio);
330 break;
331 case 3:
332 ncsr_write(CSR_HVIPRIO2H, hviprio);
333 break;
334 #endif
335 default:
336 return;
337 }
338 }
339
aia_rmw_iprio(struct kvm_vcpu * vcpu,unsigned int isel,unsigned long * val,unsigned long new_val,unsigned long wr_mask)340 static int aia_rmw_iprio(struct kvm_vcpu *vcpu, unsigned int isel,
341 unsigned long *val, unsigned long new_val,
342 unsigned long wr_mask)
343 {
344 int i, first_irq, nirqs;
345 unsigned long old_val;
346 u8 prio;
347
348 #ifndef CONFIG_32BIT
349 if (isel & 0x1)
350 return KVM_INSN_ILLEGAL_TRAP;
351 #endif
352
353 nirqs = 4 * (BITS_PER_LONG / 32);
354 first_irq = (isel - ISELECT_IPRIO0) * 4;
355
356 old_val = 0;
357 for (i = 0; i < nirqs; i++) {
358 prio = aia_get_iprio8(vcpu, first_irq + i);
359 old_val |= (unsigned long)prio << (TOPI_IPRIO_BITS * i);
360 }
361
362 if (val)
363 *val = old_val;
364
365 if (wr_mask) {
366 new_val = (old_val & ~wr_mask) | (new_val & wr_mask);
367 for (i = 0; i < nirqs; i++) {
368 prio = (new_val >> (TOPI_IPRIO_BITS * i)) &
369 TOPI_IPRIO_MASK;
370 aia_set_iprio8(vcpu, first_irq + i, prio);
371 }
372 }
373
374 return KVM_INSN_CONTINUE_NEXT_SEPC;
375 }
376
kvm_riscv_vcpu_aia_rmw_ireg(struct kvm_vcpu * vcpu,unsigned int csr_num,unsigned long * val,unsigned long new_val,unsigned long wr_mask)377 int kvm_riscv_vcpu_aia_rmw_ireg(struct kvm_vcpu *vcpu, unsigned int csr_num,
378 unsigned long *val, unsigned long new_val,
379 unsigned long wr_mask)
380 {
381 unsigned int isel;
382
383 /* If AIA not available then redirect trap */
384 if (!kvm_riscv_aia_available())
385 return KVM_INSN_ILLEGAL_TRAP;
386
387 /* First try to emulate in kernel space */
388 isel = ncsr_read(CSR_VSISELECT) & ISELECT_MASK;
389 if (isel >= ISELECT_IPRIO0 && isel <= ISELECT_IPRIO15)
390 return aia_rmw_iprio(vcpu, isel, val, new_val, wr_mask);
391 else if (isel >= IMSIC_FIRST && isel <= IMSIC_LAST &&
392 kvm_riscv_aia_initialized(vcpu->kvm))
393 return kvm_riscv_vcpu_aia_imsic_rmw(vcpu, isel, val, new_val,
394 wr_mask);
395
396 /* We can't handle it here so redirect to user space */
397 return KVM_INSN_EXIT_TO_USER_SPACE;
398 }
399
kvm_riscv_aia_alloc_hgei(int cpu,struct kvm_vcpu * owner,void __iomem ** hgei_va,phys_addr_t * hgei_pa)400 int kvm_riscv_aia_alloc_hgei(int cpu, struct kvm_vcpu *owner,
401 void __iomem **hgei_va, phys_addr_t *hgei_pa)
402 {
403 int ret = -ENOENT;
404 unsigned long flags;
405 const struct imsic_global_config *gc;
406 const struct imsic_local_config *lc;
407 struct aia_hgei_control *hgctrl = per_cpu_ptr(&aia_hgei, cpu);
408
409 if (!kvm_riscv_aia_available() || !hgctrl)
410 return -ENODEV;
411
412 raw_spin_lock_irqsave(&hgctrl->lock, flags);
413
414 if (hgctrl->free_bitmap) {
415 ret = __ffs(hgctrl->free_bitmap);
416 hgctrl->free_bitmap &= ~BIT(ret);
417 hgctrl->owners[ret] = owner;
418 }
419
420 raw_spin_unlock_irqrestore(&hgctrl->lock, flags);
421
422 gc = imsic_get_global_config();
423 lc = (gc) ? per_cpu_ptr(gc->local, cpu) : NULL;
424 if (lc && ret > 0) {
425 if (hgei_va)
426 *hgei_va = lc->msi_va + (ret * IMSIC_MMIO_PAGE_SZ);
427 if (hgei_pa)
428 *hgei_pa = lc->msi_pa + (ret * IMSIC_MMIO_PAGE_SZ);
429 }
430
431 return ret;
432 }
433
kvm_riscv_aia_free_hgei(int cpu,int hgei)434 void kvm_riscv_aia_free_hgei(int cpu, int hgei)
435 {
436 unsigned long flags;
437 struct aia_hgei_control *hgctrl = per_cpu_ptr(&aia_hgei, cpu);
438
439 if (!kvm_riscv_aia_available() || !hgctrl)
440 return;
441
442 raw_spin_lock_irqsave(&hgctrl->lock, flags);
443
444 if (hgei > 0 && hgei <= kvm_riscv_aia_nr_hgei) {
445 if (!(hgctrl->free_bitmap & BIT(hgei))) {
446 hgctrl->free_bitmap |= BIT(hgei);
447 hgctrl->owners[hgei] = NULL;
448 }
449 }
450
451 raw_spin_unlock_irqrestore(&hgctrl->lock, flags);
452 }
453
hgei_interrupt(int irq,void * dev_id)454 static irqreturn_t hgei_interrupt(int irq, void *dev_id)
455 {
456 int i;
457 unsigned long hgei_mask, flags;
458 struct aia_hgei_control *hgctrl = get_cpu_ptr(&aia_hgei);
459
460 hgei_mask = csr_read(CSR_HGEIP) & csr_read(CSR_HGEIE);
461 csr_clear(CSR_HGEIE, hgei_mask);
462
463 raw_spin_lock_irqsave(&hgctrl->lock, flags);
464
465 for_each_set_bit(i, &hgei_mask, BITS_PER_LONG) {
466 if (hgctrl->owners[i])
467 kvm_vcpu_kick(hgctrl->owners[i]);
468 }
469
470 raw_spin_unlock_irqrestore(&hgctrl->lock, flags);
471
472 put_cpu_ptr(&aia_hgei);
473 return IRQ_HANDLED;
474 }
475
aia_hgei_init(void)476 static int aia_hgei_init(void)
477 {
478 int cpu, rc;
479 struct irq_domain *domain;
480 struct aia_hgei_control *hgctrl;
481
482 /* Initialize per-CPU guest external interrupt line management */
483 for_each_possible_cpu(cpu) {
484 hgctrl = per_cpu_ptr(&aia_hgei, cpu);
485 raw_spin_lock_init(&hgctrl->lock);
486 if (kvm_riscv_aia_nr_hgei) {
487 hgctrl->free_bitmap =
488 BIT(kvm_riscv_aia_nr_hgei + 1) - 1;
489 hgctrl->free_bitmap &= ~BIT(0);
490 } else
491 hgctrl->free_bitmap = 0;
492 }
493
494 /* Skip SGEI interrupt setup for zero guest external interrupts */
495 if (!kvm_riscv_aia_nr_hgei)
496 goto skip_sgei_interrupt;
497
498 /* Find INTC irq domain */
499 domain = irq_find_matching_fwnode(riscv_get_intc_hwnode(),
500 DOMAIN_BUS_ANY);
501 if (!domain) {
502 kvm_err("unable to find INTC domain\n");
503 return -ENOENT;
504 }
505
506 /* Map per-CPU SGEI interrupt from INTC domain */
507 hgei_parent_irq = irq_create_mapping(domain, IRQ_S_GEXT);
508 if (!hgei_parent_irq) {
509 kvm_err("unable to map SGEI IRQ\n");
510 return -ENOMEM;
511 }
512
513 /* Request per-CPU SGEI interrupt */
514 rc = request_percpu_irq(hgei_parent_irq, hgei_interrupt,
515 "riscv-kvm", &aia_hgei);
516 if (rc) {
517 kvm_err("failed to request SGEI IRQ\n");
518 return rc;
519 }
520
521 skip_sgei_interrupt:
522 return 0;
523 }
524
aia_hgei_exit(void)525 static void aia_hgei_exit(void)
526 {
527 /* Do nothing for zero guest external interrupts */
528 if (!kvm_riscv_aia_nr_hgei)
529 return;
530
531 /* Free per-CPU SGEI interrupt */
532 free_percpu_irq(hgei_parent_irq, &aia_hgei);
533 }
534
kvm_riscv_aia_enable(void)535 void kvm_riscv_aia_enable(void)
536 {
537 if (!kvm_riscv_aia_available())
538 return;
539
540 csr_write(CSR_HVICTL, aia_hvictl_value(false));
541 csr_write(CSR_HVIPRIO1, 0x0);
542 csr_write(CSR_HVIPRIO2, 0x0);
543 #ifdef CONFIG_32BIT
544 csr_write(CSR_HVIPH, 0x0);
545 csr_write(CSR_HIDELEGH, 0x0);
546 csr_write(CSR_HVIPRIO1H, 0x0);
547 csr_write(CSR_HVIPRIO2H, 0x0);
548 #endif
549
550 /* Enable per-CPU SGEI interrupt */
551 enable_percpu_irq(hgei_parent_irq,
552 irq_get_trigger_type(hgei_parent_irq));
553 csr_set(CSR_HIE, BIT(IRQ_S_GEXT));
554 /* Enable IRQ filtering for overflow interrupt only if sscofpmf is present */
555 if (__riscv_isa_extension_available(NULL, RISCV_ISA_EXT_SSCOFPMF))
556 csr_set(CSR_HVIEN, BIT(IRQ_PMU_OVF));
557 }
558
kvm_riscv_aia_disable(void)559 void kvm_riscv_aia_disable(void)
560 {
561 int i;
562 unsigned long flags;
563 struct kvm_vcpu *vcpu;
564 struct aia_hgei_control *hgctrl;
565
566 if (!kvm_riscv_aia_available())
567 return;
568 hgctrl = get_cpu_ptr(&aia_hgei);
569
570 if (__riscv_isa_extension_available(NULL, RISCV_ISA_EXT_SSCOFPMF))
571 csr_clear(CSR_HVIEN, BIT(IRQ_PMU_OVF));
572 /* Disable per-CPU SGEI interrupt */
573 csr_clear(CSR_HIE, BIT(IRQ_S_GEXT));
574 disable_percpu_irq(hgei_parent_irq);
575
576 csr_write(CSR_HVICTL, aia_hvictl_value(false));
577
578 raw_spin_lock_irqsave(&hgctrl->lock, flags);
579
580 for (i = 0; i <= kvm_riscv_aia_nr_hgei; i++) {
581 vcpu = hgctrl->owners[i];
582 if (!vcpu)
583 continue;
584
585 /*
586 * We release hgctrl->lock before notifying IMSIC
587 * so that we don't have lock ordering issues.
588 */
589 raw_spin_unlock_irqrestore(&hgctrl->lock, flags);
590
591 /* Notify IMSIC */
592 kvm_riscv_vcpu_aia_imsic_release(vcpu);
593
594 /*
595 * Wakeup VCPU if it was blocked so that it can
596 * run on other HARTs
597 */
598 if (csr_read(CSR_HGEIE) & BIT(i)) {
599 csr_clear(CSR_HGEIE, BIT(i));
600 kvm_vcpu_kick(vcpu);
601 }
602
603 raw_spin_lock_irqsave(&hgctrl->lock, flags);
604 }
605
606 raw_spin_unlock_irqrestore(&hgctrl->lock, flags);
607
608 put_cpu_ptr(&aia_hgei);
609 }
610
kvm_riscv_aia_init(void)611 int kvm_riscv_aia_init(void)
612 {
613 int rc;
614 const struct imsic_global_config *gc;
615
616 if (!riscv_isa_extension_available(NULL, SxAIA))
617 return -ENODEV;
618 gc = imsic_get_global_config();
619
620 /* Figure-out number of bits in HGEIE */
621 csr_write(CSR_HGEIE, -1UL);
622 kvm_riscv_aia_nr_hgei = fls_long(csr_read(CSR_HGEIE));
623 csr_write(CSR_HGEIE, 0);
624 if (kvm_riscv_aia_nr_hgei)
625 kvm_riscv_aia_nr_hgei--;
626
627 /*
628 * Number of usable HGEI lines should be minimum of per-HART
629 * IMSIC guest files and number of bits in HGEIE
630 */
631 if (gc)
632 kvm_riscv_aia_nr_hgei = min((ulong)kvm_riscv_aia_nr_hgei,
633 BIT(gc->guest_index_bits) - 1);
634 else
635 kvm_riscv_aia_nr_hgei = 0;
636
637 /* Find number of guest MSI IDs */
638 kvm_riscv_aia_max_ids = IMSIC_MAX_ID;
639 if (gc && kvm_riscv_aia_nr_hgei)
640 kvm_riscv_aia_max_ids = gc->nr_guest_ids + 1;
641
642 /* Initialize guest external interrupt line management */
643 rc = aia_hgei_init();
644 if (rc)
645 return rc;
646
647 /* Register device operations */
648 rc = kvm_register_device_ops(&kvm_riscv_aia_device_ops,
649 KVM_DEV_TYPE_RISCV_AIA);
650 if (rc) {
651 aia_hgei_exit();
652 return rc;
653 }
654
655 /* Enable KVM AIA support */
656 static_branch_enable(&kvm_riscv_aia_available);
657
658 return 0;
659 }
660
kvm_riscv_aia_exit(void)661 void kvm_riscv_aia_exit(void)
662 {
663 if (!kvm_riscv_aia_available())
664 return;
665
666 /* Unregister device operations */
667 kvm_unregister_device_ops(KVM_DEV_TYPE_RISCV_AIA);
668
669 /* Cleanup the HGEI state */
670 aia_hgei_exit();
671 }
672