1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright (C) 2012 - Virtual Open Systems and Columbia University
4 * Author: Christoffer Dall <c.dall@virtualopensystems.com>
5 */
6
7 #include <linux/bug.h>
8 #include <linux/cpu_pm.h>
9 #include <linux/entry-kvm.h>
10 #include <linux/errno.h>
11 #include <linux/err.h>
12 #include <linux/kvm_host.h>
13 #include <linux/list.h>
14 #include <linux/module.h>
15 #include <linux/vmalloc.h>
16 #include <linux/fs.h>
17 #include <linux/mman.h>
18 #include <linux/sched.h>
19 #include <linux/kvm.h>
20 #include <linux/kvm_irqfd.h>
21 #include <linux/irqbypass.h>
22 #include <linux/sched/stat.h>
23 #include <linux/psci.h>
24 #include <trace/events/kvm.h>
25
26 #define CREATE_TRACE_POINTS
27 #include "trace_arm.h"
28
29 #include <linux/uaccess.h>
30 #include <asm/ptrace.h>
31 #include <asm/mman.h>
32 #include <asm/tlbflush.h>
33 #include <asm/cacheflush.h>
34 #include <asm/cpufeature.h>
35 #include <asm/virt.h>
36 #include <asm/kvm_arm.h>
37 #include <asm/kvm_asm.h>
38 #include <asm/kvm_emulate.h>
39 #include <asm/kvm_mmu.h>
40 #include <asm/kvm_nested.h>
41 #include <asm/kvm_pkvm.h>
42 #include <asm/kvm_ptrauth.h>
43 #include <asm/sections.h>
44
45 #include <kvm/arm_hypercalls.h>
46 #include <kvm/arm_pmu.h>
47 #include <kvm/arm_psci.h>
48
49 #include "sys_regs.h"
50
51 static enum kvm_mode kvm_mode = KVM_MODE_DEFAULT;
52
53 enum kvm_wfx_trap_policy {
54 KVM_WFX_NOTRAP_SINGLE_TASK, /* Default option */
55 KVM_WFX_NOTRAP,
56 KVM_WFX_TRAP,
57 };
58
59 static enum kvm_wfx_trap_policy kvm_wfi_trap_policy __read_mostly = KVM_WFX_NOTRAP_SINGLE_TASK;
60 static enum kvm_wfx_trap_policy kvm_wfe_trap_policy __read_mostly = KVM_WFX_NOTRAP_SINGLE_TASK;
61
62 DECLARE_KVM_HYP_PER_CPU(unsigned long, kvm_hyp_vector);
63
64 DEFINE_PER_CPU(unsigned long, kvm_arm_hyp_stack_base);
65 DECLARE_KVM_NVHE_PER_CPU(struct kvm_nvhe_init_params, kvm_init_params);
66
67 DECLARE_KVM_NVHE_PER_CPU(struct kvm_cpu_context, kvm_hyp_ctxt);
68
69 static bool vgic_present, kvm_arm_initialised;
70
71 static DEFINE_PER_CPU(unsigned char, kvm_hyp_initialized);
72
is_kvm_arm_initialised(void)73 bool is_kvm_arm_initialised(void)
74 {
75 return kvm_arm_initialised;
76 }
77
kvm_arch_vcpu_should_kick(struct kvm_vcpu * vcpu)78 int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)
79 {
80 return kvm_vcpu_exiting_guest_mode(vcpu) == IN_GUEST_MODE;
81 }
82
kvm_vm_ioctl_enable_cap(struct kvm * kvm,struct kvm_enable_cap * cap)83 int kvm_vm_ioctl_enable_cap(struct kvm *kvm,
84 struct kvm_enable_cap *cap)
85 {
86 int r = -EINVAL;
87
88 if (cap->flags)
89 return -EINVAL;
90
91 if (kvm_vm_is_protected(kvm) && !kvm_pvm_ext_allowed(cap->cap))
92 return -EINVAL;
93
94 switch (cap->cap) {
95 case KVM_CAP_ARM_NISV_TO_USER:
96 r = 0;
97 set_bit(KVM_ARCH_FLAG_RETURN_NISV_IO_ABORT_TO_USER,
98 &kvm->arch.flags);
99 break;
100 case KVM_CAP_ARM_MTE:
101 mutex_lock(&kvm->lock);
102 if (system_supports_mte() && !kvm->created_vcpus) {
103 r = 0;
104 set_bit(KVM_ARCH_FLAG_MTE_ENABLED, &kvm->arch.flags);
105 }
106 mutex_unlock(&kvm->lock);
107 break;
108 case KVM_CAP_ARM_SYSTEM_SUSPEND:
109 r = 0;
110 set_bit(KVM_ARCH_FLAG_SYSTEM_SUSPEND_ENABLED, &kvm->arch.flags);
111 break;
112 case KVM_CAP_ARM_EAGER_SPLIT_CHUNK_SIZE:
113 mutex_lock(&kvm->slots_lock);
114 /*
115 * To keep things simple, allow changing the chunk
116 * size only when no memory slots have been created.
117 */
118 if (kvm_are_all_memslots_empty(kvm)) {
119 u64 new_cap = cap->args[0];
120
121 if (!new_cap || kvm_is_block_size_supported(new_cap)) {
122 r = 0;
123 kvm->arch.mmu.split_page_chunk_size = new_cap;
124 }
125 }
126 mutex_unlock(&kvm->slots_lock);
127 break;
128 case KVM_CAP_ARM_WRITABLE_IMP_ID_REGS:
129 mutex_lock(&kvm->lock);
130 if (!kvm->created_vcpus) {
131 r = 0;
132 set_bit(KVM_ARCH_FLAG_WRITABLE_IMP_ID_REGS, &kvm->arch.flags);
133 }
134 mutex_unlock(&kvm->lock);
135 break;
136 default:
137 break;
138 }
139
140 return r;
141 }
142
kvm_arm_default_max_vcpus(void)143 static int kvm_arm_default_max_vcpus(void)
144 {
145 return vgic_present ? kvm_vgic_get_max_vcpus() : KVM_MAX_VCPUS;
146 }
147
148 /**
149 * kvm_arch_init_vm - initializes a VM data structure
150 * @kvm: pointer to the KVM struct
151 * @type: kvm device type
152 */
kvm_arch_init_vm(struct kvm * kvm,unsigned long type)153 int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
154 {
155 int ret;
156
157 mutex_init(&kvm->arch.config_lock);
158
159 #ifdef CONFIG_LOCKDEP
160 /* Clue in lockdep that the config_lock must be taken inside kvm->lock */
161 mutex_lock(&kvm->lock);
162 mutex_lock(&kvm->arch.config_lock);
163 mutex_unlock(&kvm->arch.config_lock);
164 mutex_unlock(&kvm->lock);
165 #endif
166
167 kvm_init_nested(kvm);
168
169 ret = kvm_share_hyp(kvm, kvm + 1);
170 if (ret)
171 return ret;
172
173 ret = pkvm_init_host_vm(kvm);
174 if (ret)
175 goto err_unshare_kvm;
176
177 if (!zalloc_cpumask_var(&kvm->arch.supported_cpus, GFP_KERNEL_ACCOUNT)) {
178 ret = -ENOMEM;
179 goto err_unshare_kvm;
180 }
181 cpumask_copy(kvm->arch.supported_cpus, cpu_possible_mask);
182
183 ret = kvm_init_stage2_mmu(kvm, &kvm->arch.mmu, type);
184 if (ret)
185 goto err_free_cpumask;
186
187 kvm_vgic_early_init(kvm);
188
189 kvm_timer_init_vm(kvm);
190
191 /* The maximum number of VCPUs is limited by the host's GIC model */
192 kvm->max_vcpus = kvm_arm_default_max_vcpus();
193
194 kvm_arm_init_hypercalls(kvm);
195
196 bitmap_zero(kvm->arch.vcpu_features, KVM_VCPU_MAX_FEATURES);
197
198 return 0;
199
200 err_free_cpumask:
201 free_cpumask_var(kvm->arch.supported_cpus);
202 err_unshare_kvm:
203 kvm_unshare_hyp(kvm, kvm + 1);
204 return ret;
205 }
206
kvm_arch_vcpu_fault(struct kvm_vcpu * vcpu,struct vm_fault * vmf)207 vm_fault_t kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf)
208 {
209 return VM_FAULT_SIGBUS;
210 }
211
kvm_arch_create_vm_debugfs(struct kvm * kvm)212 void kvm_arch_create_vm_debugfs(struct kvm *kvm)
213 {
214 kvm_sys_regs_create_debugfs(kvm);
215 kvm_s2_ptdump_create_debugfs(kvm);
216 }
217
kvm_destroy_mpidr_data(struct kvm * kvm)218 static void kvm_destroy_mpidr_data(struct kvm *kvm)
219 {
220 struct kvm_mpidr_data *data;
221
222 mutex_lock(&kvm->arch.config_lock);
223
224 data = rcu_dereference_protected(kvm->arch.mpidr_data,
225 lockdep_is_held(&kvm->arch.config_lock));
226 if (data) {
227 rcu_assign_pointer(kvm->arch.mpidr_data, NULL);
228 synchronize_rcu();
229 kfree(data);
230 }
231
232 mutex_unlock(&kvm->arch.config_lock);
233 }
234
235 /**
236 * kvm_arch_destroy_vm - destroy the VM data structure
237 * @kvm: pointer to the KVM struct
238 */
kvm_arch_destroy_vm(struct kvm * kvm)239 void kvm_arch_destroy_vm(struct kvm *kvm)
240 {
241 bitmap_free(kvm->arch.pmu_filter);
242 free_cpumask_var(kvm->arch.supported_cpus);
243
244 kvm_vgic_destroy(kvm);
245
246 if (is_protected_kvm_enabled())
247 pkvm_destroy_hyp_vm(kvm);
248
249 kvm_destroy_mpidr_data(kvm);
250
251 kfree(kvm->arch.sysreg_masks);
252 kvm_destroy_vcpus(kvm);
253
254 kvm_unshare_hyp(kvm, kvm + 1);
255
256 kvm_arm_teardown_hypercalls(kvm);
257 }
258
kvm_has_full_ptr_auth(void)259 static bool kvm_has_full_ptr_auth(void)
260 {
261 bool apa, gpa, api, gpi, apa3, gpa3;
262 u64 isar1, isar2, val;
263
264 /*
265 * Check that:
266 *
267 * - both Address and Generic auth are implemented for a given
268 * algorithm (Q5, IMPDEF or Q3)
269 * - only a single algorithm is implemented.
270 */
271 if (!system_has_full_ptr_auth())
272 return false;
273
274 isar1 = read_sanitised_ftr_reg(SYS_ID_AA64ISAR1_EL1);
275 isar2 = read_sanitised_ftr_reg(SYS_ID_AA64ISAR2_EL1);
276
277 apa = !!FIELD_GET(ID_AA64ISAR1_EL1_APA_MASK, isar1);
278 val = FIELD_GET(ID_AA64ISAR1_EL1_GPA_MASK, isar1);
279 gpa = (val == ID_AA64ISAR1_EL1_GPA_IMP);
280
281 api = !!FIELD_GET(ID_AA64ISAR1_EL1_API_MASK, isar1);
282 val = FIELD_GET(ID_AA64ISAR1_EL1_GPI_MASK, isar1);
283 gpi = (val == ID_AA64ISAR1_EL1_GPI_IMP);
284
285 apa3 = !!FIELD_GET(ID_AA64ISAR2_EL1_APA3_MASK, isar2);
286 val = FIELD_GET(ID_AA64ISAR2_EL1_GPA3_MASK, isar2);
287 gpa3 = (val == ID_AA64ISAR2_EL1_GPA3_IMP);
288
289 return (apa == gpa && api == gpi && apa3 == gpa3 &&
290 (apa + api + apa3) == 1);
291 }
292
kvm_vm_ioctl_check_extension(struct kvm * kvm,long ext)293 int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
294 {
295 int r;
296
297 if (kvm && kvm_vm_is_protected(kvm) && !kvm_pvm_ext_allowed(ext))
298 return 0;
299
300 switch (ext) {
301 case KVM_CAP_IRQCHIP:
302 r = vgic_present;
303 break;
304 case KVM_CAP_IOEVENTFD:
305 case KVM_CAP_USER_MEMORY:
306 case KVM_CAP_SYNC_MMU:
307 case KVM_CAP_DESTROY_MEMORY_REGION_WORKS:
308 case KVM_CAP_ONE_REG:
309 case KVM_CAP_ARM_PSCI:
310 case KVM_CAP_ARM_PSCI_0_2:
311 case KVM_CAP_READONLY_MEM:
312 case KVM_CAP_MP_STATE:
313 case KVM_CAP_IMMEDIATE_EXIT:
314 case KVM_CAP_VCPU_EVENTS:
315 case KVM_CAP_ARM_IRQ_LINE_LAYOUT_2:
316 case KVM_CAP_ARM_NISV_TO_USER:
317 case KVM_CAP_ARM_INJECT_EXT_DABT:
318 case KVM_CAP_SET_GUEST_DEBUG:
319 case KVM_CAP_VCPU_ATTRIBUTES:
320 case KVM_CAP_PTP_KVM:
321 case KVM_CAP_ARM_SYSTEM_SUSPEND:
322 case KVM_CAP_IRQFD_RESAMPLE:
323 case KVM_CAP_COUNTER_OFFSET:
324 case KVM_CAP_ARM_WRITABLE_IMP_ID_REGS:
325 r = 1;
326 break;
327 case KVM_CAP_SET_GUEST_DEBUG2:
328 return KVM_GUESTDBG_VALID_MASK;
329 case KVM_CAP_ARM_SET_DEVICE_ADDR:
330 r = 1;
331 break;
332 case KVM_CAP_NR_VCPUS:
333 /*
334 * ARM64 treats KVM_CAP_NR_CPUS differently from all other
335 * architectures, as it does not always bound it to
336 * KVM_CAP_MAX_VCPUS. It should not matter much because
337 * this is just an advisory value.
338 */
339 r = min_t(unsigned int, num_online_cpus(),
340 kvm_arm_default_max_vcpus());
341 break;
342 case KVM_CAP_MAX_VCPUS:
343 case KVM_CAP_MAX_VCPU_ID:
344 if (kvm)
345 r = kvm->max_vcpus;
346 else
347 r = kvm_arm_default_max_vcpus();
348 break;
349 case KVM_CAP_MSI_DEVID:
350 if (!kvm)
351 r = -EINVAL;
352 else
353 r = kvm->arch.vgic.msis_require_devid;
354 break;
355 case KVM_CAP_ARM_USER_IRQ:
356 /*
357 * 1: EL1_VTIMER, EL1_PTIMER, and PMU.
358 * (bump this number if adding more devices)
359 */
360 r = 1;
361 break;
362 case KVM_CAP_ARM_MTE:
363 r = system_supports_mte();
364 break;
365 case KVM_CAP_STEAL_TIME:
366 r = kvm_arm_pvtime_supported();
367 break;
368 case KVM_CAP_ARM_EL1_32BIT:
369 r = cpus_have_final_cap(ARM64_HAS_32BIT_EL1);
370 break;
371 case KVM_CAP_GUEST_DEBUG_HW_BPS:
372 r = get_num_brps();
373 break;
374 case KVM_CAP_GUEST_DEBUG_HW_WPS:
375 r = get_num_wrps();
376 break;
377 case KVM_CAP_ARM_PMU_V3:
378 r = kvm_supports_guest_pmuv3();
379 break;
380 case KVM_CAP_ARM_INJECT_SERROR_ESR:
381 r = cpus_have_final_cap(ARM64_HAS_RAS_EXTN);
382 break;
383 case KVM_CAP_ARM_VM_IPA_SIZE:
384 r = get_kvm_ipa_limit();
385 break;
386 case KVM_CAP_ARM_SVE:
387 r = system_supports_sve();
388 break;
389 case KVM_CAP_ARM_PTRAUTH_ADDRESS:
390 case KVM_CAP_ARM_PTRAUTH_GENERIC:
391 r = kvm_has_full_ptr_auth();
392 break;
393 case KVM_CAP_ARM_EAGER_SPLIT_CHUNK_SIZE:
394 if (kvm)
395 r = kvm->arch.mmu.split_page_chunk_size;
396 else
397 r = KVM_ARM_EAGER_SPLIT_CHUNK_SIZE_DEFAULT;
398 break;
399 case KVM_CAP_ARM_SUPPORTED_BLOCK_SIZES:
400 r = kvm_supported_block_sizes();
401 break;
402 case KVM_CAP_ARM_SUPPORTED_REG_MASK_RANGES:
403 r = BIT(0);
404 break;
405 default:
406 r = 0;
407 }
408
409 return r;
410 }
411
kvm_arch_dev_ioctl(struct file * filp,unsigned int ioctl,unsigned long arg)412 long kvm_arch_dev_ioctl(struct file *filp,
413 unsigned int ioctl, unsigned long arg)
414 {
415 return -EINVAL;
416 }
417
kvm_arch_alloc_vm(void)418 struct kvm *kvm_arch_alloc_vm(void)
419 {
420 size_t sz = sizeof(struct kvm);
421
422 if (!has_vhe())
423 return kzalloc(sz, GFP_KERNEL_ACCOUNT);
424
425 return __vmalloc(sz, GFP_KERNEL_ACCOUNT | __GFP_HIGHMEM | __GFP_ZERO);
426 }
427
kvm_arch_vcpu_precreate(struct kvm * kvm,unsigned int id)428 int kvm_arch_vcpu_precreate(struct kvm *kvm, unsigned int id)
429 {
430 if (irqchip_in_kernel(kvm) && vgic_initialized(kvm))
431 return -EBUSY;
432
433 if (id >= kvm->max_vcpus)
434 return -EINVAL;
435
436 return 0;
437 }
438
kvm_arch_vcpu_create(struct kvm_vcpu * vcpu)439 int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu)
440 {
441 int err;
442
443 spin_lock_init(&vcpu->arch.mp_state_lock);
444
445 #ifdef CONFIG_LOCKDEP
446 /* Inform lockdep that the config_lock is acquired after vcpu->mutex */
447 mutex_lock(&vcpu->mutex);
448 mutex_lock(&vcpu->kvm->arch.config_lock);
449 mutex_unlock(&vcpu->kvm->arch.config_lock);
450 mutex_unlock(&vcpu->mutex);
451 #endif
452
453 /* Force users to call KVM_ARM_VCPU_INIT */
454 vcpu_clear_flag(vcpu, VCPU_INITIALIZED);
455
456 vcpu->arch.mmu_page_cache.gfp_zero = __GFP_ZERO;
457
458 /* Set up the timer */
459 kvm_timer_vcpu_init(vcpu);
460
461 kvm_pmu_vcpu_init(vcpu);
462
463 kvm_arm_pvtime_vcpu_init(&vcpu->arch);
464
465 vcpu->arch.hw_mmu = &vcpu->kvm->arch.mmu;
466
467 /*
468 * This vCPU may have been created after mpidr_data was initialized.
469 * Throw out the pre-computed mappings if that is the case which forces
470 * KVM to fall back to iteratively searching the vCPUs.
471 */
472 kvm_destroy_mpidr_data(vcpu->kvm);
473
474 err = kvm_vgic_vcpu_init(vcpu);
475 if (err)
476 return err;
477
478 err = kvm_share_hyp(vcpu, vcpu + 1);
479 if (err)
480 kvm_vgic_vcpu_destroy(vcpu);
481
482 return err;
483 }
484
kvm_arch_vcpu_postcreate(struct kvm_vcpu * vcpu)485 void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
486 {
487 }
488
kvm_arch_vcpu_destroy(struct kvm_vcpu * vcpu)489 void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
490 {
491 if (!is_protected_kvm_enabled())
492 kvm_mmu_free_memory_cache(&vcpu->arch.mmu_page_cache);
493 else
494 free_hyp_memcache(&vcpu->arch.pkvm_memcache);
495 kvm_timer_vcpu_terminate(vcpu);
496 kvm_pmu_vcpu_destroy(vcpu);
497 kvm_vgic_vcpu_destroy(vcpu);
498 kvm_arm_vcpu_destroy(vcpu);
499 }
500
kvm_arch_vcpu_blocking(struct kvm_vcpu * vcpu)501 void kvm_arch_vcpu_blocking(struct kvm_vcpu *vcpu)
502 {
503
504 }
505
kvm_arch_vcpu_unblocking(struct kvm_vcpu * vcpu)506 void kvm_arch_vcpu_unblocking(struct kvm_vcpu *vcpu)
507 {
508
509 }
510
vcpu_set_pauth_traps(struct kvm_vcpu * vcpu)511 static void vcpu_set_pauth_traps(struct kvm_vcpu *vcpu)
512 {
513 if (vcpu_has_ptrauth(vcpu) && !is_protected_kvm_enabled()) {
514 /*
515 * Either we're running an L2 guest, and the API/APK bits come
516 * from L1's HCR_EL2, or API/APK are both set.
517 */
518 if (unlikely(vcpu_has_nv(vcpu) && !is_hyp_ctxt(vcpu))) {
519 u64 val;
520
521 val = __vcpu_sys_reg(vcpu, HCR_EL2);
522 val &= (HCR_API | HCR_APK);
523 vcpu->arch.hcr_el2 &= ~(HCR_API | HCR_APK);
524 vcpu->arch.hcr_el2 |= val;
525 } else {
526 vcpu->arch.hcr_el2 |= (HCR_API | HCR_APK);
527 }
528
529 /*
530 * Save the host keys if there is any chance for the guest
531 * to use pauth, as the entry code will reload the guest
532 * keys in that case.
533 */
534 if (vcpu->arch.hcr_el2 & (HCR_API | HCR_APK)) {
535 struct kvm_cpu_context *ctxt;
536
537 ctxt = this_cpu_ptr_hyp_sym(kvm_hyp_ctxt);
538 ptrauth_save_keys(ctxt);
539 }
540 }
541 }
542
kvm_vcpu_should_clear_twi(struct kvm_vcpu * vcpu)543 static bool kvm_vcpu_should_clear_twi(struct kvm_vcpu *vcpu)
544 {
545 if (unlikely(kvm_wfi_trap_policy != KVM_WFX_NOTRAP_SINGLE_TASK))
546 return kvm_wfi_trap_policy == KVM_WFX_NOTRAP;
547
548 return single_task_running() &&
549 (atomic_read(&vcpu->arch.vgic_cpu.vgic_v3.its_vpe.vlpi_count) ||
550 vcpu->kvm->arch.vgic.nassgireq);
551 }
552
kvm_vcpu_should_clear_twe(struct kvm_vcpu * vcpu)553 static bool kvm_vcpu_should_clear_twe(struct kvm_vcpu *vcpu)
554 {
555 if (unlikely(kvm_wfe_trap_policy != KVM_WFX_NOTRAP_SINGLE_TASK))
556 return kvm_wfe_trap_policy == KVM_WFX_NOTRAP;
557
558 return single_task_running();
559 }
560
kvm_arch_vcpu_load(struct kvm_vcpu * vcpu,int cpu)561 void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
562 {
563 struct kvm_s2_mmu *mmu;
564 int *last_ran;
565
566 if (is_protected_kvm_enabled())
567 goto nommu;
568
569 if (vcpu_has_nv(vcpu))
570 kvm_vcpu_load_hw_mmu(vcpu);
571
572 mmu = vcpu->arch.hw_mmu;
573 last_ran = this_cpu_ptr(mmu->last_vcpu_ran);
574
575 /*
576 * Ensure a VMID is allocated for the MMU before programming VTTBR_EL2,
577 * which happens eagerly in VHE.
578 *
579 * Also, the VMID allocator only preserves VMIDs that are active at the
580 * time of rollover, so KVM might need to grab a new VMID for the MMU if
581 * this is called from kvm_sched_in().
582 */
583 kvm_arm_vmid_update(&mmu->vmid);
584
585 /*
586 * We guarantee that both TLBs and I-cache are private to each
587 * vcpu. If detecting that a vcpu from the same VM has
588 * previously run on the same physical CPU, call into the
589 * hypervisor code to nuke the relevant contexts.
590 *
591 * We might get preempted before the vCPU actually runs, but
592 * over-invalidation doesn't affect correctness.
593 */
594 if (*last_ran != vcpu->vcpu_idx) {
595 kvm_call_hyp(__kvm_flush_cpu_context, mmu);
596 *last_ran = vcpu->vcpu_idx;
597 }
598
599 nommu:
600 vcpu->cpu = cpu;
601
602 /*
603 * The timer must be loaded before the vgic to correctly set up physical
604 * interrupt deactivation in nested state (e.g. timer interrupt).
605 */
606 kvm_timer_vcpu_load(vcpu);
607 kvm_vgic_load(vcpu);
608 kvm_vcpu_load_debug(vcpu);
609 if (has_vhe())
610 kvm_vcpu_load_vhe(vcpu);
611 kvm_arch_vcpu_load_fp(vcpu);
612 kvm_vcpu_pmu_restore_guest(vcpu);
613 if (kvm_arm_is_pvtime_enabled(&vcpu->arch))
614 kvm_make_request(KVM_REQ_RECORD_STEAL, vcpu);
615
616 if (kvm_vcpu_should_clear_twe(vcpu))
617 vcpu->arch.hcr_el2 &= ~HCR_TWE;
618 else
619 vcpu->arch.hcr_el2 |= HCR_TWE;
620
621 if (kvm_vcpu_should_clear_twi(vcpu))
622 vcpu->arch.hcr_el2 &= ~HCR_TWI;
623 else
624 vcpu->arch.hcr_el2 |= HCR_TWI;
625
626 vcpu_set_pauth_traps(vcpu);
627
628 if (is_protected_kvm_enabled()) {
629 kvm_call_hyp_nvhe(__pkvm_vcpu_load,
630 vcpu->kvm->arch.pkvm.handle,
631 vcpu->vcpu_idx, vcpu->arch.hcr_el2);
632 kvm_call_hyp(__vgic_v3_restore_vmcr_aprs,
633 &vcpu->arch.vgic_cpu.vgic_v3);
634 }
635
636 if (!cpumask_test_cpu(cpu, vcpu->kvm->arch.supported_cpus))
637 vcpu_set_on_unsupported_cpu(vcpu);
638 }
639
kvm_arch_vcpu_put(struct kvm_vcpu * vcpu)640 void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
641 {
642 if (is_protected_kvm_enabled()) {
643 kvm_call_hyp(__vgic_v3_save_vmcr_aprs,
644 &vcpu->arch.vgic_cpu.vgic_v3);
645 kvm_call_hyp_nvhe(__pkvm_vcpu_put);
646 }
647
648 kvm_vcpu_put_debug(vcpu);
649 kvm_arch_vcpu_put_fp(vcpu);
650 if (has_vhe())
651 kvm_vcpu_put_vhe(vcpu);
652 kvm_timer_vcpu_put(vcpu);
653 kvm_vgic_put(vcpu);
654 kvm_vcpu_pmu_restore_host(vcpu);
655 if (vcpu_has_nv(vcpu))
656 kvm_vcpu_put_hw_mmu(vcpu);
657 kvm_arm_vmid_clear_active();
658
659 vcpu_clear_on_unsupported_cpu(vcpu);
660 vcpu->cpu = -1;
661 }
662
__kvm_arm_vcpu_power_off(struct kvm_vcpu * vcpu)663 static void __kvm_arm_vcpu_power_off(struct kvm_vcpu *vcpu)
664 {
665 WRITE_ONCE(vcpu->arch.mp_state.mp_state, KVM_MP_STATE_STOPPED);
666 kvm_make_request(KVM_REQ_SLEEP, vcpu);
667 kvm_vcpu_kick(vcpu);
668 }
669
kvm_arm_vcpu_power_off(struct kvm_vcpu * vcpu)670 void kvm_arm_vcpu_power_off(struct kvm_vcpu *vcpu)
671 {
672 spin_lock(&vcpu->arch.mp_state_lock);
673 __kvm_arm_vcpu_power_off(vcpu);
674 spin_unlock(&vcpu->arch.mp_state_lock);
675 }
676
kvm_arm_vcpu_stopped(struct kvm_vcpu * vcpu)677 bool kvm_arm_vcpu_stopped(struct kvm_vcpu *vcpu)
678 {
679 return READ_ONCE(vcpu->arch.mp_state.mp_state) == KVM_MP_STATE_STOPPED;
680 }
681
kvm_arm_vcpu_suspend(struct kvm_vcpu * vcpu)682 static void kvm_arm_vcpu_suspend(struct kvm_vcpu *vcpu)
683 {
684 WRITE_ONCE(vcpu->arch.mp_state.mp_state, KVM_MP_STATE_SUSPENDED);
685 kvm_make_request(KVM_REQ_SUSPEND, vcpu);
686 kvm_vcpu_kick(vcpu);
687 }
688
kvm_arm_vcpu_suspended(struct kvm_vcpu * vcpu)689 static bool kvm_arm_vcpu_suspended(struct kvm_vcpu *vcpu)
690 {
691 return READ_ONCE(vcpu->arch.mp_state.mp_state) == KVM_MP_STATE_SUSPENDED;
692 }
693
kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu * vcpu,struct kvm_mp_state * mp_state)694 int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
695 struct kvm_mp_state *mp_state)
696 {
697 *mp_state = READ_ONCE(vcpu->arch.mp_state);
698
699 return 0;
700 }
701
kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu * vcpu,struct kvm_mp_state * mp_state)702 int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
703 struct kvm_mp_state *mp_state)
704 {
705 int ret = 0;
706
707 spin_lock(&vcpu->arch.mp_state_lock);
708
709 switch (mp_state->mp_state) {
710 case KVM_MP_STATE_RUNNABLE:
711 WRITE_ONCE(vcpu->arch.mp_state, *mp_state);
712 break;
713 case KVM_MP_STATE_STOPPED:
714 __kvm_arm_vcpu_power_off(vcpu);
715 break;
716 case KVM_MP_STATE_SUSPENDED:
717 kvm_arm_vcpu_suspend(vcpu);
718 break;
719 default:
720 ret = -EINVAL;
721 }
722
723 spin_unlock(&vcpu->arch.mp_state_lock);
724
725 return ret;
726 }
727
728 /**
729 * kvm_arch_vcpu_runnable - determine if the vcpu can be scheduled
730 * @v: The VCPU pointer
731 *
732 * If the guest CPU is not waiting for interrupts or an interrupt line is
733 * asserted, the CPU is by definition runnable.
734 */
kvm_arch_vcpu_runnable(struct kvm_vcpu * v)735 int kvm_arch_vcpu_runnable(struct kvm_vcpu *v)
736 {
737 bool irq_lines = *vcpu_hcr(v) & (HCR_VI | HCR_VF);
738 return ((irq_lines || kvm_vgic_vcpu_pending_irq(v))
739 && !kvm_arm_vcpu_stopped(v) && !v->arch.pause);
740 }
741
kvm_arch_vcpu_in_kernel(struct kvm_vcpu * vcpu)742 bool kvm_arch_vcpu_in_kernel(struct kvm_vcpu *vcpu)
743 {
744 return vcpu_mode_priv(vcpu);
745 }
746
747 #ifdef CONFIG_GUEST_PERF_EVENTS
kvm_arch_vcpu_get_ip(struct kvm_vcpu * vcpu)748 unsigned long kvm_arch_vcpu_get_ip(struct kvm_vcpu *vcpu)
749 {
750 return *vcpu_pc(vcpu);
751 }
752 #endif
753
kvm_init_mpidr_data(struct kvm * kvm)754 static void kvm_init_mpidr_data(struct kvm *kvm)
755 {
756 struct kvm_mpidr_data *data = NULL;
757 unsigned long c, mask, nr_entries;
758 u64 aff_set = 0, aff_clr = ~0UL;
759 struct kvm_vcpu *vcpu;
760
761 mutex_lock(&kvm->arch.config_lock);
762
763 if (rcu_access_pointer(kvm->arch.mpidr_data) ||
764 atomic_read(&kvm->online_vcpus) == 1)
765 goto out;
766
767 kvm_for_each_vcpu(c, vcpu, kvm) {
768 u64 aff = kvm_vcpu_get_mpidr_aff(vcpu);
769 aff_set |= aff;
770 aff_clr &= aff;
771 }
772
773 /*
774 * A significant bit can be either 0 or 1, and will only appear in
775 * aff_set. Use aff_clr to weed out the useless stuff.
776 */
777 mask = aff_set ^ aff_clr;
778 nr_entries = BIT_ULL(hweight_long(mask));
779
780 /*
781 * Don't let userspace fool us. If we need more than a single page
782 * to describe the compressed MPIDR array, just fall back to the
783 * iterative method. Single vcpu VMs do not need this either.
784 */
785 if (struct_size(data, cmpidr_to_idx, nr_entries) <= PAGE_SIZE)
786 data = kzalloc(struct_size(data, cmpidr_to_idx, nr_entries),
787 GFP_KERNEL_ACCOUNT);
788
789 if (!data)
790 goto out;
791
792 data->mpidr_mask = mask;
793
794 kvm_for_each_vcpu(c, vcpu, kvm) {
795 u64 aff = kvm_vcpu_get_mpidr_aff(vcpu);
796 u16 index = kvm_mpidr_index(data, aff);
797
798 data->cmpidr_to_idx[index] = c;
799 }
800
801 rcu_assign_pointer(kvm->arch.mpidr_data, data);
802 out:
803 mutex_unlock(&kvm->arch.config_lock);
804 }
805
806 /*
807 * Handle both the initialisation that is being done when the vcpu is
808 * run for the first time, as well as the updates that must be
809 * performed each time we get a new thread dealing with this vcpu.
810 */
kvm_arch_vcpu_run_pid_change(struct kvm_vcpu * vcpu)811 int kvm_arch_vcpu_run_pid_change(struct kvm_vcpu *vcpu)
812 {
813 struct kvm *kvm = vcpu->kvm;
814 int ret;
815
816 if (!kvm_vcpu_initialized(vcpu))
817 return -ENOEXEC;
818
819 if (!kvm_arm_vcpu_is_finalized(vcpu))
820 return -EPERM;
821
822 ret = kvm_arch_vcpu_run_map_fp(vcpu);
823 if (ret)
824 return ret;
825
826 if (likely(vcpu_has_run_once(vcpu)))
827 return 0;
828
829 kvm_init_mpidr_data(kvm);
830
831 if (likely(irqchip_in_kernel(kvm))) {
832 /*
833 * Map the VGIC hardware resources before running a vcpu the
834 * first time on this VM.
835 */
836 ret = kvm_vgic_map_resources(kvm);
837 if (ret)
838 return ret;
839 }
840
841 ret = kvm_finalize_sys_regs(vcpu);
842 if (ret)
843 return ret;
844
845 if (vcpu_has_nv(vcpu)) {
846 ret = kvm_vgic_vcpu_nv_init(vcpu);
847 if (ret)
848 return ret;
849 }
850
851 /*
852 * This needs to happen after any restriction has been applied
853 * to the feature set.
854 */
855 kvm_calculate_traps(vcpu);
856
857 ret = kvm_timer_enable(vcpu);
858 if (ret)
859 return ret;
860
861 if (kvm_vcpu_has_pmu(vcpu)) {
862 ret = kvm_arm_pmu_v3_enable(vcpu);
863 if (ret)
864 return ret;
865 }
866
867 if (is_protected_kvm_enabled()) {
868 ret = pkvm_create_hyp_vm(kvm);
869 if (ret)
870 return ret;
871
872 ret = pkvm_create_hyp_vcpu(vcpu);
873 if (ret)
874 return ret;
875 }
876
877 mutex_lock(&kvm->arch.config_lock);
878 set_bit(KVM_ARCH_FLAG_HAS_RAN_ONCE, &kvm->arch.flags);
879 mutex_unlock(&kvm->arch.config_lock);
880
881 return ret;
882 }
883
kvm_arch_intc_initialized(struct kvm * kvm)884 bool kvm_arch_intc_initialized(struct kvm *kvm)
885 {
886 return vgic_initialized(kvm);
887 }
888
kvm_arm_halt_guest(struct kvm * kvm)889 void kvm_arm_halt_guest(struct kvm *kvm)
890 {
891 unsigned long i;
892 struct kvm_vcpu *vcpu;
893
894 kvm_for_each_vcpu(i, vcpu, kvm)
895 vcpu->arch.pause = true;
896 kvm_make_all_cpus_request(kvm, KVM_REQ_SLEEP);
897 }
898
kvm_arm_resume_guest(struct kvm * kvm)899 void kvm_arm_resume_guest(struct kvm *kvm)
900 {
901 unsigned long i;
902 struct kvm_vcpu *vcpu;
903
904 kvm_for_each_vcpu(i, vcpu, kvm) {
905 vcpu->arch.pause = false;
906 __kvm_vcpu_wake_up(vcpu);
907 }
908 }
909
kvm_vcpu_sleep(struct kvm_vcpu * vcpu)910 static void kvm_vcpu_sleep(struct kvm_vcpu *vcpu)
911 {
912 struct rcuwait *wait = kvm_arch_vcpu_get_wait(vcpu);
913
914 rcuwait_wait_event(wait,
915 (!kvm_arm_vcpu_stopped(vcpu)) && (!vcpu->arch.pause),
916 TASK_INTERRUPTIBLE);
917
918 if (kvm_arm_vcpu_stopped(vcpu) || vcpu->arch.pause) {
919 /* Awaken to handle a signal, request we sleep again later. */
920 kvm_make_request(KVM_REQ_SLEEP, vcpu);
921 }
922
923 /*
924 * Make sure we will observe a potential reset request if we've
925 * observed a change to the power state. Pairs with the smp_wmb() in
926 * kvm_psci_vcpu_on().
927 */
928 smp_rmb();
929 }
930
931 /**
932 * kvm_vcpu_wfi - emulate Wait-For-Interrupt behavior
933 * @vcpu: The VCPU pointer
934 *
935 * Suspend execution of a vCPU until a valid wake event is detected, i.e. until
936 * the vCPU is runnable. The vCPU may or may not be scheduled out, depending
937 * on when a wake event arrives, e.g. there may already be a pending wake event.
938 */
kvm_vcpu_wfi(struct kvm_vcpu * vcpu)939 void kvm_vcpu_wfi(struct kvm_vcpu *vcpu)
940 {
941 /*
942 * Sync back the state of the GIC CPU interface so that we have
943 * the latest PMR and group enables. This ensures that
944 * kvm_arch_vcpu_runnable has up-to-date data to decide whether
945 * we have pending interrupts, e.g. when determining if the
946 * vCPU should block.
947 *
948 * For the same reason, we want to tell GICv4 that we need
949 * doorbells to be signalled, should an interrupt become pending.
950 */
951 preempt_disable();
952 vcpu_set_flag(vcpu, IN_WFI);
953 kvm_vgic_put(vcpu);
954 preempt_enable();
955
956 kvm_vcpu_halt(vcpu);
957 vcpu_clear_flag(vcpu, IN_WFIT);
958
959 preempt_disable();
960 vcpu_clear_flag(vcpu, IN_WFI);
961 kvm_vgic_load(vcpu);
962 preempt_enable();
963 }
964
kvm_vcpu_suspend(struct kvm_vcpu * vcpu)965 static int kvm_vcpu_suspend(struct kvm_vcpu *vcpu)
966 {
967 if (!kvm_arm_vcpu_suspended(vcpu))
968 return 1;
969
970 kvm_vcpu_wfi(vcpu);
971
972 /*
973 * The suspend state is sticky; we do not leave it until userspace
974 * explicitly marks the vCPU as runnable. Request that we suspend again
975 * later.
976 */
977 kvm_make_request(KVM_REQ_SUSPEND, vcpu);
978
979 /*
980 * Check to make sure the vCPU is actually runnable. If so, exit to
981 * userspace informing it of the wakeup condition.
982 */
983 if (kvm_arch_vcpu_runnable(vcpu)) {
984 memset(&vcpu->run->system_event, 0, sizeof(vcpu->run->system_event));
985 vcpu->run->system_event.type = KVM_SYSTEM_EVENT_WAKEUP;
986 vcpu->run->exit_reason = KVM_EXIT_SYSTEM_EVENT;
987 return 0;
988 }
989
990 /*
991 * Otherwise, we were unblocked to process a different event, such as a
992 * pending signal. Return 1 and allow kvm_arch_vcpu_ioctl_run() to
993 * process the event.
994 */
995 return 1;
996 }
997
998 /**
999 * check_vcpu_requests - check and handle pending vCPU requests
1000 * @vcpu: the VCPU pointer
1001 *
1002 * Return: 1 if we should enter the guest
1003 * 0 if we should exit to userspace
1004 * < 0 if we should exit to userspace, where the return value indicates
1005 * an error
1006 */
check_vcpu_requests(struct kvm_vcpu * vcpu)1007 static int check_vcpu_requests(struct kvm_vcpu *vcpu)
1008 {
1009 if (kvm_request_pending(vcpu)) {
1010 if (kvm_check_request(KVM_REQ_VM_DEAD, vcpu))
1011 return -EIO;
1012
1013 if (kvm_check_request(KVM_REQ_SLEEP, vcpu))
1014 kvm_vcpu_sleep(vcpu);
1015
1016 if (kvm_check_request(KVM_REQ_VCPU_RESET, vcpu))
1017 kvm_reset_vcpu(vcpu);
1018
1019 /*
1020 * Clear IRQ_PENDING requests that were made to guarantee
1021 * that a VCPU sees new virtual interrupts.
1022 */
1023 kvm_check_request(KVM_REQ_IRQ_PENDING, vcpu);
1024
1025 if (kvm_check_request(KVM_REQ_RECORD_STEAL, vcpu))
1026 kvm_update_stolen_time(vcpu);
1027
1028 if (kvm_check_request(KVM_REQ_RELOAD_GICv4, vcpu)) {
1029 /* The distributor enable bits were changed */
1030 preempt_disable();
1031 vgic_v4_put(vcpu);
1032 vgic_v4_load(vcpu);
1033 preempt_enable();
1034 }
1035
1036 if (kvm_check_request(KVM_REQ_RELOAD_PMU, vcpu))
1037 kvm_vcpu_reload_pmu(vcpu);
1038
1039 if (kvm_check_request(KVM_REQ_RESYNC_PMU_EL0, vcpu))
1040 kvm_vcpu_pmu_restore_guest(vcpu);
1041
1042 if (kvm_check_request(KVM_REQ_SUSPEND, vcpu))
1043 return kvm_vcpu_suspend(vcpu);
1044
1045 if (kvm_dirty_ring_check_request(vcpu))
1046 return 0;
1047
1048 check_nested_vcpu_requests(vcpu);
1049 }
1050
1051 return 1;
1052 }
1053
vcpu_mode_is_bad_32bit(struct kvm_vcpu * vcpu)1054 static bool vcpu_mode_is_bad_32bit(struct kvm_vcpu *vcpu)
1055 {
1056 if (likely(!vcpu_mode_is_32bit(vcpu)))
1057 return false;
1058
1059 if (vcpu_has_nv(vcpu))
1060 return true;
1061
1062 return !kvm_supports_32bit_el0();
1063 }
1064
1065 /**
1066 * kvm_vcpu_exit_request - returns true if the VCPU should *not* enter the guest
1067 * @vcpu: The VCPU pointer
1068 * @ret: Pointer to write optional return code
1069 *
1070 * Returns: true if the VCPU needs to return to a preemptible + interruptible
1071 * and skip guest entry.
1072 *
1073 * This function disambiguates between two different types of exits: exits to a
1074 * preemptible + interruptible kernel context and exits to userspace. For an
1075 * exit to userspace, this function will write the return code to ret and return
1076 * true. For an exit to preemptible + interruptible kernel context (i.e. check
1077 * for pending work and re-enter), return true without writing to ret.
1078 */
kvm_vcpu_exit_request(struct kvm_vcpu * vcpu,int * ret)1079 static bool kvm_vcpu_exit_request(struct kvm_vcpu *vcpu, int *ret)
1080 {
1081 struct kvm_run *run = vcpu->run;
1082
1083 /*
1084 * If we're using a userspace irqchip, then check if we need
1085 * to tell a userspace irqchip about timer or PMU level
1086 * changes and if so, exit to userspace (the actual level
1087 * state gets updated in kvm_timer_update_run and
1088 * kvm_pmu_update_run below).
1089 */
1090 if (unlikely(!irqchip_in_kernel(vcpu->kvm))) {
1091 if (kvm_timer_should_notify_user(vcpu) ||
1092 kvm_pmu_should_notify_user(vcpu)) {
1093 *ret = -EINTR;
1094 run->exit_reason = KVM_EXIT_INTR;
1095 return true;
1096 }
1097 }
1098
1099 if (unlikely(vcpu_on_unsupported_cpu(vcpu))) {
1100 run->exit_reason = KVM_EXIT_FAIL_ENTRY;
1101 run->fail_entry.hardware_entry_failure_reason = KVM_EXIT_FAIL_ENTRY_CPU_UNSUPPORTED;
1102 run->fail_entry.cpu = smp_processor_id();
1103 *ret = 0;
1104 return true;
1105 }
1106
1107 return kvm_request_pending(vcpu) ||
1108 xfer_to_guest_mode_work_pending();
1109 }
1110
1111 /*
1112 * Actually run the vCPU, entering an RCU extended quiescent state (EQS) while
1113 * the vCPU is running.
1114 *
1115 * This must be noinstr as instrumentation may make use of RCU, and this is not
1116 * safe during the EQS.
1117 */
kvm_arm_vcpu_enter_exit(struct kvm_vcpu * vcpu)1118 static int noinstr kvm_arm_vcpu_enter_exit(struct kvm_vcpu *vcpu)
1119 {
1120 int ret;
1121
1122 guest_state_enter_irqoff();
1123 ret = kvm_call_hyp_ret(__kvm_vcpu_run, vcpu);
1124 guest_state_exit_irqoff();
1125
1126 return ret;
1127 }
1128
1129 /**
1130 * kvm_arch_vcpu_ioctl_run - the main VCPU run function to execute guest code
1131 * @vcpu: The VCPU pointer
1132 *
1133 * This function is called through the VCPU_RUN ioctl called from user space. It
1134 * will execute VM code in a loop until the time slice for the process is used
1135 * or some emulation is needed from user space in which case the function will
1136 * return with return value 0 and with the kvm_run structure filled in with the
1137 * required data for the requested emulation.
1138 */
kvm_arch_vcpu_ioctl_run(struct kvm_vcpu * vcpu)1139 int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu)
1140 {
1141 struct kvm_run *run = vcpu->run;
1142 int ret;
1143
1144 if (run->exit_reason == KVM_EXIT_MMIO) {
1145 ret = kvm_handle_mmio_return(vcpu);
1146 if (ret <= 0)
1147 return ret;
1148 }
1149
1150 vcpu_load(vcpu);
1151
1152 if (!vcpu->wants_to_run) {
1153 ret = -EINTR;
1154 goto out;
1155 }
1156
1157 kvm_sigset_activate(vcpu);
1158
1159 ret = 1;
1160 run->exit_reason = KVM_EXIT_UNKNOWN;
1161 run->flags = 0;
1162 while (ret > 0) {
1163 /*
1164 * Check conditions before entering the guest
1165 */
1166 ret = xfer_to_guest_mode_handle_work(vcpu);
1167 if (!ret)
1168 ret = 1;
1169
1170 if (ret > 0)
1171 ret = check_vcpu_requests(vcpu);
1172
1173 /*
1174 * Preparing the interrupts to be injected also
1175 * involves poking the GIC, which must be done in a
1176 * non-preemptible context.
1177 */
1178 preempt_disable();
1179
1180 if (kvm_vcpu_has_pmu(vcpu))
1181 kvm_pmu_flush_hwstate(vcpu);
1182
1183 local_irq_disable();
1184
1185 kvm_vgic_flush_hwstate(vcpu);
1186
1187 kvm_pmu_update_vcpu_events(vcpu);
1188
1189 /*
1190 * Ensure we set mode to IN_GUEST_MODE after we disable
1191 * interrupts and before the final VCPU requests check.
1192 * See the comment in kvm_vcpu_exiting_guest_mode() and
1193 * Documentation/virt/kvm/vcpu-requests.rst
1194 */
1195 smp_store_mb(vcpu->mode, IN_GUEST_MODE);
1196
1197 if (ret <= 0 || kvm_vcpu_exit_request(vcpu, &ret)) {
1198 vcpu->mode = OUTSIDE_GUEST_MODE;
1199 isb(); /* Ensure work in x_flush_hwstate is committed */
1200 if (kvm_vcpu_has_pmu(vcpu))
1201 kvm_pmu_sync_hwstate(vcpu);
1202 if (unlikely(!irqchip_in_kernel(vcpu->kvm)))
1203 kvm_timer_sync_user(vcpu);
1204 kvm_vgic_sync_hwstate(vcpu);
1205 local_irq_enable();
1206 preempt_enable();
1207 continue;
1208 }
1209
1210 kvm_arch_vcpu_ctxflush_fp(vcpu);
1211
1212 /**************************************************************
1213 * Enter the guest
1214 */
1215 trace_kvm_entry(*vcpu_pc(vcpu));
1216 guest_timing_enter_irqoff();
1217
1218 ret = kvm_arm_vcpu_enter_exit(vcpu);
1219
1220 vcpu->mode = OUTSIDE_GUEST_MODE;
1221 vcpu->stat.exits++;
1222 /*
1223 * Back from guest
1224 *************************************************************/
1225
1226 /*
1227 * We must sync the PMU state before the vgic state so
1228 * that the vgic can properly sample the updated state of the
1229 * interrupt line.
1230 */
1231 if (kvm_vcpu_has_pmu(vcpu))
1232 kvm_pmu_sync_hwstate(vcpu);
1233
1234 /*
1235 * Sync the vgic state before syncing the timer state because
1236 * the timer code needs to know if the virtual timer
1237 * interrupts are active.
1238 */
1239 kvm_vgic_sync_hwstate(vcpu);
1240
1241 /*
1242 * Sync the timer hardware state before enabling interrupts as
1243 * we don't want vtimer interrupts to race with syncing the
1244 * timer virtual interrupt state.
1245 */
1246 if (unlikely(!irqchip_in_kernel(vcpu->kvm)))
1247 kvm_timer_sync_user(vcpu);
1248
1249 if (is_hyp_ctxt(vcpu))
1250 kvm_timer_sync_nested(vcpu);
1251
1252 kvm_arch_vcpu_ctxsync_fp(vcpu);
1253
1254 /*
1255 * We must ensure that any pending interrupts are taken before
1256 * we exit guest timing so that timer ticks are accounted as
1257 * guest time. Transiently unmask interrupts so that any
1258 * pending interrupts are taken.
1259 *
1260 * Per ARM DDI 0487G.b section D1.13.4, an ISB (or other
1261 * context synchronization event) is necessary to ensure that
1262 * pending interrupts are taken.
1263 */
1264 if (ARM_EXCEPTION_CODE(ret) == ARM_EXCEPTION_IRQ) {
1265 local_irq_enable();
1266 isb();
1267 local_irq_disable();
1268 }
1269
1270 guest_timing_exit_irqoff();
1271
1272 local_irq_enable();
1273
1274 trace_kvm_exit(ret, kvm_vcpu_trap_get_class(vcpu), *vcpu_pc(vcpu));
1275
1276 /* Exit types that need handling before we can be preempted */
1277 handle_exit_early(vcpu, ret);
1278
1279 preempt_enable();
1280
1281 /*
1282 * The ARMv8 architecture doesn't give the hypervisor
1283 * a mechanism to prevent a guest from dropping to AArch32 EL0
1284 * if implemented by the CPU. If we spot the guest in such
1285 * state and that we decided it wasn't supposed to do so (like
1286 * with the asymmetric AArch32 case), return to userspace with
1287 * a fatal error.
1288 */
1289 if (vcpu_mode_is_bad_32bit(vcpu)) {
1290 /*
1291 * As we have caught the guest red-handed, decide that
1292 * it isn't fit for purpose anymore by making the vcpu
1293 * invalid. The VMM can try and fix it by issuing a
1294 * KVM_ARM_VCPU_INIT if it really wants to.
1295 */
1296 vcpu_clear_flag(vcpu, VCPU_INITIALIZED);
1297 ret = ARM_EXCEPTION_IL;
1298 }
1299
1300 ret = handle_exit(vcpu, ret);
1301 }
1302
1303 /* Tell userspace about in-kernel device output levels */
1304 if (unlikely(!irqchip_in_kernel(vcpu->kvm))) {
1305 kvm_timer_update_run(vcpu);
1306 kvm_pmu_update_run(vcpu);
1307 }
1308
1309 kvm_sigset_deactivate(vcpu);
1310
1311 out:
1312 /*
1313 * In the unlikely event that we are returning to userspace
1314 * with pending exceptions or PC adjustment, commit these
1315 * adjustments in order to give userspace a consistent view of
1316 * the vcpu state. Note that this relies on __kvm_adjust_pc()
1317 * being preempt-safe on VHE.
1318 */
1319 if (unlikely(vcpu_get_flag(vcpu, PENDING_EXCEPTION) ||
1320 vcpu_get_flag(vcpu, INCREMENT_PC)))
1321 kvm_call_hyp(__kvm_adjust_pc, vcpu);
1322
1323 vcpu_put(vcpu);
1324 return ret;
1325 }
1326
vcpu_interrupt_line(struct kvm_vcpu * vcpu,int number,bool level)1327 static int vcpu_interrupt_line(struct kvm_vcpu *vcpu, int number, bool level)
1328 {
1329 int bit_index;
1330 bool set;
1331 unsigned long *hcr;
1332
1333 if (number == KVM_ARM_IRQ_CPU_IRQ)
1334 bit_index = __ffs(HCR_VI);
1335 else /* KVM_ARM_IRQ_CPU_FIQ */
1336 bit_index = __ffs(HCR_VF);
1337
1338 hcr = vcpu_hcr(vcpu);
1339 if (level)
1340 set = test_and_set_bit(bit_index, hcr);
1341 else
1342 set = test_and_clear_bit(bit_index, hcr);
1343
1344 /*
1345 * If we didn't change anything, no need to wake up or kick other CPUs
1346 */
1347 if (set == level)
1348 return 0;
1349
1350 /*
1351 * The vcpu irq_lines field was updated, wake up sleeping VCPUs and
1352 * trigger a world-switch round on the running physical CPU to set the
1353 * virtual IRQ/FIQ fields in the HCR appropriately.
1354 */
1355 kvm_make_request(KVM_REQ_IRQ_PENDING, vcpu);
1356 kvm_vcpu_kick(vcpu);
1357
1358 return 0;
1359 }
1360
kvm_vm_ioctl_irq_line(struct kvm * kvm,struct kvm_irq_level * irq_level,bool line_status)1361 int kvm_vm_ioctl_irq_line(struct kvm *kvm, struct kvm_irq_level *irq_level,
1362 bool line_status)
1363 {
1364 u32 irq = irq_level->irq;
1365 unsigned int irq_type, vcpu_id, irq_num;
1366 struct kvm_vcpu *vcpu = NULL;
1367 bool level = irq_level->level;
1368
1369 irq_type = (irq >> KVM_ARM_IRQ_TYPE_SHIFT) & KVM_ARM_IRQ_TYPE_MASK;
1370 vcpu_id = (irq >> KVM_ARM_IRQ_VCPU_SHIFT) & KVM_ARM_IRQ_VCPU_MASK;
1371 vcpu_id += ((irq >> KVM_ARM_IRQ_VCPU2_SHIFT) & KVM_ARM_IRQ_VCPU2_MASK) * (KVM_ARM_IRQ_VCPU_MASK + 1);
1372 irq_num = (irq >> KVM_ARM_IRQ_NUM_SHIFT) & KVM_ARM_IRQ_NUM_MASK;
1373
1374 trace_kvm_irq_line(irq_type, vcpu_id, irq_num, irq_level->level);
1375
1376 switch (irq_type) {
1377 case KVM_ARM_IRQ_TYPE_CPU:
1378 if (irqchip_in_kernel(kvm))
1379 return -ENXIO;
1380
1381 vcpu = kvm_get_vcpu_by_id(kvm, vcpu_id);
1382 if (!vcpu)
1383 return -EINVAL;
1384
1385 if (irq_num > KVM_ARM_IRQ_CPU_FIQ)
1386 return -EINVAL;
1387
1388 return vcpu_interrupt_line(vcpu, irq_num, level);
1389 case KVM_ARM_IRQ_TYPE_PPI:
1390 if (!irqchip_in_kernel(kvm))
1391 return -ENXIO;
1392
1393 vcpu = kvm_get_vcpu_by_id(kvm, vcpu_id);
1394 if (!vcpu)
1395 return -EINVAL;
1396
1397 if (irq_num < VGIC_NR_SGIS || irq_num >= VGIC_NR_PRIVATE_IRQS)
1398 return -EINVAL;
1399
1400 return kvm_vgic_inject_irq(kvm, vcpu, irq_num, level, NULL);
1401 case KVM_ARM_IRQ_TYPE_SPI:
1402 if (!irqchip_in_kernel(kvm))
1403 return -ENXIO;
1404
1405 if (irq_num < VGIC_NR_PRIVATE_IRQS)
1406 return -EINVAL;
1407
1408 return kvm_vgic_inject_irq(kvm, NULL, irq_num, level, NULL);
1409 }
1410
1411 return -EINVAL;
1412 }
1413
system_supported_vcpu_features(void)1414 static unsigned long system_supported_vcpu_features(void)
1415 {
1416 unsigned long features = KVM_VCPU_VALID_FEATURES;
1417
1418 if (!cpus_have_final_cap(ARM64_HAS_32BIT_EL1))
1419 clear_bit(KVM_ARM_VCPU_EL1_32BIT, &features);
1420
1421 if (!kvm_supports_guest_pmuv3())
1422 clear_bit(KVM_ARM_VCPU_PMU_V3, &features);
1423
1424 if (!system_supports_sve())
1425 clear_bit(KVM_ARM_VCPU_SVE, &features);
1426
1427 if (!kvm_has_full_ptr_auth()) {
1428 clear_bit(KVM_ARM_VCPU_PTRAUTH_ADDRESS, &features);
1429 clear_bit(KVM_ARM_VCPU_PTRAUTH_GENERIC, &features);
1430 }
1431
1432 if (!cpus_have_final_cap(ARM64_HAS_NESTED_VIRT))
1433 clear_bit(KVM_ARM_VCPU_HAS_EL2, &features);
1434
1435 return features;
1436 }
1437
kvm_vcpu_init_check_features(struct kvm_vcpu * vcpu,const struct kvm_vcpu_init * init)1438 static int kvm_vcpu_init_check_features(struct kvm_vcpu *vcpu,
1439 const struct kvm_vcpu_init *init)
1440 {
1441 unsigned long features = init->features[0];
1442 int i;
1443
1444 if (features & ~KVM_VCPU_VALID_FEATURES)
1445 return -ENOENT;
1446
1447 for (i = 1; i < ARRAY_SIZE(init->features); i++) {
1448 if (init->features[i])
1449 return -ENOENT;
1450 }
1451
1452 if (features & ~system_supported_vcpu_features())
1453 return -EINVAL;
1454
1455 /*
1456 * For now make sure that both address/generic pointer authentication
1457 * features are requested by the userspace together.
1458 */
1459 if (test_bit(KVM_ARM_VCPU_PTRAUTH_ADDRESS, &features) !=
1460 test_bit(KVM_ARM_VCPU_PTRAUTH_GENERIC, &features))
1461 return -EINVAL;
1462
1463 if (!test_bit(KVM_ARM_VCPU_EL1_32BIT, &features))
1464 return 0;
1465
1466 /* MTE is incompatible with AArch32 */
1467 if (kvm_has_mte(vcpu->kvm))
1468 return -EINVAL;
1469
1470 /* NV is incompatible with AArch32 */
1471 if (test_bit(KVM_ARM_VCPU_HAS_EL2, &features))
1472 return -EINVAL;
1473
1474 return 0;
1475 }
1476
kvm_vcpu_init_changed(struct kvm_vcpu * vcpu,const struct kvm_vcpu_init * init)1477 static bool kvm_vcpu_init_changed(struct kvm_vcpu *vcpu,
1478 const struct kvm_vcpu_init *init)
1479 {
1480 unsigned long features = init->features[0];
1481
1482 return !bitmap_equal(vcpu->kvm->arch.vcpu_features, &features,
1483 KVM_VCPU_MAX_FEATURES);
1484 }
1485
kvm_setup_vcpu(struct kvm_vcpu * vcpu)1486 static int kvm_setup_vcpu(struct kvm_vcpu *vcpu)
1487 {
1488 struct kvm *kvm = vcpu->kvm;
1489 int ret = 0;
1490
1491 /*
1492 * When the vCPU has a PMU, but no PMU is set for the guest
1493 * yet, set the default one.
1494 */
1495 if (kvm_vcpu_has_pmu(vcpu) && !kvm->arch.arm_pmu)
1496 ret = kvm_arm_set_default_pmu(kvm);
1497
1498 /* Prepare for nested if required */
1499 if (!ret && vcpu_has_nv(vcpu))
1500 ret = kvm_vcpu_init_nested(vcpu);
1501
1502 return ret;
1503 }
1504
__kvm_vcpu_set_target(struct kvm_vcpu * vcpu,const struct kvm_vcpu_init * init)1505 static int __kvm_vcpu_set_target(struct kvm_vcpu *vcpu,
1506 const struct kvm_vcpu_init *init)
1507 {
1508 unsigned long features = init->features[0];
1509 struct kvm *kvm = vcpu->kvm;
1510 int ret = -EINVAL;
1511
1512 mutex_lock(&kvm->arch.config_lock);
1513
1514 if (test_bit(KVM_ARCH_FLAG_VCPU_FEATURES_CONFIGURED, &kvm->arch.flags) &&
1515 kvm_vcpu_init_changed(vcpu, init))
1516 goto out_unlock;
1517
1518 bitmap_copy(kvm->arch.vcpu_features, &features, KVM_VCPU_MAX_FEATURES);
1519
1520 ret = kvm_setup_vcpu(vcpu);
1521 if (ret)
1522 goto out_unlock;
1523
1524 /* Now we know what it is, we can reset it. */
1525 kvm_reset_vcpu(vcpu);
1526
1527 set_bit(KVM_ARCH_FLAG_VCPU_FEATURES_CONFIGURED, &kvm->arch.flags);
1528 vcpu_set_flag(vcpu, VCPU_INITIALIZED);
1529 ret = 0;
1530 out_unlock:
1531 mutex_unlock(&kvm->arch.config_lock);
1532 return ret;
1533 }
1534
kvm_vcpu_set_target(struct kvm_vcpu * vcpu,const struct kvm_vcpu_init * init)1535 static int kvm_vcpu_set_target(struct kvm_vcpu *vcpu,
1536 const struct kvm_vcpu_init *init)
1537 {
1538 int ret;
1539
1540 if (init->target != KVM_ARM_TARGET_GENERIC_V8 &&
1541 init->target != kvm_target_cpu())
1542 return -EINVAL;
1543
1544 ret = kvm_vcpu_init_check_features(vcpu, init);
1545 if (ret)
1546 return ret;
1547
1548 if (!kvm_vcpu_initialized(vcpu))
1549 return __kvm_vcpu_set_target(vcpu, init);
1550
1551 if (kvm_vcpu_init_changed(vcpu, init))
1552 return -EINVAL;
1553
1554 kvm_reset_vcpu(vcpu);
1555 return 0;
1556 }
1557
kvm_arch_vcpu_ioctl_vcpu_init(struct kvm_vcpu * vcpu,struct kvm_vcpu_init * init)1558 static int kvm_arch_vcpu_ioctl_vcpu_init(struct kvm_vcpu *vcpu,
1559 struct kvm_vcpu_init *init)
1560 {
1561 bool power_off = false;
1562 int ret;
1563
1564 /*
1565 * Treat the power-off vCPU feature as ephemeral. Clear the bit to avoid
1566 * reflecting it in the finalized feature set, thus limiting its scope
1567 * to a single KVM_ARM_VCPU_INIT call.
1568 */
1569 if (init->features[0] & BIT(KVM_ARM_VCPU_POWER_OFF)) {
1570 init->features[0] &= ~BIT(KVM_ARM_VCPU_POWER_OFF);
1571 power_off = true;
1572 }
1573
1574 ret = kvm_vcpu_set_target(vcpu, init);
1575 if (ret)
1576 return ret;
1577
1578 /*
1579 * Ensure a rebooted VM will fault in RAM pages and detect if the
1580 * guest MMU is turned off and flush the caches as needed.
1581 *
1582 * S2FWB enforces all memory accesses to RAM being cacheable,
1583 * ensuring that the data side is always coherent. We still
1584 * need to invalidate the I-cache though, as FWB does *not*
1585 * imply CTR_EL0.DIC.
1586 */
1587 if (vcpu_has_run_once(vcpu)) {
1588 if (!cpus_have_final_cap(ARM64_HAS_STAGE2_FWB))
1589 stage2_unmap_vm(vcpu->kvm);
1590 else
1591 icache_inval_all_pou();
1592 }
1593
1594 vcpu_reset_hcr(vcpu);
1595
1596 /*
1597 * Handle the "start in power-off" case.
1598 */
1599 spin_lock(&vcpu->arch.mp_state_lock);
1600
1601 if (power_off)
1602 __kvm_arm_vcpu_power_off(vcpu);
1603 else
1604 WRITE_ONCE(vcpu->arch.mp_state.mp_state, KVM_MP_STATE_RUNNABLE);
1605
1606 spin_unlock(&vcpu->arch.mp_state_lock);
1607
1608 return 0;
1609 }
1610
kvm_arm_vcpu_set_attr(struct kvm_vcpu * vcpu,struct kvm_device_attr * attr)1611 static int kvm_arm_vcpu_set_attr(struct kvm_vcpu *vcpu,
1612 struct kvm_device_attr *attr)
1613 {
1614 int ret = -ENXIO;
1615
1616 switch (attr->group) {
1617 default:
1618 ret = kvm_arm_vcpu_arch_set_attr(vcpu, attr);
1619 break;
1620 }
1621
1622 return ret;
1623 }
1624
kvm_arm_vcpu_get_attr(struct kvm_vcpu * vcpu,struct kvm_device_attr * attr)1625 static int kvm_arm_vcpu_get_attr(struct kvm_vcpu *vcpu,
1626 struct kvm_device_attr *attr)
1627 {
1628 int ret = -ENXIO;
1629
1630 switch (attr->group) {
1631 default:
1632 ret = kvm_arm_vcpu_arch_get_attr(vcpu, attr);
1633 break;
1634 }
1635
1636 return ret;
1637 }
1638
kvm_arm_vcpu_has_attr(struct kvm_vcpu * vcpu,struct kvm_device_attr * attr)1639 static int kvm_arm_vcpu_has_attr(struct kvm_vcpu *vcpu,
1640 struct kvm_device_attr *attr)
1641 {
1642 int ret = -ENXIO;
1643
1644 switch (attr->group) {
1645 default:
1646 ret = kvm_arm_vcpu_arch_has_attr(vcpu, attr);
1647 break;
1648 }
1649
1650 return ret;
1651 }
1652
kvm_arm_vcpu_get_events(struct kvm_vcpu * vcpu,struct kvm_vcpu_events * events)1653 static int kvm_arm_vcpu_get_events(struct kvm_vcpu *vcpu,
1654 struct kvm_vcpu_events *events)
1655 {
1656 memset(events, 0, sizeof(*events));
1657
1658 return __kvm_arm_vcpu_get_events(vcpu, events);
1659 }
1660
kvm_arm_vcpu_set_events(struct kvm_vcpu * vcpu,struct kvm_vcpu_events * events)1661 static int kvm_arm_vcpu_set_events(struct kvm_vcpu *vcpu,
1662 struct kvm_vcpu_events *events)
1663 {
1664 int i;
1665
1666 /* check whether the reserved field is zero */
1667 for (i = 0; i < ARRAY_SIZE(events->reserved); i++)
1668 if (events->reserved[i])
1669 return -EINVAL;
1670
1671 /* check whether the pad field is zero */
1672 for (i = 0; i < ARRAY_SIZE(events->exception.pad); i++)
1673 if (events->exception.pad[i])
1674 return -EINVAL;
1675
1676 return __kvm_arm_vcpu_set_events(vcpu, events);
1677 }
1678
kvm_arch_vcpu_ioctl(struct file * filp,unsigned int ioctl,unsigned long arg)1679 long kvm_arch_vcpu_ioctl(struct file *filp,
1680 unsigned int ioctl, unsigned long arg)
1681 {
1682 struct kvm_vcpu *vcpu = filp->private_data;
1683 void __user *argp = (void __user *)arg;
1684 struct kvm_device_attr attr;
1685 long r;
1686
1687 switch (ioctl) {
1688 case KVM_ARM_VCPU_INIT: {
1689 struct kvm_vcpu_init init;
1690
1691 r = -EFAULT;
1692 if (copy_from_user(&init, argp, sizeof(init)))
1693 break;
1694
1695 r = kvm_arch_vcpu_ioctl_vcpu_init(vcpu, &init);
1696 break;
1697 }
1698 case KVM_SET_ONE_REG:
1699 case KVM_GET_ONE_REG: {
1700 struct kvm_one_reg reg;
1701
1702 r = -ENOEXEC;
1703 if (unlikely(!kvm_vcpu_initialized(vcpu)))
1704 break;
1705
1706 r = -EFAULT;
1707 if (copy_from_user(®, argp, sizeof(reg)))
1708 break;
1709
1710 /*
1711 * We could owe a reset due to PSCI. Handle the pending reset
1712 * here to ensure userspace register accesses are ordered after
1713 * the reset.
1714 */
1715 if (kvm_check_request(KVM_REQ_VCPU_RESET, vcpu))
1716 kvm_reset_vcpu(vcpu);
1717
1718 if (ioctl == KVM_SET_ONE_REG)
1719 r = kvm_arm_set_reg(vcpu, ®);
1720 else
1721 r = kvm_arm_get_reg(vcpu, ®);
1722 break;
1723 }
1724 case KVM_GET_REG_LIST: {
1725 struct kvm_reg_list __user *user_list = argp;
1726 struct kvm_reg_list reg_list;
1727 unsigned n;
1728
1729 r = -ENOEXEC;
1730 if (unlikely(!kvm_vcpu_initialized(vcpu)))
1731 break;
1732
1733 r = -EPERM;
1734 if (!kvm_arm_vcpu_is_finalized(vcpu))
1735 break;
1736
1737 r = -EFAULT;
1738 if (copy_from_user(®_list, user_list, sizeof(reg_list)))
1739 break;
1740 n = reg_list.n;
1741 reg_list.n = kvm_arm_num_regs(vcpu);
1742 if (copy_to_user(user_list, ®_list, sizeof(reg_list)))
1743 break;
1744 r = -E2BIG;
1745 if (n < reg_list.n)
1746 break;
1747 r = kvm_arm_copy_reg_indices(vcpu, user_list->reg);
1748 break;
1749 }
1750 case KVM_SET_DEVICE_ATTR: {
1751 r = -EFAULT;
1752 if (copy_from_user(&attr, argp, sizeof(attr)))
1753 break;
1754 r = kvm_arm_vcpu_set_attr(vcpu, &attr);
1755 break;
1756 }
1757 case KVM_GET_DEVICE_ATTR: {
1758 r = -EFAULT;
1759 if (copy_from_user(&attr, argp, sizeof(attr)))
1760 break;
1761 r = kvm_arm_vcpu_get_attr(vcpu, &attr);
1762 break;
1763 }
1764 case KVM_HAS_DEVICE_ATTR: {
1765 r = -EFAULT;
1766 if (copy_from_user(&attr, argp, sizeof(attr)))
1767 break;
1768 r = kvm_arm_vcpu_has_attr(vcpu, &attr);
1769 break;
1770 }
1771 case KVM_GET_VCPU_EVENTS: {
1772 struct kvm_vcpu_events events;
1773
1774 if (kvm_arm_vcpu_get_events(vcpu, &events))
1775 return -EINVAL;
1776
1777 if (copy_to_user(argp, &events, sizeof(events)))
1778 return -EFAULT;
1779
1780 return 0;
1781 }
1782 case KVM_SET_VCPU_EVENTS: {
1783 struct kvm_vcpu_events events;
1784
1785 if (copy_from_user(&events, argp, sizeof(events)))
1786 return -EFAULT;
1787
1788 return kvm_arm_vcpu_set_events(vcpu, &events);
1789 }
1790 case KVM_ARM_VCPU_FINALIZE: {
1791 int what;
1792
1793 if (!kvm_vcpu_initialized(vcpu))
1794 return -ENOEXEC;
1795
1796 if (get_user(what, (const int __user *)argp))
1797 return -EFAULT;
1798
1799 return kvm_arm_vcpu_finalize(vcpu, what);
1800 }
1801 default:
1802 r = -EINVAL;
1803 }
1804
1805 return r;
1806 }
1807
kvm_arch_sync_dirty_log(struct kvm * kvm,struct kvm_memory_slot * memslot)1808 void kvm_arch_sync_dirty_log(struct kvm *kvm, struct kvm_memory_slot *memslot)
1809 {
1810
1811 }
1812
kvm_vm_ioctl_set_device_addr(struct kvm * kvm,struct kvm_arm_device_addr * dev_addr)1813 static int kvm_vm_ioctl_set_device_addr(struct kvm *kvm,
1814 struct kvm_arm_device_addr *dev_addr)
1815 {
1816 switch (FIELD_GET(KVM_ARM_DEVICE_ID_MASK, dev_addr->id)) {
1817 case KVM_ARM_DEVICE_VGIC_V2:
1818 if (!vgic_present)
1819 return -ENXIO;
1820 return kvm_set_legacy_vgic_v2_addr(kvm, dev_addr);
1821 default:
1822 return -ENODEV;
1823 }
1824 }
1825
kvm_vm_has_attr(struct kvm * kvm,struct kvm_device_attr * attr)1826 static int kvm_vm_has_attr(struct kvm *kvm, struct kvm_device_attr *attr)
1827 {
1828 switch (attr->group) {
1829 case KVM_ARM_VM_SMCCC_CTRL:
1830 return kvm_vm_smccc_has_attr(kvm, attr);
1831 default:
1832 return -ENXIO;
1833 }
1834 }
1835
kvm_vm_set_attr(struct kvm * kvm,struct kvm_device_attr * attr)1836 static int kvm_vm_set_attr(struct kvm *kvm, struct kvm_device_attr *attr)
1837 {
1838 switch (attr->group) {
1839 case KVM_ARM_VM_SMCCC_CTRL:
1840 return kvm_vm_smccc_set_attr(kvm, attr);
1841 default:
1842 return -ENXIO;
1843 }
1844 }
1845
kvm_arch_vm_ioctl(struct file * filp,unsigned int ioctl,unsigned long arg)1846 int kvm_arch_vm_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg)
1847 {
1848 struct kvm *kvm = filp->private_data;
1849 void __user *argp = (void __user *)arg;
1850 struct kvm_device_attr attr;
1851
1852 switch (ioctl) {
1853 case KVM_CREATE_IRQCHIP: {
1854 int ret;
1855 if (!vgic_present)
1856 return -ENXIO;
1857 mutex_lock(&kvm->lock);
1858 ret = kvm_vgic_create(kvm, KVM_DEV_TYPE_ARM_VGIC_V2);
1859 mutex_unlock(&kvm->lock);
1860 return ret;
1861 }
1862 case KVM_ARM_SET_DEVICE_ADDR: {
1863 struct kvm_arm_device_addr dev_addr;
1864
1865 if (copy_from_user(&dev_addr, argp, sizeof(dev_addr)))
1866 return -EFAULT;
1867 return kvm_vm_ioctl_set_device_addr(kvm, &dev_addr);
1868 }
1869 case KVM_ARM_PREFERRED_TARGET: {
1870 struct kvm_vcpu_init init = {
1871 .target = KVM_ARM_TARGET_GENERIC_V8,
1872 };
1873
1874 if (copy_to_user(argp, &init, sizeof(init)))
1875 return -EFAULT;
1876
1877 return 0;
1878 }
1879 case KVM_ARM_MTE_COPY_TAGS: {
1880 struct kvm_arm_copy_mte_tags copy_tags;
1881
1882 if (copy_from_user(©_tags, argp, sizeof(copy_tags)))
1883 return -EFAULT;
1884 return kvm_vm_ioctl_mte_copy_tags(kvm, ©_tags);
1885 }
1886 case KVM_ARM_SET_COUNTER_OFFSET: {
1887 struct kvm_arm_counter_offset offset;
1888
1889 if (copy_from_user(&offset, argp, sizeof(offset)))
1890 return -EFAULT;
1891 return kvm_vm_ioctl_set_counter_offset(kvm, &offset);
1892 }
1893 case KVM_HAS_DEVICE_ATTR: {
1894 if (copy_from_user(&attr, argp, sizeof(attr)))
1895 return -EFAULT;
1896
1897 return kvm_vm_has_attr(kvm, &attr);
1898 }
1899 case KVM_SET_DEVICE_ATTR: {
1900 if (copy_from_user(&attr, argp, sizeof(attr)))
1901 return -EFAULT;
1902
1903 return kvm_vm_set_attr(kvm, &attr);
1904 }
1905 case KVM_ARM_GET_REG_WRITABLE_MASKS: {
1906 struct reg_mask_range range;
1907
1908 if (copy_from_user(&range, argp, sizeof(range)))
1909 return -EFAULT;
1910 return kvm_vm_ioctl_get_reg_writable_masks(kvm, &range);
1911 }
1912 default:
1913 return -EINVAL;
1914 }
1915 }
1916
1917 /* unlocks vcpus from @vcpu_lock_idx and smaller */
unlock_vcpus(struct kvm * kvm,int vcpu_lock_idx)1918 static void unlock_vcpus(struct kvm *kvm, int vcpu_lock_idx)
1919 {
1920 struct kvm_vcpu *tmp_vcpu;
1921
1922 for (; vcpu_lock_idx >= 0; vcpu_lock_idx--) {
1923 tmp_vcpu = kvm_get_vcpu(kvm, vcpu_lock_idx);
1924 mutex_unlock(&tmp_vcpu->mutex);
1925 }
1926 }
1927
unlock_all_vcpus(struct kvm * kvm)1928 void unlock_all_vcpus(struct kvm *kvm)
1929 {
1930 lockdep_assert_held(&kvm->lock);
1931
1932 unlock_vcpus(kvm, atomic_read(&kvm->online_vcpus) - 1);
1933 }
1934
1935 /* Returns true if all vcpus were locked, false otherwise */
lock_all_vcpus(struct kvm * kvm)1936 bool lock_all_vcpus(struct kvm *kvm)
1937 {
1938 struct kvm_vcpu *tmp_vcpu;
1939 unsigned long c;
1940
1941 lockdep_assert_held(&kvm->lock);
1942
1943 /*
1944 * Any time a vcpu is in an ioctl (including running), the
1945 * core KVM code tries to grab the vcpu->mutex.
1946 *
1947 * By grabbing the vcpu->mutex of all VCPUs we ensure that no
1948 * other VCPUs can fiddle with the state while we access it.
1949 */
1950 kvm_for_each_vcpu(c, tmp_vcpu, kvm) {
1951 if (!mutex_trylock(&tmp_vcpu->mutex)) {
1952 unlock_vcpus(kvm, c - 1);
1953 return false;
1954 }
1955 }
1956
1957 return true;
1958 }
1959
nvhe_percpu_size(void)1960 static unsigned long nvhe_percpu_size(void)
1961 {
1962 return (unsigned long)CHOOSE_NVHE_SYM(__per_cpu_end) -
1963 (unsigned long)CHOOSE_NVHE_SYM(__per_cpu_start);
1964 }
1965
nvhe_percpu_order(void)1966 static unsigned long nvhe_percpu_order(void)
1967 {
1968 unsigned long size = nvhe_percpu_size();
1969
1970 return size ? get_order(size) : 0;
1971 }
1972
pkvm_host_sve_state_order(void)1973 static size_t pkvm_host_sve_state_order(void)
1974 {
1975 return get_order(pkvm_host_sve_state_size());
1976 }
1977
1978 /* A lookup table holding the hypervisor VA for each vector slot */
1979 static void *hyp_spectre_vector_selector[BP_HARDEN_EL2_SLOTS];
1980
kvm_init_vector_slot(void * base,enum arm64_hyp_spectre_vector slot)1981 static void kvm_init_vector_slot(void *base, enum arm64_hyp_spectre_vector slot)
1982 {
1983 hyp_spectre_vector_selector[slot] = __kvm_vector_slot2addr(base, slot);
1984 }
1985
kvm_init_vector_slots(void)1986 static int kvm_init_vector_slots(void)
1987 {
1988 int err;
1989 void *base;
1990
1991 base = kern_hyp_va(kvm_ksym_ref(__kvm_hyp_vector));
1992 kvm_init_vector_slot(base, HYP_VECTOR_DIRECT);
1993
1994 base = kern_hyp_va(kvm_ksym_ref(__bp_harden_hyp_vecs));
1995 kvm_init_vector_slot(base, HYP_VECTOR_SPECTRE_DIRECT);
1996
1997 if (kvm_system_needs_idmapped_vectors() &&
1998 !is_protected_kvm_enabled()) {
1999 err = create_hyp_exec_mappings(__pa_symbol(__bp_harden_hyp_vecs),
2000 __BP_HARDEN_HYP_VECS_SZ, &base);
2001 if (err)
2002 return err;
2003 }
2004
2005 kvm_init_vector_slot(base, HYP_VECTOR_INDIRECT);
2006 kvm_init_vector_slot(base, HYP_VECTOR_SPECTRE_INDIRECT);
2007 return 0;
2008 }
2009
cpu_prepare_hyp_mode(int cpu,u32 hyp_va_bits)2010 static void __init cpu_prepare_hyp_mode(int cpu, u32 hyp_va_bits)
2011 {
2012 struct kvm_nvhe_init_params *params = per_cpu_ptr_nvhe_sym(kvm_init_params, cpu);
2013 unsigned long tcr;
2014
2015 /*
2016 * Calculate the raw per-cpu offset without a translation from the
2017 * kernel's mapping to the linear mapping, and store it in tpidr_el2
2018 * so that we can use adr_l to access per-cpu variables in EL2.
2019 * Also drop the KASAN tag which gets in the way...
2020 */
2021 params->tpidr_el2 = (unsigned long)kasan_reset_tag(per_cpu_ptr_nvhe_sym(__per_cpu_start, cpu)) -
2022 (unsigned long)kvm_ksym_ref(CHOOSE_NVHE_SYM(__per_cpu_start));
2023
2024 params->mair_el2 = read_sysreg(mair_el1);
2025
2026 tcr = read_sysreg(tcr_el1);
2027 if (cpus_have_final_cap(ARM64_KVM_HVHE)) {
2028 tcr &= ~(TCR_HD | TCR_HA | TCR_A1 | TCR_T0SZ_MASK);
2029 tcr |= TCR_EPD1_MASK;
2030 } else {
2031 unsigned long ips = FIELD_GET(TCR_IPS_MASK, tcr);
2032
2033 tcr &= TCR_EL2_MASK;
2034 tcr |= TCR_EL2_RES1 | FIELD_PREP(TCR_EL2_PS_MASK, ips);
2035 if (lpa2_is_enabled())
2036 tcr |= TCR_EL2_DS;
2037 }
2038 tcr |= TCR_T0SZ(hyp_va_bits);
2039 params->tcr_el2 = tcr;
2040
2041 params->pgd_pa = kvm_mmu_get_httbr();
2042 if (is_protected_kvm_enabled())
2043 params->hcr_el2 = HCR_HOST_NVHE_PROTECTED_FLAGS;
2044 else
2045 params->hcr_el2 = HCR_HOST_NVHE_FLAGS;
2046 if (cpus_have_final_cap(ARM64_KVM_HVHE))
2047 params->hcr_el2 |= HCR_E2H;
2048 params->vttbr = params->vtcr = 0;
2049
2050 /*
2051 * Flush the init params from the data cache because the struct will
2052 * be read while the MMU is off.
2053 */
2054 kvm_flush_dcache_to_poc(params, sizeof(*params));
2055 }
2056
hyp_install_host_vector(void)2057 static void hyp_install_host_vector(void)
2058 {
2059 struct kvm_nvhe_init_params *params;
2060 struct arm_smccc_res res;
2061
2062 /* Switch from the HYP stub to our own HYP init vector */
2063 __hyp_set_vectors(kvm_get_idmap_vector());
2064
2065 /*
2066 * Call initialization code, and switch to the full blown HYP code.
2067 * If the cpucaps haven't been finalized yet, something has gone very
2068 * wrong, and hyp will crash and burn when it uses any
2069 * cpus_have_*_cap() wrapper.
2070 */
2071 BUG_ON(!system_capabilities_finalized());
2072 params = this_cpu_ptr_nvhe_sym(kvm_init_params);
2073 arm_smccc_1_1_hvc(KVM_HOST_SMCCC_FUNC(__kvm_hyp_init), virt_to_phys(params), &res);
2074 WARN_ON(res.a0 != SMCCC_RET_SUCCESS);
2075 }
2076
cpu_init_hyp_mode(void)2077 static void cpu_init_hyp_mode(void)
2078 {
2079 hyp_install_host_vector();
2080
2081 /*
2082 * Disabling SSBD on a non-VHE system requires us to enable SSBS
2083 * at EL2.
2084 */
2085 if (this_cpu_has_cap(ARM64_SSBS) &&
2086 arm64_get_spectre_v4_state() == SPECTRE_VULNERABLE) {
2087 kvm_call_hyp_nvhe(__kvm_enable_ssbs);
2088 }
2089 }
2090
cpu_hyp_reset(void)2091 static void cpu_hyp_reset(void)
2092 {
2093 if (!is_kernel_in_hyp_mode())
2094 __hyp_reset_vectors();
2095 }
2096
2097 /*
2098 * EL2 vectors can be mapped and rerouted in a number of ways,
2099 * depending on the kernel configuration and CPU present:
2100 *
2101 * - If the CPU is affected by Spectre-v2, the hardening sequence is
2102 * placed in one of the vector slots, which is executed before jumping
2103 * to the real vectors.
2104 *
2105 * - If the CPU also has the ARM64_SPECTRE_V3A cap, the slot
2106 * containing the hardening sequence is mapped next to the idmap page,
2107 * and executed before jumping to the real vectors.
2108 *
2109 * - If the CPU only has the ARM64_SPECTRE_V3A cap, then an
2110 * empty slot is selected, mapped next to the idmap page, and
2111 * executed before jumping to the real vectors.
2112 *
2113 * Note that ARM64_SPECTRE_V3A is somewhat incompatible with
2114 * VHE, as we don't have hypervisor-specific mappings. If the system
2115 * is VHE and yet selects this capability, it will be ignored.
2116 */
cpu_set_hyp_vector(void)2117 static void cpu_set_hyp_vector(void)
2118 {
2119 struct bp_hardening_data *data = this_cpu_ptr(&bp_hardening_data);
2120 void *vector = hyp_spectre_vector_selector[data->slot];
2121
2122 if (!is_protected_kvm_enabled())
2123 *this_cpu_ptr_hyp_sym(kvm_hyp_vector) = (unsigned long)vector;
2124 else
2125 kvm_call_hyp_nvhe(__pkvm_cpu_set_vector, data->slot);
2126 }
2127
cpu_hyp_init_context(void)2128 static void cpu_hyp_init_context(void)
2129 {
2130 kvm_init_host_cpu_context(host_data_ptr(host_ctxt));
2131 kvm_init_host_debug_data();
2132
2133 if (!is_kernel_in_hyp_mode())
2134 cpu_init_hyp_mode();
2135 }
2136
cpu_hyp_init_features(void)2137 static void cpu_hyp_init_features(void)
2138 {
2139 cpu_set_hyp_vector();
2140
2141 if (is_kernel_in_hyp_mode())
2142 kvm_timer_init_vhe();
2143
2144 if (vgic_present)
2145 kvm_vgic_init_cpu_hardware();
2146 }
2147
cpu_hyp_reinit(void)2148 static void cpu_hyp_reinit(void)
2149 {
2150 cpu_hyp_reset();
2151 cpu_hyp_init_context();
2152 cpu_hyp_init_features();
2153 }
2154
cpu_hyp_init(void * discard)2155 static void cpu_hyp_init(void *discard)
2156 {
2157 if (!__this_cpu_read(kvm_hyp_initialized)) {
2158 cpu_hyp_reinit();
2159 __this_cpu_write(kvm_hyp_initialized, 1);
2160 }
2161 }
2162
cpu_hyp_uninit(void * discard)2163 static void cpu_hyp_uninit(void *discard)
2164 {
2165 if (__this_cpu_read(kvm_hyp_initialized)) {
2166 cpu_hyp_reset();
2167 __this_cpu_write(kvm_hyp_initialized, 0);
2168 }
2169 }
2170
kvm_arch_enable_virtualization_cpu(void)2171 int kvm_arch_enable_virtualization_cpu(void)
2172 {
2173 /*
2174 * Most calls to this function are made with migration
2175 * disabled, but not with preemption disabled. The former is
2176 * enough to ensure correctness, but most of the helpers
2177 * expect the later and will throw a tantrum otherwise.
2178 */
2179 preempt_disable();
2180
2181 cpu_hyp_init(NULL);
2182
2183 kvm_vgic_cpu_up();
2184 kvm_timer_cpu_up();
2185
2186 preempt_enable();
2187
2188 return 0;
2189 }
2190
kvm_arch_disable_virtualization_cpu(void)2191 void kvm_arch_disable_virtualization_cpu(void)
2192 {
2193 kvm_timer_cpu_down();
2194 kvm_vgic_cpu_down();
2195
2196 if (!is_protected_kvm_enabled())
2197 cpu_hyp_uninit(NULL);
2198 }
2199
2200 #ifdef CONFIG_CPU_PM
hyp_init_cpu_pm_notifier(struct notifier_block * self,unsigned long cmd,void * v)2201 static int hyp_init_cpu_pm_notifier(struct notifier_block *self,
2202 unsigned long cmd,
2203 void *v)
2204 {
2205 /*
2206 * kvm_hyp_initialized is left with its old value over
2207 * PM_ENTER->PM_EXIT. It is used to indicate PM_EXIT should
2208 * re-enable hyp.
2209 */
2210 switch (cmd) {
2211 case CPU_PM_ENTER:
2212 if (__this_cpu_read(kvm_hyp_initialized))
2213 /*
2214 * don't update kvm_hyp_initialized here
2215 * so that the hyp will be re-enabled
2216 * when we resume. See below.
2217 */
2218 cpu_hyp_reset();
2219
2220 return NOTIFY_OK;
2221 case CPU_PM_ENTER_FAILED:
2222 case CPU_PM_EXIT:
2223 if (__this_cpu_read(kvm_hyp_initialized))
2224 /* The hyp was enabled before suspend. */
2225 cpu_hyp_reinit();
2226
2227 return NOTIFY_OK;
2228
2229 default:
2230 return NOTIFY_DONE;
2231 }
2232 }
2233
2234 static struct notifier_block hyp_init_cpu_pm_nb = {
2235 .notifier_call = hyp_init_cpu_pm_notifier,
2236 };
2237
hyp_cpu_pm_init(void)2238 static void __init hyp_cpu_pm_init(void)
2239 {
2240 if (!is_protected_kvm_enabled())
2241 cpu_pm_register_notifier(&hyp_init_cpu_pm_nb);
2242 }
hyp_cpu_pm_exit(void)2243 static void __init hyp_cpu_pm_exit(void)
2244 {
2245 if (!is_protected_kvm_enabled())
2246 cpu_pm_unregister_notifier(&hyp_init_cpu_pm_nb);
2247 }
2248 #else
hyp_cpu_pm_init(void)2249 static inline void __init hyp_cpu_pm_init(void)
2250 {
2251 }
hyp_cpu_pm_exit(void)2252 static inline void __init hyp_cpu_pm_exit(void)
2253 {
2254 }
2255 #endif
2256
init_cpu_logical_map(void)2257 static void __init init_cpu_logical_map(void)
2258 {
2259 unsigned int cpu;
2260
2261 /*
2262 * Copy the MPIDR <-> logical CPU ID mapping to hyp.
2263 * Only copy the set of online CPUs whose features have been checked
2264 * against the finalized system capabilities. The hypervisor will not
2265 * allow any other CPUs from the `possible` set to boot.
2266 */
2267 for_each_online_cpu(cpu)
2268 hyp_cpu_logical_map[cpu] = cpu_logical_map(cpu);
2269 }
2270
2271 #define init_psci_0_1_impl_state(config, what) \
2272 config.psci_0_1_ ## what ## _implemented = psci_ops.what
2273
init_psci_relay(void)2274 static bool __init init_psci_relay(void)
2275 {
2276 /*
2277 * If PSCI has not been initialized, protected KVM cannot install
2278 * itself on newly booted CPUs.
2279 */
2280 if (!psci_ops.get_version) {
2281 kvm_err("Cannot initialize protected mode without PSCI\n");
2282 return false;
2283 }
2284
2285 kvm_host_psci_config.version = psci_ops.get_version();
2286 kvm_host_psci_config.smccc_version = arm_smccc_get_version();
2287
2288 if (kvm_host_psci_config.version == PSCI_VERSION(0, 1)) {
2289 kvm_host_psci_config.function_ids_0_1 = get_psci_0_1_function_ids();
2290 init_psci_0_1_impl_state(kvm_host_psci_config, cpu_suspend);
2291 init_psci_0_1_impl_state(kvm_host_psci_config, cpu_on);
2292 init_psci_0_1_impl_state(kvm_host_psci_config, cpu_off);
2293 init_psci_0_1_impl_state(kvm_host_psci_config, migrate);
2294 }
2295 return true;
2296 }
2297
init_subsystems(void)2298 static int __init init_subsystems(void)
2299 {
2300 int err = 0;
2301
2302 /*
2303 * Enable hardware so that subsystem initialisation can access EL2.
2304 */
2305 on_each_cpu(cpu_hyp_init, NULL, 1);
2306
2307 /*
2308 * Register CPU lower-power notifier
2309 */
2310 hyp_cpu_pm_init();
2311
2312 /*
2313 * Init HYP view of VGIC
2314 */
2315 err = kvm_vgic_hyp_init();
2316 switch (err) {
2317 case 0:
2318 vgic_present = true;
2319 break;
2320 case -ENODEV:
2321 case -ENXIO:
2322 /*
2323 * No VGIC? No pKVM for you.
2324 *
2325 * Protected mode assumes that VGICv3 is present, so no point
2326 * in trying to hobble along if vgic initialization fails.
2327 */
2328 if (is_protected_kvm_enabled())
2329 goto out;
2330
2331 /*
2332 * Otherwise, userspace could choose to implement a GIC for its
2333 * guest on non-cooperative hardware.
2334 */
2335 vgic_present = false;
2336 err = 0;
2337 break;
2338 default:
2339 goto out;
2340 }
2341
2342 if (kvm_mode == KVM_MODE_NV &&
2343 !(vgic_present && kvm_vgic_global_state.type == VGIC_V3)) {
2344 kvm_err("NV support requires GICv3, giving up\n");
2345 err = -EINVAL;
2346 goto out;
2347 }
2348
2349 /*
2350 * Init HYP architected timer support
2351 */
2352 err = kvm_timer_hyp_init(vgic_present);
2353 if (err)
2354 goto out;
2355
2356 kvm_register_perf_callbacks(NULL);
2357
2358 out:
2359 if (err)
2360 hyp_cpu_pm_exit();
2361
2362 if (err || !is_protected_kvm_enabled())
2363 on_each_cpu(cpu_hyp_uninit, NULL, 1);
2364
2365 return err;
2366 }
2367
teardown_subsystems(void)2368 static void __init teardown_subsystems(void)
2369 {
2370 kvm_unregister_perf_callbacks();
2371 hyp_cpu_pm_exit();
2372 }
2373
teardown_hyp_mode(void)2374 static void __init teardown_hyp_mode(void)
2375 {
2376 bool free_sve = system_supports_sve() && is_protected_kvm_enabled();
2377 int cpu;
2378
2379 free_hyp_pgds();
2380 for_each_possible_cpu(cpu) {
2381 free_pages(per_cpu(kvm_arm_hyp_stack_base, cpu), NVHE_STACK_SHIFT - PAGE_SHIFT);
2382 free_pages(kvm_nvhe_sym(kvm_arm_hyp_percpu_base)[cpu], nvhe_percpu_order());
2383
2384 if (free_sve) {
2385 struct cpu_sve_state *sve_state;
2386
2387 sve_state = per_cpu_ptr_nvhe_sym(kvm_host_data, cpu)->sve_state;
2388 free_pages((unsigned long) sve_state, pkvm_host_sve_state_order());
2389 }
2390 }
2391 }
2392
do_pkvm_init(u32 hyp_va_bits)2393 static int __init do_pkvm_init(u32 hyp_va_bits)
2394 {
2395 void *per_cpu_base = kvm_ksym_ref(kvm_nvhe_sym(kvm_arm_hyp_percpu_base));
2396 int ret;
2397
2398 preempt_disable();
2399 cpu_hyp_init_context();
2400 ret = kvm_call_hyp_nvhe(__pkvm_init, hyp_mem_base, hyp_mem_size,
2401 num_possible_cpus(), kern_hyp_va(per_cpu_base),
2402 hyp_va_bits);
2403 cpu_hyp_init_features();
2404
2405 /*
2406 * The stub hypercalls are now disabled, so set our local flag to
2407 * prevent a later re-init attempt in kvm_arch_enable_virtualization_cpu().
2408 */
2409 __this_cpu_write(kvm_hyp_initialized, 1);
2410 preempt_enable();
2411
2412 return ret;
2413 }
2414
get_hyp_id_aa64pfr0_el1(void)2415 static u64 get_hyp_id_aa64pfr0_el1(void)
2416 {
2417 /*
2418 * Track whether the system isn't affected by spectre/meltdown in the
2419 * hypervisor's view of id_aa64pfr0_el1, used for protected VMs.
2420 * Although this is per-CPU, we make it global for simplicity, e.g., not
2421 * to have to worry about vcpu migration.
2422 *
2423 * Unlike for non-protected VMs, userspace cannot override this for
2424 * protected VMs.
2425 */
2426 u64 val = read_sanitised_ftr_reg(SYS_ID_AA64PFR0_EL1);
2427
2428 val &= ~(ARM64_FEATURE_MASK(ID_AA64PFR0_EL1_CSV2) |
2429 ARM64_FEATURE_MASK(ID_AA64PFR0_EL1_CSV3));
2430
2431 val |= FIELD_PREP(ARM64_FEATURE_MASK(ID_AA64PFR0_EL1_CSV2),
2432 arm64_get_spectre_v2_state() == SPECTRE_UNAFFECTED);
2433 val |= FIELD_PREP(ARM64_FEATURE_MASK(ID_AA64PFR0_EL1_CSV3),
2434 arm64_get_meltdown_state() == SPECTRE_UNAFFECTED);
2435
2436 return val;
2437 }
2438
kvm_hyp_init_symbols(void)2439 static void kvm_hyp_init_symbols(void)
2440 {
2441 kvm_nvhe_sym(id_aa64pfr0_el1_sys_val) = get_hyp_id_aa64pfr0_el1();
2442 kvm_nvhe_sym(id_aa64pfr1_el1_sys_val) = read_sanitised_ftr_reg(SYS_ID_AA64PFR1_EL1);
2443 kvm_nvhe_sym(id_aa64isar0_el1_sys_val) = read_sanitised_ftr_reg(SYS_ID_AA64ISAR0_EL1);
2444 kvm_nvhe_sym(id_aa64isar1_el1_sys_val) = read_sanitised_ftr_reg(SYS_ID_AA64ISAR1_EL1);
2445 kvm_nvhe_sym(id_aa64isar2_el1_sys_val) = read_sanitised_ftr_reg(SYS_ID_AA64ISAR2_EL1);
2446 kvm_nvhe_sym(id_aa64mmfr0_el1_sys_val) = read_sanitised_ftr_reg(SYS_ID_AA64MMFR0_EL1);
2447 kvm_nvhe_sym(id_aa64mmfr1_el1_sys_val) = read_sanitised_ftr_reg(SYS_ID_AA64MMFR1_EL1);
2448 kvm_nvhe_sym(id_aa64mmfr2_el1_sys_val) = read_sanitised_ftr_reg(SYS_ID_AA64MMFR2_EL1);
2449 kvm_nvhe_sym(id_aa64smfr0_el1_sys_val) = read_sanitised_ftr_reg(SYS_ID_AA64SMFR0_EL1);
2450 kvm_nvhe_sym(__icache_flags) = __icache_flags;
2451 kvm_nvhe_sym(kvm_arm_vmid_bits) = kvm_arm_vmid_bits;
2452
2453 /*
2454 * Flush entire BSS since part of its data containing init symbols is read
2455 * while the MMU is off.
2456 */
2457 kvm_flush_dcache_to_poc(kvm_ksym_ref(__hyp_bss_start),
2458 kvm_ksym_ref(__hyp_bss_end) - kvm_ksym_ref(__hyp_bss_start));
2459 }
2460
kvm_hyp_init_protection(u32 hyp_va_bits)2461 static int __init kvm_hyp_init_protection(u32 hyp_va_bits)
2462 {
2463 void *addr = phys_to_virt(hyp_mem_base);
2464 int ret;
2465
2466 ret = create_hyp_mappings(addr, addr + hyp_mem_size, PAGE_HYP);
2467 if (ret)
2468 return ret;
2469
2470 ret = do_pkvm_init(hyp_va_bits);
2471 if (ret)
2472 return ret;
2473
2474 free_hyp_pgds();
2475
2476 return 0;
2477 }
2478
init_pkvm_host_sve_state(void)2479 static int init_pkvm_host_sve_state(void)
2480 {
2481 int cpu;
2482
2483 if (!system_supports_sve())
2484 return 0;
2485
2486 /* Allocate pages for host sve state in protected mode. */
2487 for_each_possible_cpu(cpu) {
2488 struct page *page = alloc_pages(GFP_KERNEL, pkvm_host_sve_state_order());
2489
2490 if (!page)
2491 return -ENOMEM;
2492
2493 per_cpu_ptr_nvhe_sym(kvm_host_data, cpu)->sve_state = page_address(page);
2494 }
2495
2496 /*
2497 * Don't map the pages in hyp since these are only used in protected
2498 * mode, which will (re)create its own mapping when initialized.
2499 */
2500
2501 return 0;
2502 }
2503
2504 /*
2505 * Finalizes the initialization of hyp mode, once everything else is initialized
2506 * and the initialziation process cannot fail.
2507 */
finalize_init_hyp_mode(void)2508 static void finalize_init_hyp_mode(void)
2509 {
2510 int cpu;
2511
2512 if (system_supports_sve() && is_protected_kvm_enabled()) {
2513 for_each_possible_cpu(cpu) {
2514 struct cpu_sve_state *sve_state;
2515
2516 sve_state = per_cpu_ptr_nvhe_sym(kvm_host_data, cpu)->sve_state;
2517 per_cpu_ptr_nvhe_sym(kvm_host_data, cpu)->sve_state =
2518 kern_hyp_va(sve_state);
2519 }
2520 }
2521 }
2522
pkvm_hyp_init_ptrauth(void)2523 static void pkvm_hyp_init_ptrauth(void)
2524 {
2525 struct kvm_cpu_context *hyp_ctxt;
2526 int cpu;
2527
2528 for_each_possible_cpu(cpu) {
2529 hyp_ctxt = per_cpu_ptr_nvhe_sym(kvm_hyp_ctxt, cpu);
2530 hyp_ctxt->sys_regs[APIAKEYLO_EL1] = get_random_long();
2531 hyp_ctxt->sys_regs[APIAKEYHI_EL1] = get_random_long();
2532 hyp_ctxt->sys_regs[APIBKEYLO_EL1] = get_random_long();
2533 hyp_ctxt->sys_regs[APIBKEYHI_EL1] = get_random_long();
2534 hyp_ctxt->sys_regs[APDAKEYLO_EL1] = get_random_long();
2535 hyp_ctxt->sys_regs[APDAKEYHI_EL1] = get_random_long();
2536 hyp_ctxt->sys_regs[APDBKEYLO_EL1] = get_random_long();
2537 hyp_ctxt->sys_regs[APDBKEYHI_EL1] = get_random_long();
2538 hyp_ctxt->sys_regs[APGAKEYLO_EL1] = get_random_long();
2539 hyp_ctxt->sys_regs[APGAKEYHI_EL1] = get_random_long();
2540 }
2541 }
2542
2543 /* Inits Hyp-mode on all online CPUs */
init_hyp_mode(void)2544 static int __init init_hyp_mode(void)
2545 {
2546 u32 hyp_va_bits;
2547 int cpu;
2548 int err = -ENOMEM;
2549
2550 /*
2551 * The protected Hyp-mode cannot be initialized if the memory pool
2552 * allocation has failed.
2553 */
2554 if (is_protected_kvm_enabled() && !hyp_mem_base)
2555 goto out_err;
2556
2557 /*
2558 * Allocate Hyp PGD and setup Hyp identity mapping
2559 */
2560 err = kvm_mmu_init(&hyp_va_bits);
2561 if (err)
2562 goto out_err;
2563
2564 /*
2565 * Allocate stack pages for Hypervisor-mode
2566 */
2567 for_each_possible_cpu(cpu) {
2568 unsigned long stack_base;
2569
2570 stack_base = __get_free_pages(GFP_KERNEL, NVHE_STACK_SHIFT - PAGE_SHIFT);
2571 if (!stack_base) {
2572 err = -ENOMEM;
2573 goto out_err;
2574 }
2575
2576 per_cpu(kvm_arm_hyp_stack_base, cpu) = stack_base;
2577 }
2578
2579 /*
2580 * Allocate and initialize pages for Hypervisor-mode percpu regions.
2581 */
2582 for_each_possible_cpu(cpu) {
2583 struct page *page;
2584 void *page_addr;
2585
2586 page = alloc_pages(GFP_KERNEL, nvhe_percpu_order());
2587 if (!page) {
2588 err = -ENOMEM;
2589 goto out_err;
2590 }
2591
2592 page_addr = page_address(page);
2593 memcpy(page_addr, CHOOSE_NVHE_SYM(__per_cpu_start), nvhe_percpu_size());
2594 kvm_nvhe_sym(kvm_arm_hyp_percpu_base)[cpu] = (unsigned long)page_addr;
2595 }
2596
2597 /*
2598 * Map the Hyp-code called directly from the host
2599 */
2600 err = create_hyp_mappings(kvm_ksym_ref(__hyp_text_start),
2601 kvm_ksym_ref(__hyp_text_end), PAGE_HYP_EXEC);
2602 if (err) {
2603 kvm_err("Cannot map world-switch code\n");
2604 goto out_err;
2605 }
2606
2607 err = create_hyp_mappings(kvm_ksym_ref(__hyp_rodata_start),
2608 kvm_ksym_ref(__hyp_rodata_end), PAGE_HYP_RO);
2609 if (err) {
2610 kvm_err("Cannot map .hyp.rodata section\n");
2611 goto out_err;
2612 }
2613
2614 err = create_hyp_mappings(kvm_ksym_ref(__start_rodata),
2615 kvm_ksym_ref(__end_rodata), PAGE_HYP_RO);
2616 if (err) {
2617 kvm_err("Cannot map rodata section\n");
2618 goto out_err;
2619 }
2620
2621 /*
2622 * .hyp.bss is guaranteed to be placed at the beginning of the .bss
2623 * section thanks to an assertion in the linker script. Map it RW and
2624 * the rest of .bss RO.
2625 */
2626 err = create_hyp_mappings(kvm_ksym_ref(__hyp_bss_start),
2627 kvm_ksym_ref(__hyp_bss_end), PAGE_HYP);
2628 if (err) {
2629 kvm_err("Cannot map hyp bss section: %d\n", err);
2630 goto out_err;
2631 }
2632
2633 err = create_hyp_mappings(kvm_ksym_ref(__hyp_bss_end),
2634 kvm_ksym_ref(__bss_stop), PAGE_HYP_RO);
2635 if (err) {
2636 kvm_err("Cannot map bss section\n");
2637 goto out_err;
2638 }
2639
2640 /*
2641 * Map the Hyp stack pages
2642 */
2643 for_each_possible_cpu(cpu) {
2644 struct kvm_nvhe_init_params *params = per_cpu_ptr_nvhe_sym(kvm_init_params, cpu);
2645 char *stack_base = (char *)per_cpu(kvm_arm_hyp_stack_base, cpu);
2646
2647 err = create_hyp_stack(__pa(stack_base), ¶ms->stack_hyp_va);
2648 if (err) {
2649 kvm_err("Cannot map hyp stack\n");
2650 goto out_err;
2651 }
2652
2653 /*
2654 * Save the stack PA in nvhe_init_params. This will be needed
2655 * to recreate the stack mapping in protected nVHE mode.
2656 * __hyp_pa() won't do the right thing there, since the stack
2657 * has been mapped in the flexible private VA space.
2658 */
2659 params->stack_pa = __pa(stack_base);
2660 }
2661
2662 for_each_possible_cpu(cpu) {
2663 char *percpu_begin = (char *)kvm_nvhe_sym(kvm_arm_hyp_percpu_base)[cpu];
2664 char *percpu_end = percpu_begin + nvhe_percpu_size();
2665
2666 /* Map Hyp percpu pages */
2667 err = create_hyp_mappings(percpu_begin, percpu_end, PAGE_HYP);
2668 if (err) {
2669 kvm_err("Cannot map hyp percpu region\n");
2670 goto out_err;
2671 }
2672
2673 /* Prepare the CPU initialization parameters */
2674 cpu_prepare_hyp_mode(cpu, hyp_va_bits);
2675 }
2676
2677 kvm_hyp_init_symbols();
2678
2679 if (is_protected_kvm_enabled()) {
2680 if (IS_ENABLED(CONFIG_ARM64_PTR_AUTH_KERNEL) &&
2681 cpus_have_final_cap(ARM64_HAS_ADDRESS_AUTH))
2682 pkvm_hyp_init_ptrauth();
2683
2684 init_cpu_logical_map();
2685
2686 if (!init_psci_relay()) {
2687 err = -ENODEV;
2688 goto out_err;
2689 }
2690
2691 err = init_pkvm_host_sve_state();
2692 if (err)
2693 goto out_err;
2694
2695 err = kvm_hyp_init_protection(hyp_va_bits);
2696 if (err) {
2697 kvm_err("Failed to init hyp memory protection\n");
2698 goto out_err;
2699 }
2700 }
2701
2702 return 0;
2703
2704 out_err:
2705 teardown_hyp_mode();
2706 kvm_err("error initializing Hyp mode: %d\n", err);
2707 return err;
2708 }
2709
kvm_mpidr_to_vcpu(struct kvm * kvm,unsigned long mpidr)2710 struct kvm_vcpu *kvm_mpidr_to_vcpu(struct kvm *kvm, unsigned long mpidr)
2711 {
2712 struct kvm_vcpu *vcpu = NULL;
2713 struct kvm_mpidr_data *data;
2714 unsigned long i;
2715
2716 mpidr &= MPIDR_HWID_BITMASK;
2717
2718 rcu_read_lock();
2719 data = rcu_dereference(kvm->arch.mpidr_data);
2720
2721 if (data) {
2722 u16 idx = kvm_mpidr_index(data, mpidr);
2723
2724 vcpu = kvm_get_vcpu(kvm, data->cmpidr_to_idx[idx]);
2725 if (mpidr != kvm_vcpu_get_mpidr_aff(vcpu))
2726 vcpu = NULL;
2727 }
2728
2729 rcu_read_unlock();
2730
2731 if (vcpu)
2732 return vcpu;
2733
2734 kvm_for_each_vcpu(i, vcpu, kvm) {
2735 if (mpidr == kvm_vcpu_get_mpidr_aff(vcpu))
2736 return vcpu;
2737 }
2738 return NULL;
2739 }
2740
kvm_arch_irqchip_in_kernel(struct kvm * kvm)2741 bool kvm_arch_irqchip_in_kernel(struct kvm *kvm)
2742 {
2743 return irqchip_in_kernel(kvm);
2744 }
2745
kvm_arch_irq_bypass_add_producer(struct irq_bypass_consumer * cons,struct irq_bypass_producer * prod)2746 int kvm_arch_irq_bypass_add_producer(struct irq_bypass_consumer *cons,
2747 struct irq_bypass_producer *prod)
2748 {
2749 struct kvm_kernel_irqfd *irqfd =
2750 container_of(cons, struct kvm_kernel_irqfd, consumer);
2751 struct kvm_kernel_irq_routing_entry *irq_entry = &irqfd->irq_entry;
2752
2753 /*
2754 * The only thing we have a chance of directly-injecting is LPIs. Maybe
2755 * one day...
2756 */
2757 if (irq_entry->type != KVM_IRQ_ROUTING_MSI)
2758 return 0;
2759
2760 return kvm_vgic_v4_set_forwarding(irqfd->kvm, prod->irq,
2761 &irqfd->irq_entry);
2762 }
kvm_arch_irq_bypass_del_producer(struct irq_bypass_consumer * cons,struct irq_bypass_producer * prod)2763 void kvm_arch_irq_bypass_del_producer(struct irq_bypass_consumer *cons,
2764 struct irq_bypass_producer *prod)
2765 {
2766 struct kvm_kernel_irqfd *irqfd =
2767 container_of(cons, struct kvm_kernel_irqfd, consumer);
2768 struct kvm_kernel_irq_routing_entry *irq_entry = &irqfd->irq_entry;
2769
2770 if (irq_entry->type != KVM_IRQ_ROUTING_MSI)
2771 return;
2772
2773 kvm_vgic_v4_unset_forwarding(irqfd->kvm, prod->irq,
2774 &irqfd->irq_entry);
2775 }
2776
kvm_arch_irq_bypass_stop(struct irq_bypass_consumer * cons)2777 void kvm_arch_irq_bypass_stop(struct irq_bypass_consumer *cons)
2778 {
2779 struct kvm_kernel_irqfd *irqfd =
2780 container_of(cons, struct kvm_kernel_irqfd, consumer);
2781
2782 kvm_arm_halt_guest(irqfd->kvm);
2783 }
2784
kvm_arch_irq_bypass_start(struct irq_bypass_consumer * cons)2785 void kvm_arch_irq_bypass_start(struct irq_bypass_consumer *cons)
2786 {
2787 struct kvm_kernel_irqfd *irqfd =
2788 container_of(cons, struct kvm_kernel_irqfd, consumer);
2789
2790 kvm_arm_resume_guest(irqfd->kvm);
2791 }
2792
2793 /* Initialize Hyp-mode and memory mappings on all CPUs */
kvm_arm_init(void)2794 static __init int kvm_arm_init(void)
2795 {
2796 int err;
2797 bool in_hyp_mode;
2798
2799 if (!is_hyp_mode_available()) {
2800 kvm_info("HYP mode not available\n");
2801 return -ENODEV;
2802 }
2803
2804 if (kvm_get_mode() == KVM_MODE_NONE) {
2805 kvm_info("KVM disabled from command line\n");
2806 return -ENODEV;
2807 }
2808
2809 err = kvm_sys_reg_table_init();
2810 if (err) {
2811 kvm_info("Error initializing system register tables");
2812 return err;
2813 }
2814
2815 in_hyp_mode = is_kernel_in_hyp_mode();
2816
2817 if (cpus_have_final_cap(ARM64_WORKAROUND_DEVICE_LOAD_ACQUIRE) ||
2818 cpus_have_final_cap(ARM64_WORKAROUND_1508412))
2819 kvm_info("Guests without required CPU erratum workarounds can deadlock system!\n" \
2820 "Only trusted guests should be used on this system.\n");
2821
2822 err = kvm_set_ipa_limit();
2823 if (err)
2824 return err;
2825
2826 err = kvm_arm_init_sve();
2827 if (err)
2828 return err;
2829
2830 err = kvm_arm_vmid_alloc_init();
2831 if (err) {
2832 kvm_err("Failed to initialize VMID allocator.\n");
2833 return err;
2834 }
2835
2836 if (!in_hyp_mode) {
2837 err = init_hyp_mode();
2838 if (err)
2839 goto out_err;
2840 }
2841
2842 err = kvm_init_vector_slots();
2843 if (err) {
2844 kvm_err("Cannot initialise vector slots\n");
2845 goto out_hyp;
2846 }
2847
2848 err = init_subsystems();
2849 if (err)
2850 goto out_hyp;
2851
2852 kvm_info("%s%sVHE%s mode initialized successfully\n",
2853 in_hyp_mode ? "" : (is_protected_kvm_enabled() ?
2854 "Protected " : "Hyp "),
2855 in_hyp_mode ? "" : (cpus_have_final_cap(ARM64_KVM_HVHE) ?
2856 "h" : "n"),
2857 cpus_have_final_cap(ARM64_HAS_NESTED_VIRT) ? "+NV2": "");
2858
2859 /*
2860 * FIXME: Do something reasonable if kvm_init() fails after pKVM
2861 * hypervisor protection is finalized.
2862 */
2863 err = kvm_init(sizeof(struct kvm_vcpu), 0, THIS_MODULE);
2864 if (err)
2865 goto out_subs;
2866
2867 /*
2868 * This should be called after initialization is done and failure isn't
2869 * possible anymore.
2870 */
2871 if (!in_hyp_mode)
2872 finalize_init_hyp_mode();
2873
2874 kvm_arm_initialised = true;
2875
2876 return 0;
2877
2878 out_subs:
2879 teardown_subsystems();
2880 out_hyp:
2881 if (!in_hyp_mode)
2882 teardown_hyp_mode();
2883 out_err:
2884 kvm_arm_vmid_alloc_free();
2885 return err;
2886 }
2887
early_kvm_mode_cfg(char * arg)2888 static int __init early_kvm_mode_cfg(char *arg)
2889 {
2890 if (!arg)
2891 return -EINVAL;
2892
2893 if (strcmp(arg, "none") == 0) {
2894 kvm_mode = KVM_MODE_NONE;
2895 return 0;
2896 }
2897
2898 if (!is_hyp_mode_available()) {
2899 pr_warn_once("KVM is not available. Ignoring kvm-arm.mode\n");
2900 return 0;
2901 }
2902
2903 if (strcmp(arg, "protected") == 0) {
2904 if (!is_kernel_in_hyp_mode())
2905 kvm_mode = KVM_MODE_PROTECTED;
2906 else
2907 pr_warn_once("Protected KVM not available with VHE\n");
2908
2909 return 0;
2910 }
2911
2912 if (strcmp(arg, "nvhe") == 0 && !WARN_ON(is_kernel_in_hyp_mode())) {
2913 kvm_mode = KVM_MODE_DEFAULT;
2914 return 0;
2915 }
2916
2917 if (strcmp(arg, "nested") == 0 && !WARN_ON(!is_kernel_in_hyp_mode())) {
2918 kvm_mode = KVM_MODE_NV;
2919 return 0;
2920 }
2921
2922 return -EINVAL;
2923 }
2924 early_param("kvm-arm.mode", early_kvm_mode_cfg);
2925
early_kvm_wfx_trap_policy_cfg(char * arg,enum kvm_wfx_trap_policy * p)2926 static int __init early_kvm_wfx_trap_policy_cfg(char *arg, enum kvm_wfx_trap_policy *p)
2927 {
2928 if (!arg)
2929 return -EINVAL;
2930
2931 if (strcmp(arg, "trap") == 0) {
2932 *p = KVM_WFX_TRAP;
2933 return 0;
2934 }
2935
2936 if (strcmp(arg, "notrap") == 0) {
2937 *p = KVM_WFX_NOTRAP;
2938 return 0;
2939 }
2940
2941 return -EINVAL;
2942 }
2943
early_kvm_wfi_trap_policy_cfg(char * arg)2944 static int __init early_kvm_wfi_trap_policy_cfg(char *arg)
2945 {
2946 return early_kvm_wfx_trap_policy_cfg(arg, &kvm_wfi_trap_policy);
2947 }
2948 early_param("kvm-arm.wfi_trap_policy", early_kvm_wfi_trap_policy_cfg);
2949
early_kvm_wfe_trap_policy_cfg(char * arg)2950 static int __init early_kvm_wfe_trap_policy_cfg(char *arg)
2951 {
2952 return early_kvm_wfx_trap_policy_cfg(arg, &kvm_wfe_trap_policy);
2953 }
2954 early_param("kvm-arm.wfe_trap_policy", early_kvm_wfe_trap_policy_cfg);
2955
kvm_get_mode(void)2956 enum kvm_mode kvm_get_mode(void)
2957 {
2958 return kvm_mode;
2959 }
2960
2961 module_init(kvm_arm_init);
2962