1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Copyright (C) 2012 - Virtual Open Systems and Columbia University
4  * Author: Christoffer Dall <c.dall@virtualopensystems.com>
5  */
6 
7 #include <linux/bug.h>
8 #include <linux/cpu_pm.h>
9 #include <linux/entry-kvm.h>
10 #include <linux/errno.h>
11 #include <linux/err.h>
12 #include <linux/kvm_host.h>
13 #include <linux/list.h>
14 #include <linux/module.h>
15 #include <linux/vmalloc.h>
16 #include <linux/fs.h>
17 #include <linux/mman.h>
18 #include <linux/sched.h>
19 #include <linux/kvm.h>
20 #include <linux/kvm_irqfd.h>
21 #include <linux/irqbypass.h>
22 #include <linux/sched/stat.h>
23 #include <linux/psci.h>
24 #include <trace/events/kvm.h>
25 
26 #define CREATE_TRACE_POINTS
27 #include "trace_arm.h"
28 
29 #include <linux/uaccess.h>
30 #include <asm/ptrace.h>
31 #include <asm/mman.h>
32 #include <asm/tlbflush.h>
33 #include <asm/cacheflush.h>
34 #include <asm/cpufeature.h>
35 #include <asm/virt.h>
36 #include <asm/kvm_arm.h>
37 #include <asm/kvm_asm.h>
38 #include <asm/kvm_emulate.h>
39 #include <asm/kvm_mmu.h>
40 #include <asm/kvm_nested.h>
41 #include <asm/kvm_pkvm.h>
42 #include <asm/kvm_ptrauth.h>
43 #include <asm/sections.h>
44 
45 #include <kvm/arm_hypercalls.h>
46 #include <kvm/arm_pmu.h>
47 #include <kvm/arm_psci.h>
48 
49 #include "sys_regs.h"
50 
51 static enum kvm_mode kvm_mode = KVM_MODE_DEFAULT;
52 
53 enum kvm_wfx_trap_policy {
54 	KVM_WFX_NOTRAP_SINGLE_TASK, /* Default option */
55 	KVM_WFX_NOTRAP,
56 	KVM_WFX_TRAP,
57 };
58 
59 static enum kvm_wfx_trap_policy kvm_wfi_trap_policy __read_mostly = KVM_WFX_NOTRAP_SINGLE_TASK;
60 static enum kvm_wfx_trap_policy kvm_wfe_trap_policy __read_mostly = KVM_WFX_NOTRAP_SINGLE_TASK;
61 
62 DECLARE_KVM_HYP_PER_CPU(unsigned long, kvm_hyp_vector);
63 
64 DEFINE_PER_CPU(unsigned long, kvm_arm_hyp_stack_base);
65 DECLARE_KVM_NVHE_PER_CPU(struct kvm_nvhe_init_params, kvm_init_params);
66 
67 DECLARE_KVM_NVHE_PER_CPU(struct kvm_cpu_context, kvm_hyp_ctxt);
68 
69 static bool vgic_present, kvm_arm_initialised;
70 
71 static DEFINE_PER_CPU(unsigned char, kvm_hyp_initialized);
72 
is_kvm_arm_initialised(void)73 bool is_kvm_arm_initialised(void)
74 {
75 	return kvm_arm_initialised;
76 }
77 
kvm_arch_vcpu_should_kick(struct kvm_vcpu * vcpu)78 int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)
79 {
80 	return kvm_vcpu_exiting_guest_mode(vcpu) == IN_GUEST_MODE;
81 }
82 
kvm_vm_ioctl_enable_cap(struct kvm * kvm,struct kvm_enable_cap * cap)83 int kvm_vm_ioctl_enable_cap(struct kvm *kvm,
84 			    struct kvm_enable_cap *cap)
85 {
86 	int r = -EINVAL;
87 
88 	if (cap->flags)
89 		return -EINVAL;
90 
91 	if (kvm_vm_is_protected(kvm) && !kvm_pvm_ext_allowed(cap->cap))
92 		return -EINVAL;
93 
94 	switch (cap->cap) {
95 	case KVM_CAP_ARM_NISV_TO_USER:
96 		r = 0;
97 		set_bit(KVM_ARCH_FLAG_RETURN_NISV_IO_ABORT_TO_USER,
98 			&kvm->arch.flags);
99 		break;
100 	case KVM_CAP_ARM_MTE:
101 		mutex_lock(&kvm->lock);
102 		if (system_supports_mte() && !kvm->created_vcpus) {
103 			r = 0;
104 			set_bit(KVM_ARCH_FLAG_MTE_ENABLED, &kvm->arch.flags);
105 		}
106 		mutex_unlock(&kvm->lock);
107 		break;
108 	case KVM_CAP_ARM_SYSTEM_SUSPEND:
109 		r = 0;
110 		set_bit(KVM_ARCH_FLAG_SYSTEM_SUSPEND_ENABLED, &kvm->arch.flags);
111 		break;
112 	case KVM_CAP_ARM_EAGER_SPLIT_CHUNK_SIZE:
113 		mutex_lock(&kvm->slots_lock);
114 		/*
115 		 * To keep things simple, allow changing the chunk
116 		 * size only when no memory slots have been created.
117 		 */
118 		if (kvm_are_all_memslots_empty(kvm)) {
119 			u64 new_cap = cap->args[0];
120 
121 			if (!new_cap || kvm_is_block_size_supported(new_cap)) {
122 				r = 0;
123 				kvm->arch.mmu.split_page_chunk_size = new_cap;
124 			}
125 		}
126 		mutex_unlock(&kvm->slots_lock);
127 		break;
128 	case KVM_CAP_ARM_WRITABLE_IMP_ID_REGS:
129 		mutex_lock(&kvm->lock);
130 		if (!kvm->created_vcpus) {
131 			r = 0;
132 			set_bit(KVM_ARCH_FLAG_WRITABLE_IMP_ID_REGS, &kvm->arch.flags);
133 		}
134 		mutex_unlock(&kvm->lock);
135 		break;
136 	default:
137 		break;
138 	}
139 
140 	return r;
141 }
142 
kvm_arm_default_max_vcpus(void)143 static int kvm_arm_default_max_vcpus(void)
144 {
145 	return vgic_present ? kvm_vgic_get_max_vcpus() : KVM_MAX_VCPUS;
146 }
147 
148 /**
149  * kvm_arch_init_vm - initializes a VM data structure
150  * @kvm:	pointer to the KVM struct
151  * @type:	kvm device type
152  */
kvm_arch_init_vm(struct kvm * kvm,unsigned long type)153 int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
154 {
155 	int ret;
156 
157 	mutex_init(&kvm->arch.config_lock);
158 
159 #ifdef CONFIG_LOCKDEP
160 	/* Clue in lockdep that the config_lock must be taken inside kvm->lock */
161 	mutex_lock(&kvm->lock);
162 	mutex_lock(&kvm->arch.config_lock);
163 	mutex_unlock(&kvm->arch.config_lock);
164 	mutex_unlock(&kvm->lock);
165 #endif
166 
167 	kvm_init_nested(kvm);
168 
169 	ret = kvm_share_hyp(kvm, kvm + 1);
170 	if (ret)
171 		return ret;
172 
173 	ret = pkvm_init_host_vm(kvm);
174 	if (ret)
175 		goto err_unshare_kvm;
176 
177 	if (!zalloc_cpumask_var(&kvm->arch.supported_cpus, GFP_KERNEL_ACCOUNT)) {
178 		ret = -ENOMEM;
179 		goto err_unshare_kvm;
180 	}
181 	cpumask_copy(kvm->arch.supported_cpus, cpu_possible_mask);
182 
183 	ret = kvm_init_stage2_mmu(kvm, &kvm->arch.mmu, type);
184 	if (ret)
185 		goto err_free_cpumask;
186 
187 	kvm_vgic_early_init(kvm);
188 
189 	kvm_timer_init_vm(kvm);
190 
191 	/* The maximum number of VCPUs is limited by the host's GIC model */
192 	kvm->max_vcpus = kvm_arm_default_max_vcpus();
193 
194 	kvm_arm_init_hypercalls(kvm);
195 
196 	bitmap_zero(kvm->arch.vcpu_features, KVM_VCPU_MAX_FEATURES);
197 
198 	return 0;
199 
200 err_free_cpumask:
201 	free_cpumask_var(kvm->arch.supported_cpus);
202 err_unshare_kvm:
203 	kvm_unshare_hyp(kvm, kvm + 1);
204 	return ret;
205 }
206 
kvm_arch_vcpu_fault(struct kvm_vcpu * vcpu,struct vm_fault * vmf)207 vm_fault_t kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf)
208 {
209 	return VM_FAULT_SIGBUS;
210 }
211 
kvm_arch_create_vm_debugfs(struct kvm * kvm)212 void kvm_arch_create_vm_debugfs(struct kvm *kvm)
213 {
214 	kvm_sys_regs_create_debugfs(kvm);
215 	kvm_s2_ptdump_create_debugfs(kvm);
216 }
217 
kvm_destroy_mpidr_data(struct kvm * kvm)218 static void kvm_destroy_mpidr_data(struct kvm *kvm)
219 {
220 	struct kvm_mpidr_data *data;
221 
222 	mutex_lock(&kvm->arch.config_lock);
223 
224 	data = rcu_dereference_protected(kvm->arch.mpidr_data,
225 					 lockdep_is_held(&kvm->arch.config_lock));
226 	if (data) {
227 		rcu_assign_pointer(kvm->arch.mpidr_data, NULL);
228 		synchronize_rcu();
229 		kfree(data);
230 	}
231 
232 	mutex_unlock(&kvm->arch.config_lock);
233 }
234 
235 /**
236  * kvm_arch_destroy_vm - destroy the VM data structure
237  * @kvm:	pointer to the KVM struct
238  */
kvm_arch_destroy_vm(struct kvm * kvm)239 void kvm_arch_destroy_vm(struct kvm *kvm)
240 {
241 	bitmap_free(kvm->arch.pmu_filter);
242 	free_cpumask_var(kvm->arch.supported_cpus);
243 
244 	kvm_vgic_destroy(kvm);
245 
246 	if (is_protected_kvm_enabled())
247 		pkvm_destroy_hyp_vm(kvm);
248 
249 	kvm_destroy_mpidr_data(kvm);
250 
251 	kfree(kvm->arch.sysreg_masks);
252 	kvm_destroy_vcpus(kvm);
253 
254 	kvm_unshare_hyp(kvm, kvm + 1);
255 
256 	kvm_arm_teardown_hypercalls(kvm);
257 }
258 
kvm_has_full_ptr_auth(void)259 static bool kvm_has_full_ptr_auth(void)
260 {
261 	bool apa, gpa, api, gpi, apa3, gpa3;
262 	u64 isar1, isar2, val;
263 
264 	/*
265 	 * Check that:
266 	 *
267 	 * - both Address and Generic auth are implemented for a given
268          *   algorithm (Q5, IMPDEF or Q3)
269 	 * - only a single algorithm is implemented.
270 	 */
271 	if (!system_has_full_ptr_auth())
272 		return false;
273 
274 	isar1 = read_sanitised_ftr_reg(SYS_ID_AA64ISAR1_EL1);
275 	isar2 = read_sanitised_ftr_reg(SYS_ID_AA64ISAR2_EL1);
276 
277 	apa = !!FIELD_GET(ID_AA64ISAR1_EL1_APA_MASK, isar1);
278 	val = FIELD_GET(ID_AA64ISAR1_EL1_GPA_MASK, isar1);
279 	gpa = (val == ID_AA64ISAR1_EL1_GPA_IMP);
280 
281 	api = !!FIELD_GET(ID_AA64ISAR1_EL1_API_MASK, isar1);
282 	val = FIELD_GET(ID_AA64ISAR1_EL1_GPI_MASK, isar1);
283 	gpi = (val == ID_AA64ISAR1_EL1_GPI_IMP);
284 
285 	apa3 = !!FIELD_GET(ID_AA64ISAR2_EL1_APA3_MASK, isar2);
286 	val  = FIELD_GET(ID_AA64ISAR2_EL1_GPA3_MASK, isar2);
287 	gpa3 = (val == ID_AA64ISAR2_EL1_GPA3_IMP);
288 
289 	return (apa == gpa && api == gpi && apa3 == gpa3 &&
290 		(apa + api + apa3) == 1);
291 }
292 
kvm_vm_ioctl_check_extension(struct kvm * kvm,long ext)293 int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
294 {
295 	int r;
296 
297 	if (kvm && kvm_vm_is_protected(kvm) && !kvm_pvm_ext_allowed(ext))
298 		return 0;
299 
300 	switch (ext) {
301 	case KVM_CAP_IRQCHIP:
302 		r = vgic_present;
303 		break;
304 	case KVM_CAP_IOEVENTFD:
305 	case KVM_CAP_USER_MEMORY:
306 	case KVM_CAP_SYNC_MMU:
307 	case KVM_CAP_DESTROY_MEMORY_REGION_WORKS:
308 	case KVM_CAP_ONE_REG:
309 	case KVM_CAP_ARM_PSCI:
310 	case KVM_CAP_ARM_PSCI_0_2:
311 	case KVM_CAP_READONLY_MEM:
312 	case KVM_CAP_MP_STATE:
313 	case KVM_CAP_IMMEDIATE_EXIT:
314 	case KVM_CAP_VCPU_EVENTS:
315 	case KVM_CAP_ARM_IRQ_LINE_LAYOUT_2:
316 	case KVM_CAP_ARM_NISV_TO_USER:
317 	case KVM_CAP_ARM_INJECT_EXT_DABT:
318 	case KVM_CAP_SET_GUEST_DEBUG:
319 	case KVM_CAP_VCPU_ATTRIBUTES:
320 	case KVM_CAP_PTP_KVM:
321 	case KVM_CAP_ARM_SYSTEM_SUSPEND:
322 	case KVM_CAP_IRQFD_RESAMPLE:
323 	case KVM_CAP_COUNTER_OFFSET:
324 	case KVM_CAP_ARM_WRITABLE_IMP_ID_REGS:
325 		r = 1;
326 		break;
327 	case KVM_CAP_SET_GUEST_DEBUG2:
328 		return KVM_GUESTDBG_VALID_MASK;
329 	case KVM_CAP_ARM_SET_DEVICE_ADDR:
330 		r = 1;
331 		break;
332 	case KVM_CAP_NR_VCPUS:
333 		/*
334 		 * ARM64 treats KVM_CAP_NR_CPUS differently from all other
335 		 * architectures, as it does not always bound it to
336 		 * KVM_CAP_MAX_VCPUS. It should not matter much because
337 		 * this is just an advisory value.
338 		 */
339 		r = min_t(unsigned int, num_online_cpus(),
340 			  kvm_arm_default_max_vcpus());
341 		break;
342 	case KVM_CAP_MAX_VCPUS:
343 	case KVM_CAP_MAX_VCPU_ID:
344 		if (kvm)
345 			r = kvm->max_vcpus;
346 		else
347 			r = kvm_arm_default_max_vcpus();
348 		break;
349 	case KVM_CAP_MSI_DEVID:
350 		if (!kvm)
351 			r = -EINVAL;
352 		else
353 			r = kvm->arch.vgic.msis_require_devid;
354 		break;
355 	case KVM_CAP_ARM_USER_IRQ:
356 		/*
357 		 * 1: EL1_VTIMER, EL1_PTIMER, and PMU.
358 		 * (bump this number if adding more devices)
359 		 */
360 		r = 1;
361 		break;
362 	case KVM_CAP_ARM_MTE:
363 		r = system_supports_mte();
364 		break;
365 	case KVM_CAP_STEAL_TIME:
366 		r = kvm_arm_pvtime_supported();
367 		break;
368 	case KVM_CAP_ARM_EL1_32BIT:
369 		r = cpus_have_final_cap(ARM64_HAS_32BIT_EL1);
370 		break;
371 	case KVM_CAP_GUEST_DEBUG_HW_BPS:
372 		r = get_num_brps();
373 		break;
374 	case KVM_CAP_GUEST_DEBUG_HW_WPS:
375 		r = get_num_wrps();
376 		break;
377 	case KVM_CAP_ARM_PMU_V3:
378 		r = kvm_supports_guest_pmuv3();
379 		break;
380 	case KVM_CAP_ARM_INJECT_SERROR_ESR:
381 		r = cpus_have_final_cap(ARM64_HAS_RAS_EXTN);
382 		break;
383 	case KVM_CAP_ARM_VM_IPA_SIZE:
384 		r = get_kvm_ipa_limit();
385 		break;
386 	case KVM_CAP_ARM_SVE:
387 		r = system_supports_sve();
388 		break;
389 	case KVM_CAP_ARM_PTRAUTH_ADDRESS:
390 	case KVM_CAP_ARM_PTRAUTH_GENERIC:
391 		r = kvm_has_full_ptr_auth();
392 		break;
393 	case KVM_CAP_ARM_EAGER_SPLIT_CHUNK_SIZE:
394 		if (kvm)
395 			r = kvm->arch.mmu.split_page_chunk_size;
396 		else
397 			r = KVM_ARM_EAGER_SPLIT_CHUNK_SIZE_DEFAULT;
398 		break;
399 	case KVM_CAP_ARM_SUPPORTED_BLOCK_SIZES:
400 		r = kvm_supported_block_sizes();
401 		break;
402 	case KVM_CAP_ARM_SUPPORTED_REG_MASK_RANGES:
403 		r = BIT(0);
404 		break;
405 	default:
406 		r = 0;
407 	}
408 
409 	return r;
410 }
411 
kvm_arch_dev_ioctl(struct file * filp,unsigned int ioctl,unsigned long arg)412 long kvm_arch_dev_ioctl(struct file *filp,
413 			unsigned int ioctl, unsigned long arg)
414 {
415 	return -EINVAL;
416 }
417 
kvm_arch_alloc_vm(void)418 struct kvm *kvm_arch_alloc_vm(void)
419 {
420 	size_t sz = sizeof(struct kvm);
421 
422 	if (!has_vhe())
423 		return kzalloc(sz, GFP_KERNEL_ACCOUNT);
424 
425 	return __vmalloc(sz, GFP_KERNEL_ACCOUNT | __GFP_HIGHMEM | __GFP_ZERO);
426 }
427 
kvm_arch_vcpu_precreate(struct kvm * kvm,unsigned int id)428 int kvm_arch_vcpu_precreate(struct kvm *kvm, unsigned int id)
429 {
430 	if (irqchip_in_kernel(kvm) && vgic_initialized(kvm))
431 		return -EBUSY;
432 
433 	if (id >= kvm->max_vcpus)
434 		return -EINVAL;
435 
436 	return 0;
437 }
438 
kvm_arch_vcpu_create(struct kvm_vcpu * vcpu)439 int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu)
440 {
441 	int err;
442 
443 	spin_lock_init(&vcpu->arch.mp_state_lock);
444 
445 #ifdef CONFIG_LOCKDEP
446 	/* Inform lockdep that the config_lock is acquired after vcpu->mutex */
447 	mutex_lock(&vcpu->mutex);
448 	mutex_lock(&vcpu->kvm->arch.config_lock);
449 	mutex_unlock(&vcpu->kvm->arch.config_lock);
450 	mutex_unlock(&vcpu->mutex);
451 #endif
452 
453 	/* Force users to call KVM_ARM_VCPU_INIT */
454 	vcpu_clear_flag(vcpu, VCPU_INITIALIZED);
455 
456 	vcpu->arch.mmu_page_cache.gfp_zero = __GFP_ZERO;
457 
458 	/* Set up the timer */
459 	kvm_timer_vcpu_init(vcpu);
460 
461 	kvm_pmu_vcpu_init(vcpu);
462 
463 	kvm_arm_pvtime_vcpu_init(&vcpu->arch);
464 
465 	vcpu->arch.hw_mmu = &vcpu->kvm->arch.mmu;
466 
467 	/*
468 	 * This vCPU may have been created after mpidr_data was initialized.
469 	 * Throw out the pre-computed mappings if that is the case which forces
470 	 * KVM to fall back to iteratively searching the vCPUs.
471 	 */
472 	kvm_destroy_mpidr_data(vcpu->kvm);
473 
474 	err = kvm_vgic_vcpu_init(vcpu);
475 	if (err)
476 		return err;
477 
478 	err = kvm_share_hyp(vcpu, vcpu + 1);
479 	if (err)
480 		kvm_vgic_vcpu_destroy(vcpu);
481 
482 	return err;
483 }
484 
kvm_arch_vcpu_postcreate(struct kvm_vcpu * vcpu)485 void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
486 {
487 }
488 
kvm_arch_vcpu_destroy(struct kvm_vcpu * vcpu)489 void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
490 {
491 	if (!is_protected_kvm_enabled())
492 		kvm_mmu_free_memory_cache(&vcpu->arch.mmu_page_cache);
493 	else
494 		free_hyp_memcache(&vcpu->arch.pkvm_memcache);
495 	kvm_timer_vcpu_terminate(vcpu);
496 	kvm_pmu_vcpu_destroy(vcpu);
497 	kvm_vgic_vcpu_destroy(vcpu);
498 	kvm_arm_vcpu_destroy(vcpu);
499 }
500 
kvm_arch_vcpu_blocking(struct kvm_vcpu * vcpu)501 void kvm_arch_vcpu_blocking(struct kvm_vcpu *vcpu)
502 {
503 
504 }
505 
kvm_arch_vcpu_unblocking(struct kvm_vcpu * vcpu)506 void kvm_arch_vcpu_unblocking(struct kvm_vcpu *vcpu)
507 {
508 
509 }
510 
vcpu_set_pauth_traps(struct kvm_vcpu * vcpu)511 static void vcpu_set_pauth_traps(struct kvm_vcpu *vcpu)
512 {
513 	if (vcpu_has_ptrauth(vcpu) && !is_protected_kvm_enabled()) {
514 		/*
515 		 * Either we're running an L2 guest, and the API/APK bits come
516 		 * from L1's HCR_EL2, or API/APK are both set.
517 		 */
518 		if (unlikely(vcpu_has_nv(vcpu) && !is_hyp_ctxt(vcpu))) {
519 			u64 val;
520 
521 			val = __vcpu_sys_reg(vcpu, HCR_EL2);
522 			val &= (HCR_API | HCR_APK);
523 			vcpu->arch.hcr_el2 &= ~(HCR_API | HCR_APK);
524 			vcpu->arch.hcr_el2 |= val;
525 		} else {
526 			vcpu->arch.hcr_el2 |= (HCR_API | HCR_APK);
527 		}
528 
529 		/*
530 		 * Save the host keys if there is any chance for the guest
531 		 * to use pauth, as the entry code will reload the guest
532 		 * keys in that case.
533 		 */
534 		if (vcpu->arch.hcr_el2 & (HCR_API | HCR_APK)) {
535 			struct kvm_cpu_context *ctxt;
536 
537 			ctxt = this_cpu_ptr_hyp_sym(kvm_hyp_ctxt);
538 			ptrauth_save_keys(ctxt);
539 		}
540 	}
541 }
542 
kvm_vcpu_should_clear_twi(struct kvm_vcpu * vcpu)543 static bool kvm_vcpu_should_clear_twi(struct kvm_vcpu *vcpu)
544 {
545 	if (unlikely(kvm_wfi_trap_policy != KVM_WFX_NOTRAP_SINGLE_TASK))
546 		return kvm_wfi_trap_policy == KVM_WFX_NOTRAP;
547 
548 	return single_task_running() &&
549 	       (atomic_read(&vcpu->arch.vgic_cpu.vgic_v3.its_vpe.vlpi_count) ||
550 		vcpu->kvm->arch.vgic.nassgireq);
551 }
552 
kvm_vcpu_should_clear_twe(struct kvm_vcpu * vcpu)553 static bool kvm_vcpu_should_clear_twe(struct kvm_vcpu *vcpu)
554 {
555 	if (unlikely(kvm_wfe_trap_policy != KVM_WFX_NOTRAP_SINGLE_TASK))
556 		return kvm_wfe_trap_policy == KVM_WFX_NOTRAP;
557 
558 	return single_task_running();
559 }
560 
kvm_arch_vcpu_load(struct kvm_vcpu * vcpu,int cpu)561 void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
562 {
563 	struct kvm_s2_mmu *mmu;
564 	int *last_ran;
565 
566 	if (is_protected_kvm_enabled())
567 		goto nommu;
568 
569 	if (vcpu_has_nv(vcpu))
570 		kvm_vcpu_load_hw_mmu(vcpu);
571 
572 	mmu = vcpu->arch.hw_mmu;
573 	last_ran = this_cpu_ptr(mmu->last_vcpu_ran);
574 
575 	/*
576 	 * Ensure a VMID is allocated for the MMU before programming VTTBR_EL2,
577 	 * which happens eagerly in VHE.
578 	 *
579 	 * Also, the VMID allocator only preserves VMIDs that are active at the
580 	 * time of rollover, so KVM might need to grab a new VMID for the MMU if
581 	 * this is called from kvm_sched_in().
582 	 */
583 	kvm_arm_vmid_update(&mmu->vmid);
584 
585 	/*
586 	 * We guarantee that both TLBs and I-cache are private to each
587 	 * vcpu. If detecting that a vcpu from the same VM has
588 	 * previously run on the same physical CPU, call into the
589 	 * hypervisor code to nuke the relevant contexts.
590 	 *
591 	 * We might get preempted before the vCPU actually runs, but
592 	 * over-invalidation doesn't affect correctness.
593 	 */
594 	if (*last_ran != vcpu->vcpu_idx) {
595 		kvm_call_hyp(__kvm_flush_cpu_context, mmu);
596 		*last_ran = vcpu->vcpu_idx;
597 	}
598 
599 nommu:
600 	vcpu->cpu = cpu;
601 
602 	/*
603 	 * The timer must be loaded before the vgic to correctly set up physical
604 	 * interrupt deactivation in nested state (e.g. timer interrupt).
605 	 */
606 	kvm_timer_vcpu_load(vcpu);
607 	kvm_vgic_load(vcpu);
608 	kvm_vcpu_load_debug(vcpu);
609 	if (has_vhe())
610 		kvm_vcpu_load_vhe(vcpu);
611 	kvm_arch_vcpu_load_fp(vcpu);
612 	kvm_vcpu_pmu_restore_guest(vcpu);
613 	if (kvm_arm_is_pvtime_enabled(&vcpu->arch))
614 		kvm_make_request(KVM_REQ_RECORD_STEAL, vcpu);
615 
616 	if (kvm_vcpu_should_clear_twe(vcpu))
617 		vcpu->arch.hcr_el2 &= ~HCR_TWE;
618 	else
619 		vcpu->arch.hcr_el2 |= HCR_TWE;
620 
621 	if (kvm_vcpu_should_clear_twi(vcpu))
622 		vcpu->arch.hcr_el2 &= ~HCR_TWI;
623 	else
624 		vcpu->arch.hcr_el2 |= HCR_TWI;
625 
626 	vcpu_set_pauth_traps(vcpu);
627 
628 	if (is_protected_kvm_enabled()) {
629 		kvm_call_hyp_nvhe(__pkvm_vcpu_load,
630 				  vcpu->kvm->arch.pkvm.handle,
631 				  vcpu->vcpu_idx, vcpu->arch.hcr_el2);
632 		kvm_call_hyp(__vgic_v3_restore_vmcr_aprs,
633 			     &vcpu->arch.vgic_cpu.vgic_v3);
634 	}
635 
636 	if (!cpumask_test_cpu(cpu, vcpu->kvm->arch.supported_cpus))
637 		vcpu_set_on_unsupported_cpu(vcpu);
638 }
639 
kvm_arch_vcpu_put(struct kvm_vcpu * vcpu)640 void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
641 {
642 	if (is_protected_kvm_enabled()) {
643 		kvm_call_hyp(__vgic_v3_save_vmcr_aprs,
644 			     &vcpu->arch.vgic_cpu.vgic_v3);
645 		kvm_call_hyp_nvhe(__pkvm_vcpu_put);
646 	}
647 
648 	kvm_vcpu_put_debug(vcpu);
649 	kvm_arch_vcpu_put_fp(vcpu);
650 	if (has_vhe())
651 		kvm_vcpu_put_vhe(vcpu);
652 	kvm_timer_vcpu_put(vcpu);
653 	kvm_vgic_put(vcpu);
654 	kvm_vcpu_pmu_restore_host(vcpu);
655 	if (vcpu_has_nv(vcpu))
656 		kvm_vcpu_put_hw_mmu(vcpu);
657 	kvm_arm_vmid_clear_active();
658 
659 	vcpu_clear_on_unsupported_cpu(vcpu);
660 	vcpu->cpu = -1;
661 }
662 
__kvm_arm_vcpu_power_off(struct kvm_vcpu * vcpu)663 static void __kvm_arm_vcpu_power_off(struct kvm_vcpu *vcpu)
664 {
665 	WRITE_ONCE(vcpu->arch.mp_state.mp_state, KVM_MP_STATE_STOPPED);
666 	kvm_make_request(KVM_REQ_SLEEP, vcpu);
667 	kvm_vcpu_kick(vcpu);
668 }
669 
kvm_arm_vcpu_power_off(struct kvm_vcpu * vcpu)670 void kvm_arm_vcpu_power_off(struct kvm_vcpu *vcpu)
671 {
672 	spin_lock(&vcpu->arch.mp_state_lock);
673 	__kvm_arm_vcpu_power_off(vcpu);
674 	spin_unlock(&vcpu->arch.mp_state_lock);
675 }
676 
kvm_arm_vcpu_stopped(struct kvm_vcpu * vcpu)677 bool kvm_arm_vcpu_stopped(struct kvm_vcpu *vcpu)
678 {
679 	return READ_ONCE(vcpu->arch.mp_state.mp_state) == KVM_MP_STATE_STOPPED;
680 }
681 
kvm_arm_vcpu_suspend(struct kvm_vcpu * vcpu)682 static void kvm_arm_vcpu_suspend(struct kvm_vcpu *vcpu)
683 {
684 	WRITE_ONCE(vcpu->arch.mp_state.mp_state, KVM_MP_STATE_SUSPENDED);
685 	kvm_make_request(KVM_REQ_SUSPEND, vcpu);
686 	kvm_vcpu_kick(vcpu);
687 }
688 
kvm_arm_vcpu_suspended(struct kvm_vcpu * vcpu)689 static bool kvm_arm_vcpu_suspended(struct kvm_vcpu *vcpu)
690 {
691 	return READ_ONCE(vcpu->arch.mp_state.mp_state) == KVM_MP_STATE_SUSPENDED;
692 }
693 
kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu * vcpu,struct kvm_mp_state * mp_state)694 int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
695 				    struct kvm_mp_state *mp_state)
696 {
697 	*mp_state = READ_ONCE(vcpu->arch.mp_state);
698 
699 	return 0;
700 }
701 
kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu * vcpu,struct kvm_mp_state * mp_state)702 int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
703 				    struct kvm_mp_state *mp_state)
704 {
705 	int ret = 0;
706 
707 	spin_lock(&vcpu->arch.mp_state_lock);
708 
709 	switch (mp_state->mp_state) {
710 	case KVM_MP_STATE_RUNNABLE:
711 		WRITE_ONCE(vcpu->arch.mp_state, *mp_state);
712 		break;
713 	case KVM_MP_STATE_STOPPED:
714 		__kvm_arm_vcpu_power_off(vcpu);
715 		break;
716 	case KVM_MP_STATE_SUSPENDED:
717 		kvm_arm_vcpu_suspend(vcpu);
718 		break;
719 	default:
720 		ret = -EINVAL;
721 	}
722 
723 	spin_unlock(&vcpu->arch.mp_state_lock);
724 
725 	return ret;
726 }
727 
728 /**
729  * kvm_arch_vcpu_runnable - determine if the vcpu can be scheduled
730  * @v:		The VCPU pointer
731  *
732  * If the guest CPU is not waiting for interrupts or an interrupt line is
733  * asserted, the CPU is by definition runnable.
734  */
kvm_arch_vcpu_runnable(struct kvm_vcpu * v)735 int kvm_arch_vcpu_runnable(struct kvm_vcpu *v)
736 {
737 	bool irq_lines = *vcpu_hcr(v) & (HCR_VI | HCR_VF);
738 	return ((irq_lines || kvm_vgic_vcpu_pending_irq(v))
739 		&& !kvm_arm_vcpu_stopped(v) && !v->arch.pause);
740 }
741 
kvm_arch_vcpu_in_kernel(struct kvm_vcpu * vcpu)742 bool kvm_arch_vcpu_in_kernel(struct kvm_vcpu *vcpu)
743 {
744 	return vcpu_mode_priv(vcpu);
745 }
746 
747 #ifdef CONFIG_GUEST_PERF_EVENTS
kvm_arch_vcpu_get_ip(struct kvm_vcpu * vcpu)748 unsigned long kvm_arch_vcpu_get_ip(struct kvm_vcpu *vcpu)
749 {
750 	return *vcpu_pc(vcpu);
751 }
752 #endif
753 
kvm_init_mpidr_data(struct kvm * kvm)754 static void kvm_init_mpidr_data(struct kvm *kvm)
755 {
756 	struct kvm_mpidr_data *data = NULL;
757 	unsigned long c, mask, nr_entries;
758 	u64 aff_set = 0, aff_clr = ~0UL;
759 	struct kvm_vcpu *vcpu;
760 
761 	mutex_lock(&kvm->arch.config_lock);
762 
763 	if (rcu_access_pointer(kvm->arch.mpidr_data) ||
764 	    atomic_read(&kvm->online_vcpus) == 1)
765 		goto out;
766 
767 	kvm_for_each_vcpu(c, vcpu, kvm) {
768 		u64 aff = kvm_vcpu_get_mpidr_aff(vcpu);
769 		aff_set |= aff;
770 		aff_clr &= aff;
771 	}
772 
773 	/*
774 	 * A significant bit can be either 0 or 1, and will only appear in
775 	 * aff_set. Use aff_clr to weed out the useless stuff.
776 	 */
777 	mask = aff_set ^ aff_clr;
778 	nr_entries = BIT_ULL(hweight_long(mask));
779 
780 	/*
781 	 * Don't let userspace fool us. If we need more than a single page
782 	 * to describe the compressed MPIDR array, just fall back to the
783 	 * iterative method. Single vcpu VMs do not need this either.
784 	 */
785 	if (struct_size(data, cmpidr_to_idx, nr_entries) <= PAGE_SIZE)
786 		data = kzalloc(struct_size(data, cmpidr_to_idx, nr_entries),
787 			       GFP_KERNEL_ACCOUNT);
788 
789 	if (!data)
790 		goto out;
791 
792 	data->mpidr_mask = mask;
793 
794 	kvm_for_each_vcpu(c, vcpu, kvm) {
795 		u64 aff = kvm_vcpu_get_mpidr_aff(vcpu);
796 		u16 index = kvm_mpidr_index(data, aff);
797 
798 		data->cmpidr_to_idx[index] = c;
799 	}
800 
801 	rcu_assign_pointer(kvm->arch.mpidr_data, data);
802 out:
803 	mutex_unlock(&kvm->arch.config_lock);
804 }
805 
806 /*
807  * Handle both the initialisation that is being done when the vcpu is
808  * run for the first time, as well as the updates that must be
809  * performed each time we get a new thread dealing with this vcpu.
810  */
kvm_arch_vcpu_run_pid_change(struct kvm_vcpu * vcpu)811 int kvm_arch_vcpu_run_pid_change(struct kvm_vcpu *vcpu)
812 {
813 	struct kvm *kvm = vcpu->kvm;
814 	int ret;
815 
816 	if (!kvm_vcpu_initialized(vcpu))
817 		return -ENOEXEC;
818 
819 	if (!kvm_arm_vcpu_is_finalized(vcpu))
820 		return -EPERM;
821 
822 	ret = kvm_arch_vcpu_run_map_fp(vcpu);
823 	if (ret)
824 		return ret;
825 
826 	if (likely(vcpu_has_run_once(vcpu)))
827 		return 0;
828 
829 	kvm_init_mpidr_data(kvm);
830 
831 	if (likely(irqchip_in_kernel(kvm))) {
832 		/*
833 		 * Map the VGIC hardware resources before running a vcpu the
834 		 * first time on this VM.
835 		 */
836 		ret = kvm_vgic_map_resources(kvm);
837 		if (ret)
838 			return ret;
839 	}
840 
841 	ret = kvm_finalize_sys_regs(vcpu);
842 	if (ret)
843 		return ret;
844 
845 	if (vcpu_has_nv(vcpu)) {
846 		ret = kvm_vgic_vcpu_nv_init(vcpu);
847 		if (ret)
848 			return ret;
849 	}
850 
851 	/*
852 	 * This needs to happen after any restriction has been applied
853 	 * to the feature set.
854 	 */
855 	kvm_calculate_traps(vcpu);
856 
857 	ret = kvm_timer_enable(vcpu);
858 	if (ret)
859 		return ret;
860 
861 	if (kvm_vcpu_has_pmu(vcpu)) {
862 		ret = kvm_arm_pmu_v3_enable(vcpu);
863 		if (ret)
864 			return ret;
865 	}
866 
867 	if (is_protected_kvm_enabled()) {
868 		ret = pkvm_create_hyp_vm(kvm);
869 		if (ret)
870 			return ret;
871 
872 		ret = pkvm_create_hyp_vcpu(vcpu);
873 		if (ret)
874 			return ret;
875 	}
876 
877 	mutex_lock(&kvm->arch.config_lock);
878 	set_bit(KVM_ARCH_FLAG_HAS_RAN_ONCE, &kvm->arch.flags);
879 	mutex_unlock(&kvm->arch.config_lock);
880 
881 	return ret;
882 }
883 
kvm_arch_intc_initialized(struct kvm * kvm)884 bool kvm_arch_intc_initialized(struct kvm *kvm)
885 {
886 	return vgic_initialized(kvm);
887 }
888 
kvm_arm_halt_guest(struct kvm * kvm)889 void kvm_arm_halt_guest(struct kvm *kvm)
890 {
891 	unsigned long i;
892 	struct kvm_vcpu *vcpu;
893 
894 	kvm_for_each_vcpu(i, vcpu, kvm)
895 		vcpu->arch.pause = true;
896 	kvm_make_all_cpus_request(kvm, KVM_REQ_SLEEP);
897 }
898 
kvm_arm_resume_guest(struct kvm * kvm)899 void kvm_arm_resume_guest(struct kvm *kvm)
900 {
901 	unsigned long i;
902 	struct kvm_vcpu *vcpu;
903 
904 	kvm_for_each_vcpu(i, vcpu, kvm) {
905 		vcpu->arch.pause = false;
906 		__kvm_vcpu_wake_up(vcpu);
907 	}
908 }
909 
kvm_vcpu_sleep(struct kvm_vcpu * vcpu)910 static void kvm_vcpu_sleep(struct kvm_vcpu *vcpu)
911 {
912 	struct rcuwait *wait = kvm_arch_vcpu_get_wait(vcpu);
913 
914 	rcuwait_wait_event(wait,
915 			   (!kvm_arm_vcpu_stopped(vcpu)) && (!vcpu->arch.pause),
916 			   TASK_INTERRUPTIBLE);
917 
918 	if (kvm_arm_vcpu_stopped(vcpu) || vcpu->arch.pause) {
919 		/* Awaken to handle a signal, request we sleep again later. */
920 		kvm_make_request(KVM_REQ_SLEEP, vcpu);
921 	}
922 
923 	/*
924 	 * Make sure we will observe a potential reset request if we've
925 	 * observed a change to the power state. Pairs with the smp_wmb() in
926 	 * kvm_psci_vcpu_on().
927 	 */
928 	smp_rmb();
929 }
930 
931 /**
932  * kvm_vcpu_wfi - emulate Wait-For-Interrupt behavior
933  * @vcpu:	The VCPU pointer
934  *
935  * Suspend execution of a vCPU until a valid wake event is detected, i.e. until
936  * the vCPU is runnable.  The vCPU may or may not be scheduled out, depending
937  * on when a wake event arrives, e.g. there may already be a pending wake event.
938  */
kvm_vcpu_wfi(struct kvm_vcpu * vcpu)939 void kvm_vcpu_wfi(struct kvm_vcpu *vcpu)
940 {
941 	/*
942 	 * Sync back the state of the GIC CPU interface so that we have
943 	 * the latest PMR and group enables. This ensures that
944 	 * kvm_arch_vcpu_runnable has up-to-date data to decide whether
945 	 * we have pending interrupts, e.g. when determining if the
946 	 * vCPU should block.
947 	 *
948 	 * For the same reason, we want to tell GICv4 that we need
949 	 * doorbells to be signalled, should an interrupt become pending.
950 	 */
951 	preempt_disable();
952 	vcpu_set_flag(vcpu, IN_WFI);
953 	kvm_vgic_put(vcpu);
954 	preempt_enable();
955 
956 	kvm_vcpu_halt(vcpu);
957 	vcpu_clear_flag(vcpu, IN_WFIT);
958 
959 	preempt_disable();
960 	vcpu_clear_flag(vcpu, IN_WFI);
961 	kvm_vgic_load(vcpu);
962 	preempt_enable();
963 }
964 
kvm_vcpu_suspend(struct kvm_vcpu * vcpu)965 static int kvm_vcpu_suspend(struct kvm_vcpu *vcpu)
966 {
967 	if (!kvm_arm_vcpu_suspended(vcpu))
968 		return 1;
969 
970 	kvm_vcpu_wfi(vcpu);
971 
972 	/*
973 	 * The suspend state is sticky; we do not leave it until userspace
974 	 * explicitly marks the vCPU as runnable. Request that we suspend again
975 	 * later.
976 	 */
977 	kvm_make_request(KVM_REQ_SUSPEND, vcpu);
978 
979 	/*
980 	 * Check to make sure the vCPU is actually runnable. If so, exit to
981 	 * userspace informing it of the wakeup condition.
982 	 */
983 	if (kvm_arch_vcpu_runnable(vcpu)) {
984 		memset(&vcpu->run->system_event, 0, sizeof(vcpu->run->system_event));
985 		vcpu->run->system_event.type = KVM_SYSTEM_EVENT_WAKEUP;
986 		vcpu->run->exit_reason = KVM_EXIT_SYSTEM_EVENT;
987 		return 0;
988 	}
989 
990 	/*
991 	 * Otherwise, we were unblocked to process a different event, such as a
992 	 * pending signal. Return 1 and allow kvm_arch_vcpu_ioctl_run() to
993 	 * process the event.
994 	 */
995 	return 1;
996 }
997 
998 /**
999  * check_vcpu_requests - check and handle pending vCPU requests
1000  * @vcpu:	the VCPU pointer
1001  *
1002  * Return: 1 if we should enter the guest
1003  *	   0 if we should exit to userspace
1004  *	   < 0 if we should exit to userspace, where the return value indicates
1005  *	   an error
1006  */
check_vcpu_requests(struct kvm_vcpu * vcpu)1007 static int check_vcpu_requests(struct kvm_vcpu *vcpu)
1008 {
1009 	if (kvm_request_pending(vcpu)) {
1010 		if (kvm_check_request(KVM_REQ_VM_DEAD, vcpu))
1011 			return -EIO;
1012 
1013 		if (kvm_check_request(KVM_REQ_SLEEP, vcpu))
1014 			kvm_vcpu_sleep(vcpu);
1015 
1016 		if (kvm_check_request(KVM_REQ_VCPU_RESET, vcpu))
1017 			kvm_reset_vcpu(vcpu);
1018 
1019 		/*
1020 		 * Clear IRQ_PENDING requests that were made to guarantee
1021 		 * that a VCPU sees new virtual interrupts.
1022 		 */
1023 		kvm_check_request(KVM_REQ_IRQ_PENDING, vcpu);
1024 
1025 		if (kvm_check_request(KVM_REQ_RECORD_STEAL, vcpu))
1026 			kvm_update_stolen_time(vcpu);
1027 
1028 		if (kvm_check_request(KVM_REQ_RELOAD_GICv4, vcpu)) {
1029 			/* The distributor enable bits were changed */
1030 			preempt_disable();
1031 			vgic_v4_put(vcpu);
1032 			vgic_v4_load(vcpu);
1033 			preempt_enable();
1034 		}
1035 
1036 		if (kvm_check_request(KVM_REQ_RELOAD_PMU, vcpu))
1037 			kvm_vcpu_reload_pmu(vcpu);
1038 
1039 		if (kvm_check_request(KVM_REQ_RESYNC_PMU_EL0, vcpu))
1040 			kvm_vcpu_pmu_restore_guest(vcpu);
1041 
1042 		if (kvm_check_request(KVM_REQ_SUSPEND, vcpu))
1043 			return kvm_vcpu_suspend(vcpu);
1044 
1045 		if (kvm_dirty_ring_check_request(vcpu))
1046 			return 0;
1047 
1048 		check_nested_vcpu_requests(vcpu);
1049 	}
1050 
1051 	return 1;
1052 }
1053 
vcpu_mode_is_bad_32bit(struct kvm_vcpu * vcpu)1054 static bool vcpu_mode_is_bad_32bit(struct kvm_vcpu *vcpu)
1055 {
1056 	if (likely(!vcpu_mode_is_32bit(vcpu)))
1057 		return false;
1058 
1059 	if (vcpu_has_nv(vcpu))
1060 		return true;
1061 
1062 	return !kvm_supports_32bit_el0();
1063 }
1064 
1065 /**
1066  * kvm_vcpu_exit_request - returns true if the VCPU should *not* enter the guest
1067  * @vcpu:	The VCPU pointer
1068  * @ret:	Pointer to write optional return code
1069  *
1070  * Returns: true if the VCPU needs to return to a preemptible + interruptible
1071  *	    and skip guest entry.
1072  *
1073  * This function disambiguates between two different types of exits: exits to a
1074  * preemptible + interruptible kernel context and exits to userspace. For an
1075  * exit to userspace, this function will write the return code to ret and return
1076  * true. For an exit to preemptible + interruptible kernel context (i.e. check
1077  * for pending work and re-enter), return true without writing to ret.
1078  */
kvm_vcpu_exit_request(struct kvm_vcpu * vcpu,int * ret)1079 static bool kvm_vcpu_exit_request(struct kvm_vcpu *vcpu, int *ret)
1080 {
1081 	struct kvm_run *run = vcpu->run;
1082 
1083 	/*
1084 	 * If we're using a userspace irqchip, then check if we need
1085 	 * to tell a userspace irqchip about timer or PMU level
1086 	 * changes and if so, exit to userspace (the actual level
1087 	 * state gets updated in kvm_timer_update_run and
1088 	 * kvm_pmu_update_run below).
1089 	 */
1090 	if (unlikely(!irqchip_in_kernel(vcpu->kvm))) {
1091 		if (kvm_timer_should_notify_user(vcpu) ||
1092 		    kvm_pmu_should_notify_user(vcpu)) {
1093 			*ret = -EINTR;
1094 			run->exit_reason = KVM_EXIT_INTR;
1095 			return true;
1096 		}
1097 	}
1098 
1099 	if (unlikely(vcpu_on_unsupported_cpu(vcpu))) {
1100 		run->exit_reason = KVM_EXIT_FAIL_ENTRY;
1101 		run->fail_entry.hardware_entry_failure_reason = KVM_EXIT_FAIL_ENTRY_CPU_UNSUPPORTED;
1102 		run->fail_entry.cpu = smp_processor_id();
1103 		*ret = 0;
1104 		return true;
1105 	}
1106 
1107 	return kvm_request_pending(vcpu) ||
1108 			xfer_to_guest_mode_work_pending();
1109 }
1110 
1111 /*
1112  * Actually run the vCPU, entering an RCU extended quiescent state (EQS) while
1113  * the vCPU is running.
1114  *
1115  * This must be noinstr as instrumentation may make use of RCU, and this is not
1116  * safe during the EQS.
1117  */
kvm_arm_vcpu_enter_exit(struct kvm_vcpu * vcpu)1118 static int noinstr kvm_arm_vcpu_enter_exit(struct kvm_vcpu *vcpu)
1119 {
1120 	int ret;
1121 
1122 	guest_state_enter_irqoff();
1123 	ret = kvm_call_hyp_ret(__kvm_vcpu_run, vcpu);
1124 	guest_state_exit_irqoff();
1125 
1126 	return ret;
1127 }
1128 
1129 /**
1130  * kvm_arch_vcpu_ioctl_run - the main VCPU run function to execute guest code
1131  * @vcpu:	The VCPU pointer
1132  *
1133  * This function is called through the VCPU_RUN ioctl called from user space. It
1134  * will execute VM code in a loop until the time slice for the process is used
1135  * or some emulation is needed from user space in which case the function will
1136  * return with return value 0 and with the kvm_run structure filled in with the
1137  * required data for the requested emulation.
1138  */
kvm_arch_vcpu_ioctl_run(struct kvm_vcpu * vcpu)1139 int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu)
1140 {
1141 	struct kvm_run *run = vcpu->run;
1142 	int ret;
1143 
1144 	if (run->exit_reason == KVM_EXIT_MMIO) {
1145 		ret = kvm_handle_mmio_return(vcpu);
1146 		if (ret <= 0)
1147 			return ret;
1148 	}
1149 
1150 	vcpu_load(vcpu);
1151 
1152 	if (!vcpu->wants_to_run) {
1153 		ret = -EINTR;
1154 		goto out;
1155 	}
1156 
1157 	kvm_sigset_activate(vcpu);
1158 
1159 	ret = 1;
1160 	run->exit_reason = KVM_EXIT_UNKNOWN;
1161 	run->flags = 0;
1162 	while (ret > 0) {
1163 		/*
1164 		 * Check conditions before entering the guest
1165 		 */
1166 		ret = xfer_to_guest_mode_handle_work(vcpu);
1167 		if (!ret)
1168 			ret = 1;
1169 
1170 		if (ret > 0)
1171 			ret = check_vcpu_requests(vcpu);
1172 
1173 		/*
1174 		 * Preparing the interrupts to be injected also
1175 		 * involves poking the GIC, which must be done in a
1176 		 * non-preemptible context.
1177 		 */
1178 		preempt_disable();
1179 
1180 		if (kvm_vcpu_has_pmu(vcpu))
1181 			kvm_pmu_flush_hwstate(vcpu);
1182 
1183 		local_irq_disable();
1184 
1185 		kvm_vgic_flush_hwstate(vcpu);
1186 
1187 		kvm_pmu_update_vcpu_events(vcpu);
1188 
1189 		/*
1190 		 * Ensure we set mode to IN_GUEST_MODE after we disable
1191 		 * interrupts and before the final VCPU requests check.
1192 		 * See the comment in kvm_vcpu_exiting_guest_mode() and
1193 		 * Documentation/virt/kvm/vcpu-requests.rst
1194 		 */
1195 		smp_store_mb(vcpu->mode, IN_GUEST_MODE);
1196 
1197 		if (ret <= 0 || kvm_vcpu_exit_request(vcpu, &ret)) {
1198 			vcpu->mode = OUTSIDE_GUEST_MODE;
1199 			isb(); /* Ensure work in x_flush_hwstate is committed */
1200 			if (kvm_vcpu_has_pmu(vcpu))
1201 				kvm_pmu_sync_hwstate(vcpu);
1202 			if (unlikely(!irqchip_in_kernel(vcpu->kvm)))
1203 				kvm_timer_sync_user(vcpu);
1204 			kvm_vgic_sync_hwstate(vcpu);
1205 			local_irq_enable();
1206 			preempt_enable();
1207 			continue;
1208 		}
1209 
1210 		kvm_arch_vcpu_ctxflush_fp(vcpu);
1211 
1212 		/**************************************************************
1213 		 * Enter the guest
1214 		 */
1215 		trace_kvm_entry(*vcpu_pc(vcpu));
1216 		guest_timing_enter_irqoff();
1217 
1218 		ret = kvm_arm_vcpu_enter_exit(vcpu);
1219 
1220 		vcpu->mode = OUTSIDE_GUEST_MODE;
1221 		vcpu->stat.exits++;
1222 		/*
1223 		 * Back from guest
1224 		 *************************************************************/
1225 
1226 		/*
1227 		 * We must sync the PMU state before the vgic state so
1228 		 * that the vgic can properly sample the updated state of the
1229 		 * interrupt line.
1230 		 */
1231 		if (kvm_vcpu_has_pmu(vcpu))
1232 			kvm_pmu_sync_hwstate(vcpu);
1233 
1234 		/*
1235 		 * Sync the vgic state before syncing the timer state because
1236 		 * the timer code needs to know if the virtual timer
1237 		 * interrupts are active.
1238 		 */
1239 		kvm_vgic_sync_hwstate(vcpu);
1240 
1241 		/*
1242 		 * Sync the timer hardware state before enabling interrupts as
1243 		 * we don't want vtimer interrupts to race with syncing the
1244 		 * timer virtual interrupt state.
1245 		 */
1246 		if (unlikely(!irqchip_in_kernel(vcpu->kvm)))
1247 			kvm_timer_sync_user(vcpu);
1248 
1249 		if (is_hyp_ctxt(vcpu))
1250 			kvm_timer_sync_nested(vcpu);
1251 
1252 		kvm_arch_vcpu_ctxsync_fp(vcpu);
1253 
1254 		/*
1255 		 * We must ensure that any pending interrupts are taken before
1256 		 * we exit guest timing so that timer ticks are accounted as
1257 		 * guest time. Transiently unmask interrupts so that any
1258 		 * pending interrupts are taken.
1259 		 *
1260 		 * Per ARM DDI 0487G.b section D1.13.4, an ISB (or other
1261 		 * context synchronization event) is necessary to ensure that
1262 		 * pending interrupts are taken.
1263 		 */
1264 		if (ARM_EXCEPTION_CODE(ret) == ARM_EXCEPTION_IRQ) {
1265 			local_irq_enable();
1266 			isb();
1267 			local_irq_disable();
1268 		}
1269 
1270 		guest_timing_exit_irqoff();
1271 
1272 		local_irq_enable();
1273 
1274 		trace_kvm_exit(ret, kvm_vcpu_trap_get_class(vcpu), *vcpu_pc(vcpu));
1275 
1276 		/* Exit types that need handling before we can be preempted */
1277 		handle_exit_early(vcpu, ret);
1278 
1279 		preempt_enable();
1280 
1281 		/*
1282 		 * The ARMv8 architecture doesn't give the hypervisor
1283 		 * a mechanism to prevent a guest from dropping to AArch32 EL0
1284 		 * if implemented by the CPU. If we spot the guest in such
1285 		 * state and that we decided it wasn't supposed to do so (like
1286 		 * with the asymmetric AArch32 case), return to userspace with
1287 		 * a fatal error.
1288 		 */
1289 		if (vcpu_mode_is_bad_32bit(vcpu)) {
1290 			/*
1291 			 * As we have caught the guest red-handed, decide that
1292 			 * it isn't fit for purpose anymore by making the vcpu
1293 			 * invalid. The VMM can try and fix it by issuing  a
1294 			 * KVM_ARM_VCPU_INIT if it really wants to.
1295 			 */
1296 			vcpu_clear_flag(vcpu, VCPU_INITIALIZED);
1297 			ret = ARM_EXCEPTION_IL;
1298 		}
1299 
1300 		ret = handle_exit(vcpu, ret);
1301 	}
1302 
1303 	/* Tell userspace about in-kernel device output levels */
1304 	if (unlikely(!irqchip_in_kernel(vcpu->kvm))) {
1305 		kvm_timer_update_run(vcpu);
1306 		kvm_pmu_update_run(vcpu);
1307 	}
1308 
1309 	kvm_sigset_deactivate(vcpu);
1310 
1311 out:
1312 	/*
1313 	 * In the unlikely event that we are returning to userspace
1314 	 * with pending exceptions or PC adjustment, commit these
1315 	 * adjustments in order to give userspace a consistent view of
1316 	 * the vcpu state. Note that this relies on __kvm_adjust_pc()
1317 	 * being preempt-safe on VHE.
1318 	 */
1319 	if (unlikely(vcpu_get_flag(vcpu, PENDING_EXCEPTION) ||
1320 		     vcpu_get_flag(vcpu, INCREMENT_PC)))
1321 		kvm_call_hyp(__kvm_adjust_pc, vcpu);
1322 
1323 	vcpu_put(vcpu);
1324 	return ret;
1325 }
1326 
vcpu_interrupt_line(struct kvm_vcpu * vcpu,int number,bool level)1327 static int vcpu_interrupt_line(struct kvm_vcpu *vcpu, int number, bool level)
1328 {
1329 	int bit_index;
1330 	bool set;
1331 	unsigned long *hcr;
1332 
1333 	if (number == KVM_ARM_IRQ_CPU_IRQ)
1334 		bit_index = __ffs(HCR_VI);
1335 	else /* KVM_ARM_IRQ_CPU_FIQ */
1336 		bit_index = __ffs(HCR_VF);
1337 
1338 	hcr = vcpu_hcr(vcpu);
1339 	if (level)
1340 		set = test_and_set_bit(bit_index, hcr);
1341 	else
1342 		set = test_and_clear_bit(bit_index, hcr);
1343 
1344 	/*
1345 	 * If we didn't change anything, no need to wake up or kick other CPUs
1346 	 */
1347 	if (set == level)
1348 		return 0;
1349 
1350 	/*
1351 	 * The vcpu irq_lines field was updated, wake up sleeping VCPUs and
1352 	 * trigger a world-switch round on the running physical CPU to set the
1353 	 * virtual IRQ/FIQ fields in the HCR appropriately.
1354 	 */
1355 	kvm_make_request(KVM_REQ_IRQ_PENDING, vcpu);
1356 	kvm_vcpu_kick(vcpu);
1357 
1358 	return 0;
1359 }
1360 
kvm_vm_ioctl_irq_line(struct kvm * kvm,struct kvm_irq_level * irq_level,bool line_status)1361 int kvm_vm_ioctl_irq_line(struct kvm *kvm, struct kvm_irq_level *irq_level,
1362 			  bool line_status)
1363 {
1364 	u32 irq = irq_level->irq;
1365 	unsigned int irq_type, vcpu_id, irq_num;
1366 	struct kvm_vcpu *vcpu = NULL;
1367 	bool level = irq_level->level;
1368 
1369 	irq_type = (irq >> KVM_ARM_IRQ_TYPE_SHIFT) & KVM_ARM_IRQ_TYPE_MASK;
1370 	vcpu_id = (irq >> KVM_ARM_IRQ_VCPU_SHIFT) & KVM_ARM_IRQ_VCPU_MASK;
1371 	vcpu_id += ((irq >> KVM_ARM_IRQ_VCPU2_SHIFT) & KVM_ARM_IRQ_VCPU2_MASK) * (KVM_ARM_IRQ_VCPU_MASK + 1);
1372 	irq_num = (irq >> KVM_ARM_IRQ_NUM_SHIFT) & KVM_ARM_IRQ_NUM_MASK;
1373 
1374 	trace_kvm_irq_line(irq_type, vcpu_id, irq_num, irq_level->level);
1375 
1376 	switch (irq_type) {
1377 	case KVM_ARM_IRQ_TYPE_CPU:
1378 		if (irqchip_in_kernel(kvm))
1379 			return -ENXIO;
1380 
1381 		vcpu = kvm_get_vcpu_by_id(kvm, vcpu_id);
1382 		if (!vcpu)
1383 			return -EINVAL;
1384 
1385 		if (irq_num > KVM_ARM_IRQ_CPU_FIQ)
1386 			return -EINVAL;
1387 
1388 		return vcpu_interrupt_line(vcpu, irq_num, level);
1389 	case KVM_ARM_IRQ_TYPE_PPI:
1390 		if (!irqchip_in_kernel(kvm))
1391 			return -ENXIO;
1392 
1393 		vcpu = kvm_get_vcpu_by_id(kvm, vcpu_id);
1394 		if (!vcpu)
1395 			return -EINVAL;
1396 
1397 		if (irq_num < VGIC_NR_SGIS || irq_num >= VGIC_NR_PRIVATE_IRQS)
1398 			return -EINVAL;
1399 
1400 		return kvm_vgic_inject_irq(kvm, vcpu, irq_num, level, NULL);
1401 	case KVM_ARM_IRQ_TYPE_SPI:
1402 		if (!irqchip_in_kernel(kvm))
1403 			return -ENXIO;
1404 
1405 		if (irq_num < VGIC_NR_PRIVATE_IRQS)
1406 			return -EINVAL;
1407 
1408 		return kvm_vgic_inject_irq(kvm, NULL, irq_num, level, NULL);
1409 	}
1410 
1411 	return -EINVAL;
1412 }
1413 
system_supported_vcpu_features(void)1414 static unsigned long system_supported_vcpu_features(void)
1415 {
1416 	unsigned long features = KVM_VCPU_VALID_FEATURES;
1417 
1418 	if (!cpus_have_final_cap(ARM64_HAS_32BIT_EL1))
1419 		clear_bit(KVM_ARM_VCPU_EL1_32BIT, &features);
1420 
1421 	if (!kvm_supports_guest_pmuv3())
1422 		clear_bit(KVM_ARM_VCPU_PMU_V3, &features);
1423 
1424 	if (!system_supports_sve())
1425 		clear_bit(KVM_ARM_VCPU_SVE, &features);
1426 
1427 	if (!kvm_has_full_ptr_auth()) {
1428 		clear_bit(KVM_ARM_VCPU_PTRAUTH_ADDRESS, &features);
1429 		clear_bit(KVM_ARM_VCPU_PTRAUTH_GENERIC, &features);
1430 	}
1431 
1432 	if (!cpus_have_final_cap(ARM64_HAS_NESTED_VIRT))
1433 		clear_bit(KVM_ARM_VCPU_HAS_EL2, &features);
1434 
1435 	return features;
1436 }
1437 
kvm_vcpu_init_check_features(struct kvm_vcpu * vcpu,const struct kvm_vcpu_init * init)1438 static int kvm_vcpu_init_check_features(struct kvm_vcpu *vcpu,
1439 					const struct kvm_vcpu_init *init)
1440 {
1441 	unsigned long features = init->features[0];
1442 	int i;
1443 
1444 	if (features & ~KVM_VCPU_VALID_FEATURES)
1445 		return -ENOENT;
1446 
1447 	for (i = 1; i < ARRAY_SIZE(init->features); i++) {
1448 		if (init->features[i])
1449 			return -ENOENT;
1450 	}
1451 
1452 	if (features & ~system_supported_vcpu_features())
1453 		return -EINVAL;
1454 
1455 	/*
1456 	 * For now make sure that both address/generic pointer authentication
1457 	 * features are requested by the userspace together.
1458 	 */
1459 	if (test_bit(KVM_ARM_VCPU_PTRAUTH_ADDRESS, &features) !=
1460 	    test_bit(KVM_ARM_VCPU_PTRAUTH_GENERIC, &features))
1461 		return -EINVAL;
1462 
1463 	if (!test_bit(KVM_ARM_VCPU_EL1_32BIT, &features))
1464 		return 0;
1465 
1466 	/* MTE is incompatible with AArch32 */
1467 	if (kvm_has_mte(vcpu->kvm))
1468 		return -EINVAL;
1469 
1470 	/* NV is incompatible with AArch32 */
1471 	if (test_bit(KVM_ARM_VCPU_HAS_EL2, &features))
1472 		return -EINVAL;
1473 
1474 	return 0;
1475 }
1476 
kvm_vcpu_init_changed(struct kvm_vcpu * vcpu,const struct kvm_vcpu_init * init)1477 static bool kvm_vcpu_init_changed(struct kvm_vcpu *vcpu,
1478 				  const struct kvm_vcpu_init *init)
1479 {
1480 	unsigned long features = init->features[0];
1481 
1482 	return !bitmap_equal(vcpu->kvm->arch.vcpu_features, &features,
1483 			     KVM_VCPU_MAX_FEATURES);
1484 }
1485 
kvm_setup_vcpu(struct kvm_vcpu * vcpu)1486 static int kvm_setup_vcpu(struct kvm_vcpu *vcpu)
1487 {
1488 	struct kvm *kvm = vcpu->kvm;
1489 	int ret = 0;
1490 
1491 	/*
1492 	 * When the vCPU has a PMU, but no PMU is set for the guest
1493 	 * yet, set the default one.
1494 	 */
1495 	if (kvm_vcpu_has_pmu(vcpu) && !kvm->arch.arm_pmu)
1496 		ret = kvm_arm_set_default_pmu(kvm);
1497 
1498 	/* Prepare for nested if required */
1499 	if (!ret && vcpu_has_nv(vcpu))
1500 		ret = kvm_vcpu_init_nested(vcpu);
1501 
1502 	return ret;
1503 }
1504 
__kvm_vcpu_set_target(struct kvm_vcpu * vcpu,const struct kvm_vcpu_init * init)1505 static int __kvm_vcpu_set_target(struct kvm_vcpu *vcpu,
1506 				 const struct kvm_vcpu_init *init)
1507 {
1508 	unsigned long features = init->features[0];
1509 	struct kvm *kvm = vcpu->kvm;
1510 	int ret = -EINVAL;
1511 
1512 	mutex_lock(&kvm->arch.config_lock);
1513 
1514 	if (test_bit(KVM_ARCH_FLAG_VCPU_FEATURES_CONFIGURED, &kvm->arch.flags) &&
1515 	    kvm_vcpu_init_changed(vcpu, init))
1516 		goto out_unlock;
1517 
1518 	bitmap_copy(kvm->arch.vcpu_features, &features, KVM_VCPU_MAX_FEATURES);
1519 
1520 	ret = kvm_setup_vcpu(vcpu);
1521 	if (ret)
1522 		goto out_unlock;
1523 
1524 	/* Now we know what it is, we can reset it. */
1525 	kvm_reset_vcpu(vcpu);
1526 
1527 	set_bit(KVM_ARCH_FLAG_VCPU_FEATURES_CONFIGURED, &kvm->arch.flags);
1528 	vcpu_set_flag(vcpu, VCPU_INITIALIZED);
1529 	ret = 0;
1530 out_unlock:
1531 	mutex_unlock(&kvm->arch.config_lock);
1532 	return ret;
1533 }
1534 
kvm_vcpu_set_target(struct kvm_vcpu * vcpu,const struct kvm_vcpu_init * init)1535 static int kvm_vcpu_set_target(struct kvm_vcpu *vcpu,
1536 			       const struct kvm_vcpu_init *init)
1537 {
1538 	int ret;
1539 
1540 	if (init->target != KVM_ARM_TARGET_GENERIC_V8 &&
1541 	    init->target != kvm_target_cpu())
1542 		return -EINVAL;
1543 
1544 	ret = kvm_vcpu_init_check_features(vcpu, init);
1545 	if (ret)
1546 		return ret;
1547 
1548 	if (!kvm_vcpu_initialized(vcpu))
1549 		return __kvm_vcpu_set_target(vcpu, init);
1550 
1551 	if (kvm_vcpu_init_changed(vcpu, init))
1552 		return -EINVAL;
1553 
1554 	kvm_reset_vcpu(vcpu);
1555 	return 0;
1556 }
1557 
kvm_arch_vcpu_ioctl_vcpu_init(struct kvm_vcpu * vcpu,struct kvm_vcpu_init * init)1558 static int kvm_arch_vcpu_ioctl_vcpu_init(struct kvm_vcpu *vcpu,
1559 					 struct kvm_vcpu_init *init)
1560 {
1561 	bool power_off = false;
1562 	int ret;
1563 
1564 	/*
1565 	 * Treat the power-off vCPU feature as ephemeral. Clear the bit to avoid
1566 	 * reflecting it in the finalized feature set, thus limiting its scope
1567 	 * to a single KVM_ARM_VCPU_INIT call.
1568 	 */
1569 	if (init->features[0] & BIT(KVM_ARM_VCPU_POWER_OFF)) {
1570 		init->features[0] &= ~BIT(KVM_ARM_VCPU_POWER_OFF);
1571 		power_off = true;
1572 	}
1573 
1574 	ret = kvm_vcpu_set_target(vcpu, init);
1575 	if (ret)
1576 		return ret;
1577 
1578 	/*
1579 	 * Ensure a rebooted VM will fault in RAM pages and detect if the
1580 	 * guest MMU is turned off and flush the caches as needed.
1581 	 *
1582 	 * S2FWB enforces all memory accesses to RAM being cacheable,
1583 	 * ensuring that the data side is always coherent. We still
1584 	 * need to invalidate the I-cache though, as FWB does *not*
1585 	 * imply CTR_EL0.DIC.
1586 	 */
1587 	if (vcpu_has_run_once(vcpu)) {
1588 		if (!cpus_have_final_cap(ARM64_HAS_STAGE2_FWB))
1589 			stage2_unmap_vm(vcpu->kvm);
1590 		else
1591 			icache_inval_all_pou();
1592 	}
1593 
1594 	vcpu_reset_hcr(vcpu);
1595 
1596 	/*
1597 	 * Handle the "start in power-off" case.
1598 	 */
1599 	spin_lock(&vcpu->arch.mp_state_lock);
1600 
1601 	if (power_off)
1602 		__kvm_arm_vcpu_power_off(vcpu);
1603 	else
1604 		WRITE_ONCE(vcpu->arch.mp_state.mp_state, KVM_MP_STATE_RUNNABLE);
1605 
1606 	spin_unlock(&vcpu->arch.mp_state_lock);
1607 
1608 	return 0;
1609 }
1610 
kvm_arm_vcpu_set_attr(struct kvm_vcpu * vcpu,struct kvm_device_attr * attr)1611 static int kvm_arm_vcpu_set_attr(struct kvm_vcpu *vcpu,
1612 				 struct kvm_device_attr *attr)
1613 {
1614 	int ret = -ENXIO;
1615 
1616 	switch (attr->group) {
1617 	default:
1618 		ret = kvm_arm_vcpu_arch_set_attr(vcpu, attr);
1619 		break;
1620 	}
1621 
1622 	return ret;
1623 }
1624 
kvm_arm_vcpu_get_attr(struct kvm_vcpu * vcpu,struct kvm_device_attr * attr)1625 static int kvm_arm_vcpu_get_attr(struct kvm_vcpu *vcpu,
1626 				 struct kvm_device_attr *attr)
1627 {
1628 	int ret = -ENXIO;
1629 
1630 	switch (attr->group) {
1631 	default:
1632 		ret = kvm_arm_vcpu_arch_get_attr(vcpu, attr);
1633 		break;
1634 	}
1635 
1636 	return ret;
1637 }
1638 
kvm_arm_vcpu_has_attr(struct kvm_vcpu * vcpu,struct kvm_device_attr * attr)1639 static int kvm_arm_vcpu_has_attr(struct kvm_vcpu *vcpu,
1640 				 struct kvm_device_attr *attr)
1641 {
1642 	int ret = -ENXIO;
1643 
1644 	switch (attr->group) {
1645 	default:
1646 		ret = kvm_arm_vcpu_arch_has_attr(vcpu, attr);
1647 		break;
1648 	}
1649 
1650 	return ret;
1651 }
1652 
kvm_arm_vcpu_get_events(struct kvm_vcpu * vcpu,struct kvm_vcpu_events * events)1653 static int kvm_arm_vcpu_get_events(struct kvm_vcpu *vcpu,
1654 				   struct kvm_vcpu_events *events)
1655 {
1656 	memset(events, 0, sizeof(*events));
1657 
1658 	return __kvm_arm_vcpu_get_events(vcpu, events);
1659 }
1660 
kvm_arm_vcpu_set_events(struct kvm_vcpu * vcpu,struct kvm_vcpu_events * events)1661 static int kvm_arm_vcpu_set_events(struct kvm_vcpu *vcpu,
1662 				   struct kvm_vcpu_events *events)
1663 {
1664 	int i;
1665 
1666 	/* check whether the reserved field is zero */
1667 	for (i = 0; i < ARRAY_SIZE(events->reserved); i++)
1668 		if (events->reserved[i])
1669 			return -EINVAL;
1670 
1671 	/* check whether the pad field is zero */
1672 	for (i = 0; i < ARRAY_SIZE(events->exception.pad); i++)
1673 		if (events->exception.pad[i])
1674 			return -EINVAL;
1675 
1676 	return __kvm_arm_vcpu_set_events(vcpu, events);
1677 }
1678 
kvm_arch_vcpu_ioctl(struct file * filp,unsigned int ioctl,unsigned long arg)1679 long kvm_arch_vcpu_ioctl(struct file *filp,
1680 			 unsigned int ioctl, unsigned long arg)
1681 {
1682 	struct kvm_vcpu *vcpu = filp->private_data;
1683 	void __user *argp = (void __user *)arg;
1684 	struct kvm_device_attr attr;
1685 	long r;
1686 
1687 	switch (ioctl) {
1688 	case KVM_ARM_VCPU_INIT: {
1689 		struct kvm_vcpu_init init;
1690 
1691 		r = -EFAULT;
1692 		if (copy_from_user(&init, argp, sizeof(init)))
1693 			break;
1694 
1695 		r = kvm_arch_vcpu_ioctl_vcpu_init(vcpu, &init);
1696 		break;
1697 	}
1698 	case KVM_SET_ONE_REG:
1699 	case KVM_GET_ONE_REG: {
1700 		struct kvm_one_reg reg;
1701 
1702 		r = -ENOEXEC;
1703 		if (unlikely(!kvm_vcpu_initialized(vcpu)))
1704 			break;
1705 
1706 		r = -EFAULT;
1707 		if (copy_from_user(&reg, argp, sizeof(reg)))
1708 			break;
1709 
1710 		/*
1711 		 * We could owe a reset due to PSCI. Handle the pending reset
1712 		 * here to ensure userspace register accesses are ordered after
1713 		 * the reset.
1714 		 */
1715 		if (kvm_check_request(KVM_REQ_VCPU_RESET, vcpu))
1716 			kvm_reset_vcpu(vcpu);
1717 
1718 		if (ioctl == KVM_SET_ONE_REG)
1719 			r = kvm_arm_set_reg(vcpu, &reg);
1720 		else
1721 			r = kvm_arm_get_reg(vcpu, &reg);
1722 		break;
1723 	}
1724 	case KVM_GET_REG_LIST: {
1725 		struct kvm_reg_list __user *user_list = argp;
1726 		struct kvm_reg_list reg_list;
1727 		unsigned n;
1728 
1729 		r = -ENOEXEC;
1730 		if (unlikely(!kvm_vcpu_initialized(vcpu)))
1731 			break;
1732 
1733 		r = -EPERM;
1734 		if (!kvm_arm_vcpu_is_finalized(vcpu))
1735 			break;
1736 
1737 		r = -EFAULT;
1738 		if (copy_from_user(&reg_list, user_list, sizeof(reg_list)))
1739 			break;
1740 		n = reg_list.n;
1741 		reg_list.n = kvm_arm_num_regs(vcpu);
1742 		if (copy_to_user(user_list, &reg_list, sizeof(reg_list)))
1743 			break;
1744 		r = -E2BIG;
1745 		if (n < reg_list.n)
1746 			break;
1747 		r = kvm_arm_copy_reg_indices(vcpu, user_list->reg);
1748 		break;
1749 	}
1750 	case KVM_SET_DEVICE_ATTR: {
1751 		r = -EFAULT;
1752 		if (copy_from_user(&attr, argp, sizeof(attr)))
1753 			break;
1754 		r = kvm_arm_vcpu_set_attr(vcpu, &attr);
1755 		break;
1756 	}
1757 	case KVM_GET_DEVICE_ATTR: {
1758 		r = -EFAULT;
1759 		if (copy_from_user(&attr, argp, sizeof(attr)))
1760 			break;
1761 		r = kvm_arm_vcpu_get_attr(vcpu, &attr);
1762 		break;
1763 	}
1764 	case KVM_HAS_DEVICE_ATTR: {
1765 		r = -EFAULT;
1766 		if (copy_from_user(&attr, argp, sizeof(attr)))
1767 			break;
1768 		r = kvm_arm_vcpu_has_attr(vcpu, &attr);
1769 		break;
1770 	}
1771 	case KVM_GET_VCPU_EVENTS: {
1772 		struct kvm_vcpu_events events;
1773 
1774 		if (kvm_arm_vcpu_get_events(vcpu, &events))
1775 			return -EINVAL;
1776 
1777 		if (copy_to_user(argp, &events, sizeof(events)))
1778 			return -EFAULT;
1779 
1780 		return 0;
1781 	}
1782 	case KVM_SET_VCPU_EVENTS: {
1783 		struct kvm_vcpu_events events;
1784 
1785 		if (copy_from_user(&events, argp, sizeof(events)))
1786 			return -EFAULT;
1787 
1788 		return kvm_arm_vcpu_set_events(vcpu, &events);
1789 	}
1790 	case KVM_ARM_VCPU_FINALIZE: {
1791 		int what;
1792 
1793 		if (!kvm_vcpu_initialized(vcpu))
1794 			return -ENOEXEC;
1795 
1796 		if (get_user(what, (const int __user *)argp))
1797 			return -EFAULT;
1798 
1799 		return kvm_arm_vcpu_finalize(vcpu, what);
1800 	}
1801 	default:
1802 		r = -EINVAL;
1803 	}
1804 
1805 	return r;
1806 }
1807 
kvm_arch_sync_dirty_log(struct kvm * kvm,struct kvm_memory_slot * memslot)1808 void kvm_arch_sync_dirty_log(struct kvm *kvm, struct kvm_memory_slot *memslot)
1809 {
1810 
1811 }
1812 
kvm_vm_ioctl_set_device_addr(struct kvm * kvm,struct kvm_arm_device_addr * dev_addr)1813 static int kvm_vm_ioctl_set_device_addr(struct kvm *kvm,
1814 					struct kvm_arm_device_addr *dev_addr)
1815 {
1816 	switch (FIELD_GET(KVM_ARM_DEVICE_ID_MASK, dev_addr->id)) {
1817 	case KVM_ARM_DEVICE_VGIC_V2:
1818 		if (!vgic_present)
1819 			return -ENXIO;
1820 		return kvm_set_legacy_vgic_v2_addr(kvm, dev_addr);
1821 	default:
1822 		return -ENODEV;
1823 	}
1824 }
1825 
kvm_vm_has_attr(struct kvm * kvm,struct kvm_device_attr * attr)1826 static int kvm_vm_has_attr(struct kvm *kvm, struct kvm_device_attr *attr)
1827 {
1828 	switch (attr->group) {
1829 	case KVM_ARM_VM_SMCCC_CTRL:
1830 		return kvm_vm_smccc_has_attr(kvm, attr);
1831 	default:
1832 		return -ENXIO;
1833 	}
1834 }
1835 
kvm_vm_set_attr(struct kvm * kvm,struct kvm_device_attr * attr)1836 static int kvm_vm_set_attr(struct kvm *kvm, struct kvm_device_attr *attr)
1837 {
1838 	switch (attr->group) {
1839 	case KVM_ARM_VM_SMCCC_CTRL:
1840 		return kvm_vm_smccc_set_attr(kvm, attr);
1841 	default:
1842 		return -ENXIO;
1843 	}
1844 }
1845 
kvm_arch_vm_ioctl(struct file * filp,unsigned int ioctl,unsigned long arg)1846 int kvm_arch_vm_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg)
1847 {
1848 	struct kvm *kvm = filp->private_data;
1849 	void __user *argp = (void __user *)arg;
1850 	struct kvm_device_attr attr;
1851 
1852 	switch (ioctl) {
1853 	case KVM_CREATE_IRQCHIP: {
1854 		int ret;
1855 		if (!vgic_present)
1856 			return -ENXIO;
1857 		mutex_lock(&kvm->lock);
1858 		ret = kvm_vgic_create(kvm, KVM_DEV_TYPE_ARM_VGIC_V2);
1859 		mutex_unlock(&kvm->lock);
1860 		return ret;
1861 	}
1862 	case KVM_ARM_SET_DEVICE_ADDR: {
1863 		struct kvm_arm_device_addr dev_addr;
1864 
1865 		if (copy_from_user(&dev_addr, argp, sizeof(dev_addr)))
1866 			return -EFAULT;
1867 		return kvm_vm_ioctl_set_device_addr(kvm, &dev_addr);
1868 	}
1869 	case KVM_ARM_PREFERRED_TARGET: {
1870 		struct kvm_vcpu_init init = {
1871 			.target = KVM_ARM_TARGET_GENERIC_V8,
1872 		};
1873 
1874 		if (copy_to_user(argp, &init, sizeof(init)))
1875 			return -EFAULT;
1876 
1877 		return 0;
1878 	}
1879 	case KVM_ARM_MTE_COPY_TAGS: {
1880 		struct kvm_arm_copy_mte_tags copy_tags;
1881 
1882 		if (copy_from_user(&copy_tags, argp, sizeof(copy_tags)))
1883 			return -EFAULT;
1884 		return kvm_vm_ioctl_mte_copy_tags(kvm, &copy_tags);
1885 	}
1886 	case KVM_ARM_SET_COUNTER_OFFSET: {
1887 		struct kvm_arm_counter_offset offset;
1888 
1889 		if (copy_from_user(&offset, argp, sizeof(offset)))
1890 			return -EFAULT;
1891 		return kvm_vm_ioctl_set_counter_offset(kvm, &offset);
1892 	}
1893 	case KVM_HAS_DEVICE_ATTR: {
1894 		if (copy_from_user(&attr, argp, sizeof(attr)))
1895 			return -EFAULT;
1896 
1897 		return kvm_vm_has_attr(kvm, &attr);
1898 	}
1899 	case KVM_SET_DEVICE_ATTR: {
1900 		if (copy_from_user(&attr, argp, sizeof(attr)))
1901 			return -EFAULT;
1902 
1903 		return kvm_vm_set_attr(kvm, &attr);
1904 	}
1905 	case KVM_ARM_GET_REG_WRITABLE_MASKS: {
1906 		struct reg_mask_range range;
1907 
1908 		if (copy_from_user(&range, argp, sizeof(range)))
1909 			return -EFAULT;
1910 		return kvm_vm_ioctl_get_reg_writable_masks(kvm, &range);
1911 	}
1912 	default:
1913 		return -EINVAL;
1914 	}
1915 }
1916 
1917 /* unlocks vcpus from @vcpu_lock_idx and smaller */
unlock_vcpus(struct kvm * kvm,int vcpu_lock_idx)1918 static void unlock_vcpus(struct kvm *kvm, int vcpu_lock_idx)
1919 {
1920 	struct kvm_vcpu *tmp_vcpu;
1921 
1922 	for (; vcpu_lock_idx >= 0; vcpu_lock_idx--) {
1923 		tmp_vcpu = kvm_get_vcpu(kvm, vcpu_lock_idx);
1924 		mutex_unlock(&tmp_vcpu->mutex);
1925 	}
1926 }
1927 
unlock_all_vcpus(struct kvm * kvm)1928 void unlock_all_vcpus(struct kvm *kvm)
1929 {
1930 	lockdep_assert_held(&kvm->lock);
1931 
1932 	unlock_vcpus(kvm, atomic_read(&kvm->online_vcpus) - 1);
1933 }
1934 
1935 /* Returns true if all vcpus were locked, false otherwise */
lock_all_vcpus(struct kvm * kvm)1936 bool lock_all_vcpus(struct kvm *kvm)
1937 {
1938 	struct kvm_vcpu *tmp_vcpu;
1939 	unsigned long c;
1940 
1941 	lockdep_assert_held(&kvm->lock);
1942 
1943 	/*
1944 	 * Any time a vcpu is in an ioctl (including running), the
1945 	 * core KVM code tries to grab the vcpu->mutex.
1946 	 *
1947 	 * By grabbing the vcpu->mutex of all VCPUs we ensure that no
1948 	 * other VCPUs can fiddle with the state while we access it.
1949 	 */
1950 	kvm_for_each_vcpu(c, tmp_vcpu, kvm) {
1951 		if (!mutex_trylock(&tmp_vcpu->mutex)) {
1952 			unlock_vcpus(kvm, c - 1);
1953 			return false;
1954 		}
1955 	}
1956 
1957 	return true;
1958 }
1959 
nvhe_percpu_size(void)1960 static unsigned long nvhe_percpu_size(void)
1961 {
1962 	return (unsigned long)CHOOSE_NVHE_SYM(__per_cpu_end) -
1963 		(unsigned long)CHOOSE_NVHE_SYM(__per_cpu_start);
1964 }
1965 
nvhe_percpu_order(void)1966 static unsigned long nvhe_percpu_order(void)
1967 {
1968 	unsigned long size = nvhe_percpu_size();
1969 
1970 	return size ? get_order(size) : 0;
1971 }
1972 
pkvm_host_sve_state_order(void)1973 static size_t pkvm_host_sve_state_order(void)
1974 {
1975 	return get_order(pkvm_host_sve_state_size());
1976 }
1977 
1978 /* A lookup table holding the hypervisor VA for each vector slot */
1979 static void *hyp_spectre_vector_selector[BP_HARDEN_EL2_SLOTS];
1980 
kvm_init_vector_slot(void * base,enum arm64_hyp_spectre_vector slot)1981 static void kvm_init_vector_slot(void *base, enum arm64_hyp_spectre_vector slot)
1982 {
1983 	hyp_spectre_vector_selector[slot] = __kvm_vector_slot2addr(base, slot);
1984 }
1985 
kvm_init_vector_slots(void)1986 static int kvm_init_vector_slots(void)
1987 {
1988 	int err;
1989 	void *base;
1990 
1991 	base = kern_hyp_va(kvm_ksym_ref(__kvm_hyp_vector));
1992 	kvm_init_vector_slot(base, HYP_VECTOR_DIRECT);
1993 
1994 	base = kern_hyp_va(kvm_ksym_ref(__bp_harden_hyp_vecs));
1995 	kvm_init_vector_slot(base, HYP_VECTOR_SPECTRE_DIRECT);
1996 
1997 	if (kvm_system_needs_idmapped_vectors() &&
1998 	    !is_protected_kvm_enabled()) {
1999 		err = create_hyp_exec_mappings(__pa_symbol(__bp_harden_hyp_vecs),
2000 					       __BP_HARDEN_HYP_VECS_SZ, &base);
2001 		if (err)
2002 			return err;
2003 	}
2004 
2005 	kvm_init_vector_slot(base, HYP_VECTOR_INDIRECT);
2006 	kvm_init_vector_slot(base, HYP_VECTOR_SPECTRE_INDIRECT);
2007 	return 0;
2008 }
2009 
cpu_prepare_hyp_mode(int cpu,u32 hyp_va_bits)2010 static void __init cpu_prepare_hyp_mode(int cpu, u32 hyp_va_bits)
2011 {
2012 	struct kvm_nvhe_init_params *params = per_cpu_ptr_nvhe_sym(kvm_init_params, cpu);
2013 	unsigned long tcr;
2014 
2015 	/*
2016 	 * Calculate the raw per-cpu offset without a translation from the
2017 	 * kernel's mapping to the linear mapping, and store it in tpidr_el2
2018 	 * so that we can use adr_l to access per-cpu variables in EL2.
2019 	 * Also drop the KASAN tag which gets in the way...
2020 	 */
2021 	params->tpidr_el2 = (unsigned long)kasan_reset_tag(per_cpu_ptr_nvhe_sym(__per_cpu_start, cpu)) -
2022 			    (unsigned long)kvm_ksym_ref(CHOOSE_NVHE_SYM(__per_cpu_start));
2023 
2024 	params->mair_el2 = read_sysreg(mair_el1);
2025 
2026 	tcr = read_sysreg(tcr_el1);
2027 	if (cpus_have_final_cap(ARM64_KVM_HVHE)) {
2028 		tcr &= ~(TCR_HD | TCR_HA | TCR_A1 | TCR_T0SZ_MASK);
2029 		tcr |= TCR_EPD1_MASK;
2030 	} else {
2031 		unsigned long ips = FIELD_GET(TCR_IPS_MASK, tcr);
2032 
2033 		tcr &= TCR_EL2_MASK;
2034 		tcr |= TCR_EL2_RES1 | FIELD_PREP(TCR_EL2_PS_MASK, ips);
2035 		if (lpa2_is_enabled())
2036 			tcr |= TCR_EL2_DS;
2037 	}
2038 	tcr |= TCR_T0SZ(hyp_va_bits);
2039 	params->tcr_el2 = tcr;
2040 
2041 	params->pgd_pa = kvm_mmu_get_httbr();
2042 	if (is_protected_kvm_enabled())
2043 		params->hcr_el2 = HCR_HOST_NVHE_PROTECTED_FLAGS;
2044 	else
2045 		params->hcr_el2 = HCR_HOST_NVHE_FLAGS;
2046 	if (cpus_have_final_cap(ARM64_KVM_HVHE))
2047 		params->hcr_el2 |= HCR_E2H;
2048 	params->vttbr = params->vtcr = 0;
2049 
2050 	/*
2051 	 * Flush the init params from the data cache because the struct will
2052 	 * be read while the MMU is off.
2053 	 */
2054 	kvm_flush_dcache_to_poc(params, sizeof(*params));
2055 }
2056 
hyp_install_host_vector(void)2057 static void hyp_install_host_vector(void)
2058 {
2059 	struct kvm_nvhe_init_params *params;
2060 	struct arm_smccc_res res;
2061 
2062 	/* Switch from the HYP stub to our own HYP init vector */
2063 	__hyp_set_vectors(kvm_get_idmap_vector());
2064 
2065 	/*
2066 	 * Call initialization code, and switch to the full blown HYP code.
2067 	 * If the cpucaps haven't been finalized yet, something has gone very
2068 	 * wrong, and hyp will crash and burn when it uses any
2069 	 * cpus_have_*_cap() wrapper.
2070 	 */
2071 	BUG_ON(!system_capabilities_finalized());
2072 	params = this_cpu_ptr_nvhe_sym(kvm_init_params);
2073 	arm_smccc_1_1_hvc(KVM_HOST_SMCCC_FUNC(__kvm_hyp_init), virt_to_phys(params), &res);
2074 	WARN_ON(res.a0 != SMCCC_RET_SUCCESS);
2075 }
2076 
cpu_init_hyp_mode(void)2077 static void cpu_init_hyp_mode(void)
2078 {
2079 	hyp_install_host_vector();
2080 
2081 	/*
2082 	 * Disabling SSBD on a non-VHE system requires us to enable SSBS
2083 	 * at EL2.
2084 	 */
2085 	if (this_cpu_has_cap(ARM64_SSBS) &&
2086 	    arm64_get_spectre_v4_state() == SPECTRE_VULNERABLE) {
2087 		kvm_call_hyp_nvhe(__kvm_enable_ssbs);
2088 	}
2089 }
2090 
cpu_hyp_reset(void)2091 static void cpu_hyp_reset(void)
2092 {
2093 	if (!is_kernel_in_hyp_mode())
2094 		__hyp_reset_vectors();
2095 }
2096 
2097 /*
2098  * EL2 vectors can be mapped and rerouted in a number of ways,
2099  * depending on the kernel configuration and CPU present:
2100  *
2101  * - If the CPU is affected by Spectre-v2, the hardening sequence is
2102  *   placed in one of the vector slots, which is executed before jumping
2103  *   to the real vectors.
2104  *
2105  * - If the CPU also has the ARM64_SPECTRE_V3A cap, the slot
2106  *   containing the hardening sequence is mapped next to the idmap page,
2107  *   and executed before jumping to the real vectors.
2108  *
2109  * - If the CPU only has the ARM64_SPECTRE_V3A cap, then an
2110  *   empty slot is selected, mapped next to the idmap page, and
2111  *   executed before jumping to the real vectors.
2112  *
2113  * Note that ARM64_SPECTRE_V3A is somewhat incompatible with
2114  * VHE, as we don't have hypervisor-specific mappings. If the system
2115  * is VHE and yet selects this capability, it will be ignored.
2116  */
cpu_set_hyp_vector(void)2117 static void cpu_set_hyp_vector(void)
2118 {
2119 	struct bp_hardening_data *data = this_cpu_ptr(&bp_hardening_data);
2120 	void *vector = hyp_spectre_vector_selector[data->slot];
2121 
2122 	if (!is_protected_kvm_enabled())
2123 		*this_cpu_ptr_hyp_sym(kvm_hyp_vector) = (unsigned long)vector;
2124 	else
2125 		kvm_call_hyp_nvhe(__pkvm_cpu_set_vector, data->slot);
2126 }
2127 
cpu_hyp_init_context(void)2128 static void cpu_hyp_init_context(void)
2129 {
2130 	kvm_init_host_cpu_context(host_data_ptr(host_ctxt));
2131 	kvm_init_host_debug_data();
2132 
2133 	if (!is_kernel_in_hyp_mode())
2134 		cpu_init_hyp_mode();
2135 }
2136 
cpu_hyp_init_features(void)2137 static void cpu_hyp_init_features(void)
2138 {
2139 	cpu_set_hyp_vector();
2140 
2141 	if (is_kernel_in_hyp_mode())
2142 		kvm_timer_init_vhe();
2143 
2144 	if (vgic_present)
2145 		kvm_vgic_init_cpu_hardware();
2146 }
2147 
cpu_hyp_reinit(void)2148 static void cpu_hyp_reinit(void)
2149 {
2150 	cpu_hyp_reset();
2151 	cpu_hyp_init_context();
2152 	cpu_hyp_init_features();
2153 }
2154 
cpu_hyp_init(void * discard)2155 static void cpu_hyp_init(void *discard)
2156 {
2157 	if (!__this_cpu_read(kvm_hyp_initialized)) {
2158 		cpu_hyp_reinit();
2159 		__this_cpu_write(kvm_hyp_initialized, 1);
2160 	}
2161 }
2162 
cpu_hyp_uninit(void * discard)2163 static void cpu_hyp_uninit(void *discard)
2164 {
2165 	if (__this_cpu_read(kvm_hyp_initialized)) {
2166 		cpu_hyp_reset();
2167 		__this_cpu_write(kvm_hyp_initialized, 0);
2168 	}
2169 }
2170 
kvm_arch_enable_virtualization_cpu(void)2171 int kvm_arch_enable_virtualization_cpu(void)
2172 {
2173 	/*
2174 	 * Most calls to this function are made with migration
2175 	 * disabled, but not with preemption disabled. The former is
2176 	 * enough to ensure correctness, but most of the helpers
2177 	 * expect the later and will throw a tantrum otherwise.
2178 	 */
2179 	preempt_disable();
2180 
2181 	cpu_hyp_init(NULL);
2182 
2183 	kvm_vgic_cpu_up();
2184 	kvm_timer_cpu_up();
2185 
2186 	preempt_enable();
2187 
2188 	return 0;
2189 }
2190 
kvm_arch_disable_virtualization_cpu(void)2191 void kvm_arch_disable_virtualization_cpu(void)
2192 {
2193 	kvm_timer_cpu_down();
2194 	kvm_vgic_cpu_down();
2195 
2196 	if (!is_protected_kvm_enabled())
2197 		cpu_hyp_uninit(NULL);
2198 }
2199 
2200 #ifdef CONFIG_CPU_PM
hyp_init_cpu_pm_notifier(struct notifier_block * self,unsigned long cmd,void * v)2201 static int hyp_init_cpu_pm_notifier(struct notifier_block *self,
2202 				    unsigned long cmd,
2203 				    void *v)
2204 {
2205 	/*
2206 	 * kvm_hyp_initialized is left with its old value over
2207 	 * PM_ENTER->PM_EXIT. It is used to indicate PM_EXIT should
2208 	 * re-enable hyp.
2209 	 */
2210 	switch (cmd) {
2211 	case CPU_PM_ENTER:
2212 		if (__this_cpu_read(kvm_hyp_initialized))
2213 			/*
2214 			 * don't update kvm_hyp_initialized here
2215 			 * so that the hyp will be re-enabled
2216 			 * when we resume. See below.
2217 			 */
2218 			cpu_hyp_reset();
2219 
2220 		return NOTIFY_OK;
2221 	case CPU_PM_ENTER_FAILED:
2222 	case CPU_PM_EXIT:
2223 		if (__this_cpu_read(kvm_hyp_initialized))
2224 			/* The hyp was enabled before suspend. */
2225 			cpu_hyp_reinit();
2226 
2227 		return NOTIFY_OK;
2228 
2229 	default:
2230 		return NOTIFY_DONE;
2231 	}
2232 }
2233 
2234 static struct notifier_block hyp_init_cpu_pm_nb = {
2235 	.notifier_call = hyp_init_cpu_pm_notifier,
2236 };
2237 
hyp_cpu_pm_init(void)2238 static void __init hyp_cpu_pm_init(void)
2239 {
2240 	if (!is_protected_kvm_enabled())
2241 		cpu_pm_register_notifier(&hyp_init_cpu_pm_nb);
2242 }
hyp_cpu_pm_exit(void)2243 static void __init hyp_cpu_pm_exit(void)
2244 {
2245 	if (!is_protected_kvm_enabled())
2246 		cpu_pm_unregister_notifier(&hyp_init_cpu_pm_nb);
2247 }
2248 #else
hyp_cpu_pm_init(void)2249 static inline void __init hyp_cpu_pm_init(void)
2250 {
2251 }
hyp_cpu_pm_exit(void)2252 static inline void __init hyp_cpu_pm_exit(void)
2253 {
2254 }
2255 #endif
2256 
init_cpu_logical_map(void)2257 static void __init init_cpu_logical_map(void)
2258 {
2259 	unsigned int cpu;
2260 
2261 	/*
2262 	 * Copy the MPIDR <-> logical CPU ID mapping to hyp.
2263 	 * Only copy the set of online CPUs whose features have been checked
2264 	 * against the finalized system capabilities. The hypervisor will not
2265 	 * allow any other CPUs from the `possible` set to boot.
2266 	 */
2267 	for_each_online_cpu(cpu)
2268 		hyp_cpu_logical_map[cpu] = cpu_logical_map(cpu);
2269 }
2270 
2271 #define init_psci_0_1_impl_state(config, what)	\
2272 	config.psci_0_1_ ## what ## _implemented = psci_ops.what
2273 
init_psci_relay(void)2274 static bool __init init_psci_relay(void)
2275 {
2276 	/*
2277 	 * If PSCI has not been initialized, protected KVM cannot install
2278 	 * itself on newly booted CPUs.
2279 	 */
2280 	if (!psci_ops.get_version) {
2281 		kvm_err("Cannot initialize protected mode without PSCI\n");
2282 		return false;
2283 	}
2284 
2285 	kvm_host_psci_config.version = psci_ops.get_version();
2286 	kvm_host_psci_config.smccc_version = arm_smccc_get_version();
2287 
2288 	if (kvm_host_psci_config.version == PSCI_VERSION(0, 1)) {
2289 		kvm_host_psci_config.function_ids_0_1 = get_psci_0_1_function_ids();
2290 		init_psci_0_1_impl_state(kvm_host_psci_config, cpu_suspend);
2291 		init_psci_0_1_impl_state(kvm_host_psci_config, cpu_on);
2292 		init_psci_0_1_impl_state(kvm_host_psci_config, cpu_off);
2293 		init_psci_0_1_impl_state(kvm_host_psci_config, migrate);
2294 	}
2295 	return true;
2296 }
2297 
init_subsystems(void)2298 static int __init init_subsystems(void)
2299 {
2300 	int err = 0;
2301 
2302 	/*
2303 	 * Enable hardware so that subsystem initialisation can access EL2.
2304 	 */
2305 	on_each_cpu(cpu_hyp_init, NULL, 1);
2306 
2307 	/*
2308 	 * Register CPU lower-power notifier
2309 	 */
2310 	hyp_cpu_pm_init();
2311 
2312 	/*
2313 	 * Init HYP view of VGIC
2314 	 */
2315 	err = kvm_vgic_hyp_init();
2316 	switch (err) {
2317 	case 0:
2318 		vgic_present = true;
2319 		break;
2320 	case -ENODEV:
2321 	case -ENXIO:
2322 		/*
2323 		 * No VGIC? No pKVM for you.
2324 		 *
2325 		 * Protected mode assumes that VGICv3 is present, so no point
2326 		 * in trying to hobble along if vgic initialization fails.
2327 		 */
2328 		if (is_protected_kvm_enabled())
2329 			goto out;
2330 
2331 		/*
2332 		 * Otherwise, userspace could choose to implement a GIC for its
2333 		 * guest on non-cooperative hardware.
2334 		 */
2335 		vgic_present = false;
2336 		err = 0;
2337 		break;
2338 	default:
2339 		goto out;
2340 	}
2341 
2342 	if (kvm_mode == KVM_MODE_NV &&
2343 	   !(vgic_present && kvm_vgic_global_state.type == VGIC_V3)) {
2344 		kvm_err("NV support requires GICv3, giving up\n");
2345 		err = -EINVAL;
2346 		goto out;
2347 	}
2348 
2349 	/*
2350 	 * Init HYP architected timer support
2351 	 */
2352 	err = kvm_timer_hyp_init(vgic_present);
2353 	if (err)
2354 		goto out;
2355 
2356 	kvm_register_perf_callbacks(NULL);
2357 
2358 out:
2359 	if (err)
2360 		hyp_cpu_pm_exit();
2361 
2362 	if (err || !is_protected_kvm_enabled())
2363 		on_each_cpu(cpu_hyp_uninit, NULL, 1);
2364 
2365 	return err;
2366 }
2367 
teardown_subsystems(void)2368 static void __init teardown_subsystems(void)
2369 {
2370 	kvm_unregister_perf_callbacks();
2371 	hyp_cpu_pm_exit();
2372 }
2373 
teardown_hyp_mode(void)2374 static void __init teardown_hyp_mode(void)
2375 {
2376 	bool free_sve = system_supports_sve() && is_protected_kvm_enabled();
2377 	int cpu;
2378 
2379 	free_hyp_pgds();
2380 	for_each_possible_cpu(cpu) {
2381 		free_pages(per_cpu(kvm_arm_hyp_stack_base, cpu), NVHE_STACK_SHIFT - PAGE_SHIFT);
2382 		free_pages(kvm_nvhe_sym(kvm_arm_hyp_percpu_base)[cpu], nvhe_percpu_order());
2383 
2384 		if (free_sve) {
2385 			struct cpu_sve_state *sve_state;
2386 
2387 			sve_state = per_cpu_ptr_nvhe_sym(kvm_host_data, cpu)->sve_state;
2388 			free_pages((unsigned long) sve_state, pkvm_host_sve_state_order());
2389 		}
2390 	}
2391 }
2392 
do_pkvm_init(u32 hyp_va_bits)2393 static int __init do_pkvm_init(u32 hyp_va_bits)
2394 {
2395 	void *per_cpu_base = kvm_ksym_ref(kvm_nvhe_sym(kvm_arm_hyp_percpu_base));
2396 	int ret;
2397 
2398 	preempt_disable();
2399 	cpu_hyp_init_context();
2400 	ret = kvm_call_hyp_nvhe(__pkvm_init, hyp_mem_base, hyp_mem_size,
2401 				num_possible_cpus(), kern_hyp_va(per_cpu_base),
2402 				hyp_va_bits);
2403 	cpu_hyp_init_features();
2404 
2405 	/*
2406 	 * The stub hypercalls are now disabled, so set our local flag to
2407 	 * prevent a later re-init attempt in kvm_arch_enable_virtualization_cpu().
2408 	 */
2409 	__this_cpu_write(kvm_hyp_initialized, 1);
2410 	preempt_enable();
2411 
2412 	return ret;
2413 }
2414 
get_hyp_id_aa64pfr0_el1(void)2415 static u64 get_hyp_id_aa64pfr0_el1(void)
2416 {
2417 	/*
2418 	 * Track whether the system isn't affected by spectre/meltdown in the
2419 	 * hypervisor's view of id_aa64pfr0_el1, used for protected VMs.
2420 	 * Although this is per-CPU, we make it global for simplicity, e.g., not
2421 	 * to have to worry about vcpu migration.
2422 	 *
2423 	 * Unlike for non-protected VMs, userspace cannot override this for
2424 	 * protected VMs.
2425 	 */
2426 	u64 val = read_sanitised_ftr_reg(SYS_ID_AA64PFR0_EL1);
2427 
2428 	val &= ~(ARM64_FEATURE_MASK(ID_AA64PFR0_EL1_CSV2) |
2429 		 ARM64_FEATURE_MASK(ID_AA64PFR0_EL1_CSV3));
2430 
2431 	val |= FIELD_PREP(ARM64_FEATURE_MASK(ID_AA64PFR0_EL1_CSV2),
2432 			  arm64_get_spectre_v2_state() == SPECTRE_UNAFFECTED);
2433 	val |= FIELD_PREP(ARM64_FEATURE_MASK(ID_AA64PFR0_EL1_CSV3),
2434 			  arm64_get_meltdown_state() == SPECTRE_UNAFFECTED);
2435 
2436 	return val;
2437 }
2438 
kvm_hyp_init_symbols(void)2439 static void kvm_hyp_init_symbols(void)
2440 {
2441 	kvm_nvhe_sym(id_aa64pfr0_el1_sys_val) = get_hyp_id_aa64pfr0_el1();
2442 	kvm_nvhe_sym(id_aa64pfr1_el1_sys_val) = read_sanitised_ftr_reg(SYS_ID_AA64PFR1_EL1);
2443 	kvm_nvhe_sym(id_aa64isar0_el1_sys_val) = read_sanitised_ftr_reg(SYS_ID_AA64ISAR0_EL1);
2444 	kvm_nvhe_sym(id_aa64isar1_el1_sys_val) = read_sanitised_ftr_reg(SYS_ID_AA64ISAR1_EL1);
2445 	kvm_nvhe_sym(id_aa64isar2_el1_sys_val) = read_sanitised_ftr_reg(SYS_ID_AA64ISAR2_EL1);
2446 	kvm_nvhe_sym(id_aa64mmfr0_el1_sys_val) = read_sanitised_ftr_reg(SYS_ID_AA64MMFR0_EL1);
2447 	kvm_nvhe_sym(id_aa64mmfr1_el1_sys_val) = read_sanitised_ftr_reg(SYS_ID_AA64MMFR1_EL1);
2448 	kvm_nvhe_sym(id_aa64mmfr2_el1_sys_val) = read_sanitised_ftr_reg(SYS_ID_AA64MMFR2_EL1);
2449 	kvm_nvhe_sym(id_aa64smfr0_el1_sys_val) = read_sanitised_ftr_reg(SYS_ID_AA64SMFR0_EL1);
2450 	kvm_nvhe_sym(__icache_flags) = __icache_flags;
2451 	kvm_nvhe_sym(kvm_arm_vmid_bits) = kvm_arm_vmid_bits;
2452 
2453 	/*
2454 	 * Flush entire BSS since part of its data containing init symbols is read
2455 	 * while the MMU is off.
2456 	 */
2457 	kvm_flush_dcache_to_poc(kvm_ksym_ref(__hyp_bss_start),
2458 				kvm_ksym_ref(__hyp_bss_end) - kvm_ksym_ref(__hyp_bss_start));
2459 }
2460 
kvm_hyp_init_protection(u32 hyp_va_bits)2461 static int __init kvm_hyp_init_protection(u32 hyp_va_bits)
2462 {
2463 	void *addr = phys_to_virt(hyp_mem_base);
2464 	int ret;
2465 
2466 	ret = create_hyp_mappings(addr, addr + hyp_mem_size, PAGE_HYP);
2467 	if (ret)
2468 		return ret;
2469 
2470 	ret = do_pkvm_init(hyp_va_bits);
2471 	if (ret)
2472 		return ret;
2473 
2474 	free_hyp_pgds();
2475 
2476 	return 0;
2477 }
2478 
init_pkvm_host_sve_state(void)2479 static int init_pkvm_host_sve_state(void)
2480 {
2481 	int cpu;
2482 
2483 	if (!system_supports_sve())
2484 		return 0;
2485 
2486 	/* Allocate pages for host sve state in protected mode. */
2487 	for_each_possible_cpu(cpu) {
2488 		struct page *page = alloc_pages(GFP_KERNEL, pkvm_host_sve_state_order());
2489 
2490 		if (!page)
2491 			return -ENOMEM;
2492 
2493 		per_cpu_ptr_nvhe_sym(kvm_host_data, cpu)->sve_state = page_address(page);
2494 	}
2495 
2496 	/*
2497 	 * Don't map the pages in hyp since these are only used in protected
2498 	 * mode, which will (re)create its own mapping when initialized.
2499 	 */
2500 
2501 	return 0;
2502 }
2503 
2504 /*
2505  * Finalizes the initialization of hyp mode, once everything else is initialized
2506  * and the initialziation process cannot fail.
2507  */
finalize_init_hyp_mode(void)2508 static void finalize_init_hyp_mode(void)
2509 {
2510 	int cpu;
2511 
2512 	if (system_supports_sve() && is_protected_kvm_enabled()) {
2513 		for_each_possible_cpu(cpu) {
2514 			struct cpu_sve_state *sve_state;
2515 
2516 			sve_state = per_cpu_ptr_nvhe_sym(kvm_host_data, cpu)->sve_state;
2517 			per_cpu_ptr_nvhe_sym(kvm_host_data, cpu)->sve_state =
2518 				kern_hyp_va(sve_state);
2519 		}
2520 	}
2521 }
2522 
pkvm_hyp_init_ptrauth(void)2523 static void pkvm_hyp_init_ptrauth(void)
2524 {
2525 	struct kvm_cpu_context *hyp_ctxt;
2526 	int cpu;
2527 
2528 	for_each_possible_cpu(cpu) {
2529 		hyp_ctxt = per_cpu_ptr_nvhe_sym(kvm_hyp_ctxt, cpu);
2530 		hyp_ctxt->sys_regs[APIAKEYLO_EL1] = get_random_long();
2531 		hyp_ctxt->sys_regs[APIAKEYHI_EL1] = get_random_long();
2532 		hyp_ctxt->sys_regs[APIBKEYLO_EL1] = get_random_long();
2533 		hyp_ctxt->sys_regs[APIBKEYHI_EL1] = get_random_long();
2534 		hyp_ctxt->sys_regs[APDAKEYLO_EL1] = get_random_long();
2535 		hyp_ctxt->sys_regs[APDAKEYHI_EL1] = get_random_long();
2536 		hyp_ctxt->sys_regs[APDBKEYLO_EL1] = get_random_long();
2537 		hyp_ctxt->sys_regs[APDBKEYHI_EL1] = get_random_long();
2538 		hyp_ctxt->sys_regs[APGAKEYLO_EL1] = get_random_long();
2539 		hyp_ctxt->sys_regs[APGAKEYHI_EL1] = get_random_long();
2540 	}
2541 }
2542 
2543 /* Inits Hyp-mode on all online CPUs */
init_hyp_mode(void)2544 static int __init init_hyp_mode(void)
2545 {
2546 	u32 hyp_va_bits;
2547 	int cpu;
2548 	int err = -ENOMEM;
2549 
2550 	/*
2551 	 * The protected Hyp-mode cannot be initialized if the memory pool
2552 	 * allocation has failed.
2553 	 */
2554 	if (is_protected_kvm_enabled() && !hyp_mem_base)
2555 		goto out_err;
2556 
2557 	/*
2558 	 * Allocate Hyp PGD and setup Hyp identity mapping
2559 	 */
2560 	err = kvm_mmu_init(&hyp_va_bits);
2561 	if (err)
2562 		goto out_err;
2563 
2564 	/*
2565 	 * Allocate stack pages for Hypervisor-mode
2566 	 */
2567 	for_each_possible_cpu(cpu) {
2568 		unsigned long stack_base;
2569 
2570 		stack_base = __get_free_pages(GFP_KERNEL, NVHE_STACK_SHIFT - PAGE_SHIFT);
2571 		if (!stack_base) {
2572 			err = -ENOMEM;
2573 			goto out_err;
2574 		}
2575 
2576 		per_cpu(kvm_arm_hyp_stack_base, cpu) = stack_base;
2577 	}
2578 
2579 	/*
2580 	 * Allocate and initialize pages for Hypervisor-mode percpu regions.
2581 	 */
2582 	for_each_possible_cpu(cpu) {
2583 		struct page *page;
2584 		void *page_addr;
2585 
2586 		page = alloc_pages(GFP_KERNEL, nvhe_percpu_order());
2587 		if (!page) {
2588 			err = -ENOMEM;
2589 			goto out_err;
2590 		}
2591 
2592 		page_addr = page_address(page);
2593 		memcpy(page_addr, CHOOSE_NVHE_SYM(__per_cpu_start), nvhe_percpu_size());
2594 		kvm_nvhe_sym(kvm_arm_hyp_percpu_base)[cpu] = (unsigned long)page_addr;
2595 	}
2596 
2597 	/*
2598 	 * Map the Hyp-code called directly from the host
2599 	 */
2600 	err = create_hyp_mappings(kvm_ksym_ref(__hyp_text_start),
2601 				  kvm_ksym_ref(__hyp_text_end), PAGE_HYP_EXEC);
2602 	if (err) {
2603 		kvm_err("Cannot map world-switch code\n");
2604 		goto out_err;
2605 	}
2606 
2607 	err = create_hyp_mappings(kvm_ksym_ref(__hyp_rodata_start),
2608 				  kvm_ksym_ref(__hyp_rodata_end), PAGE_HYP_RO);
2609 	if (err) {
2610 		kvm_err("Cannot map .hyp.rodata section\n");
2611 		goto out_err;
2612 	}
2613 
2614 	err = create_hyp_mappings(kvm_ksym_ref(__start_rodata),
2615 				  kvm_ksym_ref(__end_rodata), PAGE_HYP_RO);
2616 	if (err) {
2617 		kvm_err("Cannot map rodata section\n");
2618 		goto out_err;
2619 	}
2620 
2621 	/*
2622 	 * .hyp.bss is guaranteed to be placed at the beginning of the .bss
2623 	 * section thanks to an assertion in the linker script. Map it RW and
2624 	 * the rest of .bss RO.
2625 	 */
2626 	err = create_hyp_mappings(kvm_ksym_ref(__hyp_bss_start),
2627 				  kvm_ksym_ref(__hyp_bss_end), PAGE_HYP);
2628 	if (err) {
2629 		kvm_err("Cannot map hyp bss section: %d\n", err);
2630 		goto out_err;
2631 	}
2632 
2633 	err = create_hyp_mappings(kvm_ksym_ref(__hyp_bss_end),
2634 				  kvm_ksym_ref(__bss_stop), PAGE_HYP_RO);
2635 	if (err) {
2636 		kvm_err("Cannot map bss section\n");
2637 		goto out_err;
2638 	}
2639 
2640 	/*
2641 	 * Map the Hyp stack pages
2642 	 */
2643 	for_each_possible_cpu(cpu) {
2644 		struct kvm_nvhe_init_params *params = per_cpu_ptr_nvhe_sym(kvm_init_params, cpu);
2645 		char *stack_base = (char *)per_cpu(kvm_arm_hyp_stack_base, cpu);
2646 
2647 		err = create_hyp_stack(__pa(stack_base), &params->stack_hyp_va);
2648 		if (err) {
2649 			kvm_err("Cannot map hyp stack\n");
2650 			goto out_err;
2651 		}
2652 
2653 		/*
2654 		 * Save the stack PA in nvhe_init_params. This will be needed
2655 		 * to recreate the stack mapping in protected nVHE mode.
2656 		 * __hyp_pa() won't do the right thing there, since the stack
2657 		 * has been mapped in the flexible private VA space.
2658 		 */
2659 		params->stack_pa = __pa(stack_base);
2660 	}
2661 
2662 	for_each_possible_cpu(cpu) {
2663 		char *percpu_begin = (char *)kvm_nvhe_sym(kvm_arm_hyp_percpu_base)[cpu];
2664 		char *percpu_end = percpu_begin + nvhe_percpu_size();
2665 
2666 		/* Map Hyp percpu pages */
2667 		err = create_hyp_mappings(percpu_begin, percpu_end, PAGE_HYP);
2668 		if (err) {
2669 			kvm_err("Cannot map hyp percpu region\n");
2670 			goto out_err;
2671 		}
2672 
2673 		/* Prepare the CPU initialization parameters */
2674 		cpu_prepare_hyp_mode(cpu, hyp_va_bits);
2675 	}
2676 
2677 	kvm_hyp_init_symbols();
2678 
2679 	if (is_protected_kvm_enabled()) {
2680 		if (IS_ENABLED(CONFIG_ARM64_PTR_AUTH_KERNEL) &&
2681 		    cpus_have_final_cap(ARM64_HAS_ADDRESS_AUTH))
2682 			pkvm_hyp_init_ptrauth();
2683 
2684 		init_cpu_logical_map();
2685 
2686 		if (!init_psci_relay()) {
2687 			err = -ENODEV;
2688 			goto out_err;
2689 		}
2690 
2691 		err = init_pkvm_host_sve_state();
2692 		if (err)
2693 			goto out_err;
2694 
2695 		err = kvm_hyp_init_protection(hyp_va_bits);
2696 		if (err) {
2697 			kvm_err("Failed to init hyp memory protection\n");
2698 			goto out_err;
2699 		}
2700 	}
2701 
2702 	return 0;
2703 
2704 out_err:
2705 	teardown_hyp_mode();
2706 	kvm_err("error initializing Hyp mode: %d\n", err);
2707 	return err;
2708 }
2709 
kvm_mpidr_to_vcpu(struct kvm * kvm,unsigned long mpidr)2710 struct kvm_vcpu *kvm_mpidr_to_vcpu(struct kvm *kvm, unsigned long mpidr)
2711 {
2712 	struct kvm_vcpu *vcpu = NULL;
2713 	struct kvm_mpidr_data *data;
2714 	unsigned long i;
2715 
2716 	mpidr &= MPIDR_HWID_BITMASK;
2717 
2718 	rcu_read_lock();
2719 	data = rcu_dereference(kvm->arch.mpidr_data);
2720 
2721 	if (data) {
2722 		u16 idx = kvm_mpidr_index(data, mpidr);
2723 
2724 		vcpu = kvm_get_vcpu(kvm, data->cmpidr_to_idx[idx]);
2725 		if (mpidr != kvm_vcpu_get_mpidr_aff(vcpu))
2726 			vcpu = NULL;
2727 	}
2728 
2729 	rcu_read_unlock();
2730 
2731 	if (vcpu)
2732 		return vcpu;
2733 
2734 	kvm_for_each_vcpu(i, vcpu, kvm) {
2735 		if (mpidr == kvm_vcpu_get_mpidr_aff(vcpu))
2736 			return vcpu;
2737 	}
2738 	return NULL;
2739 }
2740 
kvm_arch_irqchip_in_kernel(struct kvm * kvm)2741 bool kvm_arch_irqchip_in_kernel(struct kvm *kvm)
2742 {
2743 	return irqchip_in_kernel(kvm);
2744 }
2745 
kvm_arch_irq_bypass_add_producer(struct irq_bypass_consumer * cons,struct irq_bypass_producer * prod)2746 int kvm_arch_irq_bypass_add_producer(struct irq_bypass_consumer *cons,
2747 				      struct irq_bypass_producer *prod)
2748 {
2749 	struct kvm_kernel_irqfd *irqfd =
2750 		container_of(cons, struct kvm_kernel_irqfd, consumer);
2751 	struct kvm_kernel_irq_routing_entry *irq_entry = &irqfd->irq_entry;
2752 
2753 	/*
2754 	 * The only thing we have a chance of directly-injecting is LPIs. Maybe
2755 	 * one day...
2756 	 */
2757 	if (irq_entry->type != KVM_IRQ_ROUTING_MSI)
2758 		return 0;
2759 
2760 	return kvm_vgic_v4_set_forwarding(irqfd->kvm, prod->irq,
2761 					  &irqfd->irq_entry);
2762 }
kvm_arch_irq_bypass_del_producer(struct irq_bypass_consumer * cons,struct irq_bypass_producer * prod)2763 void kvm_arch_irq_bypass_del_producer(struct irq_bypass_consumer *cons,
2764 				      struct irq_bypass_producer *prod)
2765 {
2766 	struct kvm_kernel_irqfd *irqfd =
2767 		container_of(cons, struct kvm_kernel_irqfd, consumer);
2768 	struct kvm_kernel_irq_routing_entry *irq_entry = &irqfd->irq_entry;
2769 
2770 	if (irq_entry->type != KVM_IRQ_ROUTING_MSI)
2771 		return;
2772 
2773 	kvm_vgic_v4_unset_forwarding(irqfd->kvm, prod->irq,
2774 				     &irqfd->irq_entry);
2775 }
2776 
kvm_arch_irq_bypass_stop(struct irq_bypass_consumer * cons)2777 void kvm_arch_irq_bypass_stop(struct irq_bypass_consumer *cons)
2778 {
2779 	struct kvm_kernel_irqfd *irqfd =
2780 		container_of(cons, struct kvm_kernel_irqfd, consumer);
2781 
2782 	kvm_arm_halt_guest(irqfd->kvm);
2783 }
2784 
kvm_arch_irq_bypass_start(struct irq_bypass_consumer * cons)2785 void kvm_arch_irq_bypass_start(struct irq_bypass_consumer *cons)
2786 {
2787 	struct kvm_kernel_irqfd *irqfd =
2788 		container_of(cons, struct kvm_kernel_irqfd, consumer);
2789 
2790 	kvm_arm_resume_guest(irqfd->kvm);
2791 }
2792 
2793 /* Initialize Hyp-mode and memory mappings on all CPUs */
kvm_arm_init(void)2794 static __init int kvm_arm_init(void)
2795 {
2796 	int err;
2797 	bool in_hyp_mode;
2798 
2799 	if (!is_hyp_mode_available()) {
2800 		kvm_info("HYP mode not available\n");
2801 		return -ENODEV;
2802 	}
2803 
2804 	if (kvm_get_mode() == KVM_MODE_NONE) {
2805 		kvm_info("KVM disabled from command line\n");
2806 		return -ENODEV;
2807 	}
2808 
2809 	err = kvm_sys_reg_table_init();
2810 	if (err) {
2811 		kvm_info("Error initializing system register tables");
2812 		return err;
2813 	}
2814 
2815 	in_hyp_mode = is_kernel_in_hyp_mode();
2816 
2817 	if (cpus_have_final_cap(ARM64_WORKAROUND_DEVICE_LOAD_ACQUIRE) ||
2818 	    cpus_have_final_cap(ARM64_WORKAROUND_1508412))
2819 		kvm_info("Guests without required CPU erratum workarounds can deadlock system!\n" \
2820 			 "Only trusted guests should be used on this system.\n");
2821 
2822 	err = kvm_set_ipa_limit();
2823 	if (err)
2824 		return err;
2825 
2826 	err = kvm_arm_init_sve();
2827 	if (err)
2828 		return err;
2829 
2830 	err = kvm_arm_vmid_alloc_init();
2831 	if (err) {
2832 		kvm_err("Failed to initialize VMID allocator.\n");
2833 		return err;
2834 	}
2835 
2836 	if (!in_hyp_mode) {
2837 		err = init_hyp_mode();
2838 		if (err)
2839 			goto out_err;
2840 	}
2841 
2842 	err = kvm_init_vector_slots();
2843 	if (err) {
2844 		kvm_err("Cannot initialise vector slots\n");
2845 		goto out_hyp;
2846 	}
2847 
2848 	err = init_subsystems();
2849 	if (err)
2850 		goto out_hyp;
2851 
2852 	kvm_info("%s%sVHE%s mode initialized successfully\n",
2853 		 in_hyp_mode ? "" : (is_protected_kvm_enabled() ?
2854 				     "Protected " : "Hyp "),
2855 		 in_hyp_mode ? "" : (cpus_have_final_cap(ARM64_KVM_HVHE) ?
2856 				     "h" : "n"),
2857 		 cpus_have_final_cap(ARM64_HAS_NESTED_VIRT) ? "+NV2": "");
2858 
2859 	/*
2860 	 * FIXME: Do something reasonable if kvm_init() fails after pKVM
2861 	 * hypervisor protection is finalized.
2862 	 */
2863 	err = kvm_init(sizeof(struct kvm_vcpu), 0, THIS_MODULE);
2864 	if (err)
2865 		goto out_subs;
2866 
2867 	/*
2868 	 * This should be called after initialization is done and failure isn't
2869 	 * possible anymore.
2870 	 */
2871 	if (!in_hyp_mode)
2872 		finalize_init_hyp_mode();
2873 
2874 	kvm_arm_initialised = true;
2875 
2876 	return 0;
2877 
2878 out_subs:
2879 	teardown_subsystems();
2880 out_hyp:
2881 	if (!in_hyp_mode)
2882 		teardown_hyp_mode();
2883 out_err:
2884 	kvm_arm_vmid_alloc_free();
2885 	return err;
2886 }
2887 
early_kvm_mode_cfg(char * arg)2888 static int __init early_kvm_mode_cfg(char *arg)
2889 {
2890 	if (!arg)
2891 		return -EINVAL;
2892 
2893 	if (strcmp(arg, "none") == 0) {
2894 		kvm_mode = KVM_MODE_NONE;
2895 		return 0;
2896 	}
2897 
2898 	if (!is_hyp_mode_available()) {
2899 		pr_warn_once("KVM is not available. Ignoring kvm-arm.mode\n");
2900 		return 0;
2901 	}
2902 
2903 	if (strcmp(arg, "protected") == 0) {
2904 		if (!is_kernel_in_hyp_mode())
2905 			kvm_mode = KVM_MODE_PROTECTED;
2906 		else
2907 			pr_warn_once("Protected KVM not available with VHE\n");
2908 
2909 		return 0;
2910 	}
2911 
2912 	if (strcmp(arg, "nvhe") == 0 && !WARN_ON(is_kernel_in_hyp_mode())) {
2913 		kvm_mode = KVM_MODE_DEFAULT;
2914 		return 0;
2915 	}
2916 
2917 	if (strcmp(arg, "nested") == 0 && !WARN_ON(!is_kernel_in_hyp_mode())) {
2918 		kvm_mode = KVM_MODE_NV;
2919 		return 0;
2920 	}
2921 
2922 	return -EINVAL;
2923 }
2924 early_param("kvm-arm.mode", early_kvm_mode_cfg);
2925 
early_kvm_wfx_trap_policy_cfg(char * arg,enum kvm_wfx_trap_policy * p)2926 static int __init early_kvm_wfx_trap_policy_cfg(char *arg, enum kvm_wfx_trap_policy *p)
2927 {
2928 	if (!arg)
2929 		return -EINVAL;
2930 
2931 	if (strcmp(arg, "trap") == 0) {
2932 		*p = KVM_WFX_TRAP;
2933 		return 0;
2934 	}
2935 
2936 	if (strcmp(arg, "notrap") == 0) {
2937 		*p = KVM_WFX_NOTRAP;
2938 		return 0;
2939 	}
2940 
2941 	return -EINVAL;
2942 }
2943 
early_kvm_wfi_trap_policy_cfg(char * arg)2944 static int __init early_kvm_wfi_trap_policy_cfg(char *arg)
2945 {
2946 	return early_kvm_wfx_trap_policy_cfg(arg, &kvm_wfi_trap_policy);
2947 }
2948 early_param("kvm-arm.wfi_trap_policy", early_kvm_wfi_trap_policy_cfg);
2949 
early_kvm_wfe_trap_policy_cfg(char * arg)2950 static int __init early_kvm_wfe_trap_policy_cfg(char *arg)
2951 {
2952 	return early_kvm_wfx_trap_policy_cfg(arg, &kvm_wfe_trap_policy);
2953 }
2954 early_param("kvm-arm.wfe_trap_policy", early_kvm_wfe_trap_policy_cfg);
2955 
kvm_get_mode(void)2956 enum kvm_mode kvm_get_mode(void)
2957 {
2958 	return kvm_mode;
2959 }
2960 
2961 module_init(kvm_arm_init);
2962