6 // Use of this source code is governed by a BSD-style license that can be
7 // found in the LICENSE-BSD-3-Clause file.
11 // SPDX-License-Identifier: Apache-2.0 AND BSD-3-Clause
94 /// Extract the specified bits of a 64-bit integer.
101         ($value >> $offset) & (!0u64 >> (64 - $length))
108         $value & (!0u64 >> (64 - $length))
155     ThreadCleanup(std::boxed::Box<dyn std::any::Any + std::marker::Send>),
337 /// A wrapper around creating and using a kvm-based VCPU.
354     /// * `id` - Represents the CPU number between [0, max vcpus).
355     /// * `vm` - The virtual machine this vcpu will get attached to.
356     /// * `vm_ops` - Optional object for exit handling.
357     /// * `cpu_vendor` - CPU vendor as reported by __cpuid(0x0)
364     ) -> Result<Self> {  in new()
384     /// * `kernel_entry_point` - Kernel entry point address in guest memory and boot protocol used.
385     /// * `guest_memory` - Guest memory.
386     /// * `cpuid` - (x86_64) CpuId, wrapper over the `kvm_cpuid2` structure.
394     ) -> Result<()> {  in configure()
421     pub fn get_mpidr(&self) -> u64 {  in get_mpidr()
427     pub fn get_saved_state(&self) -> Option<CpuState> {  in get_saved_state()
433     pub fn init(&self, vm: &Arc<dyn hypervisor::Vm>) -> Result<()> {  in init()
463     pub fn run(&self) -> std::result::Result<VmExit, HypervisorCpuError> {  in run()
468     pub fn set_sev_control_register(&self, vmsa_pfn: u64) -> Result<()> {  in set_sev_control_register()
482     ) -> Result<()> {  in set_gic_redistributor_addr()
494     fn id(&self) -> String {  in id()
498     fn snapshot(&mut self) -> std::result::Result<Snapshot, MigratableError> {  in snapshot()
585     fn write(&mut self, _base: u64, offset: u64, data: &[u8]) -> Option<Arc<Barrier>> {  in write()
638     fn active(&self) -> bool {  in active()
660     fn join_thread(&mut self) -> Result<()> {  in join_thread()
690     ) -> Result<Arc<Mutex<CpuManager>>> {  in new()
793     ) -> Result<()> {  in populate_cpuid()
820     fn create_vcpu(&mut self, cpu_id: u8, snapshot: Option<Snapshot>) -> Result<Arc<Mutex<Vcpu>>> {  in create_vcpu()
866     ) -> Result<()> {  in configure_vcpu()
877             // Traditional way to configure vcpu doesn't work for SEV-SNP guests.  in configure_vcpu()
878             // All the vCPU configuration for SEV-SNP guest is provided via VMSA.  in configure_vcpu()
912     ) -> Result<Vec<Arc<Mutex<Vcpu>>>> {  in create_vcpus()
940     pub fn init_pmu(&self, irq: u32) -> Result<bool> {  in init_pmu()
958     pub fn vcpus(&self) -> Vec<Arc<Mutex<Vcpu>>> {  in vcpus()
968     ) -> Result<()> {  in start_vcpu()
1075 …                      // has re-entered the kernel with KVM_RUN.  The kernel side will first finish  in start_vcpu()
1079 …                       // completed before performing a live migration.  Userspace can re-enter the  in start_vcpu()
1127                             // vcpu.run() returns false on a triple-fault so trigger a reset  in start_vcpu()
1228     ) -> Result<()> {  in activate_vcpus()
1234             (desired_vcpus - self.present_vcpus() + 1) as usize,  in activate_vcpus()
1270     pub fn check_pending_removed_vcpu(&mut self) -> bool {  in check_pending_removed_vcpu()
1279     fn remove_vcpu(&mut self, cpu_id: u8) -> Result<()> {  in remove_vcpu()
1297     ) -> Result<Vec<Arc<Mutex<Vcpu>>>> {  in create_boot_vcpus()
1304     pub fn start_boot_vcpus(&mut self, paused: bool) -> Result<()> {  in start_boot_vcpus()
1308     pub fn start_restored_vcpus(&mut self) -> Result<()> {  in start_restored_vcpus()
1317     pub fn resize(&mut self, desired_vcpus: u8) -> Result<bool> {  in resize()
1351     pub fn shutdown(&mut self) -> Result<()> {  in shutdown()
1379     pub fn initialize_tdx(&self, hob_address: u64) -> Result<()> {  in initialize_tdx()
1390     pub fn boot_vcpus(&self) -> u8 {  in boot_vcpus()
1394     pub fn max_vcpus(&self) -> u8 {  in max_vcpus()
1399     pub fn common_cpuid(&self) -> Vec<CpuIdEntry> {  in common_cpuid()
1404     fn present_vcpus(&self) -> u8 {  in present_vcpus()
1411     pub fn get_mpidrs(&self) -> Vec<u64> {  in get_mpidrs()
1419     pub fn get_saved_states(&self) -> Vec<CpuState> {  in get_saved_states()
1426     pub fn get_vcpu_topology(&self) -> Option<(u8, u8, u8)> {  in get_vcpu_topology()
1434     pub fn create_madt(&self) -> Sdt {  in create_madt()
1566     pub fn create_pptt(&self) -> Sdt {  in create_pptt()
1580                 let cluster_offset = pptt.len() - pptt_start;  in create_pptt()
1593                     let core_offset = pptt.len() - pptt_start;  in create_pptt()
1643     fn create_standard_regs(&self, cpu_id: u8) -> StandardRegisters {  in create_standard_regs()
1652     fn get_regs(&self, cpu_id: u8) -> Result<StandardRegisters> {  in get_regs()
1662     fn set_regs(&self, cpu_id: u8, regs: &StandardRegisters) -> Result<()> {  in set_regs()
1672     fn get_sregs(&self, cpu_id: u8) -> Result<SpecialRegisters> {  in get_sregs()
1682     fn set_sregs(&self, cpu_id: u8, sregs: &SpecialRegisters) -> Result<()> {  in set_sregs()
1697     ) -> Result<u64> {  in translate_gva()
1714     /// - Exception Level 1
1715     /// - Translate high address range only (kernel space)
1717     /// This implementation supports following Arm-v8a features related to
1719     /// - FEAT_LPA
1720     /// - FEAT_LVA
1721     /// - FEAT_LPA2
1729     ) -> Result<u64> {  in translate_gva()
1761         // Indication of 48-bits (0) or 52-bits (1) for FEAT_LPA2  in translate_gva()
1770         let va_size = 64 - tsz;  in translate_gva()
1778         let mut level = 4 - (va_size - 4) / stride;  in translate_gva()
1802         let indexmask_grainsize = (!0u64) >> (64 - (stride + 3));  in translate_gva()
1803         let mut indexmask = (!0u64) >> (64 - (va_size - (stride * (4 - level))));  in translate_gva()
1808             !0u64 >> (64 - 50) // mask with 50 least significant bits  in translate_gva()
1810             !0u64 >> (64 - 48) // mask with 48 least significant bits  in translate_gva()
1825             let table_offset: u64 = (gva >> (stride * (4 - level))) & indexmask;  in translate_gva()
1837             // In the case of FEAT_LPA, the next-level translation table address  in translate_gva()
1839             // For FEAT_LPA2, the next-level translation table address  in translate_gva()
1864         // - a page entry at level 3 or  in translate_gva()
1865         // - a block entry at level 1 or 2  in translate_gva()
1866         let page_size = 1u64 << ((stride * (4 - level)) + 3);  in translate_gva()
1867         descaddr &= !(page_size - 1);  in translate_gva()
1868         descaddr |= gva & (page_size - 1);  in translate_gva()
1884     pub(crate) fn vcpus_kill_signalled(&self) -> &Arc<AtomicBool> {  in vcpus_kill_signalled()
1896     ) -> Result<[u32; 4]> {  in get_cpuid_leaf()
1907     pub(crate) fn sev_snp_enabled(&self) -> bool {  in sev_snp_enabled()
1911     pub(crate) fn nmi(&self) -> Result<()> {  in nmi()
1940     fn generate_mat(&self) -> Vec<u8> {  in generate_mat()
2000                     // even it if is disabled in the MADT (non-boot CPU)  in to_aml_bytes()
2037                     // even it if is disabled in the MADT (non-boot CPU)  in to_aml_bytes()
2198                             acpi_address.0 + CPU_MANAGER_ACPI_SIZE as u64 - 1,  in to_aml_bytes()
2275     fn pause(&mut self) -> std::result::Result<(), MigratableError> {  in pause()
2314     fn resume(&mut self) -> std::result::Result<(), MigratableError> {  in resume()
2335     fn id(&self) -> String {  in id()
2339     fn snapshot(&mut self) -> std::result::Result<Snapshot, MigratableError> {  in snapshot()
2363     ) -> std::result::Result<(), DebuggableError> {  in set_guest_debug()
2372     fn debug_pause(&mut self) -> std::result::Result<(), DebuggableError> {  in debug_pause()
2376     fn debug_resume(&mut self) -> std::result::Result<(), DebuggableError> {  in debug_resume()
2381     fn read_regs(&self, cpu_id: usize) -> std::result::Result<CoreRegs, DebuggableError> {  in read_regs()
2382         // General registers: RAX, RBX, RCX, RDX, RSI, RDI, RBP, RSP, r8-r15  in read_regs()
2405         // GDB exposes 32-bit eflags instead of 64-bit rflags.  in read_regs()
2406         // https://github.com/bminor/binutils-gdb/blob/master/gdb/features/i386/64bit-core.xml  in read_regs()
2435     fn read_regs(&self, cpu_id: usize) -> std::result::Result<CoreRegs, DebuggableError> {  in read_regs()
2452     ) -> std::result::Result<(), DebuggableError> {  in write_regs()
2474         // Update the lower 32-bit of rflags.  in write_regs()
2505     ) -> std::result::Result<(), DebuggableError> {  in write_regs()
2526     ) -> std::result::Result<Vec<u8>, DebuggableError> {  in read_mem()
2538             let read_len = std::cmp::min(len as u64 - total_read, psize - (paddr & (psize - 1)));  in read_mem()
2557     ) -> std::result::Result<(), DebuggableError> {  in write_mem()
2569                 data.len() as u64 - total_written,  in write_mem()
2570                 psize - (paddr & (psize - 1)),  in write_mem()
2584     fn active_vcpus(&self) -> usize {  in active_vcpus()
2597     ) -> std::result::Result<(), GuestDebuggableError> {  in cpu_write_elf64_note()
2624             pos += descsz - size_of::<X86_64UserRegs>() - size_of::<u64>();  in cpu_write_elf64_note()
2705     ) -> std::result::Result<(), GuestDebuggableError> {  in cpu_write_vmm_note()
2873         // TODO: auto-generate kvm related structures with PartialEq on.  in test_setup_fpu()
3029         fn hypervisor_cpu_error_to_raw_os_error(error: &anyhow::Error) -> libc::c_int {  in test_save_restore_core_regs()