1 // Copyright 2022 Arm Limited (or its affiliates). All rights reserved. 2 // Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 // SPDX-License-Identifier: Apache-2.0 4 5 use kvm_ioctls::DeviceFd; 6 7 use crate::arch::aarch64::gic::{Error, Result}; 8 use crate::device::HypervisorDeviceError; 9 use crate::kvm::kvm_bindings::{ 10 kvm_device_attr, KVM_DEV_ARM_VGIC_GRP_CPU_SYSREGS, KVM_REG_ARM64_SYSREG_CRM_MASK, 11 KVM_REG_ARM64_SYSREG_CRM_SHIFT, KVM_REG_ARM64_SYSREG_CRN_MASK, KVM_REG_ARM64_SYSREG_CRN_SHIFT, 12 KVM_REG_ARM64_SYSREG_OP0_MASK, KVM_REG_ARM64_SYSREG_OP0_SHIFT, KVM_REG_ARM64_SYSREG_OP1_MASK, 13 KVM_REG_ARM64_SYSREG_OP1_SHIFT, KVM_REG_ARM64_SYSREG_OP2_MASK, KVM_REG_ARM64_SYSREG_OP2_SHIFT, 14 }; 15 16 const KVM_DEV_ARM_VGIC_V3_MPIDR_SHIFT: u32 = 32; 17 const KVM_DEV_ARM_VGIC_V3_MPIDR_MASK: u64 = 0xffffffff << KVM_DEV_ARM_VGIC_V3_MPIDR_SHIFT as u64; 18 19 const ICC_CTLR_EL1_PRIBITS_SHIFT: u32 = 8; 20 const ICC_CTLR_EL1_PRIBITS_MASK: u32 = 7 << ICC_CTLR_EL1_PRIBITS_SHIFT; 21 22 macro_rules! arm64_vgic_sys_reg { 23 ($name: tt, $op0: tt, $op1: tt, $crn: tt, $crm: tt, $op2: expr) => { 24 const $name: u64 = ((($op0 as u64) << KVM_REG_ARM64_SYSREG_OP0_SHIFT) 25 & KVM_REG_ARM64_SYSREG_OP0_MASK as u64) 26 | ((($op1 as u64) << KVM_REG_ARM64_SYSREG_OP1_SHIFT) 27 & KVM_REG_ARM64_SYSREG_OP1_MASK as u64) 28 | ((($crn as u64) << KVM_REG_ARM64_SYSREG_CRN_SHIFT) 29 & KVM_REG_ARM64_SYSREG_CRN_MASK as u64) 30 | ((($crm as u64) << KVM_REG_ARM64_SYSREG_CRM_SHIFT) 31 & KVM_REG_ARM64_SYSREG_CRM_MASK as u64) 32 | ((($op2 as u64) << KVM_REG_ARM64_SYSREG_OP2_SHIFT) 33 & KVM_REG_ARM64_SYSREG_OP2_MASK as u64); 34 }; 35 } 36 37 macro_rules! SYS_ICC_AP0Rn_EL1 { 38 ($name: tt, $n: tt) => { 39 arm64_vgic_sys_reg!($name, 3, 0, 12, 8, (4 | $n)); 40 }; 41 } 42 43 macro_rules! SYS_ICC_AP1Rn_EL1 { 44 ($name: tt, $n: tt) => { 45 arm64_vgic_sys_reg!($name, 3, 0, 12, 9, $n); 46 }; 47 } 48 49 arm64_vgic_sys_reg!(SYS_ICC_SRE_EL1, 3, 0, 12, 12, 5); 50 arm64_vgic_sys_reg!(SYS_ICC_CTLR_EL1, 3, 0, 12, 12, 4); 51 arm64_vgic_sys_reg!(SYS_ICC_IGRPEN0_EL1, 3, 0, 12, 12, 6); 52 arm64_vgic_sys_reg!(SYS_ICC_IGRPEN1_EL1, 3, 0, 12, 12, 7); 53 arm64_vgic_sys_reg!(SYS_ICC_PMR_EL1, 3, 0, 4, 6, 0); 54 arm64_vgic_sys_reg!(SYS_ICC_BPR0_EL1, 3, 0, 12, 8, 3); 55 arm64_vgic_sys_reg!(SYS_ICC_BPR1_EL1, 3, 0, 12, 12, 3); 56 SYS_ICC_AP0Rn_EL1!(SYS_ICC_AP0R0_EL1, 0); 57 SYS_ICC_AP0Rn_EL1!(SYS_ICC_AP0R1_EL1, 1); 58 SYS_ICC_AP0Rn_EL1!(SYS_ICC_AP0R2_EL1, 2); 59 SYS_ICC_AP0Rn_EL1!(SYS_ICC_AP0R3_EL1, 3); 60 SYS_ICC_AP1Rn_EL1!(SYS_ICC_AP1R0_EL1, 0); 61 SYS_ICC_AP1Rn_EL1!(SYS_ICC_AP1R1_EL1, 1); 62 SYS_ICC_AP1Rn_EL1!(SYS_ICC_AP1R2_EL1, 2); 63 SYS_ICC_AP1Rn_EL1!(SYS_ICC_AP1R3_EL1, 3); 64 65 static VGIC_ICC_REGS: &[u64] = &[ 66 SYS_ICC_SRE_EL1, 67 SYS_ICC_CTLR_EL1, 68 SYS_ICC_IGRPEN0_EL1, 69 SYS_ICC_IGRPEN1_EL1, 70 SYS_ICC_PMR_EL1, 71 SYS_ICC_BPR0_EL1, 72 SYS_ICC_BPR1_EL1, 73 SYS_ICC_AP0R0_EL1, 74 SYS_ICC_AP0R1_EL1, 75 SYS_ICC_AP0R2_EL1, 76 SYS_ICC_AP0R3_EL1, 77 SYS_ICC_AP1R0_EL1, 78 SYS_ICC_AP1R1_EL1, 79 SYS_ICC_AP1R2_EL1, 80 SYS_ICC_AP1R3_EL1, 81 ]; 82 83 fn icc_attr_set(gic: &DeviceFd, offset: u64, typer: u64, val: u32) -> Result<()> { 84 let gic_icc_attr = kvm_device_attr { 85 group: KVM_DEV_ARM_VGIC_GRP_CPU_SYSREGS, 86 attr: ((typer & KVM_DEV_ARM_VGIC_V3_MPIDR_MASK) | offset), // this needs the mpidr 87 addr: &val as *const u32 as u64, 88 flags: 0, 89 }; 90 91 gic.set_device_attr(&gic_icc_attr).map_err(|e| { 92 Error::SetDeviceAttribute(HypervisorDeviceError::SetDeviceAttribute(e.into())) 93 })?; 94 95 Ok(()) 96 } 97 98 fn icc_attr_get(gic: &DeviceFd, offset: u64, typer: u64) -> Result<u32> { 99 let mut val = 0; 100 101 let mut gic_icc_attr = kvm_device_attr { 102 group: KVM_DEV_ARM_VGIC_GRP_CPU_SYSREGS, 103 attr: ((typer & KVM_DEV_ARM_VGIC_V3_MPIDR_MASK) | offset), // this needs the mpidr 104 addr: &mut val as *mut u32 as u64, 105 flags: 0, 106 }; 107 108 // SAFETY: gic_icc_attr.addr is safe to write to. 109 unsafe { gic.get_device_attr(&mut gic_icc_attr) }.map_err(|e| { 110 Error::GetDeviceAttribute(HypervisorDeviceError::GetDeviceAttribute(e.into())) 111 })?; 112 113 Ok(val) 114 } 115 116 /// Get ICC registers. 117 pub fn get_icc_regs(gic: &DeviceFd, gicr_typer: &[u64]) -> Result<Vec<u32>> { 118 let mut state: Vec<u32> = Vec::new(); 119 // We need this for the ICC_AP<m>R<n>_EL1 registers. 120 let mut num_priority_bits = 0; 121 122 for ix in gicr_typer { 123 let i = *ix; 124 for icc_offset in VGIC_ICC_REGS { 125 if *icc_offset == SYS_ICC_CTLR_EL1 { 126 // calculate priority bits by reading the ctrl_el1 register. 127 let val = icc_attr_get(gic, *icc_offset, i)?; 128 // The priority bits are found in the ICC_CTLR_EL1 register (bits from 10:8). 129 // See page 194 from https://static.docs.arm.com/ihi0069/c/IHI0069C_gic_ 130 // architecture_specification.pdf. 131 // Citation: 132 // "Priority bits. Read-only and writes are ignored. The number of priority bits 133 // implemented, minus one." 134 num_priority_bits = 135 ((val & ICC_CTLR_EL1_PRIBITS_MASK) >> ICC_CTLR_EL1_PRIBITS_SHIFT) + 1; 136 state.push(val); 137 } 138 // As per ARMv8 documentation: https://static.docs.arm.com/ihi0069/c/IHI0069C_ 139 // gic_architecture_specification.pdf 140 // page 178, 141 // ICC_AP0R1_EL1 is only implemented in implementations that support 6 or more bits of 142 // priority. 143 // ICC_AP0R2_EL1 and ICC_AP0R3_EL1 are only implemented in implementations that support 144 // 7 bits of priority. 145 else if *icc_offset == SYS_ICC_AP0R1_EL1 || *icc_offset == SYS_ICC_AP1R1_EL1 { 146 if num_priority_bits >= 6 { 147 state.push(icc_attr_get(gic, *icc_offset, i)?); 148 } 149 } else if *icc_offset == SYS_ICC_AP0R2_EL1 150 || *icc_offset == SYS_ICC_AP0R3_EL1 151 || *icc_offset == SYS_ICC_AP1R2_EL1 152 || *icc_offset == SYS_ICC_AP1R3_EL1 153 { 154 if num_priority_bits == 7 { 155 state.push(icc_attr_get(gic, *icc_offset, i)?); 156 } 157 } else { 158 state.push(icc_attr_get(gic, *icc_offset, i)?); 159 } 160 } 161 } 162 Ok(state) 163 } 164 165 /// Set ICC registers. 166 pub fn set_icc_regs(gic: &DeviceFd, gicr_typer: &[u64], state: &[u32]) -> Result<()> { 167 let mut num_priority_bits = 0; 168 let mut idx = 0; 169 for ix in gicr_typer { 170 let i = *ix; 171 for icc_offset in VGIC_ICC_REGS { 172 if *icc_offset == SYS_ICC_CTLR_EL1 { 173 let ctrl_el1 = state[idx]; 174 num_priority_bits = 175 ((ctrl_el1 & ICC_CTLR_EL1_PRIBITS_MASK) >> ICC_CTLR_EL1_PRIBITS_SHIFT) + 1; 176 } 177 if *icc_offset == SYS_ICC_AP0R1_EL1 || *icc_offset == SYS_ICC_AP1R1_EL1 { 178 if num_priority_bits >= 6 { 179 icc_attr_set(gic, *icc_offset, i, state[idx])?; 180 idx += 1; 181 } 182 continue; 183 } 184 if *icc_offset == SYS_ICC_AP0R2_EL1 185 || *icc_offset == SYS_ICC_AP0R3_EL1 186 || *icc_offset == SYS_ICC_AP1R2_EL1 187 || *icc_offset == SYS_ICC_AP1R3_EL1 188 { 189 if num_priority_bits == 7 { 190 icc_attr_set(gic, *icc_offset, i, state[idx])?; 191 idx += 1; 192 } 193 continue; 194 } 195 icc_attr_set(gic, *icc_offset, i, state[idx])?; 196 idx += 1; 197 } 198 } 199 Ok(()) 200 } 201