1 // SPDX-License-Identifier: GPL-2.0-only 2 // Miscellaneous Arm SMMU implementation and integration quirks 3 // Copyright (C) 2019 Arm Limited 4 5 #define pr_fmt(fmt) "arm-smmu: " fmt 6 7 #include <linux/bitfield.h> 8 #include <linux/of.h> 9 10 #include "arm-smmu.h" 11 12 13 static int arm_smmu_gr0_ns(int offset) 14 { 15 switch (offset) { 16 case ARM_SMMU_GR0_sCR0: 17 case ARM_SMMU_GR0_sACR: 18 case ARM_SMMU_GR0_sGFSR: 19 case ARM_SMMU_GR0_sGFSYNR0: 20 case ARM_SMMU_GR0_sGFSYNR1: 21 case ARM_SMMU_GR0_sGFSYNR2: 22 return offset + 0x400; 23 default: 24 return offset; 25 } 26 } 27 28 static u32 arm_smmu_read_ns(struct arm_smmu_device *smmu, int page, 29 int offset) 30 { 31 if (page == ARM_SMMU_GR0) 32 offset = arm_smmu_gr0_ns(offset); 33 return readl_relaxed(arm_smmu_page(smmu, page) + offset); 34 } 35 36 static void arm_smmu_write_ns(struct arm_smmu_device *smmu, int page, 37 int offset, u32 val) 38 { 39 if (page == ARM_SMMU_GR0) 40 offset = arm_smmu_gr0_ns(offset); 41 writel_relaxed(val, arm_smmu_page(smmu, page) + offset); 42 } 43 44 /* Since we don't care for sGFAR, we can do without 64-bit accessors */ 45 static const struct arm_smmu_impl calxeda_impl = { 46 .read_reg = arm_smmu_read_ns, 47 .write_reg = arm_smmu_write_ns, 48 }; 49 50 51 struct cavium_smmu { 52 struct arm_smmu_device smmu; 53 u32 id_base; 54 }; 55 56 static int cavium_cfg_probe(struct arm_smmu_device *smmu) 57 { 58 static atomic_t context_count = ATOMIC_INIT(0); 59 struct cavium_smmu *cs = container_of(smmu, struct cavium_smmu, smmu); 60 /* 61 * Cavium CN88xx erratum #27704. 62 * Ensure ASID and VMID allocation is unique across all SMMUs in 63 * the system. 64 */ 65 cs->id_base = atomic_fetch_add(smmu->num_context_banks, &context_count); 66 dev_notice(smmu->dev, "\tenabling workaround for Cavium erratum 27704\n"); 67 68 return 0; 69 } 70 71 static int cavium_init_context(struct arm_smmu_domain *smmu_domain, 72 struct io_pgtable_cfg *pgtbl_cfg, struct device *dev) 73 { 74 struct cavium_smmu *cs = container_of(smmu_domain->smmu, 75 struct cavium_smmu, smmu); 76 77 if (smmu_domain->stage == ARM_SMMU_DOMAIN_S2) 78 smmu_domain->cfg.vmid += cs->id_base; 79 else 80 smmu_domain->cfg.asid += cs->id_base; 81 82 return 0; 83 } 84 85 static const struct arm_smmu_impl cavium_impl = { 86 .cfg_probe = cavium_cfg_probe, 87 .init_context = cavium_init_context, 88 }; 89 90 static struct arm_smmu_device *cavium_smmu_impl_init(struct arm_smmu_device *smmu) 91 { 92 struct cavium_smmu *cs; 93 94 cs = devm_krealloc(smmu->dev, smmu, sizeof(*cs), GFP_KERNEL); 95 if (!cs) 96 return ERR_PTR(-ENOMEM); 97 98 cs->smmu.impl = &cavium_impl; 99 100 return &cs->smmu; 101 } 102 103 104 #define ARM_MMU500_ACTLR_CPRE (1 << 1) 105 106 #define ARM_MMU500_ACR_CACHE_LOCK (1 << 26) 107 #define ARM_MMU500_ACR_S2CRB_TLBEN (1 << 10) 108 #define ARM_MMU500_ACR_SMTNMB_TLBEN (1 << 8) 109 110 int arm_mmu500_reset(struct arm_smmu_device *smmu) 111 { 112 u32 reg, major; 113 /* 114 * On MMU-500 r2p0 onwards we need to clear ACR.CACHE_LOCK before 115 * writes to the context bank ACTLRs will stick. And we just hope that 116 * Secure has also cleared SACR.CACHE_LOCK for this to take effect... 117 */ 118 reg = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_ID7); 119 major = FIELD_GET(ARM_SMMU_ID7_MAJOR, reg); 120 reg = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sACR); 121 if (major >= 2) 122 reg &= ~ARM_MMU500_ACR_CACHE_LOCK; 123 /* 124 * Allow unmatched Stream IDs to allocate bypass 125 * TLB entries for reduced latency. 126 */ 127 reg |= ARM_MMU500_ACR_SMTNMB_TLBEN | ARM_MMU500_ACR_S2CRB_TLBEN; 128 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sACR, reg); 129 130 #ifdef CONFIG_ARM_SMMU_MMU_500_CPRE_ERRATA 131 /* 132 * Disable MMU-500's not-particularly-beneficial next-page 133 * prefetcher for the sake of at least 5 known errata. 134 */ 135 for (int i = 0; i < smmu->num_context_banks; ++i) { 136 reg = arm_smmu_cb_read(smmu, i, ARM_SMMU_CB_ACTLR); 137 reg &= ~ARM_MMU500_ACTLR_CPRE; 138 arm_smmu_cb_write(smmu, i, ARM_SMMU_CB_ACTLR, reg); 139 reg = arm_smmu_cb_read(smmu, i, ARM_SMMU_CB_ACTLR); 140 if (reg & ARM_MMU500_ACTLR_CPRE) 141 dev_warn_once(smmu->dev, "Failed to disable prefetcher for errata workarounds, check SACR.CACHE_LOCK\n"); 142 } 143 #endif 144 145 return 0; 146 } 147 148 static const struct arm_smmu_impl arm_mmu500_impl = { 149 .reset = arm_mmu500_reset, 150 }; 151 152 static u64 mrvl_mmu500_readq(struct arm_smmu_device *smmu, int page, int off) 153 { 154 /* 155 * Marvell Armada-AP806 erratum #582743. 156 * Split all the readq to double readl 157 */ 158 return hi_lo_readq_relaxed(arm_smmu_page(smmu, page) + off); 159 } 160 161 static void mrvl_mmu500_writeq(struct arm_smmu_device *smmu, int page, int off, 162 u64 val) 163 { 164 /* 165 * Marvell Armada-AP806 erratum #582743. 166 * Split all the writeq to double writel 167 */ 168 hi_lo_writeq_relaxed(val, arm_smmu_page(smmu, page) + off); 169 } 170 171 static int mrvl_mmu500_cfg_probe(struct arm_smmu_device *smmu) 172 { 173 174 /* 175 * Armada-AP806 erratum #582743. 176 * Hide the SMMU_IDR2.PTFSv8 fields to sidestep the AArch64 177 * formats altogether and allow using 32 bits access on the 178 * interconnect. 179 */ 180 smmu->features &= ~(ARM_SMMU_FEAT_FMT_AARCH64_4K | 181 ARM_SMMU_FEAT_FMT_AARCH64_16K | 182 ARM_SMMU_FEAT_FMT_AARCH64_64K); 183 184 return 0; 185 } 186 187 static const struct arm_smmu_impl mrvl_mmu500_impl = { 188 .read_reg64 = mrvl_mmu500_readq, 189 .write_reg64 = mrvl_mmu500_writeq, 190 .cfg_probe = mrvl_mmu500_cfg_probe, 191 .reset = arm_mmu500_reset, 192 }; 193 194 195 struct arm_smmu_device *arm_smmu_impl_init(struct arm_smmu_device *smmu) 196 { 197 const struct device_node *np = smmu->dev->of_node; 198 199 /* 200 * Set the impl for model-specific implementation quirks first, 201 * such that platform integration quirks can pick it up and 202 * inherit from it if necessary. 203 */ 204 switch (smmu->model) { 205 case ARM_MMU500: 206 smmu->impl = &arm_mmu500_impl; 207 break; 208 case CAVIUM_SMMUV2: 209 return cavium_smmu_impl_init(smmu); 210 default: 211 break; 212 } 213 214 /* This is implicitly MMU-400 */ 215 if (of_property_read_bool(np, "calxeda,smmu-secure-config-access")) 216 smmu->impl = &calxeda_impl; 217 218 if (of_device_is_compatible(np, "nvidia,tegra234-smmu") || 219 of_device_is_compatible(np, "nvidia,tegra194-smmu") || 220 of_device_is_compatible(np, "nvidia,tegra186-smmu")) 221 return nvidia_smmu_impl_init(smmu); 222 223 if (IS_ENABLED(CONFIG_ARM_SMMU_QCOM)) 224 smmu = qcom_smmu_impl_init(smmu); 225 226 if (of_device_is_compatible(np, "marvell,ap806-smmu-500")) 227 smmu->impl = &mrvl_mmu500_impl; 228 229 return smmu; 230 } 231