1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * IOMMU API for ARM architected SMMU implementations. 4 * 5 * Copyright (C) 2013 ARM Limited 6 * 7 * Author: Will Deacon <will.deacon@arm.com> 8 * 9 * This driver currently supports: 10 * - SMMUv1 and v2 implementations 11 * - Stream-matching and stream-indexing 12 * - v7/v8 long-descriptor format 13 * - Non-secure access to the SMMU 14 * - Context fault reporting 15 * - Extended Stream ID (16 bit) 16 */ 17 18 #define pr_fmt(fmt) "arm-smmu: " fmt 19 20 #include <linux/acpi.h> 21 #include <linux/acpi_iort.h> 22 #include <linux/bitfield.h> 23 #include <linux/delay.h> 24 #include <linux/dma-mapping.h> 25 #include <linux/err.h> 26 #include <linux/interrupt.h> 27 #include <linux/io.h> 28 #include <linux/iopoll.h> 29 #include <linux/module.h> 30 #include <linux/of.h> 31 #include <linux/of_address.h> 32 #include <linux/pci.h> 33 #include <linux/platform_device.h> 34 #include <linux/pm_runtime.h> 35 #include <linux/ratelimit.h> 36 #include <linux/slab.h> 37 #include <linux/string_choices.h> 38 39 #include <linux/fsl/mc.h> 40 41 #include "arm-smmu.h" 42 #include "../../dma-iommu.h" 43 44 /* 45 * Apparently, some Qualcomm arm64 platforms which appear to expose their SMMU 46 * global register space are still, in fact, using a hypervisor to mediate it 47 * by trapping and emulating register accesses. Sadly, some deployed versions 48 * of said trapping code have bugs wherein they go horribly wrong for stores 49 * using r31 (i.e. XZR/WZR) as the source register. 50 */ 51 #define QCOM_DUMMY_VAL -1 52 53 #define MSI_IOVA_BASE 0x8000000 54 #define MSI_IOVA_LENGTH 0x100000 55 56 static int force_stage; 57 module_param(force_stage, int, S_IRUGO); 58 MODULE_PARM_DESC(force_stage, 59 "Force SMMU mappings to be installed at a particular stage of translation. A value of '1' or '2' forces the corresponding stage. All other values are ignored (i.e. no stage is forced). Note that selecting a specific stage will disable support for nested translation."); 60 static bool disable_bypass = 61 IS_ENABLED(CONFIG_ARM_SMMU_DISABLE_BYPASS_BY_DEFAULT); 62 module_param(disable_bypass, bool, S_IRUGO); 63 MODULE_PARM_DESC(disable_bypass, 64 "Disable bypass streams such that incoming transactions from devices that are not attached to an iommu domain will report an abort back to the device and will not be allowed to pass through the SMMU."); 65 66 #define s2cr_init_val (struct arm_smmu_s2cr){ \ 67 .type = disable_bypass ? S2CR_TYPE_FAULT : S2CR_TYPE_BYPASS, \ 68 } 69 70 static bool using_legacy_binding, using_generic_binding; 71 72 static inline int arm_smmu_rpm_get(struct arm_smmu_device *smmu) 73 { 74 if (pm_runtime_enabled(smmu->dev)) 75 return pm_runtime_resume_and_get(smmu->dev); 76 77 return 0; 78 } 79 80 static inline void arm_smmu_rpm_put(struct arm_smmu_device *smmu) 81 { 82 if (pm_runtime_enabled(smmu->dev)) { 83 pm_runtime_mark_last_busy(smmu->dev); 84 __pm_runtime_put_autosuspend(smmu->dev); 85 86 } 87 } 88 89 static void arm_smmu_rpm_use_autosuspend(struct arm_smmu_device *smmu) 90 { 91 /* 92 * Setup an autosuspend delay to avoid bouncing runpm state. 93 * Otherwise, if a driver for a suspended consumer device 94 * unmaps buffers, it will runpm resume/suspend for each one. 95 * 96 * For example, when used by a GPU device, when an application 97 * or game exits, it can trigger unmapping 100s or 1000s of 98 * buffers. With a runpm cycle for each buffer, that adds up 99 * to 5-10sec worth of reprogramming the context bank, while 100 * the system appears to be locked up to the user. 101 */ 102 pm_runtime_set_autosuspend_delay(smmu->dev, 20); 103 pm_runtime_use_autosuspend(smmu->dev); 104 } 105 106 static struct arm_smmu_domain *to_smmu_domain(struct iommu_domain *dom) 107 { 108 return container_of(dom, struct arm_smmu_domain, domain); 109 } 110 111 static struct platform_driver arm_smmu_driver; 112 static struct iommu_ops arm_smmu_ops; 113 114 #ifdef CONFIG_ARM_SMMU_LEGACY_DT_BINDINGS 115 static struct device_node *dev_get_dev_node(struct device *dev) 116 { 117 if (dev_is_pci(dev)) { 118 struct pci_bus *bus = to_pci_dev(dev)->bus; 119 120 while (!pci_is_root_bus(bus)) 121 bus = bus->parent; 122 return of_node_get(bus->bridge->parent->of_node); 123 } 124 125 return of_node_get(dev->of_node); 126 } 127 128 static int __arm_smmu_get_pci_sid(struct pci_dev *pdev, u16 alias, void *data) 129 { 130 *((__be32 *)data) = cpu_to_be32(alias); 131 return 0; /* Continue walking */ 132 } 133 134 static int __find_legacy_master_phandle(struct device *dev, void *data) 135 { 136 struct of_phandle_iterator *it = *(void **)data; 137 struct device_node *np = it->node; 138 int err; 139 140 of_for_each_phandle(it, err, dev->of_node, "mmu-masters", 141 "#stream-id-cells", -1) 142 if (it->node == np) { 143 *(void **)data = dev; 144 return 1; 145 } 146 it->node = np; 147 return err == -ENOENT ? 0 : err; 148 } 149 150 static int arm_smmu_register_legacy_master(struct device *dev, 151 struct arm_smmu_device **smmu) 152 { 153 struct device *smmu_dev; 154 struct device_node *np; 155 struct of_phandle_iterator it; 156 void *data = ⁢ 157 u32 *sids; 158 __be32 pci_sid; 159 int err; 160 161 np = dev_get_dev_node(dev); 162 if (!np || !of_property_present(np, "#stream-id-cells")) { 163 of_node_put(np); 164 return -ENODEV; 165 } 166 167 it.node = np; 168 err = driver_for_each_device(&arm_smmu_driver.driver, NULL, &data, 169 __find_legacy_master_phandle); 170 smmu_dev = data; 171 of_node_put(np); 172 if (err == 0) 173 return -ENODEV; 174 if (err < 0) 175 return err; 176 177 if (dev_is_pci(dev)) { 178 /* "mmu-masters" assumes Stream ID == Requester ID */ 179 pci_for_each_dma_alias(to_pci_dev(dev), __arm_smmu_get_pci_sid, 180 &pci_sid); 181 it.cur = &pci_sid; 182 it.cur_count = 1; 183 } 184 185 err = iommu_fwspec_init(dev, NULL); 186 if (err) 187 return err; 188 189 sids = kcalloc(it.cur_count, sizeof(*sids), GFP_KERNEL); 190 if (!sids) 191 return -ENOMEM; 192 193 *smmu = dev_get_drvdata(smmu_dev); 194 of_phandle_iterator_args(&it, sids, it.cur_count); 195 err = iommu_fwspec_add_ids(dev, sids, it.cur_count); 196 kfree(sids); 197 return err; 198 } 199 #else 200 static int arm_smmu_register_legacy_master(struct device *dev, 201 struct arm_smmu_device **smmu) 202 { 203 return -ENODEV; 204 } 205 #endif /* CONFIG_ARM_SMMU_LEGACY_DT_BINDINGS */ 206 207 static void __arm_smmu_free_bitmap(unsigned long *map, int idx) 208 { 209 clear_bit(idx, map); 210 } 211 212 /* Wait for any pending TLB invalidations to complete */ 213 static void __arm_smmu_tlb_sync(struct arm_smmu_device *smmu, int page, 214 int sync, int status) 215 { 216 unsigned int spin_cnt, delay; 217 u32 reg; 218 219 if (smmu->impl && unlikely(smmu->impl->tlb_sync)) 220 return smmu->impl->tlb_sync(smmu, page, sync, status); 221 222 arm_smmu_writel(smmu, page, sync, QCOM_DUMMY_VAL); 223 for (delay = 1; delay < TLB_LOOP_TIMEOUT; delay *= 2) { 224 for (spin_cnt = TLB_SPIN_COUNT; spin_cnt > 0; spin_cnt--) { 225 reg = arm_smmu_readl(smmu, page, status); 226 if (!(reg & ARM_SMMU_sTLBGSTATUS_GSACTIVE)) 227 return; 228 cpu_relax(); 229 } 230 udelay(delay); 231 } 232 dev_err_ratelimited(smmu->dev, 233 "TLB sync timed out -- SMMU may be deadlocked\n"); 234 } 235 236 static void arm_smmu_tlb_sync_global(struct arm_smmu_device *smmu) 237 { 238 unsigned long flags; 239 240 spin_lock_irqsave(&smmu->global_sync_lock, flags); 241 __arm_smmu_tlb_sync(smmu, ARM_SMMU_GR0, ARM_SMMU_GR0_sTLBGSYNC, 242 ARM_SMMU_GR0_sTLBGSTATUS); 243 spin_unlock_irqrestore(&smmu->global_sync_lock, flags); 244 } 245 246 static void arm_smmu_tlb_sync_context(struct arm_smmu_domain *smmu_domain) 247 { 248 struct arm_smmu_device *smmu = smmu_domain->smmu; 249 unsigned long flags; 250 251 spin_lock_irqsave(&smmu_domain->cb_lock, flags); 252 __arm_smmu_tlb_sync(smmu, ARM_SMMU_CB(smmu, smmu_domain->cfg.cbndx), 253 ARM_SMMU_CB_TLBSYNC, ARM_SMMU_CB_TLBSTATUS); 254 spin_unlock_irqrestore(&smmu_domain->cb_lock, flags); 255 } 256 257 static void arm_smmu_tlb_inv_context_s1(void *cookie) 258 { 259 struct arm_smmu_domain *smmu_domain = cookie; 260 /* 261 * The TLBI write may be relaxed, so ensure that PTEs cleared by the 262 * current CPU are visible beforehand. 263 */ 264 wmb(); 265 arm_smmu_cb_write(smmu_domain->smmu, smmu_domain->cfg.cbndx, 266 ARM_SMMU_CB_S1_TLBIASID, smmu_domain->cfg.asid); 267 arm_smmu_tlb_sync_context(smmu_domain); 268 } 269 270 static void arm_smmu_tlb_inv_context_s2(void *cookie) 271 { 272 struct arm_smmu_domain *smmu_domain = cookie; 273 struct arm_smmu_device *smmu = smmu_domain->smmu; 274 275 /* See above */ 276 wmb(); 277 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_TLBIVMID, smmu_domain->cfg.vmid); 278 arm_smmu_tlb_sync_global(smmu); 279 } 280 281 static void arm_smmu_tlb_inv_range_s1(unsigned long iova, size_t size, 282 size_t granule, void *cookie, int reg) 283 { 284 struct arm_smmu_domain *smmu_domain = cookie; 285 struct arm_smmu_device *smmu = smmu_domain->smmu; 286 struct arm_smmu_cfg *cfg = &smmu_domain->cfg; 287 int idx = cfg->cbndx; 288 289 if (smmu->features & ARM_SMMU_FEAT_COHERENT_WALK) 290 wmb(); 291 292 if (cfg->fmt != ARM_SMMU_CTX_FMT_AARCH64) { 293 iova = (iova >> 12) << 12; 294 iova |= cfg->asid; 295 do { 296 arm_smmu_cb_write(smmu, idx, reg, iova); 297 iova += granule; 298 } while (size -= granule); 299 } else { 300 iova >>= 12; 301 iova |= (u64)cfg->asid << 48; 302 do { 303 arm_smmu_cb_writeq(smmu, idx, reg, iova); 304 iova += granule >> 12; 305 } while (size -= granule); 306 } 307 } 308 309 static void arm_smmu_tlb_inv_range_s2(unsigned long iova, size_t size, 310 size_t granule, void *cookie, int reg) 311 { 312 struct arm_smmu_domain *smmu_domain = cookie; 313 struct arm_smmu_device *smmu = smmu_domain->smmu; 314 int idx = smmu_domain->cfg.cbndx; 315 316 if (smmu->features & ARM_SMMU_FEAT_COHERENT_WALK) 317 wmb(); 318 319 iova >>= 12; 320 do { 321 if (smmu_domain->cfg.fmt == ARM_SMMU_CTX_FMT_AARCH64) 322 arm_smmu_cb_writeq(smmu, idx, reg, iova); 323 else 324 arm_smmu_cb_write(smmu, idx, reg, iova); 325 iova += granule >> 12; 326 } while (size -= granule); 327 } 328 329 static void arm_smmu_tlb_inv_walk_s1(unsigned long iova, size_t size, 330 size_t granule, void *cookie) 331 { 332 struct arm_smmu_domain *smmu_domain = cookie; 333 struct arm_smmu_cfg *cfg = &smmu_domain->cfg; 334 335 if (cfg->flush_walk_prefer_tlbiasid) { 336 arm_smmu_tlb_inv_context_s1(cookie); 337 } else { 338 arm_smmu_tlb_inv_range_s1(iova, size, granule, cookie, 339 ARM_SMMU_CB_S1_TLBIVA); 340 arm_smmu_tlb_sync_context(cookie); 341 } 342 } 343 344 static void arm_smmu_tlb_add_page_s1(struct iommu_iotlb_gather *gather, 345 unsigned long iova, size_t granule, 346 void *cookie) 347 { 348 arm_smmu_tlb_inv_range_s1(iova, granule, granule, cookie, 349 ARM_SMMU_CB_S1_TLBIVAL); 350 } 351 352 static void arm_smmu_tlb_inv_walk_s2(unsigned long iova, size_t size, 353 size_t granule, void *cookie) 354 { 355 arm_smmu_tlb_inv_range_s2(iova, size, granule, cookie, 356 ARM_SMMU_CB_S2_TLBIIPAS2); 357 arm_smmu_tlb_sync_context(cookie); 358 } 359 360 static void arm_smmu_tlb_add_page_s2(struct iommu_iotlb_gather *gather, 361 unsigned long iova, size_t granule, 362 void *cookie) 363 { 364 arm_smmu_tlb_inv_range_s2(iova, granule, granule, cookie, 365 ARM_SMMU_CB_S2_TLBIIPAS2L); 366 } 367 368 static void arm_smmu_tlb_inv_walk_s2_v1(unsigned long iova, size_t size, 369 size_t granule, void *cookie) 370 { 371 arm_smmu_tlb_inv_context_s2(cookie); 372 } 373 /* 374 * On MMU-401 at least, the cost of firing off multiple TLBIVMIDs appears 375 * almost negligible, but the benefit of getting the first one in as far ahead 376 * of the sync as possible is significant, hence we don't just make this a 377 * no-op and call arm_smmu_tlb_inv_context_s2() from .iotlb_sync as you might 378 * think. 379 */ 380 static void arm_smmu_tlb_add_page_s2_v1(struct iommu_iotlb_gather *gather, 381 unsigned long iova, size_t granule, 382 void *cookie) 383 { 384 struct arm_smmu_domain *smmu_domain = cookie; 385 struct arm_smmu_device *smmu = smmu_domain->smmu; 386 387 if (smmu->features & ARM_SMMU_FEAT_COHERENT_WALK) 388 wmb(); 389 390 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_TLBIVMID, smmu_domain->cfg.vmid); 391 } 392 393 static const struct iommu_flush_ops arm_smmu_s1_tlb_ops = { 394 .tlb_flush_all = arm_smmu_tlb_inv_context_s1, 395 .tlb_flush_walk = arm_smmu_tlb_inv_walk_s1, 396 .tlb_add_page = arm_smmu_tlb_add_page_s1, 397 }; 398 399 static const struct iommu_flush_ops arm_smmu_s2_tlb_ops_v2 = { 400 .tlb_flush_all = arm_smmu_tlb_inv_context_s2, 401 .tlb_flush_walk = arm_smmu_tlb_inv_walk_s2, 402 .tlb_add_page = arm_smmu_tlb_add_page_s2, 403 }; 404 405 static const struct iommu_flush_ops arm_smmu_s2_tlb_ops_v1 = { 406 .tlb_flush_all = arm_smmu_tlb_inv_context_s2, 407 .tlb_flush_walk = arm_smmu_tlb_inv_walk_s2_v1, 408 .tlb_add_page = arm_smmu_tlb_add_page_s2_v1, 409 }; 410 411 412 void arm_smmu_read_context_fault_info(struct arm_smmu_device *smmu, int idx, 413 struct arm_smmu_context_fault_info *cfi) 414 { 415 cfi->iova = arm_smmu_cb_readq(smmu, idx, ARM_SMMU_CB_FAR); 416 cfi->fsr = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_FSR); 417 cfi->fsynr = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_FSYNR0); 418 cfi->cbfrsynra = arm_smmu_gr1_read(smmu, ARM_SMMU_GR1_CBFRSYNRA(idx)); 419 } 420 421 void arm_smmu_print_context_fault_info(struct arm_smmu_device *smmu, int idx, 422 const struct arm_smmu_context_fault_info *cfi) 423 { 424 dev_err(smmu->dev, 425 "Unhandled context fault: fsr=0x%x, iova=0x%08lx, fsynr=0x%x, cbfrsynra=0x%x, cb=%d\n", 426 cfi->fsr, cfi->iova, cfi->fsynr, cfi->cbfrsynra, idx); 427 428 dev_err(smmu->dev, "FSR = %08x [%s%sFormat=%u%s%s%s%s%s%s%s%s], SID=0x%x\n", 429 cfi->fsr, 430 (cfi->fsr & ARM_SMMU_CB_FSR_MULTI) ? "MULTI " : "", 431 (cfi->fsr & ARM_SMMU_CB_FSR_SS) ? "SS " : "", 432 (u32)FIELD_GET(ARM_SMMU_CB_FSR_FORMAT, cfi->fsr), 433 (cfi->fsr & ARM_SMMU_CB_FSR_UUT) ? " UUT" : "", 434 (cfi->fsr & ARM_SMMU_CB_FSR_ASF) ? " ASF" : "", 435 (cfi->fsr & ARM_SMMU_CB_FSR_TLBLKF) ? " TLBLKF" : "", 436 (cfi->fsr & ARM_SMMU_CB_FSR_TLBMCF) ? " TLBMCF" : "", 437 (cfi->fsr & ARM_SMMU_CB_FSR_EF) ? " EF" : "", 438 (cfi->fsr & ARM_SMMU_CB_FSR_PF) ? " PF" : "", 439 (cfi->fsr & ARM_SMMU_CB_FSR_AFF) ? " AFF" : "", 440 (cfi->fsr & ARM_SMMU_CB_FSR_TF) ? " TF" : "", 441 cfi->cbfrsynra); 442 443 dev_err(smmu->dev, "FSYNR0 = %08x [S1CBNDX=%u%s%s%s%s%s%s PLVL=%u]\n", 444 cfi->fsynr, 445 (u32)FIELD_GET(ARM_SMMU_CB_FSYNR0_S1CBNDX, cfi->fsynr), 446 (cfi->fsynr & ARM_SMMU_CB_FSYNR0_AFR) ? " AFR" : "", 447 (cfi->fsynr & ARM_SMMU_CB_FSYNR0_PTWF) ? " PTWF" : "", 448 (cfi->fsynr & ARM_SMMU_CB_FSYNR0_NSATTR) ? " NSATTR" : "", 449 (cfi->fsynr & ARM_SMMU_CB_FSYNR0_IND) ? " IND" : "", 450 (cfi->fsynr & ARM_SMMU_CB_FSYNR0_PNU) ? " PNU" : "", 451 (cfi->fsynr & ARM_SMMU_CB_FSYNR0_WNR) ? " WNR" : "", 452 (u32)FIELD_GET(ARM_SMMU_CB_FSYNR0_PLVL, cfi->fsynr)); 453 } 454 455 static irqreturn_t arm_smmu_context_fault(int irq, void *dev) 456 { 457 struct arm_smmu_context_fault_info cfi; 458 struct arm_smmu_domain *smmu_domain = dev; 459 struct arm_smmu_device *smmu = smmu_domain->smmu; 460 static DEFINE_RATELIMIT_STATE(rs, DEFAULT_RATELIMIT_INTERVAL, 461 DEFAULT_RATELIMIT_BURST); 462 int idx = smmu_domain->cfg.cbndx; 463 int ret; 464 465 arm_smmu_read_context_fault_info(smmu, idx, &cfi); 466 467 if (!(cfi.fsr & ARM_SMMU_CB_FSR_FAULT)) 468 return IRQ_NONE; 469 470 ret = report_iommu_fault(&smmu_domain->domain, NULL, cfi.iova, 471 cfi.fsynr & ARM_SMMU_CB_FSYNR0_WNR ? IOMMU_FAULT_WRITE : IOMMU_FAULT_READ); 472 473 if (ret == -ENOSYS && __ratelimit(&rs)) 474 arm_smmu_print_context_fault_info(smmu, idx, &cfi); 475 476 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_FSR, cfi.fsr); 477 return IRQ_HANDLED; 478 } 479 480 static irqreturn_t arm_smmu_global_fault(int irq, void *dev) 481 { 482 u32 gfsr, gfsynr0, gfsynr1, gfsynr2; 483 struct arm_smmu_device *smmu = dev; 484 static DEFINE_RATELIMIT_STATE(rs, DEFAULT_RATELIMIT_INTERVAL, 485 DEFAULT_RATELIMIT_BURST); 486 487 gfsr = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSR); 488 gfsynr0 = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSYNR0); 489 gfsynr1 = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSYNR1); 490 gfsynr2 = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSYNR2); 491 492 if (!gfsr) 493 return IRQ_NONE; 494 495 if (__ratelimit(&rs)) { 496 if (IS_ENABLED(CONFIG_ARM_SMMU_DISABLE_BYPASS_BY_DEFAULT) && 497 (gfsr & ARM_SMMU_sGFSR_USF)) 498 dev_err(smmu->dev, 499 "Blocked unknown Stream ID 0x%hx; boot with \"arm-smmu.disable_bypass=0\" to allow, but this may have security implications\n", 500 (u16)gfsynr1); 501 else 502 dev_err(smmu->dev, 503 "Unexpected global fault, this could be serious\n"); 504 dev_err(smmu->dev, 505 "\tGFSR 0x%08x, GFSYNR0 0x%08x, GFSYNR1 0x%08x, GFSYNR2 0x%08x\n", 506 gfsr, gfsynr0, gfsynr1, gfsynr2); 507 } 508 509 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sGFSR, gfsr); 510 return IRQ_HANDLED; 511 } 512 513 static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain, 514 struct io_pgtable_cfg *pgtbl_cfg) 515 { 516 struct arm_smmu_cfg *cfg = &smmu_domain->cfg; 517 struct arm_smmu_cb *cb = &smmu_domain->smmu->cbs[cfg->cbndx]; 518 bool stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS; 519 520 cb->cfg = cfg; 521 522 /* TCR */ 523 if (stage1) { 524 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) { 525 cb->tcr[0] = pgtbl_cfg->arm_v7s_cfg.tcr; 526 } else { 527 cb->tcr[0] = arm_smmu_lpae_tcr(pgtbl_cfg); 528 cb->tcr[1] = arm_smmu_lpae_tcr2(pgtbl_cfg); 529 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) 530 cb->tcr[1] |= ARM_SMMU_TCR2_AS; 531 else 532 cb->tcr[0] |= ARM_SMMU_TCR_EAE; 533 } 534 } else { 535 cb->tcr[0] = arm_smmu_lpae_vtcr(pgtbl_cfg); 536 } 537 538 /* TTBRs */ 539 if (stage1) { 540 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) { 541 cb->ttbr[0] = pgtbl_cfg->arm_v7s_cfg.ttbr; 542 cb->ttbr[1] = 0; 543 } else { 544 cb->ttbr[0] = FIELD_PREP(ARM_SMMU_TTBRn_ASID, 545 cfg->asid); 546 cb->ttbr[1] = FIELD_PREP(ARM_SMMU_TTBRn_ASID, 547 cfg->asid); 548 549 if (pgtbl_cfg->quirks & IO_PGTABLE_QUIRK_ARM_TTBR1) 550 cb->ttbr[1] |= pgtbl_cfg->arm_lpae_s1_cfg.ttbr; 551 else 552 cb->ttbr[0] |= pgtbl_cfg->arm_lpae_s1_cfg.ttbr; 553 } 554 } else { 555 cb->ttbr[0] = pgtbl_cfg->arm_lpae_s2_cfg.vttbr; 556 } 557 558 /* MAIRs (stage-1 only) */ 559 if (stage1) { 560 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) { 561 cb->mair[0] = pgtbl_cfg->arm_v7s_cfg.prrr; 562 cb->mair[1] = pgtbl_cfg->arm_v7s_cfg.nmrr; 563 } else { 564 cb->mair[0] = pgtbl_cfg->arm_lpae_s1_cfg.mair; 565 cb->mair[1] = pgtbl_cfg->arm_lpae_s1_cfg.mair >> 32; 566 } 567 } 568 } 569 570 void arm_smmu_write_context_bank(struct arm_smmu_device *smmu, int idx) 571 { 572 u32 reg; 573 bool stage1; 574 struct arm_smmu_cb *cb = &smmu->cbs[idx]; 575 struct arm_smmu_cfg *cfg = cb->cfg; 576 577 /* Unassigned context banks only need disabling */ 578 if (!cfg) { 579 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_SCTLR, 0); 580 return; 581 } 582 583 stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS; 584 585 /* CBA2R */ 586 if (smmu->version > ARM_SMMU_V1) { 587 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) 588 reg = ARM_SMMU_CBA2R_VA64; 589 else 590 reg = 0; 591 /* 16-bit VMIDs live in CBA2R */ 592 if (smmu->features & ARM_SMMU_FEAT_VMID16) 593 reg |= FIELD_PREP(ARM_SMMU_CBA2R_VMID16, cfg->vmid); 594 595 arm_smmu_gr1_write(smmu, ARM_SMMU_GR1_CBA2R(idx), reg); 596 } 597 598 /* CBAR */ 599 reg = FIELD_PREP(ARM_SMMU_CBAR_TYPE, cfg->cbar); 600 if (smmu->version < ARM_SMMU_V2) 601 reg |= FIELD_PREP(ARM_SMMU_CBAR_IRPTNDX, cfg->irptndx); 602 603 /* 604 * Use the weakest shareability/memory types, so they are 605 * overridden by the ttbcr/pte. 606 */ 607 if (stage1) { 608 reg |= FIELD_PREP(ARM_SMMU_CBAR_S1_BPSHCFG, 609 ARM_SMMU_CBAR_S1_BPSHCFG_NSH) | 610 FIELD_PREP(ARM_SMMU_CBAR_S1_MEMATTR, 611 ARM_SMMU_CBAR_S1_MEMATTR_WB); 612 } else if (!(smmu->features & ARM_SMMU_FEAT_VMID16)) { 613 /* 8-bit VMIDs live in CBAR */ 614 reg |= FIELD_PREP(ARM_SMMU_CBAR_VMID, cfg->vmid); 615 } 616 arm_smmu_gr1_write(smmu, ARM_SMMU_GR1_CBAR(idx), reg); 617 618 /* 619 * TCR 620 * We must write this before the TTBRs, since it determines the 621 * access behaviour of some fields (in particular, ASID[15:8]). 622 */ 623 if (stage1 && smmu->version > ARM_SMMU_V1) 624 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_TCR2, cb->tcr[1]); 625 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_TCR, cb->tcr[0]); 626 627 /* TTBRs */ 628 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) { 629 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_CONTEXTIDR, cfg->asid); 630 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_TTBR0, cb->ttbr[0]); 631 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_TTBR1, cb->ttbr[1]); 632 } else { 633 arm_smmu_cb_writeq(smmu, idx, ARM_SMMU_CB_TTBR0, cb->ttbr[0]); 634 if (stage1) 635 arm_smmu_cb_writeq(smmu, idx, ARM_SMMU_CB_TTBR1, 636 cb->ttbr[1]); 637 } 638 639 /* MAIRs (stage-1 only) */ 640 if (stage1) { 641 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_S1_MAIR0, cb->mair[0]); 642 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_S1_MAIR1, cb->mair[1]); 643 } 644 645 /* SCTLR */ 646 reg = ARM_SMMU_SCTLR_CFIE | ARM_SMMU_SCTLR_CFRE | ARM_SMMU_SCTLR_AFE | 647 ARM_SMMU_SCTLR_TRE | ARM_SMMU_SCTLR_M; 648 if (stage1) 649 reg |= ARM_SMMU_SCTLR_S1_ASIDPNE; 650 if (IS_ENABLED(CONFIG_CPU_BIG_ENDIAN)) 651 reg |= ARM_SMMU_SCTLR_E; 652 653 if (smmu->impl && smmu->impl->write_sctlr) 654 smmu->impl->write_sctlr(smmu, idx, reg); 655 else 656 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_SCTLR, reg); 657 } 658 659 static int arm_smmu_alloc_context_bank(struct arm_smmu_domain *smmu_domain, 660 struct arm_smmu_device *smmu, 661 struct device *dev, unsigned int start) 662 { 663 if (smmu->impl && smmu->impl->alloc_context_bank) 664 return smmu->impl->alloc_context_bank(smmu_domain, smmu, dev, start); 665 666 return __arm_smmu_alloc_bitmap(smmu->context_map, start, smmu->num_context_banks); 667 } 668 669 static int arm_smmu_init_domain_context(struct arm_smmu_domain *smmu_domain, 670 struct arm_smmu_device *smmu, 671 struct device *dev) 672 { 673 int irq, start, ret = 0; 674 unsigned long ias, oas; 675 struct io_pgtable_ops *pgtbl_ops; 676 struct io_pgtable_cfg pgtbl_cfg; 677 enum io_pgtable_fmt fmt; 678 struct iommu_domain *domain = &smmu_domain->domain; 679 struct arm_smmu_cfg *cfg = &smmu_domain->cfg; 680 irqreturn_t (*context_fault)(int irq, void *dev); 681 682 mutex_lock(&smmu_domain->init_mutex); 683 if (smmu_domain->smmu) 684 goto out_unlock; 685 686 /* 687 * Mapping the requested stage onto what we support is surprisingly 688 * complicated, mainly because the spec allows S1+S2 SMMUs without 689 * support for nested translation. That means we end up with the 690 * following table: 691 * 692 * Requested Supported Actual 693 * S1 N S1 694 * S1 S1+S2 S1 695 * S1 S2 S2 696 * S1 S1 S1 697 * N N N 698 * N S1+S2 S2 699 * N S2 S2 700 * N S1 S1 701 * 702 * Note that you can't actually request stage-2 mappings. 703 */ 704 if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S1)) 705 smmu_domain->stage = ARM_SMMU_DOMAIN_S2; 706 if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S2)) 707 smmu_domain->stage = ARM_SMMU_DOMAIN_S1; 708 709 /* 710 * Choosing a suitable context format is even more fiddly. Until we 711 * grow some way for the caller to express a preference, and/or move 712 * the decision into the io-pgtable code where it arguably belongs, 713 * just aim for the closest thing to the rest of the system, and hope 714 * that the hardware isn't esoteric enough that we can't assume AArch64 715 * support to be a superset of AArch32 support... 716 */ 717 if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_L) 718 cfg->fmt = ARM_SMMU_CTX_FMT_AARCH32_L; 719 if (IS_ENABLED(CONFIG_IOMMU_IO_PGTABLE_ARMV7S) && 720 !IS_ENABLED(CONFIG_64BIT) && !IS_ENABLED(CONFIG_ARM_LPAE) && 721 (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_S) && 722 (smmu_domain->stage == ARM_SMMU_DOMAIN_S1)) 723 cfg->fmt = ARM_SMMU_CTX_FMT_AARCH32_S; 724 if ((IS_ENABLED(CONFIG_64BIT) || cfg->fmt == ARM_SMMU_CTX_FMT_NONE) && 725 (smmu->features & (ARM_SMMU_FEAT_FMT_AARCH64_64K | 726 ARM_SMMU_FEAT_FMT_AARCH64_16K | 727 ARM_SMMU_FEAT_FMT_AARCH64_4K))) 728 cfg->fmt = ARM_SMMU_CTX_FMT_AARCH64; 729 730 if (cfg->fmt == ARM_SMMU_CTX_FMT_NONE) { 731 ret = -EINVAL; 732 goto out_unlock; 733 } 734 735 switch (smmu_domain->stage) { 736 case ARM_SMMU_DOMAIN_S1: 737 cfg->cbar = CBAR_TYPE_S1_TRANS_S2_BYPASS; 738 start = smmu->num_s2_context_banks; 739 ias = smmu->va_size; 740 oas = smmu->ipa_size; 741 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) { 742 fmt = ARM_64_LPAE_S1; 743 } else if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_L) { 744 fmt = ARM_32_LPAE_S1; 745 ias = min(ias, 32UL); 746 oas = min(oas, 40UL); 747 } else { 748 fmt = ARM_V7S; 749 ias = min(ias, 32UL); 750 oas = min(oas, 32UL); 751 } 752 smmu_domain->flush_ops = &arm_smmu_s1_tlb_ops; 753 break; 754 case ARM_SMMU_DOMAIN_NESTED: 755 /* 756 * We will likely want to change this if/when KVM gets 757 * involved. 758 */ 759 case ARM_SMMU_DOMAIN_S2: 760 cfg->cbar = CBAR_TYPE_S2_TRANS; 761 start = 0; 762 ias = smmu->ipa_size; 763 oas = smmu->pa_size; 764 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) { 765 fmt = ARM_64_LPAE_S2; 766 } else { 767 fmt = ARM_32_LPAE_S2; 768 ias = min(ias, 40UL); 769 oas = min(oas, 40UL); 770 } 771 if (smmu->version == ARM_SMMU_V2) 772 smmu_domain->flush_ops = &arm_smmu_s2_tlb_ops_v2; 773 else 774 smmu_domain->flush_ops = &arm_smmu_s2_tlb_ops_v1; 775 break; 776 default: 777 ret = -EINVAL; 778 goto out_unlock; 779 } 780 781 ret = arm_smmu_alloc_context_bank(smmu_domain, smmu, dev, start); 782 if (ret < 0) { 783 goto out_unlock; 784 } 785 786 smmu_domain->smmu = smmu; 787 788 cfg->cbndx = ret; 789 if (smmu->version < ARM_SMMU_V2) { 790 cfg->irptndx = atomic_inc_return(&smmu->irptndx); 791 cfg->irptndx %= smmu->num_context_irqs; 792 } else { 793 cfg->irptndx = cfg->cbndx; 794 } 795 796 if (smmu_domain->stage == ARM_SMMU_DOMAIN_S2) 797 cfg->vmid = cfg->cbndx + 1; 798 else 799 cfg->asid = cfg->cbndx; 800 801 pgtbl_cfg = (struct io_pgtable_cfg) { 802 .pgsize_bitmap = smmu->pgsize_bitmap, 803 .ias = ias, 804 .oas = oas, 805 .coherent_walk = smmu->features & ARM_SMMU_FEAT_COHERENT_WALK, 806 .tlb = smmu_domain->flush_ops, 807 .iommu_dev = smmu->dev, 808 }; 809 810 if (smmu->impl && smmu->impl->init_context) { 811 ret = smmu->impl->init_context(smmu_domain, &pgtbl_cfg, dev); 812 if (ret) 813 goto out_clear_smmu; 814 } 815 816 if (smmu_domain->pgtbl_quirks) 817 pgtbl_cfg.quirks |= smmu_domain->pgtbl_quirks; 818 819 pgtbl_ops = alloc_io_pgtable_ops(fmt, &pgtbl_cfg, smmu_domain); 820 if (!pgtbl_ops) { 821 ret = -ENOMEM; 822 goto out_clear_smmu; 823 } 824 825 /* Update the domain's page sizes to reflect the page table format */ 826 domain->pgsize_bitmap = pgtbl_cfg.pgsize_bitmap; 827 828 if (pgtbl_cfg.quirks & IO_PGTABLE_QUIRK_ARM_TTBR1) { 829 domain->geometry.aperture_start = ~0UL << ias; 830 domain->geometry.aperture_end = ~0UL; 831 } else { 832 domain->geometry.aperture_end = (1UL << ias) - 1; 833 } 834 835 domain->geometry.force_aperture = true; 836 837 /* Initialise the context bank with our page table cfg */ 838 arm_smmu_init_context_bank(smmu_domain, &pgtbl_cfg); 839 arm_smmu_write_context_bank(smmu, cfg->cbndx); 840 841 /* 842 * Request context fault interrupt. Do this last to avoid the 843 * handler seeing a half-initialised domain state. 844 */ 845 irq = smmu->irqs[cfg->irptndx]; 846 847 if (smmu->impl && smmu->impl->context_fault) 848 context_fault = smmu->impl->context_fault; 849 else 850 context_fault = arm_smmu_context_fault; 851 852 if (smmu->impl && smmu->impl->context_fault_needs_threaded_irq) 853 ret = devm_request_threaded_irq(smmu->dev, irq, NULL, 854 context_fault, 855 IRQF_ONESHOT | IRQF_SHARED, 856 "arm-smmu-context-fault", 857 smmu_domain); 858 else 859 ret = devm_request_irq(smmu->dev, irq, context_fault, IRQF_SHARED, 860 "arm-smmu-context-fault", smmu_domain); 861 862 if (ret < 0) { 863 dev_err(smmu->dev, "failed to request context IRQ %d (%u)\n", 864 cfg->irptndx, irq); 865 cfg->irptndx = ARM_SMMU_INVALID_IRPTNDX; 866 } 867 868 mutex_unlock(&smmu_domain->init_mutex); 869 870 /* Publish page table ops for map/unmap */ 871 smmu_domain->pgtbl_ops = pgtbl_ops; 872 return 0; 873 874 out_clear_smmu: 875 __arm_smmu_free_bitmap(smmu->context_map, cfg->cbndx); 876 smmu_domain->smmu = NULL; 877 out_unlock: 878 mutex_unlock(&smmu_domain->init_mutex); 879 return ret; 880 } 881 882 static void arm_smmu_destroy_domain_context(struct arm_smmu_domain *smmu_domain) 883 { 884 struct arm_smmu_device *smmu = smmu_domain->smmu; 885 struct arm_smmu_cfg *cfg = &smmu_domain->cfg; 886 int ret, irq; 887 888 if (!smmu) 889 return; 890 891 ret = arm_smmu_rpm_get(smmu); 892 if (ret < 0) 893 return; 894 895 /* 896 * Disable the context bank and free the page tables before freeing 897 * it. 898 */ 899 smmu->cbs[cfg->cbndx].cfg = NULL; 900 arm_smmu_write_context_bank(smmu, cfg->cbndx); 901 902 if (cfg->irptndx != ARM_SMMU_INVALID_IRPTNDX) { 903 irq = smmu->irqs[cfg->irptndx]; 904 devm_free_irq(smmu->dev, irq, smmu_domain); 905 } 906 907 free_io_pgtable_ops(smmu_domain->pgtbl_ops); 908 __arm_smmu_free_bitmap(smmu->context_map, cfg->cbndx); 909 910 arm_smmu_rpm_put(smmu); 911 } 912 913 static struct iommu_domain *arm_smmu_domain_alloc_paging(struct device *dev) 914 { 915 struct arm_smmu_domain *smmu_domain; 916 917 /* 918 * Allocate the domain and initialise some of its data structures. 919 * We can't really do anything meaningful until we've added a 920 * master. 921 */ 922 smmu_domain = kzalloc(sizeof(*smmu_domain), GFP_KERNEL); 923 if (!smmu_domain) 924 return NULL; 925 926 mutex_init(&smmu_domain->init_mutex); 927 spin_lock_init(&smmu_domain->cb_lock); 928 929 return &smmu_domain->domain; 930 } 931 932 static void arm_smmu_domain_free(struct iommu_domain *domain) 933 { 934 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); 935 936 /* 937 * Free the domain resources. We assume that all devices have 938 * already been detached. 939 */ 940 arm_smmu_destroy_domain_context(smmu_domain); 941 kfree(smmu_domain); 942 } 943 944 static void arm_smmu_write_smr(struct arm_smmu_device *smmu, int idx) 945 { 946 struct arm_smmu_smr *smr = smmu->smrs + idx; 947 u32 reg = FIELD_PREP(ARM_SMMU_SMR_ID, smr->id) | 948 FIELD_PREP(ARM_SMMU_SMR_MASK, smr->mask); 949 950 if (!(smmu->features & ARM_SMMU_FEAT_EXIDS) && smr->valid) 951 reg |= ARM_SMMU_SMR_VALID; 952 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_SMR(idx), reg); 953 } 954 955 static void arm_smmu_write_s2cr(struct arm_smmu_device *smmu, int idx) 956 { 957 struct arm_smmu_s2cr *s2cr = smmu->s2crs + idx; 958 u32 reg; 959 960 if (smmu->impl && smmu->impl->write_s2cr) { 961 smmu->impl->write_s2cr(smmu, idx); 962 return; 963 } 964 965 reg = FIELD_PREP(ARM_SMMU_S2CR_TYPE, s2cr->type) | 966 FIELD_PREP(ARM_SMMU_S2CR_CBNDX, s2cr->cbndx) | 967 FIELD_PREP(ARM_SMMU_S2CR_PRIVCFG, s2cr->privcfg); 968 969 if (smmu->features & ARM_SMMU_FEAT_EXIDS && smmu->smrs && 970 smmu->smrs[idx].valid) 971 reg |= ARM_SMMU_S2CR_EXIDVALID; 972 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_S2CR(idx), reg); 973 } 974 975 static void arm_smmu_write_sme(struct arm_smmu_device *smmu, int idx) 976 { 977 arm_smmu_write_s2cr(smmu, idx); 978 if (smmu->smrs) 979 arm_smmu_write_smr(smmu, idx); 980 } 981 982 /* 983 * The width of SMR's mask field depends on sCR0_EXIDENABLE, so this function 984 * should be called after sCR0 is written. 985 */ 986 static void arm_smmu_test_smr_masks(struct arm_smmu_device *smmu) 987 { 988 u32 smr; 989 int i; 990 991 if (!smmu->smrs) 992 return; 993 /* 994 * If we've had to accommodate firmware memory regions, we may 995 * have live SMRs by now; tread carefully... 996 * 997 * Somewhat perversely, not having a free SMR for this test implies we 998 * can get away without it anyway, as we'll only be able to 'allocate' 999 * these SMRs for the ID/mask values we're already trusting to be OK. 1000 */ 1001 for (i = 0; i < smmu->num_mapping_groups; i++) 1002 if (!smmu->smrs[i].valid) 1003 goto smr_ok; 1004 return; 1005 smr_ok: 1006 /* 1007 * SMR.ID bits may not be preserved if the corresponding MASK 1008 * bits are set, so check each one separately. We can reject 1009 * masters later if they try to claim IDs outside these masks. 1010 */ 1011 smr = FIELD_PREP(ARM_SMMU_SMR_ID, smmu->streamid_mask); 1012 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_SMR(i), smr); 1013 smr = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_SMR(i)); 1014 smmu->streamid_mask = FIELD_GET(ARM_SMMU_SMR_ID, smr); 1015 1016 smr = FIELD_PREP(ARM_SMMU_SMR_MASK, smmu->streamid_mask); 1017 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_SMR(i), smr); 1018 smr = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_SMR(i)); 1019 smmu->smr_mask_mask = FIELD_GET(ARM_SMMU_SMR_MASK, smr); 1020 } 1021 1022 static int arm_smmu_find_sme(struct arm_smmu_device *smmu, u16 id, u16 mask) 1023 { 1024 struct arm_smmu_smr *smrs = smmu->smrs; 1025 int i, free_idx = -ENOSPC; 1026 1027 /* Stream indexing is blissfully easy */ 1028 if (!smrs) 1029 return id; 1030 1031 /* Validating SMRs is... less so */ 1032 for (i = 0; i < smmu->num_mapping_groups; ++i) { 1033 if (!smrs[i].valid) { 1034 /* 1035 * Note the first free entry we come across, which 1036 * we'll claim in the end if nothing else matches. 1037 */ 1038 if (free_idx < 0) 1039 free_idx = i; 1040 continue; 1041 } 1042 /* 1043 * If the new entry is _entirely_ matched by an existing entry, 1044 * then reuse that, with the guarantee that there also cannot 1045 * be any subsequent conflicting entries. In normal use we'd 1046 * expect simply identical entries for this case, but there's 1047 * no harm in accommodating the generalisation. 1048 */ 1049 if ((mask & smrs[i].mask) == mask && 1050 !((id ^ smrs[i].id) & ~smrs[i].mask)) 1051 return i; 1052 /* 1053 * If the new entry has any other overlap with an existing one, 1054 * though, then there always exists at least one stream ID 1055 * which would cause a conflict, and we can't allow that risk. 1056 */ 1057 if (!((id ^ smrs[i].id) & ~(smrs[i].mask | mask))) 1058 return -EINVAL; 1059 } 1060 1061 return free_idx; 1062 } 1063 1064 static bool arm_smmu_free_sme(struct arm_smmu_device *smmu, int idx) 1065 { 1066 if (--smmu->s2crs[idx].count) 1067 return false; 1068 1069 smmu->s2crs[idx] = s2cr_init_val; 1070 if (smmu->smrs) 1071 smmu->smrs[idx].valid = false; 1072 1073 return true; 1074 } 1075 1076 static int arm_smmu_master_alloc_smes(struct device *dev) 1077 { 1078 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); 1079 struct arm_smmu_master_cfg *cfg = dev_iommu_priv_get(dev); 1080 struct arm_smmu_device *smmu = cfg->smmu; 1081 struct arm_smmu_smr *smrs = smmu->smrs; 1082 int i, idx, ret; 1083 1084 mutex_lock(&smmu->stream_map_mutex); 1085 /* Figure out a viable stream map entry allocation */ 1086 for_each_cfg_sme(cfg, fwspec, i, idx) { 1087 u16 sid = FIELD_GET(ARM_SMMU_SMR_ID, fwspec->ids[i]); 1088 u16 mask = FIELD_GET(ARM_SMMU_SMR_MASK, fwspec->ids[i]); 1089 1090 if (idx != INVALID_SMENDX) { 1091 ret = -EEXIST; 1092 goto out_err; 1093 } 1094 1095 ret = arm_smmu_find_sme(smmu, sid, mask); 1096 if (ret < 0) 1097 goto out_err; 1098 1099 idx = ret; 1100 if (smrs && smmu->s2crs[idx].count == 0) { 1101 smrs[idx].id = sid; 1102 smrs[idx].mask = mask; 1103 smrs[idx].valid = true; 1104 } 1105 smmu->s2crs[idx].count++; 1106 cfg->smendx[i] = (s16)idx; 1107 } 1108 1109 /* It worked! Now, poke the actual hardware */ 1110 for_each_cfg_sme(cfg, fwspec, i, idx) 1111 arm_smmu_write_sme(smmu, idx); 1112 1113 mutex_unlock(&smmu->stream_map_mutex); 1114 return 0; 1115 1116 out_err: 1117 while (i--) { 1118 arm_smmu_free_sme(smmu, cfg->smendx[i]); 1119 cfg->smendx[i] = INVALID_SMENDX; 1120 } 1121 mutex_unlock(&smmu->stream_map_mutex); 1122 return ret; 1123 } 1124 1125 static void arm_smmu_master_free_smes(struct arm_smmu_master_cfg *cfg, 1126 struct iommu_fwspec *fwspec) 1127 { 1128 struct arm_smmu_device *smmu = cfg->smmu; 1129 int i, idx; 1130 1131 mutex_lock(&smmu->stream_map_mutex); 1132 for_each_cfg_sme(cfg, fwspec, i, idx) { 1133 if (arm_smmu_free_sme(smmu, idx)) 1134 arm_smmu_write_sme(smmu, idx); 1135 cfg->smendx[i] = INVALID_SMENDX; 1136 } 1137 mutex_unlock(&smmu->stream_map_mutex); 1138 } 1139 1140 static void arm_smmu_master_install_s2crs(struct arm_smmu_master_cfg *cfg, 1141 enum arm_smmu_s2cr_type type, 1142 u8 cbndx, struct iommu_fwspec *fwspec) 1143 { 1144 struct arm_smmu_device *smmu = cfg->smmu; 1145 struct arm_smmu_s2cr *s2cr = smmu->s2crs; 1146 int i, idx; 1147 1148 for_each_cfg_sme(cfg, fwspec, i, idx) { 1149 if (type == s2cr[idx].type && cbndx == s2cr[idx].cbndx) 1150 continue; 1151 1152 s2cr[idx].type = type; 1153 s2cr[idx].privcfg = S2CR_PRIVCFG_DEFAULT; 1154 s2cr[idx].cbndx = cbndx; 1155 arm_smmu_write_s2cr(smmu, idx); 1156 } 1157 } 1158 1159 static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev) 1160 { 1161 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); 1162 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); 1163 struct arm_smmu_master_cfg *cfg; 1164 struct arm_smmu_device *smmu; 1165 int ret; 1166 1167 /* 1168 * FIXME: The arch/arm DMA API code tries to attach devices to its own 1169 * domains between of_xlate() and probe_device() - we have no way to cope 1170 * with that, so until ARM gets converted to rely on groups and default 1171 * domains, just say no (but more politely than by dereferencing NULL). 1172 * This should be at least a WARN_ON once that's sorted. 1173 */ 1174 cfg = dev_iommu_priv_get(dev); 1175 if (!cfg) 1176 return -ENODEV; 1177 1178 smmu = cfg->smmu; 1179 1180 ret = arm_smmu_rpm_get(smmu); 1181 if (ret < 0) 1182 return ret; 1183 1184 /* Ensure that the domain is finalised */ 1185 ret = arm_smmu_init_domain_context(smmu_domain, smmu, dev); 1186 if (ret < 0) 1187 goto rpm_put; 1188 1189 /* 1190 * Sanity check the domain. We don't support domains across 1191 * different SMMUs. 1192 */ 1193 if (smmu_domain->smmu != smmu) { 1194 ret = -EINVAL; 1195 goto rpm_put; 1196 } 1197 1198 /* Looks ok, so add the device to the domain */ 1199 arm_smmu_master_install_s2crs(cfg, S2CR_TYPE_TRANS, 1200 smmu_domain->cfg.cbndx, fwspec); 1201 rpm_put: 1202 arm_smmu_rpm_put(smmu); 1203 return ret; 1204 } 1205 1206 static int arm_smmu_attach_dev_type(struct device *dev, 1207 enum arm_smmu_s2cr_type type) 1208 { 1209 struct arm_smmu_master_cfg *cfg = dev_iommu_priv_get(dev); 1210 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); 1211 struct arm_smmu_device *smmu; 1212 int ret; 1213 1214 if (!cfg) 1215 return -ENODEV; 1216 smmu = cfg->smmu; 1217 1218 ret = arm_smmu_rpm_get(smmu); 1219 if (ret < 0) 1220 return ret; 1221 1222 arm_smmu_master_install_s2crs(cfg, type, 0, fwspec); 1223 arm_smmu_rpm_put(smmu); 1224 return 0; 1225 } 1226 1227 static int arm_smmu_attach_dev_identity(struct iommu_domain *domain, 1228 struct device *dev) 1229 { 1230 return arm_smmu_attach_dev_type(dev, S2CR_TYPE_BYPASS); 1231 } 1232 1233 static const struct iommu_domain_ops arm_smmu_identity_ops = { 1234 .attach_dev = arm_smmu_attach_dev_identity, 1235 }; 1236 1237 static struct iommu_domain arm_smmu_identity_domain = { 1238 .type = IOMMU_DOMAIN_IDENTITY, 1239 .ops = &arm_smmu_identity_ops, 1240 }; 1241 1242 static int arm_smmu_attach_dev_blocked(struct iommu_domain *domain, 1243 struct device *dev) 1244 { 1245 return arm_smmu_attach_dev_type(dev, S2CR_TYPE_FAULT); 1246 } 1247 1248 static const struct iommu_domain_ops arm_smmu_blocked_ops = { 1249 .attach_dev = arm_smmu_attach_dev_blocked, 1250 }; 1251 1252 static struct iommu_domain arm_smmu_blocked_domain = { 1253 .type = IOMMU_DOMAIN_BLOCKED, 1254 .ops = &arm_smmu_blocked_ops, 1255 }; 1256 1257 static int arm_smmu_map_pages(struct iommu_domain *domain, unsigned long iova, 1258 phys_addr_t paddr, size_t pgsize, size_t pgcount, 1259 int prot, gfp_t gfp, size_t *mapped) 1260 { 1261 struct io_pgtable_ops *ops = to_smmu_domain(domain)->pgtbl_ops; 1262 struct arm_smmu_device *smmu = to_smmu_domain(domain)->smmu; 1263 int ret; 1264 1265 if (!ops) 1266 return -ENODEV; 1267 1268 arm_smmu_rpm_get(smmu); 1269 ret = ops->map_pages(ops, iova, paddr, pgsize, pgcount, prot, gfp, mapped); 1270 arm_smmu_rpm_put(smmu); 1271 1272 return ret; 1273 } 1274 1275 static size_t arm_smmu_unmap_pages(struct iommu_domain *domain, unsigned long iova, 1276 size_t pgsize, size_t pgcount, 1277 struct iommu_iotlb_gather *iotlb_gather) 1278 { 1279 struct io_pgtable_ops *ops = to_smmu_domain(domain)->pgtbl_ops; 1280 struct arm_smmu_device *smmu = to_smmu_domain(domain)->smmu; 1281 size_t ret; 1282 1283 if (!ops) 1284 return 0; 1285 1286 arm_smmu_rpm_get(smmu); 1287 ret = ops->unmap_pages(ops, iova, pgsize, pgcount, iotlb_gather); 1288 arm_smmu_rpm_put(smmu); 1289 1290 return ret; 1291 } 1292 1293 static void arm_smmu_flush_iotlb_all(struct iommu_domain *domain) 1294 { 1295 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); 1296 struct arm_smmu_device *smmu = smmu_domain->smmu; 1297 1298 if (smmu_domain->flush_ops) { 1299 arm_smmu_rpm_get(smmu); 1300 smmu_domain->flush_ops->tlb_flush_all(smmu_domain); 1301 arm_smmu_rpm_put(smmu); 1302 } 1303 } 1304 1305 static void arm_smmu_iotlb_sync(struct iommu_domain *domain, 1306 struct iommu_iotlb_gather *gather) 1307 { 1308 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); 1309 struct arm_smmu_device *smmu = smmu_domain->smmu; 1310 1311 if (!smmu) 1312 return; 1313 1314 arm_smmu_rpm_get(smmu); 1315 if (smmu->version == ARM_SMMU_V2 || 1316 smmu_domain->stage == ARM_SMMU_DOMAIN_S1) 1317 arm_smmu_tlb_sync_context(smmu_domain); 1318 else 1319 arm_smmu_tlb_sync_global(smmu); 1320 arm_smmu_rpm_put(smmu); 1321 } 1322 1323 static phys_addr_t arm_smmu_iova_to_phys_hard(struct iommu_domain *domain, 1324 dma_addr_t iova) 1325 { 1326 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); 1327 struct arm_smmu_device *smmu = smmu_domain->smmu; 1328 struct arm_smmu_cfg *cfg = &smmu_domain->cfg; 1329 struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops; 1330 struct device *dev = smmu->dev; 1331 void __iomem *reg; 1332 u32 tmp; 1333 u64 phys; 1334 unsigned long va, flags; 1335 int ret, idx = cfg->cbndx; 1336 phys_addr_t addr = 0; 1337 1338 ret = arm_smmu_rpm_get(smmu); 1339 if (ret < 0) 1340 return 0; 1341 1342 spin_lock_irqsave(&smmu_domain->cb_lock, flags); 1343 va = iova & ~0xfffUL; 1344 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) 1345 arm_smmu_cb_writeq(smmu, idx, ARM_SMMU_CB_ATS1PR, va); 1346 else 1347 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_ATS1PR, va); 1348 1349 reg = arm_smmu_page(smmu, ARM_SMMU_CB(smmu, idx)) + ARM_SMMU_CB_ATSR; 1350 if (readl_poll_timeout_atomic(reg, tmp, !(tmp & ARM_SMMU_CB_ATSR_ACTIVE), 1351 5, 50)) { 1352 spin_unlock_irqrestore(&smmu_domain->cb_lock, flags); 1353 dev_err(dev, 1354 "iova to phys timed out on %pad. Falling back to software table walk.\n", 1355 &iova); 1356 arm_smmu_rpm_put(smmu); 1357 return ops->iova_to_phys(ops, iova); 1358 } 1359 1360 phys = arm_smmu_cb_readq(smmu, idx, ARM_SMMU_CB_PAR); 1361 spin_unlock_irqrestore(&smmu_domain->cb_lock, flags); 1362 if (phys & ARM_SMMU_CB_PAR_F) { 1363 dev_err(dev, "translation fault!\n"); 1364 dev_err(dev, "PAR = 0x%llx\n", phys); 1365 goto out; 1366 } 1367 1368 addr = (phys & GENMASK_ULL(39, 12)) | (iova & 0xfff); 1369 out: 1370 arm_smmu_rpm_put(smmu); 1371 1372 return addr; 1373 } 1374 1375 static phys_addr_t arm_smmu_iova_to_phys(struct iommu_domain *domain, 1376 dma_addr_t iova) 1377 { 1378 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); 1379 struct io_pgtable_ops *ops = smmu_domain->pgtbl_ops; 1380 1381 if (!ops) 1382 return 0; 1383 1384 if (smmu_domain->smmu->features & ARM_SMMU_FEAT_TRANS_OPS && 1385 smmu_domain->stage == ARM_SMMU_DOMAIN_S1) 1386 return arm_smmu_iova_to_phys_hard(domain, iova); 1387 1388 return ops->iova_to_phys(ops, iova); 1389 } 1390 1391 static bool arm_smmu_capable(struct device *dev, enum iommu_cap cap) 1392 { 1393 struct arm_smmu_master_cfg *cfg = dev_iommu_priv_get(dev); 1394 1395 switch (cap) { 1396 case IOMMU_CAP_CACHE_COHERENCY: 1397 /* 1398 * It's overwhelmingly the case in practice that when the pagetable 1399 * walk interface is connected to a coherent interconnect, all the 1400 * translation interfaces are too. Furthermore if the device is 1401 * natively coherent, then its translation interface must also be. 1402 */ 1403 return cfg->smmu->features & ARM_SMMU_FEAT_COHERENT_WALK || 1404 device_get_dma_attr(dev) == DEV_DMA_COHERENT; 1405 case IOMMU_CAP_NOEXEC: 1406 case IOMMU_CAP_DEFERRED_FLUSH: 1407 return true; 1408 default: 1409 return false; 1410 } 1411 } 1412 1413 static 1414 struct arm_smmu_device *arm_smmu_get_by_fwnode(struct fwnode_handle *fwnode) 1415 { 1416 struct device *dev = bus_find_device_by_fwnode(&platform_bus_type, fwnode); 1417 1418 put_device(dev); 1419 return dev ? dev_get_drvdata(dev) : NULL; 1420 } 1421 1422 static struct iommu_device *arm_smmu_probe_device(struct device *dev) 1423 { 1424 struct arm_smmu_device *smmu = NULL; 1425 struct arm_smmu_master_cfg *cfg; 1426 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); 1427 int i, ret; 1428 1429 if (using_legacy_binding) { 1430 ret = arm_smmu_register_legacy_master(dev, &smmu); 1431 1432 /* 1433 * If dev->iommu_fwspec is initally NULL, arm_smmu_register_legacy_master() 1434 * will allocate/initialise a new one. Thus we need to update fwspec for 1435 * later use. 1436 */ 1437 fwspec = dev_iommu_fwspec_get(dev); 1438 if (ret) 1439 goto out_free; 1440 } else { 1441 smmu = arm_smmu_get_by_fwnode(fwspec->iommu_fwnode); 1442 } 1443 1444 ret = -EINVAL; 1445 for (i = 0; i < fwspec->num_ids; i++) { 1446 u16 sid = FIELD_GET(ARM_SMMU_SMR_ID, fwspec->ids[i]); 1447 u16 mask = FIELD_GET(ARM_SMMU_SMR_MASK, fwspec->ids[i]); 1448 1449 if (sid & ~smmu->streamid_mask) { 1450 dev_err(dev, "stream ID 0x%x out of range for SMMU (0x%x)\n", 1451 sid, smmu->streamid_mask); 1452 goto out_free; 1453 } 1454 if (mask & ~smmu->smr_mask_mask) { 1455 dev_err(dev, "SMR mask 0x%x out of range for SMMU (0x%x)\n", 1456 mask, smmu->smr_mask_mask); 1457 goto out_free; 1458 } 1459 } 1460 1461 ret = -ENOMEM; 1462 cfg = kzalloc(offsetof(struct arm_smmu_master_cfg, smendx[i]), 1463 GFP_KERNEL); 1464 if (!cfg) 1465 goto out_free; 1466 1467 cfg->smmu = smmu; 1468 dev_iommu_priv_set(dev, cfg); 1469 while (i--) 1470 cfg->smendx[i] = INVALID_SMENDX; 1471 1472 ret = arm_smmu_rpm_get(smmu); 1473 if (ret < 0) 1474 goto out_cfg_free; 1475 1476 ret = arm_smmu_master_alloc_smes(dev); 1477 arm_smmu_rpm_put(smmu); 1478 1479 if (ret) 1480 goto out_cfg_free; 1481 1482 device_link_add(dev, smmu->dev, 1483 DL_FLAG_PM_RUNTIME | DL_FLAG_AUTOREMOVE_SUPPLIER); 1484 1485 return &smmu->iommu; 1486 1487 out_cfg_free: 1488 kfree(cfg); 1489 out_free: 1490 return ERR_PTR(ret); 1491 } 1492 1493 static void arm_smmu_release_device(struct device *dev) 1494 { 1495 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); 1496 struct arm_smmu_master_cfg *cfg = dev_iommu_priv_get(dev); 1497 int ret; 1498 1499 ret = arm_smmu_rpm_get(cfg->smmu); 1500 if (ret < 0) 1501 return; 1502 1503 arm_smmu_master_free_smes(cfg, fwspec); 1504 1505 arm_smmu_rpm_put(cfg->smmu); 1506 1507 kfree(cfg); 1508 } 1509 1510 static void arm_smmu_probe_finalize(struct device *dev) 1511 { 1512 struct arm_smmu_master_cfg *cfg; 1513 struct arm_smmu_device *smmu; 1514 1515 cfg = dev_iommu_priv_get(dev); 1516 smmu = cfg->smmu; 1517 1518 if (smmu->impl && smmu->impl->probe_finalize) 1519 smmu->impl->probe_finalize(smmu, dev); 1520 } 1521 1522 static struct iommu_group *arm_smmu_device_group(struct device *dev) 1523 { 1524 struct arm_smmu_master_cfg *cfg = dev_iommu_priv_get(dev); 1525 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); 1526 struct arm_smmu_device *smmu = cfg->smmu; 1527 struct iommu_group *group = NULL; 1528 int i, idx; 1529 1530 mutex_lock(&smmu->stream_map_mutex); 1531 for_each_cfg_sme(cfg, fwspec, i, idx) { 1532 if (group && smmu->s2crs[idx].group && 1533 group != smmu->s2crs[idx].group) { 1534 mutex_unlock(&smmu->stream_map_mutex); 1535 return ERR_PTR(-EINVAL); 1536 } 1537 1538 group = smmu->s2crs[idx].group; 1539 } 1540 1541 if (group) { 1542 mutex_unlock(&smmu->stream_map_mutex); 1543 return iommu_group_ref_get(group); 1544 } 1545 1546 if (dev_is_pci(dev)) 1547 group = pci_device_group(dev); 1548 else if (dev_is_fsl_mc(dev)) 1549 group = fsl_mc_device_group(dev); 1550 else 1551 group = generic_device_group(dev); 1552 1553 /* Remember group for faster lookups */ 1554 if (!IS_ERR(group)) 1555 for_each_cfg_sme(cfg, fwspec, i, idx) 1556 smmu->s2crs[idx].group = group; 1557 1558 mutex_unlock(&smmu->stream_map_mutex); 1559 return group; 1560 } 1561 1562 static int arm_smmu_set_pgtable_quirks(struct iommu_domain *domain, 1563 unsigned long quirks) 1564 { 1565 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); 1566 int ret = 0; 1567 1568 mutex_lock(&smmu_domain->init_mutex); 1569 if (smmu_domain->smmu) 1570 ret = -EPERM; 1571 else 1572 smmu_domain->pgtbl_quirks = quirks; 1573 mutex_unlock(&smmu_domain->init_mutex); 1574 1575 return ret; 1576 } 1577 1578 static int arm_smmu_of_xlate(struct device *dev, 1579 const struct of_phandle_args *args) 1580 { 1581 u32 mask, fwid = 0; 1582 1583 if (args->args_count > 0) 1584 fwid |= FIELD_PREP(ARM_SMMU_SMR_ID, args->args[0]); 1585 1586 if (args->args_count > 1) 1587 fwid |= FIELD_PREP(ARM_SMMU_SMR_MASK, args->args[1]); 1588 else if (!of_property_read_u32(args->np, "stream-match-mask", &mask)) 1589 fwid |= FIELD_PREP(ARM_SMMU_SMR_MASK, mask); 1590 1591 return iommu_fwspec_add_ids(dev, &fwid, 1); 1592 } 1593 1594 static void arm_smmu_get_resv_regions(struct device *dev, 1595 struct list_head *head) 1596 { 1597 struct iommu_resv_region *region; 1598 int prot = IOMMU_WRITE | IOMMU_NOEXEC | IOMMU_MMIO; 1599 1600 region = iommu_alloc_resv_region(MSI_IOVA_BASE, MSI_IOVA_LENGTH, 1601 prot, IOMMU_RESV_SW_MSI, GFP_KERNEL); 1602 if (!region) 1603 return; 1604 1605 list_add_tail(®ion->list, head); 1606 1607 iommu_dma_get_resv_regions(dev, head); 1608 } 1609 1610 static int arm_smmu_def_domain_type(struct device *dev) 1611 { 1612 struct arm_smmu_master_cfg *cfg = dev_iommu_priv_get(dev); 1613 const struct arm_smmu_impl *impl = cfg->smmu->impl; 1614 1615 if (using_legacy_binding) 1616 return IOMMU_DOMAIN_IDENTITY; 1617 1618 if (impl && impl->def_domain_type) 1619 return impl->def_domain_type(dev); 1620 1621 return 0; 1622 } 1623 1624 static struct iommu_ops arm_smmu_ops = { 1625 .identity_domain = &arm_smmu_identity_domain, 1626 .blocked_domain = &arm_smmu_blocked_domain, 1627 .capable = arm_smmu_capable, 1628 .domain_alloc_paging = arm_smmu_domain_alloc_paging, 1629 .probe_device = arm_smmu_probe_device, 1630 .release_device = arm_smmu_release_device, 1631 .probe_finalize = arm_smmu_probe_finalize, 1632 .device_group = arm_smmu_device_group, 1633 .of_xlate = arm_smmu_of_xlate, 1634 .get_resv_regions = arm_smmu_get_resv_regions, 1635 .def_domain_type = arm_smmu_def_domain_type, 1636 .pgsize_bitmap = -1UL, /* Restricted during device attach */ 1637 .owner = THIS_MODULE, 1638 .default_domain_ops = &(const struct iommu_domain_ops) { 1639 .attach_dev = arm_smmu_attach_dev, 1640 .map_pages = arm_smmu_map_pages, 1641 .unmap_pages = arm_smmu_unmap_pages, 1642 .flush_iotlb_all = arm_smmu_flush_iotlb_all, 1643 .iotlb_sync = arm_smmu_iotlb_sync, 1644 .iova_to_phys = arm_smmu_iova_to_phys, 1645 .set_pgtable_quirks = arm_smmu_set_pgtable_quirks, 1646 .free = arm_smmu_domain_free, 1647 } 1648 }; 1649 1650 static void arm_smmu_device_reset(struct arm_smmu_device *smmu) 1651 { 1652 int i; 1653 u32 reg; 1654 1655 /* clear global FSR */ 1656 reg = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSR); 1657 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sGFSR, reg); 1658 1659 /* 1660 * Reset stream mapping groups: Initial values mark all SMRn as 1661 * invalid and all S2CRn as bypass unless overridden. 1662 */ 1663 for (i = 0; i < smmu->num_mapping_groups; ++i) 1664 arm_smmu_write_sme(smmu, i); 1665 1666 /* Make sure all context banks are disabled and clear CB_FSR */ 1667 for (i = 0; i < smmu->num_context_banks; ++i) { 1668 arm_smmu_write_context_bank(smmu, i); 1669 arm_smmu_cb_write(smmu, i, ARM_SMMU_CB_FSR, ARM_SMMU_CB_FSR_FAULT); 1670 } 1671 1672 /* Invalidate the TLB, just in case */ 1673 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_TLBIALLH, QCOM_DUMMY_VAL); 1674 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_TLBIALLNSNH, QCOM_DUMMY_VAL); 1675 1676 reg = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sCR0); 1677 1678 /* Enable fault reporting */ 1679 reg |= (ARM_SMMU_sCR0_GFRE | ARM_SMMU_sCR0_GFIE | 1680 ARM_SMMU_sCR0_GCFGFRE | ARM_SMMU_sCR0_GCFGFIE); 1681 1682 /* Disable TLB broadcasting. */ 1683 reg |= (ARM_SMMU_sCR0_VMIDPNE | ARM_SMMU_sCR0_PTM); 1684 1685 /* Enable client access, handling unmatched streams as appropriate */ 1686 reg &= ~ARM_SMMU_sCR0_CLIENTPD; 1687 if (disable_bypass) 1688 reg |= ARM_SMMU_sCR0_USFCFG; 1689 else 1690 reg &= ~ARM_SMMU_sCR0_USFCFG; 1691 1692 /* Disable forced broadcasting */ 1693 reg &= ~ARM_SMMU_sCR0_FB; 1694 1695 /* Don't upgrade barriers */ 1696 reg &= ~(ARM_SMMU_sCR0_BSU); 1697 1698 if (smmu->features & ARM_SMMU_FEAT_VMID16) 1699 reg |= ARM_SMMU_sCR0_VMID16EN; 1700 1701 if (smmu->features & ARM_SMMU_FEAT_EXIDS) 1702 reg |= ARM_SMMU_sCR0_EXIDENABLE; 1703 1704 if (smmu->impl && smmu->impl->reset) 1705 smmu->impl->reset(smmu); 1706 1707 /* Push the button */ 1708 arm_smmu_tlb_sync_global(smmu); 1709 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sCR0, reg); 1710 } 1711 1712 static int arm_smmu_id_size_to_bits(int size) 1713 { 1714 switch (size) { 1715 case 0: 1716 return 32; 1717 case 1: 1718 return 36; 1719 case 2: 1720 return 40; 1721 case 3: 1722 return 42; 1723 case 4: 1724 return 44; 1725 case 5: 1726 default: 1727 return 48; 1728 } 1729 } 1730 1731 static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu) 1732 { 1733 unsigned int size; 1734 u32 id; 1735 bool cttw_reg, cttw_fw = smmu->features & ARM_SMMU_FEAT_COHERENT_WALK; 1736 int i, ret; 1737 1738 dev_notice(smmu->dev, "probing hardware configuration...\n"); 1739 dev_notice(smmu->dev, "SMMUv%d with:\n", 1740 smmu->version == ARM_SMMU_V2 ? 2 : 1); 1741 1742 /* ID0 */ 1743 id = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_ID0); 1744 1745 /* Restrict available stages based on module parameter */ 1746 if (force_stage == 1) 1747 id &= ~(ARM_SMMU_ID0_S2TS | ARM_SMMU_ID0_NTS); 1748 else if (force_stage == 2) 1749 id &= ~(ARM_SMMU_ID0_S1TS | ARM_SMMU_ID0_NTS); 1750 1751 if (id & ARM_SMMU_ID0_S1TS) { 1752 smmu->features |= ARM_SMMU_FEAT_TRANS_S1; 1753 dev_notice(smmu->dev, "\tstage 1 translation\n"); 1754 } 1755 1756 if (id & ARM_SMMU_ID0_S2TS) { 1757 smmu->features |= ARM_SMMU_FEAT_TRANS_S2; 1758 dev_notice(smmu->dev, "\tstage 2 translation\n"); 1759 } 1760 1761 if (id & ARM_SMMU_ID0_NTS) { 1762 smmu->features |= ARM_SMMU_FEAT_TRANS_NESTED; 1763 dev_notice(smmu->dev, "\tnested translation\n"); 1764 } 1765 1766 if (!(smmu->features & 1767 (ARM_SMMU_FEAT_TRANS_S1 | ARM_SMMU_FEAT_TRANS_S2))) { 1768 dev_err(smmu->dev, "\tno translation support!\n"); 1769 return -ENODEV; 1770 } 1771 1772 if ((id & ARM_SMMU_ID0_S1TS) && 1773 ((smmu->version < ARM_SMMU_V2) || !(id & ARM_SMMU_ID0_ATOSNS))) { 1774 smmu->features |= ARM_SMMU_FEAT_TRANS_OPS; 1775 dev_notice(smmu->dev, "\taddress translation ops\n"); 1776 } 1777 1778 /* 1779 * In order for DMA API calls to work properly, we must defer to what 1780 * the FW says about coherency, regardless of what the hardware claims. 1781 * Fortunately, this also opens up a workaround for systems where the 1782 * ID register value has ended up configured incorrectly. 1783 */ 1784 cttw_reg = !!(id & ARM_SMMU_ID0_CTTW); 1785 if (cttw_fw || cttw_reg) 1786 dev_notice(smmu->dev, "\t%scoherent table walk\n", 1787 cttw_fw ? "" : "non-"); 1788 if (cttw_fw != cttw_reg) 1789 dev_notice(smmu->dev, 1790 "\t(IDR0.CTTW overridden by FW configuration)\n"); 1791 1792 /* Max. number of entries we have for stream matching/indexing */ 1793 if (smmu->version == ARM_SMMU_V2 && id & ARM_SMMU_ID0_EXIDS) { 1794 smmu->features |= ARM_SMMU_FEAT_EXIDS; 1795 size = 1 << 16; 1796 } else { 1797 size = 1 << FIELD_GET(ARM_SMMU_ID0_NUMSIDB, id); 1798 } 1799 smmu->streamid_mask = size - 1; 1800 if (id & ARM_SMMU_ID0_SMS) { 1801 smmu->features |= ARM_SMMU_FEAT_STREAM_MATCH; 1802 size = FIELD_GET(ARM_SMMU_ID0_NUMSMRG, id); 1803 if (size == 0) { 1804 dev_err(smmu->dev, 1805 "stream-matching supported, but no SMRs present!\n"); 1806 return -ENODEV; 1807 } 1808 1809 /* Zero-initialised to mark as invalid */ 1810 smmu->smrs = devm_kcalloc(smmu->dev, size, sizeof(*smmu->smrs), 1811 GFP_KERNEL); 1812 if (!smmu->smrs) 1813 return -ENOMEM; 1814 1815 dev_notice(smmu->dev, 1816 "\tstream matching with %u register groups", size); 1817 } 1818 /* s2cr->type == 0 means translation, so initialise explicitly */ 1819 smmu->s2crs = devm_kmalloc_array(smmu->dev, size, sizeof(*smmu->s2crs), 1820 GFP_KERNEL); 1821 if (!smmu->s2crs) 1822 return -ENOMEM; 1823 for (i = 0; i < size; i++) 1824 smmu->s2crs[i] = s2cr_init_val; 1825 1826 smmu->num_mapping_groups = size; 1827 mutex_init(&smmu->stream_map_mutex); 1828 spin_lock_init(&smmu->global_sync_lock); 1829 1830 if (smmu->version < ARM_SMMU_V2 || 1831 !(id & ARM_SMMU_ID0_PTFS_NO_AARCH32)) { 1832 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH32_L; 1833 if (!(id & ARM_SMMU_ID0_PTFS_NO_AARCH32S)) 1834 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH32_S; 1835 } 1836 1837 /* ID1 */ 1838 id = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_ID1); 1839 smmu->pgshift = (id & ARM_SMMU_ID1_PAGESIZE) ? 16 : 12; 1840 1841 /* Check for size mismatch of SMMU address space from mapped region */ 1842 size = 1 << (FIELD_GET(ARM_SMMU_ID1_NUMPAGENDXB, id) + 1); 1843 if (smmu->numpage != 2 * size << smmu->pgshift) 1844 dev_warn(smmu->dev, 1845 "SMMU address space size (0x%x) differs from mapped region size (0x%x)!\n", 1846 2 * size << smmu->pgshift, smmu->numpage); 1847 /* Now properly encode NUMPAGE to subsequently derive SMMU_CB_BASE */ 1848 smmu->numpage = size; 1849 1850 smmu->num_s2_context_banks = FIELD_GET(ARM_SMMU_ID1_NUMS2CB, id); 1851 smmu->num_context_banks = FIELD_GET(ARM_SMMU_ID1_NUMCB, id); 1852 if (smmu->num_s2_context_banks > smmu->num_context_banks) { 1853 dev_err(smmu->dev, "impossible number of S2 context banks!\n"); 1854 return -ENODEV; 1855 } 1856 dev_notice(smmu->dev, "\t%u context banks (%u stage-2 only)\n", 1857 smmu->num_context_banks, smmu->num_s2_context_banks); 1858 smmu->cbs = devm_kcalloc(smmu->dev, smmu->num_context_banks, 1859 sizeof(*smmu->cbs), GFP_KERNEL); 1860 if (!smmu->cbs) 1861 return -ENOMEM; 1862 1863 /* ID2 */ 1864 id = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_ID2); 1865 size = arm_smmu_id_size_to_bits(FIELD_GET(ARM_SMMU_ID2_IAS, id)); 1866 smmu->ipa_size = size; 1867 1868 /* The output mask is also applied for bypass */ 1869 size = arm_smmu_id_size_to_bits(FIELD_GET(ARM_SMMU_ID2_OAS, id)); 1870 smmu->pa_size = size; 1871 1872 if (id & ARM_SMMU_ID2_VMID16) 1873 smmu->features |= ARM_SMMU_FEAT_VMID16; 1874 1875 /* 1876 * What the page table walker can address actually depends on which 1877 * descriptor format is in use, but since a) we don't know that yet, 1878 * and b) it can vary per context bank, this will have to do... 1879 */ 1880 if (dma_set_mask_and_coherent(smmu->dev, DMA_BIT_MASK(size))) 1881 dev_warn(smmu->dev, 1882 "failed to set DMA mask for table walker\n"); 1883 1884 if (smmu->version < ARM_SMMU_V2) { 1885 smmu->va_size = smmu->ipa_size; 1886 if (smmu->version == ARM_SMMU_V1_64K) 1887 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_64K; 1888 } else { 1889 size = FIELD_GET(ARM_SMMU_ID2_UBS, id); 1890 smmu->va_size = arm_smmu_id_size_to_bits(size); 1891 if (id & ARM_SMMU_ID2_PTFS_4K) 1892 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_4K; 1893 if (id & ARM_SMMU_ID2_PTFS_16K) 1894 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_16K; 1895 if (id & ARM_SMMU_ID2_PTFS_64K) 1896 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_64K; 1897 } 1898 1899 if (smmu->impl && smmu->impl->cfg_probe) { 1900 ret = smmu->impl->cfg_probe(smmu); 1901 if (ret) 1902 return ret; 1903 } 1904 1905 /* Now we've corralled the various formats, what'll it do? */ 1906 if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_S) 1907 smmu->pgsize_bitmap |= SZ_4K | SZ_64K | SZ_1M | SZ_16M; 1908 if (smmu->features & 1909 (ARM_SMMU_FEAT_FMT_AARCH32_L | ARM_SMMU_FEAT_FMT_AARCH64_4K)) 1910 smmu->pgsize_bitmap |= SZ_4K | SZ_2M | SZ_1G; 1911 if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH64_16K) 1912 smmu->pgsize_bitmap |= SZ_16K | SZ_32M; 1913 if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH64_64K) 1914 smmu->pgsize_bitmap |= SZ_64K | SZ_512M; 1915 1916 if (arm_smmu_ops.pgsize_bitmap == -1UL) 1917 arm_smmu_ops.pgsize_bitmap = smmu->pgsize_bitmap; 1918 else 1919 arm_smmu_ops.pgsize_bitmap |= smmu->pgsize_bitmap; 1920 dev_notice(smmu->dev, "\tSupported page sizes: 0x%08lx\n", 1921 smmu->pgsize_bitmap); 1922 1923 1924 if (smmu->features & ARM_SMMU_FEAT_TRANS_S1) 1925 dev_notice(smmu->dev, "\tStage-1: %lu-bit VA -> %lu-bit IPA\n", 1926 smmu->va_size, smmu->ipa_size); 1927 1928 if (smmu->features & ARM_SMMU_FEAT_TRANS_S2) 1929 dev_notice(smmu->dev, "\tStage-2: %lu-bit IPA -> %lu-bit PA\n", 1930 smmu->ipa_size, smmu->pa_size); 1931 1932 return 0; 1933 } 1934 1935 struct arm_smmu_match_data { 1936 enum arm_smmu_arch_version version; 1937 enum arm_smmu_implementation model; 1938 }; 1939 1940 #define ARM_SMMU_MATCH_DATA(name, ver, imp) \ 1941 static const struct arm_smmu_match_data name = { .version = ver, .model = imp } 1942 1943 ARM_SMMU_MATCH_DATA(smmu_generic_v1, ARM_SMMU_V1, GENERIC_SMMU); 1944 ARM_SMMU_MATCH_DATA(smmu_generic_v2, ARM_SMMU_V2, GENERIC_SMMU); 1945 ARM_SMMU_MATCH_DATA(arm_mmu401, ARM_SMMU_V1_64K, GENERIC_SMMU); 1946 ARM_SMMU_MATCH_DATA(arm_mmu500, ARM_SMMU_V2, ARM_MMU500); 1947 ARM_SMMU_MATCH_DATA(cavium_smmuv2, ARM_SMMU_V2, CAVIUM_SMMUV2); 1948 ARM_SMMU_MATCH_DATA(qcom_smmuv2, ARM_SMMU_V2, QCOM_SMMUV2); 1949 1950 static const struct of_device_id arm_smmu_of_match[] = { 1951 { .compatible = "arm,smmu-v1", .data = &smmu_generic_v1 }, 1952 { .compatible = "arm,smmu-v2", .data = &smmu_generic_v2 }, 1953 { .compatible = "arm,mmu-400", .data = &smmu_generic_v1 }, 1954 { .compatible = "arm,mmu-401", .data = &arm_mmu401 }, 1955 { .compatible = "arm,mmu-500", .data = &arm_mmu500 }, 1956 { .compatible = "cavium,smmu-v2", .data = &cavium_smmuv2 }, 1957 { .compatible = "nvidia,smmu-500", .data = &arm_mmu500 }, 1958 { .compatible = "qcom,smmu-v2", .data = &qcom_smmuv2 }, 1959 { }, 1960 }; 1961 MODULE_DEVICE_TABLE(of, arm_smmu_of_match); 1962 1963 #ifdef CONFIG_ACPI 1964 static int acpi_smmu_get_data(u32 model, struct arm_smmu_device *smmu) 1965 { 1966 int ret = 0; 1967 1968 switch (model) { 1969 case ACPI_IORT_SMMU_V1: 1970 case ACPI_IORT_SMMU_CORELINK_MMU400: 1971 smmu->version = ARM_SMMU_V1; 1972 smmu->model = GENERIC_SMMU; 1973 break; 1974 case ACPI_IORT_SMMU_CORELINK_MMU401: 1975 smmu->version = ARM_SMMU_V1_64K; 1976 smmu->model = GENERIC_SMMU; 1977 break; 1978 case ACPI_IORT_SMMU_V2: 1979 smmu->version = ARM_SMMU_V2; 1980 smmu->model = GENERIC_SMMU; 1981 break; 1982 case ACPI_IORT_SMMU_CORELINK_MMU500: 1983 smmu->version = ARM_SMMU_V2; 1984 smmu->model = ARM_MMU500; 1985 break; 1986 case ACPI_IORT_SMMU_CAVIUM_THUNDERX: 1987 smmu->version = ARM_SMMU_V2; 1988 smmu->model = CAVIUM_SMMUV2; 1989 break; 1990 default: 1991 ret = -ENODEV; 1992 } 1993 1994 return ret; 1995 } 1996 1997 static int arm_smmu_device_acpi_probe(struct arm_smmu_device *smmu, 1998 u32 *global_irqs, u32 *pmu_irqs) 1999 { 2000 struct device *dev = smmu->dev; 2001 struct acpi_iort_node *node = 2002 *(struct acpi_iort_node **)dev_get_platdata(dev); 2003 struct acpi_iort_smmu *iort_smmu; 2004 int ret; 2005 2006 /* Retrieve SMMU1/2 specific data */ 2007 iort_smmu = (struct acpi_iort_smmu *)node->node_data; 2008 2009 ret = acpi_smmu_get_data(iort_smmu->model, smmu); 2010 if (ret < 0) 2011 return ret; 2012 2013 /* Ignore the configuration access interrupt */ 2014 *global_irqs = 1; 2015 *pmu_irqs = 0; 2016 2017 if (iort_smmu->flags & ACPI_IORT_SMMU_COHERENT_WALK) 2018 smmu->features |= ARM_SMMU_FEAT_COHERENT_WALK; 2019 2020 return 0; 2021 } 2022 #else 2023 static inline int arm_smmu_device_acpi_probe(struct arm_smmu_device *smmu, 2024 u32 *global_irqs, u32 *pmu_irqs) 2025 { 2026 return -ENODEV; 2027 } 2028 #endif 2029 2030 static int arm_smmu_device_dt_probe(struct arm_smmu_device *smmu, 2031 u32 *global_irqs, u32 *pmu_irqs) 2032 { 2033 const struct arm_smmu_match_data *data; 2034 struct device *dev = smmu->dev; 2035 bool legacy_binding; 2036 2037 if (of_property_read_u32(dev->of_node, "#global-interrupts", global_irqs)) 2038 return dev_err_probe(dev, -ENODEV, 2039 "missing #global-interrupts property\n"); 2040 *pmu_irqs = 0; 2041 2042 data = of_device_get_match_data(dev); 2043 smmu->version = data->version; 2044 smmu->model = data->model; 2045 2046 legacy_binding = of_find_property(dev->of_node, "mmu-masters", NULL); 2047 if (legacy_binding && !using_generic_binding) { 2048 if (!using_legacy_binding) { 2049 pr_notice("deprecated \"mmu-masters\" DT property in use; %s support unavailable\n", 2050 IS_ENABLED(CONFIG_ARM_SMMU_LEGACY_DT_BINDINGS) ? "DMA API" : "SMMU"); 2051 } 2052 using_legacy_binding = true; 2053 } else if (!legacy_binding && !using_legacy_binding) { 2054 using_generic_binding = true; 2055 } else { 2056 dev_err(dev, "not probing due to mismatched DT properties\n"); 2057 return -ENODEV; 2058 } 2059 2060 if (of_dma_is_coherent(dev->of_node)) 2061 smmu->features |= ARM_SMMU_FEAT_COHERENT_WALK; 2062 2063 return 0; 2064 } 2065 2066 static void arm_smmu_rmr_install_bypass_smr(struct arm_smmu_device *smmu) 2067 { 2068 struct list_head rmr_list; 2069 struct iommu_resv_region *e; 2070 int idx, cnt = 0; 2071 u32 reg; 2072 2073 INIT_LIST_HEAD(&rmr_list); 2074 iort_get_rmr_sids(dev_fwnode(smmu->dev), &rmr_list); 2075 2076 /* 2077 * Rather than trying to look at existing mappings that 2078 * are setup by the firmware and then invalidate the ones 2079 * that do no have matching RMR entries, just disable the 2080 * SMMU until it gets enabled again in the reset routine. 2081 */ 2082 reg = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sCR0); 2083 reg |= ARM_SMMU_sCR0_CLIENTPD; 2084 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sCR0, reg); 2085 2086 list_for_each_entry(e, &rmr_list, list) { 2087 struct iommu_iort_rmr_data *rmr; 2088 int i; 2089 2090 rmr = container_of(e, struct iommu_iort_rmr_data, rr); 2091 for (i = 0; i < rmr->num_sids; i++) { 2092 idx = arm_smmu_find_sme(smmu, rmr->sids[i], ~0); 2093 if (idx < 0) 2094 continue; 2095 2096 if (smmu->s2crs[idx].count == 0) { 2097 smmu->smrs[idx].id = rmr->sids[i]; 2098 smmu->smrs[idx].mask = 0; 2099 smmu->smrs[idx].valid = true; 2100 } 2101 smmu->s2crs[idx].count++; 2102 smmu->s2crs[idx].type = S2CR_TYPE_BYPASS; 2103 smmu->s2crs[idx].privcfg = S2CR_PRIVCFG_DEFAULT; 2104 2105 cnt++; 2106 } 2107 } 2108 2109 dev_notice(smmu->dev, "\tpreserved %d boot mapping%s\n", cnt, 2110 str_plural(cnt)); 2111 iort_put_rmr_sids(dev_fwnode(smmu->dev), &rmr_list); 2112 } 2113 2114 static int arm_smmu_device_probe(struct platform_device *pdev) 2115 { 2116 struct resource *res; 2117 struct arm_smmu_device *smmu; 2118 struct device *dev = &pdev->dev; 2119 int num_irqs, i, err; 2120 u32 global_irqs, pmu_irqs; 2121 irqreturn_t (*global_fault)(int irq, void *dev); 2122 2123 smmu = devm_kzalloc(dev, sizeof(*smmu), GFP_KERNEL); 2124 if (!smmu) { 2125 dev_err(dev, "failed to allocate arm_smmu_device\n"); 2126 return -ENOMEM; 2127 } 2128 smmu->dev = dev; 2129 2130 if (dev->of_node) 2131 err = arm_smmu_device_dt_probe(smmu, &global_irqs, &pmu_irqs); 2132 else 2133 err = arm_smmu_device_acpi_probe(smmu, &global_irqs, &pmu_irqs); 2134 if (err) 2135 return err; 2136 2137 smmu->base = devm_platform_get_and_ioremap_resource(pdev, 0, &res); 2138 if (IS_ERR(smmu->base)) 2139 return PTR_ERR(smmu->base); 2140 smmu->ioaddr = res->start; 2141 2142 /* 2143 * The resource size should effectively match the value of SMMU_TOP; 2144 * stash that temporarily until we know PAGESIZE to validate it with. 2145 */ 2146 smmu->numpage = resource_size(res); 2147 2148 smmu = arm_smmu_impl_init(smmu); 2149 if (IS_ERR(smmu)) 2150 return PTR_ERR(smmu); 2151 2152 num_irqs = platform_irq_count(pdev); 2153 2154 smmu->num_context_irqs = num_irqs - global_irqs - pmu_irqs; 2155 if (smmu->num_context_irqs <= 0) 2156 return dev_err_probe(dev, -ENODEV, 2157 "found %d interrupts but expected at least %d\n", 2158 num_irqs, global_irqs + pmu_irqs + 1); 2159 2160 smmu->irqs = devm_kcalloc(dev, smmu->num_context_irqs, 2161 sizeof(*smmu->irqs), GFP_KERNEL); 2162 if (!smmu->irqs) 2163 return dev_err_probe(dev, -ENOMEM, "failed to allocate %d irqs\n", 2164 smmu->num_context_irqs); 2165 2166 for (i = 0; i < smmu->num_context_irqs; i++) { 2167 int irq = platform_get_irq(pdev, global_irqs + pmu_irqs + i); 2168 2169 if (irq < 0) 2170 return irq; 2171 smmu->irqs[i] = irq; 2172 } 2173 2174 err = devm_clk_bulk_get_all(dev, &smmu->clks); 2175 if (err < 0) { 2176 dev_err(dev, "failed to get clocks %d\n", err); 2177 return err; 2178 } 2179 smmu->num_clks = err; 2180 2181 err = clk_bulk_prepare_enable(smmu->num_clks, smmu->clks); 2182 if (err) 2183 return err; 2184 2185 err = arm_smmu_device_cfg_probe(smmu); 2186 if (err) 2187 return err; 2188 2189 if (smmu->version == ARM_SMMU_V2) { 2190 if (smmu->num_context_banks > smmu->num_context_irqs) { 2191 dev_err(dev, 2192 "found only %d context irq(s) but %d required\n", 2193 smmu->num_context_irqs, smmu->num_context_banks); 2194 return -ENODEV; 2195 } 2196 2197 /* Ignore superfluous interrupts */ 2198 smmu->num_context_irqs = smmu->num_context_banks; 2199 } 2200 2201 if (smmu->impl && smmu->impl->global_fault) 2202 global_fault = smmu->impl->global_fault; 2203 else 2204 global_fault = arm_smmu_global_fault; 2205 2206 for (i = 0; i < global_irqs; i++) { 2207 int irq = platform_get_irq(pdev, i); 2208 2209 if (irq < 0) 2210 return irq; 2211 2212 err = devm_request_irq(dev, irq, global_fault, IRQF_SHARED, 2213 "arm-smmu global fault", smmu); 2214 if (err) 2215 return dev_err_probe(dev, err, 2216 "failed to request global IRQ %d (%u)\n", 2217 i, irq); 2218 } 2219 2220 platform_set_drvdata(pdev, smmu); 2221 2222 /* Check for RMRs and install bypass SMRs if any */ 2223 arm_smmu_rmr_install_bypass_smr(smmu); 2224 2225 arm_smmu_device_reset(smmu); 2226 arm_smmu_test_smr_masks(smmu); 2227 2228 err = iommu_device_sysfs_add(&smmu->iommu, smmu->dev, NULL, 2229 "smmu.%pa", &smmu->ioaddr); 2230 if (err) 2231 return dev_err_probe(dev, err, "Failed to register iommu in sysfs\n"); 2232 2233 err = iommu_device_register(&smmu->iommu, &arm_smmu_ops, 2234 using_legacy_binding ? NULL : dev); 2235 if (err) { 2236 iommu_device_sysfs_remove(&smmu->iommu); 2237 return dev_err_probe(dev, err, "Failed to register iommu\n"); 2238 } 2239 2240 /* 2241 * We want to avoid touching dev->power.lock in fastpaths unless 2242 * it's really going to do something useful - pm_runtime_enabled() 2243 * can serve as an ideal proxy for that decision. So, conditionally 2244 * enable pm_runtime. 2245 */ 2246 if (dev->pm_domain) { 2247 pm_runtime_set_active(dev); 2248 pm_runtime_enable(dev); 2249 arm_smmu_rpm_use_autosuspend(smmu); 2250 } 2251 2252 return 0; 2253 } 2254 2255 static void arm_smmu_device_shutdown(struct platform_device *pdev) 2256 { 2257 struct arm_smmu_device *smmu = platform_get_drvdata(pdev); 2258 2259 if (!bitmap_empty(smmu->context_map, ARM_SMMU_MAX_CBS)) 2260 dev_notice(&pdev->dev, "disabling translation\n"); 2261 2262 arm_smmu_rpm_get(smmu); 2263 /* Turn the thing off */ 2264 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sCR0, ARM_SMMU_sCR0_CLIENTPD); 2265 arm_smmu_rpm_put(smmu); 2266 2267 if (pm_runtime_enabled(smmu->dev)) 2268 pm_runtime_force_suspend(smmu->dev); 2269 else 2270 clk_bulk_disable(smmu->num_clks, smmu->clks); 2271 2272 clk_bulk_unprepare(smmu->num_clks, smmu->clks); 2273 } 2274 2275 static void arm_smmu_device_remove(struct platform_device *pdev) 2276 { 2277 struct arm_smmu_device *smmu = platform_get_drvdata(pdev); 2278 2279 iommu_device_unregister(&smmu->iommu); 2280 iommu_device_sysfs_remove(&smmu->iommu); 2281 2282 arm_smmu_device_shutdown(pdev); 2283 } 2284 2285 static int __maybe_unused arm_smmu_runtime_resume(struct device *dev) 2286 { 2287 struct arm_smmu_device *smmu = dev_get_drvdata(dev); 2288 int ret; 2289 2290 ret = clk_bulk_enable(smmu->num_clks, smmu->clks); 2291 if (ret) 2292 return ret; 2293 2294 arm_smmu_device_reset(smmu); 2295 2296 return 0; 2297 } 2298 2299 static int __maybe_unused arm_smmu_runtime_suspend(struct device *dev) 2300 { 2301 struct arm_smmu_device *smmu = dev_get_drvdata(dev); 2302 2303 clk_bulk_disable(smmu->num_clks, smmu->clks); 2304 2305 return 0; 2306 } 2307 2308 static int __maybe_unused arm_smmu_pm_resume(struct device *dev) 2309 { 2310 int ret; 2311 struct arm_smmu_device *smmu = dev_get_drvdata(dev); 2312 2313 ret = clk_bulk_prepare(smmu->num_clks, smmu->clks); 2314 if (ret) 2315 return ret; 2316 2317 if (pm_runtime_suspended(dev)) 2318 return 0; 2319 2320 ret = arm_smmu_runtime_resume(dev); 2321 if (ret) 2322 clk_bulk_unprepare(smmu->num_clks, smmu->clks); 2323 2324 return ret; 2325 } 2326 2327 static int __maybe_unused arm_smmu_pm_suspend(struct device *dev) 2328 { 2329 int ret = 0; 2330 struct arm_smmu_device *smmu = dev_get_drvdata(dev); 2331 2332 if (pm_runtime_suspended(dev)) 2333 goto clk_unprepare; 2334 2335 ret = arm_smmu_runtime_suspend(dev); 2336 if (ret) 2337 return ret; 2338 2339 clk_unprepare: 2340 clk_bulk_unprepare(smmu->num_clks, smmu->clks); 2341 return ret; 2342 } 2343 2344 static const struct dev_pm_ops arm_smmu_pm_ops = { 2345 SET_SYSTEM_SLEEP_PM_OPS(arm_smmu_pm_suspend, arm_smmu_pm_resume) 2346 SET_RUNTIME_PM_OPS(arm_smmu_runtime_suspend, 2347 arm_smmu_runtime_resume, NULL) 2348 }; 2349 2350 static struct platform_driver arm_smmu_driver = { 2351 .driver = { 2352 .name = "arm-smmu", 2353 .of_match_table = arm_smmu_of_match, 2354 .pm = &arm_smmu_pm_ops, 2355 .suppress_bind_attrs = true, 2356 }, 2357 .probe = arm_smmu_device_probe, 2358 .remove = arm_smmu_device_remove, 2359 .shutdown = arm_smmu_device_shutdown, 2360 }; 2361 module_platform_driver(arm_smmu_driver); 2362 2363 MODULE_DESCRIPTION("IOMMU API for ARM architected SMMU implementations"); 2364 MODULE_AUTHOR("Will Deacon <will@kernel.org>"); 2365 MODULE_ALIAS("platform:arm-smmu"); 2366 MODULE_LICENSE("GPL v2"); 2367