1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * IOMMU API for ARM architected SMMU implementations. 4 * 5 * Copyright (C) 2013 ARM Limited 6 * 7 * Author: Will Deacon <will.deacon@arm.com> 8 * 9 * This driver currently supports: 10 * - SMMUv1 and v2 implementations 11 * - Stream-matching and stream-indexing 12 * - v7/v8 long-descriptor format 13 * - Non-secure access to the SMMU 14 * - Context fault reporting 15 * - Extended Stream ID (16 bit) 16 */ 17 18 #define pr_fmt(fmt) "arm-smmu: " fmt 19 20 #include <linux/acpi.h> 21 #include <linux/acpi_iort.h> 22 #include <linux/bitfield.h> 23 #include <linux/delay.h> 24 #include <linux/dma-mapping.h> 25 #include <linux/err.h> 26 #include <linux/interrupt.h> 27 #include <linux/io.h> 28 #include <linux/iopoll.h> 29 #include <linux/module.h> 30 #include <linux/of.h> 31 #include <linux/of_address.h> 32 #include <linux/pci.h> 33 #include <linux/platform_device.h> 34 #include <linux/pm_runtime.h> 35 #include <linux/ratelimit.h> 36 #include <linux/slab.h> 37 #include <linux/string_choices.h> 38 39 #include <linux/fsl/mc.h> 40 41 #include "arm-smmu.h" 42 #include "../../dma-iommu.h" 43 44 /* 45 * Apparently, some Qualcomm arm64 platforms which appear to expose their SMMU 46 * global register space are still, in fact, using a hypervisor to mediate it 47 * by trapping and emulating register accesses. Sadly, some deployed versions 48 * of said trapping code have bugs wherein they go horribly wrong for stores 49 * using r31 (i.e. XZR/WZR) as the source register. 50 */ 51 #define QCOM_DUMMY_VAL -1 52 53 #define MSI_IOVA_BASE 0x8000000 54 #define MSI_IOVA_LENGTH 0x100000 55 56 static int force_stage; 57 module_param(force_stage, int, S_IRUGO); 58 MODULE_PARM_DESC(force_stage, 59 "Force SMMU mappings to be installed at a particular stage of translation. A value of '1' or '2' forces the corresponding stage. All other values are ignored (i.e. no stage is forced). Note that selecting a specific stage will disable support for nested translation."); 60 static bool disable_bypass = 61 IS_ENABLED(CONFIG_ARM_SMMU_DISABLE_BYPASS_BY_DEFAULT); 62 module_param(disable_bypass, bool, S_IRUGO); 63 MODULE_PARM_DESC(disable_bypass, 64 "Disable bypass streams such that incoming transactions from devices that are not attached to an iommu domain will report an abort back to the device and will not be allowed to pass through the SMMU."); 65 66 #define s2cr_init_val (struct arm_smmu_s2cr){ \ 67 .type = disable_bypass ? S2CR_TYPE_FAULT : S2CR_TYPE_BYPASS, \ 68 } 69 70 static bool using_legacy_binding, using_generic_binding; 71 72 static inline int arm_smmu_rpm_get(struct arm_smmu_device *smmu) 73 { 74 if (pm_runtime_enabled(smmu->dev)) 75 return pm_runtime_resume_and_get(smmu->dev); 76 77 return 0; 78 } 79 80 static inline void arm_smmu_rpm_put(struct arm_smmu_device *smmu) 81 { 82 if (pm_runtime_enabled(smmu->dev)) { 83 pm_runtime_mark_last_busy(smmu->dev); 84 __pm_runtime_put_autosuspend(smmu->dev); 85 86 } 87 } 88 89 static void arm_smmu_rpm_use_autosuspend(struct arm_smmu_device *smmu) 90 { 91 /* 92 * Setup an autosuspend delay to avoid bouncing runpm state. 93 * Otherwise, if a driver for a suspended consumer device 94 * unmaps buffers, it will runpm resume/suspend for each one. 95 * 96 * For example, when used by a GPU device, when an application 97 * or game exits, it can trigger unmapping 100s or 1000s of 98 * buffers. With a runpm cycle for each buffer, that adds up 99 * to 5-10sec worth of reprogramming the context bank, while 100 * the system appears to be locked up to the user. 101 */ 102 pm_runtime_set_autosuspend_delay(smmu->dev, 20); 103 pm_runtime_use_autosuspend(smmu->dev); 104 } 105 106 static struct arm_smmu_domain *to_smmu_domain(struct iommu_domain *dom) 107 { 108 return container_of(dom, struct arm_smmu_domain, domain); 109 } 110 111 static struct platform_driver arm_smmu_driver; 112 static struct iommu_ops arm_smmu_ops; 113 114 #ifdef CONFIG_ARM_SMMU_LEGACY_DT_BINDINGS 115 static struct device_node *dev_get_dev_node(struct device *dev) 116 { 117 if (dev_is_pci(dev)) { 118 struct pci_bus *bus = to_pci_dev(dev)->bus; 119 120 while (!pci_is_root_bus(bus)) 121 bus = bus->parent; 122 return of_node_get(bus->bridge->parent->of_node); 123 } 124 125 return of_node_get(dev->of_node); 126 } 127 128 static int __arm_smmu_get_pci_sid(struct pci_dev *pdev, u16 alias, void *data) 129 { 130 *((__be32 *)data) = cpu_to_be32(alias); 131 return 0; /* Continue walking */ 132 } 133 134 static int __find_legacy_master_phandle(struct device *dev, void *data) 135 { 136 struct of_phandle_iterator *it = *(void **)data; 137 struct device_node *np = it->node; 138 int err; 139 140 of_for_each_phandle(it, err, dev->of_node, "mmu-masters", 141 "#stream-id-cells", -1) 142 if (it->node == np) { 143 *(void **)data = dev; 144 return 1; 145 } 146 it->node = np; 147 return err == -ENOENT ? 0 : err; 148 } 149 150 static int arm_smmu_register_legacy_master(struct device *dev, 151 struct arm_smmu_device **smmu) 152 { 153 struct device *smmu_dev; 154 struct device_node *np; 155 struct of_phandle_iterator it; 156 void *data = ⁢ 157 u32 *sids; 158 __be32 pci_sid; 159 int err; 160 161 np = dev_get_dev_node(dev); 162 if (!np || !of_property_present(np, "#stream-id-cells")) { 163 of_node_put(np); 164 return -ENODEV; 165 } 166 167 it.node = np; 168 err = driver_for_each_device(&arm_smmu_driver.driver, NULL, &data, 169 __find_legacy_master_phandle); 170 smmu_dev = data; 171 of_node_put(np); 172 if (err == 0) 173 return -ENODEV; 174 if (err < 0) 175 return err; 176 177 if (dev_is_pci(dev)) { 178 /* "mmu-masters" assumes Stream ID == Requester ID */ 179 pci_for_each_dma_alias(to_pci_dev(dev), __arm_smmu_get_pci_sid, 180 &pci_sid); 181 it.cur = &pci_sid; 182 it.cur_count = 1; 183 } 184 185 err = iommu_fwspec_init(dev, NULL); 186 if (err) 187 return err; 188 189 sids = kcalloc(it.cur_count, sizeof(*sids), GFP_KERNEL); 190 if (!sids) 191 return -ENOMEM; 192 193 *smmu = dev_get_drvdata(smmu_dev); 194 of_phandle_iterator_args(&it, sids, it.cur_count); 195 err = iommu_fwspec_add_ids(dev, sids, it.cur_count); 196 kfree(sids); 197 return err; 198 } 199 #else 200 static int arm_smmu_register_legacy_master(struct device *dev, 201 struct arm_smmu_device **smmu) 202 { 203 return -ENODEV; 204 } 205 #endif /* CONFIG_ARM_SMMU_LEGACY_DT_BINDINGS */ 206 207 static void __arm_smmu_free_bitmap(unsigned long *map, int idx) 208 { 209 clear_bit(idx, map); 210 } 211 212 /* Wait for any pending TLB invalidations to complete */ 213 static void __arm_smmu_tlb_sync(struct arm_smmu_device *smmu, int page, 214 int sync, int status) 215 { 216 unsigned int spin_cnt, delay; 217 u32 reg; 218 219 if (smmu->impl && unlikely(smmu->impl->tlb_sync)) 220 return smmu->impl->tlb_sync(smmu, page, sync, status); 221 222 arm_smmu_writel(smmu, page, sync, QCOM_DUMMY_VAL); 223 for (delay = 1; delay < TLB_LOOP_TIMEOUT; delay *= 2) { 224 for (spin_cnt = TLB_SPIN_COUNT; spin_cnt > 0; spin_cnt--) { 225 reg = arm_smmu_readl(smmu, page, status); 226 if (!(reg & ARM_SMMU_sTLBGSTATUS_GSACTIVE)) 227 return; 228 cpu_relax(); 229 } 230 udelay(delay); 231 } 232 dev_err_ratelimited(smmu->dev, 233 "TLB sync timed out -- SMMU may be deadlocked\n"); 234 } 235 236 static void arm_smmu_tlb_sync_global(struct arm_smmu_device *smmu) 237 { 238 unsigned long flags; 239 240 spin_lock_irqsave(&smmu->global_sync_lock, flags); 241 __arm_smmu_tlb_sync(smmu, ARM_SMMU_GR0, ARM_SMMU_GR0_sTLBGSYNC, 242 ARM_SMMU_GR0_sTLBGSTATUS); 243 spin_unlock_irqrestore(&smmu->global_sync_lock, flags); 244 } 245 246 static void arm_smmu_tlb_sync_context(struct arm_smmu_domain *smmu_domain) 247 { 248 struct arm_smmu_device *smmu = smmu_domain->smmu; 249 unsigned long flags; 250 251 spin_lock_irqsave(&smmu_domain->cb_lock, flags); 252 __arm_smmu_tlb_sync(smmu, ARM_SMMU_CB(smmu, smmu_domain->cfg.cbndx), 253 ARM_SMMU_CB_TLBSYNC, ARM_SMMU_CB_TLBSTATUS); 254 spin_unlock_irqrestore(&smmu_domain->cb_lock, flags); 255 } 256 257 static void arm_smmu_tlb_inv_context_s1(void *cookie) 258 { 259 struct arm_smmu_domain *smmu_domain = cookie; 260 /* 261 * The TLBI write may be relaxed, so ensure that PTEs cleared by the 262 * current CPU are visible beforehand. 263 */ 264 wmb(); 265 arm_smmu_cb_write(smmu_domain->smmu, smmu_domain->cfg.cbndx, 266 ARM_SMMU_CB_S1_TLBIASID, smmu_domain->cfg.asid); 267 arm_smmu_tlb_sync_context(smmu_domain); 268 } 269 270 static void arm_smmu_tlb_inv_context_s2(void *cookie) 271 { 272 struct arm_smmu_domain *smmu_domain = cookie; 273 struct arm_smmu_device *smmu = smmu_domain->smmu; 274 275 /* See above */ 276 wmb(); 277 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_TLBIVMID, smmu_domain->cfg.vmid); 278 arm_smmu_tlb_sync_global(smmu); 279 } 280 281 static void arm_smmu_tlb_inv_range_s1(unsigned long iova, size_t size, 282 size_t granule, void *cookie, int reg) 283 { 284 struct arm_smmu_domain *smmu_domain = cookie; 285 struct arm_smmu_device *smmu = smmu_domain->smmu; 286 struct arm_smmu_cfg *cfg = &smmu_domain->cfg; 287 int idx = cfg->cbndx; 288 289 if (smmu->features & ARM_SMMU_FEAT_COHERENT_WALK) 290 wmb(); 291 292 if (cfg->fmt != ARM_SMMU_CTX_FMT_AARCH64) { 293 iova = (iova >> 12) << 12; 294 iova |= cfg->asid; 295 do { 296 arm_smmu_cb_write(smmu, idx, reg, iova); 297 iova += granule; 298 } while (size -= granule); 299 } else { 300 iova >>= 12; 301 iova |= (u64)cfg->asid << 48; 302 do { 303 arm_smmu_cb_writeq(smmu, idx, reg, iova); 304 iova += granule >> 12; 305 } while (size -= granule); 306 } 307 } 308 309 static void arm_smmu_tlb_inv_range_s2(unsigned long iova, size_t size, 310 size_t granule, void *cookie, int reg) 311 { 312 struct arm_smmu_domain *smmu_domain = cookie; 313 struct arm_smmu_device *smmu = smmu_domain->smmu; 314 int idx = smmu_domain->cfg.cbndx; 315 316 if (smmu->features & ARM_SMMU_FEAT_COHERENT_WALK) 317 wmb(); 318 319 iova >>= 12; 320 do { 321 if (smmu_domain->cfg.fmt == ARM_SMMU_CTX_FMT_AARCH64) 322 arm_smmu_cb_writeq(smmu, idx, reg, iova); 323 else 324 arm_smmu_cb_write(smmu, idx, reg, iova); 325 iova += granule >> 12; 326 } while (size -= granule); 327 } 328 329 static void arm_smmu_tlb_inv_walk_s1(unsigned long iova, size_t size, 330 size_t granule, void *cookie) 331 { 332 struct arm_smmu_domain *smmu_domain = cookie; 333 struct arm_smmu_cfg *cfg = &smmu_domain->cfg; 334 335 if (cfg->flush_walk_prefer_tlbiasid) { 336 arm_smmu_tlb_inv_context_s1(cookie); 337 } else { 338 arm_smmu_tlb_inv_range_s1(iova, size, granule, cookie, 339 ARM_SMMU_CB_S1_TLBIVA); 340 arm_smmu_tlb_sync_context(cookie); 341 } 342 } 343 344 static void arm_smmu_tlb_add_page_s1(struct iommu_iotlb_gather *gather, 345 unsigned long iova, size_t granule, 346 void *cookie) 347 { 348 arm_smmu_tlb_inv_range_s1(iova, granule, granule, cookie, 349 ARM_SMMU_CB_S1_TLBIVAL); 350 } 351 352 static void arm_smmu_tlb_inv_walk_s2(unsigned long iova, size_t size, 353 size_t granule, void *cookie) 354 { 355 arm_smmu_tlb_inv_range_s2(iova, size, granule, cookie, 356 ARM_SMMU_CB_S2_TLBIIPAS2); 357 arm_smmu_tlb_sync_context(cookie); 358 } 359 360 static void arm_smmu_tlb_add_page_s2(struct iommu_iotlb_gather *gather, 361 unsigned long iova, size_t granule, 362 void *cookie) 363 { 364 arm_smmu_tlb_inv_range_s2(iova, granule, granule, cookie, 365 ARM_SMMU_CB_S2_TLBIIPAS2L); 366 } 367 368 static void arm_smmu_tlb_inv_walk_s2_v1(unsigned long iova, size_t size, 369 size_t granule, void *cookie) 370 { 371 arm_smmu_tlb_inv_context_s2(cookie); 372 } 373 /* 374 * On MMU-401 at least, the cost of firing off multiple TLBIVMIDs appears 375 * almost negligible, but the benefit of getting the first one in as far ahead 376 * of the sync as possible is significant, hence we don't just make this a 377 * no-op and call arm_smmu_tlb_inv_context_s2() from .iotlb_sync as you might 378 * think. 379 */ 380 static void arm_smmu_tlb_add_page_s2_v1(struct iommu_iotlb_gather *gather, 381 unsigned long iova, size_t granule, 382 void *cookie) 383 { 384 struct arm_smmu_domain *smmu_domain = cookie; 385 struct arm_smmu_device *smmu = smmu_domain->smmu; 386 387 if (smmu->features & ARM_SMMU_FEAT_COHERENT_WALK) 388 wmb(); 389 390 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_TLBIVMID, smmu_domain->cfg.vmid); 391 } 392 393 static const struct iommu_flush_ops arm_smmu_s1_tlb_ops = { 394 .tlb_flush_all = arm_smmu_tlb_inv_context_s1, 395 .tlb_flush_walk = arm_smmu_tlb_inv_walk_s1, 396 .tlb_add_page = arm_smmu_tlb_add_page_s1, 397 }; 398 399 static const struct iommu_flush_ops arm_smmu_s2_tlb_ops_v2 = { 400 .tlb_flush_all = arm_smmu_tlb_inv_context_s2, 401 .tlb_flush_walk = arm_smmu_tlb_inv_walk_s2, 402 .tlb_add_page = arm_smmu_tlb_add_page_s2, 403 }; 404 405 static const struct iommu_flush_ops arm_smmu_s2_tlb_ops_v1 = { 406 .tlb_flush_all = arm_smmu_tlb_inv_context_s2, 407 .tlb_flush_walk = arm_smmu_tlb_inv_walk_s2_v1, 408 .tlb_add_page = arm_smmu_tlb_add_page_s2_v1, 409 }; 410 411 412 void arm_smmu_read_context_fault_info(struct arm_smmu_device *smmu, int idx, 413 struct arm_smmu_context_fault_info *cfi) 414 { 415 cfi->iova = arm_smmu_cb_readq(smmu, idx, ARM_SMMU_CB_FAR); 416 cfi->fsr = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_FSR); 417 cfi->fsynr = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_FSYNR0); 418 cfi->cbfrsynra = arm_smmu_gr1_read(smmu, ARM_SMMU_GR1_CBFRSYNRA(idx)); 419 } 420 421 void arm_smmu_print_context_fault_info(struct arm_smmu_device *smmu, int idx, 422 const struct arm_smmu_context_fault_info *cfi) 423 { 424 dev_err(smmu->dev, 425 "Unhandled context fault: fsr=0x%x, iova=0x%08lx, fsynr=0x%x, cbfrsynra=0x%x, cb=%d\n", 426 cfi->fsr, cfi->iova, cfi->fsynr, cfi->cbfrsynra, idx); 427 428 dev_err(smmu->dev, "FSR = %08x [%s%sFormat=%u%s%s%s%s%s%s%s%s], SID=0x%x\n", 429 cfi->fsr, 430 (cfi->fsr & ARM_SMMU_CB_FSR_MULTI) ? "MULTI " : "", 431 (cfi->fsr & ARM_SMMU_CB_FSR_SS) ? "SS " : "", 432 (u32)FIELD_GET(ARM_SMMU_CB_FSR_FORMAT, cfi->fsr), 433 (cfi->fsr & ARM_SMMU_CB_FSR_UUT) ? " UUT" : "", 434 (cfi->fsr & ARM_SMMU_CB_FSR_ASF) ? " ASF" : "", 435 (cfi->fsr & ARM_SMMU_CB_FSR_TLBLKF) ? " TLBLKF" : "", 436 (cfi->fsr & ARM_SMMU_CB_FSR_TLBMCF) ? " TLBMCF" : "", 437 (cfi->fsr & ARM_SMMU_CB_FSR_EF) ? " EF" : "", 438 (cfi->fsr & ARM_SMMU_CB_FSR_PF) ? " PF" : "", 439 (cfi->fsr & ARM_SMMU_CB_FSR_AFF) ? " AFF" : "", 440 (cfi->fsr & ARM_SMMU_CB_FSR_TF) ? " TF" : "", 441 cfi->cbfrsynra); 442 443 dev_err(smmu->dev, "FSYNR0 = %08x [S1CBNDX=%u%s%s%s%s%s%s PLVL=%u]\n", 444 cfi->fsynr, 445 (u32)FIELD_GET(ARM_SMMU_CB_FSYNR0_S1CBNDX, cfi->fsynr), 446 (cfi->fsynr & ARM_SMMU_CB_FSYNR0_AFR) ? " AFR" : "", 447 (cfi->fsynr & ARM_SMMU_CB_FSYNR0_PTWF) ? " PTWF" : "", 448 (cfi->fsynr & ARM_SMMU_CB_FSYNR0_NSATTR) ? " NSATTR" : "", 449 (cfi->fsynr & ARM_SMMU_CB_FSYNR0_IND) ? " IND" : "", 450 (cfi->fsynr & ARM_SMMU_CB_FSYNR0_PNU) ? " PNU" : "", 451 (cfi->fsynr & ARM_SMMU_CB_FSYNR0_WNR) ? " WNR" : "", 452 (u32)FIELD_GET(ARM_SMMU_CB_FSYNR0_PLVL, cfi->fsynr)); 453 } 454 455 static irqreturn_t arm_smmu_context_fault(int irq, void *dev) 456 { 457 struct arm_smmu_context_fault_info cfi; 458 struct arm_smmu_domain *smmu_domain = dev; 459 struct arm_smmu_device *smmu = smmu_domain->smmu; 460 static DEFINE_RATELIMIT_STATE(rs, DEFAULT_RATELIMIT_INTERVAL, 461 DEFAULT_RATELIMIT_BURST); 462 int idx = smmu_domain->cfg.cbndx; 463 int ret; 464 465 arm_smmu_read_context_fault_info(smmu, idx, &cfi); 466 467 if (!(cfi.fsr & ARM_SMMU_CB_FSR_FAULT)) 468 return IRQ_NONE; 469 470 ret = report_iommu_fault(&smmu_domain->domain, NULL, cfi.iova, 471 cfi.fsynr & ARM_SMMU_CB_FSYNR0_WNR ? IOMMU_FAULT_WRITE : IOMMU_FAULT_READ); 472 473 if (ret == -ENOSYS && __ratelimit(&rs)) 474 arm_smmu_print_context_fault_info(smmu, idx, &cfi); 475 476 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_FSR, cfi.fsr); 477 478 if (cfi.fsr & ARM_SMMU_CB_FSR_SS) { 479 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_RESUME, 480 ret == -EAGAIN ? 0 : ARM_SMMU_RESUME_TERMINATE); 481 } 482 483 return IRQ_HANDLED; 484 } 485 486 static irqreturn_t arm_smmu_global_fault(int irq, void *dev) 487 { 488 u32 gfsr, gfsynr0, gfsynr1, gfsynr2; 489 struct arm_smmu_device *smmu = dev; 490 static DEFINE_RATELIMIT_STATE(rs, DEFAULT_RATELIMIT_INTERVAL, 491 DEFAULT_RATELIMIT_BURST); 492 493 gfsr = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSR); 494 gfsynr0 = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSYNR0); 495 gfsynr1 = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSYNR1); 496 gfsynr2 = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSYNR2); 497 498 if (!gfsr) 499 return IRQ_NONE; 500 501 if (__ratelimit(&rs)) { 502 if (IS_ENABLED(CONFIG_ARM_SMMU_DISABLE_BYPASS_BY_DEFAULT) && 503 (gfsr & ARM_SMMU_sGFSR_USF)) 504 dev_err(smmu->dev, 505 "Blocked unknown Stream ID 0x%hx; boot with \"arm-smmu.disable_bypass=0\" to allow, but this may have security implications\n", 506 (u16)gfsynr1); 507 else 508 dev_err(smmu->dev, 509 "Unexpected global fault, this could be serious\n"); 510 dev_err(smmu->dev, 511 "\tGFSR 0x%08x, GFSYNR0 0x%08x, GFSYNR1 0x%08x, GFSYNR2 0x%08x\n", 512 gfsr, gfsynr0, gfsynr1, gfsynr2); 513 } 514 515 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sGFSR, gfsr); 516 return IRQ_HANDLED; 517 } 518 519 static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain, 520 struct io_pgtable_cfg *pgtbl_cfg) 521 { 522 struct arm_smmu_cfg *cfg = &smmu_domain->cfg; 523 struct arm_smmu_cb *cb = &smmu_domain->smmu->cbs[cfg->cbndx]; 524 bool stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS; 525 526 cb->cfg = cfg; 527 528 /* TCR */ 529 if (stage1) { 530 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) { 531 cb->tcr[0] = pgtbl_cfg->arm_v7s_cfg.tcr; 532 } else { 533 cb->tcr[0] = arm_smmu_lpae_tcr(pgtbl_cfg); 534 cb->tcr[1] = arm_smmu_lpae_tcr2(pgtbl_cfg); 535 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) 536 cb->tcr[1] |= ARM_SMMU_TCR2_AS; 537 else 538 cb->tcr[0] |= ARM_SMMU_TCR_EAE; 539 } 540 } else { 541 cb->tcr[0] = arm_smmu_lpae_vtcr(pgtbl_cfg); 542 } 543 544 /* TTBRs */ 545 if (stage1) { 546 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) { 547 cb->ttbr[0] = pgtbl_cfg->arm_v7s_cfg.ttbr; 548 cb->ttbr[1] = 0; 549 } else { 550 cb->ttbr[0] = FIELD_PREP(ARM_SMMU_TTBRn_ASID, 551 cfg->asid); 552 cb->ttbr[1] = FIELD_PREP(ARM_SMMU_TTBRn_ASID, 553 cfg->asid); 554 555 if (pgtbl_cfg->quirks & IO_PGTABLE_QUIRK_ARM_TTBR1) 556 cb->ttbr[1] |= pgtbl_cfg->arm_lpae_s1_cfg.ttbr; 557 else 558 cb->ttbr[0] |= pgtbl_cfg->arm_lpae_s1_cfg.ttbr; 559 } 560 } else { 561 cb->ttbr[0] = pgtbl_cfg->arm_lpae_s2_cfg.vttbr; 562 } 563 564 /* MAIRs (stage-1 only) */ 565 if (stage1) { 566 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) { 567 cb->mair[0] = pgtbl_cfg->arm_v7s_cfg.prrr; 568 cb->mair[1] = pgtbl_cfg->arm_v7s_cfg.nmrr; 569 } else { 570 cb->mair[0] = pgtbl_cfg->arm_lpae_s1_cfg.mair; 571 cb->mair[1] = pgtbl_cfg->arm_lpae_s1_cfg.mair >> 32; 572 } 573 } 574 } 575 576 void arm_smmu_write_context_bank(struct arm_smmu_device *smmu, int idx) 577 { 578 u32 reg; 579 bool stage1; 580 struct arm_smmu_cb *cb = &smmu->cbs[idx]; 581 struct arm_smmu_cfg *cfg = cb->cfg; 582 583 /* Unassigned context banks only need disabling */ 584 if (!cfg) { 585 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_SCTLR, 0); 586 return; 587 } 588 589 stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS; 590 591 /* CBA2R */ 592 if (smmu->version > ARM_SMMU_V1) { 593 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) 594 reg = ARM_SMMU_CBA2R_VA64; 595 else 596 reg = 0; 597 /* 16-bit VMIDs live in CBA2R */ 598 if (smmu->features & ARM_SMMU_FEAT_VMID16) 599 reg |= FIELD_PREP(ARM_SMMU_CBA2R_VMID16, cfg->vmid); 600 601 arm_smmu_gr1_write(smmu, ARM_SMMU_GR1_CBA2R(idx), reg); 602 } 603 604 /* CBAR */ 605 reg = FIELD_PREP(ARM_SMMU_CBAR_TYPE, cfg->cbar); 606 if (smmu->version < ARM_SMMU_V2) 607 reg |= FIELD_PREP(ARM_SMMU_CBAR_IRPTNDX, cfg->irptndx); 608 609 /* 610 * Use the weakest shareability/memory types, so they are 611 * overridden by the ttbcr/pte. 612 */ 613 if (stage1) { 614 reg |= FIELD_PREP(ARM_SMMU_CBAR_S1_BPSHCFG, 615 ARM_SMMU_CBAR_S1_BPSHCFG_NSH) | 616 FIELD_PREP(ARM_SMMU_CBAR_S1_MEMATTR, 617 ARM_SMMU_CBAR_S1_MEMATTR_WB); 618 } else if (!(smmu->features & ARM_SMMU_FEAT_VMID16)) { 619 /* 8-bit VMIDs live in CBAR */ 620 reg |= FIELD_PREP(ARM_SMMU_CBAR_VMID, cfg->vmid); 621 } 622 arm_smmu_gr1_write(smmu, ARM_SMMU_GR1_CBAR(idx), reg); 623 624 /* 625 * TCR 626 * We must write this before the TTBRs, since it determines the 627 * access behaviour of some fields (in particular, ASID[15:8]). 628 */ 629 if (stage1 && smmu->version > ARM_SMMU_V1) 630 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_TCR2, cb->tcr[1]); 631 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_TCR, cb->tcr[0]); 632 633 /* TTBRs */ 634 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) { 635 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_CONTEXTIDR, cfg->asid); 636 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_TTBR0, cb->ttbr[0]); 637 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_TTBR1, cb->ttbr[1]); 638 } else { 639 arm_smmu_cb_writeq(smmu, idx, ARM_SMMU_CB_TTBR0, cb->ttbr[0]); 640 if (stage1) 641 arm_smmu_cb_writeq(smmu, idx, ARM_SMMU_CB_TTBR1, 642 cb->ttbr[1]); 643 } 644 645 /* MAIRs (stage-1 only) */ 646 if (stage1) { 647 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_S1_MAIR0, cb->mair[0]); 648 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_S1_MAIR1, cb->mair[1]); 649 } 650 651 /* SCTLR */ 652 reg = ARM_SMMU_SCTLR_CFIE | ARM_SMMU_SCTLR_CFRE | ARM_SMMU_SCTLR_AFE | 653 ARM_SMMU_SCTLR_TRE | ARM_SMMU_SCTLR_M; 654 if (stage1) 655 reg |= ARM_SMMU_SCTLR_S1_ASIDPNE; 656 if (IS_ENABLED(CONFIG_CPU_BIG_ENDIAN)) 657 reg |= ARM_SMMU_SCTLR_E; 658 659 if (smmu->impl && smmu->impl->write_sctlr) 660 smmu->impl->write_sctlr(smmu, idx, reg); 661 else 662 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_SCTLR, reg); 663 } 664 665 static int arm_smmu_alloc_context_bank(struct arm_smmu_domain *smmu_domain, 666 struct arm_smmu_device *smmu, 667 struct device *dev, unsigned int start) 668 { 669 if (smmu->impl && smmu->impl->alloc_context_bank) 670 return smmu->impl->alloc_context_bank(smmu_domain, smmu, dev, start); 671 672 return __arm_smmu_alloc_bitmap(smmu->context_map, start, smmu->num_context_banks); 673 } 674 675 static int arm_smmu_init_domain_context(struct arm_smmu_domain *smmu_domain, 676 struct arm_smmu_device *smmu, 677 struct device *dev) 678 { 679 int irq, start, ret = 0; 680 unsigned long ias, oas; 681 struct io_pgtable_ops *pgtbl_ops; 682 struct io_pgtable_cfg pgtbl_cfg; 683 enum io_pgtable_fmt fmt; 684 struct iommu_domain *domain = &smmu_domain->domain; 685 struct arm_smmu_cfg *cfg = &smmu_domain->cfg; 686 irqreturn_t (*context_fault)(int irq, void *dev); 687 688 mutex_lock(&smmu_domain->init_mutex); 689 if (smmu_domain->smmu) 690 goto out_unlock; 691 692 /* 693 * Mapping the requested stage onto what we support is surprisingly 694 * complicated, mainly because the spec allows S1+S2 SMMUs without 695 * support for nested translation. That means we end up with the 696 * following table: 697 * 698 * Requested Supported Actual 699 * S1 N S1 700 * S1 S1+S2 S1 701 * S1 S2 S2 702 * S1 S1 S1 703 * N N N 704 * N S1+S2 S2 705 * N S2 S2 706 * N S1 S1 707 * 708 * Note that you can't actually request stage-2 mappings. 709 */ 710 if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S1)) 711 smmu_domain->stage = ARM_SMMU_DOMAIN_S2; 712 if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S2)) 713 smmu_domain->stage = ARM_SMMU_DOMAIN_S1; 714 715 /* 716 * Choosing a suitable context format is even more fiddly. Until we 717 * grow some way for the caller to express a preference, and/or move 718 * the decision into the io-pgtable code where it arguably belongs, 719 * just aim for the closest thing to the rest of the system, and hope 720 * that the hardware isn't esoteric enough that we can't assume AArch64 721 * support to be a superset of AArch32 support... 722 */ 723 if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_L) 724 cfg->fmt = ARM_SMMU_CTX_FMT_AARCH32_L; 725 if (IS_ENABLED(CONFIG_IOMMU_IO_PGTABLE_ARMV7S) && 726 !IS_ENABLED(CONFIG_64BIT) && !IS_ENABLED(CONFIG_ARM_LPAE) && 727 (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_S) && 728 (smmu_domain->stage == ARM_SMMU_DOMAIN_S1)) 729 cfg->fmt = ARM_SMMU_CTX_FMT_AARCH32_S; 730 if ((IS_ENABLED(CONFIG_64BIT) || cfg->fmt == ARM_SMMU_CTX_FMT_NONE) && 731 (smmu->features & (ARM_SMMU_FEAT_FMT_AARCH64_64K | 732 ARM_SMMU_FEAT_FMT_AARCH64_16K | 733 ARM_SMMU_FEAT_FMT_AARCH64_4K))) 734 cfg->fmt = ARM_SMMU_CTX_FMT_AARCH64; 735 736 if (cfg->fmt == ARM_SMMU_CTX_FMT_NONE) { 737 ret = -EINVAL; 738 goto out_unlock; 739 } 740 741 switch (smmu_domain->stage) { 742 case ARM_SMMU_DOMAIN_S1: 743 cfg->cbar = CBAR_TYPE_S1_TRANS_S2_BYPASS; 744 start = smmu->num_s2_context_banks; 745 ias = smmu->va_size; 746 oas = smmu->ipa_size; 747 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) { 748 fmt = ARM_64_LPAE_S1; 749 } else if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_L) { 750 fmt = ARM_32_LPAE_S1; 751 ias = min(ias, 32UL); 752 oas = min(oas, 40UL); 753 } else { 754 fmt = ARM_V7S; 755 ias = min(ias, 32UL); 756 oas = min(oas, 32UL); 757 } 758 smmu_domain->flush_ops = &arm_smmu_s1_tlb_ops; 759 break; 760 case ARM_SMMU_DOMAIN_NESTED: 761 /* 762 * We will likely want to change this if/when KVM gets 763 * involved. 764 */ 765 case ARM_SMMU_DOMAIN_S2: 766 cfg->cbar = CBAR_TYPE_S2_TRANS; 767 start = 0; 768 ias = smmu->ipa_size; 769 oas = smmu->pa_size; 770 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) { 771 fmt = ARM_64_LPAE_S2; 772 } else { 773 fmt = ARM_32_LPAE_S2; 774 ias = min(ias, 40UL); 775 oas = min(oas, 40UL); 776 } 777 if (smmu->version == ARM_SMMU_V2) 778 smmu_domain->flush_ops = &arm_smmu_s2_tlb_ops_v2; 779 else 780 smmu_domain->flush_ops = &arm_smmu_s2_tlb_ops_v1; 781 break; 782 default: 783 ret = -EINVAL; 784 goto out_unlock; 785 } 786 787 ret = arm_smmu_alloc_context_bank(smmu_domain, smmu, dev, start); 788 if (ret < 0) { 789 goto out_unlock; 790 } 791 792 smmu_domain->smmu = smmu; 793 794 cfg->cbndx = ret; 795 if (smmu->version < ARM_SMMU_V2) { 796 cfg->irptndx = atomic_inc_return(&smmu->irptndx); 797 cfg->irptndx %= smmu->num_context_irqs; 798 } else { 799 cfg->irptndx = cfg->cbndx; 800 } 801 802 if (smmu_domain->stage == ARM_SMMU_DOMAIN_S2) 803 cfg->vmid = cfg->cbndx + 1; 804 else 805 cfg->asid = cfg->cbndx; 806 807 pgtbl_cfg = (struct io_pgtable_cfg) { 808 .pgsize_bitmap = smmu->pgsize_bitmap, 809 .ias = ias, 810 .oas = oas, 811 .coherent_walk = smmu->features & ARM_SMMU_FEAT_COHERENT_WALK, 812 .tlb = smmu_domain->flush_ops, 813 .iommu_dev = smmu->dev, 814 }; 815 816 if (smmu->impl && smmu->impl->init_context) { 817 ret = smmu->impl->init_context(smmu_domain, &pgtbl_cfg, dev); 818 if (ret) 819 goto out_clear_smmu; 820 } 821 822 if (smmu_domain->pgtbl_quirks) 823 pgtbl_cfg.quirks |= smmu_domain->pgtbl_quirks; 824 825 pgtbl_ops = alloc_io_pgtable_ops(fmt, &pgtbl_cfg, smmu_domain); 826 if (!pgtbl_ops) { 827 ret = -ENOMEM; 828 goto out_clear_smmu; 829 } 830 831 /* Update the domain's page sizes to reflect the page table format */ 832 domain->pgsize_bitmap = pgtbl_cfg.pgsize_bitmap; 833 834 if (pgtbl_cfg.quirks & IO_PGTABLE_QUIRK_ARM_TTBR1) { 835 domain->geometry.aperture_start = ~0UL << ias; 836 domain->geometry.aperture_end = ~0UL; 837 } else { 838 domain->geometry.aperture_end = (1UL << ias) - 1; 839 } 840 841 domain->geometry.force_aperture = true; 842 843 /* Initialise the context bank with our page table cfg */ 844 arm_smmu_init_context_bank(smmu_domain, &pgtbl_cfg); 845 arm_smmu_write_context_bank(smmu, cfg->cbndx); 846 847 /* 848 * Request context fault interrupt. Do this last to avoid the 849 * handler seeing a half-initialised domain state. 850 */ 851 irq = smmu->irqs[cfg->irptndx]; 852 853 if (smmu->impl && smmu->impl->context_fault) 854 context_fault = smmu->impl->context_fault; 855 else 856 context_fault = arm_smmu_context_fault; 857 858 if (smmu->impl && smmu->impl->context_fault_needs_threaded_irq) 859 ret = devm_request_threaded_irq(smmu->dev, irq, NULL, 860 context_fault, 861 IRQF_ONESHOT | IRQF_SHARED, 862 "arm-smmu-context-fault", 863 smmu_domain); 864 else 865 ret = devm_request_irq(smmu->dev, irq, context_fault, IRQF_SHARED, 866 "arm-smmu-context-fault", smmu_domain); 867 868 if (ret < 0) { 869 dev_err(smmu->dev, "failed to request context IRQ %d (%u)\n", 870 cfg->irptndx, irq); 871 cfg->irptndx = ARM_SMMU_INVALID_IRPTNDX; 872 } 873 874 mutex_unlock(&smmu_domain->init_mutex); 875 876 /* Publish page table ops for map/unmap */ 877 smmu_domain->pgtbl_ops = pgtbl_ops; 878 return 0; 879 880 out_clear_smmu: 881 __arm_smmu_free_bitmap(smmu->context_map, cfg->cbndx); 882 smmu_domain->smmu = NULL; 883 out_unlock: 884 mutex_unlock(&smmu_domain->init_mutex); 885 return ret; 886 } 887 888 static void arm_smmu_destroy_domain_context(struct arm_smmu_domain *smmu_domain) 889 { 890 struct arm_smmu_device *smmu = smmu_domain->smmu; 891 struct arm_smmu_cfg *cfg = &smmu_domain->cfg; 892 int ret, irq; 893 894 if (!smmu) 895 return; 896 897 ret = arm_smmu_rpm_get(smmu); 898 if (ret < 0) 899 return; 900 901 /* 902 * Disable the context bank and free the page tables before freeing 903 * it. 904 */ 905 smmu->cbs[cfg->cbndx].cfg = NULL; 906 arm_smmu_write_context_bank(smmu, cfg->cbndx); 907 908 if (cfg->irptndx != ARM_SMMU_INVALID_IRPTNDX) { 909 irq = smmu->irqs[cfg->irptndx]; 910 devm_free_irq(smmu->dev, irq, smmu_domain); 911 } 912 913 free_io_pgtable_ops(smmu_domain->pgtbl_ops); 914 __arm_smmu_free_bitmap(smmu->context_map, cfg->cbndx); 915 916 arm_smmu_rpm_put(smmu); 917 } 918 919 static struct iommu_domain *arm_smmu_domain_alloc_paging(struct device *dev) 920 { 921 struct arm_smmu_domain *smmu_domain; 922 923 /* 924 * Allocate the domain and initialise some of its data structures. 925 * We can't really do anything meaningful until we've added a 926 * master. 927 */ 928 smmu_domain = kzalloc(sizeof(*smmu_domain), GFP_KERNEL); 929 if (!smmu_domain) 930 return NULL; 931 932 mutex_init(&smmu_domain->init_mutex); 933 spin_lock_init(&smmu_domain->cb_lock); 934 935 return &smmu_domain->domain; 936 } 937 938 static void arm_smmu_domain_free(struct iommu_domain *domain) 939 { 940 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); 941 942 /* 943 * Free the domain resources. We assume that all devices have 944 * already been detached. 945 */ 946 arm_smmu_destroy_domain_context(smmu_domain); 947 kfree(smmu_domain); 948 } 949 950 static void arm_smmu_write_smr(struct arm_smmu_device *smmu, int idx) 951 { 952 struct arm_smmu_smr *smr = smmu->smrs + idx; 953 u32 reg = FIELD_PREP(ARM_SMMU_SMR_ID, smr->id) | 954 FIELD_PREP(ARM_SMMU_SMR_MASK, smr->mask); 955 956 if (!(smmu->features & ARM_SMMU_FEAT_EXIDS) && smr->valid) 957 reg |= ARM_SMMU_SMR_VALID; 958 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_SMR(idx), reg); 959 } 960 961 static void arm_smmu_write_s2cr(struct arm_smmu_device *smmu, int idx) 962 { 963 struct arm_smmu_s2cr *s2cr = smmu->s2crs + idx; 964 u32 reg; 965 966 if (smmu->impl && smmu->impl->write_s2cr) { 967 smmu->impl->write_s2cr(smmu, idx); 968 return; 969 } 970 971 reg = FIELD_PREP(ARM_SMMU_S2CR_TYPE, s2cr->type) | 972 FIELD_PREP(ARM_SMMU_S2CR_CBNDX, s2cr->cbndx) | 973 FIELD_PREP(ARM_SMMU_S2CR_PRIVCFG, s2cr->privcfg); 974 975 if (smmu->features & ARM_SMMU_FEAT_EXIDS && smmu->smrs && 976 smmu->smrs[idx].valid) 977 reg |= ARM_SMMU_S2CR_EXIDVALID; 978 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_S2CR(idx), reg); 979 } 980 981 static void arm_smmu_write_sme(struct arm_smmu_device *smmu, int idx) 982 { 983 arm_smmu_write_s2cr(smmu, idx); 984 if (smmu->smrs) 985 arm_smmu_write_smr(smmu, idx); 986 } 987 988 /* 989 * The width of SMR's mask field depends on sCR0_EXIDENABLE, so this function 990 * should be called after sCR0 is written. 991 */ 992 static void arm_smmu_test_smr_masks(struct arm_smmu_device *smmu) 993 { 994 u32 smr; 995 int i; 996 997 if (!smmu->smrs) 998 return; 999 /* 1000 * If we've had to accommodate firmware memory regions, we may 1001 * have live SMRs by now; tread carefully... 1002 * 1003 * Somewhat perversely, not having a free SMR for this test implies we 1004 * can get away without it anyway, as we'll only be able to 'allocate' 1005 * these SMRs for the ID/mask values we're already trusting to be OK. 1006 */ 1007 for (i = 0; i < smmu->num_mapping_groups; i++) 1008 if (!smmu->smrs[i].valid) 1009 goto smr_ok; 1010 return; 1011 smr_ok: 1012 /* 1013 * SMR.ID bits may not be preserved if the corresponding MASK 1014 * bits are set, so check each one separately. We can reject 1015 * masters later if they try to claim IDs outside these masks. 1016 */ 1017 smr = FIELD_PREP(ARM_SMMU_SMR_ID, smmu->streamid_mask); 1018 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_SMR(i), smr); 1019 smr = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_SMR(i)); 1020 smmu->streamid_mask = FIELD_GET(ARM_SMMU_SMR_ID, smr); 1021 1022 smr = FIELD_PREP(ARM_SMMU_SMR_MASK, smmu->streamid_mask); 1023 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_SMR(i), smr); 1024 smr = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_SMR(i)); 1025 smmu->smr_mask_mask = FIELD_GET(ARM_SMMU_SMR_MASK, smr); 1026 } 1027 1028 static int arm_smmu_find_sme(struct arm_smmu_device *smmu, u16 id, u16 mask) 1029 { 1030 struct arm_smmu_smr *smrs = smmu->smrs; 1031 int i, free_idx = -ENOSPC; 1032 1033 /* Stream indexing is blissfully easy */ 1034 if (!smrs) 1035 return id; 1036 1037 /* Validating SMRs is... less so */ 1038 for (i = 0; i < smmu->num_mapping_groups; ++i) { 1039 if (!smrs[i].valid) { 1040 /* 1041 * Note the first free entry we come across, which 1042 * we'll claim in the end if nothing else matches. 1043 */ 1044 if (free_idx < 0) 1045 free_idx = i; 1046 continue; 1047 } 1048 /* 1049 * If the new entry is _entirely_ matched by an existing entry, 1050 * then reuse that, with the guarantee that there also cannot 1051 * be any subsequent conflicting entries. In normal use we'd 1052 * expect simply identical entries for this case, but there's 1053 * no harm in accommodating the generalisation. 1054 */ 1055 if ((mask & smrs[i].mask) == mask && 1056 !((id ^ smrs[i].id) & ~smrs[i].mask)) 1057 return i; 1058 /* 1059 * If the new entry has any other overlap with an existing one, 1060 * though, then there always exists at least one stream ID 1061 * which would cause a conflict, and we can't allow that risk. 1062 */ 1063 if (!((id ^ smrs[i].id) & ~(smrs[i].mask | mask))) 1064 return -EINVAL; 1065 } 1066 1067 return free_idx; 1068 } 1069 1070 static bool arm_smmu_free_sme(struct arm_smmu_device *smmu, int idx) 1071 { 1072 if (--smmu->s2crs[idx].count) 1073 return false; 1074 1075 smmu->s2crs[idx] = s2cr_init_val; 1076 if (smmu->smrs) 1077 smmu->smrs[idx].valid = false; 1078 1079 return true; 1080 } 1081 1082 static int arm_smmu_master_alloc_smes(struct device *dev) 1083 { 1084 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); 1085 struct arm_smmu_master_cfg *cfg = dev_iommu_priv_get(dev); 1086 struct arm_smmu_device *smmu = cfg->smmu; 1087 struct arm_smmu_smr *smrs = smmu->smrs; 1088 int i, idx, ret; 1089 1090 mutex_lock(&smmu->stream_map_mutex); 1091 /* Figure out a viable stream map entry allocation */ 1092 for_each_cfg_sme(cfg, fwspec, i, idx) { 1093 u16 sid = FIELD_GET(ARM_SMMU_SMR_ID, fwspec->ids[i]); 1094 u16 mask = FIELD_GET(ARM_SMMU_SMR_MASK, fwspec->ids[i]); 1095 1096 if (idx != INVALID_SMENDX) { 1097 ret = -EEXIST; 1098 goto out_err; 1099 } 1100 1101 ret = arm_smmu_find_sme(smmu, sid, mask); 1102 if (ret < 0) 1103 goto out_err; 1104 1105 idx = ret; 1106 if (smrs && smmu->s2crs[idx].count == 0) { 1107 smrs[idx].id = sid; 1108 smrs[idx].mask = mask; 1109 smrs[idx].valid = true; 1110 } 1111 smmu->s2crs[idx].count++; 1112 cfg->smendx[i] = (s16)idx; 1113 } 1114 1115 /* It worked! Now, poke the actual hardware */ 1116 for_each_cfg_sme(cfg, fwspec, i, idx) 1117 arm_smmu_write_sme(smmu, idx); 1118 1119 mutex_unlock(&smmu->stream_map_mutex); 1120 return 0; 1121 1122 out_err: 1123 while (i--) { 1124 arm_smmu_free_sme(smmu, cfg->smendx[i]); 1125 cfg->smendx[i] = INVALID_SMENDX; 1126 } 1127 mutex_unlock(&smmu->stream_map_mutex); 1128 return ret; 1129 } 1130 1131 static void arm_smmu_master_free_smes(struct arm_smmu_master_cfg *cfg, 1132 struct iommu_fwspec *fwspec) 1133 { 1134 struct arm_smmu_device *smmu = cfg->smmu; 1135 int i, idx; 1136 1137 mutex_lock(&smmu->stream_map_mutex); 1138 for_each_cfg_sme(cfg, fwspec, i, idx) { 1139 if (arm_smmu_free_sme(smmu, idx)) 1140 arm_smmu_write_sme(smmu, idx); 1141 cfg->smendx[i] = INVALID_SMENDX; 1142 } 1143 mutex_unlock(&smmu->stream_map_mutex); 1144 } 1145 1146 static void arm_smmu_master_install_s2crs(struct arm_smmu_master_cfg *cfg, 1147 enum arm_smmu_s2cr_type type, 1148 u8 cbndx, struct iommu_fwspec *fwspec) 1149 { 1150 struct arm_smmu_device *smmu = cfg->smmu; 1151 struct arm_smmu_s2cr *s2cr = smmu->s2crs; 1152 int i, idx; 1153 1154 for_each_cfg_sme(cfg, fwspec, i, idx) { 1155 if (type == s2cr[idx].type && cbndx == s2cr[idx].cbndx) 1156 continue; 1157 1158 s2cr[idx].type = type; 1159 s2cr[idx].privcfg = S2CR_PRIVCFG_DEFAULT; 1160 s2cr[idx].cbndx = cbndx; 1161 arm_smmu_write_s2cr(smmu, idx); 1162 } 1163 } 1164 1165 static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev) 1166 { 1167 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); 1168 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); 1169 struct arm_smmu_master_cfg *cfg; 1170 struct arm_smmu_device *smmu; 1171 int ret; 1172 1173 /* 1174 * FIXME: The arch/arm DMA API code tries to attach devices to its own 1175 * domains between of_xlate() and probe_device() - we have no way to cope 1176 * with that, so until ARM gets converted to rely on groups and default 1177 * domains, just say no (but more politely than by dereferencing NULL). 1178 * This should be at least a WARN_ON once that's sorted. 1179 */ 1180 cfg = dev_iommu_priv_get(dev); 1181 if (!cfg) 1182 return -ENODEV; 1183 1184 smmu = cfg->smmu; 1185 1186 ret = arm_smmu_rpm_get(smmu); 1187 if (ret < 0) 1188 return ret; 1189 1190 /* Ensure that the domain is finalised */ 1191 ret = arm_smmu_init_domain_context(smmu_domain, smmu, dev); 1192 if (ret < 0) 1193 goto rpm_put; 1194 1195 /* 1196 * Sanity check the domain. We don't support domains across 1197 * different SMMUs. 1198 */ 1199 if (smmu_domain->smmu != smmu) { 1200 ret = -EINVAL; 1201 goto rpm_put; 1202 } 1203 1204 /* Looks ok, so add the device to the domain */ 1205 arm_smmu_master_install_s2crs(cfg, S2CR_TYPE_TRANS, 1206 smmu_domain->cfg.cbndx, fwspec); 1207 rpm_put: 1208 arm_smmu_rpm_put(smmu); 1209 return ret; 1210 } 1211 1212 static int arm_smmu_attach_dev_type(struct device *dev, 1213 enum arm_smmu_s2cr_type type) 1214 { 1215 struct arm_smmu_master_cfg *cfg = dev_iommu_priv_get(dev); 1216 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); 1217 struct arm_smmu_device *smmu; 1218 int ret; 1219 1220 if (!cfg) 1221 return -ENODEV; 1222 smmu = cfg->smmu; 1223 1224 ret = arm_smmu_rpm_get(smmu); 1225 if (ret < 0) 1226 return ret; 1227 1228 arm_smmu_master_install_s2crs(cfg, type, 0, fwspec); 1229 arm_smmu_rpm_put(smmu); 1230 return 0; 1231 } 1232 1233 static int arm_smmu_attach_dev_identity(struct iommu_domain *domain, 1234 struct device *dev) 1235 { 1236 return arm_smmu_attach_dev_type(dev, S2CR_TYPE_BYPASS); 1237 } 1238 1239 static const struct iommu_domain_ops arm_smmu_identity_ops = { 1240 .attach_dev = arm_smmu_attach_dev_identity, 1241 }; 1242 1243 static struct iommu_domain arm_smmu_identity_domain = { 1244 .type = IOMMU_DOMAIN_IDENTITY, 1245 .ops = &arm_smmu_identity_ops, 1246 }; 1247 1248 static int arm_smmu_attach_dev_blocked(struct iommu_domain *domain, 1249 struct device *dev) 1250 { 1251 return arm_smmu_attach_dev_type(dev, S2CR_TYPE_FAULT); 1252 } 1253 1254 static const struct iommu_domain_ops arm_smmu_blocked_ops = { 1255 .attach_dev = arm_smmu_attach_dev_blocked, 1256 }; 1257 1258 static struct iommu_domain arm_smmu_blocked_domain = { 1259 .type = IOMMU_DOMAIN_BLOCKED, 1260 .ops = &arm_smmu_blocked_ops, 1261 }; 1262 1263 static int arm_smmu_map_pages(struct iommu_domain *domain, unsigned long iova, 1264 phys_addr_t paddr, size_t pgsize, size_t pgcount, 1265 int prot, gfp_t gfp, size_t *mapped) 1266 { 1267 struct io_pgtable_ops *ops = to_smmu_domain(domain)->pgtbl_ops; 1268 struct arm_smmu_device *smmu = to_smmu_domain(domain)->smmu; 1269 int ret; 1270 1271 if (!ops) 1272 return -ENODEV; 1273 1274 arm_smmu_rpm_get(smmu); 1275 ret = ops->map_pages(ops, iova, paddr, pgsize, pgcount, prot, gfp, mapped); 1276 arm_smmu_rpm_put(smmu); 1277 1278 return ret; 1279 } 1280 1281 static size_t arm_smmu_unmap_pages(struct iommu_domain *domain, unsigned long iova, 1282 size_t pgsize, size_t pgcount, 1283 struct iommu_iotlb_gather *iotlb_gather) 1284 { 1285 struct io_pgtable_ops *ops = to_smmu_domain(domain)->pgtbl_ops; 1286 struct arm_smmu_device *smmu = to_smmu_domain(domain)->smmu; 1287 size_t ret; 1288 1289 if (!ops) 1290 return 0; 1291 1292 arm_smmu_rpm_get(smmu); 1293 ret = ops->unmap_pages(ops, iova, pgsize, pgcount, iotlb_gather); 1294 arm_smmu_rpm_put(smmu); 1295 1296 return ret; 1297 } 1298 1299 static void arm_smmu_flush_iotlb_all(struct iommu_domain *domain) 1300 { 1301 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); 1302 struct arm_smmu_device *smmu = smmu_domain->smmu; 1303 1304 if (smmu_domain->flush_ops) { 1305 arm_smmu_rpm_get(smmu); 1306 smmu_domain->flush_ops->tlb_flush_all(smmu_domain); 1307 arm_smmu_rpm_put(smmu); 1308 } 1309 } 1310 1311 static void arm_smmu_iotlb_sync(struct iommu_domain *domain, 1312 struct iommu_iotlb_gather *gather) 1313 { 1314 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); 1315 struct arm_smmu_device *smmu = smmu_domain->smmu; 1316 1317 if (!smmu) 1318 return; 1319 1320 arm_smmu_rpm_get(smmu); 1321 if (smmu->version == ARM_SMMU_V2 || 1322 smmu_domain->stage == ARM_SMMU_DOMAIN_S1) 1323 arm_smmu_tlb_sync_context(smmu_domain); 1324 else 1325 arm_smmu_tlb_sync_global(smmu); 1326 arm_smmu_rpm_put(smmu); 1327 } 1328 1329 static phys_addr_t arm_smmu_iova_to_phys_hard(struct iommu_domain *domain, 1330 dma_addr_t iova) 1331 { 1332 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); 1333 struct arm_smmu_device *smmu = smmu_domain->smmu; 1334 struct arm_smmu_cfg *cfg = &smmu_domain->cfg; 1335 struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops; 1336 struct device *dev = smmu->dev; 1337 void __iomem *reg; 1338 u32 tmp; 1339 u64 phys; 1340 unsigned long va, flags; 1341 int ret, idx = cfg->cbndx; 1342 phys_addr_t addr = 0; 1343 1344 ret = arm_smmu_rpm_get(smmu); 1345 if (ret < 0) 1346 return 0; 1347 1348 spin_lock_irqsave(&smmu_domain->cb_lock, flags); 1349 va = iova & ~0xfffUL; 1350 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) 1351 arm_smmu_cb_writeq(smmu, idx, ARM_SMMU_CB_ATS1PR, va); 1352 else 1353 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_ATS1PR, va); 1354 1355 reg = arm_smmu_page(smmu, ARM_SMMU_CB(smmu, idx)) + ARM_SMMU_CB_ATSR; 1356 if (readl_poll_timeout_atomic(reg, tmp, !(tmp & ARM_SMMU_CB_ATSR_ACTIVE), 1357 5, 50)) { 1358 spin_unlock_irqrestore(&smmu_domain->cb_lock, flags); 1359 dev_err(dev, 1360 "iova to phys timed out on %pad. Falling back to software table walk.\n", 1361 &iova); 1362 arm_smmu_rpm_put(smmu); 1363 return ops->iova_to_phys(ops, iova); 1364 } 1365 1366 phys = arm_smmu_cb_readq(smmu, idx, ARM_SMMU_CB_PAR); 1367 spin_unlock_irqrestore(&smmu_domain->cb_lock, flags); 1368 if (phys & ARM_SMMU_CB_PAR_F) { 1369 dev_err(dev, "translation fault!\n"); 1370 dev_err(dev, "PAR = 0x%llx\n", phys); 1371 goto out; 1372 } 1373 1374 addr = (phys & GENMASK_ULL(39, 12)) | (iova & 0xfff); 1375 out: 1376 arm_smmu_rpm_put(smmu); 1377 1378 return addr; 1379 } 1380 1381 static phys_addr_t arm_smmu_iova_to_phys(struct iommu_domain *domain, 1382 dma_addr_t iova) 1383 { 1384 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); 1385 struct io_pgtable_ops *ops = smmu_domain->pgtbl_ops; 1386 1387 if (!ops) 1388 return 0; 1389 1390 if (smmu_domain->smmu->features & ARM_SMMU_FEAT_TRANS_OPS && 1391 smmu_domain->stage == ARM_SMMU_DOMAIN_S1) 1392 return arm_smmu_iova_to_phys_hard(domain, iova); 1393 1394 return ops->iova_to_phys(ops, iova); 1395 } 1396 1397 static bool arm_smmu_capable(struct device *dev, enum iommu_cap cap) 1398 { 1399 struct arm_smmu_master_cfg *cfg = dev_iommu_priv_get(dev); 1400 1401 switch (cap) { 1402 case IOMMU_CAP_CACHE_COHERENCY: 1403 /* 1404 * It's overwhelmingly the case in practice that when the pagetable 1405 * walk interface is connected to a coherent interconnect, all the 1406 * translation interfaces are too. Furthermore if the device is 1407 * natively coherent, then its translation interface must also be. 1408 */ 1409 return cfg->smmu->features & ARM_SMMU_FEAT_COHERENT_WALK || 1410 device_get_dma_attr(dev) == DEV_DMA_COHERENT; 1411 case IOMMU_CAP_NOEXEC: 1412 case IOMMU_CAP_DEFERRED_FLUSH: 1413 return true; 1414 default: 1415 return false; 1416 } 1417 } 1418 1419 static 1420 struct arm_smmu_device *arm_smmu_get_by_fwnode(struct fwnode_handle *fwnode) 1421 { 1422 struct device *dev = bus_find_device_by_fwnode(&platform_bus_type, fwnode); 1423 1424 put_device(dev); 1425 return dev ? dev_get_drvdata(dev) : NULL; 1426 } 1427 1428 static struct iommu_device *arm_smmu_probe_device(struct device *dev) 1429 { 1430 struct arm_smmu_device *smmu = NULL; 1431 struct arm_smmu_master_cfg *cfg; 1432 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); 1433 int i, ret; 1434 1435 if (using_legacy_binding) { 1436 ret = arm_smmu_register_legacy_master(dev, &smmu); 1437 1438 /* 1439 * If dev->iommu_fwspec is initally NULL, arm_smmu_register_legacy_master() 1440 * will allocate/initialise a new one. Thus we need to update fwspec for 1441 * later use. 1442 */ 1443 fwspec = dev_iommu_fwspec_get(dev); 1444 if (ret) 1445 goto out_free; 1446 } else { 1447 smmu = arm_smmu_get_by_fwnode(fwspec->iommu_fwnode); 1448 } 1449 1450 ret = -EINVAL; 1451 for (i = 0; i < fwspec->num_ids; i++) { 1452 u16 sid = FIELD_GET(ARM_SMMU_SMR_ID, fwspec->ids[i]); 1453 u16 mask = FIELD_GET(ARM_SMMU_SMR_MASK, fwspec->ids[i]); 1454 1455 if (sid & ~smmu->streamid_mask) { 1456 dev_err(dev, "stream ID 0x%x out of range for SMMU (0x%x)\n", 1457 sid, smmu->streamid_mask); 1458 goto out_free; 1459 } 1460 if (mask & ~smmu->smr_mask_mask) { 1461 dev_err(dev, "SMR mask 0x%x out of range for SMMU (0x%x)\n", 1462 mask, smmu->smr_mask_mask); 1463 goto out_free; 1464 } 1465 } 1466 1467 ret = -ENOMEM; 1468 cfg = kzalloc(offsetof(struct arm_smmu_master_cfg, smendx[i]), 1469 GFP_KERNEL); 1470 if (!cfg) 1471 goto out_free; 1472 1473 cfg->smmu = smmu; 1474 dev_iommu_priv_set(dev, cfg); 1475 while (i--) 1476 cfg->smendx[i] = INVALID_SMENDX; 1477 1478 ret = arm_smmu_rpm_get(smmu); 1479 if (ret < 0) 1480 goto out_cfg_free; 1481 1482 ret = arm_smmu_master_alloc_smes(dev); 1483 arm_smmu_rpm_put(smmu); 1484 1485 if (ret) 1486 goto out_cfg_free; 1487 1488 device_link_add(dev, smmu->dev, 1489 DL_FLAG_PM_RUNTIME | DL_FLAG_AUTOREMOVE_SUPPLIER); 1490 1491 return &smmu->iommu; 1492 1493 out_cfg_free: 1494 kfree(cfg); 1495 out_free: 1496 return ERR_PTR(ret); 1497 } 1498 1499 static void arm_smmu_release_device(struct device *dev) 1500 { 1501 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); 1502 struct arm_smmu_master_cfg *cfg = dev_iommu_priv_get(dev); 1503 int ret; 1504 1505 ret = arm_smmu_rpm_get(cfg->smmu); 1506 if (ret < 0) 1507 return; 1508 1509 arm_smmu_master_free_smes(cfg, fwspec); 1510 1511 arm_smmu_rpm_put(cfg->smmu); 1512 1513 kfree(cfg); 1514 } 1515 1516 static void arm_smmu_probe_finalize(struct device *dev) 1517 { 1518 struct arm_smmu_master_cfg *cfg; 1519 struct arm_smmu_device *smmu; 1520 1521 cfg = dev_iommu_priv_get(dev); 1522 smmu = cfg->smmu; 1523 1524 if (smmu->impl && smmu->impl->probe_finalize) 1525 smmu->impl->probe_finalize(smmu, dev); 1526 } 1527 1528 static struct iommu_group *arm_smmu_device_group(struct device *dev) 1529 { 1530 struct arm_smmu_master_cfg *cfg = dev_iommu_priv_get(dev); 1531 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); 1532 struct arm_smmu_device *smmu = cfg->smmu; 1533 struct iommu_group *group = NULL; 1534 int i, idx; 1535 1536 mutex_lock(&smmu->stream_map_mutex); 1537 for_each_cfg_sme(cfg, fwspec, i, idx) { 1538 if (group && smmu->s2crs[idx].group && 1539 group != smmu->s2crs[idx].group) { 1540 mutex_unlock(&smmu->stream_map_mutex); 1541 return ERR_PTR(-EINVAL); 1542 } 1543 1544 group = smmu->s2crs[idx].group; 1545 } 1546 1547 if (group) { 1548 mutex_unlock(&smmu->stream_map_mutex); 1549 return iommu_group_ref_get(group); 1550 } 1551 1552 if (dev_is_pci(dev)) 1553 group = pci_device_group(dev); 1554 else if (dev_is_fsl_mc(dev)) 1555 group = fsl_mc_device_group(dev); 1556 else 1557 group = generic_device_group(dev); 1558 1559 /* Remember group for faster lookups */ 1560 if (!IS_ERR(group)) 1561 for_each_cfg_sme(cfg, fwspec, i, idx) 1562 smmu->s2crs[idx].group = group; 1563 1564 mutex_unlock(&smmu->stream_map_mutex); 1565 return group; 1566 } 1567 1568 static int arm_smmu_set_pgtable_quirks(struct iommu_domain *domain, 1569 unsigned long quirks) 1570 { 1571 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); 1572 int ret = 0; 1573 1574 mutex_lock(&smmu_domain->init_mutex); 1575 if (smmu_domain->smmu) 1576 ret = -EPERM; 1577 else 1578 smmu_domain->pgtbl_quirks = quirks; 1579 mutex_unlock(&smmu_domain->init_mutex); 1580 1581 return ret; 1582 } 1583 1584 static int arm_smmu_of_xlate(struct device *dev, 1585 const struct of_phandle_args *args) 1586 { 1587 u32 mask, fwid = 0; 1588 1589 if (args->args_count > 0) 1590 fwid |= FIELD_PREP(ARM_SMMU_SMR_ID, args->args[0]); 1591 1592 if (args->args_count > 1) 1593 fwid |= FIELD_PREP(ARM_SMMU_SMR_MASK, args->args[1]); 1594 else if (!of_property_read_u32(args->np, "stream-match-mask", &mask)) 1595 fwid |= FIELD_PREP(ARM_SMMU_SMR_MASK, mask); 1596 1597 return iommu_fwspec_add_ids(dev, &fwid, 1); 1598 } 1599 1600 static void arm_smmu_get_resv_regions(struct device *dev, 1601 struct list_head *head) 1602 { 1603 struct iommu_resv_region *region; 1604 int prot = IOMMU_WRITE | IOMMU_NOEXEC | IOMMU_MMIO; 1605 1606 region = iommu_alloc_resv_region(MSI_IOVA_BASE, MSI_IOVA_LENGTH, 1607 prot, IOMMU_RESV_SW_MSI, GFP_KERNEL); 1608 if (!region) 1609 return; 1610 1611 list_add_tail(®ion->list, head); 1612 1613 iommu_dma_get_resv_regions(dev, head); 1614 } 1615 1616 static int arm_smmu_def_domain_type(struct device *dev) 1617 { 1618 struct arm_smmu_master_cfg *cfg = dev_iommu_priv_get(dev); 1619 const struct arm_smmu_impl *impl = cfg->smmu->impl; 1620 1621 if (using_legacy_binding) 1622 return IOMMU_DOMAIN_IDENTITY; 1623 1624 if (impl && impl->def_domain_type) 1625 return impl->def_domain_type(dev); 1626 1627 return 0; 1628 } 1629 1630 static struct iommu_ops arm_smmu_ops = { 1631 .identity_domain = &arm_smmu_identity_domain, 1632 .blocked_domain = &arm_smmu_blocked_domain, 1633 .capable = arm_smmu_capable, 1634 .domain_alloc_paging = arm_smmu_domain_alloc_paging, 1635 .probe_device = arm_smmu_probe_device, 1636 .release_device = arm_smmu_release_device, 1637 .probe_finalize = arm_smmu_probe_finalize, 1638 .device_group = arm_smmu_device_group, 1639 .of_xlate = arm_smmu_of_xlate, 1640 .get_resv_regions = arm_smmu_get_resv_regions, 1641 .def_domain_type = arm_smmu_def_domain_type, 1642 .pgsize_bitmap = -1UL, /* Restricted during device attach */ 1643 .owner = THIS_MODULE, 1644 .default_domain_ops = &(const struct iommu_domain_ops) { 1645 .attach_dev = arm_smmu_attach_dev, 1646 .map_pages = arm_smmu_map_pages, 1647 .unmap_pages = arm_smmu_unmap_pages, 1648 .flush_iotlb_all = arm_smmu_flush_iotlb_all, 1649 .iotlb_sync = arm_smmu_iotlb_sync, 1650 .iova_to_phys = arm_smmu_iova_to_phys, 1651 .set_pgtable_quirks = arm_smmu_set_pgtable_quirks, 1652 .free = arm_smmu_domain_free, 1653 } 1654 }; 1655 1656 static void arm_smmu_device_reset(struct arm_smmu_device *smmu) 1657 { 1658 int i; 1659 u32 reg; 1660 1661 /* clear global FSR */ 1662 reg = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSR); 1663 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sGFSR, reg); 1664 1665 /* 1666 * Reset stream mapping groups: Initial values mark all SMRn as 1667 * invalid and all S2CRn as bypass unless overridden. 1668 */ 1669 for (i = 0; i < smmu->num_mapping_groups; ++i) 1670 arm_smmu_write_sme(smmu, i); 1671 1672 /* Make sure all context banks are disabled and clear CB_FSR */ 1673 for (i = 0; i < smmu->num_context_banks; ++i) { 1674 arm_smmu_write_context_bank(smmu, i); 1675 arm_smmu_cb_write(smmu, i, ARM_SMMU_CB_FSR, ARM_SMMU_CB_FSR_FAULT); 1676 } 1677 1678 /* Invalidate the TLB, just in case */ 1679 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_TLBIALLH, QCOM_DUMMY_VAL); 1680 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_TLBIALLNSNH, QCOM_DUMMY_VAL); 1681 1682 reg = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sCR0); 1683 1684 /* Enable fault reporting */ 1685 reg |= (ARM_SMMU_sCR0_GFRE | ARM_SMMU_sCR0_GFIE | 1686 ARM_SMMU_sCR0_GCFGFRE | ARM_SMMU_sCR0_GCFGFIE); 1687 1688 /* Disable TLB broadcasting. */ 1689 reg |= (ARM_SMMU_sCR0_VMIDPNE | ARM_SMMU_sCR0_PTM); 1690 1691 /* Enable client access, handling unmatched streams as appropriate */ 1692 reg &= ~ARM_SMMU_sCR0_CLIENTPD; 1693 if (disable_bypass) 1694 reg |= ARM_SMMU_sCR0_USFCFG; 1695 else 1696 reg &= ~ARM_SMMU_sCR0_USFCFG; 1697 1698 /* Disable forced broadcasting */ 1699 reg &= ~ARM_SMMU_sCR0_FB; 1700 1701 /* Don't upgrade barriers */ 1702 reg &= ~(ARM_SMMU_sCR0_BSU); 1703 1704 if (smmu->features & ARM_SMMU_FEAT_VMID16) 1705 reg |= ARM_SMMU_sCR0_VMID16EN; 1706 1707 if (smmu->features & ARM_SMMU_FEAT_EXIDS) 1708 reg |= ARM_SMMU_sCR0_EXIDENABLE; 1709 1710 if (smmu->impl && smmu->impl->reset) 1711 smmu->impl->reset(smmu); 1712 1713 /* Push the button */ 1714 arm_smmu_tlb_sync_global(smmu); 1715 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sCR0, reg); 1716 } 1717 1718 static int arm_smmu_id_size_to_bits(int size) 1719 { 1720 switch (size) { 1721 case 0: 1722 return 32; 1723 case 1: 1724 return 36; 1725 case 2: 1726 return 40; 1727 case 3: 1728 return 42; 1729 case 4: 1730 return 44; 1731 case 5: 1732 default: 1733 return 48; 1734 } 1735 } 1736 1737 static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu) 1738 { 1739 unsigned int size; 1740 u32 id; 1741 bool cttw_reg, cttw_fw = smmu->features & ARM_SMMU_FEAT_COHERENT_WALK; 1742 int i, ret; 1743 1744 dev_notice(smmu->dev, "probing hardware configuration...\n"); 1745 dev_notice(smmu->dev, "SMMUv%d with:\n", 1746 smmu->version == ARM_SMMU_V2 ? 2 : 1); 1747 1748 /* ID0 */ 1749 id = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_ID0); 1750 1751 /* Restrict available stages based on module parameter */ 1752 if (force_stage == 1) 1753 id &= ~(ARM_SMMU_ID0_S2TS | ARM_SMMU_ID0_NTS); 1754 else if (force_stage == 2) 1755 id &= ~(ARM_SMMU_ID0_S1TS | ARM_SMMU_ID0_NTS); 1756 1757 if (id & ARM_SMMU_ID0_S1TS) { 1758 smmu->features |= ARM_SMMU_FEAT_TRANS_S1; 1759 dev_notice(smmu->dev, "\tstage 1 translation\n"); 1760 } 1761 1762 if (id & ARM_SMMU_ID0_S2TS) { 1763 smmu->features |= ARM_SMMU_FEAT_TRANS_S2; 1764 dev_notice(smmu->dev, "\tstage 2 translation\n"); 1765 } 1766 1767 if (id & ARM_SMMU_ID0_NTS) { 1768 smmu->features |= ARM_SMMU_FEAT_TRANS_NESTED; 1769 dev_notice(smmu->dev, "\tnested translation\n"); 1770 } 1771 1772 if (!(smmu->features & 1773 (ARM_SMMU_FEAT_TRANS_S1 | ARM_SMMU_FEAT_TRANS_S2))) { 1774 dev_err(smmu->dev, "\tno translation support!\n"); 1775 return -ENODEV; 1776 } 1777 1778 if ((id & ARM_SMMU_ID0_S1TS) && 1779 ((smmu->version < ARM_SMMU_V2) || !(id & ARM_SMMU_ID0_ATOSNS))) { 1780 smmu->features |= ARM_SMMU_FEAT_TRANS_OPS; 1781 dev_notice(smmu->dev, "\taddress translation ops\n"); 1782 } 1783 1784 /* 1785 * In order for DMA API calls to work properly, we must defer to what 1786 * the FW says about coherency, regardless of what the hardware claims. 1787 * Fortunately, this also opens up a workaround for systems where the 1788 * ID register value has ended up configured incorrectly. 1789 */ 1790 cttw_reg = !!(id & ARM_SMMU_ID0_CTTW); 1791 if (cttw_fw || cttw_reg) 1792 dev_notice(smmu->dev, "\t%scoherent table walk\n", 1793 cttw_fw ? "" : "non-"); 1794 if (cttw_fw != cttw_reg) 1795 dev_notice(smmu->dev, 1796 "\t(IDR0.CTTW overridden by FW configuration)\n"); 1797 1798 /* Max. number of entries we have for stream matching/indexing */ 1799 if (smmu->version == ARM_SMMU_V2 && id & ARM_SMMU_ID0_EXIDS) { 1800 smmu->features |= ARM_SMMU_FEAT_EXIDS; 1801 size = 1 << 16; 1802 } else { 1803 size = 1 << FIELD_GET(ARM_SMMU_ID0_NUMSIDB, id); 1804 } 1805 smmu->streamid_mask = size - 1; 1806 if (id & ARM_SMMU_ID0_SMS) { 1807 smmu->features |= ARM_SMMU_FEAT_STREAM_MATCH; 1808 size = FIELD_GET(ARM_SMMU_ID0_NUMSMRG, id); 1809 if (size == 0) { 1810 dev_err(smmu->dev, 1811 "stream-matching supported, but no SMRs present!\n"); 1812 return -ENODEV; 1813 } 1814 1815 /* Zero-initialised to mark as invalid */ 1816 smmu->smrs = devm_kcalloc(smmu->dev, size, sizeof(*smmu->smrs), 1817 GFP_KERNEL); 1818 if (!smmu->smrs) 1819 return -ENOMEM; 1820 1821 dev_notice(smmu->dev, 1822 "\tstream matching with %u register groups", size); 1823 } 1824 /* s2cr->type == 0 means translation, so initialise explicitly */ 1825 smmu->s2crs = devm_kmalloc_array(smmu->dev, size, sizeof(*smmu->s2crs), 1826 GFP_KERNEL); 1827 if (!smmu->s2crs) 1828 return -ENOMEM; 1829 for (i = 0; i < size; i++) 1830 smmu->s2crs[i] = s2cr_init_val; 1831 1832 smmu->num_mapping_groups = size; 1833 mutex_init(&smmu->stream_map_mutex); 1834 spin_lock_init(&smmu->global_sync_lock); 1835 1836 if (smmu->version < ARM_SMMU_V2 || 1837 !(id & ARM_SMMU_ID0_PTFS_NO_AARCH32)) { 1838 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH32_L; 1839 if (!(id & ARM_SMMU_ID0_PTFS_NO_AARCH32S)) 1840 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH32_S; 1841 } 1842 1843 /* ID1 */ 1844 id = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_ID1); 1845 smmu->pgshift = (id & ARM_SMMU_ID1_PAGESIZE) ? 16 : 12; 1846 1847 /* Check for size mismatch of SMMU address space from mapped region */ 1848 size = 1 << (FIELD_GET(ARM_SMMU_ID1_NUMPAGENDXB, id) + 1); 1849 if (smmu->numpage != 2 * size << smmu->pgshift) 1850 dev_warn(smmu->dev, 1851 "SMMU address space size (0x%x) differs from mapped region size (0x%x)!\n", 1852 2 * size << smmu->pgshift, smmu->numpage); 1853 /* Now properly encode NUMPAGE to subsequently derive SMMU_CB_BASE */ 1854 smmu->numpage = size; 1855 1856 smmu->num_s2_context_banks = FIELD_GET(ARM_SMMU_ID1_NUMS2CB, id); 1857 smmu->num_context_banks = FIELD_GET(ARM_SMMU_ID1_NUMCB, id); 1858 if (smmu->num_s2_context_banks > smmu->num_context_banks) { 1859 dev_err(smmu->dev, "impossible number of S2 context banks!\n"); 1860 return -ENODEV; 1861 } 1862 dev_notice(smmu->dev, "\t%u context banks (%u stage-2 only)\n", 1863 smmu->num_context_banks, smmu->num_s2_context_banks); 1864 smmu->cbs = devm_kcalloc(smmu->dev, smmu->num_context_banks, 1865 sizeof(*smmu->cbs), GFP_KERNEL); 1866 if (!smmu->cbs) 1867 return -ENOMEM; 1868 1869 /* ID2 */ 1870 id = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_ID2); 1871 size = arm_smmu_id_size_to_bits(FIELD_GET(ARM_SMMU_ID2_IAS, id)); 1872 smmu->ipa_size = size; 1873 1874 /* The output mask is also applied for bypass */ 1875 size = arm_smmu_id_size_to_bits(FIELD_GET(ARM_SMMU_ID2_OAS, id)); 1876 smmu->pa_size = size; 1877 1878 if (id & ARM_SMMU_ID2_VMID16) 1879 smmu->features |= ARM_SMMU_FEAT_VMID16; 1880 1881 /* 1882 * What the page table walker can address actually depends on which 1883 * descriptor format is in use, but since a) we don't know that yet, 1884 * and b) it can vary per context bank, this will have to do... 1885 */ 1886 if (dma_set_mask_and_coherent(smmu->dev, DMA_BIT_MASK(size))) 1887 dev_warn(smmu->dev, 1888 "failed to set DMA mask for table walker\n"); 1889 1890 if (smmu->version < ARM_SMMU_V2) { 1891 smmu->va_size = smmu->ipa_size; 1892 if (smmu->version == ARM_SMMU_V1_64K) 1893 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_64K; 1894 } else { 1895 size = FIELD_GET(ARM_SMMU_ID2_UBS, id); 1896 smmu->va_size = arm_smmu_id_size_to_bits(size); 1897 if (id & ARM_SMMU_ID2_PTFS_4K) 1898 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_4K; 1899 if (id & ARM_SMMU_ID2_PTFS_16K) 1900 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_16K; 1901 if (id & ARM_SMMU_ID2_PTFS_64K) 1902 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_64K; 1903 } 1904 1905 if (smmu->impl && smmu->impl->cfg_probe) { 1906 ret = smmu->impl->cfg_probe(smmu); 1907 if (ret) 1908 return ret; 1909 } 1910 1911 /* Now we've corralled the various formats, what'll it do? */ 1912 if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_S) 1913 smmu->pgsize_bitmap |= SZ_4K | SZ_64K | SZ_1M | SZ_16M; 1914 if (smmu->features & 1915 (ARM_SMMU_FEAT_FMT_AARCH32_L | ARM_SMMU_FEAT_FMT_AARCH64_4K)) 1916 smmu->pgsize_bitmap |= SZ_4K | SZ_2M | SZ_1G; 1917 if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH64_16K) 1918 smmu->pgsize_bitmap |= SZ_16K | SZ_32M; 1919 if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH64_64K) 1920 smmu->pgsize_bitmap |= SZ_64K | SZ_512M; 1921 1922 if (arm_smmu_ops.pgsize_bitmap == -1UL) 1923 arm_smmu_ops.pgsize_bitmap = smmu->pgsize_bitmap; 1924 else 1925 arm_smmu_ops.pgsize_bitmap |= smmu->pgsize_bitmap; 1926 dev_notice(smmu->dev, "\tSupported page sizes: 0x%08lx\n", 1927 smmu->pgsize_bitmap); 1928 1929 1930 if (smmu->features & ARM_SMMU_FEAT_TRANS_S1) 1931 dev_notice(smmu->dev, "\tStage-1: %lu-bit VA -> %lu-bit IPA\n", 1932 smmu->va_size, smmu->ipa_size); 1933 1934 if (smmu->features & ARM_SMMU_FEAT_TRANS_S2) 1935 dev_notice(smmu->dev, "\tStage-2: %lu-bit IPA -> %lu-bit PA\n", 1936 smmu->ipa_size, smmu->pa_size); 1937 1938 return 0; 1939 } 1940 1941 struct arm_smmu_match_data { 1942 enum arm_smmu_arch_version version; 1943 enum arm_smmu_implementation model; 1944 }; 1945 1946 #define ARM_SMMU_MATCH_DATA(name, ver, imp) \ 1947 static const struct arm_smmu_match_data name = { .version = ver, .model = imp } 1948 1949 ARM_SMMU_MATCH_DATA(smmu_generic_v1, ARM_SMMU_V1, GENERIC_SMMU); 1950 ARM_SMMU_MATCH_DATA(smmu_generic_v2, ARM_SMMU_V2, GENERIC_SMMU); 1951 ARM_SMMU_MATCH_DATA(arm_mmu401, ARM_SMMU_V1_64K, GENERIC_SMMU); 1952 ARM_SMMU_MATCH_DATA(arm_mmu500, ARM_SMMU_V2, ARM_MMU500); 1953 ARM_SMMU_MATCH_DATA(cavium_smmuv2, ARM_SMMU_V2, CAVIUM_SMMUV2); 1954 ARM_SMMU_MATCH_DATA(qcom_smmuv2, ARM_SMMU_V2, QCOM_SMMUV2); 1955 1956 static const struct of_device_id arm_smmu_of_match[] = { 1957 { .compatible = "arm,smmu-v1", .data = &smmu_generic_v1 }, 1958 { .compatible = "arm,smmu-v2", .data = &smmu_generic_v2 }, 1959 { .compatible = "arm,mmu-400", .data = &smmu_generic_v1 }, 1960 { .compatible = "arm,mmu-401", .data = &arm_mmu401 }, 1961 { .compatible = "arm,mmu-500", .data = &arm_mmu500 }, 1962 { .compatible = "cavium,smmu-v2", .data = &cavium_smmuv2 }, 1963 { .compatible = "nvidia,smmu-500", .data = &arm_mmu500 }, 1964 { .compatible = "qcom,smmu-v2", .data = &qcom_smmuv2 }, 1965 { }, 1966 }; 1967 MODULE_DEVICE_TABLE(of, arm_smmu_of_match); 1968 1969 #ifdef CONFIG_ACPI 1970 static int acpi_smmu_get_data(u32 model, struct arm_smmu_device *smmu) 1971 { 1972 int ret = 0; 1973 1974 switch (model) { 1975 case ACPI_IORT_SMMU_V1: 1976 case ACPI_IORT_SMMU_CORELINK_MMU400: 1977 smmu->version = ARM_SMMU_V1; 1978 smmu->model = GENERIC_SMMU; 1979 break; 1980 case ACPI_IORT_SMMU_CORELINK_MMU401: 1981 smmu->version = ARM_SMMU_V1_64K; 1982 smmu->model = GENERIC_SMMU; 1983 break; 1984 case ACPI_IORT_SMMU_V2: 1985 smmu->version = ARM_SMMU_V2; 1986 smmu->model = GENERIC_SMMU; 1987 break; 1988 case ACPI_IORT_SMMU_CORELINK_MMU500: 1989 smmu->version = ARM_SMMU_V2; 1990 smmu->model = ARM_MMU500; 1991 break; 1992 case ACPI_IORT_SMMU_CAVIUM_THUNDERX: 1993 smmu->version = ARM_SMMU_V2; 1994 smmu->model = CAVIUM_SMMUV2; 1995 break; 1996 default: 1997 ret = -ENODEV; 1998 } 1999 2000 return ret; 2001 } 2002 2003 static int arm_smmu_device_acpi_probe(struct arm_smmu_device *smmu, 2004 u32 *global_irqs, u32 *pmu_irqs) 2005 { 2006 struct device *dev = smmu->dev; 2007 struct acpi_iort_node *node = 2008 *(struct acpi_iort_node **)dev_get_platdata(dev); 2009 struct acpi_iort_smmu *iort_smmu; 2010 int ret; 2011 2012 /* Retrieve SMMU1/2 specific data */ 2013 iort_smmu = (struct acpi_iort_smmu *)node->node_data; 2014 2015 ret = acpi_smmu_get_data(iort_smmu->model, smmu); 2016 if (ret < 0) 2017 return ret; 2018 2019 /* Ignore the configuration access interrupt */ 2020 *global_irqs = 1; 2021 *pmu_irqs = 0; 2022 2023 if (iort_smmu->flags & ACPI_IORT_SMMU_COHERENT_WALK) 2024 smmu->features |= ARM_SMMU_FEAT_COHERENT_WALK; 2025 2026 return 0; 2027 } 2028 #else 2029 static inline int arm_smmu_device_acpi_probe(struct arm_smmu_device *smmu, 2030 u32 *global_irqs, u32 *pmu_irqs) 2031 { 2032 return -ENODEV; 2033 } 2034 #endif 2035 2036 static int arm_smmu_device_dt_probe(struct arm_smmu_device *smmu, 2037 u32 *global_irqs, u32 *pmu_irqs) 2038 { 2039 const struct arm_smmu_match_data *data; 2040 struct device *dev = smmu->dev; 2041 bool legacy_binding; 2042 2043 if (of_property_read_u32(dev->of_node, "#global-interrupts", global_irqs)) 2044 return dev_err_probe(dev, -ENODEV, 2045 "missing #global-interrupts property\n"); 2046 *pmu_irqs = 0; 2047 2048 data = of_device_get_match_data(dev); 2049 smmu->version = data->version; 2050 smmu->model = data->model; 2051 2052 legacy_binding = of_find_property(dev->of_node, "mmu-masters", NULL); 2053 if (legacy_binding && !using_generic_binding) { 2054 if (!using_legacy_binding) { 2055 pr_notice("deprecated \"mmu-masters\" DT property in use; %s support unavailable\n", 2056 IS_ENABLED(CONFIG_ARM_SMMU_LEGACY_DT_BINDINGS) ? "DMA API" : "SMMU"); 2057 } 2058 using_legacy_binding = true; 2059 } else if (!legacy_binding && !using_legacy_binding) { 2060 using_generic_binding = true; 2061 } else { 2062 dev_err(dev, "not probing due to mismatched DT properties\n"); 2063 return -ENODEV; 2064 } 2065 2066 if (of_dma_is_coherent(dev->of_node)) 2067 smmu->features |= ARM_SMMU_FEAT_COHERENT_WALK; 2068 2069 return 0; 2070 } 2071 2072 static void arm_smmu_rmr_install_bypass_smr(struct arm_smmu_device *smmu) 2073 { 2074 struct list_head rmr_list; 2075 struct iommu_resv_region *e; 2076 int idx, cnt = 0; 2077 u32 reg; 2078 2079 INIT_LIST_HEAD(&rmr_list); 2080 iort_get_rmr_sids(dev_fwnode(smmu->dev), &rmr_list); 2081 2082 /* 2083 * Rather than trying to look at existing mappings that 2084 * are setup by the firmware and then invalidate the ones 2085 * that do no have matching RMR entries, just disable the 2086 * SMMU until it gets enabled again in the reset routine. 2087 */ 2088 reg = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sCR0); 2089 reg |= ARM_SMMU_sCR0_CLIENTPD; 2090 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sCR0, reg); 2091 2092 list_for_each_entry(e, &rmr_list, list) { 2093 struct iommu_iort_rmr_data *rmr; 2094 int i; 2095 2096 rmr = container_of(e, struct iommu_iort_rmr_data, rr); 2097 for (i = 0; i < rmr->num_sids; i++) { 2098 idx = arm_smmu_find_sme(smmu, rmr->sids[i], ~0); 2099 if (idx < 0) 2100 continue; 2101 2102 if (smmu->s2crs[idx].count == 0) { 2103 smmu->smrs[idx].id = rmr->sids[i]; 2104 smmu->smrs[idx].mask = 0; 2105 smmu->smrs[idx].valid = true; 2106 } 2107 smmu->s2crs[idx].count++; 2108 smmu->s2crs[idx].type = S2CR_TYPE_BYPASS; 2109 smmu->s2crs[idx].privcfg = S2CR_PRIVCFG_DEFAULT; 2110 2111 cnt++; 2112 } 2113 } 2114 2115 dev_notice(smmu->dev, "\tpreserved %d boot mapping%s\n", cnt, 2116 str_plural(cnt)); 2117 iort_put_rmr_sids(dev_fwnode(smmu->dev), &rmr_list); 2118 } 2119 2120 static int arm_smmu_device_probe(struct platform_device *pdev) 2121 { 2122 struct resource *res; 2123 struct arm_smmu_device *smmu; 2124 struct device *dev = &pdev->dev; 2125 int num_irqs, i, err; 2126 u32 global_irqs, pmu_irqs; 2127 irqreturn_t (*global_fault)(int irq, void *dev); 2128 2129 smmu = devm_kzalloc(dev, sizeof(*smmu), GFP_KERNEL); 2130 if (!smmu) { 2131 dev_err(dev, "failed to allocate arm_smmu_device\n"); 2132 return -ENOMEM; 2133 } 2134 smmu->dev = dev; 2135 2136 if (dev->of_node) 2137 err = arm_smmu_device_dt_probe(smmu, &global_irqs, &pmu_irqs); 2138 else 2139 err = arm_smmu_device_acpi_probe(smmu, &global_irqs, &pmu_irqs); 2140 if (err) 2141 return err; 2142 2143 smmu->base = devm_platform_get_and_ioremap_resource(pdev, 0, &res); 2144 if (IS_ERR(smmu->base)) 2145 return PTR_ERR(smmu->base); 2146 smmu->ioaddr = res->start; 2147 2148 /* 2149 * The resource size should effectively match the value of SMMU_TOP; 2150 * stash that temporarily until we know PAGESIZE to validate it with. 2151 */ 2152 smmu->numpage = resource_size(res); 2153 2154 smmu = arm_smmu_impl_init(smmu); 2155 if (IS_ERR(smmu)) 2156 return PTR_ERR(smmu); 2157 2158 num_irqs = platform_irq_count(pdev); 2159 2160 smmu->num_context_irqs = num_irqs - global_irqs - pmu_irqs; 2161 if (smmu->num_context_irqs <= 0) 2162 return dev_err_probe(dev, -ENODEV, 2163 "found %d interrupts but expected at least %d\n", 2164 num_irqs, global_irqs + pmu_irqs + 1); 2165 2166 smmu->irqs = devm_kcalloc(dev, smmu->num_context_irqs, 2167 sizeof(*smmu->irqs), GFP_KERNEL); 2168 if (!smmu->irqs) 2169 return dev_err_probe(dev, -ENOMEM, "failed to allocate %d irqs\n", 2170 smmu->num_context_irqs); 2171 2172 for (i = 0; i < smmu->num_context_irqs; i++) { 2173 int irq = platform_get_irq(pdev, global_irqs + pmu_irqs + i); 2174 2175 if (irq < 0) 2176 return irq; 2177 smmu->irqs[i] = irq; 2178 } 2179 2180 err = devm_clk_bulk_get_all(dev, &smmu->clks); 2181 if (err < 0) { 2182 dev_err(dev, "failed to get clocks %d\n", err); 2183 return err; 2184 } 2185 smmu->num_clks = err; 2186 2187 err = clk_bulk_prepare_enable(smmu->num_clks, smmu->clks); 2188 if (err) 2189 return err; 2190 2191 err = arm_smmu_device_cfg_probe(smmu); 2192 if (err) 2193 return err; 2194 2195 if (smmu->version == ARM_SMMU_V2) { 2196 if (smmu->num_context_banks > smmu->num_context_irqs) { 2197 dev_err(dev, 2198 "found only %d context irq(s) but %d required\n", 2199 smmu->num_context_irqs, smmu->num_context_banks); 2200 return -ENODEV; 2201 } 2202 2203 /* Ignore superfluous interrupts */ 2204 smmu->num_context_irqs = smmu->num_context_banks; 2205 } 2206 2207 if (smmu->impl && smmu->impl->global_fault) 2208 global_fault = smmu->impl->global_fault; 2209 else 2210 global_fault = arm_smmu_global_fault; 2211 2212 for (i = 0; i < global_irqs; i++) { 2213 int irq = platform_get_irq(pdev, i); 2214 2215 if (irq < 0) 2216 return irq; 2217 2218 err = devm_request_irq(dev, irq, global_fault, IRQF_SHARED, 2219 "arm-smmu global fault", smmu); 2220 if (err) 2221 return dev_err_probe(dev, err, 2222 "failed to request global IRQ %d (%u)\n", 2223 i, irq); 2224 } 2225 2226 platform_set_drvdata(pdev, smmu); 2227 2228 /* Check for RMRs and install bypass SMRs if any */ 2229 arm_smmu_rmr_install_bypass_smr(smmu); 2230 2231 arm_smmu_device_reset(smmu); 2232 arm_smmu_test_smr_masks(smmu); 2233 2234 err = iommu_device_sysfs_add(&smmu->iommu, smmu->dev, NULL, 2235 "smmu.%pa", &smmu->ioaddr); 2236 if (err) 2237 return dev_err_probe(dev, err, "Failed to register iommu in sysfs\n"); 2238 2239 err = iommu_device_register(&smmu->iommu, &arm_smmu_ops, 2240 using_legacy_binding ? NULL : dev); 2241 if (err) { 2242 iommu_device_sysfs_remove(&smmu->iommu); 2243 return dev_err_probe(dev, err, "Failed to register iommu\n"); 2244 } 2245 2246 /* 2247 * We want to avoid touching dev->power.lock in fastpaths unless 2248 * it's really going to do something useful - pm_runtime_enabled() 2249 * can serve as an ideal proxy for that decision. So, conditionally 2250 * enable pm_runtime. 2251 */ 2252 if (dev->pm_domain) { 2253 pm_runtime_set_active(dev); 2254 pm_runtime_enable(dev); 2255 arm_smmu_rpm_use_autosuspend(smmu); 2256 } 2257 2258 return 0; 2259 } 2260 2261 static void arm_smmu_device_shutdown(struct platform_device *pdev) 2262 { 2263 struct arm_smmu_device *smmu = platform_get_drvdata(pdev); 2264 2265 if (!bitmap_empty(smmu->context_map, ARM_SMMU_MAX_CBS)) 2266 dev_notice(&pdev->dev, "disabling translation\n"); 2267 2268 arm_smmu_rpm_get(smmu); 2269 /* Turn the thing off */ 2270 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sCR0, ARM_SMMU_sCR0_CLIENTPD); 2271 arm_smmu_rpm_put(smmu); 2272 2273 if (pm_runtime_enabled(smmu->dev)) 2274 pm_runtime_force_suspend(smmu->dev); 2275 else 2276 clk_bulk_disable(smmu->num_clks, smmu->clks); 2277 2278 clk_bulk_unprepare(smmu->num_clks, smmu->clks); 2279 } 2280 2281 static void arm_smmu_device_remove(struct platform_device *pdev) 2282 { 2283 struct arm_smmu_device *smmu = platform_get_drvdata(pdev); 2284 2285 iommu_device_unregister(&smmu->iommu); 2286 iommu_device_sysfs_remove(&smmu->iommu); 2287 2288 arm_smmu_device_shutdown(pdev); 2289 } 2290 2291 static int __maybe_unused arm_smmu_runtime_resume(struct device *dev) 2292 { 2293 struct arm_smmu_device *smmu = dev_get_drvdata(dev); 2294 int ret; 2295 2296 ret = clk_bulk_enable(smmu->num_clks, smmu->clks); 2297 if (ret) 2298 return ret; 2299 2300 arm_smmu_device_reset(smmu); 2301 2302 return 0; 2303 } 2304 2305 static int __maybe_unused arm_smmu_runtime_suspend(struct device *dev) 2306 { 2307 struct arm_smmu_device *smmu = dev_get_drvdata(dev); 2308 2309 clk_bulk_disable(smmu->num_clks, smmu->clks); 2310 2311 return 0; 2312 } 2313 2314 static int __maybe_unused arm_smmu_pm_resume(struct device *dev) 2315 { 2316 int ret; 2317 struct arm_smmu_device *smmu = dev_get_drvdata(dev); 2318 2319 ret = clk_bulk_prepare(smmu->num_clks, smmu->clks); 2320 if (ret) 2321 return ret; 2322 2323 if (pm_runtime_suspended(dev)) 2324 return 0; 2325 2326 ret = arm_smmu_runtime_resume(dev); 2327 if (ret) 2328 clk_bulk_unprepare(smmu->num_clks, smmu->clks); 2329 2330 return ret; 2331 } 2332 2333 static int __maybe_unused arm_smmu_pm_suspend(struct device *dev) 2334 { 2335 int ret = 0; 2336 struct arm_smmu_device *smmu = dev_get_drvdata(dev); 2337 2338 if (pm_runtime_suspended(dev)) 2339 goto clk_unprepare; 2340 2341 ret = arm_smmu_runtime_suspend(dev); 2342 if (ret) 2343 return ret; 2344 2345 clk_unprepare: 2346 clk_bulk_unprepare(smmu->num_clks, smmu->clks); 2347 return ret; 2348 } 2349 2350 static const struct dev_pm_ops arm_smmu_pm_ops = { 2351 SET_SYSTEM_SLEEP_PM_OPS(arm_smmu_pm_suspend, arm_smmu_pm_resume) 2352 SET_RUNTIME_PM_OPS(arm_smmu_runtime_suspend, 2353 arm_smmu_runtime_resume, NULL) 2354 }; 2355 2356 static struct platform_driver arm_smmu_driver = { 2357 .driver = { 2358 .name = "arm-smmu", 2359 .of_match_table = arm_smmu_of_match, 2360 .pm = &arm_smmu_pm_ops, 2361 .suppress_bind_attrs = true, 2362 }, 2363 .probe = arm_smmu_device_probe, 2364 .remove = arm_smmu_device_remove, 2365 .shutdown = arm_smmu_device_shutdown, 2366 }; 2367 module_platform_driver(arm_smmu_driver); 2368 2369 MODULE_DESCRIPTION("IOMMU API for ARM architected SMMU implementations"); 2370 MODULE_AUTHOR("Will Deacon <will@kernel.org>"); 2371 MODULE_ALIAS("platform:arm-smmu"); 2372 MODULE_LICENSE("GPL v2"); 2373