1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Apple DART (Device Address Resolution Table) IOMMU driver 4 * 5 * Copyright (C) 2021 The Asahi Linux Contributors 6 * 7 * Based on arm/arm-smmu/arm-ssmu.c and arm/arm-smmu-v3/arm-smmu-v3.c 8 * Copyright (C) 2013 ARM Limited 9 * Copyright (C) 2015 ARM Limited 10 * and on exynos-iommu.c 11 * Copyright (c) 2011,2016 Samsung Electronics Co., Ltd. 12 */ 13 14 #include <linux/atomic.h> 15 #include <linux/bitfield.h> 16 #include <linux/clk.h> 17 #include <linux/dev_printk.h> 18 #include <linux/dma-mapping.h> 19 #include <linux/err.h> 20 #include <linux/interrupt.h> 21 #include <linux/io-pgtable.h> 22 #include <linux/iommu.h> 23 #include <linux/iopoll.h> 24 #include <linux/module.h> 25 #include <linux/of.h> 26 #include <linux/of_address.h> 27 #include <linux/of_iommu.h> 28 #include <linux/of_platform.h> 29 #include <linux/pci.h> 30 #include <linux/platform_device.h> 31 #include <linux/slab.h> 32 #include <linux/swab.h> 33 #include <linux/types.h> 34 35 #include "dma-iommu.h" 36 37 #define DART_MAX_STREAMS 256 38 #define DART_MAX_TTBR 4 39 #define MAX_DARTS_PER_DEVICE 3 40 41 /* Common registers */ 42 43 #define DART_PARAMS1 0x00 44 #define DART_PARAMS1_PAGE_SHIFT GENMASK(27, 24) 45 46 #define DART_PARAMS2 0x04 47 #define DART_PARAMS2_BYPASS_SUPPORT BIT(0) 48 49 /* T8020/T6000 registers */ 50 51 #define DART_T8020_STREAM_COMMAND 0x20 52 #define DART_T8020_STREAM_COMMAND_BUSY BIT(2) 53 #define DART_T8020_STREAM_COMMAND_INVALIDATE BIT(20) 54 55 #define DART_T8020_STREAM_SELECT 0x34 56 57 #define DART_T8020_ERROR 0x40 58 #define DART_T8020_ERROR_STREAM GENMASK(27, 24) 59 #define DART_T8020_ERROR_CODE GENMASK(11, 0) 60 #define DART_T8020_ERROR_FLAG BIT(31) 61 62 #define DART_T8020_ERROR_READ_FAULT BIT(4) 63 #define DART_T8020_ERROR_WRITE_FAULT BIT(3) 64 #define DART_T8020_ERROR_NO_PTE BIT(2) 65 #define DART_T8020_ERROR_NO_PMD BIT(1) 66 #define DART_T8020_ERROR_NO_TTBR BIT(0) 67 68 #define DART_T8020_CONFIG 0x60 69 #define DART_T8020_CONFIG_LOCK BIT(15) 70 71 #define DART_STREAM_COMMAND_BUSY_TIMEOUT 100 72 73 #define DART_T8020_ERROR_ADDR_HI 0x54 74 #define DART_T8020_ERROR_ADDR_LO 0x50 75 76 #define DART_T8020_STREAMS_ENABLE 0xfc 77 78 #define DART_T8020_TCR 0x100 79 #define DART_T8020_TCR_TRANSLATE_ENABLE BIT(7) 80 #define DART_T8020_TCR_BYPASS_DART BIT(8) 81 #define DART_T8020_TCR_BYPASS_DAPF BIT(12) 82 83 #define DART_T8020_TTBR 0x200 84 #define DART_T8020_USB4_TTBR 0x400 85 #define DART_T8020_TTBR_VALID BIT(31) 86 #define DART_T8020_TTBR_ADDR_FIELD_SHIFT 0 87 #define DART_T8020_TTBR_SHIFT 12 88 89 /* T8110 registers */ 90 91 #define DART_T8110_PARAMS3 0x08 92 #define DART_T8110_PARAMS3_PA_WIDTH GENMASK(29, 24) 93 #define DART_T8110_PARAMS3_VA_WIDTH GENMASK(21, 16) 94 #define DART_T8110_PARAMS3_VER_MAJ GENMASK(15, 8) 95 #define DART_T8110_PARAMS3_VER_MIN GENMASK(7, 0) 96 97 #define DART_T8110_PARAMS4 0x0c 98 #define DART_T8110_PARAMS4_NUM_CLIENTS GENMASK(24, 16) 99 #define DART_T8110_PARAMS4_NUM_SIDS GENMASK(8, 0) 100 101 #define DART_T8110_TLB_CMD 0x80 102 #define DART_T8110_TLB_CMD_BUSY BIT(31) 103 #define DART_T8110_TLB_CMD_OP GENMASK(10, 8) 104 #define DART_T8110_TLB_CMD_OP_FLUSH_ALL 0 105 #define DART_T8110_TLB_CMD_OP_FLUSH_SID 1 106 #define DART_T8110_TLB_CMD_STREAM GENMASK(7, 0) 107 108 #define DART_T8110_ERROR 0x100 109 #define DART_T8110_ERROR_STREAM GENMASK(27, 20) 110 #define DART_T8110_ERROR_CODE GENMASK(14, 0) 111 #define DART_T8110_ERROR_FLAG BIT(31) 112 113 #define DART_T8110_ERROR_MASK 0x104 114 115 #define DART_T8110_ERROR_READ_FAULT BIT(5) 116 #define DART_T8110_ERROR_WRITE_FAULT BIT(4) 117 #define DART_T8110_ERROR_NO_PTE BIT(3) 118 #define DART_T8110_ERROR_NO_PMD BIT(2) 119 #define DART_T8110_ERROR_NO_PGD BIT(1) 120 #define DART_T8110_ERROR_NO_TTBR BIT(0) 121 122 #define DART_T8110_ERROR_ADDR_LO 0x170 123 #define DART_T8110_ERROR_ADDR_HI 0x174 124 125 #define DART_T8110_PROTECT 0x200 126 #define DART_T8110_UNPROTECT 0x204 127 #define DART_T8110_PROTECT_LOCK 0x208 128 #define DART_T8110_PROTECT_TTBR_TCR BIT(0) 129 130 #define DART_T8110_ENABLE_STREAMS 0xc00 131 #define DART_T8110_DISABLE_STREAMS 0xc20 132 133 #define DART_T8110_TCR 0x1000 134 #define DART_T8110_TCR_REMAP GENMASK(11, 8) 135 #define DART_T8110_TCR_REMAP_EN BIT(7) 136 #define DART_T8110_TCR_BYPASS_DAPF BIT(2) 137 #define DART_T8110_TCR_BYPASS_DART BIT(1) 138 #define DART_T8110_TCR_TRANSLATE_ENABLE BIT(0) 139 140 #define DART_T8110_TTBR 0x1400 141 #define DART_T8110_TTBR_VALID BIT(0) 142 #define DART_T8110_TTBR_ADDR_FIELD_SHIFT 2 143 #define DART_T8110_TTBR_SHIFT 14 144 145 #define DART_TCR(dart, sid) ((dart)->hw->tcr + ((sid) << 2)) 146 147 #define DART_TTBR(dart, sid, idx) ((dart)->hw->ttbr + \ 148 (((dart)->hw->ttbr_count * (sid)) << 2) + \ 149 ((idx) << 2)) 150 151 struct apple_dart_stream_map; 152 153 enum dart_type { 154 DART_T8020, 155 DART_T6000, 156 DART_T8110, 157 }; 158 159 struct apple_dart_hw { 160 enum dart_type type; 161 irqreturn_t (*irq_handler)(int irq, void *dev); 162 int (*invalidate_tlb)(struct apple_dart_stream_map *stream_map); 163 164 u32 oas; 165 enum io_pgtable_fmt fmt; 166 167 int max_sid_count; 168 169 u64 lock; 170 u64 lock_bit; 171 172 u64 error; 173 174 u64 enable_streams; 175 176 u64 tcr; 177 u64 tcr_enabled; 178 u64 tcr_disabled; 179 u64 tcr_bypass; 180 181 u64 ttbr; 182 u64 ttbr_valid; 183 u64 ttbr_addr_field_shift; 184 u64 ttbr_shift; 185 int ttbr_count; 186 }; 187 188 /* 189 * Private structure associated with each DART device. 190 * 191 * @dev: device struct 192 * @hw: SoC-specific hardware data 193 * @regs: mapped MMIO region 194 * @irq: interrupt number, can be shared with other DARTs 195 * @clks: clocks associated with this DART 196 * @num_clks: number of @clks 197 * @lock: lock for hardware operations involving this dart 198 * @pgsize: pagesize supported by this DART 199 * @supports_bypass: indicates if this DART supports bypass mode 200 * @sid2group: maps stream ids to iommu_groups 201 * @iommu: iommu core device 202 */ 203 struct apple_dart { 204 struct device *dev; 205 const struct apple_dart_hw *hw; 206 207 void __iomem *regs; 208 209 int irq; 210 struct clk_bulk_data *clks; 211 int num_clks; 212 213 spinlock_t lock; 214 215 u32 ias; 216 u32 oas; 217 u32 pgsize; 218 u32 num_streams; 219 u32 supports_bypass : 1; 220 221 struct iommu_group *sid2group[DART_MAX_STREAMS]; 222 struct iommu_device iommu; 223 224 u32 save_tcr[DART_MAX_STREAMS]; 225 u32 save_ttbr[DART_MAX_STREAMS][DART_MAX_TTBR]; 226 }; 227 228 /* 229 * Convenience struct to identify streams. 230 * 231 * The normal variant is used inside apple_dart_master_cfg which isn't written 232 * to concurrently. 233 * The atomic variant is used inside apple_dart_domain where we have to guard 234 * against races from potential parallel calls to attach/detach_device. 235 * Note that even inside the atomic variant the apple_dart pointer is not 236 * protected: This pointer is initialized once under the domain init mutex 237 * and never changed again afterwards. Devices with different dart pointers 238 * cannot be attached to the same domain. 239 * 240 * @dart dart pointer 241 * @sid stream id bitmap 242 */ 243 struct apple_dart_stream_map { 244 struct apple_dart *dart; 245 DECLARE_BITMAP(sidmap, DART_MAX_STREAMS); 246 }; 247 struct apple_dart_atomic_stream_map { 248 struct apple_dart *dart; 249 atomic_long_t sidmap[BITS_TO_LONGS(DART_MAX_STREAMS)]; 250 }; 251 252 /* 253 * This structure is attached to each iommu domain handled by a DART. 254 * 255 * @pgtbl_ops: pagetable ops allocated by io-pgtable 256 * @finalized: true if the domain has been completely initialized 257 * @init_lock: protects domain initialization 258 * @stream_maps: streams attached to this domain (valid for DMA/UNMANAGED only) 259 * @domain: core iommu domain pointer 260 */ 261 struct apple_dart_domain { 262 struct io_pgtable_ops *pgtbl_ops; 263 264 bool finalized; 265 struct mutex init_lock; 266 struct apple_dart_atomic_stream_map stream_maps[MAX_DARTS_PER_DEVICE]; 267 268 struct iommu_domain domain; 269 }; 270 271 /* 272 * This structure is attached to devices with dev_iommu_priv_set() on of_xlate 273 * and contains a list of streams bound to this device. 274 * So far the worst case seen is a single device with two streams 275 * from different darts, such that this simple static array is enough. 276 * 277 * @streams: streams for this device 278 */ 279 struct apple_dart_master_cfg { 280 /* Intersection of DART capabilitles */ 281 u32 supports_bypass : 1; 282 283 struct apple_dart_stream_map stream_maps[MAX_DARTS_PER_DEVICE]; 284 }; 285 286 /* 287 * Helper macro to iterate over apple_dart_master_cfg.stream_maps and 288 * apple_dart_domain.stream_maps 289 * 290 * @i int used as loop variable 291 * @base pointer to base struct (apple_dart_master_cfg or apple_dart_domain) 292 * @stream pointer to the apple_dart_streams struct for each loop iteration 293 */ 294 #define for_each_stream_map(i, base, stream_map) \ 295 for (i = 0, stream_map = &(base)->stream_maps[0]; \ 296 i < MAX_DARTS_PER_DEVICE && stream_map->dart; \ 297 stream_map = &(base)->stream_maps[++i]) 298 299 static struct platform_driver apple_dart_driver; 300 static const struct iommu_ops apple_dart_iommu_ops; 301 302 static struct apple_dart_domain *to_dart_domain(struct iommu_domain *dom) 303 { 304 return container_of(dom, struct apple_dart_domain, domain); 305 } 306 307 static void 308 apple_dart_hw_enable_translation(struct apple_dart_stream_map *stream_map) 309 { 310 struct apple_dart *dart = stream_map->dart; 311 int sid; 312 313 for_each_set_bit(sid, stream_map->sidmap, dart->num_streams) 314 writel(dart->hw->tcr_enabled, dart->regs + DART_TCR(dart, sid)); 315 } 316 317 static void apple_dart_hw_disable_dma(struct apple_dart_stream_map *stream_map) 318 { 319 struct apple_dart *dart = stream_map->dart; 320 int sid; 321 322 for_each_set_bit(sid, stream_map->sidmap, dart->num_streams) 323 writel(dart->hw->tcr_disabled, dart->regs + DART_TCR(dart, sid)); 324 } 325 326 static void 327 apple_dart_hw_enable_bypass(struct apple_dart_stream_map *stream_map) 328 { 329 struct apple_dart *dart = stream_map->dart; 330 int sid; 331 332 WARN_ON(!stream_map->dart->supports_bypass); 333 for_each_set_bit(sid, stream_map->sidmap, dart->num_streams) 334 writel(dart->hw->tcr_bypass, 335 dart->regs + DART_TCR(dart, sid)); 336 } 337 338 static void apple_dart_hw_set_ttbr(struct apple_dart_stream_map *stream_map, 339 u8 idx, phys_addr_t paddr) 340 { 341 struct apple_dart *dart = stream_map->dart; 342 int sid; 343 344 WARN_ON(paddr & ((1 << dart->hw->ttbr_shift) - 1)); 345 for_each_set_bit(sid, stream_map->sidmap, dart->num_streams) 346 writel(dart->hw->ttbr_valid | 347 (paddr >> dart->hw->ttbr_shift) << dart->hw->ttbr_addr_field_shift, 348 dart->regs + DART_TTBR(dart, sid, idx)); 349 } 350 351 static void apple_dart_hw_clear_ttbr(struct apple_dart_stream_map *stream_map, 352 u8 idx) 353 { 354 struct apple_dart *dart = stream_map->dart; 355 int sid; 356 357 for_each_set_bit(sid, stream_map->sidmap, dart->num_streams) 358 writel(0, dart->regs + DART_TTBR(dart, sid, idx)); 359 } 360 361 static void 362 apple_dart_hw_clear_all_ttbrs(struct apple_dart_stream_map *stream_map) 363 { 364 int i; 365 366 for (i = 0; i < stream_map->dart->hw->ttbr_count; ++i) 367 apple_dart_hw_clear_ttbr(stream_map, i); 368 } 369 370 static int 371 apple_dart_t8020_hw_stream_command(struct apple_dart_stream_map *stream_map, 372 u32 command) 373 { 374 unsigned long flags; 375 int ret, i; 376 u32 command_reg; 377 378 spin_lock_irqsave(&stream_map->dart->lock, flags); 379 380 for (i = 0; i < BITS_TO_U32(stream_map->dart->num_streams); i++) 381 writel(stream_map->sidmap[i], 382 stream_map->dart->regs + DART_T8020_STREAM_SELECT + 4 * i); 383 writel(command, stream_map->dart->regs + DART_T8020_STREAM_COMMAND); 384 385 ret = readl_poll_timeout_atomic( 386 stream_map->dart->regs + DART_T8020_STREAM_COMMAND, command_reg, 387 !(command_reg & DART_T8020_STREAM_COMMAND_BUSY), 1, 388 DART_STREAM_COMMAND_BUSY_TIMEOUT); 389 390 spin_unlock_irqrestore(&stream_map->dart->lock, flags); 391 392 if (ret) { 393 dev_err(stream_map->dart->dev, 394 "busy bit did not clear after command %x for streams %lx\n", 395 command, stream_map->sidmap[0]); 396 return ret; 397 } 398 399 return 0; 400 } 401 402 static int 403 apple_dart_t8110_hw_tlb_command(struct apple_dart_stream_map *stream_map, 404 u32 command) 405 { 406 struct apple_dart *dart = stream_map->dart; 407 unsigned long flags; 408 int ret = 0; 409 int sid; 410 411 spin_lock_irqsave(&dart->lock, flags); 412 413 for_each_set_bit(sid, stream_map->sidmap, dart->num_streams) { 414 u32 val = FIELD_PREP(DART_T8110_TLB_CMD_OP, command) | 415 FIELD_PREP(DART_T8110_TLB_CMD_STREAM, sid); 416 writel(val, dart->regs + DART_T8110_TLB_CMD); 417 418 ret = readl_poll_timeout_atomic( 419 dart->regs + DART_T8110_TLB_CMD, val, 420 !(val & DART_T8110_TLB_CMD_BUSY), 1, 421 DART_STREAM_COMMAND_BUSY_TIMEOUT); 422 423 if (ret) 424 break; 425 426 } 427 428 spin_unlock_irqrestore(&dart->lock, flags); 429 430 if (ret) { 431 dev_err(stream_map->dart->dev, 432 "busy bit did not clear after command %x for stream %d\n", 433 command, sid); 434 return ret; 435 } 436 437 return 0; 438 } 439 440 static int 441 apple_dart_t8020_hw_invalidate_tlb(struct apple_dart_stream_map *stream_map) 442 { 443 return apple_dart_t8020_hw_stream_command( 444 stream_map, DART_T8020_STREAM_COMMAND_INVALIDATE); 445 } 446 447 static int 448 apple_dart_t8110_hw_invalidate_tlb(struct apple_dart_stream_map *stream_map) 449 { 450 return apple_dart_t8110_hw_tlb_command( 451 stream_map, DART_T8110_TLB_CMD_OP_FLUSH_SID); 452 } 453 454 static int apple_dart_hw_reset(struct apple_dart *dart) 455 { 456 u32 config; 457 struct apple_dart_stream_map stream_map; 458 int i; 459 460 config = readl(dart->regs + dart->hw->lock); 461 if (config & dart->hw->lock_bit) { 462 dev_err(dart->dev, "DART is locked down until reboot: %08x\n", 463 config); 464 return -EINVAL; 465 } 466 467 stream_map.dart = dart; 468 bitmap_zero(stream_map.sidmap, DART_MAX_STREAMS); 469 bitmap_set(stream_map.sidmap, 0, dart->num_streams); 470 apple_dart_hw_disable_dma(&stream_map); 471 apple_dart_hw_clear_all_ttbrs(&stream_map); 472 473 /* enable all streams globally since TCR is used to control isolation */ 474 for (i = 0; i < BITS_TO_U32(dart->num_streams); i++) 475 writel(U32_MAX, dart->regs + dart->hw->enable_streams + 4 * i); 476 477 /* clear any pending errors before the interrupt is unmasked */ 478 writel(readl(dart->regs + dart->hw->error), dart->regs + dart->hw->error); 479 480 if (dart->hw->type == DART_T8110) 481 writel(0, dart->regs + DART_T8110_ERROR_MASK); 482 483 return dart->hw->invalidate_tlb(&stream_map); 484 } 485 486 static void apple_dart_domain_flush_tlb(struct apple_dart_domain *domain) 487 { 488 int i, j; 489 struct apple_dart_atomic_stream_map *domain_stream_map; 490 struct apple_dart_stream_map stream_map; 491 492 for_each_stream_map(i, domain, domain_stream_map) { 493 stream_map.dart = domain_stream_map->dart; 494 495 for (j = 0; j < BITS_TO_LONGS(stream_map.dart->num_streams); j++) 496 stream_map.sidmap[j] = atomic_long_read(&domain_stream_map->sidmap[j]); 497 498 stream_map.dart->hw->invalidate_tlb(&stream_map); 499 } 500 } 501 502 static void apple_dart_flush_iotlb_all(struct iommu_domain *domain) 503 { 504 apple_dart_domain_flush_tlb(to_dart_domain(domain)); 505 } 506 507 static void apple_dart_iotlb_sync(struct iommu_domain *domain, 508 struct iommu_iotlb_gather *gather) 509 { 510 apple_dart_domain_flush_tlb(to_dart_domain(domain)); 511 } 512 513 static int apple_dart_iotlb_sync_map(struct iommu_domain *domain, 514 unsigned long iova, size_t size) 515 { 516 apple_dart_domain_flush_tlb(to_dart_domain(domain)); 517 return 0; 518 } 519 520 static phys_addr_t apple_dart_iova_to_phys(struct iommu_domain *domain, 521 dma_addr_t iova) 522 { 523 struct apple_dart_domain *dart_domain = to_dart_domain(domain); 524 struct io_pgtable_ops *ops = dart_domain->pgtbl_ops; 525 526 if (!ops) 527 return 0; 528 529 return ops->iova_to_phys(ops, iova); 530 } 531 532 static int apple_dart_map_pages(struct iommu_domain *domain, unsigned long iova, 533 phys_addr_t paddr, size_t pgsize, 534 size_t pgcount, int prot, gfp_t gfp, 535 size_t *mapped) 536 { 537 struct apple_dart_domain *dart_domain = to_dart_domain(domain); 538 struct io_pgtable_ops *ops = dart_domain->pgtbl_ops; 539 540 if (!ops) 541 return -ENODEV; 542 543 return ops->map_pages(ops, iova, paddr, pgsize, pgcount, prot, gfp, 544 mapped); 545 } 546 547 static size_t apple_dart_unmap_pages(struct iommu_domain *domain, 548 unsigned long iova, size_t pgsize, 549 size_t pgcount, 550 struct iommu_iotlb_gather *gather) 551 { 552 struct apple_dart_domain *dart_domain = to_dart_domain(domain); 553 struct io_pgtable_ops *ops = dart_domain->pgtbl_ops; 554 555 return ops->unmap_pages(ops, iova, pgsize, pgcount, gather); 556 } 557 558 static void 559 apple_dart_setup_translation(struct apple_dart_domain *domain, 560 struct apple_dart_stream_map *stream_map) 561 { 562 int i; 563 struct io_pgtable_cfg *pgtbl_cfg = 564 &io_pgtable_ops_to_pgtable(domain->pgtbl_ops)->cfg; 565 566 for (i = 0; i < pgtbl_cfg->apple_dart_cfg.n_ttbrs; ++i) 567 apple_dart_hw_set_ttbr(stream_map, i, 568 pgtbl_cfg->apple_dart_cfg.ttbr[i]); 569 for (; i < stream_map->dart->hw->ttbr_count; ++i) 570 apple_dart_hw_clear_ttbr(stream_map, i); 571 572 apple_dart_hw_enable_translation(stream_map); 573 stream_map->dart->hw->invalidate_tlb(stream_map); 574 } 575 576 static int apple_dart_finalize_domain(struct apple_dart_domain *dart_domain, 577 struct apple_dart_master_cfg *cfg) 578 { 579 struct apple_dart *dart = cfg->stream_maps[0].dart; 580 struct io_pgtable_cfg pgtbl_cfg; 581 int ret = 0; 582 int i, j; 583 584 if (dart->pgsize > PAGE_SIZE) 585 return -EINVAL; 586 587 mutex_lock(&dart_domain->init_lock); 588 589 if (dart_domain->finalized) 590 goto done; 591 592 for (i = 0; i < MAX_DARTS_PER_DEVICE; ++i) { 593 dart_domain->stream_maps[i].dart = cfg->stream_maps[i].dart; 594 for (j = 0; j < BITS_TO_LONGS(dart->num_streams); j++) 595 atomic_long_set(&dart_domain->stream_maps[i].sidmap[j], 596 cfg->stream_maps[i].sidmap[j]); 597 } 598 599 pgtbl_cfg = (struct io_pgtable_cfg){ 600 .pgsize_bitmap = dart->pgsize, 601 .ias = dart->ias, 602 .oas = dart->oas, 603 .coherent_walk = 1, 604 .iommu_dev = dart->dev, 605 }; 606 607 dart_domain->pgtbl_ops = alloc_io_pgtable_ops(dart->hw->fmt, &pgtbl_cfg, 608 &dart_domain->domain); 609 if (!dart_domain->pgtbl_ops) { 610 ret = -ENOMEM; 611 goto done; 612 } 613 614 dart_domain->domain.pgsize_bitmap = pgtbl_cfg.pgsize_bitmap; 615 dart_domain->domain.geometry.aperture_start = 0; 616 dart_domain->domain.geometry.aperture_end = 617 (dma_addr_t)DMA_BIT_MASK(dart->ias); 618 dart_domain->domain.geometry.force_aperture = true; 619 620 dart_domain->finalized = true; 621 622 done: 623 mutex_unlock(&dart_domain->init_lock); 624 return ret; 625 } 626 627 static int 628 apple_dart_mod_streams(struct apple_dart_atomic_stream_map *domain_maps, 629 struct apple_dart_stream_map *master_maps, 630 bool add_streams) 631 { 632 int i, j; 633 634 for (i = 0; i < MAX_DARTS_PER_DEVICE; ++i) { 635 if (domain_maps[i].dart != master_maps[i].dart) 636 return -EINVAL; 637 } 638 639 for (i = 0; i < MAX_DARTS_PER_DEVICE; ++i) { 640 if (!domain_maps[i].dart) 641 break; 642 for (j = 0; j < BITS_TO_LONGS(domain_maps[i].dart->num_streams); j++) { 643 if (add_streams) 644 atomic_long_or(master_maps[i].sidmap[j], 645 &domain_maps[i].sidmap[j]); 646 else 647 atomic_long_and(~master_maps[i].sidmap[j], 648 &domain_maps[i].sidmap[j]); 649 } 650 } 651 652 return 0; 653 } 654 655 static int apple_dart_domain_add_streams(struct apple_dart_domain *domain, 656 struct apple_dart_master_cfg *cfg) 657 { 658 return apple_dart_mod_streams(domain->stream_maps, cfg->stream_maps, 659 true); 660 } 661 662 static int apple_dart_attach_dev_paging(struct iommu_domain *domain, 663 struct device *dev) 664 { 665 int ret, i; 666 struct apple_dart_stream_map *stream_map; 667 struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev); 668 struct apple_dart_domain *dart_domain = to_dart_domain(domain); 669 670 ret = apple_dart_finalize_domain(dart_domain, cfg); 671 if (ret) 672 return ret; 673 674 ret = apple_dart_domain_add_streams(dart_domain, cfg); 675 if (ret) 676 return ret; 677 678 for_each_stream_map(i, cfg, stream_map) 679 apple_dart_setup_translation(dart_domain, stream_map); 680 return 0; 681 } 682 683 static int apple_dart_attach_dev_identity(struct iommu_domain *domain, 684 struct device *dev) 685 { 686 struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev); 687 struct apple_dart_stream_map *stream_map; 688 int i; 689 690 if (!cfg->supports_bypass) 691 return -EINVAL; 692 693 for_each_stream_map(i, cfg, stream_map) 694 apple_dart_hw_enable_bypass(stream_map); 695 return 0; 696 } 697 698 static const struct iommu_domain_ops apple_dart_identity_ops = { 699 .attach_dev = apple_dart_attach_dev_identity, 700 }; 701 702 static struct iommu_domain apple_dart_identity_domain = { 703 .type = IOMMU_DOMAIN_IDENTITY, 704 .ops = &apple_dart_identity_ops, 705 }; 706 707 static int apple_dart_attach_dev_blocked(struct iommu_domain *domain, 708 struct device *dev) 709 { 710 struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev); 711 struct apple_dart_stream_map *stream_map; 712 int i; 713 714 for_each_stream_map(i, cfg, stream_map) 715 apple_dart_hw_disable_dma(stream_map); 716 return 0; 717 } 718 719 static const struct iommu_domain_ops apple_dart_blocked_ops = { 720 .attach_dev = apple_dart_attach_dev_blocked, 721 }; 722 723 static struct iommu_domain apple_dart_blocked_domain = { 724 .type = IOMMU_DOMAIN_BLOCKED, 725 .ops = &apple_dart_blocked_ops, 726 }; 727 728 static struct iommu_device *apple_dart_probe_device(struct device *dev) 729 { 730 struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev); 731 struct apple_dart_stream_map *stream_map; 732 int i; 733 734 if (!cfg) 735 return ERR_PTR(-ENODEV); 736 737 for_each_stream_map(i, cfg, stream_map) 738 device_link_add( 739 dev, stream_map->dart->dev, 740 DL_FLAG_PM_RUNTIME | DL_FLAG_AUTOREMOVE_SUPPLIER); 741 742 return &cfg->stream_maps[0].dart->iommu; 743 } 744 745 static void apple_dart_release_device(struct device *dev) 746 { 747 struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev); 748 749 kfree(cfg); 750 } 751 752 static struct iommu_domain *apple_dart_domain_alloc_paging(struct device *dev) 753 { 754 struct apple_dart_domain *dart_domain; 755 756 dart_domain = kzalloc(sizeof(*dart_domain), GFP_KERNEL); 757 if (!dart_domain) 758 return NULL; 759 760 mutex_init(&dart_domain->init_lock); 761 762 if (dev) { 763 struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev); 764 int ret; 765 766 ret = apple_dart_finalize_domain(dart_domain, cfg); 767 if (ret) { 768 kfree(dart_domain); 769 return ERR_PTR(ret); 770 } 771 } 772 return &dart_domain->domain; 773 } 774 775 static void apple_dart_domain_free(struct iommu_domain *domain) 776 { 777 struct apple_dart_domain *dart_domain = to_dart_domain(domain); 778 779 free_io_pgtable_ops(dart_domain->pgtbl_ops); 780 781 kfree(dart_domain); 782 } 783 784 static int apple_dart_of_xlate(struct device *dev, 785 const struct of_phandle_args *args) 786 { 787 struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev); 788 struct platform_device *iommu_pdev = of_find_device_by_node(args->np); 789 struct apple_dart *dart = platform_get_drvdata(iommu_pdev); 790 struct apple_dart *cfg_dart; 791 int i, sid; 792 793 if (args->args_count != 1) 794 return -EINVAL; 795 sid = args->args[0]; 796 797 if (!cfg) { 798 cfg = kzalloc(sizeof(*cfg), GFP_KERNEL); 799 if (!cfg) 800 return -ENOMEM; 801 /* Will be ANDed with DART capabilities */ 802 cfg->supports_bypass = true; 803 } 804 dev_iommu_priv_set(dev, cfg); 805 806 cfg_dart = cfg->stream_maps[0].dart; 807 if (cfg_dart) { 808 if (cfg_dart->pgsize != dart->pgsize) 809 return -EINVAL; 810 } 811 812 cfg->supports_bypass &= dart->supports_bypass; 813 814 for (i = 0; i < MAX_DARTS_PER_DEVICE; ++i) { 815 if (cfg->stream_maps[i].dart == dart) { 816 set_bit(sid, cfg->stream_maps[i].sidmap); 817 return 0; 818 } 819 } 820 for (i = 0; i < MAX_DARTS_PER_DEVICE; ++i) { 821 if (!cfg->stream_maps[i].dart) { 822 cfg->stream_maps[i].dart = dart; 823 set_bit(sid, cfg->stream_maps[i].sidmap); 824 return 0; 825 } 826 } 827 828 return -EINVAL; 829 } 830 831 static DEFINE_MUTEX(apple_dart_groups_lock); 832 833 static void apple_dart_release_group(void *iommu_data) 834 { 835 int i, sid; 836 struct apple_dart_stream_map *stream_map; 837 struct apple_dart_master_cfg *group_master_cfg = iommu_data; 838 839 mutex_lock(&apple_dart_groups_lock); 840 841 for_each_stream_map(i, group_master_cfg, stream_map) 842 for_each_set_bit(sid, stream_map->sidmap, stream_map->dart->num_streams) 843 stream_map->dart->sid2group[sid] = NULL; 844 845 kfree(iommu_data); 846 mutex_unlock(&apple_dart_groups_lock); 847 } 848 849 static int apple_dart_merge_master_cfg(struct apple_dart_master_cfg *dst, 850 struct apple_dart_master_cfg *src) 851 { 852 /* 853 * We know that this function is only called for groups returned from 854 * pci_device_group and that all Apple Silicon platforms never spread 855 * PCIe devices from the same bus across multiple DARTs such that we can 856 * just assume that both src and dst only have the same single DART. 857 */ 858 if (src->stream_maps[1].dart) 859 return -EINVAL; 860 if (dst->stream_maps[1].dart) 861 return -EINVAL; 862 if (src->stream_maps[0].dart != dst->stream_maps[0].dart) 863 return -EINVAL; 864 865 bitmap_or(dst->stream_maps[0].sidmap, 866 dst->stream_maps[0].sidmap, 867 src->stream_maps[0].sidmap, 868 dst->stream_maps[0].dart->num_streams); 869 return 0; 870 } 871 872 static struct iommu_group *apple_dart_device_group(struct device *dev) 873 { 874 int i, sid; 875 struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev); 876 struct apple_dart_stream_map *stream_map; 877 struct apple_dart_master_cfg *group_master_cfg; 878 struct iommu_group *group = NULL; 879 struct iommu_group *res = ERR_PTR(-EINVAL); 880 881 mutex_lock(&apple_dart_groups_lock); 882 883 for_each_stream_map(i, cfg, stream_map) { 884 for_each_set_bit(sid, stream_map->sidmap, stream_map->dart->num_streams) { 885 struct iommu_group *stream_group = 886 stream_map->dart->sid2group[sid]; 887 888 if (group && group != stream_group) { 889 res = ERR_PTR(-EINVAL); 890 goto out; 891 } 892 893 group = stream_group; 894 } 895 } 896 897 if (group) { 898 res = iommu_group_ref_get(group); 899 goto out; 900 } 901 902 #ifdef CONFIG_PCI 903 if (dev_is_pci(dev)) 904 group = pci_device_group(dev); 905 else 906 #endif 907 group = generic_device_group(dev); 908 909 res = ERR_PTR(-ENOMEM); 910 if (!group) 911 goto out; 912 913 group_master_cfg = iommu_group_get_iommudata(group); 914 if (group_master_cfg) { 915 int ret; 916 917 ret = apple_dart_merge_master_cfg(group_master_cfg, cfg); 918 if (ret) { 919 dev_err(dev, "Failed to merge DART IOMMU groups.\n"); 920 iommu_group_put(group); 921 res = ERR_PTR(ret); 922 goto out; 923 } 924 } else { 925 group_master_cfg = kmemdup(cfg, sizeof(*group_master_cfg), 926 GFP_KERNEL); 927 if (!group_master_cfg) { 928 iommu_group_put(group); 929 goto out; 930 } 931 932 iommu_group_set_iommudata(group, group_master_cfg, 933 apple_dart_release_group); 934 } 935 936 for_each_stream_map(i, cfg, stream_map) 937 for_each_set_bit(sid, stream_map->sidmap, stream_map->dart->num_streams) 938 stream_map->dart->sid2group[sid] = group; 939 940 res = group; 941 942 out: 943 mutex_unlock(&apple_dart_groups_lock); 944 return res; 945 } 946 947 static int apple_dart_def_domain_type(struct device *dev) 948 { 949 struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev); 950 951 if (cfg->stream_maps[0].dart->pgsize > PAGE_SIZE) 952 return IOMMU_DOMAIN_IDENTITY; 953 if (!cfg->supports_bypass) 954 return IOMMU_DOMAIN_DMA; 955 956 return 0; 957 } 958 959 #ifndef CONFIG_PCIE_APPLE_MSI_DOORBELL_ADDR 960 /* Keep things compiling when CONFIG_PCI_APPLE isn't selected */ 961 #define CONFIG_PCIE_APPLE_MSI_DOORBELL_ADDR 0 962 #endif 963 #define DOORBELL_ADDR (CONFIG_PCIE_APPLE_MSI_DOORBELL_ADDR & PAGE_MASK) 964 965 static void apple_dart_get_resv_regions(struct device *dev, 966 struct list_head *head) 967 { 968 if (IS_ENABLED(CONFIG_PCIE_APPLE) && dev_is_pci(dev)) { 969 struct iommu_resv_region *region; 970 int prot = IOMMU_WRITE | IOMMU_NOEXEC | IOMMU_MMIO; 971 972 region = iommu_alloc_resv_region(DOORBELL_ADDR, 973 PAGE_SIZE, prot, 974 IOMMU_RESV_MSI, GFP_KERNEL); 975 if (!region) 976 return; 977 978 list_add_tail(®ion->list, head); 979 } 980 981 iommu_dma_get_resv_regions(dev, head); 982 } 983 984 static const struct iommu_ops apple_dart_iommu_ops = { 985 .identity_domain = &apple_dart_identity_domain, 986 .blocked_domain = &apple_dart_blocked_domain, 987 .domain_alloc_paging = apple_dart_domain_alloc_paging, 988 .probe_device = apple_dart_probe_device, 989 .release_device = apple_dart_release_device, 990 .device_group = apple_dart_device_group, 991 .of_xlate = apple_dart_of_xlate, 992 .def_domain_type = apple_dart_def_domain_type, 993 .get_resv_regions = apple_dart_get_resv_regions, 994 .pgsize_bitmap = -1UL, /* Restricted during dart probe */ 995 .owner = THIS_MODULE, 996 .default_domain_ops = &(const struct iommu_domain_ops) { 997 .attach_dev = apple_dart_attach_dev_paging, 998 .map_pages = apple_dart_map_pages, 999 .unmap_pages = apple_dart_unmap_pages, 1000 .flush_iotlb_all = apple_dart_flush_iotlb_all, 1001 .iotlb_sync = apple_dart_iotlb_sync, 1002 .iotlb_sync_map = apple_dart_iotlb_sync_map, 1003 .iova_to_phys = apple_dart_iova_to_phys, 1004 .free = apple_dart_domain_free, 1005 } 1006 }; 1007 1008 static irqreturn_t apple_dart_t8020_irq(int irq, void *dev) 1009 { 1010 struct apple_dart *dart = dev; 1011 const char *fault_name = NULL; 1012 u32 error = readl(dart->regs + DART_T8020_ERROR); 1013 u32 error_code = FIELD_GET(DART_T8020_ERROR_CODE, error); 1014 u32 addr_lo = readl(dart->regs + DART_T8020_ERROR_ADDR_LO); 1015 u32 addr_hi = readl(dart->regs + DART_T8020_ERROR_ADDR_HI); 1016 u64 addr = addr_lo | (((u64)addr_hi) << 32); 1017 u8 stream_idx = FIELD_GET(DART_T8020_ERROR_STREAM, error); 1018 1019 if (!(error & DART_T8020_ERROR_FLAG)) 1020 return IRQ_NONE; 1021 1022 /* there should only be a single bit set but let's use == to be sure */ 1023 if (error_code == DART_T8020_ERROR_READ_FAULT) 1024 fault_name = "READ FAULT"; 1025 else if (error_code == DART_T8020_ERROR_WRITE_FAULT) 1026 fault_name = "WRITE FAULT"; 1027 else if (error_code == DART_T8020_ERROR_NO_PTE) 1028 fault_name = "NO PTE FOR IOVA"; 1029 else if (error_code == DART_T8020_ERROR_NO_PMD) 1030 fault_name = "NO PMD FOR IOVA"; 1031 else if (error_code == DART_T8020_ERROR_NO_TTBR) 1032 fault_name = "NO TTBR FOR IOVA"; 1033 else 1034 fault_name = "unknown"; 1035 1036 dev_err_ratelimited( 1037 dart->dev, 1038 "translation fault: status:0x%x stream:%d code:0x%x (%s) at 0x%llx", 1039 error, stream_idx, error_code, fault_name, addr); 1040 1041 writel(error, dart->regs + DART_T8020_ERROR); 1042 return IRQ_HANDLED; 1043 } 1044 1045 static irqreturn_t apple_dart_t8110_irq(int irq, void *dev) 1046 { 1047 struct apple_dart *dart = dev; 1048 const char *fault_name = NULL; 1049 u32 error = readl(dart->regs + DART_T8110_ERROR); 1050 u32 error_code = FIELD_GET(DART_T8110_ERROR_CODE, error); 1051 u32 addr_lo = readl(dart->regs + DART_T8110_ERROR_ADDR_LO); 1052 u32 addr_hi = readl(dart->regs + DART_T8110_ERROR_ADDR_HI); 1053 u64 addr = addr_lo | (((u64)addr_hi) << 32); 1054 u8 stream_idx = FIELD_GET(DART_T8110_ERROR_STREAM, error); 1055 1056 if (!(error & DART_T8110_ERROR_FLAG)) 1057 return IRQ_NONE; 1058 1059 /* there should only be a single bit set but let's use == to be sure */ 1060 if (error_code == DART_T8110_ERROR_READ_FAULT) 1061 fault_name = "READ FAULT"; 1062 else if (error_code == DART_T8110_ERROR_WRITE_FAULT) 1063 fault_name = "WRITE FAULT"; 1064 else if (error_code == DART_T8110_ERROR_NO_PTE) 1065 fault_name = "NO PTE FOR IOVA"; 1066 else if (error_code == DART_T8110_ERROR_NO_PMD) 1067 fault_name = "NO PMD FOR IOVA"; 1068 else if (error_code == DART_T8110_ERROR_NO_PGD) 1069 fault_name = "NO PGD FOR IOVA"; 1070 else if (error_code == DART_T8110_ERROR_NO_TTBR) 1071 fault_name = "NO TTBR FOR IOVA"; 1072 else 1073 fault_name = "unknown"; 1074 1075 dev_err_ratelimited( 1076 dart->dev, 1077 "translation fault: status:0x%x stream:%d code:0x%x (%s) at 0x%llx", 1078 error, stream_idx, error_code, fault_name, addr); 1079 1080 writel(error, dart->regs + DART_T8110_ERROR); 1081 return IRQ_HANDLED; 1082 } 1083 1084 static int apple_dart_probe(struct platform_device *pdev) 1085 { 1086 int ret; 1087 u32 dart_params[4]; 1088 struct resource *res; 1089 struct apple_dart *dart; 1090 struct device *dev = &pdev->dev; 1091 1092 dart = devm_kzalloc(dev, sizeof(*dart), GFP_KERNEL); 1093 if (!dart) 1094 return -ENOMEM; 1095 1096 dart->dev = dev; 1097 dart->hw = of_device_get_match_data(dev); 1098 spin_lock_init(&dart->lock); 1099 1100 dart->regs = devm_platform_get_and_ioremap_resource(pdev, 0, &res); 1101 if (IS_ERR(dart->regs)) 1102 return PTR_ERR(dart->regs); 1103 1104 if (resource_size(res) < 0x4000) { 1105 dev_err(dev, "MMIO region too small (%pr)\n", res); 1106 return -EINVAL; 1107 } 1108 1109 dart->irq = platform_get_irq(pdev, 0); 1110 if (dart->irq < 0) 1111 return -ENODEV; 1112 1113 ret = devm_clk_bulk_get_all(dev, &dart->clks); 1114 if (ret < 0) 1115 return ret; 1116 dart->num_clks = ret; 1117 1118 ret = clk_bulk_prepare_enable(dart->num_clks, dart->clks); 1119 if (ret) 1120 return ret; 1121 1122 dart_params[0] = readl(dart->regs + DART_PARAMS1); 1123 dart_params[1] = readl(dart->regs + DART_PARAMS2); 1124 dart->pgsize = 1 << FIELD_GET(DART_PARAMS1_PAGE_SHIFT, dart_params[0]); 1125 dart->supports_bypass = dart_params[1] & DART_PARAMS2_BYPASS_SUPPORT; 1126 1127 switch (dart->hw->type) { 1128 case DART_T8020: 1129 case DART_T6000: 1130 dart->ias = 32; 1131 dart->oas = dart->hw->oas; 1132 dart->num_streams = dart->hw->max_sid_count; 1133 break; 1134 1135 case DART_T8110: 1136 dart_params[2] = readl(dart->regs + DART_T8110_PARAMS3); 1137 dart_params[3] = readl(dart->regs + DART_T8110_PARAMS4); 1138 dart->ias = FIELD_GET(DART_T8110_PARAMS3_VA_WIDTH, dart_params[2]); 1139 dart->oas = FIELD_GET(DART_T8110_PARAMS3_PA_WIDTH, dart_params[2]); 1140 dart->num_streams = FIELD_GET(DART_T8110_PARAMS4_NUM_SIDS, dart_params[3]); 1141 break; 1142 } 1143 1144 if (dart->num_streams > DART_MAX_STREAMS) { 1145 dev_err(&pdev->dev, "Too many streams (%d > %d)\n", 1146 dart->num_streams, DART_MAX_STREAMS); 1147 ret = -EINVAL; 1148 goto err_clk_disable; 1149 } 1150 1151 ret = apple_dart_hw_reset(dart); 1152 if (ret) 1153 goto err_clk_disable; 1154 1155 ret = request_irq(dart->irq, dart->hw->irq_handler, IRQF_SHARED, 1156 "apple-dart fault handler", dart); 1157 if (ret) 1158 goto err_clk_disable; 1159 1160 platform_set_drvdata(pdev, dart); 1161 1162 ret = iommu_device_sysfs_add(&dart->iommu, dev, NULL, "apple-dart.%s", 1163 dev_name(&pdev->dev)); 1164 if (ret) 1165 goto err_free_irq; 1166 1167 ret = iommu_device_register(&dart->iommu, &apple_dart_iommu_ops, dev); 1168 if (ret) 1169 goto err_sysfs_remove; 1170 1171 dev_info( 1172 &pdev->dev, 1173 "DART [pagesize %x, %d streams, bypass support: %d, bypass forced: %d] initialized\n", 1174 dart->pgsize, dart->num_streams, dart->supports_bypass, 1175 dart->pgsize > PAGE_SIZE); 1176 return 0; 1177 1178 err_sysfs_remove: 1179 iommu_device_sysfs_remove(&dart->iommu); 1180 err_free_irq: 1181 free_irq(dart->irq, dart); 1182 err_clk_disable: 1183 clk_bulk_disable_unprepare(dart->num_clks, dart->clks); 1184 1185 return ret; 1186 } 1187 1188 static void apple_dart_remove(struct platform_device *pdev) 1189 { 1190 struct apple_dart *dart = platform_get_drvdata(pdev); 1191 1192 apple_dart_hw_reset(dart); 1193 free_irq(dart->irq, dart); 1194 1195 iommu_device_unregister(&dart->iommu); 1196 iommu_device_sysfs_remove(&dart->iommu); 1197 1198 clk_bulk_disable_unprepare(dart->num_clks, dart->clks); 1199 } 1200 1201 static const struct apple_dart_hw apple_dart_hw_t8103 = { 1202 .type = DART_T8020, 1203 .irq_handler = apple_dart_t8020_irq, 1204 .invalidate_tlb = apple_dart_t8020_hw_invalidate_tlb, 1205 .oas = 36, 1206 .fmt = APPLE_DART, 1207 .max_sid_count = 16, 1208 1209 .enable_streams = DART_T8020_STREAMS_ENABLE, 1210 .lock = DART_T8020_CONFIG, 1211 .lock_bit = DART_T8020_CONFIG_LOCK, 1212 1213 .error = DART_T8020_ERROR, 1214 1215 .tcr = DART_T8020_TCR, 1216 .tcr_enabled = DART_T8020_TCR_TRANSLATE_ENABLE, 1217 .tcr_disabled = 0, 1218 .tcr_bypass = DART_T8020_TCR_BYPASS_DAPF | DART_T8020_TCR_BYPASS_DART, 1219 1220 .ttbr = DART_T8020_TTBR, 1221 .ttbr_valid = DART_T8020_TTBR_VALID, 1222 .ttbr_addr_field_shift = DART_T8020_TTBR_ADDR_FIELD_SHIFT, 1223 .ttbr_shift = DART_T8020_TTBR_SHIFT, 1224 .ttbr_count = 4, 1225 }; 1226 1227 static const struct apple_dart_hw apple_dart_hw_t8103_usb4 = { 1228 .type = DART_T8020, 1229 .irq_handler = apple_dart_t8020_irq, 1230 .invalidate_tlb = apple_dart_t8020_hw_invalidate_tlb, 1231 .oas = 36, 1232 .fmt = APPLE_DART, 1233 .max_sid_count = 64, 1234 1235 .enable_streams = DART_T8020_STREAMS_ENABLE, 1236 .lock = DART_T8020_CONFIG, 1237 .lock_bit = DART_T8020_CONFIG_LOCK, 1238 1239 .error = DART_T8020_ERROR, 1240 1241 .tcr = DART_T8020_TCR, 1242 .tcr_enabled = DART_T8020_TCR_TRANSLATE_ENABLE, 1243 .tcr_disabled = 0, 1244 .tcr_bypass = 0, 1245 1246 .ttbr = DART_T8020_USB4_TTBR, 1247 .ttbr_valid = DART_T8020_TTBR_VALID, 1248 .ttbr_addr_field_shift = DART_T8020_TTBR_ADDR_FIELD_SHIFT, 1249 .ttbr_shift = DART_T8020_TTBR_SHIFT, 1250 .ttbr_count = 4, 1251 }; 1252 1253 static const struct apple_dart_hw apple_dart_hw_t6000 = { 1254 .type = DART_T6000, 1255 .irq_handler = apple_dart_t8020_irq, 1256 .invalidate_tlb = apple_dart_t8020_hw_invalidate_tlb, 1257 .oas = 42, 1258 .fmt = APPLE_DART2, 1259 .max_sid_count = 16, 1260 1261 .enable_streams = DART_T8020_STREAMS_ENABLE, 1262 .lock = DART_T8020_CONFIG, 1263 .lock_bit = DART_T8020_CONFIG_LOCK, 1264 1265 .error = DART_T8020_ERROR, 1266 1267 .tcr = DART_T8020_TCR, 1268 .tcr_enabled = DART_T8020_TCR_TRANSLATE_ENABLE, 1269 .tcr_disabled = 0, 1270 .tcr_bypass = DART_T8020_TCR_BYPASS_DAPF | DART_T8020_TCR_BYPASS_DART, 1271 1272 .ttbr = DART_T8020_TTBR, 1273 .ttbr_valid = DART_T8020_TTBR_VALID, 1274 .ttbr_addr_field_shift = DART_T8020_TTBR_ADDR_FIELD_SHIFT, 1275 .ttbr_shift = DART_T8020_TTBR_SHIFT, 1276 .ttbr_count = 4, 1277 }; 1278 1279 static const struct apple_dart_hw apple_dart_hw_t8110 = { 1280 .type = DART_T8110, 1281 .irq_handler = apple_dart_t8110_irq, 1282 .invalidate_tlb = apple_dart_t8110_hw_invalidate_tlb, 1283 .fmt = APPLE_DART2, 1284 .max_sid_count = 256, 1285 1286 .enable_streams = DART_T8110_ENABLE_STREAMS, 1287 .lock = DART_T8110_PROTECT, 1288 .lock_bit = DART_T8110_PROTECT_TTBR_TCR, 1289 1290 .error = DART_T8110_ERROR, 1291 1292 .tcr = DART_T8110_TCR, 1293 .tcr_enabled = DART_T8110_TCR_TRANSLATE_ENABLE, 1294 .tcr_disabled = 0, 1295 .tcr_bypass = DART_T8110_TCR_BYPASS_DAPF | DART_T8110_TCR_BYPASS_DART, 1296 1297 .ttbr = DART_T8110_TTBR, 1298 .ttbr_valid = DART_T8110_TTBR_VALID, 1299 .ttbr_addr_field_shift = DART_T8110_TTBR_ADDR_FIELD_SHIFT, 1300 .ttbr_shift = DART_T8110_TTBR_SHIFT, 1301 .ttbr_count = 1, 1302 }; 1303 1304 static __maybe_unused int apple_dart_suspend(struct device *dev) 1305 { 1306 struct apple_dart *dart = dev_get_drvdata(dev); 1307 unsigned int sid, idx; 1308 1309 for (sid = 0; sid < dart->num_streams; sid++) { 1310 dart->save_tcr[sid] = readl(dart->regs + DART_TCR(dart, sid)); 1311 for (idx = 0; idx < dart->hw->ttbr_count; idx++) 1312 dart->save_ttbr[sid][idx] = 1313 readl(dart->regs + DART_TTBR(dart, sid, idx)); 1314 } 1315 1316 return 0; 1317 } 1318 1319 static __maybe_unused int apple_dart_resume(struct device *dev) 1320 { 1321 struct apple_dart *dart = dev_get_drvdata(dev); 1322 unsigned int sid, idx; 1323 int ret; 1324 1325 ret = apple_dart_hw_reset(dart); 1326 if (ret) { 1327 dev_err(dev, "Failed to reset DART on resume\n"); 1328 return ret; 1329 } 1330 1331 for (sid = 0; sid < dart->num_streams; sid++) { 1332 for (idx = 0; idx < dart->hw->ttbr_count; idx++) 1333 writel(dart->save_ttbr[sid][idx], 1334 dart->regs + DART_TTBR(dart, sid, idx)); 1335 writel(dart->save_tcr[sid], dart->regs + DART_TCR(dart, sid)); 1336 } 1337 1338 return 0; 1339 } 1340 1341 static DEFINE_SIMPLE_DEV_PM_OPS(apple_dart_pm_ops, apple_dart_suspend, apple_dart_resume); 1342 1343 static const struct of_device_id apple_dart_of_match[] = { 1344 { .compatible = "apple,t8103-dart", .data = &apple_dart_hw_t8103 }, 1345 { .compatible = "apple,t8103-usb4-dart", .data = &apple_dart_hw_t8103_usb4 }, 1346 { .compatible = "apple,t8110-dart", .data = &apple_dart_hw_t8110 }, 1347 { .compatible = "apple,t6000-dart", .data = &apple_dart_hw_t6000 }, 1348 {}, 1349 }; 1350 MODULE_DEVICE_TABLE(of, apple_dart_of_match); 1351 1352 static struct platform_driver apple_dart_driver = { 1353 .driver = { 1354 .name = "apple-dart", 1355 .of_match_table = apple_dart_of_match, 1356 .suppress_bind_attrs = true, 1357 .pm = pm_sleep_ptr(&apple_dart_pm_ops), 1358 }, 1359 .probe = apple_dart_probe, 1360 .remove = apple_dart_remove, 1361 }; 1362 1363 module_platform_driver(apple_dart_driver); 1364 1365 MODULE_DESCRIPTION("IOMMU API for Apple's DART"); 1366 MODULE_AUTHOR("Sven Peter <sven@svenpeter.dev>"); 1367 MODULE_LICENSE("GPL v2"); 1368