Lines Matching full:dart
3 * Apple DART (Device Address Resolution Table) IOMMU driver
145 #define DART_TCR(dart, sid) ((dart)->hw->tcr + ((sid) << 2)) argument
147 #define DART_TTBR(dart, sid, idx) ((dart)->hw->ttbr + \ argument
148 (((dart)->hw->ttbr_count * (sid)) << 2) + \
189 * Private structure associated with each DART device.
195 * @clks: clocks associated with this DART
197 * @lock: lock for hardware operations involving this dart
198 * @pgsize: pagesize supported by this DART
199 * @supports_bypass: indicates if this DART supports bypass mode
237 * and never changed again afterwards. Devices with different dart pointers
240 * @dart dart pointer
244 struct apple_dart *dart; member
248 struct apple_dart *dart; member
253 * This structure is attached to each iommu domain handled by a DART.
280 /* Intersection of DART capabilitles */
296 i < MAX_DARTS_PER_DEVICE && stream_map->dart; \
310 struct apple_dart *dart = stream_map->dart; in apple_dart_hw_enable_translation() local
313 for_each_set_bit(sid, stream_map->sidmap, dart->num_streams) in apple_dart_hw_enable_translation()
314 writel(dart->hw->tcr_enabled, dart->regs + DART_TCR(dart, sid)); in apple_dart_hw_enable_translation()
319 struct apple_dart *dart = stream_map->dart; in apple_dart_hw_disable_dma() local
322 for_each_set_bit(sid, stream_map->sidmap, dart->num_streams) in apple_dart_hw_disable_dma()
323 writel(dart->hw->tcr_disabled, dart->regs + DART_TCR(dart, sid)); in apple_dart_hw_disable_dma()
329 struct apple_dart *dart = stream_map->dart; in apple_dart_hw_enable_bypass() local
332 WARN_ON(!stream_map->dart->supports_bypass); in apple_dart_hw_enable_bypass()
333 for_each_set_bit(sid, stream_map->sidmap, dart->num_streams) in apple_dart_hw_enable_bypass()
334 writel(dart->hw->tcr_bypass, in apple_dart_hw_enable_bypass()
335 dart->regs + DART_TCR(dart, sid)); in apple_dart_hw_enable_bypass()
341 struct apple_dart *dart = stream_map->dart; in apple_dart_hw_set_ttbr() local
344 WARN_ON(paddr & ((1 << dart->hw->ttbr_shift) - 1)); in apple_dart_hw_set_ttbr()
345 for_each_set_bit(sid, stream_map->sidmap, dart->num_streams) in apple_dart_hw_set_ttbr()
346 writel(dart->hw->ttbr_valid | in apple_dart_hw_set_ttbr()
347 (paddr >> dart->hw->ttbr_shift) << dart->hw->ttbr_addr_field_shift, in apple_dart_hw_set_ttbr()
348 dart->regs + DART_TTBR(dart, sid, idx)); in apple_dart_hw_set_ttbr()
354 struct apple_dart *dart = stream_map->dart; in apple_dart_hw_clear_ttbr() local
357 for_each_set_bit(sid, stream_map->sidmap, dart->num_streams) in apple_dart_hw_clear_ttbr()
358 writel(0, dart->regs + DART_TTBR(dart, sid, idx)); in apple_dart_hw_clear_ttbr()
366 for (i = 0; i < stream_map->dart->hw->ttbr_count; ++i) in apple_dart_hw_clear_all_ttbrs()
378 spin_lock_irqsave(&stream_map->dart->lock, flags); in apple_dart_t8020_hw_stream_command()
380 for (i = 0; i < BITS_TO_U32(stream_map->dart->num_streams); i++) in apple_dart_t8020_hw_stream_command()
382 stream_map->dart->regs + DART_T8020_STREAM_SELECT + 4 * i); in apple_dart_t8020_hw_stream_command()
383 writel(command, stream_map->dart->regs + DART_T8020_STREAM_COMMAND); in apple_dart_t8020_hw_stream_command()
386 stream_map->dart->regs + DART_T8020_STREAM_COMMAND, command_reg, in apple_dart_t8020_hw_stream_command()
390 spin_unlock_irqrestore(&stream_map->dart->lock, flags); in apple_dart_t8020_hw_stream_command()
393 dev_err(stream_map->dart->dev, in apple_dart_t8020_hw_stream_command()
406 struct apple_dart *dart = stream_map->dart; in apple_dart_t8110_hw_tlb_command() local
411 spin_lock_irqsave(&dart->lock, flags); in apple_dart_t8110_hw_tlb_command()
413 for_each_set_bit(sid, stream_map->sidmap, dart->num_streams) { in apple_dart_t8110_hw_tlb_command()
416 writel(val, dart->regs + DART_T8110_TLB_CMD); in apple_dart_t8110_hw_tlb_command()
419 dart->regs + DART_T8110_TLB_CMD, val, in apple_dart_t8110_hw_tlb_command()
428 spin_unlock_irqrestore(&dart->lock, flags); in apple_dart_t8110_hw_tlb_command()
431 dev_err(stream_map->dart->dev, in apple_dart_t8110_hw_tlb_command()
454 static int apple_dart_hw_reset(struct apple_dart *dart) in apple_dart_hw_reset() argument
460 config = readl(dart->regs + dart->hw->lock); in apple_dart_hw_reset()
461 if (config & dart->hw->lock_bit) { in apple_dart_hw_reset()
462 dev_err(dart->dev, "DART is locked down until reboot: %08x\n", in apple_dart_hw_reset()
467 stream_map.dart = dart; in apple_dart_hw_reset()
469 bitmap_set(stream_map.sidmap, 0, dart->num_streams); in apple_dart_hw_reset()
474 for (i = 0; i < BITS_TO_U32(dart->num_streams); i++) in apple_dart_hw_reset()
475 writel(U32_MAX, dart->regs + dart->hw->enable_streams + 4 * i); in apple_dart_hw_reset()
478 writel(readl(dart->regs + dart->hw->error), dart->regs + dart->hw->error); in apple_dart_hw_reset()
480 if (dart->hw->type == DART_T8110) in apple_dart_hw_reset()
481 writel(0, dart->regs + DART_T8110_ERROR_MASK); in apple_dart_hw_reset()
483 return dart->hw->invalidate_tlb(&stream_map); in apple_dart_hw_reset()
493 stream_map.dart = domain_stream_map->dart; in apple_dart_domain_flush_tlb()
495 for (j = 0; j < BITS_TO_LONGS(stream_map.dart->num_streams); j++) in apple_dart_domain_flush_tlb()
498 stream_map.dart->hw->invalidate_tlb(&stream_map); in apple_dart_domain_flush_tlb()
569 for (; i < stream_map->dart->hw->ttbr_count; ++i) in apple_dart_setup_translation()
573 stream_map->dart->hw->invalidate_tlb(stream_map); in apple_dart_setup_translation()
579 struct apple_dart *dart = cfg->stream_maps[0].dart; in apple_dart_finalize_domain() local
584 if (dart->pgsize > PAGE_SIZE) in apple_dart_finalize_domain()
593 dart_domain->stream_maps[i].dart = cfg->stream_maps[i].dart; in apple_dart_finalize_domain()
594 for (j = 0; j < BITS_TO_LONGS(dart->num_streams); j++) in apple_dart_finalize_domain()
600 .pgsize_bitmap = dart->pgsize, in apple_dart_finalize_domain()
601 .ias = dart->ias, in apple_dart_finalize_domain()
602 .oas = dart->oas, in apple_dart_finalize_domain()
604 .iommu_dev = dart->dev, in apple_dart_finalize_domain()
607 dart_domain->pgtbl_ops = alloc_io_pgtable_ops(dart->hw->fmt, &pgtbl_cfg, in apple_dart_finalize_domain()
617 (dma_addr_t)DMA_BIT_MASK(dart->ias); in apple_dart_finalize_domain()
635 if (domain_maps[i].dart != master_maps[i].dart) in apple_dart_mod_streams()
640 if (!domain_maps[i].dart) in apple_dart_mod_streams()
642 for (j = 0; j < BITS_TO_LONGS(domain_maps[i].dart->num_streams); j++) { in apple_dart_mod_streams()
739 dev, stream_map->dart->dev, in apple_dart_probe_device()
742 return &cfg->stream_maps[0].dart->iommu; in apple_dart_probe_device()
790 struct apple_dart *dart = platform_get_drvdata(iommu_pdev); in apple_dart_of_xlate() local
802 /* Will be ANDed with DART capabilities */ in apple_dart_of_xlate()
807 cfg_dart = cfg->stream_maps[0].dart; in apple_dart_of_xlate()
809 if (cfg_dart->pgsize != dart->pgsize) in apple_dart_of_xlate()
813 cfg->supports_bypass &= dart->supports_bypass; in apple_dart_of_xlate()
816 if (cfg->stream_maps[i].dart == dart) { in apple_dart_of_xlate()
822 if (!cfg->stream_maps[i].dart) { in apple_dart_of_xlate()
823 cfg->stream_maps[i].dart = dart; in apple_dart_of_xlate()
843 for_each_set_bit(sid, stream_map->sidmap, stream_map->dart->num_streams) in apple_dart_release_group()
844 stream_map->dart->sid2group[sid] = NULL; in apple_dart_release_group()
857 * just assume that both src and dst only have the same single DART. in apple_dart_merge_master_cfg()
859 if (src->stream_maps[1].dart) in apple_dart_merge_master_cfg()
861 if (dst->stream_maps[1].dart) in apple_dart_merge_master_cfg()
863 if (src->stream_maps[0].dart != dst->stream_maps[0].dart) in apple_dart_merge_master_cfg()
869 dst->stream_maps[0].dart->num_streams); in apple_dart_merge_master_cfg()
885 for_each_set_bit(sid, stream_map->sidmap, stream_map->dart->num_streams) { in apple_dart_device_group()
887 stream_map->dart->sid2group[sid]; in apple_dart_device_group()
920 dev_err(dev, "Failed to merge DART IOMMU groups.\n"); in apple_dart_device_group()
938 for_each_set_bit(sid, stream_map->sidmap, stream_map->dart->num_streams) in apple_dart_device_group()
939 stream_map->dart->sid2group[sid] = group; in apple_dart_device_group()
952 if (cfg->stream_maps[0].dart->pgsize > PAGE_SIZE) in apple_dart_def_domain_type()
995 .pgsize_bitmap = -1UL, /* Restricted during dart probe */
1011 struct apple_dart *dart = dev; in apple_dart_t8020_irq() local
1013 u32 error = readl(dart->regs + DART_T8020_ERROR); in apple_dart_t8020_irq()
1015 u32 addr_lo = readl(dart->regs + DART_T8020_ERROR_ADDR_LO); in apple_dart_t8020_irq()
1016 u32 addr_hi = readl(dart->regs + DART_T8020_ERROR_ADDR_HI); in apple_dart_t8020_irq()
1038 dart->dev, in apple_dart_t8020_irq()
1042 writel(error, dart->regs + DART_T8020_ERROR); in apple_dart_t8020_irq()
1048 struct apple_dart *dart = dev; in apple_dart_t8110_irq() local
1050 u32 error = readl(dart->regs + DART_T8110_ERROR); in apple_dart_t8110_irq()
1052 u32 addr_lo = readl(dart->regs + DART_T8110_ERROR_ADDR_LO); in apple_dart_t8110_irq()
1053 u32 addr_hi = readl(dart->regs + DART_T8110_ERROR_ADDR_HI); in apple_dart_t8110_irq()
1077 dart->dev, in apple_dart_t8110_irq()
1081 writel(error, dart->regs + DART_T8110_ERROR); in apple_dart_t8110_irq()
1090 struct apple_dart *dart; in apple_dart_probe() local
1093 dart = devm_kzalloc(dev, sizeof(*dart), GFP_KERNEL); in apple_dart_probe()
1094 if (!dart) in apple_dart_probe()
1097 dart->dev = dev; in apple_dart_probe()
1098 dart->hw = of_device_get_match_data(dev); in apple_dart_probe()
1099 spin_lock_init(&dart->lock); in apple_dart_probe()
1101 dart->regs = devm_platform_get_and_ioremap_resource(pdev, 0, &res); in apple_dart_probe()
1102 if (IS_ERR(dart->regs)) in apple_dart_probe()
1103 return PTR_ERR(dart->regs); in apple_dart_probe()
1110 dart->irq = platform_get_irq(pdev, 0); in apple_dart_probe()
1111 if (dart->irq < 0) in apple_dart_probe()
1114 ret = devm_clk_bulk_get_all(dev, &dart->clks); in apple_dart_probe()
1117 dart->num_clks = ret; in apple_dart_probe()
1119 ret = clk_bulk_prepare_enable(dart->num_clks, dart->clks); in apple_dart_probe()
1123 dart_params[0] = readl(dart->regs + DART_PARAMS1); in apple_dart_probe()
1124 dart_params[1] = readl(dart->regs + DART_PARAMS2); in apple_dart_probe()
1125 dart->pgsize = 1 << FIELD_GET(DART_PARAMS1_PAGE_SHIFT, dart_params[0]); in apple_dart_probe()
1126 dart->supports_bypass = dart_params[1] & DART_PARAMS2_BYPASS_SUPPORT; in apple_dart_probe()
1128 switch (dart->hw->type) { in apple_dart_probe()
1131 dart->ias = 32; in apple_dart_probe()
1132 dart->oas = dart->hw->oas; in apple_dart_probe()
1133 dart->num_streams = dart->hw->max_sid_count; in apple_dart_probe()
1137 dart_params[2] = readl(dart->regs + DART_T8110_PARAMS3); in apple_dart_probe()
1138 dart_params[3] = readl(dart->regs + DART_T8110_PARAMS4); in apple_dart_probe()
1139 dart->ias = FIELD_GET(DART_T8110_PARAMS3_VA_WIDTH, dart_params[2]); in apple_dart_probe()
1140 dart->oas = FIELD_GET(DART_T8110_PARAMS3_PA_WIDTH, dart_params[2]); in apple_dart_probe()
1141 dart->num_streams = FIELD_GET(DART_T8110_PARAMS4_NUM_SIDS, dart_params[3]); in apple_dart_probe()
1145 if (dart->num_streams > DART_MAX_STREAMS) { in apple_dart_probe()
1147 dart->num_streams, DART_MAX_STREAMS); in apple_dart_probe()
1152 ret = apple_dart_hw_reset(dart); in apple_dart_probe()
1156 ret = request_irq(dart->irq, dart->hw->irq_handler, IRQF_SHARED, in apple_dart_probe()
1157 "apple-dart fault handler", dart); in apple_dart_probe()
1161 platform_set_drvdata(pdev, dart); in apple_dart_probe()
1163 ret = iommu_device_sysfs_add(&dart->iommu, dev, NULL, "apple-dart.%s", in apple_dart_probe()
1168 ret = iommu_device_register(&dart->iommu, &apple_dart_iommu_ops, dev); in apple_dart_probe()
1174 "DART [pagesize %x, %d streams, bypass support: %d, bypass forced: %d] initialized\n", in apple_dart_probe()
1175 dart->pgsize, dart->num_streams, dart->supports_bypass, in apple_dart_probe()
1176 dart->pgsize > PAGE_SIZE); in apple_dart_probe()
1180 iommu_device_sysfs_remove(&dart->iommu); in apple_dart_probe()
1182 free_irq(dart->irq, dart); in apple_dart_probe()
1184 clk_bulk_disable_unprepare(dart->num_clks, dart->clks); in apple_dart_probe()
1191 struct apple_dart *dart = platform_get_drvdata(pdev); in apple_dart_remove() local
1193 apple_dart_hw_reset(dart); in apple_dart_remove()
1194 free_irq(dart->irq, dart); in apple_dart_remove()
1196 iommu_device_unregister(&dart->iommu); in apple_dart_remove()
1197 iommu_device_sysfs_remove(&dart->iommu); in apple_dart_remove()
1199 clk_bulk_disable_unprepare(dart->num_clks, dart->clks); in apple_dart_remove()
1307 struct apple_dart *dart = dev_get_drvdata(dev); in apple_dart_suspend() local
1310 for (sid = 0; sid < dart->num_streams; sid++) { in apple_dart_suspend()
1311 dart->save_tcr[sid] = readl(dart->regs + DART_TCR(dart, sid)); in apple_dart_suspend()
1312 for (idx = 0; idx < dart->hw->ttbr_count; idx++) in apple_dart_suspend()
1313 dart->save_ttbr[sid][idx] = in apple_dart_suspend()
1314 readl(dart->regs + DART_TTBR(dart, sid, idx)); in apple_dart_suspend()
1322 struct apple_dart *dart = dev_get_drvdata(dev); in apple_dart_resume() local
1326 ret = apple_dart_hw_reset(dart); in apple_dart_resume()
1328 dev_err(dev, "Failed to reset DART on resume\n"); in apple_dart_resume()
1332 for (sid = 0; sid < dart->num_streams; sid++) { in apple_dart_resume()
1333 for (idx = 0; idx < dart->hw->ttbr_count; idx++) in apple_dart_resume()
1334 writel(dart->save_ttbr[sid][idx], in apple_dart_resume()
1335 dart->regs + DART_TTBR(dart, sid, idx)); in apple_dart_resume()
1336 writel(dart->save_tcr[sid], dart->regs + DART_TCR(dart, sid)); in apple_dart_resume()
1345 { .compatible = "apple,t8103-dart", .data = &apple_dart_hw_t8103 },
1346 { .compatible = "apple,t8103-usb4-dart", .data = &apple_dart_hw_t8103_usb4 },
1347 { .compatible = "apple,t8110-dart", .data = &apple_dart_hw_t8110 },
1348 { .compatible = "apple,t6000-dart", .data = &apple_dart_hw_t6000 },
1355 .name = "apple-dart",
1366 MODULE_DESCRIPTION("IOMMU API for Apple's DART");