1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Apple DART (Device Address Resolution Table) IOMMU driver
4 *
5 * Copyright (C) 2021 The Asahi Linux Contributors
6 *
7 * Based on arm/arm-smmu/arm-ssmu.c and arm/arm-smmu-v3/arm-smmu-v3.c
8 * Copyright (C) 2013 ARM Limited
9 * Copyright (C) 2015 ARM Limited
10 * and on exynos-iommu.c
11 * Copyright (c) 2011,2016 Samsung Electronics Co., Ltd.
12 */
13
14 #include <linux/atomic.h>
15 #include <linux/bitfield.h>
16 #include <linux/clk.h>
17 #include <linux/dev_printk.h>
18 #include <linux/dma-mapping.h>
19 #include <linux/err.h>
20 #include <linux/interrupt.h>
21 #include <linux/io-pgtable.h>
22 #include <linux/iommu.h>
23 #include <linux/iopoll.h>
24 #include <linux/module.h>
25 #include <linux/of.h>
26 #include <linux/of_address.h>
27 #include <linux/of_iommu.h>
28 #include <linux/of_platform.h>
29 #include <linux/pci.h>
30 #include <linux/platform_device.h>
31 #include <linux/slab.h>
32 #include <linux/swab.h>
33 #include <linux/types.h>
34
35 #include "dma-iommu.h"
36
37 #define DART_MAX_STREAMS 256
38 #define DART_MAX_TTBR 4
39 #define MAX_DARTS_PER_DEVICE 3
40
41 /* Common registers */
42
43 #define DART_PARAMS1 0x00
44 #define DART_PARAMS1_PAGE_SHIFT GENMASK(27, 24)
45
46 #define DART_PARAMS2 0x04
47 #define DART_PARAMS2_BYPASS_SUPPORT BIT(0)
48
49 /* T8020/T6000 registers */
50
51 #define DART_T8020_STREAM_COMMAND 0x20
52 #define DART_T8020_STREAM_COMMAND_BUSY BIT(2)
53 #define DART_T8020_STREAM_COMMAND_INVALIDATE BIT(20)
54
55 #define DART_T8020_STREAM_SELECT 0x34
56
57 #define DART_T8020_ERROR 0x40
58 #define DART_T8020_ERROR_STREAM GENMASK(27, 24)
59 #define DART_T8020_ERROR_CODE GENMASK(11, 0)
60 #define DART_T8020_ERROR_FLAG BIT(31)
61
62 #define DART_T8020_ERROR_READ_FAULT BIT(4)
63 #define DART_T8020_ERROR_WRITE_FAULT BIT(3)
64 #define DART_T8020_ERROR_NO_PTE BIT(2)
65 #define DART_T8020_ERROR_NO_PMD BIT(1)
66 #define DART_T8020_ERROR_NO_TTBR BIT(0)
67
68 #define DART_T8020_CONFIG 0x60
69 #define DART_T8020_CONFIG_LOCK BIT(15)
70
71 #define DART_STREAM_COMMAND_BUSY_TIMEOUT 100
72
73 #define DART_T8020_ERROR_ADDR_HI 0x54
74 #define DART_T8020_ERROR_ADDR_LO 0x50
75
76 #define DART_T8020_STREAMS_ENABLE 0xfc
77
78 #define DART_T8020_TCR 0x100
79 #define DART_T8020_TCR_TRANSLATE_ENABLE BIT(7)
80 #define DART_T8020_TCR_BYPASS_DART BIT(8)
81 #define DART_T8020_TCR_BYPASS_DAPF BIT(12)
82
83 #define DART_T8020_TTBR 0x200
84 #define DART_T8020_USB4_TTBR 0x400
85 #define DART_T8020_TTBR_VALID BIT(31)
86 #define DART_T8020_TTBR_ADDR_FIELD_SHIFT 0
87 #define DART_T8020_TTBR_SHIFT 12
88
89 /* T8110 registers */
90
91 #define DART_T8110_PARAMS3 0x08
92 #define DART_T8110_PARAMS3_PA_WIDTH GENMASK(29, 24)
93 #define DART_T8110_PARAMS3_VA_WIDTH GENMASK(21, 16)
94 #define DART_T8110_PARAMS3_VER_MAJ GENMASK(15, 8)
95 #define DART_T8110_PARAMS3_VER_MIN GENMASK(7, 0)
96
97 #define DART_T8110_PARAMS4 0x0c
98 #define DART_T8110_PARAMS4_NUM_CLIENTS GENMASK(24, 16)
99 #define DART_T8110_PARAMS4_NUM_SIDS GENMASK(8, 0)
100
101 #define DART_T8110_TLB_CMD 0x80
102 #define DART_T8110_TLB_CMD_BUSY BIT(31)
103 #define DART_T8110_TLB_CMD_OP GENMASK(10, 8)
104 #define DART_T8110_TLB_CMD_OP_FLUSH_ALL 0
105 #define DART_T8110_TLB_CMD_OP_FLUSH_SID 1
106 #define DART_T8110_TLB_CMD_STREAM GENMASK(7, 0)
107
108 #define DART_T8110_ERROR 0x100
109 #define DART_T8110_ERROR_STREAM GENMASK(27, 20)
110 #define DART_T8110_ERROR_CODE GENMASK(14, 0)
111 #define DART_T8110_ERROR_FLAG BIT(31)
112
113 #define DART_T8110_ERROR_MASK 0x104
114
115 #define DART_T8110_ERROR_READ_FAULT BIT(5)
116 #define DART_T8110_ERROR_WRITE_FAULT BIT(4)
117 #define DART_T8110_ERROR_NO_PTE BIT(3)
118 #define DART_T8110_ERROR_NO_PMD BIT(2)
119 #define DART_T8110_ERROR_NO_PGD BIT(1)
120 #define DART_T8110_ERROR_NO_TTBR BIT(0)
121
122 #define DART_T8110_ERROR_ADDR_LO 0x170
123 #define DART_T8110_ERROR_ADDR_HI 0x174
124
125 #define DART_T8110_ERROR_STREAMS 0x1c0
126
127 #define DART_T8110_PROTECT 0x200
128 #define DART_T8110_UNPROTECT 0x204
129 #define DART_T8110_PROTECT_LOCK 0x208
130 #define DART_T8110_PROTECT_TTBR_TCR BIT(0)
131
132 #define DART_T8110_ENABLE_STREAMS 0xc00
133 #define DART_T8110_DISABLE_STREAMS 0xc20
134
135 #define DART_T8110_TCR 0x1000
136 #define DART_T8110_TCR_REMAP GENMASK(11, 8)
137 #define DART_T8110_TCR_REMAP_EN BIT(7)
138 #define DART_T8110_TCR_FOUR_LEVEL BIT(3)
139 #define DART_T8110_TCR_BYPASS_DAPF BIT(2)
140 #define DART_T8110_TCR_BYPASS_DART BIT(1)
141 #define DART_T8110_TCR_TRANSLATE_ENABLE BIT(0)
142
143 #define DART_T8110_TTBR 0x1400
144 #define DART_T8110_TTBR_VALID BIT(0)
145 #define DART_T8110_TTBR_ADDR_FIELD_SHIFT 2
146 #define DART_T8110_TTBR_SHIFT 14
147
148 #define DART_TCR(dart, sid) ((dart)->hw->tcr + ((sid) << 2))
149
150 #define DART_TTBR(dart, sid, idx) ((dart)->hw->ttbr + \
151 (((dart)->hw->ttbr_count * (sid)) << 2) + \
152 ((idx) << 2))
153
154 struct apple_dart_stream_map;
155
156 enum dart_type {
157 DART_T8020,
158 DART_T6000,
159 DART_T8110,
160 };
161
162 struct apple_dart_hw {
163 enum dart_type type;
164 irqreturn_t (*irq_handler)(int irq, void *dev);
165 int (*invalidate_tlb)(struct apple_dart_stream_map *stream_map);
166
167 u32 oas;
168 enum io_pgtable_fmt fmt;
169
170 int max_sid_count;
171
172 u32 lock;
173 u32 lock_bit;
174
175 u32 error;
176
177 u32 enable_streams;
178
179 u32 tcr;
180 u32 tcr_enabled;
181 u32 tcr_disabled;
182 u32 tcr_bypass;
183 u32 tcr_4level;
184
185 u32 ttbr;
186 u32 ttbr_valid;
187 u32 ttbr_addr_field_shift;
188 u32 ttbr_shift;
189 int ttbr_count;
190 };
191
192 /*
193 * Private structure associated with each DART device.
194 *
195 * @dev: device struct
196 * @hw: SoC-specific hardware data
197 * @regs: mapped MMIO region
198 * @irq: interrupt number, can be shared with other DARTs
199 * @clks: clocks associated with this DART
200 * @num_clks: number of @clks
201 * @lock: lock for hardware operations involving this dart
202 * @pgsize: pagesize supported by this DART
203 * @supports_bypass: indicates if this DART supports bypass mode
204 * @sid2group: maps stream ids to iommu_groups
205 * @iommu: iommu core device
206 */
207 struct apple_dart {
208 struct device *dev;
209 const struct apple_dart_hw *hw;
210
211 void __iomem *regs;
212
213 int irq;
214 struct clk_bulk_data *clks;
215 int num_clks;
216
217 spinlock_t lock;
218
219 u32 ias;
220 u32 oas;
221 u32 pgsize;
222 u32 num_streams;
223 u32 supports_bypass : 1;
224 u32 four_level : 1;
225
226 struct iommu_group *sid2group[DART_MAX_STREAMS];
227 struct iommu_device iommu;
228
229 u32 save_tcr[DART_MAX_STREAMS];
230 u32 save_ttbr[DART_MAX_STREAMS][DART_MAX_TTBR];
231 };
232
233 /*
234 * Convenience struct to identify streams.
235 *
236 * The normal variant is used inside apple_dart_master_cfg which isn't written
237 * to concurrently.
238 * The atomic variant is used inside apple_dart_domain where we have to guard
239 * against races from potential parallel calls to attach/detach_device.
240 * Note that even inside the atomic variant the apple_dart pointer is not
241 * protected: This pointer is initialized once under the domain init mutex
242 * and never changed again afterwards. Devices with different dart pointers
243 * cannot be attached to the same domain.
244 *
245 * @dart dart pointer
246 * @sid stream id bitmap
247 */
248 struct apple_dart_stream_map {
249 struct apple_dart *dart;
250 DECLARE_BITMAP(sidmap, DART_MAX_STREAMS);
251 };
252 struct apple_dart_atomic_stream_map {
253 struct apple_dart *dart;
254 atomic_long_t sidmap[BITS_TO_LONGS(DART_MAX_STREAMS)];
255 };
256
257 /*
258 * This structure is attached to each iommu domain handled by a DART.
259 *
260 * @pgtbl_ops: pagetable ops allocated by io-pgtable
261 * @finalized: true if the domain has been completely initialized
262 * @init_lock: protects domain initialization
263 * @stream_maps: streams attached to this domain (valid for DMA/UNMANAGED only)
264 * @domain: core iommu domain pointer
265 */
266 struct apple_dart_domain {
267 struct io_pgtable_ops *pgtbl_ops;
268
269 bool finalized;
270 struct mutex init_lock;
271 struct apple_dart_atomic_stream_map stream_maps[MAX_DARTS_PER_DEVICE];
272
273 struct iommu_domain domain;
274 };
275
276 /*
277 * This structure is attached to devices with dev_iommu_priv_set() on of_xlate
278 * and contains a list of streams bound to this device.
279 * So far the worst case seen is a single device with two streams
280 * from different darts, such that this simple static array is enough.
281 *
282 * @streams: streams for this device
283 */
284 struct apple_dart_master_cfg {
285 /* Intersection of DART capabilitles */
286 u32 supports_bypass : 1;
287
288 struct apple_dart_stream_map stream_maps[MAX_DARTS_PER_DEVICE];
289 };
290
291 /*
292 * Helper macro to iterate over apple_dart_master_cfg.stream_maps and
293 * apple_dart_domain.stream_maps
294 *
295 * @i int used as loop variable
296 * @base pointer to base struct (apple_dart_master_cfg or apple_dart_domain)
297 * @stream pointer to the apple_dart_streams struct for each loop iteration
298 */
299 #define for_each_stream_map(i, base, stream_map) \
300 for (i = 0, stream_map = &(base)->stream_maps[0]; \
301 i < MAX_DARTS_PER_DEVICE && stream_map->dart; \
302 stream_map = &(base)->stream_maps[++i])
303
304 static struct platform_driver apple_dart_driver;
305 static const struct iommu_ops apple_dart_iommu_ops;
306
to_dart_domain(struct iommu_domain * dom)307 static struct apple_dart_domain *to_dart_domain(struct iommu_domain *dom)
308 {
309 return container_of(dom, struct apple_dart_domain, domain);
310 }
311
312 static void
apple_dart_hw_enable_translation(struct apple_dart_stream_map * stream_map,int levels)313 apple_dart_hw_enable_translation(struct apple_dart_stream_map *stream_map, int levels)
314 {
315 struct apple_dart *dart = stream_map->dart;
316 u32 tcr = dart->hw->tcr_enabled;
317 int sid;
318
319 if (levels == 4)
320 tcr |= dart->hw->tcr_4level;
321
322 WARN_ON(levels != 3 && levels != 4);
323 WARN_ON(levels == 4 && !dart->four_level);
324 for_each_set_bit(sid, stream_map->sidmap, dart->num_streams)
325 writel(tcr, dart->regs + DART_TCR(dart, sid));
326 }
327
apple_dart_hw_disable_dma(struct apple_dart_stream_map * stream_map)328 static void apple_dart_hw_disable_dma(struct apple_dart_stream_map *stream_map)
329 {
330 struct apple_dart *dart = stream_map->dart;
331 int sid;
332
333 for_each_set_bit(sid, stream_map->sidmap, dart->num_streams)
334 writel(dart->hw->tcr_disabled, dart->regs + DART_TCR(dart, sid));
335 }
336
337 static void
apple_dart_hw_enable_bypass(struct apple_dart_stream_map * stream_map)338 apple_dart_hw_enable_bypass(struct apple_dart_stream_map *stream_map)
339 {
340 struct apple_dart *dart = stream_map->dart;
341 int sid;
342
343 WARN_ON(!stream_map->dart->supports_bypass);
344 for_each_set_bit(sid, stream_map->sidmap, dart->num_streams)
345 writel(dart->hw->tcr_bypass,
346 dart->regs + DART_TCR(dart, sid));
347 }
348
apple_dart_hw_set_ttbr(struct apple_dart_stream_map * stream_map,u8 idx,phys_addr_t paddr)349 static void apple_dart_hw_set_ttbr(struct apple_dart_stream_map *stream_map,
350 u8 idx, phys_addr_t paddr)
351 {
352 struct apple_dart *dart = stream_map->dart;
353 int sid;
354
355 WARN_ON(paddr & ((1 << dart->hw->ttbr_shift) - 1));
356 for_each_set_bit(sid, stream_map->sidmap, dart->num_streams)
357 writel(dart->hw->ttbr_valid |
358 (paddr >> dart->hw->ttbr_shift) << dart->hw->ttbr_addr_field_shift,
359 dart->regs + DART_TTBR(dart, sid, idx));
360 }
361
apple_dart_hw_clear_ttbr(struct apple_dart_stream_map * stream_map,u8 idx)362 static void apple_dart_hw_clear_ttbr(struct apple_dart_stream_map *stream_map,
363 u8 idx)
364 {
365 struct apple_dart *dart = stream_map->dart;
366 int sid;
367
368 for_each_set_bit(sid, stream_map->sidmap, dart->num_streams)
369 writel(0, dart->regs + DART_TTBR(dart, sid, idx));
370 }
371
372 static void
apple_dart_hw_clear_all_ttbrs(struct apple_dart_stream_map * stream_map)373 apple_dart_hw_clear_all_ttbrs(struct apple_dart_stream_map *stream_map)
374 {
375 int i;
376
377 for (i = 0; i < stream_map->dart->hw->ttbr_count; ++i)
378 apple_dart_hw_clear_ttbr(stream_map, i);
379 }
380
381 static int
apple_dart_t8020_hw_stream_command(struct apple_dart_stream_map * stream_map,u32 command)382 apple_dart_t8020_hw_stream_command(struct apple_dart_stream_map *stream_map,
383 u32 command)
384 {
385 unsigned long flags;
386 int ret, i;
387 u32 command_reg;
388
389 spin_lock_irqsave(&stream_map->dart->lock, flags);
390
391 for (i = 0; i < BITS_TO_U32(stream_map->dart->num_streams); i++)
392 writel(stream_map->sidmap[i],
393 stream_map->dart->regs + DART_T8020_STREAM_SELECT + 4 * i);
394 writel(command, stream_map->dart->regs + DART_T8020_STREAM_COMMAND);
395
396 ret = readl_poll_timeout_atomic(
397 stream_map->dart->regs + DART_T8020_STREAM_COMMAND, command_reg,
398 !(command_reg & DART_T8020_STREAM_COMMAND_BUSY), 1,
399 DART_STREAM_COMMAND_BUSY_TIMEOUT);
400
401 spin_unlock_irqrestore(&stream_map->dart->lock, flags);
402
403 if (ret) {
404 dev_err(stream_map->dart->dev,
405 "busy bit did not clear after command %x for streams %lx\n",
406 command, stream_map->sidmap[0]);
407 return ret;
408 }
409
410 return 0;
411 }
412
413 static int
apple_dart_t8110_hw_tlb_command(struct apple_dart_stream_map * stream_map,u32 command)414 apple_dart_t8110_hw_tlb_command(struct apple_dart_stream_map *stream_map,
415 u32 command)
416 {
417 struct apple_dart *dart = stream_map->dart;
418 unsigned long flags;
419 int ret = 0;
420 int sid;
421
422 spin_lock_irqsave(&dart->lock, flags);
423
424 for_each_set_bit(sid, stream_map->sidmap, dart->num_streams) {
425 u32 val = FIELD_PREP(DART_T8110_TLB_CMD_OP, command) |
426 FIELD_PREP(DART_T8110_TLB_CMD_STREAM, sid);
427 writel(val, dart->regs + DART_T8110_TLB_CMD);
428
429 ret = readl_poll_timeout_atomic(
430 dart->regs + DART_T8110_TLB_CMD, val,
431 !(val & DART_T8110_TLB_CMD_BUSY), 1,
432 DART_STREAM_COMMAND_BUSY_TIMEOUT);
433
434 if (ret)
435 break;
436
437 }
438
439 spin_unlock_irqrestore(&dart->lock, flags);
440
441 if (ret) {
442 dev_err(stream_map->dart->dev,
443 "busy bit did not clear after command %x for stream %d\n",
444 command, sid);
445 return ret;
446 }
447
448 return 0;
449 }
450
451 static int
apple_dart_t8020_hw_invalidate_tlb(struct apple_dart_stream_map * stream_map)452 apple_dart_t8020_hw_invalidate_tlb(struct apple_dart_stream_map *stream_map)
453 {
454 return apple_dart_t8020_hw_stream_command(
455 stream_map, DART_T8020_STREAM_COMMAND_INVALIDATE);
456 }
457
458 static int
apple_dart_t8110_hw_invalidate_tlb(struct apple_dart_stream_map * stream_map)459 apple_dart_t8110_hw_invalidate_tlb(struct apple_dart_stream_map *stream_map)
460 {
461 return apple_dart_t8110_hw_tlb_command(
462 stream_map, DART_T8110_TLB_CMD_OP_FLUSH_SID);
463 }
464
apple_dart_hw_reset(struct apple_dart * dart)465 static int apple_dart_hw_reset(struct apple_dart *dart)
466 {
467 u32 config;
468 struct apple_dart_stream_map stream_map;
469 int i;
470
471 config = readl(dart->regs + dart->hw->lock);
472 if (config & dart->hw->lock_bit) {
473 dev_err(dart->dev, "DART is locked down until reboot: %08x\n",
474 config);
475 return -EINVAL;
476 }
477
478 stream_map.dart = dart;
479 bitmap_zero(stream_map.sidmap, DART_MAX_STREAMS);
480 bitmap_set(stream_map.sidmap, 0, dart->num_streams);
481 apple_dart_hw_disable_dma(&stream_map);
482 apple_dart_hw_clear_all_ttbrs(&stream_map);
483
484 /* enable all streams globally since TCR is used to control isolation */
485 for (i = 0; i < BITS_TO_U32(dart->num_streams); i++)
486 writel(U32_MAX, dart->regs + dart->hw->enable_streams + 4 * i);
487
488 /* clear any pending errors before the interrupt is unmasked */
489 writel(readl(dart->regs + dart->hw->error), dart->regs + dart->hw->error);
490
491 if (dart->hw->type == DART_T8110)
492 writel(0, dart->regs + DART_T8110_ERROR_MASK);
493
494 return dart->hw->invalidate_tlb(&stream_map);
495 }
496
apple_dart_domain_flush_tlb(struct apple_dart_domain * domain)497 static void apple_dart_domain_flush_tlb(struct apple_dart_domain *domain)
498 {
499 int i, j;
500 struct apple_dart_atomic_stream_map *domain_stream_map;
501 struct apple_dart_stream_map stream_map;
502
503 for_each_stream_map(i, domain, domain_stream_map) {
504 stream_map.dart = domain_stream_map->dart;
505
506 for (j = 0; j < BITS_TO_LONGS(stream_map.dart->num_streams); j++)
507 stream_map.sidmap[j] = atomic_long_read(&domain_stream_map->sidmap[j]);
508
509 stream_map.dart->hw->invalidate_tlb(&stream_map);
510 }
511 }
512
apple_dart_flush_iotlb_all(struct iommu_domain * domain)513 static void apple_dart_flush_iotlb_all(struct iommu_domain *domain)
514 {
515 apple_dart_domain_flush_tlb(to_dart_domain(domain));
516 }
517
apple_dart_iotlb_sync(struct iommu_domain * domain,struct iommu_iotlb_gather * gather)518 static void apple_dart_iotlb_sync(struct iommu_domain *domain,
519 struct iommu_iotlb_gather *gather)
520 {
521 apple_dart_domain_flush_tlb(to_dart_domain(domain));
522 }
523
apple_dart_iotlb_sync_map(struct iommu_domain * domain,unsigned long iova,size_t size)524 static int apple_dart_iotlb_sync_map(struct iommu_domain *domain,
525 unsigned long iova, size_t size)
526 {
527 apple_dart_domain_flush_tlb(to_dart_domain(domain));
528 return 0;
529 }
530
apple_dart_iova_to_phys(struct iommu_domain * domain,dma_addr_t iova)531 static phys_addr_t apple_dart_iova_to_phys(struct iommu_domain *domain,
532 dma_addr_t iova)
533 {
534 struct apple_dart_domain *dart_domain = to_dart_domain(domain);
535 struct io_pgtable_ops *ops = dart_domain->pgtbl_ops;
536
537 if (!ops)
538 return 0;
539
540 return ops->iova_to_phys(ops, iova);
541 }
542
apple_dart_map_pages(struct iommu_domain * domain,unsigned long iova,phys_addr_t paddr,size_t pgsize,size_t pgcount,int prot,gfp_t gfp,size_t * mapped)543 static int apple_dart_map_pages(struct iommu_domain *domain, unsigned long iova,
544 phys_addr_t paddr, size_t pgsize,
545 size_t pgcount, int prot, gfp_t gfp,
546 size_t *mapped)
547 {
548 struct apple_dart_domain *dart_domain = to_dart_domain(domain);
549 struct io_pgtable_ops *ops = dart_domain->pgtbl_ops;
550
551 if (!ops)
552 return -ENODEV;
553
554 return ops->map_pages(ops, iova, paddr, pgsize, pgcount, prot, gfp,
555 mapped);
556 }
557
apple_dart_unmap_pages(struct iommu_domain * domain,unsigned long iova,size_t pgsize,size_t pgcount,struct iommu_iotlb_gather * gather)558 static size_t apple_dart_unmap_pages(struct iommu_domain *domain,
559 unsigned long iova, size_t pgsize,
560 size_t pgcount,
561 struct iommu_iotlb_gather *gather)
562 {
563 struct apple_dart_domain *dart_domain = to_dart_domain(domain);
564 struct io_pgtable_ops *ops = dart_domain->pgtbl_ops;
565
566 return ops->unmap_pages(ops, iova, pgsize, pgcount, gather);
567 }
568
569 static void
apple_dart_setup_translation(struct apple_dart_domain * domain,struct apple_dart_stream_map * stream_map)570 apple_dart_setup_translation(struct apple_dart_domain *domain,
571 struct apple_dart_stream_map *stream_map)
572 {
573 int i;
574 struct io_pgtable_cfg *pgtbl_cfg =
575 &io_pgtable_ops_to_pgtable(domain->pgtbl_ops)->cfg;
576
577 for (i = 0; i < pgtbl_cfg->apple_dart_cfg.n_ttbrs; ++i)
578 apple_dart_hw_set_ttbr(stream_map, i,
579 pgtbl_cfg->apple_dart_cfg.ttbr[i]);
580 for (; i < stream_map->dart->hw->ttbr_count; ++i)
581 apple_dart_hw_clear_ttbr(stream_map, i);
582
583 apple_dart_hw_enable_translation(stream_map,
584 pgtbl_cfg->apple_dart_cfg.n_levels);
585 stream_map->dart->hw->invalidate_tlb(stream_map);
586 }
587
apple_dart_finalize_domain(struct apple_dart_domain * dart_domain,struct apple_dart_master_cfg * cfg)588 static int apple_dart_finalize_domain(struct apple_dart_domain *dart_domain,
589 struct apple_dart_master_cfg *cfg)
590 {
591 struct apple_dart *dart = cfg->stream_maps[0].dart;
592 struct io_pgtable_cfg pgtbl_cfg;
593 int ret = 0;
594 int i, j;
595
596 if (dart->pgsize > PAGE_SIZE)
597 return -EINVAL;
598
599 mutex_lock(&dart_domain->init_lock);
600
601 if (dart_domain->finalized)
602 goto done;
603
604 for (i = 0; i < MAX_DARTS_PER_DEVICE; ++i) {
605 dart_domain->stream_maps[i].dart = cfg->stream_maps[i].dart;
606 for (j = 0; j < BITS_TO_LONGS(dart->num_streams); j++)
607 atomic_long_set(&dart_domain->stream_maps[i].sidmap[j],
608 cfg->stream_maps[i].sidmap[j]);
609 }
610
611 pgtbl_cfg = (struct io_pgtable_cfg){
612 .pgsize_bitmap = dart->pgsize,
613 .ias = dart->ias,
614 .oas = dart->oas,
615 .coherent_walk = 1,
616 .iommu_dev = dart->dev,
617 };
618
619 dart_domain->pgtbl_ops = alloc_io_pgtable_ops(dart->hw->fmt, &pgtbl_cfg,
620 &dart_domain->domain);
621 if (!dart_domain->pgtbl_ops) {
622 ret = -ENOMEM;
623 goto done;
624 }
625
626 dart_domain->domain.pgsize_bitmap = pgtbl_cfg.pgsize_bitmap;
627 dart_domain->domain.geometry.aperture_start = 0;
628 dart_domain->domain.geometry.aperture_end =
629 (dma_addr_t)DMA_BIT_MASK(pgtbl_cfg.ias);
630 dart_domain->domain.geometry.force_aperture = true;
631
632 dart_domain->finalized = true;
633
634 done:
635 mutex_unlock(&dart_domain->init_lock);
636 return ret;
637 }
638
639 static int
apple_dart_mod_streams(struct apple_dart_atomic_stream_map * domain_maps,struct apple_dart_stream_map * master_maps,bool add_streams)640 apple_dart_mod_streams(struct apple_dart_atomic_stream_map *domain_maps,
641 struct apple_dart_stream_map *master_maps,
642 bool add_streams)
643 {
644 int i, j;
645
646 for (i = 0; i < MAX_DARTS_PER_DEVICE; ++i) {
647 if (domain_maps[i].dart != master_maps[i].dart)
648 return -EINVAL;
649 }
650
651 for (i = 0; i < MAX_DARTS_PER_DEVICE; ++i) {
652 if (!domain_maps[i].dart)
653 break;
654 for (j = 0; j < BITS_TO_LONGS(domain_maps[i].dart->num_streams); j++) {
655 if (add_streams)
656 atomic_long_or(master_maps[i].sidmap[j],
657 &domain_maps[i].sidmap[j]);
658 else
659 atomic_long_and(~master_maps[i].sidmap[j],
660 &domain_maps[i].sidmap[j]);
661 }
662 }
663
664 return 0;
665 }
666
apple_dart_domain_add_streams(struct apple_dart_domain * domain,struct apple_dart_master_cfg * cfg)667 static int apple_dart_domain_add_streams(struct apple_dart_domain *domain,
668 struct apple_dart_master_cfg *cfg)
669 {
670 return apple_dart_mod_streams(domain->stream_maps, cfg->stream_maps,
671 true);
672 }
673
apple_dart_attach_dev_paging(struct iommu_domain * domain,struct device * dev)674 static int apple_dart_attach_dev_paging(struct iommu_domain *domain,
675 struct device *dev)
676 {
677 int ret, i;
678 struct apple_dart_stream_map *stream_map;
679 struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev);
680 struct apple_dart_domain *dart_domain = to_dart_domain(domain);
681
682 ret = apple_dart_finalize_domain(dart_domain, cfg);
683 if (ret)
684 return ret;
685
686 ret = apple_dart_domain_add_streams(dart_domain, cfg);
687 if (ret)
688 return ret;
689
690 for_each_stream_map(i, cfg, stream_map)
691 apple_dart_setup_translation(dart_domain, stream_map);
692 return 0;
693 }
694
apple_dart_attach_dev_identity(struct iommu_domain * domain,struct device * dev)695 static int apple_dart_attach_dev_identity(struct iommu_domain *domain,
696 struct device *dev)
697 {
698 struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev);
699 struct apple_dart_stream_map *stream_map;
700 int i;
701
702 if (!cfg->supports_bypass)
703 return -EINVAL;
704
705 for_each_stream_map(i, cfg, stream_map)
706 apple_dart_hw_enable_bypass(stream_map);
707 return 0;
708 }
709
710 static const struct iommu_domain_ops apple_dart_identity_ops = {
711 .attach_dev = apple_dart_attach_dev_identity,
712 };
713
714 static struct iommu_domain apple_dart_identity_domain = {
715 .type = IOMMU_DOMAIN_IDENTITY,
716 .ops = &apple_dart_identity_ops,
717 };
718
apple_dart_attach_dev_blocked(struct iommu_domain * domain,struct device * dev)719 static int apple_dart_attach_dev_blocked(struct iommu_domain *domain,
720 struct device *dev)
721 {
722 struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev);
723 struct apple_dart_stream_map *stream_map;
724 int i;
725
726 for_each_stream_map(i, cfg, stream_map)
727 apple_dart_hw_disable_dma(stream_map);
728 return 0;
729 }
730
731 static const struct iommu_domain_ops apple_dart_blocked_ops = {
732 .attach_dev = apple_dart_attach_dev_blocked,
733 };
734
735 static struct iommu_domain apple_dart_blocked_domain = {
736 .type = IOMMU_DOMAIN_BLOCKED,
737 .ops = &apple_dart_blocked_ops,
738 };
739
apple_dart_probe_device(struct device * dev)740 static struct iommu_device *apple_dart_probe_device(struct device *dev)
741 {
742 struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev);
743 struct apple_dart_stream_map *stream_map;
744 int i;
745
746 if (!cfg)
747 return ERR_PTR(-ENODEV);
748
749 for_each_stream_map(i, cfg, stream_map)
750 device_link_add(
751 dev, stream_map->dart->dev,
752 DL_FLAG_PM_RUNTIME | DL_FLAG_AUTOREMOVE_SUPPLIER);
753
754 return &cfg->stream_maps[0].dart->iommu;
755 }
756
apple_dart_release_device(struct device * dev)757 static void apple_dart_release_device(struct device *dev)
758 {
759 struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev);
760
761 kfree(cfg);
762 }
763
apple_dart_domain_alloc_paging(struct device * dev)764 static struct iommu_domain *apple_dart_domain_alloc_paging(struct device *dev)
765 {
766 struct apple_dart_domain *dart_domain;
767
768 dart_domain = kzalloc(sizeof(*dart_domain), GFP_KERNEL);
769 if (!dart_domain)
770 return NULL;
771
772 mutex_init(&dart_domain->init_lock);
773
774 if (dev) {
775 struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev);
776 int ret;
777
778 ret = apple_dart_finalize_domain(dart_domain, cfg);
779 if (ret) {
780 kfree(dart_domain);
781 return ERR_PTR(ret);
782 }
783 }
784 return &dart_domain->domain;
785 }
786
apple_dart_domain_free(struct iommu_domain * domain)787 static void apple_dart_domain_free(struct iommu_domain *domain)
788 {
789 struct apple_dart_domain *dart_domain = to_dart_domain(domain);
790
791 free_io_pgtable_ops(dart_domain->pgtbl_ops);
792
793 kfree(dart_domain);
794 }
795
apple_dart_of_xlate(struct device * dev,const struct of_phandle_args * args)796 static int apple_dart_of_xlate(struct device *dev,
797 const struct of_phandle_args *args)
798 {
799 struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev);
800 struct platform_device *iommu_pdev = of_find_device_by_node(args->np);
801 struct apple_dart *dart = platform_get_drvdata(iommu_pdev);
802 struct apple_dart *cfg_dart;
803 int i, sid;
804
805 if (args->args_count != 1)
806 return -EINVAL;
807 sid = args->args[0];
808
809 if (!cfg) {
810 cfg = kzalloc(sizeof(*cfg), GFP_KERNEL);
811 if (!cfg)
812 return -ENOMEM;
813 /* Will be ANDed with DART capabilities */
814 cfg->supports_bypass = true;
815 }
816 dev_iommu_priv_set(dev, cfg);
817
818 cfg_dart = cfg->stream_maps[0].dart;
819 if (cfg_dart) {
820 if (cfg_dart->pgsize != dart->pgsize)
821 return -EINVAL;
822 if (cfg_dart->ias != dart->ias)
823 return -EINVAL;
824 }
825
826 cfg->supports_bypass &= dart->supports_bypass;
827
828 for (i = 0; i < MAX_DARTS_PER_DEVICE; ++i) {
829 if (cfg->stream_maps[i].dart == dart) {
830 set_bit(sid, cfg->stream_maps[i].sidmap);
831 return 0;
832 }
833 }
834 for (i = 0; i < MAX_DARTS_PER_DEVICE; ++i) {
835 if (!cfg->stream_maps[i].dart) {
836 cfg->stream_maps[i].dart = dart;
837 set_bit(sid, cfg->stream_maps[i].sidmap);
838 return 0;
839 }
840 }
841
842 return -EINVAL;
843 }
844
845 static DEFINE_MUTEX(apple_dart_groups_lock);
846
apple_dart_release_group(void * iommu_data)847 static void apple_dart_release_group(void *iommu_data)
848 {
849 int i, sid;
850 struct apple_dart_stream_map *stream_map;
851 struct apple_dart_master_cfg *group_master_cfg = iommu_data;
852
853 mutex_lock(&apple_dart_groups_lock);
854
855 for_each_stream_map(i, group_master_cfg, stream_map)
856 for_each_set_bit(sid, stream_map->sidmap, stream_map->dart->num_streams)
857 stream_map->dart->sid2group[sid] = NULL;
858
859 kfree(iommu_data);
860 mutex_unlock(&apple_dart_groups_lock);
861 }
862
apple_dart_merge_master_cfg(struct apple_dart_master_cfg * dst,struct apple_dart_master_cfg * src)863 static int apple_dart_merge_master_cfg(struct apple_dart_master_cfg *dst,
864 struct apple_dart_master_cfg *src)
865 {
866 /*
867 * We know that this function is only called for groups returned from
868 * pci_device_group and that all Apple Silicon platforms never spread
869 * PCIe devices from the same bus across multiple DARTs such that we can
870 * just assume that both src and dst only have the same single DART.
871 */
872 if (src->stream_maps[1].dart)
873 return -EINVAL;
874 if (dst->stream_maps[1].dart)
875 return -EINVAL;
876 if (src->stream_maps[0].dart != dst->stream_maps[0].dart)
877 return -EINVAL;
878
879 bitmap_or(dst->stream_maps[0].sidmap,
880 dst->stream_maps[0].sidmap,
881 src->stream_maps[0].sidmap,
882 dst->stream_maps[0].dart->num_streams);
883 return 0;
884 }
885
apple_dart_device_group(struct device * dev)886 static struct iommu_group *apple_dart_device_group(struct device *dev)
887 {
888 int i, sid;
889 struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev);
890 struct apple_dart_stream_map *stream_map;
891 struct apple_dart_master_cfg *group_master_cfg;
892 struct iommu_group *group = NULL;
893 struct iommu_group *res = ERR_PTR(-EINVAL);
894
895 mutex_lock(&apple_dart_groups_lock);
896
897 for_each_stream_map(i, cfg, stream_map) {
898 for_each_set_bit(sid, stream_map->sidmap, stream_map->dart->num_streams) {
899 struct iommu_group *stream_group =
900 stream_map->dart->sid2group[sid];
901
902 if (group && group != stream_group) {
903 res = ERR_PTR(-EINVAL);
904 goto out;
905 }
906
907 group = stream_group;
908 }
909 }
910
911 if (group) {
912 res = iommu_group_ref_get(group);
913 goto out;
914 }
915
916 #ifdef CONFIG_PCI
917 if (dev_is_pci(dev))
918 group = pci_device_group(dev);
919 else
920 #endif
921 group = generic_device_group(dev);
922
923 res = ERR_PTR(-ENOMEM);
924 if (!group)
925 goto out;
926
927 group_master_cfg = iommu_group_get_iommudata(group);
928 if (group_master_cfg) {
929 int ret;
930
931 ret = apple_dart_merge_master_cfg(group_master_cfg, cfg);
932 if (ret) {
933 dev_err(dev, "Failed to merge DART IOMMU groups.\n");
934 iommu_group_put(group);
935 res = ERR_PTR(ret);
936 goto out;
937 }
938 } else {
939 group_master_cfg = kmemdup(cfg, sizeof(*group_master_cfg),
940 GFP_KERNEL);
941 if (!group_master_cfg) {
942 iommu_group_put(group);
943 goto out;
944 }
945
946 iommu_group_set_iommudata(group, group_master_cfg,
947 apple_dart_release_group);
948 }
949
950 for_each_stream_map(i, cfg, stream_map)
951 for_each_set_bit(sid, stream_map->sidmap, stream_map->dart->num_streams)
952 stream_map->dart->sid2group[sid] = group;
953
954 res = group;
955
956 out:
957 mutex_unlock(&apple_dart_groups_lock);
958 return res;
959 }
960
apple_dart_def_domain_type(struct device * dev)961 static int apple_dart_def_domain_type(struct device *dev)
962 {
963 struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev);
964
965 if (cfg->stream_maps[0].dart->pgsize > PAGE_SIZE)
966 return IOMMU_DOMAIN_IDENTITY;
967 if (!cfg->supports_bypass)
968 return IOMMU_DOMAIN_DMA;
969
970 return 0;
971 }
972
973 #ifndef CONFIG_PCIE_APPLE_MSI_DOORBELL_ADDR
974 /* Keep things compiling when CONFIG_PCI_APPLE isn't selected */
975 #define CONFIG_PCIE_APPLE_MSI_DOORBELL_ADDR 0
976 #endif
977 #define DOORBELL_ADDR (CONFIG_PCIE_APPLE_MSI_DOORBELL_ADDR & PAGE_MASK)
978
apple_dart_get_resv_regions(struct device * dev,struct list_head * head)979 static void apple_dart_get_resv_regions(struct device *dev,
980 struct list_head *head)
981 {
982 if (IS_ENABLED(CONFIG_PCIE_APPLE) && dev_is_pci(dev)) {
983 struct iommu_resv_region *region;
984 int prot = IOMMU_WRITE | IOMMU_NOEXEC | IOMMU_MMIO;
985
986 region = iommu_alloc_resv_region(DOORBELL_ADDR,
987 PAGE_SIZE, prot,
988 IOMMU_RESV_MSI, GFP_KERNEL);
989 if (!region)
990 return;
991
992 list_add_tail(®ion->list, head);
993 }
994
995 iommu_dma_get_resv_regions(dev, head);
996 }
997
998 static const struct iommu_ops apple_dart_iommu_ops = {
999 .identity_domain = &apple_dart_identity_domain,
1000 .blocked_domain = &apple_dart_blocked_domain,
1001 .domain_alloc_paging = apple_dart_domain_alloc_paging,
1002 .probe_device = apple_dart_probe_device,
1003 .release_device = apple_dart_release_device,
1004 .device_group = apple_dart_device_group,
1005 .of_xlate = apple_dart_of_xlate,
1006 .def_domain_type = apple_dart_def_domain_type,
1007 .get_resv_regions = apple_dart_get_resv_regions,
1008 .owner = THIS_MODULE,
1009 .default_domain_ops = &(const struct iommu_domain_ops) {
1010 .attach_dev = apple_dart_attach_dev_paging,
1011 .map_pages = apple_dart_map_pages,
1012 .unmap_pages = apple_dart_unmap_pages,
1013 .flush_iotlb_all = apple_dart_flush_iotlb_all,
1014 .iotlb_sync = apple_dart_iotlb_sync,
1015 .iotlb_sync_map = apple_dart_iotlb_sync_map,
1016 .iova_to_phys = apple_dart_iova_to_phys,
1017 .free = apple_dart_domain_free,
1018 }
1019 };
1020
apple_dart_t8020_irq(int irq,void * dev)1021 static irqreturn_t apple_dart_t8020_irq(int irq, void *dev)
1022 {
1023 struct apple_dart *dart = dev;
1024 const char *fault_name = NULL;
1025 u32 error = readl(dart->regs + DART_T8020_ERROR);
1026 u32 error_code = FIELD_GET(DART_T8020_ERROR_CODE, error);
1027 u32 addr_lo = readl(dart->regs + DART_T8020_ERROR_ADDR_LO);
1028 u32 addr_hi = readl(dart->regs + DART_T8020_ERROR_ADDR_HI);
1029 u64 addr = addr_lo | (((u64)addr_hi) << 32);
1030 u8 stream_idx = FIELD_GET(DART_T8020_ERROR_STREAM, error);
1031
1032 if (!(error & DART_T8020_ERROR_FLAG))
1033 return IRQ_NONE;
1034
1035 /* there should only be a single bit set but let's use == to be sure */
1036 if (error_code == DART_T8020_ERROR_READ_FAULT)
1037 fault_name = "READ FAULT";
1038 else if (error_code == DART_T8020_ERROR_WRITE_FAULT)
1039 fault_name = "WRITE FAULT";
1040 else if (error_code == DART_T8020_ERROR_NO_PTE)
1041 fault_name = "NO PTE FOR IOVA";
1042 else if (error_code == DART_T8020_ERROR_NO_PMD)
1043 fault_name = "NO PMD FOR IOVA";
1044 else if (error_code == DART_T8020_ERROR_NO_TTBR)
1045 fault_name = "NO TTBR FOR IOVA";
1046 else
1047 fault_name = "unknown";
1048
1049 dev_err_ratelimited(
1050 dart->dev,
1051 "translation fault: status:0x%x stream:%d code:0x%x (%s) at 0x%llx",
1052 error, stream_idx, error_code, fault_name, addr);
1053
1054 writel(error, dart->regs + DART_T8020_ERROR);
1055 return IRQ_HANDLED;
1056 }
1057
apple_dart_t8110_irq(int irq,void * dev)1058 static irqreturn_t apple_dart_t8110_irq(int irq, void *dev)
1059 {
1060 struct apple_dart *dart = dev;
1061 const char *fault_name = NULL;
1062 u32 error = readl(dart->regs + DART_T8110_ERROR);
1063 u32 error_code = FIELD_GET(DART_T8110_ERROR_CODE, error);
1064 u32 addr_lo = readl(dart->regs + DART_T8110_ERROR_ADDR_LO);
1065 u32 addr_hi = readl(dart->regs + DART_T8110_ERROR_ADDR_HI);
1066 u64 addr = addr_lo | (((u64)addr_hi) << 32);
1067 u8 stream_idx = FIELD_GET(DART_T8110_ERROR_STREAM, error);
1068
1069 if (!(error & DART_T8110_ERROR_FLAG))
1070 return IRQ_NONE;
1071
1072 /* there should only be a single bit set but let's use == to be sure */
1073 if (error_code == DART_T8110_ERROR_READ_FAULT)
1074 fault_name = "READ FAULT";
1075 else if (error_code == DART_T8110_ERROR_WRITE_FAULT)
1076 fault_name = "WRITE FAULT";
1077 else if (error_code == DART_T8110_ERROR_NO_PTE)
1078 fault_name = "NO PTE FOR IOVA";
1079 else if (error_code == DART_T8110_ERROR_NO_PMD)
1080 fault_name = "NO PMD FOR IOVA";
1081 else if (error_code == DART_T8110_ERROR_NO_PGD)
1082 fault_name = "NO PGD FOR IOVA";
1083 else if (error_code == DART_T8110_ERROR_NO_TTBR)
1084 fault_name = "NO TTBR FOR IOVA";
1085 else
1086 fault_name = "unknown";
1087
1088 dev_err_ratelimited(
1089 dart->dev,
1090 "translation fault: status:0x%x stream:%d code:0x%x (%s) at 0x%llx",
1091 error, stream_idx, error_code, fault_name, addr);
1092
1093 writel(error, dart->regs + DART_T8110_ERROR);
1094 for (int i = 0; i < BITS_TO_U32(dart->num_streams); i++)
1095 writel(U32_MAX, dart->regs + DART_T8110_ERROR_STREAMS + 4 * i);
1096
1097 return IRQ_HANDLED;
1098 }
1099
apple_dart_probe(struct platform_device * pdev)1100 static int apple_dart_probe(struct platform_device *pdev)
1101 {
1102 int ret;
1103 u32 dart_params[4];
1104 struct resource *res;
1105 struct apple_dart *dart;
1106 struct device *dev = &pdev->dev;
1107
1108 dart = devm_kzalloc(dev, sizeof(*dart), GFP_KERNEL);
1109 if (!dart)
1110 return -ENOMEM;
1111
1112 dart->dev = dev;
1113 dart->hw = of_device_get_match_data(dev);
1114 spin_lock_init(&dart->lock);
1115
1116 dart->regs = devm_platform_get_and_ioremap_resource(pdev, 0, &res);
1117 if (IS_ERR(dart->regs))
1118 return PTR_ERR(dart->regs);
1119
1120 if (resource_size(res) < 0x4000) {
1121 dev_err(dev, "MMIO region too small (%pr)\n", res);
1122 return -EINVAL;
1123 }
1124
1125 dart->irq = platform_get_irq(pdev, 0);
1126 if (dart->irq < 0)
1127 return -ENODEV;
1128
1129 ret = devm_clk_bulk_get_all(dev, &dart->clks);
1130 if (ret < 0)
1131 return ret;
1132 dart->num_clks = ret;
1133
1134 ret = clk_bulk_prepare_enable(dart->num_clks, dart->clks);
1135 if (ret)
1136 return ret;
1137
1138 dart_params[0] = readl(dart->regs + DART_PARAMS1);
1139 dart_params[1] = readl(dart->regs + DART_PARAMS2);
1140 dart->pgsize = 1 << FIELD_GET(DART_PARAMS1_PAGE_SHIFT, dart_params[0]);
1141 dart->supports_bypass = dart_params[1] & DART_PARAMS2_BYPASS_SUPPORT;
1142
1143 switch (dart->hw->type) {
1144 case DART_T8020:
1145 case DART_T6000:
1146 dart->ias = 32;
1147 dart->oas = dart->hw->oas;
1148 dart->num_streams = dart->hw->max_sid_count;
1149 break;
1150
1151 case DART_T8110:
1152 dart_params[2] = readl(dart->regs + DART_T8110_PARAMS3);
1153 dart_params[3] = readl(dart->regs + DART_T8110_PARAMS4);
1154 dart->ias = FIELD_GET(DART_T8110_PARAMS3_VA_WIDTH, dart_params[2]);
1155 dart->oas = FIELD_GET(DART_T8110_PARAMS3_PA_WIDTH, dart_params[2]);
1156 dart->num_streams = FIELD_GET(DART_T8110_PARAMS4_NUM_SIDS, dart_params[3]);
1157 dart->four_level = dart->ias > 36;
1158 break;
1159 }
1160
1161 if (dart->num_streams > DART_MAX_STREAMS) {
1162 dev_err(&pdev->dev, "Too many streams (%d > %d)\n",
1163 dart->num_streams, DART_MAX_STREAMS);
1164 ret = -EINVAL;
1165 goto err_clk_disable;
1166 }
1167
1168 ret = apple_dart_hw_reset(dart);
1169 if (ret)
1170 goto err_clk_disable;
1171
1172 ret = request_irq(dart->irq, dart->hw->irq_handler, IRQF_SHARED,
1173 "apple-dart fault handler", dart);
1174 if (ret)
1175 goto err_clk_disable;
1176
1177 platform_set_drvdata(pdev, dart);
1178
1179 ret = iommu_device_sysfs_add(&dart->iommu, dev, NULL, "apple-dart.%s",
1180 dev_name(&pdev->dev));
1181 if (ret)
1182 goto err_free_irq;
1183
1184 ret = iommu_device_register(&dart->iommu, &apple_dart_iommu_ops, dev);
1185 if (ret)
1186 goto err_sysfs_remove;
1187
1188 dev_info(
1189 &pdev->dev,
1190 "DART [pagesize %x, %d streams, bypass support: %d, bypass forced: %d, AS %d -> %d] initialized\n",
1191 dart->pgsize, dart->num_streams, dart->supports_bypass,
1192 dart->pgsize > PAGE_SIZE, dart->ias, dart->oas);
1193 return 0;
1194
1195 err_sysfs_remove:
1196 iommu_device_sysfs_remove(&dart->iommu);
1197 err_free_irq:
1198 free_irq(dart->irq, dart);
1199 err_clk_disable:
1200 clk_bulk_disable_unprepare(dart->num_clks, dart->clks);
1201
1202 return ret;
1203 }
1204
apple_dart_remove(struct platform_device * pdev)1205 static void apple_dart_remove(struct platform_device *pdev)
1206 {
1207 struct apple_dart *dart = platform_get_drvdata(pdev);
1208
1209 apple_dart_hw_reset(dart);
1210 free_irq(dart->irq, dart);
1211
1212 iommu_device_unregister(&dart->iommu);
1213 iommu_device_sysfs_remove(&dart->iommu);
1214
1215 clk_bulk_disable_unprepare(dart->num_clks, dart->clks);
1216 }
1217
1218 static const struct apple_dart_hw apple_dart_hw_t8103 = {
1219 .type = DART_T8020,
1220 .irq_handler = apple_dart_t8020_irq,
1221 .invalidate_tlb = apple_dart_t8020_hw_invalidate_tlb,
1222 .oas = 36,
1223 .fmt = APPLE_DART,
1224 .max_sid_count = 16,
1225
1226 .enable_streams = DART_T8020_STREAMS_ENABLE,
1227 .lock = DART_T8020_CONFIG,
1228 .lock_bit = DART_T8020_CONFIG_LOCK,
1229
1230 .error = DART_T8020_ERROR,
1231
1232 .tcr = DART_T8020_TCR,
1233 .tcr_enabled = DART_T8020_TCR_TRANSLATE_ENABLE,
1234 .tcr_disabled = 0,
1235 .tcr_bypass = DART_T8020_TCR_BYPASS_DAPF | DART_T8020_TCR_BYPASS_DART,
1236
1237 .ttbr = DART_T8020_TTBR,
1238 .ttbr_valid = DART_T8020_TTBR_VALID,
1239 .ttbr_addr_field_shift = DART_T8020_TTBR_ADDR_FIELD_SHIFT,
1240 .ttbr_shift = DART_T8020_TTBR_SHIFT,
1241 .ttbr_count = 4,
1242 };
1243
1244 static const struct apple_dart_hw apple_dart_hw_t8103_usb4 = {
1245 .type = DART_T8020,
1246 .irq_handler = apple_dart_t8020_irq,
1247 .invalidate_tlb = apple_dart_t8020_hw_invalidate_tlb,
1248 .oas = 36,
1249 .fmt = APPLE_DART,
1250 .max_sid_count = 64,
1251
1252 .enable_streams = DART_T8020_STREAMS_ENABLE,
1253 .lock = DART_T8020_CONFIG,
1254 .lock_bit = DART_T8020_CONFIG_LOCK,
1255
1256 .error = DART_T8020_ERROR,
1257
1258 .tcr = DART_T8020_TCR,
1259 .tcr_enabled = DART_T8020_TCR_TRANSLATE_ENABLE,
1260 .tcr_disabled = 0,
1261 .tcr_bypass = 0,
1262
1263 .ttbr = DART_T8020_USB4_TTBR,
1264 .ttbr_valid = DART_T8020_TTBR_VALID,
1265 .ttbr_addr_field_shift = DART_T8020_TTBR_ADDR_FIELD_SHIFT,
1266 .ttbr_shift = DART_T8020_TTBR_SHIFT,
1267 .ttbr_count = 4,
1268 };
1269
1270 static const struct apple_dart_hw apple_dart_hw_t6000 = {
1271 .type = DART_T6000,
1272 .irq_handler = apple_dart_t8020_irq,
1273 .invalidate_tlb = apple_dart_t8020_hw_invalidate_tlb,
1274 .oas = 42,
1275 .fmt = APPLE_DART2,
1276 .max_sid_count = 16,
1277
1278 .enable_streams = DART_T8020_STREAMS_ENABLE,
1279 .lock = DART_T8020_CONFIG,
1280 .lock_bit = DART_T8020_CONFIG_LOCK,
1281
1282 .error = DART_T8020_ERROR,
1283
1284 .tcr = DART_T8020_TCR,
1285 .tcr_enabled = DART_T8020_TCR_TRANSLATE_ENABLE,
1286 .tcr_disabled = 0,
1287 .tcr_bypass = DART_T8020_TCR_BYPASS_DAPF | DART_T8020_TCR_BYPASS_DART,
1288
1289 .ttbr = DART_T8020_TTBR,
1290 .ttbr_valid = DART_T8020_TTBR_VALID,
1291 .ttbr_addr_field_shift = DART_T8020_TTBR_ADDR_FIELD_SHIFT,
1292 .ttbr_shift = DART_T8020_TTBR_SHIFT,
1293 .ttbr_count = 4,
1294 };
1295
1296 static const struct apple_dart_hw apple_dart_hw_t8110 = {
1297 .type = DART_T8110,
1298 .irq_handler = apple_dart_t8110_irq,
1299 .invalidate_tlb = apple_dart_t8110_hw_invalidate_tlb,
1300 .fmt = APPLE_DART2,
1301 .max_sid_count = 256,
1302
1303 .enable_streams = DART_T8110_ENABLE_STREAMS,
1304 .lock = DART_T8110_PROTECT,
1305 .lock_bit = DART_T8110_PROTECT_TTBR_TCR,
1306
1307 .error = DART_T8110_ERROR,
1308
1309 .tcr = DART_T8110_TCR,
1310 .tcr_enabled = DART_T8110_TCR_TRANSLATE_ENABLE,
1311 .tcr_disabled = 0,
1312 .tcr_bypass = DART_T8110_TCR_BYPASS_DAPF | DART_T8110_TCR_BYPASS_DART,
1313 .tcr_4level = DART_T8110_TCR_FOUR_LEVEL,
1314
1315 .ttbr = DART_T8110_TTBR,
1316 .ttbr_valid = DART_T8110_TTBR_VALID,
1317 .ttbr_addr_field_shift = DART_T8110_TTBR_ADDR_FIELD_SHIFT,
1318 .ttbr_shift = DART_T8110_TTBR_SHIFT,
1319 .ttbr_count = 1,
1320 };
1321
apple_dart_suspend(struct device * dev)1322 static __maybe_unused int apple_dart_suspend(struct device *dev)
1323 {
1324 struct apple_dart *dart = dev_get_drvdata(dev);
1325 unsigned int sid, idx;
1326
1327 for (sid = 0; sid < dart->num_streams; sid++) {
1328 dart->save_tcr[sid] = readl(dart->regs + DART_TCR(dart, sid));
1329 for (idx = 0; idx < dart->hw->ttbr_count; idx++)
1330 dart->save_ttbr[sid][idx] =
1331 readl(dart->regs + DART_TTBR(dart, sid, idx));
1332 }
1333
1334 return 0;
1335 }
1336
apple_dart_resume(struct device * dev)1337 static __maybe_unused int apple_dart_resume(struct device *dev)
1338 {
1339 struct apple_dart *dart = dev_get_drvdata(dev);
1340 unsigned int sid, idx;
1341 int ret;
1342
1343 ret = apple_dart_hw_reset(dart);
1344 if (ret) {
1345 dev_err(dev, "Failed to reset DART on resume\n");
1346 return ret;
1347 }
1348
1349 for (sid = 0; sid < dart->num_streams; sid++) {
1350 for (idx = 0; idx < dart->hw->ttbr_count; idx++)
1351 writel(dart->save_ttbr[sid][idx],
1352 dart->regs + DART_TTBR(dart, sid, idx));
1353 writel(dart->save_tcr[sid], dart->regs + DART_TCR(dart, sid));
1354 }
1355
1356 return 0;
1357 }
1358
1359 static DEFINE_SIMPLE_DEV_PM_OPS(apple_dart_pm_ops, apple_dart_suspend, apple_dart_resume);
1360
1361 static const struct of_device_id apple_dart_of_match[] = {
1362 { .compatible = "apple,t8103-dart", .data = &apple_dart_hw_t8103 },
1363 { .compatible = "apple,t8103-usb4-dart", .data = &apple_dart_hw_t8103_usb4 },
1364 { .compatible = "apple,t8110-dart", .data = &apple_dart_hw_t8110 },
1365 { .compatible = "apple,t6000-dart", .data = &apple_dart_hw_t6000 },
1366 {},
1367 };
1368 MODULE_DEVICE_TABLE(of, apple_dart_of_match);
1369
1370 static struct platform_driver apple_dart_driver = {
1371 .driver = {
1372 .name = "apple-dart",
1373 .of_match_table = apple_dart_of_match,
1374 .suppress_bind_attrs = true,
1375 .pm = pm_sleep_ptr(&apple_dart_pm_ops),
1376 },
1377 .probe = apple_dart_probe,
1378 .remove = apple_dart_remove,
1379 };
1380
1381 module_platform_driver(apple_dart_driver);
1382
1383 MODULE_DESCRIPTION("IOMMU API for Apple's DART");
1384 MODULE_AUTHOR("Sven Peter <sven@svenpeter.dev>");
1385 MODULE_LICENSE("GPL v2");
1386