1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * IOMMU API for ARM architected SMMU implementations.
4 *
5 * Copyright (C) 2013 ARM Limited
6 *
7 * Author: Will Deacon <will.deacon@arm.com>
8 *
9 * This driver currently supports:
10 * - SMMUv1 and v2 implementations
11 * - Stream-matching and stream-indexing
12 * - v7/v8 long-descriptor format
13 * - Non-secure access to the SMMU
14 * - Context fault reporting
15 * - Extended Stream ID (16 bit)
16 */
17
18 #define pr_fmt(fmt) "arm-smmu: " fmt
19
20 #include <linux/acpi.h>
21 #include <linux/acpi_iort.h>
22 #include <linux/bitfield.h>
23 #include <linux/delay.h>
24 #include <linux/dma-mapping.h>
25 #include <linux/err.h>
26 #include <linux/interrupt.h>
27 #include <linux/io.h>
28 #include <linux/iopoll.h>
29 #include <linux/module.h>
30 #include <linux/of.h>
31 #include <linux/of_address.h>
32 #include <linux/pci.h>
33 #include <linux/platform_device.h>
34 #include <linux/pm_runtime.h>
35 #include <linux/ratelimit.h>
36 #include <linux/slab.h>
37
38 #include <linux/fsl/mc.h>
39
40 #include "arm-smmu.h"
41 #include "../../dma-iommu.h"
42
43 /*
44 * Apparently, some Qualcomm arm64 platforms which appear to expose their SMMU
45 * global register space are still, in fact, using a hypervisor to mediate it
46 * by trapping and emulating register accesses. Sadly, some deployed versions
47 * of said trapping code have bugs wherein they go horribly wrong for stores
48 * using r31 (i.e. XZR/WZR) as the source register.
49 */
50 #define QCOM_DUMMY_VAL -1
51
52 #define MSI_IOVA_BASE 0x8000000
53 #define MSI_IOVA_LENGTH 0x100000
54
55 static int force_stage;
56 module_param(force_stage, int, S_IRUGO);
57 MODULE_PARM_DESC(force_stage,
58 "Force SMMU mappings to be installed at a particular stage of translation. A value of '1' or '2' forces the corresponding stage. All other values are ignored (i.e. no stage is forced). Note that selecting a specific stage will disable support for nested translation.");
59 static bool disable_bypass =
60 IS_ENABLED(CONFIG_ARM_SMMU_DISABLE_BYPASS_BY_DEFAULT);
61 module_param(disable_bypass, bool, S_IRUGO);
62 MODULE_PARM_DESC(disable_bypass,
63 "Disable bypass streams such that incoming transactions from devices that are not attached to an iommu domain will report an abort back to the device and will not be allowed to pass through the SMMU.");
64
65 #define s2cr_init_val (struct arm_smmu_s2cr){ \
66 .type = disable_bypass ? S2CR_TYPE_FAULT : S2CR_TYPE_BYPASS, \
67 }
68
69 static bool using_legacy_binding, using_generic_binding;
70
arm_smmu_rpm_get(struct arm_smmu_device * smmu)71 static inline int arm_smmu_rpm_get(struct arm_smmu_device *smmu)
72 {
73 if (pm_runtime_enabled(smmu->dev))
74 return pm_runtime_resume_and_get(smmu->dev);
75
76 return 0;
77 }
78
arm_smmu_rpm_put(struct arm_smmu_device * smmu)79 static inline void arm_smmu_rpm_put(struct arm_smmu_device *smmu)
80 {
81 if (pm_runtime_enabled(smmu->dev))
82 pm_runtime_put_autosuspend(smmu->dev);
83 }
84
arm_smmu_rpm_use_autosuspend(struct arm_smmu_device * smmu)85 static void arm_smmu_rpm_use_autosuspend(struct arm_smmu_device *smmu)
86 {
87 /*
88 * Setup an autosuspend delay to avoid bouncing runpm state.
89 * Otherwise, if a driver for a suspended consumer device
90 * unmaps buffers, it will runpm resume/suspend for each one.
91 *
92 * For example, when used by a GPU device, when an application
93 * or game exits, it can trigger unmapping 100s or 1000s of
94 * buffers. With a runpm cycle for each buffer, that adds up
95 * to 5-10sec worth of reprogramming the context bank, while
96 * the system appears to be locked up to the user.
97 */
98 pm_runtime_set_autosuspend_delay(smmu->dev, 20);
99 pm_runtime_use_autosuspend(smmu->dev);
100 }
101
to_smmu_domain(struct iommu_domain * dom)102 static struct arm_smmu_domain *to_smmu_domain(struct iommu_domain *dom)
103 {
104 return container_of(dom, struct arm_smmu_domain, domain);
105 }
106
107 static struct platform_driver arm_smmu_driver;
108 static struct iommu_ops arm_smmu_ops;
109
110 #ifdef CONFIG_ARM_SMMU_LEGACY_DT_BINDINGS
dev_get_dev_node(struct device * dev)111 static struct device_node *dev_get_dev_node(struct device *dev)
112 {
113 if (dev_is_pci(dev)) {
114 struct pci_bus *bus = to_pci_dev(dev)->bus;
115
116 while (!pci_is_root_bus(bus))
117 bus = bus->parent;
118 return of_node_get(bus->bridge->parent->of_node);
119 }
120
121 return of_node_get(dev->of_node);
122 }
123
__arm_smmu_get_pci_sid(struct pci_dev * pdev,u16 alias,void * data)124 static int __arm_smmu_get_pci_sid(struct pci_dev *pdev, u16 alias, void *data)
125 {
126 *((__be32 *)data) = cpu_to_be32(alias);
127 return 0; /* Continue walking */
128 }
129
__find_legacy_master_phandle(struct device * dev,void * data)130 static int __find_legacy_master_phandle(struct device *dev, void *data)
131 {
132 struct of_phandle_iterator *it = *(void **)data;
133 struct device_node *np = it->node;
134 int err;
135
136 of_for_each_phandle(it, err, dev->of_node, "mmu-masters",
137 "#stream-id-cells", -1)
138 if (it->node == np) {
139 *(void **)data = dev;
140 return 1;
141 }
142 it->node = np;
143 return err == -ENOENT ? 0 : err;
144 }
145
arm_smmu_register_legacy_master(struct device * dev,struct arm_smmu_device ** smmu)146 static int arm_smmu_register_legacy_master(struct device *dev,
147 struct arm_smmu_device **smmu)
148 {
149 struct device *smmu_dev;
150 struct device_node *np;
151 struct of_phandle_iterator it;
152 void *data = ⁢
153 u32 *sids;
154 __be32 pci_sid;
155 int err;
156
157 np = dev_get_dev_node(dev);
158 if (!np || !of_property_present(np, "#stream-id-cells")) {
159 of_node_put(np);
160 return -ENODEV;
161 }
162
163 it.node = np;
164 err = driver_for_each_device(&arm_smmu_driver.driver, NULL, &data,
165 __find_legacy_master_phandle);
166 smmu_dev = data;
167 of_node_put(np);
168 if (err == 0)
169 return -ENODEV;
170 if (err < 0)
171 return err;
172
173 if (dev_is_pci(dev)) {
174 /* "mmu-masters" assumes Stream ID == Requester ID */
175 pci_for_each_dma_alias(to_pci_dev(dev), __arm_smmu_get_pci_sid,
176 &pci_sid);
177 it.cur = &pci_sid;
178 it.cur_count = 1;
179 }
180
181 err = iommu_fwspec_init(dev, &smmu_dev->of_node->fwnode,
182 &arm_smmu_ops);
183 if (err)
184 return err;
185
186 sids = kcalloc(it.cur_count, sizeof(*sids), GFP_KERNEL);
187 if (!sids)
188 return -ENOMEM;
189
190 *smmu = dev_get_drvdata(smmu_dev);
191 of_phandle_iterator_args(&it, sids, it.cur_count);
192 err = iommu_fwspec_add_ids(dev, sids, it.cur_count);
193 kfree(sids);
194 return err;
195 }
196 #else
arm_smmu_register_legacy_master(struct device * dev,struct arm_smmu_device ** smmu)197 static int arm_smmu_register_legacy_master(struct device *dev,
198 struct arm_smmu_device **smmu)
199 {
200 return -ENODEV;
201 }
202 #endif /* CONFIG_ARM_SMMU_LEGACY_DT_BINDINGS */
203
__arm_smmu_free_bitmap(unsigned long * map,int idx)204 static void __arm_smmu_free_bitmap(unsigned long *map, int idx)
205 {
206 clear_bit(idx, map);
207 }
208
209 /* Wait for any pending TLB invalidations to complete */
__arm_smmu_tlb_sync(struct arm_smmu_device * smmu,int page,int sync,int status)210 static void __arm_smmu_tlb_sync(struct arm_smmu_device *smmu, int page,
211 int sync, int status)
212 {
213 unsigned int spin_cnt, delay;
214 u32 reg;
215
216 if (smmu->impl && unlikely(smmu->impl->tlb_sync))
217 return smmu->impl->tlb_sync(smmu, page, sync, status);
218
219 arm_smmu_writel(smmu, page, sync, QCOM_DUMMY_VAL);
220 for (delay = 1; delay < TLB_LOOP_TIMEOUT; delay *= 2) {
221 for (spin_cnt = TLB_SPIN_COUNT; spin_cnt > 0; spin_cnt--) {
222 reg = arm_smmu_readl(smmu, page, status);
223 if (!(reg & ARM_SMMU_sTLBGSTATUS_GSACTIVE))
224 return;
225 cpu_relax();
226 }
227 udelay(delay);
228 }
229 dev_err_ratelimited(smmu->dev,
230 "TLB sync timed out -- SMMU may be deadlocked\n");
231 }
232
arm_smmu_tlb_sync_global(struct arm_smmu_device * smmu)233 static void arm_smmu_tlb_sync_global(struct arm_smmu_device *smmu)
234 {
235 unsigned long flags;
236
237 spin_lock_irqsave(&smmu->global_sync_lock, flags);
238 __arm_smmu_tlb_sync(smmu, ARM_SMMU_GR0, ARM_SMMU_GR0_sTLBGSYNC,
239 ARM_SMMU_GR0_sTLBGSTATUS);
240 spin_unlock_irqrestore(&smmu->global_sync_lock, flags);
241 }
242
arm_smmu_tlb_sync_context(struct arm_smmu_domain * smmu_domain)243 static void arm_smmu_tlb_sync_context(struct arm_smmu_domain *smmu_domain)
244 {
245 struct arm_smmu_device *smmu = smmu_domain->smmu;
246 unsigned long flags;
247
248 spin_lock_irqsave(&smmu_domain->cb_lock, flags);
249 __arm_smmu_tlb_sync(smmu, ARM_SMMU_CB(smmu, smmu_domain->cfg.cbndx),
250 ARM_SMMU_CB_TLBSYNC, ARM_SMMU_CB_TLBSTATUS);
251 spin_unlock_irqrestore(&smmu_domain->cb_lock, flags);
252 }
253
arm_smmu_tlb_inv_context_s1(void * cookie)254 static void arm_smmu_tlb_inv_context_s1(void *cookie)
255 {
256 struct arm_smmu_domain *smmu_domain = cookie;
257 /*
258 * The TLBI write may be relaxed, so ensure that PTEs cleared by the
259 * current CPU are visible beforehand.
260 */
261 wmb();
262 arm_smmu_cb_write(smmu_domain->smmu, smmu_domain->cfg.cbndx,
263 ARM_SMMU_CB_S1_TLBIASID, smmu_domain->cfg.asid);
264 arm_smmu_tlb_sync_context(smmu_domain);
265 }
266
arm_smmu_tlb_inv_context_s2(void * cookie)267 static void arm_smmu_tlb_inv_context_s2(void *cookie)
268 {
269 struct arm_smmu_domain *smmu_domain = cookie;
270 struct arm_smmu_device *smmu = smmu_domain->smmu;
271
272 /* See above */
273 wmb();
274 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_TLBIVMID, smmu_domain->cfg.vmid);
275 arm_smmu_tlb_sync_global(smmu);
276 }
277
arm_smmu_tlb_inv_range_s1(unsigned long iova,size_t size,size_t granule,void * cookie,int reg)278 static void arm_smmu_tlb_inv_range_s1(unsigned long iova, size_t size,
279 size_t granule, void *cookie, int reg)
280 {
281 struct arm_smmu_domain *smmu_domain = cookie;
282 struct arm_smmu_device *smmu = smmu_domain->smmu;
283 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
284 int idx = cfg->cbndx;
285
286 if (smmu->features & ARM_SMMU_FEAT_COHERENT_WALK)
287 wmb();
288
289 if (cfg->fmt != ARM_SMMU_CTX_FMT_AARCH64) {
290 iova = (iova >> 12) << 12;
291 iova |= cfg->asid;
292 do {
293 arm_smmu_cb_write(smmu, idx, reg, iova);
294 iova += granule;
295 } while (size -= granule);
296 } else {
297 iova >>= 12;
298 iova |= (u64)cfg->asid << 48;
299 do {
300 arm_smmu_cb_writeq(smmu, idx, reg, iova);
301 iova += granule >> 12;
302 } while (size -= granule);
303 }
304 }
305
arm_smmu_tlb_inv_range_s2(unsigned long iova,size_t size,size_t granule,void * cookie,int reg)306 static void arm_smmu_tlb_inv_range_s2(unsigned long iova, size_t size,
307 size_t granule, void *cookie, int reg)
308 {
309 struct arm_smmu_domain *smmu_domain = cookie;
310 struct arm_smmu_device *smmu = smmu_domain->smmu;
311 int idx = smmu_domain->cfg.cbndx;
312
313 if (smmu->features & ARM_SMMU_FEAT_COHERENT_WALK)
314 wmb();
315
316 iova >>= 12;
317 do {
318 if (smmu_domain->cfg.fmt == ARM_SMMU_CTX_FMT_AARCH64)
319 arm_smmu_cb_writeq(smmu, idx, reg, iova);
320 else
321 arm_smmu_cb_write(smmu, idx, reg, iova);
322 iova += granule >> 12;
323 } while (size -= granule);
324 }
325
arm_smmu_tlb_inv_walk_s1(unsigned long iova,size_t size,size_t granule,void * cookie)326 static void arm_smmu_tlb_inv_walk_s1(unsigned long iova, size_t size,
327 size_t granule, void *cookie)
328 {
329 struct arm_smmu_domain *smmu_domain = cookie;
330 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
331
332 if (cfg->flush_walk_prefer_tlbiasid) {
333 arm_smmu_tlb_inv_context_s1(cookie);
334 } else {
335 arm_smmu_tlb_inv_range_s1(iova, size, granule, cookie,
336 ARM_SMMU_CB_S1_TLBIVA);
337 arm_smmu_tlb_sync_context(cookie);
338 }
339 }
340
arm_smmu_tlb_add_page_s1(struct iommu_iotlb_gather * gather,unsigned long iova,size_t granule,void * cookie)341 static void arm_smmu_tlb_add_page_s1(struct iommu_iotlb_gather *gather,
342 unsigned long iova, size_t granule,
343 void *cookie)
344 {
345 arm_smmu_tlb_inv_range_s1(iova, granule, granule, cookie,
346 ARM_SMMU_CB_S1_TLBIVAL);
347 }
348
arm_smmu_tlb_inv_walk_s2(unsigned long iova,size_t size,size_t granule,void * cookie)349 static void arm_smmu_tlb_inv_walk_s2(unsigned long iova, size_t size,
350 size_t granule, void *cookie)
351 {
352 arm_smmu_tlb_inv_range_s2(iova, size, granule, cookie,
353 ARM_SMMU_CB_S2_TLBIIPAS2);
354 arm_smmu_tlb_sync_context(cookie);
355 }
356
arm_smmu_tlb_add_page_s2(struct iommu_iotlb_gather * gather,unsigned long iova,size_t granule,void * cookie)357 static void arm_smmu_tlb_add_page_s2(struct iommu_iotlb_gather *gather,
358 unsigned long iova, size_t granule,
359 void *cookie)
360 {
361 arm_smmu_tlb_inv_range_s2(iova, granule, granule, cookie,
362 ARM_SMMU_CB_S2_TLBIIPAS2L);
363 }
364
arm_smmu_tlb_inv_walk_s2_v1(unsigned long iova,size_t size,size_t granule,void * cookie)365 static void arm_smmu_tlb_inv_walk_s2_v1(unsigned long iova, size_t size,
366 size_t granule, void *cookie)
367 {
368 arm_smmu_tlb_inv_context_s2(cookie);
369 }
370 /*
371 * On MMU-401 at least, the cost of firing off multiple TLBIVMIDs appears
372 * almost negligible, but the benefit of getting the first one in as far ahead
373 * of the sync as possible is significant, hence we don't just make this a
374 * no-op and call arm_smmu_tlb_inv_context_s2() from .iotlb_sync as you might
375 * think.
376 */
arm_smmu_tlb_add_page_s2_v1(struct iommu_iotlb_gather * gather,unsigned long iova,size_t granule,void * cookie)377 static void arm_smmu_tlb_add_page_s2_v1(struct iommu_iotlb_gather *gather,
378 unsigned long iova, size_t granule,
379 void *cookie)
380 {
381 struct arm_smmu_domain *smmu_domain = cookie;
382 struct arm_smmu_device *smmu = smmu_domain->smmu;
383
384 if (smmu->features & ARM_SMMU_FEAT_COHERENT_WALK)
385 wmb();
386
387 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_TLBIVMID, smmu_domain->cfg.vmid);
388 }
389
390 static const struct iommu_flush_ops arm_smmu_s1_tlb_ops = {
391 .tlb_flush_all = arm_smmu_tlb_inv_context_s1,
392 .tlb_flush_walk = arm_smmu_tlb_inv_walk_s1,
393 .tlb_add_page = arm_smmu_tlb_add_page_s1,
394 };
395
396 static const struct iommu_flush_ops arm_smmu_s2_tlb_ops_v2 = {
397 .tlb_flush_all = arm_smmu_tlb_inv_context_s2,
398 .tlb_flush_walk = arm_smmu_tlb_inv_walk_s2,
399 .tlb_add_page = arm_smmu_tlb_add_page_s2,
400 };
401
402 static const struct iommu_flush_ops arm_smmu_s2_tlb_ops_v1 = {
403 .tlb_flush_all = arm_smmu_tlb_inv_context_s2,
404 .tlb_flush_walk = arm_smmu_tlb_inv_walk_s2_v1,
405 .tlb_add_page = arm_smmu_tlb_add_page_s2_v1,
406 };
407
arm_smmu_context_fault(int irq,void * dev)408 static irqreturn_t arm_smmu_context_fault(int irq, void *dev)
409 {
410 u32 fsr, fsynr, cbfrsynra;
411 unsigned long iova;
412 struct arm_smmu_domain *smmu_domain = dev;
413 struct arm_smmu_device *smmu = smmu_domain->smmu;
414 int idx = smmu_domain->cfg.cbndx;
415 int ret;
416
417 fsr = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_FSR);
418 if (!(fsr & ARM_SMMU_FSR_FAULT))
419 return IRQ_NONE;
420
421 fsynr = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_FSYNR0);
422 iova = arm_smmu_cb_readq(smmu, idx, ARM_SMMU_CB_FAR);
423 cbfrsynra = arm_smmu_gr1_read(smmu, ARM_SMMU_GR1_CBFRSYNRA(idx));
424
425 ret = report_iommu_fault(&smmu_domain->domain, NULL, iova,
426 fsynr & ARM_SMMU_FSYNR0_WNR ? IOMMU_FAULT_WRITE : IOMMU_FAULT_READ);
427
428 if (ret == -ENOSYS)
429 dev_err_ratelimited(smmu->dev,
430 "Unhandled context fault: fsr=0x%x, iova=0x%08lx, fsynr=0x%x, cbfrsynra=0x%x, cb=%d\n",
431 fsr, iova, fsynr, cbfrsynra, idx);
432
433 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_FSR, fsr);
434 return IRQ_HANDLED;
435 }
436
arm_smmu_global_fault(int irq,void * dev)437 static irqreturn_t arm_smmu_global_fault(int irq, void *dev)
438 {
439 u32 gfsr, gfsynr0, gfsynr1, gfsynr2;
440 struct arm_smmu_device *smmu = dev;
441 static DEFINE_RATELIMIT_STATE(rs, DEFAULT_RATELIMIT_INTERVAL,
442 DEFAULT_RATELIMIT_BURST);
443
444 gfsr = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSR);
445 gfsynr0 = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSYNR0);
446 gfsynr1 = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSYNR1);
447 gfsynr2 = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSYNR2);
448
449 if (!gfsr)
450 return IRQ_NONE;
451
452 if (__ratelimit(&rs)) {
453 if (IS_ENABLED(CONFIG_ARM_SMMU_DISABLE_BYPASS_BY_DEFAULT) &&
454 (gfsr & ARM_SMMU_sGFSR_USF))
455 dev_err(smmu->dev,
456 "Blocked unknown Stream ID 0x%hx; boot with \"arm-smmu.disable_bypass=0\" to allow, but this may have security implications\n",
457 (u16)gfsynr1);
458 else
459 dev_err(smmu->dev,
460 "Unexpected global fault, this could be serious\n");
461 dev_err(smmu->dev,
462 "\tGFSR 0x%08x, GFSYNR0 0x%08x, GFSYNR1 0x%08x, GFSYNR2 0x%08x\n",
463 gfsr, gfsynr0, gfsynr1, gfsynr2);
464 }
465
466 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sGFSR, gfsr);
467 return IRQ_HANDLED;
468 }
469
arm_smmu_init_context_bank(struct arm_smmu_domain * smmu_domain,struct io_pgtable_cfg * pgtbl_cfg)470 static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain,
471 struct io_pgtable_cfg *pgtbl_cfg)
472 {
473 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
474 struct arm_smmu_cb *cb = &smmu_domain->smmu->cbs[cfg->cbndx];
475 bool stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS;
476
477 cb->cfg = cfg;
478
479 /* TCR */
480 if (stage1) {
481 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
482 cb->tcr[0] = pgtbl_cfg->arm_v7s_cfg.tcr;
483 } else {
484 cb->tcr[0] = arm_smmu_lpae_tcr(pgtbl_cfg);
485 cb->tcr[1] = arm_smmu_lpae_tcr2(pgtbl_cfg);
486 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64)
487 cb->tcr[1] |= ARM_SMMU_TCR2_AS;
488 else
489 cb->tcr[0] |= ARM_SMMU_TCR_EAE;
490 }
491 } else {
492 cb->tcr[0] = arm_smmu_lpae_vtcr(pgtbl_cfg);
493 }
494
495 /* TTBRs */
496 if (stage1) {
497 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
498 cb->ttbr[0] = pgtbl_cfg->arm_v7s_cfg.ttbr;
499 cb->ttbr[1] = 0;
500 } else {
501 cb->ttbr[0] = FIELD_PREP(ARM_SMMU_TTBRn_ASID,
502 cfg->asid);
503 cb->ttbr[1] = FIELD_PREP(ARM_SMMU_TTBRn_ASID,
504 cfg->asid);
505
506 if (pgtbl_cfg->quirks & IO_PGTABLE_QUIRK_ARM_TTBR1)
507 cb->ttbr[1] |= pgtbl_cfg->arm_lpae_s1_cfg.ttbr;
508 else
509 cb->ttbr[0] |= pgtbl_cfg->arm_lpae_s1_cfg.ttbr;
510 }
511 } else {
512 cb->ttbr[0] = pgtbl_cfg->arm_lpae_s2_cfg.vttbr;
513 }
514
515 /* MAIRs (stage-1 only) */
516 if (stage1) {
517 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
518 cb->mair[0] = pgtbl_cfg->arm_v7s_cfg.prrr;
519 cb->mair[1] = pgtbl_cfg->arm_v7s_cfg.nmrr;
520 } else {
521 cb->mair[0] = pgtbl_cfg->arm_lpae_s1_cfg.mair;
522 cb->mair[1] = pgtbl_cfg->arm_lpae_s1_cfg.mair >> 32;
523 }
524 }
525 }
526
arm_smmu_write_context_bank(struct arm_smmu_device * smmu,int idx)527 void arm_smmu_write_context_bank(struct arm_smmu_device *smmu, int idx)
528 {
529 u32 reg;
530 bool stage1;
531 struct arm_smmu_cb *cb = &smmu->cbs[idx];
532 struct arm_smmu_cfg *cfg = cb->cfg;
533
534 /* Unassigned context banks only need disabling */
535 if (!cfg) {
536 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_SCTLR, 0);
537 return;
538 }
539
540 stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS;
541
542 /* CBA2R */
543 if (smmu->version > ARM_SMMU_V1) {
544 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64)
545 reg = ARM_SMMU_CBA2R_VA64;
546 else
547 reg = 0;
548 /* 16-bit VMIDs live in CBA2R */
549 if (smmu->features & ARM_SMMU_FEAT_VMID16)
550 reg |= FIELD_PREP(ARM_SMMU_CBA2R_VMID16, cfg->vmid);
551
552 arm_smmu_gr1_write(smmu, ARM_SMMU_GR1_CBA2R(idx), reg);
553 }
554
555 /* CBAR */
556 reg = FIELD_PREP(ARM_SMMU_CBAR_TYPE, cfg->cbar);
557 if (smmu->version < ARM_SMMU_V2)
558 reg |= FIELD_PREP(ARM_SMMU_CBAR_IRPTNDX, cfg->irptndx);
559
560 /*
561 * Use the weakest shareability/memory types, so they are
562 * overridden by the ttbcr/pte.
563 */
564 if (stage1) {
565 reg |= FIELD_PREP(ARM_SMMU_CBAR_S1_BPSHCFG,
566 ARM_SMMU_CBAR_S1_BPSHCFG_NSH) |
567 FIELD_PREP(ARM_SMMU_CBAR_S1_MEMATTR,
568 ARM_SMMU_CBAR_S1_MEMATTR_WB);
569 } else if (!(smmu->features & ARM_SMMU_FEAT_VMID16)) {
570 /* 8-bit VMIDs live in CBAR */
571 reg |= FIELD_PREP(ARM_SMMU_CBAR_VMID, cfg->vmid);
572 }
573 arm_smmu_gr1_write(smmu, ARM_SMMU_GR1_CBAR(idx), reg);
574
575 /*
576 * TCR
577 * We must write this before the TTBRs, since it determines the
578 * access behaviour of some fields (in particular, ASID[15:8]).
579 */
580 if (stage1 && smmu->version > ARM_SMMU_V1)
581 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_TCR2, cb->tcr[1]);
582 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_TCR, cb->tcr[0]);
583
584 /* TTBRs */
585 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
586 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_CONTEXTIDR, cfg->asid);
587 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_TTBR0, cb->ttbr[0]);
588 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_TTBR1, cb->ttbr[1]);
589 } else {
590 arm_smmu_cb_writeq(smmu, idx, ARM_SMMU_CB_TTBR0, cb->ttbr[0]);
591 if (stage1)
592 arm_smmu_cb_writeq(smmu, idx, ARM_SMMU_CB_TTBR1,
593 cb->ttbr[1]);
594 }
595
596 /* MAIRs (stage-1 only) */
597 if (stage1) {
598 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_S1_MAIR0, cb->mair[0]);
599 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_S1_MAIR1, cb->mair[1]);
600 }
601
602 /* SCTLR */
603 reg = ARM_SMMU_SCTLR_CFIE | ARM_SMMU_SCTLR_CFRE | ARM_SMMU_SCTLR_AFE |
604 ARM_SMMU_SCTLR_TRE | ARM_SMMU_SCTLR_M;
605 if (stage1)
606 reg |= ARM_SMMU_SCTLR_S1_ASIDPNE;
607 if (IS_ENABLED(CONFIG_CPU_BIG_ENDIAN))
608 reg |= ARM_SMMU_SCTLR_E;
609
610 if (smmu->impl && smmu->impl->write_sctlr)
611 smmu->impl->write_sctlr(smmu, idx, reg);
612 else
613 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_SCTLR, reg);
614 }
615
arm_smmu_alloc_context_bank(struct arm_smmu_domain * smmu_domain,struct arm_smmu_device * smmu,struct device * dev,unsigned int start)616 static int arm_smmu_alloc_context_bank(struct arm_smmu_domain *smmu_domain,
617 struct arm_smmu_device *smmu,
618 struct device *dev, unsigned int start)
619 {
620 if (smmu->impl && smmu->impl->alloc_context_bank)
621 return smmu->impl->alloc_context_bank(smmu_domain, smmu, dev, start);
622
623 return __arm_smmu_alloc_bitmap(smmu->context_map, start, smmu->num_context_banks);
624 }
625
arm_smmu_init_domain_context(struct arm_smmu_domain * smmu_domain,struct arm_smmu_device * smmu,struct device * dev)626 static int arm_smmu_init_domain_context(struct arm_smmu_domain *smmu_domain,
627 struct arm_smmu_device *smmu,
628 struct device *dev)
629 {
630 int irq, start, ret = 0;
631 unsigned long ias, oas;
632 struct io_pgtable_ops *pgtbl_ops;
633 struct io_pgtable_cfg pgtbl_cfg;
634 enum io_pgtable_fmt fmt;
635 struct iommu_domain *domain = &smmu_domain->domain;
636 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
637 irqreturn_t (*context_fault)(int irq, void *dev);
638
639 mutex_lock(&smmu_domain->init_mutex);
640 if (smmu_domain->smmu)
641 goto out_unlock;
642
643 /*
644 * Mapping the requested stage onto what we support is surprisingly
645 * complicated, mainly because the spec allows S1+S2 SMMUs without
646 * support for nested translation. That means we end up with the
647 * following table:
648 *
649 * Requested Supported Actual
650 * S1 N S1
651 * S1 S1+S2 S1
652 * S1 S2 S2
653 * S1 S1 S1
654 * N N N
655 * N S1+S2 S2
656 * N S2 S2
657 * N S1 S1
658 *
659 * Note that you can't actually request stage-2 mappings.
660 */
661 if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S1))
662 smmu_domain->stage = ARM_SMMU_DOMAIN_S2;
663 if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S2))
664 smmu_domain->stage = ARM_SMMU_DOMAIN_S1;
665
666 /*
667 * Choosing a suitable context format is even more fiddly. Until we
668 * grow some way for the caller to express a preference, and/or move
669 * the decision into the io-pgtable code where it arguably belongs,
670 * just aim for the closest thing to the rest of the system, and hope
671 * that the hardware isn't esoteric enough that we can't assume AArch64
672 * support to be a superset of AArch32 support...
673 */
674 if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_L)
675 cfg->fmt = ARM_SMMU_CTX_FMT_AARCH32_L;
676 if (IS_ENABLED(CONFIG_IOMMU_IO_PGTABLE_ARMV7S) &&
677 !IS_ENABLED(CONFIG_64BIT) && !IS_ENABLED(CONFIG_ARM_LPAE) &&
678 (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_S) &&
679 (smmu_domain->stage == ARM_SMMU_DOMAIN_S1))
680 cfg->fmt = ARM_SMMU_CTX_FMT_AARCH32_S;
681 if ((IS_ENABLED(CONFIG_64BIT) || cfg->fmt == ARM_SMMU_CTX_FMT_NONE) &&
682 (smmu->features & (ARM_SMMU_FEAT_FMT_AARCH64_64K |
683 ARM_SMMU_FEAT_FMT_AARCH64_16K |
684 ARM_SMMU_FEAT_FMT_AARCH64_4K)))
685 cfg->fmt = ARM_SMMU_CTX_FMT_AARCH64;
686
687 if (cfg->fmt == ARM_SMMU_CTX_FMT_NONE) {
688 ret = -EINVAL;
689 goto out_unlock;
690 }
691
692 switch (smmu_domain->stage) {
693 case ARM_SMMU_DOMAIN_S1:
694 cfg->cbar = CBAR_TYPE_S1_TRANS_S2_BYPASS;
695 start = smmu->num_s2_context_banks;
696 ias = smmu->va_size;
697 oas = smmu->ipa_size;
698 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) {
699 fmt = ARM_64_LPAE_S1;
700 } else if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_L) {
701 fmt = ARM_32_LPAE_S1;
702 ias = min(ias, 32UL);
703 oas = min(oas, 40UL);
704 } else {
705 fmt = ARM_V7S;
706 ias = min(ias, 32UL);
707 oas = min(oas, 32UL);
708 }
709 smmu_domain->flush_ops = &arm_smmu_s1_tlb_ops;
710 break;
711 case ARM_SMMU_DOMAIN_NESTED:
712 /*
713 * We will likely want to change this if/when KVM gets
714 * involved.
715 */
716 case ARM_SMMU_DOMAIN_S2:
717 cfg->cbar = CBAR_TYPE_S2_TRANS;
718 start = 0;
719 ias = smmu->ipa_size;
720 oas = smmu->pa_size;
721 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) {
722 fmt = ARM_64_LPAE_S2;
723 } else {
724 fmt = ARM_32_LPAE_S2;
725 ias = min(ias, 40UL);
726 oas = min(oas, 40UL);
727 }
728 if (smmu->version == ARM_SMMU_V2)
729 smmu_domain->flush_ops = &arm_smmu_s2_tlb_ops_v2;
730 else
731 smmu_domain->flush_ops = &arm_smmu_s2_tlb_ops_v1;
732 break;
733 default:
734 ret = -EINVAL;
735 goto out_unlock;
736 }
737
738 ret = arm_smmu_alloc_context_bank(smmu_domain, smmu, dev, start);
739 if (ret < 0) {
740 goto out_unlock;
741 }
742
743 smmu_domain->smmu = smmu;
744
745 cfg->cbndx = ret;
746 if (smmu->version < ARM_SMMU_V2) {
747 cfg->irptndx = atomic_inc_return(&smmu->irptndx);
748 cfg->irptndx %= smmu->num_context_irqs;
749 } else {
750 cfg->irptndx = cfg->cbndx;
751 }
752
753 if (smmu_domain->stage == ARM_SMMU_DOMAIN_S2)
754 cfg->vmid = cfg->cbndx + 1;
755 else
756 cfg->asid = cfg->cbndx;
757
758 pgtbl_cfg = (struct io_pgtable_cfg) {
759 .pgsize_bitmap = smmu->pgsize_bitmap,
760 .ias = ias,
761 .oas = oas,
762 .coherent_walk = smmu->features & ARM_SMMU_FEAT_COHERENT_WALK,
763 .tlb = smmu_domain->flush_ops,
764 .iommu_dev = smmu->dev,
765 };
766
767 if (smmu->impl && smmu->impl->init_context) {
768 ret = smmu->impl->init_context(smmu_domain, &pgtbl_cfg, dev);
769 if (ret)
770 goto out_clear_smmu;
771 }
772
773 if (smmu_domain->pgtbl_quirks)
774 pgtbl_cfg.quirks |= smmu_domain->pgtbl_quirks;
775
776 pgtbl_ops = alloc_io_pgtable_ops(fmt, &pgtbl_cfg, smmu_domain);
777 if (!pgtbl_ops) {
778 ret = -ENOMEM;
779 goto out_clear_smmu;
780 }
781
782 /* Update the domain's page sizes to reflect the page table format */
783 domain->pgsize_bitmap = pgtbl_cfg.pgsize_bitmap;
784
785 if (pgtbl_cfg.quirks & IO_PGTABLE_QUIRK_ARM_TTBR1) {
786 domain->geometry.aperture_start = ~0UL << ias;
787 domain->geometry.aperture_end = ~0UL;
788 } else {
789 domain->geometry.aperture_end = (1UL << ias) - 1;
790 }
791
792 domain->geometry.force_aperture = true;
793
794 /* Initialise the context bank with our page table cfg */
795 arm_smmu_init_context_bank(smmu_domain, &pgtbl_cfg);
796 arm_smmu_write_context_bank(smmu, cfg->cbndx);
797
798 /*
799 * Request context fault interrupt. Do this last to avoid the
800 * handler seeing a half-initialised domain state.
801 */
802 irq = smmu->irqs[cfg->irptndx];
803
804 if (smmu->impl && smmu->impl->context_fault)
805 context_fault = smmu->impl->context_fault;
806 else
807 context_fault = arm_smmu_context_fault;
808
809 ret = devm_request_irq(smmu->dev, irq, context_fault, IRQF_SHARED,
810 "arm-smmu-context-fault", smmu_domain);
811 if (ret < 0) {
812 dev_err(smmu->dev, "failed to request context IRQ %d (%u)\n",
813 cfg->irptndx, irq);
814 cfg->irptndx = ARM_SMMU_INVALID_IRPTNDX;
815 }
816
817 mutex_unlock(&smmu_domain->init_mutex);
818
819 /* Publish page table ops for map/unmap */
820 smmu_domain->pgtbl_ops = pgtbl_ops;
821 return 0;
822
823 out_clear_smmu:
824 __arm_smmu_free_bitmap(smmu->context_map, cfg->cbndx);
825 smmu_domain->smmu = NULL;
826 out_unlock:
827 mutex_unlock(&smmu_domain->init_mutex);
828 return ret;
829 }
830
arm_smmu_destroy_domain_context(struct arm_smmu_domain * smmu_domain)831 static void arm_smmu_destroy_domain_context(struct arm_smmu_domain *smmu_domain)
832 {
833 struct arm_smmu_device *smmu = smmu_domain->smmu;
834 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
835 int ret, irq;
836
837 if (!smmu)
838 return;
839
840 ret = arm_smmu_rpm_get(smmu);
841 if (ret < 0)
842 return;
843
844 /*
845 * Disable the context bank and free the page tables before freeing
846 * it.
847 */
848 smmu->cbs[cfg->cbndx].cfg = NULL;
849 arm_smmu_write_context_bank(smmu, cfg->cbndx);
850
851 if (cfg->irptndx != ARM_SMMU_INVALID_IRPTNDX) {
852 irq = smmu->irqs[cfg->irptndx];
853 devm_free_irq(smmu->dev, irq, smmu_domain);
854 }
855
856 free_io_pgtable_ops(smmu_domain->pgtbl_ops);
857 __arm_smmu_free_bitmap(smmu->context_map, cfg->cbndx);
858
859 arm_smmu_rpm_put(smmu);
860 }
861
arm_smmu_domain_alloc(unsigned type)862 static struct iommu_domain *arm_smmu_domain_alloc(unsigned type)
863 {
864 struct arm_smmu_domain *smmu_domain;
865
866 if (type != IOMMU_DOMAIN_UNMANAGED) {
867 if (using_legacy_binding || type != IOMMU_DOMAIN_DMA)
868 return NULL;
869 }
870 /*
871 * Allocate the domain and initialise some of its data structures.
872 * We can't really do anything meaningful until we've added a
873 * master.
874 */
875 smmu_domain = kzalloc(sizeof(*smmu_domain), GFP_KERNEL);
876 if (!smmu_domain)
877 return NULL;
878
879 mutex_init(&smmu_domain->init_mutex);
880 spin_lock_init(&smmu_domain->cb_lock);
881
882 return &smmu_domain->domain;
883 }
884
arm_smmu_domain_free(struct iommu_domain * domain)885 static void arm_smmu_domain_free(struct iommu_domain *domain)
886 {
887 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
888
889 /*
890 * Free the domain resources. We assume that all devices have
891 * already been detached.
892 */
893 arm_smmu_destroy_domain_context(smmu_domain);
894 kfree(smmu_domain);
895 }
896
arm_smmu_write_smr(struct arm_smmu_device * smmu,int idx)897 static void arm_smmu_write_smr(struct arm_smmu_device *smmu, int idx)
898 {
899 struct arm_smmu_smr *smr = smmu->smrs + idx;
900 u32 reg = FIELD_PREP(ARM_SMMU_SMR_ID, smr->id) |
901 FIELD_PREP(ARM_SMMU_SMR_MASK, smr->mask);
902
903 if (!(smmu->features & ARM_SMMU_FEAT_EXIDS) && smr->valid)
904 reg |= ARM_SMMU_SMR_VALID;
905 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_SMR(idx), reg);
906 }
907
arm_smmu_write_s2cr(struct arm_smmu_device * smmu,int idx)908 static void arm_smmu_write_s2cr(struct arm_smmu_device *smmu, int idx)
909 {
910 struct arm_smmu_s2cr *s2cr = smmu->s2crs + idx;
911 u32 reg;
912
913 if (smmu->impl && smmu->impl->write_s2cr) {
914 smmu->impl->write_s2cr(smmu, idx);
915 return;
916 }
917
918 reg = FIELD_PREP(ARM_SMMU_S2CR_TYPE, s2cr->type) |
919 FIELD_PREP(ARM_SMMU_S2CR_CBNDX, s2cr->cbndx) |
920 FIELD_PREP(ARM_SMMU_S2CR_PRIVCFG, s2cr->privcfg);
921
922 if (smmu->features & ARM_SMMU_FEAT_EXIDS && smmu->smrs &&
923 smmu->smrs[idx].valid)
924 reg |= ARM_SMMU_S2CR_EXIDVALID;
925 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_S2CR(idx), reg);
926 }
927
arm_smmu_write_sme(struct arm_smmu_device * smmu,int idx)928 static void arm_smmu_write_sme(struct arm_smmu_device *smmu, int idx)
929 {
930 arm_smmu_write_s2cr(smmu, idx);
931 if (smmu->smrs)
932 arm_smmu_write_smr(smmu, idx);
933 }
934
935 /*
936 * The width of SMR's mask field depends on sCR0_EXIDENABLE, so this function
937 * should be called after sCR0 is written.
938 */
arm_smmu_test_smr_masks(struct arm_smmu_device * smmu)939 static void arm_smmu_test_smr_masks(struct arm_smmu_device *smmu)
940 {
941 u32 smr;
942 int i;
943
944 if (!smmu->smrs)
945 return;
946 /*
947 * If we've had to accommodate firmware memory regions, we may
948 * have live SMRs by now; tread carefully...
949 *
950 * Somewhat perversely, not having a free SMR for this test implies we
951 * can get away without it anyway, as we'll only be able to 'allocate'
952 * these SMRs for the ID/mask values we're already trusting to be OK.
953 */
954 for (i = 0; i < smmu->num_mapping_groups; i++)
955 if (!smmu->smrs[i].valid)
956 goto smr_ok;
957 return;
958 smr_ok:
959 /*
960 * SMR.ID bits may not be preserved if the corresponding MASK
961 * bits are set, so check each one separately. We can reject
962 * masters later if they try to claim IDs outside these masks.
963 */
964 smr = FIELD_PREP(ARM_SMMU_SMR_ID, smmu->streamid_mask);
965 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_SMR(i), smr);
966 smr = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_SMR(i));
967 smmu->streamid_mask = FIELD_GET(ARM_SMMU_SMR_ID, smr);
968
969 smr = FIELD_PREP(ARM_SMMU_SMR_MASK, smmu->streamid_mask);
970 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_SMR(i), smr);
971 smr = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_SMR(i));
972 smmu->smr_mask_mask = FIELD_GET(ARM_SMMU_SMR_MASK, smr);
973 }
974
arm_smmu_find_sme(struct arm_smmu_device * smmu,u16 id,u16 mask)975 static int arm_smmu_find_sme(struct arm_smmu_device *smmu, u16 id, u16 mask)
976 {
977 struct arm_smmu_smr *smrs = smmu->smrs;
978 int i, free_idx = -ENOSPC;
979
980 /* Stream indexing is blissfully easy */
981 if (!smrs)
982 return id;
983
984 /* Validating SMRs is... less so */
985 for (i = 0; i < smmu->num_mapping_groups; ++i) {
986 if (!smrs[i].valid) {
987 /*
988 * Note the first free entry we come across, which
989 * we'll claim in the end if nothing else matches.
990 */
991 if (free_idx < 0)
992 free_idx = i;
993 continue;
994 }
995 /*
996 * If the new entry is _entirely_ matched by an existing entry,
997 * then reuse that, with the guarantee that there also cannot
998 * be any subsequent conflicting entries. In normal use we'd
999 * expect simply identical entries for this case, but there's
1000 * no harm in accommodating the generalisation.
1001 */
1002 if ((mask & smrs[i].mask) == mask &&
1003 !((id ^ smrs[i].id) & ~smrs[i].mask))
1004 return i;
1005 /*
1006 * If the new entry has any other overlap with an existing one,
1007 * though, then there always exists at least one stream ID
1008 * which would cause a conflict, and we can't allow that risk.
1009 */
1010 if (!((id ^ smrs[i].id) & ~(smrs[i].mask | mask)))
1011 return -EINVAL;
1012 }
1013
1014 return free_idx;
1015 }
1016
arm_smmu_free_sme(struct arm_smmu_device * smmu,int idx)1017 static bool arm_smmu_free_sme(struct arm_smmu_device *smmu, int idx)
1018 {
1019 if (--smmu->s2crs[idx].count)
1020 return false;
1021
1022 smmu->s2crs[idx] = s2cr_init_val;
1023 if (smmu->smrs)
1024 smmu->smrs[idx].valid = false;
1025
1026 return true;
1027 }
1028
arm_smmu_master_alloc_smes(struct device * dev)1029 static int arm_smmu_master_alloc_smes(struct device *dev)
1030 {
1031 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1032 struct arm_smmu_master_cfg *cfg = dev_iommu_priv_get(dev);
1033 struct arm_smmu_device *smmu = cfg->smmu;
1034 struct arm_smmu_smr *smrs = smmu->smrs;
1035 int i, idx, ret;
1036
1037 mutex_lock(&smmu->stream_map_mutex);
1038 /* Figure out a viable stream map entry allocation */
1039 for_each_cfg_sme(cfg, fwspec, i, idx) {
1040 u16 sid = FIELD_GET(ARM_SMMU_SMR_ID, fwspec->ids[i]);
1041 u16 mask = FIELD_GET(ARM_SMMU_SMR_MASK, fwspec->ids[i]);
1042
1043 if (idx != INVALID_SMENDX) {
1044 ret = -EEXIST;
1045 goto out_err;
1046 }
1047
1048 ret = arm_smmu_find_sme(smmu, sid, mask);
1049 if (ret < 0)
1050 goto out_err;
1051
1052 idx = ret;
1053 if (smrs && smmu->s2crs[idx].count == 0) {
1054 smrs[idx].id = sid;
1055 smrs[idx].mask = mask;
1056 smrs[idx].valid = true;
1057 }
1058 smmu->s2crs[idx].count++;
1059 cfg->smendx[i] = (s16)idx;
1060 }
1061
1062 /* It worked! Now, poke the actual hardware */
1063 for_each_cfg_sme(cfg, fwspec, i, idx)
1064 arm_smmu_write_sme(smmu, idx);
1065
1066 mutex_unlock(&smmu->stream_map_mutex);
1067 return 0;
1068
1069 out_err:
1070 while (i--) {
1071 arm_smmu_free_sme(smmu, cfg->smendx[i]);
1072 cfg->smendx[i] = INVALID_SMENDX;
1073 }
1074 mutex_unlock(&smmu->stream_map_mutex);
1075 return ret;
1076 }
1077
arm_smmu_master_free_smes(struct arm_smmu_master_cfg * cfg,struct iommu_fwspec * fwspec)1078 static void arm_smmu_master_free_smes(struct arm_smmu_master_cfg *cfg,
1079 struct iommu_fwspec *fwspec)
1080 {
1081 struct arm_smmu_device *smmu = cfg->smmu;
1082 int i, idx;
1083
1084 mutex_lock(&smmu->stream_map_mutex);
1085 for_each_cfg_sme(cfg, fwspec, i, idx) {
1086 if (arm_smmu_free_sme(smmu, idx))
1087 arm_smmu_write_sme(smmu, idx);
1088 cfg->smendx[i] = INVALID_SMENDX;
1089 }
1090 mutex_unlock(&smmu->stream_map_mutex);
1091 }
1092
arm_smmu_master_install_s2crs(struct arm_smmu_master_cfg * cfg,enum arm_smmu_s2cr_type type,u8 cbndx,struct iommu_fwspec * fwspec)1093 static void arm_smmu_master_install_s2crs(struct arm_smmu_master_cfg *cfg,
1094 enum arm_smmu_s2cr_type type,
1095 u8 cbndx, struct iommu_fwspec *fwspec)
1096 {
1097 struct arm_smmu_device *smmu = cfg->smmu;
1098 struct arm_smmu_s2cr *s2cr = smmu->s2crs;
1099 int i, idx;
1100
1101 for_each_cfg_sme(cfg, fwspec, i, idx) {
1102 if (type == s2cr[idx].type && cbndx == s2cr[idx].cbndx)
1103 continue;
1104
1105 s2cr[idx].type = type;
1106 s2cr[idx].privcfg = S2CR_PRIVCFG_DEFAULT;
1107 s2cr[idx].cbndx = cbndx;
1108 arm_smmu_write_s2cr(smmu, idx);
1109 }
1110 }
1111
arm_smmu_attach_dev(struct iommu_domain * domain,struct device * dev)1112 static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev)
1113 {
1114 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1115 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1116 struct arm_smmu_master_cfg *cfg;
1117 struct arm_smmu_device *smmu;
1118 int ret;
1119
1120 /*
1121 * FIXME: The arch/arm DMA API code tries to attach devices to its own
1122 * domains between of_xlate() and probe_device() - we have no way to cope
1123 * with that, so until ARM gets converted to rely on groups and default
1124 * domains, just say no (but more politely than by dereferencing NULL).
1125 * This should be at least a WARN_ON once that's sorted.
1126 */
1127 cfg = dev_iommu_priv_get(dev);
1128 if (!cfg)
1129 return -ENODEV;
1130
1131 smmu = cfg->smmu;
1132
1133 ret = arm_smmu_rpm_get(smmu);
1134 if (ret < 0)
1135 return ret;
1136
1137 /* Ensure that the domain is finalised */
1138 ret = arm_smmu_init_domain_context(smmu_domain, smmu, dev);
1139 if (ret < 0)
1140 goto rpm_put;
1141
1142 /*
1143 * Sanity check the domain. We don't support domains across
1144 * different SMMUs.
1145 */
1146 if (smmu_domain->smmu != smmu) {
1147 ret = -EINVAL;
1148 goto rpm_put;
1149 }
1150
1151 /* Looks ok, so add the device to the domain */
1152 arm_smmu_master_install_s2crs(cfg, S2CR_TYPE_TRANS,
1153 smmu_domain->cfg.cbndx, fwspec);
1154 arm_smmu_rpm_use_autosuspend(smmu);
1155 rpm_put:
1156 arm_smmu_rpm_put(smmu);
1157 return ret;
1158 }
1159
arm_smmu_attach_dev_type(struct device * dev,enum arm_smmu_s2cr_type type)1160 static int arm_smmu_attach_dev_type(struct device *dev,
1161 enum arm_smmu_s2cr_type type)
1162 {
1163 struct arm_smmu_master_cfg *cfg = dev_iommu_priv_get(dev);
1164 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1165 struct arm_smmu_device *smmu;
1166 int ret;
1167
1168 if (!cfg)
1169 return -ENODEV;
1170 smmu = cfg->smmu;
1171
1172 ret = arm_smmu_rpm_get(smmu);
1173 if (ret < 0)
1174 return ret;
1175
1176 arm_smmu_master_install_s2crs(cfg, type, 0, fwspec);
1177 arm_smmu_rpm_use_autosuspend(smmu);
1178 arm_smmu_rpm_put(smmu);
1179 return 0;
1180 }
1181
arm_smmu_attach_dev_identity(struct iommu_domain * domain,struct device * dev)1182 static int arm_smmu_attach_dev_identity(struct iommu_domain *domain,
1183 struct device *dev)
1184 {
1185 return arm_smmu_attach_dev_type(dev, S2CR_TYPE_BYPASS);
1186 }
1187
1188 static const struct iommu_domain_ops arm_smmu_identity_ops = {
1189 .attach_dev = arm_smmu_attach_dev_identity,
1190 };
1191
1192 static struct iommu_domain arm_smmu_identity_domain = {
1193 .type = IOMMU_DOMAIN_IDENTITY,
1194 .ops = &arm_smmu_identity_ops,
1195 };
1196
arm_smmu_attach_dev_blocked(struct iommu_domain * domain,struct device * dev)1197 static int arm_smmu_attach_dev_blocked(struct iommu_domain *domain,
1198 struct device *dev)
1199 {
1200 return arm_smmu_attach_dev_type(dev, S2CR_TYPE_FAULT);
1201 }
1202
1203 static const struct iommu_domain_ops arm_smmu_blocked_ops = {
1204 .attach_dev = arm_smmu_attach_dev_blocked,
1205 };
1206
1207 static struct iommu_domain arm_smmu_blocked_domain = {
1208 .type = IOMMU_DOMAIN_BLOCKED,
1209 .ops = &arm_smmu_blocked_ops,
1210 };
1211
arm_smmu_map_pages(struct iommu_domain * domain,unsigned long iova,phys_addr_t paddr,size_t pgsize,size_t pgcount,int prot,gfp_t gfp,size_t * mapped)1212 static int arm_smmu_map_pages(struct iommu_domain *domain, unsigned long iova,
1213 phys_addr_t paddr, size_t pgsize, size_t pgcount,
1214 int prot, gfp_t gfp, size_t *mapped)
1215 {
1216 struct io_pgtable_ops *ops = to_smmu_domain(domain)->pgtbl_ops;
1217 struct arm_smmu_device *smmu = to_smmu_domain(domain)->smmu;
1218 int ret;
1219
1220 if (!ops)
1221 return -ENODEV;
1222
1223 arm_smmu_rpm_get(smmu);
1224 ret = ops->map_pages(ops, iova, paddr, pgsize, pgcount, prot, gfp, mapped);
1225 arm_smmu_rpm_put(smmu);
1226
1227 return ret;
1228 }
1229
arm_smmu_unmap_pages(struct iommu_domain * domain,unsigned long iova,size_t pgsize,size_t pgcount,struct iommu_iotlb_gather * iotlb_gather)1230 static size_t arm_smmu_unmap_pages(struct iommu_domain *domain, unsigned long iova,
1231 size_t pgsize, size_t pgcount,
1232 struct iommu_iotlb_gather *iotlb_gather)
1233 {
1234 struct io_pgtable_ops *ops = to_smmu_domain(domain)->pgtbl_ops;
1235 struct arm_smmu_device *smmu = to_smmu_domain(domain)->smmu;
1236 size_t ret;
1237
1238 if (!ops)
1239 return 0;
1240
1241 arm_smmu_rpm_get(smmu);
1242 ret = ops->unmap_pages(ops, iova, pgsize, pgcount, iotlb_gather);
1243 arm_smmu_rpm_put(smmu);
1244
1245 return ret;
1246 }
1247
arm_smmu_flush_iotlb_all(struct iommu_domain * domain)1248 static void arm_smmu_flush_iotlb_all(struct iommu_domain *domain)
1249 {
1250 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1251 struct arm_smmu_device *smmu = smmu_domain->smmu;
1252
1253 if (smmu_domain->flush_ops) {
1254 arm_smmu_rpm_get(smmu);
1255 smmu_domain->flush_ops->tlb_flush_all(smmu_domain);
1256 arm_smmu_rpm_put(smmu);
1257 }
1258 }
1259
arm_smmu_iotlb_sync(struct iommu_domain * domain,struct iommu_iotlb_gather * gather)1260 static void arm_smmu_iotlb_sync(struct iommu_domain *domain,
1261 struct iommu_iotlb_gather *gather)
1262 {
1263 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1264 struct arm_smmu_device *smmu = smmu_domain->smmu;
1265
1266 if (!smmu)
1267 return;
1268
1269 arm_smmu_rpm_get(smmu);
1270 if (smmu->version == ARM_SMMU_V2 ||
1271 smmu_domain->stage == ARM_SMMU_DOMAIN_S1)
1272 arm_smmu_tlb_sync_context(smmu_domain);
1273 else
1274 arm_smmu_tlb_sync_global(smmu);
1275 arm_smmu_rpm_put(smmu);
1276 }
1277
arm_smmu_iova_to_phys_hard(struct iommu_domain * domain,dma_addr_t iova)1278 static phys_addr_t arm_smmu_iova_to_phys_hard(struct iommu_domain *domain,
1279 dma_addr_t iova)
1280 {
1281 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1282 struct arm_smmu_device *smmu = smmu_domain->smmu;
1283 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
1284 struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops;
1285 struct device *dev = smmu->dev;
1286 void __iomem *reg;
1287 u32 tmp;
1288 u64 phys;
1289 unsigned long va, flags;
1290 int ret, idx = cfg->cbndx;
1291 phys_addr_t addr = 0;
1292
1293 ret = arm_smmu_rpm_get(smmu);
1294 if (ret < 0)
1295 return 0;
1296
1297 spin_lock_irqsave(&smmu_domain->cb_lock, flags);
1298 va = iova & ~0xfffUL;
1299 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64)
1300 arm_smmu_cb_writeq(smmu, idx, ARM_SMMU_CB_ATS1PR, va);
1301 else
1302 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_ATS1PR, va);
1303
1304 reg = arm_smmu_page(smmu, ARM_SMMU_CB(smmu, idx)) + ARM_SMMU_CB_ATSR;
1305 if (readl_poll_timeout_atomic(reg, tmp, !(tmp & ARM_SMMU_ATSR_ACTIVE),
1306 5, 50)) {
1307 spin_unlock_irqrestore(&smmu_domain->cb_lock, flags);
1308 dev_err(dev,
1309 "iova to phys timed out on %pad. Falling back to software table walk.\n",
1310 &iova);
1311 arm_smmu_rpm_put(smmu);
1312 return ops->iova_to_phys(ops, iova);
1313 }
1314
1315 phys = arm_smmu_cb_readq(smmu, idx, ARM_SMMU_CB_PAR);
1316 spin_unlock_irqrestore(&smmu_domain->cb_lock, flags);
1317 if (phys & ARM_SMMU_CB_PAR_F) {
1318 dev_err(dev, "translation fault!\n");
1319 dev_err(dev, "PAR = 0x%llx\n", phys);
1320 goto out;
1321 }
1322
1323 addr = (phys & GENMASK_ULL(39, 12)) | (iova & 0xfff);
1324 out:
1325 arm_smmu_rpm_put(smmu);
1326
1327 return addr;
1328 }
1329
arm_smmu_iova_to_phys(struct iommu_domain * domain,dma_addr_t iova)1330 static phys_addr_t arm_smmu_iova_to_phys(struct iommu_domain *domain,
1331 dma_addr_t iova)
1332 {
1333 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1334 struct io_pgtable_ops *ops = smmu_domain->pgtbl_ops;
1335
1336 if (!ops)
1337 return 0;
1338
1339 if (smmu_domain->smmu->features & ARM_SMMU_FEAT_TRANS_OPS &&
1340 smmu_domain->stage == ARM_SMMU_DOMAIN_S1)
1341 return arm_smmu_iova_to_phys_hard(domain, iova);
1342
1343 return ops->iova_to_phys(ops, iova);
1344 }
1345
arm_smmu_capable(struct device * dev,enum iommu_cap cap)1346 static bool arm_smmu_capable(struct device *dev, enum iommu_cap cap)
1347 {
1348 struct arm_smmu_master_cfg *cfg = dev_iommu_priv_get(dev);
1349
1350 switch (cap) {
1351 case IOMMU_CAP_CACHE_COHERENCY:
1352 /*
1353 * It's overwhelmingly the case in practice that when the pagetable
1354 * walk interface is connected to a coherent interconnect, all the
1355 * translation interfaces are too. Furthermore if the device is
1356 * natively coherent, then its translation interface must also be.
1357 */
1358 return cfg->smmu->features & ARM_SMMU_FEAT_COHERENT_WALK ||
1359 device_get_dma_attr(dev) == DEV_DMA_COHERENT;
1360 case IOMMU_CAP_NOEXEC:
1361 case IOMMU_CAP_DEFERRED_FLUSH:
1362 return true;
1363 default:
1364 return false;
1365 }
1366 }
1367
1368 static
arm_smmu_get_by_fwnode(struct fwnode_handle * fwnode)1369 struct arm_smmu_device *arm_smmu_get_by_fwnode(struct fwnode_handle *fwnode)
1370 {
1371 struct device *dev = driver_find_device_by_fwnode(&arm_smmu_driver.driver,
1372 fwnode);
1373 put_device(dev);
1374 return dev ? dev_get_drvdata(dev) : NULL;
1375 }
1376
arm_smmu_probe_device(struct device * dev)1377 static struct iommu_device *arm_smmu_probe_device(struct device *dev)
1378 {
1379 struct arm_smmu_device *smmu = NULL;
1380 struct arm_smmu_master_cfg *cfg;
1381 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1382 int i, ret;
1383
1384 if (using_legacy_binding) {
1385 ret = arm_smmu_register_legacy_master(dev, &smmu);
1386
1387 /*
1388 * If dev->iommu_fwspec is initally NULL, arm_smmu_register_legacy_master()
1389 * will allocate/initialise a new one. Thus we need to update fwspec for
1390 * later use.
1391 */
1392 fwspec = dev_iommu_fwspec_get(dev);
1393 if (ret)
1394 goto out_free;
1395 } else {
1396 smmu = arm_smmu_get_by_fwnode(fwspec->iommu_fwnode);
1397 }
1398
1399 ret = -EINVAL;
1400 for (i = 0; i < fwspec->num_ids; i++) {
1401 u16 sid = FIELD_GET(ARM_SMMU_SMR_ID, fwspec->ids[i]);
1402 u16 mask = FIELD_GET(ARM_SMMU_SMR_MASK, fwspec->ids[i]);
1403
1404 if (sid & ~smmu->streamid_mask) {
1405 dev_err(dev, "stream ID 0x%x out of range for SMMU (0x%x)\n",
1406 sid, smmu->streamid_mask);
1407 goto out_free;
1408 }
1409 if (mask & ~smmu->smr_mask_mask) {
1410 dev_err(dev, "SMR mask 0x%x out of range for SMMU (0x%x)\n",
1411 mask, smmu->smr_mask_mask);
1412 goto out_free;
1413 }
1414 }
1415
1416 ret = -ENOMEM;
1417 cfg = kzalloc(offsetof(struct arm_smmu_master_cfg, smendx[i]),
1418 GFP_KERNEL);
1419 if (!cfg)
1420 goto out_free;
1421
1422 cfg->smmu = smmu;
1423 dev_iommu_priv_set(dev, cfg);
1424 while (i--)
1425 cfg->smendx[i] = INVALID_SMENDX;
1426
1427 ret = arm_smmu_rpm_get(smmu);
1428 if (ret < 0)
1429 goto out_cfg_free;
1430
1431 ret = arm_smmu_master_alloc_smes(dev);
1432 arm_smmu_rpm_put(smmu);
1433
1434 if (ret)
1435 goto out_cfg_free;
1436
1437 device_link_add(dev, smmu->dev,
1438 DL_FLAG_PM_RUNTIME | DL_FLAG_AUTOREMOVE_SUPPLIER);
1439
1440 return &smmu->iommu;
1441
1442 out_cfg_free:
1443 kfree(cfg);
1444 out_free:
1445 iommu_fwspec_free(dev);
1446 return ERR_PTR(ret);
1447 }
1448
arm_smmu_release_device(struct device * dev)1449 static void arm_smmu_release_device(struct device *dev)
1450 {
1451 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1452 struct arm_smmu_master_cfg *cfg = dev_iommu_priv_get(dev);
1453 int ret;
1454
1455 ret = arm_smmu_rpm_get(cfg->smmu);
1456 if (ret < 0)
1457 return;
1458
1459 arm_smmu_master_free_smes(cfg, fwspec);
1460
1461 arm_smmu_rpm_put(cfg->smmu);
1462
1463 kfree(cfg);
1464 }
1465
arm_smmu_probe_finalize(struct device * dev)1466 static void arm_smmu_probe_finalize(struct device *dev)
1467 {
1468 struct arm_smmu_master_cfg *cfg;
1469 struct arm_smmu_device *smmu;
1470
1471 cfg = dev_iommu_priv_get(dev);
1472 smmu = cfg->smmu;
1473
1474 if (smmu->impl && smmu->impl->probe_finalize)
1475 smmu->impl->probe_finalize(smmu, dev);
1476 }
1477
arm_smmu_device_group(struct device * dev)1478 static struct iommu_group *arm_smmu_device_group(struct device *dev)
1479 {
1480 struct arm_smmu_master_cfg *cfg = dev_iommu_priv_get(dev);
1481 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1482 struct arm_smmu_device *smmu = cfg->smmu;
1483 struct iommu_group *group = NULL;
1484 int i, idx;
1485
1486 mutex_lock(&smmu->stream_map_mutex);
1487 for_each_cfg_sme(cfg, fwspec, i, idx) {
1488 if (group && smmu->s2crs[idx].group &&
1489 group != smmu->s2crs[idx].group) {
1490 mutex_unlock(&smmu->stream_map_mutex);
1491 return ERR_PTR(-EINVAL);
1492 }
1493
1494 group = smmu->s2crs[idx].group;
1495 }
1496
1497 if (group) {
1498 mutex_unlock(&smmu->stream_map_mutex);
1499 return iommu_group_ref_get(group);
1500 }
1501
1502 if (dev_is_pci(dev))
1503 group = pci_device_group(dev);
1504 else if (dev_is_fsl_mc(dev))
1505 group = fsl_mc_device_group(dev);
1506 else
1507 group = generic_device_group(dev);
1508
1509 /* Remember group for faster lookups */
1510 if (!IS_ERR(group))
1511 for_each_cfg_sme(cfg, fwspec, i, idx)
1512 smmu->s2crs[idx].group = group;
1513
1514 mutex_unlock(&smmu->stream_map_mutex);
1515 return group;
1516 }
1517
arm_smmu_enable_nesting(struct iommu_domain * domain)1518 static int arm_smmu_enable_nesting(struct iommu_domain *domain)
1519 {
1520 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1521 int ret = 0;
1522
1523 mutex_lock(&smmu_domain->init_mutex);
1524 if (smmu_domain->smmu)
1525 ret = -EPERM;
1526 else
1527 smmu_domain->stage = ARM_SMMU_DOMAIN_NESTED;
1528 mutex_unlock(&smmu_domain->init_mutex);
1529
1530 return ret;
1531 }
1532
arm_smmu_set_pgtable_quirks(struct iommu_domain * domain,unsigned long quirks)1533 static int arm_smmu_set_pgtable_quirks(struct iommu_domain *domain,
1534 unsigned long quirks)
1535 {
1536 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1537 int ret = 0;
1538
1539 mutex_lock(&smmu_domain->init_mutex);
1540 if (smmu_domain->smmu)
1541 ret = -EPERM;
1542 else
1543 smmu_domain->pgtbl_quirks = quirks;
1544 mutex_unlock(&smmu_domain->init_mutex);
1545
1546 return ret;
1547 }
1548
arm_smmu_of_xlate(struct device * dev,struct of_phandle_args * args)1549 static int arm_smmu_of_xlate(struct device *dev, struct of_phandle_args *args)
1550 {
1551 u32 mask, fwid = 0;
1552
1553 if (args->args_count > 0)
1554 fwid |= FIELD_PREP(ARM_SMMU_SMR_ID, args->args[0]);
1555
1556 if (args->args_count > 1)
1557 fwid |= FIELD_PREP(ARM_SMMU_SMR_MASK, args->args[1]);
1558 else if (!of_property_read_u32(args->np, "stream-match-mask", &mask))
1559 fwid |= FIELD_PREP(ARM_SMMU_SMR_MASK, mask);
1560
1561 return iommu_fwspec_add_ids(dev, &fwid, 1);
1562 }
1563
arm_smmu_get_resv_regions(struct device * dev,struct list_head * head)1564 static void arm_smmu_get_resv_regions(struct device *dev,
1565 struct list_head *head)
1566 {
1567 struct iommu_resv_region *region;
1568 int prot = IOMMU_WRITE | IOMMU_NOEXEC | IOMMU_MMIO;
1569
1570 region = iommu_alloc_resv_region(MSI_IOVA_BASE, MSI_IOVA_LENGTH,
1571 prot, IOMMU_RESV_SW_MSI, GFP_KERNEL);
1572 if (!region)
1573 return;
1574
1575 list_add_tail(®ion->list, head);
1576
1577 iommu_dma_get_resv_regions(dev, head);
1578 }
1579
arm_smmu_def_domain_type(struct device * dev)1580 static int arm_smmu_def_domain_type(struct device *dev)
1581 {
1582 struct arm_smmu_master_cfg *cfg = dev_iommu_priv_get(dev);
1583 const struct arm_smmu_impl *impl = cfg->smmu->impl;
1584
1585 if (using_legacy_binding)
1586 return IOMMU_DOMAIN_IDENTITY;
1587
1588 if (impl && impl->def_domain_type)
1589 return impl->def_domain_type(dev);
1590
1591 return 0;
1592 }
1593
1594 static struct iommu_ops arm_smmu_ops = {
1595 .identity_domain = &arm_smmu_identity_domain,
1596 .blocked_domain = &arm_smmu_blocked_domain,
1597 .capable = arm_smmu_capable,
1598 .domain_alloc = arm_smmu_domain_alloc,
1599 .probe_device = arm_smmu_probe_device,
1600 .release_device = arm_smmu_release_device,
1601 .probe_finalize = arm_smmu_probe_finalize,
1602 .device_group = arm_smmu_device_group,
1603 .of_xlate = arm_smmu_of_xlate,
1604 .get_resv_regions = arm_smmu_get_resv_regions,
1605 .def_domain_type = arm_smmu_def_domain_type,
1606 .pgsize_bitmap = -1UL, /* Restricted during device attach */
1607 .owner = THIS_MODULE,
1608 .default_domain_ops = &(const struct iommu_domain_ops) {
1609 .attach_dev = arm_smmu_attach_dev,
1610 .map_pages = arm_smmu_map_pages,
1611 .unmap_pages = arm_smmu_unmap_pages,
1612 .flush_iotlb_all = arm_smmu_flush_iotlb_all,
1613 .iotlb_sync = arm_smmu_iotlb_sync,
1614 .iova_to_phys = arm_smmu_iova_to_phys,
1615 .enable_nesting = arm_smmu_enable_nesting,
1616 .set_pgtable_quirks = arm_smmu_set_pgtable_quirks,
1617 .free = arm_smmu_domain_free,
1618 }
1619 };
1620
arm_smmu_device_reset(struct arm_smmu_device * smmu)1621 static void arm_smmu_device_reset(struct arm_smmu_device *smmu)
1622 {
1623 int i;
1624 u32 reg;
1625
1626 /* clear global FSR */
1627 reg = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSR);
1628 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sGFSR, reg);
1629
1630 /*
1631 * Reset stream mapping groups: Initial values mark all SMRn as
1632 * invalid and all S2CRn as bypass unless overridden.
1633 */
1634 for (i = 0; i < smmu->num_mapping_groups; ++i)
1635 arm_smmu_write_sme(smmu, i);
1636
1637 /* Make sure all context banks are disabled and clear CB_FSR */
1638 for (i = 0; i < smmu->num_context_banks; ++i) {
1639 arm_smmu_write_context_bank(smmu, i);
1640 arm_smmu_cb_write(smmu, i, ARM_SMMU_CB_FSR, ARM_SMMU_FSR_FAULT);
1641 }
1642
1643 /* Invalidate the TLB, just in case */
1644 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_TLBIALLH, QCOM_DUMMY_VAL);
1645 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_TLBIALLNSNH, QCOM_DUMMY_VAL);
1646
1647 reg = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sCR0);
1648
1649 /* Enable fault reporting */
1650 reg |= (ARM_SMMU_sCR0_GFRE | ARM_SMMU_sCR0_GFIE |
1651 ARM_SMMU_sCR0_GCFGFRE | ARM_SMMU_sCR0_GCFGFIE);
1652
1653 /* Disable TLB broadcasting. */
1654 reg |= (ARM_SMMU_sCR0_VMIDPNE | ARM_SMMU_sCR0_PTM);
1655
1656 /* Enable client access, handling unmatched streams as appropriate */
1657 reg &= ~ARM_SMMU_sCR0_CLIENTPD;
1658 if (disable_bypass)
1659 reg |= ARM_SMMU_sCR0_USFCFG;
1660 else
1661 reg &= ~ARM_SMMU_sCR0_USFCFG;
1662
1663 /* Disable forced broadcasting */
1664 reg &= ~ARM_SMMU_sCR0_FB;
1665
1666 /* Don't upgrade barriers */
1667 reg &= ~(ARM_SMMU_sCR0_BSU);
1668
1669 if (smmu->features & ARM_SMMU_FEAT_VMID16)
1670 reg |= ARM_SMMU_sCR0_VMID16EN;
1671
1672 if (smmu->features & ARM_SMMU_FEAT_EXIDS)
1673 reg |= ARM_SMMU_sCR0_EXIDENABLE;
1674
1675 if (smmu->impl && smmu->impl->reset)
1676 smmu->impl->reset(smmu);
1677
1678 /* Push the button */
1679 arm_smmu_tlb_sync_global(smmu);
1680 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sCR0, reg);
1681 }
1682
arm_smmu_id_size_to_bits(int size)1683 static int arm_smmu_id_size_to_bits(int size)
1684 {
1685 switch (size) {
1686 case 0:
1687 return 32;
1688 case 1:
1689 return 36;
1690 case 2:
1691 return 40;
1692 case 3:
1693 return 42;
1694 case 4:
1695 return 44;
1696 case 5:
1697 default:
1698 return 48;
1699 }
1700 }
1701
arm_smmu_device_cfg_probe(struct arm_smmu_device * smmu)1702 static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu)
1703 {
1704 unsigned int size;
1705 u32 id;
1706 bool cttw_reg, cttw_fw = smmu->features & ARM_SMMU_FEAT_COHERENT_WALK;
1707 int i, ret;
1708
1709 dev_notice(smmu->dev, "probing hardware configuration...\n");
1710 dev_notice(smmu->dev, "SMMUv%d with:\n",
1711 smmu->version == ARM_SMMU_V2 ? 2 : 1);
1712
1713 /* ID0 */
1714 id = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_ID0);
1715
1716 /* Restrict available stages based on module parameter */
1717 if (force_stage == 1)
1718 id &= ~(ARM_SMMU_ID0_S2TS | ARM_SMMU_ID0_NTS);
1719 else if (force_stage == 2)
1720 id &= ~(ARM_SMMU_ID0_S1TS | ARM_SMMU_ID0_NTS);
1721
1722 if (id & ARM_SMMU_ID0_S1TS) {
1723 smmu->features |= ARM_SMMU_FEAT_TRANS_S1;
1724 dev_notice(smmu->dev, "\tstage 1 translation\n");
1725 }
1726
1727 if (id & ARM_SMMU_ID0_S2TS) {
1728 smmu->features |= ARM_SMMU_FEAT_TRANS_S2;
1729 dev_notice(smmu->dev, "\tstage 2 translation\n");
1730 }
1731
1732 if (id & ARM_SMMU_ID0_NTS) {
1733 smmu->features |= ARM_SMMU_FEAT_TRANS_NESTED;
1734 dev_notice(smmu->dev, "\tnested translation\n");
1735 }
1736
1737 if (!(smmu->features &
1738 (ARM_SMMU_FEAT_TRANS_S1 | ARM_SMMU_FEAT_TRANS_S2))) {
1739 dev_err(smmu->dev, "\tno translation support!\n");
1740 return -ENODEV;
1741 }
1742
1743 if ((id & ARM_SMMU_ID0_S1TS) &&
1744 ((smmu->version < ARM_SMMU_V2) || !(id & ARM_SMMU_ID0_ATOSNS))) {
1745 smmu->features |= ARM_SMMU_FEAT_TRANS_OPS;
1746 dev_notice(smmu->dev, "\taddress translation ops\n");
1747 }
1748
1749 /*
1750 * In order for DMA API calls to work properly, we must defer to what
1751 * the FW says about coherency, regardless of what the hardware claims.
1752 * Fortunately, this also opens up a workaround for systems where the
1753 * ID register value has ended up configured incorrectly.
1754 */
1755 cttw_reg = !!(id & ARM_SMMU_ID0_CTTW);
1756 if (cttw_fw || cttw_reg)
1757 dev_notice(smmu->dev, "\t%scoherent table walk\n",
1758 cttw_fw ? "" : "non-");
1759 if (cttw_fw != cttw_reg)
1760 dev_notice(smmu->dev,
1761 "\t(IDR0.CTTW overridden by FW configuration)\n");
1762
1763 /* Max. number of entries we have for stream matching/indexing */
1764 if (smmu->version == ARM_SMMU_V2 && id & ARM_SMMU_ID0_EXIDS) {
1765 smmu->features |= ARM_SMMU_FEAT_EXIDS;
1766 size = 1 << 16;
1767 } else {
1768 size = 1 << FIELD_GET(ARM_SMMU_ID0_NUMSIDB, id);
1769 }
1770 smmu->streamid_mask = size - 1;
1771 if (id & ARM_SMMU_ID0_SMS) {
1772 smmu->features |= ARM_SMMU_FEAT_STREAM_MATCH;
1773 size = FIELD_GET(ARM_SMMU_ID0_NUMSMRG, id);
1774 if (size == 0) {
1775 dev_err(smmu->dev,
1776 "stream-matching supported, but no SMRs present!\n");
1777 return -ENODEV;
1778 }
1779
1780 /* Zero-initialised to mark as invalid */
1781 smmu->smrs = devm_kcalloc(smmu->dev, size, sizeof(*smmu->smrs),
1782 GFP_KERNEL);
1783 if (!smmu->smrs)
1784 return -ENOMEM;
1785
1786 dev_notice(smmu->dev,
1787 "\tstream matching with %u register groups", size);
1788 }
1789 /* s2cr->type == 0 means translation, so initialise explicitly */
1790 smmu->s2crs = devm_kmalloc_array(smmu->dev, size, sizeof(*smmu->s2crs),
1791 GFP_KERNEL);
1792 if (!smmu->s2crs)
1793 return -ENOMEM;
1794 for (i = 0; i < size; i++)
1795 smmu->s2crs[i] = s2cr_init_val;
1796
1797 smmu->num_mapping_groups = size;
1798 mutex_init(&smmu->stream_map_mutex);
1799 spin_lock_init(&smmu->global_sync_lock);
1800
1801 if (smmu->version < ARM_SMMU_V2 ||
1802 !(id & ARM_SMMU_ID0_PTFS_NO_AARCH32)) {
1803 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH32_L;
1804 if (!(id & ARM_SMMU_ID0_PTFS_NO_AARCH32S))
1805 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH32_S;
1806 }
1807
1808 /* ID1 */
1809 id = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_ID1);
1810 smmu->pgshift = (id & ARM_SMMU_ID1_PAGESIZE) ? 16 : 12;
1811
1812 /* Check for size mismatch of SMMU address space from mapped region */
1813 size = 1 << (FIELD_GET(ARM_SMMU_ID1_NUMPAGENDXB, id) + 1);
1814 if (smmu->numpage != 2 * size << smmu->pgshift)
1815 dev_warn(smmu->dev,
1816 "SMMU address space size (0x%x) differs from mapped region size (0x%x)!\n",
1817 2 * size << smmu->pgshift, smmu->numpage);
1818 /* Now properly encode NUMPAGE to subsequently derive SMMU_CB_BASE */
1819 smmu->numpage = size;
1820
1821 smmu->num_s2_context_banks = FIELD_GET(ARM_SMMU_ID1_NUMS2CB, id);
1822 smmu->num_context_banks = FIELD_GET(ARM_SMMU_ID1_NUMCB, id);
1823 if (smmu->num_s2_context_banks > smmu->num_context_banks) {
1824 dev_err(smmu->dev, "impossible number of S2 context banks!\n");
1825 return -ENODEV;
1826 }
1827 dev_notice(smmu->dev, "\t%u context banks (%u stage-2 only)\n",
1828 smmu->num_context_banks, smmu->num_s2_context_banks);
1829 smmu->cbs = devm_kcalloc(smmu->dev, smmu->num_context_banks,
1830 sizeof(*smmu->cbs), GFP_KERNEL);
1831 if (!smmu->cbs)
1832 return -ENOMEM;
1833
1834 /* ID2 */
1835 id = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_ID2);
1836 size = arm_smmu_id_size_to_bits(FIELD_GET(ARM_SMMU_ID2_IAS, id));
1837 smmu->ipa_size = size;
1838
1839 /* The output mask is also applied for bypass */
1840 size = arm_smmu_id_size_to_bits(FIELD_GET(ARM_SMMU_ID2_OAS, id));
1841 smmu->pa_size = size;
1842
1843 if (id & ARM_SMMU_ID2_VMID16)
1844 smmu->features |= ARM_SMMU_FEAT_VMID16;
1845
1846 /*
1847 * What the page table walker can address actually depends on which
1848 * descriptor format is in use, but since a) we don't know that yet,
1849 * and b) it can vary per context bank, this will have to do...
1850 */
1851 if (dma_set_mask_and_coherent(smmu->dev, DMA_BIT_MASK(size)))
1852 dev_warn(smmu->dev,
1853 "failed to set DMA mask for table walker\n");
1854
1855 if (smmu->version < ARM_SMMU_V2) {
1856 smmu->va_size = smmu->ipa_size;
1857 if (smmu->version == ARM_SMMU_V1_64K)
1858 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_64K;
1859 } else {
1860 size = FIELD_GET(ARM_SMMU_ID2_UBS, id);
1861 smmu->va_size = arm_smmu_id_size_to_bits(size);
1862 if (id & ARM_SMMU_ID2_PTFS_4K)
1863 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_4K;
1864 if (id & ARM_SMMU_ID2_PTFS_16K)
1865 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_16K;
1866 if (id & ARM_SMMU_ID2_PTFS_64K)
1867 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_64K;
1868 }
1869
1870 if (smmu->impl && smmu->impl->cfg_probe) {
1871 ret = smmu->impl->cfg_probe(smmu);
1872 if (ret)
1873 return ret;
1874 }
1875
1876 /* Now we've corralled the various formats, what'll it do? */
1877 if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_S)
1878 smmu->pgsize_bitmap |= SZ_4K | SZ_64K | SZ_1M | SZ_16M;
1879 if (smmu->features &
1880 (ARM_SMMU_FEAT_FMT_AARCH32_L | ARM_SMMU_FEAT_FMT_AARCH64_4K))
1881 smmu->pgsize_bitmap |= SZ_4K | SZ_2M | SZ_1G;
1882 if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH64_16K)
1883 smmu->pgsize_bitmap |= SZ_16K | SZ_32M;
1884 if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH64_64K)
1885 smmu->pgsize_bitmap |= SZ_64K | SZ_512M;
1886
1887 if (arm_smmu_ops.pgsize_bitmap == -1UL)
1888 arm_smmu_ops.pgsize_bitmap = smmu->pgsize_bitmap;
1889 else
1890 arm_smmu_ops.pgsize_bitmap |= smmu->pgsize_bitmap;
1891 dev_notice(smmu->dev, "\tSupported page sizes: 0x%08lx\n",
1892 smmu->pgsize_bitmap);
1893
1894
1895 if (smmu->features & ARM_SMMU_FEAT_TRANS_S1)
1896 dev_notice(smmu->dev, "\tStage-1: %lu-bit VA -> %lu-bit IPA\n",
1897 smmu->va_size, smmu->ipa_size);
1898
1899 if (smmu->features & ARM_SMMU_FEAT_TRANS_S2)
1900 dev_notice(smmu->dev, "\tStage-2: %lu-bit IPA -> %lu-bit PA\n",
1901 smmu->ipa_size, smmu->pa_size);
1902
1903 return 0;
1904 }
1905
1906 struct arm_smmu_match_data {
1907 enum arm_smmu_arch_version version;
1908 enum arm_smmu_implementation model;
1909 };
1910
1911 #define ARM_SMMU_MATCH_DATA(name, ver, imp) \
1912 static const struct arm_smmu_match_data name = { .version = ver, .model = imp }
1913
1914 ARM_SMMU_MATCH_DATA(smmu_generic_v1, ARM_SMMU_V1, GENERIC_SMMU);
1915 ARM_SMMU_MATCH_DATA(smmu_generic_v2, ARM_SMMU_V2, GENERIC_SMMU);
1916 ARM_SMMU_MATCH_DATA(arm_mmu401, ARM_SMMU_V1_64K, GENERIC_SMMU);
1917 ARM_SMMU_MATCH_DATA(arm_mmu500, ARM_SMMU_V2, ARM_MMU500);
1918 ARM_SMMU_MATCH_DATA(cavium_smmuv2, ARM_SMMU_V2, CAVIUM_SMMUV2);
1919 ARM_SMMU_MATCH_DATA(qcom_smmuv2, ARM_SMMU_V2, QCOM_SMMUV2);
1920
1921 static const struct of_device_id arm_smmu_of_match[] = {
1922 { .compatible = "arm,smmu-v1", .data = &smmu_generic_v1 },
1923 { .compatible = "arm,smmu-v2", .data = &smmu_generic_v2 },
1924 { .compatible = "arm,mmu-400", .data = &smmu_generic_v1 },
1925 { .compatible = "arm,mmu-401", .data = &arm_mmu401 },
1926 { .compatible = "arm,mmu-500", .data = &arm_mmu500 },
1927 { .compatible = "cavium,smmu-v2", .data = &cavium_smmuv2 },
1928 { .compatible = "nvidia,smmu-500", .data = &arm_mmu500 },
1929 { .compatible = "qcom,smmu-v2", .data = &qcom_smmuv2 },
1930 { },
1931 };
1932 MODULE_DEVICE_TABLE(of, arm_smmu_of_match);
1933
1934 #ifdef CONFIG_ACPI
acpi_smmu_get_data(u32 model,struct arm_smmu_device * smmu)1935 static int acpi_smmu_get_data(u32 model, struct arm_smmu_device *smmu)
1936 {
1937 int ret = 0;
1938
1939 switch (model) {
1940 case ACPI_IORT_SMMU_V1:
1941 case ACPI_IORT_SMMU_CORELINK_MMU400:
1942 smmu->version = ARM_SMMU_V1;
1943 smmu->model = GENERIC_SMMU;
1944 break;
1945 case ACPI_IORT_SMMU_CORELINK_MMU401:
1946 smmu->version = ARM_SMMU_V1_64K;
1947 smmu->model = GENERIC_SMMU;
1948 break;
1949 case ACPI_IORT_SMMU_V2:
1950 smmu->version = ARM_SMMU_V2;
1951 smmu->model = GENERIC_SMMU;
1952 break;
1953 case ACPI_IORT_SMMU_CORELINK_MMU500:
1954 smmu->version = ARM_SMMU_V2;
1955 smmu->model = ARM_MMU500;
1956 break;
1957 case ACPI_IORT_SMMU_CAVIUM_THUNDERX:
1958 smmu->version = ARM_SMMU_V2;
1959 smmu->model = CAVIUM_SMMUV2;
1960 break;
1961 default:
1962 ret = -ENODEV;
1963 }
1964
1965 return ret;
1966 }
1967
arm_smmu_device_acpi_probe(struct arm_smmu_device * smmu,u32 * global_irqs,u32 * pmu_irqs)1968 static int arm_smmu_device_acpi_probe(struct arm_smmu_device *smmu,
1969 u32 *global_irqs, u32 *pmu_irqs)
1970 {
1971 struct device *dev = smmu->dev;
1972 struct acpi_iort_node *node =
1973 *(struct acpi_iort_node **)dev_get_platdata(dev);
1974 struct acpi_iort_smmu *iort_smmu;
1975 int ret;
1976
1977 /* Retrieve SMMU1/2 specific data */
1978 iort_smmu = (struct acpi_iort_smmu *)node->node_data;
1979
1980 ret = acpi_smmu_get_data(iort_smmu->model, smmu);
1981 if (ret < 0)
1982 return ret;
1983
1984 /* Ignore the configuration access interrupt */
1985 *global_irqs = 1;
1986 *pmu_irqs = 0;
1987
1988 if (iort_smmu->flags & ACPI_IORT_SMMU_COHERENT_WALK)
1989 smmu->features |= ARM_SMMU_FEAT_COHERENT_WALK;
1990
1991 return 0;
1992 }
1993 #else
arm_smmu_device_acpi_probe(struct arm_smmu_device * smmu,u32 * global_irqs,u32 * pmu_irqs)1994 static inline int arm_smmu_device_acpi_probe(struct arm_smmu_device *smmu,
1995 u32 *global_irqs, u32 *pmu_irqs)
1996 {
1997 return -ENODEV;
1998 }
1999 #endif
2000
arm_smmu_device_dt_probe(struct arm_smmu_device * smmu,u32 * global_irqs,u32 * pmu_irqs)2001 static int arm_smmu_device_dt_probe(struct arm_smmu_device *smmu,
2002 u32 *global_irqs, u32 *pmu_irqs)
2003 {
2004 const struct arm_smmu_match_data *data;
2005 struct device *dev = smmu->dev;
2006 bool legacy_binding;
2007
2008 if (of_property_read_u32(dev->of_node, "#global-interrupts", global_irqs))
2009 return dev_err_probe(dev, -ENODEV,
2010 "missing #global-interrupts property\n");
2011 *pmu_irqs = 0;
2012
2013 data = of_device_get_match_data(dev);
2014 smmu->version = data->version;
2015 smmu->model = data->model;
2016
2017 legacy_binding = of_find_property(dev->of_node, "mmu-masters", NULL);
2018 if (legacy_binding && !using_generic_binding) {
2019 if (!using_legacy_binding) {
2020 pr_notice("deprecated \"mmu-masters\" DT property in use; %s support unavailable\n",
2021 IS_ENABLED(CONFIG_ARM_SMMU_LEGACY_DT_BINDINGS) ? "DMA API" : "SMMU");
2022 }
2023 using_legacy_binding = true;
2024 } else if (!legacy_binding && !using_legacy_binding) {
2025 using_generic_binding = true;
2026 } else {
2027 dev_err(dev, "not probing due to mismatched DT properties\n");
2028 return -ENODEV;
2029 }
2030
2031 if (of_dma_is_coherent(dev->of_node))
2032 smmu->features |= ARM_SMMU_FEAT_COHERENT_WALK;
2033
2034 return 0;
2035 }
2036
arm_smmu_rmr_install_bypass_smr(struct arm_smmu_device * smmu)2037 static void arm_smmu_rmr_install_bypass_smr(struct arm_smmu_device *smmu)
2038 {
2039 struct list_head rmr_list;
2040 struct iommu_resv_region *e;
2041 int idx, cnt = 0;
2042 u32 reg;
2043
2044 INIT_LIST_HEAD(&rmr_list);
2045 iort_get_rmr_sids(dev_fwnode(smmu->dev), &rmr_list);
2046
2047 /*
2048 * Rather than trying to look at existing mappings that
2049 * are setup by the firmware and then invalidate the ones
2050 * that do no have matching RMR entries, just disable the
2051 * SMMU until it gets enabled again in the reset routine.
2052 */
2053 reg = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sCR0);
2054 reg |= ARM_SMMU_sCR0_CLIENTPD;
2055 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sCR0, reg);
2056
2057 list_for_each_entry(e, &rmr_list, list) {
2058 struct iommu_iort_rmr_data *rmr;
2059 int i;
2060
2061 rmr = container_of(e, struct iommu_iort_rmr_data, rr);
2062 for (i = 0; i < rmr->num_sids; i++) {
2063 idx = arm_smmu_find_sme(smmu, rmr->sids[i], ~0);
2064 if (idx < 0)
2065 continue;
2066
2067 if (smmu->s2crs[idx].count == 0) {
2068 smmu->smrs[idx].id = rmr->sids[i];
2069 smmu->smrs[idx].mask = 0;
2070 smmu->smrs[idx].valid = true;
2071 }
2072 smmu->s2crs[idx].count++;
2073 smmu->s2crs[idx].type = S2CR_TYPE_BYPASS;
2074 smmu->s2crs[idx].privcfg = S2CR_PRIVCFG_DEFAULT;
2075
2076 cnt++;
2077 }
2078 }
2079
2080 dev_notice(smmu->dev, "\tpreserved %d boot mapping%s\n", cnt,
2081 cnt == 1 ? "" : "s");
2082 iort_put_rmr_sids(dev_fwnode(smmu->dev), &rmr_list);
2083 }
2084
arm_smmu_device_probe(struct platform_device * pdev)2085 static int arm_smmu_device_probe(struct platform_device *pdev)
2086 {
2087 struct resource *res;
2088 struct arm_smmu_device *smmu;
2089 struct device *dev = &pdev->dev;
2090 int num_irqs, i, err;
2091 u32 global_irqs, pmu_irqs;
2092 irqreturn_t (*global_fault)(int irq, void *dev);
2093
2094 smmu = devm_kzalloc(dev, sizeof(*smmu), GFP_KERNEL);
2095 if (!smmu) {
2096 dev_err(dev, "failed to allocate arm_smmu_device\n");
2097 return -ENOMEM;
2098 }
2099 smmu->dev = dev;
2100
2101 if (dev->of_node)
2102 err = arm_smmu_device_dt_probe(smmu, &global_irqs, &pmu_irqs);
2103 else
2104 err = arm_smmu_device_acpi_probe(smmu, &global_irqs, &pmu_irqs);
2105 if (err)
2106 return err;
2107
2108 smmu->base = devm_platform_get_and_ioremap_resource(pdev, 0, &res);
2109 if (IS_ERR(smmu->base))
2110 return PTR_ERR(smmu->base);
2111 smmu->ioaddr = res->start;
2112
2113 /*
2114 * The resource size should effectively match the value of SMMU_TOP;
2115 * stash that temporarily until we know PAGESIZE to validate it with.
2116 */
2117 smmu->numpage = resource_size(res);
2118
2119 smmu = arm_smmu_impl_init(smmu);
2120 if (IS_ERR(smmu))
2121 return PTR_ERR(smmu);
2122
2123 num_irqs = platform_irq_count(pdev);
2124
2125 smmu->num_context_irqs = num_irqs - global_irqs - pmu_irqs;
2126 if (smmu->num_context_irqs <= 0)
2127 return dev_err_probe(dev, -ENODEV,
2128 "found %d interrupts but expected at least %d\n",
2129 num_irqs, global_irqs + pmu_irqs + 1);
2130
2131 smmu->irqs = devm_kcalloc(dev, smmu->num_context_irqs,
2132 sizeof(*smmu->irqs), GFP_KERNEL);
2133 if (!smmu->irqs)
2134 return dev_err_probe(dev, -ENOMEM, "failed to allocate %d irqs\n",
2135 smmu->num_context_irqs);
2136
2137 for (i = 0; i < smmu->num_context_irqs; i++) {
2138 int irq = platform_get_irq(pdev, global_irqs + pmu_irqs + i);
2139
2140 if (irq < 0)
2141 return irq;
2142 smmu->irqs[i] = irq;
2143 }
2144
2145 err = devm_clk_bulk_get_all(dev, &smmu->clks);
2146 if (err < 0) {
2147 dev_err(dev, "failed to get clocks %d\n", err);
2148 return err;
2149 }
2150 smmu->num_clks = err;
2151
2152 err = clk_bulk_prepare_enable(smmu->num_clks, smmu->clks);
2153 if (err)
2154 return err;
2155
2156 err = arm_smmu_device_cfg_probe(smmu);
2157 if (err)
2158 return err;
2159
2160 if (smmu->version == ARM_SMMU_V2) {
2161 if (smmu->num_context_banks > smmu->num_context_irqs) {
2162 dev_err(dev,
2163 "found only %d context irq(s) but %d required\n",
2164 smmu->num_context_irqs, smmu->num_context_banks);
2165 return -ENODEV;
2166 }
2167
2168 /* Ignore superfluous interrupts */
2169 smmu->num_context_irqs = smmu->num_context_banks;
2170 }
2171
2172 if (smmu->impl && smmu->impl->global_fault)
2173 global_fault = smmu->impl->global_fault;
2174 else
2175 global_fault = arm_smmu_global_fault;
2176
2177 for (i = 0; i < global_irqs; i++) {
2178 int irq = platform_get_irq(pdev, i);
2179
2180 if (irq < 0)
2181 return irq;
2182
2183 err = devm_request_irq(dev, irq, global_fault, IRQF_SHARED,
2184 "arm-smmu global fault", smmu);
2185 if (err)
2186 return dev_err_probe(dev, err,
2187 "failed to request global IRQ %d (%u)\n",
2188 i, irq);
2189 }
2190
2191 err = iommu_device_sysfs_add(&smmu->iommu, smmu->dev, NULL,
2192 "smmu.%pa", &smmu->ioaddr);
2193 if (err) {
2194 dev_err(dev, "Failed to register iommu in sysfs\n");
2195 return err;
2196 }
2197
2198 err = iommu_device_register(&smmu->iommu, &arm_smmu_ops,
2199 using_legacy_binding ? NULL : dev);
2200 if (err) {
2201 dev_err(dev, "Failed to register iommu\n");
2202 iommu_device_sysfs_remove(&smmu->iommu);
2203 return err;
2204 }
2205
2206 platform_set_drvdata(pdev, smmu);
2207
2208 /* Check for RMRs and install bypass SMRs if any */
2209 arm_smmu_rmr_install_bypass_smr(smmu);
2210
2211 arm_smmu_device_reset(smmu);
2212 arm_smmu_test_smr_masks(smmu);
2213
2214 /*
2215 * We want to avoid touching dev->power.lock in fastpaths unless
2216 * it's really going to do something useful - pm_runtime_enabled()
2217 * can serve as an ideal proxy for that decision. So, conditionally
2218 * enable pm_runtime.
2219 */
2220 if (dev->pm_domain) {
2221 pm_runtime_set_active(dev);
2222 pm_runtime_enable(dev);
2223 }
2224
2225 return 0;
2226 }
2227
arm_smmu_device_shutdown(struct platform_device * pdev)2228 static void arm_smmu_device_shutdown(struct platform_device *pdev)
2229 {
2230 struct arm_smmu_device *smmu = platform_get_drvdata(pdev);
2231
2232 if (!bitmap_empty(smmu->context_map, ARM_SMMU_MAX_CBS))
2233 dev_notice(&pdev->dev, "disabling translation\n");
2234
2235 arm_smmu_rpm_get(smmu);
2236 /* Turn the thing off */
2237 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sCR0, ARM_SMMU_sCR0_CLIENTPD);
2238 arm_smmu_rpm_put(smmu);
2239
2240 if (pm_runtime_enabled(smmu->dev))
2241 pm_runtime_force_suspend(smmu->dev);
2242 else
2243 clk_bulk_disable(smmu->num_clks, smmu->clks);
2244
2245 clk_bulk_unprepare(smmu->num_clks, smmu->clks);
2246 }
2247
arm_smmu_device_remove(struct platform_device * pdev)2248 static void arm_smmu_device_remove(struct platform_device *pdev)
2249 {
2250 struct arm_smmu_device *smmu = platform_get_drvdata(pdev);
2251
2252 iommu_device_unregister(&smmu->iommu);
2253 iommu_device_sysfs_remove(&smmu->iommu);
2254
2255 arm_smmu_device_shutdown(pdev);
2256 }
2257
arm_smmu_runtime_resume(struct device * dev)2258 static int __maybe_unused arm_smmu_runtime_resume(struct device *dev)
2259 {
2260 struct arm_smmu_device *smmu = dev_get_drvdata(dev);
2261 int ret;
2262
2263 ret = clk_bulk_enable(smmu->num_clks, smmu->clks);
2264 if (ret)
2265 return ret;
2266
2267 arm_smmu_device_reset(smmu);
2268
2269 return 0;
2270 }
2271
arm_smmu_runtime_suspend(struct device * dev)2272 static int __maybe_unused arm_smmu_runtime_suspend(struct device *dev)
2273 {
2274 struct arm_smmu_device *smmu = dev_get_drvdata(dev);
2275
2276 clk_bulk_disable(smmu->num_clks, smmu->clks);
2277
2278 return 0;
2279 }
2280
arm_smmu_pm_resume(struct device * dev)2281 static int __maybe_unused arm_smmu_pm_resume(struct device *dev)
2282 {
2283 int ret;
2284 struct arm_smmu_device *smmu = dev_get_drvdata(dev);
2285
2286 ret = clk_bulk_prepare(smmu->num_clks, smmu->clks);
2287 if (ret)
2288 return ret;
2289
2290 if (pm_runtime_suspended(dev))
2291 return 0;
2292
2293 ret = arm_smmu_runtime_resume(dev);
2294 if (ret)
2295 clk_bulk_unprepare(smmu->num_clks, smmu->clks);
2296
2297 return ret;
2298 }
2299
arm_smmu_pm_suspend(struct device * dev)2300 static int __maybe_unused arm_smmu_pm_suspend(struct device *dev)
2301 {
2302 int ret = 0;
2303 struct arm_smmu_device *smmu = dev_get_drvdata(dev);
2304
2305 if (pm_runtime_suspended(dev))
2306 goto clk_unprepare;
2307
2308 ret = arm_smmu_runtime_suspend(dev);
2309 if (ret)
2310 return ret;
2311
2312 clk_unprepare:
2313 clk_bulk_unprepare(smmu->num_clks, smmu->clks);
2314 return ret;
2315 }
2316
2317 static const struct dev_pm_ops arm_smmu_pm_ops = {
2318 SET_SYSTEM_SLEEP_PM_OPS(arm_smmu_pm_suspend, arm_smmu_pm_resume)
2319 SET_RUNTIME_PM_OPS(arm_smmu_runtime_suspend,
2320 arm_smmu_runtime_resume, NULL)
2321 };
2322
2323 static struct platform_driver arm_smmu_driver = {
2324 .driver = {
2325 .name = "arm-smmu",
2326 .of_match_table = arm_smmu_of_match,
2327 .pm = &arm_smmu_pm_ops,
2328 .suppress_bind_attrs = true,
2329 },
2330 .probe = arm_smmu_device_probe,
2331 .remove_new = arm_smmu_device_remove,
2332 .shutdown = arm_smmu_device_shutdown,
2333 };
2334 module_platform_driver(arm_smmu_driver);
2335
2336 MODULE_DESCRIPTION("IOMMU API for ARM architected SMMU implementations");
2337 MODULE_AUTHOR("Will Deacon <will@kernel.org>");
2338 MODULE_ALIAS("platform:arm-smmu");
2339 MODULE_LICENSE("GPL v2");
2340