1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * IOMMU API for ARM architected SMMUv3 implementations.
4 *
5 * Copyright (C) 2015 ARM Limited
6 *
7 * Author: Will Deacon <will.deacon@arm.com>
8 *
9 * This driver is powered by bad coffee and bombay mix.
10 */
11
12 #include <linux/acpi.h>
13 #include <linux/acpi_iort.h>
14 #include <linux/bitops.h>
15 #include <linux/crash_dump.h>
16 #include <linux/delay.h>
17 #include <linux/err.h>
18 #include <linux/interrupt.h>
19 #include <linux/io-pgtable.h>
20 #include <linux/iopoll.h>
21 #include <linux/module.h>
22 #include <linux/msi.h>
23 #include <linux/of.h>
24 #include <linux/of_address.h>
25 #include <linux/of_platform.h>
26 #include <linux/pci.h>
27 #include <linux/pci-ats.h>
28 #include <linux/platform_device.h>
29 #include <linux/string_choices.h>
30 #include <kunit/visibility.h>
31 #include <uapi/linux/iommufd.h>
32
33 #include "arm-smmu-v3.h"
34 #include "../../dma-iommu.h"
35
36 static bool disable_msipolling;
37 module_param(disable_msipolling, bool, 0444);
38 MODULE_PARM_DESC(disable_msipolling,
39 "Disable MSI-based polling for CMD_SYNC completion.");
40
41 static struct iommu_ops arm_smmu_ops;
42 static struct iommu_dirty_ops arm_smmu_dirty_ops;
43
44 enum arm_smmu_msi_index {
45 EVTQ_MSI_INDEX,
46 GERROR_MSI_INDEX,
47 PRIQ_MSI_INDEX,
48 ARM_SMMU_MAX_MSIS,
49 };
50
51 #define NUM_ENTRY_QWORDS 8
52 static_assert(sizeof(struct arm_smmu_ste) == NUM_ENTRY_QWORDS * sizeof(u64));
53 static_assert(sizeof(struct arm_smmu_cd) == NUM_ENTRY_QWORDS * sizeof(u64));
54
55 static phys_addr_t arm_smmu_msi_cfg[ARM_SMMU_MAX_MSIS][3] = {
56 [EVTQ_MSI_INDEX] = {
57 ARM_SMMU_EVTQ_IRQ_CFG0,
58 ARM_SMMU_EVTQ_IRQ_CFG1,
59 ARM_SMMU_EVTQ_IRQ_CFG2,
60 },
61 [GERROR_MSI_INDEX] = {
62 ARM_SMMU_GERROR_IRQ_CFG0,
63 ARM_SMMU_GERROR_IRQ_CFG1,
64 ARM_SMMU_GERROR_IRQ_CFG2,
65 },
66 [PRIQ_MSI_INDEX] = {
67 ARM_SMMU_PRIQ_IRQ_CFG0,
68 ARM_SMMU_PRIQ_IRQ_CFG1,
69 ARM_SMMU_PRIQ_IRQ_CFG2,
70 },
71 };
72
73 struct arm_smmu_option_prop {
74 u32 opt;
75 const char *prop;
76 };
77
78 DEFINE_XARRAY_ALLOC1(arm_smmu_asid_xa);
79 DEFINE_MUTEX(arm_smmu_asid_lock);
80
81 static struct arm_smmu_option_prop arm_smmu_options[] = {
82 { ARM_SMMU_OPT_SKIP_PREFETCH, "hisilicon,broken-prefetch-cmd" },
83 { ARM_SMMU_OPT_PAGE0_REGS_ONLY, "cavium,cn9900-broken-page1-regspace"},
84 { 0, NULL},
85 };
86
87 static const char * const event_str[] = {
88 [EVT_ID_BAD_STREAMID_CONFIG] = "C_BAD_STREAMID",
89 [EVT_ID_STE_FETCH_FAULT] = "F_STE_FETCH",
90 [EVT_ID_BAD_STE_CONFIG] = "C_BAD_STE",
91 [EVT_ID_STREAM_DISABLED_FAULT] = "F_STREAM_DISABLED",
92 [EVT_ID_BAD_SUBSTREAMID_CONFIG] = "C_BAD_SUBSTREAMID",
93 [EVT_ID_CD_FETCH_FAULT] = "F_CD_FETCH",
94 [EVT_ID_BAD_CD_CONFIG] = "C_BAD_CD",
95 [EVT_ID_TRANSLATION_FAULT] = "F_TRANSLATION",
96 [EVT_ID_ADDR_SIZE_FAULT] = "F_ADDR_SIZE",
97 [EVT_ID_ACCESS_FAULT] = "F_ACCESS",
98 [EVT_ID_PERMISSION_FAULT] = "F_PERMISSION",
99 [EVT_ID_VMS_FETCH_FAULT] = "F_VMS_FETCH",
100 };
101
102 static const char * const event_class_str[] = {
103 [0] = "CD fetch",
104 [1] = "Stage 1 translation table fetch",
105 [2] = "Input address caused fault",
106 [3] = "Reserved",
107 };
108
109 static int arm_smmu_alloc_cd_tables(struct arm_smmu_master *master);
110
parse_driver_options(struct arm_smmu_device * smmu)111 static void parse_driver_options(struct arm_smmu_device *smmu)
112 {
113 int i = 0;
114
115 do {
116 if (of_property_read_bool(smmu->dev->of_node,
117 arm_smmu_options[i].prop)) {
118 smmu->options |= arm_smmu_options[i].opt;
119 dev_notice(smmu->dev, "option %s\n",
120 arm_smmu_options[i].prop);
121 }
122 } while (arm_smmu_options[++i].opt);
123 }
124
125 /* Low-level queue manipulation functions */
queue_has_space(struct arm_smmu_ll_queue * q,u32 n)126 static bool queue_has_space(struct arm_smmu_ll_queue *q, u32 n)
127 {
128 u32 space, prod, cons;
129
130 prod = Q_IDX(q, q->prod);
131 cons = Q_IDX(q, q->cons);
132
133 if (Q_WRP(q, q->prod) == Q_WRP(q, q->cons))
134 space = (1 << q->max_n_shift) - (prod - cons);
135 else
136 space = cons - prod;
137
138 return space >= n;
139 }
140
queue_full(struct arm_smmu_ll_queue * q)141 static bool queue_full(struct arm_smmu_ll_queue *q)
142 {
143 return Q_IDX(q, q->prod) == Q_IDX(q, q->cons) &&
144 Q_WRP(q, q->prod) != Q_WRP(q, q->cons);
145 }
146
queue_empty(struct arm_smmu_ll_queue * q)147 static bool queue_empty(struct arm_smmu_ll_queue *q)
148 {
149 return Q_IDX(q, q->prod) == Q_IDX(q, q->cons) &&
150 Q_WRP(q, q->prod) == Q_WRP(q, q->cons);
151 }
152
queue_consumed(struct arm_smmu_ll_queue * q,u32 prod)153 static bool queue_consumed(struct arm_smmu_ll_queue *q, u32 prod)
154 {
155 return ((Q_WRP(q, q->cons) == Q_WRP(q, prod)) &&
156 (Q_IDX(q, q->cons) > Q_IDX(q, prod))) ||
157 ((Q_WRP(q, q->cons) != Q_WRP(q, prod)) &&
158 (Q_IDX(q, q->cons) <= Q_IDX(q, prod)));
159 }
160
queue_sync_cons_out(struct arm_smmu_queue * q)161 static void queue_sync_cons_out(struct arm_smmu_queue *q)
162 {
163 /*
164 * Ensure that all CPU accesses (reads and writes) to the queue
165 * are complete before we update the cons pointer.
166 */
167 __iomb();
168 writel_relaxed(q->llq.cons, q->cons_reg);
169 }
170
queue_inc_cons(struct arm_smmu_ll_queue * q)171 static void queue_inc_cons(struct arm_smmu_ll_queue *q)
172 {
173 u32 cons = (Q_WRP(q, q->cons) | Q_IDX(q, q->cons)) + 1;
174 q->cons = Q_OVF(q->cons) | Q_WRP(q, cons) | Q_IDX(q, cons);
175 }
176
queue_sync_cons_ovf(struct arm_smmu_queue * q)177 static void queue_sync_cons_ovf(struct arm_smmu_queue *q)
178 {
179 struct arm_smmu_ll_queue *llq = &q->llq;
180
181 if (likely(Q_OVF(llq->prod) == Q_OVF(llq->cons)))
182 return;
183
184 llq->cons = Q_OVF(llq->prod) | Q_WRP(llq, llq->cons) |
185 Q_IDX(llq, llq->cons);
186 queue_sync_cons_out(q);
187 }
188
queue_sync_prod_in(struct arm_smmu_queue * q)189 static int queue_sync_prod_in(struct arm_smmu_queue *q)
190 {
191 u32 prod;
192 int ret = 0;
193
194 /*
195 * We can't use the _relaxed() variant here, as we must prevent
196 * speculative reads of the queue before we have determined that
197 * prod has indeed moved.
198 */
199 prod = readl(q->prod_reg);
200
201 if (Q_OVF(prod) != Q_OVF(q->llq.prod))
202 ret = -EOVERFLOW;
203
204 q->llq.prod = prod;
205 return ret;
206 }
207
queue_inc_prod_n(struct arm_smmu_ll_queue * q,int n)208 static u32 queue_inc_prod_n(struct arm_smmu_ll_queue *q, int n)
209 {
210 u32 prod = (Q_WRP(q, q->prod) | Q_IDX(q, q->prod)) + n;
211 return Q_OVF(q->prod) | Q_WRP(q, prod) | Q_IDX(q, prod);
212 }
213
queue_poll_init(struct arm_smmu_device * smmu,struct arm_smmu_queue_poll * qp)214 static void queue_poll_init(struct arm_smmu_device *smmu,
215 struct arm_smmu_queue_poll *qp)
216 {
217 qp->delay = 1;
218 qp->spin_cnt = 0;
219 qp->wfe = !!(smmu->features & ARM_SMMU_FEAT_SEV);
220 qp->timeout = ktime_add_us(ktime_get(), ARM_SMMU_POLL_TIMEOUT_US);
221 }
222
queue_poll(struct arm_smmu_queue_poll * qp)223 static int queue_poll(struct arm_smmu_queue_poll *qp)
224 {
225 if (ktime_compare(ktime_get(), qp->timeout) > 0)
226 return -ETIMEDOUT;
227
228 if (qp->wfe) {
229 wfe();
230 } else if (++qp->spin_cnt < ARM_SMMU_POLL_SPIN_COUNT) {
231 cpu_relax();
232 } else {
233 udelay(qp->delay);
234 qp->delay *= 2;
235 qp->spin_cnt = 0;
236 }
237
238 return 0;
239 }
240
queue_write(__le64 * dst,u64 * src,size_t n_dwords)241 static void queue_write(__le64 *dst, u64 *src, size_t n_dwords)
242 {
243 int i;
244
245 for (i = 0; i < n_dwords; ++i)
246 *dst++ = cpu_to_le64(*src++);
247 }
248
queue_read(u64 * dst,__le64 * src,size_t n_dwords)249 static void queue_read(u64 *dst, __le64 *src, size_t n_dwords)
250 {
251 int i;
252
253 for (i = 0; i < n_dwords; ++i)
254 *dst++ = le64_to_cpu(*src++);
255 }
256
queue_remove_raw(struct arm_smmu_queue * q,u64 * ent)257 static int queue_remove_raw(struct arm_smmu_queue *q, u64 *ent)
258 {
259 if (queue_empty(&q->llq))
260 return -EAGAIN;
261
262 queue_read(ent, Q_ENT(q, q->llq.cons), q->ent_dwords);
263 queue_inc_cons(&q->llq);
264 queue_sync_cons_out(q);
265 return 0;
266 }
267
268 /* High-level queue accessors */
arm_smmu_cmdq_build_cmd(u64 * cmd,struct arm_smmu_cmdq_ent * ent)269 static int arm_smmu_cmdq_build_cmd(u64 *cmd, struct arm_smmu_cmdq_ent *ent)
270 {
271 memset(cmd, 0, 1 << CMDQ_ENT_SZ_SHIFT);
272 cmd[0] |= FIELD_PREP(CMDQ_0_OP, ent->opcode);
273
274 switch (ent->opcode) {
275 case CMDQ_OP_TLBI_EL2_ALL:
276 case CMDQ_OP_TLBI_NSNH_ALL:
277 break;
278 case CMDQ_OP_PREFETCH_CFG:
279 cmd[0] |= FIELD_PREP(CMDQ_PREFETCH_0_SID, ent->prefetch.sid);
280 break;
281 case CMDQ_OP_CFGI_CD:
282 cmd[0] |= FIELD_PREP(CMDQ_CFGI_0_SSID, ent->cfgi.ssid);
283 fallthrough;
284 case CMDQ_OP_CFGI_STE:
285 cmd[0] |= FIELD_PREP(CMDQ_CFGI_0_SID, ent->cfgi.sid);
286 cmd[1] |= FIELD_PREP(CMDQ_CFGI_1_LEAF, ent->cfgi.leaf);
287 break;
288 case CMDQ_OP_CFGI_CD_ALL:
289 cmd[0] |= FIELD_PREP(CMDQ_CFGI_0_SID, ent->cfgi.sid);
290 break;
291 case CMDQ_OP_CFGI_ALL:
292 /* Cover the entire SID range */
293 cmd[1] |= FIELD_PREP(CMDQ_CFGI_1_RANGE, 31);
294 break;
295 case CMDQ_OP_TLBI_NH_VA:
296 cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_VMID, ent->tlbi.vmid);
297 fallthrough;
298 case CMDQ_OP_TLBI_EL2_VA:
299 cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_NUM, ent->tlbi.num);
300 cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_SCALE, ent->tlbi.scale);
301 cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_ASID, ent->tlbi.asid);
302 cmd[1] |= FIELD_PREP(CMDQ_TLBI_1_LEAF, ent->tlbi.leaf);
303 cmd[1] |= FIELD_PREP(CMDQ_TLBI_1_TTL, ent->tlbi.ttl);
304 cmd[1] |= FIELD_PREP(CMDQ_TLBI_1_TG, ent->tlbi.tg);
305 cmd[1] |= ent->tlbi.addr & CMDQ_TLBI_1_VA_MASK;
306 break;
307 case CMDQ_OP_TLBI_S2_IPA:
308 cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_NUM, ent->tlbi.num);
309 cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_SCALE, ent->tlbi.scale);
310 cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_VMID, ent->tlbi.vmid);
311 cmd[1] |= FIELD_PREP(CMDQ_TLBI_1_LEAF, ent->tlbi.leaf);
312 cmd[1] |= FIELD_PREP(CMDQ_TLBI_1_TTL, ent->tlbi.ttl);
313 cmd[1] |= FIELD_PREP(CMDQ_TLBI_1_TG, ent->tlbi.tg);
314 cmd[1] |= ent->tlbi.addr & CMDQ_TLBI_1_IPA_MASK;
315 break;
316 case CMDQ_OP_TLBI_NH_ASID:
317 cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_ASID, ent->tlbi.asid);
318 fallthrough;
319 case CMDQ_OP_TLBI_NH_ALL:
320 case CMDQ_OP_TLBI_S12_VMALL:
321 cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_VMID, ent->tlbi.vmid);
322 break;
323 case CMDQ_OP_TLBI_EL2_ASID:
324 cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_ASID, ent->tlbi.asid);
325 break;
326 case CMDQ_OP_ATC_INV:
327 cmd[0] |= FIELD_PREP(CMDQ_0_SSV, ent->substream_valid);
328 cmd[0] |= FIELD_PREP(CMDQ_ATC_0_GLOBAL, ent->atc.global);
329 cmd[0] |= FIELD_PREP(CMDQ_ATC_0_SSID, ent->atc.ssid);
330 cmd[0] |= FIELD_PREP(CMDQ_ATC_0_SID, ent->atc.sid);
331 cmd[1] |= FIELD_PREP(CMDQ_ATC_1_SIZE, ent->atc.size);
332 cmd[1] |= ent->atc.addr & CMDQ_ATC_1_ADDR_MASK;
333 break;
334 case CMDQ_OP_PRI_RESP:
335 cmd[0] |= FIELD_PREP(CMDQ_0_SSV, ent->substream_valid);
336 cmd[0] |= FIELD_PREP(CMDQ_PRI_0_SSID, ent->pri.ssid);
337 cmd[0] |= FIELD_PREP(CMDQ_PRI_0_SID, ent->pri.sid);
338 cmd[1] |= FIELD_PREP(CMDQ_PRI_1_GRPID, ent->pri.grpid);
339 switch (ent->pri.resp) {
340 case PRI_RESP_DENY:
341 case PRI_RESP_FAIL:
342 case PRI_RESP_SUCC:
343 break;
344 default:
345 return -EINVAL;
346 }
347 cmd[1] |= FIELD_PREP(CMDQ_PRI_1_RESP, ent->pri.resp);
348 break;
349 case CMDQ_OP_RESUME:
350 cmd[0] |= FIELD_PREP(CMDQ_RESUME_0_SID, ent->resume.sid);
351 cmd[0] |= FIELD_PREP(CMDQ_RESUME_0_RESP, ent->resume.resp);
352 cmd[1] |= FIELD_PREP(CMDQ_RESUME_1_STAG, ent->resume.stag);
353 break;
354 case CMDQ_OP_CMD_SYNC:
355 if (ent->sync.msiaddr) {
356 cmd[0] |= FIELD_PREP(CMDQ_SYNC_0_CS, CMDQ_SYNC_0_CS_IRQ);
357 cmd[1] |= ent->sync.msiaddr & CMDQ_SYNC_1_MSIADDR_MASK;
358 } else {
359 cmd[0] |= FIELD_PREP(CMDQ_SYNC_0_CS, CMDQ_SYNC_0_CS_SEV);
360 }
361 cmd[0] |= FIELD_PREP(CMDQ_SYNC_0_MSH, ARM_SMMU_SH_ISH);
362 cmd[0] |= FIELD_PREP(CMDQ_SYNC_0_MSIATTR, ARM_SMMU_MEMATTR_OIWB);
363 break;
364 default:
365 return -ENOENT;
366 }
367
368 return 0;
369 }
370
arm_smmu_get_cmdq(struct arm_smmu_device * smmu,struct arm_smmu_cmdq_ent * ent)371 static struct arm_smmu_cmdq *arm_smmu_get_cmdq(struct arm_smmu_device *smmu,
372 struct arm_smmu_cmdq_ent *ent)
373 {
374 struct arm_smmu_cmdq *cmdq = NULL;
375
376 if (smmu->impl_ops && smmu->impl_ops->get_secondary_cmdq)
377 cmdq = smmu->impl_ops->get_secondary_cmdq(smmu, ent);
378
379 return cmdq ?: &smmu->cmdq;
380 }
381
arm_smmu_cmdq_needs_busy_polling(struct arm_smmu_device * smmu,struct arm_smmu_cmdq * cmdq)382 static bool arm_smmu_cmdq_needs_busy_polling(struct arm_smmu_device *smmu,
383 struct arm_smmu_cmdq *cmdq)
384 {
385 if (cmdq == &smmu->cmdq)
386 return false;
387
388 return smmu->options & ARM_SMMU_OPT_TEGRA241_CMDQV;
389 }
390
arm_smmu_cmdq_build_sync_cmd(u64 * cmd,struct arm_smmu_device * smmu,struct arm_smmu_cmdq * cmdq,u32 prod)391 static void arm_smmu_cmdq_build_sync_cmd(u64 *cmd, struct arm_smmu_device *smmu,
392 struct arm_smmu_cmdq *cmdq, u32 prod)
393 {
394 struct arm_smmu_queue *q = &cmdq->q;
395 struct arm_smmu_cmdq_ent ent = {
396 .opcode = CMDQ_OP_CMD_SYNC,
397 };
398
399 /*
400 * Beware that Hi16xx adds an extra 32 bits of goodness to its MSI
401 * payload, so the write will zero the entire command on that platform.
402 */
403 if (smmu->options & ARM_SMMU_OPT_MSIPOLL) {
404 ent.sync.msiaddr = q->base_dma + Q_IDX(&q->llq, prod) *
405 q->ent_dwords * 8;
406 }
407
408 arm_smmu_cmdq_build_cmd(cmd, &ent);
409 if (arm_smmu_cmdq_needs_busy_polling(smmu, cmdq))
410 u64p_replace_bits(cmd, CMDQ_SYNC_0_CS_NONE, CMDQ_SYNC_0_CS);
411 }
412
__arm_smmu_cmdq_skip_err(struct arm_smmu_device * smmu,struct arm_smmu_cmdq * cmdq)413 void __arm_smmu_cmdq_skip_err(struct arm_smmu_device *smmu,
414 struct arm_smmu_cmdq *cmdq)
415 {
416 static const char * const cerror_str[] = {
417 [CMDQ_ERR_CERROR_NONE_IDX] = "No error",
418 [CMDQ_ERR_CERROR_ILL_IDX] = "Illegal command",
419 [CMDQ_ERR_CERROR_ABT_IDX] = "Abort on command fetch",
420 [CMDQ_ERR_CERROR_ATC_INV_IDX] = "ATC invalidate timeout",
421 };
422 struct arm_smmu_queue *q = &cmdq->q;
423
424 int i;
425 u64 cmd[CMDQ_ENT_DWORDS];
426 u32 cons = readl_relaxed(q->cons_reg);
427 u32 idx = FIELD_GET(CMDQ_CONS_ERR, cons);
428 struct arm_smmu_cmdq_ent cmd_sync = {
429 .opcode = CMDQ_OP_CMD_SYNC,
430 };
431
432 dev_err(smmu->dev, "CMDQ error (cons 0x%08x): %s\n", cons,
433 idx < ARRAY_SIZE(cerror_str) ? cerror_str[idx] : "Unknown");
434
435 switch (idx) {
436 case CMDQ_ERR_CERROR_ABT_IDX:
437 dev_err(smmu->dev, "retrying command fetch\n");
438 return;
439 case CMDQ_ERR_CERROR_NONE_IDX:
440 return;
441 case CMDQ_ERR_CERROR_ATC_INV_IDX:
442 /*
443 * ATC Invalidation Completion timeout. CONS is still pointing
444 * at the CMD_SYNC. Attempt to complete other pending commands
445 * by repeating the CMD_SYNC, though we might well end up back
446 * here since the ATC invalidation may still be pending.
447 */
448 return;
449 case CMDQ_ERR_CERROR_ILL_IDX:
450 default:
451 break;
452 }
453
454 /*
455 * We may have concurrent producers, so we need to be careful
456 * not to touch any of the shadow cmdq state.
457 */
458 queue_read(cmd, Q_ENT(q, cons), q->ent_dwords);
459 dev_err(smmu->dev, "skipping command in error state:\n");
460 for (i = 0; i < ARRAY_SIZE(cmd); ++i)
461 dev_err(smmu->dev, "\t0x%016llx\n", (unsigned long long)cmd[i]);
462
463 /* Convert the erroneous command into a CMD_SYNC */
464 arm_smmu_cmdq_build_cmd(cmd, &cmd_sync);
465 if (arm_smmu_cmdq_needs_busy_polling(smmu, cmdq))
466 u64p_replace_bits(cmd, CMDQ_SYNC_0_CS_NONE, CMDQ_SYNC_0_CS);
467
468 queue_write(Q_ENT(q, cons), cmd, q->ent_dwords);
469 }
470
arm_smmu_cmdq_skip_err(struct arm_smmu_device * smmu)471 static void arm_smmu_cmdq_skip_err(struct arm_smmu_device *smmu)
472 {
473 __arm_smmu_cmdq_skip_err(smmu, &smmu->cmdq);
474 }
475
476 /*
477 * Command queue locking.
478 * This is a form of bastardised rwlock with the following major changes:
479 *
480 * - The only LOCK routines are exclusive_trylock() and shared_lock().
481 * Neither have barrier semantics, and instead provide only a control
482 * dependency.
483 *
484 * - The UNLOCK routines are supplemented with shared_tryunlock(), which
485 * fails if the caller appears to be the last lock holder (yes, this is
486 * racy). All successful UNLOCK routines have RELEASE semantics.
487 */
arm_smmu_cmdq_shared_lock(struct arm_smmu_cmdq * cmdq)488 static void arm_smmu_cmdq_shared_lock(struct arm_smmu_cmdq *cmdq)
489 {
490 int val;
491
492 /*
493 * We can try to avoid the cmpxchg() loop by simply incrementing the
494 * lock counter. When held in exclusive state, the lock counter is set
495 * to INT_MIN so these increments won't hurt as the value will remain
496 * negative.
497 */
498 if (atomic_fetch_inc_relaxed(&cmdq->lock) >= 0)
499 return;
500
501 do {
502 val = atomic_cond_read_relaxed(&cmdq->lock, VAL >= 0);
503 } while (atomic_cmpxchg_relaxed(&cmdq->lock, val, val + 1) != val);
504 }
505
arm_smmu_cmdq_shared_unlock(struct arm_smmu_cmdq * cmdq)506 static void arm_smmu_cmdq_shared_unlock(struct arm_smmu_cmdq *cmdq)
507 {
508 (void)atomic_dec_return_release(&cmdq->lock);
509 }
510
arm_smmu_cmdq_shared_tryunlock(struct arm_smmu_cmdq * cmdq)511 static bool arm_smmu_cmdq_shared_tryunlock(struct arm_smmu_cmdq *cmdq)
512 {
513 if (atomic_read(&cmdq->lock) == 1)
514 return false;
515
516 arm_smmu_cmdq_shared_unlock(cmdq);
517 return true;
518 }
519
520 #define arm_smmu_cmdq_exclusive_trylock_irqsave(cmdq, flags) \
521 ({ \
522 bool __ret; \
523 local_irq_save(flags); \
524 __ret = !atomic_cmpxchg_relaxed(&cmdq->lock, 0, INT_MIN); \
525 if (!__ret) \
526 local_irq_restore(flags); \
527 __ret; \
528 })
529
530 #define arm_smmu_cmdq_exclusive_unlock_irqrestore(cmdq, flags) \
531 ({ \
532 atomic_set_release(&cmdq->lock, 0); \
533 local_irq_restore(flags); \
534 })
535
536
537 /*
538 * Command queue insertion.
539 * This is made fiddly by our attempts to achieve some sort of scalability
540 * since there is one queue shared amongst all of the CPUs in the system. If
541 * you like mixed-size concurrency, dependency ordering and relaxed atomics,
542 * then you'll *love* this monstrosity.
543 *
544 * The basic idea is to split the queue up into ranges of commands that are
545 * owned by a given CPU; the owner may not have written all of the commands
546 * itself, but is responsible for advancing the hardware prod pointer when
547 * the time comes. The algorithm is roughly:
548 *
549 * 1. Allocate some space in the queue. At this point we also discover
550 * whether the head of the queue is currently owned by another CPU,
551 * or whether we are the owner.
552 *
553 * 2. Write our commands into our allocated slots in the queue.
554 *
555 * 3. Mark our slots as valid in arm_smmu_cmdq.valid_map.
556 *
557 * 4. If we are an owner:
558 * a. Wait for the previous owner to finish.
559 * b. Mark the queue head as unowned, which tells us the range
560 * that we are responsible for publishing.
561 * c. Wait for all commands in our owned range to become valid.
562 * d. Advance the hardware prod pointer.
563 * e. Tell the next owner we've finished.
564 *
565 * 5. If we are inserting a CMD_SYNC (we may or may not have been an
566 * owner), then we need to stick around until it has completed:
567 * a. If we have MSIs, the SMMU can write back into the CMD_SYNC
568 * to clear the first 4 bytes.
569 * b. Otherwise, we spin waiting for the hardware cons pointer to
570 * advance past our command.
571 *
572 * The devil is in the details, particularly the use of locking for handling
573 * SYNC completion and freeing up space in the queue before we think that it is
574 * full.
575 */
__arm_smmu_cmdq_poll_set_valid_map(struct arm_smmu_cmdq * cmdq,u32 sprod,u32 eprod,bool set)576 static void __arm_smmu_cmdq_poll_set_valid_map(struct arm_smmu_cmdq *cmdq,
577 u32 sprod, u32 eprod, bool set)
578 {
579 u32 swidx, sbidx, ewidx, ebidx;
580 struct arm_smmu_ll_queue llq = {
581 .max_n_shift = cmdq->q.llq.max_n_shift,
582 .prod = sprod,
583 };
584
585 ewidx = BIT_WORD(Q_IDX(&llq, eprod));
586 ebidx = Q_IDX(&llq, eprod) % BITS_PER_LONG;
587
588 while (llq.prod != eprod) {
589 unsigned long mask;
590 atomic_long_t *ptr;
591 u32 limit = BITS_PER_LONG;
592
593 swidx = BIT_WORD(Q_IDX(&llq, llq.prod));
594 sbidx = Q_IDX(&llq, llq.prod) % BITS_PER_LONG;
595
596 ptr = &cmdq->valid_map[swidx];
597
598 if ((swidx == ewidx) && (sbidx < ebidx))
599 limit = ebidx;
600
601 mask = GENMASK(limit - 1, sbidx);
602
603 /*
604 * The valid bit is the inverse of the wrap bit. This means
605 * that a zero-initialised queue is invalid and, after marking
606 * all entries as valid, they become invalid again when we
607 * wrap.
608 */
609 if (set) {
610 atomic_long_xor(mask, ptr);
611 } else { /* Poll */
612 unsigned long valid;
613
614 valid = (ULONG_MAX + !!Q_WRP(&llq, llq.prod)) & mask;
615 atomic_long_cond_read_relaxed(ptr, (VAL & mask) == valid);
616 }
617
618 llq.prod = queue_inc_prod_n(&llq, limit - sbidx);
619 }
620 }
621
622 /* Mark all entries in the range [sprod, eprod) as valid */
arm_smmu_cmdq_set_valid_map(struct arm_smmu_cmdq * cmdq,u32 sprod,u32 eprod)623 static void arm_smmu_cmdq_set_valid_map(struct arm_smmu_cmdq *cmdq,
624 u32 sprod, u32 eprod)
625 {
626 __arm_smmu_cmdq_poll_set_valid_map(cmdq, sprod, eprod, true);
627 }
628
629 /* Wait for all entries in the range [sprod, eprod) to become valid */
arm_smmu_cmdq_poll_valid_map(struct arm_smmu_cmdq * cmdq,u32 sprod,u32 eprod)630 static void arm_smmu_cmdq_poll_valid_map(struct arm_smmu_cmdq *cmdq,
631 u32 sprod, u32 eprod)
632 {
633 __arm_smmu_cmdq_poll_set_valid_map(cmdq, sprod, eprod, false);
634 }
635
636 /* Wait for the command queue to become non-full */
arm_smmu_cmdq_poll_until_not_full(struct arm_smmu_device * smmu,struct arm_smmu_cmdq * cmdq,struct arm_smmu_ll_queue * llq)637 static int arm_smmu_cmdq_poll_until_not_full(struct arm_smmu_device *smmu,
638 struct arm_smmu_cmdq *cmdq,
639 struct arm_smmu_ll_queue *llq)
640 {
641 unsigned long flags;
642 struct arm_smmu_queue_poll qp;
643 int ret = 0;
644
645 /*
646 * Try to update our copy of cons by grabbing exclusive cmdq access. If
647 * that fails, spin until somebody else updates it for us.
648 */
649 if (arm_smmu_cmdq_exclusive_trylock_irqsave(cmdq, flags)) {
650 WRITE_ONCE(cmdq->q.llq.cons, readl_relaxed(cmdq->q.cons_reg));
651 arm_smmu_cmdq_exclusive_unlock_irqrestore(cmdq, flags);
652 llq->val = READ_ONCE(cmdq->q.llq.val);
653 return 0;
654 }
655
656 queue_poll_init(smmu, &qp);
657 do {
658 llq->val = READ_ONCE(cmdq->q.llq.val);
659 if (!queue_full(llq))
660 break;
661
662 ret = queue_poll(&qp);
663 } while (!ret);
664
665 return ret;
666 }
667
668 /*
669 * Wait until the SMMU signals a CMD_SYNC completion MSI.
670 * Must be called with the cmdq lock held in some capacity.
671 */
__arm_smmu_cmdq_poll_until_msi(struct arm_smmu_device * smmu,struct arm_smmu_cmdq * cmdq,struct arm_smmu_ll_queue * llq)672 static int __arm_smmu_cmdq_poll_until_msi(struct arm_smmu_device *smmu,
673 struct arm_smmu_cmdq *cmdq,
674 struct arm_smmu_ll_queue *llq)
675 {
676 int ret = 0;
677 struct arm_smmu_queue_poll qp;
678 u32 *cmd = (u32 *)(Q_ENT(&cmdq->q, llq->prod));
679
680 queue_poll_init(smmu, &qp);
681
682 /*
683 * The MSI won't generate an event, since it's being written back
684 * into the command queue.
685 */
686 qp.wfe = false;
687 smp_cond_load_relaxed(cmd, !VAL || (ret = queue_poll(&qp)));
688 llq->cons = ret ? llq->prod : queue_inc_prod_n(llq, 1);
689 return ret;
690 }
691
692 /*
693 * Wait until the SMMU cons index passes llq->prod.
694 * Must be called with the cmdq lock held in some capacity.
695 */
__arm_smmu_cmdq_poll_until_consumed(struct arm_smmu_device * smmu,struct arm_smmu_cmdq * cmdq,struct arm_smmu_ll_queue * llq)696 static int __arm_smmu_cmdq_poll_until_consumed(struct arm_smmu_device *smmu,
697 struct arm_smmu_cmdq *cmdq,
698 struct arm_smmu_ll_queue *llq)
699 {
700 struct arm_smmu_queue_poll qp;
701 u32 prod = llq->prod;
702 int ret = 0;
703
704 queue_poll_init(smmu, &qp);
705 llq->val = READ_ONCE(cmdq->q.llq.val);
706 do {
707 if (queue_consumed(llq, prod))
708 break;
709
710 ret = queue_poll(&qp);
711
712 /*
713 * This needs to be a readl() so that our subsequent call
714 * to arm_smmu_cmdq_shared_tryunlock() can fail accurately.
715 *
716 * Specifically, we need to ensure that we observe all
717 * shared_lock()s by other CMD_SYNCs that share our owner,
718 * so that a failing call to tryunlock() means that we're
719 * the last one out and therefore we can safely advance
720 * cmdq->q.llq.cons. Roughly speaking:
721 *
722 * CPU 0 CPU1 CPU2 (us)
723 *
724 * if (sync)
725 * shared_lock();
726 *
727 * dma_wmb();
728 * set_valid_map();
729 *
730 * if (owner) {
731 * poll_valid_map();
732 * <control dependency>
733 * writel(prod_reg);
734 *
735 * readl(cons_reg);
736 * tryunlock();
737 *
738 * Requires us to see CPU 0's shared_lock() acquisition.
739 */
740 llq->cons = readl(cmdq->q.cons_reg);
741 } while (!ret);
742
743 return ret;
744 }
745
arm_smmu_cmdq_poll_until_sync(struct arm_smmu_device * smmu,struct arm_smmu_cmdq * cmdq,struct arm_smmu_ll_queue * llq)746 static int arm_smmu_cmdq_poll_until_sync(struct arm_smmu_device *smmu,
747 struct arm_smmu_cmdq *cmdq,
748 struct arm_smmu_ll_queue *llq)
749 {
750 if (smmu->options & ARM_SMMU_OPT_MSIPOLL &&
751 !arm_smmu_cmdq_needs_busy_polling(smmu, cmdq))
752 return __arm_smmu_cmdq_poll_until_msi(smmu, cmdq, llq);
753
754 return __arm_smmu_cmdq_poll_until_consumed(smmu, cmdq, llq);
755 }
756
arm_smmu_cmdq_write_entries(struct arm_smmu_cmdq * cmdq,u64 * cmds,u32 prod,int n)757 static void arm_smmu_cmdq_write_entries(struct arm_smmu_cmdq *cmdq, u64 *cmds,
758 u32 prod, int n)
759 {
760 int i;
761 struct arm_smmu_ll_queue llq = {
762 .max_n_shift = cmdq->q.llq.max_n_shift,
763 .prod = prod,
764 };
765
766 for (i = 0; i < n; ++i) {
767 u64 *cmd = &cmds[i * CMDQ_ENT_DWORDS];
768
769 prod = queue_inc_prod_n(&llq, i);
770 queue_write(Q_ENT(&cmdq->q, prod), cmd, CMDQ_ENT_DWORDS);
771 }
772 }
773
774 /*
775 * This is the actual insertion function, and provides the following
776 * ordering guarantees to callers:
777 *
778 * - There is a dma_wmb() before publishing any commands to the queue.
779 * This can be relied upon to order prior writes to data structures
780 * in memory (such as a CD or an STE) before the command.
781 *
782 * - On completion of a CMD_SYNC, there is a control dependency.
783 * This can be relied upon to order subsequent writes to memory (e.g.
784 * freeing an IOVA) after completion of the CMD_SYNC.
785 *
786 * - Command insertion is totally ordered, so if two CPUs each race to
787 * insert their own list of commands then all of the commands from one
788 * CPU will appear before any of the commands from the other CPU.
789 */
arm_smmu_cmdq_issue_cmdlist(struct arm_smmu_device * smmu,struct arm_smmu_cmdq * cmdq,u64 * cmds,int n,bool sync)790 int arm_smmu_cmdq_issue_cmdlist(struct arm_smmu_device *smmu,
791 struct arm_smmu_cmdq *cmdq, u64 *cmds, int n,
792 bool sync)
793 {
794 u64 cmd_sync[CMDQ_ENT_DWORDS];
795 u32 prod;
796 unsigned long flags;
797 bool owner;
798 struct arm_smmu_ll_queue llq, head;
799 int ret = 0;
800
801 llq.max_n_shift = cmdq->q.llq.max_n_shift;
802
803 /* 1. Allocate some space in the queue */
804 local_irq_save(flags);
805 llq.val = READ_ONCE(cmdq->q.llq.val);
806 do {
807 u64 old;
808
809 while (!queue_has_space(&llq, n + sync)) {
810 local_irq_restore(flags);
811 if (arm_smmu_cmdq_poll_until_not_full(smmu, cmdq, &llq))
812 dev_err_ratelimited(smmu->dev, "CMDQ timeout\n");
813 local_irq_save(flags);
814 }
815
816 head.cons = llq.cons;
817 head.prod = queue_inc_prod_n(&llq, n + sync) |
818 CMDQ_PROD_OWNED_FLAG;
819
820 old = cmpxchg_relaxed(&cmdq->q.llq.val, llq.val, head.val);
821 if (old == llq.val)
822 break;
823
824 llq.val = old;
825 } while (1);
826 owner = !(llq.prod & CMDQ_PROD_OWNED_FLAG);
827 head.prod &= ~CMDQ_PROD_OWNED_FLAG;
828 llq.prod &= ~CMDQ_PROD_OWNED_FLAG;
829
830 /*
831 * 2. Write our commands into the queue
832 * Dependency ordering from the cmpxchg() loop above.
833 */
834 arm_smmu_cmdq_write_entries(cmdq, cmds, llq.prod, n);
835 if (sync) {
836 prod = queue_inc_prod_n(&llq, n);
837 arm_smmu_cmdq_build_sync_cmd(cmd_sync, smmu, cmdq, prod);
838 queue_write(Q_ENT(&cmdq->q, prod), cmd_sync, CMDQ_ENT_DWORDS);
839
840 /*
841 * In order to determine completion of our CMD_SYNC, we must
842 * ensure that the queue can't wrap twice without us noticing.
843 * We achieve that by taking the cmdq lock as shared before
844 * marking our slot as valid.
845 */
846 arm_smmu_cmdq_shared_lock(cmdq);
847 }
848
849 /* 3. Mark our slots as valid, ensuring commands are visible first */
850 dma_wmb();
851 arm_smmu_cmdq_set_valid_map(cmdq, llq.prod, head.prod);
852
853 /* 4. If we are the owner, take control of the SMMU hardware */
854 if (owner) {
855 /* a. Wait for previous owner to finish */
856 atomic_cond_read_relaxed(&cmdq->owner_prod, VAL == llq.prod);
857
858 /* b. Stop gathering work by clearing the owned flag */
859 prod = atomic_fetch_andnot_relaxed(CMDQ_PROD_OWNED_FLAG,
860 &cmdq->q.llq.atomic.prod);
861 prod &= ~CMDQ_PROD_OWNED_FLAG;
862
863 /*
864 * c. Wait for any gathered work to be written to the queue.
865 * Note that we read our own entries so that we have the control
866 * dependency required by (d).
867 */
868 arm_smmu_cmdq_poll_valid_map(cmdq, llq.prod, prod);
869
870 /*
871 * d. Advance the hardware prod pointer
872 * Control dependency ordering from the entries becoming valid.
873 */
874 writel_relaxed(prod, cmdq->q.prod_reg);
875
876 /*
877 * e. Tell the next owner we're done
878 * Make sure we've updated the hardware first, so that we don't
879 * race to update prod and potentially move it backwards.
880 */
881 atomic_set_release(&cmdq->owner_prod, prod);
882 }
883
884 /* 5. If we are inserting a CMD_SYNC, we must wait for it to complete */
885 if (sync) {
886 llq.prod = queue_inc_prod_n(&llq, n);
887 ret = arm_smmu_cmdq_poll_until_sync(smmu, cmdq, &llq);
888 if (ret) {
889 dev_err_ratelimited(smmu->dev,
890 "CMD_SYNC timeout at 0x%08x [hwprod 0x%08x, hwcons 0x%08x]\n",
891 llq.prod,
892 readl_relaxed(cmdq->q.prod_reg),
893 readl_relaxed(cmdq->q.cons_reg));
894 }
895
896 /*
897 * Try to unlock the cmdq lock. This will fail if we're the last
898 * reader, in which case we can safely update cmdq->q.llq.cons
899 */
900 if (!arm_smmu_cmdq_shared_tryunlock(cmdq)) {
901 WRITE_ONCE(cmdq->q.llq.cons, llq.cons);
902 arm_smmu_cmdq_shared_unlock(cmdq);
903 }
904 }
905
906 local_irq_restore(flags);
907 return ret;
908 }
909
__arm_smmu_cmdq_issue_cmd(struct arm_smmu_device * smmu,struct arm_smmu_cmdq_ent * ent,bool sync)910 static int __arm_smmu_cmdq_issue_cmd(struct arm_smmu_device *smmu,
911 struct arm_smmu_cmdq_ent *ent,
912 bool sync)
913 {
914 u64 cmd[CMDQ_ENT_DWORDS];
915
916 if (unlikely(arm_smmu_cmdq_build_cmd(cmd, ent))) {
917 dev_warn(smmu->dev, "ignoring unknown CMDQ opcode 0x%x\n",
918 ent->opcode);
919 return -EINVAL;
920 }
921
922 return arm_smmu_cmdq_issue_cmdlist(
923 smmu, arm_smmu_get_cmdq(smmu, ent), cmd, 1, sync);
924 }
925
arm_smmu_cmdq_issue_cmd(struct arm_smmu_device * smmu,struct arm_smmu_cmdq_ent * ent)926 static int arm_smmu_cmdq_issue_cmd(struct arm_smmu_device *smmu,
927 struct arm_smmu_cmdq_ent *ent)
928 {
929 return __arm_smmu_cmdq_issue_cmd(smmu, ent, false);
930 }
931
arm_smmu_cmdq_issue_cmd_with_sync(struct arm_smmu_device * smmu,struct arm_smmu_cmdq_ent * ent)932 static int arm_smmu_cmdq_issue_cmd_with_sync(struct arm_smmu_device *smmu,
933 struct arm_smmu_cmdq_ent *ent)
934 {
935 return __arm_smmu_cmdq_issue_cmd(smmu, ent, true);
936 }
937
arm_smmu_cmdq_batch_init(struct arm_smmu_device * smmu,struct arm_smmu_cmdq_batch * cmds,struct arm_smmu_cmdq_ent * ent)938 static void arm_smmu_cmdq_batch_init(struct arm_smmu_device *smmu,
939 struct arm_smmu_cmdq_batch *cmds,
940 struct arm_smmu_cmdq_ent *ent)
941 {
942 cmds->num = 0;
943 cmds->cmdq = arm_smmu_get_cmdq(smmu, ent);
944 }
945
arm_smmu_cmdq_batch_add(struct arm_smmu_device * smmu,struct arm_smmu_cmdq_batch * cmds,struct arm_smmu_cmdq_ent * cmd)946 static void arm_smmu_cmdq_batch_add(struct arm_smmu_device *smmu,
947 struct arm_smmu_cmdq_batch *cmds,
948 struct arm_smmu_cmdq_ent *cmd)
949 {
950 bool unsupported_cmd = !arm_smmu_cmdq_supports_cmd(cmds->cmdq, cmd);
951 bool force_sync = (cmds->num == CMDQ_BATCH_ENTRIES - 1) &&
952 (smmu->options & ARM_SMMU_OPT_CMDQ_FORCE_SYNC);
953 int index;
954
955 if (force_sync || unsupported_cmd) {
956 arm_smmu_cmdq_issue_cmdlist(smmu, cmds->cmdq, cmds->cmds,
957 cmds->num, true);
958 arm_smmu_cmdq_batch_init(smmu, cmds, cmd);
959 }
960
961 if (cmds->num == CMDQ_BATCH_ENTRIES) {
962 arm_smmu_cmdq_issue_cmdlist(smmu, cmds->cmdq, cmds->cmds,
963 cmds->num, false);
964 arm_smmu_cmdq_batch_init(smmu, cmds, cmd);
965 }
966
967 index = cmds->num * CMDQ_ENT_DWORDS;
968 if (unlikely(arm_smmu_cmdq_build_cmd(&cmds->cmds[index], cmd))) {
969 dev_warn(smmu->dev, "ignoring unknown CMDQ opcode 0x%x\n",
970 cmd->opcode);
971 return;
972 }
973
974 cmds->num++;
975 }
976
arm_smmu_cmdq_batch_submit(struct arm_smmu_device * smmu,struct arm_smmu_cmdq_batch * cmds)977 static int arm_smmu_cmdq_batch_submit(struct arm_smmu_device *smmu,
978 struct arm_smmu_cmdq_batch *cmds)
979 {
980 return arm_smmu_cmdq_issue_cmdlist(smmu, cmds->cmdq, cmds->cmds,
981 cmds->num, true);
982 }
983
arm_smmu_page_response(struct device * dev,struct iopf_fault * unused,struct iommu_page_response * resp)984 static void arm_smmu_page_response(struct device *dev, struct iopf_fault *unused,
985 struct iommu_page_response *resp)
986 {
987 struct arm_smmu_cmdq_ent cmd = {0};
988 struct arm_smmu_master *master = dev_iommu_priv_get(dev);
989 int sid = master->streams[0].id;
990
991 if (WARN_ON(!master->stall_enabled))
992 return;
993
994 cmd.opcode = CMDQ_OP_RESUME;
995 cmd.resume.sid = sid;
996 cmd.resume.stag = resp->grpid;
997 switch (resp->code) {
998 case IOMMU_PAGE_RESP_INVALID:
999 case IOMMU_PAGE_RESP_FAILURE:
1000 cmd.resume.resp = CMDQ_RESUME_0_RESP_ABORT;
1001 break;
1002 case IOMMU_PAGE_RESP_SUCCESS:
1003 cmd.resume.resp = CMDQ_RESUME_0_RESP_RETRY;
1004 break;
1005 default:
1006 break;
1007 }
1008
1009 arm_smmu_cmdq_issue_cmd(master->smmu, &cmd);
1010 /*
1011 * Don't send a SYNC, it doesn't do anything for RESUME or PRI_RESP.
1012 * RESUME consumption guarantees that the stalled transaction will be
1013 * terminated... at some point in the future. PRI_RESP is fire and
1014 * forget.
1015 */
1016 }
1017
1018 /* Context descriptor manipulation functions */
arm_smmu_tlb_inv_asid(struct arm_smmu_device * smmu,u16 asid)1019 void arm_smmu_tlb_inv_asid(struct arm_smmu_device *smmu, u16 asid)
1020 {
1021 struct arm_smmu_cmdq_ent cmd = {
1022 .opcode = smmu->features & ARM_SMMU_FEAT_E2H ?
1023 CMDQ_OP_TLBI_EL2_ASID : CMDQ_OP_TLBI_NH_ASID,
1024 .tlbi.asid = asid,
1025 };
1026
1027 arm_smmu_cmdq_issue_cmd_with_sync(smmu, &cmd);
1028 }
1029
1030 /*
1031 * Based on the value of ent report which bits of the STE the HW will access. It
1032 * would be nice if this was complete according to the spec, but minimally it
1033 * has to capture the bits this driver uses.
1034 */
1035 VISIBLE_IF_KUNIT
arm_smmu_get_ste_used(const __le64 * ent,__le64 * used_bits)1036 void arm_smmu_get_ste_used(const __le64 *ent, __le64 *used_bits)
1037 {
1038 unsigned int cfg = FIELD_GET(STRTAB_STE_0_CFG, le64_to_cpu(ent[0]));
1039
1040 used_bits[0] = cpu_to_le64(STRTAB_STE_0_V);
1041 if (!(ent[0] & cpu_to_le64(STRTAB_STE_0_V)))
1042 return;
1043
1044 used_bits[0] |= cpu_to_le64(STRTAB_STE_0_CFG);
1045
1046 /* S1 translates */
1047 if (cfg & BIT(0)) {
1048 used_bits[0] |= cpu_to_le64(STRTAB_STE_0_S1FMT |
1049 STRTAB_STE_0_S1CTXPTR_MASK |
1050 STRTAB_STE_0_S1CDMAX);
1051 used_bits[1] |=
1052 cpu_to_le64(STRTAB_STE_1_S1DSS | STRTAB_STE_1_S1CIR |
1053 STRTAB_STE_1_S1COR | STRTAB_STE_1_S1CSH |
1054 STRTAB_STE_1_S1STALLD | STRTAB_STE_1_STRW |
1055 STRTAB_STE_1_EATS | STRTAB_STE_1_MEV);
1056 used_bits[2] |= cpu_to_le64(STRTAB_STE_2_S2VMID);
1057
1058 /*
1059 * See 13.5 Summary of attribute/permission configuration fields
1060 * for the SHCFG behavior.
1061 */
1062 if (FIELD_GET(STRTAB_STE_1_S1DSS, le64_to_cpu(ent[1])) ==
1063 STRTAB_STE_1_S1DSS_BYPASS)
1064 used_bits[1] |= cpu_to_le64(STRTAB_STE_1_SHCFG);
1065 }
1066
1067 /* S2 translates */
1068 if (cfg & BIT(1)) {
1069 used_bits[1] |=
1070 cpu_to_le64(STRTAB_STE_1_S2FWB | STRTAB_STE_1_EATS |
1071 STRTAB_STE_1_SHCFG | STRTAB_STE_1_MEV);
1072 used_bits[2] |=
1073 cpu_to_le64(STRTAB_STE_2_S2VMID | STRTAB_STE_2_VTCR |
1074 STRTAB_STE_2_S2AA64 | STRTAB_STE_2_S2ENDI |
1075 STRTAB_STE_2_S2PTW | STRTAB_STE_2_S2S |
1076 STRTAB_STE_2_S2R);
1077 used_bits[3] |= cpu_to_le64(STRTAB_STE_3_S2TTB_MASK);
1078 }
1079
1080 if (cfg == STRTAB_STE_0_CFG_BYPASS)
1081 used_bits[1] |= cpu_to_le64(STRTAB_STE_1_SHCFG);
1082 }
1083 EXPORT_SYMBOL_IF_KUNIT(arm_smmu_get_ste_used);
1084
1085 /*
1086 * Figure out if we can do a hitless update of entry to become target. Returns a
1087 * bit mask where 1 indicates that qword needs to be set disruptively.
1088 * unused_update is an intermediate value of entry that has unused bits set to
1089 * their new values.
1090 */
arm_smmu_entry_qword_diff(struct arm_smmu_entry_writer * writer,const __le64 * entry,const __le64 * target,__le64 * unused_update)1091 static u8 arm_smmu_entry_qword_diff(struct arm_smmu_entry_writer *writer,
1092 const __le64 *entry, const __le64 *target,
1093 __le64 *unused_update)
1094 {
1095 __le64 target_used[NUM_ENTRY_QWORDS] = {};
1096 __le64 cur_used[NUM_ENTRY_QWORDS] = {};
1097 u8 used_qword_diff = 0;
1098 unsigned int i;
1099
1100 writer->ops->get_used(entry, cur_used);
1101 writer->ops->get_used(target, target_used);
1102
1103 for (i = 0; i != NUM_ENTRY_QWORDS; i++) {
1104 /*
1105 * Check that masks are up to date, the make functions are not
1106 * allowed to set a bit to 1 if the used function doesn't say it
1107 * is used.
1108 */
1109 WARN_ON_ONCE(target[i] & ~target_used[i]);
1110
1111 /* Bits can change because they are not currently being used */
1112 unused_update[i] = (entry[i] & cur_used[i]) |
1113 (target[i] & ~cur_used[i]);
1114 /*
1115 * Each bit indicates that a used bit in a qword needs to be
1116 * changed after unused_update is applied.
1117 */
1118 if ((unused_update[i] & target_used[i]) != target[i])
1119 used_qword_diff |= 1 << i;
1120 }
1121 return used_qword_diff;
1122 }
1123
entry_set(struct arm_smmu_entry_writer * writer,__le64 * entry,const __le64 * target,unsigned int start,unsigned int len)1124 static bool entry_set(struct arm_smmu_entry_writer *writer, __le64 *entry,
1125 const __le64 *target, unsigned int start,
1126 unsigned int len)
1127 {
1128 bool changed = false;
1129 unsigned int i;
1130
1131 for (i = start; len != 0; len--, i++) {
1132 if (entry[i] != target[i]) {
1133 WRITE_ONCE(entry[i], target[i]);
1134 changed = true;
1135 }
1136 }
1137
1138 if (changed)
1139 writer->ops->sync(writer);
1140 return changed;
1141 }
1142
1143 /*
1144 * Update the STE/CD to the target configuration. The transition from the
1145 * current entry to the target entry takes place over multiple steps that
1146 * attempts to make the transition hitless if possible. This function takes care
1147 * not to create a situation where the HW can perceive a corrupted entry. HW is
1148 * only required to have a 64 bit atomicity with stores from the CPU, while
1149 * entries are many 64 bit values big.
1150 *
1151 * The difference between the current value and the target value is analyzed to
1152 * determine which of three updates are required - disruptive, hitless or no
1153 * change.
1154 *
1155 * In the most general disruptive case we can make any update in three steps:
1156 * - Disrupting the entry (V=0)
1157 * - Fill now unused qwords, execpt qword 0 which contains V
1158 * - Make qword 0 have the final value and valid (V=1) with a single 64
1159 * bit store
1160 *
1161 * However this disrupts the HW while it is happening. There are several
1162 * interesting cases where a STE/CD can be updated without disturbing the HW
1163 * because only a small number of bits are changing (S1DSS, CONFIG, etc) or
1164 * because the used bits don't intersect. We can detect this by calculating how
1165 * many 64 bit values need update after adjusting the unused bits and skip the
1166 * V=0 process. This relies on the IGNORED behavior described in the
1167 * specification.
1168 */
1169 VISIBLE_IF_KUNIT
arm_smmu_write_entry(struct arm_smmu_entry_writer * writer,__le64 * entry,const __le64 * target)1170 void arm_smmu_write_entry(struct arm_smmu_entry_writer *writer, __le64 *entry,
1171 const __le64 *target)
1172 {
1173 __le64 unused_update[NUM_ENTRY_QWORDS];
1174 u8 used_qword_diff;
1175
1176 used_qword_diff =
1177 arm_smmu_entry_qword_diff(writer, entry, target, unused_update);
1178 if (hweight8(used_qword_diff) == 1) {
1179 /*
1180 * Only one qword needs its used bits to be changed. This is a
1181 * hitless update, update all bits the current STE/CD is
1182 * ignoring to their new values, then update a single "critical
1183 * qword" to change the STE/CD and finally 0 out any bits that
1184 * are now unused in the target configuration.
1185 */
1186 unsigned int critical_qword_index = ffs(used_qword_diff) - 1;
1187
1188 /*
1189 * Skip writing unused bits in the critical qword since we'll be
1190 * writing it in the next step anyways. This can save a sync
1191 * when the only change is in that qword.
1192 */
1193 unused_update[critical_qword_index] =
1194 entry[critical_qword_index];
1195 entry_set(writer, entry, unused_update, 0, NUM_ENTRY_QWORDS);
1196 entry_set(writer, entry, target, critical_qword_index, 1);
1197 entry_set(writer, entry, target, 0, NUM_ENTRY_QWORDS);
1198 } else if (used_qword_diff) {
1199 /*
1200 * At least two qwords need their inuse bits to be changed. This
1201 * requires a breaking update, zero the V bit, write all qwords
1202 * but 0, then set qword 0
1203 */
1204 unused_update[0] = 0;
1205 entry_set(writer, entry, unused_update, 0, 1);
1206 entry_set(writer, entry, target, 1, NUM_ENTRY_QWORDS - 1);
1207 entry_set(writer, entry, target, 0, 1);
1208 } else {
1209 /*
1210 * No inuse bit changed. Sanity check that all unused bits are 0
1211 * in the entry. The target was already sanity checked by
1212 * compute_qword_diff().
1213 */
1214 WARN_ON_ONCE(
1215 entry_set(writer, entry, target, 0, NUM_ENTRY_QWORDS));
1216 }
1217 }
1218 EXPORT_SYMBOL_IF_KUNIT(arm_smmu_write_entry);
1219
arm_smmu_sync_cd(struct arm_smmu_master * master,int ssid,bool leaf)1220 static void arm_smmu_sync_cd(struct arm_smmu_master *master,
1221 int ssid, bool leaf)
1222 {
1223 size_t i;
1224 struct arm_smmu_cmdq_batch cmds;
1225 struct arm_smmu_device *smmu = master->smmu;
1226 struct arm_smmu_cmdq_ent cmd = {
1227 .opcode = CMDQ_OP_CFGI_CD,
1228 .cfgi = {
1229 .ssid = ssid,
1230 .leaf = leaf,
1231 },
1232 };
1233
1234 arm_smmu_cmdq_batch_init(smmu, &cmds, &cmd);
1235 for (i = 0; i < master->num_streams; i++) {
1236 cmd.cfgi.sid = master->streams[i].id;
1237 arm_smmu_cmdq_batch_add(smmu, &cmds, &cmd);
1238 }
1239
1240 arm_smmu_cmdq_batch_submit(smmu, &cmds);
1241 }
1242
arm_smmu_write_cd_l1_desc(struct arm_smmu_cdtab_l1 * dst,dma_addr_t l2ptr_dma)1243 static void arm_smmu_write_cd_l1_desc(struct arm_smmu_cdtab_l1 *dst,
1244 dma_addr_t l2ptr_dma)
1245 {
1246 u64 val = (l2ptr_dma & CTXDESC_L1_DESC_L2PTR_MASK) | CTXDESC_L1_DESC_V;
1247
1248 /* The HW has 64 bit atomicity with stores to the L2 CD table */
1249 WRITE_ONCE(dst->l2ptr, cpu_to_le64(val));
1250 }
1251
arm_smmu_cd_l1_get_desc(const struct arm_smmu_cdtab_l1 * src)1252 static dma_addr_t arm_smmu_cd_l1_get_desc(const struct arm_smmu_cdtab_l1 *src)
1253 {
1254 return le64_to_cpu(src->l2ptr) & CTXDESC_L1_DESC_L2PTR_MASK;
1255 }
1256
arm_smmu_get_cd_ptr(struct arm_smmu_master * master,u32 ssid)1257 struct arm_smmu_cd *arm_smmu_get_cd_ptr(struct arm_smmu_master *master,
1258 u32 ssid)
1259 {
1260 struct arm_smmu_cdtab_l2 *l2;
1261 struct arm_smmu_ctx_desc_cfg *cd_table = &master->cd_table;
1262
1263 if (!arm_smmu_cdtab_allocated(cd_table))
1264 return NULL;
1265
1266 if (cd_table->s1fmt == STRTAB_STE_0_S1FMT_LINEAR)
1267 return &cd_table->linear.table[ssid];
1268
1269 l2 = cd_table->l2.l2ptrs[arm_smmu_cdtab_l1_idx(ssid)];
1270 if (!l2)
1271 return NULL;
1272 return &l2->cds[arm_smmu_cdtab_l2_idx(ssid)];
1273 }
1274
arm_smmu_alloc_cd_ptr(struct arm_smmu_master * master,u32 ssid)1275 static struct arm_smmu_cd *arm_smmu_alloc_cd_ptr(struct arm_smmu_master *master,
1276 u32 ssid)
1277 {
1278 struct arm_smmu_ctx_desc_cfg *cd_table = &master->cd_table;
1279 struct arm_smmu_device *smmu = master->smmu;
1280
1281 might_sleep();
1282 iommu_group_mutex_assert(master->dev);
1283
1284 if (!arm_smmu_cdtab_allocated(cd_table)) {
1285 if (arm_smmu_alloc_cd_tables(master))
1286 return NULL;
1287 }
1288
1289 if (cd_table->s1fmt == STRTAB_STE_0_S1FMT_64K_L2) {
1290 unsigned int idx = arm_smmu_cdtab_l1_idx(ssid);
1291 struct arm_smmu_cdtab_l2 **l2ptr = &cd_table->l2.l2ptrs[idx];
1292
1293 if (!*l2ptr) {
1294 dma_addr_t l2ptr_dma;
1295
1296 *l2ptr = dma_alloc_coherent(smmu->dev, sizeof(**l2ptr),
1297 &l2ptr_dma, GFP_KERNEL);
1298 if (!*l2ptr)
1299 return NULL;
1300
1301 arm_smmu_write_cd_l1_desc(&cd_table->l2.l1tab[idx],
1302 l2ptr_dma);
1303 /* An invalid L1CD can be cached */
1304 arm_smmu_sync_cd(master, ssid, false);
1305 }
1306 }
1307 return arm_smmu_get_cd_ptr(master, ssid);
1308 }
1309
1310 struct arm_smmu_cd_writer {
1311 struct arm_smmu_entry_writer writer;
1312 unsigned int ssid;
1313 };
1314
1315 VISIBLE_IF_KUNIT
arm_smmu_get_cd_used(const __le64 * ent,__le64 * used_bits)1316 void arm_smmu_get_cd_used(const __le64 *ent, __le64 *used_bits)
1317 {
1318 used_bits[0] = cpu_to_le64(CTXDESC_CD_0_V);
1319 if (!(ent[0] & cpu_to_le64(CTXDESC_CD_0_V)))
1320 return;
1321 memset(used_bits, 0xFF, sizeof(struct arm_smmu_cd));
1322
1323 /*
1324 * If EPD0 is set by the make function it means
1325 * T0SZ/TG0/IR0/OR0/SH0/TTB0 are IGNORED
1326 */
1327 if (ent[0] & cpu_to_le64(CTXDESC_CD_0_TCR_EPD0)) {
1328 used_bits[0] &= ~cpu_to_le64(
1329 CTXDESC_CD_0_TCR_T0SZ | CTXDESC_CD_0_TCR_TG0 |
1330 CTXDESC_CD_0_TCR_IRGN0 | CTXDESC_CD_0_TCR_ORGN0 |
1331 CTXDESC_CD_0_TCR_SH0);
1332 used_bits[1] &= ~cpu_to_le64(CTXDESC_CD_1_TTB0_MASK);
1333 }
1334 }
1335 EXPORT_SYMBOL_IF_KUNIT(arm_smmu_get_cd_used);
1336
arm_smmu_cd_writer_sync_entry(struct arm_smmu_entry_writer * writer)1337 static void arm_smmu_cd_writer_sync_entry(struct arm_smmu_entry_writer *writer)
1338 {
1339 struct arm_smmu_cd_writer *cd_writer =
1340 container_of(writer, struct arm_smmu_cd_writer, writer);
1341
1342 arm_smmu_sync_cd(writer->master, cd_writer->ssid, true);
1343 }
1344
1345 static const struct arm_smmu_entry_writer_ops arm_smmu_cd_writer_ops = {
1346 .sync = arm_smmu_cd_writer_sync_entry,
1347 .get_used = arm_smmu_get_cd_used,
1348 };
1349
arm_smmu_write_cd_entry(struct arm_smmu_master * master,int ssid,struct arm_smmu_cd * cdptr,const struct arm_smmu_cd * target)1350 void arm_smmu_write_cd_entry(struct arm_smmu_master *master, int ssid,
1351 struct arm_smmu_cd *cdptr,
1352 const struct arm_smmu_cd *target)
1353 {
1354 bool target_valid = target->data[0] & cpu_to_le64(CTXDESC_CD_0_V);
1355 bool cur_valid = cdptr->data[0] & cpu_to_le64(CTXDESC_CD_0_V);
1356 struct arm_smmu_cd_writer cd_writer = {
1357 .writer = {
1358 .ops = &arm_smmu_cd_writer_ops,
1359 .master = master,
1360 },
1361 .ssid = ssid,
1362 };
1363
1364 if (ssid != IOMMU_NO_PASID && cur_valid != target_valid) {
1365 if (cur_valid)
1366 master->cd_table.used_ssids--;
1367 else
1368 master->cd_table.used_ssids++;
1369 }
1370
1371 arm_smmu_write_entry(&cd_writer.writer, cdptr->data, target->data);
1372 }
1373
arm_smmu_make_s1_cd(struct arm_smmu_cd * target,struct arm_smmu_master * master,struct arm_smmu_domain * smmu_domain)1374 void arm_smmu_make_s1_cd(struct arm_smmu_cd *target,
1375 struct arm_smmu_master *master,
1376 struct arm_smmu_domain *smmu_domain)
1377 {
1378 struct arm_smmu_ctx_desc *cd = &smmu_domain->cd;
1379 const struct io_pgtable_cfg *pgtbl_cfg =
1380 &io_pgtable_ops_to_pgtable(smmu_domain->pgtbl_ops)->cfg;
1381 typeof(&pgtbl_cfg->arm_lpae_s1_cfg.tcr) tcr =
1382 &pgtbl_cfg->arm_lpae_s1_cfg.tcr;
1383
1384 memset(target, 0, sizeof(*target));
1385
1386 target->data[0] = cpu_to_le64(
1387 FIELD_PREP(CTXDESC_CD_0_TCR_T0SZ, tcr->tsz) |
1388 FIELD_PREP(CTXDESC_CD_0_TCR_TG0, tcr->tg) |
1389 FIELD_PREP(CTXDESC_CD_0_TCR_IRGN0, tcr->irgn) |
1390 FIELD_PREP(CTXDESC_CD_0_TCR_ORGN0, tcr->orgn) |
1391 FIELD_PREP(CTXDESC_CD_0_TCR_SH0, tcr->sh) |
1392 #ifdef __BIG_ENDIAN
1393 CTXDESC_CD_0_ENDI |
1394 #endif
1395 CTXDESC_CD_0_TCR_EPD1 |
1396 CTXDESC_CD_0_V |
1397 FIELD_PREP(CTXDESC_CD_0_TCR_IPS, tcr->ips) |
1398 CTXDESC_CD_0_AA64 |
1399 (master->stall_enabled ? CTXDESC_CD_0_S : 0) |
1400 CTXDESC_CD_0_R |
1401 CTXDESC_CD_0_A |
1402 CTXDESC_CD_0_ASET |
1403 FIELD_PREP(CTXDESC_CD_0_ASID, cd->asid)
1404 );
1405
1406 /* To enable dirty flag update, set both Access flag and dirty state update */
1407 if (pgtbl_cfg->quirks & IO_PGTABLE_QUIRK_ARM_HD)
1408 target->data[0] |= cpu_to_le64(CTXDESC_CD_0_TCR_HA |
1409 CTXDESC_CD_0_TCR_HD);
1410
1411 target->data[1] = cpu_to_le64(pgtbl_cfg->arm_lpae_s1_cfg.ttbr &
1412 CTXDESC_CD_1_TTB0_MASK);
1413 target->data[3] = cpu_to_le64(pgtbl_cfg->arm_lpae_s1_cfg.mair);
1414 }
1415 EXPORT_SYMBOL_IF_KUNIT(arm_smmu_make_s1_cd);
1416
arm_smmu_clear_cd(struct arm_smmu_master * master,ioasid_t ssid)1417 void arm_smmu_clear_cd(struct arm_smmu_master *master, ioasid_t ssid)
1418 {
1419 struct arm_smmu_cd target = {};
1420 struct arm_smmu_cd *cdptr;
1421
1422 if (!arm_smmu_cdtab_allocated(&master->cd_table))
1423 return;
1424 cdptr = arm_smmu_get_cd_ptr(master, ssid);
1425 if (WARN_ON(!cdptr))
1426 return;
1427 arm_smmu_write_cd_entry(master, ssid, cdptr, &target);
1428 }
1429
arm_smmu_alloc_cd_tables(struct arm_smmu_master * master)1430 static int arm_smmu_alloc_cd_tables(struct arm_smmu_master *master)
1431 {
1432 int ret;
1433 size_t l1size;
1434 size_t max_contexts;
1435 struct arm_smmu_device *smmu = master->smmu;
1436 struct arm_smmu_ctx_desc_cfg *cd_table = &master->cd_table;
1437
1438 cd_table->s1cdmax = master->ssid_bits;
1439 max_contexts = 1 << cd_table->s1cdmax;
1440
1441 if (!(smmu->features & ARM_SMMU_FEAT_2_LVL_CDTAB) ||
1442 max_contexts <= CTXDESC_L2_ENTRIES) {
1443 cd_table->s1fmt = STRTAB_STE_0_S1FMT_LINEAR;
1444 cd_table->linear.num_ents = max_contexts;
1445
1446 l1size = max_contexts * sizeof(struct arm_smmu_cd);
1447 cd_table->linear.table = dma_alloc_coherent(smmu->dev, l1size,
1448 &cd_table->cdtab_dma,
1449 GFP_KERNEL);
1450 if (!cd_table->linear.table)
1451 return -ENOMEM;
1452 } else {
1453 cd_table->s1fmt = STRTAB_STE_0_S1FMT_64K_L2;
1454 cd_table->l2.num_l1_ents =
1455 DIV_ROUND_UP(max_contexts, CTXDESC_L2_ENTRIES);
1456
1457 cd_table->l2.l2ptrs = kcalloc(cd_table->l2.num_l1_ents,
1458 sizeof(*cd_table->l2.l2ptrs),
1459 GFP_KERNEL);
1460 if (!cd_table->l2.l2ptrs)
1461 return -ENOMEM;
1462
1463 l1size = cd_table->l2.num_l1_ents * sizeof(struct arm_smmu_cdtab_l1);
1464 cd_table->l2.l1tab = dma_alloc_coherent(smmu->dev, l1size,
1465 &cd_table->cdtab_dma,
1466 GFP_KERNEL);
1467 if (!cd_table->l2.l2ptrs) {
1468 ret = -ENOMEM;
1469 goto err_free_l2ptrs;
1470 }
1471 }
1472 return 0;
1473
1474 err_free_l2ptrs:
1475 kfree(cd_table->l2.l2ptrs);
1476 cd_table->l2.l2ptrs = NULL;
1477 return ret;
1478 }
1479
arm_smmu_free_cd_tables(struct arm_smmu_master * master)1480 static void arm_smmu_free_cd_tables(struct arm_smmu_master *master)
1481 {
1482 int i;
1483 struct arm_smmu_device *smmu = master->smmu;
1484 struct arm_smmu_ctx_desc_cfg *cd_table = &master->cd_table;
1485
1486 if (cd_table->s1fmt != STRTAB_STE_0_S1FMT_LINEAR) {
1487 for (i = 0; i < cd_table->l2.num_l1_ents; i++) {
1488 if (!cd_table->l2.l2ptrs[i])
1489 continue;
1490
1491 dma_free_coherent(smmu->dev,
1492 sizeof(*cd_table->l2.l2ptrs[i]),
1493 cd_table->l2.l2ptrs[i],
1494 arm_smmu_cd_l1_get_desc(&cd_table->l2.l1tab[i]));
1495 }
1496 kfree(cd_table->l2.l2ptrs);
1497
1498 dma_free_coherent(smmu->dev,
1499 cd_table->l2.num_l1_ents *
1500 sizeof(struct arm_smmu_cdtab_l1),
1501 cd_table->l2.l1tab, cd_table->cdtab_dma);
1502 } else {
1503 dma_free_coherent(smmu->dev,
1504 cd_table->linear.num_ents *
1505 sizeof(struct arm_smmu_cd),
1506 cd_table->linear.table, cd_table->cdtab_dma);
1507 }
1508 }
1509
1510 /* Stream table manipulation functions */
arm_smmu_write_strtab_l1_desc(struct arm_smmu_strtab_l1 * dst,dma_addr_t l2ptr_dma)1511 static void arm_smmu_write_strtab_l1_desc(struct arm_smmu_strtab_l1 *dst,
1512 dma_addr_t l2ptr_dma)
1513 {
1514 u64 val = 0;
1515
1516 val |= FIELD_PREP(STRTAB_L1_DESC_SPAN, STRTAB_SPLIT + 1);
1517 val |= l2ptr_dma & STRTAB_L1_DESC_L2PTR_MASK;
1518
1519 /* The HW has 64 bit atomicity with stores to the L2 STE table */
1520 WRITE_ONCE(dst->l2ptr, cpu_to_le64(val));
1521 }
1522
1523 struct arm_smmu_ste_writer {
1524 struct arm_smmu_entry_writer writer;
1525 u32 sid;
1526 };
1527
arm_smmu_ste_writer_sync_entry(struct arm_smmu_entry_writer * writer)1528 static void arm_smmu_ste_writer_sync_entry(struct arm_smmu_entry_writer *writer)
1529 {
1530 struct arm_smmu_ste_writer *ste_writer =
1531 container_of(writer, struct arm_smmu_ste_writer, writer);
1532 struct arm_smmu_cmdq_ent cmd = {
1533 .opcode = CMDQ_OP_CFGI_STE,
1534 .cfgi = {
1535 .sid = ste_writer->sid,
1536 .leaf = true,
1537 },
1538 };
1539
1540 arm_smmu_cmdq_issue_cmd_with_sync(writer->master->smmu, &cmd);
1541 }
1542
1543 static const struct arm_smmu_entry_writer_ops arm_smmu_ste_writer_ops = {
1544 .sync = arm_smmu_ste_writer_sync_entry,
1545 .get_used = arm_smmu_get_ste_used,
1546 };
1547
arm_smmu_write_ste(struct arm_smmu_master * master,u32 sid,struct arm_smmu_ste * ste,const struct arm_smmu_ste * target)1548 static void arm_smmu_write_ste(struct arm_smmu_master *master, u32 sid,
1549 struct arm_smmu_ste *ste,
1550 const struct arm_smmu_ste *target)
1551 {
1552 struct arm_smmu_device *smmu = master->smmu;
1553 struct arm_smmu_ste_writer ste_writer = {
1554 .writer = {
1555 .ops = &arm_smmu_ste_writer_ops,
1556 .master = master,
1557 },
1558 .sid = sid,
1559 };
1560
1561 arm_smmu_write_entry(&ste_writer.writer, ste->data, target->data);
1562
1563 /* It's likely that we'll want to use the new STE soon */
1564 if (!(smmu->options & ARM_SMMU_OPT_SKIP_PREFETCH)) {
1565 struct arm_smmu_cmdq_ent
1566 prefetch_cmd = { .opcode = CMDQ_OP_PREFETCH_CFG,
1567 .prefetch = {
1568 .sid = sid,
1569 } };
1570
1571 arm_smmu_cmdq_issue_cmd(smmu, &prefetch_cmd);
1572 }
1573 }
1574
arm_smmu_make_abort_ste(struct arm_smmu_ste * target)1575 void arm_smmu_make_abort_ste(struct arm_smmu_ste *target)
1576 {
1577 memset(target, 0, sizeof(*target));
1578 target->data[0] = cpu_to_le64(
1579 STRTAB_STE_0_V |
1580 FIELD_PREP(STRTAB_STE_0_CFG, STRTAB_STE_0_CFG_ABORT));
1581 }
1582 EXPORT_SYMBOL_IF_KUNIT(arm_smmu_make_abort_ste);
1583
1584 VISIBLE_IF_KUNIT
arm_smmu_make_bypass_ste(struct arm_smmu_device * smmu,struct arm_smmu_ste * target)1585 void arm_smmu_make_bypass_ste(struct arm_smmu_device *smmu,
1586 struct arm_smmu_ste *target)
1587 {
1588 memset(target, 0, sizeof(*target));
1589 target->data[0] = cpu_to_le64(
1590 STRTAB_STE_0_V |
1591 FIELD_PREP(STRTAB_STE_0_CFG, STRTAB_STE_0_CFG_BYPASS));
1592
1593 if (smmu->features & ARM_SMMU_FEAT_ATTR_TYPES_OVR)
1594 target->data[1] = cpu_to_le64(FIELD_PREP(STRTAB_STE_1_SHCFG,
1595 STRTAB_STE_1_SHCFG_INCOMING));
1596 }
1597 EXPORT_SYMBOL_IF_KUNIT(arm_smmu_make_bypass_ste);
1598
1599 VISIBLE_IF_KUNIT
arm_smmu_make_cdtable_ste(struct arm_smmu_ste * target,struct arm_smmu_master * master,bool ats_enabled,unsigned int s1dss)1600 void arm_smmu_make_cdtable_ste(struct arm_smmu_ste *target,
1601 struct arm_smmu_master *master, bool ats_enabled,
1602 unsigned int s1dss)
1603 {
1604 struct arm_smmu_ctx_desc_cfg *cd_table = &master->cd_table;
1605 struct arm_smmu_device *smmu = master->smmu;
1606
1607 memset(target, 0, sizeof(*target));
1608 target->data[0] = cpu_to_le64(
1609 STRTAB_STE_0_V |
1610 FIELD_PREP(STRTAB_STE_0_CFG, STRTAB_STE_0_CFG_S1_TRANS) |
1611 FIELD_PREP(STRTAB_STE_0_S1FMT, cd_table->s1fmt) |
1612 (cd_table->cdtab_dma & STRTAB_STE_0_S1CTXPTR_MASK) |
1613 FIELD_PREP(STRTAB_STE_0_S1CDMAX, cd_table->s1cdmax));
1614
1615 target->data[1] = cpu_to_le64(
1616 FIELD_PREP(STRTAB_STE_1_S1DSS, s1dss) |
1617 FIELD_PREP(STRTAB_STE_1_S1CIR, STRTAB_STE_1_S1C_CACHE_WBRA) |
1618 FIELD_PREP(STRTAB_STE_1_S1COR, STRTAB_STE_1_S1C_CACHE_WBRA) |
1619 FIELD_PREP(STRTAB_STE_1_S1CSH, ARM_SMMU_SH_ISH) |
1620 ((smmu->features & ARM_SMMU_FEAT_STALLS &&
1621 !master->stall_enabled) ?
1622 STRTAB_STE_1_S1STALLD :
1623 0) |
1624 FIELD_PREP(STRTAB_STE_1_EATS,
1625 ats_enabled ? STRTAB_STE_1_EATS_TRANS : 0));
1626
1627 if ((smmu->features & ARM_SMMU_FEAT_ATTR_TYPES_OVR) &&
1628 s1dss == STRTAB_STE_1_S1DSS_BYPASS)
1629 target->data[1] |= cpu_to_le64(FIELD_PREP(
1630 STRTAB_STE_1_SHCFG, STRTAB_STE_1_SHCFG_INCOMING));
1631
1632 if (smmu->features & ARM_SMMU_FEAT_E2H) {
1633 /*
1634 * To support BTM the streamworld needs to match the
1635 * configuration of the CPU so that the ASID broadcasts are
1636 * properly matched. This means either S/NS-EL2-E2H (hypervisor)
1637 * or NS-EL1 (guest). Since an SVA domain can be installed in a
1638 * PASID this should always use a BTM compatible configuration
1639 * if the HW supports it.
1640 */
1641 target->data[1] |= cpu_to_le64(
1642 FIELD_PREP(STRTAB_STE_1_STRW, STRTAB_STE_1_STRW_EL2));
1643 } else {
1644 target->data[1] |= cpu_to_le64(
1645 FIELD_PREP(STRTAB_STE_1_STRW, STRTAB_STE_1_STRW_NSEL1));
1646
1647 /*
1648 * VMID 0 is reserved for stage-2 bypass EL1 STEs, see
1649 * arm_smmu_domain_alloc_id()
1650 */
1651 target->data[2] =
1652 cpu_to_le64(FIELD_PREP(STRTAB_STE_2_S2VMID, 0));
1653 }
1654 }
1655 EXPORT_SYMBOL_IF_KUNIT(arm_smmu_make_cdtable_ste);
1656
arm_smmu_make_s2_domain_ste(struct arm_smmu_ste * target,struct arm_smmu_master * master,struct arm_smmu_domain * smmu_domain,bool ats_enabled)1657 void arm_smmu_make_s2_domain_ste(struct arm_smmu_ste *target,
1658 struct arm_smmu_master *master,
1659 struct arm_smmu_domain *smmu_domain,
1660 bool ats_enabled)
1661 {
1662 struct arm_smmu_s2_cfg *s2_cfg = &smmu_domain->s2_cfg;
1663 const struct io_pgtable_cfg *pgtbl_cfg =
1664 &io_pgtable_ops_to_pgtable(smmu_domain->pgtbl_ops)->cfg;
1665 typeof(&pgtbl_cfg->arm_lpae_s2_cfg.vtcr) vtcr =
1666 &pgtbl_cfg->arm_lpae_s2_cfg.vtcr;
1667 u64 vtcr_val;
1668 struct arm_smmu_device *smmu = master->smmu;
1669
1670 memset(target, 0, sizeof(*target));
1671 target->data[0] = cpu_to_le64(
1672 STRTAB_STE_0_V |
1673 FIELD_PREP(STRTAB_STE_0_CFG, STRTAB_STE_0_CFG_S2_TRANS));
1674
1675 target->data[1] = cpu_to_le64(
1676 FIELD_PREP(STRTAB_STE_1_EATS,
1677 ats_enabled ? STRTAB_STE_1_EATS_TRANS : 0));
1678
1679 if (pgtbl_cfg->quirks & IO_PGTABLE_QUIRK_ARM_S2FWB)
1680 target->data[1] |= cpu_to_le64(STRTAB_STE_1_S2FWB);
1681 if (smmu->features & ARM_SMMU_FEAT_ATTR_TYPES_OVR)
1682 target->data[1] |= cpu_to_le64(FIELD_PREP(STRTAB_STE_1_SHCFG,
1683 STRTAB_STE_1_SHCFG_INCOMING));
1684
1685 vtcr_val = FIELD_PREP(STRTAB_STE_2_VTCR_S2T0SZ, vtcr->tsz) |
1686 FIELD_PREP(STRTAB_STE_2_VTCR_S2SL0, vtcr->sl) |
1687 FIELD_PREP(STRTAB_STE_2_VTCR_S2IR0, vtcr->irgn) |
1688 FIELD_PREP(STRTAB_STE_2_VTCR_S2OR0, vtcr->orgn) |
1689 FIELD_PREP(STRTAB_STE_2_VTCR_S2SH0, vtcr->sh) |
1690 FIELD_PREP(STRTAB_STE_2_VTCR_S2TG, vtcr->tg) |
1691 FIELD_PREP(STRTAB_STE_2_VTCR_S2PS, vtcr->ps);
1692 target->data[2] = cpu_to_le64(
1693 FIELD_PREP(STRTAB_STE_2_S2VMID, s2_cfg->vmid) |
1694 FIELD_PREP(STRTAB_STE_2_VTCR, vtcr_val) |
1695 STRTAB_STE_2_S2AA64 |
1696 #ifdef __BIG_ENDIAN
1697 STRTAB_STE_2_S2ENDI |
1698 #endif
1699 STRTAB_STE_2_S2PTW |
1700 (master->stall_enabled ? STRTAB_STE_2_S2S : 0) |
1701 STRTAB_STE_2_S2R);
1702
1703 target->data[3] = cpu_to_le64(pgtbl_cfg->arm_lpae_s2_cfg.vttbr &
1704 STRTAB_STE_3_S2TTB_MASK);
1705 }
1706 EXPORT_SYMBOL_IF_KUNIT(arm_smmu_make_s2_domain_ste);
1707
1708 /*
1709 * This can safely directly manipulate the STE memory without a sync sequence
1710 * because the STE table has not been installed in the SMMU yet.
1711 */
arm_smmu_init_initial_stes(struct arm_smmu_ste * strtab,unsigned int nent)1712 static void arm_smmu_init_initial_stes(struct arm_smmu_ste *strtab,
1713 unsigned int nent)
1714 {
1715 unsigned int i;
1716
1717 for (i = 0; i < nent; ++i) {
1718 arm_smmu_make_abort_ste(strtab);
1719 strtab++;
1720 }
1721 }
1722
arm_smmu_init_l2_strtab(struct arm_smmu_device * smmu,u32 sid)1723 static int arm_smmu_init_l2_strtab(struct arm_smmu_device *smmu, u32 sid)
1724 {
1725 dma_addr_t l2ptr_dma;
1726 struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg;
1727 struct arm_smmu_strtab_l2 **l2table;
1728
1729 l2table = &cfg->l2.l2ptrs[arm_smmu_strtab_l1_idx(sid)];
1730 if (*l2table)
1731 return 0;
1732
1733 *l2table = dmam_alloc_coherent(smmu->dev, sizeof(**l2table),
1734 &l2ptr_dma, GFP_KERNEL);
1735 if (!*l2table) {
1736 dev_err(smmu->dev,
1737 "failed to allocate l2 stream table for SID %u\n",
1738 sid);
1739 return -ENOMEM;
1740 }
1741
1742 arm_smmu_init_initial_stes((*l2table)->stes,
1743 ARRAY_SIZE((*l2table)->stes));
1744 arm_smmu_write_strtab_l1_desc(&cfg->l2.l1tab[arm_smmu_strtab_l1_idx(sid)],
1745 l2ptr_dma);
1746 return 0;
1747 }
1748
arm_smmu_streams_cmp_key(const void * lhs,const struct rb_node * rhs)1749 static int arm_smmu_streams_cmp_key(const void *lhs, const struct rb_node *rhs)
1750 {
1751 struct arm_smmu_stream *stream_rhs =
1752 rb_entry(rhs, struct arm_smmu_stream, node);
1753 const u32 *sid_lhs = lhs;
1754
1755 if (*sid_lhs < stream_rhs->id)
1756 return -1;
1757 if (*sid_lhs > stream_rhs->id)
1758 return 1;
1759 return 0;
1760 }
1761
arm_smmu_streams_cmp_node(struct rb_node * lhs,const struct rb_node * rhs)1762 static int arm_smmu_streams_cmp_node(struct rb_node *lhs,
1763 const struct rb_node *rhs)
1764 {
1765 return arm_smmu_streams_cmp_key(
1766 &rb_entry(lhs, struct arm_smmu_stream, node)->id, rhs);
1767 }
1768
1769 static struct arm_smmu_master *
arm_smmu_find_master(struct arm_smmu_device * smmu,u32 sid)1770 arm_smmu_find_master(struct arm_smmu_device *smmu, u32 sid)
1771 {
1772 struct rb_node *node;
1773
1774 lockdep_assert_held(&smmu->streams_mutex);
1775
1776 node = rb_find(&sid, &smmu->streams, arm_smmu_streams_cmp_key);
1777 if (!node)
1778 return NULL;
1779 return rb_entry(node, struct arm_smmu_stream, node)->master;
1780 }
1781
1782 /* IRQ and event handlers */
arm_smmu_decode_event(struct arm_smmu_device * smmu,u64 * raw,struct arm_smmu_event * event)1783 static void arm_smmu_decode_event(struct arm_smmu_device *smmu, u64 *raw,
1784 struct arm_smmu_event *event)
1785 {
1786 struct arm_smmu_master *master;
1787
1788 event->id = FIELD_GET(EVTQ_0_ID, raw[0]);
1789 event->sid = FIELD_GET(EVTQ_0_SID, raw[0]);
1790 event->ssv = FIELD_GET(EVTQ_0_SSV, raw[0]);
1791 event->ssid = event->ssv ? FIELD_GET(EVTQ_0_SSID, raw[0]) : IOMMU_NO_PASID;
1792 event->privileged = FIELD_GET(EVTQ_1_PnU, raw[1]);
1793 event->instruction = FIELD_GET(EVTQ_1_InD, raw[1]);
1794 event->s2 = FIELD_GET(EVTQ_1_S2, raw[1]);
1795 event->read = FIELD_GET(EVTQ_1_RnW, raw[1]);
1796 event->stag = FIELD_GET(EVTQ_1_STAG, raw[1]);
1797 event->stall = FIELD_GET(EVTQ_1_STALL, raw[1]);
1798 event->class = FIELD_GET(EVTQ_1_CLASS, raw[1]);
1799 event->iova = FIELD_GET(EVTQ_2_ADDR, raw[2]);
1800 event->ipa = raw[3] & EVTQ_3_IPA;
1801 event->fetch_addr = raw[3] & EVTQ_3_FETCH_ADDR;
1802 event->ttrnw = FIELD_GET(EVTQ_1_TT_READ, raw[1]);
1803 event->class_tt = false;
1804 event->dev = NULL;
1805
1806 if (event->id == EVT_ID_PERMISSION_FAULT)
1807 event->class_tt = (event->class == EVTQ_1_CLASS_TT);
1808
1809 mutex_lock(&smmu->streams_mutex);
1810 master = arm_smmu_find_master(smmu, event->sid);
1811 if (master)
1812 event->dev = get_device(master->dev);
1813 mutex_unlock(&smmu->streams_mutex);
1814 }
1815
arm_smmu_handle_event(struct arm_smmu_device * smmu,u64 * evt,struct arm_smmu_event * event)1816 static int arm_smmu_handle_event(struct arm_smmu_device *smmu, u64 *evt,
1817 struct arm_smmu_event *event)
1818 {
1819 int ret = 0;
1820 u32 perm = 0;
1821 struct arm_smmu_master *master;
1822 struct iopf_fault fault_evt = { };
1823 struct iommu_fault *flt = &fault_evt.fault;
1824
1825 switch (event->id) {
1826 case EVT_ID_BAD_STE_CONFIG:
1827 case EVT_ID_STREAM_DISABLED_FAULT:
1828 case EVT_ID_BAD_SUBSTREAMID_CONFIG:
1829 case EVT_ID_BAD_CD_CONFIG:
1830 case EVT_ID_TRANSLATION_FAULT:
1831 case EVT_ID_ADDR_SIZE_FAULT:
1832 case EVT_ID_ACCESS_FAULT:
1833 case EVT_ID_PERMISSION_FAULT:
1834 break;
1835 default:
1836 return -EOPNOTSUPP;
1837 }
1838
1839 if (event->stall) {
1840 if (event->read)
1841 perm |= IOMMU_FAULT_PERM_READ;
1842 else
1843 perm |= IOMMU_FAULT_PERM_WRITE;
1844
1845 if (event->instruction)
1846 perm |= IOMMU_FAULT_PERM_EXEC;
1847
1848 if (event->privileged)
1849 perm |= IOMMU_FAULT_PERM_PRIV;
1850
1851 flt->type = IOMMU_FAULT_PAGE_REQ;
1852 flt->prm = (struct iommu_fault_page_request){
1853 .flags = IOMMU_FAULT_PAGE_REQUEST_LAST_PAGE,
1854 .grpid = event->stag,
1855 .perm = perm,
1856 .addr = event->iova,
1857 };
1858
1859 if (event->ssv) {
1860 flt->prm.flags |= IOMMU_FAULT_PAGE_REQUEST_PASID_VALID;
1861 flt->prm.pasid = event->ssid;
1862 }
1863 }
1864
1865 mutex_lock(&smmu->streams_mutex);
1866 master = arm_smmu_find_master(smmu, event->sid);
1867 if (!master) {
1868 ret = -EINVAL;
1869 goto out_unlock;
1870 }
1871
1872 if (event->stall)
1873 ret = iommu_report_device_fault(master->dev, &fault_evt);
1874 else if (master->vmaster && !event->s2)
1875 ret = arm_vmaster_report_event(master->vmaster, evt);
1876 else
1877 ret = -EOPNOTSUPP; /* Unhandled events should be pinned */
1878 out_unlock:
1879 mutex_unlock(&smmu->streams_mutex);
1880 return ret;
1881 }
1882
arm_smmu_dump_raw_event(struct arm_smmu_device * smmu,u64 * raw,struct arm_smmu_event * event)1883 static void arm_smmu_dump_raw_event(struct arm_smmu_device *smmu, u64 *raw,
1884 struct arm_smmu_event *event)
1885 {
1886 int i;
1887
1888 dev_err(smmu->dev, "event 0x%02x received:\n", event->id);
1889
1890 for (i = 0; i < EVTQ_ENT_DWORDS; ++i)
1891 dev_err(smmu->dev, "\t0x%016llx\n", raw[i]);
1892 }
1893
1894 #define ARM_SMMU_EVT_KNOWN(e) ((e)->id < ARRAY_SIZE(event_str) && event_str[(e)->id])
1895 #define ARM_SMMU_LOG_EVT_STR(e) ARM_SMMU_EVT_KNOWN(e) ? event_str[(e)->id] : "UNKNOWN"
1896 #define ARM_SMMU_LOG_CLIENT(e) (e)->dev ? dev_name((e)->dev) : "(unassigned sid)"
1897
arm_smmu_dump_event(struct arm_smmu_device * smmu,u64 * raw,struct arm_smmu_event * evt,struct ratelimit_state * rs)1898 static void arm_smmu_dump_event(struct arm_smmu_device *smmu, u64 *raw,
1899 struct arm_smmu_event *evt,
1900 struct ratelimit_state *rs)
1901 {
1902 if (!__ratelimit(rs))
1903 return;
1904
1905 arm_smmu_dump_raw_event(smmu, raw, evt);
1906
1907 switch (evt->id) {
1908 case EVT_ID_TRANSLATION_FAULT:
1909 case EVT_ID_ADDR_SIZE_FAULT:
1910 case EVT_ID_ACCESS_FAULT:
1911 case EVT_ID_PERMISSION_FAULT:
1912 dev_err(smmu->dev, "event: %s client: %s sid: %#x ssid: %#x iova: %#llx ipa: %#llx",
1913 ARM_SMMU_LOG_EVT_STR(evt), ARM_SMMU_LOG_CLIENT(evt),
1914 evt->sid, evt->ssid, evt->iova, evt->ipa);
1915
1916 dev_err(smmu->dev, "%s %s %s %s \"%s\"%s%s stag: %#x",
1917 evt->privileged ? "priv" : "unpriv",
1918 evt->instruction ? "inst" : "data",
1919 str_read_write(evt->read),
1920 evt->s2 ? "s2" : "s1", event_class_str[evt->class],
1921 evt->class_tt ? (evt->ttrnw ? " ttd_read" : " ttd_write") : "",
1922 evt->stall ? " stall" : "", evt->stag);
1923
1924 break;
1925
1926 case EVT_ID_STE_FETCH_FAULT:
1927 case EVT_ID_CD_FETCH_FAULT:
1928 case EVT_ID_VMS_FETCH_FAULT:
1929 dev_err(smmu->dev, "event: %s client: %s sid: %#x ssid: %#x fetch_addr: %#llx",
1930 ARM_SMMU_LOG_EVT_STR(evt), ARM_SMMU_LOG_CLIENT(evt),
1931 evt->sid, evt->ssid, evt->fetch_addr);
1932
1933 break;
1934
1935 default:
1936 dev_err(smmu->dev, "event: %s client: %s sid: %#x ssid: %#x",
1937 ARM_SMMU_LOG_EVT_STR(evt), ARM_SMMU_LOG_CLIENT(evt),
1938 evt->sid, evt->ssid);
1939 }
1940 }
1941
arm_smmu_evtq_thread(int irq,void * dev)1942 static irqreturn_t arm_smmu_evtq_thread(int irq, void *dev)
1943 {
1944 u64 evt[EVTQ_ENT_DWORDS];
1945 struct arm_smmu_event event = {0};
1946 struct arm_smmu_device *smmu = dev;
1947 struct arm_smmu_queue *q = &smmu->evtq.q;
1948 struct arm_smmu_ll_queue *llq = &q->llq;
1949 static DEFINE_RATELIMIT_STATE(rs, DEFAULT_RATELIMIT_INTERVAL,
1950 DEFAULT_RATELIMIT_BURST);
1951
1952 do {
1953 while (!queue_remove_raw(q, evt)) {
1954 arm_smmu_decode_event(smmu, evt, &event);
1955 if (arm_smmu_handle_event(smmu, evt, &event))
1956 arm_smmu_dump_event(smmu, evt, &event, &rs);
1957
1958 put_device(event.dev);
1959 cond_resched();
1960 }
1961
1962 /*
1963 * Not much we can do on overflow, so scream and pretend we're
1964 * trying harder.
1965 */
1966 if (queue_sync_prod_in(q) == -EOVERFLOW)
1967 dev_err(smmu->dev, "EVTQ overflow detected -- events lost\n");
1968 } while (!queue_empty(llq));
1969
1970 /* Sync our overflow flag, as we believe we're up to speed */
1971 queue_sync_cons_ovf(q);
1972 return IRQ_HANDLED;
1973 }
1974
arm_smmu_handle_ppr(struct arm_smmu_device * smmu,u64 * evt)1975 static void arm_smmu_handle_ppr(struct arm_smmu_device *smmu, u64 *evt)
1976 {
1977 u32 sid, ssid;
1978 u16 grpid;
1979 bool ssv, last;
1980
1981 sid = FIELD_GET(PRIQ_0_SID, evt[0]);
1982 ssv = FIELD_GET(PRIQ_0_SSID_V, evt[0]);
1983 ssid = ssv ? FIELD_GET(PRIQ_0_SSID, evt[0]) : IOMMU_NO_PASID;
1984 last = FIELD_GET(PRIQ_0_PRG_LAST, evt[0]);
1985 grpid = FIELD_GET(PRIQ_1_PRG_IDX, evt[1]);
1986
1987 dev_info(smmu->dev, "unexpected PRI request received:\n");
1988 dev_info(smmu->dev,
1989 "\tsid 0x%08x.0x%05x: [%u%s] %sprivileged %s%s%s access at iova 0x%016llx\n",
1990 sid, ssid, grpid, last ? "L" : "",
1991 evt[0] & PRIQ_0_PERM_PRIV ? "" : "un",
1992 evt[0] & PRIQ_0_PERM_READ ? "R" : "",
1993 evt[0] & PRIQ_0_PERM_WRITE ? "W" : "",
1994 evt[0] & PRIQ_0_PERM_EXEC ? "X" : "",
1995 evt[1] & PRIQ_1_ADDR_MASK);
1996
1997 if (last) {
1998 struct arm_smmu_cmdq_ent cmd = {
1999 .opcode = CMDQ_OP_PRI_RESP,
2000 .substream_valid = ssv,
2001 .pri = {
2002 .sid = sid,
2003 .ssid = ssid,
2004 .grpid = grpid,
2005 .resp = PRI_RESP_DENY,
2006 },
2007 };
2008
2009 arm_smmu_cmdq_issue_cmd(smmu, &cmd);
2010 }
2011 }
2012
arm_smmu_priq_thread(int irq,void * dev)2013 static irqreturn_t arm_smmu_priq_thread(int irq, void *dev)
2014 {
2015 struct arm_smmu_device *smmu = dev;
2016 struct arm_smmu_queue *q = &smmu->priq.q;
2017 struct arm_smmu_ll_queue *llq = &q->llq;
2018 u64 evt[PRIQ_ENT_DWORDS];
2019
2020 do {
2021 while (!queue_remove_raw(q, evt))
2022 arm_smmu_handle_ppr(smmu, evt);
2023
2024 if (queue_sync_prod_in(q) == -EOVERFLOW)
2025 dev_err(smmu->dev, "PRIQ overflow detected -- requests lost\n");
2026 } while (!queue_empty(llq));
2027
2028 /* Sync our overflow flag, as we believe we're up to speed */
2029 queue_sync_cons_ovf(q);
2030 return IRQ_HANDLED;
2031 }
2032
2033 static int arm_smmu_device_disable(struct arm_smmu_device *smmu);
2034
arm_smmu_gerror_handler(int irq,void * dev)2035 static irqreturn_t arm_smmu_gerror_handler(int irq, void *dev)
2036 {
2037 u32 gerror, gerrorn, active;
2038 struct arm_smmu_device *smmu = dev;
2039
2040 gerror = readl_relaxed(smmu->base + ARM_SMMU_GERROR);
2041 gerrorn = readl_relaxed(smmu->base + ARM_SMMU_GERRORN);
2042
2043 active = gerror ^ gerrorn;
2044 if (!(active & GERROR_ERR_MASK))
2045 return IRQ_NONE; /* No errors pending */
2046
2047 dev_warn(smmu->dev,
2048 "unexpected global error reported (0x%08x), this could be serious\n",
2049 active);
2050
2051 if (active & GERROR_SFM_ERR) {
2052 dev_err(smmu->dev, "device has entered Service Failure Mode!\n");
2053 arm_smmu_device_disable(smmu);
2054 }
2055
2056 if (active & GERROR_MSI_GERROR_ABT_ERR)
2057 dev_warn(smmu->dev, "GERROR MSI write aborted\n");
2058
2059 if (active & GERROR_MSI_PRIQ_ABT_ERR)
2060 dev_warn(smmu->dev, "PRIQ MSI write aborted\n");
2061
2062 if (active & GERROR_MSI_EVTQ_ABT_ERR)
2063 dev_warn(smmu->dev, "EVTQ MSI write aborted\n");
2064
2065 if (active & GERROR_MSI_CMDQ_ABT_ERR)
2066 dev_warn(smmu->dev, "CMDQ MSI write aborted\n");
2067
2068 if (active & GERROR_PRIQ_ABT_ERR)
2069 dev_err(smmu->dev, "PRIQ write aborted -- events may have been lost\n");
2070
2071 if (active & GERROR_EVTQ_ABT_ERR)
2072 dev_err(smmu->dev, "EVTQ write aborted -- events may have been lost\n");
2073
2074 if (active & GERROR_CMDQ_ERR)
2075 arm_smmu_cmdq_skip_err(smmu);
2076
2077 writel(gerror, smmu->base + ARM_SMMU_GERRORN);
2078 return IRQ_HANDLED;
2079 }
2080
arm_smmu_combined_irq_thread(int irq,void * dev)2081 static irqreturn_t arm_smmu_combined_irq_thread(int irq, void *dev)
2082 {
2083 struct arm_smmu_device *smmu = dev;
2084
2085 arm_smmu_evtq_thread(irq, dev);
2086 if (smmu->features & ARM_SMMU_FEAT_PRI)
2087 arm_smmu_priq_thread(irq, dev);
2088
2089 return IRQ_HANDLED;
2090 }
2091
arm_smmu_combined_irq_handler(int irq,void * dev)2092 static irqreturn_t arm_smmu_combined_irq_handler(int irq, void *dev)
2093 {
2094 arm_smmu_gerror_handler(irq, dev);
2095 return IRQ_WAKE_THREAD;
2096 }
2097
2098 static void
arm_smmu_atc_inv_to_cmd(int ssid,unsigned long iova,size_t size,struct arm_smmu_cmdq_ent * cmd)2099 arm_smmu_atc_inv_to_cmd(int ssid, unsigned long iova, size_t size,
2100 struct arm_smmu_cmdq_ent *cmd)
2101 {
2102 size_t log2_span;
2103 size_t span_mask;
2104 /* ATC invalidates are always on 4096-bytes pages */
2105 size_t inval_grain_shift = 12;
2106 unsigned long page_start, page_end;
2107
2108 /*
2109 * ATS and PASID:
2110 *
2111 * If substream_valid is clear, the PCIe TLP is sent without a PASID
2112 * prefix. In that case all ATC entries within the address range are
2113 * invalidated, including those that were requested with a PASID! There
2114 * is no way to invalidate only entries without PASID.
2115 *
2116 * When using STRTAB_STE_1_S1DSS_SSID0 (reserving CD 0 for non-PASID
2117 * traffic), translation requests without PASID create ATC entries
2118 * without PASID, which must be invalidated with substream_valid clear.
2119 * This has the unpleasant side-effect of invalidating all PASID-tagged
2120 * ATC entries within the address range.
2121 */
2122 *cmd = (struct arm_smmu_cmdq_ent) {
2123 .opcode = CMDQ_OP_ATC_INV,
2124 .substream_valid = (ssid != IOMMU_NO_PASID),
2125 .atc.ssid = ssid,
2126 };
2127
2128 if (!size) {
2129 cmd->atc.size = ATC_INV_SIZE_ALL;
2130 return;
2131 }
2132
2133 page_start = iova >> inval_grain_shift;
2134 page_end = (iova + size - 1) >> inval_grain_shift;
2135
2136 /*
2137 * In an ATS Invalidate Request, the address must be aligned on the
2138 * range size, which must be a power of two number of page sizes. We
2139 * thus have to choose between grossly over-invalidating the region, or
2140 * splitting the invalidation into multiple commands. For simplicity
2141 * we'll go with the first solution, but should refine it in the future
2142 * if multiple commands are shown to be more efficient.
2143 *
2144 * Find the smallest power of two that covers the range. The most
2145 * significant differing bit between the start and end addresses,
2146 * fls(start ^ end), indicates the required span. For example:
2147 *
2148 * We want to invalidate pages [8; 11]. This is already the ideal range:
2149 * x = 0b1000 ^ 0b1011 = 0b11
2150 * span = 1 << fls(x) = 4
2151 *
2152 * To invalidate pages [7; 10], we need to invalidate [0; 15]:
2153 * x = 0b0111 ^ 0b1010 = 0b1101
2154 * span = 1 << fls(x) = 16
2155 */
2156 log2_span = fls_long(page_start ^ page_end);
2157 span_mask = (1ULL << log2_span) - 1;
2158
2159 page_start &= ~span_mask;
2160
2161 cmd->atc.addr = page_start << inval_grain_shift;
2162 cmd->atc.size = log2_span;
2163 }
2164
arm_smmu_atc_inv_master(struct arm_smmu_master * master,ioasid_t ssid)2165 static int arm_smmu_atc_inv_master(struct arm_smmu_master *master,
2166 ioasid_t ssid)
2167 {
2168 int i;
2169 struct arm_smmu_cmdq_ent cmd;
2170 struct arm_smmu_cmdq_batch cmds;
2171
2172 arm_smmu_atc_inv_to_cmd(ssid, 0, 0, &cmd);
2173
2174 arm_smmu_cmdq_batch_init(master->smmu, &cmds, &cmd);
2175 for (i = 0; i < master->num_streams; i++) {
2176 cmd.atc.sid = master->streams[i].id;
2177 arm_smmu_cmdq_batch_add(master->smmu, &cmds, &cmd);
2178 }
2179
2180 return arm_smmu_cmdq_batch_submit(master->smmu, &cmds);
2181 }
2182
arm_smmu_atc_inv_domain(struct arm_smmu_domain * smmu_domain,unsigned long iova,size_t size)2183 int arm_smmu_atc_inv_domain(struct arm_smmu_domain *smmu_domain,
2184 unsigned long iova, size_t size)
2185 {
2186 struct arm_smmu_master_domain *master_domain;
2187 int i;
2188 unsigned long flags;
2189 struct arm_smmu_cmdq_ent cmd = {
2190 .opcode = CMDQ_OP_ATC_INV,
2191 };
2192 struct arm_smmu_cmdq_batch cmds;
2193
2194 if (!(smmu_domain->smmu->features & ARM_SMMU_FEAT_ATS))
2195 return 0;
2196
2197 /*
2198 * Ensure that we've completed prior invalidation of the main TLBs
2199 * before we read 'nr_ats_masters' in case of a concurrent call to
2200 * arm_smmu_enable_ats():
2201 *
2202 * // unmap() // arm_smmu_enable_ats()
2203 * TLBI+SYNC atomic_inc(&nr_ats_masters);
2204 * smp_mb(); [...]
2205 * atomic_read(&nr_ats_masters); pci_enable_ats() // writel()
2206 *
2207 * Ensures that we always see the incremented 'nr_ats_masters' count if
2208 * ATS was enabled at the PCI device before completion of the TLBI.
2209 */
2210 smp_mb();
2211 if (!atomic_read(&smmu_domain->nr_ats_masters))
2212 return 0;
2213
2214 arm_smmu_cmdq_batch_init(smmu_domain->smmu, &cmds, &cmd);
2215
2216 spin_lock_irqsave(&smmu_domain->devices_lock, flags);
2217 list_for_each_entry(master_domain, &smmu_domain->devices,
2218 devices_elm) {
2219 struct arm_smmu_master *master = master_domain->master;
2220
2221 if (!master->ats_enabled)
2222 continue;
2223
2224 if (master_domain->nested_ats_flush) {
2225 /*
2226 * If a S2 used as a nesting parent is changed we have
2227 * no option but to completely flush the ATC.
2228 */
2229 arm_smmu_atc_inv_to_cmd(IOMMU_NO_PASID, 0, 0, &cmd);
2230 } else {
2231 arm_smmu_atc_inv_to_cmd(master_domain->ssid, iova, size,
2232 &cmd);
2233 }
2234
2235 for (i = 0; i < master->num_streams; i++) {
2236 cmd.atc.sid = master->streams[i].id;
2237 arm_smmu_cmdq_batch_add(smmu_domain->smmu, &cmds, &cmd);
2238 }
2239 }
2240 spin_unlock_irqrestore(&smmu_domain->devices_lock, flags);
2241
2242 return arm_smmu_cmdq_batch_submit(smmu_domain->smmu, &cmds);
2243 }
2244
2245 /* IO_PGTABLE API */
arm_smmu_tlb_inv_context(void * cookie)2246 static void arm_smmu_tlb_inv_context(void *cookie)
2247 {
2248 struct arm_smmu_domain *smmu_domain = cookie;
2249 struct arm_smmu_device *smmu = smmu_domain->smmu;
2250 struct arm_smmu_cmdq_ent cmd;
2251
2252 /*
2253 * NOTE: when io-pgtable is in non-strict mode, we may get here with
2254 * PTEs previously cleared by unmaps on the current CPU not yet visible
2255 * to the SMMU. We are relying on the dma_wmb() implicit during cmd
2256 * insertion to guarantee those are observed before the TLBI. Do be
2257 * careful, 007.
2258 */
2259 if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {
2260 arm_smmu_tlb_inv_asid(smmu, smmu_domain->cd.asid);
2261 } else {
2262 cmd.opcode = CMDQ_OP_TLBI_S12_VMALL;
2263 cmd.tlbi.vmid = smmu_domain->s2_cfg.vmid;
2264 arm_smmu_cmdq_issue_cmd_with_sync(smmu, &cmd);
2265 }
2266 arm_smmu_atc_inv_domain(smmu_domain, 0, 0);
2267 }
2268
__arm_smmu_tlb_inv_range(struct arm_smmu_cmdq_ent * cmd,unsigned long iova,size_t size,size_t granule,struct arm_smmu_domain * smmu_domain)2269 static void __arm_smmu_tlb_inv_range(struct arm_smmu_cmdq_ent *cmd,
2270 unsigned long iova, size_t size,
2271 size_t granule,
2272 struct arm_smmu_domain *smmu_domain)
2273 {
2274 struct arm_smmu_device *smmu = smmu_domain->smmu;
2275 unsigned long end = iova + size, num_pages = 0, tg = 0;
2276 size_t inv_range = granule;
2277 struct arm_smmu_cmdq_batch cmds;
2278
2279 if (!size)
2280 return;
2281
2282 if (smmu->features & ARM_SMMU_FEAT_RANGE_INV) {
2283 /* Get the leaf page size */
2284 tg = __ffs(smmu_domain->domain.pgsize_bitmap);
2285
2286 num_pages = size >> tg;
2287
2288 /* Convert page size of 12,14,16 (log2) to 1,2,3 */
2289 cmd->tlbi.tg = (tg - 10) / 2;
2290
2291 /*
2292 * Determine what level the granule is at. For non-leaf, both
2293 * io-pgtable and SVA pass a nominal last-level granule because
2294 * they don't know what level(s) actually apply, so ignore that
2295 * and leave TTL=0. However for various errata reasons we still
2296 * want to use a range command, so avoid the SVA corner case
2297 * where both scale and num could be 0 as well.
2298 */
2299 if (cmd->tlbi.leaf)
2300 cmd->tlbi.ttl = 4 - ((ilog2(granule) - 3) / (tg - 3));
2301 else if ((num_pages & CMDQ_TLBI_RANGE_NUM_MAX) == 1)
2302 num_pages++;
2303 }
2304
2305 arm_smmu_cmdq_batch_init(smmu, &cmds, cmd);
2306
2307 while (iova < end) {
2308 if (smmu->features & ARM_SMMU_FEAT_RANGE_INV) {
2309 /*
2310 * On each iteration of the loop, the range is 5 bits
2311 * worth of the aligned size remaining.
2312 * The range in pages is:
2313 *
2314 * range = (num_pages & (0x1f << __ffs(num_pages)))
2315 */
2316 unsigned long scale, num;
2317
2318 /* Determine the power of 2 multiple number of pages */
2319 scale = __ffs(num_pages);
2320 cmd->tlbi.scale = scale;
2321
2322 /* Determine how many chunks of 2^scale size we have */
2323 num = (num_pages >> scale) & CMDQ_TLBI_RANGE_NUM_MAX;
2324 cmd->tlbi.num = num - 1;
2325
2326 /* range is num * 2^scale * pgsize */
2327 inv_range = num << (scale + tg);
2328
2329 /* Clear out the lower order bits for the next iteration */
2330 num_pages -= num << scale;
2331 }
2332
2333 cmd->tlbi.addr = iova;
2334 arm_smmu_cmdq_batch_add(smmu, &cmds, cmd);
2335 iova += inv_range;
2336 }
2337 arm_smmu_cmdq_batch_submit(smmu, &cmds);
2338 }
2339
arm_smmu_tlb_inv_range_domain(unsigned long iova,size_t size,size_t granule,bool leaf,struct arm_smmu_domain * smmu_domain)2340 static void arm_smmu_tlb_inv_range_domain(unsigned long iova, size_t size,
2341 size_t granule, bool leaf,
2342 struct arm_smmu_domain *smmu_domain)
2343 {
2344 struct arm_smmu_cmdq_ent cmd = {
2345 .tlbi = {
2346 .leaf = leaf,
2347 },
2348 };
2349
2350 if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {
2351 cmd.opcode = smmu_domain->smmu->features & ARM_SMMU_FEAT_E2H ?
2352 CMDQ_OP_TLBI_EL2_VA : CMDQ_OP_TLBI_NH_VA;
2353 cmd.tlbi.asid = smmu_domain->cd.asid;
2354 } else {
2355 cmd.opcode = CMDQ_OP_TLBI_S2_IPA;
2356 cmd.tlbi.vmid = smmu_domain->s2_cfg.vmid;
2357 }
2358 __arm_smmu_tlb_inv_range(&cmd, iova, size, granule, smmu_domain);
2359
2360 if (smmu_domain->nest_parent) {
2361 /*
2362 * When the S2 domain changes all the nested S1 ASIDs have to be
2363 * flushed too.
2364 */
2365 cmd.opcode = CMDQ_OP_TLBI_NH_ALL;
2366 arm_smmu_cmdq_issue_cmd_with_sync(smmu_domain->smmu, &cmd);
2367 }
2368
2369 /*
2370 * Unfortunately, this can't be leaf-only since we may have
2371 * zapped an entire table.
2372 */
2373 arm_smmu_atc_inv_domain(smmu_domain, iova, size);
2374 }
2375
arm_smmu_tlb_inv_range_asid(unsigned long iova,size_t size,int asid,size_t granule,bool leaf,struct arm_smmu_domain * smmu_domain)2376 void arm_smmu_tlb_inv_range_asid(unsigned long iova, size_t size, int asid,
2377 size_t granule, bool leaf,
2378 struct arm_smmu_domain *smmu_domain)
2379 {
2380 struct arm_smmu_cmdq_ent cmd = {
2381 .opcode = smmu_domain->smmu->features & ARM_SMMU_FEAT_E2H ?
2382 CMDQ_OP_TLBI_EL2_VA : CMDQ_OP_TLBI_NH_VA,
2383 .tlbi = {
2384 .asid = asid,
2385 .leaf = leaf,
2386 },
2387 };
2388
2389 __arm_smmu_tlb_inv_range(&cmd, iova, size, granule, smmu_domain);
2390 }
2391
arm_smmu_tlb_inv_page_nosync(struct iommu_iotlb_gather * gather,unsigned long iova,size_t granule,void * cookie)2392 static void arm_smmu_tlb_inv_page_nosync(struct iommu_iotlb_gather *gather,
2393 unsigned long iova, size_t granule,
2394 void *cookie)
2395 {
2396 struct arm_smmu_domain *smmu_domain = cookie;
2397 struct iommu_domain *domain = &smmu_domain->domain;
2398
2399 iommu_iotlb_gather_add_page(domain, gather, iova, granule);
2400 }
2401
arm_smmu_tlb_inv_walk(unsigned long iova,size_t size,size_t granule,void * cookie)2402 static void arm_smmu_tlb_inv_walk(unsigned long iova, size_t size,
2403 size_t granule, void *cookie)
2404 {
2405 arm_smmu_tlb_inv_range_domain(iova, size, granule, false, cookie);
2406 }
2407
2408 static const struct iommu_flush_ops arm_smmu_flush_ops = {
2409 .tlb_flush_all = arm_smmu_tlb_inv_context,
2410 .tlb_flush_walk = arm_smmu_tlb_inv_walk,
2411 .tlb_add_page = arm_smmu_tlb_inv_page_nosync,
2412 };
2413
arm_smmu_dbm_capable(struct arm_smmu_device * smmu)2414 static bool arm_smmu_dbm_capable(struct arm_smmu_device *smmu)
2415 {
2416 u32 features = (ARM_SMMU_FEAT_HD | ARM_SMMU_FEAT_COHERENCY);
2417
2418 return (smmu->features & features) == features;
2419 }
2420
2421 /* IOMMU API */
arm_smmu_capable(struct device * dev,enum iommu_cap cap)2422 static bool arm_smmu_capable(struct device *dev, enum iommu_cap cap)
2423 {
2424 struct arm_smmu_master *master = dev_iommu_priv_get(dev);
2425
2426 switch (cap) {
2427 case IOMMU_CAP_CACHE_COHERENCY:
2428 /* Assume that a coherent TCU implies coherent TBUs */
2429 return master->smmu->features & ARM_SMMU_FEAT_COHERENCY;
2430 case IOMMU_CAP_ENFORCE_CACHE_COHERENCY:
2431 return arm_smmu_master_canwbs(master);
2432 case IOMMU_CAP_NOEXEC:
2433 case IOMMU_CAP_DEFERRED_FLUSH:
2434 return true;
2435 case IOMMU_CAP_DIRTY_TRACKING:
2436 return arm_smmu_dbm_capable(master->smmu);
2437 default:
2438 return false;
2439 }
2440 }
2441
arm_smmu_enforce_cache_coherency(struct iommu_domain * domain)2442 static bool arm_smmu_enforce_cache_coherency(struct iommu_domain *domain)
2443 {
2444 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
2445 struct arm_smmu_master_domain *master_domain;
2446 unsigned long flags;
2447 bool ret = true;
2448
2449 spin_lock_irqsave(&smmu_domain->devices_lock, flags);
2450 list_for_each_entry(master_domain, &smmu_domain->devices,
2451 devices_elm) {
2452 if (!arm_smmu_master_canwbs(master_domain->master)) {
2453 ret = false;
2454 break;
2455 }
2456 }
2457 smmu_domain->enforce_cache_coherency = ret;
2458 spin_unlock_irqrestore(&smmu_domain->devices_lock, flags);
2459 return ret;
2460 }
2461
arm_smmu_domain_alloc(void)2462 struct arm_smmu_domain *arm_smmu_domain_alloc(void)
2463 {
2464 struct arm_smmu_domain *smmu_domain;
2465
2466 smmu_domain = kzalloc(sizeof(*smmu_domain), GFP_KERNEL);
2467 if (!smmu_domain)
2468 return ERR_PTR(-ENOMEM);
2469
2470 INIT_LIST_HEAD(&smmu_domain->devices);
2471 spin_lock_init(&smmu_domain->devices_lock);
2472
2473 return smmu_domain;
2474 }
2475
arm_smmu_domain_free_paging(struct iommu_domain * domain)2476 static void arm_smmu_domain_free_paging(struct iommu_domain *domain)
2477 {
2478 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
2479 struct arm_smmu_device *smmu = smmu_domain->smmu;
2480
2481 free_io_pgtable_ops(smmu_domain->pgtbl_ops);
2482
2483 /* Free the ASID or VMID */
2484 if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {
2485 /* Prevent SVA from touching the CD while we're freeing it */
2486 mutex_lock(&arm_smmu_asid_lock);
2487 xa_erase(&arm_smmu_asid_xa, smmu_domain->cd.asid);
2488 mutex_unlock(&arm_smmu_asid_lock);
2489 } else {
2490 struct arm_smmu_s2_cfg *cfg = &smmu_domain->s2_cfg;
2491 if (cfg->vmid)
2492 ida_free(&smmu->vmid_map, cfg->vmid);
2493 }
2494
2495 kfree(smmu_domain);
2496 }
2497
arm_smmu_domain_finalise_s1(struct arm_smmu_device * smmu,struct arm_smmu_domain * smmu_domain)2498 static int arm_smmu_domain_finalise_s1(struct arm_smmu_device *smmu,
2499 struct arm_smmu_domain *smmu_domain)
2500 {
2501 int ret;
2502 u32 asid = 0;
2503 struct arm_smmu_ctx_desc *cd = &smmu_domain->cd;
2504
2505 /* Prevent SVA from modifying the ASID until it is written to the CD */
2506 mutex_lock(&arm_smmu_asid_lock);
2507 ret = xa_alloc(&arm_smmu_asid_xa, &asid, smmu_domain,
2508 XA_LIMIT(1, (1 << smmu->asid_bits) - 1), GFP_KERNEL);
2509 cd->asid = (u16)asid;
2510 mutex_unlock(&arm_smmu_asid_lock);
2511 return ret;
2512 }
2513
arm_smmu_domain_finalise_s2(struct arm_smmu_device * smmu,struct arm_smmu_domain * smmu_domain)2514 static int arm_smmu_domain_finalise_s2(struct arm_smmu_device *smmu,
2515 struct arm_smmu_domain *smmu_domain)
2516 {
2517 int vmid;
2518 struct arm_smmu_s2_cfg *cfg = &smmu_domain->s2_cfg;
2519
2520 /* Reserve VMID 0 for stage-2 bypass STEs */
2521 vmid = ida_alloc_range(&smmu->vmid_map, 1, (1 << smmu->vmid_bits) - 1,
2522 GFP_KERNEL);
2523 if (vmid < 0)
2524 return vmid;
2525
2526 cfg->vmid = (u16)vmid;
2527 return 0;
2528 }
2529
arm_smmu_domain_finalise(struct arm_smmu_domain * smmu_domain,struct arm_smmu_device * smmu,u32 flags)2530 static int arm_smmu_domain_finalise(struct arm_smmu_domain *smmu_domain,
2531 struct arm_smmu_device *smmu, u32 flags)
2532 {
2533 int ret;
2534 enum io_pgtable_fmt fmt;
2535 struct io_pgtable_cfg pgtbl_cfg;
2536 struct io_pgtable_ops *pgtbl_ops;
2537 int (*finalise_stage_fn)(struct arm_smmu_device *smmu,
2538 struct arm_smmu_domain *smmu_domain);
2539 bool enable_dirty = flags & IOMMU_HWPT_ALLOC_DIRTY_TRACKING;
2540
2541 pgtbl_cfg = (struct io_pgtable_cfg) {
2542 .pgsize_bitmap = smmu->pgsize_bitmap,
2543 .coherent_walk = smmu->features & ARM_SMMU_FEAT_COHERENCY,
2544 .tlb = &arm_smmu_flush_ops,
2545 .iommu_dev = smmu->dev,
2546 };
2547
2548 switch (smmu_domain->stage) {
2549 case ARM_SMMU_DOMAIN_S1: {
2550 unsigned long ias = (smmu->features &
2551 ARM_SMMU_FEAT_VAX) ? 52 : 48;
2552
2553 pgtbl_cfg.ias = min_t(unsigned long, ias, VA_BITS);
2554 pgtbl_cfg.oas = smmu->ias;
2555 if (enable_dirty)
2556 pgtbl_cfg.quirks |= IO_PGTABLE_QUIRK_ARM_HD;
2557 fmt = ARM_64_LPAE_S1;
2558 finalise_stage_fn = arm_smmu_domain_finalise_s1;
2559 break;
2560 }
2561 case ARM_SMMU_DOMAIN_S2:
2562 if (enable_dirty)
2563 return -EOPNOTSUPP;
2564 pgtbl_cfg.ias = smmu->ias;
2565 pgtbl_cfg.oas = smmu->oas;
2566 fmt = ARM_64_LPAE_S2;
2567 finalise_stage_fn = arm_smmu_domain_finalise_s2;
2568 if ((smmu->features & ARM_SMMU_FEAT_S2FWB) &&
2569 (flags & IOMMU_HWPT_ALLOC_NEST_PARENT))
2570 pgtbl_cfg.quirks |= IO_PGTABLE_QUIRK_ARM_S2FWB;
2571 break;
2572 default:
2573 return -EINVAL;
2574 }
2575
2576 pgtbl_ops = alloc_io_pgtable_ops(fmt, &pgtbl_cfg, smmu_domain);
2577 if (!pgtbl_ops)
2578 return -ENOMEM;
2579
2580 smmu_domain->domain.pgsize_bitmap = pgtbl_cfg.pgsize_bitmap;
2581 smmu_domain->domain.geometry.aperture_end = (1UL << pgtbl_cfg.ias) - 1;
2582 smmu_domain->domain.geometry.force_aperture = true;
2583 if (enable_dirty && smmu_domain->stage == ARM_SMMU_DOMAIN_S1)
2584 smmu_domain->domain.dirty_ops = &arm_smmu_dirty_ops;
2585
2586 ret = finalise_stage_fn(smmu, smmu_domain);
2587 if (ret < 0) {
2588 free_io_pgtable_ops(pgtbl_ops);
2589 return ret;
2590 }
2591
2592 smmu_domain->pgtbl_ops = pgtbl_ops;
2593 smmu_domain->smmu = smmu;
2594 return 0;
2595 }
2596
2597 static struct arm_smmu_ste *
arm_smmu_get_step_for_sid(struct arm_smmu_device * smmu,u32 sid)2598 arm_smmu_get_step_for_sid(struct arm_smmu_device *smmu, u32 sid)
2599 {
2600 struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg;
2601
2602 if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB) {
2603 /* Two-level walk */
2604 return &cfg->l2.l2ptrs[arm_smmu_strtab_l1_idx(sid)]
2605 ->stes[arm_smmu_strtab_l2_idx(sid)];
2606 } else {
2607 /* Simple linear lookup */
2608 return &cfg->linear.table[sid];
2609 }
2610 }
2611
arm_smmu_install_ste_for_dev(struct arm_smmu_master * master,const struct arm_smmu_ste * target)2612 void arm_smmu_install_ste_for_dev(struct arm_smmu_master *master,
2613 const struct arm_smmu_ste *target)
2614 {
2615 int i, j;
2616 struct arm_smmu_device *smmu = master->smmu;
2617
2618 master->cd_table.in_ste =
2619 FIELD_GET(STRTAB_STE_0_CFG, le64_to_cpu(target->data[0])) ==
2620 STRTAB_STE_0_CFG_S1_TRANS;
2621 master->ste_ats_enabled =
2622 FIELD_GET(STRTAB_STE_1_EATS, le64_to_cpu(target->data[1])) ==
2623 STRTAB_STE_1_EATS_TRANS;
2624
2625 for (i = 0; i < master->num_streams; ++i) {
2626 u32 sid = master->streams[i].id;
2627 struct arm_smmu_ste *step =
2628 arm_smmu_get_step_for_sid(smmu, sid);
2629
2630 /* Bridged PCI devices may end up with duplicated IDs */
2631 for (j = 0; j < i; j++)
2632 if (master->streams[j].id == sid)
2633 break;
2634 if (j < i)
2635 continue;
2636
2637 arm_smmu_write_ste(master, sid, step, target);
2638 }
2639 }
2640
arm_smmu_ats_supported(struct arm_smmu_master * master)2641 static bool arm_smmu_ats_supported(struct arm_smmu_master *master)
2642 {
2643 struct device *dev = master->dev;
2644 struct arm_smmu_device *smmu = master->smmu;
2645 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
2646
2647 if (!(smmu->features & ARM_SMMU_FEAT_ATS))
2648 return false;
2649
2650 if (!(fwspec->flags & IOMMU_FWSPEC_PCI_RC_ATS))
2651 return false;
2652
2653 return dev_is_pci(dev) && pci_ats_supported(to_pci_dev(dev));
2654 }
2655
arm_smmu_enable_ats(struct arm_smmu_master * master)2656 static void arm_smmu_enable_ats(struct arm_smmu_master *master)
2657 {
2658 size_t stu;
2659 struct pci_dev *pdev;
2660 struct arm_smmu_device *smmu = master->smmu;
2661
2662 /* Smallest Translation Unit: log2 of the smallest supported granule */
2663 stu = __ffs(smmu->pgsize_bitmap);
2664 pdev = to_pci_dev(master->dev);
2665
2666 /*
2667 * ATC invalidation of PASID 0 causes the entire ATC to be flushed.
2668 */
2669 arm_smmu_atc_inv_master(master, IOMMU_NO_PASID);
2670 if (pci_enable_ats(pdev, stu))
2671 dev_err(master->dev, "Failed to enable ATS (STU %zu)\n", stu);
2672 }
2673
arm_smmu_enable_pasid(struct arm_smmu_master * master)2674 static int arm_smmu_enable_pasid(struct arm_smmu_master *master)
2675 {
2676 int ret;
2677 int features;
2678 int num_pasids;
2679 struct pci_dev *pdev;
2680
2681 if (!dev_is_pci(master->dev))
2682 return -ENODEV;
2683
2684 pdev = to_pci_dev(master->dev);
2685
2686 features = pci_pasid_features(pdev);
2687 if (features < 0)
2688 return features;
2689
2690 num_pasids = pci_max_pasids(pdev);
2691 if (num_pasids <= 0)
2692 return num_pasids;
2693
2694 ret = pci_enable_pasid(pdev, features);
2695 if (ret) {
2696 dev_err(&pdev->dev, "Failed to enable PASID\n");
2697 return ret;
2698 }
2699
2700 master->ssid_bits = min_t(u8, ilog2(num_pasids),
2701 master->smmu->ssid_bits);
2702 return 0;
2703 }
2704
arm_smmu_disable_pasid(struct arm_smmu_master * master)2705 static void arm_smmu_disable_pasid(struct arm_smmu_master *master)
2706 {
2707 struct pci_dev *pdev;
2708
2709 if (!dev_is_pci(master->dev))
2710 return;
2711
2712 pdev = to_pci_dev(master->dev);
2713
2714 if (!pdev->pasid_enabled)
2715 return;
2716
2717 master->ssid_bits = 0;
2718 pci_disable_pasid(pdev);
2719 }
2720
2721 static struct arm_smmu_master_domain *
arm_smmu_find_master_domain(struct arm_smmu_domain * smmu_domain,struct arm_smmu_master * master,ioasid_t ssid,bool nested_ats_flush)2722 arm_smmu_find_master_domain(struct arm_smmu_domain *smmu_domain,
2723 struct arm_smmu_master *master,
2724 ioasid_t ssid, bool nested_ats_flush)
2725 {
2726 struct arm_smmu_master_domain *master_domain;
2727
2728 lockdep_assert_held(&smmu_domain->devices_lock);
2729
2730 list_for_each_entry(master_domain, &smmu_domain->devices,
2731 devices_elm) {
2732 if (master_domain->master == master &&
2733 master_domain->ssid == ssid &&
2734 master_domain->nested_ats_flush == nested_ats_flush)
2735 return master_domain;
2736 }
2737 return NULL;
2738 }
2739
2740 /*
2741 * If the domain uses the smmu_domain->devices list return the arm_smmu_domain
2742 * structure, otherwise NULL. These domains track attached devices so they can
2743 * issue invalidations.
2744 */
2745 static struct arm_smmu_domain *
to_smmu_domain_devices(struct iommu_domain * domain)2746 to_smmu_domain_devices(struct iommu_domain *domain)
2747 {
2748 /* The domain can be NULL only when processing the first attach */
2749 if (!domain)
2750 return NULL;
2751 if ((domain->type & __IOMMU_DOMAIN_PAGING) ||
2752 domain->type == IOMMU_DOMAIN_SVA)
2753 return to_smmu_domain(domain);
2754 if (domain->type == IOMMU_DOMAIN_NESTED)
2755 return to_smmu_nested_domain(domain)->vsmmu->s2_parent;
2756 return NULL;
2757 }
2758
arm_smmu_remove_master_domain(struct arm_smmu_master * master,struct iommu_domain * domain,ioasid_t ssid)2759 static void arm_smmu_remove_master_domain(struct arm_smmu_master *master,
2760 struct iommu_domain *domain,
2761 ioasid_t ssid)
2762 {
2763 struct arm_smmu_domain *smmu_domain = to_smmu_domain_devices(domain);
2764 struct arm_smmu_master_domain *master_domain;
2765 bool nested_ats_flush = false;
2766 unsigned long flags;
2767
2768 if (!smmu_domain)
2769 return;
2770
2771 if (domain->type == IOMMU_DOMAIN_NESTED)
2772 nested_ats_flush = to_smmu_nested_domain(domain)->enable_ats;
2773
2774 spin_lock_irqsave(&smmu_domain->devices_lock, flags);
2775 master_domain = arm_smmu_find_master_domain(smmu_domain, master, ssid,
2776 nested_ats_flush);
2777 if (master_domain) {
2778 list_del(&master_domain->devices_elm);
2779 kfree(master_domain);
2780 if (master->ats_enabled)
2781 atomic_dec(&smmu_domain->nr_ats_masters);
2782 }
2783 spin_unlock_irqrestore(&smmu_domain->devices_lock, flags);
2784 }
2785
2786 /*
2787 * Start the sequence to attach a domain to a master. The sequence contains three
2788 * steps:
2789 * arm_smmu_attach_prepare()
2790 * arm_smmu_install_ste_for_dev()
2791 * arm_smmu_attach_commit()
2792 *
2793 * If prepare succeeds then the sequence must be completed. The STE installed
2794 * must set the STE.EATS field according to state.ats_enabled.
2795 *
2796 * If the device supports ATS then this determines if EATS should be enabled
2797 * in the STE, and starts sequencing EATS disable if required.
2798 *
2799 * The change of the EATS in the STE and the PCI ATS config space is managed by
2800 * this sequence to be in the right order so that if PCI ATS is enabled then
2801 * STE.ETAS is enabled.
2802 *
2803 * new_domain can be a non-paging domain. In this case ATS will not be enabled,
2804 * and invalidations won't be tracked.
2805 */
arm_smmu_attach_prepare(struct arm_smmu_attach_state * state,struct iommu_domain * new_domain)2806 int arm_smmu_attach_prepare(struct arm_smmu_attach_state *state,
2807 struct iommu_domain *new_domain)
2808 {
2809 struct arm_smmu_master *master = state->master;
2810 struct arm_smmu_master_domain *master_domain;
2811 struct arm_smmu_domain *smmu_domain =
2812 to_smmu_domain_devices(new_domain);
2813 unsigned long flags;
2814 int ret;
2815
2816 /*
2817 * arm_smmu_share_asid() must not see two domains pointing to the same
2818 * arm_smmu_master_domain contents otherwise it could randomly write one
2819 * or the other to the CD.
2820 */
2821 lockdep_assert_held(&arm_smmu_asid_lock);
2822
2823 if (smmu_domain || state->cd_needs_ats) {
2824 /*
2825 * The SMMU does not support enabling ATS with bypass/abort.
2826 * When the STE is in bypass (STE.Config[2:0] == 0b100), ATS
2827 * Translation Requests and Translated transactions are denied
2828 * as though ATS is disabled for the stream (STE.EATS == 0b00),
2829 * causing F_BAD_ATS_TREQ and F_TRANSL_FORBIDDEN events
2830 * (IHI0070Ea 5.2 Stream Table Entry).
2831 *
2832 * However, if we have installed a CD table and are using S1DSS
2833 * then ATS will work in S1DSS bypass. See "13.6.4 Full ATS
2834 * skipping stage 1".
2835 *
2836 * Disable ATS if we are going to create a normal 0b100 bypass
2837 * STE.
2838 */
2839 state->ats_enabled = !state->disable_ats &&
2840 arm_smmu_ats_supported(master);
2841 }
2842
2843 if (smmu_domain) {
2844 if (new_domain->type == IOMMU_DOMAIN_NESTED) {
2845 ret = arm_smmu_attach_prepare_vmaster(
2846 state, to_smmu_nested_domain(new_domain));
2847 if (ret)
2848 return ret;
2849 }
2850
2851 master_domain = kzalloc(sizeof(*master_domain), GFP_KERNEL);
2852 if (!master_domain) {
2853 kfree(state->vmaster);
2854 return -ENOMEM;
2855 }
2856 master_domain->master = master;
2857 master_domain->ssid = state->ssid;
2858 if (new_domain->type == IOMMU_DOMAIN_NESTED)
2859 master_domain->nested_ats_flush =
2860 to_smmu_nested_domain(new_domain)->enable_ats;
2861
2862 /*
2863 * During prepare we want the current smmu_domain and new
2864 * smmu_domain to be in the devices list before we change any
2865 * HW. This ensures that both domains will send ATS
2866 * invalidations to the master until we are done.
2867 *
2868 * It is tempting to make this list only track masters that are
2869 * using ATS, but arm_smmu_share_asid() also uses this to change
2870 * the ASID of a domain, unrelated to ATS.
2871 *
2872 * Notice if we are re-attaching the same domain then the list
2873 * will have two identical entries and commit will remove only
2874 * one of them.
2875 */
2876 spin_lock_irqsave(&smmu_domain->devices_lock, flags);
2877 if (smmu_domain->enforce_cache_coherency &&
2878 !arm_smmu_master_canwbs(master)) {
2879 spin_unlock_irqrestore(&smmu_domain->devices_lock,
2880 flags);
2881 kfree(master_domain);
2882 kfree(state->vmaster);
2883 return -EINVAL;
2884 }
2885
2886 if (state->ats_enabled)
2887 atomic_inc(&smmu_domain->nr_ats_masters);
2888 list_add(&master_domain->devices_elm, &smmu_domain->devices);
2889 spin_unlock_irqrestore(&smmu_domain->devices_lock, flags);
2890 }
2891
2892 if (!state->ats_enabled && master->ats_enabled) {
2893 pci_disable_ats(to_pci_dev(master->dev));
2894 /*
2895 * This is probably overkill, but the config write for disabling
2896 * ATS should complete before the STE is configured to generate
2897 * UR to avoid AER noise.
2898 */
2899 wmb();
2900 }
2901 return 0;
2902 }
2903
2904 /*
2905 * Commit is done after the STE/CD are configured with the EATS setting. It
2906 * completes synchronizing the PCI device's ATC and finishes manipulating the
2907 * smmu_domain->devices list.
2908 */
arm_smmu_attach_commit(struct arm_smmu_attach_state * state)2909 void arm_smmu_attach_commit(struct arm_smmu_attach_state *state)
2910 {
2911 struct arm_smmu_master *master = state->master;
2912
2913 lockdep_assert_held(&arm_smmu_asid_lock);
2914
2915 arm_smmu_attach_commit_vmaster(state);
2916
2917 if (state->ats_enabled && !master->ats_enabled) {
2918 arm_smmu_enable_ats(master);
2919 } else if (state->ats_enabled && master->ats_enabled) {
2920 /*
2921 * The translation has changed, flush the ATC. At this point the
2922 * SMMU is translating for the new domain and both the old&new
2923 * domain will issue invalidations.
2924 */
2925 arm_smmu_atc_inv_master(master, state->ssid);
2926 } else if (!state->ats_enabled && master->ats_enabled) {
2927 /* ATS is being switched off, invalidate the entire ATC */
2928 arm_smmu_atc_inv_master(master, IOMMU_NO_PASID);
2929 }
2930 master->ats_enabled = state->ats_enabled;
2931
2932 arm_smmu_remove_master_domain(master, state->old_domain, state->ssid);
2933 }
2934
arm_smmu_attach_dev(struct iommu_domain * domain,struct device * dev)2935 static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev)
2936 {
2937 int ret = 0;
2938 struct arm_smmu_ste target;
2939 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
2940 struct arm_smmu_device *smmu;
2941 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
2942 struct arm_smmu_attach_state state = {
2943 .old_domain = iommu_get_domain_for_dev(dev),
2944 .ssid = IOMMU_NO_PASID,
2945 };
2946 struct arm_smmu_master *master;
2947 struct arm_smmu_cd *cdptr;
2948
2949 if (!fwspec)
2950 return -ENOENT;
2951
2952 state.master = master = dev_iommu_priv_get(dev);
2953 smmu = master->smmu;
2954
2955 if (smmu_domain->smmu != smmu)
2956 return ret;
2957
2958 if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {
2959 cdptr = arm_smmu_alloc_cd_ptr(master, IOMMU_NO_PASID);
2960 if (!cdptr)
2961 return -ENOMEM;
2962 } else if (arm_smmu_ssids_in_use(&master->cd_table))
2963 return -EBUSY;
2964
2965 /*
2966 * Prevent arm_smmu_share_asid() from trying to change the ASID
2967 * of either the old or new domain while we are working on it.
2968 * This allows the STE and the smmu_domain->devices list to
2969 * be inconsistent during this routine.
2970 */
2971 mutex_lock(&arm_smmu_asid_lock);
2972
2973 ret = arm_smmu_attach_prepare(&state, domain);
2974 if (ret) {
2975 mutex_unlock(&arm_smmu_asid_lock);
2976 return ret;
2977 }
2978
2979 switch (smmu_domain->stage) {
2980 case ARM_SMMU_DOMAIN_S1: {
2981 struct arm_smmu_cd target_cd;
2982
2983 arm_smmu_make_s1_cd(&target_cd, master, smmu_domain);
2984 arm_smmu_write_cd_entry(master, IOMMU_NO_PASID, cdptr,
2985 &target_cd);
2986 arm_smmu_make_cdtable_ste(&target, master, state.ats_enabled,
2987 STRTAB_STE_1_S1DSS_SSID0);
2988 arm_smmu_install_ste_for_dev(master, &target);
2989 break;
2990 }
2991 case ARM_SMMU_DOMAIN_S2:
2992 arm_smmu_make_s2_domain_ste(&target, master, smmu_domain,
2993 state.ats_enabled);
2994 arm_smmu_install_ste_for_dev(master, &target);
2995 arm_smmu_clear_cd(master, IOMMU_NO_PASID);
2996 break;
2997 }
2998
2999 arm_smmu_attach_commit(&state);
3000 mutex_unlock(&arm_smmu_asid_lock);
3001 return 0;
3002 }
3003
arm_smmu_s1_set_dev_pasid(struct iommu_domain * domain,struct device * dev,ioasid_t id,struct iommu_domain * old)3004 static int arm_smmu_s1_set_dev_pasid(struct iommu_domain *domain,
3005 struct device *dev, ioasid_t id,
3006 struct iommu_domain *old)
3007 {
3008 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
3009 struct arm_smmu_master *master = dev_iommu_priv_get(dev);
3010 struct arm_smmu_device *smmu = master->smmu;
3011 struct arm_smmu_cd target_cd;
3012
3013 if (smmu_domain->smmu != smmu)
3014 return -EINVAL;
3015
3016 if (smmu_domain->stage != ARM_SMMU_DOMAIN_S1)
3017 return -EINVAL;
3018
3019 /*
3020 * We can read cd.asid outside the lock because arm_smmu_set_pasid()
3021 * will fix it
3022 */
3023 arm_smmu_make_s1_cd(&target_cd, master, smmu_domain);
3024 return arm_smmu_set_pasid(master, to_smmu_domain(domain), id,
3025 &target_cd, old);
3026 }
3027
arm_smmu_update_ste(struct arm_smmu_master * master,struct iommu_domain * sid_domain,bool ats_enabled)3028 static void arm_smmu_update_ste(struct arm_smmu_master *master,
3029 struct iommu_domain *sid_domain,
3030 bool ats_enabled)
3031 {
3032 unsigned int s1dss = STRTAB_STE_1_S1DSS_TERMINATE;
3033 struct arm_smmu_ste ste;
3034
3035 if (master->cd_table.in_ste && master->ste_ats_enabled == ats_enabled)
3036 return;
3037
3038 if (sid_domain->type == IOMMU_DOMAIN_IDENTITY)
3039 s1dss = STRTAB_STE_1_S1DSS_BYPASS;
3040 else
3041 WARN_ON(sid_domain->type != IOMMU_DOMAIN_BLOCKED);
3042
3043 /*
3044 * Change the STE into a cdtable one with SID IDENTITY/BLOCKED behavior
3045 * using s1dss if necessary. If the cd_table is already installed then
3046 * the S1DSS is correct and this will just update the EATS. Otherwise it
3047 * installs the entire thing. This will be hitless.
3048 */
3049 arm_smmu_make_cdtable_ste(&ste, master, ats_enabled, s1dss);
3050 arm_smmu_install_ste_for_dev(master, &ste);
3051 }
3052
arm_smmu_set_pasid(struct arm_smmu_master * master,struct arm_smmu_domain * smmu_domain,ioasid_t pasid,struct arm_smmu_cd * cd,struct iommu_domain * old)3053 int arm_smmu_set_pasid(struct arm_smmu_master *master,
3054 struct arm_smmu_domain *smmu_domain, ioasid_t pasid,
3055 struct arm_smmu_cd *cd, struct iommu_domain *old)
3056 {
3057 struct iommu_domain *sid_domain = iommu_get_domain_for_dev(master->dev);
3058 struct arm_smmu_attach_state state = {
3059 .master = master,
3060 .ssid = pasid,
3061 .old_domain = old,
3062 };
3063 struct arm_smmu_cd *cdptr;
3064 int ret;
3065
3066 /* The core code validates pasid */
3067
3068 if (smmu_domain->smmu != master->smmu)
3069 return -EINVAL;
3070
3071 if (!master->cd_table.in_ste &&
3072 sid_domain->type != IOMMU_DOMAIN_IDENTITY &&
3073 sid_domain->type != IOMMU_DOMAIN_BLOCKED)
3074 return -EINVAL;
3075
3076 cdptr = arm_smmu_alloc_cd_ptr(master, pasid);
3077 if (!cdptr)
3078 return -ENOMEM;
3079
3080 mutex_lock(&arm_smmu_asid_lock);
3081 ret = arm_smmu_attach_prepare(&state, &smmu_domain->domain);
3082 if (ret)
3083 goto out_unlock;
3084
3085 /*
3086 * We don't want to obtain to the asid_lock too early, so fix up the
3087 * caller set ASID under the lock in case it changed.
3088 */
3089 cd->data[0] &= ~cpu_to_le64(CTXDESC_CD_0_ASID);
3090 cd->data[0] |= cpu_to_le64(
3091 FIELD_PREP(CTXDESC_CD_0_ASID, smmu_domain->cd.asid));
3092
3093 arm_smmu_write_cd_entry(master, pasid, cdptr, cd);
3094 arm_smmu_update_ste(master, sid_domain, state.ats_enabled);
3095
3096 arm_smmu_attach_commit(&state);
3097
3098 out_unlock:
3099 mutex_unlock(&arm_smmu_asid_lock);
3100 return ret;
3101 }
3102
arm_smmu_blocking_set_dev_pasid(struct iommu_domain * new_domain,struct device * dev,ioasid_t pasid,struct iommu_domain * old_domain)3103 static int arm_smmu_blocking_set_dev_pasid(struct iommu_domain *new_domain,
3104 struct device *dev, ioasid_t pasid,
3105 struct iommu_domain *old_domain)
3106 {
3107 struct arm_smmu_domain *smmu_domain = to_smmu_domain(old_domain);
3108 struct arm_smmu_master *master = dev_iommu_priv_get(dev);
3109
3110 mutex_lock(&arm_smmu_asid_lock);
3111 arm_smmu_clear_cd(master, pasid);
3112 if (master->ats_enabled)
3113 arm_smmu_atc_inv_master(master, pasid);
3114 arm_smmu_remove_master_domain(master, &smmu_domain->domain, pasid);
3115 mutex_unlock(&arm_smmu_asid_lock);
3116
3117 /*
3118 * When the last user of the CD table goes away downgrade the STE back
3119 * to a non-cd_table one.
3120 */
3121 if (!arm_smmu_ssids_in_use(&master->cd_table)) {
3122 struct iommu_domain *sid_domain =
3123 iommu_get_domain_for_dev(master->dev);
3124
3125 if (sid_domain->type == IOMMU_DOMAIN_IDENTITY ||
3126 sid_domain->type == IOMMU_DOMAIN_BLOCKED)
3127 sid_domain->ops->attach_dev(sid_domain, dev);
3128 }
3129 return 0;
3130 }
3131
arm_smmu_attach_dev_ste(struct iommu_domain * domain,struct device * dev,struct arm_smmu_ste * ste,unsigned int s1dss)3132 static void arm_smmu_attach_dev_ste(struct iommu_domain *domain,
3133 struct device *dev,
3134 struct arm_smmu_ste *ste,
3135 unsigned int s1dss)
3136 {
3137 struct arm_smmu_master *master = dev_iommu_priv_get(dev);
3138 struct arm_smmu_attach_state state = {
3139 .master = master,
3140 .old_domain = iommu_get_domain_for_dev(dev),
3141 .ssid = IOMMU_NO_PASID,
3142 };
3143
3144 /*
3145 * Do not allow any ASID to be changed while are working on the STE,
3146 * otherwise we could miss invalidations.
3147 */
3148 mutex_lock(&arm_smmu_asid_lock);
3149
3150 /*
3151 * If the CD table is not in use we can use the provided STE, otherwise
3152 * we use a cdtable STE with the provided S1DSS.
3153 */
3154 if (arm_smmu_ssids_in_use(&master->cd_table)) {
3155 /*
3156 * If a CD table has to be present then we need to run with ATS
3157 * on because we have to assume a PASID is using ATS. For
3158 * IDENTITY this will setup things so that S1DSS=bypass which
3159 * follows the explanation in "13.6.4 Full ATS skipping stage 1"
3160 * and allows for ATS on the RID to work.
3161 */
3162 state.cd_needs_ats = true;
3163 arm_smmu_attach_prepare(&state, domain);
3164 arm_smmu_make_cdtable_ste(ste, master, state.ats_enabled, s1dss);
3165 } else {
3166 arm_smmu_attach_prepare(&state, domain);
3167 }
3168 arm_smmu_install_ste_for_dev(master, ste);
3169 arm_smmu_attach_commit(&state);
3170 mutex_unlock(&arm_smmu_asid_lock);
3171
3172 /*
3173 * This has to be done after removing the master from the
3174 * arm_smmu_domain->devices to avoid races updating the same context
3175 * descriptor from arm_smmu_share_asid().
3176 */
3177 arm_smmu_clear_cd(master, IOMMU_NO_PASID);
3178 }
3179
arm_smmu_attach_dev_identity(struct iommu_domain * domain,struct device * dev)3180 static int arm_smmu_attach_dev_identity(struct iommu_domain *domain,
3181 struct device *dev)
3182 {
3183 struct arm_smmu_ste ste;
3184 struct arm_smmu_master *master = dev_iommu_priv_get(dev);
3185
3186 arm_smmu_master_clear_vmaster(master);
3187 arm_smmu_make_bypass_ste(master->smmu, &ste);
3188 arm_smmu_attach_dev_ste(domain, dev, &ste, STRTAB_STE_1_S1DSS_BYPASS);
3189 return 0;
3190 }
3191
3192 static const struct iommu_domain_ops arm_smmu_identity_ops = {
3193 .attach_dev = arm_smmu_attach_dev_identity,
3194 };
3195
3196 static struct iommu_domain arm_smmu_identity_domain = {
3197 .type = IOMMU_DOMAIN_IDENTITY,
3198 .ops = &arm_smmu_identity_ops,
3199 };
3200
arm_smmu_attach_dev_blocked(struct iommu_domain * domain,struct device * dev)3201 static int arm_smmu_attach_dev_blocked(struct iommu_domain *domain,
3202 struct device *dev)
3203 {
3204 struct arm_smmu_ste ste;
3205 struct arm_smmu_master *master = dev_iommu_priv_get(dev);
3206
3207 arm_smmu_master_clear_vmaster(master);
3208 arm_smmu_make_abort_ste(&ste);
3209 arm_smmu_attach_dev_ste(domain, dev, &ste,
3210 STRTAB_STE_1_S1DSS_TERMINATE);
3211 return 0;
3212 }
3213
3214 static const struct iommu_domain_ops arm_smmu_blocked_ops = {
3215 .attach_dev = arm_smmu_attach_dev_blocked,
3216 .set_dev_pasid = arm_smmu_blocking_set_dev_pasid,
3217 };
3218
3219 static struct iommu_domain arm_smmu_blocked_domain = {
3220 .type = IOMMU_DOMAIN_BLOCKED,
3221 .ops = &arm_smmu_blocked_ops,
3222 };
3223
3224 static struct iommu_domain *
arm_smmu_domain_alloc_paging_flags(struct device * dev,u32 flags,const struct iommu_user_data * user_data)3225 arm_smmu_domain_alloc_paging_flags(struct device *dev, u32 flags,
3226 const struct iommu_user_data *user_data)
3227 {
3228 struct arm_smmu_master *master = dev_iommu_priv_get(dev);
3229 struct arm_smmu_device *smmu = master->smmu;
3230 const u32 PAGING_FLAGS = IOMMU_HWPT_ALLOC_DIRTY_TRACKING |
3231 IOMMU_HWPT_ALLOC_PASID |
3232 IOMMU_HWPT_ALLOC_NEST_PARENT;
3233 struct arm_smmu_domain *smmu_domain;
3234 int ret;
3235
3236 if (flags & ~PAGING_FLAGS)
3237 return ERR_PTR(-EOPNOTSUPP);
3238 if (user_data)
3239 return ERR_PTR(-EOPNOTSUPP);
3240
3241 smmu_domain = arm_smmu_domain_alloc();
3242 if (IS_ERR(smmu_domain))
3243 return ERR_CAST(smmu_domain);
3244
3245 switch (flags) {
3246 case 0:
3247 /* Prefer S1 if available */
3248 if (smmu->features & ARM_SMMU_FEAT_TRANS_S1)
3249 smmu_domain->stage = ARM_SMMU_DOMAIN_S1;
3250 else
3251 smmu_domain->stage = ARM_SMMU_DOMAIN_S2;
3252 break;
3253 case IOMMU_HWPT_ALLOC_NEST_PARENT:
3254 if (!(smmu->features & ARM_SMMU_FEAT_NESTING)) {
3255 ret = -EOPNOTSUPP;
3256 goto err_free;
3257 }
3258 smmu_domain->stage = ARM_SMMU_DOMAIN_S2;
3259 smmu_domain->nest_parent = true;
3260 break;
3261 case IOMMU_HWPT_ALLOC_DIRTY_TRACKING:
3262 case IOMMU_HWPT_ALLOC_DIRTY_TRACKING | IOMMU_HWPT_ALLOC_PASID:
3263 case IOMMU_HWPT_ALLOC_PASID:
3264 if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S1)) {
3265 ret = -EOPNOTSUPP;
3266 goto err_free;
3267 }
3268 smmu_domain->stage = ARM_SMMU_DOMAIN_S1;
3269 break;
3270 default:
3271 ret = -EOPNOTSUPP;
3272 goto err_free;
3273 }
3274
3275 smmu_domain->domain.type = IOMMU_DOMAIN_UNMANAGED;
3276 smmu_domain->domain.ops = arm_smmu_ops.default_domain_ops;
3277 ret = arm_smmu_domain_finalise(smmu_domain, smmu, flags);
3278 if (ret)
3279 goto err_free;
3280 return &smmu_domain->domain;
3281
3282 err_free:
3283 kfree(smmu_domain);
3284 return ERR_PTR(ret);
3285 }
3286
arm_smmu_map_pages(struct iommu_domain * domain,unsigned long iova,phys_addr_t paddr,size_t pgsize,size_t pgcount,int prot,gfp_t gfp,size_t * mapped)3287 static int arm_smmu_map_pages(struct iommu_domain *domain, unsigned long iova,
3288 phys_addr_t paddr, size_t pgsize, size_t pgcount,
3289 int prot, gfp_t gfp, size_t *mapped)
3290 {
3291 struct io_pgtable_ops *ops = to_smmu_domain(domain)->pgtbl_ops;
3292
3293 if (!ops)
3294 return -ENODEV;
3295
3296 return ops->map_pages(ops, iova, paddr, pgsize, pgcount, prot, gfp, mapped);
3297 }
3298
arm_smmu_unmap_pages(struct iommu_domain * domain,unsigned long iova,size_t pgsize,size_t pgcount,struct iommu_iotlb_gather * gather)3299 static size_t arm_smmu_unmap_pages(struct iommu_domain *domain, unsigned long iova,
3300 size_t pgsize, size_t pgcount,
3301 struct iommu_iotlb_gather *gather)
3302 {
3303 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
3304 struct io_pgtable_ops *ops = smmu_domain->pgtbl_ops;
3305
3306 if (!ops)
3307 return 0;
3308
3309 return ops->unmap_pages(ops, iova, pgsize, pgcount, gather);
3310 }
3311
arm_smmu_flush_iotlb_all(struct iommu_domain * domain)3312 static void arm_smmu_flush_iotlb_all(struct iommu_domain *domain)
3313 {
3314 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
3315
3316 if (smmu_domain->smmu)
3317 arm_smmu_tlb_inv_context(smmu_domain);
3318 }
3319
arm_smmu_iotlb_sync(struct iommu_domain * domain,struct iommu_iotlb_gather * gather)3320 static void arm_smmu_iotlb_sync(struct iommu_domain *domain,
3321 struct iommu_iotlb_gather *gather)
3322 {
3323 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
3324
3325 if (!gather->pgsize)
3326 return;
3327
3328 arm_smmu_tlb_inv_range_domain(gather->start,
3329 gather->end - gather->start + 1,
3330 gather->pgsize, true, smmu_domain);
3331 }
3332
3333 static phys_addr_t
arm_smmu_iova_to_phys(struct iommu_domain * domain,dma_addr_t iova)3334 arm_smmu_iova_to_phys(struct iommu_domain *domain, dma_addr_t iova)
3335 {
3336 struct io_pgtable_ops *ops = to_smmu_domain(domain)->pgtbl_ops;
3337
3338 if (!ops)
3339 return 0;
3340
3341 return ops->iova_to_phys(ops, iova);
3342 }
3343
3344 static struct platform_driver arm_smmu_driver;
3345
3346 static
arm_smmu_get_by_fwnode(struct fwnode_handle * fwnode)3347 struct arm_smmu_device *arm_smmu_get_by_fwnode(struct fwnode_handle *fwnode)
3348 {
3349 struct device *dev = bus_find_device_by_fwnode(&platform_bus_type, fwnode);
3350
3351 put_device(dev);
3352 return dev ? dev_get_drvdata(dev) : NULL;
3353 }
3354
arm_smmu_sid_in_range(struct arm_smmu_device * smmu,u32 sid)3355 static bool arm_smmu_sid_in_range(struct arm_smmu_device *smmu, u32 sid)
3356 {
3357 if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB)
3358 return arm_smmu_strtab_l1_idx(sid) < smmu->strtab_cfg.l2.num_l1_ents;
3359 return sid < smmu->strtab_cfg.linear.num_ents;
3360 }
3361
arm_smmu_init_sid_strtab(struct arm_smmu_device * smmu,u32 sid)3362 static int arm_smmu_init_sid_strtab(struct arm_smmu_device *smmu, u32 sid)
3363 {
3364 /* Check the SIDs are in range of the SMMU and our stream table */
3365 if (!arm_smmu_sid_in_range(smmu, sid))
3366 return -ERANGE;
3367
3368 /* Ensure l2 strtab is initialised */
3369 if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB)
3370 return arm_smmu_init_l2_strtab(smmu, sid);
3371
3372 return 0;
3373 }
3374
arm_smmu_insert_master(struct arm_smmu_device * smmu,struct arm_smmu_master * master)3375 static int arm_smmu_insert_master(struct arm_smmu_device *smmu,
3376 struct arm_smmu_master *master)
3377 {
3378 int i;
3379 int ret = 0;
3380 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(master->dev);
3381
3382 master->streams = kcalloc(fwspec->num_ids, sizeof(*master->streams),
3383 GFP_KERNEL);
3384 if (!master->streams)
3385 return -ENOMEM;
3386 master->num_streams = fwspec->num_ids;
3387
3388 mutex_lock(&smmu->streams_mutex);
3389 for (i = 0; i < fwspec->num_ids; i++) {
3390 struct arm_smmu_stream *new_stream = &master->streams[i];
3391 struct rb_node *existing;
3392 u32 sid = fwspec->ids[i];
3393
3394 new_stream->id = sid;
3395 new_stream->master = master;
3396
3397 ret = arm_smmu_init_sid_strtab(smmu, sid);
3398 if (ret)
3399 break;
3400
3401 /* Insert into SID tree */
3402 existing = rb_find_add(&new_stream->node, &smmu->streams,
3403 arm_smmu_streams_cmp_node);
3404 if (existing) {
3405 struct arm_smmu_master *existing_master =
3406 rb_entry(existing, struct arm_smmu_stream, node)
3407 ->master;
3408
3409 /* Bridged PCI devices may end up with duplicated IDs */
3410 if (existing_master == master)
3411 continue;
3412
3413 dev_warn(master->dev,
3414 "Aliasing StreamID 0x%x (from %s) unsupported, expect DMA to be broken\n",
3415 sid, dev_name(existing_master->dev));
3416 ret = -ENODEV;
3417 break;
3418 }
3419 }
3420
3421 if (ret) {
3422 for (i--; i >= 0; i--)
3423 rb_erase(&master->streams[i].node, &smmu->streams);
3424 kfree(master->streams);
3425 }
3426 mutex_unlock(&smmu->streams_mutex);
3427
3428 return ret;
3429 }
3430
arm_smmu_remove_master(struct arm_smmu_master * master)3431 static void arm_smmu_remove_master(struct arm_smmu_master *master)
3432 {
3433 int i;
3434 struct arm_smmu_device *smmu = master->smmu;
3435 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(master->dev);
3436
3437 if (!smmu || !master->streams)
3438 return;
3439
3440 mutex_lock(&smmu->streams_mutex);
3441 for (i = 0; i < fwspec->num_ids; i++)
3442 rb_erase(&master->streams[i].node, &smmu->streams);
3443 mutex_unlock(&smmu->streams_mutex);
3444
3445 kfree(master->streams);
3446 }
3447
arm_smmu_probe_device(struct device * dev)3448 static struct iommu_device *arm_smmu_probe_device(struct device *dev)
3449 {
3450 int ret;
3451 struct arm_smmu_device *smmu;
3452 struct arm_smmu_master *master;
3453 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
3454
3455 if (WARN_ON_ONCE(dev_iommu_priv_get(dev)))
3456 return ERR_PTR(-EBUSY);
3457
3458 smmu = arm_smmu_get_by_fwnode(fwspec->iommu_fwnode);
3459 if (!smmu)
3460 return ERR_PTR(-ENODEV);
3461
3462 master = kzalloc(sizeof(*master), GFP_KERNEL);
3463 if (!master)
3464 return ERR_PTR(-ENOMEM);
3465
3466 master->dev = dev;
3467 master->smmu = smmu;
3468 dev_iommu_priv_set(dev, master);
3469
3470 ret = arm_smmu_insert_master(smmu, master);
3471 if (ret)
3472 goto err_free_master;
3473
3474 device_property_read_u32(dev, "pasid-num-bits", &master->ssid_bits);
3475 master->ssid_bits = min(smmu->ssid_bits, master->ssid_bits);
3476
3477 /*
3478 * Note that PASID must be enabled before, and disabled after ATS:
3479 * PCI Express Base 4.0r1.0 - 10.5.1.3 ATS Control Register
3480 *
3481 * Behavior is undefined if this bit is Set and the value of the PASID
3482 * Enable, Execute Requested Enable, or Privileged Mode Requested bits
3483 * are changed.
3484 */
3485 arm_smmu_enable_pasid(master);
3486
3487 if (!(smmu->features & ARM_SMMU_FEAT_2_LVL_CDTAB))
3488 master->ssid_bits = min_t(u8, master->ssid_bits,
3489 CTXDESC_LINEAR_CDMAX);
3490
3491 if ((smmu->features & ARM_SMMU_FEAT_STALLS &&
3492 device_property_read_bool(dev, "dma-can-stall")) ||
3493 smmu->features & ARM_SMMU_FEAT_STALL_FORCE)
3494 master->stall_enabled = true;
3495
3496 if (dev_is_pci(dev)) {
3497 unsigned int stu = __ffs(smmu->pgsize_bitmap);
3498
3499 pci_prepare_ats(to_pci_dev(dev), stu);
3500 }
3501
3502 return &smmu->iommu;
3503
3504 err_free_master:
3505 kfree(master);
3506 return ERR_PTR(ret);
3507 }
3508
arm_smmu_release_device(struct device * dev)3509 static void arm_smmu_release_device(struct device *dev)
3510 {
3511 struct arm_smmu_master *master = dev_iommu_priv_get(dev);
3512
3513 if (WARN_ON(arm_smmu_master_sva_enabled(master)))
3514 iopf_queue_remove_device(master->smmu->evtq.iopf, dev);
3515
3516 /* Put the STE back to what arm_smmu_init_strtab() sets */
3517 if (dev->iommu->require_direct)
3518 arm_smmu_attach_dev_identity(&arm_smmu_identity_domain, dev);
3519 else
3520 arm_smmu_attach_dev_blocked(&arm_smmu_blocked_domain, dev);
3521
3522 arm_smmu_disable_pasid(master);
3523 arm_smmu_remove_master(master);
3524 if (arm_smmu_cdtab_allocated(&master->cd_table))
3525 arm_smmu_free_cd_tables(master);
3526 kfree(master);
3527 }
3528
arm_smmu_read_and_clear_dirty(struct iommu_domain * domain,unsigned long iova,size_t size,unsigned long flags,struct iommu_dirty_bitmap * dirty)3529 static int arm_smmu_read_and_clear_dirty(struct iommu_domain *domain,
3530 unsigned long iova, size_t size,
3531 unsigned long flags,
3532 struct iommu_dirty_bitmap *dirty)
3533 {
3534 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
3535 struct io_pgtable_ops *ops = smmu_domain->pgtbl_ops;
3536
3537 return ops->read_and_clear_dirty(ops, iova, size, flags, dirty);
3538 }
3539
arm_smmu_set_dirty_tracking(struct iommu_domain * domain,bool enabled)3540 static int arm_smmu_set_dirty_tracking(struct iommu_domain *domain,
3541 bool enabled)
3542 {
3543 /*
3544 * Always enabled and the dirty bitmap is cleared prior to
3545 * set_dirty_tracking().
3546 */
3547 return 0;
3548 }
3549
arm_smmu_device_group(struct device * dev)3550 static struct iommu_group *arm_smmu_device_group(struct device *dev)
3551 {
3552 struct iommu_group *group;
3553
3554 /*
3555 * We don't support devices sharing stream IDs other than PCI RID
3556 * aliases, since the necessary ID-to-device lookup becomes rather
3557 * impractical given a potential sparse 32-bit stream ID space.
3558 */
3559 if (dev_is_pci(dev))
3560 group = pci_device_group(dev);
3561 else
3562 group = generic_device_group(dev);
3563
3564 return group;
3565 }
3566
arm_smmu_of_xlate(struct device * dev,const struct of_phandle_args * args)3567 static int arm_smmu_of_xlate(struct device *dev,
3568 const struct of_phandle_args *args)
3569 {
3570 return iommu_fwspec_add_ids(dev, args->args, 1);
3571 }
3572
arm_smmu_get_resv_regions(struct device * dev,struct list_head * head)3573 static void arm_smmu_get_resv_regions(struct device *dev,
3574 struct list_head *head)
3575 {
3576 struct iommu_resv_region *region;
3577 int prot = IOMMU_WRITE | IOMMU_NOEXEC | IOMMU_MMIO;
3578
3579 region = iommu_alloc_resv_region(MSI_IOVA_BASE, MSI_IOVA_LENGTH,
3580 prot, IOMMU_RESV_SW_MSI, GFP_KERNEL);
3581 if (!region)
3582 return;
3583
3584 list_add_tail(®ion->list, head);
3585
3586 iommu_dma_get_resv_regions(dev, head);
3587 }
3588
arm_smmu_dev_enable_feature(struct device * dev,enum iommu_dev_features feat)3589 static int arm_smmu_dev_enable_feature(struct device *dev,
3590 enum iommu_dev_features feat)
3591 {
3592 struct arm_smmu_master *master = dev_iommu_priv_get(dev);
3593
3594 if (!master)
3595 return -ENODEV;
3596
3597 switch (feat) {
3598 case IOMMU_DEV_FEAT_IOPF:
3599 if (!arm_smmu_master_iopf_supported(master))
3600 return -EINVAL;
3601 if (master->iopf_enabled)
3602 return -EBUSY;
3603 master->iopf_enabled = true;
3604 return 0;
3605 case IOMMU_DEV_FEAT_SVA:
3606 if (!arm_smmu_master_sva_supported(master))
3607 return -EINVAL;
3608 if (arm_smmu_master_sva_enabled(master))
3609 return -EBUSY;
3610 return arm_smmu_master_enable_sva(master);
3611 default:
3612 return -EINVAL;
3613 }
3614 }
3615
arm_smmu_dev_disable_feature(struct device * dev,enum iommu_dev_features feat)3616 static int arm_smmu_dev_disable_feature(struct device *dev,
3617 enum iommu_dev_features feat)
3618 {
3619 struct arm_smmu_master *master = dev_iommu_priv_get(dev);
3620
3621 if (!master)
3622 return -EINVAL;
3623
3624 switch (feat) {
3625 case IOMMU_DEV_FEAT_IOPF:
3626 if (!master->iopf_enabled)
3627 return -EINVAL;
3628 if (master->sva_enabled)
3629 return -EBUSY;
3630 master->iopf_enabled = false;
3631 return 0;
3632 case IOMMU_DEV_FEAT_SVA:
3633 if (!arm_smmu_master_sva_enabled(master))
3634 return -EINVAL;
3635 return arm_smmu_master_disable_sva(master);
3636 default:
3637 return -EINVAL;
3638 }
3639 }
3640
3641 /*
3642 * HiSilicon PCIe tune and trace device can be used to trace TLP headers on the
3643 * PCIe link and save the data to memory by DMA. The hardware is restricted to
3644 * use identity mapping only.
3645 */
3646 #define IS_HISI_PTT_DEVICE(pdev) ((pdev)->vendor == PCI_VENDOR_ID_HUAWEI && \
3647 (pdev)->device == 0xa12e)
3648
arm_smmu_def_domain_type(struct device * dev)3649 static int arm_smmu_def_domain_type(struct device *dev)
3650 {
3651 if (dev_is_pci(dev)) {
3652 struct pci_dev *pdev = to_pci_dev(dev);
3653
3654 if (IS_HISI_PTT_DEVICE(pdev))
3655 return IOMMU_DOMAIN_IDENTITY;
3656 }
3657
3658 return 0;
3659 }
3660
3661 static struct iommu_ops arm_smmu_ops = {
3662 .identity_domain = &arm_smmu_identity_domain,
3663 .blocked_domain = &arm_smmu_blocked_domain,
3664 .capable = arm_smmu_capable,
3665 .hw_info = arm_smmu_hw_info,
3666 .domain_alloc_sva = arm_smmu_sva_domain_alloc,
3667 .domain_alloc_paging_flags = arm_smmu_domain_alloc_paging_flags,
3668 .probe_device = arm_smmu_probe_device,
3669 .release_device = arm_smmu_release_device,
3670 .device_group = arm_smmu_device_group,
3671 .of_xlate = arm_smmu_of_xlate,
3672 .get_resv_regions = arm_smmu_get_resv_regions,
3673 .dev_enable_feat = arm_smmu_dev_enable_feature,
3674 .dev_disable_feat = arm_smmu_dev_disable_feature,
3675 .page_response = arm_smmu_page_response,
3676 .def_domain_type = arm_smmu_def_domain_type,
3677 .viommu_alloc = arm_vsmmu_alloc,
3678 .user_pasid_table = 1,
3679 .pgsize_bitmap = -1UL, /* Restricted during device attach */
3680 .owner = THIS_MODULE,
3681 .default_domain_ops = &(const struct iommu_domain_ops) {
3682 .attach_dev = arm_smmu_attach_dev,
3683 .enforce_cache_coherency = arm_smmu_enforce_cache_coherency,
3684 .set_dev_pasid = arm_smmu_s1_set_dev_pasid,
3685 .map_pages = arm_smmu_map_pages,
3686 .unmap_pages = arm_smmu_unmap_pages,
3687 .flush_iotlb_all = arm_smmu_flush_iotlb_all,
3688 .iotlb_sync = arm_smmu_iotlb_sync,
3689 .iova_to_phys = arm_smmu_iova_to_phys,
3690 .free = arm_smmu_domain_free_paging,
3691 }
3692 };
3693
3694 static struct iommu_dirty_ops arm_smmu_dirty_ops = {
3695 .read_and_clear_dirty = arm_smmu_read_and_clear_dirty,
3696 .set_dirty_tracking = arm_smmu_set_dirty_tracking,
3697 };
3698
3699 /* Probing and initialisation functions */
arm_smmu_init_one_queue(struct arm_smmu_device * smmu,struct arm_smmu_queue * q,void __iomem * page,unsigned long prod_off,unsigned long cons_off,size_t dwords,const char * name)3700 int arm_smmu_init_one_queue(struct arm_smmu_device *smmu,
3701 struct arm_smmu_queue *q, void __iomem *page,
3702 unsigned long prod_off, unsigned long cons_off,
3703 size_t dwords, const char *name)
3704 {
3705 size_t qsz;
3706
3707 do {
3708 qsz = ((1 << q->llq.max_n_shift) * dwords) << 3;
3709 q->base = dmam_alloc_coherent(smmu->dev, qsz, &q->base_dma,
3710 GFP_KERNEL);
3711 if (q->base || qsz < PAGE_SIZE)
3712 break;
3713
3714 q->llq.max_n_shift--;
3715 } while (1);
3716
3717 if (!q->base) {
3718 dev_err(smmu->dev,
3719 "failed to allocate queue (0x%zx bytes) for %s\n",
3720 qsz, name);
3721 return -ENOMEM;
3722 }
3723
3724 if (!WARN_ON(q->base_dma & (qsz - 1))) {
3725 dev_info(smmu->dev, "allocated %u entries for %s\n",
3726 1 << q->llq.max_n_shift, name);
3727 }
3728
3729 q->prod_reg = page + prod_off;
3730 q->cons_reg = page + cons_off;
3731 q->ent_dwords = dwords;
3732
3733 q->q_base = Q_BASE_RWA;
3734 q->q_base |= q->base_dma & Q_BASE_ADDR_MASK;
3735 q->q_base |= FIELD_PREP(Q_BASE_LOG2SIZE, q->llq.max_n_shift);
3736
3737 q->llq.prod = q->llq.cons = 0;
3738 return 0;
3739 }
3740
arm_smmu_cmdq_init(struct arm_smmu_device * smmu,struct arm_smmu_cmdq * cmdq)3741 int arm_smmu_cmdq_init(struct arm_smmu_device *smmu,
3742 struct arm_smmu_cmdq *cmdq)
3743 {
3744 unsigned int nents = 1 << cmdq->q.llq.max_n_shift;
3745
3746 atomic_set(&cmdq->owner_prod, 0);
3747 atomic_set(&cmdq->lock, 0);
3748
3749 cmdq->valid_map = (atomic_long_t *)devm_bitmap_zalloc(smmu->dev, nents,
3750 GFP_KERNEL);
3751 if (!cmdq->valid_map)
3752 return -ENOMEM;
3753
3754 return 0;
3755 }
3756
arm_smmu_init_queues(struct arm_smmu_device * smmu)3757 static int arm_smmu_init_queues(struct arm_smmu_device *smmu)
3758 {
3759 int ret;
3760
3761 /* cmdq */
3762 ret = arm_smmu_init_one_queue(smmu, &smmu->cmdq.q, smmu->base,
3763 ARM_SMMU_CMDQ_PROD, ARM_SMMU_CMDQ_CONS,
3764 CMDQ_ENT_DWORDS, "cmdq");
3765 if (ret)
3766 return ret;
3767
3768 ret = arm_smmu_cmdq_init(smmu, &smmu->cmdq);
3769 if (ret)
3770 return ret;
3771
3772 /* evtq */
3773 ret = arm_smmu_init_one_queue(smmu, &smmu->evtq.q, smmu->page1,
3774 ARM_SMMU_EVTQ_PROD, ARM_SMMU_EVTQ_CONS,
3775 EVTQ_ENT_DWORDS, "evtq");
3776 if (ret)
3777 return ret;
3778
3779 if ((smmu->features & ARM_SMMU_FEAT_SVA) &&
3780 (smmu->features & ARM_SMMU_FEAT_STALLS)) {
3781 smmu->evtq.iopf = iopf_queue_alloc(dev_name(smmu->dev));
3782 if (!smmu->evtq.iopf)
3783 return -ENOMEM;
3784 }
3785
3786 /* priq */
3787 if (!(smmu->features & ARM_SMMU_FEAT_PRI))
3788 return 0;
3789
3790 return arm_smmu_init_one_queue(smmu, &smmu->priq.q, smmu->page1,
3791 ARM_SMMU_PRIQ_PROD, ARM_SMMU_PRIQ_CONS,
3792 PRIQ_ENT_DWORDS, "priq");
3793 }
3794
arm_smmu_init_strtab_2lvl(struct arm_smmu_device * smmu)3795 static int arm_smmu_init_strtab_2lvl(struct arm_smmu_device *smmu)
3796 {
3797 u32 l1size;
3798 struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg;
3799 unsigned int last_sid_idx =
3800 arm_smmu_strtab_l1_idx((1ULL << smmu->sid_bits) - 1);
3801
3802 /* Calculate the L1 size, capped to the SIDSIZE. */
3803 cfg->l2.num_l1_ents = min(last_sid_idx + 1, STRTAB_MAX_L1_ENTRIES);
3804 if (cfg->l2.num_l1_ents <= last_sid_idx)
3805 dev_warn(smmu->dev,
3806 "2-level strtab only covers %u/%u bits of SID\n",
3807 ilog2(cfg->l2.num_l1_ents * STRTAB_NUM_L2_STES),
3808 smmu->sid_bits);
3809
3810 l1size = cfg->l2.num_l1_ents * sizeof(struct arm_smmu_strtab_l1);
3811 cfg->l2.l1tab = dmam_alloc_coherent(smmu->dev, l1size, &cfg->l2.l1_dma,
3812 GFP_KERNEL);
3813 if (!cfg->l2.l1tab) {
3814 dev_err(smmu->dev,
3815 "failed to allocate l1 stream table (%u bytes)\n",
3816 l1size);
3817 return -ENOMEM;
3818 }
3819
3820 cfg->l2.l2ptrs = devm_kcalloc(smmu->dev, cfg->l2.num_l1_ents,
3821 sizeof(*cfg->l2.l2ptrs), GFP_KERNEL);
3822 if (!cfg->l2.l2ptrs)
3823 return -ENOMEM;
3824
3825 return 0;
3826 }
3827
arm_smmu_init_strtab_linear(struct arm_smmu_device * smmu)3828 static int arm_smmu_init_strtab_linear(struct arm_smmu_device *smmu)
3829 {
3830 u32 size;
3831 struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg;
3832
3833 size = (1 << smmu->sid_bits) * sizeof(struct arm_smmu_ste);
3834 cfg->linear.table = dmam_alloc_coherent(smmu->dev, size,
3835 &cfg->linear.ste_dma,
3836 GFP_KERNEL);
3837 if (!cfg->linear.table) {
3838 dev_err(smmu->dev,
3839 "failed to allocate linear stream table (%u bytes)\n",
3840 size);
3841 return -ENOMEM;
3842 }
3843 cfg->linear.num_ents = 1 << smmu->sid_bits;
3844
3845 arm_smmu_init_initial_stes(cfg->linear.table, cfg->linear.num_ents);
3846 return 0;
3847 }
3848
arm_smmu_init_strtab(struct arm_smmu_device * smmu)3849 static int arm_smmu_init_strtab(struct arm_smmu_device *smmu)
3850 {
3851 int ret;
3852
3853 if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB)
3854 ret = arm_smmu_init_strtab_2lvl(smmu);
3855 else
3856 ret = arm_smmu_init_strtab_linear(smmu);
3857 if (ret)
3858 return ret;
3859
3860 ida_init(&smmu->vmid_map);
3861
3862 return 0;
3863 }
3864
arm_smmu_init_structures(struct arm_smmu_device * smmu)3865 static int arm_smmu_init_structures(struct arm_smmu_device *smmu)
3866 {
3867 int ret;
3868
3869 mutex_init(&smmu->streams_mutex);
3870 smmu->streams = RB_ROOT;
3871
3872 ret = arm_smmu_init_queues(smmu);
3873 if (ret)
3874 return ret;
3875
3876 ret = arm_smmu_init_strtab(smmu);
3877 if (ret)
3878 return ret;
3879
3880 if (smmu->impl_ops && smmu->impl_ops->init_structures)
3881 return smmu->impl_ops->init_structures(smmu);
3882
3883 return 0;
3884 }
3885
arm_smmu_write_reg_sync(struct arm_smmu_device * smmu,u32 val,unsigned int reg_off,unsigned int ack_off)3886 static int arm_smmu_write_reg_sync(struct arm_smmu_device *smmu, u32 val,
3887 unsigned int reg_off, unsigned int ack_off)
3888 {
3889 u32 reg;
3890
3891 writel_relaxed(val, smmu->base + reg_off);
3892 return readl_relaxed_poll_timeout(smmu->base + ack_off, reg, reg == val,
3893 1, ARM_SMMU_POLL_TIMEOUT_US);
3894 }
3895
3896 /* GBPA is "special" */
arm_smmu_update_gbpa(struct arm_smmu_device * smmu,u32 set,u32 clr)3897 static int arm_smmu_update_gbpa(struct arm_smmu_device *smmu, u32 set, u32 clr)
3898 {
3899 int ret;
3900 u32 reg, __iomem *gbpa = smmu->base + ARM_SMMU_GBPA;
3901
3902 ret = readl_relaxed_poll_timeout(gbpa, reg, !(reg & GBPA_UPDATE),
3903 1, ARM_SMMU_POLL_TIMEOUT_US);
3904 if (ret)
3905 return ret;
3906
3907 reg &= ~clr;
3908 reg |= set;
3909 writel_relaxed(reg | GBPA_UPDATE, gbpa);
3910 ret = readl_relaxed_poll_timeout(gbpa, reg, !(reg & GBPA_UPDATE),
3911 1, ARM_SMMU_POLL_TIMEOUT_US);
3912
3913 if (ret)
3914 dev_err(smmu->dev, "GBPA not responding to update\n");
3915 return ret;
3916 }
3917
arm_smmu_free_msis(void * data)3918 static void arm_smmu_free_msis(void *data)
3919 {
3920 struct device *dev = data;
3921
3922 platform_device_msi_free_irqs_all(dev);
3923 }
3924
arm_smmu_write_msi_msg(struct msi_desc * desc,struct msi_msg * msg)3925 static void arm_smmu_write_msi_msg(struct msi_desc *desc, struct msi_msg *msg)
3926 {
3927 phys_addr_t doorbell;
3928 struct device *dev = msi_desc_to_dev(desc);
3929 struct arm_smmu_device *smmu = dev_get_drvdata(dev);
3930 phys_addr_t *cfg = arm_smmu_msi_cfg[desc->msi_index];
3931
3932 doorbell = (((u64)msg->address_hi) << 32) | msg->address_lo;
3933 doorbell &= MSI_CFG0_ADDR_MASK;
3934
3935 writeq_relaxed(doorbell, smmu->base + cfg[0]);
3936 writel_relaxed(msg->data, smmu->base + cfg[1]);
3937 writel_relaxed(ARM_SMMU_MEMATTR_DEVICE_nGnRE, smmu->base + cfg[2]);
3938 }
3939
arm_smmu_setup_msis(struct arm_smmu_device * smmu)3940 static void arm_smmu_setup_msis(struct arm_smmu_device *smmu)
3941 {
3942 int ret, nvec = ARM_SMMU_MAX_MSIS;
3943 struct device *dev = smmu->dev;
3944
3945 /* Clear the MSI address regs */
3946 writeq_relaxed(0, smmu->base + ARM_SMMU_GERROR_IRQ_CFG0);
3947 writeq_relaxed(0, smmu->base + ARM_SMMU_EVTQ_IRQ_CFG0);
3948
3949 if (smmu->features & ARM_SMMU_FEAT_PRI)
3950 writeq_relaxed(0, smmu->base + ARM_SMMU_PRIQ_IRQ_CFG0);
3951 else
3952 nvec--;
3953
3954 if (!(smmu->features & ARM_SMMU_FEAT_MSI))
3955 return;
3956
3957 if (!dev->msi.domain) {
3958 dev_info(smmu->dev, "msi_domain absent - falling back to wired irqs\n");
3959 return;
3960 }
3961
3962 /* Allocate MSIs for evtq, gerror and priq. Ignore cmdq */
3963 ret = platform_device_msi_init_and_alloc_irqs(dev, nvec, arm_smmu_write_msi_msg);
3964 if (ret) {
3965 dev_warn(dev, "failed to allocate MSIs - falling back to wired irqs\n");
3966 return;
3967 }
3968
3969 smmu->evtq.q.irq = msi_get_virq(dev, EVTQ_MSI_INDEX);
3970 smmu->gerr_irq = msi_get_virq(dev, GERROR_MSI_INDEX);
3971 smmu->priq.q.irq = msi_get_virq(dev, PRIQ_MSI_INDEX);
3972
3973 /* Add callback to free MSIs on teardown */
3974 devm_add_action_or_reset(dev, arm_smmu_free_msis, dev);
3975 }
3976
arm_smmu_setup_unique_irqs(struct arm_smmu_device * smmu)3977 static void arm_smmu_setup_unique_irqs(struct arm_smmu_device *smmu)
3978 {
3979 int irq, ret;
3980
3981 arm_smmu_setup_msis(smmu);
3982
3983 /* Request interrupt lines */
3984 irq = smmu->evtq.q.irq;
3985 if (irq) {
3986 ret = devm_request_threaded_irq(smmu->dev, irq, NULL,
3987 arm_smmu_evtq_thread,
3988 IRQF_ONESHOT,
3989 "arm-smmu-v3-evtq", smmu);
3990 if (ret < 0)
3991 dev_warn(smmu->dev, "failed to enable evtq irq\n");
3992 } else {
3993 dev_warn(smmu->dev, "no evtq irq - events will not be reported!\n");
3994 }
3995
3996 irq = smmu->gerr_irq;
3997 if (irq) {
3998 ret = devm_request_irq(smmu->dev, irq, arm_smmu_gerror_handler,
3999 0, "arm-smmu-v3-gerror", smmu);
4000 if (ret < 0)
4001 dev_warn(smmu->dev, "failed to enable gerror irq\n");
4002 } else {
4003 dev_warn(smmu->dev, "no gerr irq - errors will not be reported!\n");
4004 }
4005
4006 if (smmu->features & ARM_SMMU_FEAT_PRI) {
4007 irq = smmu->priq.q.irq;
4008 if (irq) {
4009 ret = devm_request_threaded_irq(smmu->dev, irq, NULL,
4010 arm_smmu_priq_thread,
4011 IRQF_ONESHOT,
4012 "arm-smmu-v3-priq",
4013 smmu);
4014 if (ret < 0)
4015 dev_warn(smmu->dev,
4016 "failed to enable priq irq\n");
4017 } else {
4018 dev_warn(smmu->dev, "no priq irq - PRI will be broken\n");
4019 }
4020 }
4021 }
4022
arm_smmu_setup_irqs(struct arm_smmu_device * smmu)4023 static int arm_smmu_setup_irqs(struct arm_smmu_device *smmu)
4024 {
4025 int ret, irq;
4026 u32 irqen_flags = IRQ_CTRL_EVTQ_IRQEN | IRQ_CTRL_GERROR_IRQEN;
4027
4028 /* Disable IRQs first */
4029 ret = arm_smmu_write_reg_sync(smmu, 0, ARM_SMMU_IRQ_CTRL,
4030 ARM_SMMU_IRQ_CTRLACK);
4031 if (ret) {
4032 dev_err(smmu->dev, "failed to disable irqs\n");
4033 return ret;
4034 }
4035
4036 irq = smmu->combined_irq;
4037 if (irq) {
4038 /*
4039 * Cavium ThunderX2 implementation doesn't support unique irq
4040 * lines. Use a single irq line for all the SMMUv3 interrupts.
4041 */
4042 ret = devm_request_threaded_irq(smmu->dev, irq,
4043 arm_smmu_combined_irq_handler,
4044 arm_smmu_combined_irq_thread,
4045 IRQF_ONESHOT,
4046 "arm-smmu-v3-combined-irq", smmu);
4047 if (ret < 0)
4048 dev_warn(smmu->dev, "failed to enable combined irq\n");
4049 } else
4050 arm_smmu_setup_unique_irqs(smmu);
4051
4052 if (smmu->features & ARM_SMMU_FEAT_PRI)
4053 irqen_flags |= IRQ_CTRL_PRIQ_IRQEN;
4054
4055 /* Enable interrupt generation on the SMMU */
4056 ret = arm_smmu_write_reg_sync(smmu, irqen_flags,
4057 ARM_SMMU_IRQ_CTRL, ARM_SMMU_IRQ_CTRLACK);
4058 if (ret)
4059 dev_warn(smmu->dev, "failed to enable irqs\n");
4060
4061 return 0;
4062 }
4063
arm_smmu_device_disable(struct arm_smmu_device * smmu)4064 static int arm_smmu_device_disable(struct arm_smmu_device *smmu)
4065 {
4066 int ret;
4067
4068 ret = arm_smmu_write_reg_sync(smmu, 0, ARM_SMMU_CR0, ARM_SMMU_CR0ACK);
4069 if (ret)
4070 dev_err(smmu->dev, "failed to clear cr0\n");
4071
4072 return ret;
4073 }
4074
arm_smmu_write_strtab(struct arm_smmu_device * smmu)4075 static void arm_smmu_write_strtab(struct arm_smmu_device *smmu)
4076 {
4077 struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg;
4078 dma_addr_t dma;
4079 u32 reg;
4080
4081 if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB) {
4082 reg = FIELD_PREP(STRTAB_BASE_CFG_FMT,
4083 STRTAB_BASE_CFG_FMT_2LVL) |
4084 FIELD_PREP(STRTAB_BASE_CFG_LOG2SIZE,
4085 ilog2(cfg->l2.num_l1_ents) + STRTAB_SPLIT) |
4086 FIELD_PREP(STRTAB_BASE_CFG_SPLIT, STRTAB_SPLIT);
4087 dma = cfg->l2.l1_dma;
4088 } else {
4089 reg = FIELD_PREP(STRTAB_BASE_CFG_FMT,
4090 STRTAB_BASE_CFG_FMT_LINEAR) |
4091 FIELD_PREP(STRTAB_BASE_CFG_LOG2SIZE, smmu->sid_bits);
4092 dma = cfg->linear.ste_dma;
4093 }
4094 writeq_relaxed((dma & STRTAB_BASE_ADDR_MASK) | STRTAB_BASE_RA,
4095 smmu->base + ARM_SMMU_STRTAB_BASE);
4096 writel_relaxed(reg, smmu->base + ARM_SMMU_STRTAB_BASE_CFG);
4097 }
4098
arm_smmu_device_reset(struct arm_smmu_device * smmu)4099 static int arm_smmu_device_reset(struct arm_smmu_device *smmu)
4100 {
4101 int ret;
4102 u32 reg, enables;
4103 struct arm_smmu_cmdq_ent cmd;
4104
4105 /* Clear CR0 and sync (disables SMMU and queue processing) */
4106 reg = readl_relaxed(smmu->base + ARM_SMMU_CR0);
4107 if (reg & CR0_SMMUEN) {
4108 dev_warn(smmu->dev, "SMMU currently enabled! Resetting...\n");
4109 arm_smmu_update_gbpa(smmu, GBPA_ABORT, 0);
4110 }
4111
4112 ret = arm_smmu_device_disable(smmu);
4113 if (ret)
4114 return ret;
4115
4116 /* CR1 (table and queue memory attributes) */
4117 reg = FIELD_PREP(CR1_TABLE_SH, ARM_SMMU_SH_ISH) |
4118 FIELD_PREP(CR1_TABLE_OC, CR1_CACHE_WB) |
4119 FIELD_PREP(CR1_TABLE_IC, CR1_CACHE_WB) |
4120 FIELD_PREP(CR1_QUEUE_SH, ARM_SMMU_SH_ISH) |
4121 FIELD_PREP(CR1_QUEUE_OC, CR1_CACHE_WB) |
4122 FIELD_PREP(CR1_QUEUE_IC, CR1_CACHE_WB);
4123 writel_relaxed(reg, smmu->base + ARM_SMMU_CR1);
4124
4125 /* CR2 (random crap) */
4126 reg = CR2_PTM | CR2_RECINVSID;
4127
4128 if (smmu->features & ARM_SMMU_FEAT_E2H)
4129 reg |= CR2_E2H;
4130
4131 writel_relaxed(reg, smmu->base + ARM_SMMU_CR2);
4132
4133 /* Stream table */
4134 arm_smmu_write_strtab(smmu);
4135
4136 /* Command queue */
4137 writeq_relaxed(smmu->cmdq.q.q_base, smmu->base + ARM_SMMU_CMDQ_BASE);
4138 writel_relaxed(smmu->cmdq.q.llq.prod, smmu->base + ARM_SMMU_CMDQ_PROD);
4139 writel_relaxed(smmu->cmdq.q.llq.cons, smmu->base + ARM_SMMU_CMDQ_CONS);
4140
4141 enables = CR0_CMDQEN;
4142 ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0,
4143 ARM_SMMU_CR0ACK);
4144 if (ret) {
4145 dev_err(smmu->dev, "failed to enable command queue\n");
4146 return ret;
4147 }
4148
4149 /* Invalidate any cached configuration */
4150 cmd.opcode = CMDQ_OP_CFGI_ALL;
4151 arm_smmu_cmdq_issue_cmd_with_sync(smmu, &cmd);
4152
4153 /* Invalidate any stale TLB entries */
4154 if (smmu->features & ARM_SMMU_FEAT_HYP) {
4155 cmd.opcode = CMDQ_OP_TLBI_EL2_ALL;
4156 arm_smmu_cmdq_issue_cmd_with_sync(smmu, &cmd);
4157 }
4158
4159 cmd.opcode = CMDQ_OP_TLBI_NSNH_ALL;
4160 arm_smmu_cmdq_issue_cmd_with_sync(smmu, &cmd);
4161
4162 /* Event queue */
4163 writeq_relaxed(smmu->evtq.q.q_base, smmu->base + ARM_SMMU_EVTQ_BASE);
4164 writel_relaxed(smmu->evtq.q.llq.prod, smmu->page1 + ARM_SMMU_EVTQ_PROD);
4165 writel_relaxed(smmu->evtq.q.llq.cons, smmu->page1 + ARM_SMMU_EVTQ_CONS);
4166
4167 enables |= CR0_EVTQEN;
4168 ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0,
4169 ARM_SMMU_CR0ACK);
4170 if (ret) {
4171 dev_err(smmu->dev, "failed to enable event queue\n");
4172 return ret;
4173 }
4174
4175 /* PRI queue */
4176 if (smmu->features & ARM_SMMU_FEAT_PRI) {
4177 writeq_relaxed(smmu->priq.q.q_base,
4178 smmu->base + ARM_SMMU_PRIQ_BASE);
4179 writel_relaxed(smmu->priq.q.llq.prod,
4180 smmu->page1 + ARM_SMMU_PRIQ_PROD);
4181 writel_relaxed(smmu->priq.q.llq.cons,
4182 smmu->page1 + ARM_SMMU_PRIQ_CONS);
4183
4184 enables |= CR0_PRIQEN;
4185 ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0,
4186 ARM_SMMU_CR0ACK);
4187 if (ret) {
4188 dev_err(smmu->dev, "failed to enable PRI queue\n");
4189 return ret;
4190 }
4191 }
4192
4193 if (smmu->features & ARM_SMMU_FEAT_ATS) {
4194 enables |= CR0_ATSCHK;
4195 ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0,
4196 ARM_SMMU_CR0ACK);
4197 if (ret) {
4198 dev_err(smmu->dev, "failed to enable ATS check\n");
4199 return ret;
4200 }
4201 }
4202
4203 ret = arm_smmu_setup_irqs(smmu);
4204 if (ret) {
4205 dev_err(smmu->dev, "failed to setup irqs\n");
4206 return ret;
4207 }
4208
4209 if (is_kdump_kernel())
4210 enables &= ~(CR0_EVTQEN | CR0_PRIQEN);
4211
4212 /* Enable the SMMU interface */
4213 enables |= CR0_SMMUEN;
4214 ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0,
4215 ARM_SMMU_CR0ACK);
4216 if (ret) {
4217 dev_err(smmu->dev, "failed to enable SMMU interface\n");
4218 return ret;
4219 }
4220
4221 if (smmu->impl_ops && smmu->impl_ops->device_reset) {
4222 ret = smmu->impl_ops->device_reset(smmu);
4223 if (ret) {
4224 dev_err(smmu->dev, "failed to reset impl\n");
4225 return ret;
4226 }
4227 }
4228
4229 return 0;
4230 }
4231
4232 #define IIDR_IMPLEMENTER_ARM 0x43b
4233 #define IIDR_PRODUCTID_ARM_MMU_600 0x483
4234 #define IIDR_PRODUCTID_ARM_MMU_700 0x487
4235
arm_smmu_device_iidr_probe(struct arm_smmu_device * smmu)4236 static void arm_smmu_device_iidr_probe(struct arm_smmu_device *smmu)
4237 {
4238 u32 reg;
4239 unsigned int implementer, productid, variant, revision;
4240
4241 reg = readl_relaxed(smmu->base + ARM_SMMU_IIDR);
4242 implementer = FIELD_GET(IIDR_IMPLEMENTER, reg);
4243 productid = FIELD_GET(IIDR_PRODUCTID, reg);
4244 variant = FIELD_GET(IIDR_VARIANT, reg);
4245 revision = FIELD_GET(IIDR_REVISION, reg);
4246
4247 switch (implementer) {
4248 case IIDR_IMPLEMENTER_ARM:
4249 switch (productid) {
4250 case IIDR_PRODUCTID_ARM_MMU_600:
4251 /* Arm erratum 1076982 */
4252 if (variant == 0 && revision <= 2)
4253 smmu->features &= ~ARM_SMMU_FEAT_SEV;
4254 /* Arm erratum 1209401 */
4255 if (variant < 2)
4256 smmu->features &= ~ARM_SMMU_FEAT_NESTING;
4257 break;
4258 case IIDR_PRODUCTID_ARM_MMU_700:
4259 /* Arm erratum 2812531 */
4260 smmu->features &= ~ARM_SMMU_FEAT_BTM;
4261 smmu->options |= ARM_SMMU_OPT_CMDQ_FORCE_SYNC;
4262 /* Arm errata 2268618, 2812531 */
4263 smmu->features &= ~ARM_SMMU_FEAT_NESTING;
4264 break;
4265 }
4266 break;
4267 }
4268 }
4269
arm_smmu_get_httu(struct arm_smmu_device * smmu,u32 reg)4270 static void arm_smmu_get_httu(struct arm_smmu_device *smmu, u32 reg)
4271 {
4272 u32 fw_features = smmu->features & (ARM_SMMU_FEAT_HA | ARM_SMMU_FEAT_HD);
4273 u32 hw_features = 0;
4274
4275 switch (FIELD_GET(IDR0_HTTU, reg)) {
4276 case IDR0_HTTU_ACCESS_DIRTY:
4277 hw_features |= ARM_SMMU_FEAT_HD;
4278 fallthrough;
4279 case IDR0_HTTU_ACCESS:
4280 hw_features |= ARM_SMMU_FEAT_HA;
4281 }
4282
4283 if (smmu->dev->of_node)
4284 smmu->features |= hw_features;
4285 else if (hw_features != fw_features)
4286 /* ACPI IORT sets the HTTU bits */
4287 dev_warn(smmu->dev,
4288 "IDR0.HTTU features(0x%x) overridden by FW configuration (0x%x)\n",
4289 hw_features, fw_features);
4290 }
4291
arm_smmu_device_hw_probe(struct arm_smmu_device * smmu)4292 static int arm_smmu_device_hw_probe(struct arm_smmu_device *smmu)
4293 {
4294 u32 reg;
4295 bool coherent = smmu->features & ARM_SMMU_FEAT_COHERENCY;
4296
4297 /* IDR0 */
4298 reg = readl_relaxed(smmu->base + ARM_SMMU_IDR0);
4299
4300 /* 2-level structures */
4301 if (FIELD_GET(IDR0_ST_LVL, reg) == IDR0_ST_LVL_2LVL)
4302 smmu->features |= ARM_SMMU_FEAT_2_LVL_STRTAB;
4303
4304 if (reg & IDR0_CD2L)
4305 smmu->features |= ARM_SMMU_FEAT_2_LVL_CDTAB;
4306
4307 /*
4308 * Translation table endianness.
4309 * We currently require the same endianness as the CPU, but this
4310 * could be changed later by adding a new IO_PGTABLE_QUIRK.
4311 */
4312 switch (FIELD_GET(IDR0_TTENDIAN, reg)) {
4313 case IDR0_TTENDIAN_MIXED:
4314 smmu->features |= ARM_SMMU_FEAT_TT_LE | ARM_SMMU_FEAT_TT_BE;
4315 break;
4316 #ifdef __BIG_ENDIAN
4317 case IDR0_TTENDIAN_BE:
4318 smmu->features |= ARM_SMMU_FEAT_TT_BE;
4319 break;
4320 #else
4321 case IDR0_TTENDIAN_LE:
4322 smmu->features |= ARM_SMMU_FEAT_TT_LE;
4323 break;
4324 #endif
4325 default:
4326 dev_err(smmu->dev, "unknown/unsupported TT endianness!\n");
4327 return -ENXIO;
4328 }
4329
4330 /* Boolean feature flags */
4331 if (IS_ENABLED(CONFIG_PCI_PRI) && reg & IDR0_PRI)
4332 smmu->features |= ARM_SMMU_FEAT_PRI;
4333
4334 if (IS_ENABLED(CONFIG_PCI_ATS) && reg & IDR0_ATS)
4335 smmu->features |= ARM_SMMU_FEAT_ATS;
4336
4337 if (reg & IDR0_SEV)
4338 smmu->features |= ARM_SMMU_FEAT_SEV;
4339
4340 if (reg & IDR0_MSI) {
4341 smmu->features |= ARM_SMMU_FEAT_MSI;
4342 if (coherent && !disable_msipolling)
4343 smmu->options |= ARM_SMMU_OPT_MSIPOLL;
4344 }
4345
4346 if (reg & IDR0_HYP) {
4347 smmu->features |= ARM_SMMU_FEAT_HYP;
4348 if (cpus_have_cap(ARM64_HAS_VIRT_HOST_EXTN))
4349 smmu->features |= ARM_SMMU_FEAT_E2H;
4350 }
4351
4352 arm_smmu_get_httu(smmu, reg);
4353
4354 /*
4355 * The coherency feature as set by FW is used in preference to the ID
4356 * register, but warn on mismatch.
4357 */
4358 if (!!(reg & IDR0_COHACC) != coherent)
4359 dev_warn(smmu->dev, "IDR0.COHACC overridden by FW configuration (%s)\n",
4360 str_true_false(coherent));
4361
4362 switch (FIELD_GET(IDR0_STALL_MODEL, reg)) {
4363 case IDR0_STALL_MODEL_FORCE:
4364 smmu->features |= ARM_SMMU_FEAT_STALL_FORCE;
4365 fallthrough;
4366 case IDR0_STALL_MODEL_STALL:
4367 smmu->features |= ARM_SMMU_FEAT_STALLS;
4368 }
4369
4370 if (reg & IDR0_S1P)
4371 smmu->features |= ARM_SMMU_FEAT_TRANS_S1;
4372
4373 if (reg & IDR0_S2P)
4374 smmu->features |= ARM_SMMU_FEAT_TRANS_S2;
4375
4376 if (!(reg & (IDR0_S1P | IDR0_S2P))) {
4377 dev_err(smmu->dev, "no translation support!\n");
4378 return -ENXIO;
4379 }
4380
4381 /* We only support the AArch64 table format at present */
4382 switch (FIELD_GET(IDR0_TTF, reg)) {
4383 case IDR0_TTF_AARCH32_64:
4384 smmu->ias = 40;
4385 fallthrough;
4386 case IDR0_TTF_AARCH64:
4387 break;
4388 default:
4389 dev_err(smmu->dev, "AArch64 table format not supported!\n");
4390 return -ENXIO;
4391 }
4392
4393 /* ASID/VMID sizes */
4394 smmu->asid_bits = reg & IDR0_ASID16 ? 16 : 8;
4395 smmu->vmid_bits = reg & IDR0_VMID16 ? 16 : 8;
4396
4397 /* IDR1 */
4398 reg = readl_relaxed(smmu->base + ARM_SMMU_IDR1);
4399 if (reg & (IDR1_TABLES_PRESET | IDR1_QUEUES_PRESET | IDR1_REL)) {
4400 dev_err(smmu->dev, "embedded implementation not supported\n");
4401 return -ENXIO;
4402 }
4403
4404 if (reg & IDR1_ATTR_TYPES_OVR)
4405 smmu->features |= ARM_SMMU_FEAT_ATTR_TYPES_OVR;
4406
4407 /* Queue sizes, capped to ensure natural alignment */
4408 smmu->cmdq.q.llq.max_n_shift = min_t(u32, CMDQ_MAX_SZ_SHIFT,
4409 FIELD_GET(IDR1_CMDQS, reg));
4410 if (smmu->cmdq.q.llq.max_n_shift <= ilog2(CMDQ_BATCH_ENTRIES)) {
4411 /*
4412 * We don't support splitting up batches, so one batch of
4413 * commands plus an extra sync needs to fit inside the command
4414 * queue. There's also no way we can handle the weird alignment
4415 * restrictions on the base pointer for a unit-length queue.
4416 */
4417 dev_err(smmu->dev, "command queue size <= %d entries not supported\n",
4418 CMDQ_BATCH_ENTRIES);
4419 return -ENXIO;
4420 }
4421
4422 smmu->evtq.q.llq.max_n_shift = min_t(u32, EVTQ_MAX_SZ_SHIFT,
4423 FIELD_GET(IDR1_EVTQS, reg));
4424 smmu->priq.q.llq.max_n_shift = min_t(u32, PRIQ_MAX_SZ_SHIFT,
4425 FIELD_GET(IDR1_PRIQS, reg));
4426
4427 /* SID/SSID sizes */
4428 smmu->ssid_bits = FIELD_GET(IDR1_SSIDSIZE, reg);
4429 smmu->sid_bits = FIELD_GET(IDR1_SIDSIZE, reg);
4430 smmu->iommu.max_pasids = 1UL << smmu->ssid_bits;
4431
4432 /*
4433 * If the SMMU supports fewer bits than would fill a single L2 stream
4434 * table, use a linear table instead.
4435 */
4436 if (smmu->sid_bits <= STRTAB_SPLIT)
4437 smmu->features &= ~ARM_SMMU_FEAT_2_LVL_STRTAB;
4438
4439 /* IDR3 */
4440 reg = readl_relaxed(smmu->base + ARM_SMMU_IDR3);
4441 if (FIELD_GET(IDR3_RIL, reg))
4442 smmu->features |= ARM_SMMU_FEAT_RANGE_INV;
4443 if (FIELD_GET(IDR3_FWB, reg))
4444 smmu->features |= ARM_SMMU_FEAT_S2FWB;
4445
4446 /* IDR5 */
4447 reg = readl_relaxed(smmu->base + ARM_SMMU_IDR5);
4448
4449 /* Maximum number of outstanding stalls */
4450 smmu->evtq.max_stalls = FIELD_GET(IDR5_STALL_MAX, reg);
4451
4452 /* Page sizes */
4453 if (reg & IDR5_GRAN64K)
4454 smmu->pgsize_bitmap |= SZ_64K | SZ_512M;
4455 if (reg & IDR5_GRAN16K)
4456 smmu->pgsize_bitmap |= SZ_16K | SZ_32M;
4457 if (reg & IDR5_GRAN4K)
4458 smmu->pgsize_bitmap |= SZ_4K | SZ_2M | SZ_1G;
4459
4460 /* Input address size */
4461 if (FIELD_GET(IDR5_VAX, reg) == IDR5_VAX_52_BIT)
4462 smmu->features |= ARM_SMMU_FEAT_VAX;
4463
4464 /* Output address size */
4465 switch (FIELD_GET(IDR5_OAS, reg)) {
4466 case IDR5_OAS_32_BIT:
4467 smmu->oas = 32;
4468 break;
4469 case IDR5_OAS_36_BIT:
4470 smmu->oas = 36;
4471 break;
4472 case IDR5_OAS_40_BIT:
4473 smmu->oas = 40;
4474 break;
4475 case IDR5_OAS_42_BIT:
4476 smmu->oas = 42;
4477 break;
4478 case IDR5_OAS_44_BIT:
4479 smmu->oas = 44;
4480 break;
4481 case IDR5_OAS_52_BIT:
4482 smmu->oas = 52;
4483 smmu->pgsize_bitmap |= 1ULL << 42; /* 4TB */
4484 break;
4485 default:
4486 dev_info(smmu->dev,
4487 "unknown output address size. Truncating to 48-bit\n");
4488 fallthrough;
4489 case IDR5_OAS_48_BIT:
4490 smmu->oas = 48;
4491 }
4492
4493 if (arm_smmu_ops.pgsize_bitmap == -1UL)
4494 arm_smmu_ops.pgsize_bitmap = smmu->pgsize_bitmap;
4495 else
4496 arm_smmu_ops.pgsize_bitmap |= smmu->pgsize_bitmap;
4497
4498 /* Set the DMA mask for our table walker */
4499 if (dma_set_mask_and_coherent(smmu->dev, DMA_BIT_MASK(smmu->oas)))
4500 dev_warn(smmu->dev,
4501 "failed to set DMA mask for table walker\n");
4502
4503 smmu->ias = max(smmu->ias, smmu->oas);
4504
4505 if ((smmu->features & ARM_SMMU_FEAT_TRANS_S1) &&
4506 (smmu->features & ARM_SMMU_FEAT_TRANS_S2))
4507 smmu->features |= ARM_SMMU_FEAT_NESTING;
4508
4509 arm_smmu_device_iidr_probe(smmu);
4510
4511 if (arm_smmu_sva_supported(smmu))
4512 smmu->features |= ARM_SMMU_FEAT_SVA;
4513
4514 dev_info(smmu->dev, "ias %lu-bit, oas %lu-bit (features 0x%08x)\n",
4515 smmu->ias, smmu->oas, smmu->features);
4516 return 0;
4517 }
4518
4519 #ifdef CONFIG_ACPI
4520 #ifdef CONFIG_TEGRA241_CMDQV
acpi_smmu_dsdt_probe_tegra241_cmdqv(struct acpi_iort_node * node,struct arm_smmu_device * smmu)4521 static void acpi_smmu_dsdt_probe_tegra241_cmdqv(struct acpi_iort_node *node,
4522 struct arm_smmu_device *smmu)
4523 {
4524 const char *uid = kasprintf(GFP_KERNEL, "%u", node->identifier);
4525 struct acpi_device *adev;
4526
4527 /* Look for an NVDA200C node whose _UID matches the SMMU node ID */
4528 adev = acpi_dev_get_first_match_dev("NVDA200C", uid, -1);
4529 if (adev) {
4530 /* Tegra241 CMDQV driver is responsible for put_device() */
4531 smmu->impl_dev = &adev->dev;
4532 smmu->options |= ARM_SMMU_OPT_TEGRA241_CMDQV;
4533 dev_info(smmu->dev, "found companion CMDQV device: %s\n",
4534 dev_name(smmu->impl_dev));
4535 }
4536 kfree(uid);
4537 }
4538 #else
acpi_smmu_dsdt_probe_tegra241_cmdqv(struct acpi_iort_node * node,struct arm_smmu_device * smmu)4539 static void acpi_smmu_dsdt_probe_tegra241_cmdqv(struct acpi_iort_node *node,
4540 struct arm_smmu_device *smmu)
4541 {
4542 }
4543 #endif
4544
acpi_smmu_iort_probe_model(struct acpi_iort_node * node,struct arm_smmu_device * smmu)4545 static int acpi_smmu_iort_probe_model(struct acpi_iort_node *node,
4546 struct arm_smmu_device *smmu)
4547 {
4548 struct acpi_iort_smmu_v3 *iort_smmu =
4549 (struct acpi_iort_smmu_v3 *)node->node_data;
4550
4551 switch (iort_smmu->model) {
4552 case ACPI_IORT_SMMU_V3_CAVIUM_CN99XX:
4553 smmu->options |= ARM_SMMU_OPT_PAGE0_REGS_ONLY;
4554 break;
4555 case ACPI_IORT_SMMU_V3_HISILICON_HI161X:
4556 smmu->options |= ARM_SMMU_OPT_SKIP_PREFETCH;
4557 break;
4558 case ACPI_IORT_SMMU_V3_GENERIC:
4559 /*
4560 * Tegra241 implementation stores its SMMU options and impl_dev
4561 * in DSDT. Thus, go through the ACPI tables unconditionally.
4562 */
4563 acpi_smmu_dsdt_probe_tegra241_cmdqv(node, smmu);
4564 break;
4565 }
4566
4567 dev_notice(smmu->dev, "option mask 0x%x\n", smmu->options);
4568 return 0;
4569 }
4570
arm_smmu_device_acpi_probe(struct platform_device * pdev,struct arm_smmu_device * smmu)4571 static int arm_smmu_device_acpi_probe(struct platform_device *pdev,
4572 struct arm_smmu_device *smmu)
4573 {
4574 struct acpi_iort_smmu_v3 *iort_smmu;
4575 struct device *dev = smmu->dev;
4576 struct acpi_iort_node *node;
4577
4578 node = *(struct acpi_iort_node **)dev_get_platdata(dev);
4579
4580 /* Retrieve SMMUv3 specific data */
4581 iort_smmu = (struct acpi_iort_smmu_v3 *)node->node_data;
4582
4583 if (iort_smmu->flags & ACPI_IORT_SMMU_V3_COHACC_OVERRIDE)
4584 smmu->features |= ARM_SMMU_FEAT_COHERENCY;
4585
4586 switch (FIELD_GET(ACPI_IORT_SMMU_V3_HTTU_OVERRIDE, iort_smmu->flags)) {
4587 case IDR0_HTTU_ACCESS_DIRTY:
4588 smmu->features |= ARM_SMMU_FEAT_HD;
4589 fallthrough;
4590 case IDR0_HTTU_ACCESS:
4591 smmu->features |= ARM_SMMU_FEAT_HA;
4592 }
4593
4594 return acpi_smmu_iort_probe_model(node, smmu);
4595 }
4596 #else
arm_smmu_device_acpi_probe(struct platform_device * pdev,struct arm_smmu_device * smmu)4597 static inline int arm_smmu_device_acpi_probe(struct platform_device *pdev,
4598 struct arm_smmu_device *smmu)
4599 {
4600 return -ENODEV;
4601 }
4602 #endif
4603
arm_smmu_device_dt_probe(struct platform_device * pdev,struct arm_smmu_device * smmu)4604 static int arm_smmu_device_dt_probe(struct platform_device *pdev,
4605 struct arm_smmu_device *smmu)
4606 {
4607 struct device *dev = &pdev->dev;
4608 u32 cells;
4609 int ret = -EINVAL;
4610
4611 if (of_property_read_u32(dev->of_node, "#iommu-cells", &cells))
4612 dev_err(dev, "missing #iommu-cells property\n");
4613 else if (cells != 1)
4614 dev_err(dev, "invalid #iommu-cells value (%d)\n", cells);
4615 else
4616 ret = 0;
4617
4618 parse_driver_options(smmu);
4619
4620 if (of_dma_is_coherent(dev->of_node))
4621 smmu->features |= ARM_SMMU_FEAT_COHERENCY;
4622
4623 return ret;
4624 }
4625
arm_smmu_resource_size(struct arm_smmu_device * smmu)4626 static unsigned long arm_smmu_resource_size(struct arm_smmu_device *smmu)
4627 {
4628 if (smmu->options & ARM_SMMU_OPT_PAGE0_REGS_ONLY)
4629 return SZ_64K;
4630 else
4631 return SZ_128K;
4632 }
4633
arm_smmu_ioremap(struct device * dev,resource_size_t start,resource_size_t size)4634 static void __iomem *arm_smmu_ioremap(struct device *dev, resource_size_t start,
4635 resource_size_t size)
4636 {
4637 struct resource res = DEFINE_RES_MEM(start, size);
4638
4639 return devm_ioremap_resource(dev, &res);
4640 }
4641
arm_smmu_rmr_install_bypass_ste(struct arm_smmu_device * smmu)4642 static void arm_smmu_rmr_install_bypass_ste(struct arm_smmu_device *smmu)
4643 {
4644 struct list_head rmr_list;
4645 struct iommu_resv_region *e;
4646
4647 INIT_LIST_HEAD(&rmr_list);
4648 iort_get_rmr_sids(dev_fwnode(smmu->dev), &rmr_list);
4649
4650 list_for_each_entry(e, &rmr_list, list) {
4651 struct iommu_iort_rmr_data *rmr;
4652 int ret, i;
4653
4654 rmr = container_of(e, struct iommu_iort_rmr_data, rr);
4655 for (i = 0; i < rmr->num_sids; i++) {
4656 ret = arm_smmu_init_sid_strtab(smmu, rmr->sids[i]);
4657 if (ret) {
4658 dev_err(smmu->dev, "RMR SID(0x%x) bypass failed\n",
4659 rmr->sids[i]);
4660 continue;
4661 }
4662
4663 /*
4664 * STE table is not programmed to HW, see
4665 * arm_smmu_initial_bypass_stes()
4666 */
4667 arm_smmu_make_bypass_ste(smmu,
4668 arm_smmu_get_step_for_sid(smmu, rmr->sids[i]));
4669 }
4670 }
4671
4672 iort_put_rmr_sids(dev_fwnode(smmu->dev), &rmr_list);
4673 }
4674
arm_smmu_impl_remove(void * data)4675 static void arm_smmu_impl_remove(void *data)
4676 {
4677 struct arm_smmu_device *smmu = data;
4678
4679 if (smmu->impl_ops && smmu->impl_ops->device_remove)
4680 smmu->impl_ops->device_remove(smmu);
4681 }
4682
4683 /*
4684 * Probe all the compiled in implementations. Each one checks to see if it
4685 * matches this HW and if so returns a devm_krealloc'd arm_smmu_device which
4686 * replaces the callers. Otherwise the original is returned or ERR_PTR.
4687 */
arm_smmu_impl_probe(struct arm_smmu_device * smmu)4688 static struct arm_smmu_device *arm_smmu_impl_probe(struct arm_smmu_device *smmu)
4689 {
4690 struct arm_smmu_device *new_smmu = ERR_PTR(-ENODEV);
4691 int ret;
4692
4693 if (smmu->impl_dev && (smmu->options & ARM_SMMU_OPT_TEGRA241_CMDQV))
4694 new_smmu = tegra241_cmdqv_probe(smmu);
4695
4696 if (new_smmu == ERR_PTR(-ENODEV))
4697 return smmu;
4698 if (IS_ERR(new_smmu))
4699 return new_smmu;
4700
4701 ret = devm_add_action_or_reset(new_smmu->dev, arm_smmu_impl_remove,
4702 new_smmu);
4703 if (ret)
4704 return ERR_PTR(ret);
4705 return new_smmu;
4706 }
4707
arm_smmu_device_probe(struct platform_device * pdev)4708 static int arm_smmu_device_probe(struct platform_device *pdev)
4709 {
4710 int irq, ret;
4711 struct resource *res;
4712 resource_size_t ioaddr;
4713 struct arm_smmu_device *smmu;
4714 struct device *dev = &pdev->dev;
4715
4716 smmu = devm_kzalloc(dev, sizeof(*smmu), GFP_KERNEL);
4717 if (!smmu)
4718 return -ENOMEM;
4719 smmu->dev = dev;
4720
4721 if (dev->of_node) {
4722 ret = arm_smmu_device_dt_probe(pdev, smmu);
4723 } else {
4724 ret = arm_smmu_device_acpi_probe(pdev, smmu);
4725 }
4726 if (ret)
4727 return ret;
4728
4729 smmu = arm_smmu_impl_probe(smmu);
4730 if (IS_ERR(smmu))
4731 return PTR_ERR(smmu);
4732
4733 /* Base address */
4734 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
4735 if (!res)
4736 return -EINVAL;
4737 if (resource_size(res) < arm_smmu_resource_size(smmu)) {
4738 dev_err(dev, "MMIO region too small (%pr)\n", res);
4739 return -EINVAL;
4740 }
4741 ioaddr = res->start;
4742
4743 /*
4744 * Don't map the IMPLEMENTATION DEFINED regions, since they may contain
4745 * the PMCG registers which are reserved by the PMU driver.
4746 */
4747 smmu->base = arm_smmu_ioremap(dev, ioaddr, ARM_SMMU_REG_SZ);
4748 if (IS_ERR(smmu->base))
4749 return PTR_ERR(smmu->base);
4750
4751 if (arm_smmu_resource_size(smmu) > SZ_64K) {
4752 smmu->page1 = arm_smmu_ioremap(dev, ioaddr + SZ_64K,
4753 ARM_SMMU_REG_SZ);
4754 if (IS_ERR(smmu->page1))
4755 return PTR_ERR(smmu->page1);
4756 } else {
4757 smmu->page1 = smmu->base;
4758 }
4759
4760 /* Interrupt lines */
4761
4762 irq = platform_get_irq_byname_optional(pdev, "combined");
4763 if (irq > 0)
4764 smmu->combined_irq = irq;
4765 else {
4766 irq = platform_get_irq_byname_optional(pdev, "eventq");
4767 if (irq > 0)
4768 smmu->evtq.q.irq = irq;
4769
4770 irq = platform_get_irq_byname_optional(pdev, "priq");
4771 if (irq > 0)
4772 smmu->priq.q.irq = irq;
4773
4774 irq = platform_get_irq_byname_optional(pdev, "gerror");
4775 if (irq > 0)
4776 smmu->gerr_irq = irq;
4777 }
4778 /* Probe the h/w */
4779 ret = arm_smmu_device_hw_probe(smmu);
4780 if (ret)
4781 return ret;
4782
4783 /* Initialise in-memory data structures */
4784 ret = arm_smmu_init_structures(smmu);
4785 if (ret)
4786 goto err_free_iopf;
4787
4788 /* Record our private device structure */
4789 platform_set_drvdata(pdev, smmu);
4790
4791 /* Check for RMRs and install bypass STEs if any */
4792 arm_smmu_rmr_install_bypass_ste(smmu);
4793
4794 /* Reset the device */
4795 ret = arm_smmu_device_reset(smmu);
4796 if (ret)
4797 goto err_disable;
4798
4799 /* And we're up. Go go go! */
4800 ret = iommu_device_sysfs_add(&smmu->iommu, dev, NULL,
4801 "smmu3.%pa", &ioaddr);
4802 if (ret)
4803 goto err_disable;
4804
4805 ret = iommu_device_register(&smmu->iommu, &arm_smmu_ops, dev);
4806 if (ret) {
4807 dev_err(dev, "Failed to register iommu\n");
4808 goto err_free_sysfs;
4809 }
4810
4811 return 0;
4812
4813 err_free_sysfs:
4814 iommu_device_sysfs_remove(&smmu->iommu);
4815 err_disable:
4816 arm_smmu_device_disable(smmu);
4817 err_free_iopf:
4818 iopf_queue_free(smmu->evtq.iopf);
4819 return ret;
4820 }
4821
arm_smmu_device_remove(struct platform_device * pdev)4822 static void arm_smmu_device_remove(struct platform_device *pdev)
4823 {
4824 struct arm_smmu_device *smmu = platform_get_drvdata(pdev);
4825
4826 iommu_device_unregister(&smmu->iommu);
4827 iommu_device_sysfs_remove(&smmu->iommu);
4828 arm_smmu_device_disable(smmu);
4829 iopf_queue_free(smmu->evtq.iopf);
4830 ida_destroy(&smmu->vmid_map);
4831 }
4832
arm_smmu_device_shutdown(struct platform_device * pdev)4833 static void arm_smmu_device_shutdown(struct platform_device *pdev)
4834 {
4835 struct arm_smmu_device *smmu = platform_get_drvdata(pdev);
4836
4837 arm_smmu_device_disable(smmu);
4838 }
4839
4840 static const struct of_device_id arm_smmu_of_match[] = {
4841 { .compatible = "arm,smmu-v3", },
4842 { },
4843 };
4844 MODULE_DEVICE_TABLE(of, arm_smmu_of_match);
4845
arm_smmu_driver_unregister(struct platform_driver * drv)4846 static void arm_smmu_driver_unregister(struct platform_driver *drv)
4847 {
4848 arm_smmu_sva_notifier_synchronize();
4849 platform_driver_unregister(drv);
4850 }
4851
4852 static struct platform_driver arm_smmu_driver = {
4853 .driver = {
4854 .name = "arm-smmu-v3",
4855 .of_match_table = arm_smmu_of_match,
4856 .suppress_bind_attrs = true,
4857 },
4858 .probe = arm_smmu_device_probe,
4859 .remove = arm_smmu_device_remove,
4860 .shutdown = arm_smmu_device_shutdown,
4861 };
4862 module_driver(arm_smmu_driver, platform_driver_register,
4863 arm_smmu_driver_unregister);
4864
4865 MODULE_DESCRIPTION("IOMMU API for ARM architected SMMUv3 implementations");
4866 MODULE_AUTHOR("Will Deacon <will@kernel.org>");
4867 MODULE_ALIAS("platform:arm-smmu-v3");
4868 MODULE_LICENSE("GPL v2");
4869