1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * IOMMU API for ARM architected SMMUv3 implementations.
4  *
5  * Copyright (C) 2015 ARM Limited
6  *
7  * Author: Will Deacon <will.deacon@arm.com>
8  *
9  * This driver is powered by bad coffee and bombay mix.
10  */
11 
12 #include <linux/acpi.h>
13 #include <linux/acpi_iort.h>
14 #include <linux/bitops.h>
15 #include <linux/crash_dump.h>
16 #include <linux/delay.h>
17 #include <linux/err.h>
18 #include <linux/interrupt.h>
19 #include <linux/io-pgtable.h>
20 #include <linux/iopoll.h>
21 #include <linux/module.h>
22 #include <linux/msi.h>
23 #include <linux/of.h>
24 #include <linux/of_address.h>
25 #include <linux/of_platform.h>
26 #include <linux/pci.h>
27 #include <linux/pci-ats.h>
28 #include <linux/platform_device.h>
29 #include <linux/string_choices.h>
30 #include <kunit/visibility.h>
31 #include <uapi/linux/iommufd.h>
32 
33 #include "arm-smmu-v3.h"
34 #include "../../dma-iommu.h"
35 
36 static bool disable_msipolling;
37 module_param(disable_msipolling, bool, 0444);
38 MODULE_PARM_DESC(disable_msipolling,
39 	"Disable MSI-based polling for CMD_SYNC completion.");
40 
41 static struct iommu_ops arm_smmu_ops;
42 static struct iommu_dirty_ops arm_smmu_dirty_ops;
43 
44 enum arm_smmu_msi_index {
45 	EVTQ_MSI_INDEX,
46 	GERROR_MSI_INDEX,
47 	PRIQ_MSI_INDEX,
48 	ARM_SMMU_MAX_MSIS,
49 };
50 
51 #define NUM_ENTRY_QWORDS 8
52 static_assert(sizeof(struct arm_smmu_ste) == NUM_ENTRY_QWORDS * sizeof(u64));
53 static_assert(sizeof(struct arm_smmu_cd) == NUM_ENTRY_QWORDS * sizeof(u64));
54 
55 static phys_addr_t arm_smmu_msi_cfg[ARM_SMMU_MAX_MSIS][3] = {
56 	[EVTQ_MSI_INDEX] = {
57 		ARM_SMMU_EVTQ_IRQ_CFG0,
58 		ARM_SMMU_EVTQ_IRQ_CFG1,
59 		ARM_SMMU_EVTQ_IRQ_CFG2,
60 	},
61 	[GERROR_MSI_INDEX] = {
62 		ARM_SMMU_GERROR_IRQ_CFG0,
63 		ARM_SMMU_GERROR_IRQ_CFG1,
64 		ARM_SMMU_GERROR_IRQ_CFG2,
65 	},
66 	[PRIQ_MSI_INDEX] = {
67 		ARM_SMMU_PRIQ_IRQ_CFG0,
68 		ARM_SMMU_PRIQ_IRQ_CFG1,
69 		ARM_SMMU_PRIQ_IRQ_CFG2,
70 	},
71 };
72 
73 struct arm_smmu_option_prop {
74 	u32 opt;
75 	const char *prop;
76 };
77 
78 DEFINE_XARRAY_ALLOC1(arm_smmu_asid_xa);
79 DEFINE_MUTEX(arm_smmu_asid_lock);
80 
81 static struct arm_smmu_option_prop arm_smmu_options[] = {
82 	{ ARM_SMMU_OPT_SKIP_PREFETCH, "hisilicon,broken-prefetch-cmd" },
83 	{ ARM_SMMU_OPT_PAGE0_REGS_ONLY, "cavium,cn9900-broken-page1-regspace"},
84 	{ 0, NULL},
85 };
86 
87 static const char * const event_str[] = {
88 	[EVT_ID_BAD_STREAMID_CONFIG] = "C_BAD_STREAMID",
89 	[EVT_ID_STE_FETCH_FAULT] = "F_STE_FETCH",
90 	[EVT_ID_BAD_STE_CONFIG] = "C_BAD_STE",
91 	[EVT_ID_STREAM_DISABLED_FAULT] = "F_STREAM_DISABLED",
92 	[EVT_ID_BAD_SUBSTREAMID_CONFIG] = "C_BAD_SUBSTREAMID",
93 	[EVT_ID_CD_FETCH_FAULT] = "F_CD_FETCH",
94 	[EVT_ID_BAD_CD_CONFIG] = "C_BAD_CD",
95 	[EVT_ID_TRANSLATION_FAULT] = "F_TRANSLATION",
96 	[EVT_ID_ADDR_SIZE_FAULT] = "F_ADDR_SIZE",
97 	[EVT_ID_ACCESS_FAULT] = "F_ACCESS",
98 	[EVT_ID_PERMISSION_FAULT] = "F_PERMISSION",
99 	[EVT_ID_VMS_FETCH_FAULT] = "F_VMS_FETCH",
100 };
101 
102 static const char * const event_class_str[] = {
103 	[0] = "CD fetch",
104 	[1] = "Stage 1 translation table fetch",
105 	[2] = "Input address caused fault",
106 	[3] = "Reserved",
107 };
108 
109 static int arm_smmu_alloc_cd_tables(struct arm_smmu_master *master);
110 
parse_driver_options(struct arm_smmu_device * smmu)111 static void parse_driver_options(struct arm_smmu_device *smmu)
112 {
113 	int i = 0;
114 
115 	do {
116 		if (of_property_read_bool(smmu->dev->of_node,
117 						arm_smmu_options[i].prop)) {
118 			smmu->options |= arm_smmu_options[i].opt;
119 			dev_notice(smmu->dev, "option %s\n",
120 				arm_smmu_options[i].prop);
121 		}
122 	} while (arm_smmu_options[++i].opt);
123 }
124 
125 /* Low-level queue manipulation functions */
queue_has_space(struct arm_smmu_ll_queue * q,u32 n)126 static bool queue_has_space(struct arm_smmu_ll_queue *q, u32 n)
127 {
128 	u32 space, prod, cons;
129 
130 	prod = Q_IDX(q, q->prod);
131 	cons = Q_IDX(q, q->cons);
132 
133 	if (Q_WRP(q, q->prod) == Q_WRP(q, q->cons))
134 		space = (1 << q->max_n_shift) - (prod - cons);
135 	else
136 		space = cons - prod;
137 
138 	return space >= n;
139 }
140 
queue_full(struct arm_smmu_ll_queue * q)141 static bool queue_full(struct arm_smmu_ll_queue *q)
142 {
143 	return Q_IDX(q, q->prod) == Q_IDX(q, q->cons) &&
144 	       Q_WRP(q, q->prod) != Q_WRP(q, q->cons);
145 }
146 
queue_empty(struct arm_smmu_ll_queue * q)147 static bool queue_empty(struct arm_smmu_ll_queue *q)
148 {
149 	return Q_IDX(q, q->prod) == Q_IDX(q, q->cons) &&
150 	       Q_WRP(q, q->prod) == Q_WRP(q, q->cons);
151 }
152 
queue_consumed(struct arm_smmu_ll_queue * q,u32 prod)153 static bool queue_consumed(struct arm_smmu_ll_queue *q, u32 prod)
154 {
155 	return ((Q_WRP(q, q->cons) == Q_WRP(q, prod)) &&
156 		(Q_IDX(q, q->cons) > Q_IDX(q, prod))) ||
157 	       ((Q_WRP(q, q->cons) != Q_WRP(q, prod)) &&
158 		(Q_IDX(q, q->cons) <= Q_IDX(q, prod)));
159 }
160 
queue_sync_cons_out(struct arm_smmu_queue * q)161 static void queue_sync_cons_out(struct arm_smmu_queue *q)
162 {
163 	/*
164 	 * Ensure that all CPU accesses (reads and writes) to the queue
165 	 * are complete before we update the cons pointer.
166 	 */
167 	__iomb();
168 	writel_relaxed(q->llq.cons, q->cons_reg);
169 }
170 
queue_inc_cons(struct arm_smmu_ll_queue * q)171 static void queue_inc_cons(struct arm_smmu_ll_queue *q)
172 {
173 	u32 cons = (Q_WRP(q, q->cons) | Q_IDX(q, q->cons)) + 1;
174 	q->cons = Q_OVF(q->cons) | Q_WRP(q, cons) | Q_IDX(q, cons);
175 }
176 
queue_sync_cons_ovf(struct arm_smmu_queue * q)177 static void queue_sync_cons_ovf(struct arm_smmu_queue *q)
178 {
179 	struct arm_smmu_ll_queue *llq = &q->llq;
180 
181 	if (likely(Q_OVF(llq->prod) == Q_OVF(llq->cons)))
182 		return;
183 
184 	llq->cons = Q_OVF(llq->prod) | Q_WRP(llq, llq->cons) |
185 		      Q_IDX(llq, llq->cons);
186 	queue_sync_cons_out(q);
187 }
188 
queue_sync_prod_in(struct arm_smmu_queue * q)189 static int queue_sync_prod_in(struct arm_smmu_queue *q)
190 {
191 	u32 prod;
192 	int ret = 0;
193 
194 	/*
195 	 * We can't use the _relaxed() variant here, as we must prevent
196 	 * speculative reads of the queue before we have determined that
197 	 * prod has indeed moved.
198 	 */
199 	prod = readl(q->prod_reg);
200 
201 	if (Q_OVF(prod) != Q_OVF(q->llq.prod))
202 		ret = -EOVERFLOW;
203 
204 	q->llq.prod = prod;
205 	return ret;
206 }
207 
queue_inc_prod_n(struct arm_smmu_ll_queue * q,int n)208 static u32 queue_inc_prod_n(struct arm_smmu_ll_queue *q, int n)
209 {
210 	u32 prod = (Q_WRP(q, q->prod) | Q_IDX(q, q->prod)) + n;
211 	return Q_OVF(q->prod) | Q_WRP(q, prod) | Q_IDX(q, prod);
212 }
213 
queue_poll_init(struct arm_smmu_device * smmu,struct arm_smmu_queue_poll * qp)214 static void queue_poll_init(struct arm_smmu_device *smmu,
215 			    struct arm_smmu_queue_poll *qp)
216 {
217 	qp->delay = 1;
218 	qp->spin_cnt = 0;
219 	qp->wfe = !!(smmu->features & ARM_SMMU_FEAT_SEV);
220 	qp->timeout = ktime_add_us(ktime_get(), ARM_SMMU_POLL_TIMEOUT_US);
221 }
222 
queue_poll(struct arm_smmu_queue_poll * qp)223 static int queue_poll(struct arm_smmu_queue_poll *qp)
224 {
225 	if (ktime_compare(ktime_get(), qp->timeout) > 0)
226 		return -ETIMEDOUT;
227 
228 	if (qp->wfe) {
229 		wfe();
230 	} else if (++qp->spin_cnt < ARM_SMMU_POLL_SPIN_COUNT) {
231 		cpu_relax();
232 	} else {
233 		udelay(qp->delay);
234 		qp->delay *= 2;
235 		qp->spin_cnt = 0;
236 	}
237 
238 	return 0;
239 }
240 
queue_write(__le64 * dst,u64 * src,size_t n_dwords)241 static void queue_write(__le64 *dst, u64 *src, size_t n_dwords)
242 {
243 	int i;
244 
245 	for (i = 0; i < n_dwords; ++i)
246 		*dst++ = cpu_to_le64(*src++);
247 }
248 
queue_read(u64 * dst,__le64 * src,size_t n_dwords)249 static void queue_read(u64 *dst, __le64 *src, size_t n_dwords)
250 {
251 	int i;
252 
253 	for (i = 0; i < n_dwords; ++i)
254 		*dst++ = le64_to_cpu(*src++);
255 }
256 
queue_remove_raw(struct arm_smmu_queue * q,u64 * ent)257 static int queue_remove_raw(struct arm_smmu_queue *q, u64 *ent)
258 {
259 	if (queue_empty(&q->llq))
260 		return -EAGAIN;
261 
262 	queue_read(ent, Q_ENT(q, q->llq.cons), q->ent_dwords);
263 	queue_inc_cons(&q->llq);
264 	queue_sync_cons_out(q);
265 	return 0;
266 }
267 
268 /* High-level queue accessors */
arm_smmu_cmdq_build_cmd(u64 * cmd,struct arm_smmu_cmdq_ent * ent)269 static int arm_smmu_cmdq_build_cmd(u64 *cmd, struct arm_smmu_cmdq_ent *ent)
270 {
271 	memset(cmd, 0, 1 << CMDQ_ENT_SZ_SHIFT);
272 	cmd[0] |= FIELD_PREP(CMDQ_0_OP, ent->opcode);
273 
274 	switch (ent->opcode) {
275 	case CMDQ_OP_TLBI_EL2_ALL:
276 	case CMDQ_OP_TLBI_NSNH_ALL:
277 		break;
278 	case CMDQ_OP_PREFETCH_CFG:
279 		cmd[0] |= FIELD_PREP(CMDQ_PREFETCH_0_SID, ent->prefetch.sid);
280 		break;
281 	case CMDQ_OP_CFGI_CD:
282 		cmd[0] |= FIELD_PREP(CMDQ_CFGI_0_SSID, ent->cfgi.ssid);
283 		fallthrough;
284 	case CMDQ_OP_CFGI_STE:
285 		cmd[0] |= FIELD_PREP(CMDQ_CFGI_0_SID, ent->cfgi.sid);
286 		cmd[1] |= FIELD_PREP(CMDQ_CFGI_1_LEAF, ent->cfgi.leaf);
287 		break;
288 	case CMDQ_OP_CFGI_CD_ALL:
289 		cmd[0] |= FIELD_PREP(CMDQ_CFGI_0_SID, ent->cfgi.sid);
290 		break;
291 	case CMDQ_OP_CFGI_ALL:
292 		/* Cover the entire SID range */
293 		cmd[1] |= FIELD_PREP(CMDQ_CFGI_1_RANGE, 31);
294 		break;
295 	case CMDQ_OP_TLBI_NH_VA:
296 		cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_VMID, ent->tlbi.vmid);
297 		fallthrough;
298 	case CMDQ_OP_TLBI_EL2_VA:
299 		cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_NUM, ent->tlbi.num);
300 		cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_SCALE, ent->tlbi.scale);
301 		cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_ASID, ent->tlbi.asid);
302 		cmd[1] |= FIELD_PREP(CMDQ_TLBI_1_LEAF, ent->tlbi.leaf);
303 		cmd[1] |= FIELD_PREP(CMDQ_TLBI_1_TTL, ent->tlbi.ttl);
304 		cmd[1] |= FIELD_PREP(CMDQ_TLBI_1_TG, ent->tlbi.tg);
305 		cmd[1] |= ent->tlbi.addr & CMDQ_TLBI_1_VA_MASK;
306 		break;
307 	case CMDQ_OP_TLBI_S2_IPA:
308 		cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_NUM, ent->tlbi.num);
309 		cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_SCALE, ent->tlbi.scale);
310 		cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_VMID, ent->tlbi.vmid);
311 		cmd[1] |= FIELD_PREP(CMDQ_TLBI_1_LEAF, ent->tlbi.leaf);
312 		cmd[1] |= FIELD_PREP(CMDQ_TLBI_1_TTL, ent->tlbi.ttl);
313 		cmd[1] |= FIELD_PREP(CMDQ_TLBI_1_TG, ent->tlbi.tg);
314 		cmd[1] |= ent->tlbi.addr & CMDQ_TLBI_1_IPA_MASK;
315 		break;
316 	case CMDQ_OP_TLBI_NH_ASID:
317 		cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_ASID, ent->tlbi.asid);
318 		fallthrough;
319 	case CMDQ_OP_TLBI_NH_ALL:
320 	case CMDQ_OP_TLBI_S12_VMALL:
321 		cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_VMID, ent->tlbi.vmid);
322 		break;
323 	case CMDQ_OP_TLBI_EL2_ASID:
324 		cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_ASID, ent->tlbi.asid);
325 		break;
326 	case CMDQ_OP_ATC_INV:
327 		cmd[0] |= FIELD_PREP(CMDQ_0_SSV, ent->substream_valid);
328 		cmd[0] |= FIELD_PREP(CMDQ_ATC_0_GLOBAL, ent->atc.global);
329 		cmd[0] |= FIELD_PREP(CMDQ_ATC_0_SSID, ent->atc.ssid);
330 		cmd[0] |= FIELD_PREP(CMDQ_ATC_0_SID, ent->atc.sid);
331 		cmd[1] |= FIELD_PREP(CMDQ_ATC_1_SIZE, ent->atc.size);
332 		cmd[1] |= ent->atc.addr & CMDQ_ATC_1_ADDR_MASK;
333 		break;
334 	case CMDQ_OP_PRI_RESP:
335 		cmd[0] |= FIELD_PREP(CMDQ_0_SSV, ent->substream_valid);
336 		cmd[0] |= FIELD_PREP(CMDQ_PRI_0_SSID, ent->pri.ssid);
337 		cmd[0] |= FIELD_PREP(CMDQ_PRI_0_SID, ent->pri.sid);
338 		cmd[1] |= FIELD_PREP(CMDQ_PRI_1_GRPID, ent->pri.grpid);
339 		switch (ent->pri.resp) {
340 		case PRI_RESP_DENY:
341 		case PRI_RESP_FAIL:
342 		case PRI_RESP_SUCC:
343 			break;
344 		default:
345 			return -EINVAL;
346 		}
347 		cmd[1] |= FIELD_PREP(CMDQ_PRI_1_RESP, ent->pri.resp);
348 		break;
349 	case CMDQ_OP_RESUME:
350 		cmd[0] |= FIELD_PREP(CMDQ_RESUME_0_SID, ent->resume.sid);
351 		cmd[0] |= FIELD_PREP(CMDQ_RESUME_0_RESP, ent->resume.resp);
352 		cmd[1] |= FIELD_PREP(CMDQ_RESUME_1_STAG, ent->resume.stag);
353 		break;
354 	case CMDQ_OP_CMD_SYNC:
355 		if (ent->sync.msiaddr) {
356 			cmd[0] |= FIELD_PREP(CMDQ_SYNC_0_CS, CMDQ_SYNC_0_CS_IRQ);
357 			cmd[1] |= ent->sync.msiaddr & CMDQ_SYNC_1_MSIADDR_MASK;
358 		} else {
359 			cmd[0] |= FIELD_PREP(CMDQ_SYNC_0_CS, CMDQ_SYNC_0_CS_SEV);
360 		}
361 		cmd[0] |= FIELD_PREP(CMDQ_SYNC_0_MSH, ARM_SMMU_SH_ISH);
362 		cmd[0] |= FIELD_PREP(CMDQ_SYNC_0_MSIATTR, ARM_SMMU_MEMATTR_OIWB);
363 		break;
364 	default:
365 		return -ENOENT;
366 	}
367 
368 	return 0;
369 }
370 
arm_smmu_get_cmdq(struct arm_smmu_device * smmu,struct arm_smmu_cmdq_ent * ent)371 static struct arm_smmu_cmdq *arm_smmu_get_cmdq(struct arm_smmu_device *smmu,
372 					       struct arm_smmu_cmdq_ent *ent)
373 {
374 	struct arm_smmu_cmdq *cmdq = NULL;
375 
376 	if (smmu->impl_ops && smmu->impl_ops->get_secondary_cmdq)
377 		cmdq = smmu->impl_ops->get_secondary_cmdq(smmu, ent);
378 
379 	return cmdq ?: &smmu->cmdq;
380 }
381 
arm_smmu_cmdq_needs_busy_polling(struct arm_smmu_device * smmu,struct arm_smmu_cmdq * cmdq)382 static bool arm_smmu_cmdq_needs_busy_polling(struct arm_smmu_device *smmu,
383 					     struct arm_smmu_cmdq *cmdq)
384 {
385 	if (cmdq == &smmu->cmdq)
386 		return false;
387 
388 	return smmu->options & ARM_SMMU_OPT_TEGRA241_CMDQV;
389 }
390 
arm_smmu_cmdq_build_sync_cmd(u64 * cmd,struct arm_smmu_device * smmu,struct arm_smmu_cmdq * cmdq,u32 prod)391 static void arm_smmu_cmdq_build_sync_cmd(u64 *cmd, struct arm_smmu_device *smmu,
392 					 struct arm_smmu_cmdq *cmdq, u32 prod)
393 {
394 	struct arm_smmu_queue *q = &cmdq->q;
395 	struct arm_smmu_cmdq_ent ent = {
396 		.opcode = CMDQ_OP_CMD_SYNC,
397 	};
398 
399 	/*
400 	 * Beware that Hi16xx adds an extra 32 bits of goodness to its MSI
401 	 * payload, so the write will zero the entire command on that platform.
402 	 */
403 	if (smmu->options & ARM_SMMU_OPT_MSIPOLL) {
404 		ent.sync.msiaddr = q->base_dma + Q_IDX(&q->llq, prod) *
405 				   q->ent_dwords * 8;
406 	}
407 
408 	arm_smmu_cmdq_build_cmd(cmd, &ent);
409 	if (arm_smmu_cmdq_needs_busy_polling(smmu, cmdq))
410 		u64p_replace_bits(cmd, CMDQ_SYNC_0_CS_NONE, CMDQ_SYNC_0_CS);
411 }
412 
__arm_smmu_cmdq_skip_err(struct arm_smmu_device * smmu,struct arm_smmu_cmdq * cmdq)413 void __arm_smmu_cmdq_skip_err(struct arm_smmu_device *smmu,
414 			      struct arm_smmu_cmdq *cmdq)
415 {
416 	static const char * const cerror_str[] = {
417 		[CMDQ_ERR_CERROR_NONE_IDX]	= "No error",
418 		[CMDQ_ERR_CERROR_ILL_IDX]	= "Illegal command",
419 		[CMDQ_ERR_CERROR_ABT_IDX]	= "Abort on command fetch",
420 		[CMDQ_ERR_CERROR_ATC_INV_IDX]	= "ATC invalidate timeout",
421 	};
422 	struct arm_smmu_queue *q = &cmdq->q;
423 
424 	int i;
425 	u64 cmd[CMDQ_ENT_DWORDS];
426 	u32 cons = readl_relaxed(q->cons_reg);
427 	u32 idx = FIELD_GET(CMDQ_CONS_ERR, cons);
428 	struct arm_smmu_cmdq_ent cmd_sync = {
429 		.opcode = CMDQ_OP_CMD_SYNC,
430 	};
431 
432 	dev_err(smmu->dev, "CMDQ error (cons 0x%08x): %s\n", cons,
433 		idx < ARRAY_SIZE(cerror_str) ?  cerror_str[idx] : "Unknown");
434 
435 	switch (idx) {
436 	case CMDQ_ERR_CERROR_ABT_IDX:
437 		dev_err(smmu->dev, "retrying command fetch\n");
438 		return;
439 	case CMDQ_ERR_CERROR_NONE_IDX:
440 		return;
441 	case CMDQ_ERR_CERROR_ATC_INV_IDX:
442 		/*
443 		 * ATC Invalidation Completion timeout. CONS is still pointing
444 		 * at the CMD_SYNC. Attempt to complete other pending commands
445 		 * by repeating the CMD_SYNC, though we might well end up back
446 		 * here since the ATC invalidation may still be pending.
447 		 */
448 		return;
449 	case CMDQ_ERR_CERROR_ILL_IDX:
450 	default:
451 		break;
452 	}
453 
454 	/*
455 	 * We may have concurrent producers, so we need to be careful
456 	 * not to touch any of the shadow cmdq state.
457 	 */
458 	queue_read(cmd, Q_ENT(q, cons), q->ent_dwords);
459 	dev_err(smmu->dev, "skipping command in error state:\n");
460 	for (i = 0; i < ARRAY_SIZE(cmd); ++i)
461 		dev_err(smmu->dev, "\t0x%016llx\n", (unsigned long long)cmd[i]);
462 
463 	/* Convert the erroneous command into a CMD_SYNC */
464 	arm_smmu_cmdq_build_cmd(cmd, &cmd_sync);
465 	if (arm_smmu_cmdq_needs_busy_polling(smmu, cmdq))
466 		u64p_replace_bits(cmd, CMDQ_SYNC_0_CS_NONE, CMDQ_SYNC_0_CS);
467 
468 	queue_write(Q_ENT(q, cons), cmd, q->ent_dwords);
469 }
470 
arm_smmu_cmdq_skip_err(struct arm_smmu_device * smmu)471 static void arm_smmu_cmdq_skip_err(struct arm_smmu_device *smmu)
472 {
473 	__arm_smmu_cmdq_skip_err(smmu, &smmu->cmdq);
474 }
475 
476 /*
477  * Command queue locking.
478  * This is a form of bastardised rwlock with the following major changes:
479  *
480  * - The only LOCK routines are exclusive_trylock() and shared_lock().
481  *   Neither have barrier semantics, and instead provide only a control
482  *   dependency.
483  *
484  * - The UNLOCK routines are supplemented with shared_tryunlock(), which
485  *   fails if the caller appears to be the last lock holder (yes, this is
486  *   racy). All successful UNLOCK routines have RELEASE semantics.
487  */
arm_smmu_cmdq_shared_lock(struct arm_smmu_cmdq * cmdq)488 static void arm_smmu_cmdq_shared_lock(struct arm_smmu_cmdq *cmdq)
489 {
490 	int val;
491 
492 	/*
493 	 * We can try to avoid the cmpxchg() loop by simply incrementing the
494 	 * lock counter. When held in exclusive state, the lock counter is set
495 	 * to INT_MIN so these increments won't hurt as the value will remain
496 	 * negative.
497 	 */
498 	if (atomic_fetch_inc_relaxed(&cmdq->lock) >= 0)
499 		return;
500 
501 	do {
502 		val = atomic_cond_read_relaxed(&cmdq->lock, VAL >= 0);
503 	} while (atomic_cmpxchg_relaxed(&cmdq->lock, val, val + 1) != val);
504 }
505 
arm_smmu_cmdq_shared_unlock(struct arm_smmu_cmdq * cmdq)506 static void arm_smmu_cmdq_shared_unlock(struct arm_smmu_cmdq *cmdq)
507 {
508 	(void)atomic_dec_return_release(&cmdq->lock);
509 }
510 
arm_smmu_cmdq_shared_tryunlock(struct arm_smmu_cmdq * cmdq)511 static bool arm_smmu_cmdq_shared_tryunlock(struct arm_smmu_cmdq *cmdq)
512 {
513 	if (atomic_read(&cmdq->lock) == 1)
514 		return false;
515 
516 	arm_smmu_cmdq_shared_unlock(cmdq);
517 	return true;
518 }
519 
520 #define arm_smmu_cmdq_exclusive_trylock_irqsave(cmdq, flags)		\
521 ({									\
522 	bool __ret;							\
523 	local_irq_save(flags);						\
524 	__ret = !atomic_cmpxchg_relaxed(&cmdq->lock, 0, INT_MIN);	\
525 	if (!__ret)							\
526 		local_irq_restore(flags);				\
527 	__ret;								\
528 })
529 
530 #define arm_smmu_cmdq_exclusive_unlock_irqrestore(cmdq, flags)		\
531 ({									\
532 	atomic_set_release(&cmdq->lock, 0);				\
533 	local_irq_restore(flags);					\
534 })
535 
536 
537 /*
538  * Command queue insertion.
539  * This is made fiddly by our attempts to achieve some sort of scalability
540  * since there is one queue shared amongst all of the CPUs in the system.  If
541  * you like mixed-size concurrency, dependency ordering and relaxed atomics,
542  * then you'll *love* this monstrosity.
543  *
544  * The basic idea is to split the queue up into ranges of commands that are
545  * owned by a given CPU; the owner may not have written all of the commands
546  * itself, but is responsible for advancing the hardware prod pointer when
547  * the time comes. The algorithm is roughly:
548  *
549  * 	1. Allocate some space in the queue. At this point we also discover
550  *	   whether the head of the queue is currently owned by another CPU,
551  *	   or whether we are the owner.
552  *
553  *	2. Write our commands into our allocated slots in the queue.
554  *
555  *	3. Mark our slots as valid in arm_smmu_cmdq.valid_map.
556  *
557  *	4. If we are an owner:
558  *		a. Wait for the previous owner to finish.
559  *		b. Mark the queue head as unowned, which tells us the range
560  *		   that we are responsible for publishing.
561  *		c. Wait for all commands in our owned range to become valid.
562  *		d. Advance the hardware prod pointer.
563  *		e. Tell the next owner we've finished.
564  *
565  *	5. If we are inserting a CMD_SYNC (we may or may not have been an
566  *	   owner), then we need to stick around until it has completed:
567  *		a. If we have MSIs, the SMMU can write back into the CMD_SYNC
568  *		   to clear the first 4 bytes.
569  *		b. Otherwise, we spin waiting for the hardware cons pointer to
570  *		   advance past our command.
571  *
572  * The devil is in the details, particularly the use of locking for handling
573  * SYNC completion and freeing up space in the queue before we think that it is
574  * full.
575  */
__arm_smmu_cmdq_poll_set_valid_map(struct arm_smmu_cmdq * cmdq,u32 sprod,u32 eprod,bool set)576 static void __arm_smmu_cmdq_poll_set_valid_map(struct arm_smmu_cmdq *cmdq,
577 					       u32 sprod, u32 eprod, bool set)
578 {
579 	u32 swidx, sbidx, ewidx, ebidx;
580 	struct arm_smmu_ll_queue llq = {
581 		.max_n_shift	= cmdq->q.llq.max_n_shift,
582 		.prod		= sprod,
583 	};
584 
585 	ewidx = BIT_WORD(Q_IDX(&llq, eprod));
586 	ebidx = Q_IDX(&llq, eprod) % BITS_PER_LONG;
587 
588 	while (llq.prod != eprod) {
589 		unsigned long mask;
590 		atomic_long_t *ptr;
591 		u32 limit = BITS_PER_LONG;
592 
593 		swidx = BIT_WORD(Q_IDX(&llq, llq.prod));
594 		sbidx = Q_IDX(&llq, llq.prod) % BITS_PER_LONG;
595 
596 		ptr = &cmdq->valid_map[swidx];
597 
598 		if ((swidx == ewidx) && (sbidx < ebidx))
599 			limit = ebidx;
600 
601 		mask = GENMASK(limit - 1, sbidx);
602 
603 		/*
604 		 * The valid bit is the inverse of the wrap bit. This means
605 		 * that a zero-initialised queue is invalid and, after marking
606 		 * all entries as valid, they become invalid again when we
607 		 * wrap.
608 		 */
609 		if (set) {
610 			atomic_long_xor(mask, ptr);
611 		} else { /* Poll */
612 			unsigned long valid;
613 
614 			valid = (ULONG_MAX + !!Q_WRP(&llq, llq.prod)) & mask;
615 			atomic_long_cond_read_relaxed(ptr, (VAL & mask) == valid);
616 		}
617 
618 		llq.prod = queue_inc_prod_n(&llq, limit - sbidx);
619 	}
620 }
621 
622 /* Mark all entries in the range [sprod, eprod) as valid */
arm_smmu_cmdq_set_valid_map(struct arm_smmu_cmdq * cmdq,u32 sprod,u32 eprod)623 static void arm_smmu_cmdq_set_valid_map(struct arm_smmu_cmdq *cmdq,
624 					u32 sprod, u32 eprod)
625 {
626 	__arm_smmu_cmdq_poll_set_valid_map(cmdq, sprod, eprod, true);
627 }
628 
629 /* Wait for all entries in the range [sprod, eprod) to become valid */
arm_smmu_cmdq_poll_valid_map(struct arm_smmu_cmdq * cmdq,u32 sprod,u32 eprod)630 static void arm_smmu_cmdq_poll_valid_map(struct arm_smmu_cmdq *cmdq,
631 					 u32 sprod, u32 eprod)
632 {
633 	__arm_smmu_cmdq_poll_set_valid_map(cmdq, sprod, eprod, false);
634 }
635 
636 /* Wait for the command queue to become non-full */
arm_smmu_cmdq_poll_until_not_full(struct arm_smmu_device * smmu,struct arm_smmu_cmdq * cmdq,struct arm_smmu_ll_queue * llq)637 static int arm_smmu_cmdq_poll_until_not_full(struct arm_smmu_device *smmu,
638 					     struct arm_smmu_cmdq *cmdq,
639 					     struct arm_smmu_ll_queue *llq)
640 {
641 	unsigned long flags;
642 	struct arm_smmu_queue_poll qp;
643 	int ret = 0;
644 
645 	/*
646 	 * Try to update our copy of cons by grabbing exclusive cmdq access. If
647 	 * that fails, spin until somebody else updates it for us.
648 	 */
649 	if (arm_smmu_cmdq_exclusive_trylock_irqsave(cmdq, flags)) {
650 		WRITE_ONCE(cmdq->q.llq.cons, readl_relaxed(cmdq->q.cons_reg));
651 		arm_smmu_cmdq_exclusive_unlock_irqrestore(cmdq, flags);
652 		llq->val = READ_ONCE(cmdq->q.llq.val);
653 		return 0;
654 	}
655 
656 	queue_poll_init(smmu, &qp);
657 	do {
658 		llq->val = READ_ONCE(cmdq->q.llq.val);
659 		if (!queue_full(llq))
660 			break;
661 
662 		ret = queue_poll(&qp);
663 	} while (!ret);
664 
665 	return ret;
666 }
667 
668 /*
669  * Wait until the SMMU signals a CMD_SYNC completion MSI.
670  * Must be called with the cmdq lock held in some capacity.
671  */
__arm_smmu_cmdq_poll_until_msi(struct arm_smmu_device * smmu,struct arm_smmu_cmdq * cmdq,struct arm_smmu_ll_queue * llq)672 static int __arm_smmu_cmdq_poll_until_msi(struct arm_smmu_device *smmu,
673 					  struct arm_smmu_cmdq *cmdq,
674 					  struct arm_smmu_ll_queue *llq)
675 {
676 	int ret = 0;
677 	struct arm_smmu_queue_poll qp;
678 	u32 *cmd = (u32 *)(Q_ENT(&cmdq->q, llq->prod));
679 
680 	queue_poll_init(smmu, &qp);
681 
682 	/*
683 	 * The MSI won't generate an event, since it's being written back
684 	 * into the command queue.
685 	 */
686 	qp.wfe = false;
687 	smp_cond_load_relaxed(cmd, !VAL || (ret = queue_poll(&qp)));
688 	llq->cons = ret ? llq->prod : queue_inc_prod_n(llq, 1);
689 	return ret;
690 }
691 
692 /*
693  * Wait until the SMMU cons index passes llq->prod.
694  * Must be called with the cmdq lock held in some capacity.
695  */
__arm_smmu_cmdq_poll_until_consumed(struct arm_smmu_device * smmu,struct arm_smmu_cmdq * cmdq,struct arm_smmu_ll_queue * llq)696 static int __arm_smmu_cmdq_poll_until_consumed(struct arm_smmu_device *smmu,
697 					       struct arm_smmu_cmdq *cmdq,
698 					       struct arm_smmu_ll_queue *llq)
699 {
700 	struct arm_smmu_queue_poll qp;
701 	u32 prod = llq->prod;
702 	int ret = 0;
703 
704 	queue_poll_init(smmu, &qp);
705 	llq->val = READ_ONCE(cmdq->q.llq.val);
706 	do {
707 		if (queue_consumed(llq, prod))
708 			break;
709 
710 		ret = queue_poll(&qp);
711 
712 		/*
713 		 * This needs to be a readl() so that our subsequent call
714 		 * to arm_smmu_cmdq_shared_tryunlock() can fail accurately.
715 		 *
716 		 * Specifically, we need to ensure that we observe all
717 		 * shared_lock()s by other CMD_SYNCs that share our owner,
718 		 * so that a failing call to tryunlock() means that we're
719 		 * the last one out and therefore we can safely advance
720 		 * cmdq->q.llq.cons. Roughly speaking:
721 		 *
722 		 * CPU 0		CPU1			CPU2 (us)
723 		 *
724 		 * if (sync)
725 		 * 	shared_lock();
726 		 *
727 		 * dma_wmb();
728 		 * set_valid_map();
729 		 *
730 		 * 			if (owner) {
731 		 *				poll_valid_map();
732 		 *				<control dependency>
733 		 *				writel(prod_reg);
734 		 *
735 		 *						readl(cons_reg);
736 		 *						tryunlock();
737 		 *
738 		 * Requires us to see CPU 0's shared_lock() acquisition.
739 		 */
740 		llq->cons = readl(cmdq->q.cons_reg);
741 	} while (!ret);
742 
743 	return ret;
744 }
745 
arm_smmu_cmdq_poll_until_sync(struct arm_smmu_device * smmu,struct arm_smmu_cmdq * cmdq,struct arm_smmu_ll_queue * llq)746 static int arm_smmu_cmdq_poll_until_sync(struct arm_smmu_device *smmu,
747 					 struct arm_smmu_cmdq *cmdq,
748 					 struct arm_smmu_ll_queue *llq)
749 {
750 	if (smmu->options & ARM_SMMU_OPT_MSIPOLL &&
751 	    !arm_smmu_cmdq_needs_busy_polling(smmu, cmdq))
752 		return __arm_smmu_cmdq_poll_until_msi(smmu, cmdq, llq);
753 
754 	return __arm_smmu_cmdq_poll_until_consumed(smmu, cmdq, llq);
755 }
756 
arm_smmu_cmdq_write_entries(struct arm_smmu_cmdq * cmdq,u64 * cmds,u32 prod,int n)757 static void arm_smmu_cmdq_write_entries(struct arm_smmu_cmdq *cmdq, u64 *cmds,
758 					u32 prod, int n)
759 {
760 	int i;
761 	struct arm_smmu_ll_queue llq = {
762 		.max_n_shift	= cmdq->q.llq.max_n_shift,
763 		.prod		= prod,
764 	};
765 
766 	for (i = 0; i < n; ++i) {
767 		u64 *cmd = &cmds[i * CMDQ_ENT_DWORDS];
768 
769 		prod = queue_inc_prod_n(&llq, i);
770 		queue_write(Q_ENT(&cmdq->q, prod), cmd, CMDQ_ENT_DWORDS);
771 	}
772 }
773 
774 /*
775  * This is the actual insertion function, and provides the following
776  * ordering guarantees to callers:
777  *
778  * - There is a dma_wmb() before publishing any commands to the queue.
779  *   This can be relied upon to order prior writes to data structures
780  *   in memory (such as a CD or an STE) before the command.
781  *
782  * - On completion of a CMD_SYNC, there is a control dependency.
783  *   This can be relied upon to order subsequent writes to memory (e.g.
784  *   freeing an IOVA) after completion of the CMD_SYNC.
785  *
786  * - Command insertion is totally ordered, so if two CPUs each race to
787  *   insert their own list of commands then all of the commands from one
788  *   CPU will appear before any of the commands from the other CPU.
789  */
arm_smmu_cmdq_issue_cmdlist(struct arm_smmu_device * smmu,struct arm_smmu_cmdq * cmdq,u64 * cmds,int n,bool sync)790 int arm_smmu_cmdq_issue_cmdlist(struct arm_smmu_device *smmu,
791 				struct arm_smmu_cmdq *cmdq, u64 *cmds, int n,
792 				bool sync)
793 {
794 	u64 cmd_sync[CMDQ_ENT_DWORDS];
795 	u32 prod;
796 	unsigned long flags;
797 	bool owner;
798 	struct arm_smmu_ll_queue llq, head;
799 	int ret = 0;
800 
801 	llq.max_n_shift = cmdq->q.llq.max_n_shift;
802 
803 	/* 1. Allocate some space in the queue */
804 	local_irq_save(flags);
805 	llq.val = READ_ONCE(cmdq->q.llq.val);
806 	do {
807 		u64 old;
808 
809 		while (!queue_has_space(&llq, n + sync)) {
810 			local_irq_restore(flags);
811 			if (arm_smmu_cmdq_poll_until_not_full(smmu, cmdq, &llq))
812 				dev_err_ratelimited(smmu->dev, "CMDQ timeout\n");
813 			local_irq_save(flags);
814 		}
815 
816 		head.cons = llq.cons;
817 		head.prod = queue_inc_prod_n(&llq, n + sync) |
818 					     CMDQ_PROD_OWNED_FLAG;
819 
820 		old = cmpxchg_relaxed(&cmdq->q.llq.val, llq.val, head.val);
821 		if (old == llq.val)
822 			break;
823 
824 		llq.val = old;
825 	} while (1);
826 	owner = !(llq.prod & CMDQ_PROD_OWNED_FLAG);
827 	head.prod &= ~CMDQ_PROD_OWNED_FLAG;
828 	llq.prod &= ~CMDQ_PROD_OWNED_FLAG;
829 
830 	/*
831 	 * 2. Write our commands into the queue
832 	 * Dependency ordering from the cmpxchg() loop above.
833 	 */
834 	arm_smmu_cmdq_write_entries(cmdq, cmds, llq.prod, n);
835 	if (sync) {
836 		prod = queue_inc_prod_n(&llq, n);
837 		arm_smmu_cmdq_build_sync_cmd(cmd_sync, smmu, cmdq, prod);
838 		queue_write(Q_ENT(&cmdq->q, prod), cmd_sync, CMDQ_ENT_DWORDS);
839 
840 		/*
841 		 * In order to determine completion of our CMD_SYNC, we must
842 		 * ensure that the queue can't wrap twice without us noticing.
843 		 * We achieve that by taking the cmdq lock as shared before
844 		 * marking our slot as valid.
845 		 */
846 		arm_smmu_cmdq_shared_lock(cmdq);
847 	}
848 
849 	/* 3. Mark our slots as valid, ensuring commands are visible first */
850 	dma_wmb();
851 	arm_smmu_cmdq_set_valid_map(cmdq, llq.prod, head.prod);
852 
853 	/* 4. If we are the owner, take control of the SMMU hardware */
854 	if (owner) {
855 		/* a. Wait for previous owner to finish */
856 		atomic_cond_read_relaxed(&cmdq->owner_prod, VAL == llq.prod);
857 
858 		/* b. Stop gathering work by clearing the owned flag */
859 		prod = atomic_fetch_andnot_relaxed(CMDQ_PROD_OWNED_FLAG,
860 						   &cmdq->q.llq.atomic.prod);
861 		prod &= ~CMDQ_PROD_OWNED_FLAG;
862 
863 		/*
864 		 * c. Wait for any gathered work to be written to the queue.
865 		 * Note that we read our own entries so that we have the control
866 		 * dependency required by (d).
867 		 */
868 		arm_smmu_cmdq_poll_valid_map(cmdq, llq.prod, prod);
869 
870 		/*
871 		 * d. Advance the hardware prod pointer
872 		 * Control dependency ordering from the entries becoming valid.
873 		 */
874 		writel_relaxed(prod, cmdq->q.prod_reg);
875 
876 		/*
877 		 * e. Tell the next owner we're done
878 		 * Make sure we've updated the hardware first, so that we don't
879 		 * race to update prod and potentially move it backwards.
880 		 */
881 		atomic_set_release(&cmdq->owner_prod, prod);
882 	}
883 
884 	/* 5. If we are inserting a CMD_SYNC, we must wait for it to complete */
885 	if (sync) {
886 		llq.prod = queue_inc_prod_n(&llq, n);
887 		ret = arm_smmu_cmdq_poll_until_sync(smmu, cmdq, &llq);
888 		if (ret) {
889 			dev_err_ratelimited(smmu->dev,
890 					    "CMD_SYNC timeout at 0x%08x [hwprod 0x%08x, hwcons 0x%08x]\n",
891 					    llq.prod,
892 					    readl_relaxed(cmdq->q.prod_reg),
893 					    readl_relaxed(cmdq->q.cons_reg));
894 		}
895 
896 		/*
897 		 * Try to unlock the cmdq lock. This will fail if we're the last
898 		 * reader, in which case we can safely update cmdq->q.llq.cons
899 		 */
900 		if (!arm_smmu_cmdq_shared_tryunlock(cmdq)) {
901 			WRITE_ONCE(cmdq->q.llq.cons, llq.cons);
902 			arm_smmu_cmdq_shared_unlock(cmdq);
903 		}
904 	}
905 
906 	local_irq_restore(flags);
907 	return ret;
908 }
909 
__arm_smmu_cmdq_issue_cmd(struct arm_smmu_device * smmu,struct arm_smmu_cmdq_ent * ent,bool sync)910 static int __arm_smmu_cmdq_issue_cmd(struct arm_smmu_device *smmu,
911 				     struct arm_smmu_cmdq_ent *ent,
912 				     bool sync)
913 {
914 	u64 cmd[CMDQ_ENT_DWORDS];
915 
916 	if (unlikely(arm_smmu_cmdq_build_cmd(cmd, ent))) {
917 		dev_warn(smmu->dev, "ignoring unknown CMDQ opcode 0x%x\n",
918 			 ent->opcode);
919 		return -EINVAL;
920 	}
921 
922 	return arm_smmu_cmdq_issue_cmdlist(
923 		smmu, arm_smmu_get_cmdq(smmu, ent), cmd, 1, sync);
924 }
925 
arm_smmu_cmdq_issue_cmd(struct arm_smmu_device * smmu,struct arm_smmu_cmdq_ent * ent)926 static int arm_smmu_cmdq_issue_cmd(struct arm_smmu_device *smmu,
927 				   struct arm_smmu_cmdq_ent *ent)
928 {
929 	return __arm_smmu_cmdq_issue_cmd(smmu, ent, false);
930 }
931 
arm_smmu_cmdq_issue_cmd_with_sync(struct arm_smmu_device * smmu,struct arm_smmu_cmdq_ent * ent)932 static int arm_smmu_cmdq_issue_cmd_with_sync(struct arm_smmu_device *smmu,
933 					     struct arm_smmu_cmdq_ent *ent)
934 {
935 	return __arm_smmu_cmdq_issue_cmd(smmu, ent, true);
936 }
937 
arm_smmu_cmdq_batch_init(struct arm_smmu_device * smmu,struct arm_smmu_cmdq_batch * cmds,struct arm_smmu_cmdq_ent * ent)938 static void arm_smmu_cmdq_batch_init(struct arm_smmu_device *smmu,
939 				     struct arm_smmu_cmdq_batch *cmds,
940 				     struct arm_smmu_cmdq_ent *ent)
941 {
942 	cmds->num = 0;
943 	cmds->cmdq = arm_smmu_get_cmdq(smmu, ent);
944 }
945 
arm_smmu_cmdq_batch_add(struct arm_smmu_device * smmu,struct arm_smmu_cmdq_batch * cmds,struct arm_smmu_cmdq_ent * cmd)946 static void arm_smmu_cmdq_batch_add(struct arm_smmu_device *smmu,
947 				    struct arm_smmu_cmdq_batch *cmds,
948 				    struct arm_smmu_cmdq_ent *cmd)
949 {
950 	bool unsupported_cmd = !arm_smmu_cmdq_supports_cmd(cmds->cmdq, cmd);
951 	bool force_sync = (cmds->num == CMDQ_BATCH_ENTRIES - 1) &&
952 			  (smmu->options & ARM_SMMU_OPT_CMDQ_FORCE_SYNC);
953 	int index;
954 
955 	if (force_sync || unsupported_cmd) {
956 		arm_smmu_cmdq_issue_cmdlist(smmu, cmds->cmdq, cmds->cmds,
957 					    cmds->num, true);
958 		arm_smmu_cmdq_batch_init(smmu, cmds, cmd);
959 	}
960 
961 	if (cmds->num == CMDQ_BATCH_ENTRIES) {
962 		arm_smmu_cmdq_issue_cmdlist(smmu, cmds->cmdq, cmds->cmds,
963 					    cmds->num, false);
964 		arm_smmu_cmdq_batch_init(smmu, cmds, cmd);
965 	}
966 
967 	index = cmds->num * CMDQ_ENT_DWORDS;
968 	if (unlikely(arm_smmu_cmdq_build_cmd(&cmds->cmds[index], cmd))) {
969 		dev_warn(smmu->dev, "ignoring unknown CMDQ opcode 0x%x\n",
970 			 cmd->opcode);
971 		return;
972 	}
973 
974 	cmds->num++;
975 }
976 
arm_smmu_cmdq_batch_submit(struct arm_smmu_device * smmu,struct arm_smmu_cmdq_batch * cmds)977 static int arm_smmu_cmdq_batch_submit(struct arm_smmu_device *smmu,
978 				      struct arm_smmu_cmdq_batch *cmds)
979 {
980 	return arm_smmu_cmdq_issue_cmdlist(smmu, cmds->cmdq, cmds->cmds,
981 					   cmds->num, true);
982 }
983 
arm_smmu_page_response(struct device * dev,struct iopf_fault * unused,struct iommu_page_response * resp)984 static void arm_smmu_page_response(struct device *dev, struct iopf_fault *unused,
985 				   struct iommu_page_response *resp)
986 {
987 	struct arm_smmu_cmdq_ent cmd = {0};
988 	struct arm_smmu_master *master = dev_iommu_priv_get(dev);
989 	int sid = master->streams[0].id;
990 
991 	if (WARN_ON(!master->stall_enabled))
992 		return;
993 
994 	cmd.opcode		= CMDQ_OP_RESUME;
995 	cmd.resume.sid		= sid;
996 	cmd.resume.stag		= resp->grpid;
997 	switch (resp->code) {
998 	case IOMMU_PAGE_RESP_INVALID:
999 	case IOMMU_PAGE_RESP_FAILURE:
1000 		cmd.resume.resp = CMDQ_RESUME_0_RESP_ABORT;
1001 		break;
1002 	case IOMMU_PAGE_RESP_SUCCESS:
1003 		cmd.resume.resp = CMDQ_RESUME_0_RESP_RETRY;
1004 		break;
1005 	default:
1006 		break;
1007 	}
1008 
1009 	arm_smmu_cmdq_issue_cmd(master->smmu, &cmd);
1010 	/*
1011 	 * Don't send a SYNC, it doesn't do anything for RESUME or PRI_RESP.
1012 	 * RESUME consumption guarantees that the stalled transaction will be
1013 	 * terminated... at some point in the future. PRI_RESP is fire and
1014 	 * forget.
1015 	 */
1016 }
1017 
1018 /* Context descriptor manipulation functions */
arm_smmu_tlb_inv_asid(struct arm_smmu_device * smmu,u16 asid)1019 void arm_smmu_tlb_inv_asid(struct arm_smmu_device *smmu, u16 asid)
1020 {
1021 	struct arm_smmu_cmdq_ent cmd = {
1022 		.opcode	= smmu->features & ARM_SMMU_FEAT_E2H ?
1023 			CMDQ_OP_TLBI_EL2_ASID : CMDQ_OP_TLBI_NH_ASID,
1024 		.tlbi.asid = asid,
1025 	};
1026 
1027 	arm_smmu_cmdq_issue_cmd_with_sync(smmu, &cmd);
1028 }
1029 
1030 /*
1031  * Based on the value of ent report which bits of the STE the HW will access. It
1032  * would be nice if this was complete according to the spec, but minimally it
1033  * has to capture the bits this driver uses.
1034  */
1035 VISIBLE_IF_KUNIT
arm_smmu_get_ste_used(const __le64 * ent,__le64 * used_bits)1036 void arm_smmu_get_ste_used(const __le64 *ent, __le64 *used_bits)
1037 {
1038 	unsigned int cfg = FIELD_GET(STRTAB_STE_0_CFG, le64_to_cpu(ent[0]));
1039 
1040 	used_bits[0] = cpu_to_le64(STRTAB_STE_0_V);
1041 	if (!(ent[0] & cpu_to_le64(STRTAB_STE_0_V)))
1042 		return;
1043 
1044 	used_bits[0] |= cpu_to_le64(STRTAB_STE_0_CFG);
1045 
1046 	/* S1 translates */
1047 	if (cfg & BIT(0)) {
1048 		used_bits[0] |= cpu_to_le64(STRTAB_STE_0_S1FMT |
1049 					    STRTAB_STE_0_S1CTXPTR_MASK |
1050 					    STRTAB_STE_0_S1CDMAX);
1051 		used_bits[1] |=
1052 			cpu_to_le64(STRTAB_STE_1_S1DSS | STRTAB_STE_1_S1CIR |
1053 				    STRTAB_STE_1_S1COR | STRTAB_STE_1_S1CSH |
1054 				    STRTAB_STE_1_S1STALLD | STRTAB_STE_1_STRW |
1055 				    STRTAB_STE_1_EATS | STRTAB_STE_1_MEV);
1056 		used_bits[2] |= cpu_to_le64(STRTAB_STE_2_S2VMID);
1057 
1058 		/*
1059 		 * See 13.5 Summary of attribute/permission configuration fields
1060 		 * for the SHCFG behavior.
1061 		 */
1062 		if (FIELD_GET(STRTAB_STE_1_S1DSS, le64_to_cpu(ent[1])) ==
1063 		    STRTAB_STE_1_S1DSS_BYPASS)
1064 			used_bits[1] |= cpu_to_le64(STRTAB_STE_1_SHCFG);
1065 	}
1066 
1067 	/* S2 translates */
1068 	if (cfg & BIT(1)) {
1069 		used_bits[1] |=
1070 			cpu_to_le64(STRTAB_STE_1_S2FWB | STRTAB_STE_1_EATS |
1071 				    STRTAB_STE_1_SHCFG | STRTAB_STE_1_MEV);
1072 		used_bits[2] |=
1073 			cpu_to_le64(STRTAB_STE_2_S2VMID | STRTAB_STE_2_VTCR |
1074 				    STRTAB_STE_2_S2AA64 | STRTAB_STE_2_S2ENDI |
1075 				    STRTAB_STE_2_S2PTW | STRTAB_STE_2_S2S |
1076 				    STRTAB_STE_2_S2R);
1077 		used_bits[3] |= cpu_to_le64(STRTAB_STE_3_S2TTB_MASK);
1078 	}
1079 
1080 	if (cfg == STRTAB_STE_0_CFG_BYPASS)
1081 		used_bits[1] |= cpu_to_le64(STRTAB_STE_1_SHCFG);
1082 }
1083 EXPORT_SYMBOL_IF_KUNIT(arm_smmu_get_ste_used);
1084 
1085 /*
1086  * Figure out if we can do a hitless update of entry to become target. Returns a
1087  * bit mask where 1 indicates that qword needs to be set disruptively.
1088  * unused_update is an intermediate value of entry that has unused bits set to
1089  * their new values.
1090  */
arm_smmu_entry_qword_diff(struct arm_smmu_entry_writer * writer,const __le64 * entry,const __le64 * target,__le64 * unused_update)1091 static u8 arm_smmu_entry_qword_diff(struct arm_smmu_entry_writer *writer,
1092 				    const __le64 *entry, const __le64 *target,
1093 				    __le64 *unused_update)
1094 {
1095 	__le64 target_used[NUM_ENTRY_QWORDS] = {};
1096 	__le64 cur_used[NUM_ENTRY_QWORDS] = {};
1097 	u8 used_qword_diff = 0;
1098 	unsigned int i;
1099 
1100 	writer->ops->get_used(entry, cur_used);
1101 	writer->ops->get_used(target, target_used);
1102 
1103 	for (i = 0; i != NUM_ENTRY_QWORDS; i++) {
1104 		/*
1105 		 * Check that masks are up to date, the make functions are not
1106 		 * allowed to set a bit to 1 if the used function doesn't say it
1107 		 * is used.
1108 		 */
1109 		WARN_ON_ONCE(target[i] & ~target_used[i]);
1110 
1111 		/* Bits can change because they are not currently being used */
1112 		unused_update[i] = (entry[i] & cur_used[i]) |
1113 				   (target[i] & ~cur_used[i]);
1114 		/*
1115 		 * Each bit indicates that a used bit in a qword needs to be
1116 		 * changed after unused_update is applied.
1117 		 */
1118 		if ((unused_update[i] & target_used[i]) != target[i])
1119 			used_qword_diff |= 1 << i;
1120 	}
1121 	return used_qword_diff;
1122 }
1123 
entry_set(struct arm_smmu_entry_writer * writer,__le64 * entry,const __le64 * target,unsigned int start,unsigned int len)1124 static bool entry_set(struct arm_smmu_entry_writer *writer, __le64 *entry,
1125 		      const __le64 *target, unsigned int start,
1126 		      unsigned int len)
1127 {
1128 	bool changed = false;
1129 	unsigned int i;
1130 
1131 	for (i = start; len != 0; len--, i++) {
1132 		if (entry[i] != target[i]) {
1133 			WRITE_ONCE(entry[i], target[i]);
1134 			changed = true;
1135 		}
1136 	}
1137 
1138 	if (changed)
1139 		writer->ops->sync(writer);
1140 	return changed;
1141 }
1142 
1143 /*
1144  * Update the STE/CD to the target configuration. The transition from the
1145  * current entry to the target entry takes place over multiple steps that
1146  * attempts to make the transition hitless if possible. This function takes care
1147  * not to create a situation where the HW can perceive a corrupted entry. HW is
1148  * only required to have a 64 bit atomicity with stores from the CPU, while
1149  * entries are many 64 bit values big.
1150  *
1151  * The difference between the current value and the target value is analyzed to
1152  * determine which of three updates are required - disruptive, hitless or no
1153  * change.
1154  *
1155  * In the most general disruptive case we can make any update in three steps:
1156  *  - Disrupting the entry (V=0)
1157  *  - Fill now unused qwords, execpt qword 0 which contains V
1158  *  - Make qword 0 have the final value and valid (V=1) with a single 64
1159  *    bit store
1160  *
1161  * However this disrupts the HW while it is happening. There are several
1162  * interesting cases where a STE/CD can be updated without disturbing the HW
1163  * because only a small number of bits are changing (S1DSS, CONFIG, etc) or
1164  * because the used bits don't intersect. We can detect this by calculating how
1165  * many 64 bit values need update after adjusting the unused bits and skip the
1166  * V=0 process. This relies on the IGNORED behavior described in the
1167  * specification.
1168  */
1169 VISIBLE_IF_KUNIT
arm_smmu_write_entry(struct arm_smmu_entry_writer * writer,__le64 * entry,const __le64 * target)1170 void arm_smmu_write_entry(struct arm_smmu_entry_writer *writer, __le64 *entry,
1171 			  const __le64 *target)
1172 {
1173 	__le64 unused_update[NUM_ENTRY_QWORDS];
1174 	u8 used_qword_diff;
1175 
1176 	used_qword_diff =
1177 		arm_smmu_entry_qword_diff(writer, entry, target, unused_update);
1178 	if (hweight8(used_qword_diff) == 1) {
1179 		/*
1180 		 * Only one qword needs its used bits to be changed. This is a
1181 		 * hitless update, update all bits the current STE/CD is
1182 		 * ignoring to their new values, then update a single "critical
1183 		 * qword" to change the STE/CD and finally 0 out any bits that
1184 		 * are now unused in the target configuration.
1185 		 */
1186 		unsigned int critical_qword_index = ffs(used_qword_diff) - 1;
1187 
1188 		/*
1189 		 * Skip writing unused bits in the critical qword since we'll be
1190 		 * writing it in the next step anyways. This can save a sync
1191 		 * when the only change is in that qword.
1192 		 */
1193 		unused_update[critical_qword_index] =
1194 			entry[critical_qword_index];
1195 		entry_set(writer, entry, unused_update, 0, NUM_ENTRY_QWORDS);
1196 		entry_set(writer, entry, target, critical_qword_index, 1);
1197 		entry_set(writer, entry, target, 0, NUM_ENTRY_QWORDS);
1198 	} else if (used_qword_diff) {
1199 		/*
1200 		 * At least two qwords need their inuse bits to be changed. This
1201 		 * requires a breaking update, zero the V bit, write all qwords
1202 		 * but 0, then set qword 0
1203 		 */
1204 		unused_update[0] = 0;
1205 		entry_set(writer, entry, unused_update, 0, 1);
1206 		entry_set(writer, entry, target, 1, NUM_ENTRY_QWORDS - 1);
1207 		entry_set(writer, entry, target, 0, 1);
1208 	} else {
1209 		/*
1210 		 * No inuse bit changed. Sanity check that all unused bits are 0
1211 		 * in the entry. The target was already sanity checked by
1212 		 * compute_qword_diff().
1213 		 */
1214 		WARN_ON_ONCE(
1215 			entry_set(writer, entry, target, 0, NUM_ENTRY_QWORDS));
1216 	}
1217 }
1218 EXPORT_SYMBOL_IF_KUNIT(arm_smmu_write_entry);
1219 
arm_smmu_sync_cd(struct arm_smmu_master * master,int ssid,bool leaf)1220 static void arm_smmu_sync_cd(struct arm_smmu_master *master,
1221 			     int ssid, bool leaf)
1222 {
1223 	size_t i;
1224 	struct arm_smmu_cmdq_batch cmds;
1225 	struct arm_smmu_device *smmu = master->smmu;
1226 	struct arm_smmu_cmdq_ent cmd = {
1227 		.opcode	= CMDQ_OP_CFGI_CD,
1228 		.cfgi	= {
1229 			.ssid	= ssid,
1230 			.leaf	= leaf,
1231 		},
1232 	};
1233 
1234 	arm_smmu_cmdq_batch_init(smmu, &cmds, &cmd);
1235 	for (i = 0; i < master->num_streams; i++) {
1236 		cmd.cfgi.sid = master->streams[i].id;
1237 		arm_smmu_cmdq_batch_add(smmu, &cmds, &cmd);
1238 	}
1239 
1240 	arm_smmu_cmdq_batch_submit(smmu, &cmds);
1241 }
1242 
arm_smmu_write_cd_l1_desc(struct arm_smmu_cdtab_l1 * dst,dma_addr_t l2ptr_dma)1243 static void arm_smmu_write_cd_l1_desc(struct arm_smmu_cdtab_l1 *dst,
1244 				      dma_addr_t l2ptr_dma)
1245 {
1246 	u64 val = (l2ptr_dma & CTXDESC_L1_DESC_L2PTR_MASK) | CTXDESC_L1_DESC_V;
1247 
1248 	/* The HW has 64 bit atomicity with stores to the L2 CD table */
1249 	WRITE_ONCE(dst->l2ptr, cpu_to_le64(val));
1250 }
1251 
arm_smmu_cd_l1_get_desc(const struct arm_smmu_cdtab_l1 * src)1252 static dma_addr_t arm_smmu_cd_l1_get_desc(const struct arm_smmu_cdtab_l1 *src)
1253 {
1254 	return le64_to_cpu(src->l2ptr) & CTXDESC_L1_DESC_L2PTR_MASK;
1255 }
1256 
arm_smmu_get_cd_ptr(struct arm_smmu_master * master,u32 ssid)1257 struct arm_smmu_cd *arm_smmu_get_cd_ptr(struct arm_smmu_master *master,
1258 					u32 ssid)
1259 {
1260 	struct arm_smmu_cdtab_l2 *l2;
1261 	struct arm_smmu_ctx_desc_cfg *cd_table = &master->cd_table;
1262 
1263 	if (!arm_smmu_cdtab_allocated(cd_table))
1264 		return NULL;
1265 
1266 	if (cd_table->s1fmt == STRTAB_STE_0_S1FMT_LINEAR)
1267 		return &cd_table->linear.table[ssid];
1268 
1269 	l2 = cd_table->l2.l2ptrs[arm_smmu_cdtab_l1_idx(ssid)];
1270 	if (!l2)
1271 		return NULL;
1272 	return &l2->cds[arm_smmu_cdtab_l2_idx(ssid)];
1273 }
1274 
arm_smmu_alloc_cd_ptr(struct arm_smmu_master * master,u32 ssid)1275 static struct arm_smmu_cd *arm_smmu_alloc_cd_ptr(struct arm_smmu_master *master,
1276 						 u32 ssid)
1277 {
1278 	struct arm_smmu_ctx_desc_cfg *cd_table = &master->cd_table;
1279 	struct arm_smmu_device *smmu = master->smmu;
1280 
1281 	might_sleep();
1282 	iommu_group_mutex_assert(master->dev);
1283 
1284 	if (!arm_smmu_cdtab_allocated(cd_table)) {
1285 		if (arm_smmu_alloc_cd_tables(master))
1286 			return NULL;
1287 	}
1288 
1289 	if (cd_table->s1fmt == STRTAB_STE_0_S1FMT_64K_L2) {
1290 		unsigned int idx = arm_smmu_cdtab_l1_idx(ssid);
1291 		struct arm_smmu_cdtab_l2 **l2ptr = &cd_table->l2.l2ptrs[idx];
1292 
1293 		if (!*l2ptr) {
1294 			dma_addr_t l2ptr_dma;
1295 
1296 			*l2ptr = dma_alloc_coherent(smmu->dev, sizeof(**l2ptr),
1297 						    &l2ptr_dma, GFP_KERNEL);
1298 			if (!*l2ptr)
1299 				return NULL;
1300 
1301 			arm_smmu_write_cd_l1_desc(&cd_table->l2.l1tab[idx],
1302 						  l2ptr_dma);
1303 			/* An invalid L1CD can be cached */
1304 			arm_smmu_sync_cd(master, ssid, false);
1305 		}
1306 	}
1307 	return arm_smmu_get_cd_ptr(master, ssid);
1308 }
1309 
1310 struct arm_smmu_cd_writer {
1311 	struct arm_smmu_entry_writer writer;
1312 	unsigned int ssid;
1313 };
1314 
1315 VISIBLE_IF_KUNIT
arm_smmu_get_cd_used(const __le64 * ent,__le64 * used_bits)1316 void arm_smmu_get_cd_used(const __le64 *ent, __le64 *used_bits)
1317 {
1318 	used_bits[0] = cpu_to_le64(CTXDESC_CD_0_V);
1319 	if (!(ent[0] & cpu_to_le64(CTXDESC_CD_0_V)))
1320 		return;
1321 	memset(used_bits, 0xFF, sizeof(struct arm_smmu_cd));
1322 
1323 	/*
1324 	 * If EPD0 is set by the make function it means
1325 	 * T0SZ/TG0/IR0/OR0/SH0/TTB0 are IGNORED
1326 	 */
1327 	if (ent[0] & cpu_to_le64(CTXDESC_CD_0_TCR_EPD0)) {
1328 		used_bits[0] &= ~cpu_to_le64(
1329 			CTXDESC_CD_0_TCR_T0SZ | CTXDESC_CD_0_TCR_TG0 |
1330 			CTXDESC_CD_0_TCR_IRGN0 | CTXDESC_CD_0_TCR_ORGN0 |
1331 			CTXDESC_CD_0_TCR_SH0);
1332 		used_bits[1] &= ~cpu_to_le64(CTXDESC_CD_1_TTB0_MASK);
1333 	}
1334 }
1335 EXPORT_SYMBOL_IF_KUNIT(arm_smmu_get_cd_used);
1336 
arm_smmu_cd_writer_sync_entry(struct arm_smmu_entry_writer * writer)1337 static void arm_smmu_cd_writer_sync_entry(struct arm_smmu_entry_writer *writer)
1338 {
1339 	struct arm_smmu_cd_writer *cd_writer =
1340 		container_of(writer, struct arm_smmu_cd_writer, writer);
1341 
1342 	arm_smmu_sync_cd(writer->master, cd_writer->ssid, true);
1343 }
1344 
1345 static const struct arm_smmu_entry_writer_ops arm_smmu_cd_writer_ops = {
1346 	.sync = arm_smmu_cd_writer_sync_entry,
1347 	.get_used = arm_smmu_get_cd_used,
1348 };
1349 
arm_smmu_write_cd_entry(struct arm_smmu_master * master,int ssid,struct arm_smmu_cd * cdptr,const struct arm_smmu_cd * target)1350 void arm_smmu_write_cd_entry(struct arm_smmu_master *master, int ssid,
1351 			     struct arm_smmu_cd *cdptr,
1352 			     const struct arm_smmu_cd *target)
1353 {
1354 	bool target_valid = target->data[0] & cpu_to_le64(CTXDESC_CD_0_V);
1355 	bool cur_valid = cdptr->data[0] & cpu_to_le64(CTXDESC_CD_0_V);
1356 	struct arm_smmu_cd_writer cd_writer = {
1357 		.writer = {
1358 			.ops = &arm_smmu_cd_writer_ops,
1359 			.master = master,
1360 		},
1361 		.ssid = ssid,
1362 	};
1363 
1364 	if (ssid != IOMMU_NO_PASID && cur_valid != target_valid) {
1365 		if (cur_valid)
1366 			master->cd_table.used_ssids--;
1367 		else
1368 			master->cd_table.used_ssids++;
1369 	}
1370 
1371 	arm_smmu_write_entry(&cd_writer.writer, cdptr->data, target->data);
1372 }
1373 
arm_smmu_make_s1_cd(struct arm_smmu_cd * target,struct arm_smmu_master * master,struct arm_smmu_domain * smmu_domain)1374 void arm_smmu_make_s1_cd(struct arm_smmu_cd *target,
1375 			 struct arm_smmu_master *master,
1376 			 struct arm_smmu_domain *smmu_domain)
1377 {
1378 	struct arm_smmu_ctx_desc *cd = &smmu_domain->cd;
1379 	const struct io_pgtable_cfg *pgtbl_cfg =
1380 		&io_pgtable_ops_to_pgtable(smmu_domain->pgtbl_ops)->cfg;
1381 	typeof(&pgtbl_cfg->arm_lpae_s1_cfg.tcr) tcr =
1382 		&pgtbl_cfg->arm_lpae_s1_cfg.tcr;
1383 
1384 	memset(target, 0, sizeof(*target));
1385 
1386 	target->data[0] = cpu_to_le64(
1387 		FIELD_PREP(CTXDESC_CD_0_TCR_T0SZ, tcr->tsz) |
1388 		FIELD_PREP(CTXDESC_CD_0_TCR_TG0, tcr->tg) |
1389 		FIELD_PREP(CTXDESC_CD_0_TCR_IRGN0, tcr->irgn) |
1390 		FIELD_PREP(CTXDESC_CD_0_TCR_ORGN0, tcr->orgn) |
1391 		FIELD_PREP(CTXDESC_CD_0_TCR_SH0, tcr->sh) |
1392 #ifdef __BIG_ENDIAN
1393 		CTXDESC_CD_0_ENDI |
1394 #endif
1395 		CTXDESC_CD_0_TCR_EPD1 |
1396 		CTXDESC_CD_0_V |
1397 		FIELD_PREP(CTXDESC_CD_0_TCR_IPS, tcr->ips) |
1398 		CTXDESC_CD_0_AA64 |
1399 		(master->stall_enabled ? CTXDESC_CD_0_S : 0) |
1400 		CTXDESC_CD_0_R |
1401 		CTXDESC_CD_0_A |
1402 		CTXDESC_CD_0_ASET |
1403 		FIELD_PREP(CTXDESC_CD_0_ASID, cd->asid)
1404 		);
1405 
1406 	/* To enable dirty flag update, set both Access flag and dirty state update */
1407 	if (pgtbl_cfg->quirks & IO_PGTABLE_QUIRK_ARM_HD)
1408 		target->data[0] |= cpu_to_le64(CTXDESC_CD_0_TCR_HA |
1409 					       CTXDESC_CD_0_TCR_HD);
1410 
1411 	target->data[1] = cpu_to_le64(pgtbl_cfg->arm_lpae_s1_cfg.ttbr &
1412 				      CTXDESC_CD_1_TTB0_MASK);
1413 	target->data[3] = cpu_to_le64(pgtbl_cfg->arm_lpae_s1_cfg.mair);
1414 }
1415 EXPORT_SYMBOL_IF_KUNIT(arm_smmu_make_s1_cd);
1416 
arm_smmu_clear_cd(struct arm_smmu_master * master,ioasid_t ssid)1417 void arm_smmu_clear_cd(struct arm_smmu_master *master, ioasid_t ssid)
1418 {
1419 	struct arm_smmu_cd target = {};
1420 	struct arm_smmu_cd *cdptr;
1421 
1422 	if (!arm_smmu_cdtab_allocated(&master->cd_table))
1423 		return;
1424 	cdptr = arm_smmu_get_cd_ptr(master, ssid);
1425 	if (WARN_ON(!cdptr))
1426 		return;
1427 	arm_smmu_write_cd_entry(master, ssid, cdptr, &target);
1428 }
1429 
arm_smmu_alloc_cd_tables(struct arm_smmu_master * master)1430 static int arm_smmu_alloc_cd_tables(struct arm_smmu_master *master)
1431 {
1432 	int ret;
1433 	size_t l1size;
1434 	size_t max_contexts;
1435 	struct arm_smmu_device *smmu = master->smmu;
1436 	struct arm_smmu_ctx_desc_cfg *cd_table = &master->cd_table;
1437 
1438 	cd_table->s1cdmax = master->ssid_bits;
1439 	max_contexts = 1 << cd_table->s1cdmax;
1440 
1441 	if (!(smmu->features & ARM_SMMU_FEAT_2_LVL_CDTAB) ||
1442 	    max_contexts <= CTXDESC_L2_ENTRIES) {
1443 		cd_table->s1fmt = STRTAB_STE_0_S1FMT_LINEAR;
1444 		cd_table->linear.num_ents = max_contexts;
1445 
1446 		l1size = max_contexts * sizeof(struct arm_smmu_cd);
1447 		cd_table->linear.table = dma_alloc_coherent(smmu->dev, l1size,
1448 							    &cd_table->cdtab_dma,
1449 							    GFP_KERNEL);
1450 		if (!cd_table->linear.table)
1451 			return -ENOMEM;
1452 	} else {
1453 		cd_table->s1fmt = STRTAB_STE_0_S1FMT_64K_L2;
1454 		cd_table->l2.num_l1_ents =
1455 			DIV_ROUND_UP(max_contexts, CTXDESC_L2_ENTRIES);
1456 
1457 		cd_table->l2.l2ptrs = kcalloc(cd_table->l2.num_l1_ents,
1458 					     sizeof(*cd_table->l2.l2ptrs),
1459 					     GFP_KERNEL);
1460 		if (!cd_table->l2.l2ptrs)
1461 			return -ENOMEM;
1462 
1463 		l1size = cd_table->l2.num_l1_ents * sizeof(struct arm_smmu_cdtab_l1);
1464 		cd_table->l2.l1tab = dma_alloc_coherent(smmu->dev, l1size,
1465 							&cd_table->cdtab_dma,
1466 							GFP_KERNEL);
1467 		if (!cd_table->l2.l2ptrs) {
1468 			ret = -ENOMEM;
1469 			goto err_free_l2ptrs;
1470 		}
1471 	}
1472 	return 0;
1473 
1474 err_free_l2ptrs:
1475 	kfree(cd_table->l2.l2ptrs);
1476 	cd_table->l2.l2ptrs = NULL;
1477 	return ret;
1478 }
1479 
arm_smmu_free_cd_tables(struct arm_smmu_master * master)1480 static void arm_smmu_free_cd_tables(struct arm_smmu_master *master)
1481 {
1482 	int i;
1483 	struct arm_smmu_device *smmu = master->smmu;
1484 	struct arm_smmu_ctx_desc_cfg *cd_table = &master->cd_table;
1485 
1486 	if (cd_table->s1fmt != STRTAB_STE_0_S1FMT_LINEAR) {
1487 		for (i = 0; i < cd_table->l2.num_l1_ents; i++) {
1488 			if (!cd_table->l2.l2ptrs[i])
1489 				continue;
1490 
1491 			dma_free_coherent(smmu->dev,
1492 					  sizeof(*cd_table->l2.l2ptrs[i]),
1493 					  cd_table->l2.l2ptrs[i],
1494 					  arm_smmu_cd_l1_get_desc(&cd_table->l2.l1tab[i]));
1495 		}
1496 		kfree(cd_table->l2.l2ptrs);
1497 
1498 		dma_free_coherent(smmu->dev,
1499 				  cd_table->l2.num_l1_ents *
1500 					  sizeof(struct arm_smmu_cdtab_l1),
1501 				  cd_table->l2.l1tab, cd_table->cdtab_dma);
1502 	} else {
1503 		dma_free_coherent(smmu->dev,
1504 				  cd_table->linear.num_ents *
1505 					  sizeof(struct arm_smmu_cd),
1506 				  cd_table->linear.table, cd_table->cdtab_dma);
1507 	}
1508 }
1509 
1510 /* Stream table manipulation functions */
arm_smmu_write_strtab_l1_desc(struct arm_smmu_strtab_l1 * dst,dma_addr_t l2ptr_dma)1511 static void arm_smmu_write_strtab_l1_desc(struct arm_smmu_strtab_l1 *dst,
1512 					  dma_addr_t l2ptr_dma)
1513 {
1514 	u64 val = 0;
1515 
1516 	val |= FIELD_PREP(STRTAB_L1_DESC_SPAN, STRTAB_SPLIT + 1);
1517 	val |= l2ptr_dma & STRTAB_L1_DESC_L2PTR_MASK;
1518 
1519 	/* The HW has 64 bit atomicity with stores to the L2 STE table */
1520 	WRITE_ONCE(dst->l2ptr, cpu_to_le64(val));
1521 }
1522 
1523 struct arm_smmu_ste_writer {
1524 	struct arm_smmu_entry_writer writer;
1525 	u32 sid;
1526 };
1527 
arm_smmu_ste_writer_sync_entry(struct arm_smmu_entry_writer * writer)1528 static void arm_smmu_ste_writer_sync_entry(struct arm_smmu_entry_writer *writer)
1529 {
1530 	struct arm_smmu_ste_writer *ste_writer =
1531 		container_of(writer, struct arm_smmu_ste_writer, writer);
1532 	struct arm_smmu_cmdq_ent cmd = {
1533 		.opcode	= CMDQ_OP_CFGI_STE,
1534 		.cfgi	= {
1535 			.sid	= ste_writer->sid,
1536 			.leaf	= true,
1537 		},
1538 	};
1539 
1540 	arm_smmu_cmdq_issue_cmd_with_sync(writer->master->smmu, &cmd);
1541 }
1542 
1543 static const struct arm_smmu_entry_writer_ops arm_smmu_ste_writer_ops = {
1544 	.sync = arm_smmu_ste_writer_sync_entry,
1545 	.get_used = arm_smmu_get_ste_used,
1546 };
1547 
arm_smmu_write_ste(struct arm_smmu_master * master,u32 sid,struct arm_smmu_ste * ste,const struct arm_smmu_ste * target)1548 static void arm_smmu_write_ste(struct arm_smmu_master *master, u32 sid,
1549 			       struct arm_smmu_ste *ste,
1550 			       const struct arm_smmu_ste *target)
1551 {
1552 	struct arm_smmu_device *smmu = master->smmu;
1553 	struct arm_smmu_ste_writer ste_writer = {
1554 		.writer = {
1555 			.ops = &arm_smmu_ste_writer_ops,
1556 			.master = master,
1557 		},
1558 		.sid = sid,
1559 	};
1560 
1561 	arm_smmu_write_entry(&ste_writer.writer, ste->data, target->data);
1562 
1563 	/* It's likely that we'll want to use the new STE soon */
1564 	if (!(smmu->options & ARM_SMMU_OPT_SKIP_PREFETCH)) {
1565 		struct arm_smmu_cmdq_ent
1566 			prefetch_cmd = { .opcode = CMDQ_OP_PREFETCH_CFG,
1567 					 .prefetch = {
1568 						 .sid = sid,
1569 					 } };
1570 
1571 		arm_smmu_cmdq_issue_cmd(smmu, &prefetch_cmd);
1572 	}
1573 }
1574 
arm_smmu_make_abort_ste(struct arm_smmu_ste * target)1575 void arm_smmu_make_abort_ste(struct arm_smmu_ste *target)
1576 {
1577 	memset(target, 0, sizeof(*target));
1578 	target->data[0] = cpu_to_le64(
1579 		STRTAB_STE_0_V |
1580 		FIELD_PREP(STRTAB_STE_0_CFG, STRTAB_STE_0_CFG_ABORT));
1581 }
1582 EXPORT_SYMBOL_IF_KUNIT(arm_smmu_make_abort_ste);
1583 
1584 VISIBLE_IF_KUNIT
arm_smmu_make_bypass_ste(struct arm_smmu_device * smmu,struct arm_smmu_ste * target)1585 void arm_smmu_make_bypass_ste(struct arm_smmu_device *smmu,
1586 			      struct arm_smmu_ste *target)
1587 {
1588 	memset(target, 0, sizeof(*target));
1589 	target->data[0] = cpu_to_le64(
1590 		STRTAB_STE_0_V |
1591 		FIELD_PREP(STRTAB_STE_0_CFG, STRTAB_STE_0_CFG_BYPASS));
1592 
1593 	if (smmu->features & ARM_SMMU_FEAT_ATTR_TYPES_OVR)
1594 		target->data[1] = cpu_to_le64(FIELD_PREP(STRTAB_STE_1_SHCFG,
1595 							 STRTAB_STE_1_SHCFG_INCOMING));
1596 }
1597 EXPORT_SYMBOL_IF_KUNIT(arm_smmu_make_bypass_ste);
1598 
1599 VISIBLE_IF_KUNIT
arm_smmu_make_cdtable_ste(struct arm_smmu_ste * target,struct arm_smmu_master * master,bool ats_enabled,unsigned int s1dss)1600 void arm_smmu_make_cdtable_ste(struct arm_smmu_ste *target,
1601 			       struct arm_smmu_master *master, bool ats_enabled,
1602 			       unsigned int s1dss)
1603 {
1604 	struct arm_smmu_ctx_desc_cfg *cd_table = &master->cd_table;
1605 	struct arm_smmu_device *smmu = master->smmu;
1606 
1607 	memset(target, 0, sizeof(*target));
1608 	target->data[0] = cpu_to_le64(
1609 		STRTAB_STE_0_V |
1610 		FIELD_PREP(STRTAB_STE_0_CFG, STRTAB_STE_0_CFG_S1_TRANS) |
1611 		FIELD_PREP(STRTAB_STE_0_S1FMT, cd_table->s1fmt) |
1612 		(cd_table->cdtab_dma & STRTAB_STE_0_S1CTXPTR_MASK) |
1613 		FIELD_PREP(STRTAB_STE_0_S1CDMAX, cd_table->s1cdmax));
1614 
1615 	target->data[1] = cpu_to_le64(
1616 		FIELD_PREP(STRTAB_STE_1_S1DSS, s1dss) |
1617 		FIELD_PREP(STRTAB_STE_1_S1CIR, STRTAB_STE_1_S1C_CACHE_WBRA) |
1618 		FIELD_PREP(STRTAB_STE_1_S1COR, STRTAB_STE_1_S1C_CACHE_WBRA) |
1619 		FIELD_PREP(STRTAB_STE_1_S1CSH, ARM_SMMU_SH_ISH) |
1620 		((smmu->features & ARM_SMMU_FEAT_STALLS &&
1621 		  !master->stall_enabled) ?
1622 			 STRTAB_STE_1_S1STALLD :
1623 			 0) |
1624 		FIELD_PREP(STRTAB_STE_1_EATS,
1625 			   ats_enabled ? STRTAB_STE_1_EATS_TRANS : 0));
1626 
1627 	if ((smmu->features & ARM_SMMU_FEAT_ATTR_TYPES_OVR) &&
1628 	    s1dss == STRTAB_STE_1_S1DSS_BYPASS)
1629 		target->data[1] |= cpu_to_le64(FIELD_PREP(
1630 			STRTAB_STE_1_SHCFG, STRTAB_STE_1_SHCFG_INCOMING));
1631 
1632 	if (smmu->features & ARM_SMMU_FEAT_E2H) {
1633 		/*
1634 		 * To support BTM the streamworld needs to match the
1635 		 * configuration of the CPU so that the ASID broadcasts are
1636 		 * properly matched. This means either S/NS-EL2-E2H (hypervisor)
1637 		 * or NS-EL1 (guest). Since an SVA domain can be installed in a
1638 		 * PASID this should always use a BTM compatible configuration
1639 		 * if the HW supports it.
1640 		 */
1641 		target->data[1] |= cpu_to_le64(
1642 			FIELD_PREP(STRTAB_STE_1_STRW, STRTAB_STE_1_STRW_EL2));
1643 	} else {
1644 		target->data[1] |= cpu_to_le64(
1645 			FIELD_PREP(STRTAB_STE_1_STRW, STRTAB_STE_1_STRW_NSEL1));
1646 
1647 		/*
1648 		 * VMID 0 is reserved for stage-2 bypass EL1 STEs, see
1649 		 * arm_smmu_domain_alloc_id()
1650 		 */
1651 		target->data[2] =
1652 			cpu_to_le64(FIELD_PREP(STRTAB_STE_2_S2VMID, 0));
1653 	}
1654 }
1655 EXPORT_SYMBOL_IF_KUNIT(arm_smmu_make_cdtable_ste);
1656 
arm_smmu_make_s2_domain_ste(struct arm_smmu_ste * target,struct arm_smmu_master * master,struct arm_smmu_domain * smmu_domain,bool ats_enabled)1657 void arm_smmu_make_s2_domain_ste(struct arm_smmu_ste *target,
1658 				 struct arm_smmu_master *master,
1659 				 struct arm_smmu_domain *smmu_domain,
1660 				 bool ats_enabled)
1661 {
1662 	struct arm_smmu_s2_cfg *s2_cfg = &smmu_domain->s2_cfg;
1663 	const struct io_pgtable_cfg *pgtbl_cfg =
1664 		&io_pgtable_ops_to_pgtable(smmu_domain->pgtbl_ops)->cfg;
1665 	typeof(&pgtbl_cfg->arm_lpae_s2_cfg.vtcr) vtcr =
1666 		&pgtbl_cfg->arm_lpae_s2_cfg.vtcr;
1667 	u64 vtcr_val;
1668 	struct arm_smmu_device *smmu = master->smmu;
1669 
1670 	memset(target, 0, sizeof(*target));
1671 	target->data[0] = cpu_to_le64(
1672 		STRTAB_STE_0_V |
1673 		FIELD_PREP(STRTAB_STE_0_CFG, STRTAB_STE_0_CFG_S2_TRANS));
1674 
1675 	target->data[1] = cpu_to_le64(
1676 		FIELD_PREP(STRTAB_STE_1_EATS,
1677 			   ats_enabled ? STRTAB_STE_1_EATS_TRANS : 0));
1678 
1679 	if (pgtbl_cfg->quirks & IO_PGTABLE_QUIRK_ARM_S2FWB)
1680 		target->data[1] |= cpu_to_le64(STRTAB_STE_1_S2FWB);
1681 	if (smmu->features & ARM_SMMU_FEAT_ATTR_TYPES_OVR)
1682 		target->data[1] |= cpu_to_le64(FIELD_PREP(STRTAB_STE_1_SHCFG,
1683 							  STRTAB_STE_1_SHCFG_INCOMING));
1684 
1685 	vtcr_val = FIELD_PREP(STRTAB_STE_2_VTCR_S2T0SZ, vtcr->tsz) |
1686 		   FIELD_PREP(STRTAB_STE_2_VTCR_S2SL0, vtcr->sl) |
1687 		   FIELD_PREP(STRTAB_STE_2_VTCR_S2IR0, vtcr->irgn) |
1688 		   FIELD_PREP(STRTAB_STE_2_VTCR_S2OR0, vtcr->orgn) |
1689 		   FIELD_PREP(STRTAB_STE_2_VTCR_S2SH0, vtcr->sh) |
1690 		   FIELD_PREP(STRTAB_STE_2_VTCR_S2TG, vtcr->tg) |
1691 		   FIELD_PREP(STRTAB_STE_2_VTCR_S2PS, vtcr->ps);
1692 	target->data[2] = cpu_to_le64(
1693 		FIELD_PREP(STRTAB_STE_2_S2VMID, s2_cfg->vmid) |
1694 		FIELD_PREP(STRTAB_STE_2_VTCR, vtcr_val) |
1695 		STRTAB_STE_2_S2AA64 |
1696 #ifdef __BIG_ENDIAN
1697 		STRTAB_STE_2_S2ENDI |
1698 #endif
1699 		STRTAB_STE_2_S2PTW |
1700 		(master->stall_enabled ? STRTAB_STE_2_S2S : 0) |
1701 		STRTAB_STE_2_S2R);
1702 
1703 	target->data[3] = cpu_to_le64(pgtbl_cfg->arm_lpae_s2_cfg.vttbr &
1704 				      STRTAB_STE_3_S2TTB_MASK);
1705 }
1706 EXPORT_SYMBOL_IF_KUNIT(arm_smmu_make_s2_domain_ste);
1707 
1708 /*
1709  * This can safely directly manipulate the STE memory without a sync sequence
1710  * because the STE table has not been installed in the SMMU yet.
1711  */
arm_smmu_init_initial_stes(struct arm_smmu_ste * strtab,unsigned int nent)1712 static void arm_smmu_init_initial_stes(struct arm_smmu_ste *strtab,
1713 				       unsigned int nent)
1714 {
1715 	unsigned int i;
1716 
1717 	for (i = 0; i < nent; ++i) {
1718 		arm_smmu_make_abort_ste(strtab);
1719 		strtab++;
1720 	}
1721 }
1722 
arm_smmu_init_l2_strtab(struct arm_smmu_device * smmu,u32 sid)1723 static int arm_smmu_init_l2_strtab(struct arm_smmu_device *smmu, u32 sid)
1724 {
1725 	dma_addr_t l2ptr_dma;
1726 	struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg;
1727 	struct arm_smmu_strtab_l2 **l2table;
1728 
1729 	l2table = &cfg->l2.l2ptrs[arm_smmu_strtab_l1_idx(sid)];
1730 	if (*l2table)
1731 		return 0;
1732 
1733 	*l2table = dmam_alloc_coherent(smmu->dev, sizeof(**l2table),
1734 				       &l2ptr_dma, GFP_KERNEL);
1735 	if (!*l2table) {
1736 		dev_err(smmu->dev,
1737 			"failed to allocate l2 stream table for SID %u\n",
1738 			sid);
1739 		return -ENOMEM;
1740 	}
1741 
1742 	arm_smmu_init_initial_stes((*l2table)->stes,
1743 				   ARRAY_SIZE((*l2table)->stes));
1744 	arm_smmu_write_strtab_l1_desc(&cfg->l2.l1tab[arm_smmu_strtab_l1_idx(sid)],
1745 				      l2ptr_dma);
1746 	return 0;
1747 }
1748 
arm_smmu_streams_cmp_key(const void * lhs,const struct rb_node * rhs)1749 static int arm_smmu_streams_cmp_key(const void *lhs, const struct rb_node *rhs)
1750 {
1751 	struct arm_smmu_stream *stream_rhs =
1752 		rb_entry(rhs, struct arm_smmu_stream, node);
1753 	const u32 *sid_lhs = lhs;
1754 
1755 	if (*sid_lhs < stream_rhs->id)
1756 		return -1;
1757 	if (*sid_lhs > stream_rhs->id)
1758 		return 1;
1759 	return 0;
1760 }
1761 
arm_smmu_streams_cmp_node(struct rb_node * lhs,const struct rb_node * rhs)1762 static int arm_smmu_streams_cmp_node(struct rb_node *lhs,
1763 				     const struct rb_node *rhs)
1764 {
1765 	return arm_smmu_streams_cmp_key(
1766 		&rb_entry(lhs, struct arm_smmu_stream, node)->id, rhs);
1767 }
1768 
1769 static struct arm_smmu_master *
arm_smmu_find_master(struct arm_smmu_device * smmu,u32 sid)1770 arm_smmu_find_master(struct arm_smmu_device *smmu, u32 sid)
1771 {
1772 	struct rb_node *node;
1773 
1774 	lockdep_assert_held(&smmu->streams_mutex);
1775 
1776 	node = rb_find(&sid, &smmu->streams, arm_smmu_streams_cmp_key);
1777 	if (!node)
1778 		return NULL;
1779 	return rb_entry(node, struct arm_smmu_stream, node)->master;
1780 }
1781 
1782 /* IRQ and event handlers */
arm_smmu_decode_event(struct arm_smmu_device * smmu,u64 * raw,struct arm_smmu_event * event)1783 static void arm_smmu_decode_event(struct arm_smmu_device *smmu, u64 *raw,
1784 				  struct arm_smmu_event *event)
1785 {
1786 	struct arm_smmu_master *master;
1787 
1788 	event->id = FIELD_GET(EVTQ_0_ID, raw[0]);
1789 	event->sid = FIELD_GET(EVTQ_0_SID, raw[0]);
1790 	event->ssv = FIELD_GET(EVTQ_0_SSV, raw[0]);
1791 	event->ssid = event->ssv ? FIELD_GET(EVTQ_0_SSID, raw[0]) : IOMMU_NO_PASID;
1792 	event->privileged = FIELD_GET(EVTQ_1_PnU, raw[1]);
1793 	event->instruction = FIELD_GET(EVTQ_1_InD, raw[1]);
1794 	event->s2 = FIELD_GET(EVTQ_1_S2, raw[1]);
1795 	event->read = FIELD_GET(EVTQ_1_RnW, raw[1]);
1796 	event->stag = FIELD_GET(EVTQ_1_STAG, raw[1]);
1797 	event->stall = FIELD_GET(EVTQ_1_STALL, raw[1]);
1798 	event->class = FIELD_GET(EVTQ_1_CLASS, raw[1]);
1799 	event->iova = FIELD_GET(EVTQ_2_ADDR, raw[2]);
1800 	event->ipa = raw[3] & EVTQ_3_IPA;
1801 	event->fetch_addr = raw[3] & EVTQ_3_FETCH_ADDR;
1802 	event->ttrnw = FIELD_GET(EVTQ_1_TT_READ, raw[1]);
1803 	event->class_tt = false;
1804 	event->dev = NULL;
1805 
1806 	if (event->id == EVT_ID_PERMISSION_FAULT)
1807 		event->class_tt = (event->class == EVTQ_1_CLASS_TT);
1808 
1809 	mutex_lock(&smmu->streams_mutex);
1810 	master = arm_smmu_find_master(smmu, event->sid);
1811 	if (master)
1812 		event->dev = get_device(master->dev);
1813 	mutex_unlock(&smmu->streams_mutex);
1814 }
1815 
arm_smmu_handle_event(struct arm_smmu_device * smmu,u64 * evt,struct arm_smmu_event * event)1816 static int arm_smmu_handle_event(struct arm_smmu_device *smmu, u64 *evt,
1817 				 struct arm_smmu_event *event)
1818 {
1819 	int ret = 0;
1820 	u32 perm = 0;
1821 	struct arm_smmu_master *master;
1822 	struct iopf_fault fault_evt = { };
1823 	struct iommu_fault *flt = &fault_evt.fault;
1824 
1825 	switch (event->id) {
1826 	case EVT_ID_BAD_STE_CONFIG:
1827 	case EVT_ID_STREAM_DISABLED_FAULT:
1828 	case EVT_ID_BAD_SUBSTREAMID_CONFIG:
1829 	case EVT_ID_BAD_CD_CONFIG:
1830 	case EVT_ID_TRANSLATION_FAULT:
1831 	case EVT_ID_ADDR_SIZE_FAULT:
1832 	case EVT_ID_ACCESS_FAULT:
1833 	case EVT_ID_PERMISSION_FAULT:
1834 		break;
1835 	default:
1836 		return -EOPNOTSUPP;
1837 	}
1838 
1839 	if (event->stall) {
1840 		if (event->read)
1841 			perm |= IOMMU_FAULT_PERM_READ;
1842 		else
1843 			perm |= IOMMU_FAULT_PERM_WRITE;
1844 
1845 		if (event->instruction)
1846 			perm |= IOMMU_FAULT_PERM_EXEC;
1847 
1848 		if (event->privileged)
1849 			perm |= IOMMU_FAULT_PERM_PRIV;
1850 
1851 		flt->type = IOMMU_FAULT_PAGE_REQ;
1852 		flt->prm = (struct iommu_fault_page_request){
1853 			.flags = IOMMU_FAULT_PAGE_REQUEST_LAST_PAGE,
1854 			.grpid = event->stag,
1855 			.perm = perm,
1856 			.addr = event->iova,
1857 		};
1858 
1859 		if (event->ssv) {
1860 			flt->prm.flags |= IOMMU_FAULT_PAGE_REQUEST_PASID_VALID;
1861 			flt->prm.pasid = event->ssid;
1862 		}
1863 	}
1864 
1865 	mutex_lock(&smmu->streams_mutex);
1866 	master = arm_smmu_find_master(smmu, event->sid);
1867 	if (!master) {
1868 		ret = -EINVAL;
1869 		goto out_unlock;
1870 	}
1871 
1872 	if (event->stall)
1873 		ret = iommu_report_device_fault(master->dev, &fault_evt);
1874 	else if (master->vmaster && !event->s2)
1875 		ret = arm_vmaster_report_event(master->vmaster, evt);
1876 	else
1877 		ret = -EOPNOTSUPP; /* Unhandled events should be pinned */
1878 out_unlock:
1879 	mutex_unlock(&smmu->streams_mutex);
1880 	return ret;
1881 }
1882 
arm_smmu_dump_raw_event(struct arm_smmu_device * smmu,u64 * raw,struct arm_smmu_event * event)1883 static void arm_smmu_dump_raw_event(struct arm_smmu_device *smmu, u64 *raw,
1884 				    struct arm_smmu_event *event)
1885 {
1886 	int i;
1887 
1888 	dev_err(smmu->dev, "event 0x%02x received:\n", event->id);
1889 
1890 	for (i = 0; i < EVTQ_ENT_DWORDS; ++i)
1891 		dev_err(smmu->dev, "\t0x%016llx\n", raw[i]);
1892 }
1893 
1894 #define ARM_SMMU_EVT_KNOWN(e)	((e)->id < ARRAY_SIZE(event_str) && event_str[(e)->id])
1895 #define ARM_SMMU_LOG_EVT_STR(e) ARM_SMMU_EVT_KNOWN(e) ? event_str[(e)->id] : "UNKNOWN"
1896 #define ARM_SMMU_LOG_CLIENT(e)	(e)->dev ? dev_name((e)->dev) : "(unassigned sid)"
1897 
arm_smmu_dump_event(struct arm_smmu_device * smmu,u64 * raw,struct arm_smmu_event * evt,struct ratelimit_state * rs)1898 static void arm_smmu_dump_event(struct arm_smmu_device *smmu, u64 *raw,
1899 				struct arm_smmu_event *evt,
1900 				struct ratelimit_state *rs)
1901 {
1902 	if (!__ratelimit(rs))
1903 		return;
1904 
1905 	arm_smmu_dump_raw_event(smmu, raw, evt);
1906 
1907 	switch (evt->id) {
1908 	case EVT_ID_TRANSLATION_FAULT:
1909 	case EVT_ID_ADDR_SIZE_FAULT:
1910 	case EVT_ID_ACCESS_FAULT:
1911 	case EVT_ID_PERMISSION_FAULT:
1912 		dev_err(smmu->dev, "event: %s client: %s sid: %#x ssid: %#x iova: %#llx ipa: %#llx",
1913 			ARM_SMMU_LOG_EVT_STR(evt), ARM_SMMU_LOG_CLIENT(evt),
1914 			evt->sid, evt->ssid, evt->iova, evt->ipa);
1915 
1916 		dev_err(smmu->dev, "%s %s %s %s \"%s\"%s%s stag: %#x",
1917 			evt->privileged ? "priv" : "unpriv",
1918 			evt->instruction ? "inst" : "data",
1919 			str_read_write(evt->read),
1920 			evt->s2 ? "s2" : "s1", event_class_str[evt->class],
1921 			evt->class_tt ? (evt->ttrnw ? " ttd_read" : " ttd_write") : "",
1922 			evt->stall ? " stall" : "", evt->stag);
1923 
1924 		break;
1925 
1926 	case EVT_ID_STE_FETCH_FAULT:
1927 	case EVT_ID_CD_FETCH_FAULT:
1928 	case EVT_ID_VMS_FETCH_FAULT:
1929 		dev_err(smmu->dev, "event: %s client: %s sid: %#x ssid: %#x fetch_addr: %#llx",
1930 			ARM_SMMU_LOG_EVT_STR(evt), ARM_SMMU_LOG_CLIENT(evt),
1931 			evt->sid, evt->ssid, evt->fetch_addr);
1932 
1933 		break;
1934 
1935 	default:
1936 		dev_err(smmu->dev, "event: %s client: %s sid: %#x ssid: %#x",
1937 			ARM_SMMU_LOG_EVT_STR(evt), ARM_SMMU_LOG_CLIENT(evt),
1938 			evt->sid, evt->ssid);
1939 	}
1940 }
1941 
arm_smmu_evtq_thread(int irq,void * dev)1942 static irqreturn_t arm_smmu_evtq_thread(int irq, void *dev)
1943 {
1944 	u64 evt[EVTQ_ENT_DWORDS];
1945 	struct arm_smmu_event event = {0};
1946 	struct arm_smmu_device *smmu = dev;
1947 	struct arm_smmu_queue *q = &smmu->evtq.q;
1948 	struct arm_smmu_ll_queue *llq = &q->llq;
1949 	static DEFINE_RATELIMIT_STATE(rs, DEFAULT_RATELIMIT_INTERVAL,
1950 				      DEFAULT_RATELIMIT_BURST);
1951 
1952 	do {
1953 		while (!queue_remove_raw(q, evt)) {
1954 			arm_smmu_decode_event(smmu, evt, &event);
1955 			if (arm_smmu_handle_event(smmu, evt, &event))
1956 				arm_smmu_dump_event(smmu, evt, &event, &rs);
1957 
1958 			put_device(event.dev);
1959 			cond_resched();
1960 		}
1961 
1962 		/*
1963 		 * Not much we can do on overflow, so scream and pretend we're
1964 		 * trying harder.
1965 		 */
1966 		if (queue_sync_prod_in(q) == -EOVERFLOW)
1967 			dev_err(smmu->dev, "EVTQ overflow detected -- events lost\n");
1968 	} while (!queue_empty(llq));
1969 
1970 	/* Sync our overflow flag, as we believe we're up to speed */
1971 	queue_sync_cons_ovf(q);
1972 	return IRQ_HANDLED;
1973 }
1974 
arm_smmu_handle_ppr(struct arm_smmu_device * smmu,u64 * evt)1975 static void arm_smmu_handle_ppr(struct arm_smmu_device *smmu, u64 *evt)
1976 {
1977 	u32 sid, ssid;
1978 	u16 grpid;
1979 	bool ssv, last;
1980 
1981 	sid = FIELD_GET(PRIQ_0_SID, evt[0]);
1982 	ssv = FIELD_GET(PRIQ_0_SSID_V, evt[0]);
1983 	ssid = ssv ? FIELD_GET(PRIQ_0_SSID, evt[0]) : IOMMU_NO_PASID;
1984 	last = FIELD_GET(PRIQ_0_PRG_LAST, evt[0]);
1985 	grpid = FIELD_GET(PRIQ_1_PRG_IDX, evt[1]);
1986 
1987 	dev_info(smmu->dev, "unexpected PRI request received:\n");
1988 	dev_info(smmu->dev,
1989 		 "\tsid 0x%08x.0x%05x: [%u%s] %sprivileged %s%s%s access at iova 0x%016llx\n",
1990 		 sid, ssid, grpid, last ? "L" : "",
1991 		 evt[0] & PRIQ_0_PERM_PRIV ? "" : "un",
1992 		 evt[0] & PRIQ_0_PERM_READ ? "R" : "",
1993 		 evt[0] & PRIQ_0_PERM_WRITE ? "W" : "",
1994 		 evt[0] & PRIQ_0_PERM_EXEC ? "X" : "",
1995 		 evt[1] & PRIQ_1_ADDR_MASK);
1996 
1997 	if (last) {
1998 		struct arm_smmu_cmdq_ent cmd = {
1999 			.opcode			= CMDQ_OP_PRI_RESP,
2000 			.substream_valid	= ssv,
2001 			.pri			= {
2002 				.sid	= sid,
2003 				.ssid	= ssid,
2004 				.grpid	= grpid,
2005 				.resp	= PRI_RESP_DENY,
2006 			},
2007 		};
2008 
2009 		arm_smmu_cmdq_issue_cmd(smmu, &cmd);
2010 	}
2011 }
2012 
arm_smmu_priq_thread(int irq,void * dev)2013 static irqreturn_t arm_smmu_priq_thread(int irq, void *dev)
2014 {
2015 	struct arm_smmu_device *smmu = dev;
2016 	struct arm_smmu_queue *q = &smmu->priq.q;
2017 	struct arm_smmu_ll_queue *llq = &q->llq;
2018 	u64 evt[PRIQ_ENT_DWORDS];
2019 
2020 	do {
2021 		while (!queue_remove_raw(q, evt))
2022 			arm_smmu_handle_ppr(smmu, evt);
2023 
2024 		if (queue_sync_prod_in(q) == -EOVERFLOW)
2025 			dev_err(smmu->dev, "PRIQ overflow detected -- requests lost\n");
2026 	} while (!queue_empty(llq));
2027 
2028 	/* Sync our overflow flag, as we believe we're up to speed */
2029 	queue_sync_cons_ovf(q);
2030 	return IRQ_HANDLED;
2031 }
2032 
2033 static int arm_smmu_device_disable(struct arm_smmu_device *smmu);
2034 
arm_smmu_gerror_handler(int irq,void * dev)2035 static irqreturn_t arm_smmu_gerror_handler(int irq, void *dev)
2036 {
2037 	u32 gerror, gerrorn, active;
2038 	struct arm_smmu_device *smmu = dev;
2039 
2040 	gerror = readl_relaxed(smmu->base + ARM_SMMU_GERROR);
2041 	gerrorn = readl_relaxed(smmu->base + ARM_SMMU_GERRORN);
2042 
2043 	active = gerror ^ gerrorn;
2044 	if (!(active & GERROR_ERR_MASK))
2045 		return IRQ_NONE; /* No errors pending */
2046 
2047 	dev_warn(smmu->dev,
2048 		 "unexpected global error reported (0x%08x), this could be serious\n",
2049 		 active);
2050 
2051 	if (active & GERROR_SFM_ERR) {
2052 		dev_err(smmu->dev, "device has entered Service Failure Mode!\n");
2053 		arm_smmu_device_disable(smmu);
2054 	}
2055 
2056 	if (active & GERROR_MSI_GERROR_ABT_ERR)
2057 		dev_warn(smmu->dev, "GERROR MSI write aborted\n");
2058 
2059 	if (active & GERROR_MSI_PRIQ_ABT_ERR)
2060 		dev_warn(smmu->dev, "PRIQ MSI write aborted\n");
2061 
2062 	if (active & GERROR_MSI_EVTQ_ABT_ERR)
2063 		dev_warn(smmu->dev, "EVTQ MSI write aborted\n");
2064 
2065 	if (active & GERROR_MSI_CMDQ_ABT_ERR)
2066 		dev_warn(smmu->dev, "CMDQ MSI write aborted\n");
2067 
2068 	if (active & GERROR_PRIQ_ABT_ERR)
2069 		dev_err(smmu->dev, "PRIQ write aborted -- events may have been lost\n");
2070 
2071 	if (active & GERROR_EVTQ_ABT_ERR)
2072 		dev_err(smmu->dev, "EVTQ write aborted -- events may have been lost\n");
2073 
2074 	if (active & GERROR_CMDQ_ERR)
2075 		arm_smmu_cmdq_skip_err(smmu);
2076 
2077 	writel(gerror, smmu->base + ARM_SMMU_GERRORN);
2078 	return IRQ_HANDLED;
2079 }
2080 
arm_smmu_combined_irq_thread(int irq,void * dev)2081 static irqreturn_t arm_smmu_combined_irq_thread(int irq, void *dev)
2082 {
2083 	struct arm_smmu_device *smmu = dev;
2084 
2085 	arm_smmu_evtq_thread(irq, dev);
2086 	if (smmu->features & ARM_SMMU_FEAT_PRI)
2087 		arm_smmu_priq_thread(irq, dev);
2088 
2089 	return IRQ_HANDLED;
2090 }
2091 
arm_smmu_combined_irq_handler(int irq,void * dev)2092 static irqreturn_t arm_smmu_combined_irq_handler(int irq, void *dev)
2093 {
2094 	arm_smmu_gerror_handler(irq, dev);
2095 	return IRQ_WAKE_THREAD;
2096 }
2097 
2098 static void
arm_smmu_atc_inv_to_cmd(int ssid,unsigned long iova,size_t size,struct arm_smmu_cmdq_ent * cmd)2099 arm_smmu_atc_inv_to_cmd(int ssid, unsigned long iova, size_t size,
2100 			struct arm_smmu_cmdq_ent *cmd)
2101 {
2102 	size_t log2_span;
2103 	size_t span_mask;
2104 	/* ATC invalidates are always on 4096-bytes pages */
2105 	size_t inval_grain_shift = 12;
2106 	unsigned long page_start, page_end;
2107 
2108 	/*
2109 	 * ATS and PASID:
2110 	 *
2111 	 * If substream_valid is clear, the PCIe TLP is sent without a PASID
2112 	 * prefix. In that case all ATC entries within the address range are
2113 	 * invalidated, including those that were requested with a PASID! There
2114 	 * is no way to invalidate only entries without PASID.
2115 	 *
2116 	 * When using STRTAB_STE_1_S1DSS_SSID0 (reserving CD 0 for non-PASID
2117 	 * traffic), translation requests without PASID create ATC entries
2118 	 * without PASID, which must be invalidated with substream_valid clear.
2119 	 * This has the unpleasant side-effect of invalidating all PASID-tagged
2120 	 * ATC entries within the address range.
2121 	 */
2122 	*cmd = (struct arm_smmu_cmdq_ent) {
2123 		.opcode			= CMDQ_OP_ATC_INV,
2124 		.substream_valid	= (ssid != IOMMU_NO_PASID),
2125 		.atc.ssid		= ssid,
2126 	};
2127 
2128 	if (!size) {
2129 		cmd->atc.size = ATC_INV_SIZE_ALL;
2130 		return;
2131 	}
2132 
2133 	page_start	= iova >> inval_grain_shift;
2134 	page_end	= (iova + size - 1) >> inval_grain_shift;
2135 
2136 	/*
2137 	 * In an ATS Invalidate Request, the address must be aligned on the
2138 	 * range size, which must be a power of two number of page sizes. We
2139 	 * thus have to choose between grossly over-invalidating the region, or
2140 	 * splitting the invalidation into multiple commands. For simplicity
2141 	 * we'll go with the first solution, but should refine it in the future
2142 	 * if multiple commands are shown to be more efficient.
2143 	 *
2144 	 * Find the smallest power of two that covers the range. The most
2145 	 * significant differing bit between the start and end addresses,
2146 	 * fls(start ^ end), indicates the required span. For example:
2147 	 *
2148 	 * We want to invalidate pages [8; 11]. This is already the ideal range:
2149 	 *		x = 0b1000 ^ 0b1011 = 0b11
2150 	 *		span = 1 << fls(x) = 4
2151 	 *
2152 	 * To invalidate pages [7; 10], we need to invalidate [0; 15]:
2153 	 *		x = 0b0111 ^ 0b1010 = 0b1101
2154 	 *		span = 1 << fls(x) = 16
2155 	 */
2156 	log2_span	= fls_long(page_start ^ page_end);
2157 	span_mask	= (1ULL << log2_span) - 1;
2158 
2159 	page_start	&= ~span_mask;
2160 
2161 	cmd->atc.addr	= page_start << inval_grain_shift;
2162 	cmd->atc.size	= log2_span;
2163 }
2164 
arm_smmu_atc_inv_master(struct arm_smmu_master * master,ioasid_t ssid)2165 static int arm_smmu_atc_inv_master(struct arm_smmu_master *master,
2166 				   ioasid_t ssid)
2167 {
2168 	int i;
2169 	struct arm_smmu_cmdq_ent cmd;
2170 	struct arm_smmu_cmdq_batch cmds;
2171 
2172 	arm_smmu_atc_inv_to_cmd(ssid, 0, 0, &cmd);
2173 
2174 	arm_smmu_cmdq_batch_init(master->smmu, &cmds, &cmd);
2175 	for (i = 0; i < master->num_streams; i++) {
2176 		cmd.atc.sid = master->streams[i].id;
2177 		arm_smmu_cmdq_batch_add(master->smmu, &cmds, &cmd);
2178 	}
2179 
2180 	return arm_smmu_cmdq_batch_submit(master->smmu, &cmds);
2181 }
2182 
arm_smmu_atc_inv_domain(struct arm_smmu_domain * smmu_domain,unsigned long iova,size_t size)2183 int arm_smmu_atc_inv_domain(struct arm_smmu_domain *smmu_domain,
2184 			    unsigned long iova, size_t size)
2185 {
2186 	struct arm_smmu_master_domain *master_domain;
2187 	int i;
2188 	unsigned long flags;
2189 	struct arm_smmu_cmdq_ent cmd = {
2190 		.opcode = CMDQ_OP_ATC_INV,
2191 	};
2192 	struct arm_smmu_cmdq_batch cmds;
2193 
2194 	if (!(smmu_domain->smmu->features & ARM_SMMU_FEAT_ATS))
2195 		return 0;
2196 
2197 	/*
2198 	 * Ensure that we've completed prior invalidation of the main TLBs
2199 	 * before we read 'nr_ats_masters' in case of a concurrent call to
2200 	 * arm_smmu_enable_ats():
2201 	 *
2202 	 *	// unmap()			// arm_smmu_enable_ats()
2203 	 *	TLBI+SYNC			atomic_inc(&nr_ats_masters);
2204 	 *	smp_mb();			[...]
2205 	 *	atomic_read(&nr_ats_masters);	pci_enable_ats() // writel()
2206 	 *
2207 	 * Ensures that we always see the incremented 'nr_ats_masters' count if
2208 	 * ATS was enabled at the PCI device before completion of the TLBI.
2209 	 */
2210 	smp_mb();
2211 	if (!atomic_read(&smmu_domain->nr_ats_masters))
2212 		return 0;
2213 
2214 	arm_smmu_cmdq_batch_init(smmu_domain->smmu, &cmds, &cmd);
2215 
2216 	spin_lock_irqsave(&smmu_domain->devices_lock, flags);
2217 	list_for_each_entry(master_domain, &smmu_domain->devices,
2218 			    devices_elm) {
2219 		struct arm_smmu_master *master = master_domain->master;
2220 
2221 		if (!master->ats_enabled)
2222 			continue;
2223 
2224 		if (master_domain->nested_ats_flush) {
2225 			/*
2226 			 * If a S2 used as a nesting parent is changed we have
2227 			 * no option but to completely flush the ATC.
2228 			 */
2229 			arm_smmu_atc_inv_to_cmd(IOMMU_NO_PASID, 0, 0, &cmd);
2230 		} else {
2231 			arm_smmu_atc_inv_to_cmd(master_domain->ssid, iova, size,
2232 						&cmd);
2233 		}
2234 
2235 		for (i = 0; i < master->num_streams; i++) {
2236 			cmd.atc.sid = master->streams[i].id;
2237 			arm_smmu_cmdq_batch_add(smmu_domain->smmu, &cmds, &cmd);
2238 		}
2239 	}
2240 	spin_unlock_irqrestore(&smmu_domain->devices_lock, flags);
2241 
2242 	return arm_smmu_cmdq_batch_submit(smmu_domain->smmu, &cmds);
2243 }
2244 
2245 /* IO_PGTABLE API */
arm_smmu_tlb_inv_context(void * cookie)2246 static void arm_smmu_tlb_inv_context(void *cookie)
2247 {
2248 	struct arm_smmu_domain *smmu_domain = cookie;
2249 	struct arm_smmu_device *smmu = smmu_domain->smmu;
2250 	struct arm_smmu_cmdq_ent cmd;
2251 
2252 	/*
2253 	 * NOTE: when io-pgtable is in non-strict mode, we may get here with
2254 	 * PTEs previously cleared by unmaps on the current CPU not yet visible
2255 	 * to the SMMU. We are relying on the dma_wmb() implicit during cmd
2256 	 * insertion to guarantee those are observed before the TLBI. Do be
2257 	 * careful, 007.
2258 	 */
2259 	if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {
2260 		arm_smmu_tlb_inv_asid(smmu, smmu_domain->cd.asid);
2261 	} else {
2262 		cmd.opcode	= CMDQ_OP_TLBI_S12_VMALL;
2263 		cmd.tlbi.vmid	= smmu_domain->s2_cfg.vmid;
2264 		arm_smmu_cmdq_issue_cmd_with_sync(smmu, &cmd);
2265 	}
2266 	arm_smmu_atc_inv_domain(smmu_domain, 0, 0);
2267 }
2268 
__arm_smmu_tlb_inv_range(struct arm_smmu_cmdq_ent * cmd,unsigned long iova,size_t size,size_t granule,struct arm_smmu_domain * smmu_domain)2269 static void __arm_smmu_tlb_inv_range(struct arm_smmu_cmdq_ent *cmd,
2270 				     unsigned long iova, size_t size,
2271 				     size_t granule,
2272 				     struct arm_smmu_domain *smmu_domain)
2273 {
2274 	struct arm_smmu_device *smmu = smmu_domain->smmu;
2275 	unsigned long end = iova + size, num_pages = 0, tg = 0;
2276 	size_t inv_range = granule;
2277 	struct arm_smmu_cmdq_batch cmds;
2278 
2279 	if (!size)
2280 		return;
2281 
2282 	if (smmu->features & ARM_SMMU_FEAT_RANGE_INV) {
2283 		/* Get the leaf page size */
2284 		tg = __ffs(smmu_domain->domain.pgsize_bitmap);
2285 
2286 		num_pages = size >> tg;
2287 
2288 		/* Convert page size of 12,14,16 (log2) to 1,2,3 */
2289 		cmd->tlbi.tg = (tg - 10) / 2;
2290 
2291 		/*
2292 		 * Determine what level the granule is at. For non-leaf, both
2293 		 * io-pgtable and SVA pass a nominal last-level granule because
2294 		 * they don't know what level(s) actually apply, so ignore that
2295 		 * and leave TTL=0. However for various errata reasons we still
2296 		 * want to use a range command, so avoid the SVA corner case
2297 		 * where both scale and num could be 0 as well.
2298 		 */
2299 		if (cmd->tlbi.leaf)
2300 			cmd->tlbi.ttl = 4 - ((ilog2(granule) - 3) / (tg - 3));
2301 		else if ((num_pages & CMDQ_TLBI_RANGE_NUM_MAX) == 1)
2302 			num_pages++;
2303 	}
2304 
2305 	arm_smmu_cmdq_batch_init(smmu, &cmds, cmd);
2306 
2307 	while (iova < end) {
2308 		if (smmu->features & ARM_SMMU_FEAT_RANGE_INV) {
2309 			/*
2310 			 * On each iteration of the loop, the range is 5 bits
2311 			 * worth of the aligned size remaining.
2312 			 * The range in pages is:
2313 			 *
2314 			 * range = (num_pages & (0x1f << __ffs(num_pages)))
2315 			 */
2316 			unsigned long scale, num;
2317 
2318 			/* Determine the power of 2 multiple number of pages */
2319 			scale = __ffs(num_pages);
2320 			cmd->tlbi.scale = scale;
2321 
2322 			/* Determine how many chunks of 2^scale size we have */
2323 			num = (num_pages >> scale) & CMDQ_TLBI_RANGE_NUM_MAX;
2324 			cmd->tlbi.num = num - 1;
2325 
2326 			/* range is num * 2^scale * pgsize */
2327 			inv_range = num << (scale + tg);
2328 
2329 			/* Clear out the lower order bits for the next iteration */
2330 			num_pages -= num << scale;
2331 		}
2332 
2333 		cmd->tlbi.addr = iova;
2334 		arm_smmu_cmdq_batch_add(smmu, &cmds, cmd);
2335 		iova += inv_range;
2336 	}
2337 	arm_smmu_cmdq_batch_submit(smmu, &cmds);
2338 }
2339 
arm_smmu_tlb_inv_range_domain(unsigned long iova,size_t size,size_t granule,bool leaf,struct arm_smmu_domain * smmu_domain)2340 static void arm_smmu_tlb_inv_range_domain(unsigned long iova, size_t size,
2341 					  size_t granule, bool leaf,
2342 					  struct arm_smmu_domain *smmu_domain)
2343 {
2344 	struct arm_smmu_cmdq_ent cmd = {
2345 		.tlbi = {
2346 			.leaf	= leaf,
2347 		},
2348 	};
2349 
2350 	if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {
2351 		cmd.opcode	= smmu_domain->smmu->features & ARM_SMMU_FEAT_E2H ?
2352 				  CMDQ_OP_TLBI_EL2_VA : CMDQ_OP_TLBI_NH_VA;
2353 		cmd.tlbi.asid	= smmu_domain->cd.asid;
2354 	} else {
2355 		cmd.opcode	= CMDQ_OP_TLBI_S2_IPA;
2356 		cmd.tlbi.vmid	= smmu_domain->s2_cfg.vmid;
2357 	}
2358 	__arm_smmu_tlb_inv_range(&cmd, iova, size, granule, smmu_domain);
2359 
2360 	if (smmu_domain->nest_parent) {
2361 		/*
2362 		 * When the S2 domain changes all the nested S1 ASIDs have to be
2363 		 * flushed too.
2364 		 */
2365 		cmd.opcode = CMDQ_OP_TLBI_NH_ALL;
2366 		arm_smmu_cmdq_issue_cmd_with_sync(smmu_domain->smmu, &cmd);
2367 	}
2368 
2369 	/*
2370 	 * Unfortunately, this can't be leaf-only since we may have
2371 	 * zapped an entire table.
2372 	 */
2373 	arm_smmu_atc_inv_domain(smmu_domain, iova, size);
2374 }
2375 
arm_smmu_tlb_inv_range_asid(unsigned long iova,size_t size,int asid,size_t granule,bool leaf,struct arm_smmu_domain * smmu_domain)2376 void arm_smmu_tlb_inv_range_asid(unsigned long iova, size_t size, int asid,
2377 				 size_t granule, bool leaf,
2378 				 struct arm_smmu_domain *smmu_domain)
2379 {
2380 	struct arm_smmu_cmdq_ent cmd = {
2381 		.opcode	= smmu_domain->smmu->features & ARM_SMMU_FEAT_E2H ?
2382 			  CMDQ_OP_TLBI_EL2_VA : CMDQ_OP_TLBI_NH_VA,
2383 		.tlbi = {
2384 			.asid	= asid,
2385 			.leaf	= leaf,
2386 		},
2387 	};
2388 
2389 	__arm_smmu_tlb_inv_range(&cmd, iova, size, granule, smmu_domain);
2390 }
2391 
arm_smmu_tlb_inv_page_nosync(struct iommu_iotlb_gather * gather,unsigned long iova,size_t granule,void * cookie)2392 static void arm_smmu_tlb_inv_page_nosync(struct iommu_iotlb_gather *gather,
2393 					 unsigned long iova, size_t granule,
2394 					 void *cookie)
2395 {
2396 	struct arm_smmu_domain *smmu_domain = cookie;
2397 	struct iommu_domain *domain = &smmu_domain->domain;
2398 
2399 	iommu_iotlb_gather_add_page(domain, gather, iova, granule);
2400 }
2401 
arm_smmu_tlb_inv_walk(unsigned long iova,size_t size,size_t granule,void * cookie)2402 static void arm_smmu_tlb_inv_walk(unsigned long iova, size_t size,
2403 				  size_t granule, void *cookie)
2404 {
2405 	arm_smmu_tlb_inv_range_domain(iova, size, granule, false, cookie);
2406 }
2407 
2408 static const struct iommu_flush_ops arm_smmu_flush_ops = {
2409 	.tlb_flush_all	= arm_smmu_tlb_inv_context,
2410 	.tlb_flush_walk = arm_smmu_tlb_inv_walk,
2411 	.tlb_add_page	= arm_smmu_tlb_inv_page_nosync,
2412 };
2413 
arm_smmu_dbm_capable(struct arm_smmu_device * smmu)2414 static bool arm_smmu_dbm_capable(struct arm_smmu_device *smmu)
2415 {
2416 	u32 features = (ARM_SMMU_FEAT_HD | ARM_SMMU_FEAT_COHERENCY);
2417 
2418 	return (smmu->features & features) == features;
2419 }
2420 
2421 /* IOMMU API */
arm_smmu_capable(struct device * dev,enum iommu_cap cap)2422 static bool arm_smmu_capable(struct device *dev, enum iommu_cap cap)
2423 {
2424 	struct arm_smmu_master *master = dev_iommu_priv_get(dev);
2425 
2426 	switch (cap) {
2427 	case IOMMU_CAP_CACHE_COHERENCY:
2428 		/* Assume that a coherent TCU implies coherent TBUs */
2429 		return master->smmu->features & ARM_SMMU_FEAT_COHERENCY;
2430 	case IOMMU_CAP_ENFORCE_CACHE_COHERENCY:
2431 		return arm_smmu_master_canwbs(master);
2432 	case IOMMU_CAP_NOEXEC:
2433 	case IOMMU_CAP_DEFERRED_FLUSH:
2434 		return true;
2435 	case IOMMU_CAP_DIRTY_TRACKING:
2436 		return arm_smmu_dbm_capable(master->smmu);
2437 	default:
2438 		return false;
2439 	}
2440 }
2441 
arm_smmu_enforce_cache_coherency(struct iommu_domain * domain)2442 static bool arm_smmu_enforce_cache_coherency(struct iommu_domain *domain)
2443 {
2444 	struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
2445 	struct arm_smmu_master_domain *master_domain;
2446 	unsigned long flags;
2447 	bool ret = true;
2448 
2449 	spin_lock_irqsave(&smmu_domain->devices_lock, flags);
2450 	list_for_each_entry(master_domain, &smmu_domain->devices,
2451 			    devices_elm) {
2452 		if (!arm_smmu_master_canwbs(master_domain->master)) {
2453 			ret = false;
2454 			break;
2455 		}
2456 	}
2457 	smmu_domain->enforce_cache_coherency = ret;
2458 	spin_unlock_irqrestore(&smmu_domain->devices_lock, flags);
2459 	return ret;
2460 }
2461 
arm_smmu_domain_alloc(void)2462 struct arm_smmu_domain *arm_smmu_domain_alloc(void)
2463 {
2464 	struct arm_smmu_domain *smmu_domain;
2465 
2466 	smmu_domain = kzalloc(sizeof(*smmu_domain), GFP_KERNEL);
2467 	if (!smmu_domain)
2468 		return ERR_PTR(-ENOMEM);
2469 
2470 	INIT_LIST_HEAD(&smmu_domain->devices);
2471 	spin_lock_init(&smmu_domain->devices_lock);
2472 
2473 	return smmu_domain;
2474 }
2475 
arm_smmu_domain_free_paging(struct iommu_domain * domain)2476 static void arm_smmu_domain_free_paging(struct iommu_domain *domain)
2477 {
2478 	struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
2479 	struct arm_smmu_device *smmu = smmu_domain->smmu;
2480 
2481 	free_io_pgtable_ops(smmu_domain->pgtbl_ops);
2482 
2483 	/* Free the ASID or VMID */
2484 	if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {
2485 		/* Prevent SVA from touching the CD while we're freeing it */
2486 		mutex_lock(&arm_smmu_asid_lock);
2487 		xa_erase(&arm_smmu_asid_xa, smmu_domain->cd.asid);
2488 		mutex_unlock(&arm_smmu_asid_lock);
2489 	} else {
2490 		struct arm_smmu_s2_cfg *cfg = &smmu_domain->s2_cfg;
2491 		if (cfg->vmid)
2492 			ida_free(&smmu->vmid_map, cfg->vmid);
2493 	}
2494 
2495 	kfree(smmu_domain);
2496 }
2497 
arm_smmu_domain_finalise_s1(struct arm_smmu_device * smmu,struct arm_smmu_domain * smmu_domain)2498 static int arm_smmu_domain_finalise_s1(struct arm_smmu_device *smmu,
2499 				       struct arm_smmu_domain *smmu_domain)
2500 {
2501 	int ret;
2502 	u32 asid = 0;
2503 	struct arm_smmu_ctx_desc *cd = &smmu_domain->cd;
2504 
2505 	/* Prevent SVA from modifying the ASID until it is written to the CD */
2506 	mutex_lock(&arm_smmu_asid_lock);
2507 	ret = xa_alloc(&arm_smmu_asid_xa, &asid, smmu_domain,
2508 		       XA_LIMIT(1, (1 << smmu->asid_bits) - 1), GFP_KERNEL);
2509 	cd->asid	= (u16)asid;
2510 	mutex_unlock(&arm_smmu_asid_lock);
2511 	return ret;
2512 }
2513 
arm_smmu_domain_finalise_s2(struct arm_smmu_device * smmu,struct arm_smmu_domain * smmu_domain)2514 static int arm_smmu_domain_finalise_s2(struct arm_smmu_device *smmu,
2515 				       struct arm_smmu_domain *smmu_domain)
2516 {
2517 	int vmid;
2518 	struct arm_smmu_s2_cfg *cfg = &smmu_domain->s2_cfg;
2519 
2520 	/* Reserve VMID 0 for stage-2 bypass STEs */
2521 	vmid = ida_alloc_range(&smmu->vmid_map, 1, (1 << smmu->vmid_bits) - 1,
2522 			       GFP_KERNEL);
2523 	if (vmid < 0)
2524 		return vmid;
2525 
2526 	cfg->vmid	= (u16)vmid;
2527 	return 0;
2528 }
2529 
arm_smmu_domain_finalise(struct arm_smmu_domain * smmu_domain,struct arm_smmu_device * smmu,u32 flags)2530 static int arm_smmu_domain_finalise(struct arm_smmu_domain *smmu_domain,
2531 				    struct arm_smmu_device *smmu, u32 flags)
2532 {
2533 	int ret;
2534 	enum io_pgtable_fmt fmt;
2535 	struct io_pgtable_cfg pgtbl_cfg;
2536 	struct io_pgtable_ops *pgtbl_ops;
2537 	int (*finalise_stage_fn)(struct arm_smmu_device *smmu,
2538 				 struct arm_smmu_domain *smmu_domain);
2539 	bool enable_dirty = flags & IOMMU_HWPT_ALLOC_DIRTY_TRACKING;
2540 
2541 	pgtbl_cfg = (struct io_pgtable_cfg) {
2542 		.pgsize_bitmap	= smmu->pgsize_bitmap,
2543 		.coherent_walk	= smmu->features & ARM_SMMU_FEAT_COHERENCY,
2544 		.tlb		= &arm_smmu_flush_ops,
2545 		.iommu_dev	= smmu->dev,
2546 	};
2547 
2548 	switch (smmu_domain->stage) {
2549 	case ARM_SMMU_DOMAIN_S1: {
2550 		unsigned long ias = (smmu->features &
2551 				     ARM_SMMU_FEAT_VAX) ? 52 : 48;
2552 
2553 		pgtbl_cfg.ias = min_t(unsigned long, ias, VA_BITS);
2554 		pgtbl_cfg.oas = smmu->ias;
2555 		if (enable_dirty)
2556 			pgtbl_cfg.quirks |= IO_PGTABLE_QUIRK_ARM_HD;
2557 		fmt = ARM_64_LPAE_S1;
2558 		finalise_stage_fn = arm_smmu_domain_finalise_s1;
2559 		break;
2560 	}
2561 	case ARM_SMMU_DOMAIN_S2:
2562 		if (enable_dirty)
2563 			return -EOPNOTSUPP;
2564 		pgtbl_cfg.ias = smmu->ias;
2565 		pgtbl_cfg.oas = smmu->oas;
2566 		fmt = ARM_64_LPAE_S2;
2567 		finalise_stage_fn = arm_smmu_domain_finalise_s2;
2568 		if ((smmu->features & ARM_SMMU_FEAT_S2FWB) &&
2569 		    (flags & IOMMU_HWPT_ALLOC_NEST_PARENT))
2570 			pgtbl_cfg.quirks |= IO_PGTABLE_QUIRK_ARM_S2FWB;
2571 		break;
2572 	default:
2573 		return -EINVAL;
2574 	}
2575 
2576 	pgtbl_ops = alloc_io_pgtable_ops(fmt, &pgtbl_cfg, smmu_domain);
2577 	if (!pgtbl_ops)
2578 		return -ENOMEM;
2579 
2580 	smmu_domain->domain.pgsize_bitmap = pgtbl_cfg.pgsize_bitmap;
2581 	smmu_domain->domain.geometry.aperture_end = (1UL << pgtbl_cfg.ias) - 1;
2582 	smmu_domain->domain.geometry.force_aperture = true;
2583 	if (enable_dirty && smmu_domain->stage == ARM_SMMU_DOMAIN_S1)
2584 		smmu_domain->domain.dirty_ops = &arm_smmu_dirty_ops;
2585 
2586 	ret = finalise_stage_fn(smmu, smmu_domain);
2587 	if (ret < 0) {
2588 		free_io_pgtable_ops(pgtbl_ops);
2589 		return ret;
2590 	}
2591 
2592 	smmu_domain->pgtbl_ops = pgtbl_ops;
2593 	smmu_domain->smmu = smmu;
2594 	return 0;
2595 }
2596 
2597 static struct arm_smmu_ste *
arm_smmu_get_step_for_sid(struct arm_smmu_device * smmu,u32 sid)2598 arm_smmu_get_step_for_sid(struct arm_smmu_device *smmu, u32 sid)
2599 {
2600 	struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg;
2601 
2602 	if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB) {
2603 		/* Two-level walk */
2604 		return &cfg->l2.l2ptrs[arm_smmu_strtab_l1_idx(sid)]
2605 				->stes[arm_smmu_strtab_l2_idx(sid)];
2606 	} else {
2607 		/* Simple linear lookup */
2608 		return &cfg->linear.table[sid];
2609 	}
2610 }
2611 
arm_smmu_install_ste_for_dev(struct arm_smmu_master * master,const struct arm_smmu_ste * target)2612 void arm_smmu_install_ste_for_dev(struct arm_smmu_master *master,
2613 				  const struct arm_smmu_ste *target)
2614 {
2615 	int i, j;
2616 	struct arm_smmu_device *smmu = master->smmu;
2617 
2618 	master->cd_table.in_ste =
2619 		FIELD_GET(STRTAB_STE_0_CFG, le64_to_cpu(target->data[0])) ==
2620 		STRTAB_STE_0_CFG_S1_TRANS;
2621 	master->ste_ats_enabled =
2622 		FIELD_GET(STRTAB_STE_1_EATS, le64_to_cpu(target->data[1])) ==
2623 		STRTAB_STE_1_EATS_TRANS;
2624 
2625 	for (i = 0; i < master->num_streams; ++i) {
2626 		u32 sid = master->streams[i].id;
2627 		struct arm_smmu_ste *step =
2628 			arm_smmu_get_step_for_sid(smmu, sid);
2629 
2630 		/* Bridged PCI devices may end up with duplicated IDs */
2631 		for (j = 0; j < i; j++)
2632 			if (master->streams[j].id == sid)
2633 				break;
2634 		if (j < i)
2635 			continue;
2636 
2637 		arm_smmu_write_ste(master, sid, step, target);
2638 	}
2639 }
2640 
arm_smmu_ats_supported(struct arm_smmu_master * master)2641 static bool arm_smmu_ats_supported(struct arm_smmu_master *master)
2642 {
2643 	struct device *dev = master->dev;
2644 	struct arm_smmu_device *smmu = master->smmu;
2645 	struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
2646 
2647 	if (!(smmu->features & ARM_SMMU_FEAT_ATS))
2648 		return false;
2649 
2650 	if (!(fwspec->flags & IOMMU_FWSPEC_PCI_RC_ATS))
2651 		return false;
2652 
2653 	return dev_is_pci(dev) && pci_ats_supported(to_pci_dev(dev));
2654 }
2655 
arm_smmu_enable_ats(struct arm_smmu_master * master)2656 static void arm_smmu_enable_ats(struct arm_smmu_master *master)
2657 {
2658 	size_t stu;
2659 	struct pci_dev *pdev;
2660 	struct arm_smmu_device *smmu = master->smmu;
2661 
2662 	/* Smallest Translation Unit: log2 of the smallest supported granule */
2663 	stu = __ffs(smmu->pgsize_bitmap);
2664 	pdev = to_pci_dev(master->dev);
2665 
2666 	/*
2667 	 * ATC invalidation of PASID 0 causes the entire ATC to be flushed.
2668 	 */
2669 	arm_smmu_atc_inv_master(master, IOMMU_NO_PASID);
2670 	if (pci_enable_ats(pdev, stu))
2671 		dev_err(master->dev, "Failed to enable ATS (STU %zu)\n", stu);
2672 }
2673 
arm_smmu_enable_pasid(struct arm_smmu_master * master)2674 static int arm_smmu_enable_pasid(struct arm_smmu_master *master)
2675 {
2676 	int ret;
2677 	int features;
2678 	int num_pasids;
2679 	struct pci_dev *pdev;
2680 
2681 	if (!dev_is_pci(master->dev))
2682 		return -ENODEV;
2683 
2684 	pdev = to_pci_dev(master->dev);
2685 
2686 	features = pci_pasid_features(pdev);
2687 	if (features < 0)
2688 		return features;
2689 
2690 	num_pasids = pci_max_pasids(pdev);
2691 	if (num_pasids <= 0)
2692 		return num_pasids;
2693 
2694 	ret = pci_enable_pasid(pdev, features);
2695 	if (ret) {
2696 		dev_err(&pdev->dev, "Failed to enable PASID\n");
2697 		return ret;
2698 	}
2699 
2700 	master->ssid_bits = min_t(u8, ilog2(num_pasids),
2701 				  master->smmu->ssid_bits);
2702 	return 0;
2703 }
2704 
arm_smmu_disable_pasid(struct arm_smmu_master * master)2705 static void arm_smmu_disable_pasid(struct arm_smmu_master *master)
2706 {
2707 	struct pci_dev *pdev;
2708 
2709 	if (!dev_is_pci(master->dev))
2710 		return;
2711 
2712 	pdev = to_pci_dev(master->dev);
2713 
2714 	if (!pdev->pasid_enabled)
2715 		return;
2716 
2717 	master->ssid_bits = 0;
2718 	pci_disable_pasid(pdev);
2719 }
2720 
2721 static struct arm_smmu_master_domain *
arm_smmu_find_master_domain(struct arm_smmu_domain * smmu_domain,struct arm_smmu_master * master,ioasid_t ssid,bool nested_ats_flush)2722 arm_smmu_find_master_domain(struct arm_smmu_domain *smmu_domain,
2723 			    struct arm_smmu_master *master,
2724 			    ioasid_t ssid, bool nested_ats_flush)
2725 {
2726 	struct arm_smmu_master_domain *master_domain;
2727 
2728 	lockdep_assert_held(&smmu_domain->devices_lock);
2729 
2730 	list_for_each_entry(master_domain, &smmu_domain->devices,
2731 			    devices_elm) {
2732 		if (master_domain->master == master &&
2733 		    master_domain->ssid == ssid &&
2734 		    master_domain->nested_ats_flush == nested_ats_flush)
2735 			return master_domain;
2736 	}
2737 	return NULL;
2738 }
2739 
2740 /*
2741  * If the domain uses the smmu_domain->devices list return the arm_smmu_domain
2742  * structure, otherwise NULL. These domains track attached devices so they can
2743  * issue invalidations.
2744  */
2745 static struct arm_smmu_domain *
to_smmu_domain_devices(struct iommu_domain * domain)2746 to_smmu_domain_devices(struct iommu_domain *domain)
2747 {
2748 	/* The domain can be NULL only when processing the first attach */
2749 	if (!domain)
2750 		return NULL;
2751 	if ((domain->type & __IOMMU_DOMAIN_PAGING) ||
2752 	    domain->type == IOMMU_DOMAIN_SVA)
2753 		return to_smmu_domain(domain);
2754 	if (domain->type == IOMMU_DOMAIN_NESTED)
2755 		return to_smmu_nested_domain(domain)->vsmmu->s2_parent;
2756 	return NULL;
2757 }
2758 
arm_smmu_remove_master_domain(struct arm_smmu_master * master,struct iommu_domain * domain,ioasid_t ssid)2759 static void arm_smmu_remove_master_domain(struct arm_smmu_master *master,
2760 					  struct iommu_domain *domain,
2761 					  ioasid_t ssid)
2762 {
2763 	struct arm_smmu_domain *smmu_domain = to_smmu_domain_devices(domain);
2764 	struct arm_smmu_master_domain *master_domain;
2765 	bool nested_ats_flush = false;
2766 	unsigned long flags;
2767 
2768 	if (!smmu_domain)
2769 		return;
2770 
2771 	if (domain->type == IOMMU_DOMAIN_NESTED)
2772 		nested_ats_flush = to_smmu_nested_domain(domain)->enable_ats;
2773 
2774 	spin_lock_irqsave(&smmu_domain->devices_lock, flags);
2775 	master_domain = arm_smmu_find_master_domain(smmu_domain, master, ssid,
2776 						    nested_ats_flush);
2777 	if (master_domain) {
2778 		list_del(&master_domain->devices_elm);
2779 		kfree(master_domain);
2780 		if (master->ats_enabled)
2781 			atomic_dec(&smmu_domain->nr_ats_masters);
2782 	}
2783 	spin_unlock_irqrestore(&smmu_domain->devices_lock, flags);
2784 }
2785 
2786 /*
2787  * Start the sequence to attach a domain to a master. The sequence contains three
2788  * steps:
2789  *  arm_smmu_attach_prepare()
2790  *  arm_smmu_install_ste_for_dev()
2791  *  arm_smmu_attach_commit()
2792  *
2793  * If prepare succeeds then the sequence must be completed. The STE installed
2794  * must set the STE.EATS field according to state.ats_enabled.
2795  *
2796  * If the device supports ATS then this determines if EATS should be enabled
2797  * in the STE, and starts sequencing EATS disable if required.
2798  *
2799  * The change of the EATS in the STE and the PCI ATS config space is managed by
2800  * this sequence to be in the right order so that if PCI ATS is enabled then
2801  * STE.ETAS is enabled.
2802  *
2803  * new_domain can be a non-paging domain. In this case ATS will not be enabled,
2804  * and invalidations won't be tracked.
2805  */
arm_smmu_attach_prepare(struct arm_smmu_attach_state * state,struct iommu_domain * new_domain)2806 int arm_smmu_attach_prepare(struct arm_smmu_attach_state *state,
2807 			    struct iommu_domain *new_domain)
2808 {
2809 	struct arm_smmu_master *master = state->master;
2810 	struct arm_smmu_master_domain *master_domain;
2811 	struct arm_smmu_domain *smmu_domain =
2812 		to_smmu_domain_devices(new_domain);
2813 	unsigned long flags;
2814 	int ret;
2815 
2816 	/*
2817 	 * arm_smmu_share_asid() must not see two domains pointing to the same
2818 	 * arm_smmu_master_domain contents otherwise it could randomly write one
2819 	 * or the other to the CD.
2820 	 */
2821 	lockdep_assert_held(&arm_smmu_asid_lock);
2822 
2823 	if (smmu_domain || state->cd_needs_ats) {
2824 		/*
2825 		 * The SMMU does not support enabling ATS with bypass/abort.
2826 		 * When the STE is in bypass (STE.Config[2:0] == 0b100), ATS
2827 		 * Translation Requests and Translated transactions are denied
2828 		 * as though ATS is disabled for the stream (STE.EATS == 0b00),
2829 		 * causing F_BAD_ATS_TREQ and F_TRANSL_FORBIDDEN events
2830 		 * (IHI0070Ea 5.2 Stream Table Entry).
2831 		 *
2832 		 * However, if we have installed a CD table and are using S1DSS
2833 		 * then ATS will work in S1DSS bypass. See "13.6.4 Full ATS
2834 		 * skipping stage 1".
2835 		 *
2836 		 * Disable ATS if we are going to create a normal 0b100 bypass
2837 		 * STE.
2838 		 */
2839 		state->ats_enabled = !state->disable_ats &&
2840 				     arm_smmu_ats_supported(master);
2841 	}
2842 
2843 	if (smmu_domain) {
2844 		if (new_domain->type == IOMMU_DOMAIN_NESTED) {
2845 			ret = arm_smmu_attach_prepare_vmaster(
2846 				state, to_smmu_nested_domain(new_domain));
2847 			if (ret)
2848 				return ret;
2849 		}
2850 
2851 		master_domain = kzalloc(sizeof(*master_domain), GFP_KERNEL);
2852 		if (!master_domain) {
2853 			kfree(state->vmaster);
2854 			return -ENOMEM;
2855 		}
2856 		master_domain->master = master;
2857 		master_domain->ssid = state->ssid;
2858 		if (new_domain->type == IOMMU_DOMAIN_NESTED)
2859 			master_domain->nested_ats_flush =
2860 				to_smmu_nested_domain(new_domain)->enable_ats;
2861 
2862 		/*
2863 		 * During prepare we want the current smmu_domain and new
2864 		 * smmu_domain to be in the devices list before we change any
2865 		 * HW. This ensures that both domains will send ATS
2866 		 * invalidations to the master until we are done.
2867 		 *
2868 		 * It is tempting to make this list only track masters that are
2869 		 * using ATS, but arm_smmu_share_asid() also uses this to change
2870 		 * the ASID of a domain, unrelated to ATS.
2871 		 *
2872 		 * Notice if we are re-attaching the same domain then the list
2873 		 * will have two identical entries and commit will remove only
2874 		 * one of them.
2875 		 */
2876 		spin_lock_irqsave(&smmu_domain->devices_lock, flags);
2877 		if (smmu_domain->enforce_cache_coherency &&
2878 		    !arm_smmu_master_canwbs(master)) {
2879 			spin_unlock_irqrestore(&smmu_domain->devices_lock,
2880 					       flags);
2881 			kfree(master_domain);
2882 			kfree(state->vmaster);
2883 			return -EINVAL;
2884 		}
2885 
2886 		if (state->ats_enabled)
2887 			atomic_inc(&smmu_domain->nr_ats_masters);
2888 		list_add(&master_domain->devices_elm, &smmu_domain->devices);
2889 		spin_unlock_irqrestore(&smmu_domain->devices_lock, flags);
2890 	}
2891 
2892 	if (!state->ats_enabled && master->ats_enabled) {
2893 		pci_disable_ats(to_pci_dev(master->dev));
2894 		/*
2895 		 * This is probably overkill, but the config write for disabling
2896 		 * ATS should complete before the STE is configured to generate
2897 		 * UR to avoid AER noise.
2898 		 */
2899 		wmb();
2900 	}
2901 	return 0;
2902 }
2903 
2904 /*
2905  * Commit is done after the STE/CD are configured with the EATS setting. It
2906  * completes synchronizing the PCI device's ATC and finishes manipulating the
2907  * smmu_domain->devices list.
2908  */
arm_smmu_attach_commit(struct arm_smmu_attach_state * state)2909 void arm_smmu_attach_commit(struct arm_smmu_attach_state *state)
2910 {
2911 	struct arm_smmu_master *master = state->master;
2912 
2913 	lockdep_assert_held(&arm_smmu_asid_lock);
2914 
2915 	arm_smmu_attach_commit_vmaster(state);
2916 
2917 	if (state->ats_enabled && !master->ats_enabled) {
2918 		arm_smmu_enable_ats(master);
2919 	} else if (state->ats_enabled && master->ats_enabled) {
2920 		/*
2921 		 * The translation has changed, flush the ATC. At this point the
2922 		 * SMMU is translating for the new domain and both the old&new
2923 		 * domain will issue invalidations.
2924 		 */
2925 		arm_smmu_atc_inv_master(master, state->ssid);
2926 	} else if (!state->ats_enabled && master->ats_enabled) {
2927 		/* ATS is being switched off, invalidate the entire ATC */
2928 		arm_smmu_atc_inv_master(master, IOMMU_NO_PASID);
2929 	}
2930 	master->ats_enabled = state->ats_enabled;
2931 
2932 	arm_smmu_remove_master_domain(master, state->old_domain, state->ssid);
2933 }
2934 
arm_smmu_attach_dev(struct iommu_domain * domain,struct device * dev)2935 static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev)
2936 {
2937 	int ret = 0;
2938 	struct arm_smmu_ste target;
2939 	struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
2940 	struct arm_smmu_device *smmu;
2941 	struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
2942 	struct arm_smmu_attach_state state = {
2943 		.old_domain = iommu_get_domain_for_dev(dev),
2944 		.ssid = IOMMU_NO_PASID,
2945 	};
2946 	struct arm_smmu_master *master;
2947 	struct arm_smmu_cd *cdptr;
2948 
2949 	if (!fwspec)
2950 		return -ENOENT;
2951 
2952 	state.master = master = dev_iommu_priv_get(dev);
2953 	smmu = master->smmu;
2954 
2955 	if (smmu_domain->smmu != smmu)
2956 		return ret;
2957 
2958 	if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {
2959 		cdptr = arm_smmu_alloc_cd_ptr(master, IOMMU_NO_PASID);
2960 		if (!cdptr)
2961 			return -ENOMEM;
2962 	} else if (arm_smmu_ssids_in_use(&master->cd_table))
2963 		return -EBUSY;
2964 
2965 	/*
2966 	 * Prevent arm_smmu_share_asid() from trying to change the ASID
2967 	 * of either the old or new domain while we are working on it.
2968 	 * This allows the STE and the smmu_domain->devices list to
2969 	 * be inconsistent during this routine.
2970 	 */
2971 	mutex_lock(&arm_smmu_asid_lock);
2972 
2973 	ret = arm_smmu_attach_prepare(&state, domain);
2974 	if (ret) {
2975 		mutex_unlock(&arm_smmu_asid_lock);
2976 		return ret;
2977 	}
2978 
2979 	switch (smmu_domain->stage) {
2980 	case ARM_SMMU_DOMAIN_S1: {
2981 		struct arm_smmu_cd target_cd;
2982 
2983 		arm_smmu_make_s1_cd(&target_cd, master, smmu_domain);
2984 		arm_smmu_write_cd_entry(master, IOMMU_NO_PASID, cdptr,
2985 					&target_cd);
2986 		arm_smmu_make_cdtable_ste(&target, master, state.ats_enabled,
2987 					  STRTAB_STE_1_S1DSS_SSID0);
2988 		arm_smmu_install_ste_for_dev(master, &target);
2989 		break;
2990 	}
2991 	case ARM_SMMU_DOMAIN_S2:
2992 		arm_smmu_make_s2_domain_ste(&target, master, smmu_domain,
2993 					    state.ats_enabled);
2994 		arm_smmu_install_ste_for_dev(master, &target);
2995 		arm_smmu_clear_cd(master, IOMMU_NO_PASID);
2996 		break;
2997 	}
2998 
2999 	arm_smmu_attach_commit(&state);
3000 	mutex_unlock(&arm_smmu_asid_lock);
3001 	return 0;
3002 }
3003 
arm_smmu_s1_set_dev_pasid(struct iommu_domain * domain,struct device * dev,ioasid_t id,struct iommu_domain * old)3004 static int arm_smmu_s1_set_dev_pasid(struct iommu_domain *domain,
3005 				     struct device *dev, ioasid_t id,
3006 				     struct iommu_domain *old)
3007 {
3008 	struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
3009 	struct arm_smmu_master *master = dev_iommu_priv_get(dev);
3010 	struct arm_smmu_device *smmu = master->smmu;
3011 	struct arm_smmu_cd target_cd;
3012 
3013 	if (smmu_domain->smmu != smmu)
3014 		return -EINVAL;
3015 
3016 	if (smmu_domain->stage != ARM_SMMU_DOMAIN_S1)
3017 		return -EINVAL;
3018 
3019 	/*
3020 	 * We can read cd.asid outside the lock because arm_smmu_set_pasid()
3021 	 * will fix it
3022 	 */
3023 	arm_smmu_make_s1_cd(&target_cd, master, smmu_domain);
3024 	return arm_smmu_set_pasid(master, to_smmu_domain(domain), id,
3025 				  &target_cd, old);
3026 }
3027 
arm_smmu_update_ste(struct arm_smmu_master * master,struct iommu_domain * sid_domain,bool ats_enabled)3028 static void arm_smmu_update_ste(struct arm_smmu_master *master,
3029 				struct iommu_domain *sid_domain,
3030 				bool ats_enabled)
3031 {
3032 	unsigned int s1dss = STRTAB_STE_1_S1DSS_TERMINATE;
3033 	struct arm_smmu_ste ste;
3034 
3035 	if (master->cd_table.in_ste && master->ste_ats_enabled == ats_enabled)
3036 		return;
3037 
3038 	if (sid_domain->type == IOMMU_DOMAIN_IDENTITY)
3039 		s1dss = STRTAB_STE_1_S1DSS_BYPASS;
3040 	else
3041 		WARN_ON(sid_domain->type != IOMMU_DOMAIN_BLOCKED);
3042 
3043 	/*
3044 	 * Change the STE into a cdtable one with SID IDENTITY/BLOCKED behavior
3045 	 * using s1dss if necessary. If the cd_table is already installed then
3046 	 * the S1DSS is correct and this will just update the EATS. Otherwise it
3047 	 * installs the entire thing. This will be hitless.
3048 	 */
3049 	arm_smmu_make_cdtable_ste(&ste, master, ats_enabled, s1dss);
3050 	arm_smmu_install_ste_for_dev(master, &ste);
3051 }
3052 
arm_smmu_set_pasid(struct arm_smmu_master * master,struct arm_smmu_domain * smmu_domain,ioasid_t pasid,struct arm_smmu_cd * cd,struct iommu_domain * old)3053 int arm_smmu_set_pasid(struct arm_smmu_master *master,
3054 		       struct arm_smmu_domain *smmu_domain, ioasid_t pasid,
3055 		       struct arm_smmu_cd *cd, struct iommu_domain *old)
3056 {
3057 	struct iommu_domain *sid_domain = iommu_get_domain_for_dev(master->dev);
3058 	struct arm_smmu_attach_state state = {
3059 		.master = master,
3060 		.ssid = pasid,
3061 		.old_domain = old,
3062 	};
3063 	struct arm_smmu_cd *cdptr;
3064 	int ret;
3065 
3066 	/* The core code validates pasid */
3067 
3068 	if (smmu_domain->smmu != master->smmu)
3069 		return -EINVAL;
3070 
3071 	if (!master->cd_table.in_ste &&
3072 	    sid_domain->type != IOMMU_DOMAIN_IDENTITY &&
3073 	    sid_domain->type != IOMMU_DOMAIN_BLOCKED)
3074 		return -EINVAL;
3075 
3076 	cdptr = arm_smmu_alloc_cd_ptr(master, pasid);
3077 	if (!cdptr)
3078 		return -ENOMEM;
3079 
3080 	mutex_lock(&arm_smmu_asid_lock);
3081 	ret = arm_smmu_attach_prepare(&state, &smmu_domain->domain);
3082 	if (ret)
3083 		goto out_unlock;
3084 
3085 	/*
3086 	 * We don't want to obtain to the asid_lock too early, so fix up the
3087 	 * caller set ASID under the lock in case it changed.
3088 	 */
3089 	cd->data[0] &= ~cpu_to_le64(CTXDESC_CD_0_ASID);
3090 	cd->data[0] |= cpu_to_le64(
3091 		FIELD_PREP(CTXDESC_CD_0_ASID, smmu_domain->cd.asid));
3092 
3093 	arm_smmu_write_cd_entry(master, pasid, cdptr, cd);
3094 	arm_smmu_update_ste(master, sid_domain, state.ats_enabled);
3095 
3096 	arm_smmu_attach_commit(&state);
3097 
3098 out_unlock:
3099 	mutex_unlock(&arm_smmu_asid_lock);
3100 	return ret;
3101 }
3102 
arm_smmu_blocking_set_dev_pasid(struct iommu_domain * new_domain,struct device * dev,ioasid_t pasid,struct iommu_domain * old_domain)3103 static int arm_smmu_blocking_set_dev_pasid(struct iommu_domain *new_domain,
3104 					   struct device *dev, ioasid_t pasid,
3105 					   struct iommu_domain *old_domain)
3106 {
3107 	struct arm_smmu_domain *smmu_domain = to_smmu_domain(old_domain);
3108 	struct arm_smmu_master *master = dev_iommu_priv_get(dev);
3109 
3110 	mutex_lock(&arm_smmu_asid_lock);
3111 	arm_smmu_clear_cd(master, pasid);
3112 	if (master->ats_enabled)
3113 		arm_smmu_atc_inv_master(master, pasid);
3114 	arm_smmu_remove_master_domain(master, &smmu_domain->domain, pasid);
3115 	mutex_unlock(&arm_smmu_asid_lock);
3116 
3117 	/*
3118 	 * When the last user of the CD table goes away downgrade the STE back
3119 	 * to a non-cd_table one.
3120 	 */
3121 	if (!arm_smmu_ssids_in_use(&master->cd_table)) {
3122 		struct iommu_domain *sid_domain =
3123 			iommu_get_domain_for_dev(master->dev);
3124 
3125 		if (sid_domain->type == IOMMU_DOMAIN_IDENTITY ||
3126 		    sid_domain->type == IOMMU_DOMAIN_BLOCKED)
3127 			sid_domain->ops->attach_dev(sid_domain, dev);
3128 	}
3129 	return 0;
3130 }
3131 
arm_smmu_attach_dev_ste(struct iommu_domain * domain,struct device * dev,struct arm_smmu_ste * ste,unsigned int s1dss)3132 static void arm_smmu_attach_dev_ste(struct iommu_domain *domain,
3133 				    struct device *dev,
3134 				    struct arm_smmu_ste *ste,
3135 				    unsigned int s1dss)
3136 {
3137 	struct arm_smmu_master *master = dev_iommu_priv_get(dev);
3138 	struct arm_smmu_attach_state state = {
3139 		.master = master,
3140 		.old_domain = iommu_get_domain_for_dev(dev),
3141 		.ssid = IOMMU_NO_PASID,
3142 	};
3143 
3144 	/*
3145 	 * Do not allow any ASID to be changed while are working on the STE,
3146 	 * otherwise we could miss invalidations.
3147 	 */
3148 	mutex_lock(&arm_smmu_asid_lock);
3149 
3150 	/*
3151 	 * If the CD table is not in use we can use the provided STE, otherwise
3152 	 * we use a cdtable STE with the provided S1DSS.
3153 	 */
3154 	if (arm_smmu_ssids_in_use(&master->cd_table)) {
3155 		/*
3156 		 * If a CD table has to be present then we need to run with ATS
3157 		 * on because we have to assume a PASID is using ATS. For
3158 		 * IDENTITY this will setup things so that S1DSS=bypass which
3159 		 * follows the explanation in "13.6.4 Full ATS skipping stage 1"
3160 		 * and allows for ATS on the RID to work.
3161 		 */
3162 		state.cd_needs_ats = true;
3163 		arm_smmu_attach_prepare(&state, domain);
3164 		arm_smmu_make_cdtable_ste(ste, master, state.ats_enabled, s1dss);
3165 	} else {
3166 		arm_smmu_attach_prepare(&state, domain);
3167 	}
3168 	arm_smmu_install_ste_for_dev(master, ste);
3169 	arm_smmu_attach_commit(&state);
3170 	mutex_unlock(&arm_smmu_asid_lock);
3171 
3172 	/*
3173 	 * This has to be done after removing the master from the
3174 	 * arm_smmu_domain->devices to avoid races updating the same context
3175 	 * descriptor from arm_smmu_share_asid().
3176 	 */
3177 	arm_smmu_clear_cd(master, IOMMU_NO_PASID);
3178 }
3179 
arm_smmu_attach_dev_identity(struct iommu_domain * domain,struct device * dev)3180 static int arm_smmu_attach_dev_identity(struct iommu_domain *domain,
3181 					struct device *dev)
3182 {
3183 	struct arm_smmu_ste ste;
3184 	struct arm_smmu_master *master = dev_iommu_priv_get(dev);
3185 
3186 	arm_smmu_master_clear_vmaster(master);
3187 	arm_smmu_make_bypass_ste(master->smmu, &ste);
3188 	arm_smmu_attach_dev_ste(domain, dev, &ste, STRTAB_STE_1_S1DSS_BYPASS);
3189 	return 0;
3190 }
3191 
3192 static const struct iommu_domain_ops arm_smmu_identity_ops = {
3193 	.attach_dev = arm_smmu_attach_dev_identity,
3194 };
3195 
3196 static struct iommu_domain arm_smmu_identity_domain = {
3197 	.type = IOMMU_DOMAIN_IDENTITY,
3198 	.ops = &arm_smmu_identity_ops,
3199 };
3200 
arm_smmu_attach_dev_blocked(struct iommu_domain * domain,struct device * dev)3201 static int arm_smmu_attach_dev_blocked(struct iommu_domain *domain,
3202 					struct device *dev)
3203 {
3204 	struct arm_smmu_ste ste;
3205 	struct arm_smmu_master *master = dev_iommu_priv_get(dev);
3206 
3207 	arm_smmu_master_clear_vmaster(master);
3208 	arm_smmu_make_abort_ste(&ste);
3209 	arm_smmu_attach_dev_ste(domain, dev, &ste,
3210 				STRTAB_STE_1_S1DSS_TERMINATE);
3211 	return 0;
3212 }
3213 
3214 static const struct iommu_domain_ops arm_smmu_blocked_ops = {
3215 	.attach_dev = arm_smmu_attach_dev_blocked,
3216 	.set_dev_pasid = arm_smmu_blocking_set_dev_pasid,
3217 };
3218 
3219 static struct iommu_domain arm_smmu_blocked_domain = {
3220 	.type = IOMMU_DOMAIN_BLOCKED,
3221 	.ops = &arm_smmu_blocked_ops,
3222 };
3223 
3224 static struct iommu_domain *
arm_smmu_domain_alloc_paging_flags(struct device * dev,u32 flags,const struct iommu_user_data * user_data)3225 arm_smmu_domain_alloc_paging_flags(struct device *dev, u32 flags,
3226 				   const struct iommu_user_data *user_data)
3227 {
3228 	struct arm_smmu_master *master = dev_iommu_priv_get(dev);
3229 	struct arm_smmu_device *smmu = master->smmu;
3230 	const u32 PAGING_FLAGS = IOMMU_HWPT_ALLOC_DIRTY_TRACKING |
3231 				 IOMMU_HWPT_ALLOC_PASID |
3232 				 IOMMU_HWPT_ALLOC_NEST_PARENT;
3233 	struct arm_smmu_domain *smmu_domain;
3234 	int ret;
3235 
3236 	if (flags & ~PAGING_FLAGS)
3237 		return ERR_PTR(-EOPNOTSUPP);
3238 	if (user_data)
3239 		return ERR_PTR(-EOPNOTSUPP);
3240 
3241 	smmu_domain = arm_smmu_domain_alloc();
3242 	if (IS_ERR(smmu_domain))
3243 		return ERR_CAST(smmu_domain);
3244 
3245 	switch (flags) {
3246 	case 0:
3247 		/* Prefer S1 if available */
3248 		if (smmu->features & ARM_SMMU_FEAT_TRANS_S1)
3249 			smmu_domain->stage = ARM_SMMU_DOMAIN_S1;
3250 		else
3251 			smmu_domain->stage = ARM_SMMU_DOMAIN_S2;
3252 		break;
3253 	case IOMMU_HWPT_ALLOC_NEST_PARENT:
3254 		if (!(smmu->features & ARM_SMMU_FEAT_NESTING)) {
3255 			ret = -EOPNOTSUPP;
3256 			goto err_free;
3257 		}
3258 		smmu_domain->stage = ARM_SMMU_DOMAIN_S2;
3259 		smmu_domain->nest_parent = true;
3260 		break;
3261 	case IOMMU_HWPT_ALLOC_DIRTY_TRACKING:
3262 	case IOMMU_HWPT_ALLOC_DIRTY_TRACKING | IOMMU_HWPT_ALLOC_PASID:
3263 	case IOMMU_HWPT_ALLOC_PASID:
3264 		if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S1)) {
3265 			ret = -EOPNOTSUPP;
3266 			goto err_free;
3267 		}
3268 		smmu_domain->stage = ARM_SMMU_DOMAIN_S1;
3269 		break;
3270 	default:
3271 		ret = -EOPNOTSUPP;
3272 		goto err_free;
3273 	}
3274 
3275 	smmu_domain->domain.type = IOMMU_DOMAIN_UNMANAGED;
3276 	smmu_domain->domain.ops = arm_smmu_ops.default_domain_ops;
3277 	ret = arm_smmu_domain_finalise(smmu_domain, smmu, flags);
3278 	if (ret)
3279 		goto err_free;
3280 	return &smmu_domain->domain;
3281 
3282 err_free:
3283 	kfree(smmu_domain);
3284 	return ERR_PTR(ret);
3285 }
3286 
arm_smmu_map_pages(struct iommu_domain * domain,unsigned long iova,phys_addr_t paddr,size_t pgsize,size_t pgcount,int prot,gfp_t gfp,size_t * mapped)3287 static int arm_smmu_map_pages(struct iommu_domain *domain, unsigned long iova,
3288 			      phys_addr_t paddr, size_t pgsize, size_t pgcount,
3289 			      int prot, gfp_t gfp, size_t *mapped)
3290 {
3291 	struct io_pgtable_ops *ops = to_smmu_domain(domain)->pgtbl_ops;
3292 
3293 	if (!ops)
3294 		return -ENODEV;
3295 
3296 	return ops->map_pages(ops, iova, paddr, pgsize, pgcount, prot, gfp, mapped);
3297 }
3298 
arm_smmu_unmap_pages(struct iommu_domain * domain,unsigned long iova,size_t pgsize,size_t pgcount,struct iommu_iotlb_gather * gather)3299 static size_t arm_smmu_unmap_pages(struct iommu_domain *domain, unsigned long iova,
3300 				   size_t pgsize, size_t pgcount,
3301 				   struct iommu_iotlb_gather *gather)
3302 {
3303 	struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
3304 	struct io_pgtable_ops *ops = smmu_domain->pgtbl_ops;
3305 
3306 	if (!ops)
3307 		return 0;
3308 
3309 	return ops->unmap_pages(ops, iova, pgsize, pgcount, gather);
3310 }
3311 
arm_smmu_flush_iotlb_all(struct iommu_domain * domain)3312 static void arm_smmu_flush_iotlb_all(struct iommu_domain *domain)
3313 {
3314 	struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
3315 
3316 	if (smmu_domain->smmu)
3317 		arm_smmu_tlb_inv_context(smmu_domain);
3318 }
3319 
arm_smmu_iotlb_sync(struct iommu_domain * domain,struct iommu_iotlb_gather * gather)3320 static void arm_smmu_iotlb_sync(struct iommu_domain *domain,
3321 				struct iommu_iotlb_gather *gather)
3322 {
3323 	struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
3324 
3325 	if (!gather->pgsize)
3326 		return;
3327 
3328 	arm_smmu_tlb_inv_range_domain(gather->start,
3329 				      gather->end - gather->start + 1,
3330 				      gather->pgsize, true, smmu_domain);
3331 }
3332 
3333 static phys_addr_t
arm_smmu_iova_to_phys(struct iommu_domain * domain,dma_addr_t iova)3334 arm_smmu_iova_to_phys(struct iommu_domain *domain, dma_addr_t iova)
3335 {
3336 	struct io_pgtable_ops *ops = to_smmu_domain(domain)->pgtbl_ops;
3337 
3338 	if (!ops)
3339 		return 0;
3340 
3341 	return ops->iova_to_phys(ops, iova);
3342 }
3343 
3344 static struct platform_driver arm_smmu_driver;
3345 
3346 static
arm_smmu_get_by_fwnode(struct fwnode_handle * fwnode)3347 struct arm_smmu_device *arm_smmu_get_by_fwnode(struct fwnode_handle *fwnode)
3348 {
3349 	struct device *dev = bus_find_device_by_fwnode(&platform_bus_type, fwnode);
3350 
3351 	put_device(dev);
3352 	return dev ? dev_get_drvdata(dev) : NULL;
3353 }
3354 
arm_smmu_sid_in_range(struct arm_smmu_device * smmu,u32 sid)3355 static bool arm_smmu_sid_in_range(struct arm_smmu_device *smmu, u32 sid)
3356 {
3357 	if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB)
3358 		return arm_smmu_strtab_l1_idx(sid) < smmu->strtab_cfg.l2.num_l1_ents;
3359 	return sid < smmu->strtab_cfg.linear.num_ents;
3360 }
3361 
arm_smmu_init_sid_strtab(struct arm_smmu_device * smmu,u32 sid)3362 static int arm_smmu_init_sid_strtab(struct arm_smmu_device *smmu, u32 sid)
3363 {
3364 	/* Check the SIDs are in range of the SMMU and our stream table */
3365 	if (!arm_smmu_sid_in_range(smmu, sid))
3366 		return -ERANGE;
3367 
3368 	/* Ensure l2 strtab is initialised */
3369 	if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB)
3370 		return arm_smmu_init_l2_strtab(smmu, sid);
3371 
3372 	return 0;
3373 }
3374 
arm_smmu_insert_master(struct arm_smmu_device * smmu,struct arm_smmu_master * master)3375 static int arm_smmu_insert_master(struct arm_smmu_device *smmu,
3376 				  struct arm_smmu_master *master)
3377 {
3378 	int i;
3379 	int ret = 0;
3380 	struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(master->dev);
3381 
3382 	master->streams = kcalloc(fwspec->num_ids, sizeof(*master->streams),
3383 				  GFP_KERNEL);
3384 	if (!master->streams)
3385 		return -ENOMEM;
3386 	master->num_streams = fwspec->num_ids;
3387 
3388 	mutex_lock(&smmu->streams_mutex);
3389 	for (i = 0; i < fwspec->num_ids; i++) {
3390 		struct arm_smmu_stream *new_stream = &master->streams[i];
3391 		struct rb_node *existing;
3392 		u32 sid = fwspec->ids[i];
3393 
3394 		new_stream->id = sid;
3395 		new_stream->master = master;
3396 
3397 		ret = arm_smmu_init_sid_strtab(smmu, sid);
3398 		if (ret)
3399 			break;
3400 
3401 		/* Insert into SID tree */
3402 		existing = rb_find_add(&new_stream->node, &smmu->streams,
3403 				       arm_smmu_streams_cmp_node);
3404 		if (existing) {
3405 			struct arm_smmu_master *existing_master =
3406 				rb_entry(existing, struct arm_smmu_stream, node)
3407 					->master;
3408 
3409 			/* Bridged PCI devices may end up with duplicated IDs */
3410 			if (existing_master == master)
3411 				continue;
3412 
3413 			dev_warn(master->dev,
3414 				 "Aliasing StreamID 0x%x (from %s) unsupported, expect DMA to be broken\n",
3415 				 sid, dev_name(existing_master->dev));
3416 			ret = -ENODEV;
3417 			break;
3418 		}
3419 	}
3420 
3421 	if (ret) {
3422 		for (i--; i >= 0; i--)
3423 			rb_erase(&master->streams[i].node, &smmu->streams);
3424 		kfree(master->streams);
3425 	}
3426 	mutex_unlock(&smmu->streams_mutex);
3427 
3428 	return ret;
3429 }
3430 
arm_smmu_remove_master(struct arm_smmu_master * master)3431 static void arm_smmu_remove_master(struct arm_smmu_master *master)
3432 {
3433 	int i;
3434 	struct arm_smmu_device *smmu = master->smmu;
3435 	struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(master->dev);
3436 
3437 	if (!smmu || !master->streams)
3438 		return;
3439 
3440 	mutex_lock(&smmu->streams_mutex);
3441 	for (i = 0; i < fwspec->num_ids; i++)
3442 		rb_erase(&master->streams[i].node, &smmu->streams);
3443 	mutex_unlock(&smmu->streams_mutex);
3444 
3445 	kfree(master->streams);
3446 }
3447 
arm_smmu_probe_device(struct device * dev)3448 static struct iommu_device *arm_smmu_probe_device(struct device *dev)
3449 {
3450 	int ret;
3451 	struct arm_smmu_device *smmu;
3452 	struct arm_smmu_master *master;
3453 	struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
3454 
3455 	if (WARN_ON_ONCE(dev_iommu_priv_get(dev)))
3456 		return ERR_PTR(-EBUSY);
3457 
3458 	smmu = arm_smmu_get_by_fwnode(fwspec->iommu_fwnode);
3459 	if (!smmu)
3460 		return ERR_PTR(-ENODEV);
3461 
3462 	master = kzalloc(sizeof(*master), GFP_KERNEL);
3463 	if (!master)
3464 		return ERR_PTR(-ENOMEM);
3465 
3466 	master->dev = dev;
3467 	master->smmu = smmu;
3468 	dev_iommu_priv_set(dev, master);
3469 
3470 	ret = arm_smmu_insert_master(smmu, master);
3471 	if (ret)
3472 		goto err_free_master;
3473 
3474 	device_property_read_u32(dev, "pasid-num-bits", &master->ssid_bits);
3475 	master->ssid_bits = min(smmu->ssid_bits, master->ssid_bits);
3476 
3477 	/*
3478 	 * Note that PASID must be enabled before, and disabled after ATS:
3479 	 * PCI Express Base 4.0r1.0 - 10.5.1.3 ATS Control Register
3480 	 *
3481 	 *   Behavior is undefined if this bit is Set and the value of the PASID
3482 	 *   Enable, Execute Requested Enable, or Privileged Mode Requested bits
3483 	 *   are changed.
3484 	 */
3485 	arm_smmu_enable_pasid(master);
3486 
3487 	if (!(smmu->features & ARM_SMMU_FEAT_2_LVL_CDTAB))
3488 		master->ssid_bits = min_t(u8, master->ssid_bits,
3489 					  CTXDESC_LINEAR_CDMAX);
3490 
3491 	if ((smmu->features & ARM_SMMU_FEAT_STALLS &&
3492 	     device_property_read_bool(dev, "dma-can-stall")) ||
3493 	    smmu->features & ARM_SMMU_FEAT_STALL_FORCE)
3494 		master->stall_enabled = true;
3495 
3496 	if (dev_is_pci(dev)) {
3497 		unsigned int stu = __ffs(smmu->pgsize_bitmap);
3498 
3499 		pci_prepare_ats(to_pci_dev(dev), stu);
3500 	}
3501 
3502 	return &smmu->iommu;
3503 
3504 err_free_master:
3505 	kfree(master);
3506 	return ERR_PTR(ret);
3507 }
3508 
arm_smmu_release_device(struct device * dev)3509 static void arm_smmu_release_device(struct device *dev)
3510 {
3511 	struct arm_smmu_master *master = dev_iommu_priv_get(dev);
3512 
3513 	if (WARN_ON(arm_smmu_master_sva_enabled(master)))
3514 		iopf_queue_remove_device(master->smmu->evtq.iopf, dev);
3515 
3516 	/* Put the STE back to what arm_smmu_init_strtab() sets */
3517 	if (dev->iommu->require_direct)
3518 		arm_smmu_attach_dev_identity(&arm_smmu_identity_domain, dev);
3519 	else
3520 		arm_smmu_attach_dev_blocked(&arm_smmu_blocked_domain, dev);
3521 
3522 	arm_smmu_disable_pasid(master);
3523 	arm_smmu_remove_master(master);
3524 	if (arm_smmu_cdtab_allocated(&master->cd_table))
3525 		arm_smmu_free_cd_tables(master);
3526 	kfree(master);
3527 }
3528 
arm_smmu_read_and_clear_dirty(struct iommu_domain * domain,unsigned long iova,size_t size,unsigned long flags,struct iommu_dirty_bitmap * dirty)3529 static int arm_smmu_read_and_clear_dirty(struct iommu_domain *domain,
3530 					 unsigned long iova, size_t size,
3531 					 unsigned long flags,
3532 					 struct iommu_dirty_bitmap *dirty)
3533 {
3534 	struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
3535 	struct io_pgtable_ops *ops = smmu_domain->pgtbl_ops;
3536 
3537 	return ops->read_and_clear_dirty(ops, iova, size, flags, dirty);
3538 }
3539 
arm_smmu_set_dirty_tracking(struct iommu_domain * domain,bool enabled)3540 static int arm_smmu_set_dirty_tracking(struct iommu_domain *domain,
3541 				       bool enabled)
3542 {
3543 	/*
3544 	 * Always enabled and the dirty bitmap is cleared prior to
3545 	 * set_dirty_tracking().
3546 	 */
3547 	return 0;
3548 }
3549 
arm_smmu_device_group(struct device * dev)3550 static struct iommu_group *arm_smmu_device_group(struct device *dev)
3551 {
3552 	struct iommu_group *group;
3553 
3554 	/*
3555 	 * We don't support devices sharing stream IDs other than PCI RID
3556 	 * aliases, since the necessary ID-to-device lookup becomes rather
3557 	 * impractical given a potential sparse 32-bit stream ID space.
3558 	 */
3559 	if (dev_is_pci(dev))
3560 		group = pci_device_group(dev);
3561 	else
3562 		group = generic_device_group(dev);
3563 
3564 	return group;
3565 }
3566 
arm_smmu_of_xlate(struct device * dev,const struct of_phandle_args * args)3567 static int arm_smmu_of_xlate(struct device *dev,
3568 			     const struct of_phandle_args *args)
3569 {
3570 	return iommu_fwspec_add_ids(dev, args->args, 1);
3571 }
3572 
arm_smmu_get_resv_regions(struct device * dev,struct list_head * head)3573 static void arm_smmu_get_resv_regions(struct device *dev,
3574 				      struct list_head *head)
3575 {
3576 	struct iommu_resv_region *region;
3577 	int prot = IOMMU_WRITE | IOMMU_NOEXEC | IOMMU_MMIO;
3578 
3579 	region = iommu_alloc_resv_region(MSI_IOVA_BASE, MSI_IOVA_LENGTH,
3580 					 prot, IOMMU_RESV_SW_MSI, GFP_KERNEL);
3581 	if (!region)
3582 		return;
3583 
3584 	list_add_tail(&region->list, head);
3585 
3586 	iommu_dma_get_resv_regions(dev, head);
3587 }
3588 
arm_smmu_dev_enable_feature(struct device * dev,enum iommu_dev_features feat)3589 static int arm_smmu_dev_enable_feature(struct device *dev,
3590 				       enum iommu_dev_features feat)
3591 {
3592 	struct arm_smmu_master *master = dev_iommu_priv_get(dev);
3593 
3594 	if (!master)
3595 		return -ENODEV;
3596 
3597 	switch (feat) {
3598 	case IOMMU_DEV_FEAT_IOPF:
3599 		if (!arm_smmu_master_iopf_supported(master))
3600 			return -EINVAL;
3601 		if (master->iopf_enabled)
3602 			return -EBUSY;
3603 		master->iopf_enabled = true;
3604 		return 0;
3605 	case IOMMU_DEV_FEAT_SVA:
3606 		if (!arm_smmu_master_sva_supported(master))
3607 			return -EINVAL;
3608 		if (arm_smmu_master_sva_enabled(master))
3609 			return -EBUSY;
3610 		return arm_smmu_master_enable_sva(master);
3611 	default:
3612 		return -EINVAL;
3613 	}
3614 }
3615 
arm_smmu_dev_disable_feature(struct device * dev,enum iommu_dev_features feat)3616 static int arm_smmu_dev_disable_feature(struct device *dev,
3617 					enum iommu_dev_features feat)
3618 {
3619 	struct arm_smmu_master *master = dev_iommu_priv_get(dev);
3620 
3621 	if (!master)
3622 		return -EINVAL;
3623 
3624 	switch (feat) {
3625 	case IOMMU_DEV_FEAT_IOPF:
3626 		if (!master->iopf_enabled)
3627 			return -EINVAL;
3628 		if (master->sva_enabled)
3629 			return -EBUSY;
3630 		master->iopf_enabled = false;
3631 		return 0;
3632 	case IOMMU_DEV_FEAT_SVA:
3633 		if (!arm_smmu_master_sva_enabled(master))
3634 			return -EINVAL;
3635 		return arm_smmu_master_disable_sva(master);
3636 	default:
3637 		return -EINVAL;
3638 	}
3639 }
3640 
3641 /*
3642  * HiSilicon PCIe tune and trace device can be used to trace TLP headers on the
3643  * PCIe link and save the data to memory by DMA. The hardware is restricted to
3644  * use identity mapping only.
3645  */
3646 #define IS_HISI_PTT_DEVICE(pdev)	((pdev)->vendor == PCI_VENDOR_ID_HUAWEI && \
3647 					 (pdev)->device == 0xa12e)
3648 
arm_smmu_def_domain_type(struct device * dev)3649 static int arm_smmu_def_domain_type(struct device *dev)
3650 {
3651 	if (dev_is_pci(dev)) {
3652 		struct pci_dev *pdev = to_pci_dev(dev);
3653 
3654 		if (IS_HISI_PTT_DEVICE(pdev))
3655 			return IOMMU_DOMAIN_IDENTITY;
3656 	}
3657 
3658 	return 0;
3659 }
3660 
3661 static struct iommu_ops arm_smmu_ops = {
3662 	.identity_domain	= &arm_smmu_identity_domain,
3663 	.blocked_domain		= &arm_smmu_blocked_domain,
3664 	.capable		= arm_smmu_capable,
3665 	.hw_info		= arm_smmu_hw_info,
3666 	.domain_alloc_sva       = arm_smmu_sva_domain_alloc,
3667 	.domain_alloc_paging_flags = arm_smmu_domain_alloc_paging_flags,
3668 	.probe_device		= arm_smmu_probe_device,
3669 	.release_device		= arm_smmu_release_device,
3670 	.device_group		= arm_smmu_device_group,
3671 	.of_xlate		= arm_smmu_of_xlate,
3672 	.get_resv_regions	= arm_smmu_get_resv_regions,
3673 	.dev_enable_feat	= arm_smmu_dev_enable_feature,
3674 	.dev_disable_feat	= arm_smmu_dev_disable_feature,
3675 	.page_response		= arm_smmu_page_response,
3676 	.def_domain_type	= arm_smmu_def_domain_type,
3677 	.viommu_alloc		= arm_vsmmu_alloc,
3678 	.user_pasid_table	= 1,
3679 	.pgsize_bitmap		= -1UL, /* Restricted during device attach */
3680 	.owner			= THIS_MODULE,
3681 	.default_domain_ops = &(const struct iommu_domain_ops) {
3682 		.attach_dev		= arm_smmu_attach_dev,
3683 		.enforce_cache_coherency = arm_smmu_enforce_cache_coherency,
3684 		.set_dev_pasid		= arm_smmu_s1_set_dev_pasid,
3685 		.map_pages		= arm_smmu_map_pages,
3686 		.unmap_pages		= arm_smmu_unmap_pages,
3687 		.flush_iotlb_all	= arm_smmu_flush_iotlb_all,
3688 		.iotlb_sync		= arm_smmu_iotlb_sync,
3689 		.iova_to_phys		= arm_smmu_iova_to_phys,
3690 		.free			= arm_smmu_domain_free_paging,
3691 	}
3692 };
3693 
3694 static struct iommu_dirty_ops arm_smmu_dirty_ops = {
3695 	.read_and_clear_dirty	= arm_smmu_read_and_clear_dirty,
3696 	.set_dirty_tracking     = arm_smmu_set_dirty_tracking,
3697 };
3698 
3699 /* Probing and initialisation functions */
arm_smmu_init_one_queue(struct arm_smmu_device * smmu,struct arm_smmu_queue * q,void __iomem * page,unsigned long prod_off,unsigned long cons_off,size_t dwords,const char * name)3700 int arm_smmu_init_one_queue(struct arm_smmu_device *smmu,
3701 			    struct arm_smmu_queue *q, void __iomem *page,
3702 			    unsigned long prod_off, unsigned long cons_off,
3703 			    size_t dwords, const char *name)
3704 {
3705 	size_t qsz;
3706 
3707 	do {
3708 		qsz = ((1 << q->llq.max_n_shift) * dwords) << 3;
3709 		q->base = dmam_alloc_coherent(smmu->dev, qsz, &q->base_dma,
3710 					      GFP_KERNEL);
3711 		if (q->base || qsz < PAGE_SIZE)
3712 			break;
3713 
3714 		q->llq.max_n_shift--;
3715 	} while (1);
3716 
3717 	if (!q->base) {
3718 		dev_err(smmu->dev,
3719 			"failed to allocate queue (0x%zx bytes) for %s\n",
3720 			qsz, name);
3721 		return -ENOMEM;
3722 	}
3723 
3724 	if (!WARN_ON(q->base_dma & (qsz - 1))) {
3725 		dev_info(smmu->dev, "allocated %u entries for %s\n",
3726 			 1 << q->llq.max_n_shift, name);
3727 	}
3728 
3729 	q->prod_reg	= page + prod_off;
3730 	q->cons_reg	= page + cons_off;
3731 	q->ent_dwords	= dwords;
3732 
3733 	q->q_base  = Q_BASE_RWA;
3734 	q->q_base |= q->base_dma & Q_BASE_ADDR_MASK;
3735 	q->q_base |= FIELD_PREP(Q_BASE_LOG2SIZE, q->llq.max_n_shift);
3736 
3737 	q->llq.prod = q->llq.cons = 0;
3738 	return 0;
3739 }
3740 
arm_smmu_cmdq_init(struct arm_smmu_device * smmu,struct arm_smmu_cmdq * cmdq)3741 int arm_smmu_cmdq_init(struct arm_smmu_device *smmu,
3742 		       struct arm_smmu_cmdq *cmdq)
3743 {
3744 	unsigned int nents = 1 << cmdq->q.llq.max_n_shift;
3745 
3746 	atomic_set(&cmdq->owner_prod, 0);
3747 	atomic_set(&cmdq->lock, 0);
3748 
3749 	cmdq->valid_map = (atomic_long_t *)devm_bitmap_zalloc(smmu->dev, nents,
3750 							      GFP_KERNEL);
3751 	if (!cmdq->valid_map)
3752 		return -ENOMEM;
3753 
3754 	return 0;
3755 }
3756 
arm_smmu_init_queues(struct arm_smmu_device * smmu)3757 static int arm_smmu_init_queues(struct arm_smmu_device *smmu)
3758 {
3759 	int ret;
3760 
3761 	/* cmdq */
3762 	ret = arm_smmu_init_one_queue(smmu, &smmu->cmdq.q, smmu->base,
3763 				      ARM_SMMU_CMDQ_PROD, ARM_SMMU_CMDQ_CONS,
3764 				      CMDQ_ENT_DWORDS, "cmdq");
3765 	if (ret)
3766 		return ret;
3767 
3768 	ret = arm_smmu_cmdq_init(smmu, &smmu->cmdq);
3769 	if (ret)
3770 		return ret;
3771 
3772 	/* evtq */
3773 	ret = arm_smmu_init_one_queue(smmu, &smmu->evtq.q, smmu->page1,
3774 				      ARM_SMMU_EVTQ_PROD, ARM_SMMU_EVTQ_CONS,
3775 				      EVTQ_ENT_DWORDS, "evtq");
3776 	if (ret)
3777 		return ret;
3778 
3779 	if ((smmu->features & ARM_SMMU_FEAT_SVA) &&
3780 	    (smmu->features & ARM_SMMU_FEAT_STALLS)) {
3781 		smmu->evtq.iopf = iopf_queue_alloc(dev_name(smmu->dev));
3782 		if (!smmu->evtq.iopf)
3783 			return -ENOMEM;
3784 	}
3785 
3786 	/* priq */
3787 	if (!(smmu->features & ARM_SMMU_FEAT_PRI))
3788 		return 0;
3789 
3790 	return arm_smmu_init_one_queue(smmu, &smmu->priq.q, smmu->page1,
3791 				       ARM_SMMU_PRIQ_PROD, ARM_SMMU_PRIQ_CONS,
3792 				       PRIQ_ENT_DWORDS, "priq");
3793 }
3794 
arm_smmu_init_strtab_2lvl(struct arm_smmu_device * smmu)3795 static int arm_smmu_init_strtab_2lvl(struct arm_smmu_device *smmu)
3796 {
3797 	u32 l1size;
3798 	struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg;
3799 	unsigned int last_sid_idx =
3800 		arm_smmu_strtab_l1_idx((1ULL << smmu->sid_bits) - 1);
3801 
3802 	/* Calculate the L1 size, capped to the SIDSIZE. */
3803 	cfg->l2.num_l1_ents = min(last_sid_idx + 1, STRTAB_MAX_L1_ENTRIES);
3804 	if (cfg->l2.num_l1_ents <= last_sid_idx)
3805 		dev_warn(smmu->dev,
3806 			 "2-level strtab only covers %u/%u bits of SID\n",
3807 			 ilog2(cfg->l2.num_l1_ents * STRTAB_NUM_L2_STES),
3808 			 smmu->sid_bits);
3809 
3810 	l1size = cfg->l2.num_l1_ents * sizeof(struct arm_smmu_strtab_l1);
3811 	cfg->l2.l1tab = dmam_alloc_coherent(smmu->dev, l1size, &cfg->l2.l1_dma,
3812 					    GFP_KERNEL);
3813 	if (!cfg->l2.l1tab) {
3814 		dev_err(smmu->dev,
3815 			"failed to allocate l1 stream table (%u bytes)\n",
3816 			l1size);
3817 		return -ENOMEM;
3818 	}
3819 
3820 	cfg->l2.l2ptrs = devm_kcalloc(smmu->dev, cfg->l2.num_l1_ents,
3821 				      sizeof(*cfg->l2.l2ptrs), GFP_KERNEL);
3822 	if (!cfg->l2.l2ptrs)
3823 		return -ENOMEM;
3824 
3825 	return 0;
3826 }
3827 
arm_smmu_init_strtab_linear(struct arm_smmu_device * smmu)3828 static int arm_smmu_init_strtab_linear(struct arm_smmu_device *smmu)
3829 {
3830 	u32 size;
3831 	struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg;
3832 
3833 	size = (1 << smmu->sid_bits) * sizeof(struct arm_smmu_ste);
3834 	cfg->linear.table = dmam_alloc_coherent(smmu->dev, size,
3835 						&cfg->linear.ste_dma,
3836 						GFP_KERNEL);
3837 	if (!cfg->linear.table) {
3838 		dev_err(smmu->dev,
3839 			"failed to allocate linear stream table (%u bytes)\n",
3840 			size);
3841 		return -ENOMEM;
3842 	}
3843 	cfg->linear.num_ents = 1 << smmu->sid_bits;
3844 
3845 	arm_smmu_init_initial_stes(cfg->linear.table, cfg->linear.num_ents);
3846 	return 0;
3847 }
3848 
arm_smmu_init_strtab(struct arm_smmu_device * smmu)3849 static int arm_smmu_init_strtab(struct arm_smmu_device *smmu)
3850 {
3851 	int ret;
3852 
3853 	if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB)
3854 		ret = arm_smmu_init_strtab_2lvl(smmu);
3855 	else
3856 		ret = arm_smmu_init_strtab_linear(smmu);
3857 	if (ret)
3858 		return ret;
3859 
3860 	ida_init(&smmu->vmid_map);
3861 
3862 	return 0;
3863 }
3864 
arm_smmu_init_structures(struct arm_smmu_device * smmu)3865 static int arm_smmu_init_structures(struct arm_smmu_device *smmu)
3866 {
3867 	int ret;
3868 
3869 	mutex_init(&smmu->streams_mutex);
3870 	smmu->streams = RB_ROOT;
3871 
3872 	ret = arm_smmu_init_queues(smmu);
3873 	if (ret)
3874 		return ret;
3875 
3876 	ret = arm_smmu_init_strtab(smmu);
3877 	if (ret)
3878 		return ret;
3879 
3880 	if (smmu->impl_ops && smmu->impl_ops->init_structures)
3881 		return smmu->impl_ops->init_structures(smmu);
3882 
3883 	return 0;
3884 }
3885 
arm_smmu_write_reg_sync(struct arm_smmu_device * smmu,u32 val,unsigned int reg_off,unsigned int ack_off)3886 static int arm_smmu_write_reg_sync(struct arm_smmu_device *smmu, u32 val,
3887 				   unsigned int reg_off, unsigned int ack_off)
3888 {
3889 	u32 reg;
3890 
3891 	writel_relaxed(val, smmu->base + reg_off);
3892 	return readl_relaxed_poll_timeout(smmu->base + ack_off, reg, reg == val,
3893 					  1, ARM_SMMU_POLL_TIMEOUT_US);
3894 }
3895 
3896 /* GBPA is "special" */
arm_smmu_update_gbpa(struct arm_smmu_device * smmu,u32 set,u32 clr)3897 static int arm_smmu_update_gbpa(struct arm_smmu_device *smmu, u32 set, u32 clr)
3898 {
3899 	int ret;
3900 	u32 reg, __iomem *gbpa = smmu->base + ARM_SMMU_GBPA;
3901 
3902 	ret = readl_relaxed_poll_timeout(gbpa, reg, !(reg & GBPA_UPDATE),
3903 					 1, ARM_SMMU_POLL_TIMEOUT_US);
3904 	if (ret)
3905 		return ret;
3906 
3907 	reg &= ~clr;
3908 	reg |= set;
3909 	writel_relaxed(reg | GBPA_UPDATE, gbpa);
3910 	ret = readl_relaxed_poll_timeout(gbpa, reg, !(reg & GBPA_UPDATE),
3911 					 1, ARM_SMMU_POLL_TIMEOUT_US);
3912 
3913 	if (ret)
3914 		dev_err(smmu->dev, "GBPA not responding to update\n");
3915 	return ret;
3916 }
3917 
arm_smmu_free_msis(void * data)3918 static void arm_smmu_free_msis(void *data)
3919 {
3920 	struct device *dev = data;
3921 
3922 	platform_device_msi_free_irqs_all(dev);
3923 }
3924 
arm_smmu_write_msi_msg(struct msi_desc * desc,struct msi_msg * msg)3925 static void arm_smmu_write_msi_msg(struct msi_desc *desc, struct msi_msg *msg)
3926 {
3927 	phys_addr_t doorbell;
3928 	struct device *dev = msi_desc_to_dev(desc);
3929 	struct arm_smmu_device *smmu = dev_get_drvdata(dev);
3930 	phys_addr_t *cfg = arm_smmu_msi_cfg[desc->msi_index];
3931 
3932 	doorbell = (((u64)msg->address_hi) << 32) | msg->address_lo;
3933 	doorbell &= MSI_CFG0_ADDR_MASK;
3934 
3935 	writeq_relaxed(doorbell, smmu->base + cfg[0]);
3936 	writel_relaxed(msg->data, smmu->base + cfg[1]);
3937 	writel_relaxed(ARM_SMMU_MEMATTR_DEVICE_nGnRE, smmu->base + cfg[2]);
3938 }
3939 
arm_smmu_setup_msis(struct arm_smmu_device * smmu)3940 static void arm_smmu_setup_msis(struct arm_smmu_device *smmu)
3941 {
3942 	int ret, nvec = ARM_SMMU_MAX_MSIS;
3943 	struct device *dev = smmu->dev;
3944 
3945 	/* Clear the MSI address regs */
3946 	writeq_relaxed(0, smmu->base + ARM_SMMU_GERROR_IRQ_CFG0);
3947 	writeq_relaxed(0, smmu->base + ARM_SMMU_EVTQ_IRQ_CFG0);
3948 
3949 	if (smmu->features & ARM_SMMU_FEAT_PRI)
3950 		writeq_relaxed(0, smmu->base + ARM_SMMU_PRIQ_IRQ_CFG0);
3951 	else
3952 		nvec--;
3953 
3954 	if (!(smmu->features & ARM_SMMU_FEAT_MSI))
3955 		return;
3956 
3957 	if (!dev->msi.domain) {
3958 		dev_info(smmu->dev, "msi_domain absent - falling back to wired irqs\n");
3959 		return;
3960 	}
3961 
3962 	/* Allocate MSIs for evtq, gerror and priq. Ignore cmdq */
3963 	ret = platform_device_msi_init_and_alloc_irqs(dev, nvec, arm_smmu_write_msi_msg);
3964 	if (ret) {
3965 		dev_warn(dev, "failed to allocate MSIs - falling back to wired irqs\n");
3966 		return;
3967 	}
3968 
3969 	smmu->evtq.q.irq = msi_get_virq(dev, EVTQ_MSI_INDEX);
3970 	smmu->gerr_irq = msi_get_virq(dev, GERROR_MSI_INDEX);
3971 	smmu->priq.q.irq = msi_get_virq(dev, PRIQ_MSI_INDEX);
3972 
3973 	/* Add callback to free MSIs on teardown */
3974 	devm_add_action_or_reset(dev, arm_smmu_free_msis, dev);
3975 }
3976 
arm_smmu_setup_unique_irqs(struct arm_smmu_device * smmu)3977 static void arm_smmu_setup_unique_irqs(struct arm_smmu_device *smmu)
3978 {
3979 	int irq, ret;
3980 
3981 	arm_smmu_setup_msis(smmu);
3982 
3983 	/* Request interrupt lines */
3984 	irq = smmu->evtq.q.irq;
3985 	if (irq) {
3986 		ret = devm_request_threaded_irq(smmu->dev, irq, NULL,
3987 						arm_smmu_evtq_thread,
3988 						IRQF_ONESHOT,
3989 						"arm-smmu-v3-evtq", smmu);
3990 		if (ret < 0)
3991 			dev_warn(smmu->dev, "failed to enable evtq irq\n");
3992 	} else {
3993 		dev_warn(smmu->dev, "no evtq irq - events will not be reported!\n");
3994 	}
3995 
3996 	irq = smmu->gerr_irq;
3997 	if (irq) {
3998 		ret = devm_request_irq(smmu->dev, irq, arm_smmu_gerror_handler,
3999 				       0, "arm-smmu-v3-gerror", smmu);
4000 		if (ret < 0)
4001 			dev_warn(smmu->dev, "failed to enable gerror irq\n");
4002 	} else {
4003 		dev_warn(smmu->dev, "no gerr irq - errors will not be reported!\n");
4004 	}
4005 
4006 	if (smmu->features & ARM_SMMU_FEAT_PRI) {
4007 		irq = smmu->priq.q.irq;
4008 		if (irq) {
4009 			ret = devm_request_threaded_irq(smmu->dev, irq, NULL,
4010 							arm_smmu_priq_thread,
4011 							IRQF_ONESHOT,
4012 							"arm-smmu-v3-priq",
4013 							smmu);
4014 			if (ret < 0)
4015 				dev_warn(smmu->dev,
4016 					 "failed to enable priq irq\n");
4017 		} else {
4018 			dev_warn(smmu->dev, "no priq irq - PRI will be broken\n");
4019 		}
4020 	}
4021 }
4022 
arm_smmu_setup_irqs(struct arm_smmu_device * smmu)4023 static int arm_smmu_setup_irqs(struct arm_smmu_device *smmu)
4024 {
4025 	int ret, irq;
4026 	u32 irqen_flags = IRQ_CTRL_EVTQ_IRQEN | IRQ_CTRL_GERROR_IRQEN;
4027 
4028 	/* Disable IRQs first */
4029 	ret = arm_smmu_write_reg_sync(smmu, 0, ARM_SMMU_IRQ_CTRL,
4030 				      ARM_SMMU_IRQ_CTRLACK);
4031 	if (ret) {
4032 		dev_err(smmu->dev, "failed to disable irqs\n");
4033 		return ret;
4034 	}
4035 
4036 	irq = smmu->combined_irq;
4037 	if (irq) {
4038 		/*
4039 		 * Cavium ThunderX2 implementation doesn't support unique irq
4040 		 * lines. Use a single irq line for all the SMMUv3 interrupts.
4041 		 */
4042 		ret = devm_request_threaded_irq(smmu->dev, irq,
4043 					arm_smmu_combined_irq_handler,
4044 					arm_smmu_combined_irq_thread,
4045 					IRQF_ONESHOT,
4046 					"arm-smmu-v3-combined-irq", smmu);
4047 		if (ret < 0)
4048 			dev_warn(smmu->dev, "failed to enable combined irq\n");
4049 	} else
4050 		arm_smmu_setup_unique_irqs(smmu);
4051 
4052 	if (smmu->features & ARM_SMMU_FEAT_PRI)
4053 		irqen_flags |= IRQ_CTRL_PRIQ_IRQEN;
4054 
4055 	/* Enable interrupt generation on the SMMU */
4056 	ret = arm_smmu_write_reg_sync(smmu, irqen_flags,
4057 				      ARM_SMMU_IRQ_CTRL, ARM_SMMU_IRQ_CTRLACK);
4058 	if (ret)
4059 		dev_warn(smmu->dev, "failed to enable irqs\n");
4060 
4061 	return 0;
4062 }
4063 
arm_smmu_device_disable(struct arm_smmu_device * smmu)4064 static int arm_smmu_device_disable(struct arm_smmu_device *smmu)
4065 {
4066 	int ret;
4067 
4068 	ret = arm_smmu_write_reg_sync(smmu, 0, ARM_SMMU_CR0, ARM_SMMU_CR0ACK);
4069 	if (ret)
4070 		dev_err(smmu->dev, "failed to clear cr0\n");
4071 
4072 	return ret;
4073 }
4074 
arm_smmu_write_strtab(struct arm_smmu_device * smmu)4075 static void arm_smmu_write_strtab(struct arm_smmu_device *smmu)
4076 {
4077 	struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg;
4078 	dma_addr_t dma;
4079 	u32 reg;
4080 
4081 	if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB) {
4082 		reg = FIELD_PREP(STRTAB_BASE_CFG_FMT,
4083 				 STRTAB_BASE_CFG_FMT_2LVL) |
4084 		      FIELD_PREP(STRTAB_BASE_CFG_LOG2SIZE,
4085 				 ilog2(cfg->l2.num_l1_ents) + STRTAB_SPLIT) |
4086 		      FIELD_PREP(STRTAB_BASE_CFG_SPLIT, STRTAB_SPLIT);
4087 		dma = cfg->l2.l1_dma;
4088 	} else {
4089 		reg = FIELD_PREP(STRTAB_BASE_CFG_FMT,
4090 				 STRTAB_BASE_CFG_FMT_LINEAR) |
4091 		      FIELD_PREP(STRTAB_BASE_CFG_LOG2SIZE, smmu->sid_bits);
4092 		dma = cfg->linear.ste_dma;
4093 	}
4094 	writeq_relaxed((dma & STRTAB_BASE_ADDR_MASK) | STRTAB_BASE_RA,
4095 		       smmu->base + ARM_SMMU_STRTAB_BASE);
4096 	writel_relaxed(reg, smmu->base + ARM_SMMU_STRTAB_BASE_CFG);
4097 }
4098 
arm_smmu_device_reset(struct arm_smmu_device * smmu)4099 static int arm_smmu_device_reset(struct arm_smmu_device *smmu)
4100 {
4101 	int ret;
4102 	u32 reg, enables;
4103 	struct arm_smmu_cmdq_ent cmd;
4104 
4105 	/* Clear CR0 and sync (disables SMMU and queue processing) */
4106 	reg = readl_relaxed(smmu->base + ARM_SMMU_CR0);
4107 	if (reg & CR0_SMMUEN) {
4108 		dev_warn(smmu->dev, "SMMU currently enabled! Resetting...\n");
4109 		arm_smmu_update_gbpa(smmu, GBPA_ABORT, 0);
4110 	}
4111 
4112 	ret = arm_smmu_device_disable(smmu);
4113 	if (ret)
4114 		return ret;
4115 
4116 	/* CR1 (table and queue memory attributes) */
4117 	reg = FIELD_PREP(CR1_TABLE_SH, ARM_SMMU_SH_ISH) |
4118 	      FIELD_PREP(CR1_TABLE_OC, CR1_CACHE_WB) |
4119 	      FIELD_PREP(CR1_TABLE_IC, CR1_CACHE_WB) |
4120 	      FIELD_PREP(CR1_QUEUE_SH, ARM_SMMU_SH_ISH) |
4121 	      FIELD_PREP(CR1_QUEUE_OC, CR1_CACHE_WB) |
4122 	      FIELD_PREP(CR1_QUEUE_IC, CR1_CACHE_WB);
4123 	writel_relaxed(reg, smmu->base + ARM_SMMU_CR1);
4124 
4125 	/* CR2 (random crap) */
4126 	reg = CR2_PTM | CR2_RECINVSID;
4127 
4128 	if (smmu->features & ARM_SMMU_FEAT_E2H)
4129 		reg |= CR2_E2H;
4130 
4131 	writel_relaxed(reg, smmu->base + ARM_SMMU_CR2);
4132 
4133 	/* Stream table */
4134 	arm_smmu_write_strtab(smmu);
4135 
4136 	/* Command queue */
4137 	writeq_relaxed(smmu->cmdq.q.q_base, smmu->base + ARM_SMMU_CMDQ_BASE);
4138 	writel_relaxed(smmu->cmdq.q.llq.prod, smmu->base + ARM_SMMU_CMDQ_PROD);
4139 	writel_relaxed(smmu->cmdq.q.llq.cons, smmu->base + ARM_SMMU_CMDQ_CONS);
4140 
4141 	enables = CR0_CMDQEN;
4142 	ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0,
4143 				      ARM_SMMU_CR0ACK);
4144 	if (ret) {
4145 		dev_err(smmu->dev, "failed to enable command queue\n");
4146 		return ret;
4147 	}
4148 
4149 	/* Invalidate any cached configuration */
4150 	cmd.opcode = CMDQ_OP_CFGI_ALL;
4151 	arm_smmu_cmdq_issue_cmd_with_sync(smmu, &cmd);
4152 
4153 	/* Invalidate any stale TLB entries */
4154 	if (smmu->features & ARM_SMMU_FEAT_HYP) {
4155 		cmd.opcode = CMDQ_OP_TLBI_EL2_ALL;
4156 		arm_smmu_cmdq_issue_cmd_with_sync(smmu, &cmd);
4157 	}
4158 
4159 	cmd.opcode = CMDQ_OP_TLBI_NSNH_ALL;
4160 	arm_smmu_cmdq_issue_cmd_with_sync(smmu, &cmd);
4161 
4162 	/* Event queue */
4163 	writeq_relaxed(smmu->evtq.q.q_base, smmu->base + ARM_SMMU_EVTQ_BASE);
4164 	writel_relaxed(smmu->evtq.q.llq.prod, smmu->page1 + ARM_SMMU_EVTQ_PROD);
4165 	writel_relaxed(smmu->evtq.q.llq.cons, smmu->page1 + ARM_SMMU_EVTQ_CONS);
4166 
4167 	enables |= CR0_EVTQEN;
4168 	ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0,
4169 				      ARM_SMMU_CR0ACK);
4170 	if (ret) {
4171 		dev_err(smmu->dev, "failed to enable event queue\n");
4172 		return ret;
4173 	}
4174 
4175 	/* PRI queue */
4176 	if (smmu->features & ARM_SMMU_FEAT_PRI) {
4177 		writeq_relaxed(smmu->priq.q.q_base,
4178 			       smmu->base + ARM_SMMU_PRIQ_BASE);
4179 		writel_relaxed(smmu->priq.q.llq.prod,
4180 			       smmu->page1 + ARM_SMMU_PRIQ_PROD);
4181 		writel_relaxed(smmu->priq.q.llq.cons,
4182 			       smmu->page1 + ARM_SMMU_PRIQ_CONS);
4183 
4184 		enables |= CR0_PRIQEN;
4185 		ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0,
4186 					      ARM_SMMU_CR0ACK);
4187 		if (ret) {
4188 			dev_err(smmu->dev, "failed to enable PRI queue\n");
4189 			return ret;
4190 		}
4191 	}
4192 
4193 	if (smmu->features & ARM_SMMU_FEAT_ATS) {
4194 		enables |= CR0_ATSCHK;
4195 		ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0,
4196 					      ARM_SMMU_CR0ACK);
4197 		if (ret) {
4198 			dev_err(smmu->dev, "failed to enable ATS check\n");
4199 			return ret;
4200 		}
4201 	}
4202 
4203 	ret = arm_smmu_setup_irqs(smmu);
4204 	if (ret) {
4205 		dev_err(smmu->dev, "failed to setup irqs\n");
4206 		return ret;
4207 	}
4208 
4209 	if (is_kdump_kernel())
4210 		enables &= ~(CR0_EVTQEN | CR0_PRIQEN);
4211 
4212 	/* Enable the SMMU interface */
4213 	enables |= CR0_SMMUEN;
4214 	ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0,
4215 				      ARM_SMMU_CR0ACK);
4216 	if (ret) {
4217 		dev_err(smmu->dev, "failed to enable SMMU interface\n");
4218 		return ret;
4219 	}
4220 
4221 	if (smmu->impl_ops && smmu->impl_ops->device_reset) {
4222 		ret = smmu->impl_ops->device_reset(smmu);
4223 		if (ret) {
4224 			dev_err(smmu->dev, "failed to reset impl\n");
4225 			return ret;
4226 		}
4227 	}
4228 
4229 	return 0;
4230 }
4231 
4232 #define IIDR_IMPLEMENTER_ARM		0x43b
4233 #define IIDR_PRODUCTID_ARM_MMU_600	0x483
4234 #define IIDR_PRODUCTID_ARM_MMU_700	0x487
4235 
arm_smmu_device_iidr_probe(struct arm_smmu_device * smmu)4236 static void arm_smmu_device_iidr_probe(struct arm_smmu_device *smmu)
4237 {
4238 	u32 reg;
4239 	unsigned int implementer, productid, variant, revision;
4240 
4241 	reg = readl_relaxed(smmu->base + ARM_SMMU_IIDR);
4242 	implementer = FIELD_GET(IIDR_IMPLEMENTER, reg);
4243 	productid = FIELD_GET(IIDR_PRODUCTID, reg);
4244 	variant = FIELD_GET(IIDR_VARIANT, reg);
4245 	revision = FIELD_GET(IIDR_REVISION, reg);
4246 
4247 	switch (implementer) {
4248 	case IIDR_IMPLEMENTER_ARM:
4249 		switch (productid) {
4250 		case IIDR_PRODUCTID_ARM_MMU_600:
4251 			/* Arm erratum 1076982 */
4252 			if (variant == 0 && revision <= 2)
4253 				smmu->features &= ~ARM_SMMU_FEAT_SEV;
4254 			/* Arm erratum 1209401 */
4255 			if (variant < 2)
4256 				smmu->features &= ~ARM_SMMU_FEAT_NESTING;
4257 			break;
4258 		case IIDR_PRODUCTID_ARM_MMU_700:
4259 			/* Arm erratum 2812531 */
4260 			smmu->features &= ~ARM_SMMU_FEAT_BTM;
4261 			smmu->options |= ARM_SMMU_OPT_CMDQ_FORCE_SYNC;
4262 			/* Arm errata 2268618, 2812531 */
4263 			smmu->features &= ~ARM_SMMU_FEAT_NESTING;
4264 			break;
4265 		}
4266 		break;
4267 	}
4268 }
4269 
arm_smmu_get_httu(struct arm_smmu_device * smmu,u32 reg)4270 static void arm_smmu_get_httu(struct arm_smmu_device *smmu, u32 reg)
4271 {
4272 	u32 fw_features = smmu->features & (ARM_SMMU_FEAT_HA | ARM_SMMU_FEAT_HD);
4273 	u32 hw_features = 0;
4274 
4275 	switch (FIELD_GET(IDR0_HTTU, reg)) {
4276 	case IDR0_HTTU_ACCESS_DIRTY:
4277 		hw_features |= ARM_SMMU_FEAT_HD;
4278 		fallthrough;
4279 	case IDR0_HTTU_ACCESS:
4280 		hw_features |= ARM_SMMU_FEAT_HA;
4281 	}
4282 
4283 	if (smmu->dev->of_node)
4284 		smmu->features |= hw_features;
4285 	else if (hw_features != fw_features)
4286 		/* ACPI IORT sets the HTTU bits */
4287 		dev_warn(smmu->dev,
4288 			 "IDR0.HTTU features(0x%x) overridden by FW configuration (0x%x)\n",
4289 			  hw_features, fw_features);
4290 }
4291 
arm_smmu_device_hw_probe(struct arm_smmu_device * smmu)4292 static int arm_smmu_device_hw_probe(struct arm_smmu_device *smmu)
4293 {
4294 	u32 reg;
4295 	bool coherent = smmu->features & ARM_SMMU_FEAT_COHERENCY;
4296 
4297 	/* IDR0 */
4298 	reg = readl_relaxed(smmu->base + ARM_SMMU_IDR0);
4299 
4300 	/* 2-level structures */
4301 	if (FIELD_GET(IDR0_ST_LVL, reg) == IDR0_ST_LVL_2LVL)
4302 		smmu->features |= ARM_SMMU_FEAT_2_LVL_STRTAB;
4303 
4304 	if (reg & IDR0_CD2L)
4305 		smmu->features |= ARM_SMMU_FEAT_2_LVL_CDTAB;
4306 
4307 	/*
4308 	 * Translation table endianness.
4309 	 * We currently require the same endianness as the CPU, but this
4310 	 * could be changed later by adding a new IO_PGTABLE_QUIRK.
4311 	 */
4312 	switch (FIELD_GET(IDR0_TTENDIAN, reg)) {
4313 	case IDR0_TTENDIAN_MIXED:
4314 		smmu->features |= ARM_SMMU_FEAT_TT_LE | ARM_SMMU_FEAT_TT_BE;
4315 		break;
4316 #ifdef __BIG_ENDIAN
4317 	case IDR0_TTENDIAN_BE:
4318 		smmu->features |= ARM_SMMU_FEAT_TT_BE;
4319 		break;
4320 #else
4321 	case IDR0_TTENDIAN_LE:
4322 		smmu->features |= ARM_SMMU_FEAT_TT_LE;
4323 		break;
4324 #endif
4325 	default:
4326 		dev_err(smmu->dev, "unknown/unsupported TT endianness!\n");
4327 		return -ENXIO;
4328 	}
4329 
4330 	/* Boolean feature flags */
4331 	if (IS_ENABLED(CONFIG_PCI_PRI) && reg & IDR0_PRI)
4332 		smmu->features |= ARM_SMMU_FEAT_PRI;
4333 
4334 	if (IS_ENABLED(CONFIG_PCI_ATS) && reg & IDR0_ATS)
4335 		smmu->features |= ARM_SMMU_FEAT_ATS;
4336 
4337 	if (reg & IDR0_SEV)
4338 		smmu->features |= ARM_SMMU_FEAT_SEV;
4339 
4340 	if (reg & IDR0_MSI) {
4341 		smmu->features |= ARM_SMMU_FEAT_MSI;
4342 		if (coherent && !disable_msipolling)
4343 			smmu->options |= ARM_SMMU_OPT_MSIPOLL;
4344 	}
4345 
4346 	if (reg & IDR0_HYP) {
4347 		smmu->features |= ARM_SMMU_FEAT_HYP;
4348 		if (cpus_have_cap(ARM64_HAS_VIRT_HOST_EXTN))
4349 			smmu->features |= ARM_SMMU_FEAT_E2H;
4350 	}
4351 
4352 	arm_smmu_get_httu(smmu, reg);
4353 
4354 	/*
4355 	 * The coherency feature as set by FW is used in preference to the ID
4356 	 * register, but warn on mismatch.
4357 	 */
4358 	if (!!(reg & IDR0_COHACC) != coherent)
4359 		dev_warn(smmu->dev, "IDR0.COHACC overridden by FW configuration (%s)\n",
4360 			 str_true_false(coherent));
4361 
4362 	switch (FIELD_GET(IDR0_STALL_MODEL, reg)) {
4363 	case IDR0_STALL_MODEL_FORCE:
4364 		smmu->features |= ARM_SMMU_FEAT_STALL_FORCE;
4365 		fallthrough;
4366 	case IDR0_STALL_MODEL_STALL:
4367 		smmu->features |= ARM_SMMU_FEAT_STALLS;
4368 	}
4369 
4370 	if (reg & IDR0_S1P)
4371 		smmu->features |= ARM_SMMU_FEAT_TRANS_S1;
4372 
4373 	if (reg & IDR0_S2P)
4374 		smmu->features |= ARM_SMMU_FEAT_TRANS_S2;
4375 
4376 	if (!(reg & (IDR0_S1P | IDR0_S2P))) {
4377 		dev_err(smmu->dev, "no translation support!\n");
4378 		return -ENXIO;
4379 	}
4380 
4381 	/* We only support the AArch64 table format at present */
4382 	switch (FIELD_GET(IDR0_TTF, reg)) {
4383 	case IDR0_TTF_AARCH32_64:
4384 		smmu->ias = 40;
4385 		fallthrough;
4386 	case IDR0_TTF_AARCH64:
4387 		break;
4388 	default:
4389 		dev_err(smmu->dev, "AArch64 table format not supported!\n");
4390 		return -ENXIO;
4391 	}
4392 
4393 	/* ASID/VMID sizes */
4394 	smmu->asid_bits = reg & IDR0_ASID16 ? 16 : 8;
4395 	smmu->vmid_bits = reg & IDR0_VMID16 ? 16 : 8;
4396 
4397 	/* IDR1 */
4398 	reg = readl_relaxed(smmu->base + ARM_SMMU_IDR1);
4399 	if (reg & (IDR1_TABLES_PRESET | IDR1_QUEUES_PRESET | IDR1_REL)) {
4400 		dev_err(smmu->dev, "embedded implementation not supported\n");
4401 		return -ENXIO;
4402 	}
4403 
4404 	if (reg & IDR1_ATTR_TYPES_OVR)
4405 		smmu->features |= ARM_SMMU_FEAT_ATTR_TYPES_OVR;
4406 
4407 	/* Queue sizes, capped to ensure natural alignment */
4408 	smmu->cmdq.q.llq.max_n_shift = min_t(u32, CMDQ_MAX_SZ_SHIFT,
4409 					     FIELD_GET(IDR1_CMDQS, reg));
4410 	if (smmu->cmdq.q.llq.max_n_shift <= ilog2(CMDQ_BATCH_ENTRIES)) {
4411 		/*
4412 		 * We don't support splitting up batches, so one batch of
4413 		 * commands plus an extra sync needs to fit inside the command
4414 		 * queue. There's also no way we can handle the weird alignment
4415 		 * restrictions on the base pointer for a unit-length queue.
4416 		 */
4417 		dev_err(smmu->dev, "command queue size <= %d entries not supported\n",
4418 			CMDQ_BATCH_ENTRIES);
4419 		return -ENXIO;
4420 	}
4421 
4422 	smmu->evtq.q.llq.max_n_shift = min_t(u32, EVTQ_MAX_SZ_SHIFT,
4423 					     FIELD_GET(IDR1_EVTQS, reg));
4424 	smmu->priq.q.llq.max_n_shift = min_t(u32, PRIQ_MAX_SZ_SHIFT,
4425 					     FIELD_GET(IDR1_PRIQS, reg));
4426 
4427 	/* SID/SSID sizes */
4428 	smmu->ssid_bits = FIELD_GET(IDR1_SSIDSIZE, reg);
4429 	smmu->sid_bits = FIELD_GET(IDR1_SIDSIZE, reg);
4430 	smmu->iommu.max_pasids = 1UL << smmu->ssid_bits;
4431 
4432 	/*
4433 	 * If the SMMU supports fewer bits than would fill a single L2 stream
4434 	 * table, use a linear table instead.
4435 	 */
4436 	if (smmu->sid_bits <= STRTAB_SPLIT)
4437 		smmu->features &= ~ARM_SMMU_FEAT_2_LVL_STRTAB;
4438 
4439 	/* IDR3 */
4440 	reg = readl_relaxed(smmu->base + ARM_SMMU_IDR3);
4441 	if (FIELD_GET(IDR3_RIL, reg))
4442 		smmu->features |= ARM_SMMU_FEAT_RANGE_INV;
4443 	if (FIELD_GET(IDR3_FWB, reg))
4444 		smmu->features |= ARM_SMMU_FEAT_S2FWB;
4445 
4446 	/* IDR5 */
4447 	reg = readl_relaxed(smmu->base + ARM_SMMU_IDR5);
4448 
4449 	/* Maximum number of outstanding stalls */
4450 	smmu->evtq.max_stalls = FIELD_GET(IDR5_STALL_MAX, reg);
4451 
4452 	/* Page sizes */
4453 	if (reg & IDR5_GRAN64K)
4454 		smmu->pgsize_bitmap |= SZ_64K | SZ_512M;
4455 	if (reg & IDR5_GRAN16K)
4456 		smmu->pgsize_bitmap |= SZ_16K | SZ_32M;
4457 	if (reg & IDR5_GRAN4K)
4458 		smmu->pgsize_bitmap |= SZ_4K | SZ_2M | SZ_1G;
4459 
4460 	/* Input address size */
4461 	if (FIELD_GET(IDR5_VAX, reg) == IDR5_VAX_52_BIT)
4462 		smmu->features |= ARM_SMMU_FEAT_VAX;
4463 
4464 	/* Output address size */
4465 	switch (FIELD_GET(IDR5_OAS, reg)) {
4466 	case IDR5_OAS_32_BIT:
4467 		smmu->oas = 32;
4468 		break;
4469 	case IDR5_OAS_36_BIT:
4470 		smmu->oas = 36;
4471 		break;
4472 	case IDR5_OAS_40_BIT:
4473 		smmu->oas = 40;
4474 		break;
4475 	case IDR5_OAS_42_BIT:
4476 		smmu->oas = 42;
4477 		break;
4478 	case IDR5_OAS_44_BIT:
4479 		smmu->oas = 44;
4480 		break;
4481 	case IDR5_OAS_52_BIT:
4482 		smmu->oas = 52;
4483 		smmu->pgsize_bitmap |= 1ULL << 42; /* 4TB */
4484 		break;
4485 	default:
4486 		dev_info(smmu->dev,
4487 			"unknown output address size. Truncating to 48-bit\n");
4488 		fallthrough;
4489 	case IDR5_OAS_48_BIT:
4490 		smmu->oas = 48;
4491 	}
4492 
4493 	if (arm_smmu_ops.pgsize_bitmap == -1UL)
4494 		arm_smmu_ops.pgsize_bitmap = smmu->pgsize_bitmap;
4495 	else
4496 		arm_smmu_ops.pgsize_bitmap |= smmu->pgsize_bitmap;
4497 
4498 	/* Set the DMA mask for our table walker */
4499 	if (dma_set_mask_and_coherent(smmu->dev, DMA_BIT_MASK(smmu->oas)))
4500 		dev_warn(smmu->dev,
4501 			 "failed to set DMA mask for table walker\n");
4502 
4503 	smmu->ias = max(smmu->ias, smmu->oas);
4504 
4505 	if ((smmu->features & ARM_SMMU_FEAT_TRANS_S1) &&
4506 	    (smmu->features & ARM_SMMU_FEAT_TRANS_S2))
4507 		smmu->features |= ARM_SMMU_FEAT_NESTING;
4508 
4509 	arm_smmu_device_iidr_probe(smmu);
4510 
4511 	if (arm_smmu_sva_supported(smmu))
4512 		smmu->features |= ARM_SMMU_FEAT_SVA;
4513 
4514 	dev_info(smmu->dev, "ias %lu-bit, oas %lu-bit (features 0x%08x)\n",
4515 		 smmu->ias, smmu->oas, smmu->features);
4516 	return 0;
4517 }
4518 
4519 #ifdef CONFIG_ACPI
4520 #ifdef CONFIG_TEGRA241_CMDQV
acpi_smmu_dsdt_probe_tegra241_cmdqv(struct acpi_iort_node * node,struct arm_smmu_device * smmu)4521 static void acpi_smmu_dsdt_probe_tegra241_cmdqv(struct acpi_iort_node *node,
4522 						struct arm_smmu_device *smmu)
4523 {
4524 	const char *uid = kasprintf(GFP_KERNEL, "%u", node->identifier);
4525 	struct acpi_device *adev;
4526 
4527 	/* Look for an NVDA200C node whose _UID matches the SMMU node ID */
4528 	adev = acpi_dev_get_first_match_dev("NVDA200C", uid, -1);
4529 	if (adev) {
4530 		/* Tegra241 CMDQV driver is responsible for put_device() */
4531 		smmu->impl_dev = &adev->dev;
4532 		smmu->options |= ARM_SMMU_OPT_TEGRA241_CMDQV;
4533 		dev_info(smmu->dev, "found companion CMDQV device: %s\n",
4534 			 dev_name(smmu->impl_dev));
4535 	}
4536 	kfree(uid);
4537 }
4538 #else
acpi_smmu_dsdt_probe_tegra241_cmdqv(struct acpi_iort_node * node,struct arm_smmu_device * smmu)4539 static void acpi_smmu_dsdt_probe_tegra241_cmdqv(struct acpi_iort_node *node,
4540 						struct arm_smmu_device *smmu)
4541 {
4542 }
4543 #endif
4544 
acpi_smmu_iort_probe_model(struct acpi_iort_node * node,struct arm_smmu_device * smmu)4545 static int acpi_smmu_iort_probe_model(struct acpi_iort_node *node,
4546 				      struct arm_smmu_device *smmu)
4547 {
4548 	struct acpi_iort_smmu_v3 *iort_smmu =
4549 		(struct acpi_iort_smmu_v3 *)node->node_data;
4550 
4551 	switch (iort_smmu->model) {
4552 	case ACPI_IORT_SMMU_V3_CAVIUM_CN99XX:
4553 		smmu->options |= ARM_SMMU_OPT_PAGE0_REGS_ONLY;
4554 		break;
4555 	case ACPI_IORT_SMMU_V3_HISILICON_HI161X:
4556 		smmu->options |= ARM_SMMU_OPT_SKIP_PREFETCH;
4557 		break;
4558 	case ACPI_IORT_SMMU_V3_GENERIC:
4559 		/*
4560 		 * Tegra241 implementation stores its SMMU options and impl_dev
4561 		 * in DSDT. Thus, go through the ACPI tables unconditionally.
4562 		 */
4563 		acpi_smmu_dsdt_probe_tegra241_cmdqv(node, smmu);
4564 		break;
4565 	}
4566 
4567 	dev_notice(smmu->dev, "option mask 0x%x\n", smmu->options);
4568 	return 0;
4569 }
4570 
arm_smmu_device_acpi_probe(struct platform_device * pdev,struct arm_smmu_device * smmu)4571 static int arm_smmu_device_acpi_probe(struct platform_device *pdev,
4572 				      struct arm_smmu_device *smmu)
4573 {
4574 	struct acpi_iort_smmu_v3 *iort_smmu;
4575 	struct device *dev = smmu->dev;
4576 	struct acpi_iort_node *node;
4577 
4578 	node = *(struct acpi_iort_node **)dev_get_platdata(dev);
4579 
4580 	/* Retrieve SMMUv3 specific data */
4581 	iort_smmu = (struct acpi_iort_smmu_v3 *)node->node_data;
4582 
4583 	if (iort_smmu->flags & ACPI_IORT_SMMU_V3_COHACC_OVERRIDE)
4584 		smmu->features |= ARM_SMMU_FEAT_COHERENCY;
4585 
4586 	switch (FIELD_GET(ACPI_IORT_SMMU_V3_HTTU_OVERRIDE, iort_smmu->flags)) {
4587 	case IDR0_HTTU_ACCESS_DIRTY:
4588 		smmu->features |= ARM_SMMU_FEAT_HD;
4589 		fallthrough;
4590 	case IDR0_HTTU_ACCESS:
4591 		smmu->features |= ARM_SMMU_FEAT_HA;
4592 	}
4593 
4594 	return acpi_smmu_iort_probe_model(node, smmu);
4595 }
4596 #else
arm_smmu_device_acpi_probe(struct platform_device * pdev,struct arm_smmu_device * smmu)4597 static inline int arm_smmu_device_acpi_probe(struct platform_device *pdev,
4598 					     struct arm_smmu_device *smmu)
4599 {
4600 	return -ENODEV;
4601 }
4602 #endif
4603 
arm_smmu_device_dt_probe(struct platform_device * pdev,struct arm_smmu_device * smmu)4604 static int arm_smmu_device_dt_probe(struct platform_device *pdev,
4605 				    struct arm_smmu_device *smmu)
4606 {
4607 	struct device *dev = &pdev->dev;
4608 	u32 cells;
4609 	int ret = -EINVAL;
4610 
4611 	if (of_property_read_u32(dev->of_node, "#iommu-cells", &cells))
4612 		dev_err(dev, "missing #iommu-cells property\n");
4613 	else if (cells != 1)
4614 		dev_err(dev, "invalid #iommu-cells value (%d)\n", cells);
4615 	else
4616 		ret = 0;
4617 
4618 	parse_driver_options(smmu);
4619 
4620 	if (of_dma_is_coherent(dev->of_node))
4621 		smmu->features |= ARM_SMMU_FEAT_COHERENCY;
4622 
4623 	return ret;
4624 }
4625 
arm_smmu_resource_size(struct arm_smmu_device * smmu)4626 static unsigned long arm_smmu_resource_size(struct arm_smmu_device *smmu)
4627 {
4628 	if (smmu->options & ARM_SMMU_OPT_PAGE0_REGS_ONLY)
4629 		return SZ_64K;
4630 	else
4631 		return SZ_128K;
4632 }
4633 
arm_smmu_ioremap(struct device * dev,resource_size_t start,resource_size_t size)4634 static void __iomem *arm_smmu_ioremap(struct device *dev, resource_size_t start,
4635 				      resource_size_t size)
4636 {
4637 	struct resource res = DEFINE_RES_MEM(start, size);
4638 
4639 	return devm_ioremap_resource(dev, &res);
4640 }
4641 
arm_smmu_rmr_install_bypass_ste(struct arm_smmu_device * smmu)4642 static void arm_smmu_rmr_install_bypass_ste(struct arm_smmu_device *smmu)
4643 {
4644 	struct list_head rmr_list;
4645 	struct iommu_resv_region *e;
4646 
4647 	INIT_LIST_HEAD(&rmr_list);
4648 	iort_get_rmr_sids(dev_fwnode(smmu->dev), &rmr_list);
4649 
4650 	list_for_each_entry(e, &rmr_list, list) {
4651 		struct iommu_iort_rmr_data *rmr;
4652 		int ret, i;
4653 
4654 		rmr = container_of(e, struct iommu_iort_rmr_data, rr);
4655 		for (i = 0; i < rmr->num_sids; i++) {
4656 			ret = arm_smmu_init_sid_strtab(smmu, rmr->sids[i]);
4657 			if (ret) {
4658 				dev_err(smmu->dev, "RMR SID(0x%x) bypass failed\n",
4659 					rmr->sids[i]);
4660 				continue;
4661 			}
4662 
4663 			/*
4664 			 * STE table is not programmed to HW, see
4665 			 * arm_smmu_initial_bypass_stes()
4666 			 */
4667 			arm_smmu_make_bypass_ste(smmu,
4668 				arm_smmu_get_step_for_sid(smmu, rmr->sids[i]));
4669 		}
4670 	}
4671 
4672 	iort_put_rmr_sids(dev_fwnode(smmu->dev), &rmr_list);
4673 }
4674 
arm_smmu_impl_remove(void * data)4675 static void arm_smmu_impl_remove(void *data)
4676 {
4677 	struct arm_smmu_device *smmu = data;
4678 
4679 	if (smmu->impl_ops && smmu->impl_ops->device_remove)
4680 		smmu->impl_ops->device_remove(smmu);
4681 }
4682 
4683 /*
4684  * Probe all the compiled in implementations. Each one checks to see if it
4685  * matches this HW and if so returns a devm_krealloc'd arm_smmu_device which
4686  * replaces the callers. Otherwise the original is returned or ERR_PTR.
4687  */
arm_smmu_impl_probe(struct arm_smmu_device * smmu)4688 static struct arm_smmu_device *arm_smmu_impl_probe(struct arm_smmu_device *smmu)
4689 {
4690 	struct arm_smmu_device *new_smmu = ERR_PTR(-ENODEV);
4691 	int ret;
4692 
4693 	if (smmu->impl_dev && (smmu->options & ARM_SMMU_OPT_TEGRA241_CMDQV))
4694 		new_smmu = tegra241_cmdqv_probe(smmu);
4695 
4696 	if (new_smmu == ERR_PTR(-ENODEV))
4697 		return smmu;
4698 	if (IS_ERR(new_smmu))
4699 		return new_smmu;
4700 
4701 	ret = devm_add_action_or_reset(new_smmu->dev, arm_smmu_impl_remove,
4702 				       new_smmu);
4703 	if (ret)
4704 		return ERR_PTR(ret);
4705 	return new_smmu;
4706 }
4707 
arm_smmu_device_probe(struct platform_device * pdev)4708 static int arm_smmu_device_probe(struct platform_device *pdev)
4709 {
4710 	int irq, ret;
4711 	struct resource *res;
4712 	resource_size_t ioaddr;
4713 	struct arm_smmu_device *smmu;
4714 	struct device *dev = &pdev->dev;
4715 
4716 	smmu = devm_kzalloc(dev, sizeof(*smmu), GFP_KERNEL);
4717 	if (!smmu)
4718 		return -ENOMEM;
4719 	smmu->dev = dev;
4720 
4721 	if (dev->of_node) {
4722 		ret = arm_smmu_device_dt_probe(pdev, smmu);
4723 	} else {
4724 		ret = arm_smmu_device_acpi_probe(pdev, smmu);
4725 	}
4726 	if (ret)
4727 		return ret;
4728 
4729 	smmu = arm_smmu_impl_probe(smmu);
4730 	if (IS_ERR(smmu))
4731 		return PTR_ERR(smmu);
4732 
4733 	/* Base address */
4734 	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
4735 	if (!res)
4736 		return -EINVAL;
4737 	if (resource_size(res) < arm_smmu_resource_size(smmu)) {
4738 		dev_err(dev, "MMIO region too small (%pr)\n", res);
4739 		return -EINVAL;
4740 	}
4741 	ioaddr = res->start;
4742 
4743 	/*
4744 	 * Don't map the IMPLEMENTATION DEFINED regions, since they may contain
4745 	 * the PMCG registers which are reserved by the PMU driver.
4746 	 */
4747 	smmu->base = arm_smmu_ioremap(dev, ioaddr, ARM_SMMU_REG_SZ);
4748 	if (IS_ERR(smmu->base))
4749 		return PTR_ERR(smmu->base);
4750 
4751 	if (arm_smmu_resource_size(smmu) > SZ_64K) {
4752 		smmu->page1 = arm_smmu_ioremap(dev, ioaddr + SZ_64K,
4753 					       ARM_SMMU_REG_SZ);
4754 		if (IS_ERR(smmu->page1))
4755 			return PTR_ERR(smmu->page1);
4756 	} else {
4757 		smmu->page1 = smmu->base;
4758 	}
4759 
4760 	/* Interrupt lines */
4761 
4762 	irq = platform_get_irq_byname_optional(pdev, "combined");
4763 	if (irq > 0)
4764 		smmu->combined_irq = irq;
4765 	else {
4766 		irq = platform_get_irq_byname_optional(pdev, "eventq");
4767 		if (irq > 0)
4768 			smmu->evtq.q.irq = irq;
4769 
4770 		irq = platform_get_irq_byname_optional(pdev, "priq");
4771 		if (irq > 0)
4772 			smmu->priq.q.irq = irq;
4773 
4774 		irq = platform_get_irq_byname_optional(pdev, "gerror");
4775 		if (irq > 0)
4776 			smmu->gerr_irq = irq;
4777 	}
4778 	/* Probe the h/w */
4779 	ret = arm_smmu_device_hw_probe(smmu);
4780 	if (ret)
4781 		return ret;
4782 
4783 	/* Initialise in-memory data structures */
4784 	ret = arm_smmu_init_structures(smmu);
4785 	if (ret)
4786 		goto err_free_iopf;
4787 
4788 	/* Record our private device structure */
4789 	platform_set_drvdata(pdev, smmu);
4790 
4791 	/* Check for RMRs and install bypass STEs if any */
4792 	arm_smmu_rmr_install_bypass_ste(smmu);
4793 
4794 	/* Reset the device */
4795 	ret = arm_smmu_device_reset(smmu);
4796 	if (ret)
4797 		goto err_disable;
4798 
4799 	/* And we're up. Go go go! */
4800 	ret = iommu_device_sysfs_add(&smmu->iommu, dev, NULL,
4801 				     "smmu3.%pa", &ioaddr);
4802 	if (ret)
4803 		goto err_disable;
4804 
4805 	ret = iommu_device_register(&smmu->iommu, &arm_smmu_ops, dev);
4806 	if (ret) {
4807 		dev_err(dev, "Failed to register iommu\n");
4808 		goto err_free_sysfs;
4809 	}
4810 
4811 	return 0;
4812 
4813 err_free_sysfs:
4814 	iommu_device_sysfs_remove(&smmu->iommu);
4815 err_disable:
4816 	arm_smmu_device_disable(smmu);
4817 err_free_iopf:
4818 	iopf_queue_free(smmu->evtq.iopf);
4819 	return ret;
4820 }
4821 
arm_smmu_device_remove(struct platform_device * pdev)4822 static void arm_smmu_device_remove(struct platform_device *pdev)
4823 {
4824 	struct arm_smmu_device *smmu = platform_get_drvdata(pdev);
4825 
4826 	iommu_device_unregister(&smmu->iommu);
4827 	iommu_device_sysfs_remove(&smmu->iommu);
4828 	arm_smmu_device_disable(smmu);
4829 	iopf_queue_free(smmu->evtq.iopf);
4830 	ida_destroy(&smmu->vmid_map);
4831 }
4832 
arm_smmu_device_shutdown(struct platform_device * pdev)4833 static void arm_smmu_device_shutdown(struct platform_device *pdev)
4834 {
4835 	struct arm_smmu_device *smmu = platform_get_drvdata(pdev);
4836 
4837 	arm_smmu_device_disable(smmu);
4838 }
4839 
4840 static const struct of_device_id arm_smmu_of_match[] = {
4841 	{ .compatible = "arm,smmu-v3", },
4842 	{ },
4843 };
4844 MODULE_DEVICE_TABLE(of, arm_smmu_of_match);
4845 
arm_smmu_driver_unregister(struct platform_driver * drv)4846 static void arm_smmu_driver_unregister(struct platform_driver *drv)
4847 {
4848 	arm_smmu_sva_notifier_synchronize();
4849 	platform_driver_unregister(drv);
4850 }
4851 
4852 static struct platform_driver arm_smmu_driver = {
4853 	.driver	= {
4854 		.name			= "arm-smmu-v3",
4855 		.of_match_table		= arm_smmu_of_match,
4856 		.suppress_bind_attrs	= true,
4857 	},
4858 	.probe	= arm_smmu_device_probe,
4859 	.remove = arm_smmu_device_remove,
4860 	.shutdown = arm_smmu_device_shutdown,
4861 };
4862 module_driver(arm_smmu_driver, platform_driver_register,
4863 	      arm_smmu_driver_unregister);
4864 
4865 MODULE_DESCRIPTION("IOMMU API for ARM architected SMMUv3 implementations");
4866 MODULE_AUTHOR("Will Deacon <will@kernel.org>");
4867 MODULE_ALIAS("platform:arm-smmu-v3");
4868 MODULE_LICENSE("GPL v2");
4869