1 // SPDX-License-Identifier: MIT
2 /*
3  * Copyright © 2021-2023 Intel Corporation
4  */
5 
6 #include "xe_mmio.h"
7 
8 #include <linux/delay.h>
9 #include <linux/io-64-nonatomic-lo-hi.h>
10 #include <linux/minmax.h>
11 #include <linux/pci.h>
12 
13 #include <drm/drm_managed.h>
14 #include <drm/drm_print.h>
15 
16 #include "regs/xe_bars.h"
17 #include "regs/xe_regs.h"
18 #include "xe_device.h"
19 #include "xe_gt.h"
20 #include "xe_gt_printk.h"
21 #include "xe_gt_sriov_vf.h"
22 #include "xe_macros.h"
23 #include "xe_sriov.h"
24 #include "xe_trace.h"
25 
tiles_fini(void * arg)26 static void tiles_fini(void *arg)
27 {
28 	struct xe_device *xe = arg;
29 	struct xe_tile *tile;
30 	int id;
31 
32 	for_each_remote_tile(tile, xe, id)
33 		tile->mmio.regs = NULL;
34 }
35 
36 /*
37  * On multi-tile devices, partition the BAR space for MMIO on each tile,
38  * possibly accounting for register override on the number of tiles available.
39  * tile_mmio_size contains both the tile's 4MB register space, as well as
40  * additional space for the GTT and other (possibly unused) regions).
41  * Resulting memory layout is like below:
42  *
43  * .----------------------. <- tile_count * tile_mmio_size
44  * |         ....         |
45  * |----------------------| <- 2 * tile_mmio_size
46  * |   tile1 GTT + other  |
47  * |----------------------| <- 1 * tile_mmio_size + 4MB
48  * |   tile1->mmio.regs   |
49  * |----------------------| <- 1 * tile_mmio_size
50  * |   tile0 GTT + other  |
51  * |----------------------| <- 4MB
52  * |   tile0->mmio.regs   |
53  * '----------------------' <- 0MB
54  */
mmio_multi_tile_setup(struct xe_device * xe,size_t tile_mmio_size)55 static void mmio_multi_tile_setup(struct xe_device *xe, size_t tile_mmio_size)
56 {
57 	struct xe_tile *tile;
58 	u8 id;
59 
60 	/*
61 	 * Nothing to be done as tile 0 has already been setup earlier with the
62 	 * entire BAR mapped - see xe_mmio_probe_early()
63 	 */
64 	if (xe->info.tile_count == 1)
65 		return;
66 
67 	/* Possibly override number of tile based on configuration register */
68 	if (!xe->info.skip_mtcfg) {
69 		struct xe_mmio *mmio = xe_root_tile_mmio(xe);
70 		u8 tile_count;
71 		u32 mtcfg;
72 
73 		/*
74 		 * Although the per-tile mmio regs are not yet initialized, this
75 		 * is fine as it's going to the root tile's mmio, that's
76 		 * guaranteed to be initialized earlier in xe_mmio_probe_early()
77 		 */
78 		mtcfg = xe_mmio_read32(mmio, XEHP_MTCFG_ADDR);
79 		tile_count = REG_FIELD_GET(TILE_COUNT, mtcfg) + 1;
80 
81 		if (tile_count < xe->info.tile_count) {
82 			drm_info(&xe->drm, "tile_count: %d, reduced_tile_count %d\n",
83 				 xe->info.tile_count, tile_count);
84 			xe->info.tile_count = tile_count;
85 
86 			/*
87 			 * FIXME: Needs some work for standalone media, but
88 			 * should be impossible with multi-tile for now:
89 			 * multi-tile platform with standalone media doesn't
90 			 * exist
91 			 */
92 			xe->info.gt_count = xe->info.tile_count;
93 		}
94 	}
95 
96 	for_each_remote_tile(tile, xe, id)
97 		xe_mmio_init(&tile->mmio, tile, xe->mmio.regs + id * tile_mmio_size, SZ_4M);
98 }
99 
xe_mmio_probe_tiles(struct xe_device * xe)100 int xe_mmio_probe_tiles(struct xe_device *xe)
101 {
102 	size_t tile_mmio_size = SZ_16M;
103 
104 	mmio_multi_tile_setup(xe, tile_mmio_size);
105 
106 	return devm_add_action_or_reset(xe->drm.dev, tiles_fini, xe);
107 }
108 
mmio_fini(void * arg)109 static void mmio_fini(void *arg)
110 {
111 	struct xe_device *xe = arg;
112 	struct xe_tile *root_tile = xe_device_get_root_tile(xe);
113 
114 	pci_iounmap(to_pci_dev(xe->drm.dev), xe->mmio.regs);
115 	xe->mmio.regs = NULL;
116 	root_tile->mmio.regs = NULL;
117 }
118 
xe_mmio_probe_early(struct xe_device * xe)119 int xe_mmio_probe_early(struct xe_device *xe)
120 {
121 	struct xe_tile *root_tile = xe_device_get_root_tile(xe);
122 	struct pci_dev *pdev = to_pci_dev(xe->drm.dev);
123 
124 	/*
125 	 * Map the entire BAR.
126 	 * The first 16MB of the BAR, belong to the root tile, and include:
127 	 * registers (0-4MB), reserved space (4MB-8MB) and GGTT (8MB-16MB).
128 	 */
129 	xe->mmio.size = pci_resource_len(pdev, GTTMMADR_BAR);
130 	xe->mmio.regs = pci_iomap(pdev, GTTMMADR_BAR, 0);
131 	if (!xe->mmio.regs) {
132 		drm_err(&xe->drm, "failed to map registers\n");
133 		return -EIO;
134 	}
135 
136 	/* Setup first tile; other tiles (if present) will be setup later. */
137 	xe_mmio_init(&root_tile->mmio, root_tile, xe->mmio.regs, SZ_4M);
138 
139 	return devm_add_action_or_reset(xe->drm.dev, mmio_fini, xe);
140 }
141 
142 /**
143  * xe_mmio_init() - Initialize an MMIO instance
144  * @mmio: Pointer to the MMIO instance to initialize
145  * @tile: The tile to which the MMIO region belongs
146  * @ptr: Pointer to the start of the MMIO region
147  * @size: The size of the MMIO region in bytes
148  *
149  * This is a convenience function for minimal initialization of struct xe_mmio.
150  */
xe_mmio_init(struct xe_mmio * mmio,struct xe_tile * tile,void __iomem * ptr,u32 size)151 void xe_mmio_init(struct xe_mmio *mmio, struct xe_tile *tile, void __iomem *ptr, u32 size)
152 {
153 	xe_tile_assert(tile, size <= XE_REG_ADDR_MAX);
154 
155 	mmio->regs = ptr;
156 	mmio->regs_size = size;
157 	mmio->tile = tile;
158 }
159 
mmio_flush_pending_writes(struct xe_mmio * mmio)160 static void mmio_flush_pending_writes(struct xe_mmio *mmio)
161 {
162 #define DUMMY_REG_OFFSET	0x130030
163 	int i;
164 
165 	if (mmio->tile->xe->info.platform != XE_LUNARLAKE)
166 		return;
167 
168 	/* 4 dummy writes */
169 	for (i = 0; i < 4; i++)
170 		writel(0, mmio->regs + DUMMY_REG_OFFSET);
171 }
172 
xe_mmio_read8(struct xe_mmio * mmio,struct xe_reg reg)173 u8 xe_mmio_read8(struct xe_mmio *mmio, struct xe_reg reg)
174 {
175 	u32 addr = xe_mmio_adjusted_addr(mmio, reg.addr);
176 	u8 val;
177 
178 	/* Wa_15015404425 */
179 	mmio_flush_pending_writes(mmio);
180 
181 	val = readb(mmio->regs + addr);
182 	trace_xe_reg_rw(mmio, false, addr, val, sizeof(val));
183 
184 	return val;
185 }
186 
xe_mmio_read16(struct xe_mmio * mmio,struct xe_reg reg)187 u16 xe_mmio_read16(struct xe_mmio *mmio, struct xe_reg reg)
188 {
189 	u32 addr = xe_mmio_adjusted_addr(mmio, reg.addr);
190 	u16 val;
191 
192 	/* Wa_15015404425 */
193 	mmio_flush_pending_writes(mmio);
194 
195 	val = readw(mmio->regs + addr);
196 	trace_xe_reg_rw(mmio, false, addr, val, sizeof(val));
197 
198 	return val;
199 }
200 
xe_mmio_write32(struct xe_mmio * mmio,struct xe_reg reg,u32 val)201 void xe_mmio_write32(struct xe_mmio *mmio, struct xe_reg reg, u32 val)
202 {
203 	u32 addr = xe_mmio_adjusted_addr(mmio, reg.addr);
204 
205 	trace_xe_reg_rw(mmio, true, addr, val, sizeof(val));
206 
207 	if (!reg.vf && mmio->sriov_vf_gt)
208 		xe_gt_sriov_vf_write32(mmio->sriov_vf_gt, reg, val);
209 	else
210 		writel(val, mmio->regs + addr);
211 }
212 
xe_mmio_read32(struct xe_mmio * mmio,struct xe_reg reg)213 u32 xe_mmio_read32(struct xe_mmio *mmio, struct xe_reg reg)
214 {
215 	u32 addr = xe_mmio_adjusted_addr(mmio, reg.addr);
216 	u32 val;
217 
218 	/* Wa_15015404425 */
219 	mmio_flush_pending_writes(mmio);
220 
221 	if (!reg.vf && mmio->sriov_vf_gt)
222 		val = xe_gt_sriov_vf_read32(mmio->sriov_vf_gt, reg);
223 	else
224 		val = readl(mmio->regs + addr);
225 
226 	trace_xe_reg_rw(mmio, false, addr, val, sizeof(val));
227 
228 	return val;
229 }
230 
xe_mmio_rmw32(struct xe_mmio * mmio,struct xe_reg reg,u32 clr,u32 set)231 u32 xe_mmio_rmw32(struct xe_mmio *mmio, struct xe_reg reg, u32 clr, u32 set)
232 {
233 	u32 old, reg_val;
234 
235 	old = xe_mmio_read32(mmio, reg);
236 	reg_val = (old & ~clr) | set;
237 	xe_mmio_write32(mmio, reg, reg_val);
238 
239 	return old;
240 }
241 
xe_mmio_write32_and_verify(struct xe_mmio * mmio,struct xe_reg reg,u32 val,u32 mask,u32 eval)242 int xe_mmio_write32_and_verify(struct xe_mmio *mmio,
243 			       struct xe_reg reg, u32 val, u32 mask, u32 eval)
244 {
245 	u32 reg_val;
246 
247 	xe_mmio_write32(mmio, reg, val);
248 	reg_val = xe_mmio_read32(mmio, reg);
249 
250 	return (reg_val & mask) != eval ? -EINVAL : 0;
251 }
252 
xe_mmio_in_range(const struct xe_mmio * mmio,const struct xe_mmio_range * range,struct xe_reg reg)253 bool xe_mmio_in_range(const struct xe_mmio *mmio,
254 		      const struct xe_mmio_range *range,
255 		      struct xe_reg reg)
256 {
257 	u32 addr = xe_mmio_adjusted_addr(mmio, reg.addr);
258 
259 	return range && addr >= range->start && addr <= range->end;
260 }
261 
262 /**
263  * xe_mmio_read64_2x32() - Read a 64-bit register as two 32-bit reads
264  * @mmio: MMIO target
265  * @reg: register to read value from
266  *
267  * Although Intel GPUs have some 64-bit registers, the hardware officially
268  * only supports GTTMMADR register reads of 32 bits or smaller.  Even if
269  * a readq operation may return a reasonable value, that violation of the
270  * spec shouldn't be relied upon and all 64-bit register reads should be
271  * performed as two 32-bit reads of the upper and lower dwords.
272  *
273  * When reading registers that may be changing (such as
274  * counters), a rollover of the lower dword between the two 32-bit reads
275  * can be problematic.  This function attempts to ensure the upper dword has
276  * stabilized before returning the 64-bit value.
277  *
278  * Note that because this function may re-read the register multiple times
279  * while waiting for the value to stabilize it should not be used to read
280  * any registers where read operations have side effects.
281  *
282  * Returns the value of the 64-bit register.
283  */
xe_mmio_read64_2x32(struct xe_mmio * mmio,struct xe_reg reg)284 u64 xe_mmio_read64_2x32(struct xe_mmio *mmio, struct xe_reg reg)
285 {
286 	struct xe_reg reg_udw = { .addr = reg.addr + 0x4 };
287 	u32 ldw, udw, oldudw, retries;
288 
289 	reg.addr = xe_mmio_adjusted_addr(mmio, reg.addr);
290 	reg_udw.addr = xe_mmio_adjusted_addr(mmio, reg_udw.addr);
291 
292 	/* we shouldn't adjust just one register address */
293 	xe_tile_assert(mmio->tile, reg_udw.addr == reg.addr + 0x4);
294 
295 	oldudw = xe_mmio_read32(mmio, reg_udw);
296 	for (retries = 5; retries; --retries) {
297 		ldw = xe_mmio_read32(mmio, reg);
298 		udw = xe_mmio_read32(mmio, reg_udw);
299 
300 		if (udw == oldudw)
301 			break;
302 
303 		oldudw = udw;
304 	}
305 
306 	drm_WARN(&mmio->tile->xe->drm, retries == 0,
307 		 "64-bit read of %#x did not stabilize\n", reg.addr);
308 
309 	return (u64)udw << 32 | ldw;
310 }
311 
__xe_mmio_wait32(struct xe_mmio * mmio,struct xe_reg reg,u32 mask,u32 val,u32 timeout_us,u32 * out_val,bool atomic,bool expect_match)312 static int __xe_mmio_wait32(struct xe_mmio *mmio, struct xe_reg reg, u32 mask, u32 val,
313 			    u32 timeout_us, u32 *out_val, bool atomic, bool expect_match)
314 {
315 	ktime_t cur = ktime_get_raw();
316 	const ktime_t end = ktime_add_us(cur, timeout_us);
317 	int ret = -ETIMEDOUT;
318 	s64 wait = 10;
319 	u32 read;
320 	bool check;
321 
322 	for (;;) {
323 		read = xe_mmio_read32(mmio, reg);
324 
325 		check = (read & mask) == val;
326 		if (!expect_match)
327 			check = !check;
328 
329 		if (check) {
330 			ret = 0;
331 			break;
332 		}
333 
334 		cur = ktime_get_raw();
335 		if (!ktime_before(cur, end))
336 			break;
337 
338 		if (ktime_after(ktime_add_us(cur, wait), end))
339 			wait = ktime_us_delta(end, cur);
340 
341 		if (atomic)
342 			udelay(wait);
343 		else
344 			usleep_range(wait, wait << 1);
345 		wait <<= 1;
346 	}
347 
348 	if (ret != 0) {
349 		read = xe_mmio_read32(mmio, reg);
350 
351 		check = (read & mask) == val;
352 		if (!expect_match)
353 			check = !check;
354 
355 		if (check)
356 			ret = 0;
357 	}
358 
359 	if (out_val)
360 		*out_val = read;
361 
362 	return ret;
363 }
364 
365 /**
366  * xe_mmio_wait32() - Wait for a register to match the desired masked value
367  * @mmio: MMIO target
368  * @reg: register to read value from
369  * @mask: mask to be applied to the value read from the register
370  * @val: desired value after applying the mask
371  * @timeout_us: time out after this period of time. Wait logic tries to be
372  * smart, applying an exponential backoff until @timeout_us is reached.
373  * @out_val: if not NULL, points where to store the last unmasked value
374  * @atomic: needs to be true if calling from an atomic context
375  *
376  * This function polls for the desired masked value and returns zero on success
377  * or -ETIMEDOUT if timed out.
378  *
379  * Note that @timeout_us represents the minimum amount of time to wait before
380  * giving up. The actual time taken by this function can be a little more than
381  * @timeout_us for different reasons, specially in non-atomic contexts. Thus,
382  * it is possible that this function succeeds even after @timeout_us has passed.
383  */
xe_mmio_wait32(struct xe_mmio * mmio,struct xe_reg reg,u32 mask,u32 val,u32 timeout_us,u32 * out_val,bool atomic)384 int xe_mmio_wait32(struct xe_mmio *mmio, struct xe_reg reg, u32 mask, u32 val, u32 timeout_us,
385 		   u32 *out_val, bool atomic)
386 {
387 	return __xe_mmio_wait32(mmio, reg, mask, val, timeout_us, out_val, atomic, true);
388 }
389 
390 /**
391  * xe_mmio_wait32_not() - Wait for a register to return anything other than the given masked value
392  * @mmio: MMIO target
393  * @reg: register to read value from
394  * @mask: mask to be applied to the value read from the register
395  * @val: value not to be matched after applying the mask
396  * @timeout_us: time out after this period of time
397  * @out_val: if not NULL, points where to store the last unmasked value
398  * @atomic: needs to be true if calling from an atomic context
399  *
400  * This function works exactly like xe_mmio_wait32() with the exception that
401  * @val is expected not to be matched.
402  */
xe_mmio_wait32_not(struct xe_mmio * mmio,struct xe_reg reg,u32 mask,u32 val,u32 timeout_us,u32 * out_val,bool atomic)403 int xe_mmio_wait32_not(struct xe_mmio *mmio, struct xe_reg reg, u32 mask, u32 val, u32 timeout_us,
404 		       u32 *out_val, bool atomic)
405 {
406 	return __xe_mmio_wait32(mmio, reg, mask, val, timeout_us, out_val, atomic, false);
407 }
408