1 // SPDX-License-Identifier: MIT
2 /*
3 * Copyright © 2022 Intel Corporation
4 */
5
6 #include "i915_drv.h"
7 #include "intel_gt.h"
8 #include "intel_gt_mcr.h"
9 #include "intel_gt_print.h"
10 #include "intel_gt_regs.h"
11
12 /**
13 * DOC: GT Multicast/Replicated (MCR) Register Support
14 *
15 * Some GT registers are designed as "multicast" or "replicated" registers:
16 * multiple instances of the same register share a single MMIO offset. MCR
17 * registers are generally used when the hardware needs to potentially track
18 * independent values of a register per hardware unit (e.g., per-subslice,
19 * per-L3bank, etc.). The specific types of replication that exist vary
20 * per-platform.
21 *
22 * MMIO accesses to MCR registers are controlled according to the settings
23 * programmed in the platform's MCR_SELECTOR register(s). MMIO writes to MCR
24 * registers can be done in either a (i.e., a single write updates all
25 * instances of the register to the same value) or unicast (a write updates only
26 * one specific instance). Reads of MCR registers always operate in a unicast
27 * manner regardless of how the multicast/unicast bit is set in MCR_SELECTOR.
28 * Selection of a specific MCR instance for unicast operations is referred to
29 * as "steering."
30 *
31 * If MCR register operations are steered toward a hardware unit that is
32 * fused off or currently powered down due to power gating, the MMIO operation
33 * is "terminated" by the hardware. Terminated read operations will return a
34 * value of zero and terminated unicast write operations will be silently
35 * ignored.
36 */
37
38 #define HAS_MSLICE_STEERING(i915) (INTEL_INFO(i915)->has_mslice_steering)
39
40 static const char * const intel_steering_types[] = {
41 "L3BANK",
42 "MSLICE",
43 "LNCF",
44 "GAM",
45 "DSS",
46 "OADDRM",
47 "INSTANCE 0",
48 };
49
50 static const struct intel_mmio_range icl_l3bank_steering_table[] = {
51 { 0x00B100, 0x00B3FF },
52 {},
53 };
54
55 /*
56 * Although the bspec lists more "MSLICE" ranges than shown here, some of those
57 * are of a "GAM" subclass that has special rules. Thus we use a separate
58 * GAM table farther down for those.
59 */
60 static const struct intel_mmio_range dg2_mslice_steering_table[] = {
61 { 0x00DD00, 0x00DDFF },
62 { 0x00E900, 0x00FFFF }, /* 0xEA00 - OxEFFF is unused */
63 {},
64 };
65
66 static const struct intel_mmio_range dg2_lncf_steering_table[] = {
67 { 0x00B000, 0x00B0FF },
68 { 0x00D880, 0x00D8FF },
69 {},
70 };
71
72 static const struct intel_mmio_range xelpg_instance0_steering_table[] = {
73 { 0x000B00, 0x000BFF }, /* SQIDI */
74 { 0x001000, 0x001FFF }, /* SQIDI */
75 { 0x004000, 0x0048FF }, /* GAM */
76 { 0x008700, 0x0087FF }, /* SQIDI */
77 { 0x00B000, 0x00B0FF }, /* NODE */
78 { 0x00C800, 0x00CFFF }, /* GAM */
79 { 0x00D880, 0x00D8FF }, /* NODE */
80 { 0x00DD00, 0x00DDFF }, /* OAAL2 */
81 {},
82 };
83
84 static const struct intel_mmio_range xelpg_l3bank_steering_table[] = {
85 { 0x00B100, 0x00B3FF },
86 {},
87 };
88
89 /* DSS steering is used for SLICE ranges as well */
90 static const struct intel_mmio_range xelpg_dss_steering_table[] = {
91 { 0x005200, 0x0052FF }, /* SLICE */
92 { 0x005500, 0x007FFF }, /* SLICE */
93 { 0x008140, 0x00815F }, /* SLICE (0x8140-0x814F), DSS (0x8150-0x815F) */
94 { 0x0094D0, 0x00955F }, /* SLICE (0x94D0-0x951F), DSS (0x9520-0x955F) */
95 { 0x009680, 0x0096FF }, /* DSS */
96 { 0x00D800, 0x00D87F }, /* SLICE */
97 { 0x00DC00, 0x00DCFF }, /* SLICE */
98 { 0x00DE80, 0x00E8FF }, /* DSS (0xE000-0xE0FF reserved) */
99 {},
100 };
101
102 static const struct intel_mmio_range xelpmp_oaddrm_steering_table[] = {
103 { 0x393200, 0x39323F },
104 { 0x393400, 0x3934FF },
105 {},
106 };
107
intel_gt_mcr_init(struct intel_gt * gt)108 void intel_gt_mcr_init(struct intel_gt *gt)
109 {
110 struct drm_i915_private *i915 = gt->i915;
111 unsigned long fuse;
112 int i;
113
114 spin_lock_init(>->mcr_lock);
115
116 /*
117 * An mslice is unavailable only if both the meml3 for the slice is
118 * disabled *and* all of the DSS in the slice (quadrant) are disabled.
119 */
120 if (HAS_MSLICE_STEERING(i915)) {
121 gt->info.mslice_mask =
122 intel_slicemask_from_xehp_dssmask(gt->info.sseu.subslice_mask,
123 GEN_DSS_PER_MSLICE);
124 gt->info.mslice_mask |= REG_FIELD_GET(GEN12_MEML3_EN_MASK,
125 intel_uncore_read(gt->uncore, GEN10_MIRROR_FUSE3));
126
127 if (!gt->info.mslice_mask) /* should be impossible! */
128 gt_warn(gt, "mslice mask all zero!\n");
129 }
130
131 if (MEDIA_VER(i915) >= 13 && gt->type == GT_MEDIA) {
132 gt->steering_table[OADDRM] = xelpmp_oaddrm_steering_table;
133 } else if (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 70)) {
134 /* Wa_14016747170 */
135 if (IS_GFX_GT_IP_STEP(gt, IP_VER(12, 70), STEP_A0, STEP_B0) ||
136 IS_GFX_GT_IP_STEP(gt, IP_VER(12, 71), STEP_A0, STEP_B0))
137 fuse = REG_FIELD_GET(MTL_GT_L3_EXC_MASK,
138 intel_uncore_read(gt->uncore,
139 MTL_GT_ACTIVITY_FACTOR));
140 else
141 fuse = REG_FIELD_GET(GT_L3_EXC_MASK,
142 intel_uncore_read(gt->uncore, XEHP_FUSE4));
143
144 /*
145 * Despite the register field being named "exclude mask" the
146 * bits actually represent enabled banks (two banks per bit).
147 */
148 for_each_set_bit(i, &fuse, 3)
149 gt->info.l3bank_mask |= 0x3 << 2 * i;
150
151 gt->steering_table[INSTANCE0] = xelpg_instance0_steering_table;
152 gt->steering_table[L3BANK] = xelpg_l3bank_steering_table;
153 gt->steering_table[DSS] = xelpg_dss_steering_table;
154 } else if (IS_DG2(i915)) {
155 gt->steering_table[MSLICE] = dg2_mslice_steering_table;
156 gt->steering_table[LNCF] = dg2_lncf_steering_table;
157 /*
158 * No need to hook up the GAM table since it has a dedicated
159 * steering control register on DG2 and can use implicit
160 * steering.
161 */
162 } else if (GRAPHICS_VER(i915) >= 11 &&
163 GRAPHICS_VER_FULL(i915) < IP_VER(12, 55)) {
164 gt->steering_table[L3BANK] = icl_l3bank_steering_table;
165 gt->info.l3bank_mask =
166 ~intel_uncore_read(gt->uncore, GEN10_MIRROR_FUSE3) &
167 GEN10_L3BANK_MASK;
168 if (!gt->info.l3bank_mask) /* should be impossible! */
169 gt_warn(gt, "L3 bank mask is all zero!\n");
170 } else if (GRAPHICS_VER(i915) >= 11) {
171 /*
172 * We expect all modern platforms to have at least some
173 * type of steering that needs to be initialized.
174 */
175 MISSING_CASE(INTEL_INFO(i915)->platform);
176 }
177 }
178
179 /*
180 * Although the rest of the driver should use MCR-specific functions to
181 * read/write MCR registers, we still use the regular intel_uncore_* functions
182 * internally to implement those, so we need a way for the functions in this
183 * file to "cast" an i915_mcr_reg_t into an i915_reg_t.
184 */
mcr_reg_cast(const i915_mcr_reg_t mcr)185 static i915_reg_t mcr_reg_cast(const i915_mcr_reg_t mcr)
186 {
187 i915_reg_t r = { .reg = mcr.reg };
188
189 return r;
190 }
191
192 /*
193 * rw_with_mcr_steering_fw - Access a register with specific MCR steering
194 * @gt: GT to read register from
195 * @reg: register being accessed
196 * @rw_flag: FW_REG_READ for read access or FW_REG_WRITE for write access
197 * @group: group number (documented as "sliceid" on older platforms)
198 * @instance: instance number (documented as "subsliceid" on older platforms)
199 * @value: register value to be written (ignored for read)
200 *
201 * Context: The caller must hold the MCR lock
202 * Return: 0 for write access. register value for read access.
203 *
204 * Caller needs to make sure the relevant forcewake wells are up.
205 */
rw_with_mcr_steering_fw(struct intel_gt * gt,i915_mcr_reg_t reg,u8 rw_flag,int group,int instance,u32 value)206 static u32 rw_with_mcr_steering_fw(struct intel_gt *gt,
207 i915_mcr_reg_t reg, u8 rw_flag,
208 int group, int instance, u32 value)
209 {
210 struct intel_uncore *uncore = gt->uncore;
211 u32 mcr_mask, mcr_ss, mcr, old_mcr, val = 0;
212
213 lockdep_assert_held(>->mcr_lock);
214
215 if (GRAPHICS_VER_FULL(uncore->i915) >= IP_VER(12, 70)) {
216 /*
217 * Always leave the hardware in multicast mode when doing reads
218 * (see comment about Wa_22013088509 below) and only change it
219 * to unicast mode when doing writes of a specific instance.
220 *
221 * No need to save old steering reg value.
222 */
223 intel_uncore_write_fw(uncore, MTL_MCR_SELECTOR,
224 REG_FIELD_PREP(MTL_MCR_GROUPID, group) |
225 REG_FIELD_PREP(MTL_MCR_INSTANCEID, instance) |
226 (rw_flag == FW_REG_READ ? GEN11_MCR_MULTICAST : 0));
227 } else if (GRAPHICS_VER(uncore->i915) >= 11) {
228 mcr_mask = GEN11_MCR_SLICE_MASK | GEN11_MCR_SUBSLICE_MASK;
229 mcr_ss = GEN11_MCR_SLICE(group) | GEN11_MCR_SUBSLICE(instance);
230
231 /*
232 * Wa_22013088509
233 *
234 * The setting of the multicast/unicast bit usually wouldn't
235 * matter for read operations (which always return the value
236 * from a single register instance regardless of how that bit
237 * is set), but some platforms have a workaround requiring us
238 * to remain in multicast mode for reads. There's no real
239 * downside to this, so we'll just go ahead and do so on all
240 * platforms; we'll only clear the multicast bit from the mask
241 * when explicitly doing a write operation.
242 */
243 if (rw_flag == FW_REG_WRITE)
244 mcr_mask |= GEN11_MCR_MULTICAST;
245
246 mcr = intel_uncore_read_fw(uncore, GEN8_MCR_SELECTOR);
247 old_mcr = mcr;
248
249 mcr &= ~mcr_mask;
250 mcr |= mcr_ss;
251 intel_uncore_write_fw(uncore, GEN8_MCR_SELECTOR, mcr);
252 } else {
253 mcr_mask = GEN8_MCR_SLICE_MASK | GEN8_MCR_SUBSLICE_MASK;
254 mcr_ss = GEN8_MCR_SLICE(group) | GEN8_MCR_SUBSLICE(instance);
255
256 mcr = intel_uncore_read_fw(uncore, GEN8_MCR_SELECTOR);
257 old_mcr = mcr;
258
259 mcr &= ~mcr_mask;
260 mcr |= mcr_ss;
261 intel_uncore_write_fw(uncore, GEN8_MCR_SELECTOR, mcr);
262 }
263
264 if (rw_flag == FW_REG_READ)
265 val = intel_uncore_read_fw(uncore, mcr_reg_cast(reg));
266 else
267 intel_uncore_write_fw(uncore, mcr_reg_cast(reg), value);
268
269 /*
270 * For pre-MTL platforms, we need to restore the old value of the
271 * steering control register to ensure that implicit steering continues
272 * to behave as expected. For MTL and beyond, we need only reinstate
273 * the 'multicast' bit (and only if we did a write that cleared it).
274 */
275 if (GRAPHICS_VER_FULL(uncore->i915) >= IP_VER(12, 70) && rw_flag == FW_REG_WRITE)
276 intel_uncore_write_fw(uncore, MTL_MCR_SELECTOR, GEN11_MCR_MULTICAST);
277 else if (GRAPHICS_VER_FULL(uncore->i915) < IP_VER(12, 70))
278 intel_uncore_write_fw(uncore, GEN8_MCR_SELECTOR, old_mcr);
279
280 return val;
281 }
282
rw_with_mcr_steering(struct intel_gt * gt,i915_mcr_reg_t reg,u8 rw_flag,int group,int instance,u32 value)283 static u32 rw_with_mcr_steering(struct intel_gt *gt,
284 i915_mcr_reg_t reg, u8 rw_flag,
285 int group, int instance,
286 u32 value)
287 {
288 struct intel_uncore *uncore = gt->uncore;
289 enum forcewake_domains fw_domains;
290 unsigned long flags;
291 u32 val;
292
293 fw_domains = intel_uncore_forcewake_for_reg(uncore, mcr_reg_cast(reg),
294 rw_flag);
295 fw_domains |= intel_uncore_forcewake_for_reg(uncore,
296 GEN8_MCR_SELECTOR,
297 FW_REG_READ | FW_REG_WRITE);
298
299 intel_gt_mcr_lock(gt, &flags);
300 spin_lock(&uncore->lock);
301 intel_uncore_forcewake_get__locked(uncore, fw_domains);
302
303 val = rw_with_mcr_steering_fw(gt, reg, rw_flag, group, instance, value);
304
305 intel_uncore_forcewake_put__locked(uncore, fw_domains);
306 spin_unlock(&uncore->lock);
307 intel_gt_mcr_unlock(gt, flags);
308
309 return val;
310 }
311
312 /**
313 * intel_gt_mcr_lock - Acquire MCR steering lock
314 * @gt: GT structure
315 * @flags: storage to save IRQ flags to
316 *
317 * Performs locking to protect the steering for the duration of an MCR
318 * operation. On MTL and beyond, a hardware lock will also be taken to
319 * serialize access not only for the driver, but also for external hardware and
320 * firmware agents.
321 *
322 * Context: Takes gt->mcr_lock. uncore->lock should *not* be held when this
323 * function is called, although it may be acquired after this
324 * function call.
325 */
intel_gt_mcr_lock(struct intel_gt * gt,unsigned long * flags)326 void intel_gt_mcr_lock(struct intel_gt *gt, unsigned long *flags)
327 __acquires(>->mcr_lock)
328 {
329 unsigned long __flags;
330 int err = 0;
331
332 lockdep_assert_not_held(>->uncore->lock);
333
334 /*
335 * Starting with MTL, we need to coordinate not only with other
336 * driver threads, but also with hardware/firmware agents. A dedicated
337 * locking register is used.
338 */
339 if (GRAPHICS_VER_FULL(gt->i915) >= IP_VER(12, 70)) {
340 /*
341 * The steering control and semaphore registers are inside an
342 * "always on" power domain with respect to RC6. However there
343 * are some issues if higher-level platform sleep states are
344 * entering/exiting at the same time these registers are
345 * accessed. Grabbing GT forcewake and holding it over the
346 * entire lock/steer/unlock cycle ensures that those sleep
347 * states have been fully exited before we access these
348 * registers. This wakeref will be released in the unlock
349 * routine.
350 *
351 * Wa_22018931422
352 */
353 intel_uncore_forcewake_get(gt->uncore, FORCEWAKE_GT);
354
355 err = wait_for(intel_uncore_read_fw(gt->uncore,
356 MTL_STEER_SEMAPHORE) == 0x1, 100);
357 }
358
359 /*
360 * Even on platforms with a hardware lock, we'll continue to grab
361 * a software spinlock too for lockdep purposes. If the hardware lock
362 * was already acquired, there should never be contention on the
363 * software lock.
364 */
365 spin_lock_irqsave(>->mcr_lock, __flags);
366
367 *flags = __flags;
368
369 /*
370 * In theory we should never fail to acquire the HW semaphore; this
371 * would indicate some hardware/firmware is misbehaving and not
372 * releasing it properly.
373 */
374 if (err == -ETIMEDOUT) {
375 gt_err_ratelimited(gt, "hardware MCR steering semaphore timed out");
376 add_taint_for_CI(gt->i915, TAINT_WARN); /* CI is now unreliable */
377 }
378 }
379
380 /**
381 * intel_gt_mcr_unlock - Release MCR steering lock
382 * @gt: GT structure
383 * @flags: IRQ flags to restore
384 *
385 * Releases the lock acquired by intel_gt_mcr_lock().
386 *
387 * Context: Releases gt->mcr_lock
388 */
intel_gt_mcr_unlock(struct intel_gt * gt,unsigned long flags)389 void intel_gt_mcr_unlock(struct intel_gt *gt, unsigned long flags)
390 __releases(>->mcr_lock)
391 {
392 spin_unlock_irqrestore(>->mcr_lock, flags);
393
394 if (GRAPHICS_VER_FULL(gt->i915) >= IP_VER(12, 70)) {
395 intel_uncore_write_fw(gt->uncore, MTL_STEER_SEMAPHORE, 0x1);
396
397 intel_uncore_forcewake_put(gt->uncore, FORCEWAKE_GT);
398 }
399 }
400
401 /**
402 * intel_gt_mcr_lock_sanitize - Sanitize MCR steering lock
403 * @gt: GT structure
404 *
405 * This will be used to sanitize the initial status of the hardware lock
406 * during driver load and resume since there won't be any concurrent access
407 * from other agents at those times, but it's possible that boot firmware
408 * may have left the lock in a bad state.
409 *
410 */
intel_gt_mcr_lock_sanitize(struct intel_gt * gt)411 void intel_gt_mcr_lock_sanitize(struct intel_gt *gt)
412 {
413 /*
414 * This gets called at load/resume time, so we shouldn't be
415 * racing with other driver threads grabbing the mcr lock.
416 */
417 lockdep_assert_not_held(>->mcr_lock);
418
419 if (GRAPHICS_VER_FULL(gt->i915) >= IP_VER(12, 70))
420 intel_uncore_write_fw(gt->uncore, MTL_STEER_SEMAPHORE, 0x1);
421 }
422
423 /**
424 * intel_gt_mcr_read - read a specific instance of an MCR register
425 * @gt: GT structure
426 * @reg: the MCR register to read
427 * @group: the MCR group
428 * @instance: the MCR instance
429 *
430 * Context: Takes and releases gt->mcr_lock
431 *
432 * Returns the value read from an MCR register after steering toward a specific
433 * group/instance.
434 */
intel_gt_mcr_read(struct intel_gt * gt,i915_mcr_reg_t reg,int group,int instance)435 u32 intel_gt_mcr_read(struct intel_gt *gt,
436 i915_mcr_reg_t reg,
437 int group, int instance)
438 {
439 return rw_with_mcr_steering(gt, reg, FW_REG_READ, group, instance, 0);
440 }
441
442 /**
443 * intel_gt_mcr_unicast_write - write a specific instance of an MCR register
444 * @gt: GT structure
445 * @reg: the MCR register to write
446 * @value: value to write
447 * @group: the MCR group
448 * @instance: the MCR instance
449 *
450 * Write an MCR register in unicast mode after steering toward a specific
451 * group/instance.
452 *
453 * Context: Calls a function that takes and releases gt->mcr_lock
454 */
intel_gt_mcr_unicast_write(struct intel_gt * gt,i915_mcr_reg_t reg,u32 value,int group,int instance)455 void intel_gt_mcr_unicast_write(struct intel_gt *gt, i915_mcr_reg_t reg, u32 value,
456 int group, int instance)
457 {
458 rw_with_mcr_steering(gt, reg, FW_REG_WRITE, group, instance, value);
459 }
460
461 /**
462 * intel_gt_mcr_multicast_write - write a value to all instances of an MCR register
463 * @gt: GT structure
464 * @reg: the MCR register to write
465 * @value: value to write
466 *
467 * Write an MCR register in multicast mode to update all instances.
468 *
469 * Context: Takes and releases gt->mcr_lock
470 */
intel_gt_mcr_multicast_write(struct intel_gt * gt,i915_mcr_reg_t reg,u32 value)471 void intel_gt_mcr_multicast_write(struct intel_gt *gt,
472 i915_mcr_reg_t reg, u32 value)
473 {
474 unsigned long flags;
475
476 intel_gt_mcr_lock(gt, &flags);
477
478 /*
479 * Ensure we have multicast behavior, just in case some non-i915 agent
480 * left the hardware in unicast mode.
481 */
482 if (GRAPHICS_VER_FULL(gt->i915) >= IP_VER(12, 70))
483 intel_uncore_write_fw(gt->uncore, MTL_MCR_SELECTOR, GEN11_MCR_MULTICAST);
484
485 intel_uncore_write(gt->uncore, mcr_reg_cast(reg), value);
486
487 intel_gt_mcr_unlock(gt, flags);
488 }
489
490 /**
491 * intel_gt_mcr_multicast_write_fw - write a value to all instances of an MCR register
492 * @gt: GT structure
493 * @reg: the MCR register to write
494 * @value: value to write
495 *
496 * Write an MCR register in multicast mode to update all instances. This
497 * function assumes the caller is already holding any necessary forcewake
498 * domains; use intel_gt_mcr_multicast_write() in cases where forcewake should
499 * be obtained automatically.
500 *
501 * Context: The caller must hold gt->mcr_lock.
502 */
intel_gt_mcr_multicast_write_fw(struct intel_gt * gt,i915_mcr_reg_t reg,u32 value)503 void intel_gt_mcr_multicast_write_fw(struct intel_gt *gt, i915_mcr_reg_t reg, u32 value)
504 {
505 lockdep_assert_held(>->mcr_lock);
506
507 /*
508 * Ensure we have multicast behavior, just in case some non-i915 agent
509 * left the hardware in unicast mode.
510 */
511 if (GRAPHICS_VER_FULL(gt->i915) >= IP_VER(12, 70))
512 intel_uncore_write_fw(gt->uncore, MTL_MCR_SELECTOR, GEN11_MCR_MULTICAST);
513
514 intel_uncore_write_fw(gt->uncore, mcr_reg_cast(reg), value);
515 }
516
517 /**
518 * intel_gt_mcr_multicast_rmw - Performs a multicast RMW operations
519 * @gt: GT structure
520 * @reg: the MCR register to read and write
521 * @clear: bits to clear during RMW
522 * @set: bits to set during RMW
523 *
524 * Performs a read-modify-write on an MCR register in a multicast manner.
525 * This operation only makes sense on MCR registers where all instances are
526 * expected to have the same value. The read will target any non-terminated
527 * instance and the write will be applied to all instances.
528 *
529 * This function assumes the caller is already holding any necessary forcewake
530 * domains; use intel_gt_mcr_multicast_rmw() in cases where forcewake should
531 * be obtained automatically.
532 *
533 * Context: Calls functions that take and release gt->mcr_lock
534 *
535 * Returns the old (unmodified) value read.
536 */
intel_gt_mcr_multicast_rmw(struct intel_gt * gt,i915_mcr_reg_t reg,u32 clear,u32 set)537 u32 intel_gt_mcr_multicast_rmw(struct intel_gt *gt, i915_mcr_reg_t reg,
538 u32 clear, u32 set)
539 {
540 u32 val = intel_gt_mcr_read_any(gt, reg);
541
542 intel_gt_mcr_multicast_write(gt, reg, (val & ~clear) | set);
543
544 return val;
545 }
546
547 /*
548 * reg_needs_read_steering - determine whether a register read requires
549 * explicit steering
550 * @gt: GT structure
551 * @reg: the register to check steering requirements for
552 * @type: type of multicast steering to check
553 *
554 * Determines whether @reg needs explicit steering of a specific type for
555 * reads.
556 *
557 * Returns false if @reg does not belong to a register range of the given
558 * steering type, or if the default (subslice-based) steering IDs are suitable
559 * for @type steering too.
560 */
reg_needs_read_steering(struct intel_gt * gt,i915_mcr_reg_t reg,enum intel_steering_type type)561 static bool reg_needs_read_steering(struct intel_gt *gt,
562 i915_mcr_reg_t reg,
563 enum intel_steering_type type)
564 {
565 u32 offset = i915_mmio_reg_offset(reg);
566 const struct intel_mmio_range *entry;
567
568 if (likely(!gt->steering_table[type]))
569 return false;
570
571 if (IS_GSI_REG(offset))
572 offset += gt->uncore->gsi_offset;
573
574 for (entry = gt->steering_table[type]; entry->end; entry++) {
575 if (offset >= entry->start && offset <= entry->end)
576 return true;
577 }
578
579 return false;
580 }
581
582 /*
583 * get_nonterminated_steering - determines valid IDs for a class of MCR steering
584 * @gt: GT structure
585 * @type: multicast register type
586 * @group: Group ID returned
587 * @instance: Instance ID returned
588 *
589 * Determines group and instance values that will steer reads of the specified
590 * MCR class to a non-terminated instance.
591 */
get_nonterminated_steering(struct intel_gt * gt,enum intel_steering_type type,u8 * group,u8 * instance)592 static void get_nonterminated_steering(struct intel_gt *gt,
593 enum intel_steering_type type,
594 u8 *group, u8 *instance)
595 {
596 u32 dss;
597
598 switch (type) {
599 case L3BANK:
600 *group = 0; /* unused */
601 *instance = __ffs(gt->info.l3bank_mask);
602 break;
603 case MSLICE:
604 GEM_WARN_ON(!HAS_MSLICE_STEERING(gt->i915));
605 *group = __ffs(gt->info.mslice_mask);
606 *instance = 0; /* unused */
607 break;
608 case LNCF:
609 /*
610 * An LNCF is always present if its mslice is present, so we
611 * can safely just steer to LNCF 0 in all cases.
612 */
613 GEM_WARN_ON(!HAS_MSLICE_STEERING(gt->i915));
614 *group = __ffs(gt->info.mslice_mask) << 1;
615 *instance = 0; /* unused */
616 break;
617 case GAM:
618 *group = IS_DG2(gt->i915) ? 1 : 0;
619 *instance = 0;
620 break;
621 case DSS:
622 dss = intel_sseu_find_first_xehp_dss(>->info.sseu, 0, 0);
623 *group = dss / GEN_DSS_PER_GSLICE;
624 *instance = dss % GEN_DSS_PER_GSLICE;
625 break;
626 case INSTANCE0:
627 /*
628 * There are a lot of MCR types for which instance (0, 0)
629 * will always provide a non-terminated value.
630 */
631 *group = 0;
632 *instance = 0;
633 break;
634 case OADDRM:
635 if ((VDBOX_MASK(gt) | VEBOX_MASK(gt) | gt->info.sfc_mask) & BIT(0))
636 *group = 0;
637 else
638 *group = 1;
639 *instance = 0;
640 break;
641 default:
642 MISSING_CASE(type);
643 *group = 0;
644 *instance = 0;
645 }
646 }
647
648 /**
649 * intel_gt_mcr_get_nonterminated_steering - find group/instance values that
650 * will steer a register to a non-terminated instance
651 * @gt: GT structure
652 * @reg: register for which the steering is required
653 * @group: return variable for group steering
654 * @instance: return variable for instance steering
655 *
656 * This function returns a group/instance pair that is guaranteed to work for
657 * read steering of the given register. Note that a value will be returned even
658 * if the register is not replicated and therefore does not actually require
659 * steering.
660 */
intel_gt_mcr_get_nonterminated_steering(struct intel_gt * gt,i915_mcr_reg_t reg,u8 * group,u8 * instance)661 void intel_gt_mcr_get_nonterminated_steering(struct intel_gt *gt,
662 i915_mcr_reg_t reg,
663 u8 *group, u8 *instance)
664 {
665 int type;
666
667 for (type = 0; type < NUM_STEERING_TYPES; type++) {
668 if (reg_needs_read_steering(gt, reg, type)) {
669 get_nonterminated_steering(gt, type, group, instance);
670 return;
671 }
672 }
673
674 *group = gt->default_steering.groupid;
675 *instance = gt->default_steering.instanceid;
676 }
677
678 /**
679 * intel_gt_mcr_read_any_fw - reads one instance of an MCR register
680 * @gt: GT structure
681 * @reg: register to read
682 *
683 * Reads a GT MCR register. The read will be steered to a non-terminated
684 * instance (i.e., one that isn't fused off or powered down by power gating).
685 * This function assumes the caller is already holding any necessary forcewake
686 * domains; use intel_gt_mcr_read_any() in cases where forcewake should be
687 * obtained automatically.
688 *
689 * Context: The caller must hold gt->mcr_lock.
690 *
691 * Returns the value from a non-terminated instance of @reg.
692 */
intel_gt_mcr_read_any_fw(struct intel_gt * gt,i915_mcr_reg_t reg)693 u32 intel_gt_mcr_read_any_fw(struct intel_gt *gt, i915_mcr_reg_t reg)
694 {
695 int type;
696 u8 group, instance;
697
698 lockdep_assert_held(>->mcr_lock);
699
700 for (type = 0; type < NUM_STEERING_TYPES; type++) {
701 if (reg_needs_read_steering(gt, reg, type)) {
702 get_nonterminated_steering(gt, type, &group, &instance);
703 return rw_with_mcr_steering_fw(gt, reg,
704 FW_REG_READ,
705 group, instance, 0);
706 }
707 }
708
709 return intel_uncore_read_fw(gt->uncore, mcr_reg_cast(reg));
710 }
711
712 /**
713 * intel_gt_mcr_read_any - reads one instance of an MCR register
714 * @gt: GT structure
715 * @reg: register to read
716 *
717 * Reads a GT MCR register. The read will be steered to a non-terminated
718 * instance (i.e., one that isn't fused off or powered down by power gating).
719 *
720 * Context: Calls a function that takes and releases gt->mcr_lock.
721 *
722 * Returns the value from a non-terminated instance of @reg.
723 */
intel_gt_mcr_read_any(struct intel_gt * gt,i915_mcr_reg_t reg)724 u32 intel_gt_mcr_read_any(struct intel_gt *gt, i915_mcr_reg_t reg)
725 {
726 int type;
727 u8 group, instance;
728
729 for (type = 0; type < NUM_STEERING_TYPES; type++) {
730 if (reg_needs_read_steering(gt, reg, type)) {
731 get_nonterminated_steering(gt, type, &group, &instance);
732 return rw_with_mcr_steering(gt, reg,
733 FW_REG_READ,
734 group, instance, 0);
735 }
736 }
737
738 return intel_uncore_read(gt->uncore, mcr_reg_cast(reg));
739 }
740
report_steering_type(struct drm_printer * p,struct intel_gt * gt,enum intel_steering_type type,bool dump_table)741 static void report_steering_type(struct drm_printer *p,
742 struct intel_gt *gt,
743 enum intel_steering_type type,
744 bool dump_table)
745 {
746 const struct intel_mmio_range *entry;
747 u8 group, instance;
748
749 BUILD_BUG_ON(ARRAY_SIZE(intel_steering_types) != NUM_STEERING_TYPES);
750
751 if (!gt->steering_table[type]) {
752 drm_printf(p, "%s steering: uses default steering\n",
753 intel_steering_types[type]);
754 return;
755 }
756
757 get_nonterminated_steering(gt, type, &group, &instance);
758 drm_printf(p, "%s steering: group=0x%x, instance=0x%x\n",
759 intel_steering_types[type], group, instance);
760
761 if (!dump_table)
762 return;
763
764 for (entry = gt->steering_table[type]; entry->end; entry++)
765 drm_printf(p, "\t0x%06x - 0x%06x\n", entry->start, entry->end);
766 }
767
intel_gt_mcr_report_steering(struct drm_printer * p,struct intel_gt * gt,bool dump_table)768 void intel_gt_mcr_report_steering(struct drm_printer *p, struct intel_gt *gt,
769 bool dump_table)
770 {
771 /*
772 * Starting with MTL we no longer have default steering;
773 * all ranges are explicitly steered.
774 */
775 if (GRAPHICS_VER_FULL(gt->i915) < IP_VER(12, 70))
776 drm_printf(p, "Default steering: group=0x%x, instance=0x%x\n",
777 gt->default_steering.groupid,
778 gt->default_steering.instanceid);
779
780 if (GRAPHICS_VER_FULL(gt->i915) >= IP_VER(12, 70)) {
781 for (int i = 0; i < NUM_STEERING_TYPES; i++)
782 if (gt->steering_table[i])
783 report_steering_type(p, gt, i, dump_table);
784 } else if (HAS_MSLICE_STEERING(gt->i915)) {
785 report_steering_type(p, gt, MSLICE, dump_table);
786 report_steering_type(p, gt, LNCF, dump_table);
787 }
788 }
789
790 /**
791 * intel_gt_mcr_get_ss_steering - returns the group/instance steering for a SS
792 * @gt: GT structure
793 * @dss: DSS ID to obtain steering for
794 * @group: pointer to storage for steering group ID
795 * @instance: pointer to storage for steering instance ID
796 *
797 * Returns the steering IDs (via the @group and @instance parameters) that
798 * correspond to a specific subslice/DSS ID.
799 */
intel_gt_mcr_get_ss_steering(struct intel_gt * gt,unsigned int dss,unsigned int * group,unsigned int * instance)800 void intel_gt_mcr_get_ss_steering(struct intel_gt *gt, unsigned int dss,
801 unsigned int *group, unsigned int *instance)
802 {
803 if (GRAPHICS_VER_FULL(gt->i915) >= IP_VER(12, 55)) {
804 *group = dss / GEN_DSS_PER_GSLICE;
805 *instance = dss % GEN_DSS_PER_GSLICE;
806 } else {
807 *group = dss / GEN_MAX_SS_PER_HSW_SLICE;
808 *instance = dss % GEN_MAX_SS_PER_HSW_SLICE;
809 return;
810 }
811 }
812
813 /**
814 * intel_gt_mcr_wait_for_reg - wait until MCR register matches expected state
815 * @gt: GT structure
816 * @reg: the register to read
817 * @mask: mask to apply to register value
818 * @value: value to wait for
819 * @fast_timeout_us: fast timeout in microsecond for atomic/tight wait
820 * @slow_timeout_ms: slow timeout in millisecond
821 *
822 * This routine waits until the target register @reg contains the expected
823 * @value after applying the @mask, i.e. it waits until ::
824 *
825 * (intel_gt_mcr_read_any_fw(gt, reg) & mask) == value
826 *
827 * Otherwise, the wait will timeout after @slow_timeout_ms milliseconds.
828 * For atomic context @slow_timeout_ms must be zero and @fast_timeout_us
829 * must be not larger than 20,0000 microseconds.
830 *
831 * This function is basically an MCR-friendly version of
832 * __intel_wait_for_register_fw(). Generally this function will only be used
833 * on GAM registers which are a bit special --- although they're MCR registers,
834 * reads (e.g., waiting for status updates) are always directed to the primary
835 * instance.
836 *
837 * Note that this routine assumes the caller holds forcewake asserted, it is
838 * not suitable for very long waits.
839 *
840 * Context: Calls a function that takes and releases gt->mcr_lock
841 * Return: 0 if the register matches the desired condition, or -ETIMEDOUT.
842 */
intel_gt_mcr_wait_for_reg(struct intel_gt * gt,i915_mcr_reg_t reg,u32 mask,u32 value,unsigned int fast_timeout_us,unsigned int slow_timeout_ms)843 int intel_gt_mcr_wait_for_reg(struct intel_gt *gt,
844 i915_mcr_reg_t reg,
845 u32 mask,
846 u32 value,
847 unsigned int fast_timeout_us,
848 unsigned int slow_timeout_ms)
849 {
850 int ret;
851
852 lockdep_assert_not_held(>->mcr_lock);
853
854 #define done ((intel_gt_mcr_read_any(gt, reg) & mask) == value)
855
856 /* Catch any overuse of this function */
857 might_sleep_if(slow_timeout_ms);
858 GEM_BUG_ON(fast_timeout_us > 20000);
859 GEM_BUG_ON(!fast_timeout_us && !slow_timeout_ms);
860
861 ret = -ETIMEDOUT;
862 if (fast_timeout_us && fast_timeout_us <= 20000)
863 ret = _wait_for_atomic(done, fast_timeout_us, 0);
864 if (ret && slow_timeout_ms)
865 ret = wait_for(done, slow_timeout_ms);
866
867 return ret;
868 #undef done
869 }
870