1 // SPDX-License-Identifier: MIT
2 /*
3 * Copyright © 2022 Intel Corporation
4 */
5
6 #include "xe_guc.h"
7
8 #include <drm/drm_managed.h>
9
10 #include <generated/xe_wa_oob.h>
11
12 #include "abi/guc_actions_abi.h"
13 #include "abi/guc_errors_abi.h"
14 #include "regs/xe_gt_regs.h"
15 #include "regs/xe_gtt_defs.h"
16 #include "regs/xe_guc_regs.h"
17 #include "regs/xe_irq_regs.h"
18 #include "xe_bo.h"
19 #include "xe_device.h"
20 #include "xe_force_wake.h"
21 #include "xe_gt.h"
22 #include "xe_gt_printk.h"
23 #include "xe_gt_sriov_vf.h"
24 #include "xe_gt_throttle.h"
25 #include "xe_guc_ads.h"
26 #include "xe_guc_buf.h"
27 #include "xe_guc_capture.h"
28 #include "xe_guc_ct.h"
29 #include "xe_guc_db_mgr.h"
30 #include "xe_guc_engine_activity.h"
31 #include "xe_guc_hwconfig.h"
32 #include "xe_guc_klv_helpers.h"
33 #include "xe_guc_log.h"
34 #include "xe_guc_pc.h"
35 #include "xe_guc_relay.h"
36 #include "xe_guc_submit.h"
37 #include "xe_memirq.h"
38 #include "xe_mmio.h"
39 #include "xe_platform_types.h"
40 #include "xe_sriov.h"
41 #include "xe_uc.h"
42 #include "xe_uc_fw.h"
43 #include "xe_wa.h"
44 #include "xe_wopcm.h"
45
guc_bo_ggtt_addr(struct xe_guc * guc,struct xe_bo * bo)46 static u32 guc_bo_ggtt_addr(struct xe_guc *guc,
47 struct xe_bo *bo)
48 {
49 struct xe_device *xe = guc_to_xe(guc);
50 u32 addr;
51
52 /*
53 * For most BOs, the address on the allocating tile is fine. However for
54 * some, e.g. G2G CTB, the address on a specific tile is required as it
55 * might be different for each tile. So, just always ask for the address
56 * on the target GuC.
57 */
58 addr = __xe_bo_ggtt_addr(bo, gt_to_tile(guc_to_gt(guc))->id);
59
60 /* GuC addresses above GUC_GGTT_TOP don't map through the GTT */
61 xe_assert(xe, addr >= xe_wopcm_size(guc_to_xe(guc)));
62 xe_assert(xe, addr < GUC_GGTT_TOP);
63 xe_assert(xe, xe_bo_size(bo) <= GUC_GGTT_TOP - addr);
64
65 return addr;
66 }
67
guc_ctl_debug_flags(struct xe_guc * guc)68 static u32 guc_ctl_debug_flags(struct xe_guc *guc)
69 {
70 u32 level = xe_guc_log_get_level(&guc->log);
71 u32 flags = 0;
72
73 if (!GUC_LOG_LEVEL_IS_VERBOSE(level))
74 flags |= GUC_LOG_DISABLED;
75 else
76 flags |= GUC_LOG_LEVEL_TO_VERBOSITY(level) <<
77 GUC_LOG_VERBOSITY_SHIFT;
78
79 return flags;
80 }
81
guc_ctl_feature_flags(struct xe_guc * guc)82 static u32 guc_ctl_feature_flags(struct xe_guc *guc)
83 {
84 u32 flags = GUC_CTL_ENABLE_LITE_RESTORE;
85
86 if (!guc_to_xe(guc)->info.skip_guc_pc)
87 flags |= GUC_CTL_ENABLE_SLPC;
88
89 return flags;
90 }
91
guc_ctl_log_params_flags(struct xe_guc * guc)92 static u32 guc_ctl_log_params_flags(struct xe_guc *guc)
93 {
94 u32 offset = guc_bo_ggtt_addr(guc, guc->log.bo) >> PAGE_SHIFT;
95 u32 flags;
96
97 #if (((CRASH_BUFFER_SIZE) % SZ_1M) == 0)
98 #define LOG_UNIT SZ_1M
99 #define LOG_FLAG GUC_LOG_LOG_ALLOC_UNITS
100 #else
101 #define LOG_UNIT SZ_4K
102 #define LOG_FLAG 0
103 #endif
104
105 #if (((CAPTURE_BUFFER_SIZE) % SZ_1M) == 0)
106 #define CAPTURE_UNIT SZ_1M
107 #define CAPTURE_FLAG GUC_LOG_CAPTURE_ALLOC_UNITS
108 #else
109 #define CAPTURE_UNIT SZ_4K
110 #define CAPTURE_FLAG 0
111 #endif
112
113 BUILD_BUG_ON(!CRASH_BUFFER_SIZE);
114 BUILD_BUG_ON(!IS_ALIGNED(CRASH_BUFFER_SIZE, LOG_UNIT));
115 BUILD_BUG_ON(!DEBUG_BUFFER_SIZE);
116 BUILD_BUG_ON(!IS_ALIGNED(DEBUG_BUFFER_SIZE, LOG_UNIT));
117 BUILD_BUG_ON(!CAPTURE_BUFFER_SIZE);
118 BUILD_BUG_ON(!IS_ALIGNED(CAPTURE_BUFFER_SIZE, CAPTURE_UNIT));
119
120 BUILD_BUG_ON((CRASH_BUFFER_SIZE / LOG_UNIT - 1) >
121 (GUC_LOG_CRASH_MASK >> GUC_LOG_CRASH_SHIFT));
122 BUILD_BUG_ON((DEBUG_BUFFER_SIZE / LOG_UNIT - 1) >
123 (GUC_LOG_DEBUG_MASK >> GUC_LOG_DEBUG_SHIFT));
124 BUILD_BUG_ON((CAPTURE_BUFFER_SIZE / CAPTURE_UNIT - 1) >
125 (GUC_LOG_CAPTURE_MASK >> GUC_LOG_CAPTURE_SHIFT));
126
127 flags = GUC_LOG_VALID |
128 GUC_LOG_NOTIFY_ON_HALF_FULL |
129 CAPTURE_FLAG |
130 LOG_FLAG |
131 ((CRASH_BUFFER_SIZE / LOG_UNIT - 1) << GUC_LOG_CRASH_SHIFT) |
132 ((DEBUG_BUFFER_SIZE / LOG_UNIT - 1) << GUC_LOG_DEBUG_SHIFT) |
133 ((CAPTURE_BUFFER_SIZE / CAPTURE_UNIT - 1) <<
134 GUC_LOG_CAPTURE_SHIFT) |
135 (offset << GUC_LOG_BUF_ADDR_SHIFT);
136
137 #undef LOG_UNIT
138 #undef LOG_FLAG
139 #undef CAPTURE_UNIT
140 #undef CAPTURE_FLAG
141
142 return flags;
143 }
144
guc_ctl_ads_flags(struct xe_guc * guc)145 static u32 guc_ctl_ads_flags(struct xe_guc *guc)
146 {
147 u32 ads = guc_bo_ggtt_addr(guc, guc->ads.bo) >> PAGE_SHIFT;
148 u32 flags = ads << GUC_ADS_ADDR_SHIFT;
149
150 return flags;
151 }
152
needs_wa_dual_queue(struct xe_gt * gt)153 static bool needs_wa_dual_queue(struct xe_gt *gt)
154 {
155 /*
156 * The DUAL_QUEUE_WA tells the GuC to not allow concurrent submissions
157 * on RCS and CCSes with different address spaces, which on DG2 is
158 * required as a WA for an HW bug.
159 */
160 if (XE_WA(gt, 22011391025))
161 return true;
162
163 /*
164 * On newer platforms, the HW has been updated to not allow parallel
165 * execution of different address spaces, so the RCS/CCS will stall the
166 * context switch if one of the other RCS/CCSes is busy with a different
167 * address space. While functionally correct, having a submission
168 * stalled on the HW limits the GuC ability to shuffle things around and
169 * can cause complications if the non-stalled submission runs for a long
170 * time, because the GuC doesn't know that the stalled submission isn't
171 * actually running and might declare it as hung. Therefore, we enable
172 * the DUAL_QUEUE_WA on all newer platforms on GTs that have CCS engines
173 * to move management back to the GuC.
174 */
175 if (CCS_MASK(gt) && GRAPHICS_VERx100(gt_to_xe(gt)) >= 1270)
176 return true;
177
178 return false;
179 }
180
guc_ctl_wa_flags(struct xe_guc * guc)181 static u32 guc_ctl_wa_flags(struct xe_guc *guc)
182 {
183 struct xe_device *xe = guc_to_xe(guc);
184 struct xe_gt *gt = guc_to_gt(guc);
185 u32 flags = 0;
186
187 if (XE_WA(gt, 22012773006))
188 flags |= GUC_WA_POLLCS;
189
190 if (XE_WA(gt, 14014475959))
191 flags |= GUC_WA_HOLD_CCS_SWITCHOUT;
192
193 if (needs_wa_dual_queue(gt))
194 flags |= GUC_WA_DUAL_QUEUE;
195
196 /*
197 * Wa_22011802037: FIXME - there's more to be done than simply setting
198 * this flag: make sure each CS is stopped when preparing for GT reset
199 * and wait for pending MI_FW.
200 */
201 if (GRAPHICS_VERx100(xe) < 1270)
202 flags |= GUC_WA_PRE_PARSER;
203
204 if (XE_WA(gt, 22012727170) || XE_WA(gt, 22012727685))
205 flags |= GUC_WA_CONTEXT_ISOLATION;
206
207 if (XE_WA(gt, 18020744125) &&
208 !xe_hw_engine_mask_per_class(gt, XE_ENGINE_CLASS_RENDER))
209 flags |= GUC_WA_RCS_REGS_IN_CCS_REGS_LIST;
210
211 if (XE_WA(gt, 1509372804))
212 flags |= GUC_WA_RENDER_RST_RC6_EXIT;
213
214 if (XE_WA(gt, 14018913170))
215 flags |= GUC_WA_ENABLE_TSC_CHECK_ON_RC6;
216
217 return flags;
218 }
219
guc_ctl_devid(struct xe_guc * guc)220 static u32 guc_ctl_devid(struct xe_guc *guc)
221 {
222 struct xe_device *xe = guc_to_xe(guc);
223
224 return (((u32)xe->info.devid) << 16) | xe->info.revid;
225 }
226
guc_print_params(struct xe_guc * guc)227 static void guc_print_params(struct xe_guc *guc)
228 {
229 struct xe_gt *gt = guc_to_gt(guc);
230 u32 *params = guc->params;
231 int i;
232
233 BUILD_BUG_ON(sizeof(guc->params) != GUC_CTL_MAX_DWORDS * sizeof(u32));
234 BUILD_BUG_ON(GUC_CTL_MAX_DWORDS + 2 != SOFT_SCRATCH_COUNT);
235
236 for (i = 0; i < GUC_CTL_MAX_DWORDS; i++)
237 xe_gt_dbg(gt, "GuC param[%2d] = 0x%08x\n", i, params[i]);
238 }
239
guc_init_params(struct xe_guc * guc)240 static void guc_init_params(struct xe_guc *guc)
241 {
242 u32 *params = guc->params;
243
244 params[GUC_CTL_LOG_PARAMS] = guc_ctl_log_params_flags(guc);
245 params[GUC_CTL_FEATURE] = 0;
246 params[GUC_CTL_DEBUG] = guc_ctl_debug_flags(guc);
247 params[GUC_CTL_ADS] = guc_ctl_ads_flags(guc);
248 params[GUC_CTL_WA] = 0;
249 params[GUC_CTL_DEVID] = guc_ctl_devid(guc);
250
251 guc_print_params(guc);
252 }
253
guc_init_params_post_hwconfig(struct xe_guc * guc)254 static void guc_init_params_post_hwconfig(struct xe_guc *guc)
255 {
256 u32 *params = guc->params;
257
258 params[GUC_CTL_LOG_PARAMS] = guc_ctl_log_params_flags(guc);
259 params[GUC_CTL_FEATURE] = guc_ctl_feature_flags(guc);
260 params[GUC_CTL_DEBUG] = guc_ctl_debug_flags(guc);
261 params[GUC_CTL_ADS] = guc_ctl_ads_flags(guc);
262 params[GUC_CTL_WA] = guc_ctl_wa_flags(guc);
263 params[GUC_CTL_DEVID] = guc_ctl_devid(guc);
264
265 guc_print_params(guc);
266 }
267
268 /*
269 * Initialize the GuC parameter block before starting the firmware
270 * transfer. These parameters are read by the firmware on startup
271 * and cannot be changed thereafter.
272 */
guc_write_params(struct xe_guc * guc)273 static void guc_write_params(struct xe_guc *guc)
274 {
275 struct xe_gt *gt = guc_to_gt(guc);
276 int i;
277
278 xe_force_wake_assert_held(gt_to_fw(gt), XE_FW_GT);
279
280 xe_mmio_write32(>->mmio, SOFT_SCRATCH(0), 0);
281
282 for (i = 0; i < GUC_CTL_MAX_DWORDS; i++)
283 xe_mmio_write32(>->mmio, SOFT_SCRATCH(1 + i), guc->params[i]);
284 }
285
guc_action_register_g2g_buffer(struct xe_guc * guc,u32 type,u32 dst_tile,u32 dst_dev,u32 desc_addr,u32 buff_addr,u32 size)286 static int guc_action_register_g2g_buffer(struct xe_guc *guc, u32 type, u32 dst_tile, u32 dst_dev,
287 u32 desc_addr, u32 buff_addr, u32 size)
288 {
289 struct xe_gt *gt = guc_to_gt(guc);
290 struct xe_device *xe = gt_to_xe(gt);
291 u32 action[] = {
292 XE_GUC_ACTION_REGISTER_G2G,
293 FIELD_PREP(XE_G2G_REGISTER_SIZE, size / SZ_4K - 1) |
294 FIELD_PREP(XE_G2G_REGISTER_TYPE, type) |
295 FIELD_PREP(XE_G2G_REGISTER_TILE, dst_tile) |
296 FIELD_PREP(XE_G2G_REGISTER_DEVICE, dst_dev),
297 desc_addr,
298 buff_addr,
299 };
300
301 xe_assert(xe, (type == XE_G2G_TYPE_IN) || (type == XE_G2G_TYPE_OUT));
302 xe_assert(xe, !(size % SZ_4K));
303
304 return xe_guc_ct_send_block(&guc->ct, action, ARRAY_SIZE(action));
305 }
306
guc_action_deregister_g2g_buffer(struct xe_guc * guc,u32 type,u32 dst_tile,u32 dst_dev)307 static int guc_action_deregister_g2g_buffer(struct xe_guc *guc, u32 type, u32 dst_tile, u32 dst_dev)
308 {
309 struct xe_gt *gt = guc_to_gt(guc);
310 struct xe_device *xe = gt_to_xe(gt);
311 u32 action[] = {
312 XE_GUC_ACTION_DEREGISTER_G2G,
313 FIELD_PREP(XE_G2G_DEREGISTER_TYPE, type) |
314 FIELD_PREP(XE_G2G_DEREGISTER_TILE, dst_tile) |
315 FIELD_PREP(XE_G2G_DEREGISTER_DEVICE, dst_dev),
316 };
317
318 xe_assert(xe, (type == XE_G2G_TYPE_IN) || (type == XE_G2G_TYPE_OUT));
319
320 return xe_guc_ct_send_block(&guc->ct, action, ARRAY_SIZE(action));
321 }
322
323 #define G2G_DEV(gt) (((gt)->info.type == XE_GT_TYPE_MAIN) ? 0 : 1)
324
325 #define G2G_BUFFER_SIZE (SZ_4K)
326 #define G2G_DESC_SIZE (64)
327 #define G2G_DESC_AREA_SIZE (SZ_4K)
328
329 /*
330 * Generate a unique id for each bi-directional CTB for each pair of
331 * near and far tiles/devices. The id can then be used as an index into
332 * a single allocation that is sub-divided into multiple CTBs.
333 *
334 * For example, with two devices per tile and two tiles, the table should
335 * look like:
336 * Far <tile>.<dev>
337 * 0.0 0.1 1.0 1.1
338 * N 0.0 --/-- 00/01 02/03 04/05
339 * e 0.1 01/00 --/-- 06/07 08/09
340 * a 1.0 03/02 07/06 --/-- 10/11
341 * r 1.1 05/04 09/08 11/10 --/--
342 *
343 * Where each entry is Rx/Tx channel id.
344 *
345 * So GuC #3 (tile 1, dev 1) talking to GuC #2 (tile 1, dev 0) would
346 * be reading from channel #11 and writing to channel #10. Whereas,
347 * GuC #2 talking to GuC #3 would be read on #10 and write to #11.
348 */
g2g_slot(u32 near_tile,u32 near_dev,u32 far_tile,u32 far_dev,u32 type,u32 max_inst,bool have_dev)349 static unsigned int g2g_slot(u32 near_tile, u32 near_dev, u32 far_tile, u32 far_dev,
350 u32 type, u32 max_inst, bool have_dev)
351 {
352 u32 near = near_tile, far = far_tile;
353 u32 idx = 0, x, y, direction;
354 int i;
355
356 if (have_dev) {
357 near = (near << 1) | near_dev;
358 far = (far << 1) | far_dev;
359 }
360
361 /* No need to send to one's self */
362 if (far == near)
363 return -1;
364
365 if (far > near) {
366 /* Top right table half */
367 x = far;
368 y = near;
369
370 /* T/R is 'forwards' direction */
371 direction = type;
372 } else {
373 /* Bottom left table half */
374 x = near;
375 y = far;
376
377 /* B/L is 'backwards' direction */
378 direction = (1 - type);
379 }
380
381 /* Count the rows prior to the target */
382 for (i = y; i > 0; i--)
383 idx += max_inst - i;
384
385 /* Count this row up to the target */
386 idx += (x - 1 - y);
387
388 /* Slots are in Rx/Tx pairs */
389 idx *= 2;
390
391 /* Pick Rx/Tx direction */
392 idx += direction;
393
394 return idx;
395 }
396
guc_g2g_register(struct xe_guc * near_guc,struct xe_gt * far_gt,u32 type,bool have_dev)397 static int guc_g2g_register(struct xe_guc *near_guc, struct xe_gt *far_gt, u32 type, bool have_dev)
398 {
399 struct xe_gt *near_gt = guc_to_gt(near_guc);
400 struct xe_device *xe = gt_to_xe(near_gt);
401 struct xe_bo *g2g_bo;
402 u32 near_tile = gt_to_tile(near_gt)->id;
403 u32 near_dev = G2G_DEV(near_gt);
404 u32 far_tile = gt_to_tile(far_gt)->id;
405 u32 far_dev = G2G_DEV(far_gt);
406 u32 max = xe->info.gt_count;
407 u32 base, desc, buf;
408 int slot;
409
410 /* G2G is not allowed between different cards */
411 xe_assert(xe, xe == gt_to_xe(far_gt));
412
413 g2g_bo = near_guc->g2g.bo;
414 xe_assert(xe, g2g_bo);
415
416 slot = g2g_slot(near_tile, near_dev, far_tile, far_dev, type, max, have_dev);
417 xe_assert(xe, slot >= 0);
418
419 base = guc_bo_ggtt_addr(near_guc, g2g_bo);
420 desc = base + slot * G2G_DESC_SIZE;
421 buf = base + G2G_DESC_AREA_SIZE + slot * G2G_BUFFER_SIZE;
422
423 xe_assert(xe, (desc - base + G2G_DESC_SIZE) <= G2G_DESC_AREA_SIZE);
424 xe_assert(xe, (buf - base + G2G_BUFFER_SIZE) <= xe_bo_size(g2g_bo));
425
426 return guc_action_register_g2g_buffer(near_guc, type, far_tile, far_dev,
427 desc, buf, G2G_BUFFER_SIZE);
428 }
429
guc_g2g_deregister(struct xe_guc * guc,u32 far_tile,u32 far_dev,u32 type)430 static void guc_g2g_deregister(struct xe_guc *guc, u32 far_tile, u32 far_dev, u32 type)
431 {
432 guc_action_deregister_g2g_buffer(guc, type, far_tile, far_dev);
433 }
434
guc_g2g_size(struct xe_guc * guc)435 static u32 guc_g2g_size(struct xe_guc *guc)
436 {
437 struct xe_gt *gt = guc_to_gt(guc);
438 struct xe_device *xe = gt_to_xe(gt);
439 unsigned int count = xe->info.gt_count;
440 u32 num_channels = (count * (count - 1)) / 2;
441
442 xe_assert(xe, num_channels * XE_G2G_TYPE_LIMIT * G2G_DESC_SIZE <= G2G_DESC_AREA_SIZE);
443
444 return num_channels * XE_G2G_TYPE_LIMIT * G2G_BUFFER_SIZE + G2G_DESC_AREA_SIZE;
445 }
446
xe_guc_g2g_wanted(struct xe_device * xe)447 static bool xe_guc_g2g_wanted(struct xe_device *xe)
448 {
449 /* Can't do GuC to GuC communication if there is only one GuC */
450 if (xe->info.gt_count <= 1)
451 return false;
452
453 /* No current user */
454 return false;
455 }
456
guc_g2g_alloc(struct xe_guc * guc)457 static int guc_g2g_alloc(struct xe_guc *guc)
458 {
459 struct xe_gt *gt = guc_to_gt(guc);
460 struct xe_device *xe = gt_to_xe(gt);
461 struct xe_tile *tile = gt_to_tile(gt);
462 struct xe_bo *bo;
463 u32 g2g_size;
464
465 if (guc->g2g.bo)
466 return 0;
467
468 if (gt->info.id != 0) {
469 struct xe_gt *root_gt = xe_device_get_gt(xe, 0);
470 struct xe_guc *root_guc = &root_gt->uc.guc;
471 struct xe_bo *bo;
472
473 bo = xe_bo_get(root_guc->g2g.bo);
474 if (!bo)
475 return -ENODEV;
476
477 guc->g2g.bo = bo;
478 guc->g2g.owned = false;
479 return 0;
480 }
481
482 g2g_size = guc_g2g_size(guc);
483 bo = xe_managed_bo_create_pin_map(xe, tile, g2g_size,
484 XE_BO_FLAG_VRAM_IF_DGFX(tile) |
485 XE_BO_FLAG_GGTT |
486 XE_BO_FLAG_GGTT_ALL |
487 XE_BO_FLAG_GGTT_INVALIDATE |
488 XE_BO_FLAG_PINNED_NORESTORE);
489 if (IS_ERR(bo))
490 return PTR_ERR(bo);
491
492 xe_map_memset(xe, &bo->vmap, 0, 0, g2g_size);
493 guc->g2g.bo = bo;
494 guc->g2g.owned = true;
495
496 return 0;
497 }
498
guc_g2g_fini(struct xe_guc * guc)499 static void guc_g2g_fini(struct xe_guc *guc)
500 {
501 if (!guc->g2g.bo)
502 return;
503
504 /* Unpinning the owned object is handled by generic shutdown */
505 if (!guc->g2g.owned)
506 xe_bo_put(guc->g2g.bo);
507
508 guc->g2g.bo = NULL;
509 }
510
guc_g2g_start(struct xe_guc * guc)511 static int guc_g2g_start(struct xe_guc *guc)
512 {
513 struct xe_gt *far_gt, *gt = guc_to_gt(guc);
514 struct xe_device *xe = gt_to_xe(gt);
515 unsigned int i, j;
516 int t, err;
517 bool have_dev;
518
519 if (!guc->g2g.bo) {
520 int ret;
521
522 ret = guc_g2g_alloc(guc);
523 if (ret)
524 return ret;
525 }
526
527 /* GuC interface will need extending if more GT device types are ever created. */
528 xe_gt_assert(gt, (gt->info.type == XE_GT_TYPE_MAIN) || (gt->info.type == XE_GT_TYPE_MEDIA));
529
530 /* Channel numbering depends on whether there are multiple GTs per tile */
531 have_dev = xe->info.gt_count > xe->info.tile_count;
532
533 for_each_gt(far_gt, xe, i) {
534 u32 far_tile, far_dev;
535
536 if (far_gt->info.id == gt->info.id)
537 continue;
538
539 far_tile = gt_to_tile(far_gt)->id;
540 far_dev = G2G_DEV(far_gt);
541
542 for (t = 0; t < XE_G2G_TYPE_LIMIT; t++) {
543 err = guc_g2g_register(guc, far_gt, t, have_dev);
544 if (err) {
545 while (--t >= 0)
546 guc_g2g_deregister(guc, far_tile, far_dev, t);
547 goto err_deregister;
548 }
549 }
550 }
551
552 return 0;
553
554 err_deregister:
555 for_each_gt(far_gt, xe, j) {
556 u32 tile, dev;
557
558 if (far_gt->info.id == gt->info.id)
559 continue;
560
561 if (j >= i)
562 break;
563
564 tile = gt_to_tile(far_gt)->id;
565 dev = G2G_DEV(far_gt);
566
567 for (t = 0; t < XE_G2G_TYPE_LIMIT; t++)
568 guc_g2g_deregister(guc, tile, dev, t);
569 }
570
571 return err;
572 }
573
__guc_opt_in_features_enable(struct xe_guc * guc,u64 addr,u32 num_dwords)574 static int __guc_opt_in_features_enable(struct xe_guc *guc, u64 addr, u32 num_dwords)
575 {
576 u32 action[] = {
577 XE_GUC_ACTION_OPT_IN_FEATURE_KLV,
578 lower_32_bits(addr),
579 upper_32_bits(addr),
580 num_dwords
581 };
582
583 return xe_guc_ct_send_block(&guc->ct, action, ARRAY_SIZE(action));
584 }
585
supports_dynamic_ics(struct xe_guc * guc)586 static bool supports_dynamic_ics(struct xe_guc *guc)
587 {
588 struct xe_device *xe = guc_to_xe(guc);
589 struct xe_gt *gt = guc_to_gt(guc);
590
591 /* Dynamic ICS is available for PVC and Xe2 and newer platforms. */
592 if (xe->info.platform != XE_PVC && GRAPHICS_VER(xe) < 20)
593 return false;
594
595 /*
596 * The feature is currently not compatible with multi-lrc, so the GuC
597 * does not support it at all on the media engines (which are the main
598 * users of mlrc). On the primary GT side, to avoid it being used in
599 * conjunction with mlrc, we only enable it if we are in single CCS
600 * mode.
601 */
602 if (xe_gt_is_media_type(gt) || gt->ccs_mode > 1)
603 return false;
604
605 /*
606 * Dynamic ICS requires GuC v70.40.1, which maps to compatibility
607 * version v1.18.4.
608 */
609 return GUC_SUBMIT_VER(guc) >= MAKE_GUC_VER(1, 18, 4);
610 }
611
612 #define OPT_IN_MAX_DWORDS 16
xe_guc_opt_in_features_enable(struct xe_guc * guc)613 int xe_guc_opt_in_features_enable(struct xe_guc *guc)
614 {
615 struct xe_device *xe = guc_to_xe(guc);
616 CLASS(xe_guc_buf, buf)(&guc->buf, OPT_IN_MAX_DWORDS);
617 u32 count = 0;
618 u32 *klvs;
619 int ret;
620
621 if (!xe_guc_buf_is_valid(buf))
622 return -ENOBUFS;
623
624 klvs = xe_guc_buf_cpu_ptr(buf);
625
626 /*
627 * The extra CAT error type opt-in was added in GuC v70.17.0, which maps
628 * to compatibility version v1.7.0.
629 * Note that the GuC allows enabling this KLV even on platforms that do
630 * not support the extra type; in such case the returned type variable
631 * will be set to a known invalid value which we can check against.
632 */
633 if (GUC_SUBMIT_VER(guc) >= MAKE_GUC_VER(1, 7, 0))
634 klvs[count++] = PREP_GUC_KLV_TAG(OPT_IN_FEATURE_EXT_CAT_ERR_TYPE);
635
636 if (supports_dynamic_ics(guc))
637 klvs[count++] = PREP_GUC_KLV_TAG(OPT_IN_FEATURE_DYNAMIC_INHIBIT_CONTEXT_SWITCH);
638
639 if (count) {
640 xe_assert(xe, count <= OPT_IN_MAX_DWORDS);
641
642 ret = __guc_opt_in_features_enable(guc, xe_guc_buf_flush(buf), count);
643 if (ret < 0) {
644 xe_gt_err(guc_to_gt(guc),
645 "failed to enable GuC opt-in features: %pe\n",
646 ERR_PTR(ret));
647 return ret;
648 }
649 }
650
651 return 0;
652 }
653
guc_fini_hw(void * arg)654 static void guc_fini_hw(void *arg)
655 {
656 struct xe_guc *guc = arg;
657 struct xe_gt *gt = guc_to_gt(guc);
658 unsigned int fw_ref;
659
660 fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL);
661 xe_uc_sanitize_reset(&guc_to_gt(guc)->uc);
662 xe_force_wake_put(gt_to_fw(gt), fw_ref);
663
664 guc_g2g_fini(guc);
665 }
666
667 /**
668 * xe_guc_comm_init_early - early initialization of GuC communication
669 * @guc: the &xe_guc to initialize
670 *
671 * Must be called prior to first MMIO communication with GuC firmware.
672 */
xe_guc_comm_init_early(struct xe_guc * guc)673 void xe_guc_comm_init_early(struct xe_guc *guc)
674 {
675 struct xe_gt *gt = guc_to_gt(guc);
676
677 if (xe_gt_is_media_type(gt))
678 guc->notify_reg = MED_GUC_HOST_INTERRUPT;
679 else
680 guc->notify_reg = GUC_HOST_INTERRUPT;
681 }
682
xe_guc_realloc_post_hwconfig(struct xe_guc * guc)683 static int xe_guc_realloc_post_hwconfig(struct xe_guc *guc)
684 {
685 struct xe_tile *tile = gt_to_tile(guc_to_gt(guc));
686 struct xe_device *xe = guc_to_xe(guc);
687 int ret;
688
689 if (!IS_DGFX(guc_to_xe(guc)))
690 return 0;
691
692 ret = xe_managed_bo_reinit_in_vram(xe, tile, &guc->fw.bo);
693 if (ret)
694 return ret;
695
696 ret = xe_managed_bo_reinit_in_vram(xe, tile, &guc->log.bo);
697 if (ret)
698 return ret;
699
700 ret = xe_managed_bo_reinit_in_vram(xe, tile, &guc->ads.bo);
701 if (ret)
702 return ret;
703
704 ret = xe_managed_bo_reinit_in_vram(xe, tile, &guc->ct.bo);
705 if (ret)
706 return ret;
707
708 return 0;
709 }
710
vf_guc_init_noalloc(struct xe_guc * guc)711 static int vf_guc_init_noalloc(struct xe_guc *guc)
712 {
713 struct xe_gt *gt = guc_to_gt(guc);
714 int err;
715
716 err = xe_gt_sriov_vf_bootstrap(gt);
717 if (err)
718 return err;
719
720 err = xe_gt_sriov_vf_query_config(gt);
721 if (err)
722 return err;
723
724 return 0;
725 }
726
xe_guc_init_noalloc(struct xe_guc * guc)727 int xe_guc_init_noalloc(struct xe_guc *guc)
728 {
729 struct xe_device *xe = guc_to_xe(guc);
730 struct xe_gt *gt = guc_to_gt(guc);
731 int ret;
732
733 xe_guc_comm_init_early(guc);
734
735 ret = xe_guc_ct_init_noalloc(&guc->ct);
736 if (ret)
737 goto out;
738
739 ret = xe_guc_relay_init(&guc->relay);
740 if (ret)
741 goto out;
742
743 if (IS_SRIOV_VF(xe)) {
744 ret = vf_guc_init_noalloc(guc);
745 if (ret)
746 goto out;
747 }
748
749 return 0;
750
751 out:
752 xe_gt_err(gt, "GuC init failed with %pe\n", ERR_PTR(ret));
753 return ret;
754 }
755
xe_guc_init(struct xe_guc * guc)756 int xe_guc_init(struct xe_guc *guc)
757 {
758 struct xe_device *xe = guc_to_xe(guc);
759 struct xe_gt *gt = guc_to_gt(guc);
760 int ret;
761
762 guc->fw.type = XE_UC_FW_TYPE_GUC;
763 ret = xe_uc_fw_init(&guc->fw);
764 if (ret)
765 return ret;
766
767 if (!xe_uc_fw_is_enabled(&guc->fw))
768 return 0;
769
770 if (IS_SRIOV_VF(xe)) {
771 ret = xe_guc_ct_init(&guc->ct);
772 if (ret)
773 goto out;
774 return 0;
775 }
776
777 ret = xe_guc_log_init(&guc->log);
778 if (ret)
779 goto out;
780
781 ret = xe_guc_capture_init(guc);
782 if (ret)
783 goto out;
784
785 ret = xe_guc_ads_init(&guc->ads);
786 if (ret)
787 goto out;
788
789 ret = xe_guc_ct_init(&guc->ct);
790 if (ret)
791 goto out;
792
793 xe_uc_fw_change_status(&guc->fw, XE_UC_FIRMWARE_LOADABLE);
794
795 ret = devm_add_action_or_reset(xe->drm.dev, guc_fini_hw, guc);
796 if (ret)
797 goto out;
798
799 guc_init_params(guc);
800
801 return 0;
802
803 out:
804 xe_gt_err(gt, "GuC init failed with %pe\n", ERR_PTR(ret));
805 return ret;
806 }
807
vf_guc_init_post_hwconfig(struct xe_guc * guc)808 static int vf_guc_init_post_hwconfig(struct xe_guc *guc)
809 {
810 int err;
811
812 err = xe_guc_submit_init(guc, xe_gt_sriov_vf_guc_ids(guc_to_gt(guc)));
813 if (err)
814 return err;
815
816 err = xe_guc_buf_cache_init(&guc->buf);
817 if (err)
818 return err;
819
820 /* XXX xe_guc_db_mgr_init not needed for now */
821
822 return 0;
823 }
824
825 /**
826 * xe_guc_init_post_hwconfig - initialize GuC post hwconfig load
827 * @guc: The GuC object
828 *
829 * Return: 0 on success, negative error code on error.
830 */
xe_guc_init_post_hwconfig(struct xe_guc * guc)831 int xe_guc_init_post_hwconfig(struct xe_guc *guc)
832 {
833 int ret;
834
835 if (IS_SRIOV_VF(guc_to_xe(guc)))
836 return vf_guc_init_post_hwconfig(guc);
837
838 ret = xe_guc_realloc_post_hwconfig(guc);
839 if (ret)
840 return ret;
841
842 guc_init_params_post_hwconfig(guc);
843
844 ret = xe_guc_submit_init(guc, ~0);
845 if (ret)
846 return ret;
847
848 ret = xe_guc_db_mgr_init(&guc->dbm, ~0);
849 if (ret)
850 return ret;
851
852 ret = xe_guc_pc_init(&guc->pc);
853 if (ret)
854 return ret;
855
856 ret = xe_guc_engine_activity_init(guc);
857 if (ret)
858 return ret;
859
860 ret = xe_guc_buf_cache_init(&guc->buf);
861 if (ret)
862 return ret;
863
864 return xe_guc_ads_init_post_hwconfig(&guc->ads);
865 }
866
xe_guc_post_load_init(struct xe_guc * guc)867 int xe_guc_post_load_init(struct xe_guc *guc)
868 {
869 int ret;
870
871 xe_guc_ads_populate_post_load(&guc->ads);
872
873 ret = xe_guc_opt_in_features_enable(guc);
874 if (ret)
875 return ret;
876
877 if (xe_guc_g2g_wanted(guc_to_xe(guc))) {
878 ret = guc_g2g_start(guc);
879 if (ret)
880 return ret;
881 }
882
883 guc->submission_state.enabled = true;
884
885 return 0;
886 }
887
xe_guc_reset(struct xe_guc * guc)888 int xe_guc_reset(struct xe_guc *guc)
889 {
890 struct xe_gt *gt = guc_to_gt(guc);
891 struct xe_mmio *mmio = >->mmio;
892 u32 guc_status, gdrst;
893 int ret;
894
895 xe_force_wake_assert_held(gt_to_fw(gt), XE_FW_GT);
896
897 if (IS_SRIOV_VF(gt_to_xe(gt)))
898 return xe_gt_sriov_vf_bootstrap(gt);
899
900 xe_mmio_write32(mmio, GDRST, GRDOM_GUC);
901
902 ret = xe_mmio_wait32(mmio, GDRST, GRDOM_GUC, 0, 5000, &gdrst, false);
903 if (ret) {
904 xe_gt_err(gt, "GuC reset timed out, GDRST=%#x\n", gdrst);
905 goto err_out;
906 }
907
908 guc_status = xe_mmio_read32(mmio, GUC_STATUS);
909 if (!(guc_status & GS_MIA_IN_RESET)) {
910 xe_gt_err(gt, "GuC status: %#x, MIA core expected to be in reset\n",
911 guc_status);
912 ret = -EIO;
913 goto err_out;
914 }
915
916 return 0;
917
918 err_out:
919
920 return ret;
921 }
922
guc_prepare_xfer(struct xe_guc * guc)923 static void guc_prepare_xfer(struct xe_guc *guc)
924 {
925 struct xe_gt *gt = guc_to_gt(guc);
926 struct xe_mmio *mmio = >->mmio;
927 struct xe_device *xe = guc_to_xe(guc);
928 u32 shim_flags = GUC_ENABLE_READ_CACHE_LOGIC |
929 GUC_ENABLE_READ_CACHE_FOR_SRAM_DATA |
930 GUC_ENABLE_READ_CACHE_FOR_WOPCM_DATA |
931 GUC_ENABLE_MIA_CLOCK_GATING;
932
933 if (GRAPHICS_VERx100(xe) < 1250)
934 shim_flags |= GUC_DISABLE_SRAM_INIT_TO_ZEROES |
935 GUC_ENABLE_MIA_CACHING;
936
937 if (GRAPHICS_VER(xe) >= 20 || xe->info.platform == XE_PVC)
938 shim_flags |= REG_FIELD_PREP(GUC_MOCS_INDEX_MASK, gt->mocs.uc_index);
939
940 /* Must program this register before loading the ucode with DMA */
941 xe_mmio_write32(mmio, GUC_SHIM_CONTROL, shim_flags);
942
943 xe_mmio_write32(mmio, GT_PM_CONFIG, GT_DOORBELL_ENABLE);
944
945 /* Make sure GuC receives ARAT interrupts */
946 xe_mmio_rmw32(mmio, PMINTRMSK, ARAT_EXPIRED_INTRMSK, 0);
947 }
948
949 /*
950 * Supporting MMIO & in memory RSA
951 */
guc_xfer_rsa(struct xe_guc * guc)952 static int guc_xfer_rsa(struct xe_guc *guc)
953 {
954 struct xe_gt *gt = guc_to_gt(guc);
955 u32 rsa[UOS_RSA_SCRATCH_COUNT];
956 size_t copied;
957 int i;
958
959 if (guc->fw.rsa_size > 256) {
960 u32 rsa_ggtt_addr = xe_bo_ggtt_addr(guc->fw.bo) +
961 xe_uc_fw_rsa_offset(&guc->fw);
962 xe_mmio_write32(>->mmio, UOS_RSA_SCRATCH(0), rsa_ggtt_addr);
963 return 0;
964 }
965
966 copied = xe_uc_fw_copy_rsa(&guc->fw, rsa, sizeof(rsa));
967 if (copied < sizeof(rsa))
968 return -ENOMEM;
969
970 for (i = 0; i < UOS_RSA_SCRATCH_COUNT; i++)
971 xe_mmio_write32(>->mmio, UOS_RSA_SCRATCH(i), rsa[i]);
972
973 return 0;
974 }
975
976 /*
977 * Check a previously read GuC status register (GUC_STATUS) looking for
978 * known terminal states (either completion or failure) of either the
979 * microkernel status field or the boot ROM status field. Returns +1 for
980 * successful completion, -1 for failure and 0 for any intermediate state.
981 */
guc_load_done(u32 status)982 static int guc_load_done(u32 status)
983 {
984 u32 uk_val = REG_FIELD_GET(GS_UKERNEL_MASK, status);
985 u32 br_val = REG_FIELD_GET(GS_BOOTROM_MASK, status);
986
987 switch (uk_val) {
988 case XE_GUC_LOAD_STATUS_READY:
989 return 1;
990
991 case XE_GUC_LOAD_STATUS_ERROR_DEVID_BUILD_MISMATCH:
992 case XE_GUC_LOAD_STATUS_GUC_PREPROD_BUILD_MISMATCH:
993 case XE_GUC_LOAD_STATUS_ERROR_DEVID_INVALID_GUCTYPE:
994 case XE_GUC_LOAD_STATUS_HWCONFIG_ERROR:
995 case XE_GUC_LOAD_STATUS_DPC_ERROR:
996 case XE_GUC_LOAD_STATUS_EXCEPTION:
997 case XE_GUC_LOAD_STATUS_INIT_DATA_INVALID:
998 case XE_GUC_LOAD_STATUS_MPU_DATA_INVALID:
999 case XE_GUC_LOAD_STATUS_INIT_MMIO_SAVE_RESTORE_INVALID:
1000 return -1;
1001 }
1002
1003 switch (br_val) {
1004 case XE_BOOTROM_STATUS_NO_KEY_FOUND:
1005 case XE_BOOTROM_STATUS_RSA_FAILED:
1006 case XE_BOOTROM_STATUS_PAVPC_FAILED:
1007 case XE_BOOTROM_STATUS_WOPCM_FAILED:
1008 case XE_BOOTROM_STATUS_LOADLOC_FAILED:
1009 case XE_BOOTROM_STATUS_JUMP_FAILED:
1010 case XE_BOOTROM_STATUS_RC6CTXCONFIG_FAILED:
1011 case XE_BOOTROM_STATUS_MPUMAP_INCORRECT:
1012 case XE_BOOTROM_STATUS_EXCEPTION:
1013 case XE_BOOTROM_STATUS_PROD_KEY_CHECK_FAILURE:
1014 return -1;
1015 }
1016
1017 return 0;
1018 }
1019
guc_pc_get_cur_freq(struct xe_guc_pc * guc_pc)1020 static s32 guc_pc_get_cur_freq(struct xe_guc_pc *guc_pc)
1021 {
1022 u32 freq;
1023 int ret = xe_guc_pc_get_cur_freq(guc_pc, &freq);
1024
1025 return ret ? ret : freq;
1026 }
1027
1028 /*
1029 * Wait for the GuC to start up.
1030 *
1031 * Measurements indicate this should take no more than 20ms (assuming the GT
1032 * clock is at maximum frequency). However, thermal throttling and other issues
1033 * can prevent the clock hitting max and thus making the load take significantly
1034 * longer. Allow up to 200ms as a safety margin for real world worst case situations.
1035 *
1036 * However, bugs anywhere from KMD to GuC to PCODE to fan failure in a CI farm can
1037 * lead to even longer times. E.g. if the GT is clamped to minimum frequency then
1038 * the load times can be in the seconds range. So the timeout is increased for debug
1039 * builds to ensure that problems can be correctly analysed. For release builds, the
1040 * timeout is kept short so that users don't wait forever to find out that there is a
1041 * problem. In either case, if the load took longer than is reasonable even with some
1042 * 'sensible' throttling, then flag a warning because something is not right.
1043 *
1044 * Note that there is a limit on how long an individual usleep_range() can wait for,
1045 * hence longer waits require wrapping a shorter wait in a loop.
1046 *
1047 * Note that the only reason an end user should hit the shorter timeout is in case of
1048 * extreme thermal throttling. And a system that is that hot during boot is probably
1049 * dead anyway!
1050 */
1051 #if IS_ENABLED(CONFIG_DRM_XE_DEBUG)
1052 #define GUC_LOAD_RETRY_LIMIT 20
1053 #else
1054 #define GUC_LOAD_RETRY_LIMIT 3
1055 #endif
1056 #define GUC_LOAD_TIME_WARN_MS 200
1057
guc_wait_ucode(struct xe_guc * guc)1058 static void guc_wait_ucode(struct xe_guc *guc)
1059 {
1060 struct xe_gt *gt = guc_to_gt(guc);
1061 struct xe_mmio *mmio = >->mmio;
1062 struct xe_guc_pc *guc_pc = >->uc.guc.pc;
1063 ktime_t before, after, delta;
1064 int load_done;
1065 u32 status = 0;
1066 int count = 0;
1067 u64 delta_ms;
1068 u32 before_freq;
1069
1070 before_freq = xe_guc_pc_get_act_freq(guc_pc);
1071 before = ktime_get();
1072 /*
1073 * Note, can't use any kind of timing information from the call to xe_mmio_wait.
1074 * It could return a thousand intermediate stages at random times. Instead, must
1075 * manually track the total time taken and locally implement the timeout.
1076 */
1077 do {
1078 u32 last_status = status & (GS_UKERNEL_MASK | GS_BOOTROM_MASK);
1079 int ret;
1080
1081 /*
1082 * Wait for any change (intermediate or terminal) in the status register.
1083 * Note, the return value is a don't care. The only failure code is timeout
1084 * but the timeouts need to be accumulated over all the intermediate partial
1085 * timeouts rather than allowing a huge timeout each time. So basically, need
1086 * to treat a timeout no different to a value change.
1087 */
1088 ret = xe_mmio_wait32_not(mmio, GUC_STATUS, GS_UKERNEL_MASK | GS_BOOTROM_MASK,
1089 last_status, 1000 * 1000, &status, false);
1090 if (ret < 0)
1091 count++;
1092 after = ktime_get();
1093 delta = ktime_sub(after, before);
1094 delta_ms = ktime_to_ms(delta);
1095
1096 load_done = guc_load_done(status);
1097 if (load_done != 0)
1098 break;
1099
1100 if (delta_ms >= (GUC_LOAD_RETRY_LIMIT * 1000))
1101 break;
1102
1103 xe_gt_dbg(gt, "load still in progress, timeouts = %d, freq = %dMHz (req %dMHz), status = 0x%08X [0x%02X/%02X]\n",
1104 count, xe_guc_pc_get_act_freq(guc_pc),
1105 guc_pc_get_cur_freq(guc_pc), status,
1106 REG_FIELD_GET(GS_BOOTROM_MASK, status),
1107 REG_FIELD_GET(GS_UKERNEL_MASK, status));
1108 } while (1);
1109
1110 if (load_done != 1) {
1111 u32 ukernel = REG_FIELD_GET(GS_UKERNEL_MASK, status);
1112 u32 bootrom = REG_FIELD_GET(GS_BOOTROM_MASK, status);
1113
1114 xe_gt_err(gt, "load failed: status = 0x%08X, time = %lldms, freq = %dMHz (req %dMHz), done = %d\n",
1115 status, delta_ms, xe_guc_pc_get_act_freq(guc_pc),
1116 guc_pc_get_cur_freq(guc_pc), load_done);
1117 xe_gt_err(gt, "load failed: status: Reset = %d, BootROM = 0x%02X, UKernel = 0x%02X, MIA = 0x%02X, Auth = 0x%02X\n",
1118 REG_FIELD_GET(GS_MIA_IN_RESET, status),
1119 bootrom, ukernel,
1120 REG_FIELD_GET(GS_MIA_MASK, status),
1121 REG_FIELD_GET(GS_AUTH_STATUS_MASK, status));
1122
1123 switch (bootrom) {
1124 case XE_BOOTROM_STATUS_NO_KEY_FOUND:
1125 xe_gt_err(gt, "invalid key requested, header = 0x%08X\n",
1126 xe_mmio_read32(mmio, GUC_HEADER_INFO));
1127 break;
1128
1129 case XE_BOOTROM_STATUS_RSA_FAILED:
1130 xe_gt_err(gt, "firmware signature verification failed\n");
1131 break;
1132
1133 case XE_BOOTROM_STATUS_PROD_KEY_CHECK_FAILURE:
1134 xe_gt_err(gt, "firmware production part check failure\n");
1135 break;
1136 }
1137
1138 switch (ukernel) {
1139 case XE_GUC_LOAD_STATUS_EXCEPTION:
1140 xe_gt_err(gt, "firmware exception. EIP: %#x\n",
1141 xe_mmio_read32(mmio, SOFT_SCRATCH(13)));
1142 break;
1143
1144 case XE_GUC_LOAD_STATUS_INIT_MMIO_SAVE_RESTORE_INVALID:
1145 xe_gt_err(gt, "illegal register in save/restore workaround list\n");
1146 break;
1147
1148 case XE_GUC_LOAD_STATUS_HWCONFIG_START:
1149 xe_gt_err(gt, "still extracting hwconfig table.\n");
1150 break;
1151 }
1152
1153 xe_device_declare_wedged(gt_to_xe(gt));
1154 } else if (delta_ms > GUC_LOAD_TIME_WARN_MS) {
1155 xe_gt_warn(gt, "excessive init time: %lldms! [status = 0x%08X, timeouts = %d]\n",
1156 delta_ms, status, count);
1157 xe_gt_warn(gt, "excessive init time: [freq = %dMHz (req = %dMHz), before = %dMHz, perf_limit_reasons = 0x%08X]\n",
1158 xe_guc_pc_get_act_freq(guc_pc), guc_pc_get_cur_freq(guc_pc),
1159 before_freq, xe_gt_throttle_get_limit_reasons(gt));
1160 } else {
1161 xe_gt_dbg(gt, "init took %lldms, freq = %dMHz (req = %dMHz), before = %dMHz, status = 0x%08X, timeouts = %d\n",
1162 delta_ms, xe_guc_pc_get_act_freq(guc_pc), guc_pc_get_cur_freq(guc_pc),
1163 before_freq, status, count);
1164 }
1165 }
1166
__xe_guc_upload(struct xe_guc * guc)1167 static int __xe_guc_upload(struct xe_guc *guc)
1168 {
1169 int ret;
1170
1171 /* Raise GT freq to speed up HuC/GuC load */
1172 xe_guc_pc_raise_unslice(&guc->pc);
1173
1174 guc_write_params(guc);
1175 guc_prepare_xfer(guc);
1176
1177 /*
1178 * Note that GuC needs the CSS header plus uKernel code to be copied
1179 * by the DMA engine in one operation, whereas the RSA signature is
1180 * loaded separately, either by copying it to the UOS_RSA_SCRATCH
1181 * register (if key size <= 256) or through a ggtt-pinned vma (if key
1182 * size > 256). The RSA size and therefore the way we provide it to the
1183 * HW is fixed for each platform and hard-coded in the bootrom.
1184 */
1185 ret = guc_xfer_rsa(guc);
1186 if (ret)
1187 goto out;
1188 /*
1189 * Current uCode expects the code to be loaded at 8k; locations below
1190 * this are used for the stack.
1191 */
1192 ret = xe_uc_fw_upload(&guc->fw, 0x2000, UOS_MOVE);
1193 if (ret)
1194 goto out;
1195
1196 /* Wait for authentication */
1197 guc_wait_ucode(guc);
1198
1199 xe_uc_fw_change_status(&guc->fw, XE_UC_FIRMWARE_RUNNING);
1200 return 0;
1201
1202 out:
1203 xe_uc_fw_change_status(&guc->fw, XE_UC_FIRMWARE_LOAD_FAIL);
1204 return 0 /* FIXME: ret, don't want to stop load currently */;
1205 }
1206
vf_guc_min_load_for_hwconfig(struct xe_guc * guc)1207 static int vf_guc_min_load_for_hwconfig(struct xe_guc *guc)
1208 {
1209 struct xe_gt *gt = guc_to_gt(guc);
1210 int ret;
1211
1212 ret = xe_guc_hwconfig_init(guc);
1213 if (ret)
1214 return ret;
1215
1216 ret = xe_guc_enable_communication(guc);
1217 if (ret)
1218 return ret;
1219
1220 ret = xe_gt_sriov_vf_connect(gt);
1221 if (ret)
1222 goto err_out;
1223
1224 ret = xe_gt_sriov_vf_query_runtime(gt);
1225 if (ret)
1226 goto err_out;
1227
1228 return 0;
1229
1230 err_out:
1231 xe_guc_sanitize(guc);
1232 return ret;
1233 }
1234
1235 /**
1236 * xe_guc_min_load_for_hwconfig - load minimal GuC and read hwconfig table
1237 * @guc: The GuC object
1238 *
1239 * This function uploads a minimal GuC that does not support submissions but
1240 * in a state where the hwconfig table can be read. Next, it reads and parses
1241 * the hwconfig table so it can be used for subsequent steps in the driver load.
1242 * Lastly, it enables CT communication (XXX: this is needed for PFs/VFs only).
1243 *
1244 * Return: 0 on success, negative error code on error.
1245 */
xe_guc_min_load_for_hwconfig(struct xe_guc * guc)1246 int xe_guc_min_load_for_hwconfig(struct xe_guc *guc)
1247 {
1248 int ret;
1249
1250 if (IS_SRIOV_VF(guc_to_xe(guc)))
1251 return vf_guc_min_load_for_hwconfig(guc);
1252
1253 xe_guc_ads_populate_minimal(&guc->ads);
1254
1255 xe_guc_pc_init_early(&guc->pc);
1256
1257 ret = __xe_guc_upload(guc);
1258 if (ret)
1259 return ret;
1260
1261 ret = xe_guc_hwconfig_init(guc);
1262 if (ret)
1263 return ret;
1264
1265 ret = xe_guc_enable_communication(guc);
1266 if (ret)
1267 return ret;
1268
1269 return 0;
1270 }
1271
xe_guc_upload(struct xe_guc * guc)1272 int xe_guc_upload(struct xe_guc *guc)
1273 {
1274 xe_guc_ads_populate(&guc->ads);
1275
1276 return __xe_guc_upload(guc);
1277 }
1278
guc_handle_mmio_msg(struct xe_guc * guc)1279 static void guc_handle_mmio_msg(struct xe_guc *guc)
1280 {
1281 struct xe_gt *gt = guc_to_gt(guc);
1282 u32 msg;
1283
1284 if (IS_SRIOV_VF(guc_to_xe(guc)))
1285 return;
1286
1287 xe_force_wake_assert_held(gt_to_fw(gt), XE_FW_GT);
1288
1289 msg = xe_mmio_read32(>->mmio, SOFT_SCRATCH(15));
1290 msg &= XE_GUC_RECV_MSG_EXCEPTION |
1291 XE_GUC_RECV_MSG_CRASH_DUMP_POSTED;
1292 xe_mmio_write32(>->mmio, SOFT_SCRATCH(15), 0);
1293
1294 if (msg & XE_GUC_RECV_MSG_CRASH_DUMP_POSTED)
1295 xe_gt_err(gt, "Received early GuC crash dump notification!\n");
1296
1297 if (msg & XE_GUC_RECV_MSG_EXCEPTION)
1298 xe_gt_err(gt, "Received early GuC exception notification!\n");
1299 }
1300
guc_enable_irq(struct xe_guc * guc)1301 static void guc_enable_irq(struct xe_guc *guc)
1302 {
1303 struct xe_gt *gt = guc_to_gt(guc);
1304 u32 events = xe_gt_is_media_type(gt) ?
1305 REG_FIELD_PREP(ENGINE0_MASK, GUC_INTR_GUC2HOST) :
1306 REG_FIELD_PREP(ENGINE1_MASK, GUC_INTR_GUC2HOST);
1307
1308 /* Primary GuC and media GuC share a single enable bit */
1309 xe_mmio_write32(>->mmio, GUC_SG_INTR_ENABLE,
1310 REG_FIELD_PREP(ENGINE1_MASK, GUC_INTR_GUC2HOST));
1311
1312 /*
1313 * There are separate mask bits for primary and media GuCs, so use
1314 * a RMW operation to avoid clobbering the other GuC's setting.
1315 */
1316 xe_mmio_rmw32(>->mmio, GUC_SG_INTR_MASK, events, 0);
1317 }
1318
xe_guc_enable_communication(struct xe_guc * guc)1319 int xe_guc_enable_communication(struct xe_guc *guc)
1320 {
1321 struct xe_device *xe = guc_to_xe(guc);
1322 int err;
1323
1324 if (IS_SRIOV_VF(xe) && xe_device_has_memirq(xe)) {
1325 struct xe_gt *gt = guc_to_gt(guc);
1326 struct xe_tile *tile = gt_to_tile(gt);
1327
1328 err = xe_memirq_init_guc(&tile->memirq, guc);
1329 if (err)
1330 return err;
1331 } else {
1332 guc_enable_irq(guc);
1333 }
1334
1335 err = xe_guc_ct_enable(&guc->ct);
1336 if (err)
1337 return err;
1338
1339 guc_handle_mmio_msg(guc);
1340
1341 return 0;
1342 }
1343
xe_guc_suspend(struct xe_guc * guc)1344 int xe_guc_suspend(struct xe_guc *guc)
1345 {
1346 struct xe_gt *gt = guc_to_gt(guc);
1347 u32 action[] = {
1348 XE_GUC_ACTION_CLIENT_SOFT_RESET,
1349 };
1350 int ret;
1351
1352 ret = xe_guc_mmio_send(guc, action, ARRAY_SIZE(action));
1353 if (ret) {
1354 xe_gt_err(gt, "GuC suspend failed: %pe\n", ERR_PTR(ret));
1355 return ret;
1356 }
1357
1358 xe_guc_sanitize(guc);
1359 return 0;
1360 }
1361
xe_guc_notify(struct xe_guc * guc)1362 void xe_guc_notify(struct xe_guc *guc)
1363 {
1364 struct xe_gt *gt = guc_to_gt(guc);
1365 const u32 default_notify_data = 0;
1366
1367 /*
1368 * Both GUC_HOST_INTERRUPT and MED_GUC_HOST_INTERRUPT can pass
1369 * additional payload data to the GuC but this capability is not
1370 * used by the firmware yet. Use default value in the meantime.
1371 */
1372 xe_mmio_write32(>->mmio, guc->notify_reg, default_notify_data);
1373 }
1374
xe_guc_auth_huc(struct xe_guc * guc,u32 rsa_addr)1375 int xe_guc_auth_huc(struct xe_guc *guc, u32 rsa_addr)
1376 {
1377 u32 action[] = {
1378 XE_GUC_ACTION_AUTHENTICATE_HUC,
1379 rsa_addr
1380 };
1381
1382 return xe_guc_ct_send_block(&guc->ct, action, ARRAY_SIZE(action));
1383 }
1384
xe_guc_mmio_send_recv(struct xe_guc * guc,const u32 * request,u32 len,u32 * response_buf)1385 int xe_guc_mmio_send_recv(struct xe_guc *guc, const u32 *request,
1386 u32 len, u32 *response_buf)
1387 {
1388 struct xe_device *xe = guc_to_xe(guc);
1389 struct xe_gt *gt = guc_to_gt(guc);
1390 struct xe_mmio *mmio = >->mmio;
1391 u32 header, reply;
1392 struct xe_reg reply_reg = xe_gt_is_media_type(gt) ?
1393 MED_VF_SW_FLAG(0) : VF_SW_FLAG(0);
1394 const u32 LAST_INDEX = VF_SW_FLAG_COUNT - 1;
1395 bool lost = false;
1396 int ret;
1397 int i;
1398
1399 BUILD_BUG_ON(VF_SW_FLAG_COUNT != MED_VF_SW_FLAG_COUNT);
1400
1401 xe_assert(xe, len);
1402 xe_assert(xe, len <= VF_SW_FLAG_COUNT);
1403 xe_assert(xe, len <= MED_VF_SW_FLAG_COUNT);
1404 xe_assert(xe, FIELD_GET(GUC_HXG_MSG_0_ORIGIN, request[0]) ==
1405 GUC_HXG_ORIGIN_HOST);
1406 xe_assert(xe, FIELD_GET(GUC_HXG_MSG_0_TYPE, request[0]) ==
1407 GUC_HXG_TYPE_REQUEST);
1408
1409 retry:
1410 /* Not in critical data-path, just do if else for GT type */
1411 if (xe_gt_is_media_type(gt)) {
1412 for (i = 0; i < len; ++i)
1413 xe_mmio_write32(mmio, MED_VF_SW_FLAG(i),
1414 request[i]);
1415 xe_mmio_read32(mmio, MED_VF_SW_FLAG(LAST_INDEX));
1416 } else {
1417 for (i = 0; i < len; ++i)
1418 xe_mmio_write32(mmio, VF_SW_FLAG(i),
1419 request[i]);
1420 xe_mmio_read32(mmio, VF_SW_FLAG(LAST_INDEX));
1421 }
1422
1423 xe_guc_notify(guc);
1424
1425 ret = xe_mmio_wait32(mmio, reply_reg, GUC_HXG_MSG_0_ORIGIN,
1426 FIELD_PREP(GUC_HXG_MSG_0_ORIGIN, GUC_HXG_ORIGIN_GUC),
1427 50000, &reply, false);
1428 if (ret) {
1429 /* scratch registers might be cleared during FLR, try once more */
1430 if (!reply && !lost) {
1431 xe_gt_dbg(gt, "GuC mmio request %#x: lost, trying again\n", request[0]);
1432 lost = true;
1433 goto retry;
1434 }
1435 timeout:
1436 xe_gt_err(gt, "GuC mmio request %#x: no reply %#x\n",
1437 request[0], reply);
1438 return ret;
1439 }
1440
1441 header = xe_mmio_read32(mmio, reply_reg);
1442 if (FIELD_GET(GUC_HXG_MSG_0_TYPE, header) ==
1443 GUC_HXG_TYPE_NO_RESPONSE_BUSY) {
1444 /*
1445 * Once we got a BUSY reply we must wait again for the final
1446 * response but this time we can't use ORIGIN mask anymore.
1447 * To spot a right change in the reply, we take advantage that
1448 * response SUCCESS and FAILURE differ only by the single bit
1449 * and all other bits are set and can be used as a new mask.
1450 */
1451 u32 resp_bits = GUC_HXG_TYPE_RESPONSE_SUCCESS & GUC_HXG_TYPE_RESPONSE_FAILURE;
1452 u32 resp_mask = FIELD_PREP(GUC_HXG_MSG_0_TYPE, resp_bits);
1453
1454 BUILD_BUG_ON(FIELD_MAX(GUC_HXG_MSG_0_TYPE) != GUC_HXG_TYPE_RESPONSE_SUCCESS);
1455 BUILD_BUG_ON((GUC_HXG_TYPE_RESPONSE_SUCCESS ^ GUC_HXG_TYPE_RESPONSE_FAILURE) != 1);
1456
1457 ret = xe_mmio_wait32(mmio, reply_reg, resp_mask, resp_mask,
1458 1000000, &header, false);
1459
1460 if (unlikely(FIELD_GET(GUC_HXG_MSG_0_ORIGIN, header) !=
1461 GUC_HXG_ORIGIN_GUC))
1462 goto proto;
1463 if (unlikely(ret)) {
1464 if (FIELD_GET(GUC_HXG_MSG_0_TYPE, header) !=
1465 GUC_HXG_TYPE_NO_RESPONSE_BUSY)
1466 goto proto;
1467 goto timeout;
1468 }
1469 }
1470
1471 if (FIELD_GET(GUC_HXG_MSG_0_TYPE, header) ==
1472 GUC_HXG_TYPE_NO_RESPONSE_RETRY) {
1473 u32 reason = FIELD_GET(GUC_HXG_RETRY_MSG_0_REASON, header);
1474
1475 xe_gt_dbg(gt, "GuC mmio request %#x: retrying, reason %#x\n",
1476 request[0], reason);
1477 goto retry;
1478 }
1479
1480 if (FIELD_GET(GUC_HXG_MSG_0_TYPE, header) ==
1481 GUC_HXG_TYPE_RESPONSE_FAILURE) {
1482 u32 hint = FIELD_GET(GUC_HXG_FAILURE_MSG_0_HINT, header);
1483 u32 error = FIELD_GET(GUC_HXG_FAILURE_MSG_0_ERROR, header);
1484
1485 xe_gt_err(gt, "GuC mmio request %#x: failure %#x hint %#x\n",
1486 request[0], error, hint);
1487 return -ENXIO;
1488 }
1489
1490 if (FIELD_GET(GUC_HXG_MSG_0_TYPE, header) !=
1491 GUC_HXG_TYPE_RESPONSE_SUCCESS) {
1492 proto:
1493 xe_gt_err(gt, "GuC mmio request %#x: unexpected reply %#x\n",
1494 request[0], header);
1495 return -EPROTO;
1496 }
1497
1498 /* Just copy entire possible message response */
1499 if (response_buf) {
1500 response_buf[0] = header;
1501
1502 for (i = 1; i < VF_SW_FLAG_COUNT; i++) {
1503 reply_reg.addr += sizeof(u32);
1504 response_buf[i] = xe_mmio_read32(mmio, reply_reg);
1505 }
1506 }
1507
1508 /* Use data from the GuC response as our return value */
1509 return FIELD_GET(GUC_HXG_RESPONSE_MSG_0_DATA0, header);
1510 }
1511 ALLOW_ERROR_INJECTION(xe_guc_mmio_send_recv, ERRNO);
1512
xe_guc_mmio_send(struct xe_guc * guc,const u32 * request,u32 len)1513 int xe_guc_mmio_send(struct xe_guc *guc, const u32 *request, u32 len)
1514 {
1515 return xe_guc_mmio_send_recv(guc, request, len, NULL);
1516 }
1517
guc_self_cfg(struct xe_guc * guc,u16 key,u16 len,u64 val)1518 static int guc_self_cfg(struct xe_guc *guc, u16 key, u16 len, u64 val)
1519 {
1520 struct xe_device *xe = guc_to_xe(guc);
1521 u32 request[HOST2GUC_SELF_CFG_REQUEST_MSG_LEN] = {
1522 FIELD_PREP(GUC_HXG_MSG_0_ORIGIN, GUC_HXG_ORIGIN_HOST) |
1523 FIELD_PREP(GUC_HXG_MSG_0_TYPE, GUC_HXG_TYPE_REQUEST) |
1524 FIELD_PREP(GUC_HXG_REQUEST_MSG_0_ACTION,
1525 GUC_ACTION_HOST2GUC_SELF_CFG),
1526 FIELD_PREP(HOST2GUC_SELF_CFG_REQUEST_MSG_1_KLV_KEY, key) |
1527 FIELD_PREP(HOST2GUC_SELF_CFG_REQUEST_MSG_1_KLV_LEN, len),
1528 FIELD_PREP(HOST2GUC_SELF_CFG_REQUEST_MSG_2_VALUE32,
1529 lower_32_bits(val)),
1530 FIELD_PREP(HOST2GUC_SELF_CFG_REQUEST_MSG_3_VALUE64,
1531 upper_32_bits(val)),
1532 };
1533 int ret;
1534
1535 xe_assert(xe, len <= 2);
1536 xe_assert(xe, len != 1 || !upper_32_bits(val));
1537
1538 /* Self config must go over MMIO */
1539 ret = xe_guc_mmio_send(guc, request, ARRAY_SIZE(request));
1540
1541 if (unlikely(ret < 0))
1542 return ret;
1543 if (unlikely(ret > 1))
1544 return -EPROTO;
1545 if (unlikely(!ret))
1546 return -ENOKEY;
1547
1548 return 0;
1549 }
1550
xe_guc_self_cfg32(struct xe_guc * guc,u16 key,u32 val)1551 int xe_guc_self_cfg32(struct xe_guc *guc, u16 key, u32 val)
1552 {
1553 return guc_self_cfg(guc, key, 1, val);
1554 }
1555
xe_guc_self_cfg64(struct xe_guc * guc,u16 key,u64 val)1556 int xe_guc_self_cfg64(struct xe_guc *guc, u16 key, u64 val)
1557 {
1558 return guc_self_cfg(guc, key, 2, val);
1559 }
1560
xe_guc_sw_0_irq_handler(struct xe_guc * guc)1561 static void xe_guc_sw_0_irq_handler(struct xe_guc *guc)
1562 {
1563 struct xe_gt *gt = guc_to_gt(guc);
1564
1565 if (IS_SRIOV_VF(gt_to_xe(gt)))
1566 xe_gt_sriov_vf_migrated_event_handler(gt);
1567 }
1568
xe_guc_irq_handler(struct xe_guc * guc,const u16 iir)1569 void xe_guc_irq_handler(struct xe_guc *guc, const u16 iir)
1570 {
1571 if (iir & GUC_INTR_GUC2HOST)
1572 xe_guc_ct_irq_handler(&guc->ct);
1573
1574 if (iir & GUC_INTR_SW_INT_0)
1575 xe_guc_sw_0_irq_handler(guc);
1576 }
1577
xe_guc_sanitize(struct xe_guc * guc)1578 void xe_guc_sanitize(struct xe_guc *guc)
1579 {
1580 xe_uc_fw_sanitize(&guc->fw);
1581 xe_guc_ct_disable(&guc->ct);
1582 guc->submission_state.enabled = false;
1583 }
1584
xe_guc_reset_prepare(struct xe_guc * guc)1585 int xe_guc_reset_prepare(struct xe_guc *guc)
1586 {
1587 return xe_guc_submit_reset_prepare(guc);
1588 }
1589
xe_guc_reset_wait(struct xe_guc * guc)1590 void xe_guc_reset_wait(struct xe_guc *guc)
1591 {
1592 xe_guc_submit_reset_wait(guc);
1593 }
1594
xe_guc_stop_prepare(struct xe_guc * guc)1595 void xe_guc_stop_prepare(struct xe_guc *guc)
1596 {
1597 if (!IS_SRIOV_VF(guc_to_xe(guc))) {
1598 int err;
1599
1600 err = xe_guc_pc_stop(&guc->pc);
1601 xe_gt_WARN(guc_to_gt(guc), err, "Failed to stop GuC PC: %pe\n",
1602 ERR_PTR(err));
1603 }
1604 }
1605
xe_guc_stop(struct xe_guc * guc)1606 void xe_guc_stop(struct xe_guc *guc)
1607 {
1608 xe_guc_ct_stop(&guc->ct);
1609
1610 xe_guc_submit_stop(guc);
1611 }
1612
xe_guc_start(struct xe_guc * guc)1613 int xe_guc_start(struct xe_guc *guc)
1614 {
1615 return xe_guc_submit_start(guc);
1616 }
1617
xe_guc_print_info(struct xe_guc * guc,struct drm_printer * p)1618 void xe_guc_print_info(struct xe_guc *guc, struct drm_printer *p)
1619 {
1620 struct xe_gt *gt = guc_to_gt(guc);
1621 unsigned int fw_ref;
1622 u32 status;
1623 int i;
1624
1625 xe_uc_fw_print(&guc->fw, p);
1626
1627 if (!IS_SRIOV_VF(gt_to_xe(gt))) {
1628 fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
1629 if (!fw_ref)
1630 return;
1631
1632 status = xe_mmio_read32(>->mmio, GUC_STATUS);
1633
1634 drm_printf(p, "\nGuC status 0x%08x:\n", status);
1635 drm_printf(p, "\tBootrom status = 0x%x\n",
1636 REG_FIELD_GET(GS_BOOTROM_MASK, status));
1637 drm_printf(p, "\tuKernel status = 0x%x\n",
1638 REG_FIELD_GET(GS_UKERNEL_MASK, status));
1639 drm_printf(p, "\tMIA Core status = 0x%x\n",
1640 REG_FIELD_GET(GS_MIA_MASK, status));
1641 drm_printf(p, "\tLog level = %d\n",
1642 xe_guc_log_get_level(&guc->log));
1643
1644 drm_puts(p, "\nScratch registers:\n");
1645 for (i = 0; i < SOFT_SCRATCH_COUNT; i++) {
1646 drm_printf(p, "\t%2d: \t0x%x\n",
1647 i, xe_mmio_read32(>->mmio, SOFT_SCRATCH(i)));
1648 }
1649
1650 xe_force_wake_put(gt_to_fw(gt), fw_ref);
1651 }
1652
1653 drm_puts(p, "\n");
1654 xe_guc_ct_print(&guc->ct, p, false);
1655
1656 drm_puts(p, "\n");
1657 xe_guc_submit_print(guc, p);
1658 }
1659
1660 /**
1661 * xe_guc_declare_wedged() - Declare GuC wedged
1662 * @guc: the GuC object
1663 *
1664 * Wedge the GuC which stops all submission, saves desired debug state, and
1665 * cleans up anything which could timeout.
1666 */
xe_guc_declare_wedged(struct xe_guc * guc)1667 void xe_guc_declare_wedged(struct xe_guc *guc)
1668 {
1669 xe_gt_assert(guc_to_gt(guc), guc_to_xe(guc)->wedged.mode);
1670
1671 xe_guc_reset_prepare(guc);
1672 xe_guc_ct_stop(&guc->ct);
1673 xe_guc_submit_wedge(guc);
1674 }
1675