1 // SPDX-License-Identifier: MIT
2 /*
3 * Copyright © 2022 Intel Corporation
4 */
5
6 #include "xe_gt.h"
7
8 #include <linux/minmax.h>
9
10 #include <drm/drm_managed.h>
11 #include <uapi/drm/xe_drm.h>
12
13 #include <generated/xe_wa_oob.h>
14
15 #include "instructions/xe_alu_commands.h"
16 #include "instructions/xe_gfxpipe_commands.h"
17 #include "instructions/xe_mi_commands.h"
18 #include "regs/xe_engine_regs.h"
19 #include "regs/xe_gt_regs.h"
20 #include "xe_assert.h"
21 #include "xe_bb.h"
22 #include "xe_bo.h"
23 #include "xe_device.h"
24 #include "xe_eu_stall.h"
25 #include "xe_exec_queue.h"
26 #include "xe_execlist.h"
27 #include "xe_force_wake.h"
28 #include "xe_ggtt.h"
29 #include "xe_gsc.h"
30 #include "xe_gt_ccs_mode.h"
31 #include "xe_gt_clock.h"
32 #include "xe_gt_freq.h"
33 #include "xe_gt_idle.h"
34 #include "xe_gt_mcr.h"
35 #include "xe_gt_pagefault.h"
36 #include "xe_gt_printk.h"
37 #include "xe_gt_sriov_pf.h"
38 #include "xe_gt_sriov_vf.h"
39 #include "xe_gt_sysfs.h"
40 #include "xe_gt_tlb_invalidation.h"
41 #include "xe_gt_topology.h"
42 #include "xe_guc_exec_queue_types.h"
43 #include "xe_guc_pc.h"
44 #include "xe_hw_fence.h"
45 #include "xe_hw_engine_class_sysfs.h"
46 #include "xe_irq.h"
47 #include "xe_lmtt.h"
48 #include "xe_lrc.h"
49 #include "xe_map.h"
50 #include "xe_migrate.h"
51 #include "xe_mmio.h"
52 #include "xe_pat.h"
53 #include "xe_pm.h"
54 #include "xe_mocs.h"
55 #include "xe_reg_sr.h"
56 #include "xe_ring_ops.h"
57 #include "xe_sa.h"
58 #include "xe_sched_job.h"
59 #include "xe_sriov.h"
60 #include "xe_tuning.h"
61 #include "xe_uc.h"
62 #include "xe_uc_fw.h"
63 #include "xe_vm.h"
64 #include "xe_wa.h"
65 #include "xe_wopcm.h"
66
gt_fini(struct drm_device * drm,void * arg)67 static void gt_fini(struct drm_device *drm, void *arg)
68 {
69 struct xe_gt *gt = arg;
70
71 destroy_workqueue(gt->ordered_wq);
72 }
73
xe_gt_alloc(struct xe_tile * tile)74 struct xe_gt *xe_gt_alloc(struct xe_tile *tile)
75 {
76 struct xe_gt *gt;
77 int err;
78
79 gt = drmm_kzalloc(&tile_to_xe(tile)->drm, sizeof(*gt), GFP_KERNEL);
80 if (!gt)
81 return ERR_PTR(-ENOMEM);
82
83 gt->tile = tile;
84 gt->ordered_wq = alloc_ordered_workqueue("gt-ordered-wq",
85 WQ_MEM_RECLAIM);
86
87 err = drmm_add_action_or_reset(>_to_xe(gt)->drm, gt_fini, gt);
88 if (err)
89 return ERR_PTR(err);
90
91 return gt;
92 }
93
xe_gt_sanitize(struct xe_gt * gt)94 void xe_gt_sanitize(struct xe_gt *gt)
95 {
96 /*
97 * FIXME: if xe_uc_sanitize is called here, on TGL driver will not
98 * reload
99 */
100 gt->uc.guc.submission_state.enabled = false;
101 }
102
xe_gt_enable_host_l2_vram(struct xe_gt * gt)103 static void xe_gt_enable_host_l2_vram(struct xe_gt *gt)
104 {
105 unsigned int fw_ref;
106 u32 reg;
107
108 if (!XE_WA(gt, 16023588340))
109 return;
110
111 fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
112 if (!fw_ref)
113 return;
114
115 if (xe_gt_is_main_type(gt)) {
116 reg = xe_gt_mcr_unicast_read_any(gt, XE2_GAMREQSTRM_CTRL);
117 reg |= CG_DIS_CNTLBUS;
118 xe_gt_mcr_multicast_write(gt, XE2_GAMREQSTRM_CTRL, reg);
119 }
120
121 xe_gt_mcr_multicast_write(gt, XEHPC_L3CLOS_MASK(3), 0xF);
122 xe_force_wake_put(gt_to_fw(gt), fw_ref);
123 }
124
xe_gt_disable_host_l2_vram(struct xe_gt * gt)125 static void xe_gt_disable_host_l2_vram(struct xe_gt *gt)
126 {
127 unsigned int fw_ref;
128 u32 reg;
129
130 if (!XE_WA(gt, 16023588340))
131 return;
132
133 if (xe_gt_is_media_type(gt))
134 return;
135
136 fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
137 if (!fw_ref)
138 return;
139
140 reg = xe_gt_mcr_unicast_read_any(gt, XE2_GAMREQSTRM_CTRL);
141 reg &= ~CG_DIS_CNTLBUS;
142 xe_gt_mcr_multicast_write(gt, XE2_GAMREQSTRM_CTRL, reg);
143
144 xe_force_wake_put(gt_to_fw(gt), fw_ref);
145 }
146
147 static void gt_reset_worker(struct work_struct *w);
148
emit_job_sync(struct xe_exec_queue * q,struct xe_bb * bb,long timeout_jiffies)149 static int emit_job_sync(struct xe_exec_queue *q, struct xe_bb *bb,
150 long timeout_jiffies)
151 {
152 struct xe_sched_job *job;
153 struct dma_fence *fence;
154 long timeout;
155
156 job = xe_bb_create_job(q, bb);
157 if (IS_ERR(job))
158 return PTR_ERR(job);
159
160 xe_sched_job_arm(job);
161 fence = dma_fence_get(&job->drm.s_fence->finished);
162 xe_sched_job_push(job);
163
164 timeout = dma_fence_wait_timeout(fence, false, timeout_jiffies);
165 dma_fence_put(fence);
166 if (timeout < 0)
167 return timeout;
168 else if (!timeout)
169 return -ETIME;
170
171 return 0;
172 }
173
emit_nop_job(struct xe_gt * gt,struct xe_exec_queue * q)174 static int emit_nop_job(struct xe_gt *gt, struct xe_exec_queue *q)
175 {
176 struct xe_bb *bb;
177 int ret;
178
179 bb = xe_bb_new(gt, 4, false);
180 if (IS_ERR(bb))
181 return PTR_ERR(bb);
182
183 ret = emit_job_sync(q, bb, HZ);
184 xe_bb_free(bb, NULL);
185
186 return ret;
187 }
188
emit_wa_job(struct xe_gt * gt,struct xe_exec_queue * q)189 static int emit_wa_job(struct xe_gt *gt, struct xe_exec_queue *q)
190 {
191 struct xe_reg_sr *sr = &q->hwe->reg_lrc;
192 struct xe_reg_sr_entry *entry;
193 int count_rmw = 0, count = 0, ret;
194 unsigned long idx;
195 struct xe_bb *bb;
196 size_t bb_len = 0;
197 u32 *cs;
198
199 /* count RMW registers as those will be handled separately */
200 xa_for_each(&sr->xa, idx, entry) {
201 if (entry->reg.masked || entry->clr_bits == ~0)
202 ++count;
203 else
204 ++count_rmw;
205 }
206
207 if (count)
208 bb_len += count * 2 + 1;
209
210 if (count_rmw)
211 bb_len += count_rmw * 20 + 7;
212
213 if (q->hwe->class == XE_ENGINE_CLASS_RENDER)
214 /*
215 * Big enough to emit all of the context's 3DSTATE via
216 * xe_lrc_emit_hwe_state_instructions()
217 */
218 bb_len += xe_gt_lrc_size(gt, q->hwe->class) / sizeof(u32);
219
220 xe_gt_dbg(gt, "LRC %s WA job: %zu dwords\n", q->hwe->name, bb_len);
221
222 bb = xe_bb_new(gt, bb_len, false);
223 if (IS_ERR(bb))
224 return PTR_ERR(bb);
225
226 cs = bb->cs;
227
228 if (count) {
229 /*
230 * Emit single LRI with all non RMW regs: 1 leading dw + 2dw per
231 * reg + 1
232 */
233
234 *cs++ = MI_LOAD_REGISTER_IMM | MI_LRI_NUM_REGS(count);
235
236 xa_for_each(&sr->xa, idx, entry) {
237 struct xe_reg reg = entry->reg;
238 u32 val;
239
240 if (reg.masked)
241 val = entry->clr_bits << 16;
242 else if (entry->clr_bits == ~0)
243 val = 0;
244 else
245 continue;
246
247 val |= entry->set_bits;
248
249 *cs++ = reg.addr;
250 *cs++ = val;
251 xe_gt_dbg(gt, "REG[0x%x] = 0x%08x", reg.addr, val);
252 }
253 }
254
255 if (count_rmw) {
256 /* Emit MI_MATH for each RMW reg: 20dw per reg + 7 trailing dw */
257
258 xa_for_each(&sr->xa, idx, entry) {
259 if (entry->reg.masked || entry->clr_bits == ~0)
260 continue;
261
262 *cs++ = MI_LOAD_REGISTER_REG | MI_LRR_DST_CS_MMIO;
263 *cs++ = entry->reg.addr;
264 *cs++ = CS_GPR_REG(0, 0).addr;
265
266 *cs++ = MI_LOAD_REGISTER_IMM | MI_LRI_NUM_REGS(2) |
267 MI_LRI_LRM_CS_MMIO;
268 *cs++ = CS_GPR_REG(0, 1).addr;
269 *cs++ = entry->clr_bits;
270 *cs++ = CS_GPR_REG(0, 2).addr;
271 *cs++ = entry->set_bits;
272
273 *cs++ = MI_MATH(8);
274 *cs++ = CS_ALU_INSTR_LOAD(SRCA, REG0);
275 *cs++ = CS_ALU_INSTR_LOADINV(SRCB, REG1);
276 *cs++ = CS_ALU_INSTR_AND;
277 *cs++ = CS_ALU_INSTR_STORE(REG0, ACCU);
278 *cs++ = CS_ALU_INSTR_LOAD(SRCA, REG0);
279 *cs++ = CS_ALU_INSTR_LOAD(SRCB, REG2);
280 *cs++ = CS_ALU_INSTR_OR;
281 *cs++ = CS_ALU_INSTR_STORE(REG0, ACCU);
282
283 *cs++ = MI_LOAD_REGISTER_REG | MI_LRR_SRC_CS_MMIO;
284 *cs++ = CS_GPR_REG(0, 0).addr;
285 *cs++ = entry->reg.addr;
286
287 xe_gt_dbg(gt, "REG[%#x] = ~%#x|%#x\n",
288 entry->reg.addr, entry->clr_bits, entry->set_bits);
289 }
290
291 /* reset used GPR */
292 *cs++ = MI_LOAD_REGISTER_IMM | MI_LRI_NUM_REGS(3) |
293 MI_LRI_LRM_CS_MMIO;
294 *cs++ = CS_GPR_REG(0, 0).addr;
295 *cs++ = 0;
296 *cs++ = CS_GPR_REG(0, 1).addr;
297 *cs++ = 0;
298 *cs++ = CS_GPR_REG(0, 2).addr;
299 *cs++ = 0;
300 }
301
302 cs = xe_lrc_emit_hwe_state_instructions(q, cs);
303
304 bb->len = cs - bb->cs;
305
306 ret = emit_job_sync(q, bb, HZ);
307
308 xe_bb_free(bb, NULL);
309
310 return ret;
311 }
312
xe_gt_record_default_lrcs(struct xe_gt * gt)313 int xe_gt_record_default_lrcs(struct xe_gt *gt)
314 {
315 struct xe_device *xe = gt_to_xe(gt);
316 struct xe_hw_engine *hwe;
317 enum xe_hw_engine_id id;
318 int err = 0;
319
320 for_each_hw_engine(hwe, gt, id) {
321 struct xe_exec_queue *q, *nop_q;
322 void *default_lrc;
323
324 if (gt->default_lrc[hwe->class])
325 continue;
326
327 xe_reg_sr_init(&hwe->reg_lrc, hwe->name, xe);
328 xe_wa_process_lrc(hwe);
329 xe_hw_engine_setup_default_lrc_state(hwe);
330 xe_tuning_process_lrc(hwe);
331
332 default_lrc = drmm_kzalloc(&xe->drm,
333 xe_gt_lrc_size(gt, hwe->class),
334 GFP_KERNEL);
335 if (!default_lrc)
336 return -ENOMEM;
337
338 q = xe_exec_queue_create(xe, NULL, BIT(hwe->logical_instance), 1,
339 hwe, EXEC_QUEUE_FLAG_KERNEL, 0);
340 if (IS_ERR(q)) {
341 err = PTR_ERR(q);
342 xe_gt_err(gt, "hwe %s: xe_exec_queue_create failed (%pe)\n",
343 hwe->name, q);
344 return err;
345 }
346
347 /* Prime golden LRC with known good state */
348 err = emit_wa_job(gt, q);
349 if (err) {
350 xe_gt_err(gt, "hwe %s: emit_wa_job failed (%pe) guc_id=%u\n",
351 hwe->name, ERR_PTR(err), q->guc->id);
352 goto put_exec_queue;
353 }
354
355 nop_q = xe_exec_queue_create(xe, NULL, BIT(hwe->logical_instance),
356 1, hwe, EXEC_QUEUE_FLAG_KERNEL, 0);
357 if (IS_ERR(nop_q)) {
358 err = PTR_ERR(nop_q);
359 xe_gt_err(gt, "hwe %s: nop xe_exec_queue_create failed (%pe)\n",
360 hwe->name, nop_q);
361 goto put_exec_queue;
362 }
363
364 /* Switch to different LRC */
365 err = emit_nop_job(gt, nop_q);
366 if (err) {
367 xe_gt_err(gt, "hwe %s: nop emit_nop_job failed (%pe) guc_id=%u\n",
368 hwe->name, ERR_PTR(err), nop_q->guc->id);
369 goto put_nop_q;
370 }
371
372 xe_map_memcpy_from(xe, default_lrc,
373 &q->lrc[0]->bo->vmap,
374 xe_lrc_pphwsp_offset(q->lrc[0]),
375 xe_gt_lrc_size(gt, hwe->class));
376
377 gt->default_lrc[hwe->class] = default_lrc;
378 put_nop_q:
379 xe_exec_queue_put(nop_q);
380 put_exec_queue:
381 xe_exec_queue_put(q);
382 if (err)
383 break;
384 }
385
386 return err;
387 }
388
xe_gt_init_early(struct xe_gt * gt)389 int xe_gt_init_early(struct xe_gt *gt)
390 {
391 unsigned int fw_ref;
392 int err;
393
394 if (IS_SRIOV_PF(gt_to_xe(gt))) {
395 err = xe_gt_sriov_pf_init_early(gt);
396 if (err)
397 return err;
398 }
399
400 xe_reg_sr_init(>->reg_sr, "GT", gt_to_xe(gt));
401
402 err = xe_wa_init(gt);
403 if (err)
404 return err;
405
406 err = xe_tuning_init(gt);
407 if (err)
408 return err;
409
410 xe_wa_process_oob(gt);
411
412 xe_force_wake_init_gt(gt, gt_to_fw(gt));
413 spin_lock_init(>->global_invl_lock);
414
415 err = xe_gt_tlb_invalidation_init_early(gt);
416 if (err)
417 return err;
418
419 xe_mocs_init_early(gt);
420
421 /*
422 * Only after this point can GT-specific MMIO operations
423 * (including things like communication with the GuC)
424 * be performed.
425 */
426 xe_gt_mmio_init(gt);
427
428 err = xe_uc_init_noalloc(>->uc);
429 if (err)
430 return err;
431
432 fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
433 if (!fw_ref)
434 return -ETIMEDOUT;
435
436 xe_gt_mcr_init_early(gt);
437 xe_pat_init(gt);
438 xe_force_wake_put(gt_to_fw(gt), fw_ref);
439
440 return 0;
441 }
442
dump_pat_on_error(struct xe_gt * gt)443 static void dump_pat_on_error(struct xe_gt *gt)
444 {
445 struct drm_printer p;
446 char prefix[32];
447
448 snprintf(prefix, sizeof(prefix), "[GT%u Error]", gt->info.id);
449 p = drm_dbg_printer(>_to_xe(gt)->drm, DRM_UT_DRIVER, prefix);
450
451 xe_pat_dump(gt, &p);
452 }
453
gt_init_with_gt_forcewake(struct xe_gt * gt)454 static int gt_init_with_gt_forcewake(struct xe_gt *gt)
455 {
456 unsigned int fw_ref;
457 int err;
458
459 fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
460 if (!fw_ref)
461 return -ETIMEDOUT;
462
463 err = xe_uc_init(>->uc);
464 if (err)
465 goto err_force_wake;
466
467 xe_gt_topology_init(gt);
468 xe_gt_mcr_init(gt);
469 xe_gt_enable_host_l2_vram(gt);
470
471 if (xe_gt_is_main_type(gt)) {
472 err = xe_ggtt_init(gt_to_tile(gt)->mem.ggtt);
473 if (err)
474 goto err_force_wake;
475 if (IS_SRIOV_PF(gt_to_xe(gt)))
476 xe_lmtt_init(>_to_tile(gt)->sriov.pf.lmtt);
477 }
478
479 /* Enable per hw engine IRQs */
480 xe_irq_enable_hwe(gt);
481
482 /* Rerun MCR init as we now have hw engine list */
483 xe_gt_mcr_init(gt);
484
485 err = xe_hw_engines_init_early(gt);
486 if (err) {
487 dump_pat_on_error(gt);
488 goto err_force_wake;
489 }
490
491 err = xe_hw_engine_class_sysfs_init(gt);
492 if (err)
493 goto err_force_wake;
494
495 /* Initialize CCS mode sysfs after early initialization of HW engines */
496 err = xe_gt_ccs_mode_sysfs_init(gt);
497 if (err)
498 goto err_force_wake;
499
500 /*
501 * Stash hardware-reported version. Since this register does not exist
502 * on pre-MTL platforms, reading it there will (correctly) return 0.
503 */
504 gt->info.gmdid = xe_mmio_read32(>->mmio, GMD_ID);
505
506 xe_force_wake_put(gt_to_fw(gt), fw_ref);
507 return 0;
508
509 err_force_wake:
510 xe_force_wake_put(gt_to_fw(gt), fw_ref);
511
512 return err;
513 }
514
gt_init_with_all_forcewake(struct xe_gt * gt)515 static int gt_init_with_all_forcewake(struct xe_gt *gt)
516 {
517 unsigned int fw_ref;
518 int err;
519
520 fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL);
521 if (!xe_force_wake_ref_has_domain(fw_ref, XE_FORCEWAKE_ALL)) {
522 err = -ETIMEDOUT;
523 goto err_force_wake;
524 }
525
526 xe_gt_mcr_set_implicit_defaults(gt);
527 xe_wa_process_gt(gt);
528 xe_tuning_process_gt(gt);
529 xe_reg_sr_apply_mmio(>->reg_sr, gt);
530
531 err = xe_gt_clock_init(gt);
532 if (err)
533 goto err_force_wake;
534
535 xe_mocs_init(gt);
536 err = xe_execlist_init(gt);
537 if (err)
538 goto err_force_wake;
539
540 err = xe_hw_engines_init(gt);
541 if (err)
542 goto err_force_wake;
543
544 err = xe_uc_init_post_hwconfig(>->uc);
545 if (err)
546 goto err_force_wake;
547
548 if (xe_gt_is_main_type(gt)) {
549 /*
550 * USM has its only SA pool to non-block behind user operations
551 */
552 if (gt_to_xe(gt)->info.has_usm) {
553 struct xe_device *xe = gt_to_xe(gt);
554
555 gt->usm.bb_pool = xe_sa_bo_manager_init(gt_to_tile(gt),
556 IS_DGFX(xe) ? SZ_1M : SZ_512K, 16);
557 if (IS_ERR(gt->usm.bb_pool)) {
558 err = PTR_ERR(gt->usm.bb_pool);
559 goto err_force_wake;
560 }
561 }
562 }
563
564 if (xe_gt_is_main_type(gt)) {
565 struct xe_tile *tile = gt_to_tile(gt);
566
567 tile->migrate = xe_migrate_init(tile);
568 if (IS_ERR(tile->migrate)) {
569 err = PTR_ERR(tile->migrate);
570 goto err_force_wake;
571 }
572 }
573
574 err = xe_uc_load_hw(>->uc);
575 if (err)
576 goto err_force_wake;
577
578 /* Configure default CCS mode of 1 engine with all resources */
579 if (xe_gt_ccs_mode_enabled(gt)) {
580 gt->ccs_mode = 1;
581 xe_gt_apply_ccs_mode(gt);
582 }
583
584 if (IS_SRIOV_PF(gt_to_xe(gt)) && xe_gt_is_main_type(gt))
585 xe_lmtt_init_hw(>_to_tile(gt)->sriov.pf.lmtt);
586
587 if (IS_SRIOV_PF(gt_to_xe(gt))) {
588 xe_gt_sriov_pf_init(gt);
589 xe_gt_sriov_pf_init_hw(gt);
590 }
591
592 xe_force_wake_put(gt_to_fw(gt), fw_ref);
593
594 return 0;
595
596 err_force_wake:
597 xe_force_wake_put(gt_to_fw(gt), fw_ref);
598
599 return err;
600 }
601
xe_gt_fini(void * arg)602 static void xe_gt_fini(void *arg)
603 {
604 struct xe_gt *gt = arg;
605 int i;
606
607 for (i = 0; i < XE_ENGINE_CLASS_MAX; ++i)
608 xe_hw_fence_irq_finish(>->fence_irq[i]);
609
610 xe_gt_disable_host_l2_vram(gt);
611 }
612
xe_gt_init(struct xe_gt * gt)613 int xe_gt_init(struct xe_gt *gt)
614 {
615 int err;
616 int i;
617
618 INIT_WORK(>->reset.worker, gt_reset_worker);
619
620 for (i = 0; i < XE_ENGINE_CLASS_MAX; ++i) {
621 gt->ring_ops[i] = xe_ring_ops_get(gt, i);
622 xe_hw_fence_irq_init(>->fence_irq[i]);
623 }
624
625 err = devm_add_action_or_reset(gt_to_xe(gt)->drm.dev, xe_gt_fini, gt);
626 if (err)
627 return err;
628
629 err = xe_gt_sysfs_init(gt);
630 if (err)
631 return err;
632
633 err = gt_init_with_gt_forcewake(gt);
634 if (err)
635 return err;
636
637 err = xe_gt_pagefault_init(gt);
638 if (err)
639 return err;
640
641 err = xe_gt_idle_init(>->gtidle);
642 if (err)
643 return err;
644
645 err = xe_gt_freq_init(gt);
646 if (err)
647 return err;
648
649 xe_force_wake_init_engines(gt, gt_to_fw(gt));
650
651 err = gt_init_with_all_forcewake(gt);
652 if (err)
653 return err;
654
655 xe_gt_record_user_engines(gt);
656
657 err = xe_eu_stall_init(gt);
658 if (err)
659 return err;
660
661 return 0;
662 }
663
664 /**
665 * xe_gt_mmio_init() - Initialize GT's MMIO access
666 * @gt: the GT object
667 *
668 * Initialize GT's MMIO accessor, which will be used to access registers inside
669 * this GT.
670 */
xe_gt_mmio_init(struct xe_gt * gt)671 void xe_gt_mmio_init(struct xe_gt *gt)
672 {
673 struct xe_tile *tile = gt_to_tile(gt);
674 struct xe_device *xe = tile_to_xe(tile);
675
676 xe_mmio_init(>->mmio, tile, tile->mmio.regs, tile->mmio.regs_size);
677
678 if (gt->info.type == XE_GT_TYPE_MEDIA) {
679 gt->mmio.adj_offset = MEDIA_GT_GSI_OFFSET;
680 gt->mmio.adj_limit = MEDIA_GT_GSI_LENGTH;
681 } else {
682 gt->mmio.adj_offset = 0;
683 gt->mmio.adj_limit = 0;
684 }
685
686 if (IS_SRIOV_VF(xe))
687 gt->mmio.sriov_vf_gt = gt;
688 }
689
xe_gt_record_user_engines(struct xe_gt * gt)690 void xe_gt_record_user_engines(struct xe_gt *gt)
691 {
692 struct xe_hw_engine *hwe;
693 enum xe_hw_engine_id id;
694
695 gt->user_engines.mask = 0;
696 memset(gt->user_engines.instances_per_class, 0,
697 sizeof(gt->user_engines.instances_per_class));
698
699 for_each_hw_engine(hwe, gt, id) {
700 if (xe_hw_engine_is_reserved(hwe))
701 continue;
702
703 gt->user_engines.mask |= BIT_ULL(id);
704 gt->user_engines.instances_per_class[hwe->class]++;
705 }
706
707 xe_gt_assert(gt, (gt->user_engines.mask | gt->info.engine_mask)
708 == gt->info.engine_mask);
709 }
710
do_gt_reset(struct xe_gt * gt)711 static int do_gt_reset(struct xe_gt *gt)
712 {
713 int err;
714
715 if (IS_SRIOV_VF(gt_to_xe(gt)))
716 return xe_gt_sriov_vf_reset(gt);
717
718 xe_gsc_wa_14015076503(gt, true);
719
720 xe_mmio_write32(>->mmio, GDRST, GRDOM_FULL);
721 err = xe_mmio_wait32(>->mmio, GDRST, GRDOM_FULL, 0, 5000, NULL, false);
722 if (err)
723 xe_gt_err(gt, "failed to clear GRDOM_FULL (%pe)\n",
724 ERR_PTR(err));
725
726 xe_gsc_wa_14015076503(gt, false);
727
728 return err;
729 }
730
vf_gt_restart(struct xe_gt * gt)731 static int vf_gt_restart(struct xe_gt *gt)
732 {
733 int err;
734
735 err = xe_uc_sanitize_reset(>->uc);
736 if (err)
737 return err;
738
739 err = xe_uc_load_hw(>->uc);
740 if (err)
741 return err;
742
743 err = xe_uc_start(>->uc);
744 if (err)
745 return err;
746
747 return 0;
748 }
749
do_gt_restart(struct xe_gt * gt)750 static int do_gt_restart(struct xe_gt *gt)
751 {
752 struct xe_hw_engine *hwe;
753 enum xe_hw_engine_id id;
754 int err;
755
756 if (IS_SRIOV_VF(gt_to_xe(gt)))
757 return vf_gt_restart(gt);
758
759 xe_pat_init(gt);
760
761 xe_gt_enable_host_l2_vram(gt);
762
763 xe_gt_mcr_set_implicit_defaults(gt);
764 xe_reg_sr_apply_mmio(>->reg_sr, gt);
765
766 err = xe_wopcm_init(>->uc.wopcm);
767 if (err)
768 return err;
769
770 for_each_hw_engine(hwe, gt, id)
771 xe_hw_engine_enable_ring(hwe);
772
773 err = xe_uc_sanitize_reset(>->uc);
774 if (err)
775 return err;
776
777 err = xe_uc_load_hw(>->uc);
778 if (err)
779 return err;
780
781 if (IS_SRIOV_PF(gt_to_xe(gt)) && xe_gt_is_main_type(gt))
782 xe_lmtt_init_hw(>_to_tile(gt)->sriov.pf.lmtt);
783
784 if (IS_SRIOV_PF(gt_to_xe(gt)))
785 xe_gt_sriov_pf_init_hw(gt);
786
787 xe_mocs_init(gt);
788 err = xe_uc_start(>->uc);
789 if (err)
790 return err;
791
792 for_each_hw_engine(hwe, gt, id)
793 xe_reg_sr_apply_mmio(&hwe->reg_sr, gt);
794
795 /* Get CCS mode in sync between sw/hw */
796 xe_gt_apply_ccs_mode(gt);
797
798 /* Restore GT freq to expected values */
799 xe_gt_sanitize_freq(gt);
800
801 if (IS_SRIOV_PF(gt_to_xe(gt)))
802 xe_gt_sriov_pf_restart(gt);
803
804 return 0;
805 }
806
gt_reset(struct xe_gt * gt)807 static int gt_reset(struct xe_gt *gt)
808 {
809 unsigned int fw_ref;
810 int err;
811
812 if (xe_device_wedged(gt_to_xe(gt)))
813 return -ECANCELED;
814
815 /* We only support GT resets with GuC submission */
816 if (!xe_device_uc_enabled(gt_to_xe(gt)))
817 return -ENODEV;
818
819 xe_gt_info(gt, "reset started\n");
820
821 xe_pm_runtime_get(gt_to_xe(gt));
822
823 if (xe_fault_inject_gt_reset()) {
824 err = -ECANCELED;
825 goto err_fail;
826 }
827
828 xe_gt_sanitize(gt);
829
830 fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL);
831 if (!xe_force_wake_ref_has_domain(fw_ref, XE_FORCEWAKE_ALL)) {
832 err = -ETIMEDOUT;
833 goto err_out;
834 }
835
836 if (IS_SRIOV_PF(gt_to_xe(gt)))
837 xe_gt_sriov_pf_stop_prepare(gt);
838
839 xe_uc_gucrc_disable(>->uc);
840 xe_uc_stop_prepare(>->uc);
841 xe_gt_pagefault_reset(gt);
842
843 xe_uc_stop(>->uc);
844
845 xe_gt_tlb_invalidation_reset(gt);
846
847 err = do_gt_reset(gt);
848 if (err)
849 goto err_out;
850
851 err = do_gt_restart(gt);
852 if (err)
853 goto err_out;
854
855 xe_force_wake_put(gt_to_fw(gt), fw_ref);
856 xe_pm_runtime_put(gt_to_xe(gt));
857
858 xe_gt_info(gt, "reset done\n");
859
860 return 0;
861
862 err_out:
863 xe_force_wake_put(gt_to_fw(gt), fw_ref);
864 XE_WARN_ON(xe_uc_start(>->uc));
865 err_fail:
866 xe_gt_err(gt, "reset failed (%pe)\n", ERR_PTR(err));
867
868 xe_device_declare_wedged(gt_to_xe(gt));
869 xe_pm_runtime_put(gt_to_xe(gt));
870
871 return err;
872 }
873
gt_reset_worker(struct work_struct * w)874 static void gt_reset_worker(struct work_struct *w)
875 {
876 struct xe_gt *gt = container_of(w, typeof(*gt), reset.worker);
877
878 gt_reset(gt);
879 }
880
xe_gt_reset_async(struct xe_gt * gt)881 void xe_gt_reset_async(struct xe_gt *gt)
882 {
883 xe_gt_info(gt, "trying reset from %ps\n", __builtin_return_address(0));
884
885 /* Don't do a reset while one is already in flight */
886 if (!xe_fault_inject_gt_reset() && xe_uc_reset_prepare(>->uc))
887 return;
888
889 xe_gt_info(gt, "reset queued\n");
890 queue_work(gt->ordered_wq, >->reset.worker);
891 }
892
xe_gt_suspend_prepare(struct xe_gt * gt)893 void xe_gt_suspend_prepare(struct xe_gt *gt)
894 {
895 unsigned int fw_ref;
896
897 fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL);
898
899 xe_uc_suspend_prepare(>->uc);
900
901 xe_force_wake_put(gt_to_fw(gt), fw_ref);
902 }
903
xe_gt_suspend(struct xe_gt * gt)904 int xe_gt_suspend(struct xe_gt *gt)
905 {
906 unsigned int fw_ref;
907 int err;
908
909 xe_gt_dbg(gt, "suspending\n");
910 xe_gt_sanitize(gt);
911
912 fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL);
913 if (!xe_force_wake_ref_has_domain(fw_ref, XE_FORCEWAKE_ALL))
914 goto err_msg;
915
916 err = xe_uc_suspend(>->uc);
917 if (err)
918 goto err_force_wake;
919
920 xe_gt_idle_disable_pg(gt);
921
922 xe_gt_disable_host_l2_vram(gt);
923
924 xe_force_wake_put(gt_to_fw(gt), fw_ref);
925 xe_gt_dbg(gt, "suspended\n");
926
927 return 0;
928
929 err_msg:
930 err = -ETIMEDOUT;
931 err_force_wake:
932 xe_force_wake_put(gt_to_fw(gt), fw_ref);
933 xe_gt_err(gt, "suspend failed (%pe)\n", ERR_PTR(err));
934
935 return err;
936 }
937
xe_gt_shutdown(struct xe_gt * gt)938 void xe_gt_shutdown(struct xe_gt *gt)
939 {
940 unsigned int fw_ref;
941
942 fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL);
943 do_gt_reset(gt);
944 xe_force_wake_put(gt_to_fw(gt), fw_ref);
945 }
946
947 /**
948 * xe_gt_sanitize_freq() - Restore saved frequencies if necessary.
949 * @gt: the GT object
950 *
951 * Called after driver init/GSC load completes to restore GT frequencies if we
952 * limited them for any WAs.
953 */
xe_gt_sanitize_freq(struct xe_gt * gt)954 int xe_gt_sanitize_freq(struct xe_gt *gt)
955 {
956 int ret = 0;
957
958 if ((!xe_uc_fw_is_available(>->uc.gsc.fw) ||
959 xe_uc_fw_is_loaded(>->uc.gsc.fw) ||
960 xe_uc_fw_is_in_error_state(>->uc.gsc.fw)) &&
961 XE_WA(gt, 22019338487))
962 ret = xe_guc_pc_restore_stashed_freq(>->uc.guc.pc);
963
964 return ret;
965 }
966
xe_gt_resume(struct xe_gt * gt)967 int xe_gt_resume(struct xe_gt *gt)
968 {
969 unsigned int fw_ref;
970 int err;
971
972 xe_gt_dbg(gt, "resuming\n");
973 fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL);
974 if (!xe_force_wake_ref_has_domain(fw_ref, XE_FORCEWAKE_ALL))
975 goto err_msg;
976
977 err = do_gt_restart(gt);
978 if (err)
979 goto err_force_wake;
980
981 xe_gt_idle_enable_pg(gt);
982
983 xe_force_wake_put(gt_to_fw(gt), fw_ref);
984 xe_gt_dbg(gt, "resumed\n");
985
986 return 0;
987
988 err_msg:
989 err = -ETIMEDOUT;
990 err_force_wake:
991 xe_force_wake_put(gt_to_fw(gt), fw_ref);
992 xe_gt_err(gt, "resume failed (%pe)\n", ERR_PTR(err));
993
994 return err;
995 }
996
xe_gt_hw_engine(struct xe_gt * gt,enum xe_engine_class class,u16 instance,bool logical)997 struct xe_hw_engine *xe_gt_hw_engine(struct xe_gt *gt,
998 enum xe_engine_class class,
999 u16 instance, bool logical)
1000 {
1001 struct xe_hw_engine *hwe;
1002 enum xe_hw_engine_id id;
1003
1004 for_each_hw_engine(hwe, gt, id)
1005 if (hwe->class == class &&
1006 ((!logical && hwe->instance == instance) ||
1007 (logical && hwe->logical_instance == instance)))
1008 return hwe;
1009
1010 return NULL;
1011 }
1012
xe_gt_any_hw_engine_by_reset_domain(struct xe_gt * gt,enum xe_engine_class class)1013 struct xe_hw_engine *xe_gt_any_hw_engine_by_reset_domain(struct xe_gt *gt,
1014 enum xe_engine_class class)
1015 {
1016 struct xe_hw_engine *hwe;
1017 enum xe_hw_engine_id id;
1018
1019 for_each_hw_engine(hwe, gt, id) {
1020 switch (class) {
1021 case XE_ENGINE_CLASS_RENDER:
1022 case XE_ENGINE_CLASS_COMPUTE:
1023 if (hwe->class == XE_ENGINE_CLASS_RENDER ||
1024 hwe->class == XE_ENGINE_CLASS_COMPUTE)
1025 return hwe;
1026 break;
1027 default:
1028 if (hwe->class == class)
1029 return hwe;
1030 }
1031 }
1032
1033 return NULL;
1034 }
1035
xe_gt_any_hw_engine(struct xe_gt * gt)1036 struct xe_hw_engine *xe_gt_any_hw_engine(struct xe_gt *gt)
1037 {
1038 struct xe_hw_engine *hwe;
1039 enum xe_hw_engine_id id;
1040
1041 for_each_hw_engine(hwe, gt, id)
1042 return hwe;
1043
1044 return NULL;
1045 }
1046
1047 /**
1048 * xe_gt_declare_wedged() - Declare GT wedged
1049 * @gt: the GT object
1050 *
1051 * Wedge the GT which stops all submission, saves desired debug state, and
1052 * cleans up anything which could timeout.
1053 */
xe_gt_declare_wedged(struct xe_gt * gt)1054 void xe_gt_declare_wedged(struct xe_gt *gt)
1055 {
1056 xe_gt_assert(gt, gt_to_xe(gt)->wedged.mode);
1057
1058 xe_uc_declare_wedged(>->uc);
1059 xe_gt_tlb_invalidation_reset(gt);
1060 }
1061