1 /*
2 * Copyright 2021 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 */
23
24 #include <linux/firmware.h>
25 #include "amdgpu.h"
26 #include "amdgpu_vcn.h"
27 #include "amdgpu_pm.h"
28 #include "amdgpu_cs.h"
29 #include "soc15.h"
30 #include "soc15d.h"
31 #include "soc15_hw_ip.h"
32 #include "vcn_v2_0.h"
33 #include "mmsch_v4_0.h"
34 #include "vcn_v4_0.h"
35
36 #include "vcn/vcn_4_0_0_offset.h"
37 #include "vcn/vcn_4_0_0_sh_mask.h"
38 #include "ivsrcid/vcn/irqsrcs_vcn_4_0.h"
39
40 #include <drm/drm_drv.h>
41
42 #define mmUVD_DPG_LMA_CTL regUVD_DPG_LMA_CTL
43 #define mmUVD_DPG_LMA_CTL_BASE_IDX regUVD_DPG_LMA_CTL_BASE_IDX
44 #define mmUVD_DPG_LMA_DATA regUVD_DPG_LMA_DATA
45 #define mmUVD_DPG_LMA_DATA_BASE_IDX regUVD_DPG_LMA_DATA_BASE_IDX
46
47 #define VCN_VID_SOC_ADDRESS_2_0 0x1fb00
48 #define VCN1_VID_SOC_ADDRESS_3_0 0x48300
49 #define VCN1_AON_SOC_ADDRESS_3_0 0x48000
50
51 #define VCN_HARVEST_MMSCH 0
52
53 #define RDECODE_MSG_CREATE 0x00000000
54 #define RDECODE_MESSAGE_CREATE 0x00000001
55
56 static const struct amdgpu_hwip_reg_entry vcn_reg_list_4_0[] = {
57 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_POWER_STATUS),
58 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_STATUS),
59 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_CONTEXT_ID),
60 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_CONTEXT_ID2),
61 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_GPCOM_VCPU_DATA0),
62 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_GPCOM_VCPU_DATA1),
63 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_GPCOM_VCPU_CMD),
64 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_BASE_HI),
65 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_BASE_LO),
66 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_BASE_HI2),
67 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_BASE_LO2),
68 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_BASE_HI3),
69 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_BASE_LO3),
70 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_BASE_HI4),
71 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_BASE_LO4),
72 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_RPTR),
73 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_WPTR),
74 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_RPTR2),
75 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_WPTR2),
76 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_RPTR3),
77 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_WPTR3),
78 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_RPTR4),
79 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_WPTR4),
80 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_SIZE),
81 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_SIZE2),
82 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_SIZE3),
83 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_SIZE4),
84 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_PGFSM_CONFIG),
85 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_PGFSM_STATUS),
86 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_DPG_LMA_CTL),
87 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_DPG_LMA_DATA),
88 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_DPG_LMA_MASK),
89 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_DPG_PAUSE)
90 };
91
92 static int amdgpu_ih_clientid_vcns[] = {
93 SOC15_IH_CLIENTID_VCN,
94 SOC15_IH_CLIENTID_VCN1
95 };
96
97 static int vcn_v4_0_start_sriov(struct amdgpu_device *adev);
98 static void vcn_v4_0_set_unified_ring_funcs(struct amdgpu_device *adev);
99 static void vcn_v4_0_set_irq_funcs(struct amdgpu_device *adev);
100 static int vcn_v4_0_set_pg_state(struct amdgpu_vcn_inst *vinst,
101 enum amd_powergating_state state);
102 static int vcn_v4_0_pause_dpg_mode(struct amdgpu_vcn_inst *vinst,
103 struct dpg_pause_state *new_state);
104 static void vcn_v4_0_unified_ring_set_wptr(struct amdgpu_ring *ring);
105 static void vcn_v4_0_set_ras_funcs(struct amdgpu_device *adev);
106
107 /**
108 * vcn_v4_0_early_init - set function pointers and load microcode
109 *
110 * @ip_block: Pointer to the amdgpu_ip_block for this hw instance.
111 *
112 * Set ring and irq function pointers
113 * Load microcode from filesystem
114 */
vcn_v4_0_early_init(struct amdgpu_ip_block * ip_block)115 static int vcn_v4_0_early_init(struct amdgpu_ip_block *ip_block)
116 {
117 struct amdgpu_device *adev = ip_block->adev;
118 int i, r;
119
120 if (amdgpu_sriov_vf(adev)) {
121 adev->vcn.harvest_config = VCN_HARVEST_MMSCH;
122 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
123 if (amdgpu_vcn_is_disabled_vcn(adev, VCN_ENCODE_RING, i)) {
124 adev->vcn.harvest_config |= 1 << i;
125 dev_info(adev->dev, "VCN%d is disabled by hypervisor\n", i);
126 }
127 }
128 }
129
130 for (i = 0; i < adev->vcn.num_vcn_inst; ++i)
131 /* re-use enc ring as unified ring */
132 adev->vcn.inst[i].num_enc_rings = 1;
133
134 vcn_v4_0_set_unified_ring_funcs(adev);
135 vcn_v4_0_set_irq_funcs(adev);
136 vcn_v4_0_set_ras_funcs(adev);
137
138 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
139 adev->vcn.inst[i].set_pg_state = vcn_v4_0_set_pg_state;
140
141 r = amdgpu_vcn_early_init(adev, i);
142 if (r)
143 return r;
144 }
145
146 return 0;
147 }
148
vcn_v4_0_fw_shared_init(struct amdgpu_device * adev,int inst_idx)149 static int vcn_v4_0_fw_shared_init(struct amdgpu_device *adev, int inst_idx)
150 {
151 volatile struct amdgpu_vcn4_fw_shared *fw_shared;
152
153 fw_shared = adev->vcn.inst[inst_idx].fw_shared.cpu_addr;
154 fw_shared->present_flag_0 = cpu_to_le32(AMDGPU_FW_SHARED_FLAG_0_UNIFIED_QUEUE);
155 fw_shared->sq.is_enabled = 1;
156
157 fw_shared->present_flag_0 |= cpu_to_le32(AMDGPU_VCN_SMU_DPM_INTERFACE_FLAG);
158 fw_shared->smu_dpm_interface.smu_interface_type = (adev->flags & AMD_IS_APU) ?
159 AMDGPU_VCN_SMU_DPM_INTERFACE_APU : AMDGPU_VCN_SMU_DPM_INTERFACE_DGPU;
160
161 if (amdgpu_ip_version(adev, VCN_HWIP, 0) ==
162 IP_VERSION(4, 0, 2)) {
163 fw_shared->present_flag_0 |= AMDGPU_FW_SHARED_FLAG_0_DRM_KEY_INJECT;
164 fw_shared->drm_key_wa.method =
165 AMDGPU_DRM_KEY_INJECT_WORKAROUND_VCNFW_ASD_HANDSHAKING;
166 }
167
168 if (amdgpu_vcnfw_log)
169 amdgpu_vcn_fwlog_init(&adev->vcn.inst[inst_idx]);
170
171 return 0;
172 }
173
174 /**
175 * vcn_v4_0_sw_init - sw init for VCN block
176 *
177 * @ip_block: Pointer to the amdgpu_ip_block for this hw instance.
178 *
179 * Load firmware and sw initialization
180 */
vcn_v4_0_sw_init(struct amdgpu_ip_block * ip_block)181 static int vcn_v4_0_sw_init(struct amdgpu_ip_block *ip_block)
182 {
183 struct amdgpu_ring *ring;
184 struct amdgpu_device *adev = ip_block->adev;
185 int i, r;
186 uint32_t reg_count = ARRAY_SIZE(vcn_reg_list_4_0);
187 uint32_t *ptr;
188
189 for (i = 0; i < adev->vcn.num_vcn_inst; i++) {
190 if (adev->vcn.harvest_config & (1 << i))
191 continue;
192
193 r = amdgpu_vcn_sw_init(adev, i);
194 if (r)
195 return r;
196
197 amdgpu_vcn_setup_ucode(adev, i);
198
199 r = amdgpu_vcn_resume(adev, i);
200 if (r)
201 return r;
202
203 /* Init instance 0 sched_score to 1, so it's scheduled after other instances */
204 if (i == 0)
205 atomic_set(&adev->vcn.inst[i].sched_score, 1);
206 else
207 atomic_set(&adev->vcn.inst[i].sched_score, 0);
208
209 /* VCN UNIFIED TRAP */
210 r = amdgpu_irq_add_id(adev, amdgpu_ih_clientid_vcns[i],
211 VCN_4_0__SRCID__UVD_ENC_GENERAL_PURPOSE, &adev->vcn.inst[i].irq);
212 if (r)
213 return r;
214
215 /* VCN POISON TRAP */
216 r = amdgpu_irq_add_id(adev, amdgpu_ih_clientid_vcns[i],
217 VCN_4_0__SRCID_UVD_POISON, &adev->vcn.inst[i].ras_poison_irq);
218 if (r)
219 return r;
220
221 ring = &adev->vcn.inst[i].ring_enc[0];
222 ring->use_doorbell = true;
223 if (amdgpu_sriov_vf(adev))
224 ring->doorbell_index = (adev->doorbell_index.vcn.vcn_ring0_1 << 1) + i *
225 (adev->vcn.inst[i].num_enc_rings + 1) + 1;
226 else
227 ring->doorbell_index = (adev->doorbell_index.vcn.vcn_ring0_1 << 1) + 2 + 8 * i;
228 ring->vm_hub = AMDGPU_MMHUB0(0);
229 sprintf(ring->name, "vcn_unified_%d", i);
230
231 r = amdgpu_ring_init(adev, ring, 512, &adev->vcn.inst[i].irq, 0,
232 AMDGPU_RING_PRIO_0, &adev->vcn.inst[i].sched_score);
233 if (r)
234 return r;
235
236 vcn_v4_0_fw_shared_init(adev, i);
237
238 if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG)
239 adev->vcn.inst[i].pause_dpg_mode = vcn_v4_0_pause_dpg_mode;
240 }
241
242 adev->vcn.supported_reset =
243 amdgpu_get_soft_full_reset_mask(&adev->vcn.inst[0].ring_enc[0]);
244 if (!amdgpu_sriov_vf(adev))
245 adev->vcn.supported_reset |= AMDGPU_RESET_TYPE_PER_QUEUE;
246
247 if (amdgpu_sriov_vf(adev)) {
248 r = amdgpu_virt_alloc_mm_table(adev);
249 if (r)
250 return r;
251 }
252
253
254 r = amdgpu_vcn_ras_sw_init(adev);
255 if (r)
256 return r;
257
258 /* Allocate memory for VCN IP Dump buffer */
259 ptr = kcalloc(adev->vcn.num_vcn_inst * reg_count, sizeof(uint32_t), GFP_KERNEL);
260 if (!ptr) {
261 DRM_ERROR("Failed to allocate memory for VCN IP Dump\n");
262 adev->vcn.ip_dump = NULL;
263 } else {
264 adev->vcn.ip_dump = ptr;
265 }
266
267 r = amdgpu_vcn_sysfs_reset_mask_init(adev);
268 if (r)
269 return r;
270
271 return 0;
272 }
273
274 /**
275 * vcn_v4_0_sw_fini - sw fini for VCN block
276 *
277 * @ip_block: Pointer to the amdgpu_ip_block for this hw instance.
278 *
279 * VCN suspend and free up sw allocation
280 */
vcn_v4_0_sw_fini(struct amdgpu_ip_block * ip_block)281 static int vcn_v4_0_sw_fini(struct amdgpu_ip_block *ip_block)
282 {
283 struct amdgpu_device *adev = ip_block->adev;
284 int i, r, idx;
285
286 if (drm_dev_enter(adev_to_drm(adev), &idx)) {
287 for (i = 0; i < adev->vcn.num_vcn_inst; i++) {
288 volatile struct amdgpu_vcn4_fw_shared *fw_shared;
289
290 if (adev->vcn.harvest_config & (1 << i))
291 continue;
292
293 fw_shared = adev->vcn.inst[i].fw_shared.cpu_addr;
294 fw_shared->present_flag_0 = 0;
295 fw_shared->sq.is_enabled = 0;
296 }
297
298 drm_dev_exit(idx);
299 }
300
301 if (amdgpu_sriov_vf(adev))
302 amdgpu_virt_free_mm_table(adev);
303
304 for (i = 0; i < adev->vcn.num_vcn_inst; i++) {
305 r = amdgpu_vcn_suspend(adev, i);
306 if (r)
307 return r;
308 }
309
310 amdgpu_vcn_sysfs_reset_mask_fini(adev);
311
312 for (i = 0; i < adev->vcn.num_vcn_inst; i++) {
313 r = amdgpu_vcn_sw_fini(adev, i);
314 if (r)
315 return r;
316 }
317
318 kfree(adev->vcn.ip_dump);
319
320 return 0;
321 }
322
323 /**
324 * vcn_v4_0_hw_init - start and test VCN block
325 *
326 * @ip_block: Pointer to the amdgpu_ip_block for this hw instance.
327 *
328 * Initialize the hardware, boot up the VCPU and do some testing
329 */
vcn_v4_0_hw_init(struct amdgpu_ip_block * ip_block)330 static int vcn_v4_0_hw_init(struct amdgpu_ip_block *ip_block)
331 {
332 struct amdgpu_device *adev = ip_block->adev;
333 struct amdgpu_ring *ring;
334 int i, r;
335
336 if (amdgpu_sriov_vf(adev)) {
337 r = vcn_v4_0_start_sriov(adev);
338 if (r)
339 return r;
340
341 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
342 if (adev->vcn.harvest_config & (1 << i))
343 continue;
344
345 ring = &adev->vcn.inst[i].ring_enc[0];
346 ring->wptr = 0;
347 ring->wptr_old = 0;
348 vcn_v4_0_unified_ring_set_wptr(ring);
349 ring->sched.ready = true;
350 }
351 } else {
352 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
353 if (adev->vcn.harvest_config & (1 << i))
354 continue;
355
356 ring = &adev->vcn.inst[i].ring_enc[0];
357
358 adev->nbio.funcs->vcn_doorbell_range(adev, ring->use_doorbell,
359 ((adev->doorbell_index.vcn.vcn_ring0_1 << 1) + 8 * i), i);
360
361 r = amdgpu_ring_test_helper(ring);
362 if (r)
363 return r;
364 }
365 }
366
367 return 0;
368 }
369
370 /**
371 * vcn_v4_0_hw_fini - stop the hardware block
372 *
373 * @ip_block: Pointer to the amdgpu_ip_block for this hw instance.
374 *
375 * Stop the VCN block, mark ring as not ready any more
376 */
vcn_v4_0_hw_fini(struct amdgpu_ip_block * ip_block)377 static int vcn_v4_0_hw_fini(struct amdgpu_ip_block *ip_block)
378 {
379 struct amdgpu_device *adev = ip_block->adev;
380 int i;
381
382 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
383 struct amdgpu_vcn_inst *vinst = &adev->vcn.inst[i];
384
385 if (adev->vcn.harvest_config & (1 << i))
386 continue;
387
388 cancel_delayed_work_sync(&vinst->idle_work);
389
390 if (!amdgpu_sriov_vf(adev)) {
391 if ((adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG) ||
392 (vinst->cur_state != AMD_PG_STATE_GATE &&
393 RREG32_SOC15(VCN, i, regUVD_STATUS))) {
394 vinst->set_pg_state(vinst, AMD_PG_STATE_GATE);
395 }
396 }
397 if (amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__VCN))
398 amdgpu_irq_put(adev, &vinst->ras_poison_irq, 0);
399 }
400
401 return 0;
402 }
403
404 /**
405 * vcn_v4_0_suspend - suspend VCN block
406 *
407 * @ip_block: Pointer to the amdgpu_ip_block for this hw instance.
408 *
409 * HW fini and suspend VCN block
410 */
vcn_v4_0_suspend(struct amdgpu_ip_block * ip_block)411 static int vcn_v4_0_suspend(struct amdgpu_ip_block *ip_block)
412 {
413 struct amdgpu_device *adev = ip_block->adev;
414 int r, i;
415
416 r = vcn_v4_0_hw_fini(ip_block);
417 if (r)
418 return r;
419
420 for (i = 0; i < adev->vcn.num_vcn_inst; i++) {
421 r = amdgpu_vcn_suspend(ip_block->adev, i);
422 if (r)
423 return r;
424 }
425
426 return 0;
427 }
428
429 /**
430 * vcn_v4_0_resume - resume VCN block
431 *
432 * @ip_block: Pointer to the amdgpu_ip_block for this hw instance.
433 *
434 * Resume firmware and hw init VCN block
435 */
vcn_v4_0_resume(struct amdgpu_ip_block * ip_block)436 static int vcn_v4_0_resume(struct amdgpu_ip_block *ip_block)
437 {
438 struct amdgpu_device *adev = ip_block->adev;
439 int r, i;
440
441 for (i = 0; i < adev->vcn.num_vcn_inst; i++) {
442 r = amdgpu_vcn_resume(ip_block->adev, i);
443 if (r)
444 return r;
445 }
446
447 r = vcn_v4_0_hw_init(ip_block);
448
449 return r;
450 }
451
452 /**
453 * vcn_v4_0_mc_resume - memory controller programming
454 *
455 * @vinst: VCN instance
456 *
457 * Let the VCN memory controller know it's offsets
458 */
vcn_v4_0_mc_resume(struct amdgpu_vcn_inst * vinst)459 static void vcn_v4_0_mc_resume(struct amdgpu_vcn_inst *vinst)
460 {
461 struct amdgpu_device *adev = vinst->adev;
462 int inst = vinst->inst;
463 uint32_t offset, size;
464 const struct common_firmware_header *hdr;
465
466 hdr = (const struct common_firmware_header *)adev->vcn.inst[inst].fw->data;
467 size = AMDGPU_GPU_PAGE_ALIGN(le32_to_cpu(hdr->ucode_size_bytes) + 8);
468
469 /* cache window 0: fw */
470 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
471 WREG32_SOC15(VCN, inst, regUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW,
472 (adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + inst].tmr_mc_addr_lo));
473 WREG32_SOC15(VCN, inst, regUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH,
474 (adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + inst].tmr_mc_addr_hi));
475 WREG32_SOC15(VCN, inst, regUVD_VCPU_CACHE_OFFSET0, 0);
476 offset = 0;
477 } else {
478 WREG32_SOC15(VCN, inst, regUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW,
479 lower_32_bits(adev->vcn.inst[inst].gpu_addr));
480 WREG32_SOC15(VCN, inst, regUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH,
481 upper_32_bits(adev->vcn.inst[inst].gpu_addr));
482 offset = size;
483 WREG32_SOC15(VCN, inst, regUVD_VCPU_CACHE_OFFSET0, AMDGPU_UVD_FIRMWARE_OFFSET >> 3);
484 }
485 WREG32_SOC15(VCN, inst, regUVD_VCPU_CACHE_SIZE0, size);
486
487 /* cache window 1: stack */
488 WREG32_SOC15(VCN, inst, regUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW,
489 lower_32_bits(adev->vcn.inst[inst].gpu_addr + offset));
490 WREG32_SOC15(VCN, inst, regUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH,
491 upper_32_bits(adev->vcn.inst[inst].gpu_addr + offset));
492 WREG32_SOC15(VCN, inst, regUVD_VCPU_CACHE_OFFSET1, 0);
493 WREG32_SOC15(VCN, inst, regUVD_VCPU_CACHE_SIZE1, AMDGPU_VCN_STACK_SIZE);
494
495 /* cache window 2: context */
496 WREG32_SOC15(VCN, inst, regUVD_LMI_VCPU_CACHE2_64BIT_BAR_LOW,
497 lower_32_bits(adev->vcn.inst[inst].gpu_addr + offset + AMDGPU_VCN_STACK_SIZE));
498 WREG32_SOC15(VCN, inst, regUVD_LMI_VCPU_CACHE2_64BIT_BAR_HIGH,
499 upper_32_bits(adev->vcn.inst[inst].gpu_addr + offset + AMDGPU_VCN_STACK_SIZE));
500 WREG32_SOC15(VCN, inst, regUVD_VCPU_CACHE_OFFSET2, 0);
501 WREG32_SOC15(VCN, inst, regUVD_VCPU_CACHE_SIZE2, AMDGPU_VCN_CONTEXT_SIZE);
502
503 /* non-cache window */
504 WREG32_SOC15(VCN, inst, regUVD_LMI_VCPU_NC0_64BIT_BAR_LOW,
505 lower_32_bits(adev->vcn.inst[inst].fw_shared.gpu_addr));
506 WREG32_SOC15(VCN, inst, regUVD_LMI_VCPU_NC0_64BIT_BAR_HIGH,
507 upper_32_bits(adev->vcn.inst[inst].fw_shared.gpu_addr));
508 WREG32_SOC15(VCN, inst, regUVD_VCPU_NONCACHE_OFFSET0, 0);
509 WREG32_SOC15(VCN, inst, regUVD_VCPU_NONCACHE_SIZE0,
510 AMDGPU_GPU_PAGE_ALIGN(sizeof(struct amdgpu_vcn4_fw_shared)));
511 }
512
513 /**
514 * vcn_v4_0_mc_resume_dpg_mode - memory controller programming for dpg mode
515 *
516 * @vinst: VCN instance
517 * @indirect: indirectly write sram
518 *
519 * Let the VCN memory controller know it's offsets with dpg mode
520 */
vcn_v4_0_mc_resume_dpg_mode(struct amdgpu_vcn_inst * vinst,bool indirect)521 static void vcn_v4_0_mc_resume_dpg_mode(struct amdgpu_vcn_inst *vinst,
522 bool indirect)
523 {
524 struct amdgpu_device *adev = vinst->adev;
525 int inst_idx = vinst->inst;
526 uint32_t offset, size;
527 const struct common_firmware_header *hdr;
528 hdr = (const struct common_firmware_header *)adev->vcn.inst[inst_idx].fw->data;
529 size = AMDGPU_GPU_PAGE_ALIGN(le32_to_cpu(hdr->ucode_size_bytes) + 8);
530
531 /* cache window 0: fw */
532 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
533 if (!indirect) {
534 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
535 VCN, inst_idx, regUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW),
536 (adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + inst_idx].tmr_mc_addr_lo), 0, indirect);
537 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
538 VCN, inst_idx, regUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH),
539 (adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + inst_idx].tmr_mc_addr_hi), 0, indirect);
540 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
541 VCN, inst_idx, regUVD_VCPU_CACHE_OFFSET0), 0, 0, indirect);
542 } else {
543 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
544 VCN, inst_idx, regUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW), 0, 0, indirect);
545 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
546 VCN, inst_idx, regUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH), 0, 0, indirect);
547 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
548 VCN, inst_idx, regUVD_VCPU_CACHE_OFFSET0), 0, 0, indirect);
549 }
550 offset = 0;
551 } else {
552 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
553 VCN, inst_idx, regUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW),
554 lower_32_bits(adev->vcn.inst[inst_idx].gpu_addr), 0, indirect);
555 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
556 VCN, inst_idx, regUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH),
557 upper_32_bits(adev->vcn.inst[inst_idx].gpu_addr), 0, indirect);
558 offset = size;
559 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
560 VCN, inst_idx, regUVD_VCPU_CACHE_OFFSET0),
561 AMDGPU_UVD_FIRMWARE_OFFSET >> 3, 0, indirect);
562 }
563
564 if (!indirect)
565 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
566 VCN, inst_idx, regUVD_VCPU_CACHE_SIZE0), size, 0, indirect);
567 else
568 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
569 VCN, inst_idx, regUVD_VCPU_CACHE_SIZE0), 0, 0, indirect);
570
571 /* cache window 1: stack */
572 if (!indirect) {
573 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
574 VCN, inst_idx, regUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW),
575 lower_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset), 0, indirect);
576 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
577 VCN, inst_idx, regUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH),
578 upper_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset), 0, indirect);
579 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
580 VCN, inst_idx, regUVD_VCPU_CACHE_OFFSET1), 0, 0, indirect);
581 } else {
582 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
583 VCN, inst_idx, regUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW), 0, 0, indirect);
584 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
585 VCN, inst_idx, regUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH), 0, 0, indirect);
586 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
587 VCN, inst_idx, regUVD_VCPU_CACHE_OFFSET1), 0, 0, indirect);
588 }
589 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
590 VCN, inst_idx, regUVD_VCPU_CACHE_SIZE1), AMDGPU_VCN_STACK_SIZE, 0, indirect);
591
592 /* cache window 2: context */
593 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
594 VCN, inst_idx, regUVD_LMI_VCPU_CACHE2_64BIT_BAR_LOW),
595 lower_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset + AMDGPU_VCN_STACK_SIZE), 0, indirect);
596 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
597 VCN, inst_idx, regUVD_LMI_VCPU_CACHE2_64BIT_BAR_HIGH),
598 upper_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset + AMDGPU_VCN_STACK_SIZE), 0, indirect);
599 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
600 VCN, inst_idx, regUVD_VCPU_CACHE_OFFSET2), 0, 0, indirect);
601 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
602 VCN, inst_idx, regUVD_VCPU_CACHE_SIZE2), AMDGPU_VCN_CONTEXT_SIZE, 0, indirect);
603
604 /* non-cache window */
605 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
606 VCN, inst_idx, regUVD_LMI_VCPU_NC0_64BIT_BAR_LOW),
607 lower_32_bits(adev->vcn.inst[inst_idx].fw_shared.gpu_addr), 0, indirect);
608 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
609 VCN, inst_idx, regUVD_LMI_VCPU_NC0_64BIT_BAR_HIGH),
610 upper_32_bits(adev->vcn.inst[inst_idx].fw_shared.gpu_addr), 0, indirect);
611 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
612 VCN, inst_idx, regUVD_VCPU_NONCACHE_OFFSET0), 0, 0, indirect);
613 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
614 VCN, inst_idx, regUVD_VCPU_NONCACHE_SIZE0),
615 AMDGPU_GPU_PAGE_ALIGN(sizeof(struct amdgpu_vcn4_fw_shared)), 0, indirect);
616
617 /* VCN global tiling registers */
618 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
619 VCN, inst_idx, regUVD_GFX10_ADDR_CONFIG),
620 adev->gfx.config.gb_addr_config, 0, indirect);
621 }
622
623 /**
624 * vcn_v4_0_disable_static_power_gating - disable VCN static power gating
625 *
626 * @vinst: VCN instance
627 *
628 * Disable static power gating for VCN block
629 */
vcn_v4_0_disable_static_power_gating(struct amdgpu_vcn_inst * vinst)630 static void vcn_v4_0_disable_static_power_gating(struct amdgpu_vcn_inst *vinst)
631 {
632 struct amdgpu_device *adev = vinst->adev;
633 int inst = vinst->inst;
634 uint32_t data = 0;
635
636 if (adev->pg_flags & AMD_PG_SUPPORT_VCN) {
637 data = (1 << UVD_PGFSM_CONFIG__UVDM_PWR_CONFIG__SHIFT
638 | 1 << UVD_PGFSM_CONFIG__UVDS_PWR_CONFIG__SHIFT
639 | 1 << UVD_PGFSM_CONFIG__UVDLM_PWR_CONFIG__SHIFT
640 | 2 << UVD_PGFSM_CONFIG__UVDF_PWR_CONFIG__SHIFT
641 | 2 << UVD_PGFSM_CONFIG__UVDTC_PWR_CONFIG__SHIFT
642 | 2 << UVD_PGFSM_CONFIG__UVDB_PWR_CONFIG__SHIFT
643 | 2 << UVD_PGFSM_CONFIG__UVDTA_PWR_CONFIG__SHIFT
644 | 2 << UVD_PGFSM_CONFIG__UVDTD_PWR_CONFIG__SHIFT
645 | 2 << UVD_PGFSM_CONFIG__UVDTE_PWR_CONFIG__SHIFT
646 | 2 << UVD_PGFSM_CONFIG__UVDE_PWR_CONFIG__SHIFT
647 | 2 << UVD_PGFSM_CONFIG__UVDAB_PWR_CONFIG__SHIFT
648 | 2 << UVD_PGFSM_CONFIG__UVDTB_PWR_CONFIG__SHIFT
649 | 2 << UVD_PGFSM_CONFIG__UVDNA_PWR_CONFIG__SHIFT
650 | 2 << UVD_PGFSM_CONFIG__UVDNB_PWR_CONFIG__SHIFT);
651
652 WREG32_SOC15(VCN, inst, regUVD_PGFSM_CONFIG, data);
653 SOC15_WAIT_ON_RREG(VCN, inst, regUVD_PGFSM_STATUS,
654 UVD_PGFSM_STATUS__UVDM_UVDU_UVDLM_PWR_ON_3_0, 0x3F3FFFFF);
655 } else {
656 uint32_t value;
657
658 value = (inst) ? 0x2200800 : 0;
659 data = (1 << UVD_PGFSM_CONFIG__UVDM_PWR_CONFIG__SHIFT
660 | 1 << UVD_PGFSM_CONFIG__UVDS_PWR_CONFIG__SHIFT
661 | 1 << UVD_PGFSM_CONFIG__UVDLM_PWR_CONFIG__SHIFT
662 | 1 << UVD_PGFSM_CONFIG__UVDF_PWR_CONFIG__SHIFT
663 | 1 << UVD_PGFSM_CONFIG__UVDTC_PWR_CONFIG__SHIFT
664 | 1 << UVD_PGFSM_CONFIG__UVDB_PWR_CONFIG__SHIFT
665 | 1 << UVD_PGFSM_CONFIG__UVDTA_PWR_CONFIG__SHIFT
666 | 1 << UVD_PGFSM_CONFIG__UVDTD_PWR_CONFIG__SHIFT
667 | 1 << UVD_PGFSM_CONFIG__UVDTE_PWR_CONFIG__SHIFT
668 | 1 << UVD_PGFSM_CONFIG__UVDE_PWR_CONFIG__SHIFT
669 | 1 << UVD_PGFSM_CONFIG__UVDAB_PWR_CONFIG__SHIFT
670 | 1 << UVD_PGFSM_CONFIG__UVDTB_PWR_CONFIG__SHIFT
671 | 1 << UVD_PGFSM_CONFIG__UVDNA_PWR_CONFIG__SHIFT
672 | 1 << UVD_PGFSM_CONFIG__UVDNB_PWR_CONFIG__SHIFT);
673
674 WREG32_SOC15(VCN, inst, regUVD_PGFSM_CONFIG, data);
675 SOC15_WAIT_ON_RREG(VCN, inst, regUVD_PGFSM_STATUS, value, 0x3F3FFFFF);
676 }
677
678 data = RREG32_SOC15(VCN, inst, regUVD_POWER_STATUS);
679 data &= ~0x103;
680 if (adev->pg_flags & AMD_PG_SUPPORT_VCN)
681 data |= UVD_PGFSM_CONFIG__UVDM_UVDU_PWR_ON |
682 UVD_POWER_STATUS__UVD_PG_EN_MASK;
683
684 WREG32_SOC15(VCN, inst, regUVD_POWER_STATUS, data);
685
686 return;
687 }
688
689 /**
690 * vcn_v4_0_enable_static_power_gating - enable VCN static power gating
691 *
692 * @vinst: VCN instance
693 *
694 * Enable static power gating for VCN block
695 */
vcn_v4_0_enable_static_power_gating(struct amdgpu_vcn_inst * vinst)696 static void vcn_v4_0_enable_static_power_gating(struct amdgpu_vcn_inst *vinst)
697 {
698 struct amdgpu_device *adev = vinst->adev;
699 int inst = vinst->inst;
700 uint32_t data;
701
702 if (adev->pg_flags & AMD_PG_SUPPORT_VCN) {
703 /* Before power off, this indicator has to be turned on */
704 data = RREG32_SOC15(VCN, inst, regUVD_POWER_STATUS);
705 data &= ~UVD_POWER_STATUS__UVD_POWER_STATUS_MASK;
706 data |= UVD_POWER_STATUS__UVD_POWER_STATUS_TILES_OFF;
707 WREG32_SOC15(VCN, inst, regUVD_POWER_STATUS, data);
708
709 data = (2 << UVD_PGFSM_CONFIG__UVDM_PWR_CONFIG__SHIFT
710 | 2 << UVD_PGFSM_CONFIG__UVDS_PWR_CONFIG__SHIFT
711 | 2 << UVD_PGFSM_CONFIG__UVDF_PWR_CONFIG__SHIFT
712 | 2 << UVD_PGFSM_CONFIG__UVDTC_PWR_CONFIG__SHIFT
713 | 2 << UVD_PGFSM_CONFIG__UVDB_PWR_CONFIG__SHIFT
714 | 2 << UVD_PGFSM_CONFIG__UVDTA_PWR_CONFIG__SHIFT
715 | 2 << UVD_PGFSM_CONFIG__UVDLM_PWR_CONFIG__SHIFT
716 | 2 << UVD_PGFSM_CONFIG__UVDTD_PWR_CONFIG__SHIFT
717 | 2 << UVD_PGFSM_CONFIG__UVDTE_PWR_CONFIG__SHIFT
718 | 2 << UVD_PGFSM_CONFIG__UVDE_PWR_CONFIG__SHIFT
719 | 2 << UVD_PGFSM_CONFIG__UVDAB_PWR_CONFIG__SHIFT
720 | 2 << UVD_PGFSM_CONFIG__UVDTB_PWR_CONFIG__SHIFT
721 | 2 << UVD_PGFSM_CONFIG__UVDNA_PWR_CONFIG__SHIFT
722 | 2 << UVD_PGFSM_CONFIG__UVDNB_PWR_CONFIG__SHIFT);
723 WREG32_SOC15(VCN, inst, regUVD_PGFSM_CONFIG, data);
724
725 data = (2 << UVD_PGFSM_STATUS__UVDM_PWR_STATUS__SHIFT
726 | 2 << UVD_PGFSM_STATUS__UVDS_PWR_STATUS__SHIFT
727 | 2 << UVD_PGFSM_STATUS__UVDF_PWR_STATUS__SHIFT
728 | 2 << UVD_PGFSM_STATUS__UVDTC_PWR_STATUS__SHIFT
729 | 2 << UVD_PGFSM_STATUS__UVDB_PWR_STATUS__SHIFT
730 | 2 << UVD_PGFSM_STATUS__UVDTA_PWR_STATUS__SHIFT
731 | 2 << UVD_PGFSM_STATUS__UVDLM_PWR_STATUS__SHIFT
732 | 2 << UVD_PGFSM_STATUS__UVDTD_PWR_STATUS__SHIFT
733 | 2 << UVD_PGFSM_STATUS__UVDTE_PWR_STATUS__SHIFT
734 | 2 << UVD_PGFSM_STATUS__UVDE_PWR_STATUS__SHIFT
735 | 2 << UVD_PGFSM_STATUS__UVDAB_PWR_STATUS__SHIFT
736 | 2 << UVD_PGFSM_STATUS__UVDTB_PWR_STATUS__SHIFT
737 | 2 << UVD_PGFSM_STATUS__UVDNA_PWR_STATUS__SHIFT
738 | 2 << UVD_PGFSM_STATUS__UVDNB_PWR_STATUS__SHIFT);
739 SOC15_WAIT_ON_RREG(VCN, inst, regUVD_PGFSM_STATUS, data, 0x3F3FFFFF);
740 }
741
742 return;
743 }
744
745 /**
746 * vcn_v4_0_disable_clock_gating - disable VCN clock gating
747 *
748 * @vinst: VCN instance
749 *
750 * Disable clock gating for VCN block
751 */
vcn_v4_0_disable_clock_gating(struct amdgpu_vcn_inst * vinst)752 static void vcn_v4_0_disable_clock_gating(struct amdgpu_vcn_inst *vinst)
753 {
754 struct amdgpu_device *adev = vinst->adev;
755 int inst = vinst->inst;
756 uint32_t data;
757
758 if (adev->cg_flags & AMD_CG_SUPPORT_VCN_MGCG)
759 return;
760
761 /* VCN disable CGC */
762 data = RREG32_SOC15(VCN, inst, regUVD_CGC_CTRL);
763 data &= ~UVD_CGC_CTRL__DYN_CLOCK_MODE_MASK;
764 data |= 1 << UVD_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT;
765 data |= 4 << UVD_CGC_CTRL__CLK_OFF_DELAY__SHIFT;
766 WREG32_SOC15(VCN, inst, regUVD_CGC_CTRL, data);
767
768 data = RREG32_SOC15(VCN, inst, regUVD_CGC_GATE);
769 data &= ~(UVD_CGC_GATE__SYS_MASK
770 | UVD_CGC_GATE__UDEC_MASK
771 | UVD_CGC_GATE__MPEG2_MASK
772 | UVD_CGC_GATE__REGS_MASK
773 | UVD_CGC_GATE__RBC_MASK
774 | UVD_CGC_GATE__LMI_MC_MASK
775 | UVD_CGC_GATE__LMI_UMC_MASK
776 | UVD_CGC_GATE__IDCT_MASK
777 | UVD_CGC_GATE__MPRD_MASK
778 | UVD_CGC_GATE__MPC_MASK
779 | UVD_CGC_GATE__LBSI_MASK
780 | UVD_CGC_GATE__LRBBM_MASK
781 | UVD_CGC_GATE__UDEC_RE_MASK
782 | UVD_CGC_GATE__UDEC_CM_MASK
783 | UVD_CGC_GATE__UDEC_IT_MASK
784 | UVD_CGC_GATE__UDEC_DB_MASK
785 | UVD_CGC_GATE__UDEC_MP_MASK
786 | UVD_CGC_GATE__WCB_MASK
787 | UVD_CGC_GATE__VCPU_MASK
788 | UVD_CGC_GATE__MMSCH_MASK);
789
790 WREG32_SOC15(VCN, inst, regUVD_CGC_GATE, data);
791 SOC15_WAIT_ON_RREG(VCN, inst, regUVD_CGC_GATE, 0, 0xFFFFFFFF);
792
793 data = RREG32_SOC15(VCN, inst, regUVD_CGC_CTRL);
794 data &= ~(UVD_CGC_CTRL__UDEC_RE_MODE_MASK
795 | UVD_CGC_CTRL__UDEC_CM_MODE_MASK
796 | UVD_CGC_CTRL__UDEC_IT_MODE_MASK
797 | UVD_CGC_CTRL__UDEC_DB_MODE_MASK
798 | UVD_CGC_CTRL__UDEC_MP_MODE_MASK
799 | UVD_CGC_CTRL__SYS_MODE_MASK
800 | UVD_CGC_CTRL__UDEC_MODE_MASK
801 | UVD_CGC_CTRL__MPEG2_MODE_MASK
802 | UVD_CGC_CTRL__REGS_MODE_MASK
803 | UVD_CGC_CTRL__RBC_MODE_MASK
804 | UVD_CGC_CTRL__LMI_MC_MODE_MASK
805 | UVD_CGC_CTRL__LMI_UMC_MODE_MASK
806 | UVD_CGC_CTRL__IDCT_MODE_MASK
807 | UVD_CGC_CTRL__MPRD_MODE_MASK
808 | UVD_CGC_CTRL__MPC_MODE_MASK
809 | UVD_CGC_CTRL__LBSI_MODE_MASK
810 | UVD_CGC_CTRL__LRBBM_MODE_MASK
811 | UVD_CGC_CTRL__WCB_MODE_MASK
812 | UVD_CGC_CTRL__VCPU_MODE_MASK
813 | UVD_CGC_CTRL__MMSCH_MODE_MASK);
814 WREG32_SOC15(VCN, inst, regUVD_CGC_CTRL, data);
815
816 data = RREG32_SOC15(VCN, inst, regUVD_SUVD_CGC_GATE);
817 data |= (UVD_SUVD_CGC_GATE__SRE_MASK
818 | UVD_SUVD_CGC_GATE__SIT_MASK
819 | UVD_SUVD_CGC_GATE__SMP_MASK
820 | UVD_SUVD_CGC_GATE__SCM_MASK
821 | UVD_SUVD_CGC_GATE__SDB_MASK
822 | UVD_SUVD_CGC_GATE__SRE_H264_MASK
823 | UVD_SUVD_CGC_GATE__SRE_HEVC_MASK
824 | UVD_SUVD_CGC_GATE__SIT_H264_MASK
825 | UVD_SUVD_CGC_GATE__SIT_HEVC_MASK
826 | UVD_SUVD_CGC_GATE__SCM_H264_MASK
827 | UVD_SUVD_CGC_GATE__SCM_HEVC_MASK
828 | UVD_SUVD_CGC_GATE__SDB_H264_MASK
829 | UVD_SUVD_CGC_GATE__SDB_HEVC_MASK
830 | UVD_SUVD_CGC_GATE__SCLR_MASK
831 | UVD_SUVD_CGC_GATE__UVD_SC_MASK
832 | UVD_SUVD_CGC_GATE__ENT_MASK
833 | UVD_SUVD_CGC_GATE__SIT_HEVC_DEC_MASK
834 | UVD_SUVD_CGC_GATE__SIT_HEVC_ENC_MASK
835 | UVD_SUVD_CGC_GATE__SITE_MASK
836 | UVD_SUVD_CGC_GATE__SRE_VP9_MASK
837 | UVD_SUVD_CGC_GATE__SCM_VP9_MASK
838 | UVD_SUVD_CGC_GATE__SIT_VP9_DEC_MASK
839 | UVD_SUVD_CGC_GATE__SDB_VP9_MASK
840 | UVD_SUVD_CGC_GATE__IME_HEVC_MASK);
841 WREG32_SOC15(VCN, inst, regUVD_SUVD_CGC_GATE, data);
842
843 data = RREG32_SOC15(VCN, inst, regUVD_SUVD_CGC_CTRL);
844 data &= ~(UVD_SUVD_CGC_CTRL__SRE_MODE_MASK
845 | UVD_SUVD_CGC_CTRL__SIT_MODE_MASK
846 | UVD_SUVD_CGC_CTRL__SMP_MODE_MASK
847 | UVD_SUVD_CGC_CTRL__SCM_MODE_MASK
848 | UVD_SUVD_CGC_CTRL__SDB_MODE_MASK
849 | UVD_SUVD_CGC_CTRL__SCLR_MODE_MASK
850 | UVD_SUVD_CGC_CTRL__UVD_SC_MODE_MASK
851 | UVD_SUVD_CGC_CTRL__ENT_MODE_MASK
852 | UVD_SUVD_CGC_CTRL__IME_MODE_MASK
853 | UVD_SUVD_CGC_CTRL__SITE_MODE_MASK);
854 WREG32_SOC15(VCN, inst, regUVD_SUVD_CGC_CTRL, data);
855 }
856
857 /**
858 * vcn_v4_0_disable_clock_gating_dpg_mode - disable VCN clock gating dpg mode
859 *
860 * @vinst: VCN instance
861 * @sram_sel: sram select
862 * @indirect: indirectly write sram
863 *
864 * Disable clock gating for VCN block with dpg mode
865 */
vcn_v4_0_disable_clock_gating_dpg_mode(struct amdgpu_vcn_inst * vinst,uint8_t sram_sel,uint8_t indirect)866 static void vcn_v4_0_disable_clock_gating_dpg_mode(struct amdgpu_vcn_inst *vinst,
867 uint8_t sram_sel,
868 uint8_t indirect)
869 {
870 struct amdgpu_device *adev = vinst->adev;
871 int inst_idx = vinst->inst;
872 uint32_t reg_data = 0;
873
874 if (adev->cg_flags & AMD_CG_SUPPORT_VCN_MGCG)
875 return;
876
877 /* enable sw clock gating control */
878 reg_data = 0 << UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
879 reg_data |= 1 << UVD_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT;
880 reg_data |= 4 << UVD_CGC_CTRL__CLK_OFF_DELAY__SHIFT;
881 reg_data &= ~(UVD_CGC_CTRL__UDEC_RE_MODE_MASK |
882 UVD_CGC_CTRL__UDEC_CM_MODE_MASK |
883 UVD_CGC_CTRL__UDEC_IT_MODE_MASK |
884 UVD_CGC_CTRL__UDEC_DB_MODE_MASK |
885 UVD_CGC_CTRL__UDEC_MP_MODE_MASK |
886 UVD_CGC_CTRL__SYS_MODE_MASK |
887 UVD_CGC_CTRL__UDEC_MODE_MASK |
888 UVD_CGC_CTRL__MPEG2_MODE_MASK |
889 UVD_CGC_CTRL__REGS_MODE_MASK |
890 UVD_CGC_CTRL__RBC_MODE_MASK |
891 UVD_CGC_CTRL__LMI_MC_MODE_MASK |
892 UVD_CGC_CTRL__LMI_UMC_MODE_MASK |
893 UVD_CGC_CTRL__IDCT_MODE_MASK |
894 UVD_CGC_CTRL__MPRD_MODE_MASK |
895 UVD_CGC_CTRL__MPC_MODE_MASK |
896 UVD_CGC_CTRL__LBSI_MODE_MASK |
897 UVD_CGC_CTRL__LRBBM_MODE_MASK |
898 UVD_CGC_CTRL__WCB_MODE_MASK |
899 UVD_CGC_CTRL__VCPU_MODE_MASK);
900 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
901 VCN, inst_idx, regUVD_CGC_CTRL), reg_data, sram_sel, indirect);
902
903 /* turn off clock gating */
904 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
905 VCN, inst_idx, regUVD_CGC_GATE), 0, sram_sel, indirect);
906
907 /* turn on SUVD clock gating */
908 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
909 VCN, inst_idx, regUVD_SUVD_CGC_GATE), 1, sram_sel, indirect);
910
911 /* turn on sw mode in UVD_SUVD_CGC_CTRL */
912 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
913 VCN, inst_idx, regUVD_SUVD_CGC_CTRL), 0, sram_sel, indirect);
914 }
915
916 /**
917 * vcn_v4_0_enable_clock_gating - enable VCN clock gating
918 *
919 * @vinst: VCN instance
920 *
921 * Enable clock gating for VCN block
922 */
vcn_v4_0_enable_clock_gating(struct amdgpu_vcn_inst * vinst)923 static void vcn_v4_0_enable_clock_gating(struct amdgpu_vcn_inst *vinst)
924 {
925 struct amdgpu_device *adev = vinst->adev;
926 int inst = vinst->inst;
927 uint32_t data;
928
929 if (adev->cg_flags & AMD_CG_SUPPORT_VCN_MGCG)
930 return;
931
932 /* enable VCN CGC */
933 data = RREG32_SOC15(VCN, inst, regUVD_CGC_CTRL);
934 data |= 0 << UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
935 data |= 1 << UVD_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT;
936 data |= 4 << UVD_CGC_CTRL__CLK_OFF_DELAY__SHIFT;
937 WREG32_SOC15(VCN, inst, regUVD_CGC_CTRL, data);
938
939 data = RREG32_SOC15(VCN, inst, regUVD_CGC_CTRL);
940 data |= (UVD_CGC_CTRL__UDEC_RE_MODE_MASK
941 | UVD_CGC_CTRL__UDEC_CM_MODE_MASK
942 | UVD_CGC_CTRL__UDEC_IT_MODE_MASK
943 | UVD_CGC_CTRL__UDEC_DB_MODE_MASK
944 | UVD_CGC_CTRL__UDEC_MP_MODE_MASK
945 | UVD_CGC_CTRL__SYS_MODE_MASK
946 | UVD_CGC_CTRL__UDEC_MODE_MASK
947 | UVD_CGC_CTRL__MPEG2_MODE_MASK
948 | UVD_CGC_CTRL__REGS_MODE_MASK
949 | UVD_CGC_CTRL__RBC_MODE_MASK
950 | UVD_CGC_CTRL__LMI_MC_MODE_MASK
951 | UVD_CGC_CTRL__LMI_UMC_MODE_MASK
952 | UVD_CGC_CTRL__IDCT_MODE_MASK
953 | UVD_CGC_CTRL__MPRD_MODE_MASK
954 | UVD_CGC_CTRL__MPC_MODE_MASK
955 | UVD_CGC_CTRL__LBSI_MODE_MASK
956 | UVD_CGC_CTRL__LRBBM_MODE_MASK
957 | UVD_CGC_CTRL__WCB_MODE_MASK
958 | UVD_CGC_CTRL__VCPU_MODE_MASK
959 | UVD_CGC_CTRL__MMSCH_MODE_MASK);
960 WREG32_SOC15(VCN, inst, regUVD_CGC_CTRL, data);
961
962 data = RREG32_SOC15(VCN, inst, regUVD_SUVD_CGC_CTRL);
963 data |= (UVD_SUVD_CGC_CTRL__SRE_MODE_MASK
964 | UVD_SUVD_CGC_CTRL__SIT_MODE_MASK
965 | UVD_SUVD_CGC_CTRL__SMP_MODE_MASK
966 | UVD_SUVD_CGC_CTRL__SCM_MODE_MASK
967 | UVD_SUVD_CGC_CTRL__SDB_MODE_MASK
968 | UVD_SUVD_CGC_CTRL__SCLR_MODE_MASK
969 | UVD_SUVD_CGC_CTRL__UVD_SC_MODE_MASK
970 | UVD_SUVD_CGC_CTRL__ENT_MODE_MASK
971 | UVD_SUVD_CGC_CTRL__IME_MODE_MASK
972 | UVD_SUVD_CGC_CTRL__SITE_MODE_MASK);
973 WREG32_SOC15(VCN, inst, regUVD_SUVD_CGC_CTRL, data);
974 }
975
vcn_v4_0_enable_ras(struct amdgpu_vcn_inst * vinst,bool indirect)976 static void vcn_v4_0_enable_ras(struct amdgpu_vcn_inst *vinst,
977 bool indirect)
978 {
979 struct amdgpu_device *adev = vinst->adev;
980 int inst_idx = vinst->inst;
981 uint32_t tmp;
982
983 if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__VCN))
984 return;
985
986 tmp = VCN_RAS_CNTL__VCPU_VCODEC_REARM_MASK |
987 VCN_RAS_CNTL__VCPU_VCODEC_IH_EN_MASK |
988 VCN_RAS_CNTL__VCPU_VCODEC_PMI_EN_MASK |
989 VCN_RAS_CNTL__VCPU_VCODEC_STALL_EN_MASK;
990 WREG32_SOC15_DPG_MODE(inst_idx,
991 SOC15_DPG_MODE_OFFSET(VCN, 0, regVCN_RAS_CNTL),
992 tmp, 0, indirect);
993
994 tmp = UVD_SYS_INT_EN__RASCNTL_VCPU_VCODEC_EN_MASK;
995 WREG32_SOC15_DPG_MODE(inst_idx,
996 SOC15_DPG_MODE_OFFSET(VCN, 0, regUVD_SYS_INT_EN),
997 tmp, 0, indirect);
998 }
999
1000 /**
1001 * vcn_v4_0_start_dpg_mode - VCN start with dpg mode
1002 *
1003 * @vinst: VCN instance
1004 * @indirect: indirectly write sram
1005 *
1006 * Start VCN block with dpg mode
1007 */
vcn_v4_0_start_dpg_mode(struct amdgpu_vcn_inst * vinst,bool indirect)1008 static int vcn_v4_0_start_dpg_mode(struct amdgpu_vcn_inst *vinst, bool indirect)
1009 {
1010 struct amdgpu_device *adev = vinst->adev;
1011 int inst_idx = vinst->inst;
1012 volatile struct amdgpu_vcn4_fw_shared *fw_shared = adev->vcn.inst[inst_idx].fw_shared.cpu_addr;
1013 struct amdgpu_ring *ring;
1014 uint32_t tmp;
1015
1016 /* disable register anti-hang mechanism */
1017 WREG32_P(SOC15_REG_OFFSET(VCN, inst_idx, regUVD_POWER_STATUS), 1,
1018 ~UVD_POWER_STATUS__UVD_POWER_STATUS_MASK);
1019 /* enable dynamic power gating mode */
1020 tmp = RREG32_SOC15(VCN, inst_idx, regUVD_POWER_STATUS);
1021 tmp |= UVD_POWER_STATUS__UVD_PG_MODE_MASK;
1022 tmp |= UVD_POWER_STATUS__UVD_PG_EN_MASK;
1023 WREG32_SOC15(VCN, inst_idx, regUVD_POWER_STATUS, tmp);
1024
1025 if (indirect)
1026 adev->vcn.inst[inst_idx].dpg_sram_curr_addr = (uint32_t *)adev->vcn.inst[inst_idx].dpg_sram_cpu_addr;
1027
1028 /* enable clock gating */
1029 vcn_v4_0_disable_clock_gating_dpg_mode(vinst, 0, indirect);
1030
1031 /* enable VCPU clock */
1032 tmp = (0xFF << UVD_VCPU_CNTL__PRB_TIMEOUT_VAL__SHIFT);
1033 tmp |= UVD_VCPU_CNTL__CLK_EN_MASK | UVD_VCPU_CNTL__BLK_RST_MASK;
1034 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
1035 VCN, inst_idx, regUVD_VCPU_CNTL), tmp, 0, indirect);
1036
1037 /* disable master interupt */
1038 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
1039 VCN, inst_idx, regUVD_MASTINT_EN), 0, 0, indirect);
1040
1041 /* setup regUVD_LMI_CTRL */
1042 tmp = (UVD_LMI_CTRL__WRITE_CLEAN_TIMER_EN_MASK |
1043 UVD_LMI_CTRL__REQ_MODE_MASK |
1044 UVD_LMI_CTRL__CRC_RESET_MASK |
1045 UVD_LMI_CTRL__MASK_MC_URGENT_MASK |
1046 UVD_LMI_CTRL__DATA_COHERENCY_EN_MASK |
1047 UVD_LMI_CTRL__VCPU_DATA_COHERENCY_EN_MASK |
1048 (8 << UVD_LMI_CTRL__WRITE_CLEAN_TIMER__SHIFT) |
1049 0x00100000L);
1050 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
1051 VCN, inst_idx, regUVD_LMI_CTRL), tmp, 0, indirect);
1052
1053 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
1054 VCN, inst_idx, regUVD_MPC_CNTL),
1055 0x2 << UVD_MPC_CNTL__REPLACEMENT_MODE__SHIFT, 0, indirect);
1056
1057 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
1058 VCN, inst_idx, regUVD_MPC_SET_MUXA0),
1059 ((0x1 << UVD_MPC_SET_MUXA0__VARA_1__SHIFT) |
1060 (0x2 << UVD_MPC_SET_MUXA0__VARA_2__SHIFT) |
1061 (0x3 << UVD_MPC_SET_MUXA0__VARA_3__SHIFT) |
1062 (0x4 << UVD_MPC_SET_MUXA0__VARA_4__SHIFT)), 0, indirect);
1063
1064 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
1065 VCN, inst_idx, regUVD_MPC_SET_MUXB0),
1066 ((0x1 << UVD_MPC_SET_MUXB0__VARB_1__SHIFT) |
1067 (0x2 << UVD_MPC_SET_MUXB0__VARB_2__SHIFT) |
1068 (0x3 << UVD_MPC_SET_MUXB0__VARB_3__SHIFT) |
1069 (0x4 << UVD_MPC_SET_MUXB0__VARB_4__SHIFT)), 0, indirect);
1070
1071 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
1072 VCN, inst_idx, regUVD_MPC_SET_MUX),
1073 ((0x0 << UVD_MPC_SET_MUX__SET_0__SHIFT) |
1074 (0x1 << UVD_MPC_SET_MUX__SET_1__SHIFT) |
1075 (0x2 << UVD_MPC_SET_MUX__SET_2__SHIFT)), 0, indirect);
1076
1077 vcn_v4_0_mc_resume_dpg_mode(vinst, indirect);
1078
1079 tmp = (0xFF << UVD_VCPU_CNTL__PRB_TIMEOUT_VAL__SHIFT);
1080 tmp |= UVD_VCPU_CNTL__CLK_EN_MASK;
1081 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
1082 VCN, inst_idx, regUVD_VCPU_CNTL), tmp, 0, indirect);
1083
1084 /* enable LMI MC and UMC channels */
1085 tmp = 0x1f << UVD_LMI_CTRL2__RE_OFLD_MIF_WR_REQ_NUM__SHIFT;
1086 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
1087 VCN, inst_idx, regUVD_LMI_CTRL2), tmp, 0, indirect);
1088
1089 vcn_v4_0_enable_ras(vinst, indirect);
1090
1091 /* enable master interrupt */
1092 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
1093 VCN, inst_idx, regUVD_MASTINT_EN),
1094 UVD_MASTINT_EN__VCPU_EN_MASK, 0, indirect);
1095
1096
1097 if (indirect)
1098 amdgpu_vcn_psp_update_sram(adev, inst_idx, 0);
1099
1100 ring = &adev->vcn.inst[inst_idx].ring_enc[0];
1101
1102 WREG32_SOC15(VCN, inst_idx, regUVD_RB_BASE_LO, ring->gpu_addr);
1103 WREG32_SOC15(VCN, inst_idx, regUVD_RB_BASE_HI, upper_32_bits(ring->gpu_addr));
1104 WREG32_SOC15(VCN, inst_idx, regUVD_RB_SIZE, ring->ring_size / 4);
1105
1106 tmp = RREG32_SOC15(VCN, inst_idx, regVCN_RB_ENABLE);
1107 tmp &= ~(VCN_RB_ENABLE__RB1_EN_MASK);
1108 WREG32_SOC15(VCN, inst_idx, regVCN_RB_ENABLE, tmp);
1109 fw_shared->sq.queue_mode |= FW_QUEUE_RING_RESET;
1110 WREG32_SOC15(VCN, inst_idx, regUVD_RB_RPTR, 0);
1111 WREG32_SOC15(VCN, inst_idx, regUVD_RB_WPTR, 0);
1112
1113 tmp = RREG32_SOC15(VCN, inst_idx, regUVD_RB_RPTR);
1114 WREG32_SOC15(VCN, inst_idx, regUVD_RB_WPTR, tmp);
1115 ring->wptr = RREG32_SOC15(VCN, inst_idx, regUVD_RB_WPTR);
1116
1117 tmp = RREG32_SOC15(VCN, inst_idx, regVCN_RB_ENABLE);
1118 tmp |= VCN_RB_ENABLE__RB1_EN_MASK;
1119 WREG32_SOC15(VCN, inst_idx, regVCN_RB_ENABLE, tmp);
1120 fw_shared->sq.queue_mode &= ~(FW_QUEUE_RING_RESET | FW_QUEUE_DPG_HOLD_OFF);
1121
1122 WREG32_SOC15(VCN, inst_idx, regVCN_RB1_DB_CTRL,
1123 ring->doorbell_index << VCN_RB1_DB_CTRL__OFFSET__SHIFT |
1124 VCN_RB1_DB_CTRL__EN_MASK);
1125
1126 /* Keeping one read-back to ensure all register writes are done,
1127 * otherwise it may introduce race conditions.
1128 */
1129 RREG32_SOC15(VCN, inst_idx, regUVD_STATUS);
1130
1131 return 0;
1132 }
1133
1134
1135 /**
1136 * vcn_v4_0_start - VCN start
1137 *
1138 * @vinst: VCN instance
1139 *
1140 * Start VCN block
1141 */
vcn_v4_0_start(struct amdgpu_vcn_inst * vinst)1142 static int vcn_v4_0_start(struct amdgpu_vcn_inst *vinst)
1143 {
1144 struct amdgpu_device *adev = vinst->adev;
1145 int i = vinst->inst;
1146 volatile struct amdgpu_vcn4_fw_shared *fw_shared;
1147 struct amdgpu_ring *ring;
1148 uint32_t tmp;
1149 int j, k, r;
1150
1151 if (adev->vcn.harvest_config & (1 << i))
1152 return 0;
1153
1154 if (adev->pm.dpm_enabled)
1155 amdgpu_dpm_enable_vcn(adev, true, i);
1156
1157 fw_shared = adev->vcn.inst[i].fw_shared.cpu_addr;
1158
1159 if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG)
1160 return vcn_v4_0_start_dpg_mode(vinst, adev->vcn.inst[i].indirect_sram);
1161
1162 /* disable VCN power gating */
1163 vcn_v4_0_disable_static_power_gating(vinst);
1164
1165 /* set VCN status busy */
1166 tmp = RREG32_SOC15(VCN, i, regUVD_STATUS) | UVD_STATUS__UVD_BUSY;
1167 WREG32_SOC15(VCN, i, regUVD_STATUS, tmp);
1168
1169 /*SW clock gating */
1170 vcn_v4_0_disable_clock_gating(vinst);
1171
1172 /* enable VCPU clock */
1173 WREG32_P(SOC15_REG_OFFSET(VCN, i, regUVD_VCPU_CNTL),
1174 UVD_VCPU_CNTL__CLK_EN_MASK, ~UVD_VCPU_CNTL__CLK_EN_MASK);
1175
1176 /* disable master interrupt */
1177 WREG32_P(SOC15_REG_OFFSET(VCN, i, regUVD_MASTINT_EN), 0,
1178 ~UVD_MASTINT_EN__VCPU_EN_MASK);
1179
1180 /* enable LMI MC and UMC channels */
1181 WREG32_P(SOC15_REG_OFFSET(VCN, i, regUVD_LMI_CTRL2), 0,
1182 ~UVD_LMI_CTRL2__STALL_ARB_UMC_MASK);
1183
1184 tmp = RREG32_SOC15(VCN, i, regUVD_SOFT_RESET);
1185 tmp &= ~UVD_SOFT_RESET__LMI_SOFT_RESET_MASK;
1186 tmp &= ~UVD_SOFT_RESET__LMI_UMC_SOFT_RESET_MASK;
1187 WREG32_SOC15(VCN, i, regUVD_SOFT_RESET, tmp);
1188
1189 /* setup regUVD_LMI_CTRL */
1190 tmp = RREG32_SOC15(VCN, i, regUVD_LMI_CTRL);
1191 WREG32_SOC15(VCN, i, regUVD_LMI_CTRL, tmp |
1192 UVD_LMI_CTRL__WRITE_CLEAN_TIMER_EN_MASK |
1193 UVD_LMI_CTRL__MASK_MC_URGENT_MASK |
1194 UVD_LMI_CTRL__DATA_COHERENCY_EN_MASK |
1195 UVD_LMI_CTRL__VCPU_DATA_COHERENCY_EN_MASK);
1196
1197 /* setup regUVD_MPC_CNTL */
1198 tmp = RREG32_SOC15(VCN, i, regUVD_MPC_CNTL);
1199 tmp &= ~UVD_MPC_CNTL__REPLACEMENT_MODE_MASK;
1200 tmp |= 0x2 << UVD_MPC_CNTL__REPLACEMENT_MODE__SHIFT;
1201 WREG32_SOC15(VCN, i, regUVD_MPC_CNTL, tmp);
1202
1203 /* setup UVD_MPC_SET_MUXA0 */
1204 WREG32_SOC15(VCN, i, regUVD_MPC_SET_MUXA0,
1205 ((0x1 << UVD_MPC_SET_MUXA0__VARA_1__SHIFT) |
1206 (0x2 << UVD_MPC_SET_MUXA0__VARA_2__SHIFT) |
1207 (0x3 << UVD_MPC_SET_MUXA0__VARA_3__SHIFT) |
1208 (0x4 << UVD_MPC_SET_MUXA0__VARA_4__SHIFT)));
1209
1210 /* setup UVD_MPC_SET_MUXB0 */
1211 WREG32_SOC15(VCN, i, regUVD_MPC_SET_MUXB0,
1212 ((0x1 << UVD_MPC_SET_MUXB0__VARB_1__SHIFT) |
1213 (0x2 << UVD_MPC_SET_MUXB0__VARB_2__SHIFT) |
1214 (0x3 << UVD_MPC_SET_MUXB0__VARB_3__SHIFT) |
1215 (0x4 << UVD_MPC_SET_MUXB0__VARB_4__SHIFT)));
1216
1217 /* setup UVD_MPC_SET_MUX */
1218 WREG32_SOC15(VCN, i, regUVD_MPC_SET_MUX,
1219 ((0x0 << UVD_MPC_SET_MUX__SET_0__SHIFT) |
1220 (0x1 << UVD_MPC_SET_MUX__SET_1__SHIFT) |
1221 (0x2 << UVD_MPC_SET_MUX__SET_2__SHIFT)));
1222
1223 vcn_v4_0_mc_resume(vinst);
1224
1225 /* VCN global tiling registers */
1226 WREG32_SOC15(VCN, i, regUVD_GFX10_ADDR_CONFIG,
1227 adev->gfx.config.gb_addr_config);
1228
1229 /* unblock VCPU register access */
1230 WREG32_P(SOC15_REG_OFFSET(VCN, i, regUVD_RB_ARB_CTRL), 0,
1231 ~UVD_RB_ARB_CTRL__VCPU_DIS_MASK);
1232
1233 /* release VCPU reset to boot */
1234 WREG32_P(SOC15_REG_OFFSET(VCN, i, regUVD_VCPU_CNTL), 0,
1235 ~UVD_VCPU_CNTL__BLK_RST_MASK);
1236
1237 for (j = 0; j < 10; ++j) {
1238 uint32_t status;
1239
1240 for (k = 0; k < 100; ++k) {
1241 status = RREG32_SOC15(VCN, i, regUVD_STATUS);
1242 if (status & 2)
1243 break;
1244 mdelay(10);
1245 if (amdgpu_emu_mode == 1)
1246 msleep(1);
1247 }
1248
1249 if (amdgpu_emu_mode == 1) {
1250 r = -1;
1251 if (status & 2) {
1252 r = 0;
1253 break;
1254 }
1255 } else {
1256 r = 0;
1257 if (status & 2)
1258 break;
1259
1260 dev_err(adev->dev, "VCN[%d] is not responding, trying to reset the VCPU!!!\n", i);
1261 WREG32_P(SOC15_REG_OFFSET(VCN, i, regUVD_VCPU_CNTL),
1262 UVD_VCPU_CNTL__BLK_RST_MASK,
1263 ~UVD_VCPU_CNTL__BLK_RST_MASK);
1264 mdelay(10);
1265 WREG32_P(SOC15_REG_OFFSET(VCN, i, regUVD_VCPU_CNTL), 0,
1266 ~UVD_VCPU_CNTL__BLK_RST_MASK);
1267
1268 mdelay(10);
1269 r = -1;
1270 }
1271 }
1272
1273 if (r) {
1274 dev_err(adev->dev, "VCN[%d] is not responding, giving up!!!\n", i);
1275 return r;
1276 }
1277
1278 /* enable master interrupt */
1279 WREG32_P(SOC15_REG_OFFSET(VCN, i, regUVD_MASTINT_EN),
1280 UVD_MASTINT_EN__VCPU_EN_MASK,
1281 ~UVD_MASTINT_EN__VCPU_EN_MASK);
1282
1283 /* clear the busy bit of VCN_STATUS */
1284 WREG32_P(SOC15_REG_OFFSET(VCN, i, regUVD_STATUS), 0,
1285 ~(2 << UVD_STATUS__VCPU_REPORT__SHIFT));
1286
1287 ring = &adev->vcn.inst[i].ring_enc[0];
1288 WREG32_SOC15(VCN, i, regVCN_RB1_DB_CTRL,
1289 ring->doorbell_index << VCN_RB1_DB_CTRL__OFFSET__SHIFT |
1290 VCN_RB1_DB_CTRL__EN_MASK);
1291
1292 WREG32_SOC15(VCN, i, regUVD_RB_BASE_LO, ring->gpu_addr);
1293 WREG32_SOC15(VCN, i, regUVD_RB_BASE_HI, upper_32_bits(ring->gpu_addr));
1294 WREG32_SOC15(VCN, i, regUVD_RB_SIZE, ring->ring_size / 4);
1295
1296 tmp = RREG32_SOC15(VCN, i, regVCN_RB_ENABLE);
1297 tmp &= ~(VCN_RB_ENABLE__RB1_EN_MASK);
1298 WREG32_SOC15(VCN, i, regVCN_RB_ENABLE, tmp);
1299 fw_shared->sq.queue_mode |= FW_QUEUE_RING_RESET;
1300 WREG32_SOC15(VCN, i, regUVD_RB_RPTR, 0);
1301 WREG32_SOC15(VCN, i, regUVD_RB_WPTR, 0);
1302
1303 tmp = RREG32_SOC15(VCN, i, regUVD_RB_RPTR);
1304 WREG32_SOC15(VCN, i, regUVD_RB_WPTR, tmp);
1305 ring->wptr = RREG32_SOC15(VCN, i, regUVD_RB_WPTR);
1306
1307 tmp = RREG32_SOC15(VCN, i, regVCN_RB_ENABLE);
1308 tmp |= VCN_RB_ENABLE__RB1_EN_MASK;
1309 WREG32_SOC15(VCN, i, regVCN_RB_ENABLE, tmp);
1310 fw_shared->sq.queue_mode &= ~(FW_QUEUE_RING_RESET | FW_QUEUE_DPG_HOLD_OFF);
1311
1312 /* Keeping one read-back to ensure all register writes are done,
1313 * otherwise it may introduce race conditions.
1314 */
1315 RREG32_SOC15(VCN, i, regUVD_STATUS);
1316
1317 return 0;
1318 }
1319
vcn_v4_0_init_ring_metadata(struct amdgpu_device * adev,uint32_t vcn_inst,struct amdgpu_ring * ring_enc)1320 static int vcn_v4_0_init_ring_metadata(struct amdgpu_device *adev, uint32_t vcn_inst, struct amdgpu_ring *ring_enc)
1321 {
1322 struct amdgpu_vcn_rb_metadata *rb_metadata = NULL;
1323 uint8_t *rb_ptr = (uint8_t *)ring_enc->ring;
1324
1325 rb_ptr += ring_enc->ring_size;
1326 rb_metadata = (struct amdgpu_vcn_rb_metadata *)rb_ptr;
1327
1328 memset(rb_metadata, 0, sizeof(struct amdgpu_vcn_rb_metadata));
1329 rb_metadata->size = sizeof(struct amdgpu_vcn_rb_metadata);
1330 rb_metadata->present_flag_0 |= cpu_to_le32(AMDGPU_VCN_VF_RB_SETUP_FLAG);
1331 rb_metadata->present_flag_0 |= cpu_to_le32(AMDGPU_VCN_VF_RB_DECOUPLE_FLAG);
1332 rb_metadata->version = 1;
1333 rb_metadata->ring_id = vcn_inst & 0xFF;
1334
1335 return 0;
1336 }
1337
vcn_v4_0_start_sriov(struct amdgpu_device * adev)1338 static int vcn_v4_0_start_sriov(struct amdgpu_device *adev)
1339 {
1340 int i;
1341 struct amdgpu_ring *ring_enc;
1342 uint64_t cache_addr;
1343 uint64_t rb_enc_addr;
1344 uint64_t ctx_addr;
1345 uint32_t param, resp, expected;
1346 uint32_t offset, cache_size;
1347 uint32_t tmp, timeout;
1348
1349 struct amdgpu_mm_table *table = &adev->virt.mm_table;
1350 uint32_t *table_loc;
1351 uint32_t table_size;
1352 uint32_t size, size_dw;
1353 uint32_t init_status;
1354 uint32_t enabled_vcn;
1355
1356 struct mmsch_v4_0_cmd_direct_write
1357 direct_wt = { {0} };
1358 struct mmsch_v4_0_cmd_direct_read_modify_write
1359 direct_rd_mod_wt = { {0} };
1360 struct mmsch_v4_0_cmd_end end = { {0} };
1361 struct mmsch_v4_0_init_header header;
1362
1363 volatile struct amdgpu_vcn4_fw_shared *fw_shared;
1364 volatile struct amdgpu_fw_shared_rb_setup *rb_setup;
1365
1366 direct_wt.cmd_header.command_type =
1367 MMSCH_COMMAND__DIRECT_REG_WRITE;
1368 direct_rd_mod_wt.cmd_header.command_type =
1369 MMSCH_COMMAND__DIRECT_REG_READ_MODIFY_WRITE;
1370 end.cmd_header.command_type =
1371 MMSCH_COMMAND__END;
1372
1373 header.version = MMSCH_VERSION;
1374 header.total_size = sizeof(struct mmsch_v4_0_init_header) >> 2;
1375 for (i = 0; i < MMSCH_V4_0_VCN_INSTANCES; i++) {
1376 header.inst[i].init_status = 0;
1377 header.inst[i].table_offset = 0;
1378 header.inst[i].table_size = 0;
1379 }
1380
1381 table_loc = (uint32_t *)table->cpu_addr;
1382 table_loc += header.total_size;
1383 for (i = 0; i < adev->vcn.num_vcn_inst; i++) {
1384 if (adev->vcn.harvest_config & (1 << i))
1385 continue;
1386
1387 // Must re/init fw_shared at beginning
1388 vcn_v4_0_fw_shared_init(adev, i);
1389
1390 table_size = 0;
1391
1392 MMSCH_V4_0_INSERT_DIRECT_RD_MOD_WT(SOC15_REG_OFFSET(VCN, i,
1393 regUVD_STATUS),
1394 ~UVD_STATUS__UVD_BUSY, UVD_STATUS__UVD_BUSY);
1395
1396 cache_size = AMDGPU_GPU_PAGE_ALIGN(adev->vcn.inst[i].fw->size + 4);
1397
1398 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
1399 MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1400 regUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW),
1401 adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + i].tmr_mc_addr_lo);
1402 MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1403 regUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH),
1404 adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + i].tmr_mc_addr_hi);
1405 offset = 0;
1406 MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1407 regUVD_VCPU_CACHE_OFFSET0),
1408 0);
1409 } else {
1410 MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1411 regUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW),
1412 lower_32_bits(adev->vcn.inst[i].gpu_addr));
1413 MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1414 regUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH),
1415 upper_32_bits(adev->vcn.inst[i].gpu_addr));
1416 offset = cache_size;
1417 MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1418 regUVD_VCPU_CACHE_OFFSET0),
1419 AMDGPU_UVD_FIRMWARE_OFFSET >> 3);
1420 }
1421
1422 MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1423 regUVD_VCPU_CACHE_SIZE0),
1424 cache_size);
1425
1426 cache_addr = adev->vcn.inst[i].gpu_addr + offset;
1427 MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1428 regUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW),
1429 lower_32_bits(cache_addr));
1430 MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1431 regUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH),
1432 upper_32_bits(cache_addr));
1433 MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1434 regUVD_VCPU_CACHE_OFFSET1),
1435 0);
1436 MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1437 regUVD_VCPU_CACHE_SIZE1),
1438 AMDGPU_VCN_STACK_SIZE);
1439
1440 cache_addr = adev->vcn.inst[i].gpu_addr + offset +
1441 AMDGPU_VCN_STACK_SIZE;
1442 MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1443 regUVD_LMI_VCPU_CACHE2_64BIT_BAR_LOW),
1444 lower_32_bits(cache_addr));
1445 MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1446 regUVD_LMI_VCPU_CACHE2_64BIT_BAR_HIGH),
1447 upper_32_bits(cache_addr));
1448 MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1449 regUVD_VCPU_CACHE_OFFSET2),
1450 0);
1451 MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1452 regUVD_VCPU_CACHE_SIZE2),
1453 AMDGPU_VCN_CONTEXT_SIZE);
1454
1455 fw_shared = adev->vcn.inst[i].fw_shared.cpu_addr;
1456 rb_setup = &fw_shared->rb_setup;
1457
1458 ring_enc = &adev->vcn.inst[i].ring_enc[0];
1459 ring_enc->wptr = 0;
1460 rb_enc_addr = ring_enc->gpu_addr;
1461
1462 rb_setup->is_rb_enabled_flags |= RB_ENABLED;
1463 fw_shared->present_flag_0 |= cpu_to_le32(AMDGPU_VCN_VF_RB_SETUP_FLAG);
1464
1465 if (amdgpu_sriov_is_vcn_rb_decouple(adev)) {
1466 vcn_v4_0_init_ring_metadata(adev, i, ring_enc);
1467
1468 memset((void *)&rb_setup->rb_info, 0, sizeof(struct amdgpu_vcn_rb_setup_info) * MAX_NUM_VCN_RB_SETUP);
1469 if (!(adev->vcn.harvest_config & (1 << 0))) {
1470 rb_setup->rb_info[0].rb_addr_lo = lower_32_bits(adev->vcn.inst[0].ring_enc[0].gpu_addr);
1471 rb_setup->rb_info[0].rb_addr_hi = upper_32_bits(adev->vcn.inst[0].ring_enc[0].gpu_addr);
1472 rb_setup->rb_info[0].rb_size = adev->vcn.inst[0].ring_enc[0].ring_size / 4;
1473 }
1474 if (!(adev->vcn.harvest_config & (1 << 1))) {
1475 rb_setup->rb_info[2].rb_addr_lo = lower_32_bits(adev->vcn.inst[1].ring_enc[0].gpu_addr);
1476 rb_setup->rb_info[2].rb_addr_hi = upper_32_bits(adev->vcn.inst[1].ring_enc[0].gpu_addr);
1477 rb_setup->rb_info[2].rb_size = adev->vcn.inst[1].ring_enc[0].ring_size / 4;
1478 }
1479 fw_shared->decouple.is_enabled = 1;
1480 fw_shared->present_flag_0 |= cpu_to_le32(AMDGPU_VCN_VF_RB_DECOUPLE_FLAG);
1481 } else {
1482 rb_setup->rb_addr_lo = lower_32_bits(rb_enc_addr);
1483 rb_setup->rb_addr_hi = upper_32_bits(rb_enc_addr);
1484 rb_setup->rb_size = ring_enc->ring_size / 4;
1485 }
1486
1487 MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1488 regUVD_LMI_VCPU_NC0_64BIT_BAR_LOW),
1489 lower_32_bits(adev->vcn.inst[i].fw_shared.gpu_addr));
1490 MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1491 regUVD_LMI_VCPU_NC0_64BIT_BAR_HIGH),
1492 upper_32_bits(adev->vcn.inst[i].fw_shared.gpu_addr));
1493 MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1494 regUVD_VCPU_NONCACHE_SIZE0),
1495 AMDGPU_GPU_PAGE_ALIGN(sizeof(struct amdgpu_vcn4_fw_shared)));
1496
1497 /* add end packet */
1498 MMSCH_V4_0_INSERT_END();
1499
1500 /* refine header */
1501 header.inst[i].init_status = 0;
1502 header.inst[i].table_offset = header.total_size;
1503 header.inst[i].table_size = table_size;
1504 header.total_size += table_size;
1505 }
1506
1507 /* Update init table header in memory */
1508 size = sizeof(struct mmsch_v4_0_init_header);
1509 table_loc = (uint32_t *)table->cpu_addr;
1510 memcpy((void *)table_loc, &header, size);
1511
1512 /* message MMSCH (in VCN[0]) to initialize this client
1513 * 1, write to mmsch_vf_ctx_addr_lo/hi register with GPU mc addr
1514 * of memory descriptor location
1515 */
1516 ctx_addr = table->gpu_addr;
1517 WREG32_SOC15(VCN, 0, regMMSCH_VF_CTX_ADDR_LO, lower_32_bits(ctx_addr));
1518 WREG32_SOC15(VCN, 0, regMMSCH_VF_CTX_ADDR_HI, upper_32_bits(ctx_addr));
1519
1520 /* 2, update vmid of descriptor */
1521 tmp = RREG32_SOC15(VCN, 0, regMMSCH_VF_VMID);
1522 tmp &= ~MMSCH_VF_VMID__VF_CTX_VMID_MASK;
1523 /* use domain0 for MM scheduler */
1524 tmp |= (0 << MMSCH_VF_VMID__VF_CTX_VMID__SHIFT);
1525 WREG32_SOC15(VCN, 0, regMMSCH_VF_VMID, tmp);
1526
1527 /* 3, notify mmsch about the size of this descriptor */
1528 size = header.total_size;
1529 WREG32_SOC15(VCN, 0, regMMSCH_VF_CTX_SIZE, size);
1530
1531 /* 4, set resp to zero */
1532 WREG32_SOC15(VCN, 0, regMMSCH_VF_MAILBOX_RESP, 0);
1533
1534 /* 5, kick off the initialization and wait until
1535 * MMSCH_VF_MAILBOX_RESP becomes non-zero
1536 */
1537 param = 0x00000001;
1538 WREG32_SOC15(VCN, 0, regMMSCH_VF_MAILBOX_HOST, param);
1539 tmp = 0;
1540 timeout = 1000;
1541 resp = 0;
1542 expected = MMSCH_VF_MAILBOX_RESP__OK;
1543 while (resp != expected) {
1544 resp = RREG32_SOC15(VCN, 0, regMMSCH_VF_MAILBOX_RESP);
1545 if (resp != 0)
1546 break;
1547
1548 udelay(10);
1549 tmp = tmp + 10;
1550 if (tmp >= timeout) {
1551 DRM_ERROR("failed to init MMSCH. TIME-OUT after %d usec"\
1552 " waiting for regMMSCH_VF_MAILBOX_RESP "\
1553 "(expected=0x%08x, readback=0x%08x)\n",
1554 tmp, expected, resp);
1555 return -EBUSY;
1556 }
1557 }
1558 enabled_vcn = amdgpu_vcn_is_disabled_vcn(adev, VCN_DECODE_RING, 0) ? 1 : 0;
1559 init_status = ((struct mmsch_v4_0_init_header *)(table_loc))->inst[enabled_vcn].init_status;
1560 if (resp != expected && resp != MMSCH_VF_MAILBOX_RESP__INCOMPLETE
1561 && init_status != MMSCH_VF_ENGINE_STATUS__PASS)
1562 DRM_ERROR("MMSCH init status is incorrect! readback=0x%08x, header init "\
1563 "status for VCN%x: 0x%x\n", resp, enabled_vcn, init_status);
1564
1565 return 0;
1566 }
1567
1568 /**
1569 * vcn_v4_0_stop_dpg_mode - VCN stop with dpg mode
1570 *
1571 * @vinst: VCN instance
1572 *
1573 * Stop VCN block with dpg mode
1574 */
vcn_v4_0_stop_dpg_mode(struct amdgpu_vcn_inst * vinst)1575 static void vcn_v4_0_stop_dpg_mode(struct amdgpu_vcn_inst *vinst)
1576 {
1577 struct amdgpu_device *adev = vinst->adev;
1578 int inst_idx = vinst->inst;
1579 struct dpg_pause_state state = {.fw_based = VCN_DPG_STATE__UNPAUSE};
1580 uint32_t tmp;
1581
1582 vcn_v4_0_pause_dpg_mode(vinst, &state);
1583 /* Wait for power status to be 1 */
1584 SOC15_WAIT_ON_RREG(VCN, inst_idx, regUVD_POWER_STATUS, 1,
1585 UVD_POWER_STATUS__UVD_POWER_STATUS_MASK);
1586
1587 /* wait for read ptr to be equal to write ptr */
1588 tmp = RREG32_SOC15(VCN, inst_idx, regUVD_RB_WPTR);
1589 SOC15_WAIT_ON_RREG(VCN, inst_idx, regUVD_RB_RPTR, tmp, 0xFFFFFFFF);
1590
1591 SOC15_WAIT_ON_RREG(VCN, inst_idx, regUVD_POWER_STATUS, 1,
1592 UVD_POWER_STATUS__UVD_POWER_STATUS_MASK);
1593
1594 /* disable dynamic power gating mode */
1595 WREG32_P(SOC15_REG_OFFSET(VCN, inst_idx, regUVD_POWER_STATUS), 0,
1596 ~UVD_POWER_STATUS__UVD_PG_MODE_MASK);
1597
1598 /* Keeping one read-back to ensure all register writes are done,
1599 * otherwise it may introduce race conditions.
1600 */
1601 RREG32_SOC15(VCN, inst_idx, regUVD_STATUS);
1602 }
1603
1604 /**
1605 * vcn_v4_0_stop - VCN stop
1606 *
1607 * @vinst: VCN instance
1608 *
1609 * Stop VCN block
1610 */
vcn_v4_0_stop(struct amdgpu_vcn_inst * vinst)1611 static int vcn_v4_0_stop(struct amdgpu_vcn_inst *vinst)
1612 {
1613 struct amdgpu_device *adev = vinst->adev;
1614 int i = vinst->inst;
1615 volatile struct amdgpu_vcn4_fw_shared *fw_shared;
1616 uint32_t tmp;
1617 int r = 0;
1618
1619 if (adev->vcn.harvest_config & (1 << i))
1620 return 0;
1621
1622 fw_shared = adev->vcn.inst[i].fw_shared.cpu_addr;
1623 fw_shared->sq.queue_mode |= FW_QUEUE_DPG_HOLD_OFF;
1624
1625 if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG) {
1626 vcn_v4_0_stop_dpg_mode(vinst);
1627 r = 0;
1628 goto done;
1629 }
1630
1631 /* wait for vcn idle */
1632 r = SOC15_WAIT_ON_RREG(VCN, i, regUVD_STATUS, UVD_STATUS__IDLE, 0x7);
1633 if (r)
1634 goto done;
1635
1636 tmp = UVD_LMI_STATUS__VCPU_LMI_WRITE_CLEAN_MASK |
1637 UVD_LMI_STATUS__READ_CLEAN_MASK |
1638 UVD_LMI_STATUS__WRITE_CLEAN_MASK |
1639 UVD_LMI_STATUS__WRITE_CLEAN_RAW_MASK;
1640 r = SOC15_WAIT_ON_RREG(VCN, i, regUVD_LMI_STATUS, tmp, tmp);
1641 if (r)
1642 goto done;
1643
1644 /* disable LMI UMC channel */
1645 tmp = RREG32_SOC15(VCN, i, regUVD_LMI_CTRL2);
1646 tmp |= UVD_LMI_CTRL2__STALL_ARB_UMC_MASK;
1647 WREG32_SOC15(VCN, i, regUVD_LMI_CTRL2, tmp);
1648 tmp = UVD_LMI_STATUS__UMC_READ_CLEAN_RAW_MASK |
1649 UVD_LMI_STATUS__UMC_WRITE_CLEAN_RAW_MASK;
1650 r = SOC15_WAIT_ON_RREG(VCN, i, regUVD_LMI_STATUS, tmp, tmp);
1651 if (r)
1652 goto done;
1653
1654 /* block VCPU register access */
1655 WREG32_P(SOC15_REG_OFFSET(VCN, i, regUVD_RB_ARB_CTRL),
1656 UVD_RB_ARB_CTRL__VCPU_DIS_MASK,
1657 ~UVD_RB_ARB_CTRL__VCPU_DIS_MASK);
1658
1659 /* reset VCPU */
1660 WREG32_P(SOC15_REG_OFFSET(VCN, i, regUVD_VCPU_CNTL),
1661 UVD_VCPU_CNTL__BLK_RST_MASK,
1662 ~UVD_VCPU_CNTL__BLK_RST_MASK);
1663
1664 /* disable VCPU clock */
1665 WREG32_P(SOC15_REG_OFFSET(VCN, i, regUVD_VCPU_CNTL), 0,
1666 ~(UVD_VCPU_CNTL__CLK_EN_MASK));
1667
1668 /* apply soft reset */
1669 tmp = RREG32_SOC15(VCN, i, regUVD_SOFT_RESET);
1670 tmp |= UVD_SOFT_RESET__LMI_UMC_SOFT_RESET_MASK;
1671 WREG32_SOC15(VCN, i, regUVD_SOFT_RESET, tmp);
1672 tmp = RREG32_SOC15(VCN, i, regUVD_SOFT_RESET);
1673 tmp |= UVD_SOFT_RESET__LMI_SOFT_RESET_MASK;
1674 WREG32_SOC15(VCN, i, regUVD_SOFT_RESET, tmp);
1675
1676 /* clear status */
1677 WREG32_SOC15(VCN, i, regUVD_STATUS, 0);
1678
1679 /* apply HW clock gating */
1680 vcn_v4_0_enable_clock_gating(vinst);
1681
1682 /* enable VCN power gating */
1683 vcn_v4_0_enable_static_power_gating(vinst);
1684
1685 /* Keeping one read-back to ensure all register writes are done,
1686 * otherwise it may introduce race conditions.
1687 */
1688 RREG32_SOC15(VCN, i, regUVD_STATUS);
1689
1690 done:
1691 if (adev->pm.dpm_enabled)
1692 amdgpu_dpm_enable_vcn(adev, false, i);
1693
1694 return 0;
1695 }
1696
1697 /**
1698 * vcn_v4_0_pause_dpg_mode - VCN pause with dpg mode
1699 *
1700 * @vinst: VCN instance
1701 * @new_state: pause state
1702 *
1703 * Pause dpg mode for VCN block
1704 */
vcn_v4_0_pause_dpg_mode(struct amdgpu_vcn_inst * vinst,struct dpg_pause_state * new_state)1705 static int vcn_v4_0_pause_dpg_mode(struct amdgpu_vcn_inst *vinst,
1706 struct dpg_pause_state *new_state)
1707 {
1708 struct amdgpu_device *adev = vinst->adev;
1709 int inst_idx = vinst->inst;
1710 uint32_t reg_data = 0;
1711 int ret_code;
1712
1713 /* pause/unpause if state is changed */
1714 if (adev->vcn.inst[inst_idx].pause_state.fw_based != new_state->fw_based) {
1715 DRM_DEV_DEBUG(adev->dev, "dpg pause state changed %d -> %d",
1716 adev->vcn.inst[inst_idx].pause_state.fw_based, new_state->fw_based);
1717 reg_data = RREG32_SOC15(VCN, inst_idx, regUVD_DPG_PAUSE) &
1718 (~UVD_DPG_PAUSE__NJ_PAUSE_DPG_ACK_MASK);
1719
1720 if (new_state->fw_based == VCN_DPG_STATE__PAUSE) {
1721 ret_code = SOC15_WAIT_ON_RREG(VCN, inst_idx, regUVD_POWER_STATUS, 0x1,
1722 UVD_POWER_STATUS__UVD_POWER_STATUS_MASK);
1723
1724 if (!ret_code) {
1725 /* pause DPG */
1726 reg_data |= UVD_DPG_PAUSE__NJ_PAUSE_DPG_REQ_MASK;
1727 WREG32_SOC15(VCN, inst_idx, regUVD_DPG_PAUSE, reg_data);
1728
1729 /* wait for ACK */
1730 SOC15_WAIT_ON_RREG(VCN, inst_idx, regUVD_DPG_PAUSE,
1731 UVD_DPG_PAUSE__NJ_PAUSE_DPG_ACK_MASK,
1732 UVD_DPG_PAUSE__NJ_PAUSE_DPG_ACK_MASK);
1733
1734 SOC15_WAIT_ON_RREG(VCN, inst_idx, regUVD_POWER_STATUS,
1735 UVD_PGFSM_CONFIG__UVDM_UVDU_PWR_ON, UVD_POWER_STATUS__UVD_POWER_STATUS_MASK);
1736 }
1737 } else {
1738 /* unpause dpg, no need to wait */
1739 reg_data &= ~UVD_DPG_PAUSE__NJ_PAUSE_DPG_REQ_MASK;
1740 WREG32_SOC15(VCN, inst_idx, regUVD_DPG_PAUSE, reg_data);
1741 }
1742 adev->vcn.inst[inst_idx].pause_state.fw_based = new_state->fw_based;
1743 }
1744
1745 return 0;
1746 }
1747
1748 /**
1749 * vcn_v4_0_unified_ring_get_rptr - get unified read pointer
1750 *
1751 * @ring: amdgpu_ring pointer
1752 *
1753 * Returns the current hardware unified read pointer
1754 */
vcn_v4_0_unified_ring_get_rptr(struct amdgpu_ring * ring)1755 static uint64_t vcn_v4_0_unified_ring_get_rptr(struct amdgpu_ring *ring)
1756 {
1757 struct amdgpu_device *adev = ring->adev;
1758
1759 if (ring != &adev->vcn.inst[ring->me].ring_enc[0])
1760 DRM_ERROR("wrong ring id is identified in %s", __func__);
1761
1762 return RREG32_SOC15(VCN, ring->me, regUVD_RB_RPTR);
1763 }
1764
1765 /**
1766 * vcn_v4_0_unified_ring_get_wptr - get unified write pointer
1767 *
1768 * @ring: amdgpu_ring pointer
1769 *
1770 * Returns the current hardware unified write pointer
1771 */
vcn_v4_0_unified_ring_get_wptr(struct amdgpu_ring * ring)1772 static uint64_t vcn_v4_0_unified_ring_get_wptr(struct amdgpu_ring *ring)
1773 {
1774 struct amdgpu_device *adev = ring->adev;
1775
1776 if (ring != &adev->vcn.inst[ring->me].ring_enc[0])
1777 DRM_ERROR("wrong ring id is identified in %s", __func__);
1778
1779 if (ring->use_doorbell)
1780 return *ring->wptr_cpu_addr;
1781 else
1782 return RREG32_SOC15(VCN, ring->me, regUVD_RB_WPTR);
1783 }
1784
1785 /**
1786 * vcn_v4_0_unified_ring_set_wptr - set enc write pointer
1787 *
1788 * @ring: amdgpu_ring pointer
1789 *
1790 * Commits the enc write pointer to the hardware
1791 */
vcn_v4_0_unified_ring_set_wptr(struct amdgpu_ring * ring)1792 static void vcn_v4_0_unified_ring_set_wptr(struct amdgpu_ring *ring)
1793 {
1794 struct amdgpu_device *adev = ring->adev;
1795
1796 if (ring != &adev->vcn.inst[ring->me].ring_enc[0])
1797 DRM_ERROR("wrong ring id is identified in %s", __func__);
1798
1799 if (ring->use_doorbell) {
1800 *ring->wptr_cpu_addr = lower_32_bits(ring->wptr);
1801 WDOORBELL32(ring->doorbell_index, lower_32_bits(ring->wptr));
1802 } else {
1803 WREG32_SOC15(VCN, ring->me, regUVD_RB_WPTR, lower_32_bits(ring->wptr));
1804 }
1805 }
1806
vcn_v4_0_limit_sched(struct amdgpu_cs_parser * p,struct amdgpu_job * job)1807 static int vcn_v4_0_limit_sched(struct amdgpu_cs_parser *p,
1808 struct amdgpu_job *job)
1809 {
1810 struct drm_gpu_scheduler **scheds;
1811
1812 /* The create msg must be in the first IB submitted */
1813 if (atomic_read(&job->base.entity->fence_seq))
1814 return -EINVAL;
1815
1816 /* if VCN0 is harvested, we can't support AV1 */
1817 if (p->adev->vcn.harvest_config & AMDGPU_VCN_HARVEST_VCN0)
1818 return -EINVAL;
1819
1820 scheds = p->adev->gpu_sched[AMDGPU_HW_IP_VCN_ENC]
1821 [AMDGPU_RING_PRIO_0].sched;
1822 drm_sched_entity_modify_sched(job->base.entity, scheds, 1);
1823 return 0;
1824 }
1825
vcn_v4_0_dec_msg(struct amdgpu_cs_parser * p,struct amdgpu_job * job,uint64_t addr)1826 static int vcn_v4_0_dec_msg(struct amdgpu_cs_parser *p, struct amdgpu_job *job,
1827 uint64_t addr)
1828 {
1829 struct ttm_operation_ctx ctx = { false, false };
1830 struct amdgpu_bo_va_mapping *map;
1831 uint32_t *msg, num_buffers;
1832 struct amdgpu_bo *bo;
1833 uint64_t start, end;
1834 unsigned int i;
1835 void *ptr;
1836 int r;
1837
1838 addr &= AMDGPU_GMC_HOLE_MASK;
1839 r = amdgpu_cs_find_mapping(p, addr, &bo, &map);
1840 if (r) {
1841 DRM_ERROR("Can't find BO for addr 0x%08llx\n", addr);
1842 return r;
1843 }
1844
1845 start = map->start * AMDGPU_GPU_PAGE_SIZE;
1846 end = (map->last + 1) * AMDGPU_GPU_PAGE_SIZE;
1847 if (addr & 0x7) {
1848 DRM_ERROR("VCN messages must be 8 byte aligned!\n");
1849 return -EINVAL;
1850 }
1851
1852 bo->flags |= AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;
1853 amdgpu_bo_placement_from_domain(bo, bo->allowed_domains);
1854 r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
1855 if (r) {
1856 DRM_ERROR("Failed validating the VCN message BO (%d)!\n", r);
1857 return r;
1858 }
1859
1860 r = amdgpu_bo_kmap(bo, &ptr);
1861 if (r) {
1862 DRM_ERROR("Failed mapping the VCN message (%d)!\n", r);
1863 return r;
1864 }
1865
1866 msg = ptr + addr - start;
1867
1868 /* Check length */
1869 if (msg[1] > end - addr) {
1870 r = -EINVAL;
1871 goto out;
1872 }
1873
1874 if (msg[3] != RDECODE_MSG_CREATE)
1875 goto out;
1876
1877 num_buffers = msg[2];
1878 for (i = 0, msg = &msg[6]; i < num_buffers; ++i, msg += 4) {
1879 uint32_t offset, size, *create;
1880
1881 if (msg[0] != RDECODE_MESSAGE_CREATE)
1882 continue;
1883
1884 offset = msg[1];
1885 size = msg[2];
1886
1887 if (offset + size > end) {
1888 r = -EINVAL;
1889 goto out;
1890 }
1891
1892 create = ptr + addr + offset - start;
1893
1894 /* H264, HEVC and VP9 can run on any instance */
1895 if (create[0] == 0x7 || create[0] == 0x10 || create[0] == 0x11)
1896 continue;
1897
1898 r = vcn_v4_0_limit_sched(p, job);
1899 if (r)
1900 goto out;
1901 }
1902
1903 out:
1904 amdgpu_bo_kunmap(bo);
1905 return r;
1906 }
1907
1908 #define RADEON_VCN_ENGINE_TYPE_ENCODE (0x00000002)
1909 #define RADEON_VCN_ENGINE_TYPE_DECODE (0x00000003)
1910
1911 #define RADEON_VCN_ENGINE_INFO (0x30000001)
1912 #define RADEON_VCN_ENGINE_INFO_MAX_OFFSET 16
1913
1914 #define RENCODE_ENCODE_STANDARD_AV1 2
1915 #define RENCODE_IB_PARAM_SESSION_INIT 0x00000003
1916 #define RENCODE_IB_PARAM_SESSION_INIT_MAX_OFFSET 64
1917
1918 /* return the offset in ib if id is found, -1 otherwise
1919 * to speed up the searching we only search upto max_offset
1920 */
vcn_v4_0_enc_find_ib_param(struct amdgpu_ib * ib,uint32_t id,int max_offset)1921 static int vcn_v4_0_enc_find_ib_param(struct amdgpu_ib *ib, uint32_t id, int max_offset)
1922 {
1923 int i;
1924
1925 for (i = 0; i < ib->length_dw && i < max_offset && ib->ptr[i] >= 8; i += ib->ptr[i]/4) {
1926 if (ib->ptr[i + 1] == id)
1927 return i;
1928 }
1929 return -1;
1930 }
1931
vcn_v4_0_ring_patch_cs_in_place(struct amdgpu_cs_parser * p,struct amdgpu_job * job,struct amdgpu_ib * ib)1932 static int vcn_v4_0_ring_patch_cs_in_place(struct amdgpu_cs_parser *p,
1933 struct amdgpu_job *job,
1934 struct amdgpu_ib *ib)
1935 {
1936 struct amdgpu_ring *ring = amdgpu_job_ring(job);
1937 struct amdgpu_vcn_decode_buffer *decode_buffer;
1938 uint64_t addr;
1939 uint32_t val;
1940 int idx;
1941
1942 /* The first instance can decode anything */
1943 if (!ring->me)
1944 return 0;
1945
1946 /* RADEON_VCN_ENGINE_INFO is at the top of ib block */
1947 idx = vcn_v4_0_enc_find_ib_param(ib, RADEON_VCN_ENGINE_INFO,
1948 RADEON_VCN_ENGINE_INFO_MAX_OFFSET);
1949 if (idx < 0) /* engine info is missing */
1950 return 0;
1951
1952 val = amdgpu_ib_get_value(ib, idx + 2); /* RADEON_VCN_ENGINE_TYPE */
1953 if (val == RADEON_VCN_ENGINE_TYPE_DECODE) {
1954 decode_buffer = (struct amdgpu_vcn_decode_buffer *)&ib->ptr[idx + 6];
1955
1956 if (!(decode_buffer->valid_buf_flag & 0x1))
1957 return 0;
1958
1959 addr = ((u64)decode_buffer->msg_buffer_address_hi) << 32 |
1960 decode_buffer->msg_buffer_address_lo;
1961 return vcn_v4_0_dec_msg(p, job, addr);
1962 } else if (val == RADEON_VCN_ENGINE_TYPE_ENCODE) {
1963 idx = vcn_v4_0_enc_find_ib_param(ib, RENCODE_IB_PARAM_SESSION_INIT,
1964 RENCODE_IB_PARAM_SESSION_INIT_MAX_OFFSET);
1965 if (idx >= 0 && ib->ptr[idx + 2] == RENCODE_ENCODE_STANDARD_AV1)
1966 return vcn_v4_0_limit_sched(p, job);
1967 }
1968 return 0;
1969 }
1970
vcn_v4_0_ring_reset(struct amdgpu_ring * ring,unsigned int vmid,struct amdgpu_fence * timedout_fence)1971 static int vcn_v4_0_ring_reset(struct amdgpu_ring *ring,
1972 unsigned int vmid,
1973 struct amdgpu_fence *timedout_fence)
1974 {
1975 struct amdgpu_device *adev = ring->adev;
1976 struct amdgpu_vcn_inst *vinst = &adev->vcn.inst[ring->me];
1977 int r;
1978
1979 amdgpu_ring_reset_helper_begin(ring, timedout_fence);
1980 r = vcn_v4_0_stop(vinst);
1981 if (r)
1982 return r;
1983 r = vcn_v4_0_start(vinst);
1984 if (r)
1985 return r;
1986 return amdgpu_ring_reset_helper_end(ring, timedout_fence);
1987 }
1988
1989 static struct amdgpu_ring_funcs vcn_v4_0_unified_ring_vm_funcs = {
1990 .type = AMDGPU_RING_TYPE_VCN_ENC,
1991 .align_mask = 0x3f,
1992 .nop = VCN_ENC_CMD_NO_OP,
1993 .extra_dw = sizeof(struct amdgpu_vcn_rb_metadata),
1994 .get_rptr = vcn_v4_0_unified_ring_get_rptr,
1995 .get_wptr = vcn_v4_0_unified_ring_get_wptr,
1996 .set_wptr = vcn_v4_0_unified_ring_set_wptr,
1997 .patch_cs_in_place = vcn_v4_0_ring_patch_cs_in_place,
1998 .emit_frame_size =
1999 SOC15_FLUSH_GPU_TLB_NUM_WREG * 3 +
2000 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 4 +
2001 4 + /* vcn_v2_0_enc_ring_emit_vm_flush */
2002 5 + 5 + /* vcn_v2_0_enc_ring_emit_fence x2 vm fence */
2003 1, /* vcn_v2_0_enc_ring_insert_end */
2004 .emit_ib_size = 5, /* vcn_v2_0_enc_ring_emit_ib */
2005 .emit_ib = vcn_v2_0_enc_ring_emit_ib,
2006 .emit_fence = vcn_v2_0_enc_ring_emit_fence,
2007 .emit_vm_flush = vcn_v2_0_enc_ring_emit_vm_flush,
2008 .test_ring = amdgpu_vcn_enc_ring_test_ring,
2009 .test_ib = amdgpu_vcn_unified_ring_test_ib,
2010 .insert_nop = amdgpu_ring_insert_nop,
2011 .insert_end = vcn_v2_0_enc_ring_insert_end,
2012 .pad_ib = amdgpu_ring_generic_pad_ib,
2013 .begin_use = amdgpu_vcn_ring_begin_use,
2014 .end_use = amdgpu_vcn_ring_end_use,
2015 .emit_wreg = vcn_v2_0_enc_ring_emit_wreg,
2016 .emit_reg_wait = vcn_v2_0_enc_ring_emit_reg_wait,
2017 .emit_reg_write_reg_wait = amdgpu_ring_emit_reg_write_reg_wait_helper,
2018 .reset = vcn_v4_0_ring_reset,
2019 };
2020
2021 /**
2022 * vcn_v4_0_set_unified_ring_funcs - set unified ring functions
2023 *
2024 * @adev: amdgpu_device pointer
2025 *
2026 * Set unified ring functions
2027 */
vcn_v4_0_set_unified_ring_funcs(struct amdgpu_device * adev)2028 static void vcn_v4_0_set_unified_ring_funcs(struct amdgpu_device *adev)
2029 {
2030 int i;
2031
2032 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
2033 if (adev->vcn.harvest_config & (1 << i))
2034 continue;
2035
2036 if (amdgpu_ip_version(adev, VCN_HWIP, 0) == IP_VERSION(4, 0, 2))
2037 vcn_v4_0_unified_ring_vm_funcs.secure_submission_supported = true;
2038
2039 adev->vcn.inst[i].ring_enc[0].funcs =
2040 (const struct amdgpu_ring_funcs *)&vcn_v4_0_unified_ring_vm_funcs;
2041 adev->vcn.inst[i].ring_enc[0].me = i;
2042 }
2043 }
2044
2045 /**
2046 * vcn_v4_0_is_idle - check VCN block is idle
2047 *
2048 * @ip_block: Pointer to the amdgpu_ip_block structure
2049 *
2050 * Check whether VCN block is idle
2051 */
vcn_v4_0_is_idle(struct amdgpu_ip_block * ip_block)2052 static bool vcn_v4_0_is_idle(struct amdgpu_ip_block *ip_block)
2053 {
2054 struct amdgpu_device *adev = ip_block->adev;
2055 int i, ret = 1;
2056
2057 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
2058 if (adev->vcn.harvest_config & (1 << i))
2059 continue;
2060
2061 ret &= (RREG32_SOC15(VCN, i, regUVD_STATUS) == UVD_STATUS__IDLE);
2062 }
2063
2064 return ret;
2065 }
2066
2067 /**
2068 * vcn_v4_0_wait_for_idle - wait for VCN block idle
2069 *
2070 * @ip_block: Pointer to the amdgpu_ip_block for this hw instance.
2071 *
2072 * Wait for VCN block idle
2073 */
vcn_v4_0_wait_for_idle(struct amdgpu_ip_block * ip_block)2074 static int vcn_v4_0_wait_for_idle(struct amdgpu_ip_block *ip_block)
2075 {
2076 struct amdgpu_device *adev = ip_block->adev;
2077 int i, ret = 0;
2078
2079 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
2080 if (adev->vcn.harvest_config & (1 << i))
2081 continue;
2082
2083 ret = SOC15_WAIT_ON_RREG(VCN, i, regUVD_STATUS, UVD_STATUS__IDLE,
2084 UVD_STATUS__IDLE);
2085 if (ret)
2086 return ret;
2087 }
2088
2089 return ret;
2090 }
2091
2092 /**
2093 * vcn_v4_0_set_clockgating_state - set VCN block clockgating state
2094 *
2095 * @ip_block: amdgpu_ip_block pointer
2096 * @state: clock gating state
2097 *
2098 * Set VCN block clockgating state
2099 */
vcn_v4_0_set_clockgating_state(struct amdgpu_ip_block * ip_block,enum amd_clockgating_state state)2100 static int vcn_v4_0_set_clockgating_state(struct amdgpu_ip_block *ip_block,
2101 enum amd_clockgating_state state)
2102 {
2103 struct amdgpu_device *adev = ip_block->adev;
2104 bool enable = state == AMD_CG_STATE_GATE;
2105 int i;
2106
2107 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
2108 struct amdgpu_vcn_inst *vinst = &adev->vcn.inst[i];
2109
2110 if (adev->vcn.harvest_config & (1 << i))
2111 continue;
2112
2113 if (enable) {
2114 if (RREG32_SOC15(VCN, i, regUVD_STATUS) != UVD_STATUS__IDLE)
2115 return -EBUSY;
2116 vcn_v4_0_enable_clock_gating(vinst);
2117 } else {
2118 vcn_v4_0_disable_clock_gating(vinst);
2119 }
2120 }
2121
2122 return 0;
2123 }
2124
vcn_v4_0_set_pg_state(struct amdgpu_vcn_inst * vinst,enum amd_powergating_state state)2125 static int vcn_v4_0_set_pg_state(struct amdgpu_vcn_inst *vinst,
2126 enum amd_powergating_state state)
2127 {
2128 struct amdgpu_device *adev = vinst->adev;
2129 int ret = 0;
2130
2131 /* for SRIOV, guest should not control VCN Power-gating
2132 * MMSCH FW should control Power-gating and clock-gating
2133 * guest should avoid touching CGC and PG
2134 */
2135 if (amdgpu_sriov_vf(adev)) {
2136 vinst->cur_state = AMD_PG_STATE_UNGATE;
2137 return 0;
2138 }
2139
2140 if (state == vinst->cur_state)
2141 return 0;
2142
2143 if (state == AMD_PG_STATE_GATE)
2144 ret = vcn_v4_0_stop(vinst);
2145 else
2146 ret = vcn_v4_0_start(vinst);
2147
2148 if (!ret)
2149 vinst->cur_state = state;
2150
2151 return ret;
2152 }
2153
2154 /**
2155 * vcn_v4_0_set_ras_interrupt_state - set VCN block RAS interrupt state
2156 *
2157 * @adev: amdgpu_device pointer
2158 * @source: interrupt sources
2159 * @type: interrupt types
2160 * @state: interrupt states
2161 *
2162 * Set VCN block RAS interrupt state
2163 */
vcn_v4_0_set_ras_interrupt_state(struct amdgpu_device * adev,struct amdgpu_irq_src * source,unsigned int type,enum amdgpu_interrupt_state state)2164 static int vcn_v4_0_set_ras_interrupt_state(struct amdgpu_device *adev,
2165 struct amdgpu_irq_src *source,
2166 unsigned int type,
2167 enum amdgpu_interrupt_state state)
2168 {
2169 return 0;
2170 }
2171
2172 /**
2173 * vcn_v4_0_process_interrupt - process VCN block interrupt
2174 *
2175 * @adev: amdgpu_device pointer
2176 * @source: interrupt sources
2177 * @entry: interrupt entry from clients and sources
2178 *
2179 * Process VCN block interrupt
2180 */
vcn_v4_0_process_interrupt(struct amdgpu_device * adev,struct amdgpu_irq_src * source,struct amdgpu_iv_entry * entry)2181 static int vcn_v4_0_process_interrupt(struct amdgpu_device *adev, struct amdgpu_irq_src *source,
2182 struct amdgpu_iv_entry *entry)
2183 {
2184 uint32_t ip_instance;
2185
2186 if (amdgpu_sriov_is_vcn_rb_decouple(adev)) {
2187 ip_instance = entry->ring_id;
2188 } else {
2189 switch (entry->client_id) {
2190 case SOC15_IH_CLIENTID_VCN:
2191 ip_instance = 0;
2192 break;
2193 case SOC15_IH_CLIENTID_VCN1:
2194 ip_instance = 1;
2195 break;
2196 default:
2197 DRM_ERROR("Unhandled client id: %d\n", entry->client_id);
2198 return 0;
2199 }
2200 }
2201
2202 DRM_DEBUG("IH: VCN TRAP\n");
2203
2204 switch (entry->src_id) {
2205 case VCN_4_0__SRCID__UVD_ENC_GENERAL_PURPOSE:
2206 amdgpu_fence_process(&adev->vcn.inst[ip_instance].ring_enc[0]);
2207 break;
2208 default:
2209 DRM_ERROR("Unhandled interrupt: %d %d\n",
2210 entry->src_id, entry->src_data[0]);
2211 break;
2212 }
2213
2214 return 0;
2215 }
2216
2217 static const struct amdgpu_irq_src_funcs vcn_v4_0_irq_funcs = {
2218 .process = vcn_v4_0_process_interrupt,
2219 };
2220
2221 static const struct amdgpu_irq_src_funcs vcn_v4_0_ras_irq_funcs = {
2222 .set = vcn_v4_0_set_ras_interrupt_state,
2223 .process = amdgpu_vcn_process_poison_irq,
2224 };
2225
2226 /**
2227 * vcn_v4_0_set_irq_funcs - set VCN block interrupt irq functions
2228 *
2229 * @adev: amdgpu_device pointer
2230 *
2231 * Set VCN block interrupt irq functions
2232 */
vcn_v4_0_set_irq_funcs(struct amdgpu_device * adev)2233 static void vcn_v4_0_set_irq_funcs(struct amdgpu_device *adev)
2234 {
2235 int i;
2236
2237 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
2238 if (adev->vcn.harvest_config & (1 << i))
2239 continue;
2240
2241 adev->vcn.inst[i].irq.num_types = adev->vcn.inst[i].num_enc_rings + 1;
2242 adev->vcn.inst[i].irq.funcs = &vcn_v4_0_irq_funcs;
2243
2244 adev->vcn.inst[i].ras_poison_irq.num_types = adev->vcn.inst[i].num_enc_rings + 1;
2245 adev->vcn.inst[i].ras_poison_irq.funcs = &vcn_v4_0_ras_irq_funcs;
2246 }
2247 }
2248
vcn_v4_0_print_ip_state(struct amdgpu_ip_block * ip_block,struct drm_printer * p)2249 static void vcn_v4_0_print_ip_state(struct amdgpu_ip_block *ip_block, struct drm_printer *p)
2250 {
2251 struct amdgpu_device *adev = ip_block->adev;
2252 int i, j;
2253 uint32_t reg_count = ARRAY_SIZE(vcn_reg_list_4_0);
2254 uint32_t inst_off, is_powered;
2255
2256 if (!adev->vcn.ip_dump)
2257 return;
2258
2259 drm_printf(p, "num_instances:%d\n", adev->vcn.num_vcn_inst);
2260 for (i = 0; i < adev->vcn.num_vcn_inst; i++) {
2261 if (adev->vcn.harvest_config & (1 << i)) {
2262 drm_printf(p, "\nHarvested Instance:VCN%d Skipping dump\n", i);
2263 continue;
2264 }
2265
2266 inst_off = i * reg_count;
2267 is_powered = (adev->vcn.ip_dump[inst_off] &
2268 UVD_POWER_STATUS__UVD_POWER_STATUS_MASK) != 1;
2269
2270 if (is_powered) {
2271 drm_printf(p, "\nActive Instance:VCN%d\n", i);
2272 for (j = 0; j < reg_count; j++)
2273 drm_printf(p, "%-50s \t 0x%08x\n", vcn_reg_list_4_0[j].reg_name,
2274 adev->vcn.ip_dump[inst_off + j]);
2275 } else {
2276 drm_printf(p, "\nInactive Instance:VCN%d\n", i);
2277 }
2278 }
2279 }
2280
vcn_v4_0_dump_ip_state(struct amdgpu_ip_block * ip_block)2281 static void vcn_v4_0_dump_ip_state(struct amdgpu_ip_block *ip_block)
2282 {
2283 struct amdgpu_device *adev = ip_block->adev;
2284 int i, j;
2285 bool is_powered;
2286 uint32_t inst_off;
2287 uint32_t reg_count = ARRAY_SIZE(vcn_reg_list_4_0);
2288
2289 if (!adev->vcn.ip_dump)
2290 return;
2291
2292 for (i = 0; i < adev->vcn.num_vcn_inst; i++) {
2293 if (adev->vcn.harvest_config & (1 << i))
2294 continue;
2295
2296 inst_off = i * reg_count;
2297 /* mmUVD_POWER_STATUS is always readable and is first element of the array */
2298 adev->vcn.ip_dump[inst_off] = RREG32_SOC15(VCN, i, regUVD_POWER_STATUS);
2299 is_powered = (adev->vcn.ip_dump[inst_off] &
2300 UVD_POWER_STATUS__UVD_POWER_STATUS_MASK) != 1;
2301
2302 if (is_powered)
2303 for (j = 1; j < reg_count; j++)
2304 adev->vcn.ip_dump[inst_off + j] =
2305 RREG32(SOC15_REG_ENTRY_OFFSET_INST(vcn_reg_list_4_0[j],
2306 i));
2307 }
2308 }
2309
2310 static const struct amd_ip_funcs vcn_v4_0_ip_funcs = {
2311 .name = "vcn_v4_0",
2312 .early_init = vcn_v4_0_early_init,
2313 .sw_init = vcn_v4_0_sw_init,
2314 .sw_fini = vcn_v4_0_sw_fini,
2315 .hw_init = vcn_v4_0_hw_init,
2316 .hw_fini = vcn_v4_0_hw_fini,
2317 .suspend = vcn_v4_0_suspend,
2318 .resume = vcn_v4_0_resume,
2319 .is_idle = vcn_v4_0_is_idle,
2320 .wait_for_idle = vcn_v4_0_wait_for_idle,
2321 .set_clockgating_state = vcn_v4_0_set_clockgating_state,
2322 .set_powergating_state = vcn_set_powergating_state,
2323 .dump_ip_state = vcn_v4_0_dump_ip_state,
2324 .print_ip_state = vcn_v4_0_print_ip_state,
2325 };
2326
2327 const struct amdgpu_ip_block_version vcn_v4_0_ip_block = {
2328 .type = AMD_IP_BLOCK_TYPE_VCN,
2329 .major = 4,
2330 .minor = 0,
2331 .rev = 0,
2332 .funcs = &vcn_v4_0_ip_funcs,
2333 };
2334
vcn_v4_0_query_poison_by_instance(struct amdgpu_device * adev,uint32_t instance,uint32_t sub_block)2335 static uint32_t vcn_v4_0_query_poison_by_instance(struct amdgpu_device *adev,
2336 uint32_t instance, uint32_t sub_block)
2337 {
2338 uint32_t poison_stat = 0, reg_value = 0;
2339
2340 switch (sub_block) {
2341 case AMDGPU_VCN_V4_0_VCPU_VCODEC:
2342 reg_value = RREG32_SOC15(VCN, instance, regUVD_RAS_VCPU_VCODEC_STATUS);
2343 poison_stat = REG_GET_FIELD(reg_value, UVD_RAS_VCPU_VCODEC_STATUS, POISONED_PF);
2344 break;
2345 default:
2346 break;
2347 }
2348
2349 if (poison_stat)
2350 dev_info(adev->dev, "Poison detected in VCN%d, sub_block%d\n",
2351 instance, sub_block);
2352
2353 return poison_stat;
2354 }
2355
vcn_v4_0_query_ras_poison_status(struct amdgpu_device * adev)2356 static bool vcn_v4_0_query_ras_poison_status(struct amdgpu_device *adev)
2357 {
2358 uint32_t inst, sub;
2359 uint32_t poison_stat = 0;
2360
2361 for (inst = 0; inst < adev->vcn.num_vcn_inst; inst++)
2362 for (sub = 0; sub < AMDGPU_VCN_V4_0_MAX_SUB_BLOCK; sub++)
2363 poison_stat +=
2364 vcn_v4_0_query_poison_by_instance(adev, inst, sub);
2365
2366 return !!poison_stat;
2367 }
2368
2369 const struct amdgpu_ras_block_hw_ops vcn_v4_0_ras_hw_ops = {
2370 .query_poison_status = vcn_v4_0_query_ras_poison_status,
2371 };
2372
2373 static struct amdgpu_vcn_ras vcn_v4_0_ras = {
2374 .ras_block = {
2375 .hw_ops = &vcn_v4_0_ras_hw_ops,
2376 .ras_late_init = amdgpu_vcn_ras_late_init,
2377 },
2378 };
2379
vcn_v4_0_set_ras_funcs(struct amdgpu_device * adev)2380 static void vcn_v4_0_set_ras_funcs(struct amdgpu_device *adev)
2381 {
2382 switch (amdgpu_ip_version(adev, VCN_HWIP, 0)) {
2383 case IP_VERSION(4, 0, 0):
2384 adev->vcn.ras = &vcn_v4_0_ras;
2385 break;
2386 default:
2387 break;
2388 }
2389 }
2390