1 /*
2 * Copyright 2016 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 */
23
24 #include <linux/delay.h>
25 #include <linux/kernel.h>
26 #include <linux/firmware.h>
27 #include <linux/module.h>
28 #include <linux/pci.h>
29
30 #include "amdgpu.h"
31 #include "amdgpu_gfx.h"
32 #include "soc15.h"
33 #include "soc15d.h"
34 #include "amdgpu_atomfirmware.h"
35 #include "amdgpu_pm.h"
36
37 #include "gc/gc_9_0_offset.h"
38 #include "gc/gc_9_0_sh_mask.h"
39
40 #include "vega10_enum.h"
41
42 #include "soc15_common.h"
43 #include "clearstate_gfx9.h"
44 #include "v9_structs.h"
45
46 #include "ivsrcid/gfx/irqsrcs_gfx_9_0.h"
47
48 #include "amdgpu_ras.h"
49
50 #include "amdgpu_ring_mux.h"
51 #include "gfx_v9_4.h"
52 #include "gfx_v9_0.h"
53 #include "gfx_v9_0_cleaner_shader.h"
54 #include "gfx_v9_4_2.h"
55
56 #include "asic_reg/pwr/pwr_10_0_offset.h"
57 #include "asic_reg/pwr/pwr_10_0_sh_mask.h"
58 #include "asic_reg/gc/gc_9_0_default.h"
59
60 #define GFX9_NUM_GFX_RINGS 1
61 #define GFX9_NUM_SW_GFX_RINGS 2
62 #define GFX9_MEC_HPD_SIZE 4096
63 #define RLCG_UCODE_LOADING_START_ADDRESS 0x00002000L
64 #define RLC_SAVE_RESTORE_ADDR_STARTING_OFFSET 0x00000000L
65
66 #define mmGCEA_PROBE_MAP 0x070c
67 #define mmGCEA_PROBE_MAP_BASE_IDX 0
68
69 MODULE_FIRMWARE("amdgpu/vega10_ce.bin");
70 MODULE_FIRMWARE("amdgpu/vega10_pfp.bin");
71 MODULE_FIRMWARE("amdgpu/vega10_me.bin");
72 MODULE_FIRMWARE("amdgpu/vega10_mec.bin");
73 MODULE_FIRMWARE("amdgpu/vega10_mec2.bin");
74 MODULE_FIRMWARE("amdgpu/vega10_rlc.bin");
75
76 MODULE_FIRMWARE("amdgpu/vega12_ce.bin");
77 MODULE_FIRMWARE("amdgpu/vega12_pfp.bin");
78 MODULE_FIRMWARE("amdgpu/vega12_me.bin");
79 MODULE_FIRMWARE("amdgpu/vega12_mec.bin");
80 MODULE_FIRMWARE("amdgpu/vega12_mec2.bin");
81 MODULE_FIRMWARE("amdgpu/vega12_rlc.bin");
82
83 MODULE_FIRMWARE("amdgpu/vega20_ce.bin");
84 MODULE_FIRMWARE("amdgpu/vega20_pfp.bin");
85 MODULE_FIRMWARE("amdgpu/vega20_me.bin");
86 MODULE_FIRMWARE("amdgpu/vega20_mec.bin");
87 MODULE_FIRMWARE("amdgpu/vega20_mec2.bin");
88 MODULE_FIRMWARE("amdgpu/vega20_rlc.bin");
89
90 MODULE_FIRMWARE("amdgpu/raven_ce.bin");
91 MODULE_FIRMWARE("amdgpu/raven_pfp.bin");
92 MODULE_FIRMWARE("amdgpu/raven_me.bin");
93 MODULE_FIRMWARE("amdgpu/raven_mec.bin");
94 MODULE_FIRMWARE("amdgpu/raven_mec2.bin");
95 MODULE_FIRMWARE("amdgpu/raven_rlc.bin");
96
97 MODULE_FIRMWARE("amdgpu/picasso_ce.bin");
98 MODULE_FIRMWARE("amdgpu/picasso_pfp.bin");
99 MODULE_FIRMWARE("amdgpu/picasso_me.bin");
100 MODULE_FIRMWARE("amdgpu/picasso_mec.bin");
101 MODULE_FIRMWARE("amdgpu/picasso_mec2.bin");
102 MODULE_FIRMWARE("amdgpu/picasso_rlc.bin");
103 MODULE_FIRMWARE("amdgpu/picasso_rlc_am4.bin");
104
105 MODULE_FIRMWARE("amdgpu/raven2_ce.bin");
106 MODULE_FIRMWARE("amdgpu/raven2_pfp.bin");
107 MODULE_FIRMWARE("amdgpu/raven2_me.bin");
108 MODULE_FIRMWARE("amdgpu/raven2_mec.bin");
109 MODULE_FIRMWARE("amdgpu/raven2_mec2.bin");
110 MODULE_FIRMWARE("amdgpu/raven2_rlc.bin");
111 MODULE_FIRMWARE("amdgpu/raven_kicker_rlc.bin");
112
113 MODULE_FIRMWARE("amdgpu/arcturus_mec.bin");
114 MODULE_FIRMWARE("amdgpu/arcturus_rlc.bin");
115
116 MODULE_FIRMWARE("amdgpu/renoir_ce.bin");
117 MODULE_FIRMWARE("amdgpu/renoir_pfp.bin");
118 MODULE_FIRMWARE("amdgpu/renoir_me.bin");
119 MODULE_FIRMWARE("amdgpu/renoir_mec.bin");
120 MODULE_FIRMWARE("amdgpu/renoir_rlc.bin");
121
122 MODULE_FIRMWARE("amdgpu/green_sardine_ce.bin");
123 MODULE_FIRMWARE("amdgpu/green_sardine_pfp.bin");
124 MODULE_FIRMWARE("amdgpu/green_sardine_me.bin");
125 MODULE_FIRMWARE("amdgpu/green_sardine_mec.bin");
126 MODULE_FIRMWARE("amdgpu/green_sardine_mec2.bin");
127 MODULE_FIRMWARE("amdgpu/green_sardine_rlc.bin");
128
129 MODULE_FIRMWARE("amdgpu/aldebaran_mec.bin");
130 MODULE_FIRMWARE("amdgpu/aldebaran_mec2.bin");
131 MODULE_FIRMWARE("amdgpu/aldebaran_rlc.bin");
132 MODULE_FIRMWARE("amdgpu/aldebaran_sjt_mec.bin");
133 MODULE_FIRMWARE("amdgpu/aldebaran_sjt_mec2.bin");
134
135 #define mmTCP_CHAN_STEER_0_ARCT 0x0b03
136 #define mmTCP_CHAN_STEER_0_ARCT_BASE_IDX 0
137 #define mmTCP_CHAN_STEER_1_ARCT 0x0b04
138 #define mmTCP_CHAN_STEER_1_ARCT_BASE_IDX 0
139 #define mmTCP_CHAN_STEER_2_ARCT 0x0b09
140 #define mmTCP_CHAN_STEER_2_ARCT_BASE_IDX 0
141 #define mmTCP_CHAN_STEER_3_ARCT 0x0b0a
142 #define mmTCP_CHAN_STEER_3_ARCT_BASE_IDX 0
143 #define mmTCP_CHAN_STEER_4_ARCT 0x0b0b
144 #define mmTCP_CHAN_STEER_4_ARCT_BASE_IDX 0
145 #define mmTCP_CHAN_STEER_5_ARCT 0x0b0c
146 #define mmTCP_CHAN_STEER_5_ARCT_BASE_IDX 0
147
148 #define mmGOLDEN_TSC_COUNT_UPPER_Renoir 0x0025
149 #define mmGOLDEN_TSC_COUNT_UPPER_Renoir_BASE_IDX 1
150 #define mmGOLDEN_TSC_COUNT_LOWER_Renoir 0x0026
151 #define mmGOLDEN_TSC_COUNT_LOWER_Renoir_BASE_IDX 1
152
153 static const struct amdgpu_hwip_reg_entry gc_reg_list_9[] = {
154 SOC15_REG_ENTRY_STR(GC, 0, mmGRBM_STATUS),
155 SOC15_REG_ENTRY_STR(GC, 0, mmGRBM_STATUS2),
156 SOC15_REG_ENTRY_STR(GC, 0, mmCP_STALLED_STAT1),
157 SOC15_REG_ENTRY_STR(GC, 0, mmCP_STALLED_STAT2),
158 SOC15_REG_ENTRY_STR(GC, 0, mmCP_CPC_STALLED_STAT1),
159 SOC15_REG_ENTRY_STR(GC, 0, mmCP_CPF_STALLED_STAT1),
160 SOC15_REG_ENTRY_STR(GC, 0, mmCP_BUSY_STAT),
161 SOC15_REG_ENTRY_STR(GC, 0, mmCP_CPC_BUSY_STAT),
162 SOC15_REG_ENTRY_STR(GC, 0, mmCP_CPF_BUSY_STAT),
163 SOC15_REG_ENTRY_STR(GC, 0, mmCP_CPF_STATUS),
164 SOC15_REG_ENTRY_STR(GC, 0, mmCP_GFX_ERROR),
165 SOC15_REG_ENTRY_STR(GC, 0, mmCP_RB_BASE),
166 SOC15_REG_ENTRY_STR(GC, 0, mmCP_RB_RPTR),
167 SOC15_REG_ENTRY_STR(GC, 0, mmCP_RB_WPTR),
168 SOC15_REG_ENTRY_STR(GC, 0, mmCP_RB0_BASE),
169 SOC15_REG_ENTRY_STR(GC, 0, mmCP_RB0_RPTR),
170 SOC15_REG_ENTRY_STR(GC, 0, mmCP_RB0_WPTR),
171 SOC15_REG_ENTRY_STR(GC, 0, mmCP_RB1_BASE),
172 SOC15_REG_ENTRY_STR(GC, 0, mmCP_RB1_RPTR),
173 SOC15_REG_ENTRY_STR(GC, 0, mmCP_RB1_WPTR),
174 SOC15_REG_ENTRY_STR(GC, 0, mmCP_RB2_BASE),
175 SOC15_REG_ENTRY_STR(GC, 0, mmCP_RB2_WPTR),
176 SOC15_REG_ENTRY_STR(GC, 0, mmCP_RB2_WPTR),
177 SOC15_REG_ENTRY_STR(GC, 0, mmCP_CE_IB1_CMD_BUFSZ),
178 SOC15_REG_ENTRY_STR(GC, 0, mmCP_CE_IB2_CMD_BUFSZ),
179 SOC15_REG_ENTRY_STR(GC, 0, mmCP_IB1_CMD_BUFSZ),
180 SOC15_REG_ENTRY_STR(GC, 0, mmCP_IB2_CMD_BUFSZ),
181 SOC15_REG_ENTRY_STR(GC, 0, mmCP_CE_IB1_BASE_LO),
182 SOC15_REG_ENTRY_STR(GC, 0, mmCP_CE_IB1_BASE_HI),
183 SOC15_REG_ENTRY_STR(GC, 0, mmCP_CE_IB1_BUFSZ),
184 SOC15_REG_ENTRY_STR(GC, 0, mmCP_CE_IB2_BASE_LO),
185 SOC15_REG_ENTRY_STR(GC, 0, mmCP_CE_IB2_BASE_HI),
186 SOC15_REG_ENTRY_STR(GC, 0, mmCP_CE_IB2_BUFSZ),
187 SOC15_REG_ENTRY_STR(GC, 0, mmCP_IB1_BASE_LO),
188 SOC15_REG_ENTRY_STR(GC, 0, mmCP_IB1_BASE_HI),
189 SOC15_REG_ENTRY_STR(GC, 0, mmCP_IB1_BUFSZ),
190 SOC15_REG_ENTRY_STR(GC, 0, mmCP_IB2_BASE_LO),
191 SOC15_REG_ENTRY_STR(GC, 0, mmCP_IB2_BASE_HI),
192 SOC15_REG_ENTRY_STR(GC, 0, mmCP_IB2_BUFSZ),
193 SOC15_REG_ENTRY_STR(GC, 0, mmCPF_UTCL1_STATUS),
194 SOC15_REG_ENTRY_STR(GC, 0, mmCPC_UTCL1_STATUS),
195 SOC15_REG_ENTRY_STR(GC, 0, mmCPG_UTCL1_STATUS),
196 SOC15_REG_ENTRY_STR(GC, 0, mmGDS_PROTECTION_FAULT),
197 SOC15_REG_ENTRY_STR(GC, 0, mmGDS_VM_PROTECTION_FAULT),
198 SOC15_REG_ENTRY_STR(GC, 0, mmIA_UTCL1_STATUS),
199 SOC15_REG_ENTRY_STR(GC, 0, mmIA_UTCL1_CNTL),
200 SOC15_REG_ENTRY_STR(GC, 0, mmPA_CL_CNTL_STATUS),
201 SOC15_REG_ENTRY_STR(GC, 0, mmRLC_UTCL1_STATUS),
202 SOC15_REG_ENTRY_STR(GC, 0, mmRMI_UTCL1_STATUS),
203 SOC15_REG_ENTRY_STR(GC, 0, mmSQC_DCACHE_UTCL1_STATUS),
204 SOC15_REG_ENTRY_STR(GC, 0, mmSQC_ICACHE_UTCL1_STATUS),
205 SOC15_REG_ENTRY_STR(GC, 0, mmSQ_UTCL1_STATUS),
206 SOC15_REG_ENTRY_STR(GC, 0, mmTCP_UTCL1_STATUS),
207 SOC15_REG_ENTRY_STR(GC, 0, mmWD_UTCL1_STATUS),
208 SOC15_REG_ENTRY_STR(GC, 0, mmVM_L2_PROTECTION_FAULT_CNTL),
209 SOC15_REG_ENTRY_STR(GC, 0, mmVM_L2_PROTECTION_FAULT_STATUS),
210 SOC15_REG_ENTRY_STR(GC, 0, mmCP_DEBUG),
211 SOC15_REG_ENTRY_STR(GC, 0, mmCP_MEC_CNTL),
212 SOC15_REG_ENTRY_STR(GC, 0, mmCP_CE_INSTR_PNTR),
213 SOC15_REG_ENTRY_STR(GC, 0, mmCP_MEC1_INSTR_PNTR),
214 SOC15_REG_ENTRY_STR(GC, 0, mmCP_MEC2_INSTR_PNTR),
215 SOC15_REG_ENTRY_STR(GC, 0, mmCP_ME_INSTR_PNTR),
216 SOC15_REG_ENTRY_STR(GC, 0, mmCP_PFP_INSTR_PNTR),
217 SOC15_REG_ENTRY_STR(GC, 0, mmCP_CPC_STATUS),
218 SOC15_REG_ENTRY_STR(GC, 0, mmRLC_STAT),
219 SOC15_REG_ENTRY_STR(GC, 0, mmRLC_SMU_COMMAND),
220 SOC15_REG_ENTRY_STR(GC, 0, mmRLC_SMU_MESSAGE),
221 SOC15_REG_ENTRY_STR(GC, 0, mmRLC_SMU_ARGUMENT_1),
222 SOC15_REG_ENTRY_STR(GC, 0, mmRLC_SMU_ARGUMENT_2),
223 SOC15_REG_ENTRY_STR(GC, 0, mmSMU_RLC_RESPONSE),
224 SOC15_REG_ENTRY_STR(GC, 0, mmRLC_SAFE_MODE),
225 SOC15_REG_ENTRY_STR(GC, 0, mmRLC_SMU_SAFE_MODE),
226 SOC15_REG_ENTRY_STR(GC, 0, mmRLC_INT_STAT),
227 SOC15_REG_ENTRY_STR(GC, 0, mmRLC_GPM_GENERAL_6),
228 /* cp header registers */
229 SOC15_REG_ENTRY_STR(GC, 0, mmCP_CE_HEADER_DUMP),
230 SOC15_REG_ENTRY_STR(GC, 0, mmCP_MEC_ME1_HEADER_DUMP),
231 SOC15_REG_ENTRY_STR(GC, 0, mmCP_MEC_ME2_HEADER_DUMP),
232 SOC15_REG_ENTRY_STR(GC, 0, mmCP_PFP_HEADER_DUMP),
233 SOC15_REG_ENTRY_STR(GC, 0, mmCP_ME_HEADER_DUMP),
234 /* SE status registers */
235 SOC15_REG_ENTRY_STR(GC, 0, mmGRBM_STATUS_SE0),
236 SOC15_REG_ENTRY_STR(GC, 0, mmGRBM_STATUS_SE1),
237 SOC15_REG_ENTRY_STR(GC, 0, mmGRBM_STATUS_SE2),
238 SOC15_REG_ENTRY_STR(GC, 0, mmGRBM_STATUS_SE3)
239 };
240
241 static const struct amdgpu_hwip_reg_entry gc_cp_reg_list_9[] = {
242 /* compute queue registers */
243 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_VMID),
244 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_ACTIVE),
245 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_PERSISTENT_STATE),
246 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_PIPE_PRIORITY),
247 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_QUEUE_PRIORITY),
248 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_QUANTUM),
249 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_PQ_BASE),
250 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_PQ_BASE_HI),
251 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_PQ_RPTR),
252 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_PQ_WPTR_POLL_ADDR),
253 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_PQ_WPTR_POLL_ADDR_HI),
254 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL),
255 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_PQ_CONTROL),
256 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_IB_BASE_ADDR),
257 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_IB_BASE_ADDR_HI),
258 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_IB_RPTR),
259 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_IB_CONTROL),
260 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_DEQUEUE_REQUEST),
261 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_EOP_BASE_ADDR),
262 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_EOP_BASE_ADDR_HI),
263 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_EOP_CONTROL),
264 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_EOP_RPTR),
265 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_EOP_WPTR),
266 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_EOP_EVENTS),
267 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_CTX_SAVE_BASE_ADDR_LO),
268 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_CTX_SAVE_BASE_ADDR_HI),
269 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_CTX_SAVE_CONTROL),
270 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_CNTL_STACK_OFFSET),
271 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_CNTL_STACK_SIZE),
272 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_WG_STATE_OFFSET),
273 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_CTX_SAVE_SIZE),
274 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_GDS_RESOURCE_STATE),
275 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_ERROR),
276 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_EOP_WPTR_MEM),
277 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_PQ_WPTR_LO),
278 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_PQ_WPTR_HI),
279 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_GFX_STATUS),
280 };
281
282 enum ta_ras_gfx_subblock {
283 /*CPC*/
284 TA_RAS_BLOCK__GFX_CPC_INDEX_START = 0,
285 TA_RAS_BLOCK__GFX_CPC_SCRATCH = TA_RAS_BLOCK__GFX_CPC_INDEX_START,
286 TA_RAS_BLOCK__GFX_CPC_UCODE,
287 TA_RAS_BLOCK__GFX_DC_STATE_ME1,
288 TA_RAS_BLOCK__GFX_DC_CSINVOC_ME1,
289 TA_RAS_BLOCK__GFX_DC_RESTORE_ME1,
290 TA_RAS_BLOCK__GFX_DC_STATE_ME2,
291 TA_RAS_BLOCK__GFX_DC_CSINVOC_ME2,
292 TA_RAS_BLOCK__GFX_DC_RESTORE_ME2,
293 TA_RAS_BLOCK__GFX_CPC_INDEX_END = TA_RAS_BLOCK__GFX_DC_RESTORE_ME2,
294 /* CPF*/
295 TA_RAS_BLOCK__GFX_CPF_INDEX_START,
296 TA_RAS_BLOCK__GFX_CPF_ROQ_ME2 = TA_RAS_BLOCK__GFX_CPF_INDEX_START,
297 TA_RAS_BLOCK__GFX_CPF_ROQ_ME1,
298 TA_RAS_BLOCK__GFX_CPF_TAG,
299 TA_RAS_BLOCK__GFX_CPF_INDEX_END = TA_RAS_BLOCK__GFX_CPF_TAG,
300 /* CPG*/
301 TA_RAS_BLOCK__GFX_CPG_INDEX_START,
302 TA_RAS_BLOCK__GFX_CPG_DMA_ROQ = TA_RAS_BLOCK__GFX_CPG_INDEX_START,
303 TA_RAS_BLOCK__GFX_CPG_DMA_TAG,
304 TA_RAS_BLOCK__GFX_CPG_TAG,
305 TA_RAS_BLOCK__GFX_CPG_INDEX_END = TA_RAS_BLOCK__GFX_CPG_TAG,
306 /* GDS*/
307 TA_RAS_BLOCK__GFX_GDS_INDEX_START,
308 TA_RAS_BLOCK__GFX_GDS_MEM = TA_RAS_BLOCK__GFX_GDS_INDEX_START,
309 TA_RAS_BLOCK__GFX_GDS_INPUT_QUEUE,
310 TA_RAS_BLOCK__GFX_GDS_OA_PHY_CMD_RAM_MEM,
311 TA_RAS_BLOCK__GFX_GDS_OA_PHY_DATA_RAM_MEM,
312 TA_RAS_BLOCK__GFX_GDS_OA_PIPE_MEM,
313 TA_RAS_BLOCK__GFX_GDS_INDEX_END = TA_RAS_BLOCK__GFX_GDS_OA_PIPE_MEM,
314 /* SPI*/
315 TA_RAS_BLOCK__GFX_SPI_SR_MEM,
316 /* SQ*/
317 TA_RAS_BLOCK__GFX_SQ_INDEX_START,
318 TA_RAS_BLOCK__GFX_SQ_SGPR = TA_RAS_BLOCK__GFX_SQ_INDEX_START,
319 TA_RAS_BLOCK__GFX_SQ_LDS_D,
320 TA_RAS_BLOCK__GFX_SQ_LDS_I,
321 TA_RAS_BLOCK__GFX_SQ_VGPR, /* VGPR = SP*/
322 TA_RAS_BLOCK__GFX_SQ_INDEX_END = TA_RAS_BLOCK__GFX_SQ_VGPR,
323 /* SQC (3 ranges)*/
324 TA_RAS_BLOCK__GFX_SQC_INDEX_START,
325 /* SQC range 0*/
326 TA_RAS_BLOCK__GFX_SQC_INDEX0_START = TA_RAS_BLOCK__GFX_SQC_INDEX_START,
327 TA_RAS_BLOCK__GFX_SQC_INST_UTCL1_LFIFO =
328 TA_RAS_BLOCK__GFX_SQC_INDEX0_START,
329 TA_RAS_BLOCK__GFX_SQC_DATA_CU0_WRITE_DATA_BUF,
330 TA_RAS_BLOCK__GFX_SQC_DATA_CU0_UTCL1_LFIFO,
331 TA_RAS_BLOCK__GFX_SQC_DATA_CU1_WRITE_DATA_BUF,
332 TA_RAS_BLOCK__GFX_SQC_DATA_CU1_UTCL1_LFIFO,
333 TA_RAS_BLOCK__GFX_SQC_DATA_CU2_WRITE_DATA_BUF,
334 TA_RAS_BLOCK__GFX_SQC_DATA_CU2_UTCL1_LFIFO,
335 TA_RAS_BLOCK__GFX_SQC_INDEX0_END =
336 TA_RAS_BLOCK__GFX_SQC_DATA_CU2_UTCL1_LFIFO,
337 /* SQC range 1*/
338 TA_RAS_BLOCK__GFX_SQC_INDEX1_START,
339 TA_RAS_BLOCK__GFX_SQC_INST_BANKA_TAG_RAM =
340 TA_RAS_BLOCK__GFX_SQC_INDEX1_START,
341 TA_RAS_BLOCK__GFX_SQC_INST_BANKA_UTCL1_MISS_FIFO,
342 TA_RAS_BLOCK__GFX_SQC_INST_BANKA_MISS_FIFO,
343 TA_RAS_BLOCK__GFX_SQC_INST_BANKA_BANK_RAM,
344 TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_TAG_RAM,
345 TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_HIT_FIFO,
346 TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_MISS_FIFO,
347 TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_DIRTY_BIT_RAM,
348 TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_BANK_RAM,
349 TA_RAS_BLOCK__GFX_SQC_INDEX1_END =
350 TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_BANK_RAM,
351 /* SQC range 2*/
352 TA_RAS_BLOCK__GFX_SQC_INDEX2_START,
353 TA_RAS_BLOCK__GFX_SQC_INST_BANKB_TAG_RAM =
354 TA_RAS_BLOCK__GFX_SQC_INDEX2_START,
355 TA_RAS_BLOCK__GFX_SQC_INST_BANKB_UTCL1_MISS_FIFO,
356 TA_RAS_BLOCK__GFX_SQC_INST_BANKB_MISS_FIFO,
357 TA_RAS_BLOCK__GFX_SQC_INST_BANKB_BANK_RAM,
358 TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_TAG_RAM,
359 TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_HIT_FIFO,
360 TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_MISS_FIFO,
361 TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_DIRTY_BIT_RAM,
362 TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_BANK_RAM,
363 TA_RAS_BLOCK__GFX_SQC_INDEX2_END =
364 TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_BANK_RAM,
365 TA_RAS_BLOCK__GFX_SQC_INDEX_END = TA_RAS_BLOCK__GFX_SQC_INDEX2_END,
366 /* TA*/
367 TA_RAS_BLOCK__GFX_TA_INDEX_START,
368 TA_RAS_BLOCK__GFX_TA_FS_DFIFO = TA_RAS_BLOCK__GFX_TA_INDEX_START,
369 TA_RAS_BLOCK__GFX_TA_FS_AFIFO,
370 TA_RAS_BLOCK__GFX_TA_FL_LFIFO,
371 TA_RAS_BLOCK__GFX_TA_FX_LFIFO,
372 TA_RAS_BLOCK__GFX_TA_FS_CFIFO,
373 TA_RAS_BLOCK__GFX_TA_INDEX_END = TA_RAS_BLOCK__GFX_TA_FS_CFIFO,
374 /* TCA*/
375 TA_RAS_BLOCK__GFX_TCA_INDEX_START,
376 TA_RAS_BLOCK__GFX_TCA_HOLE_FIFO = TA_RAS_BLOCK__GFX_TCA_INDEX_START,
377 TA_RAS_BLOCK__GFX_TCA_REQ_FIFO,
378 TA_RAS_BLOCK__GFX_TCA_INDEX_END = TA_RAS_BLOCK__GFX_TCA_REQ_FIFO,
379 /* TCC (5 sub-ranges)*/
380 TA_RAS_BLOCK__GFX_TCC_INDEX_START,
381 /* TCC range 0*/
382 TA_RAS_BLOCK__GFX_TCC_INDEX0_START = TA_RAS_BLOCK__GFX_TCC_INDEX_START,
383 TA_RAS_BLOCK__GFX_TCC_CACHE_DATA = TA_RAS_BLOCK__GFX_TCC_INDEX0_START,
384 TA_RAS_BLOCK__GFX_TCC_CACHE_DATA_BANK_0_1,
385 TA_RAS_BLOCK__GFX_TCC_CACHE_DATA_BANK_1_0,
386 TA_RAS_BLOCK__GFX_TCC_CACHE_DATA_BANK_1_1,
387 TA_RAS_BLOCK__GFX_TCC_CACHE_DIRTY_BANK_0,
388 TA_RAS_BLOCK__GFX_TCC_CACHE_DIRTY_BANK_1,
389 TA_RAS_BLOCK__GFX_TCC_HIGH_RATE_TAG,
390 TA_RAS_BLOCK__GFX_TCC_LOW_RATE_TAG,
391 TA_RAS_BLOCK__GFX_TCC_INDEX0_END = TA_RAS_BLOCK__GFX_TCC_LOW_RATE_TAG,
392 /* TCC range 1*/
393 TA_RAS_BLOCK__GFX_TCC_INDEX1_START,
394 TA_RAS_BLOCK__GFX_TCC_IN_USE_DEC = TA_RAS_BLOCK__GFX_TCC_INDEX1_START,
395 TA_RAS_BLOCK__GFX_TCC_IN_USE_TRANSFER,
396 TA_RAS_BLOCK__GFX_TCC_INDEX1_END =
397 TA_RAS_BLOCK__GFX_TCC_IN_USE_TRANSFER,
398 /* TCC range 2*/
399 TA_RAS_BLOCK__GFX_TCC_INDEX2_START,
400 TA_RAS_BLOCK__GFX_TCC_RETURN_DATA = TA_RAS_BLOCK__GFX_TCC_INDEX2_START,
401 TA_RAS_BLOCK__GFX_TCC_RETURN_CONTROL,
402 TA_RAS_BLOCK__GFX_TCC_UC_ATOMIC_FIFO,
403 TA_RAS_BLOCK__GFX_TCC_WRITE_RETURN,
404 TA_RAS_BLOCK__GFX_TCC_WRITE_CACHE_READ,
405 TA_RAS_BLOCK__GFX_TCC_SRC_FIFO,
406 TA_RAS_BLOCK__GFX_TCC_SRC_FIFO_NEXT_RAM,
407 TA_RAS_BLOCK__GFX_TCC_CACHE_TAG_PROBE_FIFO,
408 TA_RAS_BLOCK__GFX_TCC_INDEX2_END =
409 TA_RAS_BLOCK__GFX_TCC_CACHE_TAG_PROBE_FIFO,
410 /* TCC range 3*/
411 TA_RAS_BLOCK__GFX_TCC_INDEX3_START,
412 TA_RAS_BLOCK__GFX_TCC_LATENCY_FIFO = TA_RAS_BLOCK__GFX_TCC_INDEX3_START,
413 TA_RAS_BLOCK__GFX_TCC_LATENCY_FIFO_NEXT_RAM,
414 TA_RAS_BLOCK__GFX_TCC_INDEX3_END =
415 TA_RAS_BLOCK__GFX_TCC_LATENCY_FIFO_NEXT_RAM,
416 /* TCC range 4*/
417 TA_RAS_BLOCK__GFX_TCC_INDEX4_START,
418 TA_RAS_BLOCK__GFX_TCC_WRRET_TAG_WRITE_RETURN =
419 TA_RAS_BLOCK__GFX_TCC_INDEX4_START,
420 TA_RAS_BLOCK__GFX_TCC_ATOMIC_RETURN_BUFFER,
421 TA_RAS_BLOCK__GFX_TCC_INDEX4_END =
422 TA_RAS_BLOCK__GFX_TCC_ATOMIC_RETURN_BUFFER,
423 TA_RAS_BLOCK__GFX_TCC_INDEX_END = TA_RAS_BLOCK__GFX_TCC_INDEX4_END,
424 /* TCI*/
425 TA_RAS_BLOCK__GFX_TCI_WRITE_RAM,
426 /* TCP*/
427 TA_RAS_BLOCK__GFX_TCP_INDEX_START,
428 TA_RAS_BLOCK__GFX_TCP_CACHE_RAM = TA_RAS_BLOCK__GFX_TCP_INDEX_START,
429 TA_RAS_BLOCK__GFX_TCP_LFIFO_RAM,
430 TA_RAS_BLOCK__GFX_TCP_CMD_FIFO,
431 TA_RAS_BLOCK__GFX_TCP_VM_FIFO,
432 TA_RAS_BLOCK__GFX_TCP_DB_RAM,
433 TA_RAS_BLOCK__GFX_TCP_UTCL1_LFIFO0,
434 TA_RAS_BLOCK__GFX_TCP_UTCL1_LFIFO1,
435 TA_RAS_BLOCK__GFX_TCP_INDEX_END = TA_RAS_BLOCK__GFX_TCP_UTCL1_LFIFO1,
436 /* TD*/
437 TA_RAS_BLOCK__GFX_TD_INDEX_START,
438 TA_RAS_BLOCK__GFX_TD_SS_FIFO_LO = TA_RAS_BLOCK__GFX_TD_INDEX_START,
439 TA_RAS_BLOCK__GFX_TD_SS_FIFO_HI,
440 TA_RAS_BLOCK__GFX_TD_CS_FIFO,
441 TA_RAS_BLOCK__GFX_TD_INDEX_END = TA_RAS_BLOCK__GFX_TD_CS_FIFO,
442 /* EA (3 sub-ranges)*/
443 TA_RAS_BLOCK__GFX_EA_INDEX_START,
444 /* EA range 0*/
445 TA_RAS_BLOCK__GFX_EA_INDEX0_START = TA_RAS_BLOCK__GFX_EA_INDEX_START,
446 TA_RAS_BLOCK__GFX_EA_DRAMRD_CMDMEM = TA_RAS_BLOCK__GFX_EA_INDEX0_START,
447 TA_RAS_BLOCK__GFX_EA_DRAMWR_CMDMEM,
448 TA_RAS_BLOCK__GFX_EA_DRAMWR_DATAMEM,
449 TA_RAS_BLOCK__GFX_EA_RRET_TAGMEM,
450 TA_RAS_BLOCK__GFX_EA_WRET_TAGMEM,
451 TA_RAS_BLOCK__GFX_EA_GMIRD_CMDMEM,
452 TA_RAS_BLOCK__GFX_EA_GMIWR_CMDMEM,
453 TA_RAS_BLOCK__GFX_EA_GMIWR_DATAMEM,
454 TA_RAS_BLOCK__GFX_EA_INDEX0_END = TA_RAS_BLOCK__GFX_EA_GMIWR_DATAMEM,
455 /* EA range 1*/
456 TA_RAS_BLOCK__GFX_EA_INDEX1_START,
457 TA_RAS_BLOCK__GFX_EA_DRAMRD_PAGEMEM = TA_RAS_BLOCK__GFX_EA_INDEX1_START,
458 TA_RAS_BLOCK__GFX_EA_DRAMWR_PAGEMEM,
459 TA_RAS_BLOCK__GFX_EA_IORD_CMDMEM,
460 TA_RAS_BLOCK__GFX_EA_IOWR_CMDMEM,
461 TA_RAS_BLOCK__GFX_EA_IOWR_DATAMEM,
462 TA_RAS_BLOCK__GFX_EA_GMIRD_PAGEMEM,
463 TA_RAS_BLOCK__GFX_EA_GMIWR_PAGEMEM,
464 TA_RAS_BLOCK__GFX_EA_INDEX1_END = TA_RAS_BLOCK__GFX_EA_GMIWR_PAGEMEM,
465 /* EA range 2*/
466 TA_RAS_BLOCK__GFX_EA_INDEX2_START,
467 TA_RAS_BLOCK__GFX_EA_MAM_D0MEM = TA_RAS_BLOCK__GFX_EA_INDEX2_START,
468 TA_RAS_BLOCK__GFX_EA_MAM_D1MEM,
469 TA_RAS_BLOCK__GFX_EA_MAM_D2MEM,
470 TA_RAS_BLOCK__GFX_EA_MAM_D3MEM,
471 TA_RAS_BLOCK__GFX_EA_INDEX2_END = TA_RAS_BLOCK__GFX_EA_MAM_D3MEM,
472 TA_RAS_BLOCK__GFX_EA_INDEX_END = TA_RAS_BLOCK__GFX_EA_INDEX2_END,
473 /* UTC VM L2 bank*/
474 TA_RAS_BLOCK__UTC_VML2_BANK_CACHE,
475 /* UTC VM walker*/
476 TA_RAS_BLOCK__UTC_VML2_WALKER,
477 /* UTC ATC L2 2MB cache*/
478 TA_RAS_BLOCK__UTC_ATCL2_CACHE_2M_BANK,
479 /* UTC ATC L2 4KB cache*/
480 TA_RAS_BLOCK__UTC_ATCL2_CACHE_4K_BANK,
481 TA_RAS_BLOCK__GFX_MAX
482 };
483
484 struct ras_gfx_subblock {
485 unsigned char *name;
486 int ta_subblock;
487 int hw_supported_error_type;
488 int sw_supported_error_type;
489 };
490
491 #define AMDGPU_RAS_SUB_BLOCK(subblock, a, b, c, d, e, f, g, h) \
492 [AMDGPU_RAS_BLOCK__##subblock] = { \
493 #subblock, \
494 TA_RAS_BLOCK__##subblock, \
495 ((a) | ((b) << 1) | ((c) << 2) | ((d) << 3)), \
496 (((e) << 1) | ((f) << 3) | (g) | ((h) << 2)), \
497 }
498
499 static const struct ras_gfx_subblock ras_gfx_subblocks[] = {
500 AMDGPU_RAS_SUB_BLOCK(GFX_CPC_SCRATCH, 0, 1, 1, 1, 1, 0, 0, 1),
501 AMDGPU_RAS_SUB_BLOCK(GFX_CPC_UCODE, 0, 1, 1, 1, 1, 0, 0, 1),
502 AMDGPU_RAS_SUB_BLOCK(GFX_DC_STATE_ME1, 1, 0, 0, 1, 0, 0, 1, 0),
503 AMDGPU_RAS_SUB_BLOCK(GFX_DC_CSINVOC_ME1, 1, 0, 0, 1, 0, 0, 0, 0),
504 AMDGPU_RAS_SUB_BLOCK(GFX_DC_RESTORE_ME1, 1, 0, 0, 1, 0, 0, 0, 0),
505 AMDGPU_RAS_SUB_BLOCK(GFX_DC_STATE_ME2, 1, 0, 0, 1, 0, 0, 0, 0),
506 AMDGPU_RAS_SUB_BLOCK(GFX_DC_CSINVOC_ME2, 1, 0, 0, 1, 0, 0, 0, 0),
507 AMDGPU_RAS_SUB_BLOCK(GFX_DC_RESTORE_ME2, 1, 0, 0, 1, 0, 0, 0, 0),
508 AMDGPU_RAS_SUB_BLOCK(GFX_CPF_ROQ_ME2, 1, 0, 0, 1, 0, 0, 0, 0),
509 AMDGPU_RAS_SUB_BLOCK(GFX_CPF_ROQ_ME1, 1, 0, 0, 1, 0, 0, 1, 0),
510 AMDGPU_RAS_SUB_BLOCK(GFX_CPF_TAG, 0, 1, 1, 1, 1, 0, 0, 1),
511 AMDGPU_RAS_SUB_BLOCK(GFX_CPG_DMA_ROQ, 1, 0, 0, 1, 0, 0, 1, 0),
512 AMDGPU_RAS_SUB_BLOCK(GFX_CPG_DMA_TAG, 0, 1, 1, 1, 0, 1, 0, 1),
513 AMDGPU_RAS_SUB_BLOCK(GFX_CPG_TAG, 0, 1, 1, 1, 1, 1, 0, 1),
514 AMDGPU_RAS_SUB_BLOCK(GFX_GDS_MEM, 0, 1, 1, 1, 0, 0, 0, 0),
515 AMDGPU_RAS_SUB_BLOCK(GFX_GDS_INPUT_QUEUE, 1, 0, 0, 1, 0, 0, 0, 0),
516 AMDGPU_RAS_SUB_BLOCK(GFX_GDS_OA_PHY_CMD_RAM_MEM, 0, 1, 1, 1, 0, 0, 0,
517 0),
518 AMDGPU_RAS_SUB_BLOCK(GFX_GDS_OA_PHY_DATA_RAM_MEM, 1, 0, 0, 1, 0, 0, 0,
519 0),
520 AMDGPU_RAS_SUB_BLOCK(GFX_GDS_OA_PIPE_MEM, 0, 1, 1, 1, 0, 0, 0, 0),
521 AMDGPU_RAS_SUB_BLOCK(GFX_SPI_SR_MEM, 1, 0, 0, 1, 0, 0, 0, 0),
522 AMDGPU_RAS_SUB_BLOCK(GFX_SQ_SGPR, 0, 1, 1, 1, 0, 0, 0, 0),
523 AMDGPU_RAS_SUB_BLOCK(GFX_SQ_LDS_D, 0, 1, 1, 1, 1, 0, 0, 1),
524 AMDGPU_RAS_SUB_BLOCK(GFX_SQ_LDS_I, 0, 1, 1, 1, 0, 0, 0, 0),
525 AMDGPU_RAS_SUB_BLOCK(GFX_SQ_VGPR, 0, 1, 1, 1, 0, 0, 0, 0),
526 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_UTCL1_LFIFO, 0, 1, 1, 1, 0, 0, 0, 1),
527 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU0_WRITE_DATA_BUF, 0, 1, 1, 1, 0, 0,
528 0, 0),
529 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU0_UTCL1_LFIFO, 0, 1, 1, 1, 0, 0, 0,
530 0),
531 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU1_WRITE_DATA_BUF, 0, 1, 1, 1, 0, 0,
532 0, 0),
533 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU1_UTCL1_LFIFO, 0, 1, 1, 1, 1, 0, 0,
534 0),
535 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU2_WRITE_DATA_BUF, 0, 1, 1, 1, 0, 0,
536 0, 0),
537 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU2_UTCL1_LFIFO, 0, 1, 1, 1, 0, 0, 0,
538 0),
539 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKA_TAG_RAM, 0, 1, 1, 1, 1, 0, 0,
540 1),
541 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKA_UTCL1_MISS_FIFO, 1, 0, 0, 1, 0,
542 0, 0, 0),
543 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKA_MISS_FIFO, 1, 0, 0, 1, 0, 0, 0,
544 0),
545 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKA_BANK_RAM, 0, 1, 1, 1, 0, 0, 0,
546 0),
547 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_TAG_RAM, 0, 1, 1, 1, 0, 0, 0,
548 0),
549 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_HIT_FIFO, 1, 0, 0, 1, 0, 0, 0,
550 0),
551 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_MISS_FIFO, 1, 0, 0, 1, 0, 0, 0,
552 0),
553 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_DIRTY_BIT_RAM, 1, 0, 0, 1, 0, 0,
554 0, 0),
555 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_BANK_RAM, 0, 1, 1, 1, 0, 0, 0,
556 0),
557 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKB_TAG_RAM, 0, 1, 1, 1, 1, 0, 0,
558 0),
559 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKB_UTCL1_MISS_FIFO, 1, 0, 0, 1, 0,
560 0, 0, 0),
561 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKB_MISS_FIFO, 1, 0, 0, 1, 0, 0, 0,
562 0),
563 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKB_BANK_RAM, 0, 1, 1, 1, 0, 0, 0,
564 0),
565 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_TAG_RAM, 0, 1, 1, 1, 0, 0, 0,
566 0),
567 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_HIT_FIFO, 1, 0, 0, 1, 0, 0, 0,
568 0),
569 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_MISS_FIFO, 1, 0, 0, 1, 0, 0, 0,
570 0),
571 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_DIRTY_BIT_RAM, 1, 0, 0, 1, 0, 0,
572 0, 0),
573 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_BANK_RAM, 0, 1, 1, 1, 0, 0, 0,
574 0),
575 AMDGPU_RAS_SUB_BLOCK(GFX_TA_FS_DFIFO, 0, 1, 1, 1, 1, 0, 0, 1),
576 AMDGPU_RAS_SUB_BLOCK(GFX_TA_FS_AFIFO, 1, 0, 0, 1, 0, 0, 0, 0),
577 AMDGPU_RAS_SUB_BLOCK(GFX_TA_FL_LFIFO, 1, 0, 0, 1, 0, 0, 0, 0),
578 AMDGPU_RAS_SUB_BLOCK(GFX_TA_FX_LFIFO, 1, 0, 0, 1, 0, 0, 0, 0),
579 AMDGPU_RAS_SUB_BLOCK(GFX_TA_FS_CFIFO, 1, 0, 0, 1, 0, 0, 0, 0),
580 AMDGPU_RAS_SUB_BLOCK(GFX_TCA_HOLE_FIFO, 1, 0, 0, 1, 0, 1, 1, 0),
581 AMDGPU_RAS_SUB_BLOCK(GFX_TCA_REQ_FIFO, 1, 0, 0, 1, 0, 0, 0, 0),
582 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DATA, 0, 1, 1, 1, 1, 0, 0, 1),
583 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DATA_BANK_0_1, 0, 1, 1, 1, 1, 0, 0,
584 1),
585 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DATA_BANK_1_0, 0, 1, 1, 1, 1, 0, 0,
586 1),
587 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DATA_BANK_1_1, 0, 1, 1, 1, 1, 0, 0,
588 1),
589 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DIRTY_BANK_0, 0, 1, 1, 1, 0, 0, 0,
590 0),
591 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DIRTY_BANK_1, 0, 1, 1, 1, 0, 0, 0,
592 0),
593 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_HIGH_RATE_TAG, 0, 1, 1, 1, 0, 0, 0, 0),
594 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_LOW_RATE_TAG, 0, 1, 1, 1, 0, 0, 0, 0),
595 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_IN_USE_DEC, 1, 0, 0, 1, 0, 0, 0, 0),
596 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_IN_USE_TRANSFER, 1, 0, 0, 1, 0, 0, 0, 0),
597 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_RETURN_DATA, 1, 0, 0, 1, 0, 0, 0, 0),
598 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_RETURN_CONTROL, 1, 0, 0, 1, 0, 0, 0, 0),
599 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_UC_ATOMIC_FIFO, 1, 0, 0, 1, 0, 0, 0, 0),
600 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_WRITE_RETURN, 1, 0, 0, 1, 0, 1, 1, 0),
601 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_WRITE_CACHE_READ, 1, 0, 0, 1, 0, 0, 0, 0),
602 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_SRC_FIFO, 0, 1, 1, 1, 0, 0, 0, 0),
603 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_SRC_FIFO_NEXT_RAM, 1, 0, 0, 1, 0, 0, 1, 0),
604 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_TAG_PROBE_FIFO, 1, 0, 0, 1, 0, 0, 0,
605 0),
606 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_LATENCY_FIFO, 1, 0, 0, 1, 0, 0, 0, 0),
607 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_LATENCY_FIFO_NEXT_RAM, 1, 0, 0, 1, 0, 0, 0,
608 0),
609 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_WRRET_TAG_WRITE_RETURN, 1, 0, 0, 1, 0, 0,
610 0, 0),
611 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_ATOMIC_RETURN_BUFFER, 1, 0, 0, 1, 0, 0, 0,
612 0),
613 AMDGPU_RAS_SUB_BLOCK(GFX_TCI_WRITE_RAM, 1, 0, 0, 1, 0, 0, 0, 0),
614 AMDGPU_RAS_SUB_BLOCK(GFX_TCP_CACHE_RAM, 0, 1, 1, 1, 1, 0, 0, 1),
615 AMDGPU_RAS_SUB_BLOCK(GFX_TCP_LFIFO_RAM, 0, 1, 1, 1, 0, 0, 0, 0),
616 AMDGPU_RAS_SUB_BLOCK(GFX_TCP_CMD_FIFO, 1, 0, 0, 1, 0, 0, 0, 0),
617 AMDGPU_RAS_SUB_BLOCK(GFX_TCP_VM_FIFO, 0, 1, 1, 1, 0, 0, 0, 0),
618 AMDGPU_RAS_SUB_BLOCK(GFX_TCP_DB_RAM, 1, 0, 0, 1, 0, 0, 0, 0),
619 AMDGPU_RAS_SUB_BLOCK(GFX_TCP_UTCL1_LFIFO0, 0, 1, 1, 1, 0, 0, 0, 0),
620 AMDGPU_RAS_SUB_BLOCK(GFX_TCP_UTCL1_LFIFO1, 0, 1, 1, 1, 0, 0, 0, 0),
621 AMDGPU_RAS_SUB_BLOCK(GFX_TD_SS_FIFO_LO, 0, 1, 1, 1, 1, 0, 0, 1),
622 AMDGPU_RAS_SUB_BLOCK(GFX_TD_SS_FIFO_HI, 0, 1, 1, 1, 0, 0, 0, 0),
623 AMDGPU_RAS_SUB_BLOCK(GFX_TD_CS_FIFO, 1, 0, 0, 1, 0, 0, 0, 0),
624 AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMRD_CMDMEM, 0, 1, 1, 1, 1, 0, 0, 1),
625 AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMWR_CMDMEM, 0, 1, 1, 1, 0, 0, 0, 0),
626 AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMWR_DATAMEM, 0, 1, 1, 1, 0, 0, 0, 0),
627 AMDGPU_RAS_SUB_BLOCK(GFX_EA_RRET_TAGMEM, 0, 1, 1, 1, 0, 0, 0, 0),
628 AMDGPU_RAS_SUB_BLOCK(GFX_EA_WRET_TAGMEM, 0, 1, 1, 1, 0, 0, 0, 0),
629 AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIRD_CMDMEM, 0, 1, 1, 1, 0, 0, 0, 0),
630 AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIWR_CMDMEM, 0, 1, 1, 1, 0, 0, 0, 0),
631 AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIWR_DATAMEM, 0, 1, 1, 1, 0, 0, 0, 0),
632 AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMRD_PAGEMEM, 1, 0, 0, 1, 0, 0, 0, 0),
633 AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMWR_PAGEMEM, 1, 0, 0, 1, 0, 0, 0, 0),
634 AMDGPU_RAS_SUB_BLOCK(GFX_EA_IORD_CMDMEM, 1, 0, 0, 1, 0, 0, 0, 0),
635 AMDGPU_RAS_SUB_BLOCK(GFX_EA_IOWR_CMDMEM, 1, 0, 0, 1, 0, 0, 0, 0),
636 AMDGPU_RAS_SUB_BLOCK(GFX_EA_IOWR_DATAMEM, 1, 0, 0, 1, 0, 0, 0, 0),
637 AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIRD_PAGEMEM, 1, 0, 0, 1, 0, 0, 0, 0),
638 AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIWR_PAGEMEM, 1, 0, 0, 1, 0, 0, 0, 0),
639 AMDGPU_RAS_SUB_BLOCK(GFX_EA_MAM_D0MEM, 1, 0, 0, 1, 0, 0, 0, 0),
640 AMDGPU_RAS_SUB_BLOCK(GFX_EA_MAM_D1MEM, 1, 0, 0, 1, 0, 0, 0, 0),
641 AMDGPU_RAS_SUB_BLOCK(GFX_EA_MAM_D2MEM, 1, 0, 0, 1, 0, 0, 0, 0),
642 AMDGPU_RAS_SUB_BLOCK(GFX_EA_MAM_D3MEM, 1, 0, 0, 1, 0, 0, 0, 0),
643 AMDGPU_RAS_SUB_BLOCK(UTC_VML2_BANK_CACHE, 0, 1, 1, 1, 0, 0, 0, 0),
644 AMDGPU_RAS_SUB_BLOCK(UTC_VML2_WALKER, 0, 1, 1, 1, 0, 0, 0, 0),
645 AMDGPU_RAS_SUB_BLOCK(UTC_ATCL2_CACHE_2M_BANK, 1, 0, 0, 1, 0, 0, 0, 0),
646 AMDGPU_RAS_SUB_BLOCK(UTC_ATCL2_CACHE_4K_BANK, 0, 1, 1, 1, 0, 0, 0, 0),
647 };
648
649 static const struct soc15_reg_golden golden_settings_gc_9_0[] =
650 {
651 SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0xf00fffff, 0x00000400),
652 SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG3, 0x80000000, 0x80000000),
653 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_GPU_ID, 0x0000000f, 0x00000000),
654 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_BINNER_EVENT_CNTL_3, 0x00000003, 0x82400024),
655 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE, 0x3fffffff, 0x00000001),
656 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000),
657 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSH_MEM_CONFIG, 0x00001000, 0x00001000),
658 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_CU_0, 0x0007ffff, 0x00000800),
659 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_CU_1, 0x0007ffff, 0x00000800),
660 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_EN_CU_0, 0x01ffffff, 0x00ffff87),
661 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_EN_CU_1, 0x01ffffff, 0x00ffff8f),
662 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQC_CONFIG, 0x03000000, 0x020a2000),
663 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x010b0000),
664 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x4a2c0e68),
665 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0xb5d3f197),
666 SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_CACHE_INVALIDATION, 0x3fff3af3, 0x19200000),
667 SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_GS_MAX_WAVE_ID, 0x00000fff, 0x000003ff),
668 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC1_F32_INT_DIS, 0x00000800, 0x00000800),
669 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC2_F32_INT_DIS, 0x00000800, 0x00000800),
670 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_DEBUG, 0x00008000, 0x00008000)
671 };
672
673 static const struct soc15_reg_golden golden_settings_gc_9_0_vg10[] =
674 {
675 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0x0000f000, 0x00012107),
676 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_3, 0x30000000, 0x10000000),
677 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPC_UTCL1_CNTL, 0x08000000, 0x08000080),
678 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPF_UTCL1_CNTL, 0x08000000, 0x08000080),
679 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPG_UTCL1_CNTL, 0x08000000, 0x08000080),
680 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x2a114042),
681 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xffff77ff, 0x2a114042),
682 SOC15_REG_GOLDEN_VALUE(GC, 0, mmIA_UTCL1_CNTL, 0x08000000, 0x08000080),
683 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0x00008000, 0x00048000),
684 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_0, 0x08000000, 0x08000080),
685 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_1, 0x08000000, 0x08000080),
686 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_2, 0x08000000, 0x08000080),
687 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_PREWALKER_UTCL1_CNTL, 0x08000000, 0x08000080),
688 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_SPM_UTCL1_CNTL, 0x08000000, 0x08000080),
689 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRMI_UTCL1_CNTL2, 0x00030000, 0x00020000),
690 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_CONFIG_CNTL_1, 0x0000000f, 0x01000107),
691 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x00001800, 0x00000800),
692 SOC15_REG_GOLDEN_VALUE(GC, 0, mmWD_UTCL1_CNTL, 0x08000000, 0x08000080)
693 };
694
695 static const struct soc15_reg_golden golden_settings_gc_9_0_vg20[] =
696 {
697 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_DCC_CONFIG, 0x0f000080, 0x04000080),
698 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_2, 0x0f000000, 0x0a000000),
699 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_3, 0x30000000, 0x10000000),
700 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xf3e777ff, 0x22014042),
701 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xf3e777ff, 0x22014042),
702 SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0x00003e00, 0x00000400),
703 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xff840000, 0x04040000),
704 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRMI_UTCL1_CNTL2, 0x00030000, 0x00030000),
705 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_CONFIG_CNTL_1, 0xffff010f, 0x01000107),
706 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0x000b0000, 0x000b0000),
707 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x01000000, 0x01000000)
708 };
709
710 static const struct soc15_reg_golden golden_settings_gc_9_1[] =
711 {
712 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0xfffdf3cf, 0x00014104),
713 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPC_UTCL1_CNTL, 0x08000000, 0x08000080),
714 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPF_UTCL1_CNTL, 0x08000000, 0x08000080),
715 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPG_UTCL1_CNTL, 0x08000000, 0x08000080),
716 SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0xf00fffff, 0x00000420),
717 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_GPU_ID, 0x0000000f, 0x00000000),
718 SOC15_REG_GOLDEN_VALUE(GC, 0, mmIA_UTCL1_CNTL, 0x08000000, 0x08000080),
719 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_BINNER_EVENT_CNTL_3, 0x00000003, 0x82400024),
720 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE, 0x3fffffff, 0x00000001),
721 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000),
722 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_0, 0x08000000, 0x08000080),
723 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_1, 0x08000000, 0x08000080),
724 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_2, 0x08000000, 0x08000080),
725 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_PREWALKER_UTCL1_CNTL, 0x08000000, 0x08000080),
726 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_SPM_UTCL1_CNTL, 0x08000000, 0x08000080),
727 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x010b0000),
728 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000),
729 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00003120),
730 SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_CACHE_INVALIDATION, 0x3fff3af3, 0x19200000),
731 SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_GS_MAX_WAVE_ID, 0x00000fff, 0x000000ff),
732 SOC15_REG_GOLDEN_VALUE(GC, 0, mmWD_UTCL1_CNTL, 0x08000000, 0x08000080),
733 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC1_F32_INT_DIS, 0x00000800, 0x00000800),
734 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC2_F32_INT_DIS, 0x00000800, 0x00000800),
735 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_DEBUG, 0x00008000, 0x00008000)
736 };
737
738 static const struct soc15_reg_golden golden_settings_gc_9_1_rv1[] =
739 {
740 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_3, 0x30000000, 0x10000000),
741 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x24000042),
742 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xffff77ff, 0x24000042),
743 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xffffffff, 0x04048000),
744 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_MODE_CNTL_1, 0x06000000, 0x06000000),
745 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRMI_UTCL1_CNTL2, 0x00030000, 0x00020000),
746 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x01bd9f33, 0x00000800)
747 };
748
749 static const struct soc15_reg_golden golden_settings_gc_9_1_rv2[] =
750 {
751 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_DCC_CONFIG, 0xff7fffff, 0x04000000),
752 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0xfffdf3cf, 0x00014104),
753 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_2, 0xff7fffff, 0x0a000000),
754 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPC_UTCL1_CNTL, 0x7f0fffff, 0x08000080),
755 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPF_UTCL1_CNTL, 0xff8fffff, 0x08000080),
756 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPG_UTCL1_CNTL, 0x7f8fffff, 0x08000080),
757 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x26013041),
758 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xffff77ff, 0x26013041),
759 SOC15_REG_GOLDEN_VALUE(GC, 0, mmIA_UTCL1_CNTL, 0x3f8fffff, 0x08000080),
760 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xffffffff, 0x04040000),
761 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_0, 0xff0fffff, 0x08000080),
762 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_1, 0xff0fffff, 0x08000080),
763 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_2, 0xff0fffff, 0x08000080),
764 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_PREWALKER_UTCL1_CNTL, 0xff0fffff, 0x08000080),
765 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_SPM_UTCL1_CNTL, 0xff0fffff, 0x08000080),
766 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000),
767 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00000010),
768 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x01bd9f33, 0x01000000),
769 SOC15_REG_GOLDEN_VALUE(GC, 0, mmWD_UTCL1_CNTL, 0x3f8fffff, 0x08000080),
770 };
771
772 static const struct soc15_reg_golden golden_settings_gc_9_1_rn[] =
773 {
774 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0xfffdf3cf, 0x00014104),
775 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_2, 0xff7fffff, 0x0a000000),
776 SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0xf00fffff, 0x00000400),
777 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xf3e777ff, 0x24000042),
778 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xf3e777ff, 0x24000042),
779 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE, 0x3fffffff, 0x00000001),
780 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xffffffff, 0x04040000),
781 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000),
782 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x010b0000),
783 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000),
784 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00003120),
785 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGCEA_PROBE_MAP, 0xffffffff, 0x0000cccc),
786 };
787
788 static const struct soc15_reg_golden golden_settings_gc_9_x_common[] =
789 {
790 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_SD_CNTL, 0xffffffff, 0x000001ff),
791 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGRBM_CAM_INDEX, 0xffffffff, 0x00000000),
792 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGRBM_CAM_DATA, 0xffffffff, 0x2544c382)
793 };
794
795 static const struct soc15_reg_golden golden_settings_gc_9_2_1[] =
796 {
797 SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0xf00fffff, 0x00000420),
798 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_GPU_ID, 0x0000000f, 0x00000000),
799 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_BINNER_EVENT_CNTL_3, 0x00000003, 0x82400024),
800 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE, 0x3fffffff, 0x00000001),
801 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000),
802 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSH_MEM_CONFIG, 0x00001000, 0x00001000),
803 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_CU_0, 0x0007ffff, 0x00000800),
804 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_CU_1, 0x0007ffff, 0x00000800),
805 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_EN_CU_0, 0x01ffffff, 0x0000ff87),
806 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_EN_CU_1, 0x01ffffff, 0x0000ff8f),
807 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQC_CONFIG, 0x03000000, 0x020a2000),
808 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x010b0000),
809 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x4a2c0e68),
810 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0xb5d3f197),
811 SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_CACHE_INVALIDATION, 0x3fff3af3, 0x19200000),
812 SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_GS_MAX_WAVE_ID, 0x00000fff, 0x000003ff)
813 };
814
815 static const struct soc15_reg_golden golden_settings_gc_9_2_1_vg12[] =
816 {
817 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_DCC_CONFIG, 0x00000080, 0x04000080),
818 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0xfffdf3cf, 0x00014104),
819 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_2, 0x0f000000, 0x0a000000),
820 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x24104041),
821 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xffff77ff, 0x24104041),
822 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xffffffff, 0x04040000),
823 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_CONFIG_CNTL_1, 0xffff03ff, 0x01000107),
824 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000),
825 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0x76325410),
826 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x01bd9f33, 0x01000000),
827 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC1_F32_INT_DIS, 0x00000800, 0x00000800),
828 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC2_F32_INT_DIS, 0x00000800, 0x00000800),
829 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_DEBUG, 0x00008000, 0x00008000)
830 };
831
832 static const struct soc15_reg_golden golden_settings_gc_9_4_1_arct[] =
833 {
834 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x2a114042),
835 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x10b0000),
836 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_0_ARCT, 0x3fffffff, 0x346f0a4e),
837 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_1_ARCT, 0x3fffffff, 0x1c642ca),
838 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_2_ARCT, 0x3fffffff, 0x26f45098),
839 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_3_ARCT, 0x3fffffff, 0x2ebd9fe3),
840 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_4_ARCT, 0x3fffffff, 0xb90f5b1),
841 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_5_ARCT, 0x3ff, 0x135),
842 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQ_CONFIG, 0xffffffff, 0x011A0000),
843 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQ_FIFO_SIZES, 0xffffffff, 0x00000f00),
844 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_UTCL1_CNTL1, 0x30000000, 0x30000000)
845 };
846
847 static const struct soc15_reg_rlcg rlcg_access_gc_9_0[] = {
848 {SOC15_REG_ENTRY(GC, 0, mmGRBM_GFX_INDEX)},
849 {SOC15_REG_ENTRY(GC, 0, mmSQ_IND_INDEX)},
850 };
851
852 static const u32 GFX_RLC_SRM_INDEX_CNTL_ADDR_OFFSETS[] =
853 {
854 mmRLC_SRM_INDEX_CNTL_ADDR_0 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
855 mmRLC_SRM_INDEX_CNTL_ADDR_1 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
856 mmRLC_SRM_INDEX_CNTL_ADDR_2 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
857 mmRLC_SRM_INDEX_CNTL_ADDR_3 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
858 mmRLC_SRM_INDEX_CNTL_ADDR_4 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
859 mmRLC_SRM_INDEX_CNTL_ADDR_5 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
860 mmRLC_SRM_INDEX_CNTL_ADDR_6 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
861 mmRLC_SRM_INDEX_CNTL_ADDR_7 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
862 };
863
864 static const u32 GFX_RLC_SRM_INDEX_CNTL_DATA_OFFSETS[] =
865 {
866 mmRLC_SRM_INDEX_CNTL_DATA_0 - mmRLC_SRM_INDEX_CNTL_DATA_0,
867 mmRLC_SRM_INDEX_CNTL_DATA_1 - mmRLC_SRM_INDEX_CNTL_DATA_0,
868 mmRLC_SRM_INDEX_CNTL_DATA_2 - mmRLC_SRM_INDEX_CNTL_DATA_0,
869 mmRLC_SRM_INDEX_CNTL_DATA_3 - mmRLC_SRM_INDEX_CNTL_DATA_0,
870 mmRLC_SRM_INDEX_CNTL_DATA_4 - mmRLC_SRM_INDEX_CNTL_DATA_0,
871 mmRLC_SRM_INDEX_CNTL_DATA_5 - mmRLC_SRM_INDEX_CNTL_DATA_0,
872 mmRLC_SRM_INDEX_CNTL_DATA_6 - mmRLC_SRM_INDEX_CNTL_DATA_0,
873 mmRLC_SRM_INDEX_CNTL_DATA_7 - mmRLC_SRM_INDEX_CNTL_DATA_0,
874 };
875
876 #define VEGA10_GB_ADDR_CONFIG_GOLDEN 0x2a114042
877 #define VEGA12_GB_ADDR_CONFIG_GOLDEN 0x24104041
878 #define RAVEN_GB_ADDR_CONFIG_GOLDEN 0x24000042
879 #define RAVEN2_GB_ADDR_CONFIG_GOLDEN 0x26013041
880
881 static void gfx_v9_0_set_ring_funcs(struct amdgpu_device *adev);
882 static void gfx_v9_0_set_irq_funcs(struct amdgpu_device *adev);
883 static void gfx_v9_0_set_gds_init(struct amdgpu_device *adev);
884 static void gfx_v9_0_set_rlc_funcs(struct amdgpu_device *adev);
885 static int gfx_v9_0_get_cu_info(struct amdgpu_device *adev,
886 struct amdgpu_cu_info *cu_info);
887 static uint64_t gfx_v9_0_get_gpu_clock_counter(struct amdgpu_device *adev);
888 static void gfx_v9_0_ring_emit_de_meta(struct amdgpu_ring *ring, bool resume, bool usegds);
889 static u64 gfx_v9_0_ring_get_rptr_compute(struct amdgpu_ring *ring);
890 static void gfx_v9_0_query_ras_error_count(struct amdgpu_device *adev,
891 void *ras_error_status);
892 static int gfx_v9_0_ras_error_inject(struct amdgpu_device *adev,
893 void *inject_if, uint32_t instance_mask);
894 static void gfx_v9_0_reset_ras_error_count(struct amdgpu_device *adev);
895 static void gfx_v9_0_update_spm_vmid_internal(struct amdgpu_device *adev,
896 unsigned int vmid);
897 static void gfx_v9_0_set_safe_mode(struct amdgpu_device *adev, int xcc_id);
898 static void gfx_v9_0_unset_safe_mode(struct amdgpu_device *adev, int xcc_id);
899
gfx_v9_0_kiq_set_resources(struct amdgpu_ring * kiq_ring,uint64_t queue_mask)900 static void gfx_v9_0_kiq_set_resources(struct amdgpu_ring *kiq_ring,
901 uint64_t queue_mask)
902 {
903 struct amdgpu_device *adev = kiq_ring->adev;
904 u64 shader_mc_addr;
905
906 /* Cleaner shader MC address */
907 shader_mc_addr = adev->gfx.cleaner_shader_gpu_addr >> 8;
908
909 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_SET_RESOURCES, 6));
910 amdgpu_ring_write(kiq_ring,
911 PACKET3_SET_RESOURCES_VMID_MASK(0) |
912 /* vmid_mask:0* queue_type:0 (KIQ) */
913 PACKET3_SET_RESOURCES_QUEUE_TYPE(0));
914 amdgpu_ring_write(kiq_ring,
915 lower_32_bits(queue_mask)); /* queue mask lo */
916 amdgpu_ring_write(kiq_ring,
917 upper_32_bits(queue_mask)); /* queue mask hi */
918 amdgpu_ring_write(kiq_ring, lower_32_bits(shader_mc_addr)); /* cleaner shader addr lo */
919 amdgpu_ring_write(kiq_ring, upper_32_bits(shader_mc_addr)); /* cleaner shader addr hi */
920 amdgpu_ring_write(kiq_ring, 0); /* oac mask */
921 amdgpu_ring_write(kiq_ring, 0); /* gds heap base:0, gds heap size:0 */
922 }
923
gfx_v9_0_kiq_map_queues(struct amdgpu_ring * kiq_ring,struct amdgpu_ring * ring)924 static void gfx_v9_0_kiq_map_queues(struct amdgpu_ring *kiq_ring,
925 struct amdgpu_ring *ring)
926 {
927 uint64_t mqd_addr = amdgpu_bo_gpu_offset(ring->mqd_obj);
928 uint64_t wptr_addr = ring->wptr_gpu_addr;
929 uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0;
930
931 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_MAP_QUEUES, 5));
932 /* Q_sel:0, vmid:0, vidmem: 1, engine:0, num_Q:1*/
933 amdgpu_ring_write(kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */
934 PACKET3_MAP_QUEUES_QUEUE_SEL(0) | /* Queue_Sel */
935 PACKET3_MAP_QUEUES_VMID(0) | /* VMID */
936 PACKET3_MAP_QUEUES_QUEUE(ring->queue) |
937 PACKET3_MAP_QUEUES_PIPE(ring->pipe) |
938 PACKET3_MAP_QUEUES_ME((ring->me == 1 ? 0 : 1)) |
939 /*queue_type: normal compute queue */
940 PACKET3_MAP_QUEUES_QUEUE_TYPE(0) |
941 /* alloc format: all_on_one_pipe */
942 PACKET3_MAP_QUEUES_ALLOC_FORMAT(0) |
943 PACKET3_MAP_QUEUES_ENGINE_SEL(eng_sel) |
944 /* num_queues: must be 1 */
945 PACKET3_MAP_QUEUES_NUM_QUEUES(1));
946 amdgpu_ring_write(kiq_ring,
947 PACKET3_MAP_QUEUES_DOORBELL_OFFSET(ring->doorbell_index));
948 amdgpu_ring_write(kiq_ring, lower_32_bits(mqd_addr));
949 amdgpu_ring_write(kiq_ring, upper_32_bits(mqd_addr));
950 amdgpu_ring_write(kiq_ring, lower_32_bits(wptr_addr));
951 amdgpu_ring_write(kiq_ring, upper_32_bits(wptr_addr));
952 }
953
gfx_v9_0_kiq_unmap_queues(struct amdgpu_ring * kiq_ring,struct amdgpu_ring * ring,enum amdgpu_unmap_queues_action action,u64 gpu_addr,u64 seq)954 static void gfx_v9_0_kiq_unmap_queues(struct amdgpu_ring *kiq_ring,
955 struct amdgpu_ring *ring,
956 enum amdgpu_unmap_queues_action action,
957 u64 gpu_addr, u64 seq)
958 {
959 uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0;
960
961 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_UNMAP_QUEUES, 4));
962 amdgpu_ring_write(kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */
963 PACKET3_UNMAP_QUEUES_ACTION(action) |
964 PACKET3_UNMAP_QUEUES_QUEUE_SEL(0) |
965 PACKET3_UNMAP_QUEUES_ENGINE_SEL(eng_sel) |
966 PACKET3_UNMAP_QUEUES_NUM_QUEUES(1));
967 amdgpu_ring_write(kiq_ring,
968 PACKET3_UNMAP_QUEUES_DOORBELL_OFFSET0(ring->doorbell_index));
969
970 if (action == PREEMPT_QUEUES_NO_UNMAP) {
971 amdgpu_ring_write(kiq_ring, lower_32_bits(ring->wptr & ring->buf_mask));
972 amdgpu_ring_write(kiq_ring, 0);
973 amdgpu_ring_write(kiq_ring, 0);
974
975 } else {
976 amdgpu_ring_write(kiq_ring, 0);
977 amdgpu_ring_write(kiq_ring, 0);
978 amdgpu_ring_write(kiq_ring, 0);
979 }
980 }
981
gfx_v9_0_kiq_query_status(struct amdgpu_ring * kiq_ring,struct amdgpu_ring * ring,u64 addr,u64 seq)982 static void gfx_v9_0_kiq_query_status(struct amdgpu_ring *kiq_ring,
983 struct amdgpu_ring *ring,
984 u64 addr,
985 u64 seq)
986 {
987 uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0;
988
989 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_QUERY_STATUS, 5));
990 amdgpu_ring_write(kiq_ring,
991 PACKET3_QUERY_STATUS_CONTEXT_ID(0) |
992 PACKET3_QUERY_STATUS_INTERRUPT_SEL(0) |
993 PACKET3_QUERY_STATUS_COMMAND(2));
994 /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */
995 amdgpu_ring_write(kiq_ring,
996 PACKET3_QUERY_STATUS_DOORBELL_OFFSET(ring->doorbell_index) |
997 PACKET3_QUERY_STATUS_ENG_SEL(eng_sel));
998 amdgpu_ring_write(kiq_ring, lower_32_bits(addr));
999 amdgpu_ring_write(kiq_ring, upper_32_bits(addr));
1000 amdgpu_ring_write(kiq_ring, lower_32_bits(seq));
1001 amdgpu_ring_write(kiq_ring, upper_32_bits(seq));
1002 }
1003
gfx_v9_0_kiq_invalidate_tlbs(struct amdgpu_ring * kiq_ring,uint16_t pasid,uint32_t flush_type,bool all_hub)1004 static void gfx_v9_0_kiq_invalidate_tlbs(struct amdgpu_ring *kiq_ring,
1005 uint16_t pasid, uint32_t flush_type,
1006 bool all_hub)
1007 {
1008 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_INVALIDATE_TLBS, 0));
1009 amdgpu_ring_write(kiq_ring,
1010 PACKET3_INVALIDATE_TLBS_DST_SEL(1) |
1011 PACKET3_INVALIDATE_TLBS_ALL_HUB(all_hub) |
1012 PACKET3_INVALIDATE_TLBS_PASID(pasid) |
1013 PACKET3_INVALIDATE_TLBS_FLUSH_TYPE(flush_type));
1014 }
1015
1016
gfx_v9_0_kiq_reset_hw_queue(struct amdgpu_ring * kiq_ring,uint32_t queue_type,uint32_t me_id,uint32_t pipe_id,uint32_t queue_id,uint32_t xcc_id,uint32_t vmid)1017 static void gfx_v9_0_kiq_reset_hw_queue(struct amdgpu_ring *kiq_ring, uint32_t queue_type,
1018 uint32_t me_id, uint32_t pipe_id, uint32_t queue_id,
1019 uint32_t xcc_id, uint32_t vmid)
1020 {
1021 struct amdgpu_device *adev = kiq_ring->adev;
1022 unsigned i;
1023
1024 /* enter save mode */
1025 amdgpu_gfx_rlc_enter_safe_mode(adev, xcc_id);
1026 mutex_lock(&adev->srbm_mutex);
1027 soc15_grbm_select(adev, me_id, pipe_id, queue_id, 0, 0);
1028
1029 if (queue_type == AMDGPU_RING_TYPE_COMPUTE) {
1030 WREG32_SOC15(GC, 0, mmCP_HQD_DEQUEUE_REQUEST, 0x2);
1031 WREG32_SOC15(GC, 0, mmSPI_COMPUTE_QUEUE_RESET, 0x1);
1032 /* wait till dequeue take effects */
1033 for (i = 0; i < adev->usec_timeout; i++) {
1034 if (!(RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1))
1035 break;
1036 udelay(1);
1037 }
1038 if (i >= adev->usec_timeout)
1039 dev_err(adev->dev, "fail to wait on hqd deactive\n");
1040 } else {
1041 dev_err(adev->dev, "reset queue_type(%d) not supported\n", queue_type);
1042 }
1043
1044 soc15_grbm_select(adev, 0, 0, 0, 0, 0);
1045 mutex_unlock(&adev->srbm_mutex);
1046 /* exit safe mode */
1047 amdgpu_gfx_rlc_exit_safe_mode(adev, xcc_id);
1048 }
1049
1050 static const struct kiq_pm4_funcs gfx_v9_0_kiq_pm4_funcs = {
1051 .kiq_set_resources = gfx_v9_0_kiq_set_resources,
1052 .kiq_map_queues = gfx_v9_0_kiq_map_queues,
1053 .kiq_unmap_queues = gfx_v9_0_kiq_unmap_queues,
1054 .kiq_query_status = gfx_v9_0_kiq_query_status,
1055 .kiq_invalidate_tlbs = gfx_v9_0_kiq_invalidate_tlbs,
1056 .kiq_reset_hw_queue = gfx_v9_0_kiq_reset_hw_queue,
1057 .set_resources_size = 8,
1058 .map_queues_size = 7,
1059 .unmap_queues_size = 6,
1060 .query_status_size = 7,
1061 .invalidate_tlbs_size = 2,
1062 };
1063
gfx_v9_0_set_kiq_pm4_funcs(struct amdgpu_device * adev)1064 static void gfx_v9_0_set_kiq_pm4_funcs(struct amdgpu_device *adev)
1065 {
1066 adev->gfx.kiq[0].pmf = &gfx_v9_0_kiq_pm4_funcs;
1067 }
1068
gfx_v9_0_init_golden_registers(struct amdgpu_device * adev)1069 static void gfx_v9_0_init_golden_registers(struct amdgpu_device *adev)
1070 {
1071 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
1072 case IP_VERSION(9, 0, 1):
1073 soc15_program_register_sequence(adev,
1074 golden_settings_gc_9_0,
1075 ARRAY_SIZE(golden_settings_gc_9_0));
1076 soc15_program_register_sequence(adev,
1077 golden_settings_gc_9_0_vg10,
1078 ARRAY_SIZE(golden_settings_gc_9_0_vg10));
1079 break;
1080 case IP_VERSION(9, 2, 1):
1081 soc15_program_register_sequence(adev,
1082 golden_settings_gc_9_2_1,
1083 ARRAY_SIZE(golden_settings_gc_9_2_1));
1084 soc15_program_register_sequence(adev,
1085 golden_settings_gc_9_2_1_vg12,
1086 ARRAY_SIZE(golden_settings_gc_9_2_1_vg12));
1087 break;
1088 case IP_VERSION(9, 4, 0):
1089 soc15_program_register_sequence(adev,
1090 golden_settings_gc_9_0,
1091 ARRAY_SIZE(golden_settings_gc_9_0));
1092 soc15_program_register_sequence(adev,
1093 golden_settings_gc_9_0_vg20,
1094 ARRAY_SIZE(golden_settings_gc_9_0_vg20));
1095 break;
1096 case IP_VERSION(9, 4, 1):
1097 soc15_program_register_sequence(adev,
1098 golden_settings_gc_9_4_1_arct,
1099 ARRAY_SIZE(golden_settings_gc_9_4_1_arct));
1100 break;
1101 case IP_VERSION(9, 2, 2):
1102 case IP_VERSION(9, 1, 0):
1103 soc15_program_register_sequence(adev, golden_settings_gc_9_1,
1104 ARRAY_SIZE(golden_settings_gc_9_1));
1105 if (adev->apu_flags & AMD_APU_IS_RAVEN2)
1106 soc15_program_register_sequence(adev,
1107 golden_settings_gc_9_1_rv2,
1108 ARRAY_SIZE(golden_settings_gc_9_1_rv2));
1109 else
1110 soc15_program_register_sequence(adev,
1111 golden_settings_gc_9_1_rv1,
1112 ARRAY_SIZE(golden_settings_gc_9_1_rv1));
1113 break;
1114 case IP_VERSION(9, 3, 0):
1115 soc15_program_register_sequence(adev,
1116 golden_settings_gc_9_1_rn,
1117 ARRAY_SIZE(golden_settings_gc_9_1_rn));
1118 return; /* for renoir, don't need common goldensetting */
1119 case IP_VERSION(9, 4, 2):
1120 gfx_v9_4_2_init_golden_registers(adev,
1121 adev->smuio.funcs->get_die_id(adev));
1122 break;
1123 default:
1124 break;
1125 }
1126
1127 if ((amdgpu_ip_version(adev, GC_HWIP, 0) != IP_VERSION(9, 4, 1)) &&
1128 (amdgpu_ip_version(adev, GC_HWIP, 0) != IP_VERSION(9, 4, 2)))
1129 soc15_program_register_sequence(adev, golden_settings_gc_9_x_common,
1130 (const u32)ARRAY_SIZE(golden_settings_gc_9_x_common));
1131 }
1132
gfx_v9_0_write_data_to_reg(struct amdgpu_ring * ring,int eng_sel,bool wc,uint32_t reg,uint32_t val)1133 static void gfx_v9_0_write_data_to_reg(struct amdgpu_ring *ring, int eng_sel,
1134 bool wc, uint32_t reg, uint32_t val)
1135 {
1136 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
1137 amdgpu_ring_write(ring, WRITE_DATA_ENGINE_SEL(eng_sel) |
1138 WRITE_DATA_DST_SEL(0) |
1139 (wc ? WR_CONFIRM : 0));
1140 amdgpu_ring_write(ring, reg);
1141 amdgpu_ring_write(ring, 0);
1142 amdgpu_ring_write(ring, val);
1143 }
1144
gfx_v9_0_wait_reg_mem(struct amdgpu_ring * ring,int eng_sel,int mem_space,int opt,uint32_t addr0,uint32_t addr1,uint32_t ref,uint32_t mask,uint32_t inv)1145 static void gfx_v9_0_wait_reg_mem(struct amdgpu_ring *ring, int eng_sel,
1146 int mem_space, int opt, uint32_t addr0,
1147 uint32_t addr1, uint32_t ref, uint32_t mask,
1148 uint32_t inv)
1149 {
1150 amdgpu_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
1151 amdgpu_ring_write(ring,
1152 /* memory (1) or register (0) */
1153 (WAIT_REG_MEM_MEM_SPACE(mem_space) |
1154 WAIT_REG_MEM_OPERATION(opt) | /* wait */
1155 WAIT_REG_MEM_FUNCTION(3) | /* equal */
1156 WAIT_REG_MEM_ENGINE(eng_sel)));
1157
1158 if (mem_space)
1159 BUG_ON(addr0 & 0x3); /* Dword align */
1160 amdgpu_ring_write(ring, addr0);
1161 amdgpu_ring_write(ring, addr1);
1162 amdgpu_ring_write(ring, ref);
1163 amdgpu_ring_write(ring, mask);
1164 amdgpu_ring_write(ring, inv); /* poll interval */
1165 }
1166
gfx_v9_0_ring_test_ring(struct amdgpu_ring * ring)1167 static int gfx_v9_0_ring_test_ring(struct amdgpu_ring *ring)
1168 {
1169 struct amdgpu_device *adev = ring->adev;
1170 uint32_t scratch = SOC15_REG_OFFSET(GC, 0, mmSCRATCH_REG0);
1171 uint32_t tmp = 0;
1172 unsigned i;
1173 int r;
1174
1175 WREG32(scratch, 0xCAFEDEAD);
1176 r = amdgpu_ring_alloc(ring, 3);
1177 if (r)
1178 return r;
1179
1180 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
1181 amdgpu_ring_write(ring, scratch - PACKET3_SET_UCONFIG_REG_START);
1182 amdgpu_ring_write(ring, 0xDEADBEEF);
1183 amdgpu_ring_commit(ring);
1184
1185 for (i = 0; i < adev->usec_timeout; i++) {
1186 tmp = RREG32(scratch);
1187 if (tmp == 0xDEADBEEF)
1188 break;
1189 udelay(1);
1190 }
1191
1192 if (i >= adev->usec_timeout)
1193 r = -ETIMEDOUT;
1194 return r;
1195 }
1196
gfx_v9_0_ring_test_ib(struct amdgpu_ring * ring,long timeout)1197 static int gfx_v9_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)
1198 {
1199 struct amdgpu_device *adev = ring->adev;
1200 struct amdgpu_ib ib;
1201 struct dma_fence *f = NULL;
1202
1203 unsigned index;
1204 uint64_t gpu_addr;
1205 uint32_t tmp;
1206 long r;
1207
1208 r = amdgpu_device_wb_get(adev, &index);
1209 if (r)
1210 return r;
1211
1212 gpu_addr = adev->wb.gpu_addr + (index * 4);
1213 adev->wb.wb[index] = cpu_to_le32(0xCAFEDEAD);
1214 memset(&ib, 0, sizeof(ib));
1215
1216 r = amdgpu_ib_get(adev, NULL, 20, AMDGPU_IB_POOL_DIRECT, &ib);
1217 if (r)
1218 goto err1;
1219
1220 ib.ptr[0] = PACKET3(PACKET3_WRITE_DATA, 3);
1221 ib.ptr[1] = WRITE_DATA_DST_SEL(5) | WR_CONFIRM;
1222 ib.ptr[2] = lower_32_bits(gpu_addr);
1223 ib.ptr[3] = upper_32_bits(gpu_addr);
1224 ib.ptr[4] = 0xDEADBEEF;
1225 ib.length_dw = 5;
1226
1227 r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f);
1228 if (r)
1229 goto err2;
1230
1231 r = dma_fence_wait_timeout(f, false, timeout);
1232 if (r == 0) {
1233 r = -ETIMEDOUT;
1234 goto err2;
1235 } else if (r < 0) {
1236 goto err2;
1237 }
1238
1239 tmp = adev->wb.wb[index];
1240 if (tmp == 0xDEADBEEF)
1241 r = 0;
1242 else
1243 r = -EINVAL;
1244
1245 err2:
1246 amdgpu_ib_free(&ib, NULL);
1247 dma_fence_put(f);
1248 err1:
1249 amdgpu_device_wb_free(adev, index);
1250 return r;
1251 }
1252
1253
gfx_v9_0_free_microcode(struct amdgpu_device * adev)1254 static void gfx_v9_0_free_microcode(struct amdgpu_device *adev)
1255 {
1256 amdgpu_ucode_release(&adev->gfx.pfp_fw);
1257 amdgpu_ucode_release(&adev->gfx.me_fw);
1258 amdgpu_ucode_release(&adev->gfx.ce_fw);
1259 amdgpu_ucode_release(&adev->gfx.rlc_fw);
1260 amdgpu_ucode_release(&adev->gfx.mec_fw);
1261 amdgpu_ucode_release(&adev->gfx.mec2_fw);
1262
1263 kfree(adev->gfx.rlc.register_list_format);
1264 }
1265
gfx_v9_0_check_fw_write_wait(struct amdgpu_device * adev)1266 static void gfx_v9_0_check_fw_write_wait(struct amdgpu_device *adev)
1267 {
1268 adev->gfx.me_fw_write_wait = false;
1269 adev->gfx.mec_fw_write_wait = false;
1270
1271 if ((amdgpu_ip_version(adev, GC_HWIP, 0) != IP_VERSION(9, 4, 1)) &&
1272 (amdgpu_ip_version(adev, GC_HWIP, 0) != IP_VERSION(9, 4, 2)) &&
1273 ((adev->gfx.mec_fw_version < 0x000001a5) ||
1274 (adev->gfx.mec_feature_version < 46) ||
1275 (adev->gfx.pfp_fw_version < 0x000000b7) ||
1276 (adev->gfx.pfp_feature_version < 46)))
1277 DRM_WARN_ONCE("CP firmware version too old, please update!");
1278
1279 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
1280 case IP_VERSION(9, 0, 1):
1281 if ((adev->gfx.me_fw_version >= 0x0000009c) &&
1282 (adev->gfx.me_feature_version >= 42) &&
1283 (adev->gfx.pfp_fw_version >= 0x000000b1) &&
1284 (adev->gfx.pfp_feature_version >= 42))
1285 adev->gfx.me_fw_write_wait = true;
1286
1287 if ((adev->gfx.mec_fw_version >= 0x00000193) &&
1288 (adev->gfx.mec_feature_version >= 42))
1289 adev->gfx.mec_fw_write_wait = true;
1290 break;
1291 case IP_VERSION(9, 2, 1):
1292 if ((adev->gfx.me_fw_version >= 0x0000009c) &&
1293 (adev->gfx.me_feature_version >= 44) &&
1294 (adev->gfx.pfp_fw_version >= 0x000000b2) &&
1295 (adev->gfx.pfp_feature_version >= 44))
1296 adev->gfx.me_fw_write_wait = true;
1297
1298 if ((adev->gfx.mec_fw_version >= 0x00000196) &&
1299 (adev->gfx.mec_feature_version >= 44))
1300 adev->gfx.mec_fw_write_wait = true;
1301 break;
1302 case IP_VERSION(9, 4, 0):
1303 if ((adev->gfx.me_fw_version >= 0x0000009c) &&
1304 (adev->gfx.me_feature_version >= 44) &&
1305 (adev->gfx.pfp_fw_version >= 0x000000b2) &&
1306 (adev->gfx.pfp_feature_version >= 44))
1307 adev->gfx.me_fw_write_wait = true;
1308
1309 if ((adev->gfx.mec_fw_version >= 0x00000197) &&
1310 (adev->gfx.mec_feature_version >= 44))
1311 adev->gfx.mec_fw_write_wait = true;
1312 break;
1313 case IP_VERSION(9, 1, 0):
1314 case IP_VERSION(9, 2, 2):
1315 if ((adev->gfx.me_fw_version >= 0x0000009c) &&
1316 (adev->gfx.me_feature_version >= 42) &&
1317 (adev->gfx.pfp_fw_version >= 0x000000b1) &&
1318 (adev->gfx.pfp_feature_version >= 42))
1319 adev->gfx.me_fw_write_wait = true;
1320
1321 if ((adev->gfx.mec_fw_version >= 0x00000192) &&
1322 (adev->gfx.mec_feature_version >= 42))
1323 adev->gfx.mec_fw_write_wait = true;
1324 break;
1325 default:
1326 adev->gfx.me_fw_write_wait = true;
1327 adev->gfx.mec_fw_write_wait = true;
1328 break;
1329 }
1330 }
1331
1332 struct amdgpu_gfxoff_quirk {
1333 u16 chip_vendor;
1334 u16 chip_device;
1335 u16 subsys_vendor;
1336 u16 subsys_device;
1337 u8 revision;
1338 };
1339
1340 static const struct amdgpu_gfxoff_quirk amdgpu_gfxoff_quirk_list[] = {
1341 /* https://bugzilla.kernel.org/show_bug.cgi?id=204689 */
1342 { 0x1002, 0x15dd, 0x1002, 0x15dd, 0xc8 },
1343 /* https://bugzilla.kernel.org/show_bug.cgi?id=207171 */
1344 { 0x1002, 0x15dd, 0x103c, 0x83e7, 0xd3 },
1345 /* GFXOFF is unstable on C6 parts with a VBIOS 113-RAVEN-114 */
1346 { 0x1002, 0x15dd, 0x1002, 0x15dd, 0xc6 },
1347 /* Apple MacBook Pro (15-inch, 2019) Radeon Pro Vega 20 4 GB */
1348 { 0x1002, 0x69af, 0x106b, 0x019a, 0xc0 },
1349 /* https://bbs.openkylin.top/t/topic/171497 */
1350 { 0x1002, 0x15d8, 0x19e5, 0x3e14, 0xc2 },
1351 /* HP 705G4 DM with R5 2400G */
1352 { 0x1002, 0x15dd, 0x103c, 0x8464, 0xd6 },
1353 { 0, 0, 0, 0, 0 },
1354 };
1355
gfx_v9_0_should_disable_gfxoff(struct pci_dev * pdev)1356 static bool gfx_v9_0_should_disable_gfxoff(struct pci_dev *pdev)
1357 {
1358 const struct amdgpu_gfxoff_quirk *p = amdgpu_gfxoff_quirk_list;
1359
1360 while (p && p->chip_device != 0) {
1361 if (pdev->vendor == p->chip_vendor &&
1362 pdev->device == p->chip_device &&
1363 pdev->subsystem_vendor == p->subsys_vendor &&
1364 pdev->subsystem_device == p->subsys_device &&
1365 pdev->revision == p->revision) {
1366 return true;
1367 }
1368 ++p;
1369 }
1370 return false;
1371 }
1372
is_raven_kicker(struct amdgpu_device * adev)1373 static bool is_raven_kicker(struct amdgpu_device *adev)
1374 {
1375 if (adev->pm.fw_version >= 0x41e2b)
1376 return true;
1377 else
1378 return false;
1379 }
1380
check_if_enlarge_doorbell_range(struct amdgpu_device * adev)1381 static bool check_if_enlarge_doorbell_range(struct amdgpu_device *adev)
1382 {
1383 if ((amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 3, 0)) &&
1384 (adev->gfx.me_fw_version >= 0x000000a5) &&
1385 (adev->gfx.me_feature_version >= 52))
1386 return true;
1387 else
1388 return false;
1389 }
1390
gfx_v9_0_check_if_need_gfxoff(struct amdgpu_device * adev)1391 static void gfx_v9_0_check_if_need_gfxoff(struct amdgpu_device *adev)
1392 {
1393 if (gfx_v9_0_should_disable_gfxoff(adev->pdev))
1394 adev->pm.pp_feature &= ~PP_GFXOFF_MASK;
1395
1396 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
1397 case IP_VERSION(9, 0, 1):
1398 case IP_VERSION(9, 2, 1):
1399 case IP_VERSION(9, 4, 0):
1400 break;
1401 case IP_VERSION(9, 2, 2):
1402 case IP_VERSION(9, 1, 0):
1403 if (!((adev->apu_flags & AMD_APU_IS_RAVEN2) ||
1404 (adev->apu_flags & AMD_APU_IS_PICASSO)) &&
1405 ((!is_raven_kicker(adev) &&
1406 adev->gfx.rlc_fw_version < 531) ||
1407 (adev->gfx.rlc_feature_version < 1) ||
1408 !adev->gfx.rlc.is_rlc_v2_1))
1409 adev->pm.pp_feature &= ~PP_GFXOFF_MASK;
1410
1411 if (adev->pm.pp_feature & PP_GFXOFF_MASK)
1412 adev->pg_flags |= AMD_PG_SUPPORT_GFX_PG |
1413 AMD_PG_SUPPORT_CP |
1414 AMD_PG_SUPPORT_RLC_SMU_HS;
1415 break;
1416 case IP_VERSION(9, 3, 0):
1417 if (adev->pm.pp_feature & PP_GFXOFF_MASK)
1418 adev->pg_flags |= AMD_PG_SUPPORT_GFX_PG |
1419 AMD_PG_SUPPORT_CP |
1420 AMD_PG_SUPPORT_RLC_SMU_HS;
1421 break;
1422 default:
1423 break;
1424 }
1425 }
1426
gfx_v9_0_init_cp_gfx_microcode(struct amdgpu_device * adev,char * chip_name)1427 static int gfx_v9_0_init_cp_gfx_microcode(struct amdgpu_device *adev,
1428 char *chip_name)
1429 {
1430 int err;
1431
1432 err = amdgpu_ucode_request(adev, &adev->gfx.pfp_fw,
1433 AMDGPU_UCODE_REQUIRED,
1434 "amdgpu/%s_pfp.bin", chip_name);
1435 if (err)
1436 goto out;
1437 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_PFP);
1438
1439 err = amdgpu_ucode_request(adev, &adev->gfx.me_fw,
1440 AMDGPU_UCODE_REQUIRED,
1441 "amdgpu/%s_me.bin", chip_name);
1442 if (err)
1443 goto out;
1444 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_ME);
1445
1446 err = amdgpu_ucode_request(adev, &adev->gfx.ce_fw,
1447 AMDGPU_UCODE_REQUIRED,
1448 "amdgpu/%s_ce.bin", chip_name);
1449 if (err)
1450 goto out;
1451 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_CE);
1452
1453 out:
1454 if (err) {
1455 amdgpu_ucode_release(&adev->gfx.pfp_fw);
1456 amdgpu_ucode_release(&adev->gfx.me_fw);
1457 amdgpu_ucode_release(&adev->gfx.ce_fw);
1458 }
1459 return err;
1460 }
1461
gfx_v9_0_init_rlc_microcode(struct amdgpu_device * adev,char * chip_name)1462 static int gfx_v9_0_init_rlc_microcode(struct amdgpu_device *adev,
1463 char *chip_name)
1464 {
1465 int err;
1466 const struct rlc_firmware_header_v2_0 *rlc_hdr;
1467 uint16_t version_major;
1468 uint16_t version_minor;
1469 uint32_t smu_version;
1470
1471 /*
1472 * For Picasso && AM4 SOCKET board, we use picasso_rlc_am4.bin
1473 * instead of picasso_rlc.bin.
1474 * Judgment method:
1475 * PCO AM4: revision >= 0xC8 && revision <= 0xCF
1476 * or revision >= 0xD8 && revision <= 0xDF
1477 * otherwise is PCO FP5
1478 */
1479 if (!strcmp(chip_name, "picasso") &&
1480 (((adev->pdev->revision >= 0xC8) && (adev->pdev->revision <= 0xCF)) ||
1481 ((adev->pdev->revision >= 0xD8) && (adev->pdev->revision <= 0xDF))))
1482 err = amdgpu_ucode_request(adev, &adev->gfx.rlc_fw,
1483 AMDGPU_UCODE_REQUIRED,
1484 "amdgpu/%s_rlc_am4.bin", chip_name);
1485 else if (!strcmp(chip_name, "raven") && (amdgpu_pm_load_smu_firmware(adev, &smu_version) == 0) &&
1486 (smu_version >= 0x41e2b))
1487 /**
1488 *SMC is loaded by SBIOS on APU and it's able to get the SMU version directly.
1489 */
1490 err = amdgpu_ucode_request(adev, &adev->gfx.rlc_fw,
1491 AMDGPU_UCODE_REQUIRED,
1492 "amdgpu/%s_kicker_rlc.bin", chip_name);
1493 else
1494 err = amdgpu_ucode_request(adev, &adev->gfx.rlc_fw,
1495 AMDGPU_UCODE_REQUIRED,
1496 "amdgpu/%s_rlc.bin", chip_name);
1497 if (err)
1498 goto out;
1499
1500 rlc_hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data;
1501 version_major = le16_to_cpu(rlc_hdr->header.header_version_major);
1502 version_minor = le16_to_cpu(rlc_hdr->header.header_version_minor);
1503 err = amdgpu_gfx_rlc_init_microcode(adev, version_major, version_minor);
1504 out:
1505 if (err)
1506 amdgpu_ucode_release(&adev->gfx.rlc_fw);
1507
1508 return err;
1509 }
1510
gfx_v9_0_load_mec2_fw_bin_support(struct amdgpu_device * adev)1511 static bool gfx_v9_0_load_mec2_fw_bin_support(struct amdgpu_device *adev)
1512 {
1513 if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 2) ||
1514 amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 1) ||
1515 amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 3, 0))
1516 return false;
1517
1518 return true;
1519 }
1520
gfx_v9_0_init_cp_compute_microcode(struct amdgpu_device * adev,char * chip_name)1521 static int gfx_v9_0_init_cp_compute_microcode(struct amdgpu_device *adev,
1522 char *chip_name)
1523 {
1524 int err;
1525
1526 if (amdgpu_sriov_vf(adev) && (adev->asic_type == CHIP_ALDEBARAN))
1527 err = amdgpu_ucode_request(adev, &adev->gfx.mec_fw,
1528 AMDGPU_UCODE_REQUIRED,
1529 "amdgpu/%s_sjt_mec.bin", chip_name);
1530 else
1531 err = amdgpu_ucode_request(adev, &adev->gfx.mec_fw,
1532 AMDGPU_UCODE_REQUIRED,
1533 "amdgpu/%s_mec.bin", chip_name);
1534 if (err)
1535 goto out;
1536
1537 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_MEC1);
1538 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_MEC1_JT);
1539
1540 if (gfx_v9_0_load_mec2_fw_bin_support(adev)) {
1541 if (amdgpu_sriov_vf(adev) && (adev->asic_type == CHIP_ALDEBARAN))
1542 err = amdgpu_ucode_request(adev, &adev->gfx.mec2_fw,
1543 AMDGPU_UCODE_REQUIRED,
1544 "amdgpu/%s_sjt_mec2.bin", chip_name);
1545 else
1546 err = amdgpu_ucode_request(adev, &adev->gfx.mec2_fw,
1547 AMDGPU_UCODE_REQUIRED,
1548 "amdgpu/%s_mec2.bin", chip_name);
1549 if (!err) {
1550 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_MEC2);
1551 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_MEC2_JT);
1552 } else {
1553 err = 0;
1554 amdgpu_ucode_release(&adev->gfx.mec2_fw);
1555 }
1556 } else {
1557 adev->gfx.mec2_fw_version = adev->gfx.mec_fw_version;
1558 adev->gfx.mec2_feature_version = adev->gfx.mec_feature_version;
1559 }
1560
1561 gfx_v9_0_check_if_need_gfxoff(adev);
1562 gfx_v9_0_check_fw_write_wait(adev);
1563
1564 out:
1565 if (err)
1566 amdgpu_ucode_release(&adev->gfx.mec_fw);
1567 return err;
1568 }
1569
gfx_v9_0_init_microcode(struct amdgpu_device * adev)1570 static int gfx_v9_0_init_microcode(struct amdgpu_device *adev)
1571 {
1572 char ucode_prefix[30];
1573 int r;
1574
1575 DRM_DEBUG("\n");
1576 amdgpu_ucode_ip_version_decode(adev, GC_HWIP, ucode_prefix, sizeof(ucode_prefix));
1577
1578 /* No CPG in Arcturus */
1579 if (adev->gfx.num_gfx_rings) {
1580 r = gfx_v9_0_init_cp_gfx_microcode(adev, ucode_prefix);
1581 if (r)
1582 return r;
1583 }
1584
1585 r = gfx_v9_0_init_rlc_microcode(adev, ucode_prefix);
1586 if (r)
1587 return r;
1588
1589 r = gfx_v9_0_init_cp_compute_microcode(adev, ucode_prefix);
1590 if (r)
1591 return r;
1592
1593 return r;
1594 }
1595
gfx_v9_0_get_csb_size(struct amdgpu_device * adev)1596 static u32 gfx_v9_0_get_csb_size(struct amdgpu_device *adev)
1597 {
1598 u32 count = 0;
1599 const struct cs_section_def *sect = NULL;
1600 const struct cs_extent_def *ext = NULL;
1601
1602 /* begin clear state */
1603 count += 2;
1604 /* context control state */
1605 count += 3;
1606
1607 for (sect = gfx9_cs_data; sect->section != NULL; ++sect) {
1608 for (ext = sect->section; ext->extent != NULL; ++ext) {
1609 if (sect->id == SECT_CONTEXT)
1610 count += 2 + ext->reg_count;
1611 else
1612 return 0;
1613 }
1614 }
1615
1616 /* end clear state */
1617 count += 2;
1618 /* clear state */
1619 count += 2;
1620
1621 return count;
1622 }
1623
gfx_v9_0_get_csb_buffer(struct amdgpu_device * adev,volatile u32 * buffer)1624 static void gfx_v9_0_get_csb_buffer(struct amdgpu_device *adev,
1625 volatile u32 *buffer)
1626 {
1627 u32 count = 0, i;
1628 const struct cs_section_def *sect = NULL;
1629 const struct cs_extent_def *ext = NULL;
1630
1631 if (adev->gfx.rlc.cs_data == NULL)
1632 return;
1633 if (buffer == NULL)
1634 return;
1635
1636 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
1637 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
1638
1639 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CONTEXT_CONTROL, 1));
1640 buffer[count++] = cpu_to_le32(0x80000000);
1641 buffer[count++] = cpu_to_le32(0x80000000);
1642
1643 for (sect = adev->gfx.rlc.cs_data; sect->section != NULL; ++sect) {
1644 for (ext = sect->section; ext->extent != NULL; ++ext) {
1645 if (sect->id == SECT_CONTEXT) {
1646 buffer[count++] =
1647 cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count));
1648 buffer[count++] = cpu_to_le32(ext->reg_index -
1649 PACKET3_SET_CONTEXT_REG_START);
1650 for (i = 0; i < ext->reg_count; i++)
1651 buffer[count++] = cpu_to_le32(ext->extent[i]);
1652 } else {
1653 return;
1654 }
1655 }
1656 }
1657
1658 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
1659 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_END_CLEAR_STATE);
1660
1661 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CLEAR_STATE, 0));
1662 buffer[count++] = cpu_to_le32(0);
1663 }
1664
gfx_v9_0_init_always_on_cu_mask(struct amdgpu_device * adev)1665 static void gfx_v9_0_init_always_on_cu_mask(struct amdgpu_device *adev)
1666 {
1667 struct amdgpu_cu_info *cu_info = &adev->gfx.cu_info;
1668 uint32_t pg_always_on_cu_num = 2;
1669 uint32_t always_on_cu_num;
1670 uint32_t i, j, k;
1671 uint32_t mask, cu_bitmap, counter;
1672
1673 if (adev->flags & AMD_IS_APU)
1674 always_on_cu_num = 4;
1675 else if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 2, 1))
1676 always_on_cu_num = 8;
1677 else
1678 always_on_cu_num = 12;
1679
1680 mutex_lock(&adev->grbm_idx_mutex);
1681 for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
1682 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {
1683 mask = 1;
1684 cu_bitmap = 0;
1685 counter = 0;
1686 amdgpu_gfx_select_se_sh(adev, i, j, 0xffffffff, 0);
1687
1688 for (k = 0; k < adev->gfx.config.max_cu_per_sh; k ++) {
1689 if (cu_info->bitmap[0][i][j] & mask) {
1690 if (counter == pg_always_on_cu_num)
1691 WREG32_SOC15(GC, 0, mmRLC_PG_ALWAYS_ON_CU_MASK, cu_bitmap);
1692 if (counter < always_on_cu_num)
1693 cu_bitmap |= mask;
1694 else
1695 break;
1696 counter++;
1697 }
1698 mask <<= 1;
1699 }
1700
1701 WREG32_SOC15(GC, 0, mmRLC_LB_ALWAYS_ACTIVE_CU_MASK, cu_bitmap);
1702 cu_info->ao_cu_bitmap[i][j] = cu_bitmap;
1703 }
1704 }
1705 amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0);
1706 mutex_unlock(&adev->grbm_idx_mutex);
1707 }
1708
gfx_v9_0_init_lbpw(struct amdgpu_device * adev)1709 static void gfx_v9_0_init_lbpw(struct amdgpu_device *adev)
1710 {
1711 uint32_t data;
1712
1713 /* set mmRLC_LB_THR_CONFIG_1/2/3/4 */
1714 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_1, 0x0000007F);
1715 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_2, 0x0333A5A7);
1716 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_3, 0x00000077);
1717 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_4, (0x30 | 0x40 << 8 | 0x02FA << 16));
1718
1719 /* set mmRLC_LB_CNTR_INIT = 0x0000_0000 */
1720 WREG32_SOC15(GC, 0, mmRLC_LB_CNTR_INIT, 0x00000000);
1721
1722 /* set mmRLC_LB_CNTR_MAX = 0x0000_0500 */
1723 WREG32_SOC15(GC, 0, mmRLC_LB_CNTR_MAX, 0x00000500);
1724
1725 mutex_lock(&adev->grbm_idx_mutex);
1726 /* set mmRLC_LB_INIT_CU_MASK thru broadcast mode to enable all SE/SH*/
1727 amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0);
1728 WREG32_SOC15(GC, 0, mmRLC_LB_INIT_CU_MASK, 0xffffffff);
1729
1730 /* set mmRLC_LB_PARAMS = 0x003F_1006 */
1731 data = REG_SET_FIELD(0, RLC_LB_PARAMS, FIFO_SAMPLES, 0x0003);
1732 data |= REG_SET_FIELD(data, RLC_LB_PARAMS, PG_IDLE_SAMPLES, 0x0010);
1733 data |= REG_SET_FIELD(data, RLC_LB_PARAMS, PG_IDLE_SAMPLE_INTERVAL, 0x033F);
1734 WREG32_SOC15(GC, 0, mmRLC_LB_PARAMS, data);
1735
1736 /* set mmRLC_GPM_GENERAL_7[31-16] = 0x00C0 */
1737 data = RREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_7);
1738 data &= 0x0000FFFF;
1739 data |= 0x00C00000;
1740 WREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_7, data);
1741
1742 /*
1743 * RLC_LB_ALWAYS_ACTIVE_CU_MASK = 0xF (4 CUs AON for Raven),
1744 * programmed in gfx_v9_0_init_always_on_cu_mask()
1745 */
1746
1747 /* set RLC_LB_CNTL = 0x8000_0095, 31 bit is reserved,
1748 * but used for RLC_LB_CNTL configuration */
1749 data = RLC_LB_CNTL__LB_CNT_SPIM_ACTIVE_MASK;
1750 data |= REG_SET_FIELD(data, RLC_LB_CNTL, CU_MASK_USED_OFF_HYST, 0x09);
1751 data |= REG_SET_FIELD(data, RLC_LB_CNTL, RESERVED, 0x80000);
1752 WREG32_SOC15(GC, 0, mmRLC_LB_CNTL, data);
1753 mutex_unlock(&adev->grbm_idx_mutex);
1754
1755 gfx_v9_0_init_always_on_cu_mask(adev);
1756 }
1757
gfx_v9_4_init_lbpw(struct amdgpu_device * adev)1758 static void gfx_v9_4_init_lbpw(struct amdgpu_device *adev)
1759 {
1760 uint32_t data;
1761
1762 /* set mmRLC_LB_THR_CONFIG_1/2/3/4 */
1763 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_1, 0x0000007F);
1764 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_2, 0x033388F8);
1765 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_3, 0x00000077);
1766 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_4, (0x10 | 0x27 << 8 | 0x02FA << 16));
1767
1768 /* set mmRLC_LB_CNTR_INIT = 0x0000_0000 */
1769 WREG32_SOC15(GC, 0, mmRLC_LB_CNTR_INIT, 0x00000000);
1770
1771 /* set mmRLC_LB_CNTR_MAX = 0x0000_0500 */
1772 WREG32_SOC15(GC, 0, mmRLC_LB_CNTR_MAX, 0x00000800);
1773
1774 mutex_lock(&adev->grbm_idx_mutex);
1775 /* set mmRLC_LB_INIT_CU_MASK thru broadcast mode to enable all SE/SH*/
1776 amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0);
1777 WREG32_SOC15(GC, 0, mmRLC_LB_INIT_CU_MASK, 0xffffffff);
1778
1779 /* set mmRLC_LB_PARAMS = 0x003F_1006 */
1780 data = REG_SET_FIELD(0, RLC_LB_PARAMS, FIFO_SAMPLES, 0x0003);
1781 data |= REG_SET_FIELD(data, RLC_LB_PARAMS, PG_IDLE_SAMPLES, 0x0010);
1782 data |= REG_SET_FIELD(data, RLC_LB_PARAMS, PG_IDLE_SAMPLE_INTERVAL, 0x033F);
1783 WREG32_SOC15(GC, 0, mmRLC_LB_PARAMS, data);
1784
1785 /* set mmRLC_GPM_GENERAL_7[31-16] = 0x00C0 */
1786 data = RREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_7);
1787 data &= 0x0000FFFF;
1788 data |= 0x00C00000;
1789 WREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_7, data);
1790
1791 /*
1792 * RLC_LB_ALWAYS_ACTIVE_CU_MASK = 0xFFF (12 CUs AON),
1793 * programmed in gfx_v9_0_init_always_on_cu_mask()
1794 */
1795
1796 /* set RLC_LB_CNTL = 0x8000_0095, 31 bit is reserved,
1797 * but used for RLC_LB_CNTL configuration */
1798 data = RLC_LB_CNTL__LB_CNT_SPIM_ACTIVE_MASK;
1799 data |= REG_SET_FIELD(data, RLC_LB_CNTL, CU_MASK_USED_OFF_HYST, 0x09);
1800 data |= REG_SET_FIELD(data, RLC_LB_CNTL, RESERVED, 0x80000);
1801 WREG32_SOC15(GC, 0, mmRLC_LB_CNTL, data);
1802 mutex_unlock(&adev->grbm_idx_mutex);
1803
1804 gfx_v9_0_init_always_on_cu_mask(adev);
1805 }
1806
gfx_v9_0_enable_lbpw(struct amdgpu_device * adev,bool enable)1807 static void gfx_v9_0_enable_lbpw(struct amdgpu_device *adev, bool enable)
1808 {
1809 WREG32_FIELD15(GC, 0, RLC_LB_CNTL, LOAD_BALANCE_ENABLE, enable ? 1 : 0);
1810 }
1811
gfx_v9_0_cp_jump_table_num(struct amdgpu_device * adev)1812 static int gfx_v9_0_cp_jump_table_num(struct amdgpu_device *adev)
1813 {
1814 if (gfx_v9_0_load_mec2_fw_bin_support(adev))
1815 return 5;
1816 else
1817 return 4;
1818 }
1819
gfx_v9_0_init_rlcg_reg_access_ctrl(struct amdgpu_device * adev)1820 static void gfx_v9_0_init_rlcg_reg_access_ctrl(struct amdgpu_device *adev)
1821 {
1822 struct amdgpu_rlcg_reg_access_ctrl *reg_access_ctrl;
1823
1824 reg_access_ctrl = &adev->gfx.rlc.reg_access_ctrl[0];
1825 reg_access_ctrl->scratch_reg0 = SOC15_REG_OFFSET(GC, 0, mmSCRATCH_REG0);
1826 reg_access_ctrl->scratch_reg1 = SOC15_REG_OFFSET(GC, 0, mmSCRATCH_REG1);
1827 reg_access_ctrl->scratch_reg2 = SOC15_REG_OFFSET(GC, 0, mmSCRATCH_REG2);
1828 reg_access_ctrl->scratch_reg3 = SOC15_REG_OFFSET(GC, 0, mmSCRATCH_REG3);
1829 reg_access_ctrl->grbm_cntl = SOC15_REG_OFFSET(GC, 0, mmGRBM_GFX_CNTL);
1830 reg_access_ctrl->grbm_idx = SOC15_REG_OFFSET(GC, 0, mmGRBM_GFX_INDEX);
1831 reg_access_ctrl->spare_int = SOC15_REG_OFFSET(GC, 0, mmRLC_SPARE_INT);
1832 adev->gfx.rlc.rlcg_reg_access_supported = true;
1833 }
1834
gfx_v9_0_rlc_init(struct amdgpu_device * adev)1835 static int gfx_v9_0_rlc_init(struct amdgpu_device *adev)
1836 {
1837 const struct cs_section_def *cs_data;
1838 int r;
1839
1840 adev->gfx.rlc.cs_data = gfx9_cs_data;
1841
1842 cs_data = adev->gfx.rlc.cs_data;
1843
1844 if (cs_data) {
1845 /* init clear state block */
1846 r = amdgpu_gfx_rlc_init_csb(adev);
1847 if (r)
1848 return r;
1849 }
1850
1851 if (adev->flags & AMD_IS_APU) {
1852 /* TODO: double check the cp_table_size for RV */
1853 adev->gfx.rlc.cp_table_size = ALIGN(96 * 5 * 4, 2048) + (64 * 1024); /* JT + GDS */
1854 r = amdgpu_gfx_rlc_init_cpt(adev);
1855 if (r)
1856 return r;
1857 }
1858
1859 return 0;
1860 }
1861
gfx_v9_0_mec_fini(struct amdgpu_device * adev)1862 static void gfx_v9_0_mec_fini(struct amdgpu_device *adev)
1863 {
1864 amdgpu_bo_free_kernel(&adev->gfx.mec.hpd_eop_obj, NULL, NULL);
1865 amdgpu_bo_free_kernel(&adev->gfx.mec.mec_fw_obj, NULL, NULL);
1866 }
1867
gfx_v9_0_mec_init(struct amdgpu_device * adev)1868 static int gfx_v9_0_mec_init(struct amdgpu_device *adev)
1869 {
1870 int r;
1871 u32 *hpd;
1872 const __le32 *fw_data;
1873 unsigned fw_size;
1874 u32 *fw;
1875 size_t mec_hpd_size;
1876
1877 const struct gfx_firmware_header_v1_0 *mec_hdr;
1878
1879 bitmap_zero(adev->gfx.mec_bitmap[0].queue_bitmap, AMDGPU_MAX_COMPUTE_QUEUES);
1880
1881 /* take ownership of the relevant compute queues */
1882 amdgpu_gfx_compute_queue_acquire(adev);
1883 mec_hpd_size = adev->gfx.num_compute_rings * GFX9_MEC_HPD_SIZE;
1884 if (mec_hpd_size) {
1885 r = amdgpu_bo_create_reserved(adev, mec_hpd_size, PAGE_SIZE,
1886 AMDGPU_GEM_DOMAIN_VRAM |
1887 AMDGPU_GEM_DOMAIN_GTT,
1888 &adev->gfx.mec.hpd_eop_obj,
1889 &adev->gfx.mec.hpd_eop_gpu_addr,
1890 (void **)&hpd);
1891 if (r) {
1892 dev_warn(adev->dev, "(%d) create HDP EOP bo failed\n", r);
1893 gfx_v9_0_mec_fini(adev);
1894 return r;
1895 }
1896
1897 memset(hpd, 0, mec_hpd_size);
1898
1899 amdgpu_bo_kunmap(adev->gfx.mec.hpd_eop_obj);
1900 amdgpu_bo_unreserve(adev->gfx.mec.hpd_eop_obj);
1901 }
1902
1903 mec_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data;
1904
1905 fw_data = (const __le32 *)
1906 (adev->gfx.mec_fw->data +
1907 le32_to_cpu(mec_hdr->header.ucode_array_offset_bytes));
1908 fw_size = le32_to_cpu(mec_hdr->header.ucode_size_bytes);
1909
1910 r = amdgpu_bo_create_reserved(adev, mec_hdr->header.ucode_size_bytes,
1911 PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT,
1912 &adev->gfx.mec.mec_fw_obj,
1913 &adev->gfx.mec.mec_fw_gpu_addr,
1914 (void **)&fw);
1915 if (r) {
1916 dev_warn(adev->dev, "(%d) create mec firmware bo failed\n", r);
1917 gfx_v9_0_mec_fini(adev);
1918 return r;
1919 }
1920
1921 memcpy(fw, fw_data, fw_size);
1922
1923 amdgpu_bo_kunmap(adev->gfx.mec.mec_fw_obj);
1924 amdgpu_bo_unreserve(adev->gfx.mec.mec_fw_obj);
1925
1926 return 0;
1927 }
1928
wave_read_ind(struct amdgpu_device * adev,uint32_t simd,uint32_t wave,uint32_t address)1929 static uint32_t wave_read_ind(struct amdgpu_device *adev, uint32_t simd, uint32_t wave, uint32_t address)
1930 {
1931 WREG32_SOC15_RLC(GC, 0, mmSQ_IND_INDEX,
1932 (wave << SQ_IND_INDEX__WAVE_ID__SHIFT) |
1933 (simd << SQ_IND_INDEX__SIMD_ID__SHIFT) |
1934 (address << SQ_IND_INDEX__INDEX__SHIFT) |
1935 (SQ_IND_INDEX__FORCE_READ_MASK));
1936 return RREG32_SOC15(GC, 0, mmSQ_IND_DATA);
1937 }
1938
wave_read_regs(struct amdgpu_device * adev,uint32_t simd,uint32_t wave,uint32_t thread,uint32_t regno,uint32_t num,uint32_t * out)1939 static void wave_read_regs(struct amdgpu_device *adev, uint32_t simd,
1940 uint32_t wave, uint32_t thread,
1941 uint32_t regno, uint32_t num, uint32_t *out)
1942 {
1943 WREG32_SOC15_RLC(GC, 0, mmSQ_IND_INDEX,
1944 (wave << SQ_IND_INDEX__WAVE_ID__SHIFT) |
1945 (simd << SQ_IND_INDEX__SIMD_ID__SHIFT) |
1946 (regno << SQ_IND_INDEX__INDEX__SHIFT) |
1947 (thread << SQ_IND_INDEX__THREAD_ID__SHIFT) |
1948 (SQ_IND_INDEX__FORCE_READ_MASK) |
1949 (SQ_IND_INDEX__AUTO_INCR_MASK));
1950 while (num--)
1951 *(out++) = RREG32_SOC15(GC, 0, mmSQ_IND_DATA);
1952 }
1953
gfx_v9_0_read_wave_data(struct amdgpu_device * adev,uint32_t xcc_id,uint32_t simd,uint32_t wave,uint32_t * dst,int * no_fields)1954 static void gfx_v9_0_read_wave_data(struct amdgpu_device *adev, uint32_t xcc_id, uint32_t simd, uint32_t wave, uint32_t *dst, int *no_fields)
1955 {
1956 /* type 1 wave data */
1957 dst[(*no_fields)++] = 1;
1958 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_STATUS);
1959 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_PC_LO);
1960 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_PC_HI);
1961 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_EXEC_LO);
1962 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_EXEC_HI);
1963 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_HW_ID);
1964 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_INST_DW0);
1965 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_INST_DW1);
1966 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_GPR_ALLOC);
1967 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_LDS_ALLOC);
1968 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_TRAPSTS);
1969 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_IB_STS);
1970 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_IB_DBG0);
1971 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_M0);
1972 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_MODE);
1973 }
1974
gfx_v9_0_read_wave_sgprs(struct amdgpu_device * adev,uint32_t xcc_id,uint32_t simd,uint32_t wave,uint32_t start,uint32_t size,uint32_t * dst)1975 static void gfx_v9_0_read_wave_sgprs(struct amdgpu_device *adev, uint32_t xcc_id, uint32_t simd,
1976 uint32_t wave, uint32_t start,
1977 uint32_t size, uint32_t *dst)
1978 {
1979 wave_read_regs(
1980 adev, simd, wave, 0,
1981 start + SQIND_WAVE_SGPRS_OFFSET, size, dst);
1982 }
1983
gfx_v9_0_read_wave_vgprs(struct amdgpu_device * adev,uint32_t xcc_id,uint32_t simd,uint32_t wave,uint32_t thread,uint32_t start,uint32_t size,uint32_t * dst)1984 static void gfx_v9_0_read_wave_vgprs(struct amdgpu_device *adev, uint32_t xcc_id, uint32_t simd,
1985 uint32_t wave, uint32_t thread,
1986 uint32_t start, uint32_t size,
1987 uint32_t *dst)
1988 {
1989 wave_read_regs(
1990 adev, simd, wave, thread,
1991 start + SQIND_WAVE_VGPRS_OFFSET, size, dst);
1992 }
1993
gfx_v9_0_select_me_pipe_q(struct amdgpu_device * adev,u32 me,u32 pipe,u32 q,u32 vm,u32 xcc_id)1994 static void gfx_v9_0_select_me_pipe_q(struct amdgpu_device *adev,
1995 u32 me, u32 pipe, u32 q, u32 vm, u32 xcc_id)
1996 {
1997 soc15_grbm_select(adev, me, pipe, q, vm, 0);
1998 }
1999
2000 static const struct amdgpu_gfx_funcs gfx_v9_0_gfx_funcs = {
2001 .get_gpu_clock_counter = &gfx_v9_0_get_gpu_clock_counter,
2002 .select_se_sh = &gfx_v9_0_select_se_sh,
2003 .read_wave_data = &gfx_v9_0_read_wave_data,
2004 .read_wave_sgprs = &gfx_v9_0_read_wave_sgprs,
2005 .read_wave_vgprs = &gfx_v9_0_read_wave_vgprs,
2006 .select_me_pipe_q = &gfx_v9_0_select_me_pipe_q,
2007 };
2008
2009 const struct amdgpu_ras_block_hw_ops gfx_v9_0_ras_ops = {
2010 .ras_error_inject = &gfx_v9_0_ras_error_inject,
2011 .query_ras_error_count = &gfx_v9_0_query_ras_error_count,
2012 .reset_ras_error_count = &gfx_v9_0_reset_ras_error_count,
2013 };
2014
2015 static struct amdgpu_gfx_ras gfx_v9_0_ras = {
2016 .ras_block = {
2017 .hw_ops = &gfx_v9_0_ras_ops,
2018 },
2019 };
2020
gfx_v9_0_gpu_early_init(struct amdgpu_device * adev)2021 static int gfx_v9_0_gpu_early_init(struct amdgpu_device *adev)
2022 {
2023 u32 gb_addr_config;
2024 int err;
2025
2026 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
2027 case IP_VERSION(9, 0, 1):
2028 adev->gfx.config.max_hw_contexts = 8;
2029 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
2030 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
2031 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
2032 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
2033 gb_addr_config = VEGA10_GB_ADDR_CONFIG_GOLDEN;
2034 break;
2035 case IP_VERSION(9, 2, 1):
2036 adev->gfx.config.max_hw_contexts = 8;
2037 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
2038 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
2039 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
2040 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
2041 gb_addr_config = VEGA12_GB_ADDR_CONFIG_GOLDEN;
2042 DRM_INFO("fix gfx.config for vega12\n");
2043 break;
2044 case IP_VERSION(9, 4, 0):
2045 adev->gfx.ras = &gfx_v9_0_ras;
2046 adev->gfx.config.max_hw_contexts = 8;
2047 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
2048 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
2049 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
2050 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
2051 gb_addr_config = RREG32_SOC15(GC, 0, mmGB_ADDR_CONFIG);
2052 gb_addr_config &= ~0xf3e777ff;
2053 gb_addr_config |= 0x22014042;
2054 /* check vbios table if gpu info is not available */
2055 err = amdgpu_atomfirmware_get_gfx_info(adev);
2056 if (err)
2057 return err;
2058 break;
2059 case IP_VERSION(9, 2, 2):
2060 case IP_VERSION(9, 1, 0):
2061 adev->gfx.config.max_hw_contexts = 8;
2062 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
2063 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
2064 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
2065 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
2066 if (adev->apu_flags & AMD_APU_IS_RAVEN2)
2067 gb_addr_config = RAVEN2_GB_ADDR_CONFIG_GOLDEN;
2068 else
2069 gb_addr_config = RAVEN_GB_ADDR_CONFIG_GOLDEN;
2070 break;
2071 case IP_VERSION(9, 4, 1):
2072 adev->gfx.ras = &gfx_v9_4_ras;
2073 adev->gfx.config.max_hw_contexts = 8;
2074 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
2075 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
2076 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
2077 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
2078 gb_addr_config = RREG32_SOC15(GC, 0, mmGB_ADDR_CONFIG);
2079 gb_addr_config &= ~0xf3e777ff;
2080 gb_addr_config |= 0x22014042;
2081 break;
2082 case IP_VERSION(9, 3, 0):
2083 adev->gfx.config.max_hw_contexts = 8;
2084 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
2085 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
2086 adev->gfx.config.sc_hiz_tile_fifo_size = 0x80;
2087 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
2088 gb_addr_config = RREG32_SOC15(GC, 0, mmGB_ADDR_CONFIG);
2089 gb_addr_config &= ~0xf3e777ff;
2090 gb_addr_config |= 0x22010042;
2091 break;
2092 case IP_VERSION(9, 4, 2):
2093 adev->gfx.ras = &gfx_v9_4_2_ras;
2094 adev->gfx.config.max_hw_contexts = 8;
2095 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
2096 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
2097 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
2098 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
2099 gb_addr_config = RREG32_SOC15(GC, 0, mmGB_ADDR_CONFIG);
2100 gb_addr_config &= ~0xf3e777ff;
2101 gb_addr_config |= 0x22014042;
2102 /* check vbios table if gpu info is not available */
2103 err = amdgpu_atomfirmware_get_gfx_info(adev);
2104 if (err)
2105 return err;
2106 break;
2107 default:
2108 BUG();
2109 break;
2110 }
2111
2112 adev->gfx.config.gb_addr_config = gb_addr_config;
2113
2114 adev->gfx.config.gb_addr_config_fields.num_pipes = 1 <<
2115 REG_GET_FIELD(
2116 adev->gfx.config.gb_addr_config,
2117 GB_ADDR_CONFIG,
2118 NUM_PIPES);
2119
2120 adev->gfx.config.max_tile_pipes =
2121 adev->gfx.config.gb_addr_config_fields.num_pipes;
2122
2123 adev->gfx.config.gb_addr_config_fields.num_banks = 1 <<
2124 REG_GET_FIELD(
2125 adev->gfx.config.gb_addr_config,
2126 GB_ADDR_CONFIG,
2127 NUM_BANKS);
2128 adev->gfx.config.gb_addr_config_fields.max_compress_frags = 1 <<
2129 REG_GET_FIELD(
2130 adev->gfx.config.gb_addr_config,
2131 GB_ADDR_CONFIG,
2132 MAX_COMPRESSED_FRAGS);
2133 adev->gfx.config.gb_addr_config_fields.num_rb_per_se = 1 <<
2134 REG_GET_FIELD(
2135 adev->gfx.config.gb_addr_config,
2136 GB_ADDR_CONFIG,
2137 NUM_RB_PER_SE);
2138 adev->gfx.config.gb_addr_config_fields.num_se = 1 <<
2139 REG_GET_FIELD(
2140 adev->gfx.config.gb_addr_config,
2141 GB_ADDR_CONFIG,
2142 NUM_SHADER_ENGINES);
2143 adev->gfx.config.gb_addr_config_fields.pipe_interleave_size = 1 << (8 +
2144 REG_GET_FIELD(
2145 adev->gfx.config.gb_addr_config,
2146 GB_ADDR_CONFIG,
2147 PIPE_INTERLEAVE_SIZE));
2148
2149 return 0;
2150 }
2151
gfx_v9_0_compute_ring_init(struct amdgpu_device * adev,int ring_id,int mec,int pipe,int queue)2152 static int gfx_v9_0_compute_ring_init(struct amdgpu_device *adev, int ring_id,
2153 int mec, int pipe, int queue)
2154 {
2155 unsigned irq_type;
2156 struct amdgpu_ring *ring = &adev->gfx.compute_ring[ring_id];
2157 unsigned int hw_prio;
2158
2159 ring = &adev->gfx.compute_ring[ring_id];
2160
2161 /* mec0 is me1 */
2162 ring->me = mec + 1;
2163 ring->pipe = pipe;
2164 ring->queue = queue;
2165
2166 ring->ring_obj = NULL;
2167 ring->use_doorbell = true;
2168 ring->doorbell_index = (adev->doorbell_index.mec_ring0 + ring_id) << 1;
2169 ring->eop_gpu_addr = adev->gfx.mec.hpd_eop_gpu_addr
2170 + (ring_id * GFX9_MEC_HPD_SIZE);
2171 ring->vm_hub = AMDGPU_GFXHUB(0);
2172 sprintf(ring->name, "comp_%d.%d.%d", ring->me, ring->pipe, ring->queue);
2173
2174 irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP
2175 + ((ring->me - 1) * adev->gfx.mec.num_pipe_per_mec)
2176 + ring->pipe;
2177 hw_prio = amdgpu_gfx_is_high_priority_compute_queue(adev, ring) ?
2178 AMDGPU_RING_PRIO_2 : AMDGPU_RING_PRIO_DEFAULT;
2179 /* type-2 packets are deprecated on MEC, use type-3 instead */
2180 return amdgpu_ring_init(adev, ring, 1024, &adev->gfx.eop_irq, irq_type,
2181 hw_prio, NULL);
2182 }
2183
gfx_v9_0_alloc_ip_dump(struct amdgpu_device * adev)2184 static void gfx_v9_0_alloc_ip_dump(struct amdgpu_device *adev)
2185 {
2186 uint32_t reg_count = ARRAY_SIZE(gc_reg_list_9);
2187 uint32_t *ptr;
2188 uint32_t inst;
2189
2190 ptr = kcalloc(reg_count, sizeof(uint32_t), GFP_KERNEL);
2191 if (!ptr) {
2192 DRM_ERROR("Failed to allocate memory for GFX IP Dump\n");
2193 adev->gfx.ip_dump_core = NULL;
2194 } else {
2195 adev->gfx.ip_dump_core = ptr;
2196 }
2197
2198 /* Allocate memory for compute queue registers for all the instances */
2199 reg_count = ARRAY_SIZE(gc_cp_reg_list_9);
2200 inst = adev->gfx.mec.num_mec * adev->gfx.mec.num_pipe_per_mec *
2201 adev->gfx.mec.num_queue_per_pipe;
2202
2203 ptr = kcalloc(reg_count * inst, sizeof(uint32_t), GFP_KERNEL);
2204 if (!ptr) {
2205 DRM_ERROR("Failed to allocate memory for Compute Queues IP Dump\n");
2206 adev->gfx.ip_dump_compute_queues = NULL;
2207 } else {
2208 adev->gfx.ip_dump_compute_queues = ptr;
2209 }
2210 }
2211
gfx_v9_0_sw_init(struct amdgpu_ip_block * ip_block)2212 static int gfx_v9_0_sw_init(struct amdgpu_ip_block *ip_block)
2213 {
2214 int i, j, k, r, ring_id;
2215 int xcc_id = 0;
2216 struct amdgpu_ring *ring;
2217 struct amdgpu_device *adev = ip_block->adev;
2218 unsigned int hw_prio;
2219
2220 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
2221 case IP_VERSION(9, 0, 1):
2222 case IP_VERSION(9, 2, 1):
2223 case IP_VERSION(9, 4, 0):
2224 case IP_VERSION(9, 2, 2):
2225 case IP_VERSION(9, 1, 0):
2226 case IP_VERSION(9, 4, 1):
2227 case IP_VERSION(9, 3, 0):
2228 case IP_VERSION(9, 4, 2):
2229 adev->gfx.mec.num_mec = 2;
2230 break;
2231 default:
2232 adev->gfx.mec.num_mec = 1;
2233 break;
2234 }
2235
2236 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
2237 case IP_VERSION(9, 4, 2):
2238 adev->gfx.cleaner_shader_ptr = gfx_9_4_2_cleaner_shader_hex;
2239 adev->gfx.cleaner_shader_size = sizeof(gfx_9_4_2_cleaner_shader_hex);
2240 if (adev->gfx.mec_fw_version >= 88) {
2241 adev->gfx.enable_cleaner_shader = true;
2242 r = amdgpu_gfx_cleaner_shader_sw_init(adev, adev->gfx.cleaner_shader_size);
2243 if (r) {
2244 adev->gfx.enable_cleaner_shader = false;
2245 dev_err(adev->dev, "Failed to initialize cleaner shader\n");
2246 }
2247 }
2248 break;
2249 default:
2250 adev->gfx.enable_cleaner_shader = false;
2251 break;
2252 }
2253
2254 adev->gfx.mec.num_pipe_per_mec = 4;
2255 adev->gfx.mec.num_queue_per_pipe = 8;
2256
2257 /* EOP Event */
2258 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_EOP_INTERRUPT, &adev->gfx.eop_irq);
2259 if (r)
2260 return r;
2261
2262 /* Bad opcode Event */
2263 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP,
2264 GFX_9_0__SRCID__CP_BAD_OPCODE_ERROR,
2265 &adev->gfx.bad_op_irq);
2266 if (r)
2267 return r;
2268
2269 /* Privileged reg */
2270 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_PRIV_REG_FAULT,
2271 &adev->gfx.priv_reg_irq);
2272 if (r)
2273 return r;
2274
2275 /* Privileged inst */
2276 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_PRIV_INSTR_FAULT,
2277 &adev->gfx.priv_inst_irq);
2278 if (r)
2279 return r;
2280
2281 /* ECC error */
2282 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_ECC_ERROR,
2283 &adev->gfx.cp_ecc_error_irq);
2284 if (r)
2285 return r;
2286
2287 /* FUE error */
2288 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_FUE_ERROR,
2289 &adev->gfx.cp_ecc_error_irq);
2290 if (r)
2291 return r;
2292
2293 adev->gfx.gfx_current_status = AMDGPU_GFX_NORMAL_MODE;
2294
2295 if (adev->gfx.rlc.funcs) {
2296 if (adev->gfx.rlc.funcs->init) {
2297 r = adev->gfx.rlc.funcs->init(adev);
2298 if (r) {
2299 dev_err(adev->dev, "Failed to init rlc BOs!\n");
2300 return r;
2301 }
2302 }
2303 }
2304
2305 r = gfx_v9_0_mec_init(adev);
2306 if (r) {
2307 DRM_ERROR("Failed to init MEC BOs!\n");
2308 return r;
2309 }
2310
2311 /* set up the gfx ring */
2312 for (i = 0; i < adev->gfx.num_gfx_rings; i++) {
2313 ring = &adev->gfx.gfx_ring[i];
2314 ring->ring_obj = NULL;
2315 if (!i)
2316 sprintf(ring->name, "gfx");
2317 else
2318 sprintf(ring->name, "gfx_%d", i);
2319 ring->use_doorbell = true;
2320 ring->doorbell_index = adev->doorbell_index.gfx_ring0 << 1;
2321
2322 /* disable scheduler on the real ring */
2323 ring->no_scheduler = adev->gfx.mcbp;
2324 ring->vm_hub = AMDGPU_GFXHUB(0);
2325 r = amdgpu_ring_init(adev, ring, 1024, &adev->gfx.eop_irq,
2326 AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP,
2327 AMDGPU_RING_PRIO_DEFAULT, NULL);
2328 if (r)
2329 return r;
2330 }
2331
2332 /* set up the software rings */
2333 if (adev->gfx.mcbp && adev->gfx.num_gfx_rings) {
2334 for (i = 0; i < GFX9_NUM_SW_GFX_RINGS; i++) {
2335 ring = &adev->gfx.sw_gfx_ring[i];
2336 ring->ring_obj = NULL;
2337 sprintf(ring->name, amdgpu_sw_ring_name(i));
2338 ring->use_doorbell = true;
2339 ring->doorbell_index = adev->doorbell_index.gfx_ring0 << 1;
2340 ring->is_sw_ring = true;
2341 hw_prio = amdgpu_sw_ring_priority(i);
2342 ring->vm_hub = AMDGPU_GFXHUB(0);
2343 r = amdgpu_ring_init(adev, ring, 1024, &adev->gfx.eop_irq,
2344 AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP, hw_prio,
2345 NULL);
2346 if (r)
2347 return r;
2348 ring->wptr = 0;
2349 }
2350
2351 /* init the muxer and add software rings */
2352 r = amdgpu_ring_mux_init(&adev->gfx.muxer, &adev->gfx.gfx_ring[0],
2353 GFX9_NUM_SW_GFX_RINGS);
2354 if (r) {
2355 DRM_ERROR("amdgpu_ring_mux_init failed(%d)\n", r);
2356 return r;
2357 }
2358 for (i = 0; i < GFX9_NUM_SW_GFX_RINGS; i++) {
2359 r = amdgpu_ring_mux_add_sw_ring(&adev->gfx.muxer,
2360 &adev->gfx.sw_gfx_ring[i]);
2361 if (r) {
2362 DRM_ERROR("amdgpu_ring_mux_add_sw_ring failed(%d)\n", r);
2363 return r;
2364 }
2365 }
2366 }
2367
2368 /* set up the compute queues - allocate horizontally across pipes */
2369 ring_id = 0;
2370 for (i = 0; i < adev->gfx.mec.num_mec; ++i) {
2371 for (j = 0; j < adev->gfx.mec.num_queue_per_pipe; j++) {
2372 for (k = 0; k < adev->gfx.mec.num_pipe_per_mec; k++) {
2373 if (!amdgpu_gfx_is_mec_queue_enabled(adev, 0, i,
2374 k, j))
2375 continue;
2376
2377 r = gfx_v9_0_compute_ring_init(adev,
2378 ring_id,
2379 i, k, j);
2380 if (r)
2381 return r;
2382
2383 ring_id++;
2384 }
2385 }
2386 }
2387
2388 /* TODO: Add queue reset mask when FW fully supports it */
2389 adev->gfx.gfx_supported_reset =
2390 amdgpu_get_soft_full_reset_mask(&adev->gfx.gfx_ring[0]);
2391 adev->gfx.compute_supported_reset =
2392 amdgpu_get_soft_full_reset_mask(&adev->gfx.compute_ring[0]);
2393
2394 r = amdgpu_gfx_kiq_init(adev, GFX9_MEC_HPD_SIZE, 0);
2395 if (r) {
2396 DRM_ERROR("Failed to init KIQ BOs!\n");
2397 return r;
2398 }
2399
2400 r = amdgpu_gfx_kiq_init_ring(adev, xcc_id);
2401 if (r)
2402 return r;
2403
2404 /* create MQD for all compute queues as wel as KIQ for SRIOV case */
2405 r = amdgpu_gfx_mqd_sw_init(adev, sizeof(struct v9_mqd_allocation), 0);
2406 if (r)
2407 return r;
2408
2409 adev->gfx.ce_ram_size = 0x8000;
2410
2411 r = gfx_v9_0_gpu_early_init(adev);
2412 if (r)
2413 return r;
2414
2415 if (amdgpu_gfx_ras_sw_init(adev)) {
2416 dev_err(adev->dev, "Failed to initialize gfx ras block!\n");
2417 return -EINVAL;
2418 }
2419
2420 gfx_v9_0_alloc_ip_dump(adev);
2421
2422 r = amdgpu_gfx_sysfs_init(adev);
2423 if (r)
2424 return r;
2425
2426 return 0;
2427 }
2428
2429
gfx_v9_0_sw_fini(struct amdgpu_ip_block * ip_block)2430 static int gfx_v9_0_sw_fini(struct amdgpu_ip_block *ip_block)
2431 {
2432 int i;
2433 struct amdgpu_device *adev = ip_block->adev;
2434
2435 if (adev->gfx.mcbp && adev->gfx.num_gfx_rings) {
2436 for (i = 0; i < GFX9_NUM_SW_GFX_RINGS; i++)
2437 amdgpu_ring_fini(&adev->gfx.sw_gfx_ring[i]);
2438 amdgpu_ring_mux_fini(&adev->gfx.muxer);
2439 }
2440
2441 for (i = 0; i < adev->gfx.num_gfx_rings; i++)
2442 amdgpu_ring_fini(&adev->gfx.gfx_ring[i]);
2443 for (i = 0; i < adev->gfx.num_compute_rings; i++)
2444 amdgpu_ring_fini(&adev->gfx.compute_ring[i]);
2445
2446 amdgpu_gfx_mqd_sw_fini(adev, 0);
2447 amdgpu_gfx_kiq_free_ring(&adev->gfx.kiq[0].ring);
2448 amdgpu_gfx_kiq_fini(adev, 0);
2449
2450 amdgpu_gfx_cleaner_shader_sw_fini(adev);
2451
2452 gfx_v9_0_mec_fini(adev);
2453 amdgpu_bo_free_kernel(&adev->gfx.rlc.clear_state_obj,
2454 &adev->gfx.rlc.clear_state_gpu_addr,
2455 (void **)&adev->gfx.rlc.cs_ptr);
2456 if (adev->flags & AMD_IS_APU) {
2457 amdgpu_bo_free_kernel(&adev->gfx.rlc.cp_table_obj,
2458 &adev->gfx.rlc.cp_table_gpu_addr,
2459 (void **)&adev->gfx.rlc.cp_table_ptr);
2460 }
2461 gfx_v9_0_free_microcode(adev);
2462
2463 amdgpu_gfx_sysfs_fini(adev);
2464
2465 kfree(adev->gfx.ip_dump_core);
2466 kfree(adev->gfx.ip_dump_compute_queues);
2467
2468 return 0;
2469 }
2470
2471
gfx_v9_0_tiling_mode_table_init(struct amdgpu_device * adev)2472 static void gfx_v9_0_tiling_mode_table_init(struct amdgpu_device *adev)
2473 {
2474 /* TODO */
2475 }
2476
gfx_v9_0_select_se_sh(struct amdgpu_device * adev,u32 se_num,u32 sh_num,u32 instance,int xcc_id)2477 void gfx_v9_0_select_se_sh(struct amdgpu_device *adev, u32 se_num, u32 sh_num,
2478 u32 instance, int xcc_id)
2479 {
2480 u32 data;
2481
2482 if (instance == 0xffffffff)
2483 data = REG_SET_FIELD(0, GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES, 1);
2484 else
2485 data = REG_SET_FIELD(0, GRBM_GFX_INDEX, INSTANCE_INDEX, instance);
2486
2487 if (se_num == 0xffffffff)
2488 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_BROADCAST_WRITES, 1);
2489 else
2490 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_INDEX, se_num);
2491
2492 if (sh_num == 0xffffffff)
2493 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SH_BROADCAST_WRITES, 1);
2494 else
2495 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SH_INDEX, sh_num);
2496
2497 WREG32_SOC15_RLC_SHADOW(GC, 0, mmGRBM_GFX_INDEX, data);
2498 }
2499
gfx_v9_0_get_rb_active_bitmap(struct amdgpu_device * adev)2500 static u32 gfx_v9_0_get_rb_active_bitmap(struct amdgpu_device *adev)
2501 {
2502 u32 data, mask;
2503
2504 data = RREG32_SOC15(GC, 0, mmCC_RB_BACKEND_DISABLE);
2505 data |= RREG32_SOC15(GC, 0, mmGC_USER_RB_BACKEND_DISABLE);
2506
2507 data &= CC_RB_BACKEND_DISABLE__BACKEND_DISABLE_MASK;
2508 data >>= GC_USER_RB_BACKEND_DISABLE__BACKEND_DISABLE__SHIFT;
2509
2510 mask = amdgpu_gfx_create_bitmask(adev->gfx.config.max_backends_per_se /
2511 adev->gfx.config.max_sh_per_se);
2512
2513 return (~data) & mask;
2514 }
2515
gfx_v9_0_setup_rb(struct amdgpu_device * adev)2516 static void gfx_v9_0_setup_rb(struct amdgpu_device *adev)
2517 {
2518 int i, j;
2519 u32 data;
2520 u32 active_rbs = 0;
2521 u32 rb_bitmap_width_per_sh = adev->gfx.config.max_backends_per_se /
2522 adev->gfx.config.max_sh_per_se;
2523
2524 mutex_lock(&adev->grbm_idx_mutex);
2525 for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
2526 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {
2527 amdgpu_gfx_select_se_sh(adev, i, j, 0xffffffff, 0);
2528 data = gfx_v9_0_get_rb_active_bitmap(adev);
2529 active_rbs |= data << ((i * adev->gfx.config.max_sh_per_se + j) *
2530 rb_bitmap_width_per_sh);
2531 }
2532 }
2533 amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0);
2534 mutex_unlock(&adev->grbm_idx_mutex);
2535
2536 adev->gfx.config.backend_enable_mask = active_rbs;
2537 adev->gfx.config.num_rbs = hweight32(active_rbs);
2538 }
2539
gfx_v9_0_debug_trap_config_init(struct amdgpu_device * adev,uint32_t first_vmid,uint32_t last_vmid)2540 static void gfx_v9_0_debug_trap_config_init(struct amdgpu_device *adev,
2541 uint32_t first_vmid,
2542 uint32_t last_vmid)
2543 {
2544 uint32_t data;
2545 uint32_t trap_config_vmid_mask = 0;
2546 int i;
2547
2548 /* Calculate trap config vmid mask */
2549 for (i = first_vmid; i < last_vmid; i++)
2550 trap_config_vmid_mask |= (1 << i);
2551
2552 data = REG_SET_FIELD(0, SPI_GDBG_TRAP_CONFIG,
2553 VMID_SEL, trap_config_vmid_mask);
2554 data = REG_SET_FIELD(data, SPI_GDBG_TRAP_CONFIG,
2555 TRAP_EN, 1);
2556 WREG32(SOC15_REG_OFFSET(GC, 0, mmSPI_GDBG_TRAP_CONFIG), data);
2557 WREG32(SOC15_REG_OFFSET(GC, 0, mmSPI_GDBG_TRAP_MASK), 0);
2558
2559 WREG32(SOC15_REG_OFFSET(GC, 0, mmSPI_GDBG_TRAP_DATA0), 0);
2560 WREG32(SOC15_REG_OFFSET(GC, 0, mmSPI_GDBG_TRAP_DATA1), 0);
2561 }
2562
2563 #define DEFAULT_SH_MEM_BASES (0x6000)
gfx_v9_0_init_compute_vmid(struct amdgpu_device * adev)2564 static void gfx_v9_0_init_compute_vmid(struct amdgpu_device *adev)
2565 {
2566 int i;
2567 uint32_t sh_mem_config;
2568 uint32_t sh_mem_bases;
2569
2570 /*
2571 * Configure apertures:
2572 * LDS: 0x60000000'00000000 - 0x60000001'00000000 (4GB)
2573 * Scratch: 0x60000001'00000000 - 0x60000002'00000000 (4GB)
2574 * GPUVM: 0x60010000'00000000 - 0x60020000'00000000 (1TB)
2575 */
2576 sh_mem_bases = DEFAULT_SH_MEM_BASES | (DEFAULT_SH_MEM_BASES << 16);
2577
2578 sh_mem_config = SH_MEM_ADDRESS_MODE_64 |
2579 SH_MEM_ALIGNMENT_MODE_UNALIGNED <<
2580 SH_MEM_CONFIG__ALIGNMENT_MODE__SHIFT;
2581
2582 mutex_lock(&adev->srbm_mutex);
2583 for (i = adev->vm_manager.first_kfd_vmid; i < AMDGPU_NUM_VMID; i++) {
2584 soc15_grbm_select(adev, 0, 0, 0, i, 0);
2585 /* CP and shaders */
2586 WREG32_SOC15_RLC(GC, 0, mmSH_MEM_CONFIG, sh_mem_config);
2587 WREG32_SOC15_RLC(GC, 0, mmSH_MEM_BASES, sh_mem_bases);
2588 }
2589 soc15_grbm_select(adev, 0, 0, 0, 0, 0);
2590 mutex_unlock(&adev->srbm_mutex);
2591
2592 /* Initialize all compute VMIDs to have no GDS, GWS, or OA
2593 access. These should be enabled by FW for target VMIDs. */
2594 for (i = adev->vm_manager.first_kfd_vmid; i < AMDGPU_NUM_VMID; i++) {
2595 WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_BASE, 2 * i, 0);
2596 WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_SIZE, 2 * i, 0);
2597 WREG32_SOC15_OFFSET(GC, 0, mmGDS_GWS_VMID0, i, 0);
2598 WREG32_SOC15_OFFSET(GC, 0, mmGDS_OA_VMID0, i, 0);
2599 }
2600 }
2601
gfx_v9_0_init_gds_vmid(struct amdgpu_device * adev)2602 static void gfx_v9_0_init_gds_vmid(struct amdgpu_device *adev)
2603 {
2604 int vmid;
2605
2606 /*
2607 * Initialize all compute and user-gfx VMIDs to have no GDS, GWS, or OA
2608 * access. Compute VMIDs should be enabled by FW for target VMIDs,
2609 * the driver can enable them for graphics. VMID0 should maintain
2610 * access so that HWS firmware can save/restore entries.
2611 */
2612 for (vmid = 1; vmid < AMDGPU_NUM_VMID; vmid++) {
2613 WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_BASE, 2 * vmid, 0);
2614 WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_SIZE, 2 * vmid, 0);
2615 WREG32_SOC15_OFFSET(GC, 0, mmGDS_GWS_VMID0, vmid, 0);
2616 WREG32_SOC15_OFFSET(GC, 0, mmGDS_OA_VMID0, vmid, 0);
2617 }
2618 }
2619
gfx_v9_0_init_sq_config(struct amdgpu_device * adev)2620 static void gfx_v9_0_init_sq_config(struct amdgpu_device *adev)
2621 {
2622 uint32_t tmp;
2623
2624 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
2625 case IP_VERSION(9, 4, 1):
2626 tmp = RREG32_SOC15(GC, 0, mmSQ_CONFIG);
2627 tmp = REG_SET_FIELD(tmp, SQ_CONFIG, DISABLE_BARRIER_WAITCNT,
2628 !READ_ONCE(adev->barrier_has_auto_waitcnt));
2629 WREG32_SOC15(GC, 0, mmSQ_CONFIG, tmp);
2630 break;
2631 default:
2632 break;
2633 }
2634 }
2635
gfx_v9_0_constants_init(struct amdgpu_device * adev)2636 static void gfx_v9_0_constants_init(struct amdgpu_device *adev)
2637 {
2638 u32 tmp;
2639 int i;
2640
2641 if (!amdgpu_sriov_vf(adev) ||
2642 amdgpu_ip_version(adev, GC_HWIP, 0) != IP_VERSION(9, 4, 2)) {
2643 WREG32_FIELD15_RLC(GC, 0, GRBM_CNTL, READ_TIMEOUT, 0xff);
2644 }
2645
2646 gfx_v9_0_tiling_mode_table_init(adev);
2647
2648 if (adev->gfx.num_gfx_rings)
2649 gfx_v9_0_setup_rb(adev);
2650 gfx_v9_0_get_cu_info(adev, &adev->gfx.cu_info);
2651 adev->gfx.config.db_debug2 = RREG32_SOC15(GC, 0, mmDB_DEBUG2);
2652
2653 /* XXX SH_MEM regs */
2654 /* where to put LDS, scratch, GPUVM in FSA64 space */
2655 mutex_lock(&adev->srbm_mutex);
2656 for (i = 0; i < adev->vm_manager.id_mgr[AMDGPU_GFXHUB(0)].num_ids; i++) {
2657 soc15_grbm_select(adev, 0, 0, 0, i, 0);
2658 /* CP and shaders */
2659 if (i == 0) {
2660 tmp = REG_SET_FIELD(0, SH_MEM_CONFIG, ALIGNMENT_MODE,
2661 SH_MEM_ALIGNMENT_MODE_UNALIGNED);
2662 tmp = REG_SET_FIELD(tmp, SH_MEM_CONFIG, RETRY_DISABLE,
2663 !!adev->gmc.noretry);
2664 WREG32_SOC15_RLC(GC, 0, mmSH_MEM_CONFIG, tmp);
2665 WREG32_SOC15_RLC(GC, 0, mmSH_MEM_BASES, 0);
2666 } else {
2667 tmp = REG_SET_FIELD(0, SH_MEM_CONFIG, ALIGNMENT_MODE,
2668 SH_MEM_ALIGNMENT_MODE_UNALIGNED);
2669 tmp = REG_SET_FIELD(tmp, SH_MEM_CONFIG, RETRY_DISABLE,
2670 !!adev->gmc.noretry);
2671 WREG32_SOC15_RLC(GC, 0, mmSH_MEM_CONFIG, tmp);
2672 tmp = REG_SET_FIELD(0, SH_MEM_BASES, PRIVATE_BASE,
2673 (adev->gmc.private_aperture_start >> 48));
2674 tmp = REG_SET_FIELD(tmp, SH_MEM_BASES, SHARED_BASE,
2675 (adev->gmc.shared_aperture_start >> 48));
2676 WREG32_SOC15_RLC(GC, 0, mmSH_MEM_BASES, tmp);
2677 }
2678 }
2679 soc15_grbm_select(adev, 0, 0, 0, 0, 0);
2680
2681 mutex_unlock(&adev->srbm_mutex);
2682
2683 gfx_v9_0_init_compute_vmid(adev);
2684 gfx_v9_0_init_gds_vmid(adev);
2685 gfx_v9_0_init_sq_config(adev);
2686 }
2687
gfx_v9_0_wait_for_rlc_serdes(struct amdgpu_device * adev)2688 static void gfx_v9_0_wait_for_rlc_serdes(struct amdgpu_device *adev)
2689 {
2690 u32 i, j, k;
2691 u32 mask;
2692
2693 mutex_lock(&adev->grbm_idx_mutex);
2694 for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
2695 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {
2696 amdgpu_gfx_select_se_sh(adev, i, j, 0xffffffff, 0);
2697 for (k = 0; k < adev->usec_timeout; k++) {
2698 if (RREG32_SOC15(GC, 0, mmRLC_SERDES_CU_MASTER_BUSY) == 0)
2699 break;
2700 udelay(1);
2701 }
2702 if (k == adev->usec_timeout) {
2703 amdgpu_gfx_select_se_sh(adev, 0xffffffff,
2704 0xffffffff, 0xffffffff, 0);
2705 mutex_unlock(&adev->grbm_idx_mutex);
2706 DRM_INFO("Timeout wait for RLC serdes %u,%u\n",
2707 i, j);
2708 return;
2709 }
2710 }
2711 }
2712 amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0);
2713 mutex_unlock(&adev->grbm_idx_mutex);
2714
2715 mask = RLC_SERDES_NONCU_MASTER_BUSY__SE_MASTER_BUSY_MASK |
2716 RLC_SERDES_NONCU_MASTER_BUSY__GC_MASTER_BUSY_MASK |
2717 RLC_SERDES_NONCU_MASTER_BUSY__TC0_MASTER_BUSY_MASK |
2718 RLC_SERDES_NONCU_MASTER_BUSY__TC1_MASTER_BUSY_MASK;
2719 for (k = 0; k < adev->usec_timeout; k++) {
2720 if ((RREG32_SOC15(GC, 0, mmRLC_SERDES_NONCU_MASTER_BUSY) & mask) == 0)
2721 break;
2722 udelay(1);
2723 }
2724 }
2725
gfx_v9_0_enable_gui_idle_interrupt(struct amdgpu_device * adev,bool enable)2726 static void gfx_v9_0_enable_gui_idle_interrupt(struct amdgpu_device *adev,
2727 bool enable)
2728 {
2729 u32 tmp;
2730
2731 /* These interrupts should be enabled to drive DS clock */
2732
2733 tmp= RREG32_SOC15(GC, 0, mmCP_INT_CNTL_RING0);
2734
2735 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CNTX_BUSY_INT_ENABLE, enable ? 1 : 0);
2736 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CNTX_EMPTY_INT_ENABLE, enable ? 1 : 0);
2737 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CMP_BUSY_INT_ENABLE, enable ? 1 : 0);
2738 if (adev->gfx.num_gfx_rings)
2739 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, GFX_IDLE_INT_ENABLE, enable ? 1 : 0);
2740
2741 WREG32_SOC15(GC, 0, mmCP_INT_CNTL_RING0, tmp);
2742 }
2743
gfx_v9_0_init_csb(struct amdgpu_device * adev)2744 static void gfx_v9_0_init_csb(struct amdgpu_device *adev)
2745 {
2746 adev->gfx.rlc.funcs->get_csb_buffer(adev, adev->gfx.rlc.cs_ptr);
2747 /* csib */
2748 WREG32_RLC(SOC15_REG_OFFSET(GC, 0, mmRLC_CSIB_ADDR_HI),
2749 adev->gfx.rlc.clear_state_gpu_addr >> 32);
2750 WREG32_RLC(SOC15_REG_OFFSET(GC, 0, mmRLC_CSIB_ADDR_LO),
2751 adev->gfx.rlc.clear_state_gpu_addr & 0xfffffffc);
2752 WREG32_RLC(SOC15_REG_OFFSET(GC, 0, mmRLC_CSIB_LENGTH),
2753 adev->gfx.rlc.clear_state_size);
2754 }
2755
gfx_v9_1_parse_ind_reg_list(int * register_list_format,int indirect_offset,int list_size,int * unique_indirect_regs,int unique_indirect_reg_count,int * indirect_start_offsets,int * indirect_start_offsets_count,int max_start_offsets_count)2756 static void gfx_v9_1_parse_ind_reg_list(int *register_list_format,
2757 int indirect_offset,
2758 int list_size,
2759 int *unique_indirect_regs,
2760 int unique_indirect_reg_count,
2761 int *indirect_start_offsets,
2762 int *indirect_start_offsets_count,
2763 int max_start_offsets_count)
2764 {
2765 int idx;
2766
2767 for (; indirect_offset < list_size; indirect_offset++) {
2768 WARN_ON(*indirect_start_offsets_count >= max_start_offsets_count);
2769 indirect_start_offsets[*indirect_start_offsets_count] = indirect_offset;
2770 *indirect_start_offsets_count = *indirect_start_offsets_count + 1;
2771
2772 while (register_list_format[indirect_offset] != 0xFFFFFFFF) {
2773 indirect_offset += 2;
2774
2775 /* look for the matching indice */
2776 for (idx = 0; idx < unique_indirect_reg_count; idx++) {
2777 if (unique_indirect_regs[idx] ==
2778 register_list_format[indirect_offset] ||
2779 !unique_indirect_regs[idx])
2780 break;
2781 }
2782
2783 BUG_ON(idx >= unique_indirect_reg_count);
2784
2785 if (!unique_indirect_regs[idx])
2786 unique_indirect_regs[idx] = register_list_format[indirect_offset];
2787
2788 indirect_offset++;
2789 }
2790 }
2791 }
2792
gfx_v9_1_init_rlc_save_restore_list(struct amdgpu_device * adev)2793 static int gfx_v9_1_init_rlc_save_restore_list(struct amdgpu_device *adev)
2794 {
2795 int unique_indirect_regs[] = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0};
2796 int unique_indirect_reg_count = 0;
2797
2798 int indirect_start_offsets[] = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0};
2799 int indirect_start_offsets_count = 0;
2800
2801 int list_size = 0;
2802 int i = 0, j = 0;
2803 u32 tmp = 0;
2804
2805 u32 *register_list_format =
2806 kmemdup(adev->gfx.rlc.register_list_format,
2807 adev->gfx.rlc.reg_list_format_size_bytes, GFP_KERNEL);
2808 if (!register_list_format)
2809 return -ENOMEM;
2810
2811 /* setup unique_indirect_regs array and indirect_start_offsets array */
2812 unique_indirect_reg_count = ARRAY_SIZE(unique_indirect_regs);
2813 gfx_v9_1_parse_ind_reg_list(register_list_format,
2814 adev->gfx.rlc.reg_list_format_direct_reg_list_length,
2815 adev->gfx.rlc.reg_list_format_size_bytes >> 2,
2816 unique_indirect_regs,
2817 unique_indirect_reg_count,
2818 indirect_start_offsets,
2819 &indirect_start_offsets_count,
2820 ARRAY_SIZE(indirect_start_offsets));
2821
2822 /* enable auto inc in case it is disabled */
2823 tmp = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_CNTL));
2824 tmp |= RLC_SRM_CNTL__AUTO_INCR_ADDR_MASK;
2825 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_CNTL), tmp);
2826
2827 /* write register_restore table to offset 0x0 using RLC_SRM_ARAM_ADDR/DATA */
2828 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_ARAM_ADDR),
2829 RLC_SAVE_RESTORE_ADDR_STARTING_OFFSET);
2830 for (i = 0; i < adev->gfx.rlc.reg_list_size_bytes >> 2; i++)
2831 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_ARAM_DATA),
2832 adev->gfx.rlc.register_restore[i]);
2833
2834 /* load indirect register */
2835 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_ADDR),
2836 adev->gfx.rlc.reg_list_format_start);
2837
2838 /* direct register portion */
2839 for (i = 0; i < adev->gfx.rlc.reg_list_format_direct_reg_list_length; i++)
2840 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_DATA),
2841 register_list_format[i]);
2842
2843 /* indirect register portion */
2844 while (i < (adev->gfx.rlc.reg_list_format_size_bytes >> 2)) {
2845 if (register_list_format[i] == 0xFFFFFFFF) {
2846 WREG32_SOC15(GC, 0, mmRLC_GPM_SCRATCH_DATA, register_list_format[i++]);
2847 continue;
2848 }
2849
2850 WREG32_SOC15(GC, 0, mmRLC_GPM_SCRATCH_DATA, register_list_format[i++]);
2851 WREG32_SOC15(GC, 0, mmRLC_GPM_SCRATCH_DATA, register_list_format[i++]);
2852
2853 for (j = 0; j < unique_indirect_reg_count; j++) {
2854 if (register_list_format[i] == unique_indirect_regs[j]) {
2855 WREG32_SOC15(GC, 0, mmRLC_GPM_SCRATCH_DATA, j);
2856 break;
2857 }
2858 }
2859
2860 BUG_ON(j >= unique_indirect_reg_count);
2861
2862 i++;
2863 }
2864
2865 /* set save/restore list size */
2866 list_size = adev->gfx.rlc.reg_list_size_bytes >> 2;
2867 list_size = list_size >> 1;
2868 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_ADDR),
2869 adev->gfx.rlc.reg_restore_list_size);
2870 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_DATA), list_size);
2871
2872 /* write the starting offsets to RLC scratch ram */
2873 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_ADDR),
2874 adev->gfx.rlc.starting_offsets_start);
2875 for (i = 0; i < ARRAY_SIZE(indirect_start_offsets); i++)
2876 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_DATA),
2877 indirect_start_offsets[i]);
2878
2879 /* load unique indirect regs*/
2880 for (i = 0; i < ARRAY_SIZE(unique_indirect_regs); i++) {
2881 if (unique_indirect_regs[i] != 0) {
2882 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_INDEX_CNTL_ADDR_0)
2883 + GFX_RLC_SRM_INDEX_CNTL_ADDR_OFFSETS[i],
2884 unique_indirect_regs[i] & 0x3FFFF);
2885
2886 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_INDEX_CNTL_DATA_0)
2887 + GFX_RLC_SRM_INDEX_CNTL_DATA_OFFSETS[i],
2888 unique_indirect_regs[i] >> 20);
2889 }
2890 }
2891
2892 kfree(register_list_format);
2893 return 0;
2894 }
2895
gfx_v9_0_enable_save_restore_machine(struct amdgpu_device * adev)2896 static void gfx_v9_0_enable_save_restore_machine(struct amdgpu_device *adev)
2897 {
2898 WREG32_FIELD15(GC, 0, RLC_SRM_CNTL, SRM_ENABLE, 1);
2899 }
2900
pwr_10_0_gfxip_control_over_cgpg(struct amdgpu_device * adev,bool enable)2901 static void pwr_10_0_gfxip_control_over_cgpg(struct amdgpu_device *adev,
2902 bool enable)
2903 {
2904 uint32_t data = 0;
2905 uint32_t default_data = 0;
2906
2907 default_data = data = RREG32(SOC15_REG_OFFSET(PWR, 0, mmPWR_MISC_CNTL_STATUS));
2908 if (enable) {
2909 /* enable GFXIP control over CGPG */
2910 data |= PWR_MISC_CNTL_STATUS__PWR_GFX_RLC_CGPG_EN_MASK;
2911 if(default_data != data)
2912 WREG32(SOC15_REG_OFFSET(PWR, 0, mmPWR_MISC_CNTL_STATUS), data);
2913
2914 /* update status */
2915 data &= ~PWR_MISC_CNTL_STATUS__PWR_GFXOFF_STATUS_MASK;
2916 data |= (2 << PWR_MISC_CNTL_STATUS__PWR_GFXOFF_STATUS__SHIFT);
2917 if(default_data != data)
2918 WREG32(SOC15_REG_OFFSET(PWR, 0, mmPWR_MISC_CNTL_STATUS), data);
2919 } else {
2920 /* restore GFXIP control over GCPG */
2921 data &= ~PWR_MISC_CNTL_STATUS__PWR_GFX_RLC_CGPG_EN_MASK;
2922 if(default_data != data)
2923 WREG32(SOC15_REG_OFFSET(PWR, 0, mmPWR_MISC_CNTL_STATUS), data);
2924 }
2925 }
2926
gfx_v9_0_init_gfx_power_gating(struct amdgpu_device * adev)2927 static void gfx_v9_0_init_gfx_power_gating(struct amdgpu_device *adev)
2928 {
2929 uint32_t data = 0;
2930
2931 if (adev->pg_flags & (AMD_PG_SUPPORT_GFX_PG |
2932 AMD_PG_SUPPORT_GFX_SMG |
2933 AMD_PG_SUPPORT_GFX_DMG)) {
2934 /* init IDLE_POLL_COUNT = 60 */
2935 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_RB_WPTR_POLL_CNTL));
2936 data &= ~CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT_MASK;
2937 data |= (0x60 << CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT);
2938 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_RB_WPTR_POLL_CNTL), data);
2939
2940 /* init RLC PG Delay */
2941 data = 0;
2942 data |= (0x10 << RLC_PG_DELAY__POWER_UP_DELAY__SHIFT);
2943 data |= (0x10 << RLC_PG_DELAY__POWER_DOWN_DELAY__SHIFT);
2944 data |= (0x10 << RLC_PG_DELAY__CMD_PROPAGATE_DELAY__SHIFT);
2945 data |= (0x40 << RLC_PG_DELAY__MEM_SLEEP_DELAY__SHIFT);
2946 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY), data);
2947
2948 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY_2));
2949 data &= ~RLC_PG_DELAY_2__SERDES_CMD_DELAY_MASK;
2950 data |= (0x4 << RLC_PG_DELAY_2__SERDES_CMD_DELAY__SHIFT);
2951 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY_2), data);
2952
2953 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY_3));
2954 data &= ~RLC_PG_DELAY_3__CGCG_ACTIVE_BEFORE_CGPG_MASK;
2955 data |= (0xff << RLC_PG_DELAY_3__CGCG_ACTIVE_BEFORE_CGPG__SHIFT);
2956 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY_3), data);
2957
2958 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_AUTO_PG_CTRL));
2959 data &= ~RLC_AUTO_PG_CTRL__GRBM_REG_SAVE_GFX_IDLE_THRESHOLD_MASK;
2960
2961 /* program GRBM_REG_SAVE_GFX_IDLE_THRESHOLD to 0x55f0 */
2962 data |= (0x55f0 << RLC_AUTO_PG_CTRL__GRBM_REG_SAVE_GFX_IDLE_THRESHOLD__SHIFT);
2963 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_AUTO_PG_CTRL), data);
2964 if (amdgpu_ip_version(adev, GC_HWIP, 0) != IP_VERSION(9, 3, 0))
2965 pwr_10_0_gfxip_control_over_cgpg(adev, true);
2966 }
2967 }
2968
gfx_v9_0_enable_sck_slow_down_on_power_up(struct amdgpu_device * adev,bool enable)2969 static void gfx_v9_0_enable_sck_slow_down_on_power_up(struct amdgpu_device *adev,
2970 bool enable)
2971 {
2972 uint32_t data = 0;
2973 uint32_t default_data = 0;
2974
2975 default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));
2976 data = REG_SET_FIELD(data, RLC_PG_CNTL,
2977 SMU_CLK_SLOWDOWN_ON_PU_ENABLE,
2978 enable ? 1 : 0);
2979 if (default_data != data)
2980 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
2981 }
2982
gfx_v9_0_enable_sck_slow_down_on_power_down(struct amdgpu_device * adev,bool enable)2983 static void gfx_v9_0_enable_sck_slow_down_on_power_down(struct amdgpu_device *adev,
2984 bool enable)
2985 {
2986 uint32_t data = 0;
2987 uint32_t default_data = 0;
2988
2989 default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));
2990 data = REG_SET_FIELD(data, RLC_PG_CNTL,
2991 SMU_CLK_SLOWDOWN_ON_PD_ENABLE,
2992 enable ? 1 : 0);
2993 if(default_data != data)
2994 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
2995 }
2996
gfx_v9_0_enable_cp_power_gating(struct amdgpu_device * adev,bool enable)2997 static void gfx_v9_0_enable_cp_power_gating(struct amdgpu_device *adev,
2998 bool enable)
2999 {
3000 uint32_t data = 0;
3001 uint32_t default_data = 0;
3002
3003 default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));
3004 data = REG_SET_FIELD(data, RLC_PG_CNTL,
3005 CP_PG_DISABLE,
3006 enable ? 0 : 1);
3007 if(default_data != data)
3008 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
3009 }
3010
gfx_v9_0_enable_gfx_cg_power_gating(struct amdgpu_device * adev,bool enable)3011 static void gfx_v9_0_enable_gfx_cg_power_gating(struct amdgpu_device *adev,
3012 bool enable)
3013 {
3014 uint32_t data, default_data;
3015
3016 default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));
3017 data = REG_SET_FIELD(data, RLC_PG_CNTL,
3018 GFX_POWER_GATING_ENABLE,
3019 enable ? 1 : 0);
3020 if(default_data != data)
3021 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
3022 }
3023
gfx_v9_0_enable_gfx_pipeline_powergating(struct amdgpu_device * adev,bool enable)3024 static void gfx_v9_0_enable_gfx_pipeline_powergating(struct amdgpu_device *adev,
3025 bool enable)
3026 {
3027 uint32_t data, default_data;
3028
3029 default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));
3030 data = REG_SET_FIELD(data, RLC_PG_CNTL,
3031 GFX_PIPELINE_PG_ENABLE,
3032 enable ? 1 : 0);
3033 if(default_data != data)
3034 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
3035
3036 if (!enable)
3037 /* read any GFX register to wake up GFX */
3038 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmDB_RENDER_CONTROL));
3039 }
3040
gfx_v9_0_enable_gfx_static_mg_power_gating(struct amdgpu_device * adev,bool enable)3041 static void gfx_v9_0_enable_gfx_static_mg_power_gating(struct amdgpu_device *adev,
3042 bool enable)
3043 {
3044 uint32_t data, default_data;
3045
3046 default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));
3047 data = REG_SET_FIELD(data, RLC_PG_CNTL,
3048 STATIC_PER_CU_PG_ENABLE,
3049 enable ? 1 : 0);
3050 if(default_data != data)
3051 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
3052 }
3053
gfx_v9_0_enable_gfx_dynamic_mg_power_gating(struct amdgpu_device * adev,bool enable)3054 static void gfx_v9_0_enable_gfx_dynamic_mg_power_gating(struct amdgpu_device *adev,
3055 bool enable)
3056 {
3057 uint32_t data, default_data;
3058
3059 default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));
3060 data = REG_SET_FIELD(data, RLC_PG_CNTL,
3061 DYN_PER_CU_PG_ENABLE,
3062 enable ? 1 : 0);
3063 if(default_data != data)
3064 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
3065 }
3066
gfx_v9_0_init_pg(struct amdgpu_device * adev)3067 static void gfx_v9_0_init_pg(struct amdgpu_device *adev)
3068 {
3069 gfx_v9_0_init_csb(adev);
3070
3071 /*
3072 * Rlc save restore list is workable since v2_1.
3073 * And it's needed by gfxoff feature.
3074 */
3075 if (adev->gfx.rlc.is_rlc_v2_1) {
3076 if (amdgpu_ip_version(adev, GC_HWIP, 0) ==
3077 IP_VERSION(9, 2, 1) ||
3078 (adev->apu_flags & AMD_APU_IS_RAVEN2))
3079 gfx_v9_1_init_rlc_save_restore_list(adev);
3080 gfx_v9_0_enable_save_restore_machine(adev);
3081 }
3082
3083 if (adev->pg_flags & (AMD_PG_SUPPORT_GFX_PG |
3084 AMD_PG_SUPPORT_GFX_SMG |
3085 AMD_PG_SUPPORT_GFX_DMG |
3086 AMD_PG_SUPPORT_CP |
3087 AMD_PG_SUPPORT_GDS |
3088 AMD_PG_SUPPORT_RLC_SMU_HS)) {
3089 WREG32_SOC15(GC, 0, mmRLC_JUMP_TABLE_RESTORE,
3090 adev->gfx.rlc.cp_table_gpu_addr >> 8);
3091 gfx_v9_0_init_gfx_power_gating(adev);
3092 }
3093 }
3094
gfx_v9_0_rlc_stop(struct amdgpu_device * adev)3095 static void gfx_v9_0_rlc_stop(struct amdgpu_device *adev)
3096 {
3097 WREG32_FIELD15(GC, 0, RLC_CNTL, RLC_ENABLE_F32, 0);
3098 gfx_v9_0_enable_gui_idle_interrupt(adev, false);
3099 gfx_v9_0_wait_for_rlc_serdes(adev);
3100 }
3101
gfx_v9_0_rlc_reset(struct amdgpu_device * adev)3102 static void gfx_v9_0_rlc_reset(struct amdgpu_device *adev)
3103 {
3104 WREG32_FIELD15(GC, 0, GRBM_SOFT_RESET, SOFT_RESET_RLC, 1);
3105 udelay(50);
3106 WREG32_FIELD15(GC, 0, GRBM_SOFT_RESET, SOFT_RESET_RLC, 0);
3107 udelay(50);
3108 }
3109
gfx_v9_0_rlc_start(struct amdgpu_device * adev)3110 static void gfx_v9_0_rlc_start(struct amdgpu_device *adev)
3111 {
3112 #ifdef AMDGPU_RLC_DEBUG_RETRY
3113 u32 rlc_ucode_ver;
3114 #endif
3115
3116 WREG32_FIELD15(GC, 0, RLC_CNTL, RLC_ENABLE_F32, 1);
3117 udelay(50);
3118
3119 /* carrizo do enable cp interrupt after cp inited */
3120 if (!(adev->flags & AMD_IS_APU)) {
3121 gfx_v9_0_enable_gui_idle_interrupt(adev, true);
3122 udelay(50);
3123 }
3124
3125 #ifdef AMDGPU_RLC_DEBUG_RETRY
3126 /* RLC_GPM_GENERAL_6 : RLC Ucode version */
3127 rlc_ucode_ver = RREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_6);
3128 if(rlc_ucode_ver == 0x108) {
3129 DRM_INFO("Using rlc debug ucode. mmRLC_GPM_GENERAL_6 ==0x08%x / fw_ver == %i \n",
3130 rlc_ucode_ver, adev->gfx.rlc_fw_version);
3131 /* RLC_GPM_TIMER_INT_3 : Timer interval in RefCLK cycles,
3132 * default is 0x9C4 to create a 100us interval */
3133 WREG32_SOC15(GC, 0, mmRLC_GPM_TIMER_INT_3, 0x9C4);
3134 /* RLC_GPM_GENERAL_12 : Minimum gap between wptr and rptr
3135 * to disable the page fault retry interrupts, default is
3136 * 0x100 (256) */
3137 WREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_12, 0x100);
3138 }
3139 #endif
3140 }
3141
gfx_v9_0_rlc_load_microcode(struct amdgpu_device * adev)3142 static int gfx_v9_0_rlc_load_microcode(struct amdgpu_device *adev)
3143 {
3144 const struct rlc_firmware_header_v2_0 *hdr;
3145 const __le32 *fw_data;
3146 unsigned i, fw_size;
3147
3148 if (!adev->gfx.rlc_fw)
3149 return -EINVAL;
3150
3151 hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data;
3152 amdgpu_ucode_print_rlc_hdr(&hdr->header);
3153
3154 fw_data = (const __le32 *)(adev->gfx.rlc_fw->data +
3155 le32_to_cpu(hdr->header.ucode_array_offset_bytes));
3156 fw_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
3157
3158 WREG32_SOC15(GC, 0, mmRLC_GPM_UCODE_ADDR,
3159 RLCG_UCODE_LOADING_START_ADDRESS);
3160 for (i = 0; i < fw_size; i++)
3161 WREG32_SOC15(GC, 0, mmRLC_GPM_UCODE_DATA, le32_to_cpup(fw_data++));
3162 WREG32_SOC15(GC, 0, mmRLC_GPM_UCODE_ADDR, adev->gfx.rlc_fw_version);
3163
3164 return 0;
3165 }
3166
gfx_v9_0_rlc_resume(struct amdgpu_device * adev)3167 static int gfx_v9_0_rlc_resume(struct amdgpu_device *adev)
3168 {
3169 int r;
3170
3171 if (amdgpu_sriov_vf(adev)) {
3172 gfx_v9_0_init_csb(adev);
3173 return 0;
3174 }
3175
3176 adev->gfx.rlc.funcs->stop(adev);
3177
3178 /* disable CG */
3179 WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL, 0);
3180
3181 gfx_v9_0_init_pg(adev);
3182
3183 if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP) {
3184 /* legacy rlc firmware loading */
3185 r = gfx_v9_0_rlc_load_microcode(adev);
3186 if (r)
3187 return r;
3188 }
3189
3190 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
3191 case IP_VERSION(9, 2, 2):
3192 case IP_VERSION(9, 1, 0):
3193 gfx_v9_0_init_lbpw(adev);
3194 if (amdgpu_lbpw == 0)
3195 gfx_v9_0_enable_lbpw(adev, false);
3196 else
3197 gfx_v9_0_enable_lbpw(adev, true);
3198 break;
3199 case IP_VERSION(9, 4, 0):
3200 gfx_v9_4_init_lbpw(adev);
3201 if (amdgpu_lbpw > 0)
3202 gfx_v9_0_enable_lbpw(adev, true);
3203 else
3204 gfx_v9_0_enable_lbpw(adev, false);
3205 break;
3206 default:
3207 break;
3208 }
3209
3210 gfx_v9_0_update_spm_vmid_internal(adev, 0xf);
3211
3212 adev->gfx.rlc.funcs->start(adev);
3213
3214 return 0;
3215 }
3216
gfx_v9_0_cp_gfx_enable(struct amdgpu_device * adev,bool enable)3217 static void gfx_v9_0_cp_gfx_enable(struct amdgpu_device *adev, bool enable)
3218 {
3219 u32 tmp = RREG32_SOC15(GC, 0, mmCP_ME_CNTL);
3220
3221 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, CE_INVALIDATE_ICACHE, enable ? 0 : 1);
3222 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_INVALIDATE_ICACHE, enable ? 0 : 1);
3223 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_INVALIDATE_ICACHE, enable ? 0 : 1);
3224 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, CE_PIPE0_RESET, enable ? 0 : 1);
3225 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, CE_PIPE1_RESET, enable ? 0 : 1);
3226 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_PIPE0_RESET, enable ? 0 : 1);
3227 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_PIPE1_RESET, enable ? 0 : 1);
3228 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_PIPE0_RESET, enable ? 0 : 1);
3229 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_PIPE1_RESET, enable ? 0 : 1);
3230 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_HALT, enable ? 0 : 1);
3231 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_HALT, enable ? 0 : 1);
3232 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, CE_HALT, enable ? 0 : 1);
3233 WREG32_SOC15_RLC(GC, 0, mmCP_ME_CNTL, tmp);
3234 udelay(50);
3235 }
3236
gfx_v9_0_cp_gfx_load_microcode(struct amdgpu_device * adev)3237 static int gfx_v9_0_cp_gfx_load_microcode(struct amdgpu_device *adev)
3238 {
3239 const struct gfx_firmware_header_v1_0 *pfp_hdr;
3240 const struct gfx_firmware_header_v1_0 *ce_hdr;
3241 const struct gfx_firmware_header_v1_0 *me_hdr;
3242 const __le32 *fw_data;
3243 unsigned i, fw_size;
3244
3245 if (!adev->gfx.me_fw || !adev->gfx.pfp_fw || !adev->gfx.ce_fw)
3246 return -EINVAL;
3247
3248 pfp_hdr = (const struct gfx_firmware_header_v1_0 *)
3249 adev->gfx.pfp_fw->data;
3250 ce_hdr = (const struct gfx_firmware_header_v1_0 *)
3251 adev->gfx.ce_fw->data;
3252 me_hdr = (const struct gfx_firmware_header_v1_0 *)
3253 adev->gfx.me_fw->data;
3254
3255 amdgpu_ucode_print_gfx_hdr(&pfp_hdr->header);
3256 amdgpu_ucode_print_gfx_hdr(&ce_hdr->header);
3257 amdgpu_ucode_print_gfx_hdr(&me_hdr->header);
3258
3259 gfx_v9_0_cp_gfx_enable(adev, false);
3260
3261 /* PFP */
3262 fw_data = (const __le32 *)
3263 (adev->gfx.pfp_fw->data +
3264 le32_to_cpu(pfp_hdr->header.ucode_array_offset_bytes));
3265 fw_size = le32_to_cpu(pfp_hdr->header.ucode_size_bytes) / 4;
3266 WREG32_SOC15(GC, 0, mmCP_PFP_UCODE_ADDR, 0);
3267 for (i = 0; i < fw_size; i++)
3268 WREG32_SOC15(GC, 0, mmCP_PFP_UCODE_DATA, le32_to_cpup(fw_data++));
3269 WREG32_SOC15(GC, 0, mmCP_PFP_UCODE_ADDR, adev->gfx.pfp_fw_version);
3270
3271 /* CE */
3272 fw_data = (const __le32 *)
3273 (adev->gfx.ce_fw->data +
3274 le32_to_cpu(ce_hdr->header.ucode_array_offset_bytes));
3275 fw_size = le32_to_cpu(ce_hdr->header.ucode_size_bytes) / 4;
3276 WREG32_SOC15(GC, 0, mmCP_CE_UCODE_ADDR, 0);
3277 for (i = 0; i < fw_size; i++)
3278 WREG32_SOC15(GC, 0, mmCP_CE_UCODE_DATA, le32_to_cpup(fw_data++));
3279 WREG32_SOC15(GC, 0, mmCP_CE_UCODE_ADDR, adev->gfx.ce_fw_version);
3280
3281 /* ME */
3282 fw_data = (const __le32 *)
3283 (adev->gfx.me_fw->data +
3284 le32_to_cpu(me_hdr->header.ucode_array_offset_bytes));
3285 fw_size = le32_to_cpu(me_hdr->header.ucode_size_bytes) / 4;
3286 WREG32_SOC15(GC, 0, mmCP_ME_RAM_WADDR, 0);
3287 for (i = 0; i < fw_size; i++)
3288 WREG32_SOC15(GC, 0, mmCP_ME_RAM_DATA, le32_to_cpup(fw_data++));
3289 WREG32_SOC15(GC, 0, mmCP_ME_RAM_WADDR, adev->gfx.me_fw_version);
3290
3291 return 0;
3292 }
3293
gfx_v9_0_cp_gfx_start(struct amdgpu_device * adev)3294 static int gfx_v9_0_cp_gfx_start(struct amdgpu_device *adev)
3295 {
3296 struct amdgpu_ring *ring = &adev->gfx.gfx_ring[0];
3297 const struct cs_section_def *sect = NULL;
3298 const struct cs_extent_def *ext = NULL;
3299 int r, i, tmp;
3300
3301 /* init the CP */
3302 WREG32_SOC15(GC, 0, mmCP_MAX_CONTEXT, adev->gfx.config.max_hw_contexts - 1);
3303 WREG32_SOC15(GC, 0, mmCP_DEVICE_ID, 1);
3304
3305 gfx_v9_0_cp_gfx_enable(adev, true);
3306
3307 /* Now only limit the quirk on the APU gfx9 series and already
3308 * confirmed that the APU gfx10/gfx11 needn't such update.
3309 */
3310 if (adev->flags & AMD_IS_APU &&
3311 adev->in_s3 && !pm_resume_via_firmware()) {
3312 DRM_INFO("Will skip the CSB packet resubmit\n");
3313 return 0;
3314 }
3315 r = amdgpu_ring_alloc(ring, gfx_v9_0_get_csb_size(adev) + 4 + 3);
3316 if (r) {
3317 DRM_ERROR("amdgpu: cp failed to lock ring (%d).\n", r);
3318 return r;
3319 }
3320
3321 amdgpu_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3322 amdgpu_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
3323
3324 amdgpu_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1));
3325 amdgpu_ring_write(ring, 0x80000000);
3326 amdgpu_ring_write(ring, 0x80000000);
3327
3328 for (sect = gfx9_cs_data; sect->section != NULL; ++sect) {
3329 for (ext = sect->section; ext->extent != NULL; ++ext) {
3330 if (sect->id == SECT_CONTEXT) {
3331 amdgpu_ring_write(ring,
3332 PACKET3(PACKET3_SET_CONTEXT_REG,
3333 ext->reg_count));
3334 amdgpu_ring_write(ring,
3335 ext->reg_index - PACKET3_SET_CONTEXT_REG_START);
3336 for (i = 0; i < ext->reg_count; i++)
3337 amdgpu_ring_write(ring, ext->extent[i]);
3338 }
3339 }
3340 }
3341
3342 amdgpu_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3343 amdgpu_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
3344
3345 amdgpu_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
3346 amdgpu_ring_write(ring, 0);
3347
3348 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2));
3349 amdgpu_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE));
3350 amdgpu_ring_write(ring, 0x8000);
3351 amdgpu_ring_write(ring, 0x8000);
3352
3353 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG,1));
3354 tmp = (PACKET3_SET_UCONFIG_REG_INDEX_TYPE |
3355 (SOC15_REG_OFFSET(GC, 0, mmVGT_INDEX_TYPE) - PACKET3_SET_UCONFIG_REG_START));
3356 amdgpu_ring_write(ring, tmp);
3357 amdgpu_ring_write(ring, 0);
3358
3359 amdgpu_ring_commit(ring);
3360
3361 return 0;
3362 }
3363
gfx_v9_0_cp_gfx_resume(struct amdgpu_device * adev)3364 static int gfx_v9_0_cp_gfx_resume(struct amdgpu_device *adev)
3365 {
3366 struct amdgpu_ring *ring;
3367 u32 tmp;
3368 u32 rb_bufsz;
3369 u64 rb_addr, rptr_addr, wptr_gpu_addr;
3370
3371 /* Set the write pointer delay */
3372 WREG32_SOC15(GC, 0, mmCP_RB_WPTR_DELAY, 0);
3373
3374 /* set the RB to use vmid 0 */
3375 WREG32_SOC15(GC, 0, mmCP_RB_VMID, 0);
3376
3377 /* Set ring buffer size */
3378 ring = &adev->gfx.gfx_ring[0];
3379 rb_bufsz = order_base_2(ring->ring_size / 8);
3380 tmp = REG_SET_FIELD(0, CP_RB0_CNTL, RB_BUFSZ, rb_bufsz);
3381 tmp = REG_SET_FIELD(tmp, CP_RB0_CNTL, RB_BLKSZ, rb_bufsz - 2);
3382 #ifdef __BIG_ENDIAN
3383 tmp = REG_SET_FIELD(tmp, CP_RB0_CNTL, BUF_SWAP, 1);
3384 #endif
3385 WREG32_SOC15(GC, 0, mmCP_RB0_CNTL, tmp);
3386
3387 /* Initialize the ring buffer's write pointers */
3388 ring->wptr = 0;
3389 WREG32_SOC15(GC, 0, mmCP_RB0_WPTR, lower_32_bits(ring->wptr));
3390 WREG32_SOC15(GC, 0, mmCP_RB0_WPTR_HI, upper_32_bits(ring->wptr));
3391
3392 /* set the wb address whether it's enabled or not */
3393 rptr_addr = ring->rptr_gpu_addr;
3394 WREG32_SOC15(GC, 0, mmCP_RB0_RPTR_ADDR, lower_32_bits(rptr_addr));
3395 WREG32_SOC15(GC, 0, mmCP_RB0_RPTR_ADDR_HI, upper_32_bits(rptr_addr) & CP_RB_RPTR_ADDR_HI__RB_RPTR_ADDR_HI_MASK);
3396
3397 wptr_gpu_addr = ring->wptr_gpu_addr;
3398 WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_ADDR_LO, lower_32_bits(wptr_gpu_addr));
3399 WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_ADDR_HI, upper_32_bits(wptr_gpu_addr));
3400
3401 mdelay(1);
3402 WREG32_SOC15(GC, 0, mmCP_RB0_CNTL, tmp);
3403
3404 rb_addr = ring->gpu_addr >> 8;
3405 WREG32_SOC15(GC, 0, mmCP_RB0_BASE, rb_addr);
3406 WREG32_SOC15(GC, 0, mmCP_RB0_BASE_HI, upper_32_bits(rb_addr));
3407
3408 tmp = RREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_CONTROL);
3409 if (ring->use_doorbell) {
3410 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL,
3411 DOORBELL_OFFSET, ring->doorbell_index);
3412 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL,
3413 DOORBELL_EN, 1);
3414 } else {
3415 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL, DOORBELL_EN, 0);
3416 }
3417 WREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_CONTROL, tmp);
3418
3419 tmp = REG_SET_FIELD(0, CP_RB_DOORBELL_RANGE_LOWER,
3420 DOORBELL_RANGE_LOWER, ring->doorbell_index);
3421 WREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_RANGE_LOWER, tmp);
3422
3423 WREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_RANGE_UPPER,
3424 CP_RB_DOORBELL_RANGE_UPPER__DOORBELL_RANGE_UPPER_MASK);
3425
3426
3427 /* start the ring */
3428 gfx_v9_0_cp_gfx_start(adev);
3429
3430 return 0;
3431 }
3432
gfx_v9_0_cp_compute_enable(struct amdgpu_device * adev,bool enable)3433 static void gfx_v9_0_cp_compute_enable(struct amdgpu_device *adev, bool enable)
3434 {
3435 if (enable) {
3436 WREG32_SOC15_RLC(GC, 0, mmCP_MEC_CNTL, 0);
3437 } else {
3438 WREG32_SOC15_RLC(GC, 0, mmCP_MEC_CNTL,
3439 (CP_MEC_CNTL__MEC_INVALIDATE_ICACHE_MASK |
3440 CP_MEC_CNTL__MEC_ME1_PIPE0_RESET_MASK |
3441 CP_MEC_CNTL__MEC_ME1_PIPE1_RESET_MASK |
3442 CP_MEC_CNTL__MEC_ME1_PIPE2_RESET_MASK |
3443 CP_MEC_CNTL__MEC_ME1_PIPE3_RESET_MASK |
3444 CP_MEC_CNTL__MEC_ME2_PIPE0_RESET_MASK |
3445 CP_MEC_CNTL__MEC_ME2_PIPE1_RESET_MASK |
3446 CP_MEC_CNTL__MEC_ME1_HALT_MASK |
3447 CP_MEC_CNTL__MEC_ME2_HALT_MASK));
3448 adev->gfx.kiq[0].ring.sched.ready = false;
3449 }
3450 udelay(50);
3451 }
3452
gfx_v9_0_cp_compute_load_microcode(struct amdgpu_device * adev)3453 static int gfx_v9_0_cp_compute_load_microcode(struct amdgpu_device *adev)
3454 {
3455 const struct gfx_firmware_header_v1_0 *mec_hdr;
3456 const __le32 *fw_data;
3457 unsigned i;
3458 u32 tmp;
3459
3460 if (!adev->gfx.mec_fw)
3461 return -EINVAL;
3462
3463 gfx_v9_0_cp_compute_enable(adev, false);
3464
3465 mec_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data;
3466 amdgpu_ucode_print_gfx_hdr(&mec_hdr->header);
3467
3468 fw_data = (const __le32 *)
3469 (adev->gfx.mec_fw->data +
3470 le32_to_cpu(mec_hdr->header.ucode_array_offset_bytes));
3471 tmp = 0;
3472 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, VMID, 0);
3473 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, CACHE_POLICY, 0);
3474 WREG32_SOC15(GC, 0, mmCP_CPC_IC_BASE_CNTL, tmp);
3475
3476 WREG32_SOC15(GC, 0, mmCP_CPC_IC_BASE_LO,
3477 adev->gfx.mec.mec_fw_gpu_addr & 0xFFFFF000);
3478 WREG32_SOC15(GC, 0, mmCP_CPC_IC_BASE_HI,
3479 upper_32_bits(adev->gfx.mec.mec_fw_gpu_addr));
3480
3481 /* MEC1 */
3482 WREG32_SOC15(GC, 0, mmCP_MEC_ME1_UCODE_ADDR,
3483 mec_hdr->jt_offset);
3484 for (i = 0; i < mec_hdr->jt_size; i++)
3485 WREG32_SOC15(GC, 0, mmCP_MEC_ME1_UCODE_DATA,
3486 le32_to_cpup(fw_data + mec_hdr->jt_offset + i));
3487
3488 WREG32_SOC15(GC, 0, mmCP_MEC_ME1_UCODE_ADDR,
3489 adev->gfx.mec_fw_version);
3490 /* Todo : Loading MEC2 firmware is only necessary if MEC2 should run different microcode than MEC1. */
3491
3492 return 0;
3493 }
3494
3495 /* KIQ functions */
gfx_v9_0_kiq_setting(struct amdgpu_ring * ring)3496 static void gfx_v9_0_kiq_setting(struct amdgpu_ring *ring)
3497 {
3498 uint32_t tmp;
3499 struct amdgpu_device *adev = ring->adev;
3500
3501 /* tell RLC which is KIQ queue */
3502 tmp = RREG32_SOC15(GC, 0, mmRLC_CP_SCHEDULERS);
3503 tmp &= 0xffffff00;
3504 tmp |= (ring->me << 5) | (ring->pipe << 3) | (ring->queue);
3505 WREG32_SOC15_RLC(GC, 0, mmRLC_CP_SCHEDULERS, tmp | 0x80);
3506 }
3507
gfx_v9_0_mqd_set_priority(struct amdgpu_ring * ring,struct v9_mqd * mqd)3508 static void gfx_v9_0_mqd_set_priority(struct amdgpu_ring *ring, struct v9_mqd *mqd)
3509 {
3510 struct amdgpu_device *adev = ring->adev;
3511
3512 if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) {
3513 if (amdgpu_gfx_is_high_priority_compute_queue(adev, ring)) {
3514 mqd->cp_hqd_pipe_priority = AMDGPU_GFX_PIPE_PRIO_HIGH;
3515 mqd->cp_hqd_queue_priority =
3516 AMDGPU_GFX_QUEUE_PRIORITY_MAXIMUM;
3517 }
3518 }
3519 }
3520
gfx_v9_0_mqd_init(struct amdgpu_ring * ring)3521 static int gfx_v9_0_mqd_init(struct amdgpu_ring *ring)
3522 {
3523 struct amdgpu_device *adev = ring->adev;
3524 struct v9_mqd *mqd = ring->mqd_ptr;
3525 uint64_t hqd_gpu_addr, wb_gpu_addr, eop_base_addr;
3526 uint32_t tmp;
3527
3528 mqd->header = 0xC0310800;
3529 mqd->compute_pipelinestat_enable = 0x00000001;
3530 mqd->compute_static_thread_mgmt_se0 = 0xffffffff;
3531 mqd->compute_static_thread_mgmt_se1 = 0xffffffff;
3532 mqd->compute_static_thread_mgmt_se2 = 0xffffffff;
3533 mqd->compute_static_thread_mgmt_se3 = 0xffffffff;
3534 mqd->compute_static_thread_mgmt_se4 = 0xffffffff;
3535 mqd->compute_static_thread_mgmt_se5 = 0xffffffff;
3536 mqd->compute_static_thread_mgmt_se6 = 0xffffffff;
3537 mqd->compute_static_thread_mgmt_se7 = 0xffffffff;
3538 mqd->compute_misc_reserved = 0x00000003;
3539
3540 mqd->dynamic_cu_mask_addr_lo =
3541 lower_32_bits(ring->mqd_gpu_addr
3542 + offsetof(struct v9_mqd_allocation, dynamic_cu_mask));
3543 mqd->dynamic_cu_mask_addr_hi =
3544 upper_32_bits(ring->mqd_gpu_addr
3545 + offsetof(struct v9_mqd_allocation, dynamic_cu_mask));
3546
3547 eop_base_addr = ring->eop_gpu_addr >> 8;
3548 mqd->cp_hqd_eop_base_addr_lo = eop_base_addr;
3549 mqd->cp_hqd_eop_base_addr_hi = upper_32_bits(eop_base_addr);
3550
3551 /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
3552 tmp = RREG32_SOC15(GC, 0, mmCP_HQD_EOP_CONTROL);
3553 tmp = REG_SET_FIELD(tmp, CP_HQD_EOP_CONTROL, EOP_SIZE,
3554 (order_base_2(GFX9_MEC_HPD_SIZE / 4) - 1));
3555
3556 mqd->cp_hqd_eop_control = tmp;
3557
3558 /* enable doorbell? */
3559 tmp = RREG32_SOC15(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL);
3560
3561 if (ring->use_doorbell) {
3562 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3563 DOORBELL_OFFSET, ring->doorbell_index);
3564 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3565 DOORBELL_EN, 1);
3566 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3567 DOORBELL_SOURCE, 0);
3568 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3569 DOORBELL_HIT, 0);
3570 } else {
3571 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3572 DOORBELL_EN, 0);
3573 }
3574
3575 mqd->cp_hqd_pq_doorbell_control = tmp;
3576
3577 /* disable the queue if it's active */
3578 ring->wptr = 0;
3579 mqd->cp_hqd_dequeue_request = 0;
3580 mqd->cp_hqd_pq_rptr = 0;
3581 mqd->cp_hqd_pq_wptr_lo = 0;
3582 mqd->cp_hqd_pq_wptr_hi = 0;
3583
3584 /* set the pointer to the MQD */
3585 mqd->cp_mqd_base_addr_lo = ring->mqd_gpu_addr & 0xfffffffc;
3586 mqd->cp_mqd_base_addr_hi = upper_32_bits(ring->mqd_gpu_addr);
3587
3588 /* set MQD vmid to 0 */
3589 tmp = RREG32_SOC15(GC, 0, mmCP_MQD_CONTROL);
3590 tmp = REG_SET_FIELD(tmp, CP_MQD_CONTROL, VMID, 0);
3591 mqd->cp_mqd_control = tmp;
3592
3593 /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
3594 hqd_gpu_addr = ring->gpu_addr >> 8;
3595 mqd->cp_hqd_pq_base_lo = hqd_gpu_addr;
3596 mqd->cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr);
3597
3598 /* set up the HQD, this is similar to CP_RB0_CNTL */
3599 tmp = RREG32_SOC15(GC, 0, mmCP_HQD_PQ_CONTROL);
3600 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, QUEUE_SIZE,
3601 (order_base_2(ring->ring_size / 4) - 1));
3602 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, RPTR_BLOCK_SIZE,
3603 (order_base_2(AMDGPU_GPU_PAGE_SIZE / 4) - 1));
3604 #ifdef __BIG_ENDIAN
3605 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, ENDIAN_SWAP, 1);
3606 #endif
3607 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, UNORD_DISPATCH, 0);
3608 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, ROQ_PQ_IB_FLIP, 0);
3609 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, PRIV_STATE, 1);
3610 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, KMD_QUEUE, 1);
3611 mqd->cp_hqd_pq_control = tmp;
3612
3613 /* set the wb address whether it's enabled or not */
3614 wb_gpu_addr = ring->rptr_gpu_addr;
3615 mqd->cp_hqd_pq_rptr_report_addr_lo = wb_gpu_addr & 0xfffffffc;
3616 mqd->cp_hqd_pq_rptr_report_addr_hi =
3617 upper_32_bits(wb_gpu_addr) & 0xffff;
3618
3619 /* only used if CP_PQ_WPTR_POLL_CNTL.CP_PQ_WPTR_POLL_CNTL__EN_MASK=1 */
3620 wb_gpu_addr = ring->wptr_gpu_addr;
3621 mqd->cp_hqd_pq_wptr_poll_addr_lo = wb_gpu_addr & 0xfffffffc;
3622 mqd->cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff;
3623
3624 /* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */
3625 ring->wptr = 0;
3626 mqd->cp_hqd_pq_rptr = RREG32_SOC15(GC, 0, mmCP_HQD_PQ_RPTR);
3627
3628 /* set the vmid for the queue */
3629 mqd->cp_hqd_vmid = 0;
3630
3631 tmp = RREG32_SOC15(GC, 0, mmCP_HQD_PERSISTENT_STATE);
3632 tmp = REG_SET_FIELD(tmp, CP_HQD_PERSISTENT_STATE, PRELOAD_SIZE, 0x53);
3633 mqd->cp_hqd_persistent_state = tmp;
3634
3635 /* set MIN_IB_AVAIL_SIZE */
3636 tmp = RREG32_SOC15(GC, 0, mmCP_HQD_IB_CONTROL);
3637 tmp = REG_SET_FIELD(tmp, CP_HQD_IB_CONTROL, MIN_IB_AVAIL_SIZE, 3);
3638 mqd->cp_hqd_ib_control = tmp;
3639
3640 /* set static priority for a queue/ring */
3641 gfx_v9_0_mqd_set_priority(ring, mqd);
3642 mqd->cp_hqd_quantum = RREG32_SOC15(GC, 0, mmCP_HQD_QUANTUM);
3643
3644 /* map_queues packet doesn't need activate the queue,
3645 * so only kiq need set this field.
3646 */
3647 if (ring->funcs->type == AMDGPU_RING_TYPE_KIQ)
3648 mqd->cp_hqd_active = 1;
3649
3650 return 0;
3651 }
3652
gfx_v9_0_kiq_init_register(struct amdgpu_ring * ring)3653 static int gfx_v9_0_kiq_init_register(struct amdgpu_ring *ring)
3654 {
3655 struct amdgpu_device *adev = ring->adev;
3656 struct v9_mqd *mqd = ring->mqd_ptr;
3657 int j;
3658
3659 /* disable wptr polling */
3660 WREG32_FIELD15(GC, 0, CP_PQ_WPTR_POLL_CNTL, EN, 0);
3661
3662 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_EOP_BASE_ADDR,
3663 mqd->cp_hqd_eop_base_addr_lo);
3664 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_EOP_BASE_ADDR_HI,
3665 mqd->cp_hqd_eop_base_addr_hi);
3666
3667 /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
3668 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_EOP_CONTROL,
3669 mqd->cp_hqd_eop_control);
3670
3671 /* enable doorbell? */
3672 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL,
3673 mqd->cp_hqd_pq_doorbell_control);
3674
3675 /* disable the queue if it's active */
3676 if (RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1) {
3677 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_DEQUEUE_REQUEST, 1);
3678 for (j = 0; j < adev->usec_timeout; j++) {
3679 if (!(RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1))
3680 break;
3681 udelay(1);
3682 }
3683 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_DEQUEUE_REQUEST,
3684 mqd->cp_hqd_dequeue_request);
3685 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_RPTR,
3686 mqd->cp_hqd_pq_rptr);
3687 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_LO,
3688 mqd->cp_hqd_pq_wptr_lo);
3689 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_HI,
3690 mqd->cp_hqd_pq_wptr_hi);
3691 }
3692
3693 /* set the pointer to the MQD */
3694 WREG32_SOC15_RLC(GC, 0, mmCP_MQD_BASE_ADDR,
3695 mqd->cp_mqd_base_addr_lo);
3696 WREG32_SOC15_RLC(GC, 0, mmCP_MQD_BASE_ADDR_HI,
3697 mqd->cp_mqd_base_addr_hi);
3698
3699 /* set MQD vmid to 0 */
3700 WREG32_SOC15_RLC(GC, 0, mmCP_MQD_CONTROL,
3701 mqd->cp_mqd_control);
3702
3703 /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
3704 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_BASE,
3705 mqd->cp_hqd_pq_base_lo);
3706 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_BASE_HI,
3707 mqd->cp_hqd_pq_base_hi);
3708
3709 /* set up the HQD, this is similar to CP_RB0_CNTL */
3710 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_CONTROL,
3711 mqd->cp_hqd_pq_control);
3712
3713 /* set the wb address whether it's enabled or not */
3714 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_RPTR_REPORT_ADDR,
3715 mqd->cp_hqd_pq_rptr_report_addr_lo);
3716 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_RPTR_REPORT_ADDR_HI,
3717 mqd->cp_hqd_pq_rptr_report_addr_hi);
3718
3719 /* only used if CP_PQ_WPTR_POLL_CNTL.CP_PQ_WPTR_POLL_CNTL__EN_MASK=1 */
3720 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_POLL_ADDR,
3721 mqd->cp_hqd_pq_wptr_poll_addr_lo);
3722 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_POLL_ADDR_HI,
3723 mqd->cp_hqd_pq_wptr_poll_addr_hi);
3724
3725 /* enable the doorbell if requested */
3726 if (ring->use_doorbell) {
3727 WREG32_SOC15(GC, 0, mmCP_MEC_DOORBELL_RANGE_LOWER,
3728 (adev->doorbell_index.kiq * 2) << 2);
3729 /* If GC has entered CGPG, ringing doorbell > first page
3730 * doesn't wakeup GC. Enlarge CP_MEC_DOORBELL_RANGE_UPPER to
3731 * workaround this issue. And this change has to align with firmware
3732 * update.
3733 */
3734 if (check_if_enlarge_doorbell_range(adev))
3735 WREG32_SOC15(GC, 0, mmCP_MEC_DOORBELL_RANGE_UPPER,
3736 (adev->doorbell.size - 4));
3737 else
3738 WREG32_SOC15(GC, 0, mmCP_MEC_DOORBELL_RANGE_UPPER,
3739 (adev->doorbell_index.userqueue_end * 2) << 2);
3740 }
3741
3742 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL,
3743 mqd->cp_hqd_pq_doorbell_control);
3744
3745 /* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */
3746 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_LO,
3747 mqd->cp_hqd_pq_wptr_lo);
3748 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_HI,
3749 mqd->cp_hqd_pq_wptr_hi);
3750
3751 /* set the vmid for the queue */
3752 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_VMID, mqd->cp_hqd_vmid);
3753
3754 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PERSISTENT_STATE,
3755 mqd->cp_hqd_persistent_state);
3756
3757 /* activate the queue */
3758 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_ACTIVE,
3759 mqd->cp_hqd_active);
3760
3761 if (ring->use_doorbell)
3762 WREG32_FIELD15(GC, 0, CP_PQ_STATUS, DOORBELL_ENABLE, 1);
3763
3764 return 0;
3765 }
3766
gfx_v9_0_kiq_fini_register(struct amdgpu_ring * ring)3767 static int gfx_v9_0_kiq_fini_register(struct amdgpu_ring *ring)
3768 {
3769 struct amdgpu_device *adev = ring->adev;
3770 int j;
3771
3772 /* disable the queue if it's active */
3773 if (RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1) {
3774
3775 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_DEQUEUE_REQUEST, 1);
3776
3777 for (j = 0; j < adev->usec_timeout; j++) {
3778 if (!(RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1))
3779 break;
3780 udelay(1);
3781 }
3782
3783 if (j == AMDGPU_MAX_USEC_TIMEOUT) {
3784 DRM_DEBUG("KIQ dequeue request failed.\n");
3785
3786 /* Manual disable if dequeue request times out */
3787 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_ACTIVE, 0);
3788 }
3789
3790 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_DEQUEUE_REQUEST,
3791 0);
3792 }
3793
3794 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_IQ_TIMER, 0);
3795 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_IB_CONTROL, 0);
3796 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PERSISTENT_STATE, 0);
3797 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL, 0x40000000);
3798 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL, 0);
3799 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_RPTR, 0);
3800 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_HI, 0);
3801 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_LO, 0);
3802
3803 return 0;
3804 }
3805
gfx_v9_0_kiq_init_queue(struct amdgpu_ring * ring)3806 static int gfx_v9_0_kiq_init_queue(struct amdgpu_ring *ring)
3807 {
3808 struct amdgpu_device *adev = ring->adev;
3809 struct v9_mqd *mqd = ring->mqd_ptr;
3810 struct v9_mqd *tmp_mqd;
3811
3812 gfx_v9_0_kiq_setting(ring);
3813
3814 /* GPU could be in bad state during probe, driver trigger the reset
3815 * after load the SMU, in this case , the mqd is not be initialized.
3816 * driver need to re-init the mqd.
3817 * check mqd->cp_hqd_pq_control since this value should not be 0
3818 */
3819 tmp_mqd = (struct v9_mqd *)adev->gfx.kiq[0].mqd_backup;
3820 if (amdgpu_in_reset(adev) && tmp_mqd->cp_hqd_pq_control){
3821 /* for GPU_RESET case , reset MQD to a clean status */
3822 if (adev->gfx.kiq[0].mqd_backup)
3823 memcpy(mqd, adev->gfx.kiq[0].mqd_backup, sizeof(struct v9_mqd_allocation));
3824
3825 /* reset ring buffer */
3826 ring->wptr = 0;
3827 amdgpu_ring_clear_ring(ring);
3828
3829 mutex_lock(&adev->srbm_mutex);
3830 soc15_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0, 0);
3831 gfx_v9_0_kiq_init_register(ring);
3832 soc15_grbm_select(adev, 0, 0, 0, 0, 0);
3833 mutex_unlock(&adev->srbm_mutex);
3834 } else {
3835 memset((void *)mqd, 0, sizeof(struct v9_mqd_allocation));
3836 ((struct v9_mqd_allocation *)mqd)->dynamic_cu_mask = 0xFFFFFFFF;
3837 ((struct v9_mqd_allocation *)mqd)->dynamic_rb_mask = 0xFFFFFFFF;
3838 if (amdgpu_sriov_vf(adev) && adev->in_suspend)
3839 amdgpu_ring_clear_ring(ring);
3840 mutex_lock(&adev->srbm_mutex);
3841 soc15_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0, 0);
3842 gfx_v9_0_mqd_init(ring);
3843 gfx_v9_0_kiq_init_register(ring);
3844 soc15_grbm_select(adev, 0, 0, 0, 0, 0);
3845 mutex_unlock(&adev->srbm_mutex);
3846
3847 if (adev->gfx.kiq[0].mqd_backup)
3848 memcpy(adev->gfx.kiq[0].mqd_backup, mqd, sizeof(struct v9_mqd_allocation));
3849 }
3850
3851 return 0;
3852 }
3853
gfx_v9_0_kcq_init_queue(struct amdgpu_ring * ring,bool restore)3854 static int gfx_v9_0_kcq_init_queue(struct amdgpu_ring *ring, bool restore)
3855 {
3856 struct amdgpu_device *adev = ring->adev;
3857 struct v9_mqd *mqd = ring->mqd_ptr;
3858 int mqd_idx = ring - &adev->gfx.compute_ring[0];
3859 struct v9_mqd *tmp_mqd;
3860
3861 /* Same as above kiq init, driver need to re-init the mqd if mqd->cp_hqd_pq_control
3862 * is not be initialized before
3863 */
3864 tmp_mqd = (struct v9_mqd *)adev->gfx.mec.mqd_backup[mqd_idx];
3865
3866 if (!restore && (!tmp_mqd->cp_hqd_pq_control ||
3867 (!amdgpu_in_reset(adev) && !adev->in_suspend))) {
3868 memset((void *)mqd, 0, sizeof(struct v9_mqd_allocation));
3869 ((struct v9_mqd_allocation *)mqd)->dynamic_cu_mask = 0xFFFFFFFF;
3870 ((struct v9_mqd_allocation *)mqd)->dynamic_rb_mask = 0xFFFFFFFF;
3871 mutex_lock(&adev->srbm_mutex);
3872 soc15_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0, 0);
3873 gfx_v9_0_mqd_init(ring);
3874 soc15_grbm_select(adev, 0, 0, 0, 0, 0);
3875 mutex_unlock(&adev->srbm_mutex);
3876
3877 if (adev->gfx.mec.mqd_backup[mqd_idx])
3878 memcpy(adev->gfx.mec.mqd_backup[mqd_idx], mqd, sizeof(struct v9_mqd_allocation));
3879 } else {
3880 /* restore MQD to a clean status */
3881 if (adev->gfx.mec.mqd_backup[mqd_idx])
3882 memcpy(mqd, adev->gfx.mec.mqd_backup[mqd_idx], sizeof(struct v9_mqd_allocation));
3883 /* reset ring buffer */
3884 ring->wptr = 0;
3885 atomic64_set((atomic64_t *)ring->wptr_cpu_addr, 0);
3886 amdgpu_ring_clear_ring(ring);
3887 }
3888
3889 return 0;
3890 }
3891
gfx_v9_0_kiq_resume(struct amdgpu_device * adev)3892 static int gfx_v9_0_kiq_resume(struct amdgpu_device *adev)
3893 {
3894 gfx_v9_0_kiq_init_queue(&adev->gfx.kiq[0].ring);
3895 return 0;
3896 }
3897
gfx_v9_0_kcq_resume(struct amdgpu_device * adev)3898 static int gfx_v9_0_kcq_resume(struct amdgpu_device *adev)
3899 {
3900 int i, r;
3901
3902 gfx_v9_0_cp_compute_enable(adev, true);
3903
3904 for (i = 0; i < adev->gfx.num_compute_rings; i++) {
3905 r = gfx_v9_0_kcq_init_queue(&adev->gfx.compute_ring[i], false);
3906 if (r)
3907 return r;
3908 }
3909
3910 return amdgpu_gfx_enable_kcq(adev, 0);
3911 }
3912
gfx_v9_0_cp_resume(struct amdgpu_device * adev)3913 static int gfx_v9_0_cp_resume(struct amdgpu_device *adev)
3914 {
3915 int r, i;
3916 struct amdgpu_ring *ring;
3917
3918 if (!(adev->flags & AMD_IS_APU))
3919 gfx_v9_0_enable_gui_idle_interrupt(adev, false);
3920
3921 if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP) {
3922 if (adev->gfx.num_gfx_rings) {
3923 /* legacy firmware loading */
3924 r = gfx_v9_0_cp_gfx_load_microcode(adev);
3925 if (r)
3926 return r;
3927 }
3928
3929 r = gfx_v9_0_cp_compute_load_microcode(adev);
3930 if (r)
3931 return r;
3932 }
3933
3934 if (adev->gfx.num_gfx_rings)
3935 gfx_v9_0_cp_gfx_enable(adev, false);
3936 gfx_v9_0_cp_compute_enable(adev, false);
3937
3938 r = gfx_v9_0_kiq_resume(adev);
3939 if (r)
3940 return r;
3941
3942 if (adev->gfx.num_gfx_rings) {
3943 r = gfx_v9_0_cp_gfx_resume(adev);
3944 if (r)
3945 return r;
3946 }
3947
3948 r = gfx_v9_0_kcq_resume(adev);
3949 if (r)
3950 return r;
3951
3952 if (adev->gfx.num_gfx_rings) {
3953 ring = &adev->gfx.gfx_ring[0];
3954 r = amdgpu_ring_test_helper(ring);
3955 if (r)
3956 return r;
3957 }
3958
3959 for (i = 0; i < adev->gfx.num_compute_rings; i++) {
3960 ring = &adev->gfx.compute_ring[i];
3961 amdgpu_ring_test_helper(ring);
3962 }
3963
3964 gfx_v9_0_enable_gui_idle_interrupt(adev, true);
3965
3966 return 0;
3967 }
3968
gfx_v9_0_init_tcp_config(struct amdgpu_device * adev)3969 static void gfx_v9_0_init_tcp_config(struct amdgpu_device *adev)
3970 {
3971 u32 tmp;
3972
3973 if (amdgpu_ip_version(adev, GC_HWIP, 0) != IP_VERSION(9, 4, 1) &&
3974 amdgpu_ip_version(adev, GC_HWIP, 0) != IP_VERSION(9, 4, 2))
3975 return;
3976
3977 tmp = RREG32_SOC15(GC, 0, mmTCP_ADDR_CONFIG);
3978 tmp = REG_SET_FIELD(tmp, TCP_ADDR_CONFIG, ENABLE64KHASH,
3979 adev->df.hash_status.hash_64k);
3980 tmp = REG_SET_FIELD(tmp, TCP_ADDR_CONFIG, ENABLE2MHASH,
3981 adev->df.hash_status.hash_2m);
3982 tmp = REG_SET_FIELD(tmp, TCP_ADDR_CONFIG, ENABLE1GHASH,
3983 adev->df.hash_status.hash_1g);
3984 WREG32_SOC15(GC, 0, mmTCP_ADDR_CONFIG, tmp);
3985 }
3986
gfx_v9_0_cp_enable(struct amdgpu_device * adev,bool enable)3987 static void gfx_v9_0_cp_enable(struct amdgpu_device *adev, bool enable)
3988 {
3989 if (adev->gfx.num_gfx_rings)
3990 gfx_v9_0_cp_gfx_enable(adev, enable);
3991 gfx_v9_0_cp_compute_enable(adev, enable);
3992 }
3993
gfx_v9_0_hw_init(struct amdgpu_ip_block * ip_block)3994 static int gfx_v9_0_hw_init(struct amdgpu_ip_block *ip_block)
3995 {
3996 int r;
3997 struct amdgpu_device *adev = ip_block->adev;
3998
3999 amdgpu_gfx_cleaner_shader_init(adev, adev->gfx.cleaner_shader_size,
4000 adev->gfx.cleaner_shader_ptr);
4001
4002 if (!amdgpu_sriov_vf(adev))
4003 gfx_v9_0_init_golden_registers(adev);
4004
4005 gfx_v9_0_constants_init(adev);
4006
4007 gfx_v9_0_init_tcp_config(adev);
4008
4009 r = adev->gfx.rlc.funcs->resume(adev);
4010 if (r)
4011 return r;
4012
4013 r = gfx_v9_0_cp_resume(adev);
4014 if (r)
4015 return r;
4016
4017 if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 2) &&
4018 !amdgpu_sriov_vf(adev))
4019 gfx_v9_4_2_set_power_brake_sequence(adev);
4020
4021 return r;
4022 }
4023
gfx_v9_0_hw_fini(struct amdgpu_ip_block * ip_block)4024 static int gfx_v9_0_hw_fini(struct amdgpu_ip_block *ip_block)
4025 {
4026 struct amdgpu_device *adev = ip_block->adev;
4027
4028 if (amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX))
4029 amdgpu_irq_put(adev, &adev->gfx.cp_ecc_error_irq, 0);
4030 amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0);
4031 amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0);
4032 amdgpu_irq_put(adev, &adev->gfx.bad_op_irq, 0);
4033
4034 /* DF freeze and kcq disable will fail */
4035 if (!amdgpu_ras_intr_triggered())
4036 /* disable KCQ to avoid CPC touch memory not valid anymore */
4037 amdgpu_gfx_disable_kcq(adev, 0);
4038
4039 if (amdgpu_sriov_vf(adev)) {
4040 gfx_v9_0_cp_gfx_enable(adev, false);
4041 /* must disable polling for SRIOV when hw finished, otherwise
4042 * CPC engine may still keep fetching WB address which is already
4043 * invalid after sw finished and trigger DMAR reading error in
4044 * hypervisor side.
4045 */
4046 WREG32_FIELD15(GC, 0, CP_PQ_WPTR_POLL_CNTL, EN, 0);
4047 return 0;
4048 }
4049
4050 /* Use deinitialize sequence from CAIL when unbinding device from driver,
4051 * otherwise KIQ is hanging when binding back
4052 */
4053 if (!amdgpu_in_reset(adev) && !adev->in_suspend) {
4054 mutex_lock(&adev->srbm_mutex);
4055 soc15_grbm_select(adev, adev->gfx.kiq[0].ring.me,
4056 adev->gfx.kiq[0].ring.pipe,
4057 adev->gfx.kiq[0].ring.queue, 0, 0);
4058 gfx_v9_0_kiq_fini_register(&adev->gfx.kiq[0].ring);
4059 soc15_grbm_select(adev, 0, 0, 0, 0, 0);
4060 mutex_unlock(&adev->srbm_mutex);
4061 }
4062
4063 gfx_v9_0_cp_enable(adev, false);
4064
4065 /* Skip stopping RLC with A+A reset or when RLC controls GFX clock */
4066 if ((adev->gmc.xgmi.connected_to_cpu && amdgpu_in_reset(adev)) ||
4067 (amdgpu_ip_version(adev, GC_HWIP, 0) >= IP_VERSION(9, 4, 2))) {
4068 dev_dbg(adev->dev, "Skipping RLC halt\n");
4069 return 0;
4070 }
4071
4072 adev->gfx.rlc.funcs->stop(adev);
4073 return 0;
4074 }
4075
gfx_v9_0_suspend(struct amdgpu_ip_block * ip_block)4076 static int gfx_v9_0_suspend(struct amdgpu_ip_block *ip_block)
4077 {
4078 return gfx_v9_0_hw_fini(ip_block);
4079 }
4080
gfx_v9_0_resume(struct amdgpu_ip_block * ip_block)4081 static int gfx_v9_0_resume(struct amdgpu_ip_block *ip_block)
4082 {
4083 return gfx_v9_0_hw_init(ip_block);
4084 }
4085
gfx_v9_0_is_idle(struct amdgpu_ip_block * ip_block)4086 static bool gfx_v9_0_is_idle(struct amdgpu_ip_block *ip_block)
4087 {
4088 struct amdgpu_device *adev = ip_block->adev;
4089
4090 if (REG_GET_FIELD(RREG32_SOC15(GC, 0, mmGRBM_STATUS),
4091 GRBM_STATUS, GUI_ACTIVE))
4092 return false;
4093 else
4094 return true;
4095 }
4096
gfx_v9_0_wait_for_idle(struct amdgpu_ip_block * ip_block)4097 static int gfx_v9_0_wait_for_idle(struct amdgpu_ip_block *ip_block)
4098 {
4099 unsigned i;
4100 struct amdgpu_device *adev = ip_block->adev;
4101
4102 for (i = 0; i < adev->usec_timeout; i++) {
4103 if (gfx_v9_0_is_idle(ip_block))
4104 return 0;
4105 udelay(1);
4106 }
4107 return -ETIMEDOUT;
4108 }
4109
gfx_v9_0_soft_reset(struct amdgpu_ip_block * ip_block)4110 static int gfx_v9_0_soft_reset(struct amdgpu_ip_block *ip_block)
4111 {
4112 u32 grbm_soft_reset = 0;
4113 u32 tmp;
4114 struct amdgpu_device *adev = ip_block->adev;
4115
4116 /* GRBM_STATUS */
4117 tmp = RREG32_SOC15(GC, 0, mmGRBM_STATUS);
4118 if (tmp & (GRBM_STATUS__PA_BUSY_MASK | GRBM_STATUS__SC_BUSY_MASK |
4119 GRBM_STATUS__BCI_BUSY_MASK | GRBM_STATUS__SX_BUSY_MASK |
4120 GRBM_STATUS__TA_BUSY_MASK | GRBM_STATUS__VGT_BUSY_MASK |
4121 GRBM_STATUS__DB_BUSY_MASK | GRBM_STATUS__CB_BUSY_MASK |
4122 GRBM_STATUS__GDS_BUSY_MASK | GRBM_STATUS__SPI_BUSY_MASK |
4123 GRBM_STATUS__IA_BUSY_MASK | GRBM_STATUS__IA_BUSY_NO_DMA_MASK)) {
4124 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset,
4125 GRBM_SOFT_RESET, SOFT_RESET_CP, 1);
4126 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset,
4127 GRBM_SOFT_RESET, SOFT_RESET_GFX, 1);
4128 }
4129
4130 if (tmp & (GRBM_STATUS__CP_BUSY_MASK | GRBM_STATUS__CP_COHERENCY_BUSY_MASK)) {
4131 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset,
4132 GRBM_SOFT_RESET, SOFT_RESET_CP, 1);
4133 }
4134
4135 /* GRBM_STATUS2 */
4136 tmp = RREG32_SOC15(GC, 0, mmGRBM_STATUS2);
4137 if (REG_GET_FIELD(tmp, GRBM_STATUS2, RLC_BUSY))
4138 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset,
4139 GRBM_SOFT_RESET, SOFT_RESET_RLC, 1);
4140
4141
4142 if (grbm_soft_reset) {
4143 /* stop the rlc */
4144 adev->gfx.rlc.funcs->stop(adev);
4145
4146 if (adev->gfx.num_gfx_rings)
4147 /* Disable GFX parsing/prefetching */
4148 gfx_v9_0_cp_gfx_enable(adev, false);
4149
4150 /* Disable MEC parsing/prefetching */
4151 gfx_v9_0_cp_compute_enable(adev, false);
4152
4153 if (grbm_soft_reset) {
4154 tmp = RREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET);
4155 tmp |= grbm_soft_reset;
4156 dev_info(adev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
4157 WREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET, tmp);
4158 tmp = RREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET);
4159
4160 udelay(50);
4161
4162 tmp &= ~grbm_soft_reset;
4163 WREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET, tmp);
4164 tmp = RREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET);
4165 }
4166
4167 /* Wait a little for things to settle down */
4168 udelay(50);
4169 }
4170 return 0;
4171 }
4172
gfx_v9_0_kiq_read_clock(struct amdgpu_device * adev)4173 static uint64_t gfx_v9_0_kiq_read_clock(struct amdgpu_device *adev)
4174 {
4175 signed long r, cnt = 0;
4176 unsigned long flags;
4177 uint32_t seq, reg_val_offs = 0;
4178 uint64_t value = 0;
4179 struct amdgpu_kiq *kiq = &adev->gfx.kiq[0];
4180 struct amdgpu_ring *ring = &kiq->ring;
4181
4182 BUG_ON(!ring->funcs->emit_rreg);
4183
4184 spin_lock_irqsave(&kiq->ring_lock, flags);
4185 if (amdgpu_device_wb_get(adev, ®_val_offs)) {
4186 pr_err("critical bug! too many kiq readers\n");
4187 goto failed_unlock;
4188 }
4189 amdgpu_ring_alloc(ring, 32);
4190 amdgpu_ring_write(ring, PACKET3(PACKET3_COPY_DATA, 4));
4191 amdgpu_ring_write(ring, 9 | /* src: register*/
4192 (5 << 8) | /* dst: memory */
4193 (1 << 16) | /* count sel */
4194 (1 << 20)); /* write confirm */
4195 amdgpu_ring_write(ring, 0);
4196 amdgpu_ring_write(ring, 0);
4197 amdgpu_ring_write(ring, lower_32_bits(adev->wb.gpu_addr +
4198 reg_val_offs * 4));
4199 amdgpu_ring_write(ring, upper_32_bits(adev->wb.gpu_addr +
4200 reg_val_offs * 4));
4201 r = amdgpu_fence_emit_polling(ring, &seq, MAX_KIQ_REG_WAIT);
4202 if (r)
4203 goto failed_undo;
4204
4205 amdgpu_ring_commit(ring);
4206 spin_unlock_irqrestore(&kiq->ring_lock, flags);
4207
4208 r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT);
4209
4210 /* don't wait anymore for gpu reset case because this way may
4211 * block gpu_recover() routine forever, e.g. this virt_kiq_rreg
4212 * is triggered in TTM and ttm_bo_lock_delayed_workqueue() will
4213 * never return if we keep waiting in virt_kiq_rreg, which cause
4214 * gpu_recover() hang there.
4215 *
4216 * also don't wait anymore for IRQ context
4217 * */
4218 if (r < 1 && (amdgpu_in_reset(adev)))
4219 goto failed_kiq_read;
4220
4221 might_sleep();
4222 while (r < 1 && cnt++ < MAX_KIQ_REG_TRY) {
4223 msleep(MAX_KIQ_REG_BAILOUT_INTERVAL);
4224 r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT);
4225 }
4226
4227 if (cnt > MAX_KIQ_REG_TRY)
4228 goto failed_kiq_read;
4229
4230 mb();
4231 value = (uint64_t)adev->wb.wb[reg_val_offs] |
4232 (uint64_t)adev->wb.wb[reg_val_offs + 1 ] << 32ULL;
4233 amdgpu_device_wb_free(adev, reg_val_offs);
4234 return value;
4235
4236 failed_undo:
4237 amdgpu_ring_undo(ring);
4238 failed_unlock:
4239 spin_unlock_irqrestore(&kiq->ring_lock, flags);
4240 failed_kiq_read:
4241 if (reg_val_offs)
4242 amdgpu_device_wb_free(adev, reg_val_offs);
4243 pr_err("failed to read gpu clock\n");
4244 return ~0;
4245 }
4246
gfx_v9_0_get_gpu_clock_counter(struct amdgpu_device * adev)4247 static uint64_t gfx_v9_0_get_gpu_clock_counter(struct amdgpu_device *adev)
4248 {
4249 uint64_t clock, clock_lo, clock_hi, hi_check;
4250
4251 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
4252 case IP_VERSION(9, 3, 0):
4253 preempt_disable();
4254 clock_hi = RREG32_SOC15_NO_KIQ(SMUIO, 0, mmGOLDEN_TSC_COUNT_UPPER_Renoir);
4255 clock_lo = RREG32_SOC15_NO_KIQ(SMUIO, 0, mmGOLDEN_TSC_COUNT_LOWER_Renoir);
4256 hi_check = RREG32_SOC15_NO_KIQ(SMUIO, 0, mmGOLDEN_TSC_COUNT_UPPER_Renoir);
4257 /* The SMUIO TSC clock frequency is 100MHz, which sets 32-bit carry over
4258 * roughly every 42 seconds.
4259 */
4260 if (hi_check != clock_hi) {
4261 clock_lo = RREG32_SOC15_NO_KIQ(SMUIO, 0, mmGOLDEN_TSC_COUNT_LOWER_Renoir);
4262 clock_hi = hi_check;
4263 }
4264 preempt_enable();
4265 clock = clock_lo | (clock_hi << 32ULL);
4266 break;
4267 default:
4268 amdgpu_gfx_off_ctrl(adev, false);
4269 mutex_lock(&adev->gfx.gpu_clock_mutex);
4270 if (amdgpu_ip_version(adev, GC_HWIP, 0) ==
4271 IP_VERSION(9, 0, 1) &&
4272 amdgpu_sriov_runtime(adev)) {
4273 clock = gfx_v9_0_kiq_read_clock(adev);
4274 } else {
4275 WREG32_SOC15(GC, 0, mmRLC_CAPTURE_GPU_CLOCK_COUNT, 1);
4276 clock = (uint64_t)RREG32_SOC15(GC, 0, mmRLC_GPU_CLOCK_COUNT_LSB) |
4277 ((uint64_t)RREG32_SOC15(GC, 0, mmRLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
4278 }
4279 mutex_unlock(&adev->gfx.gpu_clock_mutex);
4280 amdgpu_gfx_off_ctrl(adev, true);
4281 break;
4282 }
4283 return clock;
4284 }
4285
gfx_v9_0_ring_emit_gds_switch(struct amdgpu_ring * ring,uint32_t vmid,uint32_t gds_base,uint32_t gds_size,uint32_t gws_base,uint32_t gws_size,uint32_t oa_base,uint32_t oa_size)4286 static void gfx_v9_0_ring_emit_gds_switch(struct amdgpu_ring *ring,
4287 uint32_t vmid,
4288 uint32_t gds_base, uint32_t gds_size,
4289 uint32_t gws_base, uint32_t gws_size,
4290 uint32_t oa_base, uint32_t oa_size)
4291 {
4292 struct amdgpu_device *adev = ring->adev;
4293
4294 /* GDS Base */
4295 gfx_v9_0_write_data_to_reg(ring, 0, false,
4296 SOC15_REG_OFFSET(GC, 0, mmGDS_VMID0_BASE) + 2 * vmid,
4297 gds_base);
4298
4299 /* GDS Size */
4300 gfx_v9_0_write_data_to_reg(ring, 0, false,
4301 SOC15_REG_OFFSET(GC, 0, mmGDS_VMID0_SIZE) + 2 * vmid,
4302 gds_size);
4303
4304 /* GWS */
4305 gfx_v9_0_write_data_to_reg(ring, 0, false,
4306 SOC15_REG_OFFSET(GC, 0, mmGDS_GWS_VMID0) + vmid,
4307 gws_size << GDS_GWS_VMID0__SIZE__SHIFT | gws_base);
4308
4309 /* OA */
4310 gfx_v9_0_write_data_to_reg(ring, 0, false,
4311 SOC15_REG_OFFSET(GC, 0, mmGDS_OA_VMID0) + vmid,
4312 (1 << (oa_size + oa_base)) - (1 << oa_base));
4313 }
4314
4315 static const u32 vgpr_init_compute_shader[] =
4316 {
4317 0xb07c0000, 0xbe8000ff,
4318 0x000000f8, 0xbf110800,
4319 0x7e000280, 0x7e020280,
4320 0x7e040280, 0x7e060280,
4321 0x7e080280, 0x7e0a0280,
4322 0x7e0c0280, 0x7e0e0280,
4323 0x80808800, 0xbe803200,
4324 0xbf84fff5, 0xbf9c0000,
4325 0xd28c0001, 0x0001007f,
4326 0xd28d0001, 0x0002027e,
4327 0x10020288, 0xb8810904,
4328 0xb7814000, 0xd1196a01,
4329 0x00000301, 0xbe800087,
4330 0xbefc00c1, 0xd89c4000,
4331 0x00020201, 0xd89cc080,
4332 0x00040401, 0x320202ff,
4333 0x00000800, 0x80808100,
4334 0xbf84fff8, 0x7e020280,
4335 0xbf810000, 0x00000000,
4336 };
4337
4338 static const u32 sgpr_init_compute_shader[] =
4339 {
4340 0xb07c0000, 0xbe8000ff,
4341 0x0000005f, 0xbee50080,
4342 0xbe812c65, 0xbe822c65,
4343 0xbe832c65, 0xbe842c65,
4344 0xbe852c65, 0xb77c0005,
4345 0x80808500, 0xbf84fff8,
4346 0xbe800080, 0xbf810000,
4347 };
4348
4349 static const u32 vgpr_init_compute_shader_arcturus[] = {
4350 0xd3d94000, 0x18000080, 0xd3d94001, 0x18000080, 0xd3d94002, 0x18000080,
4351 0xd3d94003, 0x18000080, 0xd3d94004, 0x18000080, 0xd3d94005, 0x18000080,
4352 0xd3d94006, 0x18000080, 0xd3d94007, 0x18000080, 0xd3d94008, 0x18000080,
4353 0xd3d94009, 0x18000080, 0xd3d9400a, 0x18000080, 0xd3d9400b, 0x18000080,
4354 0xd3d9400c, 0x18000080, 0xd3d9400d, 0x18000080, 0xd3d9400e, 0x18000080,
4355 0xd3d9400f, 0x18000080, 0xd3d94010, 0x18000080, 0xd3d94011, 0x18000080,
4356 0xd3d94012, 0x18000080, 0xd3d94013, 0x18000080, 0xd3d94014, 0x18000080,
4357 0xd3d94015, 0x18000080, 0xd3d94016, 0x18000080, 0xd3d94017, 0x18000080,
4358 0xd3d94018, 0x18000080, 0xd3d94019, 0x18000080, 0xd3d9401a, 0x18000080,
4359 0xd3d9401b, 0x18000080, 0xd3d9401c, 0x18000080, 0xd3d9401d, 0x18000080,
4360 0xd3d9401e, 0x18000080, 0xd3d9401f, 0x18000080, 0xd3d94020, 0x18000080,
4361 0xd3d94021, 0x18000080, 0xd3d94022, 0x18000080, 0xd3d94023, 0x18000080,
4362 0xd3d94024, 0x18000080, 0xd3d94025, 0x18000080, 0xd3d94026, 0x18000080,
4363 0xd3d94027, 0x18000080, 0xd3d94028, 0x18000080, 0xd3d94029, 0x18000080,
4364 0xd3d9402a, 0x18000080, 0xd3d9402b, 0x18000080, 0xd3d9402c, 0x18000080,
4365 0xd3d9402d, 0x18000080, 0xd3d9402e, 0x18000080, 0xd3d9402f, 0x18000080,
4366 0xd3d94030, 0x18000080, 0xd3d94031, 0x18000080, 0xd3d94032, 0x18000080,
4367 0xd3d94033, 0x18000080, 0xd3d94034, 0x18000080, 0xd3d94035, 0x18000080,
4368 0xd3d94036, 0x18000080, 0xd3d94037, 0x18000080, 0xd3d94038, 0x18000080,
4369 0xd3d94039, 0x18000080, 0xd3d9403a, 0x18000080, 0xd3d9403b, 0x18000080,
4370 0xd3d9403c, 0x18000080, 0xd3d9403d, 0x18000080, 0xd3d9403e, 0x18000080,
4371 0xd3d9403f, 0x18000080, 0xd3d94040, 0x18000080, 0xd3d94041, 0x18000080,
4372 0xd3d94042, 0x18000080, 0xd3d94043, 0x18000080, 0xd3d94044, 0x18000080,
4373 0xd3d94045, 0x18000080, 0xd3d94046, 0x18000080, 0xd3d94047, 0x18000080,
4374 0xd3d94048, 0x18000080, 0xd3d94049, 0x18000080, 0xd3d9404a, 0x18000080,
4375 0xd3d9404b, 0x18000080, 0xd3d9404c, 0x18000080, 0xd3d9404d, 0x18000080,
4376 0xd3d9404e, 0x18000080, 0xd3d9404f, 0x18000080, 0xd3d94050, 0x18000080,
4377 0xd3d94051, 0x18000080, 0xd3d94052, 0x18000080, 0xd3d94053, 0x18000080,
4378 0xd3d94054, 0x18000080, 0xd3d94055, 0x18000080, 0xd3d94056, 0x18000080,
4379 0xd3d94057, 0x18000080, 0xd3d94058, 0x18000080, 0xd3d94059, 0x18000080,
4380 0xd3d9405a, 0x18000080, 0xd3d9405b, 0x18000080, 0xd3d9405c, 0x18000080,
4381 0xd3d9405d, 0x18000080, 0xd3d9405e, 0x18000080, 0xd3d9405f, 0x18000080,
4382 0xd3d94060, 0x18000080, 0xd3d94061, 0x18000080, 0xd3d94062, 0x18000080,
4383 0xd3d94063, 0x18000080, 0xd3d94064, 0x18000080, 0xd3d94065, 0x18000080,
4384 0xd3d94066, 0x18000080, 0xd3d94067, 0x18000080, 0xd3d94068, 0x18000080,
4385 0xd3d94069, 0x18000080, 0xd3d9406a, 0x18000080, 0xd3d9406b, 0x18000080,
4386 0xd3d9406c, 0x18000080, 0xd3d9406d, 0x18000080, 0xd3d9406e, 0x18000080,
4387 0xd3d9406f, 0x18000080, 0xd3d94070, 0x18000080, 0xd3d94071, 0x18000080,
4388 0xd3d94072, 0x18000080, 0xd3d94073, 0x18000080, 0xd3d94074, 0x18000080,
4389 0xd3d94075, 0x18000080, 0xd3d94076, 0x18000080, 0xd3d94077, 0x18000080,
4390 0xd3d94078, 0x18000080, 0xd3d94079, 0x18000080, 0xd3d9407a, 0x18000080,
4391 0xd3d9407b, 0x18000080, 0xd3d9407c, 0x18000080, 0xd3d9407d, 0x18000080,
4392 0xd3d9407e, 0x18000080, 0xd3d9407f, 0x18000080, 0xd3d94080, 0x18000080,
4393 0xd3d94081, 0x18000080, 0xd3d94082, 0x18000080, 0xd3d94083, 0x18000080,
4394 0xd3d94084, 0x18000080, 0xd3d94085, 0x18000080, 0xd3d94086, 0x18000080,
4395 0xd3d94087, 0x18000080, 0xd3d94088, 0x18000080, 0xd3d94089, 0x18000080,
4396 0xd3d9408a, 0x18000080, 0xd3d9408b, 0x18000080, 0xd3d9408c, 0x18000080,
4397 0xd3d9408d, 0x18000080, 0xd3d9408e, 0x18000080, 0xd3d9408f, 0x18000080,
4398 0xd3d94090, 0x18000080, 0xd3d94091, 0x18000080, 0xd3d94092, 0x18000080,
4399 0xd3d94093, 0x18000080, 0xd3d94094, 0x18000080, 0xd3d94095, 0x18000080,
4400 0xd3d94096, 0x18000080, 0xd3d94097, 0x18000080, 0xd3d94098, 0x18000080,
4401 0xd3d94099, 0x18000080, 0xd3d9409a, 0x18000080, 0xd3d9409b, 0x18000080,
4402 0xd3d9409c, 0x18000080, 0xd3d9409d, 0x18000080, 0xd3d9409e, 0x18000080,
4403 0xd3d9409f, 0x18000080, 0xd3d940a0, 0x18000080, 0xd3d940a1, 0x18000080,
4404 0xd3d940a2, 0x18000080, 0xd3d940a3, 0x18000080, 0xd3d940a4, 0x18000080,
4405 0xd3d940a5, 0x18000080, 0xd3d940a6, 0x18000080, 0xd3d940a7, 0x18000080,
4406 0xd3d940a8, 0x18000080, 0xd3d940a9, 0x18000080, 0xd3d940aa, 0x18000080,
4407 0xd3d940ab, 0x18000080, 0xd3d940ac, 0x18000080, 0xd3d940ad, 0x18000080,
4408 0xd3d940ae, 0x18000080, 0xd3d940af, 0x18000080, 0xd3d940b0, 0x18000080,
4409 0xd3d940b1, 0x18000080, 0xd3d940b2, 0x18000080, 0xd3d940b3, 0x18000080,
4410 0xd3d940b4, 0x18000080, 0xd3d940b5, 0x18000080, 0xd3d940b6, 0x18000080,
4411 0xd3d940b7, 0x18000080, 0xd3d940b8, 0x18000080, 0xd3d940b9, 0x18000080,
4412 0xd3d940ba, 0x18000080, 0xd3d940bb, 0x18000080, 0xd3d940bc, 0x18000080,
4413 0xd3d940bd, 0x18000080, 0xd3d940be, 0x18000080, 0xd3d940bf, 0x18000080,
4414 0xd3d940c0, 0x18000080, 0xd3d940c1, 0x18000080, 0xd3d940c2, 0x18000080,
4415 0xd3d940c3, 0x18000080, 0xd3d940c4, 0x18000080, 0xd3d940c5, 0x18000080,
4416 0xd3d940c6, 0x18000080, 0xd3d940c7, 0x18000080, 0xd3d940c8, 0x18000080,
4417 0xd3d940c9, 0x18000080, 0xd3d940ca, 0x18000080, 0xd3d940cb, 0x18000080,
4418 0xd3d940cc, 0x18000080, 0xd3d940cd, 0x18000080, 0xd3d940ce, 0x18000080,
4419 0xd3d940cf, 0x18000080, 0xd3d940d0, 0x18000080, 0xd3d940d1, 0x18000080,
4420 0xd3d940d2, 0x18000080, 0xd3d940d3, 0x18000080, 0xd3d940d4, 0x18000080,
4421 0xd3d940d5, 0x18000080, 0xd3d940d6, 0x18000080, 0xd3d940d7, 0x18000080,
4422 0xd3d940d8, 0x18000080, 0xd3d940d9, 0x18000080, 0xd3d940da, 0x18000080,
4423 0xd3d940db, 0x18000080, 0xd3d940dc, 0x18000080, 0xd3d940dd, 0x18000080,
4424 0xd3d940de, 0x18000080, 0xd3d940df, 0x18000080, 0xd3d940e0, 0x18000080,
4425 0xd3d940e1, 0x18000080, 0xd3d940e2, 0x18000080, 0xd3d940e3, 0x18000080,
4426 0xd3d940e4, 0x18000080, 0xd3d940e5, 0x18000080, 0xd3d940e6, 0x18000080,
4427 0xd3d940e7, 0x18000080, 0xd3d940e8, 0x18000080, 0xd3d940e9, 0x18000080,
4428 0xd3d940ea, 0x18000080, 0xd3d940eb, 0x18000080, 0xd3d940ec, 0x18000080,
4429 0xd3d940ed, 0x18000080, 0xd3d940ee, 0x18000080, 0xd3d940ef, 0x18000080,
4430 0xd3d940f0, 0x18000080, 0xd3d940f1, 0x18000080, 0xd3d940f2, 0x18000080,
4431 0xd3d940f3, 0x18000080, 0xd3d940f4, 0x18000080, 0xd3d940f5, 0x18000080,
4432 0xd3d940f6, 0x18000080, 0xd3d940f7, 0x18000080, 0xd3d940f8, 0x18000080,
4433 0xd3d940f9, 0x18000080, 0xd3d940fa, 0x18000080, 0xd3d940fb, 0x18000080,
4434 0xd3d940fc, 0x18000080, 0xd3d940fd, 0x18000080, 0xd3d940fe, 0x18000080,
4435 0xd3d940ff, 0x18000080, 0xb07c0000, 0xbe8a00ff, 0x000000f8, 0xbf11080a,
4436 0x7e000280, 0x7e020280, 0x7e040280, 0x7e060280, 0x7e080280, 0x7e0a0280,
4437 0x7e0c0280, 0x7e0e0280, 0x808a880a, 0xbe80320a, 0xbf84fff5, 0xbf9c0000,
4438 0xd28c0001, 0x0001007f, 0xd28d0001, 0x0002027e, 0x10020288, 0xb88b0904,
4439 0xb78b4000, 0xd1196a01, 0x00001701, 0xbe8a0087, 0xbefc00c1, 0xd89c4000,
4440 0x00020201, 0xd89cc080, 0x00040401, 0x320202ff, 0x00000800, 0x808a810a,
4441 0xbf84fff8, 0xbf810000,
4442 };
4443
4444 /* When below register arrays changed, please update gpr_reg_size,
4445 and sec_ded_counter_reg_size in function gfx_v9_0_do_edc_gpr_workarounds,
4446 to cover all gfx9 ASICs */
4447 static const struct soc15_reg_entry vgpr_init_regs[] = {
4448 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_RESOURCE_LIMITS), 0x0000000 },
4449 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_X), 0x40 },
4450 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Y), 4 },
4451 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Z), 1 },
4452 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC1), 0x3f },
4453 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC2), 0x400000 }, /* 64KB LDS */
4454 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE0), 0xffffffff },
4455 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE1), 0xffffffff },
4456 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE2), 0xffffffff },
4457 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE3), 0xffffffff },
4458 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE4), 0xffffffff },
4459 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE5), 0xffffffff },
4460 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE6), 0xffffffff },
4461 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE7), 0xffffffff },
4462 };
4463
4464 static const struct soc15_reg_entry vgpr_init_regs_arcturus[] = {
4465 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_RESOURCE_LIMITS), 0x0000000 },
4466 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_X), 0x40 },
4467 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Y), 4 },
4468 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Z), 1 },
4469 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC1), 0xbf },
4470 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC2), 0x400000 }, /* 64KB LDS */
4471 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE0), 0xffffffff },
4472 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE1), 0xffffffff },
4473 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE2), 0xffffffff },
4474 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE3), 0xffffffff },
4475 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE4), 0xffffffff },
4476 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE5), 0xffffffff },
4477 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE6), 0xffffffff },
4478 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE7), 0xffffffff },
4479 };
4480
4481 static const struct soc15_reg_entry sgpr1_init_regs[] = {
4482 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_RESOURCE_LIMITS), 0x0000000 },
4483 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_X), 0x40 },
4484 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Y), 8 },
4485 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Z), 1 },
4486 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC1), 0x240 }, /* (80 GPRS) */
4487 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC2), 0x0 },
4488 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE0), 0x000000ff },
4489 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE1), 0x000000ff },
4490 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE2), 0x000000ff },
4491 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE3), 0x000000ff },
4492 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE4), 0x000000ff },
4493 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE5), 0x000000ff },
4494 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE6), 0x000000ff },
4495 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE7), 0x000000ff },
4496 };
4497
4498 static const struct soc15_reg_entry sgpr2_init_regs[] = {
4499 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_RESOURCE_LIMITS), 0x0000000 },
4500 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_X), 0x40 },
4501 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Y), 8 },
4502 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Z), 1 },
4503 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC1), 0x240 }, /* (80 GPRS) */
4504 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC2), 0x0 },
4505 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE0), 0x0000ff00 },
4506 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE1), 0x0000ff00 },
4507 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE2), 0x0000ff00 },
4508 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE3), 0x0000ff00 },
4509 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE4), 0x0000ff00 },
4510 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE5), 0x0000ff00 },
4511 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE6), 0x0000ff00 },
4512 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE7), 0x0000ff00 },
4513 };
4514
4515 static const struct soc15_reg_entry gfx_v9_0_edc_counter_regs[] = {
4516 { SOC15_REG_ENTRY(GC, 0, mmCPC_EDC_SCRATCH_CNT), 0, 1, 1},
4517 { SOC15_REG_ENTRY(GC, 0, mmCPC_EDC_UCODE_CNT), 0, 1, 1},
4518 { SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_ROQ_CNT), 0, 1, 1},
4519 { SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_TAG_CNT), 0, 1, 1},
4520 { SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_DMA_CNT), 0, 1, 1},
4521 { SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_TAG_CNT), 0, 1, 1},
4522 { SOC15_REG_ENTRY(GC, 0, mmDC_EDC_CSINVOC_CNT), 0, 1, 1},
4523 { SOC15_REG_ENTRY(GC, 0, mmDC_EDC_RESTORE_CNT), 0, 1, 1},
4524 { SOC15_REG_ENTRY(GC, 0, mmDC_EDC_STATE_CNT), 0, 1, 1},
4525 { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_CNT), 0, 1, 1},
4526 { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_GRBM_CNT), 0, 1, 1},
4527 { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_DED), 0, 1, 1},
4528 { SOC15_REG_ENTRY(GC, 0, mmSPI_EDC_CNT), 0, 4, 1},
4529 { SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT), 0, 4, 6},
4530 { SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_DED_CNT), 0, 4, 16},
4531 { SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_INFO), 0, 4, 16},
4532 { SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_SEC_CNT), 0, 4, 16},
4533 { SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 0, 1, 16},
4534 { SOC15_REG_ENTRY(GC, 0, mmTCP_ATC_EDC_GATCL1_CNT), 0, 4, 16},
4535 { SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT), 0, 4, 16},
4536 { SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW), 0, 4, 16},
4537 { SOC15_REG_ENTRY(GC, 0, mmTD_EDC_CNT), 0, 4, 16},
4538 { SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 0, 4, 6},
4539 { SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT), 0, 4, 16},
4540 { SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT), 0, 4, 16},
4541 { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PHY_CNT), 0, 1, 1},
4542 { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT), 0, 1, 1},
4543 { SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 0, 1, 32},
4544 { SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 0, 1, 32},
4545 { SOC15_REG_ENTRY(GC, 0, mmTCI_EDC_CNT), 0, 1, 72},
4546 { SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2), 0, 1, 16},
4547 { SOC15_REG_ENTRY(GC, 0, mmTCA_EDC_CNT), 0, 1, 2},
4548 { SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 0, 4, 6},
4549 };
4550
gfx_v9_0_do_edc_gds_workarounds(struct amdgpu_device * adev)4551 static int gfx_v9_0_do_edc_gds_workarounds(struct amdgpu_device *adev)
4552 {
4553 struct amdgpu_ring *ring = &adev->gfx.compute_ring[0];
4554 int i, r;
4555
4556 /* only support when RAS is enabled */
4557 if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX))
4558 return 0;
4559
4560 r = amdgpu_ring_alloc(ring, 7);
4561 if (r) {
4562 DRM_ERROR("amdgpu: GDS workarounds failed to lock ring %s (%d).\n",
4563 ring->name, r);
4564 return r;
4565 }
4566
4567 WREG32_SOC15(GC, 0, mmGDS_VMID0_BASE, 0x00000000);
4568 WREG32_SOC15(GC, 0, mmGDS_VMID0_SIZE, adev->gds.gds_size);
4569
4570 amdgpu_ring_write(ring, PACKET3(PACKET3_DMA_DATA, 5));
4571 amdgpu_ring_write(ring, (PACKET3_DMA_DATA_CP_SYNC |
4572 PACKET3_DMA_DATA_DST_SEL(1) |
4573 PACKET3_DMA_DATA_SRC_SEL(2) |
4574 PACKET3_DMA_DATA_ENGINE(0)));
4575 amdgpu_ring_write(ring, 0);
4576 amdgpu_ring_write(ring, 0);
4577 amdgpu_ring_write(ring, 0);
4578 amdgpu_ring_write(ring, 0);
4579 amdgpu_ring_write(ring, PACKET3_DMA_DATA_CMD_RAW_WAIT |
4580 adev->gds.gds_size);
4581
4582 amdgpu_ring_commit(ring);
4583
4584 for (i = 0; i < adev->usec_timeout; i++) {
4585 if (ring->wptr == gfx_v9_0_ring_get_rptr_compute(ring))
4586 break;
4587 udelay(1);
4588 }
4589
4590 if (i >= adev->usec_timeout)
4591 r = -ETIMEDOUT;
4592
4593 WREG32_SOC15(GC, 0, mmGDS_VMID0_SIZE, 0x00000000);
4594
4595 return r;
4596 }
4597
gfx_v9_0_do_edc_gpr_workarounds(struct amdgpu_device * adev)4598 static int gfx_v9_0_do_edc_gpr_workarounds(struct amdgpu_device *adev)
4599 {
4600 struct amdgpu_ring *ring = &adev->gfx.compute_ring[0];
4601 struct amdgpu_ib ib;
4602 struct dma_fence *f = NULL;
4603 int r, i;
4604 unsigned total_size, vgpr_offset, sgpr_offset;
4605 u64 gpu_addr;
4606
4607 int compute_dim_x = adev->gfx.config.max_shader_engines *
4608 adev->gfx.config.max_cu_per_sh *
4609 adev->gfx.config.max_sh_per_se;
4610 int sgpr_work_group_size = 5;
4611 int gpr_reg_size = adev->gfx.config.max_shader_engines + 6;
4612 int vgpr_init_shader_size;
4613 const u32 *vgpr_init_shader_ptr;
4614 const struct soc15_reg_entry *vgpr_init_regs_ptr;
4615
4616 /* only support when RAS is enabled */
4617 if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX))
4618 return 0;
4619
4620 /* bail if the compute ring is not ready */
4621 if (!ring->sched.ready)
4622 return 0;
4623
4624 if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 1)) {
4625 vgpr_init_shader_ptr = vgpr_init_compute_shader_arcturus;
4626 vgpr_init_shader_size = sizeof(vgpr_init_compute_shader_arcturus);
4627 vgpr_init_regs_ptr = vgpr_init_regs_arcturus;
4628 } else {
4629 vgpr_init_shader_ptr = vgpr_init_compute_shader;
4630 vgpr_init_shader_size = sizeof(vgpr_init_compute_shader);
4631 vgpr_init_regs_ptr = vgpr_init_regs;
4632 }
4633
4634 total_size =
4635 (gpr_reg_size * 3 + 4 + 5 + 2) * 4; /* VGPRS */
4636 total_size +=
4637 (gpr_reg_size * 3 + 4 + 5 + 2) * 4; /* SGPRS1 */
4638 total_size +=
4639 (gpr_reg_size * 3 + 4 + 5 + 2) * 4; /* SGPRS2 */
4640 total_size = ALIGN(total_size, 256);
4641 vgpr_offset = total_size;
4642 total_size += ALIGN(vgpr_init_shader_size, 256);
4643 sgpr_offset = total_size;
4644 total_size += sizeof(sgpr_init_compute_shader);
4645
4646 /* allocate an indirect buffer to put the commands in */
4647 memset(&ib, 0, sizeof(ib));
4648 r = amdgpu_ib_get(adev, NULL, total_size,
4649 AMDGPU_IB_POOL_DIRECT, &ib);
4650 if (r) {
4651 DRM_ERROR("amdgpu: failed to get ib (%d).\n", r);
4652 return r;
4653 }
4654
4655 /* load the compute shaders */
4656 for (i = 0; i < vgpr_init_shader_size/sizeof(u32); i++)
4657 ib.ptr[i + (vgpr_offset / 4)] = vgpr_init_shader_ptr[i];
4658
4659 for (i = 0; i < ARRAY_SIZE(sgpr_init_compute_shader); i++)
4660 ib.ptr[i + (sgpr_offset / 4)] = sgpr_init_compute_shader[i];
4661
4662 /* init the ib length to 0 */
4663 ib.length_dw = 0;
4664
4665 /* VGPR */
4666 /* write the register state for the compute dispatch */
4667 for (i = 0; i < gpr_reg_size; i++) {
4668 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1);
4669 ib.ptr[ib.length_dw++] = SOC15_REG_ENTRY_OFFSET(vgpr_init_regs_ptr[i])
4670 - PACKET3_SET_SH_REG_START;
4671 ib.ptr[ib.length_dw++] = vgpr_init_regs_ptr[i].reg_value;
4672 }
4673 /* write the shader start address: mmCOMPUTE_PGM_LO, mmCOMPUTE_PGM_HI */
4674 gpu_addr = (ib.gpu_addr + (u64)vgpr_offset) >> 8;
4675 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2);
4676 ib.ptr[ib.length_dw++] = SOC15_REG_OFFSET(GC, 0, mmCOMPUTE_PGM_LO)
4677 - PACKET3_SET_SH_REG_START;
4678 ib.ptr[ib.length_dw++] = lower_32_bits(gpu_addr);
4679 ib.ptr[ib.length_dw++] = upper_32_bits(gpu_addr);
4680
4681 /* write dispatch packet */
4682 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3);
4683 ib.ptr[ib.length_dw++] = compute_dim_x * 2; /* x */
4684 ib.ptr[ib.length_dw++] = 1; /* y */
4685 ib.ptr[ib.length_dw++] = 1; /* z */
4686 ib.ptr[ib.length_dw++] =
4687 REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, 1);
4688
4689 /* write CS partial flush packet */
4690 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0);
4691 ib.ptr[ib.length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4);
4692
4693 /* SGPR1 */
4694 /* write the register state for the compute dispatch */
4695 for (i = 0; i < gpr_reg_size; i++) {
4696 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1);
4697 ib.ptr[ib.length_dw++] = SOC15_REG_ENTRY_OFFSET(sgpr1_init_regs[i])
4698 - PACKET3_SET_SH_REG_START;
4699 ib.ptr[ib.length_dw++] = sgpr1_init_regs[i].reg_value;
4700 }
4701 /* write the shader start address: mmCOMPUTE_PGM_LO, mmCOMPUTE_PGM_HI */
4702 gpu_addr = (ib.gpu_addr + (u64)sgpr_offset) >> 8;
4703 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2);
4704 ib.ptr[ib.length_dw++] = SOC15_REG_OFFSET(GC, 0, mmCOMPUTE_PGM_LO)
4705 - PACKET3_SET_SH_REG_START;
4706 ib.ptr[ib.length_dw++] = lower_32_bits(gpu_addr);
4707 ib.ptr[ib.length_dw++] = upper_32_bits(gpu_addr);
4708
4709 /* write dispatch packet */
4710 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3);
4711 ib.ptr[ib.length_dw++] = compute_dim_x / 2 * sgpr_work_group_size; /* x */
4712 ib.ptr[ib.length_dw++] = 1; /* y */
4713 ib.ptr[ib.length_dw++] = 1; /* z */
4714 ib.ptr[ib.length_dw++] =
4715 REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, 1);
4716
4717 /* write CS partial flush packet */
4718 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0);
4719 ib.ptr[ib.length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4);
4720
4721 /* SGPR2 */
4722 /* write the register state for the compute dispatch */
4723 for (i = 0; i < gpr_reg_size; i++) {
4724 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1);
4725 ib.ptr[ib.length_dw++] = SOC15_REG_ENTRY_OFFSET(sgpr2_init_regs[i])
4726 - PACKET3_SET_SH_REG_START;
4727 ib.ptr[ib.length_dw++] = sgpr2_init_regs[i].reg_value;
4728 }
4729 /* write the shader start address: mmCOMPUTE_PGM_LO, mmCOMPUTE_PGM_HI */
4730 gpu_addr = (ib.gpu_addr + (u64)sgpr_offset) >> 8;
4731 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2);
4732 ib.ptr[ib.length_dw++] = SOC15_REG_OFFSET(GC, 0, mmCOMPUTE_PGM_LO)
4733 - PACKET3_SET_SH_REG_START;
4734 ib.ptr[ib.length_dw++] = lower_32_bits(gpu_addr);
4735 ib.ptr[ib.length_dw++] = upper_32_bits(gpu_addr);
4736
4737 /* write dispatch packet */
4738 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3);
4739 ib.ptr[ib.length_dw++] = compute_dim_x / 2 * sgpr_work_group_size; /* x */
4740 ib.ptr[ib.length_dw++] = 1; /* y */
4741 ib.ptr[ib.length_dw++] = 1; /* z */
4742 ib.ptr[ib.length_dw++] =
4743 REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, 1);
4744
4745 /* write CS partial flush packet */
4746 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0);
4747 ib.ptr[ib.length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4);
4748
4749 /* shedule the ib on the ring */
4750 r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f);
4751 if (r) {
4752 DRM_ERROR("amdgpu: ib submit failed (%d).\n", r);
4753 goto fail;
4754 }
4755
4756 /* wait for the GPU to finish processing the IB */
4757 r = dma_fence_wait(f, false);
4758 if (r) {
4759 DRM_ERROR("amdgpu: fence wait failed (%d).\n", r);
4760 goto fail;
4761 }
4762
4763 fail:
4764 amdgpu_ib_free(&ib, NULL);
4765 dma_fence_put(f);
4766
4767 return r;
4768 }
4769
gfx_v9_0_early_init(struct amdgpu_ip_block * ip_block)4770 static int gfx_v9_0_early_init(struct amdgpu_ip_block *ip_block)
4771 {
4772 struct amdgpu_device *adev = ip_block->adev;
4773
4774 adev->gfx.funcs = &gfx_v9_0_gfx_funcs;
4775
4776 if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 1) ||
4777 amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 2))
4778 adev->gfx.num_gfx_rings = 0;
4779 else
4780 adev->gfx.num_gfx_rings = GFX9_NUM_GFX_RINGS;
4781 adev->gfx.xcc_mask = 1;
4782 adev->gfx.num_compute_rings = min(amdgpu_gfx_get_num_kcq(adev),
4783 AMDGPU_MAX_COMPUTE_RINGS);
4784 gfx_v9_0_set_kiq_pm4_funcs(adev);
4785 gfx_v9_0_set_ring_funcs(adev);
4786 gfx_v9_0_set_irq_funcs(adev);
4787 gfx_v9_0_set_gds_init(adev);
4788 gfx_v9_0_set_rlc_funcs(adev);
4789
4790 /* init rlcg reg access ctrl */
4791 gfx_v9_0_init_rlcg_reg_access_ctrl(adev);
4792
4793 return gfx_v9_0_init_microcode(adev);
4794 }
4795
gfx_v9_0_ecc_late_init(struct amdgpu_ip_block * ip_block)4796 static int gfx_v9_0_ecc_late_init(struct amdgpu_ip_block *ip_block)
4797 {
4798 struct amdgpu_device *adev = ip_block->adev;
4799 int r;
4800
4801 /*
4802 * Temp workaround to fix the issue that CP firmware fails to
4803 * update read pointer when CPDMA is writing clearing operation
4804 * to GDS in suspend/resume sequence on several cards. So just
4805 * limit this operation in cold boot sequence.
4806 */
4807 if ((!adev->in_suspend) &&
4808 (adev->gds.gds_size)) {
4809 r = gfx_v9_0_do_edc_gds_workarounds(adev);
4810 if (r)
4811 return r;
4812 }
4813
4814 /* requires IBs so do in late init after IB pool is initialized */
4815 if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 2))
4816 r = gfx_v9_4_2_do_edc_gpr_workarounds(adev);
4817 else
4818 r = gfx_v9_0_do_edc_gpr_workarounds(adev);
4819
4820 if (r)
4821 return r;
4822
4823 if (adev->gfx.ras &&
4824 adev->gfx.ras->enable_watchdog_timer)
4825 adev->gfx.ras->enable_watchdog_timer(adev);
4826
4827 return 0;
4828 }
4829
gfx_v9_0_late_init(struct amdgpu_ip_block * ip_block)4830 static int gfx_v9_0_late_init(struct amdgpu_ip_block *ip_block)
4831 {
4832 struct amdgpu_device *adev = ip_block->adev;
4833 int r;
4834
4835 r = amdgpu_irq_get(adev, &adev->gfx.priv_reg_irq, 0);
4836 if (r)
4837 return r;
4838
4839 r = amdgpu_irq_get(adev, &adev->gfx.priv_inst_irq, 0);
4840 if (r)
4841 return r;
4842
4843 r = amdgpu_irq_get(adev, &adev->gfx.bad_op_irq, 0);
4844 if (r)
4845 return r;
4846
4847 r = gfx_v9_0_ecc_late_init(ip_block);
4848 if (r)
4849 return r;
4850
4851 if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 2))
4852 gfx_v9_4_2_debug_trap_config_init(adev,
4853 adev->vm_manager.first_kfd_vmid, AMDGPU_NUM_VMID);
4854 else
4855 gfx_v9_0_debug_trap_config_init(adev,
4856 adev->vm_manager.first_kfd_vmid, AMDGPU_NUM_VMID);
4857
4858 return 0;
4859 }
4860
gfx_v9_0_is_rlc_enabled(struct amdgpu_device * adev)4861 static bool gfx_v9_0_is_rlc_enabled(struct amdgpu_device *adev)
4862 {
4863 uint32_t rlc_setting;
4864
4865 /* if RLC is not enabled, do nothing */
4866 rlc_setting = RREG32_SOC15(GC, 0, mmRLC_CNTL);
4867 if (!(rlc_setting & RLC_CNTL__RLC_ENABLE_F32_MASK))
4868 return false;
4869
4870 return true;
4871 }
4872
gfx_v9_0_set_safe_mode(struct amdgpu_device * adev,int xcc_id)4873 static void gfx_v9_0_set_safe_mode(struct amdgpu_device *adev, int xcc_id)
4874 {
4875 uint32_t data;
4876 unsigned i;
4877
4878 data = RLC_SAFE_MODE__CMD_MASK;
4879 data |= (1 << RLC_SAFE_MODE__MESSAGE__SHIFT);
4880 WREG32_SOC15(GC, 0, mmRLC_SAFE_MODE, data);
4881
4882 /* wait for RLC_SAFE_MODE */
4883 for (i = 0; i < adev->usec_timeout; i++) {
4884 if (!REG_GET_FIELD(RREG32_SOC15(GC, 0, mmRLC_SAFE_MODE), RLC_SAFE_MODE, CMD))
4885 break;
4886 udelay(1);
4887 }
4888 }
4889
gfx_v9_0_unset_safe_mode(struct amdgpu_device * adev,int xcc_id)4890 static void gfx_v9_0_unset_safe_mode(struct amdgpu_device *adev, int xcc_id)
4891 {
4892 uint32_t data;
4893
4894 data = RLC_SAFE_MODE__CMD_MASK;
4895 WREG32_SOC15(GC, 0, mmRLC_SAFE_MODE, data);
4896 }
4897
gfx_v9_0_update_gfx_cg_power_gating(struct amdgpu_device * adev,bool enable)4898 static void gfx_v9_0_update_gfx_cg_power_gating(struct amdgpu_device *adev,
4899 bool enable)
4900 {
4901 amdgpu_gfx_rlc_enter_safe_mode(adev, 0);
4902
4903 if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_PG) && enable) {
4904 gfx_v9_0_enable_gfx_cg_power_gating(adev, true);
4905 if (adev->pg_flags & AMD_PG_SUPPORT_GFX_PIPELINE)
4906 gfx_v9_0_enable_gfx_pipeline_powergating(adev, true);
4907 } else {
4908 gfx_v9_0_enable_gfx_cg_power_gating(adev, false);
4909 if (adev->pg_flags & AMD_PG_SUPPORT_GFX_PIPELINE)
4910 gfx_v9_0_enable_gfx_pipeline_powergating(adev, false);
4911 }
4912
4913 amdgpu_gfx_rlc_exit_safe_mode(adev, 0);
4914 }
4915
gfx_v9_0_update_gfx_mg_power_gating(struct amdgpu_device * adev,bool enable)4916 static void gfx_v9_0_update_gfx_mg_power_gating(struct amdgpu_device *adev,
4917 bool enable)
4918 {
4919 /* TODO: double check if we need to perform under safe mode */
4920 /* gfx_v9_0_enter_rlc_safe_mode(adev); */
4921
4922 if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_SMG) && enable)
4923 gfx_v9_0_enable_gfx_static_mg_power_gating(adev, true);
4924 else
4925 gfx_v9_0_enable_gfx_static_mg_power_gating(adev, false);
4926
4927 if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_DMG) && enable)
4928 gfx_v9_0_enable_gfx_dynamic_mg_power_gating(adev, true);
4929 else
4930 gfx_v9_0_enable_gfx_dynamic_mg_power_gating(adev, false);
4931
4932 /* gfx_v9_0_exit_rlc_safe_mode(adev); */
4933 }
4934
gfx_v9_0_update_medium_grain_clock_gating(struct amdgpu_device * adev,bool enable)4935 static void gfx_v9_0_update_medium_grain_clock_gating(struct amdgpu_device *adev,
4936 bool enable)
4937 {
4938 uint32_t data, def;
4939
4940 /* It is disabled by HW by default */
4941 if (enable && (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGCG)) {
4942 /* 1 - RLC_CGTT_MGCG_OVERRIDE */
4943 def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE);
4944
4945 if (amdgpu_ip_version(adev, GC_HWIP, 0) != IP_VERSION(9, 2, 1))
4946 data &= ~RLC_CGTT_MGCG_OVERRIDE__CPF_CGTT_SCLK_OVERRIDE_MASK;
4947
4948 data &= ~(RLC_CGTT_MGCG_OVERRIDE__GRBM_CGTT_SCLK_OVERRIDE_MASK |
4949 RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK |
4950 RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGLS_OVERRIDE_MASK);
4951
4952 /* only for Vega10 & Raven1 */
4953 data |= RLC_CGTT_MGCG_OVERRIDE__RLC_CGTT_SCLK_OVERRIDE_MASK;
4954
4955 if (def != data)
4956 WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data);
4957
4958 /* MGLS is a global flag to control all MGLS in GFX */
4959 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGLS) {
4960 /* 2 - RLC memory Light sleep */
4961 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_RLC_LS) {
4962 def = data = RREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL);
4963 data |= RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK;
4964 if (def != data)
4965 WREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL, data);
4966 }
4967 /* 3 - CP memory Light sleep */
4968 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CP_LS) {
4969 def = data = RREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL);
4970 data |= CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK;
4971 if (def != data)
4972 WREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL, data);
4973 }
4974 }
4975 } else {
4976 /* 1 - MGCG_OVERRIDE */
4977 def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE);
4978
4979 if (amdgpu_ip_version(adev, GC_HWIP, 0) != IP_VERSION(9, 2, 1))
4980 data |= RLC_CGTT_MGCG_OVERRIDE__CPF_CGTT_SCLK_OVERRIDE_MASK;
4981
4982 data |= (RLC_CGTT_MGCG_OVERRIDE__RLC_CGTT_SCLK_OVERRIDE_MASK |
4983 RLC_CGTT_MGCG_OVERRIDE__GRBM_CGTT_SCLK_OVERRIDE_MASK |
4984 RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK |
4985 RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGLS_OVERRIDE_MASK);
4986
4987 if (def != data)
4988 WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data);
4989
4990 /* 2 - disable MGLS in RLC */
4991 data = RREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL);
4992 if (data & RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK) {
4993 data &= ~RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK;
4994 WREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL, data);
4995 }
4996
4997 /* 3 - disable MGLS in CP */
4998 data = RREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL);
4999 if (data & CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK) {
5000 data &= ~CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK;
5001 WREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL, data);
5002 }
5003 }
5004 }
5005
gfx_v9_0_update_3d_clock_gating(struct amdgpu_device * adev,bool enable)5006 static void gfx_v9_0_update_3d_clock_gating(struct amdgpu_device *adev,
5007 bool enable)
5008 {
5009 uint32_t data, def;
5010
5011 if (!adev->gfx.num_gfx_rings)
5012 return;
5013
5014 /* Enable 3D CGCG/CGLS */
5015 if (enable) {
5016 /* write cmd to clear cgcg/cgls ov */
5017 def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE);
5018 /* unset CGCG override */
5019 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_GFX3D_CG_OVERRIDE_MASK;
5020 /* update CGCG and CGLS override bits */
5021 if (def != data)
5022 WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data);
5023
5024 /* enable 3Dcgcg FSM(0x0000363f) */
5025 def = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D);
5026
5027 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGCG)
5028 data = (0x36 << RLC_CGCG_CGLS_CTRL_3D__CGCG_GFX_IDLE_THRESHOLD__SHIFT) |
5029 RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK;
5030 else
5031 data = 0x0 << RLC_CGCG_CGLS_CTRL_3D__CGCG_GFX_IDLE_THRESHOLD__SHIFT;
5032
5033 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGLS)
5034 data |= (0x000F << RLC_CGCG_CGLS_CTRL_3D__CGLS_REP_COMPANSAT_DELAY__SHIFT) |
5035 RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK;
5036 if (def != data)
5037 WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D, data);
5038
5039 /* set IDLE_POLL_COUNT(0x00900100) */
5040 def = RREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL);
5041 data = (0x0100 << CP_RB_WPTR_POLL_CNTL__POLL_FREQUENCY__SHIFT) |
5042 (0x0090 << CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT);
5043 if (def != data)
5044 WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL, data);
5045 } else {
5046 /* Disable CGCG/CGLS */
5047 def = data = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D);
5048 /* disable cgcg, cgls should be disabled */
5049 data &= ~(RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK |
5050 RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK);
5051 /* disable cgcg and cgls in FSM */
5052 if (def != data)
5053 WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D, data);
5054 }
5055 }
5056
gfx_v9_0_update_coarse_grain_clock_gating(struct amdgpu_device * adev,bool enable)5057 static void gfx_v9_0_update_coarse_grain_clock_gating(struct amdgpu_device *adev,
5058 bool enable)
5059 {
5060 uint32_t def, data;
5061
5062 if (enable && (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGCG)) {
5063 def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE);
5064 /* unset CGCG override */
5065 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGCG_OVERRIDE_MASK;
5066 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS)
5067 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGLS_OVERRIDE_MASK;
5068 else
5069 data |= RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGLS_OVERRIDE_MASK;
5070 /* update CGCG and CGLS override bits */
5071 if (def != data)
5072 WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data);
5073
5074 /* enable cgcg FSM(0x0000363F) */
5075 def = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL);
5076
5077 if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 1))
5078 data = (0x2000 << RLC_CGCG_CGLS_CTRL__CGCG_GFX_IDLE_THRESHOLD__SHIFT) |
5079 RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK;
5080 else
5081 data = (0x36 << RLC_CGCG_CGLS_CTRL__CGCG_GFX_IDLE_THRESHOLD__SHIFT) |
5082 RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK;
5083 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS)
5084 data |= (0x000F << RLC_CGCG_CGLS_CTRL__CGLS_REP_COMPANSAT_DELAY__SHIFT) |
5085 RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK;
5086 if (def != data)
5087 WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL, data);
5088
5089 /* set IDLE_POLL_COUNT(0x00900100) */
5090 def = RREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL);
5091 data = (0x0100 << CP_RB_WPTR_POLL_CNTL__POLL_FREQUENCY__SHIFT) |
5092 (0x0090 << CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT);
5093 if (def != data)
5094 WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL, data);
5095 } else {
5096 def = data = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL);
5097 /* reset CGCG/CGLS bits */
5098 data &= ~(RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK | RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK);
5099 /* disable cgcg and cgls in FSM */
5100 if (def != data)
5101 WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL, data);
5102 }
5103 }
5104
gfx_v9_0_update_gfx_clock_gating(struct amdgpu_device * adev,bool enable)5105 static int gfx_v9_0_update_gfx_clock_gating(struct amdgpu_device *adev,
5106 bool enable)
5107 {
5108 amdgpu_gfx_rlc_enter_safe_mode(adev, 0);
5109 if (enable) {
5110 /* CGCG/CGLS should be enabled after MGCG/MGLS
5111 * === MGCG + MGLS ===
5112 */
5113 gfx_v9_0_update_medium_grain_clock_gating(adev, enable);
5114 /* === CGCG /CGLS for GFX 3D Only === */
5115 gfx_v9_0_update_3d_clock_gating(adev, enable);
5116 /* === CGCG + CGLS === */
5117 gfx_v9_0_update_coarse_grain_clock_gating(adev, enable);
5118 } else {
5119 /* CGCG/CGLS should be disabled before MGCG/MGLS
5120 * === CGCG + CGLS ===
5121 */
5122 gfx_v9_0_update_coarse_grain_clock_gating(adev, enable);
5123 /* === CGCG /CGLS for GFX 3D Only === */
5124 gfx_v9_0_update_3d_clock_gating(adev, enable);
5125 /* === MGCG + MGLS === */
5126 gfx_v9_0_update_medium_grain_clock_gating(adev, enable);
5127 }
5128 amdgpu_gfx_rlc_exit_safe_mode(adev, 0);
5129 return 0;
5130 }
5131
gfx_v9_0_update_spm_vmid_internal(struct amdgpu_device * adev,unsigned int vmid)5132 static void gfx_v9_0_update_spm_vmid_internal(struct amdgpu_device *adev,
5133 unsigned int vmid)
5134 {
5135 u32 reg, data;
5136
5137 reg = SOC15_REG_OFFSET(GC, 0, mmRLC_SPM_MC_CNTL);
5138 if (amdgpu_sriov_is_pp_one_vf(adev))
5139 data = RREG32_NO_KIQ(reg);
5140 else
5141 data = RREG32_SOC15(GC, 0, mmRLC_SPM_MC_CNTL);
5142
5143 data &= ~RLC_SPM_MC_CNTL__RLC_SPM_VMID_MASK;
5144 data |= (vmid & RLC_SPM_MC_CNTL__RLC_SPM_VMID_MASK) << RLC_SPM_MC_CNTL__RLC_SPM_VMID__SHIFT;
5145
5146 if (amdgpu_sriov_is_pp_one_vf(adev))
5147 WREG32_SOC15_NO_KIQ(GC, 0, mmRLC_SPM_MC_CNTL, data);
5148 else
5149 WREG32_SOC15(GC, 0, mmRLC_SPM_MC_CNTL, data);
5150 }
5151
gfx_v9_0_update_spm_vmid(struct amdgpu_device * adev,struct amdgpu_ring * ring,unsigned int vmid)5152 static void gfx_v9_0_update_spm_vmid(struct amdgpu_device *adev, struct amdgpu_ring *ring, unsigned int vmid)
5153 {
5154 amdgpu_gfx_off_ctrl(adev, false);
5155
5156 gfx_v9_0_update_spm_vmid_internal(adev, vmid);
5157
5158 amdgpu_gfx_off_ctrl(adev, true);
5159 }
5160
gfx_v9_0_check_rlcg_range(struct amdgpu_device * adev,uint32_t offset,struct soc15_reg_rlcg * entries,int arr_size)5161 static bool gfx_v9_0_check_rlcg_range(struct amdgpu_device *adev,
5162 uint32_t offset,
5163 struct soc15_reg_rlcg *entries, int arr_size)
5164 {
5165 int i;
5166 uint32_t reg;
5167
5168 if (!entries)
5169 return false;
5170
5171 for (i = 0; i < arr_size; i++) {
5172 const struct soc15_reg_rlcg *entry;
5173
5174 entry = &entries[i];
5175 reg = adev->reg_offset[entry->hwip][entry->instance][entry->segment] + entry->reg;
5176 if (offset == reg)
5177 return true;
5178 }
5179
5180 return false;
5181 }
5182
gfx_v9_0_is_rlcg_access_range(struct amdgpu_device * adev,u32 offset)5183 static bool gfx_v9_0_is_rlcg_access_range(struct amdgpu_device *adev, u32 offset)
5184 {
5185 return gfx_v9_0_check_rlcg_range(adev, offset,
5186 (void *)rlcg_access_gc_9_0,
5187 ARRAY_SIZE(rlcg_access_gc_9_0));
5188 }
5189
5190 static const struct amdgpu_rlc_funcs gfx_v9_0_rlc_funcs = {
5191 .is_rlc_enabled = gfx_v9_0_is_rlc_enabled,
5192 .set_safe_mode = gfx_v9_0_set_safe_mode,
5193 .unset_safe_mode = gfx_v9_0_unset_safe_mode,
5194 .init = gfx_v9_0_rlc_init,
5195 .get_csb_size = gfx_v9_0_get_csb_size,
5196 .get_csb_buffer = gfx_v9_0_get_csb_buffer,
5197 .get_cp_table_num = gfx_v9_0_cp_jump_table_num,
5198 .resume = gfx_v9_0_rlc_resume,
5199 .stop = gfx_v9_0_rlc_stop,
5200 .reset = gfx_v9_0_rlc_reset,
5201 .start = gfx_v9_0_rlc_start,
5202 .update_spm_vmid = gfx_v9_0_update_spm_vmid,
5203 .is_rlcg_access_range = gfx_v9_0_is_rlcg_access_range,
5204 };
5205
gfx_v9_0_set_powergating_state(struct amdgpu_ip_block * ip_block,enum amd_powergating_state state)5206 static int gfx_v9_0_set_powergating_state(struct amdgpu_ip_block *ip_block,
5207 enum amd_powergating_state state)
5208 {
5209 struct amdgpu_device *adev = ip_block->adev;
5210 bool enable = (state == AMD_PG_STATE_GATE);
5211
5212 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
5213 case IP_VERSION(9, 2, 2):
5214 case IP_VERSION(9, 1, 0):
5215 case IP_VERSION(9, 3, 0):
5216 if (!enable)
5217 amdgpu_gfx_off_ctrl_immediate(adev, false);
5218
5219 if (adev->pg_flags & AMD_PG_SUPPORT_RLC_SMU_HS) {
5220 gfx_v9_0_enable_sck_slow_down_on_power_up(adev, true);
5221 gfx_v9_0_enable_sck_slow_down_on_power_down(adev, true);
5222 } else {
5223 gfx_v9_0_enable_sck_slow_down_on_power_up(adev, false);
5224 gfx_v9_0_enable_sck_slow_down_on_power_down(adev, false);
5225 }
5226
5227 if (adev->pg_flags & AMD_PG_SUPPORT_CP)
5228 gfx_v9_0_enable_cp_power_gating(adev, true);
5229 else
5230 gfx_v9_0_enable_cp_power_gating(adev, false);
5231
5232 /* update gfx cgpg state */
5233 gfx_v9_0_update_gfx_cg_power_gating(adev, enable);
5234
5235 /* update mgcg state */
5236 gfx_v9_0_update_gfx_mg_power_gating(adev, enable);
5237
5238 if (enable)
5239 amdgpu_gfx_off_ctrl_immediate(adev, true);
5240 break;
5241 case IP_VERSION(9, 2, 1):
5242 amdgpu_gfx_off_ctrl_immediate(adev, enable);
5243 break;
5244 default:
5245 break;
5246 }
5247
5248 return 0;
5249 }
5250
gfx_v9_0_set_clockgating_state(struct amdgpu_ip_block * ip_block,enum amd_clockgating_state state)5251 static int gfx_v9_0_set_clockgating_state(struct amdgpu_ip_block *ip_block,
5252 enum amd_clockgating_state state)
5253 {
5254 struct amdgpu_device *adev = ip_block->adev;
5255
5256 if (amdgpu_sriov_vf(adev))
5257 return 0;
5258
5259 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
5260 case IP_VERSION(9, 0, 1):
5261 case IP_VERSION(9, 2, 1):
5262 case IP_VERSION(9, 4, 0):
5263 case IP_VERSION(9, 2, 2):
5264 case IP_VERSION(9, 1, 0):
5265 case IP_VERSION(9, 4, 1):
5266 case IP_VERSION(9, 3, 0):
5267 case IP_VERSION(9, 4, 2):
5268 gfx_v9_0_update_gfx_clock_gating(adev,
5269 state == AMD_CG_STATE_GATE);
5270 break;
5271 default:
5272 break;
5273 }
5274 return 0;
5275 }
5276
gfx_v9_0_get_clockgating_state(struct amdgpu_ip_block * ip_block,u64 * flags)5277 static void gfx_v9_0_get_clockgating_state(struct amdgpu_ip_block *ip_block, u64 *flags)
5278 {
5279 struct amdgpu_device *adev = ip_block->adev;
5280 int data;
5281
5282 if (amdgpu_sriov_vf(adev))
5283 *flags = 0;
5284
5285 /* AMD_CG_SUPPORT_GFX_MGCG */
5286 data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE));
5287 if (!(data & RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK))
5288 *flags |= AMD_CG_SUPPORT_GFX_MGCG;
5289
5290 /* AMD_CG_SUPPORT_GFX_CGCG */
5291 data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmRLC_CGCG_CGLS_CTRL));
5292 if (data & RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK)
5293 *flags |= AMD_CG_SUPPORT_GFX_CGCG;
5294
5295 /* AMD_CG_SUPPORT_GFX_CGLS */
5296 if (data & RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK)
5297 *flags |= AMD_CG_SUPPORT_GFX_CGLS;
5298
5299 /* AMD_CG_SUPPORT_GFX_RLC_LS */
5300 data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmRLC_MEM_SLP_CNTL));
5301 if (data & RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK)
5302 *flags |= AMD_CG_SUPPORT_GFX_RLC_LS | AMD_CG_SUPPORT_GFX_MGLS;
5303
5304 /* AMD_CG_SUPPORT_GFX_CP_LS */
5305 data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmCP_MEM_SLP_CNTL));
5306 if (data & CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK)
5307 *flags |= AMD_CG_SUPPORT_GFX_CP_LS | AMD_CG_SUPPORT_GFX_MGLS;
5308
5309 if (amdgpu_ip_version(adev, GC_HWIP, 0) != IP_VERSION(9, 4, 1)) {
5310 /* AMD_CG_SUPPORT_GFX_3D_CGCG */
5311 data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D));
5312 if (data & RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK)
5313 *flags |= AMD_CG_SUPPORT_GFX_3D_CGCG;
5314
5315 /* AMD_CG_SUPPORT_GFX_3D_CGLS */
5316 if (data & RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK)
5317 *flags |= AMD_CG_SUPPORT_GFX_3D_CGLS;
5318 }
5319 }
5320
gfx_v9_0_ring_get_rptr_gfx(struct amdgpu_ring * ring)5321 static u64 gfx_v9_0_ring_get_rptr_gfx(struct amdgpu_ring *ring)
5322 {
5323 return *ring->rptr_cpu_addr; /* gfx9 is 32bit rptr*/
5324 }
5325
gfx_v9_0_ring_get_wptr_gfx(struct amdgpu_ring * ring)5326 static u64 gfx_v9_0_ring_get_wptr_gfx(struct amdgpu_ring *ring)
5327 {
5328 struct amdgpu_device *adev = ring->adev;
5329 u64 wptr;
5330
5331 /* XXX check if swapping is necessary on BE */
5332 if (ring->use_doorbell) {
5333 wptr = atomic64_read((atomic64_t *)ring->wptr_cpu_addr);
5334 } else {
5335 wptr = RREG32_SOC15(GC, 0, mmCP_RB0_WPTR);
5336 wptr += (u64)RREG32_SOC15(GC, 0, mmCP_RB0_WPTR_HI) << 32;
5337 }
5338
5339 return wptr;
5340 }
5341
gfx_v9_0_ring_set_wptr_gfx(struct amdgpu_ring * ring)5342 static void gfx_v9_0_ring_set_wptr_gfx(struct amdgpu_ring *ring)
5343 {
5344 struct amdgpu_device *adev = ring->adev;
5345
5346 if (ring->use_doorbell) {
5347 /* XXX check if swapping is necessary on BE */
5348 atomic64_set((atomic64_t *)ring->wptr_cpu_addr, ring->wptr);
5349 WDOORBELL64(ring->doorbell_index, ring->wptr);
5350 } else {
5351 WREG32_SOC15(GC, 0, mmCP_RB0_WPTR, lower_32_bits(ring->wptr));
5352 WREG32_SOC15(GC, 0, mmCP_RB0_WPTR_HI, upper_32_bits(ring->wptr));
5353 }
5354 }
5355
gfx_v9_0_ring_emit_hdp_flush(struct amdgpu_ring * ring)5356 static void gfx_v9_0_ring_emit_hdp_flush(struct amdgpu_ring *ring)
5357 {
5358 struct amdgpu_device *adev = ring->adev;
5359 u32 ref_and_mask, reg_mem_engine;
5360 const struct nbio_hdp_flush_reg *nbio_hf_reg = adev->nbio.hdp_flush_reg;
5361
5362 if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) {
5363 switch (ring->me) {
5364 case 1:
5365 ref_and_mask = nbio_hf_reg->ref_and_mask_cp2 << ring->pipe;
5366 break;
5367 case 2:
5368 ref_and_mask = nbio_hf_reg->ref_and_mask_cp6 << ring->pipe;
5369 break;
5370 default:
5371 return;
5372 }
5373 reg_mem_engine = 0;
5374 } else {
5375 ref_and_mask = nbio_hf_reg->ref_and_mask_cp0;
5376 reg_mem_engine = 1; /* pfp */
5377 }
5378
5379 gfx_v9_0_wait_reg_mem(ring, reg_mem_engine, 0, 1,
5380 adev->nbio.funcs->get_hdp_flush_req_offset(adev),
5381 adev->nbio.funcs->get_hdp_flush_done_offset(adev),
5382 ref_and_mask, ref_and_mask, 0x20);
5383 }
5384
gfx_v9_0_ring_emit_ib_gfx(struct amdgpu_ring * ring,struct amdgpu_job * job,struct amdgpu_ib * ib,uint32_t flags)5385 static void gfx_v9_0_ring_emit_ib_gfx(struct amdgpu_ring *ring,
5386 struct amdgpu_job *job,
5387 struct amdgpu_ib *ib,
5388 uint32_t flags)
5389 {
5390 unsigned vmid = AMDGPU_JOB_GET_VMID(job);
5391 u32 header, control = 0;
5392
5393 if (ib->flags & AMDGPU_IB_FLAG_CE)
5394 header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2);
5395 else
5396 header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
5397
5398 control |= ib->length_dw | (vmid << 24);
5399
5400 if (ib->flags & AMDGPU_IB_FLAG_PREEMPT) {
5401 control |= INDIRECT_BUFFER_PRE_ENB(1);
5402
5403 if (flags & AMDGPU_IB_PREEMPTED)
5404 control |= INDIRECT_BUFFER_PRE_RESUME(1);
5405
5406 if (!(ib->flags & AMDGPU_IB_FLAG_CE) && vmid)
5407 gfx_v9_0_ring_emit_de_meta(ring,
5408 (!amdgpu_sriov_vf(ring->adev) &&
5409 flags & AMDGPU_IB_PREEMPTED) ?
5410 true : false,
5411 job->gds_size > 0 && job->gds_base != 0);
5412 }
5413
5414 amdgpu_ring_write(ring, header);
5415 BUG_ON(ib->gpu_addr & 0x3); /* Dword align */
5416 amdgpu_ring_write(ring,
5417 #ifdef __BIG_ENDIAN
5418 (2 << 0) |
5419 #endif
5420 lower_32_bits(ib->gpu_addr));
5421 amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr));
5422 amdgpu_ring_ib_on_emit_cntl(ring);
5423 amdgpu_ring_write(ring, control);
5424 }
5425
gfx_v9_0_ring_patch_cntl(struct amdgpu_ring * ring,unsigned offset)5426 static void gfx_v9_0_ring_patch_cntl(struct amdgpu_ring *ring,
5427 unsigned offset)
5428 {
5429 u32 control = ring->ring[offset];
5430
5431 control |= INDIRECT_BUFFER_PRE_RESUME(1);
5432 ring->ring[offset] = control;
5433 }
5434
gfx_v9_0_ring_patch_ce_meta(struct amdgpu_ring * ring,unsigned offset)5435 static void gfx_v9_0_ring_patch_ce_meta(struct amdgpu_ring *ring,
5436 unsigned offset)
5437 {
5438 struct amdgpu_device *adev = ring->adev;
5439 void *ce_payload_cpu_addr;
5440 uint64_t payload_offset, payload_size;
5441
5442 payload_size = sizeof(struct v9_ce_ib_state);
5443
5444 if (ring->is_mes_queue) {
5445 payload_offset = offsetof(struct amdgpu_mes_ctx_meta_data,
5446 gfx[0].gfx_meta_data) +
5447 offsetof(struct v9_gfx_meta_data, ce_payload);
5448 ce_payload_cpu_addr =
5449 amdgpu_mes_ctx_get_offs_cpu_addr(ring, payload_offset);
5450 } else {
5451 payload_offset = offsetof(struct v9_gfx_meta_data, ce_payload);
5452 ce_payload_cpu_addr = adev->virt.csa_cpu_addr + payload_offset;
5453 }
5454
5455 if (offset + (payload_size >> 2) <= ring->buf_mask + 1) {
5456 memcpy((void *)&ring->ring[offset], ce_payload_cpu_addr, payload_size);
5457 } else {
5458 memcpy((void *)&ring->ring[offset], ce_payload_cpu_addr,
5459 (ring->buf_mask + 1 - offset) << 2);
5460 payload_size -= (ring->buf_mask + 1 - offset) << 2;
5461 memcpy((void *)&ring->ring[0],
5462 ce_payload_cpu_addr + ((ring->buf_mask + 1 - offset) << 2),
5463 payload_size);
5464 }
5465 }
5466
gfx_v9_0_ring_patch_de_meta(struct amdgpu_ring * ring,unsigned offset)5467 static void gfx_v9_0_ring_patch_de_meta(struct amdgpu_ring *ring,
5468 unsigned offset)
5469 {
5470 struct amdgpu_device *adev = ring->adev;
5471 void *de_payload_cpu_addr;
5472 uint64_t payload_offset, payload_size;
5473
5474 payload_size = sizeof(struct v9_de_ib_state);
5475
5476 if (ring->is_mes_queue) {
5477 payload_offset = offsetof(struct amdgpu_mes_ctx_meta_data,
5478 gfx[0].gfx_meta_data) +
5479 offsetof(struct v9_gfx_meta_data, de_payload);
5480 de_payload_cpu_addr =
5481 amdgpu_mes_ctx_get_offs_cpu_addr(ring, payload_offset);
5482 } else {
5483 payload_offset = offsetof(struct v9_gfx_meta_data, de_payload);
5484 de_payload_cpu_addr = adev->virt.csa_cpu_addr + payload_offset;
5485 }
5486
5487 ((struct v9_de_ib_state *)de_payload_cpu_addr)->ib_completion_status =
5488 IB_COMPLETION_STATUS_PREEMPTED;
5489
5490 if (offset + (payload_size >> 2) <= ring->buf_mask + 1) {
5491 memcpy((void *)&ring->ring[offset], de_payload_cpu_addr, payload_size);
5492 } else {
5493 memcpy((void *)&ring->ring[offset], de_payload_cpu_addr,
5494 (ring->buf_mask + 1 - offset) << 2);
5495 payload_size -= (ring->buf_mask + 1 - offset) << 2;
5496 memcpy((void *)&ring->ring[0],
5497 de_payload_cpu_addr + ((ring->buf_mask + 1 - offset) << 2),
5498 payload_size);
5499 }
5500 }
5501
gfx_v9_0_ring_emit_ib_compute(struct amdgpu_ring * ring,struct amdgpu_job * job,struct amdgpu_ib * ib,uint32_t flags)5502 static void gfx_v9_0_ring_emit_ib_compute(struct amdgpu_ring *ring,
5503 struct amdgpu_job *job,
5504 struct amdgpu_ib *ib,
5505 uint32_t flags)
5506 {
5507 unsigned vmid = AMDGPU_JOB_GET_VMID(job);
5508 u32 control = INDIRECT_BUFFER_VALID | ib->length_dw | (vmid << 24);
5509
5510 /* Currently, there is a high possibility to get wave ID mismatch
5511 * between ME and GDS, leading to a hw deadlock, because ME generates
5512 * different wave IDs than the GDS expects. This situation happens
5513 * randomly when at least 5 compute pipes use GDS ordered append.
5514 * The wave IDs generated by ME are also wrong after suspend/resume.
5515 * Those are probably bugs somewhere else in the kernel driver.
5516 *
5517 * Writing GDS_COMPUTE_MAX_WAVE_ID resets wave ID counters in ME and
5518 * GDS to 0 for this ring (me/pipe).
5519 */
5520 if (ib->flags & AMDGPU_IB_FLAG_RESET_GDS_MAX_WAVE_ID) {
5521 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
5522 amdgpu_ring_write(ring, mmGDS_COMPUTE_MAX_WAVE_ID);
5523 amdgpu_ring_write(ring, ring->adev->gds.gds_compute_max_wave_id);
5524 }
5525
5526 amdgpu_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
5527 BUG_ON(ib->gpu_addr & 0x3); /* Dword align */
5528 amdgpu_ring_write(ring,
5529 #ifdef __BIG_ENDIAN
5530 (2 << 0) |
5531 #endif
5532 lower_32_bits(ib->gpu_addr));
5533 amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr));
5534 amdgpu_ring_write(ring, control);
5535 }
5536
gfx_v9_0_ring_emit_fence(struct amdgpu_ring * ring,u64 addr,u64 seq,unsigned flags)5537 static void gfx_v9_0_ring_emit_fence(struct amdgpu_ring *ring, u64 addr,
5538 u64 seq, unsigned flags)
5539 {
5540 bool write64bit = flags & AMDGPU_FENCE_FLAG_64BIT;
5541 bool int_sel = flags & AMDGPU_FENCE_FLAG_INT;
5542 bool writeback = flags & AMDGPU_FENCE_FLAG_TC_WB_ONLY;
5543 bool exec = flags & AMDGPU_FENCE_FLAG_EXEC;
5544 uint32_t dw2 = 0;
5545
5546 /* RELEASE_MEM - flush caches, send int */
5547 amdgpu_ring_write(ring, PACKET3(PACKET3_RELEASE_MEM, 6));
5548
5549 if (writeback) {
5550 dw2 = EOP_TC_NC_ACTION_EN;
5551 } else {
5552 dw2 = EOP_TCL1_ACTION_EN | EOP_TC_ACTION_EN |
5553 EOP_TC_MD_ACTION_EN;
5554 }
5555 dw2 |= EOP_TC_WB_ACTION_EN | EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
5556 EVENT_INDEX(5);
5557 if (exec)
5558 dw2 |= EOP_EXEC;
5559
5560 amdgpu_ring_write(ring, dw2);
5561 amdgpu_ring_write(ring, DATA_SEL(write64bit ? 2 : 1) | INT_SEL(int_sel ? 2 : 0));
5562
5563 /*
5564 * the address should be Qword aligned if 64bit write, Dword
5565 * aligned if only send 32bit data low (discard data high)
5566 */
5567 if (write64bit)
5568 BUG_ON(addr & 0x7);
5569 else
5570 BUG_ON(addr & 0x3);
5571 amdgpu_ring_write(ring, lower_32_bits(addr));
5572 amdgpu_ring_write(ring, upper_32_bits(addr));
5573 amdgpu_ring_write(ring, lower_32_bits(seq));
5574 amdgpu_ring_write(ring, upper_32_bits(seq));
5575 amdgpu_ring_write(ring, 0);
5576 }
5577
gfx_v9_0_ring_emit_pipeline_sync(struct amdgpu_ring * ring)5578 static void gfx_v9_0_ring_emit_pipeline_sync(struct amdgpu_ring *ring)
5579 {
5580 int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX);
5581 uint32_t seq = ring->fence_drv.sync_seq;
5582 uint64_t addr = ring->fence_drv.gpu_addr;
5583
5584 gfx_v9_0_wait_reg_mem(ring, usepfp, 1, 0,
5585 lower_32_bits(addr), upper_32_bits(addr),
5586 seq, 0xffffffff, 4);
5587 }
5588
gfx_v9_0_ring_emit_vm_flush(struct amdgpu_ring * ring,unsigned vmid,uint64_t pd_addr)5589 static void gfx_v9_0_ring_emit_vm_flush(struct amdgpu_ring *ring,
5590 unsigned vmid, uint64_t pd_addr)
5591 {
5592 amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr);
5593
5594 /* compute doesn't have PFP */
5595 if (ring->funcs->type == AMDGPU_RING_TYPE_GFX) {
5596 /* sync PFP to ME, otherwise we might get invalid PFP reads */
5597 amdgpu_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
5598 amdgpu_ring_write(ring, 0x0);
5599 }
5600 }
5601
gfx_v9_0_ring_get_rptr_compute(struct amdgpu_ring * ring)5602 static u64 gfx_v9_0_ring_get_rptr_compute(struct amdgpu_ring *ring)
5603 {
5604 return *ring->rptr_cpu_addr; /* gfx9 hardware is 32bit rptr */
5605 }
5606
gfx_v9_0_ring_get_wptr_compute(struct amdgpu_ring * ring)5607 static u64 gfx_v9_0_ring_get_wptr_compute(struct amdgpu_ring *ring)
5608 {
5609 u64 wptr;
5610
5611 /* XXX check if swapping is necessary on BE */
5612 if (ring->use_doorbell)
5613 wptr = atomic64_read((atomic64_t *)ring->wptr_cpu_addr);
5614 else
5615 BUG();
5616 return wptr;
5617 }
5618
gfx_v9_0_ring_set_wptr_compute(struct amdgpu_ring * ring)5619 static void gfx_v9_0_ring_set_wptr_compute(struct amdgpu_ring *ring)
5620 {
5621 struct amdgpu_device *adev = ring->adev;
5622
5623 /* XXX check if swapping is necessary on BE */
5624 if (ring->use_doorbell) {
5625 atomic64_set((atomic64_t *)ring->wptr_cpu_addr, ring->wptr);
5626 WDOORBELL64(ring->doorbell_index, ring->wptr);
5627 } else{
5628 BUG(); /* only DOORBELL method supported on gfx9 now */
5629 }
5630 }
5631
gfx_v9_0_ring_emit_fence_kiq(struct amdgpu_ring * ring,u64 addr,u64 seq,unsigned int flags)5632 static void gfx_v9_0_ring_emit_fence_kiq(struct amdgpu_ring *ring, u64 addr,
5633 u64 seq, unsigned int flags)
5634 {
5635 struct amdgpu_device *adev = ring->adev;
5636
5637 /* we only allocate 32bit for each seq wb address */
5638 BUG_ON(flags & AMDGPU_FENCE_FLAG_64BIT);
5639
5640 /* write fence seq to the "addr" */
5641 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5642 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5643 WRITE_DATA_DST_SEL(5) | WR_CONFIRM));
5644 amdgpu_ring_write(ring, lower_32_bits(addr));
5645 amdgpu_ring_write(ring, upper_32_bits(addr));
5646 amdgpu_ring_write(ring, lower_32_bits(seq));
5647
5648 if (flags & AMDGPU_FENCE_FLAG_INT) {
5649 /* set register to trigger INT */
5650 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5651 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5652 WRITE_DATA_DST_SEL(0) | WR_CONFIRM));
5653 amdgpu_ring_write(ring, SOC15_REG_OFFSET(GC, 0, mmCPC_INT_STATUS));
5654 amdgpu_ring_write(ring, 0);
5655 amdgpu_ring_write(ring, 0x20000000); /* src_id is 178 */
5656 }
5657 }
5658
gfx_v9_ring_emit_sb(struct amdgpu_ring * ring)5659 static void gfx_v9_ring_emit_sb(struct amdgpu_ring *ring)
5660 {
5661 amdgpu_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
5662 amdgpu_ring_write(ring, 0);
5663 }
5664
gfx_v9_0_ring_emit_ce_meta(struct amdgpu_ring * ring,bool resume)5665 static void gfx_v9_0_ring_emit_ce_meta(struct amdgpu_ring *ring, bool resume)
5666 {
5667 struct amdgpu_device *adev = ring->adev;
5668 struct v9_ce_ib_state ce_payload = {0};
5669 uint64_t offset, ce_payload_gpu_addr;
5670 void *ce_payload_cpu_addr;
5671 int cnt;
5672
5673 cnt = (sizeof(ce_payload) >> 2) + 4 - 2;
5674
5675 if (ring->is_mes_queue) {
5676 offset = offsetof(struct amdgpu_mes_ctx_meta_data,
5677 gfx[0].gfx_meta_data) +
5678 offsetof(struct v9_gfx_meta_data, ce_payload);
5679 ce_payload_gpu_addr =
5680 amdgpu_mes_ctx_get_offs_gpu_addr(ring, offset);
5681 ce_payload_cpu_addr =
5682 amdgpu_mes_ctx_get_offs_cpu_addr(ring, offset);
5683 } else {
5684 offset = offsetof(struct v9_gfx_meta_data, ce_payload);
5685 ce_payload_gpu_addr = amdgpu_csa_vaddr(ring->adev) + offset;
5686 ce_payload_cpu_addr = adev->virt.csa_cpu_addr + offset;
5687 }
5688
5689 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, cnt));
5690 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(2) |
5691 WRITE_DATA_DST_SEL(8) |
5692 WR_CONFIRM) |
5693 WRITE_DATA_CACHE_POLICY(0));
5694 amdgpu_ring_write(ring, lower_32_bits(ce_payload_gpu_addr));
5695 amdgpu_ring_write(ring, upper_32_bits(ce_payload_gpu_addr));
5696
5697 amdgpu_ring_ib_on_emit_ce(ring);
5698
5699 if (resume)
5700 amdgpu_ring_write_multiple(ring, ce_payload_cpu_addr,
5701 sizeof(ce_payload) >> 2);
5702 else
5703 amdgpu_ring_write_multiple(ring, (void *)&ce_payload,
5704 sizeof(ce_payload) >> 2);
5705 }
5706
gfx_v9_0_ring_preempt_ib(struct amdgpu_ring * ring)5707 static int gfx_v9_0_ring_preempt_ib(struct amdgpu_ring *ring)
5708 {
5709 int i, r = 0;
5710 struct amdgpu_device *adev = ring->adev;
5711 struct amdgpu_kiq *kiq = &adev->gfx.kiq[0];
5712 struct amdgpu_ring *kiq_ring = &kiq->ring;
5713 unsigned long flags;
5714
5715 if (!kiq->pmf || !kiq->pmf->kiq_unmap_queues)
5716 return -EINVAL;
5717
5718 spin_lock_irqsave(&kiq->ring_lock, flags);
5719
5720 if (amdgpu_ring_alloc(kiq_ring, kiq->pmf->unmap_queues_size)) {
5721 spin_unlock_irqrestore(&kiq->ring_lock, flags);
5722 return -ENOMEM;
5723 }
5724
5725 /* assert preemption condition */
5726 amdgpu_ring_set_preempt_cond_exec(ring, false);
5727
5728 ring->trail_seq += 1;
5729 amdgpu_ring_alloc(ring, 13);
5730 gfx_v9_0_ring_emit_fence(ring, ring->trail_fence_gpu_addr,
5731 ring->trail_seq, AMDGPU_FENCE_FLAG_EXEC | AMDGPU_FENCE_FLAG_INT);
5732
5733 /* assert IB preemption, emit the trailing fence */
5734 kiq->pmf->kiq_unmap_queues(kiq_ring, ring, PREEMPT_QUEUES_NO_UNMAP,
5735 ring->trail_fence_gpu_addr,
5736 ring->trail_seq);
5737
5738 amdgpu_ring_commit(kiq_ring);
5739 spin_unlock_irqrestore(&kiq->ring_lock, flags);
5740
5741 /* poll the trailing fence */
5742 for (i = 0; i < adev->usec_timeout; i++) {
5743 if (ring->trail_seq ==
5744 le32_to_cpu(*ring->trail_fence_cpu_addr))
5745 break;
5746 udelay(1);
5747 }
5748
5749 if (i >= adev->usec_timeout) {
5750 r = -EINVAL;
5751 DRM_WARN("ring %d timeout to preempt ib\n", ring->idx);
5752 }
5753
5754 /*reset the CP_VMID_PREEMPT after trailing fence*/
5755 amdgpu_ring_emit_wreg(ring,
5756 SOC15_REG_OFFSET(GC, 0, mmCP_VMID_PREEMPT),
5757 0x0);
5758 amdgpu_ring_commit(ring);
5759
5760 /* deassert preemption condition */
5761 amdgpu_ring_set_preempt_cond_exec(ring, true);
5762 return r;
5763 }
5764
gfx_v9_0_ring_emit_de_meta(struct amdgpu_ring * ring,bool resume,bool usegds)5765 static void gfx_v9_0_ring_emit_de_meta(struct amdgpu_ring *ring, bool resume, bool usegds)
5766 {
5767 struct amdgpu_device *adev = ring->adev;
5768 struct v9_de_ib_state de_payload = {0};
5769 uint64_t offset, gds_addr, de_payload_gpu_addr;
5770 void *de_payload_cpu_addr;
5771 int cnt;
5772
5773 if (ring->is_mes_queue) {
5774 offset = offsetof(struct amdgpu_mes_ctx_meta_data,
5775 gfx[0].gfx_meta_data) +
5776 offsetof(struct v9_gfx_meta_data, de_payload);
5777 de_payload_gpu_addr =
5778 amdgpu_mes_ctx_get_offs_gpu_addr(ring, offset);
5779 de_payload_cpu_addr =
5780 amdgpu_mes_ctx_get_offs_cpu_addr(ring, offset);
5781
5782 offset = offsetof(struct amdgpu_mes_ctx_meta_data,
5783 gfx[0].gds_backup) +
5784 offsetof(struct v9_gfx_meta_data, de_payload);
5785 gds_addr = amdgpu_mes_ctx_get_offs_gpu_addr(ring, offset);
5786 } else {
5787 offset = offsetof(struct v9_gfx_meta_data, de_payload);
5788 de_payload_gpu_addr = amdgpu_csa_vaddr(ring->adev) + offset;
5789 de_payload_cpu_addr = adev->virt.csa_cpu_addr + offset;
5790
5791 gds_addr = ALIGN(amdgpu_csa_vaddr(ring->adev) +
5792 AMDGPU_CSA_SIZE - adev->gds.gds_size,
5793 PAGE_SIZE);
5794 }
5795
5796 if (usegds) {
5797 de_payload.gds_backup_addrlo = lower_32_bits(gds_addr);
5798 de_payload.gds_backup_addrhi = upper_32_bits(gds_addr);
5799 }
5800
5801 cnt = (sizeof(de_payload) >> 2) + 4 - 2;
5802 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, cnt));
5803 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(1) |
5804 WRITE_DATA_DST_SEL(8) |
5805 WR_CONFIRM) |
5806 WRITE_DATA_CACHE_POLICY(0));
5807 amdgpu_ring_write(ring, lower_32_bits(de_payload_gpu_addr));
5808 amdgpu_ring_write(ring, upper_32_bits(de_payload_gpu_addr));
5809
5810 amdgpu_ring_ib_on_emit_de(ring);
5811 if (resume)
5812 amdgpu_ring_write_multiple(ring, de_payload_cpu_addr,
5813 sizeof(de_payload) >> 2);
5814 else
5815 amdgpu_ring_write_multiple(ring, (void *)&de_payload,
5816 sizeof(de_payload) >> 2);
5817 }
5818
gfx_v9_0_ring_emit_frame_cntl(struct amdgpu_ring * ring,bool start,bool secure)5819 static void gfx_v9_0_ring_emit_frame_cntl(struct amdgpu_ring *ring, bool start,
5820 bool secure)
5821 {
5822 uint32_t v = secure ? FRAME_TMZ : 0;
5823
5824 amdgpu_ring_write(ring, PACKET3(PACKET3_FRAME_CONTROL, 0));
5825 amdgpu_ring_write(ring, v | FRAME_CMD(start ? 0 : 1));
5826 }
5827
gfx_v9_ring_emit_cntxcntl(struct amdgpu_ring * ring,uint32_t flags)5828 static void gfx_v9_ring_emit_cntxcntl(struct amdgpu_ring *ring, uint32_t flags)
5829 {
5830 uint32_t dw2 = 0;
5831
5832 gfx_v9_0_ring_emit_ce_meta(ring,
5833 (!amdgpu_sriov_vf(ring->adev) &&
5834 flags & AMDGPU_IB_PREEMPTED) ? true : false);
5835
5836 dw2 |= 0x80000000; /* set load_enable otherwise this package is just NOPs */
5837 if (flags & AMDGPU_HAVE_CTX_SWITCH) {
5838 /* set load_global_config & load_global_uconfig */
5839 dw2 |= 0x8001;
5840 /* set load_cs_sh_regs */
5841 dw2 |= 0x01000000;
5842 /* set load_per_context_state & load_gfx_sh_regs for GFX */
5843 dw2 |= 0x10002;
5844
5845 /* set load_ce_ram if preamble presented */
5846 if (AMDGPU_PREAMBLE_IB_PRESENT & flags)
5847 dw2 |= 0x10000000;
5848 } else {
5849 /* still load_ce_ram if this is the first time preamble presented
5850 * although there is no context switch happens.
5851 */
5852 if (AMDGPU_PREAMBLE_IB_PRESENT_FIRST & flags)
5853 dw2 |= 0x10000000;
5854 }
5855
5856 amdgpu_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1));
5857 amdgpu_ring_write(ring, dw2);
5858 amdgpu_ring_write(ring, 0);
5859 }
5860
gfx_v9_0_ring_emit_init_cond_exec(struct amdgpu_ring * ring,uint64_t addr)5861 static unsigned gfx_v9_0_ring_emit_init_cond_exec(struct amdgpu_ring *ring,
5862 uint64_t addr)
5863 {
5864 unsigned ret;
5865 amdgpu_ring_write(ring, PACKET3(PACKET3_COND_EXEC, 3));
5866 amdgpu_ring_write(ring, lower_32_bits(addr));
5867 amdgpu_ring_write(ring, upper_32_bits(addr));
5868 /* discard following DWs if *cond_exec_gpu_addr==0 */
5869 amdgpu_ring_write(ring, 0);
5870 ret = ring->wptr & ring->buf_mask;
5871 /* patch dummy value later */
5872 amdgpu_ring_write(ring, 0);
5873 return ret;
5874 }
5875
gfx_v9_0_ring_emit_rreg(struct amdgpu_ring * ring,uint32_t reg,uint32_t reg_val_offs)5876 static void gfx_v9_0_ring_emit_rreg(struct amdgpu_ring *ring, uint32_t reg,
5877 uint32_t reg_val_offs)
5878 {
5879 struct amdgpu_device *adev = ring->adev;
5880
5881 amdgpu_ring_write(ring, PACKET3(PACKET3_COPY_DATA, 4));
5882 amdgpu_ring_write(ring, 0 | /* src: register*/
5883 (5 << 8) | /* dst: memory */
5884 (1 << 20)); /* write confirm */
5885 amdgpu_ring_write(ring, reg);
5886 amdgpu_ring_write(ring, 0);
5887 amdgpu_ring_write(ring, lower_32_bits(adev->wb.gpu_addr +
5888 reg_val_offs * 4));
5889 amdgpu_ring_write(ring, upper_32_bits(adev->wb.gpu_addr +
5890 reg_val_offs * 4));
5891 }
5892
gfx_v9_0_ring_emit_wreg(struct amdgpu_ring * ring,uint32_t reg,uint32_t val)5893 static void gfx_v9_0_ring_emit_wreg(struct amdgpu_ring *ring, uint32_t reg,
5894 uint32_t val)
5895 {
5896 uint32_t cmd = 0;
5897
5898 switch (ring->funcs->type) {
5899 case AMDGPU_RING_TYPE_GFX:
5900 cmd = WRITE_DATA_ENGINE_SEL(1) | WR_CONFIRM;
5901 break;
5902 case AMDGPU_RING_TYPE_KIQ:
5903 cmd = (1 << 16); /* no inc addr */
5904 break;
5905 default:
5906 cmd = WR_CONFIRM;
5907 break;
5908 }
5909 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5910 amdgpu_ring_write(ring, cmd);
5911 amdgpu_ring_write(ring, reg);
5912 amdgpu_ring_write(ring, 0);
5913 amdgpu_ring_write(ring, val);
5914 }
5915
gfx_v9_0_ring_emit_reg_wait(struct amdgpu_ring * ring,uint32_t reg,uint32_t val,uint32_t mask)5916 static void gfx_v9_0_ring_emit_reg_wait(struct amdgpu_ring *ring, uint32_t reg,
5917 uint32_t val, uint32_t mask)
5918 {
5919 gfx_v9_0_wait_reg_mem(ring, 0, 0, 0, reg, 0, val, mask, 0x20);
5920 }
5921
gfx_v9_0_ring_emit_reg_write_reg_wait(struct amdgpu_ring * ring,uint32_t reg0,uint32_t reg1,uint32_t ref,uint32_t mask)5922 static void gfx_v9_0_ring_emit_reg_write_reg_wait(struct amdgpu_ring *ring,
5923 uint32_t reg0, uint32_t reg1,
5924 uint32_t ref, uint32_t mask)
5925 {
5926 int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX);
5927 struct amdgpu_device *adev = ring->adev;
5928 bool fw_version_ok = (ring->funcs->type == AMDGPU_RING_TYPE_GFX) ?
5929 adev->gfx.me_fw_write_wait : adev->gfx.mec_fw_write_wait;
5930
5931 if (fw_version_ok)
5932 gfx_v9_0_wait_reg_mem(ring, usepfp, 0, 1, reg0, reg1,
5933 ref, mask, 0x20);
5934 else
5935 amdgpu_ring_emit_reg_write_reg_wait_helper(ring, reg0, reg1,
5936 ref, mask);
5937 }
5938
gfx_v9_0_ring_soft_recovery(struct amdgpu_ring * ring,unsigned vmid)5939 static void gfx_v9_0_ring_soft_recovery(struct amdgpu_ring *ring, unsigned vmid)
5940 {
5941 struct amdgpu_device *adev = ring->adev;
5942 uint32_t value = 0;
5943
5944 value = REG_SET_FIELD(value, SQ_CMD, CMD, 0x03);
5945 value = REG_SET_FIELD(value, SQ_CMD, MODE, 0x01);
5946 value = REG_SET_FIELD(value, SQ_CMD, CHECK_VMID, 1);
5947 value = REG_SET_FIELD(value, SQ_CMD, VM_ID, vmid);
5948 amdgpu_gfx_rlc_enter_safe_mode(adev, 0);
5949 WREG32_SOC15(GC, 0, mmSQ_CMD, value);
5950 amdgpu_gfx_rlc_exit_safe_mode(adev, 0);
5951 }
5952
gfx_v9_0_set_gfx_eop_interrupt_state(struct amdgpu_device * adev,enum amdgpu_interrupt_state state)5953 static void gfx_v9_0_set_gfx_eop_interrupt_state(struct amdgpu_device *adev,
5954 enum amdgpu_interrupt_state state)
5955 {
5956 switch (state) {
5957 case AMDGPU_IRQ_STATE_DISABLE:
5958 case AMDGPU_IRQ_STATE_ENABLE:
5959 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0,
5960 TIME_STAMP_INT_ENABLE,
5961 state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
5962 break;
5963 default:
5964 break;
5965 }
5966 }
5967
gfx_v9_0_set_compute_eop_interrupt_state(struct amdgpu_device * adev,int me,int pipe,enum amdgpu_interrupt_state state)5968 static void gfx_v9_0_set_compute_eop_interrupt_state(struct amdgpu_device *adev,
5969 int me, int pipe,
5970 enum amdgpu_interrupt_state state)
5971 {
5972 u32 mec_int_cntl, mec_int_cntl_reg;
5973
5974 /*
5975 * amdgpu controls only the first MEC. That's why this function only
5976 * handles the setting of interrupts for this specific MEC. All other
5977 * pipes' interrupts are set by amdkfd.
5978 */
5979
5980 if (me == 1) {
5981 switch (pipe) {
5982 case 0:
5983 mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE0_INT_CNTL);
5984 break;
5985 case 1:
5986 mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE1_INT_CNTL);
5987 break;
5988 case 2:
5989 mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE2_INT_CNTL);
5990 break;
5991 case 3:
5992 mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE3_INT_CNTL);
5993 break;
5994 default:
5995 DRM_DEBUG("invalid pipe %d\n", pipe);
5996 return;
5997 }
5998 } else {
5999 DRM_DEBUG("invalid me %d\n", me);
6000 return;
6001 }
6002
6003 switch (state) {
6004 case AMDGPU_IRQ_STATE_DISABLE:
6005 mec_int_cntl = RREG32_SOC15_IP(GC,mec_int_cntl_reg);
6006 mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL,
6007 TIME_STAMP_INT_ENABLE, 0);
6008 WREG32_SOC15_IP(GC, mec_int_cntl_reg, mec_int_cntl);
6009 break;
6010 case AMDGPU_IRQ_STATE_ENABLE:
6011 mec_int_cntl = RREG32_SOC15_IP(GC, mec_int_cntl_reg);
6012 mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL,
6013 TIME_STAMP_INT_ENABLE, 1);
6014 WREG32_SOC15_IP(GC, mec_int_cntl_reg, mec_int_cntl);
6015 break;
6016 default:
6017 break;
6018 }
6019 }
6020
gfx_v9_0_get_cpc_int_cntl(struct amdgpu_device * adev,int me,int pipe)6021 static u32 gfx_v9_0_get_cpc_int_cntl(struct amdgpu_device *adev,
6022 int me, int pipe)
6023 {
6024 /*
6025 * amdgpu controls only the first MEC. That's why this function only
6026 * handles the setting of interrupts for this specific MEC. All other
6027 * pipes' interrupts are set by amdkfd.
6028 */
6029 if (me != 1)
6030 return 0;
6031
6032 switch (pipe) {
6033 case 0:
6034 return SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE0_INT_CNTL);
6035 case 1:
6036 return SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE1_INT_CNTL);
6037 case 2:
6038 return SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE2_INT_CNTL);
6039 case 3:
6040 return SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE3_INT_CNTL);
6041 default:
6042 return 0;
6043 }
6044 }
6045
gfx_v9_0_set_priv_reg_fault_state(struct amdgpu_device * adev,struct amdgpu_irq_src * source,unsigned type,enum amdgpu_interrupt_state state)6046 static int gfx_v9_0_set_priv_reg_fault_state(struct amdgpu_device *adev,
6047 struct amdgpu_irq_src *source,
6048 unsigned type,
6049 enum amdgpu_interrupt_state state)
6050 {
6051 u32 cp_int_cntl_reg, cp_int_cntl;
6052 int i, j;
6053
6054 switch (state) {
6055 case AMDGPU_IRQ_STATE_DISABLE:
6056 case AMDGPU_IRQ_STATE_ENABLE:
6057 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0,
6058 PRIV_REG_INT_ENABLE,
6059 state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
6060 for (i = 0; i < adev->gfx.mec.num_mec; i++) {
6061 for (j = 0; j < adev->gfx.mec.num_pipe_per_mec; j++) {
6062 /* MECs start at 1 */
6063 cp_int_cntl_reg = gfx_v9_0_get_cpc_int_cntl(adev, i + 1, j);
6064
6065 if (cp_int_cntl_reg) {
6066 cp_int_cntl = RREG32_SOC15_IP(GC, cp_int_cntl_reg);
6067 cp_int_cntl = REG_SET_FIELD(cp_int_cntl, CP_ME1_PIPE0_INT_CNTL,
6068 PRIV_REG_INT_ENABLE,
6069 state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
6070 WREG32_SOC15_IP(GC, cp_int_cntl_reg, cp_int_cntl);
6071 }
6072 }
6073 }
6074 break;
6075 default:
6076 break;
6077 }
6078
6079 return 0;
6080 }
6081
gfx_v9_0_set_bad_op_fault_state(struct amdgpu_device * adev,struct amdgpu_irq_src * source,unsigned type,enum amdgpu_interrupt_state state)6082 static int gfx_v9_0_set_bad_op_fault_state(struct amdgpu_device *adev,
6083 struct amdgpu_irq_src *source,
6084 unsigned type,
6085 enum amdgpu_interrupt_state state)
6086 {
6087 u32 cp_int_cntl_reg, cp_int_cntl;
6088 int i, j;
6089
6090 switch (state) {
6091 case AMDGPU_IRQ_STATE_DISABLE:
6092 case AMDGPU_IRQ_STATE_ENABLE:
6093 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0,
6094 OPCODE_ERROR_INT_ENABLE,
6095 state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
6096 for (i = 0; i < adev->gfx.mec.num_mec; i++) {
6097 for (j = 0; j < adev->gfx.mec.num_pipe_per_mec; j++) {
6098 /* MECs start at 1 */
6099 cp_int_cntl_reg = gfx_v9_0_get_cpc_int_cntl(adev, i + 1, j);
6100
6101 if (cp_int_cntl_reg) {
6102 cp_int_cntl = RREG32_SOC15_IP(GC, cp_int_cntl_reg);
6103 cp_int_cntl = REG_SET_FIELD(cp_int_cntl, CP_ME1_PIPE0_INT_CNTL,
6104 OPCODE_ERROR_INT_ENABLE,
6105 state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
6106 WREG32_SOC15_IP(GC, cp_int_cntl_reg, cp_int_cntl);
6107 }
6108 }
6109 }
6110 break;
6111 default:
6112 break;
6113 }
6114
6115 return 0;
6116 }
6117
gfx_v9_0_set_priv_inst_fault_state(struct amdgpu_device * adev,struct amdgpu_irq_src * source,unsigned type,enum amdgpu_interrupt_state state)6118 static int gfx_v9_0_set_priv_inst_fault_state(struct amdgpu_device *adev,
6119 struct amdgpu_irq_src *source,
6120 unsigned type,
6121 enum amdgpu_interrupt_state state)
6122 {
6123 switch (state) {
6124 case AMDGPU_IRQ_STATE_DISABLE:
6125 case AMDGPU_IRQ_STATE_ENABLE:
6126 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0,
6127 PRIV_INSTR_INT_ENABLE,
6128 state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
6129 break;
6130 default:
6131 break;
6132 }
6133
6134 return 0;
6135 }
6136
6137 #define ENABLE_ECC_ON_ME_PIPE(me, pipe) \
6138 WREG32_FIELD15(GC, 0, CP_ME##me##_PIPE##pipe##_INT_CNTL,\
6139 CP_ECC_ERROR_INT_ENABLE, 1)
6140
6141 #define DISABLE_ECC_ON_ME_PIPE(me, pipe) \
6142 WREG32_FIELD15(GC, 0, CP_ME##me##_PIPE##pipe##_INT_CNTL,\
6143 CP_ECC_ERROR_INT_ENABLE, 0)
6144
gfx_v9_0_set_cp_ecc_error_state(struct amdgpu_device * adev,struct amdgpu_irq_src * source,unsigned type,enum amdgpu_interrupt_state state)6145 static int gfx_v9_0_set_cp_ecc_error_state(struct amdgpu_device *adev,
6146 struct amdgpu_irq_src *source,
6147 unsigned type,
6148 enum amdgpu_interrupt_state state)
6149 {
6150 switch (state) {
6151 case AMDGPU_IRQ_STATE_DISABLE:
6152 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0,
6153 CP_ECC_ERROR_INT_ENABLE, 0);
6154 DISABLE_ECC_ON_ME_PIPE(1, 0);
6155 DISABLE_ECC_ON_ME_PIPE(1, 1);
6156 DISABLE_ECC_ON_ME_PIPE(1, 2);
6157 DISABLE_ECC_ON_ME_PIPE(1, 3);
6158 break;
6159
6160 case AMDGPU_IRQ_STATE_ENABLE:
6161 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0,
6162 CP_ECC_ERROR_INT_ENABLE, 1);
6163 ENABLE_ECC_ON_ME_PIPE(1, 0);
6164 ENABLE_ECC_ON_ME_PIPE(1, 1);
6165 ENABLE_ECC_ON_ME_PIPE(1, 2);
6166 ENABLE_ECC_ON_ME_PIPE(1, 3);
6167 break;
6168 default:
6169 break;
6170 }
6171
6172 return 0;
6173 }
6174
6175
gfx_v9_0_set_eop_interrupt_state(struct amdgpu_device * adev,struct amdgpu_irq_src * src,unsigned type,enum amdgpu_interrupt_state state)6176 static int gfx_v9_0_set_eop_interrupt_state(struct amdgpu_device *adev,
6177 struct amdgpu_irq_src *src,
6178 unsigned type,
6179 enum amdgpu_interrupt_state state)
6180 {
6181 switch (type) {
6182 case AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP:
6183 gfx_v9_0_set_gfx_eop_interrupt_state(adev, state);
6184 break;
6185 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP:
6186 gfx_v9_0_set_compute_eop_interrupt_state(adev, 1, 0, state);
6187 break;
6188 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE1_EOP:
6189 gfx_v9_0_set_compute_eop_interrupt_state(adev, 1, 1, state);
6190 break;
6191 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE2_EOP:
6192 gfx_v9_0_set_compute_eop_interrupt_state(adev, 1, 2, state);
6193 break;
6194 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE3_EOP:
6195 gfx_v9_0_set_compute_eop_interrupt_state(adev, 1, 3, state);
6196 break;
6197 case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE0_EOP:
6198 gfx_v9_0_set_compute_eop_interrupt_state(adev, 2, 0, state);
6199 break;
6200 case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE1_EOP:
6201 gfx_v9_0_set_compute_eop_interrupt_state(adev, 2, 1, state);
6202 break;
6203 case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE2_EOP:
6204 gfx_v9_0_set_compute_eop_interrupt_state(adev, 2, 2, state);
6205 break;
6206 case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE3_EOP:
6207 gfx_v9_0_set_compute_eop_interrupt_state(adev, 2, 3, state);
6208 break;
6209 default:
6210 break;
6211 }
6212 return 0;
6213 }
6214
gfx_v9_0_eop_irq(struct amdgpu_device * adev,struct amdgpu_irq_src * source,struct amdgpu_iv_entry * entry)6215 static int gfx_v9_0_eop_irq(struct amdgpu_device *adev,
6216 struct amdgpu_irq_src *source,
6217 struct amdgpu_iv_entry *entry)
6218 {
6219 int i;
6220 u8 me_id, pipe_id, queue_id;
6221 struct amdgpu_ring *ring;
6222
6223 DRM_DEBUG("IH: CP EOP\n");
6224 me_id = (entry->ring_id & 0x0c) >> 2;
6225 pipe_id = (entry->ring_id & 0x03) >> 0;
6226 queue_id = (entry->ring_id & 0x70) >> 4;
6227
6228 switch (me_id) {
6229 case 0:
6230 if (adev->gfx.num_gfx_rings) {
6231 if (!adev->gfx.mcbp) {
6232 amdgpu_fence_process(&adev->gfx.gfx_ring[0]);
6233 } else if (!amdgpu_mcbp_handle_trailing_fence_irq(&adev->gfx.muxer)) {
6234 /* Fence signals are handled on the software rings*/
6235 for (i = 0; i < GFX9_NUM_SW_GFX_RINGS; i++)
6236 amdgpu_fence_process(&adev->gfx.sw_gfx_ring[i]);
6237 }
6238 }
6239 break;
6240 case 1:
6241 case 2:
6242 for (i = 0; i < adev->gfx.num_compute_rings; i++) {
6243 ring = &adev->gfx.compute_ring[i];
6244 /* Per-queue interrupt is supported for MEC starting from VI.
6245 * The interrupt can only be enabled/disabled per pipe instead of per queue.
6246 */
6247 if ((ring->me == me_id) && (ring->pipe == pipe_id) && (ring->queue == queue_id))
6248 amdgpu_fence_process(ring);
6249 }
6250 break;
6251 }
6252 return 0;
6253 }
6254
gfx_v9_0_fault(struct amdgpu_device * adev,struct amdgpu_iv_entry * entry)6255 static void gfx_v9_0_fault(struct amdgpu_device *adev,
6256 struct amdgpu_iv_entry *entry)
6257 {
6258 u8 me_id, pipe_id, queue_id;
6259 struct amdgpu_ring *ring;
6260 int i;
6261
6262 me_id = (entry->ring_id & 0x0c) >> 2;
6263 pipe_id = (entry->ring_id & 0x03) >> 0;
6264 queue_id = (entry->ring_id & 0x70) >> 4;
6265
6266 switch (me_id) {
6267 case 0:
6268 drm_sched_fault(&adev->gfx.gfx_ring[0].sched);
6269 break;
6270 case 1:
6271 case 2:
6272 for (i = 0; i < adev->gfx.num_compute_rings; i++) {
6273 ring = &adev->gfx.compute_ring[i];
6274 if (ring->me == me_id && ring->pipe == pipe_id &&
6275 ring->queue == queue_id)
6276 drm_sched_fault(&ring->sched);
6277 }
6278 break;
6279 }
6280 }
6281
gfx_v9_0_priv_reg_irq(struct amdgpu_device * adev,struct amdgpu_irq_src * source,struct amdgpu_iv_entry * entry)6282 static int gfx_v9_0_priv_reg_irq(struct amdgpu_device *adev,
6283 struct amdgpu_irq_src *source,
6284 struct amdgpu_iv_entry *entry)
6285 {
6286 DRM_ERROR("Illegal register access in command stream\n");
6287 gfx_v9_0_fault(adev, entry);
6288 return 0;
6289 }
6290
gfx_v9_0_bad_op_irq(struct amdgpu_device * adev,struct amdgpu_irq_src * source,struct amdgpu_iv_entry * entry)6291 static int gfx_v9_0_bad_op_irq(struct amdgpu_device *adev,
6292 struct amdgpu_irq_src *source,
6293 struct amdgpu_iv_entry *entry)
6294 {
6295 DRM_ERROR("Illegal opcode in command stream\n");
6296 gfx_v9_0_fault(adev, entry);
6297 return 0;
6298 }
6299
gfx_v9_0_priv_inst_irq(struct amdgpu_device * adev,struct amdgpu_irq_src * source,struct amdgpu_iv_entry * entry)6300 static int gfx_v9_0_priv_inst_irq(struct amdgpu_device *adev,
6301 struct amdgpu_irq_src *source,
6302 struct amdgpu_iv_entry *entry)
6303 {
6304 DRM_ERROR("Illegal instruction in command stream\n");
6305 gfx_v9_0_fault(adev, entry);
6306 return 0;
6307 }
6308
6309
6310 static const struct soc15_ras_field_entry gfx_v9_0_ras_fields[] = {
6311 { "CPC_SCRATCH", SOC15_REG_ENTRY(GC, 0, mmCPC_EDC_SCRATCH_CNT),
6312 SOC15_REG_FIELD(CPC_EDC_SCRATCH_CNT, SEC_COUNT),
6313 SOC15_REG_FIELD(CPC_EDC_SCRATCH_CNT, DED_COUNT)
6314 },
6315 { "CPC_UCODE", SOC15_REG_ENTRY(GC, 0, mmCPC_EDC_UCODE_CNT),
6316 SOC15_REG_FIELD(CPC_EDC_UCODE_CNT, SEC_COUNT),
6317 SOC15_REG_FIELD(CPC_EDC_UCODE_CNT, DED_COUNT)
6318 },
6319 { "CPF_ROQ_ME1", SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_ROQ_CNT),
6320 SOC15_REG_FIELD(CPF_EDC_ROQ_CNT, COUNT_ME1),
6321 0, 0
6322 },
6323 { "CPF_ROQ_ME2", SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_ROQ_CNT),
6324 SOC15_REG_FIELD(CPF_EDC_ROQ_CNT, COUNT_ME2),
6325 0, 0
6326 },
6327 { "CPF_TAG", SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_TAG_CNT),
6328 SOC15_REG_FIELD(CPF_EDC_TAG_CNT, SEC_COUNT),
6329 SOC15_REG_FIELD(CPF_EDC_TAG_CNT, DED_COUNT)
6330 },
6331 { "CPG_DMA_ROQ", SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_DMA_CNT),
6332 SOC15_REG_FIELD(CPG_EDC_DMA_CNT, ROQ_COUNT),
6333 0, 0
6334 },
6335 { "CPG_DMA_TAG", SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_DMA_CNT),
6336 SOC15_REG_FIELD(CPG_EDC_DMA_CNT, TAG_SEC_COUNT),
6337 SOC15_REG_FIELD(CPG_EDC_DMA_CNT, TAG_DED_COUNT)
6338 },
6339 { "CPG_TAG", SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_TAG_CNT),
6340 SOC15_REG_FIELD(CPG_EDC_TAG_CNT, SEC_COUNT),
6341 SOC15_REG_FIELD(CPG_EDC_TAG_CNT, DED_COUNT)
6342 },
6343 { "DC_CSINVOC", SOC15_REG_ENTRY(GC, 0, mmDC_EDC_CSINVOC_CNT),
6344 SOC15_REG_FIELD(DC_EDC_CSINVOC_CNT, COUNT_ME1),
6345 0, 0
6346 },
6347 { "DC_RESTORE", SOC15_REG_ENTRY(GC, 0, mmDC_EDC_RESTORE_CNT),
6348 SOC15_REG_FIELD(DC_EDC_RESTORE_CNT, COUNT_ME1),
6349 0, 0
6350 },
6351 { "DC_STATE", SOC15_REG_ENTRY(GC, 0, mmDC_EDC_STATE_CNT),
6352 SOC15_REG_FIELD(DC_EDC_STATE_CNT, COUNT_ME1),
6353 0, 0
6354 },
6355 { "GDS_MEM", SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_CNT),
6356 SOC15_REG_FIELD(GDS_EDC_CNT, GDS_MEM_SEC),
6357 SOC15_REG_FIELD(GDS_EDC_CNT, GDS_MEM_DED)
6358 },
6359 { "GDS_INPUT_QUEUE", SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_CNT),
6360 SOC15_REG_FIELD(GDS_EDC_CNT, GDS_INPUT_QUEUE_SED),
6361 0, 0
6362 },
6363 { "GDS_ME0_CS_PIPE_MEM", SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PHY_CNT),
6364 SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, ME0_CS_PIPE_MEM_SEC),
6365 SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, ME0_CS_PIPE_MEM_DED)
6366 },
6367 { "GDS_OA_PHY_PHY_CMD_RAM_MEM",
6368 SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PHY_CNT),
6369 SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, PHY_CMD_RAM_MEM_SEC),
6370 SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, PHY_CMD_RAM_MEM_DED)
6371 },
6372 { "GDS_OA_PHY_PHY_DATA_RAM_MEM",
6373 SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PHY_CNT),
6374 SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, PHY_DATA_RAM_MEM_SED),
6375 0, 0
6376 },
6377 { "GDS_OA_PIPE_ME1_PIPE0_PIPE_MEM",
6378 SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT),
6379 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE0_PIPE_MEM_SEC),
6380 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE0_PIPE_MEM_DED)
6381 },
6382 { "GDS_OA_PIPE_ME1_PIPE1_PIPE_MEM",
6383 SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT),
6384 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE1_PIPE_MEM_SEC),
6385 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE1_PIPE_MEM_DED)
6386 },
6387 { "GDS_OA_PIPE_ME1_PIPE2_PIPE_MEM",
6388 SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT),
6389 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE2_PIPE_MEM_SEC),
6390 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE2_PIPE_MEM_DED)
6391 },
6392 { "GDS_OA_PIPE_ME1_PIPE3_PIPE_MEM",
6393 SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT),
6394 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE3_PIPE_MEM_SEC),
6395 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE3_PIPE_MEM_DED)
6396 },
6397 { "SPI_SR_MEM", SOC15_REG_ENTRY(GC, 0, mmSPI_EDC_CNT),
6398 SOC15_REG_FIELD(SPI_EDC_CNT, SPI_SR_MEM_SED_COUNT),
6399 0, 0
6400 },
6401 { "TA_FS_DFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT),
6402 SOC15_REG_FIELD(TA_EDC_CNT, TA_FS_DFIFO_SEC_COUNT),
6403 SOC15_REG_FIELD(TA_EDC_CNT, TA_FS_DFIFO_DED_COUNT)
6404 },
6405 { "TA_FS_AFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT),
6406 SOC15_REG_FIELD(TA_EDC_CNT, TA_FS_AFIFO_SED_COUNT),
6407 0, 0
6408 },
6409 { "TA_FL_LFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT),
6410 SOC15_REG_FIELD(TA_EDC_CNT, TA_FL_LFIFO_SED_COUNT),
6411 0, 0
6412 },
6413 { "TA_FX_LFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT),
6414 SOC15_REG_FIELD(TA_EDC_CNT, TA_FX_LFIFO_SED_COUNT),
6415 0, 0
6416 },
6417 { "TA_FS_CFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT),
6418 SOC15_REG_FIELD(TA_EDC_CNT, TA_FS_CFIFO_SED_COUNT),
6419 0, 0
6420 },
6421 { "TCA_HOLE_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCA_EDC_CNT),
6422 SOC15_REG_FIELD(TCA_EDC_CNT, HOLE_FIFO_SED_COUNT),
6423 0, 0
6424 },
6425 { "TCA_REQ_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCA_EDC_CNT),
6426 SOC15_REG_FIELD(TCA_EDC_CNT, REQ_FIFO_SED_COUNT),
6427 0, 0
6428 },
6429 { "TCC_CACHE_DATA", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6430 SOC15_REG_FIELD(TCC_EDC_CNT, CACHE_DATA_SEC_COUNT),
6431 SOC15_REG_FIELD(TCC_EDC_CNT, CACHE_DATA_DED_COUNT)
6432 },
6433 { "TCC_CACHE_DIRTY", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6434 SOC15_REG_FIELD(TCC_EDC_CNT, CACHE_DIRTY_SEC_COUNT),
6435 SOC15_REG_FIELD(TCC_EDC_CNT, CACHE_DIRTY_DED_COUNT)
6436 },
6437 { "TCC_HIGH_RATE_TAG", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6438 SOC15_REG_FIELD(TCC_EDC_CNT, HIGH_RATE_TAG_SEC_COUNT),
6439 SOC15_REG_FIELD(TCC_EDC_CNT, HIGH_RATE_TAG_DED_COUNT)
6440 },
6441 { "TCC_LOW_RATE_TAG", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6442 SOC15_REG_FIELD(TCC_EDC_CNT, LOW_RATE_TAG_SEC_COUNT),
6443 SOC15_REG_FIELD(TCC_EDC_CNT, LOW_RATE_TAG_DED_COUNT)
6444 },
6445 { "TCC_SRC_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6446 SOC15_REG_FIELD(TCC_EDC_CNT, SRC_FIFO_SEC_COUNT),
6447 SOC15_REG_FIELD(TCC_EDC_CNT, SRC_FIFO_DED_COUNT)
6448 },
6449 { "TCC_IN_USE_DEC", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6450 SOC15_REG_FIELD(TCC_EDC_CNT, IN_USE_DEC_SED_COUNT),
6451 0, 0
6452 },
6453 { "TCC_IN_USE_TRANSFER", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6454 SOC15_REG_FIELD(TCC_EDC_CNT, IN_USE_TRANSFER_SED_COUNT),
6455 0, 0
6456 },
6457 { "TCC_LATENCY_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6458 SOC15_REG_FIELD(TCC_EDC_CNT, LATENCY_FIFO_SED_COUNT),
6459 0, 0
6460 },
6461 { "TCC_RETURN_DATA", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6462 SOC15_REG_FIELD(TCC_EDC_CNT, RETURN_DATA_SED_COUNT),
6463 0, 0
6464 },
6465 { "TCC_RETURN_CONTROL", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6466 SOC15_REG_FIELD(TCC_EDC_CNT, RETURN_CONTROL_SED_COUNT),
6467 0, 0
6468 },
6469 { "TCC_UC_ATOMIC_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6470 SOC15_REG_FIELD(TCC_EDC_CNT, UC_ATOMIC_FIFO_SED_COUNT),
6471 0, 0
6472 },
6473 { "TCC_WRITE_RETURN", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2),
6474 SOC15_REG_FIELD(TCC_EDC_CNT2, WRITE_RETURN_SED_COUNT),
6475 0, 0
6476 },
6477 { "TCC_WRITE_CACHE_READ", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2),
6478 SOC15_REG_FIELD(TCC_EDC_CNT2, WRITE_CACHE_READ_SED_COUNT),
6479 0, 0
6480 },
6481 { "TCC_SRC_FIFO_NEXT_RAM", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2),
6482 SOC15_REG_FIELD(TCC_EDC_CNT2, SRC_FIFO_NEXT_RAM_SED_COUNT),
6483 0, 0
6484 },
6485 { "TCC_LATENCY_FIFO_NEXT_RAM", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2),
6486 SOC15_REG_FIELD(TCC_EDC_CNT2, LATENCY_FIFO_NEXT_RAM_SED_COUNT),
6487 0, 0
6488 },
6489 { "TCC_CACHE_TAG_PROBE_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2),
6490 SOC15_REG_FIELD(TCC_EDC_CNT2, CACHE_TAG_PROBE_FIFO_SED_COUNT),
6491 0, 0
6492 },
6493 { "TCC_WRRET_TAG_WRITE_RETURN", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2),
6494 SOC15_REG_FIELD(TCC_EDC_CNT2, WRRET_TAG_WRITE_RETURN_SED_COUNT),
6495 0, 0
6496 },
6497 { "TCC_ATOMIC_RETURN_BUFFER", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2),
6498 SOC15_REG_FIELD(TCC_EDC_CNT2, ATOMIC_RETURN_BUFFER_SED_COUNT),
6499 0, 0
6500 },
6501 { "TCI_WRITE_RAM", SOC15_REG_ENTRY(GC, 0, mmTCI_EDC_CNT),
6502 SOC15_REG_FIELD(TCI_EDC_CNT, WRITE_RAM_SED_COUNT),
6503 0, 0
6504 },
6505 { "TCP_CACHE_RAM", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW),
6506 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, CACHE_RAM_SEC_COUNT),
6507 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, CACHE_RAM_DED_COUNT)
6508 },
6509 { "TCP_LFIFO_RAM", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW),
6510 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, LFIFO_RAM_SEC_COUNT),
6511 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, LFIFO_RAM_DED_COUNT)
6512 },
6513 { "TCP_CMD_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW),
6514 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, CMD_FIFO_SED_COUNT),
6515 0, 0
6516 },
6517 { "TCP_VM_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW),
6518 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, VM_FIFO_SEC_COUNT),
6519 0, 0
6520 },
6521 { "TCP_DB_RAM", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW),
6522 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, DB_RAM_SED_COUNT),
6523 0, 0
6524 },
6525 { "TCP_UTCL1_LFIFO0", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW),
6526 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, UTCL1_LFIFO0_SEC_COUNT),
6527 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, UTCL1_LFIFO0_DED_COUNT)
6528 },
6529 { "TCP_UTCL1_LFIFO1", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW),
6530 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, UTCL1_LFIFO1_SEC_COUNT),
6531 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, UTCL1_LFIFO1_DED_COUNT)
6532 },
6533 { "TD_SS_FIFO_LO", SOC15_REG_ENTRY(GC, 0, mmTD_EDC_CNT),
6534 SOC15_REG_FIELD(TD_EDC_CNT, SS_FIFO_LO_SEC_COUNT),
6535 SOC15_REG_FIELD(TD_EDC_CNT, SS_FIFO_LO_DED_COUNT)
6536 },
6537 { "TD_SS_FIFO_HI", SOC15_REG_ENTRY(GC, 0, mmTD_EDC_CNT),
6538 SOC15_REG_FIELD(TD_EDC_CNT, SS_FIFO_HI_SEC_COUNT),
6539 SOC15_REG_FIELD(TD_EDC_CNT, SS_FIFO_HI_DED_COUNT)
6540 },
6541 { "TD_CS_FIFO", SOC15_REG_ENTRY(GC, 0, mmTD_EDC_CNT),
6542 SOC15_REG_FIELD(TD_EDC_CNT, CS_FIFO_SED_COUNT),
6543 0, 0
6544 },
6545 { "SQ_LDS_D", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT),
6546 SOC15_REG_FIELD(SQ_EDC_CNT, LDS_D_SEC_COUNT),
6547 SOC15_REG_FIELD(SQ_EDC_CNT, LDS_D_DED_COUNT)
6548 },
6549 { "SQ_LDS_I", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT),
6550 SOC15_REG_FIELD(SQ_EDC_CNT, LDS_I_SEC_COUNT),
6551 SOC15_REG_FIELD(SQ_EDC_CNT, LDS_I_DED_COUNT)
6552 },
6553 { "SQ_SGPR", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT),
6554 SOC15_REG_FIELD(SQ_EDC_CNT, SGPR_SEC_COUNT),
6555 SOC15_REG_FIELD(SQ_EDC_CNT, SGPR_DED_COUNT)
6556 },
6557 { "SQ_VGPR0", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT),
6558 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR0_SEC_COUNT),
6559 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR0_DED_COUNT)
6560 },
6561 { "SQ_VGPR1", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT),
6562 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR1_SEC_COUNT),
6563 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR1_DED_COUNT)
6564 },
6565 { "SQ_VGPR2", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT),
6566 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR2_SEC_COUNT),
6567 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR2_DED_COUNT)
6568 },
6569 { "SQ_VGPR3", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT),
6570 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR3_SEC_COUNT),
6571 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR3_DED_COUNT)
6572 },
6573 { "SQC_DATA_CU0_WRITE_DATA_BUF", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT),
6574 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU0_WRITE_DATA_BUF_SEC_COUNT),
6575 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU0_WRITE_DATA_BUF_DED_COUNT)
6576 },
6577 { "SQC_DATA_CU0_UTCL1_LFIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT),
6578 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU0_UTCL1_LFIFO_SEC_COUNT),
6579 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU0_UTCL1_LFIFO_DED_COUNT)
6580 },
6581 { "SQC_DATA_CU1_WRITE_DATA_BUF", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT),
6582 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU1_WRITE_DATA_BUF_SEC_COUNT),
6583 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU1_WRITE_DATA_BUF_DED_COUNT)
6584 },
6585 { "SQC_DATA_CU1_UTCL1_LFIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT),
6586 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU1_UTCL1_LFIFO_SEC_COUNT),
6587 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU1_UTCL1_LFIFO_DED_COUNT)
6588 },
6589 { "SQC_DATA_CU2_WRITE_DATA_BUF", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT),
6590 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU2_WRITE_DATA_BUF_SEC_COUNT),
6591 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU2_WRITE_DATA_BUF_DED_COUNT)
6592 },
6593 { "SQC_DATA_CU2_UTCL1_LFIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT),
6594 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU2_UTCL1_LFIFO_SEC_COUNT),
6595 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU2_UTCL1_LFIFO_DED_COUNT)
6596 },
6597 { "SQC_INST_BANKA_TAG_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6598 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_TAG_RAM_SEC_COUNT),
6599 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_TAG_RAM_DED_COUNT)
6600 },
6601 { "SQC_INST_BANKA_BANK_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6602 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_BANK_RAM_SEC_COUNT),
6603 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_BANK_RAM_DED_COUNT)
6604 },
6605 { "SQC_DATA_BANKA_TAG_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6606 SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_TAG_RAM_SEC_COUNT),
6607 SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_TAG_RAM_DED_COUNT)
6608 },
6609 { "SQC_DATA_BANKA_BANK_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6610 SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_BANK_RAM_SEC_COUNT),
6611 SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_BANK_RAM_DED_COUNT)
6612 },
6613 { "SQC_INST_BANKA_UTCL1_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6614 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_UTCL1_MISS_FIFO_SED_COUNT),
6615 0, 0
6616 },
6617 { "SQC_INST_BANKA_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6618 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_MISS_FIFO_SED_COUNT),
6619 0, 0
6620 },
6621 { "SQC_DATA_BANKA_HIT_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6622 SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_HIT_FIFO_SED_COUNT),
6623 0, 0
6624 },
6625 { "SQC_DATA_BANKA_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6626 SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_MISS_FIFO_SED_COUNT),
6627 0, 0
6628 },
6629 { "SQC_DATA_BANKA_DIRTY_BIT_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6630 SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_DIRTY_BIT_RAM_SED_COUNT),
6631 0, 0
6632 },
6633 { "SQC_INST_UTCL1_LFIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6634 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_UTCL1_LFIFO_SEC_COUNT),
6635 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_UTCL1_LFIFO_DED_COUNT)
6636 },
6637 { "SQC_INST_BANKB_TAG_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6638 SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_TAG_RAM_SEC_COUNT),
6639 SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_TAG_RAM_DED_COUNT)
6640 },
6641 { "SQC_INST_BANKB_BANK_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6642 SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_BANK_RAM_SEC_COUNT),
6643 SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_BANK_RAM_DED_COUNT)
6644 },
6645 { "SQC_DATA_BANKB_TAG_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6646 SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_TAG_RAM_SEC_COUNT),
6647 SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_TAG_RAM_DED_COUNT)
6648 },
6649 { "SQC_DATA_BANKB_BANK_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6650 SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_BANK_RAM_SEC_COUNT),
6651 SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_BANK_RAM_DED_COUNT)
6652 },
6653 { "SQC_INST_BANKB_UTCL1_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6654 SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_UTCL1_MISS_FIFO_SED_COUNT),
6655 0, 0
6656 },
6657 { "SQC_INST_BANKB_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6658 SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_MISS_FIFO_SED_COUNT),
6659 0, 0
6660 },
6661 { "SQC_DATA_BANKB_HIT_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6662 SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_HIT_FIFO_SED_COUNT),
6663 0, 0
6664 },
6665 { "SQC_DATA_BANKB_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6666 SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_MISS_FIFO_SED_COUNT),
6667 0, 0
6668 },
6669 { "SQC_DATA_BANKB_DIRTY_BIT_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6670 SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_DIRTY_BIT_RAM_SED_COUNT),
6671 0, 0
6672 },
6673 { "EA_DRAMRD_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6674 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMRD_CMDMEM_SEC_COUNT),
6675 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMRD_CMDMEM_DED_COUNT)
6676 },
6677 { "EA_DRAMWR_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6678 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_CMDMEM_SEC_COUNT),
6679 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_CMDMEM_DED_COUNT)
6680 },
6681 { "EA_DRAMWR_DATAMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6682 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_DATAMEM_SEC_COUNT),
6683 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_DATAMEM_DED_COUNT)
6684 },
6685 { "EA_RRET_TAGMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6686 SOC15_REG_FIELD(GCEA_EDC_CNT, RRET_TAGMEM_SEC_COUNT),
6687 SOC15_REG_FIELD(GCEA_EDC_CNT, RRET_TAGMEM_DED_COUNT)
6688 },
6689 { "EA_WRET_TAGMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6690 SOC15_REG_FIELD(GCEA_EDC_CNT, WRET_TAGMEM_SEC_COUNT),
6691 SOC15_REG_FIELD(GCEA_EDC_CNT, WRET_TAGMEM_DED_COUNT)
6692 },
6693 { "EA_DRAMRD_PAGEMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6694 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMRD_PAGEMEM_SED_COUNT),
6695 0, 0
6696 },
6697 { "EA_DRAMWR_PAGEMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6698 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_PAGEMEM_SED_COUNT),
6699 0, 0
6700 },
6701 { "EA_IORD_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6702 SOC15_REG_FIELD(GCEA_EDC_CNT, IORD_CMDMEM_SED_COUNT),
6703 0, 0
6704 },
6705 { "EA_IOWR_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6706 SOC15_REG_FIELD(GCEA_EDC_CNT, IOWR_CMDMEM_SED_COUNT),
6707 0, 0
6708 },
6709 { "EA_IOWR_DATAMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6710 SOC15_REG_FIELD(GCEA_EDC_CNT, IOWR_DATAMEM_SED_COUNT),
6711 0, 0
6712 },
6713 { "GMIRD_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6714 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIRD_CMDMEM_SEC_COUNT),
6715 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIRD_CMDMEM_DED_COUNT)
6716 },
6717 { "GMIWR_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6718 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_CMDMEM_SEC_COUNT),
6719 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_CMDMEM_DED_COUNT)
6720 },
6721 { "GMIWR_DATAMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6722 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_DATAMEM_SEC_COUNT),
6723 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_DATAMEM_DED_COUNT)
6724 },
6725 { "GMIRD_PAGEMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6726 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIRD_PAGEMEM_SED_COUNT),
6727 0, 0
6728 },
6729 { "GMIWR_PAGEMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6730 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_PAGEMEM_SED_COUNT),
6731 0, 0
6732 },
6733 { "MAM_D0MEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6734 SOC15_REG_FIELD(GCEA_EDC_CNT2, MAM_D0MEM_SED_COUNT),
6735 0, 0
6736 },
6737 { "MAM_D1MEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6738 SOC15_REG_FIELD(GCEA_EDC_CNT2, MAM_D1MEM_SED_COUNT),
6739 0, 0
6740 },
6741 { "MAM_D2MEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6742 SOC15_REG_FIELD(GCEA_EDC_CNT2, MAM_D2MEM_SED_COUNT),
6743 0, 0
6744 },
6745 { "MAM_D3MEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6746 SOC15_REG_FIELD(GCEA_EDC_CNT2, MAM_D3MEM_SED_COUNT),
6747 0, 0
6748 }
6749 };
6750
gfx_v9_0_ras_error_inject(struct amdgpu_device * adev,void * inject_if,uint32_t instance_mask)6751 static int gfx_v9_0_ras_error_inject(struct amdgpu_device *adev,
6752 void *inject_if, uint32_t instance_mask)
6753 {
6754 struct ras_inject_if *info = (struct ras_inject_if *)inject_if;
6755 int ret;
6756 struct ta_ras_trigger_error_input block_info = { 0 };
6757
6758 if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX))
6759 return -EINVAL;
6760
6761 if (info->head.sub_block_index >= ARRAY_SIZE(ras_gfx_subblocks))
6762 return -EINVAL;
6763
6764 if (!ras_gfx_subblocks[info->head.sub_block_index].name)
6765 return -EPERM;
6766
6767 if (!(ras_gfx_subblocks[info->head.sub_block_index].hw_supported_error_type &
6768 info->head.type)) {
6769 DRM_ERROR("GFX Subblock %s, hardware do not support type 0x%x\n",
6770 ras_gfx_subblocks[info->head.sub_block_index].name,
6771 info->head.type);
6772 return -EPERM;
6773 }
6774
6775 if (!(ras_gfx_subblocks[info->head.sub_block_index].sw_supported_error_type &
6776 info->head.type)) {
6777 DRM_ERROR("GFX Subblock %s, driver do not support type 0x%x\n",
6778 ras_gfx_subblocks[info->head.sub_block_index].name,
6779 info->head.type);
6780 return -EPERM;
6781 }
6782
6783 block_info.block_id = amdgpu_ras_block_to_ta(info->head.block);
6784 block_info.sub_block_index =
6785 ras_gfx_subblocks[info->head.sub_block_index].ta_subblock;
6786 block_info.inject_error_type = amdgpu_ras_error_to_ta(info->head.type);
6787 block_info.address = info->address;
6788 block_info.value = info->value;
6789
6790 mutex_lock(&adev->grbm_idx_mutex);
6791 ret = psp_ras_trigger_error(&adev->psp, &block_info, instance_mask);
6792 mutex_unlock(&adev->grbm_idx_mutex);
6793
6794 return ret;
6795 }
6796
6797 static const char * const vml2_mems[] = {
6798 "UTC_VML2_BANK_CACHE_0_BIGK_MEM0",
6799 "UTC_VML2_BANK_CACHE_0_BIGK_MEM1",
6800 "UTC_VML2_BANK_CACHE_0_4K_MEM0",
6801 "UTC_VML2_BANK_CACHE_0_4K_MEM1",
6802 "UTC_VML2_BANK_CACHE_1_BIGK_MEM0",
6803 "UTC_VML2_BANK_CACHE_1_BIGK_MEM1",
6804 "UTC_VML2_BANK_CACHE_1_4K_MEM0",
6805 "UTC_VML2_BANK_CACHE_1_4K_MEM1",
6806 "UTC_VML2_BANK_CACHE_2_BIGK_MEM0",
6807 "UTC_VML2_BANK_CACHE_2_BIGK_MEM1",
6808 "UTC_VML2_BANK_CACHE_2_4K_MEM0",
6809 "UTC_VML2_BANK_CACHE_2_4K_MEM1",
6810 "UTC_VML2_BANK_CACHE_3_BIGK_MEM0",
6811 "UTC_VML2_BANK_CACHE_3_BIGK_MEM1",
6812 "UTC_VML2_BANK_CACHE_3_4K_MEM0",
6813 "UTC_VML2_BANK_CACHE_3_4K_MEM1",
6814 };
6815
6816 static const char * const vml2_walker_mems[] = {
6817 "UTC_VML2_CACHE_PDE0_MEM0",
6818 "UTC_VML2_CACHE_PDE0_MEM1",
6819 "UTC_VML2_CACHE_PDE1_MEM0",
6820 "UTC_VML2_CACHE_PDE1_MEM1",
6821 "UTC_VML2_CACHE_PDE2_MEM0",
6822 "UTC_VML2_CACHE_PDE2_MEM1",
6823 "UTC_VML2_RDIF_LOG_FIFO",
6824 };
6825
6826 static const char * const atc_l2_cache_2m_mems[] = {
6827 "UTC_ATCL2_CACHE_2M_BANK0_WAY0_MEM",
6828 "UTC_ATCL2_CACHE_2M_BANK0_WAY1_MEM",
6829 "UTC_ATCL2_CACHE_2M_BANK1_WAY0_MEM",
6830 "UTC_ATCL2_CACHE_2M_BANK1_WAY1_MEM",
6831 };
6832
6833 static const char *atc_l2_cache_4k_mems[] = {
6834 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM0",
6835 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM1",
6836 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM2",
6837 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM3",
6838 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM4",
6839 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM5",
6840 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM6",
6841 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM7",
6842 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM0",
6843 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM1",
6844 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM2",
6845 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM3",
6846 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM4",
6847 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM5",
6848 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM6",
6849 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM7",
6850 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM0",
6851 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM1",
6852 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM2",
6853 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM3",
6854 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM4",
6855 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM5",
6856 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM6",
6857 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM7",
6858 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM0",
6859 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM1",
6860 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM2",
6861 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM3",
6862 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM4",
6863 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM5",
6864 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM6",
6865 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM7",
6866 };
6867
gfx_v9_0_query_utc_edc_status(struct amdgpu_device * adev,struct ras_err_data * err_data)6868 static int gfx_v9_0_query_utc_edc_status(struct amdgpu_device *adev,
6869 struct ras_err_data *err_data)
6870 {
6871 uint32_t i, data;
6872 uint32_t sec_count, ded_count;
6873
6874 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, 255);
6875 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_CNT, 0);
6876 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, 255);
6877 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_CNT, 0);
6878 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, 255);
6879 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_CNT, 0);
6880 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, 255);
6881 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_CNT, 0);
6882
6883 for (i = 0; i < ARRAY_SIZE(vml2_mems); i++) {
6884 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, i);
6885 data = RREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_CNT);
6886
6887 sec_count = REG_GET_FIELD(data, VM_L2_MEM_ECC_CNT, SEC_COUNT);
6888 if (sec_count) {
6889 dev_info(adev->dev, "Instance[%d]: SubBlock %s, "
6890 "SEC %d\n", i, vml2_mems[i], sec_count);
6891 err_data->ce_count += sec_count;
6892 }
6893
6894 ded_count = REG_GET_FIELD(data, VM_L2_MEM_ECC_CNT, DED_COUNT);
6895 if (ded_count) {
6896 dev_info(adev->dev, "Instance[%d]: SubBlock %s, "
6897 "DED %d\n", i, vml2_mems[i], ded_count);
6898 err_data->ue_count += ded_count;
6899 }
6900 }
6901
6902 for (i = 0; i < ARRAY_SIZE(vml2_walker_mems); i++) {
6903 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, i);
6904 data = RREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_CNT);
6905
6906 sec_count = REG_GET_FIELD(data, VM_L2_WALKER_MEM_ECC_CNT,
6907 SEC_COUNT);
6908 if (sec_count) {
6909 dev_info(adev->dev, "Instance[%d]: SubBlock %s, "
6910 "SEC %d\n", i, vml2_walker_mems[i], sec_count);
6911 err_data->ce_count += sec_count;
6912 }
6913
6914 ded_count = REG_GET_FIELD(data, VM_L2_WALKER_MEM_ECC_CNT,
6915 DED_COUNT);
6916 if (ded_count) {
6917 dev_info(adev->dev, "Instance[%d]: SubBlock %s, "
6918 "DED %d\n", i, vml2_walker_mems[i], ded_count);
6919 err_data->ue_count += ded_count;
6920 }
6921 }
6922
6923 for (i = 0; i < ARRAY_SIZE(atc_l2_cache_2m_mems); i++) {
6924 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, i);
6925 data = RREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_CNT);
6926
6927 sec_count = (data & 0x00006000L) >> 0xd;
6928 if (sec_count) {
6929 dev_info(adev->dev, "Instance[%d]: SubBlock %s, "
6930 "SEC %d\n", i, atc_l2_cache_2m_mems[i],
6931 sec_count);
6932 err_data->ce_count += sec_count;
6933 }
6934 }
6935
6936 for (i = 0; i < ARRAY_SIZE(atc_l2_cache_4k_mems); i++) {
6937 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, i);
6938 data = RREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_CNT);
6939
6940 sec_count = (data & 0x00006000L) >> 0xd;
6941 if (sec_count) {
6942 dev_info(adev->dev, "Instance[%d]: SubBlock %s, "
6943 "SEC %d\n", i, atc_l2_cache_4k_mems[i],
6944 sec_count);
6945 err_data->ce_count += sec_count;
6946 }
6947
6948 ded_count = (data & 0x00018000L) >> 0xf;
6949 if (ded_count) {
6950 dev_info(adev->dev, "Instance[%d]: SubBlock %s, "
6951 "DED %d\n", i, atc_l2_cache_4k_mems[i],
6952 ded_count);
6953 err_data->ue_count += ded_count;
6954 }
6955 }
6956
6957 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, 255);
6958 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, 255);
6959 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, 255);
6960 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, 255);
6961
6962 return 0;
6963 }
6964
gfx_v9_0_ras_error_count(struct amdgpu_device * adev,const struct soc15_reg_entry * reg,uint32_t se_id,uint32_t inst_id,uint32_t value,uint32_t * sec_count,uint32_t * ded_count)6965 static int gfx_v9_0_ras_error_count(struct amdgpu_device *adev,
6966 const struct soc15_reg_entry *reg,
6967 uint32_t se_id, uint32_t inst_id, uint32_t value,
6968 uint32_t *sec_count, uint32_t *ded_count)
6969 {
6970 uint32_t i;
6971 uint32_t sec_cnt, ded_cnt;
6972
6973 for (i = 0; i < ARRAY_SIZE(gfx_v9_0_ras_fields); i++) {
6974 if(gfx_v9_0_ras_fields[i].reg_offset != reg->reg_offset ||
6975 gfx_v9_0_ras_fields[i].seg != reg->seg ||
6976 gfx_v9_0_ras_fields[i].inst != reg->inst)
6977 continue;
6978
6979 sec_cnt = (value &
6980 gfx_v9_0_ras_fields[i].sec_count_mask) >>
6981 gfx_v9_0_ras_fields[i].sec_count_shift;
6982 if (sec_cnt) {
6983 dev_info(adev->dev, "GFX SubBlock %s, "
6984 "Instance[%d][%d], SEC %d\n",
6985 gfx_v9_0_ras_fields[i].name,
6986 se_id, inst_id,
6987 sec_cnt);
6988 *sec_count += sec_cnt;
6989 }
6990
6991 ded_cnt = (value &
6992 gfx_v9_0_ras_fields[i].ded_count_mask) >>
6993 gfx_v9_0_ras_fields[i].ded_count_shift;
6994 if (ded_cnt) {
6995 dev_info(adev->dev, "GFX SubBlock %s, "
6996 "Instance[%d][%d], DED %d\n",
6997 gfx_v9_0_ras_fields[i].name,
6998 se_id, inst_id,
6999 ded_cnt);
7000 *ded_count += ded_cnt;
7001 }
7002 }
7003
7004 return 0;
7005 }
7006
gfx_v9_0_reset_ras_error_count(struct amdgpu_device * adev)7007 static void gfx_v9_0_reset_ras_error_count(struct amdgpu_device *adev)
7008 {
7009 int i, j, k;
7010
7011 if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX))
7012 return;
7013
7014 /* read back registers to clear the counters */
7015 mutex_lock(&adev->grbm_idx_mutex);
7016 for (i = 0; i < ARRAY_SIZE(gfx_v9_0_edc_counter_regs); i++) {
7017 for (j = 0; j < gfx_v9_0_edc_counter_regs[i].se_num; j++) {
7018 for (k = 0; k < gfx_v9_0_edc_counter_regs[i].instance; k++) {
7019 amdgpu_gfx_select_se_sh(adev, j, 0x0, k, 0);
7020 RREG32(SOC15_REG_ENTRY_OFFSET(gfx_v9_0_edc_counter_regs[i]));
7021 }
7022 }
7023 }
7024 WREG32_SOC15(GC, 0, mmGRBM_GFX_INDEX, 0xe0000000);
7025 mutex_unlock(&adev->grbm_idx_mutex);
7026
7027 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, 255);
7028 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_CNT, 0);
7029 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, 255);
7030 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_CNT, 0);
7031 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, 255);
7032 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_CNT, 0);
7033 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, 255);
7034 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_CNT, 0);
7035
7036 for (i = 0; i < ARRAY_SIZE(vml2_mems); i++) {
7037 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, i);
7038 RREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_CNT);
7039 }
7040
7041 for (i = 0; i < ARRAY_SIZE(vml2_walker_mems); i++) {
7042 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, i);
7043 RREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_CNT);
7044 }
7045
7046 for (i = 0; i < ARRAY_SIZE(atc_l2_cache_2m_mems); i++) {
7047 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, i);
7048 RREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_CNT);
7049 }
7050
7051 for (i = 0; i < ARRAY_SIZE(atc_l2_cache_4k_mems); i++) {
7052 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, i);
7053 RREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_CNT);
7054 }
7055
7056 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, 255);
7057 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, 255);
7058 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, 255);
7059 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, 255);
7060 }
7061
gfx_v9_0_query_ras_error_count(struct amdgpu_device * adev,void * ras_error_status)7062 static void gfx_v9_0_query_ras_error_count(struct amdgpu_device *adev,
7063 void *ras_error_status)
7064 {
7065 struct ras_err_data *err_data = (struct ras_err_data *)ras_error_status;
7066 uint32_t sec_count = 0, ded_count = 0;
7067 uint32_t i, j, k;
7068 uint32_t reg_value;
7069
7070 if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX))
7071 return;
7072
7073 err_data->ue_count = 0;
7074 err_data->ce_count = 0;
7075
7076 mutex_lock(&adev->grbm_idx_mutex);
7077
7078 for (i = 0; i < ARRAY_SIZE(gfx_v9_0_edc_counter_regs); i++) {
7079 for (j = 0; j < gfx_v9_0_edc_counter_regs[i].se_num; j++) {
7080 for (k = 0; k < gfx_v9_0_edc_counter_regs[i].instance; k++) {
7081 amdgpu_gfx_select_se_sh(adev, j, 0, k, 0);
7082 reg_value =
7083 RREG32(SOC15_REG_ENTRY_OFFSET(gfx_v9_0_edc_counter_regs[i]));
7084 if (reg_value)
7085 gfx_v9_0_ras_error_count(adev,
7086 &gfx_v9_0_edc_counter_regs[i],
7087 j, k, reg_value,
7088 &sec_count, &ded_count);
7089 }
7090 }
7091 }
7092
7093 err_data->ce_count += sec_count;
7094 err_data->ue_count += ded_count;
7095
7096 amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0);
7097 mutex_unlock(&adev->grbm_idx_mutex);
7098
7099 gfx_v9_0_query_utc_edc_status(adev, err_data);
7100 }
7101
gfx_v9_0_emit_mem_sync(struct amdgpu_ring * ring)7102 static void gfx_v9_0_emit_mem_sync(struct amdgpu_ring *ring)
7103 {
7104 const unsigned int cp_coher_cntl =
7105 PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_SH_ICACHE_ACTION_ENA(1) |
7106 PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_SH_KCACHE_ACTION_ENA(1) |
7107 PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_TC_ACTION_ENA(1) |
7108 PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_TCL1_ACTION_ENA(1) |
7109 PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_TC_WB_ACTION_ENA(1);
7110
7111 /* ACQUIRE_MEM -make one or more surfaces valid for use by the subsequent operations */
7112 amdgpu_ring_write(ring, PACKET3(PACKET3_ACQUIRE_MEM, 5));
7113 amdgpu_ring_write(ring, cp_coher_cntl); /* CP_COHER_CNTL */
7114 amdgpu_ring_write(ring, 0xffffffff); /* CP_COHER_SIZE */
7115 amdgpu_ring_write(ring, 0xffffff); /* CP_COHER_SIZE_HI */
7116 amdgpu_ring_write(ring, 0); /* CP_COHER_BASE */
7117 amdgpu_ring_write(ring, 0); /* CP_COHER_BASE_HI */
7118 amdgpu_ring_write(ring, 0x0000000A); /* POLL_INTERVAL */
7119 }
7120
gfx_v9_0_emit_wave_limit_cs(struct amdgpu_ring * ring,uint32_t pipe,bool enable)7121 static void gfx_v9_0_emit_wave_limit_cs(struct amdgpu_ring *ring,
7122 uint32_t pipe, bool enable)
7123 {
7124 struct amdgpu_device *adev = ring->adev;
7125 uint32_t val;
7126 uint32_t wcl_cs_reg;
7127
7128 /* mmSPI_WCL_PIPE_PERCENT_CS[0-7]_DEFAULT values are same */
7129 val = enable ? 0x1 : mmSPI_WCL_PIPE_PERCENT_CS0_DEFAULT;
7130
7131 switch (pipe) {
7132 case 0:
7133 wcl_cs_reg = SOC15_REG_OFFSET(GC, 0, mmSPI_WCL_PIPE_PERCENT_CS0);
7134 break;
7135 case 1:
7136 wcl_cs_reg = SOC15_REG_OFFSET(GC, 0, mmSPI_WCL_PIPE_PERCENT_CS1);
7137 break;
7138 case 2:
7139 wcl_cs_reg = SOC15_REG_OFFSET(GC, 0, mmSPI_WCL_PIPE_PERCENT_CS2);
7140 break;
7141 case 3:
7142 wcl_cs_reg = SOC15_REG_OFFSET(GC, 0, mmSPI_WCL_PIPE_PERCENT_CS3);
7143 break;
7144 default:
7145 DRM_DEBUG("invalid pipe %d\n", pipe);
7146 return;
7147 }
7148
7149 amdgpu_ring_emit_wreg(ring, wcl_cs_reg, val);
7150
7151 }
gfx_v9_0_emit_wave_limit(struct amdgpu_ring * ring,bool enable)7152 static void gfx_v9_0_emit_wave_limit(struct amdgpu_ring *ring, bool enable)
7153 {
7154 struct amdgpu_device *adev = ring->adev;
7155 uint32_t val;
7156 int i;
7157
7158
7159 /* mmSPI_WCL_PIPE_PERCENT_GFX is 7 bit multiplier register to limit
7160 * number of gfx waves. Setting 5 bit will make sure gfx only gets
7161 * around 25% of gpu resources.
7162 */
7163 val = enable ? 0x1f : mmSPI_WCL_PIPE_PERCENT_GFX_DEFAULT;
7164 amdgpu_ring_emit_wreg(ring,
7165 SOC15_REG_OFFSET(GC, 0, mmSPI_WCL_PIPE_PERCENT_GFX),
7166 val);
7167
7168 /* Restrict waves for normal/low priority compute queues as well
7169 * to get best QoS for high priority compute jobs.
7170 *
7171 * amdgpu controls only 1st ME(0-3 CS pipes).
7172 */
7173 for (i = 0; i < adev->gfx.mec.num_pipe_per_mec; i++) {
7174 if (i != ring->pipe)
7175 gfx_v9_0_emit_wave_limit_cs(ring, i, enable);
7176
7177 }
7178 }
7179
gfx_v9_ring_insert_nop(struct amdgpu_ring * ring,uint32_t num_nop)7180 static void gfx_v9_ring_insert_nop(struct amdgpu_ring *ring, uint32_t num_nop)
7181 {
7182 /* Header itself is a NOP packet */
7183 if (num_nop == 1) {
7184 amdgpu_ring_write(ring, ring->funcs->nop);
7185 return;
7186 }
7187
7188 /* Max HW optimization till 0x3ffe, followed by remaining one NOP at a time*/
7189 amdgpu_ring_write(ring, PACKET3(PACKET3_NOP, min(num_nop - 2, 0x3ffe)));
7190
7191 /* Header is at index 0, followed by num_nops - 1 NOP packet's */
7192 amdgpu_ring_insert_nop(ring, num_nop - 1);
7193 }
7194
gfx_v9_0_reset_kgq(struct amdgpu_ring * ring,unsigned int vmid)7195 static int gfx_v9_0_reset_kgq(struct amdgpu_ring *ring, unsigned int vmid)
7196 {
7197 struct amdgpu_device *adev = ring->adev;
7198 struct amdgpu_kiq *kiq = &adev->gfx.kiq[0];
7199 struct amdgpu_ring *kiq_ring = &kiq->ring;
7200 unsigned long flags;
7201 u32 tmp;
7202 int r;
7203
7204 if (amdgpu_sriov_vf(adev))
7205 return -EINVAL;
7206
7207 if (!kiq->pmf || !kiq->pmf->kiq_unmap_queues)
7208 return -EINVAL;
7209
7210 spin_lock_irqsave(&kiq->ring_lock, flags);
7211
7212 if (amdgpu_ring_alloc(kiq_ring, 5)) {
7213 spin_unlock_irqrestore(&kiq->ring_lock, flags);
7214 return -ENOMEM;
7215 }
7216
7217 tmp = REG_SET_FIELD(0, CP_VMID_RESET, RESET_REQUEST, 1 << vmid);
7218 gfx_v9_0_ring_emit_wreg(kiq_ring,
7219 SOC15_REG_OFFSET(GC, 0, mmCP_VMID_RESET), tmp);
7220 amdgpu_ring_commit(kiq_ring);
7221
7222 spin_unlock_irqrestore(&kiq->ring_lock, flags);
7223
7224 r = amdgpu_ring_test_ring(kiq_ring);
7225 if (r)
7226 return r;
7227
7228 if (amdgpu_ring_alloc(ring, 7 + 7 + 5))
7229 return -ENOMEM;
7230 gfx_v9_0_ring_emit_fence(ring, ring->fence_drv.gpu_addr,
7231 ring->fence_drv.sync_seq, AMDGPU_FENCE_FLAG_EXEC);
7232 gfx_v9_0_ring_emit_reg_wait(ring,
7233 SOC15_REG_OFFSET(GC, 0, mmCP_VMID_RESET), 0, 0xffff);
7234 gfx_v9_0_ring_emit_wreg(ring,
7235 SOC15_REG_OFFSET(GC, 0, mmCP_VMID_RESET), 0);
7236
7237 return amdgpu_ring_test_ring(ring);
7238 }
7239
gfx_v9_0_reset_kcq(struct amdgpu_ring * ring,unsigned int vmid)7240 static int gfx_v9_0_reset_kcq(struct amdgpu_ring *ring,
7241 unsigned int vmid)
7242 {
7243 struct amdgpu_device *adev = ring->adev;
7244 struct amdgpu_kiq *kiq = &adev->gfx.kiq[0];
7245 struct amdgpu_ring *kiq_ring = &kiq->ring;
7246 unsigned long flags;
7247 int i, r;
7248
7249 if (amdgpu_sriov_vf(adev))
7250 return -EINVAL;
7251
7252 if (!kiq->pmf || !kiq->pmf->kiq_unmap_queues)
7253 return -EINVAL;
7254
7255 spin_lock_irqsave(&kiq->ring_lock, flags);
7256
7257 if (amdgpu_ring_alloc(kiq_ring, kiq->pmf->unmap_queues_size)) {
7258 spin_unlock_irqrestore(&kiq->ring_lock, flags);
7259 return -ENOMEM;
7260 }
7261
7262 kiq->pmf->kiq_unmap_queues(kiq_ring, ring, RESET_QUEUES,
7263 0, 0);
7264 amdgpu_ring_commit(kiq_ring);
7265
7266 spin_unlock_irqrestore(&kiq->ring_lock, flags);
7267
7268 r = amdgpu_ring_test_ring(kiq_ring);
7269 if (r)
7270 return r;
7271
7272 /* make sure dequeue is complete*/
7273 amdgpu_gfx_rlc_enter_safe_mode(adev, 0);
7274 mutex_lock(&adev->srbm_mutex);
7275 soc15_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0, 0);
7276 for (i = 0; i < adev->usec_timeout; i++) {
7277 if (!(RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1))
7278 break;
7279 udelay(1);
7280 }
7281 if (i >= adev->usec_timeout)
7282 r = -ETIMEDOUT;
7283 soc15_grbm_select(adev, 0, 0, 0, 0, 0);
7284 mutex_unlock(&adev->srbm_mutex);
7285 amdgpu_gfx_rlc_exit_safe_mode(adev, 0);
7286 if (r) {
7287 dev_err(adev->dev, "fail to wait on hqd deactive\n");
7288 return r;
7289 }
7290
7291 r = gfx_v9_0_kcq_init_queue(ring, true);
7292 if (r) {
7293 dev_err(adev->dev, "fail to init kcq\n");
7294 return r;
7295 }
7296 spin_lock_irqsave(&kiq->ring_lock, flags);
7297 r = amdgpu_ring_alloc(kiq_ring, kiq->pmf->map_queues_size);
7298 if (r) {
7299 spin_unlock_irqrestore(&kiq->ring_lock, flags);
7300 return -ENOMEM;
7301 }
7302 kiq->pmf->kiq_map_queues(kiq_ring, ring);
7303 amdgpu_ring_commit(kiq_ring);
7304 spin_unlock_irqrestore(&kiq->ring_lock, flags);
7305 r = amdgpu_ring_test_ring(kiq_ring);
7306 if (r) {
7307 DRM_ERROR("fail to remap queue\n");
7308 return r;
7309 }
7310 return amdgpu_ring_test_ring(ring);
7311 }
7312
gfx_v9_ip_print(struct amdgpu_ip_block * ip_block,struct drm_printer * p)7313 static void gfx_v9_ip_print(struct amdgpu_ip_block *ip_block, struct drm_printer *p)
7314 {
7315 struct amdgpu_device *adev = ip_block->adev;
7316 uint32_t i, j, k, reg, index = 0;
7317 uint32_t reg_count = ARRAY_SIZE(gc_reg_list_9);
7318
7319 if (!adev->gfx.ip_dump_core)
7320 return;
7321
7322 for (i = 0; i < reg_count; i++)
7323 drm_printf(p, "%-50s \t 0x%08x\n",
7324 gc_reg_list_9[i].reg_name,
7325 adev->gfx.ip_dump_core[i]);
7326
7327 /* print compute queue registers for all instances */
7328 if (!adev->gfx.ip_dump_compute_queues)
7329 return;
7330
7331 reg_count = ARRAY_SIZE(gc_cp_reg_list_9);
7332 drm_printf(p, "\nnum_mec: %d num_pipe: %d num_queue: %d\n",
7333 adev->gfx.mec.num_mec,
7334 adev->gfx.mec.num_pipe_per_mec,
7335 adev->gfx.mec.num_queue_per_pipe);
7336
7337 for (i = 0; i < adev->gfx.mec.num_mec; i++) {
7338 for (j = 0; j < adev->gfx.mec.num_pipe_per_mec; j++) {
7339 for (k = 0; k < adev->gfx.mec.num_queue_per_pipe; k++) {
7340 drm_printf(p, "\nmec %d, pipe %d, queue %d\n", i, j, k);
7341 for (reg = 0; reg < reg_count; reg++) {
7342 drm_printf(p, "%-50s \t 0x%08x\n",
7343 gc_cp_reg_list_9[reg].reg_name,
7344 adev->gfx.ip_dump_compute_queues[index + reg]);
7345 }
7346 index += reg_count;
7347 }
7348 }
7349 }
7350
7351 }
7352
gfx_v9_ip_dump(struct amdgpu_ip_block * ip_block)7353 static void gfx_v9_ip_dump(struct amdgpu_ip_block *ip_block)
7354 {
7355 struct amdgpu_device *adev = ip_block->adev;
7356 uint32_t i, j, k, reg, index = 0;
7357 uint32_t reg_count = ARRAY_SIZE(gc_reg_list_9);
7358
7359 if (!adev->gfx.ip_dump_core || !adev->gfx.num_gfx_rings)
7360 return;
7361
7362 amdgpu_gfx_off_ctrl(adev, false);
7363 for (i = 0; i < reg_count; i++)
7364 adev->gfx.ip_dump_core[i] = RREG32(SOC15_REG_ENTRY_OFFSET(gc_reg_list_9[i]));
7365 amdgpu_gfx_off_ctrl(adev, true);
7366
7367 /* dump compute queue registers for all instances */
7368 if (!adev->gfx.ip_dump_compute_queues)
7369 return;
7370
7371 reg_count = ARRAY_SIZE(gc_cp_reg_list_9);
7372 amdgpu_gfx_off_ctrl(adev, false);
7373 mutex_lock(&adev->srbm_mutex);
7374 for (i = 0; i < adev->gfx.mec.num_mec; i++) {
7375 for (j = 0; j < adev->gfx.mec.num_pipe_per_mec; j++) {
7376 for (k = 0; k < adev->gfx.mec.num_queue_per_pipe; k++) {
7377 /* ME0 is for GFX so start from 1 for CP */
7378 soc15_grbm_select(adev, 1 + i, j, k, 0, 0);
7379
7380 for (reg = 0; reg < reg_count; reg++) {
7381 adev->gfx.ip_dump_compute_queues[index + reg] =
7382 RREG32(SOC15_REG_ENTRY_OFFSET(
7383 gc_cp_reg_list_9[reg]));
7384 }
7385 index += reg_count;
7386 }
7387 }
7388 }
7389 soc15_grbm_select(adev, 0, 0, 0, 0, 0);
7390 mutex_unlock(&adev->srbm_mutex);
7391 amdgpu_gfx_off_ctrl(adev, true);
7392
7393 }
7394
gfx_v9_0_ring_emit_cleaner_shader(struct amdgpu_ring * ring)7395 static void gfx_v9_0_ring_emit_cleaner_shader(struct amdgpu_ring *ring)
7396 {
7397 /* Emit the cleaner shader */
7398 amdgpu_ring_write(ring, PACKET3(PACKET3_RUN_CLEANER_SHADER, 0));
7399 amdgpu_ring_write(ring, 0); /* RESERVED field, programmed to zero */
7400 }
7401
gfx_v9_0_ring_begin_use_compute(struct amdgpu_ring * ring)7402 static void gfx_v9_0_ring_begin_use_compute(struct amdgpu_ring *ring)
7403 {
7404 struct amdgpu_device *adev = ring->adev;
7405 struct amdgpu_ip_block *gfx_block =
7406 amdgpu_device_ip_get_ip_block(adev, AMD_IP_BLOCK_TYPE_GFX);
7407
7408 amdgpu_gfx_enforce_isolation_ring_begin_use(ring);
7409
7410 /* Raven and PCO APUs seem to have stability issues
7411 * with compute and gfxoff and gfx pg. Disable gfx pg during
7412 * submission and allow again afterwards.
7413 */
7414 if (gfx_block && amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 1, 0))
7415 gfx_v9_0_set_powergating_state(gfx_block, AMD_PG_STATE_UNGATE);
7416 }
7417
gfx_v9_0_ring_end_use_compute(struct amdgpu_ring * ring)7418 static void gfx_v9_0_ring_end_use_compute(struct amdgpu_ring *ring)
7419 {
7420 struct amdgpu_device *adev = ring->adev;
7421 struct amdgpu_ip_block *gfx_block =
7422 amdgpu_device_ip_get_ip_block(adev, AMD_IP_BLOCK_TYPE_GFX);
7423
7424 /* Raven and PCO APUs seem to have stability issues
7425 * with compute and gfxoff and gfx pg. Disable gfx pg during
7426 * submission and allow again afterwards.
7427 */
7428 if (gfx_block && amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 1, 0))
7429 gfx_v9_0_set_powergating_state(gfx_block, AMD_PG_STATE_GATE);
7430
7431 amdgpu_gfx_enforce_isolation_ring_end_use(ring);
7432 }
7433
7434 static const struct amd_ip_funcs gfx_v9_0_ip_funcs = {
7435 .name = "gfx_v9_0",
7436 .early_init = gfx_v9_0_early_init,
7437 .late_init = gfx_v9_0_late_init,
7438 .sw_init = gfx_v9_0_sw_init,
7439 .sw_fini = gfx_v9_0_sw_fini,
7440 .hw_init = gfx_v9_0_hw_init,
7441 .hw_fini = gfx_v9_0_hw_fini,
7442 .suspend = gfx_v9_0_suspend,
7443 .resume = gfx_v9_0_resume,
7444 .is_idle = gfx_v9_0_is_idle,
7445 .wait_for_idle = gfx_v9_0_wait_for_idle,
7446 .soft_reset = gfx_v9_0_soft_reset,
7447 .set_clockgating_state = gfx_v9_0_set_clockgating_state,
7448 .set_powergating_state = gfx_v9_0_set_powergating_state,
7449 .get_clockgating_state = gfx_v9_0_get_clockgating_state,
7450 .dump_ip_state = gfx_v9_ip_dump,
7451 .print_ip_state = gfx_v9_ip_print,
7452 };
7453
7454 static const struct amdgpu_ring_funcs gfx_v9_0_ring_funcs_gfx = {
7455 .type = AMDGPU_RING_TYPE_GFX,
7456 .align_mask = 0xff,
7457 .nop = PACKET3(PACKET3_NOP, 0x3FFF),
7458 .support_64bit_ptrs = true,
7459 .secure_submission_supported = true,
7460 .get_rptr = gfx_v9_0_ring_get_rptr_gfx,
7461 .get_wptr = gfx_v9_0_ring_get_wptr_gfx,
7462 .set_wptr = gfx_v9_0_ring_set_wptr_gfx,
7463 .emit_frame_size = /* totally 242 maximum if 16 IBs */
7464 5 + /* COND_EXEC */
7465 7 + /* PIPELINE_SYNC */
7466 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 +
7467 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 +
7468 2 + /* VM_FLUSH */
7469 8 + /* FENCE for VM_FLUSH */
7470 20 + /* GDS switch */
7471 4 + /* double SWITCH_BUFFER,
7472 the first COND_EXEC jump to the place just
7473 prior to this double SWITCH_BUFFER */
7474 5 + /* COND_EXEC */
7475 7 + /* HDP_flush */
7476 4 + /* VGT_flush */
7477 14 + /* CE_META */
7478 31 + /* DE_META */
7479 3 + /* CNTX_CTRL */
7480 5 + /* HDP_INVL */
7481 8 + 8 + /* FENCE x2 */
7482 2 + /* SWITCH_BUFFER */
7483 7 + /* gfx_v9_0_emit_mem_sync */
7484 2, /* gfx_v9_0_ring_emit_cleaner_shader */
7485 .emit_ib_size = 4, /* gfx_v9_0_ring_emit_ib_gfx */
7486 .emit_ib = gfx_v9_0_ring_emit_ib_gfx,
7487 .emit_fence = gfx_v9_0_ring_emit_fence,
7488 .emit_pipeline_sync = gfx_v9_0_ring_emit_pipeline_sync,
7489 .emit_vm_flush = gfx_v9_0_ring_emit_vm_flush,
7490 .emit_gds_switch = gfx_v9_0_ring_emit_gds_switch,
7491 .emit_hdp_flush = gfx_v9_0_ring_emit_hdp_flush,
7492 .test_ring = gfx_v9_0_ring_test_ring,
7493 .insert_nop = gfx_v9_ring_insert_nop,
7494 .pad_ib = amdgpu_ring_generic_pad_ib,
7495 .emit_switch_buffer = gfx_v9_ring_emit_sb,
7496 .emit_cntxcntl = gfx_v9_ring_emit_cntxcntl,
7497 .init_cond_exec = gfx_v9_0_ring_emit_init_cond_exec,
7498 .preempt_ib = gfx_v9_0_ring_preempt_ib,
7499 .emit_frame_cntl = gfx_v9_0_ring_emit_frame_cntl,
7500 .emit_wreg = gfx_v9_0_ring_emit_wreg,
7501 .emit_reg_wait = gfx_v9_0_ring_emit_reg_wait,
7502 .emit_reg_write_reg_wait = gfx_v9_0_ring_emit_reg_write_reg_wait,
7503 .soft_recovery = gfx_v9_0_ring_soft_recovery,
7504 .emit_mem_sync = gfx_v9_0_emit_mem_sync,
7505 .reset = gfx_v9_0_reset_kgq,
7506 .emit_cleaner_shader = gfx_v9_0_ring_emit_cleaner_shader,
7507 .begin_use = amdgpu_gfx_enforce_isolation_ring_begin_use,
7508 .end_use = amdgpu_gfx_enforce_isolation_ring_end_use,
7509 };
7510
7511 static const struct amdgpu_ring_funcs gfx_v9_0_sw_ring_funcs_gfx = {
7512 .type = AMDGPU_RING_TYPE_GFX,
7513 .align_mask = 0xff,
7514 .nop = PACKET3(PACKET3_NOP, 0x3FFF),
7515 .support_64bit_ptrs = true,
7516 .secure_submission_supported = true,
7517 .get_rptr = amdgpu_sw_ring_get_rptr_gfx,
7518 .get_wptr = amdgpu_sw_ring_get_wptr_gfx,
7519 .set_wptr = amdgpu_sw_ring_set_wptr_gfx,
7520 .emit_frame_size = /* totally 242 maximum if 16 IBs */
7521 5 + /* COND_EXEC */
7522 7 + /* PIPELINE_SYNC */
7523 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 +
7524 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 +
7525 2 + /* VM_FLUSH */
7526 8 + /* FENCE for VM_FLUSH */
7527 20 + /* GDS switch */
7528 4 + /* double SWITCH_BUFFER,
7529 * the first COND_EXEC jump to the place just
7530 * prior to this double SWITCH_BUFFER
7531 */
7532 5 + /* COND_EXEC */
7533 7 + /* HDP_flush */
7534 4 + /* VGT_flush */
7535 14 + /* CE_META */
7536 31 + /* DE_META */
7537 3 + /* CNTX_CTRL */
7538 5 + /* HDP_INVL */
7539 8 + 8 + /* FENCE x2 */
7540 2 + /* SWITCH_BUFFER */
7541 7 + /* gfx_v9_0_emit_mem_sync */
7542 2, /* gfx_v9_0_ring_emit_cleaner_shader */
7543 .emit_ib_size = 4, /* gfx_v9_0_ring_emit_ib_gfx */
7544 .emit_ib = gfx_v9_0_ring_emit_ib_gfx,
7545 .emit_fence = gfx_v9_0_ring_emit_fence,
7546 .emit_pipeline_sync = gfx_v9_0_ring_emit_pipeline_sync,
7547 .emit_vm_flush = gfx_v9_0_ring_emit_vm_flush,
7548 .emit_gds_switch = gfx_v9_0_ring_emit_gds_switch,
7549 .emit_hdp_flush = gfx_v9_0_ring_emit_hdp_flush,
7550 .test_ring = gfx_v9_0_ring_test_ring,
7551 .test_ib = gfx_v9_0_ring_test_ib,
7552 .insert_nop = gfx_v9_ring_insert_nop,
7553 .pad_ib = amdgpu_ring_generic_pad_ib,
7554 .emit_switch_buffer = gfx_v9_ring_emit_sb,
7555 .emit_cntxcntl = gfx_v9_ring_emit_cntxcntl,
7556 .init_cond_exec = gfx_v9_0_ring_emit_init_cond_exec,
7557 .emit_frame_cntl = gfx_v9_0_ring_emit_frame_cntl,
7558 .emit_wreg = gfx_v9_0_ring_emit_wreg,
7559 .emit_reg_wait = gfx_v9_0_ring_emit_reg_wait,
7560 .emit_reg_write_reg_wait = gfx_v9_0_ring_emit_reg_write_reg_wait,
7561 .soft_recovery = gfx_v9_0_ring_soft_recovery,
7562 .emit_mem_sync = gfx_v9_0_emit_mem_sync,
7563 .patch_cntl = gfx_v9_0_ring_patch_cntl,
7564 .patch_de = gfx_v9_0_ring_patch_de_meta,
7565 .patch_ce = gfx_v9_0_ring_patch_ce_meta,
7566 .emit_cleaner_shader = gfx_v9_0_ring_emit_cleaner_shader,
7567 .begin_use = amdgpu_gfx_enforce_isolation_ring_begin_use,
7568 .end_use = amdgpu_gfx_enforce_isolation_ring_end_use,
7569 };
7570
7571 static const struct amdgpu_ring_funcs gfx_v9_0_ring_funcs_compute = {
7572 .type = AMDGPU_RING_TYPE_COMPUTE,
7573 .align_mask = 0xff,
7574 .nop = PACKET3(PACKET3_NOP, 0x3FFF),
7575 .support_64bit_ptrs = true,
7576 .get_rptr = gfx_v9_0_ring_get_rptr_compute,
7577 .get_wptr = gfx_v9_0_ring_get_wptr_compute,
7578 .set_wptr = gfx_v9_0_ring_set_wptr_compute,
7579 .emit_frame_size =
7580 20 + /* gfx_v9_0_ring_emit_gds_switch */
7581 7 + /* gfx_v9_0_ring_emit_hdp_flush */
7582 5 + /* hdp invalidate */
7583 7 + /* gfx_v9_0_ring_emit_pipeline_sync */
7584 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 +
7585 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 +
7586 8 + 8 + 8 + /* gfx_v9_0_ring_emit_fence x3 for user fence, vm fence */
7587 7 + /* gfx_v9_0_emit_mem_sync */
7588 5 + /* gfx_v9_0_emit_wave_limit for updating mmSPI_WCL_PIPE_PERCENT_GFX register */
7589 15 + /* for updating 3 mmSPI_WCL_PIPE_PERCENT_CS registers */
7590 2, /* gfx_v9_0_ring_emit_cleaner_shader */
7591 .emit_ib_size = 7, /* gfx_v9_0_ring_emit_ib_compute */
7592 .emit_ib = gfx_v9_0_ring_emit_ib_compute,
7593 .emit_fence = gfx_v9_0_ring_emit_fence,
7594 .emit_pipeline_sync = gfx_v9_0_ring_emit_pipeline_sync,
7595 .emit_vm_flush = gfx_v9_0_ring_emit_vm_flush,
7596 .emit_gds_switch = gfx_v9_0_ring_emit_gds_switch,
7597 .emit_hdp_flush = gfx_v9_0_ring_emit_hdp_flush,
7598 .test_ring = gfx_v9_0_ring_test_ring,
7599 .test_ib = gfx_v9_0_ring_test_ib,
7600 .insert_nop = gfx_v9_ring_insert_nop,
7601 .pad_ib = amdgpu_ring_generic_pad_ib,
7602 .emit_wreg = gfx_v9_0_ring_emit_wreg,
7603 .emit_reg_wait = gfx_v9_0_ring_emit_reg_wait,
7604 .emit_reg_write_reg_wait = gfx_v9_0_ring_emit_reg_write_reg_wait,
7605 .soft_recovery = gfx_v9_0_ring_soft_recovery,
7606 .emit_mem_sync = gfx_v9_0_emit_mem_sync,
7607 .emit_wave_limit = gfx_v9_0_emit_wave_limit,
7608 .reset = gfx_v9_0_reset_kcq,
7609 .emit_cleaner_shader = gfx_v9_0_ring_emit_cleaner_shader,
7610 .begin_use = gfx_v9_0_ring_begin_use_compute,
7611 .end_use = gfx_v9_0_ring_end_use_compute,
7612 };
7613
7614 static const struct amdgpu_ring_funcs gfx_v9_0_ring_funcs_kiq = {
7615 .type = AMDGPU_RING_TYPE_KIQ,
7616 .align_mask = 0xff,
7617 .nop = PACKET3(PACKET3_NOP, 0x3FFF),
7618 .support_64bit_ptrs = true,
7619 .get_rptr = gfx_v9_0_ring_get_rptr_compute,
7620 .get_wptr = gfx_v9_0_ring_get_wptr_compute,
7621 .set_wptr = gfx_v9_0_ring_set_wptr_compute,
7622 .emit_frame_size =
7623 20 + /* gfx_v9_0_ring_emit_gds_switch */
7624 7 + /* gfx_v9_0_ring_emit_hdp_flush */
7625 5 + /* hdp invalidate */
7626 7 + /* gfx_v9_0_ring_emit_pipeline_sync */
7627 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 +
7628 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 +
7629 8 + 8 + 8, /* gfx_v9_0_ring_emit_fence_kiq x3 for user fence, vm fence */
7630 .emit_ib_size = 7, /* gfx_v9_0_ring_emit_ib_compute */
7631 .emit_fence = gfx_v9_0_ring_emit_fence_kiq,
7632 .test_ring = gfx_v9_0_ring_test_ring,
7633 .insert_nop = amdgpu_ring_insert_nop,
7634 .pad_ib = amdgpu_ring_generic_pad_ib,
7635 .emit_rreg = gfx_v9_0_ring_emit_rreg,
7636 .emit_wreg = gfx_v9_0_ring_emit_wreg,
7637 .emit_reg_wait = gfx_v9_0_ring_emit_reg_wait,
7638 .emit_reg_write_reg_wait = gfx_v9_0_ring_emit_reg_write_reg_wait,
7639 };
7640
gfx_v9_0_set_ring_funcs(struct amdgpu_device * adev)7641 static void gfx_v9_0_set_ring_funcs(struct amdgpu_device *adev)
7642 {
7643 int i;
7644
7645 adev->gfx.kiq[0].ring.funcs = &gfx_v9_0_ring_funcs_kiq;
7646
7647 for (i = 0; i < adev->gfx.num_gfx_rings; i++)
7648 adev->gfx.gfx_ring[i].funcs = &gfx_v9_0_ring_funcs_gfx;
7649
7650 if (adev->gfx.mcbp && adev->gfx.num_gfx_rings) {
7651 for (i = 0; i < GFX9_NUM_SW_GFX_RINGS; i++)
7652 adev->gfx.sw_gfx_ring[i].funcs = &gfx_v9_0_sw_ring_funcs_gfx;
7653 }
7654
7655 for (i = 0; i < adev->gfx.num_compute_rings; i++)
7656 adev->gfx.compute_ring[i].funcs = &gfx_v9_0_ring_funcs_compute;
7657 }
7658
7659 static const struct amdgpu_irq_src_funcs gfx_v9_0_eop_irq_funcs = {
7660 .set = gfx_v9_0_set_eop_interrupt_state,
7661 .process = gfx_v9_0_eop_irq,
7662 };
7663
7664 static const struct amdgpu_irq_src_funcs gfx_v9_0_priv_reg_irq_funcs = {
7665 .set = gfx_v9_0_set_priv_reg_fault_state,
7666 .process = gfx_v9_0_priv_reg_irq,
7667 };
7668
7669 static const struct amdgpu_irq_src_funcs gfx_v9_0_bad_op_irq_funcs = {
7670 .set = gfx_v9_0_set_bad_op_fault_state,
7671 .process = gfx_v9_0_bad_op_irq,
7672 };
7673
7674 static const struct amdgpu_irq_src_funcs gfx_v9_0_priv_inst_irq_funcs = {
7675 .set = gfx_v9_0_set_priv_inst_fault_state,
7676 .process = gfx_v9_0_priv_inst_irq,
7677 };
7678
7679 static const struct amdgpu_irq_src_funcs gfx_v9_0_cp_ecc_error_irq_funcs = {
7680 .set = gfx_v9_0_set_cp_ecc_error_state,
7681 .process = amdgpu_gfx_cp_ecc_error_irq,
7682 };
7683
7684
gfx_v9_0_set_irq_funcs(struct amdgpu_device * adev)7685 static void gfx_v9_0_set_irq_funcs(struct amdgpu_device *adev)
7686 {
7687 adev->gfx.eop_irq.num_types = AMDGPU_CP_IRQ_LAST;
7688 adev->gfx.eop_irq.funcs = &gfx_v9_0_eop_irq_funcs;
7689
7690 adev->gfx.priv_reg_irq.num_types = 1;
7691 adev->gfx.priv_reg_irq.funcs = &gfx_v9_0_priv_reg_irq_funcs;
7692
7693 adev->gfx.bad_op_irq.num_types = 1;
7694 adev->gfx.bad_op_irq.funcs = &gfx_v9_0_bad_op_irq_funcs;
7695
7696 adev->gfx.priv_inst_irq.num_types = 1;
7697 adev->gfx.priv_inst_irq.funcs = &gfx_v9_0_priv_inst_irq_funcs;
7698
7699 adev->gfx.cp_ecc_error_irq.num_types = 2; /*C5 ECC error and C9 FUE error*/
7700 adev->gfx.cp_ecc_error_irq.funcs = &gfx_v9_0_cp_ecc_error_irq_funcs;
7701 }
7702
gfx_v9_0_set_rlc_funcs(struct amdgpu_device * adev)7703 static void gfx_v9_0_set_rlc_funcs(struct amdgpu_device *adev)
7704 {
7705 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
7706 case IP_VERSION(9, 0, 1):
7707 case IP_VERSION(9, 2, 1):
7708 case IP_VERSION(9, 4, 0):
7709 case IP_VERSION(9, 2, 2):
7710 case IP_VERSION(9, 1, 0):
7711 case IP_VERSION(9, 4, 1):
7712 case IP_VERSION(9, 3, 0):
7713 case IP_VERSION(9, 4, 2):
7714 adev->gfx.rlc.funcs = &gfx_v9_0_rlc_funcs;
7715 break;
7716 default:
7717 break;
7718 }
7719 }
7720
gfx_v9_0_set_gds_init(struct amdgpu_device * adev)7721 static void gfx_v9_0_set_gds_init(struct amdgpu_device *adev)
7722 {
7723 /* init asci gds info */
7724 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
7725 case IP_VERSION(9, 0, 1):
7726 case IP_VERSION(9, 2, 1):
7727 case IP_VERSION(9, 4, 0):
7728 adev->gds.gds_size = 0x10000;
7729 break;
7730 case IP_VERSION(9, 2, 2):
7731 case IP_VERSION(9, 1, 0):
7732 case IP_VERSION(9, 4, 1):
7733 adev->gds.gds_size = 0x1000;
7734 break;
7735 case IP_VERSION(9, 4, 2):
7736 /* aldebaran removed all the GDS internal memory,
7737 * only support GWS opcode in kernel, like barrier
7738 * semaphore.etc */
7739 adev->gds.gds_size = 0;
7740 break;
7741 default:
7742 adev->gds.gds_size = 0x10000;
7743 break;
7744 }
7745
7746 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
7747 case IP_VERSION(9, 0, 1):
7748 case IP_VERSION(9, 4, 0):
7749 adev->gds.gds_compute_max_wave_id = 0x7ff;
7750 break;
7751 case IP_VERSION(9, 2, 1):
7752 adev->gds.gds_compute_max_wave_id = 0x27f;
7753 break;
7754 case IP_VERSION(9, 2, 2):
7755 case IP_VERSION(9, 1, 0):
7756 if (adev->apu_flags & AMD_APU_IS_RAVEN2)
7757 adev->gds.gds_compute_max_wave_id = 0x77; /* raven2 */
7758 else
7759 adev->gds.gds_compute_max_wave_id = 0x15f; /* raven1 */
7760 break;
7761 case IP_VERSION(9, 4, 1):
7762 adev->gds.gds_compute_max_wave_id = 0xfff;
7763 break;
7764 case IP_VERSION(9, 4, 2):
7765 /* deprecated for Aldebaran, no usage at all */
7766 adev->gds.gds_compute_max_wave_id = 0;
7767 break;
7768 default:
7769 /* this really depends on the chip */
7770 adev->gds.gds_compute_max_wave_id = 0x7ff;
7771 break;
7772 }
7773
7774 adev->gds.gws_size = 64;
7775 adev->gds.oa_size = 16;
7776 }
7777
gfx_v9_0_set_user_cu_inactive_bitmap(struct amdgpu_device * adev,u32 bitmap)7778 static void gfx_v9_0_set_user_cu_inactive_bitmap(struct amdgpu_device *adev,
7779 u32 bitmap)
7780 {
7781 u32 data;
7782
7783 if (!bitmap)
7784 return;
7785
7786 data = bitmap << GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_CUS__SHIFT;
7787 data &= GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_CUS_MASK;
7788
7789 WREG32_SOC15(GC, 0, mmGC_USER_SHADER_ARRAY_CONFIG, data);
7790 }
7791
gfx_v9_0_get_cu_active_bitmap(struct amdgpu_device * adev)7792 static u32 gfx_v9_0_get_cu_active_bitmap(struct amdgpu_device *adev)
7793 {
7794 u32 data, mask;
7795
7796 data = RREG32_SOC15(GC, 0, mmCC_GC_SHADER_ARRAY_CONFIG);
7797 data |= RREG32_SOC15(GC, 0, mmGC_USER_SHADER_ARRAY_CONFIG);
7798
7799 data &= CC_GC_SHADER_ARRAY_CONFIG__INACTIVE_CUS_MASK;
7800 data >>= CC_GC_SHADER_ARRAY_CONFIG__INACTIVE_CUS__SHIFT;
7801
7802 mask = amdgpu_gfx_create_bitmask(adev->gfx.config.max_cu_per_sh);
7803
7804 return (~data) & mask;
7805 }
7806
gfx_v9_0_get_cu_info(struct amdgpu_device * adev,struct amdgpu_cu_info * cu_info)7807 static int gfx_v9_0_get_cu_info(struct amdgpu_device *adev,
7808 struct amdgpu_cu_info *cu_info)
7809 {
7810 int i, j, k, counter, active_cu_number = 0;
7811 u32 mask, bitmap, ao_bitmap, ao_cu_mask = 0;
7812 unsigned disable_masks[4 * 4];
7813
7814 if (!adev || !cu_info)
7815 return -EINVAL;
7816
7817 /*
7818 * 16 comes from bitmap array size 4*4, and it can cover all gfx9 ASICs
7819 */
7820 if (adev->gfx.config.max_shader_engines *
7821 adev->gfx.config.max_sh_per_se > 16)
7822 return -EINVAL;
7823
7824 amdgpu_gfx_parse_disable_cu(disable_masks,
7825 adev->gfx.config.max_shader_engines,
7826 adev->gfx.config.max_sh_per_se);
7827
7828 mutex_lock(&adev->grbm_idx_mutex);
7829 for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
7830 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {
7831 mask = 1;
7832 ao_bitmap = 0;
7833 counter = 0;
7834 amdgpu_gfx_select_se_sh(adev, i, j, 0xffffffff, 0);
7835 gfx_v9_0_set_user_cu_inactive_bitmap(
7836 adev, disable_masks[i * adev->gfx.config.max_sh_per_se + j]);
7837 bitmap = gfx_v9_0_get_cu_active_bitmap(adev);
7838
7839 /*
7840 * The bitmap(and ao_cu_bitmap) in cu_info structure is
7841 * 4x4 size array, and it's usually suitable for Vega
7842 * ASICs which has 4*2 SE/SH layout.
7843 * But for Arcturus, SE/SH layout is changed to 8*1.
7844 * To mostly reduce the impact, we make it compatible
7845 * with current bitmap array as below:
7846 * SE4,SH0 --> bitmap[0][1]
7847 * SE5,SH0 --> bitmap[1][1]
7848 * SE6,SH0 --> bitmap[2][1]
7849 * SE7,SH0 --> bitmap[3][1]
7850 */
7851 cu_info->bitmap[0][i % 4][j + i / 4] = bitmap;
7852
7853 for (k = 0; k < adev->gfx.config.max_cu_per_sh; k ++) {
7854 if (bitmap & mask) {
7855 if (counter < adev->gfx.config.max_cu_per_sh)
7856 ao_bitmap |= mask;
7857 counter ++;
7858 }
7859 mask <<= 1;
7860 }
7861 active_cu_number += counter;
7862 if (i < 2 && j < 2)
7863 ao_cu_mask |= (ao_bitmap << (i * 16 + j * 8));
7864 cu_info->ao_cu_bitmap[i % 4][j + i / 4] = ao_bitmap;
7865 }
7866 }
7867 amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0);
7868 mutex_unlock(&adev->grbm_idx_mutex);
7869
7870 cu_info->number = active_cu_number;
7871 cu_info->ao_cu_mask = ao_cu_mask;
7872 cu_info->simd_per_cu = NUM_SIMD_PER_CU;
7873
7874 return 0;
7875 }
7876
7877 const struct amdgpu_ip_block_version gfx_v9_0_ip_block =
7878 {
7879 .type = AMD_IP_BLOCK_TYPE_GFX,
7880 .major = 9,
7881 .minor = 0,
7882 .rev = 0,
7883 .funcs = &gfx_v9_0_ip_funcs,
7884 };
7885