1 /*
2 * Copyright 2018 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 *
23 */
24 #include <linux/list.h>
25 #include "amdgpu.h"
26 #include "amdgpu_xgmi.h"
27 #include "amdgpu_ras.h"
28 #include "soc15.h"
29 #include "df/df_3_6_offset.h"
30 #include "xgmi/xgmi_4_0_0_smn.h"
31 #include "xgmi/xgmi_4_0_0_sh_mask.h"
32 #include "xgmi/xgmi_6_1_0_sh_mask.h"
33 #include "wafl/wafl2_4_0_0_smn.h"
34 #include "wafl/wafl2_4_0_0_sh_mask.h"
35
36 #include "amdgpu_reset.h"
37
38 #define smnPCS_XGMI3X16_PCS_ERROR_STATUS 0x11a0020c
39 #define smnPCS_XGMI3X16_PCS_ERROR_NONCORRECTABLE_MASK 0x11a00218
40 #define smnPCS_GOPX1_PCS_ERROR_STATUS 0x12200210
41 #define smnPCS_GOPX1_PCS_ERROR_NONCORRECTABLE_MASK 0x12200218
42
43 #define XGMI_STATE_DISABLE 0xD1
44 #define XGMI_STATE_LS0 0x81
45 #define XGMI_LINK_ACTIVE 1
46 #define XGMI_LINK_INACTIVE 0
47
48 static DEFINE_MUTEX(xgmi_mutex);
49
50 #define AMDGPU_MAX_XGMI_DEVICE_PER_HIVE 4
51
52 static LIST_HEAD(xgmi_hive_list);
53
54 static const int xgmi_pcs_err_status_reg_vg20[] = {
55 smnXGMI0_PCS_GOPX16_PCS_ERROR_STATUS,
56 smnXGMI0_PCS_GOPX16_PCS_ERROR_STATUS + 0x100000,
57 };
58
59 static const int wafl_pcs_err_status_reg_vg20[] = {
60 smnPCS_GOPX1_0_PCS_GOPX1_PCS_ERROR_STATUS,
61 smnPCS_GOPX1_0_PCS_GOPX1_PCS_ERROR_STATUS + 0x100000,
62 };
63
64 static const int xgmi_pcs_err_status_reg_arct[] = {
65 smnXGMI0_PCS_GOPX16_PCS_ERROR_STATUS,
66 smnXGMI0_PCS_GOPX16_PCS_ERROR_STATUS + 0x100000,
67 smnXGMI0_PCS_GOPX16_PCS_ERROR_STATUS + 0x500000,
68 smnXGMI0_PCS_GOPX16_PCS_ERROR_STATUS + 0x600000,
69 smnXGMI0_PCS_GOPX16_PCS_ERROR_STATUS + 0x700000,
70 smnXGMI0_PCS_GOPX16_PCS_ERROR_STATUS + 0x800000,
71 };
72
73 /* same as vg20*/
74 static const int wafl_pcs_err_status_reg_arct[] = {
75 smnPCS_GOPX1_0_PCS_GOPX1_PCS_ERROR_STATUS,
76 smnPCS_GOPX1_0_PCS_GOPX1_PCS_ERROR_STATUS + 0x100000,
77 };
78
79 static const int xgmi3x16_pcs_err_status_reg_aldebaran[] = {
80 smnPCS_XGMI3X16_PCS_ERROR_STATUS,
81 smnPCS_XGMI3X16_PCS_ERROR_STATUS + 0x100000,
82 smnPCS_XGMI3X16_PCS_ERROR_STATUS + 0x200000,
83 smnPCS_XGMI3X16_PCS_ERROR_STATUS + 0x300000,
84 smnPCS_XGMI3X16_PCS_ERROR_STATUS + 0x400000,
85 smnPCS_XGMI3X16_PCS_ERROR_STATUS + 0x500000,
86 smnPCS_XGMI3X16_PCS_ERROR_STATUS + 0x600000,
87 smnPCS_XGMI3X16_PCS_ERROR_STATUS + 0x700000
88 };
89
90 static const int xgmi3x16_pcs_err_noncorrectable_mask_reg_aldebaran[] = {
91 smnPCS_XGMI3X16_PCS_ERROR_NONCORRECTABLE_MASK,
92 smnPCS_XGMI3X16_PCS_ERROR_NONCORRECTABLE_MASK + 0x100000,
93 smnPCS_XGMI3X16_PCS_ERROR_NONCORRECTABLE_MASK + 0x200000,
94 smnPCS_XGMI3X16_PCS_ERROR_NONCORRECTABLE_MASK + 0x300000,
95 smnPCS_XGMI3X16_PCS_ERROR_NONCORRECTABLE_MASK + 0x400000,
96 smnPCS_XGMI3X16_PCS_ERROR_NONCORRECTABLE_MASK + 0x500000,
97 smnPCS_XGMI3X16_PCS_ERROR_NONCORRECTABLE_MASK + 0x600000,
98 smnPCS_XGMI3X16_PCS_ERROR_NONCORRECTABLE_MASK + 0x700000
99 };
100
101 static const int walf_pcs_err_status_reg_aldebaran[] = {
102 smnPCS_GOPX1_PCS_ERROR_STATUS,
103 smnPCS_GOPX1_PCS_ERROR_STATUS + 0x100000
104 };
105
106 static const int walf_pcs_err_noncorrectable_mask_reg_aldebaran[] = {
107 smnPCS_GOPX1_PCS_ERROR_NONCORRECTABLE_MASK,
108 smnPCS_GOPX1_PCS_ERROR_NONCORRECTABLE_MASK + 0x100000
109 };
110
111 static const int xgmi3x16_pcs_err_status_reg_v6_4[] = {
112 smnPCS_XGMI3X16_PCS_ERROR_STATUS,
113 smnPCS_XGMI3X16_PCS_ERROR_STATUS + 0x100000
114 };
115
116 static const int xgmi3x16_pcs_err_noncorrectable_mask_reg_v6_4[] = {
117 smnPCS_XGMI3X16_PCS_ERROR_NONCORRECTABLE_MASK,
118 smnPCS_XGMI3X16_PCS_ERROR_NONCORRECTABLE_MASK + 0x100000
119 };
120
121 static const u64 xgmi_v6_4_0_mca_base_array[] = {
122 0x11a09200,
123 0x11b09200,
124 };
125
126 static const char *xgmi_v6_4_0_ras_error_code_ext[32] = {
127 [0x00] = "XGMI PCS DataLossErr",
128 [0x01] = "XGMI PCS TrainingErr",
129 [0x02] = "XGMI PCS FlowCtrlAckErr",
130 [0x03] = "XGMI PCS RxFifoUnderflowErr",
131 [0x04] = "XGMI PCS RxFifoOverflowErr",
132 [0x05] = "XGMI PCS CRCErr",
133 [0x06] = "XGMI PCS BERExceededErr",
134 [0x07] = "XGMI PCS TxMetaDataErr",
135 [0x08] = "XGMI PCS ReplayBufParityErr",
136 [0x09] = "XGMI PCS DataParityErr",
137 [0x0a] = "XGMI PCS ReplayFifoOverflowErr",
138 [0x0b] = "XGMI PCS ReplayFifoUnderflowErr",
139 [0x0c] = "XGMI PCS ElasticFifoOverflowErr",
140 [0x0d] = "XGMI PCS DeskewErr",
141 [0x0e] = "XGMI PCS FlowCtrlCRCErr",
142 [0x0f] = "XGMI PCS DataStartupLimitErr",
143 [0x10] = "XGMI PCS FCInitTimeoutErr",
144 [0x11] = "XGMI PCS RecoveryTimeoutErr",
145 [0x12] = "XGMI PCS ReadySerialTimeoutErr",
146 [0x13] = "XGMI PCS ReadySerialAttemptErr",
147 [0x14] = "XGMI PCS RecoveryAttemptErr",
148 [0x15] = "XGMI PCS RecoveryRelockAttemptErr",
149 [0x16] = "XGMI PCS ReplayAttemptErr",
150 [0x17] = "XGMI PCS SyncHdrErr",
151 [0x18] = "XGMI PCS TxReplayTimeoutErr",
152 [0x19] = "XGMI PCS RxReplayTimeoutErr",
153 [0x1a] = "XGMI PCS LinkSubTxTimeoutErr",
154 [0x1b] = "XGMI PCS LinkSubRxTimeoutErr",
155 [0x1c] = "XGMI PCS RxCMDPktErr",
156 };
157
158 static const struct amdgpu_pcs_ras_field xgmi_pcs_ras_fields[] = {
159 {"XGMI PCS DataLossErr",
160 SOC15_REG_FIELD(XGMI0_PCS_GOPX16_PCS_ERROR_STATUS, DataLossErr)},
161 {"XGMI PCS TrainingErr",
162 SOC15_REG_FIELD(XGMI0_PCS_GOPX16_PCS_ERROR_STATUS, TrainingErr)},
163 {"XGMI PCS CRCErr",
164 SOC15_REG_FIELD(XGMI0_PCS_GOPX16_PCS_ERROR_STATUS, CRCErr)},
165 {"XGMI PCS BERExceededErr",
166 SOC15_REG_FIELD(XGMI0_PCS_GOPX16_PCS_ERROR_STATUS, BERExceededErr)},
167 {"XGMI PCS TxMetaDataErr",
168 SOC15_REG_FIELD(XGMI0_PCS_GOPX16_PCS_ERROR_STATUS, TxMetaDataErr)},
169 {"XGMI PCS ReplayBufParityErr",
170 SOC15_REG_FIELD(XGMI0_PCS_GOPX16_PCS_ERROR_STATUS, ReplayBufParityErr)},
171 {"XGMI PCS DataParityErr",
172 SOC15_REG_FIELD(XGMI0_PCS_GOPX16_PCS_ERROR_STATUS, DataParityErr)},
173 {"XGMI PCS ReplayFifoOverflowErr",
174 SOC15_REG_FIELD(XGMI0_PCS_GOPX16_PCS_ERROR_STATUS, ReplayFifoOverflowErr)},
175 {"XGMI PCS ReplayFifoUnderflowErr",
176 SOC15_REG_FIELD(XGMI0_PCS_GOPX16_PCS_ERROR_STATUS, ReplayFifoUnderflowErr)},
177 {"XGMI PCS ElasticFifoOverflowErr",
178 SOC15_REG_FIELD(XGMI0_PCS_GOPX16_PCS_ERROR_STATUS, ElasticFifoOverflowErr)},
179 {"XGMI PCS DeskewErr",
180 SOC15_REG_FIELD(XGMI0_PCS_GOPX16_PCS_ERROR_STATUS, DeskewErr)},
181 {"XGMI PCS DataStartupLimitErr",
182 SOC15_REG_FIELD(XGMI0_PCS_GOPX16_PCS_ERROR_STATUS, DataStartupLimitErr)},
183 {"XGMI PCS FCInitTimeoutErr",
184 SOC15_REG_FIELD(XGMI0_PCS_GOPX16_PCS_ERROR_STATUS, FCInitTimeoutErr)},
185 {"XGMI PCS RecoveryTimeoutErr",
186 SOC15_REG_FIELD(XGMI0_PCS_GOPX16_PCS_ERROR_STATUS, RecoveryTimeoutErr)},
187 {"XGMI PCS ReadySerialTimeoutErr",
188 SOC15_REG_FIELD(XGMI0_PCS_GOPX16_PCS_ERROR_STATUS, ReadySerialTimeoutErr)},
189 {"XGMI PCS ReadySerialAttemptErr",
190 SOC15_REG_FIELD(XGMI0_PCS_GOPX16_PCS_ERROR_STATUS, ReadySerialAttemptErr)},
191 {"XGMI PCS RecoveryAttemptErr",
192 SOC15_REG_FIELD(XGMI0_PCS_GOPX16_PCS_ERROR_STATUS, RecoveryAttemptErr)},
193 {"XGMI PCS RecoveryRelockAttemptErr",
194 SOC15_REG_FIELD(XGMI0_PCS_GOPX16_PCS_ERROR_STATUS, RecoveryRelockAttemptErr)},
195 };
196
197 static const struct amdgpu_pcs_ras_field wafl_pcs_ras_fields[] = {
198 {"WAFL PCS DataLossErr",
199 SOC15_REG_FIELD(PCS_GOPX1_0_PCS_GOPX1_PCS_ERROR_STATUS, DataLossErr)},
200 {"WAFL PCS TrainingErr",
201 SOC15_REG_FIELD(PCS_GOPX1_0_PCS_GOPX1_PCS_ERROR_STATUS, TrainingErr)},
202 {"WAFL PCS CRCErr",
203 SOC15_REG_FIELD(PCS_GOPX1_0_PCS_GOPX1_PCS_ERROR_STATUS, CRCErr)},
204 {"WAFL PCS BERExceededErr",
205 SOC15_REG_FIELD(PCS_GOPX1_0_PCS_GOPX1_PCS_ERROR_STATUS, BERExceededErr)},
206 {"WAFL PCS TxMetaDataErr",
207 SOC15_REG_FIELD(PCS_GOPX1_0_PCS_GOPX1_PCS_ERROR_STATUS, TxMetaDataErr)},
208 {"WAFL PCS ReplayBufParityErr",
209 SOC15_REG_FIELD(PCS_GOPX1_0_PCS_GOPX1_PCS_ERROR_STATUS, ReplayBufParityErr)},
210 {"WAFL PCS DataParityErr",
211 SOC15_REG_FIELD(PCS_GOPX1_0_PCS_GOPX1_PCS_ERROR_STATUS, DataParityErr)},
212 {"WAFL PCS ReplayFifoOverflowErr",
213 SOC15_REG_FIELD(PCS_GOPX1_0_PCS_GOPX1_PCS_ERROR_STATUS, ReplayFifoOverflowErr)},
214 {"WAFL PCS ReplayFifoUnderflowErr",
215 SOC15_REG_FIELD(PCS_GOPX1_0_PCS_GOPX1_PCS_ERROR_STATUS, ReplayFifoUnderflowErr)},
216 {"WAFL PCS ElasticFifoOverflowErr",
217 SOC15_REG_FIELD(PCS_GOPX1_0_PCS_GOPX1_PCS_ERROR_STATUS, ElasticFifoOverflowErr)},
218 {"WAFL PCS DeskewErr",
219 SOC15_REG_FIELD(PCS_GOPX1_0_PCS_GOPX1_PCS_ERROR_STATUS, DeskewErr)},
220 {"WAFL PCS DataStartupLimitErr",
221 SOC15_REG_FIELD(PCS_GOPX1_0_PCS_GOPX1_PCS_ERROR_STATUS, DataStartupLimitErr)},
222 {"WAFL PCS FCInitTimeoutErr",
223 SOC15_REG_FIELD(PCS_GOPX1_0_PCS_GOPX1_PCS_ERROR_STATUS, FCInitTimeoutErr)},
224 {"WAFL PCS RecoveryTimeoutErr",
225 SOC15_REG_FIELD(PCS_GOPX1_0_PCS_GOPX1_PCS_ERROR_STATUS, RecoveryTimeoutErr)},
226 {"WAFL PCS ReadySerialTimeoutErr",
227 SOC15_REG_FIELD(PCS_GOPX1_0_PCS_GOPX1_PCS_ERROR_STATUS, ReadySerialTimeoutErr)},
228 {"WAFL PCS ReadySerialAttemptErr",
229 SOC15_REG_FIELD(PCS_GOPX1_0_PCS_GOPX1_PCS_ERROR_STATUS, ReadySerialAttemptErr)},
230 {"WAFL PCS RecoveryAttemptErr",
231 SOC15_REG_FIELD(PCS_GOPX1_0_PCS_GOPX1_PCS_ERROR_STATUS, RecoveryAttemptErr)},
232 {"WAFL PCS RecoveryRelockAttemptErr",
233 SOC15_REG_FIELD(PCS_GOPX1_0_PCS_GOPX1_PCS_ERROR_STATUS, RecoveryRelockAttemptErr)},
234 };
235
236 static const struct amdgpu_pcs_ras_field xgmi3x16_pcs_ras_fields[] = {
237 {"XGMI3X16 PCS DataLossErr",
238 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, DataLossErr)},
239 {"XGMI3X16 PCS TrainingErr",
240 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, TrainingErr)},
241 {"XGMI3X16 PCS FlowCtrlAckErr",
242 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, FlowCtrlAckErr)},
243 {"XGMI3X16 PCS RxFifoUnderflowErr",
244 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, RxFifoUnderflowErr)},
245 {"XGMI3X16 PCS RxFifoOverflowErr",
246 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, RxFifoOverflowErr)},
247 {"XGMI3X16 PCS CRCErr",
248 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, CRCErr)},
249 {"XGMI3X16 PCS BERExceededErr",
250 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, BERExceededErr)},
251 {"XGMI3X16 PCS TxVcidDataErr",
252 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, TxVcidDataErr)},
253 {"XGMI3X16 PCS ReplayBufParityErr",
254 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, ReplayBufParityErr)},
255 {"XGMI3X16 PCS DataParityErr",
256 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, DataParityErr)},
257 {"XGMI3X16 PCS ReplayFifoOverflowErr",
258 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, ReplayFifoOverflowErr)},
259 {"XGMI3X16 PCS ReplayFifoUnderflowErr",
260 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, ReplayFifoUnderflowErr)},
261 {"XGMI3X16 PCS ElasticFifoOverflowErr",
262 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, ElasticFifoOverflowErr)},
263 {"XGMI3X16 PCS DeskewErr",
264 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, DeskewErr)},
265 {"XGMI3X16 PCS FlowCtrlCRCErr",
266 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, FlowCtrlCRCErr)},
267 {"XGMI3X16 PCS DataStartupLimitErr",
268 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, DataStartupLimitErr)},
269 {"XGMI3X16 PCS FCInitTimeoutErr",
270 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, FCInitTimeoutErr)},
271 {"XGMI3X16 PCS RecoveryTimeoutErr",
272 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, RecoveryTimeoutErr)},
273 {"XGMI3X16 PCS ReadySerialTimeoutErr",
274 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, ReadySerialTimeoutErr)},
275 {"XGMI3X16 PCS ReadySerialAttemptErr",
276 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, ReadySerialAttemptErr)},
277 {"XGMI3X16 PCS RecoveryAttemptErr",
278 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, RecoveryAttemptErr)},
279 {"XGMI3X16 PCS RecoveryRelockAttemptErr",
280 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, RecoveryRelockAttemptErr)},
281 {"XGMI3X16 PCS ReplayAttemptErr",
282 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, ReplayAttemptErr)},
283 {"XGMI3X16 PCS SyncHdrErr",
284 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, SyncHdrErr)},
285 {"XGMI3X16 PCS TxReplayTimeoutErr",
286 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, TxReplayTimeoutErr)},
287 {"XGMI3X16 PCS RxReplayTimeoutErr",
288 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, RxReplayTimeoutErr)},
289 {"XGMI3X16 PCS LinkSubTxTimeoutErr",
290 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, LinkSubTxTimeoutErr)},
291 {"XGMI3X16 PCS LinkSubRxTimeoutErr",
292 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, LinkSubRxTimeoutErr)},
293 {"XGMI3X16 PCS RxCMDPktErr",
294 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, RxCMDPktErr)},
295 };
296
xgmi_v6_4_get_link_status(struct amdgpu_device * adev,int global_link_num)297 static u32 xgmi_v6_4_get_link_status(struct amdgpu_device *adev, int global_link_num)
298 {
299 const u32 smnpcs_xgmi3x16_pcs_state_hist1 = 0x11a00070;
300 const int xgmi_inst = 2;
301 u32 link_inst;
302 u64 addr;
303
304 link_inst = global_link_num % xgmi_inst;
305
306 addr = (smnpcs_xgmi3x16_pcs_state_hist1 | (link_inst << 20)) +
307 adev->asic_funcs->encode_ext_smn_addressing(global_link_num / xgmi_inst);
308
309 return RREG32_PCIE_EXT(addr);
310 }
311
amdgpu_get_xgmi_link_status(struct amdgpu_device * adev,int global_link_num)312 int amdgpu_get_xgmi_link_status(struct amdgpu_device *adev, int global_link_num)
313 {
314 u32 xgmi_state_reg_val;
315
316 switch (amdgpu_ip_version(adev, XGMI_HWIP, 0)) {
317 case IP_VERSION(6, 4, 0):
318 case IP_VERSION(6, 4, 1):
319 xgmi_state_reg_val = xgmi_v6_4_get_link_status(adev, global_link_num);
320 break;
321 default:
322 return -EOPNOTSUPP;
323 }
324
325 if ((xgmi_state_reg_val & 0xFF) == XGMI_STATE_DISABLE)
326 return -ENOLINK;
327
328 if ((xgmi_state_reg_val & 0xFF) == XGMI_STATE_LS0)
329 return XGMI_LINK_ACTIVE;
330
331 return XGMI_LINK_INACTIVE;
332 }
333
334 /**
335 * DOC: AMDGPU XGMI Support
336 *
337 * XGMI is a high speed interconnect that joins multiple GPU cards
338 * into a homogeneous memory space that is organized by a collective
339 * hive ID and individual node IDs, both of which are 64-bit numbers.
340 *
341 * The file xgmi_device_id contains the unique per GPU device ID and
342 * is stored in the /sys/class/drm/card${cardno}/device/ directory.
343 *
344 * Inside the device directory a sub-directory 'xgmi_hive_info' is
345 * created which contains the hive ID and the list of nodes.
346 *
347 * The hive ID is stored in:
348 * /sys/class/drm/card${cardno}/device/xgmi_hive_info/xgmi_hive_id
349 *
350 * The node information is stored in numbered directories:
351 * /sys/class/drm/card${cardno}/device/xgmi_hive_info/node${nodeno}/xgmi_device_id
352 *
353 * Each device has their own xgmi_hive_info direction with a mirror
354 * set of node sub-directories.
355 *
356 * The XGMI memory space is built by contiguously adding the power of
357 * two padded VRAM space from each node to each other.
358 *
359 */
360
361 static struct attribute amdgpu_xgmi_hive_id = {
362 .name = "xgmi_hive_id",
363 .mode = S_IRUGO
364 };
365
366 static struct attribute *amdgpu_xgmi_hive_attrs[] = {
367 &amdgpu_xgmi_hive_id,
368 NULL
369 };
370 ATTRIBUTE_GROUPS(amdgpu_xgmi_hive);
371
amdgpu_xgmi_show_attrs(struct kobject * kobj,struct attribute * attr,char * buf)372 static ssize_t amdgpu_xgmi_show_attrs(struct kobject *kobj,
373 struct attribute *attr, char *buf)
374 {
375 struct amdgpu_hive_info *hive = container_of(
376 kobj, struct amdgpu_hive_info, kobj);
377
378 if (attr == &amdgpu_xgmi_hive_id)
379 return snprintf(buf, PAGE_SIZE, "%llu\n", hive->hive_id);
380
381 return 0;
382 }
383
amdgpu_xgmi_hive_release(struct kobject * kobj)384 static void amdgpu_xgmi_hive_release(struct kobject *kobj)
385 {
386 struct amdgpu_hive_info *hive = container_of(
387 kobj, struct amdgpu_hive_info, kobj);
388
389 amdgpu_reset_put_reset_domain(hive->reset_domain);
390 hive->reset_domain = NULL;
391
392 mutex_destroy(&hive->hive_lock);
393 kfree(hive);
394 }
395
396 static const struct sysfs_ops amdgpu_xgmi_hive_ops = {
397 .show = amdgpu_xgmi_show_attrs,
398 };
399
400 static const struct kobj_type amdgpu_xgmi_hive_type = {
401 .release = amdgpu_xgmi_hive_release,
402 .sysfs_ops = &amdgpu_xgmi_hive_ops,
403 .default_groups = amdgpu_xgmi_hive_groups,
404 };
405
amdgpu_xgmi_show_device_id(struct device * dev,struct device_attribute * attr,char * buf)406 static ssize_t amdgpu_xgmi_show_device_id(struct device *dev,
407 struct device_attribute *attr,
408 char *buf)
409 {
410 struct drm_device *ddev = dev_get_drvdata(dev);
411 struct amdgpu_device *adev = drm_to_adev(ddev);
412
413 return sysfs_emit(buf, "%llu\n", adev->gmc.xgmi.node_id);
414
415 }
416
amdgpu_xgmi_show_physical_id(struct device * dev,struct device_attribute * attr,char * buf)417 static ssize_t amdgpu_xgmi_show_physical_id(struct device *dev,
418 struct device_attribute *attr,
419 char *buf)
420 {
421 struct drm_device *ddev = dev_get_drvdata(dev);
422 struct amdgpu_device *adev = drm_to_adev(ddev);
423
424 return sysfs_emit(buf, "%u\n", adev->gmc.xgmi.physical_node_id);
425
426 }
427
amdgpu_xgmi_show_num_hops(struct device * dev,struct device_attribute * attr,char * buf)428 static ssize_t amdgpu_xgmi_show_num_hops(struct device *dev,
429 struct device_attribute *attr,
430 char *buf)
431 {
432 struct drm_device *ddev = dev_get_drvdata(dev);
433 struct amdgpu_device *adev = drm_to_adev(ddev);
434 struct psp_xgmi_topology_info *top = &adev->psp.xgmi_context.top_info;
435 int i;
436
437 for (i = 0; i < top->num_nodes; i++)
438 sprintf(buf + 3 * i, "%02x ", top->nodes[i].num_hops);
439
440 return sysfs_emit(buf, "%s\n", buf);
441 }
442
amdgpu_xgmi_show_num_links(struct device * dev,struct device_attribute * attr,char * buf)443 static ssize_t amdgpu_xgmi_show_num_links(struct device *dev,
444 struct device_attribute *attr,
445 char *buf)
446 {
447 struct drm_device *ddev = dev_get_drvdata(dev);
448 struct amdgpu_device *adev = drm_to_adev(ddev);
449 struct psp_xgmi_topology_info *top = &adev->psp.xgmi_context.top_info;
450 int i;
451
452 for (i = 0; i < top->num_nodes; i++)
453 sprintf(buf + 3 * i, "%02x ", top->nodes[i].num_links);
454
455 return sysfs_emit(buf, "%s\n", buf);
456 }
457
amdgpu_xgmi_show_connected_port_num(struct device * dev,struct device_attribute * attr,char * buf)458 static ssize_t amdgpu_xgmi_show_connected_port_num(struct device *dev,
459 struct device_attribute *attr,
460 char *buf)
461 {
462 struct drm_device *ddev = dev_get_drvdata(dev);
463 struct amdgpu_device *adev = drm_to_adev(ddev);
464 struct psp_xgmi_topology_info *top = &adev->psp.xgmi_context.top_info;
465 int i, j, size = 0;
466 int current_node;
467 /*
468 * get the node id in the sysfs for the current socket and show
469 * it in the port num info output in the sysfs for easy reading.
470 * it is NOT the one retrieved from xgmi ta.
471 */
472 for (i = 0; i < top->num_nodes; i++) {
473 if (top->nodes[i].node_id == adev->gmc.xgmi.node_id) {
474 current_node = i;
475 break;
476 }
477 }
478
479 if (i == top->num_nodes)
480 return -EINVAL;
481
482 for (i = 0; i < top->num_nodes; i++) {
483 for (j = 0; j < top->nodes[i].num_links; j++)
484 /* node id in sysfs starts from 1 rather than 0 so +1 here */
485 size += sysfs_emit_at(buf, size, "%02x:%02x -> %02x:%02x\n", current_node + 1,
486 top->nodes[i].port_num[j].src_xgmi_port_num, i + 1,
487 top->nodes[i].port_num[j].dst_xgmi_port_num);
488 }
489
490 return size;
491 }
492
493 #define AMDGPU_XGMI_SET_FICAA(o) ((o) | 0x456801)
amdgpu_xgmi_show_error(struct device * dev,struct device_attribute * attr,char * buf)494 static ssize_t amdgpu_xgmi_show_error(struct device *dev,
495 struct device_attribute *attr,
496 char *buf)
497 {
498 struct drm_device *ddev = dev_get_drvdata(dev);
499 struct amdgpu_device *adev = drm_to_adev(ddev);
500 uint32_t ficaa_pie_ctl_in, ficaa_pie_status_in;
501 uint64_t fica_out;
502 unsigned int error_count = 0;
503
504 ficaa_pie_ctl_in = AMDGPU_XGMI_SET_FICAA(0x200);
505 ficaa_pie_status_in = AMDGPU_XGMI_SET_FICAA(0x208);
506
507 if ((!adev->df.funcs) ||
508 (!adev->df.funcs->get_fica) ||
509 (!adev->df.funcs->set_fica))
510 return -EINVAL;
511
512 fica_out = adev->df.funcs->get_fica(adev, ficaa_pie_ctl_in);
513 if (fica_out != 0x1f)
514 pr_err("xGMI error counters not enabled!\n");
515
516 fica_out = adev->df.funcs->get_fica(adev, ficaa_pie_status_in);
517
518 if ((fica_out & 0xffff) == 2)
519 error_count = ((fica_out >> 62) & 0x1) + (fica_out >> 63);
520
521 adev->df.funcs->set_fica(adev, ficaa_pie_status_in, 0, 0);
522
523 return sysfs_emit(buf, "%u\n", error_count);
524 }
525
526
527 static DEVICE_ATTR(xgmi_device_id, S_IRUGO, amdgpu_xgmi_show_device_id, NULL);
528 static DEVICE_ATTR(xgmi_physical_id, 0444, amdgpu_xgmi_show_physical_id, NULL);
529 static DEVICE_ATTR(xgmi_error, S_IRUGO, amdgpu_xgmi_show_error, NULL);
530 static DEVICE_ATTR(xgmi_num_hops, S_IRUGO, amdgpu_xgmi_show_num_hops, NULL);
531 static DEVICE_ATTR(xgmi_num_links, S_IRUGO, amdgpu_xgmi_show_num_links, NULL);
532 static DEVICE_ATTR(xgmi_port_num, S_IRUGO, amdgpu_xgmi_show_connected_port_num, NULL);
533
amdgpu_xgmi_sysfs_add_dev_info(struct amdgpu_device * adev,struct amdgpu_hive_info * hive)534 static int amdgpu_xgmi_sysfs_add_dev_info(struct amdgpu_device *adev,
535 struct amdgpu_hive_info *hive)
536 {
537 int ret = 0;
538 char node[10] = { 0 };
539
540 /* Create xgmi device id file */
541 ret = device_create_file(adev->dev, &dev_attr_xgmi_device_id);
542 if (ret) {
543 dev_err(adev->dev, "XGMI: Failed to create device file xgmi_device_id\n");
544 return ret;
545 }
546
547 ret = device_create_file(adev->dev, &dev_attr_xgmi_physical_id);
548 if (ret) {
549 dev_err(adev->dev, "XGMI: Failed to create device file xgmi_physical_id\n");
550 return ret;
551 }
552
553 /* Create xgmi error file */
554 ret = device_create_file(adev->dev, &dev_attr_xgmi_error);
555 if (ret)
556 pr_err("failed to create xgmi_error\n");
557
558 /* Create xgmi num hops file */
559 ret = device_create_file(adev->dev, &dev_attr_xgmi_num_hops);
560 if (ret)
561 pr_err("failed to create xgmi_num_hops\n");
562
563 /* Create xgmi num links file */
564 ret = device_create_file(adev->dev, &dev_attr_xgmi_num_links);
565 if (ret)
566 pr_err("failed to create xgmi_num_links\n");
567
568 /* Create xgmi port num file if supported */
569 if (adev->psp.xgmi_context.xgmi_ta_caps & EXTEND_PEER_LINK_INFO_CMD_FLAG) {
570 ret = device_create_file(adev->dev, &dev_attr_xgmi_port_num);
571 if (ret)
572 dev_err(adev->dev, "failed to create xgmi_port_num\n");
573 }
574
575 /* Create sysfs link to hive info folder on the first device */
576 if (hive->kobj.parent != (&adev->dev->kobj)) {
577 ret = sysfs_create_link(&adev->dev->kobj, &hive->kobj,
578 "xgmi_hive_info");
579 if (ret) {
580 dev_err(adev->dev, "XGMI: Failed to create link to hive info");
581 goto remove_file;
582 }
583 }
584
585 sprintf(node, "node%d", atomic_read(&hive->number_devices));
586 /* Create sysfs link form the hive folder to yourself */
587 ret = sysfs_create_link(&hive->kobj, &adev->dev->kobj, node);
588 if (ret) {
589 dev_err(adev->dev, "XGMI: Failed to create link from hive info");
590 goto remove_link;
591 }
592
593 goto success;
594
595
596 remove_link:
597 sysfs_remove_link(&adev->dev->kobj, adev_to_drm(adev)->unique);
598
599 remove_file:
600 device_remove_file(adev->dev, &dev_attr_xgmi_device_id);
601 device_remove_file(adev->dev, &dev_attr_xgmi_physical_id);
602 device_remove_file(adev->dev, &dev_attr_xgmi_error);
603 device_remove_file(adev->dev, &dev_attr_xgmi_num_hops);
604 device_remove_file(adev->dev, &dev_attr_xgmi_num_links);
605 if (adev->psp.xgmi_context.xgmi_ta_caps & EXTEND_PEER_LINK_INFO_CMD_FLAG)
606 device_remove_file(adev->dev, &dev_attr_xgmi_port_num);
607
608 success:
609 return ret;
610 }
611
amdgpu_xgmi_sysfs_rem_dev_info(struct amdgpu_device * adev,struct amdgpu_hive_info * hive)612 static void amdgpu_xgmi_sysfs_rem_dev_info(struct amdgpu_device *adev,
613 struct amdgpu_hive_info *hive)
614 {
615 char node[10];
616 memset(node, 0, sizeof(node));
617
618 device_remove_file(adev->dev, &dev_attr_xgmi_device_id);
619 device_remove_file(adev->dev, &dev_attr_xgmi_physical_id);
620 device_remove_file(adev->dev, &dev_attr_xgmi_error);
621 device_remove_file(adev->dev, &dev_attr_xgmi_num_hops);
622 device_remove_file(adev->dev, &dev_attr_xgmi_num_links);
623 if (adev->psp.xgmi_context.xgmi_ta_caps & EXTEND_PEER_LINK_INFO_CMD_FLAG)
624 device_remove_file(adev->dev, &dev_attr_xgmi_port_num);
625
626 if (hive->kobj.parent != (&adev->dev->kobj))
627 sysfs_remove_link(&adev->dev->kobj,"xgmi_hive_info");
628
629 sprintf(node, "node%d", atomic_read(&hive->number_devices));
630 sysfs_remove_link(&hive->kobj, node);
631
632 }
633
634
635
amdgpu_get_xgmi_hive(struct amdgpu_device * adev)636 struct amdgpu_hive_info *amdgpu_get_xgmi_hive(struct amdgpu_device *adev)
637 {
638 struct amdgpu_hive_info *hive = NULL;
639 int ret;
640
641 if (!adev->gmc.xgmi.hive_id)
642 return NULL;
643
644 if (adev->hive) {
645 kobject_get(&adev->hive->kobj);
646 return adev->hive;
647 }
648
649 mutex_lock(&xgmi_mutex);
650
651 list_for_each_entry(hive, &xgmi_hive_list, node) {
652 if (hive->hive_id == adev->gmc.xgmi.hive_id)
653 goto pro_end;
654 }
655
656 hive = kzalloc(sizeof(*hive), GFP_KERNEL);
657 if (!hive) {
658 dev_err(adev->dev, "XGMI: allocation failed\n");
659 ret = -ENOMEM;
660 hive = NULL;
661 goto pro_end;
662 }
663
664 /* initialize new hive if not exist */
665 ret = kobject_init_and_add(&hive->kobj,
666 &amdgpu_xgmi_hive_type,
667 &adev->dev->kobj,
668 "%s", "xgmi_hive_info");
669 if (ret) {
670 dev_err(adev->dev, "XGMI: failed initializing kobject for xgmi hive\n");
671 kobject_put(&hive->kobj);
672 hive = NULL;
673 goto pro_end;
674 }
675
676 /**
677 * Only init hive->reset_domain for none SRIOV configuration. For SRIOV,
678 * Host driver decide how to reset the GPU either through FLR or chain reset.
679 * Guest side will get individual notifications from the host for the FLR
680 * if necessary.
681 */
682 if (!amdgpu_sriov_vf(adev)) {
683 /**
684 * Avoid recreating reset domain when hive is reconstructed for the case
685 * of reset the devices in the XGMI hive during probe for passthrough GPU
686 * See https://www.spinics.net/lists/amd-gfx/msg58836.html
687 */
688 if (adev->reset_domain->type != XGMI_HIVE) {
689 hive->reset_domain =
690 amdgpu_reset_create_reset_domain(XGMI_HIVE, "amdgpu-reset-hive");
691 if (!hive->reset_domain) {
692 dev_err(adev->dev, "XGMI: failed initializing reset domain for xgmi hive\n");
693 ret = -ENOMEM;
694 kobject_put(&hive->kobj);
695 hive = NULL;
696 goto pro_end;
697 }
698 } else {
699 amdgpu_reset_get_reset_domain(adev->reset_domain);
700 hive->reset_domain = adev->reset_domain;
701 }
702 }
703
704 hive->hive_id = adev->gmc.xgmi.hive_id;
705 INIT_LIST_HEAD(&hive->device_list);
706 INIT_LIST_HEAD(&hive->node);
707 mutex_init(&hive->hive_lock);
708 atomic_set(&hive->number_devices, 0);
709 task_barrier_init(&hive->tb);
710 hive->pstate = AMDGPU_XGMI_PSTATE_UNKNOWN;
711 hive->hi_req_gpu = NULL;
712 atomic_set(&hive->requested_nps_mode, UNKNOWN_MEMORY_PARTITION_MODE);
713
714 /*
715 * hive pstate on boot is high in vega20 so we have to go to low
716 * pstate on after boot.
717 */
718 hive->hi_req_count = AMDGPU_MAX_XGMI_DEVICE_PER_HIVE;
719 list_add_tail(&hive->node, &xgmi_hive_list);
720
721 pro_end:
722 if (hive)
723 kobject_get(&hive->kobj);
724 mutex_unlock(&xgmi_mutex);
725 return hive;
726 }
727
amdgpu_put_xgmi_hive(struct amdgpu_hive_info * hive)728 void amdgpu_put_xgmi_hive(struct amdgpu_hive_info *hive)
729 {
730 if (hive)
731 kobject_put(&hive->kobj);
732 }
733
amdgpu_xgmi_set_pstate(struct amdgpu_device * adev,int pstate)734 int amdgpu_xgmi_set_pstate(struct amdgpu_device *adev, int pstate)
735 {
736 int ret = 0;
737 struct amdgpu_hive_info *hive;
738 struct amdgpu_device *request_adev;
739 bool is_hi_req = pstate == AMDGPU_XGMI_PSTATE_MAX_VEGA20;
740 bool init_low;
741
742 hive = amdgpu_get_xgmi_hive(adev);
743 if (!hive)
744 return 0;
745
746 request_adev = hive->hi_req_gpu ? hive->hi_req_gpu : adev;
747 init_low = hive->pstate == AMDGPU_XGMI_PSTATE_UNKNOWN;
748 amdgpu_put_xgmi_hive(hive);
749 /* fw bug so temporarily disable pstate switching */
750 return 0;
751
752 if (!hive || adev->asic_type != CHIP_VEGA20)
753 return 0;
754
755 mutex_lock(&hive->hive_lock);
756
757 if (is_hi_req)
758 hive->hi_req_count++;
759 else
760 hive->hi_req_count--;
761
762 /*
763 * Vega20 only needs single peer to request pstate high for the hive to
764 * go high but all peers must request pstate low for the hive to go low
765 */
766 if (hive->pstate == pstate ||
767 (!is_hi_req && hive->hi_req_count && !init_low))
768 goto out;
769
770 dev_dbg(request_adev->dev, "Set xgmi pstate %d.\n", pstate);
771
772 ret = amdgpu_dpm_set_xgmi_pstate(request_adev, pstate);
773 if (ret) {
774 dev_err(request_adev->dev,
775 "XGMI: Set pstate failure on device %llx, hive %llx, ret %d",
776 request_adev->gmc.xgmi.node_id,
777 request_adev->gmc.xgmi.hive_id, ret);
778 goto out;
779 }
780
781 if (init_low)
782 hive->pstate = hive->hi_req_count ?
783 hive->pstate : AMDGPU_XGMI_PSTATE_MIN;
784 else {
785 hive->pstate = pstate;
786 hive->hi_req_gpu = pstate != AMDGPU_XGMI_PSTATE_MIN ?
787 adev : NULL;
788 }
789 out:
790 mutex_unlock(&hive->hive_lock);
791 return ret;
792 }
793
amdgpu_xgmi_update_topology(struct amdgpu_hive_info * hive,struct amdgpu_device * adev)794 int amdgpu_xgmi_update_topology(struct amdgpu_hive_info *hive, struct amdgpu_device *adev)
795 {
796 int ret;
797
798 if (amdgpu_sriov_vf(adev))
799 return 0;
800
801 /* Each psp need to set the latest topology */
802 ret = psp_xgmi_set_topology_info(&adev->psp,
803 atomic_read(&hive->number_devices),
804 &adev->psp.xgmi_context.top_info);
805 if (ret)
806 dev_err(adev->dev,
807 "XGMI: Set topology failure on device %llx, hive %llx, ret %d",
808 adev->gmc.xgmi.node_id,
809 adev->gmc.xgmi.hive_id, ret);
810
811 return ret;
812 }
813
814
815 /*
816 * NOTE psp_xgmi_node_info.num_hops layout is as follows:
817 * num_hops[7:6] = link type (0 = xGMI2, 1 = xGMI3, 2/3 = reserved)
818 * num_hops[5:3] = reserved
819 * num_hops[2:0] = number of hops
820 */
amdgpu_xgmi_get_hops_count(struct amdgpu_device * adev,struct amdgpu_device * peer_adev)821 int amdgpu_xgmi_get_hops_count(struct amdgpu_device *adev,
822 struct amdgpu_device *peer_adev)
823 {
824 struct psp_xgmi_topology_info *top = &adev->psp.xgmi_context.top_info;
825 uint8_t num_hops_mask = 0x7;
826 int i;
827
828 if (!adev->gmc.xgmi.supported)
829 return 0;
830
831 for (i = 0 ; i < top->num_nodes; ++i)
832 if (top->nodes[i].node_id == peer_adev->gmc.xgmi.node_id)
833 return top->nodes[i].num_hops & num_hops_mask;
834
835 dev_err(adev->dev, "Failed to get xgmi hops count for peer %d.\n",
836 peer_adev->gmc.xgmi.physical_node_id);
837
838 return 0;
839 }
840
amdgpu_xgmi_get_bandwidth(struct amdgpu_device * adev,struct amdgpu_device * peer_adev,enum amdgpu_xgmi_bw_mode bw_mode,enum amdgpu_xgmi_bw_unit bw_unit,uint32_t * min_bw,uint32_t * max_bw)841 int amdgpu_xgmi_get_bandwidth(struct amdgpu_device *adev, struct amdgpu_device *peer_adev,
842 enum amdgpu_xgmi_bw_mode bw_mode, enum amdgpu_xgmi_bw_unit bw_unit,
843 uint32_t *min_bw, uint32_t *max_bw)
844 {
845 bool peer_mode = bw_mode == AMDGPU_XGMI_BW_MODE_PER_PEER;
846 int unit_scale = bw_unit == AMDGPU_XGMI_BW_UNIT_MBYTES ? 1000 : 1;
847 int num_lanes = adev->gmc.xgmi.max_width;
848 int speed = adev->gmc.xgmi.max_speed;
849 int num_links = !peer_mode ? 1 : -1;
850
851 if (!(min_bw && max_bw))
852 return -EINVAL;
853
854 *min_bw = 0;
855 *max_bw = 0;
856
857 if (!adev->gmc.xgmi.supported)
858 return -ENODATA;
859
860 if (peer_mode && !peer_adev)
861 return -EINVAL;
862
863 if (peer_mode) {
864 struct psp_xgmi_topology_info *top = &adev->psp.xgmi_context.top_info;
865 int i;
866
867 for (i = 0 ; i < top->num_nodes; ++i) {
868 if (top->nodes[i].node_id != peer_adev->gmc.xgmi.node_id)
869 continue;
870
871 num_links = top->nodes[i].num_links;
872 break;
873 }
874 }
875
876 if (num_links == -1) {
877 dev_err(adev->dev, "Failed to get number of xgmi links for peer %d.\n",
878 peer_adev->gmc.xgmi.physical_node_id);
879 } else if (num_links) {
880 int per_link_bw = (speed * num_lanes * unit_scale)/BITS_PER_BYTE;
881
882 *min_bw = per_link_bw;
883 *max_bw = num_links * per_link_bw;
884 }
885
886 return 0;
887 }
888
amdgpu_xgmi_get_is_sharing_enabled(struct amdgpu_device * adev,struct amdgpu_device * peer_adev)889 bool amdgpu_xgmi_get_is_sharing_enabled(struct amdgpu_device *adev,
890 struct amdgpu_device *peer_adev)
891 {
892 struct psp_xgmi_topology_info *top = &adev->psp.xgmi_context.top_info;
893 int i;
894
895 /* Sharing should always be enabled for non-SRIOV. */
896 if (!amdgpu_sriov_vf(adev))
897 return true;
898
899 for (i = 0 ; i < top->num_nodes; ++i)
900 if (top->nodes[i].node_id == peer_adev->gmc.xgmi.node_id)
901 return !!top->nodes[i].is_sharing_enabled;
902
903 return false;
904 }
905
906 /*
907 * Devices that support extended data require the entire hive to initialize with
908 * the shared memory buffer flag set.
909 *
910 * Hive locks and conditions apply - see amdgpu_xgmi_add_device
911 */
amdgpu_xgmi_initialize_hive_get_data_partition(struct amdgpu_hive_info * hive,bool set_extended_data)912 static int amdgpu_xgmi_initialize_hive_get_data_partition(struct amdgpu_hive_info *hive,
913 bool set_extended_data)
914 {
915 struct amdgpu_device *tmp_adev;
916 int ret;
917
918 list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head) {
919 ret = psp_xgmi_initialize(&tmp_adev->psp, set_extended_data, false);
920 if (ret) {
921 dev_err(tmp_adev->dev,
922 "XGMI: Failed to initialize xgmi session for data partition %i\n",
923 set_extended_data);
924 return ret;
925 }
926
927 }
928
929 return 0;
930 }
931
amdgpu_xgmi_fill_topology_info(struct amdgpu_device * adev,struct amdgpu_device * peer_adev)932 static void amdgpu_xgmi_fill_topology_info(struct amdgpu_device *adev,
933 struct amdgpu_device *peer_adev)
934 {
935 struct psp_xgmi_topology_info *top_info = &adev->psp.xgmi_context.top_info;
936 struct psp_xgmi_topology_info *peer_info = &peer_adev->psp.xgmi_context.top_info;
937
938 for (int i = 0; i < peer_info->num_nodes; i++) {
939 if (peer_info->nodes[i].node_id == adev->gmc.xgmi.node_id) {
940 for (int j = 0; j < top_info->num_nodes; j++) {
941 if (top_info->nodes[j].node_id == peer_adev->gmc.xgmi.node_id) {
942 peer_info->nodes[i].num_hops = top_info->nodes[j].num_hops;
943 peer_info->nodes[i].is_sharing_enabled =
944 top_info->nodes[j].is_sharing_enabled;
945 peer_info->nodes[i].num_links =
946 top_info->nodes[j].num_links;
947 return;
948 }
949 }
950 }
951 }
952 }
953
amdgpu_xgmi_add_device(struct amdgpu_device * adev)954 int amdgpu_xgmi_add_device(struct amdgpu_device *adev)
955 {
956 struct psp_xgmi_topology_info *top_info;
957 struct amdgpu_hive_info *hive;
958 struct amdgpu_xgmi *entry;
959 struct amdgpu_device *tmp_adev = NULL;
960
961 int count = 0, ret = 0;
962
963 if (!adev->gmc.xgmi.supported)
964 return 0;
965
966 if (amdgpu_device_ip_get_ip_block(adev, AMD_IP_BLOCK_TYPE_PSP)) {
967 ret = psp_xgmi_initialize(&adev->psp, false, true);
968 if (ret) {
969 dev_err(adev->dev,
970 "XGMI: Failed to initialize xgmi session\n");
971 return ret;
972 }
973
974 ret = psp_xgmi_get_hive_id(&adev->psp, &adev->gmc.xgmi.hive_id);
975 if (ret) {
976 dev_err(adev->dev,
977 "XGMI: Failed to get hive id\n");
978 return ret;
979 }
980
981 ret = psp_xgmi_get_node_id(&adev->psp, &adev->gmc.xgmi.node_id);
982 if (ret) {
983 dev_err(adev->dev,
984 "XGMI: Failed to get node id\n");
985 return ret;
986 }
987 } else {
988 adev->gmc.xgmi.hive_id = 16;
989 adev->gmc.xgmi.node_id = adev->gmc.xgmi.physical_node_id + 16;
990 }
991
992 hive = amdgpu_get_xgmi_hive(adev);
993 if (!hive) {
994 ret = -EINVAL;
995 dev_err(adev->dev,
996 "XGMI: node 0x%llx, can not match hive 0x%llx in the hive list.\n",
997 adev->gmc.xgmi.node_id, adev->gmc.xgmi.hive_id);
998 goto exit;
999 }
1000 mutex_lock(&hive->hive_lock);
1001
1002 top_info = &adev->psp.xgmi_context.top_info;
1003
1004 list_add_tail(&adev->gmc.xgmi.head, &hive->device_list);
1005 list_for_each_entry(entry, &hive->device_list, head)
1006 top_info->nodes[count++].node_id = entry->node_id;
1007 top_info->num_nodes = count;
1008 atomic_set(&hive->number_devices, count);
1009
1010 task_barrier_add_task(&hive->tb);
1011
1012 if (amdgpu_device_ip_get_ip_block(adev, AMD_IP_BLOCK_TYPE_PSP)) {
1013 list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head) {
1014 /* update node list for other device in the hive */
1015 if (tmp_adev != adev) {
1016 top_info = &tmp_adev->psp.xgmi_context.top_info;
1017 top_info->nodes[count - 1].node_id =
1018 adev->gmc.xgmi.node_id;
1019 top_info->num_nodes = count;
1020 }
1021 ret = amdgpu_xgmi_update_topology(hive, tmp_adev);
1022 if (ret)
1023 goto exit_unlock;
1024 }
1025
1026 if (amdgpu_sriov_vf(adev) &&
1027 adev->psp.xgmi_context.xgmi_ta_caps & EXTEND_PEER_LINK_INFO_CMD_FLAG) {
1028 /* only get topology for VF being init if it can support full duplex */
1029 ret = psp_xgmi_get_topology_info(&adev->psp, count,
1030 &adev->psp.xgmi_context.top_info, false);
1031 if (ret) {
1032 dev_err(adev->dev,
1033 "XGMI: Get topology failure on device %llx, hive %llx, ret %d",
1034 adev->gmc.xgmi.node_id,
1035 adev->gmc.xgmi.hive_id, ret);
1036 /* To do: continue with some node failed or disable the whole hive*/
1037 goto exit_unlock;
1038 }
1039
1040 /* fill the topology info for peers instead of getting from PSP */
1041 list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head) {
1042 amdgpu_xgmi_fill_topology_info(adev, tmp_adev);
1043 }
1044 } else {
1045 /* get latest topology info for each device from psp */
1046 list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head) {
1047 ret = psp_xgmi_get_topology_info(&tmp_adev->psp, count,
1048 &tmp_adev->psp.xgmi_context.top_info, false);
1049 if (ret) {
1050 dev_err(tmp_adev->dev,
1051 "XGMI: Get topology failure on device %llx, hive %llx, ret %d",
1052 tmp_adev->gmc.xgmi.node_id,
1053 tmp_adev->gmc.xgmi.hive_id, ret);
1054 /* To do : continue with some node failed or disable the whole hive */
1055 goto exit_unlock;
1056 }
1057 }
1058 }
1059
1060 /* get topology again for hives that support extended data */
1061 if (adev->psp.xgmi_context.supports_extended_data) {
1062
1063 /* initialize the hive to get extended data. */
1064 ret = amdgpu_xgmi_initialize_hive_get_data_partition(hive, true);
1065 if (ret)
1066 goto exit_unlock;
1067
1068 /* get the extended data. */
1069 list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head) {
1070 ret = psp_xgmi_get_topology_info(&tmp_adev->psp, count,
1071 &tmp_adev->psp.xgmi_context.top_info, true);
1072 if (ret) {
1073 dev_err(tmp_adev->dev,
1074 "XGMI: Get topology for extended data failure on device %llx, hive %llx, ret %d",
1075 tmp_adev->gmc.xgmi.node_id,
1076 tmp_adev->gmc.xgmi.hive_id, ret);
1077 goto exit_unlock;
1078 }
1079 }
1080
1081 /* initialize the hive to get non-extended data for the next round. */
1082 ret = amdgpu_xgmi_initialize_hive_get_data_partition(hive, false);
1083 if (ret)
1084 goto exit_unlock;
1085
1086 }
1087 }
1088
1089 if (!ret)
1090 ret = amdgpu_xgmi_sysfs_add_dev_info(adev, hive);
1091
1092 exit_unlock:
1093 mutex_unlock(&hive->hive_lock);
1094 exit:
1095 if (!ret) {
1096 adev->hive = hive;
1097 dev_info(adev->dev, "XGMI: Add node %d, hive 0x%llx.\n",
1098 adev->gmc.xgmi.physical_node_id, adev->gmc.xgmi.hive_id);
1099 } else {
1100 amdgpu_put_xgmi_hive(hive);
1101 dev_err(adev->dev, "XGMI: Failed to add node %d, hive 0x%llx ret: %d\n",
1102 adev->gmc.xgmi.physical_node_id, adev->gmc.xgmi.hive_id,
1103 ret);
1104 }
1105
1106 return ret;
1107 }
1108
amdgpu_xgmi_remove_device(struct amdgpu_device * adev)1109 int amdgpu_xgmi_remove_device(struct amdgpu_device *adev)
1110 {
1111 struct amdgpu_hive_info *hive = adev->hive;
1112
1113 if (!adev->gmc.xgmi.supported)
1114 return -EINVAL;
1115
1116 if (!hive)
1117 return -EINVAL;
1118
1119 mutex_lock(&hive->hive_lock);
1120 task_barrier_rem_task(&hive->tb);
1121 amdgpu_xgmi_sysfs_rem_dev_info(adev, hive);
1122 if (hive->hi_req_gpu == adev)
1123 hive->hi_req_gpu = NULL;
1124 list_del(&adev->gmc.xgmi.head);
1125 mutex_unlock(&hive->hive_lock);
1126
1127 amdgpu_put_xgmi_hive(hive);
1128 adev->hive = NULL;
1129
1130 if (atomic_dec_return(&hive->number_devices) == 0) {
1131 /* Remove the hive from global hive list */
1132 mutex_lock(&xgmi_mutex);
1133 list_del(&hive->node);
1134 mutex_unlock(&xgmi_mutex);
1135
1136 amdgpu_put_xgmi_hive(hive);
1137 }
1138
1139 return 0;
1140 }
1141
xgmi_v6_4_0_aca_bank_parser(struct aca_handle * handle,struct aca_bank * bank,enum aca_smu_type type,void * data)1142 static int xgmi_v6_4_0_aca_bank_parser(struct aca_handle *handle, struct aca_bank *bank,
1143 enum aca_smu_type type, void *data)
1144 {
1145 struct amdgpu_device *adev = handle->adev;
1146 struct aca_bank_info info;
1147 const char *error_str;
1148 u64 status, count;
1149 int ret, ext_error_code;
1150
1151 ret = aca_bank_info_decode(bank, &info);
1152 if (ret)
1153 return ret;
1154
1155 status = bank->regs[ACA_REG_IDX_STATUS];
1156 ext_error_code = ACA_REG__STATUS__ERRORCODEEXT(status);
1157
1158 error_str = ext_error_code < ARRAY_SIZE(xgmi_v6_4_0_ras_error_code_ext) ?
1159 xgmi_v6_4_0_ras_error_code_ext[ext_error_code] : NULL;
1160 if (error_str)
1161 dev_info(adev->dev, "%s detected\n", error_str);
1162
1163 count = ACA_REG__MISC0__ERRCNT(bank->regs[ACA_REG_IDX_MISC0]);
1164
1165 switch (type) {
1166 case ACA_SMU_TYPE_UE:
1167 if (ext_error_code != 0 && ext_error_code != 9)
1168 count = 0ULL;
1169
1170 bank->aca_err_type = ACA_ERROR_TYPE_UE;
1171 ret = aca_error_cache_log_bank_error(handle, &info, ACA_ERROR_TYPE_UE, count);
1172 break;
1173 case ACA_SMU_TYPE_CE:
1174 count = ext_error_code == 6 ? count : 0ULL;
1175 bank->aca_err_type = ACA_ERROR_TYPE_CE;
1176 ret = aca_error_cache_log_bank_error(handle, &info, bank->aca_err_type, count);
1177 break;
1178 default:
1179 return -EINVAL;
1180 }
1181
1182 return ret;
1183 }
1184
1185 static const struct aca_bank_ops xgmi_v6_4_0_aca_bank_ops = {
1186 .aca_bank_parser = xgmi_v6_4_0_aca_bank_parser,
1187 };
1188
1189 static const struct aca_info xgmi_v6_4_0_aca_info = {
1190 .hwip = ACA_HWIP_TYPE_PCS_XGMI,
1191 .mask = ACA_ERROR_UE_MASK | ACA_ERROR_CE_MASK,
1192 .bank_ops = &xgmi_v6_4_0_aca_bank_ops,
1193 };
1194
amdgpu_xgmi_ras_late_init(struct amdgpu_device * adev,struct ras_common_if * ras_block)1195 static int amdgpu_xgmi_ras_late_init(struct amdgpu_device *adev, struct ras_common_if *ras_block)
1196 {
1197 int r;
1198
1199 if (!adev->gmc.xgmi.supported ||
1200 adev->gmc.xgmi.num_physical_nodes == 0)
1201 return 0;
1202
1203 amdgpu_ras_reset_error_count(adev, AMDGPU_RAS_BLOCK__XGMI_WAFL);
1204
1205 r = amdgpu_ras_block_late_init(adev, ras_block);
1206 if (r)
1207 return r;
1208
1209 switch (amdgpu_ip_version(adev, XGMI_HWIP, 0)) {
1210 case IP_VERSION(6, 4, 0):
1211 case IP_VERSION(6, 4, 1):
1212 r = amdgpu_ras_bind_aca(adev, AMDGPU_RAS_BLOCK__XGMI_WAFL,
1213 &xgmi_v6_4_0_aca_info, NULL);
1214 if (r)
1215 goto late_fini;
1216 break;
1217 default:
1218 break;
1219 }
1220
1221 return 0;
1222
1223 late_fini:
1224 amdgpu_ras_block_late_fini(adev, ras_block);
1225
1226 return r;
1227 }
1228
amdgpu_xgmi_get_relative_phy_addr(struct amdgpu_device * adev,uint64_t addr)1229 uint64_t amdgpu_xgmi_get_relative_phy_addr(struct amdgpu_device *adev,
1230 uint64_t addr)
1231 {
1232 struct amdgpu_xgmi *xgmi = &adev->gmc.xgmi;
1233 return (addr + xgmi->physical_node_id * xgmi->node_segment_size);
1234 }
1235
pcs_clear_status(struct amdgpu_device * adev,uint32_t pcs_status_reg)1236 static void pcs_clear_status(struct amdgpu_device *adev, uint32_t pcs_status_reg)
1237 {
1238 WREG32_PCIE(pcs_status_reg, 0xFFFFFFFF);
1239 WREG32_PCIE(pcs_status_reg, 0);
1240 }
1241
amdgpu_xgmi_legacy_reset_ras_error_count(struct amdgpu_device * adev)1242 static void amdgpu_xgmi_legacy_reset_ras_error_count(struct amdgpu_device *adev)
1243 {
1244 uint32_t i;
1245
1246 switch (adev->asic_type) {
1247 case CHIP_ARCTURUS:
1248 for (i = 0; i < ARRAY_SIZE(xgmi_pcs_err_status_reg_arct); i++)
1249 pcs_clear_status(adev,
1250 xgmi_pcs_err_status_reg_arct[i]);
1251 break;
1252 case CHIP_VEGA20:
1253 for (i = 0; i < ARRAY_SIZE(xgmi_pcs_err_status_reg_vg20); i++)
1254 pcs_clear_status(adev,
1255 xgmi_pcs_err_status_reg_vg20[i]);
1256 break;
1257 case CHIP_ALDEBARAN:
1258 for (i = 0; i < ARRAY_SIZE(xgmi3x16_pcs_err_status_reg_aldebaran); i++)
1259 pcs_clear_status(adev,
1260 xgmi3x16_pcs_err_status_reg_aldebaran[i]);
1261 for (i = 0; i < ARRAY_SIZE(walf_pcs_err_status_reg_aldebaran); i++)
1262 pcs_clear_status(adev,
1263 walf_pcs_err_status_reg_aldebaran[i]);
1264 break;
1265 default:
1266 break;
1267 }
1268
1269 switch (amdgpu_ip_version(adev, XGMI_HWIP, 0)) {
1270 case IP_VERSION(6, 4, 0):
1271 case IP_VERSION(6, 4, 1):
1272 for (i = 0; i < ARRAY_SIZE(xgmi3x16_pcs_err_status_reg_v6_4); i++)
1273 pcs_clear_status(adev,
1274 xgmi3x16_pcs_err_status_reg_v6_4[i]);
1275 break;
1276 default:
1277 break;
1278 }
1279 }
1280
__xgmi_v6_4_0_reset_error_count(struct amdgpu_device * adev,int xgmi_inst,u64 mca_base)1281 static void __xgmi_v6_4_0_reset_error_count(struct amdgpu_device *adev, int xgmi_inst, u64 mca_base)
1282 {
1283 WREG64_MCA(xgmi_inst, mca_base, ACA_REG_IDX_STATUS, 0ULL);
1284 }
1285
xgmi_v6_4_0_reset_error_count(struct amdgpu_device * adev,int xgmi_inst)1286 static void xgmi_v6_4_0_reset_error_count(struct amdgpu_device *adev, int xgmi_inst)
1287 {
1288 int i;
1289
1290 for (i = 0; i < ARRAY_SIZE(xgmi_v6_4_0_mca_base_array); i++)
1291 __xgmi_v6_4_0_reset_error_count(adev, xgmi_inst, xgmi_v6_4_0_mca_base_array[i]);
1292 }
1293
xgmi_v6_4_0_reset_ras_error_count(struct amdgpu_device * adev)1294 static void xgmi_v6_4_0_reset_ras_error_count(struct amdgpu_device *adev)
1295 {
1296 int i;
1297
1298 for_each_inst(i, adev->aid_mask)
1299 xgmi_v6_4_0_reset_error_count(adev, i);
1300 }
1301
amdgpu_xgmi_reset_ras_error_count(struct amdgpu_device * adev)1302 static void amdgpu_xgmi_reset_ras_error_count(struct amdgpu_device *adev)
1303 {
1304 switch (amdgpu_ip_version(adev, XGMI_HWIP, 0)) {
1305 case IP_VERSION(6, 4, 0):
1306 case IP_VERSION(6, 4, 1):
1307 xgmi_v6_4_0_reset_ras_error_count(adev);
1308 break;
1309 default:
1310 amdgpu_xgmi_legacy_reset_ras_error_count(adev);
1311 break;
1312 }
1313 }
1314
amdgpu_xgmi_query_pcs_error_status(struct amdgpu_device * adev,uint32_t value,uint32_t mask_value,uint32_t * ue_count,uint32_t * ce_count,bool is_xgmi_pcs,bool check_mask)1315 static int amdgpu_xgmi_query_pcs_error_status(struct amdgpu_device *adev,
1316 uint32_t value,
1317 uint32_t mask_value,
1318 uint32_t *ue_count,
1319 uint32_t *ce_count,
1320 bool is_xgmi_pcs,
1321 bool check_mask)
1322 {
1323 int i;
1324 int ue_cnt = 0;
1325 const struct amdgpu_pcs_ras_field *pcs_ras_fields = NULL;
1326 uint32_t field_array_size = 0;
1327
1328 if (is_xgmi_pcs) {
1329 if (amdgpu_ip_version(adev, XGMI_HWIP, 0) ==
1330 IP_VERSION(6, 1, 0) ||
1331 amdgpu_ip_version(adev, XGMI_HWIP, 0) ==
1332 IP_VERSION(6, 4, 0) ||
1333 amdgpu_ip_version(adev, XGMI_HWIP, 0) ==
1334 IP_VERSION(6, 4, 1)) {
1335 pcs_ras_fields = &xgmi3x16_pcs_ras_fields[0];
1336 field_array_size = ARRAY_SIZE(xgmi3x16_pcs_ras_fields);
1337 } else {
1338 pcs_ras_fields = &xgmi_pcs_ras_fields[0];
1339 field_array_size = ARRAY_SIZE(xgmi_pcs_ras_fields);
1340 }
1341 } else {
1342 pcs_ras_fields = &wafl_pcs_ras_fields[0];
1343 field_array_size = ARRAY_SIZE(wafl_pcs_ras_fields);
1344 }
1345
1346 if (check_mask)
1347 value = value & ~mask_value;
1348
1349 /* query xgmi/walf pcs error status,
1350 * only ue is supported */
1351 for (i = 0; value && i < field_array_size; i++) {
1352 ue_cnt = (value &
1353 pcs_ras_fields[i].pcs_err_mask) >>
1354 pcs_ras_fields[i].pcs_err_shift;
1355 if (ue_cnt) {
1356 dev_info(adev->dev, "%s detected\n",
1357 pcs_ras_fields[i].err_name);
1358 *ue_count += ue_cnt;
1359 }
1360
1361 /* reset bit value if the bit is checked */
1362 value &= ~(pcs_ras_fields[i].pcs_err_mask);
1363 }
1364
1365 return 0;
1366 }
1367
amdgpu_xgmi_legacy_query_ras_error_count(struct amdgpu_device * adev,void * ras_error_status)1368 static void amdgpu_xgmi_legacy_query_ras_error_count(struct amdgpu_device *adev,
1369 void *ras_error_status)
1370 {
1371 struct ras_err_data *err_data = (struct ras_err_data *)ras_error_status;
1372 int i, supported = 1;
1373 uint32_t data, mask_data = 0;
1374 uint32_t ue_cnt = 0, ce_cnt = 0;
1375
1376 if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__XGMI_WAFL))
1377 return ;
1378
1379 err_data->ue_count = 0;
1380 err_data->ce_count = 0;
1381
1382 switch (adev->asic_type) {
1383 case CHIP_ARCTURUS:
1384 /* check xgmi pcs error */
1385 for (i = 0; i < ARRAY_SIZE(xgmi_pcs_err_status_reg_arct); i++) {
1386 data = RREG32_PCIE(xgmi_pcs_err_status_reg_arct[i]);
1387 if (data)
1388 amdgpu_xgmi_query_pcs_error_status(adev, data,
1389 mask_data, &ue_cnt, &ce_cnt, true, false);
1390 }
1391 /* check wafl pcs error */
1392 for (i = 0; i < ARRAY_SIZE(wafl_pcs_err_status_reg_arct); i++) {
1393 data = RREG32_PCIE(wafl_pcs_err_status_reg_arct[i]);
1394 if (data)
1395 amdgpu_xgmi_query_pcs_error_status(adev, data,
1396 mask_data, &ue_cnt, &ce_cnt, false, false);
1397 }
1398 break;
1399 case CHIP_VEGA20:
1400 /* check xgmi pcs error */
1401 for (i = 0; i < ARRAY_SIZE(xgmi_pcs_err_status_reg_vg20); i++) {
1402 data = RREG32_PCIE(xgmi_pcs_err_status_reg_vg20[i]);
1403 if (data)
1404 amdgpu_xgmi_query_pcs_error_status(adev, data,
1405 mask_data, &ue_cnt, &ce_cnt, true, false);
1406 }
1407 /* check wafl pcs error */
1408 for (i = 0; i < ARRAY_SIZE(wafl_pcs_err_status_reg_vg20); i++) {
1409 data = RREG32_PCIE(wafl_pcs_err_status_reg_vg20[i]);
1410 if (data)
1411 amdgpu_xgmi_query_pcs_error_status(adev, data,
1412 mask_data, &ue_cnt, &ce_cnt, false, false);
1413 }
1414 break;
1415 case CHIP_ALDEBARAN:
1416 /* check xgmi3x16 pcs error */
1417 for (i = 0; i < ARRAY_SIZE(xgmi3x16_pcs_err_status_reg_aldebaran); i++) {
1418 data = RREG32_PCIE(xgmi3x16_pcs_err_status_reg_aldebaran[i]);
1419 mask_data =
1420 RREG32_PCIE(xgmi3x16_pcs_err_noncorrectable_mask_reg_aldebaran[i]);
1421 if (data)
1422 amdgpu_xgmi_query_pcs_error_status(adev, data,
1423 mask_data, &ue_cnt, &ce_cnt, true, true);
1424 }
1425 /* check wafl pcs error */
1426 for (i = 0; i < ARRAY_SIZE(walf_pcs_err_status_reg_aldebaran); i++) {
1427 data = RREG32_PCIE(walf_pcs_err_status_reg_aldebaran[i]);
1428 mask_data =
1429 RREG32_PCIE(walf_pcs_err_noncorrectable_mask_reg_aldebaran[i]);
1430 if (data)
1431 amdgpu_xgmi_query_pcs_error_status(adev, data,
1432 mask_data, &ue_cnt, &ce_cnt, false, true);
1433 }
1434 break;
1435 default:
1436 supported = 0;
1437 break;
1438 }
1439
1440 switch (amdgpu_ip_version(adev, XGMI_HWIP, 0)) {
1441 case IP_VERSION(6, 4, 0):
1442 case IP_VERSION(6, 4, 1):
1443 /* check xgmi3x16 pcs error */
1444 for (i = 0; i < ARRAY_SIZE(xgmi3x16_pcs_err_status_reg_v6_4); i++) {
1445 data = RREG32_PCIE(xgmi3x16_pcs_err_status_reg_v6_4[i]);
1446 mask_data =
1447 RREG32_PCIE(xgmi3x16_pcs_err_noncorrectable_mask_reg_v6_4[i]);
1448 if (data)
1449 amdgpu_xgmi_query_pcs_error_status(adev, data,
1450 mask_data, &ue_cnt, &ce_cnt, true, true);
1451 }
1452 break;
1453 default:
1454 if (!supported)
1455 dev_warn(adev->dev, "XGMI RAS error query not supported");
1456 break;
1457 }
1458
1459 amdgpu_ras_reset_error_count(adev, AMDGPU_RAS_BLOCK__XGMI_WAFL);
1460
1461 err_data->ue_count += ue_cnt;
1462 err_data->ce_count += ce_cnt;
1463 }
1464
xgmi_v6_4_0_pcs_mca_get_error_type(struct amdgpu_device * adev,u64 status)1465 static enum aca_error_type xgmi_v6_4_0_pcs_mca_get_error_type(struct amdgpu_device *adev, u64 status)
1466 {
1467 const char *error_str;
1468 int ext_error_code;
1469
1470 ext_error_code = ACA_REG__STATUS__ERRORCODEEXT(status);
1471
1472 error_str = ext_error_code < ARRAY_SIZE(xgmi_v6_4_0_ras_error_code_ext) ?
1473 xgmi_v6_4_0_ras_error_code_ext[ext_error_code] : NULL;
1474 if (error_str)
1475 dev_info(adev->dev, "%s detected\n", error_str);
1476
1477 switch (ext_error_code) {
1478 case 0:
1479 return ACA_ERROR_TYPE_UE;
1480 case 6:
1481 return ACA_ERROR_TYPE_CE;
1482 default:
1483 return -EINVAL;
1484 }
1485
1486 return -EINVAL;
1487 }
1488
__xgmi_v6_4_0_query_error_count(struct amdgpu_device * adev,struct amdgpu_smuio_mcm_config_info * mcm_info,u64 mca_base,struct ras_err_data * err_data)1489 static void __xgmi_v6_4_0_query_error_count(struct amdgpu_device *adev, struct amdgpu_smuio_mcm_config_info *mcm_info,
1490 u64 mca_base, struct ras_err_data *err_data)
1491 {
1492 int xgmi_inst = mcm_info->die_id;
1493 u64 status = 0;
1494
1495 status = RREG64_MCA(xgmi_inst, mca_base, ACA_REG_IDX_STATUS);
1496 if (!ACA_REG__STATUS__VAL(status))
1497 return;
1498
1499 switch (xgmi_v6_4_0_pcs_mca_get_error_type(adev, status)) {
1500 case ACA_ERROR_TYPE_UE:
1501 amdgpu_ras_error_statistic_ue_count(err_data, mcm_info, 1ULL);
1502 break;
1503 case ACA_ERROR_TYPE_CE:
1504 amdgpu_ras_error_statistic_ce_count(err_data, mcm_info, 1ULL);
1505 break;
1506 default:
1507 break;
1508 }
1509
1510 WREG64_MCA(xgmi_inst, mca_base, ACA_REG_IDX_STATUS, 0ULL);
1511 }
1512
xgmi_v6_4_0_query_error_count(struct amdgpu_device * adev,int xgmi_inst,struct ras_err_data * err_data)1513 static void xgmi_v6_4_0_query_error_count(struct amdgpu_device *adev, int xgmi_inst, struct ras_err_data *err_data)
1514 {
1515 struct amdgpu_smuio_mcm_config_info mcm_info = {
1516 .socket_id = adev->smuio.funcs->get_socket_id(adev),
1517 .die_id = xgmi_inst,
1518 };
1519 int i;
1520
1521 for (i = 0; i < ARRAY_SIZE(xgmi_v6_4_0_mca_base_array); i++)
1522 __xgmi_v6_4_0_query_error_count(adev, &mcm_info, xgmi_v6_4_0_mca_base_array[i], err_data);
1523 }
1524
xgmi_v6_4_0_query_ras_error_count(struct amdgpu_device * adev,void * ras_error_status)1525 static void xgmi_v6_4_0_query_ras_error_count(struct amdgpu_device *adev, void *ras_error_status)
1526 {
1527 struct ras_err_data *err_data = (struct ras_err_data *)ras_error_status;
1528 int i;
1529
1530 for_each_inst(i, adev->aid_mask)
1531 xgmi_v6_4_0_query_error_count(adev, i, err_data);
1532 }
1533
amdgpu_xgmi_query_ras_error_count(struct amdgpu_device * adev,void * ras_error_status)1534 static void amdgpu_xgmi_query_ras_error_count(struct amdgpu_device *adev,
1535 void *ras_error_status)
1536 {
1537 switch (amdgpu_ip_version(adev, XGMI_HWIP, 0)) {
1538 case IP_VERSION(6, 4, 0):
1539 case IP_VERSION(6, 4, 1):
1540 xgmi_v6_4_0_query_ras_error_count(adev, ras_error_status);
1541 break;
1542 default:
1543 amdgpu_xgmi_legacy_query_ras_error_count(adev, ras_error_status);
1544 break;
1545 }
1546 }
1547
1548 /* Trigger XGMI/WAFL error */
amdgpu_ras_error_inject_xgmi(struct amdgpu_device * adev,void * inject_if,uint32_t instance_mask)1549 static int amdgpu_ras_error_inject_xgmi(struct amdgpu_device *adev,
1550 void *inject_if, uint32_t instance_mask)
1551 {
1552 int ret1, ret2;
1553 struct ta_ras_trigger_error_input *block_info =
1554 (struct ta_ras_trigger_error_input *)inject_if;
1555
1556 if (amdgpu_dpm_set_df_cstate(adev, DF_CSTATE_DISALLOW))
1557 dev_warn(adev->dev, "Failed to disallow df cstate");
1558
1559 ret1 = amdgpu_dpm_set_pm_policy(adev, PP_PM_POLICY_XGMI_PLPD, XGMI_PLPD_DISALLOW);
1560 if (ret1 && ret1 != -EOPNOTSUPP)
1561 dev_warn(adev->dev, "Failed to disallow XGMI power down");
1562
1563 ret2 = psp_ras_trigger_error(&adev->psp, block_info, instance_mask);
1564
1565 if (amdgpu_ras_intr_triggered())
1566 return ret2;
1567
1568 ret1 = amdgpu_dpm_set_pm_policy(adev, PP_PM_POLICY_XGMI_PLPD, XGMI_PLPD_DEFAULT);
1569 if (ret1 && ret1 != -EOPNOTSUPP)
1570 dev_warn(adev->dev, "Failed to allow XGMI power down");
1571
1572 if (amdgpu_dpm_set_df_cstate(adev, DF_CSTATE_ALLOW))
1573 dev_warn(adev->dev, "Failed to allow df cstate");
1574
1575 return ret2;
1576 }
1577
1578 struct amdgpu_ras_block_hw_ops xgmi_ras_hw_ops = {
1579 .query_ras_error_count = amdgpu_xgmi_query_ras_error_count,
1580 .reset_ras_error_count = amdgpu_xgmi_reset_ras_error_count,
1581 .ras_error_inject = amdgpu_ras_error_inject_xgmi,
1582 };
1583
1584 struct amdgpu_xgmi_ras xgmi_ras = {
1585 .ras_block = {
1586 .hw_ops = &xgmi_ras_hw_ops,
1587 .ras_late_init = amdgpu_xgmi_ras_late_init,
1588 },
1589 };
1590
amdgpu_xgmi_ras_sw_init(struct amdgpu_device * adev)1591 int amdgpu_xgmi_ras_sw_init(struct amdgpu_device *adev)
1592 {
1593 int err;
1594 struct amdgpu_xgmi_ras *ras;
1595
1596 if (!adev->gmc.xgmi.ras)
1597 return 0;
1598
1599 ras = adev->gmc.xgmi.ras;
1600 err = amdgpu_ras_register_ras_block(adev, &ras->ras_block);
1601 if (err) {
1602 dev_err(adev->dev, "Failed to register xgmi_wafl_pcs ras block!\n");
1603 return err;
1604 }
1605
1606 strcpy(ras->ras_block.ras_comm.name, "xgmi_wafl");
1607 ras->ras_block.ras_comm.block = AMDGPU_RAS_BLOCK__XGMI_WAFL;
1608 ras->ras_block.ras_comm.type = AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE;
1609 adev->gmc.xgmi.ras_if = &ras->ras_block.ras_comm;
1610
1611 return 0;
1612 }
1613
amdgpu_xgmi_reset_on_init_work(struct work_struct * work)1614 static void amdgpu_xgmi_reset_on_init_work(struct work_struct *work)
1615 {
1616 struct amdgpu_hive_info *hive =
1617 container_of(work, struct amdgpu_hive_info, reset_on_init_work);
1618 struct amdgpu_reset_context reset_context;
1619 struct amdgpu_device *tmp_adev;
1620 struct list_head device_list;
1621 int r;
1622
1623 mutex_lock(&hive->hive_lock);
1624
1625 INIT_LIST_HEAD(&device_list);
1626 list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head)
1627 list_add_tail(&tmp_adev->reset_list, &device_list);
1628
1629 tmp_adev = list_first_entry(&device_list, struct amdgpu_device,
1630 reset_list);
1631 amdgpu_device_lock_reset_domain(tmp_adev->reset_domain);
1632
1633 reset_context.method = AMD_RESET_METHOD_ON_INIT;
1634 reset_context.reset_req_dev = tmp_adev;
1635 reset_context.hive = hive;
1636 reset_context.reset_device_list = &device_list;
1637 set_bit(AMDGPU_NEED_FULL_RESET, &reset_context.flags);
1638 set_bit(AMDGPU_SKIP_COREDUMP, &reset_context.flags);
1639
1640 amdgpu_reset_do_xgmi_reset_on_init(&reset_context);
1641 mutex_unlock(&hive->hive_lock);
1642 amdgpu_device_unlock_reset_domain(tmp_adev->reset_domain);
1643
1644 list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head) {
1645 r = amdgpu_ras_init_badpage_info(tmp_adev);
1646 if (r && r != -EHWPOISON)
1647 dev_err(tmp_adev->dev,
1648 "error during bad page data initialization");
1649 }
1650 }
1651
amdgpu_xgmi_schedule_reset_on_init(struct amdgpu_hive_info * hive)1652 static void amdgpu_xgmi_schedule_reset_on_init(struct amdgpu_hive_info *hive)
1653 {
1654 INIT_WORK(&hive->reset_on_init_work, amdgpu_xgmi_reset_on_init_work);
1655 amdgpu_reset_domain_schedule(hive->reset_domain,
1656 &hive->reset_on_init_work);
1657 }
1658
amdgpu_xgmi_reset_on_init(struct amdgpu_device * adev)1659 int amdgpu_xgmi_reset_on_init(struct amdgpu_device *adev)
1660 {
1661 struct amdgpu_hive_info *hive;
1662 bool reset_scheduled;
1663 int num_devs;
1664
1665 hive = amdgpu_get_xgmi_hive(adev);
1666 if (!hive)
1667 return -EINVAL;
1668
1669 mutex_lock(&hive->hive_lock);
1670 num_devs = atomic_read(&hive->number_devices);
1671 reset_scheduled = false;
1672 if (num_devs == adev->gmc.xgmi.num_physical_nodes) {
1673 amdgpu_xgmi_schedule_reset_on_init(hive);
1674 reset_scheduled = true;
1675 }
1676
1677 mutex_unlock(&hive->hive_lock);
1678 amdgpu_put_xgmi_hive(hive);
1679
1680 if (reset_scheduled)
1681 flush_work(&hive->reset_on_init_work);
1682
1683 return 0;
1684 }
1685
amdgpu_xgmi_request_nps_change(struct amdgpu_device * adev,struct amdgpu_hive_info * hive,int req_nps_mode)1686 int amdgpu_xgmi_request_nps_change(struct amdgpu_device *adev,
1687 struct amdgpu_hive_info *hive,
1688 int req_nps_mode)
1689 {
1690 struct amdgpu_device *tmp_adev;
1691 int cur_nps_mode, r;
1692
1693 /* This is expected to be called only during unload of driver. The
1694 * request needs to be placed only once for all devices in the hive. If
1695 * one of them fail, revert the request for previous successful devices.
1696 * After placing the request, make hive mode as UNKNOWN so that other
1697 * devices don't request anymore.
1698 */
1699 mutex_lock(&hive->hive_lock);
1700 if (atomic_read(&hive->requested_nps_mode) ==
1701 UNKNOWN_MEMORY_PARTITION_MODE) {
1702 dev_dbg(adev->dev, "Unexpected entry for hive NPS change");
1703 mutex_unlock(&hive->hive_lock);
1704 return 0;
1705 }
1706 list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head) {
1707 r = adev->gmc.gmc_funcs->request_mem_partition_mode(
1708 tmp_adev, req_nps_mode);
1709 if (r)
1710 break;
1711 }
1712 if (r) {
1713 /* Request back current mode if one of the requests failed */
1714 cur_nps_mode =
1715 adev->gmc.gmc_funcs->query_mem_partition_mode(tmp_adev);
1716 list_for_each_entry_continue_reverse(
1717 tmp_adev, &hive->device_list, gmc.xgmi.head)
1718 adev->gmc.gmc_funcs->request_mem_partition_mode(
1719 tmp_adev, cur_nps_mode);
1720 }
1721 /* Set to UNKNOWN so that other devices don't request anymore */
1722 atomic_set(&hive->requested_nps_mode, UNKNOWN_MEMORY_PARTITION_MODE);
1723 mutex_unlock(&hive->hive_lock);
1724
1725 return r;
1726 }
1727
amdgpu_xgmi_same_hive(struct amdgpu_device * adev,struct amdgpu_device * bo_adev)1728 bool amdgpu_xgmi_same_hive(struct amdgpu_device *adev,
1729 struct amdgpu_device *bo_adev)
1730 {
1731 return (amdgpu_use_xgmi_p2p && adev != bo_adev &&
1732 adev->gmc.xgmi.hive_id &&
1733 adev->gmc.xgmi.hive_id == bo_adev->gmc.xgmi.hive_id);
1734 }
1735
amdgpu_xgmi_early_init(struct amdgpu_device * adev)1736 void amdgpu_xgmi_early_init(struct amdgpu_device *adev)
1737 {
1738 if (!adev->gmc.xgmi.supported)
1739 return;
1740
1741 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
1742 case IP_VERSION(9, 4, 0):
1743 case IP_VERSION(9, 4, 1):
1744 case IP_VERSION(9, 4, 2):
1745 adev->gmc.xgmi.max_speed = XGMI_SPEED_25GT;
1746 adev->gmc.xgmi.max_width = 16;
1747 break;
1748 case IP_VERSION(9, 4, 3):
1749 case IP_VERSION(9, 4, 4):
1750 case IP_VERSION(9, 5, 0):
1751 adev->gmc.xgmi.max_speed = XGMI_SPEED_32GT;
1752 adev->gmc.xgmi.max_width = 16;
1753 break;
1754 default:
1755 break;
1756 }
1757 }
1758