1 /*
2 * Copyright 2018 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 *
23 */
24 #include <linux/list.h>
25 #include "amdgpu.h"
26 #include "amdgpu_xgmi.h"
27 #include "amdgpu_ras.h"
28 #include "soc15.h"
29 #include "df/df_3_6_offset.h"
30 #include "xgmi/xgmi_4_0_0_smn.h"
31 #include "xgmi/xgmi_4_0_0_sh_mask.h"
32 #include "xgmi/xgmi_6_1_0_sh_mask.h"
33 #include "wafl/wafl2_4_0_0_smn.h"
34 #include "wafl/wafl2_4_0_0_sh_mask.h"
35
36 #include "amdgpu_reset.h"
37
38 #define smnPCS_XGMI3X16_PCS_ERROR_STATUS 0x11a0020c
39 #define smnPCS_XGMI3X16_PCS_ERROR_NONCORRECTABLE_MASK 0x11a00218
40 #define smnPCS_GOPX1_PCS_ERROR_STATUS 0x12200210
41 #define smnPCS_GOPX1_PCS_ERROR_NONCORRECTABLE_MASK 0x12200218
42
43 static DEFINE_MUTEX(xgmi_mutex);
44
45 #define AMDGPU_MAX_XGMI_DEVICE_PER_HIVE 4
46
47 static DRM_LIST_HEAD(xgmi_hive_list);
48
49 static const int xgmi_pcs_err_status_reg_vg20[] = {
50 smnXGMI0_PCS_GOPX16_PCS_ERROR_STATUS,
51 smnXGMI0_PCS_GOPX16_PCS_ERROR_STATUS + 0x100000,
52 };
53
54 static const int wafl_pcs_err_status_reg_vg20[] = {
55 smnPCS_GOPX1_0_PCS_GOPX1_PCS_ERROR_STATUS,
56 smnPCS_GOPX1_0_PCS_GOPX1_PCS_ERROR_STATUS + 0x100000,
57 };
58
59 static const int xgmi_pcs_err_status_reg_arct[] = {
60 smnXGMI0_PCS_GOPX16_PCS_ERROR_STATUS,
61 smnXGMI0_PCS_GOPX16_PCS_ERROR_STATUS + 0x100000,
62 smnXGMI0_PCS_GOPX16_PCS_ERROR_STATUS + 0x500000,
63 smnXGMI0_PCS_GOPX16_PCS_ERROR_STATUS + 0x600000,
64 smnXGMI0_PCS_GOPX16_PCS_ERROR_STATUS + 0x700000,
65 smnXGMI0_PCS_GOPX16_PCS_ERROR_STATUS + 0x800000,
66 };
67
68 /* same as vg20*/
69 static const int wafl_pcs_err_status_reg_arct[] = {
70 smnPCS_GOPX1_0_PCS_GOPX1_PCS_ERROR_STATUS,
71 smnPCS_GOPX1_0_PCS_GOPX1_PCS_ERROR_STATUS + 0x100000,
72 };
73
74 static const int xgmi3x16_pcs_err_status_reg_aldebaran[] = {
75 smnPCS_XGMI3X16_PCS_ERROR_STATUS,
76 smnPCS_XGMI3X16_PCS_ERROR_STATUS + 0x100000,
77 smnPCS_XGMI3X16_PCS_ERROR_STATUS + 0x200000,
78 smnPCS_XGMI3X16_PCS_ERROR_STATUS + 0x300000,
79 smnPCS_XGMI3X16_PCS_ERROR_STATUS + 0x400000,
80 smnPCS_XGMI3X16_PCS_ERROR_STATUS + 0x500000,
81 smnPCS_XGMI3X16_PCS_ERROR_STATUS + 0x600000,
82 smnPCS_XGMI3X16_PCS_ERROR_STATUS + 0x700000
83 };
84
85 static const int xgmi3x16_pcs_err_noncorrectable_mask_reg_aldebaran[] = {
86 smnPCS_XGMI3X16_PCS_ERROR_NONCORRECTABLE_MASK,
87 smnPCS_XGMI3X16_PCS_ERROR_NONCORRECTABLE_MASK + 0x100000,
88 smnPCS_XGMI3X16_PCS_ERROR_NONCORRECTABLE_MASK + 0x200000,
89 smnPCS_XGMI3X16_PCS_ERROR_NONCORRECTABLE_MASK + 0x300000,
90 smnPCS_XGMI3X16_PCS_ERROR_NONCORRECTABLE_MASK + 0x400000,
91 smnPCS_XGMI3X16_PCS_ERROR_NONCORRECTABLE_MASK + 0x500000,
92 smnPCS_XGMI3X16_PCS_ERROR_NONCORRECTABLE_MASK + 0x600000,
93 smnPCS_XGMI3X16_PCS_ERROR_NONCORRECTABLE_MASK + 0x700000
94 };
95
96 static const int walf_pcs_err_status_reg_aldebaran[] = {
97 smnPCS_GOPX1_PCS_ERROR_STATUS,
98 smnPCS_GOPX1_PCS_ERROR_STATUS + 0x100000
99 };
100
101 static const int walf_pcs_err_noncorrectable_mask_reg_aldebaran[] = {
102 smnPCS_GOPX1_PCS_ERROR_NONCORRECTABLE_MASK,
103 smnPCS_GOPX1_PCS_ERROR_NONCORRECTABLE_MASK + 0x100000
104 };
105
106 static const int xgmi3x16_pcs_err_status_reg_v6_4[] = {
107 smnPCS_XGMI3X16_PCS_ERROR_STATUS,
108 smnPCS_XGMI3X16_PCS_ERROR_STATUS + 0x100000
109 };
110
111 static const int xgmi3x16_pcs_err_noncorrectable_mask_reg_v6_4[] = {
112 smnPCS_XGMI3X16_PCS_ERROR_NONCORRECTABLE_MASK,
113 smnPCS_XGMI3X16_PCS_ERROR_NONCORRECTABLE_MASK + 0x100000
114 };
115
116 static const u64 xgmi_v6_4_0_mca_base_array[] = {
117 0x11a09200,
118 0x11b09200,
119 };
120
121 static const char *xgmi_v6_4_0_ras_error_code_ext[32] = {
122 [0x00] = "XGMI PCS DataLossErr",
123 [0x01] = "XGMI PCS TrainingErr",
124 [0x02] = "XGMI PCS FlowCtrlAckErr",
125 [0x03] = "XGMI PCS RxFifoUnderflowErr",
126 [0x04] = "XGMI PCS RxFifoOverflowErr",
127 [0x05] = "XGMI PCS CRCErr",
128 [0x06] = "XGMI PCS BERExceededErr",
129 [0x07] = "XGMI PCS TxMetaDataErr",
130 [0x08] = "XGMI PCS ReplayBufParityErr",
131 [0x09] = "XGMI PCS DataParityErr",
132 [0x0a] = "XGMI PCS ReplayFifoOverflowErr",
133 [0x0b] = "XGMI PCS ReplayFifoUnderflowErr",
134 [0x0c] = "XGMI PCS ElasticFifoOverflowErr",
135 [0x0d] = "XGMI PCS DeskewErr",
136 [0x0e] = "XGMI PCS FlowCtrlCRCErr",
137 [0x0f] = "XGMI PCS DataStartupLimitErr",
138 [0x10] = "XGMI PCS FCInitTimeoutErr",
139 [0x11] = "XGMI PCS RecoveryTimeoutErr",
140 [0x12] = "XGMI PCS ReadySerialTimeoutErr",
141 [0x13] = "XGMI PCS ReadySerialAttemptErr",
142 [0x14] = "XGMI PCS RecoveryAttemptErr",
143 [0x15] = "XGMI PCS RecoveryRelockAttemptErr",
144 [0x16] = "XGMI PCS ReplayAttemptErr",
145 [0x17] = "XGMI PCS SyncHdrErr",
146 [0x18] = "XGMI PCS TxReplayTimeoutErr",
147 [0x19] = "XGMI PCS RxReplayTimeoutErr",
148 [0x1a] = "XGMI PCS LinkSubTxTimeoutErr",
149 [0x1b] = "XGMI PCS LinkSubRxTimeoutErr",
150 [0x1c] = "XGMI PCS RxCMDPktErr",
151 };
152
153 static const struct amdgpu_pcs_ras_field xgmi_pcs_ras_fields[] = {
154 {"XGMI PCS DataLossErr",
155 SOC15_REG_FIELD(XGMI0_PCS_GOPX16_PCS_ERROR_STATUS, DataLossErr)},
156 {"XGMI PCS TrainingErr",
157 SOC15_REG_FIELD(XGMI0_PCS_GOPX16_PCS_ERROR_STATUS, TrainingErr)},
158 {"XGMI PCS CRCErr",
159 SOC15_REG_FIELD(XGMI0_PCS_GOPX16_PCS_ERROR_STATUS, CRCErr)},
160 {"XGMI PCS BERExceededErr",
161 SOC15_REG_FIELD(XGMI0_PCS_GOPX16_PCS_ERROR_STATUS, BERExceededErr)},
162 {"XGMI PCS TxMetaDataErr",
163 SOC15_REG_FIELD(XGMI0_PCS_GOPX16_PCS_ERROR_STATUS, TxMetaDataErr)},
164 {"XGMI PCS ReplayBufParityErr",
165 SOC15_REG_FIELD(XGMI0_PCS_GOPX16_PCS_ERROR_STATUS, ReplayBufParityErr)},
166 {"XGMI PCS DataParityErr",
167 SOC15_REG_FIELD(XGMI0_PCS_GOPX16_PCS_ERROR_STATUS, DataParityErr)},
168 {"XGMI PCS ReplayFifoOverflowErr",
169 SOC15_REG_FIELD(XGMI0_PCS_GOPX16_PCS_ERROR_STATUS, ReplayFifoOverflowErr)},
170 {"XGMI PCS ReplayFifoUnderflowErr",
171 SOC15_REG_FIELD(XGMI0_PCS_GOPX16_PCS_ERROR_STATUS, ReplayFifoUnderflowErr)},
172 {"XGMI PCS ElasticFifoOverflowErr",
173 SOC15_REG_FIELD(XGMI0_PCS_GOPX16_PCS_ERROR_STATUS, ElasticFifoOverflowErr)},
174 {"XGMI PCS DeskewErr",
175 SOC15_REG_FIELD(XGMI0_PCS_GOPX16_PCS_ERROR_STATUS, DeskewErr)},
176 {"XGMI PCS DataStartupLimitErr",
177 SOC15_REG_FIELD(XGMI0_PCS_GOPX16_PCS_ERROR_STATUS, DataStartupLimitErr)},
178 {"XGMI PCS FCInitTimeoutErr",
179 SOC15_REG_FIELD(XGMI0_PCS_GOPX16_PCS_ERROR_STATUS, FCInitTimeoutErr)},
180 {"XGMI PCS RecoveryTimeoutErr",
181 SOC15_REG_FIELD(XGMI0_PCS_GOPX16_PCS_ERROR_STATUS, RecoveryTimeoutErr)},
182 {"XGMI PCS ReadySerialTimeoutErr",
183 SOC15_REG_FIELD(XGMI0_PCS_GOPX16_PCS_ERROR_STATUS, ReadySerialTimeoutErr)},
184 {"XGMI PCS ReadySerialAttemptErr",
185 SOC15_REG_FIELD(XGMI0_PCS_GOPX16_PCS_ERROR_STATUS, ReadySerialAttemptErr)},
186 {"XGMI PCS RecoveryAttemptErr",
187 SOC15_REG_FIELD(XGMI0_PCS_GOPX16_PCS_ERROR_STATUS, RecoveryAttemptErr)},
188 {"XGMI PCS RecoveryRelockAttemptErr",
189 SOC15_REG_FIELD(XGMI0_PCS_GOPX16_PCS_ERROR_STATUS, RecoveryRelockAttemptErr)},
190 };
191
192 static const struct amdgpu_pcs_ras_field wafl_pcs_ras_fields[] = {
193 {"WAFL PCS DataLossErr",
194 SOC15_REG_FIELD(PCS_GOPX1_0_PCS_GOPX1_PCS_ERROR_STATUS, DataLossErr)},
195 {"WAFL PCS TrainingErr",
196 SOC15_REG_FIELD(PCS_GOPX1_0_PCS_GOPX1_PCS_ERROR_STATUS, TrainingErr)},
197 {"WAFL PCS CRCErr",
198 SOC15_REG_FIELD(PCS_GOPX1_0_PCS_GOPX1_PCS_ERROR_STATUS, CRCErr)},
199 {"WAFL PCS BERExceededErr",
200 SOC15_REG_FIELD(PCS_GOPX1_0_PCS_GOPX1_PCS_ERROR_STATUS, BERExceededErr)},
201 {"WAFL PCS TxMetaDataErr",
202 SOC15_REG_FIELD(PCS_GOPX1_0_PCS_GOPX1_PCS_ERROR_STATUS, TxMetaDataErr)},
203 {"WAFL PCS ReplayBufParityErr",
204 SOC15_REG_FIELD(PCS_GOPX1_0_PCS_GOPX1_PCS_ERROR_STATUS, ReplayBufParityErr)},
205 {"WAFL PCS DataParityErr",
206 SOC15_REG_FIELD(PCS_GOPX1_0_PCS_GOPX1_PCS_ERROR_STATUS, DataParityErr)},
207 {"WAFL PCS ReplayFifoOverflowErr",
208 SOC15_REG_FIELD(PCS_GOPX1_0_PCS_GOPX1_PCS_ERROR_STATUS, ReplayFifoOverflowErr)},
209 {"WAFL PCS ReplayFifoUnderflowErr",
210 SOC15_REG_FIELD(PCS_GOPX1_0_PCS_GOPX1_PCS_ERROR_STATUS, ReplayFifoUnderflowErr)},
211 {"WAFL PCS ElasticFifoOverflowErr",
212 SOC15_REG_FIELD(PCS_GOPX1_0_PCS_GOPX1_PCS_ERROR_STATUS, ElasticFifoOverflowErr)},
213 {"WAFL PCS DeskewErr",
214 SOC15_REG_FIELD(PCS_GOPX1_0_PCS_GOPX1_PCS_ERROR_STATUS, DeskewErr)},
215 {"WAFL PCS DataStartupLimitErr",
216 SOC15_REG_FIELD(PCS_GOPX1_0_PCS_GOPX1_PCS_ERROR_STATUS, DataStartupLimitErr)},
217 {"WAFL PCS FCInitTimeoutErr",
218 SOC15_REG_FIELD(PCS_GOPX1_0_PCS_GOPX1_PCS_ERROR_STATUS, FCInitTimeoutErr)},
219 {"WAFL PCS RecoveryTimeoutErr",
220 SOC15_REG_FIELD(PCS_GOPX1_0_PCS_GOPX1_PCS_ERROR_STATUS, RecoveryTimeoutErr)},
221 {"WAFL PCS ReadySerialTimeoutErr",
222 SOC15_REG_FIELD(PCS_GOPX1_0_PCS_GOPX1_PCS_ERROR_STATUS, ReadySerialTimeoutErr)},
223 {"WAFL PCS ReadySerialAttemptErr",
224 SOC15_REG_FIELD(PCS_GOPX1_0_PCS_GOPX1_PCS_ERROR_STATUS, ReadySerialAttemptErr)},
225 {"WAFL PCS RecoveryAttemptErr",
226 SOC15_REG_FIELD(PCS_GOPX1_0_PCS_GOPX1_PCS_ERROR_STATUS, RecoveryAttemptErr)},
227 {"WAFL PCS RecoveryRelockAttemptErr",
228 SOC15_REG_FIELD(PCS_GOPX1_0_PCS_GOPX1_PCS_ERROR_STATUS, RecoveryRelockAttemptErr)},
229 };
230
231 static const struct amdgpu_pcs_ras_field xgmi3x16_pcs_ras_fields[] = {
232 {"XGMI3X16 PCS DataLossErr",
233 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, DataLossErr)},
234 {"XGMI3X16 PCS TrainingErr",
235 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, TrainingErr)},
236 {"XGMI3X16 PCS FlowCtrlAckErr",
237 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, FlowCtrlAckErr)},
238 {"XGMI3X16 PCS RxFifoUnderflowErr",
239 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, RxFifoUnderflowErr)},
240 {"XGMI3X16 PCS RxFifoOverflowErr",
241 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, RxFifoOverflowErr)},
242 {"XGMI3X16 PCS CRCErr",
243 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, CRCErr)},
244 {"XGMI3X16 PCS BERExceededErr",
245 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, BERExceededErr)},
246 {"XGMI3X16 PCS TxVcidDataErr",
247 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, TxVcidDataErr)},
248 {"XGMI3X16 PCS ReplayBufParityErr",
249 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, ReplayBufParityErr)},
250 {"XGMI3X16 PCS DataParityErr",
251 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, DataParityErr)},
252 {"XGMI3X16 PCS ReplayFifoOverflowErr",
253 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, ReplayFifoOverflowErr)},
254 {"XGMI3X16 PCS ReplayFifoUnderflowErr",
255 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, ReplayFifoUnderflowErr)},
256 {"XGMI3X16 PCS ElasticFifoOverflowErr",
257 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, ElasticFifoOverflowErr)},
258 {"XGMI3X16 PCS DeskewErr",
259 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, DeskewErr)},
260 {"XGMI3X16 PCS FlowCtrlCRCErr",
261 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, FlowCtrlCRCErr)},
262 {"XGMI3X16 PCS DataStartupLimitErr",
263 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, DataStartupLimitErr)},
264 {"XGMI3X16 PCS FCInitTimeoutErr",
265 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, FCInitTimeoutErr)},
266 {"XGMI3X16 PCS RecoveryTimeoutErr",
267 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, RecoveryTimeoutErr)},
268 {"XGMI3X16 PCS ReadySerialTimeoutErr",
269 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, ReadySerialTimeoutErr)},
270 {"XGMI3X16 PCS ReadySerialAttemptErr",
271 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, ReadySerialAttemptErr)},
272 {"XGMI3X16 PCS RecoveryAttemptErr",
273 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, RecoveryAttemptErr)},
274 {"XGMI3X16 PCS RecoveryRelockAttemptErr",
275 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, RecoveryRelockAttemptErr)},
276 {"XGMI3X16 PCS ReplayAttemptErr",
277 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, ReplayAttemptErr)},
278 {"XGMI3X16 PCS SyncHdrErr",
279 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, SyncHdrErr)},
280 {"XGMI3X16 PCS TxReplayTimeoutErr",
281 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, TxReplayTimeoutErr)},
282 {"XGMI3X16 PCS RxReplayTimeoutErr",
283 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, RxReplayTimeoutErr)},
284 {"XGMI3X16 PCS LinkSubTxTimeoutErr",
285 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, LinkSubTxTimeoutErr)},
286 {"XGMI3X16 PCS LinkSubRxTimeoutErr",
287 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, LinkSubRxTimeoutErr)},
288 {"XGMI3X16 PCS RxCMDPktErr",
289 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, RxCMDPktErr)},
290 };
291
292 /**
293 * DOC: AMDGPU XGMI Support
294 *
295 * XGMI is a high speed interconnect that joins multiple GPU cards
296 * into a homogeneous memory space that is organized by a collective
297 * hive ID and individual node IDs, both of which are 64-bit numbers.
298 *
299 * The file xgmi_device_id contains the unique per GPU device ID and
300 * is stored in the /sys/class/drm/card${cardno}/device/ directory.
301 *
302 * Inside the device directory a sub-directory 'xgmi_hive_info' is
303 * created which contains the hive ID and the list of nodes.
304 *
305 * The hive ID is stored in:
306 * /sys/class/drm/card${cardno}/device/xgmi_hive_info/xgmi_hive_id
307 *
308 * The node information is stored in numbered directories:
309 * /sys/class/drm/card${cardno}/device/xgmi_hive_info/node${nodeno}/xgmi_device_id
310 *
311 * Each device has their own xgmi_hive_info direction with a mirror
312 * set of node sub-directories.
313 *
314 * The XGMI memory space is built by contiguously adding the power of
315 * two padded VRAM space from each node to each other.
316 *
317 */
318
319 static struct attribute amdgpu_xgmi_hive_id = {
320 .name = "xgmi_hive_id",
321 #ifdef notyet
322 .mode = S_IRUGO
323 #endif
324 };
325
326 static struct attribute *amdgpu_xgmi_hive_attrs[] = {
327 &amdgpu_xgmi_hive_id,
328 NULL
329 };
330 ATTRIBUTE_GROUPS(amdgpu_xgmi_hive);
331
amdgpu_xgmi_show_attrs(struct kobject * kobj,struct attribute * attr,char * buf)332 static ssize_t amdgpu_xgmi_show_attrs(struct kobject *kobj,
333 struct attribute *attr, char *buf)
334 {
335 struct amdgpu_hive_info *hive = container_of(
336 kobj, struct amdgpu_hive_info, kobj);
337
338 if (attr == &amdgpu_xgmi_hive_id)
339 return snprintf(buf, PAGE_SIZE, "%llu\n", hive->hive_id);
340
341 return 0;
342 }
343
amdgpu_xgmi_hive_release(struct kobject * kobj)344 static void amdgpu_xgmi_hive_release(struct kobject *kobj)
345 {
346 struct amdgpu_hive_info *hive = container_of(
347 kobj, struct amdgpu_hive_info, kobj);
348
349 amdgpu_reset_put_reset_domain(hive->reset_domain);
350 hive->reset_domain = NULL;
351
352 mutex_destroy(&hive->hive_lock);
353 kfree(hive);
354 }
355
356 #ifdef notyet
357 static const struct sysfs_ops amdgpu_xgmi_hive_ops = {
358 .show = amdgpu_xgmi_show_attrs,
359 };
360 #endif
361
362 static const struct kobj_type amdgpu_xgmi_hive_type = {
363 .release = amdgpu_xgmi_hive_release,
364 #ifdef notyet
365 .sysfs_ops = &amdgpu_xgmi_hive_ops,
366 .default_groups = amdgpu_xgmi_hive_groups,
367 #endif
368 };
369
amdgpu_xgmi_show_device_id(struct device * dev,struct device_attribute * attr,char * buf)370 static ssize_t amdgpu_xgmi_show_device_id(struct device *dev,
371 struct device_attribute *attr,
372 char *buf)
373 {
374 struct drm_device *ddev = dev_get_drvdata(dev);
375 struct amdgpu_device *adev = drm_to_adev(ddev);
376
377 return sysfs_emit(buf, "%llu\n", adev->gmc.xgmi.node_id);
378
379 }
380
amdgpu_xgmi_show_physical_id(struct device * dev,struct device_attribute * attr,char * buf)381 static ssize_t amdgpu_xgmi_show_physical_id(struct device *dev,
382 struct device_attribute *attr,
383 char *buf)
384 {
385 struct drm_device *ddev = dev_get_drvdata(dev);
386 struct amdgpu_device *adev = drm_to_adev(ddev);
387
388 return sysfs_emit(buf, "%u\n", adev->gmc.xgmi.physical_node_id);
389
390 }
391
amdgpu_xgmi_show_num_hops(struct device * dev,struct device_attribute * attr,char * buf)392 static ssize_t amdgpu_xgmi_show_num_hops(struct device *dev,
393 struct device_attribute *attr,
394 char *buf)
395 {
396 STUB();
397 return -ENOSYS;
398 #ifdef __linux__
399 struct drm_device *ddev = dev_get_drvdata(dev);
400 struct amdgpu_device *adev = drm_to_adev(ddev);
401 struct psp_xgmi_topology_info *top = &adev->psp.xgmi_context.top_info;
402 int i;
403
404 for (i = 0; i < top->num_nodes; i++)
405 sprintf(buf + 3 * i, "%02x ", top->nodes[i].num_hops);
406
407 return sysfs_emit(buf, "%s\n", buf);
408 #endif
409 }
410
amdgpu_xgmi_show_num_links(struct device * dev,struct device_attribute * attr,char * buf)411 static ssize_t amdgpu_xgmi_show_num_links(struct device *dev,
412 struct device_attribute *attr,
413 char *buf)
414 {
415 STUB();
416 return -ENOSYS;
417 #ifdef __linux__
418 struct drm_device *ddev = dev_get_drvdata(dev);
419 struct amdgpu_device *adev = drm_to_adev(ddev);
420 struct psp_xgmi_topology_info *top = &adev->psp.xgmi_context.top_info;
421 int i;
422
423 for (i = 0; i < top->num_nodes; i++)
424 sprintf(buf + 3 * i, "%02x ", top->nodes[i].num_links);
425
426 return sysfs_emit(buf, "%s\n", buf);
427 #endif
428 }
429
amdgpu_xgmi_show_connected_port_num(struct device * dev,struct device_attribute * attr,char * buf)430 static ssize_t amdgpu_xgmi_show_connected_port_num(struct device *dev,
431 struct device_attribute *attr,
432 char *buf)
433 {
434 struct drm_device *ddev = dev_get_drvdata(dev);
435 struct amdgpu_device *adev = drm_to_adev(ddev);
436 struct psp_xgmi_topology_info *top = &adev->psp.xgmi_context.top_info;
437 int i, j, size = 0;
438 int current_node;
439 /*
440 * get the node id in the sysfs for the current socket and show
441 * it in the port num info output in the sysfs for easy reading.
442 * it is NOT the one retrieved from xgmi ta.
443 */
444 for (i = 0; i < top->num_nodes; i++) {
445 if (top->nodes[i].node_id == adev->gmc.xgmi.node_id) {
446 current_node = i;
447 break;
448 }
449 }
450
451 if (i == top->num_nodes)
452 return -EINVAL;
453
454 for (i = 0; i < top->num_nodes; i++) {
455 for (j = 0; j < top->nodes[i].num_links; j++)
456 /* node id in sysfs starts from 1 rather than 0 so +1 here */
457 size += sysfs_emit_at(buf, size, "%02x:%02x -> %02x:%02x\n", current_node + 1,
458 top->nodes[i].port_num[j].src_xgmi_port_num, i + 1,
459 top->nodes[i].port_num[j].dst_xgmi_port_num);
460 }
461
462 return size;
463 }
464
465 #define AMDGPU_XGMI_SET_FICAA(o) ((o) | 0x456801)
amdgpu_xgmi_show_error(struct device * dev,struct device_attribute * attr,char * buf)466 static ssize_t amdgpu_xgmi_show_error(struct device *dev,
467 struct device_attribute *attr,
468 char *buf)
469 {
470 struct drm_device *ddev = dev_get_drvdata(dev);
471 struct amdgpu_device *adev = drm_to_adev(ddev);
472 uint32_t ficaa_pie_ctl_in, ficaa_pie_status_in;
473 uint64_t fica_out;
474 unsigned int error_count = 0;
475
476 ficaa_pie_ctl_in = AMDGPU_XGMI_SET_FICAA(0x200);
477 ficaa_pie_status_in = AMDGPU_XGMI_SET_FICAA(0x208);
478
479 if ((!adev->df.funcs) ||
480 (!adev->df.funcs->get_fica) ||
481 (!adev->df.funcs->set_fica))
482 return -EINVAL;
483
484 fica_out = adev->df.funcs->get_fica(adev, ficaa_pie_ctl_in);
485 if (fica_out != 0x1f)
486 pr_err("xGMI error counters not enabled!\n");
487
488 fica_out = adev->df.funcs->get_fica(adev, ficaa_pie_status_in);
489
490 if ((fica_out & 0xffff) == 2)
491 error_count = ((fica_out >> 62) & 0x1) + (fica_out >> 63);
492
493 adev->df.funcs->set_fica(adev, ficaa_pie_status_in, 0, 0);
494
495 return sysfs_emit(buf, "%u\n", error_count);
496 }
497
498
499 static DEVICE_ATTR(xgmi_device_id, S_IRUGO, amdgpu_xgmi_show_device_id, NULL);
500 static DEVICE_ATTR(xgmi_physical_id, 0444, amdgpu_xgmi_show_physical_id, NULL);
501 static DEVICE_ATTR(xgmi_error, S_IRUGO, amdgpu_xgmi_show_error, NULL);
502 static DEVICE_ATTR(xgmi_num_hops, S_IRUGO, amdgpu_xgmi_show_num_hops, NULL);
503 static DEVICE_ATTR(xgmi_num_links, S_IRUGO, amdgpu_xgmi_show_num_links, NULL);
504 static DEVICE_ATTR(xgmi_port_num, S_IRUGO, amdgpu_xgmi_show_connected_port_num, NULL);
505
amdgpu_xgmi_sysfs_add_dev_info(struct amdgpu_device * adev,struct amdgpu_hive_info * hive)506 static int amdgpu_xgmi_sysfs_add_dev_info(struct amdgpu_device *adev,
507 struct amdgpu_hive_info *hive)
508 {
509 STUB();
510 return -ENOSYS;
511 #ifdef notyet
512 int ret = 0;
513 char node[10] = { 0 };
514
515 /* Create xgmi device id file */
516 ret = device_create_file(adev->dev, &dev_attr_xgmi_device_id);
517 if (ret) {
518 dev_err(adev->dev, "XGMI: Failed to create device file xgmi_device_id\n");
519 return ret;
520 }
521
522 ret = device_create_file(adev->dev, &dev_attr_xgmi_physical_id);
523 if (ret) {
524 dev_err(adev->dev, "XGMI: Failed to create device file xgmi_physical_id\n");
525 return ret;
526 }
527
528 /* Create xgmi error file */
529 ret = device_create_file(adev->dev, &dev_attr_xgmi_error);
530 if (ret)
531 pr_err("failed to create xgmi_error\n");
532
533 /* Create xgmi num hops file */
534 ret = device_create_file(adev->dev, &dev_attr_xgmi_num_hops);
535 if (ret)
536 pr_err("failed to create xgmi_num_hops\n");
537
538 /* Create xgmi num links file */
539 ret = device_create_file(adev->dev, &dev_attr_xgmi_num_links);
540 if (ret)
541 pr_err("failed to create xgmi_num_links\n");
542
543 /* Create xgmi port num file if supported */
544 if (adev->psp.xgmi_context.xgmi_ta_caps & EXTEND_PEER_LINK_INFO_CMD_FLAG) {
545 ret = device_create_file(adev->dev, &dev_attr_xgmi_port_num);
546 if (ret)
547 dev_err(adev->dev, "failed to create xgmi_port_num\n");
548 }
549
550 /* Create sysfs link to hive info folder on the first device */
551 if (hive->kobj.parent != (&adev->dev->kobj)) {
552 ret = sysfs_create_link(&adev->dev->kobj, &hive->kobj,
553 "xgmi_hive_info");
554 if (ret) {
555 dev_err(adev->dev, "XGMI: Failed to create link to hive info");
556 goto remove_file;
557 }
558 }
559
560 snprintf(node, sizeof(node), "node%d", atomic_read(&hive->number_devices));
561 /* Create sysfs link form the hive folder to yourself */
562 ret = sysfs_create_link(&hive->kobj, &adev->dev->kobj, node);
563 if (ret) {
564 dev_err(adev->dev, "XGMI: Failed to create link from hive info");
565 goto remove_link;
566 }
567
568 goto success;
569
570
571 remove_link:
572 sysfs_remove_link(&adev->dev->kobj, adev_to_drm(adev)->unique);
573
574 remove_file:
575 device_remove_file(adev->dev, &dev_attr_xgmi_device_id);
576 device_remove_file(adev->dev, &dev_attr_xgmi_physical_id);
577 device_remove_file(adev->dev, &dev_attr_xgmi_error);
578 device_remove_file(adev->dev, &dev_attr_xgmi_num_hops);
579 device_remove_file(adev->dev, &dev_attr_xgmi_num_links);
580 if (adev->psp.xgmi_context.xgmi_ta_caps & EXTEND_PEER_LINK_INFO_CMD_FLAG)
581 device_remove_file(adev->dev, &dev_attr_xgmi_port_num);
582
583 success:
584 return ret;
585 #endif
586 }
587
amdgpu_xgmi_sysfs_rem_dev_info(struct amdgpu_device * adev,struct amdgpu_hive_info * hive)588 static void amdgpu_xgmi_sysfs_rem_dev_info(struct amdgpu_device *adev,
589 struct amdgpu_hive_info *hive)
590 {
591 #ifdef __linux__
592 char node[10];
593 memset(node, 0, sizeof(node));
594
595 device_remove_file(adev->dev, &dev_attr_xgmi_device_id);
596 device_remove_file(adev->dev, &dev_attr_xgmi_physical_id);
597 device_remove_file(adev->dev, &dev_attr_xgmi_error);
598 device_remove_file(adev->dev, &dev_attr_xgmi_num_hops);
599 device_remove_file(adev->dev, &dev_attr_xgmi_num_links);
600 if (adev->psp.xgmi_context.xgmi_ta_caps & EXTEND_PEER_LINK_INFO_CMD_FLAG)
601 device_remove_file(adev->dev, &dev_attr_xgmi_port_num);
602
603 if (hive->kobj.parent != (&adev->dev->kobj))
604 sysfs_remove_link(&adev->dev->kobj,"xgmi_hive_info");
605
606 sprintf(node, "node%d", atomic_read(&hive->number_devices));
607 sysfs_remove_link(&hive->kobj, node);
608 #endif
609 }
610
611
612
amdgpu_get_xgmi_hive(struct amdgpu_device * adev)613 struct amdgpu_hive_info *amdgpu_get_xgmi_hive(struct amdgpu_device *adev)
614 {
615 struct amdgpu_hive_info *hive = NULL;
616 int ret;
617
618 if (!adev->gmc.xgmi.hive_id)
619 return NULL;
620
621 STUB();
622 return NULL;
623 #ifdef notyet
624
625 if (adev->hive) {
626 kobject_get(&adev->hive->kobj);
627 return adev->hive;
628 }
629
630 mutex_lock(&xgmi_mutex);
631
632 list_for_each_entry(hive, &xgmi_hive_list, node) {
633 if (hive->hive_id == adev->gmc.xgmi.hive_id)
634 goto pro_end;
635 }
636
637 hive = kzalloc(sizeof(*hive), GFP_KERNEL);
638 if (!hive) {
639 dev_err(adev->dev, "XGMI: allocation failed\n");
640 ret = -ENOMEM;
641 hive = NULL;
642 goto pro_end;
643 }
644
645 /* initialize new hive if not exist */
646 ret = kobject_init_and_add(&hive->kobj,
647 &amdgpu_xgmi_hive_type,
648 &adev->dev->kobj,
649 "%s", "xgmi_hive_info");
650 if (ret) {
651 dev_err(adev->dev, "XGMI: failed initializing kobject for xgmi hive\n");
652 kobject_put(&hive->kobj);
653 hive = NULL;
654 goto pro_end;
655 }
656
657 /**
658 * Only init hive->reset_domain for none SRIOV configuration. For SRIOV,
659 * Host driver decide how to reset the GPU either through FLR or chain reset.
660 * Guest side will get individual notifications from the host for the FLR
661 * if necessary.
662 */
663 if (!amdgpu_sriov_vf(adev)) {
664 /**
665 * Avoid recreating reset domain when hive is reconstructed for the case
666 * of reset the devices in the XGMI hive during probe for passthrough GPU
667 * See https://www.spinics.net/lists/amd-gfx/msg58836.html
668 */
669 if (adev->reset_domain->type != XGMI_HIVE) {
670 hive->reset_domain =
671 amdgpu_reset_create_reset_domain(XGMI_HIVE, "amdgpu-reset-hive");
672 if (!hive->reset_domain) {
673 dev_err(adev->dev, "XGMI: failed initializing reset domain for xgmi hive\n");
674 ret = -ENOMEM;
675 kobject_put(&hive->kobj);
676 hive = NULL;
677 goto pro_end;
678 }
679 } else {
680 amdgpu_reset_get_reset_domain(adev->reset_domain);
681 hive->reset_domain = adev->reset_domain;
682 }
683 }
684
685 hive->hive_id = adev->gmc.xgmi.hive_id;
686 INIT_LIST_HEAD(&hive->device_list);
687 INIT_LIST_HEAD(&hive->node);
688 rw_init(&hive->hive_lock, "aghive");
689 atomic_set(&hive->number_devices, 0);
690 task_barrier_init(&hive->tb);
691 hive->pstate = AMDGPU_XGMI_PSTATE_UNKNOWN;
692 hive->hi_req_gpu = NULL;
693
694 /*
695 * hive pstate on boot is high in vega20 so we have to go to low
696 * pstate on after boot.
697 */
698 hive->hi_req_count = AMDGPU_MAX_XGMI_DEVICE_PER_HIVE;
699 list_add_tail(&hive->node, &xgmi_hive_list);
700
701 pro_end:
702 if (hive)
703 kobject_get(&hive->kobj);
704 mutex_unlock(&xgmi_mutex);
705 return hive;
706 #endif
707 }
708
amdgpu_put_xgmi_hive(struct amdgpu_hive_info * hive)709 void amdgpu_put_xgmi_hive(struct amdgpu_hive_info *hive)
710 {
711 if (hive)
712 kobject_put(&hive->kobj);
713 }
714
amdgpu_xgmi_set_pstate(struct amdgpu_device * adev,int pstate)715 int amdgpu_xgmi_set_pstate(struct amdgpu_device *adev, int pstate)
716 {
717 int ret = 0;
718 struct amdgpu_hive_info *hive;
719 struct amdgpu_device *request_adev;
720 bool is_hi_req = pstate == AMDGPU_XGMI_PSTATE_MAX_VEGA20;
721 bool init_low;
722
723 hive = amdgpu_get_xgmi_hive(adev);
724 if (!hive)
725 return 0;
726
727 request_adev = hive->hi_req_gpu ? hive->hi_req_gpu : adev;
728 init_low = hive->pstate == AMDGPU_XGMI_PSTATE_UNKNOWN;
729 amdgpu_put_xgmi_hive(hive);
730 /* fw bug so temporarily disable pstate switching */
731 return 0;
732
733 if (!hive || adev->asic_type != CHIP_VEGA20)
734 return 0;
735
736 mutex_lock(&hive->hive_lock);
737
738 if (is_hi_req)
739 hive->hi_req_count++;
740 else
741 hive->hi_req_count--;
742
743 /*
744 * Vega20 only needs single peer to request pstate high for the hive to
745 * go high but all peers must request pstate low for the hive to go low
746 */
747 if (hive->pstate == pstate ||
748 (!is_hi_req && hive->hi_req_count && !init_low))
749 goto out;
750
751 dev_dbg(request_adev->dev, "Set xgmi pstate %d.\n", pstate);
752
753 ret = amdgpu_dpm_set_xgmi_pstate(request_adev, pstate);
754 if (ret) {
755 dev_err(request_adev->dev,
756 "XGMI: Set pstate failure on device %llx, hive %llx, ret %d",
757 request_adev->gmc.xgmi.node_id,
758 request_adev->gmc.xgmi.hive_id, ret);
759 goto out;
760 }
761
762 if (init_low)
763 hive->pstate = hive->hi_req_count ?
764 hive->pstate : AMDGPU_XGMI_PSTATE_MIN;
765 else {
766 hive->pstate = pstate;
767 hive->hi_req_gpu = pstate != AMDGPU_XGMI_PSTATE_MIN ?
768 adev : NULL;
769 }
770 out:
771 mutex_unlock(&hive->hive_lock);
772 return ret;
773 }
774
amdgpu_xgmi_update_topology(struct amdgpu_hive_info * hive,struct amdgpu_device * adev)775 int amdgpu_xgmi_update_topology(struct amdgpu_hive_info *hive, struct amdgpu_device *adev)
776 {
777 int ret;
778
779 if (amdgpu_sriov_vf(adev))
780 return 0;
781
782 /* Each psp need to set the latest topology */
783 ret = psp_xgmi_set_topology_info(&adev->psp,
784 atomic_read(&hive->number_devices),
785 &adev->psp.xgmi_context.top_info);
786 if (ret)
787 dev_err(adev->dev,
788 "XGMI: Set topology failure on device %llx, hive %llx, ret %d",
789 adev->gmc.xgmi.node_id,
790 adev->gmc.xgmi.hive_id, ret);
791
792 return ret;
793 }
794
795
796 /*
797 * NOTE psp_xgmi_node_info.num_hops layout is as follows:
798 * num_hops[7:6] = link type (0 = xGMI2, 1 = xGMI3, 2/3 = reserved)
799 * num_hops[5:3] = reserved
800 * num_hops[2:0] = number of hops
801 */
amdgpu_xgmi_get_hops_count(struct amdgpu_device * adev,struct amdgpu_device * peer_adev)802 int amdgpu_xgmi_get_hops_count(struct amdgpu_device *adev,
803 struct amdgpu_device *peer_adev)
804 {
805 struct psp_xgmi_topology_info *top = &adev->psp.xgmi_context.top_info;
806 uint8_t num_hops_mask = 0x7;
807 int i;
808
809 for (i = 0 ; i < top->num_nodes; ++i)
810 if (top->nodes[i].node_id == peer_adev->gmc.xgmi.node_id)
811 return top->nodes[i].num_hops & num_hops_mask;
812 return -EINVAL;
813 }
814
amdgpu_xgmi_get_num_links(struct amdgpu_device * adev,struct amdgpu_device * peer_adev)815 int amdgpu_xgmi_get_num_links(struct amdgpu_device *adev,
816 struct amdgpu_device *peer_adev)
817 {
818 struct psp_xgmi_topology_info *top = &adev->psp.xgmi_context.top_info;
819 int i;
820
821 for (i = 0 ; i < top->num_nodes; ++i)
822 if (top->nodes[i].node_id == peer_adev->gmc.xgmi.node_id)
823 return top->nodes[i].num_links;
824 return -EINVAL;
825 }
826
827 /*
828 * Devices that support extended data require the entire hive to initialize with
829 * the shared memory buffer flag set.
830 *
831 * Hive locks and conditions apply - see amdgpu_xgmi_add_device
832 */
amdgpu_xgmi_initialize_hive_get_data_partition(struct amdgpu_hive_info * hive,bool set_extended_data)833 static int amdgpu_xgmi_initialize_hive_get_data_partition(struct amdgpu_hive_info *hive,
834 bool set_extended_data)
835 {
836 struct amdgpu_device *tmp_adev;
837 int ret;
838
839 list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head) {
840 ret = psp_xgmi_initialize(&tmp_adev->psp, set_extended_data, false);
841 if (ret) {
842 dev_err(tmp_adev->dev,
843 "XGMI: Failed to initialize xgmi session for data partition %i\n",
844 set_extended_data);
845 return ret;
846 }
847
848 }
849
850 return 0;
851 }
852
amdgpu_xgmi_fill_topology_info(struct amdgpu_device * adev,struct amdgpu_device * peer_adev)853 static void amdgpu_xgmi_fill_topology_info(struct amdgpu_device *adev,
854 struct amdgpu_device *peer_adev)
855 {
856 struct psp_xgmi_topology_info *top_info = &adev->psp.xgmi_context.top_info;
857 struct psp_xgmi_topology_info *peer_info = &peer_adev->psp.xgmi_context.top_info;
858
859 for (int i = 0; i < peer_info->num_nodes; i++) {
860 if (peer_info->nodes[i].node_id == adev->gmc.xgmi.node_id) {
861 for (int j = 0; j < top_info->num_nodes; j++) {
862 if (top_info->nodes[j].node_id == peer_adev->gmc.xgmi.node_id) {
863 peer_info->nodes[i].num_hops = top_info->nodes[j].num_hops;
864 peer_info->nodes[i].is_sharing_enabled =
865 top_info->nodes[j].is_sharing_enabled;
866 peer_info->nodes[i].num_links =
867 top_info->nodes[j].num_links;
868 return;
869 }
870 }
871 }
872 }
873 }
874
amdgpu_xgmi_add_device(struct amdgpu_device * adev)875 int amdgpu_xgmi_add_device(struct amdgpu_device *adev)
876 {
877 struct psp_xgmi_topology_info *top_info;
878 struct amdgpu_hive_info *hive;
879 struct amdgpu_xgmi *entry;
880 struct amdgpu_device *tmp_adev = NULL;
881
882 int count = 0, ret = 0;
883
884 if (!adev->gmc.xgmi.supported)
885 return 0;
886
887 if (!adev->gmc.xgmi.pending_reset &&
888 amdgpu_device_ip_get_ip_block(adev, AMD_IP_BLOCK_TYPE_PSP)) {
889 ret = psp_xgmi_initialize(&adev->psp, false, true);
890 if (ret) {
891 dev_err(adev->dev,
892 "XGMI: Failed to initialize xgmi session\n");
893 return ret;
894 }
895
896 ret = psp_xgmi_get_hive_id(&adev->psp, &adev->gmc.xgmi.hive_id);
897 if (ret) {
898 dev_err(adev->dev,
899 "XGMI: Failed to get hive id\n");
900 return ret;
901 }
902
903 ret = psp_xgmi_get_node_id(&adev->psp, &adev->gmc.xgmi.node_id);
904 if (ret) {
905 dev_err(adev->dev,
906 "XGMI: Failed to get node id\n");
907 return ret;
908 }
909 } else {
910 adev->gmc.xgmi.hive_id = 16;
911 adev->gmc.xgmi.node_id = adev->gmc.xgmi.physical_node_id + 16;
912 }
913
914 hive = amdgpu_get_xgmi_hive(adev);
915 if (!hive) {
916 ret = -EINVAL;
917 dev_err(adev->dev,
918 "XGMI: node 0x%llx, can not match hive 0x%llx in the hive list.\n",
919 adev->gmc.xgmi.node_id, adev->gmc.xgmi.hive_id);
920 goto exit;
921 }
922 mutex_lock(&hive->hive_lock);
923
924 top_info = &adev->psp.xgmi_context.top_info;
925
926 list_add_tail(&adev->gmc.xgmi.head, &hive->device_list);
927 list_for_each_entry(entry, &hive->device_list, head)
928 top_info->nodes[count++].node_id = entry->node_id;
929 top_info->num_nodes = count;
930 atomic_set(&hive->number_devices, count);
931
932 task_barrier_add_task(&hive->tb);
933
934 if (!adev->gmc.xgmi.pending_reset &&
935 amdgpu_device_ip_get_ip_block(adev, AMD_IP_BLOCK_TYPE_PSP)) {
936 list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head) {
937 /* update node list for other device in the hive */
938 if (tmp_adev != adev) {
939 top_info = &tmp_adev->psp.xgmi_context.top_info;
940 top_info->nodes[count - 1].node_id =
941 adev->gmc.xgmi.node_id;
942 top_info->num_nodes = count;
943 }
944 ret = amdgpu_xgmi_update_topology(hive, tmp_adev);
945 if (ret)
946 goto exit_unlock;
947 }
948
949 if (amdgpu_sriov_vf(adev) &&
950 adev->psp.xgmi_context.xgmi_ta_caps & EXTEND_PEER_LINK_INFO_CMD_FLAG) {
951 /* only get topology for VF being init if it can support full duplex */
952 ret = psp_xgmi_get_topology_info(&adev->psp, count,
953 &adev->psp.xgmi_context.top_info, false);
954 if (ret) {
955 dev_err(adev->dev,
956 "XGMI: Get topology failure on device %llx, hive %llx, ret %d",
957 adev->gmc.xgmi.node_id,
958 adev->gmc.xgmi.hive_id, ret);
959 /* To do: continue with some node failed or disable the whole hive*/
960 goto exit_unlock;
961 }
962
963 /* fill the topology info for peers instead of getting from PSP */
964 list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head) {
965 amdgpu_xgmi_fill_topology_info(adev, tmp_adev);
966 }
967 } else {
968 /* get latest topology info for each device from psp */
969 list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head) {
970 ret = psp_xgmi_get_topology_info(&tmp_adev->psp, count,
971 &tmp_adev->psp.xgmi_context.top_info, false);
972 if (ret) {
973 dev_err(tmp_adev->dev,
974 "XGMI: Get topology failure on device %llx, hive %llx, ret %d",
975 tmp_adev->gmc.xgmi.node_id,
976 tmp_adev->gmc.xgmi.hive_id, ret);
977 /* To do : continue with some node failed or disable the whole hive */
978 goto exit_unlock;
979 }
980 }
981 }
982
983 /* get topology again for hives that support extended data */
984 if (adev->psp.xgmi_context.supports_extended_data) {
985
986 /* initialize the hive to get extended data. */
987 ret = amdgpu_xgmi_initialize_hive_get_data_partition(hive, true);
988 if (ret)
989 goto exit_unlock;
990
991 /* get the extended data. */
992 list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head) {
993 ret = psp_xgmi_get_topology_info(&tmp_adev->psp, count,
994 &tmp_adev->psp.xgmi_context.top_info, true);
995 if (ret) {
996 dev_err(tmp_adev->dev,
997 "XGMI: Get topology for extended data failure on device %llx, hive %llx, ret %d",
998 tmp_adev->gmc.xgmi.node_id,
999 tmp_adev->gmc.xgmi.hive_id, ret);
1000 goto exit_unlock;
1001 }
1002 }
1003
1004 /* initialize the hive to get non-extended data for the next round. */
1005 ret = amdgpu_xgmi_initialize_hive_get_data_partition(hive, false);
1006 if (ret)
1007 goto exit_unlock;
1008
1009 }
1010 }
1011
1012 if (!ret && !adev->gmc.xgmi.pending_reset)
1013 ret = amdgpu_xgmi_sysfs_add_dev_info(adev, hive);
1014
1015 exit_unlock:
1016 mutex_unlock(&hive->hive_lock);
1017 exit:
1018 if (!ret) {
1019 adev->hive = hive;
1020 dev_info(adev->dev, "XGMI: Add node %d, hive 0x%llx.\n",
1021 adev->gmc.xgmi.physical_node_id, adev->gmc.xgmi.hive_id);
1022 } else {
1023 amdgpu_put_xgmi_hive(hive);
1024 dev_err(adev->dev, "XGMI: Failed to add node %d, hive 0x%llx ret: %d\n",
1025 adev->gmc.xgmi.physical_node_id, adev->gmc.xgmi.hive_id,
1026 ret);
1027 }
1028
1029 return ret;
1030 }
1031
amdgpu_xgmi_remove_device(struct amdgpu_device * adev)1032 int amdgpu_xgmi_remove_device(struct amdgpu_device *adev)
1033 {
1034 struct amdgpu_hive_info *hive = adev->hive;
1035
1036 if (!adev->gmc.xgmi.supported)
1037 return -EINVAL;
1038
1039 if (!hive)
1040 return -EINVAL;
1041
1042 mutex_lock(&hive->hive_lock);
1043 task_barrier_rem_task(&hive->tb);
1044 amdgpu_xgmi_sysfs_rem_dev_info(adev, hive);
1045 if (hive->hi_req_gpu == adev)
1046 hive->hi_req_gpu = NULL;
1047 list_del(&adev->gmc.xgmi.head);
1048 mutex_unlock(&hive->hive_lock);
1049
1050 amdgpu_put_xgmi_hive(hive);
1051 adev->hive = NULL;
1052
1053 if (atomic_dec_return(&hive->number_devices) == 0) {
1054 /* Remove the hive from global hive list */
1055 mutex_lock(&xgmi_mutex);
1056 list_del(&hive->node);
1057 mutex_unlock(&xgmi_mutex);
1058
1059 amdgpu_put_xgmi_hive(hive);
1060 }
1061
1062 return 0;
1063 }
1064
xgmi_v6_4_0_aca_bank_parser(struct aca_handle * handle,struct aca_bank * bank,enum aca_smu_type type,void * data)1065 static int xgmi_v6_4_0_aca_bank_parser(struct aca_handle *handle, struct aca_bank *bank,
1066 enum aca_smu_type type, void *data)
1067 {
1068 struct amdgpu_device *adev = handle->adev;
1069 struct aca_bank_info info;
1070 const char *error_str;
1071 u64 status, count;
1072 int ret, ext_error_code;
1073
1074 ret = aca_bank_info_decode(bank, &info);
1075 if (ret)
1076 return ret;
1077
1078 status = bank->regs[ACA_REG_IDX_STATUS];
1079 ext_error_code = ACA_REG__STATUS__ERRORCODEEXT(status);
1080
1081 error_str = ext_error_code < ARRAY_SIZE(xgmi_v6_4_0_ras_error_code_ext) ?
1082 xgmi_v6_4_0_ras_error_code_ext[ext_error_code] : NULL;
1083 if (error_str)
1084 dev_info(adev->dev, "%s detected\n", error_str);
1085
1086 count = ACA_REG__MISC0__ERRCNT(bank->regs[ACA_REG_IDX_MISC0]);
1087
1088 switch (type) {
1089 case ACA_SMU_TYPE_UE:
1090 if (ext_error_code != 0 && ext_error_code != 9)
1091 count = 0ULL;
1092
1093 ret = aca_error_cache_log_bank_error(handle, &info, ACA_ERROR_TYPE_UE, count);
1094 break;
1095 case ACA_SMU_TYPE_CE:
1096 count = ext_error_code == 6 ? count : 0ULL;
1097 ret = aca_error_cache_log_bank_error(handle, &info, ACA_ERROR_TYPE_CE, count);
1098 break;
1099 default:
1100 return -EINVAL;
1101 }
1102
1103 return ret;
1104 }
1105
1106 static const struct aca_bank_ops xgmi_v6_4_0_aca_bank_ops = {
1107 .aca_bank_parser = xgmi_v6_4_0_aca_bank_parser,
1108 };
1109
1110 static const struct aca_info xgmi_v6_4_0_aca_info = {
1111 .hwip = ACA_HWIP_TYPE_PCS_XGMI,
1112 .mask = ACA_ERROR_UE_MASK | ACA_ERROR_CE_MASK,
1113 .bank_ops = &xgmi_v6_4_0_aca_bank_ops,
1114 };
1115
amdgpu_xgmi_ras_late_init(struct amdgpu_device * adev,struct ras_common_if * ras_block)1116 static int amdgpu_xgmi_ras_late_init(struct amdgpu_device *adev, struct ras_common_if *ras_block)
1117 {
1118 int r;
1119
1120 if (!adev->gmc.xgmi.supported ||
1121 adev->gmc.xgmi.num_physical_nodes == 0)
1122 return 0;
1123
1124 amdgpu_ras_reset_error_count(adev, AMDGPU_RAS_BLOCK__XGMI_WAFL);
1125
1126 r = amdgpu_ras_block_late_init(adev, ras_block);
1127 if (r)
1128 return r;
1129
1130 switch (amdgpu_ip_version(adev, XGMI_HWIP, 0)) {
1131 case IP_VERSION(6, 4, 0):
1132 r = amdgpu_ras_bind_aca(adev, AMDGPU_RAS_BLOCK__XGMI_WAFL,
1133 &xgmi_v6_4_0_aca_info, NULL);
1134 if (r)
1135 goto late_fini;
1136 break;
1137 default:
1138 break;
1139 }
1140
1141 return 0;
1142
1143 late_fini:
1144 amdgpu_ras_block_late_fini(adev, ras_block);
1145
1146 return r;
1147 }
1148
amdgpu_xgmi_get_relative_phy_addr(struct amdgpu_device * adev,uint64_t addr)1149 uint64_t amdgpu_xgmi_get_relative_phy_addr(struct amdgpu_device *adev,
1150 uint64_t addr)
1151 {
1152 struct amdgpu_xgmi *xgmi = &adev->gmc.xgmi;
1153 return (addr + xgmi->physical_node_id * xgmi->node_segment_size);
1154 }
1155
pcs_clear_status(struct amdgpu_device * adev,uint32_t pcs_status_reg)1156 static void pcs_clear_status(struct amdgpu_device *adev, uint32_t pcs_status_reg)
1157 {
1158 WREG32_PCIE(pcs_status_reg, 0xFFFFFFFF);
1159 WREG32_PCIE(pcs_status_reg, 0);
1160 }
1161
amdgpu_xgmi_legacy_reset_ras_error_count(struct amdgpu_device * adev)1162 static void amdgpu_xgmi_legacy_reset_ras_error_count(struct amdgpu_device *adev)
1163 {
1164 uint32_t i;
1165
1166 switch (adev->asic_type) {
1167 case CHIP_ARCTURUS:
1168 for (i = 0; i < ARRAY_SIZE(xgmi_pcs_err_status_reg_arct); i++)
1169 pcs_clear_status(adev,
1170 xgmi_pcs_err_status_reg_arct[i]);
1171 break;
1172 case CHIP_VEGA20:
1173 for (i = 0; i < ARRAY_SIZE(xgmi_pcs_err_status_reg_vg20); i++)
1174 pcs_clear_status(adev,
1175 xgmi_pcs_err_status_reg_vg20[i]);
1176 break;
1177 case CHIP_ALDEBARAN:
1178 for (i = 0; i < ARRAY_SIZE(xgmi3x16_pcs_err_status_reg_aldebaran); i++)
1179 pcs_clear_status(adev,
1180 xgmi3x16_pcs_err_status_reg_aldebaran[i]);
1181 for (i = 0; i < ARRAY_SIZE(walf_pcs_err_status_reg_aldebaran); i++)
1182 pcs_clear_status(adev,
1183 walf_pcs_err_status_reg_aldebaran[i]);
1184 break;
1185 default:
1186 break;
1187 }
1188
1189 switch (amdgpu_ip_version(adev, XGMI_HWIP, 0)) {
1190 case IP_VERSION(6, 4, 0):
1191 for (i = 0; i < ARRAY_SIZE(xgmi3x16_pcs_err_status_reg_v6_4); i++)
1192 pcs_clear_status(adev,
1193 xgmi3x16_pcs_err_status_reg_v6_4[i]);
1194 break;
1195 default:
1196 break;
1197 }
1198 }
1199
__xgmi_v6_4_0_reset_error_count(struct amdgpu_device * adev,int xgmi_inst,u64 mca_base)1200 static void __xgmi_v6_4_0_reset_error_count(struct amdgpu_device *adev, int xgmi_inst, u64 mca_base)
1201 {
1202 WREG64_MCA(xgmi_inst, mca_base, ACA_REG_IDX_STATUS, 0ULL);
1203 }
1204
xgmi_v6_4_0_reset_error_count(struct amdgpu_device * adev,int xgmi_inst)1205 static void xgmi_v6_4_0_reset_error_count(struct amdgpu_device *adev, int xgmi_inst)
1206 {
1207 int i;
1208
1209 for (i = 0; i < ARRAY_SIZE(xgmi_v6_4_0_mca_base_array); i++)
1210 __xgmi_v6_4_0_reset_error_count(adev, xgmi_inst, xgmi_v6_4_0_mca_base_array[i]);
1211 }
1212
xgmi_v6_4_0_reset_ras_error_count(struct amdgpu_device * adev)1213 static void xgmi_v6_4_0_reset_ras_error_count(struct amdgpu_device *adev)
1214 {
1215 int i;
1216
1217 for_each_inst(i, adev->aid_mask)
1218 xgmi_v6_4_0_reset_error_count(adev, i);
1219 }
1220
amdgpu_xgmi_reset_ras_error_count(struct amdgpu_device * adev)1221 static void amdgpu_xgmi_reset_ras_error_count(struct amdgpu_device *adev)
1222 {
1223 switch (amdgpu_ip_version(adev, XGMI_HWIP, 0)) {
1224 case IP_VERSION(6, 4, 0):
1225 xgmi_v6_4_0_reset_ras_error_count(adev);
1226 break;
1227 default:
1228 amdgpu_xgmi_legacy_reset_ras_error_count(adev);
1229 break;
1230 }
1231 }
1232
amdgpu_xgmi_query_pcs_error_status(struct amdgpu_device * adev,uint32_t value,uint32_t mask_value,uint32_t * ue_count,uint32_t * ce_count,bool is_xgmi_pcs,bool check_mask)1233 static int amdgpu_xgmi_query_pcs_error_status(struct amdgpu_device *adev,
1234 uint32_t value,
1235 uint32_t mask_value,
1236 uint32_t *ue_count,
1237 uint32_t *ce_count,
1238 bool is_xgmi_pcs,
1239 bool check_mask)
1240 {
1241 int i;
1242 int ue_cnt = 0;
1243 const struct amdgpu_pcs_ras_field *pcs_ras_fields = NULL;
1244 uint32_t field_array_size = 0;
1245
1246 if (is_xgmi_pcs) {
1247 if (amdgpu_ip_version(adev, XGMI_HWIP, 0) ==
1248 IP_VERSION(6, 1, 0) ||
1249 amdgpu_ip_version(adev, XGMI_HWIP, 0) ==
1250 IP_VERSION(6, 4, 0)) {
1251 pcs_ras_fields = &xgmi3x16_pcs_ras_fields[0];
1252 field_array_size = ARRAY_SIZE(xgmi3x16_pcs_ras_fields);
1253 } else {
1254 pcs_ras_fields = &xgmi_pcs_ras_fields[0];
1255 field_array_size = ARRAY_SIZE(xgmi_pcs_ras_fields);
1256 }
1257 } else {
1258 pcs_ras_fields = &wafl_pcs_ras_fields[0];
1259 field_array_size = ARRAY_SIZE(wafl_pcs_ras_fields);
1260 }
1261
1262 if (check_mask)
1263 value = value & ~mask_value;
1264
1265 /* query xgmi/walf pcs error status,
1266 * only ue is supported */
1267 for (i = 0; value && i < field_array_size; i++) {
1268 ue_cnt = (value &
1269 pcs_ras_fields[i].pcs_err_mask) >>
1270 pcs_ras_fields[i].pcs_err_shift;
1271 if (ue_cnt) {
1272 dev_info(adev->dev, "%s detected\n",
1273 pcs_ras_fields[i].err_name);
1274 *ue_count += ue_cnt;
1275 }
1276
1277 /* reset bit value if the bit is checked */
1278 value &= ~(pcs_ras_fields[i].pcs_err_mask);
1279 }
1280
1281 return 0;
1282 }
1283
amdgpu_xgmi_legacy_query_ras_error_count(struct amdgpu_device * adev,void * ras_error_status)1284 static void amdgpu_xgmi_legacy_query_ras_error_count(struct amdgpu_device *adev,
1285 void *ras_error_status)
1286 {
1287 struct ras_err_data *err_data = (struct ras_err_data *)ras_error_status;
1288 int i, supported = 1;
1289 uint32_t data, mask_data = 0;
1290 uint32_t ue_cnt = 0, ce_cnt = 0;
1291
1292 if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__XGMI_WAFL))
1293 return ;
1294
1295 err_data->ue_count = 0;
1296 err_data->ce_count = 0;
1297
1298 switch (adev->asic_type) {
1299 case CHIP_ARCTURUS:
1300 /* check xgmi pcs error */
1301 for (i = 0; i < ARRAY_SIZE(xgmi_pcs_err_status_reg_arct); i++) {
1302 data = RREG32_PCIE(xgmi_pcs_err_status_reg_arct[i]);
1303 if (data)
1304 amdgpu_xgmi_query_pcs_error_status(adev, data,
1305 mask_data, &ue_cnt, &ce_cnt, true, false);
1306 }
1307 /* check wafl pcs error */
1308 for (i = 0; i < ARRAY_SIZE(wafl_pcs_err_status_reg_arct); i++) {
1309 data = RREG32_PCIE(wafl_pcs_err_status_reg_arct[i]);
1310 if (data)
1311 amdgpu_xgmi_query_pcs_error_status(adev, data,
1312 mask_data, &ue_cnt, &ce_cnt, false, false);
1313 }
1314 break;
1315 case CHIP_VEGA20:
1316 /* check xgmi pcs error */
1317 for (i = 0; i < ARRAY_SIZE(xgmi_pcs_err_status_reg_vg20); i++) {
1318 data = RREG32_PCIE(xgmi_pcs_err_status_reg_vg20[i]);
1319 if (data)
1320 amdgpu_xgmi_query_pcs_error_status(adev, data,
1321 mask_data, &ue_cnt, &ce_cnt, true, false);
1322 }
1323 /* check wafl pcs error */
1324 for (i = 0; i < ARRAY_SIZE(wafl_pcs_err_status_reg_vg20); i++) {
1325 data = RREG32_PCIE(wafl_pcs_err_status_reg_vg20[i]);
1326 if (data)
1327 amdgpu_xgmi_query_pcs_error_status(adev, data,
1328 mask_data, &ue_cnt, &ce_cnt, false, false);
1329 }
1330 break;
1331 case CHIP_ALDEBARAN:
1332 /* check xgmi3x16 pcs error */
1333 for (i = 0; i < ARRAY_SIZE(xgmi3x16_pcs_err_status_reg_aldebaran); i++) {
1334 data = RREG32_PCIE(xgmi3x16_pcs_err_status_reg_aldebaran[i]);
1335 mask_data =
1336 RREG32_PCIE(xgmi3x16_pcs_err_noncorrectable_mask_reg_aldebaran[i]);
1337 if (data)
1338 amdgpu_xgmi_query_pcs_error_status(adev, data,
1339 mask_data, &ue_cnt, &ce_cnt, true, true);
1340 }
1341 /* check wafl pcs error */
1342 for (i = 0; i < ARRAY_SIZE(walf_pcs_err_status_reg_aldebaran); i++) {
1343 data = RREG32_PCIE(walf_pcs_err_status_reg_aldebaran[i]);
1344 mask_data =
1345 RREG32_PCIE(walf_pcs_err_noncorrectable_mask_reg_aldebaran[i]);
1346 if (data)
1347 amdgpu_xgmi_query_pcs_error_status(adev, data,
1348 mask_data, &ue_cnt, &ce_cnt, false, true);
1349 }
1350 break;
1351 default:
1352 supported = 0;
1353 break;
1354 }
1355
1356 switch (amdgpu_ip_version(adev, XGMI_HWIP, 0)) {
1357 case IP_VERSION(6, 4, 0):
1358 /* check xgmi3x16 pcs error */
1359 for (i = 0; i < ARRAY_SIZE(xgmi3x16_pcs_err_status_reg_v6_4); i++) {
1360 data = RREG32_PCIE(xgmi3x16_pcs_err_status_reg_v6_4[i]);
1361 mask_data =
1362 RREG32_PCIE(xgmi3x16_pcs_err_noncorrectable_mask_reg_v6_4[i]);
1363 if (data)
1364 amdgpu_xgmi_query_pcs_error_status(adev, data,
1365 mask_data, &ue_cnt, &ce_cnt, true, true);
1366 }
1367 break;
1368 default:
1369 if (!supported)
1370 dev_warn(adev->dev, "XGMI RAS error query not supported");
1371 break;
1372 }
1373
1374 amdgpu_ras_reset_error_count(adev, AMDGPU_RAS_BLOCK__XGMI_WAFL);
1375
1376 err_data->ue_count += ue_cnt;
1377 err_data->ce_count += ce_cnt;
1378 }
1379
xgmi_v6_4_0_pcs_mca_get_error_type(struct amdgpu_device * adev,u64 status)1380 static enum aca_error_type xgmi_v6_4_0_pcs_mca_get_error_type(struct amdgpu_device *adev, u64 status)
1381 {
1382 const char *error_str;
1383 int ext_error_code;
1384
1385 ext_error_code = ACA_REG__STATUS__ERRORCODEEXT(status);
1386
1387 error_str = ext_error_code < ARRAY_SIZE(xgmi_v6_4_0_ras_error_code_ext) ?
1388 xgmi_v6_4_0_ras_error_code_ext[ext_error_code] : NULL;
1389 if (error_str)
1390 dev_info(adev->dev, "%s detected\n", error_str);
1391
1392 switch (ext_error_code) {
1393 case 0:
1394 return ACA_ERROR_TYPE_UE;
1395 case 6:
1396 return ACA_ERROR_TYPE_CE;
1397 default:
1398 return -EINVAL;
1399 }
1400
1401 return -EINVAL;
1402 }
1403
__xgmi_v6_4_0_query_error_count(struct amdgpu_device * adev,struct amdgpu_smuio_mcm_config_info * mcm_info,u64 mca_base,struct ras_err_data * err_data)1404 static void __xgmi_v6_4_0_query_error_count(struct amdgpu_device *adev, struct amdgpu_smuio_mcm_config_info *mcm_info,
1405 u64 mca_base, struct ras_err_data *err_data)
1406 {
1407 int xgmi_inst = mcm_info->die_id;
1408 u64 status = 0;
1409
1410 status = RREG64_MCA(xgmi_inst, mca_base, ACA_REG_IDX_STATUS);
1411 if (!ACA_REG__STATUS__VAL(status))
1412 return;
1413
1414 switch (xgmi_v6_4_0_pcs_mca_get_error_type(adev, status)) {
1415 case ACA_ERROR_TYPE_UE:
1416 amdgpu_ras_error_statistic_ue_count(err_data, mcm_info, 1ULL);
1417 break;
1418 case ACA_ERROR_TYPE_CE:
1419 amdgpu_ras_error_statistic_ce_count(err_data, mcm_info, 1ULL);
1420 break;
1421 default:
1422 break;
1423 }
1424
1425 WREG64_MCA(xgmi_inst, mca_base, ACA_REG_IDX_STATUS, 0ULL);
1426 }
1427
xgmi_v6_4_0_query_error_count(struct amdgpu_device * adev,int xgmi_inst,struct ras_err_data * err_data)1428 static void xgmi_v6_4_0_query_error_count(struct amdgpu_device *adev, int xgmi_inst, struct ras_err_data *err_data)
1429 {
1430 struct amdgpu_smuio_mcm_config_info mcm_info = {
1431 .socket_id = adev->smuio.funcs->get_socket_id(adev),
1432 .die_id = xgmi_inst,
1433 };
1434 int i;
1435
1436 for (i = 0; i < ARRAY_SIZE(xgmi_v6_4_0_mca_base_array); i++)
1437 __xgmi_v6_4_0_query_error_count(adev, &mcm_info, xgmi_v6_4_0_mca_base_array[i], err_data);
1438 }
1439
xgmi_v6_4_0_query_ras_error_count(struct amdgpu_device * adev,void * ras_error_status)1440 static void xgmi_v6_4_0_query_ras_error_count(struct amdgpu_device *adev, void *ras_error_status)
1441 {
1442 struct ras_err_data *err_data = (struct ras_err_data *)ras_error_status;
1443 int i;
1444
1445 for_each_inst(i, adev->aid_mask)
1446 xgmi_v6_4_0_query_error_count(adev, i, err_data);
1447 }
1448
amdgpu_xgmi_query_ras_error_count(struct amdgpu_device * adev,void * ras_error_status)1449 static void amdgpu_xgmi_query_ras_error_count(struct amdgpu_device *adev,
1450 void *ras_error_status)
1451 {
1452 switch (amdgpu_ip_version(adev, XGMI_HWIP, 0)) {
1453 case IP_VERSION(6, 4, 0):
1454 xgmi_v6_4_0_query_ras_error_count(adev, ras_error_status);
1455 break;
1456 default:
1457 amdgpu_xgmi_legacy_query_ras_error_count(adev, ras_error_status);
1458 break;
1459 }
1460 }
1461
1462 /* Trigger XGMI/WAFL error */
amdgpu_ras_error_inject_xgmi(struct amdgpu_device * adev,void * inject_if,uint32_t instance_mask)1463 static int amdgpu_ras_error_inject_xgmi(struct amdgpu_device *adev,
1464 void *inject_if, uint32_t instance_mask)
1465 {
1466 int ret1, ret2;
1467 struct ta_ras_trigger_error_input *block_info =
1468 (struct ta_ras_trigger_error_input *)inject_if;
1469
1470 if (amdgpu_dpm_set_df_cstate(adev, DF_CSTATE_DISALLOW))
1471 dev_warn(adev->dev, "Failed to disallow df cstate");
1472
1473 ret1 = amdgpu_dpm_set_pm_policy(adev, PP_PM_POLICY_XGMI_PLPD, XGMI_PLPD_DISALLOW);
1474 if (ret1 && ret1 != -EOPNOTSUPP)
1475 dev_warn(adev->dev, "Failed to disallow XGMI power down");
1476
1477 ret2 = psp_ras_trigger_error(&adev->psp, block_info, instance_mask);
1478
1479 if (amdgpu_ras_intr_triggered())
1480 return ret2;
1481
1482 ret1 = amdgpu_dpm_set_pm_policy(adev, PP_PM_POLICY_XGMI_PLPD, XGMI_PLPD_DEFAULT);
1483 if (ret1 && ret1 != -EOPNOTSUPP)
1484 dev_warn(adev->dev, "Failed to allow XGMI power down");
1485
1486 if (amdgpu_dpm_set_df_cstate(adev, DF_CSTATE_ALLOW))
1487 dev_warn(adev->dev, "Failed to allow df cstate");
1488
1489 return ret2;
1490 }
1491
1492 struct amdgpu_ras_block_hw_ops xgmi_ras_hw_ops = {
1493 .query_ras_error_count = amdgpu_xgmi_query_ras_error_count,
1494 .reset_ras_error_count = amdgpu_xgmi_reset_ras_error_count,
1495 .ras_error_inject = amdgpu_ras_error_inject_xgmi,
1496 };
1497
1498 struct amdgpu_xgmi_ras xgmi_ras = {
1499 .ras_block = {
1500 .hw_ops = &xgmi_ras_hw_ops,
1501 .ras_late_init = amdgpu_xgmi_ras_late_init,
1502 },
1503 };
1504
amdgpu_xgmi_ras_sw_init(struct amdgpu_device * adev)1505 int amdgpu_xgmi_ras_sw_init(struct amdgpu_device *adev)
1506 {
1507 int err;
1508 struct amdgpu_xgmi_ras *ras;
1509
1510 if (!adev->gmc.xgmi.ras)
1511 return 0;
1512
1513 ras = adev->gmc.xgmi.ras;
1514 err = amdgpu_ras_register_ras_block(adev, &ras->ras_block);
1515 if (err) {
1516 dev_err(adev->dev, "Failed to register xgmi_wafl_pcs ras block!\n");
1517 return err;
1518 }
1519
1520 strlcpy(ras->ras_block.ras_comm.name, "xgmi_wafl",
1521 sizeof(ras->ras_block.ras_comm.name));
1522 ras->ras_block.ras_comm.block = AMDGPU_RAS_BLOCK__XGMI_WAFL;
1523 ras->ras_block.ras_comm.type = AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE;
1524 adev->gmc.xgmi.ras_if = &ras->ras_block.ras_comm;
1525
1526 return 0;
1527 }
1528