projects/linux·blob master · refs

1520 lines · 46.2 KiB

1/*
2 * Copyright 2018 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 *
23 */
24#include <linux/list.h>
25#include "amdgpu.h"
26#include "amdgpu_xgmi.h"
27#include "amdgpu_ras.h"
28#include "soc15.h"
29#include "df/df_3_6_offset.h"
30#include "xgmi/xgmi_4_0_0_smn.h"
31#include "xgmi/xgmi_4_0_0_sh_mask.h"
32#include "xgmi/xgmi_6_1_0_sh_mask.h"
33#include "wafl/wafl2_4_0_0_smn.h"
34#include "wafl/wafl2_4_0_0_sh_mask.h"
35
36#include "amdgpu_reset.h"
37
38#define smnPCS_XGMI3X16_PCS_ERROR_STATUS 0x11a0020c
39#define smnPCS_XGMI3X16_PCS_ERROR_NONCORRECTABLE_MASK 0x11a00218
40#define smnPCS_GOPX1_PCS_ERROR_STATUS 0x12200210
41#define smnPCS_GOPX1_PCS_ERROR_NONCORRECTABLE_MASK 0x12200218
42
43#define XGMI_STATE_DISABLE 0xD1
44#define XGMI_STATE_LS0 0x81
45
46static DEFINE_MUTEX(xgmi_mutex);
47
48#define AMDGPU_MAX_XGMI_DEVICE_PER_HIVE 4
49
50static LIST_HEAD(xgmi_hive_list);
51
52static const int xgmi_pcs_err_status_reg_vg20[] = {
53 smnXGMI0_PCS_GOPX16_PCS_ERROR_STATUS,
54 smnXGMI0_PCS_GOPX16_PCS_ERROR_STATUS + 0x100000,
55};
56
57static const int wafl_pcs_err_status_reg_vg20[] = {
58 smnPCS_GOPX1_0_PCS_GOPX1_PCS_ERROR_STATUS,
59 smnPCS_GOPX1_0_PCS_GOPX1_PCS_ERROR_STATUS + 0x100000,
60};
61
62static const int xgmi_pcs_err_status_reg_arct[] = {
63 smnXGMI0_PCS_GOPX16_PCS_ERROR_STATUS,
64 smnXGMI0_PCS_GOPX16_PCS_ERROR_STATUS + 0x100000,
65 smnXGMI0_PCS_GOPX16_PCS_ERROR_STATUS + 0x500000,
66 smnXGMI0_PCS_GOPX16_PCS_ERROR_STATUS + 0x600000,
67 smnXGMI0_PCS_GOPX16_PCS_ERROR_STATUS + 0x700000,
68 smnXGMI0_PCS_GOPX16_PCS_ERROR_STATUS + 0x800000,
69};
70
71/* same as vg20*/
72static const int wafl_pcs_err_status_reg_arct[] = {
73 smnPCS_GOPX1_0_PCS_GOPX1_PCS_ERROR_STATUS,
74 smnPCS_GOPX1_0_PCS_GOPX1_PCS_ERROR_STATUS + 0x100000,
75};
76
77static const int xgmi3x16_pcs_err_status_reg_aldebaran[] = {
78 smnPCS_XGMI3X16_PCS_ERROR_STATUS,
79 smnPCS_XGMI3X16_PCS_ERROR_STATUS + 0x100000,
80 smnPCS_XGMI3X16_PCS_ERROR_STATUS + 0x200000,
81 smnPCS_XGMI3X16_PCS_ERROR_STATUS + 0x300000,
82 smnPCS_XGMI3X16_PCS_ERROR_STATUS + 0x400000,
83 smnPCS_XGMI3X16_PCS_ERROR_STATUS + 0x500000,
84 smnPCS_XGMI3X16_PCS_ERROR_STATUS + 0x600000,
85 smnPCS_XGMI3X16_PCS_ERROR_STATUS + 0x700000
86};
87
88static const int xgmi3x16_pcs_err_noncorrectable_mask_reg_aldebaran[] = {
89 smnPCS_XGMI3X16_PCS_ERROR_NONCORRECTABLE_MASK,
90 smnPCS_XGMI3X16_PCS_ERROR_NONCORRECTABLE_MASK + 0x100000,
91 smnPCS_XGMI3X16_PCS_ERROR_NONCORRECTABLE_MASK + 0x200000,
92 smnPCS_XGMI3X16_PCS_ERROR_NONCORRECTABLE_MASK + 0x300000,
93 smnPCS_XGMI3X16_PCS_ERROR_NONCORRECTABLE_MASK + 0x400000,
94 smnPCS_XGMI3X16_PCS_ERROR_NONCORRECTABLE_MASK + 0x500000,
95 smnPCS_XGMI3X16_PCS_ERROR_NONCORRECTABLE_MASK + 0x600000,
96 smnPCS_XGMI3X16_PCS_ERROR_NONCORRECTABLE_MASK + 0x700000
97};
98
99static const int walf_pcs_err_status_reg_aldebaran[] = {
100 smnPCS_GOPX1_PCS_ERROR_STATUS,
101 smnPCS_GOPX1_PCS_ERROR_STATUS + 0x100000
102};
103
104static const int walf_pcs_err_noncorrectable_mask_reg_aldebaran[] = {
105 smnPCS_GOPX1_PCS_ERROR_NONCORRECTABLE_MASK,
106 smnPCS_GOPX1_PCS_ERROR_NONCORRECTABLE_MASK + 0x100000
107};
108
109static const struct amdgpu_pcs_ras_field xgmi_pcs_ras_fields[] = {
110 {"XGMI PCS DataLossErr",
111 SOC15_REG_FIELD(XGMI0_PCS_GOPX16_PCS_ERROR_STATUS, DataLossErr)},
112 {"XGMI PCS TrainingErr",
113 SOC15_REG_FIELD(XGMI0_PCS_GOPX16_PCS_ERROR_STATUS, TrainingErr)},
114 {"XGMI PCS CRCErr",
115 SOC15_REG_FIELD(XGMI0_PCS_GOPX16_PCS_ERROR_STATUS, CRCErr)},
116 {"XGMI PCS BERExceededErr",
117 SOC15_REG_FIELD(XGMI0_PCS_GOPX16_PCS_ERROR_STATUS, BERExceededErr)},
118 {"XGMI PCS TxMetaDataErr",
119 SOC15_REG_FIELD(XGMI0_PCS_GOPX16_PCS_ERROR_STATUS, TxMetaDataErr)},
120 {"XGMI PCS ReplayBufParityErr",
121 SOC15_REG_FIELD(XGMI0_PCS_GOPX16_PCS_ERROR_STATUS, ReplayBufParityErr)},
122 {"XGMI PCS DataParityErr",
123 SOC15_REG_FIELD(XGMI0_PCS_GOPX16_PCS_ERROR_STATUS, DataParityErr)},
124 {"XGMI PCS ReplayFifoOverflowErr",
125 SOC15_REG_FIELD(XGMI0_PCS_GOPX16_PCS_ERROR_STATUS, ReplayFifoOverflowErr)},
126 {"XGMI PCS ReplayFifoUnderflowErr",
127 SOC15_REG_FIELD(XGMI0_PCS_GOPX16_PCS_ERROR_STATUS, ReplayFifoUnderflowErr)},
128 {"XGMI PCS ElasticFifoOverflowErr",
129 SOC15_REG_FIELD(XGMI0_PCS_GOPX16_PCS_ERROR_STATUS, ElasticFifoOverflowErr)},
130 {"XGMI PCS DeskewErr",
131 SOC15_REG_FIELD(XGMI0_PCS_GOPX16_PCS_ERROR_STATUS, DeskewErr)},
132 {"XGMI PCS DataStartupLimitErr",
133 SOC15_REG_FIELD(XGMI0_PCS_GOPX16_PCS_ERROR_STATUS, DataStartupLimitErr)},
134 {"XGMI PCS FCInitTimeoutErr",
135 SOC15_REG_FIELD(XGMI0_PCS_GOPX16_PCS_ERROR_STATUS, FCInitTimeoutErr)},
136 {"XGMI PCS RecoveryTimeoutErr",
137 SOC15_REG_FIELD(XGMI0_PCS_GOPX16_PCS_ERROR_STATUS, RecoveryTimeoutErr)},
138 {"XGMI PCS ReadySerialTimeoutErr",
139 SOC15_REG_FIELD(XGMI0_PCS_GOPX16_PCS_ERROR_STATUS, ReadySerialTimeoutErr)},
140 {"XGMI PCS ReadySerialAttemptErr",
141 SOC15_REG_FIELD(XGMI0_PCS_GOPX16_PCS_ERROR_STATUS, ReadySerialAttemptErr)},
142 {"XGMI PCS RecoveryAttemptErr",
143 SOC15_REG_FIELD(XGMI0_PCS_GOPX16_PCS_ERROR_STATUS, RecoveryAttemptErr)},
144 {"XGMI PCS RecoveryRelockAttemptErr",
145 SOC15_REG_FIELD(XGMI0_PCS_GOPX16_PCS_ERROR_STATUS, RecoveryRelockAttemptErr)},
146};
147
148static const struct amdgpu_pcs_ras_field wafl_pcs_ras_fields[] = {
149 {"WAFL PCS DataLossErr",
150 SOC15_REG_FIELD(PCS_GOPX1_0_PCS_GOPX1_PCS_ERROR_STATUS, DataLossErr)},
151 {"WAFL PCS TrainingErr",
152 SOC15_REG_FIELD(PCS_GOPX1_0_PCS_GOPX1_PCS_ERROR_STATUS, TrainingErr)},
153 {"WAFL PCS CRCErr",
154 SOC15_REG_FIELD(PCS_GOPX1_0_PCS_GOPX1_PCS_ERROR_STATUS, CRCErr)},
155 {"WAFL PCS BERExceededErr",
156 SOC15_REG_FIELD(PCS_GOPX1_0_PCS_GOPX1_PCS_ERROR_STATUS, BERExceededErr)},
157 {"WAFL PCS TxMetaDataErr",
158 SOC15_REG_FIELD(PCS_GOPX1_0_PCS_GOPX1_PCS_ERROR_STATUS, TxMetaDataErr)},
159 {"WAFL PCS ReplayBufParityErr",
160 SOC15_REG_FIELD(PCS_GOPX1_0_PCS_GOPX1_PCS_ERROR_STATUS, ReplayBufParityErr)},
161 {"WAFL PCS DataParityErr",
162 SOC15_REG_FIELD(PCS_GOPX1_0_PCS_GOPX1_PCS_ERROR_STATUS, DataParityErr)},
163 {"WAFL PCS ReplayFifoOverflowErr",
164 SOC15_REG_FIELD(PCS_GOPX1_0_PCS_GOPX1_PCS_ERROR_STATUS, ReplayFifoOverflowErr)},
165 {"WAFL PCS ReplayFifoUnderflowErr",
166 SOC15_REG_FIELD(PCS_GOPX1_0_PCS_GOPX1_PCS_ERROR_STATUS, ReplayFifoUnderflowErr)},
167 {"WAFL PCS ElasticFifoOverflowErr",
168 SOC15_REG_FIELD(PCS_GOPX1_0_PCS_GOPX1_PCS_ERROR_STATUS, ElasticFifoOverflowErr)},
169 {"WAFL PCS DeskewErr",
170 SOC15_REG_FIELD(PCS_GOPX1_0_PCS_GOPX1_PCS_ERROR_STATUS, DeskewErr)},
171 {"WAFL PCS DataStartupLimitErr",
172 SOC15_REG_FIELD(PCS_GOPX1_0_PCS_GOPX1_PCS_ERROR_STATUS, DataStartupLimitErr)},
173 {"WAFL PCS FCInitTimeoutErr",
174 SOC15_REG_FIELD(PCS_GOPX1_0_PCS_GOPX1_PCS_ERROR_STATUS, FCInitTimeoutErr)},
175 {"WAFL PCS RecoveryTimeoutErr",
176 SOC15_REG_FIELD(PCS_GOPX1_0_PCS_GOPX1_PCS_ERROR_STATUS, RecoveryTimeoutErr)},
177 {"WAFL PCS ReadySerialTimeoutErr",
178 SOC15_REG_FIELD(PCS_GOPX1_0_PCS_GOPX1_PCS_ERROR_STATUS, ReadySerialTimeoutErr)},
179 {"WAFL PCS ReadySerialAttemptErr",
180 SOC15_REG_FIELD(PCS_GOPX1_0_PCS_GOPX1_PCS_ERROR_STATUS, ReadySerialAttemptErr)},
181 {"WAFL PCS RecoveryAttemptErr",
182 SOC15_REG_FIELD(PCS_GOPX1_0_PCS_GOPX1_PCS_ERROR_STATUS, RecoveryAttemptErr)},
183 {"WAFL PCS RecoveryRelockAttemptErr",
184 SOC15_REG_FIELD(PCS_GOPX1_0_PCS_GOPX1_PCS_ERROR_STATUS, RecoveryRelockAttemptErr)},
185};
186
187static const struct amdgpu_pcs_ras_field xgmi3x16_pcs_ras_fields[] = {
188 {"XGMI3X16 PCS DataLossErr",
189 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, DataLossErr)},
190 {"XGMI3X16 PCS TrainingErr",
191 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, TrainingErr)},
192 {"XGMI3X16 PCS FlowCtrlAckErr",
193 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, FlowCtrlAckErr)},
194 {"XGMI3X16 PCS RxFifoUnderflowErr",
195 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, RxFifoUnderflowErr)},
196 {"XGMI3X16 PCS RxFifoOverflowErr",
197 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, RxFifoOverflowErr)},
198 {"XGMI3X16 PCS CRCErr",
199 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, CRCErr)},
200 {"XGMI3X16 PCS BERExceededErr",
201 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, BERExceededErr)},
202 {"XGMI3X16 PCS TxVcidDataErr",
203 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, TxVcidDataErr)},
204 {"XGMI3X16 PCS ReplayBufParityErr",
205 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, ReplayBufParityErr)},
206 {"XGMI3X16 PCS DataParityErr",
207 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, DataParityErr)},
208 {"XGMI3X16 PCS ReplayFifoOverflowErr",
209 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, ReplayFifoOverflowErr)},
210 {"XGMI3X16 PCS ReplayFifoUnderflowErr",
211 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, ReplayFifoUnderflowErr)},
212 {"XGMI3X16 PCS ElasticFifoOverflowErr",
213 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, ElasticFifoOverflowErr)},
214 {"XGMI3X16 PCS DeskewErr",
215 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, DeskewErr)},
216 {"XGMI3X16 PCS FlowCtrlCRCErr",
217 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, FlowCtrlCRCErr)},
218 {"XGMI3X16 PCS DataStartupLimitErr",
219 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, DataStartupLimitErr)},
220 {"XGMI3X16 PCS FCInitTimeoutErr",
221 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, FCInitTimeoutErr)},
222 {"XGMI3X16 PCS RecoveryTimeoutErr",
223 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, RecoveryTimeoutErr)},
224 {"XGMI3X16 PCS ReadySerialTimeoutErr",
225 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, ReadySerialTimeoutErr)},
226 {"XGMI3X16 PCS ReadySerialAttemptErr",
227 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, ReadySerialAttemptErr)},
228 {"XGMI3X16 PCS RecoveryAttemptErr",
229 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, RecoveryAttemptErr)},
230 {"XGMI3X16 PCS RecoveryRelockAttemptErr",
231 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, RecoveryRelockAttemptErr)},
232 {"XGMI3X16 PCS ReplayAttemptErr",
233 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, ReplayAttemptErr)},
234 {"XGMI3X16 PCS SyncHdrErr",
235 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, SyncHdrErr)},
236 {"XGMI3X16 PCS TxReplayTimeoutErr",
237 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, TxReplayTimeoutErr)},
238 {"XGMI3X16 PCS RxReplayTimeoutErr",
239 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, RxReplayTimeoutErr)},
240 {"XGMI3X16 PCS LinkSubTxTimeoutErr",
241 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, LinkSubTxTimeoutErr)},
242 {"XGMI3X16 PCS LinkSubRxTimeoutErr",
243 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, LinkSubRxTimeoutErr)},
244 {"XGMI3X16 PCS RxCMDPktErr",
245 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, RxCMDPktErr)},
246};
247
248int amdgpu_xgmi_get_ext_link(struct amdgpu_device *adev, int link_num)
249{
250 int link_map_6_4_x[8] = { 0, 3, 1, 2, 7, 6, 4, 5 };
251
252 if (adev->gmc.xgmi.num_physical_nodes <= 1)
253 return -EINVAL;
254
255 switch (amdgpu_ip_version(adev, XGMI_HWIP, 0)) {
256 case IP_VERSION(6, 4, 0):
257 case IP_VERSION(6, 4, 1):
258 if (link_num < ARRAY_SIZE(link_map_6_4_x))
259 return link_map_6_4_x[link_num];
260 break;
261 default:
262 return -EINVAL;
263 }
264
265 return -EINVAL;
266}
267
268static u32 xgmi_v6_4_get_link_status(struct amdgpu_device *adev, int global_link_num)
269{
270 const u32 smn_xgmi_6_4_pcs_state_hist1[2] = { 0x11a00070, 0x11b00070 };
271 const u32 smn_xgmi_6_4_1_pcs_state_hist1[2] = { 0x12100070,
272 0x11b00070 };
273 u32 i, n;
274 u64 addr;
275
276 switch (amdgpu_ip_version(adev, XGMI_HWIP, 0)) {
277 case IP_VERSION(6, 4, 0):
278 n = ARRAY_SIZE(smn_xgmi_6_4_pcs_state_hist1);
279 addr = smn_xgmi_6_4_pcs_state_hist1[global_link_num % n];
280 break;
281 case IP_VERSION(6, 4, 1):
282 n = ARRAY_SIZE(smn_xgmi_6_4_1_pcs_state_hist1);
283 addr = smn_xgmi_6_4_1_pcs_state_hist1[global_link_num % n];
284 break;
285 default:
286 return U32_MAX;
287 }
288
289 i = global_link_num / n;
290
291 if (!(adev->aid_mask & BIT(i)))
292 return U32_MAX;
293
294 addr += amdgpu_reg_get_smn_base64(adev, XGMI_HWIP, i);
295
296 return RREG32_PCIE_EXT(addr);
297}
298
299int amdgpu_get_xgmi_link_status(struct amdgpu_device *adev, int global_link_num)
300{
301 u32 xgmi_state_reg_val;
302
303 if (amdgpu_sriov_vf(adev))
304 return AMDGPU_XGMI_LINK_NA;
305
306 if (adev->gmc.xgmi.num_physical_nodes <= 1)
307 return -EINVAL;
308
309 switch (amdgpu_ip_version(adev, XGMI_HWIP, 0)) {
310 case IP_VERSION(6, 4, 0):
311 case IP_VERSION(6, 4, 1):
312 xgmi_state_reg_val = xgmi_v6_4_get_link_status(adev, global_link_num);
313 break;
314 default:
315 return -EOPNOTSUPP;
316 }
317
318 if ((xgmi_state_reg_val & 0xFF) == XGMI_STATE_DISABLE)
319 return -ENOLINK;
320
321 if ((xgmi_state_reg_val & 0xFF) == XGMI_STATE_LS0)
322 return AMDGPU_XGMI_LINK_ACTIVE;
323
324 return AMDGPU_XGMI_LINK_INACTIVE;
325}
326
327/**
328 * DOC: AMDGPU XGMI Support
329 *
330 * XGMI is a high speed interconnect that joins multiple GPU cards
331 * into a homogeneous memory space that is organized by a collective
332 * hive ID and individual node IDs, both of which are 64-bit numbers.
333 *
334 * The file xgmi_device_id contains the unique per GPU device ID and
335 * is stored in the /sys/class/drm/card${cardno}/device/ directory.
336 *
337 * Inside the device directory a sub-directory 'xgmi_hive_info' is
338 * created which contains the hive ID and the list of nodes.
339 *
340 * The hive ID is stored in:
341 * /sys/class/drm/card${cardno}/device/xgmi_hive_info/xgmi_hive_id
342 *
343 * The node information is stored in numbered directories:
344 * /sys/class/drm/card${cardno}/device/xgmi_hive_info/node${nodeno}/xgmi_device_id
345 *
346 * Each device has their own xgmi_hive_info direction with a mirror
347 * set of node sub-directories.
348 *
349 * The XGMI memory space is built by contiguously adding the power of
350 * two padded VRAM space from each node to each other.
351 *
352 */
353
354static struct attribute amdgpu_xgmi_hive_id = {
355 .name = "xgmi_hive_id",
356 .mode = S_IRUGO
357};
358
359static struct attribute *amdgpu_xgmi_hive_attrs[] = {
360 &amdgpu_xgmi_hive_id,
361 NULL
362};
363ATTRIBUTE_GROUPS(amdgpu_xgmi_hive);
364
365static ssize_t amdgpu_xgmi_show_attrs(struct kobject *kobj,
366 struct attribute *attr, char *buf)
367{
368 struct amdgpu_hive_info *hive = container_of(
369 kobj, struct amdgpu_hive_info, kobj);
370
371 if (attr == &amdgpu_xgmi_hive_id)
372 return snprintf(buf, PAGE_SIZE, "%llu\n", hive->hive_id);
373
374 return 0;
375}
376
377static void amdgpu_xgmi_hive_release(struct kobject *kobj)
378{
379 struct amdgpu_hive_info *hive = container_of(
380 kobj, struct amdgpu_hive_info, kobj);
381
382 amdgpu_reset_put_reset_domain(hive->reset_domain);
383 hive->reset_domain = NULL;
384
385 mutex_destroy(&hive->hive_lock);
386 kfree(hive);
387}
388
389static const struct sysfs_ops amdgpu_xgmi_hive_ops = {
390 .show = amdgpu_xgmi_show_attrs,
391};
392
393static const struct kobj_type amdgpu_xgmi_hive_type = {
394 .release = amdgpu_xgmi_hive_release,
395 .sysfs_ops = &amdgpu_xgmi_hive_ops,
396 .default_groups = amdgpu_xgmi_hive_groups,
397};
398
399static ssize_t amdgpu_xgmi_show_device_id(struct device *dev,
400 struct device_attribute *attr,
401 char *buf)
402{
403 struct drm_device *ddev = dev_get_drvdata(dev);
404 struct amdgpu_device *adev = drm_to_adev(ddev);
405
406 return sysfs_emit(buf, "%llu\n", adev->gmc.xgmi.node_id);
407
408}
409
410static ssize_t amdgpu_xgmi_show_physical_id(struct device *dev,
411 struct device_attribute *attr,
412 char *buf)
413{
414 struct drm_device *ddev = dev_get_drvdata(dev);
415 struct amdgpu_device *adev = drm_to_adev(ddev);
416
417 return sysfs_emit(buf, "%u\n", adev->gmc.xgmi.physical_node_id);
418
419}
420
421static ssize_t amdgpu_xgmi_show_num_hops(struct device *dev,
422 struct device_attribute *attr,
423 char *buf)
424{
425 struct drm_device *ddev = dev_get_drvdata(dev);
426 struct amdgpu_device *adev = drm_to_adev(ddev);
427 struct psp_xgmi_topology_info *top = &adev->psp.xgmi_context.top_info;
428 int i, offset = 0;
429
430 for (i = 0; i < top->num_nodes; i++)
431 offset += sysfs_emit_at(buf, offset, "%02x ", top->nodes[i].num_hops);
432
433 return offset + sysfs_emit_at(buf, offset, "\n");
434}
435
436static ssize_t amdgpu_xgmi_show_num_links(struct device *dev,
437 struct device_attribute *attr,
438 char *buf)
439{
440 struct drm_device *ddev = dev_get_drvdata(dev);
441 struct amdgpu_device *adev = drm_to_adev(ddev);
442 struct psp_xgmi_topology_info *top = &adev->psp.xgmi_context.top_info;
443 int i, offset = 0;
444
445 for (i = 0; i < top->num_nodes; i++)
446 offset += sysfs_emit_at(buf, offset, "%02x ", top->nodes[i].num_links);
447
448 return offset + sysfs_emit_at(buf, offset, "\n");
449}
450
451static ssize_t amdgpu_xgmi_show_connected_port_num(struct device *dev,
452 struct device_attribute *attr,
453 char *buf)
454{
455 struct drm_device *ddev = dev_get_drvdata(dev);
456 struct amdgpu_device *adev = drm_to_adev(ddev);
457 struct psp_xgmi_topology_info *top = &adev->psp.xgmi_context.top_info;
458 int i, j, size = 0;
459 int current_node;
460 /*
461 * get the node id in the sysfs for the current socket and show
462 * it in the port num info output in the sysfs for easy reading.
463 * it is NOT the one retrieved from xgmi ta.
464 */
465 for (i = 0; i < top->num_nodes; i++) {
466 if (top->nodes[i].node_id == adev->gmc.xgmi.node_id) {
467 current_node = i;
468 break;
469 }
470 }
471
472 if (i == top->num_nodes)
473 return -EINVAL;
474
475 for (i = 0; i < top->num_nodes; i++) {
476 for (j = 0; j < top->nodes[i].num_links; j++)
477 /* node id in sysfs starts from 1 rather than 0 so +1 here */
478 size += sysfs_emit_at(buf, size, "%02x:%02x -> %02x:%02x\n", current_node + 1,
479 top->nodes[i].port_num[j].src_xgmi_port_num, i + 1,
480 top->nodes[i].port_num[j].dst_xgmi_port_num);
481 }
482
483 return size;
484}
485
486#define AMDGPU_XGMI_SET_FICAA(o) ((o) | 0x456801)
487static ssize_t amdgpu_xgmi_show_error(struct device *dev,
488 struct device_attribute *attr,
489 char *buf)
490{
491 struct drm_device *ddev = dev_get_drvdata(dev);
492 struct amdgpu_device *adev = drm_to_adev(ddev);
493 uint32_t ficaa_pie_ctl_in, ficaa_pie_status_in;
494 uint64_t fica_out;
495 unsigned int error_count = 0;
496
497 ficaa_pie_ctl_in = AMDGPU_XGMI_SET_FICAA(0x200);
498 ficaa_pie_status_in = AMDGPU_XGMI_SET_FICAA(0x208);
499
500 if ((!adev->df.funcs) ||
501 (!adev->df.funcs->get_fica) ||
502 (!adev->df.funcs->set_fica))
503 return -EINVAL;
504
505 fica_out = adev->df.funcs->get_fica(adev, ficaa_pie_ctl_in);
506 if (fica_out != 0x1f)
507 pr_err("xGMI error counters not enabled!\n");
508
509 fica_out = adev->df.funcs->get_fica(adev, ficaa_pie_status_in);
510
511 if ((fica_out & 0xffff) == 2)
512 error_count = ((fica_out >> 62) & 0x1) + (fica_out >> 63);
513
514 adev->df.funcs->set_fica(adev, ficaa_pie_status_in, 0, 0);
515
516 return sysfs_emit(buf, "%u\n", error_count);
517}
518
519
520static DEVICE_ATTR(xgmi_device_id, S_IRUGO, amdgpu_xgmi_show_device_id, NULL);
521static DEVICE_ATTR(xgmi_physical_id, 0444, amdgpu_xgmi_show_physical_id, NULL);
522static DEVICE_ATTR(xgmi_error, S_IRUGO, amdgpu_xgmi_show_error, NULL);
523static DEVICE_ATTR(xgmi_num_hops, S_IRUGO, amdgpu_xgmi_show_num_hops, NULL);
524static DEVICE_ATTR(xgmi_num_links, S_IRUGO, amdgpu_xgmi_show_num_links, NULL);
525static DEVICE_ATTR(xgmi_port_num, S_IRUGO, amdgpu_xgmi_show_connected_port_num, NULL);
526
527static int amdgpu_xgmi_sysfs_add_dev_info(struct amdgpu_device *adev,
528 struct amdgpu_hive_info *hive)
529{
530 int ret = 0;
531 char node[10] = { 0 };
532
533 /* Create xgmi device id file */
534 ret = device_create_file(adev->dev, &dev_attr_xgmi_device_id);
535 if (ret) {
536 dev_err(adev->dev, "XGMI: Failed to create device file xgmi_device_id\n");
537 return ret;
538 }
539
540 ret = device_create_file(adev->dev, &dev_attr_xgmi_physical_id);
541 if (ret) {
542 dev_err(adev->dev, "XGMI: Failed to create device file xgmi_physical_id\n");
543 return ret;
544 }
545
546 /* Create xgmi error file */
547 ret = device_create_file(adev->dev, &dev_attr_xgmi_error);
548 if (ret)
549 pr_err("failed to create xgmi_error\n");
550
551 /* Create xgmi num hops file */
552 ret = device_create_file(adev->dev, &dev_attr_xgmi_num_hops);
553 if (ret)
554 pr_err("failed to create xgmi_num_hops\n");
555
556 /* Create xgmi num links file */
557 ret = device_create_file(adev->dev, &dev_attr_xgmi_num_links);
558 if (ret)
559 pr_err("failed to create xgmi_num_links\n");
560
561 /* Create xgmi port num file if supported */
562 if (adev->psp.xgmi_context.supports_ext_link_info) {
563 ret = device_create_file(adev->dev, &dev_attr_xgmi_port_num);
564 if (ret)
565 dev_err(adev->dev, "failed to create xgmi_port_num\n");
566 }
567
568 /* Create sysfs link to hive info folder on the first device */
569 if (hive->kobj.parent != (&adev->dev->kobj)) {
570 ret = sysfs_create_link(&adev->dev->kobj, &hive->kobj,
571 "xgmi_hive_info");
572 if (ret) {
573 dev_err(adev->dev, "XGMI: Failed to create link to hive info");
574 goto remove_file;
575 }
576 }
577
578 sprintf(node, "node%d", atomic_read(&hive->number_devices));
579 /* Create sysfs link form the hive folder to yourself */
580 ret = sysfs_create_link(&hive->kobj, &adev->dev->kobj, node);
581 if (ret) {
582 dev_err(adev->dev, "XGMI: Failed to create link from hive info");
583 goto remove_link;
584 }
585
586 goto success;
587
588
589remove_link:
590 sysfs_remove_link(&adev->dev->kobj, adev_to_drm(adev)->unique);
591
592remove_file:
593 device_remove_file(adev->dev, &dev_attr_xgmi_device_id);
594 device_remove_file(adev->dev, &dev_attr_xgmi_physical_id);
595 device_remove_file(adev->dev, &dev_attr_xgmi_error);
596 device_remove_file(adev->dev, &dev_attr_xgmi_num_hops);
597 device_remove_file(adev->dev, &dev_attr_xgmi_num_links);
598 if (adev->psp.xgmi_context.supports_ext_link_info)
599 device_remove_file(adev->dev, &dev_attr_xgmi_port_num);
600
601success:
602 return ret;
603}
604
605static void amdgpu_xgmi_sysfs_rem_dev_info(struct amdgpu_device *adev,
606 struct amdgpu_hive_info *hive)
607{
608 char node[10];
609 memset(node, 0, sizeof(node));
610
611 device_remove_file(adev->dev, &dev_attr_xgmi_device_id);
612 device_remove_file(adev->dev, &dev_attr_xgmi_physical_id);
613 device_remove_file(adev->dev, &dev_attr_xgmi_error);
614 device_remove_file(adev->dev, &dev_attr_xgmi_num_hops);
615 device_remove_file(adev->dev, &dev_attr_xgmi_num_links);
616 if (adev->psp.xgmi_context.supports_ext_link_info)
617 device_remove_file(adev->dev, &dev_attr_xgmi_port_num);
618
619 if (hive->kobj.parent != (&adev->dev->kobj))
620 sysfs_remove_link(&adev->dev->kobj,"xgmi_hive_info");
621
622 sprintf(node, "node%d", atomic_read(&hive->number_devices));
623 sysfs_remove_link(&hive->kobj, node);
624
625}
626
627
628
629struct amdgpu_hive_info *amdgpu_get_xgmi_hive(struct amdgpu_device *adev)
630{
631 struct amdgpu_hive_info *hive = NULL;
632 int ret;
633
634 if (!adev->gmc.xgmi.hive_id)
635 return NULL;
636
637 if (adev->hive) {
638 kobject_get(&adev->hive->kobj);
639 return adev->hive;
640 }
641
642 mutex_lock(&xgmi_mutex);
643
644 list_for_each_entry(hive, &xgmi_hive_list, node) {
645 if (hive->hive_id == adev->gmc.xgmi.hive_id)
646 goto pro_end;
647 }
648
649 hive = kzalloc_obj(*hive);
650 if (!hive) {
651 dev_err(adev->dev, "XGMI: allocation failed\n");
652 ret = -ENOMEM;
653 hive = NULL;
654 goto pro_end;
655 }
656
657 /* initialize new hive if not exist */
658 ret = kobject_init_and_add(&hive->kobj,
659 &amdgpu_xgmi_hive_type,
660 &adev->dev->kobj,
661 "%s", "xgmi_hive_info");
662 if (ret) {
663 dev_err(adev->dev, "XGMI: failed initializing kobject for xgmi hive\n");
664 kobject_put(&hive->kobj);
665 hive = NULL;
666 goto pro_end;
667 }
668
669 /**
670 * Only init hive->reset_domain for none SRIOV configuration. For SRIOV,
671 * Host driver decide how to reset the GPU either through FLR or chain reset.
672 * Guest side will get individual notifications from the host for the FLR
673 * if necessary.
674 */
675 if (!amdgpu_sriov_vf(adev)) {
676 /**
677 * Avoid recreating reset domain when hive is reconstructed for the case
678 * of reset the devices in the XGMI hive during probe for passthrough GPU
679 * See https://www.spinics.net/lists/amd-gfx/msg58836.html
680 */
681 if (adev->reset_domain->type != XGMI_HIVE) {
682 hive->reset_domain =
683 amdgpu_reset_create_reset_domain(XGMI_HIVE, "amdgpu-reset-hive");
684 if (!hive->reset_domain) {
685 dev_err(adev->dev, "XGMI: failed initializing reset domain for xgmi hive\n");
686 ret = -ENOMEM;
687 kobject_put(&hive->kobj);
688 hive = NULL;
689 goto pro_end;
690 }
691 } else {
692 amdgpu_reset_get_reset_domain(adev->reset_domain);
693 hive->reset_domain = adev->reset_domain;
694 }
695 }
696
697 hive->hive_id = adev->gmc.xgmi.hive_id;
698 INIT_LIST_HEAD(&hive->device_list);
699 INIT_LIST_HEAD(&hive->node);
700 mutex_init(&hive->hive_lock);
701 atomic_set(&hive->number_devices, 0);
702 task_barrier_init(&hive->tb);
703 hive->pstate = AMDGPU_XGMI_PSTATE_UNKNOWN;
704 hive->hi_req_gpu = NULL;
705 atomic_set(&hive->requested_nps_mode, UNKNOWN_MEMORY_PARTITION_MODE);
706
707 /*
708 * hive pstate on boot is high in vega20 so we have to go to low
709 * pstate on after boot.
710 */
711 hive->hi_req_count = AMDGPU_MAX_XGMI_DEVICE_PER_HIVE;
712 list_add_tail(&hive->node, &xgmi_hive_list);
713
714pro_end:
715 if (hive)
716 kobject_get(&hive->kobj);
717 mutex_unlock(&xgmi_mutex);
718 return hive;
719}
720
721void amdgpu_put_xgmi_hive(struct amdgpu_hive_info *hive)
722{
723 if (hive)
724 kobject_put(&hive->kobj);
725}
726
727int amdgpu_xgmi_set_pstate(struct amdgpu_device *adev, int pstate)
728{
729 int ret = 0;
730 struct amdgpu_hive_info *hive;
731 struct amdgpu_device *request_adev;
732 bool is_hi_req = pstate == AMDGPU_XGMI_PSTATE_MAX_VEGA20;
733 bool init_low;
734
735 hive = amdgpu_get_xgmi_hive(adev);
736 if (!hive)
737 return 0;
738
739 request_adev = hive->hi_req_gpu ? hive->hi_req_gpu : adev;
740 init_low = hive->pstate == AMDGPU_XGMI_PSTATE_UNKNOWN;
741 amdgpu_put_xgmi_hive(hive);
742 /* fw bug so temporarily disable pstate switching */
743 return 0;
744
745 if (!hive || adev->asic_type != CHIP_VEGA20)
746 return 0;
747
748 mutex_lock(&hive->hive_lock);
749
750 if (is_hi_req)
751 hive->hi_req_count++;
752 else
753 hive->hi_req_count--;
754
755 /*
756 * Vega20 only needs single peer to request pstate high for the hive to
757 * go high but all peers must request pstate low for the hive to go low
758 */
759 if (hive->pstate == pstate ||
760 (!is_hi_req && hive->hi_req_count && !init_low))
761 goto out;
762
763 dev_dbg(request_adev->dev, "Set xgmi pstate %d.\n", pstate);
764
765 ret = amdgpu_dpm_set_xgmi_pstate(request_adev, pstate);
766 if (ret) {
767 dev_err(request_adev->dev,
768 "XGMI: Set pstate failure on device %llx, hive %llx, ret %d",
769 request_adev->gmc.xgmi.node_id,
770 request_adev->gmc.xgmi.hive_id, ret);
771 goto out;
772 }
773
774 if (init_low)
775 hive->pstate = hive->hi_req_count ?
776 hive->pstate : AMDGPU_XGMI_PSTATE_MIN;
777 else {
778 hive->pstate = pstate;
779 hive->hi_req_gpu = pstate != AMDGPU_XGMI_PSTATE_MIN ?
780 adev : NULL;
781 }
782out:
783 mutex_unlock(&hive->hive_lock);
784 return ret;
785}
786
787int amdgpu_xgmi_update_topology(struct amdgpu_hive_info *hive, struct amdgpu_device *adev)
788{
789 int ret;
790
791 if (amdgpu_sriov_vf(adev))
792 return 0;
793
794 /* Each psp need to set the latest topology */
795 ret = psp_xgmi_set_topology_info(&adev->psp,
796 atomic_read(&hive->number_devices),
797 &adev->psp.xgmi_context.top_info);
798 if (ret)
799 dev_err(adev->dev,
800 "XGMI: Set topology failure on device %llx, hive %llx, ret %d",
801 adev->gmc.xgmi.node_id,
802 adev->gmc.xgmi.hive_id, ret);
803
804 return ret;
805}
806
807
808/*
809 * NOTE psp_xgmi_node_info.num_hops layout is as follows:
810 * num_hops[7:6] = link type (0 = xGMI2, 1 = xGMI3, 2/3 = reserved)
811 * num_hops[5:3] = reserved
812 * num_hops[2:0] = number of hops
813 */
814int amdgpu_xgmi_get_hops_count(struct amdgpu_device *adev,
815 struct amdgpu_device *peer_adev)
816{
817 struct psp_xgmi_topology_info *top = &adev->psp.xgmi_context.top_info;
818 uint8_t num_hops_mask = 0x7;
819 int i;
820
821 if (!adev->gmc.xgmi.supported)
822 return 0;
823
824 for (i = 0 ; i < top->num_nodes; ++i)
825 if (top->nodes[i].node_id == peer_adev->gmc.xgmi.node_id)
826 return top->nodes[i].num_hops & num_hops_mask;
827
828 dev_err(adev->dev, "Failed to get xgmi hops count for peer %d.\n",
829 peer_adev->gmc.xgmi.physical_node_id);
830
831 return 0;
832}
833
834int amdgpu_xgmi_get_bandwidth(struct amdgpu_device *adev, struct amdgpu_device *peer_adev,
835 enum amdgpu_xgmi_bw_mode bw_mode, enum amdgpu_xgmi_bw_unit bw_unit,
836 uint32_t *min_bw, uint32_t *max_bw)
837{
838 bool peer_mode = bw_mode == AMDGPU_XGMI_BW_MODE_PER_PEER;
839 int unit_scale = bw_unit == AMDGPU_XGMI_BW_UNIT_MBYTES ? 1000 : 1;
840 int num_lanes = adev->gmc.xgmi.max_width;
841 int speed = adev->gmc.xgmi.max_speed;
842 int num_links = !peer_mode ? 1 : -1;
843
844 if (!(min_bw && max_bw))
845 return -EINVAL;
846
847 *min_bw = 0;
848 *max_bw = 0;
849
850 if (!adev->gmc.xgmi.supported)
851 return -ENODATA;
852
853 if (peer_mode && !peer_adev)
854 return -EINVAL;
855
856 if (peer_mode) {
857 struct psp_xgmi_topology_info *top = &adev->psp.xgmi_context.top_info;
858 int i;
859
860 for (i = 0 ; i < top->num_nodes; ++i) {
861 if (top->nodes[i].node_id != peer_adev->gmc.xgmi.node_id)
862 continue;
863
864 num_links = top->nodes[i].num_links;
865 break;
866 }
867 }
868
869 if (num_links == -1) {
870 dev_err(adev->dev, "Failed to get number of xgmi links for peer %d.\n",
871 peer_adev->gmc.xgmi.physical_node_id);
872 } else if (num_links) {
873 int per_link_bw = (speed * num_lanes * unit_scale)/BITS_PER_BYTE;
874
875 *min_bw = per_link_bw;
876 *max_bw = num_links * per_link_bw;
877 }
878
879 return 0;
880}
881
882bool amdgpu_xgmi_get_is_sharing_enabled(struct amdgpu_device *adev,
883 struct amdgpu_device *peer_adev)
884{
885 struct psp_xgmi_topology_info *top = &adev->psp.xgmi_context.top_info;
886 int i;
887
888 /* Sharing should always be enabled for non-SRIOV. */
889 if (!amdgpu_sriov_vf(adev))
890 return true;
891
892 for (i = 0 ; i < top->num_nodes; ++i)
893 if (top->nodes[i].node_id == peer_adev->gmc.xgmi.node_id)
894 return !!top->nodes[i].is_sharing_enabled;
895
896 return false;
897}
898
899/*
900 * Devices that support extended data require the entire hive to initialize with
901 * the shared memory buffer flag set.
902 *
903 * Hive locks and conditions apply - see amdgpu_xgmi_add_device
904 */
905static int amdgpu_xgmi_initialize_hive_get_data_partition(struct amdgpu_hive_info *hive,
906 bool set_extended_data)
907{
908 struct amdgpu_device *tmp_adev;
909 int ret;
910
911 list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head) {
912 ret = psp_xgmi_initialize(&tmp_adev->psp, set_extended_data, false);
913 if (ret) {
914 dev_err(tmp_adev->dev,
915 "XGMI: Failed to initialize xgmi session for data partition %i\n",
916 set_extended_data);
917 return ret;
918 }
919
920 }
921
922 return 0;
923}
924
925int amdgpu_xgmi_add_device(struct amdgpu_device *adev)
926{
927 struct psp_xgmi_topology_info *top_info;
928 struct amdgpu_hive_info *hive;
929 struct amdgpu_xgmi *entry;
930 struct amdgpu_device *tmp_adev = NULL;
931
932 int count = 0, ret = 0;
933
934 if (!adev->gmc.xgmi.supported)
935 return 0;
936
937 if (amdgpu_device_ip_get_ip_block(adev, AMD_IP_BLOCK_TYPE_PSP)) {
938 ret = psp_xgmi_initialize(&adev->psp, false, true);
939 if (ret) {
940 dev_err(adev->dev,
941 "XGMI: Failed to initialize xgmi session\n");
942 return ret;
943 }
944
945 ret = psp_xgmi_get_hive_id(&adev->psp, &adev->gmc.xgmi.hive_id);
946 if (ret) {
947 dev_err(adev->dev,
948 "XGMI: Failed to get hive id\n");
949 return ret;
950 }
951
952 ret = psp_xgmi_get_node_id(&adev->psp, &adev->gmc.xgmi.node_id);
953 if (ret) {
954 dev_err(adev->dev,
955 "XGMI: Failed to get node id\n");
956 return ret;
957 }
958 } else {
959 adev->gmc.xgmi.hive_id = 16;
960 adev->gmc.xgmi.node_id = adev->gmc.xgmi.physical_node_id + 16;
961 }
962
963 hive = amdgpu_get_xgmi_hive(adev);
964 if (!hive) {
965 ret = -EINVAL;
966 dev_err(adev->dev,
967 "XGMI: node 0x%llx, can not match hive 0x%llx in the hive list.\n",
968 adev->gmc.xgmi.node_id, adev->gmc.xgmi.hive_id);
969 goto exit;
970 }
971 mutex_lock(&hive->hive_lock);
972
973 top_info = &adev->psp.xgmi_context.top_info;
974
975 list_add_tail(&adev->gmc.xgmi.head, &hive->device_list);
976 list_for_each_entry(entry, &hive->device_list, head)
977 top_info->nodes[count++].node_id = entry->node_id;
978 top_info->num_nodes = count;
979 atomic_set(&hive->number_devices, count);
980
981 task_barrier_add_task(&hive->tb);
982
983 if (amdgpu_device_ip_get_ip_block(adev, AMD_IP_BLOCK_TYPE_PSP)) {
984 list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head) {
985 /* update node list for other device in the hive */
986 if (tmp_adev != adev) {
987 top_info = &tmp_adev->psp.xgmi_context.top_info;
988 top_info->nodes[count - 1].node_id =
989 adev->gmc.xgmi.node_id;
990 top_info->num_nodes = count;
991 }
992 ret = amdgpu_xgmi_update_topology(hive, tmp_adev);
993 if (ret)
994 goto exit_unlock;
995 }
996
997 if (amdgpu_sriov_vf(adev) &&
998 adev->psp.xgmi_context.xgmi_ta_caps & EXTEND_PEER_LINK_INFO_CMD_FLAG) {
999 /* only get topology for VF being init if it can support full duplex */
1000 ret = psp_xgmi_get_topology_info(&adev->psp, count,
1001 &adev->psp.xgmi_context.top_info, false);
1002 if (ret) {
1003 dev_err(adev->dev,
1004 "XGMI: Get topology failure on device %llx, hive %llx, ret %d",
1005 adev->gmc.xgmi.node_id,
1006 adev->gmc.xgmi.hive_id, ret);
1007 /* To do: continue with some node failed or disable the whole hive*/
1008 goto exit_unlock;
1009 }
1010 } else {
1011 /* get latest topology info for each device from psp */
1012 list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head) {
1013 ret = psp_xgmi_get_topology_info(&tmp_adev->psp, count,
1014 &tmp_adev->psp.xgmi_context.top_info, false);
1015 if (ret) {
1016 dev_err(tmp_adev->dev,
1017 "XGMI: Get topology failure on device %llx, hive %llx, ret %d",
1018 tmp_adev->gmc.xgmi.node_id,
1019 tmp_adev->gmc.xgmi.hive_id, ret);
1020 /* To do : continue with some node failed or disable the whole hive */
1021 goto exit_unlock;
1022 }
1023 }
1024 }
1025
1026 /* get topology again for hives that support extended data */
1027 if (adev->psp.xgmi_context.supports_extended_data) {
1028
1029 /* initialize the hive to get extended data. */
1030 ret = amdgpu_xgmi_initialize_hive_get_data_partition(hive, true);
1031 if (ret)
1032 goto exit_unlock;
1033
1034 /* get the extended data. */
1035 list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head) {
1036 ret = psp_xgmi_get_topology_info(&tmp_adev->psp, count,
1037 &tmp_adev->psp.xgmi_context.top_info, true);
1038 if (ret) {
1039 dev_err(tmp_adev->dev,
1040 "XGMI: Get topology for extended data failure on device %llx, hive %llx, ret %d",
1041 tmp_adev->gmc.xgmi.node_id,
1042 tmp_adev->gmc.xgmi.hive_id, ret);
1043 goto exit_unlock;
1044 }
1045 }
1046
1047 /* initialize the hive to get non-extended data for the next round. */
1048 ret = amdgpu_xgmi_initialize_hive_get_data_partition(hive, false);
1049 if (ret)
1050 goto exit_unlock;
1051
1052 }
1053 }
1054
1055 if (!ret)
1056 ret = amdgpu_xgmi_sysfs_add_dev_info(adev, hive);
1057
1058exit_unlock:
1059 mutex_unlock(&hive->hive_lock);
1060exit:
1061 if (!ret) {
1062 adev->hive = hive;
1063 dev_info(adev->dev, "XGMI: Add node %d, hive 0x%llx.\n",
1064 adev->gmc.xgmi.physical_node_id, adev->gmc.xgmi.hive_id);
1065 } else {
1066 amdgpu_put_xgmi_hive(hive);
1067 dev_err(adev->dev, "XGMI: Failed to add node %d, hive 0x%llx ret: %d\n",
1068 adev->gmc.xgmi.physical_node_id, adev->gmc.xgmi.hive_id,
1069 ret);
1070 }
1071
1072 return ret;
1073}
1074
1075int amdgpu_xgmi_remove_device(struct amdgpu_device *adev)
1076{
1077 struct amdgpu_hive_info *hive = adev->hive;
1078
1079 if (!adev->gmc.xgmi.supported)
1080 return -EINVAL;
1081
1082 if (!hive)
1083 return -EINVAL;
1084
1085 mutex_lock(&hive->hive_lock);
1086 task_barrier_rem_task(&hive->tb);
1087 amdgpu_xgmi_sysfs_rem_dev_info(adev, hive);
1088 if (hive->hi_req_gpu == adev)
1089 hive->hi_req_gpu = NULL;
1090 list_del(&adev->gmc.xgmi.head);
1091 mutex_unlock(&hive->hive_lock);
1092
1093 amdgpu_put_xgmi_hive(hive);
1094 adev->hive = NULL;
1095
1096 if (atomic_dec_return(&hive->number_devices) == 0) {
1097 /* Remove the hive from global hive list */
1098 mutex_lock(&xgmi_mutex);
1099 list_del(&hive->node);
1100 mutex_unlock(&xgmi_mutex);
1101
1102 amdgpu_put_xgmi_hive(hive);
1103 }
1104
1105 return 0;
1106}
1107
1108static int amdgpu_xgmi_ras_late_init(struct amdgpu_device *adev, struct ras_common_if *ras_block)
1109{
1110 if (!adev->gmc.xgmi.supported ||
1111 adev->gmc.xgmi.num_physical_nodes == 0)
1112 return 0;
1113
1114 amdgpu_ras_reset_error_count(adev, AMDGPU_RAS_BLOCK__XGMI_WAFL);
1115
1116 return amdgpu_ras_block_late_init(adev, ras_block);
1117}
1118
1119uint64_t amdgpu_xgmi_get_relative_phy_addr(struct amdgpu_device *adev,
1120 uint64_t addr)
1121{
1122 struct amdgpu_xgmi *xgmi = &adev->gmc.xgmi;
1123 return (addr + xgmi->physical_node_id * xgmi->node_segment_size);
1124}
1125
1126static void pcs_clear_status(struct amdgpu_device *adev, uint32_t pcs_status_reg)
1127{
1128 WREG32_PCIE(pcs_status_reg, 0xFFFFFFFF);
1129 WREG32_PCIE(pcs_status_reg, 0);
1130}
1131
1132static void amdgpu_xgmi_reset_ras_error_count(struct amdgpu_device *adev)
1133{
1134 uint32_t i;
1135
1136 switch (adev->asic_type) {
1137 case CHIP_ARCTURUS:
1138 for (i = 0; i < ARRAY_SIZE(xgmi_pcs_err_status_reg_arct); i++)
1139 pcs_clear_status(adev,
1140 xgmi_pcs_err_status_reg_arct[i]);
1141 break;
1142 case CHIP_VEGA20:
1143 for (i = 0; i < ARRAY_SIZE(xgmi_pcs_err_status_reg_vg20); i++)
1144 pcs_clear_status(adev,
1145 xgmi_pcs_err_status_reg_vg20[i]);
1146 break;
1147 case CHIP_ALDEBARAN:
1148 for (i = 0; i < ARRAY_SIZE(xgmi3x16_pcs_err_status_reg_aldebaran); i++)
1149 pcs_clear_status(adev,
1150 xgmi3x16_pcs_err_status_reg_aldebaran[i]);
1151 for (i = 0; i < ARRAY_SIZE(walf_pcs_err_status_reg_aldebaran); i++)
1152 pcs_clear_status(adev,
1153 walf_pcs_err_status_reg_aldebaran[i]);
1154 break;
1155 default:
1156 break;
1157 }
1158}
1159
1160static int amdgpu_xgmi_query_pcs_error_status(struct amdgpu_device *adev,
1161 uint32_t value,
1162 uint32_t mask_value,
1163 uint32_t *ue_count,
1164 uint32_t *ce_count,
1165 bool is_xgmi_pcs,
1166 bool check_mask)
1167{
1168 int i;
1169 int ue_cnt = 0;
1170 const struct amdgpu_pcs_ras_field *pcs_ras_fields = NULL;
1171 uint32_t field_array_size = 0;
1172
1173 if (is_xgmi_pcs) {
1174 if (amdgpu_ip_version(adev, XGMI_HWIP, 0) ==
1175 IP_VERSION(6, 1, 0)) {
1176 pcs_ras_fields = &xgmi3x16_pcs_ras_fields[0];
1177 field_array_size = ARRAY_SIZE(xgmi3x16_pcs_ras_fields);
1178 } else {
1179 pcs_ras_fields = &xgmi_pcs_ras_fields[0];
1180 field_array_size = ARRAY_SIZE(xgmi_pcs_ras_fields);
1181 }
1182 } else {
1183 pcs_ras_fields = &wafl_pcs_ras_fields[0];
1184 field_array_size = ARRAY_SIZE(wafl_pcs_ras_fields);
1185 }
1186
1187 if (check_mask)
1188 value = value & ~mask_value;
1189
1190 /* query xgmi/walf pcs error status,
1191 * only ue is supported */
1192 for (i = 0; value && i < field_array_size; i++) {
1193 ue_cnt = (value &
1194 pcs_ras_fields[i].pcs_err_mask) >>
1195 pcs_ras_fields[i].pcs_err_shift;
1196 if (ue_cnt) {
1197 dev_info(adev->dev, "%s detected\n",
1198 pcs_ras_fields[i].err_name);
1199 *ue_count += ue_cnt;
1200 }
1201
1202 /* reset bit value if the bit is checked */
1203 value &= ~(pcs_ras_fields[i].pcs_err_mask);
1204 }
1205
1206 return 0;
1207}
1208
1209static void amdgpu_xgmi_query_ras_error_count(struct amdgpu_device *adev,
1210 void *ras_error_status)
1211{
1212 struct ras_err_data *err_data = (struct ras_err_data *)ras_error_status;
1213 int i;
1214 uint32_t data, mask_data = 0;
1215 uint32_t ue_cnt = 0, ce_cnt = 0;
1216
1217 if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__XGMI_WAFL))
1218 return ;
1219
1220 err_data->ue_count = 0;
1221 err_data->ce_count = 0;
1222
1223 switch (adev->asic_type) {
1224 case CHIP_ARCTURUS:
1225 /* check xgmi pcs error */
1226 for (i = 0; i < ARRAY_SIZE(xgmi_pcs_err_status_reg_arct); i++) {
1227 data = RREG32_PCIE(xgmi_pcs_err_status_reg_arct[i]);
1228 if (data)
1229 amdgpu_xgmi_query_pcs_error_status(adev, data,
1230 mask_data, &ue_cnt, &ce_cnt, true, false);
1231 }
1232 /* check wafl pcs error */
1233 for (i = 0; i < ARRAY_SIZE(wafl_pcs_err_status_reg_arct); i++) {
1234 data = RREG32_PCIE(wafl_pcs_err_status_reg_arct[i]);
1235 if (data)
1236 amdgpu_xgmi_query_pcs_error_status(adev, data,
1237 mask_data, &ue_cnt, &ce_cnt, false, false);
1238 }
1239 break;
1240 case CHIP_VEGA20:
1241 /* check xgmi pcs error */
1242 for (i = 0; i < ARRAY_SIZE(xgmi_pcs_err_status_reg_vg20); i++) {
1243 data = RREG32_PCIE(xgmi_pcs_err_status_reg_vg20[i]);
1244 if (data)
1245 amdgpu_xgmi_query_pcs_error_status(adev, data,
1246 mask_data, &ue_cnt, &ce_cnt, true, false);
1247 }
1248 /* check wafl pcs error */
1249 for (i = 0; i < ARRAY_SIZE(wafl_pcs_err_status_reg_vg20); i++) {
1250 data = RREG32_PCIE(wafl_pcs_err_status_reg_vg20[i]);
1251 if (data)
1252 amdgpu_xgmi_query_pcs_error_status(adev, data,
1253 mask_data, &ue_cnt, &ce_cnt, false, false);
1254 }
1255 break;
1256 case CHIP_ALDEBARAN:
1257 /* check xgmi3x16 pcs error */
1258 for (i = 0; i < ARRAY_SIZE(xgmi3x16_pcs_err_status_reg_aldebaran); i++) {
1259 data = RREG32_PCIE(xgmi3x16_pcs_err_status_reg_aldebaran[i]);
1260 mask_data =
1261 RREG32_PCIE(xgmi3x16_pcs_err_noncorrectable_mask_reg_aldebaran[i]);
1262 if (data)
1263 amdgpu_xgmi_query_pcs_error_status(adev, data,
1264 mask_data, &ue_cnt, &ce_cnt, true, true);
1265 }
1266 /* check wafl pcs error */
1267 for (i = 0; i < ARRAY_SIZE(walf_pcs_err_status_reg_aldebaran); i++) {
1268 data = RREG32_PCIE(walf_pcs_err_status_reg_aldebaran[i]);
1269 mask_data =
1270 RREG32_PCIE(walf_pcs_err_noncorrectable_mask_reg_aldebaran[i]);
1271 if (data)
1272 amdgpu_xgmi_query_pcs_error_status(adev, data,
1273 mask_data, &ue_cnt, &ce_cnt, false, true);
1274 }
1275 break;
1276 default:
1277 break;
1278 }
1279
1280 amdgpu_ras_reset_error_count(adev, AMDGPU_RAS_BLOCK__XGMI_WAFL);
1281
1282 err_data->ue_count += ue_cnt;
1283 err_data->ce_count += ce_cnt;
1284}
1285
1286/* Trigger XGMI/WAFL error */
1287static int amdgpu_ras_error_inject_xgmi(struct amdgpu_device *adev,
1288 void *inject_if, uint32_t instance_mask)
1289{
1290 int ret1, ret2;
1291 struct ta_ras_trigger_error_input *block_info =
1292 (struct ta_ras_trigger_error_input *)inject_if;
1293
1294 if (amdgpu_dpm_set_df_cstate(adev, DF_CSTATE_DISALLOW))
1295 dev_warn(adev->dev, "Failed to disallow df cstate");
1296
1297 ret1 = amdgpu_dpm_set_pm_policy(adev, PP_PM_POLICY_XGMI_PLPD, XGMI_PLPD_DISALLOW);
1298 if (ret1 && ret1 != -EOPNOTSUPP)
1299 dev_warn(adev->dev, "Failed to disallow XGMI power down");
1300
1301 ret2 = psp_ras_trigger_error(&adev->psp, block_info, instance_mask);
1302
1303 if (amdgpu_ras_intr_triggered())
1304 return ret2;
1305
1306 ret1 = amdgpu_dpm_set_pm_policy(adev, PP_PM_POLICY_XGMI_PLPD, XGMI_PLPD_DEFAULT);
1307 if (ret1 && ret1 != -EOPNOTSUPP)
1308 dev_warn(adev->dev, "Failed to allow XGMI power down");
1309
1310 if (amdgpu_dpm_set_df_cstate(adev, DF_CSTATE_ALLOW))
1311 dev_warn(adev->dev, "Failed to allow df cstate");
1312
1313 return ret2;
1314}
1315
1316struct amdgpu_ras_block_hw_ops xgmi_ras_hw_ops = {
1317 .query_ras_error_count = amdgpu_xgmi_query_ras_error_count,
1318 .reset_ras_error_count = amdgpu_xgmi_reset_ras_error_count,
1319 .ras_error_inject = amdgpu_ras_error_inject_xgmi,
1320};
1321
1322struct amdgpu_xgmi_ras xgmi_ras = {
1323 .ras_block = {
1324 .hw_ops = &xgmi_ras_hw_ops,
1325 .ras_late_init = amdgpu_xgmi_ras_late_init,
1326 },
1327};
1328
1329int amdgpu_xgmi_ras_sw_init(struct amdgpu_device *adev)
1330{
1331 int err;
1332 struct amdgpu_xgmi_ras *ras;
1333
1334 if (!adev->gmc.xgmi.ras)
1335 return 0;
1336
1337 ras = adev->gmc.xgmi.ras;
1338 err = amdgpu_ras_register_ras_block(adev, &ras->ras_block);
1339 if (err) {
1340 dev_err(adev->dev, "Failed to register xgmi_wafl_pcs ras block!\n");
1341 return err;
1342 }
1343
1344 strcpy(ras->ras_block.ras_comm.name, "xgmi_wafl");
1345 ras->ras_block.ras_comm.block = AMDGPU_RAS_BLOCK__XGMI_WAFL;
1346 ras->ras_block.ras_comm.type = AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE;
1347 adev->gmc.xgmi.ras_if = &ras->ras_block.ras_comm;
1348
1349 return 0;
1350}
1351
1352static void amdgpu_xgmi_reset_on_init_work(struct work_struct *work)
1353{
1354 struct amdgpu_hive_info *hive =
1355 container_of(work, struct amdgpu_hive_info, reset_on_init_work);
1356 struct amdgpu_reset_context reset_context;
1357 struct amdgpu_device *tmp_adev;
1358 struct list_head device_list;
1359 int r;
1360
1361 mutex_lock(&hive->hive_lock);
1362
1363 INIT_LIST_HEAD(&device_list);
1364 list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head)
1365 list_add_tail(&tmp_adev->reset_list, &device_list);
1366
1367 tmp_adev = list_first_entry(&device_list, struct amdgpu_device,
1368 reset_list);
1369 amdgpu_device_lock_reset_domain(tmp_adev->reset_domain);
1370
1371 reset_context.method = AMD_RESET_METHOD_ON_INIT;
1372 reset_context.reset_req_dev = tmp_adev;
1373 reset_context.hive = hive;
1374 reset_context.reset_device_list = &device_list;
1375 set_bit(AMDGPU_NEED_FULL_RESET, &reset_context.flags);
1376 set_bit(AMDGPU_SKIP_COREDUMP, &reset_context.flags);
1377
1378 amdgpu_reset_do_xgmi_reset_on_init(&reset_context);
1379 mutex_unlock(&hive->hive_lock);
1380 amdgpu_device_unlock_reset_domain(tmp_adev->reset_domain);
1381
1382 list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head) {
1383 /* Enable ttm buffers funcs after the reset lock has been dropped. */
1384 amdgpu_ttm_enable_buffer_funcs(tmp_adev);
1385
1386 r = amdgpu_ras_init_badpage_info(tmp_adev);
1387 if (r && r != -EHWPOISON)
1388 dev_err(tmp_adev->dev,
1389 "error during bad page data initialization");
1390
1391 /*
1392 * For the reset-on-init path (e.g. an NPS memory partition
1393 * switch) the RAS IP block hw_init was skipped under the
1394 * minimal init level, so uniras was never enabled. Bring it
1395 * up now that the reset domain has been unlocked. This is a
1396 * no-op for any other reset path where RAS is already
1397 * initialized, and for non-uniras devices.
1398 */
1399 r = amdgpu_ras_resume_after_reset(tmp_adev);
1400 if (r)
1401 dev_err(tmp_adev->dev,
1402 "failed to resume RAS after XGMI reset-on-init\n");
1403 }
1404}
1405
1406static void amdgpu_xgmi_schedule_reset_on_init(struct amdgpu_hive_info *hive)
1407{
1408 INIT_WORK(&hive->reset_on_init_work, amdgpu_xgmi_reset_on_init_work);
1409 amdgpu_reset_domain_schedule(hive->reset_domain,
1410 &hive->reset_on_init_work);
1411}
1412
1413int amdgpu_xgmi_reset_on_init(struct amdgpu_device *adev)
1414{
1415 struct amdgpu_hive_info *hive;
1416 bool reset_scheduled;
1417 int num_devs;
1418
1419 hive = amdgpu_get_xgmi_hive(adev);
1420 if (!hive)
1421 return -EINVAL;
1422
1423 mutex_lock(&hive->hive_lock);
1424 num_devs = atomic_read(&hive->number_devices);
1425 reset_scheduled = false;
1426 if (num_devs == adev->gmc.xgmi.num_physical_nodes) {
1427 amdgpu_xgmi_schedule_reset_on_init(hive);
1428 reset_scheduled = true;
1429 }
1430
1431 mutex_unlock(&hive->hive_lock);
1432 amdgpu_put_xgmi_hive(hive);
1433
1434 if (reset_scheduled)
1435 flush_work(&hive->reset_on_init_work);
1436
1437 return 0;
1438}
1439
1440int amdgpu_xgmi_request_nps_change(struct amdgpu_device *adev,
1441 struct amdgpu_hive_info *hive,
1442 int req_nps_mode)
1443{
1444 struct amdgpu_device *tmp_adev;
1445 int cur_nps_mode, r;
1446
1447 /* This is expected to be called only during unload of driver. The
1448 * request needs to be placed only once for all devices in the hive. If
1449 * one of them fail, revert the request for previous successful devices.
1450 * After placing the request, make hive mode as UNKNOWN so that other
1451 * devices don't request anymore.
1452 */
1453 mutex_lock(&hive->hive_lock);
1454 if (atomic_read(&hive->requested_nps_mode) ==
1455 UNKNOWN_MEMORY_PARTITION_MODE) {
1456 dev_dbg(adev->dev, "Unexpected entry for hive NPS change");
1457 mutex_unlock(&hive->hive_lock);
1458 return 0;
1459 }
1460 list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head) {
1461 r = adev->gmc.gmc_funcs->request_mem_partition_mode(
1462 tmp_adev, req_nps_mode);
1463 if (r)
1464 break;
1465 }
1466 if (r) {
1467 /* Request back current mode if one of the requests failed */
1468 cur_nps_mode =
1469 adev->gmc.gmc_funcs->query_mem_partition_mode(tmp_adev);
1470 list_for_each_entry_continue_reverse(
1471 tmp_adev, &hive->device_list, gmc.xgmi.head)
1472 adev->gmc.gmc_funcs->request_mem_partition_mode(
1473 tmp_adev, cur_nps_mode);
1474 }
1475 /* Set to UNKNOWN so that other devices don't request anymore */
1476 atomic_set(&hive->requested_nps_mode, UNKNOWN_MEMORY_PARTITION_MODE);
1477 mutex_unlock(&hive->hive_lock);
1478
1479 return r;
1480}
1481
1482bool amdgpu_xgmi_same_hive(struct amdgpu_device *adev,
1483 struct amdgpu_device *bo_adev)
1484{
1485 return (amdgpu_use_xgmi_p2p && adev != bo_adev &&
1486 adev->gmc.xgmi.hive_id &&
1487 adev->gmc.xgmi.hive_id == bo_adev->gmc.xgmi.hive_id);
1488}
1489
1490void amdgpu_xgmi_early_init(struct amdgpu_device *adev)
1491{
1492 if (!adev->gmc.xgmi.supported)
1493 return;
1494
1495 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
1496 case IP_VERSION(9, 4, 0):
1497 case IP_VERSION(9, 4, 1):
1498 case IP_VERSION(9, 4, 2):
1499 /* 25 GT/s */
1500 adev->gmc.xgmi.max_speed = 25;
1501 adev->gmc.xgmi.max_width = 16;
1502 break;
1503 case IP_VERSION(9, 4, 3):
1504 case IP_VERSION(9, 4, 4):
1505 case IP_VERSION(9, 5, 0):
1506 /* 32 GT/s */
1507 adev->gmc.xgmi.max_speed = 32;
1508 adev->gmc.xgmi.max_width = 16;
1509 break;
1510 default:
1511 break;
1512 }
1513}
1514
1515void amgpu_xgmi_set_max_speed_width(struct amdgpu_device *adev,
1516 uint16_t max_speed, uint8_t max_width)
1517{
1518 adev->gmc.xgmi.max_speed = max_speed;
1519 adev->gmc.xgmi.max_width = max_width;
1520}