projects/linux·blob master · refs

6886 lines · 185.1 KiB

1/*
2 * Copyright 2008 Advanced Micro Devices, Inc.
3 * Copyright 2008 Red Hat Inc.
4 * Copyright 2009 Jerome Glisse.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
22 * OTHER DEALINGS IN THE SOFTWARE.
23 *
24 * Authors: Dave Airlie
25 * Alex Deucher
26 * Jerome Glisse
27 */
28
29#include <linux/aperture.h>
30#include <linux/power_supply.h>
31#include <linux/kthread.h>
32#include <linux/module.h>
33#include <linux/console.h>
34#include <linux/slab.h>
35#include <linux/iommu.h>
36#include <linux/pci.h>
37#include <linux/pci-p2pdma.h>
38#include <linux/apple-gmux.h>
39#include <linux/nospec.h>
40
41#include <drm/drm_atomic_helper.h>
42#include <drm/drm_client_event.h>
43#include <drm/drm_crtc_helper.h>
44#include <drm/drm_probe_helper.h>
45#include <drm/amdgpu_drm.h>
46#include <linux/device.h>
47#include <linux/vgaarb.h>
48#include <linux/vga_switcheroo.h>
49#include <linux/efi.h>
50#include "amdgpu.h"
51#include "amdgpu_trace.h"
52#include "amdgpu_i2c.h"
53#include "atom.h"
54#include "amdgpu_atombios.h"
55#include "amdgpu_atomfirmware.h"
56#include "amd_pcie.h"
57#ifdef CONFIG_DRM_AMDGPU_SI
58#include "si.h"
59#endif
60#ifdef CONFIG_DRM_AMDGPU_CIK
61#include "cik.h"
62#endif
63#include "vi.h"
64#include "soc15.h"
65#include "nv.h"
66#include "bif/bif_4_1_d.h"
67#include <linux/firmware.h>
68#include "amdgpu_vf_error.h"
69
70#include "amdgpu_amdkfd.h"
71#include "amdgpu_pm.h"
72
73#include "amdgpu_xgmi.h"
74#include "amdgpu_ras.h"
75#include "amdgpu_wb.h"
76#include "amdgpu_ras_mgr.h"
77#include "amdgpu_pmu.h"
78#include "amdgpu_smu.h"
79#include "amdgpu_fru_eeprom.h"
80#include "amdgpu_reset.h"
81#include "amdgpu_virt.h"
82#include "amdgpu_dev_coredump.h"
83
84#include <linux/suspend.h>
85#include <drm/task_barrier.h>
86#include <linux/pm_runtime.h>
87
88#include <drm/drm_drv.h>
89
90#if IS_ENABLED(CONFIG_X86)
91#include <asm/intel-family.h>
92#include <asm/cpu_device_id.h>
93#endif
94
95MODULE_FIRMWARE("amdgpu/vega10_gpu_info.bin");
96MODULE_FIRMWARE("amdgpu/vega12_gpu_info.bin");
97MODULE_FIRMWARE("amdgpu/raven_gpu_info.bin");
98MODULE_FIRMWARE("amdgpu/picasso_gpu_info.bin");
99MODULE_FIRMWARE("amdgpu/raven2_gpu_info.bin");
100MODULE_FIRMWARE("amdgpu/arcturus_gpu_info.bin");
101MODULE_FIRMWARE("amdgpu/navi12_gpu_info.bin");
102MODULE_FIRMWARE("amdgpu/cyan_skillfish_gpu_info.bin");
103
104#define AMDGPU_RESUME_MS 2000
105#define AMDGPU_MAX_RETRY_LIMIT 2
106#define AMDGPU_RETRY_SRIOV_RESET(r) ((r) == -EBUSY || (r) == -ETIMEDOUT || (r) == -EINVAL)
107#define AMDGPU_PCIE_INDEX_FALLBACK (0x38 >> 2)
108#define AMDGPU_PCIE_INDEX_HI_FALLBACK (0x44 >> 2)
109#define AMDGPU_PCIE_DATA_FALLBACK (0x3C >> 2)
110
111#define AMDGPU_VBIOS_SKIP (1U << 0)
112#define AMDGPU_VBIOS_OPTIONAL (1U << 1)
113
114static const struct drm_driver amdgpu_kms_driver;
115
116const char *amdgpu_asic_name[] = {
117 "TAHITI",
118 "PITCAIRN",
119 "VERDE",
120 "OLAND",
121 "HAINAN",
122 "BONAIRE",
123 "KAVERI",
124 "KABINI",
125 "HAWAII",
126 "MULLINS",
127 "TOPAZ",
128 "TONGA",
129 "FIJI",
130 "CARRIZO",
131 "STONEY",
132 "POLARIS10",
133 "POLARIS11",
134 "POLARIS12",
135 "VEGAM",
136 "VEGA10",
137 "VEGA12",
138 "VEGA20",
139 "RAVEN",
140 "ARCTURUS",
141 "RENOIR",
142 "ALDEBARAN",
143 "NAVI10",
144 "CYAN_SKILLFISH",
145 "NAVI14",
146 "NAVI12",
147 "SIENNA_CICHLID",
148 "NAVY_FLOUNDER",
149 "VANGOGH",
150 "DIMGREY_CAVEFISH",
151 "BEIGE_GOBY",
152 "YELLOW_CARP",
153 "IP DISCOVERY",
154 "LAST",
155};
156
157#define AMDGPU_IP_BLK_MASK_ALL GENMASK(AMD_IP_BLOCK_TYPE_NUM - 1, 0)
158/*
159 * Default init level where all blocks are expected to be initialized. This is
160 * the level of initialization expected by default and also after a full reset
161 * of the device.
162 */
163struct amdgpu_init_level amdgpu_init_default = {
164 .level = AMDGPU_INIT_LEVEL_DEFAULT,
165 .hwini_ip_block_mask = AMDGPU_IP_BLK_MASK_ALL,
166};
167
168struct amdgpu_init_level amdgpu_init_recovery = {
169 .level = AMDGPU_INIT_LEVEL_RESET_RECOVERY,
170 .hwini_ip_block_mask = AMDGPU_IP_BLK_MASK_ALL,
171};
172
173/*
174 * Minimal blocks needed to be initialized before a XGMI hive can be reset. This
175 * is used for cases like reset on initialization where the entire hive needs to
176 * be reset before first use.
177 */
178struct amdgpu_init_level amdgpu_init_minimal_xgmi = {
179 .level = AMDGPU_INIT_LEVEL_MINIMAL_XGMI,
180 .hwini_ip_block_mask =
181 BIT(AMD_IP_BLOCK_TYPE_GMC) | BIT(AMD_IP_BLOCK_TYPE_SMC) |
182 BIT(AMD_IP_BLOCK_TYPE_COMMON) | BIT(AMD_IP_BLOCK_TYPE_IH) |
183 BIT(AMD_IP_BLOCK_TYPE_PSP)
184};
185
186static int amdgpu_device_ip_resume_phase1(struct amdgpu_device *adev);
187static int amdgpu_device_ip_resume_phase2(struct amdgpu_device *adev);
188static int amdgpu_device_ip_resume_phase3(struct amdgpu_device *adev);
189
190static void amdgpu_device_load_switch_state(struct amdgpu_device *adev);
191
192static inline bool amdgpu_ip_member_of_hwini(struct amdgpu_device *adev,
193 enum amd_ip_block_type block)
194{
195 return (adev->init_lvl->hwini_ip_block_mask & (1U << block)) != 0;
196}
197
198void amdgpu_set_init_level(struct amdgpu_device *adev,
199 enum amdgpu_init_lvl_id lvl)
200{
201 switch (lvl) {
202 case AMDGPU_INIT_LEVEL_MINIMAL_XGMI:
203 adev->init_lvl = &amdgpu_init_minimal_xgmi;
204 break;
205 case AMDGPU_INIT_LEVEL_RESET_RECOVERY:
206 adev->init_lvl = &amdgpu_init_recovery;
207 break;
208 case AMDGPU_INIT_LEVEL_DEFAULT:
209 fallthrough;
210 default:
211 adev->init_lvl = &amdgpu_init_default;
212 break;
213 }
214}
215
216static inline void amdgpu_device_stop_pending_resets(struct amdgpu_device *adev);
217static int amdgpu_device_pm_notifier(struct notifier_block *nb, unsigned long mode,
218 void *data);
219
220/**
221 * DOC: pcie_replay_count
222 *
223 * The amdgpu driver provides a sysfs API for reporting the total number
224 * of PCIe replays (NAKs).
225 * The file pcie_replay_count is used for this and returns the total
226 * number of replays as a sum of the NAKs generated and NAKs received.
227 */
228
229static ssize_t amdgpu_device_get_pcie_replay_count(struct device *dev,
230 struct device_attribute *attr, char *buf)
231{
232 struct drm_device *ddev = dev_get_drvdata(dev);
233 struct amdgpu_device *adev = drm_to_adev(ddev);
234 uint64_t cnt = amdgpu_asic_get_pcie_replay_count(adev);
235
236 return sysfs_emit(buf, "%llu\n", cnt);
237}
238
239static DEVICE_ATTR(pcie_replay_count, 0444,
240 amdgpu_device_get_pcie_replay_count, NULL);
241
242static int amdgpu_device_attr_sysfs_init(struct amdgpu_device *adev)
243{
244 int ret = 0;
245
246 if (amdgpu_nbio_is_replay_cnt_supported(adev))
247 ret = sysfs_create_file(&adev->dev->kobj,
248 &dev_attr_pcie_replay_count.attr);
249
250 return ret;
251}
252
253static void amdgpu_device_attr_sysfs_fini(struct amdgpu_device *adev)
254{
255 if (amdgpu_nbio_is_replay_cnt_supported(adev))
256 sysfs_remove_file(&adev->dev->kobj,
257 &dev_attr_pcie_replay_count.attr);
258}
259
260static ssize_t amdgpu_sysfs_reg_state_get(struct file *f, struct kobject *kobj,
261 const struct bin_attribute *attr, char *buf,
262 loff_t ppos, size_t count)
263{
264 struct device *dev = kobj_to_dev(kobj);
265 struct drm_device *ddev = dev_get_drvdata(dev);
266 struct amdgpu_device *adev = drm_to_adev(ddev);
267 ssize_t bytes_read;
268
269 switch (ppos) {
270 case AMDGPU_SYS_REG_STATE_XGMI:
271 bytes_read = amdgpu_asic_get_reg_state(
272 adev, AMDGPU_REG_STATE_TYPE_XGMI, buf, count);
273 break;
274 case AMDGPU_SYS_REG_STATE_WAFL:
275 bytes_read = amdgpu_asic_get_reg_state(
276 adev, AMDGPU_REG_STATE_TYPE_WAFL, buf, count);
277 break;
278 case AMDGPU_SYS_REG_STATE_PCIE:
279 bytes_read = amdgpu_asic_get_reg_state(
280 adev, AMDGPU_REG_STATE_TYPE_PCIE, buf, count);
281 break;
282 case AMDGPU_SYS_REG_STATE_USR:
283 bytes_read = amdgpu_asic_get_reg_state(
284 adev, AMDGPU_REG_STATE_TYPE_USR, buf, count);
285 break;
286 case AMDGPU_SYS_REG_STATE_USR_1:
287 bytes_read = amdgpu_asic_get_reg_state(
288 adev, AMDGPU_REG_STATE_TYPE_USR_1, buf, count);
289 break;
290 default:
291 return -EINVAL;
292 }
293
294 return bytes_read;
295}
296
297static const BIN_ATTR(reg_state, 0444, amdgpu_sysfs_reg_state_get, NULL,
298 AMDGPU_SYS_REG_STATE_END);
299
300int amdgpu_reg_state_sysfs_init(struct amdgpu_device *adev)
301{
302 int ret;
303
304 if (!amdgpu_asic_get_reg_state_supported(adev))
305 return 0;
306
307 ret = sysfs_create_bin_file(&adev->dev->kobj, &bin_attr_reg_state);
308
309 return ret;
310}
311
312void amdgpu_reg_state_sysfs_fini(struct amdgpu_device *adev)
313{
314 if (!amdgpu_asic_get_reg_state_supported(adev))
315 return;
316 sysfs_remove_bin_file(&adev->dev->kobj, &bin_attr_reg_state);
317}
318
319/**
320 * DOC: board_info
321 *
322 * The amdgpu driver provides a sysfs API for giving board related information.
323 * It provides the form factor information in the format
324 *
325 * type : form factor
326 *
327 * Possible form factor values
328 *
329 * - "cem" - PCIE CEM card
330 * - "oam" - Open Compute Accelerator Module
331 * - "unknown" - Not known
332 *
333 */
334
335static ssize_t amdgpu_device_get_board_info(struct device *dev,
336 struct device_attribute *attr,
337 char *buf)
338{
339 struct drm_device *ddev = dev_get_drvdata(dev);
340 struct amdgpu_device *adev = drm_to_adev(ddev);
341 enum amdgpu_pkg_type pkg_type = AMDGPU_PKG_TYPE_CEM;
342 const char *pkg;
343
344 if (adev->smuio.funcs && adev->smuio.funcs->get_pkg_type)
345 pkg_type = adev->smuio.funcs->get_pkg_type(adev);
346
347 switch (pkg_type) {
348 case AMDGPU_PKG_TYPE_CEM:
349 pkg = "cem";
350 break;
351 case AMDGPU_PKG_TYPE_OAM:
352 pkg = "oam";
353 break;
354 default:
355 pkg = "unknown";
356 break;
357 }
358
359 return sysfs_emit(buf, "%s : %s\n", "type", pkg);
360}
361
362static DEVICE_ATTR(board_info, 0444, amdgpu_device_get_board_info, NULL);
363
364static struct attribute *amdgpu_board_attrs[] = {
365 &dev_attr_board_info.attr,
366 NULL,
367};
368
369static umode_t amdgpu_board_attrs_is_visible(struct kobject *kobj,
370 struct attribute *attr, int n)
371{
372 struct device *dev = kobj_to_dev(kobj);
373 struct drm_device *ddev = dev_get_drvdata(dev);
374 struct amdgpu_device *adev = drm_to_adev(ddev);
375
376 if (adev->flags & AMD_IS_APU)
377 return 0;
378
379 return attr->mode;
380}
381
382static const struct attribute_group amdgpu_board_attrs_group = {
383 .attrs = amdgpu_board_attrs,
384 .is_visible = amdgpu_board_attrs_is_visible
385};
386
387/**
388 * DOC: uma/carveout_options
389 *
390 * This is a read-only file that lists all available UMA allocation
391 * options and their corresponding indices. Example output::
392 *
393 * $ cat uma/carveout_options
394 * 0: Minimum (512 MB)
395 * 1: (1 GB)
396 * 2: (2 GB)
397 * 3: (4 GB)
398 * 4: (6 GB)
399 * 5: (8 GB)
400 * 6: (12 GB)
401 * 7: Medium (16 GB)
402 * 8: (24 GB)
403 * 9: High (32 GB)
404 */
405static ssize_t carveout_options_show(struct device *dev,
406 struct device_attribute *attr,
407 char *buf)
408{
409 struct drm_device *ddev = dev_get_drvdata(dev);
410 struct amdgpu_device *adev = drm_to_adev(ddev);
411 struct amdgpu_uma_carveout_info *uma_info = &adev->uma_info;
412 uint32_t memory_carved;
413 ssize_t size = 0;
414
415 if (!uma_info || !uma_info->num_entries)
416 return -ENODEV;
417
418 for (int i = 0; i < uma_info->num_entries; i++) {
419 memory_carved = uma_info->entries[i].memory_carved_mb;
420 if (memory_carved >= SZ_1G/SZ_1M) {
421 size += sysfs_emit_at(buf, size, "%d: %s (%u GB)\n",
422 i,
423 uma_info->entries[i].name,
424 memory_carved >> 10);
425 } else {
426 size += sysfs_emit_at(buf, size, "%d: %s (%u MB)\n",
427 i,
428 uma_info->entries[i].name,
429 memory_carved);
430 }
431 }
432
433 return size;
434}
435static DEVICE_ATTR_RO(carveout_options);
436
437/**
438 * DOC: uma/carveout
439 *
440 * This file is both readable and writable. When read, it shows the
441 * index of the current setting. Writing a valid index to this file
442 * allows users to change the UMA carveout size to the selected option
443 * on the next boot.
444 *
445 * The available options and their corresponding indices can be read
446 * from the uma/carveout_options file.
447 */
448static ssize_t carveout_show(struct device *dev,
449 struct device_attribute *attr,
450 char *buf)
451{
452 struct drm_device *ddev = dev_get_drvdata(dev);
453 struct amdgpu_device *adev = drm_to_adev(ddev);
454
455 return sysfs_emit(buf, "%u\n", adev->uma_info.uma_option_index);
456}
457
458static ssize_t carveout_store(struct device *dev,
459 struct device_attribute *attr,
460 const char *buf, size_t count)
461{
462 struct drm_device *ddev = dev_get_drvdata(dev);
463 struct amdgpu_device *adev = drm_to_adev(ddev);
464 struct amdgpu_uma_carveout_info *uma_info = &adev->uma_info;
465 struct amdgpu_uma_carveout_option *opt;
466 unsigned long val;
467 uint8_t flags;
468 int r;
469
470 r = kstrtoul(buf, 10, &val);
471 if (r)
472 return r;
473
474 if (val >= uma_info->num_entries)
475 return -EINVAL;
476
477 val = array_index_nospec(val, uma_info->num_entries);
478 opt = &uma_info->entries[val];
479
480 if (!(opt->flags & AMDGPU_UMA_FLAG_AUTO) &&
481 !(opt->flags & AMDGPU_UMA_FLAG_CUSTOM)) {
482 drm_err_once(ddev, "Option %lu not supported due to lack of Custom/Auto flag", val);
483 return -EINVAL;
484 }
485
486 flags = opt->flags;
487 flags &= ~((flags & AMDGPU_UMA_FLAG_AUTO) >> 1);
488
489 guard(mutex)(&uma_info->update_lock);
490
491 r = amdgpu_acpi_set_uma_allocation_size(adev, val, flags);
492 if (r)
493 return r;
494
495 uma_info->uma_option_index = val;
496
497 return count;
498}
499static DEVICE_ATTR_RW(carveout);
500
501static struct attribute *amdgpu_uma_attrs[] = {
502 &dev_attr_carveout.attr,
503 &dev_attr_carveout_options.attr,
504 NULL
505};
506
507const struct attribute_group amdgpu_uma_attr_group = {
508 .name = "uma",
509 .attrs = amdgpu_uma_attrs
510};
511
512static void amdgpu_uma_sysfs_init(struct amdgpu_device *adev)
513{
514 int rc;
515
516 if (!(adev->flags & AMD_IS_APU))
517 return;
518
519 if (!amdgpu_acpi_is_set_uma_allocation_size_supported())
520 return;
521
522 rc = amdgpu_atomfirmware_get_uma_carveout_info(adev, &adev->uma_info);
523 if (rc) {
524 drm_dbg(adev_to_drm(adev),
525 "Failed to parse UMA carveout info from VBIOS: %d\n", rc);
526 goto out_info;
527 }
528
529 mutex_init(&adev->uma_info.update_lock);
530
531 rc = devm_device_add_group(adev->dev, &amdgpu_uma_attr_group);
532 if (rc) {
533 drm_dbg(adev_to_drm(adev), "Failed to add UMA carveout sysfs interfaces %d\n", rc);
534 goto out_attr;
535 }
536
537 return;
538
539out_attr:
540 mutex_destroy(&adev->uma_info.update_lock);
541out_info:
542 return;
543}
544
545static void amdgpu_uma_sysfs_fini(struct amdgpu_device *adev)
546{
547 struct amdgpu_uma_carveout_info *uma_info = &adev->uma_info;
548
549 if (!amdgpu_acpi_is_set_uma_allocation_size_supported())
550 return;
551
552 mutex_destroy(&uma_info->update_lock);
553 uma_info->num_entries = 0;
554}
555
556static void amdgpu_device_get_pcie_info(struct amdgpu_device *adev);
557
558/**
559 * amdgpu_device_supports_px - Is the device a dGPU with ATPX power control
560 *
561 * @adev: amdgpu device pointer
562 *
563 * Returns true if the device is a dGPU with ATPX power control,
564 * otherwise return false.
565 */
566bool amdgpu_device_supports_px(struct amdgpu_device *adev)
567{
568 if ((adev->flags & AMD_IS_PX) && !amdgpu_is_atpx_hybrid())
569 return true;
570 return false;
571}
572
573/**
574 * amdgpu_device_supports_boco - Is the device a dGPU with ACPI power resources
575 *
576 * @adev: amdgpu device pointer
577 *
578 * Returns true if the device is a dGPU with ACPI power control,
579 * otherwise return false.
580 */
581bool amdgpu_device_supports_boco(struct amdgpu_device *adev)
582{
583 if (!IS_ENABLED(CONFIG_HOTPLUG_PCI_PCIE))
584 return false;
585
586 if (adev->has_pr3 ||
587 ((adev->flags & AMD_IS_PX) && amdgpu_is_atpx_hybrid()))
588 return true;
589 return false;
590}
591
592/**
593 * amdgpu_device_supports_baco - Does the device support BACO
594 *
595 * @adev: amdgpu device pointer
596 *
597 * Return:
598 * 1 if the device supports BACO;
599 * 3 if the device supports MACO (only works if BACO is supported)
600 * otherwise return 0.
601 */
602int amdgpu_device_supports_baco(struct amdgpu_device *adev)
603{
604 return amdgpu_asic_supports_baco(adev);
605}
606
607void amdgpu_device_detect_runtime_pm_mode(struct amdgpu_device *adev)
608{
609 int bamaco_support;
610
611 adev->pm.rpm_mode = AMDGPU_RUNPM_NONE;
612 if (pci_is_thunderbolt_attached(adev->pdev) ||
613 dev_is_removable(&adev->pdev->dev)) {
614 dev_info(adev->dev,
615 "Runtime PM disabled for externally attached device\n");
616 return;
617 }
618
619 bamaco_support = amdgpu_device_supports_baco(adev);
620
621 switch (amdgpu_runtime_pm) {
622 case 2:
623 if (bamaco_support & MACO_SUPPORT) {
624 adev->pm.rpm_mode = AMDGPU_RUNPM_BAMACO;
625 dev_info(adev->dev, "Forcing BAMACO for runtime pm\n");
626 } else if (bamaco_support == BACO_SUPPORT) {
627 adev->pm.rpm_mode = AMDGPU_RUNPM_BACO;
628 dev_info(adev->dev, "Requested mode BAMACO not available,fallback to use BACO\n");
629 }
630 break;
631 case 1:
632 if (bamaco_support & BACO_SUPPORT) {
633 adev->pm.rpm_mode = AMDGPU_RUNPM_BACO;
634 dev_info(adev->dev, "Forcing BACO for runtime pm\n");
635 }
636 break;
637 case -1:
638 case -2:
639 if (amdgpu_device_supports_px(adev)) {
640 /* enable PX as runtime mode */
641 adev->pm.rpm_mode = AMDGPU_RUNPM_PX;
642 dev_info(adev->dev, "Using ATPX for runtime pm\n");
643 } else if (amdgpu_device_supports_boco(adev)) {
644 /* enable boco as runtime mode */
645 adev->pm.rpm_mode = AMDGPU_RUNPM_BOCO;
646 dev_info(adev->dev, "Using BOCO for runtime pm\n");
647 } else {
648 if (!bamaco_support)
649 goto no_runtime_pm;
650
651 switch (adev->asic_type) {
652 case CHIP_VEGA20:
653 case CHIP_ARCTURUS:
654 /* BACO are not supported on vega20 and arctrus */
655 break;
656 case CHIP_VEGA10:
657 /* enable BACO as runpm mode if noretry=0 */
658 if (!adev->gmc.noretry && !amdgpu_passthrough(adev))
659 adev->pm.rpm_mode = AMDGPU_RUNPM_BACO;
660 break;
661 default:
662 /* enable BACO as runpm mode on CI+ */
663 if (!amdgpu_passthrough(adev))
664 adev->pm.rpm_mode = AMDGPU_RUNPM_BACO;
665 break;
666 }
667
668 if (adev->pm.rpm_mode == AMDGPU_RUNPM_BACO) {
669 if (bamaco_support & MACO_SUPPORT) {
670 adev->pm.rpm_mode = AMDGPU_RUNPM_BAMACO;
671 dev_info(adev->dev, "Using BAMACO for runtime pm\n");
672 } else {
673 dev_info(adev->dev, "Using BACO for runtime pm\n");
674 }
675 }
676 }
677 break;
678 case 0:
679 dev_info(adev->dev, "runtime pm is manually disabled\n");
680 break;
681 default:
682 break;
683 }
684
685no_runtime_pm:
686 if (adev->pm.rpm_mode == AMDGPU_RUNPM_NONE)
687 dev_info(adev->dev, "Runtime PM not available\n");
688}
689/**
690 * amdgpu_device_supports_smart_shift - Is the device dGPU with
691 * smart shift support
692 *
693 * @adev: amdgpu device pointer
694 *
695 * Returns true if the device is a dGPU with Smart Shift support,
696 * otherwise returns false.
697 */
698bool amdgpu_device_supports_smart_shift(struct amdgpu_device *adev)
699{
700 return (amdgpu_device_supports_boco(adev) &&
701 amdgpu_acpi_is_power_shift_control_supported());
702}
703
704/*
705 * VRAM access helper functions
706 */
707
708/**
709 * amdgpu_device_mm_access - access vram by MM_INDEX/MM_DATA
710 *
711 * @adev: amdgpu_device pointer
712 * @pos: offset of the buffer in vram
713 * @buf: virtual address of the buffer in system memory
714 * @size: read/write size, sizeof(@buf) must > @size
715 * @write: true - write to vram, otherwise - read from vram
716 */
717void amdgpu_device_mm_access(struct amdgpu_device *adev, loff_t pos,
718 void *buf, size_t size, bool write)
719{
720 unsigned long flags;
721 uint32_t hi = ~0, tmp = 0;
722 uint32_t *data = buf;
723 uint64_t last;
724 int idx;
725
726 if (!drm_dev_enter(adev_to_drm(adev), &idx))
727 return;
728
729 if (!IS_ALIGNED(pos, 4) || !IS_ALIGNED(size, 4)) {
730 dev_err(adev->dev, "unaligned pos/size (pos=0x%llx, size=0x%zx)\n",
731 pos, size);
732 drm_dev_exit(idx);
733 return;
734 }
735
736 spin_lock_irqsave(&adev->mmio_idx_lock, flags);
737 for (last = pos + size; pos < last; pos += 4) {
738 tmp = pos >> 31;
739
740 WREG32_NO_KIQ(mmMM_INDEX, ((uint32_t)pos) | 0x80000000);
741 if (tmp != hi) {
742 WREG32_NO_KIQ(mmMM_INDEX_HI, tmp);
743 hi = tmp;
744 }
745 if (write)
746 WREG32_NO_KIQ(mmMM_DATA, *data++);
747 else
748 *data++ = RREG32_NO_KIQ(mmMM_DATA);
749 }
750
751 spin_unlock_irqrestore(&adev->mmio_idx_lock, flags);
752 drm_dev_exit(idx);
753}
754
755#ifdef CONFIG_64BIT
756/*
757 * During early SR-IOV VF init, host-provided init data can live in FB before
758 * the normal VRAM aperture mapping is ready. Use a temporary BAR0 mapping for
759 * reads only, and verify it matches the VRAM aperture when aperture information
760 * is already available.
761 */
762static int amdgpu_device_read_fb_via_bar0(struct amdgpu_device *adev,
763 u64 offset, void *buf, size_t size)
764{
765 resource_size_t aper_base, aper_size, bar_start, bar_size, map_base;
766 void __iomem *vram;
767 size_t map_offset, map_size;
768 unsigned long flags;
769 u64 end;
770
771 if (!buf || !size)
772 return -EINVAL;
773
774 if (!amdgpu_sriov_vf(adev))
775 return -EINVAL;
776
777 flags = pci_resource_flags(adev->pdev, 0);
778 if ((flags & IORESOURCE_UNSET) || !(flags & IORESOURCE_MEM))
779 return -EINVAL;
780
781 bar_start = pci_resource_start(adev->pdev, 0);
782 bar_size = pci_resource_len(adev->pdev, 0);
783 if (!bar_size)
784 return -ENODEV;
785
786 aper_base = adev->gmc.aper_base;
787 aper_size = adev->gmc.visible_vram_size ? adev->gmc.visible_vram_size :
788 adev->gmc.aper_size;
789
790 if (aper_base || aper_size) {
791 if (aper_base != bar_start || aper_size > bar_size)
792 return -EINVAL;
793 } else {
794 aper_base = bar_start;
795 aper_size = bar_size;
796 }
797
798 if (check_add_overflow(offset, size, &end) || end > aper_size)
799 return -EINVAL;
800
801 map_offset = offset_in_page(offset);
802 map_base = aper_base + (offset & PAGE_MASK);
803 map_size = PAGE_ALIGN(map_offset + size);
804
805 vram = ioremap_wc(map_base, map_size);
806 if (!vram)
807 return -ENOMEM;
808
809 memcpy_fromio(buf, (u8 __iomem *)vram + map_offset, size);
810 iounmap(vram);
811
812 return 0;
813}
814#endif
815
816/**
817 * amdgpu_device_aper_access - access vram by vram aperture
818 *
819 * @adev: amdgpu_device pointer
820 * @pos: offset of the buffer in vram
821 * @buf: virtual address of the buffer in system memory
822 * @size: read/write size, sizeof(@buf) must > @size
823 * @write: true - write to vram, otherwise - read from vram
824 *
825 * The return value means how many bytes have been transferred.
826 */
827size_t amdgpu_device_aper_access(struct amdgpu_device *adev, loff_t pos,
828 void *buf, size_t size, bool write)
829{
830#ifdef CONFIG_64BIT
831 void __iomem *addr;
832 size_t count = 0;
833 uint64_t last;
834
835 if (!adev->mman.aper_base_kaddr) {
836 /* Writes still require the regular aperture/MM path. */
837 if (!write && !amdgpu_device_read_fb_via_bar0(adev, pos, buf, size))
838 return size;
839 return 0;
840 }
841
842 last = min(pos + size, adev->gmc.visible_vram_size);
843 if (last > pos) {
844 addr = adev->mman.aper_base_kaddr + pos;
845 count = last - pos;
846
847 if (write) {
848 memcpy_toio(addr, buf, count);
849 /* Make sure HDP write cache flush happens without any reordering
850 * after the system memory contents are sent over PCIe device
851 */
852 mb();
853 amdgpu_device_flush_hdp(adev, NULL);
854 } else {
855 amdgpu_device_invalidate_hdp(adev, NULL);
856 /* Make sure HDP read cache is invalidated before issuing a read
857 * to the PCIe device
858 */
859 mb();
860 memcpy_fromio(buf, addr, count);
861 }
862
863 }
864
865 return count;
866#else
867 return 0;
868#endif
869}
870
871/**
872 * amdgpu_device_vram_access - read/write a buffer in vram
873 *
874 * @adev: amdgpu_device pointer
875 * @pos: offset of the buffer in vram
876 * @buf: virtual address of the buffer in system memory
877 * @size: read/write size, sizeof(@buf) must > @size
878 * @write: true - write to vram, otherwise - read from vram
879 */
880void amdgpu_device_vram_access(struct amdgpu_device *adev, loff_t pos,
881 void *buf, size_t size, bool write)
882{
883 size_t count;
884
885 /* try to using vram apreature to access vram first */
886 count = amdgpu_device_aper_access(adev, pos, buf, size, write);
887 size -= count;
888 if (size) {
889 /* using MM to access rest vram */
890 pos += count;
891 buf += count;
892 amdgpu_device_mm_access(adev, pos, buf, size, write);
893 }
894}
895
896/*
897 * register access helper functions.
898 */
899
900/* Check if hw access should be skipped because of hotplug or device error */
901bool amdgpu_device_skip_hw_access(struct amdgpu_device *adev)
902{
903 if (adev->no_hw_access)
904 return true;
905
906#ifdef CONFIG_LOCKDEP
907 /*
908 * This is a bit complicated to understand, so worth a comment. What we assert
909 * here is that the GPU reset is not running on another thread in parallel.
910 *
911 * For this we trylock the read side of the reset semaphore, if that succeeds
912 * we know that the reset is not running in parallel.
913 *
914 * If the trylock fails we assert that we are either already holding the read
915 * side of the lock or are the reset thread itself and hold the write side of
916 * the lock.
917 */
918 if (in_task()) {
919 if (down_read_trylock(&adev->reset_domain->sem))
920 up_read(&adev->reset_domain->sem);
921 else
922 lockdep_assert_held(&adev->reset_domain->sem);
923 }
924#endif
925 return false;
926}
927
928/**
929 * amdgpu_device_get_rev_id - query device rev_id
930 *
931 * @adev: amdgpu_device pointer
932 *
933 * Return device rev_id
934 */
935u32 amdgpu_device_get_rev_id(struct amdgpu_device *adev)
936{
937 return adev->nbio.funcs->get_rev_id(adev);
938}
939
940static uint32_t amdgpu_device_get_vbios_flags(struct amdgpu_device *adev)
941{
942 if (hweight32(adev->aid_mask) && (adev->flags & AMD_IS_APU))
943 return AMDGPU_VBIOS_SKIP;
944
945 if (hweight32(adev->aid_mask) && amdgpu_passthrough(adev))
946 return AMDGPU_VBIOS_OPTIONAL;
947
948 return 0;
949}
950
951/**
952 * amdgpu_device_asic_init - Wrapper for atom asic_init
953 *
954 * @adev: amdgpu_device pointer
955 *
956 * Does any asic specific work and then calls atom asic init.
957 */
958static int amdgpu_device_asic_init(struct amdgpu_device *adev)
959{
960 uint32_t flags;
961 bool optional;
962 int ret;
963
964 amdgpu_asic_pre_asic_init(adev);
965 flags = amdgpu_device_get_vbios_flags(adev);
966 optional = !!(flags & (AMDGPU_VBIOS_OPTIONAL | AMDGPU_VBIOS_SKIP));
967
968 if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 3) ||
969 amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 4) ||
970 amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 5, 0) ||
971 amdgpu_ip_version(adev, GC_HWIP, 0) >= IP_VERSION(11, 0, 0)) {
972 amdgpu_psp_wait_for_bootloader(adev);
973 if (optional && !adev->bios)
974 return 0;
975
976 ret = amdgpu_atomfirmware_asic_init(adev, true);
977 return ret;
978 } else {
979 if (optional && !adev->bios)
980 return 0;
981
982 return amdgpu_atom_asic_init(adev->mode_info.atom_context);
983 }
984
985 return 0;
986}
987
988/**
989 * amdgpu_device_mem_scratch_init - allocate the VRAM scratch page
990 *
991 * @adev: amdgpu_device pointer
992 *
993 * Allocates a scratch page of VRAM for use by various things in the
994 * driver.
995 */
996static int amdgpu_device_mem_scratch_init(struct amdgpu_device *adev)
997{
998 return amdgpu_bo_create_kernel(adev, AMDGPU_GPU_PAGE_SIZE, PAGE_SIZE,
999 AMDGPU_GEM_DOMAIN_VRAM |
1000 AMDGPU_GEM_DOMAIN_GTT,
1001 &adev->mem_scratch.robj,
1002 &adev->mem_scratch.gpu_addr,
1003 (void **)&adev->mem_scratch.ptr);
1004}
1005
1006/**
1007 * amdgpu_device_mem_scratch_fini - Free the VRAM scratch page
1008 *
1009 * @adev: amdgpu_device pointer
1010 *
1011 * Frees the VRAM scratch page.
1012 */
1013static void amdgpu_device_mem_scratch_fini(struct amdgpu_device *adev)
1014{
1015 amdgpu_bo_free_kernel(&adev->mem_scratch.robj, NULL, NULL);
1016}
1017
1018/**
1019 * amdgpu_device_program_register_sequence - program an array of registers.
1020 *
1021 * @adev: amdgpu_device pointer
1022 * @registers: pointer to the register array
1023 * @array_size: size of the register array
1024 *
1025 * Programs an array or registers with and or masks.
1026 * This is a helper for setting golden registers.
1027 */
1028void amdgpu_device_program_register_sequence(struct amdgpu_device *adev,
1029 const u32 *registers,
1030 const u32 array_size)
1031{
1032 u32 tmp, reg, and_mask, or_mask;
1033 int i;
1034
1035 if (array_size % 3)
1036 return;
1037
1038 for (i = 0; i < array_size; i += 3) {
1039 reg = registers[i + 0];
1040 and_mask = registers[i + 1];
1041 or_mask = registers[i + 2];
1042
1043 if (and_mask == 0xffffffff) {
1044 tmp = or_mask;
1045 } else {
1046 tmp = RREG32(reg);
1047 tmp &= ~and_mask;
1048 if (adev->family >= AMDGPU_FAMILY_AI)
1049 tmp |= (or_mask & and_mask);
1050 else
1051 tmp |= or_mask;
1052 }
1053 WREG32(reg, tmp);
1054 }
1055}
1056
1057/**
1058 * amdgpu_device_pci_config_reset - reset the GPU
1059 *
1060 * @adev: amdgpu_device pointer
1061 *
1062 * Resets the GPU using the pci config reset sequence.
1063 * Only applicable to asics prior to vega10.
1064 */
1065void amdgpu_device_pci_config_reset(struct amdgpu_device *adev)
1066{
1067 pci_write_config_dword(adev->pdev, 0x7c, AMDGPU_ASIC_RESET_DATA);
1068}
1069
1070/**
1071 * amdgpu_device_pci_reset - reset the GPU using generic PCI means
1072 *
1073 * @adev: amdgpu_device pointer
1074 *
1075 * Resets the GPU using generic pci reset interfaces (FLR, SBR, etc.).
1076 */
1077int amdgpu_device_pci_reset(struct amdgpu_device *adev)
1078{
1079 return pci_reset_function(adev->pdev);
1080}
1081
1082/**
1083 * amdgpu_device_resize_fb_bar - try to resize FB BAR
1084 *
1085 * @adev: amdgpu_device pointer
1086 *
1087 * Try to resize FB BAR to make all VRAM CPU accessible. We try very hard not
1088 * to fail, but if any of the BARs is not accessible after the size we abort
1089 * driver loading by returning -ENODEV.
1090 */
1091int amdgpu_device_resize_fb_bar(struct amdgpu_device *adev)
1092{
1093 int rbar_size = pci_rebar_bytes_to_size(adev->gmc.real_vram_size);
1094 struct pci_bus *root;
1095 struct resource *res;
1096 int max_size, r;
1097 unsigned int i;
1098 u16 cmd;
1099
1100 if (!IS_ENABLED(CONFIG_PHYS_ADDR_T_64BIT))
1101 return 0;
1102
1103 /* Bypass for VF */
1104 if (amdgpu_sriov_vf(adev))
1105 return 0;
1106
1107 if (!amdgpu_rebar)
1108 return 0;
1109
1110 /* resizing on Dell G5 SE platforms causes problems with runtime pm */
1111 if ((amdgpu_runtime_pm != 0) &&
1112 adev->pdev->vendor == PCI_VENDOR_ID_ATI &&
1113 adev->pdev->device == 0x731f &&
1114 adev->pdev->subsystem_vendor == PCI_VENDOR_ID_DELL)
1115 return 0;
1116
1117 /* PCI_EXT_CAP_ID_VNDR extended capability is located at 0x100 */
1118 if (!pci_find_ext_capability(adev->pdev, PCI_EXT_CAP_ID_VNDR))
1119 dev_warn(
1120 adev->dev,
1121 "System can't access extended configuration space, please check!!\n");
1122
1123 /* skip if the bios has already enabled large BAR */
1124 if (adev->gmc.real_vram_size &&
1125 (pci_resource_len(adev->pdev, 0) >= adev->gmc.real_vram_size))
1126 return 0;
1127
1128 /* Check if the root BUS has 64bit memory resources */
1129 root = adev->pdev->bus;
1130 while (root->parent)
1131 root = root->parent;
1132
1133 pci_bus_for_each_resource(root, res, i) {
1134 if (res && res->flags & (IORESOURCE_MEM | IORESOURCE_MEM_64) &&
1135 res->start > 0x100000000ull)
1136 break;
1137 }
1138
1139 /* Trying to resize is pointless without a root hub window above 4GB */
1140 if (!res)
1141 return 0;
1142
1143 /* Limit the BAR size to what is available */
1144 max_size = pci_rebar_get_max_size(adev->pdev, 0);
1145 if (max_size < 0)
1146 return 0;
1147 rbar_size = min(max_size, rbar_size);
1148
1149 /* Disable memory decoding while we change the BAR addresses and size */
1150 pci_read_config_word(adev->pdev, PCI_COMMAND, &cmd);
1151 pci_write_config_word(adev->pdev, PCI_COMMAND,
1152 cmd & ~PCI_COMMAND_MEMORY);
1153
1154 /* Tear down doorbell as resizing will release BARs */
1155 amdgpu_doorbell_fini(adev);
1156
1157 r = pci_resize_resource(adev->pdev, 0, rbar_size,
1158 (adev->asic_type >= CHIP_BONAIRE) ? 1 << 5
1159 : 1 << 2);
1160 if (r == -ENOSPC)
1161 dev_info(adev->dev,
1162 "Not enough PCI address space for a large BAR.");
1163 else if (r && r != -ENOTSUPP)
1164 dev_err(adev->dev, "Problem resizing BAR0 (%d).", r);
1165
1166 /* When the doorbell or fb BAR isn't available we have no chance of
1167 * using the device.
1168 */
1169 r = amdgpu_doorbell_init(adev);
1170 if (r || (pci_resource_flags(adev->pdev, 0) & IORESOURCE_UNSET))
1171 return -ENODEV;
1172
1173 pci_write_config_word(adev->pdev, PCI_COMMAND, cmd);
1174
1175 return 0;
1176}
1177
1178/*
1179 * GPU helpers function.
1180 */
1181/**
1182 * amdgpu_device_need_post - check if the hw need post or not
1183 *
1184 * @adev: amdgpu_device pointer
1185 *
1186 * Check if the asic has been initialized (all asics) at driver startup
1187 * or post is needed if hw reset is performed.
1188 * Returns true if need or false if not.
1189 */
1190bool amdgpu_device_need_post(struct amdgpu_device *adev)
1191{
1192 uint32_t reg, flags;
1193
1194 if (amdgpu_sriov_vf(adev))
1195 return false;
1196
1197 flags = amdgpu_device_get_vbios_flags(adev);
1198 if (flags & AMDGPU_VBIOS_SKIP)
1199 return false;
1200 if ((flags & AMDGPU_VBIOS_OPTIONAL) && !adev->bios)
1201 return false;
1202
1203 if (amdgpu_passthrough(adev)) {
1204 /* for FIJI: In whole GPU pass-through virtualization case, after VM reboot
1205 * some old smc fw still need driver do vPost otherwise gpu hang, while
1206 * those smc fw version above 22.15 doesn't have this flaw, so we force
1207 * vpost executed for smc version below 22.15
1208 */
1209 if (adev->asic_type == CHIP_FIJI) {
1210 int err;
1211 uint32_t fw_ver;
1212
1213 err = request_firmware(&adev->pm.fw, "amdgpu/fiji_smc.bin", adev->dev);
1214 /* force vPost if error occurred */
1215 if (err)
1216 return true;
1217
1218 fw_ver = *((uint32_t *)adev->pm.fw->data + 69);
1219 release_firmware(adev->pm.fw);
1220 if (fw_ver < 0x00160e00)
1221 return true;
1222 }
1223 }
1224
1225 /* Don't post if we need to reset whole hive on init */
1226 if (adev->init_lvl->level == AMDGPU_INIT_LEVEL_MINIMAL_XGMI)
1227 return false;
1228
1229 if (adev->has_hw_reset) {
1230 adev->has_hw_reset = false;
1231 return true;
1232 }
1233
1234 /* bios scratch used on CIK+ */
1235 if (adev->asic_type >= CHIP_BONAIRE)
1236 return amdgpu_atombios_scratch_need_asic_init(adev);
1237
1238 /* check MEM_SIZE for older asics */
1239 reg = amdgpu_asic_get_config_memsize(adev);
1240
1241 if ((reg != 0) && (reg != 0xffffffff))
1242 return false;
1243
1244 return true;
1245}
1246
1247/*
1248 * Check whether seamless boot is supported.
1249 *
1250 * So far we only support seamless boot on DCE 3.0 or later.
1251 * If users report that it works on older ASICS as well, we may
1252 * loosen this.
1253 */
1254bool amdgpu_device_seamless_boot_supported(struct amdgpu_device *adev)
1255{
1256 switch (amdgpu_seamless) {
1257 case -1:
1258 break;
1259 case 1:
1260 return true;
1261 case 0:
1262 return false;
1263 default:
1264 dev_err(adev->dev, "Invalid value for amdgpu.seamless: %d\n",
1265 amdgpu_seamless);
1266 return false;
1267 }
1268
1269 if (!(adev->flags & AMD_IS_APU))
1270 return false;
1271
1272 if (adev->mman.keep_stolen_vga_memory)
1273 return false;
1274
1275 return amdgpu_ip_version(adev, DCE_HWIP, 0) >= IP_VERSION(3, 0, 0);
1276}
1277
1278#if IS_ENABLED(CONFIG_X86)
1279static const struct x86_cpu_id amdgpu_pcie_dynamic_switching_quirks[] = {
1280 /*
1281 * Intel hosts such as Rocket Lake, Alder Lake, Raptor Lake and Sapphire Rapids
1282 * don't support dynamic speed switching. Until we have confirmation from Intel
1283 * that a specific host supports it, it's safer that we keep it disabled for all.
1284 *
1285 * https://edc.intel.com/content/www/us/en/design/products/platforms/details/raptor-lake-s/13th-generation-core-processors-datasheet-volume-1-of-2/005/pci-express-support/
1286 * https://gitlab.freedesktop.org/drm/amd/-/issues/2663
1287 */
1288 X86_MATCH_VENDOR_FAM(INTEL, X86_FAMILY_ANY, NULL),
1289 /*
1290 * AMD Ryzen Pinnacle Ridge (Zen+, family 0x17 model 0x08) CPUs don't
1291 * support PCIe dynamic speed switching.
1292 * https://gitlab.freedesktop.org/drm/amd/-/work_items/5436
1293 */
1294 X86_MATCH_VENDOR_FAM_MODEL(AMD, 0x17, 0x08, NULL),
1295 {}
1296};
1297
1298static bool amdgpu_device_pcie_dynamic_switching_supported(struct amdgpu_device *adev)
1299{
1300 /* eGPU change speeds based on USB4 fabric conditions */
1301 if (dev_is_removable(adev->dev))
1302 return true;
1303
1304 /* Hosts have problems with dynamic speed switching */
1305 if (x86_match_cpu(amdgpu_pcie_dynamic_switching_quirks))
1306 return false;
1307
1308 return true;
1309}
1310#else
1311static inline bool amdgpu_device_pcie_dynamic_switching_supported(struct amdgpu_device *adev)
1312{
1313 return true;
1314}
1315#endif
1316
1317static bool amdgpu_device_aspm_support_quirk(struct amdgpu_device *adev)
1318{
1319 /* Enabling ASPM causes randoms hangs on Tahiti and Oland on Zen4.
1320 * It's unclear if this is a platform-specific or GPU-specific issue.
1321 * Disable ASPM on SI for the time being.
1322 */
1323 if (adev->family == AMDGPU_FAMILY_SI ||
1324 (!(adev->pm.pp_feature & PP_PCIE_DPM_MASK) && adev->family == AMDGPU_FAMILY_VI))
1325 return true;
1326
1327#if IS_ENABLED(CONFIG_X86)
1328 struct cpuinfo_x86 *c = &cpu_data(0);
1329
1330 if (c->x86_vendor == X86_VENDOR_INTEL) {
1331 switch (c->x86_model) {
1332 case VFM_MODEL(INTEL_ALDERLAKE):
1333 case VFM_MODEL(INTEL_ALDERLAKE_L):
1334 case VFM_MODEL(INTEL_RAPTORLAKE):
1335 case VFM_MODEL(INTEL_RAPTORLAKE_P):
1336 case VFM_MODEL(INTEL_RAPTORLAKE_S):
1337 case VFM_MODEL(INTEL_TIGERLAKE):
1338 case VFM_MODEL(INTEL_TIGERLAKE_L):
1339 return true;
1340 default:
1341 return false;
1342 }
1343 } else {
1344 return false;
1345 }
1346#else
1347 return false;
1348#endif
1349}
1350
1351/*
1352 * Some dGPUs expose their display endpoint below an internal PCIe switch.
1353 * Use the switch upstream port to query the host-facing link.
1354 */
1355static struct pci_dev *amdgpu_device_get_aspm_pdev(struct amdgpu_device *adev)
1356{
1357 struct pci_dev *swds, *swus;
1358
1359 swds = pci_upstream_bridge(adev->pdev);
1360 if (!swds ||
1361 (swds->vendor != PCI_VENDOR_ID_ATI &&
1362 swds->vendor != PCI_VENDOR_ID_AMD) ||
1363 pci_pcie_type(swds) != PCI_EXP_TYPE_DOWNSTREAM)
1364 return adev->pdev;
1365
1366 swus = pci_upstream_bridge(swds);
1367 if (!swus ||
1368 (swus->vendor != PCI_VENDOR_ID_ATI &&
1369 swus->vendor != PCI_VENDOR_ID_AMD) ||
1370 pci_pcie_type(swus) != PCI_EXP_TYPE_UPSTREAM)
1371 return adev->pdev;
1372
1373 return swus;
1374}
1375
1376/**
1377 * amdgpu_device_should_use_aspm - check if the device should program ASPM
1378 *
1379 * @adev: amdgpu_device pointer
1380 *
1381 * Confirm whether the module parameter and pcie bridge agree that ASPM should
1382 * be set for this device.
1383 *
1384 * Returns true if it should be used or false if not.
1385 */
1386bool amdgpu_device_should_use_aspm(struct amdgpu_device *adev)
1387{
1388 struct pci_dev *aspm_pdev, *parent;
1389 bool enabled;
1390
1391 switch (amdgpu_aspm) {
1392 case -1:
1393 break;
1394 case 0:
1395 return false;
1396 case 1:
1397 return true;
1398 default:
1399 return false;
1400 }
1401 if (adev->flags & AMD_IS_APU)
1402 return false;
1403 if (amdgpu_device_aspm_support_quirk(adev))
1404 return false;
1405
1406 /*
1407 * pcie_aspm_enabled() checks the link between its argument and
1408 * the immediate upstream bridge. Use SWUS for dGPUs with an
1409 * internal switch so that this is the host-facing link.
1410 */
1411 aspm_pdev = amdgpu_device_get_aspm_pdev(adev);
1412 parent = pci_upstream_bridge(aspm_pdev);
1413 if (!parent) {
1414 dev_dbg(adev->dev, "ASPM: no upstream PCIe link for %s\n",
1415 pci_name(aspm_pdev));
1416 return false;
1417 }
1418
1419 enabled = pcie_aspm_enabled(aspm_pdev);
1420 /* Report the exact link used for the automatic ASPM decision. */
1421 dev_dbg(adev->dev, "ASPM: link %s <-> %s is %s\n",
1422 pci_name(parent), pci_name(aspm_pdev),
1423 enabled ? "enabled" : "disabled");
1424
1425 return enabled;
1426}
1427
1428/* if we get transitioned to only one device, take VGA back */
1429/**
1430 * amdgpu_device_vga_set_decode - enable/disable vga decode
1431 *
1432 * @pdev: PCI device pointer
1433 * @state: enable/disable vga decode
1434 *
1435 * Enable/disable vga decode (all asics).
1436 * Returns VGA resource flags.
1437 */
1438static unsigned int amdgpu_device_vga_set_decode(struct pci_dev *pdev,
1439 bool state)
1440{
1441 struct amdgpu_device *adev = drm_to_adev(pci_get_drvdata(pdev));
1442
1443 amdgpu_asic_set_vga_state(adev, state);
1444 if (state)
1445 return VGA_RSRC_LEGACY_IO | VGA_RSRC_LEGACY_MEM |
1446 VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM;
1447 else
1448 return VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM;
1449}
1450
1451/**
1452 * amdgpu_device_check_block_size - validate the vm block size
1453 *
1454 * @adev: amdgpu_device pointer
1455 *
1456 * Validates the vm block size specified via module parameter.
1457 * The vm block size defines number of bits in page table versus page directory,
1458 * a page is 4KB so we have 12 bits offset, minimum 9 bits in the
1459 * page table and the remaining bits are in the page directory.
1460 */
1461static void amdgpu_device_check_block_size(struct amdgpu_device *adev)
1462{
1463 /* defines number of bits in page table versus page directory,
1464 * a page is 4KB so we have 12 bits offset, minimum 9 bits in the
1465 * page table and the remaining bits are in the page directory
1466 */
1467 if (amdgpu_vm_block_size == -1)
1468 return;
1469
1470 if (amdgpu_vm_block_size < 9) {
1471 dev_warn(adev->dev, "VM page table size (%d) too small\n",
1472 amdgpu_vm_block_size);
1473 amdgpu_vm_block_size = -1;
1474 }
1475}
1476
1477/**
1478 * amdgpu_device_check_vm_size - validate the vm size
1479 *
1480 * @adev: amdgpu_device pointer
1481 *
1482 * Validates the vm size in GB specified via module parameter.
1483 * The VM size is the size of the GPU virtual memory space in GB.
1484 */
1485static void amdgpu_device_check_vm_size(struct amdgpu_device *adev)
1486{
1487 /* no need to check the default value */
1488 if (amdgpu_vm_size == -1)
1489 return;
1490
1491 if (amdgpu_vm_size < 1) {
1492 dev_warn(adev->dev, "VM size (%d) too small, min is 1GB\n",
1493 amdgpu_vm_size);
1494 amdgpu_vm_size = -1;
1495 }
1496}
1497
1498static void amdgpu_device_check_smu_prv_buffer_size(struct amdgpu_device *adev)
1499{
1500 struct sysinfo si;
1501 bool is_os_64 = (sizeof(void *) == 8);
1502 uint64_t total_memory;
1503 uint64_t dram_size_seven_GB = 0x1B8000000;
1504 uint64_t dram_size_three_GB = 0xB8000000;
1505
1506 if (amdgpu_smu_memory_pool_size == 0)
1507 return;
1508
1509 if (!is_os_64) {
1510 dev_warn(adev->dev, "Not 64-bit OS, feature not supported\n");
1511 goto def_value;
1512 }
1513 si_meminfo(&si);
1514 total_memory = (uint64_t)si.totalram * si.mem_unit;
1515
1516 if ((amdgpu_smu_memory_pool_size == 1) ||
1517 (amdgpu_smu_memory_pool_size == 2)) {
1518 if (total_memory < dram_size_three_GB)
1519 goto def_value1;
1520 } else if ((amdgpu_smu_memory_pool_size == 4) ||
1521 (amdgpu_smu_memory_pool_size == 8)) {
1522 if (total_memory < dram_size_seven_GB)
1523 goto def_value1;
1524 } else {
1525 dev_warn(adev->dev, "Smu memory pool size not supported\n");
1526 goto def_value;
1527 }
1528 adev->pm.smu_prv_buffer_size = amdgpu_smu_memory_pool_size << 28;
1529
1530 return;
1531
1532def_value1:
1533 dev_warn(adev->dev, "No enough system memory\n");
1534def_value:
1535 adev->pm.smu_prv_buffer_size = 0;
1536}
1537
1538static int amdgpu_device_init_apu_flags(struct amdgpu_device *adev)
1539{
1540 if (!(adev->flags & AMD_IS_APU) ||
1541 adev->asic_type < CHIP_RAVEN)
1542 return 0;
1543
1544 switch (adev->asic_type) {
1545 case CHIP_RAVEN:
1546 if (adev->pdev->device == 0x15dd)
1547 adev->apu_flags |= AMD_APU_IS_RAVEN;
1548 if (adev->pdev->device == 0x15d8)
1549 adev->apu_flags |= AMD_APU_IS_PICASSO;
1550 break;
1551 case CHIP_RENOIR:
1552 if ((adev->pdev->device == 0x1636) ||
1553 (adev->pdev->device == 0x164c))
1554 adev->apu_flags |= AMD_APU_IS_RENOIR;
1555 else
1556 adev->apu_flags |= AMD_APU_IS_GREEN_SARDINE;
1557 break;
1558 case CHIP_VANGOGH:
1559 adev->apu_flags |= AMD_APU_IS_VANGOGH;
1560 break;
1561 case CHIP_YELLOW_CARP:
1562 break;
1563 case CHIP_CYAN_SKILLFISH:
1564 if ((adev->pdev->device == 0x13FE) ||
1565 (adev->pdev->device == 0x143F))
1566 adev->apu_flags |= AMD_APU_IS_CYAN_SKILLFISH2;
1567 break;
1568 default:
1569 break;
1570 }
1571
1572 return 0;
1573}
1574
1575/**
1576 * amdgpu_device_check_arguments - validate module params
1577 *
1578 * @adev: amdgpu_device pointer
1579 *
1580 * Validates certain module parameters and updates
1581 * the associated values used by the driver (all asics).
1582 */
1583static int amdgpu_device_check_arguments(struct amdgpu_device *adev)
1584{
1585 int i;
1586
1587 if (amdgpu_sched_jobs < 4) {
1588 dev_warn(adev->dev, "sched jobs (%d) must be at least 4\n",
1589 amdgpu_sched_jobs);
1590 amdgpu_sched_jobs = 4;
1591 } else if (!is_power_of_2(amdgpu_sched_jobs)) {
1592 dev_warn(adev->dev, "sched jobs (%d) must be a power of 2\n",
1593 amdgpu_sched_jobs);
1594 amdgpu_sched_jobs = roundup_pow_of_two(amdgpu_sched_jobs);
1595 }
1596
1597 if (amdgpu_gart_size != -1 && amdgpu_gart_size < 32) {
1598 /* gart size must be greater or equal to 32M */
1599 dev_warn(adev->dev, "gart size (%d) too small\n",
1600 amdgpu_gart_size);
1601 amdgpu_gart_size = -1;
1602 }
1603
1604 if (amdgpu_gtt_size != -1 && amdgpu_gtt_size < 32) {
1605 /* gtt size must be greater or equal to 32M */
1606 dev_warn(adev->dev, "gtt size (%d) too small\n",
1607 amdgpu_gtt_size);
1608 amdgpu_gtt_size = -1;
1609 }
1610
1611 /* valid range is between 4 and 9 inclusive */
1612 if (amdgpu_vm_fragment_size != -1 &&
1613 (amdgpu_vm_fragment_size > 9 || amdgpu_vm_fragment_size < 4)) {
1614 dev_warn(adev->dev, "valid range is between 4 and 9\n");
1615 amdgpu_vm_fragment_size = -1;
1616 }
1617
1618 if (amdgpu_sched_hw_submission < 2) {
1619 dev_warn(adev->dev, "sched hw submission jobs (%d) must be at least 2\n",
1620 amdgpu_sched_hw_submission);
1621 amdgpu_sched_hw_submission = 2;
1622 } else if (!is_power_of_2(amdgpu_sched_hw_submission)) {
1623 dev_warn(adev->dev, "sched hw submission jobs (%d) must be a power of 2\n",
1624 amdgpu_sched_hw_submission);
1625 amdgpu_sched_hw_submission = roundup_pow_of_two(amdgpu_sched_hw_submission);
1626 }
1627
1628 if (amdgpu_reset_method < -1 || amdgpu_reset_method > 4) {
1629 dev_warn(adev->dev, "invalid option for reset method, reverting to default\n");
1630 amdgpu_reset_method = -1;
1631 }
1632
1633 amdgpu_device_check_smu_prv_buffer_size(adev);
1634
1635 amdgpu_device_check_vm_size(adev);
1636
1637 amdgpu_device_check_block_size(adev);
1638
1639 adev->firmware.load_type = amdgpu_ucode_get_load_type(adev, amdgpu_fw_load_type);
1640
1641 for (i = 0; i < MAX_XCP; i++) {
1642 switch (amdgpu_enforce_isolation) {
1643 case -1:
1644 case 0:
1645 default:
1646 /* disable */
1647 adev->enforce_isolation[i] = AMDGPU_ENFORCE_ISOLATION_DISABLE;
1648 break;
1649 case 1:
1650 /* enable */
1651 adev->enforce_isolation[i] =
1652 AMDGPU_ENFORCE_ISOLATION_ENABLE;
1653 break;
1654 case 2:
1655 /* enable legacy mode */
1656 adev->enforce_isolation[i] =
1657 AMDGPU_ENFORCE_ISOLATION_ENABLE_LEGACY;
1658 break;
1659 case 3:
1660 /* enable only process isolation without submitting cleaner shader */
1661 adev->enforce_isolation[i] =
1662 AMDGPU_ENFORCE_ISOLATION_NO_CLEANER_SHADER;
1663 break;
1664 }
1665 }
1666
1667 return 0;
1668}
1669
1670/**
1671 * amdgpu_switcheroo_set_state - set switcheroo state
1672 *
1673 * @pdev: pci dev pointer
1674 * @state: vga_switcheroo state
1675 *
1676 * Callback for the switcheroo driver. Suspends or resumes
1677 * the asics before or after it is powered up using ACPI methods.
1678 */
1679static void amdgpu_switcheroo_set_state(struct pci_dev *pdev,
1680 enum vga_switcheroo_state state)
1681{
1682 struct drm_device *dev = pci_get_drvdata(pdev);
1683 int r;
1684
1685 if (amdgpu_device_supports_px(drm_to_adev(dev)) &&
1686 state == VGA_SWITCHEROO_OFF)
1687 return;
1688
1689 if (state == VGA_SWITCHEROO_ON) {
1690 pr_info("switched on\n");
1691 /* don't suspend or resume card normally */
1692 dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;
1693
1694 pci_set_power_state(pdev, PCI_D0);
1695 amdgpu_device_load_pci_state(pdev);
1696 r = pci_enable_device(pdev);
1697 if (r)
1698 dev_warn(&pdev->dev, "pci_enable_device failed (%d)\n",
1699 r);
1700 amdgpu_device_resume(dev, true);
1701
1702 dev->switch_power_state = DRM_SWITCH_POWER_ON;
1703 } else {
1704 dev_info(&pdev->dev, "switched off\n");
1705 dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;
1706 amdgpu_device_prepare(dev);
1707 amdgpu_device_suspend(dev, true);
1708 amdgpu_device_cache_pci_state(pdev);
1709 /* Shut down the device */
1710 pci_disable_device(pdev);
1711 pci_set_power_state(pdev, PCI_D3cold);
1712 dev->switch_power_state = DRM_SWITCH_POWER_OFF;
1713 }
1714}
1715
1716/**
1717 * amdgpu_switcheroo_can_switch - see if switcheroo state can change
1718 *
1719 * @pdev: pci dev pointer
1720 *
1721 * Callback for the switcheroo driver. Check of the switcheroo
1722 * state can be changed.
1723 * Returns true if the state can be changed, false if not.
1724 */
1725static bool amdgpu_switcheroo_can_switch(struct pci_dev *pdev)
1726{
1727 struct drm_device *dev = pci_get_drvdata(pdev);
1728
1729 /*
1730 * FIXME: open_count is protected by drm_global_mutex but that would lead to
1731 * locking inversion with the driver load path. And the access here is
1732 * completely racy anyway. So don't bother with locking for now.
1733 */
1734 return atomic_read(&dev->open_count) == 0;
1735}
1736
1737static const struct vga_switcheroo_client_ops amdgpu_switcheroo_ops = {
1738 .set_gpu_state = amdgpu_switcheroo_set_state,
1739 .reprobe = NULL,
1740 .can_switch = amdgpu_switcheroo_can_switch,
1741};
1742
1743/**
1744 * amdgpu_device_enable_virtual_display - enable virtual display feature
1745 *
1746 * @adev: amdgpu_device pointer
1747 *
1748 * Enabled the virtual display feature if the user has enabled it via
1749 * the module parameter virtual_display. This feature provides a virtual
1750 * display hardware on headless boards or in virtualized environments.
1751 * This function parses and validates the configuration string specified by
1752 * the user and configures the virtual display configuration (number of
1753 * virtual connectors, crtcs, etc.) specified.
1754 */
1755static void amdgpu_device_enable_virtual_display(struct amdgpu_device *adev)
1756{
1757 adev->enable_virtual_display = false;
1758
1759 if (amdgpu_virtual_display) {
1760 const char *pci_address_name = pci_name(adev->pdev);
1761 char *pciaddstr, *pciaddstr_tmp, *pciaddname_tmp, *pciaddname;
1762
1763 pciaddstr = kstrdup(amdgpu_virtual_display, GFP_KERNEL);
1764 pciaddstr_tmp = pciaddstr;
1765 while ((pciaddname_tmp = strsep(&pciaddstr_tmp, ";"))) {
1766 pciaddname = strsep(&pciaddname_tmp, ",");
1767 if (!strcmp("all", pciaddname)
1768 || !strcmp(pci_address_name, pciaddname)) {
1769 long num_crtc;
1770 int res = -1;
1771
1772 adev->enable_virtual_display = true;
1773
1774 if (pciaddname_tmp)
1775 res = kstrtol(pciaddname_tmp, 10,
1776 &num_crtc);
1777
1778 if (!res) {
1779 if (num_crtc < 1)
1780 num_crtc = 1;
1781 if (num_crtc > 6)
1782 num_crtc = 6;
1783 adev->mode_info.num_crtc = num_crtc;
1784 } else {
1785 adev->mode_info.num_crtc = 1;
1786 }
1787 break;
1788 }
1789 }
1790
1791 dev_info(
1792 adev->dev,
1793 "virtual display string:%s, %s:virtual_display:%d, num_crtc:%d\n",
1794 amdgpu_virtual_display, pci_address_name,
1795 adev->enable_virtual_display, adev->mode_info.num_crtc);
1796
1797 kfree(pciaddstr);
1798 }
1799}
1800
1801void amdgpu_device_set_sriov_virtual_display(struct amdgpu_device *adev)
1802{
1803 if (amdgpu_sriov_vf(adev) && !adev->enable_virtual_display) {
1804 adev->mode_info.num_crtc = 1;
1805 adev->enable_virtual_display = true;
1806 dev_info(adev->dev, "virtual_display:%d, num_crtc:%d\n",
1807 adev->enable_virtual_display,
1808 adev->mode_info.num_crtc);
1809 }
1810}
1811
1812/**
1813 * amdgpu_device_parse_gpu_info_fw - parse gpu info firmware
1814 *
1815 * @adev: amdgpu_device pointer
1816 *
1817 * Parses the asic configuration parameters specified in the gpu info
1818 * firmware and makes them available to the driver for use in configuring
1819 * the asic.
1820 * Returns 0 on success, -EINVAL on failure.
1821 */
1822static int amdgpu_device_parse_gpu_info_fw(struct amdgpu_device *adev)
1823{
1824 const char *chip_name;
1825 int err;
1826 const struct gpu_info_firmware_header_v1_0 *hdr;
1827
1828 adev->firmware.gpu_info_fw = NULL;
1829
1830 switch (adev->asic_type) {
1831 default:
1832 return 0;
1833 case CHIP_VEGA10:
1834 chip_name = "vega10";
1835 break;
1836 case CHIP_VEGA12:
1837 chip_name = "vega12";
1838 break;
1839 case CHIP_RAVEN:
1840 if (adev->apu_flags & AMD_APU_IS_RAVEN2)
1841 chip_name = "raven2";
1842 else if (adev->apu_flags & AMD_APU_IS_PICASSO)
1843 chip_name = "picasso";
1844 else
1845 chip_name = "raven";
1846 break;
1847 case CHIP_ARCTURUS:
1848 chip_name = "arcturus";
1849 break;
1850 case CHIP_NAVI12:
1851 if (adev->discovery.bin)
1852 return 0;
1853 chip_name = "navi12";
1854 break;
1855 case CHIP_CYAN_SKILLFISH:
1856 if (adev->discovery.bin)
1857 return 0;
1858 chip_name = "cyan_skillfish";
1859 break;
1860 }
1861
1862 err = amdgpu_ucode_request(adev, &adev->firmware.gpu_info_fw,
1863 AMDGPU_UCODE_OPTIONAL,
1864 "amdgpu/%s_gpu_info.bin", chip_name);
1865 if (err) {
1866 dev_err(adev->dev,
1867 "Failed to get gpu_info firmware \"%s_gpu_info.bin\"\n",
1868 chip_name);
1869 goto out;
1870 }
1871
1872 hdr = (const struct gpu_info_firmware_header_v1_0 *)adev->firmware.gpu_info_fw->data;
1873 amdgpu_ucode_print_gpu_info_hdr(&hdr->header);
1874
1875 switch (hdr->version_major) {
1876 case 1:
1877 {
1878 const struct gpu_info_firmware_v1_0 *gpu_info_fw =
1879 (const struct gpu_info_firmware_v1_0 *)(adev->firmware.gpu_info_fw->data +
1880 le32_to_cpu(hdr->header.ucode_array_offset_bytes));
1881
1882 /*
1883 * Should be dropped when DAL no longer needs it.
1884 */
1885 if (adev->asic_type == CHIP_NAVI12)
1886 goto parse_soc_bounding_box;
1887
1888 adev->gfx.config.max_shader_engines = le32_to_cpu(gpu_info_fw->gc_num_se);
1889 adev->gfx.config.max_cu_per_sh = le32_to_cpu(gpu_info_fw->gc_num_cu_per_sh);
1890 adev->gfx.config.max_sh_per_se = le32_to_cpu(gpu_info_fw->gc_num_sh_per_se);
1891 adev->gfx.config.max_backends_per_se = le32_to_cpu(gpu_info_fw->gc_num_rb_per_se);
1892 adev->gfx.config.max_texture_channel_caches =
1893 le32_to_cpu(gpu_info_fw->gc_num_tccs);
1894 adev->gfx.config.max_gprs = le32_to_cpu(gpu_info_fw->gc_num_gprs);
1895 adev->gfx.config.max_gs_threads = le32_to_cpu(gpu_info_fw->gc_num_max_gs_thds);
1896 adev->gfx.config.gs_vgt_table_depth = le32_to_cpu(gpu_info_fw->gc_gs_table_depth);
1897 adev->gfx.config.gs_prim_buffer_depth = le32_to_cpu(gpu_info_fw->gc_gsprim_buff_depth);
1898 adev->gfx.config.double_offchip_lds_buf =
1899 le32_to_cpu(gpu_info_fw->gc_double_offchip_lds_buffer);
1900 adev->gfx.cu_info.wave_front_size = le32_to_cpu(gpu_info_fw->gc_wave_size);
1901 adev->gfx.cu_info.max_waves_per_simd =
1902 le32_to_cpu(gpu_info_fw->gc_max_waves_per_simd);
1903 adev->gfx.cu_info.max_scratch_slots_per_cu =
1904 le32_to_cpu(gpu_info_fw->gc_max_scratch_slots_per_cu);
1905 adev->gfx.cu_info.lds_size = le32_to_cpu(gpu_info_fw->gc_lds_size);
1906 if (hdr->version_minor >= 1) {
1907 const struct gpu_info_firmware_v1_1 *gpu_info_fw =
1908 (const struct gpu_info_firmware_v1_1 *)(adev->firmware.gpu_info_fw->data +
1909 le32_to_cpu(hdr->header.ucode_array_offset_bytes));
1910 adev->gfx.config.num_sc_per_sh =
1911 le32_to_cpu(gpu_info_fw->num_sc_per_sh);
1912 adev->gfx.config.num_packer_per_sc =
1913 le32_to_cpu(gpu_info_fw->num_packer_per_sc);
1914 }
1915
1916parse_soc_bounding_box:
1917 /*
1918 * soc bounding box info is not integrated in disocovery table,
1919 * we always need to parse it from gpu info firmware if needed.
1920 */
1921 if (hdr->version_minor == 2) {
1922 const struct gpu_info_firmware_v1_2 *gpu_info_fw =
1923 (const struct gpu_info_firmware_v1_2 *)(adev->firmware.gpu_info_fw->data +
1924 le32_to_cpu(hdr->header.ucode_array_offset_bytes));
1925 adev->dm.soc_bounding_box = &gpu_info_fw->soc_bounding_box;
1926 }
1927 break;
1928 }
1929 default:
1930 dev_err(adev->dev,
1931 "Unsupported gpu_info table %d\n", hdr->header.ucode_version);
1932 err = -EINVAL;
1933 goto out;
1934 }
1935out:
1936 return err;
1937}
1938
1939static void amdgpu_uid_init(struct amdgpu_device *adev)
1940{
1941 /* Initialize the UID for the device */
1942 adev->uid_info = kzalloc_obj(struct amdgpu_uid);
1943 if (!adev->uid_info) {
1944 dev_warn(adev->dev, "Failed to allocate memory for UID\n");
1945 return;
1946 }
1947 adev->uid_info->adev = adev;
1948}
1949
1950static void amdgpu_uid_fini(struct amdgpu_device *adev)
1951{
1952 /* Free the UID memory */
1953 kfree(adev->uid_info);
1954 adev->uid_info = NULL;
1955}
1956
1957static void amdgpu_device_init_pcie_links(struct amdgpu_device *adev)
1958{
1959 adev->link_dev = adev->pdev;
1960 adev->link_partner = pci_upstream_bridge(adev->link_dev);
1961
1962 /* Skip upstream/downstream switches internal to the dGPU. */
1963 while (adev->link_partner &&
1964 adev->link_partner->vendor == PCI_VENDOR_ID_ATI) {
1965 adev->link_dev = adev->link_partner;
1966 adev->link_partner = pci_upstream_bridge(adev->link_dev);
1967 }
1968}
1969
1970/**
1971 * amdgpu_device_ip_early_init - run early init for hardware IPs
1972 *
1973 * @adev: amdgpu_device pointer
1974 *
1975 * Early initialization pass for hardware IPs. The hardware IPs that make
1976 * up each asic are discovered each IP's early_init callback is run. This
1977 * is the first stage in initializing the asic.
1978 * Returns 0 on success, negative error code on failure.
1979 */
1980static int amdgpu_device_ip_early_init(struct amdgpu_device *adev)
1981{
1982 struct amdgpu_ip_block *ip_block;
1983 bool total, skip_bios, early_full_gpu_access = false;
1984 uint32_t bios_flags;
1985 int i, r;
1986
1987 amdgpu_device_enable_virtual_display(adev);
1988
1989 if (amdgpu_sriov_vf(adev)) {
1990 /*
1991 * Legacy hosts do not provide init data before early init, so
1992 * keep the original early full GPU access request for them. Newer
1993 * hosts publish the init data through VF FB, which lets us defer
1994 * full GPU access until after non-GPU early init work is done.
1995 */
1996 early_full_gpu_access = (adev->virt.req_init_data_ver == 0);
1997 if (early_full_gpu_access) {
1998 r = amdgpu_virt_request_full_gpu(adev, true);
1999 if (r)
2000 return r;
2001 }
2002
2003 r = amdgpu_virt_init_critical_region(adev);
2004 if (r)
2005 return r;
2006 }
2007
2008 switch (adev->asic_type) {
2009#ifdef CONFIG_DRM_AMDGPU_SI
2010 case CHIP_VERDE:
2011 case CHIP_TAHITI:
2012 case CHIP_PITCAIRN:
2013 case CHIP_OLAND:
2014 case CHIP_HAINAN:
2015 adev->family = AMDGPU_FAMILY_SI;
2016 r = si_set_ip_blocks(adev);
2017 if (r)
2018 return r;
2019 break;
2020#endif
2021#ifdef CONFIG_DRM_AMDGPU_CIK
2022 case CHIP_BONAIRE:
2023 case CHIP_HAWAII:
2024 case CHIP_KAVERI:
2025 case CHIP_KABINI:
2026 case CHIP_MULLINS:
2027 if (adev->flags & AMD_IS_APU)
2028 adev->family = AMDGPU_FAMILY_KV;
2029 else
2030 adev->family = AMDGPU_FAMILY_CI;
2031
2032 r = cik_set_ip_blocks(adev);
2033 if (r)
2034 return r;
2035 break;
2036#endif
2037 case CHIP_TOPAZ:
2038 case CHIP_TONGA:
2039 case CHIP_FIJI:
2040 case CHIP_POLARIS10:
2041 case CHIP_POLARIS11:
2042 case CHIP_POLARIS12:
2043 case CHIP_VEGAM:
2044 case CHIP_CARRIZO:
2045 case CHIP_STONEY:
2046 if (adev->flags & AMD_IS_APU)
2047 adev->family = AMDGPU_FAMILY_CZ;
2048 else
2049 adev->family = AMDGPU_FAMILY_VI;
2050
2051 r = vi_set_ip_blocks(adev);
2052 if (r)
2053 return r;
2054 break;
2055 default:
2056 r = amdgpu_discovery_set_ip_blocks(adev);
2057 if (r) {
2058 adev->num_ip_blocks = 0;
2059 return r;
2060 }
2061 break;
2062 }
2063
2064 /* Check for IP version 9.4.3 with A0 hardware */
2065 if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 3) &&
2066 !amdgpu_device_get_rev_id(adev)) {
2067 dev_err(adev->dev, "Unsupported A0 hardware\n");
2068 return -ENODEV; /* device unsupported - no device error */
2069 }
2070
2071 if (amdgpu_has_atpx() &&
2072 (amdgpu_is_atpx_hybrid() ||
2073 amdgpu_has_atpx_dgpu_power_cntl()) &&
2074 ((adev->flags & AMD_IS_APU) == 0) &&
2075 !dev_is_removable(&adev->pdev->dev))
2076 adev->flags |= AMD_IS_PX;
2077
2078 if (!(adev->flags & AMD_IS_APU))
2079 adev->has_pr3 = adev->link_partner &&
2080 pci_pr3_present(adev->link_partner);
2081
2082 adev->pm.pp_feature = amdgpu_pp_feature_mask;
2083 if (amdgpu_sriov_vf(adev) || sched_policy == KFD_SCHED_POLICY_NO_HWS)
2084 adev->pm.pp_feature &= ~PP_GFXOFF_MASK;
2085 if (amdgpu_sriov_vf(adev) && adev->asic_type == CHIP_SIENNA_CICHLID)
2086 adev->pm.pp_feature &= ~PP_OVERDRIVE_MASK;
2087 if (!amdgpu_device_pcie_dynamic_switching_supported(adev))
2088 adev->pm.pp_feature &= ~PP_PCIE_DPM_MASK;
2089
2090 adev->virt.is_xgmi_node_migrate_enabled = false;
2091 if (amdgpu_sriov_vf(adev)) {
2092 adev->virt.is_xgmi_node_migrate_enabled =
2093 amdgpu_ip_version((adev), GC_HWIP, 0) == IP_VERSION(9, 4, 4);
2094 }
2095
2096 total = true;
2097 for (i = 0; i < adev->num_ip_blocks; i++) {
2098 ip_block = &adev->ip_blocks[i];
2099
2100 if ((amdgpu_ip_block_mask & (1 << i)) == 0) {
2101 dev_warn(adev->dev, "disabled ip block: %d <%s>\n", i,
2102 adev->ip_blocks[i].version->funcs->name);
2103 adev->ip_blocks[i].status.valid = false;
2104 } else if (ip_block->version->funcs->early_init) {
2105 r = ip_block->version->funcs->early_init(ip_block);
2106 if (r == -ENOENT) {
2107 adev->ip_blocks[i].status.valid = false;
2108 } else if (r) {
2109 dev_err(adev->dev,
2110 "early_init of IP block <%s> failed %d\n",
2111 adev->ip_blocks[i].version->funcs->name,
2112 r);
2113 total = false;
2114 } else {
2115 adev->ip_blocks[i].status.valid = true;
2116 }
2117 } else {
2118 adev->ip_blocks[i].status.valid = true;
2119 }
2120 /* get the vbios after the asic_funcs are set up */
2121 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON) {
2122 r = amdgpu_device_parse_gpu_info_fw(adev);
2123 if (r)
2124 return r;
2125
2126 bios_flags = amdgpu_device_get_vbios_flags(adev);
2127 skip_bios = !!(bios_flags & AMDGPU_VBIOS_SKIP);
2128 /* Read BIOS */
2129 if (!skip_bios) {
2130 bool optional =
2131 !!(bios_flags & AMDGPU_VBIOS_OPTIONAL);
2132 if (!amdgpu_get_bios(adev) && !optional)
2133 return -EINVAL;
2134
2135 if (optional && !adev->bios)
2136 dev_info(
2137 adev->dev,
2138 "VBIOS image optional, proceeding without VBIOS image");
2139
2140 if (adev->bios) {
2141 r = amdgpu_atombios_init(adev);
2142 if (r) {
2143 dev_err(adev->dev,
2144 "amdgpu_atombios_init failed\n");
2145 amdgpu_vf_error_put(
2146 adev,
2147 AMDGIM_ERROR_VF_ATOMBIOS_INIT_FAIL,
2148 0, 0);
2149 return r;
2150 }
2151 }
2152 }
2153
2154 /*get pf2vf msg info at it's earliest time*/
2155 if (amdgpu_sriov_vf(adev))
2156 amdgpu_virt_init_data_exchange(adev);
2157
2158 }
2159 }
2160 if (!total)
2161 return -ENODEV;
2162
2163 /* Request full GPU access only for the remaining SR-IOV init work. */
2164 if (amdgpu_sriov_vf(adev) && !early_full_gpu_access) {
2165 r = amdgpu_virt_request_full_gpu(adev, true);
2166 if (r)
2167 return r;
2168 }
2169
2170 if (adev->gmc.xgmi.supported)
2171 amdgpu_xgmi_early_init(adev);
2172
2173 if (amdgpu_is_multi_aid(adev))
2174 amdgpu_uid_init(adev);
2175 ip_block = amdgpu_device_ip_get_ip_block(adev, AMD_IP_BLOCK_TYPE_GFX);
2176 if (ip_block->status.valid != false)
2177 amdgpu_amdkfd_device_probe(adev);
2178
2179 adev->cg_flags &= amdgpu_cg_mask;
2180 adev->pg_flags &= amdgpu_pg_mask;
2181
2182 amdgpu_smu_early_init(adev);
2183
2184 return 0;
2185}
2186
2187static int amdgpu_device_ip_hw_init_phase1(struct amdgpu_device *adev)
2188{
2189 int i, r;
2190
2191 for (i = 0; i < adev->num_ip_blocks; i++) {
2192 if (!adev->ip_blocks[i].status.sw)
2193 continue;
2194 if (adev->ip_blocks[i].status.hw)
2195 continue;
2196 if (!amdgpu_ip_member_of_hwini(
2197 adev, adev->ip_blocks[i].version->type))
2198 continue;
2199 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
2200 (amdgpu_sriov_vf(adev) && (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP)) ||
2201 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH) {
2202 r = adev->ip_blocks[i].version->funcs->hw_init(&adev->ip_blocks[i]);
2203 if (r) {
2204 dev_err(adev->dev,
2205 "hw_init of IP block <%s> failed %d\n",
2206 adev->ip_blocks[i].version->funcs->name,
2207 r);
2208 return r;
2209 }
2210 adev->ip_blocks[i].status.hw = true;
2211 }
2212 }
2213
2214 return 0;
2215}
2216
2217static int amdgpu_device_ip_hw_init_phase2(struct amdgpu_device *adev)
2218{
2219 int i, r;
2220
2221 for (i = 0; i < adev->num_ip_blocks; i++) {
2222 if (!adev->ip_blocks[i].status.sw)
2223 continue;
2224 if (adev->ip_blocks[i].status.hw)
2225 continue;
2226 if (!amdgpu_ip_member_of_hwini(
2227 adev, adev->ip_blocks[i].version->type))
2228 continue;
2229 r = adev->ip_blocks[i].version->funcs->hw_init(&adev->ip_blocks[i]);
2230 if (r) {
2231 dev_err(adev->dev,
2232 "hw_init of IP block <%s> failed %d\n",
2233 adev->ip_blocks[i].version->funcs->name, r);
2234 return r;
2235 }
2236 adev->ip_blocks[i].status.hw = true;
2237 }
2238
2239 return 0;
2240}
2241
2242static int amdgpu_device_fw_loading(struct amdgpu_device *adev)
2243{
2244 int r = 0;
2245 int i;
2246 uint32_t smu_version;
2247
2248 if (adev->asic_type >= CHIP_VEGA10) {
2249 for (i = 0; i < adev->num_ip_blocks; i++) {
2250 if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_PSP)
2251 continue;
2252
2253 if (!amdgpu_ip_member_of_hwini(adev,
2254 AMD_IP_BLOCK_TYPE_PSP))
2255 break;
2256
2257 if (!adev->ip_blocks[i].status.sw)
2258 continue;
2259
2260 /* no need to do the fw loading again if already done*/
2261 if (adev->ip_blocks[i].status.hw == true)
2262 break;
2263
2264 if (amdgpu_in_reset(adev) || adev->in_suspend) {
2265 r = amdgpu_ip_block_resume(&adev->ip_blocks[i]);
2266 if (r)
2267 return r;
2268 } else {
2269 r = adev->ip_blocks[i].version->funcs->hw_init(&adev->ip_blocks[i]);
2270 if (r) {
2271 dev_err(adev->dev,
2272 "hw_init of IP block <%s> failed %d\n",
2273 adev->ip_blocks[i]
2274 .version->funcs->name,
2275 r);
2276 return r;
2277 }
2278 adev->ip_blocks[i].status.hw = true;
2279 }
2280 break;
2281 }
2282 }
2283
2284 if (!amdgpu_sriov_vf(adev) || adev->asic_type == CHIP_TONGA)
2285 r = amdgpu_pm_load_smu_firmware(adev, &smu_version);
2286
2287 return r;
2288}
2289
2290static int amdgpu_device_init_schedulers(struct amdgpu_device *adev)
2291{
2292 struct drm_sched_init_args args = {
2293 .ops = &amdgpu_sched_ops,
2294 .num_rqs = DRM_SCHED_PRIORITY_COUNT,
2295 .timeout_wq = adev->reset_domain->wq,
2296 .dev = adev->dev,
2297 };
2298 long timeout;
2299 int r, i;
2300
2301 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
2302 struct amdgpu_ring *ring = adev->rings[i];
2303
2304 /* No need to setup the GPU scheduler for rings that don't need it */
2305 if (!ring || ring->no_scheduler)
2306 continue;
2307
2308 switch (ring->funcs->type) {
2309 case AMDGPU_RING_TYPE_GFX:
2310 timeout = adev->gfx_timeout;
2311 break;
2312 case AMDGPU_RING_TYPE_COMPUTE:
2313 timeout = adev->compute_timeout;
2314 break;
2315 case AMDGPU_RING_TYPE_SDMA:
2316 timeout = adev->sdma_timeout;
2317 break;
2318 default:
2319 timeout = adev->video_timeout;
2320 break;
2321 }
2322
2323 args.timeout = timeout;
2324 args.credit_limit = ring->num_hw_submission;
2325 args.score = ring->sched_score;
2326 args.name = ring->name;
2327
2328 r = drm_sched_init(&ring->sched, &args);
2329 if (r) {
2330 dev_err(adev->dev,
2331 "Failed to create scheduler on ring %s.\n",
2332 ring->name);
2333 return r;
2334 }
2335 r = amdgpu_uvd_entity_init(adev, ring);
2336 if (r) {
2337 dev_err(adev->dev,
2338 "Failed to create UVD scheduling entity on ring %s.\n",
2339 ring->name);
2340 return r;
2341 }
2342 r = amdgpu_vce_entity_init(adev, ring);
2343 if (r) {
2344 dev_err(adev->dev,
2345 "Failed to create VCE scheduling entity on ring %s.\n",
2346 ring->name);
2347 return r;
2348 }
2349 }
2350
2351 if (adev->xcp_mgr)
2352 amdgpu_xcp_update_partition_sched_list(adev);
2353
2354 return 0;
2355}
2356
2357
2358/**
2359 * amdgpu_device_ip_init - run init for hardware IPs
2360 *
2361 * @adev: amdgpu_device pointer
2362 *
2363 * Main initialization pass for hardware IPs. The list of all the hardware
2364 * IPs that make up the asic is walked and the sw_init and hw_init callbacks
2365 * are run. sw_init initializes the software state associated with each IP
2366 * and hw_init initializes the hardware associated with each IP.
2367 * Returns 0 on success, negative error code on failure.
2368 */
2369static int amdgpu_device_ip_init(struct amdgpu_device *adev)
2370{
2371 bool init_badpage;
2372 int i, r;
2373
2374 r = amdgpu_ras_init(adev);
2375 if (r)
2376 return r;
2377
2378 for (i = 0; i < adev->num_ip_blocks; i++) {
2379 if (!adev->ip_blocks[i].status.valid)
2380 continue;
2381 if (adev->ip_blocks[i].version->funcs->sw_init) {
2382 r = adev->ip_blocks[i].version->funcs->sw_init(&adev->ip_blocks[i]);
2383 if (r) {
2384 dev_err(adev->dev,
2385 "sw_init of IP block <%s> failed %d\n",
2386 adev->ip_blocks[i].version->funcs->name,
2387 r);
2388 goto init_failed;
2389 }
2390 }
2391 adev->ip_blocks[i].status.sw = true;
2392
2393 if (!amdgpu_ip_member_of_hwini(
2394 adev, adev->ip_blocks[i].version->type))
2395 continue;
2396
2397 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON) {
2398 /* need to do common hw init early so everything is set up for gmc */
2399 r = adev->ip_blocks[i].version->funcs->hw_init(&adev->ip_blocks[i]);
2400 if (r) {
2401 dev_err(adev->dev, "hw_init %d failed %d\n", i,
2402 r);
2403 goto init_failed;
2404 }
2405 adev->ip_blocks[i].status.hw = true;
2406 } else if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) {
2407 /* need to do gmc hw init early so we can allocate gpu mem */
2408 /* Try to reserve bad pages early */
2409 if (amdgpu_sriov_vf(adev))
2410 amdgpu_virt_exchange_data(adev);
2411
2412 r = amdgpu_device_mem_scratch_init(adev);
2413 if (r) {
2414 dev_err(adev->dev,
2415 "amdgpu_mem_scratch_init failed %d\n",
2416 r);
2417 goto init_failed;
2418 }
2419 r = adev->ip_blocks[i].version->funcs->hw_init(&adev->ip_blocks[i]);
2420 if (r) {
2421 dev_err(adev->dev, "hw_init %d failed %d\n", i,
2422 r);
2423 goto init_failed;
2424 }
2425 r = amdgpu_wb_init(adev);
2426 if (r) {
2427 dev_err(adev->dev,
2428 "amdgpu_wb_init failed %d\n", r);
2429 goto init_failed;
2430 }
2431 adev->ip_blocks[i].status.hw = true;
2432
2433 /* right after GMC hw init, we create CSA */
2434 if (adev->gfx.mcbp) {
2435 r = amdgpu_allocate_static_csa(adev, &adev->virt.csa_obj,
2436 AMDGPU_GEM_DOMAIN_VRAM |
2437 AMDGPU_GEM_DOMAIN_GTT,
2438 AMDGPU_CSA_SIZE);
2439 if (r) {
2440 dev_err(adev->dev,
2441 "allocate CSA failed %d\n", r);
2442 goto init_failed;
2443 }
2444 }
2445
2446 r = amdgpu_seq64_init(adev);
2447 if (r) {
2448 dev_err(adev->dev, "allocate seq64 failed %d\n",
2449 r);
2450 goto init_failed;
2451 }
2452 }
2453 }
2454
2455 if (amdgpu_sriov_vf(adev))
2456 amdgpu_virt_init_data_exchange(adev);
2457
2458 r = amdgpu_ib_pool_init(adev);
2459 if (r) {
2460 dev_err(adev->dev, "IB initialization failed (%d).\n", r);
2461 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_IB_INIT_FAIL, 0, r);
2462 goto init_failed;
2463 }
2464
2465 r = amdgpu_ucode_create_bo(adev); /* create ucode bo when sw_init complete*/
2466 if (r)
2467 goto init_failed;
2468
2469 r = amdgpu_device_ip_hw_init_phase1(adev);
2470 if (r)
2471 goto init_failed;
2472
2473 r = amdgpu_device_fw_loading(adev);
2474 if (r)
2475 goto init_failed;
2476
2477 r = amdgpu_device_ip_hw_init_phase2(adev);
2478 if (r)
2479 goto init_failed;
2480
2481 /*
2482 * retired pages will be loaded from eeprom and reserved here,
2483 * it should be called after amdgpu_device_ip_hw_init_phase2 since
2484 * for some ASICs the RAS EEPROM code relies on SMU fully functioning
2485 * for I2C communication which only true at this point.
2486 *
2487 * amdgpu_ras_recovery_init may fail, but the upper only cares the
2488 * failure from bad gpu situation and stop amdgpu init process
2489 * accordingly. For other failed cases, it will still release all
2490 * the resource and print error message, rather than returning one
2491 * negative value to upper level.
2492 *
2493 * Note: theoretically, this should be called before all vram allocations
2494 * to protect retired page from abusing
2495 */
2496 init_badpage = (adev->init_lvl->level != AMDGPU_INIT_LEVEL_MINIMAL_XGMI);
2497 r = amdgpu_ras_recovery_init(adev, init_badpage);
2498 if (r)
2499 goto init_failed;
2500
2501 /**
2502 * In case of XGMI grab extra reference for reset domain for this device
2503 */
2504 if (adev->gmc.xgmi.num_physical_nodes > 1) {
2505 if (amdgpu_xgmi_add_device(adev) == 0) {
2506 if (!amdgpu_sriov_vf(adev)) {
2507 struct amdgpu_hive_info *hive = amdgpu_get_xgmi_hive(adev);
2508
2509 if (WARN_ON(!hive)) {
2510 r = -ENOENT;
2511 goto init_failed;
2512 }
2513
2514 if (!hive->reset_domain ||
2515 !amdgpu_reset_get_reset_domain(hive->reset_domain)) {
2516 r = -ENOENT;
2517 amdgpu_put_xgmi_hive(hive);
2518 goto init_failed;
2519 }
2520
2521 /* Drop the early temporary reset domain we created for device */
2522 amdgpu_reset_put_reset_domain(adev->reset_domain);
2523 adev->reset_domain = hive->reset_domain;
2524 amdgpu_put_xgmi_hive(hive);
2525 }
2526 }
2527 }
2528
2529 r = amdgpu_device_init_schedulers(adev);
2530 if (r)
2531 goto init_failed;
2532
2533 /* If SDMA is not brought up during hwini, the ttm buffer funcs enablement
2534 * is delayed after reset-on-init completes.
2535 */
2536 if (amdgpu_ip_member_of_hwini(adev, AMD_IP_BLOCK_TYPE_SDMA))
2537 amdgpu_ttm_enable_buffer_funcs(adev);
2538
2539 /* Don't init kfd if whole hive need to be reset during init */
2540 if (adev->init_lvl->level != AMDGPU_INIT_LEVEL_MINIMAL_XGMI) {
2541 amdgpu_amdkfd_device_init(adev);
2542 }
2543
2544 amdgpu_fru_get_product_info(adev);
2545
2546 r = amdgpu_cper_init(adev);
2547
2548init_failed:
2549
2550 return r;
2551}
2552
2553/**
2554 * amdgpu_device_fill_reset_magic - writes reset magic to gart pointer
2555 *
2556 * @adev: amdgpu_device pointer
2557 *
2558 * Writes a reset magic value to the gart pointer in VRAM. The driver calls
2559 * this function before a GPU reset. If the value is retained after a
2560 * GPU reset, VRAM has not been lost. Some GPU resets may destroy VRAM contents.
2561 */
2562static void amdgpu_device_fill_reset_magic(struct amdgpu_device *adev)
2563{
2564 memcpy(adev->reset_magic, adev->gart.ptr, AMDGPU_RESET_MAGIC_NUM);
2565}
2566
2567/**
2568 * amdgpu_device_check_vram_lost - check if vram is valid
2569 *
2570 * @adev: amdgpu_device pointer
2571 *
2572 * Checks the reset magic value written to the gart pointer in VRAM.
2573 * The driver calls this after a GPU reset to see if the contents of
2574 * VRAM is lost or now.
2575 * returns true if vram is lost, false if not.
2576 */
2577static bool amdgpu_device_check_vram_lost(struct amdgpu_device *adev)
2578{
2579 if (memcmp(adev->gart.ptr, adev->reset_magic,
2580 AMDGPU_RESET_MAGIC_NUM))
2581 return true;
2582
2583 if (!amdgpu_in_reset(adev))
2584 return false;
2585
2586 /*
2587 * For all ASICs with baco/mode1 reset, the VRAM is
2588 * always assumed to be lost.
2589 */
2590 switch (amdgpu_asic_reset_method(adev)) {
2591 case AMD_RESET_METHOD_LEGACY:
2592 case AMD_RESET_METHOD_LINK:
2593 case AMD_RESET_METHOD_BACO:
2594 case AMD_RESET_METHOD_MODE1:
2595 return true;
2596 default:
2597 return false;
2598 }
2599}
2600
2601/**
2602 * amdgpu_device_set_cg_state - set clockgating for amdgpu device
2603 *
2604 * @adev: amdgpu_device pointer
2605 * @state: clockgating state (gate or ungate)
2606 *
2607 * The list of all the hardware IPs that make up the asic is walked and the
2608 * set_clockgating_state callbacks are run.
2609 * Late initialization pass enabling clockgating for hardware IPs.
2610 * Fini or suspend, pass disabling clockgating for hardware IPs.
2611 * Returns 0 on success, negative error code on failure.
2612 */
2613
2614int amdgpu_device_set_cg_state(struct amdgpu_device *adev,
2615 enum amd_clockgating_state state)
2616{
2617 int i, j, r;
2618
2619 if (amdgpu_emu_mode == 1)
2620 return 0;
2621
2622 for (j = 0; j < adev->num_ip_blocks; j++) {
2623 i = state == AMD_CG_STATE_GATE ? j : adev->num_ip_blocks - j - 1;
2624 if (!adev->ip_blocks[i].status.late_initialized)
2625 continue;
2626 if (!adev->ip_blocks[i].version)
2627 continue;
2628 /* skip CG for GFX, SDMA on S0ix */
2629 if (adev->in_s0ix &&
2630 (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GFX ||
2631 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SDMA))
2632 continue;
2633 /* skip CG for VCE/UVD, it's handled specially */
2634 if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_UVD &&
2635 adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCE &&
2636 adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCN &&
2637 adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_JPEG &&
2638 adev->ip_blocks[i].version->funcs->set_clockgating_state) {
2639 /* enable clockgating to save power */
2640 r = adev->ip_blocks[i].version->funcs->set_clockgating_state(&adev->ip_blocks[i],
2641 state);
2642 if (r) {
2643 dev_err(adev->dev,
2644 "set_clockgating_state(gate) of IP block <%s> failed %d\n",
2645 adev->ip_blocks[i].version->funcs->name,
2646 r);
2647 return r;
2648 }
2649 }
2650 }
2651
2652 return 0;
2653}
2654
2655int amdgpu_device_set_pg_state(struct amdgpu_device *adev,
2656 enum amd_powergating_state state)
2657{
2658 int i, j, r;
2659
2660 if (amdgpu_emu_mode == 1)
2661 return 0;
2662
2663 for (j = 0; j < adev->num_ip_blocks; j++) {
2664 i = state == AMD_PG_STATE_GATE ? j : adev->num_ip_blocks - j - 1;
2665 if (!adev->ip_blocks[i].status.late_initialized)
2666 continue;
2667 if (!adev->ip_blocks[i].version)
2668 continue;
2669 /* skip PG for GFX, SDMA on S0ix */
2670 if (adev->in_s0ix &&
2671 (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GFX ||
2672 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SDMA))
2673 continue;
2674 /* skip CG for VCE/UVD, it's handled specially */
2675 if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_UVD &&
2676 adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCE &&
2677 adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCN &&
2678 adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_JPEG &&
2679 adev->ip_blocks[i].version->funcs->set_powergating_state) {
2680 /* enable powergating to save power */
2681 r = adev->ip_blocks[i].version->funcs->set_powergating_state(&adev->ip_blocks[i],
2682 state);
2683 if (r) {
2684 dev_err(adev->dev,
2685 "set_powergating_state(gate) of IP block <%s> failed %d\n",
2686 adev->ip_blocks[i].version->funcs->name,
2687 r);
2688 return r;
2689 }
2690 }
2691 }
2692 return 0;
2693}
2694
2695static int amdgpu_device_enable_mgpu_fan_boost(void)
2696{
2697 struct amdgpu_gpu_instance *gpu_ins;
2698 struct amdgpu_device *adev;
2699 int i, ret = 0;
2700
2701 mutex_lock(&mgpu_info.mutex);
2702
2703 /*
2704 * MGPU fan boost feature should be enabled
2705 * only when there are two or more dGPUs in
2706 * the system
2707 */
2708 if (mgpu_info.num_dgpu < 2)
2709 goto out;
2710
2711 for (i = 0; i < mgpu_info.num_dgpu; i++) {
2712 gpu_ins = &(mgpu_info.gpu_ins[i]);
2713 adev = gpu_ins->adev;
2714 if (!(adev->flags & AMD_IS_APU || amdgpu_sriov_multi_vf_mode(adev)) &&
2715 !gpu_ins->mgpu_fan_enabled) {
2716 ret = amdgpu_dpm_enable_mgpu_fan_boost(adev);
2717 if (ret)
2718 break;
2719
2720 gpu_ins->mgpu_fan_enabled = 1;
2721 }
2722 }
2723
2724out:
2725 mutex_unlock(&mgpu_info.mutex);
2726
2727 return ret;
2728}
2729
2730/**
2731 * amdgpu_device_ip_late_init - run late init for hardware IPs
2732 *
2733 * @adev: amdgpu_device pointer
2734 *
2735 * Late initialization pass for hardware IPs. The list of all the hardware
2736 * IPs that make up the asic is walked and the late_init callbacks are run.
2737 * late_init covers any special initialization that an IP requires
2738 * after all of the have been initialized or something that needs to happen
2739 * late in the init process.
2740 * Returns 0 on success, negative error code on failure.
2741 */
2742static int amdgpu_device_ip_late_init(struct amdgpu_device *adev)
2743{
2744 struct amdgpu_gpu_instance *gpu_instance;
2745 int i = 0, r;
2746
2747 for (i = 0; i < adev->num_ip_blocks; i++) {
2748 if (!adev->ip_blocks[i].status.hw)
2749 continue;
2750 if (adev->ip_blocks[i].version->funcs->late_init) {
2751 r = adev->ip_blocks[i].version->funcs->late_init(&adev->ip_blocks[i]);
2752 if (r) {
2753 dev_err(adev->dev,
2754 "late_init of IP block <%s> failed %d\n",
2755 adev->ip_blocks[i].version->funcs->name,
2756 r);
2757 return r;
2758 }
2759 }
2760 adev->ip_blocks[i].status.late_initialized = true;
2761 }
2762
2763 r = amdgpu_ras_late_init(adev);
2764 if (r) {
2765 dev_err(adev->dev, "amdgpu_ras_late_init failed %d", r);
2766 return r;
2767 }
2768
2769 if (!amdgpu_reset_in_recovery(adev))
2770 amdgpu_ras_set_error_query_ready(adev, true);
2771
2772 amdgpu_device_set_cg_state(adev, AMD_CG_STATE_GATE);
2773 amdgpu_device_set_pg_state(adev, AMD_PG_STATE_GATE);
2774
2775 amdgpu_device_fill_reset_magic(adev);
2776
2777 r = amdgpu_device_enable_mgpu_fan_boost();
2778 if (r)
2779 dev_err(adev->dev, "enable mgpu fan boost failed (%d).\n", r);
2780
2781 /* For passthrough configuration on arcturus and aldebaran, enable special handling SBR */
2782 if (amdgpu_passthrough(adev) &&
2783 ((adev->asic_type == CHIP_ARCTURUS && adev->gmc.xgmi.num_physical_nodes > 1) ||
2784 adev->asic_type == CHIP_ALDEBARAN))
2785 amdgpu_dpm_handle_passthrough_sbr(adev, true);
2786
2787 if (adev->gmc.xgmi.num_physical_nodes > 1) {
2788 mutex_lock(&mgpu_info.mutex);
2789
2790 /*
2791 * Reset device p-state to low as this was booted with high.
2792 *
2793 * This should be performed only after all devices from the same
2794 * hive get initialized.
2795 *
2796 * However, it's unknown how many device in the hive in advance.
2797 * As this is counted one by one during devices initializations.
2798 *
2799 * So, we wait for all XGMI interlinked devices initialized.
2800 * This may bring some delays as those devices may come from
2801 * different hives. But that should be OK.
2802 */
2803 if (mgpu_info.num_dgpu == adev->gmc.xgmi.num_physical_nodes) {
2804 for (i = 0; i < mgpu_info.num_gpu; i++) {
2805 gpu_instance = &(mgpu_info.gpu_ins[i]);
2806 if (gpu_instance->adev->flags & AMD_IS_APU)
2807 continue;
2808
2809 r = amdgpu_xgmi_set_pstate(gpu_instance->adev,
2810 AMDGPU_XGMI_PSTATE_MIN);
2811 if (r) {
2812 dev_err(adev->dev,
2813 "pstate setting failed (%d).\n",
2814 r);
2815 break;
2816 }
2817 }
2818 }
2819
2820 mutex_unlock(&mgpu_info.mutex);
2821 }
2822
2823 return 0;
2824}
2825
2826static void amdgpu_ip_block_hw_fini(struct amdgpu_ip_block *ip_block)
2827{
2828 struct amdgpu_device *adev = ip_block->adev;
2829 int r;
2830
2831 if (!ip_block->version->funcs->hw_fini) {
2832 dev_err(adev->dev, "hw_fini of IP block <%s> not defined\n",
2833 ip_block->version->funcs->name);
2834 } else {
2835 r = ip_block->version->funcs->hw_fini(ip_block);
2836 /* XXX handle errors */
2837 if (r) {
2838 dev_dbg(adev->dev,
2839 "hw_fini of IP block <%s> failed %d\n",
2840 ip_block->version->funcs->name, r);
2841 }
2842 }
2843
2844 ip_block->status.hw = false;
2845}
2846
2847/**
2848 * amdgpu_device_smu_fini_early - smu hw_fini wrapper
2849 *
2850 * @adev: amdgpu_device pointer
2851 *
2852 * For ASICs need to disable SMC first
2853 */
2854static void amdgpu_device_smu_fini_early(struct amdgpu_device *adev)
2855{
2856 int i;
2857
2858 if (amdgpu_ip_version(adev, GC_HWIP, 0) > IP_VERSION(9, 0, 0))
2859 return;
2860
2861 for (i = 0; i < adev->num_ip_blocks; i++) {
2862 if (!adev->ip_blocks[i].status.hw)
2863 continue;
2864 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) {
2865 amdgpu_ip_block_hw_fini(&adev->ip_blocks[i]);
2866 break;
2867 }
2868 }
2869}
2870
2871static int amdgpu_device_ip_fini_early(struct amdgpu_device *adev)
2872{
2873 int i, r;
2874
2875 for (i = 0; i < adev->num_ip_blocks; i++) {
2876 if (!adev->ip_blocks[i].version)
2877 continue;
2878 if (!adev->ip_blocks[i].version->funcs->early_fini)
2879 continue;
2880
2881 r = adev->ip_blocks[i].version->funcs->early_fini(&adev->ip_blocks[i]);
2882 if (r) {
2883 dev_dbg(adev->dev,
2884 "early_fini of IP block <%s> failed %d\n",
2885 adev->ip_blocks[i].version->funcs->name, r);
2886 }
2887 }
2888
2889 amdgpu_amdkfd_suspend(adev, true);
2890 amdgpu_amdkfd_teardown_processes(adev);
2891 amdgpu_userq_suspend(adev);
2892
2893 /* Workaround for ASICs need to disable SMC first */
2894 amdgpu_device_smu_fini_early(adev);
2895
2896 for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
2897 if (!adev->ip_blocks[i].status.hw)
2898 continue;
2899
2900 amdgpu_ip_block_hw_fini(&adev->ip_blocks[i]);
2901 }
2902
2903 if (amdgpu_sriov_vf(adev)) {
2904 if (amdgpu_virt_release_full_gpu(adev, false))
2905 dev_err(adev->dev,
2906 "failed to release exclusive mode on fini\n");
2907 }
2908
2909 /*
2910 * Driver reload on the APU can fail due to firmware validation because
2911 * the PSP is always running, as it is shared across the whole SoC.
2912 * This same issue does not occur on dGPU because it has a mechanism
2913 * that checks whether the PSP is running. A solution for those issues
2914 * in the APU is to trigger a GPU reset, but this should be done during
2915 * the unload phase to avoid adding boot latency and screen flicker.
2916 * GFX V11 has GC block as default off IP. Every time AMDGPU driver sends
2917 * a request to PMFW to unload MP1, PMFW will put GC in reset and power down
2918 * the voltage. Hence, skipping reset for APUs with GFX V11 or later.
2919 */
2920 if ((adev->flags & AMD_IS_APU) && !adev->gmc.is_app_apu &&
2921 amdgpu_ip_version(adev, GC_HWIP, 0) < IP_VERSION(11, 0, 0)) {
2922 r = amdgpu_asic_reset(adev);
2923 if (r)
2924 dev_err(adev->dev, "asic reset on %s failed\n", __func__);
2925 }
2926
2927 return 0;
2928}
2929
2930/**
2931 * amdgpu_device_ip_fini - run fini for hardware IPs
2932 *
2933 * @adev: amdgpu_device pointer
2934 *
2935 * Main teardown pass for hardware IPs. The list of all the hardware
2936 * IPs that make up the asic is walked and the hw_fini and sw_fini callbacks
2937 * are run. hw_fini tears down the hardware associated with each IP
2938 * and sw_fini tears down any software state associated with each IP.
2939 * Returns 0 on success, negative error code on failure.
2940 */
2941static int amdgpu_device_ip_fini(struct amdgpu_device *adev)
2942{
2943 int i, r;
2944
2945 amdgpu_cper_fini(adev);
2946
2947 if (amdgpu_sriov_vf(adev) && adev->virt.ras_init_done)
2948 amdgpu_virt_release_ras_err_handler_data(adev);
2949
2950 if (adev->gmc.xgmi.num_physical_nodes > 1)
2951 amdgpu_xgmi_remove_device(adev);
2952
2953 amdgpu_amdkfd_device_fini_sw(adev);
2954
2955 for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
2956 if (!adev->ip_blocks[i].status.sw)
2957 continue;
2958
2959 if (!adev->ip_blocks[i].version)
2960 continue;
2961 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) {
2962 amdgpu_ucode_free_bo(adev);
2963 amdgpu_free_static_csa(&adev->virt.csa_obj);
2964 amdgpu_wb_fini(adev);
2965 amdgpu_device_mem_scratch_fini(adev);
2966 amdgpu_ib_pool_fini(adev);
2967 amdgpu_seq64_fini(adev);
2968 amdgpu_doorbell_fini(adev);
2969 }
2970 if (adev->ip_blocks[i].version->funcs->sw_fini) {
2971 r = adev->ip_blocks[i].version->funcs->sw_fini(&adev->ip_blocks[i]);
2972 /* XXX handle errors */
2973 if (r) {
2974 dev_dbg(adev->dev,
2975 "sw_fini of IP block <%s> failed %d\n",
2976 adev->ip_blocks[i].version->funcs->name,
2977 r);
2978 }
2979 }
2980 adev->ip_blocks[i].status.sw = false;
2981 adev->ip_blocks[i].status.valid = false;
2982 }
2983
2984 for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
2985 if (!adev->ip_blocks[i].status.late_initialized)
2986 continue;
2987 if (!adev->ip_blocks[i].version)
2988 continue;
2989 if (adev->ip_blocks[i].version->funcs->late_fini)
2990 adev->ip_blocks[i].version->funcs->late_fini(&adev->ip_blocks[i]);
2991 adev->ip_blocks[i].status.late_initialized = false;
2992 }
2993
2994 amdgpu_ras_fini(adev);
2995 amdgpu_uid_fini(adev);
2996
2997 return 0;
2998}
2999
3000/**
3001 * amdgpu_device_delayed_init_work_handler - work handler for IB tests
3002 *
3003 * @work: work_struct.
3004 */
3005static void amdgpu_device_delayed_init_work_handler(struct work_struct *work)
3006{
3007 struct amdgpu_device *adev =
3008 container_of(work, struct amdgpu_device, delayed_init_work.work);
3009 int r;
3010
3011 r = amdgpu_ib_ring_tests(adev);
3012 if (r)
3013 dev_err(adev->dev, "ib ring test failed (%d).\n", r);
3014}
3015
3016static void amdgpu_device_delay_enable_gfx_off(struct work_struct *work)
3017{
3018 struct amdgpu_device *adev =
3019 container_of(work, struct amdgpu_device, gfx.gfx_off_delay_work.work);
3020
3021 WARN_ON_ONCE(adev->gfx.gfx_off_state);
3022 WARN_ON_ONCE(adev->gfx.gfx_off_req_count);
3023
3024 if (!amdgpu_dpm_set_powergating_by_smu(adev, AMD_IP_BLOCK_TYPE_GFX, true, 0))
3025 adev->gfx.gfx_off_state = true;
3026}
3027
3028/**
3029 * amdgpu_device_ip_suspend_phase1 - run suspend for hardware IPs (phase 1)
3030 *
3031 * @adev: amdgpu_device pointer
3032 *
3033 * Main suspend function for hardware IPs. The list of all the hardware
3034 * IPs that make up the asic is walked, clockgating is disabled and the
3035 * suspend callbacks are run. suspend puts the hardware and software state
3036 * in each IP into a state suitable for suspend.
3037 * Returns 0 on success, negative error code on failure.
3038 */
3039static int amdgpu_device_ip_suspend_phase1(struct amdgpu_device *adev)
3040{
3041 int i, r, rec;
3042
3043 amdgpu_device_set_pg_state(adev, AMD_PG_STATE_UNGATE);
3044 amdgpu_device_set_cg_state(adev, AMD_CG_STATE_UNGATE);
3045
3046 /*
3047 * Per PMFW team's suggestion, driver needs to handle gfxoff
3048 * and df cstate features disablement for gpu reset(e.g. Mode1Reset)
3049 * scenario. Add the missing df cstate disablement here.
3050 */
3051 if (amdgpu_dpm_set_df_cstate(adev, DF_CSTATE_DISALLOW))
3052 dev_warn(adev->dev, "Failed to disallow df cstate");
3053
3054 for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
3055 if (!adev->ip_blocks[i].status.valid)
3056 continue;
3057
3058 /* displays are handled separately */
3059 if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_DCE)
3060 continue;
3061
3062 r = amdgpu_ip_block_suspend(&adev->ip_blocks[i]);
3063 if (r)
3064 goto unwind;
3065 }
3066
3067 return 0;
3068unwind:
3069 rec = amdgpu_device_ip_resume_phase3(adev);
3070 if (rec)
3071 dev_err(adev->dev,
3072 "amdgpu_device_ip_resume_phase3 failed during unwind: %d\n",
3073 rec);
3074
3075 amdgpu_dpm_set_df_cstate(adev, DF_CSTATE_ALLOW);
3076
3077 amdgpu_device_set_pg_state(adev, AMD_PG_STATE_GATE);
3078 amdgpu_device_set_cg_state(adev, AMD_CG_STATE_GATE);
3079
3080 return r;
3081}
3082
3083/**
3084 * amdgpu_device_ip_suspend_phase2 - run suspend for hardware IPs (phase 2)
3085 *
3086 * @adev: amdgpu_device pointer
3087 *
3088 * Main suspend function for hardware IPs. The list of all the hardware
3089 * IPs that make up the asic is walked, clockgating is disabled and the
3090 * suspend callbacks are run. suspend puts the hardware and software state
3091 * in each IP into a state suitable for suspend.
3092 * Returns 0 on success, negative error code on failure.
3093 */
3094static int amdgpu_device_ip_suspend_phase2(struct amdgpu_device *adev)
3095{
3096 int i, r, rec;
3097
3098 if (adev->in_s0ix)
3099 amdgpu_dpm_gfx_state_change(adev, sGpuChangeState_D3Entry);
3100
3101 for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
3102 if (!adev->ip_blocks[i].status.valid || !adev->ip_blocks[i].status.hw)
3103 continue;
3104 /* displays are handled in phase1 */
3105 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_DCE)
3106 continue;
3107 /* PSP lost connection when err_event_athub occurs */
3108 if (amdgpu_ras_intr_triggered() &&
3109 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP) {
3110 adev->ip_blocks[i].status.hw = false;
3111 continue;
3112 }
3113
3114 /* skip unnecessary suspend if we do not initialize them yet */
3115 if (!amdgpu_ip_member_of_hwini(
3116 adev, adev->ip_blocks[i].version->type))
3117 continue;
3118
3119 /* Since we skip suspend for S0i3, we need to cancel the delayed
3120 * idle work here as the suspend callback never gets called.
3121 */
3122 if (adev->in_s0ix &&
3123 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GFX &&
3124 amdgpu_ip_version(adev, GC_HWIP, 0) >= IP_VERSION(10, 0, 0))
3125 cancel_delayed_work_sync(&adev->gfx.idle_work);
3126 /* skip suspend of gfx/mes and psp for S0ix
3127 * gfx is in gfxoff state, so on resume it will exit gfxoff just
3128 * like at runtime. PSP is also part of the always on hardware
3129 * so no need to suspend it.
3130 */
3131 if (adev->in_s0ix &&
3132 (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP ||
3133 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GFX ||
3134 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_MES))
3135 continue;
3136
3137 /* SDMA 5.x+ is part of GFX power domain so it's covered by GFXOFF */
3138 if (adev->in_s0ix &&
3139 (amdgpu_ip_version(adev, SDMA0_HWIP, 0) >=
3140 IP_VERSION(5, 0, 0)) &&
3141 (adev->ip_blocks[i].version->type ==
3142 AMD_IP_BLOCK_TYPE_SDMA))
3143 continue;
3144
3145 /* Once swPSP provides the IMU, RLC FW binaries to TOS during cold-boot.
3146 * These are in TMR, hence are expected to be reused by PSP-TOS to reload
3147 * from this location and RLC Autoload automatically also gets loaded
3148 * from here based on PMFW -> PSP message during re-init sequence.
3149 * Therefore, the psp suspend & resume should be skipped to avoid destroy
3150 * the TMR and reload FWs again for IMU enabled APU ASICs.
3151 */
3152 if (amdgpu_in_reset(adev) &&
3153 (adev->flags & AMD_IS_APU) && adev->gfx.imu.funcs &&
3154 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP)
3155 continue;
3156
3157 r = amdgpu_ip_block_suspend(&adev->ip_blocks[i]);
3158 if (r)
3159 goto unwind;
3160
3161 /* handle putting the SMC in the appropriate state */
3162 if (!amdgpu_sriov_vf(adev)) {
3163 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) {
3164 r = amdgpu_dpm_set_mp1_state(adev, adev->mp1_state);
3165 if (r) {
3166 dev_err(adev->dev,
3167 "SMC failed to set mp1 state %d, %d\n",
3168 adev->mp1_state, r);
3169 goto unwind;
3170 }
3171 }
3172 }
3173 }
3174
3175 return 0;
3176unwind:
3177 /* suspend phase 2 = resume phase 1 + resume phase 2 */
3178 rec = amdgpu_device_ip_resume_phase1(adev);
3179 if (rec) {
3180 dev_err(adev->dev,
3181 "amdgpu_device_ip_resume_phase1 failed during unwind: %d\n",
3182 rec);
3183 return r;
3184 }
3185
3186 rec = amdgpu_device_fw_loading(adev);
3187 if (rec) {
3188 dev_err(adev->dev,
3189 "amdgpu_device_fw_loading failed during unwind: %d\n",
3190 rec);
3191 return r;
3192 }
3193
3194 rec = amdgpu_device_ip_resume_phase2(adev);
3195 if (rec) {
3196 dev_err(adev->dev,
3197 "amdgpu_device_ip_resume_phase2 failed during unwind: %d\n",
3198 rec);
3199 return r;
3200 }
3201
3202 return r;
3203}
3204
3205/**
3206 * amdgpu_device_ip_suspend - run suspend for hardware IPs
3207 *
3208 * @adev: amdgpu_device pointer
3209 *
3210 * Main suspend function for hardware IPs. The list of all the hardware
3211 * IPs that make up the asic is walked, clockgating is disabled and the
3212 * suspend callbacks are run. suspend puts the hardware and software state
3213 * in each IP into a state suitable for suspend.
3214 * Returns 0 on success, negative error code on failure.
3215 */
3216static int amdgpu_device_ip_suspend(struct amdgpu_device *adev)
3217{
3218 int r;
3219
3220 if (amdgpu_sriov_vf(adev)) {
3221 amdgpu_virt_fini_data_exchange(adev);
3222 amdgpu_virt_request_full_gpu(adev, false);
3223 }
3224
3225 r = amdgpu_device_ip_suspend_phase1(adev);
3226 if (r)
3227 return r;
3228 r = amdgpu_device_ip_suspend_phase2(adev);
3229
3230 if (amdgpu_sriov_vf(adev))
3231 amdgpu_virt_release_full_gpu(adev, false);
3232
3233 return r;
3234}
3235
3236static int amdgpu_device_ip_reinit_early_sriov(struct amdgpu_device *adev)
3237{
3238 int i, r;
3239
3240 static enum amd_ip_block_type ip_order[] = {
3241 AMD_IP_BLOCK_TYPE_COMMON,
3242 AMD_IP_BLOCK_TYPE_GMC,
3243 AMD_IP_BLOCK_TYPE_PSP,
3244 AMD_IP_BLOCK_TYPE_IH,
3245 };
3246
3247 for (i = 0; i < adev->num_ip_blocks; i++) {
3248 int j;
3249 struct amdgpu_ip_block *block;
3250
3251 block = &adev->ip_blocks[i];
3252 block->status.hw = false;
3253
3254 for (j = 0; j < ARRAY_SIZE(ip_order); j++) {
3255
3256 if (block->version->type != ip_order[j] ||
3257 !block->status.valid)
3258 continue;
3259
3260 r = block->version->funcs->hw_init(&adev->ip_blocks[i]);
3261 if (r) {
3262 dev_err(adev->dev, "RE-INIT-early: %s failed\n",
3263 block->version->funcs->name);
3264 return r;
3265 }
3266 block->status.hw = true;
3267 }
3268 }
3269
3270 return 0;
3271}
3272
3273static int amdgpu_device_ip_reinit_late_sriov(struct amdgpu_device *adev)
3274{
3275 struct amdgpu_ip_block *block;
3276 int i, r = 0;
3277
3278 static enum amd_ip_block_type ip_order[] = {
3279 AMD_IP_BLOCK_TYPE_SMC,
3280 AMD_IP_BLOCK_TYPE_DCE,
3281 AMD_IP_BLOCK_TYPE_GFX,
3282 AMD_IP_BLOCK_TYPE_SDMA,
3283 AMD_IP_BLOCK_TYPE_MES,
3284 AMD_IP_BLOCK_TYPE_UVD,
3285 AMD_IP_BLOCK_TYPE_VCE,
3286 AMD_IP_BLOCK_TYPE_VCN,
3287 AMD_IP_BLOCK_TYPE_JPEG
3288 };
3289
3290 for (i = 0; i < ARRAY_SIZE(ip_order); i++) {
3291 block = amdgpu_device_ip_get_ip_block(adev, ip_order[i]);
3292
3293 if (!block)
3294 continue;
3295
3296 if (block->status.valid && !block->status.hw) {
3297 if (block->version->type == AMD_IP_BLOCK_TYPE_SMC) {
3298 r = amdgpu_ip_block_resume(block);
3299 } else {
3300 r = block->version->funcs->hw_init(block);
3301 }
3302
3303 if (r) {
3304 dev_err(adev->dev, "RE-INIT-late: %s failed\n",
3305 block->version->funcs->name);
3306 break;
3307 }
3308 block->status.hw = true;
3309 }
3310 }
3311
3312 return r;
3313}
3314
3315/**
3316 * amdgpu_device_ip_resume_phase1 - run resume for hardware IPs
3317 *
3318 * @adev: amdgpu_device pointer
3319 *
3320 * First resume function for hardware IPs. The list of all the hardware
3321 * IPs that make up the asic is walked and the resume callbacks are run for
3322 * COMMON, GMC, and IH. resume puts the hardware into a functional state
3323 * after a suspend and updates the software state as necessary. This
3324 * function is also used for restoring the GPU after a GPU reset.
3325 * Returns 0 on success, negative error code on failure.
3326 */
3327static int amdgpu_device_ip_resume_phase1(struct amdgpu_device *adev)
3328{
3329 int i, r;
3330
3331 for (i = 0; i < adev->num_ip_blocks; i++) {
3332 if (!adev->ip_blocks[i].status.valid || adev->ip_blocks[i].status.hw)
3333 continue;
3334 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
3335 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC ||
3336 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH ||
3337 (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP && amdgpu_sriov_vf(adev))) {
3338
3339 r = amdgpu_ip_block_resume(&adev->ip_blocks[i]);
3340 if (r)
3341 return r;
3342 }
3343 }
3344
3345 return 0;
3346}
3347
3348/**
3349 * amdgpu_device_ip_resume_phase2 - run resume for hardware IPs
3350 *
3351 * @adev: amdgpu_device pointer
3352 *
3353 * Second resume function for hardware IPs. The list of all the hardware
3354 * IPs that make up the asic is walked and the resume callbacks are run for
3355 * all blocks except COMMON, GMC, and IH. resume puts the hardware into a
3356 * functional state after a suspend and updates the software state as
3357 * necessary. This function is also used for restoring the GPU after a GPU
3358 * reset.
3359 * Returns 0 on success, negative error code on failure.
3360 */
3361static int amdgpu_device_ip_resume_phase2(struct amdgpu_device *adev)
3362{
3363 int i, r;
3364
3365 for (i = 0; i < adev->num_ip_blocks; i++) {
3366 if (!adev->ip_blocks[i].status.valid || adev->ip_blocks[i].status.hw)
3367 continue;
3368 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
3369 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC ||
3370 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH ||
3371 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_DCE ||
3372 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP)
3373 continue;
3374 r = amdgpu_ip_block_resume(&adev->ip_blocks[i]);
3375 if (r)
3376 return r;
3377 }
3378
3379 return 0;
3380}
3381
3382/**
3383 * amdgpu_device_ip_resume_phase3 - run resume for hardware IPs
3384 *
3385 * @adev: amdgpu_device pointer
3386 *
3387 * Third resume function for hardware IPs. The list of all the hardware
3388 * IPs that make up the asic is walked and the resume callbacks are run for
3389 * all DCE. resume puts the hardware into a functional state after a suspend
3390 * and updates the software state as necessary. This function is also used
3391 * for restoring the GPU after a GPU reset.
3392 *
3393 * Returns 0 on success, negative error code on failure.
3394 */
3395static int amdgpu_device_ip_resume_phase3(struct amdgpu_device *adev)
3396{
3397 int i, r;
3398
3399 for (i = 0; i < adev->num_ip_blocks; i++) {
3400 if (!adev->ip_blocks[i].status.valid || adev->ip_blocks[i].status.hw)
3401 continue;
3402 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_DCE) {
3403 r = amdgpu_ip_block_resume(&adev->ip_blocks[i]);
3404 if (r)
3405 return r;
3406 }
3407 }
3408
3409 return 0;
3410}
3411
3412/**
3413 * amdgpu_device_ip_resume - run resume for hardware IPs
3414 *
3415 * @adev: amdgpu_device pointer
3416 *
3417 * Main resume function for hardware IPs. The hardware IPs
3418 * are split into two resume functions because they are
3419 * also used in recovering from a GPU reset and some additional
3420 * steps need to be take between them. In this case (S3/S4) they are
3421 * run sequentially.
3422 * Returns 0 on success, negative error code on failure.
3423 */
3424static int amdgpu_device_ip_resume(struct amdgpu_device *adev)
3425{
3426 int r;
3427
3428 r = amdgpu_device_ip_resume_phase1(adev);
3429 if (r)
3430 return r;
3431
3432 r = amdgpu_device_fw_loading(adev);
3433 if (r)
3434 return r;
3435
3436 r = amdgpu_device_ip_resume_phase2(adev);
3437
3438 amdgpu_ttm_enable_buffer_funcs(adev);
3439
3440 if (r)
3441 return r;
3442
3443 amdgpu_fence_driver_hw_init(adev);
3444
3445 r = amdgpu_device_ip_resume_phase3(adev);
3446
3447 return r;
3448}
3449
3450/**
3451 * amdgpu_device_detect_sriov_bios - determine if the board supports SR-IOV
3452 *
3453 * @adev: amdgpu_device pointer
3454 *
3455 * Query the VBIOS data tables to determine if the board supports SR-IOV.
3456 */
3457static void amdgpu_device_detect_sriov_bios(struct amdgpu_device *adev)
3458{
3459 if (amdgpu_sriov_vf(adev)) {
3460 if (adev->is_atom_fw) {
3461 if (amdgpu_atomfirmware_gpu_virtualization_supported(adev))
3462 adev->virt.caps |= AMDGPU_SRIOV_CAPS_SRIOV_VBIOS;
3463 } else {
3464 if (amdgpu_atombios_has_gpu_virtualization_table(adev))
3465 adev->virt.caps |= AMDGPU_SRIOV_CAPS_SRIOV_VBIOS;
3466 }
3467
3468 if (!(adev->virt.caps & AMDGPU_SRIOV_CAPS_SRIOV_VBIOS))
3469 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_NO_VBIOS, 0, 0);
3470 }
3471}
3472
3473/**
3474 * amdgpu_device_asic_has_dc_support - determine if DC supports the asic
3475 *
3476 * @pdev : pci device context
3477 * @asic_type: AMD asic type
3478 *
3479 * Check if there is DC (new modesetting infrastructre) support for an asic.
3480 * returns true if DC has support, false if not.
3481 */
3482bool amdgpu_device_asic_has_dc_support(struct pci_dev *pdev,
3483 enum amd_asic_type asic_type)
3484{
3485 switch (asic_type) {
3486#ifdef CONFIG_DRM_AMDGPU_SI
3487 case CHIP_HAINAN:
3488#endif
3489 case CHIP_TOPAZ:
3490 /* chips with no display hardware */
3491 return false;
3492#if defined(CONFIG_DRM_AMD_DC)
3493 case CHIP_TAHITI:
3494 case CHIP_PITCAIRN:
3495 case CHIP_VERDE:
3496 case CHIP_OLAND:
3497 return amdgpu_dc != 0 && IS_ENABLED(CONFIG_DRM_AMD_DC_SI);
3498 default:
3499 return amdgpu_dc != 0;
3500#else
3501 default:
3502 if (amdgpu_dc > 0)
3503 dev_info_once(
3504 &pdev->dev,
3505 "Display Core has been requested via kernel parameter but isn't supported by ASIC, ignoring\n");
3506 return false;
3507#endif
3508 }
3509}
3510
3511/**
3512 * amdgpu_device_has_dc_support - check if dc is supported
3513 *
3514 * @adev: amdgpu_device pointer
3515 *
3516 * Returns true for supported, false for not supported
3517 */
3518bool amdgpu_device_has_dc_support(struct amdgpu_device *adev)
3519{
3520 if (adev->enable_virtual_display ||
3521 (adev->harvest_ip_mask & AMD_HARVEST_IP_DMU_MASK))
3522 return false;
3523
3524 return amdgpu_device_asic_has_dc_support(adev->pdev, adev->asic_type);
3525}
3526
3527static void amdgpu_device_xgmi_reset_func(struct work_struct *__work)
3528{
3529 struct amdgpu_device *adev =
3530 container_of(__work, struct amdgpu_device, xgmi_reset_work);
3531 struct amdgpu_hive_info *hive = amdgpu_get_xgmi_hive(adev);
3532
3533 /* It's a bug to not have a hive within this function */
3534 if (WARN_ON(!hive))
3535 return;
3536
3537 /*
3538 * Use task barrier to synchronize all xgmi reset works across the
3539 * hive. task_barrier_enter and task_barrier_exit will block
3540 * until all the threads running the xgmi reset works reach
3541 * those points. task_barrier_full will do both blocks.
3542 */
3543 if (amdgpu_asic_reset_method(adev) == AMD_RESET_METHOD_BACO) {
3544
3545 task_barrier_enter(&hive->tb);
3546 adev->asic_reset_res = amdgpu_device_baco_enter(adev);
3547
3548 if (adev->asic_reset_res)
3549 goto fail;
3550
3551 task_barrier_exit(&hive->tb);
3552 adev->asic_reset_res = amdgpu_device_baco_exit(adev);
3553
3554 if (adev->asic_reset_res)
3555 goto fail;
3556
3557 amdgpu_ras_reset_error_count(adev, AMDGPU_RAS_BLOCK__MMHUB);
3558 } else {
3559
3560 task_barrier_full(&hive->tb);
3561 adev->asic_reset_res = amdgpu_asic_reset(adev);
3562 }
3563
3564fail:
3565 if (adev->asic_reset_res)
3566 dev_warn(adev->dev,
3567 "ASIC reset failed with error, %d for drm dev, %s",
3568 adev->asic_reset_res, adev_to_drm(adev)->unique);
3569 amdgpu_put_xgmi_hive(hive);
3570}
3571
3572static int amdgpu_device_get_job_timeout_settings(struct amdgpu_device *adev)
3573{
3574 char buf[AMDGPU_MAX_TIMEOUT_PARAM_LENGTH];
3575 char *input = buf;
3576 char *timeout_setting = NULL;
3577 int index = 0;
3578 long timeout;
3579 int ret = 0;
3580
3581 /* By default timeout for all queues is 2 sec */
3582 adev->gfx_timeout = adev->compute_timeout = adev->sdma_timeout =
3583 adev->video_timeout = msecs_to_jiffies(2000);
3584
3585 if (!strnlen(amdgpu_lockup_timeout, AMDGPU_MAX_TIMEOUT_PARAM_LENGTH))
3586 return 0;
3587
3588 /*
3589 * strsep() destructively modifies its input by replacing delimiters
3590 * with '\0'. Use a stack copy so the global module parameter buffer
3591 * remains intact for multi-GPU systems where this function is called
3592 * once per device.
3593 */
3594 strscpy(buf, amdgpu_lockup_timeout, sizeof(buf));
3595
3596 while ((timeout_setting = strsep(&input, ",")) &&
3597 strnlen(timeout_setting, AMDGPU_MAX_TIMEOUT_PARAM_LENGTH)) {
3598 ret = kstrtol(timeout_setting, 0, &timeout);
3599 if (ret)
3600 return ret;
3601
3602 if (timeout == 0) {
3603 index++;
3604 continue;
3605 } else if (timeout < 0) {
3606 timeout = MAX_SCHEDULE_TIMEOUT;
3607 dev_warn(adev->dev, "lockup timeout disabled");
3608 add_taint(TAINT_SOFTLOCKUP, LOCKDEP_STILL_OK);
3609 } else {
3610 timeout = msecs_to_jiffies(timeout);
3611 }
3612
3613 switch (index++) {
3614 case 0:
3615 adev->gfx_timeout = timeout;
3616 break;
3617 case 1:
3618 adev->compute_timeout = timeout;
3619 break;
3620 case 2:
3621 adev->sdma_timeout = timeout;
3622 break;
3623 case 3:
3624 adev->video_timeout = timeout;
3625 break;
3626 default:
3627 break;
3628 }
3629 }
3630
3631 /* When only one value specified apply it to all queues. */
3632 if (index == 1)
3633 adev->gfx_timeout = adev->compute_timeout = adev->sdma_timeout =
3634 adev->video_timeout = timeout;
3635
3636 return ret;
3637}
3638
3639/**
3640 * amdgpu_device_check_iommu_direct_map - check if RAM direct mapped to GPU
3641 *
3642 * @adev: amdgpu_device pointer
3643 *
3644 * RAM direct mapped to GPU if IOMMU is not enabled or is pass through mode
3645 */
3646static void amdgpu_device_check_iommu_direct_map(struct amdgpu_device *adev)
3647{
3648 struct iommu_domain *domain;
3649
3650 domain = iommu_get_domain_for_dev(adev->dev);
3651 if (!domain || domain->type == IOMMU_DOMAIN_IDENTITY)
3652 adev->ram_is_direct_mapped = true;
3653}
3654
3655#if defined(CONFIG_HSA_AMD_P2P)
3656/**
3657 * amdgpu_device_check_iommu_remap - Check if DMA remapping is enabled.
3658 *
3659 * @adev: amdgpu_device pointer
3660 *
3661 * return if IOMMU remapping bar address
3662 */
3663static bool amdgpu_device_check_iommu_remap(struct amdgpu_device *adev)
3664{
3665 struct iommu_domain *domain;
3666
3667 domain = iommu_get_domain_for_dev(adev->dev);
3668 if (domain && (domain->type == IOMMU_DOMAIN_DMA ||
3669 domain->type == IOMMU_DOMAIN_DMA_FQ))
3670 return true;
3671
3672 return false;
3673}
3674#endif
3675
3676static void amdgpu_device_set_mcbp(struct amdgpu_device *adev)
3677{
3678 if (amdgpu_mcbp == 1)
3679 adev->gfx.mcbp = true;
3680 else if (amdgpu_mcbp == 0)
3681 adev->gfx.mcbp = false;
3682
3683 if (amdgpu_sriov_vf(adev))
3684 adev->gfx.mcbp = true;
3685
3686 if (adev->gfx.mcbp)
3687 dev_info(adev->dev, "MCBP is enabled\n");
3688}
3689
3690static int amdgpu_device_sys_interface_init(struct amdgpu_device *adev)
3691{
3692 int r;
3693
3694 r = amdgpu_atombios_sysfs_init(adev);
3695 if (r)
3696 drm_err(&adev->ddev,
3697 "registering atombios sysfs failed (%d).\n", r);
3698
3699 r = amdgpu_pm_sysfs_init(adev);
3700 if (r)
3701 dev_err(adev->dev, "registering pm sysfs failed (%d).\n", r);
3702
3703 r = amdgpu_ucode_sysfs_init(adev);
3704 if (r) {
3705 adev->ucode_sysfs_en = false;
3706 dev_err(adev->dev, "Creating firmware sysfs failed (%d).\n", r);
3707 } else
3708 adev->ucode_sysfs_en = true;
3709
3710 r = amdgpu_device_attr_sysfs_init(adev);
3711 if (r)
3712 dev_err(adev->dev, "Could not create amdgpu device attr\n");
3713
3714 r = devm_device_add_group(adev->dev, &amdgpu_board_attrs_group);
3715 if (r)
3716 dev_err(adev->dev,
3717 "Could not create amdgpu board attributes\n");
3718
3719 amdgpu_fru_sysfs_init(adev);
3720 amdgpu_reg_state_sysfs_init(adev);
3721 amdgpu_xcp_sysfs_init(adev);
3722 amdgpu_uma_sysfs_init(adev);
3723 amdgpu_ptl_sysfs_init(adev);
3724
3725 return r;
3726}
3727
3728static void amdgpu_device_sys_interface_fini(struct amdgpu_device *adev)
3729{
3730 if (adev->pm.sysfs_initialized)
3731 amdgpu_pm_sysfs_fini(adev);
3732 if (adev->ucode_sysfs_en)
3733 amdgpu_ucode_sysfs_fini(adev);
3734
3735 amdgpu_discovery_sysfs_fini(adev);
3736 amdgpu_preempt_mgr_sysfs_fini(adev);
3737
3738 amdgpu_device_attr_sysfs_fini(adev);
3739 amdgpu_fru_sysfs_fini(adev);
3740
3741 amdgpu_reg_state_sysfs_fini(adev);
3742 amdgpu_xcp_sysfs_fini(adev);
3743 amdgpu_uma_sysfs_fini(adev);
3744 amdgpu_ptl_sysfs_fini(adev);
3745}
3746
3747static bool
3748amdgpu_device_should_register_switcheroo(struct amdgpu_device *adev, bool px)
3749{
3750 return !pci_is_thunderbolt_attached(adev->pdev) &&
3751 (px || (!dev_is_removable(&adev->pdev->dev) &&
3752 apple_gmux_detect(NULL, NULL)));
3753}
3754
3755/**
3756 * amdgpu_device_init - initialize the driver
3757 *
3758 * @adev: amdgpu_device pointer
3759 * @flags: driver flags
3760 *
3761 * Initializes the driver info and hw (all asics).
3762 * Returns 0 for success or an error on failure.
3763 * Called at driver startup.
3764 */
3765int amdgpu_device_init(struct amdgpu_device *adev,
3766 uint32_t flags)
3767{
3768 struct pci_dev *pdev = adev->pdev;
3769 int r, i;
3770 bool px = false;
3771 u32 max_MBps;
3772 int tmp;
3773
3774 adev->shutdown = false;
3775 adev->flags = flags;
3776 amdgpu_device_init_pcie_links(adev);
3777
3778 if (amdgpu_force_asic_type >= 0 && amdgpu_force_asic_type < CHIP_LAST)
3779 adev->asic_type = amdgpu_force_asic_type;
3780 else
3781 adev->asic_type = flags & AMD_ASIC_MASK;
3782
3783 adev->usec_timeout = AMDGPU_MAX_USEC_TIMEOUT;
3784 if (amdgpu_emu_mode == 1)
3785 adev->usec_timeout *= 10;
3786 adev->gmc.gart_size = 512 * 1024 * 1024;
3787 adev->accel_working = false;
3788 adev->num_rings = 0;
3789 RCU_INIT_POINTER(adev->gang_submit, dma_fence_get_stub());
3790 adev->mman.buffer_funcs = NULL;
3791 adev->mman.num_buffer_funcs_scheds = 0;
3792 adev->vm_manager.vm_pte_funcs = NULL;
3793 adev->vm_manager.vm_pte_num_scheds = 0;
3794 adev->gmc.gmc_funcs = NULL;
3795 adev->harvest_ip_mask = 0x0;
3796 adev->fence_context = dma_fence_context_alloc(AMDGPU_MAX_RINGS);
3797 bitmap_zero(adev->gfx.pipe_reserve_bitmap, AMDGPU_MAX_COMPUTE_QUEUES);
3798
3799 amdgpu_reg_access_init(adev);
3800
3801 dev_info(
3802 adev->dev,
3803 "initializing kernel modesetting (%s 0x%04X:0x%04X 0x%04X:0x%04X 0x%02X).\n",
3804 amdgpu_asic_name[adev->asic_type], pdev->vendor, pdev->device,
3805 pdev->subsystem_vendor, pdev->subsystem_device, pdev->revision);
3806
3807 /* mutex initialization are all done here so we
3808 * can recall function without having locking issues
3809 */
3810 mutex_init(&adev->firmware.mutex);
3811 mutex_init(&adev->pm.mutex);
3812 mutex_init(&adev->gfx.gpu_clock_mutex);
3813 mutex_init(&adev->srbm_mutex);
3814 mutex_init(&adev->gfx.pipe_reserve_mutex);
3815 mutex_init(&adev->gfx.gfx_off_mutex);
3816 mutex_init(&adev->gfx.partition_mutex);
3817 mutex_init(&adev->grbm_idx_mutex);
3818 mutex_init(&adev->mn_lock);
3819 mutex_init(&adev->virt.vf_errors.lock);
3820 hash_init(adev->mn_hash);
3821 mutex_init(&adev->psp.mutex);
3822 mutex_init(&adev->psp.ptl.mutex);
3823 mutex_init(&adev->notifier_lock);
3824 mutex_init(&adev->pm.stable_pstate_ctx_lock);
3825 mutex_init(&adev->benchmark_mutex);
3826 mutex_init(&adev->gfx.reset_sem_mutex);
3827
3828 /* Associate locks with lockdep classes for ordering validation */
3829 amdgpu_lockdep_set_class(adev);
3830 /* Initialize the mutex for cleaner shader isolation between GFX and compute processes */
3831 mutex_init(&adev->enforce_isolation_mutex);
3832 for (i = 0; i < MAX_XCP; ++i) {
3833 adev->isolation[i].spearhead = dma_fence_get_stub();
3834 amdgpu_sync_create(&adev->isolation[i].active);
3835 amdgpu_sync_create(&adev->isolation[i].prev);
3836 }
3837 mutex_init(&adev->gfx.userq_sch_mutex);
3838 mutex_init(&adev->gfx.workload_profile_mutex);
3839 mutex_init(&adev->vcn.workload_profile_mutex);
3840
3841 spin_lock_init(&adev->irq.lock);
3842
3843 amdgpu_early_init_rlc_reg_funcs(adev);
3844 amdgpu_device_init_apu_flags(adev);
3845
3846 r = amdgpu_device_check_arguments(adev);
3847 if (r)
3848 return r;
3849
3850 spin_lock_init(&adev->mmio_idx_lock);
3851 spin_lock_init(&adev->mm_stats.lock);
3852 spin_lock_init(&adev->virt.rlcg_reg_lock);
3853 spin_lock_init(&adev->wb.lock);
3854
3855 INIT_LIST_HEAD(&adev->reset_list);
3856
3857 INIT_LIST_HEAD(&adev->ras_list);
3858
3859 INIT_LIST_HEAD(&adev->pm.od_kobj_list);
3860
3861 xa_init_flags(&adev->userq_doorbell_xa, XA_FLAGS_LOCK_IRQ);
3862
3863 INIT_DELAYED_WORK(&adev->delayed_init_work,
3864 amdgpu_device_delayed_init_work_handler);
3865 INIT_DELAYED_WORK(&adev->gfx.gfx_off_delay_work,
3866 amdgpu_device_delay_enable_gfx_off);
3867 /*
3868 * Initialize the enforce_isolation work structures for each XCP
3869 * partition. This work handler is responsible for enforcing shader
3870 * isolation on AMD GPUs. It counts the number of emitted fences for
3871 * each GFX and compute ring. If there are any fences, it schedules
3872 * the `enforce_isolation_work` to be run after a delay. If there are
3873 * no fences, it signals the Kernel Fusion Driver (KFD) to resume the
3874 * runqueue.
3875 */
3876 for (i = 0; i < MAX_XCP; i++) {
3877 INIT_DELAYED_WORK(&adev->gfx.enforce_isolation[i].work,
3878 amdgpu_gfx_enforce_isolation_handler);
3879 adev->gfx.enforce_isolation[i].adev = adev;
3880 adev->gfx.enforce_isolation[i].xcp_id = i;
3881 }
3882
3883 INIT_WORK(&adev->xgmi_reset_work, amdgpu_device_xgmi_reset_func);
3884
3885 amdgpu_coredump_init(adev);
3886
3887 adev->gfx.gfx_off_req_count = 1;
3888 adev->gfx.gfx_off_residency = 0;
3889 adev->gfx.gfx_off_entrycount = 0;
3890 adev->pm.ac_power = power_supply_is_system_supplied() > 0;
3891
3892 atomic_set(&adev->throttling_logging_enabled, 1);
3893 /*
3894 * If throttling continues, logging will be performed every minute
3895 * to avoid log flooding. "-1" is subtracted since the thermal
3896 * throttling interrupt comes every second. Thus, the total logging
3897 * interval is 59 seconds(retelimited printk interval) + 1(waiting
3898 * for throttling interrupt) = 60 seconds.
3899 */
3900 ratelimit_state_init(&adev->throttling_logging_rs, (60 - 1) * HZ, 1);
3901
3902 ratelimit_set_flags(&adev->throttling_logging_rs, RATELIMIT_MSG_ON_RELEASE);
3903
3904 /* Registers mapping */
3905 /* TODO: block userspace mapping of io register */
3906 if (adev->asic_type >= CHIP_BONAIRE) {
3907 adev->rmmio_base = pci_resource_start(adev->pdev, 5);
3908 adev->rmmio_size = pci_resource_len(adev->pdev, 5);
3909 } else {
3910 adev->rmmio_base = pci_resource_start(adev->pdev, 2);
3911 adev->rmmio_size = pci_resource_len(adev->pdev, 2);
3912 }
3913
3914 for (i = 0; i < AMD_IP_BLOCK_TYPE_NUM; i++)
3915 atomic_set(&adev->pm.pwr_state[i], POWER_STATE_UNKNOWN);
3916
3917 adev->rmmio = ioremap(adev->rmmio_base, adev->rmmio_size);
3918 if (!adev->rmmio)
3919 return -ENOMEM;
3920
3921 dev_info(adev->dev, "register mmio base: 0x%08X\n",
3922 (uint32_t)adev->rmmio_base);
3923 dev_info(adev->dev, "register mmio size: %u\n",
3924 (unsigned int)adev->rmmio_size);
3925
3926 /*
3927 * Reset domain needs to be present early, before XGMI hive discovered
3928 * (if any) and initialized to use reset sem and in_gpu reset flag
3929 * early on during init and before calling to RREG32.
3930 */
3931 adev->reset_domain = amdgpu_reset_create_reset_domain(SINGLE_DEVICE, "amdgpu-reset-dev");
3932 if (!adev->reset_domain)
3933 return -ENOMEM;
3934
3935 /* detect hw virtualization here */
3936 amdgpu_virt_init(adev);
3937
3938 amdgpu_device_get_pcie_info(adev);
3939
3940 r = amdgpu_device_get_job_timeout_settings(adev);
3941 if (r) {
3942 dev_err(adev->dev, "invalid lockup_timeout parameter syntax\n");
3943 return r;
3944 }
3945
3946 amdgpu_device_set_mcbp(adev);
3947
3948 /*
3949 * By default, use default mode where all blocks are expected to be
3950 * initialized. At present a 'swinit' of blocks is required to be
3951 * completed before the need for a different level is detected.
3952 */
3953 amdgpu_set_init_level(adev, AMDGPU_INIT_LEVEL_DEFAULT);
3954
3955 amdgpu_device_check_iommu_direct_map(adev);
3956
3957 /* early init functions */
3958 r = amdgpu_device_ip_early_init(adev);
3959 if (r)
3960 return r;
3961
3962 /*
3963 * No need to remove conflicting FBs for non-display class devices.
3964 * This prevents the sysfb from being freed accidently.
3965 */
3966 if ((pdev->class >> 8) == PCI_CLASS_DISPLAY_VGA ||
3967 (pdev->class >> 8) == PCI_CLASS_DISPLAY_OTHER) {
3968 /* Get rid of things like offb */
3969 r = aperture_remove_conflicting_pci_devices(adev->pdev, amdgpu_kms_driver.name);
3970 if (r)
3971 return r;
3972 }
3973
3974 /* Enable TMZ based on IP_VERSION */
3975 amdgpu_gmc_tmz_set(adev);
3976
3977 if (amdgpu_sriov_vf(adev) &&
3978 amdgpu_ip_version(adev, GC_HWIP, 0) >= IP_VERSION(10, 3, 0))
3979 /* VF MMIO access (except mailbox range) from CPU
3980 * will be blocked during sriov runtime
3981 */
3982 adev->virt.caps |= AMDGPU_VF_MMIO_ACCESS_PROTECT;
3983
3984 amdgpu_gmc_noretry_set(adev);
3985 /* Need to get xgmi info early to decide the reset behavior*/
3986 if (adev->gmc.xgmi.supported) {
3987 if (adev->gfxhub.funcs &&
3988 adev->gfxhub.funcs->get_xgmi_info) {
3989 r = adev->gfxhub.funcs->get_xgmi_info(adev);
3990 if (r)
3991 return r;
3992 }
3993 }
3994
3995 if (adev->gmc.xgmi.connected_to_cpu) {
3996 if (adev->mmhub.funcs &&
3997 adev->mmhub.funcs->get_xgmi_info) {
3998 r = adev->mmhub.funcs->get_xgmi_info(adev);
3999 if (r)
4000 return r;
4001 }
4002 }
4003
4004 /* enable PCIE atomic ops */
4005 if (amdgpu_sriov_vf(adev)) {
4006 if (adev->virt.fw_reserve.p_pf2vf)
4007 adev->have_atomics_support = ((struct amd_sriov_msg_pf2vf_info *)
4008 adev->virt.fw_reserve.p_pf2vf)->pcie_atomic_ops_support_flags ==
4009 (PCI_EXP_DEVCAP2_ATOMIC_COMP32 | PCI_EXP_DEVCAP2_ATOMIC_COMP64);
4010 /* APUs w/ gfx9 onwards doesn't reply on PCIe atomics, rather it is a
4011 * internal path natively support atomics, set have_atomics_support to true.
4012 */
4013 } else if ((adev->flags & AMD_IS_APU &&
4014 amdgpu_ip_version(adev, GC_HWIP, 0) > IP_VERSION(9, 0, 0)) ||
4015 (adev->gmc.xgmi.connected_to_cpu &&
4016 amdgpu_ip_version(adev, GC_HWIP, 0) >= IP_VERSION(12, 1, 0))) {
4017 adev->have_atomics_support = true;
4018 } else {
4019 adev->have_atomics_support =
4020 !pci_enable_atomic_ops_to_root(adev->pdev,
4021 PCI_EXP_DEVCAP2_ATOMIC_COMP32 |
4022 PCI_EXP_DEVCAP2_ATOMIC_COMP64);
4023 }
4024
4025 if (!adev->have_atomics_support)
4026 dev_info(adev->dev, "PCIE atomic ops is not supported\n");
4027
4028 /* doorbell bar mapping and doorbell index init*/
4029 amdgpu_doorbell_init(adev);
4030
4031 if (amdgpu_emu_mode == 1) {
4032 /* post the asic on emulation mode */
4033 emu_soc_asic_init(adev);
4034 goto fence_driver_init;
4035 }
4036
4037 amdgpu_reset_init(adev);
4038
4039 /* detect if we are with an SRIOV vbios */
4040 if (adev->bios)
4041 amdgpu_device_detect_sriov_bios(adev);
4042
4043 /* check if we need to reset the asic
4044 * E.g., driver was not cleanly unloaded previously, etc.
4045 */
4046 if (!amdgpu_sriov_vf(adev) && amdgpu_asic_need_reset_on_init(adev)) {
4047 if (adev->gmc.xgmi.num_physical_nodes) {
4048 dev_info(adev->dev, "Pending hive reset.\n");
4049 amdgpu_set_init_level(adev,
4050 AMDGPU_INIT_LEVEL_MINIMAL_XGMI);
4051 } else {
4052 tmp = amdgpu_reset_method;
4053 /* It should do a default reset when loading or reloading the driver,
4054 * regardless of the module parameter reset_method.
4055 */
4056 amdgpu_reset_method = AMD_RESET_METHOD_NONE;
4057 r = amdgpu_asic_reset(adev);
4058 amdgpu_reset_method = tmp;
4059 }
4060
4061 if (r) {
4062 dev_err(adev->dev, "asic reset on init failed\n");
4063 goto failed;
4064 }
4065 }
4066
4067 /* Post card if necessary */
4068 if (amdgpu_device_need_post(adev)) {
4069 if (!adev->bios) {
4070 dev_err(adev->dev, "no vBIOS found\n");
4071 r = -EINVAL;
4072 goto failed;
4073 }
4074 dev_info(adev->dev, "GPU posting now...\n");
4075 r = amdgpu_device_asic_init(adev);
4076 if (r) {
4077 dev_err(adev->dev, "gpu post error!\n");
4078 goto failed;
4079 }
4080 }
4081
4082 if (adev->bios) {
4083 if (adev->is_atom_fw) {
4084 /* Initialize clocks */
4085 r = amdgpu_atomfirmware_get_clock_info(adev);
4086 if (r) {
4087 dev_err(adev->dev, "amdgpu_atomfirmware_get_clock_info failed\n");
4088 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_ATOMBIOS_GET_CLOCK_FAIL, 0, 0);
4089 goto failed;
4090 }
4091 } else {
4092 /* Initialize clocks */
4093 r = amdgpu_atombios_get_clock_info(adev);
4094 if (r) {
4095 dev_err(adev->dev, "amdgpu_atombios_get_clock_info failed\n");
4096 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_ATOMBIOS_GET_CLOCK_FAIL, 0, 0);
4097 goto failed;
4098 }
4099 /* init i2c buses */
4100 amdgpu_i2c_init(adev);
4101 }
4102 }
4103
4104fence_driver_init:
4105 /* Fence driver */
4106 r = amdgpu_fence_driver_sw_init(adev);
4107 if (r) {
4108 dev_err(adev->dev, "amdgpu_fence_driver_sw_init failed\n");
4109 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_FENCE_INIT_FAIL, 0, 0);
4110 goto failed;
4111 }
4112
4113 /* init the mode config */
4114 drm_mode_config_init(adev_to_drm(adev));
4115
4116 r = amdgpu_device_ip_init(adev);
4117 if (r) {
4118 dev_err(adev->dev, "amdgpu_device_ip_init failed\n");
4119 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_AMDGPU_INIT_FAIL, 0, 0);
4120 goto release_ras_con;
4121 }
4122
4123 amdgpu_fence_driver_hw_init(adev);
4124
4125 dev_info(adev->dev,
4126 "SE %d, SH per SE %d, CU per SH %d, active_cu_number %d\n",
4127 adev->gfx.config.max_shader_engines,
4128 adev->gfx.config.max_sh_per_se,
4129 adev->gfx.config.max_cu_per_sh,
4130 adev->gfx.cu_info.number);
4131
4132 adev->accel_working = true;
4133
4134 amdgpu_vm_check_compute_bug(adev);
4135
4136 /* Initialize the buffer migration limit. */
4137 if (amdgpu_moverate >= 0)
4138 max_MBps = amdgpu_moverate;
4139 else
4140 max_MBps = 8; /* Allow 8 MB/s. */
4141 /* Get a log2 for easy divisions. */
4142 adev->mm_stats.log2_max_MBps = ilog2(max(1u, max_MBps));
4143
4144 /*
4145 * Register gpu instance before amdgpu_device_enable_mgpu_fan_boost.
4146 * Otherwise the mgpu fan boost feature will be skipped due to the
4147 * gpu instance is counted less.
4148 */
4149 amdgpu_register_gpu_instance(adev);
4150
4151 /* enable clockgating, etc. after ib tests, etc. since some blocks require
4152 * explicit gating rather than handling it automatically.
4153 */
4154 if (adev->init_lvl->level != AMDGPU_INIT_LEVEL_MINIMAL_XGMI) {
4155 r = amdgpu_device_ip_late_init(adev);
4156 if (r) {
4157 dev_err(adev->dev, "amdgpu_device_ip_late_init failed\n");
4158 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_AMDGPU_LATE_INIT_FAIL, 0, r);
4159 goto release_ras_con;
4160 }
4161 /* must succeed. */
4162 amdgpu_ras_resume(adev);
4163 queue_delayed_work(system_dfl_wq, &adev->delayed_init_work,
4164 msecs_to_jiffies(AMDGPU_RESUME_MS));
4165 }
4166
4167 if (amdgpu_sriov_vf(adev)) {
4168 amdgpu_virt_release_full_gpu(adev, true);
4169 flush_delayed_work(&adev->delayed_init_work);
4170 }
4171
4172 /* Don't init kfd if whole hive need to be reset during init */
4173 if (adev->init_lvl->level != AMDGPU_INIT_LEVEL_MINIMAL_XGMI) {
4174 kgd2kfd_init_zone_device(adev);
4175 kfd_update_svm_support_properties(adev);
4176 }
4177
4178 if (adev->init_lvl->level == AMDGPU_INIT_LEVEL_MINIMAL_XGMI)
4179 amdgpu_xgmi_reset_on_init(adev);
4180
4181 /*
4182 * Place those sysfs registering after `late_init`. As some of those
4183 * operations performed in `late_init` might affect the sysfs
4184 * interfaces creating.
4185 */
4186 r = amdgpu_device_sys_interface_init(adev);
4187
4188 if (IS_ENABLED(CONFIG_PERF_EVENTS))
4189 r = amdgpu_pmu_init(adev);
4190 if (r)
4191 dev_err(adev->dev, "amdgpu_pmu_init failed\n");
4192
4193 /* Have stored pci confspace at hand for restore in sudden PCI error */
4194 if (amdgpu_device_cache_pci_state(adev->pdev))
4195 pci_restore_state(pdev);
4196
4197 /* if we have > 1 VGA cards, then disable the amdgpu VGA resources */
4198 /* this will fail for cards that aren't VGA class devices, just
4199 * ignore it
4200 */
4201 if ((adev->pdev->class >> 8) == PCI_CLASS_DISPLAY_VGA)
4202 vga_client_register(adev->pdev, amdgpu_device_vga_set_decode);
4203
4204 px = amdgpu_device_supports_px(adev);
4205
4206 if (amdgpu_device_should_register_switcheroo(adev, px))
4207 vga_switcheroo_register_client(adev->pdev,
4208 &amdgpu_switcheroo_ops, px);
4209
4210 if (px)
4211 vga_switcheroo_init_domain_pm_ops(adev->dev, &adev->vga_pm_domain);
4212
4213 adev->pm_nb.notifier_call = amdgpu_device_pm_notifier;
4214 r = register_pm_notifier(&adev->pm_nb);
4215 if (r)
4216 goto failed;
4217
4218 return 0;
4219
4220release_ras_con:
4221 if (amdgpu_sriov_vf(adev))
4222 amdgpu_virt_release_full_gpu(adev, true);
4223
4224 /* failed in exclusive mode due to timeout */
4225 if (amdgpu_sriov_vf(adev) &&
4226 !amdgpu_sriov_runtime(adev) &&
4227 amdgpu_virt_mmio_blocked(adev) &&
4228 !amdgpu_virt_wait_reset(adev)) {
4229 dev_err(adev->dev, "VF exclusive mode timeout\n");
4230 /* Don't send request since VF is inactive. */
4231 adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME;
4232 adev->virt.ops = NULL;
4233 r = -EAGAIN;
4234 }
4235 amdgpu_release_ras_context(adev);
4236
4237failed:
4238 amdgpu_vf_error_trans_all(adev);
4239
4240 return r;
4241}
4242
4243static void amdgpu_device_unmap_mmio(struct amdgpu_device *adev)
4244{
4245
4246 /* Clear all CPU mappings pointing to this device */
4247 unmap_mapping_range(adev->ddev.anon_inode->i_mapping, 0, 0, 1);
4248
4249 /* Unmap all mapped bars - Doorbell, registers and VRAM */
4250 amdgpu_doorbell_fini(adev);
4251
4252 iounmap(adev->rmmio);
4253 adev->rmmio = NULL;
4254 if (adev->mman.aper_base_kaddr)
4255 iounmap(adev->mman.aper_base_kaddr);
4256 adev->mman.aper_base_kaddr = NULL;
4257
4258 /* Memory manager related */
4259 if (!adev->gmc.xgmi.connected_to_cpu && !adev->gmc.is_app_apu) {
4260 arch_phys_wc_del(adev->gmc.vram_mtrr);
4261 arch_io_free_memtype_wc(adev->gmc.aper_base, adev->gmc.aper_size);
4262 }
4263}
4264
4265/**
4266 * amdgpu_device_fini_hw - tear down the driver
4267 *
4268 * @adev: amdgpu_device pointer
4269 *
4270 * Tear down the driver info (all asics).
4271 * Called at driver shutdown.
4272 */
4273void amdgpu_device_fini_hw(struct amdgpu_device *adev)
4274{
4275 dev_info(adev->dev, "finishing device.\n");
4276 flush_delayed_work(&adev->delayed_init_work);
4277
4278 if (adev->mman.initialized)
4279 drain_workqueue(adev->mman.bdev.wq);
4280
4281 adev->shutdown = true;
4282
4283 unregister_pm_notifier(&adev->pm_nb);
4284
4285 /* make sure IB test finished before entering exclusive mode
4286 * to avoid preemption on IB test
4287 */
4288 if (amdgpu_sriov_vf(adev)) {
4289 amdgpu_virt_request_full_gpu(adev, false);
4290 amdgpu_virt_fini_data_exchange(adev);
4291 }
4292
4293 amdgpu_device_set_pg_state(adev, AMD_PG_STATE_UNGATE);
4294 amdgpu_device_set_cg_state(adev, AMD_CG_STATE_UNGATE);
4295
4296 /* disable all interrupts */
4297 amdgpu_irq_disable_all(adev);
4298 if (adev->mode_info.mode_config_initialized) {
4299 if (!drm_drv_uses_atomic_modeset(adev_to_drm(adev)))
4300 drm_helper_force_disable_all(adev_to_drm(adev));
4301 else
4302 drm_atomic_helper_shutdown(adev_to_drm(adev));
4303 }
4304 amdgpu_fence_driver_hw_fini(adev);
4305
4306 amdgpu_device_sys_interface_fini(adev);
4307
4308 /* disable ras feature must before hw fini */
4309 amdgpu_ras_pre_fini(adev);
4310
4311 amdgpu_ttm_disable_buffer_funcs(adev);
4312
4313 /*
4314 * device went through surprise hotplug; we need to destroy topology
4315 * before ip_fini_early to prevent kfd locking refcount issues by calling
4316 * amdgpu_amdkfd_suspend()
4317 */
4318 if (pci_dev_is_disconnected(adev->pdev))
4319 amdgpu_amdkfd_device_fini_sw(adev);
4320
4321 amdgpu_coredump_fini(adev);
4322 amdgpu_device_ip_fini_early(adev);
4323
4324 amdgpu_irq_fini_hw(adev);
4325
4326 if (adev->mman.initialized)
4327 ttm_device_clear_dma_mappings(&adev->mman.bdev);
4328
4329 amdgpu_gart_dummy_page_fini(adev);
4330
4331 if (pci_dev_is_disconnected(adev->pdev))
4332 amdgpu_device_unmap_mmio(adev);
4333
4334}
4335
4336void amdgpu_device_fini_sw(struct amdgpu_device *adev)
4337{
4338 int i;
4339 bool px;
4340
4341 amdgpu_device_ip_fini(adev);
4342 amdgpu_fence_driver_sw_fini(adev);
4343 amdgpu_ucode_release(&adev->firmware.gpu_info_fw);
4344 adev->accel_working = false;
4345 dma_fence_put(rcu_dereference_protected(adev->gang_submit, true));
4346 for (i = 0; i < MAX_XCP; ++i) {
4347 dma_fence_put(adev->isolation[i].spearhead);
4348 amdgpu_sync_free(&adev->isolation[i].active);
4349 amdgpu_sync_free(&adev->isolation[i].prev);
4350 }
4351
4352 amdgpu_reset_fini(adev);
4353
4354 /* free i2c buses */
4355 amdgpu_i2c_fini(adev);
4356
4357 if (adev->bios) {
4358 if (amdgpu_emu_mode != 1)
4359 amdgpu_atombios_fini(adev);
4360 amdgpu_bios_release(adev);
4361 }
4362
4363 kfree(adev->fru_info);
4364 adev->fru_info = NULL;
4365
4366 kfree(adev->xcp_mgr);
4367 adev->xcp_mgr = NULL;
4368
4369 px = amdgpu_device_supports_px(adev);
4370
4371 if (amdgpu_device_should_register_switcheroo(adev, px))
4372 vga_switcheroo_unregister_client(adev->pdev);
4373
4374 if (px)
4375 vga_switcheroo_fini_domain_pm_ops(adev->dev);
4376
4377 if ((adev->pdev->class >> 8) == PCI_CLASS_DISPLAY_VGA)
4378 vga_client_unregister(adev->pdev);
4379
4380 if (adev->rmmio) {
4381 iounmap(adev->rmmio);
4382 adev->rmmio = NULL;
4383 }
4384
4385 if (IS_ENABLED(CONFIG_PERF_EVENTS))
4386 amdgpu_pmu_fini(adev);
4387 if (adev->discovery.bin)
4388 amdgpu_discovery_fini(adev);
4389
4390 amdgpu_reset_put_reset_domain(adev->reset_domain);
4391 adev->reset_domain = NULL;
4392
4393 kfree(adev->pci_state);
4394 kfree(adev->pcie_reset_ctx.swds_pcistate);
4395 kfree(adev->pcie_reset_ctx.swus_pcistate);
4396}
4397
4398/**
4399 * amdgpu_device_evict_resources - evict device resources
4400 * @adev: amdgpu device object
4401 *
4402 * Evicts all ttm device resources(vram BOs, gart table) from the lru list
4403 * of the vram memory type. Mainly used for evicting device resources
4404 * at suspend time.
4405 *
4406 */
4407static int amdgpu_device_evict_resources(struct amdgpu_device *adev)
4408{
4409 int ret;
4410
4411 /* No need to evict vram on APUs unless going to S4 */
4412 if (!adev->in_s4 && (adev->flags & AMD_IS_APU))
4413 return 0;
4414
4415 /* No need to evict when going to S5 through S4 callbacks */
4416 if (system_state == SYSTEM_POWER_OFF)
4417 return 0;
4418
4419 ret = amdgpu_ttm_evict_resources(adev, TTM_PL_VRAM);
4420 if (ret) {
4421 dev_warn(adev->dev, "evicting device resources failed\n");
4422 return ret;
4423 }
4424
4425 if (adev->in_s4) {
4426 ret = ttm_device_prepare_hibernation(&adev->mman.bdev);
4427 if (ret)
4428 dev_err(adev->dev, "prepare hibernation failed, %d\n", ret);
4429 }
4430 return ret;
4431}
4432
4433/*
4434 * Suspend & resume.
4435 */
4436/**
4437 * amdgpu_device_pm_notifier - Notification block for Suspend/Hibernate events
4438 * @nb: notifier block
4439 * @mode: suspend mode
4440 * @data: data
4441 *
4442 * This function is called when the system is about to suspend or hibernate.
4443 * It is used to set the appropriate flags so that eviction can be optimized
4444 * in the pm prepare callback.
4445 */
4446static int amdgpu_device_pm_notifier(struct notifier_block *nb, unsigned long mode,
4447 void *data)
4448{
4449 struct amdgpu_device *adev = container_of(nb, struct amdgpu_device, pm_nb);
4450
4451 switch (mode) {
4452 case PM_HIBERNATION_PREPARE:
4453 adev->in_s4 = true;
4454 break;
4455 case PM_POST_HIBERNATION:
4456 adev->in_s4 = false;
4457 break;
4458 }
4459
4460 return NOTIFY_DONE;
4461}
4462
4463/**
4464 * amdgpu_device_prepare - prepare for device suspend
4465 *
4466 * @dev: drm dev pointer
4467 *
4468 * Prepare to put the hw in the suspend state (all asics).
4469 * Returns 0 for success or an error on failure.
4470 * Called at driver suspend.
4471 */
4472int amdgpu_device_prepare(struct drm_device *dev)
4473{
4474 struct amdgpu_device *adev = drm_to_adev(dev);
4475 int i, r;
4476
4477 if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
4478 return 0;
4479
4480 /* Evict the majority of BOs before starting suspend sequence */
4481 r = amdgpu_device_evict_resources(adev);
4482 if (r)
4483 return r;
4484
4485 flush_delayed_work(&adev->gfx.gfx_off_delay_work);
4486
4487 for (i = 0; i < adev->num_ip_blocks; i++) {
4488 if (!adev->ip_blocks[i].status.valid)
4489 continue;
4490 if (!adev->ip_blocks[i].version->funcs->prepare_suspend)
4491 continue;
4492 r = adev->ip_blocks[i].version->funcs->prepare_suspend(&adev->ip_blocks[i]);
4493 if (r)
4494 return r;
4495 }
4496
4497 return 0;
4498}
4499
4500/**
4501 * amdgpu_device_complete - complete power state transition
4502 *
4503 * @dev: drm dev pointer
4504 *
4505 * Undo the changes from amdgpu_device_prepare. This will be
4506 * called on all resume transitions, including those that failed.
4507 */
4508void amdgpu_device_complete(struct drm_device *dev)
4509{
4510 struct amdgpu_device *adev = drm_to_adev(dev);
4511 int i;
4512
4513 for (i = 0; i < adev->num_ip_blocks; i++) {
4514 if (!adev->ip_blocks[i].status.valid)
4515 continue;
4516 if (!adev->ip_blocks[i].version->funcs->complete)
4517 continue;
4518 adev->ip_blocks[i].version->funcs->complete(&adev->ip_blocks[i]);
4519 }
4520}
4521
4522/**
4523 * amdgpu_device_suspend - initiate device suspend
4524 *
4525 * @dev: drm dev pointer
4526 * @notify_clients: notify in-kernel DRM clients
4527 *
4528 * Puts the hw in the suspend state (all asics).
4529 * Returns 0 for success or an error on failure.
4530 * Called at driver suspend.
4531 */
4532int amdgpu_device_suspend(struct drm_device *dev, bool notify_clients)
4533{
4534 struct amdgpu_device *adev = drm_to_adev(dev);
4535 int r, rec;
4536
4537 if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
4538 return 0;
4539
4540 adev->in_suspend = true;
4541
4542 if (amdgpu_sriov_vf(adev)) {
4543 if (!adev->in_runpm)
4544 amdgpu_amdkfd_suspend_process(adev);
4545 amdgpu_virt_fini_data_exchange(adev);
4546 r = amdgpu_virt_request_full_gpu(adev, false);
4547 if (r)
4548 return r;
4549 }
4550
4551 r = amdgpu_acpi_smart_shift_update(adev, AMDGPU_SS_DEV_D3);
4552 if (r)
4553 goto unwind_sriov;
4554
4555 if (notify_clients)
4556 drm_client_dev_suspend(adev_to_drm(adev));
4557
4558 cancel_delayed_work_sync(&adev->delayed_init_work);
4559
4560 amdgpu_ras_suspend(adev);
4561
4562 r = amdgpu_device_ip_suspend_phase1(adev);
4563 if (r)
4564 goto unwind_smartshift;
4565
4566 amdgpu_amdkfd_suspend(adev, !amdgpu_sriov_vf(adev) && !adev->in_runpm);
4567 r = amdgpu_userq_suspend(adev);
4568 if (r)
4569 goto unwind_ip_phase1;
4570
4571 r = amdgpu_device_evict_resources(adev);
4572 if (r)
4573 goto unwind_userq;
4574
4575 amdgpu_ttm_disable_buffer_funcs(adev);
4576
4577 amdgpu_fence_driver_hw_fini(adev);
4578
4579 r = amdgpu_device_ip_suspend_phase2(adev);
4580 if (r)
4581 goto unwind_evict;
4582
4583 if (amdgpu_sriov_vf(adev))
4584 amdgpu_virt_release_full_gpu(adev, false);
4585
4586 return 0;
4587
4588unwind_evict:
4589 amdgpu_ttm_enable_buffer_funcs(adev);
4590 amdgpu_fence_driver_hw_init(adev);
4591
4592unwind_userq:
4593 rec = amdgpu_userq_resume(adev);
4594 if (rec) {
4595 dev_warn(adev->dev, "failed to re-initialize user queues: %d\n", rec);
4596 return r;
4597 }
4598 rec = amdgpu_amdkfd_resume(adev, !amdgpu_sriov_vf(adev) && !adev->in_runpm);
4599 if (rec) {
4600 dev_warn(adev->dev, "failed to re-initialize kfd: %d\n", rec);
4601 return r;
4602 }
4603
4604unwind_ip_phase1:
4605 /* suspend phase 1 = resume phase 3 */
4606 rec = amdgpu_device_ip_resume_phase3(adev);
4607 if (rec) {
4608 dev_warn(adev->dev, "failed to re-initialize IPs phase1: %d\n", rec);
4609 return r;
4610 }
4611
4612unwind_smartshift:
4613 rec = amdgpu_acpi_smart_shift_update(adev, AMDGPU_SS_DEV_D0);
4614 if (rec) {
4615 dev_warn(adev->dev, "failed to re-update smart shift: %d\n", rec);
4616 return r;
4617 }
4618
4619 if (notify_clients)
4620 drm_client_dev_resume(adev_to_drm(adev));
4621
4622 amdgpu_ras_resume(adev);
4623
4624unwind_sriov:
4625 if (amdgpu_sriov_vf(adev)) {
4626 rec = amdgpu_virt_request_full_gpu(adev, true);
4627 if (rec) {
4628 dev_warn(adev->dev, "failed to reinitialize sriov: %d\n", rec);
4629 return r;
4630 }
4631 }
4632
4633 adev->in_suspend = adev->in_s0ix = adev->in_s3 = false;
4634
4635 return r;
4636}
4637
4638static inline int amdgpu_virt_resume(struct amdgpu_device *adev)
4639{
4640 int r;
4641 unsigned int prev_physical_node_id = adev->gmc.xgmi.physical_node_id;
4642
4643 /* During VM resume, QEMU programming of VF MSIX table (register GFXMSIX_VECT0_ADDR_LO)
4644 * may not work. The access could be blocked by nBIF protection as VF isn't in
4645 * exclusive access mode. Exclusive access is enabled now, disable/enable MSIX
4646 * so that QEMU reprograms MSIX table.
4647 */
4648 amdgpu_restore_msix(adev);
4649
4650 r = adev->gfxhub.funcs->get_xgmi_info(adev);
4651 if (r)
4652 return r;
4653
4654 dev_info(adev->dev, "xgmi node, old id %d, new id %d\n",
4655 prev_physical_node_id, adev->gmc.xgmi.physical_node_id);
4656
4657 adev->vm_manager.vram_base_offset = adev->gfxhub.funcs->get_mc_fb_offset(adev);
4658 adev->vm_manager.vram_base_offset +=
4659 adev->gmc.xgmi.physical_node_id * adev->gmc.xgmi.node_segment_size;
4660
4661 return 0;
4662}
4663
4664/**
4665 * amdgpu_device_resume - initiate device resume
4666 *
4667 * @dev: drm dev pointer
4668 * @notify_clients: notify in-kernel DRM clients
4669 *
4670 * Bring the hw back to operating state (all asics).
4671 * Returns 0 for success or an error on failure.
4672 * Called at driver resume.
4673 */
4674int amdgpu_device_resume(struct drm_device *dev, bool notify_clients)
4675{
4676 struct amdgpu_device *adev = drm_to_adev(dev);
4677 int r = 0;
4678
4679 if (amdgpu_sriov_vf(adev)) {
4680 r = amdgpu_virt_request_full_gpu(adev, true);
4681 if (r)
4682 return r;
4683 }
4684
4685 if (amdgpu_virt_xgmi_migrate_enabled(adev)) {
4686 r = amdgpu_virt_resume(adev);
4687 if (r)
4688 goto exit;
4689 }
4690
4691 if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
4692 return 0;
4693
4694 if (adev->in_s0ix)
4695 amdgpu_dpm_gfx_state_change(adev, sGpuChangeState_D0Entry);
4696
4697 /* post card */
4698 if (amdgpu_device_need_post(adev)) {
4699 r = amdgpu_device_asic_init(adev);
4700 if (r)
4701 dev_err(adev->dev, "amdgpu asic init failed\n");
4702 }
4703
4704 r = amdgpu_device_ip_resume(adev);
4705
4706 if (r) {
4707 dev_err(adev->dev, "amdgpu_device_ip_resume failed (%d).\n", r);
4708 goto exit;
4709 }
4710
4711 r = amdgpu_amdkfd_resume(adev, !amdgpu_sriov_vf(adev) && !adev->in_runpm);
4712 if (r)
4713 goto exit;
4714
4715 r = amdgpu_userq_resume(adev);
4716 if (r)
4717 goto exit;
4718
4719 r = amdgpu_device_ip_late_init(adev);
4720 if (r)
4721 goto exit;
4722
4723 queue_delayed_work(system_dfl_wq, &adev->delayed_init_work,
4724 msecs_to_jiffies(AMDGPU_RESUME_MS));
4725exit:
4726 if (amdgpu_sriov_vf(adev)) {
4727 amdgpu_virt_init_data_exchange(adev);
4728 amdgpu_virt_release_full_gpu(adev, true);
4729
4730 if (!r && !adev->in_runpm)
4731 r = amdgpu_amdkfd_resume_process(adev);
4732 }
4733
4734 if (r)
4735 return r;
4736
4737 /* Make sure IB tests flushed */
4738 flush_delayed_work(&adev->delayed_init_work);
4739
4740 if (notify_clients)
4741 drm_client_dev_resume(adev_to_drm(adev));
4742
4743 amdgpu_ras_resume(adev);
4744
4745 if (adev->mode_info.num_crtc) {
4746 /*
4747 * Most of the connector probing functions try to acquire runtime pm
4748 * refs to ensure that the GPU is powered on when connector polling is
4749 * performed. Since we're calling this from a runtime PM callback,
4750 * trying to acquire rpm refs will cause us to deadlock.
4751 *
4752 * Since we're guaranteed to be holding the rpm lock, it's safe to
4753 * temporarily disable the rpm helpers so this doesn't deadlock us.
4754 */
4755#ifdef CONFIG_PM
4756 dev->dev->power.disable_depth++;
4757#endif
4758 if (!adev->dc_enabled)
4759 drm_helper_hpd_irq_event(dev);
4760 else
4761 drm_kms_helper_hotplug_event(dev);
4762#ifdef CONFIG_PM
4763 dev->dev->power.disable_depth--;
4764#endif
4765 }
4766
4767 amdgpu_vram_mgr_clear_reset_blocks(adev);
4768 adev->in_suspend = false;
4769
4770 if (amdgpu_acpi_smart_shift_update(adev, AMDGPU_SS_DEV_D0))
4771 dev_warn(adev->dev, "smart shift update failed\n");
4772
4773 return 0;
4774}
4775
4776/**
4777 * amdgpu_device_reset_sriov - reset ASIC for SR-IOV vf
4778 *
4779 * @adev: amdgpu_device pointer
4780 * @reset_context: amdgpu reset context pointer
4781 *
4782 * do VF FLR and reinitialize Asic
4783 * return 0 means succeeded otherwise failed
4784 */
4785static int amdgpu_device_reset_sriov(struct amdgpu_device *adev,
4786 struct amdgpu_reset_context *reset_context)
4787{
4788 int r;
4789 struct amdgpu_hive_info *hive = NULL;
4790
4791 if (test_bit(AMDGPU_HOST_FLR, &reset_context->flags)) {
4792 if (!amdgpu_ras_get_fed_status(adev))
4793 amdgpu_virt_ready_to_reset(adev);
4794 amdgpu_virt_wait_reset(adev);
4795 clear_bit(AMDGPU_HOST_FLR, &reset_context->flags);
4796 r = amdgpu_virt_request_full_gpu(adev, true);
4797 } else {
4798 r = amdgpu_virt_reset_gpu(adev);
4799 }
4800 if (r)
4801 return r;
4802
4803 amdgpu_ras_clear_err_state(adev);
4804 amdgpu_irq_gpu_reset_resume_helper(adev);
4805
4806 /* some sw clean up VF needs to do before recover */
4807 amdgpu_virt_post_reset(adev);
4808
4809 /* Resume IP prior to SMC */
4810 r = amdgpu_device_ip_reinit_early_sriov(adev);
4811 if (r)
4812 return r;
4813
4814 amdgpu_virt_init_data_exchange(adev);
4815
4816 r = amdgpu_device_fw_loading(adev);
4817 if (r)
4818 return r;
4819
4820 /* now we are okay to resume SMC/CP/SDMA */
4821 r = amdgpu_device_ip_reinit_late_sriov(adev);
4822 if (r)
4823 return r;
4824
4825 hive = amdgpu_get_xgmi_hive(adev);
4826 /* Update PSP FW topology after reset */
4827 if (hive && adev->gmc.xgmi.num_physical_nodes > 1)
4828 r = amdgpu_xgmi_update_topology(hive, adev);
4829 if (hive)
4830 amdgpu_put_xgmi_hive(hive);
4831 if (r)
4832 return r;
4833
4834 r = amdgpu_ib_ring_tests(adev);
4835 if (r)
4836 return r;
4837
4838 if (adev->virt.gim_feature & AMDGIM_FEATURE_GIM_FLR_VRAMLOST)
4839 amdgpu_inc_vram_lost(adev);
4840
4841 /* need to be called during full access so we can't do it later like
4842 * bare-metal does.
4843 */
4844 amdgpu_amdkfd_post_reset(adev);
4845 amdgpu_virt_release_full_gpu(adev, true);
4846
4847 /* Aldebaran and gfx_11_0_3 support ras in SRIOV, so need resume ras during reset */
4848 if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 2) ||
4849 amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 3) ||
4850 amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 4) ||
4851 amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 5, 0) ||
4852 amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(11, 0, 3))
4853 amdgpu_ras_resume(adev);
4854
4855 amdgpu_virt_ras_telemetry_post_reset(adev);
4856
4857 return 0;
4858}
4859
4860/**
4861 * amdgpu_device_has_job_running - check if there is any unfinished job
4862 *
4863 * @adev: amdgpu_device pointer
4864 *
4865 * check if there is any job running on the device when guest driver receives
4866 * FLR notification from host driver. If there are still jobs running, then
4867 * the guest driver will not respond the FLR reset. Instead, let the job hit
4868 * the timeout and guest driver then issue the reset request.
4869 */
4870bool amdgpu_device_has_job_running(struct amdgpu_device *adev)
4871{
4872 int i;
4873
4874 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
4875 struct amdgpu_ring *ring = adev->rings[i];
4876
4877 if (!amdgpu_ring_sched_ready(ring))
4878 continue;
4879
4880 if (amdgpu_fence_count_emitted(ring))
4881 return true;
4882 }
4883 return false;
4884}
4885
4886/**
4887 * amdgpu_device_should_recover_gpu - check if we should try GPU recovery
4888 *
4889 * @adev: amdgpu_device pointer
4890 *
4891 * Check amdgpu_gpu_recovery and SRIOV status to see if we should try to recover
4892 * a hung GPU.
4893 */
4894bool amdgpu_device_should_recover_gpu(struct amdgpu_device *adev)
4895{
4896
4897 if (amdgpu_gpu_recovery == 0)
4898 goto disabled;
4899
4900 /* Skip soft reset check in fatal error mode */
4901 if (!amdgpu_ras_is_poison_mode_supported(adev))
4902 return true;
4903
4904 if (amdgpu_sriov_vf(adev))
4905 return true;
4906
4907 if (amdgpu_gpu_recovery == -1) {
4908 switch (adev->asic_type) {
4909#ifdef CONFIG_DRM_AMDGPU_SI
4910 case CHIP_VERDE:
4911 case CHIP_TAHITI:
4912 case CHIP_PITCAIRN:
4913 case CHIP_OLAND:
4914 case CHIP_HAINAN:
4915#endif
4916#ifdef CONFIG_DRM_AMDGPU_CIK
4917 case CHIP_KAVERI:
4918 case CHIP_KABINI:
4919 case CHIP_MULLINS:
4920#endif
4921 case CHIP_CARRIZO:
4922 case CHIP_STONEY:
4923 case CHIP_CYAN_SKILLFISH:
4924 goto disabled;
4925 default:
4926 break;
4927 }
4928 }
4929
4930 return true;
4931
4932disabled:
4933 dev_info(adev->dev, "GPU recovery disabled.\n");
4934 return false;
4935}
4936
4937int amdgpu_device_mode1_reset(struct amdgpu_device *adev)
4938{
4939 u32 i;
4940 int ret = 0;
4941
4942 if (adev->bios)
4943 amdgpu_atombios_scratch_regs_engine_hung(adev, true);
4944
4945 dev_info(adev->dev, "GPU mode1 reset\n");
4946
4947 /* Cache the state before bus master disable. The saved config space
4948 * values are used in other cases like restore after mode-2 reset.
4949 */
4950 amdgpu_device_cache_pci_state(adev->pdev);
4951
4952 /* disable BM */
4953 pci_clear_master(adev->pdev);
4954
4955 if (amdgpu_dpm_is_mode1_reset_supported(adev)) {
4956 dev_info(adev->dev, "GPU smu mode1 reset\n");
4957 ret = amdgpu_dpm_mode1_reset(adev);
4958 } else {
4959 dev_info(adev->dev, "GPU psp mode1 reset\n");
4960 ret = psp_gpu_reset(adev);
4961 }
4962
4963 if (ret)
4964 goto mode1_reset_failed;
4965
4966 /* enable mmio access after mode 1 reset completed */
4967 adev->no_hw_access = false;
4968
4969 /* ensure no_hw_access is updated before we access hw */
4970 smp_mb();
4971
4972 amdgpu_device_load_pci_state(adev->pdev);
4973 ret = amdgpu_psp_wait_for_bootloader(adev);
4974 if (ret)
4975 goto mode1_reset_failed;
4976
4977 /* wait for asic to come out of reset */
4978 for (i = 0; i < adev->usec_timeout; i++) {
4979 u32 memsize = adev->nbio.funcs->get_memsize(adev);
4980
4981 if (memsize != 0xffffffff)
4982 break;
4983 udelay(1);
4984 }
4985
4986 if (i >= adev->usec_timeout) {
4987 ret = -ETIMEDOUT;
4988 goto mode1_reset_failed;
4989 }
4990
4991 if (adev->bios)
4992 amdgpu_atombios_scratch_regs_engine_hung(adev, false);
4993
4994 return 0;
4995
4996mode1_reset_failed:
4997 dev_err(adev->dev, "GPU mode1 reset failed\n");
4998 return ret;
4999}
5000
5001int amdgpu_device_link_reset(struct amdgpu_device *adev)
5002{
5003 int ret = 0;
5004
5005 dev_info(adev->dev, "GPU link reset\n");
5006
5007 if (!amdgpu_reset_in_dpc(adev))
5008 ret = amdgpu_dpm_link_reset(adev);
5009
5010 if (ret)
5011 goto link_reset_failed;
5012
5013 ret = amdgpu_psp_wait_for_bootloader(adev);
5014 if (ret)
5015 goto link_reset_failed;
5016
5017 return 0;
5018
5019link_reset_failed:
5020 dev_err(adev->dev, "GPU link reset failed\n");
5021 return ret;
5022}
5023
5024int amdgpu_device_pre_asic_reset(struct amdgpu_device *adev,
5025 struct amdgpu_reset_context *reset_context)
5026{
5027 struct amdgpu_job *job = NULL;
5028 struct dma_fence *fence = NULL;
5029 struct amdgpu_device *tmp_adev = reset_context->reset_req_dev;
5030 bool need_full_reset =
5031 test_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags);
5032 int i, r;
5033
5034 if (reset_context->reset_req_dev == adev)
5035 job = reset_context->job;
5036
5037 if (amdgpu_sriov_vf(adev))
5038 amdgpu_virt_pre_reset(adev);
5039
5040 amdgpu_fence_driver_isr_toggle(adev, true);
5041
5042 if (job)
5043 fence = &job->hw_fence->base;
5044
5045 /* block all schedulers and reset given job's ring */
5046 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
5047 struct amdgpu_ring *ring = adev->rings[i];
5048
5049 if (!amdgpu_ring_sched_ready(ring))
5050 continue;
5051
5052 /* after all hw jobs are reset, hw fence is meaningless, so force_completion */
5053 amdgpu_fence_driver_force_completion(ring, fence);
5054 }
5055
5056 /*
5057 * MES scheduler rings have no drm scheduler, so they are missed by the
5058 * loop above. Realign their polling fence too (one per XCC), otherwise the
5059 * first post-reset submission polls forever on a stale seq. sched.ready is
5060 * only set while the driver owns the ring.
5061 */
5062 for (i = 0; i < AMDGPU_MAX_MES_INST_PIPES; i++) {
5063 struct amdgpu_ring *mes_ring = &adev->mes.ring[i];
5064
5065 if (mes_ring->fence_drv.initialized && mes_ring->sched.ready)
5066 amdgpu_fence_driver_force_completion(mes_ring, fence);
5067 }
5068
5069 /*
5070 * KIQ rings are polling-fence/no_scheduler like MES, so realign their
5071 * fence too (one ring per XCC), otherwise the first post-reset KIQ
5072 * submission polls forever on a stale seq.
5073 */
5074 for (i = 0; i < AMDGPU_MAX_GC_INSTANCES; i++) {
5075 struct amdgpu_ring *kiq_ring = &adev->gfx.kiq[i].ring;
5076
5077 if (kiq_ring->fence_drv.initialized && kiq_ring->sched.ready)
5078 amdgpu_fence_driver_force_completion(kiq_ring, fence);
5079 }
5080
5081 amdgpu_fence_driver_isr_toggle(adev, false);
5082
5083 r = amdgpu_reset_prepare_hwcontext(adev, reset_context);
5084 /* If reset handler not implemented, continue; otherwise return */
5085 if (r == -EOPNOTSUPP)
5086 r = 0;
5087 else
5088 return r;
5089
5090 /* Don't suspend on bare metal if we are not going to HW reset the ASIC */
5091 if (!amdgpu_sriov_vf(adev)) {
5092 need_full_reset = true;
5093
5094 if (!test_bit(AMDGPU_SKIP_COREDUMP, &reset_context->flags)) {
5095 dev_info(tmp_adev->dev, "Dumping IP State\n");
5096 /* Trigger ip dump before we reset the asic */
5097 for (i = 0; i < tmp_adev->num_ip_blocks; i++)
5098 if (tmp_adev->ip_blocks[i].version->funcs->dump_ip_state)
5099 tmp_adev->ip_blocks[i].version->funcs
5100 ->dump_ip_state((void *)&tmp_adev->ip_blocks[i]);
5101 dev_info(tmp_adev->dev, "Dumping IP State Completed\n");
5102 }
5103
5104 if (need_full_reset)
5105 r = amdgpu_device_ip_suspend(adev);
5106 if (need_full_reset)
5107 set_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags);
5108 else
5109 clear_bit(AMDGPU_NEED_FULL_RESET,
5110 &reset_context->flags);
5111 }
5112
5113 return r;
5114}
5115
5116int amdgpu_device_reinit_after_reset(struct amdgpu_reset_context *reset_context)
5117{
5118 struct list_head *device_list_handle;
5119 bool full_reset, vram_lost = false;
5120 struct amdgpu_device *tmp_adev;
5121 int r, init_level;
5122
5123 device_list_handle = reset_context->reset_device_list;
5124
5125 if (!device_list_handle)
5126 return -EINVAL;
5127
5128 full_reset = test_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags);
5129
5130 /**
5131 * If it's reset on init, it's default init level, otherwise keep level
5132 * as recovery level.
5133 */
5134 if (reset_context->method == AMD_RESET_METHOD_ON_INIT)
5135 init_level = AMDGPU_INIT_LEVEL_DEFAULT;
5136 else
5137 init_level = AMDGPU_INIT_LEVEL_RESET_RECOVERY;
5138
5139 r = 0;
5140 list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
5141 amdgpu_set_init_level(tmp_adev, init_level);
5142 if (full_reset) {
5143 /* post card */
5144 amdgpu_reset_set_dpc_status(tmp_adev, false);
5145 amdgpu_ras_clear_err_state(tmp_adev);
5146 r = amdgpu_device_asic_init(tmp_adev);
5147 if (r) {
5148 dev_warn(tmp_adev->dev, "asic atom init failed!");
5149 } else {
5150 dev_info(tmp_adev->dev, "GPU reset succeeded, trying to resume\n");
5151
5152 r = amdgpu_device_ip_resume_phase1(tmp_adev);
5153 if (r)
5154 goto out;
5155
5156 vram_lost = amdgpu_device_check_vram_lost(tmp_adev);
5157
5158 if (!test_bit(AMDGPU_SKIP_COREDUMP, &reset_context->flags))
5159 amdgpu_coredump(tmp_adev, false, vram_lost, reset_context->job);
5160
5161 if (vram_lost) {
5162 dev_info(
5163 tmp_adev->dev,
5164 "VRAM is lost due to GPU reset!\n");
5165 amdgpu_inc_vram_lost(tmp_adev);
5166 }
5167
5168 r = amdgpu_device_fw_loading(tmp_adev);
5169 if (r)
5170 return r;
5171
5172 r = amdgpu_xcp_restore_partition_mode(
5173 tmp_adev->xcp_mgr);
5174 if (r)
5175 goto out;
5176
5177 r = amdgpu_device_ip_resume_phase2(tmp_adev);
5178 if (r)
5179 goto out;
5180
5181 r = amdgpu_device_ip_resume_phase3(tmp_adev);
5182 if (r)
5183 goto out;
5184
5185 if (vram_lost)
5186 amdgpu_device_fill_reset_magic(tmp_adev);
5187
5188 /*
5189 * Add this ASIC as tracked as reset was already
5190 * complete successfully.
5191 */
5192 amdgpu_register_gpu_instance(tmp_adev);
5193
5194 if (!reset_context->hive &&
5195 tmp_adev->gmc.xgmi.num_physical_nodes > 1)
5196 amdgpu_xgmi_add_device(tmp_adev);
5197
5198 r = amdgpu_device_ip_late_init(tmp_adev);
5199 if (r)
5200 goto out;
5201
5202 r = amdgpu_userq_post_reset(tmp_adev, vram_lost);
5203 if (r)
5204 goto out;
5205
5206 drm_client_dev_resume(adev_to_drm(tmp_adev));
5207
5208 /*
5209 * The GPU enters bad state once faulty pages
5210 * by ECC has reached the threshold, and ras
5211 * recovery is scheduled next. So add one check
5212 * here to break recovery if it indeed exceeds
5213 * bad page threshold, and remind user to
5214 * retire this GPU or setting one bigger
5215 * bad_page_threshold value to fix this once
5216 * probing driver again.
5217 */
5218 if (!amdgpu_ras_is_rma(tmp_adev)) {
5219 /* must succeed. */
5220 amdgpu_ras_resume(tmp_adev);
5221 } else {
5222 r = -EINVAL;
5223 goto out;
5224 }
5225
5226 /* Update PSP FW topology after reset */
5227 if (reset_context->hive &&
5228 tmp_adev->gmc.xgmi.num_physical_nodes > 1)
5229 r = amdgpu_xgmi_update_topology(
5230 reset_context->hive, tmp_adev);
5231 }
5232 }
5233
5234out:
5235 if (!r) {
5236 /* IP init is complete now, set level as default */
5237 amdgpu_set_init_level(tmp_adev,
5238 AMDGPU_INIT_LEVEL_DEFAULT);
5239 amdgpu_irq_gpu_reset_resume_helper(tmp_adev);
5240 r = amdgpu_ib_ring_tests(tmp_adev);
5241 if (r) {
5242 dev_err(tmp_adev->dev, "ib ring test failed (%d).\n", r);
5243 r = -EAGAIN;
5244 goto end;
5245 }
5246 }
5247
5248 if (r)
5249 tmp_adev->asic_reset_res = r;
5250 }
5251
5252end:
5253 return r;
5254}
5255
5256int amdgpu_do_asic_reset(struct list_head *device_list_handle,
5257 struct amdgpu_reset_context *reset_context)
5258{
5259 struct amdgpu_device *tmp_adev = NULL;
5260 bool need_full_reset, skip_hw_reset;
5261 int r = 0;
5262
5263 /* Try reset handler method first */
5264 tmp_adev = list_first_entry(device_list_handle, struct amdgpu_device,
5265 reset_list);
5266
5267 reset_context->reset_device_list = device_list_handle;
5268 r = amdgpu_reset_perform_reset(tmp_adev, reset_context);
5269 /* If reset handler not implemented, continue; otherwise return */
5270 if (r == -EOPNOTSUPP)
5271 r = 0;
5272 else
5273 return r;
5274
5275 /* Reset handler not implemented, use the default method */
5276 need_full_reset =
5277 test_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags);
5278 skip_hw_reset = test_bit(AMDGPU_SKIP_HW_RESET, &reset_context->flags);
5279
5280 /*
5281 * ASIC reset has to be done on all XGMI hive nodes ASAP
5282 * to allow proper links negotiation in FW (within 1 sec)
5283 */
5284 if (!skip_hw_reset && need_full_reset) {
5285 list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
5286 /* For XGMI run all resets in parallel to speed up the process */
5287 if (tmp_adev->gmc.xgmi.num_physical_nodes > 1) {
5288 if (!queue_work(system_dfl_wq,
5289 &tmp_adev->xgmi_reset_work))
5290 r = -EALREADY;
5291 } else
5292 r = amdgpu_asic_reset(tmp_adev);
5293
5294 if (r) {
5295 dev_err(tmp_adev->dev,
5296 "ASIC reset failed with error, %d for drm dev, %s",
5297 r, adev_to_drm(tmp_adev)->unique);
5298 goto out;
5299 }
5300 }
5301
5302 /* For XGMI wait for all resets to complete before proceed */
5303 if (!r) {
5304 list_for_each_entry(tmp_adev, device_list_handle,
5305 reset_list) {
5306 if (tmp_adev->gmc.xgmi.num_physical_nodes > 1) {
5307 flush_work(&tmp_adev->xgmi_reset_work);
5308 r = tmp_adev->asic_reset_res;
5309 if (r)
5310 break;
5311 }
5312 }
5313 }
5314 }
5315
5316 if (!r && amdgpu_ras_intr_triggered()) {
5317 list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
5318 amdgpu_ras_reset_error_count(tmp_adev,
5319 AMDGPU_RAS_BLOCK__MMHUB);
5320 }
5321
5322 amdgpu_ras_intr_cleared();
5323 }
5324
5325 r = amdgpu_device_reinit_after_reset(reset_context);
5326 if (r == -EAGAIN)
5327 set_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags);
5328 else
5329 clear_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags);
5330
5331out:
5332 return r;
5333}
5334
5335static void amdgpu_device_set_mp1_state(struct amdgpu_device *adev)
5336{
5337
5338 switch (amdgpu_asic_reset_method(adev)) {
5339 case AMD_RESET_METHOD_MODE1:
5340 case AMD_RESET_METHOD_LINK:
5341 adev->mp1_state = PP_MP1_STATE_SHUTDOWN;
5342 break;
5343 case AMD_RESET_METHOD_MODE2:
5344 adev->mp1_state = PP_MP1_STATE_RESET;
5345 break;
5346 default:
5347 adev->mp1_state = PP_MP1_STATE_NONE;
5348 break;
5349 }
5350}
5351
5352static void amdgpu_device_unset_mp1_state(struct amdgpu_device *adev)
5353{
5354 amdgpu_vf_error_trans_all(adev);
5355 adev->mp1_state = PP_MP1_STATE_NONE;
5356}
5357
5358static void amdgpu_device_resume_display_audio(struct amdgpu_device *adev)
5359{
5360 struct pci_dev *p = NULL;
5361
5362 p = pci_get_domain_bus_and_slot(pci_domain_nr(adev->pdev->bus),
5363 adev->pdev->bus->number, 1);
5364 if (p) {
5365 pm_runtime_enable(&(p->dev));
5366 pm_runtime_resume(&(p->dev));
5367 }
5368
5369 pci_dev_put(p);
5370}
5371
5372static int amdgpu_device_suspend_display_audio(struct amdgpu_device *adev)
5373{
5374 enum amd_reset_method reset_method;
5375 struct pci_dev *p = NULL;
5376 u64 expires;
5377
5378 /*
5379 * For now, only BACO and mode1 reset are confirmed
5380 * to suffer the audio issue without proper suspended.
5381 */
5382 reset_method = amdgpu_asic_reset_method(adev);
5383 if ((reset_method != AMD_RESET_METHOD_BACO) &&
5384 (reset_method != AMD_RESET_METHOD_MODE1))
5385 return -EINVAL;
5386
5387 p = pci_get_domain_bus_and_slot(pci_domain_nr(adev->pdev->bus),
5388 adev->pdev->bus->number, 1);
5389 if (!p)
5390 return -ENODEV;
5391
5392 expires = pm_runtime_autosuspend_expiration(&(p->dev));
5393 if (!expires)
5394 /*
5395 * If we cannot get the audio device autosuspend delay,
5396 * a fixed 4S interval will be used. Considering 3S is
5397 * the audio controller default autosuspend delay setting.
5398 * 4S used here is guaranteed to cover that.
5399 */
5400 expires = ktime_get_mono_fast_ns() + NSEC_PER_SEC * 4ULL;
5401
5402 while (!pm_runtime_status_suspended(&(p->dev))) {
5403 if (!pm_runtime_suspend(&(p->dev)))
5404 break;
5405
5406 if (expires < ktime_get_mono_fast_ns()) {
5407 dev_warn(adev->dev, "failed to suspend display audio\n");
5408 pci_dev_put(p);
5409 /* TODO: abort the succeeding gpu reset? */
5410 return -ETIMEDOUT;
5411 }
5412 }
5413
5414 pm_runtime_disable(&(p->dev));
5415
5416 pci_dev_put(p);
5417 return 0;
5418}
5419
5420static inline void amdgpu_device_stop_pending_resets(struct amdgpu_device *adev)
5421{
5422 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
5423
5424#if defined(CONFIG_DEBUG_FS)
5425 if (!amdgpu_sriov_vf(adev))
5426 cancel_work(&adev->reset_work);
5427#endif
5428 amdgpu_userq_mgr_cancel_reset_work(adev);
5429
5430 if (adev->kfd.dev)
5431 cancel_work(&adev->kfd.reset_work);
5432
5433 if (amdgpu_sriov_vf(adev))
5434 cancel_work(&adev->virt.flr_work);
5435
5436 if (con && adev->ras_enabled)
5437 cancel_work(&con->recovery_work);
5438
5439}
5440
5441static int amdgpu_device_health_check(struct list_head *device_list_handle)
5442{
5443 struct amdgpu_device *tmp_adev;
5444 int ret = 0;
5445
5446 list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
5447 ret |= amdgpu_device_bus_status_check(tmp_adev);
5448 }
5449
5450 return ret;
5451}
5452
5453static void amdgpu_device_recovery_prepare(struct amdgpu_device *adev,
5454 struct list_head *device_list,
5455 struct amdgpu_hive_info *hive)
5456{
5457 struct amdgpu_device *tmp_adev = NULL;
5458
5459 /*
5460 * Build list of devices to reset.
5461 * In case we are in XGMI hive mode, resort the device list
5462 * to put adev in the 1st position.
5463 */
5464 if (!amdgpu_sriov_vf(adev) && (adev->gmc.xgmi.num_physical_nodes > 1) && hive) {
5465 list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head) {
5466 list_add_tail(&tmp_adev->reset_list, device_list);
5467 if (adev->shutdown)
5468 tmp_adev->shutdown = true;
5469 }
5470 if (!list_is_first(&adev->reset_list, device_list))
5471 list_rotate_to_front(&adev->reset_list, device_list);
5472 } else {
5473 list_add_tail(&adev->reset_list, device_list);
5474 }
5475}
5476
5477static void amdgpu_device_recovery_get_reset_lock(struct amdgpu_device *adev,
5478 struct list_head *device_list)
5479{
5480 struct amdgpu_device *tmp_adev = NULL;
5481
5482 if (list_empty(device_list))
5483 return;
5484 tmp_adev =
5485 list_first_entry(device_list, struct amdgpu_device, reset_list);
5486 amdgpu_device_lock_reset_domain(tmp_adev->reset_domain);
5487}
5488
5489static void amdgpu_device_recovery_put_reset_lock(struct amdgpu_device *adev,
5490 struct list_head *device_list)
5491{
5492 struct amdgpu_device *tmp_adev = NULL;
5493
5494 if (list_empty(device_list))
5495 return;
5496 tmp_adev =
5497 list_first_entry(device_list, struct amdgpu_device, reset_list);
5498 amdgpu_device_unlock_reset_domain(tmp_adev->reset_domain);
5499}
5500
5501static void amdgpu_device_halt_activities(struct amdgpu_device *adev,
5502 struct amdgpu_job *job,
5503 struct amdgpu_reset_context *reset_context,
5504 struct list_head *device_list,
5505 struct amdgpu_hive_info *hive,
5506 bool need_emergency_restart)
5507{
5508 struct amdgpu_device *tmp_adev = NULL;
5509 int i;
5510
5511 /* block all schedulers and reset given job's ring */
5512 list_for_each_entry(tmp_adev, device_list, reset_list) {
5513 amdgpu_device_set_mp1_state(tmp_adev);
5514
5515 /*
5516 * Try to put the audio codec into suspend state
5517 * before gpu reset started.
5518 *
5519 * Due to the power domain of the graphics device
5520 * is shared with AZ power domain. Without this,
5521 * we may change the audio hardware from behind
5522 * the audio driver's back. That will trigger
5523 * some audio codec errors.
5524 */
5525 if (!amdgpu_device_suspend_display_audio(tmp_adev))
5526 tmp_adev->pcie_reset_ctx.audio_suspended = true;
5527
5528 amdgpu_ras_set_error_query_ready(tmp_adev, false);
5529
5530 cancel_delayed_work_sync(&tmp_adev->delayed_init_work);
5531
5532 amdgpu_amdkfd_pre_reset(tmp_adev, reset_context);
5533
5534 /*
5535 * Mark these ASICs to be reset as untracked first
5536 * And add them back after reset completed
5537 */
5538 amdgpu_unregister_gpu_instance(tmp_adev);
5539
5540 drm_client_dev_suspend(adev_to_drm(tmp_adev));
5541
5542 /* disable ras on ALL IPs */
5543 if (!need_emergency_restart && !amdgpu_reset_in_dpc(adev))
5544 amdgpu_ras_suspend(tmp_adev);
5545
5546 amdgpu_userq_pre_reset(tmp_adev);
5547
5548 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
5549 struct amdgpu_ring *ring = tmp_adev->rings[i];
5550
5551 if (!amdgpu_ring_sched_ready(ring))
5552 continue;
5553
5554 drm_sched_wqueue_stop(&ring->sched);
5555
5556 if (need_emergency_restart)
5557 amdgpu_job_stop_all_jobs_on_sched(&ring->sched);
5558 }
5559 atomic_inc(&tmp_adev->gpu_reset_counter);
5560 }
5561}
5562
5563static int amdgpu_device_asic_reset(struct amdgpu_device *adev,
5564 struct list_head *device_list,
5565 struct amdgpu_reset_context *reset_context)
5566{
5567 struct amdgpu_device *tmp_adev = NULL;
5568 int retry_limit = AMDGPU_MAX_RETRY_LIMIT;
5569 int r = 0;
5570
5571retry: /* Rest of adevs pre asic reset from XGMI hive. */
5572 list_for_each_entry(tmp_adev, device_list, reset_list) {
5573 r = amdgpu_device_pre_asic_reset(tmp_adev, reset_context);
5574 /*TODO Should we stop ?*/
5575 if (r) {
5576 dev_err(tmp_adev->dev, "GPU pre asic reset failed with err, %d for drm dev, %s ",
5577 r, adev_to_drm(tmp_adev)->unique);
5578 tmp_adev->asic_reset_res = r;
5579 }
5580 }
5581
5582 /* Actual ASIC resets if needed.*/
5583 /* Host driver will handle XGMI hive reset for SRIOV */
5584 if (amdgpu_sriov_vf(adev)) {
5585
5586 /* Bail out of reset early */
5587 if (amdgpu_ras_is_rma(adev))
5588 return -ENODEV;
5589
5590 if (amdgpu_ras_get_fed_status(adev) || amdgpu_virt_rcvd_ras_interrupt(adev)) {
5591 dev_dbg(adev->dev, "Detected RAS error, wait for FLR completion\n");
5592 amdgpu_ras_set_fed(adev, true);
5593 set_bit(AMDGPU_HOST_FLR, &reset_context->flags);
5594 }
5595
5596 r = amdgpu_device_reset_sriov(adev, reset_context);
5597 if (AMDGPU_RETRY_SRIOV_RESET(r) && (retry_limit--) > 0) {
5598 amdgpu_virt_release_full_gpu(adev, true);
5599 goto retry;
5600 }
5601 if (r)
5602 adev->asic_reset_res = r;
5603 } else {
5604 r = amdgpu_do_asic_reset(device_list, reset_context);
5605 if (r && r == -EAGAIN)
5606 goto retry;
5607 }
5608
5609 list_for_each_entry(tmp_adev, device_list, reset_list) {
5610 /*
5611 * Drop any pending non scheduler resets queued before reset is done.
5612 * Any reset scheduled after this point would be valid. Scheduler resets
5613 * were already dropped during drm_sched_stop and no new ones can come
5614 * in before drm_sched_start.
5615 */
5616 amdgpu_device_stop_pending_resets(tmp_adev);
5617 }
5618
5619 return r;
5620}
5621
5622static int amdgpu_device_sched_resume(struct list_head *device_list,
5623 struct amdgpu_reset_context *reset_context,
5624 bool job_signaled)
5625{
5626 struct amdgpu_device *tmp_adev = NULL;
5627 int i, r = 0;
5628
5629 /* Post ASIC reset for all devs .*/
5630 list_for_each_entry(tmp_adev, device_list, reset_list) {
5631
5632 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
5633 struct amdgpu_ring *ring = tmp_adev->rings[i];
5634
5635 if (!amdgpu_ring_sched_ready(ring))
5636 continue;
5637
5638 drm_sched_wqueue_start(&ring->sched);
5639 }
5640
5641 if (!drm_drv_uses_atomic_modeset(adev_to_drm(tmp_adev)) && !job_signaled)
5642 drm_helper_resume_force_mode(adev_to_drm(tmp_adev));
5643
5644 if (tmp_adev->asic_reset_res) {
5645 /* bad news, how to tell it to userspace ?
5646 * for ras error, we should report GPU bad status instead of
5647 * reset failure
5648 */
5649 if (reset_context->src != AMDGPU_RESET_SRC_RAS ||
5650 !amdgpu_ras_eeprom_check_err_threshold(tmp_adev))
5651 dev_info(
5652 tmp_adev->dev,
5653 "GPU reset(%d) failed with error %d\n",
5654 atomic_read(
5655 &tmp_adev->gpu_reset_counter),
5656 tmp_adev->asic_reset_res);
5657 amdgpu_vf_error_put(tmp_adev,
5658 AMDGIM_ERROR_VF_GPU_RESET_FAIL, 0,
5659 tmp_adev->asic_reset_res);
5660 if (!r)
5661 r = tmp_adev->asic_reset_res;
5662 tmp_adev->asic_reset_res = 0;
5663 } else {
5664 dev_info(tmp_adev->dev, "GPU reset(%d) succeeded!\n",
5665 atomic_read(&tmp_adev->gpu_reset_counter));
5666 if (amdgpu_acpi_smart_shift_update(tmp_adev,
5667 AMDGPU_SS_DEV_D0))
5668 dev_warn(tmp_adev->dev,
5669 "smart shift update failed\n");
5670 }
5671 }
5672
5673 return r;
5674}
5675
5676static void amdgpu_device_gpu_resume(struct amdgpu_device *adev,
5677 struct list_head *device_list,
5678 bool need_emergency_restart)
5679{
5680 struct amdgpu_device *tmp_adev = NULL;
5681
5682 list_for_each_entry(tmp_adev, device_list, reset_list) {
5683 /* unlock kfd: SRIOV would do it separately */
5684 if (!need_emergency_restart && !amdgpu_sriov_vf(tmp_adev))
5685 amdgpu_amdkfd_post_reset(tmp_adev);
5686
5687 /* kfd_post_reset will do nothing if kfd device is not initialized,
5688 * need to bring up kfd here if it's not be initialized before
5689 */
5690 if (!adev->kfd.init_complete)
5691 amdgpu_amdkfd_device_init(adev);
5692
5693 if (tmp_adev->pcie_reset_ctx.audio_suspended)
5694 amdgpu_device_resume_display_audio(tmp_adev);
5695
5696 amdgpu_device_unset_mp1_state(tmp_adev);
5697
5698 amdgpu_ras_set_error_query_ready(tmp_adev, true);
5699
5700 }
5701}
5702
5703
5704/**
5705 * amdgpu_device_gpu_recover - reset the asic and recover scheduler
5706 *
5707 * @adev: amdgpu_device pointer
5708 * @job: which job trigger hang
5709 * @reset_context: amdgpu reset context pointer
5710 *
5711 * Attempt to reset the GPU if it has hung (all asics).
5712 * Attempt to do soft-reset or full-reset and reinitialize Asic
5713 * Returns 0 for success or an error on failure.
5714 */
5715
5716int amdgpu_device_gpu_recover(struct amdgpu_device *adev,
5717 struct amdgpu_job *job,
5718 struct amdgpu_reset_context *reset_context)
5719{
5720 struct list_head device_list;
5721 bool job_signaled = false;
5722 struct amdgpu_hive_info *hive = NULL;
5723 int r = 0;
5724 bool need_emergency_restart = false;
5725 /* save the pasid here as the job may be freed before the end of the reset */
5726 int pasid = job ? job->pasid : -EINVAL;
5727
5728 /*
5729 * If it reaches here because of hang/timeout and a RAS error is
5730 * detected at the same time, let RAS recovery take care of it.
5731 */
5732 if (amdgpu_ras_is_err_state(adev, AMDGPU_RAS_BLOCK__ANY) &&
5733 !amdgpu_sriov_vf(adev) &&
5734 reset_context->src != AMDGPU_RESET_SRC_RAS) {
5735 dev_dbg(adev->dev,
5736 "Gpu recovery from source: %d yielding to RAS error recovery handling",
5737 reset_context->src);
5738 return 0;
5739 }
5740
5741 /*
5742 * Special case: RAS triggered and full reset isn't supported
5743 */
5744 need_emergency_restart = amdgpu_ras_need_emergency_restart(adev);
5745
5746 /*
5747 * Flush RAM to disk so that after reboot
5748 * the user can read log and see why the system rebooted.
5749 */
5750 if (need_emergency_restart && amdgpu_ras_get_context(adev) &&
5751 amdgpu_ras_get_context(adev)->reboot) {
5752 dev_warn(adev->dev, "Emergency reboot.");
5753
5754 ksys_sync_helper();
5755 emergency_restart();
5756 }
5757
5758 dev_info(adev->dev, "GPU %s begin!. Source: %d\n",
5759 need_emergency_restart ? "jobs stop" : "reset",
5760 reset_context->src);
5761
5762 if (!amdgpu_sriov_vf(adev))
5763 hive = amdgpu_get_xgmi_hive(adev);
5764 if (hive)
5765 mutex_lock(&hive->hive_lock);
5766
5767 reset_context->job = job;
5768 reset_context->hive = hive;
5769 INIT_LIST_HEAD(&device_list);
5770
5771 amdgpu_device_recovery_prepare(adev, &device_list, hive);
5772
5773 if (!amdgpu_sriov_vf(adev)) {
5774 r = amdgpu_device_health_check(&device_list);
5775 if (r)
5776 goto end_reset;
5777 }
5778
5779 /* Cannot be called after locking reset domain */
5780 amdgpu_ras_pre_reset(adev, &device_list);
5781
5782 /* We need to lock reset domain only once both for XGMI and single device */
5783 amdgpu_device_recovery_get_reset_lock(adev, &device_list);
5784
5785 /* unmap all the mappings of doorbell and framebuffer to prevent user space from
5786 * accessing them
5787 */
5788 unmap_mapping_range(adev->ddev.anon_inode->i_mapping, 0, 0, 1);
5789 amdgpu_amdkfd_clear_kfd_mapping(adev);
5790
5791 amdgpu_device_halt_activities(adev, job, reset_context, &device_list,
5792 hive, need_emergency_restart);
5793 if (need_emergency_restart)
5794 goto skip_sched_resume;
5795 /*
5796 * Must check guilty signal here since after this point all old
5797 * HW fences are force signaled.
5798 *
5799 * job->base holds a reference to parent fence
5800 */
5801 if (job && (dma_fence_get_status(&job->hw_fence->base) > 0)) {
5802 job_signaled = true;
5803 dev_info(adev->dev, "Guilty job already signaled, skipping HW reset");
5804 goto skip_hw_reset;
5805 }
5806
5807 r = amdgpu_device_asic_reset(adev, &device_list, reset_context);
5808 if (r)
5809 goto reset_unlock;
5810skip_hw_reset:
5811 r = amdgpu_device_sched_resume(&device_list, reset_context, job_signaled);
5812 if (r)
5813 goto reset_unlock;
5814skip_sched_resume:
5815 amdgpu_device_gpu_resume(adev, &device_list, need_emergency_restart);
5816reset_unlock:
5817 amdgpu_device_recovery_put_reset_lock(adev, &device_list);
5818 amdgpu_ras_post_reset(adev, &device_list);
5819end_reset:
5820 if (hive) {
5821 mutex_unlock(&hive->hive_lock);
5822 amdgpu_put_xgmi_hive(hive);
5823 }
5824
5825 if (r)
5826 dev_info(adev->dev, "GPU reset end with ret = %d\n", r);
5827
5828 atomic_set(&adev->reset_domain->reset_res, r);
5829
5830 if (!r) {
5831 struct amdgpu_task_info *ti = NULL;
5832
5833 /*
5834 * The job may already be freed at this point via the sched tdr workqueue so
5835 * use the cached pasid.
5836 */
5837 if (pasid >= 0)
5838 ti = amdgpu_vm_get_task_info_pasid(adev, pasid);
5839
5840 drm_dev_wedged_event(adev_to_drm(adev), DRM_WEDGE_RECOVERY_NONE,
5841 ti ? &ti->task : NULL);
5842
5843 amdgpu_vm_put_task_info(ti);
5844 }
5845
5846 return r;
5847}
5848
5849/**
5850 * amdgpu_device_partner_bandwidth - find the bandwidth of appropriate partner
5851 *
5852 * @adev: amdgpu_device pointer
5853 * @speed: pointer to the speed of the link
5854 * @width: pointer to the width of the link
5855 *
5856 * Evaluate the hierarchy to find the speed and bandwidth capabilities of the
5857 * first physical partner to an AMD dGPU.
5858 * This will exclude any virtual switches and links.
5859 */
5860static void amdgpu_device_partner_bandwidth(struct amdgpu_device *adev,
5861 enum pci_bus_speed *speed,
5862 enum pcie_link_width *width)
5863{
5864 if (!speed || !width)
5865 return;
5866
5867 *speed = PCI_SPEED_UNKNOWN;
5868 *width = PCIE_LNK_WIDTH_UNKNOWN;
5869
5870 if (amdgpu_device_pcie_dynamic_switching_supported(adev)) {
5871 if (adev->link_partner) {
5872 *speed = pcie_get_speed_cap(adev->link_partner);
5873 *width = pcie_get_width_cap(adev->link_partner);
5874 }
5875 } else {
5876 /* use the current speeds rather than max if switching is not supported */
5877 pcie_bandwidth_available(adev->pdev, NULL, speed, width);
5878 }
5879}
5880
5881/**
5882 * amdgpu_device_gpu_bandwidth - find the bandwidth of the GPU
5883 *
5884 * @adev: amdgpu_device pointer
5885 * @speed: pointer to the speed of the link
5886 * @width: pointer to the width of the link
5887 *
5888 * Evaluate the hierarchy to find the speed and bandwidth capabilities of the
5889 * AMD dGPU which may be a virtual upstream bridge.
5890 */
5891static void amdgpu_device_gpu_bandwidth(struct amdgpu_device *adev,
5892 enum pci_bus_speed *speed,
5893 enum pcie_link_width *width)
5894{
5895 if (!speed || !width)
5896 return;
5897
5898 *speed = pcie_get_speed_cap(adev->link_dev);
5899 *width = pcie_get_width_cap(adev->link_dev);
5900}
5901
5902/**
5903 * amdgpu_device_get_pcie_info - fence pcie info about the PCIE slot
5904 *
5905 * @adev: amdgpu_device pointer
5906 *
5907 * Fetches and stores in the driver the PCIE capabilities (gen speed
5908 * and lanes) of the slot the device is in. Handles APUs and
5909 * virtualized environments where PCIE config space may not be available.
5910 */
5911static void amdgpu_device_get_pcie_info(struct amdgpu_device *adev)
5912{
5913 enum pci_bus_speed speed_cap, platform_speed_cap;
5914 enum pcie_link_width platform_link_width, link_width;
5915
5916 if (amdgpu_pcie_gen_cap)
5917 adev->pm.pcie_gen_mask = amdgpu_pcie_gen_cap;
5918
5919 if (amdgpu_pcie_lane_cap)
5920 adev->pm.pcie_mlw_mask = amdgpu_pcie_lane_cap;
5921
5922 /* covers APUs as well */
5923 if (pci_is_root_bus(adev->pdev->bus) && !amdgpu_passthrough(adev)) {
5924 if (adev->pm.pcie_gen_mask == 0)
5925 adev->pm.pcie_gen_mask = AMDGPU_DEFAULT_PCIE_GEN_MASK;
5926 if (adev->pm.pcie_mlw_mask == 0)
5927 adev->pm.pcie_mlw_mask = AMDGPU_DEFAULT_PCIE_MLW_MASK;
5928 return;
5929 }
5930
5931 if (adev->pm.pcie_gen_mask && adev->pm.pcie_mlw_mask)
5932 return;
5933
5934 amdgpu_device_partner_bandwidth(adev, &platform_speed_cap,
5935 &platform_link_width);
5936 amdgpu_device_gpu_bandwidth(adev, &speed_cap, &link_width);
5937
5938 if (adev->pm.pcie_gen_mask == 0) {
5939 /* asic caps */
5940 if (speed_cap == PCI_SPEED_UNKNOWN) {
5941 adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5942 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 |
5943 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3);
5944 } else {
5945 if (speed_cap == PCIE_SPEED_32_0GT)
5946 adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5947 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 |
5948 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3 |
5949 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN4 |
5950 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN5);
5951 else if (speed_cap == PCIE_SPEED_16_0GT)
5952 adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5953 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 |
5954 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3 |
5955 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN4);
5956 else if (speed_cap == PCIE_SPEED_8_0GT)
5957 adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5958 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 |
5959 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3);
5960 else if (speed_cap == PCIE_SPEED_5_0GT)
5961 adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5962 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2);
5963 else
5964 adev->pm.pcie_gen_mask |= CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1;
5965 }
5966 /* platform caps */
5967 if (platform_speed_cap == PCI_SPEED_UNKNOWN) {
5968 adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5969 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2);
5970 } else {
5971 if (platform_speed_cap == PCIE_SPEED_32_0GT)
5972 adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5973 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 |
5974 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3 |
5975 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN4 |
5976 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN5);
5977 else if (platform_speed_cap == PCIE_SPEED_16_0GT)
5978 adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5979 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 |
5980 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3 |
5981 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN4);
5982 else if (platform_speed_cap == PCIE_SPEED_8_0GT)
5983 adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5984 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 |
5985 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3);
5986 else if (platform_speed_cap == PCIE_SPEED_5_0GT)
5987 adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5988 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2);
5989 else
5990 adev->pm.pcie_gen_mask |= CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1;
5991
5992 }
5993 }
5994 if (adev->pm.pcie_mlw_mask == 0) {
5995 /* asic caps */
5996 if (link_width == PCIE_LNK_WIDTH_UNKNOWN) {
5997 adev->pm.pcie_mlw_mask |= AMDGPU_DEFAULT_ASIC_PCIE_MLW_MASK;
5998 } else {
5999 switch (link_width) {
6000 case PCIE_LNK_X32:
6001 adev->pm.pcie_mlw_mask |= (CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X32 |
6002 CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X16 |
6003 CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X12 |
6004 CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X8 |
6005 CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X4 |
6006 CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X2 |
6007 CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X1);
6008 break;
6009 case PCIE_LNK_X16:
6010 adev->pm.pcie_mlw_mask |= (CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X16 |
6011 CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X12 |
6012 CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X8 |
6013 CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X4 |
6014 CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X2 |
6015 CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X1);
6016 break;
6017 case PCIE_LNK_X12:
6018 adev->pm.pcie_mlw_mask |= (CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X12 |
6019 CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X8 |
6020 CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X4 |
6021 CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X2 |
6022 CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X1);
6023 break;
6024 case PCIE_LNK_X8:
6025 adev->pm.pcie_mlw_mask |= (CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X8 |
6026 CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X4 |
6027 CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X2 |
6028 CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X1);
6029 break;
6030 case PCIE_LNK_X4:
6031 adev->pm.pcie_mlw_mask |= (CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X4 |
6032 CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X2 |
6033 CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X1);
6034 break;
6035 case PCIE_LNK_X2:
6036 adev->pm.pcie_mlw_mask |= (CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X2 |
6037 CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X1);
6038 break;
6039 case PCIE_LNK_X1:
6040 adev->pm.pcie_mlw_mask |= CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X1;
6041 break;
6042 default:
6043 break;
6044 }
6045 }
6046 /* platform caps */
6047 if (platform_link_width == PCIE_LNK_WIDTH_UNKNOWN) {
6048 adev->pm.pcie_mlw_mask |= AMDGPU_DEFAULT_PCIE_MLW_MASK;
6049 } else {
6050 switch (platform_link_width) {
6051 case PCIE_LNK_X32:
6052 adev->pm.pcie_mlw_mask |= (CAIL_PCIE_LINK_WIDTH_SUPPORT_X32 |
6053 CAIL_PCIE_LINK_WIDTH_SUPPORT_X16 |
6054 CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 |
6055 CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
6056 CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
6057 CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
6058 CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
6059 break;
6060 case PCIE_LNK_X16:
6061 adev->pm.pcie_mlw_mask |= (CAIL_PCIE_LINK_WIDTH_SUPPORT_X16 |
6062 CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 |
6063 CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
6064 CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
6065 CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
6066 CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
6067 break;
6068 case PCIE_LNK_X12:
6069 adev->pm.pcie_mlw_mask |= (CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 |
6070 CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
6071 CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
6072 CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
6073 CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
6074 break;
6075 case PCIE_LNK_X8:
6076 adev->pm.pcie_mlw_mask |= (CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
6077 CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
6078 CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
6079 CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
6080 break;
6081 case PCIE_LNK_X4:
6082 adev->pm.pcie_mlw_mask |= (CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
6083 CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
6084 CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
6085 break;
6086 case PCIE_LNK_X2:
6087 adev->pm.pcie_mlw_mask |= (CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
6088 CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
6089 break;
6090 case PCIE_LNK_X1:
6091 adev->pm.pcie_mlw_mask |= CAIL_PCIE_LINK_WIDTH_SUPPORT_X1;
6092 break;
6093 default:
6094 break;
6095 }
6096 }
6097 }
6098}
6099
6100/**
6101 * amdgpu_device_is_peer_accessible - Check peer access through PCIe BAR
6102 *
6103 * @adev: amdgpu_device pointer
6104 * @peer_adev: amdgpu_device pointer for peer device trying to access @adev
6105 *
6106 * Return true if @peer_adev can access (DMA) @adev through the PCIe
6107 * BAR, i.e. @adev is "large BAR" and the BAR matches the DMA mask of
6108 * @peer_adev.
6109 */
6110bool amdgpu_device_is_peer_accessible(struct amdgpu_device *adev,
6111 struct amdgpu_device *peer_adev)
6112{
6113#ifdef CONFIG_HSA_AMD_P2P
6114 bool p2p_access =
6115 !adev->gmc.xgmi.connected_to_cpu &&
6116 !(pci_p2pdma_distance(adev->pdev, peer_adev->dev, false) < 0);
6117 if (!p2p_access)
6118 dev_info(adev->dev, "PCIe P2P access from peer device %s is not supported by the chipset\n",
6119 pci_name(peer_adev->pdev));
6120
6121 bool is_large_bar = adev->gmc.visible_vram_size &&
6122 adev->gmc.real_vram_size == adev->gmc.visible_vram_size;
6123 bool p2p_addressable = amdgpu_device_check_iommu_remap(peer_adev);
6124
6125 if (!p2p_addressable) {
6126 uint64_t address_mask = peer_adev->dev->dma_mask ?
6127 ~*peer_adev->dev->dma_mask : ~((1ULL << 32) - 1);
6128 resource_size_t aper_limit =
6129 adev->gmc.aper_base + adev->gmc.aper_size - 1;
6130
6131 p2p_addressable = !(adev->gmc.aper_base & address_mask ||
6132 aper_limit & address_mask);
6133 }
6134 return pcie_p2p && is_large_bar && p2p_access && p2p_addressable;
6135#else
6136 return false;
6137#endif
6138}
6139
6140int amdgpu_device_baco_enter(struct amdgpu_device *adev)
6141{
6142 struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
6143
6144 if (!amdgpu_device_supports_baco(adev))
6145 return -ENOTSUPP;
6146
6147 if (ras && adev->ras_enabled &&
6148 adev->nbio.funcs->enable_doorbell_interrupt)
6149 adev->nbio.funcs->enable_doorbell_interrupt(adev, false);
6150
6151 return amdgpu_dpm_baco_enter(adev);
6152}
6153
6154int amdgpu_device_baco_exit(struct amdgpu_device *adev)
6155{
6156 struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
6157 int ret = 0;
6158
6159 if (!amdgpu_device_supports_baco(adev))
6160 return -ENOTSUPP;
6161
6162 ret = amdgpu_dpm_baco_exit(adev);
6163 if (ret)
6164 return ret;
6165
6166 if (ras && adev->ras_enabled &&
6167 adev->nbio.funcs->enable_doorbell_interrupt)
6168 adev->nbio.funcs->enable_doorbell_interrupt(adev, true);
6169
6170 if (amdgpu_passthrough(adev) && adev->nbio.funcs &&
6171 adev->nbio.funcs->clear_doorbell_interrupt)
6172 adev->nbio.funcs->clear_doorbell_interrupt(adev);
6173
6174 return 0;
6175}
6176
6177/**
6178 * amdgpu_pci_error_detected - Called when a PCI error is detected.
6179 * @pdev: PCI device struct
6180 * @state: PCI channel state
6181 *
6182 * Description: Called when a PCI error is detected.
6183 *
6184 * Return: PCI_ERS_RESULT_NEED_RESET or PCI_ERS_RESULT_DISCONNECT.
6185 */
6186pci_ers_result_t amdgpu_pci_error_detected(struct pci_dev *pdev, pci_channel_state_t state)
6187{
6188 struct drm_device *dev = pci_get_drvdata(pdev);
6189 struct amdgpu_device *adev = drm_to_adev(dev);
6190 struct amdgpu_hive_info *hive __free(xgmi_put_hive) =
6191 amdgpu_get_xgmi_hive(adev);
6192 struct amdgpu_reset_context reset_context;
6193 struct list_head device_list;
6194
6195 dev_info(adev->dev, "PCI error: detected callback!!\n");
6196
6197 adev->pci_channel_state = state;
6198
6199 switch (state) {
6200 case pci_channel_io_normal:
6201 dev_info(adev->dev, "pci_channel_io_normal: state(%d)!!\n", state);
6202 return PCI_ERS_RESULT_CAN_RECOVER;
6203 case pci_channel_io_frozen:
6204 /* Fatal error, prepare for slot reset */
6205 dev_info(adev->dev, "pci_channel_io_frozen: state(%d)!!\n", state);
6206 if (hive) {
6207 /* Hive devices should be able to support FW based
6208 * link reset on other devices, if not return.
6209 */
6210 if (!amdgpu_dpm_is_link_reset_supported(adev)) {
6211 dev_warn(adev->dev,
6212 "No support for XGMI hive yet...\n");
6213 return PCI_ERS_RESULT_DISCONNECT;
6214 }
6215 /* Set dpc status only if device is part of hive
6216 * Non-hive devices should be able to recover after
6217 * link reset.
6218 */
6219 amdgpu_reset_set_dpc_status(adev, true);
6220
6221 mutex_lock(&hive->hive_lock);
6222 } else {
6223 if (amdgpu_device_bus_status_check(adev))
6224 amdgpu_reset_set_dpc_status(adev, true);
6225 }
6226 memset(&reset_context, 0, sizeof(reset_context));
6227 INIT_LIST_HEAD(&device_list);
6228
6229 amdgpu_device_recovery_prepare(adev, &device_list, hive);
6230 amdgpu_device_recovery_get_reset_lock(adev, &device_list);
6231 amdgpu_device_halt_activities(adev, NULL, &reset_context, &device_list,
6232 hive, false);
6233 if (hive)
6234 mutex_unlock(&hive->hive_lock);
6235 return PCI_ERS_RESULT_NEED_RESET;
6236 case pci_channel_io_perm_failure:
6237 /* Permanent error, prepare for device removal */
6238 dev_info(adev->dev, "pci_channel_io_perm_failure: state(%d)!!\n", state);
6239 return PCI_ERS_RESULT_DISCONNECT;
6240 }
6241
6242 return PCI_ERS_RESULT_NEED_RESET;
6243}
6244
6245/**
6246 * amdgpu_pci_mmio_enabled - Enable MMIO and dump debug registers
6247 * @pdev: pointer to PCI device
6248 */
6249pci_ers_result_t amdgpu_pci_mmio_enabled(struct pci_dev *pdev)
6250{
6251 struct drm_device *dev = pci_get_drvdata(pdev);
6252 struct amdgpu_device *adev = drm_to_adev(dev);
6253
6254 dev_info(adev->dev, "PCI error: mmio enabled callback!!\n");
6255
6256 /* TODO - dump whatever for debugging purposes */
6257
6258 /* This called only if amdgpu_pci_error_detected returns
6259 * PCI_ERS_RESULT_CAN_RECOVER. Read/write to the device still
6260 * works, no need to reset slot.
6261 */
6262
6263 return PCI_ERS_RESULT_RECOVERED;
6264}
6265
6266/**
6267 * amdgpu_pci_slot_reset - Called when PCI slot has been reset.
6268 * @pdev: PCI device struct
6269 *
6270 * Description: This routine is called by the pci error recovery
6271 * code after the PCI slot has been reset, just before we
6272 * should resume normal operations.
6273 */
6274pci_ers_result_t amdgpu_pci_slot_reset(struct pci_dev *pdev)
6275{
6276 struct drm_device *dev = pci_get_drvdata(pdev);
6277 struct amdgpu_device *adev = drm_to_adev(dev);
6278 struct amdgpu_reset_context reset_context;
6279 struct amdgpu_device *tmp_adev;
6280 struct amdgpu_hive_info *hive;
6281 struct list_head device_list;
6282 struct pci_dev *link_dev;
6283 int r = 0, i, timeout;
6284 u32 memsize;
6285 u16 status;
6286
6287 dev_info(adev->dev, "PCI error: slot reset callback!!\n");
6288
6289 memset(&reset_context, 0, sizeof(reset_context));
6290 INIT_LIST_HEAD(&device_list);
6291 hive = amdgpu_get_xgmi_hive(adev);
6292 if (hive) {
6293 mutex_lock(&hive->hive_lock);
6294 list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head)
6295 list_add_tail(&tmp_adev->reset_list, &device_list);
6296 } else {
6297 list_add_tail(&adev->reset_list, &device_list);
6298 }
6299
6300 if (adev->pcie_reset_ctx.swus)
6301 link_dev = adev->pcie_reset_ctx.swus;
6302 else
6303 link_dev = adev->pdev;
6304 /* wait for asic to come out of reset, timeout = 10s */
6305 timeout = 10000;
6306 do {
6307 usleep_range(10000, 10500);
6308 r = pci_read_config_word(link_dev, PCI_VENDOR_ID, &status);
6309 timeout -= 10;
6310 } while (timeout > 0 && (status != PCI_VENDOR_ID_ATI) &&
6311 (status != PCI_VENDOR_ID_AMD));
6312
6313 if ((status != PCI_VENDOR_ID_ATI) && (status != PCI_VENDOR_ID_AMD)) {
6314 r = -ETIME;
6315 goto out;
6316 }
6317
6318 amdgpu_device_load_switch_state(adev);
6319 /* Restore PCI confspace */
6320 amdgpu_device_load_pci_state(pdev);
6321
6322 /* confirm ASIC came out of reset */
6323 for (i = 0; i < adev->usec_timeout; i++) {
6324 memsize = amdgpu_asic_get_config_memsize(adev);
6325
6326 if (memsize != 0xffffffff)
6327 break;
6328 udelay(1);
6329 }
6330 if (memsize == 0xffffffff) {
6331 r = -ETIME;
6332 goto out;
6333 }
6334
6335 reset_context.method = AMD_RESET_METHOD_NONE;
6336 reset_context.reset_req_dev = adev;
6337 set_bit(AMDGPU_NEED_FULL_RESET, &reset_context.flags);
6338 set_bit(AMDGPU_SKIP_COREDUMP, &reset_context.flags);
6339
6340 if (hive) {
6341 reset_context.hive = hive;
6342 list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head)
6343 tmp_adev->pcie_reset_ctx.in_link_reset = true;
6344 } else {
6345 adev->pcie_reset_ctx.in_link_reset = true;
6346 set_bit(AMDGPU_SKIP_HW_RESET, &reset_context.flags);
6347 }
6348
6349 r = amdgpu_device_asic_reset(adev, &device_list, &reset_context);
6350out:
6351 if (!r) {
6352 if (amdgpu_device_cache_pci_state(adev->pdev))
6353 pci_restore_state(adev->pdev);
6354 dev_info(adev->dev, "PCIe error recovery succeeded\n");
6355 } else {
6356 dev_err(adev->dev, "PCIe error recovery failed, err:%d\n", r);
6357 if (hive) {
6358 list_for_each_entry(tmp_adev, &device_list, reset_list)
6359 amdgpu_device_unset_mp1_state(tmp_adev);
6360 }
6361 amdgpu_device_recovery_put_reset_lock(adev, &device_list);
6362 }
6363
6364 if (hive) {
6365 mutex_unlock(&hive->hive_lock);
6366 amdgpu_put_xgmi_hive(hive);
6367 }
6368
6369 return r ? PCI_ERS_RESULT_DISCONNECT : PCI_ERS_RESULT_RECOVERED;
6370}
6371
6372/**
6373 * amdgpu_pci_resume() - resume normal ops after PCI reset
6374 * @pdev: pointer to PCI device
6375 *
6376 * Called when the error recovery driver tells us that its
6377 * OK to resume normal operation.
6378 */
6379void amdgpu_pci_resume(struct pci_dev *pdev)
6380{
6381 struct drm_device *dev = pci_get_drvdata(pdev);
6382 struct amdgpu_device *adev = drm_to_adev(dev);
6383 struct list_head device_list;
6384 struct amdgpu_hive_info *hive = NULL;
6385 struct amdgpu_device *tmp_adev = NULL;
6386
6387 dev_info(adev->dev, "PCI error: resume callback!!\n");
6388
6389 /* Only continue execution for the case of pci_channel_io_frozen */
6390 if (adev->pci_channel_state != pci_channel_io_frozen)
6391 return;
6392
6393 INIT_LIST_HEAD(&device_list);
6394
6395 hive = amdgpu_get_xgmi_hive(adev);
6396 if (hive) {
6397 mutex_lock(&hive->hive_lock);
6398 list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head) {
6399 tmp_adev->pcie_reset_ctx.in_link_reset = false;
6400 list_add_tail(&tmp_adev->reset_list, &device_list);
6401 }
6402 } else {
6403 adev->pcie_reset_ctx.in_link_reset = false;
6404 list_add_tail(&adev->reset_list, &device_list);
6405 }
6406 amdgpu_device_sched_resume(&device_list, NULL, NULL);
6407 amdgpu_device_gpu_resume(adev, &device_list, false);
6408 amdgpu_device_recovery_put_reset_lock(adev, &device_list);
6409
6410 if (hive) {
6411 mutex_unlock(&hive->hive_lock);
6412 amdgpu_put_xgmi_hive(hive);
6413 }
6414}
6415
6416static void amdgpu_device_cache_switch_state(struct amdgpu_device *adev)
6417{
6418 struct pci_dev *swus, *swds;
6419 int r;
6420
6421 swds = pci_upstream_bridge(adev->pdev);
6422 if (!swds || swds->vendor != PCI_VENDOR_ID_ATI ||
6423 pci_pcie_type(swds) != PCI_EXP_TYPE_DOWNSTREAM)
6424 return;
6425 swus = pci_upstream_bridge(swds);
6426 if (!swus ||
6427 (swus->vendor != PCI_VENDOR_ID_ATI &&
6428 swus->vendor != PCI_VENDOR_ID_AMD) ||
6429 pci_pcie_type(swus) != PCI_EXP_TYPE_UPSTREAM)
6430 return;
6431
6432 /* If already saved, return */
6433 if (adev->pcie_reset_ctx.swus)
6434 return;
6435 /* Upstream bridge is ATI, assume it's SWUS/DS architecture */
6436 r = pci_save_state(swds);
6437 if (r)
6438 return;
6439 adev->pcie_reset_ctx.swds_pcistate = pci_store_saved_state(swds);
6440
6441 r = pci_save_state(swus);
6442 if (r)
6443 return;
6444 adev->pcie_reset_ctx.swus_pcistate = pci_store_saved_state(swus);
6445
6446 adev->pcie_reset_ctx.swus = swus;
6447}
6448
6449static void amdgpu_device_load_switch_state(struct amdgpu_device *adev)
6450{
6451 struct pci_dev *pdev;
6452 int r;
6453
6454 if (!adev->pcie_reset_ctx.swds_pcistate ||
6455 !adev->pcie_reset_ctx.swus_pcistate)
6456 return;
6457
6458 pdev = adev->pcie_reset_ctx.swus;
6459 r = pci_load_saved_state(pdev, adev->pcie_reset_ctx.swus_pcistate);
6460 if (!r) {
6461 pci_restore_state(pdev);
6462 } else {
6463 dev_warn(adev->dev, "Failed to load SWUS state, err:%d\n", r);
6464 return;
6465 }
6466
6467 pdev = pci_upstream_bridge(adev->pdev);
6468 r = pci_load_saved_state(pdev, adev->pcie_reset_ctx.swds_pcistate);
6469 if (!r)
6470 pci_restore_state(pdev);
6471 else
6472 dev_warn(adev->dev, "Failed to load SWDS state, err:%d\n", r);
6473}
6474
6475bool amdgpu_device_cache_pci_state(struct pci_dev *pdev)
6476{
6477 struct drm_device *dev = pci_get_drvdata(pdev);
6478 struct amdgpu_device *adev = drm_to_adev(dev);
6479 int r;
6480
6481 if (amdgpu_sriov_vf(adev))
6482 return false;
6483
6484 r = pci_save_state(pdev);
6485 if (!r) {
6486 kfree(adev->pci_state);
6487
6488 adev->pci_state = pci_store_saved_state(pdev);
6489
6490 if (!adev->pci_state) {
6491 dev_err(adev->dev, "Failed to store PCI saved state");
6492 return false;
6493 }
6494 } else {
6495 dev_warn(adev->dev, "Failed to save PCI state, err:%d\n", r);
6496 return false;
6497 }
6498
6499 amdgpu_device_cache_switch_state(adev);
6500
6501 return true;
6502}
6503
6504bool amdgpu_device_load_pci_state(struct pci_dev *pdev)
6505{
6506 struct drm_device *dev = pci_get_drvdata(pdev);
6507 struct amdgpu_device *adev = drm_to_adev(dev);
6508 int r;
6509
6510 if (!adev->pci_state)
6511 return false;
6512
6513 r = pci_load_saved_state(pdev, adev->pci_state);
6514
6515 if (!r) {
6516 pci_restore_state(pdev);
6517 } else {
6518 dev_warn(adev->dev, "Failed to load PCI state, err:%d\n", r);
6519 return false;
6520 }
6521
6522 return true;
6523}
6524
6525void amdgpu_device_flush_hdp(struct amdgpu_device *adev,
6526 struct amdgpu_ring *ring)
6527{
6528#ifdef CONFIG_X86_64
6529 if ((adev->flags & AMD_IS_APU) && !amdgpu_passthrough(adev))
6530 return;
6531#endif
6532 if (adev->gmc.xgmi.connected_to_cpu)
6533 return;
6534
6535 if (ring && ring->funcs->emit_hdp_flush) {
6536 amdgpu_ring_emit_hdp_flush(ring);
6537 return;
6538 }
6539
6540 if (!ring && amdgpu_sriov_runtime(adev)) {
6541 if (!amdgpu_kiq_hdp_flush(adev))
6542 return;
6543 }
6544
6545 amdgpu_hdp_flush(adev, ring);
6546}
6547
6548void amdgpu_device_invalidate_hdp(struct amdgpu_device *adev,
6549 struct amdgpu_ring *ring)
6550{
6551#ifdef CONFIG_X86_64
6552 if ((adev->flags & AMD_IS_APU) && !amdgpu_passthrough(adev))
6553 return;
6554#endif
6555 if (adev->gmc.xgmi.connected_to_cpu)
6556 return;
6557
6558 amdgpu_hdp_invalidate(adev, ring);
6559}
6560
6561int amdgpu_in_reset(struct amdgpu_device *adev)
6562{
6563 return atomic_read(&adev->reset_domain->in_gpu_reset);
6564}
6565
6566/**
6567 * amdgpu_device_halt() - bring hardware to some kind of halt state
6568 *
6569 * @adev: amdgpu_device pointer
6570 *
6571 * Bring hardware to some kind of halt state so that no one can touch it
6572 * any more. It will help to maintain error context when error occurred.
6573 * Compare to a simple hang, the system will keep stable at least for SSH
6574 * access. Then it should be trivial to inspect the hardware state and
6575 * see what's going on. Implemented as following:
6576 *
6577 * 1. drm_dev_unplug() makes device inaccessible to user space(IOCTLs, etc),
6578 * clears all CPU mappings to device, disallows remappings through page faults
6579 * 2. amdgpu_irq_disable_all() disables all interrupts
6580 * 3. amdgpu_fence_driver_hw_fini() signals all HW fences
6581 * 4. set adev->no_hw_access to avoid potential crashes after setp 5
6582 * 5. amdgpu_device_unmap_mmio() clears all MMIO mappings
6583 * 6. pci_disable_device() and pci_wait_for_pending_transaction()
6584 * flush any in flight DMA operations
6585 */
6586void amdgpu_device_halt(struct amdgpu_device *adev)
6587{
6588 struct pci_dev *pdev = adev->pdev;
6589 struct drm_device *ddev = adev_to_drm(adev);
6590
6591 amdgpu_xcp_dev_unplug(adev);
6592 drm_dev_unplug(ddev);
6593
6594 amdgpu_device_set_pg_state(adev, AMD_PG_STATE_UNGATE);
6595 amdgpu_device_set_cg_state(adev, AMD_CG_STATE_UNGATE);
6596
6597 amdgpu_irq_disable_all(adev);
6598
6599 amdgpu_fence_driver_hw_fini(adev);
6600
6601 adev->no_hw_access = true;
6602
6603 amdgpu_device_unmap_mmio(adev);
6604
6605 pci_disable_device(pdev);
6606 pci_wait_for_pending_transaction(pdev);
6607}
6608
6609/**
6610 * amdgpu_device_get_gang - return a reference to the current gang
6611 * @adev: amdgpu_device pointer
6612 *
6613 * Returns: A new reference to the current gang leader.
6614 */
6615struct dma_fence *amdgpu_device_get_gang(struct amdgpu_device *adev)
6616{
6617 struct dma_fence *fence;
6618
6619 rcu_read_lock();
6620 fence = dma_fence_get_rcu_safe(&adev->gang_submit);
6621 rcu_read_unlock();
6622 return fence;
6623}
6624
6625/**
6626 * amdgpu_device_switch_gang - switch to a new gang
6627 * @adev: amdgpu_device pointer
6628 * @gang: the gang to switch to
6629 *
6630 * Try to switch to a new gang.
6631 * Returns: NULL if we switched to the new gang or a reference to the current
6632 * gang leader.
6633 */
6634struct dma_fence *amdgpu_device_switch_gang(struct amdgpu_device *adev,
6635 struct dma_fence *gang)
6636{
6637 struct dma_fence *old = NULL;
6638
6639 dma_fence_get(gang);
6640 do {
6641 dma_fence_put(old);
6642 old = amdgpu_device_get_gang(adev);
6643 if (old == gang)
6644 break;
6645
6646 if (!dma_fence_is_signaled(old)) {
6647 dma_fence_put(gang);
6648 return old;
6649 }
6650
6651 } while (cmpxchg((struct dma_fence __force **)&adev->gang_submit,
6652 old, gang) != old);
6653
6654 /*
6655 * Drop it once for the exchanged reference in adev and once for the
6656 * thread local reference acquired in amdgpu_device_get_gang().
6657 */
6658 dma_fence_put(old);
6659 dma_fence_put(old);
6660 return NULL;
6661}
6662
6663/**
6664 * amdgpu_device_enforce_isolation - enforce HW isolation
6665 * @adev: the amdgpu device pointer
6666 * @ring: the HW ring the job is supposed to run on
6667 * @job: the job which is about to be pushed to the HW ring
6668 *
6669 * Makes sure that only one client at a time can use the GFX block.
6670 * Returns: The dependency to wait on before the job can be pushed to the HW.
6671 * The function is called multiple times until NULL is returned.
6672 */
6673struct dma_fence *amdgpu_device_enforce_isolation(struct amdgpu_device *adev,
6674 struct amdgpu_ring *ring,
6675 struct amdgpu_job *job)
6676{
6677 struct drm_sched_fence *f = job->base.s_fence;
6678 struct amdgpu_isolation *isolation;
6679 struct dma_fence *dep;
6680 void *owner;
6681 int r;
6682
6683 /*
6684 * For now enforce isolation only for the GFX block since we only need
6685 * the cleaner shader on those rings.
6686 */
6687 if (ring->funcs->type != AMDGPU_RING_TYPE_GFX &&
6688 ring->funcs->type != AMDGPU_RING_TYPE_COMPUTE)
6689 return NULL;
6690
6691 isolation = &adev->isolation[ring->xcp_id == AMDGPU_XCP_NO_PARTITION ?
6692 0 : ring->xcp_id];
6693
6694 /*
6695 * All submissions where enforce isolation is false are handled as if
6696 * they come from a single client. Use ~0l as the owner to distinct it
6697 * from kernel submissions where the owner is NULL.
6698 */
6699 owner = job->enforce_isolation ? f->owner : (void *)~0l;
6700
6701 mutex_lock(&adev->enforce_isolation_mutex);
6702
6703 /*
6704 * The "spearhead" submission is the first one which changes the
6705 * ownership to its client. We always need to wait for it to be
6706 * pushed to the HW before proceeding with anything.
6707 */
6708 if (&f->scheduled != isolation->spearhead &&
6709 !dma_fence_is_signaled(isolation->spearhead)) {
6710 dep = isolation->spearhead;
6711 goto out_grab_ref;
6712 }
6713
6714 if (isolation->owner != owner) {
6715
6716 /*
6717 * Wait for any gang to be assembled before switching to a
6718 * different owner or otherwise we could deadlock the
6719 * submissions.
6720 */
6721 if (!job->gang_submit) {
6722 dep = amdgpu_device_get_gang(adev);
6723 if (!dma_fence_is_signaled(dep))
6724 goto out_return_dep;
6725 dma_fence_put(dep);
6726 }
6727
6728 dma_fence_put(isolation->spearhead);
6729 isolation->spearhead = dma_fence_get(&f->scheduled);
6730 amdgpu_sync_move(&isolation->active, &isolation->prev);
6731 trace_amdgpu_isolation(isolation->owner, owner);
6732 isolation->owner = owner;
6733 }
6734
6735 /*
6736 * Specifying the ring here helps to pipeline submissions even when
6737 * isolation is enabled. If that is not desired for testing NULL can be
6738 * used instead of the ring to enforce a CPU round trip while switching
6739 * between clients.
6740 */
6741 dep = amdgpu_sync_peek_fence(&isolation->prev, ring);
6742 r = amdgpu_sync_fence(&isolation->active, &f->finished, GFP_NOWAIT);
6743 if (r)
6744 dev_warn(adev->dev, "OOM tracking isolation\n");
6745
6746out_grab_ref:
6747 dma_fence_get(dep);
6748out_return_dep:
6749 mutex_unlock(&adev->enforce_isolation_mutex);
6750 return dep;
6751}
6752
6753bool amdgpu_device_has_display_hardware(struct amdgpu_device *adev)
6754{
6755 switch (adev->asic_type) {
6756#ifdef CONFIG_DRM_AMDGPU_SI
6757 case CHIP_HAINAN:
6758#endif
6759 case CHIP_TOPAZ:
6760 /* chips with no display hardware */
6761 return false;
6762#ifdef CONFIG_DRM_AMDGPU_SI
6763 case CHIP_TAHITI:
6764 case CHIP_PITCAIRN:
6765 case CHIP_VERDE:
6766 case CHIP_OLAND:
6767#endif
6768#ifdef CONFIG_DRM_AMDGPU_CIK
6769 case CHIP_BONAIRE:
6770 case CHIP_HAWAII:
6771 case CHIP_KAVERI:
6772 case CHIP_KABINI:
6773 case CHIP_MULLINS:
6774#endif
6775 case CHIP_TONGA:
6776 case CHIP_FIJI:
6777 case CHIP_POLARIS10:
6778 case CHIP_POLARIS11:
6779 case CHIP_POLARIS12:
6780 case CHIP_VEGAM:
6781 case CHIP_CARRIZO:
6782 case CHIP_STONEY:
6783 /* chips with display hardware */
6784 return true;
6785 default:
6786 /* IP discovery */
6787 if (!amdgpu_ip_version(adev, DCE_HWIP, 0) ||
6788 (adev->harvest_ip_mask & AMD_HARVEST_IP_DMU_MASK))
6789 return false;
6790 return true;
6791 }
6792}
6793
6794ssize_t amdgpu_get_soft_full_reset_mask(struct amdgpu_ring *ring)
6795{
6796 ssize_t size = 0;
6797
6798 if (!ring || !ring->adev)
6799 return size;
6800
6801 if (amdgpu_device_should_recover_gpu(ring->adev))
6802 size |= AMDGPU_RESET_TYPE_FULL;
6803
6804 if (unlikely(!ring->adev->debug_disable_soft_recovery) &&
6805 !amdgpu_sriov_vf(ring->adev) && ring->funcs->soft_recovery)
6806 size |= AMDGPU_RESET_TYPE_SOFT_RECOVERY;
6807
6808 return size;
6809}
6810
6811ssize_t amdgpu_show_reset_mask(char *buf, uint32_t supported_reset)
6812{
6813 ssize_t size = 0;
6814
6815 if (supported_reset == 0) {
6816 size += sysfs_emit_at(buf, size, "unsupported");
6817 size += sysfs_emit_at(buf, size, "\n");
6818 return size;
6819
6820 }
6821
6822 if (supported_reset & AMDGPU_RESET_TYPE_SOFT_RECOVERY)
6823 size += sysfs_emit_at(buf, size, "soft_recovery ");
6824
6825 if (supported_reset & AMDGPU_RESET_TYPE_PER_QUEUE)
6826 size += sysfs_emit_at(buf, size, "queue ");
6827
6828 if (supported_reset & AMDGPU_RESET_TYPE_PER_PIPE)
6829 size += sysfs_emit_at(buf, size, "pipe ");
6830
6831 if (supported_reset & AMDGPU_RESET_TYPE_FULL)
6832 size += sysfs_emit_at(buf, size, "full ");
6833
6834 size += sysfs_emit_at(buf, size, "\n");
6835 return size;
6836}
6837
6838void amdgpu_device_set_uid(struct amdgpu_uid *uid_info,
6839 enum amdgpu_uid_type type, uint8_t inst,
6840 uint64_t uid)
6841{
6842 if (!uid_info)
6843 return;
6844
6845 if (type >= AMDGPU_UID_TYPE_MAX) {
6846 dev_err_once(uid_info->adev->dev, "Invalid UID type %d\n",
6847 type);
6848 return;
6849 }
6850
6851 if (inst >= AMDGPU_UID_INST_MAX) {
6852 dev_err_once(uid_info->adev->dev, "Invalid UID instance %d\n",
6853 inst);
6854 return;
6855 }
6856
6857 if (uid_info->uid[type][inst] != 0) {
6858 dev_warn_once(
6859 uid_info->adev->dev,
6860 "Overwriting existing UID %llu for type %d instance %d\n",
6861 uid_info->uid[type][inst], type, inst);
6862 }
6863
6864 uid_info->uid[type][inst] = uid;
6865}
6866
6867u64 amdgpu_device_get_uid(struct amdgpu_uid *uid_info,
6868 enum amdgpu_uid_type type, uint8_t inst)
6869{
6870 if (!uid_info)
6871 return 0;
6872
6873 if (type >= AMDGPU_UID_TYPE_MAX) {
6874 dev_err_once(uid_info->adev->dev, "Invalid UID type %d\n",
6875 type);
6876 return 0;
6877 }
6878
6879 if (inst >= AMDGPU_UID_INST_MAX) {
6880 dev_err_once(uid_info->adev->dev, "Invalid UID instance %d\n",
6881 inst);
6882 return 0;
6883 }
6884
6885 return uid_info->uid[type][inst];
6886}