projects/linux·blob master · refs

1563 lines · 39.9 KiB

1// SPDX-License-Identifier: MIT
2/*
3 * Copyright © 2021 Intel Corporation
4 */
5
6#include "xe_device.h"
7
8#include <linux/aperture.h>
9#include <linux/delay.h>
10#include <linux/fault-inject.h>
11#include <linux/units.h>
12
13#include <drm/drm_client.h>
14#include <drm/drm_gem_ttm_helper.h>
15#include <drm/drm_ioctl.h>
16#include <drm/drm_managed.h>
17#include <drm/drm_pagemap_util.h>
18#include <drm/drm_print.h>
19#include <kunit/static_stub.h>
20#include <uapi/drm/xe_drm.h>
21
22#include "display/xe_display.h"
23#include "instructions/xe_gpu_commands.h"
24#include "regs/xe_gt_regs.h"
25#include "regs/xe_regs.h"
26#include "xe_bo.h"
27#include "xe_bo_evict.h"
28#include "xe_configfs.h"
29#include "xe_debugfs.h"
30#include "xe_defaults.h"
31#include "xe_devcoredump.h"
32#include "xe_device_sysfs.h"
33#include "xe_dma_buf.h"
34#include "xe_drm_client.h"
35#include "xe_drv.h"
36#include "xe_exec.h"
37#include "xe_exec_queue.h"
38#include "xe_force_wake.h"
39#include "xe_ggtt.h"
40#include "xe_gt.h"
41#include "xe_gt_mcr.h"
42#include "xe_gt_printk.h"
43#include "xe_gt_sriov_vf.h"
44#include "xe_guc.h"
45#include "xe_guc_pc.h"
46#include "xe_hw_engine_group.h"
47#include "xe_hwmon.h"
48#include "xe_i2c.h"
49#include "xe_irq.h"
50#include "xe_late_bind_fw.h"
51#include "xe_mmio.h"
52#include "xe_module.h"
53#include "xe_nvm.h"
54#include "xe_oa.h"
55#include "xe_observation.h"
56#include "xe_pagefault.h"
57#include "xe_pat.h"
58#include "xe_pcode.h"
59#include "xe_pm.h"
60#include "xe_pmu.h"
61#include "xe_psmi.h"
62#include "xe_pxp.h"
63#include "xe_query.h"
64#include "xe_ras.h"
65#include "xe_shrinker.h"
66#include "xe_soc_remapper.h"
67#include "xe_survivability_mode.h"
68#include "xe_sriov.h"
69#include "xe_svm.h"
70#include "xe_sysctrl.h"
71#include "xe_tile.h"
72#include "xe_ttm_stolen_mgr.h"
73#include "xe_ttm_sys_mgr.h"
74#include "xe_vm.h"
75#include "xe_vm_madvise.h"
76#include "xe_vram.h"
77#include "xe_vram_types.h"
78#include "xe_vsec.h"
79#include "xe_wait_user_fence.h"
80#include "xe_wa.h"
81
82#include <generated/xe_device_wa_oob.h>
83#include <generated/xe_wa_oob.h>
84
85static int xe_file_open(struct drm_device *dev, struct drm_file *file)
86{
87 struct xe_device *xe = to_xe_device(dev);
88 struct xe_drm_client *client;
89 struct xe_file *xef;
90 int ret = -ENOMEM;
91 struct task_struct *task = NULL;
92
93 xef = kzalloc_obj(*xef);
94 if (!xef)
95 return ret;
96
97 client = xe_drm_client_alloc();
98 if (!client) {
99 kfree(xef);
100 return ret;
101 }
102
103 xef->drm = file;
104 xef->client = client;
105 xef->xe = xe;
106
107 mutex_init(&xef->vm.lock);
108 xa_init_flags(&xef->vm.xa, XA_FLAGS_ALLOC1);
109
110 mutex_init(&xef->exec_queue.lock);
111 xa_init_flags(&xef->exec_queue.xa, XA_FLAGS_ALLOC1);
112
113 file->driver_priv = xef;
114 kref_init(&xef->refcount);
115
116 task = get_pid_task(rcu_access_pointer(file->pid), PIDTYPE_PID);
117 if (task) {
118 xef->process_name = kstrdup(task->comm, GFP_KERNEL);
119 xef->pid = task->pid;
120 put_task_struct(task);
121 }
122
123 return 0;
124}
125
126static void xe_file_destroy(struct kref *ref)
127{
128 struct xe_file *xef = container_of(ref, struct xe_file, refcount);
129
130 xa_destroy(&xef->exec_queue.xa);
131 mutex_destroy(&xef->exec_queue.lock);
132 xa_destroy(&xef->vm.xa);
133 mutex_destroy(&xef->vm.lock);
134
135 xe_drm_client_put(xef->client);
136 kfree(xef->process_name);
137 kfree(xef);
138}
139
140/**
141 * xe_file_get() - Take a reference to the xe file object
142 * @xef: Pointer to the xe file
143 *
144 * Anyone with a pointer to xef must take a reference to the xe file
145 * object using this call.
146 *
147 * Return: xe file pointer
148 */
149struct xe_file *xe_file_get(struct xe_file *xef)
150{
151 kref_get(&xef->refcount);
152 return xef;
153}
154
155/**
156 * xe_file_put() - Drop a reference to the xe file object
157 * @xef: Pointer to the xe file
158 *
159 * Used to drop reference to the xef object
160 */
161void xe_file_put(struct xe_file *xef)
162{
163 kref_put(&xef->refcount, xe_file_destroy);
164}
165
166static void xe_file_close(struct drm_device *dev, struct drm_file *file)
167{
168 struct xe_device *xe = to_xe_device(dev);
169 struct xe_file *xef = file->driver_priv;
170 struct xe_vm *vm;
171 struct xe_exec_queue *q;
172 unsigned long idx;
173
174 guard(xe_pm_runtime)(xe);
175
176 /*
177 * No need for exec_queue.lock here as there is no contention for it
178 * when FD is closing as IOCTLs presumably can't be modifying the
179 * xarray. Taking exec_queue.lock here causes undue dependency on
180 * vm->lock taken during xe_exec_queue_kill().
181 */
182 xa_for_each(&xef->exec_queue.xa, idx, q) {
183 if (q->vm && q->hwe->hw_engine_group)
184 xe_hw_engine_group_del_exec_queue(q->hwe->hw_engine_group, q);
185 xe_exec_queue_kill(q);
186 xe_exec_queue_put(q);
187 }
188 xa_for_each(&xef->vm.xa, idx, vm)
189 xe_vm_close_and_put(vm);
190
191 xe_file_put(xef);
192}
193
194static const struct drm_ioctl_desc xe_ioctls[] = {
195 DRM_IOCTL_DEF_DRV(XE_DEVICE_QUERY, xe_query_ioctl, DRM_RENDER_ALLOW),
196 DRM_IOCTL_DEF_DRV(XE_GEM_CREATE, xe_gem_create_ioctl, DRM_RENDER_ALLOW),
197 DRM_IOCTL_DEF_DRV(XE_GEM_MMAP_OFFSET, xe_gem_mmap_offset_ioctl,
198 DRM_RENDER_ALLOW),
199 DRM_IOCTL_DEF_DRV(XE_VM_CREATE, xe_vm_create_ioctl, DRM_RENDER_ALLOW),
200 DRM_IOCTL_DEF_DRV(XE_VM_DESTROY, xe_vm_destroy_ioctl, DRM_RENDER_ALLOW),
201 DRM_IOCTL_DEF_DRV(XE_VM_BIND, xe_vm_bind_ioctl, DRM_RENDER_ALLOW),
202 DRM_IOCTL_DEF_DRV(XE_EXEC, xe_exec_ioctl, DRM_RENDER_ALLOW),
203 DRM_IOCTL_DEF_DRV(XE_EXEC_QUEUE_CREATE, xe_exec_queue_create_ioctl,
204 DRM_RENDER_ALLOW),
205 DRM_IOCTL_DEF_DRV(XE_EXEC_QUEUE_DESTROY, xe_exec_queue_destroy_ioctl,
206 DRM_RENDER_ALLOW),
207 DRM_IOCTL_DEF_DRV(XE_EXEC_QUEUE_GET_PROPERTY, xe_exec_queue_get_property_ioctl,
208 DRM_RENDER_ALLOW),
209 DRM_IOCTL_DEF_DRV(XE_WAIT_USER_FENCE, xe_wait_user_fence_ioctl,
210 DRM_RENDER_ALLOW),
211 DRM_IOCTL_DEF_DRV(XE_OBSERVATION, xe_observation_ioctl, DRM_RENDER_ALLOW),
212 DRM_IOCTL_DEF_DRV(XE_MADVISE, xe_vm_madvise_ioctl, DRM_RENDER_ALLOW),
213 DRM_IOCTL_DEF_DRV(XE_VM_QUERY_MEM_RANGE_ATTRS, xe_vm_query_vmas_attrs_ioctl,
214 DRM_RENDER_ALLOW),
215 DRM_IOCTL_DEF_DRV(XE_EXEC_QUEUE_SET_PROPERTY, xe_exec_queue_set_property_ioctl,
216 DRM_RENDER_ALLOW),
217 DRM_IOCTL_DEF_DRV(XE_VM_GET_PROPERTY, xe_vm_get_property_ioctl,
218 DRM_RENDER_ALLOW),
219};
220
221static long xe_drm_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
222{
223 struct drm_file *file_priv = file->private_data;
224 struct xe_device *xe = to_xe_device(file_priv->minor->dev);
225 long ret;
226
227 if (xe_device_wedged(xe))
228 return -ECANCELED;
229
230 ACQUIRE(xe_pm_runtime_ioctl, pm)(xe);
231 ret = ACQUIRE_ERR(xe_pm_runtime_ioctl, &pm);
232 if (ret >= 0)
233 ret = drm_ioctl(file, cmd, arg);
234
235 return ret;
236}
237
238#ifdef CONFIG_COMPAT
239static long xe_drm_compat_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
240{
241 struct drm_file *file_priv = file->private_data;
242 struct xe_device *xe = to_xe_device(file_priv->minor->dev);
243 long ret;
244
245 if (xe_device_wedged(xe))
246 return -ECANCELED;
247
248 ACQUIRE(xe_pm_runtime_ioctl, pm)(xe);
249 ret = ACQUIRE_ERR(xe_pm_runtime_ioctl, &pm);
250 if (ret >= 0)
251 ret = drm_compat_ioctl(file, cmd, arg);
252
253 return ret;
254}
255#else
256/* similarly to drm_compat_ioctl, let's it be assigned to .compat_ioct unconditionally */
257#define xe_drm_compat_ioctl NULL
258#endif
259
260static void barrier_open(struct vm_area_struct *vma)
261{
262 drm_dev_get(vma->vm_private_data);
263}
264
265static void barrier_close(struct vm_area_struct *vma)
266{
267 drm_dev_put(vma->vm_private_data);
268}
269
270static void barrier_release_dummy_page(struct drm_device *dev, void *res)
271{
272 struct page *dummy_page = (struct page *)res;
273
274 __free_page(dummy_page);
275}
276
277static vm_fault_t barrier_fault(struct vm_fault *vmf)
278{
279 struct drm_device *dev = vmf->vma->vm_private_data;
280 struct vm_area_struct *vma = vmf->vma;
281 vm_fault_t ret = VM_FAULT_NOPAGE;
282 pgprot_t prot;
283 int idx;
284
285 prot = vma_get_page_prot(vma);
286
287 if (drm_dev_enter(dev, &idx)) {
288 unsigned long pfn;
289
290#define LAST_DB_PAGE_OFFSET 0x7ff001
291 pfn = PHYS_PFN(pci_resource_start(to_pci_dev(dev->dev), 0) +
292 LAST_DB_PAGE_OFFSET);
293 ret = vmf_insert_pfn_prot(vma, vma->vm_start, pfn,
294 pgprot_noncached(prot));
295 drm_dev_exit(idx);
296 } else {
297 struct page *page;
298
299 /* Allocate new dummy page to map all the VA range in this VMA to it*/
300 page = alloc_page(GFP_KERNEL | __GFP_ZERO);
301 if (!page)
302 return VM_FAULT_OOM;
303
304 /* Set the page to be freed using drmm release action */
305 if (drmm_add_action_or_reset(dev, barrier_release_dummy_page, page))
306 return VM_FAULT_OOM;
307
308 ret = vmf_insert_pfn_prot(vma, vma->vm_start, page_to_pfn(page),
309 prot);
310 }
311
312 return ret;
313}
314
315static const struct vm_operations_struct vm_ops_barrier = {
316 .open = barrier_open,
317 .close = barrier_close,
318 .fault = barrier_fault,
319};
320
321static int xe_pci_barrier_mmap(struct file *filp,
322 struct vm_area_struct *vma)
323{
324 struct drm_file *priv = filp->private_data;
325 struct drm_device *dev = priv->minor->dev;
326 struct xe_device *xe = to_xe_device(dev);
327
328 if (!IS_DGFX(xe))
329 return -EINVAL;
330
331 if (vma->vm_end - vma->vm_start > SZ_4K)
332 return -EINVAL;
333
334 if (vma_is_cow_mapping(vma))
335 return -EINVAL;
336
337 if (vma->vm_flags & (VM_READ | VM_EXEC))
338 return -EINVAL;
339
340 vm_flags_clear(vma, VM_MAYREAD | VM_MAYEXEC);
341 vm_flags_set(vma, VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP | VM_IO);
342 vma->vm_ops = &vm_ops_barrier;
343 vma->vm_private_data = dev;
344 drm_dev_get(vma->vm_private_data);
345
346 return 0;
347}
348
349static int xe_mmap(struct file *filp, struct vm_area_struct *vma)
350{
351 struct drm_file *priv = filp->private_data;
352 struct drm_device *dev = priv->minor->dev;
353
354 if (drm_dev_is_unplugged(dev))
355 return -ENODEV;
356
357 switch (vma->vm_pgoff) {
358 case XE_PCI_BARRIER_MMAP_OFFSET >> XE_PTE_SHIFT:
359 return xe_pci_barrier_mmap(filp, vma);
360 }
361
362 return drm_gem_mmap(filp, vma);
363}
364
365static const struct file_operations xe_driver_fops = {
366 .owner = THIS_MODULE,
367 .open = drm_open,
368 .release = drm_release_noglobal,
369 .unlocked_ioctl = xe_drm_ioctl,
370 .mmap = xe_mmap,
371 .poll = drm_poll,
372 .read = drm_read,
373 .compat_ioctl = xe_drm_compat_ioctl,
374 .llseek = noop_llseek,
375#ifdef CONFIG_PROC_FS
376 .show_fdinfo = drm_show_fdinfo,
377#endif
378 .fop_flags = FOP_UNSIGNED_OFFSET,
379};
380
381/**
382 * xe_is_xe_file() - Is the file an xe device file?
383 * @file: The file.
384 *
385 * Checks whether the file is opened against
386 * an xe device.
387 *
388 * Return: %true if an xe file, %false if not.
389 */
390bool xe_is_xe_file(const struct file *file)
391{
392 return file->f_op == &xe_driver_fops;
393}
394
395static const struct drm_driver regular_driver = {
396 .driver_features =
397 XE_DISPLAY_DRIVER_FEATURES |
398 DRIVER_GEM |
399 DRIVER_RENDER | DRIVER_SYNCOBJ |
400 DRIVER_SYNCOBJ_TIMELINE,
401 .open = xe_file_open,
402 .postclose = xe_file_close,
403
404 .gem_prime_import = xe_gem_prime_import,
405
406 .dumb_create = xe_bo_dumb_create,
407 .dumb_map_offset = drm_gem_ttm_dumb_map_offset,
408#ifdef CONFIG_PROC_FS
409 .show_fdinfo = xe_drm_client_fdinfo,
410#endif
411 .ioctls = xe_ioctls,
412 .num_ioctls = ARRAY_SIZE(xe_ioctls),
413 .fops = &xe_driver_fops,
414 .name = DRIVER_NAME,
415 .desc = DRIVER_DESC,
416 .major = DRIVER_MAJOR,
417 .minor = DRIVER_MINOR,
418 .patchlevel = DRIVER_PATCHLEVEL,
419 XE_DISPLAY_DRIVER_OPS,
420};
421
422#ifdef CONFIG_PCI_IOV
423static const struct drm_ioctl_desc xe_ioctls_admin_only[] = {
424 DRM_IOCTL_DEF_DRV(XE_DEVICE_QUERY, xe_query_ioctl, DRM_RENDER_ALLOW),
425 DRM_IOCTL_DEF_DRV(XE_OBSERVATION, xe_observation_ioctl, DRM_RENDER_ALLOW),
426};
427
428static const struct drm_driver admin_only_driver = {
429 .driver_features =
430 DRIVER_GEM | DRIVER_RENDER,
431 .open = xe_file_open,
432 .postclose = xe_file_close,
433 .ioctls = xe_ioctls_admin_only,
434 .num_ioctls = ARRAY_SIZE(xe_ioctls_admin_only),
435 .fops = &xe_driver_fops,
436 .name = DRIVER_NAME,
437 .desc = DRIVER_DESC,
438 .major = DRIVER_MAJOR,
439 .minor = DRIVER_MINOR,
440 .patchlevel = DRIVER_PATCHLEVEL,
441};
442
443/**
444 * xe_device_is_admin_only() - Check whether device is admin only or not.
445 * @xe: the &xe_device to check
446 *
447 * Return: true if the device is admin only, false otherwise.
448 */
449bool xe_device_is_admin_only(const struct xe_device *xe)
450{
451 KUNIT_STATIC_STUB_REDIRECT(xe_device_is_admin_only, xe);
452 return xe->drm.driver == &admin_only_driver;
453}
454#endif
455
456static void xe_device_destroy(struct drm_device *dev, void *dummy)
457{
458 struct xe_device *xe = to_xe_device(dev);
459
460 xe_bo_dev_fini(&xe->bo_device);
461
462 if (xe->preempt_fence_wq)
463 destroy_workqueue(xe->preempt_fence_wq);
464
465 if (xe->ordered_wq)
466 destroy_workqueue(xe->ordered_wq);
467
468 if (xe->unordered_wq)
469 destroy_workqueue(xe->unordered_wq);
470
471 if (xe->destroy_wq)
472 destroy_workqueue(xe->destroy_wq);
473
474 ttm_device_fini(&xe->ttm);
475}
476
477/**
478 * xe_device_create() - Create a new &xe_device instance
479 * @pdev: the parent &pci_dev
480 *
481 * Allocate and initialize a device managed Xe device structure.
482 *
483 * Return: pointer to new &xe_device on success, or ERR_PTR on failure.
484 */
485struct xe_device *xe_device_create(struct pci_dev *pdev)
486{
487 const struct drm_driver *driver = &regular_driver;
488 struct xe_device *xe;
489 int err;
490
491#ifdef CONFIG_PCI_IOV
492 /*
493 * Since XE device is not initialized yet, read from configfs
494 * directly to decide whether we are in admin-only PF mode or not.
495 */
496 if (xe_configfs_admin_only_pf(pdev))
497 driver = &admin_only_driver;
498#endif
499
500 err = aperture_remove_conflicting_pci_devices(pdev, driver->name);
501 if (err)
502 return ERR_PTR(err);
503
504 xe = devm_drm_dev_alloc(&pdev->dev, driver, struct xe_device, drm);
505 if (IS_ERR(xe))
506 return xe;
507
508 err = xe_device_init_early(xe);
509 if (err)
510 return ERR_PTR(err);
511
512 return xe;
513}
514ALLOW_ERROR_INJECTION(xe_device_create, ERRNO); /* See xe_pci_probe() */
515
516/**
517 * xe_device_init_early() - Initialize a new &xe_device instance
518 * @xe: the &xe_device to initialize
519 *
520 * Return: 0 on success or a negative error code on failure.
521 */
522int xe_device_init_early(struct xe_device *xe)
523{
524 int err;
525
526 err = ttm_device_init(&xe->ttm, &xe_ttm_funcs, xe->drm.dev,
527 xe->drm.anon_inode->i_mapping,
528 xe->drm.vma_offset_manager,
529 TTM_ALLOCATION_POOL_BENEFICIAL_ORDER(get_order(SZ_2M)));
530 if (err)
531 return err;
532
533 xe_bo_dev_init(&xe->bo_device);
534 err = drmm_add_action_or_reset(&xe->drm, xe_device_destroy, NULL);
535 if (err)
536 return err;
537
538 err = xe_shrinker_create(xe);
539 if (err)
540 return err;
541
542 xe->atomic_svm_timeslice_ms = 5;
543 xe->min_run_period_lr_ms = 5;
544
545 err = xe_irq_init(xe);
546 if (err)
547 return err;
548
549 xe_validation_device_init(&xe->val);
550
551 init_waitqueue_head(&xe->ufence_wq);
552
553 init_rwsem(&xe->usm.lock);
554
555 err = xe_pagemap_shrinker_create(xe);
556 if (err)
557 return err;
558
559 xa_init_flags(&xe->usm.asid_to_vm, XA_FLAGS_ALLOC);
560
561 if (IS_ENABLED(CONFIG_DRM_XE_DEBUG)) {
562 /* Trigger a large asid and an early asid wrap. */
563 u32 asid;
564
565 BUILD_BUG_ON(XE_MAX_ASID < 2);
566 err = xa_alloc_cyclic(&xe->usm.asid_to_vm, &asid, NULL,
567 XA_LIMIT(XE_MAX_ASID - 2, XE_MAX_ASID - 1),
568 &xe->usm.next_asid, GFP_KERNEL);
569 drm_WARN_ON(&xe->drm, err);
570 if (err >= 0)
571 xa_erase(&xe->usm.asid_to_vm, asid);
572 }
573
574 err = xe_bo_pinned_init(xe);
575 if (err)
576 return err;
577
578 xe->preempt_fence_wq = alloc_ordered_workqueue("xe-preempt-fence-wq",
579 WQ_MEM_RECLAIM);
580 xe->ordered_wq = alloc_ordered_workqueue("xe-ordered-wq", 0);
581 xe->unordered_wq = alloc_workqueue("xe-unordered-wq", WQ_PERCPU, 0);
582 xe->destroy_wq = alloc_workqueue("xe-destroy-wq", WQ_PERCPU | WQ_MEM_RECLAIM, 0);
583 if (!xe->ordered_wq || !xe->unordered_wq ||
584 !xe->preempt_fence_wq || !xe->destroy_wq) {
585 /*
586 * Cleanup done in xe_device_destroy via
587 * drmm_add_action_or_reset register above
588 */
589 drm_err(&xe->drm, "Failed to allocate xe workqueues\n");
590 return -ENOMEM;
591 }
592
593 err = drmm_mutex_init(&xe->drm, &xe->pmt.lock);
594 if (err)
595 return err;
596
597 err = xe_pm_init_early(xe);
598 if (err)
599 return err;
600
601 return 0;
602}
603
604static bool xe_driver_flr_disabled(struct xe_device *xe)
605{
606 if (IS_SRIOV_VF(xe))
607 return true;
608
609 if (xe_mmio_read32(xe_root_tile_mmio(xe), GU_CNTL_PROTECTED) & DRIVERINT_FLR_DIS) {
610 drm_info(&xe->drm, "Driver-FLR disabled by BIOS\n");
611 return true;
612 }
613
614 return false;
615}
616
617/*
618 * The driver-initiated FLR is the highest level of reset that we can trigger
619 * from within the driver. It is different from the PCI FLR in that it doesn't
620 * fully reset the SGUnit and doesn't modify the PCI config space and therefore
621 * it doesn't require a re-enumeration of the PCI BARs. However, the
622 * driver-initiated FLR does still cause a reset of both GT and display and a
623 * memory wipe of local and stolen memory, so recovery would require a full HW
624 * re-init and saving/restoring (or re-populating) the wiped memory. Since we
625 * perform the FLR as the very last action before releasing access to the HW
626 * during the driver release flow, we don't attempt recovery at all, because
627 * if/when a new instance of Xe is bound to the device it will do a full
628 * re-init anyway.
629 */
630static void __xe_driver_flr(struct xe_device *xe)
631{
632 const unsigned int flr_timeout = 3 * USEC_PER_SEC; /* specs recommend a 3s wait */
633 struct xe_mmio *mmio = xe_root_tile_mmio(xe);
634 int ret;
635
636 drm_dbg(&xe->drm, "Triggering Driver-FLR\n");
637
638 /*
639 * Make sure any pending FLR requests have cleared by waiting for the
640 * FLR trigger bit to go to zero. Also clear GU_DEBUG's DRIVERFLR_STATUS
641 * to make sure it's not still set from a prior attempt (it's a write to
642 * clear bit).
643 * Note that we should never be in a situation where a previous attempt
644 * is still pending (unless the HW is totally dead), but better to be
645 * safe in case something unexpected happens
646 */
647 ret = xe_mmio_wait32(mmio, GU_CNTL, DRIVERFLR, 0, flr_timeout, NULL, false);
648 if (ret) {
649 drm_err(&xe->drm, "Driver-FLR-prepare wait for ready failed! %d\n", ret);
650 return;
651 }
652 xe_mmio_write32(mmio, GU_DEBUG, DRIVERFLR_STATUS);
653
654 /* Trigger the actual Driver-FLR */
655 xe_mmio_rmw32(mmio, GU_CNTL, 0, DRIVERFLR);
656
657 /* Wait for hardware teardown to complete */
658 ret = xe_mmio_wait32(mmio, GU_CNTL, DRIVERFLR, 0, flr_timeout, NULL, false);
659 if (ret) {
660 drm_err(&xe->drm, "Driver-FLR-teardown wait completion failed! %d\n", ret);
661 return;
662 }
663
664 /* Wait for hardware/firmware re-init to complete */
665 ret = xe_mmio_wait32(mmio, GU_DEBUG, DRIVERFLR_STATUS, DRIVERFLR_STATUS,
666 flr_timeout, NULL, false);
667 if (ret) {
668 drm_err(&xe->drm, "Driver-FLR-reinit wait completion failed! %d\n", ret);
669 return;
670 }
671
672 /* Clear sticky completion status */
673 xe_mmio_write32(mmio, GU_DEBUG, DRIVERFLR_STATUS);
674}
675
676static void xe_driver_flr(struct xe_device *xe)
677{
678 if (xe_driver_flr_disabled(xe))
679 return;
680
681 __xe_driver_flr(xe);
682}
683
684static void xe_driver_flr_fini(void *arg)
685{
686 struct xe_device *xe = arg;
687
688 if (xe->needs_flr_on_fini)
689 xe_driver_flr(xe);
690}
691
692static void xe_device_sanitize(void *arg)
693{
694 struct xe_device *xe = arg;
695 struct xe_gt *gt;
696 u8 id;
697
698 for_each_gt(gt, xe, id)
699 xe_gt_sanitize(gt);
700}
701
702static int xe_set_dma_info(struct xe_device *xe)
703{
704 unsigned int mask_size = xe->info.dma_mask_size;
705 int err;
706
707 dma_set_max_seg_size(xe->drm.dev, xe_sg_segment_size(xe->drm.dev));
708
709 err = dma_set_mask(xe->drm.dev, DMA_BIT_MASK(mask_size));
710 if (err)
711 goto mask_err;
712
713 err = dma_set_coherent_mask(xe->drm.dev, DMA_BIT_MASK(mask_size));
714 if (err)
715 goto mask_err;
716
717 return 0;
718
719mask_err:
720 drm_err(&xe->drm, "Can't set DMA mask/consistent mask (%d)\n", err);
721 return err;
722}
723
724static void assert_lmem_ready(struct xe_device *xe)
725{
726 if (!IS_DGFX(xe) || IS_SRIOV_VF(xe))
727 return;
728
729 xe_assert(xe, xe_mmio_read32(xe_root_tile_mmio(xe), GU_CNTL) &
730 LMEM_INIT);
731}
732
733static void vf_update_device_info(struct xe_device *xe)
734{
735 xe_assert(xe, IS_SRIOV_VF(xe));
736 /* disable features that are not available/applicable to VFs */
737 xe->info.probe_display = 0;
738 xe->info.has_heci_cscfi = 0;
739 xe->info.has_heci_gscfi = 0;
740 xe->info.has_i2c = 0;
741 xe->info.has_late_bind = 0;
742 xe->info.skip_guc_pc = 1;
743 xe->info.skip_pcode = 1;
744 xe->info.has_drm_ras = false;
745}
746
747static int xe_device_vram_alloc(struct xe_device *xe)
748{
749 struct xe_vram_region *vram;
750
751 if (!IS_DGFX(xe))
752 return 0;
753
754 vram = drmm_kzalloc(&xe->drm, sizeof(*vram), GFP_KERNEL);
755 if (!vram)
756 return -ENOMEM;
757
758 xe->mem.vram = vram;
759 return 0;
760}
761
762/**
763 * xe_device_probe_early: Device early probe
764 * @xe: xe device instance
765 *
766 * Initialize MMIO resources that don't require any
767 * knowledge about tile count. Also initialize pcode and
768 * check vram initialization on root tile.
769 *
770 * Return: 0 on success, error code on failure
771 */
772int xe_device_probe_early(struct xe_device *xe)
773{
774 int err;
775
776 xe_wa_device_init(xe);
777 xe_wa_process_device_oob(xe);
778
779 err = xe_mmio_probe_early(xe);
780 if (err)
781 return err;
782
783 xe_sriov_probe_early(xe);
784
785 if (xe_device_is_admin_only(xe) && !IS_SRIOV_PF(xe)) {
786 xe_err(xe, "Can't run Admin-only mode without SR-IOV PF mode!\n");
787 return -ENODEV;
788 }
789
790 if (IS_SRIOV_VF(xe))
791 vf_update_device_info(xe);
792
793 /*
794 * Check for pcode uncore_init status to confirm if the SoC
795 * initialization is complete. Until done, any MMIO or lmem access from
796 * the driver will be blocked
797 */
798 err = xe_pcode_probe_early(xe);
799 if (err || xe_survivability_mode_is_requested(xe)) {
800 int save_err = err;
801
802 /*
803 * Try to leave device in survivability mode if device is
804 * possible, but still return the previous error for error
805 * propagation
806 */
807 err = xe_survivability_mode_boot_enable(xe);
808 if (err)
809 return err;
810
811 return save_err;
812 }
813
814 /*
815 * Make sure the lmem is initialized and ready to use. xe_pcode_ready()
816 * is flagged after full initialization is complete. Assert if lmem is
817 * not initialized.
818 */
819 assert_lmem_ready(xe);
820
821 xe->wedged.mode = xe_device_validate_wedged_mode(xe, xe_modparam.wedged_mode) ?
822 XE_DEFAULT_WEDGED_MODE : xe_modparam.wedged_mode;
823 drm_dbg(&xe->drm, "wedged_mode: setting mode (%u) %s\n",
824 xe->wedged.mode, xe_wedged_mode_to_string(xe->wedged.mode));
825
826 err = xe_device_vram_alloc(xe);
827 if (err)
828 return err;
829
830 return 0;
831}
832ALLOW_ERROR_INJECTION(xe_device_probe_early, ERRNO); /* See xe_pci_probe() */
833
834static int probe_has_flat_ccs(struct xe_device *xe)
835{
836 struct xe_gt *gt;
837 u32 reg;
838
839 /* Always enabled/disabled, no runtime check to do */
840 if (GRAPHICS_VER(xe) < 20 || !xe->info.has_flat_ccs || IS_SRIOV_VF(xe))
841 return 0;
842
843 gt = xe_root_mmio_gt(xe);
844 if (!gt)
845 return 0;
846
847 CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT);
848 if (!fw_ref.domains)
849 return -ETIMEDOUT;
850
851 reg = xe_gt_mcr_unicast_read_any(gt, XE2_FLAT_CCS_BASE_RANGE_LOWER);
852 xe->info.has_flat_ccs = (reg & XE2_FLAT_CCS_ENABLE);
853
854 if (!xe->info.has_flat_ccs)
855 drm_dbg(&xe->drm,
856 "Flat CCS has been disabled in bios, May lead to performance impact");
857
858 return 0;
859}
860
861/*
862 * Detect if the driver is being run on pre-production hardware. We don't
863 * keep workarounds for pre-production hardware long term, so print an
864 * error and add taint if we're being loaded on a pre-production platform
865 * for which the pre-prod workarounds have already been removed.
866 *
867 * The general policy is that we'll remove any workarounds that only apply to
868 * pre-production hardware around the time force_probe restrictions are lifted
869 * for a platform of the next major IP generation (for example, Xe2 pre-prod
870 * workarounds should be removed around the time the first Xe3 platforms have
871 * force_probe lifted).
872 */
873static void detect_preproduction_hw(struct xe_device *xe)
874{
875 struct xe_gt *gt;
876 int id;
877
878 /*
879 * SR-IOV VFs don't have access to the FUSE2 register, so we can't
880 * check pre-production status there. But the host OS will notice
881 * and report the pre-production status, which should be enough to
882 * help us catch mistaken use of pre-production hardware.
883 */
884 if (IS_SRIOV_VF(xe))
885 return;
886
887 /*
888 * The "SW_CAP" fuse contains a bit indicating whether the device is a
889 * production or pre-production device. This fuse is reflected through
890 * the GT "FUSE2" register, even though the contents of the fuse are
891 * not GT-specific. Every GT's reflection of this fuse should show the
892 * same value, so we'll just use the first available GT for lookup.
893 */
894 for_each_gt(gt, xe, id)
895 break;
896
897 if (!gt)
898 return;
899
900 CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT);
901 if (!xe_force_wake_ref_has_domain(fw_ref.domains, XE_FW_GT)) {
902 xe_gt_err(gt, "Forcewake failure; cannot determine production/pre-production hw status.\n");
903 return;
904 }
905
906 if (xe_mmio_read32(&gt->mmio, FUSE2) & PRODUCTION_HW)
907 return;
908
909 xe_info(xe, "Pre-production hardware detected.\n");
910 if (!xe->info.has_pre_prod_wa) {
911 xe_err(xe, "Pre-production workarounds for this platform have already been removed.\n");
912 add_taint(TAINT_MACHINE_CHECK, LOCKDEP_STILL_OK);
913 }
914}
915
916static void xe_device_wedged_fini(struct drm_device *drm, void *arg)
917{
918 struct xe_device *xe = arg;
919
920 if (atomic_read(&xe->wedged.flag))
921 xe_pm_runtime_put(xe);
922}
923
924#ifdef CONFIG_DRM_XE_DEBUG_PAGE_SIZE
925static int xe_debug_page_size_alloc_ctrl_init(struct xe_device *xe)
926{
927 int err;
928
929 err = drmm_mutex_init(&xe->drm, &xe->page_size_alloc_ctrl.lock);
930 if (err)
931 return err;
932
933 xe->page_size_alloc_ctrl.mode = XE_PAGE_SIZE_ALLOC_CTRL_MODE_NONE;
934 xe->page_size_alloc_ctrl.cur_index = 0;
935
936 return 0;
937}
938#else
939static int xe_debug_page_size_alloc_ctrl_init(struct xe_device *xe)
940{
941 return 0;
942}
943#endif
944
945int xe_device_probe(struct xe_device *xe)
946{
947 struct xe_tile *tile;
948 struct xe_gt *gt;
949 int err;
950 u8 id;
951
952 xe_pat_init_early(xe);
953
954 err = xe_sriov_init(xe);
955 if (err)
956 return err;
957
958 xe->info.mem_region_mask = 1;
959
960 err = xe_set_dma_info(xe);
961 if (err)
962 return err;
963
964 err = xe_mmio_probe_tiles(xe);
965 if (err)
966 return err;
967
968 for_each_gt(gt, xe, id) {
969 err = xe_gt_init_early(gt);
970 if (err)
971 return err;
972 }
973
974 /*
975 * Wa_16029380221: The affected GT will always use non-coherent
976 * access to page tables, so we must do uncached writes from the
977 * CPU.
978 */
979 for_each_gt(gt, xe, id)
980 if (XE_GT_WA(gt, 16029380221))
981 xe->info.has_cached_pt = false;
982
983 for_each_tile(tile, xe, id) {
984 err = xe_ggtt_init_early(tile->mem.ggtt);
985 if (err)
986 return err;
987 }
988
989 /*
990 * From here on, if a step fails, make sure a Driver-FLR is triggereed
991 */
992 err = devm_add_action_or_reset(xe->drm.dev, xe_driver_flr_fini, xe);
993 if (err)
994 return err;
995
996 err = probe_has_flat_ccs(xe);
997 if (err)
998 return err;
999
1000 err = xe_vram_probe(xe);
1001 if (err)
1002 return err;
1003
1004 for_each_tile(tile, xe, id) {
1005 err = xe_tile_init_noalloc(tile);
1006 if (err)
1007 return err;
1008 }
1009
1010 /*
1011 * Allow allocations only now to ensure xe_display_init_early()
1012 * is the first to allocate, always.
1013 */
1014 err = xe_ttm_sys_mgr_init(xe);
1015 if (err)
1016 return err;
1017
1018 /* Allocate and map stolen after potential VRAM resize */
1019 err = xe_ttm_stolen_mgr_init(xe);
1020 if (err)
1021 return err;
1022
1023 err = xe_soc_remapper_init(xe);
1024 if (err)
1025 return err;
1026
1027 err = xe_sysctrl_init(xe);
1028 if (err)
1029 return err;
1030
1031 xe_ras_init(xe);
1032
1033 /*
1034 * Now that GT is initialized (TTM in particular),
1035 * we can try to init display, and inherit the initial fb.
1036 * This is the reason the first allocation needs to be done
1037 * inside display.
1038 */
1039 err = xe_display_init_early(xe);
1040 if (err)
1041 return err;
1042
1043 for_each_tile(tile, xe, id) {
1044 err = xe_tile_init(tile);
1045 if (err)
1046 return err;
1047 }
1048
1049 err = xe_irq_install(xe);
1050 if (err)
1051 return err;
1052
1053 for_each_gt(gt, xe, id) {
1054 err = xe_gt_init(gt);
1055 if (err)
1056 return err;
1057 }
1058
1059 err = xe_pagefault_init(xe);
1060 if (err)
1061 return err;
1062
1063 if (xe->tiles->media_gt &&
1064 XE_GT_WA(xe->tiles->media_gt, 15015404425_disable))
1065 XE_DEVICE_WA_DISABLE(xe, 15015404425);
1066
1067 err = xe_devcoredump_init(xe);
1068 if (err)
1069 return err;
1070
1071 xe_nvm_init(xe);
1072
1073 err = xe_heci_gsc_init(xe);
1074 if (err)
1075 return err;
1076
1077 err = xe_late_bind_init(&xe->late_bind);
1078 if (err)
1079 return err;
1080
1081 err = xe_oa_init(xe);
1082 if (err)
1083 return err;
1084
1085 err = xe_display_init(xe);
1086 if (err)
1087 return err;
1088
1089 err = xe_pxp_init(xe);
1090 if (err)
1091 return err;
1092
1093 err = xe_psmi_init(xe);
1094 if (err)
1095 return err;
1096
1097 err = xe_debug_page_size_alloc_ctrl_init(xe);
1098 if (err)
1099 return err;
1100
1101 err = drm_dev_register(&xe->drm, 0);
1102 if (err)
1103 return err;
1104
1105 xe_display_register(xe);
1106
1107 err = xe_oa_register(xe);
1108 if (err)
1109 goto err_unregister_display;
1110
1111 err = xe_pmu_register(&xe->pmu);
1112 if (err)
1113 goto err_unregister_display;
1114
1115 err = xe_device_sysfs_init(xe);
1116 if (err)
1117 goto err_unregister_display;
1118
1119 xe_debugfs_register(xe);
1120
1121 err = xe_hwmon_register(xe);
1122 if (err)
1123 goto err_unregister_display;
1124
1125 err = xe_i2c_probe(xe);
1126 if (err)
1127 goto err_unregister_display;
1128
1129 for_each_gt(gt, xe, id)
1130 xe_gt_sanitize_freq(gt);
1131
1132 xe_vsec_init(xe);
1133
1134 err = xe_sriov_init_late(xe);
1135 if (err)
1136 goto err_unregister_display;
1137
1138 detect_preproduction_hw(xe);
1139
1140 err = drmm_add_action_or_reset(&xe->drm, xe_device_wedged_fini, xe);
1141 if (err)
1142 goto err_unregister_display;
1143
1144 /*
1145 * Process and log any errors detected by hardware. Possible results can
1146 * include declaring the device as wedged, which must be done only after
1147 * xe_device_wedged_fini() is registered.
1148 */
1149 xe_ras_process_errors(xe);
1150
1151 err = devm_add_action_or_reset(xe->drm.dev, xe_device_sanitize, xe);
1152 if (err)
1153 goto err_unregister_display;
1154
1155 return 0;
1156
1157err_unregister_display:
1158 xe_display_unregister(xe);
1159 drm_dev_unregister(&xe->drm);
1160
1161 return err;
1162}
1163
1164void xe_device_remove(struct xe_device *xe)
1165{
1166 xe_display_unregister(xe);
1167
1168 drm_dev_unplug(&xe->drm);
1169
1170 xe_bo_pci_dev_remove_all(xe);
1171}
1172
1173void xe_device_shutdown(struct xe_device *xe)
1174{
1175 struct xe_gt *gt;
1176 u8 id;
1177
1178 drm_dbg(&xe->drm, "Shutting down device\n");
1179
1180 xe_display_shutdown(xe);
1181
1182 xe_irq_suspend(xe);
1183
1184 for_each_gt(gt, xe, id)
1185 xe_gt_shutdown(gt);
1186
1187 xe_display_shutdown_late(xe);
1188
1189 if (!xe_driver_flr_disabled(xe)) {
1190 /* BOOM! */
1191 __xe_driver_flr(xe);
1192 }
1193}
1194
1195/**
1196 * xe_device_wmb() - Device specific write memory barrier
1197 * @xe: the &xe_device
1198 *
1199 * While wmb() is sufficient for a barrier if we use system memory, on discrete
1200 * platforms with device memory we additionally need to issue a register write.
1201 * Since it doesn't matter which register we write to, use the read-only VF_CAP
1202 * register that is also marked as accessible by the VFs.
1203 */
1204void xe_device_wmb(struct xe_device *xe)
1205{
1206 wmb();
1207 if (IS_DGFX(xe))
1208 xe_mmio_write32(xe_root_tile_mmio(xe), VF_CAP_REG, 0);
1209}
1210
1211/*
1212 * Issue a TRANSIENT_FLUSH_REQUEST and wait for completion on each gt.
1213 */
1214static void tdf_request_sync(struct xe_device *xe)
1215{
1216 struct xe_gt *gt;
1217 u8 id;
1218
1219 for_each_gt_with_type(gt, xe, id, BIT(XE_GT_TYPE_MAIN)) {
1220 CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT);
1221 if (!fw_ref.domains)
1222 return;
1223
1224 xe_mmio_write32(&gt->mmio, XE2_TDF_CTRL, TRANSIENT_FLUSH_REQUEST);
1225
1226 /*
1227 * FIXME: We can likely do better here with our choice of
1228 * timeout. Currently we just assume the worst case, i.e. 150us,
1229 * which is believed to be sufficient to cover the worst case
1230 * scenario on current platforms if all cache entries are
1231 * transient and need to be flushed..
1232 */
1233 if (xe_mmio_wait32(&gt->mmio, XE2_TDF_CTRL, TRANSIENT_FLUSH_REQUEST, 0,
1234 300, NULL, false))
1235 xe_gt_err_once(gt, "TD flush timeout\n");
1236 }
1237}
1238
1239/**
1240 * xe_device_is_l2_flush_optimized - if L2 flush is optimized by HW
1241 * @xe: The device to check.
1242 *
1243 * Return: true if the HW device optimizing L2 flush, false otherwise.
1244 */
1245bool xe_device_is_l2_flush_optimized(struct xe_device *xe)
1246{
1247 /* XA is *always* flushed, like at the end-of-submssion (and maybe other
1248 * places), just that internally as an optimisation hw doesn't need to make
1249 * that a full flush (which will also include XA) when Media is
1250 * off/powergated, since it doesn't need to worry about GT caches vs Media
1251 * coherency, and only CPU vs GPU coherency, so can make that flush a
1252 * targeted XA flush, since stuff tagged with XA now means it's shared with
1253 * the CPU. The main implication is that we now need to somehow flush non-XA before
1254 * freeing system memory pages, otherwise dirty cachelines could be flushed after the free
1255 * (like if Media suddenly turns on and does a full flush)
1256 */
1257 if (GRAPHICS_VER(xe) >= 35 && !IS_DGFX(xe))
1258 return true;
1259 return false;
1260}
1261
1262void xe_device_l2_flush(struct xe_device *xe)
1263{
1264 struct xe_gt *gt;
1265
1266 gt = xe_root_mmio_gt(xe);
1267 if (!gt)
1268 return;
1269
1270 if (!XE_GT_WA(gt, 16023588340))
1271 return;
1272
1273 CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT);
1274 if (!fw_ref.domains)
1275 return;
1276
1277 spin_lock(&gt->global_invl_lock);
1278
1279 xe_mmio_write32(&gt->mmio, XE2_GLOBAL_INVAL, 0x1);
1280 if (xe_mmio_wait32(&gt->mmio, XE2_GLOBAL_INVAL, 0x1, 0x0, 1000, NULL, true))
1281 xe_gt_err_once(gt, "Global invalidation timeout\n");
1282
1283 spin_unlock(&gt->global_invl_lock);
1284}
1285
1286/**
1287 * xe_device_td_flush() - Flush transient L3 cache entries
1288 * @xe: The device
1289 *
1290 * Display engine has direct access to memory and is never coherent with L3/L4
1291 * caches (or CPU caches), however KMD is responsible for specifically flushing
1292 * transient L3 GPU cache entries prior to the flip sequence to ensure scanout
1293 * can happen from such a surface without seeing corruption.
1294 *
1295 * Display surfaces can be tagged as transient by mapping it using one of the
1296 * various L3:XD PAT index modes on Xe2.
1297 *
1298 * Note: On non-discrete xe2 platforms, like LNL, the entire L3 cache is flushed
1299 * at the end of each submission via PIPE_CONTROL for compute/render, since SA
1300 * Media is not coherent with L3 and we want to support render-vs-media
1301 * usescases. For other engines like copy/blt the HW internally forces uncached
1302 * behaviour, hence why we can skip the TDF on such platforms.
1303 */
1304void xe_device_td_flush(struct xe_device *xe)
1305{
1306 struct xe_gt *root_gt;
1307
1308 /*
1309 * From Xe3p onward the HW takes care of flush of TD entries also along
1310 * with flushing XA entries, which will be at the usual sync points,
1311 * like at the end of submission, so no manual flush is needed here.
1312 */
1313 if (GRAPHICS_VER(xe) >= 35)
1314 return;
1315
1316 if (!IS_DGFX(xe) || GRAPHICS_VER(xe) < 20)
1317 return;
1318
1319 root_gt = xe_root_mmio_gt(xe);
1320 if (!root_gt)
1321 return;
1322
1323 if (XE_GT_WA(root_gt, 16023588340)) {
1324 /* A transient flush is not sufficient: flush the L2 */
1325 xe_device_l2_flush(xe);
1326 } else {
1327 xe_guc_pc_apply_flush_freq_limit(&root_gt->uc.guc.pc);
1328 tdf_request_sync(xe);
1329 xe_guc_pc_remove_flush_freq_limit(&root_gt->uc.guc.pc);
1330 }
1331}
1332
1333u32 xe_device_ccs_bytes(struct xe_device *xe, u64 size)
1334{
1335 return xe_device_has_flat_ccs(xe) ?
1336 DIV_ROUND_UP_ULL(size, NUM_BYTES_PER_CCS_BYTE(xe)) : 0;
1337}
1338
1339/**
1340 * xe_device_assert_mem_access - Inspect the current runtime_pm state.
1341 * @xe: xe device instance
1342 *
1343 * To be used before any kind of memory access. It will splat a debug warning
1344 * if the device is currently sleeping. But it doesn't guarantee in any way
1345 * that the device is going to remain awake. Xe PM runtime get and put
1346 * functions might be added to the outer bound of the memory access, while
1347 * this check is intended for inner usage to splat some warning if the worst
1348 * case has just happened.
1349 */
1350void xe_device_assert_mem_access(struct xe_device *xe)
1351{
1352 xe_assert(xe, !xe_pm_runtime_suspended(xe));
1353}
1354
1355void xe_device_snapshot_print(struct xe_device *xe, struct drm_printer *p)
1356{
1357 struct xe_gt *gt;
1358 u8 id;
1359
1360 drm_printf(p, "PCI ID: 0x%04x\n", xe->info.devid);
1361 drm_printf(p, "PCI revision: 0x%02x\n", xe->info.revid);
1362
1363 for_each_gt(gt, xe, id) {
1364 drm_printf(p, "GT id: %u\n", id);
1365 drm_printf(p, "\tTile: %u\n", gt->tile->id);
1366 drm_printf(p, "\tType: %s\n",
1367 gt->info.type == XE_GT_TYPE_MAIN ? "main" : "media");
1368 drm_printf(p, "\tIP ver: %u.%u.%u\n",
1369 REG_FIELD_GET(GMD_ID_ARCH_MASK, gt->info.gmdid),
1370 REG_FIELD_GET(GMD_ID_RELEASE_MASK, gt->info.gmdid),
1371 REG_FIELD_GET(GMD_ID_REVID, gt->info.gmdid));
1372 drm_printf(p, "\tCS reference clock: %u\n", gt->info.reference_clock);
1373 }
1374}
1375
1376u64 xe_device_canonicalize_addr(struct xe_device *xe, u64 address)
1377{
1378 return sign_extend64(address, xe->info.va_bits - 1);
1379}
1380
1381u64 xe_device_uncanonicalize_addr(struct xe_device *xe, u64 address)
1382{
1383 return address & GENMASK_ULL(xe->info.va_bits - 1, 0);
1384}
1385
1386/**
1387 * DOC: Xe Device Wedging
1388 *
1389 * Xe driver uses drm device wedged uevent as documented in Documentation/gpu/drm-uapi.rst.
1390 * When device is in wedged state, every IOCTL will be blocked and GT cannot
1391 * be used. The conditions under which the driver declares the device wedged
1392 * depend on the wedged mode configuration (see &enum xe_wedged_mode). The
1393 * default recovery method for a wedged state is rebind/bus-reset.
1394 *
1395 * Another recovery method is vendor-specific. Below are the cases that send
1396 * ``WEDGED=vendor-specific`` recovery method in drm device wedged uevent.
1397 *
1398 * Case: Firmware Flash
1399 * --------------------
1400 *
1401 * Identification Hint
1402 * +++++++++++++++++++
1403 *
1404 * ``WEDGED=vendor-specific`` drm device wedged uevent with
1405 * :ref:`Runtime Survivability mode <xe-survivability-mode>` is used to notify
1406 * admin/userspace consumer about the need for a firmware flash.
1407 *
1408 * Recovery Procedure
1409 * ++++++++++++++++++
1410 *
1411 * Once ``WEDGED=vendor-specific`` drm device wedged uevent is received, follow
1412 * the below steps
1413 *
1414 * - Check Runtime Survivability mode sysfs.
1415 * If enabled, firmware flash is required to recover the device.
1416 *
1417 * /sys/bus/pci/devices/<device>/survivability_mode
1418 *
1419 * - Admin/userspace consumer can use firmware flashing tools like fwupd to flash
1420 * firmware and restore device to normal operation.
1421 */
1422
1423/**
1424 * xe_device_set_wedged_method - Set wedged recovery method
1425 * @xe: xe device instance
1426 * @method: recovery method to set
1427 *
1428 * Set wedged recovery method to be sent in drm wedged uevent.
1429 */
1430void xe_device_set_wedged_method(struct xe_device *xe, unsigned long method)
1431{
1432 xe->wedged.method = method;
1433}
1434
1435/**
1436 * xe_device_declare_wedged - Declare device wedged
1437 * @xe: xe device instance
1438 *
1439 * This is a final state that can only be cleared with the recovery method
1440 * specified in the drm wedged uevent. The method can be set using
1441 * xe_device_set_wedged_method before declaring the device as wedged. If no method
1442 * is set, reprobe (unbind/re-bind) will be sent by default.
1443 *
1444 * In this state every IOCTL will be blocked so the GT cannot be used.
1445 * In general it will be called upon any critical error such as gt reset
1446 * failure or guc loading failure. Userspace will be notified of this state
1447 * through device wedged uevent.
1448 * If xe.wedged module parameter is set to 2, this function will be called
1449 * on every single execution timeout (a.k.a. GPU hang) right after devcoredump
1450 * snapshot capture. In this mode, GT reset won't be attempted so the state of
1451 * the issue is preserved for further debugging.
1452 */
1453void xe_device_declare_wedged(struct xe_device *xe)
1454{
1455 struct xe_gt *gt;
1456 u8 id;
1457
1458 if (xe->wedged.mode == XE_WEDGED_MODE_NEVER) {
1459 drm_dbg(&xe->drm, "Wedged mode is forcibly disabled\n");
1460 return;
1461 }
1462
1463 if (!atomic_xchg(&xe->wedged.flag, 1)) {
1464 xe->needs_flr_on_fini = true;
1465 xe_pm_runtime_get_noresume(xe);
1466 drm_err(&xe->drm,
1467 "CRITICAL: Xe has declared device %s as wedged.\n"
1468 "IOCTLs and executions are blocked.\n"
1469 "For recovery procedure, refer to https://docs.kernel.org/gpu/drm-uapi.html#device-wedging\n"
1470 "Please file a _new_ bug report at https://gitlab.freedesktop.org/drm/xe/kernel/issues/new\n",
1471 dev_name(xe->drm.dev));
1472 }
1473
1474 for_each_gt(gt, xe, id)
1475 xe_gt_declare_wedged(gt);
1476
1477 if (xe_device_wedged(xe)) {
1478 /*
1479 * XE_WEDGED_MODE_UPON_ANY_HANG_NO_RESET is intended for debugging
1480 * hangs, so wedge the device with 'none' recovery method and have
1481 * it available to the user for debugging.
1482 */
1483 if (xe->wedged.mode == XE_WEDGED_MODE_UPON_ANY_HANG_NO_RESET)
1484 xe_device_set_wedged_method(xe, DRM_WEDGE_RECOVERY_NONE);
1485 /* If no wedge recovery method is set, use default */
1486 else if (!xe->wedged.method)
1487 xe_device_set_wedged_method(xe, DRM_WEDGE_RECOVERY_REBIND |
1488 DRM_WEDGE_RECOVERY_BUS_RESET);
1489
1490 /* Notify userspace of wedged device */
1491 drm_dev_wedged_event(&xe->drm, xe->wedged.method, NULL);
1492 }
1493}
1494
1495/**
1496 * xe_device_validate_wedged_mode - Check if given mode is supported
1497 * @xe: the &xe_device
1498 * @mode: requested mode to validate
1499 *
1500 * Check whether the provided wedged mode is supported.
1501 *
1502 * Return: 0 if mode is supported, error code otherwise.
1503 */
1504int xe_device_validate_wedged_mode(struct xe_device *xe, unsigned int mode)
1505{
1506 if (mode > XE_WEDGED_MODE_UPON_ANY_HANG_NO_RESET) {
1507 drm_dbg(&xe->drm, "wedged_mode: invalid value (%u)\n", mode);
1508 return -EINVAL;
1509 } else if (mode == XE_WEDGED_MODE_UPON_ANY_HANG_NO_RESET && (IS_SRIOV_VF(xe) ||
1510 (IS_SRIOV_PF(xe) && !IS_ENABLED(CONFIG_DRM_XE_DEBUG)))) {
1511 drm_dbg(&xe->drm, "wedged_mode: (%u) %s mode is not supported for %s\n",
1512 mode, xe_wedged_mode_to_string(mode),
1513 xe_sriov_mode_to_string(xe_device_sriov_mode(xe)));
1514 return -EPERM;
1515 }
1516
1517 return 0;
1518}
1519
1520/**
1521 * xe_wedged_mode_to_string - Convert enum value to string.
1522 * @mode: the &xe_wedged_mode to convert
1523 *
1524 * Returns: wedged mode as a user friendly string.
1525 */
1526const char *xe_wedged_mode_to_string(enum xe_wedged_mode mode)
1527{
1528 switch (mode) {
1529 case XE_WEDGED_MODE_NEVER:
1530 return "never";
1531 case XE_WEDGED_MODE_UPON_CRITICAL_ERROR:
1532 return "upon-critical-error";
1533 case XE_WEDGED_MODE_UPON_ANY_HANG_NO_RESET:
1534 return "upon-any-hang-no-reset";
1535 default:
1536 return "<invalid>";
1537 }
1538}
1539
1540/**
1541 * xe_device_asid_to_vm() - Find VM from ASID
1542 * @xe: the &xe_device
1543 * @asid: Address space ID
1544 *
1545 * Find a VM from ASID and take a reference to VM which caller must drop.
1546 * Reclaim safe.
1547 *
1548 * Return: VM on success, ERR_PTR on failure
1549 */
1550struct xe_vm *xe_device_asid_to_vm(struct xe_device *xe, u32 asid)
1551{
1552 struct xe_vm *vm;
1553
1554 down_read(&xe->usm.lock);
1555 vm = xa_load(&xe->usm.asid_to_vm, asid);
1556 if (vm)
1557 xe_vm_get(vm);
1558 else
1559 vm = ERR_PTR(-EINVAL);
1560 up_read(&xe->usm.lock);
1561
1562 return vm;
1563}