projects/linux·blob master · refs

203 lines · 6.4 KiB

1/* SPDX-License-Identifier: MIT */
2/*
3 * Copyright 2023 Advanced Micro Devices, Inc.
4 *
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the "Software"),
7 * to deal in the Software without restriction, including without limitation
8 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9 * and/or sell copies of the Software, and to permit persons to whom the
10 * Software is furnished to do so, subject to the following conditions:
11 *
12 * The above copyright notice and this permission notice shall be included in
13 * all copies or substantial portions of the Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
19 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
20 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
21 * OTHER DEALINGS IN THE SOFTWARE.
22 *
23 */
24
25#ifndef AMDGPU_USERQ_H_
26#define AMDGPU_USERQ_H_
27#include "amdgpu_eviction_fence.h"
28
29#define AMDGPU_MAX_USERQ_COUNT 512
30
31#define to_ev_fence(f) container_of(f, struct amdgpu_eviction_fence, base)
32#define uq_mgr_to_fpriv(u) container_of(u, struct amdgpu_fpriv, userq_mgr)
33#define work_to_uq_mgr(w, name) container_of(w, struct amdgpu_userq_mgr, name)
34
35enum amdgpu_userq_state {
36 AMDGPU_USERQ_STATE_UNMAPPED = 0,
37 AMDGPU_USERQ_STATE_MAPPED,
38 AMDGPU_USERQ_STATE_PREEMPTED,
39 AMDGPU_USERQ_STATE_HUNG,
40 AMDGPU_USERQ_STATE_INVALID_VA,
41};
42
43struct amdgpu_mqd_prop;
44
45struct amdgpu_userq_obj {
46 void *cpu_ptr;
47 uint64_t gpu_addr;
48 struct amdgpu_bo *obj;
49};
50
51struct amdgpu_usermode_queue {
52 int queue_type;
53 enum amdgpu_userq_state state;
54 uint64_t doorbell_handle;
55 uint64_t doorbell_index;
56 u32 doorbell_offset;
57 uint64_t flags;
58 struct amdgpu_mqd_prop *userq_prop;
59 struct amdgpu_userq_mgr *userq_mgr;
60 struct amdgpu_vm *vm;
61 struct amdgpu_userq_obj mqd;
62 struct amdgpu_userq_obj db_obj;
63 struct amdgpu_userq_obj fw_obj;
64 struct amdgpu_userq_obj wptr_obj;
65
66 /**
67 * @fence_drv_lock: Protecting @fence_drv_xa.
68 */
69 struct mutex fence_drv_lock;
70
71 /**
72 * @fence_drv_xa:
73 *
74 * References to the external fence drivers returned by wait_ioctl.
75 * Dropped on the next signaled dma_fence or queue destruction.
76 */
77 struct xarray fence_drv_xa;
78 struct amdgpu_userq_fence_driver *fence_drv;
79 struct dma_fence *last_fence;
80 u32 xcp_id;
81 int priority;
82 struct dentry *debugfs_queue;
83
84 /**
85 * @hang_detect_work:
86 *
87 * Delayed work which runs when userq_fences time out.
88 */
89 struct delayed_work hang_detect_work;
90 struct kref refcount;
91
92 union {
93 struct {
94 u64 queue_rb;
95 u64 wptr;
96 u64 rptr;
97 u64 eop;
98 u64 shadow;
99 u64 csa;
100 } va;
101 u64 va_array[6];
102 } userq_vas;
103
104 uint32_t gang_ctx_array_index;
105};
106
107struct amdgpu_userq_funcs {
108 int (*mqd_create)(struct amdgpu_usermode_queue *queue,
109 struct drm_amdgpu_userq_in *args);
110 int (*mqd_update)(struct amdgpu_usermode_queue *queue,
111 struct drm_amdgpu_userq_in *args);
112 void (*mqd_destroy)(struct amdgpu_usermode_queue *uq);
113 int (*unmap)(struct amdgpu_usermode_queue *queue);
114 int (*map)(struct amdgpu_usermode_queue *queue);
115 int (*preempt)(struct amdgpu_usermode_queue *queue);
116 int (*restore)(struct amdgpu_usermode_queue *queue);
117 int (*reset)(struct amdgpu_usermode_queue *queue);
118};
119
120/* Usermode queues for gfx */
121struct amdgpu_userq_mgr {
122 /**
123 * @userq_xa: Per-process user queue map (queue ID → queue)
124 * Key: queue_id (unique ID within the process's userq manager)
125 * Value: struct amdgpu_usermode_queue
126 */
127 struct xarray userq_xa;
128 struct mutex userq_mutex;
129 struct amdgpu_device *adev;
130 struct delayed_work resume_work;
131 struct drm_file *file;
132 struct mutex proc_ctx_lock;
133 struct amdgpu_userq_obj proc_ctx_obj;
134
135 bool proc_ctx_allocated;
136 uint32_t proc_ctx_array_index;
137 /**
138 * @reset_work:
139 *
140 * Reset work which is used when eviction fails.
141 */
142 struct work_struct reset_work;
143 atomic_t userq_count[AMDGPU_RING_TYPE_MAX];
144};
145
146struct amdgpu_db_info {
147 uint64_t doorbell_handle;
148 uint32_t queue_type;
149 uint32_t doorbell_offset;
150 struct amdgpu_userq_obj *db_obj;
151};
152
153struct amdgpu_usermode_queue *amdgpu_userq_get(struct amdgpu_userq_mgr *uq_mgr, u32 qid);
154void amdgpu_userq_put(struct amdgpu_usermode_queue *queue);
155
156int amdgpu_userq_ioctl(struct drm_device *dev, void *data, struct drm_file *filp);
157
158int amdgpu_userq_mgr_init(struct amdgpu_userq_mgr *userq_mgr, struct drm_file *file_priv,
159 struct amdgpu_device *adev);
160
161void amdgpu_userq_mgr_cancel_reset_work(struct amdgpu_device *adev);
162void amdgpu_userq_mgr_cancel_resume(struct amdgpu_userq_mgr *userq_mgr);
163void amdgpu_userq_mgr_fini(struct amdgpu_userq_mgr *userq_mgr);
164
165void amdgpu_userq_wait_for_signal(struct amdgpu_userq_mgr *uq_mgr);
166void amdgpu_userq_evict(struct amdgpu_userq_mgr *uq_mgr);
167
168void amdgpu_userq_ensure_ev_fence(struct amdgpu_userq_mgr *userq_mgr,
169 struct amdgpu_eviction_fence_mgr *evf_mgr);
170
171u32 amdgpu_userq_get_supported_ip_mask(struct amdgpu_device *adev);
172bool amdgpu_userq_enabled(struct drm_device *dev);
173
174int amdgpu_userq_suspend(struct amdgpu_device *adev);
175int amdgpu_userq_resume(struct amdgpu_device *adev);
176
177int amdgpu_userq_stop_sched_for_enforce_isolation(struct amdgpu_device *adev,
178 u32 idx);
179int amdgpu_userq_start_sched_for_enforce_isolation(struct amdgpu_device *adev,
180 u32 idx);
181void amdgpu_userq_reset_work(struct work_struct *work);
182void amdgpu_userq_pre_reset(struct amdgpu_device *adev);
183int amdgpu_userq_post_reset(struct amdgpu_device *adev, bool vram_lost);
184void amdgpu_userq_start_hang_detect_work(struct amdgpu_usermode_queue *queue);
185void amdgpu_userq_process_fence_irq(struct amdgpu_device *adev, u32 doorbell);
186
187/*
188 * CP packs the per-process doorbell_id of the queue in
189 * CTXID0[9:0] on priv-fault (same encoding KFD uses via
190 * KFD_CTXID0_DOORBELL_ID_MASK)
191 */
192#define AMDGPU_CTXID0_DOORBELL_ID_MASK 0x3ff
193
194void amdgpu_userq_process_reset_irq(struct amdgpu_device *adev,
195 u32 pasid, u32 doorbell_offset);
196
197int amdgpu_userq_input_va_validate(struct amdgpu_device *adev,
198 struct amdgpu_usermode_queue *queue,
199 u64 addr, u64 expected_size, u64 *va_out);
200
201void amdgpu_userq_gem_va_unmap_validate(struct amdgpu_device *adev,
202 struct amdgpu_bo_va_mapping *mapping);
203#endif