projects/linux·blob master · refs

382 lines · 12.5 KiB

1/* SPDX-License-Identifier: GPL-2.0 OR MIT */
2/*
3 * Copyright 2014-2022 Advanced Micro Devices, Inc.
4 *
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the "Software"),
7 * to deal in the Software without restriction, including without limitation
8 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9 * and/or sell copies of the Software, and to permit persons to whom the
10 * Software is furnished to do so, subject to the following conditions:
11 *
12 * The above copyright notice and this permission notice shall be included in
13 * all copies or substantial portions of the Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
19 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
20 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
21 * OTHER DEALINGS IN THE SOFTWARE.
22 *
23 */
24
25#ifndef KFD_DEVICE_QUEUE_MANAGER_H_
26#define KFD_DEVICE_QUEUE_MANAGER_H_
27
28#include <linux/rwsem.h>
29#include <linux/list.h>
30#include <linux/mutex.h>
31#include <linux/sched/mm.h>
32#include "kfd_priv.h"
33#include "kfd_mqd_manager.h"
34
35#define VMID_NUM 16
36
37#define KFD_MES_PROCESS_QUANTUM 100000
38#define KFD_MES_GANG_QUANTUM 10000
39
40struct device_process_node {
41 struct qcm_process_device *qpd;
42 struct list_head list;
43};
44
45union SQ_CMD_BITS {
46 struct {
47 uint32_t cmd:3;
48 uint32_t:1;
49 uint32_t mode:3;
50 uint32_t check_vmid:1;
51 uint32_t trap_id:3;
52 uint32_t:5;
53 uint32_t wave_id:4;
54 uint32_t simd_id:2;
55 uint32_t:2;
56 uint32_t queue_id:3;
57 uint32_t:1;
58 uint32_t vm_id:4;
59 } bitfields, bits;
60 uint32_t u32All;
61 signed int i32All;
62 float f32All;
63};
64
65union GRBM_GFX_INDEX_BITS {
66 struct {
67 uint32_t instance_index:8;
68 uint32_t sh_index:8;
69 uint32_t se_index:8;
70 uint32_t:5;
71 uint32_t sh_broadcast_writes:1;
72 uint32_t instance_broadcast_writes:1;
73 uint32_t se_broadcast_writes:1;
74 } bitfields, bits;
75 uint32_t u32All;
76 signed int i32All;
77 float f32All;
78};
79
80/**
81 * struct device_queue_manager_ops
82 *
83 * @create_queue: Queue creation routine.
84 *
85 * @destroy_queue: Queue destruction routine.
86 *
87 * @update_queue: Queue update routine.
88 *
89 * @exeute_queues: Dispatches the queues list to the H/W.
90 *
91 * @register_process: This routine associates a specific process with device.
92 *
93 * @unregister_process: destroys the associations between process to device.
94 *
95 * @initialize: Initializes the pipelines and memory module for that device.
96 *
97 * @start: Initializes the resources/modules the device needs for queues
98 * execution. This function is called on device initialization and after the
99 * system woke up after suspension.
100 *
101 * @stop: This routine stops execution of all the active queue running on the
102 * H/W and basically this function called on system suspend.
103 *
104 * @uninitialize: Destroys all the device queue manager resources allocated in
105 * initialize routine.
106 *
107 * @halt: This routine unmaps queues from runlist and set halt status to true
108 * so no more queues will be mapped to runlist until unhalt.
109 *
110 * @unhalt: This routine unset halt status to flase and maps queues back to
111 * runlist.
112 *
113 * @create_kernel_queue: Creates kernel queue. Used for debug queue.
114 *
115 * @destroy_kernel_queue: Destroys kernel queue. Used for debug queue.
116 *
117 * @set_cache_memory_policy: Sets memory policy (cached/ non cached) for the
118 * memory apertures.
119 *
120 * @process_termination: Clears all process queues belongs to that device.
121 *
122 * @evict_process_queues: Evict all active queues of a process
123 *
124 * @restore_process_queues: Restore all evicted queues of a process
125 *
126 * @get_wave_state: Retrieves context save state and optionally copies the
127 * control stack, if kept in the MQD, to the given userspace address.
128 *
129 * @reset_queues: reset queues which consume RAS poison
130 * @get_queue_checkpoint_info: Retrieves queue size information for CRIU checkpoint.
131 *
132 * @checkpoint_mqd: checkpoint queue MQD contents for CRIU.
133 */
134
135struct device_queue_manager_ops {
136 int (*create_queue)(struct device_queue_manager *dqm,
137 struct queue *q,
138 struct qcm_process_device *qpd,
139 const struct kfd_criu_queue_priv_data *qd,
140 const void *restore_mqd,
141 const void *restore_ctl_stack);
142
143 int (*destroy_queue)(struct device_queue_manager *dqm,
144 struct qcm_process_device *qpd,
145 struct queue *q);
146
147 int (*update_queue)(struct device_queue_manager *dqm,
148 struct queue *q, struct mqd_update_info *minfo);
149
150 int (*register_process)(struct device_queue_manager *dqm,
151 struct qcm_process_device *qpd);
152
153 int (*unregister_process)(struct device_queue_manager *dqm,
154 struct qcm_process_device *qpd);
155
156 int (*initialize)(struct device_queue_manager *dqm);
157 int (*start)(struct device_queue_manager *dqm);
158 int (*stop)(struct device_queue_manager *dqm);
159 void (*uninitialize)(struct device_queue_manager *dqm);
160 int (*halt)(struct device_queue_manager *dqm);
161 int (*unhalt)(struct device_queue_manager *dqm);
162 int (*create_kernel_queue)(struct device_queue_manager *dqm,
163 struct kernel_queue *kq,
164 struct qcm_process_device *qpd);
165
166 void (*destroy_kernel_queue)(struct device_queue_manager *dqm,
167 struct kernel_queue *kq,
168 struct qcm_process_device *qpd);
169
170 bool (*set_cache_memory_policy)(struct device_queue_manager *dqm,
171 struct qcm_process_device *qpd,
172 enum cache_policy default_policy,
173 enum cache_policy alternate_policy,
174 void __user *alternate_aperture_base,
175 uint64_t alternate_aperture_size,
176 u32 misc_process_properties);
177
178 int (*process_termination)(struct device_queue_manager *dqm,
179 struct qcm_process_device *qpd);
180
181 int (*evict_process_queues)(struct device_queue_manager *dqm,
182 struct qcm_process_device *qpd);
183 int (*restore_process_queues)(struct device_queue_manager *dqm,
184 struct qcm_process_device *qpd);
185
186 int (*get_wave_state)(struct device_queue_manager *dqm,
187 struct queue *q,
188 void __user *ctl_stack,
189 u32 *ctl_stack_used_size,
190 u32 *save_area_used_size);
191
192 int (*reset_queues)(struct device_queue_manager *dqm,
193 uint16_t pasid);
194 int (*get_queue_checkpoint_info)(struct device_queue_manager *dqm,
195 const struct queue *q, u32 *mqd_size,
196 u32 *ctl_stack_size);
197
198 int (*checkpoint_mqd)(struct device_queue_manager *dqm,
199 const struct queue *q,
200 void *mqd,
201 void *ctl_stack);
202 void (*set_perfcount)(struct device_queue_manager *dqm,
203 int enable);
204};
205
206struct device_queue_manager_asic_ops {
207 int (*update_qpd)(struct device_queue_manager *dqm,
208 struct qcm_process_device *qpd);
209 bool (*set_cache_memory_policy)(struct device_queue_manager *dqm,
210 struct qcm_process_device *qpd,
211 enum cache_policy default_policy,
212 enum cache_policy alternate_policy,
213 void __user *alternate_aperture_base,
214 uint64_t alternate_aperture_size,
215 u32 misc_process_properties);
216 void (*init_sdma_vm)(struct device_queue_manager *dqm,
217 struct queue *q,
218 struct qcm_process_device *qpd);
219 struct mqd_manager * (*mqd_manager_init)(enum KFD_MQD_TYPE type,
220 struct kfd_node *dev);
221};
222
223struct dqm_detect_hang_info {
224 int pipe_id;
225 int queue_id;
226 int xcc_id;
227 uint64_t queue_address;
228};
229
230/**
231 * struct device_queue_manager
232 *
233 * This struct is a base class for the kfd queues scheduler in the
234 * device level. The device base class should expose the basic operations
235 * for queue creation and queue destruction. This base class hides the
236 * scheduling mode of the driver and the specific implementation of the
237 * concrete device. This class is the only class in the queues scheduler
238 * that configures the H/W.
239 *
240 */
241
242struct device_queue_manager {
243 struct device_queue_manager_ops ops;
244 struct device_queue_manager_asic_ops asic_ops;
245
246 struct mqd_manager *mqd_mgrs[KFD_MQD_TYPE_MAX];
247 struct packet_manager packet_mgr;
248 struct kfd_node *dev;
249 struct mutex lock_hidden; /* use dqm_lock/unlock(dqm) */
250 struct list_head queues;
251 unsigned int saved_flags;
252 unsigned int processes_count;
253 unsigned int active_queue_count;
254 unsigned int active_cp_queue_count;
255 unsigned int gws_queue_count;
256 unsigned int total_queue_count;
257 unsigned int next_pipe_to_allocate;
258 unsigned int *allocated_queues;
259 DECLARE_BITMAP(sdma_bitmap, KFD_MAX_SDMA_QUEUES);
260 DECLARE_BITMAP(xgmi_sdma_bitmap, KFD_MAX_SDMA_QUEUES);
261 /* the pasid mapping for each kfd vmid */
262 uint16_t vmid_pasid[VMID_NUM];
263 uint64_t pipelines_addr;
264 uint64_t fence_gpu_addr;
265 uint64_t *fence_addr;
266 struct kfd_mem_obj *fence_mem;
267 bool active_runlist;
268 int sched_policy;
269 uint32_t trap_debug_vmid;
270
271 /* hw exception */
272 bool is_hws_hang;
273 bool is_resetting;
274 struct kfd_mem_obj hiq_sdma_mqd;
275 bool sched_running;
276 bool sched_halt;
277
278 /* used for GFX 9.4.3 only */
279 uint32_t current_logical_xcc_start;
280
281 uint32_t wait_times;
282
283 wait_queue_head_t destroy_wait;
284
285 /* for per-queue reset support */
286 struct dqm_detect_hang_info *detect_hang_info;
287 size_t detect_hang_info_size;
288 int detect_hang_count;
289 /* for per-queue reset with mes */
290 u32 *hung_db_array;
291 struct amdgpu_mes_hung_queue_hqd_info *hqd_info;
292};
293
294void device_queue_manager_init_cik(
295 struct device_queue_manager_asic_ops *asic_ops);
296void device_queue_manager_init_vi(
297 struct device_queue_manager_asic_ops *asic_ops);
298void device_queue_manager_init_v9(
299 struct device_queue_manager_asic_ops *asic_ops);
300void device_queue_manager_init_v10(
301 struct device_queue_manager_asic_ops *asic_ops);
302void device_queue_manager_init_v11(
303 struct device_queue_manager_asic_ops *asic_ops);
304void device_queue_manager_init_v12(
305 struct device_queue_manager_asic_ops *asic_ops);
306void device_queue_manager_init_v12_1(
307 struct device_queue_manager_asic_ops *asic_ops);
308void program_sh_mem_settings(struct device_queue_manager *dqm,
309 struct qcm_process_device *qpd);
310unsigned int get_cp_queues_num(struct device_queue_manager *dqm);
311unsigned int get_queues_per_pipe(struct device_queue_manager *dqm);
312unsigned int get_pipes_per_mec(struct device_queue_manager *dqm);
313unsigned int get_num_sdma_queues(struct device_queue_manager *dqm);
314unsigned int get_num_xgmi_sdma_queues(struct device_queue_manager *dqm);
315int reserve_debug_trap_vmid(struct device_queue_manager *dqm,
316 struct qcm_process_device *qpd);
317int release_debug_trap_vmid(struct device_queue_manager *dqm,
318 struct qcm_process_device *qpd);
319int suspend_queues(struct kfd_process *p,
320 uint32_t num_queues,
321 uint32_t grace_period,
322 uint64_t exception_clear_mask,
323 uint32_t *usr_queue_id_array);
324int resume_queues(struct kfd_process *p,
325 uint32_t num_queues,
326 uint32_t *usr_queue_id_array);
327void set_queue_snapshot_entry(struct queue *q,
328 uint64_t exception_clear_mask,
329 struct kfd_queue_snapshot_entry *qss_entry);
330int debug_lock_and_unmap(struct device_queue_manager *dqm);
331int debug_map_and_unlock(struct device_queue_manager *dqm);
332int debug_refresh_runlist(struct device_queue_manager *dqm);
333bool kfd_dqm_is_queue_in_process(struct device_queue_manager *dqm,
334 struct qcm_process_device *qpd,
335 int doorbell_off, u32 *queue_format);
336int kfd_reset_queue_mes(struct device_queue_manager *dqm, int queue_type,
337 int pipe, int queue, unsigned int db);
338size_t mqd_size_from_queue_type(struct device_queue_manager *dqm,
339 enum kfd_queue_type type);
340
341static inline unsigned int get_sh_mem_bases_32(struct kfd_process_device *pdd)
342{
343 return (pdd->lds_base >> 16) & 0xFF;
344}
345
346static inline unsigned int
347get_sh_mem_bases_nybble_64(struct kfd_process_device *pdd)
348{
349 return (pdd->lds_base >> 60) & 0x0E;
350}
351
352/* The DQM lock can be taken in MMU notifiers. Make sure no reclaim-FS
353 * happens while holding this lock anywhere to prevent deadlocks when
354 * an MMU notifier runs in reclaim-FS context.
355 */
356static inline void dqm_lock(struct device_queue_manager *dqm)
357{
358 mutex_lock(&dqm->lock_hidden);
359 dqm->saved_flags = memalloc_noreclaim_save();
360}
361static inline void dqm_unlock(struct device_queue_manager *dqm)
362{
363 memalloc_noreclaim_restore(dqm->saved_flags);
364 mutex_unlock(&dqm->lock_hidden);
365}
366
367static inline int read_sdma_queue_counter(uint64_t __user *q_rptr, uint64_t *val)
368{
369 /* SDMA activity counter is stored at queue's RPTR + 0x8 location. */
370 return get_user(*val, q_rptr + 1);
371}
372
373static inline void update_dqm_wait_times(struct device_queue_manager *dqm)
374{
375 if (dqm->dev->kfd2kgd->get_iq_wait_times)
376 dqm->dev->kfd2kgd->get_iq_wait_times(dqm->dev->adev,
377 &dqm->wait_times,
378 ffs(dqm->dev->xcc_mask) - 1);
379}
380
381
382#endif /* KFD_DEVICE_QUEUE_MANAGER_H_ */