projects/linux·blob master · refs

646 lines · 20.5 KiB

1/*
2 * Copyright 2019 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 */
23
24#ifndef __AMDGPU_MES_H__
25#define __AMDGPU_MES_H__
26
27#include "amdgpu_irq.h"
28#include "kgd_kfd_interface.h"
29#include "amdgpu_gfx.h"
30#include "amdgpu_doorbell.h"
31#include <linux/sched/mm.h>
32
33#define AMDGPU_MES_MAX_COMPUTE_PIPES 8
34#define AMDGPU_MES_MAX_GFX_PIPES 2
35#define AMDGPU_MES_MAX_SDMA_PIPES 2
36
37#define AMDGPU_MES_API_VERSION_SHIFT 12
38#define AMDGPU_MES_FEAT_VERSION_SHIFT 24
39
40#define AMDGPU_MES_VERSION_MASK 0x00000fff
41#define AMDGPU_MES_API_VERSION_MASK 0x00fff000
42#define AMDGPU_MES_FEAT_VERSION_MASK 0xff000000
43#define AMDGPU_MES_MSCRATCH_SIZE 0x40000
44#define AMDGPU_MES_INVALID_DB_OFFSET 0xffffffff
45
46enum amdgpu_mes_priority_level {
47 AMDGPU_MES_PRIORITY_LEVEL_LOW = 0,
48 AMDGPU_MES_PRIORITY_LEVEL_NORMAL = 1,
49 AMDGPU_MES_PRIORITY_LEVEL_MEDIUM = 2,
50 AMDGPU_MES_PRIORITY_LEVEL_HIGH = 3,
51 AMDGPU_MES_PRIORITY_LEVEL_REALTIME = 4,
52 AMDGPU_MES_PRIORITY_NUM_LEVELS
53};
54
55#define AMDGPU_MES_PROC_CTX_SIZE 0x1000 /* one page area */
56#define AMDGPU_MES_GANG_CTX_SIZE 0x1000 /* one page area */
57#define AMDGPU_MES_PROC_CTX_ARRAY_MAX 128
58#define AMDGPU_MES_GANG_CTX_ARRAY_MAX 512
59struct amdgpu_mes_funcs;
60
61enum amdgpu_mes_pipe {
62 AMDGPU_MES_PIPE_0 = 0,
63 AMDGPU_MES_PIPE_1,
64 AMDGPU_MAX_MES_PIPES = 2,
65};
66
67#define AMDGPU_MES_SCHED_PIPE AMDGPU_MES_PIPE_0
68#define AMDGPU_MES_KIQ_PIPE AMDGPU_MES_PIPE_1
69
70#define AMDGPU_MAX_MES_INST_PIPES \
71 (AMDGPU_MAX_MES_PIPES * AMDGPU_MAX_GC_INSTANCES)
72
73#define MES_PIPE_INST(xcc_id, pipe_id) \
74 (xcc_id * AMDGPU_MAX_MES_PIPES + pipe_id)
75
76struct amdgpu_mes {
77 struct amdgpu_device *adev;
78
79 struct mutex mutex_hidden;
80
81 struct ida doorbell_ida;
82
83 spinlock_t queue_id_lock;
84
85 uint32_t sched_version;
86 uint32_t kiq_version;
87 uint32_t fw_version[AMDGPU_MAX_MES_PIPES];
88 bool enable_legacy_queue_map;
89
90 uint32_t total_max_queue;
91 uint32_t max_doorbell_slices;
92
93 uint64_t default_process_quantum;
94 uint64_t default_gang_quantum;
95
96 struct amdgpu_ring ring[AMDGPU_MAX_MES_INST_PIPES];
97 spinlock_t ring_lock[AMDGPU_MAX_MES_INST_PIPES];
98
99 const struct firmware *fw[AMDGPU_MAX_MES_PIPES];
100
101 /* mes ucode */
102 struct amdgpu_bo *ucode_fw_obj[AMDGPU_MAX_MES_INST_PIPES];
103 uint64_t ucode_fw_gpu_addr[AMDGPU_MAX_MES_INST_PIPES];
104 uint32_t *ucode_fw_ptr[AMDGPU_MAX_MES_INST_PIPES];
105 uint64_t uc_start_addr[AMDGPU_MAX_MES_PIPES];
106
107 /* mes ucode data */
108 struct amdgpu_bo *data_fw_obj[AMDGPU_MAX_MES_INST_PIPES];
109 uint64_t data_fw_gpu_addr[AMDGPU_MAX_MES_INST_PIPES];
110 uint32_t *data_fw_ptr[AMDGPU_MAX_MES_INST_PIPES];
111 uint64_t data_start_addr[AMDGPU_MAX_MES_PIPES];
112
113 /* eop gpu obj */
114 struct amdgpu_bo *eop_gpu_obj[AMDGPU_MAX_MES_INST_PIPES];
115 uint64_t eop_gpu_addr[AMDGPU_MAX_MES_INST_PIPES];
116
117 void *mqd_backup[AMDGPU_MAX_MES_INST_PIPES];
118 struct amdgpu_irq_src irq[AMDGPU_MAX_MES_INST_PIPES];
119
120 uint32_t vmid_mask_gfxhub;
121 uint32_t vmid_mask_mmhub;
122 uint32_t gfx_hqd_mask[AMDGPU_MES_MAX_GFX_PIPES];
123 uint32_t compute_hqd_mask[AMDGPU_MES_MAX_COMPUTE_PIPES];
124 uint32_t sdma_hqd_mask[AMDGPU_MES_MAX_SDMA_PIPES];
125 uint32_t aggregated_doorbells[AMDGPU_MES_PRIORITY_NUM_LEVELS];
126
127 uint32_t sch_ctx_offs[AMDGPU_MAX_MES_INST_PIPES];
128 uint64_t sch_ctx_gpu_addr[AMDGPU_MAX_MES_INST_PIPES];
129 uint64_t *sch_ctx_ptr[AMDGPU_MAX_MES_INST_PIPES];
130 uint32_t query_status_fence_offs[AMDGPU_MAX_MES_INST_PIPES];
131 uint64_t query_status_fence_gpu_addr[AMDGPU_MAX_MES_INST_PIPES];
132 uint64_t *query_status_fence_ptr[AMDGPU_MAX_MES_INST_PIPES];
133
134 uint32_t saved_flags;
135
136 /* initialize kiq pipe */
137 int (*kiq_hw_init)(struct amdgpu_device *adev,
138 uint32_t xcc_id);
139 int (*kiq_hw_fini)(struct amdgpu_device *adev,
140 uint32_t xcc_id);
141
142 /* MES doorbells */
143 uint32_t db_start_dw_offset;
144 uint32_t num_mes_dbs;
145 unsigned long *doorbell_bitmap;
146
147 /* MES event log buffer */
148 uint32_t event_log_size;
149 struct amdgpu_bo *event_log_gpu_obj;
150 uint64_t event_log_gpu_addr;
151 void *event_log_cpu_addr;
152
153 /* ip specific functions */
154 const struct amdgpu_mes_funcs *funcs;
155
156 /* mes resource_1 bo*/
157 struct amdgpu_bo *resource_1[AMDGPU_MAX_MES_PIPES];
158 uint64_t resource_1_gpu_addr[AMDGPU_MAX_MES_PIPES];
159 void *resource_1_addr[AMDGPU_MAX_MES_PIPES];
160
161 int hung_queue_db_array_size;
162 int hung_queue_hqd_info_offset;
163 struct amdgpu_bo *hung_queue_db_array_gpu_obj[AMDGPU_MAX_MES_INST_PIPES];
164 uint64_t hung_queue_db_array_gpu_addr[AMDGPU_MAX_MES_INST_PIPES];
165 void *hung_queue_db_array_cpu_addr[AMDGPU_MAX_MES_INST_PIPES];
166
167 /* cooperative dispatch */
168 bool enable_coop_mode;
169 int master_xcc_ids[AMDGPU_MAX_MES_INST_PIPES];
170 struct amdgpu_bo *shared_cmd_buf_obj[AMDGPU_MAX_MES_INST_PIPES];
171 uint64_t shared_cmd_buf_gpu_addr[AMDGPU_MAX_MES_INST_PIPES];
172
173 bool compute_pipe_reset_enabled;
174 bool gfx_pipe_reset_enabled;
175
176 bool use_rs64mem;
177 struct amdgpu_bo *ctx_array_size_bo;
178 uint64_t ctx_array_size_gpu_addr;
179 uint32_t *ctx_array_size_cpu_ptr;
180
181 uint32_t proc_ctx_array_size;
182 unsigned long *proc_ctx_bitmap;
183 uint32_t gang_ctx_array_size;
184 uint32_t gang_ctx_array_index;
185 unsigned long *gang_ctx_bitmap;
186};
187
188struct amdgpu_mes_hung_queue_hqd_info {
189 union {
190 struct {
191 u32 queue_type: 3; // queue type
192 u32 pipe_index: 4; // pipe index
193 u32 queue_index: 8; // queue index
194 u32 reserved: 17;
195 };
196
197 u32 bit0_31;
198 };
199};
200
201struct amdgpu_mes_gang {
202 int gang_id;
203 int priority;
204 int inprocess_gang_priority;
205 int global_priority_level;
206 struct list_head list;
207 struct amdgpu_mes_process *process;
208 struct amdgpu_bo *gang_ctx_bo;
209 uint64_t gang_ctx_gpu_addr;
210 void *gang_ctx_cpu_ptr;
211 uint64_t gang_quantum;
212 struct list_head queue_list;
213};
214
215struct amdgpu_mes_queue {
216 struct list_head list;
217 struct amdgpu_mes_gang *gang;
218 int queue_id;
219 uint64_t doorbell_off;
220 struct amdgpu_bo *mqd_obj;
221 void *mqd_cpu_ptr;
222 uint64_t mqd_gpu_addr;
223 uint64_t wptr_gpu_addr;
224 int queue_type;
225 int paging;
226 struct amdgpu_ring *ring;
227};
228
229struct amdgpu_mes_queue_properties {
230 int queue_type;
231 uint64_t hqd_base_gpu_addr;
232 uint64_t rptr_gpu_addr;
233 uint64_t wptr_gpu_addr;
234 uint64_t wptr_mc_addr;
235 uint32_t queue_size;
236 uint64_t eop_gpu_addr;
237 uint32_t hqd_pipe_priority;
238 uint32_t hqd_queue_priority;
239 bool paging;
240 struct amdgpu_ring *ring;
241 /* out */
242 uint64_t doorbell_off;
243};
244
245struct amdgpu_mes_gang_properties {
246 uint32_t priority;
247 uint32_t gang_quantum;
248 uint32_t inprocess_gang_priority;
249 uint32_t priority_level;
250 int global_priority_level;
251};
252
253struct mes_add_queue_input {
254 uint32_t xcc_id;
255 uint32_t process_id;
256 uint64_t page_table_base_addr;
257 uint64_t process_va_start;
258 uint64_t process_va_end;
259 uint64_t process_quantum;
260 uint64_t process_context_addr;
261 uint64_t gang_quantum;
262 uint64_t gang_context_addr;
263 uint32_t inprocess_gang_priority;
264 uint32_t gang_global_priority_level;
265 uint32_t doorbell_offset;
266 uint64_t mqd_addr;
267 uint64_t wptr_addr;
268 uint64_t wptr_mc_addr;
269 uint32_t queue_type;
270 uint32_t paging;
271 uint32_t gws_base;
272 uint32_t gws_size;
273 uint64_t tba_addr;
274 uint64_t tma_addr;
275 uint32_t trap_en;
276 uint32_t skip_process_ctx_clear;
277 uint32_t is_kfd_process;
278 uint32_t is_aql_queue;
279 uint32_t queue_size;
280 uint32_t exclusively_scheduled;
281 uint32_t sh_mem_config_data;
282 uint32_t vm_cntx_cntl;
283 uint32_t process_context_array_index;
284 uint32_t gang_context_array_index;
285};
286
287struct mes_remove_queue_input {
288 uint32_t xcc_id;
289 uint32_t doorbell_offset;
290 uint64_t gang_context_addr;
291 uint32_t queue_type;
292 bool remove_queue_after_reset;
293 uint32_t gang_context_array_index;
294};
295
296struct mes_map_legacy_queue_input {
297 uint32_t xcc_id;
298 uint32_t queue_type;
299 uint32_t doorbell_offset;
300 uint32_t pipe_id;
301 uint32_t queue_id;
302 uint64_t mqd_addr;
303 uint64_t wptr_addr;
304};
305
306struct mes_unmap_legacy_queue_input {
307 uint32_t xcc_id;
308 enum amdgpu_unmap_queues_action action;
309 uint32_t queue_type;
310 uint32_t doorbell_offset;
311 uint32_t pipe_id;
312 uint32_t queue_id;
313 uint64_t trail_fence_addr;
314 uint64_t trail_fence_data;
315};
316
317struct mes_suspend_gang_input {
318 uint32_t xcc_id;
319 bool suspend_all_gangs;
320 bool suspend_all_sdma_gangs;
321 uint64_t gang_context_addr;
322 uint64_t suspend_fence_addr;
323 uint32_t suspend_fence_value;
324 uint32_t doorbell_offset;
325};
326
327struct mes_resume_gang_input {
328 uint32_t xcc_id;
329 bool resume_all_gangs;
330 uint64_t gang_context_addr;
331 uint32_t doorbell_offset;
332};
333
334struct mes_reset_queue_input {
335 uint32_t xcc_id;
336 uint32_t queue_type;
337 uint32_t doorbell_offset;
338 bool use_mmio;
339 uint32_t me_id;
340 uint32_t pipe_id;
341 uint32_t queue_id;
342 uint64_t mqd_addr;
343 uint64_t wptr_addr;
344 uint32_t vmid;
345 bool legacy_gfx;
346 bool is_kq;
347};
348
349struct mes_detect_and_reset_queue_input {
350 u32 queue_type;
351 bool detect_only;
352 u32 xcc_id;
353};
354
355struct mes_inv_tlbs_pasid_input {
356 uint32_t xcc_id;
357 uint16_t pasid;
358 uint8_t hub_id;
359 uint8_t flush_type;
360};
361
362enum mes_misc_opcode {
363 MES_MISC_OP_WRITE_REG,
364 MES_MISC_OP_READ_REG,
365 MES_MISC_OP_WRM_REG_WAIT,
366 MES_MISC_OP_WRM_REG_WR_WAIT,
367 MES_MISC_OP_SET_SHADER_DEBUGGER,
368 MES_MISC_OP_CHANGE_CONFIG,
369};
370
371struct mes_misc_op_input {
372 uint32_t xcc_id;
373 enum mes_misc_opcode op;
374
375 union {
376 struct {
377 uint32_t reg_offset;
378 uint64_t buffer_addr;
379 } read_reg;
380
381 struct {
382 uint32_t reg_offset;
383 uint32_t reg_value;
384 } write_reg;
385
386 struct {
387 uint32_t ref;
388 uint32_t mask;
389 uint32_t reg0;
390 uint32_t reg1;
391 } wrm_reg;
392
393 struct {
394 uint64_t process_context_addr;
395 union {
396 struct {
397 uint32_t single_memop : 1;
398 uint32_t single_alu_op : 1;
399 uint32_t reserved: 29;
400 uint32_t process_ctx_flush: 1;
401 };
402 uint32_t u32all;
403 } flags;
404 uint32_t spi_gdbg_per_vmid_cntl;
405 uint32_t tcp_watch_cntl[4];
406 uint32_t trap_en;
407 } set_shader_debugger;
408
409 struct {
410 union {
411 struct {
412 uint32_t limit_single_process : 1;
413 uint32_t enable_hws_logging_buffer : 1;
414 uint32_t reserved : 30;
415 };
416 uint32_t all;
417 } option;
418 struct {
419 uint32_t tdr_level;
420 uint32_t tdr_delay;
421 } tdr_config;
422 } change_config;
423 };
424};
425
426struct amdgpu_mes_funcs {
427 int (*add_hw_queue)(struct amdgpu_mes *mes,
428 struct mes_add_queue_input *input);
429
430 int (*remove_hw_queue)(struct amdgpu_mes *mes,
431 struct mes_remove_queue_input *input);
432
433 int (*map_legacy_queue)(struct amdgpu_mes *mes,
434 struct mes_map_legacy_queue_input *input);
435
436 int (*unmap_legacy_queue)(struct amdgpu_mes *mes,
437 struct mes_unmap_legacy_queue_input *input);
438
439 int (*suspend_gang)(struct amdgpu_mes *mes,
440 struct mes_suspend_gang_input *input);
441
442 int (*resume_gang)(struct amdgpu_mes *mes,
443 struct mes_resume_gang_input *input);
444
445 int (*misc_op)(struct amdgpu_mes *mes,
446 struct mes_misc_op_input *input);
447
448 int (*reset_hw_queue)(struct amdgpu_mes *mes,
449 struct mes_reset_queue_input *input);
450
451 int (*detect_and_reset_hung_queues)(struct amdgpu_mes *mes,
452 struct mes_detect_and_reset_queue_input *input);
453
454
455 int (*invalidate_tlbs_pasid)(struct amdgpu_mes *mes,
456 struct mes_inv_tlbs_pasid_input *input);
457};
458
459enum amdgpu_mqd_update_flag {
460 AMDGPU_UPDATE_FLAG_DBG_WA_ENABLE = 1,
461 AMDGPU_UPDATE_FLAG_DBG_WA_DISABLE = 2,
462 AMDGPU_UPDATE_FLAG_IS_GWS = 4, /* quirk for gfx9 IP */
463};
464
465struct amdgpu_mqd_prop {
466 uint64_t mqd_gpu_addr;
467 uint64_t hqd_base_gpu_addr;
468 uint64_t rptr_gpu_addr;
469 uint64_t wptr_gpu_addr;
470 uint32_t queue_size;
471 bool use_doorbell;
472 uint32_t doorbell_index;
473 uint64_t eop_gpu_addr;
474 uint32_t hqd_pipe_priority;
475 uint32_t hqd_queue_priority;
476 uint32_t mqd_stride_size;
477 bool allow_tunneling;
478 bool hqd_active;
479 uint64_t shadow_addr;
480 uint64_t gds_bkup_addr;
481 uint64_t csa_addr;
482 uint64_t fence_address;
483 bool tmz_queue;
484 bool kernel_queue;
485 uint32_t *cu_mask;
486 uint32_t cu_mask_count;
487 uint32_t cu_flags;
488 bool is_user_cu_masked;
489};
490
491struct amdgpu_mqd {
492 unsigned mqd_size;
493 int (*init_mqd)(struct amdgpu_device *adev, void *mqd,
494 struct amdgpu_mqd_prop *p);
495};
496
497/*
498 * MES FW uses address(mqd_addr + sizeof(struct mqd) + 3*sizeof(uint32_t))
499 * as fence address and writes a 32 bit fence value to this address.
500 * Driver needs to allocate at least 4 DWs extra memory in addition to
501 * sizeof(struct mqd). Add 8 DWs and align to AMDGPU_GPU_PAGE_SIZE for safety.
502 */
503#define AMDGPU_MQD_SIZE_ALIGN(mqd_size) AMDGPU_GPU_PAGE_ALIGN(((mqd_size) + 32))
504
505#define amdgpu_mes_kiq_hw_init(adev, xcc_id) \
506 (adev)->mes.kiq_hw_init((adev), (xcc_id))
507#define amdgpu_mes_kiq_hw_fini(adev, xcc_id) \
508 (adev)->mes.kiq_hw_fini((adev), (xcc_id))
509
510int amdgpu_mes_init_microcode(struct amdgpu_device *adev, int pipe);
511void amdgpu_mes_validate_fw_version(struct amdgpu_device *adev);
512int amdgpu_mes_init(struct amdgpu_device *adev);
513void amdgpu_mes_fini(struct amdgpu_device *adev);
514
515int amdgpu_mes_suspend(struct amdgpu_device *adev, u32 xcc_id);
516int amdgpu_mes_resume(struct amdgpu_device *adev, u32 xcc_id);
517
518int amdgpu_mes_map_legacy_queue(struct amdgpu_device *adev,
519 struct amdgpu_ring *ring, uint32_t xcc_id);
520int amdgpu_mes_unmap_legacy_queue(struct amdgpu_device *adev,
521 struct amdgpu_ring *ring,
522 enum amdgpu_unmap_queues_action action,
523 u64 gpu_addr, u64 seq, uint32_t xcc_id);
524int amdgpu_mes_reset_legacy_queue(struct amdgpu_device *adev,
525 struct amdgpu_ring *ring,
526 unsigned int vmid,
527 bool use_mmio,
528 uint32_t xcc_id);
529int amdgpu_mes_reset_queue_mmio(struct amdgpu_device *adev,
530 int queue_type,
531 unsigned int vmid,
532 unsigned int me,
533 unsigned int pipe,
534 unsigned int queue,
535 uint32_t xcc_id);
536int amdgpu_mes_reset_user_queue(struct amdgpu_device *adev,
537 int queue_type,
538 unsigned int doorbell_index,
539 unsigned int xcc_id);
540
541int amdgpu_mes_get_hung_queue_db_array_size(struct amdgpu_device *adev);
542int amdgpu_mes_detect_and_reset_hung_queues(struct amdgpu_device *adev,
543 int queue_type,
544 bool detect_only,
545 unsigned int *hung_db_num,
546 u32 *hung_db_array,
547 uint32_t xcc_id);
548
549uint32_t amdgpu_mes_rreg(struct amdgpu_device *adev, uint32_t reg,
550 uint32_t xcc_id);
551int amdgpu_mes_wreg(struct amdgpu_device *adev,
552 uint32_t reg, uint32_t val, uint32_t xcc_id);
553int amdgpu_mes_reg_write_reg_wait(struct amdgpu_device *adev,
554 uint32_t reg0, uint32_t reg1,
555 uint32_t ref, uint32_t mask, uint32_t xcc_id);
556int amdgpu_mes_hdp_flush(struct amdgpu_device *adev);
557int amdgpu_mes_set_shader_debugger(struct amdgpu_device *adev,
558 uint64_t process_context_addr,
559 uint32_t spi_gdbg_per_vmid_cntl,
560 const uint32_t *tcp_watch_cntl,
561 uint32_t flags,
562 bool trap_en,
563 uint32_t xcc_id);
564int amdgpu_mes_flush_shader_debugger(struct amdgpu_device *adev,
565 uint64_t process_context_addr, uint32_t xcc_id);
566
567uint32_t amdgpu_mes_get_aggregated_doorbell_index(struct amdgpu_device *adev,
568 enum amdgpu_mes_priority_level prio);
569
570int amdgpu_mes_doorbell_process_slice(struct amdgpu_device *adev);
571
572/*
573 * MES lock can be taken in MMU notifiers.
574 *
575 * A bit more detail about why to set no-FS reclaim with MES lock:
576 *
577 * The purpose of the MMU notifier is to stop GPU access to memory so
578 * that the Linux VM subsystem can move pages around safely. This is
579 * done by preempting user mode queues for the affected process. When
580 * MES is used, MES lock needs to be taken to preempt the queues.
581 *
582 * The MMU notifier callback entry point in the driver is
583 * amdgpu_mn_invalidate_range_start_hsa. The relevant call chain from
584 * there is:
585 * amdgpu_amdkfd_evict_userptr -> kgd2kfd_quiesce_mm ->
586 * kfd_process_evict_queues -> pdd->dev->dqm->ops.evict_process_queues
587 *
588 * The last part of the chain is a function pointer where we take the
589 * MES lock.
590 *
591 * The problem with taking locks in the MMU notifier is, that MMU
592 * notifiers can be called in reclaim-FS context. That's where the
593 * kernel frees up pages to make room for new page allocations under
594 * memory pressure. While we are running in reclaim-FS context, we must
595 * not trigger another memory reclaim operation because that would
596 * recursively reenter the reclaim code and cause a deadlock. The
597 * memalloc_nofs_save/restore calls guarantee that.
598 *
599 * In addition we also need to avoid lock dependencies on other locks taken
600 * under the MES lock, for example reservation locks. Here is a possible
601 * scenario of a deadlock:
602 * Thread A: takes and holds reservation lock | triggers reclaim-FS |
603 * MMU notifier | blocks trying to take MES lock
604 * Thread B: takes and holds MES lock | blocks trying to take reservation lock
605 *
606 * In this scenario Thread B gets involved in a deadlock even without
607 * triggering a reclaim-FS operation itself.
608 * To fix this and break the lock dependency chain you'd need to either:
609 * 1. protect reservation locks with memalloc_nofs_save/restore, or
610 * 2. avoid taking reservation locks under the MES lock.
611 *
612 * Reservation locks are taken all over the kernel in different subsystems, we
613 * have no control over them and their lock dependencies.So the only workable
614 * solution is to avoid taking other locks under the MES lock.
615 * As a result, make sure no reclaim-FS happens while holding this lock anywhere
616 * to prevent deadlocks when an MMU notifier runs in reclaim-FS context.
617 */
618static inline void amdgpu_mes_lock(struct amdgpu_mes *mes)
619{
620 mutex_lock(&mes->mutex_hidden);
621 mes->saved_flags = memalloc_noreclaim_save();
622}
623
624static inline void amdgpu_mes_unlock(struct amdgpu_mes *mes)
625{
626 memalloc_noreclaim_restore(mes->saved_flags);
627 mutex_unlock(&mes->mutex_hidden);
628}
629
630bool amdgpu_mes_suspend_resume_all_supported(struct amdgpu_device *adev);
631bool amdgpu_mes_queue_reset_by_mes_supported(struct amdgpu_device *adev);
632
633int amdgpu_mes_update_enforce_isolation(struct amdgpu_device *adev);
634
635int amdgpu_mes_rs64mem_init(struct amdgpu_mes *mes);
636void amdgpu_mes_rs64mem_fini(struct amdgpu_mes *mes);
637int amdgpu_mes_rs64mem_setup_bitmaps(struct amdgpu_mes *mes);
638int amdgpu_mes_alloc_proc_ctx_index(struct amdgpu_mes *mes,
639 uint32_t *index);
640void amdgpu_mes_free_proc_ctx_index(struct amdgpu_mes *mes,
641 uint32_t index);
642int amdgpu_mes_alloc_gang_ctx_index(struct amdgpu_mes *mes,
643 uint32_t *index);
644void amdgpu_mes_free_gang_ctx_index(struct amdgpu_mes *mes,
645 uint32_t index);
646#endif /* __AMDGPU_MES_H__ */