projects/linux·blob master · refs

3098 lines · 86.3 KiB

1/*
2 * Copyright 2014 Advanced Micro Devices, Inc.
3 * Copyright 2008 Red Hat Inc.
4 * Copyright 2009 Jerome Glisse.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
22 * OTHER DEALINGS IN THE SOFTWARE.
23 *
24 */
25
26#include <linux/firmware.h>
27#include <linux/pm_runtime.h>
28
29#include "amdgpu.h"
30#include "amdgpu_gfx.h"
31#include "amdgpu_rlc.h"
32#include "amdgpu_ras.h"
33#include "amdgpu_reset.h"
34#include "amdgpu_xcp.h"
35#include "amdgpu_xgmi.h"
36#include "amdgpu_mes.h"
37#include "amdgpu_userq.h"
38#include "mes_userqueue.h"
39#include "nvd.h"
40
41/* delay 0.1 second to enable gfx off feature */
42#define GFX_OFF_DELAY_ENABLE msecs_to_jiffies(100)
43
44#define GFX_OFF_NO_DELAY 0
45
46/*
47 * GPU GFX IP block helpers function.
48 */
49
50int amdgpu_gfx_mec_queue_to_bit(struct amdgpu_device *adev, int mec,
51 int pipe, int queue)
52{
53 int bit = 0;
54
55 bit += mec * adev->gfx.mec.num_pipe_per_mec
56 * adev->gfx.mec.num_queue_per_pipe;
57 bit += pipe * adev->gfx.mec.num_queue_per_pipe;
58 bit += queue;
59
60 return bit;
61}
62
63void amdgpu_queue_mask_bit_to_mec_queue(struct amdgpu_device *adev, int bit,
64 int *mec, int *pipe, int *queue)
65{
66 *queue = bit % adev->gfx.mec.num_queue_per_pipe;
67 *pipe = (bit / adev->gfx.mec.num_queue_per_pipe)
68 % adev->gfx.mec.num_pipe_per_mec;
69 *mec = (bit / adev->gfx.mec.num_queue_per_pipe)
70 / adev->gfx.mec.num_pipe_per_mec;
71
72}
73
74bool amdgpu_gfx_is_mec_queue_enabled(struct amdgpu_device *adev,
75 int xcc_id, int mec, int pipe, int queue)
76{
77 return test_bit(amdgpu_gfx_mec_queue_to_bit(adev, mec, pipe, queue),
78 adev->gfx.mec_bitmap[xcc_id].queue_bitmap);
79}
80
81static int amdgpu_gfx_me_queue_to_bit(struct amdgpu_device *adev,
82 int me, int pipe, int queue)
83{
84 int num_queue_per_pipe = 1; /* we only enable 1 KGQ per pipe */
85 int bit = 0;
86
87 bit += me * adev->gfx.me.num_pipe_per_me
88 * num_queue_per_pipe;
89 bit += pipe * num_queue_per_pipe;
90 bit += queue;
91
92 return bit;
93}
94
95bool amdgpu_gfx_is_me_queue_enabled(struct amdgpu_device *adev,
96 int me, int pipe, int queue)
97{
98 return test_bit(amdgpu_gfx_me_queue_to_bit(adev, me, pipe, queue),
99 adev->gfx.me.queue_bitmap);
100}
101
102/**
103 * amdgpu_gfx_parse_disable_cu - Parse the disable_cu module parameter
104 *
105 * @adev: amdgpu device pointer
106 * @mask: array in which the per-shader array disable masks will be stored
107 * @max_se: number of SEs
108 * @max_sh: number of SHs
109 *
110 * The bitmask of CUs to be disabled in the shader array determined by se and
111 * sh is stored in mask[se * max_sh + sh].
112 */
113void amdgpu_gfx_parse_disable_cu(struct amdgpu_device *adev, unsigned int *mask,
114 unsigned int max_se, unsigned int max_sh)
115{
116 unsigned int se, sh, cu;
117 const char *p;
118
119 memset(mask, 0, sizeof(*mask) * max_se * max_sh);
120
121 if (!amdgpu_disable_cu || !*amdgpu_disable_cu)
122 return;
123
124 p = amdgpu_disable_cu;
125 for (;;) {
126 char *next;
127 int ret = sscanf(p, "%u.%u.%u", &se, &sh, &cu);
128
129 if (ret < 3) {
130 drm_err(adev_to_drm(adev), "could not parse disable_cu\n");
131 return;
132 }
133
134 if (se < max_se && sh < max_sh && cu < 16) {
135 drm_info(adev_to_drm(adev), "Disabling CU %u.%u.%u\n", se, sh, cu);
136 mask[se * max_sh + sh] |= 1u << cu;
137 } else {
138 drm_err(adev_to_drm(adev), "disable_cu %u.%u.%u is out of range\n",
139 se, sh, cu);
140 }
141
142 next = strchr(p, ',');
143 if (!next)
144 break;
145 p = next + 1;
146 }
147}
148
149static bool amdgpu_gfx_is_graphics_multipipe_capable(struct amdgpu_device *adev)
150{
151 return amdgpu_async_gfx_ring && adev->gfx.me.num_pipe_per_me > 1;
152}
153
154static bool amdgpu_gfx_is_compute_multipipe_capable(struct amdgpu_device *adev)
155{
156 if (amdgpu_compute_multipipe != -1) {
157 dev_info(adev->dev, " forcing compute pipe policy %d\n",
158 amdgpu_compute_multipipe);
159 return amdgpu_compute_multipipe == 1;
160 }
161
162 if (amdgpu_ip_version(adev, GC_HWIP, 0) > IP_VERSION(9, 0, 0))
163 return true;
164
165 /* FIXME: spreading the queues across pipes causes perf regressions
166 * on POLARIS11 compute workloads */
167 if (adev->asic_type == CHIP_POLARIS11)
168 return false;
169
170 return adev->gfx.mec.num_mec > 1;
171}
172
173bool amdgpu_gfx_is_high_priority_graphics_queue(struct amdgpu_device *adev,
174 struct amdgpu_ring *ring)
175{
176 int queue = ring->queue;
177 int pipe = ring->pipe;
178
179 /* Policy: use pipe1 queue0 as high priority graphics queue if we
180 * have more than one gfx pipe.
181 */
182 if (amdgpu_gfx_is_graphics_multipipe_capable(adev) &&
183 adev->gfx.num_gfx_rings > 1 && pipe == 1 && queue == 0) {
184 int me = ring->me;
185 int bit;
186
187 bit = amdgpu_gfx_me_queue_to_bit(adev, me, pipe, queue);
188 if (ring == &adev->gfx.gfx_ring[bit])
189 return true;
190 }
191
192 return false;
193}
194
195bool amdgpu_gfx_is_high_priority_compute_queue(struct amdgpu_device *adev,
196 struct amdgpu_ring *ring)
197{
198 /* Policy: use 1st queue as high priority compute queue if we
199 * have more than one compute queue.
200 */
201 if (adev->gfx.num_compute_rings > 1 &&
202 ring == &adev->gfx.compute_ring[0])
203 return true;
204
205 return false;
206}
207
208void amdgpu_gfx_compute_queue_acquire(struct amdgpu_device *adev)
209{
210 int i, j, queue, pipe;
211 bool multipipe_policy = amdgpu_gfx_is_compute_multipipe_capable(adev);
212 int max_queues_per_mec = min(adev->gfx.mec.num_pipe_per_mec *
213 adev->gfx.mec.num_queue_per_pipe,
214 adev->gfx.num_compute_rings);
215 int num_xcc = adev->gfx.xcc_mask ? NUM_XCC(adev->gfx.xcc_mask) : 1;
216
217 if (multipipe_policy) {
218 /* policy: make queues evenly cross all pipes on MEC1 only
219 * for multiple xcc, just use the original policy for simplicity */
220 for (j = 0; j < num_xcc; j++) {
221 for (i = 0; i < max_queues_per_mec; i++) {
222 pipe = i % adev->gfx.mec.num_pipe_per_mec;
223 queue = (i / adev->gfx.mec.num_pipe_per_mec) %
224 adev->gfx.mec.num_queue_per_pipe;
225
226 set_bit(pipe * adev->gfx.mec.num_queue_per_pipe + queue,
227 adev->gfx.mec_bitmap[j].queue_bitmap);
228 }
229 }
230 } else {
231 /* policy: amdgpu owns all queues in the given pipe */
232 for (j = 0; j < num_xcc; j++) {
233 for (i = 0; i < max_queues_per_mec; ++i)
234 set_bit(i, adev->gfx.mec_bitmap[j].queue_bitmap);
235 }
236 }
237
238 for (j = 0; j < num_xcc; j++) {
239 dev_dbg(adev->dev, "mec queue bitmap weight=%d\n",
240 bitmap_weight(adev->gfx.mec_bitmap[j].queue_bitmap, AMDGPU_MAX_COMPUTE_QUEUES));
241 }
242}
243
244void amdgpu_gfx_graphics_queue_acquire(struct amdgpu_device *adev)
245{
246 int i, queue, pipe;
247 bool multipipe_policy = amdgpu_gfx_is_graphics_multipipe_capable(adev);
248 int num_queue_per_pipe = 1; /* we only enable 1 KGQ per pipe */
249 int max_queues_per_me = adev->gfx.me.num_pipe_per_me * num_queue_per_pipe;
250
251 if (multipipe_policy) {
252 /* policy: amdgpu owns the first queue per pipe at this stage
253 * will extend to mulitple queues per pipe later */
254 for (i = 0; i < max_queues_per_me; i++) {
255 pipe = i % adev->gfx.me.num_pipe_per_me;
256 queue = (i / adev->gfx.me.num_pipe_per_me) %
257 num_queue_per_pipe;
258
259 set_bit(pipe * num_queue_per_pipe + queue,
260 adev->gfx.me.queue_bitmap);
261 }
262 } else {
263 for (i = 0; i < max_queues_per_me; ++i)
264 set_bit(i, adev->gfx.me.queue_bitmap);
265 }
266
267 /* update the number of active graphics rings */
268 if (adev->gfx.num_gfx_rings)
269 adev->gfx.num_gfx_rings =
270 bitmap_weight(adev->gfx.me.queue_bitmap, AMDGPU_MAX_GFX_QUEUES);
271}
272
273static int amdgpu_gfx_kiq_acquire(struct amdgpu_device *adev,
274 struct amdgpu_ring *ring, int xcc_id)
275{
276 int queue_bit;
277 int mec, pipe, queue;
278
279 queue_bit = adev->gfx.mec.num_mec
280 * adev->gfx.mec.num_pipe_per_mec
281 * adev->gfx.mec.num_queue_per_pipe;
282
283 while (--queue_bit >= 0) {
284 if (test_bit(queue_bit, adev->gfx.mec_bitmap[xcc_id].queue_bitmap))
285 continue;
286
287 amdgpu_queue_mask_bit_to_mec_queue(adev, queue_bit, &mec, &pipe, &queue);
288
289 /*
290 * 1. Using pipes 2/3 from MEC 2 seems cause problems.
291 * 2. It must use queue id 0, because CGPG_IDLE/SAVE/LOAD/RUN
292 * only can be issued on queue 0.
293 */
294 if ((mec == 1 && pipe > 1) || queue != 0)
295 continue;
296
297 ring->me = mec + 1;
298 ring->pipe = pipe;
299 ring->queue = queue;
300
301 return 0;
302 }
303
304 dev_err(adev->dev, "Failed to find a queue for KIQ\n");
305 return -EINVAL;
306}
307
308int amdgpu_gfx_kiq_init_ring(struct amdgpu_device *adev, int xcc_id)
309{
310 struct amdgpu_kiq *kiq = &adev->gfx.kiq[xcc_id];
311 struct amdgpu_irq_src *irq = &kiq->irq;
312 struct amdgpu_ring *ring = &kiq->ring;
313 int r = 0;
314
315 spin_lock_init(&kiq->ring_lock);
316
317 ring->adev = NULL;
318 ring->ring_obj = NULL;
319 ring->use_doorbell = true;
320 ring->xcc_id = xcc_id;
321 ring->vm_hub = AMDGPU_GFXHUB(xcc_id);
322 ring->doorbell_index =
323 (adev->doorbell_index.kiq +
324 xcc_id * adev->doorbell_index.xcc_doorbell_range)
325 << 1;
326
327 r = amdgpu_gfx_kiq_acquire(adev, ring, xcc_id);
328 if (r)
329 return r;
330
331 ring->eop_gpu_addr = kiq->eop_gpu_addr;
332 ring->no_scheduler = true;
333 snprintf(ring->name, sizeof(ring->name), "kiq_%hhu.%hhu.%hhu.%hhu",
334 (unsigned char)xcc_id, (unsigned char)ring->me,
335 (unsigned char)ring->pipe, (unsigned char)ring->queue);
336 r = amdgpu_ring_init(adev, ring, 1024, irq, AMDGPU_CP_KIQ_IRQ_DRIVER0,
337 AMDGPU_RING_PRIO_DEFAULT, NULL);
338 if (r)
339 dev_warn(adev->dev, "(%d) failed to init kiq ring\n", r);
340
341 return r;
342}
343
344void amdgpu_gfx_kiq_free_ring(struct amdgpu_ring *ring)
345{
346 amdgpu_ring_fini(ring);
347}
348
349void amdgpu_gfx_kiq_fini(struct amdgpu_device *adev, int xcc_id)
350{
351 struct amdgpu_kiq *kiq = &adev->gfx.kiq[xcc_id];
352
353 amdgpu_bo_free_kernel(&kiq->eop_obj, &kiq->eop_gpu_addr, NULL);
354}
355
356int amdgpu_gfx_kiq_init(struct amdgpu_device *adev,
357 unsigned int hpd_size, int xcc_id)
358{
359 int r;
360 u32 *hpd;
361 struct amdgpu_kiq *kiq = &adev->gfx.kiq[xcc_id];
362
363 r = amdgpu_bo_create_kernel(adev, hpd_size, PAGE_SIZE,
364 AMDGPU_GEM_DOMAIN_GTT, &kiq->eop_obj,
365 &kiq->eop_gpu_addr, (void **)&hpd);
366 if (r) {
367 dev_warn(adev->dev, "failed to create KIQ bo (%d).\n", r);
368 return r;
369 }
370
371 memset(hpd, 0, hpd_size);
372
373 r = amdgpu_bo_reserve(kiq->eop_obj, true);
374 if (unlikely(r != 0))
375 dev_warn(adev->dev, "(%d) reserve kiq eop bo failed\n", r);
376 amdgpu_bo_kunmap(kiq->eop_obj);
377 amdgpu_bo_unreserve(kiq->eop_obj);
378
379 return 0;
380}
381
382static void amdgpu_gfx_mqd_reset_restore(struct amdgpu_ring *ring)
383{
384 struct amdgpu_device *adev = ring->adev;
385 int mqd_idx, mqd_size;
386
387 /* restore mqd with the backup copy */
388 if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) {
389 mqd_idx = ring - &adev->gfx.compute_ring[0];
390 mqd_size = adev->mqds[AMDGPU_HW_IP_COMPUTE].mqd_size;
391 if (adev->gfx.mec.mqd_backup[mqd_idx])
392 memcpy_toio(ring->mqd_ptr, adev->gfx.mec.mqd_backup[mqd_idx], mqd_size);
393 } else if (ring->funcs->type == AMDGPU_RING_TYPE_GFX) {
394 mqd_size = adev->mqds[AMDGPU_HW_IP_GFX].mqd_size;
395 mqd_idx = ring - &adev->gfx.gfx_ring[0];
396
397 if (adev->gfx.me.mqd_backup[mqd_idx])
398 memcpy_toio(ring->mqd_ptr, adev->gfx.me.mqd_backup[mqd_idx], mqd_size);
399 }
400 /* reset the ring */
401 ring->wptr = 0;
402 atomic64_set((atomic64_t *)ring->wptr_cpu_addr, 0);
403 amdgpu_ring_clear_ring(ring);
404}
405
406/* create MQD for each compute/gfx queue */
407int amdgpu_gfx_mqd_sw_init(struct amdgpu_device *adev,
408 unsigned int mqd_size, int xcc_id)
409{
410 int r, i, j;
411 struct amdgpu_kiq *kiq = &adev->gfx.kiq[xcc_id];
412 struct amdgpu_ring *ring = &kiq->ring;
413 u32 domain = AMDGPU_GEM_DOMAIN_GTT;
414 u32 gfx_mqd_size = max(adev->mqds[AMDGPU_HW_IP_GFX].mqd_size, mqd_size);
415 u32 compute_mqd_size = max(adev->mqds[AMDGPU_HW_IP_COMPUTE].mqd_size, mqd_size);
416
417#if !defined(CONFIG_ARM) && !defined(CONFIG_ARM64)
418 /* Only enable on gfx10 and 11 for now to avoid changing behavior on older chips */
419 if (amdgpu_ip_version(adev, GC_HWIP, 0) >= IP_VERSION(10, 0, 0))
420 domain |= AMDGPU_GEM_DOMAIN_VRAM;
421#endif
422
423 /* create MQD for KIQ (on GFX8+ where we use KIQ) */
424 if (adev->asic_type >= CHIP_TOPAZ && !adev->enable_mes_kiq && !ring->mqd_obj) {
425 /* originaly the KIQ MQD is put in GTT domain, but for SRIOV VRAM domain is a must
426 * otherwise hypervisor trigger SAVE_VF fail after driver unloaded which mean MQD
427 * deallocated and gart_unbind, to strict diverage we decide to use VRAM domain for
428 * KIQ MQD no matter SRIOV or Bare-metal
429 */
430 r = amdgpu_bo_create_kernel(adev, mqd_size, PAGE_SIZE,
431 AMDGPU_GEM_DOMAIN_VRAM |
432 AMDGPU_GEM_DOMAIN_GTT,
433 &ring->mqd_obj,
434 &ring->mqd_gpu_addr,
435 &ring->mqd_ptr);
436 if (r) {
437 dev_warn(adev->dev, "failed to create ring mqd ob (%d)", r);
438 return r;
439 }
440
441 /* prepare MQD backup */
442 kiq->mqd_backup = kzalloc(mqd_size, GFP_KERNEL);
443 if (!kiq->mqd_backup) {
444 dev_warn(adev->dev,
445 "no memory to create MQD backup for ring %s\n", ring->name);
446 return -ENOMEM;
447 }
448 }
449
450 if (adev->asic_type >= CHIP_NAVI10 && amdgpu_async_gfx_ring) {
451 /* create MQD for each KGQ */
452 for (i = 0; i < adev->gfx.num_gfx_rings; i++) {
453 ring = &adev->gfx.gfx_ring[i];
454 if (!ring->mqd_obj) {
455 r = amdgpu_bo_create_kernel(adev, AMDGPU_MQD_SIZE_ALIGN(gfx_mqd_size),
456 PAGE_SIZE, domain, &ring->mqd_obj,
457 &ring->mqd_gpu_addr, &ring->mqd_ptr);
458 if (r) {
459 dev_warn(adev->dev, "failed to create ring mqd bo (%d)", r);
460 return r;
461 }
462
463 ring->mqd_size = gfx_mqd_size;
464 /* prepare MQD backup */
465 adev->gfx.me.mqd_backup[i] = kzalloc(gfx_mqd_size, GFP_KERNEL);
466 if (!adev->gfx.me.mqd_backup[i]) {
467 dev_warn(adev->dev, "no memory to create MQD backup for ring %s\n", ring->name);
468 return -ENOMEM;
469 }
470 }
471 }
472 }
473
474 /* create MQD for each KCQ */
475 for (i = 0; i < adev->gfx.num_compute_rings; i++) {
476 j = i + xcc_id * adev->gfx.num_compute_rings;
477 ring = &adev->gfx.compute_ring[j];
478 if (!ring->mqd_obj) {
479 r = amdgpu_bo_create_kernel(adev, AMDGPU_MQD_SIZE_ALIGN(compute_mqd_size),
480 PAGE_SIZE, domain, &ring->mqd_obj,
481 &ring->mqd_gpu_addr, &ring->mqd_ptr);
482 if (r) {
483 dev_warn(adev->dev, "failed to create ring mqd bo (%d)", r);
484 return r;
485 }
486
487 ring->mqd_size = compute_mqd_size;
488 /* prepare MQD backup */
489 adev->gfx.mec.mqd_backup[j] = kzalloc(compute_mqd_size, GFP_KERNEL);
490 if (!adev->gfx.mec.mqd_backup[j]) {
491 dev_warn(adev->dev, "no memory to create MQD backup for ring %s\n", ring->name);
492 return -ENOMEM;
493 }
494 }
495 }
496
497 return 0;
498}
499
500void amdgpu_gfx_mqd_sw_fini(struct amdgpu_device *adev, int xcc_id)
501{
502 struct amdgpu_ring *ring = NULL;
503 int i, j;
504 struct amdgpu_kiq *kiq = &adev->gfx.kiq[xcc_id];
505
506 if (adev->asic_type >= CHIP_NAVI10 && amdgpu_async_gfx_ring) {
507 for (i = 0; i < adev->gfx.num_gfx_rings; i++) {
508 ring = &adev->gfx.gfx_ring[i];
509 kfree(adev->gfx.me.mqd_backup[i]);
510 amdgpu_bo_free_kernel(&ring->mqd_obj,
511 &ring->mqd_gpu_addr,
512 &ring->mqd_ptr);
513 }
514 }
515
516 for (i = 0; i < adev->gfx.num_compute_rings; i++) {
517 j = i + xcc_id * adev->gfx.num_compute_rings;
518 ring = &adev->gfx.compute_ring[j];
519 kfree(adev->gfx.mec.mqd_backup[j]);
520 amdgpu_bo_free_kernel(&ring->mqd_obj,
521 &ring->mqd_gpu_addr,
522 &ring->mqd_ptr);
523 }
524
525 ring = &kiq->ring;
526 kfree(kiq->mqd_backup);
527 amdgpu_bo_free_kernel(&ring->mqd_obj,
528 &ring->mqd_gpu_addr,
529 &ring->mqd_ptr);
530}
531
532void amdgpu_gfx_mqd_symmetrically_map_cu_mask(struct amdgpu_device *adev, const uint32_t *cu_mask,
533 uint32_t cu_mask_count, uint32_t *se_mask)
534{
535 struct amdgpu_cu_info *cu_info = &adev->gfx.cu_info;
536 struct amdgpu_gfx_config *gfx_info = &adev->gfx.config;
537 uint32_t cu_per_sh[8][4] = {0};
538 int i, se, sh, cu, cu_bitmap_sh_mul;
539 int xcc_inst = ffs(adev->gfx.xcc_mask) - 1;
540 bool wgp_mode_req = amdgpu_ip_version(adev, GC_HWIP, 0) >= IP_VERSION(10, 0, 0);
541 int cu_inc = wgp_mode_req ? 2 : 1;
542 uint32_t en_mask = wgp_mode_req ? 0x3 : 0x1;
543 int num_xcc, inc, inst = 0;
544
545 if (xcc_inst < 0)
546 xcc_inst = 0;
547
548 num_xcc = hweight16(adev->gfx.xcc_mask);
549 if (!num_xcc)
550 num_xcc = 1;
551
552 inc = cu_inc * num_xcc;
553
554 cu_bitmap_sh_mul = 2;
555
556 for (se = 0; se < gfx_info->max_shader_engines; se++)
557 for (sh = 0; sh < gfx_info->max_sh_per_se; sh++)
558 cu_per_sh[se][sh] = hweight32(
559 cu_info->bitmap[xcc_inst][se % 4][sh + (se / 4) *
560 cu_bitmap_sh_mul]);
561
562 for (i = 0; i < gfx_info->max_shader_engines; i++)
563 se_mask[i] = 0;
564
565 i = inst;
566 for (cu = 0; cu < 16; cu += cu_inc) {
567 for (sh = 0; sh < gfx_info->max_sh_per_se; sh++) {
568 for (se = 0; se < gfx_info->max_shader_engines; se++) {
569 if (cu_per_sh[se][sh] > cu) {
570 if ((i / 32) < cu_mask_count && (cu_mask[i / 32] & (1 << (i % 32))))
571 se_mask[se] |= en_mask << (cu + sh * 16);
572 i += inc;
573 if (i >= cu_mask_count * 32)
574 return;
575 }
576 }
577 }
578 }
579}
580
581int amdgpu_gfx_disable_kcq(struct amdgpu_device *adev, int xcc_id)
582{
583 struct amdgpu_kiq *kiq = &adev->gfx.kiq[xcc_id];
584 struct amdgpu_ring *kiq_ring = &kiq->ring;
585 int i, r = 0;
586 int j;
587
588 if (adev->enable_mes) {
589 for (i = 0; i < adev->gfx.num_compute_rings; i++) {
590 j = i + xcc_id * adev->gfx.num_compute_rings;
591 amdgpu_mes_unmap_legacy_queue(adev,
592 &adev->gfx.compute_ring[j],
593 RESET_QUEUES, 0, 0, xcc_id);
594 }
595 return 0;
596 }
597
598 if (!kiq->pmf || !kiq->pmf->kiq_unmap_queues)
599 return -EINVAL;
600
601 if (!kiq_ring->sched.ready || amdgpu_in_reset(adev))
602 return 0;
603
604 spin_lock(&kiq->ring_lock);
605 if (amdgpu_ring_alloc(kiq_ring, kiq->pmf->unmap_queues_size *
606 adev->gfx.num_compute_rings)) {
607 spin_unlock(&kiq->ring_lock);
608 return -ENOMEM;
609 }
610
611 for (i = 0; i < adev->gfx.num_compute_rings; i++) {
612 j = i + xcc_id * adev->gfx.num_compute_rings;
613 kiq->pmf->kiq_unmap_queues(kiq_ring,
614 &adev->gfx.compute_ring[j],
615 RESET_QUEUES, 0, 0);
616 }
617 /* Submit unmap queue packet */
618 amdgpu_ring_commit(kiq_ring);
619 /*
620 * Ring test will do a basic scratch register change check. Just run
621 * this to ensure that unmap queues that is submitted before got
622 * processed successfully before returning.
623 */
624 r = amdgpu_ring_test_helper(kiq_ring);
625
626 spin_unlock(&kiq->ring_lock);
627
628 return r;
629}
630
631int amdgpu_gfx_disable_kgq(struct amdgpu_device *adev, int xcc_id)
632{
633 struct amdgpu_kiq *kiq = &adev->gfx.kiq[xcc_id];
634 struct amdgpu_ring *kiq_ring = &kiq->ring;
635 int i, r = 0;
636 int j;
637
638 if (adev->enable_mes) {
639 if (amdgpu_gfx_is_master_xcc(adev, xcc_id)) {
640 for (i = 0; i < adev->gfx.num_gfx_rings; i++) {
641 j = i + xcc_id * adev->gfx.num_gfx_rings;
642 amdgpu_mes_unmap_legacy_queue(adev,
643 &adev->gfx.gfx_ring[j],
644 PREEMPT_QUEUES, 0, 0, xcc_id);
645 }
646 }
647 return 0;
648 }
649
650 if (!kiq->pmf || !kiq->pmf->kiq_unmap_queues)
651 return -EINVAL;
652
653 if (!adev->gfx.kiq[0].ring.sched.ready || amdgpu_in_reset(adev))
654 return 0;
655
656 if (amdgpu_gfx_is_master_xcc(adev, xcc_id)) {
657 spin_lock(&kiq->ring_lock);
658 if (amdgpu_ring_alloc(kiq_ring, kiq->pmf->unmap_queues_size *
659 adev->gfx.num_gfx_rings)) {
660 spin_unlock(&kiq->ring_lock);
661 return -ENOMEM;
662 }
663
664 for (i = 0; i < adev->gfx.num_gfx_rings; i++) {
665 j = i + xcc_id * adev->gfx.num_gfx_rings;
666 kiq->pmf->kiq_unmap_queues(kiq_ring,
667 &adev->gfx.gfx_ring[j],
668 PREEMPT_QUEUES, 0, 0);
669 }
670 /* Submit unmap queue packet */
671 amdgpu_ring_commit(kiq_ring);
672
673 /*
674 * Ring test will do a basic scratch register change check.
675 * Just run this to ensure that unmap queues that is submitted
676 * before got processed successfully before returning.
677 */
678 r = amdgpu_ring_test_helper(kiq_ring);
679 spin_unlock(&kiq->ring_lock);
680 }
681
682 return r;
683}
684
685int amdgpu_queue_mask_bit_to_set_resource_bit(struct amdgpu_device *adev,
686 int queue_bit)
687{
688 int mec, pipe, queue;
689 int set_resource_bit = 0;
690
691 amdgpu_queue_mask_bit_to_mec_queue(adev, queue_bit, &mec, &pipe, &queue);
692
693 set_resource_bit = mec * 4 * 8 + pipe * 8 + queue;
694
695 return set_resource_bit;
696}
697
698static int amdgpu_gfx_mes_enable_kcq(struct amdgpu_device *adev, int xcc_id)
699{
700 struct amdgpu_kiq *kiq = &adev->gfx.kiq[xcc_id];
701 struct amdgpu_ring *kiq_ring = &kiq->ring;
702 uint64_t queue_mask = ~0ULL;
703 int r, i, j;
704
705 amdgpu_device_flush_hdp(adev, NULL);
706
707 if (!adev->enable_uni_mes) {
708 spin_lock(&kiq->ring_lock);
709 r = amdgpu_ring_alloc(kiq_ring, kiq->pmf->set_resources_size);
710 if (r) {
711 dev_err(adev->dev, "Failed to lock KIQ (%d).\n", r);
712 spin_unlock(&kiq->ring_lock);
713 return r;
714 }
715
716 kiq->pmf->kiq_set_resources(kiq_ring, queue_mask);
717 r = amdgpu_ring_test_helper(kiq_ring);
718 spin_unlock(&kiq->ring_lock);
719 if (r)
720 dev_err(adev->dev, "KIQ failed to set resources\n");
721 }
722
723 for (i = 0; i < adev->gfx.num_compute_rings; i++) {
724 j = i + xcc_id * adev->gfx.num_compute_rings;
725 r = amdgpu_mes_map_legacy_queue(adev,
726 &adev->gfx.compute_ring[j],
727 xcc_id);
728 if (r) {
729 dev_err(adev->dev, "failed to map compute queue\n");
730 return r;
731 }
732 }
733
734 return 0;
735}
736
737int amdgpu_gfx_enable_kcq(struct amdgpu_device *adev, int xcc_id)
738{
739 struct amdgpu_kiq *kiq = &adev->gfx.kiq[xcc_id];
740 struct amdgpu_ring *kiq_ring = &kiq->ring;
741 uint64_t queue_mask = 0;
742 int r, i, j;
743
744 if (adev->mes.enable_legacy_queue_map)
745 return amdgpu_gfx_mes_enable_kcq(adev, xcc_id);
746
747 if (!kiq->pmf || !kiq->pmf->kiq_map_queues || !kiq->pmf->kiq_set_resources)
748 return -EINVAL;
749
750 for (i = 0; i < AMDGPU_MAX_COMPUTE_QUEUES; ++i) {
751 if (!test_bit(i, adev->gfx.mec_bitmap[xcc_id].queue_bitmap))
752 continue;
753
754 /* This situation may be hit in the future if a new HW
755 * generation exposes more than 64 queues. If so, the
756 * definition of queue_mask needs updating */
757 if (WARN_ON(i > (sizeof(queue_mask)*8))) {
758 dev_err(adev->dev, "Invalid KCQ enabled: %d\n", i);
759 break;
760 }
761
762 queue_mask |= (1ull << amdgpu_queue_mask_bit_to_set_resource_bit(adev, i));
763 }
764
765 amdgpu_device_flush_hdp(adev, NULL);
766
767 dev_info(adev->dev, "kiq ring mec %d pipe %d q %d\n", kiq_ring->me,
768 kiq_ring->pipe, kiq_ring->queue);
769
770 spin_lock(&kiq->ring_lock);
771 r = amdgpu_ring_alloc(kiq_ring, kiq->pmf->map_queues_size *
772 adev->gfx.num_compute_rings +
773 kiq->pmf->set_resources_size);
774 if (r) {
775 dev_err(adev->dev, "Failed to lock KIQ (%d).\n", r);
776 spin_unlock(&kiq->ring_lock);
777 return r;
778 }
779
780 kiq->pmf->kiq_set_resources(kiq_ring, queue_mask);
781 for (i = 0; i < adev->gfx.num_compute_rings; i++) {
782 j = i + xcc_id * adev->gfx.num_compute_rings;
783 kiq->pmf->kiq_map_queues(kiq_ring,
784 &adev->gfx.compute_ring[j]);
785 }
786 /* Submit map queue packet */
787 amdgpu_ring_commit(kiq_ring);
788 /*
789 * Ring test will do a basic scratch register change check. Just run
790 * this to ensure that map queues that is submitted before got
791 * processed successfully before returning.
792 */
793 r = amdgpu_ring_test_helper(kiq_ring);
794 spin_unlock(&kiq->ring_lock);
795 if (r)
796 dev_err(adev->dev, "KCQ enable failed\n");
797
798 return r;
799}
800
801int amdgpu_gfx_enable_kgq(struct amdgpu_device *adev, int xcc_id)
802{
803 struct amdgpu_kiq *kiq = &adev->gfx.kiq[xcc_id];
804 struct amdgpu_ring *kiq_ring = &kiq->ring;
805 int r, i, j;
806
807 if (!kiq->pmf || !kiq->pmf->kiq_map_queues)
808 return -EINVAL;
809
810 amdgpu_device_flush_hdp(adev, NULL);
811
812 if (adev->mes.enable_legacy_queue_map) {
813 for (i = 0; i < adev->gfx.num_gfx_rings; i++) {
814 j = i + xcc_id * adev->gfx.num_gfx_rings;
815 r = amdgpu_mes_map_legacy_queue(adev,
816 &adev->gfx.gfx_ring[j],
817 xcc_id);
818 if (r) {
819 dev_err(adev->dev, "failed to map gfx queue\n");
820 return r;
821 }
822 }
823
824 return 0;
825 }
826
827 spin_lock(&kiq->ring_lock);
828 /* No need to map kcq on the slave */
829 if (amdgpu_gfx_is_master_xcc(adev, xcc_id)) {
830 r = amdgpu_ring_alloc(kiq_ring, kiq->pmf->map_queues_size *
831 adev->gfx.num_gfx_rings);
832 if (r) {
833 dev_err(adev->dev, "Failed to lock KIQ (%d).\n", r);
834 spin_unlock(&kiq->ring_lock);
835 return r;
836 }
837
838 for (i = 0; i < adev->gfx.num_gfx_rings; i++) {
839 j = i + xcc_id * adev->gfx.num_gfx_rings;
840 kiq->pmf->kiq_map_queues(kiq_ring,
841 &adev->gfx.gfx_ring[j]);
842 }
843 }
844 /* Submit map queue packet */
845 amdgpu_ring_commit(kiq_ring);
846 /*
847 * Ring test will do a basic scratch register change check. Just run
848 * this to ensure that map queues that is submitted before got
849 * processed successfully before returning.
850 */
851 r = amdgpu_ring_test_helper(kiq_ring);
852 spin_unlock(&kiq->ring_lock);
853 if (r)
854 dev_err(adev->dev, "KGQ enable failed\n");
855
856 return r;
857}
858
859/**
860 * amdgpu_gfx_handle_priv_fault - Handle privileged instruction fault
861 *
862 * @adev: amdgpu_device pointer
863 * @entry: interrupt vector entry containing fault information
864 * @me_id: micro-engine ID of the faulty ring
865 * @pipe_id: pipe ID of the faulty ring
866 * @queue_id: queue ID of the faulty ring
867 *
868 * This function handles privileged instruction faults by identifying
869 * the faulty ring (gfx or compute) and triggering a scheduler fault, or by
870 * recovering the faulting user queue.
871 */
872void amdgpu_gfx_handle_priv_fault(struct amdgpu_device *adev,
873 struct amdgpu_iv_entry *entry,
874 u8 me_id, u8 pipe_id, u8 queue_id)
875{
876 struct amdgpu_ring *ring;
877 u32 doorbell_offset;
878 int i;
879
880 /*
881 * Try KQ first by ring_id (HW slot is authoritative). The
882 * KMD compute_hqd_mask contract guarantees KCQ and user queues
883 * never share a HW slot.
884 */
885 if (!adev->gfx.disable_kq) {
886 for (i = 0; i < adev->gfx.num_gfx_rings; i++) {
887 ring = &adev->gfx.gfx_ring[i];
888 if (ring->me == me_id && ring->pipe == pipe_id &&
889 ring->queue == queue_id) {
890 drm_sched_fault(&ring->sched);
891 return;
892 }
893 }
894
895 for (i = 0; i < adev->gfx.num_compute_rings; i++) {
896 ring = &adev->gfx.compute_ring[i];
897 if (ring->me == me_id && ring->pipe == pipe_id &&
898 ring->queue == queue_id) {
899 drm_sched_fault(&ring->sched);
900 return;
901 }
902 }
903 }
904
905 /* No KQ matched: the faulting slot belongs to a user queue. */
906 if (adev->gfx.disable_uq)
907 return;
908
909 doorbell_offset = entry->src_data[0] & AMDGPU_CTXID0_DOORBELL_ID_MASK;
910
911 /*
912 * A compute user-queue fault IV carries the doorbell offset, so reset
913 * the queue directly from it. A gfx user-queue fault is raised by the
914 * ME and carries only the HW slot (no doorbell); record the slot and
915 * let the worker read the doorbell back from the HQD.
916 */
917 if (doorbell_offset) {
918 amdgpu_userq_process_reset_irq(adev, entry->pasid,
919 doorbell_offset);
920 } else {
921 set_bit(pipe_id | (queue_id << 2), &adev->gfx.userq_priv_fault_slots);
922 schedule_work(&adev->gfx.userq_priv_fault_work);
923 }
924}
925
926static void amdgpu_gfx_do_off_ctrl(struct amdgpu_device *adev, bool enable,
927 bool no_delay)
928{
929 unsigned long delay = GFX_OFF_DELAY_ENABLE;
930
931 if (!(adev->pm.pp_feature & PP_GFXOFF_MASK))
932 return;
933
934 mutex_lock(&adev->gfx.gfx_off_mutex);
935
936 if (enable) {
937 /* If the count is already 0, it means there's an imbalance bug somewhere.
938 * Note that the bug may be in a different caller than the one which triggers the
939 * WARN_ON_ONCE.
940 */
941 if (WARN_ON_ONCE(adev->gfx.gfx_off_req_count == 0))
942 goto unlock;
943
944 adev->gfx.gfx_off_req_count--;
945
946 if (adev->gfx.gfx_off_req_count == 0 &&
947 !adev->gfx.gfx_off_state) {
948 /* If going to s2idle, no need to wait */
949 if (no_delay) {
950 if (!amdgpu_dpm_set_powergating_by_smu(adev,
951 AMD_IP_BLOCK_TYPE_GFX, true, 0))
952 adev->gfx.gfx_off_state = true;
953 } else {
954 schedule_delayed_work(&adev->gfx.gfx_off_delay_work,
955 delay);
956 }
957 }
958 } else {
959 if (adev->gfx.gfx_off_req_count == 0) {
960 cancel_delayed_work_sync(&adev->gfx.gfx_off_delay_work);
961
962 if (adev->gfx.gfx_off_state &&
963 !amdgpu_dpm_set_powergating_by_smu(adev, AMD_IP_BLOCK_TYPE_GFX, false, 0)) {
964 adev->gfx.gfx_off_state = false;
965
966 if (adev->gfx.funcs->init_spm_golden) {
967 dev_dbg(adev->dev,
968 "GFXOFF is disabled, re-init SPM golden settings\n");
969 amdgpu_gfx_init_spm_golden(adev);
970 }
971 }
972 }
973
974 adev->gfx.gfx_off_req_count++;
975 }
976
977unlock:
978 mutex_unlock(&adev->gfx.gfx_off_mutex);
979}
980
981/* amdgpu_gfx_off_ctrl - Handle gfx off feature enable/disable
982 *
983 * @adev: amdgpu_device pointer
984 * @bool enable true: enable gfx off feature, false: disable gfx off feature
985 *
986 * 1. gfx off feature will be enabled by gfx ip after gfx cg pg enabled.
987 * 2. other client can send request to disable gfx off feature, the request should be honored.
988 * 3. other client can cancel their request of disable gfx off feature
989 * 4. other client should not send request to enable gfx off feature before disable gfx off feature.
990 *
991 * gfx off allow will be delayed by GFX_OFF_DELAY_ENABLE ms.
992 */
993void amdgpu_gfx_off_ctrl(struct amdgpu_device *adev, bool enable)
994{
995 /* If going to s2idle, no need to wait */
996 bool no_delay = adev->in_s0ix ? true : false;
997
998 amdgpu_gfx_do_off_ctrl(adev, enable, no_delay);
999}
1000
1001/* amdgpu_gfx_off_ctrl_immediate - Handle gfx off feature enable/disable
1002 *
1003 * @adev: amdgpu_device pointer
1004 * @bool enable true: enable gfx off feature, false: disable gfx off feature
1005 *
1006 * 1. gfx off feature will be enabled by gfx ip after gfx cg pg enabled.
1007 * 2. other client can send request to disable gfx off feature, the request should be honored.
1008 * 3. other client can cancel their request of disable gfx off feature
1009 * 4. other client should not send request to enable gfx off feature before disable gfx off feature.
1010 *
1011 * gfx off allow will be issued immediately.
1012 */
1013void amdgpu_gfx_off_ctrl_immediate(struct amdgpu_device *adev, bool enable)
1014{
1015 amdgpu_gfx_do_off_ctrl(adev, enable, true);
1016}
1017
1018int amdgpu_set_gfx_off_residency(struct amdgpu_device *adev, bool value)
1019{
1020 int r = 0;
1021
1022 mutex_lock(&adev->gfx.gfx_off_mutex);
1023
1024 r = amdgpu_dpm_set_residency_gfxoff(adev, value);
1025
1026 mutex_unlock(&adev->gfx.gfx_off_mutex);
1027
1028 return r;
1029}
1030
1031int amdgpu_get_gfx_off_residency(struct amdgpu_device *adev, u32 *value)
1032{
1033 int r = 0;
1034
1035 mutex_lock(&adev->gfx.gfx_off_mutex);
1036
1037 r = amdgpu_dpm_get_residency_gfxoff(adev, value);
1038
1039 mutex_unlock(&adev->gfx.gfx_off_mutex);
1040
1041 return r;
1042}
1043
1044int amdgpu_get_gfx_off_entrycount(struct amdgpu_device *adev, u64 *value)
1045{
1046 int r = 0;
1047
1048 mutex_lock(&adev->gfx.gfx_off_mutex);
1049
1050 r = amdgpu_dpm_get_entrycount_gfxoff(adev, value);
1051
1052 mutex_unlock(&adev->gfx.gfx_off_mutex);
1053
1054 return r;
1055}
1056
1057int amdgpu_get_gfx_off_status(struct amdgpu_device *adev, uint32_t *value)
1058{
1059
1060 int r = 0;
1061
1062 mutex_lock(&adev->gfx.gfx_off_mutex);
1063
1064 r = amdgpu_dpm_get_status_gfxoff(adev, value);
1065
1066 mutex_unlock(&adev->gfx.gfx_off_mutex);
1067
1068 return r;
1069}
1070
1071int amdgpu_gfx_ras_late_init(struct amdgpu_device *adev, struct ras_common_if *ras_block)
1072{
1073 int r;
1074
1075 if (amdgpu_ras_is_supported(adev, ras_block->block)) {
1076 if (!amdgpu_persistent_edc_harvesting_supported(adev)) {
1077 r = amdgpu_ras_reset_error_status(adev, AMDGPU_RAS_BLOCK__GFX);
1078 if (r)
1079 return r;
1080 }
1081
1082 r = amdgpu_ras_block_late_init(adev, ras_block);
1083 if (r)
1084 return r;
1085
1086 if (!amdgpu_sriov_vf(adev) && adev->gfx.cp_ecc_error_irq.funcs) {
1087 r = amdgpu_irq_get(adev, &adev->gfx.cp_ecc_error_irq, 0);
1088 if (r)
1089 goto late_fini;
1090 }
1091 } else {
1092 amdgpu_ras_feature_enable_on_boot(adev, ras_block, 0);
1093 }
1094
1095 return 0;
1096late_fini:
1097 amdgpu_ras_block_late_fini(adev, ras_block);
1098 return r;
1099}
1100
1101void amdgpu_gfx_ras_suspend(struct amdgpu_device *adev,
1102 struct ras_common_if *ras_block)
1103{
1104 if (!amdgpu_sriov_vf(adev) && adev->gfx.cp_ecc_error_irq.funcs)
1105 amdgpu_irq_put(adev, &adev->gfx.cp_ecc_error_irq, 0);
1106}
1107
1108void amdgpu_gfx_ras_fini(struct amdgpu_device *adev,
1109 struct ras_common_if *ras_block)
1110{
1111 if (!amdgpu_sriov_vf(adev) && adev->gfx.cp_ecc_error_irq.funcs)
1112 amdgpu_irq_put(adev, &adev->gfx.cp_ecc_error_irq, 0);
1113 amdgpu_ras_block_late_fini(adev, ras_block);
1114}
1115
1116int amdgpu_gfx_ras_sw_init(struct amdgpu_device *adev)
1117{
1118 int err = 0;
1119 struct amdgpu_gfx_ras *ras = NULL;
1120
1121 /* adev->gfx.ras is NULL, which means gfx does not
1122 * support ras function, then do nothing here.
1123 */
1124 if (!adev->gfx.ras)
1125 return 0;
1126
1127 ras = adev->gfx.ras;
1128
1129 err = amdgpu_ras_register_ras_block(adev, &ras->ras_block);
1130 if (err) {
1131 dev_err(adev->dev, "Failed to register gfx ras block!\n");
1132 return err;
1133 }
1134
1135 strcpy(ras->ras_block.ras_comm.name, "gfx");
1136 ras->ras_block.ras_comm.block = AMDGPU_RAS_BLOCK__GFX;
1137 ras->ras_block.ras_comm.type = AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE;
1138 adev->gfx.ras_if = &ras->ras_block.ras_comm;
1139
1140 /* If not define special ras_late_init function, use gfx default ras_late_init */
1141 if (!ras->ras_block.ras_late_init)
1142 ras->ras_block.ras_late_init = amdgpu_gfx_ras_late_init;
1143
1144 if (!ras->ras_block.ras_suspend)
1145 ras->ras_block.ras_suspend = amdgpu_gfx_ras_suspend;
1146
1147 if (!ras->ras_block.ras_fini)
1148 ras->ras_block.ras_fini = amdgpu_gfx_ras_fini;
1149
1150 /* If not defined special ras_cb function, use default ras_cb */
1151 if (!ras->ras_block.ras_cb)
1152 ras->ras_block.ras_cb = amdgpu_gfx_process_ras_data_cb;
1153
1154 return 0;
1155}
1156
1157int amdgpu_gfx_poison_consumption_handler(struct amdgpu_device *adev,
1158 struct amdgpu_iv_entry *entry)
1159{
1160 if (adev->gfx.ras && adev->gfx.ras->poison_consumption_handler)
1161 return adev->gfx.ras->poison_consumption_handler(adev, entry);
1162
1163 return 0;
1164}
1165
1166int amdgpu_gfx_process_ras_data_cb(struct amdgpu_device *adev,
1167 void *err_data,
1168 struct amdgpu_iv_entry *entry)
1169{
1170 /* TODO ue will trigger an interrupt.
1171 *
1172 * When “Full RAS” is enabled, the per-IP interrupt sources should
1173 * be disabled and the driver should only look for the aggregated
1174 * interrupt via sync flood
1175 */
1176 if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX)) {
1177 kgd2kfd_set_sram_ecc_flag(adev->kfd.dev);
1178 if (adev->gfx.ras && adev->gfx.ras->ras_block.hw_ops &&
1179 adev->gfx.ras->ras_block.hw_ops->query_ras_error_count)
1180 adev->gfx.ras->ras_block.hw_ops->query_ras_error_count(adev, err_data);
1181 amdgpu_ras_reset_gpu(adev);
1182 }
1183 return AMDGPU_RAS_SUCCESS;
1184}
1185
1186int amdgpu_gfx_cp_ecc_error_irq(struct amdgpu_device *adev,
1187 struct amdgpu_irq_src *source,
1188 struct amdgpu_iv_entry *entry)
1189{
1190 struct ras_common_if *ras_if = adev->gfx.ras_if;
1191 struct ras_dispatch_if ih_data = {
1192 .entry = entry,
1193 };
1194
1195 if (!ras_if)
1196 return 0;
1197
1198 ih_data.head = *ras_if;
1199
1200 dev_err(adev->dev, "CP ECC ERROR IRQ\n");
1201 amdgpu_ras_interrupt_dispatch(adev, &ih_data);
1202 return 0;
1203}
1204
1205void amdgpu_gfx_ras_error_func(struct amdgpu_device *adev,
1206 void *ras_error_status,
1207 void (*func)(struct amdgpu_device *adev, void *ras_error_status,
1208 int xcc_id))
1209{
1210 int i;
1211 int num_xcc = adev->gfx.xcc_mask ? NUM_XCC(adev->gfx.xcc_mask) : 1;
1212 uint32_t xcc_mask = GENMASK(num_xcc - 1, 0);
1213 struct ras_err_data *err_data = (struct ras_err_data *)ras_error_status;
1214
1215 if (err_data) {
1216 err_data->ue_count = 0;
1217 err_data->ce_count = 0;
1218 }
1219
1220 for_each_inst(i, xcc_mask)
1221 func(adev, ras_error_status, i);
1222}
1223
1224uint32_t amdgpu_kiq_rreg(struct amdgpu_device *adev, uint32_t reg, uint32_t xcc_id)
1225{
1226 signed long r, cnt = 0;
1227 unsigned long flags;
1228 uint32_t seq, reg_val_offs = 0, value = 0;
1229 struct amdgpu_kiq *kiq = &adev->gfx.kiq[xcc_id];
1230 struct amdgpu_ring *ring = &kiq->ring;
1231
1232 if (amdgpu_device_skip_hw_access(adev))
1233 return 0;
1234
1235 if (adev->mes.ring[0].sched.ready)
1236 return amdgpu_mes_rreg(adev, reg, xcc_id);
1237
1238 BUG_ON(!ring->funcs->emit_rreg);
1239
1240 spin_lock_irqsave(&kiq->ring_lock, flags);
1241 if (amdgpu_wb_get(adev, &reg_val_offs)) {
1242 pr_err("critical bug! too many kiq readers\n");
1243 goto failed_unlock;
1244 }
1245 r = amdgpu_ring_alloc(ring, 32);
1246 if (r)
1247 goto failed_unlock;
1248
1249 amdgpu_ring_emit_rreg(ring, reg, reg_val_offs);
1250 r = amdgpu_fence_emit_polling(ring, &seq, MAX_KIQ_REG_WAIT);
1251 if (r)
1252 goto failed_undo;
1253
1254 amdgpu_ring_commit(ring);
1255 spin_unlock_irqrestore(&kiq->ring_lock, flags);
1256
1257 r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT);
1258
1259 /* don't wait anymore for gpu reset case because this way may
1260 * block gpu_recover() routine forever, e.g. this virt_kiq_rreg
1261 * is triggered in TTM and ttm_bo_lock_delayed_workqueue() will
1262 * never return if we keep waiting in virt_kiq_rreg, which cause
1263 * gpu_recover() hang there.
1264 *
1265 * also don't wait anymore for IRQ context
1266 * */
1267 if (r < 1 && (amdgpu_in_reset(adev) || in_interrupt()))
1268 goto failed_kiq_read;
1269
1270 might_sleep();
1271 while (r < 1 && cnt++ < MAX_KIQ_REG_TRY) {
1272 if (amdgpu_in_reset(adev))
1273 goto failed_kiq_read;
1274
1275 msleep(MAX_KIQ_REG_BAILOUT_INTERVAL);
1276 r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT);
1277 }
1278
1279 if (cnt > MAX_KIQ_REG_TRY)
1280 goto failed_kiq_read;
1281
1282 mb();
1283 value = adev->wb.wb[reg_val_offs];
1284 amdgpu_wb_free(adev, reg_val_offs);
1285 return value;
1286
1287failed_undo:
1288 amdgpu_ring_undo(ring);
1289failed_unlock:
1290 spin_unlock_irqrestore(&kiq->ring_lock, flags);
1291failed_kiq_read:
1292 if (reg_val_offs)
1293 amdgpu_wb_free(adev, reg_val_offs);
1294 dev_err(adev->dev, "failed to read reg:%x\n", reg);
1295 return ~0;
1296}
1297
1298void amdgpu_kiq_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v, uint32_t xcc_id)
1299{
1300 signed long r, cnt = 0;
1301 unsigned long flags;
1302 uint32_t seq;
1303 struct amdgpu_kiq *kiq = &adev->gfx.kiq[xcc_id];
1304 struct amdgpu_ring *ring = &kiq->ring;
1305
1306 BUG_ON(!ring->funcs->emit_wreg);
1307
1308 if (amdgpu_device_skip_hw_access(adev))
1309 return;
1310
1311 if (adev->mes.ring[0].sched.ready) {
1312 amdgpu_mes_wreg(adev, reg, v, xcc_id);
1313 return;
1314 }
1315
1316 spin_lock_irqsave(&kiq->ring_lock, flags);
1317 r = amdgpu_ring_alloc(ring, 32);
1318 if (r)
1319 goto failed_unlock;
1320
1321 amdgpu_ring_emit_wreg(ring, reg, v);
1322 r = amdgpu_fence_emit_polling(ring, &seq, MAX_KIQ_REG_WAIT);
1323 if (r)
1324 goto failed_undo;
1325
1326 amdgpu_ring_commit(ring);
1327 spin_unlock_irqrestore(&kiq->ring_lock, flags);
1328
1329 r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT);
1330
1331 /* don't wait anymore for gpu reset case because this way may
1332 * block gpu_recover() routine forever, e.g. this virt_kiq_rreg
1333 * is triggered in TTM and ttm_bo_lock_delayed_workqueue() will
1334 * never return if we keep waiting in virt_kiq_rreg, which cause
1335 * gpu_recover() hang there.
1336 *
1337 * also don't wait anymore for IRQ context
1338 * */
1339 if (r < 1 && (amdgpu_in_reset(adev) || in_interrupt()))
1340 goto failed_kiq_write;
1341
1342 might_sleep();
1343 while (r < 1 && cnt++ < MAX_KIQ_REG_TRY) {
1344 if (amdgpu_in_reset(adev))
1345 goto failed_kiq_write;
1346
1347 msleep(MAX_KIQ_REG_BAILOUT_INTERVAL);
1348 r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT);
1349 }
1350
1351 if (cnt > MAX_KIQ_REG_TRY)
1352 goto failed_kiq_write;
1353
1354 return;
1355
1356failed_undo:
1357 amdgpu_ring_undo(ring);
1358failed_unlock:
1359 spin_unlock_irqrestore(&kiq->ring_lock, flags);
1360failed_kiq_write:
1361 dev_err(adev->dev, "failed to write reg:%x\n", reg);
1362}
1363
1364void amdgpu_gfx_get_hdp_flush_mask(struct amdgpu_ring *ring,
1365 uint32_t *hdp_flush_mask, uint32_t *reg_mem_engine)
1366{
1367
1368 if (!ring || !hdp_flush_mask || !reg_mem_engine) {
1369 DRM_INFO("%s:invalid params\n", __func__);
1370 return;
1371 }
1372
1373 const struct nbio_hdp_flush_reg *nbio_hf_reg = ring->adev->nbio.hdp_flush_reg;
1374
1375 switch (ring->funcs->type) {
1376 case AMDGPU_RING_TYPE_GFX:
1377 *hdp_flush_mask = nbio_hf_reg->ref_and_mask_cp0 << ring->pipe;
1378 *reg_mem_engine = 1; /* pfp */
1379 break;
1380 case AMDGPU_RING_TYPE_COMPUTE:
1381 *hdp_flush_mask = nbio_hf_reg->ref_and_mask_cp2 << ring->pipe;
1382 *reg_mem_engine = 0;
1383 break;
1384 case AMDGPU_RING_TYPE_MES:
1385 *hdp_flush_mask = nbio_hf_reg->ref_and_mask_cp8;
1386 *reg_mem_engine = 0;
1387 break;
1388 case AMDGPU_RING_TYPE_KIQ:
1389 *hdp_flush_mask = nbio_hf_reg->ref_and_mask_cp9;
1390 *reg_mem_engine = 0;
1391 break;
1392 default:
1393 DRM_ERROR("%s:unsupported ring type %d\n", __func__, ring->funcs->type);
1394 return;
1395 }
1396}
1397
1398int amdgpu_kiq_hdp_flush(struct amdgpu_device *adev)
1399{
1400 signed long r, cnt = 0;
1401 unsigned long flags;
1402 uint32_t seq;
1403 struct amdgpu_kiq *kiq = &adev->gfx.kiq[0];
1404 struct amdgpu_ring *ring = &kiq->ring;
1405
1406 if (amdgpu_device_skip_hw_access(adev))
1407 return 0;
1408
1409 if (adev->enable_mes_kiq && adev->mes.ring[0].sched.ready)
1410 return amdgpu_mes_hdp_flush(adev);
1411
1412 if (!ring->funcs->emit_hdp_flush) {
1413 return -EOPNOTSUPP;
1414 }
1415
1416 spin_lock_irqsave(&kiq->ring_lock, flags);
1417 r = amdgpu_ring_alloc(ring, 32);
1418 if (r)
1419 goto failed_unlock;
1420
1421 amdgpu_ring_emit_hdp_flush(ring);
1422 r = amdgpu_fence_emit_polling(ring, &seq, MAX_KIQ_REG_WAIT);
1423 if (r)
1424 goto failed_undo;
1425
1426 amdgpu_ring_commit(ring);
1427 spin_unlock_irqrestore(&kiq->ring_lock, flags);
1428
1429 r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT);
1430
1431 /* don't wait anymore for gpu reset case because this way may
1432 * block gpu_recover() routine forever, e.g. this virt_kiq_rreg
1433 * is triggered in TTM and ttm_bo_lock_delayed_workqueue() will
1434 * never return if we keep waiting in virt_kiq_rreg, which cause
1435 * gpu_recover() hang there.
1436 *
1437 * also don't wait anymore for IRQ context
1438 * */
1439 if (r < 1 && (amdgpu_in_reset(adev) || in_interrupt()))
1440 goto failed_kiq_hdp_flush;
1441
1442 might_sleep();
1443 while (r < 1 && cnt++ < MAX_KIQ_REG_TRY) {
1444 if (amdgpu_in_reset(adev))
1445 goto failed_kiq_hdp_flush;
1446
1447 msleep(MAX_KIQ_REG_BAILOUT_INTERVAL);
1448 r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT);
1449 }
1450
1451 if (cnt > MAX_KIQ_REG_TRY) {
1452 dev_err(adev->dev, "failed to flush HDP via KIQ timeout\n");
1453 return -ETIMEDOUT;
1454 }
1455
1456 return 0;
1457
1458failed_undo:
1459 amdgpu_ring_undo(ring);
1460failed_unlock:
1461 spin_unlock_irqrestore(&kiq->ring_lock, flags);
1462failed_kiq_hdp_flush:
1463 if (!amdgpu_in_reset(adev))
1464 dev_err(adev->dev, "failed to flush HDP via KIQ\n");
1465 return r < 0 ? r : -EIO;
1466}
1467
1468int amdgpu_gfx_get_num_kcq(struct amdgpu_device *adev)
1469{
1470 if (amdgpu_num_kcq == -1) {
1471 return 8;
1472 } else if (amdgpu_num_kcq > 8 || amdgpu_num_kcq < 0) {
1473 dev_warn(adev->dev, "set kernel compute queue number to 8 due to invalid parameter provided by user\n");
1474 return 8;
1475 }
1476 return amdgpu_num_kcq;
1477}
1478
1479void amdgpu_gfx_cp_init_microcode(struct amdgpu_device *adev,
1480 uint32_t ucode_id)
1481{
1482 const struct gfx_firmware_header_v1_0 *cp_hdr;
1483 const struct gfx_firmware_header_v2_0 *cp_hdr_v2_0;
1484 struct amdgpu_firmware_info *info = NULL;
1485 const struct firmware *ucode_fw;
1486 unsigned int fw_size;
1487
1488 switch (ucode_id) {
1489 case AMDGPU_UCODE_ID_CP_PFP:
1490 cp_hdr = (const struct gfx_firmware_header_v1_0 *)
1491 adev->gfx.pfp_fw->data;
1492 adev->gfx.pfp_fw_version =
1493 le32_to_cpu(cp_hdr->header.ucode_version);
1494 adev->gfx.pfp_feature_version =
1495 le32_to_cpu(cp_hdr->ucode_feature_version);
1496 ucode_fw = adev->gfx.pfp_fw;
1497 fw_size = le32_to_cpu(cp_hdr->header.ucode_size_bytes);
1498 break;
1499 case AMDGPU_UCODE_ID_CP_RS64_PFP:
1500 cp_hdr_v2_0 = (const struct gfx_firmware_header_v2_0 *)
1501 adev->gfx.pfp_fw->data;
1502 adev->gfx.pfp_fw_version =
1503 le32_to_cpu(cp_hdr_v2_0->header.ucode_version);
1504 adev->gfx.pfp_feature_version =
1505 le32_to_cpu(cp_hdr_v2_0->ucode_feature_version);
1506 ucode_fw = adev->gfx.pfp_fw;
1507 fw_size = le32_to_cpu(cp_hdr_v2_0->ucode_size_bytes);
1508 break;
1509 case AMDGPU_UCODE_ID_CP_RS64_PFP_P0_STACK:
1510 case AMDGPU_UCODE_ID_CP_RS64_PFP_P1_STACK:
1511 cp_hdr_v2_0 = (const struct gfx_firmware_header_v2_0 *)
1512 adev->gfx.pfp_fw->data;
1513 ucode_fw = adev->gfx.pfp_fw;
1514 fw_size = le32_to_cpu(cp_hdr_v2_0->data_size_bytes);
1515 break;
1516 case AMDGPU_UCODE_ID_CP_ME:
1517 cp_hdr = (const struct gfx_firmware_header_v1_0 *)
1518 adev->gfx.me_fw->data;
1519 adev->gfx.me_fw_version =
1520 le32_to_cpu(cp_hdr->header.ucode_version);
1521 adev->gfx.me_feature_version =
1522 le32_to_cpu(cp_hdr->ucode_feature_version);
1523 ucode_fw = adev->gfx.me_fw;
1524 fw_size = le32_to_cpu(cp_hdr->header.ucode_size_bytes);
1525 break;
1526 case AMDGPU_UCODE_ID_CP_RS64_ME:
1527 cp_hdr_v2_0 = (const struct gfx_firmware_header_v2_0 *)
1528 adev->gfx.me_fw->data;
1529 adev->gfx.me_fw_version =
1530 le32_to_cpu(cp_hdr_v2_0->header.ucode_version);
1531 adev->gfx.me_feature_version =
1532 le32_to_cpu(cp_hdr_v2_0->ucode_feature_version);
1533 ucode_fw = adev->gfx.me_fw;
1534 fw_size = le32_to_cpu(cp_hdr_v2_0->ucode_size_bytes);
1535 break;
1536 case AMDGPU_UCODE_ID_CP_RS64_ME_P0_STACK:
1537 case AMDGPU_UCODE_ID_CP_RS64_ME_P1_STACK:
1538 cp_hdr_v2_0 = (const struct gfx_firmware_header_v2_0 *)
1539 adev->gfx.me_fw->data;
1540 ucode_fw = adev->gfx.me_fw;
1541 fw_size = le32_to_cpu(cp_hdr_v2_0->data_size_bytes);
1542 break;
1543 case AMDGPU_UCODE_ID_CP_CE:
1544 cp_hdr = (const struct gfx_firmware_header_v1_0 *)
1545 adev->gfx.ce_fw->data;
1546 adev->gfx.ce_fw_version =
1547 le32_to_cpu(cp_hdr->header.ucode_version);
1548 adev->gfx.ce_feature_version =
1549 le32_to_cpu(cp_hdr->ucode_feature_version);
1550 ucode_fw = adev->gfx.ce_fw;
1551 fw_size = le32_to_cpu(cp_hdr->header.ucode_size_bytes);
1552 break;
1553 case AMDGPU_UCODE_ID_CP_MEC1:
1554 cp_hdr = (const struct gfx_firmware_header_v1_0 *)
1555 adev->gfx.mec_fw->data;
1556 adev->gfx.mec_fw_version =
1557 le32_to_cpu(cp_hdr->header.ucode_version);
1558 adev->gfx.mec_feature_version =
1559 le32_to_cpu(cp_hdr->ucode_feature_version);
1560 ucode_fw = adev->gfx.mec_fw;
1561 fw_size = le32_to_cpu(cp_hdr->header.ucode_size_bytes) -
1562 le32_to_cpu(cp_hdr->jt_size) * 4;
1563 break;
1564 case AMDGPU_UCODE_ID_CP_MEC1_JT:
1565 cp_hdr = (const struct gfx_firmware_header_v1_0 *)
1566 adev->gfx.mec_fw->data;
1567 ucode_fw = adev->gfx.mec_fw;
1568 fw_size = le32_to_cpu(cp_hdr->jt_size) * 4;
1569 break;
1570 case AMDGPU_UCODE_ID_CP_MEC2:
1571 cp_hdr = (const struct gfx_firmware_header_v1_0 *)
1572 adev->gfx.mec2_fw->data;
1573 adev->gfx.mec2_fw_version =
1574 le32_to_cpu(cp_hdr->header.ucode_version);
1575 adev->gfx.mec2_feature_version =
1576 le32_to_cpu(cp_hdr->ucode_feature_version);
1577 ucode_fw = adev->gfx.mec2_fw;
1578 fw_size = le32_to_cpu(cp_hdr->header.ucode_size_bytes) -
1579 le32_to_cpu(cp_hdr->jt_size) * 4;
1580 break;
1581 case AMDGPU_UCODE_ID_CP_MEC2_JT:
1582 cp_hdr = (const struct gfx_firmware_header_v1_0 *)
1583 adev->gfx.mec2_fw->data;
1584 ucode_fw = adev->gfx.mec2_fw;
1585 fw_size = le32_to_cpu(cp_hdr->jt_size) * 4;
1586 break;
1587 case AMDGPU_UCODE_ID_CP_RS64_MEC:
1588 cp_hdr_v2_0 = (const struct gfx_firmware_header_v2_0 *)
1589 adev->gfx.mec_fw->data;
1590 adev->gfx.mec_fw_version =
1591 le32_to_cpu(cp_hdr_v2_0->header.ucode_version);
1592 adev->gfx.mec_feature_version =
1593 le32_to_cpu(cp_hdr_v2_0->ucode_feature_version);
1594 ucode_fw = adev->gfx.mec_fw;
1595 fw_size = le32_to_cpu(cp_hdr_v2_0->ucode_size_bytes);
1596 break;
1597 case AMDGPU_UCODE_ID_CP_RS64_MEC_P0_STACK:
1598 case AMDGPU_UCODE_ID_CP_RS64_MEC_P1_STACK:
1599 case AMDGPU_UCODE_ID_CP_RS64_MEC_P2_STACK:
1600 case AMDGPU_UCODE_ID_CP_RS64_MEC_P3_STACK:
1601 cp_hdr_v2_0 = (const struct gfx_firmware_header_v2_0 *)
1602 adev->gfx.mec_fw->data;
1603 ucode_fw = adev->gfx.mec_fw;
1604 fw_size = le32_to_cpu(cp_hdr_v2_0->data_size_bytes);
1605 break;
1606 default:
1607 dev_err(adev->dev, "Invalid ucode id %u\n", ucode_id);
1608 return;
1609 }
1610
1611 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
1612 info = &adev->firmware.ucode[ucode_id];
1613 info->ucode_id = ucode_id;
1614 info->fw = ucode_fw;
1615 adev->firmware.fw_size += ALIGN(fw_size, PAGE_SIZE);
1616 }
1617}
1618
1619bool amdgpu_gfx_is_master_xcc(struct amdgpu_device *adev, int xcc_id)
1620{
1621 return !(xcc_id % (adev->gfx.num_xcc_per_xcp ?
1622 adev->gfx.num_xcc_per_xcp : 1));
1623}
1624
1625static ssize_t amdgpu_gfx_get_current_compute_partition(struct device *dev,
1626 struct device_attribute *addr,
1627 char *buf)
1628{
1629 struct drm_device *ddev = dev_get_drvdata(dev);
1630 struct amdgpu_device *adev = drm_to_adev(ddev);
1631 int mode;
1632
1633 /* Only minimal precaution taken to reject requests while in reset.*/
1634 if (amdgpu_in_reset(adev))
1635 return -EPERM;
1636
1637 mode = amdgpu_xcp_query_partition_mode(adev->xcp_mgr,
1638 AMDGPU_XCP_FL_NONE);
1639
1640 return sysfs_emit(buf, "%s\n", amdgpu_gfx_compute_mode_desc(mode));
1641}
1642
1643static ssize_t amdgpu_gfx_set_compute_partition(struct device *dev,
1644 struct device_attribute *addr,
1645 const char *buf, size_t count)
1646{
1647 struct drm_device *ddev = dev_get_drvdata(dev);
1648 struct amdgpu_device *adev = drm_to_adev(ddev);
1649 enum amdgpu_gfx_partition mode;
1650 int ret = 0, num_xcc;
1651
1652 num_xcc = NUM_XCC(adev->gfx.xcc_mask);
1653 if (num_xcc % 2 != 0)
1654 return -EINVAL;
1655
1656 if (!strncasecmp("SPX", buf, strlen("SPX"))) {
1657 mode = AMDGPU_SPX_PARTITION_MODE;
1658 } else if (!strncasecmp("DPX", buf, strlen("DPX"))) {
1659 /*
1660 * DPX mode needs AIDs to be in multiple of 2.
1661 * Each AID connects 2 XCCs.
1662 */
1663 if (num_xcc%4)
1664 return -EINVAL;
1665 mode = AMDGPU_DPX_PARTITION_MODE;
1666 } else if (!strncasecmp("TPX", buf, strlen("TPX"))) {
1667 if (num_xcc != 6)
1668 return -EINVAL;
1669 mode = AMDGPU_TPX_PARTITION_MODE;
1670 } else if (!strncasecmp("QPX", buf, strlen("QPX"))) {
1671 if (num_xcc != 8)
1672 return -EINVAL;
1673 mode = AMDGPU_QPX_PARTITION_MODE;
1674 } else if (!strncasecmp("CPX", buf, strlen("CPX"))) {
1675 mode = AMDGPU_CPX_PARTITION_MODE;
1676 } else {
1677 return -EINVAL;
1678 }
1679
1680 /* Don't allow a switch while under reset */
1681 if (!down_read_trylock(&adev->reset_domain->sem))
1682 return -EPERM;
1683
1684 ret = amdgpu_xcp_switch_partition_mode(adev->xcp_mgr, mode);
1685
1686 up_read(&adev->reset_domain->sem);
1687
1688 if (ret)
1689 return ret;
1690
1691 return count;
1692}
1693
1694static ssize_t compute_partition_mem_alloc_mode_show(struct device *dev,
1695 struct device_attribute *addr,
1696 char *buf)
1697{
1698 struct drm_device *ddev = dev_get_drvdata(dev);
1699 struct amdgpu_device *adev = drm_to_adev(ddev);
1700 int mode = adev->xcp_mgr->mem_alloc_mode;
1701
1702 return sysfs_emit(buf, "%s\n",
1703 amdgpu_gfx_compute_mem_alloc_mode_desc(mode));
1704}
1705
1706
1707static ssize_t compute_partition_mem_alloc_mode_store(struct device *dev,
1708 struct device_attribute *addr,
1709 const char *buf, size_t count)
1710{
1711 struct drm_device *ddev = dev_get_drvdata(dev);
1712 struct amdgpu_device *adev = drm_to_adev(ddev);
1713
1714 if (!strncasecmp("CAPPING", buf, strlen("CAPPING")))
1715 adev->xcp_mgr->mem_alloc_mode = AMDGPU_PARTITION_MEM_CAPPING_EVEN;
1716 else if (!strncasecmp("ALL", buf, strlen("ALL")))
1717 adev->xcp_mgr->mem_alloc_mode = AMDGPU_PARTITION_MEM_ALLOC_ALL;
1718 else
1719 return -EINVAL;
1720
1721 return count;
1722}
1723
1724static const char *xcp_desc[] = {
1725 [AMDGPU_SPX_PARTITION_MODE] = "SPX",
1726 [AMDGPU_DPX_PARTITION_MODE] = "DPX",
1727 [AMDGPU_TPX_PARTITION_MODE] = "TPX",
1728 [AMDGPU_QPX_PARTITION_MODE] = "QPX",
1729 [AMDGPU_CPX_PARTITION_MODE] = "CPX",
1730};
1731
1732static ssize_t amdgpu_gfx_get_available_compute_partition(struct device *dev,
1733 struct device_attribute *addr,
1734 char *buf)
1735{
1736 struct drm_device *ddev = dev_get_drvdata(dev);
1737 struct amdgpu_device *adev = drm_to_adev(ddev);
1738 struct amdgpu_xcp_mgr *xcp_mgr = adev->xcp_mgr;
1739 int size = 0, mode;
1740 char *sep = "";
1741
1742 if (!xcp_mgr || !xcp_mgr->avail_xcp_modes)
1743 return sysfs_emit(buf, "Not supported\n");
1744
1745 for_each_inst(mode, xcp_mgr->avail_xcp_modes) {
1746 size += sysfs_emit_at(buf, size, "%s%s", sep, xcp_desc[mode]);
1747 sep = ", ";
1748 }
1749
1750 size += sysfs_emit_at(buf, size, "\n");
1751
1752 return size;
1753}
1754
1755static int amdgpu_gfx_run_cleaner_shader_job(struct amdgpu_ring *ring)
1756{
1757 struct amdgpu_device *adev = ring->adev;
1758 struct drm_gpu_scheduler *sched = &ring->sched;
1759 struct drm_sched_entity entity;
1760 unsigned int ib_size_dw = 16;
1761 static atomic_t counter;
1762 struct dma_fence *f;
1763 struct amdgpu_job *job;
1764 struct amdgpu_ib *ib;
1765 void *owner;
1766 int r;
1767
1768 /* Initialize the scheduler entity */
1769 r = drm_sched_entity_init(&entity, DRM_SCHED_PRIORITY_NORMAL,
1770 &sched, 1, NULL);
1771 if (r) {
1772 dev_err(adev->dev, "Failed setting up GFX kernel entity.\n");
1773 goto err;
1774 }
1775
1776 /*
1777 * Use some unique dummy value as the owner to make sure we execute
1778 * the cleaner shader on each submission. The value just need to change
1779 * for each submission and is otherwise meaningless.
1780 */
1781 owner = (void *)(unsigned long)atomic_inc_return(&counter);
1782
1783 r = amdgpu_job_alloc_with_ib(ring->adev, &entity, owner,
1784 ib_size_dw * sizeof(uint32_t), 0,
1785 AMDGPU_KERNEL_JOB_ID_CLEANER_SHADER,
1786 &job);
1787 if (r)
1788 goto err;
1789
1790 job->enforce_isolation = true;
1791 /* always run the cleaner shader */
1792 job->run_cleaner_shader = true;
1793
1794 ib = &job->ibs[0];
1795 memset32(ib->ptr, ring->funcs->nop, ib_size_dw);
1796 ib->length_dw = ib_size_dw;
1797
1798 f = amdgpu_job_submit(job);
1799
1800 r = dma_fence_wait(f, false);
1801 if (r)
1802 goto err;
1803
1804 dma_fence_put(f);
1805
1806 /* Clean up the scheduler entity */
1807 drm_sched_entity_destroy(&entity);
1808 return 0;
1809
1810err:
1811 return r;
1812}
1813
1814static int amdgpu_gfx_run_cleaner_shader(struct amdgpu_device *adev, int xcp_id)
1815{
1816 int num_xcc = NUM_XCC(adev->gfx.xcc_mask);
1817 struct amdgpu_ring *ring;
1818 int num_xcc_to_clear;
1819 int i, r, xcc_id;
1820
1821 if (adev->gfx.num_xcc_per_xcp)
1822 num_xcc_to_clear = adev->gfx.num_xcc_per_xcp;
1823 else
1824 num_xcc_to_clear = 1;
1825
1826 for (xcc_id = 0; xcc_id < num_xcc; xcc_id++) {
1827 for (i = 0; i < adev->gfx.num_compute_rings; i++) {
1828 ring = &adev->gfx.compute_ring[i + xcc_id * adev->gfx.num_compute_rings];
1829 if ((ring->xcp_id == xcp_id) && ring->sched.ready) {
1830 r = amdgpu_gfx_run_cleaner_shader_job(ring);
1831 if (r)
1832 return r;
1833 num_xcc_to_clear--;
1834 break;
1835 }
1836 }
1837 }
1838
1839 if (num_xcc_to_clear)
1840 return -ENOENT;
1841
1842 return 0;
1843}
1844
1845/**
1846 * amdgpu_gfx_set_run_cleaner_shader - Execute the AMDGPU GFX Cleaner Shader
1847 * @dev: The device structure
1848 * @attr: The device attribute structure
1849 * @buf: The buffer containing the input data
1850 * @count: The size of the input data
1851 *
1852 * Provides the sysfs interface to manually run a cleaner shader, which is
1853 * used to clear the GPU state between different tasks. Writing a value to the
1854 * 'run_cleaner_shader' sysfs file triggers the cleaner shader execution.
1855 * The value written corresponds to the partition index on multi-partition
1856 * devices. On single-partition devices, the value should be '0'.
1857 *
1858 * The cleaner shader clears the Local Data Store (LDS) and General Purpose
1859 * Registers (GPRs) to ensure data isolation between GPU workloads.
1860 *
1861 * Return: The number of bytes written to the sysfs file.
1862 */
1863static ssize_t amdgpu_gfx_set_run_cleaner_shader(struct device *dev,
1864 struct device_attribute *attr,
1865 const char *buf,
1866 size_t count)
1867{
1868 struct drm_device *ddev = dev_get_drvdata(dev);
1869 struct amdgpu_device *adev = drm_to_adev(ddev);
1870 int ret;
1871 long value;
1872
1873 if (amdgpu_in_reset(adev))
1874 return -EPERM;
1875 if (adev->in_suspend && !adev->in_runpm)
1876 return -EPERM;
1877
1878 if (adev->gfx.disable_kq)
1879 return -EPERM;
1880
1881 ret = kstrtol(buf, 0, &value);
1882
1883 if (ret)
1884 return -EINVAL;
1885
1886 if (value < 0)
1887 return -EINVAL;
1888
1889 if (adev->xcp_mgr) {
1890 if (value >= adev->xcp_mgr->num_xcps)
1891 return -EINVAL;
1892 } else {
1893 if (value > 1)
1894 return -EINVAL;
1895 }
1896
1897 ret = pm_runtime_get_sync(ddev->dev);
1898 if (ret < 0) {
1899 pm_runtime_put_autosuspend(ddev->dev);
1900 return ret;
1901 }
1902
1903 ret = amdgpu_gfx_run_cleaner_shader(adev, value);
1904
1905 pm_runtime_put_autosuspend(ddev->dev);
1906
1907 if (ret)
1908 return ret;
1909
1910 return count;
1911}
1912
1913/**
1914 * amdgpu_gfx_get_enforce_isolation - Query AMDGPU GFX Enforce Isolation Settings
1915 * @dev: The device structure
1916 * @attr: The device attribute structure
1917 * @buf: The buffer to store the output data
1918 *
1919 * Provides the sysfs read interface to get the current settings of the 'enforce_isolation'
1920 * feature for each GPU partition. Reading from the 'enforce_isolation'
1921 * sysfs file returns the isolation settings for all partitions, where '0'
1922 * indicates disabled, '1' indicates enabled, and '2' indicates enabled in legacy mode,
1923 * and '3' indicates enabled without cleaner shader.
1924 *
1925 * Return: The number of bytes read from the sysfs file.
1926 */
1927static ssize_t amdgpu_gfx_get_enforce_isolation(struct device *dev,
1928 struct device_attribute *attr,
1929 char *buf)
1930{
1931 struct drm_device *ddev = dev_get_drvdata(dev);
1932 struct amdgpu_device *adev = drm_to_adev(ddev);
1933 int i;
1934 ssize_t size = 0;
1935
1936 if (adev->xcp_mgr) {
1937 for (i = 0; i < adev->xcp_mgr->num_xcps; i++) {
1938 size += sysfs_emit_at(buf, size, "%u", adev->enforce_isolation[i]);
1939 if (i < (adev->xcp_mgr->num_xcps - 1))
1940 size += sysfs_emit_at(buf, size, " ");
1941 }
1942 buf[size++] = '\n';
1943 } else {
1944 size = sysfs_emit_at(buf, 0, "%u\n", adev->enforce_isolation[0]);
1945 }
1946
1947 return size;
1948}
1949
1950/**
1951 * amdgpu_gfx_set_enforce_isolation - Control AMDGPU GFX Enforce Isolation
1952 * @dev: The device structure
1953 * @attr: The device attribute structure
1954 * @buf: The buffer containing the input data
1955 * @count: The size of the input data
1956 *
1957 * This function allows control over the 'enforce_isolation' feature, which
1958 * serializes access to the graphics engine. Writing '0' to disable, '1' to
1959 * enable isolation with cleaner shader, '2' to enable legacy isolation without
1960 * cleaner shader, or '3' to enable process isolation without submitting the
1961 * cleaner shader to the 'enforce_isolation' sysfs file sets the isolation mode
1962 * for each partition. The input should specify the setting for all
1963 * partitions.
1964 *
1965 * Return: The number of bytes written to the sysfs file.
1966 */
1967static ssize_t amdgpu_gfx_set_enforce_isolation(struct device *dev,
1968 struct device_attribute *attr,
1969 const char *buf, size_t count)
1970{
1971 struct drm_device *ddev = dev_get_drvdata(dev);
1972 struct amdgpu_device *adev = drm_to_adev(ddev);
1973 long partition_values[MAX_XCP] = {0};
1974 int ret, i, num_partitions;
1975 const char *input_buf = buf;
1976
1977 for (i = 0; i < (adev->xcp_mgr ? adev->xcp_mgr->num_xcps : 1); i++) {
1978 ret = sscanf(input_buf, "%ld", &partition_values[i]);
1979 if (ret <= 0)
1980 break;
1981
1982 /* Move the pointer to the next value in the string */
1983 input_buf = strchr(input_buf, ' ');
1984 if (input_buf) {
1985 input_buf++;
1986 } else {
1987 i++;
1988 break;
1989 }
1990 }
1991 num_partitions = i;
1992
1993 if (adev->xcp_mgr && num_partitions != adev->xcp_mgr->num_xcps)
1994 return -EINVAL;
1995
1996 if (!adev->xcp_mgr && num_partitions != 1)
1997 return -EINVAL;
1998
1999 for (i = 0; i < num_partitions; i++) {
2000 if (partition_values[i] != 0 &&
2001 partition_values[i] != 1 &&
2002 partition_values[i] != 2 &&
2003 partition_values[i] != 3)
2004 return -EINVAL;
2005 }
2006
2007 mutex_lock(&adev->enforce_isolation_mutex);
2008 for (i = 0; i < num_partitions; i++) {
2009 switch (partition_values[i]) {
2010 case 0:
2011 default:
2012 adev->enforce_isolation[i] = AMDGPU_ENFORCE_ISOLATION_DISABLE;
2013 break;
2014 case 1:
2015 adev->enforce_isolation[i] =
2016 AMDGPU_ENFORCE_ISOLATION_ENABLE;
2017 break;
2018 case 2:
2019 adev->enforce_isolation[i] =
2020 AMDGPU_ENFORCE_ISOLATION_ENABLE_LEGACY;
2021 break;
2022 case 3:
2023 adev->enforce_isolation[i] =
2024 AMDGPU_ENFORCE_ISOLATION_NO_CLEANER_SHADER;
2025 break;
2026 }
2027 }
2028 mutex_unlock(&adev->enforce_isolation_mutex);
2029
2030 amdgpu_mes_update_enforce_isolation(adev);
2031
2032 return count;
2033}
2034
2035static ssize_t amdgpu_gfx_get_gfx_reset_mask(struct device *dev,
2036 struct device_attribute *attr,
2037 char *buf)
2038{
2039 struct drm_device *ddev = dev_get_drvdata(dev);
2040 struct amdgpu_device *adev = drm_to_adev(ddev);
2041
2042 if (!adev)
2043 return -ENODEV;
2044
2045 return amdgpu_show_reset_mask(buf, adev->gfx.gfx_supported_reset);
2046}
2047
2048static ssize_t amdgpu_gfx_get_compute_reset_mask(struct device *dev,
2049 struct device_attribute *attr,
2050 char *buf)
2051{
2052 struct drm_device *ddev = dev_get_drvdata(dev);
2053 struct amdgpu_device *adev = drm_to_adev(ddev);
2054
2055 if (!adev)
2056 return -ENODEV;
2057
2058 return amdgpu_show_reset_mask(buf, adev->gfx.compute_supported_reset);
2059}
2060
2061static int amdgpu_gfx_mes_reset_queue_start(struct amdgpu_ring *ring,
2062 unsigned int vmid,
2063 struct amdgpu_fence *timedout_fence,
2064 bool use_mmio)
2065{
2066 struct amdgpu_device *adev = ring->adev;
2067 bool reinit_queue;
2068 int r;
2069
2070 if ((ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) &&
2071 adev->mes.compute_pipe_reset_enabled)
2072 reinit_queue = true;
2073 else if ((ring->funcs->type == AMDGPU_RING_TYPE_GFX) &&
2074 adev->mes.gfx_pipe_reset_enabled)
2075 reinit_queue = true;
2076 else
2077 reinit_queue = use_mmio;
2078
2079 amdgpu_ring_reset_helper_begin(ring, timedout_fence);
2080
2081 r = amdgpu_mes_reset_legacy_queue(ring->adev, ring, vmid, use_mmio, 0);
2082 if (r)
2083 return r;
2084
2085 if (reinit_queue) {
2086 r = amdgpu_mes_unmap_legacy_queue(adev, ring,
2087 RESET_QUEUES, 0, 0, 0);
2088 if (r)
2089 return r;
2090 amdgpu_gfx_mqd_reset_restore(ring);
2091
2092 r = amdgpu_mes_map_legacy_queue(adev, ring, 0);
2093 if (r) {
2094 dev_err(adev->dev, "failed to remap kgq\n");
2095 return r;
2096 }
2097 }
2098 return 0;
2099}
2100
2101int amdgpu_gfx_mes_reset_queue(struct amdgpu_ring *ring,
2102 unsigned int vmid,
2103 struct amdgpu_fence *timedout_fence,
2104 bool use_mmio)
2105{
2106 int r;
2107
2108 r = amdgpu_gfx_mes_reset_queue_start(ring, vmid, timedout_fence,
2109 use_mmio);
2110 if (r)
2111 return r;
2112 return amdgpu_ring_reset_helper_end(ring, timedout_fence);
2113}
2114
2115static DEVICE_ATTR(run_cleaner_shader, 0200,
2116 NULL, amdgpu_gfx_set_run_cleaner_shader);
2117
2118static DEVICE_ATTR(enforce_isolation, 0644,
2119 amdgpu_gfx_get_enforce_isolation,
2120 amdgpu_gfx_set_enforce_isolation);
2121
2122static DEVICE_ATTR(current_compute_partition, 0644,
2123 amdgpu_gfx_get_current_compute_partition,
2124 amdgpu_gfx_set_compute_partition);
2125
2126static DEVICE_ATTR(available_compute_partition, 0444,
2127 amdgpu_gfx_get_available_compute_partition, NULL);
2128static DEVICE_ATTR(gfx_reset_mask, 0444,
2129 amdgpu_gfx_get_gfx_reset_mask, NULL);
2130
2131static DEVICE_ATTR(compute_reset_mask, 0444,
2132 amdgpu_gfx_get_compute_reset_mask, NULL);
2133
2134static DEVICE_ATTR(compute_partition_mem_alloc_mode, 0644,
2135 compute_partition_mem_alloc_mode_show,
2136 compute_partition_mem_alloc_mode_store);
2137
2138static int amdgpu_gfx_sysfs_xcp_init(struct amdgpu_device *adev)
2139{
2140 struct amdgpu_xcp_mgr *xcp_mgr = adev->xcp_mgr;
2141 bool xcp_switch_supported;
2142 int r;
2143
2144 if (!xcp_mgr)
2145 return 0;
2146
2147 xcp_switch_supported =
2148 (xcp_mgr->funcs && xcp_mgr->funcs->switch_partition_mode);
2149
2150 if (!xcp_switch_supported)
2151 dev_attr_current_compute_partition.attr.mode &=
2152 ~(S_IWUSR | S_IWGRP | S_IWOTH);
2153
2154 r = device_create_file(adev->dev, &dev_attr_current_compute_partition);
2155 if (r)
2156 return r;
2157
2158 r = device_create_file(adev->dev,
2159 &dev_attr_compute_partition_mem_alloc_mode);
2160 if (r)
2161 return r;
2162
2163 if (xcp_switch_supported)
2164 r = device_create_file(adev->dev,
2165 &dev_attr_available_compute_partition);
2166
2167 return r;
2168}
2169
2170static void amdgpu_gfx_sysfs_xcp_fini(struct amdgpu_device *adev)
2171{
2172 struct amdgpu_xcp_mgr *xcp_mgr = adev->xcp_mgr;
2173 bool xcp_switch_supported;
2174
2175 if (!xcp_mgr)
2176 return;
2177
2178 xcp_switch_supported =
2179 (xcp_mgr->funcs && xcp_mgr->funcs->switch_partition_mode);
2180 device_remove_file(adev->dev, &dev_attr_current_compute_partition);
2181
2182 device_remove_file(adev->dev, &dev_attr_compute_partition_mem_alloc_mode);
2183
2184 if (xcp_switch_supported)
2185 device_remove_file(adev->dev,
2186 &dev_attr_available_compute_partition);
2187}
2188
2189static int amdgpu_gfx_sysfs_isolation_shader_init(struct amdgpu_device *adev)
2190{
2191 int r;
2192
2193 r = device_create_file(adev->dev, &dev_attr_enforce_isolation);
2194 if (r)
2195 return r;
2196 if (adev->gfx.enable_cleaner_shader)
2197 r = device_create_file(adev->dev, &dev_attr_run_cleaner_shader);
2198
2199 return r;
2200}
2201
2202static void amdgpu_gfx_sysfs_isolation_shader_fini(struct amdgpu_device *adev)
2203{
2204 device_remove_file(adev->dev, &dev_attr_enforce_isolation);
2205 if (adev->gfx.enable_cleaner_shader)
2206 device_remove_file(adev->dev, &dev_attr_run_cleaner_shader);
2207}
2208
2209static int amdgpu_gfx_sysfs_reset_mask_init(struct amdgpu_device *adev)
2210{
2211 int r = 0;
2212
2213 if (!amdgpu_gpu_recovery)
2214 return r;
2215
2216 if (adev->gfx.num_gfx_rings) {
2217 r = device_create_file(adev->dev, &dev_attr_gfx_reset_mask);
2218 if (r)
2219 return r;
2220 }
2221
2222 if (adev->gfx.num_compute_rings) {
2223 r = device_create_file(adev->dev, &dev_attr_compute_reset_mask);
2224 if (r)
2225 return r;
2226 }
2227
2228 return r;
2229}
2230
2231static void amdgpu_gfx_sysfs_reset_mask_fini(struct amdgpu_device *adev)
2232{
2233 if (!amdgpu_gpu_recovery)
2234 return;
2235
2236 if (adev->gfx.num_gfx_rings)
2237 device_remove_file(adev->dev, &dev_attr_gfx_reset_mask);
2238
2239 if (adev->gfx.num_compute_rings)
2240 device_remove_file(adev->dev, &dev_attr_compute_reset_mask);
2241}
2242
2243int amdgpu_gfx_sysfs_init(struct amdgpu_device *adev)
2244{
2245 int r;
2246
2247 r = amdgpu_gfx_sysfs_xcp_init(adev);
2248 if (r) {
2249 dev_err(adev->dev, "failed to create xcp sysfs files");
2250 return r;
2251 }
2252
2253 r = amdgpu_gfx_sysfs_isolation_shader_init(adev);
2254 if (r)
2255 dev_err(adev->dev, "failed to create isolation sysfs files");
2256
2257 r = amdgpu_gfx_sysfs_reset_mask_init(adev);
2258 if (r)
2259 dev_err(adev->dev, "failed to create reset mask sysfs files");
2260
2261 return r;
2262}
2263
2264void amdgpu_gfx_sysfs_fini(struct amdgpu_device *adev)
2265{
2266 if (adev->dev->kobj.sd) {
2267 amdgpu_gfx_sysfs_xcp_fini(adev);
2268 amdgpu_gfx_sysfs_isolation_shader_fini(adev);
2269 amdgpu_gfx_sysfs_reset_mask_fini(adev);
2270 }
2271}
2272
2273static void amdgpu_gfx_reset_start_compute_scheds(struct amdgpu_device *adev,
2274 struct amdgpu_ring *guilty_ring)
2275{
2276 struct amdgpu_ring *ring;
2277 int i;
2278
2279 for (i = 0; i < adev->gfx.num_compute_rings; i++) {
2280 ring = &adev->gfx.compute_ring[i];
2281 if (ring == guilty_ring)
2282 continue;
2283 drm_sched_wqueue_start(&ring->sched);
2284 }
2285}
2286
2287static void amdgpu_gfx_reset_stop_compute_scheds(struct amdgpu_device *adev,
2288 struct amdgpu_ring *guilty_ring)
2289{
2290 struct amdgpu_ring *ring;
2291 int i;
2292
2293 for (i = 0; i < adev->gfx.num_compute_rings; i++) {
2294 ring = &adev->gfx.compute_ring[i];
2295 if (ring == guilty_ring)
2296 continue;
2297 drm_sched_wqueue_stop(&ring->sched);
2298 }
2299}
2300
2301/*
2302 * Match the MES-reported hung doorbell against a compute ring and run
2303 * the reset. On hit, the matched ring and its guilty fence are returned
2304 * via *out_ring / *out_fence so the caller can defer reset end until
2305 * after MES has resumed all gangs.
2306 */
2307static int amdgpu_gfx_reset_mes_kcq(struct amdgpu_device *adev,
2308 struct amdgpu_ring *guilty_ring,
2309 unsigned int db,
2310 struct amdgpu_ring **out_ring,
2311 struct amdgpu_fence **out_fence)
2312{
2313 bool use_mmio = adev->gfx.mec.use_mmio_for_reset;
2314 struct amdgpu_fence *fence;
2315 struct amdgpu_ring *ring;
2316 int i, r;
2317
2318 *out_ring = NULL;
2319 *out_fence = NULL;
2320 for (i = 0; i < adev->gfx.num_compute_rings; i++) {
2321 ring = &adev->gfx.compute_ring[i];
2322 if (ring == guilty_ring)
2323 continue;
2324 if (ring->doorbell_index == db) {
2325 fence = amdgpu_ring_find_guilty_fence(ring);
2326 r = amdgpu_gfx_mes_reset_queue_start(ring, 0, fence,
2327 use_mmio);
2328 if (r)
2329 return r;
2330 *out_ring = ring;
2331 *out_fence = fence;
2332 break;
2333 }
2334 }
2335 return 0;
2336}
2337
2338int amdgpu_gfx_reset_mes_compute(struct amdgpu_device *adev,
2339 struct amdgpu_ring *ring,
2340 struct amdgpu_fence *guilty_fence,
2341 struct amdgpu_usermode_queue *uq,
2342 unsigned int *hung_queue_count,
2343 void *faulty_queue_input)
2344{
2345 struct amdgpu_mes_hung_queue_hqd_info *hqd_info =
2346 (struct amdgpu_mes_hung_queue_hqd_info *)
2347 &adev->gfx.mec.mes_hung_db_array[adev->mes.hung_queue_hqd_info_offset];
2348 int i, r, pipe, queue, queue_type;
2349 unsigned int num_hung = 0;
2350 bool use_mmio = adev->gfx.mec.use_mmio_for_reset;
2351 struct mes_remove_queue_input *queue_input = (struct mes_remove_queue_input *)faulty_queue_input;
2352 struct amdgpu_gfx_deferred_entry deferred_end[AMDGPU_MAX_COMPUTE_RINGS + 1];
2353 int n_deferred = 0;
2354 int ring_err;
2355
2356 guard(mutex)(&adev->gfx.mec.reset_mutex);
2357 /* stop the drm schedulers for all compute queues */
2358 amdgpu_gfx_reset_stop_compute_scheds(adev, ring);
2359 /* suspend all will determine which queues are hung.
2360 * reset detect will return the array of bad queue doorbells
2361 */
2362 r = amdgpu_mes_suspend(adev, 0);
2363 /* if suspend all success, it should no hang queue */
2364 if (!r)
2365 /* always reset the KCQ/userq since we need to signal the fence
2366 * and we could be stuck in a loop which is preemptable.
2367 */
2368 goto fence_reset;
2369 r = amdgpu_mes_detect_and_reset_hung_queues(adev, AMDGPU_RING_TYPE_COMPUTE,
2370 true, &num_hung, adev->gfx.mec.mes_hung_db_array, 0);
2371 if (r)
2372 goto out;
2373 if (hung_queue_count)
2374 *hung_queue_count = num_hung;
2375
2376fence_reset:
2377 /* reset the queue this came from if specified */
2378 if (ring) {
2379 r = amdgpu_gfx_mes_reset_queue_start(ring, 0, guilty_fence,
2380 use_mmio);
2381 if (r)
2382 goto out;
2383 deferred_end[n_deferred].ring = ring;
2384 deferred_end[n_deferred].fence = guilty_fence;
2385 n_deferred++;
2386 }
2387 if (uq) {
2388 r = mes_userq_reset(uq);
2389 if (r)
2390 goto out;
2391 }
2392 for (i = 0; i < num_hung; i++) {
2393 struct amdgpu_ring *hr = NULL;
2394 struct amdgpu_fence *hf = NULL;
2395
2396 pipe = hqd_info[i].pipe_index;
2397 queue = hqd_info[i].queue_index;
2398 queue_type = hqd_info[i].queue_type;
2399
2400 /* reset any KCQs */
2401 r = amdgpu_gfx_reset_mes_kcq(adev, ring,
2402 adev->gfx.mec.mes_hung_db_array[i],
2403 &hr, &hf);
2404 if (r)
2405 goto out;
2406 if (hr) {
2407 deferred_end[n_deferred].ring = hr;
2408 deferred_end[n_deferred].fence = hf;
2409 n_deferred++;
2410 }
2411 /* reset any KFD queues */
2412 r = amdgpu_amdkfd_reset_mes_queue(adev, 0, queue_type, pipe, queue,
2413 adev->gfx.mec.mes_hung_db_array[i]);
2414 if (r)
2415 goto out;
2416 /* reset KGD user queues */
2417 r = mes_userq_reset_queue(adev, uq, queue_type, pipe, queue,
2418 adev->gfx.mec.mes_hung_db_array[i]);
2419 if (r)
2420 goto out;
2421 }
2422
2423 /* MES doesn't detect any hung queue but we have a known bad queue
2424 * and it is not KCQ
2425 */
2426 if (!num_hung && queue_input && !ring) {
2427 /* MES suspend_all is successful means this bad queue is
2428 * preempted successfuly. Remove it before resume all so it
2429 * doesn't get mapped back
2430 */
2431 if (!down_read_trylock(&adev->reset_domain->sem)) {
2432 r = -EIO;
2433 goto out;
2434 }
2435 amdgpu_mes_lock(&adev->mes);
2436 r = adev->mes.funcs->remove_hw_queue(&adev->mes, queue_input);
2437 amdgpu_mes_unlock(&adev->mes);
2438 up_read(&adev->reset_domain->sem);
2439 }
2440
2441out:
2442 /* resume all will enable the non-hung queues */
2443 amdgpu_mes_resume(adev, 0);
2444
2445 /* Now CP is running again — replay backed-up commands and ring
2446 * doorbells on each reset queue.
2447 */
2448 ring_err = r;
2449 for (i = 0; i < n_deferred; i++) {
2450 int er = amdgpu_ring_reset_helper_end(deferred_end[i].ring,
2451 deferred_end[i].fence);
2452
2453 if (er && !ring_err)
2454 ring_err = er;
2455 }
2456
2457 if (!ring_err)
2458 amdgpu_gfx_reset_start_compute_scheds(adev, ring);
2459
2460 /* If this reset is triggered by non-KCQ, the KCQ result after resume must
2461 * not override the reset result; otherwise a false reset failure is returned
2462 * to the non-KCQ caller
2463 */
2464 return ring ? ring_err : r;
2465}
2466
2467int amdgpu_gfx_cleaner_shader_sw_init(struct amdgpu_device *adev,
2468 unsigned int cleaner_shader_size)
2469{
2470 if (!adev->gfx.enable_cleaner_shader)
2471 return -EOPNOTSUPP;
2472
2473 return amdgpu_bo_create_kernel(adev, cleaner_shader_size, PAGE_SIZE,
2474 AMDGPU_GEM_DOMAIN_VRAM | AMDGPU_GEM_DOMAIN_GTT,
2475 &adev->gfx.cleaner_shader_obj,
2476 &adev->gfx.cleaner_shader_gpu_addr,
2477 (void **)&adev->gfx.cleaner_shader_cpu_ptr);
2478}
2479
2480void amdgpu_gfx_cleaner_shader_sw_fini(struct amdgpu_device *adev)
2481{
2482 if (!adev->gfx.enable_cleaner_shader)
2483 return;
2484
2485 amdgpu_bo_free_kernel(&adev->gfx.cleaner_shader_obj,
2486 &adev->gfx.cleaner_shader_gpu_addr,
2487 (void **)&adev->gfx.cleaner_shader_cpu_ptr);
2488}
2489
2490void amdgpu_gfx_cleaner_shader_init(struct amdgpu_device *adev,
2491 unsigned int cleaner_shader_size,
2492 const void *cleaner_shader_ptr)
2493{
2494 if (!adev->gfx.enable_cleaner_shader)
2495 return;
2496
2497 if (adev->gfx.cleaner_shader_cpu_ptr && cleaner_shader_ptr)
2498 memcpy_toio(adev->gfx.cleaner_shader_cpu_ptr, cleaner_shader_ptr,
2499 cleaner_shader_size);
2500}
2501
2502/**
2503 * amdgpu_gfx_kfd_sch_ctrl - Control the KFD scheduler from the KGD (Graphics Driver)
2504 * @adev: amdgpu_device pointer
2505 * @idx: Index of the scheduler to control
2506 * @enable: Whether to enable or disable the KFD scheduler
2507 *
2508 * This function is used to control the KFD (Kernel Fusion Driver) scheduler
2509 * from the KGD. It is part of the cleaner shader feature. This function plays
2510 * a key role in enforcing process isolation on the GPU.
2511 *
2512 * The function uses a reference count mechanism (kfd_sch_req_count) to keep
2513 * track of the number of requests to enable the KFD scheduler. When a request
2514 * to enable the KFD scheduler is made, the reference count is decremented.
2515 * When the reference count reaches zero, a delayed work is scheduled to
2516 * enforce isolation after a delay of GFX_SLICE_PERIOD.
2517 *
2518 * When a request to disable the KFD scheduler is made, the function first
2519 * checks if the reference count is zero. If it is, it cancels the delayed work
2520 * for enforcing isolation and checks if the KFD scheduler is active. If the
2521 * KFD scheduler is active, it sends a request to stop the KFD scheduler and
2522 * sets the KFD scheduler state to inactive. Then, it increments the reference
2523 * count.
2524 *
2525 * The function is synchronized using the kfd_sch_mutex to ensure that the KFD
2526 * scheduler state and reference count are updated atomically.
2527 *
2528 * Note: If the reference count is already zero when a request to enable the
2529 * KFD scheduler is made, it means there's an imbalance bug somewhere. The
2530 * function triggers a warning in this case.
2531 */
2532static void amdgpu_gfx_kfd_sch_ctrl(struct amdgpu_device *adev, u32 idx,
2533 bool enable)
2534{
2535 mutex_lock(&adev->gfx.userq_sch_mutex);
2536
2537 if (enable) {
2538 /* If the count is already 0, it means there's an imbalance bug somewhere.
2539 * Note that the bug may be in a different caller than the one which triggers the
2540 * WARN_ON_ONCE.
2541 */
2542 if (WARN_ON_ONCE(adev->gfx.userq_sch_req_count[idx] == 0)) {
2543 dev_err(adev->dev, "Attempted to enable KFD scheduler when reference count is already zero\n");
2544 goto unlock;
2545 }
2546
2547 adev->gfx.userq_sch_req_count[idx]--;
2548
2549 if (adev->gfx.userq_sch_req_count[idx] == 0 &&
2550 adev->gfx.userq_sch_inactive[idx]) {
2551 schedule_delayed_work(&adev->gfx.enforce_isolation[idx].work,
2552 msecs_to_jiffies(adev->gfx.enforce_isolation_time[idx]));
2553 }
2554 } else {
2555 if (adev->gfx.userq_sch_req_count[idx] == 0) {
2556 cancel_delayed_work_sync(&adev->gfx.enforce_isolation[idx].work);
2557 if (!adev->gfx.userq_sch_inactive[idx]) {
2558 amdgpu_userq_stop_sched_for_enforce_isolation(adev, idx);
2559 if (adev->kfd.init_complete)
2560 amdgpu_amdkfd_stop_sched(adev, idx);
2561 adev->gfx.userq_sch_inactive[idx] = true;
2562 }
2563 }
2564
2565 adev->gfx.userq_sch_req_count[idx]++;
2566 }
2567
2568unlock:
2569 mutex_unlock(&adev->gfx.userq_sch_mutex);
2570}
2571
2572/**
2573 * amdgpu_gfx_enforce_isolation_handler - work handler for enforcing shader isolation
2574 *
2575 * @work: work_struct.
2576 *
2577 * This function is the work handler for enforcing shader isolation on AMD GPUs.
2578 * It counts the number of emitted fences for each GFX and compute ring. If there
2579 * are any fences, it schedules the `enforce_isolation_work` to be run after a
2580 * delay of `GFX_SLICE_PERIOD`. If there are no fences, it signals the Kernel Fusion
2581 * Driver (KFD) to resume the runqueue. The function is synchronized using the
2582 * `enforce_isolation_mutex`.
2583 */
2584void amdgpu_gfx_enforce_isolation_handler(struct work_struct *work)
2585{
2586 struct amdgpu_isolation_work *isolation_work =
2587 container_of(work, struct amdgpu_isolation_work, work.work);
2588 struct amdgpu_device *adev = isolation_work->adev;
2589 u32 i, idx, fences = 0;
2590
2591 if (isolation_work->xcp_id == AMDGPU_XCP_NO_PARTITION)
2592 idx = 0;
2593 else
2594 idx = isolation_work->xcp_id;
2595
2596 if (idx >= MAX_XCP)
2597 return;
2598
2599 mutex_lock(&adev->enforce_isolation_mutex);
2600 for (i = 0; i < AMDGPU_MAX_GFX_RINGS; ++i) {
2601 if (isolation_work->xcp_id == adev->gfx.gfx_ring[i].xcp_id)
2602 fences += amdgpu_fence_count_emitted(&adev->gfx.gfx_ring[i]);
2603 }
2604 for (i = 0; i < (AMDGPU_MAX_COMPUTE_RINGS * AMDGPU_MAX_GC_INSTANCES); ++i) {
2605 if (isolation_work->xcp_id == adev->gfx.compute_ring[i].xcp_id)
2606 fences += amdgpu_fence_count_emitted(&adev->gfx.compute_ring[i]);
2607 }
2608 if (fences) {
2609 /* we've already had our timeslice, so let's wrap this up */
2610 schedule_delayed_work(&adev->gfx.enforce_isolation[idx].work,
2611 msecs_to_jiffies(1));
2612 } else {
2613 /* Tell KFD to resume the runqueue */
2614 WARN_ON_ONCE(!adev->gfx.userq_sch_inactive[idx]);
2615 WARN_ON_ONCE(adev->gfx.userq_sch_req_count[idx]);
2616
2617 amdgpu_userq_start_sched_for_enforce_isolation(adev, idx);
2618 if (adev->kfd.init_complete)
2619 amdgpu_amdkfd_start_sched(adev, idx);
2620 adev->gfx.userq_sch_inactive[idx] = false;
2621 }
2622 mutex_unlock(&adev->enforce_isolation_mutex);
2623}
2624
2625/**
2626 * amdgpu_gfx_enforce_isolation_wait_for_kfd - Manage KFD wait period for process isolation
2627 * @adev: amdgpu_device pointer
2628 * @idx: Index of the GPU partition
2629 *
2630 * When kernel submissions come in, the jobs are given a time slice and once
2631 * that time slice is up, if there are KFD user queues active, kernel
2632 * submissions are blocked until KFD has had its time slice. Once the KFD time
2633 * slice is up, KFD user queues are preempted and kernel submissions are
2634 * unblocked and allowed to run again.
2635 */
2636static void
2637amdgpu_gfx_enforce_isolation_wait_for_kfd(struct amdgpu_device *adev,
2638 u32 idx)
2639{
2640 unsigned long cjiffies;
2641 bool wait = false;
2642
2643 mutex_lock(&adev->enforce_isolation_mutex);
2644 if (adev->enforce_isolation[idx] == AMDGPU_ENFORCE_ISOLATION_ENABLE) {
2645 /* set the initial values if nothing is set */
2646 if (!adev->gfx.enforce_isolation_jiffies[idx]) {
2647 adev->gfx.enforce_isolation_jiffies[idx] = jiffies;
2648 adev->gfx.enforce_isolation_time[idx] = GFX_SLICE_PERIOD_MS;
2649 }
2650 /* Make sure KFD gets a chance to run */
2651 if (amdgpu_amdkfd_compute_active(adev, idx)) {
2652 cjiffies = jiffies;
2653 if (time_after(cjiffies, adev->gfx.enforce_isolation_jiffies[idx])) {
2654 cjiffies -= adev->gfx.enforce_isolation_jiffies[idx];
2655 if ((jiffies_to_msecs(cjiffies) >= GFX_SLICE_PERIOD_MS)) {
2656 /* if our time is up, let KGD work drain before scheduling more */
2657 wait = true;
2658 /* reset the timer period */
2659 adev->gfx.enforce_isolation_time[idx] = GFX_SLICE_PERIOD_MS;
2660 } else {
2661 /* set the timer period to what's left in our time slice */
2662 adev->gfx.enforce_isolation_time[idx] =
2663 GFX_SLICE_PERIOD_MS - jiffies_to_msecs(cjiffies);
2664 }
2665 } else {
2666 /* if jiffies wrap around we will just wait a little longer */
2667 adev->gfx.enforce_isolation_jiffies[idx] = jiffies;
2668 }
2669 } else {
2670 /* if there is no KFD work, then set the full slice period */
2671 adev->gfx.enforce_isolation_jiffies[idx] = jiffies;
2672 adev->gfx.enforce_isolation_time[idx] = GFX_SLICE_PERIOD_MS;
2673 }
2674 }
2675 mutex_unlock(&adev->enforce_isolation_mutex);
2676
2677 if (wait)
2678 msleep(GFX_SLICE_PERIOD_MS);
2679}
2680
2681/**
2682 * amdgpu_gfx_enforce_isolation_ring_begin_use - Begin use of a ring with enforced isolation
2683 * @ring: Pointer to the amdgpu_ring structure
2684 *
2685 * Ring begin_use helper implementation for gfx which serializes access to the
2686 * gfx IP between kernel submission IOCTLs and KFD user queues when isolation
2687 * enforcement is enabled. The kernel submission IOCTLs and KFD user queues
2688 * each get a time slice when both are active.
2689 */
2690void amdgpu_gfx_enforce_isolation_ring_begin_use(struct amdgpu_ring *ring)
2691{
2692 struct amdgpu_device *adev = ring->adev;
2693 u32 idx;
2694 bool sched_work = false;
2695
2696 if (!adev->gfx.enable_cleaner_shader)
2697 return;
2698
2699 if (ring->xcp_id == AMDGPU_XCP_NO_PARTITION)
2700 idx = 0;
2701 else
2702 idx = ring->xcp_id;
2703
2704 if (idx >= MAX_XCP)
2705 return;
2706
2707 /* Don't submit more work until KFD has had some time */
2708 amdgpu_gfx_enforce_isolation_wait_for_kfd(adev, idx);
2709
2710 mutex_lock(&adev->enforce_isolation_mutex);
2711 if (adev->enforce_isolation[idx] == AMDGPU_ENFORCE_ISOLATION_ENABLE) {
2712 if (adev->kfd.init_complete)
2713 sched_work = true;
2714 }
2715 mutex_unlock(&adev->enforce_isolation_mutex);
2716
2717 if (sched_work)
2718 amdgpu_gfx_kfd_sch_ctrl(adev, idx, false);
2719}
2720
2721/**
2722 * amdgpu_gfx_enforce_isolation_ring_end_use - End use of a ring with enforced isolation
2723 * @ring: Pointer to the amdgpu_ring structure
2724 *
2725 * Ring end_use helper implementation for gfx which serializes access to the
2726 * gfx IP between kernel submission IOCTLs and KFD user queues when isolation
2727 * enforcement is enabled. The kernel submission IOCTLs and KFD user queues
2728 * each get a time slice when both are active.
2729 */
2730void amdgpu_gfx_enforce_isolation_ring_end_use(struct amdgpu_ring *ring)
2731{
2732 struct amdgpu_device *adev = ring->adev;
2733 u32 idx;
2734 bool sched_work = false;
2735
2736 if (!adev->gfx.enable_cleaner_shader)
2737 return;
2738
2739 if (ring->xcp_id == AMDGPU_XCP_NO_PARTITION)
2740 idx = 0;
2741 else
2742 idx = ring->xcp_id;
2743
2744 if (idx >= MAX_XCP)
2745 return;
2746
2747 mutex_lock(&adev->enforce_isolation_mutex);
2748 if (adev->enforce_isolation[idx] == AMDGPU_ENFORCE_ISOLATION_ENABLE) {
2749 if (adev->kfd.init_complete)
2750 sched_work = true;
2751 }
2752 mutex_unlock(&adev->enforce_isolation_mutex);
2753
2754 if (sched_work)
2755 amdgpu_gfx_kfd_sch_ctrl(adev, idx, true);
2756}
2757
2758void amdgpu_gfx_profile_idle_work_handler(struct work_struct *work)
2759{
2760 struct amdgpu_device *adev =
2761 container_of(work, struct amdgpu_device, gfx.idle_work.work);
2762 enum PP_SMC_POWER_PROFILE profile;
2763 u32 i, fences = 0;
2764 int r;
2765
2766 if (adev->gfx.num_gfx_rings)
2767 profile = PP_SMC_POWER_PROFILE_FULLSCREEN3D;
2768 else
2769 profile = PP_SMC_POWER_PROFILE_COMPUTE;
2770
2771 for (i = 0; i < AMDGPU_MAX_GFX_RINGS; ++i)
2772 fences += amdgpu_fence_count_emitted(&adev->gfx.gfx_ring[i]);
2773 for (i = 0; i < (AMDGPU_MAX_COMPUTE_RINGS * AMDGPU_MAX_GC_INSTANCES); ++i)
2774 fences += amdgpu_fence_count_emitted(&adev->gfx.compute_ring[i]);
2775 if (!fences && !atomic_read(&adev->gfx.total_submission_cnt)) {
2776 mutex_lock(&adev->gfx.workload_profile_mutex);
2777 if (adev->gfx.workload_profile_active) {
2778 r = amdgpu_dpm_switch_power_profile(adev, profile, false);
2779 if (r)
2780 dev_warn(adev->dev, "(%d) failed to disable %s power profile mode\n", r,
2781 profile == PP_SMC_POWER_PROFILE_FULLSCREEN3D ?
2782 "fullscreen 3D" : "compute");
2783 adev->gfx.workload_profile_active = false;
2784 }
2785 mutex_unlock(&adev->gfx.workload_profile_mutex);
2786 } else {
2787 schedule_delayed_work(&adev->gfx.idle_work, GFX_PROFILE_IDLE_TIMEOUT);
2788 }
2789}
2790
2791void amdgpu_gfx_profile_ring_begin_use(struct amdgpu_ring *ring)
2792{
2793 struct amdgpu_device *adev = ring->adev;
2794 enum PP_SMC_POWER_PROFILE profile;
2795 int r;
2796
2797 if (amdgpu_dpm_is_overdrive_enabled(adev))
2798 return;
2799
2800 if (adev->gfx.num_gfx_rings)
2801 profile = PP_SMC_POWER_PROFILE_FULLSCREEN3D;
2802 else
2803 profile = PP_SMC_POWER_PROFILE_COMPUTE;
2804
2805 if (!atomic_fetch_inc(&adev->gfx.total_submission_cnt))
2806 cancel_delayed_work_sync(&adev->gfx.idle_work);
2807
2808 /* We can safely return early here because we've cancelled the
2809 * the delayed work so there is no one else to set it to false
2810 * and we don't care if someone else sets it to true.
2811 */
2812 if (adev->gfx.workload_profile_active)
2813 return;
2814
2815 mutex_lock(&adev->gfx.workload_profile_mutex);
2816 if (!adev->gfx.workload_profile_active) {
2817 r = amdgpu_dpm_switch_power_profile(adev, profile, true);
2818 if (r)
2819 dev_warn(adev->dev, "(%d) failed to disable %s power profile mode\n", r,
2820 profile == PP_SMC_POWER_PROFILE_FULLSCREEN3D ?
2821 "fullscreen 3D" : "compute");
2822 adev->gfx.workload_profile_active = true;
2823 }
2824 mutex_unlock(&adev->gfx.workload_profile_mutex);
2825}
2826
2827void amdgpu_gfx_profile_ring_end_use(struct amdgpu_ring *ring)
2828{
2829 struct amdgpu_device *adev = ring->adev;
2830
2831 if (amdgpu_dpm_is_overdrive_enabled(adev))
2832 return;
2833
2834 if (atomic_dec_and_test(&ring->adev->gfx.total_submission_cnt))
2835 schedule_delayed_work(&ring->adev->gfx.idle_work,
2836 GFX_PROFILE_IDLE_TIMEOUT);
2837}
2838
2839/**
2840 * amdgpu_gfx_csb_preamble_start - Set CSB preamble start
2841 *
2842 * @buffer: This is an output variable that gets the PACKET3 preamble setup.
2843 *
2844 * Return:
2845 * return the latest index.
2846 */
2847u32 amdgpu_gfx_csb_preamble_start(u32 *buffer)
2848{
2849 u32 count = 0;
2850
2851 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
2852 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
2853
2854 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CONTEXT_CONTROL, 1));
2855 buffer[count++] = cpu_to_le32(0x80000000);
2856 buffer[count++] = cpu_to_le32(0x80000000);
2857
2858 return count;
2859}
2860
2861/**
2862 * amdgpu_gfx_csb_data_parser - Parser CS data
2863 *
2864 * @adev: amdgpu_device pointer used to get the CS data and other gfx info.
2865 * @buffer: This is an output variable that gets the PACKET3 preamble end.
2866 * @count: Index to start set the preemble end.
2867 *
2868 * Return:
2869 * return the latest index.
2870 */
2871u32 amdgpu_gfx_csb_data_parser(struct amdgpu_device *adev, u32 *buffer, u32 count)
2872{
2873 const struct cs_section_def *sect = NULL;
2874 const struct cs_extent_def *ext = NULL;
2875 u32 i;
2876
2877 for (sect = adev->gfx.rlc.cs_data; sect->section != NULL; ++sect) {
2878 for (ext = sect->section; ext->extent != NULL; ++ext) {
2879 if (sect->id == SECT_CONTEXT) {
2880 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count));
2881 buffer[count++] = cpu_to_le32(ext->reg_index - PACKET3_SET_CONTEXT_REG_START);
2882
2883 for (i = 0; i < ext->reg_count; i++)
2884 buffer[count++] = cpu_to_le32(ext->extent[i]);
2885 }
2886 }
2887 }
2888
2889 return count;
2890}
2891
2892/**
2893 * amdgpu_gfx_csb_preamble_end - Set CSB preamble end
2894 *
2895 * @buffer: This is an output variable that gets the PACKET3 preamble end.
2896 * @count: Index to start set the preemble end.
2897 */
2898void amdgpu_gfx_csb_preamble_end(u32 *buffer, u32 count)
2899{
2900 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
2901 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_END_CLEAR_STATE);
2902
2903 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CLEAR_STATE, 0));
2904 buffer[count++] = cpu_to_le32(0);
2905}
2906
2907/*
2908 * debugfs for to enable/disable gfx job submission to specific core.
2909 */
2910#if defined(CONFIG_DEBUG_FS)
2911static int amdgpu_debugfs_gfx_sched_mask_set(void *data, u64 val)
2912{
2913 struct amdgpu_device *adev = (struct amdgpu_device *)data;
2914 u32 i;
2915 u64 mask = 0;
2916 struct amdgpu_ring *ring;
2917
2918 if (!adev)
2919 return -ENODEV;
2920
2921 mask = (1ULL << adev->gfx.num_gfx_rings) - 1;
2922 if ((val & mask) == 0)
2923 return -EINVAL;
2924
2925 for (i = 0; i < adev->gfx.num_gfx_rings; ++i) {
2926 ring = &adev->gfx.gfx_ring[i];
2927 if (val & (1 << i))
2928 ring->sched.ready = true;
2929 else
2930 ring->sched.ready = false;
2931 }
2932 /* publish sched.ready flag update effective immediately across smp */
2933 smp_rmb();
2934 return 0;
2935}
2936
2937static int amdgpu_debugfs_gfx_sched_mask_get(void *data, u64 *val)
2938{
2939 struct amdgpu_device *adev = (struct amdgpu_device *)data;
2940 u32 i;
2941 u64 mask = 0;
2942 struct amdgpu_ring *ring;
2943
2944 if (!adev)
2945 return -ENODEV;
2946 for (i = 0; i < adev->gfx.num_gfx_rings; ++i) {
2947 ring = &adev->gfx.gfx_ring[i];
2948 if (ring->sched.ready)
2949 mask |= 1ULL << i;
2950 }
2951
2952 *val = mask;
2953 return 0;
2954}
2955
2956DEFINE_DEBUGFS_ATTRIBUTE(amdgpu_debugfs_gfx_sched_mask_fops,
2957 amdgpu_debugfs_gfx_sched_mask_get,
2958 amdgpu_debugfs_gfx_sched_mask_set, "%llx\n");
2959
2960#endif
2961
2962void amdgpu_debugfs_gfx_sched_mask_init(struct amdgpu_device *adev)
2963{
2964#if defined(CONFIG_DEBUG_FS)
2965 struct drm_minor *minor = adev_to_drm(adev)->primary;
2966 struct dentry *root = minor->debugfs_root;
2967 char name[32];
2968
2969 if (!(adev->gfx.num_gfx_rings > 1))
2970 return;
2971 sprintf(name, "amdgpu_gfx_sched_mask");
2972 debugfs_create_file(name, 0600, root, adev,
2973 &amdgpu_debugfs_gfx_sched_mask_fops);
2974#endif
2975}
2976
2977/*
2978 * debugfs for to enable/disable compute job submission to specific core.
2979 */
2980#if defined(CONFIG_DEBUG_FS)
2981static int amdgpu_debugfs_compute_sched_mask_set(void *data, u64 val)
2982{
2983 struct amdgpu_device *adev = (struct amdgpu_device *)data;
2984 u32 i;
2985 u64 mask = 0;
2986 struct amdgpu_ring *ring;
2987
2988 if (!adev)
2989 return -ENODEV;
2990
2991 mask = (1ULL << adev->gfx.num_compute_rings) - 1;
2992 if ((val & mask) == 0)
2993 return -EINVAL;
2994
2995 for (i = 0; i < adev->gfx.num_compute_rings; ++i) {
2996 ring = &adev->gfx.compute_ring[i];
2997 if (val & (1 << i))
2998 ring->sched.ready = true;
2999 else
3000 ring->sched.ready = false;
3001 }
3002
3003 /* publish sched.ready flag update effective immediately across smp */
3004 smp_rmb();
3005 return 0;
3006}
3007
3008static int amdgpu_debugfs_compute_sched_mask_get(void *data, u64 *val)
3009{
3010 struct amdgpu_device *adev = (struct amdgpu_device *)data;
3011 u32 i;
3012 u64 mask = 0;
3013 struct amdgpu_ring *ring;
3014
3015 if (!adev)
3016 return -ENODEV;
3017 for (i = 0; i < adev->gfx.num_compute_rings; ++i) {
3018 ring = &adev->gfx.compute_ring[i];
3019 if (ring->sched.ready)
3020 mask |= 1ULL << i;
3021 }
3022
3023 *val = mask;
3024 return 0;
3025}
3026
3027DEFINE_DEBUGFS_ATTRIBUTE(amdgpu_debugfs_compute_sched_mask_fops,
3028 amdgpu_debugfs_compute_sched_mask_get,
3029 amdgpu_debugfs_compute_sched_mask_set, "%llx\n");
3030
3031#endif
3032
3033void amdgpu_debugfs_compute_sched_mask_init(struct amdgpu_device *adev)
3034{
3035#if defined(CONFIG_DEBUG_FS)
3036 struct drm_minor *minor = adev_to_drm(adev)->primary;
3037 struct dentry *root = minor->debugfs_root;
3038 char name[32];
3039
3040 if (!(adev->gfx.num_compute_rings > 1))
3041 return;
3042 sprintf(name, "amdgpu_compute_sched_mask");
3043 debugfs_create_file(name, 0600, root, adev,
3044 &amdgpu_debugfs_compute_sched_mask_fops);
3045#endif
3046}
3047
3048int amdgpu_gfx_ring_preempt_ib(struct amdgpu_ring *ring)
3049{
3050 struct amdgpu_device *adev = ring->adev;
3051 struct amdgpu_kiq *kiq = &adev->gfx.kiq[0];
3052 struct amdgpu_ring *kiq_ring = &kiq->ring;
3053 unsigned long flags;
3054 int i;
3055
3056 if (adev->enable_mes)
3057 return -EINVAL;
3058
3059 if (!kiq->pmf || !kiq->pmf->kiq_unmap_queues)
3060 return -EINVAL;
3061
3062 spin_lock_irqsave(&kiq->ring_lock, flags);
3063
3064 if (amdgpu_ring_alloc(kiq_ring, kiq->pmf->unmap_queues_size)) {
3065 spin_unlock_irqrestore(&kiq->ring_lock, flags);
3066 return -ENOMEM;
3067 }
3068
3069 /* assert preemption condition */
3070 amdgpu_ring_set_preempt_cond_exec(ring, false);
3071
3072 /* assert IB preemption, emit the trailing fence */
3073 kiq->pmf->kiq_unmap_queues(kiq_ring, ring, PREEMPT_QUEUES_NO_UNMAP,
3074 ring->trail_fence_gpu_addr,
3075 ++ring->trail_seq);
3076 amdgpu_ring_commit(kiq_ring);
3077
3078 spin_unlock_irqrestore(&kiq->ring_lock, flags);
3079
3080 /* poll the trailing fence */
3081 for (i = 0; i < adev->usec_timeout; i++) {
3082 if (ring->trail_seq ==
3083 le32_to_cpu(*(ring->trail_fence_cpu_addr)))
3084 break;
3085 udelay(1);
3086 }
3087
3088 /* deassert preemption condition */
3089 amdgpu_ring_set_preempt_cond_exec(ring, true);
3090
3091 if (i >= adev->usec_timeout) {
3092 DRM_ERROR("ring %d failed to preempt ib\n", ring->idx);
3093 return -EINVAL;
3094 }
3095
3096 return 0;
3097}
3098