projects/linux·blob master · refs

1050 lines · 28.6 KiB

1/*
2 * Copyright 2009 Jerome Glisse.
3 * All Rights Reserved.
4 *
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the
7 * "Software"), to deal in the Software without restriction, including
8 * without limitation the rights to use, copy, modify, merge, publish,
9 * distribute, sub license, and/or sell copies of the Software, and to
10 * permit persons to whom the Software is furnished to do so, subject to
11 * the following conditions:
12 *
13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
16 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
17 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
18 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
19 * USE OR OTHER DEALINGS IN THE SOFTWARE.
20 *
21 * The above copyright notice and this permission notice (including the
22 * next paragraph) shall be included in all copies or substantial portions
23 * of the Software.
24 *
25 */
26/*
27 * Authors:
28 * Jerome Glisse <[email protected]>
29 * Dave Airlie
30 */
31#include <linux/seq_file.h>
32#include <linux/atomic.h>
33#include <linux/wait.h>
34#include <linux/kref.h>
35#include <linux/slab.h>
36#include <linux/firmware.h>
37#include <linux/pm_runtime.h>
38
39#include <drm/drm_drv.h>
40#include "amdgpu.h"
41#include "amdgpu_trace.h"
42#include "amdgpu_reset.h"
43
44/*
45 * Cast helper
46 */
47static const struct dma_fence_ops amdgpu_fence_ops;
48static inline struct amdgpu_fence *to_amdgpu_fence(struct dma_fence *f)
49{
50 struct amdgpu_fence *__f = container_of(f, struct amdgpu_fence, base);
51
52 return __f;
53}
54
55/**
56 * amdgpu_fence_write - write a fence value
57 *
58 * @ring: ring the fence is associated with
59 * @seq: sequence number to write
60 *
61 * Writes a fence value to memory (all asics).
62 */
63static void amdgpu_fence_write(struct amdgpu_ring *ring, u32 seq)
64{
65 struct amdgpu_fence_driver *drv = &ring->fence_drv;
66
67 if (drv->cpu_addr)
68 *drv->cpu_addr = cpu_to_le32(seq);
69}
70
71/**
72 * amdgpu_fence_read - read a fence value
73 *
74 * @ring: ring the fence is associated with
75 *
76 * Reads a fence value from memory (all asics).
77 * Returns the value of the fence read from memory.
78 */
79static u32 amdgpu_fence_read(struct amdgpu_ring *ring)
80{
81 struct amdgpu_fence_driver *drv = &ring->fence_drv;
82 u32 seq = 0;
83
84 if (drv->cpu_addr)
85 seq = le32_to_cpu(*drv->cpu_addr);
86 else
87 seq = atomic_read(&drv->last_seq);
88
89 return seq;
90}
91
92/**
93 * amdgpu_fence_emit - emit a fence on the requested ring
94 *
95 * @ring: ring the fence is associated with
96 * @af: amdgpu fence input
97 * @flags: flags to pass into the subordinate .emit_fence() call
98 *
99 * Emits a fence command on the requested ring (all asics).
100 */
101void amdgpu_fence_emit(struct amdgpu_ring *ring, struct amdgpu_fence *af,
102 unsigned int flags)
103{
104 struct amdgpu_device *adev = ring->adev;
105 struct dma_fence *fence;
106 struct dma_fence __rcu **ptr;
107 uint32_t seq;
108
109 fence = &af->base;
110 af->ring = ring;
111
112 seq = ++ring->fence_drv.sync_seq;
113 dma_fence_init(fence, &amdgpu_fence_ops,
114 &ring->fence_drv.lock,
115 adev->fence_context + ring->idx, seq);
116
117 amdgpu_ring_emit_fence(ring, ring->fence_drv.gpu_addr,
118 seq, flags | AMDGPU_FENCE_FLAG_INT);
119
120 pm_runtime_get_noresume(adev_to_drm(adev)->dev);
121 ptr = &ring->fence_drv.fences[seq & ring->fence_drv.num_fences_mask];
122 if (unlikely(rcu_dereference_protected(*ptr, 1))) {
123 struct dma_fence *old;
124
125 rcu_read_lock();
126 old = dma_fence_get_rcu_safe(ptr);
127 rcu_read_unlock();
128
129 if (old) {
130 /*
131 * dma_fence_wait(old, false) is not interruptible.
132 * It will not return an error in this case.
133 * So we can safely ignore the return value.
134 */
135 dma_fence_wait(old, false);
136 dma_fence_put(old);
137 }
138 }
139
140 to_amdgpu_fence(fence)->start_timestamp = ktime_get();
141
142 /* This function can't be called concurrently anyway, otherwise
143 * emitting the fence would mess up the hardware ring buffer.
144 */
145 rcu_assign_pointer(*ptr, dma_fence_get(fence));
146}
147
148/**
149 * amdgpu_fence_emit_polling - emit a fence on the requeste ring
150 *
151 * @ring: ring the fence is associated with
152 * @s: resulting sequence number
153 * @timeout: the timeout for waiting in usecs
154 *
155 * Emits a fence command on the requested ring (all asics).
156 * Used For polling fence.
157 * Returns 0 on success, -ENOMEM on failure.
158 */
159int amdgpu_fence_emit_polling(struct amdgpu_ring *ring, uint32_t *s,
160 uint32_t timeout)
161{
162 uint32_t seq;
163 signed long r;
164
165 if (!s)
166 return -EINVAL;
167
168 seq = ++ring->fence_drv.sync_seq;
169 r = amdgpu_fence_wait_polling(ring,
170 seq - ring->fence_drv.num_fences_mask,
171 timeout);
172 if (r < 1)
173 return -ETIMEDOUT;
174
175 amdgpu_ring_emit_fence(ring, ring->fence_drv.gpu_addr,
176 seq, 0);
177
178 *s = seq;
179
180 return 0;
181}
182
183/**
184 * amdgpu_fence_schedule_fallback - schedule fallback check
185 *
186 * @ring: pointer to struct amdgpu_ring
187 *
188 * Start a timer as fallback to our interrupts.
189 */
190static void amdgpu_fence_schedule_fallback(struct amdgpu_ring *ring)
191{
192 mod_timer(&ring->fence_drv.fallback_timer,
193 jiffies + AMDGPU_FENCE_JIFFIES_TIMEOUT);
194}
195
196/**
197 * amdgpu_fence_process - check for fence activity
198 *
199 * @ring: pointer to struct amdgpu_ring
200 *
201 * Checks the current fence value and calculates the last
202 * signalled fence value. Wakes the fence queue if the
203 * sequence number has increased.
204 *
205 * Returns true if fence was processed
206 */
207bool amdgpu_fence_process(struct amdgpu_ring *ring)
208{
209 struct amdgpu_fence_driver *drv = &ring->fence_drv;
210 struct amdgpu_device *adev = ring->adev;
211 uint32_t seq, last_seq;
212
213 do {
214 last_seq = atomic_read(&ring->fence_drv.last_seq);
215 seq = amdgpu_fence_read(ring);
216
217 } while (atomic_cmpxchg(&drv->last_seq, last_seq, seq) != last_seq);
218
219 if (timer_delete(&ring->fence_drv.fallback_timer) &&
220 seq != ring->fence_drv.sync_seq)
221 amdgpu_fence_schedule_fallback(ring);
222
223 if (unlikely(seq == last_seq))
224 return false;
225
226 last_seq &= drv->num_fences_mask;
227 seq &= drv->num_fences_mask;
228
229 do {
230 struct dma_fence *fence, **ptr;
231
232 ++last_seq;
233 last_seq &= drv->num_fences_mask;
234 ptr = &drv->fences[last_seq];
235
236 /* There is always exactly one thread signaling this fence slot */
237 fence = rcu_dereference_protected(*ptr, 1);
238 RCU_INIT_POINTER(*ptr, NULL);
239
240 if (!fence)
241 continue;
242
243 dma_fence_signal(fence);
244 dma_fence_put(fence);
245 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
246 } while (last_seq != seq);
247
248 return true;
249}
250
251/**
252 * amdgpu_fence_fallback - fallback for hardware interrupts
253 *
254 * @t: timer context used to obtain the pointer to ring structure
255 *
256 * Checks for fence activity.
257 */
258static void amdgpu_fence_fallback(struct timer_list *t)
259{
260 struct amdgpu_ring *ring = timer_container_of(ring, t,
261 fence_drv.fallback_timer);
262
263 if (amdgpu_fence_process(ring))
264 dev_warn(ring->adev->dev,
265 "Fence fallback timer expired on ring %s\n",
266 ring->name);
267}
268
269/**
270 * amdgpu_fence_wait_empty - wait for all fences to signal
271 *
272 * @ring: ring index the fence is associated with
273 *
274 * Wait for all fences on the requested ring to signal (all asics).
275 * Returns 0 if the fences have passed, error for all other cases.
276 */
277int amdgpu_fence_wait_empty(struct amdgpu_ring *ring)
278{
279 uint64_t seq = READ_ONCE(ring->fence_drv.sync_seq);
280 struct dma_fence *fence, **ptr;
281 int r;
282
283 if (!seq)
284 return 0;
285
286 ptr = &ring->fence_drv.fences[seq & ring->fence_drv.num_fences_mask];
287 rcu_read_lock();
288 fence = rcu_dereference(*ptr);
289 if (!fence || !dma_fence_get_rcu(fence)) {
290 rcu_read_unlock();
291 return 0;
292 }
293 rcu_read_unlock();
294
295 r = dma_fence_wait(fence, false);
296 dma_fence_put(fence);
297 return r;
298}
299
300/**
301 * amdgpu_fence_wait_polling - busy wait for givn sequence number
302 *
303 * @ring: ring index the fence is associated with
304 * @wait_seq: sequence number to wait
305 * @timeout: the timeout for waiting in usecs
306 *
307 * Wait for all fences on the requested ring to signal (all asics).
308 * Returns left time if no timeout, 0 or minus if timeout.
309 */
310signed long amdgpu_fence_wait_polling(struct amdgpu_ring *ring,
311 uint32_t wait_seq,
312 signed long timeout)
313{
314
315 while ((int32_t)(wait_seq - amdgpu_fence_read(ring)) > 0 && timeout > 0) {
316 udelay(2);
317 timeout -= 2;
318 }
319 return timeout > 0 ? timeout : 0;
320}
321/**
322 * amdgpu_fence_count_emitted - get the count of emitted fences
323 *
324 * @ring: ring the fence is associated with
325 *
326 * Get the number of fences emitted on the requested ring (all asics).
327 * Returns the number of emitted fences on the ring. Used by the
328 * dynpm code to ring track activity.
329 */
330unsigned int amdgpu_fence_count_emitted(struct amdgpu_ring *ring)
331{
332 uint64_t emitted;
333
334 /* We are not protected by ring lock when reading the last sequence
335 * but it's ok to report slightly wrong fence count here.
336 */
337 emitted = 0x100000000ull;
338 emitted -= atomic_read(&ring->fence_drv.last_seq);
339 emitted += READ_ONCE(ring->fence_drv.sync_seq);
340 return lower_32_bits(emitted);
341}
342
343/**
344 * amdgpu_fence_last_unsignaled_time_us - the time fence emitted until now
345 * @ring: ring the fence is associated with
346 *
347 * Find the earliest fence unsignaled until now, calculate the time delta
348 * between the time fence emitted and now.
349 */
350u64 amdgpu_fence_last_unsignaled_time_us(struct amdgpu_ring *ring)
351{
352 struct amdgpu_fence_driver *drv = &ring->fence_drv;
353 struct dma_fence *fence;
354 uint32_t last_seq, sync_seq;
355
356 last_seq = atomic_read(&ring->fence_drv.last_seq);
357 sync_seq = READ_ONCE(ring->fence_drv.sync_seq);
358 if (last_seq == sync_seq)
359 return 0;
360
361 ++last_seq;
362 last_seq &= drv->num_fences_mask;
363 fence = drv->fences[last_seq];
364 if (!fence)
365 return 0;
366
367 return ktime_us_delta(ktime_get(),
368 to_amdgpu_fence(fence)->start_timestamp);
369}
370
371/**
372 * amdgpu_fence_update_start_timestamp - update the timestamp of the fence
373 * @ring: ring the fence is associated with
374 * @seq: the fence seq number to update.
375 * @timestamp: the start timestamp to update.
376 *
377 * The function called at the time the fence and related ib is about to
378 * resubmit to gpu in MCBP scenario. Thus we do not consider race condition
379 * with amdgpu_fence_process to modify the same fence.
380 */
381void amdgpu_fence_update_start_timestamp(struct amdgpu_ring *ring, uint32_t seq, ktime_t timestamp)
382{
383 struct amdgpu_fence_driver *drv = &ring->fence_drv;
384 struct dma_fence *fence;
385
386 seq &= drv->num_fences_mask;
387 fence = drv->fences[seq];
388 if (!fence)
389 return;
390
391 to_amdgpu_fence(fence)->start_timestamp = timestamp;
392}
393
394/**
395 * amdgpu_fence_driver_start_ring - make the fence driver
396 * ready for use on the requested ring.
397 *
398 * @ring: ring to start the fence driver on
399 * @irq_src: interrupt source to use for this ring
400 * @irq_type: interrupt type to use for this ring
401 *
402 * Make the fence driver ready for processing (all asics).
403 * Not all asics have all rings, so each asic will only
404 * start the fence driver on the rings it has.
405 * Returns 0 for success, errors for failure.
406 */
407int amdgpu_fence_driver_start_ring(struct amdgpu_ring *ring,
408 struct amdgpu_irq_src *irq_src,
409 unsigned int irq_type)
410{
411 struct amdgpu_device *adev = ring->adev;
412 uint64_t index;
413
414 if (ring->funcs->type != AMDGPU_RING_TYPE_UVD) {
415 ring->fence_drv.cpu_addr = ring->fence_cpu_addr;
416 ring->fence_drv.gpu_addr = ring->fence_gpu_addr;
417 } else {
418 /* put fence directly behind firmware */
419 index = ALIGN(adev->uvd.fw->size, 8);
420 ring->fence_drv.cpu_addr = adev->uvd.inst[ring->me].cpu_addr + index;
421 ring->fence_drv.gpu_addr = adev->uvd.inst[ring->me].gpu_addr + index;
422 }
423 amdgpu_fence_write(ring, atomic_read(&ring->fence_drv.last_seq));
424
425 ring->fence_drv.irq_src = irq_src;
426 ring->fence_drv.irq_type = irq_type;
427 ring->fence_drv.initialized = true;
428
429 DRM_DEV_DEBUG(adev->dev, "fence driver on ring %s use gpu addr 0x%016llx\n",
430 ring->name, ring->fence_drv.gpu_addr);
431 return 0;
432}
433
434/**
435 * amdgpu_fence_driver_init_ring - init the fence driver
436 * for the requested ring.
437 *
438 * @ring: ring to init the fence driver on
439 *
440 * Init the fence driver for the requested ring (all asics).
441 * Helper function for amdgpu_fence_driver_sw_init().
442 */
443int amdgpu_fence_driver_init_ring(struct amdgpu_ring *ring)
444{
445 struct amdgpu_device *adev = ring->adev;
446
447 if (!adev)
448 return -EINVAL;
449
450 if (!is_power_of_2(ring->num_hw_submission))
451 return -EINVAL;
452
453 ring->fence_drv.cpu_addr = NULL;
454 ring->fence_drv.gpu_addr = 0;
455 ring->fence_drv.sync_seq = 0;
456 atomic_set(&ring->fence_drv.last_seq, 0);
457 ring->fence_drv.initialized = false;
458
459 timer_setup(&ring->fence_drv.fallback_timer, amdgpu_fence_fallback, 0);
460
461 ring->fence_drv.num_fences_mask = ring->num_hw_submission * 2 - 1;
462 spin_lock_init(&ring->fence_drv.lock);
463 ring->fence_drv.fences = kcalloc(ring->num_hw_submission * 2, sizeof(void *),
464 GFP_KERNEL);
465
466 if (!ring->fence_drv.fences)
467 return -ENOMEM;
468
469 return 0;
470}
471
472/**
473 * amdgpu_fence_driver_sw_init - init the fence driver
474 * for all possible rings.
475 *
476 * @adev: amdgpu device pointer
477 *
478 * Init the fence driver for all possible rings (all asics).
479 * Not all asics have all rings, so each asic will only
480 * start the fence driver on the rings it has using
481 * amdgpu_fence_driver_start_ring().
482 * Returns 0 for success.
483 */
484int amdgpu_fence_driver_sw_init(struct amdgpu_device *adev)
485{
486 return 0;
487}
488
489/**
490 * amdgpu_fence_need_ring_interrupt_restore - helper function to check whether
491 * fence driver interrupts need to be restored.
492 *
493 * @ring: ring that to be checked
494 *
495 * Interrupts for rings that belong to GFX IP don't need to be restored
496 * when the target power state is s0ix.
497 *
498 * Return true if need to restore interrupts, false otherwise.
499 */
500static bool amdgpu_fence_need_ring_interrupt_restore(struct amdgpu_ring *ring)
501{
502 struct amdgpu_device *adev = ring->adev;
503 bool is_gfx_power_domain = false;
504
505 switch (ring->funcs->type) {
506 case AMDGPU_RING_TYPE_SDMA:
507 /* SDMA 5.x+ is part of GFX power domain so it's covered by GFXOFF */
508 if (amdgpu_ip_version(adev, SDMA0_HWIP, 0) >=
509 IP_VERSION(5, 0, 0))
510 is_gfx_power_domain = true;
511 break;
512 case AMDGPU_RING_TYPE_GFX:
513 case AMDGPU_RING_TYPE_COMPUTE:
514 case AMDGPU_RING_TYPE_KIQ:
515 case AMDGPU_RING_TYPE_MES:
516 is_gfx_power_domain = true;
517 break;
518 default:
519 break;
520 }
521
522 return !(adev->in_s0ix && is_gfx_power_domain);
523}
524
525/**
526 * amdgpu_fence_driver_hw_fini - tear down the fence driver
527 * for all possible rings.
528 *
529 * @adev: amdgpu device pointer
530 *
531 * Tear down the fence driver for all possible rings (all asics).
532 */
533void amdgpu_fence_driver_hw_fini(struct amdgpu_device *adev)
534{
535 int i, r;
536
537 for (i = 0; i < AMDGPU_MAX_RINGS; i++) {
538 struct amdgpu_ring *ring = adev->rings[i];
539
540 if (!ring || !ring->fence_drv.initialized)
541 continue;
542
543 /* You can't wait for HW to signal if it's gone */
544 if (!drm_dev_is_unplugged(adev_to_drm(adev)))
545 r = amdgpu_fence_wait_empty(ring);
546 else
547 r = -ENODEV;
548 /* no need to trigger GPU reset as we are unloading */
549 if (r)
550 amdgpu_fence_driver_force_completion(ring, NULL);
551
552 if (!drm_dev_is_unplugged(adev_to_drm(adev)) &&
553 ring->fence_drv.irq_src &&
554 amdgpu_fence_need_ring_interrupt_restore(ring))
555 amdgpu_irq_put(adev, ring->fence_drv.irq_src,
556 ring->fence_drv.irq_type);
557
558 timer_delete_sync(&ring->fence_drv.fallback_timer);
559 }
560}
561
562/* Will either stop and flush handlers for amdgpu interrupt or reanble it */
563void amdgpu_fence_driver_isr_toggle(struct amdgpu_device *adev, bool stop)
564{
565 int i;
566
567 for (i = 0; i < AMDGPU_MAX_RINGS; i++) {
568 struct amdgpu_ring *ring = adev->rings[i];
569
570 if (!ring || !ring->fence_drv.initialized || !ring->fence_drv.irq_src)
571 continue;
572
573 if (stop)
574 disable_irq(adev->irq.irq);
575 else
576 enable_irq(adev->irq.irq);
577 }
578}
579
580void amdgpu_fence_driver_sw_fini(struct amdgpu_device *adev)
581{
582 unsigned int i, j;
583
584 for (i = 0; i < AMDGPU_MAX_RINGS; i++) {
585 struct amdgpu_ring *ring = adev->rings[i];
586
587 if (!ring || !ring->fence_drv.initialized)
588 continue;
589
590 /*
591 * Notice we check for sched.ops since there's some
592 * override on the meaning of sched.ready by amdgpu.
593 * The natural check would be sched.ready, which is
594 * set as drm_sched_init() finishes...
595 */
596 if (ring->sched.ops)
597 drm_sched_fini(&ring->sched);
598
599 for (j = 0; j <= ring->fence_drv.num_fences_mask; ++j)
600 dma_fence_put(ring->fence_drv.fences[j]);
601 kfree(ring->fence_drv.fences);
602 ring->fence_drv.fences = NULL;
603 ring->fence_drv.initialized = false;
604 }
605}
606
607/**
608 * amdgpu_fence_driver_hw_init - enable the fence driver
609 * for all possible rings.
610 *
611 * @adev: amdgpu device pointer
612 *
613 * Enable the fence driver for all possible rings (all asics).
614 * Not all asics have all rings, so each asic will only
615 * start the fence driver on the rings it has using
616 * amdgpu_fence_driver_start_ring().
617 * Returns 0 for success.
618 */
619void amdgpu_fence_driver_hw_init(struct amdgpu_device *adev)
620{
621 int i;
622
623 for (i = 0; i < AMDGPU_MAX_RINGS; i++) {
624 struct amdgpu_ring *ring = adev->rings[i];
625
626 if (!ring || !ring->fence_drv.initialized)
627 continue;
628
629 /* enable the interrupt */
630 if (ring->fence_drv.irq_src &&
631 amdgpu_fence_need_ring_interrupt_restore(ring))
632 amdgpu_irq_get(adev, ring->fence_drv.irq_src,
633 ring->fence_drv.irq_type);
634 }
635}
636
637/**
638 * amdgpu_fence_driver_set_error - set error code on fences
639 * @ring: the ring which contains the fences
640 * @error: the error code to set
641 *
642 * Set an error code to all the fences pending on the ring.
643 */
644void amdgpu_fence_driver_set_error(struct amdgpu_ring *ring, int error)
645{
646 struct amdgpu_fence_driver *drv = &ring->fence_drv;
647 unsigned long flags;
648
649 spin_lock_irqsave(&drv->lock, flags);
650 for (unsigned int i = 0; i <= drv->num_fences_mask; ++i) {
651 struct dma_fence *fence;
652
653 fence = rcu_dereference_protected(drv->fences[i],
654 lockdep_is_held(&drv->lock));
655 if (fence && !dma_fence_is_signaled_locked(fence))
656 dma_fence_set_error(fence, error);
657 }
658 spin_unlock_irqrestore(&drv->lock, flags);
659}
660
661/**
662 * amdgpu_fence_driver_force_completion - force signal latest fence of ring
663 *
664 * @ring: fence of the ring to signal
665 * @timedout_fence: fence of the timedout job
666 *
667 */
668void amdgpu_fence_driver_force_completion(struct amdgpu_ring *ring,
669 struct dma_fence *timedout_fence)
670{
671 struct amdgpu_fence_driver *drv = &ring->fence_drv;
672 unsigned long flags;
673
674 spin_lock_irqsave(&drv->lock, flags);
675 for (unsigned int i = 0; i <= drv->num_fences_mask; ++i) {
676 struct dma_fence *fence;
677
678 fence = rcu_dereference_protected(drv->fences[i],
679 lockdep_is_held(&drv->lock));
680 if (fence && !dma_fence_is_signaled_locked(fence)) {
681 if (fence == timedout_fence)
682 dma_fence_set_error(fence, -ETIME);
683 else
684 dma_fence_set_error(fence, -ECANCELED);
685 }
686 }
687 spin_unlock_irqrestore(&drv->lock, flags);
688
689 amdgpu_fence_write(ring, ring->fence_drv.sync_seq);
690 amdgpu_fence_process(ring);
691}
692
693/*
694 * Kernel queue reset handling
695 *
696 * The driver can reset individual queues for most engines, but those queues
697 * may contain work from multiple contexts. Resetting the queue will reset
698 * lose all of that state. In order to minimize the collateral damage, the
699 * driver will save the ring contents which are not associated with the guilty
700 * context prior to resetting the queue. After resetting the queue the queue
701 * contents from the other contexts is re-emitted to the rings so that it can
702 * be processed by the engine. To handle this, we save the queue's write
703 * pointer (wptr) in the fences associated with each context. If we get a
704 * queue timeout, we can then use the wptrs from the fences to determine
705 * which data needs to be saved out of the queue's ring buffer.
706 */
707
708/**
709 * amdgpu_ring_set_fence_errors_and_reemit - Set dma_fence errors and reemit
710 *
711 * @ring: the ring to operate on
712 * @guilty_fence: fence of the ring to update
713 *
714 */
715void amdgpu_ring_set_fence_errors_and_reemit(struct amdgpu_ring *ring,
716 struct amdgpu_fence *guilty_fence)
717{
718 struct dma_fence *unprocessed;
719 struct dma_fence __rcu **ptr;
720 struct amdgpu_fence *fence;
721 unsigned long flags;
722 u32 seq, last_seq;
723 unsigned int i;
724 bool is_guilty_fence;
725 bool is_guilty_context;
726
727 last_seq = amdgpu_fence_read(ring);
728 seq = ring->fence_drv.sync_seq;
729
730 /* Hardware completion can precede the interrupt which signals fences. */
731 if (last_seq == seq) {
732 amdgpu_fence_process(ring);
733 return;
734 }
735
736 /* If there is nothing to reemit, return early and set an error on the fence
737 * if applicable. If all of the fences are siganlled, this will be a nop.
738 * if there are still fences and ring_backup_entries_to_copy is 0, then
739 * we are skipping it on purpose.
740 */
741 if (!ring->ring_backup_entries_to_copy) {
742 if (guilty_fence)
743 amdgpu_fence_driver_force_completion(ring,
744 &guilty_fence->base);
745 return;
746 }
747 ring->reemit = true;
748 amdgpu_ring_alloc(ring, ring->ring_backup_entries_to_copy);
749 spin_lock_irqsave(&ring->fence_drv.lock, flags);
750 do {
751 last_seq++;
752
753 ptr = &ring->fence_drv.fences[last_seq & ring->fence_drv.num_fences_mask];
754 rcu_read_lock();
755 unprocessed = rcu_dereference(*ptr);
756
757 if (unprocessed && !dma_fence_is_signaled_locked(unprocessed)) {
758 fence = container_of(unprocessed, struct amdgpu_fence, base);
759 is_guilty_fence = fence == guilty_fence;
760 is_guilty_context = guilty_fence ?
761 (fence->context == guilty_fence->context) : false;
762
763 /* mark all fences from the guilty context with an error */
764 if (is_guilty_fence)
765 dma_fence_set_error(&fence->base, -ETIME);
766 else if (is_guilty_context)
767 dma_fence_set_error(&fence->base, -ECANCELED);
768
769 /* reemit the packet stream and update wptrs */
770 fence->ib_wptr = ring->wptr;
771 for (i = 0; i < fence->ib_dw_size; i++) {
772 /* Skip the IB(s) for the guilty context. */
773 if (is_guilty_context &&
774 i >= fence->skip_ib_dw_start_offset &&
775 i < fence->skip_ib_dw_end_offset)
776 amdgpu_ring_write(ring, ring->funcs->nop);
777 else
778 amdgpu_ring_write(ring,
779 ring->ring_backup[fence->backup_idx + i]);
780 }
781 }
782 rcu_read_unlock();
783 } while (last_seq != seq);
784 spin_unlock_irqrestore(&ring->fence_drv.lock, flags);
785 amdgpu_ring_commit(ring);
786 ring->reemit = false;
787}
788
789static void amdgpu_ring_backup_unprocessed_command(struct amdgpu_ring *ring,
790 struct amdgpu_fence *af)
791{
792 unsigned int first_idx = af->ib_wptr & ring->buf_mask;
793 unsigned int dw_size = af->ib_dw_size;
794 unsigned int i;
795
796 af->backup_idx = ring->ring_backup_entries_to_copy;
797 /* Backup the contents of the ring buffer. */
798 for (i = first_idx; dw_size > 0; ++i, i &= ring->buf_mask, --dw_size)
799 ring->ring_backup[ring->ring_backup_entries_to_copy++] = ring->ring[i];
800}
801
802void amdgpu_ring_backup_unprocessed_commands(struct amdgpu_ring *ring,
803 struct amdgpu_fence *guilty_fence)
804{
805 struct dma_fence *unprocessed;
806 struct dma_fence __rcu **ptr;
807 struct amdgpu_fence *fence;
808 u32 seq, last_seq;
809
810 last_seq = amdgpu_fence_read(ring);
811 seq = ring->fence_drv.sync_seq;
812 ring->ring_backup_entries_to_copy = 0;
813
814 /*
815 * Avoid replay loops for a repeated guilty fence. NULL identifies a
816 * collateral ring, whose pending commands still need backup.
817 */
818 if (guilty_fence && ring->guilty_fence == guilty_fence) {
819 ring->guilty_fence = NULL;
820 return;
821 }
822 ring->guilty_fence = guilty_fence;
823
824 while (last_seq != seq) {
825 last_seq++;
826
827 ptr = &ring->fence_drv.fences[last_seq & ring->fence_drv.num_fences_mask];
828 rcu_read_lock();
829 unprocessed = rcu_dereference(*ptr);
830
831 if (unprocessed && !dma_fence_is_signaled(unprocessed)) {
832 fence = container_of(unprocessed, struct amdgpu_fence, base);
833
834 amdgpu_ring_backup_unprocessed_command(ring, fence);
835 }
836 rcu_read_unlock();
837 }
838}
839
840struct amdgpu_fence *
841amdgpu_ring_find_guilty_fence(struct amdgpu_ring *ring)
842{
843 struct dma_fence *unprocessed;
844 struct dma_fence __rcu **ptr;
845 struct amdgpu_fence *fence;
846 u32 seq, last_seq;
847
848 last_seq = amdgpu_fence_read(ring) & ring->fence_drv.num_fences_mask;
849 seq = ring->fence_drv.sync_seq & ring->fence_drv.num_fences_mask;
850
851 do {
852 last_seq++;
853 last_seq &= ring->fence_drv.num_fences_mask;
854
855 ptr = &ring->fence_drv.fences[last_seq];
856 rcu_read_lock();
857 unprocessed = rcu_dereference(*ptr);
858
859 if (unprocessed && !dma_fence_is_signaled(unprocessed)) {
860 fence = container_of(unprocessed, struct amdgpu_fence, base);
861 rcu_read_unlock();
862 return fence;
863 }
864 rcu_read_unlock();
865 } while (last_seq != seq);
866
867 return NULL;
868}
869
870/*
871 * Common fence implementation
872 */
873
874static const char *amdgpu_fence_get_driver_name(struct dma_fence *fence)
875{
876 return "amdgpu";
877}
878
879static const char *amdgpu_fence_get_timeline_name(struct dma_fence *f)
880{
881 return (const char *)to_amdgpu_fence(f)->ring->name;
882}
883
884/**
885 * amdgpu_fence_enable_signaling - enable signalling on fence
886 * @f: fence
887 *
888 * This function is called with fence_queue lock held, and adds a callback
889 * to fence_queue that checks if this fence is signaled, and if so it
890 * signals the fence and removes itself.
891 */
892static bool amdgpu_fence_enable_signaling(struct dma_fence *f)
893{
894 if (!timer_pending(&to_amdgpu_fence(f)->ring->fence_drv.fallback_timer))
895 amdgpu_fence_schedule_fallback(to_amdgpu_fence(f)->ring);
896
897 return true;
898}
899
900/**
901 * amdgpu_fence_free - free up the fence memory
902 *
903 * @rcu: RCU callback head
904 *
905 * Free up the fence memory after the RCU grace period.
906 */
907static void amdgpu_fence_free(struct rcu_head *rcu)
908{
909 struct dma_fence *f = container_of(rcu, struct dma_fence, rcu);
910
911 /* free fence_slab if it's separated fence*/
912 kfree(to_amdgpu_fence(f));
913}
914
915/**
916 * amdgpu_fence_release - callback that fence can be freed
917 *
918 * @f: fence
919 *
920 * This function is called when the reference count becomes zero.
921 * It just RCU schedules freeing up the fence.
922 */
923static void amdgpu_fence_release(struct dma_fence *f)
924{
925 call_rcu(&f->rcu, amdgpu_fence_free);
926}
927
928static const struct dma_fence_ops amdgpu_fence_ops = {
929 .get_driver_name = amdgpu_fence_get_driver_name,
930 .get_timeline_name = amdgpu_fence_get_timeline_name,
931 .enable_signaling = amdgpu_fence_enable_signaling,
932 .release = amdgpu_fence_release,
933};
934
935/*
936 * Fence debugfs
937 */
938#if defined(CONFIG_DEBUG_FS)
939static int amdgpu_debugfs_fence_info_show(struct seq_file *m, void *unused)
940{
941 struct amdgpu_device *adev = m->private;
942 int i;
943
944 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
945 struct amdgpu_ring *ring = adev->rings[i];
946
947 if (!ring || !ring->fence_drv.initialized)
948 continue;
949
950 amdgpu_fence_process(ring);
951
952 seq_printf(m, "--- ring %d (%s) ---\n", i, ring->name);
953 seq_printf(m, "Last signaled fence 0x%08x\n",
954 atomic_read(&ring->fence_drv.last_seq));
955 seq_printf(m, "Last emitted 0x%08x\n",
956 ring->fence_drv.sync_seq);
957
958 if (ring->funcs->type == AMDGPU_RING_TYPE_GFX ||
959 ring->funcs->type == AMDGPU_RING_TYPE_SDMA) {
960 seq_printf(m, "Last signaled trailing fence 0x%08x\n",
961 le32_to_cpu(*ring->trail_fence_cpu_addr));
962 seq_printf(m, "Last emitted 0x%08x\n",
963 ring->trail_seq);
964 }
965
966 if (ring->funcs->type != AMDGPU_RING_TYPE_GFX)
967 continue;
968
969 /* set in CP_VMID_PREEMPT and preemption occurred */
970 seq_printf(m, "Last preempted 0x%08x\n",
971 le32_to_cpu(*(ring->fence_drv.cpu_addr + 2)));
972 /* set in CP_VMID_RESET and reset occurred */
973 seq_printf(m, "Last reset 0x%08x\n",
974 le32_to_cpu(*(ring->fence_drv.cpu_addr + 4)));
975 /* Both preemption and reset occurred */
976 seq_printf(m, "Last both 0x%08x\n",
977 le32_to_cpu(*(ring->fence_drv.cpu_addr + 6)));
978 }
979 return 0;
980}
981
982/*
983 * amdgpu_debugfs_gpu_recover - manually trigger a gpu reset & recover
984 *
985 * Manually trigger a gpu reset at the next fence wait.
986 */
987static int gpu_recover_get(void *data, u64 *val)
988{
989 struct amdgpu_device *adev = (struct amdgpu_device *)data;
990 struct drm_device *dev = adev_to_drm(adev);
991 int r;
992
993 r = pm_runtime_get_sync(dev->dev);
994 if (r < 0) {
995 pm_runtime_put_autosuspend(dev->dev);
996 return 0;
997 }
998
999 if (amdgpu_reset_domain_schedule(adev->reset_domain, &adev->reset_work))
1000 flush_work(&adev->reset_work);
1001
1002 *val = atomic_read(&adev->reset_domain->reset_res);
1003
1004 pm_runtime_put_autosuspend(dev->dev);
1005
1006 return 0;
1007}
1008
1009DEFINE_SHOW_ATTRIBUTE(amdgpu_debugfs_fence_info);
1010DEFINE_DEBUGFS_ATTRIBUTE(amdgpu_debugfs_gpu_recover_fops, gpu_recover_get, NULL,
1011 "%lld\n");
1012
1013static void amdgpu_debugfs_reset_work(struct work_struct *work)
1014{
1015 struct amdgpu_device *adev = container_of(work, struct amdgpu_device,
1016 reset_work);
1017
1018 struct amdgpu_reset_context reset_context;
1019
1020 memset(&reset_context, 0, sizeof(reset_context));
1021
1022 reset_context.method = AMD_RESET_METHOD_NONE;
1023 reset_context.reset_req_dev = adev;
1024 reset_context.src = AMDGPU_RESET_SRC_USER;
1025 set_bit(AMDGPU_NEED_FULL_RESET, &reset_context.flags);
1026 set_bit(AMDGPU_SKIP_COREDUMP, &reset_context.flags);
1027
1028 amdgpu_device_gpu_recover(adev, NULL, &reset_context);
1029}
1030
1031#endif
1032
1033void amdgpu_debugfs_fence_init(struct amdgpu_device *adev)
1034{
1035#if defined(CONFIG_DEBUG_FS)
1036 struct drm_minor *minor = adev_to_drm(adev)->primary;
1037 struct dentry *root = minor->debugfs_root;
1038
1039 debugfs_create_file("amdgpu_fence_info", 0444, root, adev,
1040 &amdgpu_debugfs_fence_info_fops);
1041
1042 if (!amdgpu_sriov_vf(adev)) {
1043
1044 INIT_WORK(&adev->reset_work, amdgpu_debugfs_reset_work);
1045 debugfs_create_file("amdgpu_gpu_recover", 0444, root, adev,
1046 &amdgpu_debugfs_gpu_recover_fops);
1047 }
1048#endif
1049}
1050