projects/linux·blob master · refs

949 lines · 23.8 KiB

1/*
2 * Copyright 2015 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 * Authors: monk liu <[email protected]>
23 */
24
25#include <drm/drm_auth.h>
26#include <drm/drm_drv.h>
27#include "amdgpu.h"
28#include "amdgpu_sched.h"
29#include "amdgpu_ras.h"
30#include <linux/nospec.h>
31
32#define to_amdgpu_ctx_entity(e) \
33 container_of((e), struct amdgpu_ctx_entity, entity)
34
35const unsigned int amdgpu_ctx_num_entities[AMDGPU_HW_IP_NUM] = {
36 [AMDGPU_HW_IP_GFX] = 1,
37 [AMDGPU_HW_IP_COMPUTE] = 4,
38 [AMDGPU_HW_IP_DMA] = 2,
39 [AMDGPU_HW_IP_UVD] = 1,
40 [AMDGPU_HW_IP_VCE] = 1,
41 [AMDGPU_HW_IP_UVD_ENC] = 1,
42 [AMDGPU_HW_IP_VCN_DEC] = 1,
43 [AMDGPU_HW_IP_VCN_ENC] = 1,
44 [AMDGPU_HW_IP_VCN_JPEG] = 1,
45 [AMDGPU_HW_IP_VPE] = 1,
46};
47
48bool amdgpu_ctx_priority_is_valid(int32_t ctx_prio)
49{
50 switch (ctx_prio) {
51 case AMDGPU_CTX_PRIORITY_VERY_LOW:
52 case AMDGPU_CTX_PRIORITY_LOW:
53 case AMDGPU_CTX_PRIORITY_NORMAL:
54 case AMDGPU_CTX_PRIORITY_HIGH:
55 case AMDGPU_CTX_PRIORITY_VERY_HIGH:
56 return true;
57 default:
58 case AMDGPU_CTX_PRIORITY_UNSET:
59 /* UNSET priority is not valid and we don't carry that
60 * around, but set it to NORMAL in the only place this
61 * function is called, amdgpu_ctx_ioctl().
62 */
63 return false;
64 }
65}
66
67static enum drm_sched_priority
68amdgpu_ctx_to_drm_sched_prio(int32_t ctx_prio)
69{
70 switch (ctx_prio) {
71 case AMDGPU_CTX_PRIORITY_UNSET:
72 pr_warn_once("AMD-->DRM context priority value UNSET-->NORMAL");
73 return DRM_SCHED_PRIORITY_NORMAL;
74
75 case AMDGPU_CTX_PRIORITY_VERY_LOW:
76 return DRM_SCHED_PRIORITY_LOW;
77
78 case AMDGPU_CTX_PRIORITY_LOW:
79 return DRM_SCHED_PRIORITY_LOW;
80
81 case AMDGPU_CTX_PRIORITY_NORMAL:
82 return DRM_SCHED_PRIORITY_NORMAL;
83
84 case AMDGPU_CTX_PRIORITY_HIGH:
85 return DRM_SCHED_PRIORITY_HIGH;
86
87 case AMDGPU_CTX_PRIORITY_VERY_HIGH:
88 return DRM_SCHED_PRIORITY_HIGH;
89
90 /* This should not happen as we sanitized userspace provided priority
91 * already, WARN if this happens.
92 */
93 default:
94 WARN(1, "Invalid context priority %d\n", ctx_prio);
95 return DRM_SCHED_PRIORITY_NORMAL;
96 }
97
98}
99
100static int amdgpu_ctx_priority_permit(struct drm_file *filp,
101 int32_t priority)
102{
103 /* NORMAL and below are accessible by everyone */
104 if (priority <= AMDGPU_CTX_PRIORITY_NORMAL)
105 return 0;
106
107 if (capable(CAP_SYS_NICE))
108 return 0;
109
110 if (drm_is_current_master(filp))
111 return 0;
112
113 return -EACCES;
114}
115
116static enum amdgpu_gfx_pipe_priority amdgpu_ctx_prio_to_gfx_pipe_prio(int32_t prio)
117{
118 switch (prio) {
119 case AMDGPU_CTX_PRIORITY_HIGH:
120 case AMDGPU_CTX_PRIORITY_VERY_HIGH:
121 return AMDGPU_GFX_PIPE_PRIO_HIGH;
122 default:
123 return AMDGPU_GFX_PIPE_PRIO_NORMAL;
124 }
125}
126
127static enum amdgpu_ring_priority_level amdgpu_ctx_sched_prio_to_ring_prio(int32_t prio)
128{
129 switch (prio) {
130 case AMDGPU_CTX_PRIORITY_HIGH:
131 return AMDGPU_RING_PRIO_1;
132 case AMDGPU_CTX_PRIORITY_VERY_HIGH:
133 return AMDGPU_RING_PRIO_2;
134 default:
135 return AMDGPU_RING_PRIO_0;
136 }
137}
138
139static unsigned int amdgpu_ctx_get_hw_prio(struct amdgpu_ctx *ctx, u32 hw_ip)
140{
141 struct amdgpu_device *adev = ctx->mgr->adev;
142 unsigned int hw_prio;
143 int32_t ctx_prio;
144
145 ctx_prio = (ctx->override_priority == AMDGPU_CTX_PRIORITY_UNSET) ?
146 ctx->init_priority : ctx->override_priority;
147
148 switch (hw_ip) {
149 case AMDGPU_HW_IP_GFX:
150 case AMDGPU_HW_IP_COMPUTE:
151 hw_prio = amdgpu_ctx_prio_to_gfx_pipe_prio(ctx_prio);
152 break;
153 case AMDGPU_HW_IP_VCE:
154 case AMDGPU_HW_IP_VCN_ENC:
155 hw_prio = amdgpu_ctx_sched_prio_to_ring_prio(ctx_prio);
156 break;
157 default:
158 hw_prio = AMDGPU_RING_PRIO_DEFAULT;
159 break;
160 }
161
162 hw_ip = array_index_nospec(hw_ip, AMDGPU_HW_IP_NUM);
163 if (adev->gpu_sched[hw_ip][hw_prio].num_scheds == 0)
164 hw_prio = AMDGPU_RING_PRIO_DEFAULT;
165
166 return hw_prio;
167}
168
169/* Calculate the time spend on the hw */
170static ktime_t amdgpu_ctx_fence_time(struct dma_fence *fence)
171{
172 struct drm_sched_fence *s_fence;
173
174 if (!fence)
175 return ns_to_ktime(0);
176
177 /* When the fence is not even scheduled it can't have spend time */
178 s_fence = to_drm_sched_fence(fence);
179 if (!test_bit(DMA_FENCE_FLAG_TIMESTAMP_BIT, &s_fence->scheduled.flags))
180 return ns_to_ktime(0);
181
182 /* When it is still running account how much already spend */
183 if (!test_bit(DMA_FENCE_FLAG_TIMESTAMP_BIT, &s_fence->finished.flags))
184 return ktime_sub(ktime_get(), s_fence->scheduled.timestamp);
185
186 return ktime_sub(s_fence->finished.timestamp,
187 s_fence->scheduled.timestamp);
188}
189
190static ktime_t amdgpu_ctx_entity_time(struct amdgpu_ctx *ctx,
191 struct amdgpu_ctx_entity *centity)
192{
193 ktime_t res = ns_to_ktime(0);
194 uint32_t i;
195
196 spin_lock(&ctx->ring_lock);
197 for (i = 0; i < amdgpu_sched_jobs; i++) {
198 res = ktime_add(res, amdgpu_ctx_fence_time(centity->fences[i]));
199 }
200 spin_unlock(&ctx->ring_lock);
201 return res;
202}
203
204static int amdgpu_ctx_init_entity(struct amdgpu_ctx *ctx, u32 hw_ip,
205 const u32 ring)
206{
207 struct drm_gpu_scheduler **scheds = NULL, *sched = NULL;
208 struct amdgpu_device *adev = ctx->mgr->adev;
209 struct amdgpu_ctx_entity *entity;
210 enum drm_sched_priority drm_prio;
211 unsigned int hw_prio, num_scheds;
212 int32_t ctx_prio;
213 int r;
214
215 entity = kzalloc_flex(*entity, fences, amdgpu_sched_jobs);
216 if (!entity)
217 return -ENOMEM;
218
219 ctx_prio = (ctx->override_priority == AMDGPU_CTX_PRIORITY_UNSET) ?
220 ctx->init_priority : ctx->override_priority;
221 entity->hw_ip = hw_ip;
222 entity->sequence = 1;
223 hw_prio = amdgpu_ctx_get_hw_prio(ctx, hw_ip);
224 drm_prio = amdgpu_ctx_to_drm_sched_prio(ctx_prio);
225
226 hw_ip = array_index_nospec(hw_ip, AMDGPU_HW_IP_NUM);
227
228 if (!(adev)->xcp_mgr) {
229 scheds = adev->gpu_sched[hw_ip][hw_prio].sched;
230 num_scheds = adev->gpu_sched[hw_ip][hw_prio].num_scheds;
231 } else {
232 struct amdgpu_fpriv *fpriv;
233
234 /* TODO: Stop using fpriv here, we only need the xcp_id. */
235 fpriv = container_of(ctx->mgr, struct amdgpu_fpriv, ctx_mgr);
236 r = amdgpu_xcp_select_scheds(adev, hw_ip, hw_prio, fpriv,
237 &num_scheds, &scheds);
238 if (r)
239 goto error_free_entity;
240 }
241
242 if (num_scheds == 0) {
243 r = -EINVAL;
244 goto error_free_entity;
245 }
246
247 /* disable load balance if the hw engine retains context among dependent jobs */
248 if (hw_ip == AMDGPU_HW_IP_VCN_ENC ||
249 hw_ip == AMDGPU_HW_IP_VCN_DEC ||
250 hw_ip == AMDGPU_HW_IP_UVD_ENC ||
251 hw_ip == AMDGPU_HW_IP_UVD) {
252 sched = drm_sched_pick_best(scheds, num_scheds);
253 scheds = &sched;
254 num_scheds = 1;
255 }
256
257 r = drm_sched_entity_init(&entity->entity, drm_prio, scheds, num_scheds,
258 NULL);
259 if (r)
260 goto error_free_entity;
261
262 /* It's not an error if we fail to install the new entity */
263 if (cmpxchg(&ctx->entities[hw_ip][ring], NULL, entity))
264 goto cleanup_entity;
265
266 return 0;
267
268cleanup_entity:
269 drm_sched_entity_fini(&entity->entity);
270
271error_free_entity:
272 kfree(entity);
273
274 return r;
275}
276
277static ktime_t amdgpu_ctx_fini_entity(struct amdgpu_device *adev,
278 struct amdgpu_ctx_entity *entity)
279{
280 ktime_t res = ns_to_ktime(0);
281 int i;
282
283 if (!entity)
284 return res;
285
286 drm_sched_entity_destroy(&entity->entity);
287
288 for (i = 0; i < amdgpu_sched_jobs; ++i) {
289 res = ktime_add(res, amdgpu_ctx_fence_time(entity->fences[i]));
290 dma_fence_put(entity->fences[i]);
291 }
292
293 amdgpu_xcp_release_sched(adev, entity);
294
295 kfree(entity);
296 return res;
297}
298
299static u32 amdgpu_get_stable_pstate(struct amdgpu_device *adev)
300{
301 switch (amdgpu_dpm_get_performance_level(adev)) {
302 case AMD_DPM_FORCED_LEVEL_PROFILE_STANDARD:
303 return AMDGPU_CTX_STABLE_PSTATE_STANDARD;
304 case AMD_DPM_FORCED_LEVEL_PROFILE_MIN_SCLK:
305 return AMDGPU_CTX_STABLE_PSTATE_MIN_SCLK;
306 case AMD_DPM_FORCED_LEVEL_PROFILE_MIN_MCLK:
307 return AMDGPU_CTX_STABLE_PSTATE_MIN_MCLK;
308 case AMD_DPM_FORCED_LEVEL_PROFILE_PEAK:
309 return AMDGPU_CTX_STABLE_PSTATE_PEAK;
310 default:
311 return AMDGPU_CTX_STABLE_PSTATE_NONE;
312 }
313}
314
315static int amdgpu_ctx_init(struct amdgpu_ctx_mgr *mgr, int32_t priority,
316 struct drm_file *filp, struct amdgpu_ctx *ctx)
317{
318 struct amdgpu_fpriv *fpriv = filp->driver_priv;
319 int r;
320
321 r = amdgpu_ctx_priority_permit(filp, priority);
322 if (r)
323 return r;
324
325 memset(ctx, 0, sizeof(*ctx));
326
327 kref_init(&ctx->refcount);
328 ctx->mgr = mgr;
329 spin_lock_init(&ctx->ring_lock);
330
331 ctx->reset_counter = atomic_read(&mgr->adev->gpu_reset_counter);
332 ctx->reset_counter_query = ctx->reset_counter;
333 ctx->generation = amdgpu_vm_generation(mgr->adev, &fpriv->vm);
334 ctx->init_priority = priority;
335 ctx->override_priority = AMDGPU_CTX_PRIORITY_UNSET;
336 ctx->stable_pstate = AMDGPU_CTX_STABLE_PSTATE_NONE;
337
338 return 0;
339}
340
341static int __amdgpu_ctx_set_stable_pstate(struct amdgpu_ctx *ctx,
342 u32 stable_pstate)
343{
344 struct amdgpu_device *adev = ctx->mgr->adev;
345 enum amd_dpm_forced_level level;
346 struct amdgpu_ctx *current_ctx;
347 u32 current_stable_pstate;
348 int r = 0;
349
350 lockdep_assert_held(&adev->pm.stable_pstate_ctx_lock);
351
352 switch (stable_pstate) {
353 case AMDGPU_CTX_STABLE_PSTATE_NONE:
354 level = AMD_DPM_FORCED_LEVEL_AUTO;
355 break;
356 case AMDGPU_CTX_STABLE_PSTATE_STANDARD:
357 level = AMD_DPM_FORCED_LEVEL_PROFILE_STANDARD;
358 break;
359 case AMDGPU_CTX_STABLE_PSTATE_MIN_SCLK:
360 level = AMD_DPM_FORCED_LEVEL_PROFILE_MIN_SCLK;
361 break;
362 case AMDGPU_CTX_STABLE_PSTATE_MIN_MCLK:
363 level = AMD_DPM_FORCED_LEVEL_PROFILE_MIN_MCLK;
364 break;
365 case AMDGPU_CTX_STABLE_PSTATE_PEAK:
366 level = AMD_DPM_FORCED_LEVEL_PROFILE_PEAK;
367 break;
368 default:
369 return -EINVAL;
370 }
371
372 current_ctx = adev->pm.stable_pstate_ctx;
373 if (current_ctx && current_ctx != ctx)
374 return -EBUSY;
375
376 current_stable_pstate = amdgpu_get_stable_pstate(adev);
377 if (current_stable_pstate == stable_pstate)
378 return 0;
379
380 r = amdgpu_dpm_force_performance_level(adev, level);
381 if (r)
382 return r;
383
384 if (!current_ctx) {
385 adev->pm.stable_pstate_ctx = ctx;
386 /*
387 * Serialized by context taking ownership for the first time
388 * while holding adev->pm.stable_pstate_ctx_lock).
389 */
390 WRITE_ONCE(ctx->stable_pstate, current_stable_pstate);
391 }
392
393 return 0;
394}
395
396static int amdgpu_ctx_set_stable_pstate(struct amdgpu_ctx *ctx,
397 u32 stable_pstate)
398{
399 struct amdgpu_device *adev = ctx->mgr->adev;
400 int r;
401
402 mutex_lock(&adev->pm.stable_pstate_ctx_lock);
403 r = __amdgpu_ctx_set_stable_pstate(ctx, stable_pstate);
404 mutex_unlock(&adev->pm.stable_pstate_ctx_lock);
405
406 return r;
407}
408
409void amdgpu_ctx_fini(struct kref *ref)
410{
411 struct amdgpu_ctx *ctx = container_of(ref, struct amdgpu_ctx, refcount);
412 struct amdgpu_ctx_mgr *mgr = ctx->mgr;
413 struct amdgpu_device *adev = mgr->adev;
414 unsigned i, j, idx;
415
416 if (!adev)
417 return;
418
419 for (i = 0; i < AMDGPU_HW_IP_NUM; ++i) {
420 for (j = 0; j < AMDGPU_MAX_ENTITY_NUM; ++j) {
421 ktime_t spend;
422
423 spend = amdgpu_ctx_fini_entity(adev, ctx->entities[i][j]);
424 atomic64_add(ktime_to_ns(spend), &mgr->time_spend[i]);
425 }
426 }
427
428 if (drm_dev_enter(adev_to_drm(adev), &idx)) {
429 mutex_lock(&adev->pm.stable_pstate_ctx_lock);
430 if (adev->pm.stable_pstate_ctx == ctx) {
431 __amdgpu_ctx_set_stable_pstate(ctx, ctx->stable_pstate);
432 adev->pm.stable_pstate_ctx = NULL;
433 }
434 mutex_unlock(&adev->pm.stable_pstate_ctx_lock);
435 drm_dev_exit(idx);
436 }
437
438 kfree(ctx);
439}
440
441int amdgpu_ctx_get_entity(struct amdgpu_ctx *ctx, u32 hw_ip, u32 instance,
442 u32 ring, struct drm_sched_entity **entity)
443{
444 int r;
445 struct drm_sched_entity *ctx_entity;
446
447 if (hw_ip >= AMDGPU_HW_IP_NUM) {
448 drm_err(adev_to_drm(ctx->mgr->adev),
449 "unknown HW IP type: %d\n", hw_ip);
450 return -EINVAL;
451 }
452
453 /* Right now all IPs have only one instance - multiple rings. */
454 if (instance != 0) {
455 drm_dbg(adev_to_drm(ctx->mgr->adev),
456 "invalid ip instance: %d\n", instance);
457 return -EINVAL;
458 }
459
460 if (ring >= amdgpu_ctx_num_entities[hw_ip]) {
461 drm_dbg(adev_to_drm(ctx->mgr->adev),
462 "invalid ring: %d %d\n", hw_ip, ring);
463 return -EINVAL;
464 }
465
466 if (ctx->entities[hw_ip][ring] == NULL) {
467 r = amdgpu_ctx_init_entity(ctx, hw_ip, ring);
468 if (r)
469 return r;
470 }
471
472 ctx_entity = &ctx->entities[hw_ip][ring]->entity;
473 r = drm_sched_entity_error(ctx_entity);
474 if (r) {
475 DRM_DEBUG("error entity %p\n", ctx_entity);
476 return r;
477 }
478
479 *entity = ctx_entity;
480 return 0;
481}
482
483static int amdgpu_ctx_alloc(struct amdgpu_device *adev,
484 struct amdgpu_fpriv *fpriv,
485 struct drm_file *filp,
486 int32_t priority,
487 uint32_t *id)
488{
489 struct amdgpu_ctx_mgr *mgr = &fpriv->ctx_mgr;
490 struct amdgpu_ctx *ctx;
491 int r;
492
493 ctx = kmalloc_obj(*ctx);
494 if (!ctx)
495 return -ENOMEM;
496
497 r = amdgpu_ctx_init(mgr, priority, filp, ctx);
498 if (r) {
499 kfree(ctx);
500 return r;
501 }
502
503 r = xa_alloc(&mgr->ctx_handles, id, ctx, xa_limit_32b, GFP_KERNEL);
504 if (r)
505 amdgpu_ctx_put(ctx);
506
507 return r;
508}
509
510static int amdgpu_ctx_free(struct amdgpu_fpriv *fpriv, uint32_t id)
511{
512 struct amdgpu_ctx *ctx;
513
514 ctx = xa_erase(&fpriv->ctx_mgr.ctx_handles, id);
515 amdgpu_ctx_put(ctx);
516
517 return ctx ? 0 : -EINVAL;
518}
519
520static int amdgpu_ctx_query(struct amdgpu_device *adev,
521 struct amdgpu_fpriv *fpriv, uint32_t id,
522 union drm_amdgpu_ctx_out *out)
523{
524 struct amdgpu_ctx *ctx;
525 unsigned reset_counter;
526
527 ctx = amdgpu_ctx_get(fpriv, id);
528 if (!ctx)
529 return -EINVAL;
530
531 /* TODO: these two are always zero */
532 out->state.flags = 0x0;
533 out->state.hangs = 0x0;
534
535 /* determine if a GPU reset has occured since the last call */
536 reset_counter = atomic_read(&adev->gpu_reset_counter);
537 /* TODO: this should ideally return NO, GUILTY, or INNOCENT. */
538 if (ctx->reset_counter_query == reset_counter)
539 out->state.reset_status = AMDGPU_CTX_NO_RESET;
540 else
541 out->state.reset_status = AMDGPU_CTX_UNKNOWN_RESET;
542 ctx->reset_counter_query = reset_counter;
543
544 amdgpu_ctx_put(ctx);
545
546 return 0;
547}
548
549#define AMDGPU_RAS_COUNTE_DELAY_MS 3000
550
551static bool amdgpu_ctx_guilty(struct amdgpu_ctx *ctx)
552{
553 int i, j, r;
554
555 for (i = 0; i < AMDGPU_HW_IP_NUM; ++i) {
556 for (j = 0; j < amdgpu_ctx_num_entities[i]; ++j) {
557 struct amdgpu_ctx_entity *ctx_entity;
558
559 ctx_entity = ctx->entities[i][j];
560 if (!ctx_entity)
561 continue;
562
563 r = drm_sched_entity_error(&ctx_entity->entity);
564 if (r == -ETIME)
565 return true;
566 }
567 }
568
569 return false;
570}
571
572static int amdgpu_ctx_query2(struct amdgpu_device *adev,
573 struct amdgpu_fpriv *fpriv, uint32_t id,
574 union drm_amdgpu_ctx_out *out)
575{
576 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
577 struct amdgpu_ctx *ctx;
578
579 ctx = amdgpu_ctx_get(fpriv, id);
580 if (!ctx)
581 return -EINVAL;
582
583 out->state.flags = 0x0;
584 out->state.hangs = 0x0;
585
586 if (ctx->reset_counter != atomic_read(&adev->gpu_reset_counter))
587 out->state.flags |= AMDGPU_CTX_QUERY2_FLAGS_RESET;
588
589 if (ctx->generation != amdgpu_vm_generation(adev, &fpriv->vm))
590 out->state.flags |= AMDGPU_CTX_QUERY2_FLAGS_VRAMLOST;
591
592 if (amdgpu_ctx_guilty(ctx))
593 out->state.flags |= AMDGPU_CTX_QUERY2_FLAGS_GUILTY;
594
595 if (amdgpu_in_reset(adev))
596 out->state.flags |= AMDGPU_CTX_QUERY2_FLAGS_RESET_IN_PROGRESS;
597
598 if (adev->ras_enabled && con) {
599 /* Return the cached values in O(1),
600 * and schedule delayed work to cache
601 * new vaues.
602 */
603 int ce_count, ue_count;
604
605 ce_count = atomic_read(&con->ras_ce_count);
606 ue_count = atomic_read(&con->ras_ue_count);
607
608 if (ce_count != ctx->ras_counter_ce) {
609 ctx->ras_counter_ce = ce_count;
610 out->state.flags |= AMDGPU_CTX_QUERY2_FLAGS_RAS_CE;
611 }
612
613 if (ue_count != ctx->ras_counter_ue) {
614 ctx->ras_counter_ue = ue_count;
615 out->state.flags |= AMDGPU_CTX_QUERY2_FLAGS_RAS_UE;
616 }
617
618 schedule_delayed_work(&con->ras_counte_delay_work,
619 msecs_to_jiffies(AMDGPU_RAS_COUNTE_DELAY_MS));
620 }
621
622 amdgpu_ctx_put(ctx);
623
624 return 0;
625}
626
627static int amdgpu_ctx_stable_pstate(struct amdgpu_device *adev,
628 struct amdgpu_fpriv *fpriv, uint32_t id,
629 bool set, u32 *stable_pstate)
630{
631 struct amdgpu_ctx *ctx;
632 int r = 0;
633
634 ctx = amdgpu_ctx_get(fpriv, id);
635 if (!ctx)
636 return -EINVAL;
637
638 /*
639 * The get path is odd in this uapi - it will check whether the context
640 * id exist, but otherwise does nothing with it. In other words, the
641 * uapi has historically been implemented as being able to query the
642 * global device state, as long as the caller supplies a random valid
643 * context id.
644 */
645
646 if (set)
647 r = amdgpu_ctx_set_stable_pstate(ctx, *stable_pstate);
648 else
649 *stable_pstate = amdgpu_get_stable_pstate(adev);
650
651 amdgpu_ctx_put(ctx);
652 return r;
653}
654
655int amdgpu_ctx_ioctl(struct drm_device *dev, void *data,
656 struct drm_file *filp)
657{
658 int r;
659 uint32_t id, stable_pstate;
660 int32_t priority;
661
662 union drm_amdgpu_ctx *args = data;
663 struct amdgpu_device *adev = drm_to_adev(dev);
664 struct amdgpu_fpriv *fpriv = filp->driver_priv;
665
666 id = args->in.ctx_id;
667 priority = args->in.priority;
668
669 /* For backwards compatibility, we need to accept ioctls with garbage
670 * in the priority field. Garbage values in the priority field, result
671 * in the priority being set to NORMAL.
672 */
673 if (!amdgpu_ctx_priority_is_valid(priority))
674 priority = AMDGPU_CTX_PRIORITY_NORMAL;
675
676 switch (args->in.op) {
677 case AMDGPU_CTX_OP_ALLOC_CTX:
678 if (args->in.flags)
679 return -EINVAL;
680 r = amdgpu_ctx_alloc(adev, fpriv, filp, priority, &id);
681 args->out.alloc.ctx_id = id;
682 break;
683 case AMDGPU_CTX_OP_FREE_CTX:
684 if (args->in.flags)
685 return -EINVAL;
686 r = amdgpu_ctx_free(fpriv, id);
687 break;
688 case AMDGPU_CTX_OP_QUERY_STATE:
689 if (args->in.flags)
690 return -EINVAL;
691 r = amdgpu_ctx_query(adev, fpriv, id, &args->out);
692 break;
693 case AMDGPU_CTX_OP_QUERY_STATE2:
694 if (args->in.flags)
695 return -EINVAL;
696 r = amdgpu_ctx_query2(adev, fpriv, id, &args->out);
697 break;
698 case AMDGPU_CTX_OP_GET_STABLE_PSTATE:
699 if (args->in.flags)
700 return -EINVAL;
701 r = amdgpu_ctx_stable_pstate(adev, fpriv, id, false, &stable_pstate);
702 if (!r)
703 args->out.pstate.flags = stable_pstate;
704 break;
705 case AMDGPU_CTX_OP_SET_STABLE_PSTATE:
706 if (args->in.flags & ~AMDGPU_CTX_STABLE_PSTATE_FLAGS_MASK)
707 return -EINVAL;
708 stable_pstate = args->in.flags & AMDGPU_CTX_STABLE_PSTATE_FLAGS_MASK;
709 if (stable_pstate > AMDGPU_CTX_STABLE_PSTATE_PEAK)
710 return -EINVAL;
711 r = amdgpu_ctx_stable_pstate(adev, fpriv, id, true, &stable_pstate);
712 break;
713 default:
714 return -EINVAL;
715 }
716
717 return r;
718}
719
720struct amdgpu_ctx *amdgpu_ctx_get(struct amdgpu_fpriv *fpriv, uint32_t id)
721{
722 struct amdgpu_ctx *ctx;
723 struct amdgpu_ctx_mgr *mgr;
724
725 if (!fpriv)
726 return NULL;
727
728 mgr = &fpriv->ctx_mgr;
729
730 xa_lock(&mgr->ctx_handles);
731 ctx = xa_load(&mgr->ctx_handles, id);
732 if (ctx)
733 kref_get(&ctx->refcount);
734 xa_unlock(&mgr->ctx_handles);
735 return ctx;
736}
737
738uint64_t amdgpu_ctx_add_fence(struct amdgpu_ctx *ctx,
739 struct drm_sched_entity *entity,
740 struct dma_fence *fence)
741{
742 struct amdgpu_ctx_entity *centity = to_amdgpu_ctx_entity(entity);
743 uint64_t seq = centity->sequence;
744 struct dma_fence *other = NULL;
745 unsigned idx = 0;
746
747 idx = seq & (amdgpu_sched_jobs - 1);
748 other = centity->fences[idx];
749 WARN_ON(other && !dma_fence_is_signaled(other));
750
751 dma_fence_get(fence);
752
753 spin_lock(&ctx->ring_lock);
754 centity->fences[idx] = fence;
755 centity->sequence++;
756 spin_unlock(&ctx->ring_lock);
757
758 atomic64_add(ktime_to_ns(amdgpu_ctx_fence_time(other)),
759 &ctx->mgr->time_spend[centity->hw_ip]);
760
761 dma_fence_put(other);
762 return seq;
763}
764
765struct dma_fence *amdgpu_ctx_get_fence(struct amdgpu_ctx *ctx,
766 struct drm_sched_entity *entity,
767 uint64_t seq)
768{
769 struct amdgpu_ctx_entity *centity = to_amdgpu_ctx_entity(entity);
770 struct dma_fence *fence;
771
772 spin_lock(&ctx->ring_lock);
773
774 if (seq == ~0ull)
775 seq = centity->sequence - 1;
776
777 if (seq >= centity->sequence) {
778 spin_unlock(&ctx->ring_lock);
779 return ERR_PTR(-EINVAL);
780 }
781
782
783 if (seq + amdgpu_sched_jobs < centity->sequence) {
784 spin_unlock(&ctx->ring_lock);
785 return NULL;
786 }
787
788 fence = dma_fence_get(centity->fences[seq & (amdgpu_sched_jobs - 1)]);
789 spin_unlock(&ctx->ring_lock);
790
791 return fence;
792}
793
794static void amdgpu_ctx_set_entity_priority(struct amdgpu_ctx *ctx,
795 struct amdgpu_ctx_entity *aentity,
796 int hw_ip,
797 int32_t priority)
798{
799 struct amdgpu_device *adev = ctx->mgr->adev;
800 unsigned int hw_prio;
801 struct drm_gpu_scheduler **scheds = NULL;
802 unsigned num_scheds;
803
804 /* set sw priority */
805 drm_sched_entity_set_priority(&aentity->entity,
806 amdgpu_ctx_to_drm_sched_prio(priority));
807
808 /* set hw priority */
809 if (hw_ip == AMDGPU_HW_IP_COMPUTE || hw_ip == AMDGPU_HW_IP_GFX) {
810 hw_prio = amdgpu_ctx_get_hw_prio(ctx, hw_ip);
811 hw_prio = array_index_nospec(hw_prio, AMDGPU_RING_PRIO_MAX);
812 scheds = adev->gpu_sched[hw_ip][hw_prio].sched;
813 num_scheds = adev->gpu_sched[hw_ip][hw_prio].num_scheds;
814 drm_sched_entity_modify_sched(&aentity->entity, scheds,
815 num_scheds);
816 }
817}
818
819void amdgpu_ctx_priority_override(struct amdgpu_ctx *ctx,
820 int32_t priority)
821{
822 int32_t ctx_prio;
823 unsigned i, j;
824
825 ctx->override_priority = priority;
826
827 ctx_prio = (ctx->override_priority == AMDGPU_CTX_PRIORITY_UNSET) ?
828 ctx->init_priority : ctx->override_priority;
829 for (i = 0; i < AMDGPU_HW_IP_NUM; ++i) {
830 for (j = 0; j < amdgpu_ctx_num_entities[i]; ++j) {
831 if (!ctx->entities[i][j])
832 continue;
833
834 amdgpu_ctx_set_entity_priority(ctx, ctx->entities[i][j],
835 i, ctx_prio);
836 }
837 }
838}
839
840int amdgpu_ctx_wait_prev_fence(struct amdgpu_ctx *ctx,
841 struct drm_sched_entity *entity)
842{
843 struct amdgpu_ctx_entity *centity = to_amdgpu_ctx_entity(entity);
844 struct dma_fence *other;
845 unsigned idx;
846 long r;
847
848 spin_lock(&ctx->ring_lock);
849 idx = centity->sequence & (amdgpu_sched_jobs - 1);
850 other = dma_fence_get(centity->fences[idx]);
851 spin_unlock(&ctx->ring_lock);
852
853 if (!other)
854 return 0;
855
856 r = dma_fence_wait(other, true);
857 if (r < 0 && r != -ERESTARTSYS)
858 drm_err(adev_to_drm(ctx->mgr->adev),
859 "AMDGPU: Error waiting for fence in ctx %p\n", ctx);
860
861 dma_fence_put(other);
862 return r;
863}
864
865void amdgpu_ctx_mgr_init(struct amdgpu_ctx_mgr *mgr,
866 struct amdgpu_device *adev)
867{
868 unsigned int i;
869
870 mgr->adev = adev;
871 xa_init_flags(&mgr->ctx_handles, XA_FLAGS_ALLOC1);
872
873 for (i = 0; i < AMDGPU_HW_IP_NUM; ++i)
874 atomic64_set(&mgr->time_spend[i], 0);
875}
876
877long amdgpu_ctx_mgr_entity_flush(struct amdgpu_ctx_mgr *mgr, long timeout)
878{
879 struct amdgpu_ctx *ctx;
880 unsigned long id;
881 int i, j;
882
883 xa_lock(&mgr->ctx_handles);
884 xa_for_each(&mgr->ctx_handles, id, ctx) {
885 kref_get(&ctx->refcount);
886 xa_unlock(&mgr->ctx_handles);
887 for (i = 0; i < AMDGPU_HW_IP_NUM; ++i) {
888 for (j = 0; j < amdgpu_ctx_num_entities[i]; ++j) {
889 struct drm_sched_entity *entity;
890
891 if (!ctx->entities[i][j])
892 continue;
893
894 entity = &ctx->entities[i][j]->entity;
895 timeout = drm_sched_entity_flush(entity, timeout);
896 }
897 }
898 amdgpu_ctx_put(ctx);
899 xa_lock(&mgr->ctx_handles);
900 }
901 xa_unlock(&mgr->ctx_handles);
902 return timeout;
903}
904
905void amdgpu_ctx_mgr_fini(struct amdgpu_ctx_mgr *mgr)
906{
907 struct amdgpu_ctx *ctx;
908 unsigned long id;
909
910 xa_for_each(&mgr->ctx_handles, id, ctx)
911 amdgpu_ctx_put(ctx);
912 xa_destroy(&mgr->ctx_handles);
913}
914
915void amdgpu_ctx_mgr_usage(struct amdgpu_ctx_mgr *mgr,
916 ktime_t usage[AMDGPU_HW_IP_NUM])
917{
918 struct amdgpu_ctx *ctx;
919 unsigned int hw_ip, i;
920 unsigned long id;
921
922 /*
923 * This is a little bit racy because it can be that a ctx or a fence are
924 * destroyed just in the moment we try to account them. But that is ok
925 * since exactly that case is explicitely allowed by the interface.
926 */
927 for (hw_ip = 0; hw_ip < AMDGPU_HW_IP_NUM; ++hw_ip) {
928 uint64_t ns = atomic64_read(&mgr->time_spend[hw_ip]);
929
930 usage[hw_ip] = ns_to_ktime(ns);
931 }
932
933 xa_lock(&mgr->ctx_handles);
934 xa_for_each(&mgr->ctx_handles, id, ctx) {
935 for (hw_ip = 0; hw_ip < AMDGPU_HW_IP_NUM; ++hw_ip) {
936 for (i = 0; i < amdgpu_ctx_num_entities[hw_ip]; ++i) {
937 struct amdgpu_ctx_entity *centity;
938 ktime_t spend;
939
940 centity = ctx->entities[hw_ip][i];
941 if (!centity)
942 continue;
943 spend = amdgpu_ctx_entity_time(ctx, centity);
944 usage[hw_ip] = ktime_add(usage[hw_ip], spend);
945 }
946 }
947 }
948 xa_unlock(&mgr->ctx_handles);
949}