projects/linux·blob master · refs

789 lines · 25.8 KiB

1/*
2 * Copyright 2015 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 */
23
24#ifndef _DRM_GPU_SCHEDULER_H_
25#define _DRM_GPU_SCHEDULER_H_
26
27#include <drm/spsc_queue.h>
28#include <linux/average.h>
29#include <linux/dma-fence.h>
30#include <linux/completion.h>
31#include <linux/xarray.h>
32#include <linux/workqueue.h>
33
34DECLARE_EWMA(drm_sched_avgtime, 6, 4);
35
36#define MAX_WAIT_SCHED_ENTITY_Q_EMPTY msecs_to_jiffies(1000)
37
38/**
39 * DRM_SCHED_FENCE_DONT_PIPELINE - Prevent dependency pipelining
40 *
41 * Setting this flag on a scheduler fence prevents pipelining of jobs depending
42 * on this fence. In other words we always insert a full CPU round trip before
43 * dependent jobs are pushed to the hw queue.
44 */
45#define DRM_SCHED_FENCE_DONT_PIPELINE DMA_FENCE_FLAG_USER_BITS
46
47/**
48 * DRM_SCHED_FENCE_FLAG_HAS_DEADLINE_BIT - A fence deadline hint has been set
49 *
50 * Because we could have a deadline hint can be set before the backing hw
51 * fence is created, we need to keep track of whether a deadline has already
52 * been set.
53 */
54#define DRM_SCHED_FENCE_FLAG_HAS_DEADLINE_BIT (DMA_FENCE_FLAG_USER_BITS + 1)
55
56enum dma_resv_usage;
57struct dma_resv;
58struct drm_gem_object;
59
60struct drm_gpu_scheduler;
61struct drm_sched_rq;
62
63struct drm_file;
64
65/* These are often used as an (initial) index
66 * to an array, and as such should start at 0.
67 */
68enum drm_sched_priority {
69 DRM_SCHED_PRIORITY_INVALID = -1, /* Internal marker - do not use. */
70 DRM_SCHED_PRIORITY_KERNEL,
71 DRM_SCHED_PRIORITY_HIGH,
72 DRM_SCHED_PRIORITY_NORMAL,
73 DRM_SCHED_PRIORITY_LOW,
74
75 DRM_SCHED_PRIORITY_COUNT
76};
77
78struct drm_sched_entity_stats;
79
80/**
81 * struct drm_sched_entity - A wrapper around a job queue (typically
82 * attached to the DRM file_priv).
83 *
84 * Entities will emit jobs in order to their corresponding hardware
85 * ring, and the scheduler will alternate between entities based on
86 * scheduling policy.
87 */
88struct drm_sched_entity {
89 /**
90 * @list:
91 *
92 * Used to append this struct to the list of entities in the runqueue
93 * @rq under &drm_sched_rq.entities.
94 *
95 * Protected by &drm_sched_rq.lock of @rq.
96 */
97 struct list_head list;
98
99 /**
100 * @lock:
101 *
102 * Lock protecting the run-queue (@rq) to which this entity belongs,
103 * @priority, the list of schedulers (@sched_list, @num_sched_list) and
104 * the @rr_ts field.
105 */
106 spinlock_t lock;
107
108 /**
109 * @rq:
110 *
111 * Runqueue on which this entity is currently scheduled.
112 *
113 * FIXME: Locking is very unclear for this. Writers are protected by
114 * @lock, but readers are generally lockless and seem to just race with
115 * not even a READ_ONCE.
116 */
117 struct drm_sched_rq *rq;
118
119 /**
120 * @stats: Stats object reference held by the entity and jobs.
121 */
122 struct drm_sched_entity_stats *stats;
123
124 /**
125 * @sched_list:
126 *
127 * A list of schedulers (struct drm_gpu_scheduler). Jobs from this entity can
128 * be scheduled on any scheduler on this list.
129 *
130 * This can be modified by calling drm_sched_entity_modify_sched().
131 * Locking is entirely up to the driver, see the above function for more
132 * details.
133 *
134 * This will be set to NULL if &num_sched_list equals 1 and @rq has been
135 * set already.
136 *
137 * FIXME: This means priority changes through
138 * drm_sched_entity_set_priority() will be lost henceforth in this case.
139 */
140 struct drm_gpu_scheduler **sched_list;
141
142 /**
143 * @num_sched_list:
144 *
145 * Number of drm_gpu_schedulers in the @sched_list.
146 */
147 unsigned int num_sched_list;
148
149 /**
150 * @priority:
151 *
152 * Priority of the entity. This can be modified by calling
153 * drm_sched_entity_set_priority(). Protected by @lock.
154 */
155 enum drm_sched_priority priority;
156
157 /**
158 * @rq_priority: Run-queue priority
159 */
160 enum drm_sched_priority rq_priority;
161
162 /**
163 * @rr_ts:
164 *
165 * Fake timestamp of the last popped job from the entity.
166 */
167 ktime_t rr_ts;
168
169 /**
170 * @job_queue: the list of jobs of this entity.
171 */
172 struct spsc_queue job_queue;
173
174 /**
175 * @fence_seq:
176 *
177 * A linearly increasing seqno incremented with each new
178 * &drm_sched_fence which is part of the entity.
179 *
180 * FIXME: Callers of drm_sched_job_arm() need to ensure correct locking,
181 * this doesn't need to be atomic.
182 */
183 atomic_t fence_seq;
184
185 /**
186 * @fence_context:
187 *
188 * A unique context for all the fences which belong to this entity. The
189 * &drm_sched_fence.scheduled uses the fence_context but
190 * &drm_sched_fence.finished uses fence_context + 1.
191 */
192 uint64_t fence_context;
193
194 /**
195 * @dependency:
196 *
197 * The dependency fence of the job which is on the top of the job queue.
198 */
199 struct dma_fence *dependency;
200
201 /**
202 * @cb:
203 *
204 * Callback for the dependency fence above.
205 */
206 struct dma_fence_cb cb;
207
208 /**
209 * @guilty:
210 *
211 * Points to entities' guilty.
212 */
213 atomic_t *guilty;
214
215 /**
216 * @last_scheduled:
217 *
218 * Points to the finished fence of the last scheduled job. Only written
219 * by drm_sched_entity_pop_job(). Can be accessed locklessly from
220 * drm_sched_job_arm() if the queue is empty.
221 */
222 struct dma_fence __rcu *last_scheduled;
223
224 /**
225 * @last_user: last group leader pushing a job into the entity.
226 */
227 struct task_struct *last_user;
228
229 /**
230 * @stopped:
231 *
232 * Marks the enity as removed from rq and destined for
233 * termination. This is set by calling drm_sched_entity_flush().
234 */
235 bool stopped;
236
237 /**
238 * @entity_idle:
239 *
240 * Signals when entity is not in use, used to sequence entity cleanup in
241 * drm_sched_entity_fini().
242 */
243 struct completion entity_idle;
244
245 /**
246 * @oldest_job_waiting:
247 *
248 * Marks earliest job waiting in SW queue
249 */
250 ktime_t oldest_job_waiting;
251
252 /**
253 * @rb_tree_node:
254 *
255 * The node used to insert this entity into time based priority queue
256 */
257 struct rb_node rb_tree_node;
258
259};
260
261/**
262 * struct drm_sched_rq - queue of entities to be scheduled.
263 *
264 * @sched: the scheduler to which this rq belongs to.
265 * @lock: protects @entities, @rb_tree_root, @rr_ts and @head_prio.
266 * @rr_ts: monotonically incrementing fake timestamp for RR mode.
267 * @entities: list of the entities to be scheduled.
268 * @rb_tree_root: root of time based priority queue of entities for FIFO scheduling
269 * @head_prio: priority of the top tree element.
270 *
271 * Run queue is a set of entities scheduling command submissions for
272 * one specific ring. It implements the scheduling policy that selects
273 * the next entity to emit commands from.
274 */
275struct drm_sched_rq {
276 struct drm_gpu_scheduler *sched;
277
278 spinlock_t lock;
279 /* Following members are protected by the @lock: */
280 ktime_t rr_ts;
281 struct list_head entities;
282 struct rb_root_cached rb_tree_root;
283 enum drm_sched_priority head_prio;
284};
285
286/**
287 * struct drm_sched_fence - fences corresponding to the scheduling of a job.
288 */
289struct drm_sched_fence {
290 /**
291 * @scheduled: this fence is what will be signaled by the scheduler
292 * when the job is scheduled.
293 */
294 struct dma_fence scheduled;
295
296 /**
297 * @finished: this fence is what will be signaled by the scheduler
298 * when the job is completed.
299 *
300 * When setting up an out fence for the job, you should use
301 * this, since it's available immediately upon
302 * drm_sched_job_init(), and the fence returned by the driver
303 * from run_job() won't be created until the dependencies have
304 * resolved.
305 */
306 struct dma_fence finished;
307
308 /**
309 * @deadline: deadline set on &drm_sched_fence.finished which
310 * potentially needs to be propagated to &drm_sched_fence.parent
311 */
312 ktime_t deadline;
313
314 /**
315 * @parent: the fence returned by &drm_sched_backend_ops.run_job
316 * when scheduling the job on hardware. We signal the
317 * &drm_sched_fence.finished fence once parent is signalled.
318 */
319 struct dma_fence *parent;
320 /**
321 * @sched: the scheduler instance to which the job having this struct
322 * belongs to.
323 */
324 struct drm_gpu_scheduler *sched;
325 /**
326 * @lock: the lock used by the scheduled and the finished fences.
327 */
328 spinlock_t lock;
329 /**
330 * @owner: job owner for debugging
331 */
332 void *owner;
333
334 /**
335 * @drm_client_id:
336 *
337 * The client_id of the drm_file which owns the job.
338 */
339 uint64_t drm_client_id;
340};
341
342struct drm_sched_fence *to_drm_sched_fence(struct dma_fence *f);
343
344/**
345 * struct drm_sched_job - A job to be run by an entity.
346 *
347 * @queue_node: used to append this struct to the queue of jobs in an entity.
348 * @list: a job participates in a "pending" and "done" lists.
349 * @sched: the scheduler instance on which this job is scheduled.
350 * @s_fence: contains the fences for the scheduling of job.
351 * @finish_cb: the callback for the finished fence.
352 * @credits: the number of credits this job contributes to the scheduler
353 * @work: Helper to reschedule job kill to different context.
354 * @karma: increment on every hang caused by this job. If this exceeds the hang
355 * limit of the scheduler then the job is marked guilty and will not
356 * be scheduled further.
357 * @s_priority: the priority of the job.
358 * @entity: the entity to which this job belongs.
359 * @cb: the callback for the parent fence in s_fence.
360 *
361 * A job is created by the driver using drm_sched_job_init(), and
362 * should call drm_sched_entity_push_job() once it wants the scheduler
363 * to schedule the job.
364 */
365struct drm_sched_job {
366 /**
367 * @submit_ts:
368 *
369 * When the job was pushed into the entity queue.
370 */
371 ktime_t submit_ts;
372
373 /**
374 * @sched:
375 *
376 * The scheduler this job is or will be scheduled on. Gets set by
377 * drm_sched_job_arm(). Valid until drm_sched_backend_ops.free_job()
378 * has finished.
379 */
380 struct drm_gpu_scheduler *sched;
381
382 struct drm_sched_fence *s_fence;
383 struct drm_sched_entity *entity;
384
385 /**
386 * @entity_stats: Stats object reference held by the job and entity.
387 */
388 struct drm_sched_entity_stats *entity_stats;
389
390 enum drm_sched_priority s_priority;
391 u32 credits;
392 /** @last_dependency: tracks @dependencies as they signal */
393 unsigned int last_dependency;
394 atomic_t karma;
395
396 struct spsc_node queue_node;
397 struct list_head list;
398
399 /*
400 * work is used only after finish_cb has been used and will not be
401 * accessed anymore.
402 */
403 union {
404 struct dma_fence_cb finish_cb;
405 struct work_struct work;
406 };
407
408 struct dma_fence_cb cb;
409
410 /**
411 * @dependencies:
412 *
413 * Contains the dependencies as struct dma_fence for this job, see
414 * drm_sched_job_add_dependency() and
415 * drm_sched_job_add_implicit_dependencies().
416 */
417 struct xarray dependencies;
418};
419
420/**
421 * enum drm_gpu_sched_stat - the scheduler's status
422 *
423 * @DRM_GPU_SCHED_STAT_NONE: Reserved. Do not use.
424 * @DRM_GPU_SCHED_STAT_RESET: The GPU hung and successfully reset.
425 * @DRM_GPU_SCHED_STAT_ENODEV: Error: Device is not available anymore.
426 * @DRM_GPU_SCHED_STAT_NO_HANG: Contrary to scheduler's assumption, the GPU
427 * did not hang and is still running.
428 */
429enum drm_gpu_sched_stat {
430 DRM_GPU_SCHED_STAT_NONE,
431 DRM_GPU_SCHED_STAT_RESET,
432 DRM_GPU_SCHED_STAT_ENODEV,
433 DRM_GPU_SCHED_STAT_NO_HANG,
434};
435
436/**
437 * struct drm_sched_backend_ops - Define the backend operations
438 * called by the scheduler
439 *
440 * These functions should be implemented in the driver side.
441 */
442struct drm_sched_backend_ops {
443 /**
444 * @prepare_job:
445 *
446 * Called when the scheduler is considering scheduling this job next, to
447 * get another struct dma_fence for this job to block on. Once it
448 * returns NULL, run_job() may be called.
449 *
450 * Can be NULL if no additional preparation to the dependencies are
451 * necessary. Skipped when jobs are killed instead of run.
452 */
453 struct dma_fence *(*prepare_job)(struct drm_sched_job *sched_job,
454 struct drm_sched_entity *s_entity);
455
456 /**
457 * @run_job: Called to execute the job once all of the dependencies
458 * have been resolved.
459 *
460 * @sched_job: the job to run
461 *
462 * The deprecated drm_sched_resubmit_jobs() (called by &struct
463 * drm_sched_backend_ops.timedout_job) can invoke this again with the
464 * same parameters. Using this is discouraged because it violates
465 * dma_fence rules, notably dma_fence_init() has to be called on
466 * already initialized fences for a second time. Moreover, this is
467 * dangerous because attempts to allocate memory might deadlock with
468 * memory management code waiting for the reset to complete.
469 *
470 * TODO: Document what drivers should do / use instead.
471 *
472 * This method is called in a workqueue context - either from the
473 * submit_wq the driver passed through drm_sched_init(), or, if the
474 * driver passed NULL, a separate, ordered workqueue the scheduler
475 * allocated.
476 *
477 * Note that the scheduler expects to 'inherit' its own reference to
478 * this fence from the callback. It does not invoke an extra
479 * dma_fence_get() on it. Consequently, this callback must take a
480 * reference for the scheduler, and additional ones for the driver's
481 * respective needs.
482 *
483 * Return:
484 * * On success: dma_fence the driver must signal once the hardware has
485 * completed the job ("hardware fence").
486 * * On failure: NULL or an ERR_PTR.
487 */
488 struct dma_fence *(*run_job)(struct drm_sched_job *sched_job);
489
490 /**
491 * @timedout_job: Called when a job has taken too long to execute,
492 * to trigger GPU recovery.
493 *
494 * @sched_job: The job that has timed out
495 *
496 * Drivers typically issue a reset to recover from GPU hangs.
497 * This procedure looks very different depending on whether a firmware
498 * or a hardware scheduler is being used.
499 *
500 * For a FIRMWARE SCHEDULER, each ring has one scheduler, and each
501 * scheduler has one entity. Hence, the steps taken typically look as
502 * follows:
503 *
504 * 1. Stop the scheduler using drm_sched_stop(). This will pause the
505 * scheduler workqueues and cancel the timeout work, guaranteeing
506 * that nothing is queued while the ring is being removed.
507 * 2. Remove the ring. The firmware will make sure that the
508 * corresponding parts of the hardware are resetted, and that other
509 * rings are not impacted.
510 * 3. Kill the entity and the associated scheduler.
511 *
512 *
513 * For a HARDWARE SCHEDULER, a scheduler instance schedules jobs from
514 * one or more entities to one ring. This implies that all entities
515 * associated with the affected scheduler cannot be torn down, because
516 * this would effectively also affect innocent userspace processes which
517 * did not submit faulty jobs (for example).
518 *
519 * Consequently, the procedure to recover with a hardware scheduler
520 * should look like this:
521 *
522 * 1. Stop all schedulers impacted by the reset using drm_sched_stop().
523 * 2. Kill the entity the faulty job stems from.
524 * 3. Issue a GPU reset on all faulty rings (driver-specific).
525 * 4. Re-submit jobs on all schedulers impacted by re-submitting them to
526 * the entities which are still alive.
527 * 5. Restart all schedulers that were stopped in step #1 using
528 * drm_sched_start().
529 *
530 * Note that some GPUs have distinct hardware queues but need to reset
531 * the GPU globally, which requires extra synchronization between the
532 * timeout handlers of different schedulers. One way to achieve this
533 * synchronization is to create an ordered workqueue (using
534 * alloc_ordered_workqueue()) at the driver level, and pass this queue
535 * as drm_sched_init()'s @timeout_wq parameter. This will guarantee
536 * that timeout handlers are executed sequentially.
537 *
538 * Return: The scheduler's status, defined by &enum drm_gpu_sched_stat
539 *
540 */
541 enum drm_gpu_sched_stat (*timedout_job)(struct drm_sched_job *sched_job);
542
543 /**
544 * @free_job: Called once the job's finished fence has been signaled
545 * and it's time to clean it up.
546 */
547 void (*free_job)(struct drm_sched_job *sched_job);
548
549 /**
550 * @cancel_job: Used by the scheduler to guarantee remaining jobs' fences
551 * get signaled in drm_sched_fini().
552 *
553 * Used by the scheduler to cancel all jobs that have not been executed
554 * with &struct drm_sched_backend_ops.run_job by the time
555 * drm_sched_fini() gets invoked.
556 *
557 * Drivers need to signal the passed job's hardware fence with an
558 * appropriate error code (e.g., -ECANCELED) in this callback. They
559 * must not free the job.
560 *
561 * The scheduler will only call this callback once it stopped calling
562 * all other callbacks forever, with the exception of &struct
563 * drm_sched_backend_ops.free_job.
564 */
565 void (*cancel_job)(struct drm_sched_job *sched_job);
566};
567
568/**
569 * struct drm_gpu_scheduler - scheduler instance-specific data
570 *
571 * @ops: backend operations provided by the driver.
572 * @credit_limit: the credit limit of this scheduler
573 * @credit_count: the current credit count of this scheduler
574 * @timeout: the time after which a job is removed from the scheduler.
575 * @name: name of the ring for which this scheduler is being used.
576 * @num_user_rqs: Number of run-queues. This is at most
577 * DRM_SCHED_PRIORITY_COUNT, as there's usually one run-queue per
578 * priority, but could be less.
579 * @num_rqs: Equal to @num_user_rqs for FIFO and RR and 1 for the FAIR policy.
580 * @sched_rq: An allocated array of run-queues of size @num_rqs;
581 * @job_scheduled: once drm_sched_entity_flush() is called the scheduler
582 * waits on this wait queue until all the scheduled jobs are
583 * finished.
584 * @job_id_count: used to assign unique id to the each job.
585 * @submit_wq: workqueue used to queue @work_run_job and @work_free_job
586 * @timeout_wq: workqueue used to queue @work_tdr
587 * @avg_job_us: Average job duration.
588 * @work_run_job: work which calls run_job op of each scheduler.
589 * @work_free_job: work which calls free_job op of each scheduler.
590 * @work_tdr: schedules a delayed call to @drm_sched_job_timedout after the
591 * timeout interval is over.
592 * @pending_list: the list of jobs which are currently in the job queue.
593 * @job_list_lock: lock to protect the pending_list.
594 * @hang_limit: once the hangs by a job crosses this limit then it is marked
595 * guilty and it will no longer be considered for scheduling.
596 * @score: score to help loadbalancer pick a idle sched
597 * @_score: score used when the driver doesn't provide one
598 * @ready: marks if the underlying HW is ready to work
599 * @free_guilty: A hit to time out handler to free the guilty job.
600 * @pause_submit: pause queuing of @work_run_job on @submit_wq
601 * @own_submit_wq: scheduler owns allocation of @submit_wq
602 * @dev: system &struct device
603 *
604 * One scheduler is implemented for each hardware ring.
605 */
606struct drm_gpu_scheduler {
607 const struct drm_sched_backend_ops *ops;
608 u32 credit_limit;
609 atomic_t credit_count;
610 long timeout;
611 const char *name;
612 u32 num_rqs;
613 u32 num_user_rqs;
614 struct drm_sched_rq **sched_rq;
615 wait_queue_head_t job_scheduled;
616 atomic64_t job_id_count;
617 struct workqueue_struct *submit_wq;
618 struct workqueue_struct *timeout_wq;
619 struct ewma_drm_sched_avgtime avg_job_us;
620 struct work_struct work_run_job;
621 struct work_struct work_free_job;
622 struct delayed_work work_tdr;
623 struct list_head pending_list;
624 spinlock_t job_list_lock;
625 int hang_limit;
626 atomic_t *score;
627 atomic_t _score;
628 bool ready;
629 bool free_guilty;
630 bool pause_submit;
631 bool own_submit_wq;
632 struct device *dev;
633};
634
635/**
636 * struct drm_sched_init_args - parameters for initializing a DRM GPU scheduler
637 *
638 * @ops: backend operations provided by the driver
639 * @submit_wq: workqueue to use for submission. If NULL, an ordered wq is
640 * allocated and used.
641 * @num_rqs: Number of run-queues. This may be at most DRM_SCHED_PRIORITY_COUNT,
642 * as there's usually one run-queue per priority, but may be less.
643 * @credit_limit: the number of credits this scheduler can hold from all jobs
644 * @hang_limit: number of times to allow a job to hang before dropping it.
645 * This mechanism is DEPRECATED. Set it to 0.
646 * @timeout: timeout value in jiffies for submitted jobs.
647 * @timeout_wq: workqueue to use for timeout work. If NULL, the system_wq is used.
648 * @score: score atomic shared with other schedulers. May be NULL.
649 * @name: name (typically the driver's name). Used for debugging
650 * @dev: associated device. Used for debugging
651 */
652struct drm_sched_init_args {
653 const struct drm_sched_backend_ops *ops;
654 struct workqueue_struct *submit_wq;
655 struct workqueue_struct *timeout_wq;
656 u32 num_rqs;
657 u32 credit_limit;
658 unsigned int hang_limit;
659 long timeout;
660 atomic_t *score;
661 const char *name;
662 struct device *dev;
663};
664
665/* Scheduler operations */
666
667int drm_sched_init(struct drm_gpu_scheduler *sched,
668 const struct drm_sched_init_args *args);
669
670void drm_sched_fini(struct drm_gpu_scheduler *sched);
671
672unsigned long drm_sched_suspend_timeout(struct drm_gpu_scheduler *sched);
673void drm_sched_resume_timeout(struct drm_gpu_scheduler *sched,
674 unsigned long remaining);
675void drm_sched_tdr_queue_imm(struct drm_gpu_scheduler *sched);
676bool drm_sched_wqueue_ready(struct drm_gpu_scheduler *sched);
677void drm_sched_wqueue_stop(struct drm_gpu_scheduler *sched);
678void drm_sched_wqueue_start(struct drm_gpu_scheduler *sched);
679void drm_sched_stop(struct drm_gpu_scheduler *sched, struct drm_sched_job *bad);
680void drm_sched_start(struct drm_gpu_scheduler *sched, int errno);
681void drm_sched_resubmit_jobs(struct drm_gpu_scheduler *sched);
682void drm_sched_fault(struct drm_gpu_scheduler *sched);
683bool drm_sched_is_stopped(struct drm_gpu_scheduler *sched);
684
685struct drm_gpu_scheduler *
686drm_sched_pick_best(struct drm_gpu_scheduler **sched_list,
687 unsigned int num_sched_list);
688
689/* Jobs */
690
691int drm_sched_job_init(struct drm_sched_job *job,
692 struct drm_sched_entity *entity,
693 u32 credits, void *owner,
694 u64 drm_client_id);
695void drm_sched_job_arm(struct drm_sched_job *job);
696void drm_sched_entity_push_job(struct drm_sched_job *sched_job);
697int drm_sched_job_add_dependency(struct drm_sched_job *job,
698 struct dma_fence *fence);
699int drm_sched_job_add_syncobj_dependency(struct drm_sched_job *job,
700 struct drm_file *file,
701 u32 handle,
702 u32 point);
703int drm_sched_job_add_resv_dependencies(struct drm_sched_job *job,
704 struct dma_resv *resv,
705 enum dma_resv_usage usage);
706int drm_sched_job_add_implicit_dependencies(struct drm_sched_job *job,
707 struct drm_gem_object *obj,
708 bool write);
709bool drm_sched_job_has_dependency(struct drm_sched_job *job,
710 struct dma_fence *fence);
711void drm_sched_job_cleanup(struct drm_sched_job *job);
712void drm_sched_increase_karma(struct drm_sched_job *bad);
713bool drm_sched_job_is_signaled(struct drm_sched_job *job);
714
715static inline bool drm_sched_invalidate_job(struct drm_sched_job *s_job,
716 int threshold)
717{
718 return s_job && atomic_inc_return(&s_job->karma) > threshold;
719}
720
721/* Entities */
722
723int drm_sched_entity_init(struct drm_sched_entity *entity,
724 enum drm_sched_priority priority,
725 struct drm_gpu_scheduler **sched_list,
726 unsigned int num_sched_list,
727 atomic_t *guilty);
728long drm_sched_entity_flush(struct drm_sched_entity *entity, long timeout);
729void drm_sched_entity_kill(struct drm_sched_entity *entity);
730void drm_sched_entity_fini(struct drm_sched_entity *entity);
731void drm_sched_entity_destroy(struct drm_sched_entity *entity);
732void drm_sched_entity_set_priority(struct drm_sched_entity *entity,
733 enum drm_sched_priority priority);
734int drm_sched_entity_error(struct drm_sched_entity *entity);
735void drm_sched_entity_modify_sched(struct drm_sched_entity *entity,
736 struct drm_gpu_scheduler **sched_list,
737 unsigned int num_sched_list);
738
739/**
740 * struct drm_sched_pending_job_iter - DRM scheduler pending job iterator state
741 * @sched: DRM scheduler associated with pending job iterator
742 */
743struct drm_sched_pending_job_iter {
744 struct drm_gpu_scheduler *sched;
745};
746
747/* Drivers should never call this directly */
748static inline struct drm_sched_pending_job_iter
749__drm_sched_pending_job_iter_begin(struct drm_gpu_scheduler *sched)
750{
751 struct drm_sched_pending_job_iter iter = {
752 .sched = sched,
753 };
754
755 WARN_ON(!drm_sched_is_stopped(sched));
756 return iter;
757}
758
759/* Drivers should never call this directly */
760static inline void
761__drm_sched_pending_job_iter_end(const struct drm_sched_pending_job_iter iter)
762{
763 WARN_ON(!drm_sched_is_stopped(iter.sched));
764}
765
766DEFINE_CLASS(drm_sched_pending_job_iter, struct drm_sched_pending_job_iter,
767 __drm_sched_pending_job_iter_end(_T),
768 __drm_sched_pending_job_iter_begin(__sched),
769 struct drm_gpu_scheduler *__sched);
770static inline void *
771class_drm_sched_pending_job_iter_lock_ptr(class_drm_sched_pending_job_iter_t *_T)
772{ return _T; }
773#define class_drm_sched_pending_job_iter_is_conditional false
774
775/**
776 * drm_sched_for_each_pending_job() - Iterator for each pending job in scheduler
777 * @__job: Current pending job being iterated over
778 * @__sched: DRM scheduler to iterate over pending jobs
779 * @__entity: DRM scheduler entity to filter jobs, NULL indicates no filter
780 *
781 * Iterator for each pending job in scheduler, filtering on an entity, and
782 * enforcing scheduler is fully stopped
783 */
784#define drm_sched_for_each_pending_job(__job, __sched, __entity) \
785 scoped_guard(drm_sched_pending_job_iter, (__sched)) \
786 list_for_each_entry((__job), &(__sched)->pending_list, list) \
787 for_each_if(!(__entity) || (__job)->entity == (__entity))
788
789#endif