projects/linux·blob master · refs

3242 lines · 84.1 KiB

1/*
2 * Copyright 2008 Advanced Micro Devices, Inc.
3 * Copyright 2008 Red Hat Inc.
4 * Copyright 2009 Jerome Glisse.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
22 * OTHER DEALINGS IN THE SOFTWARE.
23 *
24 * Authors: Dave Airlie
25 * Alex Deucher
26 * Jerome Glisse
27 */
28
29#include <linux/dma-fence-array.h>
30#include <linux/interval_tree_generic.h>
31#include <linux/idr.h>
32#include <linux/dma-buf.h>
33
34#include <drm/amdgpu_drm.h>
35#include <drm/drm_drv.h>
36#include <drm/ttm/ttm_tt.h>
37#include <drm/drm_exec.h>
38#include "amdgpu.h"
39#include "amdgpu_vm.h"
40#include "amdgpu_trace.h"
41#include "amdgpu_amdkfd.h"
42#include "amdgpu_gmc.h"
43#include "amdgpu_xgmi.h"
44#include "amdgpu_dma_buf.h"
45#include "amdgpu_res_cursor.h"
46#include "kfd_svm.h"
47
48/**
49 * DOC: GPUVM
50 *
51 * GPUVM is the MMU functionality provided on the GPU.
52 * GPUVM is similar to the legacy GART on older asics, however
53 * rather than there being a single global GART table
54 * for the entire GPU, there can be multiple GPUVM page tables active
55 * at any given time. The GPUVM page tables can contain a mix
56 * VRAM pages and system pages (both memory and MMIO) and system pages
57 * can be mapped as snooped (cached system pages) or unsnooped
58 * (uncached system pages).
59 *
60 * Each active GPUVM has an ID associated with it and there is a page table
61 * linked with each VMID. When executing a command buffer,
62 * the kernel tells the engine what VMID to use for that command
63 * buffer. VMIDs are allocated dynamically as commands are submitted.
64 * The userspace drivers maintain their own address space and the kernel
65 * sets up their pages tables accordingly when they submit their
66 * command buffers and a VMID is assigned.
67 * The hardware supports up to 16 active GPUVMs at any given time.
68 *
69 * Each GPUVM is represented by a 1-2 or 1-5 level page table, depending
70 * on the ASIC family. GPUVM supports RWX attributes on each page as well
71 * as other features such as encryption and caching attributes.
72 *
73 * VMID 0 is special. It is the GPUVM used for the kernel driver. In
74 * addition to an aperture managed by a page table, VMID 0 also has
75 * several other apertures. There is an aperture for direct access to VRAM
76 * and there is a legacy AGP aperture which just forwards accesses directly
77 * to the matching system physical addresses (or IOVAs when an IOMMU is
78 * present). These apertures provide direct access to these memories without
79 * incurring the overhead of a page table. VMID 0 is used by the kernel
80 * driver for tasks like memory management.
81 *
82 * GPU clients (i.e., engines on the GPU) use GPUVM VMIDs to access memory.
83 * For user applications, each application can have their own unique GPUVM
84 * address space. The application manages the address space and the kernel
85 * driver manages the GPUVM page tables for each process. If an GPU client
86 * accesses an invalid page, it will generate a GPU page fault, similar to
87 * accessing an invalid page on a CPU.
88 */
89
90#define START(node) ((node)->start)
91#define LAST(node) ((node)->last)
92
93INTERVAL_TREE_DEFINE(struct amdgpu_bo_va_mapping, rb, uint64_t, __subtree_last,
94 START, LAST, static, amdgpu_vm_it)
95
96#undef START
97#undef LAST
98
99/**
100 * struct amdgpu_prt_cb - Helper to disable partial resident texture feature from a fence callback
101 */
102struct amdgpu_prt_cb {
103
104 /**
105 * @adev: amdgpu device
106 */
107 struct amdgpu_device *adev;
108
109 /**
110 * @cb: callback
111 */
112 struct dma_fence_cb cb;
113};
114
115/**
116 * struct amdgpu_vm_tlb_seq_struct - Helper to increment the TLB flush sequence
117 */
118struct amdgpu_vm_tlb_seq_struct {
119 /**
120 * @vm: pointer to the amdgpu_vm structure to set the fence sequence on
121 */
122 struct amdgpu_vm *vm;
123
124 /**
125 * @cb: callback
126 */
127 struct dma_fence_cb cb;
128};
129
130/**
131 * amdgpu_vm_assert_locked - check if VM is correctly locked
132 * @vm: the VM which schould be tested
133 *
134 * Asserts that the VM root PD is locked.
135 */
136static void amdgpu_vm_assert_locked(struct amdgpu_vm *vm)
137{
138 dma_resv_assert_held(vm->root.bo->tbo.base.resv);
139}
140
141/* Initialize the amdgpu_vm_bo_status object */
142static void amdgpu_vm_bo_status_init(struct amdgpu_vm_bo_status *lists)
143{
144 INIT_LIST_HEAD(&lists->evicted);
145 INIT_LIST_HEAD(&lists->needs_update);
146 INIT_LIST_HEAD(&lists->idle);
147}
148
149/*
150 * Make sure we have the lock to modify the vm_bo status and return the object
151 * with the status lists.
152 */
153static struct amdgpu_vm_bo_status *
154amdgpu_vm_bo_lock_lists(struct amdgpu_vm_bo_base *vm_bo)
155{
156 struct amdgpu_vm *vm = vm_bo->vm;
157 struct amdgpu_bo *bo = vm_bo->bo;
158
159 if (amdgpu_vm_is_bo_always_valid(vm, bo)) {
160 /* No extra locking needed, protected by the root PD resv lock */
161 amdgpu_vm_assert_locked(vm);
162
163 if (bo->tbo.type == ttm_bo_type_kernel)
164 return &vm->kernel;
165
166 return &vm->always_valid;
167 }
168
169 spin_lock(&vm_bo->vm->individual_lock);
170 return &vm->individual;
171}
172
173/* Eventually unlock the status list lock again */
174static void amdgpu_vm_bo_unlock_lists(struct amdgpu_vm_bo_base *vm_bo)
175{
176 if (amdgpu_vm_is_bo_always_valid(vm_bo->vm, vm_bo->bo))
177 amdgpu_vm_assert_locked(vm_bo->vm);
178 else
179 spin_unlock(&vm_bo->vm->individual_lock);
180}
181
182/**
183 * amdgpu_vm_is_bo_always_valid - check if the BO is VM always valid
184 *
185 * @vm: VM to test against.
186 * @bo: BO to be tested.
187 *
188 * Returns true if the BO shares the dma_resv object with the root PD and is
189 * always guaranteed to be valid inside the VM.
190 */
191bool amdgpu_vm_is_bo_always_valid(struct amdgpu_vm *vm, struct amdgpu_bo *bo)
192{
193 return bo && bo->tbo.base.resv == vm->root.bo->tbo.base.resv;
194}
195
196/**
197 * amdgpu_vm_bo_evicted - vm_bo is evicted
198 *
199 * @vm_bo: vm_bo which is evicted
200 *
201 * State for vm_bo objects meaning the underlying BO was evicted and need to
202 * move in place again.
203 */
204static void amdgpu_vm_bo_evicted(struct amdgpu_vm_bo_base *vm_bo)
205{
206 struct amdgpu_vm_bo_status *lists;
207
208 lists = amdgpu_vm_bo_lock_lists(vm_bo);
209 vm_bo->moved = true;
210 list_move(&vm_bo->vm_status, &lists->evicted);
211 amdgpu_vm_bo_unlock_lists(vm_bo);
212}
213/**
214 * amdgpu_vm_bo_needs_update - vm_bo needs pagetable update
215 *
216 * @vm_bo: vm_bo which is out of date
217 *
218 * State for vm_bo objects meaning the underlying BO had mapping changes (move, PRT bind/unbind)
219 * but the new location is not yet reflected in the page tables.
220 */
221static void amdgpu_vm_bo_needs_update(struct amdgpu_vm_bo_base *vm_bo)
222{
223 struct amdgpu_vm_bo_status *lists;
224 struct amdgpu_bo *bo = vm_bo->bo;
225
226 /*
227 * The root PD doesn't have a parent PDE and goes directly into the
228 * idle state.
229 */
230 lists = amdgpu_vm_bo_lock_lists(vm_bo);
231 if (bo && bo->tbo.type == ttm_bo_type_kernel && !bo->parent) {
232 vm_bo->moved = false;
233 list_move(&vm_bo->vm_status, &lists->idle);
234 } else {
235 list_move(&vm_bo->vm_status, &lists->needs_update);
236 }
237 amdgpu_vm_bo_unlock_lists(vm_bo);
238}
239
240/**
241 * amdgpu_vm_bo_idle - vm_bo is idle
242 *
243 * @vm_bo: vm_bo which is now idle
244 *
245 * State for vm_bo objects meaning we are done with the state machine and no
246 * further action is necessary.
247 */
248static void amdgpu_vm_bo_idle(struct amdgpu_vm_bo_base *vm_bo)
249{
250 struct amdgpu_vm_bo_status *lists;
251
252 lists = amdgpu_vm_bo_lock_lists(vm_bo);
253 if (!amdgpu_vm_is_bo_always_valid(vm_bo->vm, vm_bo->bo))
254 vm_bo->moved = false;
255 list_move(&vm_bo->vm_status, &lists->idle);
256 amdgpu_vm_bo_unlock_lists(vm_bo);
257}
258
259/**
260 * amdgpu_vm_bo_reset_state_machine - reset the vm_bo state machine
261 * @vm: the VM which state machine to reset
262 *
263 * Move all vm_bo object in the VM into a state where their location will be
264 * updated in the page tables again.
265 */
266static void amdgpu_vm_bo_reset_state_machine(struct amdgpu_vm *vm)
267{
268 struct amdgpu_vm_bo_base *vm_bo, *tmp;
269
270 /*
271 * Don't use list splice here, we need the special handling for the root
272 * PD and set the moved flag appropriately.
273 */
274 amdgpu_vm_assert_locked(vm);
275 list_for_each_entry_safe(vm_bo, tmp, &vm->kernel.idle, vm_status)
276 amdgpu_vm_bo_needs_update(vm_bo);
277 list_for_each_entry_safe(vm_bo, tmp, &vm->always_valid.idle, vm_status)
278 amdgpu_vm_bo_needs_update(vm_bo);
279
280 spin_lock(&vm->individual_lock);
281 list_for_each_entry_safe(vm_bo, tmp, &vm->individual.idle, vm_status) {
282 vm_bo->moved = true;
283 list_move(&vm_bo->vm_status, &vm->individual.needs_update);
284 }
285 spin_unlock(&vm->individual_lock);
286}
287
288/**
289 * amdgpu_vm_update_shared - helper to update shared memory stat
290 * @base: base structure for tracking BO usage in a VM
291 *
292 * Takes the vm stats_lock and updates the shared memory stat. If the basic
293 * stat changed (e.g. buffer was moved) amdgpu_vm_update_stats need to be called
294 * as well.
295 */
296static void amdgpu_vm_update_shared(struct amdgpu_vm_bo_base *base)
297{
298 struct amdgpu_vm *vm = base->vm;
299 struct amdgpu_bo *bo = base->bo;
300 uint64_t size = amdgpu_bo_size(bo);
301 uint32_t bo_memtype = amdgpu_bo_mem_stats_placement(bo);
302 bool shared;
303
304 dma_resv_assert_held(bo->tbo.base.resv);
305 spin_lock(&vm->stats_lock);
306 shared = drm_gem_object_is_shared_for_memory_stats(&bo->tbo.base);
307 if (base->shared != shared) {
308 base->shared = shared;
309 if (shared) {
310 vm->stats[bo_memtype].drm.shared += size;
311 vm->stats[bo_memtype].drm.private -= size;
312 } else {
313 vm->stats[bo_memtype].drm.shared -= size;
314 vm->stats[bo_memtype].drm.private += size;
315 }
316 }
317 spin_unlock(&vm->stats_lock);
318}
319
320/**
321 * amdgpu_vm_bo_update_shared - callback when bo gets shared/unshared
322 * @bo: amdgpu buffer object
323 *
324 * Update the per VM stats for all the vm if needed from private to shared or
325 * vice versa.
326 */
327void amdgpu_vm_bo_update_shared(struct amdgpu_bo *bo)
328{
329 struct amdgpu_vm_bo_base *base;
330
331 for (base = bo->vm_bo; base; base = base->next)
332 amdgpu_vm_update_shared(base);
333}
334
335/**
336 * amdgpu_vm_update_stats_locked - helper to update normal memory stat
337 * @base: base structure for tracking BO usage in a VM
338 * @res: the ttm_resource to use for the purpose of accounting, may or may not
339 * be bo->tbo.resource
340 * @sign: if we should add (+1) or subtract (-1) from the stat
341 *
342 * Caller need to have the vm stats_lock held. Useful for when multiple update
343 * need to happen at the same time.
344 */
345static void amdgpu_vm_update_stats_locked(struct amdgpu_vm_bo_base *base,
346 struct ttm_resource *res, int sign)
347{
348 struct amdgpu_vm *vm = base->vm;
349 struct amdgpu_bo *bo = base->bo;
350 int64_t size = sign * amdgpu_bo_size(bo);
351 uint32_t bo_memtype = amdgpu_bo_mem_stats_placement(bo);
352
353 /* For drm-total- and drm-shared-, BO are accounted by their preferred
354 * placement, see also amdgpu_bo_mem_stats_placement.
355 */
356 if (base->shared)
357 vm->stats[bo_memtype].drm.shared += size;
358 else
359 vm->stats[bo_memtype].drm.private += size;
360
361 if (res && res->mem_type < __AMDGPU_PL_NUM) {
362 uint32_t res_memtype = res->mem_type;
363
364 vm->stats[res_memtype].drm.resident += size;
365 /* BO only count as purgeable if it is resident,
366 * since otherwise there's nothing to purge.
367 */
368 if (bo->flags & AMDGPU_GEM_CREATE_DISCARDABLE)
369 vm->stats[res_memtype].drm.purgeable += size;
370 if (!(bo->preferred_domains &
371 amdgpu_mem_type_to_domain(res_memtype)))
372 vm->stats[bo_memtype].evicted += size;
373 }
374}
375
376/**
377 * amdgpu_vm_update_stats - helper to update normal memory stat
378 * @base: base structure for tracking BO usage in a VM
379 * @res: the ttm_resource to use for the purpose of accounting, may or may not
380 * be bo->tbo.resource
381 * @sign: if we should add (+1) or subtract (-1) from the stat
382 *
383 * Updates the basic memory stat when bo is added/deleted/moved.
384 */
385void amdgpu_vm_update_stats(struct amdgpu_vm_bo_base *base,
386 struct ttm_resource *res, int sign)
387{
388 struct amdgpu_vm *vm = base->vm;
389
390 spin_lock(&vm->stats_lock);
391 amdgpu_vm_update_stats_locked(base, res, sign);
392 spin_unlock(&vm->stats_lock);
393}
394
395/**
396 * amdgpu_vm_bo_base_init - Adds bo to the list of bos associated with the vm
397 *
398 * @base: base structure for tracking BO usage in a VM
399 * @vm: vm to which bo is to be added
400 * @bo: amdgpu buffer object
401 *
402 * Initialize a bo_va_base structure and add it to the appropriate lists
403 *
404 */
405void amdgpu_vm_bo_base_init(struct amdgpu_vm_bo_base *base,
406 struct amdgpu_vm *vm, struct amdgpu_bo *bo)
407{
408 base->vm = vm;
409 base->bo = bo;
410 base->next = NULL;
411 INIT_LIST_HEAD(&base->vm_status);
412
413 dma_resv_assert_held(vm->root.bo->tbo.base.resv);
414 if (!bo)
415 return;
416
417 base->next = bo->vm_bo;
418 bo->vm_bo = base;
419
420 spin_lock(&vm->stats_lock);
421 base->shared = drm_gem_object_is_shared_for_memory_stats(&bo->tbo.base);
422 amdgpu_vm_update_stats_locked(base, bo->tbo.resource, +1);
423 spin_unlock(&vm->stats_lock);
424
425 if (!amdgpu_vm_is_bo_always_valid(vm, bo)) {
426 amdgpu_vm_bo_idle(base);
427 return;
428 }
429
430 ttm_bo_set_bulk_move(&bo->tbo, &vm->lru_bulk_move);
431
432 /*
433 * When a per VM isn't in the desired domain put it into the evicted
434 * state to make sure that it gets validated on the next best occasion.
435 */
436 if (bo->preferred_domains &
437 amdgpu_mem_type_to_domain(bo->tbo.resource->mem_type))
438 amdgpu_vm_bo_needs_update(base);
439 else
440 amdgpu_vm_bo_evicted(base);
441}
442
443/**
444 * amdgpu_vm_lock_pd - lock PD in drm_exec
445 *
446 * @vm: vm providing the BOs
447 * @exec: drm execution context
448 * @num_fences: number of extra fences to reserve
449 *
450 * Lock the VM root PD in the DRM execution context.
451 */
452int amdgpu_vm_lock_pd(struct amdgpu_vm *vm, struct drm_exec *exec,
453 unsigned int num_fences)
454{
455 /* We need at least two fences for the VM PD/PT updates */
456 return drm_exec_prepare_obj(exec, &vm->root.bo->tbo.base,
457 2 + num_fences);
458}
459
460/**
461 * amdgpu_vm_lock_individual - lock all BOs on the individual idle list
462 * @vm: vm providing the BOs
463 * @exec: drm execution context
464 * @num_fences: number of extra fences to reserve
465 *
466 * Lock the BOs on the individual idle list in the DRM execution context.
467 */
468int amdgpu_vm_lock_individual(struct amdgpu_vm *vm, struct drm_exec *exec,
469 unsigned int num_fences)
470{
471 struct list_head *prev = &vm->individual.idle;
472 struct amdgpu_bo_va *bo_va;
473 struct amdgpu_bo *bo;
474 int ret;
475
476 /* We can only trust prev->next while holding the lock */
477 spin_lock(&vm->individual_lock);
478 while (!list_is_head(prev->next, &vm->individual.idle)) {
479 bo_va = list_entry(prev->next, typeof(*bo_va), base.vm_status);
480
481 bo = bo_va->base.bo;
482 if (bo) {
483 amdgpu_bo_ref(bo);
484 spin_unlock(&vm->individual_lock);
485
486 ret = drm_exec_prepare_obj(exec, &bo->tbo.base, num_fences);
487 amdgpu_bo_unref(&bo);
488 if (unlikely(ret))
489 return ret;
490
491 spin_lock(&vm->individual_lock);
492 }
493 prev = prev->next;
494 }
495 spin_unlock(&vm->individual_lock);
496
497 return 0;
498}
499
500/**
501 * amdgpu_vm_move_to_lru_tail - move all BOs to the end of LRU
502 *
503 * @adev: amdgpu device pointer
504 * @vm: vm providing the BOs
505 *
506 * Move all BOs to the end of LRU and remember their positions to put them
507 * together.
508 */
509void amdgpu_vm_move_to_lru_tail(struct amdgpu_device *adev,
510 struct amdgpu_vm *vm)
511{
512 spin_lock(&adev->mman.bdev.lru_lock);
513 ttm_lru_bulk_move_tail(&vm->lru_bulk_move);
514 spin_unlock(&adev->mman.bdev.lru_lock);
515}
516
517/* Create scheduler entities for page table updates */
518static int amdgpu_vm_init_entities(struct amdgpu_device *adev,
519 struct amdgpu_vm *vm)
520{
521 int r;
522
523 r = drm_sched_entity_init(&vm->immediate, DRM_SCHED_PRIORITY_NORMAL,
524 adev->vm_manager.vm_pte_scheds,
525 adev->vm_manager.vm_pte_num_scheds, NULL);
526 if (r)
527 goto error;
528
529 return drm_sched_entity_init(&vm->delayed, DRM_SCHED_PRIORITY_NORMAL,
530 adev->vm_manager.vm_pte_scheds,
531 adev->vm_manager.vm_pte_num_scheds, NULL);
532
533error:
534 drm_sched_entity_destroy(&vm->immediate);
535 return r;
536}
537
538/* Destroy the entities for page table updates again */
539static void amdgpu_vm_fini_entities(struct amdgpu_vm *vm)
540{
541 drm_sched_entity_destroy(&vm->immediate);
542 drm_sched_entity_destroy(&vm->delayed);
543}
544
545/**
546 * amdgpu_vm_generation - return the page table re-generation counter
547 * @adev: the amdgpu_device
548 * @vm: optional VM to check, might be NULL
549 *
550 * Returns a page table re-generation token to allow checking if submissions
551 * are still valid to use this VM. The VM parameter might be NULL in which case
552 * just the VRAM lost counter will be used.
553 */
554uint64_t amdgpu_vm_generation(struct amdgpu_device *adev, struct amdgpu_vm *vm)
555{
556 uint64_t result = (u64)atomic_read(&adev->vram_lost_counter) << 32;
557
558 if (!vm)
559 return result;
560
561 result += lower_32_bits(vm->generation);
562 /* Add one if the page tables will be re-generated on next CS */
563 if (drm_sched_entity_error(&vm->delayed))
564 ++result;
565
566 return result;
567}
568
569/**
570 * amdgpu_vm_validate - validate evicted BOs tracked in the VM
571 *
572 * @adev: amdgpu device pointer
573 * @vm: vm providing the BOs
574 * @ticket: optional reservation ticket used to reserve the VM
575 * @validate: callback to do the validation
576 * @param: parameter for the validation callback
577 *
578 * Validate the page table BOs and per-VM BOs on command submission if
579 * necessary. If a ticket is given, also try to validate evicted user queue
580 * BOs. They must already be reserved with the given ticket.
581 *
582 * Returns:
583 * Validation result.
584 */
585int amdgpu_vm_validate(struct amdgpu_device *adev, struct amdgpu_vm *vm,
586 struct ww_acquire_ctx *ticket,
587 int (*validate)(void *p, struct amdgpu_bo *bo),
588 void *param)
589{
590 uint64_t new_vm_generation = amdgpu_vm_generation(adev, vm);
591 struct amdgpu_vm_bo_base *bo_base, *tmp;
592 int r;
593
594 dma_resv_assert_held(vm->root.bo->tbo.base.resv);
595 if (vm->generation != new_vm_generation) {
596 vm->generation = new_vm_generation;
597 amdgpu_vm_bo_reset_state_machine(vm);
598 amdgpu_vm_fini_entities(vm);
599 r = amdgpu_vm_init_entities(adev, vm);
600 if (r)
601 return r;
602 }
603
604 list_for_each_entry_safe(bo_base, tmp, &vm->kernel.evicted, vm_status) {
605 r = validate(param, bo_base->bo);
606 if (r)
607 return r;
608
609 vm->update_funcs->map_table(to_amdgpu_bo_vm(bo_base->bo));
610 bo_base->moved = true;
611 amdgpu_vm_bo_needs_update(bo_base);
612 }
613
614 /*
615 * As soon as all page tables are in place we can start updating them
616 * again.
617 */
618 amdgpu_vm_eviction_lock(vm);
619 vm->evicting = false;
620 amdgpu_vm_eviction_unlock(vm);
621
622 list_for_each_entry_safe(bo_base, tmp, &vm->always_valid.evicted,
623 vm_status) {
624 r = validate(param, bo_base->bo);
625 if (r)
626 return r;
627
628 bo_base->moved = true;
629 amdgpu_vm_bo_needs_update(bo_base);
630 }
631
632 if (!ticket)
633 return 0;
634
635 spin_lock(&vm->individual_lock);
636restart:
637 list_for_each_entry(bo_base, &vm->individual.evicted, vm_status) {
638 struct amdgpu_bo *bo = bo_base->bo;
639
640 if (dma_resv_locking_ctx(bo->tbo.base.resv) != ticket)
641 continue;
642
643 spin_unlock(&vm->individual_lock);
644
645 r = validate(param, bo);
646 if (r)
647 return r;
648
649 bo_base->moved = true;
650 amdgpu_vm_bo_needs_update(bo_base);
651
652 /* It's a bit inefficient to always jump back to the start, but
653 * we would need to re-structure the KFD for properly fixing
654 * that.
655 */
656 spin_lock(&vm->individual_lock);
657 goto restart;
658 }
659 spin_unlock(&vm->individual_lock);
660
661 return 0;
662}
663
664/**
665 * amdgpu_vm_ready - check VM is ready for updates
666 *
667 * @vm: VM to check
668 *
669 * Check if all VM PDs/PTs are ready for updates
670 *
671 * Returns:
672 * True if VM is not evicting and all VM entities are not stopped
673 */
674bool amdgpu_vm_ready(struct amdgpu_vm *vm)
675{
676 bool ret;
677
678 amdgpu_vm_assert_locked(vm);
679
680 amdgpu_vm_eviction_lock(vm);
681 ret = !vm->evicting;
682 amdgpu_vm_eviction_unlock(vm);
683
684 ret &= list_empty(&vm->kernel.evicted);
685
686 spin_lock(&vm->immediate.lock);
687 ret &= !vm->immediate.stopped;
688 spin_unlock(&vm->immediate.lock);
689
690 spin_lock(&vm->delayed.lock);
691 ret &= !vm->delayed.stopped;
692 spin_unlock(&vm->delayed.lock);
693
694 return ret;
695}
696
697/**
698 * amdgpu_vm_check_compute_bug - check whether asic has compute vm bug
699 *
700 * @adev: amdgpu_device pointer
701 */
702void amdgpu_vm_check_compute_bug(struct amdgpu_device *adev)
703{
704 const struct amdgpu_ip_block *ip_block;
705 bool has_compute_vm_bug;
706 struct amdgpu_ring *ring;
707 int i;
708
709 has_compute_vm_bug = false;
710
711 ip_block = amdgpu_device_ip_get_ip_block(adev, AMD_IP_BLOCK_TYPE_GFX);
712 if (ip_block) {
713 /* Compute has a VM bug for GFX version < 7.
714 Compute has a VM bug for GFX 8 MEC firmware version < 673.*/
715 if (ip_block->version->major <= 7)
716 has_compute_vm_bug = true;
717 else if (ip_block->version->major == 8)
718 if (adev->gfx.mec_fw_version < 673)
719 has_compute_vm_bug = true;
720 }
721
722 for (i = 0; i < adev->num_rings; i++) {
723 ring = adev->rings[i];
724 if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE)
725 /* only compute rings */
726 ring->has_compute_vm_bug = has_compute_vm_bug;
727 else
728 ring->has_compute_vm_bug = false;
729 }
730}
731
732/**
733 * amdgpu_vm_need_pipeline_sync - Check if pipe sync is needed for job.
734 *
735 * @ring: ring on which the job will be submitted
736 * @job: job to submit
737 *
738 * Returns:
739 * True if sync is needed.
740 */
741bool amdgpu_vm_need_pipeline_sync(struct amdgpu_ring *ring,
742 struct amdgpu_job *job)
743{
744 struct amdgpu_device *adev = ring->adev;
745 unsigned vmhub = ring->vm_hub;
746 struct amdgpu_vmid_mgr *id_mgr = &adev->vm_manager.id_mgr[vmhub];
747
748 if (job->vmid == 0)
749 return false;
750
751 if (job->vm_needs_flush || ring->has_compute_vm_bug)
752 return true;
753
754 if (ring->funcs->emit_gds_switch && job->gds_switch_needed)
755 return true;
756
757 if (amdgpu_vmid_had_gpu_reset(adev, &id_mgr->ids[job->vmid]))
758 return true;
759
760 return false;
761}
762
763/**
764 * amdgpu_vm_flush - hardware flush the vm
765 *
766 * @ring: ring to use for flush
767 * @job: related job
768 * @need_pipe_sync: is pipe sync needed
769 * @emit_spm_needed: does the caller need to emit spm
770 * @emit_gds_needed: does the caller need to emit gds
771 *
772 * Emit a VM flush when it is necessary.
773 */
774void amdgpu_vm_flush(struct amdgpu_ring *ring, struct amdgpu_job *job,
775 bool *need_pipe_sync, bool *emit_spm_needed,
776 bool *emit_gds_needed)
777{
778 struct amdgpu_device *adev = ring->adev;
779 struct amdgpu_isolation *isolation =
780 &adev->isolation[ring->xcp_id == AMDGPU_XCP_NO_PARTITION ?
781 0 : ring->xcp_id];
782 unsigned vmhub = ring->vm_hub;
783 struct amdgpu_vmid_mgr *id_mgr = &adev->vm_manager.id_mgr[vmhub];
784 struct amdgpu_vmid *id = &id_mgr->ids[job->vmid];
785 bool spm_update_needed = adev->gfx.rlc.funcs->update_spm_vmid &&
786 job->spm_update_needed;
787 bool gds_switch_needed = ring->funcs->emit_gds_switch &&
788 job->gds_switch_needed;
789 bool vm_flush_needed = job->vm_needs_flush;
790 bool cleaner_shader_needed = false;
791 bool pasid_mapping_needed = false;
792 struct dma_fence *fence = NULL;
793 unsigned int patch = 0;
794 bool emit_fence;
795
796 if (amdgpu_vmid_had_gpu_reset(adev, id)) {
797 gds_switch_needed = true;
798 vm_flush_needed = true;
799 pasid_mapping_needed = true;
800 spm_update_needed = true;
801 }
802
803 mutex_lock(&id_mgr->lock);
804 if (id->pasid != job->pasid || !id->pasid_mapping ||
805 !dma_fence_is_signaled(id->pasid_mapping))
806 pasid_mapping_needed = true;
807 mutex_unlock(&id_mgr->lock);
808
809 gds_switch_needed &= !!ring->funcs->emit_gds_switch;
810 spm_update_needed &= !!adev->gfx.rlc.funcs->update_spm_vmid;
811 vm_flush_needed &= !!ring->funcs->emit_vm_flush &&
812 job->vm_pd_addr != AMDGPU_BO_INVALID_OFFSET;
813 pasid_mapping_needed &= adev->gmc.gmc_funcs->emit_pasid_mapping &&
814 ring->funcs->emit_wreg;
815
816 cleaner_shader_needed = job->run_cleaner_shader &&
817 adev->gfx.enable_cleaner_shader &&
818 ring->funcs->emit_cleaner_shader && job->base.s_fence &&
819 &job->base.s_fence->scheduled == isolation->spearhead;
820
821 emit_fence = vm_flush_needed || pasid_mapping_needed ||
822 cleaner_shader_needed;
823
824 *emit_spm_needed = spm_update_needed;
825 if (spm_update_needed && emit_fence)
826 *emit_spm_needed = false;
827
828 *emit_gds_needed = gds_switch_needed;
829 if (gds_switch_needed && emit_fence)
830 *emit_gds_needed = false;
831
832 if (!emit_fence)
833 return;
834
835 amdgpu_ring_ib_begin(ring);
836
837 /* There is no matching insert_end for this on purpose for the vm flush.
838 * The IB portion of the submission has both. Having multiple
839 * insert_start sequences is ok, but you can only have one insert_end
840 * per submission based on the way VCN FW works. For JPEG
841 * you can as many insert_start and insert_end sequences as you like as
842 * long as the rest of the packets come between start and end sequences.
843 */
844 if (ring->funcs->insert_start)
845 ring->funcs->insert_start(ring);
846
847 if (ring->funcs->init_cond_exec)
848 patch = amdgpu_ring_init_cond_exec(ring,
849 ring->cond_exe_gpu_addr);
850
851 if (*need_pipe_sync) {
852 amdgpu_ring_emit_pipeline_sync(ring);
853 *need_pipe_sync = false;
854 }
855
856 if (cleaner_shader_needed)
857 ring->funcs->emit_cleaner_shader(ring);
858
859 if (vm_flush_needed) {
860 trace_amdgpu_vm_flush(ring, job->vmid, job->vm_pd_addr);
861 amdgpu_ring_emit_vm_flush(ring, job->vmid, job->vm_pd_addr);
862 }
863
864 if (pasid_mapping_needed)
865 amdgpu_gmc_emit_pasid_mapping(ring, job->vmid, job->pasid);
866
867 if (spm_update_needed)
868 adev->gfx.rlc.funcs->update_spm_vmid(adev, ring->xcc_id, ring, job->vmid);
869
870 if (gds_switch_needed)
871 amdgpu_ring_emit_gds_switch(ring, job->vmid, job->gds_base,
872 job->gds_size, job->gws_base,
873 job->gws_size, job->oa_base,
874 job->oa_size);
875
876 amdgpu_fence_emit(ring, job->hw_vm_fence, 0);
877 fence = &job->hw_vm_fence->base;
878 /* get a ref for the job */
879 dma_fence_get(fence);
880
881 if (vm_flush_needed) {
882 mutex_lock(&id_mgr->lock);
883 dma_fence_put(id->last_flush);
884 id->last_flush = dma_fence_get(fence);
885 id->current_gpu_reset_count =
886 atomic_read(&adev->gpu_reset_counter);
887 mutex_unlock(&id_mgr->lock);
888 }
889
890 if (pasid_mapping_needed) {
891 mutex_lock(&id_mgr->lock);
892 id->pasid = job->pasid;
893 dma_fence_put(id->pasid_mapping);
894 id->pasid_mapping = dma_fence_get(fence);
895 mutex_unlock(&id_mgr->lock);
896 }
897
898 /*
899 * Make sure that all other submissions wait for the cleaner shader to
900 * finish before we push them to the HW.
901 */
902 if (cleaner_shader_needed) {
903 trace_amdgpu_cleaner_shader(ring, fence);
904 mutex_lock(&adev->enforce_isolation_mutex);
905 dma_fence_put(isolation->spearhead);
906 isolation->spearhead = dma_fence_get(fence);
907 mutex_unlock(&adev->enforce_isolation_mutex);
908 }
909 dma_fence_put(fence);
910
911 amdgpu_ring_patch_cond_exec(ring, patch);
912
913 /*
914 * Sync CE with ME to prevent CE from fetching the next CE IB
915 * before the context switch is done. This is emitted before
916 * the first IB of a job submission after a context switch.
917 * The double SWITCH_BUFFER here *cannot* be skipped by COND_EXEC.
918 */
919 if (ring->funcs->emit_switch_buffer) {
920 amdgpu_ring_emit_switch_buffer(ring);
921 amdgpu_ring_emit_switch_buffer(ring);
922 }
923
924 amdgpu_ring_ib_end(ring);
925}
926
927/**
928 * amdgpu_vm_bo_find - find the bo_va for a specific vm & bo
929 *
930 * @vm: requested vm
931 * @bo: requested buffer object
932 *
933 * Find @bo inside the requested vm.
934 * Search inside the @bos vm list for the requested vm
935 * Returns the found bo_va or NULL if none is found
936 *
937 * Object has to be reserved!
938 *
939 * Returns:
940 * Found bo_va or NULL.
941 */
942struct amdgpu_bo_va *amdgpu_vm_bo_find(struct amdgpu_vm *vm,
943 struct amdgpu_bo *bo)
944{
945 struct amdgpu_vm_bo_base *base;
946
947 for (base = bo->vm_bo; base; base = base->next) {
948 if (base->vm != vm)
949 continue;
950
951 return container_of(base, struct amdgpu_bo_va, base);
952 }
953 return NULL;
954}
955
956/**
957 * amdgpu_vm_map_gart - Resolve gart mapping of addr
958 *
959 * @pages_addr: optional DMA address to use for lookup
960 * @addr: the unmapped addr
961 *
962 * Look up the physical address of the page that the pte resolves
963 * to.
964 *
965 * Returns:
966 * The pointer for the page table entry.
967 */
968uint64_t amdgpu_vm_map_gart(const dma_addr_t *pages_addr, uint64_t addr)
969{
970 uint64_t result;
971
972 /* page table offset */
973 result = pages_addr[addr >> PAGE_SHIFT];
974
975 /* in case cpu page size != gpu page size*/
976 result |= addr & (~PAGE_MASK);
977
978 result &= 0xFFFFFFFFFFFFF000ULL;
979
980 return result;
981}
982
983/**
984 * amdgpu_vm_update_pdes - make sure that all directories are valid
985 *
986 * @adev: amdgpu_device pointer
987 * @vm: requested vm
988 * @immediate: submit immediately to the paging queue
989 *
990 * Makes sure all directories are up to date.
991 *
992 * Returns:
993 * 0 for success, error for failure.
994 */
995int amdgpu_vm_update_pdes(struct amdgpu_device *adev,
996 struct amdgpu_vm *vm, bool immediate)
997{
998 struct amdgpu_vm_update_params params;
999 struct amdgpu_vm_bo_base *entry, *tmp;
1000 bool flush_tlb_needed = false;
1001 int r, idx;
1002
1003 amdgpu_vm_assert_locked(vm);
1004
1005 if (list_empty(&vm->kernel.needs_update))
1006 return 0;
1007
1008 if (!drm_dev_enter(adev_to_drm(adev), &idx))
1009 return -ENODEV;
1010
1011 memset(&params, 0, sizeof(params));
1012 params.adev = adev;
1013 params.vm = vm;
1014 params.immediate = immediate;
1015
1016 r = vm->update_funcs->prepare(&params, NULL,
1017 AMDGPU_KERNEL_JOB_ID_VM_UPDATE_PDES);
1018 if (r)
1019 goto error;
1020
1021 list_for_each_entry(entry, &vm->kernel.needs_update, vm_status) {
1022 /* vm_flush_needed after updating moved PDEs */
1023 flush_tlb_needed |= entry->moved;
1024
1025 r = amdgpu_vm_pde_update(&params, entry);
1026 if (r)
1027 goto error;
1028 }
1029
1030 r = vm->update_funcs->commit(&params, &vm->last_update);
1031 if (r)
1032 goto error;
1033
1034 if (flush_tlb_needed)
1035 atomic64_inc(&vm->tlb_seq);
1036
1037 list_for_each_entry_safe(entry, tmp, &vm->kernel.needs_update,
1038 vm_status)
1039 amdgpu_vm_bo_idle(entry);
1040
1041error:
1042 drm_dev_exit(idx);
1043 return r;
1044}
1045
1046/**
1047 * amdgpu_vm_tlb_seq_cb - make sure to increment tlb sequence
1048 * @fence: unused
1049 * @cb: the callback structure
1050 *
1051 * Increments the tlb sequence to make sure that future CS execute a VM flush.
1052 */
1053static void amdgpu_vm_tlb_seq_cb(struct dma_fence *fence,
1054 struct dma_fence_cb *cb)
1055{
1056 struct amdgpu_vm_tlb_seq_struct *tlb_cb;
1057
1058 tlb_cb = container_of(cb, typeof(*tlb_cb), cb);
1059 atomic64_inc(&tlb_cb->vm->tlb_seq);
1060 kfree(tlb_cb);
1061}
1062
1063/**
1064 * amdgpu_vm_tlb_flush - prepare TLB flush
1065 *
1066 * @params: parameters for update
1067 * @fence: input fence to sync TLB flush with
1068 * @tlb_cb: the callback structure
1069 *
1070 * Increments the tlb sequence to make sure that future CS execute a VM flush.
1071 */
1072static void
1073amdgpu_vm_tlb_flush(struct amdgpu_vm_update_params *params,
1074 struct dma_fence **fence,
1075 struct amdgpu_vm_tlb_seq_struct *tlb_cb)
1076{
1077 struct amdgpu_vm *vm = params->vm;
1078
1079 tlb_cb->vm = vm;
1080 if (!fence || !*fence) {
1081 amdgpu_vm_tlb_seq_cb(NULL, &tlb_cb->cb);
1082 return;
1083 }
1084
1085 if (!dma_fence_add_callback(*fence, &tlb_cb->cb,
1086 amdgpu_vm_tlb_seq_cb)) {
1087 dma_fence_put(vm->last_tlb_flush);
1088 vm->last_tlb_flush = dma_fence_get(*fence);
1089 } else {
1090 amdgpu_vm_tlb_seq_cb(NULL, &tlb_cb->cb);
1091 }
1092
1093 /* Prepare a TLB flush fence to be attached to PTs */
1094 /* The check for need_tlb_fence should be dropped once we
1095 * sort out the issues with KIQ/MES TLB invalidation timeouts.
1096 */
1097 if (!params->unlocked && vm->need_tlb_fence) {
1098 amdgpu_vm_tlb_fence_create(params->adev, vm, fence);
1099
1100 /* Makes sure no PD/PT is freed before the flush */
1101 dma_resv_add_fence(vm->root.bo->tbo.base.resv, *fence,
1102 DMA_RESV_USAGE_BOOKKEEP);
1103 }
1104}
1105
1106/**
1107 * amdgpu_vm_update_range - update a range in the vm page table
1108 *
1109 * @adev: amdgpu_device pointer to use for commands
1110 * @vm: the VM to update the range
1111 * @immediate: immediate submission in a page fault
1112 * @unlocked: unlocked invalidation during MM callback
1113 * @flush_tlb: trigger tlb invalidation after update completed
1114 * @allow_override: change MTYPE for local NUMA nodes
1115 * @sync: fences we need to sync to
1116 * @start: start of mapped range
1117 * @last: last mapped entry
1118 * @flags: flags for the entries
1119 * @offset: offset into nodes and pages_addr
1120 * @vram_base: base for vram mappings
1121 * @res: ttm_resource to map
1122 * @pages_addr: DMA addresses to use for mapping
1123 * @fence: optional resulting fence
1124 *
1125 * Fill in the page table entries between @start and @last.
1126 *
1127 * Returns:
1128 * 0 for success, negative erro code for failure.
1129 */
1130int amdgpu_vm_update_range(struct amdgpu_device *adev, struct amdgpu_vm *vm,
1131 bool immediate, bool unlocked, bool flush_tlb,
1132 bool allow_override, struct amdgpu_sync *sync,
1133 uint64_t start, uint64_t last, uint64_t flags,
1134 uint64_t offset, uint64_t vram_base,
1135 struct ttm_resource *res, dma_addr_t *pages_addr,
1136 struct dma_fence **fence)
1137{
1138 struct amdgpu_vm_tlb_seq_struct *tlb_cb;
1139 struct amdgpu_vm_update_params params;
1140 struct amdgpu_res_cursor cursor;
1141 int r, idx;
1142
1143 if (!drm_dev_enter(adev_to_drm(adev), &idx))
1144 return -ENODEV;
1145
1146 tlb_cb = kmalloc_obj(*tlb_cb);
1147 if (!tlb_cb) {
1148 drm_dev_exit(idx);
1149 return -ENOMEM;
1150 }
1151
1152 /* Vega20+XGMI where PTEs get inadvertently cached in L2 texture cache,
1153 * heavy-weight flush TLB unconditionally.
1154 */
1155 flush_tlb |= adev->gmc.xgmi.num_physical_nodes &&
1156 amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 0);
1157
1158 /*
1159 * On GFX8 and older any 8 PTE block with a valid bit set enters the TLB
1160 */
1161 flush_tlb |= amdgpu_ip_version(adev, GC_HWIP, 0) < IP_VERSION(9, 0, 0);
1162
1163 memset(&params, 0, sizeof(params));
1164 params.adev = adev;
1165 params.vm = vm;
1166 params.immediate = immediate;
1167 params.pages_addr = pages_addr;
1168 params.unlocked = unlocked;
1169 params.needs_flush = flush_tlb;
1170 params.override_pte = allow_override && adev->gmc.override_pte;
1171 INIT_LIST_HEAD(&params.tlb_flush_waitlist);
1172
1173 amdgpu_vm_eviction_lock(vm);
1174 if (vm->evicting) {
1175 r = -EBUSY;
1176 goto error_free;
1177 }
1178
1179 if (!unlocked && !dma_fence_is_signaled(vm->last_unlocked)) {
1180 struct dma_fence *tmp = dma_fence_get_stub();
1181
1182 amdgpu_bo_fence(vm->root.bo, vm->last_unlocked, true);
1183 swap(vm->last_unlocked, tmp);
1184 dma_fence_put(tmp);
1185 }
1186
1187 r = vm->update_funcs->prepare(&params, sync,
1188 AMDGPU_KERNEL_JOB_ID_VM_UPDATE_RANGE);
1189 if (r)
1190 goto error_free;
1191
1192 amdgpu_res_first(pages_addr ? NULL : res, offset,
1193 (last - start + 1) * AMDGPU_GPU_PAGE_SIZE, &cursor);
1194 while (cursor.remaining) {
1195 uint64_t tmp, num_entries, addr;
1196
1197 num_entries = cursor.size >> AMDGPU_GPU_PAGE_SHIFT;
1198 if (pages_addr) {
1199 bool contiguous = true;
1200
1201 if (num_entries > AMDGPU_GPU_PAGES_IN_CPU_PAGE) {
1202 uint64_t pfn = cursor.start >> PAGE_SHIFT;
1203 uint64_t count;
1204
1205 contiguous = pages_addr[pfn + 1] ==
1206 pages_addr[pfn] + PAGE_SIZE;
1207
1208 tmp = num_entries /
1209 AMDGPU_GPU_PAGES_IN_CPU_PAGE;
1210 for (count = 2; count < tmp; ++count) {
1211 uint64_t idx = pfn + count;
1212
1213 if (contiguous != (pages_addr[idx] ==
1214 pages_addr[idx - 1] + PAGE_SIZE))
1215 break;
1216 }
1217 if (!contiguous)
1218 count--;
1219 num_entries = count *
1220 AMDGPU_GPU_PAGES_IN_CPU_PAGE;
1221 }
1222
1223 if (!contiguous) {
1224 addr = cursor.start;
1225 params.pages_addr = pages_addr;
1226 } else {
1227 addr = pages_addr[cursor.start >> PAGE_SHIFT];
1228 params.pages_addr = NULL;
1229 }
1230
1231 } else if (flags & (AMDGPU_PTE_VALID | AMDGPU_PTE_PRT_FLAG(adev))) {
1232 addr = vram_base + cursor.start;
1233 } else {
1234 addr = 0;
1235 }
1236
1237 tmp = start + num_entries;
1238 r = amdgpu_vm_ptes_update(&params, start, tmp, addr, flags);
1239 if (r)
1240 goto error_free;
1241
1242 amdgpu_res_next(&cursor, num_entries * AMDGPU_GPU_PAGE_SIZE);
1243 start = tmp;
1244 }
1245
1246 r = vm->update_funcs->commit(&params, fence);
1247 if (r)
1248 goto error_free;
1249
1250 if (params.needs_flush) {
1251 amdgpu_vm_tlb_flush(&params, fence, tlb_cb);
1252 tlb_cb = NULL;
1253 }
1254
1255 amdgpu_vm_pt_free_list(adev, &params);
1256
1257error_free:
1258 kfree(tlb_cb);
1259 amdgpu_vm_eviction_unlock(vm);
1260 drm_dev_exit(idx);
1261 return r;
1262}
1263
1264void amdgpu_vm_get_memory(struct amdgpu_vm *vm,
1265 struct amdgpu_mem_stats stats[__AMDGPU_PL_NUM])
1266{
1267 spin_lock(&vm->stats_lock);
1268 memcpy(stats, vm->stats, sizeof(*stats) * __AMDGPU_PL_NUM);
1269 spin_unlock(&vm->stats_lock);
1270}
1271
1272/**
1273 * amdgpu_vm_bo_update - update all BO mappings in the vm page table
1274 *
1275 * @adev: amdgpu_device pointer
1276 * @bo_va: requested BO and VM object
1277 * @clear: if true clear the entries
1278 *
1279 * Fill in the page table entries for @bo_va.
1280 *
1281 * Returns:
1282 * 0 for success, -EINVAL for failure.
1283 */
1284int amdgpu_vm_bo_update(struct amdgpu_device *adev, struct amdgpu_bo_va *bo_va,
1285 bool clear)
1286{
1287 struct amdgpu_bo *bo = bo_va->base.bo;
1288 struct amdgpu_vm *vm = bo_va->base.vm;
1289 struct amdgpu_bo_va_mapping *mapping;
1290 struct dma_fence **last_update;
1291 dma_addr_t *pages_addr = NULL;
1292 struct ttm_resource *mem;
1293 struct amdgpu_sync sync;
1294 bool flush_tlb = clear;
1295 uint64_t vram_base;
1296 uint64_t flags;
1297 bool uncached;
1298 int r;
1299
1300 amdgpu_sync_create(&sync);
1301 if (clear) {
1302 mem = NULL;
1303
1304 /* Implicitly sync to command submissions in the same VM before
1305 * unmapping.
1306 */
1307 r = amdgpu_sync_resv(adev, &sync, vm->root.bo->tbo.base.resv,
1308 AMDGPU_SYNC_EQ_OWNER, vm);
1309 if (r)
1310 goto error_free;
1311 if (bo) {
1312 r = amdgpu_sync_kfd(&sync, bo->tbo.base.resv);
1313 if (r)
1314 goto error_free;
1315 }
1316 } else if (!bo) {
1317 mem = NULL;
1318
1319 /* PRT map operations don't need to sync to anything. */
1320
1321 } else {
1322 struct drm_gem_object *obj = &bo->tbo.base;
1323
1324 if (drm_gem_is_imported(obj) && bo_va->is_xgmi) {
1325 struct dma_buf *dma_buf = obj->import_attach->dmabuf;
1326 struct drm_gem_object *gobj = dma_buf->priv;
1327 struct amdgpu_bo *abo = gem_to_amdgpu_bo(gobj);
1328
1329 if (abo->tbo.resource &&
1330 abo->tbo.resource->mem_type == TTM_PL_VRAM)
1331 bo = gem_to_amdgpu_bo(gobj);
1332 }
1333 mem = bo->tbo.resource;
1334 if (mem && (mem->mem_type == TTM_PL_TT ||
1335 mem->mem_type == AMDGPU_PL_PREEMPT))
1336 pages_addr = bo->tbo.ttm->dma_address;
1337
1338 /* Implicitly sync to moving fences before mapping anything */
1339 r = amdgpu_sync_resv(adev, &sync, bo->tbo.base.resv,
1340 AMDGPU_SYNC_EXPLICIT, vm);
1341 if (r)
1342 goto error_free;
1343 }
1344
1345 if (bo) {
1346 struct amdgpu_device *bo_adev;
1347
1348 flags = amdgpu_ttm_tt_pte_flags(adev, bo->tbo.ttm, mem);
1349
1350 if (amdgpu_bo_encrypted(bo))
1351 flags |= AMDGPU_PTE_TMZ;
1352
1353 bo_adev = amdgpu_ttm_adev(bo->tbo.bdev);
1354 vram_base = bo_adev->vm_manager.vram_base_offset;
1355 uncached = (bo->flags & AMDGPU_GEM_CREATE_UNCACHED) != 0;
1356 } else {
1357 flags = 0x0;
1358 vram_base = 0;
1359 uncached = false;
1360 }
1361
1362 if (clear || amdgpu_vm_is_bo_always_valid(vm, bo))
1363 last_update = &vm->last_update;
1364 else
1365 last_update = &bo_va->last_pt_update;
1366
1367 if (!clear && bo_va->base.moved) {
1368 flush_tlb = true;
1369 list_splice_init(&bo_va->valids, &bo_va->invalids);
1370
1371 } else if (bo_va->cleared != clear) {
1372 list_splice_init(&bo_va->valids, &bo_va->invalids);
1373 }
1374
1375 list_for_each_entry(mapping, &bo_va->invalids, list) {
1376 uint64_t update_flags = flags;
1377
1378 /* normally,bo_va->flags only contians READABLE and WIRTEABLE bit go here
1379 * but in case of something, we filter the flags in first place
1380 */
1381 if (!(mapping->flags & AMDGPU_VM_PAGE_READABLE))
1382 update_flags &= ~AMDGPU_PTE_READABLE;
1383 if (!(mapping->flags & AMDGPU_VM_PAGE_WRITEABLE))
1384 update_flags &= ~AMDGPU_PTE_WRITEABLE;
1385
1386 /* Apply ASIC specific mapping flags */
1387 amdgpu_gmc_get_vm_pte(adev, vm, bo, mapping->flags,
1388 &update_flags);
1389
1390 trace_amdgpu_vm_bo_update(mapping);
1391
1392 r = amdgpu_vm_update_range(adev, vm, false, false, flush_tlb,
1393 !uncached, &sync, mapping->start,
1394 mapping->last, update_flags,
1395 mapping->offset, vram_base, mem,
1396 pages_addr, last_update);
1397 if (r)
1398 goto error_free;
1399 }
1400
1401 /* If the BO is not in its preferred location add it back to
1402 * the evicted list so that it gets validated again on the
1403 * next command submission.
1404 */
1405 if (amdgpu_vm_is_bo_always_valid(vm, bo)) {
1406 if (bo->tbo.resource &&
1407 !(bo->preferred_domains &
1408 amdgpu_mem_type_to_domain(bo->tbo.resource->mem_type)))
1409 amdgpu_vm_bo_evicted(&bo_va->base);
1410 else
1411 amdgpu_vm_bo_idle(&bo_va->base);
1412 } else if (bo) {
1413 /*
1414 * A PRT/sparse mapping has no BO and is kept off the vm_bo
1415 * state lists (see amdgpu_vm_bo_base_init()); putting it on the
1416 * idle list here would let amdgpu_vm_handle_moved() dereference
1417 * the NULL bo after a reset.
1418 */
1419 amdgpu_vm_bo_idle(&bo_va->base);
1420 }
1421
1422 list_splice_init(&bo_va->invalids, &bo_va->valids);
1423 bo_va->cleared = clear;
1424 bo_va->base.moved = false;
1425
1426 if (trace_amdgpu_vm_bo_mapping_enabled()) {
1427 list_for_each_entry(mapping, &bo_va->valids, list)
1428 trace_amdgpu_vm_bo_mapping(mapping);
1429 }
1430
1431error_free:
1432 amdgpu_sync_free(&sync);
1433 return r;
1434}
1435
1436/**
1437 * amdgpu_vm_update_prt_state - update the global PRT state
1438 *
1439 * @adev: amdgpu_device pointer
1440 */
1441static void amdgpu_vm_update_prt_state(struct amdgpu_device *adev)
1442{
1443 unsigned long flags;
1444 bool enable;
1445
1446 spin_lock_irqsave(&adev->vm_manager.prt_lock, flags);
1447 enable = !!atomic_read(&adev->vm_manager.num_prt_users);
1448 adev->gmc.gmc_funcs->set_prt(adev, enable);
1449 spin_unlock_irqrestore(&adev->vm_manager.prt_lock, flags);
1450}
1451
1452/**
1453 * amdgpu_vm_prt_get - add a PRT user
1454 *
1455 * @adev: amdgpu_device pointer
1456 */
1457static void amdgpu_vm_prt_get(struct amdgpu_device *adev)
1458{
1459 if (!adev->gmc.gmc_funcs->set_prt)
1460 return;
1461
1462 if (atomic_inc_return(&adev->vm_manager.num_prt_users) == 1)
1463 amdgpu_vm_update_prt_state(adev);
1464}
1465
1466/**
1467 * amdgpu_vm_prt_put - drop a PRT user
1468 *
1469 * @adev: amdgpu_device pointer
1470 */
1471static void amdgpu_vm_prt_put(struct amdgpu_device *adev)
1472{
1473 if (atomic_dec_return(&adev->vm_manager.num_prt_users) == 0)
1474 amdgpu_vm_update_prt_state(adev);
1475}
1476
1477/**
1478 * amdgpu_vm_prt_cb - callback for updating the PRT status
1479 *
1480 * @fence: fence for the callback
1481 * @_cb: the callback function
1482 */
1483static void amdgpu_vm_prt_cb(struct dma_fence *fence, struct dma_fence_cb *_cb)
1484{
1485 struct amdgpu_prt_cb *cb = container_of(_cb, struct amdgpu_prt_cb, cb);
1486
1487 amdgpu_vm_prt_put(cb->adev);
1488 kfree(cb);
1489}
1490
1491/**
1492 * amdgpu_vm_add_prt_cb - add callback for updating the PRT status
1493 *
1494 * @adev: amdgpu_device pointer
1495 * @fence: fence for the callback
1496 */
1497static void amdgpu_vm_add_prt_cb(struct amdgpu_device *adev,
1498 struct dma_fence *fence)
1499{
1500 struct amdgpu_prt_cb *cb;
1501
1502 if (!adev->gmc.gmc_funcs->set_prt)
1503 return;
1504
1505 cb = kmalloc_obj(struct amdgpu_prt_cb);
1506 if (!cb) {
1507 /* Last resort when we are OOM */
1508 if (fence)
1509 dma_fence_wait(fence, false);
1510
1511 amdgpu_vm_prt_put(adev);
1512 } else {
1513 cb->adev = adev;
1514 if (!fence || dma_fence_add_callback(fence, &cb->cb,
1515 amdgpu_vm_prt_cb))
1516 amdgpu_vm_prt_cb(fence, &cb->cb);
1517 }
1518}
1519
1520/**
1521 * amdgpu_vm_free_mapping - free a mapping
1522 *
1523 * @adev: amdgpu_device pointer
1524 * @vm: requested vm
1525 * @mapping: mapping to be freed
1526 * @fence: fence of the unmap operation
1527 *
1528 * Free a mapping and make sure we decrease the PRT usage count if applicable.
1529 */
1530static void amdgpu_vm_free_mapping(struct amdgpu_device *adev,
1531 struct amdgpu_vm *vm,
1532 struct amdgpu_bo_va_mapping *mapping,
1533 struct dma_fence *fence)
1534{
1535 if (mapping->flags & AMDGPU_VM_PAGE_PRT)
1536 amdgpu_vm_add_prt_cb(adev, fence);
1537 kfree(mapping);
1538}
1539
1540/**
1541 * amdgpu_vm_prt_fini - finish all prt mappings
1542 *
1543 * @adev: amdgpu_device pointer
1544 * @vm: requested vm
1545 *
1546 * Register a cleanup callback to disable PRT support after VM dies.
1547 */
1548static void amdgpu_vm_prt_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm)
1549{
1550 struct dma_resv *resv = vm->root.bo->tbo.base.resv;
1551 struct dma_resv_iter cursor;
1552 struct dma_fence *fence;
1553
1554 dma_resv_for_each_fence(&cursor, resv, DMA_RESV_USAGE_BOOKKEEP, fence) {
1555 /* Add a callback for each fence in the reservation object */
1556 amdgpu_vm_prt_get(adev);
1557 amdgpu_vm_add_prt_cb(adev, fence);
1558 }
1559}
1560
1561/**
1562 * amdgpu_vm_clear_freed - clear freed BOs in the PT
1563 *
1564 * @adev: amdgpu_device pointer
1565 * @vm: requested vm
1566 * @fence: optional resulting fence (unchanged if no work needed to be done
1567 * or if an error occurred)
1568 *
1569 * Make sure all freed BOs are cleared in the PT.
1570 * PTs have to be reserved and mutex must be locked!
1571 *
1572 * Returns:
1573 * 0 for success.
1574 *
1575 */
1576int amdgpu_vm_clear_freed(struct amdgpu_device *adev,
1577 struct amdgpu_vm *vm,
1578 struct dma_fence **fence)
1579{
1580 struct amdgpu_bo_va_mapping *mapping;
1581 struct dma_fence *f = NULL;
1582 struct amdgpu_sync sync;
1583 int r;
1584
1585 if (list_empty(&vm->freed))
1586 return 0;
1587
1588 /*
1589 * Implicitly sync to command submissions in the same VM before
1590 * unmapping.
1591 */
1592 amdgpu_sync_create(&sync);
1593 r = amdgpu_sync_resv(adev, &sync, vm->root.bo->tbo.base.resv,
1594 AMDGPU_SYNC_EQ_OWNER, vm);
1595 if (r)
1596 goto error_free;
1597
1598 while (!list_empty(&vm->freed)) {
1599 mapping = list_first_entry(&vm->freed,
1600 struct amdgpu_bo_va_mapping, list);
1601 list_del(&mapping->list);
1602
1603 r = amdgpu_vm_update_range(adev, vm, false, false, true, false,
1604 &sync, mapping->start, mapping->last,
1605 0, 0, 0, NULL, NULL, &f);
1606 amdgpu_vm_free_mapping(adev, vm, mapping, f);
1607 if (r) {
1608 dma_fence_put(f);
1609 goto error_free;
1610 }
1611 }
1612
1613 if (fence && f) {
1614 dma_fence_put(*fence);
1615 *fence = f;
1616 } else {
1617 dma_fence_put(f);
1618 }
1619
1620error_free:
1621 amdgpu_sync_free(&sync);
1622 return r;
1623
1624}
1625
1626/**
1627 * amdgpu_vm_handle_moved - handle moved BOs in the PT
1628 *
1629 * @adev: amdgpu_device pointer
1630 * @vm: requested vm
1631 * @ticket: optional reservation ticket used to reserve the VM
1632 *
1633 * Make sure all BOs which are moved are updated in the PTs.
1634 *
1635 * Returns:
1636 * 0 for success.
1637 *
1638 * PTs have to be reserved!
1639 */
1640int amdgpu_vm_handle_moved(struct amdgpu_device *adev,
1641 struct amdgpu_vm *vm,
1642 struct ww_acquire_ctx *ticket)
1643{
1644 struct amdgpu_bo_va *bo_va, *tmp;
1645 struct dma_resv *resv;
1646 struct amdgpu_bo *bo;
1647 bool clear, unlock;
1648 int r;
1649
1650 list_for_each_entry_safe(bo_va, tmp, &vm->always_valid.needs_update,
1651 base.vm_status) {
1652 /* Per VM BOs never need to bo cleared in the page tables */
1653 r = amdgpu_vm_bo_update(adev, bo_va, false);
1654 if (r)
1655 return r;
1656 }
1657
1658 spin_lock(&vm->individual_lock);
1659 while (!list_empty(&vm->individual.needs_update)) {
1660 bo_va = list_first_entry(&vm->individual.needs_update,
1661 typeof(*bo_va), base.vm_status);
1662 bo = bo_va->base.bo;
1663 resv = bo->tbo.base.resv;
1664 spin_unlock(&vm->individual_lock);
1665
1666 /* Try to reserve the BO to avoid clearing its ptes */
1667 if (!adev->debug_vm && !amdgpu_ttm_tt_get_usermm(bo->tbo.ttm) &&
1668 dma_resv_trylock(resv)) {
1669 clear = false;
1670 unlock = true;
1671 /* The caller is already holding the reservation lock */
1672 } else if (ticket && dma_resv_locking_ctx(resv) == ticket) {
1673 clear = false;
1674 unlock = false;
1675 /* Somebody else is using the BO right now */
1676 } else {
1677 clear = true;
1678 unlock = false;
1679 }
1680
1681 r = amdgpu_vm_bo_update(adev, bo_va, clear);
1682
1683 if (unlock)
1684 dma_resv_unlock(resv);
1685 if (r)
1686 return r;
1687
1688 /* Remember evicted DMABuf imports in compute VMs for later
1689 * validation
1690 */
1691 if (vm->is_compute_context &&
1692 drm_gem_is_imported(&bo_va->base.bo->tbo.base) &&
1693 (!bo_va->base.bo->tbo.resource ||
1694 bo_va->base.bo->tbo.resource->mem_type == TTM_PL_SYSTEM))
1695 amdgpu_vm_bo_evicted(&bo_va->base);
1696
1697 spin_lock(&vm->individual_lock);
1698 }
1699 spin_unlock(&vm->individual_lock);
1700
1701 return 0;
1702}
1703
1704/**
1705 * amdgpu_vm_flush_compute_tlb - Flush TLB on compute VM
1706 *
1707 * @adev: amdgpu_device pointer
1708 * @vm: requested vm
1709 * @flush_type: flush type
1710 * @xcc_mask: mask of XCCs that belong to the compute partition in need of a TLB flush.
1711 *
1712 * Flush TLB if needed for a compute VM.
1713 *
1714 * Returns:
1715 * 0 for success.
1716 */
1717int amdgpu_vm_flush_compute_tlb(struct amdgpu_device *adev,
1718 struct amdgpu_vm *vm,
1719 uint32_t flush_type,
1720 uint32_t xcc_mask)
1721{
1722 uint64_t tlb_seq = amdgpu_vm_tlb_seq(vm);
1723 bool all_hub = false;
1724 int xcc = 0, r = 0;
1725
1726 WARN_ON_ONCE(!vm->is_compute_context);
1727
1728 /*
1729 * It can be that we race and lose here, but that is extremely unlikely
1730 * and the worst thing which could happen is that we flush the changes
1731 * into the TLB once more which is harmless.
1732 */
1733 if (atomic64_xchg(&vm->kfd_last_flushed_seq, tlb_seq) == tlb_seq)
1734 return 0;
1735
1736 if (adev->family == AMDGPU_FAMILY_AI ||
1737 adev->family == AMDGPU_FAMILY_RV)
1738 all_hub = true;
1739
1740 for_each_inst(xcc, xcc_mask) {
1741 r = amdgpu_gmc_flush_gpu_tlb_pasid(adev, vm->pasid, flush_type,
1742 all_hub, xcc);
1743 if (r)
1744 break;
1745 }
1746 return r;
1747}
1748
1749/**
1750 * amdgpu_vm_bo_add - add a bo to a specific vm
1751 *
1752 * @adev: amdgpu_device pointer
1753 * @vm: requested vm
1754 * @bo: amdgpu buffer object
1755 *
1756 * Add @bo into the requested vm.
1757 * Add @bo to the list of bos associated with the vm
1758 *
1759 * Returns:
1760 * Newly added bo_va or NULL for failure
1761 *
1762 * Object has to be reserved!
1763 */
1764struct amdgpu_bo_va *amdgpu_vm_bo_add(struct amdgpu_device *adev,
1765 struct amdgpu_vm *vm,
1766 struct amdgpu_bo *bo)
1767{
1768 struct amdgpu_bo_va *bo_va;
1769
1770 amdgpu_vm_assert_locked(vm);
1771
1772 bo_va = kzalloc_obj(struct amdgpu_bo_va);
1773 if (bo_va == NULL) {
1774 return NULL;
1775 }
1776 amdgpu_vm_bo_base_init(&bo_va->base, vm, bo);
1777
1778 bo_va->ref_count = 1;
1779 bo_va->last_pt_update = dma_fence_get_stub();
1780 INIT_LIST_HEAD(&bo_va->valids);
1781 INIT_LIST_HEAD(&bo_va->invalids);
1782
1783 if (!bo)
1784 return bo_va;
1785
1786 dma_resv_assert_held(bo->tbo.base.resv);
1787 if (amdgpu_dmabuf_is_xgmi_accessible(adev, bo)) {
1788 bo_va->is_xgmi = true;
1789 /* Power up XGMI if it can be potentially used */
1790 amdgpu_xgmi_set_pstate(adev, AMDGPU_XGMI_PSTATE_MAX_VEGA20);
1791 }
1792
1793 return bo_va;
1794}
1795
1796
1797/**
1798 * amdgpu_vm_bo_insert_map - insert a new mapping
1799 *
1800 * @adev: amdgpu_device pointer
1801 * @bo_va: bo_va to store the address
1802 * @mapping: the mapping to insert
1803 *
1804 * Insert a new mapping into all structures.
1805 */
1806static void amdgpu_vm_bo_insert_map(struct amdgpu_device *adev,
1807 struct amdgpu_bo_va *bo_va,
1808 struct amdgpu_bo_va_mapping *mapping)
1809{
1810 struct amdgpu_vm *vm = bo_va->base.vm;
1811 struct amdgpu_bo *bo = bo_va->base.bo;
1812
1813 mapping->bo_va = bo_va;
1814 list_add(&mapping->list, &bo_va->invalids);
1815 amdgpu_vm_it_insert(mapping, &vm->va);
1816
1817 if (mapping->flags & AMDGPU_VM_PAGE_PRT)
1818 amdgpu_vm_prt_get(adev);
1819
1820 if (amdgpu_vm_is_bo_always_valid(vm, bo) && !bo_va->base.moved)
1821 amdgpu_vm_bo_needs_update(&bo_va->base);
1822
1823 trace_amdgpu_vm_bo_map(bo_va, mapping);
1824}
1825
1826/* Validate operation parameters to prevent potential abuse */
1827static int amdgpu_vm_verify_parameters(struct amdgpu_device *adev,
1828 struct amdgpu_bo *bo,
1829 uint64_t saddr,
1830 uint64_t offset,
1831 uint64_t size)
1832{
1833 uint64_t tmp, lpfn;
1834
1835 if (saddr & AMDGPU_GPU_PAGE_MASK
1836 || offset & AMDGPU_GPU_PAGE_MASK
1837 || size & AMDGPU_GPU_PAGE_MASK)
1838 return -EINVAL;
1839
1840 if (check_add_overflow(saddr, size, &tmp)
1841 || check_add_overflow(offset, size, &tmp)
1842 || size == 0 /* which also leads to end < begin */)
1843 return -EINVAL;
1844
1845 /* make sure object fit at this offset */
1846 if (bo && offset + size > amdgpu_bo_size(bo))
1847 return -EINVAL;
1848
1849 /* Ensure last pfn not exceed max_pfn */
1850 lpfn = (saddr + size - 1) >> AMDGPU_GPU_PAGE_SHIFT;
1851 if (lpfn >= adev->vm_manager.max_pfn)
1852 return -EINVAL;
1853
1854 return 0;
1855}
1856
1857/**
1858 * amdgpu_vm_bo_map - map bo inside a vm
1859 *
1860 * @adev: amdgpu_device pointer
1861 * @bo_va: bo_va to store the address
1862 * @saddr: where to map the BO
1863 * @offset: requested offset in the BO
1864 * @size: BO size in bytes
1865 * @flags: attributes of pages (read/write/valid/etc.)
1866 *
1867 * Add a mapping of the BO at the specefied addr into the VM.
1868 *
1869 * Returns:
1870 * 0 for success, error for failure.
1871 *
1872 * Object has to be reserved and unreserved outside!
1873 */
1874int amdgpu_vm_bo_map(struct amdgpu_device *adev,
1875 struct amdgpu_bo_va *bo_va,
1876 uint64_t saddr, uint64_t offset,
1877 uint64_t size, uint32_t flags)
1878{
1879 struct amdgpu_bo_va_mapping *mapping, *tmp;
1880 struct amdgpu_bo *bo = bo_va->base.bo;
1881 struct amdgpu_vm *vm = bo_va->base.vm;
1882 uint64_t eaddr;
1883 int r;
1884
1885 r = amdgpu_vm_verify_parameters(adev, bo, saddr, offset, size);
1886 if (r)
1887 return r;
1888
1889 saddr /= AMDGPU_GPU_PAGE_SIZE;
1890 eaddr = saddr + (size - 1) / AMDGPU_GPU_PAGE_SIZE;
1891
1892 tmp = amdgpu_vm_it_iter_first(&vm->va, saddr, eaddr);
1893 if (tmp) {
1894 /* bo and tmp overlap, invalid addr */
1895 dev_err(adev->dev, "bo %p va 0x%010Lx-0x%010Lx conflict with "
1896 "0x%010Lx-0x%010Lx\n", bo, saddr, eaddr,
1897 tmp->start, tmp->last + 1);
1898 return -EINVAL;
1899 }
1900
1901 mapping = kmalloc_obj(*mapping);
1902 if (!mapping)
1903 return -ENOMEM;
1904
1905 mapping->start = saddr;
1906 mapping->last = eaddr;
1907 mapping->offset = offset;
1908 mapping->flags = flags;
1909
1910 amdgpu_vm_bo_insert_map(adev, bo_va, mapping);
1911
1912 return 0;
1913}
1914
1915/**
1916 * amdgpu_vm_bo_replace_map - map bo inside a vm, replacing existing mappings
1917 *
1918 * @adev: amdgpu_device pointer
1919 * @bo_va: bo_va to store the address
1920 * @saddr: where to map the BO
1921 * @offset: requested offset in the BO
1922 * @size: BO size in bytes
1923 * @flags: attributes of pages (read/write/valid/etc.)
1924 *
1925 * Add a mapping of the BO at the specefied addr into the VM. Replace existing
1926 * mappings as we do so.
1927 *
1928 * Returns:
1929 * 0 for success, error for failure.
1930 *
1931 * Object has to be reserved and unreserved outside!
1932 */
1933int amdgpu_vm_bo_replace_map(struct amdgpu_device *adev,
1934 struct amdgpu_bo_va *bo_va,
1935 uint64_t saddr, uint64_t offset,
1936 uint64_t size, uint32_t flags)
1937{
1938 struct amdgpu_bo_va_mapping *mapping;
1939 struct amdgpu_bo *bo = bo_va->base.bo;
1940 uint64_t eaddr;
1941 int r;
1942
1943 r = amdgpu_vm_verify_parameters(adev, bo, saddr, offset, size);
1944 if (r)
1945 return r;
1946
1947 /* Allocate all the needed memory */
1948 mapping = kmalloc_obj(*mapping);
1949 if (!mapping)
1950 return -ENOMEM;
1951
1952 r = amdgpu_vm_bo_clear_mappings(adev, bo_va->base.vm, saddr, size);
1953 if (r) {
1954 kfree(mapping);
1955 return r;
1956 }
1957
1958 saddr /= AMDGPU_GPU_PAGE_SIZE;
1959 eaddr = saddr + (size - 1) / AMDGPU_GPU_PAGE_SIZE;
1960
1961 mapping->start = saddr;
1962 mapping->last = eaddr;
1963 mapping->offset = offset;
1964 mapping->flags = flags;
1965
1966 amdgpu_vm_bo_insert_map(adev, bo_va, mapping);
1967
1968 return 0;
1969}
1970
1971/**
1972 * amdgpu_vm_bo_unmap - remove bo mapping from vm
1973 *
1974 * @adev: amdgpu_device pointer
1975 * @bo_va: bo_va to remove the address from
1976 * @saddr: where to the BO is mapped
1977 *
1978 * Remove a mapping of the BO at the specefied addr from the VM.
1979 *
1980 * Returns:
1981 * 0 for success, error for failure.
1982 *
1983 * Object has to be reserved and unreserved outside!
1984 */
1985int amdgpu_vm_bo_unmap(struct amdgpu_device *adev,
1986 struct amdgpu_bo_va *bo_va,
1987 uint64_t saddr)
1988{
1989 struct amdgpu_bo_va_mapping *mapping;
1990 struct amdgpu_vm *vm = bo_va->base.vm;
1991 bool valid = true;
1992
1993 saddr /= AMDGPU_GPU_PAGE_SIZE;
1994
1995 list_for_each_entry(mapping, &bo_va->valids, list) {
1996 if (mapping->start == saddr)
1997 break;
1998 }
1999
2000 if (&mapping->list == &bo_va->valids) {
2001 valid = false;
2002
2003 list_for_each_entry(mapping, &bo_va->invalids, list) {
2004 if (mapping->start == saddr)
2005 break;
2006 }
2007
2008 if (&mapping->list == &bo_va->invalids)
2009 return -ENOENT;
2010 }
2011
2012 /* It's unlikely to happen that the mapping userq hasn't been idled
2013 * during user requests GEM unmap IOCTL except for forcing the unmap
2014 * from user space.
2015 */
2016 if (unlikely(bo_va->userq_va_mapped))
2017 amdgpu_userq_gem_va_unmap_validate(adev, mapping);
2018
2019 list_del(&mapping->list);
2020 amdgpu_vm_it_remove(mapping, &vm->va);
2021 mapping->bo_va = NULL;
2022 trace_amdgpu_vm_bo_unmap(bo_va, mapping);
2023
2024 if (valid)
2025 list_add(&mapping->list, &vm->freed);
2026 else
2027 amdgpu_vm_free_mapping(adev, vm, mapping,
2028 bo_va->last_pt_update);
2029
2030 return 0;
2031}
2032
2033/**
2034 * amdgpu_vm_bo_clear_mappings - remove all mappings in a specific range
2035 *
2036 * @adev: amdgpu_device pointer
2037 * @vm: VM structure to use
2038 * @saddr: start of the range
2039 * @size: size of the range
2040 *
2041 * Remove all mappings in a range, split them as appropriate.
2042 *
2043 * Returns:
2044 * 0 for success, error for failure.
2045 */
2046int amdgpu_vm_bo_clear_mappings(struct amdgpu_device *adev,
2047 struct amdgpu_vm *vm,
2048 uint64_t saddr, uint64_t size)
2049{
2050 struct amdgpu_bo_va_mapping *before, *after, *tmp, *next;
2051 LIST_HEAD(removed);
2052 uint64_t eaddr;
2053 int r;
2054
2055 r = amdgpu_vm_verify_parameters(adev, NULL, saddr, 0, size);
2056 if (r)
2057 return r;
2058
2059 saddr /= AMDGPU_GPU_PAGE_SIZE;
2060 eaddr = saddr + (size - 1) / AMDGPU_GPU_PAGE_SIZE;
2061
2062 /* Allocate all the needed memory */
2063 before = kzalloc_obj(*before);
2064 if (!before)
2065 return -ENOMEM;
2066 INIT_LIST_HEAD(&before->list);
2067
2068 after = kzalloc_obj(*after);
2069 if (!after) {
2070 kfree(before);
2071 return -ENOMEM;
2072 }
2073 INIT_LIST_HEAD(&after->list);
2074
2075 /* Now gather all removed mappings */
2076 tmp = amdgpu_vm_it_iter_first(&vm->va, saddr, eaddr);
2077 while (tmp) {
2078 /* Remember mapping split at the start */
2079 if (tmp->start < saddr) {
2080 before->start = tmp->start;
2081 before->last = saddr - 1;
2082 before->offset = tmp->offset;
2083 before->flags = tmp->flags;
2084 before->bo_va = tmp->bo_va;
2085 list_add(&before->list, &tmp->bo_va->invalids);
2086 }
2087
2088 /* Remember mapping split at the end */
2089 if (tmp->last > eaddr) {
2090 after->start = eaddr + 1;
2091 after->last = tmp->last;
2092 after->offset = tmp->offset;
2093 after->offset += (after->start - tmp->start) << AMDGPU_GPU_PAGE_SHIFT;
2094 after->flags = tmp->flags;
2095 after->bo_va = tmp->bo_va;
2096 list_add(&after->list, &tmp->bo_va->invalids);
2097 }
2098
2099 list_del(&tmp->list);
2100 list_add(&tmp->list, &removed);
2101
2102 tmp = amdgpu_vm_it_iter_next(tmp, saddr, eaddr);
2103 }
2104
2105 /* And free them up */
2106 list_for_each_entry_safe(tmp, next, &removed, list) {
2107 amdgpu_vm_it_remove(tmp, &vm->va);
2108 list_del(&tmp->list);
2109
2110 if (tmp->start < saddr)
2111 tmp->start = saddr;
2112 if (tmp->last > eaddr)
2113 tmp->last = eaddr;
2114
2115 tmp->bo_va = NULL;
2116 list_add(&tmp->list, &vm->freed);
2117 trace_amdgpu_vm_bo_unmap(NULL, tmp);
2118 }
2119
2120 /* Insert partial mapping before the range */
2121 if (!list_empty(&before->list)) {
2122 struct amdgpu_bo *bo = before->bo_va->base.bo;
2123
2124 amdgpu_vm_it_insert(before, &vm->va);
2125 if (before->flags & AMDGPU_VM_PAGE_PRT)
2126 amdgpu_vm_prt_get(adev);
2127
2128 if (amdgpu_vm_is_bo_always_valid(vm, bo) &&
2129 !before->bo_va->base.moved)
2130 amdgpu_vm_bo_needs_update(&before->bo_va->base);
2131 } else {
2132 kfree(before);
2133 }
2134
2135 /* Insert partial mapping after the range */
2136 if (!list_empty(&after->list)) {
2137 struct amdgpu_bo *bo = after->bo_va->base.bo;
2138
2139 amdgpu_vm_it_insert(after, &vm->va);
2140 if (after->flags & AMDGPU_VM_PAGE_PRT)
2141 amdgpu_vm_prt_get(adev);
2142
2143 if (amdgpu_vm_is_bo_always_valid(vm, bo) &&
2144 !after->bo_va->base.moved)
2145 amdgpu_vm_bo_needs_update(&after->bo_va->base);
2146 } else {
2147 kfree(after);
2148 }
2149
2150 return 0;
2151}
2152
2153/**
2154 * amdgpu_vm_bo_lookup_mapping - find mapping by address
2155 *
2156 * @vm: the requested VM
2157 * @addr: the address
2158 *
2159 * Find a mapping by it's address.
2160 *
2161 * Returns:
2162 * The amdgpu_bo_va_mapping matching for addr or NULL
2163 *
2164 */
2165struct amdgpu_bo_va_mapping *amdgpu_vm_bo_lookup_mapping(struct amdgpu_vm *vm,
2166 uint64_t addr)
2167{
2168 return amdgpu_vm_it_iter_first(&vm->va, addr, addr);
2169}
2170
2171/**
2172 * amdgpu_vm_bo_trace_cs - trace all reserved mappings
2173 *
2174 * @vm: the requested vm
2175 * @ticket: CS ticket
2176 *
2177 * Trace all mappings of BOs reserved during a command submission.
2178 */
2179void amdgpu_vm_bo_trace_cs(struct amdgpu_vm *vm, struct ww_acquire_ctx *ticket)
2180{
2181 struct amdgpu_bo_va_mapping *mapping;
2182
2183 if (!trace_amdgpu_vm_bo_cs_enabled())
2184 return;
2185
2186 for (mapping = amdgpu_vm_it_iter_first(&vm->va, 0, U64_MAX); mapping;
2187 mapping = amdgpu_vm_it_iter_next(mapping, 0, U64_MAX)) {
2188 if (mapping->bo_va && mapping->bo_va->base.bo) {
2189 struct amdgpu_bo *bo;
2190
2191 bo = mapping->bo_va->base.bo;
2192 if (dma_resv_locking_ctx(bo->tbo.base.resv) !=
2193 ticket)
2194 continue;
2195 }
2196
2197 trace_amdgpu_vm_bo_cs(mapping);
2198 }
2199}
2200
2201/**
2202 * amdgpu_vm_bo_del - remove a bo from a specific vm
2203 *
2204 * @adev: amdgpu_device pointer
2205 * @bo_va: requested bo_va
2206 *
2207 * Remove @bo_va->bo from the requested vm.
2208 *
2209 * Object have to be reserved!
2210 */
2211void amdgpu_vm_bo_del(struct amdgpu_device *adev,
2212 struct amdgpu_bo_va *bo_va)
2213{
2214 struct amdgpu_bo_va_mapping *mapping, *next;
2215 struct amdgpu_bo *bo = bo_va->base.bo;
2216 struct amdgpu_vm *vm = bo_va->base.vm;
2217 struct amdgpu_vm_bo_base **base;
2218
2219 dma_resv_assert_held(vm->root.bo->tbo.base.resv);
2220
2221 if (bo) {
2222 dma_resv_assert_held(bo->tbo.base.resv);
2223 if (amdgpu_vm_is_bo_always_valid(vm, bo))
2224 ttm_bo_set_bulk_move(&bo->tbo, NULL);
2225
2226 for (base = &bo_va->base.bo->vm_bo; *base;
2227 base = &(*base)->next) {
2228 if (*base != &bo_va->base)
2229 continue;
2230
2231 amdgpu_vm_update_stats(*base, bo->tbo.resource, -1);
2232 *base = bo_va->base.next;
2233 break;
2234 }
2235 }
2236
2237 spin_lock(&vm->individual_lock);
2238 list_del(&bo_va->base.vm_status);
2239 spin_unlock(&vm->individual_lock);
2240
2241 list_for_each_entry_safe(mapping, next, &bo_va->valids, list) {
2242 list_del(&mapping->list);
2243 amdgpu_vm_it_remove(mapping, &vm->va);
2244 mapping->bo_va = NULL;
2245 trace_amdgpu_vm_bo_unmap(bo_va, mapping);
2246 list_add(&mapping->list, &vm->freed);
2247 }
2248 list_for_each_entry_safe(mapping, next, &bo_va->invalids, list) {
2249 list_del(&mapping->list);
2250 amdgpu_vm_it_remove(mapping, &vm->va);
2251 amdgpu_vm_free_mapping(adev, vm, mapping,
2252 bo_va->last_pt_update);
2253 }
2254
2255 dma_fence_put(bo_va->last_pt_update);
2256
2257 if (bo && bo_va->is_xgmi)
2258 amdgpu_xgmi_set_pstate(adev, AMDGPU_XGMI_PSTATE_MIN);
2259
2260 kfree(bo_va);
2261}
2262
2263/**
2264 * amdgpu_vm_evictable - check if we can evict a VM
2265 *
2266 * @bo: A page table of the VM.
2267 *
2268 * Check if it is possible to evict a VM.
2269 */
2270bool amdgpu_vm_evictable(struct amdgpu_bo *bo)
2271{
2272 struct amdgpu_vm_bo_base *bo_base = bo->vm_bo;
2273
2274 /* Page tables of a destroyed VM can go away immediately */
2275 if (!bo_base || !bo_base->vm)
2276 return true;
2277
2278 /* Don't evict VM page tables while they are busy */
2279 if (!dma_resv_test_signaled(bo->tbo.base.resv, DMA_RESV_USAGE_BOOKKEEP))
2280 return false;
2281
2282 /* Try to block ongoing updates */
2283 if (!amdgpu_vm_eviction_trylock(bo_base->vm))
2284 return false;
2285
2286 /* Don't evict VM page tables while they are updated */
2287 if (!dma_fence_is_signaled(bo_base->vm->last_unlocked)) {
2288 amdgpu_vm_eviction_unlock(bo_base->vm);
2289 return false;
2290 }
2291
2292 bo_base->vm->evicting = true;
2293 amdgpu_vm_eviction_unlock(bo_base->vm);
2294 return true;
2295}
2296
2297/**
2298 * amdgpu_vm_bo_invalidate - mark the bo as invalid
2299 *
2300 * @bo: amdgpu buffer object
2301 * @evicted: is the BO evicted
2302 *
2303 * Mark @bo as invalid.
2304 */
2305void amdgpu_vm_bo_invalidate(struct amdgpu_bo *bo, bool evicted)
2306{
2307 struct amdgpu_vm_bo_base *bo_base;
2308
2309 for (bo_base = bo->vm_bo; bo_base; bo_base = bo_base->next) {
2310 struct amdgpu_vm *vm = bo_base->vm;
2311
2312 if (evicted && amdgpu_vm_is_bo_always_valid(vm, bo)) {
2313 amdgpu_vm_bo_evicted(bo_base);
2314 continue;
2315 }
2316
2317 if (bo_base->moved)
2318 continue;
2319 bo_base->moved = true;
2320 amdgpu_vm_bo_needs_update(bo_base);
2321 }
2322}
2323
2324/**
2325 * amdgpu_vm_bo_move - handle BO move
2326 *
2327 * @bo: amdgpu buffer object
2328 * @new_mem: the new placement of the BO move
2329 * @evicted: is the BO evicted
2330 *
2331 * Update the memory stats for the new placement and mark @bo as invalid.
2332 */
2333void amdgpu_vm_bo_move(struct amdgpu_bo *bo, struct ttm_resource *new_mem,
2334 bool evicted)
2335{
2336 struct amdgpu_vm_bo_base *bo_base;
2337
2338 for (bo_base = bo->vm_bo; bo_base; bo_base = bo_base->next) {
2339 struct amdgpu_vm *vm = bo_base->vm;
2340
2341 spin_lock(&vm->stats_lock);
2342 amdgpu_vm_update_stats_locked(bo_base, bo->tbo.resource, -1);
2343 amdgpu_vm_update_stats_locked(bo_base, new_mem, +1);
2344 spin_unlock(&vm->stats_lock);
2345 }
2346
2347 amdgpu_vm_bo_invalidate(bo, evicted);
2348}
2349
2350/**
2351 * amdgpu_vm_get_block_size - calculate VM page table size as power of two
2352 *
2353 * @vm_size: VM size
2354 *
2355 * Returns:
2356 * VM page table as power of two
2357 */
2358static uint32_t amdgpu_vm_get_block_size(uint64_t vm_size)
2359{
2360 /* Total bits covered by PD + PTs */
2361 unsigned bits = ilog2(vm_size) + 18;
2362
2363 /* Make sure the PD is 4K in size up to 8GB address space.
2364 Above that split equal between PD and PTs */
2365 if (vm_size <= 8)
2366 return (bits - 9);
2367 else
2368 return ((bits + 3) / 2);
2369}
2370
2371/**
2372 * amdgpu_vm_adjust_size - adjust vm size, block size and fragment size
2373 *
2374 * @adev: amdgpu_device pointer
2375 * @min_vm_size: the minimum vm size in GB if it's set auto
2376 * @fragment_size_default: Default PTE fragment size
2377 * @max_level: max VMPT level
2378 * @max_bits: max address space size in bits
2379 *
2380 */
2381void amdgpu_vm_adjust_size(struct amdgpu_device *adev, uint32_t min_vm_size,
2382 uint32_t fragment_size_default, unsigned max_level,
2383 unsigned max_bits)
2384{
2385 unsigned int max_size = 1 << (max_bits - 30);
2386 unsigned int vm_size;
2387 uint64_t tmp;
2388
2389 /* adjust vm size first */
2390 if (amdgpu_vm_size != -1) {
2391 vm_size = amdgpu_vm_size;
2392 if (vm_size > max_size) {
2393 dev_warn(adev->dev, "VM size (%d) too large, max is %u GB\n",
2394 amdgpu_vm_size, max_size);
2395 vm_size = max_size;
2396 }
2397 } else {
2398 struct sysinfo si;
2399 unsigned int phys_ram_gb;
2400
2401 /* Optimal VM size depends on the amount of physical
2402 * RAM available. Underlying requirements and
2403 * assumptions:
2404 *
2405 * - Need to map system memory and VRAM from all GPUs
2406 * - VRAM from other GPUs not known here
2407 * - Assume VRAM <= system memory
2408 * - On GFX8 and older, VM space can be segmented for
2409 * different MTYPEs
2410 * - Need to allow room for fragmentation, guard pages etc.
2411 *
2412 * This adds up to a rough guess of system memory x3.
2413 * Round up to power of two to maximize the available
2414 * VM size with the given page table size.
2415 */
2416 si_meminfo(&si);
2417 phys_ram_gb = ((uint64_t)si.totalram * si.mem_unit +
2418 (1 << 30) - 1) >> 30;
2419 vm_size = roundup_pow_of_two(
2420 clamp(phys_ram_gb * 3, min_vm_size, max_size));
2421 }
2422
2423 adev->vm_manager.max_pfn = (uint64_t)vm_size << 18;
2424 adev->vm_manager.max_level = max_level;
2425
2426 tmp = roundup_pow_of_two(adev->vm_manager.max_pfn);
2427 if (amdgpu_vm_block_size != -1)
2428 tmp >>= amdgpu_vm_block_size - 9;
2429 tmp = DIV_ROUND_UP(fls64(tmp) - 1, 9) - 1;
2430 adev->vm_manager.num_level = min_t(unsigned int, max_level, tmp);
2431 switch (adev->vm_manager.num_level) {
2432 case 4:
2433 adev->vm_manager.root_level = AMDGPU_VM_PDB3;
2434 break;
2435 case 3:
2436 adev->vm_manager.root_level = AMDGPU_VM_PDB2;
2437 break;
2438 case 2:
2439 adev->vm_manager.root_level = AMDGPU_VM_PDB1;
2440 break;
2441 case 1:
2442 adev->vm_manager.root_level = AMDGPU_VM_PDB0;
2443 break;
2444 default:
2445 dev_err(adev->dev, "VMPT only supports 2~4+1 levels\n");
2446 }
2447 /* block size depends on vm size and hw setup*/
2448 if (amdgpu_vm_block_size != -1)
2449 adev->vm_manager.block_size =
2450 min((unsigned)amdgpu_vm_block_size, max_bits
2451 - AMDGPU_GPU_PAGE_SHIFT
2452 - 9 * adev->vm_manager.num_level);
2453 else if (adev->vm_manager.num_level > 1)
2454 adev->vm_manager.block_size = 9;
2455 else
2456 adev->vm_manager.block_size = amdgpu_vm_get_block_size(tmp);
2457
2458 if (amdgpu_vm_fragment_size == -1)
2459 adev->vm_manager.fragment_size = fragment_size_default;
2460 else
2461 adev->vm_manager.fragment_size = amdgpu_vm_fragment_size;
2462
2463 dev_info(
2464 adev->dev,
2465 "vm size is %u GB, %u levels, block size is %u-bit, fragment size is %u-bit\n",
2466 vm_size, adev->vm_manager.num_level + 1,
2467 adev->vm_manager.block_size, adev->vm_manager.fragment_size);
2468}
2469
2470/**
2471 * amdgpu_vm_wait_idle - wait for the VM to become idle
2472 *
2473 * @vm: VM object to wait for
2474 * @timeout: timeout to wait for VM to become idle
2475 */
2476long amdgpu_vm_wait_idle(struct amdgpu_vm *vm, long timeout)
2477{
2478 timeout = drm_sched_entity_flush(&vm->immediate, timeout);
2479 if (timeout <= 0)
2480 return timeout;
2481
2482 return drm_sched_entity_flush(&vm->delayed, timeout);
2483}
2484
2485static void amdgpu_vm_destroy_task_info(struct kref *kref)
2486{
2487 struct amdgpu_task_info *ti = container_of(kref, struct amdgpu_task_info, refcount);
2488
2489 kfree(ti);
2490}
2491
2492/**
2493 * amdgpu_vm_put_task_info - reference down the vm task_info ptr
2494 *
2495 * @task_info: task_info struct under discussion.
2496 *
2497 * frees the vm task_info ptr at the last put
2498 */
2499void amdgpu_vm_put_task_info(struct amdgpu_task_info *task_info)
2500{
2501 if (task_info)
2502 kref_put(&task_info->refcount, amdgpu_vm_destroy_task_info);
2503}
2504
2505/**
2506 * amdgpu_vm_get_task_info_vm - Extracts task info for a vm.
2507 *
2508 * @vm: VM to get info from
2509 *
2510 * Returns the reference counted task_info structure, which must be
2511 * referenced down with amdgpu_vm_put_task_info.
2512 */
2513struct amdgpu_task_info *
2514amdgpu_vm_get_task_info_vm(struct amdgpu_vm *vm)
2515{
2516 struct amdgpu_task_info *ti = NULL;
2517
2518 if (vm) {
2519 ti = vm->task_info;
2520 kref_get(&vm->task_info->refcount);
2521 }
2522
2523 return ti;
2524}
2525
2526/**
2527 * amdgpu_vm_get_task_info_pasid - Extracts task info for a PASID.
2528 *
2529 * @adev: drm device pointer
2530 * @pasid: PASID identifier for VM
2531 *
2532 * Returns the reference counted task_info structure, which must be
2533 * referenced down with amdgpu_vm_put_task_info.
2534 */
2535struct amdgpu_task_info *
2536amdgpu_vm_get_task_info_pasid(struct amdgpu_device *adev, u32 pasid)
2537{
2538 struct amdgpu_fpriv *fpriv;
2539 struct amdgpu_task_info *ti;
2540 struct amdgpu_vm *vm;
2541 unsigned long flags;
2542
2543 amdgpu_pasid_lock(&flags);
2544 fpriv = amdgpu_pasid_get_fpriv_locked(pasid);
2545 vm = fpriv ? &fpriv->vm : NULL;
2546 ti = amdgpu_vm_get_task_info_vm(vm);
2547 amdgpu_pasid_unlock(flags);
2548
2549 return ti;
2550}
2551
2552static int amdgpu_vm_create_task_info(struct amdgpu_vm *vm)
2553{
2554 vm->task_info = kzalloc_obj(struct amdgpu_task_info);
2555 if (!vm->task_info)
2556 return -ENOMEM;
2557
2558 kref_init(&vm->task_info->refcount);
2559 return 0;
2560}
2561
2562/**
2563 * amdgpu_vm_set_task_info - Sets VMs task info.
2564 *
2565 * @vm: vm for which to set the info
2566 */
2567void amdgpu_vm_set_task_info(struct amdgpu_vm *vm)
2568{
2569 if (!vm->task_info)
2570 return;
2571
2572 if (vm->task_info->task.pid == current->pid)
2573 return;
2574
2575 vm->task_info->task.pid = current->pid;
2576 get_task_comm(vm->task_info->task.comm, current);
2577
2578 vm->task_info->tgid = current->tgid;
2579 get_task_comm(vm->task_info->process_name, current->group_leader);
2580}
2581
2582/**
2583 * amdgpu_vm_init - initialize a vm instance
2584 *
2585 * @adev: amdgpu_device pointer
2586 * @vm: requested vm
2587 * @xcp_id: GPU partition selection id
2588 *
2589 * Init @vm fields.
2590 *
2591 * Returns:
2592 * 0 for success, error for failure.
2593 */
2594int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm,
2595 int32_t xcp_id)
2596{
2597 struct amdgpu_bo *root_bo;
2598 struct amdgpu_bo_vm *root;
2599 int r, i;
2600
2601 vm->va = RB_ROOT_CACHED;
2602 for (i = 0; i < AMDGPU_MAX_VMHUBS; i++)
2603 vm->reserved_vmid[i] = NULL;
2604
2605 amdgpu_vm_bo_status_init(&vm->kernel);
2606 amdgpu_vm_bo_status_init(&vm->always_valid);
2607 spin_lock_init(&vm->individual_lock);
2608 amdgpu_vm_bo_status_init(&vm->individual);
2609 INIT_LIST_HEAD(&vm->freed);
2610 INIT_KFIFO(vm->faults);
2611 spin_lock_init(&vm->stats_lock);
2612
2613 r = amdgpu_vm_init_entities(adev, vm);
2614 if (r)
2615 return r;
2616
2617 ttm_lru_bulk_move_init(&vm->lru_bulk_move);
2618
2619 vm->is_compute_context = false;
2620 vm->need_tlb_fence = amdgpu_userq_enabled(&adev->ddev);
2621
2622 vm->use_cpu_for_update = !!(adev->vm_manager.vm_update_mode &
2623 AMDGPU_VM_USE_CPU_FOR_GFX);
2624
2625 dev_dbg(adev->dev, "VM update mode is %s\n",
2626 vm->use_cpu_for_update ? "CPU" : "SDMA");
2627 WARN_ONCE((vm->use_cpu_for_update &&
2628 !amdgpu_gmc_vram_full_visible(&adev->gmc)),
2629 "CPU update of VM recommended only for large BAR system\n");
2630
2631 if (vm->use_cpu_for_update)
2632 vm->update_funcs = &amdgpu_vm_cpu_funcs;
2633 else
2634 vm->update_funcs = &amdgpu_vm_sdma_funcs;
2635
2636 vm->last_update = dma_fence_get_stub();
2637 vm->last_unlocked = dma_fence_get_stub();
2638 vm->last_tlb_flush = dma_fence_get_stub();
2639 vm->generation = amdgpu_vm_generation(adev, NULL);
2640
2641 mutex_init(&vm->eviction_lock);
2642 vm->evicting = false;
2643 vm->tlb_fence_context = dma_fence_context_alloc(1);
2644
2645 r = amdgpu_vm_pt_create(adev, vm, adev->vm_manager.root_level,
2646 false, &root, xcp_id);
2647 if (r)
2648 goto error_free_delayed;
2649
2650 root_bo = amdgpu_bo_ref(&root->bo);
2651 r = amdgpu_bo_reserve(root_bo, true);
2652 if (r) {
2653 amdgpu_bo_unref(&root_bo);
2654 goto error_free_delayed;
2655 }
2656
2657 amdgpu_vm_bo_base_init(&vm->root, vm, root_bo);
2658 r = dma_resv_reserve_fences(root_bo->tbo.base.resv, 1);
2659 if (r)
2660 goto error_free_root;
2661
2662 r = amdgpu_vm_pt_clear(adev, vm, root, false);
2663 if (r)
2664 goto error_free_root;
2665
2666 r = amdgpu_vm_create_task_info(vm);
2667 if (r)
2668 dev_dbg(adev->dev, "Failed to create task info for VM\n");
2669
2670 amdgpu_bo_unreserve(vm->root.bo);
2671 amdgpu_bo_unref(&root_bo);
2672
2673 return 0;
2674
2675error_free_root:
2676 amdgpu_vm_pt_free_root(adev, vm);
2677 amdgpu_bo_unreserve(vm->root.bo);
2678 amdgpu_bo_unref(&root_bo);
2679
2680error_free_delayed:
2681 dma_fence_put(vm->last_update);
2682 dma_fence_put(vm->last_tlb_flush);
2683 dma_fence_put(vm->last_unlocked);
2684 ttm_lru_bulk_move_fini(&adev->mman.bdev, &vm->lru_bulk_move);
2685 amdgpu_vm_fini_entities(vm);
2686
2687 return r;
2688}
2689
2690/**
2691 * amdgpu_vm_make_compute - Turn a GFX VM into a compute VM
2692 *
2693 * @adev: amdgpu_device pointer
2694 * @vm: requested vm
2695 *
2696 * This only works on GFX VMs that don't have any BOs added and no
2697 * page tables allocated yet.
2698 *
2699 * Changes the following VM parameters:
2700 * - use_cpu_for_update
2701 * - pte_supports_ats
2702 *
2703 * Reinitializes the page directory to reflect the changed ATS
2704 * setting.
2705 *
2706 * Returns:
2707 * 0 for success, -errno for errors.
2708 */
2709int amdgpu_vm_make_compute(struct amdgpu_device *adev, struct amdgpu_vm *vm)
2710{
2711 int r;
2712
2713 r = amdgpu_bo_reserve(vm->root.bo, true);
2714 if (r)
2715 return r;
2716
2717 /* Update VM state */
2718 vm->use_cpu_for_update = !!(adev->vm_manager.vm_update_mode &
2719 AMDGPU_VM_USE_CPU_FOR_COMPUTE);
2720 dev_dbg(adev->dev, "VM update mode is %s\n",
2721 vm->use_cpu_for_update ? "CPU" : "SDMA");
2722 WARN_ONCE((vm->use_cpu_for_update &&
2723 !amdgpu_gmc_vram_full_visible(&adev->gmc)),
2724 "CPU update of VM recommended only for large BAR system\n");
2725
2726 if (vm->use_cpu_for_update) {
2727 /* Sync with last SDMA update/clear before switching to CPU */
2728 r = amdgpu_bo_sync_wait(vm->root.bo,
2729 AMDGPU_FENCE_OWNER_UNDEFINED, true);
2730 if (r)
2731 goto unreserve_bo;
2732
2733 vm->update_funcs = &amdgpu_vm_cpu_funcs;
2734 r = amdgpu_vm_pt_map_tables(adev, vm);
2735 if (r)
2736 goto unreserve_bo;
2737
2738 } else {
2739 vm->update_funcs = &amdgpu_vm_sdma_funcs;
2740 }
2741
2742 dma_fence_put(vm->last_update);
2743 vm->last_update = dma_fence_get_stub();
2744 vm->is_compute_context = true;
2745 vm->need_tlb_fence = true;
2746
2747unreserve_bo:
2748 amdgpu_bo_unreserve(vm->root.bo);
2749 return r;
2750}
2751
2752static int amdgpu_vm_stats_is_zero(struct amdgpu_vm *vm)
2753{
2754 for (int i = 0; i < __AMDGPU_PL_NUM; ++i) {
2755 if (!(drm_memory_stats_is_zero(&vm->stats[i].drm) &&
2756 vm->stats[i].evicted == 0))
2757 return false;
2758 }
2759 return true;
2760}
2761
2762/**
2763 * amdgpu_vm_fini - tear down a vm instance
2764 *
2765 * @adev: amdgpu_device pointer
2766 * @vm: requested vm
2767 *
2768 * Tear down @vm.
2769 * Unbind the VM and remove all bos from the vm bo list
2770 */
2771void amdgpu_vm_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm)
2772{
2773 struct amdgpu_bo_va_mapping *mapping, *tmp;
2774 bool prt_fini_needed = !!adev->gmc.gmc_funcs->set_prt;
2775 struct amdgpu_bo *root;
2776 unsigned long flags;
2777 int i;
2778
2779 amdgpu_amdkfd_gpuvm_destroy_cb(adev, vm);
2780
2781 root = amdgpu_bo_ref(vm->root.bo);
2782 amdgpu_bo_reserve(root, true);
2783 dma_fence_wait(vm->last_unlocked, false);
2784 dma_fence_put(vm->last_unlocked);
2785 dma_fence_wait(vm->last_tlb_flush, false);
2786 /* Make sure that all fence callbacks have completed */
2787 dma_fence_lock_irqsave(vm->last_tlb_flush, flags);
2788 dma_fence_unlock_irqrestore(vm->last_tlb_flush, flags);
2789 dma_fence_put(vm->last_tlb_flush);
2790
2791 list_for_each_entry_safe(mapping, tmp, &vm->freed, list) {
2792 if (mapping->flags & AMDGPU_VM_PAGE_PRT && prt_fini_needed) {
2793 amdgpu_vm_prt_fini(adev, vm);
2794 prt_fini_needed = false;
2795 }
2796
2797 list_del(&mapping->list);
2798 amdgpu_vm_free_mapping(adev, vm, mapping, NULL);
2799 }
2800
2801 amdgpu_vm_pt_free_root(adev, vm);
2802 amdgpu_bo_unreserve(root);
2803 amdgpu_bo_unref(&root);
2804 WARN_ON(vm->root.bo);
2805
2806 amdgpu_vm_fini_entities(vm);
2807
2808 if (!RB_EMPTY_ROOT(&vm->va.rb_root)) {
2809 dev_err(adev->dev, "still active bo inside vm\n");
2810 }
2811 rbtree_postorder_for_each_entry_safe(mapping, tmp,
2812 &vm->va.rb_root, rb) {
2813 /* Don't remove the mapping here, we don't want to trigger a
2814 * rebalance and the tree is about to be destroyed anyway.
2815 */
2816 list_del(&mapping->list);
2817 kfree(mapping);
2818 }
2819
2820 dma_fence_put(vm->last_update);
2821
2822 for (i = 0; i < AMDGPU_MAX_VMHUBS; i++) {
2823 amdgpu_vmid_free_reserved(adev, vm, i);
2824 }
2825
2826 ttm_lru_bulk_move_fini(&adev->mman.bdev, &vm->lru_bulk_move);
2827
2828 if (!amdgpu_vm_stats_is_zero(vm)) {
2829 struct amdgpu_task_info *ti = vm->task_info;
2830
2831 dev_warn(adev->dev,
2832 "VM memory stats for proc %s(%d) task %s(%d) is non-zero when fini\n",
2833 ti->process_name, ti->task.pid, ti->task.comm, ti->tgid);
2834 }
2835
2836 amdgpu_vm_put_task_info(vm->task_info);
2837}
2838
2839/**
2840 * amdgpu_vm_manager_init - init the VM manager
2841 *
2842 * @adev: amdgpu_device pointer
2843 *
2844 * Initialize the VM manager structures
2845 */
2846void amdgpu_vm_manager_init(struct amdgpu_device *adev)
2847{
2848 /* Concurrent flushes are only possible starting with Vega10 and
2849 * are broken on Navi10 and Navi14.
2850 */
2851 adev->vm_manager.concurrent_flush = !(adev->asic_type < CHIP_VEGA10 ||
2852 adev->asic_type == CHIP_NAVI10 ||
2853 adev->asic_type == CHIP_NAVI14);
2854 amdgpu_vmid_mgr_init(adev);
2855
2856 spin_lock_init(&adev->vm_manager.prt_lock);
2857 atomic_set(&adev->vm_manager.num_prt_users, 0);
2858
2859 /* If not overridden by the user, by default, only in large BAR systems
2860 * Compute VM tables will be updated by CPU
2861 */
2862#ifdef CONFIG_X86_64
2863 if (amdgpu_vm_update_mode == -1) {
2864 /* For asic with VF MMIO access protection
2865 * avoid using CPU for VM table updates
2866 */
2867 if (amdgpu_gmc_vram_full_visible(&adev->gmc) &&
2868 !amdgpu_sriov_vf_mmio_access_protection(adev))
2869 adev->vm_manager.vm_update_mode =
2870 AMDGPU_VM_USE_CPU_FOR_COMPUTE;
2871 else
2872 adev->vm_manager.vm_update_mode = 0;
2873 } else
2874 adev->vm_manager.vm_update_mode = amdgpu_vm_update_mode;
2875#else
2876 adev->vm_manager.vm_update_mode = 0;
2877#endif
2878}
2879
2880/**
2881 * amdgpu_vm_manager_fini - cleanup VM manager
2882 *
2883 * @adev: amdgpu_device pointer
2884 *
2885 * Cleanup the VM manager and free resources.
2886 */
2887void amdgpu_vm_manager_fini(struct amdgpu_device *adev)
2888{
2889 amdgpu_vmid_mgr_fini(adev);
2890 amdgpu_pasid_mgr_cleanup();
2891}
2892
2893/**
2894 * amdgpu_vm_ioctl - Manages VMID reservation for vm hubs.
2895 *
2896 * @dev: drm device pointer
2897 * @data: drm_amdgpu_vm
2898 * @filp: drm file pointer
2899 *
2900 * Returns:
2901 * 0 for success, -errno for errors.
2902 */
2903int amdgpu_vm_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
2904{
2905 union drm_amdgpu_vm *args = data;
2906 struct amdgpu_device *adev = drm_to_adev(dev);
2907 struct amdgpu_fpriv *fpriv = filp->driver_priv;
2908 struct amdgpu_vm *vm = &fpriv->vm;
2909
2910 /* No valid flags defined yet */
2911 if (args->in.flags)
2912 return -EINVAL;
2913
2914 switch (args->in.op) {
2915 case AMDGPU_VM_OP_RESERVE_VMID:
2916 /* We only have requirement to reserve vmid from gfxhub */
2917 return amdgpu_vmid_alloc_reserved(adev, vm, AMDGPU_GFXHUB(0));
2918 case AMDGPU_VM_OP_UNRESERVE_VMID:
2919 amdgpu_vmid_free_reserved(adev, vm, AMDGPU_GFXHUB(0));
2920 break;
2921 default:
2922 return -EINVAL;
2923 }
2924
2925 return 0;
2926}
2927
2928/**
2929 * amdgpu_vm_lock_by_pasid - look up a VM by PASID and lock its root PD
2930 * @adev: amdgpu device pointer
2931 * @pasid: PASID of the VM
2932 * @exec: drm_exec context to lock the root PD in
2933 *
2934 * Must be called from within a drm_exec_until_all_locked() loop; the caller
2935 * runs drm_exec_retry_on_contention() afterwards. The drm_exec context holds
2936 * a reference on the root BO until it is finalised.
2937 *
2938 * Return: the VM on success, or NULL if the PASID has no VM, the VM is being
2939 * torn down, or locking the root PD failed.
2940 */
2941struct amdgpu_vm *amdgpu_vm_lock_by_pasid(struct amdgpu_device *adev,
2942 u32 pasid, struct drm_exec *exec)
2943{
2944 unsigned long irqflags;
2945 struct amdgpu_fpriv *fpriv;
2946 struct amdgpu_bo *root;
2947 struct amdgpu_vm *vm;
2948 int r;
2949
2950 amdgpu_pasid_lock(&irqflags);
2951 fpriv = amdgpu_pasid_get_fpriv_locked(pasid);
2952 vm = fpriv ? &fpriv->vm : NULL;
2953 root = vm && vm->root.bo ? amdgpu_bo_ref(vm->root.bo) : NULL;
2954 amdgpu_pasid_unlock(irqflags);
2955
2956 if (!root)
2957 return NULL;
2958
2959 r = drm_exec_lock_obj(exec, &root->tbo.base);
2960 if (r) {
2961 amdgpu_bo_unref(&root);
2962 return NULL;
2963 }
2964
2965 /* Double check that the VM still exists */
2966 amdgpu_pasid_lock(&irqflags);
2967 fpriv = amdgpu_pasid_get_fpriv_locked(pasid);
2968 if (!fpriv) {
2969 vm = NULL;
2970 } else {
2971 vm = &fpriv->vm;
2972 if (vm->root.bo != root)
2973 vm = NULL;
2974 }
2975 amdgpu_pasid_unlock(irqflags);
2976
2977 if (!vm) {
2978 drm_exec_unlock_obj(exec, &root->tbo.base);
2979 amdgpu_bo_unref(&root);
2980 return NULL;
2981 }
2982
2983 /* The drm_exec context holds its own reference on the root BO. */
2984 amdgpu_bo_unref(&root);
2985
2986 return vm;
2987}
2988
2989/**
2990 * amdgpu_vm_handle_fault - graceful handling of VM faults.
2991 * @adev: amdgpu device pointer
2992 * @pasid: PASID of the VM
2993 * @ts: Timestamp of the fault
2994 * @vmid: VMID, only used for GFX 9.4.3.
2995 * @node_id: Node_id received in IH cookie. Only applicable for
2996 * GFX 9.4.3.
2997 * @addr: Address of the fault
2998 * @write_fault: true is write fault, false is read fault
2999 *
3000 * Try to gracefully handle a VM fault. Return true if the fault was handled and
3001 * shouldn't be reported any more.
3002 */
3003bool amdgpu_vm_handle_fault(struct amdgpu_device *adev, u32 pasid,
3004 u32 vmid, u32 node_id, uint64_t addr,
3005 uint64_t ts, bool write_fault)
3006{
3007 bool is_compute_context = false;
3008 struct drm_exec exec;
3009 uint64_t value, flags;
3010 struct amdgpu_vm *vm;
3011 int r;
3012
3013 drm_exec_init(&exec, 0, 1);
3014 drm_exec_until_all_locked(&exec) {
3015 vm = amdgpu_vm_lock_by_pasid(adev, pasid, &exec);
3016 drm_exec_retry_on_contention(&exec);
3017 if (!vm)
3018 break;
3019 }
3020 if (!vm) {
3021 drm_exec_fini(&exec);
3022 return false;
3023 }
3024
3025 is_compute_context = vm->is_compute_context;
3026
3027 if (is_compute_context) {
3028 __label__ drm_exec_retry;
3029
3030 /* Release the root PD lock since svm_range_restore_pages
3031 * might try to take it.
3032 * TODO: rework svm_range_restore_pages so that this isn't
3033 * necessary.
3034 */
3035 drm_exec_fini(&exec);
3036
3037 if (!svm_range_restore_pages(adev, pasid, vmid,
3038 node_id, addr >> PAGE_SHIFT, ts, write_fault))
3039 return true;
3040
3041 /* Re-acquire the VM lock, could be that the VM was freed in between. */
3042 drm_exec_init(&exec, 0, 1);
3043 drm_exec_until_all_locked(&exec) {
3044 vm = amdgpu_vm_lock_by_pasid(adev, pasid, &exec);
3045 drm_exec_retry_on_contention(&exec);
3046 if (!vm)
3047 break;
3048 }
3049 if (!vm) {
3050 drm_exec_fini(&exec);
3051 return false;
3052 }
3053 }
3054
3055 addr /= AMDGPU_GPU_PAGE_SIZE;
3056 flags = AMDGPU_PTE_VALID | AMDGPU_PTE_SNOOPED |
3057 AMDGPU_PTE_SYSTEM;
3058
3059 if (is_compute_context) {
3060 /* Intentionally setting invalid PTE flag
3061 * combination to force a no-retry-fault
3062 */
3063 flags = AMDGPU_VM_NORETRY_FLAGS;
3064 value = 0;
3065 } else if (amdgpu_vm_fault_stop == AMDGPU_VM_FAULT_STOP_NEVER) {
3066 /* Redirect the access to the dummy page */
3067 value = adev->dummy_page_addr;
3068 flags |= AMDGPU_PTE_EXECUTABLE | AMDGPU_PTE_READABLE |
3069 AMDGPU_PTE_WRITEABLE;
3070
3071 } else {
3072 /* Let the hw retry silently on the PTE */
3073 value = 0;
3074 }
3075
3076 r = dma_resv_reserve_fences(vm->root.bo->tbo.base.resv, 1);
3077 if (r) {
3078 pr_debug("failed %d to reserve fence slot\n", r);
3079 goto error_unlock;
3080 }
3081
3082 r = amdgpu_vm_update_range(adev, vm, true, false, false, false,
3083 NULL, addr, addr, flags, value, 0, NULL, NULL, NULL);
3084 if (r)
3085 goto error_unlock;
3086
3087 r = amdgpu_vm_update_pdes(adev, vm, true);
3088
3089error_unlock:
3090 drm_exec_fini(&exec);
3091 if (r < 0)
3092 dev_err(adev->dev, "Can't handle page fault (%d)\n", r);
3093
3094 return false;
3095}
3096
3097#if defined(CONFIG_DEBUG_FS)
3098
3099/* print the debug info for a specific set of status lists */
3100static void amdgpu_debugfs_vm_bo_status_info(struct seq_file *m,
3101 struct amdgpu_vm_bo_status *lists)
3102{
3103 struct amdgpu_vm_bo_base *base;
3104 unsigned int id;
3105
3106 id = 0;
3107 seq_puts(m, "\tEvicted BOs:\n");
3108 list_for_each_entry(base, &lists->evicted, vm_status) {
3109 if (!base->bo)
3110 continue;
3111
3112 amdgpu_bo_print_info(id++, base->bo, m);
3113 }
3114
3115 id = 0;
3116 seq_puts(m, "\tMoved BOs:\n");
3117 list_for_each_entry(base, &lists->needs_update, vm_status) {
3118 if (!base->bo)
3119 continue;
3120
3121 amdgpu_bo_print_info(id++, base->bo, m);
3122 }
3123
3124 id = 0;
3125 seq_puts(m, "\tIdle BOs:\n");
3126 list_for_each_entry(base, &lists->idle, vm_status) {
3127 if (!base->bo)
3128 continue;
3129
3130 amdgpu_bo_print_info(id++, base->bo, m);
3131 }
3132}
3133
3134/**
3135 * amdgpu_debugfs_vm_bo_info - print BO info for the VM
3136 *
3137 * @vm: Requested VM for printing BO info
3138 * @m: debugfs file
3139 *
3140 * Print BO information in debugfs file for the VM
3141 */
3142void amdgpu_debugfs_vm_bo_info(struct amdgpu_vm *vm, struct seq_file *m)
3143{
3144 amdgpu_vm_assert_locked(vm);
3145
3146 seq_puts(m, "\tKernel PT/PDs:\n");
3147 amdgpu_debugfs_vm_bo_status_info(m, &vm->kernel);
3148
3149 seq_puts(m, "\tPer VM BOs:\n");
3150 amdgpu_debugfs_vm_bo_status_info(m, &vm->always_valid);
3151
3152 seq_puts(m, "\tIndividual BOs:\n");
3153 spin_lock(&vm->individual_lock);
3154 amdgpu_debugfs_vm_bo_status_info(m, &vm->individual);
3155 spin_unlock(&vm->individual_lock);
3156}
3157#endif
3158
3159/**
3160 * amdgpu_vm_update_fault_cache - update cached fault into.
3161 * @adev: amdgpu device pointer
3162 * @pasid: PASID of the VM
3163 * @addr: Address of the fault
3164 * @status: GPUVM fault status register
3165 * @vmhub: which vmhub got the fault
3166 *
3167 * Cache the fault info for later use by userspace in debugging.
3168 */
3169void amdgpu_vm_update_fault_cache(struct amdgpu_device *adev,
3170 unsigned int pasid,
3171 uint64_t addr,
3172 uint32_t status,
3173 unsigned int vmhub)
3174{
3175 struct amdgpu_fpriv *fpriv;
3176 struct amdgpu_vm *vm;
3177 unsigned long flags;
3178
3179 amdgpu_pasid_lock(&flags);
3180
3181 fpriv = amdgpu_pasid_get_fpriv_locked(pasid);
3182 vm = fpriv ? &fpriv->vm : NULL;
3183 /* Don't update the fault cache if status is 0. In the multiple
3184 * fault case, subsequent faults will return a 0 status which is
3185 * useless for userspace and replaces the useful fault status, so
3186 * only update if status is non-0.
3187 */
3188 if (vm && status) {
3189 vm->fault_info.addr = addr;
3190 vm->fault_info.status = status;
3191 /*
3192 * Update the fault information globally for later usage
3193 * when vm could be stale or freed.
3194 */
3195 adev->vm_manager.fault_info.addr = addr;
3196 adev->vm_manager.fault_info.vmhub = vmhub;
3197 adev->vm_manager.fault_info.status = status;
3198
3199 if (AMDGPU_IS_GFXHUB(vmhub)) {
3200 vm->fault_info.vmhub = AMDGPU_VMHUB_TYPE_GFX;
3201 vm->fault_info.vmhub |=
3202 (vmhub - AMDGPU_GFXHUB_START) << AMDGPU_VMHUB_IDX_SHIFT;
3203 } else if (AMDGPU_IS_MMHUB0(vmhub)) {
3204 vm->fault_info.vmhub = AMDGPU_VMHUB_TYPE_MM0;
3205 vm->fault_info.vmhub |=
3206 (vmhub - AMDGPU_MMHUB0_START) << AMDGPU_VMHUB_IDX_SHIFT;
3207 } else if (AMDGPU_IS_MMHUB1(vmhub)) {
3208 vm->fault_info.vmhub = AMDGPU_VMHUB_TYPE_MM1;
3209 vm->fault_info.vmhub |=
3210 (vmhub - AMDGPU_MMHUB1_START) << AMDGPU_VMHUB_IDX_SHIFT;
3211 } else {
3212 WARN_ONCE(1, "Invalid vmhub %u\n", vmhub);
3213 }
3214 }
3215 amdgpu_pasid_unlock(flags);
3216}
3217
3218void amdgpu_vm_print_task_info(struct amdgpu_device *adev,
3219 struct amdgpu_task_info *task_info)
3220{
3221 dev_err(adev->dev,
3222 " Process %s pid %d thread %s pid %d\n",
3223 task_info->process_name, task_info->tgid,
3224 task_info->task.comm, task_info->task.pid);
3225}
3226
3227void amdgpu_sdma_set_vm_pte_scheds(struct amdgpu_device *adev,
3228 const struct amdgpu_vm_pte_funcs *vm_pte_funcs)
3229{
3230 struct drm_gpu_scheduler *sched;
3231 int i;
3232
3233 for (i = 0; i < adev->sdma.num_instances; i++) {
3234 if (adev->sdma.has_page_queue)
3235 sched = &adev->sdma.instance[i].page.sched;
3236 else
3237 sched = &adev->sdma.instance[i].ring.sched;
3238 adev->vm_manager.vm_pte_scheds[i] = sched;
3239 }
3240 adev->vm_manager.vm_pte_num_scheds = adev->sdma.num_instances;
3241 adev->vm_manager.vm_pte_funcs = vm_pte_funcs;
3242}