projects/linux·blob master · refs

2637 lines · 73.9 KiB

1// SPDX-License-Identifier: MIT
2/*
3 * Copyright © 2020 Intel Corporation
4 */
5
6#include "xe_migrate.h"
7
8#include <linux/bitfield.h>
9#include <linux/sizes.h>
10
11#include <drm/drm_managed.h>
12#include <drm/drm_pagemap.h>
13#include <drm/ttm/ttm_tt.h>
14#include <uapi/drm/xe_drm.h>
15
16#include <generated/xe_wa_oob.h>
17
18#include "instructions/xe_gpu_commands.h"
19#include "instructions/xe_mi_commands.h"
20#include "regs/xe_gtt_defs.h"
21#include "tests/xe_test.h"
22#include "xe_assert.h"
23#include "xe_bb.h"
24#include "xe_bo.h"
25#include "xe_exec_queue.h"
26#include "xe_ggtt.h"
27#include "xe_gt.h"
28#include "xe_gt_printk.h"
29#include "xe_hw_engine.h"
30#include "xe_lrc.h"
31#include "xe_map.h"
32#include "xe_mem_pool.h"
33#include "xe_mocs.h"
34#include "xe_pat.h"
35#include "xe_printk.h"
36#include "xe_pt.h"
37#include "xe_res_cursor.h"
38#include "xe_sa.h"
39#include "xe_sched_job.h"
40#include "xe_sriov_vf_ccs.h"
41#include "xe_svm.h"
42#include "xe_sync.h"
43#include "xe_trace_bo.h"
44#include "xe_validation.h"
45#include "xe_vm.h"
46#include "xe_vram.h"
47
48/**
49 * struct xe_migrate - migrate context.
50 */
51struct xe_migrate {
52 /** @q: Default exec queue used for migration */
53 struct xe_exec_queue *q;
54 /** @tile: Backpointer to the tile this struct xe_migrate belongs to. */
55 struct xe_tile *tile;
56 /** @job_mutex: Timeline mutex for @eng. */
57 struct mutex job_mutex;
58 /** @pt_bo: Page-table buffer object. */
59 struct xe_bo *pt_bo;
60 /** @batch_base_ofs: VM offset of the migration batch buffer */
61 u64 batch_base_ofs;
62 /** @usm_batch_base_ofs: VM offset of the usm batch buffer */
63 u64 usm_batch_base_ofs;
64 /** @cleared_mem_ofs: VM offset of @cleared_bo. */
65 u64 cleared_mem_ofs;
66 /** @large_page_copy_ofs: VM offset of 2M pages used for large copies */
67 u64 large_page_copy_ofs;
68 /**
69 * @large_page_copy_pdes: BO offset to writeout 2M pages (PDEs) used for
70 * large copies
71 */
72 u64 large_page_copy_pdes;
73 /**
74 * @fence: dma-fence representing the last migration job batch.
75 * Protected by @job_mutex.
76 */
77 struct dma_fence *fence;
78 /**
79 * @vm_update_sa: For integrated, used to suballocate page-tables
80 * out of the pt_bo.
81 */
82 struct drm_suballoc_manager vm_update_sa;
83 /** @min_chunk_size: For dgfx, Minimum chunk size */
84 u64 min_chunk_size;
85};
86
87#define MAX_PREEMPTDISABLE_TRANSFER SZ_8M /* Around 1ms. */
88#define MAX_CCS_LIMITED_TRANSFER SZ_4M /* XE_PAGE_SIZE * (FIELD_MAX(XE2_CCS_SIZE_MASK) + 1) */
89#define NUM_KERNEL_PDE 15
90#define NUM_PT_SLOTS 32
91#define LEVEL0_PAGE_TABLE_ENCODE_SIZE SZ_2M
92#define MAX_NUM_PTE 512
93#define IDENTITY_OFFSET 256ULL
94
95/*
96 * Although MI_STORE_DATA_IMM's "length" field is 10-bits, 0x3FE is the largest
97 * legal value accepted. Since that instruction field is always stored in
98 * (val-2) format, this translates to 0x400 dwords for the true maximum length
99 * of the instruction. Subtracting the instruction header (1 dword) and
100 * address (2 dwords), that leaves 0x3FD dwords (0x1FE qwords) for PTE values.
101 */
102#define MAX_PTE_PER_SDI 0x1FEU
103
104static void xe_migrate_fini(void *arg)
105{
106 struct xe_migrate *m = arg;
107
108 xe_vm_lock(m->q->vm, false);
109 xe_bo_unpin(m->pt_bo);
110 xe_vm_unlock(m->q->vm);
111
112 dma_fence_put(m->fence);
113 xe_bo_put(m->pt_bo);
114 drm_suballoc_manager_fini(&m->vm_update_sa);
115 mutex_destroy(&m->job_mutex);
116 xe_vm_close_and_put(m->q->vm);
117 xe_exec_queue_put(m->q);
118}
119
120static inline u16 xe_migrate_pat_index(struct xe_device *xe,
121 enum ttm_caching caching,
122 bool is_comp_pte)
123{
124 enum xe_cache_level cache_level;
125
126 /*
127 * Select the appropriate PAT index for buffer object PTEs programmed
128 * by emit_pte(). We choose not to mess with xe_migrate_prepare_vm()
129 * yet, for simplicity.
130 */
131 if (is_comp_pte && GRAPHICS_VERx100(xe) >= 2000)
132 cache_level = XE_CACHE_NONE_COMPRESSION;
133 else if (caching == ttm_cached)
134 cache_level = XE_CACHE_WB;
135 else
136 cache_level = XE_CACHE_NONE;
137
138 return xe_cache_pat_idx(xe, cache_level);
139}
140
141static u64 xe_migrate_vm_addr(u64 slot, u32 level)
142{
143 XE_WARN_ON(slot >= NUM_PT_SLOTS);
144
145 /* First slot is reserved for mapping of PT bo and bb, start from 1 */
146 return (slot + 1ULL) << xe_pt_shift(level + 1);
147}
148
149static u64 xe_migrate_vram_ofs(struct xe_device *xe, u64 addr, bool is_comp_pte)
150{
151 /*
152 * Remove the DPA to get a correct offset into identity table for the
153 * migrate offset
154 */
155 u64 identity_offset = IDENTITY_OFFSET;
156
157 if (GRAPHICS_VER(xe) >= 20 && is_comp_pte)
158 identity_offset += DIV_ROUND_UP_ULL(xe_vram_region_actual_physical_size
159 (xe->mem.vram), SZ_1G);
160
161 addr -= xe_vram_region_dpa_base(xe->mem.vram);
162 return addr + (identity_offset << xe_pt_shift(2));
163}
164
165static void xe_migrate_program_identity(struct xe_device *xe, struct xe_vm *vm, struct xe_bo *bo,
166 u64 map_ofs, u64 vram_offset, u16 pat_index, u64 pt_2m_ofs)
167{
168 struct xe_vram_region *vram = xe->mem.vram;
169 resource_size_t dpa_base = xe_vram_region_dpa_base(vram);
170 u64 pos, ofs, flags;
171 u64 entry;
172 /* XXX: Unclear if this should be usable_size? */
173 u64 vram_limit = xe_vram_region_actual_physical_size(vram) + dpa_base;
174 u32 level = 2;
175
176 ofs = map_ofs + XE_PAGE_SIZE * level + vram_offset * 8;
177 flags = vm->pt_ops->pte_encode_addr(xe, 0, pat_index, level,
178 true, 0);
179
180 xe_assert(xe, IS_ALIGNED(xe_vram_region_usable_size(vram), SZ_2M));
181
182 /*
183 * Use 1GB pages when possible, last chunk always use 2M
184 * pages as mixing reserved memory (stolen, WOCPM) with a single
185 * mapping is not allowed on certain platforms.
186 */
187 for (pos = dpa_base; pos < vram_limit;
188 pos += SZ_1G, ofs += 8) {
189 if (pos + SZ_1G >= vram_limit) {
190 entry = vm->pt_ops->pde_encode_bo(bo, pt_2m_ofs);
191 xe_map_wr(xe, &bo->vmap, ofs, u64, entry);
192
193 flags = vm->pt_ops->pte_encode_addr(xe, 0,
194 pat_index,
195 level - 1,
196 true, 0);
197
198 for (ofs = pt_2m_ofs; pos < vram_limit;
199 pos += SZ_2M, ofs += 8)
200 xe_map_wr(xe, &bo->vmap, ofs, u64, pos | flags);
201 break; /* Ensure pos == vram_limit assert correct */
202 }
203
204 xe_map_wr(xe, &bo->vmap, ofs, u64, pos | flags);
205 }
206
207 xe_assert(xe, pos == vram_limit);
208}
209
210static int xe_migrate_pt_bo_alloc(struct xe_tile *tile, struct xe_migrate *m,
211 struct xe_vm *vm, struct drm_exec *exec)
212{
213 struct xe_bo *bo, *batch = tile->mem.kernel_bb_pool->bo;
214 u32 num_entries = NUM_PT_SLOTS;
215
216 /* Can't bump NUM_PT_SLOTS too high */
217 BUILD_BUG_ON(NUM_PT_SLOTS > SZ_2M/XE_PAGE_SIZE);
218 /* Must be a multiple of 64K to support all platforms */
219 BUILD_BUG_ON(NUM_PT_SLOTS * XE_PAGE_SIZE % SZ_64K);
220 /* And one slot reserved for the 4KiB page table updates */
221 BUILD_BUG_ON(!(NUM_KERNEL_PDE & 1));
222
223 /* Need to be sure everything fits in the first PT, or create more */
224 xe_tile_assert(tile, m->batch_base_ofs + xe_bo_size(batch) < SZ_2M);
225
226 bo = xe_bo_create_pin_map(vm->xe, tile, vm,
227 num_entries * XE_PAGE_SIZE,
228 ttm_bo_type_kernel,
229 XE_BO_FLAG_VRAM_IF_DGFX(tile) |
230 XE_BO_FLAG_PAGETABLE, exec);
231 if (IS_ERR(bo))
232 return PTR_ERR(bo);
233
234 m->pt_bo = bo;
235 return 0;
236}
237
238static void xe_migrate_prepare_vm(struct xe_tile *tile, struct xe_migrate *m,
239 struct xe_vm *vm, u32 *ofs)
240{
241 struct xe_device *xe = tile_to_xe(tile);
242 u16 pat_index = xe_cache_pat_idx(xe, XE_CACHE_WB);
243 u8 id = tile->id;
244 u32 num_entries = NUM_PT_SLOTS, num_level = vm->pt_root[id]->level;
245#define VRAM_IDENTITY_MAP_COUNT 2
246 u32 num_setup = num_level + VRAM_IDENTITY_MAP_COUNT;
247#undef VRAM_IDENTITY_MAP_COUNT
248 u32 map_ofs, level, i;
249 struct xe_bo *bo = m->pt_bo, *batch = tile->mem.kernel_bb_pool->bo;
250 u64 entry, pt29_ofs;
251
252 /* PT30 & PT31 reserved for 2M identity map */
253 pt29_ofs = xe_bo_size(bo) - 3 * XE_PAGE_SIZE;
254 entry = vm->pt_ops->pde_encode_bo(bo, pt29_ofs);
255 xe_pt_write(xe, &vm->pt_root[id]->bo->vmap, 0, entry);
256
257 map_ofs = (num_entries - num_setup) * XE_PAGE_SIZE;
258
259 /* Map the entire BO in our level 0 pt */
260 for (i = 0, level = 0; i < num_entries; level++) {
261 entry = vm->pt_ops->pte_encode_bo(bo, i * XE_PAGE_SIZE,
262 pat_index, 0);
263
264 xe_map_wr(xe, &bo->vmap, map_ofs + level * 8, u64, entry);
265
266 if (vm->flags & XE_VM_FLAG_64K)
267 i += 16;
268 else
269 i += 1;
270 }
271
272 if (!IS_DGFX(xe)) {
273 /* Write out batch too */
274 m->batch_base_ofs = NUM_PT_SLOTS * XE_PAGE_SIZE;
275 for (i = 0; i < xe_bo_size(batch);
276 i += vm->flags & XE_VM_FLAG_64K ? XE_64K_PAGE_SIZE :
277 XE_PAGE_SIZE) {
278 entry = vm->pt_ops->pte_encode_bo(batch, i,
279 pat_index, 0);
280
281 xe_map_wr(xe, &bo->vmap, map_ofs + level * 8, u64,
282 entry);
283 level++;
284 }
285 if (xe->info.has_usm) {
286 xe_tile_assert(tile, xe_bo_size(batch) == SZ_1M);
287
288 batch = tile->primary_gt->usm.bb_pool->bo;
289 m->usm_batch_base_ofs = m->batch_base_ofs + SZ_1M;
290 xe_tile_assert(tile, xe_bo_size(batch) == SZ_512K);
291
292 for (i = 0; i < xe_bo_size(batch);
293 i += vm->flags & XE_VM_FLAG_64K ? XE_64K_PAGE_SIZE :
294 XE_PAGE_SIZE) {
295 entry = vm->pt_ops->pte_encode_bo(batch, i,
296 pat_index, 0);
297
298 xe_map_wr(xe, &bo->vmap, map_ofs + level * 8, u64,
299 entry);
300 level++;
301 }
302 }
303 } else {
304 u64 batch_addr = xe_bo_addr(batch, 0, XE_PAGE_SIZE);
305
306 m->batch_base_ofs = xe_migrate_vram_ofs(xe, batch_addr, false);
307
308 if (xe->info.has_usm) {
309 batch = tile->primary_gt->usm.bb_pool->bo;
310 batch_addr = xe_bo_addr(batch, 0, XE_PAGE_SIZE);
311 m->usm_batch_base_ofs = xe_migrate_vram_ofs(xe, batch_addr, false);
312 }
313 }
314
315 for (level = 1; level < num_level; level++) {
316 u32 flags = 0;
317
318 if (vm->flags & XE_VM_FLAG_64K && level == 1)
319 flags = XE_PDE_64K;
320
321 entry = vm->pt_ops->pde_encode_bo(bo, map_ofs + (u64)(level - 1) *
322 XE_PAGE_SIZE);
323 xe_map_wr(xe, &bo->vmap, map_ofs + XE_PAGE_SIZE * level, u64,
324 entry | flags);
325 }
326
327 /* Write PDE's that point to our BO. */
328 for (i = 0; i < map_ofs / XE_PAGE_SIZE; i++) {
329 entry = vm->pt_ops->pde_encode_bo(bo, (u64)i * XE_PAGE_SIZE);
330
331 xe_map_wr(xe, &bo->vmap, map_ofs + XE_PAGE_SIZE +
332 (i + 1) * 8, u64, entry);
333 }
334
335 /* Reserve 2M PDEs */
336 level = 1;
337 m->large_page_copy_ofs = NUM_PT_SLOTS << xe_pt_shift(level);
338 m->large_page_copy_pdes = map_ofs + XE_PAGE_SIZE * level +
339 NUM_PT_SLOTS * 8;
340
341 /* Set up a 1GiB NULL mapping at 255GiB offset. */
342 level = 2;
343 xe_map_wr(xe, &bo->vmap, map_ofs + XE_PAGE_SIZE * level + 255 * 8, u64,
344 vm->pt_ops->pte_encode_addr(xe, 0, pat_index, level, IS_DGFX(xe), 0)
345 | XE_PTE_NULL);
346 m->cleared_mem_ofs = (255ULL << xe_pt_shift(level));
347
348 /* Identity map the entire vram at 256GiB offset */
349 if (IS_DGFX(xe)) {
350 u64 pt30_ofs = xe_bo_size(bo) - 2 * XE_PAGE_SIZE;
351 resource_size_t actual_phy_size = xe_vram_region_actual_physical_size(xe->mem.vram);
352
353 xe_migrate_program_identity(xe, vm, bo, map_ofs, IDENTITY_OFFSET,
354 pat_index, pt30_ofs);
355 xe_assert(xe, actual_phy_size <= (MAX_NUM_PTE - IDENTITY_OFFSET) * SZ_1G);
356
357 /*
358 * Identity map the entire vram for compressed pat_index for xe2+
359 * if flat ccs is enabled.
360 */
361 if (GRAPHICS_VER(xe) >= 20 && xe_device_has_flat_ccs(xe)) {
362 u16 comp_pat_index = xe_cache_pat_idx(xe, XE_CACHE_NONE_COMPRESSION);
363 u64 vram_offset = IDENTITY_OFFSET +
364 DIV_ROUND_UP_ULL(actual_phy_size, SZ_1G);
365 u64 pt31_ofs = xe_bo_size(bo) - XE_PAGE_SIZE;
366
367 xe_assert(xe, actual_phy_size <= (MAX_NUM_PTE - IDENTITY_OFFSET -
368 IDENTITY_OFFSET / 2) * SZ_1G);
369 xe_migrate_program_identity(xe, vm, bo, map_ofs, vram_offset,
370 comp_pat_index, pt31_ofs);
371 }
372 }
373
374 if (ofs)
375 *ofs = map_ofs;
376}
377
378static void xe_migrate_suballoc_manager_init(struct xe_migrate *m, u32 map_ofs)
379{
380 /*
381 * Example layout created above, with root level = 3:
382 * [PT0...PT7]: kernel PT's for copy/clear; 64 or 4KiB PTE's
383 * [PT8]: Kernel PT for VM_BIND, 4 KiB PTE's
384 * [PT9...PT26]: Userspace PT's for VM_BIND, 4 KiB PTE's
385 * [PT27 = PDE 0] [PT28 = PDE 1] [PT29 = PDE 2] [PT30 & PT31 = 2M vram identity map]
386 *
387 * This makes the lowest part of the VM point to the pagetables.
388 * Hence the lowest 2M in the vm should point to itself, with a few writes
389 * and flushes, other parts of the VM can be used either for copying and
390 * clearing.
391 *
392 * For performance, the kernel reserves PDE's, so about 20 are left
393 * for async VM updates.
394 *
395 * To make it easier to work, each scratch PT is put in slot (1 + PT #)
396 * everywhere, this allows lockless updates to scratch pages by using
397 * the different addresses in VM.
398 */
399#define NUM_VMUSA_UNIT_PER_PAGE 32
400#define VM_SA_UPDATE_UNIT_SIZE (XE_PAGE_SIZE / NUM_VMUSA_UNIT_PER_PAGE)
401#define NUM_VMUSA_WRITES_PER_UNIT (VM_SA_UPDATE_UNIT_SIZE / sizeof(u64))
402 drm_suballoc_manager_init(&m->vm_update_sa,
403 (size_t)(map_ofs / XE_PAGE_SIZE - NUM_KERNEL_PDE) *
404 NUM_VMUSA_UNIT_PER_PAGE, 0);
405}
406
407static bool xe_migrate_needs_ccs_emit(struct xe_device *xe)
408{
409 return xe_device_has_flat_ccs(xe) && !(GRAPHICS_VER(xe) >= 20 && IS_DGFX(xe));
410}
411
412/**
413 * xe_migrate_alloc - Allocate a migrate struct for a given &xe_tile
414 * @tile: &xe_tile
415 *
416 * Allocates a &xe_migrate for a given tile.
417 *
418 * Return: &xe_migrate on success, or NULL when out of memory.
419 */
420struct xe_migrate *xe_migrate_alloc(struct xe_tile *tile)
421{
422 struct xe_migrate *m = drmm_kzalloc(&tile_to_xe(tile)->drm, sizeof(*m), GFP_KERNEL);
423
424 if (m)
425 m->tile = tile;
426 return m;
427}
428
429static int xe_migrate_lock_prepare_vm(struct xe_tile *tile, struct xe_migrate *m, struct xe_vm *vm)
430{
431 struct xe_device *xe = tile_to_xe(tile);
432 struct xe_validation_ctx ctx;
433 struct drm_exec exec;
434 u32 map_ofs;
435 int err = 0;
436
437 xe_validation_guard(&ctx, &xe->val, &exec, (struct xe_val_flags) {}, err) {
438 err = xe_vm_drm_exec_lock(vm, &exec);
439 if (err)
440 return err;
441
442 drm_exec_retry_on_contention(&exec);
443
444 err = xe_migrate_pt_bo_alloc(tile, m, vm, &exec);
445 if (err)
446 return err;
447
448 xe_migrate_prepare_vm(tile, m, vm, &map_ofs);
449 xe_migrate_suballoc_manager_init(m, map_ofs);
450 drm_exec_retry_on_contention(&exec);
451 xe_validation_retry_on_oom(&ctx, &err);
452 }
453
454 return err;
455}
456
457/**
458 * xe_migrate_init() - Initialize a migrate context
459 * @m: The migration context
460 *
461 * Return: 0 if successful, negative error code on failure
462 */
463int xe_migrate_init(struct xe_migrate *m)
464{
465 struct xe_tile *tile = m->tile;
466 struct xe_gt *primary_gt = tile->primary_gt;
467 struct xe_device *xe = tile_to_xe(tile);
468 struct xe_vm *vm;
469 int err;
470
471 /* Special layout, prepared below.. */
472 vm = xe_vm_create(xe, XE_VM_FLAG_MIGRATION |
473 XE_VM_FLAG_SET_TILE_ID(tile), NULL);
474 if (IS_ERR(vm))
475 return PTR_ERR(vm);
476
477 err = xe_migrate_lock_prepare_vm(tile, m, vm);
478 if (err)
479 goto err_out;
480
481 if (xe->info.has_usm) {
482 struct xe_hw_engine *hwe0 = primary_gt->usm.paging_hwe0;
483 u32 logical_mask = primary_gt->usm.paging_logical_mask;
484
485 if (!hwe0 || !logical_mask) {
486 err = -EINVAL;
487 goto err_out;
488 }
489
490 /*
491 * XXX: Currently only reserving 1 (likely slow) BCS instance on
492 * PVC, may want to revisit if performance is needed.
493 */
494 m->q = xe_exec_queue_create(xe, vm, logical_mask, 1, hwe0,
495 EXEC_QUEUE_FLAG_KERNEL |
496 EXEC_QUEUE_FLAG_PERMANENT |
497 EXEC_QUEUE_FLAG_HIGH_PRIORITY |
498 EXEC_QUEUE_FLAG_MIGRATE |
499 EXEC_QUEUE_FLAG_LOW_LATENCY, 0);
500 } else {
501 m->q = xe_exec_queue_create_class(xe, primary_gt, vm,
502 XE_ENGINE_CLASS_COPY,
503 EXEC_QUEUE_FLAG_KERNEL |
504 EXEC_QUEUE_FLAG_PERMANENT |
505 EXEC_QUEUE_FLAG_MIGRATE, 0);
506 }
507 if (IS_ERR(m->q)) {
508 err = PTR_ERR(m->q);
509 goto err_out;
510 }
511
512 mutex_init(&m->job_mutex);
513 fs_reclaim_acquire(GFP_KERNEL);
514 might_lock(&m->job_mutex);
515 fs_reclaim_release(GFP_KERNEL);
516
517 err = devm_add_action_or_reset(xe->drm.dev, xe_migrate_fini, m);
518 if (err)
519 return err;
520
521 if (IS_DGFX(xe)) {
522 if (xe_migrate_needs_ccs_emit(xe))
523 /* min chunk size corresponds to 4K of CCS Metadata */
524 m->min_chunk_size = SZ_4K * SZ_64K /
525 xe_device_ccs_bytes(xe, SZ_64K);
526 else
527 /* Somewhat arbitrary to avoid a huge amount of blits */
528 m->min_chunk_size = SZ_64K;
529 m->min_chunk_size = roundup_pow_of_two(m->min_chunk_size);
530 drm_dbg(&xe->drm, "Migrate min chunk size is 0x%08llx\n",
531 (unsigned long long)m->min_chunk_size);
532 }
533
534 return err;
535
536err_out:
537 xe_vm_close_and_put(vm);
538 return err;
539
540}
541
542static u64 max_mem_transfer_per_pass(struct xe_device *xe)
543{
544 if (!IS_DGFX(xe) && xe_device_has_flat_ccs(xe))
545 return MAX_CCS_LIMITED_TRANSFER;
546
547 return MAX_PREEMPTDISABLE_TRANSFER;
548}
549
550static u64 xe_migrate_res_sizes(struct xe_migrate *m, struct xe_res_cursor *cur)
551{
552 struct xe_device *xe = tile_to_xe(m->tile);
553 u64 size = min_t(u64, max_mem_transfer_per_pass(xe), cur->remaining);
554
555 if (mem_type_is_vram(cur->mem_type)) {
556 /*
557 * VRAM we want to blit in chunks with sizes aligned to
558 * min_chunk_size in order for the offset to CCS metadata to be
559 * page-aligned. If it's the last chunk it may be smaller.
560 *
561 * Another constraint is that we need to limit the blit to
562 * the VRAM block size, unless size is smaller than
563 * min_chunk_size.
564 */
565 u64 chunk = max_t(u64, cur->size, m->min_chunk_size);
566
567 size = min_t(u64, size, chunk);
568 if (size > m->min_chunk_size)
569 size = round_down(size, m->min_chunk_size);
570 }
571
572 return size;
573}
574
575static bool xe_migrate_allow_identity(u64 size, const struct xe_res_cursor *cur)
576{
577 /* If the chunk is not fragmented, allow identity map. */
578 return cur->size >= size;
579}
580
581#define PTE_UPDATE_FLAG_IS_VRAM BIT(0)
582#define PTE_UPDATE_FLAG_IS_COMP_PTE BIT(1)
583
584static u32 pte_update_size(struct xe_migrate *m,
585 u32 flags,
586 struct ttm_resource *res,
587 struct xe_res_cursor *cur,
588 u64 *L0, u64 *L0_ofs, u32 *L0_pt,
589 u32 cmd_size, u32 pt_ofs, u32 avail_pts)
590{
591 u32 cmds = 0;
592 bool is_vram = PTE_UPDATE_FLAG_IS_VRAM & flags;
593 bool is_comp_pte = PTE_UPDATE_FLAG_IS_COMP_PTE & flags;
594
595 *L0_pt = pt_ofs;
596 if (is_vram && xe_migrate_allow_identity(*L0, cur)) {
597 /* Offset into identity map. */
598 *L0_ofs = xe_migrate_vram_ofs(tile_to_xe(m->tile),
599 cur->start + vram_region_gpu_offset(res),
600 is_comp_pte);
601 cmds += cmd_size;
602 } else {
603 /* Clip L0 to available size */
604 u64 size = min(*L0, (u64)avail_pts * SZ_2M);
605 u32 num_4k_pages = (size + XE_PAGE_SIZE - 1) >> XE_PTE_SHIFT;
606
607 *L0 = size;
608 *L0_ofs = xe_migrate_vm_addr(pt_ofs, 0);
609
610 /* MI_STORE_DATA_IMM */
611 cmds += 3 * DIV_ROUND_UP(num_4k_pages, MAX_PTE_PER_SDI);
612
613 /* PDE qwords */
614 cmds += num_4k_pages * 2;
615
616 /* Each chunk has a single blit command */
617 cmds += cmd_size;
618 }
619
620 return cmds;
621}
622
623static void emit_pte(struct xe_migrate *m,
624 struct xe_bb *bb, u32 at_pt,
625 bool is_vram, bool is_comp_pte,
626 struct xe_res_cursor *cur,
627 u32 size, struct ttm_resource *res)
628{
629 struct xe_device *xe = tile_to_xe(m->tile);
630 struct xe_vm *vm = m->q->vm;
631 struct xe_bo *bo = ttm_to_xe_bo(res->bo);
632 enum ttm_caching caching = ttm_cached;
633 u16 pat_index;
634 u32 ptes;
635 u64 ofs = (u64)at_pt * XE_PAGE_SIZE;
636 u64 cur_ofs;
637
638 if (!is_vram && bo->ttm.ttm)
639 caching = bo->ttm.ttm->caching;
640
641 pat_index = xe_migrate_pat_index(xe, caching, is_comp_pte);
642
643 ptes = DIV_ROUND_UP(size, XE_PAGE_SIZE);
644
645 while (ptes) {
646 u32 chunk = min(MAX_PTE_PER_SDI, ptes);
647
648 bb->cs[bb->len++] = MI_STORE_DATA_IMM | MI_SDI_NUM_QW(chunk);
649 bb->cs[bb->len++] = ofs;
650 bb->cs[bb->len++] = 0;
651
652 cur_ofs = ofs;
653 ofs += chunk * 8;
654 ptes -= chunk;
655
656 while (chunk--) {
657 u64 addr, flags = 0;
658 bool devmem = false;
659
660 addr = xe_res_dma(cur) & PAGE_MASK;
661 if (is_vram) {
662 if (vm->flags & XE_VM_FLAG_64K) {
663 u64 va = cur_ofs * XE_PAGE_SIZE / 8;
664
665 xe_assert(xe, (va & (SZ_64K - 1)) ==
666 (addr & (SZ_64K - 1)));
667
668 flags |= XE_PTE_PS64;
669 }
670
671 addr += vram_region_gpu_offset(res);
672 devmem = true;
673 }
674
675 addr = vm->pt_ops->pte_encode_addr(m->tile->xe,
676 addr, pat_index,
677 0, devmem, flags);
678 bb->cs[bb->len++] = lower_32_bits(addr);
679 bb->cs[bb->len++] = upper_32_bits(addr);
680
681 xe_res_next(cur, min_t(u32, size, PAGE_SIZE));
682 cur_ofs += 8;
683 }
684 }
685}
686
687#define EMIT_COPY_CCS_DW 5
688static void emit_copy_ccs(struct xe_gt *gt, struct xe_bb *bb,
689 u64 dst_ofs, bool dst_is_indirect,
690 u64 src_ofs, bool src_is_indirect,
691 u32 size)
692{
693 struct xe_device *xe = gt_to_xe(gt);
694 u32 *cs = bb->cs + bb->len;
695 u32 num_ccs_blks;
696 u32 num_pages;
697 u32 ccs_copy_size;
698 u32 mocs;
699
700 if (GRAPHICS_VERx100(xe) >= 2000) {
701 num_pages = DIV_ROUND_UP(size, XE_PAGE_SIZE);
702 xe_gt_assert(gt, FIELD_FIT(XE2_CCS_SIZE_MASK, num_pages - 1));
703
704 ccs_copy_size = REG_FIELD_PREP(XE2_CCS_SIZE_MASK, num_pages - 1);
705 mocs = FIELD_PREP(XE2_XY_CTRL_SURF_MOCS_INDEX_MASK, gt->mocs.uc_index);
706
707 } else {
708 num_ccs_blks = DIV_ROUND_UP(xe_device_ccs_bytes(gt_to_xe(gt), size),
709 NUM_CCS_BYTES_PER_BLOCK);
710 xe_gt_assert(gt, FIELD_FIT(CCS_SIZE_MASK, num_ccs_blks - 1));
711
712 ccs_copy_size = REG_FIELD_PREP(CCS_SIZE_MASK, num_ccs_blks - 1);
713 mocs = FIELD_PREP(XY_CTRL_SURF_MOCS_MASK, gt->mocs.uc_index);
714 }
715
716 *cs++ = XY_CTRL_SURF_COPY_BLT |
717 (src_is_indirect ? 0x0 : 0x1) << SRC_ACCESS_TYPE_SHIFT |
718 (dst_is_indirect ? 0x0 : 0x1) << DST_ACCESS_TYPE_SHIFT |
719 ccs_copy_size;
720 *cs++ = lower_32_bits(src_ofs);
721 *cs++ = upper_32_bits(src_ofs) | mocs;
722 *cs++ = lower_32_bits(dst_ofs);
723 *cs++ = upper_32_bits(dst_ofs) | mocs;
724
725 bb->len = cs - bb->cs;
726}
727
728static u32 blt_fast_copy_cmd_len(struct xe_device *xe)
729{
730 return 10;
731}
732
733static u32 blt_mem_copy_cmd_len(struct xe_device *xe)
734{
735 return 10;
736}
737
738static u32 emit_copy_cmd_len(struct xe_device *xe)
739{
740 return (xe->info.has_mem_copy_instr) ? blt_mem_copy_cmd_len(xe) :
741 blt_fast_copy_cmd_len(xe);
742}
743
744static void emit_xy_fast_copy(struct xe_gt *gt, struct xe_bb *bb, u64 src_ofs,
745 u64 dst_ofs, unsigned int size,
746 unsigned int pitch)
747{
748 struct xe_device *xe = gt_to_xe(gt);
749 u32 mocs = 0;
750 u32 tile_y = 0;
751 u32 len;
752
753 xe_gt_assert(gt, !(pitch & 3));
754 xe_gt_assert(gt, size / pitch <= S16_MAX);
755 xe_gt_assert(gt, pitch / 4 <= S16_MAX);
756 xe_gt_assert(gt, pitch <= U16_MAX);
757
758 if (GRAPHICS_VER(xe) >= 20)
759 mocs = FIELD_PREP(XE2_XY_FAST_COPY_BLT_MOCS_INDEX_MASK, gt->mocs.uc_index);
760
761 if (GRAPHICS_VERx100(xe) >= 1250)
762 tile_y = XY_FAST_COPY_BLT_D1_SRC_TILE4 | XY_FAST_COPY_BLT_D1_DST_TILE4;
763
764 len = blt_fast_copy_cmd_len(xe);
765 bb->cs[bb->len++] = XY_FAST_COPY_BLT_CMD | (len - 2);
766 bb->cs[bb->len++] = XY_FAST_COPY_BLT_DEPTH_32 | pitch | tile_y | mocs;
767 bb->cs[bb->len++] = 0;
768 bb->cs[bb->len++] = (size / pitch) << 16 | pitch / 4;
769 bb->cs[bb->len++] = lower_32_bits(dst_ofs);
770 bb->cs[bb->len++] = upper_32_bits(dst_ofs);
771 bb->cs[bb->len++] = 0;
772 bb->cs[bb->len++] = pitch | mocs;
773 bb->cs[bb->len++] = lower_32_bits(src_ofs);
774 bb->cs[bb->len++] = upper_32_bits(src_ofs);
775}
776
777#define PAGE_COPY_MODE_PS SZ_256 /* hw uses 256 bytes as the page-size */
778static void emit_mem_copy(struct xe_gt *gt, struct xe_bb *bb, u64 src_ofs,
779 u64 dst_ofs, unsigned int size, unsigned int pitch)
780{
781 u32 mode, copy_type, width;
782 u32 len;
783
784 xe_gt_assert(gt, IS_ALIGNED(size, pitch));
785 xe_gt_assert(gt, pitch <= U16_MAX);
786 xe_gt_assert(gt, pitch);
787 xe_gt_assert(gt, size);
788
789 if (IS_ALIGNED(size, PAGE_COPY_MODE_PS) &&
790 IS_ALIGNED(lower_32_bits(src_ofs), PAGE_COPY_MODE_PS) &&
791 IS_ALIGNED(lower_32_bits(dst_ofs), PAGE_COPY_MODE_PS)) {
792 mode = MEM_COPY_PAGE_COPY_MODE;
793 copy_type = 0; /* linear copy */
794 width = size / PAGE_COPY_MODE_PS;
795 } else if (pitch > 1) {
796 xe_gt_assert(gt, size / pitch <= U16_MAX);
797 mode = 0; /* BYTE_COPY */
798 copy_type = MEM_COPY_MATRIX_COPY;
799 width = pitch;
800 } else {
801 mode = 0; /* BYTE_COPY */
802 copy_type = 0; /* linear copy */
803 width = size;
804 }
805
806 xe_gt_assert(gt, width <= U16_MAX);
807
808 len = blt_mem_copy_cmd_len(gt_to_xe(gt));
809
810 bb->cs[bb->len++] = MEM_COPY_CMD | mode | copy_type | (len - 2);
811 bb->cs[bb->len++] = width - 1;
812 bb->cs[bb->len++] = size / pitch - 1; /* ignored by hw for page-copy/linear above */
813 bb->cs[bb->len++] = pitch - 1;
814 bb->cs[bb->len++] = pitch - 1;
815 bb->cs[bb->len++] = lower_32_bits(src_ofs);
816 bb->cs[bb->len++] = upper_32_bits(src_ofs);
817 bb->cs[bb->len++] = lower_32_bits(dst_ofs);
818 bb->cs[bb->len++] = upper_32_bits(dst_ofs);
819 bb->cs[bb->len++] = FIELD_PREP(MEM_COPY_SRC_MOCS_INDEX_MASK, gt->mocs.uc_index) |
820 FIELD_PREP(MEM_COPY_DST_MOCS_INDEX_MASK, gt->mocs.uc_index);
821}
822
823static void emit_copy(struct xe_gt *gt, struct xe_bb *bb,
824 u64 src_ofs, u64 dst_ofs, unsigned int size,
825 unsigned int pitch)
826{
827 struct xe_device *xe = gt_to_xe(gt);
828
829 if (xe->info.has_mem_copy_instr)
830 emit_mem_copy(gt, bb, src_ofs, dst_ofs, size, pitch);
831 else
832 emit_xy_fast_copy(gt, bb, src_ofs, dst_ofs, size, pitch);
833}
834
835static u64 xe_migrate_batch_base(struct xe_migrate *m, bool usm)
836{
837 return usm ? m->usm_batch_base_ofs : m->batch_base_ofs;
838}
839
840static u32 xe_migrate_ccs_copy(struct xe_migrate *m,
841 struct xe_bb *bb,
842 u64 src_ofs, bool src_is_indirect,
843 u64 dst_ofs, bool dst_is_indirect, u32 dst_size,
844 u64 ccs_ofs, bool copy_ccs)
845{
846 struct xe_gt *gt = m->tile->primary_gt;
847 u32 flush_flags = 0;
848
849 if (!copy_ccs && dst_is_indirect) {
850 /*
851 * If the src is already in vram, then it should already
852 * have been cleared by us, or has been populated by the
853 * user. Make sure we copy the CCS aux state as-is.
854 *
855 * Otherwise if the bo doesn't have any CCS metadata attached,
856 * we still need to clear it for security reasons.
857 */
858 u64 ccs_src_ofs = src_is_indirect ? src_ofs : m->cleared_mem_ofs;
859
860 emit_copy_ccs(gt, bb,
861 dst_ofs, true,
862 ccs_src_ofs, src_is_indirect, dst_size);
863
864 flush_flags = MI_FLUSH_DW_CCS;
865 } else if (copy_ccs) {
866 if (!src_is_indirect)
867 src_ofs = ccs_ofs;
868 else if (!dst_is_indirect)
869 dst_ofs = ccs_ofs;
870
871 xe_gt_assert(gt, src_is_indirect || dst_is_indirect);
872
873 emit_copy_ccs(gt, bb, dst_ofs, dst_is_indirect, src_ofs,
874 src_is_indirect, dst_size);
875 if (dst_is_indirect)
876 flush_flags = MI_FLUSH_DW_CCS;
877 }
878
879 return flush_flags;
880}
881
882static struct dma_fence *__xe_migrate_copy(struct xe_migrate *m,
883 struct xe_bo *src_bo,
884 struct xe_bo *dst_bo,
885 struct ttm_resource *src,
886 struct ttm_resource *dst,
887 bool copy_only_ccs,
888 bool is_vram_resolve)
889{
890 struct xe_gt *gt = m->tile->primary_gt;
891 struct xe_device *xe = gt_to_xe(gt);
892 struct dma_fence *fence = NULL;
893 u64 size = xe_bo_size(src_bo);
894 struct xe_res_cursor src_it, dst_it, ccs_it;
895 u64 src_L0_ofs, dst_L0_ofs;
896 u32 src_L0_pt, dst_L0_pt;
897 u64 src_L0, dst_L0;
898 int pass = 0;
899 int err;
900 bool src_is_pltt = src->mem_type == XE_PL_TT;
901 bool dst_is_pltt = dst->mem_type == XE_PL_TT;
902 bool src_is_vram = mem_type_is_vram(src->mem_type);
903 bool dst_is_vram = mem_type_is_vram(dst->mem_type);
904 bool type_device = src_bo->ttm.type == ttm_bo_type_device;
905 bool needs_ccs_emit = type_device && xe_migrate_needs_ccs_emit(xe);
906 bool copy_ccs = xe_device_has_flat_ccs(xe) &&
907 xe_bo_needs_ccs_pages(src_bo) && xe_bo_needs_ccs_pages(dst_bo);
908 bool copy_system_ccs = copy_ccs && (!src_is_vram || !dst_is_vram);
909
910 /*
911 * For decompression operation, always use the compression PAT index.
912 * Otherwise, only use the compression PAT index for device memory
913 * when copying from VRAM to system memory.
914 */
915 bool use_comp_pat = is_vram_resolve || (type_device &&
916 xe_device_has_flat_ccs(xe) &&
917 GRAPHICS_VER(xe) >= 20 && src_is_vram && !dst_is_vram);
918
919 /* Copying CCS between two different BOs is not supported yet. */
920 if (XE_WARN_ON(copy_ccs && src_bo != dst_bo))
921 return ERR_PTR(-EINVAL);
922
923 if (src_bo != dst_bo && XE_WARN_ON(xe_bo_size(src_bo) != xe_bo_size(dst_bo)))
924 return ERR_PTR(-EINVAL);
925
926 if (!src_is_vram)
927 xe_res_first_sg(xe_bo_sg(src_bo), 0, size, &src_it);
928 else
929 xe_res_first(src, 0, size, &src_it);
930 if (!dst_is_vram)
931 xe_res_first_sg(xe_bo_sg(dst_bo), 0, size, &dst_it);
932 else
933 xe_res_first(dst, 0, size, &dst_it);
934
935 if (copy_system_ccs)
936 xe_res_first_sg(xe_bo_sg(src_bo), xe_bo_ccs_pages_start(src_bo),
937 PAGE_ALIGN(xe_device_ccs_bytes(xe, size)),
938 &ccs_it);
939
940 while (size) {
941 u32 batch_size = 1; /* MI_BATCH_BUFFER_END */
942 struct xe_sched_job *job;
943 struct xe_bb *bb;
944 u32 flush_flags = 0;
945 u32 update_idx;
946 u64 ccs_ofs, ccs_size;
947 u32 ccs_pt;
948 u32 pte_flags;
949
950 bool usm = xe->info.has_usm;
951 u32 avail_pts = max_mem_transfer_per_pass(xe) / LEVEL0_PAGE_TABLE_ENCODE_SIZE;
952
953 src_L0 = xe_migrate_res_sizes(m, &src_it);
954 dst_L0 = xe_migrate_res_sizes(m, &dst_it);
955
956 drm_dbg(&xe->drm, "Pass %u, sizes: %llu & %llu\n",
957 pass++, src_L0, dst_L0);
958
959 src_L0 = min(src_L0, dst_L0);
960
961 pte_flags = src_is_vram ? PTE_UPDATE_FLAG_IS_VRAM : 0;
962 pte_flags |= use_comp_pat ? PTE_UPDATE_FLAG_IS_COMP_PTE : 0;
963 batch_size += pte_update_size(m, pte_flags, src, &src_it, &src_L0,
964 &src_L0_ofs, &src_L0_pt, 0, 0,
965 avail_pts);
966 if (copy_only_ccs) {
967 dst_L0_ofs = src_L0_ofs;
968 } else {
969 pte_flags = dst_is_vram ? PTE_UPDATE_FLAG_IS_VRAM : 0;
970 batch_size += pte_update_size(m, pte_flags, dst,
971 &dst_it, &src_L0,
972 &dst_L0_ofs, &dst_L0_pt,
973 0, avail_pts, avail_pts);
974 }
975
976 if (copy_system_ccs) {
977 xe_assert(xe, type_device);
978 ccs_size = xe_device_ccs_bytes(xe, src_L0);
979 batch_size += pte_update_size(m, 0, NULL, &ccs_it, &ccs_size,
980 &ccs_ofs, &ccs_pt, 0,
981 2 * avail_pts,
982 avail_pts);
983 xe_assert(xe, IS_ALIGNED(ccs_it.start, PAGE_SIZE));
984 }
985
986 /* Add copy commands size here */
987 batch_size += ((copy_only_ccs) ? 0 : emit_copy_cmd_len(xe)) +
988 ((needs_ccs_emit ? EMIT_COPY_CCS_DW : 0));
989
990 bb = xe_bb_new(gt, batch_size, usm);
991 if (IS_ERR(bb)) {
992 err = PTR_ERR(bb);
993 goto err_sync;
994 }
995
996 if (src_is_vram && xe_migrate_allow_identity(src_L0, &src_it))
997 xe_res_next(&src_it, src_L0);
998 else
999 emit_pte(m, bb, src_L0_pt, src_is_vram, copy_system_ccs || use_comp_pat,
1000 &src_it, src_L0, src);
1001
1002 if (dst_is_vram && xe_migrate_allow_identity(src_L0, &dst_it))
1003 xe_res_next(&dst_it, src_L0);
1004 else if (!copy_only_ccs)
1005 emit_pte(m, bb, dst_L0_pt, dst_is_vram, copy_system_ccs,
1006 &dst_it, src_L0, dst);
1007
1008 if (copy_system_ccs)
1009 emit_pte(m, bb, ccs_pt, false, false, &ccs_it, ccs_size, src);
1010
1011 bb->cs[bb->len++] = MI_BATCH_BUFFER_END;
1012 update_idx = bb->len;
1013
1014 if (!copy_only_ccs)
1015 emit_copy(gt, bb, src_L0_ofs, dst_L0_ofs, src_L0, XE_PAGE_SIZE);
1016
1017 if (needs_ccs_emit)
1018 flush_flags = xe_migrate_ccs_copy(m, bb, src_L0_ofs,
1019 IS_DGFX(xe) ? src_is_vram : src_is_pltt,
1020 dst_L0_ofs,
1021 IS_DGFX(xe) ? dst_is_vram : dst_is_pltt,
1022 src_L0, ccs_ofs, copy_ccs);
1023
1024 job = xe_bb_create_migration_job(m->q, bb,
1025 xe_migrate_batch_base(m, usm),
1026 update_idx);
1027 if (IS_ERR(job)) {
1028 err = PTR_ERR(job);
1029 goto err;
1030 }
1031
1032 xe_sched_job_add_migrate_flush(job, flush_flags | MI_INVALIDATE_TLB);
1033 if (!fence) {
1034 err = xe_sched_job_add_deps(job, src_bo->ttm.base.resv,
1035 DMA_RESV_USAGE_BOOKKEEP);
1036 if (!err && src_bo->ttm.base.resv != dst_bo->ttm.base.resv)
1037 err = xe_sched_job_add_deps(job, dst_bo->ttm.base.resv,
1038 DMA_RESV_USAGE_BOOKKEEP);
1039 if (err)
1040 goto err_job;
1041 }
1042
1043 mutex_lock(&m->job_mutex);
1044 xe_sched_job_arm(job);
1045 dma_fence_put(fence);
1046 fence = dma_fence_get(&job->drm.s_fence->finished);
1047 xe_sched_job_push(job);
1048
1049 dma_fence_put(m->fence);
1050 m->fence = dma_fence_get(fence);
1051
1052 mutex_unlock(&m->job_mutex);
1053
1054 xe_bb_free(bb, fence);
1055 size -= src_L0;
1056 continue;
1057
1058err_job:
1059 xe_sched_job_put(job);
1060err:
1061 xe_bb_free(bb, NULL);
1062
1063err_sync:
1064 /* Sync partial copy if any. FIXME: under job_mutex? */
1065 if (fence) {
1066 dma_fence_wait(fence, false);
1067 dma_fence_put(fence);
1068 }
1069
1070 return ERR_PTR(err);
1071 }
1072
1073 return fence;
1074}
1075
1076/**
1077 * xe_migrate_copy() - Copy content of TTM resources.
1078 * @m: The migration context.
1079 * @src_bo: The buffer object @src is currently bound to.
1080 * @dst_bo: If copying between resources created for the same bo, set this to
1081 * the same value as @src_bo. If copying between buffer objects, set it to
1082 * the buffer object @dst is currently bound to.
1083 * @src: The source TTM resource.
1084 * @dst: The dst TTM resource.
1085 * @copy_only_ccs: If true copy only CCS metadata
1086 *
1087 * Copies the contents of @src to @dst: On flat CCS devices,
1088 * the CCS metadata is copied as well if needed, or if not present,
1089 * the CCS metadata of @dst is cleared for security reasons.
1090 *
1091 * Return: Pointer to a dma_fence representing the last copy batch, or
1092 * an error pointer on failure. If there is a failure, any copy operation
1093 * started by the function call has been synced.
1094 */
1095struct dma_fence *xe_migrate_copy(struct xe_migrate *m,
1096 struct xe_bo *src_bo,
1097 struct xe_bo *dst_bo,
1098 struct ttm_resource *src,
1099 struct ttm_resource *dst,
1100 bool copy_only_ccs)
1101{
1102 return __xe_migrate_copy(m, src_bo, dst_bo, src, dst, copy_only_ccs, false);
1103}
1104
1105/**
1106 * xe_migrate_resolve() - Resolve and decompress a buffer object if required.
1107 * @m: The migrate context
1108 * @bo: The buffer object to resolve
1109 * @res: The reservation object
1110 *
1111 * Wrapper around __xe_migrate_copy() with is_vram_resolve set to true
1112 * to trigger decompression if needed.
1113 *
1114 * Return: A dma_fence that signals on completion, or an ERR_PTR on failure.
1115 */
1116struct dma_fence *xe_migrate_resolve(struct xe_migrate *m,
1117 struct xe_bo *bo,
1118 struct ttm_resource *res)
1119{
1120 return __xe_migrate_copy(m, bo, bo, res, res, false, true);
1121}
1122
1123/**
1124 * xe_migrate_lrc() - Get the LRC from migrate context.
1125 * @migrate: Migrate context.
1126 *
1127 * Return: Pointer to LRC on success, error on failure
1128 */
1129struct xe_lrc *xe_migrate_lrc(struct xe_migrate *migrate)
1130{
1131 return migrate->q->lrc[0];
1132}
1133
1134static u64 migrate_vm_ppgtt_addr_tlb_inval(void)
1135{
1136 /*
1137 * The migrate VM is self-referential so it can modify its own PTEs (see
1138 * pte_update_size() or emit_pte() functions). We reserve NUM_KERNEL_PDE
1139 * entries for kernel operations (copies, clears, CCS migrate), and
1140 * suballocate the rest to user operations (binds/unbinds). With
1141 * NUM_KERNEL_PDE = 15, NUM_KERNEL_PDE - 1 is already used for PTE updates,
1142 * so assign NUM_KERNEL_PDE - 2 for TLB invalidation.
1143 */
1144 return (NUM_KERNEL_PDE - 2) * XE_PAGE_SIZE;
1145}
1146
1147static int emit_flush_invalidate(u32 *dw, int i, u32 flags)
1148{
1149 u64 addr = migrate_vm_ppgtt_addr_tlb_inval();
1150
1151 dw[i++] = MI_FLUSH_DW | MI_INVALIDATE_TLB | MI_FLUSH_DW_OP_STOREDW |
1152 MI_FLUSH_IMM_DW | flags;
1153 dw[i++] = lower_32_bits(addr);
1154 dw[i++] = upper_32_bits(addr);
1155 dw[i++] = MI_NOOP;
1156 dw[i++] = MI_NOOP;
1157
1158 return i;
1159}
1160
1161/**
1162 * xe_migrate_ccs_rw_copy() - Copy content of TTM resources.
1163 * @tile: Tile whose migration context to be used.
1164 * @q : Execution to be used along with migration context.
1165 * @src_bo: The buffer object @src is currently bound to.
1166 * @new_mem: The (not yet committed) destination resource @src_bo is being
1167 * moved into; src_bo->ttm.resource is still the old resource.
1168 * @read_write : Creates BB commands for CCS read/write.
1169 *
1170 * Creates batch buffer instructions to copy CCS metadata from CCS pool to
1171 * memory and vice versa.
1172 *
1173 * This function should only be called for IGPU.
1174 *
1175 * Return: 0 if successful, negative error code on failure.
1176 */
1177int xe_migrate_ccs_rw_copy(struct xe_tile *tile, struct xe_exec_queue *q,
1178 struct xe_bo *src_bo,
1179 struct ttm_resource *new_mem,
1180 enum xe_sriov_vf_ccs_rw_ctxs read_write)
1181
1182{
1183 bool src_is_pltt = read_write == XE_SRIOV_VF_CCS_READ_CTX;
1184 bool dst_is_pltt = read_write == XE_SRIOV_VF_CCS_WRITE_CTX;
1185 struct ttm_resource *src = new_mem;
1186 struct xe_migrate *m = tile->migrate;
1187 struct xe_gt *gt = tile->primary_gt;
1188 u32 batch_size, batch_size_allocated;
1189 struct xe_device *xe = gt_to_xe(gt);
1190 struct xe_res_cursor src_it, ccs_it;
1191 struct xe_mem_pool *bb_pool;
1192 struct xe_sriov_vf_ccs_ctx *ctx;
1193 u64 size = xe_bo_size(src_bo);
1194 struct xe_mem_pool_node *bb;
1195 u64 src_L0, src_L0_ofs;
1196 struct xe_bb xe_bb_tmp;
1197 u32 src_L0_pt;
1198 int err;
1199
1200 ctx = &xe->sriov.vf.ccs.contexts[read_write];
1201
1202 xe_res_first_sg(xe_bo_sg(src_bo), 0, size, &src_it);
1203
1204 xe_res_first_sg(xe_bo_sg(src_bo), xe_bo_ccs_pages_start(src_bo),
1205 PAGE_ALIGN(xe_device_ccs_bytes(xe, size)),
1206 &ccs_it);
1207
1208 /* Calculate Batch buffer size */
1209 batch_size = 0;
1210 while (size) {
1211 batch_size += 10; /* Flush + ggtt addr + 2 NOP */
1212 u64 ccs_ofs, ccs_size;
1213 u32 ccs_pt;
1214
1215 u32 avail_pts = max_mem_transfer_per_pass(xe) / LEVEL0_PAGE_TABLE_ENCODE_SIZE;
1216
1217 src_L0 = min_t(u64, max_mem_transfer_per_pass(xe), size);
1218
1219 batch_size += pte_update_size(m, false, src, &src_it, &src_L0,
1220 &src_L0_ofs, &src_L0_pt, 0, 0,
1221 avail_pts);
1222
1223 ccs_size = xe_device_ccs_bytes(xe, src_L0);
1224 batch_size += pte_update_size(m, 0, NULL, &ccs_it, &ccs_size, &ccs_ofs,
1225 &ccs_pt, 0, avail_pts, avail_pts);
1226 xe_assert(xe, IS_ALIGNED(ccs_it.start, PAGE_SIZE));
1227
1228 /* Add copy commands size here */
1229 batch_size += EMIT_COPY_CCS_DW;
1230
1231 size -= src_L0;
1232 }
1233
1234 bb = xe_mem_pool_alloc_node();
1235 if (IS_ERR(bb))
1236 return PTR_ERR(bb);
1237
1238 bb_pool = ctx->mem.ccs_bb_pool;
1239 scoped_guard(mutex, xe_mem_pool_bo_swap_guard(bb_pool)) {
1240 xe_mem_pool_swap_shadow_locked(bb_pool);
1241
1242 err = xe_mem_pool_insert_node(bb_pool, bb, batch_size * sizeof(u32));
1243 if (err) {
1244 xe_gt_err(gt, "BB allocation failed.\n");
1245 kfree(bb);
1246 return err;
1247 }
1248
1249 batch_size_allocated = batch_size;
1250 size = xe_bo_size(src_bo);
1251 batch_size = 0;
1252
1253 xe_bb_tmp = (struct xe_bb){ .cs = xe_mem_pool_node_cpu_addr(bb), .len = 0 };
1254 /*
1255 * Emit PTE and copy commands here.
1256 * The CCS copy command can only support limited size. If the size to be
1257 * copied is more than the limit, divide copy into chunks. So, calculate
1258 * sizes here again before copy command is emitted.
1259 */
1260
1261 while (size) {
1262 batch_size += 10; /* Flush + ggtt addr + 2 NOP */
1263 u32 flush_flags = 0;
1264 u64 ccs_ofs, ccs_size;
1265 u32 ccs_pt;
1266
1267 u32 avail_pts = max_mem_transfer_per_pass(xe) /
1268 LEVEL0_PAGE_TABLE_ENCODE_SIZE;
1269
1270 src_L0 = xe_migrate_res_sizes(m, &src_it);
1271
1272 batch_size += pte_update_size(m, false, src, &src_it, &src_L0,
1273 &src_L0_ofs, &src_L0_pt, 0, 0,
1274 avail_pts);
1275
1276 ccs_size = xe_device_ccs_bytes(xe, src_L0);
1277 batch_size += pte_update_size(m, 0, NULL, &ccs_it, &ccs_size, &ccs_ofs,
1278 &ccs_pt, 0, avail_pts, avail_pts);
1279 xe_assert(xe, IS_ALIGNED(ccs_it.start, PAGE_SIZE));
1280 batch_size += EMIT_COPY_CCS_DW;
1281
1282 emit_pte(m, &xe_bb_tmp, src_L0_pt, false, true, &src_it, src_L0, src);
1283
1284 emit_pte(m, &xe_bb_tmp, ccs_pt, false, false, &ccs_it, ccs_size, src);
1285
1286 xe_bb_tmp.len = emit_flush_invalidate(xe_bb_tmp.cs, xe_bb_tmp.len,
1287 flush_flags);
1288 flush_flags = xe_migrate_ccs_copy(m, &xe_bb_tmp, src_L0_ofs, src_is_pltt,
1289 src_L0_ofs, dst_is_pltt,
1290 src_L0, ccs_ofs, true);
1291 xe_bb_tmp.len = emit_flush_invalidate(xe_bb_tmp.cs, xe_bb_tmp.len,
1292 flush_flags);
1293
1294 size -= src_L0;
1295 }
1296
1297 xe_assert(xe, (batch_size_allocated == xe_bb_tmp.len));
1298 xe_assert(xe, bb->sa_node.size == xe_bb_tmp.len * sizeof(u32));
1299 src_bo->bb_ccs[read_write] = bb;
1300
1301 xe_sriov_vf_ccs_rw_update_bb_addr(ctx);
1302 xe_mem_pool_sync_shadow_locked(bb);
1303 }
1304
1305 return 0;
1306}
1307
1308/**
1309 * xe_migrate_ccs_rw_copy_clear() - Clear the CCS read/write batch buffer
1310 * content.
1311 * @src_bo: The buffer object @src is currently bound to.
1312 * @read_write : Creates BB commands for CCS read/write.
1313 * @bound: Device is bound
1314 *
1315 * Directly clearing the BB lacks atomicity and can lead to undefined
1316 * behavior if the vCPU is halted mid-operation during the clearing
1317 * process. To avoid this issue, we use a shadow buffer object approach.
1318 *
1319 * First swap the SA BO address with the shadow BO, perform the clearing
1320 * operation on the BB, update the shadow BO in the ring buffer, then
1321 * sync the shadow and the actual buffer to maintain consistency.
1322 *
1323 * Returns: None.
1324 */
1325void xe_migrate_ccs_rw_copy_clear(struct xe_bo *src_bo,
1326 enum xe_sriov_vf_ccs_rw_ctxs read_write,
1327 bool bound)
1328{
1329 struct xe_mem_pool_node *bb = src_bo->bb_ccs[read_write];
1330 struct xe_device *xe = xe_bo_device(src_bo);
1331 struct xe_mem_pool *bb_pool;
1332 struct xe_sriov_vf_ccs_ctx *ctx;
1333 u32 *cs;
1334
1335 xe_assert(xe, IS_SRIOV_VF(xe));
1336
1337 ctx = &xe->sriov.vf.ccs.contexts[read_write];
1338 bb_pool = ctx->mem.ccs_bb_pool;
1339
1340 scoped_guard(mutex, xe_mem_pool_bo_swap_guard(bb_pool)) {
1341 if (bound) {
1342 xe_mem_pool_swap_shadow_locked(bb_pool);
1343
1344 cs = xe_mem_pool_node_cpu_addr(bb);
1345 memset(cs, MI_NOOP, bb->sa_node.size);
1346 xe_sriov_vf_ccs_rw_update_bb_addr(ctx);
1347
1348 xe_mem_pool_sync_shadow_locked(bb);
1349 }
1350 xe_mem_pool_free_node(bb);
1351 src_bo->bb_ccs[read_write] = NULL;
1352 }
1353}
1354
1355/**
1356 * xe_migrate_exec_queue() - Get the execution queue from migrate context.
1357 * @migrate: Migrate context.
1358 *
1359 * Return: Pointer to execution queue on success, error on failure
1360 */
1361struct xe_exec_queue *xe_migrate_exec_queue(struct xe_migrate *migrate)
1362{
1363 return migrate->q;
1364}
1365
1366/**
1367 * xe_migrate_vram_copy_chunk() - Copy a chunk of a VRAM buffer object.
1368 * @vram_bo: The VRAM buffer object.
1369 * @vram_offset: The VRAM offset.
1370 * @sysmem_bo: The sysmem buffer object.
1371 * @sysmem_offset: The sysmem offset.
1372 * @size: The size of VRAM chunk to copy.
1373 * @dir: The direction of the copy operation.
1374 *
1375 * Copies a portion of a buffer object between VRAM and system memory.
1376 * On Xe2 platforms that support flat CCS, VRAM data is decompressed when
1377 * copying to system memory.
1378 *
1379 * Return: Pointer to a dma_fence representing the last copy batch, or
1380 * an error pointer on failure. If there is a failure, any copy operation
1381 * started by the function call has been synced.
1382 */
1383struct dma_fence *xe_migrate_vram_copy_chunk(struct xe_bo *vram_bo, u64 vram_offset,
1384 struct xe_bo *sysmem_bo, u64 sysmem_offset,
1385 u64 size, enum xe_migrate_copy_dir dir)
1386{
1387 struct xe_device *xe = xe_bo_device(vram_bo);
1388 struct xe_tile *tile = vram_bo->tile;
1389 struct xe_gt *gt = tile->primary_gt;
1390 struct xe_migrate *m = tile->migrate;
1391 struct dma_fence *fence = NULL;
1392 struct ttm_resource *vram = vram_bo->ttm.resource;
1393 struct ttm_resource *sysmem = sysmem_bo->ttm.resource;
1394 struct xe_res_cursor vram_it, sysmem_it;
1395 u64 vram_L0_ofs, sysmem_L0_ofs;
1396 u32 vram_L0_pt, sysmem_L0_pt;
1397 u64 vram_L0, sysmem_L0;
1398 bool to_sysmem = (dir == XE_MIGRATE_COPY_TO_SRAM);
1399 bool use_comp_pat = to_sysmem &&
1400 GRAPHICS_VER(xe) >= 20 && xe_device_has_flat_ccs(xe);
1401 int pass = 0;
1402 int err;
1403
1404 xe_assert(xe, IS_ALIGNED(vram_offset | sysmem_offset | size, PAGE_SIZE));
1405 xe_assert(xe, xe_bo_is_vram(vram_bo));
1406 xe_assert(xe, !xe_bo_is_vram(sysmem_bo));
1407 xe_assert(xe, !range_overflows(vram_offset, size, (u64)vram_bo->ttm.base.size));
1408 xe_assert(xe, !range_overflows(sysmem_offset, size, (u64)sysmem_bo->ttm.base.size));
1409
1410 xe_res_first(vram, vram_offset, size, &vram_it);
1411 xe_res_first_sg(xe_bo_sg(sysmem_bo), sysmem_offset, size, &sysmem_it);
1412
1413 while (size) {
1414 u32 pte_flags = PTE_UPDATE_FLAG_IS_VRAM;
1415 u32 batch_size = 2; /* arb_clear() + MI_BATCH_BUFFER_END */
1416 struct xe_sched_job *job;
1417 struct xe_bb *bb;
1418 u32 update_idx;
1419 bool usm = xe->info.has_usm;
1420 u32 avail_pts = max_mem_transfer_per_pass(xe) / LEVEL0_PAGE_TABLE_ENCODE_SIZE;
1421
1422 sysmem_L0 = xe_migrate_res_sizes(m, &sysmem_it);
1423 vram_L0 = min(xe_migrate_res_sizes(m, &vram_it), sysmem_L0);
1424
1425 xe_dbg(xe, "Pass %u, size: %llu\n", pass++, vram_L0);
1426
1427 pte_flags |= use_comp_pat ? PTE_UPDATE_FLAG_IS_COMP_PTE : 0;
1428 batch_size += pte_update_size(m, pte_flags, vram, &vram_it, &vram_L0,
1429 &vram_L0_ofs, &vram_L0_pt, 0, 0, avail_pts);
1430
1431 batch_size += pte_update_size(m, 0, sysmem, &sysmem_it, &vram_L0, &sysmem_L0_ofs,
1432 &sysmem_L0_pt, 0, avail_pts, avail_pts);
1433 batch_size += emit_copy_cmd_len(xe);
1434
1435 bb = xe_bb_new(gt, batch_size, usm);
1436 if (IS_ERR(bb)) {
1437 err = PTR_ERR(bb);
1438 return ERR_PTR(err);
1439 }
1440
1441 if (xe_migrate_allow_identity(vram_L0, &vram_it))
1442 xe_res_next(&vram_it, vram_L0);
1443 else
1444 emit_pte(m, bb, vram_L0_pt, true, use_comp_pat, &vram_it, vram_L0, vram);
1445
1446 emit_pte(m, bb, sysmem_L0_pt, false, false, &sysmem_it, vram_L0, sysmem);
1447
1448 bb->cs[bb->len++] = MI_BATCH_BUFFER_END;
1449 update_idx = bb->len;
1450
1451 if (to_sysmem)
1452 emit_copy(gt, bb, vram_L0_ofs, sysmem_L0_ofs, vram_L0, XE_PAGE_SIZE);
1453 else
1454 emit_copy(gt, bb, sysmem_L0_ofs, vram_L0_ofs, vram_L0, XE_PAGE_SIZE);
1455
1456 job = xe_bb_create_migration_job(m->q, bb, xe_migrate_batch_base(m, usm),
1457 update_idx);
1458 if (IS_ERR(job)) {
1459 xe_bb_free(bb, NULL);
1460 err = PTR_ERR(job);
1461 return ERR_PTR(err);
1462 }
1463
1464 xe_sched_job_add_migrate_flush(job, MI_INVALIDATE_TLB);
1465
1466 xe_assert(xe, dma_resv_test_signaled(vram_bo->ttm.base.resv,
1467 DMA_RESV_USAGE_BOOKKEEP));
1468 xe_assert(xe, dma_resv_test_signaled(sysmem_bo->ttm.base.resv,
1469 DMA_RESV_USAGE_BOOKKEEP));
1470
1471 scoped_guard(mutex, &m->job_mutex) {
1472 xe_sched_job_arm(job);
1473 dma_fence_put(fence);
1474 fence = dma_fence_get(&job->drm.s_fence->finished);
1475 xe_sched_job_push(job);
1476
1477 dma_fence_put(m->fence);
1478 m->fence = dma_fence_get(fence);
1479 }
1480
1481 xe_bb_free(bb, fence);
1482 size -= vram_L0;
1483 }
1484
1485 return fence;
1486}
1487
1488static u32 blt_mem_set_cmd_len(struct xe_device *xe)
1489{
1490 return 7;
1491}
1492
1493static void emit_clear_link_copy(struct xe_gt *gt, struct xe_bb *bb, u64 src_ofs,
1494 u32 size, u32 pitch)
1495{
1496 struct xe_device *xe = gt_to_xe(gt);
1497 u32 *cs = bb->cs + bb->len;
1498 u32 len = blt_mem_set_cmd_len(xe);
1499
1500 *cs++ = PVC_MEM_SET_CMD | PVC_MEM_SET_MATRIX | (len - 2);
1501 *cs++ = pitch - 1;
1502 *cs++ = (size / pitch) - 1;
1503 *cs++ = pitch - 1;
1504 *cs++ = lower_32_bits(src_ofs);
1505 *cs++ = upper_32_bits(src_ofs);
1506 if (GRAPHICS_VERx100(xe) >= 2000)
1507 *cs++ = FIELD_PREP(XE2_MEM_SET_MOCS_INDEX_MASK, gt->mocs.uc_index);
1508 else
1509 *cs++ = FIELD_PREP(PVC_MEM_SET_MOCS_INDEX_MASK, gt->mocs.uc_index);
1510
1511 xe_gt_assert(gt, cs - bb->cs == len + bb->len);
1512
1513 bb->len += len;
1514}
1515
1516static u32 blt_fast_color_cmd_len(struct xe_device *xe)
1517{
1518 if (GRAPHICS_VERx100(xe) >= 1250)
1519 return 16;
1520 else
1521 return 11;
1522}
1523
1524static void emit_clear_main_copy(struct xe_gt *gt, struct xe_bb *bb,
1525 u64 src_ofs, u32 size, u32 pitch, bool is_vram)
1526{
1527 struct xe_device *xe = gt_to_xe(gt);
1528 u32 *cs = bb->cs + bb->len;
1529 u32 len = blt_fast_color_cmd_len(xe);
1530
1531
1532 *cs++ = XY_FAST_COLOR_BLT_CMD | XY_FAST_COLOR_BLT_DEPTH_32 |
1533 (len - 2);
1534 if (GRAPHICS_VERx100(xe) >= 2000)
1535 *cs++ = FIELD_PREP(XE2_XY_FAST_COLOR_BLT_MOCS_INDEX_MASK, gt->mocs.uc_index) |
1536 (pitch - 1);
1537 else
1538 *cs++ = FIELD_PREP(XY_FAST_COLOR_BLT_MOCS_MASK, gt->mocs.uc_index) |
1539 (pitch - 1);
1540 *cs++ = 0;
1541 *cs++ = (size / pitch) << 16 | pitch / 4;
1542 *cs++ = lower_32_bits(src_ofs);
1543 *cs++ = upper_32_bits(src_ofs);
1544 *cs++ = (is_vram ? 0x0 : 0x1) << XY_FAST_COLOR_BLT_MEM_TYPE_SHIFT;
1545 *cs++ = 0;
1546 *cs++ = 0;
1547 *cs++ = 0;
1548 *cs++ = 0;
1549
1550 if (len > 11) {
1551 *cs++ = 0;
1552 *cs++ = 0;
1553 *cs++ = 0;
1554 *cs++ = 0;
1555 *cs++ = 0;
1556 }
1557
1558 xe_gt_assert(gt, cs - bb->cs == len + bb->len);
1559
1560 bb->len += len;
1561}
1562
1563static u32 emit_clear_cmd_len(struct xe_gt *gt)
1564{
1565 struct xe_device *xe = gt_to_xe(gt);
1566
1567 if (gt->info.has_xe2_blt_instructions)
1568 return blt_mem_set_cmd_len(xe);
1569 else
1570 return blt_fast_color_cmd_len(xe);
1571}
1572
1573static void emit_clear(struct xe_gt *gt, struct xe_bb *bb, u64 src_ofs,
1574 u32 size, u32 pitch, bool is_vram)
1575{
1576 if (gt->info.has_xe2_blt_instructions)
1577 emit_clear_link_copy(gt, bb, src_ofs, size, pitch);
1578 else
1579 emit_clear_main_copy(gt, bb, src_ofs, size, pitch,
1580 is_vram);
1581}
1582
1583/**
1584 * xe_migrate_clear() - Copy content of TTM resources.
1585 * @m: The migration context.
1586 * @bo: The buffer object @dst is currently bound to.
1587 * @dst: The dst TTM resource to be cleared.
1588 * @clear_flags: flags to specify which data to clear: CCS, BO, or both.
1589 *
1590 * Clear the contents of @dst to zero when XE_MIGRATE_CLEAR_FLAG_BO_DATA is set.
1591 * On flat CCS devices, the CCS metadata is cleared to zero with XE_MIGRATE_CLEAR_FLAG_CCS_DATA.
1592 * Set XE_MIGRATE_CLEAR_FLAG_FULL to clear bo as well as CCS metadata.
1593 * TODO: Eliminate the @bo argument.
1594 *
1595 * Return: Pointer to a dma_fence representing the last clear batch, or
1596 * an error pointer on failure. If there is a failure, any clear operation
1597 * started by the function call has been synced.
1598 */
1599struct dma_fence *xe_migrate_clear(struct xe_migrate *m,
1600 struct xe_bo *bo,
1601 struct ttm_resource *dst,
1602 u32 clear_flags)
1603{
1604 bool clear_vram = mem_type_is_vram(dst->mem_type);
1605 bool clear_bo_data = XE_MIGRATE_CLEAR_FLAG_BO_DATA & clear_flags;
1606 bool clear_ccs = XE_MIGRATE_CLEAR_FLAG_CCS_DATA & clear_flags;
1607 struct xe_gt *gt = m->tile->primary_gt;
1608 struct xe_device *xe = gt_to_xe(gt);
1609 bool clear_only_system_ccs = false;
1610 struct dma_fence *fence = NULL;
1611 u64 size = xe_bo_size(bo);
1612 struct xe_res_cursor src_it;
1613 struct ttm_resource *src = dst;
1614 int err;
1615
1616 if (WARN_ON(!clear_bo_data && !clear_ccs))
1617 return NULL;
1618
1619 if (!clear_bo_data && clear_ccs && !IS_DGFX(xe))
1620 clear_only_system_ccs = true;
1621
1622 if (!clear_vram)
1623 xe_res_first_sg(xe_bo_sg(bo), 0, xe_bo_size(bo), &src_it);
1624 else
1625 xe_res_first(src, 0, xe_bo_size(bo), &src_it);
1626
1627 while (size) {
1628 u64 clear_L0_ofs;
1629 u32 clear_L0_pt;
1630 u32 flush_flags = 0;
1631 u64 clear_L0;
1632 struct xe_sched_job *job;
1633 struct xe_bb *bb;
1634 u32 batch_size, update_idx;
1635 u32 pte_flags;
1636
1637 bool usm = xe->info.has_usm;
1638 u32 avail_pts = max_mem_transfer_per_pass(xe) / LEVEL0_PAGE_TABLE_ENCODE_SIZE;
1639
1640 clear_L0 = xe_migrate_res_sizes(m, &src_it);
1641
1642 /* Calculate final sizes and batch size.. */
1643 pte_flags = clear_vram ? PTE_UPDATE_FLAG_IS_VRAM : 0;
1644 batch_size = 1 +
1645 pte_update_size(m, pte_flags, src, &src_it,
1646 &clear_L0, &clear_L0_ofs, &clear_L0_pt,
1647 clear_bo_data ? emit_clear_cmd_len(gt) : 0, 0,
1648 avail_pts);
1649
1650 if (xe_migrate_needs_ccs_emit(xe))
1651 batch_size += EMIT_COPY_CCS_DW;
1652
1653 /* Clear commands */
1654
1655 if (WARN_ON_ONCE(!clear_L0))
1656 break;
1657
1658 bb = xe_bb_new(gt, batch_size, usm);
1659 if (IS_ERR(bb)) {
1660 err = PTR_ERR(bb);
1661 goto err_sync;
1662 }
1663
1664 size -= clear_L0;
1665 /* Preemption is enabled again by the ring ops. */
1666 if (clear_vram && xe_migrate_allow_identity(clear_L0, &src_it)) {
1667 xe_res_next(&src_it, clear_L0);
1668 } else {
1669 emit_pte(m, bb, clear_L0_pt, clear_vram,
1670 clear_only_system_ccs, &src_it, clear_L0, dst);
1671 flush_flags |= MI_INVALIDATE_TLB;
1672 }
1673
1674 bb->cs[bb->len++] = MI_BATCH_BUFFER_END;
1675 update_idx = bb->len;
1676
1677 if (clear_bo_data)
1678 emit_clear(gt, bb, clear_L0_ofs, clear_L0, XE_PAGE_SIZE, clear_vram);
1679
1680 if (xe_migrate_needs_ccs_emit(xe)) {
1681 emit_copy_ccs(gt, bb, clear_L0_ofs, true,
1682 m->cleared_mem_ofs, false, clear_L0);
1683 flush_flags |= MI_FLUSH_DW_CCS;
1684 }
1685
1686 job = xe_bb_create_migration_job(m->q, bb,
1687 xe_migrate_batch_base(m, usm),
1688 update_idx);
1689 if (IS_ERR(job)) {
1690 err = PTR_ERR(job);
1691 goto err;
1692 }
1693
1694 xe_sched_job_add_migrate_flush(job, flush_flags);
1695 if (!fence) {
1696 /*
1697 * There can't be anything userspace related at this
1698 * point, so we just need to respect any potential move
1699 * fences, which are always tracked as
1700 * DMA_RESV_USAGE_KERNEL.
1701 */
1702 err = xe_sched_job_add_deps(job, bo->ttm.base.resv,
1703 DMA_RESV_USAGE_KERNEL);
1704 if (err)
1705 goto err_job;
1706 }
1707
1708 mutex_lock(&m->job_mutex);
1709 xe_sched_job_arm(job);
1710 dma_fence_put(fence);
1711 fence = dma_fence_get(&job->drm.s_fence->finished);
1712 xe_sched_job_push(job);
1713
1714 dma_fence_put(m->fence);
1715 m->fence = dma_fence_get(fence);
1716
1717 mutex_unlock(&m->job_mutex);
1718
1719 xe_bb_free(bb, fence);
1720 continue;
1721
1722err_job:
1723 xe_sched_job_put(job);
1724err:
1725 xe_bb_free(bb, NULL);
1726err_sync:
1727 /* Sync partial copies if any. FIXME: job_mutex? */
1728 if (fence) {
1729 dma_fence_wait(fence, false);
1730 dma_fence_put(fence);
1731 }
1732
1733 return ERR_PTR(err);
1734 }
1735
1736 if (clear_ccs)
1737 bo->ccs_cleared = true;
1738
1739 return fence;
1740}
1741
1742static void write_pgtable(struct xe_tile *tile, struct xe_bb *bb, u64 ppgtt_ofs,
1743 const struct xe_vm_pgtable_update_op *pt_op,
1744 const struct xe_vm_pgtable_update *update,
1745 struct xe_migrate_pt_update *pt_update)
1746{
1747 const struct xe_migrate_pt_update_ops *ops = pt_update->ops;
1748 u32 chunk;
1749 u32 ofs = update->ofs, size = update->qwords;
1750
1751 /*
1752 * If we have 512 entries (max), we would populate it ourselves,
1753 * and update the PDE above it to the new pointer.
1754 * The only time this can only happen if we have to update the top
1755 * PDE. This requires a BO that is almost vm->size big.
1756 *
1757 * This shouldn't be possible in practice.. might change when 16K
1758 * pages are used. Hence the assert.
1759 */
1760 xe_tile_assert(tile, update->qwords < MAX_NUM_PTE);
1761 if (!ppgtt_ofs)
1762 ppgtt_ofs = xe_migrate_vram_ofs(tile_to_xe(tile),
1763 xe_bo_addr(update->pt_bo, 0,
1764 XE_PAGE_SIZE), false);
1765
1766 do {
1767 u64 addr = ppgtt_ofs + ofs * 8;
1768
1769 chunk = min(size, MAX_PTE_PER_SDI);
1770
1771 /* Ensure populatefn can do memset64 by aligning bb->cs */
1772 if (!(bb->len & 1))
1773 bb->cs[bb->len++] = MI_NOOP;
1774
1775 bb->cs[bb->len++] = MI_STORE_DATA_IMM | MI_SDI_NUM_QW(chunk);
1776 bb->cs[bb->len++] = lower_32_bits(addr);
1777 bb->cs[bb->len++] = upper_32_bits(addr);
1778 if (pt_op->bind)
1779 ops->populate(pt_update, tile, NULL, bb->cs + bb->len,
1780 ofs, chunk, update);
1781 else
1782 ops->clear(pt_update, tile, NULL, bb->cs + bb->len,
1783 ofs, chunk, update);
1784
1785 bb->len += chunk * 2;
1786 ofs += chunk;
1787 size -= chunk;
1788 } while (size);
1789}
1790
1791struct xe_vm *xe_migrate_get_vm(struct xe_migrate *m)
1792{
1793 return xe_vm_get(m->q->vm);
1794}
1795
1796#if IS_ENABLED(CONFIG_DRM_XE_KUNIT_TEST)
1797struct migrate_test_params {
1798 struct xe_test_priv base;
1799 bool force_gpu;
1800};
1801
1802#define to_migrate_test_params(_priv) \
1803 container_of(_priv, struct migrate_test_params, base)
1804#endif
1805
1806static struct dma_fence *
1807xe_migrate_update_pgtables_cpu(struct xe_migrate *m,
1808 struct xe_migrate_pt_update *pt_update)
1809{
1810 XE_TEST_DECLARE(struct migrate_test_params *test =
1811 to_migrate_test_params
1812 (xe_cur_kunit_priv(XE_TEST_LIVE_MIGRATE));)
1813 const struct xe_migrate_pt_update_ops *ops = pt_update->ops;
1814 struct xe_vm *vm = pt_update->vops->vm;
1815 struct xe_vm_pgtable_update_ops *pt_update_ops =
1816 &pt_update->vops->pt_update_ops[pt_update->tile_id];
1817 int err;
1818 u32 i, j;
1819
1820 if (XE_TEST_ONLY(test && test->force_gpu))
1821 return ERR_PTR(-ETIME);
1822
1823 if (ops->pre_commit) {
1824 pt_update->job = NULL;
1825 err = ops->pre_commit(pt_update);
1826 if (err)
1827 return ERR_PTR(err);
1828 }
1829
1830 for (i = 0; i < pt_update_ops->num_ops; ++i) {
1831 const struct xe_vm_pgtable_update_op *pt_op =
1832 &pt_update_ops->ops[i];
1833
1834 for (j = 0; j < pt_op->num_entries; j++) {
1835 const struct xe_vm_pgtable_update *update =
1836 &pt_op->entries[j];
1837
1838 if (pt_op->bind)
1839 ops->populate(pt_update, m->tile,
1840 &update->pt_bo->vmap, NULL,
1841 update->ofs, update->qwords,
1842 update);
1843 else
1844 ops->clear(pt_update, m->tile,
1845 &update->pt_bo->vmap, NULL,
1846 update->ofs, update->qwords, update);
1847 }
1848 }
1849
1850 trace_xe_vm_cpu_bind(vm);
1851 xe_device_wmb(vm->xe);
1852
1853 return dma_fence_get_stub();
1854}
1855
1856static struct dma_fence *
1857__xe_migrate_update_pgtables(struct xe_migrate *m,
1858 struct xe_migrate_pt_update *pt_update,
1859 struct xe_vm_pgtable_update_ops *pt_update_ops)
1860{
1861 const struct xe_migrate_pt_update_ops *ops = pt_update->ops;
1862 struct xe_tile *tile = m->tile;
1863 struct xe_gt *gt = tile->primary_gt;
1864 struct xe_device *xe = tile_to_xe(tile);
1865 struct xe_sched_job *job;
1866 struct dma_fence *fence;
1867 struct drm_suballoc *sa_bo = NULL;
1868 struct xe_bb *bb;
1869 u32 i, j, batch_size = 0, ppgtt_ofs, update_idx, page_ofs = 0;
1870 u32 num_updates = 0, current_update = 0;
1871 u64 addr;
1872 int err = 0;
1873 bool is_migrate = pt_update_ops->q == m->q;
1874 bool usm = is_migrate && xe->info.has_usm;
1875
1876 for (i = 0; i < pt_update_ops->num_ops; ++i) {
1877 struct xe_vm_pgtable_update_op *pt_op = &pt_update_ops->ops[i];
1878 struct xe_vm_pgtable_update *updates = pt_op->entries;
1879
1880 num_updates += pt_op->num_entries;
1881 for (j = 0; j < pt_op->num_entries; ++j) {
1882 u32 num_cmds = DIV_ROUND_UP(updates[j].qwords,
1883 MAX_PTE_PER_SDI);
1884
1885 /* align noop + MI_STORE_DATA_IMM cmd prefix */
1886 batch_size += 4 * num_cmds + updates[j].qwords * 2;
1887 }
1888 }
1889
1890 /* fixed + PTE entries */
1891 if (IS_DGFX(xe))
1892 batch_size += 2;
1893 else
1894 batch_size += 6 * (num_updates / MAX_PTE_PER_SDI + 1) +
1895 num_updates * 2;
1896
1897 bb = xe_bb_new(gt, batch_size, usm);
1898 if (IS_ERR(bb))
1899 return ERR_CAST(bb);
1900
1901 /* For sysmem PTE's, need to map them in our hole.. */
1902 if (!IS_DGFX(xe)) {
1903 u16 pat_index = xe_cache_pat_idx(xe, XE_CACHE_WB);
1904 u32 ptes, ofs;
1905
1906 ppgtt_ofs = NUM_KERNEL_PDE - 1;
1907 if (!is_migrate) {
1908 u32 num_units = DIV_ROUND_UP(num_updates,
1909 NUM_VMUSA_WRITES_PER_UNIT);
1910
1911 if (num_units > m->vm_update_sa.size) {
1912 err = -ENOBUFS;
1913 goto err_bb;
1914 }
1915 sa_bo = drm_suballoc_new(&m->vm_update_sa, num_units,
1916 GFP_KERNEL, true, 0);
1917 if (IS_ERR(sa_bo)) {
1918 err = PTR_ERR(sa_bo);
1919 goto err_bb;
1920 }
1921
1922 ppgtt_ofs = NUM_KERNEL_PDE +
1923 (drm_suballoc_soffset(sa_bo) /
1924 NUM_VMUSA_UNIT_PER_PAGE);
1925 page_ofs = (drm_suballoc_soffset(sa_bo) %
1926 NUM_VMUSA_UNIT_PER_PAGE) *
1927 VM_SA_UPDATE_UNIT_SIZE;
1928 }
1929
1930 /* Map our PT's to gtt */
1931 i = 0;
1932 j = 0;
1933 ptes = num_updates;
1934 ofs = ppgtt_ofs * XE_PAGE_SIZE + page_ofs;
1935 while (ptes) {
1936 u32 chunk = min(MAX_PTE_PER_SDI, ptes);
1937 u32 idx = 0;
1938
1939 bb->cs[bb->len++] = MI_STORE_DATA_IMM |
1940 MI_SDI_NUM_QW(chunk);
1941 bb->cs[bb->len++] = ofs;
1942 bb->cs[bb->len++] = 0; /* upper_32_bits */
1943
1944 for (; i < pt_update_ops->num_ops; ++i) {
1945 struct xe_vm_pgtable_update_op *pt_op =
1946 &pt_update_ops->ops[i];
1947 struct xe_vm_pgtable_update *updates = pt_op->entries;
1948
1949 for (; j < pt_op->num_entries; ++j, ++current_update, ++idx) {
1950 struct xe_vm *vm = pt_update->vops->vm;
1951 struct xe_bo *pt_bo = updates[j].pt_bo;
1952
1953 if (idx == chunk)
1954 goto next_cmd;
1955
1956 xe_tile_assert(tile, xe_bo_size(pt_bo) == SZ_4K);
1957
1958 /* Map a PT at most once */
1959 if (pt_bo->update_index < 0)
1960 pt_bo->update_index = current_update;
1961
1962 addr = vm->pt_ops->pte_encode_bo(pt_bo, 0,
1963 pat_index, 0);
1964 bb->cs[bb->len++] = lower_32_bits(addr);
1965 bb->cs[bb->len++] = upper_32_bits(addr);
1966 }
1967
1968 j = 0;
1969 }
1970
1971next_cmd:
1972 ptes -= chunk;
1973 ofs += chunk * sizeof(u64);
1974 }
1975
1976 bb->cs[bb->len++] = MI_BATCH_BUFFER_END;
1977 update_idx = bb->len;
1978
1979 addr = xe_migrate_vm_addr(ppgtt_ofs, 0) +
1980 (page_ofs / sizeof(u64)) * XE_PAGE_SIZE;
1981 for (i = 0; i < pt_update_ops->num_ops; ++i) {
1982 struct xe_vm_pgtable_update_op *pt_op =
1983 &pt_update_ops->ops[i];
1984 struct xe_vm_pgtable_update *updates = pt_op->entries;
1985
1986 for (j = 0; j < pt_op->num_entries; ++j) {
1987 struct xe_bo *pt_bo = updates[j].pt_bo;
1988
1989 write_pgtable(tile, bb, addr +
1990 pt_bo->update_index * XE_PAGE_SIZE,
1991 pt_op, &updates[j], pt_update);
1992 }
1993 }
1994 } else {
1995 /* phys pages, no preamble required */
1996 bb->cs[bb->len++] = MI_BATCH_BUFFER_END;
1997 update_idx = bb->len;
1998
1999 for (i = 0; i < pt_update_ops->num_ops; ++i) {
2000 struct xe_vm_pgtable_update_op *pt_op =
2001 &pt_update_ops->ops[i];
2002 struct xe_vm_pgtable_update *updates = pt_op->entries;
2003
2004 for (j = 0; j < pt_op->num_entries; ++j)
2005 write_pgtable(tile, bb, 0, pt_op, &updates[j],
2006 pt_update);
2007 }
2008 }
2009
2010 job = xe_bb_create_migration_job(pt_update_ops->q, bb,
2011 xe_migrate_batch_base(m, usm),
2012 update_idx);
2013 if (IS_ERR(job)) {
2014 err = PTR_ERR(job);
2015 goto err_sa;
2016 }
2017
2018 xe_sched_job_add_migrate_flush(job, MI_INVALIDATE_TLB);
2019
2020 if (ops->pre_commit) {
2021 pt_update->job = job;
2022 err = ops->pre_commit(pt_update);
2023 if (err)
2024 goto err_job;
2025 }
2026 if (is_migrate)
2027 mutex_lock(&m->job_mutex);
2028
2029 xe_sched_job_arm(job);
2030 fence = dma_fence_get(&job->drm.s_fence->finished);
2031 xe_sched_job_push(job);
2032
2033 if (is_migrate)
2034 mutex_unlock(&m->job_mutex);
2035
2036 xe_bb_free(bb, fence);
2037 drm_suballoc_free(sa_bo, fence);
2038
2039 return fence;
2040
2041err_job:
2042 xe_sched_job_put(job);
2043err_sa:
2044 drm_suballoc_free(sa_bo, NULL);
2045err_bb:
2046 xe_bb_free(bb, NULL);
2047 return ERR_PTR(err);
2048}
2049
2050/**
2051 * xe_migrate_update_pgtables() - Pipelined page-table update
2052 * @m: The migrate context.
2053 * @pt_update: PT update arguments
2054 *
2055 * Perform a pipelined page-table update. The update descriptors are typically
2056 * built under the same lock critical section as a call to this function. If
2057 * using the default engine for the updates, they will be performed in the
2058 * order they grab the job_mutex. If different engines are used, external
2059 * synchronization is needed for overlapping updates to maintain page-table
2060 * consistency. Note that the meaning of "overlapping" is that the updates
2061 * touch the same page-table, which might be a higher-level page-directory.
2062 * If no pipelining is needed, then updates may be performed by the cpu.
2063 *
2064 * Return: A dma_fence that, when signaled, indicates the update completion.
2065 */
2066struct dma_fence *
2067xe_migrate_update_pgtables(struct xe_migrate *m,
2068 struct xe_migrate_pt_update *pt_update)
2069
2070{
2071 struct xe_vm_pgtable_update_ops *pt_update_ops =
2072 &pt_update->vops->pt_update_ops[pt_update->tile_id];
2073 struct dma_fence *fence;
2074
2075 fence = xe_migrate_update_pgtables_cpu(m, pt_update);
2076
2077 /* -ETIME indicates a job is needed, anything else is legit error */
2078 if (!IS_ERR(fence) || PTR_ERR(fence) != -ETIME)
2079 return fence;
2080
2081 return __xe_migrate_update_pgtables(m, pt_update, pt_update_ops);
2082}
2083
2084/**
2085 * xe_migrate_wait() - Complete all operations using the xe_migrate context
2086 * @m: Migrate context to wait for.
2087 *
2088 * Waits until the GPU no longer uses the migrate context's default engine
2089 * or its page-table objects. FIXME: What about separate page-table update
2090 * engines?
2091 */
2092void xe_migrate_wait(struct xe_migrate *m)
2093{
2094 if (m->fence)
2095 dma_fence_wait(m->fence, false);
2096}
2097
2098static u32 pte_update_cmd_size(u64 size)
2099{
2100 u32 num_dword;
2101 u64 entries = DIV_U64_ROUND_UP(size, XE_PAGE_SIZE);
2102
2103 XE_WARN_ON(size > MAX_PREEMPTDISABLE_TRANSFER);
2104
2105 /*
2106 * MI_STORE_DATA_IMM command is used to update page table. Each
2107 * instruction can update maximumly MAX_PTE_PER_SDI pte entries. To
2108 * update n (n <= MAX_PTE_PER_SDI) pte entries, we need:
2109 *
2110 * - 1 dword for the MI_STORE_DATA_IMM command header (opcode etc)
2111 * - 2 dword for the page table's physical location
2112 * - 2*n dword for value of pte to fill (each pte entry is 2 dwords)
2113 */
2114 num_dword = (1 + 2) * DIV_U64_ROUND_UP(entries, MAX_PTE_PER_SDI);
2115 num_dword += entries * 2;
2116
2117 return num_dword;
2118}
2119
2120static void build_pt_update_batch_sram(struct xe_migrate *m,
2121 struct xe_bb *bb, u32 pt_offset,
2122 struct drm_pagemap_addr *sram_addr,
2123 u32 size, int level)
2124{
2125 u16 pat_index = xe_cache_pat_idx(tile_to_xe(m->tile), XE_CACHE_WB);
2126 u64 gpu_page_size = 0x1ull << xe_pt_shift(level);
2127 u32 ptes;
2128 int i = 0;
2129
2130 xe_tile_assert(m->tile, PAGE_ALIGNED(size));
2131
2132 ptes = DIV_ROUND_UP(size, gpu_page_size);
2133 while (ptes) {
2134 u32 chunk = min(MAX_PTE_PER_SDI, ptes);
2135
2136 if (!level)
2137 chunk = ALIGN_DOWN(chunk, PAGE_SIZE / XE_PAGE_SIZE);
2138
2139 bb->cs[bb->len++] = MI_STORE_DATA_IMM | MI_SDI_NUM_QW(chunk);
2140 bb->cs[bb->len++] = pt_offset;
2141 bb->cs[bb->len++] = 0;
2142
2143 pt_offset += chunk * 8;
2144 ptes -= chunk;
2145
2146 while (chunk--) {
2147 u64 addr = sram_addr[i].addr;
2148 u64 pte;
2149
2150 xe_tile_assert(m->tile, sram_addr[i].proto ==
2151 DRM_INTERCONNECT_SYSTEM ||
2152 sram_addr[i].proto == XE_INTERCONNECT_P2P);
2153 xe_tile_assert(m->tile, addr);
2154 xe_tile_assert(m->tile, PAGE_ALIGNED(addr));
2155
2156again:
2157 pte = m->q->vm->pt_ops->pte_encode_addr(m->tile->xe,
2158 addr, pat_index,
2159 level, false, 0);
2160 bb->cs[bb->len++] = lower_32_bits(pte);
2161 bb->cs[bb->len++] = upper_32_bits(pte);
2162
2163 if (gpu_page_size < PAGE_SIZE) {
2164 addr += XE_PAGE_SIZE;
2165 if (!PAGE_ALIGNED(addr)) {
2166 chunk--;
2167 goto again;
2168 }
2169 i++;
2170 } else {
2171 i += gpu_page_size / PAGE_SIZE;
2172 }
2173 }
2174 }
2175}
2176
2177static bool xe_migrate_vram_use_pde(struct drm_pagemap_addr *sram_addr,
2178 unsigned long size)
2179{
2180 u32 large_size = (0x1 << xe_pt_shift(1));
2181 unsigned long i, incr = large_size / PAGE_SIZE;
2182
2183 for (i = 0; i < DIV_ROUND_UP(size, PAGE_SIZE); i += incr)
2184 if (PAGE_SIZE << sram_addr[i].order != large_size)
2185 return false;
2186
2187 return true;
2188}
2189
2190#define XE_CACHELINE_BYTES 64ull
2191#define XE_CACHELINE_MASK (XE_CACHELINE_BYTES - 1)
2192
2193static u32 xe_migrate_copy_pitch(struct xe_device *xe, u32 len)
2194{
2195 u32 pitch;
2196
2197 if (IS_ALIGNED(len, PAGE_SIZE))
2198 pitch = PAGE_SIZE;
2199 else if (IS_ALIGNED(len, SZ_4K))
2200 pitch = SZ_4K;
2201 else if (IS_ALIGNED(len, SZ_256))
2202 pitch = SZ_256;
2203 else if (IS_ALIGNED(len, 4))
2204 pitch = 4;
2205 else
2206 pitch = 1;
2207
2208 xe_assert(xe, pitch > 1 || xe->info.has_mem_copy_instr);
2209 return pitch;
2210}
2211
2212static struct dma_fence *xe_migrate_vram(struct xe_migrate *m,
2213 unsigned long len,
2214 unsigned long sram_offset,
2215 struct drm_pagemap_addr *sram_addr,
2216 u64 vram_addr,
2217 struct dma_fence *deps,
2218 const enum xe_migrate_copy_dir dir)
2219{
2220 struct xe_gt *gt = m->tile->primary_gt;
2221 struct xe_device *xe = gt_to_xe(gt);
2222 bool use_usm_batch = xe->info.has_usm;
2223 struct dma_fence *fence = NULL;
2224 u32 batch_size = 1;
2225 u64 src_L0_ofs, dst_L0_ofs;
2226 struct xe_sched_job *job;
2227 struct xe_bb *bb;
2228 u32 update_idx, pt_slot = 0;
2229 unsigned long npages = DIV_ROUND_UP(len + sram_offset, PAGE_SIZE);
2230 unsigned int pitch = xe_migrate_copy_pitch(xe, len);
2231 int err;
2232 unsigned long i, j;
2233 bool use_pde = xe_migrate_vram_use_pde(sram_addr, len + sram_offset);
2234
2235 if (!xe->info.has_mem_copy_instr &&
2236 drm_WARN_ON(&xe->drm,
2237 (!IS_ALIGNED(len, pitch)) || (sram_offset | vram_addr) & XE_CACHELINE_MASK))
2238 return ERR_PTR(-EOPNOTSUPP);
2239
2240 xe_assert(xe, npages * PAGE_SIZE <= MAX_PREEMPTDISABLE_TRANSFER);
2241
2242 batch_size += pte_update_cmd_size(npages << PAGE_SHIFT);
2243 batch_size += emit_copy_cmd_len(xe);
2244
2245 bb = xe_bb_new(gt, batch_size, use_usm_batch);
2246 if (IS_ERR(bb)) {
2247 err = PTR_ERR(bb);
2248 return ERR_PTR(err);
2249 }
2250
2251 /*
2252 * If the order of a struct drm_pagemap_addr entry is greater than 0,
2253 * the entry is populated by GPU pagemap but subsequent entries within
2254 * the range of that order are not populated.
2255 * build_pt_update_batch_sram() expects a fully populated array of
2256 * struct drm_pagemap_addr. Ensure this is the case even with higher
2257 * orders.
2258 */
2259 for (i = 0; !use_pde && i < npages;) {
2260 unsigned int order = sram_addr[i].order;
2261
2262 for (j = 1; j < NR_PAGES(order) && i + j < npages; j++)
2263 if (!sram_addr[i + j].addr)
2264 sram_addr[i + j].addr = sram_addr[i].addr + j * PAGE_SIZE;
2265
2266 i += NR_PAGES(order);
2267 }
2268
2269 if (use_pde)
2270 build_pt_update_batch_sram(m, bb, m->large_page_copy_pdes,
2271 sram_addr, npages << PAGE_SHIFT, 1);
2272 else
2273 build_pt_update_batch_sram(m, bb, pt_slot * XE_PAGE_SIZE,
2274 sram_addr, npages << PAGE_SHIFT, 0);
2275
2276 if (dir == XE_MIGRATE_COPY_TO_VRAM) {
2277 if (use_pde)
2278 src_L0_ofs = m->large_page_copy_ofs + sram_offset;
2279 else
2280 src_L0_ofs = xe_migrate_vm_addr(pt_slot, 0) + sram_offset;
2281 dst_L0_ofs = xe_migrate_vram_ofs(xe, vram_addr, false);
2282
2283 } else {
2284 src_L0_ofs = xe_migrate_vram_ofs(xe, vram_addr, false);
2285 if (use_pde)
2286 dst_L0_ofs = m->large_page_copy_ofs + sram_offset;
2287 else
2288 dst_L0_ofs = xe_migrate_vm_addr(pt_slot, 0) + sram_offset;
2289 }
2290
2291 bb->cs[bb->len++] = MI_BATCH_BUFFER_END;
2292 update_idx = bb->len;
2293
2294 emit_copy(gt, bb, src_L0_ofs, dst_L0_ofs, len, pitch);
2295
2296 job = xe_bb_create_migration_job(m->q, bb,
2297 xe_migrate_batch_base(m, use_usm_batch),
2298 update_idx);
2299 if (IS_ERR(job)) {
2300 err = PTR_ERR(job);
2301 goto err;
2302 }
2303
2304 xe_sched_job_add_migrate_flush(job, MI_INVALIDATE_TLB);
2305
2306 if (deps && !dma_fence_is_signaled(deps)) {
2307 dma_fence_get(deps);
2308 err = drm_sched_job_add_dependency(&job->drm, deps);
2309 if (err)
2310 dma_fence_wait(deps, false);
2311 err = 0;
2312 }
2313
2314 mutex_lock(&m->job_mutex);
2315 xe_sched_job_arm(job);
2316 fence = dma_fence_get(&job->drm.s_fence->finished);
2317 xe_sched_job_push(job);
2318
2319 dma_fence_put(m->fence);
2320 m->fence = dma_fence_get(fence);
2321 mutex_unlock(&m->job_mutex);
2322
2323 xe_bb_free(bb, fence);
2324
2325 return fence;
2326
2327err:
2328 xe_bb_free(bb, NULL);
2329
2330 return ERR_PTR(err);
2331}
2332
2333/**
2334 * xe_migrate_to_vram() - Migrate to VRAM
2335 * @m: The migration context.
2336 * @npages: Number of pages to migrate.
2337 * @src_addr: Array of DMA information (source of migrate)
2338 * @dst_addr: Device physical address of VRAM (destination of migrate)
2339 * @deps: struct dma_fence representing the dependencies that need
2340 * to be signaled before migration.
2341 *
2342 * Copy from an array dma addresses to a VRAM device physical address
2343 *
2344 * Return: dma fence for migrate to signal completion on success, ERR_PTR on
2345 * failure
2346 */
2347struct dma_fence *xe_migrate_to_vram(struct xe_migrate *m,
2348 unsigned long npages,
2349 struct drm_pagemap_addr *src_addr,
2350 u64 dst_addr,
2351 struct dma_fence *deps)
2352{
2353 return xe_migrate_vram(m, npages * PAGE_SIZE, 0, src_addr, dst_addr,
2354 deps, XE_MIGRATE_COPY_TO_VRAM);
2355}
2356
2357/**
2358 * xe_migrate_from_vram() - Migrate from VRAM
2359 * @m: The migration context.
2360 * @npages: Number of pages to migrate.
2361 * @src_addr: Device physical address of VRAM (source of migrate)
2362 * @dst_addr: Array of DMA information (destination of migrate)
2363 * @deps: struct dma_fence representing the dependencies that need
2364 * to be signaled before migration.
2365 *
2366 * Copy from a VRAM device physical address to an array dma addresses
2367 *
2368 * Return: dma fence for migrate to signal completion on success, ERR_PTR on
2369 * failure
2370 */
2371struct dma_fence *xe_migrate_from_vram(struct xe_migrate *m,
2372 unsigned long npages,
2373 u64 src_addr,
2374 struct drm_pagemap_addr *dst_addr,
2375 struct dma_fence *deps)
2376{
2377 return xe_migrate_vram(m, npages * PAGE_SIZE, 0, dst_addr, src_addr,
2378 deps, XE_MIGRATE_COPY_TO_SRAM);
2379}
2380
2381static void xe_migrate_dma_unmap(struct xe_device *xe,
2382 struct drm_pagemap_addr *pagemap_addr,
2383 int len, int write)
2384{
2385 unsigned long i, npages = DIV_ROUND_UP(len, PAGE_SIZE);
2386
2387 for (i = 0; i < npages; ++i) {
2388 if (!pagemap_addr[i].addr)
2389 break;
2390
2391 dma_unmap_page(xe->drm.dev, pagemap_addr[i].addr, PAGE_SIZE,
2392 write ? DMA_TO_DEVICE : DMA_FROM_DEVICE);
2393 }
2394 kfree(pagemap_addr);
2395}
2396
2397static struct drm_pagemap_addr *xe_migrate_dma_map(struct xe_device *xe,
2398 void *buf, int len,
2399 int write)
2400{
2401 struct drm_pagemap_addr *pagemap_addr;
2402 unsigned long i, npages = DIV_ROUND_UP(len, PAGE_SIZE);
2403
2404 pagemap_addr = kzalloc_objs(*pagemap_addr, npages);
2405 if (!pagemap_addr)
2406 return ERR_PTR(-ENOMEM);
2407
2408 for (i = 0; i < npages; ++i) {
2409 dma_addr_t addr;
2410 struct page *page;
2411 enum dma_data_direction dir = write ? DMA_TO_DEVICE :
2412 DMA_FROM_DEVICE;
2413
2414 if (is_vmalloc_addr(buf))
2415 page = vmalloc_to_page(buf);
2416 else
2417 page = virt_to_page(buf);
2418
2419 addr = dma_map_page(xe->drm.dev, page, 0, PAGE_SIZE, dir);
2420 if (dma_mapping_error(xe->drm.dev, addr))
2421 goto err_fault;
2422
2423 pagemap_addr[i] =
2424 drm_pagemap_addr_encode(addr,
2425 DRM_INTERCONNECT_SYSTEM,
2426 0, dir);
2427 buf += PAGE_SIZE;
2428 }
2429
2430 return pagemap_addr;
2431
2432err_fault:
2433 xe_migrate_dma_unmap(xe, pagemap_addr, len, write);
2434 return ERR_PTR(-EFAULT);
2435}
2436
2437/**
2438 * xe_migrate_access_memory - Access memory of a BO via GPU
2439 *
2440 * @m: The migration context.
2441 * @bo: buffer object
2442 * @offset: access offset into buffer object
2443 * @buf: pointer to caller memory to read into or write from
2444 * @len: length of access
2445 * @write: write access
2446 *
2447 * Access memory of a BO via GPU either reading in or writing from a passed in
2448 * pointer. Pointer is dma mapped for GPU access and GPU commands are issued to
2449 * read to or write from pointer.
2450 *
2451 * Returns:
2452 * 0 if successful, negative error code on failure.
2453 */
2454int xe_migrate_access_memory(struct xe_migrate *m, struct xe_bo *bo,
2455 unsigned long offset, void *buf, int len,
2456 int write)
2457{
2458 struct xe_tile *tile = m->tile;
2459 struct xe_device *xe = tile_to_xe(tile);
2460 struct xe_res_cursor cursor;
2461 struct dma_fence *fence = NULL;
2462 struct drm_pagemap_addr *pagemap_addr;
2463 unsigned long page_offset = (unsigned long)buf & ~PAGE_MASK;
2464 int bytes_left = len, current_page = 0;
2465 void *orig_buf = buf;
2466
2467 xe_bo_assert_held(bo);
2468
2469 /* Use bounce buffer for small access and unaligned access */
2470 if (!xe->info.has_mem_copy_instr &&
2471 (!IS_ALIGNED(len, 4) ||
2472 !IS_ALIGNED(page_offset, XE_CACHELINE_BYTES) ||
2473 !IS_ALIGNED(offset, XE_CACHELINE_BYTES))) {
2474 int buf_offset = 0;
2475 void *bounce;
2476 int err;
2477
2478 BUILD_BUG_ON(!is_power_of_2(XE_CACHELINE_BYTES));
2479 bounce = kmalloc(XE_CACHELINE_BYTES, GFP_KERNEL);
2480 if (!bounce)
2481 return -ENOMEM;
2482
2483 /*
2484 * Less than ideal for large unaligned access but this should be
2485 * fairly rare, can fixup if this becomes common.
2486 */
2487 do {
2488 int copy_bytes = min_t(int, bytes_left,
2489 XE_CACHELINE_BYTES -
2490 (offset & XE_CACHELINE_MASK));
2491 int ptr_offset = offset & XE_CACHELINE_MASK;
2492
2493 err = xe_migrate_access_memory(m, bo,
2494 offset &
2495 ~XE_CACHELINE_MASK,
2496 bounce,
2497 XE_CACHELINE_BYTES, 0);
2498 if (err)
2499 break;
2500
2501 if (write) {
2502 memcpy(bounce + ptr_offset, buf + buf_offset, copy_bytes);
2503
2504 err = xe_migrate_access_memory(m, bo,
2505 offset & ~XE_CACHELINE_MASK,
2506 bounce,
2507 XE_CACHELINE_BYTES, write);
2508 if (err)
2509 break;
2510 } else {
2511 memcpy(buf + buf_offset, bounce + ptr_offset,
2512 copy_bytes);
2513 }
2514
2515 bytes_left -= copy_bytes;
2516 buf_offset += copy_bytes;
2517 offset += copy_bytes;
2518 } while (bytes_left);
2519
2520 kfree(bounce);
2521 return err;
2522 }
2523
2524 pagemap_addr = xe_migrate_dma_map(xe, buf, len + page_offset, write);
2525 if (IS_ERR(pagemap_addr))
2526 return PTR_ERR(pagemap_addr);
2527
2528 xe_res_first(bo->ttm.resource, offset, xe_bo_size(bo) - offset, &cursor);
2529
2530 do {
2531 struct dma_fence *__fence;
2532 u64 vram_addr = vram_region_gpu_offset(bo->ttm.resource) +
2533 cursor.start;
2534 int current_bytes;
2535 u32 pitch;
2536
2537 if (cursor.size > MAX_PREEMPTDISABLE_TRANSFER)
2538 current_bytes = min_t(int, bytes_left,
2539 MAX_PREEMPTDISABLE_TRANSFER);
2540 else
2541 current_bytes = min_t(int, bytes_left, cursor.size);
2542
2543 pitch = xe_migrate_copy_pitch(xe, current_bytes);
2544 if (xe->info.has_mem_copy_instr)
2545 current_bytes = min_t(int, current_bytes, U16_MAX * pitch);
2546 else
2547 current_bytes = min_t(int, current_bytes,
2548 round_down(S16_MAX * pitch,
2549 XE_CACHELINE_BYTES));
2550
2551 __fence = xe_migrate_vram(m, current_bytes,
2552 (unsigned long)buf & ~PAGE_MASK,
2553 &pagemap_addr[current_page],
2554 vram_addr, NULL, write ?
2555 XE_MIGRATE_COPY_TO_VRAM :
2556 XE_MIGRATE_COPY_TO_SRAM);
2557 if (IS_ERR(__fence)) {
2558 if (fence) {
2559 dma_fence_wait(fence, false);
2560 dma_fence_put(fence);
2561 }
2562 fence = __fence;
2563 goto out_err;
2564 }
2565
2566 dma_fence_put(fence);
2567 fence = __fence;
2568
2569 buf += current_bytes;
2570 offset += current_bytes;
2571 current_page = (int)(buf - orig_buf) / PAGE_SIZE;
2572 bytes_left -= current_bytes;
2573 if (bytes_left)
2574 xe_res_next(&cursor, current_bytes);
2575 } while (bytes_left);
2576
2577 dma_fence_wait(fence, false);
2578 dma_fence_put(fence);
2579
2580out_err:
2581 xe_migrate_dma_unmap(xe, pagemap_addr, len + page_offset, write);
2582 return IS_ERR(fence) ? PTR_ERR(fence) : 0;
2583}
2584
2585/**
2586 * xe_migrate_job_lock() - Lock migrate job lock
2587 * @m: The migration context.
2588 * @q: Queue associated with the operation which requires a lock
2589 *
2590 * Lock the migrate job lock if the queue is a migration queue, otherwise
2591 * assert the VM's dma-resv is held (user queue's have own locking).
2592 */
2593void xe_migrate_job_lock(struct xe_migrate *m, struct xe_exec_queue *q)
2594{
2595 bool is_migrate = q == m->q;
2596
2597 if (is_migrate)
2598 mutex_lock(&m->job_mutex);
2599 else
2600 xe_vm_assert_held(q->user_vm); /* User queues VM's should be locked */
2601}
2602
2603/**
2604 * xe_migrate_job_unlock() - Unlock migrate job lock
2605 * @m: The migration context.
2606 * @q: Queue associated with the operation which requires a lock
2607 *
2608 * Unlock the migrate job lock if the queue is a migration queue, otherwise
2609 * assert the VM's dma-resv is held (user queue's have own locking).
2610 */
2611void xe_migrate_job_unlock(struct xe_migrate *m, struct xe_exec_queue *q)
2612{
2613 bool is_migrate = q == m->q;
2614
2615 if (is_migrate)
2616 mutex_unlock(&m->job_mutex);
2617 else
2618 xe_vm_assert_held(q->user_vm); /* User queues VM's should be locked */
2619}
2620
2621#if IS_ENABLED(CONFIG_PROVE_LOCKING)
2622/**
2623 * xe_migrate_job_lock_assert() - Assert migrate job lock held of queue
2624 * @q: Migrate queue
2625 */
2626void xe_migrate_job_lock_assert(struct xe_exec_queue *q)
2627{
2628 struct xe_migrate *m = gt_to_tile(q->gt)->migrate;
2629
2630 xe_gt_assert(q->gt, q == m->q);
2631 lockdep_assert_held(&m->job_mutex);
2632}
2633#endif
2634
2635#if IS_ENABLED(CONFIG_DRM_XE_KUNIT_TEST)
2636#include "tests/xe_migrate.c"
2637#endif