linux - drivers/gpu/drm/xe/xe

5014 lines · 123.8 KiB

1	// SPDX-License-Identifier: MIT
2	/*
3	* Copyright © 2021 Intel Corporation
4	*/
5
6	#include "xe_vm.h"
7
8	#include <linux/dma-fence-array.h>
9	#include <linux/nospec.h>
10
11	#include <drm/drm_drv.h>
12	#include <drm/drm_exec.h>
13	#include <drm/drm_print.h>
14	#include <drm/ttm/ttm_tt.h>
15	#include <uapi/drm/xe_drm.h>
16	#include <linux/ascii85.h>
17	#include <linux/delay.h>
18	#include <linux/kthread.h>
19	#include <linux/mm.h>
20	#include <linux/swap.h>
21
22	#include <generated/xe_wa_oob.h>
23
24	#include "regs/xe_gtt_defs.h"
25	#include "xe_assert.h"
26	#include "xe_bo.h"
27	#include "xe_device.h"
28	#include "xe_drm_client.h"
29	#include "xe_exec_queue.h"
30	#include "xe_gt.h"
31	#include "xe_migrate.h"
32	#include "xe_pat.h"
33	#include "xe_pm.h"
34	#include "xe_preempt_fence.h"
35	#include "xe_pt.h"
36	#include "xe_pxp.h"
37	#include "xe_sriov_vf.h"
38	#include "xe_svm.h"
39	#include "xe_sync.h"
40	#include "xe_tile.h"
41	#include "xe_tlb_inval.h"
42	#include "xe_trace_bo.h"
43	#include "xe_vm_madvise.h"
44	#include "xe_wa.h"
45
46	static struct drm_gem_object xe_vm_obj(struct xe_vm vm)
47	{
48	return vm->gpuvm.r_obj;
49	}
50
51	/**
52	* xe_vm_drm_exec_lock() - Lock the vm's resv with a drm_exec transaction
53	* @vm: The vm whose resv is to be locked.
54	* @exec: The drm_exec transaction.
55	*
56	* Helper to lock the vm's resv as part of a drm_exec transaction.
57	*
58	* Return: %0 on success. See drm_exec_lock_obj() for error codes.
59	*/
60	int xe_vm_drm_exec_lock(struct xe_vm vm, struct drm_exec exec)
61	{
62	return drm_exec_lock_obj(exec, xe_vm_obj(vm));
63	}
64
65	static bool preempt_fences_waiting(struct xe_vm *vm)
66	{
67	struct xe_exec_queue *q;
68
69	lockdep_assert_held(&vm->lock);
70	xe_vm_assert_held(vm);
71
72	list_for_each_entry(q, &vm->preempt.exec_queues, lr.link) {
73	if (!q->lr.pfence \|\|
74	test_bit(DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT,
75	&q->lr.pfence->flags)) {
76	return true;
77	}
78	}
79
80	return false;
81	}
82
83	static void free_preempt_fences(struct list_head *list)
84	{
85	struct list_head link, next;
86
87	list_for_each_safe(link, next, list)
88	xe_preempt_fence_free(to_preempt_fence_from_link(link));
89	}
90
91	static int alloc_preempt_fences(struct xe_vm vm, struct list_head list,
92	unsigned int *count)
93	{
94	lockdep_assert_held(&vm->lock);
95	xe_vm_assert_held(vm);
96
97	if (*count >= vm->preempt.num_exec_queues)
98	return 0;
99
100	for (; count < vm->preempt.num_exec_queues; ++(count)) {
101	struct xe_preempt_fence *pfence = xe_preempt_fence_alloc();
102
103	if (IS_ERR(pfence))
104	return PTR_ERR(pfence);
105
106	list_move_tail(xe_preempt_fence_link(pfence), list);
107	}
108
109	return 0;
110	}
111
112	static int wait_for_existing_preempt_fences(struct xe_vm *vm)
113	{
114	struct xe_exec_queue *q;
115	bool vf_migration = IS_SRIOV_VF(vm->xe) &&
116	xe_sriov_vf_migration_supported(vm->xe);
117	signed long wait_time = vf_migration ? HZ / 5 : MAX_SCHEDULE_TIMEOUT;
118
119	xe_vm_assert_held(vm);
120
121	list_for_each_entry(q, &vm->preempt.exec_queues, lr.link) {
122	if (q->lr.pfence) {
123	long timeout;
124
125	timeout = dma_fence_wait_timeout(q->lr.pfence, false,
126	wait_time);
127	if (!timeout) {
128	xe_assert(vm->xe, vf_migration);
129	return -EAGAIN;
130	}
131
132	/* Only -ETIME on fence indicates VM needs to be killed */
133	if (timeout < 0 \|\| q->lr.pfence->error == -ETIME)
134	return -ETIME;
135
136	dma_fence_put(q->lr.pfence);
137	q->lr.pfence = NULL;
138	}
139	}
140
141	return 0;
142	}
143
144	static bool xe_vm_is_idle(struct xe_vm *vm)
145	{
146	struct xe_exec_queue *q;
147
148	xe_vm_assert_held(vm);
149	list_for_each_entry(q, &vm->preempt.exec_queues, lr.link) {
150	if (!xe_exec_queue_is_idle(q))
151	return false;
152	}
153
154	return true;
155	}
156
157	static void arm_preempt_fences(struct xe_vm vm, struct list_head list)
158	{
159	struct list_head *link;
160	struct xe_exec_queue *q;
161
162	list_for_each_entry(q, &vm->preempt.exec_queues, lr.link) {
163	struct dma_fence *fence;
164
165	link = list->next;
166	xe_assert(vm->xe, link != list);
167
168	fence = xe_preempt_fence_arm(to_preempt_fence_from_link(link),
169	q, q->lr.context,
170	++q->lr.seqno);
171	dma_fence_put(q->lr.pfence);
172	q->lr.pfence = fence;
173	}
174	}
175
176	static int add_preempt_fences(struct xe_vm vm, struct xe_bo bo)
177	{
178	struct xe_exec_queue *q;
179	int err;
180
181	xe_bo_assert_held(bo);
182
183	if (!vm->preempt.num_exec_queues)
184	return 0;
185
186	err = dma_resv_reserve_fences(bo->ttm.base.resv, vm->preempt.num_exec_queues);
187	if (err)
188	return err;
189
190	list_for_each_entry(q, &vm->preempt.exec_queues, lr.link)
191	if (q->lr.pfence) {
192	dma_resv_add_fence(bo->ttm.base.resv,
193	q->lr.pfence,
194	DMA_RESV_USAGE_BOOKKEEP);
195	}
196
197	return 0;
198	}
199
200	static void resume_and_reinstall_preempt_fences(struct xe_vm *vm,
201	struct drm_exec *exec)
202	{
203	struct xe_exec_queue *q;
204
205	lockdep_assert_held(&vm->lock);
206	xe_vm_assert_held(vm);
207
208	list_for_each_entry(q, &vm->preempt.exec_queues, lr.link) {
209	/*
210	* Only resume queues whose suspend() actually succeeded. A
211	* failed suspend() (e.g. killed/banned/wedged) leaves the queue
212	* un-suspended, so it must not be resumed.
213	*
214	* Also skip queues that have since been reset/killed/banned/
215	* wedged: their suspend may not have completed (suspend_pending
216	* can still be set, e.g. a preempt fence signalled with -ENOENT
217	* without waiting), so resuming would trip the !suspend_pending
218	* assert in the backend. Such queues are being torn down anyway,
219	* so leave them marked suspended and let teardown resolve their
220	* state.
221	*/
222	if (READ_ONCE(q->lr.suspended) && !q->ops->reset_status(q)) {
223	WRITE_ONCE(q->lr.suspended, false);
224	q->ops->resume(q);
225	}
226
227	drm_gpuvm_resv_add_fence(&vm->gpuvm, exec, q->lr.pfence,
228	DMA_RESV_USAGE_BOOKKEEP, DMA_RESV_USAGE_BOOKKEEP);
229	}
230	}
231
232	int xe_vm_add_compute_exec_queue(struct xe_vm vm, struct xe_exec_queue q)
233	{
234	struct drm_gpuvm_exec vm_exec = {
235	.vm = &vm->gpuvm,
236	.flags = DRM_EXEC_INTERRUPTIBLE_WAIT,
237	.num_fences = 1,
238	};
239	struct drm_exec *exec = &vm_exec.exec;
240	struct xe_validation_ctx ctx;
241	struct dma_fence *pfence;
242	int err;
243	bool wait;
244
245	xe_assert(vm->xe, xe_vm_in_preempt_fence_mode(vm));
246
247	down_write(&vm->lock);
248	err = xe_validation_exec_lock(&ctx, &vm_exec, &vm->xe->val);
249	if (err)
250	goto out_up_write;
251
252	pfence = xe_preempt_fence_create(q, q->lr.context,
253	++q->lr.seqno);
254	if (IS_ERR(pfence)) {
255	err = PTR_ERR(pfence);
256	goto out_fini;
257	}
258
259	list_add(&q->lr.link, &vm->preempt.exec_queues);
260	++vm->preempt.num_exec_queues;
261	q->lr.pfence = pfence;
262
263	xe_svm_notifier_lock(vm);
264
265	drm_gpuvm_resv_add_fence(&vm->gpuvm, exec, pfence,
266	DMA_RESV_USAGE_BOOKKEEP, DMA_RESV_USAGE_BOOKKEEP);
267
268	/*
269	* Check to see if a preemption on VM is in flight or userptr
270	* invalidation, if so trigger this preempt fence to sync state with
271	* other preempt fences on the VM.
272	*/
273	wait = __xe_vm_userptr_needs_repin(vm) \|\| preempt_fences_waiting(vm);
274	if (wait)
275	dma_fence_enable_signaling(pfence);
276
277	xe_svm_notifier_unlock(vm);
278
279	out_fini:
280	xe_validation_ctx_fini(&ctx);
281	out_up_write:
282	up_write(&vm->lock);
283
284	return err;
285	}
286	ALLOW_ERROR_INJECTION(xe_vm_add_compute_exec_queue, ERRNO);
287
288	/**
289	* xe_vm_remove_compute_exec_queue() - Remove compute exec queue from VM
290	* @vm: The VM.
291	* @q: The exec_queue
292	*
293	* Note that this function might be called multiple times on the same queue.
294	*/
295	void xe_vm_remove_compute_exec_queue(struct xe_vm vm, struct xe_exec_queue q)
296	{
297	if (!xe_vm_in_preempt_fence_mode(vm))
298	return;
299
300	down_write(&vm->lock);
301	if (!list_empty(&q->lr.link)) {
302	list_del_init(&q->lr.link);
303	--vm->preempt.num_exec_queues;
304	}
305	if (q->lr.pfence) {
306	dma_fence_enable_signaling(q->lr.pfence);
307	dma_fence_put(q->lr.pfence);
308	q->lr.pfence = NULL;
309	}
310	up_write(&vm->lock);
311	}
312
313	#define XE_VM_REBIND_RETRY_TIMEOUT_MS 1000
314
315	/**
316	* xe_vm_kill() - VM Kill
317	* @vm: The VM.
318	* @unlocked: Flag indicates the VM's dma-resv is not held
319	*
320	* Kill the VM by setting banned flag indicated VM is no longer available for
321	* use. If in preempt fence mode, also kill all exec queue attached to the VM.
322	*/
323	void xe_vm_kill(struct xe_vm *vm, bool unlocked)
324	{
325	struct xe_exec_queue *q;
326
327	lockdep_assert_held(&vm->lock);
328
329	if (unlocked)
330	xe_vm_lock(vm, false);
331
332	vm->flags \|= XE_VM_FLAG_BANNED;
333	trace_xe_vm_kill(vm);
334
335	list_for_each_entry(q, &vm->preempt.exec_queues, lr.link)
336	q->ops->kill(q);
337
338	if (unlocked)
339	xe_vm_unlock(vm);
340
341	/* TODO: Inform user the VM is banned */
342	}
343
344	static int xe_gpuvm_validate(struct drm_gpuvm_bo vm_bo, struct drm_exec exec)
345	{
346	struct xe_vm *vm = gpuvm_to_vm(vm_bo->vm);
347	struct xe_bo *bo = gem_to_xe_bo(vm_bo->obj);
348	struct drm_gpuva *gpuva;
349	int ret;
350
351	lockdep_assert_held(&vm->lock);
352	drm_gpuvm_bo_for_each_va(gpuva, vm_bo)
353	list_move_tail(&gpuva_to_vma(gpuva)->combined_links.rebind,
354	&vm->rebind_list);
355
356	/* Skip re-populating purged BOs, rebind maps scratch pages. */
357	if (xe_bo_is_purged(bo)) {
358	vm_bo->evicted = false;
359	return 0;
360	}
361
362	if (!try_wait_for_completion(&vm->xe->pm_block))
363	return -EAGAIN;
364
365	ret = xe_bo_validate(bo, vm, false, exec);
366	if (ret)
367	return ret;
368
369	vm_bo->evicted = false;
370	return 0;
371	}
372
373	/**
374	* xe_vm_validate_rebind() - Validate buffer objects and rebind vmas
375	* @vm: The vm for which we are rebinding.
376	* @exec: The struct drm_exec with the locked GEM objects.
377	* @num_fences: The number of fences to reserve for the operation, not
378	* including rebinds and validations.
379	*
380	* Validates all evicted gem objects and rebinds their vmas. Note that
381	* rebindings may cause evictions and hence the validation-rebind
382	* sequence is rerun until there are no more objects to validate.
383	*
384	* Return: 0 on success, negative error code on error. In particular,
385	* may return -EINTR or -ERESTARTSYS if interrupted, and -EDEADLK if
386	* the drm_exec transaction needs to be restarted.
387	*/
388	int xe_vm_validate_rebind(struct xe_vm vm, struct drm_exec exec,
389	unsigned int num_fences)
390	{
391	struct drm_gem_object *obj;
392	int ret;
393
394	do {
395	ret = drm_gpuvm_validate(&vm->gpuvm, exec);
396	if (ret)
397	return ret;
398
399	ret = xe_vm_rebind(vm, false);
400	if (ret)
401	return ret;
402	} while (!list_empty(&vm->gpuvm.evict.list));
403
404	drm_exec_for_each_locked_object(exec, obj) {
405	ret = dma_resv_reserve_fences(obj->resv, num_fences);
406	if (ret)
407	return ret;
408	}
409
410	return 0;
411	}
412
413	/**
414	* xe_vm_bump_lru() - Bump the LRU position of BOs used by a submission
415	* @vm: The vm for which we are bumping LRUs.
416	* @exec: The struct drm_exec with the locked GEM objects.
417	*
418	* Move the VM's private BOs, and all external objects locked by @exec,
419	* to the LRU tail. Private BOs are moved in bulk using @vm's
420	* &xe_vm.lru_bulk_move. External objects, which may be shared with other
421	* VMs, are not part of that bulk move and are bumped individually.
422	*/
423	void xe_vm_bump_lru(struct xe_vm vm, struct drm_exec exec)
424	{
425	spinlock_t *lru_lock = &vm->xe->ttm.lru_lock;
426	struct drm_gem_object *obj;
427
428	spin_lock(lru_lock);
429	ttm_lru_bulk_move_tail(&vm->lru_bulk_move);
430	drm_exec_for_each_locked_object(exec, obj) {
431	if (!drm_gpuvm_is_extobj(&vm->gpuvm, obj))
432	continue;
433
434	ttm_bo_move_to_lru_tail(&gem_to_xe_bo(obj)->ttm);
435	cond_resched_lock(lru_lock);
436	}
437	spin_unlock(lru_lock);
438	}
439
440	static int xe_preempt_work_begin(struct drm_exec exec, struct xe_vm vm,
441	bool *done)
442	{
443	int err;
444
445	err = drm_gpuvm_prepare_vm(&vm->gpuvm, exec, 0);
446	if (err)
447	return err;
448
449	if (xe_vm_is_idle(vm)) {
450	vm->preempt.rebind_deactivated = true;
451	*done = true;
452	return 0;
453	}
454
455	if (!preempt_fences_waiting(vm)) {
456	*done = true;
457	return 0;
458	}
459
460	err = drm_gpuvm_prepare_objects(&vm->gpuvm, exec, 0);
461	if (err)
462	return err;
463
464	err = wait_for_existing_preempt_fences(vm);
465	if (err)
466	return err;
467
468	/*
469	* Add validation and rebinding to the locking loop since both can
470	* cause evictions which may require blocing dma_resv locks.
471	* The fence reservation here is intended for the new preempt fences
472	* we attach at the end of the rebind work.
473	*/
474	return xe_vm_validate_rebind(vm, exec, vm->preempt.num_exec_queues);
475	}
476
477	static bool vm_suspend_rebind_worker(struct xe_vm *vm)
478	{
479	struct xe_device *xe = vm->xe;
480	bool ret = false;
481
482	mutex_lock(&xe->rebind_resume_lock);
483	if (!try_wait_for_completion(&vm->xe->pm_block)) {
484	ret = true;
485	list_move_tail(&vm->preempt.pm_activate_link, &xe->rebind_resume_list);
486	}
487	mutex_unlock(&xe->rebind_resume_lock);
488
489	return ret;
490	}
491
492	/**
493	* xe_vm_resume_rebind_worker() - Resume the rebind worker.
494	* @vm: The vm whose preempt worker to resume.
495	*
496	* Resume a preempt worker that was previously suspended by
497	* vm_suspend_rebind_worker().
498	*/
499	void xe_vm_resume_rebind_worker(struct xe_vm *vm)
500	{
501	queue_work(vm->xe->ordered_wq, &vm->preempt.rebind_work);
502	}
503
504	static void preempt_rebind_work_func(struct work_struct *w)
505	{
506	struct xe_vm *vm = container_of(w, struct xe_vm, preempt.rebind_work);
507	struct xe_validation_ctx ctx;
508	struct drm_exec exec;
509	unsigned int fence_count = 0;
510	LIST_HEAD(preempt_fences);
511	int err = 0;
512	long wait;
513	int __maybe_unused tries = 0;
514
515	xe_assert(vm->xe, xe_vm_in_preempt_fence_mode(vm));
516	trace_xe_vm_rebind_worker_enter(vm);
517
518	down_write(&vm->lock);
519
520	if (xe_vm_is_closed_or_banned(vm)) {
521	up_write(&vm->lock);
522	trace_xe_vm_rebind_worker_exit(vm);
523	return;
524	}
525
526	retry:
527	if (!try_wait_for_completion(&vm->xe->pm_block) && vm_suspend_rebind_worker(vm)) {
528	up_write(&vm->lock);
529	/* We don't actually block but don't make progress. */
530	xe_pm_might_block_on_suspend();
531	return;
532	}
533
534	if (xe_vm_userptr_check_repin(vm)) {
535	err = xe_vm_userptr_pin(vm);
536	if (err)
537	goto out_unlock_outer;
538	}
539
540	err = xe_validation_ctx_init(&ctx, &vm->xe->val, &exec,
541	(struct xe_val_flags) {.interruptible = true});
542	if (err)
543	goto out_unlock_outer;
544
545	drm_exec_until_all_locked(&exec) {
546	bool done = false;
547
548	err = xe_preempt_work_begin(&exec, vm, &done);
549	drm_exec_retry_on_contention(&exec);
550	xe_validation_retry_on_oom(&ctx, &err);
551	if (err \|\| done) {
552	xe_validation_ctx_fini(&ctx);
553	goto out_unlock_outer;
554	}
555	}
556
557	err = alloc_preempt_fences(vm, &preempt_fences, &fence_count);
558	if (err)
559	goto out_unlock;
560
561	xe_vm_set_validation_exec(vm, &exec);
562	err = xe_vm_rebind(vm, true);
563	xe_vm_set_validation_exec(vm, NULL);
564	if (err)
565	goto out_unlock;
566
567	/* Wait on rebinds and munmap style VM unbinds */
568	wait = dma_resv_wait_timeout(xe_vm_resv(vm),
569	DMA_RESV_USAGE_KERNEL,
570	false, MAX_SCHEDULE_TIMEOUT);
571	if (wait <= 0) {
572	err = -ETIME;
573	goto out_unlock;
574	}
575
576	#define retry_required(__tries, __vm) \
577	(IS_ENABLED(CONFIG_DRM_XE_USERPTR_INVAL_INJECT) ? \
578	(!(__tries)++ \|\| __xe_vm_userptr_needs_repin(__vm)) : \
579	__xe_vm_userptr_needs_repin(__vm))
580
581	xe_svm_notifier_lock(vm);
582	if (retry_required(tries, vm)) {
583	xe_svm_notifier_unlock(vm);
584	err = -EAGAIN;
585	goto out_unlock;
586	}
587
588	#undef retry_required
589
590	xe_vm_bump_lru(vm, &exec);
591
592	/* Point of no return. */
593	arm_preempt_fences(vm, &preempt_fences);
594	resume_and_reinstall_preempt_fences(vm, &exec);
595	xe_svm_notifier_unlock(vm);
596
597	out_unlock:
598	xe_validation_ctx_fini(&ctx);
599	out_unlock_outer:
600	if (err == -EAGAIN) {
601	trace_xe_vm_rebind_worker_retry(vm);
602
603	/*
604	* We can't block in workers on a VF which supports migration
605	* given this can block the VF post-migration workers from
606	* getting scheduled.
607	*/
608	if (IS_SRIOV_VF(vm->xe) &&
609	xe_sriov_vf_migration_supported(vm->xe)) {
610	up_write(&vm->lock);
611	xe_vm_queue_rebind_worker(vm);
612	return;
613	}
614
615	goto retry;
616	}
617
618	if (err) {
619	drm_warn(&vm->xe->drm, "VM worker error: %d\n", err);
620	xe_vm_kill(vm, true);
621	}
622	up_write(&vm->lock);
623
624	free_preempt_fences(&preempt_fences);
625
626	trace_xe_vm_rebind_worker_exit(vm);
627	}
628
629	/**
630	* xe_vm_add_fault_entry_pf() - Add pagefault to vm fault list
631	* @vm: The VM.
632	* @pf: The pagefault.
633	*
634	* This function takes the data from the pagefault @pf and saves it to @vm->faults.list.
635	*
636	* The function exits silently if the list is full, and reports a warning if the pagefault
637	* could not be saved to the list.
638	*/
639	void xe_vm_add_fault_entry_pf(struct xe_vm vm, struct xe_pagefault pf)
640	{
641	struct xe_vm_fault_entry *e;
642	struct xe_hw_engine *hwe;
643
644	/* Do not report faults on reserved engines */
645	hwe = xe_gt_hw_engine(pf->gt, pf->consumer.engine_class,
646	pf->consumer.engine_instance, false);
647	if (!hwe \|\| xe_hw_engine_is_reserved(hwe))
648	return;
649
650	e = kzalloc_obj(*e);
651	if (!e) {
652	drm_warn(&vm->xe->drm,
653	"Could not allocate memory for fault!\n");
654	return;
655	}
656
657	guard(spinlock)(&vm->faults.lock);
658
659	/*
660	* Limit the number of faults in the fault list to prevent
661	* memory overuse.
662	*/
663	if (vm->faults.len >= MAX_FAULTS_SAVED_PER_VM) {
664	kfree(e);
665	return;
666	}
667
668	e->address = pf->consumer.page_addr;
669	/*
670	* TODO:
671	* Address precision is currently always SZ_4K, but this may change
672	* in the future.
673	*/
674	e->address_precision = SZ_4K;
675	e->access_type = pf->consumer.access_type;
676	e->fault_type = FIELD_GET(XE_PAGEFAULT_TYPE_MASK,
677	pf->consumer.fault_type_level);
678	e->fault_level = FIELD_GET(XE_PAGEFAULT_LEVEL_MASK,
679	pf->consumer.fault_type_level);
680
681	list_add_tail(&e->list, &vm->faults.list);
682	vm->faults.len++;
683	}
684
685	static void xe_vm_clear_fault_entries(struct xe_vm *vm)
686	{
687	struct xe_vm_fault_entry e, tmp;
688
689	guard(spinlock)(&vm->faults.lock);
690	list_for_each_entry_safe(e, tmp, &vm->faults.list, list) {
691	list_del(&e->list);
692	kfree(e);
693	}
694	vm->faults.len = 0;
695	}
696
697	static int xe_vma_ops_alloc(struct xe_vma_ops *vops, bool array_of_binds)
698	{
699	int i;
700
701	for (i = 0; i < XE_MAX_TILES_PER_DEVICE; ++i) {
702	if (!vops->pt_update_ops[i].num_ops)
703	continue;
704
705	vops->pt_update_ops[i].ops =
706	kmalloc_objs(*vops->pt_update_ops[i].ops,
707	vops->pt_update_ops[i].num_ops,
708	GFP_KERNEL \| __GFP_RETRY_MAYFAIL \| __GFP_NOWARN);
709	if (!vops->pt_update_ops[i].ops)
710	return array_of_binds ? -ENOBUFS : -ENOMEM;
711	}
712
713	return 0;
714	}
715	ALLOW_ERROR_INJECTION(xe_vma_ops_alloc, ERRNO);
716
717	static void xe_vma_svm_prefetch_op_fini(struct xe_vma_op *op)
718	{
719	struct xe_vma *vma;
720
721	vma = gpuva_to_vma(op->base.prefetch.va);
722
723	if (op->base.op == DRM_GPUVA_OP_PREFETCH && xe_vma_is_cpu_addr_mirror(vma))
724	xa_destroy(&op->prefetch_range.range);
725	}
726
727	static void xe_vma_svm_prefetch_ops_fini(struct xe_vma_ops *vops)
728	{
729	struct xe_vma_op *op;
730
731	if (!(vops->flags & XE_VMA_OPS_FLAG_HAS_SVM_PREFETCH))
732	return;
733
734	list_for_each_entry(op, &vops->list, link)
735	xe_vma_svm_prefetch_op_fini(op);
736	}
737
738	static void xe_vma_ops_fini(struct xe_vma_ops *vops)
739	{
740	int i;
741
742	xe_vma_svm_prefetch_ops_fini(vops);
743
744	for (i = 0; i < XE_MAX_TILES_PER_DEVICE; ++i)
745	kfree(vops->pt_update_ops[i].ops);
746	}
747
748	static void xe_vma_ops_incr_pt_update_ops(struct xe_vma_ops *vops, u8 tile_mask, int inc_val)
749	{
750	int i;
751
752	if (!inc_val)
753	return;
754
755	for (i = 0; i < XE_MAX_TILES_PER_DEVICE; ++i)
756	if (BIT(i) & tile_mask)
757	vops->pt_update_ops[i].num_ops += inc_val;
758	}
759
760	#define XE_VMA_CREATE_MASK ( \
761	XE_VMA_READ_ONLY \| \
762	XE_VMA_DUMPABLE \| \
763	XE_VMA_SYSTEM_ALLOCATOR \| \
764	DRM_GPUVA_SPARSE \| \
765	XE_VMA_MADV_AUTORESET)
766
767	static void xe_vm_populate_rebind(struct xe_vma_op op, struct xe_vma vma,
768	u8 tile_mask)
769	{
770	INIT_LIST_HEAD(&op->link);
771	op->tile_mask = tile_mask;
772	op->base.op = DRM_GPUVA_OP_MAP;
773	op->base.map.va.addr = vma->gpuva.va.addr;
774	op->base.map.va.range = vma->gpuva.va.range;
775	op->base.map.gem.obj = vma->gpuva.gem.obj;
776	op->base.map.gem.offset = vma->gpuva.gem.offset;
777	op->map.vma = vma;
778	op->map.immediate = true;
779	op->map.vma_flags = vma->gpuva.flags & XE_VMA_CREATE_MASK;
780	}
781
782	static int xe_vm_ops_add_rebind(struct xe_vma_ops vops, struct xe_vma vma,
783	u8 tile_mask)
784	{
785	struct xe_vma_op *op;
786
787	op = kzalloc_obj(*op);
788	if (!op)
789	return -ENOMEM;
790
791	xe_vm_populate_rebind(op, vma, tile_mask);
792	list_add_tail(&op->link, &vops->list);
793	xe_vma_ops_incr_pt_update_ops(vops, tile_mask, 1);
794
795	return 0;
796	}
797
798	static struct dma_fence ops_execute(struct xe_vm vm,
799	struct xe_vma_ops *vops);
800	static void xe_vma_ops_init(struct xe_vma_ops vops, struct xe_vm vm,
801	struct xe_exec_queue *q,
802	struct xe_sync_entry *syncs, u32 num_syncs);
803
804	int xe_vm_rebind(struct xe_vm *vm, bool rebind_worker)
805	{
806	struct dma_fence *fence;
807	struct xe_vma vma, next;
808	struct xe_vma_ops vops;
809	struct xe_vma_op op, next_op;
810	int err, i;
811
812	lockdep_assert_held(&vm->lock);
813	if ((xe_vm_in_lr_mode(vm) && !rebind_worker) \|\|
814	list_empty(&vm->rebind_list))
815	return 0;
816
817	xe_vma_ops_init(&vops, vm, NULL, NULL, 0);
818	for (i = 0; i < XE_MAX_TILES_PER_DEVICE; ++i)
819	vops.pt_update_ops[i].wait_vm_bookkeep = true;
820
821	xe_vm_assert_held(vm);
822	list_for_each_entry(vma, &vm->rebind_list, combined_links.rebind) {
823	xe_assert(vm->xe, vma->tile_present);
824
825	if (rebind_worker)
826	trace_xe_vma_rebind_worker(vma);
827	else
828	trace_xe_vma_rebind_exec(vma);
829
830	err = xe_vm_ops_add_rebind(&vops, vma,
831	vma->tile_present);
832	if (err)
833	goto free_ops;
834	}
835
836	err = xe_vma_ops_alloc(&vops, false);
837	if (err)
838	goto free_ops;
839
840	fence = ops_execute(vm, &vops);
841	if (IS_ERR(fence)) {
842	err = PTR_ERR(fence);
843	} else {
844	dma_fence_put(fence);
845	list_for_each_entry_safe(vma, next, &vm->rebind_list,
846	combined_links.rebind)
847	list_del_init(&vma->combined_links.rebind);
848	}
849	free_ops:
850	list_for_each_entry_safe(op, next_op, &vops.list, link) {
851	list_del(&op->link);
852	kfree(op);
853	}
854	xe_vma_ops_fini(&vops);
855
856	return err;
857	}
858
859	struct dma_fence xe_vma_rebind(struct xe_vm vm, struct xe_vma *vma, u8 tile_mask)
860	{
861	struct dma_fence *fence = NULL;
862	struct xe_vma_ops vops;
863	struct xe_vma_op op, next_op;
864	struct xe_tile *tile;
865	u8 id;
866	int err;
867
868	lockdep_assert_held(&vm->lock);
869	xe_vm_assert_held(vm);
870	xe_assert(vm->xe, xe_vm_in_fault_mode(vm));
871
872	xe_vma_ops_init(&vops, vm, NULL, NULL, 0);
873	vops.flags \|= XE_VMA_OPS_FLAG_SKIP_TLB_WAIT;
874	for_each_tile(tile, vm->xe, id) {
875	vops.pt_update_ops[id].wait_vm_bookkeep = true;
876	vops.pt_update_ops[tile->id].q =
877	xe_migrate_exec_queue(tile->migrate);
878	}
879
880	err = xe_vm_ops_add_rebind(&vops, vma, tile_mask);
881	if (err)
882	return ERR_PTR(err);
883
884	err = xe_vma_ops_alloc(&vops, false);
885	if (err) {
886	fence = ERR_PTR(err);
887	goto free_ops;
888	}
889
890	fence = ops_execute(vm, &vops);
891
892	free_ops:
893	list_for_each_entry_safe(op, next_op, &vops.list, link) {
894	list_del(&op->link);
895	kfree(op);
896	}
897	xe_vma_ops_fini(&vops);
898
899	return fence;
900	}
901
902	static void xe_vm_populate_range_rebind(struct xe_vma_op *op,
903	struct xe_vma *vma,
904	struct xe_svm_range *range,
905	u8 tile_mask)
906	{
907	INIT_LIST_HEAD(&op->link);
908	op->tile_mask = tile_mask;
909	op->base.op = DRM_GPUVA_OP_DRIVER;
910	op->subop = XE_VMA_SUBOP_MAP_RANGE;
911	op->map_range.vma = vma;
912	op->map_range.range = range;
913	}
914
915	static int
916	xe_vm_ops_add_range_rebind(struct xe_vma_ops *vops,
917	struct xe_vma *vma,
918	struct xe_svm_range *range,
919	u8 tile_mask)
920	{
921	struct xe_vma_op *op;
922
923	op = kzalloc_obj(*op);
924	if (!op)
925	return -ENOMEM;
926
927	xe_vm_populate_range_rebind(op, vma, range, tile_mask);
928	list_add_tail(&op->link, &vops->list);
929	xe_vma_ops_incr_pt_update_ops(vops, tile_mask, 1);
930
931	return 0;
932	}
933
934	/**
935	* xe_vm_range_rebind() - VM range (re)bind
936	* @vm: The VM which the range belongs to.
937	* @vma: The VMA which the range belongs to.
938	* @range: SVM range to rebind.
939	* @tile_mask: Tile mask to bind the range to.
940	*
941	* (re)bind SVM range setting up GPU page tables for the range.
942	*
943	* Return: dma fence for rebind to signal completion on success, ERR_PTR on
944	* failure
945	*/
946	struct dma_fence xe_vm_range_rebind(struct xe_vm vm,
947	struct xe_vma *vma,
948	struct xe_svm_range *range,
949	u8 tile_mask)
950	{
951	struct dma_fence *fence = NULL;
952	struct xe_vma_ops vops;
953	struct xe_vma_op op, next_op;
954	struct xe_tile *tile;
955	u8 id;
956	int err;
957
958	lockdep_assert_held(&vm->lock);
959	xe_vm_assert_held(vm);
960	xe_assert(vm->xe, xe_vm_in_fault_mode(vm));
961	xe_assert(vm->xe, xe_vma_is_cpu_addr_mirror(vma));
962
963	xe_vma_ops_init(&vops, vm, NULL, NULL, 0);
964	vops.flags \|= XE_VMA_OPS_FLAG_SKIP_TLB_WAIT;
965	for_each_tile(tile, vm->xe, id) {
966	vops.pt_update_ops[id].wait_vm_bookkeep = true;
967	vops.pt_update_ops[tile->id].q =
968	xe_migrate_exec_queue(tile->migrate);
969	}
970
971	err = xe_vm_ops_add_range_rebind(&vops, vma, range, tile_mask);
972	if (err)
973	return ERR_PTR(err);
974
975	err = xe_vma_ops_alloc(&vops, false);
976	if (err) {
977	fence = ERR_PTR(err);
978	goto free_ops;
979	}
980
981	fence = ops_execute(vm, &vops);
982
983	free_ops:
984	list_for_each_entry_safe(op, next_op, &vops.list, link) {
985	list_del(&op->link);
986	kfree(op);
987	}
988	xe_vma_ops_fini(&vops);
989
990	return fence;
991	}
992
993	static void xe_vm_populate_range_unbind(struct xe_vma_op *op,
994	struct xe_svm_range *range)
995	{
996	INIT_LIST_HEAD(&op->link);
997	op->tile_mask = range->tile_present;
998	op->base.op = DRM_GPUVA_OP_DRIVER;
999	op->subop = XE_VMA_SUBOP_UNMAP_RANGE;
1000	op->unmap_range.range = range;
1001	}
1002
1003	static int
1004	xe_vm_ops_add_range_unbind(struct xe_vma_ops *vops,
1005	struct xe_svm_range *range)
1006	{
1007	struct xe_vma_op *op;
1008
1009	op = kzalloc_obj(*op);
1010	if (!op)
1011	return -ENOMEM;
1012
1013	xe_vm_populate_range_unbind(op, range);
1014	list_add_tail(&op->link, &vops->list);
1015	xe_vma_ops_incr_pt_update_ops(vops, range->tile_present, 1);
1016
1017	return 0;
1018	}
1019
1020	/**
1021	* xe_vm_range_unbind() - VM range unbind
1022	* @vm: The VM which the range belongs to.
1023	* @range: SVM range to rebind.
1024	*
1025	* Unbind SVM range removing the GPU page tables for the range.
1026	*
1027	* Return: dma fence for unbind to signal completion on success, ERR_PTR on
1028	* failure
1029	*/
1030	struct dma_fence xe_vm_range_unbind(struct xe_vm vm,
1031	struct xe_svm_range *range)
1032	{
1033	struct dma_fence *fence = NULL;
1034	struct xe_vma_ops vops;
1035	struct xe_vma_op op, next_op;
1036	struct xe_tile *tile;
1037	u8 id;
1038	int err;
1039
1040	lockdep_assert_held(&vm->lock);
1041	xe_vm_assert_held(vm);
1042	xe_assert(vm->xe, xe_vm_in_fault_mode(vm));
1043
1044	if (!range->tile_present)
1045	return dma_fence_get_stub();
1046
1047	xe_vma_ops_init(&vops, vm, NULL, NULL, 0);
1048	for_each_tile(tile, vm->xe, id) {
1049	vops.pt_update_ops[id].wait_vm_bookkeep = true;
1050	vops.pt_update_ops[tile->id].q =
1051	xe_migrate_exec_queue(tile->migrate);
1052	}
1053
1054	err = xe_vm_ops_add_range_unbind(&vops, range);
1055	if (err)
1056	return ERR_PTR(err);
1057
1058	err = xe_vma_ops_alloc(&vops, false);
1059	if (err) {
1060	fence = ERR_PTR(err);
1061	goto free_ops;
1062	}
1063
1064	fence = ops_execute(vm, &vops);
1065
1066	free_ops:
1067	list_for_each_entry_safe(op, next_op, &vops.list, link) {
1068	list_del(&op->link);
1069	kfree(op);
1070	}
1071	xe_vma_ops_fini(&vops);
1072
1073	return fence;
1074	}
1075
1076	static void xe_vma_mem_attr_fini(struct xe_vma_mem_attr *attr)
1077	{
1078	drm_pagemap_put(attr->preferred_loc.dpagemap);
1079	}
1080
1081	static void xe_vma_free(struct xe_vma *vma)
1082	{
1083	xe_vma_mem_attr_fini(&vma->attr);
1084
1085	if (xe_vma_is_userptr(vma))
1086	kfree(to_userptr_vma(vma));
1087	else
1088	kfree(vma);
1089	}
1090
1091	/**
1092	* xe_vma_mem_attr_copy() - copy an xe_vma_mem_attr structure.
1093	* @to: Destination.
1094	* @from: Source.
1095	*
1096	* Copies an xe_vma_mem_attr structure taking care to get reference
1097	* counting of individual members right.
1098	*/
1099	void xe_vma_mem_attr_copy(struct xe_vma_mem_attr to, struct xe_vma_mem_attr from)
1100	{
1101	xe_vma_mem_attr_fini(to);
1102	to = from;
1103	if (to->preferred_loc.dpagemap)
1104	drm_pagemap_get(to->preferred_loc.dpagemap);
1105	}
1106
1107	static struct xe_vma xe_vma_create(struct xe_vm vm,
1108	struct xe_bo *bo,
1109	u64 bo_offset_or_userptr,
1110	u64 start, u64 end,
1111	struct xe_vma_mem_attr *attr,
1112	unsigned int flags)
1113	{
1114	struct xe_vma *vma;
1115	struct xe_tile *tile;
1116	u8 id;
1117	bool is_null = (flags & DRM_GPUVA_SPARSE);
1118	bool is_cpu_addr_mirror = (flags & XE_VMA_SYSTEM_ALLOCATOR);
1119
1120	xe_assert(vm->xe, start < end);
1121	xe_assert(vm->xe, end < vm->size);
1122
1123	/*
1124	* Allocate and ensure that the xe_vma_is_userptr() return
1125	* matches what was allocated.
1126	*/
1127	if (!bo && !is_null && !is_cpu_addr_mirror) {
1128	struct xe_userptr_vma uvma = kzalloc_obj(uvma);
1129
1130	if (!uvma)
1131	return ERR_PTR(-ENOMEM);
1132
1133	vma = &uvma->vma;
1134	} else {
1135	vma = kzalloc_obj(*vma);
1136	if (!vma)
1137	return ERR_PTR(-ENOMEM);
1138
1139	if (bo)
1140	vma->gpuva.gem.obj = &bo->ttm.base;
1141	}
1142
1143	INIT_LIST_HEAD(&vma->combined_links.rebind);
1144
1145	INIT_LIST_HEAD(&vma->gpuva.gem.entry);
1146	vma->gpuva.vm = &vm->gpuvm;
1147	vma->gpuva.va.addr = start;
1148	vma->gpuva.va.range = end - start + 1;
1149	vma->gpuva.flags = flags;
1150
1151	for_each_tile(tile, vm->xe, id)
1152	vma->tile_mask \|= 0x1 << id;
1153
1154	if (vm->xe->info.has_atomic_enable_pte_bit)
1155	vma->gpuva.flags \|= XE_VMA_ATOMIC_PTE_BIT;
1156
1157	xe_vma_mem_attr_copy(&vma->attr, attr);
1158	if (bo) {
1159	struct drm_gpuvm_bo *vm_bo;
1160
1161	xe_bo_assert_held(bo);
1162
1163	/*
1164	* Reject only WILLNEED mappings on DONTNEED/PURGED BOs. This
1165	* gates new vm_bind ioctls (user supplies WILLNEED) while
1166	* still allowing partial-unbind / remap splits whose new VMAs
1167	* inherit the parent's DONTNEED attr. It must also run before
1168	* xe_bo_willneed_get_locked() below so a 0->1 holder bump
1169	* cannot silently promote DONTNEED back to WILLNEED.
1170	*/
1171	if (vma->attr.purgeable_state == XE_MADV_PURGEABLE_WILLNEED) {
1172	if (xe_bo_madv_is_dontneed(bo)) {
1173	xe_vma_free(vma);
1174	return ERR_PTR(-EBUSY);
1175	}
1176	if (xe_bo_is_purged(bo)) {
1177	xe_vma_free(vma);
1178	return ERR_PTR(-EINVAL);
1179	}
1180	}
1181
1182	vm_bo = drm_gpuvm_bo_obtain_locked(vma->gpuva.vm, &bo->ttm.base);
1183	if (IS_ERR(vm_bo)) {
1184	xe_vma_free(vma);
1185	return ERR_CAST(vm_bo);
1186	}
1187
1188	drm_gpuvm_bo_extobj_add(vm_bo);
1189	drm_gem_object_get(&bo->ttm.base);
1190	vma->gpuva.gem.offset = bo_offset_or_userptr;
1191	drm_gpuva_link(&vma->gpuva, vm_bo);
1192	drm_gpuvm_bo_put(vm_bo);
1193
1194	xe_bo_vma_count_inc_locked(bo);
1195	if (vma->attr.purgeable_state == XE_MADV_PURGEABLE_WILLNEED)
1196	xe_bo_willneed_get_locked(bo);
1197	} else /* userptr or null */ {
1198	if (!is_null && !is_cpu_addr_mirror) {
1199	struct xe_userptr_vma *uvma = to_userptr_vma(vma);
1200	u64 size = end - start + 1;
1201	int err;
1202
1203	vma->gpuva.gem.offset = bo_offset_or_userptr;
1204
1205	err = xe_userptr_setup(uvma, xe_vma_userptr(vma), size);
1206	if (err) {
1207	xe_vma_free(vma);
1208	return ERR_PTR(err);
1209	}
1210	}
1211
1212	xe_vm_get(vm);
1213	}
1214
1215	return vma;
1216	}
1217
1218	static void xe_vma_destroy_late(struct xe_vma *vma)
1219	{
1220	struct xe_vm *vm = xe_vma_vm(vma);
1221	struct xe_bo *bo = xe_vma_bo(vma);
1222
1223	if (vma->ufence) {
1224	xe_sync_ufence_put(vma->ufence);
1225	vma->ufence = NULL;
1226	}
1227
1228	if (xe_vma_is_userptr(vma)) {
1229	struct xe_userptr_vma *uvma = to_userptr_vma(vma);
1230
1231	xe_userptr_remove(uvma);
1232	xe_vm_put(vm);
1233	} else if (xe_vma_is_null(vma) \|\| xe_vma_is_cpu_addr_mirror(vma)) {
1234	xe_vm_put(vm);
1235	} else {
1236	xe_bo_put(bo);
1237	}
1238
1239	xe_vma_free(vma);
1240	}
1241
1242	static void vma_destroy_work_func(struct work_struct *w)
1243	{
1244	struct xe_vma *vma =
1245	container_of(w, struct xe_vma, destroy_work);
1246
1247	xe_vma_destroy_late(vma);
1248	}
1249
1250	static void vma_destroy_cb(struct dma_fence *fence,
1251	struct dma_fence_cb *cb)
1252	{
1253	struct xe_vma *vma = container_of(cb, struct xe_vma, destroy_cb);
1254
1255	INIT_WORK(&vma->destroy_work, vma_destroy_work_func);
1256	queue_work(system_dfl_wq, &vma->destroy_work);
1257	}
1258
1259	static void xe_vma_destroy(struct xe_vma vma, struct dma_fence fence)
1260	{
1261	struct xe_vm *vm = xe_vma_vm(vma);
1262	struct xe_bo *bo = xe_vma_bo(vma);
1263
1264	lockdep_assert_held_write(&vm->lock);
1265	xe_assert(vm->xe, list_empty(&vma->combined_links.destroy));
1266
1267	if (xe_vma_is_userptr(vma)) {
1268	xe_assert(vm->xe, vma->gpuva.flags & XE_VMA_DESTROYED);
1269	xe_userptr_destroy(to_userptr_vma(vma));
1270	} else if (!xe_vma_is_null(vma) && !xe_vma_is_cpu_addr_mirror(vma)) {
1271	xe_bo_assert_held(bo);
1272
1273	drm_gpuva_unlink(&vma->gpuva);
1274
1275	xe_bo_vma_count_dec_locked(bo);
1276	if (vma->attr.purgeable_state == XE_MADV_PURGEABLE_WILLNEED)
1277	xe_bo_willneed_put_locked(bo);
1278	}
1279
1280	xe_vm_assert_held(vm);
1281	if (fence) {
1282	int ret = dma_fence_add_callback(fence, &vma->destroy_cb,
1283	vma_destroy_cb);
1284
1285	if (ret) {
1286	XE_WARN_ON(ret != -ENOENT);
1287	xe_vma_destroy_late(vma);
1288	}
1289	} else {
1290	xe_vma_destroy_late(vma);
1291	}
1292	}
1293
1294	/**
1295	* xe_vm_lock_vma() - drm_exec utility to lock a vma
1296	* @exec: The drm_exec object we're currently locking for.
1297	* @vma: The vma for witch we want to lock the vm resv and any attached
1298	* object's resv.
1299	*
1300	* Return: 0 on success, negative error code on error. In particular
1301	* may return -EDEADLK on WW transaction contention and -EINTR if
1302	* an interruptible wait is terminated by a signal.
1303	*/
1304	int xe_vm_lock_vma(struct drm_exec exec, struct xe_vma vma)
1305	{
1306	struct xe_vm *vm = xe_vma_vm(vma);
1307	struct xe_bo *bo = xe_vma_bo(vma);
1308	int err;
1309
1310	XE_WARN_ON(!vm);
1311
1312	err = drm_exec_lock_obj(exec, xe_vm_obj(vm));
1313	if (!err && bo && !bo->vm)
1314	err = drm_exec_lock_obj(exec, &bo->ttm.base);
1315
1316	return err;
1317	}
1318
1319	static void xe_vma_destroy_unlocked(struct xe_vma *vma)
1320	{
1321	struct xe_device *xe = xe_vma_vm(vma)->xe;
1322	struct xe_validation_ctx ctx;
1323	struct drm_exec exec;
1324	int err = 0;
1325
1326	xe_validation_guard(&ctx, &xe->val, &exec, (struct xe_val_flags) {}, err) {
1327	err = xe_vm_lock_vma(&exec, vma);
1328	drm_exec_retry_on_contention(&exec);
1329	if (XE_WARN_ON(err))
1330	break;
1331	xe_vma_destroy(vma, NULL);
1332	}
1333	xe_assert(xe, !err);
1334	}
1335
1336	struct xe_vma *
1337	xe_vm_find_overlapping_vma(struct xe_vm *vm, u64 start, u64 range)
1338	{
1339	struct drm_gpuva *gpuva;
1340
1341	lockdep_assert_held(&vm->lock);
1342
1343	if (xe_vm_is_closed_or_banned(vm))
1344	return NULL;
1345
1346	xe_assert(vm->xe, start + range <= vm->size);
1347
1348	gpuva = drm_gpuva_find_first(&vm->gpuvm, start, range);
1349
1350	return gpuva ? gpuva_to_vma(gpuva) : NULL;
1351	}
1352
1353	static int xe_vm_insert_vma(struct xe_vm vm, struct xe_vma vma)
1354	{
1355	int err;
1356
1357	xe_assert(vm->xe, xe_vma_vm(vma) == vm);
1358	lockdep_assert_held(&vm->lock);
1359
1360	mutex_lock(&vm->snap_mutex);
1361	err = drm_gpuva_insert(&vm->gpuvm, &vma->gpuva);
1362	mutex_unlock(&vm->snap_mutex);
1363	XE_WARN_ON(err); /* Shouldn't be possible */
1364
1365	return err;
1366	}
1367
1368	static void xe_vm_remove_vma(struct xe_vm vm, struct xe_vma vma)
1369	{
1370	xe_assert(vm->xe, xe_vma_vm(vma) == vm);
1371	lockdep_assert_held(&vm->lock);
1372
1373	mutex_lock(&vm->snap_mutex);
1374	drm_gpuva_remove(&vma->gpuva);
1375	mutex_unlock(&vm->snap_mutex);
1376	if (vm->usm.last_fault_vma == vma)
1377	vm->usm.last_fault_vma = NULL;
1378	}
1379
1380	static struct drm_gpuva_op *xe_vm_op_alloc(void)
1381	{
1382	struct xe_vma_op *op;
1383
1384	op = kzalloc_obj(*op);
1385
1386	if (unlikely(!op))
1387	return NULL;
1388
1389	return &op->base;
1390	}
1391
1392	static void xe_vm_free(struct drm_gpuvm *gpuvm);
1393
1394	static const struct drm_gpuvm_ops gpuvm_ops = {
1395	.op_alloc = xe_vm_op_alloc,
1396	.vm_bo_validate = xe_gpuvm_validate,
1397	.vm_free = xe_vm_free,
1398	};
1399
1400	static u64 pde_encode_pat_index(u16 pat_index)
1401	{
1402	u64 pte = 0;
1403
1404	if (pat_index & BIT(0))
1405	pte \|= XE_PPGTT_PTE_PAT0;
1406
1407	if (pat_index & BIT(1))
1408	pte \|= XE_PPGTT_PTE_PAT1;
1409
1410	return pte;
1411	}
1412
1413	static u64 pte_encode_pat_index(u16 pat_index, u32 pt_level)
1414	{
1415	u64 pte = 0;
1416
1417	if (pat_index & BIT(0))
1418	pte \|= XE_PPGTT_PTE_PAT0;
1419
1420	if (pat_index & BIT(1))
1421	pte \|= XE_PPGTT_PTE_PAT1;
1422
1423	if (pat_index & BIT(2)) {
1424	if (pt_level)
1425	pte \|= XE_PPGTT_PDE_PDPE_PAT2;
1426	else
1427	pte \|= XE_PPGTT_PTE_PAT2;
1428	}
1429
1430	if (pat_index & BIT(3))
1431	pte \|= XELPG_PPGTT_PTE_PAT3;
1432
1433	if (pat_index & (BIT(4)))
1434	pte \|= XE2_PPGTT_PTE_PAT4;
1435
1436	return pte;
1437	}
1438
1439	static u64 pte_encode_ps(u32 pt_level)
1440	{
1441	XE_WARN_ON(pt_level > MAX_HUGEPTE_LEVEL);
1442
1443	if (pt_level == 1)
1444	return XE_PDE_PS_2M;
1445	else if (pt_level == 2)
1446	return XE_PDPE_PS_1G;
1447
1448	return 0;
1449	}
1450
1451	static u16 pde_pat_index(struct xe_bo *bo)
1452	{
1453	struct xe_device *xe = xe_bo_device(bo);
1454	u16 pat_index;
1455
1456	/*
1457	* We only have two bits to encode the PAT index in non-leaf nodes, but
1458	* these only point to other paging structures so we only need a minimal
1459	* selection of options. The user PAT index is only for encoding leaf
1460	* nodes, where we have use of more bits to do the encoding. The
1461	* non-leaf nodes are instead under driver control so the chosen index
1462	* here should be distinct from the user PAT index. Also the
1463	* corresponding coherency of the PAT index should be tied to the
1464	* allocation type of the page table (or at least we should pick
1465	* something which is always safe).
1466	*/
1467	if (!xe_bo_is_vram(bo) && bo->ttm.ttm->caching == ttm_cached)
1468	pat_index = xe_cache_pat_idx(xe, XE_CACHE_WB);
1469	else
1470	pat_index = xe_cache_pat_idx(xe, XE_CACHE_NONE);
1471
1472	xe_assert(xe, pat_index <= 3);
1473
1474	return pat_index;
1475	}
1476
1477	static u64 xelp_pde_encode_bo(struct xe_bo *bo, u64 bo_offset)
1478	{
1479	u64 pde;
1480
1481	pde = xe_bo_addr(bo, bo_offset, XE_PAGE_SIZE);
1482	pde \|= XE_PAGE_PRESENT \| XE_PAGE_RW;
1483	pde \|= pde_encode_pat_index(pde_pat_index(bo));
1484
1485	return pde;
1486	}
1487
1488	static u64 xelp_pte_encode_bo(struct xe_bo *bo, u64 bo_offset,
1489	u16 pat_index, u32 pt_level)
1490	{
1491	u64 pte;
1492
1493	pte = xe_bo_addr(bo, bo_offset, XE_PAGE_SIZE);
1494	pte \|= XE_PAGE_PRESENT \| XE_PAGE_RW;
1495	pte \|= pte_encode_pat_index(pat_index, pt_level);
1496	pte \|= pte_encode_ps(pt_level);
1497
1498	if (xe_bo_is_vram(bo) \|\| xe_bo_is_stolen_devmem(bo))
1499	pte \|= XE_PPGTT_PTE_DM;
1500
1501	return pte;
1502	}
1503
1504	static u64 xelp_pte_encode_vma(u64 pte, struct xe_vma *vma,
1505	u16 pat_index, u32 pt_level)
1506	{
1507	struct xe_bo *bo = xe_vma_bo(vma);
1508	struct xe_vm *vm = xe_vma_vm(vma);
1509
1510	pte \|= XE_PAGE_PRESENT;
1511
1512	if (likely(!xe_vma_read_only(vma)))
1513	pte \|= XE_PAGE_RW;
1514
1515	pte \|= pte_encode_pat_index(pat_index, pt_level);
1516	pte \|= pte_encode_ps(pt_level);
1517
1518	/*
1519	* NULL PTEs redirect to scratch page (return zeros on read).
1520	* Set for: 1) explicit null VMAs, 2) purged BOs on scratch VMs.
1521	* Never set NULL flag without scratch page - causes undefined behavior.
1522	*/
1523	if (unlikely(xe_vma_is_null(vma) \|\|
1524	(bo && xe_bo_is_purged(bo) && xe_vm_has_scratch(vm))))
1525	pte \|= XE_PTE_NULL;
1526
1527	return pte;
1528	}
1529
1530	static u64 xelp_pte_encode_addr(struct xe_device *xe, u64 addr,
1531	u16 pat_index,
1532	u32 pt_level, bool devmem, u64 flags)
1533	{
1534	u64 pte;
1535
1536	/* Avoid passing random bits directly as flags */
1537	xe_assert(xe, !(flags & ~XE_PTE_PS64));
1538
1539	pte = addr;
1540	pte \|= XE_PAGE_PRESENT \| XE_PAGE_RW;
1541	pte \|= pte_encode_pat_index(pat_index, pt_level);
1542	pte \|= pte_encode_ps(pt_level);
1543
1544	if (devmem)
1545	pte \|= XE_PPGTT_PTE_DM;
1546
1547	pte \|= flags;
1548
1549	return pte;
1550	}
1551
1552	static const struct xe_pt_ops xelp_pt_ops = {
1553	.pte_encode_bo = xelp_pte_encode_bo,
1554	.pte_encode_vma = xelp_pte_encode_vma,
1555	.pte_encode_addr = xelp_pte_encode_addr,
1556	.pde_encode_bo = xelp_pde_encode_bo,
1557	};
1558
1559	static void vm_destroy_work_func(struct work_struct *w);
1560
1561	/**
1562	* xe_vm_create_scratch() - Setup a scratch memory pagetable tree for the
1563	* given tile and vm.
1564	* @xe: xe device.
1565	* @tile: tile to set up for.
1566	* @vm: vm to set up for.
1567	* @exec: The struct drm_exec object used to lock the vm resv.
1568	*
1569	* Sets up a pagetable tree with one page-table per level and a single
1570	* leaf PTE. All pagetable entries point to the single page-table or,
1571	* for MAX_HUGEPTE_LEVEL, a NULL huge PTE returning 0 on read and
1572	* writes become NOPs.
1573	*
1574	* Return: 0 on success, negative error code on error.
1575	*/
1576	static int xe_vm_create_scratch(struct xe_device xe, struct xe_tile tile,
1577	struct xe_vm vm, struct drm_exec exec)
1578	{
1579	u8 id = tile->id;
1580	int i;
1581
1582	for (i = MAX_HUGEPTE_LEVEL; i < vm->pt_root[id]->level; i++) {
1583	vm->scratch_pt[id][i] = xe_pt_create(vm, tile, i, exec);
1584	if (IS_ERR(vm->scratch_pt[id][i])) {
1585	int err = PTR_ERR(vm->scratch_pt[id][i]);
1586
1587	vm->scratch_pt[id][i] = NULL;
1588	return err;
1589	}
1590	xe_pt_populate_empty(tile, vm, vm->scratch_pt[id][i]);
1591	}
1592
1593	return 0;
1594	}
1595	ALLOW_ERROR_INJECTION(xe_vm_create_scratch, ERRNO);
1596
1597	static void xe_vm_free_scratch(struct xe_vm *vm)
1598	{
1599	struct xe_tile *tile;
1600	u8 id;
1601
1602	if (!xe_vm_has_scratch(vm))
1603	return;
1604
1605	for_each_tile(tile, vm->xe, id) {
1606	u32 i;
1607
1608	if (!vm->pt_root[id])
1609	continue;
1610
1611	for (i = MAX_HUGEPTE_LEVEL; i < vm->pt_root[id]->level; ++i)
1612	if (vm->scratch_pt[id][i])
1613	xe_pt_destroy(vm->scratch_pt[id][i], vm->flags, NULL);
1614	}
1615	}
1616
1617	static void xe_vm_pt_destroy(struct xe_vm *vm)
1618	{
1619	struct xe_tile *tile;
1620	u8 id;
1621
1622	xe_vm_assert_held(vm);
1623
1624	for_each_tile(tile, vm->xe, id) {
1625	if (vm->pt_root[id]) {
1626	xe_pt_destroy(vm->pt_root[id], vm->flags, NULL);
1627	vm->pt_root[id] = NULL;
1628	}
1629	}
1630	}
1631
1632	static void xe_vm_init_prove_locking(struct xe_device xe, struct xe_vm vm)
1633	{
1634	if (!IS_ENABLED(CONFIG_PROVE_LOCKING))
1635	return;
1636
1637	fs_reclaim_acquire(GFP_KERNEL);
1638	might_lock(&vm->exec_queues.lock);
1639	fs_reclaim_release(GFP_KERNEL);
1640
1641	down_read(&vm->exec_queues.lock);
1642	might_lock(&xe_root_mmio_gt(xe)->uc.guc.ct.lock);
1643	up_read(&vm->exec_queues.lock);
1644	}
1645
1646	struct xe_vm xe_vm_create(struct xe_device xe, u32 flags, struct xe_file *xef)
1647	{
1648	struct drm_gem_object *vm_resv_obj;
1649	struct xe_validation_ctx ctx;
1650	struct drm_exec exec;
1651	struct xe_vm *vm;
1652	int err;
1653	struct xe_tile *tile;
1654	u8 id;
1655
1656	/*
1657	* Since the GSCCS is not user-accessible, we don't expect a GSC VM to
1658	* ever be in faulting mode.
1659	*/
1660	xe_assert(xe, !((flags & XE_VM_FLAG_GSC) && (flags & XE_VM_FLAG_FAULT_MODE)));
1661
1662	vm = kzalloc(sizeof(*vm), GFP_KERNEL);
1663	if (!vm)
1664	return ERR_PTR(-ENOMEM);
1665
1666	vm->xe = xe;
1667
1668	vm->size = 1ull << xe->info.va_bits;
1669	vm->flags = flags;
1670
1671	if (xef)
1672	vm->xef = xe_file_get(xef);
1673	/*
1674	* GSC VMs are kernel-owned, only used for PXP ops and can sometimes be
1675	* manipulated under the PXP mutex. However, the PXP mutex can be taken
1676	* under a user-VM lock when the PXP session is started at exec_queue
1677	* creation time. Those are different VMs and therefore there is no risk
1678	* of deadlock, but we need to tell lockdep that this is the case or it
1679	* will print a warning.
1680	*/
1681	if (flags & XE_VM_FLAG_GSC) {
1682	static struct lock_class_key gsc_vm_key;
1683
1684	__init_rwsem(&vm->lock, "gsc_vm", &gsc_vm_key);
1685	} else {
1686	init_rwsem(&vm->lock);
1687	}
1688	mutex_init(&vm->snap_mutex);
1689
1690	INIT_LIST_HEAD(&vm->rebind_list);
1691
1692	INIT_LIST_HEAD(&vm->userptr.repin_list);
1693	INIT_LIST_HEAD(&vm->userptr.invalidated);
1694	spin_lock_init(&vm->userptr.invalidated_lock);
1695
1696	INIT_LIST_HEAD(&vm->faults.list);
1697	spin_lock_init(&vm->faults.lock);
1698
1699	ttm_lru_bulk_move_init(&vm->lru_bulk_move);
1700
1701	INIT_WORK(&vm->destroy_work, vm_destroy_work_func);
1702
1703	INIT_LIST_HEAD(&vm->preempt.exec_queues);
1704	for (id = 0; id < XE_MAX_TILES_PER_DEVICE * XE_MAX_GT_PER_TILE; ++id)
1705	INIT_LIST_HEAD(&vm->exec_queues.list[id]);
1706	if (flags & XE_VM_FLAG_FAULT_MODE)
1707	vm->preempt.min_run_period_ms = xe->min_run_period_pf_ms;
1708	else
1709	vm->preempt.min_run_period_ms = xe->min_run_period_lr_ms;
1710
1711	init_rwsem(&vm->exec_queues.lock);
1712	xe_vm_init_prove_locking(xe, vm);
1713
1714	for_each_tile(tile, xe, id)
1715	xe_range_fence_tree_init(&vm->rftree[id]);
1716
1717	vm->pt_ops = &xelp_pt_ops;
1718
1719	/*
1720	* Long-running workloads are not protected by the scheduler references.
1721	* By design, run_job for long-running workloads returns NULL and the
1722	* scheduler drops all the references of it, hence protecting the VM
1723	* for this case is necessary.
1724	*/
1725	if (flags & XE_VM_FLAG_LR_MODE) {
1726	INIT_WORK(&vm->preempt.rebind_work, preempt_rebind_work_func);
1727	xe_pm_runtime_get_noresume(xe);
1728	INIT_LIST_HEAD(&vm->preempt.pm_activate_link);
1729	}
1730
1731	err = xe_svm_init(vm);
1732	if (err)
1733	goto err_no_resv;
1734
1735	vm_resv_obj = drm_gpuvm_resv_object_alloc(&xe->drm);
1736	if (!vm_resv_obj) {
1737	err = -ENOMEM;
1738	goto err_svm_fini;
1739	}
1740
1741	drm_gpuvm_init(&vm->gpuvm, "Xe VM", DRM_GPUVM_RESV_PROTECTED, &xe->drm,
1742	vm_resv_obj, 0, vm->size, 0, 0, &gpuvm_ops);
1743
1744	drm_gem_object_put(vm_resv_obj);
1745
1746	err = 0;
1747	xe_validation_guard(&ctx, &xe->val, &exec, (struct xe_val_flags) {.interruptible = true},
1748	err) {
1749	err = xe_vm_drm_exec_lock(vm, &exec);
1750	drm_exec_retry_on_contention(&exec);
1751
1752	if (IS_DGFX(xe) && xe->info.vram_flags & XE_VRAM_FLAGS_NEED64K)
1753	vm->flags \|= XE_VM_FLAG_64K;
1754
1755	for_each_tile(tile, xe, id) {
1756	if (flags & XE_VM_FLAG_MIGRATION &&
1757	tile->id != XE_VM_FLAG_TILE_ID(flags))
1758	continue;
1759
1760	vm->pt_root[id] = xe_pt_create(vm, tile, xe->info.vm_max_level,
1761	&exec);
1762	if (IS_ERR(vm->pt_root[id])) {
1763	err = PTR_ERR(vm->pt_root[id]);
1764	vm->pt_root[id] = NULL;
1765	xe_vm_pt_destroy(vm);
1766	drm_exec_retry_on_contention(&exec);
1767	xe_validation_retry_on_oom(&ctx, &err);
1768	break;
1769	}
1770	}
1771	if (err)
1772	break;
1773
1774	if (xe_vm_has_scratch(vm)) {
1775	for_each_tile(tile, xe, id) {
1776	if (!vm->pt_root[id])
1777	continue;
1778
1779	err = xe_vm_create_scratch(xe, tile, vm, &exec);
1780	if (err) {
1781	xe_vm_free_scratch(vm);
1782	xe_vm_pt_destroy(vm);
1783	drm_exec_retry_on_contention(&exec);
1784	xe_validation_retry_on_oom(&ctx, &err);
1785	break;
1786	}
1787	}
1788	if (err)
1789	break;
1790	vm->batch_invalidate_tlb = true;
1791	}
1792
1793	if (vm->flags & XE_VM_FLAG_LR_MODE)
1794	vm->batch_invalidate_tlb = false;
1795
1796	/* Fill pt_root after allocating scratch tables */
1797	for_each_tile(tile, xe, id) {
1798	if (!vm->pt_root[id])
1799	continue;
1800
1801	xe_pt_populate_empty(tile, vm, vm->pt_root[id]);
1802	}
1803	}
1804	if (err)
1805	goto err_close;
1806
1807	/* Kernel migration VM shouldn't have a circular loop.. */
1808	if (!(flags & XE_VM_FLAG_MIGRATION)) {
1809	for_each_tile(tile, xe, id) {
1810	struct xe_exec_queue *q;
1811	u32 create_flags = EXEC_QUEUE_FLAG_VM;
1812
1813	if (!vm->pt_root[id])
1814	continue;
1815
1816	if (!xef) /* Not from userspace */
1817	create_flags \|= EXEC_QUEUE_FLAG_KERNEL;
1818
1819	q = xe_exec_queue_create_bind(xe, tile, vm, create_flags, 0);
1820	if (IS_ERR(q)) {
1821	err = PTR_ERR(q);
1822	goto err_close;
1823	}
1824	vm->q[id] = q;
1825	}
1826	}
1827
1828	if (xef && xe->info.has_asid) {
1829	u32 asid;
1830
1831	down_write(&xe->usm.lock);
1832	err = xa_alloc_cyclic(&xe->usm.asid_to_vm, &asid, vm,
1833	XA_LIMIT(1, XE_MAX_ASID - 1),
1834	&xe->usm.next_asid, GFP_NOWAIT);
1835	up_write(&xe->usm.lock);
1836	if (err < 0)
1837	goto err_close;
1838
1839	vm->usm.asid = asid;
1840	}
1841
1842	trace_xe_vm_create(vm);
1843
1844	return vm;
1845
1846	err_close:
1847	xe_vm_close_and_put(vm);
1848	return ERR_PTR(err);
1849
1850	err_svm_fini:
1851	vm->size = 0; /* close the vm */
1852	if (flags & XE_VM_FLAG_FAULT_MODE)
1853	xe_svm_close(vm);
1854	xe_svm_fini(vm);
1855	err_no_resv:
1856	mutex_destroy(&vm->snap_mutex);
1857	for_each_tile(tile, xe, id)
1858	xe_range_fence_tree_fini(&vm->rftree[id]);
1859	ttm_lru_bulk_move_fini(&xe->ttm, &vm->lru_bulk_move);
1860	if (vm->xef)
1861	xe_file_put(vm->xef);
1862	kfree(vm);
1863	if (flags & XE_VM_FLAG_LR_MODE)
1864	xe_pm_runtime_put(xe);
1865	return ERR_PTR(err);
1866	}
1867
1868	static void xe_vm_close(struct xe_vm *vm)
1869	{
1870	struct xe_device *xe = vm->xe;
1871	bool bound;
1872	int idx;
1873
1874	bound = drm_dev_enter(&xe->drm, &idx);
1875
1876	down_write(&vm->lock);
1877	if (xe_vm_in_fault_mode(vm))
1878	xe_svm_notifier_lock(vm);
1879
1880	vm->size = 0;
1881
1882	if (!((vm->flags & XE_VM_FLAG_MIGRATION))) {
1883	struct xe_tile *tile;
1884	struct xe_gt *gt;
1885	u8 id;
1886
1887	/* Wait for pending binds */
1888	dma_resv_wait_timeout(xe_vm_resv(vm),
1889	DMA_RESV_USAGE_BOOKKEEP,
1890	false, MAX_SCHEDULE_TIMEOUT);
1891
1892	if (bound) {
1893	for_each_tile(tile, xe, id)
1894	if (vm->pt_root[id])
1895	xe_pt_clear(xe, vm->pt_root[id]);
1896
1897	for_each_gt(gt, xe, id)
1898	xe_tlb_inval_vm(&gt->tlb_inval, vm);
1899	}
1900	}
1901
1902	if (xe_vm_in_fault_mode(vm))
1903	xe_svm_notifier_unlock(vm);
1904	up_write(&vm->lock);
1905
1906	if (bound)
1907	drm_dev_exit(idx);
1908	}
1909
1910	void xe_vm_close_and_put(struct xe_vm *vm)
1911	{
1912	LIST_HEAD(contested);
1913	struct xe_device *xe = vm->xe;
1914	struct xe_tile *tile;
1915	struct xe_vma vma, next_vma;
1916	struct drm_gpuva gpuva, next;
1917	u8 id;
1918
1919	xe_assert(xe, !vm->preempt.num_exec_queues);
1920
1921	xe_vm_close(vm);
1922	if (xe_vm_in_preempt_fence_mode(vm)) {
1923	mutex_lock(&xe->rebind_resume_lock);
1924	list_del_init(&vm->preempt.pm_activate_link);
1925	mutex_unlock(&xe->rebind_resume_lock);
1926	flush_work(&vm->preempt.rebind_work);
1927	}
1928	if (xe_vm_in_fault_mode(vm))
1929	xe_svm_close(vm);
1930
1931	down_write(&vm->lock);
1932	for_each_tile(tile, xe, id) {
1933	if (vm->q[id]) {
1934	int i;
1935
1936	xe_exec_queue_last_fence_put(vm->q[id], vm);
1937	for_each_tlb_inval(i)
1938	xe_exec_queue_tlb_inval_last_fence_put(vm->q[id], vm, i);
1939	}
1940	}
1941	up_write(&vm->lock);
1942
1943	for_each_tile(tile, xe, id) {
1944	if (vm->q[id]) {
1945	xe_exec_queue_kill(vm->q[id]);
1946	xe_exec_queue_put(vm->q[id]);
1947	vm->q[id] = NULL;
1948	}
1949	}
1950
1951	down_write(&vm->lock);
1952	xe_vm_lock(vm, false);
1953	drm_gpuvm_for_each_va_safe(gpuva, next, &vm->gpuvm) {
1954	vma = gpuva_to_vma(gpuva);
1955
1956	if (xe_vma_has_no_bo(vma)) {
1957	xe_svm_notifier_lock(vm);
1958	vma->gpuva.flags \|= XE_VMA_DESTROYED;
1959	xe_svm_notifier_unlock(vm);
1960	}
1961
1962	xe_vm_remove_vma(vm, vma);
1963
1964	/* easy case, remove from VMA? */
1965	if (xe_vma_has_no_bo(vma) \|\| xe_vma_bo(vma)->vm) {
1966	list_del_init(&vma->combined_links.rebind);
1967	xe_vma_destroy(vma, NULL);
1968	continue;
1969	}
1970
1971	list_move_tail(&vma->combined_links.destroy, &contested);
1972	vma->gpuva.flags \|= XE_VMA_DESTROYED;
1973	}
1974
1975	xe_vm_unlock(vm);
1976
1977	/*
1978	* Unlink and destroy all contested external-BO VMAs before destroying
1979	* the page tables. Otherwise, concurrent eviction holding only bo->resv
1980	* can walk the BO's VMAs and attempt to invalidate/zap page tables that
1981	* have already been freed.
1982	*/
1983	list_for_each_entry_safe(vma, next_vma, &contested,
1984	combined_links.destroy) {
1985	list_del_init(&vma->combined_links.destroy);
1986	xe_vma_destroy_unlocked(vma);
1987	}
1988
1989	xe_vm_lock(vm, false);
1990	xe_vm_free_scratch(vm);
1991	xe_vm_pt_destroy(vm);
1992	xe_vm_unlock(vm);
1993
1994	xe_svm_fini(vm);
1995
1996	up_write(&vm->lock);
1997
1998	down_write(&xe->usm.lock);
1999	if (vm->usm.asid) {
2000	void *lookup;
2001
2002	xe_assert(xe, xe->info.has_asid);
2003	xe_assert(xe, !(vm->flags & XE_VM_FLAG_MIGRATION));
2004
2005	lookup = xa_erase(&xe->usm.asid_to_vm, vm->usm.asid);
2006	xe_assert(xe, lookup == vm);
2007	}
2008	up_write(&xe->usm.lock);
2009
2010	xe_vm_clear_fault_entries(vm);
2011
2012	for_each_tile(tile, xe, id)
2013	xe_range_fence_tree_fini(&vm->rftree[id]);
2014
2015	xe_vm_put(vm);
2016	}
2017
2018	static void vm_destroy_work_func(struct work_struct *w)
2019	{
2020	struct xe_vm *vm =
2021	container_of(w, struct xe_vm, destroy_work);
2022	struct xe_device *xe = vm->xe;
2023	struct xe_tile *tile;
2024	u8 id;
2025
2026	/* xe_vm_close_and_put was not called? */
2027	xe_assert(xe, !vm->size);
2028
2029	if (xe_vm_in_preempt_fence_mode(vm))
2030	flush_work(&vm->preempt.rebind_work);
2031
2032	mutex_destroy(&vm->snap_mutex);
2033
2034	if (vm->flags & XE_VM_FLAG_LR_MODE)
2035	xe_pm_runtime_put(xe);
2036
2037	for_each_tile(tile, xe, id)
2038	XE_WARN_ON(vm->pt_root[id]);
2039
2040	trace_xe_vm_free(vm);
2041
2042	ttm_lru_bulk_move_fini(&xe->ttm, &vm->lru_bulk_move);
2043
2044	if (vm->xef)
2045	xe_file_put(vm->xef);
2046
2047	kfree(vm);
2048	}
2049
2050	static void xe_vm_free(struct drm_gpuvm *gpuvm)
2051	{
2052	struct xe_vm *vm = container_of(gpuvm, struct xe_vm, gpuvm);
2053
2054	/* To destroy the VM we need to be able to sleep */
2055	queue_work(system_dfl_wq, &vm->destroy_work);
2056	}
2057
2058	struct xe_vm xe_vm_lookup(struct xe_file xef, u32 id)
2059	{
2060	struct xe_vm *vm;
2061
2062	mutex_lock(&xef->vm.lock);
2063	vm = xa_load(&xef->vm.xa, id);
2064	if (vm)
2065	xe_vm_get(vm);
2066	mutex_unlock(&xef->vm.lock);
2067
2068	return vm;
2069	}
2070
2071	u64 xe_vm_pdp4_descriptor(struct xe_vm vm, struct xe_tile tile)
2072	{
2073	return vm->pt_ops->pde_encode_bo(vm->pt_root[tile->id]->bo, 0);
2074	}
2075
2076	static struct xe_exec_queue *
2077	to_wait_exec_queue(struct xe_vm vm, struct xe_exec_queue q)
2078	{
2079	return q ? q : vm->q[0];
2080	}
2081
2082	static struct xe_user_fence *
2083	find_ufence_get(struct xe_sync_entry *syncs, u32 num_syncs)
2084	{
2085	unsigned int i;
2086
2087	for (i = 0; i < num_syncs; i++) {
2088	struct xe_sync_entry *e = &syncs[i];
2089
2090	if (xe_sync_is_ufence(e))
2091	return xe_sync_ufence_get(e);
2092	}
2093
2094	return NULL;
2095	}
2096
2097	#define ALL_DRM_XE_VM_CREATE_FLAGS (DRM_XE_VM_CREATE_FLAG_SCRATCH_PAGE \| \
2098	DRM_XE_VM_CREATE_FLAG_LR_MODE \| \
2099	DRM_XE_VM_CREATE_FLAG_FAULT_MODE \| \
2100	DRM_XE_VM_CREATE_FLAG_NO_VM_OVERCOMMIT)
2101
2102	int xe_vm_create_ioctl(struct drm_device dev, void data,
2103	struct drm_file *file)
2104	{
2105	struct xe_device *xe = to_xe_device(dev);
2106	struct xe_file *xef = to_xe_file(file);
2107	struct drm_xe_vm_create *args = data;
2108	struct xe_gt *wa_gt = xe_root_mmio_gt(xe);
2109	struct xe_vm *vm;
2110	u32 id;
2111	int err;
2112	u32 flags = 0;
2113
2114	if (XE_IOCTL_DBG(xe, args->extensions))
2115	return -EINVAL;
2116
2117	if (wa_gt && XE_GT_WA(wa_gt, 22014953428))
2118	args->flags \|= DRM_XE_VM_CREATE_FLAG_SCRATCH_PAGE;
2119
2120	if (XE_IOCTL_DBG(xe, args->flags & DRM_XE_VM_CREATE_FLAG_FAULT_MODE &&
2121	!xe->info.has_usm))
2122	return -EINVAL;
2123
2124	if (XE_IOCTL_DBG(xe, args->reserved[0] \|\| args->reserved[1]))
2125	return -EINVAL;
2126
2127	if (XE_IOCTL_DBG(xe, args->flags & ~ALL_DRM_XE_VM_CREATE_FLAGS))
2128	return -EINVAL;
2129
2130	if (XE_IOCTL_DBG(xe, args->flags & DRM_XE_VM_CREATE_FLAG_SCRATCH_PAGE &&
2131	args->flags & DRM_XE_VM_CREATE_FLAG_FAULT_MODE &&
2132	!xe->info.needs_scratch))
2133	return -EINVAL;
2134
2135	if (XE_IOCTL_DBG(xe, !(args->flags & DRM_XE_VM_CREATE_FLAG_LR_MODE) &&
2136	args->flags & DRM_XE_VM_CREATE_FLAG_FAULT_MODE))
2137	return -EINVAL;
2138
2139	if (XE_IOCTL_DBG(xe, !(args->flags & DRM_XE_VM_CREATE_FLAG_FAULT_MODE) &&
2140	args->flags & DRM_XE_VM_CREATE_FLAG_NO_VM_OVERCOMMIT))
2141	return -EINVAL;
2142
2143	if (args->flags & DRM_XE_VM_CREATE_FLAG_SCRATCH_PAGE)
2144	flags \|= XE_VM_FLAG_SCRATCH_PAGE;
2145	if (args->flags & DRM_XE_VM_CREATE_FLAG_LR_MODE)
2146	flags \|= XE_VM_FLAG_LR_MODE;
2147	if (args->flags & DRM_XE_VM_CREATE_FLAG_FAULT_MODE)
2148	flags \|= XE_VM_FLAG_FAULT_MODE;
2149	if (args->flags & DRM_XE_VM_CREATE_FLAG_NO_VM_OVERCOMMIT)
2150	flags \|= XE_VM_FLAG_NO_VM_OVERCOMMIT;
2151
2152	vm = xe_vm_create(xe, flags, xef);
2153	if (IS_ERR(vm))
2154	return PTR_ERR(vm);
2155
2156	#if IS_ENABLED(CONFIG_DRM_XE_DEBUG_MEM)
2157	/* Warning: Security issue - never enable by default */
2158	args->reserved[0] = xe_bo_main_addr(vm->pt_root[0]->bo, XE_PAGE_SIZE);
2159	#endif
2160
2161	/* user id alloc must always be last in ioctl to prevent UAF */
2162	err = xa_alloc(&xef->vm.xa, &id, vm, xa_limit_32b, GFP_KERNEL);
2163	if (err)
2164	goto err_close_and_put;
2165
2166	args->vm_id = id;
2167
2168	return 0;
2169
2170	err_close_and_put:
2171	xe_vm_close_and_put(vm);
2172
2173	return err;
2174	}
2175
2176	int xe_vm_destroy_ioctl(struct drm_device dev, void data,
2177	struct drm_file *file)
2178	{
2179	struct xe_device *xe = to_xe_device(dev);
2180	struct xe_file *xef = to_xe_file(file);
2181	struct drm_xe_vm_destroy *args = data;
2182	struct xe_vm *vm;
2183	int err = 0;
2184
2185	if (XE_IOCTL_DBG(xe, args->pad) \|\|
2186	XE_IOCTL_DBG(xe, args->reserved[0] \|\| args->reserved[1]))
2187	return -EINVAL;
2188
2189	mutex_lock(&xef->vm.lock);
2190	vm = xa_load(&xef->vm.xa, args->vm_id);
2191	if (XE_IOCTL_DBG(xe, !vm))
2192	err = -ENOENT;
2193	else if (XE_IOCTL_DBG(xe, vm->preempt.num_exec_queues))
2194	err = -EBUSY;
2195	else
2196	xa_erase(&xef->vm.xa, args->vm_id);
2197	mutex_unlock(&xef->vm.lock);
2198
2199	if (!err)
2200	xe_vm_close_and_put(vm);
2201
2202	return err;
2203	}
2204
2205	static int xe_vm_query_vmas(struct xe_vm *vm, u64 start, u64 end)
2206	{
2207	struct drm_gpuva *gpuva;
2208	u32 num_vmas = 0;
2209
2210	lockdep_assert_held(&vm->lock);
2211	drm_gpuvm_for_each_va_range(gpuva, &vm->gpuvm, start, end)
2212	num_vmas++;
2213
2214	return num_vmas;
2215	}
2216
2217	static int get_mem_attrs(struct xe_vm vm, u32 num_vmas, u64 start,
2218	u64 end, struct drm_xe_mem_range_attr *attrs)
2219	{
2220	struct drm_gpuva *gpuva;
2221	int i = 0;
2222
2223	lockdep_assert_held(&vm->lock);
2224
2225	drm_gpuvm_for_each_va_range(gpuva, &vm->gpuvm, start, end) {
2226	struct xe_vma *vma = gpuva_to_vma(gpuva);
2227
2228	if (i == *num_vmas)
2229	return -ENOSPC;
2230
2231	attrs[i].start = xe_vma_start(vma);
2232	attrs[i].end = xe_vma_end(vma);
2233	attrs[i].atomic.val = vma->attr.atomic_access;
2234	attrs[i].pat_index.val = vma->attr.pat_index;
2235	attrs[i].preferred_mem_loc.devmem_fd = vma->attr.preferred_loc.devmem_fd;
2236	attrs[i].preferred_mem_loc.migration_policy =
2237	vma->attr.preferred_loc.migration_policy;
2238
2239	i++;
2240	}
2241
2242	*num_vmas = i;
2243	return 0;
2244	}
2245
2246	int xe_vm_query_vmas_attrs_ioctl(struct drm_device dev, void data, struct drm_file *file)
2247	{
2248	struct xe_device *xe = to_xe_device(dev);
2249	struct xe_file *xef = to_xe_file(file);
2250	struct drm_xe_mem_range_attr *mem_attrs;
2251	struct drm_xe_vm_query_mem_range_attr *args = data;
2252	u64 __user *attrs_user = u64_to_user_ptr(args->vector_of_mem_attr);
2253	struct xe_vm *vm;
2254	int err = 0;
2255
2256	if (XE_IOCTL_DBG(xe,
2257	((args->num_mem_ranges == 0 &&
2258	(attrs_user \|\| args->sizeof_mem_range_attr != 0)) \|\|
2259	(args->num_mem_ranges > 0 &&
2260	(!attrs_user \|\|
2261	args->sizeof_mem_range_attr !=
2262	sizeof(struct drm_xe_mem_range_attr))))))
2263	return -EINVAL;
2264
2265	vm = xe_vm_lookup(xef, args->vm_id);
2266	if (XE_IOCTL_DBG(xe, !vm))
2267	return -EINVAL;
2268
2269	err = down_read_interruptible(&vm->lock);
2270	if (err)
2271	goto put_vm;
2272
2273	attrs_user = u64_to_user_ptr(args->vector_of_mem_attr);
2274
2275	if (args->num_mem_ranges == 0 && !attrs_user) {
2276	args->num_mem_ranges = xe_vm_query_vmas(vm, args->start, args->start + args->range);
2277	args->sizeof_mem_range_attr = sizeof(struct drm_xe_mem_range_attr);
2278	goto unlock_vm;
2279	}
2280
2281	mem_attrs = kvmalloc_array(args->num_mem_ranges, args->sizeof_mem_range_attr,
2282	GFP_KERNEL \| __GFP_ACCOUNT \|
2283	__GFP_RETRY_MAYFAIL \| __GFP_NOWARN);
2284	if (!mem_attrs) {
2285	err = args->num_mem_ranges > 1 ? -ENOBUFS : -ENOMEM;
2286	goto unlock_vm;
2287	}
2288
2289	memset(mem_attrs, 0, args->num_mem_ranges * args->sizeof_mem_range_attr);
2290	err = get_mem_attrs(vm, &args->num_mem_ranges, args->start,
2291	args->start + args->range, mem_attrs);
2292	if (err)
2293	goto free_mem_attrs;
2294
2295	err = copy_to_user(attrs_user, mem_attrs,
2296	args->sizeof_mem_range_attr * args->num_mem_ranges);
2297	if (err)
2298	err = -EFAULT;
2299
2300	free_mem_attrs:
2301	kvfree(mem_attrs);
2302	unlock_vm:
2303	up_read(&vm->lock);
2304	put_vm:
2305	xe_vm_put(vm);
2306	return err;
2307	}
2308
2309	static bool vma_matches(struct xe_vma *vma, u64 page_addr)
2310	{
2311	if (page_addr > xe_vma_end(vma) - 1 \|\|
2312	page_addr + SZ_4K - 1 < xe_vma_start(vma))
2313	return false;
2314
2315	return true;
2316	}
2317
2318	/**
2319	* xe_vm_find_vma_by_addr() - Find a VMA by its address
2320	*
2321	* @vm: the xe_vm the vma belongs to
2322	* @page_addr: address to look up
2323	*/
2324	struct xe_vma xe_vm_find_vma_by_addr(struct xe_vm vm, u64 page_addr)
2325	{
2326	struct xe_vma *vma = NULL;
2327
2328	if (vm->usm.last_fault_vma) { /* Fast lookup */
2329	if (vma_matches(vm->usm.last_fault_vma, page_addr))
2330	vma = vm->usm.last_fault_vma;
2331	}
2332	if (!vma)
2333	vma = xe_vm_find_overlapping_vma(vm, page_addr, SZ_4K);
2334
2335	return vma;
2336	}
2337
2338	static const u32 region_to_mem_type[] = {
2339	XE_PL_TT,
2340	XE_PL_VRAM0,
2341	XE_PL_VRAM1,
2342	};
2343
2344	static void prep_vma_destroy(struct xe_vm vm, struct xe_vma vma,
2345	bool post_commit)
2346	{
2347	xe_svm_notifier_lock(vm);
2348	vma->gpuva.flags \|= XE_VMA_DESTROYED;
2349	xe_svm_notifier_unlock(vm);
2350	if (post_commit)
2351	xe_vm_remove_vma(vm, vma);
2352	}
2353
2354	#undef ULL
2355	#define ULL unsigned long long
2356
2357	#if IS_ENABLED(CONFIG_DRM_XE_DEBUG_VM)
2358	static void print_op(struct xe_device xe, struct drm_gpuva_op op)
2359	{
2360	struct xe_vma *vma;
2361
2362	switch (op->op) {
2363	case DRM_GPUVA_OP_MAP:
2364	vm_dbg(&xe->drm, "MAP: addr=0x%016llx, range=0x%016llx",
2365	(ULL)op->map.va.addr, (ULL)op->map.va.range);
2366	break;
2367	case DRM_GPUVA_OP_REMAP:
2368	vma = gpuva_to_vma(op->remap.unmap->va);
2369	vm_dbg(&xe->drm, "REMAP:UNMAP: addr=0x%016llx, range=0x%016llx, keep=%d",
2370	(ULL)xe_vma_start(vma), (ULL)xe_vma_size(vma),
2371	op->remap.unmap->keep ? 1 : 0);
2372	if (op->remap.prev)
2373	vm_dbg(&xe->drm,
2374	"REMAP:PREV: addr=0x%016llx, range=0x%016llx",
2375	(ULL)op->remap.prev->va.addr,
2376	(ULL)op->remap.prev->va.range);
2377	if (op->remap.next)
2378	vm_dbg(&xe->drm,
2379	"REMAP:NEXT: addr=0x%016llx, range=0x%016llx",
2380	(ULL)op->remap.next->va.addr,
2381	(ULL)op->remap.next->va.range);
2382	break;
2383	case DRM_GPUVA_OP_UNMAP:
2384	vma = gpuva_to_vma(op->unmap.va);
2385	vm_dbg(&xe->drm, "UNMAP: addr=0x%016llx, range=0x%016llx, keep=%d",
2386	(ULL)xe_vma_start(vma), (ULL)xe_vma_size(vma),
2387	op->unmap.keep ? 1 : 0);
2388	break;
2389	case DRM_GPUVA_OP_PREFETCH:
2390	vma = gpuva_to_vma(op->prefetch.va);
2391	vm_dbg(&xe->drm, "PREFETCH: addr=0x%016llx, range=0x%016llx",
2392	(ULL)xe_vma_start(vma), (ULL)xe_vma_size(vma));
2393	break;
2394	default:
2395	drm_warn(&xe->drm, "NOT POSSIBLE\n");
2396	}
2397	}
2398	#else
2399	static void print_op(struct xe_device xe, struct drm_gpuva_op op)
2400	{
2401	}
2402	#endif
2403
2404	static bool __xe_vm_needs_clear_scratch_pages(struct xe_vm *vm, u32 bind_flags)
2405	{
2406	if (!xe_vm_in_fault_mode(vm))
2407	return false;
2408
2409	if (!xe_vm_has_scratch(vm))
2410	return false;
2411
2412	if (bind_flags & DRM_XE_VM_BIND_FLAG_IMMEDIATE)
2413	return false;
2414
2415	return true;
2416	}
2417
2418	static void xe_svm_prefetch_gpuva_ops_fini(struct drm_gpuva_ops *ops)
2419	{
2420	struct drm_gpuva_op *__op;
2421
2422	drm_gpuva_for_each_op(__op, ops) {
2423	struct xe_vma_op *op = gpuva_op_to_vma_op(__op);
2424
2425	xe_vma_svm_prefetch_op_fini(op);
2426	}
2427	}
2428
2429	/*
2430	* Create operations list from IOCTL arguments, setup operations fields so parse
2431	* and commit steps are decoupled from IOCTL arguments. This step can fail.
2432	*/
2433	static struct drm_gpuva_ops *
2434	vm_bind_ioctl_ops_create(struct xe_vm vm, struct xe_vma_ops vops,
2435	struct xe_bo *bo, u64 bo_offset_or_userptr,
2436	u64 addr, u64 range,
2437	u32 operation, u32 flags,
2438	u32 prefetch_region, u16 pat_index)
2439	{
2440	struct drm_gem_object *obj = bo ? &bo->ttm.base : NULL;
2441	struct drm_gpuva_ops *ops;
2442	struct drm_gpuva_op *__op;
2443	struct drm_gpuvm_bo *vm_bo;
2444	u64 range_start = addr;
2445	u64 range_end = addr + range;
2446	int err;
2447
2448	lockdep_assert_held_write(&vm->lock);
2449
2450	vm_dbg(&vm->xe->drm,
2451	"op=%d, addr=0x%016llx, range=0x%016llx, bo_offset_or_userptr=0x%016llx",
2452	operation, (ULL)addr, (ULL)range,
2453	(ULL)bo_offset_or_userptr);
2454
2455	switch (operation) {
2456	case DRM_XE_VM_BIND_OP_MAP:
2457	if (flags & DRM_XE_VM_BIND_FLAG_CPU_ADDR_MIRROR) {
2458	xe_vm_find_cpu_addr_mirror_vma_range(vm, &range_start, &range_end);
2459	vops->flags \|= XE_VMA_OPS_FLAG_ALLOW_SVM_UNMAP;
2460	}
2461
2462	fallthrough;
2463	case DRM_XE_VM_BIND_OP_MAP_USERPTR: {
2464	struct drm_gpuvm_map_req map_req = {
2465	.map.va.addr = range_start,
2466	.map.va.range = range_end - range_start,
2467	.map.gem.obj = obj,
2468	.map.gem.offset = bo_offset_or_userptr,
2469	};
2470
2471	ops = drm_gpuvm_sm_map_ops_create(&vm->gpuvm, &map_req);
2472	break;
2473	}
2474	case DRM_XE_VM_BIND_OP_UNMAP:
2475	ops = drm_gpuvm_sm_unmap_ops_create(&vm->gpuvm, addr, range);
2476	break;
2477	case DRM_XE_VM_BIND_OP_PREFETCH:
2478	ops = drm_gpuvm_prefetch_ops_create(&vm->gpuvm, addr, range);
2479	break;
2480	case DRM_XE_VM_BIND_OP_UNMAP_ALL:
2481	xe_assert(vm->xe, bo);
2482
2483	err = xe_bo_lock(bo, true);
2484	if (err)
2485	return ERR_PTR(err);
2486
2487	vm_bo = drm_gpuvm_bo_obtain_locked(&vm->gpuvm, obj);
2488	if (IS_ERR(vm_bo)) {
2489	xe_bo_unlock(bo);
2490	return ERR_CAST(vm_bo);
2491	}
2492
2493	ops = drm_gpuvm_bo_unmap_ops_create(vm_bo);
2494	drm_gpuvm_bo_put(vm_bo);
2495	xe_bo_unlock(bo);
2496	break;
2497	default:
2498	drm_warn(&vm->xe->drm, "NOT POSSIBLE\n");
2499	ops = ERR_PTR(-EINVAL);
2500	}
2501	if (IS_ERR(ops))
2502	return ops;
2503
2504	drm_gpuva_for_each_op(__op, ops) {
2505	struct xe_vma_op *op = gpuva_op_to_vma_op(__op);
2506
2507	if (__op->op == DRM_GPUVA_OP_MAP) {
2508	op->map.immediate =
2509	flags & DRM_XE_VM_BIND_FLAG_IMMEDIATE;
2510	if (flags & DRM_XE_VM_BIND_FLAG_READONLY)
2511	op->map.vma_flags \|= XE_VMA_READ_ONLY;
2512	if (flags & DRM_XE_VM_BIND_FLAG_NULL)
2513	op->map.vma_flags \|= DRM_GPUVA_SPARSE;
2514	if (flags & DRM_XE_VM_BIND_FLAG_CPU_ADDR_MIRROR)
2515	op->map.vma_flags \|= XE_VMA_SYSTEM_ALLOCATOR;
2516	if (flags & DRM_XE_VM_BIND_FLAG_DUMPABLE)
2517	op->map.vma_flags \|= XE_VMA_DUMPABLE;
2518	if (flags & DRM_XE_VM_BIND_FLAG_MADVISE_AUTORESET)
2519	op->map.vma_flags \|= XE_VMA_MADV_AUTORESET;
2520	op->map.request_decompress = flags & DRM_XE_VM_BIND_FLAG_DECOMPRESS;
2521	op->map.pat_index = pat_index;
2522	op->map.invalidate_on_bind =
2523	__xe_vm_needs_clear_scratch_pages(vm, flags);
2524	} else if (__op->op == DRM_GPUVA_OP_PREFETCH) {
2525	struct xe_vma *vma = gpuva_to_vma(op->base.prefetch.va);
2526	struct xe_tile *tile;
2527	struct xe_svm_range *svm_range;
2528	struct drm_gpusvm_ctx ctx = {};
2529	struct drm_pagemap *dpagemap = NULL;
2530	u8 id, tile_mask = 0;
2531	u32 i;
2532
2533	if (!xe_vma_is_cpu_addr_mirror(vma)) {
2534	op->prefetch.region = prefetch_region;
2535	break;
2536	}
2537
2538	ctx.read_only = xe_vma_read_only(vma);
2539	ctx.devmem_possible = IS_DGFX(vm->xe) &&
2540	IS_ENABLED(CONFIG_DRM_XE_PAGEMAP);
2541
2542	for_each_tile(tile, vm->xe, id)
2543	tile_mask \|= 0x1 << id;
2544
2545	xa_init_flags(&op->prefetch_range.range, XA_FLAGS_ALLOC);
2546	op->prefetch_range.ranges_count = 0;
2547
2548	if (prefetch_region == DRM_XE_CONSULT_MEM_ADVISE_PREF_LOC) {
2549	dpagemap = xe_vma_resolve_pagemap(vma,
2550	xe_device_get_root_tile(vm->xe));
2551	} else if (prefetch_region) {
2552	tile = &vm->xe->tiles[region_to_mem_type[prefetch_region] -
2553	XE_PL_VRAM0];
2554	dpagemap = xe_tile_local_pagemap(tile);
2555	}
2556
2557	op->prefetch_range.dpagemap = dpagemap;
2558	alloc_next_range:
2559	svm_range = xe_svm_range_find_or_insert(vm, addr, vma, &ctx);
2560
2561	if (PTR_ERR(svm_range) == -ENOENT) {
2562	u64 ret = xe_svm_find_vma_start(vm, addr, range_end, vma);
2563
2564	addr = ret == ULONG_MAX ? 0 : ret;
2565	if (addr)
2566	goto alloc_next_range;
2567	else
2568	goto print_op_label;
2569	}
2570
2571	if (IS_ERR(svm_range)) {
2572	err = PTR_ERR(svm_range);
2573	goto unwind_prefetch_ops;
2574	}
2575
2576	if (xe_svm_range_validate(vm, svm_range, tile_mask, dpagemap)) {
2577	xe_svm_range_debug(svm_range, "PREFETCH - RANGE IS VALID");
2578	goto check_next_range;
2579	}
2580
2581	err = xa_alloc(&op->prefetch_range.range,
2582	&i, svm_range, xa_limit_32b,
2583	GFP_KERNEL);
2584
2585	if (err)
2586	goto unwind_prefetch_ops;
2587
2588	op->prefetch_range.ranges_count++;
2589	vops->flags \|= XE_VMA_OPS_FLAG_HAS_SVM_PREFETCH;
2590	xe_svm_range_debug(svm_range, "PREFETCH - RANGE CREATED");
2591	check_next_range:
2592	if (range_end > xe_svm_range_end(svm_range) &&
2593	xe_svm_range_end(svm_range) < xe_vma_end(vma)) {
2594	addr = xe_svm_range_end(svm_range);
2595	goto alloc_next_range;
2596	}
2597	}
2598	print_op_label:
2599	print_op(vm->xe, __op);
2600	}
2601
2602	return ops;
2603
2604	unwind_prefetch_ops:
2605	xe_svm_prefetch_gpuva_ops_fini(ops);
2606	drm_gpuva_ops_free(&vm->gpuvm, ops);
2607	return ERR_PTR(err);
2608	}
2609
2610	ALLOW_ERROR_INJECTION(vm_bind_ioctl_ops_create, ERRNO);
2611
2612	static struct xe_vma new_vma(struct xe_vm vm, struct drm_gpuva_op_map *op,
2613	struct xe_vma_mem_attr *attr, unsigned int flags)
2614	{
2615	struct xe_bo *bo = op->gem.obj ? gem_to_xe_bo(op->gem.obj) : NULL;
2616	struct xe_validation_ctx ctx;
2617	struct drm_exec exec;
2618	struct xe_vma *vma;
2619	int err = 0;
2620
2621	lockdep_assert_held_write(&vm->lock);
2622
2623	if (bo) {
2624	err = 0;
2625	xe_validation_guard(&ctx, &vm->xe->val, &exec,
2626	(struct xe_val_flags) {.interruptible = true}, err) {
2627	if (!bo->vm) {
2628	err = drm_exec_lock_obj(&exec, xe_vm_obj(vm));
2629	drm_exec_retry_on_contention(&exec);
2630	}
2631	if (!err) {
2632	err = drm_exec_lock_obj(&exec, &bo->ttm.base);
2633	drm_exec_retry_on_contention(&exec);
2634	}
2635	if (err)
2636	return ERR_PTR(err);
2637
2638	vma = xe_vma_create(vm, bo, op->gem.offset,
2639	op->va.addr, op->va.addr +
2640	op->va.range - 1, attr, flags);
2641	if (IS_ERR(vma))
2642	return vma;
2643
2644	if (!bo->vm) {
2645	err = add_preempt_fences(vm, bo);
2646	if (err) {
2647	prep_vma_destroy(vm, vma, false);
2648	xe_vma_destroy(vma, NULL);
2649	}
2650	}
2651	}
2652	if (err)
2653	return ERR_PTR(err);
2654	} else {
2655	vma = xe_vma_create(vm, NULL, op->gem.offset,
2656	op->va.addr, op->va.addr +
2657	op->va.range - 1, attr, flags);
2658	if (IS_ERR(vma))
2659	return vma;
2660
2661	if (xe_vma_is_userptr(vma)) {
2662	err = xe_vma_userptr_pin_pages(to_userptr_vma(vma));
2663	/*
2664	* -EBUSY has dedicated meaning that a user fence
2665	* attached to the VMA is busy, in practice
2666	* xe_vma_userptr_pin_pages can only fail with -EBUSY if
2667	* we are low on memory so convert this to -ENOMEM.
2668	*/
2669	if (err == -EBUSY)
2670	err = -ENOMEM;
2671	}
2672	}
2673	if (err) {
2674	prep_vma_destroy(vm, vma, false);
2675	xe_vma_destroy_unlocked(vma);
2676	vma = ERR_PTR(err);
2677	}
2678
2679	return vma;
2680	}
2681
2682	static u64 xe_vma_max_pte_size(struct xe_vma *vma)
2683	{
2684	if (vma->gpuva.flags & XE_VMA_PTE_1G)
2685	return SZ_1G;
2686	else if (vma->gpuva.flags & (XE_VMA_PTE_2M \| XE_VMA_PTE_COMPACT))
2687	return SZ_2M;
2688	else if (vma->gpuva.flags & XE_VMA_PTE_64K)
2689	return SZ_64K;
2690	else if (vma->gpuva.flags & XE_VMA_PTE_4K)
2691	return SZ_4K;
2692
2693	return SZ_1G; /* Uninitialized, used max size */
2694	}
2695
2696	static void xe_vma_set_pte_size(struct xe_vma *vma, u64 size)
2697	{
2698	switch (size) {
2699	case SZ_1G:
2700	vma->gpuva.flags \|= XE_VMA_PTE_1G;
2701	break;
2702	case SZ_2M:
2703	vma->gpuva.flags \|= XE_VMA_PTE_2M;
2704	break;
2705	case SZ_64K:
2706	vma->gpuva.flags \|= XE_VMA_PTE_64K;
2707	break;
2708	case SZ_4K:
2709	vma->gpuva.flags \|= XE_VMA_PTE_4K;
2710	break;
2711	}
2712	}
2713
2714	static int xe_vma_op_commit(struct xe_vm vm, struct xe_vma_op op)
2715	{
2716	int err = 0;
2717
2718	lockdep_assert_held_write(&vm->lock);
2719
2720	switch (op->base.op) {
2721	case DRM_GPUVA_OP_MAP:
2722	err \|= xe_vm_insert_vma(vm, op->map.vma);
2723	if (!err)
2724	op->flags \|= XE_VMA_OP_COMMITTED;
2725	break;
2726	case DRM_GPUVA_OP_REMAP:
2727	{
2728	u8 tile_present =
2729	gpuva_to_vma(op->base.remap.unmap->va)->tile_present;
2730
2731	prep_vma_destroy(vm, gpuva_to_vma(op->base.remap.unmap->va),
2732	true);
2733	op->flags \|= XE_VMA_OP_COMMITTED;
2734
2735	if (op->remap.prev) {
2736	err \|= xe_vm_insert_vma(vm, op->remap.prev);
2737	if (!err)
2738	op->flags \|= XE_VMA_OP_PREV_COMMITTED;
2739	if (!err && op->remap.skip_prev) {
2740	op->remap.prev->tile_present =
2741	tile_present;
2742	}
2743	}
2744	if (op->remap.next) {
2745	err \|= xe_vm_insert_vma(vm, op->remap.next);
2746	if (!err)
2747	op->flags \|= XE_VMA_OP_NEXT_COMMITTED;
2748	if (!err && op->remap.skip_next) {
2749	op->remap.next->tile_present =
2750	tile_present;
2751	}
2752	}
2753
2754	/*
2755	* Adjust for partial unbind after removing VMA from VM. In case
2756	* of unwind we might need to undo this later.
2757	*/
2758	if (!err) {
2759	op->base.remap.unmap->va->va.addr = op->remap.start;
2760	op->base.remap.unmap->va->va.range = op->remap.range;
2761	}
2762	break;
2763	}
2764	case DRM_GPUVA_OP_UNMAP:
2765	prep_vma_destroy(vm, gpuva_to_vma(op->base.unmap.va), true);
2766	op->flags \|= XE_VMA_OP_COMMITTED;
2767	break;
2768	case DRM_GPUVA_OP_PREFETCH:
2769	op->flags \|= XE_VMA_OP_COMMITTED;
2770	break;
2771	default:
2772	drm_warn(&vm->xe->drm, "NOT POSSIBLE\n");
2773	}
2774
2775	return err;
2776	}
2777
2778	/**
2779	* xe_vma_has_default_mem_attrs - Check if a VMA has default memory attributes
2780	* @vma: Pointer to the xe_vma structure to check
2781	*
2782	* This function determines whether the given VMA (Virtual Memory Area)
2783	* has its memory attributes set to their default values. Specifically,
2784	* it checks the following conditions:
2785	*
2786	* - `atomic_access` is `DRM_XE_VMA_ATOMIC_UNDEFINED`
2787	* - `pat_index` is equal to `default_pat_index`
2788	* - `preferred_loc.devmem_fd` is `DRM_XE_PREFERRED_LOC_DEFAULT_DEVICE`
2789	* - `preferred_loc.migration_policy` is `DRM_XE_MIGRATE_ALL_PAGES`
2790	*
2791	* Return: true if all attributes are at their default values, false otherwise.
2792	*/
2793	bool xe_vma_has_default_mem_attrs(struct xe_vma *vma)
2794	{
2795	return (vma->attr.atomic_access == DRM_XE_ATOMIC_UNDEFINED &&
2796	vma->attr.pat_index == vma->attr.default_pat_index &&
2797	vma->attr.preferred_loc.devmem_fd == DRM_XE_PREFERRED_LOC_DEFAULT_DEVICE &&
2798	vma->attr.preferred_loc.migration_policy == DRM_XE_MIGRATE_ALL_PAGES);
2799	}
2800
2801	static int vm_bind_ioctl_ops_parse(struct xe_vm vm, struct drm_gpuva_ops ops,
2802	struct xe_vma_ops *vops)
2803	{
2804	struct xe_device *xe = vm->xe;
2805	struct drm_gpuva_op *__op;
2806	struct xe_tile *tile;
2807	u8 id, tile_mask = 0;
2808	int err = 0;
2809
2810	lockdep_assert_held_write(&vm->lock);
2811
2812	for_each_tile(tile, vm->xe, id)
2813	tile_mask \|= 0x1 << id;
2814
2815	drm_gpuva_for_each_op(__op, ops) {
2816	struct xe_vma_op *op = gpuva_op_to_vma_op(__op);
2817	struct xe_vma *vma;
2818	unsigned int flags = 0;
2819
2820	INIT_LIST_HEAD(&op->link);
2821	list_add_tail(&op->link, &vops->list);
2822	op->tile_mask = tile_mask;
2823
2824	switch (op->base.op) {
2825	case DRM_GPUVA_OP_MAP:
2826	{
2827	struct xe_vma_mem_attr default_attr = {
2828	.preferred_loc = {
2829	.devmem_fd = DRM_XE_PREFERRED_LOC_DEFAULT_DEVICE,
2830	.migration_policy = DRM_XE_MIGRATE_ALL_PAGES,
2831	},
2832	.atomic_access = DRM_XE_ATOMIC_UNDEFINED,
2833	.default_pat_index = op->map.pat_index,
2834	.pat_index = op->map.pat_index,
2835	.purgeable_state = XE_MADV_PURGEABLE_WILLNEED,
2836	};
2837
2838	flags \|= op->map.vma_flags & XE_VMA_CREATE_MASK;
2839
2840	vma = new_vma(vm, &op->base.map, &default_attr,
2841	flags);
2842	if (IS_ERR(vma))
2843	return PTR_ERR(vma);
2844
2845	op->map.vma = vma;
2846	if (((op->map.immediate \|\| !xe_vm_in_fault_mode(vm)) &&
2847	!(op->map.vma_flags & XE_VMA_SYSTEM_ALLOCATOR)) \|\|
2848	op->map.invalidate_on_bind)
2849	xe_vma_ops_incr_pt_update_ops(vops,
2850	op->tile_mask, 1);
2851	break;
2852	}
2853	case DRM_GPUVA_OP_REMAP:
2854	{
2855	struct xe_vma *old =
2856	gpuva_to_vma(op->base.remap.unmap->va);
2857	bool skip = xe_vma_is_cpu_addr_mirror(old);
2858	u64 start = xe_vma_start(old), end = xe_vma_end(old);
2859	int num_remap_ops = 0;
2860
2861	if (op->base.remap.prev)
2862	start = op->base.remap.prev->va.addr +
2863	op->base.remap.prev->va.range;
2864	if (op->base.remap.next)
2865	end = op->base.remap.next->va.addr;
2866
2867	if (xe_vma_is_cpu_addr_mirror(old) &&
2868	xe_svm_has_mapping(vm, start, end)) {
2869	if (vops->flags & XE_VMA_OPS_FLAG_MADVISE)
2870	xe_svm_unmap_address_range(vm, start, end);
2871	else
2872	return -EBUSY;
2873	}
2874
2875	op->remap.start = xe_vma_start(old);
2876	op->remap.range = xe_vma_size(old);
2877	op->remap.old_start = op->remap.start;
2878	op->remap.old_range = op->remap.range;
2879
2880	flags \|= op->base.remap.unmap->va->flags & XE_VMA_CREATE_MASK;
2881	if (op->base.remap.prev) {
2882	vma = new_vma(vm, op->base.remap.prev,
2883	&old->attr, flags);
2884	if (IS_ERR(vma))
2885	return PTR_ERR(vma);
2886
2887	op->remap.prev = vma;
2888
2889	/*
2890	* Userptr creates a new SG mapping so
2891	* we must also rebind.
2892	*/
2893	op->remap.skip_prev = skip \|\|
2894	(!xe_vma_is_userptr(old) &&
2895	IS_ALIGNED(xe_vma_end(vma),
2896	xe_vma_max_pte_size(old)));
2897	if (op->remap.skip_prev) {
2898	xe_vma_set_pte_size(vma, xe_vma_max_pte_size(old));
2899	op->remap.range -=
2900	xe_vma_end(vma) -
2901	xe_vma_start(old);
2902	op->remap.start = xe_vma_end(vma);
2903	vm_dbg(&xe->drm, "REMAP:SKIP_PREV: addr=0x%016llx, range=0x%016llx",
2904	(ULL)op->remap.start,
2905	(ULL)op->remap.range);
2906	} else {
2907	num_remap_ops++;
2908	}
2909	}
2910
2911	if (op->base.remap.next) {
2912	vma = new_vma(vm, op->base.remap.next,
2913	&old->attr, flags);
2914	if (IS_ERR(vma))
2915	return PTR_ERR(vma);
2916
2917	op->remap.next = vma;
2918
2919	/*
2920	* Userptr creates a new SG mapping so
2921	* we must also rebind.
2922	*/
2923	op->remap.skip_next = skip \|\|
2924	(!xe_vma_is_userptr(old) &&
2925	IS_ALIGNED(xe_vma_start(vma),
2926	xe_vma_max_pte_size(old)));
2927	if (op->remap.skip_next) {
2928	xe_vma_set_pte_size(vma, xe_vma_max_pte_size(old));
2929	op->remap.range -=
2930	xe_vma_end(old) -
2931	xe_vma_start(vma);
2932	vm_dbg(&xe->drm, "REMAP:SKIP_NEXT: addr=0x%016llx, range=0x%016llx",
2933	(ULL)op->remap.start,
2934	(ULL)op->remap.range);
2935	} else {
2936	num_remap_ops++;
2937	}
2938	}
2939	if (!skip)
2940	num_remap_ops++;
2941
2942	xe_vma_ops_incr_pt_update_ops(vops, op->tile_mask, num_remap_ops);
2943	break;
2944	}
2945	case DRM_GPUVA_OP_UNMAP:
2946	vma = gpuva_to_vma(op->base.unmap.va);
2947
2948	if (xe_vma_is_cpu_addr_mirror(vma) &&
2949	xe_svm_has_mapping(vm, xe_vma_start(vma),
2950	xe_vma_end(vma)) &&
2951	!(vops->flags & XE_VMA_OPS_FLAG_ALLOW_SVM_UNMAP))
2952	return -EBUSY;
2953
2954	if (!xe_vma_is_cpu_addr_mirror(vma))
2955	xe_vma_ops_incr_pt_update_ops(vops, op->tile_mask, 1);
2956	break;
2957	case DRM_GPUVA_OP_PREFETCH:
2958	vma = gpuva_to_vma(op->base.prefetch.va);
2959
2960	if (xe_vma_is_userptr(vma)) {
2961	err = xe_vma_userptr_pin_pages(to_userptr_vma(vma));
2962	if (err)
2963	return err;
2964	}
2965
2966	if (xe_vma_is_cpu_addr_mirror(vma))
2967	xe_vma_ops_incr_pt_update_ops(vops, op->tile_mask,
2968	op->prefetch_range.ranges_count);
2969	else
2970	xe_vma_ops_incr_pt_update_ops(vops, op->tile_mask, 1);
2971
2972	break;
2973	default:
2974	drm_warn(&vm->xe->drm, "NOT POSSIBLE\n");
2975	}
2976
2977	err = xe_vma_op_commit(vm, op);
2978	if (err)
2979	return err;
2980	}
2981
2982	return 0;
2983	}
2984
2985	static void xe_vma_op_unwind(struct xe_vm vm, struct xe_vma_op op,
2986	bool post_commit, bool prev_post_commit,
2987	bool next_post_commit)
2988	{
2989	lockdep_assert_held_write(&vm->lock);
2990
2991	switch (op->base.op) {
2992	case DRM_GPUVA_OP_MAP:
2993	if (op->map.vma) {
2994	prep_vma_destroy(vm, op->map.vma, post_commit);
2995	xe_vma_destroy_unlocked(op->map.vma);
2996	}
2997	break;
2998	case DRM_GPUVA_OP_UNMAP:
2999	{
3000	struct xe_vma *vma = gpuva_to_vma(op->base.unmap.va);
3001
3002	if (vma) {
3003	xe_svm_notifier_lock(vm);
3004	vma->gpuva.flags &= ~XE_VMA_DESTROYED;
3005	xe_svm_notifier_unlock(vm);
3006	if (post_commit)
3007	xe_vm_insert_vma(vm, vma);
3008	}
3009	break;
3010	}
3011	case DRM_GPUVA_OP_REMAP:
3012	{
3013	struct xe_vma *vma = gpuva_to_vma(op->base.remap.unmap->va);
3014
3015	if (op->remap.prev) {
3016	prep_vma_destroy(vm, op->remap.prev, prev_post_commit);
3017	xe_vma_destroy_unlocked(op->remap.prev);
3018	}
3019	if (op->remap.next) {
3020	prep_vma_destroy(vm, op->remap.next, next_post_commit);
3021	xe_vma_destroy_unlocked(op->remap.next);
3022	}
3023	if (vma) {
3024	xe_svm_notifier_lock(vm);
3025	vma->gpuva.flags &= ~XE_VMA_DESTROYED;
3026	xe_svm_notifier_unlock(vm);
3027	if (post_commit) {
3028	/*
3029	* Restore the old va range, in case of the
3030	* prev/next skip optimisation. Otherwise what
3031	* we re-insert here could be smaller than the
3032	* original range.
3033	*/
3034	op->base.remap.unmap->va->va.addr =
3035	op->remap.old_start;
3036	op->base.remap.unmap->va->va.range =
3037	op->remap.old_range;
3038	xe_vm_insert_vma(vm, vma);
3039	}
3040	}
3041	break;
3042	}
3043	case DRM_GPUVA_OP_PREFETCH:
3044	/* Nothing to do */
3045	break;
3046	default:
3047	drm_warn(&vm->xe->drm, "NOT POSSIBLE\n");
3048	}
3049	}
3050
3051	static void vm_bind_ioctl_ops_unwind(struct xe_vm *vm,
3052	struct drm_gpuva_ops **ops,
3053	int num_ops_list)
3054	{
3055	int i;
3056
3057	for (i = num_ops_list - 1; i >= 0; --i) {
3058	struct drm_gpuva_ops *__ops = ops[i];
3059	struct drm_gpuva_op *__op;
3060
3061	if (!__ops)
3062	continue;
3063
3064	drm_gpuva_for_each_op_reverse(__op, __ops) {
3065	struct xe_vma_op *op = gpuva_op_to_vma_op(__op);
3066
3067	xe_vma_op_unwind(vm, op,
3068	op->flags & XE_VMA_OP_COMMITTED,
3069	op->flags & XE_VMA_OP_PREV_COMMITTED,
3070	op->flags & XE_VMA_OP_NEXT_COMMITTED);
3071	}
3072	}
3073	}
3074
3075	/**
3076	* struct xe_vma_lock_and_validate_flags - Flags for vma_lock_and_validate()
3077	* @res_evict: Allow evicting resources during validation
3078	* @validate: Perform BO validation
3079	* @request_decompress: Request BO decompression
3080	* @check_purged: Reject operation if BO is DONTNEED or PURGED
3081	*/
3082	struct xe_vma_lock_and_validate_flags {
3083	u32 res_evict : 1;
3084	u32 validate : 1;
3085	u32 request_decompress : 1;
3086	u32 check_purged : 1;
3087	};
3088
3089	static int vma_lock_and_validate(struct drm_exec exec, struct xe_vma vma,
3090	struct xe_vma_lock_and_validate_flags flags)
3091	{
3092	struct xe_bo *bo = xe_vma_bo(vma);
3093	struct xe_vm *vm = xe_vma_vm(vma);
3094	bool validate_bo = flags.validate;
3095	int err = 0;
3096
3097	if (bo) {
3098	if (!bo->vm)
3099	err = drm_exec_lock_obj(exec, &bo->ttm.base);
3100
3101	/* Reject new mappings to DONTNEED/purged BOs; allow cleanup operations */
3102	if (!err && flags.check_purged) {
3103	if (xe_bo_madv_is_dontneed(bo))
3104	err = -EBUSY; /* BO marked purgeable */
3105	else if (xe_bo_is_purged(bo))
3106	err = -EINVAL; /* BO already purged */
3107	}
3108
3109	/* Don't validate the BO for DONTNEED/PURGED remap remnants. */
3110	if (vma->attr.purgeable_state != XE_MADV_PURGEABLE_WILLNEED)
3111	validate_bo = false;
3112
3113	if (!err && validate_bo)
3114	err = xe_bo_validate(bo, vm,
3115	xe_vm_allow_vm_eviction(vm) &&
3116	flags.res_evict, exec);
3117
3118	if (err)
3119	return err;
3120
3121	if (flags.request_decompress)
3122	err = xe_bo_decompress(bo);
3123	}
3124
3125	return err;
3126	}
3127
3128	static int check_ufence(struct xe_vma *vma)
3129	{
3130	if (vma->ufence) {
3131	struct xe_user_fence * const f = vma->ufence;
3132
3133	if (!xe_sync_ufence_get_status(f))
3134	return -EBUSY;
3135
3136	vma->ufence = NULL;
3137	xe_sync_ufence_put(f);
3138	}
3139
3140	return 0;
3141	}
3142
3143	static int prefetch_ranges(struct xe_vm vm, struct xe_vma_op op)
3144	{
3145	bool devmem_possible = IS_DGFX(vm->xe) && IS_ENABLED(CONFIG_DRM_XE_PAGEMAP);
3146	struct xe_vma *vma = gpuva_to_vma(op->base.prefetch.va);
3147	struct drm_pagemap *dpagemap = op->prefetch_range.dpagemap;
3148	int err = 0;
3149
3150	struct xe_svm_range *svm_range;
3151	struct drm_gpusvm_ctx ctx = {};
3152	unsigned long i;
3153
3154	if (!xe_vma_is_cpu_addr_mirror(vma))
3155	return 0;
3156
3157	ctx.read_only = xe_vma_read_only(vma);
3158	ctx.devmem_possible = devmem_possible;
3159	ctx.check_pages_threshold = devmem_possible ? SZ_64K : 0;
3160	ctx.device_private_page_owner = xe_svm_private_page_owner(vm, !dpagemap);
3161
3162	/* TODO: Threading the migration */
3163	xa_for_each(&op->prefetch_range.range, i, svm_range) {
3164	if (!dpagemap)
3165	xe_svm_range_migrate_to_smem(vm, svm_range);
3166
3167	if (IS_ENABLED(CONFIG_DRM_XE_DEBUG_VM)) {
3168	drm_dbg(&vm->xe->drm,
3169	"Prefetch pagemap is %s start 0x%016lx end 0x%016lx\n",
3170	dpagemap ? dpagemap->drm->unique : "system",
3171	xe_svm_range_start(svm_range), xe_svm_range_end(svm_range));
3172	}
3173
3174	if (xe_svm_range_needs_migrate_to_vram(svm_range, vma, dpagemap)) {
3175	err = xe_svm_alloc_vram(svm_range, &ctx, dpagemap);
3176	if (err) {
3177	drm_dbg(&vm->xe->drm, "VRAM allocation failed, retry from userspace, asid=%u, gpusvm=%p, errno=%pe\n",
3178	vm->usm.asid, &vm->svm.gpusvm, ERR_PTR(err));
3179	return -ENODATA;
3180	}
3181	xe_svm_range_debug(svm_range, "PREFETCH - RANGE MIGRATED TO VRAM");
3182	}
3183
3184	err = xe_svm_range_get_pages(vm, svm_range, &ctx);
3185	if (err) {
3186	drm_dbg(&vm->xe->drm, "Get pages failed, asid=%u, gpusvm=%p, errno=%pe\n",
3187	vm->usm.asid, &vm->svm.gpusvm, ERR_PTR(err));
3188	if (err == -EOPNOTSUPP \|\| err == -EFAULT \|\| err == -EPERM)
3189	err = -ENODATA;
3190	return err;
3191	}
3192	xe_svm_range_debug(svm_range, "PREFETCH - RANGE GET PAGES DONE");
3193	}
3194
3195	return err;
3196	}
3197
3198	static int op_lock_and_prep(struct drm_exec exec, struct xe_vm vm,
3199	struct xe_vma_ops vops, struct xe_vma_op op)
3200	{
3201	int err = 0;
3202	bool res_evict;
3203
3204	/*
3205	* We only allow evicting a BO within the VM if it is not part of an
3206	* array of binds, as an array of binds can evict another BO within the
3207	* bind.
3208	*/
3209	res_evict = !(vops->flags & XE_VMA_OPS_ARRAY_OF_BINDS);
3210
3211	switch (op->base.op) {
3212	case DRM_GPUVA_OP_MAP:
3213	if (!op->map.invalidate_on_bind)
3214	err = vma_lock_and_validate(exec, op->map.vma,
3215	(struct xe_vma_lock_and_validate_flags) {
3216	.res_evict = res_evict,
3217	.validate = !xe_vm_in_fault_mode(vm) \|\|
3218	op->map.immediate,
3219	.request_decompress =
3220	op->map.request_decompress,
3221	.check_purged = false,
3222	});
3223	break;
3224	case DRM_GPUVA_OP_REMAP:
3225	err = check_ufence(gpuva_to_vma(op->base.remap.unmap->va));
3226	if (err)
3227	break;
3228
3229	err = vma_lock_and_validate(exec,
3230	gpuva_to_vma(op->base.remap.unmap->va),
3231	(struct xe_vma_lock_and_validate_flags) {
3232	.res_evict = res_evict,
3233	.validate = false,
3234	.request_decompress = false,
3235	.check_purged = false,
3236	});
3237	if (!err && op->remap.prev)
3238	err = vma_lock_and_validate(exec, op->remap.prev,
3239	(struct xe_vma_lock_and_validate_flags) {
3240	.res_evict = res_evict,
3241	.validate = true,
3242	.request_decompress = false,
3243	.check_purged = false,
3244	});
3245	if (!err && op->remap.next)
3246	err = vma_lock_and_validate(exec, op->remap.next,
3247	(struct xe_vma_lock_and_validate_flags) {
3248	.res_evict = res_evict,
3249	.validate = true,
3250	.request_decompress = false,
3251	.check_purged = false,
3252	});
3253	break;
3254	case DRM_GPUVA_OP_UNMAP:
3255	err = check_ufence(gpuva_to_vma(op->base.unmap.va));
3256	if (err)
3257	break;
3258
3259	err = vma_lock_and_validate(exec,
3260	gpuva_to_vma(op->base.unmap.va),
3261	(struct xe_vma_lock_and_validate_flags) {
3262	.res_evict = res_evict,
3263	.validate = false,
3264	.request_decompress = false,
3265	.check_purged = false,
3266	});
3267	break;
3268	case DRM_GPUVA_OP_PREFETCH:
3269	{
3270	struct xe_vma *vma = gpuva_to_vma(op->base.prefetch.va);
3271	u32 region;
3272
3273	if (!xe_vma_is_cpu_addr_mirror(vma)) {
3274	region = op->prefetch.region;
3275	xe_assert(vm->xe, region == DRM_XE_CONSULT_MEM_ADVISE_PREF_LOC \|\|
3276	region <= ARRAY_SIZE(region_to_mem_type));
3277	}
3278
3279	/*
3280	* PREFETCH is the only op that still gates on BO purge state.
3281	* MAP/REMAP handle this inside xe_vma_create() so partial
3282	* unbind on a DONTNEED BO still works. PREFETCH skips
3283	* xe_vma_create() and would migrate a BO with no backing
3284	* store, so reject DONTNEED/PURGED here.
3285	*/
3286	err = vma_lock_and_validate(exec,
3287	gpuva_to_vma(op->base.prefetch.va),
3288	(struct xe_vma_lock_and_validate_flags) {
3289	.res_evict = res_evict,
3290	.validate = false,
3291	.request_decompress = false,
3292	.check_purged = true,
3293	});
3294	if (!err && !xe_vma_has_no_bo(vma)) {
3295	struct xe_bo *bo = xe_vma_bo(vma);
3296	u32 mem_type;
3297
3298	if (region == DRM_XE_CONSULT_MEM_ADVISE_PREF_LOC) {
3299	unsigned int i;
3300
3301	mem_type = XE_PL_TT;
3302	for (i = 0; i < bo->placement.num_placement; i++) {
3303	if (mem_type_is_vram(bo->placements[i].mem_type)) {
3304	mem_type = bo->placements[i].mem_type;
3305	break;
3306	}
3307	}
3308	} else {
3309	mem_type = region_to_mem_type[region];
3310	}
3311
3312	err = xe_bo_migrate(bo, mem_type, NULL, exec);
3313	}
3314	break;
3315	}
3316	default:
3317	drm_warn(&vm->xe->drm, "NOT POSSIBLE\n");
3318	}
3319
3320	return err;
3321	}
3322
3323	static int vm_bind_ioctl_ops_prefetch_ranges(struct xe_vm vm, struct xe_vma_ops vops)
3324	{
3325	struct xe_vma_op *op;
3326	int err;
3327
3328	if (!(vops->flags & XE_VMA_OPS_FLAG_HAS_SVM_PREFETCH))
3329	return 0;
3330
3331	list_for_each_entry(op, &vops->list, link) {
3332	if (op->base.op == DRM_GPUVA_OP_PREFETCH) {
3333	err = prefetch_ranges(vm, op);
3334	if (err)
3335	return err;
3336	}
3337	}
3338
3339	return 0;
3340	}
3341
3342	static int vm_bind_ioctl_ops_lock_and_prep(struct drm_exec *exec,
3343	struct xe_vm *vm,
3344	struct xe_vma_ops *vops)
3345	{
3346	struct xe_vma_op *op;
3347	int err;
3348
3349	err = drm_exec_lock_obj(exec, xe_vm_obj(vm));
3350	if (err)
3351	return err;
3352
3353	list_for_each_entry(op, &vops->list, link) {
3354	err = op_lock_and_prep(exec, vm, vops, op);
3355	if (err)
3356	return err;
3357	}
3358
3359	#ifdef TEST_VM_OPS_ERROR
3360	if (vops->inject_error &&
3361	vm->xe->vm_inject_error_position == FORCE_OP_ERROR_LOCK)
3362	return -ENOSPC;
3363	#endif
3364
3365	return 0;
3366	}
3367
3368	static void op_trace(struct xe_vma_op *op)
3369	{
3370	switch (op->base.op) {
3371	case DRM_GPUVA_OP_MAP:
3372	trace_xe_vma_bind(op->map.vma);
3373	break;
3374	case DRM_GPUVA_OP_REMAP:
3375	trace_xe_vma_unbind(gpuva_to_vma(op->base.remap.unmap->va));
3376	if (op->remap.prev)
3377	trace_xe_vma_bind(op->remap.prev);
3378	if (op->remap.next)
3379	trace_xe_vma_bind(op->remap.next);
3380	break;
3381	case DRM_GPUVA_OP_UNMAP:
3382	trace_xe_vma_unbind(gpuva_to_vma(op->base.unmap.va));
3383	break;
3384	case DRM_GPUVA_OP_PREFETCH:
3385	trace_xe_vma_bind(gpuva_to_vma(op->base.prefetch.va));
3386	break;
3387	case DRM_GPUVA_OP_DRIVER:
3388	break;
3389	default:
3390	XE_WARN_ON("NOT POSSIBLE");
3391	}
3392	}
3393
3394	static void trace_xe_vm_ops_execute(struct xe_vma_ops *vops)
3395	{
3396	struct xe_vma_op *op;
3397
3398	list_for_each_entry(op, &vops->list, link)
3399	op_trace(op);
3400	}
3401
3402	static int vm_ops_setup_tile_args(struct xe_vm vm, struct xe_vma_ops vops)
3403	{
3404	struct xe_exec_queue *q = vops->q;
3405	struct xe_tile *tile;
3406	int number_tiles = 0;
3407	u8 id;
3408
3409	for_each_tile(tile, vm->xe, id) {
3410	if (vops->pt_update_ops[id].num_ops)
3411	++number_tiles;
3412
3413	if (vops->pt_update_ops[id].q)
3414	continue;
3415
3416	if (q) {
3417	vops->pt_update_ops[id].q = q;
3418	if (vm->pt_root[id] && !list_empty(&q->multi_gt_list))
3419	q = list_next_entry(q, multi_gt_list);
3420	} else {
3421	vops->pt_update_ops[id].q = vm->q[id];
3422	}
3423	}
3424
3425	return number_tiles;
3426	}
3427
3428	static struct dma_fence ops_execute(struct xe_vm vm,
3429	struct xe_vma_ops *vops)
3430	{
3431	struct xe_tile *tile;
3432	struct dma_fence *fence = NULL;
3433	struct dma_fence **fences = NULL;
3434	struct dma_fence_array *cf = NULL;
3435	int number_tiles = 0, current_fence = 0, n_fence = 0, err, i;
3436	u8 id;
3437
3438	number_tiles = vm_ops_setup_tile_args(vm, vops);
3439	if (number_tiles == 0)
3440	return ERR_PTR(-ENODATA);
3441
3442	for_each_tile(tile, vm->xe, id) {
3443	++n_fence;
3444
3445	if (!(vops->flags & XE_VMA_OPS_FLAG_SKIP_TLB_WAIT))
3446	for_each_tlb_inval(i)
3447	++n_fence;
3448	}
3449
3450	fences = kmalloc_objs(*fences, n_fence);
3451	if (!fences) {
3452	fence = ERR_PTR(-ENOMEM);
3453	goto err_trace;
3454	}
3455
3456	cf = dma_fence_array_alloc(n_fence);
3457	if (!cf) {
3458	fence = ERR_PTR(-ENOMEM);
3459	goto err_out;
3460	}
3461
3462	for_each_tile(tile, vm->xe, id) {
3463	if (!vops->pt_update_ops[id].num_ops)
3464	continue;
3465
3466	err = xe_pt_update_ops_prepare(tile, vops);
3467	if (err) {
3468	fence = ERR_PTR(err);
3469	goto err_out;
3470	}
3471	}
3472
3473	trace_xe_vm_ops_execute(vops);
3474
3475	for_each_tile(tile, vm->xe, id) {
3476	struct xe_exec_queue *q = vops->pt_update_ops[tile->id].q;
3477
3478	fence = NULL;
3479	if (!vops->pt_update_ops[id].num_ops)
3480	goto collect_fences;
3481
3482	fence = xe_pt_update_ops_run(tile, vops);
3483	if (IS_ERR(fence))
3484	goto err_out;
3485
3486	collect_fences:
3487	fences[current_fence++] = fence ?: dma_fence_get_stub();
3488	if (vops->flags & XE_VMA_OPS_FLAG_SKIP_TLB_WAIT)
3489	continue;
3490
3491	xe_migrate_job_lock(tile->migrate, q);
3492	for_each_tlb_inval(i)
3493	fences[current_fence++] =
3494	xe_exec_queue_tlb_inval_last_fence_get(q, vm, i);
3495	xe_migrate_job_unlock(tile->migrate, q);
3496	}
3497
3498	xe_assert(vm->xe, current_fence == n_fence);
3499	dma_fence_array_init(cf, n_fence, fences, dma_fence_context_alloc(1),
3500	1);
3501	fence = &cf->base;
3502
3503	for_each_tile(tile, vm->xe, id) {
3504	if (!vops->pt_update_ops[id].num_ops)
3505	continue;
3506
3507	xe_pt_update_ops_fini(tile, vops);
3508	}
3509
3510	return fence;
3511
3512	err_out:
3513	for_each_tile(tile, vm->xe, id) {
3514	if (!vops->pt_update_ops[id].num_ops)
3515	continue;
3516
3517	xe_pt_update_ops_abort(tile, vops);
3518	}
3519	while (current_fence)
3520	dma_fence_put(fences[--current_fence]);
3521	kfree(fences);
3522	kfree(cf);
3523
3524	err_trace:
3525	trace_xe_vm_ops_fail(vm);
3526	return fence;
3527	}
3528
3529	static void vma_add_ufence(struct xe_vma vma, struct xe_user_fence ufence)
3530	{
3531	if (vma->ufence)
3532	xe_sync_ufence_put(vma->ufence);
3533	vma->ufence = __xe_sync_ufence_get(ufence);
3534	}
3535
3536	static void op_add_ufence(struct xe_vm vm, struct xe_vma_op op,
3537	struct xe_user_fence *ufence)
3538	{
3539	switch (op->base.op) {
3540	case DRM_GPUVA_OP_MAP:
3541	if (!xe_vma_is_cpu_addr_mirror(op->map.vma))
3542	vma_add_ufence(op->map.vma, ufence);
3543	break;
3544	case DRM_GPUVA_OP_REMAP:
3545	if (op->remap.prev)
3546	vma_add_ufence(op->remap.prev, ufence);
3547	if (op->remap.next)
3548	vma_add_ufence(op->remap.next, ufence);
3549	break;
3550	case DRM_GPUVA_OP_UNMAP:
3551	break;
3552	case DRM_GPUVA_OP_PREFETCH:
3553	vma_add_ufence(gpuva_to_vma(op->base.prefetch.va), ufence);
3554	break;
3555	default:
3556	drm_warn(&vm->xe->drm, "NOT POSSIBLE\n");
3557	}
3558	}
3559
3560	static void vm_bind_ioctl_ops_fini(struct xe_vm vm, struct xe_vma_ops vops,
3561	struct dma_fence *fence)
3562	{
3563	struct xe_user_fence *ufence;
3564	struct xe_vma_op *op;
3565	int i;
3566
3567	ufence = find_ufence_get(vops->syncs, vops->num_syncs);
3568	list_for_each_entry(op, &vops->list, link) {
3569	if (ufence)
3570	op_add_ufence(vm, op, ufence);
3571
3572	if (op->base.op == DRM_GPUVA_OP_UNMAP)
3573	xe_vma_destroy(gpuva_to_vma(op->base.unmap.va), fence);
3574	else if (op->base.op == DRM_GPUVA_OP_REMAP)
3575	xe_vma_destroy(gpuva_to_vma(op->base.remap.unmap->va),
3576	fence);
3577	}
3578	if (ufence)
3579	xe_sync_ufence_put(ufence);
3580	if (fence) {
3581	for (i = 0; i < vops->num_syncs; i++)
3582	xe_sync_entry_signal(vops->syncs + i, fence);
3583	}
3584	}
3585
3586	static struct dma_fence vm_bind_ioctl_ops_execute(struct xe_vm vm,
3587	struct xe_vma_ops *vops)
3588	{
3589	struct xe_validation_ctx ctx;
3590	struct drm_exec exec;
3591	struct dma_fence *fence;
3592	int err = 0;
3593
3594	lockdep_assert_held_write(&vm->lock);
3595
3596	xe_validation_guard(&ctx, &vm->xe->val, &exec,
3597	((struct xe_val_flags) {
3598	.interruptible = true,
3599	.exec_ignore_duplicates = true,
3600	}), err) {
3601	err = vm_bind_ioctl_ops_lock_and_prep(&exec, vm, vops);
3602	drm_exec_retry_on_contention(&exec);
3603	xe_validation_retry_on_oom(&ctx, &err);
3604	if (err)
3605	return ERR_PTR(err);
3606
3607	xe_vm_set_validation_exec(vm, &exec);
3608	fence = ops_execute(vm, vops);
3609	xe_vm_set_validation_exec(vm, NULL);
3610	if (IS_ERR(fence)) {
3611	if (PTR_ERR(fence) == -ENODATA)
3612	vm_bind_ioctl_ops_fini(vm, vops, NULL);
3613	return fence;
3614	}
3615
3616	vm_bind_ioctl_ops_fini(vm, vops, fence);
3617	}
3618
3619	return err ? ERR_PTR(err) : fence;
3620	}
3621	ALLOW_ERROR_INJECTION(vm_bind_ioctl_ops_execute, ERRNO);
3622
3623	#define SUPPORTED_FLAGS_STUB \
3624	(DRM_XE_VM_BIND_FLAG_READONLY \| \
3625	DRM_XE_VM_BIND_FLAG_IMMEDIATE \| \
3626	DRM_XE_VM_BIND_FLAG_NULL \| \
3627	DRM_XE_VM_BIND_FLAG_DUMPABLE \| \
3628	DRM_XE_VM_BIND_FLAG_CHECK_PXP \| \
3629	DRM_XE_VM_BIND_FLAG_CPU_ADDR_MIRROR \| \
3630	DRM_XE_VM_BIND_FLAG_MADVISE_AUTORESET \| \
3631	DRM_XE_VM_BIND_FLAG_DECOMPRESS)
3632
3633	#ifdef TEST_VM_OPS_ERROR
3634	#define SUPPORTED_FLAGS (SUPPORTED_FLAGS_STUB \| FORCE_OP_ERROR)
3635	#else
3636	#define SUPPORTED_FLAGS SUPPORTED_FLAGS_STUB
3637	#endif
3638
3639	#define XE_64K_PAGE_MASK 0xffffull
3640	#define ALL_DRM_XE_SYNCS_FLAGS (DRM_XE_SYNCS_FLAG_WAIT_FOR_OP)
3641
3642	static int vm_bind_ioctl_check_args(struct xe_device xe, struct xe_vm vm,
3643	struct drm_xe_vm_bind *args,
3644	struct drm_xe_vm_bind_op **bind_ops)
3645	{
3646	int err;
3647	int i;
3648
3649	if (XE_IOCTL_DBG(xe, args->pad \|\| args->pad2) \|\|
3650	XE_IOCTL_DBG(xe, args->reserved[0] \|\| args->reserved[1]))
3651	return -EINVAL;
3652
3653	if (XE_IOCTL_DBG(xe, args->extensions))
3654	return -EINVAL;
3655
3656	if (XE_IOCTL_DBG(xe, args->num_syncs > DRM_XE_MAX_SYNCS))
3657	return -EINVAL;
3658
3659	if (args->num_binds > 1) {
3660	u64 __user *bind_user =
3661	u64_to_user_ptr(args->vector_of_binds);
3662
3663	*bind_ops = kvmalloc_objs(struct drm_xe_vm_bind_op,
3664	args->num_binds,
3665	GFP_KERNEL \| __GFP_ACCOUNT \| __GFP_RETRY_MAYFAIL \| __GFP_NOWARN);
3666	if (!*bind_ops)
3667	return args->num_binds > 1 ? -ENOBUFS : -ENOMEM;
3668
3669	err = copy_from_user(*bind_ops, bind_user,
3670	sizeof(struct drm_xe_vm_bind_op) *
3671	args->num_binds);
3672	if (XE_IOCTL_DBG(xe, err)) {
3673	err = -EFAULT;
3674	goto free_bind_ops;
3675	}
3676	} else {
3677	*bind_ops = &args->bind;
3678	}
3679
3680	for (i = 0; i < args->num_binds; ++i) {
3681	u64 range = (*bind_ops)[i].range;
3682	u64 addr = (*bind_ops)[i].addr;
3683	u32 op = (*bind_ops)[i].op;
3684	u32 flags = (*bind_ops)[i].flags;
3685	u32 obj = (*bind_ops)[i].obj;
3686	u64 obj_offset = (*bind_ops)[i].obj_offset;
3687	u32 prefetch_region = (*bind_ops)[i].prefetch_mem_region_instance;
3688	bool is_null = flags & DRM_XE_VM_BIND_FLAG_NULL;
3689	bool is_cpu_addr_mirror = flags &
3690	DRM_XE_VM_BIND_FLAG_CPU_ADDR_MIRROR;
3691	bool is_decompress = flags & DRM_XE_VM_BIND_FLAG_DECOMPRESS;
3692	u16 pat_index = (*bind_ops)[i].pat_index;
3693	u16 coh_mode;
3694	bool comp_en;
3695
3696	if (XE_IOCTL_DBG(xe, is_cpu_addr_mirror &&
3697	(!xe_vm_in_fault_mode(vm) \|\|
3698	!IS_ENABLED(CONFIG_DRM_XE_GPUSVM)))) {
3699	err = -EINVAL;
3700	goto free_bind_ops;
3701	}
3702
3703	if (XE_IOCTL_DBG(xe, pat_index >= xe->pat.n_entries)) {
3704	err = -EINVAL;
3705	goto free_bind_ops;
3706	}
3707
3708	pat_index = array_index_nospec(pat_index, xe->pat.n_entries);
3709	(*bind_ops)[i].pat_index = pat_index;
3710	coh_mode = xe_pat_index_get_coh_mode(xe, pat_index);
3711	comp_en = xe_pat_index_get_comp_en(xe, pat_index);
3712	if (XE_IOCTL_DBG(xe, !coh_mode)) { /* hw reserved */
3713	err = -EINVAL;
3714	goto free_bind_ops;
3715	}
3716
3717	if (XE_WARN_ON(coh_mode > XE_COH_2WAY)) {
3718	err = -EINVAL;
3719	goto free_bind_ops;
3720	}
3721
3722	if (XE_IOCTL_DBG(xe, op > DRM_XE_VM_BIND_OP_PREFETCH) \|\|
3723	XE_IOCTL_DBG(xe, flags & ~SUPPORTED_FLAGS) \|\|
3724	XE_IOCTL_DBG(xe, obj && (is_null \|\| is_cpu_addr_mirror)) \|\|
3725	XE_IOCTL_DBG(xe, obj_offset && (is_null \|\|
3726	is_cpu_addr_mirror)) \|\|
3727	XE_IOCTL_DBG(xe, op != DRM_XE_VM_BIND_OP_MAP &&
3728	(is_decompress \|\| is_null \|\| is_cpu_addr_mirror)) \|\|
3729	XE_IOCTL_DBG(xe, is_decompress &&
3730	xe_pat_index_get_comp_en(xe, pat_index)) \|\|
3731	XE_IOCTL_DBG(xe, !obj &&
3732	op == DRM_XE_VM_BIND_OP_MAP &&
3733	!is_null && !is_cpu_addr_mirror) \|\|
3734	XE_IOCTL_DBG(xe, !obj &&
3735	op == DRM_XE_VM_BIND_OP_UNMAP_ALL) \|\|
3736	XE_IOCTL_DBG(xe, addr &&
3737	op == DRM_XE_VM_BIND_OP_UNMAP_ALL) \|\|
3738	XE_IOCTL_DBG(xe, range &&
3739	op == DRM_XE_VM_BIND_OP_UNMAP_ALL) \|\|
3740	XE_IOCTL_DBG(xe, obj &&
3741	op == DRM_XE_VM_BIND_OP_MAP_USERPTR) \|\|
3742	XE_IOCTL_DBG(xe, coh_mode == XE_COH_NONE &&
3743	op == DRM_XE_VM_BIND_OP_MAP_USERPTR) \|\|
3744	XE_IOCTL_DBG(xe, !IS_DGFX(xe) && coh_mode == XE_COH_NONE &&
3745	is_cpu_addr_mirror) \|\|
3746	XE_IOCTL_DBG(xe, xe_device_is_l2_flush_optimized(xe) &&
3747	(op == DRM_XE_VM_BIND_OP_MAP_USERPTR \|\|
3748	is_cpu_addr_mirror) &&
3749	(pat_index != 19 && coh_mode != XE_COH_2WAY)) \|\|
3750	XE_IOCTL_DBG(xe, comp_en &&
3751	op == DRM_XE_VM_BIND_OP_MAP_USERPTR) \|\|
3752	XE_IOCTL_DBG(xe, op == DRM_XE_VM_BIND_OP_MAP_USERPTR &&
3753	!IS_ENABLED(CONFIG_DRM_GPUSVM)) \|\|
3754	XE_IOCTL_DBG(xe, obj &&
3755	op == DRM_XE_VM_BIND_OP_PREFETCH) \|\|
3756	XE_IOCTL_DBG(xe, prefetch_region &&
3757	op != DRM_XE_VM_BIND_OP_PREFETCH) \|\|
3758	XE_IOCTL_DBG(xe, (prefetch_region != DRM_XE_CONSULT_MEM_ADVISE_PREF_LOC &&
3759	/* Guard against undefined shift in BIT(prefetch_region) */
3760	(prefetch_region >= (sizeof(xe->info.mem_region_mask) * 8) \|\|
3761	!(BIT(prefetch_region) & xe->info.mem_region_mask)))) \|\|
3762	XE_IOCTL_DBG(xe, obj &&
3763	op == DRM_XE_VM_BIND_OP_UNMAP) \|\|
3764	XE_IOCTL_DBG(xe, (flags & DRM_XE_VM_BIND_FLAG_MADVISE_AUTORESET) &&
3765	(!is_cpu_addr_mirror \|\| op != DRM_XE_VM_BIND_OP_MAP))) {
3766	err = -EINVAL;
3767	goto free_bind_ops;
3768	}
3769
3770	if (XE_IOCTL_DBG(xe, obj_offset & ~PAGE_MASK) \|\|
3771	XE_IOCTL_DBG(xe, addr & ~PAGE_MASK) \|\|
3772	XE_IOCTL_DBG(xe, range & ~PAGE_MASK) \|\|
3773	XE_IOCTL_DBG(xe, !range &&
3774	op != DRM_XE_VM_BIND_OP_UNMAP_ALL)) {
3775	err = -EINVAL;
3776	goto free_bind_ops;
3777	}
3778
3779	if (is_decompress && (XE_IOCTL_DBG(xe, !xe_device_has_flat_ccs(xe)) \|\|
3780	XE_IOCTL_DBG(xe, GRAPHICS_VER(xe) < 20) \|\|
3781	XE_IOCTL_DBG(xe, !IS_DGFX(xe)))) {
3782	err = -EOPNOTSUPP;
3783	goto free_bind_ops;
3784	}
3785	}
3786
3787	return 0;
3788
3789	free_bind_ops:
3790	if (args->num_binds > 1)
3791	kvfree(*bind_ops);
3792	*bind_ops = NULL;
3793	return err;
3794	}
3795
3796	static int vm_bind_ioctl_signal_fences(struct xe_vm *vm,
3797	struct xe_exec_queue *q,
3798	struct xe_sync_entry *syncs,
3799	int num_syncs)
3800	{
3801	struct dma_fence *fence = NULL;
3802	int i, err = 0;
3803
3804	if (num_syncs) {
3805	fence = xe_sync_in_fence_get(syncs, num_syncs,
3806	to_wait_exec_queue(vm, q), vm);
3807	if (IS_ERR(fence))
3808	return PTR_ERR(fence);
3809
3810	for (i = 0; i < num_syncs; i++)
3811	xe_sync_entry_signal(&syncs[i], fence);
3812	}
3813
3814	dma_fence_put(fence);
3815
3816	return err;
3817	}
3818
3819	static void xe_vma_ops_init(struct xe_vma_ops vops, struct xe_vm vm,
3820	struct xe_exec_queue *q,
3821	struct xe_sync_entry *syncs, u32 num_syncs)
3822	{
3823	memset(vops, 0, sizeof(*vops));
3824	INIT_LIST_HEAD(&vops->list);
3825	vops->vm = vm;
3826	vops->q = q;
3827	vops->syncs = syncs;
3828	vops->num_syncs = num_syncs;
3829	vops->flags = 0;
3830	}
3831
3832	static int xe_vm_bind_ioctl_validate_bo(struct xe_device xe, struct xe_bo bo,
3833	u64 addr, u64 range, u64 obj_offset,
3834	u16 pat_index, u32 op, u32 bind_flags)
3835	{
3836	u16 coh_mode;
3837	bool comp_en;
3838
3839	if (XE_IOCTL_DBG(xe, (bo->flags & XE_BO_FLAG_NO_COMPRESSION) &&
3840	xe_pat_index_get_comp_en(xe, pat_index)))
3841	return -EINVAL;
3842
3843	if (XE_IOCTL_DBG(xe, range > xe_bo_size(bo)) \|\|
3844	XE_IOCTL_DBG(xe, obj_offset >
3845	xe_bo_size(bo) - range)) {
3846	return -EINVAL;
3847	}
3848
3849	/*
3850	* Some platforms require 64k VM_BIND alignment,
3851	* specifically those with XE_VRAM_FLAGS_NEED64K.
3852	*
3853	* Other platforms may have BO's set to 64k physical placement,
3854	* but can be mapped at 4k offsets anyway. This check is only
3855	* there for the former case.
3856	*/
3857	if ((bo->flags & XE_BO_FLAG_INTERNAL_64K) &&
3858	(xe->info.vram_flags & XE_VRAM_FLAGS_NEED64K)) {
3859	if (XE_IOCTL_DBG(xe, obj_offset &
3860	XE_64K_PAGE_MASK) \|\|
3861	XE_IOCTL_DBG(xe, addr & XE_64K_PAGE_MASK) \|\|
3862	XE_IOCTL_DBG(xe, range & XE_64K_PAGE_MASK)) {
3863	return -EINVAL;
3864	}
3865	}
3866
3867	coh_mode = xe_pat_index_get_coh_mode(xe, pat_index);
3868	if (bo->cpu_caching) {
3869	if (XE_IOCTL_DBG(xe, coh_mode == XE_COH_NONE &&
3870	bo->cpu_caching == DRM_XE_GEM_CPU_CACHING_WB)) {
3871	return -EINVAL;
3872	}
3873	} else if (XE_IOCTL_DBG(xe, coh_mode == XE_COH_NONE)) {
3874	/*
3875	* Imported dma-buf from a different device should
3876	* require 1way or 2way coherency since we don't know
3877	* how it was mapped on the CPU. Just assume is it
3878	* potentially cached on CPU side.
3879	*/
3880	return -EINVAL;
3881	}
3882
3883	/*
3884	* Ensures that imported buffer objects (dma-bufs) are not mapped
3885	* with a PAT index that enables compression.
3886	*/
3887	comp_en = xe_pat_index_get_comp_en(xe, pat_index);
3888	if (XE_IOCTL_DBG(xe, bo->ttm.base.import_attach && comp_en))
3889	return -EINVAL;
3890
3891	if (XE_IOCTL_DBG(xe, bo->ttm.base.import_attach && xe_device_is_l2_flush_optimized(xe) &&
3892	(pat_index != 19 && coh_mode != XE_COH_2WAY)))
3893	return -EINVAL;
3894
3895	/* If a BO is protected it can only be mapped if the key is still valid */
3896	if ((bind_flags & DRM_XE_VM_BIND_FLAG_CHECK_PXP) && xe_bo_is_protected(bo) &&
3897	op != DRM_XE_VM_BIND_OP_UNMAP && op != DRM_XE_VM_BIND_OP_UNMAP_ALL)
3898	if (XE_IOCTL_DBG(xe, xe_pxp_bo_key_check(xe->pxp, bo) != 0))
3899	return -ENOEXEC;
3900
3901	return 0;
3902	}
3903
3904	int xe_vm_bind_ioctl(struct drm_device dev, void data, struct drm_file *file)
3905	{
3906	struct xe_device *xe = to_xe_device(dev);
3907	struct xe_file *xef = to_xe_file(file);
3908	struct drm_xe_vm_bind *args = data;
3909	struct drm_xe_sync __user *syncs_user;
3910	struct xe_bo **bos = NULL;
3911	struct drm_gpuva_ops **ops = NULL;
3912	struct xe_vm *vm;
3913	struct xe_exec_queue *q = NULL;
3914	u32 num_syncs, num_ufence = 0;
3915	struct xe_sync_entry *syncs = NULL;
3916	struct drm_xe_vm_bind_op *bind_ops = NULL;
3917	struct xe_vma_ops vops;
3918	struct dma_fence *fence;
3919	int err;
3920	int i;
3921
3922	vm = xe_vm_lookup(xef, args->vm_id);
3923	if (XE_IOCTL_DBG(xe, !vm))
3924	return -EINVAL;
3925
3926	err = vm_bind_ioctl_check_args(xe, vm, args, &bind_ops);
3927	if (err)
3928	goto put_vm;
3929
3930	if (args->exec_queue_id) {
3931	q = xe_exec_queue_lookup(xef, args->exec_queue_id);
3932	if (XE_IOCTL_DBG(xe, !q)) {
3933	err = -ENOENT;
3934	goto free_bind_ops;
3935	}
3936
3937	if (XE_IOCTL_DBG(xe, !(q->flags & EXEC_QUEUE_FLAG_VM))) {
3938	err = -EINVAL;
3939	goto put_exec_queue;
3940	}
3941	}
3942
3943	if (XE_IOCTL_DBG(xe, q && vm != q->user_vm)) {
3944	err = -EINVAL;
3945	goto put_exec_queue;
3946	}
3947
3948	/* Ensure all UNMAPs visible */
3949	xe_svm_flush(vm);
3950
3951	err = down_write_killable(&vm->lock);
3952	if (err)
3953	goto put_exec_queue;
3954
3955	if (XE_IOCTL_DBG(xe, xe_vm_is_closed_or_banned(vm))) {
3956	err = -ENOENT;
3957	goto release_vm_lock;
3958	}
3959
3960	for (i = 0; i < args->num_binds; ++i) {
3961	u64 range = bind_ops[i].range;
3962	u64 addr = bind_ops[i].addr;
3963
3964	if (XE_IOCTL_DBG(xe, range > vm->size) \|\|
3965	XE_IOCTL_DBG(xe, addr > vm->size - range)) {
3966	err = -EINVAL;
3967	goto release_vm_lock;
3968	}
3969	}
3970
3971	if (args->num_binds) {
3972	bos = kvzalloc_objs(*bos, args->num_binds,
3973	GFP_KERNEL \| __GFP_ACCOUNT \| __GFP_RETRY_MAYFAIL \| __GFP_NOWARN);
3974	if (!bos) {
3975	err = -ENOMEM;
3976	goto release_vm_lock;
3977	}
3978
3979	ops = kvzalloc_objs(*ops, args->num_binds,
3980	GFP_KERNEL \| __GFP_ACCOUNT \| __GFP_RETRY_MAYFAIL \| __GFP_NOWARN);
3981	if (!ops) {
3982	err = -ENOMEM;
3983	goto free_bos;
3984	}
3985	}
3986
3987	for (i = 0; i < args->num_binds; ++i) {
3988	struct drm_gem_object *gem_obj;
3989	u64 range = bind_ops[i].range;
3990	u64 addr = bind_ops[i].addr;
3991	u32 obj = bind_ops[i].obj;
3992	u64 obj_offset = bind_ops[i].obj_offset;
3993	u16 pat_index = bind_ops[i].pat_index;
3994	u32 op = bind_ops[i].op;
3995	u32 bind_flags = bind_ops[i].flags;
3996
3997	if (!obj)
3998	continue;
3999
4000	gem_obj = drm_gem_object_lookup(file, obj);
4001	if (XE_IOCTL_DBG(xe, !gem_obj)) {
4002	err = -ENOENT;
4003	goto put_obj;
4004	}
4005	bos[i] = gem_to_xe_bo(gem_obj);
4006
4007	err = xe_vm_bind_ioctl_validate_bo(xe, bos[i], addr, range,
4008	obj_offset, pat_index, op,
4009	bind_flags);
4010	if (err)
4011	goto put_obj;
4012	}
4013
4014	if (args->num_syncs) {
4015	syncs = kzalloc_objs(*syncs, args->num_syncs);
4016	if (!syncs) {
4017	err = -ENOMEM;
4018	goto put_obj;
4019	}
4020	}
4021
4022	syncs_user = u64_to_user_ptr(args->syncs);
4023	for (num_syncs = 0; num_syncs < args->num_syncs; num_syncs++) {
4024	struct xe_exec_queue *__q = q ?: vm->q[0];
4025
4026	err = xe_sync_entry_parse(xe, xef, &syncs[num_syncs],
4027	&syncs_user[num_syncs],
4028	__q->ufence_syncobj,
4029	++__q->ufence_timeline_value,
4030	(xe_vm_in_lr_mode(vm) ?
4031	SYNC_PARSE_FLAG_LR_MODE : 0) \|
4032	(!args->num_binds ?
4033	SYNC_PARSE_FLAG_DISALLOW_USER_FENCE : 0));
4034	if (err)
4035	goto free_syncs;
4036
4037	if (xe_sync_is_ufence(&syncs[num_syncs]))
4038	num_ufence++;
4039	}
4040
4041	if (XE_IOCTL_DBG(xe, num_ufence > 1)) {
4042	err = -EINVAL;
4043	goto free_syncs;
4044	}
4045
4046	if (!args->num_binds) {
4047	err = -ENODATA;
4048	goto free_syncs;
4049	}
4050
4051	xe_vma_ops_init(&vops, vm, q, syncs, num_syncs);
4052	if (args->num_binds > 1)
4053	vops.flags \|= XE_VMA_OPS_ARRAY_OF_BINDS;
4054	for (i = 0; i < args->num_binds; ++i) {
4055	u64 range = bind_ops[i].range;
4056	u64 addr = bind_ops[i].addr;
4057	u32 op = bind_ops[i].op;
4058	u32 flags = bind_ops[i].flags;
4059	u64 obj_offset = bind_ops[i].obj_offset;
4060	u32 prefetch_region = bind_ops[i].prefetch_mem_region_instance;
4061	u16 pat_index = bind_ops[i].pat_index;
4062
4063	ops[i] = vm_bind_ioctl_ops_create(vm, &vops, bos[i], obj_offset,
4064	addr, range, op, flags,
4065	prefetch_region, pat_index);
4066	if (IS_ERR(ops[i])) {
4067	err = PTR_ERR(ops[i]);
4068	ops[i] = NULL;
4069	goto unwind_ops;
4070	}
4071
4072	err = vm_bind_ioctl_ops_parse(vm, ops[i], &vops);
4073	if (err)
4074	goto unwind_ops;
4075
4076	#ifdef TEST_VM_OPS_ERROR
4077	if (flags & FORCE_OP_ERROR) {
4078	vops.inject_error = true;
4079	vm->xe->vm_inject_error_position =
4080	(vm->xe->vm_inject_error_position + 1) %
4081	FORCE_OP_ERROR_COUNT;
4082	}
4083	#endif
4084	}
4085
4086	/* Nothing to do */
4087	if (list_empty(&vops.list)) {
4088	err = -ENODATA;
4089	goto unwind_ops;
4090	}
4091
4092	err = xe_vma_ops_alloc(&vops, args->num_binds > 1);
4093	if (err)
4094	goto unwind_ops;
4095
4096	err = vm_bind_ioctl_ops_prefetch_ranges(vm, &vops);
4097	if (err)
4098	goto unwind_ops;
4099
4100	fence = vm_bind_ioctl_ops_execute(vm, &vops);
4101	if (IS_ERR(fence))
4102	err = PTR_ERR(fence);
4103	else
4104	dma_fence_put(fence);
4105
4106	unwind_ops:
4107	if (err && err != -ENODATA)
4108	vm_bind_ioctl_ops_unwind(vm, ops, args->num_binds);
4109	xe_vma_ops_fini(&vops);
4110	for (i = args->num_binds - 1; i >= 0; --i)
4111	if (ops[i])
4112	drm_gpuva_ops_free(&vm->gpuvm, ops[i]);
4113	free_syncs:
4114	if (err == -ENODATA)
4115	err = vm_bind_ioctl_signal_fences(vm, q, syncs, num_syncs);
4116	while (num_syncs--)
4117	xe_sync_entry_cleanup(&syncs[num_syncs]);
4118
4119	kfree(syncs);
4120	put_obj:
4121	for (i = 0; i < args->num_binds; ++i)
4122	xe_bo_put(bos[i]);
4123
4124	kvfree(ops);
4125	free_bos:
4126	kvfree(bos);
4127	release_vm_lock:
4128	up_write(&vm->lock);
4129	put_exec_queue:
4130	if (q)
4131	xe_exec_queue_put(q);
4132	free_bind_ops:
4133	if (args->num_binds > 1)
4134	kvfree(bind_ops);
4135	put_vm:
4136	xe_vm_put(vm);
4137	return err;
4138	}
4139
4140	/*
4141	* Map access type, fault type, and fault level from current bspec
4142	* specification to user spec abstraction. The current mapping is
4143	* approximately 1-to-1, with access type being the only notable
4144	* exception as it carries additional data with respect to prefetch
4145	* status that needs to be masked out.
4146	*/
4147	static u8 xe_to_user_access_type(u8 access_type)
4148	{
4149	return access_type & XE_PAGEFAULT_ACCESS_TYPE_MASK;
4150	}
4151
4152	static u8 xe_to_user_fault_type(u8 fault_type)
4153	{
4154	return fault_type;
4155	}
4156
4157	static u8 xe_to_user_fault_level(u8 fault_level)
4158	{
4159	return fault_level;
4160	}
4161
4162	static int fill_faults(struct xe_vm *vm,
4163	struct drm_xe_vm_get_property *args)
4164	{
4165	struct xe_vm_fault __user *usr_ptr = u64_to_user_ptr(args->data);
4166	struct xe_vm_fault *fault_list, fault_entry = { 0 };
4167	struct xe_vm_fault_entry *entry;
4168	int ret = 0, i = 0, count, entry_size;
4169
4170	entry_size = sizeof(struct xe_vm_fault);
4171	count = args->size / entry_size;
4172
4173	fault_list = kzalloc_objs(struct xe_vm_fault, count);
4174	if (!fault_list)
4175	return -ENOMEM;
4176
4177	spin_lock(&vm->faults.lock);
4178	list_for_each_entry(entry, &vm->faults.list, list) {
4179	if (i == count)
4180	break;
4181
4182	fault_entry.address = xe_device_canonicalize_addr(vm->xe, entry->address);
4183	fault_entry.address_precision = entry->address_precision;
4184
4185	fault_entry.access_type = xe_to_user_access_type(entry->access_type);
4186	fault_entry.fault_type = xe_to_user_fault_type(entry->fault_type);
4187	fault_entry.fault_level = xe_to_user_fault_level(entry->fault_level);
4188
4189	memcpy(&fault_list[i], &fault_entry, entry_size);
4190
4191	i++;
4192	}
4193	spin_unlock(&vm->faults.lock);
4194
4195	ret = copy_to_user(usr_ptr, fault_list, args->size);
4196
4197	kfree(fault_list);
4198	return ret ? -EFAULT : 0;
4199	}
4200
4201	static int xe_vm_get_property_helper(struct xe_vm *vm,
4202	struct drm_xe_vm_get_property *args)
4203	{
4204	size_t size;
4205
4206	switch (args->property) {
4207	case DRM_XE_VM_GET_PROPERTY_FAULTS:
4208	spin_lock(&vm->faults.lock);
4209	size = size_mul(sizeof(struct xe_vm_fault), vm->faults.len);
4210	spin_unlock(&vm->faults.lock);
4211
4212	if (!args->size) {
4213	args->size = size;
4214	return 0;
4215	}
4216
4217	/*
4218	* Number of faults may increase between calls to
4219	* xe_vm_get_property_ioctl, so just report the number of
4220	* faults the user requests if it's less than or equal to
4221	* the number of faults in the VM fault array.
4222	*
4223	* We should also at least assert that the args->size value
4224	* is a multiple of the xe_vm_fault struct size.
4225	*/
4226	if (args->size > size \|\| args->size % sizeof(struct xe_vm_fault))
4227	return -EINVAL;
4228
4229	return fill_faults(vm, args);
4230	}
4231	return -EINVAL;
4232	}
4233
4234	int xe_vm_get_property_ioctl(struct drm_device drm, void data,
4235	struct drm_file *file)
4236	{
4237	struct xe_device *xe = to_xe_device(drm);
4238	struct xe_file *xef = to_xe_file(file);
4239	struct drm_xe_vm_get_property *args = data;
4240	struct xe_vm *vm;
4241	int ret = 0;
4242
4243	if (XE_IOCTL_DBG(xe, (args->reserved[0] \|\| args->reserved[1] \|\|
4244	args->reserved[2] \|\| args->extensions \|\|
4245	args->pad)))
4246	return -EINVAL;
4247
4248	vm = xe_vm_lookup(xef, args->vm_id);
4249	if (XE_IOCTL_DBG(xe, !vm))
4250	return -ENOENT;
4251
4252	ret = xe_vm_get_property_helper(vm, args);
4253
4254	xe_vm_put(vm);
4255	return ret;
4256	}
4257
4258	/**
4259	* xe_vm_bind_kernel_bo - bind a kernel BO to a VM
4260	* @vm: VM to bind the BO to
4261	* @bo: BO to bind
4262	* @q: exec queue to use for the bind (optional)
4263	* @addr: address at which to bind the BO
4264	* @cache_lvl: PAT cache level to use
4265	*
4266	* Execute a VM bind map operation on a kernel-owned BO to bind it into a
4267	* kernel-owned VM.
4268	*
4269	* Returns a dma_fence to track the binding completion if the job to do so was
4270	* successfully submitted, an error pointer otherwise.
4271	*/
4272	struct dma_fence xe_vm_bind_kernel_bo(struct xe_vm vm, struct xe_bo *bo,
4273	struct xe_exec_queue *q, u64 addr,
4274	enum xe_cache_level cache_lvl)
4275	{
4276	struct xe_vma_ops vops;
4277	struct drm_gpuva_ops *ops = NULL;
4278	struct dma_fence *fence;
4279	int err;
4280
4281	xe_bo_get(bo);
4282	xe_vm_get(vm);
4283	if (q)
4284	xe_exec_queue_get(q);
4285
4286	down_write(&vm->lock);
4287
4288	xe_vma_ops_init(&vops, vm, q, NULL, 0);
4289
4290	ops = vm_bind_ioctl_ops_create(vm, &vops, bo, 0, addr, xe_bo_size(bo),
4291	DRM_XE_VM_BIND_OP_MAP, 0, 0,
4292	xe_cache_pat_idx(vm->xe, cache_lvl));
4293	if (IS_ERR(ops)) {
4294	err = PTR_ERR(ops);
4295	goto release_vm_lock;
4296	}
4297
4298	err = vm_bind_ioctl_ops_parse(vm, ops, &vops);
4299	if (err)
4300	goto release_vm_lock;
4301
4302	xe_assert(vm->xe, !list_empty(&vops.list));
4303
4304	err = xe_vma_ops_alloc(&vops, false);
4305	if (err)
4306	goto unwind_ops;
4307
4308	fence = vm_bind_ioctl_ops_execute(vm, &vops);
4309	if (IS_ERR(fence))
4310	err = PTR_ERR(fence);
4311
4312	unwind_ops:
4313	if (err && err != -ENODATA)
4314	vm_bind_ioctl_ops_unwind(vm, &ops, 1);
4315
4316	xe_vma_ops_fini(&vops);
4317	drm_gpuva_ops_free(&vm->gpuvm, ops);
4318
4319	release_vm_lock:
4320	up_write(&vm->lock);
4321
4322	if (q)
4323	xe_exec_queue_put(q);
4324	xe_vm_put(vm);
4325	xe_bo_put(bo);
4326
4327	if (err)
4328	fence = ERR_PTR(err);
4329
4330	return fence;
4331	}
4332
4333	/**
4334	* xe_vm_lock() - Lock the vm's dma_resv object
4335	* @vm: The struct xe_vm whose lock is to be locked
4336	* @intr: Whether to perform any wait interruptible
4337	*
4338	* Return: 0 on success, -EINTR if @intr is true and the wait for a
4339	* contended lock was interrupted. If @intr is false, the function
4340	* always returns 0.
4341	*/
4342	int xe_vm_lock(struct xe_vm *vm, bool intr)
4343	{
4344	int ret;
4345
4346	if (intr)
4347	ret = dma_resv_lock_interruptible(xe_vm_resv(vm), NULL);
4348	else
4349	ret = dma_resv_lock(xe_vm_resv(vm), NULL);
4350
4351	return ret;
4352	}
4353
4354	/**
4355	* xe_vm_unlock() - Unlock the vm's dma_resv object
4356	* @vm: The struct xe_vm whose lock is to be released.
4357	*
4358	* Unlock a buffer object lock that was locked by xe_vm_lock().
4359	*/
4360	void xe_vm_unlock(struct xe_vm *vm)
4361	{
4362	dma_resv_unlock(xe_vm_resv(vm));
4363	}
4364
4365	/**
4366	* xe_vm_invalidate_vma_submit - Submit a job to invalidate GPU mappings for
4367	* VMA.
4368	* @vma: VMA to invalidate
4369	* @batch: TLB invalidation batch to populate; caller must later call
4370	* xe_tlb_inval_batch_wait() on it to wait for completion
4371	*
4372	* Walks a list of page tables leaves which it memset the entries owned by this
4373	* VMA to zero, invalidates the TLBs, but doesn't block waiting for TLB flush
4374	* to complete, but instead populates @batch which can be waited on using
4375	* xe_tlb_inval_batch_wait().
4376	*
4377	* Returns 0 for success, negative error code otherwise.
4378	*/
4379	int xe_vm_invalidate_vma_submit(struct xe_vma vma, struct xe_tlb_inval_batch batch)
4380	{
4381	struct xe_device *xe = xe_vma_vm(vma)->xe;
4382	struct xe_vm *vm = xe_vma_vm(vma);
4383	struct xe_tile *tile;
4384	u8 tile_mask = 0;
4385	int ret = 0;
4386	u8 id;
4387
4388	xe_assert(xe, !xe_vma_is_null(vma));
4389	xe_assert(xe, !xe_vma_is_cpu_addr_mirror(vma));
4390	trace_xe_vma_invalidate(vma);
4391
4392	vm_dbg(&vm->xe->drm,
4393	"INVALIDATE: addr=0x%016llx, range=0x%016llx",
4394	xe_vma_start(vma), xe_vma_size(vma));
4395
4396	/*
4397	* Check that we don't race with page-table updates, tile_invalidated
4398	* update is safe
4399	*/
4400	if (IS_ENABLED(CONFIG_PROVE_LOCKING)) {
4401	if (xe_vma_is_userptr(vma)) {
4402	lockdep_assert(lockdep_is_held_type(&vm->svm.gpusvm.notifier_lock, 0) \|\|
4403	(lockdep_is_held_type(&vm->svm.gpusvm.notifier_lock, 1) &&
4404	lockdep_is_held(&xe_vm_resv(vm)->lock.base)));
4405
4406	WARN_ON_ONCE(!mmu_interval_check_retry
4407	(&to_userptr_vma(vma)->userptr.notifier,
4408	to_userptr_vma(vma)->userptr.pages.notifier_seq));
4409	WARN_ON_ONCE(!dma_resv_test_signaled(xe_vm_resv(vm),
4410	DMA_RESV_USAGE_BOOKKEEP));
4411
4412	} else {
4413	xe_bo_assert_held(xe_vma_bo(vma));
4414	}
4415	}
4416
4417	for_each_tile(tile, xe, id)
4418	if (xe_pt_zap_ptes(tile, vma))
4419	tile_mask \|= BIT(id);
4420
4421	xe_device_wmb(xe);
4422
4423	ret = xe_tlb_inval_range_tilemask_submit(xe, xe_vma_vm(vma)->usm.asid,
4424	xe_vma_start(vma), xe_vma_end(vma),
4425	tile_mask, batch);
4426
4427	/* WRITE_ONCE pairs with READ_ONCE in xe_vm_has_valid_gpu_mapping() */
4428	WRITE_ONCE(vma->tile_invalidated, vma->tile_mask);
4429	return ret;
4430	}
4431
4432	/**
4433	* xe_vm_invalidate_vma - invalidate GPU mappings for VMA without a lock
4434	* @vma: VMA to invalidate
4435	*
4436	* Walks a list of page tables leaves which it memset the entries owned by this
4437	* VMA to zero, invalidates the TLBs, and block until TLBs invalidation is
4438	* complete.
4439	*
4440	* Returns 0 for success, negative error code otherwise.
4441	*/
4442	int xe_vm_invalidate_vma(struct xe_vma *vma)
4443	{
4444	struct xe_tlb_inval_batch batch;
4445	int ret;
4446
4447	ret = xe_vm_invalidate_vma_submit(vma, &batch);
4448	if (ret)
4449	return ret;
4450
4451	xe_tlb_inval_batch_wait(&batch);
4452	return ret;
4453	}
4454
4455	int xe_vm_validate_protected(struct xe_vm *vm)
4456	{
4457	struct drm_gpuva *gpuva;
4458	int err = 0;
4459
4460	if (!vm)
4461	return -ENODEV;
4462
4463	mutex_lock(&vm->snap_mutex);
4464
4465	drm_gpuvm_for_each_va(gpuva, &vm->gpuvm) {
4466	struct xe_vma *vma = gpuva_to_vma(gpuva);
4467	struct xe_bo *bo = vma->gpuva.gem.obj ?
4468	gem_to_xe_bo(vma->gpuva.gem.obj) : NULL;
4469
4470	if (!bo)
4471	continue;
4472
4473	if (xe_bo_is_protected(bo)) {
4474	err = xe_pxp_bo_key_check(vm->xe->pxp, bo);
4475	if (err)
4476	break;
4477	}
4478	}
4479
4480	mutex_unlock(&vm->snap_mutex);
4481	return err;
4482	}
4483
4484	struct xe_vm_snapshot {
4485	int uapi_flags;
4486	unsigned long num_snaps;
4487	struct {
4488	u64 ofs, bo_ofs;
4489	unsigned long len;
4490	#define XE_VM_SNAP_FLAG_USERPTR BIT(0)
4491	#define XE_VM_SNAP_FLAG_READ_ONLY BIT(1)
4492	#define XE_VM_SNAP_FLAG_IS_NULL BIT(2)
4493	unsigned long flags;
4494	int uapi_mem_region;
4495	u16 pat_index;
4496	int cpu_caching;
4497	struct xe_bo *bo;
4498	void *data;
4499	struct mm_struct *mm;
4500	} snap[];
4501	};
4502
4503	struct xe_vm_snapshot xe_vm_snapshot_capture(struct xe_vm vm)
4504	{
4505	unsigned long num_snaps = 0, i;
4506	struct xe_vm_snapshot *snap = NULL;
4507	struct drm_gpuva *gpuva;
4508
4509	if (!vm)
4510	return NULL;
4511
4512	mutex_lock(&vm->snap_mutex);
4513	drm_gpuvm_for_each_va(gpuva, &vm->gpuvm) {
4514	if (gpuva->flags & XE_VMA_DUMPABLE)
4515	num_snaps++;
4516	}
4517
4518	if (num_snaps)
4519	snap = kvzalloc(offsetof(struct xe_vm_snapshot, snap[num_snaps]), GFP_NOWAIT);
4520	if (!snap) {
4521	snap = num_snaps ? ERR_PTR(-ENOMEM) : ERR_PTR(-ENODEV);
4522	goto out_unlock;
4523	}
4524
4525	if (vm->flags & XE_VM_FLAG_FAULT_MODE)
4526	snap->uapi_flags \|= DRM_XE_VM_CREATE_FLAG_FAULT_MODE;
4527	if (vm->flags & XE_VM_FLAG_LR_MODE)
4528	snap->uapi_flags \|= DRM_XE_VM_CREATE_FLAG_LR_MODE;
4529	if (vm->flags & XE_VM_FLAG_SCRATCH_PAGE)
4530	snap->uapi_flags \|= DRM_XE_VM_CREATE_FLAG_SCRATCH_PAGE;
4531
4532	snap->num_snaps = num_snaps;
4533	i = 0;
4534	drm_gpuvm_for_each_va(gpuva, &vm->gpuvm) {
4535	struct xe_vma *vma = gpuva_to_vma(gpuva);
4536	struct xe_bo *bo = vma->gpuva.gem.obj ?
4537	gem_to_xe_bo(vma->gpuva.gem.obj) : NULL;
4538
4539	if (!(gpuva->flags & XE_VMA_DUMPABLE))
4540	continue;
4541
4542	snap->snap[i].ofs = xe_vma_start(vma);
4543	snap->snap[i].len = xe_vma_size(vma);
4544	snap->snap[i].flags = xe_vma_read_only(vma) ?
4545	XE_VM_SNAP_FLAG_READ_ONLY : 0;
4546	snap->snap[i].pat_index = vma->attr.pat_index;
4547	if (bo) {
4548	snap->snap[i].cpu_caching = bo->cpu_caching;
4549	snap->snap[i].bo = xe_bo_get(bo);
4550	snap->snap[i].bo_ofs = xe_vma_bo_offset(vma);
4551	switch (bo->ttm.resource->mem_type) {
4552	case XE_PL_SYSTEM:
4553	case XE_PL_TT:
4554	snap->snap[i].uapi_mem_region = 0;
4555	break;
4556	case XE_PL_VRAM0:
4557	snap->snap[i].uapi_mem_region = 1;
4558	break;
4559	case XE_PL_VRAM1:
4560	snap->snap[i].uapi_mem_region = 2;
4561	break;
4562	}
4563	} else if (xe_vma_is_userptr(vma)) {
4564	struct mm_struct *mm =
4565	to_userptr_vma(vma)->userptr.notifier.mm;
4566
4567	if (mmget_not_zero(mm))
4568	snap->snap[i].mm = mm;
4569	else
4570	snap->snap[i].data = ERR_PTR(-EFAULT);
4571
4572	snap->snap[i].bo_ofs = xe_vma_userptr(vma);
4573	snap->snap[i].flags \|= XE_VM_SNAP_FLAG_USERPTR;
4574	snap->snap[i].uapi_mem_region = 0;
4575	} else if (xe_vma_is_null(vma)) {
4576	snap->snap[i].flags \|= XE_VM_SNAP_FLAG_IS_NULL;
4577	snap->snap[i].uapi_mem_region = -1;
4578	} else {
4579	snap->snap[i].data = ERR_PTR(-ENOENT);
4580	snap->snap[i].uapi_mem_region = -1;
4581	}
4582	i++;
4583	}
4584
4585	out_unlock:
4586	mutex_unlock(&vm->snap_mutex);
4587	return snap;
4588	}
4589
4590	void xe_vm_snapshot_capture_delayed(struct xe_vm_snapshot *snap)
4591	{
4592	if (IS_ERR_OR_NULL(snap))
4593	return;
4594
4595	for (int i = 0; i < snap->num_snaps; i++) {
4596	struct xe_bo *bo = snap->snap[i].bo;
4597	int err;
4598
4599	if (IS_ERR(snap->snap[i].data) \|\|
4600	snap->snap[i].flags & XE_VM_SNAP_FLAG_IS_NULL)
4601	continue;
4602
4603	snap->snap[i].data = kvmalloc(snap->snap[i].len, GFP_USER);
4604	if (!snap->snap[i].data) {
4605	snap->snap[i].data = ERR_PTR(-ENOMEM);
4606	goto cleanup_bo;
4607	}
4608
4609	if (bo) {
4610	err = xe_bo_read(bo, snap->snap[i].bo_ofs,
4611	snap->snap[i].data, snap->snap[i].len);
4612	} else {
4613	void __user userptr = (void __user )(size_t)snap->snap[i].bo_ofs;
4614
4615	kthread_use_mm(snap->snap[i].mm);
4616	if (!copy_from_user(snap->snap[i].data, userptr, snap->snap[i].len))
4617	err = 0;
4618	else
4619	err = -EFAULT;
4620	kthread_unuse_mm(snap->snap[i].mm);
4621
4622	mmput(snap->snap[i].mm);
4623	snap->snap[i].mm = NULL;
4624	}
4625
4626	if (err) {
4627	kvfree(snap->snap[i].data);
4628	snap->snap[i].data = ERR_PTR(err);
4629	}
4630
4631	cleanup_bo:
4632	xe_bo_put(bo);
4633	snap->snap[i].bo = NULL;
4634	}
4635	}
4636
4637	void xe_vm_snapshot_print(struct xe_vm_snapshot snap, struct drm_printer p)
4638	{
4639	unsigned long i, j;
4640
4641	if (IS_ERR_OR_NULL(snap)) {
4642	drm_printf(p, "[0].error: %li\n", PTR_ERR(snap));
4643	return;
4644	}
4645
4646	drm_printf(p, "VM.uapi_flags: 0x%x\n", snap->uapi_flags);
4647	for (i = 0; i < snap->num_snaps; i++) {
4648	drm_printf(p, "[%llx].length: 0x%lx\n", snap->snap[i].ofs, snap->snap[i].len);
4649
4650	drm_printf(p, "[%llx].properties: %s\|%s\|mem_region=0x%lx\|pat_index=%d\|cpu_caching=%d\n",
4651	snap->snap[i].ofs,
4652	snap->snap[i].flags & XE_VM_SNAP_FLAG_READ_ONLY ?
4653	"read_only" : "read_write",
4654	snap->snap[i].flags & XE_VM_SNAP_FLAG_IS_NULL ?
4655	"null_sparse" :
4656	snap->snap[i].flags & XE_VM_SNAP_FLAG_USERPTR ?
4657	"userptr" : "bo",
4658	snap->snap[i].uapi_mem_region == -1 ? 0 :
4659	BIT(snap->snap[i].uapi_mem_region),
4660	snap->snap[i].pat_index,
4661	snap->snap[i].cpu_caching);
4662
4663	if (IS_ERR(snap->snap[i].data)) {
4664	drm_printf(p, "[%llx].error: %li\n", snap->snap[i].ofs,
4665	PTR_ERR(snap->snap[i].data));
4666	continue;
4667	}
4668
4669	if (snap->snap[i].flags & XE_VM_SNAP_FLAG_IS_NULL)
4670	continue;
4671
4672	drm_printf(p, "[%llx].data: ", snap->snap[i].ofs);
4673
4674	for (j = 0; j < snap->snap[i].len; j += sizeof(u32)) {
4675	u32 *val = snap->snap[i].data + j;
4676	char dumped[ASCII85_BUFSZ];
4677
4678	drm_puts(p, ascii85_encode(*val, dumped));
4679	}
4680
4681	drm_puts(p, "\n");
4682
4683	if (drm_coredump_printer_is_full(p))
4684	return;
4685	}
4686	}
4687
4688	void xe_vm_snapshot_free(struct xe_vm_snapshot *snap)
4689	{
4690	unsigned long i;
4691
4692	if (IS_ERR_OR_NULL(snap))
4693	return;
4694
4695	for (i = 0; i < snap->num_snaps; i++) {
4696	if (!IS_ERR(snap->snap[i].data))
4697	kvfree(snap->snap[i].data);
4698	xe_bo_put(snap->snap[i].bo);
4699	if (snap->snap[i].mm)
4700	mmput(snap->snap[i].mm);
4701	}
4702	kvfree(snap);
4703	}
4704
4705	/**
4706	* xe_vma_need_vram_for_atomic - Check if VMA needs VRAM migration for atomic operations
4707	* @xe: Pointer to the Xe device structure
4708	* @vma: Pointer to the virtual memory area (VMA) structure
4709	* @is_atomic: In pagefault path and atomic operation
4710	*
4711	* This function determines whether the given VMA needs to be migrated to
4712	* VRAM in order to do atomic GPU operation.
4713	*
4714	* Return:
4715	* 1 - Migration to VRAM is required
4716	* 0 - Migration is not required
4717	* -EACCES - Invalid access for atomic memory attr
4718	*
4719	*/
4720	int xe_vma_need_vram_for_atomic(struct xe_device xe, struct xe_vma vma, bool is_atomic)
4721	{
4722	u32 atomic_access = xe_vma_bo(vma) ? xe_vma_bo(vma)->attr.atomic_access :
4723	vma->attr.atomic_access;
4724
4725	if (!IS_DGFX(xe) \|\| !is_atomic)
4726	return false;
4727
4728	/*
4729	* NOTE: The checks implemented here are platform-specific. For
4730	* instance, on a device supporting CXL atomics, these would ideally
4731	* work universally without additional handling.
4732	*/
4733	switch (atomic_access) {
4734	case DRM_XE_ATOMIC_DEVICE:
4735	return !xe->info.has_device_atomics_on_smem;
4736
4737	case DRM_XE_ATOMIC_CPU:
4738	return -EACCES;
4739
4740	case DRM_XE_ATOMIC_UNDEFINED:
4741	case DRM_XE_ATOMIC_GLOBAL:
4742	default:
4743	return 1;
4744	}
4745	}
4746
4747	static int xe_vm_alloc_vma(struct xe_vm *vm,
4748	struct drm_gpuvm_map_req *map_req,
4749	bool is_madvise)
4750	{
4751	struct xe_vma_ops vops;
4752	struct drm_gpuva_ops *ops = NULL;
4753	struct drm_gpuva_op *__op;
4754	unsigned int vma_flags = 0;
4755	bool remap_op = false;
4756	struct xe_vma_mem_attr tmp_attr = {};
4757	u16 default_pat;
4758	int err;
4759
4760	lockdep_assert_held_write(&vm->lock);
4761
4762	if (is_madvise)
4763	ops = drm_gpuvm_madvise_ops_create(&vm->gpuvm, map_req);
4764	else
4765	ops = drm_gpuvm_sm_map_ops_create(&vm->gpuvm, map_req);
4766
4767	if (IS_ERR(ops))
4768	return PTR_ERR(ops);
4769
4770	if (list_empty(&ops->list)) {
4771	err = 0;
4772	goto free_ops;
4773	}
4774
4775	drm_gpuva_for_each_op(__op, ops) {
4776	struct xe_vma_op *op = gpuva_op_to_vma_op(__op);
4777	struct xe_vma *vma = NULL;
4778
4779	if (!is_madvise) {
4780	if (__op->op == DRM_GPUVA_OP_UNMAP) {
4781	vma = gpuva_to_vma(op->base.unmap.va);
4782	XE_WARN_ON(!xe_vma_has_default_mem_attrs(vma));
4783	default_pat = vma->attr.default_pat_index;
4784	vma_flags = vma->gpuva.flags;
4785	}
4786
4787	if (__op->op == DRM_GPUVA_OP_REMAP) {
4788	vma = gpuva_to_vma(op->base.remap.unmap->va);
4789	default_pat = vma->attr.default_pat_index;
4790	vma_flags = vma->gpuva.flags;
4791	}
4792
4793	if (__op->op == DRM_GPUVA_OP_MAP) {
4794	op->map.vma_flags \|= vma_flags & XE_VMA_CREATE_MASK;
4795	op->map.pat_index = default_pat;
4796	}
4797	} else {
4798	if (__op->op == DRM_GPUVA_OP_REMAP) {
4799	vma = gpuva_to_vma(op->base.remap.unmap->va);
4800	xe_assert(vm->xe, !remap_op);
4801	xe_assert(vm->xe, xe_vma_has_no_bo(vma));
4802	remap_op = true;
4803	vma_flags = vma->gpuva.flags;
4804	}
4805
4806	if (__op->op == DRM_GPUVA_OP_MAP) {
4807	xe_assert(vm->xe, remap_op);
4808	remap_op = false;
4809	/*
4810	* In case of madvise ops DRM_GPUVA_OP_MAP is
4811	* always after DRM_GPUVA_OP_REMAP, so ensure
4812	* to propagate the flags from the vma we're
4813	* unmapping.
4814	*/
4815	op->map.vma_flags \|= vma_flags & XE_VMA_CREATE_MASK;
4816	}
4817	}
4818	print_op(vm->xe, __op);
4819	}
4820
4821	xe_vma_ops_init(&vops, vm, NULL, NULL, 0);
4822
4823	if (is_madvise)
4824	vops.flags \|= XE_VMA_OPS_FLAG_MADVISE;
4825	else
4826	vops.flags \|= XE_VMA_OPS_FLAG_ALLOW_SVM_UNMAP;
4827
4828	err = vm_bind_ioctl_ops_parse(vm, ops, &vops);
4829	if (err)
4830	goto unwind_ops;
4831
4832	xe_vm_lock(vm, false);
4833
4834	drm_gpuva_for_each_op(__op, ops) {
4835	struct xe_vma_op *op = gpuva_op_to_vma_op(__op);
4836	struct xe_vma *vma;
4837
4838	if (__op->op == DRM_GPUVA_OP_UNMAP) {
4839	vma = gpuva_to_vma(op->base.unmap.va);
4840	/* There should be no unmap for madvise */
4841	if (is_madvise)
4842	XE_WARN_ON("UNEXPECTED UNMAP");
4843
4844	xe_vma_destroy(vma, NULL);
4845	} else if (__op->op == DRM_GPUVA_OP_REMAP) {
4846	vma = gpuva_to_vma(op->base.remap.unmap->va);
4847	/* In case of madvise ops Store attributes for REMAP UNMAPPED
4848	* VMA, so they can be assigned to newly MAP created vma.
4849	*/
4850	if (is_madvise)
4851	xe_vma_mem_attr_copy(&tmp_attr, &vma->attr);
4852
4853	xe_vma_destroy(gpuva_to_vma(op->base.remap.unmap->va), NULL);
4854	} else if (__op->op == DRM_GPUVA_OP_MAP) {
4855	vma = op->map.vma;
4856	/* In case of madvise call, MAP will always be followed by REMAP.
4857	* Therefore temp_attr will always have sane values, making it safe to
4858	* copy them to new vma.
4859	*/
4860	if (is_madvise)
4861	xe_vma_mem_attr_copy(&vma->attr, &tmp_attr);
4862	}
4863	}
4864
4865	xe_vm_unlock(vm);
4866	drm_gpuva_ops_free(&vm->gpuvm, ops);
4867	xe_vma_mem_attr_fini(&tmp_attr);
4868	return 0;
4869
4870	unwind_ops:
4871	vm_bind_ioctl_ops_unwind(vm, &ops, 1);
4872	free_ops:
4873	drm_gpuva_ops_free(&vm->gpuvm, ops);
4874	return err;
4875	}
4876
4877	/**
4878	* xe_vm_alloc_madvise_vma - Allocate VMA's with madvise ops
4879	* @vm: Pointer to the xe_vm structure
4880	* @start: Starting input address
4881	* @range: Size of the input range
4882	*
4883	* This function splits existing vma to create new vma for user provided input range
4884	*
4885	* Return: 0 if success
4886	*/
4887	int xe_vm_alloc_madvise_vma(struct xe_vm *vm, uint64_t start, uint64_t range)
4888	{
4889	struct drm_gpuvm_map_req map_req = {
4890	.map.va.addr = start,
4891	.map.va.range = range,
4892	};
4893
4894	lockdep_assert_held_write(&vm->lock);
4895
4896	vm_dbg(&vm->xe->drm, "MADVISE_OPS_CREATE: addr=0x%016llx, size=0x%016llx", start, range);
4897
4898	return xe_vm_alloc_vma(vm, &map_req, true);
4899	}
4900
4901	static bool is_cpu_addr_vma_with_default_attr(struct xe_vma *vma)
4902	{
4903	return vma && xe_vma_is_cpu_addr_mirror(vma) &&
4904	xe_vma_has_default_mem_attrs(vma);
4905	}
4906
4907	/**
4908	* xe_vm_find_cpu_addr_mirror_vma_range - Extend a VMA range to include adjacent CPU-mirrored VMAs
4909	* @vm: VM to search within
4910	* @start: Input/output pointer to the starting address of the range
4911	* @end: Input/output pointer to the end address of the range
4912	*
4913	* Given a range defined by @start and @range, this function checks the VMAs
4914	* immediately before and after the range. If those neighboring VMAs are
4915	* CPU-address-mirrored and have default memory attributes, the function
4916	* updates @start and @range to include them. This extended range can then
4917	* be used for merging or other operations that require a unified VMA.
4918	*
4919	* The function does not perform the merge itself; it only computes the
4920	* mergeable boundaries.
4921	*/
4922	void xe_vm_find_cpu_addr_mirror_vma_range(struct xe_vm vm, u64 start, u64 *end)
4923	{
4924	struct xe_vma prev, next;
4925
4926	lockdep_assert_held(&vm->lock);
4927
4928	if (*start >= SZ_4K) {
4929	prev = xe_vm_find_vma_by_addr(vm, *start - SZ_4K);
4930	if (is_cpu_addr_vma_with_default_attr(prev))
4931	*start = xe_vma_start(prev);
4932	}
4933
4934	if (*end < vm->size) {
4935	next = xe_vm_find_vma_by_addr(vm, *end + 1);
4936	if (is_cpu_addr_vma_with_default_attr(next))
4937	*end = xe_vma_end(next);
4938	}
4939	}
4940
4941	/**
4942	* xe_vm_alloc_cpu_addr_mirror_vma - Allocate CPU addr mirror vma
4943	* @vm: Pointer to the xe_vm structure
4944	* @start: Starting input address
4945	* @range: Size of the input range
4946	*
4947	* This function splits/merges existing vma to create new vma for user provided input range
4948	*
4949	* Return: 0 if success
4950	*/
4951	int xe_vm_alloc_cpu_addr_mirror_vma(struct xe_vm *vm, uint64_t start, uint64_t range)
4952	{
4953	struct drm_gpuvm_map_req map_req = {
4954	.map.va.addr = start,
4955	.map.va.range = range,
4956	};
4957
4958	lockdep_assert_held_write(&vm->lock);
4959
4960	vm_dbg(&vm->xe->drm, "CPU_ADDR_MIRROR_VMA_OPS_CREATE: addr=0x%016llx, size=0x%016llx",
4961	start, range);
4962
4963	return xe_vm_alloc_vma(vm, &map_req, false);
4964	}
4965
4966	/**
4967	* xe_vm_add_exec_queue() - Add exec queue to VM
4968	* @vm: The VM.
4969	* @q: The exec_queue
4970	*
4971	* Add exec queue to VM, skipped if the device does not have context based TLB
4972	* invalidations.
4973	*/
4974	void xe_vm_add_exec_queue(struct xe_vm vm, struct xe_exec_queue q)
4975	{
4976	struct xe_device *xe = vm->xe;
4977
4978	/* User VMs and queues only */
4979	xe_assert(xe, !(q->flags & EXEC_QUEUE_FLAG_KERNEL));
4980	xe_assert(xe, !(q->flags & EXEC_QUEUE_FLAG_PERMANENT));
4981	xe_assert(xe, !(q->flags & EXEC_QUEUE_FLAG_VM));
4982	xe_assert(xe, !(q->flags & EXEC_QUEUE_FLAG_MIGRATE));
4983	xe_assert(xe, vm->xef);
4984	xe_assert(xe, vm == q->vm);
4985
4986	if (!xe->info.has_ctx_tlb_inval)
4987	return;
4988
4989	down_write(&vm->exec_queues.lock);
4990	list_add(&q->vm_exec_queue_link, &vm->exec_queues.list[q->gt->info.id]);
4991	++vm->exec_queues.count[q->gt->info.id];
4992	up_write(&vm->exec_queues.lock);
4993	}
4994
4995	/**
4996	* xe_vm_remove_exec_queue() - Remove exec queue from VM
4997	* @vm: The VM.
4998	* @q: The exec_queue
4999	*
5000	* Remove exec queue from VM, skipped if the device does not have context based
5001	* TLB invalidations.
5002	*/
5003	void xe_vm_remove_exec_queue(struct xe_vm vm, struct xe_exec_queue q)
5004	{
5005	if (!vm->xe->info.has_ctx_tlb_inval)
5006	return;
5007
5008	down_write(&vm->exec_queues.lock);
5009	if (!list_empty(&q->vm_exec_queue_link)) {
5010	list_del(&q->vm_exec_queue_link);
5011	--vm->exec_queues.count[q->gt->info.id];
5012	}
5013	up_write(&vm->exec_queues.lock);
5014	}