projects/linux·blob master · refs

429 lines · 14.9 KiB

1// SPDX-License-Identifier: GPL-2.0 OR MIT
2
3use super::*;
4
5/// The actual data that gets threaded through the callbacks.
6struct SmData<'a, 'ctx, T: DriverGpuVm + 'ctx> {
7 gpuvm: &'a mut UniqueRefGpuVm<T>,
8 user_context: &'a mut T::SmContext<'ctx>,
9}
10
11/// Adds an extra field to `SmData` for `sm_map()` callbacks.
12///
13/// # Invariants
14///
15/// `self.vm_bo.gpuvm() == self.sm_data.gpuvm`.
16#[repr(C)]
17struct SmMapData<'a, 'ctx, T: DriverGpuVm> {
18 sm_data: SmData<'a, 'ctx, T>,
19 vm_bo: &'a GpuVmBo<T>,
20}
21
22/// The argument for [`UniqueRefGpuVm::sm_map`].
23pub struct OpMapRequest<'a, 'ctx, T: DriverGpuVm + 'ctx> {
24 /// Address in GPU virtual address space.
25 pub addr: u64,
26 /// Length of mapping to create.
27 pub range: u64,
28 /// Offset in GEM object.
29 pub gem_offset: u64,
30 /// The GEM object to map.
31 pub vm_bo: &'a GpuVmBo<T>,
32 /// The user-provided context type.
33 pub context: &'a mut T::SmContext<'ctx>,
34}
35
36impl<'a, 'ctx, T: DriverGpuVm> OpMapRequest<'a, 'ctx, T> {
37 fn raw_request(&self) -> bindings::drm_gpuvm_map_req {
38 bindings::drm_gpuvm_map_req {
39 map: bindings::drm_gpuva_op_map {
40 va: bindings::drm_gpuva_op_map__bindgen_ty_1 {
41 addr: self.addr,
42 range: self.range,
43 },
44 gem: bindings::drm_gpuva_op_map__bindgen_ty_2 {
45 offset: self.gem_offset,
46 obj: self.vm_bo.obj().as_raw(),
47 },
48 },
49 }
50 }
51}
52
53/// Represents an `sm_step_map` operation that has not yet been completed.
54pub struct OpMap<'op, T: DriverGpuVm> {
55 op: &'op bindings::drm_gpuva_op_map,
56 // Since these abstractions are designed for immediate mode, the VM BO needs to be
57 // pre-allocated, so we always have it available when we reach this point.
58 vm_bo: &'op GpuVmBo<T>,
59 // This ensures that 'op is invariant, so that `OpMap<'long, T>` does not
60 // coerce to `OpMap<'short, T>`. This ensures that the user can't return
61 // the wrong `OpMapped` value.
62 _invariant: PhantomData<*mut &'op mut T>,
63}
64
65impl<'op, T: DriverGpuVm> OpMap<'op, T> {
66 /// The base address of the new mapping.
67 pub fn addr(&self) -> u64 {
68 self.op.va.addr
69 }
70
71 /// The length of the new mapping.
72 pub fn length(&self) -> u64 {
73 self.op.va.range
74 }
75
76 /// The offset within the [`drm_gem_object`](DriverGpuVm::Object).
77 pub fn gem_offset(&self) -> u64 {
78 self.op.gem.offset
79 }
80
81 /// The [`drm_gem_object`](DriverGpuVm::Object) to map.
82 pub fn obj(&self) -> &T::Object {
83 // SAFETY: The `obj` pointer is guaranteed to be valid.
84 unsafe { <T::Object as IntoGEMObject>::from_raw(self.op.gem.obj) }
85 }
86
87 /// The [`GpuVmBo`] that the new VA will be associated with.
88 pub fn vm_bo(&self) -> &GpuVmBo<T> {
89 self.vm_bo
90 }
91
92 /// Use the pre-allocated VA to carry out this map operation.
93 pub fn insert(self, va: GpuVaAlloc<T>, va_data: impl PinInit<T::VaData>) -> OpMapped<'op, T> {
94 let va = va.prepare(va_data);
95 // SAFETY: By the type invariants we may access the interval tree.
96 unsafe { bindings::drm_gpuva_map(self.vm_bo.gpuvm().as_raw(), va, self.op) };
97
98 let _gpuva_guard = self.vm_bo().lock_gpuva();
99 // SAFETY: The va is prepared for insertion, and we hold the GEM lock.
100 unsafe { bindings::drm_gpuva_link(va, self.vm_bo.as_raw()) };
101
102 OpMapped {
103 _invariant: self._invariant,
104 }
105 }
106}
107
108/// Represents a completed [`OpMap`] operation.
109pub struct OpMapped<'op, T> {
110 _invariant: PhantomData<*mut &'op mut T>,
111}
112
113/// Represents an `sm_step_unmap` operation that has not yet been completed.
114pub struct OpUnmap<'op, T: DriverGpuVm> {
115 op: &'op bindings::drm_gpuva_op_unmap,
116 // This ensures that 'op is invariant, so that `OpUnmap<'long, T>` does not
117 // coerce to `OpUnmap<'short, T>`. This ensures that the user can't return the
118 // wrong`OpUnmapped` value.
119 _invariant: PhantomData<*mut &'op mut T>,
120}
121
122impl<'op, T: DriverGpuVm> OpUnmap<'op, T> {
123 /// Indicates whether this [`GpuVa`] is physically contiguous with the
124 /// original mapping request.
125 ///
126 /// Optionally, if `keep` is set, drivers may keep the actual page table
127 /// mappings for this `drm_gpuva`, adding the missing page table entries
128 /// only and update the `drm_gpuvm` accordingly.
129 pub fn keep(&self) -> bool {
130 self.op.keep
131 }
132
133 /// The range being unmapped.
134 pub fn va(&self) -> &GpuVa<T> {
135 // SAFETY: This is a valid va. It's not the `kernel_alloc_node` because you can't unmap it,
136 // and it's not sparse by the `GpuVm<T>` type invariants.
137 unsafe { GpuVa::<T>::from_raw(self.op.va) }
138 }
139
140 /// Remove the VA.
141 pub fn remove(self) -> (OpUnmapped<'op, T>, GpuVaRemoved<T>) {
142 // SAFETY: The op references a valid drm_gpuva in the GPUVM.
143 unsafe { bindings::drm_gpuva_unmap(self.op) };
144 // SAFETY: The va is no longer in the interval tree so we may unlink it.
145 unsafe { bindings::drm_gpuva_unlink_defer(self.op.va) };
146
147 // SAFETY: We just removed this va from the `GpuVm<T>`.
148 let va = unsafe { GpuVaRemoved::from_raw(self.op.va) };
149
150 (
151 OpUnmapped {
152 _invariant: self._invariant,
153 },
154 va,
155 )
156 }
157}
158
159/// Represents a completed [`OpUnmap`] operation.
160pub struct OpUnmapped<'op, T> {
161 _invariant: PhantomData<*mut &'op mut T>,
162}
163
164/// Represents an `sm_step_remap` operation that has not yet been completed.
165pub struct OpRemap<'op, T: DriverGpuVm> {
166 op: &'op bindings::drm_gpuva_op_remap,
167 // This ensures that 'op is invariant, so that `OpRemap<'long, T>` does not
168 // coerce to `OpRemap<'short, T>`. This ensures that the user can't return the
169 // wrong`OpRemapped` value.
170 _invariant: PhantomData<*mut &'op mut T>,
171}
172
173impl<'op, T: DriverGpuVm> OpRemap<'op, T> {
174 /// The preceding part of a split mapping.
175 #[inline]
176 pub fn prev(&self) -> Option<&OpRemapMapData> {
177 // SAFETY: We checked for null, so the pointer must be valid.
178 NonNull::new(self.op.prev).map(|ptr| unsafe { OpRemapMapData::from_raw(ptr) })
179 }
180
181 /// The subsequent part of a split mapping.
182 #[inline]
183 pub fn next(&self) -> Option<&OpRemapMapData> {
184 // SAFETY: We checked for null, so the pointer must be valid.
185 NonNull::new(self.op.next).map(|ptr| unsafe { OpRemapMapData::from_raw(ptr) })
186 }
187
188 /// Indicates whether the `drm_gpuva` being removed is physically contiguous with the original
189 /// mapping request.
190 ///
191 /// Optionally, if `keep` is set, drivers may keep the actual page table mappings for this
192 /// `drm_gpuva`, adding the missing page table entries only and update the `drm_gpuvm`
193 /// accordingly.
194 #[inline]
195 pub fn keep(&self) -> bool {
196 // SAFETY: The unmap pointer is always valid.
197 unsafe { (*self.op.unmap).keep }
198 }
199
200 /// The range being unmapped.
201 #[inline]
202 pub fn va_to_unmap(&self) -> &GpuVa<T> {
203 // SAFETY: This is a valid va. It's not the `kernel_alloc_node` because you can't unmap it,
204 // and it's not sparse by the `GpuVm<T>` type invariants.
205 unsafe { GpuVa::<T>::from_raw((*self.op.unmap).va) }
206 }
207
208 /// The [`drm_gem_object`](DriverGpuVm::Object) whose VA is being remapped.
209 #[inline]
210 pub fn obj(&self) -> &T::Object {
211 self.va_to_unmap().obj()
212 }
213
214 /// The [`GpuVmBo`] that is being remapped.
215 #[inline]
216 pub fn vm_bo(&self) -> &GpuVmBo<T> {
217 self.va_to_unmap().vm_bo()
218 }
219
220 /// Update the GPUVM to perform the remapping.
221 pub fn remap(
222 self,
223 va_alloc: [GpuVaAlloc<T>; 2],
224 prev_data: impl PinInit<T::VaData>,
225 next_data: impl PinInit<T::VaData>,
226 ) -> (OpRemapped<'op, T>, OpRemapRet<T>) {
227 let [va1, va2] = va_alloc;
228
229 let mut unused_va = None;
230 let mut prev_ptr = ptr::null_mut();
231 let mut next_ptr = ptr::null_mut();
232 if self.prev().is_some() {
233 prev_ptr = va1.prepare(prev_data);
234 } else {
235 unused_va = Some(va1);
236 }
237 if self.next().is_some() {
238 next_ptr = va2.prepare(next_data);
239 } else {
240 unused_va = Some(va2);
241 }
242
243 // SAFETY: the pointers are non-null when required
244 unsafe { bindings::drm_gpuva_remap(prev_ptr, next_ptr, self.op) };
245
246 let gpuva_guard = self.vm_bo().lock_gpuva();
247 if !prev_ptr.is_null() {
248 // SAFETY: The prev_ptr is a valid drm_gpuva prepared for insertion. The vm_bo is still
249 // valid as the not-yet-unlinked gpuva holds a refcount on the vm_bo.
250 unsafe { bindings::drm_gpuva_link(prev_ptr, self.vm_bo().as_raw()) };
251 }
252 if !next_ptr.is_null() {
253 // SAFETY: The next_ptr is a valid drm_gpuva prepared for insertion. The vm_bo is still
254 // valid as the not-yet-unlinked gpuva holds a refcount on the vm_bo.
255 unsafe { bindings::drm_gpuva_link(next_ptr, self.vm_bo().as_raw()) };
256 }
257 drop(gpuva_guard);
258
259 // SAFETY: The va is no longer in the interval tree so we may unlink it.
260 unsafe { bindings::drm_gpuva_unlink_defer((*self.op.unmap).va) };
261
262 (
263 OpRemapped {
264 _invariant: self._invariant,
265 },
266 OpRemapRet {
267 // SAFETY: We just removed this va from the `GpuVm<T>`.
268 unmapped_va: unsafe { GpuVaRemoved::from_raw((*self.op.unmap).va) },
269 unused_va,
270 },
271 )
272 }
273}
274
275/// Part of an [`OpRemap`] that represents a new mapping.
276#[repr(transparent)]
277pub struct OpRemapMapData(bindings::drm_gpuva_op_map);
278
279impl OpRemapMapData {
280 /// # Safety
281 /// Must reference a valid `drm_gpuva_op_map` for duration of `'a`.
282 unsafe fn from_raw<'a>(ptr: NonNull<bindings::drm_gpuva_op_map>) -> &'a Self {
283 // SAFETY: ok per safety requirements
284 unsafe { ptr.cast().as_ref() }
285 }
286
287 /// The base address of the new mapping.
288 pub fn addr(&self) -> u64 {
289 self.0.va.addr
290 }
291
292 /// The length of the new mapping.
293 pub fn length(&self) -> u64 {
294 self.0.va.range
295 }
296
297 /// The offset within the [`drm_gem_object`](DriverGpuVm::Object).
298 pub fn gem_offset(&self) -> u64 {
299 self.0.gem.offset
300 }
301}
302
303/// Struct containing objects removed or not used by [`OpRemap::remap`].
304pub struct OpRemapRet<T: DriverGpuVm> {
305 /// The `drm_gpuva` that was removed.
306 pub unmapped_va: GpuVaRemoved<T>,
307 /// If the remap did not split the region into two pieces, then the unused `drm_gpuva` is
308 /// returned here.
309 pub unused_va: Option<GpuVaAlloc<T>>,
310}
311
312/// Represents a completed [`OpRemap`] operation.
313pub struct OpRemapped<'op, T> {
314 _invariant: PhantomData<*mut &'op mut T>,
315}
316
317impl<T: DriverGpuVm> UniqueRefGpuVm<T> {
318 /// Create a mapping, removing or remapping anything that overlaps.
319 ///
320 /// Internally calls the [`DriverGpuVm`] callbacks similar to [`Self::sm_unmap`], except that
321 /// the [`DriverGpuVm::sm_step_map`] is called once to create the requested mapping.
322 #[inline]
323 pub fn sm_map(&mut self, req: OpMapRequest<'_, '_, T>) -> Result {
324 if req.vm_bo.gpuvm() != &**self {
325 return Err(EINVAL);
326 }
327
328 let gpuvm = self.as_raw();
329 let raw_req = req.raw_request();
330 // INVARIANT: Checked above that `vm_bo.gpuvm() == self`.
331 let mut p = SmMapData {
332 sm_data: SmData {
333 gpuvm: self,
334 user_context: req.context,
335 },
336 vm_bo: req.vm_bo,
337 };
338 // SAFETY:
339 // * raw_request() creates a valid request.
340 // * The private data is valid to be interpreted as both SmData and SmMapData since the
341 // first field of SmMapData is SmData.
342 to_result(unsafe {
343 bindings::drm_gpuvm_sm_map(gpuvm, (&raw mut p).cast(), &raw const raw_req)
344 })
345 }
346
347 /// Remove any mappings in the given region.
348 ///
349 /// Internally calls [`DriverGpuVm::sm_step_unmap`] for ranges entirely contained within the
350 /// given range, and [`DriverGpuVm::sm_step_remap`] for ranges that overlap with the range.
351 #[inline]
352 pub fn sm_unmap(&mut self, addr: u64, length: u64, context: &mut T::SmContext<'_>) -> Result {
353 let gpuvm = self.as_raw();
354 let mut p = SmData {
355 gpuvm: self,
356 user_context: context,
357 };
358 // SAFETY:
359 // * raw_request() creates a valid request.
360 // * The private data is a valid SmData.
361 to_result(unsafe { bindings::drm_gpuvm_sm_unmap(gpuvm, (&raw mut p).cast(), addr, length) })
362 }
363}
364
365impl<T: DriverGpuVm> GpuVm<T> {
366 /// # Safety
367 /// Must be called from `sm_map` with a pointer to `SmMapData`.
368 pub(super) unsafe extern "C" fn sm_step_map(
369 op: *mut bindings::drm_gpuva_op,
370 p: *mut c_void,
371 ) -> c_int {
372 // SAFETY: If we reach `sm_step_map` then we were called from `sm_map` which always passes
373 // an `SmMapData` as private data.
374 let p = unsafe { &mut *p.cast::<SmMapData<'_, '_, T>>() };
375 let op = OpMap {
376 // SAFETY: sm_step_map is called with a map operation.
377 op: unsafe { &(*op).__bindgen_anon_1.map },
378 vm_bo: p.vm_bo,
379 _invariant: PhantomData,
380 };
381 match p
382 .sm_data
383 .gpuvm
384 .data()
385 .sm_step_map(op, p.sm_data.user_context)
386 {
387 Ok(OpMapped { .. }) => 0,
388 Err(err) => err.to_errno(),
389 }
390 }
391
392 /// # Safety
393 /// Must be called from `sm_map` or `sm_unmap` with a pointer to `SmMapData` or `SmData`.
394 pub(super) unsafe extern "C" fn sm_step_unmap(
395 op: *mut bindings::drm_gpuva_op,
396 p: *mut c_void,
397 ) -> c_int {
398 // SAFETY: The caller provides a pointer that can be treated as `SmData`.
399 let p = unsafe { &mut *p.cast::<SmData<'_, '_, T>>() };
400 let op = OpUnmap {
401 // SAFETY: sm_step_unmap is called with an unmap operation.
402 op: unsafe { &(*op).__bindgen_anon_1.unmap },
403 _invariant: PhantomData,
404 };
405 match p.gpuvm.data().sm_step_unmap(op, p.user_context) {
406 Ok(OpUnmapped { .. }) => 0,
407 Err(err) => err.to_errno(),
408 }
409 }
410
411 /// # Safety
412 /// Must be called from `sm_map` or `sm_unmap` with a pointer to `SmMapData` or `SmData`.
413 pub(super) unsafe extern "C" fn sm_step_remap(
414 op: *mut bindings::drm_gpuva_op,
415 p: *mut c_void,
416 ) -> c_int {
417 // SAFETY: The caller provides a pointer that can be treated as `SmData`.
418 let p = unsafe { &mut *p.cast::<SmData<'_, '_, T>>() };
419 let op = OpRemap {
420 // SAFETY: sm_step_remap is called with a remap operation.
421 op: unsafe { &(*op).__bindgen_anon_1.remap },
422 _invariant: PhantomData,
423 };
424 match p.gpuvm.data().sm_step_remap(op, p.user_context) {
425 Ok(OpRemapped { .. }) => 0,
426 Err(err) => err.to_errno(),
427 }
428 }
429}