projects/linux·blob master · refs

334 lines · 10.7 KiB

1// SPDX-License-Identifier: GPL-2.0 OR MIT
2
3#![cfg(CONFIG_RUST_DRM_GPUVM)]
4
5//! DRM GPUVM in immediate mode
6//!
7//! Rust abstractions for using GPUVM in immediate mode. This is when the GPUVM state is updated
8//! during `run_job()`, i.e., in the DMA fence signalling critical path, to ensure that the GPUVM
9//! and the GPU's virtual address space has the same state at all times.
10//!
11//! C header: [`include/drm/drm_gpuvm.h`](srctree/include/drm/drm_gpuvm.h)
12
13use kernel::{
14 alloc::{
15 AllocError,
16 Flags as AllocFlags, //
17 },
18 bindings,
19 drm,
20 drm::gem::IntoGEMObject,
21 error::to_result,
22 prelude::*,
23 sync::aref::{
24 ARef,
25 AlwaysRefCounted, //
26 },
27 types::Opaque, //
28};
29
30use core::{
31 cell::UnsafeCell,
32 marker::PhantomData,
33 mem::{
34 ManuallyDrop,
35 MaybeUninit, //
36 },
37 ops::{
38 Deref,
39 DerefMut,
40 Range, //
41 },
42 ptr::{
43 self,
44 NonNull, //
45 }, //
46};
47
48mod sm_ops;
49pub use self::sm_ops::*;
50
51mod vm_bo;
52pub use self::vm_bo::*;
53
54mod va;
55pub use self::va::*;
56
57/// A DRM GPU VA manager.
58///
59/// This object is refcounted, but the locations of mapped ranges may only be accessed or changed
60/// via the special unique handle [`UniqueRefGpuVm`].
61///
62/// # Invariants
63///
64/// * Stored in an allocation managed by the refcount in `self.vm`.
65/// * Access to `data` and the gpuvm interval tree is controlled via the [`UniqueRefGpuVm`] type.
66/// * Does not contain any sparse [`GpuVa<T>`] instances.
67#[pin_data]
68pub struct GpuVm<T: DriverGpuVm> {
69 #[pin]
70 vm: Opaque<bindings::drm_gpuvm>,
71 /// Accessed only through the [`UniqueRefGpuVm`] reference.
72 data: UnsafeCell<T>,
73}
74
75// SAFETY: It is safe to send a `GpuVm<T>` to another thread: all data reachable through it
76// (`T`, `T::VmBoData`, and the GEM `T::Object`) is `Send` by the `DriverGpuVm` bounds.
77unsafe impl<T: DriverGpuVm> Send for GpuVm<T> {}
78// SAFETY: It is safe to share a `&GpuVm<T>` between threads: `&self` methods only alias data
79// that is `Sync` by the `DriverGpuVm` bounds, and any thread may drop that data, or upgrade the
80// reference and ultimately drop `T`, which the same bounds make `Send`.
81unsafe impl<T: DriverGpuVm> Sync for GpuVm<T> {}
82
83// SAFETY: By type invariants, the allocation is managed by the refcount in `self.vm`.
84unsafe impl<T: DriverGpuVm> AlwaysRefCounted for GpuVm<T> {
85 fn inc_ref(&self) {
86 // SAFETY: By type invariants, the allocation is managed by the refcount in `self.vm`.
87 unsafe { bindings::drm_gpuvm_get(self.vm.get()) };
88 }
89
90 unsafe fn dec_ref(obj: NonNull<Self>) {
91 // SAFETY: By type invariants, the allocation is managed by the refcount in `self.vm`.
92 unsafe { bindings::drm_gpuvm_put((*obj.as_ptr()).vm.get()) };
93 }
94}
95
96impl<T: DriverGpuVm> PartialEq for GpuVm<T> {
97 #[inline]
98 fn eq(&self, other: &Self) -> bool {
99 core::ptr::eq(self.as_raw(), other.as_raw())
100 }
101}
102impl<T: DriverGpuVm> Eq for GpuVm<T> {}
103
104impl<T: DriverGpuVm> GpuVm<T> {
105 const fn vtable() -> &'static bindings::drm_gpuvm_ops {
106 &bindings::drm_gpuvm_ops {
107 vm_free: Some(Self::vm_free),
108 op_alloc: None,
109 op_free: None,
110 vm_bo_alloc: GpuVmBo::<T>::ALLOC_FN,
111 vm_bo_free: GpuVmBo::<T>::FREE_FN,
112 vm_bo_validate: None,
113 sm_step_map: Some(Self::sm_step_map),
114 sm_step_unmap: Some(Self::sm_step_unmap),
115 sm_step_remap: Some(Self::sm_step_remap),
116 }
117 }
118
119 /// Creates a GPUVM instance.
120 #[expect(clippy::new_ret_no_self)]
121 pub fn new<E, Ctx: drm::DeviceContext>(
122 name: &'static CStr,
123 dev: &drm::Device<T::Driver, Ctx>,
124 r_obj: &T::Object,
125 range: Range<u64>,
126 reserve_range: Range<u64>,
127 data: T,
128 ) -> Result<UniqueRefGpuVm<T>, E>
129 where
130 E: From<AllocError>,
131 E: From<core::convert::Infallible>,
132 {
133 let obj = KBox::try_pin_init::<E>(
134 try_pin_init!(Self {
135 data: UnsafeCell::new(data),
136 vm <- Opaque::ffi_init(|vm| {
137 // SAFETY: These arguments are valid. `vm` is valid until refcount drops to
138 // zero. The `vm` is zeroed before calling this method by `__GFP_ZERO` flag
139 // below.
140 unsafe {
141 bindings::drm_gpuvm_init(
142 vm,
143 name.as_char_ptr(),
144 bindings::drm_gpuvm_flags_DRM_GPUVM_IMMEDIATE_MODE
145 | bindings::drm_gpuvm_flags_DRM_GPUVM_RESV_PROTECTED,
146 dev.as_raw(),
147 r_obj.as_raw(),
148 range.start,
149 range.end - range.start,
150 reserve_range.start,
151 reserve_range.end - reserve_range.start,
152 const { Self::vtable() },
153 )
154 }
155 }),
156 }? E),
157 GFP_KERNEL | __GFP_ZERO,
158 )?;
159 // SAFETY: This transfers the initial refcount to the ARef.
160 let aref = unsafe {
161 ARef::from_raw(NonNull::new_unchecked(KBox::into_raw(
162 Pin::into_inner_unchecked(obj),
163 )))
164 };
165 // INVARIANT: This reference is unique.
166 Ok(UniqueRefGpuVm(aref))
167 }
168
169 /// Access this [`GpuVm`] from a raw pointer.
170 ///
171 /// # Safety
172 ///
173 /// The pointer must reference the `struct drm_gpuvm` in a valid [`GpuVm<T>`] that remains
174 /// valid for at least `'a`.
175 #[inline]
176 pub unsafe fn from_raw<'a>(ptr: *mut bindings::drm_gpuvm) -> &'a Self {
177 // SAFETY: Caller passes a pointer to the `drm_gpuvm` in a `GpuVm<T>`. Caller ensures the
178 // pointer is valid for 'a.
179 unsafe { &*kernel::container_of!(Opaque::cast_from(ptr), Self, vm) }
180 }
181
182 /// Returns a raw pointer to the embedded `struct drm_gpuvm`.
183 #[inline]
184 pub fn as_raw(&self) -> *mut bindings::drm_gpuvm {
185 self.vm.get()
186 }
187
188 /// The start of the VA space.
189 #[inline]
190 pub fn va_start(&self) -> u64 {
191 // SAFETY: The `mm_start` field is immutable.
192 unsafe { (*self.as_raw()).mm_start }
193 }
194
195 /// The length of the GPU's virtual address space.
196 #[inline]
197 pub fn va_length(&self) -> u64 {
198 // SAFETY: The `mm_range` field is immutable.
199 unsafe { (*self.as_raw()).mm_range }
200 }
201
202 /// Returns the range of the GPU virtual address space.
203 #[inline]
204 pub fn va_range(&self) -> Range<u64> {
205 let start = self.va_start();
206 // OVERFLOW: This reconstructs the Range<u64> passed to the constructor, so it won't fail.
207 let end = start + self.va_length();
208 Range { start, end }
209 }
210
211 /// Get or create the [`GpuVmBo`] for this gem object.
212 #[inline]
213 pub fn obtain(
214 &self,
215 obj: &T::Object,
216 data: impl PinInit<T::VmBoData>,
217 ) -> Result<ARef<GpuVmBo<T>>, AllocError> {
218 Ok(GpuVmBoAlloc::new(self, obj, data)?.obtain())
219 }
220
221 /// Clean up buffer objects that are no longer used.
222 #[inline]
223 pub fn deferred_cleanup(&self) {
224 // SAFETY: This GPUVM uses immediate mode.
225 unsafe { bindings::drm_gpuvm_bo_deferred_cleanup(self.as_raw()) }
226 }
227
228 /// Check if this GEM object is an external object for this GPUVM.
229 #[inline]
230 pub fn is_extobj(&self, obj: &T::Object) -> bool {
231 // SAFETY: We may call this with any GPUVM and GEM object.
232 unsafe { bindings::drm_gpuvm_is_extobj(self.as_raw(), obj.as_raw()) }
233 }
234
235 /// Free this GPUVM.
236 ///
237 /// # Safety
238 ///
239 /// Called when refcount hits zero.
240 unsafe extern "C" fn vm_free(me: *mut bindings::drm_gpuvm) {
241 // SAFETY: Caller passes a pointer to the `drm_gpuvm` in a `GpuVm<T>`.
242 let me = unsafe { kernel::container_of!(Opaque::cast_from(me), Self, vm).cast_mut() };
243 // SAFETY: By type invariants we can free it when refcount hits zero.
244 drop(unsafe { KBox::from_raw(me) })
245 }
246
247 #[inline]
248 fn raw_resv(&self) -> *mut bindings::dma_resv {
249 // SAFETY: `r_obj` is immutable and valid for duration of GPUVM.
250 unsafe { (*(*self.as_raw()).r_obj).resv }
251 }
252}
253
254/// The manager for a GPUVM.
255pub trait DriverGpuVm: Sized + Send + Sync {
256 /// Parent `Driver` for this object.
257 type Driver: drm::Driver;
258
259 /// The kind of GEM object stored in this GPUVM.
260 type Object: drm::driver::AllocImpl<Driver = Self::Driver> + Send + Sync;
261
262 /// Data stored with each [`struct drm_gpuva`](struct@GpuVa).
263 ///
264 /// Only `Send` is required: the data has a single owner at all times, moving
265 /// between threads by value (handed back as a [`GpuVaRemoved`]) but never
266 /// accessed by two threads concurrently.
267 type VaData: Send;
268
269 /// Data stored with each [`struct drm_gpuvm_bo`](struct@GpuVmBo).
270 type VmBoData: Send + Sync;
271
272 /// The private data passed to callbacks.
273 type SmContext<'ctx>
274 where
275 Self: 'ctx;
276
277 /// Indicates that a new mapping should be created.
278 fn sm_step_map<'op, 'ctx>(
279 &mut self,
280 op: OpMap<'op, Self>,
281 context: &mut Self::SmContext<'ctx>,
282 ) -> Result<OpMapped<'op, Self>, Error>;
283
284 /// Indicates that an existing mapping should be removed.
285 fn sm_step_unmap<'op, 'ctx>(
286 &mut self,
287 op: OpUnmap<'op, Self>,
288 context: &mut Self::SmContext<'ctx>,
289 ) -> Result<OpUnmapped<'op, Self>, Error>;
290
291 /// Indicates that an existing mapping should be split up.
292 fn sm_step_remap<'op, 'ctx>(
293 &mut self,
294 op: OpRemap<'op, Self>,
295 context: &mut Self::SmContext<'ctx>,
296 ) -> Result<OpRemapped<'op, Self>, Error>;
297}
298
299/// The core of the DRM GPU VA manager.
300///
301/// This object is a unique reference to the VM that can access the interval tree and the Rust
302/// `data` field.
303///
304/// # Invariants
305///
306/// Each `GpuVm` instance has at most one `UniqueRefGpuVm` reference.
307// `Send`/`Sync` derive from `ARef<GpuVm<T>>`; the trait bounds make them correct for the unique
308// handle's `&mut T` access.
309pub struct UniqueRefGpuVm<T: DriverGpuVm>(ARef<GpuVm<T>>);
310
311impl<T: DriverGpuVm> UniqueRefGpuVm<T> {
312 /// Access the data owned by this `UniqueRefGpuVm` immutably.
313 #[inline]
314 pub fn data_ref(&self) -> &T {
315 // SAFETY: By the type invariants we may access `data`.
316 unsafe { &*self.0.data.get() }
317 }
318
319 /// Access the data owned by this `UniqueRefGpuVm` mutably.
320 #[inline]
321 pub fn data(&mut self) -> &mut T {
322 // SAFETY: By the type invariants we may access `data`.
323 unsafe { &mut *self.0.data.get() }
324 }
325}
326
327impl<T: DriverGpuVm> Deref for UniqueRefGpuVm<T> {
328 type Target = GpuVm<T>;
329
330 #[inline]
331 fn deref(&self) -> &GpuVm<T> {
332 &self.0
333 }
334}