projects/linux·blob master · refs

258 lines · 10.0 KiB

1// SPDX-License-Identifier: GPL-2.0 OR MIT
2
3use super::*;
4
5/// Represents that a given GEM object has at least one mapping on this [`GpuVm`] instance.
6///
7/// Does not assume that GEM lock is held.
8///
9/// # Invariants
10///
11/// * Allocated with `kmalloc` and refcounted via `inner`.
12/// * Is present in the gem list.
13#[repr(C)]
14#[pin_data]
15pub struct GpuVmBo<T: DriverGpuVm> {
16 #[pin]
17 inner: Opaque<bindings::drm_gpuvm_bo>,
18 #[pin]
19 data: T::VmBoData,
20}
21
22// SAFETY: It is safe to send a `GpuVmBo<T>` to another thread: dropping it there drops
23// `T::VmBoData` and the GEM `T::Object`, both `Send` by the `DriverGpuVm` bounds.
24unsafe impl<T: DriverGpuVm> Send for GpuVmBo<T> {}
25
26// SAFETY: It is safe to share a `&GpuVmBo<T>` between threads: it effectively shares
27// `&T::VmBoData` and the GEM `&T::Object` (both `Sync`), and any thread may upgrade to an
28// `ARef` and ultimately drop them (both `Send`), per the `DriverGpuVm` bounds.
29unsafe impl<T: DriverGpuVm> Sync for GpuVmBo<T> {}
30
31// SAFETY: By type invariants, the allocation is managed by the refcount in `self.inner`.
32unsafe impl<T: DriverGpuVm> AlwaysRefCounted for GpuVmBo<T> {
33 fn inc_ref(&self) {
34 // SAFETY: By type invariants, the allocation is managed by the refcount in `self.inner`.
35 unsafe { bindings::drm_gpuvm_bo_get(self.inner.get()) };
36 }
37
38 unsafe fn dec_ref(obj: NonNull<Self>) {
39 // CAST: `drm_gpuvm_bo` is first field of repr(C) struct.
40 // SAFETY: By type invariants, the allocation is managed by the refcount in `self.inner`.
41 // This GPUVM instance uses immediate mode, so we may put the refcount using the deferred
42 // mechanism.
43 unsafe { bindings::drm_gpuvm_bo_put_deferred(obj.as_ptr().cast()) };
44 }
45}
46
47impl<T: DriverGpuVm> PartialEq for GpuVmBo<T> {
48 #[inline]
49 fn eq(&self, other: &Self) -> bool {
50 core::ptr::eq(self.as_raw(), other.as_raw())
51 }
52}
53impl<T: DriverGpuVm> Eq for GpuVmBo<T> {}
54
55impl<T: DriverGpuVm> GpuVmBo<T> {
56 /// The function pointer for allocating a GpuVmBo stored in the gpuvm vtable.
57 ///
58 /// Allocation is always implemented according to [`Self::vm_bo_alloc`], but it is set to
59 /// `None` if the default gpuvm behavior is the same as `vm_bo_alloc`.
60 ///
61 /// This may be `Some` even if `FREE_FN` is `None`, or vice-versa.
62 pub(super) const ALLOC_FN: Option<unsafe extern "C" fn() -> *mut bindings::drm_gpuvm_bo> = {
63 use core::alloc::Layout;
64 let base = Layout::new::<bindings::drm_gpuvm_bo>();
65 let rust = Layout::new::<Self>();
66 assert!(base.size() <= rust.size());
67 if base.size() != rust.size() || base.align() != rust.align() {
68 Some(Self::vm_bo_alloc)
69 } else {
70 // This causes GPUVM to allocate a `GpuVmBo<T>` with `kzalloc(sizeof(drm_gpuvm_bo))`.
71 None
72 }
73 };
74
75 /// The function pointer for freeing a GpuVmBo stored in the gpuvm vtable.
76 ///
77 /// Freeing is always implemented according to [`Self::vm_bo_free`], but it is set to `None` if
78 /// the default gpuvm behavior is the same as `vm_bo_free`.
79 ///
80 /// This may be `Some` even if `ALLOC_FN` is `None`, or vice-versa.
81 pub(super) const FREE_FN: Option<unsafe extern "C" fn(*mut bindings::drm_gpuvm_bo)> = {
82 if core::mem::needs_drop::<Self>() {
83 Some(Self::vm_bo_free)
84 } else {
85 // This causes GPUVM to free a `GpuVmBo<T>` with `kfree`.
86 None
87 }
88 };
89
90 /// Custom function for allocating a `drm_gpuvm_bo`.
91 ///
92 /// # Safety
93 ///
94 /// Always safe to call.
95 unsafe extern "C" fn vm_bo_alloc() -> *mut bindings::drm_gpuvm_bo {
96 let raw_ptr = KBox::<Self>::new_uninit(GFP_KERNEL | __GFP_ZERO)
97 .map(KBox::into_raw)
98 .unwrap_or(ptr::null_mut());
99
100 // CAST: `drm_gpuvm_bo` is first field of `Self`.
101 raw_ptr.cast()
102 }
103
104 /// Custom function for freeing a `drm_gpuvm_bo`.
105 ///
106 /// # Safety
107 ///
108 /// The pointer must have been allocated with [`GpuVmBo::ALLOC_FN`], and must not be used after
109 /// this call.
110 unsafe extern "C" fn vm_bo_free(ptr: *mut bindings::drm_gpuvm_bo) {
111 // CAST: `drm_gpuvm_bo` is first field of `Self`.
112 // SAFETY:
113 // * The ptr was allocated from kmalloc with the layout of `GpuVmBo<T>`.
114 // * `ptr->inner` has no destructor.
115 // * `ptr->data` contains a valid `T::VmBoData` that we can drop.
116 drop(unsafe { KBox::<Self>::from_raw(ptr.cast()) });
117 }
118
119 /// Access this [`GpuVmBo`] from a raw pointer.
120 ///
121 /// # Safety
122 ///
123 /// For the duration of `'a`, the pointer must reference a valid `drm_gpuvm_bo` associated with
124 /// a [`GpuVm<T>`]. The BO must also be present in the GEM list.
125 #[inline]
126 pub(crate) unsafe fn from_raw<'a>(ptr: *mut bindings::drm_gpuvm_bo) -> &'a Self {
127 // SAFETY: `drm_gpuvm_bo` is first field and `repr(C)`.
128 unsafe { &*ptr.cast() }
129 }
130
131 /// Returns a raw pointer to underlying C value.
132 #[inline]
133 pub fn as_raw(&self) -> *mut bindings::drm_gpuvm_bo {
134 self.inner.get()
135 }
136
137 /// The [`GpuVm`] that this GEM object is mapped in.
138 #[inline]
139 pub fn gpuvm(&self) -> &GpuVm<T> {
140 // SAFETY: The `obj` pointer is guaranteed to be valid.
141 unsafe { GpuVm::<T>::from_raw((*self.inner.get()).vm) }
142 }
143
144 /// The [`drm_gem_object`](DriverGpuVm::Object) for these mappings.
145 #[inline]
146 pub fn obj(&self) -> &T::Object {
147 // SAFETY: The `obj` pointer is guaranteed to be valid.
148 unsafe { <T::Object as IntoGEMObject>::from_raw((*self.inner.get()).obj) }
149 }
150
151 /// The driver data with this buffer object.
152 #[inline]
153 pub fn data(&self) -> &T::VmBoData {
154 &self.data
155 }
156
157 pub(super) fn lock_gpuva(&self) -> crate::sync::MutexGuard<'_, ()> {
158 // SAFETY: The GEM object is valid.
159 let ptr = unsafe { &raw mut (*self.obj().as_raw()).gpuva.lock };
160 // SAFETY: The GEM object is valid, so the mutex is properly initialized.
161 let mutex = unsafe { crate::sync::Mutex::from_raw(ptr) };
162 mutex.lock()
163 }
164}
165
166/// A pre-allocated [`GpuVmBo`] object.
167///
168/// # Invariants
169///
170/// Points at a `drm_gpuvm_bo` that contains a valid `T::VmBoData`, has a refcount of one, and is
171/// absent from any gem, extobj, or evict lists.
172pub(super) struct GpuVmBoAlloc<T: DriverGpuVm>(NonNull<GpuVmBo<T>>);
173
174impl<T: DriverGpuVm> GpuVmBoAlloc<T> {
175 /// Create a new pre-allocated [`GpuVmBo`].
176 ///
177 /// It's intentional that the initializer is infallible because `drm_gpuvm_bo_put` will call
178 /// drop on the data, so we don't have a way to free it when the data is missing.
179 #[inline]
180 pub(super) fn new(
181 gpuvm: &GpuVm<T>,
182 gem: &T::Object,
183 value: impl PinInit<T::VmBoData>,
184 ) -> Result<GpuVmBoAlloc<T>, AllocError> {
185 // CAST: `GpuVmBoAlloc::vm_bo_alloc` ensures that this memory was allocated with the layout
186 // of `GpuVmBo<T>`. The type is repr(C), so `container_of` is not required.
187 // SAFETY: The provided gpuvm and gem ptrs are valid for the duration of this call.
188 let raw_ptr = unsafe {
189 bindings::drm_gpuvm_bo_create(gpuvm.as_raw(), gem.as_raw()).cast::<GpuVmBo<T>>()
190 };
191 let ptr = NonNull::new(raw_ptr).ok_or(AllocError)?;
192 // SAFETY: `ptr->data` is a valid pinned location.
193 unsafe { pin_init::raw_init(&raw mut (*raw_ptr).data, value) };
194 // INVARIANTS: We just created the vm_bo so it's absent from lists, and the data is valid
195 // as we just initialized it.
196 Ok(GpuVmBoAlloc(ptr))
197 }
198
199 /// Returns a raw pointer to underlying C value.
200 #[inline]
201 pub(super) fn as_raw(&self) -> *mut bindings::drm_gpuvm_bo {
202 // SAFETY: The pointer references a valid `drm_gpuvm_bo`.
203 unsafe { (*self.0.as_ptr()).inner.get() }
204 }
205
206 /// Look up whether there is an existing [`GpuVmBo`] for this gem object.
207 ///
208 /// The caller should not hold the GEM mutex or DMA resv lock.
209 #[inline]
210 pub(super) fn obtain(self) -> ARef<GpuVmBo<T>> {
211 let me = ManuallyDrop::new(self);
212 // SAFETY: Valid `drm_gpuvm_bo` not already in the lists. We do not access `me` after this
213 // call.
214 let ptr = unsafe { bindings::drm_gpuvm_bo_obtain_prealloc(me.as_raw()) };
215
216 // SAFETY: `drm_gpuvm_bo_obtain_prealloc` always returns a non-null ptr
217 let nonnull = unsafe { NonNull::new_unchecked(ptr.cast()) };
218
219 // INVARIANTS: `drm_gpuvm_bo_obtain_prealloc` ensures that the bo is in the GEM list.
220 // SAFETY: We received one refcount from `drm_gpuvm_bo_obtain_prealloc`.
221 let ret = unsafe { ARef::<GpuVmBo<T>>::from_raw(nonnull) };
222
223 // Ensure that external objects are in the extobj list.
224 //
225 // Note that we must call `extobj_add` even if `ptr != me` to avoid a race condition where
226 // we could end up using the extobj before the thread with `ptr == me` calls extobj_add.
227 if ret.gpuvm().is_extobj(ret.obj()) {
228 let resv_lock = ret.gpuvm().raw_resv();
229 // TODO: Use a proper lock guard here once a dma_resv lock abstraction exists.
230 // SAFETY: The GPUVM is still alive, so its resv lock is too.
231 unsafe { bindings::dma_resv_lock(resv_lock, ptr::null_mut()) };
232 // SAFETY: We hold the GPUVMs resv lock.
233 unsafe { bindings::drm_gpuvm_bo_extobj_add(ptr) };
234 // SAFETY: We took the lock, so we can unlock it.
235 unsafe { bindings::dma_resv_unlock(resv_lock) };
236 }
237
238 ret
239 }
240}
241
242impl<T: DriverGpuVm> Deref for GpuVmBoAlloc<T> {
243 type Target = GpuVmBo<T>;
244 #[inline]
245 fn deref(&self) -> &GpuVmBo<T> {
246 // SAFETY: By the type invariants we may deref while `Self` exists.
247 unsafe { self.0.as_ref() }
248 }
249}
250
251impl<T: DriverGpuVm> Drop for GpuVmBoAlloc<T> {
252 #[inline]
253 fn drop(&mut self) {
254 // TODO: Call drm_gpuvm_bo_destroy_not_in_lists() directly.
255 // SAFETY: It's safe to perform a deferred put in any context.
256 unsafe { bindings::drm_gpuvm_bo_put_deferred(self.as_raw()) };
257 }
258}