projects/linux·blob master · refs

297 lines · 9.7 KiB

1// SPDX-License-Identifier: GPL-2.0
2
3// Copyright (C) 2024 Google LLC.
4
5//! Memory management.
6//!
7//! This module deals with managing the address space of userspace processes. Each process has an
8//! instance of [`Mm`], which keeps track of multiple VMAs (virtual memory areas). Each VMA
9//! corresponds to a region of memory that the userspace process can access, and the VMA lets you
10//! control what happens when userspace reads or writes to that region of memory.
11//!
12//! C header: [`include/linux/mm.h`](srctree/include/linux/mm.h)
13
14use crate::{
15 bindings,
16 sync::aref::{ARef, AlwaysRefCounted},
17 types::{NotThreadSafe, Opaque},
18};
19use core::{ops::Deref, ptr::NonNull};
20
21pub mod virt;
22use virt::VmaRef;
23
24#[cfg(CONFIG_MMU)]
25pub use mmput_async::MmWithUserAsync;
26mod mmput_async;
27
28/// A wrapper for the kernel's `struct mm_struct`.
29///
30/// This represents the address space of a userspace process, so each process has one `Mm`
31/// instance. It may hold many VMAs internally.
32///
33/// There is a counter called `mm_users` that counts the users of the address space; this includes
34/// the userspace process itself, but can also include kernel threads accessing the address space.
35/// Once `mm_users` reaches zero, this indicates that the address space can be destroyed. To access
36/// the address space, you must prevent `mm_users` from reaching zero while you are accessing it.
37/// The [`MmWithUser`] type represents an address space where this is guaranteed, and you can
38/// create one using [`mmget_not_zero`].
39///
40/// The `ARef<Mm>` smart pointer holds an `mmgrab` refcount. Its destructor may sleep.
41///
42/// # Invariants
43///
44/// Values of this type are always refcounted using `mmgrab`.
45///
46/// [`mmget_not_zero`]: Mm::mmget_not_zero
47#[repr(transparent)]
48pub struct Mm {
49 mm: Opaque<bindings::mm_struct>,
50}
51
52// SAFETY: It is safe to call `mmdrop` on another thread than where `mmgrab` was called.
53unsafe impl Send for Mm {}
54// SAFETY: All methods on `Mm` can be called in parallel from several threads.
55unsafe impl Sync for Mm {}
56
57// SAFETY: By the type invariants, this type is always refcounted.
58unsafe impl AlwaysRefCounted for Mm {
59 #[inline]
60 fn inc_ref(&self) {
61 // SAFETY: The pointer is valid since self is a reference.
62 unsafe { bindings::mmgrab(self.as_raw()) };
63 }
64
65 #[inline]
66 unsafe fn dec_ref(obj: NonNull<Self>) {
67 // SAFETY: The caller is giving up their refcount.
68 unsafe { bindings::mmdrop(obj.cast().as_ptr()) };
69 }
70}
71
72/// A wrapper for the kernel's `struct mm_struct`.
73///
74/// This type is like [`Mm`], but with non-zero `mm_users`. It can only be used when `mm_users` can
75/// be proven to be non-zero at compile-time, usually because the relevant code holds an `mmget`
76/// refcount. It can be used to access the associated address space.
77///
78/// The `ARef<MmWithUser>` smart pointer holds an `mmget` refcount. Its destructor may sleep.
79///
80/// # Invariants
81///
82/// Values of this type are always refcounted using `mmget`. The value of `mm_users` is non-zero.
83#[repr(transparent)]
84pub struct MmWithUser {
85 mm: Mm,
86}
87
88// SAFETY: It is safe to call `mmput` on another thread than where `mmget` was called.
89unsafe impl Send for MmWithUser {}
90// SAFETY: All methods on `MmWithUser` can be called in parallel from several threads.
91unsafe impl Sync for MmWithUser {}
92
93// SAFETY: By the type invariants, this type is always refcounted.
94unsafe impl AlwaysRefCounted for MmWithUser {
95 #[inline]
96 fn inc_ref(&self) {
97 // SAFETY: The pointer is valid since self is a reference.
98 unsafe { bindings::mmget(self.as_raw()) };
99 }
100
101 #[inline]
102 unsafe fn dec_ref(obj: NonNull<Self>) {
103 // SAFETY: The caller is giving up their refcount.
104 unsafe { bindings::mmput(obj.cast().as_ptr()) };
105 }
106}
107
108// Make all `Mm` methods available on `MmWithUser`.
109impl Deref for MmWithUser {
110 type Target = Mm;
111
112 #[inline]
113 fn deref(&self) -> &Mm {
114 &self.mm
115 }
116}
117
118// These methods are safe to call even if `mm_users` is zero.
119impl Mm {
120 /// Returns a raw pointer to the inner `mm_struct`.
121 #[inline]
122 pub fn as_raw(&self) -> *mut bindings::mm_struct {
123 self.mm.get()
124 }
125
126 /// Obtain a reference from a raw pointer.
127 ///
128 /// # Safety
129 ///
130 /// The caller must ensure that `ptr` points at an `mm_struct`, and that it is not deallocated
131 /// during the lifetime 'a.
132 #[inline]
133 pub unsafe fn from_raw<'a>(ptr: *const bindings::mm_struct) -> &'a Mm {
134 // SAFETY: Caller promises that the pointer is valid for 'a. Layouts are compatible due to
135 // repr(transparent).
136 unsafe { &*ptr.cast() }
137 }
138
139 /// Calls `mmget_not_zero` and returns a handle if it succeeds.
140 #[inline]
141 pub fn mmget_not_zero(&self) -> Option<ARef<MmWithUser>> {
142 // SAFETY: The pointer is valid since self is a reference.
143 let success = unsafe { bindings::mmget_not_zero(self.as_raw()) };
144
145 if success {
146 // SAFETY: We just created an `mmget` refcount.
147 Some(unsafe { ARef::from_raw(NonNull::new_unchecked(self.as_raw().cast())) })
148 } else {
149 None
150 }
151 }
152}
153
154// These methods require `mm_users` to be non-zero.
155impl MmWithUser {
156 /// Obtain a reference from a raw pointer.
157 ///
158 /// # Safety
159 ///
160 /// The caller must ensure that `ptr` points at an `mm_struct`, and that `mm_users` remains
161 /// non-zero for the duration of the lifetime 'a.
162 #[inline]
163 pub unsafe fn from_raw<'a>(ptr: *const bindings::mm_struct) -> &'a MmWithUser {
164 // SAFETY: Caller promises that the pointer is valid for 'a. The layout is compatible due
165 // to repr(transparent).
166 unsafe { &*ptr.cast() }
167 }
168
169 /// Attempt to access a vma using the vma read lock.
170 ///
171 /// This is an optimistic trylock operation, so it may fail if there is contention. In that
172 /// case, you should fall back to taking the mmap read lock.
173 ///
174 /// When per-vma locks are disabled, this always returns `None`.
175 #[inline]
176 pub fn lock_vma_under_rcu(&self, vma_addr: usize) -> Option<VmaReadGuard<'_>> {
177 #[cfg(CONFIG_PER_VMA_LOCK)]
178 {
179 // SAFETY: Calling `bindings::lock_vma_under_rcu` is always okay given an mm where
180 // `mm_users` is non-zero.
181 let vma = unsafe { bindings::lock_vma_under_rcu(self.as_raw(), vma_addr) };
182 if !vma.is_null() {
183 return Some(VmaReadGuard {
184 // SAFETY: If `lock_vma_under_rcu` returns a non-null ptr, then it points at a
185 // valid vma. The vma is stable for as long as the vma read lock is held.
186 vma: unsafe { VmaRef::from_raw(vma) },
187 _nts: NotThreadSafe,
188 });
189 }
190 }
191
192 // Silence warnings about unused variables.
193 #[cfg(not(CONFIG_PER_VMA_LOCK))]
194 let _ = vma_addr;
195
196 None
197 }
198
199 /// Lock the mmap read lock.
200 #[inline]
201 pub fn mmap_read_lock(&self) -> MmapReadGuard<'_> {
202 // SAFETY: The pointer is valid since self is a reference.
203 unsafe { bindings::mmap_read_lock(self.as_raw()) };
204
205 // INVARIANT: We just acquired the read lock.
206 MmapReadGuard {
207 mm: self,
208 _nts: NotThreadSafe,
209 }
210 }
211
212 /// Try to lock the mmap read lock.
213 #[inline]
214 pub fn mmap_read_trylock(&self) -> Option<MmapReadGuard<'_>> {
215 // SAFETY: The pointer is valid since self is a reference.
216 let success = unsafe { bindings::mmap_read_trylock(self.as_raw()) };
217
218 if success {
219 // INVARIANT: We just acquired the read lock.
220 Some(MmapReadGuard {
221 mm: self,
222 _nts: NotThreadSafe,
223 })
224 } else {
225 None
226 }
227 }
228}
229
230/// A guard for the mmap read lock.
231///
232/// # Invariants
233///
234/// This `MmapReadGuard` guard owns the mmap read lock.
235pub struct MmapReadGuard<'a> {
236 mm: &'a MmWithUser,
237 // `mmap_read_lock` and `mmap_read_unlock` must be called on the same thread
238 _nts: NotThreadSafe,
239}
240
241impl<'a> MmapReadGuard<'a> {
242 /// Look up a vma at the given address.
243 #[inline]
244 pub fn vma_lookup(&self, vma_addr: usize) -> Option<&virt::VmaRef> {
245 // SAFETY: By the type invariants we hold the mmap read guard, so we can safely call this
246 // method. Any value is okay for `vma_addr`.
247 let vma = unsafe { bindings::vma_lookup(self.mm.as_raw(), vma_addr) };
248
249 if vma.is_null() {
250 None
251 } else {
252 // SAFETY: We just checked that a vma was found, so the pointer references a valid vma.
253 //
254 // Furthermore, the returned vma is still under the protection of the read lock guard
255 // and can be used while the mmap read lock is still held. That the vma is not used
256 // after the MmapReadGuard gets dropped is enforced by the borrow-checker.
257 unsafe { Some(virt::VmaRef::from_raw(vma)) }
258 }
259 }
260}
261
262impl Drop for MmapReadGuard<'_> {
263 #[inline]
264 fn drop(&mut self) {
265 // SAFETY: We hold the read lock by the type invariants.
266 unsafe { bindings::mmap_read_unlock(self.mm.as_raw()) };
267 }
268}
269
270/// A guard for the vma read lock.
271///
272/// # Invariants
273///
274/// This `VmaReadGuard` guard owns the vma read lock.
275pub struct VmaReadGuard<'a> {
276 vma: &'a VmaRef,
277 // `vma_end_read` must be called on the same thread as where the lock was taken
278 _nts: NotThreadSafe,
279}
280
281// Make all `VmaRef` methods available on `VmaReadGuard`.
282impl Deref for VmaReadGuard<'_> {
283 type Target = VmaRef;
284
285 #[inline]
286 fn deref(&self) -> &VmaRef {
287 self.vma
288 }
289}
290
291impl Drop for VmaReadGuard<'_> {
292 #[inline]
293 fn drop(&mut self) {
294 // SAFETY: We hold the read lock by the type invariants.
295 unsafe { bindings::vma_end_read(self.vma.as_ptr()) };
296 }
297}