projects/linux·blob master · refs

323 lines · 10.9 KiB

1// SPDX-License-Identifier: GPL-2.0
2
3//! Allocator support.
4//!
5//! Documentation for the kernel's memory allocators can found in the "Memory Allocation Guide"
6//! linked below. For instance, this includes the concept of "get free page" (GFP) flags and the
7//! typical application of the different kernel allocators.
8//!
9//! Reference: <https://docs.kernel.org/core-api/memory-allocation.html>
10
11use super::{
12 AllocError,
13 Allocator,
14 Flags,
15 NumaNode, //
16};
17
18use crate::{
19 bindings,
20 page, //
21};
22
23use core::{
24 alloc::Layout,
25 ptr::{
26 self,
27 NonNull, //
28 }, //
29};
30
31const ARCH_KMALLOC_MINALIGN: usize = bindings::ARCH_KMALLOC_MINALIGN;
32
33mod iter;
34pub use self::iter::VmallocPageIter;
35
36/// The contiguous kernel allocator.
37///
38/// `Kmalloc` is typically used for physically contiguous allocations up to page size, but also
39/// supports larger allocations up to `bindings::KMALLOC_MAX_SIZE`, which is hardware specific.
40///
41/// For more details see [self].
42pub struct Kmalloc;
43
44/// The virtually contiguous kernel allocator.
45///
46/// `Vmalloc` allocates pages from the page level allocator and maps them into the contiguous kernel
47/// virtual space. It is typically used for large allocations. The memory allocated with this
48/// allocator is not physically contiguous.
49///
50/// For more details see [self].
51pub struct Vmalloc;
52
53/// The kvmalloc kernel allocator.
54///
55/// `KVmalloc` attempts to allocate memory with `Kmalloc` first, but falls back to `Vmalloc` upon
56/// failure. This allocator is typically used when the size for the requested allocation is not
57/// known and may exceed the capabilities of `Kmalloc`.
58///
59/// For more details see [self].
60pub struct KVmalloc;
61
62/// # Invariants
63///
64/// One of the following: `krealloc_node_align`, `vrealloc_node_align`, `kvrealloc_node_align`.
65struct ReallocFunc(
66 unsafe extern "C" fn(
67 *const crate::ffi::c_void,
68 usize,
69 crate::ffi::c_ulong,
70 u32,
71 crate::ffi::c_int,
72 ) -> *mut crate::ffi::c_void,
73);
74
75impl ReallocFunc {
76 // INVARIANT: `krealloc_node_align` satisfies the type invariants.
77 const KREALLOC: Self = Self(bindings::krealloc_node_align);
78
79 // INVARIANT: `vrealloc_node_align` satisfies the type invariants.
80 const VREALLOC: Self = Self(bindings::vrealloc_node_align);
81
82 // INVARIANT: `kvrealloc_node_align` satisfies the type invariants.
83 const KVREALLOC: Self = Self(bindings::kvrealloc_node_align);
84
85 /// # Safety
86 ///
87 /// This method has the same safety requirements as [`Allocator::realloc`].
88 ///
89 /// # Guarantees
90 ///
91 /// This method has the same guarantees as `Allocator::realloc`. Additionally
92 /// - it accepts any pointer to a valid memory allocation allocated by this function.
93 /// - memory allocated by this function remains valid until it is passed to this function.
94 #[inline]
95 unsafe fn call(
96 &self,
97 ptr: Option<NonNull<u8>>,
98 layout: Layout,
99 old_layout: Layout,
100 flags: Flags,
101 nid: NumaNode,
102 ) -> Result<NonNull<[u8]>, AllocError> {
103 let size = layout.size();
104 let ptr = match ptr {
105 Some(ptr) => {
106 if old_layout.size() == 0 {
107 ptr::null()
108 } else {
109 ptr.as_ptr()
110 }
111 }
112 None => ptr::null(),
113 };
114
115 // SAFETY:
116 // - `self.0` is one of `krealloc`, `vrealloc`, `kvrealloc` and thus only requires that
117 // `ptr` is NULL or valid.
118 // - `ptr` is either NULL or valid by the safety requirements of this function.
119 //
120 // GUARANTEE:
121 // - `self.0` is one of `krealloc`, `vrealloc`, `kvrealloc`.
122 // - Those functions provide the guarantees of this function.
123 let raw_ptr = unsafe {
124 // If `size == 0` and `ptr != NULL` the memory behind the pointer is freed.
125 self.0(ptr.cast(), size, layout.align(), flags.0, nid.0).cast()
126 };
127
128 let ptr = if size == 0 {
129 crate::alloc::dangling_from_layout(layout)
130 } else {
131 NonNull::new(raw_ptr).ok_or(AllocError)?
132 };
133
134 Ok(NonNull::slice_from_raw_parts(ptr, size))
135 }
136}
137
138impl Kmalloc {
139 /// Returns a [`Layout`] that makes [`Kmalloc`] fulfill the requested size and alignment of
140 /// `layout`.
141 pub fn aligned_layout(layout: Layout) -> Layout {
142 // Note that `layout.size()` (after padding) is guaranteed to be a multiple of
143 // `layout.align()` which together with the slab guarantees means that `Kmalloc` will return
144 // a properly aligned object (see comments in `kmalloc()` for more information).
145 layout.pad_to_align()
146 }
147}
148
149// SAFETY: `realloc` delegates to `ReallocFunc::call`, which guarantees that
150// - memory remains valid until it is explicitly freed,
151// - passing a pointer to a valid memory allocation is OK,
152// - `realloc` satisfies the guarantees, since `ReallocFunc::call` has the same.
153unsafe impl Allocator for Kmalloc {
154 const MIN_ALIGN: usize = ARCH_KMALLOC_MINALIGN;
155
156 #[inline]
157 unsafe fn realloc(
158 ptr: Option<NonNull<u8>>,
159 layout: Layout,
160 old_layout: Layout,
161 flags: Flags,
162 nid: NumaNode,
163 ) -> Result<NonNull<[u8]>, AllocError> {
164 let layout = Kmalloc::aligned_layout(layout);
165
166 // SAFETY: `ReallocFunc::call` has the same safety requirements as `Allocator::realloc`.
167 unsafe { ReallocFunc::KREALLOC.call(ptr, layout, old_layout, flags, nid) }
168 }
169}
170
171impl Vmalloc {
172 /// Convert a pointer to a [`Vmalloc`] allocation to a [`page::BorrowedPage`].
173 ///
174 /// # Examples
175 ///
176 /// ```
177 /// # use core::ptr::{
178 /// # from_mut,
179 /// # NonNull, //
180 /// # };
181 /// # use kernel::page;
182 /// use kernel::alloc::allocator::Vmalloc;
183 ///
184 /// let mut vbox = VBox::<[u8; page::PAGE_SIZE]>::new_uninit(GFP_KERNEL)?;
185 ///
186 /// {
187 /// // SAFETY: By the type invariant of `Box` the inner pointer of `vbox` is non-null.
188 /// let ptr = unsafe { NonNull::new_unchecked(from_mut(&mut *vbox)) };
189 ///
190 /// // SAFETY:
191 /// // `ptr` is a valid pointer to a `Vmalloc` allocation.
192 /// // `ptr` is valid for the entire lifetime of `page`.
193 /// let page = unsafe { Vmalloc::to_page(ptr.cast()) };
194 ///
195 /// // SAFETY: There is no concurrent read or write to the same page.
196 /// unsafe { page.fill_zero_raw(0, page::PAGE_SIZE)? };
197 /// }
198 /// # Ok::<(), Error>(())
199 /// ```
200 ///
201 /// # Safety
202 ///
203 /// - `ptr` must be a valid pointer to a [`Vmalloc`] allocation.
204 /// - `ptr` must remain valid for the entire duration of `'a`.
205 pub unsafe fn to_page<'a>(ptr: NonNull<u8>) -> page::BorrowedPage<'a> {
206 // SAFETY: `ptr` is a valid pointer to `Vmalloc` memory.
207 let page = unsafe { bindings::vmalloc_to_page(ptr.as_ptr().cast()) };
208
209 // SAFETY: `vmalloc_to_page` returns a valid pointer to a `struct page` for a valid pointer
210 // to `Vmalloc` memory.
211 let page = unsafe { NonNull::new_unchecked(page) };
212
213 // SAFETY:
214 // - `page` is a valid pointer to a `struct page`, given that by the safety requirements of
215 // this function `ptr` is a valid pointer to a `Vmalloc` allocation.
216 // - By the safety requirements of this function `ptr` is valid for the entire lifetime of
217 // `'a`.
218 unsafe { page::BorrowedPage::from_raw(page) }
219 }
220}
221
222// SAFETY: `realloc` delegates to `ReallocFunc::call`, which guarantees that
223// - memory remains valid until it is explicitly freed,
224// - passing a pointer to a valid memory allocation is OK,
225// - `realloc` satisfies the guarantees, since `ReallocFunc::call` has the same.
226unsafe impl Allocator for Vmalloc {
227 const MIN_ALIGN: usize = kernel::page::PAGE_SIZE;
228
229 #[inline]
230 unsafe fn realloc(
231 ptr: Option<NonNull<u8>>,
232 layout: Layout,
233 old_layout: Layout,
234 flags: Flags,
235 nid: NumaNode,
236 ) -> Result<NonNull<[u8]>, AllocError> {
237 // SAFETY: If not `None`, `ptr` is guaranteed to point to valid memory, which was previously
238 // allocated with this `Allocator`.
239 unsafe { ReallocFunc::VREALLOC.call(ptr, layout, old_layout, flags, nid) }
240 }
241}
242
243// SAFETY: `realloc` delegates to `ReallocFunc::call`, which guarantees that
244// - memory remains valid until it is explicitly freed,
245// - passing a pointer to a valid memory allocation is OK,
246// - `realloc` satisfies the guarantees, since `ReallocFunc::call` has the same.
247unsafe impl Allocator for KVmalloc {
248 const MIN_ALIGN: usize = ARCH_KMALLOC_MINALIGN;
249
250 #[inline]
251 unsafe fn realloc(
252 ptr: Option<NonNull<u8>>,
253 layout: Layout,
254 old_layout: Layout,
255 flags: Flags,
256 nid: NumaNode,
257 ) -> Result<NonNull<[u8]>, AllocError> {
258 // `KVmalloc` may use the `Kmalloc` backend, hence we have to enforce a `Kmalloc`
259 // compatible layout.
260 let layout = Kmalloc::aligned_layout(layout);
261
262 // SAFETY: If not `None`, `ptr` is guaranteed to point to valid memory, which was previously
263 // allocated with this `Allocator`.
264 unsafe { ReallocFunc::KVREALLOC.call(ptr, layout, old_layout, flags, nid) }
265 }
266}
267
268#[cfg(CONFIG_RUST_ALLOCATOR_KUNIT_TEST)]
269#[macros::kunit_tests(rust_allocator)]
270mod tests {
271 use super::*;
272 use core::mem::MaybeUninit;
273 use kernel::prelude::*;
274
275 #[test]
276 fn test_alignment() -> Result {
277 const TEST_SIZE: usize = 1024;
278 const TEST_LARGE_ALIGN_SIZE: usize = kernel::page::PAGE_SIZE * 4;
279
280 // These two structs are used to test allocating aligned memory.
281 // they don't need to be accessed, so they're marked as dead_code.
282 #[expect(dead_code)]
283 #[repr(align(128))]
284 struct Blob([u8; TEST_SIZE]);
285 #[expect(dead_code)]
286 #[repr(align(8192))]
287 struct LargeAlignBlob([u8; TEST_LARGE_ALIGN_SIZE]);
288
289 struct TestAlign<T, A: Allocator>(Box<MaybeUninit<T>, A>);
290 impl<T, A: Allocator> TestAlign<T, A> {
291 fn new() -> Result<Self> {
292 Ok(Self(Box::<_, A>::new_uninit(GFP_KERNEL)?))
293 }
294
295 fn is_aligned_to(&self, align: usize) -> bool {
296 assert!(align.is_power_of_two());
297
298 let addr = self.0.as_ptr() as usize;
299 addr & (align - 1) == 0
300 }
301 }
302
303 let ta = TestAlign::<Blob, Kmalloc>::new()?;
304 assert!(ta.is_aligned_to(128));
305
306 let ta = TestAlign::<LargeAlignBlob, Kmalloc>::new()?;
307 assert!(ta.is_aligned_to(8192));
308
309 let ta = TestAlign::<Blob, Vmalloc>::new()?;
310 assert!(ta.is_aligned_to(128));
311
312 let ta = TestAlign::<LargeAlignBlob, Vmalloc>::new()?;
313 assert!(ta.is_aligned_to(8192));
314
315 let ta = TestAlign::<Blob, KVmalloc>::new()?;
316 assert!(ta.is_aligned_to(128));
317
318 let ta = TestAlign::<LargeAlignBlob, KVmalloc>::new()?;
319 assert!(ta.is_aligned_to(8192));
320
321 Ok(())
322 }
323}