projects/linux·blob v6.13 · refs

260 lines · 10.7 KiB

1// SPDX-License-Identifier: GPL-2.0
2
3//! Kernel page allocation and management.
4
5use crate::{
6 alloc::{AllocError, Flags},
7 bindings,
8 error::code::*,
9 error::Result,
10 uaccess::UserSliceReader,
11};
12use core::ptr::{self, NonNull};
13
14/// A bitwise shift for the page size.
15pub const PAGE_SHIFT: usize = bindings::PAGE_SHIFT as usize;
16
17/// The number of bytes in a page.
18pub const PAGE_SIZE: usize = bindings::PAGE_SIZE;
19
20/// A bitmask that gives the page containing a given address.
21pub const PAGE_MASK: usize = !(PAGE_SIZE - 1);
22
23/// Round up the given number to the next multiple of [`PAGE_SIZE`].
24///
25/// It is incorrect to pass an address where the next multiple of [`PAGE_SIZE`] doesn't fit in a
26/// [`usize`].
27pub const fn page_align(addr: usize) -> usize {
28 // Parentheses around `PAGE_SIZE - 1` to avoid triggering overflow sanitizers in the wrong
29 // cases.
30 (addr + (PAGE_SIZE - 1)) & PAGE_MASK
31}
32
33/// A pointer to a page that owns the page allocation.
34///
35/// # Invariants
36///
37/// The pointer is valid, and has ownership over the page.
38pub struct Page {
39 page: NonNull<bindings::page>,
40}
41
42// SAFETY: Pages have no logic that relies on them staying on a given thread, so moving them across
43// threads is safe.
44unsafe impl Send for Page {}
45
46// SAFETY: Pages have no logic that relies on them not being accessed concurrently, so accessing
47// them concurrently is safe.
48unsafe impl Sync for Page {}
49
50impl Page {
51 /// Allocates a new page.
52 ///
53 /// # Examples
54 ///
55 /// Allocate memory for a page.
56 ///
57 /// ```
58 /// use kernel::page::Page;
59 ///
60 /// # fn dox() -> Result<(), kernel::alloc::AllocError> {
61 /// let page = Page::alloc_page(GFP_KERNEL)?;
62 /// # Ok(()) }
63 /// ```
64 ///
65 /// Allocate memory for a page and zero its contents.
66 ///
67 /// ```
68 /// use kernel::page::Page;
69 ///
70 /// # fn dox() -> Result<(), kernel::alloc::AllocError> {
71 /// let page = Page::alloc_page(GFP_KERNEL | __GFP_ZERO)?;
72 /// # Ok(()) }
73 /// ```
74 pub fn alloc_page(flags: Flags) -> Result<Self, AllocError> {
75 // SAFETY: Depending on the value of `gfp_flags`, this call may sleep. Other than that, it
76 // is always safe to call this method.
77 let page = unsafe { bindings::alloc_pages(flags.as_raw(), 0) };
78 let page = NonNull::new(page).ok_or(AllocError)?;
79 // INVARIANT: We just successfully allocated a page, so we now have ownership of the newly
80 // allocated page. We transfer that ownership to the new `Page` object.
81 Ok(Self { page })
82 }
83
84 /// Returns a raw pointer to the page.
85 pub fn as_ptr(&self) -> *mut bindings::page {
86 self.page.as_ptr()
87 }
88
89 /// Runs a piece of code with this page mapped to an address.
90 ///
91 /// The page is unmapped when this call returns.
92 ///
93 /// # Using the raw pointer
94 ///
95 /// It is up to the caller to use the provided raw pointer correctly. The pointer is valid for
96 /// `PAGE_SIZE` bytes and for the duration in which the closure is called. The pointer might
97 /// only be mapped on the current thread, and when that is the case, dereferencing it on other
98 /// threads is UB. Other than that, the usual rules for dereferencing a raw pointer apply: don't
99 /// cause data races, the memory may be uninitialized, and so on.
100 ///
101 /// If multiple threads map the same page at the same time, then they may reference with
102 /// different addresses. However, even if the addresses are different, the underlying memory is
103 /// still the same for these purposes (e.g., it's still a data race if they both write to the
104 /// same underlying byte at the same time).
105 fn with_page_mapped<T>(&self, f: impl FnOnce(*mut u8) -> T) -> T {
106 // SAFETY: `page` is valid due to the type invariants on `Page`.
107 let mapped_addr = unsafe { bindings::kmap_local_page(self.as_ptr()) };
108
109 let res = f(mapped_addr.cast());
110
111 // This unmaps the page mapped above.
112 //
113 // SAFETY: Since this API takes the user code as a closure, it can only be used in a manner
114 // where the pages are unmapped in reverse order. This is as required by `kunmap_local`.
115 //
116 // In other words, if this call to `kunmap_local` happens when a different page should be
117 // unmapped first, then there must necessarily be a call to `kmap_local_page` other than the
118 // call just above in `with_page_mapped` that made that possible. In this case, it is the
119 // unsafe block that wraps that other call that is incorrect.
120 unsafe { bindings::kunmap_local(mapped_addr) };
121
122 res
123 }
124
125 /// Runs a piece of code with a raw pointer to a slice of this page, with bounds checking.
126 ///
127 /// If `f` is called, then it will be called with a pointer that points at `off` bytes into the
128 /// page, and the pointer will be valid for at least `len` bytes. The pointer is only valid on
129 /// this task, as this method uses a local mapping.
130 ///
131 /// If `off` and `len` refers to a region outside of this page, then this method returns
132 /// [`EINVAL`] and does not call `f`.
133 ///
134 /// # Using the raw pointer
135 ///
136 /// It is up to the caller to use the provided raw pointer correctly. The pointer is valid for
137 /// `len` bytes and for the duration in which the closure is called. The pointer might only be
138 /// mapped on the current thread, and when that is the case, dereferencing it on other threads
139 /// is UB. Other than that, the usual rules for dereferencing a raw pointer apply: don't cause
140 /// data races, the memory may be uninitialized, and so on.
141 ///
142 /// If multiple threads map the same page at the same time, then they may reference with
143 /// different addresses. However, even if the addresses are different, the underlying memory is
144 /// still the same for these purposes (e.g., it's still a data race if they both write to the
145 /// same underlying byte at the same time).
146 fn with_pointer_into_page<T>(
147 &self,
148 off: usize,
149 len: usize,
150 f: impl FnOnce(*mut u8) -> Result<T>,
151 ) -> Result<T> {
152 let bounds_ok = off <= PAGE_SIZE && len <= PAGE_SIZE && (off + len) <= PAGE_SIZE;
153
154 if bounds_ok {
155 self.with_page_mapped(move |page_addr| {
156 // SAFETY: The `off` integer is at most `PAGE_SIZE`, so this pointer offset will
157 // result in a pointer that is in bounds or one off the end of the page.
158 f(unsafe { page_addr.add(off) })
159 })
160 } else {
161 Err(EINVAL)
162 }
163 }
164
165 /// Maps the page and reads from it into the given buffer.
166 ///
167 /// This method will perform bounds checks on the page offset. If `offset .. offset+len` goes
168 /// outside of the page, then this call returns [`EINVAL`].
169 ///
170 /// # Safety
171 ///
172 /// * Callers must ensure that `dst` is valid for writing `len` bytes.
173 /// * Callers must ensure that this call does not race with a write to the same page that
174 /// overlaps with this read.
175 pub unsafe fn read_raw(&self, dst: *mut u8, offset: usize, len: usize) -> Result {
176 self.with_pointer_into_page(offset, len, move |src| {
177 // SAFETY: If `with_pointer_into_page` calls into this closure, then
178 // it has performed a bounds check and guarantees that `src` is
179 // valid for `len` bytes.
180 //
181 // There caller guarantees that there is no data race.
182 unsafe { ptr::copy_nonoverlapping(src, dst, len) };
183 Ok(())
184 })
185 }
186
187 /// Maps the page and writes into it from the given buffer.
188 ///
189 /// This method will perform bounds checks on the page offset. If `offset .. offset+len` goes
190 /// outside of the page, then this call returns [`EINVAL`].
191 ///
192 /// # Safety
193 ///
194 /// * Callers must ensure that `src` is valid for reading `len` bytes.
195 /// * Callers must ensure that this call does not race with a read or write to the same page
196 /// that overlaps with this write.
197 pub unsafe fn write_raw(&self, src: *const u8, offset: usize, len: usize) -> Result {
198 self.with_pointer_into_page(offset, len, move |dst| {
199 // SAFETY: If `with_pointer_into_page` calls into this closure, then it has performed a
200 // bounds check and guarantees that `dst` is valid for `len` bytes.
201 //
202 // There caller guarantees that there is no data race.
203 unsafe { ptr::copy_nonoverlapping(src, dst, len) };
204 Ok(())
205 })
206 }
207
208 /// Maps the page and zeroes the given slice.
209 ///
210 /// This method will perform bounds checks on the page offset. If `offset .. offset+len` goes
211 /// outside of the page, then this call returns [`EINVAL`].
212 ///
213 /// # Safety
214 ///
215 /// Callers must ensure that this call does not race with a read or write to the same page that
216 /// overlaps with this write.
217 pub unsafe fn fill_zero_raw(&self, offset: usize, len: usize) -> Result {
218 self.with_pointer_into_page(offset, len, move |dst| {
219 // SAFETY: If `with_pointer_into_page` calls into this closure, then it has performed a
220 // bounds check and guarantees that `dst` is valid for `len` bytes.
221 //
222 // There caller guarantees that there is no data race.
223 unsafe { ptr::write_bytes(dst, 0u8, len) };
224 Ok(())
225 })
226 }
227
228 /// Copies data from userspace into this page.
229 ///
230 /// This method will perform bounds checks on the page offset. If `offset .. offset+len` goes
231 /// outside of the page, then this call returns [`EINVAL`].
232 ///
233 /// Like the other `UserSliceReader` methods, data races are allowed on the userspace address.
234 /// However, they are not allowed on the page you are copying into.
235 ///
236 /// # Safety
237 ///
238 /// Callers must ensure that this call does not race with a read or write to the same page that
239 /// overlaps with this write.
240 pub unsafe fn copy_from_user_slice_raw(
241 &self,
242 reader: &mut UserSliceReader,
243 offset: usize,
244 len: usize,
245 ) -> Result {
246 self.with_pointer_into_page(offset, len, move |dst| {
247 // SAFETY: If `with_pointer_into_page` calls into this closure, then it has performed a
248 // bounds check and guarantees that `dst` is valid for `len` bytes. Furthermore, we have
249 // exclusive access to the slice since the caller guarantees that there are no races.
250 reader.read_raw(unsafe { core::slice::from_raw_parts_mut(dst.cast(), len) })
251 })
252 }
253}
254
255impl Drop for Page {
256 fn drop(&mut self) {
257 // SAFETY: By the type invariants, we have ownership of the page and can free it.
258 unsafe { bindings::__free_pages(self.page.as_ptr(), 0) };
259 }
260}