projects/linux·blob master · refs

264 lines · 9.0 KiB

1// SPDX-License-Identifier: GPL-2.0
2
3//! IOMMU page table management.
4//!
5//! C header: [`include/linux/io-pgtable.h`](srctree/include/linux/io-pgtable.h)
6
7use core::{
8 marker::PhantomData,
9 ptr::NonNull, //
10};
11
12use crate::{
13 alloc,
14 bindings,
15 device::{
16 Bound,
17 Device, //
18 },
19 error::to_result,
20 io::PhysAddr,
21 prelude::*, //
22};
23
24use bindings::io_pgtable_fmt;
25
26/// Protection flags used with IOMMU mappings.
27pub mod prot {
28 /// Read access.
29 pub const READ: u32 = bindings::IOMMU_READ;
30 /// Write access.
31 pub const WRITE: u32 = bindings::IOMMU_WRITE;
32 /// Request cache coherency.
33 pub const CACHE: u32 = bindings::IOMMU_CACHE;
34 /// Request no-execute permission.
35 pub const NOEXEC: u32 = bindings::IOMMU_NOEXEC;
36 /// MMIO peripheral mapping.
37 pub const MMIO: u32 = bindings::IOMMU_MMIO;
38 /// Privileged mapping.
39 pub const PRIVILEGED: u32 = bindings::IOMMU_PRIV;
40}
41
42/// Represents a requested `io_pgtable` configuration.
43pub struct Config {
44 /// Quirk bitmask (type-specific).
45 pub quirks: usize,
46 /// Valid page sizes, as a bitmask of powers of two.
47 pub pgsize_bitmap: usize,
48 /// Input address space size in bits.
49 pub ias: u32,
50 /// Output address space size in bits.
51 pub oas: u32,
52 /// IOMMU uses coherent accesses for page table walks.
53 pub coherent_walk: bool,
54}
55
56/// An io page table using a specific format.
57///
58/// # Invariants
59///
60/// The pointer references a valid io page table.
61pub struct IoPageTable<'a, F: IoPageTableFmt> {
62 ptr: NonNull<bindings::io_pgtable_ops>,
63 _dev: PhantomData<&'a Device<Bound>>,
64 _marker: PhantomData<F>,
65}
66
67// SAFETY: `struct io_pgtable_ops` is not restricted to a single thread.
68unsafe impl<F: IoPageTableFmt> Send for IoPageTable<'_, F> {}
69// SAFETY: `struct io_pgtable_ops` may be accessed concurrently.
70unsafe impl<F: IoPageTableFmt> Sync for IoPageTable<'_, F> {}
71
72/// The format used by this page table.
73pub trait IoPageTableFmt: 'static {
74 /// The value representing this format.
75 const FORMAT: io_pgtable_fmt;
76}
77
78impl<'a, F: IoPageTableFmt> IoPageTable<'a, F> {
79 /// Create a new `IoPageTable`.
80 #[inline]
81 pub fn new(dev: &'a Device<Bound>, config: Config) -> Result<IoPageTable<'a, F>> {
82 let mut raw_cfg = bindings::io_pgtable_cfg {
83 quirks: config.quirks,
84 pgsize_bitmap: config.pgsize_bitmap,
85 ias: config.ias,
86 oas: config.oas,
87 coherent_walk: config.coherent_walk,
88 tlb: &raw const NOOP_FLUSH_OPS,
89 iommu_dev: dev.as_raw(),
90 ..Zeroable::zeroed()
91 };
92
93 // SAFETY:
94 // * The raw_cfg pointer is valid for the duration of this call.
95 // * The provided `FLUSH_OPS` contains valid function pointers that accept a null pointer
96 // as cookie.
97 // * The caller ensures that the io pgtable does not outlive the device.
98 let ops = unsafe {
99 bindings::alloc_io_pgtable_ops(F::FORMAT, &mut raw_cfg, core::ptr::null_mut())
100 };
101
102 // INVARIANT: We successfully created a valid page table.
103 Ok(IoPageTable {
104 ptr: NonNull::new(ops).ok_or(ENOMEM)?,
105 _dev: PhantomData,
106 _marker: PhantomData,
107 })
108 }
109
110 /// Obtain a raw pointer to the underlying `struct io_pgtable_ops`.
111 #[inline]
112 pub fn raw_ops(&self) -> *mut bindings::io_pgtable_ops {
113 self.ptr.as_ptr()
114 }
115
116 /// Obtain a raw pointer to the underlying `struct io_pgtable`.
117 #[inline]
118 pub fn raw_pgtable(&self) -> *mut bindings::io_pgtable {
119 // SAFETY: The io_pgtable_ops of an io-pgtable is always the ops field of a io_pgtable.
120 unsafe { kernel::container_of!(self.raw_ops(), bindings::io_pgtable, ops) }
121 }
122
123 /// Obtain a raw pointer to the underlying `struct io_pgtable_cfg`.
124 #[inline]
125 pub fn raw_cfg(&self) -> *mut bindings::io_pgtable_cfg {
126 // SAFETY: The `raw_pgtable()` method returns a valid pointer.
127 unsafe { &raw mut (*self.raw_pgtable()).cfg }
128 }
129
130 /// Map a physically contiguous range of pages of the same size.
131 ///
132 /// Even if successful, this operation may not map the entire range. In that case, only a
133 /// prefix of the range is mapped, and the returned integer indicates its length in bytes. In
134 /// this case, the caller will usually call `map_pages` again for the remaining range.
135 ///
136 /// The returned [`Result`] indicates whether an error was encountered while mapping pages.
137 /// Note that this may return a non-zero length even if an error was encountered. The caller
138 /// will usually [unmap the relevant pages](Self::unmap_pages) on error.
139 ///
140 /// The caller must flush the TLB before using the pgtable to access the newly created mapping.
141 ///
142 /// # Safety
143 ///
144 /// * No other io-pgtable operation may access the range `iova .. iova+pgsize*pgcount` while
145 /// this `map_pages` operation executes.
146 /// * This page table must not contain any mapping that overlaps with the mapping created by
147 /// this call.
148 /// * If this page table is live, then the caller must ensure that it's okay to access the
149 /// physical address being mapped for the duration in which it is mapped.
150 #[inline]
151 pub unsafe fn map_pages(
152 &self,
153 iova: usize,
154 paddr: PhysAddr,
155 pgsize: usize,
156 pgcount: usize,
157 prot: u32,
158 flags: alloc::Flags,
159 ) -> (usize, Result) {
160 let mut mapped: usize = 0;
161
162 // SAFETY: The `map_pages` function in `io_pgtable_ops` is never null.
163 let map_pages = unsafe { (*self.raw_ops()).map_pages.unwrap_unchecked() };
164
165 // SAFETY: The safety requirements of this method are sufficient to call `map_pages`.
166 let ret = to_result(unsafe {
167 (map_pages)(
168 self.raw_ops(),
169 iova,
170 paddr,
171 pgsize,
172 pgcount,
173 prot as i32,
174 flags.as_raw(),
175 &mut mapped,
176 )
177 });
178
179 (mapped, ret)
180 }
181
182 /// Unmap a range of virtually contiguous pages of the same size.
183 ///
184 /// This may not unmap the entire range, and returns the length of the unmapped prefix in
185 /// bytes.
186 ///
187 /// # Safety
188 ///
189 /// * No other io-pgtable operation may access the range `iova .. iova+pgsize*pgcount` while
190 /// this `unmap_pages` operation executes.
191 /// * This page table must contain one or more consecutive mappings starting at `iova` whose
192 /// total size is `pgcount * pgsize`.
193 #[inline]
194 #[must_use]
195 pub unsafe fn unmap_pages(&self, iova: usize, pgsize: usize, pgcount: usize) -> usize {
196 // SAFETY: The `unmap_pages` function in `io_pgtable_ops` is never null.
197 let unmap_pages = unsafe { (*self.raw_ops()).unmap_pages.unwrap_unchecked() };
198
199 // SAFETY: The safety requirements of this method are sufficient to call `unmap_pages`.
200 unsafe { (unmap_pages)(self.raw_ops(), iova, pgsize, pgcount, core::ptr::null_mut()) }
201 }
202}
203
204// For the initial users of these rust bindings, the GPU FW is managing the IOTLB and performs all
205// required invalidations using a range. There is no need for it get ARM style invalidation
206// instructions from the page table code.
207//
208// Support for flushing the TLB with ARM style invalidation instructions may be added in the
209// future.
210static NOOP_FLUSH_OPS: bindings::iommu_flush_ops = bindings::iommu_flush_ops {
211 tlb_flush_all: Some(rust_tlb_flush_all_noop),
212 tlb_flush_walk: Some(rust_tlb_flush_walk_noop),
213 tlb_add_page: None,
214};
215
216#[no_mangle]
217extern "C" fn rust_tlb_flush_all_noop(_cookie: *mut core::ffi::c_void) {}
218
219#[no_mangle]
220extern "C" fn rust_tlb_flush_walk_noop(
221 _iova: usize,
222 _size: usize,
223 _granule: usize,
224 _cookie: *mut core::ffi::c_void,
225) {
226}
227
228impl<F: IoPageTableFmt> Drop for IoPageTable<'_, F> {
229 fn drop(&mut self) {
230 // SAFETY: The caller of `Self::ttbr()` promised that the page table is not live when this
231 // destructor runs.
232 unsafe { bindings::free_io_pgtable_ops(self.raw_ops()) };
233 }
234}
235
236/// The `ARM_64_LPAE_S1` page table format.
237pub enum ARM64LPAES1 {}
238
239impl IoPageTableFmt for ARM64LPAES1 {
240 const FORMAT: io_pgtable_fmt = bindings::io_pgtable_fmt_ARM_64_LPAE_S1 as io_pgtable_fmt;
241}
242
243impl IoPageTable<'_, ARM64LPAES1> {
244 /// Access the `ttbr` field of the configuration.
245 ///
246 /// This is the physical address of the page table, which may be passed to the device that
247 /// needs to use it.
248 ///
249 /// # Safety
250 ///
251 /// The caller must ensure that the device stops using the page table before dropping it.
252 #[inline]
253 pub unsafe fn ttbr(&self) -> u64 {
254 // SAFETY: `arm_lpae_s1_cfg` is the right cfg type for `ARM64LPAES1`.
255 unsafe { (*self.raw_cfg()).__bindgen_anon_1.arm_lpae_s1_cfg.ttbr }
256 }
257
258 /// Access the `mair` field of the configuration.
259 #[inline]
260 pub fn mair(&self) -> u64 {
261 // SAFETY: `arm_lpae_s1_cfg` is the right cfg type for `ARM64LPAES1`.
262 unsafe { (*self.raw_cfg()).__bindgen_anon_1.arm_lpae_s1_cfg.mair }
263 }
264}