projects/linux·blob uaccess-cleanups · refs

1754 lines · 53.5 KiB

1// SPDX-License-Identifier: GPL-2.0
2
3//! Memory-mapped IO.
4//!
5//! C header: [`include/asm-generic/io.h`](srctree/include/asm-generic/io.h)
6
7use core::{
8 marker::PhantomData,
9 mem::MaybeUninit, //
10};
11
12use crate::{
13 bindings,
14 prelude::*,
15 ptr::{
16 Alignment,
17 KnownSize, //
18 }, //
19};
20
21#[cfg(CONFIG_HAS_IOMEM)]
22pub mod mem;
23pub mod poll;
24pub mod register;
25pub mod resource;
26
27pub use crate::register;
28pub use resource::Resource;
29
30use register::LocatedRegister;
31
32/// Physical address type.
33///
34/// This is a type alias to either `u32` or `u64` depending on the config option
35/// `CONFIG_PHYS_ADDR_T_64BIT`, and it can be a u64 even on 32-bit architectures.
36pub type PhysAddr = bindings::phys_addr_t;
37
38/// Resource Size type.
39///
40/// This is a type alias to either `u32` or `u64` depending on the config option
41/// `CONFIG_PHYS_ADDR_T_64BIT`, and it can be a u64 even on 32-bit architectures.
42pub type ResourceSize = bindings::resource_size_t;
43
44/// Untyped I/O region.
45///
46/// This type can be used when an I/O region without known type information has a compile-time known
47/// minimum size (and a runtime known actual size).
48///
49/// # Invariants
50///
51/// - Size of the region is at least as large as the `SIZE` generic parameter.
52/// - Size of the region is multiple of 4.
53#[repr(C, align(4))]
54#[derive(FromBytes)]
55pub struct Region<const SIZE: usize = 0> {
56 inner: [u8],
57}
58
59impl<const SIZE: usize> Region<SIZE> {
60 /// Create a raw mutable pointer from given base address and size.
61 ///
62 /// `size` should be at least as large as the minimum size `SIZE`, and `base` and `size` should
63 /// be 4-byte aligned to uphold the type invariant.
64 ///
65 /// Just like other methods on raw pointers, it is not unsafe to create a raw pointer
66 /// that does not uphold the type invariants. However such pointers are not valid.
67 #[inline]
68 pub fn ptr_from_raw_parts_mut(base: *mut u8, size: usize) -> *mut Self {
69 core::ptr::slice_from_raw_parts_mut(base, size) as *mut Region<SIZE>
70 }
71
72 /// Create a raw mutable pointer from given base address and size.
73 ///
74 /// The alignment of `base` is checked, and `size` is checked against the minimum size specified
75 /// via const generics.
76 #[inline]
77 pub fn ptr_try_from_raw_parts_mut(base: *mut u8, size: usize) -> Result<*mut Self> {
78 if size < SIZE || base.align_offset(4) != 0 || !size.is_multiple_of(4) {
79 return Err(EINVAL);
80 }
81
82 Ok(Self::ptr_from_raw_parts_mut(base, size))
83 }
84}
85
86impl<const SIZE: usize> KnownSize for Region<SIZE> {
87 const MIN_SIZE: usize = SIZE;
88 // Alignment of 4 is the most common; different base types can be added once required.
89 const MIN_ALIGN: Alignment = Alignment::new::<4>();
90
91 #[inline(always)]
92 fn size(p: *const Self) -> usize {
93 (p as *const [u8]).len()
94 }
95}
96
97// SAFETY:
98// - Values read from I/O are always treated as initialized.
99// - Per type invariant the size is multiple of 4 and the type is 4-byte aligned, so it is padding
100// free.
101//
102// This cannot be derived as `derive(IntoBytes)` as the padding free property comes from type
103// invariant which the macro does not know.
104unsafe impl<const SIZE: usize> IntoBytes for Region<SIZE> {
105 #[inline]
106 #[allow(unused)] // Rust 1.87+ stops requiring this and will emit unused warnings.
107 fn only_derive_is_allowed_to_implement_this_trait() {}
108}
109
110/// Raw representation of an MMIO region.
111///
112/// `MmioRaw<T>` is equivalent to `T __iomem *` in C.
113///
114/// By itself, the existence of an instance of this structure does not provide any guarantees that
115/// the represented MMIO region does exist or is properly mapped.
116///
117/// Instead, the bus specific MMIO implementation must convert this raw representation into an
118/// `Mmio` instance providing the actual memory accessors. Only by the conversion into an `Mmio`
119/// structure any guarantees are given.
120pub struct MmioRaw<T: ?Sized> {
121 /// Pointer is in I/O address space.
122 ///
123 /// The provenance does not matter, only the address and metadata do.
124 ptr: *mut T,
125}
126
127impl<T: ?Sized> Copy for MmioRaw<T> {}
128impl<T: ?Sized> Clone for MmioRaw<T> {
129 #[inline]
130 fn clone(&self) -> Self {
131 *self
132 }
133}
134
135// SAFETY: `MmioRaw` is just an address, so is thread-safe.
136unsafe impl<T: ?Sized> Send for MmioRaw<T> {}
137// SAFETY: `MmioRaw` is just an address, so is thread-safe.
138unsafe impl<T: ?Sized> Sync for MmioRaw<T> {}
139
140impl<T> MmioRaw<T> {
141 /// Create a `MmioRaw` from address.
142 #[inline]
143 pub fn new(addr: usize) -> Self {
144 Self {
145 ptr: core::ptr::without_provenance_mut(addr),
146 }
147 }
148}
149
150impl<const SIZE: usize> MmioRaw<Region<SIZE>> {
151 /// Create a `MmioRaw` representing a I/O region with given size.
152 ///
153 /// The size is checked against the minimum size specified via const generics.
154 #[inline]
155 pub fn new_region(addr: usize, size: usize) -> Result<Self> {
156 Ok(Self {
157 ptr: Region::ptr_try_from_raw_parts_mut(core::ptr::without_provenance_mut(addr), size)?,
158 })
159 }
160}
161
162impl<T: ?Sized + KnownSize> MmioRaw<T> {
163 /// Returns the base address of the MMIO region.
164 #[inline]
165 pub fn addr(&self) -> usize {
166 self.ptr.addr()
167 }
168
169 /// Returns the size of the MMIO region.
170 #[inline]
171 pub fn size(&self) -> usize {
172 KnownSize::size(self.ptr)
173 }
174}
175
176/// Checks whether an access of type `U` at the given `base` and the given `offset`
177/// is valid within this region.
178///
179/// The `base` is used for alignment checking only. This can be set to 0 to skip the check.
180#[inline]
181const fn offset_valid<U>(base: usize, offset: usize, size: usize) -> bool {
182 if let Some(end) = offset.checked_add(size_of::<U>()) {
183 end <= size && (base.wrapping_add(offset) % align_of::<U>() == 0)
184 } else {
185 false
186 }
187}
188
189/// Returns a view for a given `offset`, performing compile-time bound checks.
190// Always inline to optimize out error path of `build_assert`.
191#[inline(always)]
192fn io_view_assert<'a, IO: Io<'a>, U>(
193 this: IO,
194 offset: usize,
195) -> <IO::Backend as IoBackend>::View<'a, U> {
196 // We cannot check alignment with `offset_valid` using `ptr.addr()`. So set 0 for it and
197 // ensure alignment by checking that the alignment of `U` is smaller or equal to the
198 // alignment of `IO::Target`.
199 const_assert!(Alignment::of::<U>().as_usize() <= IO::Target::MIN_ALIGN.as_usize());
200 build_assert!(offset_valid::<U>(0, offset, IO::Target::MIN_SIZE));
201
202 let view = this.as_view();
203 let ptr = IO::Backend::as_ptr(view);
204 let projected_ptr = ptr.cast::<U>().wrapping_byte_add(offset);
205 // SAFETY: `offset_valid` checks for size and alignment and therefore `projected_ptr` is a
206 // valid projection.
207 unsafe { IO::Backend::project_view(view, projected_ptr) }
208}
209
210/// Returns a view for a given `offset`, performing runtime bound checks.
211#[inline]
212fn io_view<'a, IO: Io<'a>, U>(
213 this: IO,
214 offset: usize,
215) -> Result<<IO::Backend as IoBackend>::View<'a, U>> {
216 let view = this.as_view();
217 let ptr = IO::Backend::as_ptr(view);
218
219 if !offset_valid::<U>(ptr.addr(), offset, KnownSize::size(ptr)) {
220 return Err(EINVAL);
221 }
222
223 let projected_ptr = ptr.cast::<U>().wrapping_byte_add(offset);
224 // SAFETY: `offset_valid` checks for size and alignment and therefore `projected_ptr` is a
225 // valid projection.
226 Ok(unsafe { IO::Backend::project_view(view, projected_ptr) })
227}
228
229/// I/O backends.
230///
231/// This is an abstract representation to be implemented by arbitrary I/O
232/// backends (e.g. MMIO, PCI config space, etc.).
233///
234/// The base trait only defines the projection operations; which I/O methods are available depends
235/// on which [`IoCapable<T>`] traits are implemented for the type. For example, for MMIO regions,
236/// all widths (u8, u16, u32, and u64 on 64-bit systems) are typically supported. For PCI
237/// configuration space, u8, u16, and u32 are supported but u64 is not.
238///
239/// This trait is separate from the `Io` trait as multiple different I/O types may share the same
240/// operation.
241pub trait IoBackend {
242 /// View type for this I/O backend.
243 type View<'a, T: ?Sized + KnownSize>: IoBase<'a, Backend = Self, Target = T>;
244
245 /// Convert a `view` to a raw pointer for projection.
246 ///
247 /// The returned pointer is private implementation detail of the backend; it is likely not
248 /// valid. It should not be dereferenced.
249 fn as_ptr<'a, T: ?Sized + KnownSize>(view: Self::View<'a, T>) -> *mut T;
250
251 /// Project `view` to its subregion indicated by `ptr`.
252 ///
253 /// If input `view` is valid, returned view must also be valid.
254 ///
255 /// # Safety
256 ///
257 /// `ptr` must be a projection of `Self::as_ptr(view)`.
258 unsafe fn project_view<'a, T: ?Sized + KnownSize, U: ?Sized + KnownSize>(
259 view: Self::View<'a, T>,
260 ptr: *mut U,
261 ) -> Self::View<'a, U>;
262}
263
264/// Trait indicating that an I/O backend supports operations of a certain type and providing an
265/// implementation for these operations.
266///
267/// Different I/O backends can implement this trait to expose only the operations they support.
268///
269/// For example, a PCI configuration space may implement `IoCapable<u8>`, `IoCapable<u16>`,
270/// and `IoCapable<u32>`, but not `IoCapable<u64>`, while an MMIO region on a 64-bit
271/// system might implement all four.
272pub trait IoCapable<T>: IoBackend {
273 /// Performs an I/O read of type `T` at `view` and returns the result.
274 fn io_read<'a>(view: Self::View<'a, T>) -> T;
275
276 /// Performs an I/O write of `value` at `view`.
277 fn io_write<'a>(view: Self::View<'a, T>, value: T);
278}
279
280/// Trait indicating that an I/O backend supports memory copy operations.
281pub trait IoCopyable: IoBackend {
282 /// Copy contents of `view` to `buffer`.
283 ///
284 /// # Safety
285 ///
286 /// - `buffer` is valid for volatile write for `view.size()` bytes.
287 /// - `buffer` should not overlap with `view`.
288 unsafe fn copy_from_io(view: Self::View<'_, [u8]>, buffer: *mut u8);
289
290 /// Copy contents from `buffer` to `view`.
291 ///
292 /// # Safety
293 ///
294 /// - `buffer` is valid for volatile read for `view.size()` bytes.
295 /// - `buffer` should not overlap with `view`.
296 unsafe fn copy_to_io(view: Self::View<'_, [u8]>, buffer: *const u8);
297
298 /// Copy from `view` and return the value.
299 #[inline]
300 fn copy_read<T: FromBytes>(view: Self::View<'_, T>) -> T {
301 // Project `self` to `[u8]`.
302 let ptr = Self::as_ptr(view);
303 // SAFETY: This is a identity projection.
304 let slice_view = unsafe {
305 Self::project_view(
306 view,
307 core::ptr::slice_from_raw_parts_mut::<u8>(ptr.cast(), size_of::<T>()),
308 )
309 };
310
311 let mut buf = MaybeUninit::<T>::uninit();
312 // SAFETY:
313 // - `buf.as_mut_ptr()` is valid for write for `size_of::<T>()` bytes.
314 // - `buf` is local so `buf.as_mut_ptr()` cannot overlap with `slice_view`.
315 unsafe { Self::copy_from_io(slice_view, buf.as_mut_ptr().cast()) };
316 // SAFETY: `T: FromBytes` guarantee that all bit patterns are valid.
317 unsafe { buf.assume_init() }
318 }
319
320 /// Copy `value` to `view`.
321 ///
322 /// Destructor of `value` will not be executed, consistent with [`zerocopy::transmute`].
323 #[inline]
324 fn copy_write<T: IntoBytes>(view: Self::View<'_, T>, value: T) {
325 // Project `self` to `[u8]`.
326 let ptr = Self::as_ptr(view);
327 // SAFETY: This is a identity projection.
328 let slice_view = unsafe {
329 Self::project_view(
330 view,
331 core::ptr::slice_from_raw_parts_mut::<u8>(ptr.cast(), size_of::<T>()),
332 )
333 };
334
335 // SAFETY:
336 // - `&raw const value` is valid for read for `size_of::<T>()` bytes.
337 // - `value` is local so `&raw const value` cannot overlap with `slice_view`.
338 unsafe { Self::copy_to_io(slice_view, (&raw const value).cast()) };
339 core::mem::forget(value);
340 }
341}
342
343/// Describes a given I/O location: its offset, width, and type to convert the raw value from and
344/// into.
345///
346/// This trait is the key abstraction allowing [`Io::read`], [`Io::write`], and [`Io::update`] (and
347/// their fallible [`try_read`](Io::try_read), [`try_write`](Io::try_write) and
348/// [`try_update`](Io::try_update) counterparts) to work uniformly with both raw [`usize`] offsets
349/// (for primitive types like [`u32`]) and typed ones (like those generated by the [`register!`]
350/// macro).
351///
352/// An `IoLoc<Base, T>` carries the following pieces of information:
353///
354/// - The valid `Base` to operate on. For most registers, this should be [`Region`].
355/// - The offset to access (returned by [`IoLoc::offset`]),
356/// - The width of the access (determined by [`IoLoc::IoType`]),
357/// - The type `T` in which the raw data is returned or provided.
358///
359/// `T` and `IoLoc::IoType` may differ: for instance, a typed register has `T` = the register type
360/// with its bitfields, and `IoType` = its backing primitive (e.g. `u32`).
361pub trait IoLoc<Base: ?Sized, T> {
362 /// Size ([`u8`], [`u16`], etc) of the I/O performed on the returned [`offset`](IoLoc::offset).
363 type IoType: Into<T> + From<T>;
364
365 /// Consumes `self` and returns the offset of this location.
366 fn offset(self) -> usize;
367}
368
369/// Implements [`IoLoc<Region<SIZE>, $ty>`] for [`usize`], allowing [`usize`] to be used as a
370/// parameter of [`Io::read`] and [`Io::write`].
371macro_rules! impl_usize_ioloc {
372 ($($ty:ty),*) => {
373 $(
374 impl<const SIZE: usize> IoLoc<Region<SIZE>, $ty> for usize {
375 type IoType = $ty;
376
377 #[inline(always)]
378 fn offset(self) -> usize {
379 self
380 }
381 }
382 )*
383 }
384}
385
386// Provide the ability to read any primitive type from a [`usize`].
387impl_usize_ioloc!(u8, u16, u32, u64);
388
389/// Types implementing this trait (e.g. MMIO BARs or PCI config regions)
390/// can perform I/O operations on regions of memory.
391///
392/// This trait defines which backend shall be used for I/O operations and provides a method to
393/// convert into [`IoBackend::View`]. Users should use the [`Io`] trait which provides the actual
394/// methods to perform I/O operations.
395///
396/// This should be implemented on cheaply copyable handles, such as references or view types.
397pub trait IoBase<'a>: Copy {
398 /// Type that defines all I/O operations.
399 type Backend: IoBackend;
400
401 /// Type of this I/O region. For untyped regions, [`Region`] can be used.
402 type Target: ?Sized + KnownSize;
403
404 /// Return a view that covers the full region.
405 fn as_view(self) -> <Self::Backend as IoBackend>::View<'a, Self::Target>;
406}
407
408/// Extension trait to provide I/O operation methods to types that implement [`IoBase`].
409///
410/// This trait provides:
411/// - Helper methods for offset validation and address calculation
412/// - Fallible (runtime checked) accessors for different data widths
413///
414/// Which I/O methods are available depends on the associated [`IoBackend`] implementation.
415pub trait Io<'a>: IoBase<'a> {
416 /// Returns the size of this I/O region.
417 #[inline]
418 fn size(self) -> usize {
419 KnownSize::size(Self::Backend::as_ptr(self.as_view()))
420 }
421
422 /// Returns the length of the slice in number of elements.
423 #[inline]
424 fn len<T>(self) -> usize
425 where
426 Self: Io<'a, Target = [T]>,
427 {
428 Self::Backend::as_ptr(self.as_view()).len()
429 }
430
431 /// Returns `true` if the slice has a length of 0.
432 #[inline]
433 fn is_empty<T>(self) -> bool
434 where
435 Self: Io<'a, Target = [T]>,
436 {
437 self.len() == 0
438 }
439
440 /// Try to convert into a different typed I/O view.
441 ///
442 /// A runtime check is performed to ensure that the target type is of same or smaller size to
443 /// current type, and the current view is properly aligned for the target type. Returns
444 /// `Err(EINVAL)` if the runtime check fails.
445 ///
446 /// # Examples
447 ///
448 /// ```no_run
449 /// use kernel::io::{
450 /// io_project,
451 /// Mmio,
452 /// Io,
453 /// Region,
454 /// };
455 /// #[derive(FromBytes, IntoBytes)]
456 /// #[repr(C)]
457 /// struct MyStruct { field: u32, }
458 ///
459 /// # fn test(mmio: &Mmio<'_, Region>) -> Result {
460 /// // let mmio: Mmio<'_, Region>;
461 /// let whole: Mmio<'_, MyStruct> = mmio.try_cast()?;
462 /// # Ok::<(), Error>(()) }
463 /// ```
464 #[inline]
465 fn try_cast<U>(self) -> Result<<Self::Backend as IoBackend>::View<'a, U>>
466 where
467 Self::Target: FromBytes + IntoBytes,
468 U: FromBytes + IntoBytes,
469 {
470 let view = self.as_view();
471 let ptr = Self::Backend::as_ptr(view);
472
473 if size_of::<U>() > KnownSize::size(ptr) {
474 return Err(EINVAL);
475 }
476
477 if ptr.addr() % align_of::<U>() != 0 {
478 return Err(EINVAL);
479 }
480
481 // SAFETY: We have checked bounds and alignment, so this is a valid projection.
482 Ok(unsafe { Self::Backend::project_view(view, ptr.cast()) })
483 }
484
485 /// Read a value from I/O.
486 ///
487 /// This only works for primitives supported by the I/O backend.
488 ///
489 /// # Examples
490 ///
491 /// ```no_run
492 /// # use kernel::io::*;
493 /// # fn test_read_val(mmio: Mmio<'_, u32>) {
494 /// // let mmio: Mmio<'_, u32>;
495 /// let val: u32 = mmio.read_val();
496 /// # }
497 /// ```
498 #[inline]
499 fn read_val(self) -> Self::Target
500 where
501 Self::Backend: IoCapable<Self::Target>,
502 Self::Target: Sized,
503 {
504 Self::Backend::io_read(self.as_view())
505 }
506
507 /// Write a value to I/O.
508 ///
509 /// This only works for primitives supported by the I/O backend.
510 ///
511 /// # Examples
512 ///
513 /// ```no_run
514 /// # use kernel::io::*;
515 /// # fn test_write_val(mmio: Mmio<'_, u32>) {
516 /// // let mmio: Mmio<'_, u32>;
517 /// mmio.write_val(1u32);
518 /// # }
519 /// ```
520 #[inline]
521 fn write_val(self, value: Self::Target)
522 where
523 Self::Backend: IoCapable<Self::Target>,
524 Self::Target: Sized,
525 {
526 Self::Backend::io_write(self.as_view(), value)
527 }
528
529 /// Copy-read from I/O memory.
530 ///
531 /// This is equivalent to reading from the I/O memory with byte-wise copy, although the actual
532 /// implementation might be more efficient. There is no atomicity guarantee. Note that for some
533 /// backends (e.g. `Mmio`), this can read different value compared to [`read_val`] as
534 /// byte-swapping is not performed.
535 ///
536 /// [`read_val`]: Io::read_val
537 ///
538 /// # Examples
539 ///
540 /// ```no_run
541 /// # use kernel::io::*;
542 /// # fn test_copy_read(mmio: Mmio<'_, [u8; 6]>) {
543 /// // let mmio: Mmio<'_, [u8; 6]>;
544 /// let val: [u8; 6] = mmio.copy_read();
545 /// # }
546 /// ```
547 #[inline]
548 fn copy_read(self) -> Self::Target
549 where
550 Self::Backend: IoCopyable,
551 Self::Target: Sized + FromBytes,
552 {
553 Self::Backend::copy_read(self.as_view())
554 }
555
556 /// Copy-write to I/O memory.
557 ///
558 /// This is equivalent to writing to the I/O memory with byte-wise copy, although the actual
559 /// implementation might be more efficient. There is no atomicity guarantee. Note that for some
560 /// backends (e.g. `Mmio`), this can write different value compared to [`write_val`] as
561 /// byte-swapping is not performed.
562 ///
563 /// [`write_val`]: Io::write_val
564 ///
565 /// # Examples
566 ///
567 /// ```no_run
568 /// # use kernel::io::*;
569 /// # fn test_copy_write(mmio: Mmio<'_, [u8; 6]>) {
570 /// // let mmio: Mmio<'_, [u8; 6]>;
571 /// mmio.copy_write([0xAA, 0xBB, 0xCC, 0xDD, 0xEE, 0xFF]);
572 /// # }
573 /// ```
574 #[inline]
575 fn copy_write(self, value: Self::Target)
576 where
577 Self::Backend: IoCopyable,
578 Self::Target: Sized + IntoBytes,
579 {
580 Self::Backend::copy_write(self.as_view(), value);
581 }
582
583 /// Copy bytes from `data` to I/O memory.
584 ///
585 /// # Panics
586 ///
587 /// This function will panic if the length of `self` differs from the length of `data`, similar
588 /// to [`[u8]::copy_from_slice`].
589 ///
590 /// # Examples
591 ///
592 /// ```no_run
593 /// # use kernel::io::*;
594 /// # fn test_copy_write(mmio: Mmio<'_, [u8]>) {
595 /// // let mmio: Mmio<'_, [u8]>;
596 /// mmio.copy_from_slice(&[0xAA, 0xBB, 0xCC, 0xDD, 0xEE, 0xFF]);
597 /// # }
598 /// ```
599 #[inline]
600 fn copy_from_slice(self, data: &[u8])
601 where
602 Self::Backend: IoCopyable,
603 Self: Io<'a, Target = [u8]>,
604 {
605 assert_eq!(self.len(), data.len());
606
607 // SAFETY: `data.as_ptr()` is valid for read for `self.size()` bytes.
608 unsafe {
609 Self::Backend::copy_to_io(self.as_view(), data.as_ptr());
610 }
611 }
612
613 /// Copy bytes from I/O memory to `data`.
614 ///
615 /// # Panics
616 ///
617 /// This function will panic if the length of `self` differs from the length of `data`, similar
618 /// to [`[u8]::copy_from_slice`].
619 ///
620 /// # Examples
621 ///
622 /// ```no_run
623 /// # use kernel::io::*;
624 /// # fn test_copy_write(mmio: Mmio<'_, [u8]>) {
625 /// // let mmio: Mmio<'_, [u8]>;
626 /// let mut buf = [0; 6];
627 /// mmio.copy_to_slice(&mut buf);
628 /// # }
629 /// ```
630 #[inline]
631 fn copy_to_slice(self, data: &mut [u8])
632 where
633 Self::Backend: IoCopyable,
634 Self: Io<'a, Target = [u8]>,
635 {
636 assert_eq!(self.len(), data.len());
637
638 // SAFETY: `data.as_mut_ptr()` is valid for write for `self.size()` bytes.
639 unsafe {
640 Self::Backend::copy_from_io(self.as_view(), data.as_mut_ptr());
641 }
642 }
643
644 /// Fallible 8-bit read with runtime bounds check.
645 #[inline(always)]
646 fn try_read8(self, offset: usize) -> Result<u8>
647 where
648 usize: IoLoc<Self::Target, u8, IoType = u8>,
649 Self::Backend: IoCapable<u8>,
650 {
651 self.try_read(offset)
652 }
653
654 /// Fallible 16-bit read with runtime bounds check.
655 #[inline(always)]
656 fn try_read16(self, offset: usize) -> Result<u16>
657 where
658 usize: IoLoc<Self::Target, u16, IoType = u16>,
659 Self::Backend: IoCapable<u16>,
660 {
661 self.try_read(offset)
662 }
663
664 /// Fallible 32-bit read with runtime bounds check.
665 #[inline(always)]
666 fn try_read32(self, offset: usize) -> Result<u32>
667 where
668 usize: IoLoc<Self::Target, u32, IoType = u32>,
669 Self::Backend: IoCapable<u32>,
670 {
671 self.try_read(offset)
672 }
673
674 /// Fallible 64-bit read with runtime bounds check.
675 #[inline(always)]
676 fn try_read64(self, offset: usize) -> Result<u64>
677 where
678 usize: IoLoc<Self::Target, u64, IoType = u64>,
679 Self::Backend: IoCapable<u64>,
680 {
681 self.try_read(offset)
682 }
683
684 /// Fallible 8-bit write with runtime bounds check.
685 #[inline(always)]
686 fn try_write8(self, value: u8, offset: usize) -> Result
687 where
688 usize: IoLoc<Self::Target, u8, IoType = u8>,
689 Self::Backend: IoCapable<u8>,
690 {
691 self.try_write(offset, value)
692 }
693
694 /// Fallible 16-bit write with runtime bounds check.
695 #[inline(always)]
696 fn try_write16(self, value: u16, offset: usize) -> Result
697 where
698 usize: IoLoc<Self::Target, u16, IoType = u16>,
699 Self::Backend: IoCapable<u16>,
700 {
701 self.try_write(offset, value)
702 }
703
704 /// Fallible 32-bit write with runtime bounds check.
705 #[inline(always)]
706 fn try_write32(self, value: u32, offset: usize) -> Result
707 where
708 usize: IoLoc<Self::Target, u32, IoType = u32>,
709 Self::Backend: IoCapable<u32>,
710 {
711 self.try_write(offset, value)
712 }
713
714 /// Fallible 64-bit write with runtime bounds check.
715 #[inline(always)]
716 fn try_write64(self, value: u64, offset: usize) -> Result
717 where
718 usize: IoLoc<Self::Target, u64, IoType = u64>,
719 Self::Backend: IoCapable<u64>,
720 {
721 self.try_write(offset, value)
722 }
723
724 /// Infallible 8-bit read with compile-time bounds check.
725 ///
726 /// `offset` should be constant.
727 #[inline(always)]
728 fn read8(self, offset: usize) -> u8
729 where
730 usize: IoLoc<Self::Target, u8, IoType = u8>,
731 Self::Backend: IoCapable<u8>,
732 {
733 self.read(offset)
734 }
735
736 /// Infallible 16-bit read with compile-time bounds check.
737 ///
738 /// `offset` should be constant.
739 #[inline(always)]
740 fn read16(self, offset: usize) -> u16
741 where
742 usize: IoLoc<Self::Target, u16, IoType = u16>,
743 Self::Backend: IoCapable<u16>,
744 {
745 self.read(offset)
746 }
747
748 /// Infallible 32-bit read with compile-time bounds check.
749 ///
750 /// `offset` should be constant.
751 #[inline(always)]
752 fn read32(self, offset: usize) -> u32
753 where
754 usize: IoLoc<Self::Target, u32, IoType = u32>,
755 Self::Backend: IoCapable<u32>,
756 {
757 self.read(offset)
758 }
759
760 /// Infallible 64-bit read with compile-time bounds check.
761 ///
762 /// `offset` should be constant.
763 #[inline(always)]
764 fn read64(self, offset: usize) -> u64
765 where
766 usize: IoLoc<Self::Target, u64, IoType = u64>,
767 Self::Backend: IoCapable<u64>,
768 {
769 self.read(offset)
770 }
771
772 /// Infallible 8-bit write with compile-time bounds check.
773 ///
774 /// `offset` should be constant.
775 #[inline(always)]
776 fn write8(self, value: u8, offset: usize)
777 where
778 usize: IoLoc<Self::Target, u8, IoType = u8>,
779 Self::Backend: IoCapable<u8>,
780 {
781 self.write(offset, value)
782 }
783
784 /// Infallible 16-bit write with compile-time bounds check.
785 ///
786 /// `offset` should be constant.
787 #[inline(always)]
788 fn write16(self, value: u16, offset: usize)
789 where
790 usize: IoLoc<Self::Target, u16, IoType = u16>,
791 Self::Backend: IoCapable<u16>,
792 {
793 self.write(offset, value)
794 }
795
796 /// Infallible 32-bit write with compile-time bounds check.
797 ///
798 /// `offset` should be constant.
799 #[inline(always)]
800 fn write32(self, value: u32, offset: usize)
801 where
802 usize: IoLoc<Self::Target, u32, IoType = u32>,
803 Self::Backend: IoCapable<u32>,
804 {
805 self.write(offset, value)
806 }
807
808 /// Infallible 64-bit write with compile-time bounds check.
809 ///
810 /// `offset` should be constant.
811 #[inline(always)]
812 fn write64(self, value: u64, offset: usize)
813 where
814 usize: IoLoc<Self::Target, u64, IoType = u64>,
815 Self::Backend: IoCapable<u64>,
816 {
817 self.write(offset, value)
818 }
819
820 /// Generic fallible read with runtime bounds check.
821 ///
822 /// # Examples
823 ///
824 /// Read a primitive type from an I/O address:
825 ///
826 /// ```no_run
827 /// use kernel::io::{
828 /// Io,
829 /// Mmio,
830 /// Region,
831 /// };
832 ///
833 /// fn do_reads(io: Mmio<'_, Region>) -> Result {
834 /// // 32-bit read from address `0x10`.
835 /// let v: u32 = io.try_read(0x10)?;
836 ///
837 /// // 8-bit read from address `0xfff`.
838 /// let v: u8 = io.try_read(0xfff)?;
839 ///
840 /// Ok(())
841 /// }
842 /// ```
843 #[inline(always)]
844 fn try_read<T, L>(self, location: L) -> Result<T>
845 where
846 L: IoLoc<Self::Target, T>,
847 Self::Backend: IoCapable<L::IoType>,
848 {
849 let view = io_view::<Self, L::IoType>(self, location.offset())?;
850 Ok(Self::Backend::io_read(view).into())
851 }
852
853 /// Generic fallible write with runtime bounds check.
854 ///
855 /// # Examples
856 ///
857 /// Write a primitive type to an I/O address:
858 ///
859 /// ```no_run
860 /// use kernel::io::{
861 /// Io,
862 /// Mmio,
863 /// Region,
864 /// };
865 ///
866 /// fn do_writes(io: Mmio<'_, Region>) -> Result {
867 /// // 32-bit write of value `1` at address `0x10`.
868 /// io.try_write(0x10, 1u32)?;
869 ///
870 /// // 8-bit write of value `0xff` at address `0xfff`.
871 /// io.try_write(0xfff, 0xffu8)?;
872 ///
873 /// Ok(())
874 /// }
875 /// ```
876 #[inline(always)]
877 fn try_write<T, L>(self, location: L, value: T) -> Result
878 where
879 L: IoLoc<Self::Target, T>,
880 Self::Backend: IoCapable<L::IoType>,
881 {
882 let view = io_view::<Self, L::IoType>(self, location.offset())?;
883 let io_value = value.into();
884 Self::Backend::io_write(view, io_value);
885 Ok(())
886 }
887
888 /// Generic fallible write of a fully-located register value.
889 ///
890 /// # Examples
891 ///
892 /// Tuples carrying a location and a value can be used with this method:
893 ///
894 /// ```no_run
895 /// use kernel::io::{
896 /// register,
897 /// Io,
898 /// Mmio,
899 /// Region,
900 /// };
901 ///
902 /// register! {
903 /// VERSION(u32) @ 0x100 {
904 /// 15:8 major;
905 /// 7:0 minor;
906 /// }
907 /// }
908 ///
909 /// impl VERSION {
910 /// fn new(major: u8, minor: u8) -> Self {
911 /// VERSION::zeroed().with_major(major).with_minor(minor)
912 /// }
913 /// }
914 ///
915 /// fn do_write_reg(io: Mmio<'_, Region>) -> Result {
916 ///
917 /// io.try_write_reg(VERSION::new(1, 0))
918 /// }
919 /// ```
920 #[inline(always)]
921 fn try_write_reg<T, L, V>(self, value: V) -> Result
922 where
923 L: IoLoc<Self::Target, T>,
924 V: LocatedRegister<Self::Target, Location = L, Value = T>,
925 Self::Backend: IoCapable<L::IoType>,
926 {
927 let (location, value) = value.into_io_op();
928
929 self.try_write(location, value)
930 }
931
932 /// Generic fallible update with runtime bounds check.
933 ///
934 /// Note: this does not perform any synchronization. The caller is responsible for ensuring
935 /// exclusive access if required.
936 ///
937 /// # Examples
938 ///
939 /// Read the u32 value at address `0x10`, increment it, and store the updated value back:
940 ///
941 /// ```no_run
942 /// use kernel::io::{
943 /// Io,
944 /// Mmio,
945 /// Region,
946 /// };
947 ///
948 /// fn do_update(io: Mmio<'_, Region<0x1000>>) -> Result {
949 /// io.try_update(0x10, |v: u32| {
950 /// v + 1
951 /// })
952 /// }
953 /// ```
954 #[inline(always)]
955 fn try_update<T, L, F>(self, location: L, f: F) -> Result
956 where
957 L: IoLoc<Self::Target, T>,
958 Self::Backend: IoCapable<L::IoType>,
959 F: FnOnce(T) -> T,
960 {
961 let view = io_view::<Self, L::IoType>(self, location.offset())?;
962
963 let value: T = Self::Backend::io_read(view).into();
964 let io_value = f(value).into();
965 Self::Backend::io_write(view, io_value);
966
967 Ok(())
968 }
969
970 /// Generic infallible read with compile-time bounds check.
971 ///
972 /// # Examples
973 ///
974 /// Read a primitive type from an I/O address:
975 ///
976 /// ```no_run
977 /// use kernel::io::{
978 /// Io,
979 /// Mmio,
980 /// Region,
981 /// };
982 ///
983 /// fn do_reads(io: Mmio<'_, Region<0x1000>>) {
984 /// // 32-bit read from address `0x10`.
985 /// let v: u32 = io.read(0x10);
986 ///
987 /// // 8-bit read from the top of the I/O space.
988 /// let v: u8 = io.read(0xfff);
989 /// }
990 /// ```
991 #[inline(always)]
992 fn read<T, L>(self, location: L) -> T
993 where
994 L: IoLoc<Self::Target, T>,
995 Self::Backend: IoCapable<L::IoType>,
996 {
997 let view = io_view_assert::<Self, L::IoType>(self, location.offset());
998 Self::Backend::io_read(view).into()
999 }
1000
1001 /// Generic infallible write with compile-time bounds check.
1002 ///
1003 /// # Examples
1004 ///
1005 /// Write a primitive type to an I/O address:
1006 ///
1007 /// ```no_run
1008 /// use kernel::io::{
1009 /// Io,
1010 /// Mmio,
1011 /// Region,
1012 /// };
1013 ///
1014 /// fn do_writes(io: Mmio<'_, Region<0x1000>>) {
1015 /// // 32-bit write of value `1` at address `0x10`.
1016 /// io.write(0x10, 1u32);
1017 ///
1018 /// // 8-bit write of value `0xff` at the top of the I/O space.
1019 /// io.write(0xfff, 0xffu8);
1020 /// }
1021 /// ```
1022 #[inline(always)]
1023 fn write<T, L>(self, location: L, value: T)
1024 where
1025 L: IoLoc<Self::Target, T>,
1026 Self::Backend: IoCapable<L::IoType>,
1027 {
1028 let view = io_view_assert::<Self, L::IoType>(self, location.offset());
1029 let io_value = value.into();
1030 Self::Backend::io_write(view, io_value);
1031 }
1032
1033 /// Generic infallible write of a fully-located register value.
1034 ///
1035 /// # Examples
1036 ///
1037 /// Tuples carrying a location and a value can be used with this method:
1038 ///
1039 /// ```no_run
1040 /// use kernel::io::{
1041 /// register,
1042 /// Io,
1043 /// Mmio,
1044 /// Region,
1045 /// };
1046 ///
1047 /// register! {
1048 /// VERSION(u32) @ 0x100 {
1049 /// 15:8 major;
1050 /// 7:0 minor;
1051 /// }
1052 /// }
1053 ///
1054 /// impl VERSION {
1055 /// fn new(major: u8, minor: u8) -> Self {
1056 /// VERSION::zeroed().with_major(major).with_minor(minor)
1057 /// }
1058 /// }
1059 ///
1060 /// fn do_write_reg(io: Mmio<'_, Region<0x1000>>) {
1061 /// io.write_reg(VERSION::new(1, 0));
1062 /// }
1063 /// ```
1064 #[inline(always)]
1065 fn write_reg<T, L, V>(self, value: V)
1066 where
1067 L: IoLoc<Self::Target, T>,
1068 V: LocatedRegister<Self::Target, Location = L, Value = T>,
1069 Self::Backend: IoCapable<L::IoType>,
1070 {
1071 let (location, value) = value.into_io_op();
1072
1073 self.write(location, value)
1074 }
1075
1076 /// Generic infallible update with compile-time bounds check.
1077 ///
1078 /// Note: this does not perform any synchronization. The caller is responsible for ensuring
1079 /// exclusive access if required.
1080 ///
1081 /// # Examples
1082 ///
1083 /// Read the u32 value at address `0x10`, increment it, and store the updated value back:
1084 ///
1085 /// ```no_run
1086 /// use kernel::io::{
1087 /// Io,
1088 /// Mmio,
1089 /// Region,
1090 /// };
1091 ///
1092 /// fn do_update(io: Mmio<'_, Region<0x1000>>) {
1093 /// io.update(0x10, |v: u32| {
1094 /// v + 1
1095 /// })
1096 /// }
1097 /// ```
1098 #[inline(always)]
1099 fn update<T, L, F>(self, location: L, f: F)
1100 where
1101 L: IoLoc<Self::Target, T>,
1102 Self::Backend: IoCapable<L::IoType>,
1103 F: FnOnce(T) -> T,
1104 {
1105 let view = io_view_assert::<Self, L::IoType>(self, location.offset());
1106 let value: T = Self::Backend::io_read(view).into();
1107 let io_value = f(value).into();
1108 Self::Backend::io_write(view, io_value);
1109 }
1110}
1111
1112// Blanket implementation ensures that provided methods cannot be arbitrarily overridden by
1113// implementers, which is relied upon for correctness and soundness.
1114impl<'a, T: IoBase<'a>> Io<'a> for T {}
1115
1116/// A view of memory-mapped I/O region.
1117///
1118/// # Invariant
1119///
1120/// `ptr` points to a valid and aligned memory-mapped I/O region for the duration lifetime `'a`.
1121pub struct Mmio<'a, T: ?Sized> {
1122 ptr: *mut T,
1123 phantom: PhantomData<&'a ()>,
1124}
1125
1126impl<T: ?Sized> Copy for Mmio<'_, T> {}
1127impl<T: ?Sized> Clone for Mmio<'_, T> {
1128 #[inline]
1129 fn clone(&self) -> Self {
1130 *self
1131 }
1132}
1133
1134impl<'a, T: ?Sized> Mmio<'a, T> {
1135 /// Create a `Mmio`, providing the accessors to the MMIO mapping.
1136 ///
1137 /// # Safety
1138 ///
1139 /// `raw` represents a valid and aligned memory-mapped I/O region while `'a` is alive.
1140 #[inline]
1141 pub unsafe fn from_raw(raw: MmioRaw<T>) -> Self {
1142 // INVARIANT: Per safety requirement.
1143 Self {
1144 ptr: raw.ptr,
1145 phantom: PhantomData,
1146 }
1147 }
1148}
1149
1150// SAFETY: `Mmio<'_, T>` is conceptually `&T` but in I/O memory.
1151unsafe impl<T: ?Sized + Sync> Send for Mmio<'_, T> {}
1152
1153// SAFETY: `Mmio<'_, T>` is conceptually `&T` but in I/O memory.
1154unsafe impl<T: ?Sized + Sync> Sync for Mmio<'_, T> {}
1155
1156impl<'a, T: ?Sized + KnownSize> IoBase<'a> for Mmio<'a, T> {
1157 type Backend = MmioBackend;
1158 type Target = T;
1159
1160 #[inline]
1161 fn as_view(self) -> Mmio<'a, T> {
1162 self
1163 }
1164}
1165
1166/// I/O Backend for memory-mapped I/O.
1167pub struct MmioBackend;
1168
1169impl IoBackend for MmioBackend {
1170 type View<'a, T: ?Sized + KnownSize> = Mmio<'a, T>;
1171
1172 #[inline]
1173 fn as_ptr<'a, T: ?Sized + KnownSize>(view: Self::View<'a, T>) -> *mut T {
1174 view.ptr
1175 }
1176
1177 #[inline]
1178 unsafe fn project_view<'a, T: ?Sized + KnownSize, U: ?Sized + KnownSize>(
1179 _view: Self::View<'a, T>,
1180 ptr: *mut U,
1181 ) -> Self::View<'a, U> {
1182 // INVARIANT: Per safety requirement, `ptr` is projection from `view`, so it is also a valid
1183 // memory-mapped I/O region.
1184 Mmio {
1185 ptr,
1186 phantom: PhantomData,
1187 }
1188 }
1189}
1190
1191/// Implements [`IoCapable`] on `$backend` for `$ty` using `$read_fn` and `$write_fn`.
1192macro_rules! impl_mmio_io_capable {
1193 ($backend: ident, $ty:ty, $read_fn:ident, $write_fn:ident) => {
1194 impl IoCapable<$ty> for $backend {
1195 #[inline]
1196 fn io_read(view: <$backend as IoBackend>::View<'_, $ty>) -> $ty {
1197 // SAFETY: `$backend::as_ptr(view)` is a valid pointer for MMIO operations for both
1198 // `MmioBackend` and `RelaxedMmioBackend`.
1199 unsafe { bindings::$read_fn($backend::as_ptr(view).cast_const().cast()) }
1200 }
1201
1202 #[inline]
1203 fn io_write(view: <$backend as IoBackend>::View<'_, $ty>, value: $ty) {
1204 // SAFETY: `$backend::as_ptr(view)` is a valid pointer for MMIO operations for both
1205 // `MmioBackend` and `RelaxedMmioBackend`.
1206 unsafe { bindings::$write_fn(value, $backend::as_ptr(view).cast()) }
1207 }
1208 }
1209 };
1210}
1211
1212// MMIO regions support 8, 16, and 32-bit accesses.
1213impl_mmio_io_capable!(MmioBackend, u8, readb, writeb);
1214impl_mmio_io_capable!(MmioBackend, u16, readw, writew);
1215impl_mmio_io_capable!(MmioBackend, u32, readl, writel);
1216// MMIO regions on 64-bit systems also support 64-bit accesses.
1217#[cfg(CONFIG_64BIT)]
1218impl_mmio_io_capable!(MmioBackend, u64, readq, writeq);
1219
1220impl IoCopyable for MmioBackend {
1221 #[inline]
1222 unsafe fn copy_from_io(view: Self::View<'_, [u8]>, buffer: *mut u8) {
1223 // SAFETY:
1224 // - `view.ptr` is valid MMIO memory for `view.size()` bytes.
1225 // - `buffer` is valid for write for `view.size()` bytes.
1226 unsafe {
1227 bindings::memcpy_fromio(buffer.cast(), view.ptr.cast(), view.size());
1228 }
1229 }
1230
1231 #[inline]
1232 unsafe fn copy_to_io(view: Self::View<'_, [u8]>, buffer: *const u8) {
1233 // SAFETY:
1234 // - `view.ptr` is valid MMIO memory for `view.size()` bytes.
1235 // - `buffer` is valid for read for `view.size()` bytes.
1236 unsafe {
1237 bindings::memcpy_toio(view.ptr.cast(), buffer.cast(), view.size());
1238 }
1239 }
1240}
1241
1242/// [`Mmio`] but using relaxed accessors.
1243///
1244/// This type provides an implementation of [`Io`] that uses relaxed I/O MMIO operands instead of
1245/// the regular ones.
1246///
1247/// See [`Mmio::relaxed`] for a usage example.
1248pub struct RelaxedMmio<'a, T: ?Sized>(Mmio<'a, T>);
1249
1250impl<T: ?Sized> Copy for RelaxedMmio<'_, T> {}
1251impl<T: ?Sized> Clone for RelaxedMmio<'_, T> {
1252 #[inline]
1253 fn clone(&self) -> Self {
1254 *self
1255 }
1256}
1257
1258/// I/O Backend for memory-mapped I/O, with relaxed access semantics.
1259pub struct RelaxedMmioBackend;
1260
1261impl IoBackend for RelaxedMmioBackend {
1262 type View<'a, T: ?Sized + KnownSize> = RelaxedMmio<'a, T>;
1263
1264 #[inline]
1265 fn as_ptr<'a, T: ?Sized + KnownSize>(view: Self::View<'a, T>) -> *mut T {
1266 MmioBackend::as_ptr(view.0)
1267 }
1268
1269 #[inline]
1270 unsafe fn project_view<'a, T: ?Sized + KnownSize, U: ?Sized + KnownSize>(
1271 view: Self::View<'a, T>,
1272 ptr: *mut U,
1273 ) -> Self::View<'a, U> {
1274 // SAFETY: Per safety requirement.
1275 RelaxedMmio(unsafe { MmioBackend::project_view(view.0, ptr) })
1276 }
1277}
1278
1279impl<'a, T: ?Sized + KnownSize> IoBase<'a> for RelaxedMmio<'a, T> {
1280 type Backend = RelaxedMmioBackend;
1281 type Target = T;
1282
1283 #[inline]
1284 fn as_view(self) -> RelaxedMmio<'a, T> {
1285 self
1286 }
1287}
1288
1289impl<'a, T: ?Sized> Mmio<'a, T> {
1290 /// Returns a [`RelaxedMmio`] that performs relaxed I/O operations.
1291 ///
1292 /// Relaxed accessors do not provide ordering guarantees with respect to DMA or memory accesses
1293 /// and can be used when such ordering is not required.
1294 ///
1295 /// # Examples
1296 ///
1297 /// ```no_run
1298 /// use kernel::io::{
1299 /// Io,
1300 /// Mmio,
1301 /// Region,
1302 /// RelaxedMmio,
1303 /// };
1304 ///
1305 /// fn do_io(io: Mmio<'_, Region<0x100>>) {
1306 /// // The access is performed using `readl_relaxed` instead of `readl`.
1307 /// let v = io.relaxed().read32(0x10);
1308 /// }
1309 ///
1310 /// ```
1311 #[inline]
1312 pub fn relaxed(self) -> RelaxedMmio<'a, T> {
1313 RelaxedMmio(self)
1314 }
1315}
1316
1317// MMIO regions support 8, 16, and 32-bit accesses.
1318impl_mmio_io_capable!(RelaxedMmioBackend, u8, readb_relaxed, writeb_relaxed);
1319impl_mmio_io_capable!(RelaxedMmioBackend, u16, readw_relaxed, writew_relaxed);
1320impl_mmio_io_capable!(RelaxedMmioBackend, u32, readl_relaxed, writel_relaxed);
1321// MMIO regions on 64-bit systems also support 64-bit accesses.
1322#[cfg(CONFIG_64BIT)]
1323impl_mmio_io_capable!(RelaxedMmioBackend, u64, readq_relaxed, writeq_relaxed);
1324
1325/// I/O Backend for system memory.
1326pub struct SysMemBackend;
1327
1328impl IoBackend for SysMemBackend {
1329 type View<'a, T: ?Sized + KnownSize> = SysMem<'a, T>;
1330
1331 #[inline]
1332 fn as_ptr<'a, T: ?Sized + KnownSize>(view: Self::View<'a, T>) -> *mut T {
1333 view.ptr
1334 }
1335
1336 #[inline]
1337 unsafe fn project_view<'a, T: ?Sized + KnownSize, U: ?Sized + KnownSize>(
1338 _view: Self::View<'a, T>,
1339 ptr: *mut U,
1340 ) -> Self::View<'a, U> {
1341 // INVARIANT: Per safety requirement, `ptr` is projection from `view`, so it is also a valid
1342 // kernel accessible memory region.
1343 SysMem {
1344 ptr,
1345 phantom: PhantomData,
1346 }
1347 }
1348}
1349
1350/// Implements [`IoCapable`] on `SysMemBackend` for `$ty` using `read_volatile` and
1351/// `write_volatile`.
1352macro_rules! impl_sysmem_io_capable {
1353 ($ty:ty) => {
1354 impl IoCapable<$ty> for SysMemBackend {
1355 #[inline]
1356 fn io_read(view: SysMem<'_, $ty>) -> $ty {
1357 // SAFETY:
1358 // - Per type invariant, `ptr` is valid and aligned.
1359 // - Using read_volatile() here so that race with hardware is well-defined.
1360 // - Using read_volatile() here is not sound if it races with other CPU per Rust
1361 // rules, but this is allowed per LKMM.
1362 // - The macro is only used on primitives so all bit patterns are valid.
1363 unsafe { view.ptr.read_volatile() }
1364 }
1365
1366 #[inline]
1367 fn io_write(view: SysMem<'_, $ty>, value: $ty) {
1368 // SAFETY:
1369 // - Per type invariant, `ptr` is valid and aligned.
1370 // - Using write_volatile() here so that race with hardware is well-defined.
1371 // - Using write_volatile() here is not sound if it races with other CPU per Rust
1372 // rules, but this is allowed per LKMM.
1373 unsafe { view.ptr.write_volatile(value) }
1374 }
1375 }
1376 };
1377}
1378
1379impl_sysmem_io_capable!(u8);
1380impl_sysmem_io_capable!(u16);
1381impl_sysmem_io_capable!(u32);
1382#[cfg(CONFIG_64BIT)]
1383impl_sysmem_io_capable!(u64);
1384
1385impl IoCopyable for SysMemBackend {
1386 #[inline]
1387 unsafe fn copy_from_io(view: Self::View<'_, [u8]>, buffer: *mut u8) {
1388 // Use `bindings::memcpy` instead of `copy_nonoverlapping` for volatile.
1389 // SAFETY:
1390 // - `view.ptr` is in CPU address space and valid for read.
1391 // - `buffer` is valid for write for `view.size()` bytes which is equal to `view.ptr.len()`.
1392 unsafe { bindings::memcpy(buffer.cast(), view.ptr.cast(), view.ptr.len()) };
1393 }
1394
1395 #[inline]
1396 unsafe fn copy_to_io(view: Self::View<'_, [u8]>, buffer: *const u8) {
1397 // Use `bindings::memcpy` instead of `copy_nonoverlapping` for volatile.
1398 // SAFETY:
1399 // - `view.ptr` is in CPU address space and valid for write.
1400 // - `buffer` is valid for read for `view.size()` bytes which is equal to `view.ptr.len()`.
1401 unsafe { bindings::memcpy(view.ptr.cast(), buffer.cast(), view.ptr.len()) };
1402 }
1403
1404 #[inline]
1405 fn copy_read<T: FromBytes>(view: Self::View<'_, T>) -> T {
1406 // SAFETY:
1407 // - Per type invariant, `ptr` is valid and aligned.
1408 // - Using read_volatile() here so that race with hardware is well-defined.
1409 // - Using read_volatile() here is not sound if it races with other CPU per Rust
1410 // rules, but this is allowed per LKMM.
1411 // - `T: FromBytes` so all bit patterns are valid.
1412 unsafe { view.ptr.read_volatile() }
1413 }
1414
1415 #[inline]
1416 fn copy_write<T: IntoBytes>(view: Self::View<'_, T>, value: T) {
1417 // SAFETY:
1418 // - Per type invariant, `ptr` is valid and aligned.
1419 // - Using write_volatile() here so that race with hardware is well-defined.
1420 // - Using write_volatile() here is not sound if it races with other CPU per Rust
1421 // rules, but this is allowed per LKMM.
1422 unsafe { view.ptr.write_volatile(value) }
1423 }
1424}
1425
1426/// A view of a system memory region.
1427///
1428/// Provides `Io` trait implementation for kernel virtual address ranges,
1429/// using volatile read/write to safely access shared memory that may be
1430/// concurrently accessed by external hardware.
1431///
1432/// # Invariants
1433///
1434/// `self.ptr.addr() .. self.ptr.addr() + KnownSize::size(self.ptr)` is valid and aligned kernel
1435/// accessible memory region for the lifetime `'a`.
1436pub struct SysMem<'a, T: ?Sized> {
1437 ptr: *mut T,
1438 phantom: PhantomData<&'a ()>,
1439}
1440
1441impl<T: ?Sized> Copy for SysMem<'_, T> {}
1442impl<T: ?Sized> Clone for SysMem<'_, T> {
1443 #[inline]
1444 fn clone(&self) -> Self {
1445 *self
1446 }
1447}
1448
1449// SAFETY: `SysMem<'_, T>` is conceptually `&T`.
1450unsafe impl<T: ?Sized + Sync> Send for SysMem<'_, T> {}
1451
1452// SAFETY: `SysMem<'_, T>` is conceptually `&T`.
1453unsafe impl<T: ?Sized + Sync> Sync for SysMem<'_, T> {}
1454
1455impl<'a, T: ?Sized> SysMem<'a, T> {
1456 /// Create a `SysMem` from a raw pointer.
1457 ///
1458 /// # Safety
1459 ///
1460 /// `ptr.addr() .. ptr.addr() + KnownSize::size(ptr)` must be valid and aligned kernel
1461 /// accessible memory region for the lifetime `'a`.
1462 #[inline]
1463 pub unsafe fn new(ptr: *mut T) -> Self {
1464 // INVARIANT: Per safety requirement.
1465 Self {
1466 ptr,
1467 phantom: PhantomData,
1468 }
1469 }
1470
1471 /// Obtain the raw pointer to the memory.
1472 #[inline]
1473 pub fn as_ptr(self) -> *mut T {
1474 self.ptr
1475 }
1476}
1477
1478impl<'a, T: ?Sized + KnownSize> IoBase<'a> for SysMem<'a, T> {
1479 type Backend = SysMemBackend;
1480 type Target = T;
1481
1482 #[inline]
1483 fn as_view(self) -> <Self::Backend as IoBackend>::View<'a, Self::Target> {
1484 self
1485 }
1486}
1487
1488/// I/O Backend for [`IoSysMap`].
1489pub struct IoSysMapBackend;
1490
1491/// Either [`Mmio`] or [`SysMem`].
1492///
1493/// This can be used when a piece of logic may wish to handle both MMIO or system memory but does
1494/// not want or cannot be generic over I/O backends. This serves a similar purpose to
1495/// [`include/linux/iosys-map.h`] in C.
1496///
1497/// This type can be used like any other types that implements [`Io`]; this also include
1498/// [`io_project!`], [`io_read!`], [`io_write!`].
1499///
1500/// [`include/linux/iosys-map.h`]: srctree/include/linux/iosys-map.h
1501pub enum IoSysMap<'a, T: ?Sized> {
1502 /// The view is I/O memory.
1503 Io(Mmio<'a, T>),
1504 /// The view is system memory.
1505 Sys(SysMem<'a, T>),
1506}
1507
1508impl<T: ?Sized> Copy for IoSysMap<'_, T> {}
1509impl<T: ?Sized> Clone for IoSysMap<'_, T> {
1510 #[inline]
1511 fn clone(&self) -> Self {
1512 *self
1513 }
1514}
1515
1516impl<'a, T: ?Sized> From<Mmio<'a, T>> for IoSysMap<'a, T> {
1517 #[inline]
1518 fn from(value: Mmio<'a, T>) -> Self {
1519 IoSysMap::Io(value)
1520 }
1521}
1522
1523impl<'a, T: ?Sized> From<SysMem<'a, T>> for IoSysMap<'a, T> {
1524 #[inline]
1525 fn from(value: SysMem<'a, T>) -> Self {
1526 IoSysMap::Sys(value)
1527 }
1528}
1529
1530impl IoBackend for IoSysMapBackend {
1531 type View<'a, T: ?Sized + KnownSize> = IoSysMap<'a, T>;
1532
1533 #[inline]
1534 fn as_ptr<'a, T: ?Sized + KnownSize>(view: Self::View<'a, T>) -> *mut T {
1535 match view {
1536 IoSysMap::Io(l) => MmioBackend::as_ptr(l),
1537 IoSysMap::Sys(r) => SysMemBackend::as_ptr(r),
1538 }
1539 }
1540
1541 #[inline]
1542 unsafe fn project_view<'a, T: ?Sized + KnownSize, U: ?Sized + KnownSize>(
1543 view: Self::View<'a, T>,
1544 ptr: *mut U,
1545 ) -> Self::View<'a, U> {
1546 match view {
1547 // SAFETY: Per safety requirement.
1548 IoSysMap::Io(l) => IoSysMap::Io(unsafe { MmioBackend::project_view(l, ptr) }),
1549 // SAFETY: Per safety requirement.
1550 IoSysMap::Sys(r) => IoSysMap::Sys(unsafe { SysMemBackend::project_view(r, ptr) }),
1551 }
1552 }
1553}
1554
1555impl<T> IoCapable<T> for IoSysMapBackend
1556where
1557 MmioBackend: IoCapable<T>,
1558 SysMemBackend: IoCapable<T>,
1559{
1560 #[inline]
1561 fn io_read(view: Self::View<'_, T>) -> T {
1562 match view {
1563 IoSysMap::Io(l) => MmioBackend::io_read(l),
1564 IoSysMap::Sys(r) => SysMemBackend::io_read(r),
1565 }
1566 }
1567
1568 #[inline]
1569 fn io_write<'a>(view: Self::View<'a, T>, value: T) {
1570 match view {
1571 IoSysMap::Io(l) => MmioBackend::io_write(l, value),
1572 IoSysMap::Sys(r) => SysMemBackend::io_write(r, value),
1573 }
1574 }
1575}
1576
1577impl IoCopyable for IoSysMapBackend {
1578 #[inline]
1579 unsafe fn copy_from_io(view: Self::View<'_, [u8]>, buffer: *mut u8) {
1580 match view {
1581 // SAFETY: Per safety requirement.
1582 IoSysMap::Io(l) => unsafe { MmioBackend::copy_from_io(l, buffer) },
1583 // SAFETY: Per safety requirement.
1584 IoSysMap::Sys(r) => unsafe { SysMemBackend::copy_from_io(r, buffer) },
1585 }
1586 }
1587
1588 #[inline]
1589 unsafe fn copy_to_io(view: Self::View<'_, [u8]>, buffer: *const u8) {
1590 match view {
1591 // SAFETY: Per safety requirement.
1592 IoSysMap::Io(l) => unsafe { MmioBackend::copy_to_io(l, buffer) },
1593 // SAFETY: Per safety requirement.
1594 IoSysMap::Sys(r) => unsafe { SysMemBackend::copy_to_io(r, buffer) },
1595 }
1596 }
1597
1598 #[inline]
1599 fn copy_read<T: FromBytes>(view: Self::View<'_, T>) -> T {
1600 match view {
1601 IoSysMap::Io(l) => MmioBackend::copy_read(l),
1602 IoSysMap::Sys(r) => SysMemBackend::copy_read(r),
1603 }
1604 }
1605
1606 #[inline]
1607 fn copy_write<T: IntoBytes>(view: Self::View<'_, T>, value: T) {
1608 match view {
1609 IoSysMap::Io(l) => MmioBackend::copy_write(l, value),
1610 IoSysMap::Sys(r) => SysMemBackend::copy_write(r, value),
1611 }
1612 }
1613}
1614
1615impl<'a, T: ?Sized + KnownSize> IoBase<'a> for IoSysMap<'a, T> {
1616 type Backend = IoSysMapBackend;
1617 type Target = T;
1618
1619 #[inline]
1620 fn as_view(self) -> IoSysMap<'a, T> {
1621 self
1622 }
1623}
1624
1625// This helper turns associated functions to methods so it can be invoked in macro.
1626// Used by `io_project!()` only.
1627#[doc(hidden)]
1628#[derive(Clone, Copy)]
1629pub struct ProjectHelper<T>(pub T);
1630
1631impl<'a, T> ProjectHelper<T>
1632where
1633 T: Io<'a, Backend: IoBackend<View<'a, T::Target> = T>>,
1634{
1635 // These helper methods must not have symbols present in the binary to avoid confusion.
1636 #[inline(always)]
1637 pub fn as_ptr(self) -> *mut T::Target {
1638 T::Backend::as_ptr(self.0)
1639 }
1640
1641 /// # Safety
1642 ///
1643 /// Same as `IoBackend::project_view`
1644 #[inline(always)]
1645 pub unsafe fn project_view<U: ?Sized + KnownSize>(
1646 self,
1647 ptr: *mut U,
1648 ) -> <T::Backend as IoBackend>::View<'a, U> {
1649 // SAFETY: Per safety requirement.
1650 unsafe { T::Backend::project_view::<T::Target, _>(self.0, ptr) }
1651 }
1652}
1653
1654/// Project an I/O type to a subview of it.
1655///
1656/// The syntax is of form `io_project!(io, proj)` where `io` is an expression to a type that
1657/// implements [`Io`] and `proj` is a [projection specification](kernel::ptr::project!).
1658///
1659/// # Examples
1660///
1661/// ```
1662/// use kernel::io::{
1663/// io_project,
1664/// Mmio,
1665/// };
1666/// #[repr(C)]
1667/// struct MyStruct { field: u32, }
1668///
1669/// # fn test(mmio: Mmio<'_, [MyStruct]>) -> Result {
1670/// // let mmio: Mmio<[MyStruct]>;
1671/// let field: Mmio<'_, u32> = io_project!(mmio, [try: 1].field);
1672/// let whole: Mmio<'_, MyStruct> = io_project!(mmio, [try: 2]);
1673/// let nested: Mmio<'_, u32> = io_project!(whole, .field);
1674/// # Ok::<(), Error>(()) }
1675/// ```
1676#[macro_export]
1677#[doc(hidden)]
1678macro_rules! io_project {
1679 ($io:expr, $($proj:tt)*) => {{
1680 #[allow(unused)]
1681 use $crate::io::IoBase as _;
1682 let view = $crate::io::ProjectHelper($io.as_view());
1683 let ptr = $crate::ptr::project!(
1684 mut view.as_ptr(), $($proj)*
1685 );
1686 #[allow(unused_unsafe)]
1687 // SAFETY: `ptr` is a projection.
1688 unsafe { view.project_view(ptr) }
1689 }};
1690}
1691#[doc(inline)]
1692pub use crate::io_project;
1693
1694/// Read from I/O memory.
1695///
1696/// The syntax is of form `io_read!(io, proj)` where `io` is an expression to a type that
1697/// implements [`Io`] and `proj` is a [projection specification](kernel::ptr::project!).
1698///
1699/// # Examples
1700///
1701/// ```
1702/// #[repr(C)]
1703/// struct MyStruct { field: u32, }
1704///
1705/// # fn test(mmio: kernel::io::Mmio<'_, [MyStruct]>) -> Result {
1706/// // let mmio: Mmio<'_, [MyStruct]>;
1707/// let field: u32 = kernel::io::io_read!(mmio, [try: 2].field);
1708/// # Ok::<(), Error>(()) }
1709/// ```
1710#[macro_export]
1711#[doc(hidden)]
1712macro_rules! io_read {
1713 ($io:expr, $($proj:tt)*) => {
1714 $crate::io::Io::read_val($crate::io_project!($io, $($proj)*))
1715 };
1716}
1717#[doc(inline)]
1718pub use crate::io_read;
1719
1720/// Writes to I/O memory.
1721///
1722/// The syntax is of form `io_write!(io, proj, val)` where `io` is an expression to a type that
1723/// implements [`Io`] and `proj` is a [projection specification](kernel::ptr::project!),
1724/// and `val` is the value to be written to the projected location.
1725///
1726/// # Examples
1727///
1728/// ```
1729/// #[repr(C)]
1730/// struct MyStruct { field: u32, }
1731///
1732/// # fn test(mmio: kernel::io::Mmio<'_, [MyStruct]>) -> Result {
1733/// // let mmio: Mmio<'_, [MyStruct]>;
1734/// kernel::io::io_write!(mmio, [try: 2].field, 10);
1735/// # Ok::<(), Error>(()) }
1736/// ```
1737#[macro_export]
1738#[doc(hidden)]
1739macro_rules! io_write {
1740 (@parse [$io:expr] [$($proj:tt)*] [, $val:expr]) => {
1741 $crate::io::Io::write_val($crate::io_project!($io, $($proj)*), $val)
1742 };
1743 (@parse [$io:expr] [$($proj:tt)*] [.$field:tt $($rest:tt)*]) => {
1744 $crate::io_write!(@parse [$io] [$($proj)* .$field] [$($rest)*])
1745 };
1746 (@parse [$io:expr] [$($proj:tt)*] [[$flavor:ident: $index:expr] $($rest:tt)*]) => {
1747 $crate::io_write!(@parse [$io] [$($proj)* [$flavor: $index]] [$($rest)*])
1748 };
1749 ($io:expr, $($rest:tt)*) => {
1750 $crate::io_write!(@parse [$io] [] [$($rest)*])
1751 };
1752}
1753#[doc(inline)]
1754pub use crate::io_write;