projects/linux·blob master · refs

1251 lines · 41.1 KiB

1// SPDX-License-Identifier: GPL-2.0
2
3//! Direct memory access (DMA).
4//!
5//! C header: [`include/linux/dma-mapping.h`](srctree/include/linux/dma-mapping.h)
6
7use crate::{
8 bindings,
9 debugfs,
10 device::{
11 self,
12 Bound,
13 Core, //
14 },
15 error::to_result,
16 fs::file,
17 io::{
18 IoBackend,
19 IoBase,
20 IoCapable,
21 IoCopyable,
22 SysMem,
23 SysMemBackend, //
24 },
25 prelude::*,
26 ptr::KnownSize,
27 sync::aref::ARef,
28 transmute::{
29 AsBytes,
30 FromBytes, //
31 },
32 uaccess::UserSliceWriter, //
33};
34use core::{
35 ops::{
36 Deref,
37 DerefMut, //
38 },
39 ptr::NonNull, //
40};
41
42/// DMA address type.
43///
44/// Represents a bus address used for Direct Memory Access (DMA) operations.
45///
46/// This is an alias of the kernel's `dma_addr_t`, which may be `u32` or `u64` depending on
47/// `CONFIG_ARCH_DMA_ADDR_T_64BIT`.
48///
49/// Note that this may be `u64` even on 32-bit architectures.
50pub type DmaAddress = bindings::dma_addr_t;
51
52/// Trait to be implemented by DMA capable bus devices.
53///
54/// The [`dma::Device`](Device) trait should be implemented by bus specific device representations,
55/// where the underlying bus is DMA capable, such as:
56#[cfg_attr(CONFIG_PCI, doc = "* [`pci::Device`](kernel::pci::Device)")]
57/// * [`platform::Device`](::kernel::platform::Device)
58pub trait Device<'a>: AsRef<device::Device<Core<'a>>> {
59 /// Set up the device's DMA streaming addressing capabilities.
60 ///
61 /// This method is usually called once from `probe()` as soon as the device capabilities are
62 /// known.
63 ///
64 /// # Safety
65 ///
66 /// This method must not be called concurrently with any DMA allocation or mapping primitives,
67 /// such as [`Coherent::zeroed`].
68 unsafe fn dma_set_mask(&self, mask: DmaMask) -> Result {
69 // SAFETY:
70 // - By the type invariant of `device::Device`, `self.as_ref().as_raw()` is valid.
71 // - The safety requirement of this function guarantees that there are no concurrent calls
72 // to DMA allocation and mapping primitives using this mask.
73 to_result(unsafe { bindings::dma_set_mask(self.as_ref().as_raw(), mask.value()) })
74 }
75
76 /// Set up the device's DMA coherent addressing capabilities.
77 ///
78 /// This method is usually called once from `probe()` as soon as the device capabilities are
79 /// known.
80 ///
81 /// # Safety
82 ///
83 /// This method must not be called concurrently with any DMA allocation or mapping primitives,
84 /// such as [`Coherent::zeroed`].
85 unsafe fn dma_set_coherent_mask(&self, mask: DmaMask) -> Result {
86 // SAFETY:
87 // - By the type invariant of `device::Device`, `self.as_ref().as_raw()` is valid.
88 // - The safety requirement of this function guarantees that there are no concurrent calls
89 // to DMA allocation and mapping primitives using this mask.
90 to_result(unsafe { bindings::dma_set_coherent_mask(self.as_ref().as_raw(), mask.value()) })
91 }
92
93 /// Set up the device's DMA addressing capabilities.
94 ///
95 /// This is a combination of [`Device::dma_set_mask`] and [`Device::dma_set_coherent_mask`].
96 ///
97 /// This method is usually called once from `probe()` as soon as the device capabilities are
98 /// known.
99 ///
100 /// # Safety
101 ///
102 /// This method must not be called concurrently with any DMA allocation or mapping primitives,
103 /// such as [`Coherent::zeroed`].
104 unsafe fn dma_set_mask_and_coherent(&self, mask: DmaMask) -> Result {
105 // SAFETY:
106 // - By the type invariant of `device::Device`, `self.as_ref().as_raw()` is valid.
107 // - The safety requirement of this function guarantees that there are no concurrent calls
108 // to DMA allocation and mapping primitives using this mask.
109 to_result(unsafe {
110 bindings::dma_set_mask_and_coherent(self.as_ref().as_raw(), mask.value())
111 })
112 }
113
114 /// Set the maximum size of a single DMA segment the device may request.
115 ///
116 /// This method is usually called once from `probe()` as soon as the device capabilities are
117 /// known.
118 ///
119 /// # Safety
120 ///
121 /// This method must not be called concurrently with any DMA allocation or mapping primitives,
122 /// such as [`Coherent::zeroed`].
123 unsafe fn dma_set_max_seg_size(&self, size: u32) {
124 // SAFETY:
125 // - By the type invariant of `device::Device`, `self.as_ref().as_raw()` is valid.
126 // - The safety requirement of this function guarantees that there are no concurrent calls
127 // to DMA allocation and mapping primitives using this parameter.
128 unsafe { bindings::dma_set_max_seg_size(self.as_ref().as_raw(), size) }
129 }
130}
131
132/// A DMA mask that holds a bitmask with the lowest `n` bits set.
133///
134/// Use [`DmaMask::new`] or [`DmaMask::try_new`] to construct a value. Values
135/// are guaranteed to never exceed the bit width of `u64`.
136///
137/// This is the Rust equivalent of the C macro `DMA_BIT_MASK()`.
138#[derive(Debug, Clone, Copy, PartialEq, Eq)]
139pub struct DmaMask(u64);
140
141impl DmaMask {
142 /// Constructs a `DmaMask` with the lowest `n` bits set to `1`.
143 ///
144 /// For `n <= 64`, sets exactly the lowest `n` bits.
145 /// For `n > 64`, results in a build error.
146 ///
147 /// # Examples
148 ///
149 /// ```
150 /// use kernel::dma::DmaMask;
151 ///
152 /// let mask0 = DmaMask::new::<0>();
153 /// assert_eq!(mask0.value(), 0);
154 ///
155 /// let mask1 = DmaMask::new::<1>();
156 /// assert_eq!(mask1.value(), 0b1);
157 ///
158 /// let mask64 = DmaMask::new::<64>();
159 /// assert_eq!(mask64.value(), u64::MAX);
160 ///
161 /// // Build failure.
162 /// // let mask_overflow = DmaMask::new::<100>();
163 /// ```
164 #[inline]
165 pub const fn new<const N: u32>() -> Self {
166 let Ok(mask) = Self::try_new(N) else {
167 build_error!("Invalid DMA Mask.");
168 };
169
170 mask
171 }
172
173 /// Constructs a `DmaMask` with the lowest `n` bits set to `1`.
174 ///
175 /// For `n <= 64`, sets exactly the lowest `n` bits.
176 /// For `n > 64`, returns [`EINVAL`].
177 ///
178 /// # Examples
179 ///
180 /// ```
181 /// use kernel::dma::DmaMask;
182 ///
183 /// let mask0 = DmaMask::try_new(0)?;
184 /// assert_eq!(mask0.value(), 0);
185 ///
186 /// let mask1 = DmaMask::try_new(1)?;
187 /// assert_eq!(mask1.value(), 0b1);
188 ///
189 /// let mask64 = DmaMask::try_new(64)?;
190 /// assert_eq!(mask64.value(), u64::MAX);
191 ///
192 /// let mask_overflow = DmaMask::try_new(100);
193 /// assert!(mask_overflow.is_err());
194 /// # Ok::<(), Error>(())
195 /// ```
196 #[inline]
197 pub const fn try_new(n: u32) -> Result<Self> {
198 Ok(Self(match n {
199 0 => 0,
200 1..=64 => u64::MAX >> (64 - n),
201 _ => return Err(EINVAL),
202 }))
203 }
204
205 /// Returns the underlying `u64` bitmask value.
206 #[inline]
207 pub const fn value(&self) -> u64 {
208 self.0
209 }
210}
211
212/// Possible attributes associated with a DMA mapping.
213///
214/// They can be combined with the operators `|`, `&`, and `!`.
215///
216/// Values can be used from the [`attrs`] module.
217///
218/// # Examples
219///
220/// ```
221/// # use kernel::device::{Bound, Device};
222/// use kernel::dma::{attrs::*, Coherent};
223///
224/// # fn test(dev: &Device<Bound>) -> Result {
225/// let attribs = DMA_ATTR_FORCE_CONTIGUOUS | DMA_ATTR_NO_WARN;
226/// let c: Coherent<[u64]> =
227/// Coherent::zeroed_slice_with_attrs(dev, 4, GFP_KERNEL, attribs)?;
228/// # Ok::<(), Error>(()) }
229/// ```
230#[derive(Clone, Copy, PartialEq)]
231#[repr(transparent)]
232pub struct Attrs(u32);
233
234impl Attrs {
235 /// Get the raw representation of this attribute.
236 pub(crate) fn as_raw(self) -> crate::ffi::c_ulong {
237 self.0 as crate::ffi::c_ulong
238 }
239
240 /// Check whether `flags` is contained in `self`.
241 pub fn contains(self, flags: Attrs) -> bool {
242 (self & flags) == flags
243 }
244}
245
246impl core::ops::BitOr for Attrs {
247 type Output = Self;
248 fn bitor(self, rhs: Self) -> Self::Output {
249 Self(self.0 | rhs.0)
250 }
251}
252
253impl core::ops::BitAnd for Attrs {
254 type Output = Self;
255 fn bitand(self, rhs: Self) -> Self::Output {
256 Self(self.0 & rhs.0)
257 }
258}
259
260impl core::ops::Not for Attrs {
261 type Output = Self;
262 fn not(self) -> Self::Output {
263 Self(!self.0)
264 }
265}
266
267/// DMA mapping attributes.
268pub mod attrs {
269 use super::Attrs;
270
271 /// Specifies that reads and writes to the mapping may be weakly ordered, that is that reads
272 /// and writes may pass each other.
273 pub const DMA_ATTR_WEAK_ORDERING: Attrs = Attrs(bindings::DMA_ATTR_WEAK_ORDERING);
274
275 /// Specifies that writes to the mapping may be buffered to improve performance.
276 pub const DMA_ATTR_WRITE_COMBINE: Attrs = Attrs(bindings::DMA_ATTR_WRITE_COMBINE);
277
278 /// Allows platform code to skip synchronization of the CPU cache for the given buffer assuming
279 /// that it has been already transferred to 'device' domain.
280 pub const DMA_ATTR_SKIP_CPU_SYNC: Attrs = Attrs(bindings::DMA_ATTR_SKIP_CPU_SYNC);
281
282 /// Forces contiguous allocation of the buffer in physical memory.
283 pub const DMA_ATTR_FORCE_CONTIGUOUS: Attrs = Attrs(bindings::DMA_ATTR_FORCE_CONTIGUOUS);
284
285 /// Hints DMA-mapping subsystem that it's probably not worth the time to try
286 /// to allocate memory to in a way that gives better TLB efficiency.
287 pub const DMA_ATTR_ALLOC_SINGLE_PAGES: Attrs = Attrs(bindings::DMA_ATTR_ALLOC_SINGLE_PAGES);
288
289 /// This tells the DMA-mapping subsystem to suppress allocation failure reports (similarly to
290 /// `__GFP_NOWARN`).
291 pub const DMA_ATTR_NO_WARN: Attrs = Attrs(bindings::DMA_ATTR_NO_WARN);
292
293 /// Indicates that the buffer is fully accessible at an elevated privilege level (and
294 /// ideally inaccessible or at least read-only at lesser-privileged levels).
295 pub const DMA_ATTR_PRIVILEGED: Attrs = Attrs(bindings::DMA_ATTR_PRIVILEGED);
296
297 /// Indicates that the buffer is MMIO memory.
298 pub const DMA_ATTR_MMIO: Attrs = Attrs(bindings::DMA_ATTR_MMIO);
299}
300
301/// DMA data direction.
302///
303/// Corresponds to the C [`enum dma_data_direction`].
304///
305/// [`enum dma_data_direction`]: srctree/include/linux/dma-direction.h
306#[derive(Copy, Clone, PartialEq, Eq, Debug)]
307#[repr(u32)]
308pub enum DataDirection {
309 /// The DMA mapping is for bidirectional data transfer.
310 ///
311 /// This is used when the buffer can be both read from and written to by the device.
312 /// The cache for the corresponding memory region is both flushed and invalidated.
313 Bidirectional = Self::const_cast(bindings::dma_data_direction_DMA_BIDIRECTIONAL),
314
315 /// The DMA mapping is for data transfer from memory to the device (write).
316 ///
317 /// The CPU has prepared data in the buffer, and the device will read it.
318 /// The cache for the corresponding memory region is flushed before device access.
319 ToDevice = Self::const_cast(bindings::dma_data_direction_DMA_TO_DEVICE),
320
321 /// The DMA mapping is for data transfer from the device to memory (read).
322 ///
323 /// The device will write data into the buffer for the CPU to read.
324 /// The cache for the corresponding memory region is invalidated before CPU access.
325 FromDevice = Self::const_cast(bindings::dma_data_direction_DMA_FROM_DEVICE),
326
327 /// The DMA mapping is not for data transfer.
328 ///
329 /// This is primarily for debugging purposes. With this direction, the DMA mapping API
330 /// will not perform any cache coherency operations.
331 None = Self::const_cast(bindings::dma_data_direction_DMA_NONE),
332}
333
334impl DataDirection {
335 /// Casts the bindgen-generated enum type to a `u32` at compile time.
336 ///
337 /// This function will cause a compile-time error if the underlying value of the
338 /// C enum is out of bounds for `u32`.
339 const fn const_cast(val: bindings::dma_data_direction) -> u32 {
340 // CAST: The C standard allows compilers to choose different integer types for enums.
341 // To safely check the value, we cast it to a wide signed integer type (`i128`)
342 // which can hold any standard C integer enum type without truncation.
343 let wide_val = val as i128;
344
345 // Check if the value is outside the valid range for the target type `u32`.
346 // CAST: `u32::MAX` is cast to `i128` to match the type of `wide_val` for the comparison.
347 if wide_val < 0 || wide_val > u32::MAX as i128 {
348 // Trigger a compile-time error in a const context.
349 build_error!("C enum value is out of bounds for the target type `u32`.");
350 }
351
352 // CAST: This cast is valid because the check above guarantees that `wide_val`
353 // is within the representable range of `u32`.
354 wide_val as u32
355 }
356}
357
358impl From<DataDirection> for bindings::dma_data_direction {
359 /// Returns the raw representation of [`enum dma_data_direction`].
360 fn from(direction: DataDirection) -> Self {
361 // CAST: `direction as u32` gets the underlying representation of our `#[repr(u32)]` enum.
362 // The subsequent cast to `Self` (the bindgen type) assumes the C enum is compatible
363 // with the enum variants of `DataDirection`, which is a valid assumption given our
364 // compile-time checks.
365 direction as u32 as Self
366 }
367}
368
369/// CPU-owned DMA allocation that can be converted into a device-shared [`Coherent`] object.
370///
371/// Unlike [`Coherent`], a [`CoherentBox`] is guaranteed to be fully owned by the CPU -- its DMA
372/// address is not exposed and it cannot be accessed by a device. This means it can safely be used
373/// like a normal boxed allocation (e.g. direct reads, writes, and mutable slices are all safe).
374///
375/// A typical use is to allocate a [`CoherentBox`], populate it with normal CPU access, and then
376/// convert it into a [`Coherent`] object to share it with the device.
377///
378/// # Examples
379///
380/// `CoherentBox<T>`:
381///
382/// ```
383/// # use kernel::device::{
384/// # Bound,
385/// # Device,
386/// # };
387/// use kernel::dma::{attrs::*,
388/// Coherent,
389/// CoherentBox,
390/// };
391///
392/// # fn test(dev: &Device<Bound>) -> Result {
393/// let mut dmem: CoherentBox<u64> = CoherentBox::zeroed(dev, GFP_KERNEL)?;
394/// *dmem = 42;
395/// let dmem: Coherent<u64> = dmem.into();
396/// # Ok::<(), Error>(()) }
397/// ```
398///
399/// `CoherentBox<[T]>`:
400///
401///
402/// ```
403/// # use kernel::device::{
404/// # Bound,
405/// # Device,
406/// # };
407/// use kernel::dma::{attrs::*,
408/// Coherent,
409/// CoherentBox,
410/// };
411///
412/// # fn test(dev: &Device<Bound>) -> Result {
413/// let mut dmem: CoherentBox<[u64]> = CoherentBox::zeroed_slice(dev, 4, GFP_KERNEL)?;
414/// dmem.fill(42);
415/// let dmem: Coherent<[u64]> = dmem.into();
416/// # Ok::<(), Error>(()) }
417/// ```
418pub struct CoherentBox<T: KnownSize + ?Sized>(Coherent<T>);
419
420impl<T: AsBytes + FromBytes> CoherentBox<[T]> {
421 /// [`CoherentBox`] variant of [`Coherent::zeroed_slice_with_attrs`].
422 #[inline]
423 pub fn zeroed_slice_with_attrs(
424 dev: &device::Device<Bound>,
425 count: usize,
426 gfp_flags: kernel::alloc::Flags,
427 dma_attrs: Attrs,
428 ) -> Result<Self> {
429 Coherent::zeroed_slice_with_attrs(dev, count, gfp_flags, dma_attrs).map(Self)
430 }
431
432 /// Same as [CoherentBox::zeroed_slice_with_attrs], but with `dma::Attrs(0)`.
433 #[inline]
434 pub fn zeroed_slice(
435 dev: &device::Device<Bound>,
436 count: usize,
437 gfp_flags: kernel::alloc::Flags,
438 ) -> Result<Self> {
439 Self::zeroed_slice_with_attrs(dev, count, gfp_flags, Attrs(0))
440 }
441
442 /// Initializes the element at `i` using the given initializer.
443 ///
444 /// Returns `EINVAL` if `i` is out of bounds.
445 pub fn init_at<E>(&mut self, i: usize, init: impl Init<T, E>) -> Result
446 where
447 Error: From<E>,
448 {
449 if i >= self.0.len() {
450 return Err(EINVAL);
451 }
452
453 let ptr = &raw mut self[i];
454
455 // SAFETY:
456 // - `ptr` is valid, properly aligned, and within this allocation.
457 // - `T: AsBytes + FromBytes` guarantees all bit patterns are valid, so partial writes on
458 // error cannot leave the element in an invalid state.
459 // - The DMA address has not been exposed yet, so there is no concurrent device access.
460 unsafe { pin_init::raw_try_init(ptr, init)? };
461
462 Ok(())
463 }
464
465 /// Allocates a region of coherent memory of the same size as `data` and initializes it with a
466 /// copy of its contents.
467 ///
468 /// This is the [`CoherentBox`] variant of [`Coherent::from_slice_with_attrs`].
469 ///
470 /// # Examples
471 ///
472 /// ```
473 /// use core::ops::Deref;
474 ///
475 /// # use kernel::device::{Bound, Device};
476 /// use kernel::dma::{
477 /// attrs::*,
478 /// CoherentBox
479 /// };
480 ///
481 /// # fn test(dev: &Device<Bound>) -> Result {
482 /// let data = [0u8, 1u8, 2u8, 3u8];
483 /// let c: CoherentBox<[u8]> =
484 /// CoherentBox::from_slice_with_attrs(dev, &data, GFP_KERNEL, DMA_ATTR_NO_WARN)?;
485 ///
486 /// assert_eq!(c.deref(), &data);
487 /// # Ok::<(), Error>(()) }
488 /// ```
489 pub fn from_slice_with_attrs(
490 dev: &device::Device<Bound>,
491 data: &[T],
492 gfp_flags: kernel::alloc::Flags,
493 dma_attrs: Attrs,
494 ) -> Result<Self>
495 where
496 T: Copy,
497 {
498 let mut slice = Self(Coherent::<T>::alloc_slice_with_attrs(
499 dev,
500 data.len(),
501 gfp_flags,
502 dma_attrs,
503 )?);
504
505 // PANIC: `slice` was created with length `data.len()`.
506 slice.copy_from_slice(data);
507
508 Ok(slice)
509 }
510
511 /// Performs the same functionality as [`CoherentBox::from_slice_with_attrs`], except the
512 /// `dma_attrs` is 0 by default.
513 #[inline]
514 pub fn from_slice(
515 dev: &device::Device<Bound>,
516 data: &[T],
517 gfp_flags: kernel::alloc::Flags,
518 ) -> Result<Self>
519 where
520 T: Copy,
521 {
522 Self::from_slice_with_attrs(dev, data, gfp_flags, Attrs(0))
523 }
524}
525
526impl<T: AsBytes + FromBytes> CoherentBox<T> {
527 /// Same as [`CoherentBox::zeroed_slice_with_attrs`], but for a single element.
528 #[inline]
529 pub fn zeroed_with_attrs(
530 dev: &device::Device<Bound>,
531 gfp_flags: kernel::alloc::Flags,
532 dma_attrs: Attrs,
533 ) -> Result<Self> {
534 Coherent::zeroed_with_attrs(dev, gfp_flags, dma_attrs).map(Self)
535 }
536
537 /// Same as [`CoherentBox::zeroed_slice`], but for a single element.
538 #[inline]
539 pub fn zeroed(dev: &device::Device<Bound>, gfp_flags: kernel::alloc::Flags) -> Result<Self> {
540 Self::zeroed_with_attrs(dev, gfp_flags, Attrs(0))
541 }
542}
543
544impl<T: KnownSize + ?Sized> Deref for CoherentBox<T> {
545 type Target = T;
546
547 #[inline]
548 fn deref(&self) -> &Self::Target {
549 // SAFETY:
550 // - We have not exposed the DMA address yet, so there can't be any concurrent access by a
551 // device.
552 // - We have exclusive access to `self.0`.
553 unsafe { self.0.as_ref() }
554 }
555}
556
557impl<T: AsBytes + FromBytes + KnownSize + ?Sized> DerefMut for CoherentBox<T> {
558 #[inline]
559 fn deref_mut(&mut self) -> &mut Self::Target {
560 // SAFETY:
561 // - We have not exposed the DMA address yet, so there can't be any concurrent access by a
562 // device.
563 // - We have exclusive access to `self.0`.
564 unsafe { self.0.as_mut() }
565 }
566}
567
568impl<T: AsBytes + FromBytes + KnownSize + ?Sized> From<CoherentBox<T>> for Coherent<T> {
569 #[inline]
570 fn from(value: CoherentBox<T>) -> Self {
571 value.0
572 }
573}
574
575/// An abstraction of the `dma_alloc_coherent` API.
576///
577/// This is an abstraction around the `dma_alloc_coherent` API which is used to allocate and map
578/// large coherent DMA regions.
579///
580/// A [`Coherent`] instance contains a pointer to the allocated region (in the
581/// processor's virtual address space) and the device address which can be given to the device
582/// as the DMA address base of the region. The region is released once [`Coherent`]
583/// is dropped.
584///
585/// # Invariants
586///
587/// - For the lifetime of an instance of [`Coherent`], the `cpu_addr` is a valid pointer
588/// to an allocated region of coherent memory and `dma_addr` is the DMA address base of the
589/// region.
590/// - The size in bytes of the allocation is equal to size information via pointer.
591// TODO
592//
593// DMA allocations potentially carry device resources (e.g.IOMMU mappings), hence for soundness
594// reasons DMA allocation would need to be embedded in a `Devres` container, in order to ensure
595// that device resources can never survive device unbind.
596//
597// However, it is neither desirable nor necessary to protect the allocated memory of the DMA
598// allocation from surviving device unbind; it would require RCU read side critical sections to
599// access the memory, which may require subsequent unnecessary copies.
600//
601// Hence, find a way to revoke the device resources of a `Coherent`, but not the
602// entire `Coherent` including the allocated memory itself.
603pub struct Coherent<T: KnownSize + ?Sized> {
604 dev: ARef<device::Device>,
605 dma_addr: DmaAddress,
606 cpu_addr: NonNull<T>,
607 dma_attrs: Attrs,
608}
609
610impl<T: KnownSize + ?Sized> Coherent<T> {
611 /// Returns the size in bytes of this allocation.
612 #[inline]
613 pub fn size(&self) -> usize {
614 T::size(self.cpu_addr.as_ptr())
615 }
616
617 /// Returns the raw pointer to the allocated region in the CPU's virtual address space.
618 #[inline]
619 pub fn as_ptr(&self) -> *const T {
620 self.cpu_addr.as_ptr()
621 }
622
623 /// Returns the raw pointer to the allocated region in the CPU's virtual address space as
624 /// a mutable pointer.
625 #[inline]
626 pub fn as_mut_ptr(&self) -> *mut T {
627 self.cpu_addr.as_ptr()
628 }
629
630 /// Returns a DMA address which may be given to the device as the base of the region.
631 #[inline]
632 pub fn dma_address(&self) -> DmaAddress {
633 self.dma_addr
634 }
635
636 /// Returns a reference to the data in the region.
637 ///
638 /// # Safety
639 ///
640 /// * Callers must ensure that the device does not read/write to/from memory while the returned
641 /// slice is live.
642 /// * Callers must ensure that this call does not race with a write to the same region while
643 /// the returned slice is live.
644 #[inline]
645 pub unsafe fn as_ref(&self) -> &T {
646 // SAFETY: per safety requirement.
647 unsafe { &*self.as_ptr() }
648 }
649
650 /// Returns a mutable reference to the data in the region.
651 ///
652 /// # Safety
653 ///
654 /// * Callers must ensure that the device does not read/write to/from memory while the returned
655 /// slice is live.
656 /// * Callers must ensure that this call does not race with a read or write to the same region
657 /// while the returned slice is live.
658 #[expect(clippy::mut_from_ref, reason = "unsafe to use API")]
659 #[inline]
660 pub unsafe fn as_mut(&self) -> &mut T {
661 // SAFETY: per safety requirement.
662 unsafe { &mut *self.as_mut_ptr() }
663 }
664}
665
666impl<T: AsBytes + FromBytes> Coherent<T> {
667 /// Allocates a region of `T` of coherent memory.
668 fn alloc_with_attrs(
669 dev: &device::Device<Bound>,
670 gfp_flags: kernel::alloc::Flags,
671 dma_attrs: Attrs,
672 ) -> Result<Self> {
673 const {
674 assert!(
675 core::mem::size_of::<T>() > 0,
676 "It doesn't make sense for the allocated type to be a ZST"
677 );
678 }
679
680 let mut dma_addr = 0;
681 // SAFETY: Device pointer is guaranteed as valid by the type invariant on `Device`.
682 let addr = unsafe {
683 bindings::dma_alloc_attrs(
684 dev.as_raw(),
685 core::mem::size_of::<T>(),
686 &mut dma_addr,
687 gfp_flags.as_raw(),
688 dma_attrs.as_raw(),
689 )
690 };
691 let cpu_addr = NonNull::new(addr.cast()).ok_or(ENOMEM)?;
692 // INVARIANT:
693 // - We just successfully allocated a coherent region which is adequately sized for `T`,
694 // hence the cpu address is valid.
695 // - We also hold a refcounted reference to the device.
696 Ok(Self {
697 dev: dev.into(),
698 dma_addr,
699 cpu_addr,
700 dma_attrs,
701 })
702 }
703
704 /// Allocates a region of type `T` of coherent memory.
705 ///
706 /// # Examples
707 ///
708 /// ```
709 /// # use kernel::device::{
710 /// # Bound,
711 /// # Device,
712 /// # };
713 /// use kernel::dma::{
714 /// attrs::*,
715 /// Coherent,
716 /// };
717 ///
718 /// # fn test(dev: &Device<Bound>) -> Result {
719 /// let c: Coherent<[u64; 4]> =
720 /// Coherent::zeroed_with_attrs(dev, GFP_KERNEL, DMA_ATTR_NO_WARN)?;
721 /// # Ok::<(), Error>(()) }
722 /// ```
723 #[inline]
724 pub fn zeroed_with_attrs(
725 dev: &device::Device<Bound>,
726 gfp_flags: kernel::alloc::Flags,
727 dma_attrs: Attrs,
728 ) -> Result<Self> {
729 Self::alloc_with_attrs(dev, gfp_flags | __GFP_ZERO, dma_attrs)
730 }
731
732 /// Performs the same functionality as [`Coherent::zeroed_with_attrs`], except the
733 /// `dma_attrs` is 0 by default.
734 #[inline]
735 pub fn zeroed(dev: &device::Device<Bound>, gfp_flags: kernel::alloc::Flags) -> Result<Self> {
736 Self::zeroed_with_attrs(dev, gfp_flags, Attrs(0))
737 }
738
739 /// Same as [`Coherent::zeroed_with_attrs`], but instead of a zero-initialization the memory is
740 /// initialized with `init`.
741 pub fn init_with_attrs<E>(
742 dev: &device::Device<Bound>,
743 gfp_flags: kernel::alloc::Flags,
744 dma_attrs: Attrs,
745 init: impl Init<T, E>,
746 ) -> Result<Self>
747 where
748 Error: From<E>,
749 {
750 let dmem = Self::alloc_with_attrs(dev, gfp_flags, dma_attrs)?;
751 let ptr = dmem.as_mut_ptr();
752
753 // SAFETY:
754 // - `ptr` is valid, properly aligned, and points to exclusively owned memory.
755 // - If `raw_try_init` fails, `self` is dropped, which safely frees the underlying
756 // `Coherent`'s DMA memory. `T: AsBytes + FromBytes` ensures there are no complex `Drop`
757 // requirements we are bypassing.
758 unsafe { pin_init::raw_try_init(ptr, init)? };
759
760 Ok(dmem)
761 }
762
763 /// Same as [`Coherent::zeroed`], but instead of a zero-initialization the memory is initialized
764 /// with `init`.
765 #[inline]
766 pub fn init<E>(
767 dev: &device::Device<Bound>,
768 gfp_flags: kernel::alloc::Flags,
769 init: impl Init<T, E>,
770 ) -> Result<Self>
771 where
772 Error: From<E>,
773 {
774 Self::init_with_attrs(dev, gfp_flags, Attrs(0), init)
775 }
776
777 /// Allocates a region of `[T; len]` of coherent memory.
778 fn alloc_slice_with_attrs(
779 dev: &device::Device<Bound>,
780 len: usize,
781 gfp_flags: kernel::alloc::Flags,
782 dma_attrs: Attrs,
783 ) -> Result<Coherent<[T]>> {
784 const {
785 assert!(
786 core::mem::size_of::<T>() > 0,
787 "It doesn't make sense for the allocated type to be a ZST"
788 );
789 }
790
791 // `dma_alloc_attrs` cannot handle zero-length allocation, bail early.
792 if len == 0 {
793 Err(EINVAL)?;
794 }
795
796 let size = core::mem::size_of::<T>().checked_mul(len).ok_or(ENOMEM)?;
797 let mut dma_addr = 0;
798 // SAFETY: Device pointer is guaranteed as valid by the type invariant on `Device`.
799 let addr = unsafe {
800 bindings::dma_alloc_attrs(
801 dev.as_raw(),
802 size,
803 &mut dma_addr,
804 gfp_flags.as_raw(),
805 dma_attrs.as_raw(),
806 )
807 };
808 let cpu_addr = NonNull::slice_from_raw_parts(NonNull::new(addr.cast()).ok_or(ENOMEM)?, len);
809 // INVARIANT:
810 // - We just successfully allocated a coherent region which is adequately sized for
811 // `[T; len]`, hence the cpu address is valid.
812 // - We also hold a refcounted reference to the device.
813 Ok(Coherent {
814 dev: dev.into(),
815 dma_addr,
816 cpu_addr,
817 dma_attrs,
818 })
819 }
820
821 /// Allocates a zeroed region of type `T` of coherent memory.
822 ///
823 /// Unlike `Coherent::<[T; N]>::zeroed_with_attrs`, `Coherent::<T>::zeroed_slices` support
824 /// a runtime length.
825 ///
826 /// # Examples
827 ///
828 /// ```
829 /// # use kernel::device::{
830 /// # Bound,
831 /// # Device,
832 /// # };
833 /// use kernel::dma::{
834 /// attrs::*,
835 /// Coherent,
836 /// };
837 ///
838 /// # fn test(dev: &Device<Bound>) -> Result {
839 /// let c: Coherent<[u64]> =
840 /// Coherent::zeroed_slice_with_attrs(dev, 4, GFP_KERNEL, DMA_ATTR_NO_WARN)?;
841 /// # Ok::<(), Error>(()) }
842 /// ```
843 #[inline]
844 pub fn zeroed_slice_with_attrs(
845 dev: &device::Device<Bound>,
846 len: usize,
847 gfp_flags: kernel::alloc::Flags,
848 dma_attrs: Attrs,
849 ) -> Result<Coherent<[T]>> {
850 Coherent::alloc_slice_with_attrs(dev, len, gfp_flags | __GFP_ZERO, dma_attrs)
851 }
852
853 /// Performs the same functionality as [`Coherent::zeroed_slice_with_attrs`], except the
854 /// `dma_attrs` is 0 by default.
855 #[inline]
856 pub fn zeroed_slice(
857 dev: &device::Device<Bound>,
858 len: usize,
859 gfp_flags: kernel::alloc::Flags,
860 ) -> Result<Coherent<[T]>> {
861 Self::zeroed_slice_with_attrs(dev, len, gfp_flags, Attrs(0))
862 }
863
864 /// Allocates a region of coherent memory of the same size as `data` and initializes it with a
865 /// copy of its contents.
866 ///
867 /// # Examples
868 ///
869 /// ```
870 /// # use kernel::device::{Bound, Device};
871 /// use kernel::dma::{
872 /// attrs::*,
873 /// Coherent
874 /// };
875 ///
876 /// # fn test(dev: &Device<Bound>) -> Result {
877 /// let data = [0u8, 1u8, 2u8, 3u8];
878 /// // `c` has the same content as `data`.
879 /// let c: Coherent<[u8]> =
880 /// Coherent::from_slice_with_attrs(dev, &data, GFP_KERNEL, DMA_ATTR_NO_WARN)?;
881 ///
882 /// # Ok::<(), Error>(()) }
883 /// ```
884 #[inline]
885 pub fn from_slice_with_attrs(
886 dev: &device::Device<Bound>,
887 data: &[T],
888 gfp_flags: kernel::alloc::Flags,
889 dma_attrs: Attrs,
890 ) -> Result<Coherent<[T]>>
891 where
892 T: Copy,
893 {
894 CoherentBox::from_slice_with_attrs(dev, data, gfp_flags, dma_attrs).map(Into::into)
895 }
896
897 /// Performs the same functionality as [`Coherent::from_slice_with_attrs`], except the
898 /// `dma_attrs` is 0 by default.
899 #[inline]
900 pub fn from_slice(
901 dev: &device::Device<Bound>,
902 data: &[T],
903 gfp_flags: kernel::alloc::Flags,
904 ) -> Result<Coherent<[T]>>
905 where
906 T: Copy,
907 {
908 Self::from_slice_with_attrs(dev, data, gfp_flags, Attrs(0))
909 }
910}
911
912impl<T> Coherent<[T]> {
913 /// Returns the number of elements `T` in this allocation.
914 ///
915 /// Note that this is not the size of the allocation in bytes, which is provided by
916 /// [`Self::size`].
917 #[inline]
918 #[expect(clippy::len_without_is_empty, reason = "Coherent slice is never empty")]
919 pub fn len(&self) -> usize {
920 self.cpu_addr.len()
921 }
922}
923
924/// Note that the device configured to do DMA must be halted before this object is dropped.
925impl<T: KnownSize + ?Sized> Drop for Coherent<T> {
926 fn drop(&mut self) {
927 let size = T::size(self.cpu_addr.as_ptr());
928 // SAFETY: Device pointer is guaranteed as valid by the type invariant on `Device`.
929 // The cpu address, and the dma address are valid due to the type invariants on
930 // `Coherent`.
931 unsafe {
932 bindings::dma_free_attrs(
933 self.dev.as_raw(),
934 size,
935 self.cpu_addr.as_ptr().cast(),
936 self.dma_addr,
937 self.dma_attrs.as_raw(),
938 )
939 }
940 }
941}
942
943// SAFETY: It is safe to send a `Coherent` to another thread if `T`
944// can be sent to another thread.
945unsafe impl<T: KnownSize + Send + ?Sized> Send for Coherent<T> {}
946
947// SAFETY: Sharing `&Coherent` across threads is safe if `T` is `Sync`, because all
948// methods that access the buffer contents (`field_read`, `field_write`, `as_slice`,
949// `as_slice_mut`) are `unsafe`, and callers are responsible for ensuring no data races occur.
950// The safe methods only return metadata or raw pointers whose use requires `unsafe`.
951unsafe impl<T: KnownSize + ?Sized + AsBytes + FromBytes + Sync> Sync for Coherent<T> {}
952
953impl<T: KnownSize + AsBytes + ?Sized> debugfs::BinaryWriter for Coherent<T> {
954 fn write_to_slice(
955 &self,
956 writer: &mut UserSliceWriter,
957 offset: &mut file::Offset,
958 ) -> Result<usize> {
959 if offset.is_negative() {
960 return Err(EINVAL);
961 }
962
963 // If the offset is too large for a usize (e.g. on 32-bit platforms),
964 // then consider that as past EOF and just return 0 bytes.
965 let Ok(offset_val) = usize::try_from(*offset) else {
966 return Ok(0);
967 };
968
969 if offset_val >= self.size() {
970 return Ok(0);
971 }
972
973 let count = (self.size() - offset_val).min(writer.len());
974
975 writer.write_dma(self, offset_val, count)?;
976
977 *offset += count as i64;
978 Ok(count)
979 }
980}
981
982/// An opaque DMA allocation without a kernel virtual mapping.
983///
984/// Unlike [`Coherent`], a `CoherentHandle` does not provide CPU access to the allocated memory.
985/// The allocation is always performed with `DMA_ATTR_NO_KERNEL_MAPPING`, meaning no kernel
986/// virtual mapping is created for the buffer. The value returned by the C API as the CPU
987/// address is an opaque handle used only to free the allocation.
988///
989/// This is useful for buffers that are only ever accessed by hardware.
990///
991/// # Invariants
992///
993/// - `cpu_handle` holds the opaque handle returned by `dma_alloc_attrs` with
994/// `DMA_ATTR_NO_KERNEL_MAPPING` set, and is only valid for passing back to `dma_free_attrs`.
995/// - `dma_addr` is the corresponding bus address for device DMA.
996/// - `size` is the allocation size in bytes as passed to `dma_alloc_attrs`.
997/// - `dma_attrs` contains the attributes used for the allocation, always including
998/// `DMA_ATTR_NO_KERNEL_MAPPING`.
999pub struct CoherentHandle {
1000 dev: ARef<device::Device>,
1001 dma_addr: DmaAddress,
1002 cpu_handle: NonNull<c_void>,
1003 size: usize,
1004 dma_attrs: Attrs,
1005}
1006
1007impl CoherentHandle {
1008 /// Allocates `size` bytes of coherent DMA memory without creating a kernel virtual mapping.
1009 ///
1010 /// Additional DMA attributes may be passed via `dma_attrs`; `DMA_ATTR_NO_KERNEL_MAPPING` is
1011 /// always set implicitly.
1012 ///
1013 /// Returns `EINVAL` if `size` is zero, `ENOMEM` if the allocation fails.
1014 pub fn alloc_with_attrs(
1015 dev: &device::Device<Bound>,
1016 size: usize,
1017 gfp_flags: kernel::alloc::Flags,
1018 dma_attrs: Attrs,
1019 ) -> Result<Self> {
1020 if size == 0 {
1021 return Err(EINVAL);
1022 }
1023
1024 let dma_attrs = dma_attrs | Attrs(bindings::DMA_ATTR_NO_KERNEL_MAPPING);
1025 let mut dma_addr = 0;
1026 // SAFETY: `dev.as_raw()` is valid by the type invariant on `device::Device`.
1027 let cpu_handle = unsafe {
1028 bindings::dma_alloc_attrs(
1029 dev.as_raw(),
1030 size,
1031 &mut dma_addr,
1032 gfp_flags.as_raw(),
1033 dma_attrs.as_raw(),
1034 )
1035 };
1036
1037 let cpu_handle = NonNull::new(cpu_handle).ok_or(ENOMEM)?;
1038
1039 // INVARIANT: `cpu_handle` is the opaque handle from a successful `dma_alloc_attrs` call
1040 // with `DMA_ATTR_NO_KERNEL_MAPPING`, `dma_addr` is the corresponding DMA address,
1041 // and we hold a refcounted reference to the device.
1042 Ok(Self {
1043 dev: dev.into(),
1044 dma_addr,
1045 cpu_handle,
1046 size,
1047 dma_attrs,
1048 })
1049 }
1050
1051 /// Allocates `size` bytes of coherent DMA memory without creating a kernel virtual mapping.
1052 #[inline]
1053 pub fn alloc(
1054 dev: &device::Device<Bound>,
1055 size: usize,
1056 gfp_flags: kernel::alloc::Flags,
1057 ) -> Result<Self> {
1058 Self::alloc_with_attrs(dev, size, gfp_flags, Attrs(0))
1059 }
1060
1061 /// Returns the DMA address for this allocation.
1062 ///
1063 /// This address can be programmed into device hardware for DMA access.
1064 #[inline]
1065 pub fn dma_address(&self) -> DmaAddress {
1066 self.dma_addr
1067 }
1068
1069 /// Returns the size in bytes of this allocation.
1070 #[inline]
1071 pub fn size(&self) -> usize {
1072 self.size
1073 }
1074}
1075
1076impl Drop for CoherentHandle {
1077 fn drop(&mut self) {
1078 // SAFETY: All values are valid by the type invariants on `CoherentHandle`.
1079 // `cpu_handle` is the opaque handle from `dma_alloc_attrs` and is passed back unchanged.
1080 unsafe {
1081 bindings::dma_free_attrs(
1082 self.dev.as_raw(),
1083 self.size,
1084 self.cpu_handle.as_ptr(),
1085 self.dma_addr,
1086 self.dma_attrs.as_raw(),
1087 )
1088 }
1089 }
1090}
1091
1092// SAFETY: `CoherentHandle` only holds a device reference, a DMA address, an opaque CPU handle,
1093// and a size. None of these are tied to a specific thread.
1094unsafe impl Send for CoherentHandle {}
1095
1096// SAFETY: `CoherentHandle` provides no CPU access to the underlying allocation. The only
1097// operations on `&CoherentHandle` are reading the DMA address and size, both of which are
1098// plain `Copy` values.
1099unsafe impl Sync for CoherentHandle {}
1100
1101/// View type for `Coherent`.
1102///
1103/// This is same as [`SysMem`] but with additional information that allows handing out a DMA
1104/// address.
1105pub struct CoherentView<'a, T: ?Sized> {
1106 cpu_addr: SysMem<'a, T>,
1107 dma_addr: DmaAddress,
1108}
1109
1110impl<T: ?Sized> Copy for CoherentView<'_, T> {}
1111impl<T: ?Sized> Clone for CoherentView<'_, T> {
1112 #[inline]
1113 fn clone(&self) -> Self {
1114 *self
1115 }
1116}
1117
1118impl<'a, T: ?Sized> CoherentView<'a, T> {
1119 /// Erase the DMA address information and obtain a [`SysMem`] view of the same memory region.
1120 #[inline]
1121 pub fn as_sys_mem(self) -> SysMem<'a, T> {
1122 self.cpu_addr
1123 }
1124
1125 /// Returns the DMA address which may be given to the device as base of the region.
1126 #[inline]
1127 pub fn dma_address(self) -> DmaAddress {
1128 self.dma_addr
1129 }
1130
1131 /// Returns a reference to the data in the region.
1132 ///
1133 /// # Safety
1134 ///
1135 /// * Callers must ensure that the device does not read/write to/from memory while the returned
1136 /// reference is live.
1137 /// * Callers must ensure that this call does not race with a write (including call to `as_mut`)
1138 /// to the same region while the returned reference is live.
1139 #[inline]
1140 pub unsafe fn as_ref(self) -> &'a T {
1141 // SAFETY: pointer is aligned and valid per type invariant. Aliasing rule is satisfied per
1142 // safety requirement.
1143 unsafe { &*self.cpu_addr.as_ptr() }
1144 }
1145
1146 /// Returns a mutable reference to the data in the region.
1147 ///
1148 /// # Safety
1149 ///
1150 /// * Callers must ensure that the device does not read/write to/from memory while the returned
1151 /// reference is live.
1152 /// * Callers must ensure that this call does not race with a read (including call to `as_ref`)
1153 /// or write (including call to `as_mut`) to the same region while the returned reference is
1154 /// live.
1155 #[inline]
1156 pub unsafe fn as_mut(self) -> &'a mut T {
1157 // SAFETY: pointer is aligned and valid per type invariant. Aliasing rule is satisfied per
1158 // safety requirement.
1159 unsafe { &mut *self.cpu_addr.as_ptr() }
1160 }
1161}
1162
1163/// `IoBackend` implementation for `Coherent`.
1164pub struct CoherentIoBackend;
1165
1166impl IoBackend for CoherentIoBackend {
1167 type View<'a, T: ?Sized + KnownSize> = CoherentView<'a, T>;
1168
1169 #[inline]
1170 fn as_ptr<'a, T: ?Sized + KnownSize>(view: Self::View<'a, T>) -> *mut T {
1171 SysMemBackend::as_ptr(view.cpu_addr)
1172 }
1173
1174 #[inline]
1175 unsafe fn project_view<'a, T: ?Sized + KnownSize, U: ?Sized + KnownSize>(
1176 view: Self::View<'a, T>,
1177 ptr: *mut U,
1178 ) -> Self::View<'a, U> {
1179 let offset = ptr.addr() - view.cpu_addr.as_ptr().addr();
1180 // CAST: The offset DMA address can never overflow.
1181 let dma_addr = view.dma_addr + offset as DmaAddress;
1182 CoherentView {
1183 dma_addr,
1184 // SAFETY: Per safety requirement.
1185 cpu_addr: unsafe { SysMemBackend::project_view(view.cpu_addr, ptr) },
1186 }
1187 }
1188}
1189
1190impl<T> IoCapable<T> for CoherentIoBackend
1191where
1192 SysMemBackend: IoCapable<T>,
1193{
1194 #[inline]
1195 fn io_read<'a>(view: Self::View<'a, T>) -> T {
1196 SysMemBackend::io_read(view.cpu_addr)
1197 }
1198
1199 #[inline]
1200 fn io_write<'a>(view: Self::View<'a, T>, value: T) {
1201 SysMemBackend::io_write(view.cpu_addr, value)
1202 }
1203}
1204
1205impl IoCopyable for CoherentIoBackend {
1206 #[inline]
1207 unsafe fn copy_from_io(view: Self::View<'_, [u8]>, buffer: *mut u8) {
1208 // SAFETY: Per safety requirement.
1209 unsafe { SysMemBackend::copy_from_io(view.cpu_addr, buffer) }
1210 }
1211
1212 #[inline]
1213 unsafe fn copy_to_io(view: Self::View<'_, [u8]>, buffer: *const u8) {
1214 // SAFETY: Per safety requirement.
1215 unsafe { SysMemBackend::copy_to_io(view.cpu_addr, buffer) }
1216 }
1217
1218 #[inline]
1219 fn copy_read<T: zerocopy::FromBytes>(view: Self::View<'_, T>) -> T {
1220 SysMemBackend::copy_read(view.cpu_addr)
1221 }
1222
1223 #[inline]
1224 fn copy_write<T: zerocopy::IntoBytes>(view: Self::View<'_, T>, value: T) {
1225 SysMemBackend::copy_write(view.cpu_addr, value)
1226 }
1227}
1228
1229impl<'a, T: ?Sized + KnownSize> IoBase<'a> for CoherentView<'a, T> {
1230 type Backend = CoherentIoBackend;
1231 type Target = T;
1232
1233 #[inline]
1234 fn as_view(self) -> CoherentView<'a, Self::Target> {
1235 self
1236 }
1237}
1238
1239impl<'a, T: ?Sized + KnownSize> IoBase<'a> for &'a Coherent<T> {
1240 type Backend = CoherentIoBackend;
1241 type Target = T;
1242
1243 #[inline]
1244 fn as_view(self) -> CoherentView<'a, Self::Target> {
1245 CoherentView {
1246 // SAFETY: `cpu_addr` is valid and aligned kernel accessible memory.
1247 cpu_addr: unsafe { SysMem::new(self.cpu_addr.as_ptr()) },
1248 dma_addr: self.dma_addr,
1249 }
1250 }
1251}