projects/linux·blob v7.2 · refs

1216 lines · 41.8 KiB

1// SPDX-License-Identifier: GPL-2.0
2
3//! Direct memory access (DMA).
4//!
5//! C header: [`include/linux/dma-mapping.h`](srctree/include/linux/dma-mapping.h)
6
7use crate::{
8 bindings,
9 debugfs,
10 device::{
11 self,
12 Bound,
13 Core, //
14 },
15 error::to_result,
16 fs::file,
17 prelude::*,
18 ptr::KnownSize,
19 sync::aref::ARef,
20 transmute::{
21 AsBytes,
22 FromBytes, //
23 }, //
24 uaccess::UserSliceWriter,
25};
26use core::{
27 ops::{
28 Deref,
29 DerefMut, //
30 },
31 ptr::NonNull, //
32};
33
34/// DMA address type.
35///
36/// Represents a bus address used for Direct Memory Access (DMA) operations.
37///
38/// This is an alias of the kernel's `dma_addr_t`, which may be `u32` or `u64` depending on
39/// `CONFIG_ARCH_DMA_ADDR_T_64BIT`.
40///
41/// Note that this may be `u64` even on 32-bit architectures.
42pub type DmaAddress = bindings::dma_addr_t;
43
44/// Trait to be implemented by DMA capable bus devices.
45///
46/// The [`dma::Device`](Device) trait should be implemented by bus specific device representations,
47/// where the underlying bus is DMA capable, such as:
48#[cfg_attr(CONFIG_PCI, doc = "* [`pci::Device`](kernel::pci::Device)")]
49/// * [`platform::Device`](::kernel::platform::Device)
50pub trait Device<'a>: AsRef<device::Device<Core<'a>>> {
51 /// Set up the device's DMA streaming addressing capabilities.
52 ///
53 /// This method is usually called once from `probe()` as soon as the device capabilities are
54 /// known.
55 ///
56 /// # Safety
57 ///
58 /// This method must not be called concurrently with any DMA allocation or mapping primitives,
59 /// such as [`Coherent::zeroed`].
60 unsafe fn dma_set_mask(&self, mask: DmaMask) -> Result {
61 // SAFETY:
62 // - By the type invariant of `device::Device`, `self.as_ref().as_raw()` is valid.
63 // - The safety requirement of this function guarantees that there are no concurrent calls
64 // to DMA allocation and mapping primitives using this mask.
65 to_result(unsafe { bindings::dma_set_mask(self.as_ref().as_raw(), mask.value()) })
66 }
67
68 /// Set up the device's DMA coherent addressing capabilities.
69 ///
70 /// This method is usually called once from `probe()` as soon as the device capabilities are
71 /// known.
72 ///
73 /// # Safety
74 ///
75 /// This method must not be called concurrently with any DMA allocation or mapping primitives,
76 /// such as [`Coherent::zeroed`].
77 unsafe fn dma_set_coherent_mask(&self, mask: DmaMask) -> Result {
78 // SAFETY:
79 // - By the type invariant of `device::Device`, `self.as_ref().as_raw()` is valid.
80 // - The safety requirement of this function guarantees that there are no concurrent calls
81 // to DMA allocation and mapping primitives using this mask.
82 to_result(unsafe { bindings::dma_set_coherent_mask(self.as_ref().as_raw(), mask.value()) })
83 }
84
85 /// Set up the device's DMA addressing capabilities.
86 ///
87 /// This is a combination of [`Device::dma_set_mask`] and [`Device::dma_set_coherent_mask`].
88 ///
89 /// This method is usually called once from `probe()` as soon as the device capabilities are
90 /// known.
91 ///
92 /// # Safety
93 ///
94 /// This method must not be called concurrently with any DMA allocation or mapping primitives,
95 /// such as [`Coherent::zeroed`].
96 unsafe fn dma_set_mask_and_coherent(&self, mask: DmaMask) -> Result {
97 // SAFETY:
98 // - By the type invariant of `device::Device`, `self.as_ref().as_raw()` is valid.
99 // - The safety requirement of this function guarantees that there are no concurrent calls
100 // to DMA allocation and mapping primitives using this mask.
101 to_result(unsafe {
102 bindings::dma_set_mask_and_coherent(self.as_ref().as_raw(), mask.value())
103 })
104 }
105
106 /// Set the maximum size of a single DMA segment the device may request.
107 ///
108 /// This method is usually called once from `probe()` as soon as the device capabilities are
109 /// known.
110 ///
111 /// # Safety
112 ///
113 /// This method must not be called concurrently with any DMA allocation or mapping primitives,
114 /// such as [`Coherent::zeroed`].
115 unsafe fn dma_set_max_seg_size(&self, size: u32) {
116 // SAFETY:
117 // - By the type invariant of `device::Device`, `self.as_ref().as_raw()` is valid.
118 // - The safety requirement of this function guarantees that there are no concurrent calls
119 // to DMA allocation and mapping primitives using this parameter.
120 unsafe { bindings::dma_set_max_seg_size(self.as_ref().as_raw(), size) }
121 }
122}
123
124/// A DMA mask that holds a bitmask with the lowest `n` bits set.
125///
126/// Use [`DmaMask::new`] or [`DmaMask::try_new`] to construct a value. Values
127/// are guaranteed to never exceed the bit width of `u64`.
128///
129/// This is the Rust equivalent of the C macro `DMA_BIT_MASK()`.
130#[derive(Debug, Clone, Copy, PartialEq, Eq)]
131pub struct DmaMask(u64);
132
133impl DmaMask {
134 /// Constructs a `DmaMask` with the lowest `n` bits set to `1`.
135 ///
136 /// For `n <= 64`, sets exactly the lowest `n` bits.
137 /// For `n > 64`, results in a build error.
138 ///
139 /// # Examples
140 ///
141 /// ```
142 /// use kernel::dma::DmaMask;
143 ///
144 /// let mask0 = DmaMask::new::<0>();
145 /// assert_eq!(mask0.value(), 0);
146 ///
147 /// let mask1 = DmaMask::new::<1>();
148 /// assert_eq!(mask1.value(), 0b1);
149 ///
150 /// let mask64 = DmaMask::new::<64>();
151 /// assert_eq!(mask64.value(), u64::MAX);
152 ///
153 /// // Build failure.
154 /// // let mask_overflow = DmaMask::new::<100>();
155 /// ```
156 #[inline]
157 pub const fn new<const N: u32>() -> Self {
158 let Ok(mask) = Self::try_new(N) else {
159 build_error!("Invalid DMA Mask.");
160 };
161
162 mask
163 }
164
165 /// Constructs a `DmaMask` with the lowest `n` bits set to `1`.
166 ///
167 /// For `n <= 64`, sets exactly the lowest `n` bits.
168 /// For `n > 64`, returns [`EINVAL`].
169 ///
170 /// # Examples
171 ///
172 /// ```
173 /// use kernel::dma::DmaMask;
174 ///
175 /// let mask0 = DmaMask::try_new(0)?;
176 /// assert_eq!(mask0.value(), 0);
177 ///
178 /// let mask1 = DmaMask::try_new(1)?;
179 /// assert_eq!(mask1.value(), 0b1);
180 ///
181 /// let mask64 = DmaMask::try_new(64)?;
182 /// assert_eq!(mask64.value(), u64::MAX);
183 ///
184 /// let mask_overflow = DmaMask::try_new(100);
185 /// assert!(mask_overflow.is_err());
186 /// # Ok::<(), Error>(())
187 /// ```
188 #[inline]
189 pub const fn try_new(n: u32) -> Result<Self> {
190 Ok(Self(match n {
191 0 => 0,
192 1..=64 => u64::MAX >> (64 - n),
193 _ => return Err(EINVAL),
194 }))
195 }
196
197 /// Returns the underlying `u64` bitmask value.
198 #[inline]
199 pub const fn value(&self) -> u64 {
200 self.0
201 }
202}
203
204/// Possible attributes associated with a DMA mapping.
205///
206/// They can be combined with the operators `|`, `&`, and `!`.
207///
208/// Values can be used from the [`attrs`] module.
209///
210/// # Examples
211///
212/// ```
213/// # use kernel::device::{Bound, Device};
214/// use kernel::dma::{attrs::*, Coherent};
215///
216/// # fn test(dev: &Device<Bound>) -> Result {
217/// let attribs = DMA_ATTR_FORCE_CONTIGUOUS | DMA_ATTR_NO_WARN;
218/// let c: Coherent<[u64]> =
219/// Coherent::zeroed_slice_with_attrs(dev, 4, GFP_KERNEL, attribs)?;
220/// # Ok::<(), Error>(()) }
221/// ```
222#[derive(Clone, Copy, PartialEq)]
223#[repr(transparent)]
224pub struct Attrs(u32);
225
226impl Attrs {
227 /// Get the raw representation of this attribute.
228 pub(crate) fn as_raw(self) -> crate::ffi::c_ulong {
229 self.0 as crate::ffi::c_ulong
230 }
231
232 /// Check whether `flags` is contained in `self`.
233 pub fn contains(self, flags: Attrs) -> bool {
234 (self & flags) == flags
235 }
236}
237
238impl core::ops::BitOr for Attrs {
239 type Output = Self;
240 fn bitor(self, rhs: Self) -> Self::Output {
241 Self(self.0 | rhs.0)
242 }
243}
244
245impl core::ops::BitAnd for Attrs {
246 type Output = Self;
247 fn bitand(self, rhs: Self) -> Self::Output {
248 Self(self.0 & rhs.0)
249 }
250}
251
252impl core::ops::Not for Attrs {
253 type Output = Self;
254 fn not(self) -> Self::Output {
255 Self(!self.0)
256 }
257}
258
259/// DMA mapping attributes.
260pub mod attrs {
261 use super::Attrs;
262
263 /// Specifies that reads and writes to the mapping may be weakly ordered, that is that reads
264 /// and writes may pass each other.
265 pub const DMA_ATTR_WEAK_ORDERING: Attrs = Attrs(bindings::DMA_ATTR_WEAK_ORDERING);
266
267 /// Specifies that writes to the mapping may be buffered to improve performance.
268 pub const DMA_ATTR_WRITE_COMBINE: Attrs = Attrs(bindings::DMA_ATTR_WRITE_COMBINE);
269
270 /// Allows platform code to skip synchronization of the CPU cache for the given buffer assuming
271 /// that it has been already transferred to 'device' domain.
272 pub const DMA_ATTR_SKIP_CPU_SYNC: Attrs = Attrs(bindings::DMA_ATTR_SKIP_CPU_SYNC);
273
274 /// Forces contiguous allocation of the buffer in physical memory.
275 pub const DMA_ATTR_FORCE_CONTIGUOUS: Attrs = Attrs(bindings::DMA_ATTR_FORCE_CONTIGUOUS);
276
277 /// Hints DMA-mapping subsystem that it's probably not worth the time to try
278 /// to allocate memory to in a way that gives better TLB efficiency.
279 pub const DMA_ATTR_ALLOC_SINGLE_PAGES: Attrs = Attrs(bindings::DMA_ATTR_ALLOC_SINGLE_PAGES);
280
281 /// This tells the DMA-mapping subsystem to suppress allocation failure reports (similarly to
282 /// `__GFP_NOWARN`).
283 pub const DMA_ATTR_NO_WARN: Attrs = Attrs(bindings::DMA_ATTR_NO_WARN);
284
285 /// Indicates that the buffer is fully accessible at an elevated privilege level (and
286 /// ideally inaccessible or at least read-only at lesser-privileged levels).
287 pub const DMA_ATTR_PRIVILEGED: Attrs = Attrs(bindings::DMA_ATTR_PRIVILEGED);
288
289 /// Indicates that the buffer is MMIO memory.
290 pub const DMA_ATTR_MMIO: Attrs = Attrs(bindings::DMA_ATTR_MMIO);
291}
292
293/// DMA data direction.
294///
295/// Corresponds to the C [`enum dma_data_direction`].
296///
297/// [`enum dma_data_direction`]: srctree/include/linux/dma-direction.h
298#[derive(Copy, Clone, PartialEq, Eq, Debug)]
299#[repr(u32)]
300pub enum DataDirection {
301 /// The DMA mapping is for bidirectional data transfer.
302 ///
303 /// This is used when the buffer can be both read from and written to by the device.
304 /// The cache for the corresponding memory region is both flushed and invalidated.
305 Bidirectional = Self::const_cast(bindings::dma_data_direction_DMA_BIDIRECTIONAL),
306
307 /// The DMA mapping is for data transfer from memory to the device (write).
308 ///
309 /// The CPU has prepared data in the buffer, and the device will read it.
310 /// The cache for the corresponding memory region is flushed before device access.
311 ToDevice = Self::const_cast(bindings::dma_data_direction_DMA_TO_DEVICE),
312
313 /// The DMA mapping is for data transfer from the device to memory (read).
314 ///
315 /// The device will write data into the buffer for the CPU to read.
316 /// The cache for the corresponding memory region is invalidated before CPU access.
317 FromDevice = Self::const_cast(bindings::dma_data_direction_DMA_FROM_DEVICE),
318
319 /// The DMA mapping is not for data transfer.
320 ///
321 /// This is primarily for debugging purposes. With this direction, the DMA mapping API
322 /// will not perform any cache coherency operations.
323 None = Self::const_cast(bindings::dma_data_direction_DMA_NONE),
324}
325
326impl DataDirection {
327 /// Casts the bindgen-generated enum type to a `u32` at compile time.
328 ///
329 /// This function will cause a compile-time error if the underlying value of the
330 /// C enum is out of bounds for `u32`.
331 const fn const_cast(val: bindings::dma_data_direction) -> u32 {
332 // CAST: The C standard allows compilers to choose different integer types for enums.
333 // To safely check the value, we cast it to a wide signed integer type (`i128`)
334 // which can hold any standard C integer enum type without truncation.
335 let wide_val = val as i128;
336
337 // Check if the value is outside the valid range for the target type `u32`.
338 // CAST: `u32::MAX` is cast to `i128` to match the type of `wide_val` for the comparison.
339 if wide_val < 0 || wide_val > u32::MAX as i128 {
340 // Trigger a compile-time error in a const context.
341 build_error!("C enum value is out of bounds for the target type `u32`.");
342 }
343
344 // CAST: This cast is valid because the check above guarantees that `wide_val`
345 // is within the representable range of `u32`.
346 wide_val as u32
347 }
348}
349
350impl From<DataDirection> for bindings::dma_data_direction {
351 /// Returns the raw representation of [`enum dma_data_direction`].
352 fn from(direction: DataDirection) -> Self {
353 // CAST: `direction as u32` gets the underlying representation of our `#[repr(u32)]` enum.
354 // The subsequent cast to `Self` (the bindgen type) assumes the C enum is compatible
355 // with the enum variants of `DataDirection`, which is a valid assumption given our
356 // compile-time checks.
357 direction as u32 as Self
358 }
359}
360
361/// CPU-owned DMA allocation that can be converted into a device-shared [`Coherent`] object.
362///
363/// Unlike [`Coherent`], a [`CoherentBox`] is guaranteed to be fully owned by the CPU -- its DMA
364/// address is not exposed and it cannot be accessed by a device. This means it can safely be used
365/// like a normal boxed allocation (e.g. direct reads, writes, and mutable slices are all safe).
366///
367/// A typical use is to allocate a [`CoherentBox`], populate it with normal CPU access, and then
368/// convert it into a [`Coherent`] object to share it with the device.
369///
370/// # Examples
371///
372/// `CoherentBox<T>`:
373///
374/// ```
375/// # use kernel::device::{
376/// # Bound,
377/// # Device,
378/// # };
379/// use kernel::dma::{attrs::*,
380/// Coherent,
381/// CoherentBox,
382/// };
383///
384/// # fn test(dev: &Device<Bound>) -> Result {
385/// let mut dmem: CoherentBox<u64> = CoherentBox::zeroed(dev, GFP_KERNEL)?;
386/// *dmem = 42;
387/// let dmem: Coherent<u64> = dmem.into();
388/// # Ok::<(), Error>(()) }
389/// ```
390///
391/// `CoherentBox<[T]>`:
392///
393///
394/// ```
395/// # use kernel::device::{
396/// # Bound,
397/// # Device,
398/// # };
399/// use kernel::dma::{attrs::*,
400/// Coherent,
401/// CoherentBox,
402/// };
403///
404/// # fn test(dev: &Device<Bound>) -> Result {
405/// let mut dmem: CoherentBox<[u64]> = CoherentBox::zeroed_slice(dev, 4, GFP_KERNEL)?;
406/// dmem.fill(42);
407/// let dmem: Coherent<[u64]> = dmem.into();
408/// # Ok::<(), Error>(()) }
409/// ```
410pub struct CoherentBox<T: KnownSize + ?Sized>(Coherent<T>);
411
412impl<T: AsBytes + FromBytes> CoherentBox<[T]> {
413 /// [`CoherentBox`] variant of [`Coherent::zeroed_slice_with_attrs`].
414 #[inline]
415 pub fn zeroed_slice_with_attrs(
416 dev: &device::Device<Bound>,
417 count: usize,
418 gfp_flags: kernel::alloc::Flags,
419 dma_attrs: Attrs,
420 ) -> Result<Self> {
421 Coherent::zeroed_slice_with_attrs(dev, count, gfp_flags, dma_attrs).map(Self)
422 }
423
424 /// Same as [CoherentBox::zeroed_slice_with_attrs], but with `dma::Attrs(0)`.
425 #[inline]
426 pub fn zeroed_slice(
427 dev: &device::Device<Bound>,
428 count: usize,
429 gfp_flags: kernel::alloc::Flags,
430 ) -> Result<Self> {
431 Self::zeroed_slice_with_attrs(dev, count, gfp_flags, Attrs(0))
432 }
433
434 /// Initializes the element at `i` using the given initializer.
435 ///
436 /// Returns `EINVAL` if `i` is out of bounds.
437 pub fn init_at<E>(&mut self, i: usize, init: impl Init<T, E>) -> Result
438 where
439 Error: From<E>,
440 {
441 if i >= self.0.len() {
442 return Err(EINVAL);
443 }
444
445 let ptr = &raw mut self[i];
446
447 // SAFETY:
448 // - `ptr` is valid, properly aligned, and within this allocation.
449 // - `T: AsBytes + FromBytes` guarantees all bit patterns are valid, so partial writes on
450 // error cannot leave the element in an invalid state.
451 // - The DMA address has not been exposed yet, so there is no concurrent device access.
452 unsafe { init.__init(ptr)? };
453
454 Ok(())
455 }
456
457 /// Allocates a region of coherent memory of the same size as `data` and initializes it with a
458 /// copy of its contents.
459 ///
460 /// This is the [`CoherentBox`] variant of [`Coherent::from_slice_with_attrs`].
461 ///
462 /// # Examples
463 ///
464 /// ```
465 /// use core::ops::Deref;
466 ///
467 /// # use kernel::device::{Bound, Device};
468 /// use kernel::dma::{
469 /// attrs::*,
470 /// CoherentBox
471 /// };
472 ///
473 /// # fn test(dev: &Device<Bound>) -> Result {
474 /// let data = [0u8, 1u8, 2u8, 3u8];
475 /// let c: CoherentBox<[u8]> =
476 /// CoherentBox::from_slice_with_attrs(dev, &data, GFP_KERNEL, DMA_ATTR_NO_WARN)?;
477 ///
478 /// assert_eq!(c.deref(), &data);
479 /// # Ok::<(), Error>(()) }
480 /// ```
481 pub fn from_slice_with_attrs(
482 dev: &device::Device<Bound>,
483 data: &[T],
484 gfp_flags: kernel::alloc::Flags,
485 dma_attrs: Attrs,
486 ) -> Result<Self>
487 where
488 T: Copy,
489 {
490 let mut slice = Self(Coherent::<T>::alloc_slice_with_attrs(
491 dev,
492 data.len(),
493 gfp_flags,
494 dma_attrs,
495 )?);
496
497 // PANIC: `slice` was created with length `data.len()`.
498 slice.copy_from_slice(data);
499
500 Ok(slice)
501 }
502
503 /// Performs the same functionality as [`CoherentBox::from_slice_with_attrs`], except the
504 /// `dma_attrs` is 0 by default.
505 #[inline]
506 pub fn from_slice(
507 dev: &device::Device<Bound>,
508 data: &[T],
509 gfp_flags: kernel::alloc::Flags,
510 ) -> Result<Self>
511 where
512 T: Copy,
513 {
514 Self::from_slice_with_attrs(dev, data, gfp_flags, Attrs(0))
515 }
516}
517
518impl<T: AsBytes + FromBytes> CoherentBox<T> {
519 /// Same as [`CoherentBox::zeroed_slice_with_attrs`], but for a single element.
520 #[inline]
521 pub fn zeroed_with_attrs(
522 dev: &device::Device<Bound>,
523 gfp_flags: kernel::alloc::Flags,
524 dma_attrs: Attrs,
525 ) -> Result<Self> {
526 Coherent::zeroed_with_attrs(dev, gfp_flags, dma_attrs).map(Self)
527 }
528
529 /// Same as [`CoherentBox::zeroed_slice`], but for a single element.
530 #[inline]
531 pub fn zeroed(dev: &device::Device<Bound>, gfp_flags: kernel::alloc::Flags) -> Result<Self> {
532 Self::zeroed_with_attrs(dev, gfp_flags, Attrs(0))
533 }
534}
535
536impl<T: KnownSize + ?Sized> Deref for CoherentBox<T> {
537 type Target = T;
538
539 #[inline]
540 fn deref(&self) -> &Self::Target {
541 // SAFETY:
542 // - We have not exposed the DMA address yet, so there can't be any concurrent access by a
543 // device.
544 // - We have exclusive access to `self.0`.
545 unsafe { self.0.as_ref() }
546 }
547}
548
549impl<T: AsBytes + FromBytes + KnownSize + ?Sized> DerefMut for CoherentBox<T> {
550 #[inline]
551 fn deref_mut(&mut self) -> &mut Self::Target {
552 // SAFETY:
553 // - We have not exposed the DMA address yet, so there can't be any concurrent access by a
554 // device.
555 // - We have exclusive access to `self.0`.
556 unsafe { self.0.as_mut() }
557 }
558}
559
560impl<T: AsBytes + FromBytes + KnownSize + ?Sized> From<CoherentBox<T>> for Coherent<T> {
561 #[inline]
562 fn from(value: CoherentBox<T>) -> Self {
563 value.0
564 }
565}
566
567/// An abstraction of the `dma_alloc_coherent` API.
568///
569/// This is an abstraction around the `dma_alloc_coherent` API which is used to allocate and map
570/// large coherent DMA regions.
571///
572/// A [`Coherent`] instance contains a pointer to the allocated region (in the
573/// processor's virtual address space) and the device address which can be given to the device
574/// as the DMA address base of the region. The region is released once [`Coherent`]
575/// is dropped.
576///
577/// # Invariants
578///
579/// - For the lifetime of an instance of [`Coherent`], the `cpu_addr` is a valid pointer
580/// to an allocated region of coherent memory and `dma_handle` is the DMA address base of the
581/// region.
582/// - The size in bytes of the allocation is equal to size information via pointer.
583// TODO
584//
585// DMA allocations potentially carry device resources (e.g.IOMMU mappings), hence for soundness
586// reasons DMA allocation would need to be embedded in a `Devres` container, in order to ensure
587// that device resources can never survive device unbind.
588//
589// However, it is neither desirable nor necessary to protect the allocated memory of the DMA
590// allocation from surviving device unbind; it would require RCU read side critical sections to
591// access the memory, which may require subsequent unnecessary copies.
592//
593// Hence, find a way to revoke the device resources of a `Coherent`, but not the
594// entire `Coherent` including the allocated memory itself.
595pub struct Coherent<T: KnownSize + ?Sized> {
596 dev: ARef<device::Device>,
597 dma_handle: DmaAddress,
598 cpu_addr: NonNull<T>,
599 dma_attrs: Attrs,
600}
601
602impl<T: KnownSize + ?Sized> Coherent<T> {
603 /// Returns the size in bytes of this allocation.
604 #[inline]
605 pub fn size(&self) -> usize {
606 T::size(self.cpu_addr.as_ptr())
607 }
608
609 /// Returns the raw pointer to the allocated region in the CPU's virtual address space.
610 #[inline]
611 pub fn as_ptr(&self) -> *const T {
612 self.cpu_addr.as_ptr()
613 }
614
615 /// Returns the raw pointer to the allocated region in the CPU's virtual address space as
616 /// a mutable pointer.
617 #[inline]
618 pub fn as_mut_ptr(&self) -> *mut T {
619 self.cpu_addr.as_ptr()
620 }
621
622 /// Returns a DMA handle which may be given to the device as the DMA address base of
623 /// the region.
624 #[inline]
625 pub fn dma_handle(&self) -> DmaAddress {
626 self.dma_handle
627 }
628
629 /// Returns a reference to the data in the region.
630 ///
631 /// # Safety
632 ///
633 /// * Callers must ensure that the device does not read/write to/from memory while the returned
634 /// slice is live.
635 /// * Callers must ensure that this call does not race with a write to the same region while
636 /// the returned slice is live.
637 #[inline]
638 pub unsafe fn as_ref(&self) -> &T {
639 // SAFETY: per safety requirement.
640 unsafe { &*self.as_ptr() }
641 }
642
643 /// Returns a mutable reference to the data in the region.
644 ///
645 /// # Safety
646 ///
647 /// * Callers must ensure that the device does not read/write to/from memory while the returned
648 /// slice is live.
649 /// * Callers must ensure that this call does not race with a read or write to the same region
650 /// while the returned slice is live.
651 #[expect(clippy::mut_from_ref, reason = "unsafe to use API")]
652 #[inline]
653 pub unsafe fn as_mut(&self) -> &mut T {
654 // SAFETY: per safety requirement.
655 unsafe { &mut *self.as_mut_ptr() }
656 }
657
658 /// Reads the value of `field` and ensures that its type is [`FromBytes`].
659 ///
660 /// # Safety
661 ///
662 /// This must be called from the [`dma_read`] macro which ensures that the `field` pointer is
663 /// validated beforehand.
664 ///
665 /// Public but hidden since it should only be used from [`dma_read`] macro.
666 #[doc(hidden)]
667 pub unsafe fn field_read<F: FromBytes>(&self, field: *const F) -> F {
668 // SAFETY:
669 // - By the safety requirements field is valid.
670 // - Using read_volatile() here is not sound as per the usual rules, the usage here is
671 // a special exception with the following notes in place. When dealing with a potential
672 // race from a hardware or code outside kernel (e.g. user-space program), we need that
673 // read on a valid memory is not UB. Currently read_volatile() is used for this, and the
674 // rationale behind is that it should generate the same code as READ_ONCE() which the
675 // kernel already relies on to avoid UB on data races. Note that the usage of
676 // read_volatile() is limited to this particular case, it cannot be used to prevent
677 // the UB caused by racing between two kernel functions nor do they provide atomicity.
678 unsafe { field.read_volatile() }
679 }
680
681 /// Writes a value to `field` and ensures that its type is [`AsBytes`].
682 ///
683 /// # Safety
684 ///
685 /// This must be called from the [`dma_write`] macro which ensures that the `field` pointer is
686 /// validated beforehand.
687 ///
688 /// Public but hidden since it should only be used from [`dma_write`] macro.
689 #[doc(hidden)]
690 pub unsafe fn field_write<F: AsBytes>(&self, field: *mut F, val: F) {
691 // SAFETY:
692 // - By the safety requirements field is valid.
693 // - Using write_volatile() here is not sound as per the usual rules, the usage here is
694 // a special exception with the following notes in place. When dealing with a potential
695 // race from a hardware or code outside kernel (e.g. user-space program), we need that
696 // write on a valid memory is not UB. Currently write_volatile() is used for this, and the
697 // rationale behind is that it should generate the same code as WRITE_ONCE() which the
698 // kernel already relies on to avoid UB on data races. Note that the usage of
699 // write_volatile() is limited to this particular case, it cannot be used to prevent
700 // the UB caused by racing between two kernel functions nor do they provide atomicity.
701 unsafe { field.write_volatile(val) }
702 }
703}
704
705impl<T: AsBytes + FromBytes> Coherent<T> {
706 /// Allocates a region of `T` of coherent memory.
707 fn alloc_with_attrs(
708 dev: &device::Device<Bound>,
709 gfp_flags: kernel::alloc::Flags,
710 dma_attrs: Attrs,
711 ) -> Result<Self> {
712 const {
713 assert!(
714 core::mem::size_of::<T>() > 0,
715 "It doesn't make sense for the allocated type to be a ZST"
716 );
717 }
718
719 let mut dma_handle = 0;
720 // SAFETY: Device pointer is guaranteed as valid by the type invariant on `Device`.
721 let addr = unsafe {
722 bindings::dma_alloc_attrs(
723 dev.as_raw(),
724 core::mem::size_of::<T>(),
725 &mut dma_handle,
726 gfp_flags.as_raw(),
727 dma_attrs.as_raw(),
728 )
729 };
730 let cpu_addr = NonNull::new(addr.cast()).ok_or(ENOMEM)?;
731 // INVARIANT:
732 // - We just successfully allocated a coherent region which is adequately sized for `T`,
733 // hence the cpu address is valid.
734 // - We also hold a refcounted reference to the device.
735 Ok(Self {
736 dev: dev.into(),
737 dma_handle,
738 cpu_addr,
739 dma_attrs,
740 })
741 }
742
743 /// Allocates a region of type `T` of coherent memory.
744 ///
745 /// # Examples
746 ///
747 /// ```
748 /// # use kernel::device::{
749 /// # Bound,
750 /// # Device,
751 /// # };
752 /// use kernel::dma::{
753 /// attrs::*,
754 /// Coherent,
755 /// };
756 ///
757 /// # fn test(dev: &Device<Bound>) -> Result {
758 /// let c: Coherent<[u64; 4]> =
759 /// Coherent::zeroed_with_attrs(dev, GFP_KERNEL, DMA_ATTR_NO_WARN)?;
760 /// # Ok::<(), Error>(()) }
761 /// ```
762 #[inline]
763 pub fn zeroed_with_attrs(
764 dev: &device::Device<Bound>,
765 gfp_flags: kernel::alloc::Flags,
766 dma_attrs: Attrs,
767 ) -> Result<Self> {
768 Self::alloc_with_attrs(dev, gfp_flags | __GFP_ZERO, dma_attrs)
769 }
770
771 /// Performs the same functionality as [`Coherent::zeroed_with_attrs`], except the
772 /// `dma_attrs` is 0 by default.
773 #[inline]
774 pub fn zeroed(dev: &device::Device<Bound>, gfp_flags: kernel::alloc::Flags) -> Result<Self> {
775 Self::zeroed_with_attrs(dev, gfp_flags, Attrs(0))
776 }
777
778 /// Same as [`Coherent::zeroed_with_attrs`], but instead of a zero-initialization the memory is
779 /// initialized with `init`.
780 pub fn init_with_attrs<E>(
781 dev: &device::Device<Bound>,
782 gfp_flags: kernel::alloc::Flags,
783 dma_attrs: Attrs,
784 init: impl Init<T, E>,
785 ) -> Result<Self>
786 where
787 Error: From<E>,
788 {
789 let dmem = Self::alloc_with_attrs(dev, gfp_flags, dma_attrs)?;
790 let ptr = dmem.as_mut_ptr();
791
792 // SAFETY:
793 // - `ptr` is valid, properly aligned, and points to exclusively owned memory.
794 // - If `__init` fails, `self` is dropped, which safely frees the underlying `Coherent`'s
795 // DMA memory. `T: AsBytes + FromBytes` ensures there are no complex `Drop` requirements
796 // we are bypassing.
797 unsafe { init.__init(ptr)? };
798
799 Ok(dmem)
800 }
801
802 /// Same as [`Coherent::zeroed`], but instead of a zero-initialization the memory is initialized
803 /// with `init`.
804 #[inline]
805 pub fn init<E>(
806 dev: &device::Device<Bound>,
807 gfp_flags: kernel::alloc::Flags,
808 init: impl Init<T, E>,
809 ) -> Result<Self>
810 where
811 Error: From<E>,
812 {
813 Self::init_with_attrs(dev, gfp_flags, Attrs(0), init)
814 }
815
816 /// Allocates a region of `[T; len]` of coherent memory.
817 fn alloc_slice_with_attrs(
818 dev: &device::Device<Bound>,
819 len: usize,
820 gfp_flags: kernel::alloc::Flags,
821 dma_attrs: Attrs,
822 ) -> Result<Coherent<[T]>> {
823 const {
824 assert!(
825 core::mem::size_of::<T>() > 0,
826 "It doesn't make sense for the allocated type to be a ZST"
827 );
828 }
829
830 // `dma_alloc_attrs` cannot handle zero-length allocation, bail early.
831 if len == 0 {
832 Err(EINVAL)?;
833 }
834
835 let size = core::mem::size_of::<T>().checked_mul(len).ok_or(ENOMEM)?;
836 let mut dma_handle = 0;
837 // SAFETY: Device pointer is guaranteed as valid by the type invariant on `Device`.
838 let addr = unsafe {
839 bindings::dma_alloc_attrs(
840 dev.as_raw(),
841 size,
842 &mut dma_handle,
843 gfp_flags.as_raw(),
844 dma_attrs.as_raw(),
845 )
846 };
847 let cpu_addr = NonNull::slice_from_raw_parts(NonNull::new(addr.cast()).ok_or(ENOMEM)?, len);
848 // INVARIANT:
849 // - We just successfully allocated a coherent region which is adequately sized for
850 // `[T; len]`, hence the cpu address is valid.
851 // - We also hold a refcounted reference to the device.
852 Ok(Coherent {
853 dev: dev.into(),
854 dma_handle,
855 cpu_addr,
856 dma_attrs,
857 })
858 }
859
860 /// Allocates a zeroed region of type `T` of coherent memory.
861 ///
862 /// Unlike `Coherent::<[T; N]>::zeroed_with_attrs`, `Coherent::<T>::zeroed_slices` support
863 /// a runtime length.
864 ///
865 /// # Examples
866 ///
867 /// ```
868 /// # use kernel::device::{
869 /// # Bound,
870 /// # Device,
871 /// # };
872 /// use kernel::dma::{
873 /// attrs::*,
874 /// Coherent,
875 /// };
876 ///
877 /// # fn test(dev: &Device<Bound>) -> Result {
878 /// let c: Coherent<[u64]> =
879 /// Coherent::zeroed_slice_with_attrs(dev, 4, GFP_KERNEL, DMA_ATTR_NO_WARN)?;
880 /// # Ok::<(), Error>(()) }
881 /// ```
882 #[inline]
883 pub fn zeroed_slice_with_attrs(
884 dev: &device::Device<Bound>,
885 len: usize,
886 gfp_flags: kernel::alloc::Flags,
887 dma_attrs: Attrs,
888 ) -> Result<Coherent<[T]>> {
889 Coherent::alloc_slice_with_attrs(dev, len, gfp_flags | __GFP_ZERO, dma_attrs)
890 }
891
892 /// Performs the same functionality as [`Coherent::zeroed_slice_with_attrs`], except the
893 /// `dma_attrs` is 0 by default.
894 #[inline]
895 pub fn zeroed_slice(
896 dev: &device::Device<Bound>,
897 len: usize,
898 gfp_flags: kernel::alloc::Flags,
899 ) -> Result<Coherent<[T]>> {
900 Self::zeroed_slice_with_attrs(dev, len, gfp_flags, Attrs(0))
901 }
902
903 /// Allocates a region of coherent memory of the same size as `data` and initializes it with a
904 /// copy of its contents.
905 ///
906 /// # Examples
907 ///
908 /// ```
909 /// # use kernel::device::{Bound, Device};
910 /// use kernel::dma::{
911 /// attrs::*,
912 /// Coherent
913 /// };
914 ///
915 /// # fn test(dev: &Device<Bound>) -> Result {
916 /// let data = [0u8, 1u8, 2u8, 3u8];
917 /// // `c` has the same content as `data`.
918 /// let c: Coherent<[u8]> =
919 /// Coherent::from_slice_with_attrs(dev, &data, GFP_KERNEL, DMA_ATTR_NO_WARN)?;
920 ///
921 /// # Ok::<(), Error>(()) }
922 /// ```
923 #[inline]
924 pub fn from_slice_with_attrs(
925 dev: &device::Device<Bound>,
926 data: &[T],
927 gfp_flags: kernel::alloc::Flags,
928 dma_attrs: Attrs,
929 ) -> Result<Coherent<[T]>>
930 where
931 T: Copy,
932 {
933 CoherentBox::from_slice_with_attrs(dev, data, gfp_flags, dma_attrs).map(Into::into)
934 }
935
936 /// Performs the same functionality as [`Coherent::from_slice_with_attrs`], except the
937 /// `dma_attrs` is 0 by default.
938 #[inline]
939 pub fn from_slice(
940 dev: &device::Device<Bound>,
941 data: &[T],
942 gfp_flags: kernel::alloc::Flags,
943 ) -> Result<Coherent<[T]>>
944 where
945 T: Copy,
946 {
947 Self::from_slice_with_attrs(dev, data, gfp_flags, Attrs(0))
948 }
949}
950
951impl<T> Coherent<[T]> {
952 /// Returns the number of elements `T` in this allocation.
953 ///
954 /// Note that this is not the size of the allocation in bytes, which is provided by
955 /// [`Self::size`].
956 #[inline]
957 #[expect(clippy::len_without_is_empty, reason = "Coherent slice is never empty")]
958 pub fn len(&self) -> usize {
959 self.cpu_addr.len()
960 }
961}
962
963/// Note that the device configured to do DMA must be halted before this object is dropped.
964impl<T: KnownSize + ?Sized> Drop for Coherent<T> {
965 fn drop(&mut self) {
966 let size = T::size(self.cpu_addr.as_ptr());
967 // SAFETY: Device pointer is guaranteed as valid by the type invariant on `Device`.
968 // The cpu address, and the dma handle are valid due to the type invariants on
969 // `Coherent`.
970 unsafe {
971 bindings::dma_free_attrs(
972 self.dev.as_raw(),
973 size,
974 self.cpu_addr.as_ptr().cast(),
975 self.dma_handle,
976 self.dma_attrs.as_raw(),
977 )
978 }
979 }
980}
981
982// SAFETY: It is safe to send a `Coherent` to another thread if `T`
983// can be sent to another thread.
984unsafe impl<T: KnownSize + Send + ?Sized> Send for Coherent<T> {}
985
986// SAFETY: Sharing `&Coherent` across threads is safe if `T` is `Sync`, because all
987// methods that access the buffer contents (`field_read`, `field_write`, `as_slice`,
988// `as_slice_mut`) are `unsafe`, and callers are responsible for ensuring no data races occur.
989// The safe methods only return metadata or raw pointers whose use requires `unsafe`.
990unsafe impl<T: KnownSize + ?Sized + AsBytes + FromBytes + Sync> Sync for Coherent<T> {}
991
992impl<T: KnownSize + AsBytes + ?Sized> debugfs::BinaryWriter for Coherent<T> {
993 fn write_to_slice(
994 &self,
995 writer: &mut UserSliceWriter,
996 offset: &mut file::Offset,
997 ) -> Result<usize> {
998 if offset.is_negative() {
999 return Err(EINVAL);
1000 }
1001
1002 // If the offset is too large for a usize (e.g. on 32-bit platforms),
1003 // then consider that as past EOF and just return 0 bytes.
1004 let Ok(offset_val) = usize::try_from(*offset) else {
1005 return Ok(0);
1006 };
1007
1008 let count = self.size().saturating_sub(offset_val).min(writer.len());
1009
1010 writer.write_dma(self, offset_val, count)?;
1011
1012 *offset += count as i64;
1013 Ok(count)
1014 }
1015}
1016
1017/// An opaque DMA allocation without a kernel virtual mapping.
1018///
1019/// Unlike [`Coherent`], a `CoherentHandle` does not provide CPU access to the allocated memory.
1020/// The allocation is always performed with `DMA_ATTR_NO_KERNEL_MAPPING`, meaning no kernel
1021/// virtual mapping is created for the buffer. The value returned by the C API as the CPU
1022/// address is an opaque handle used only to free the allocation.
1023///
1024/// This is useful for buffers that are only ever accessed by hardware.
1025///
1026/// # Invariants
1027///
1028/// - `cpu_handle` holds the opaque handle returned by `dma_alloc_attrs` with
1029/// `DMA_ATTR_NO_KERNEL_MAPPING` set, and is only valid for passing back to `dma_free_attrs`.
1030/// - `dma_handle` is the corresponding bus address for device DMA.
1031/// - `size` is the allocation size in bytes as passed to `dma_alloc_attrs`.
1032/// - `dma_attrs` contains the attributes used for the allocation, always including
1033/// `DMA_ATTR_NO_KERNEL_MAPPING`.
1034pub struct CoherentHandle {
1035 dev: ARef<device::Device>,
1036 dma_handle: DmaAddress,
1037 cpu_handle: NonNull<c_void>,
1038 size: usize,
1039 dma_attrs: Attrs,
1040}
1041
1042impl CoherentHandle {
1043 /// Allocates `size` bytes of coherent DMA memory without creating a kernel virtual mapping.
1044 ///
1045 /// Additional DMA attributes may be passed via `dma_attrs`; `DMA_ATTR_NO_KERNEL_MAPPING` is
1046 /// always set implicitly.
1047 ///
1048 /// Returns `EINVAL` if `size` is zero, `ENOMEM` if the allocation fails.
1049 pub fn alloc_with_attrs(
1050 dev: &device::Device<Bound>,
1051 size: usize,
1052 gfp_flags: kernel::alloc::Flags,
1053 dma_attrs: Attrs,
1054 ) -> Result<Self> {
1055 if size == 0 {
1056 return Err(EINVAL);
1057 }
1058
1059 let dma_attrs = dma_attrs | Attrs(bindings::DMA_ATTR_NO_KERNEL_MAPPING);
1060 let mut dma_handle = 0;
1061 // SAFETY: `dev.as_raw()` is valid by the type invariant on `device::Device`.
1062 let cpu_handle = unsafe {
1063 bindings::dma_alloc_attrs(
1064 dev.as_raw(),
1065 size,
1066 &mut dma_handle,
1067 gfp_flags.as_raw(),
1068 dma_attrs.as_raw(),
1069 )
1070 };
1071
1072 let cpu_handle = NonNull::new(cpu_handle).ok_or(ENOMEM)?;
1073
1074 // INVARIANT: `cpu_handle` is the opaque handle from a successful `dma_alloc_attrs` call
1075 // with `DMA_ATTR_NO_KERNEL_MAPPING`, `dma_handle` is the corresponding DMA address,
1076 // and we hold a refcounted reference to the device.
1077 Ok(Self {
1078 dev: dev.into(),
1079 dma_handle,
1080 cpu_handle,
1081 size,
1082 dma_attrs,
1083 })
1084 }
1085
1086 /// Allocates `size` bytes of coherent DMA memory without creating a kernel virtual mapping.
1087 #[inline]
1088 pub fn alloc(
1089 dev: &device::Device<Bound>,
1090 size: usize,
1091 gfp_flags: kernel::alloc::Flags,
1092 ) -> Result<Self> {
1093 Self::alloc_with_attrs(dev, size, gfp_flags, Attrs(0))
1094 }
1095
1096 /// Returns the DMA handle for this allocation.
1097 ///
1098 /// This address can be programmed into device hardware for DMA access.
1099 #[inline]
1100 pub fn dma_handle(&self) -> DmaAddress {
1101 self.dma_handle
1102 }
1103
1104 /// Returns the size in bytes of this allocation.
1105 #[inline]
1106 pub fn size(&self) -> usize {
1107 self.size
1108 }
1109}
1110
1111impl Drop for CoherentHandle {
1112 fn drop(&mut self) {
1113 // SAFETY: All values are valid by the type invariants on `CoherentHandle`.
1114 // `cpu_handle` is the opaque handle from `dma_alloc_attrs` and is passed back unchanged.
1115 unsafe {
1116 bindings::dma_free_attrs(
1117 self.dev.as_raw(),
1118 self.size,
1119 self.cpu_handle.as_ptr(),
1120 self.dma_handle,
1121 self.dma_attrs.as_raw(),
1122 )
1123 }
1124 }
1125}
1126
1127// SAFETY: `CoherentHandle` only holds a device reference, a DMA handle, an opaque CPU handle,
1128// and a size. None of these are tied to a specific thread.
1129unsafe impl Send for CoherentHandle {}
1130
1131// SAFETY: `CoherentHandle` provides no CPU access to the underlying allocation. The only
1132// operations on `&CoherentHandle` are reading the DMA handle and size, both of which are
1133// plain `Copy` values.
1134unsafe impl Sync for CoherentHandle {}
1135
1136/// Reads a field of an item from an allocated region of structs.
1137///
1138/// The syntax is of the form `kernel::dma_read!(dma, proj)` where `dma` is an expression evaluating
1139/// to a [`Coherent`] and `proj` is a [projection specification](kernel::ptr::project!).
1140///
1141/// # Examples
1142///
1143/// ```
1144/// use kernel::device::Device;
1145/// use kernel::dma::{attrs::*, Coherent};
1146///
1147/// struct MyStruct { field: u32, }
1148///
1149/// // SAFETY: All bit patterns are acceptable values for `MyStruct`.
1150/// unsafe impl kernel::transmute::FromBytes for MyStruct{};
1151/// // SAFETY: Instances of `MyStruct` have no uninitialized portions.
1152/// unsafe impl kernel::transmute::AsBytes for MyStruct{};
1153///
1154/// # fn test(alloc: &kernel::dma::Coherent<[MyStruct]>) -> Result {
1155/// let whole = kernel::dma_read!(alloc, [try: 2]);
1156/// let field = kernel::dma_read!(alloc, [panic: 1].field);
1157/// # Ok::<(), Error>(()) }
1158/// ```
1159#[macro_export]
1160macro_rules! dma_read {
1161 ($dma:expr, $($proj:tt)*) => {{
1162 let dma = &$dma;
1163 let ptr = $crate::ptr::project!(
1164 $crate::dma::Coherent::as_ptr(dma), $($proj)*
1165 );
1166 // SAFETY: The pointer created by the projection is within the DMA region.
1167 unsafe { $crate::dma::Coherent::field_read(dma, ptr) }
1168 }};
1169}
1170
1171/// Writes to a field of an item from an allocated region of structs.
1172///
1173/// The syntax is of the form `kernel::dma_write!(dma, proj, val)` where `dma` is an expression
1174/// evaluating to a [`Coherent`], `proj` is a
1175/// [projection specification](kernel::ptr::project!), and `val` is the value to be written to the
1176/// projected location.
1177///
1178/// # Examples
1179///
1180/// ```
1181/// use kernel::device::Device;
1182/// use kernel::dma::{attrs::*, Coherent};
1183///
1184/// struct MyStruct { member: u32, }
1185///
1186/// // SAFETY: All bit patterns are acceptable values for `MyStruct`.
1187/// unsafe impl kernel::transmute::FromBytes for MyStruct{};
1188/// // SAFETY: Instances of `MyStruct` have no uninitialized portions.
1189/// unsafe impl kernel::transmute::AsBytes for MyStruct{};
1190///
1191/// # fn test(alloc: &kernel::dma::Coherent<[MyStruct]>) -> Result {
1192/// kernel::dma_write!(alloc, [try: 2].member, 0xf);
1193/// kernel::dma_write!(alloc, [panic: 1], MyStruct { member: 0xf });
1194/// # Ok::<(), Error>(()) }
1195/// ```
1196#[macro_export]
1197macro_rules! dma_write {
1198 (@parse [$dma:expr] [$($proj:tt)*] [, $val:expr]) => {{
1199 let dma = &$dma;
1200 let ptr = $crate::ptr::project!(
1201 mut $crate::dma::Coherent::as_mut_ptr(dma), $($proj)*
1202 );
1203 let val = $val;
1204 // SAFETY: The pointer created by the projection is within the DMA region.
1205 unsafe { $crate::dma::Coherent::field_write(dma, ptr, val) }
1206 }};
1207 (@parse [$dma:expr] [$($proj:tt)*] [.$field:tt $($rest:tt)*]) => {
1208 $crate::dma_write!(@parse [$dma] [$($proj)* .$field] [$($rest)*])
1209 };
1210 (@parse [$dma:expr] [$($proj:tt)*] [[$flavor:ident: $index:expr] $($rest:tt)*]) => {
1211 $crate::dma_write!(@parse [$dma] [$($proj)* [$flavor: $index]] [$($rest)*])
1212 };
1213 ($dma:expr, $($rest:tt)*) => {
1214 $crate::dma_write!(@parse [$dma] [] [$($rest)*])
1215 };
1216}