projects/linux·blob master · refs

944 lines · 35.3 KiB

1// SPDX-License-Identifier: (BSD-2-Clause OR Apache-2.0) OR MIT
2//
3// Copyright 2023 The Fuchsia Authors
4//
5// Licensed under a BSD-style license <LICENSE-BSD>, Apache License, Version 2.0
6// <LICENSE-APACHE or https://www.apache.org/licenses/LICENSE-2.0>, or the MIT
7// license <LICENSE-MIT or https://opensource.org/licenses/MIT>, at your option.
8// This file may not be copied, modified, or distributed except according to
9// those terms.
10
11#[macro_use]
12pub(crate) mod macros;
13
14#[doc(hidden)]
15pub mod macro_util;
16
17use core::{
18 marker::PhantomData,
19 mem::{self, ManuallyDrop},
20 num::NonZeroUsize,
21 ptr::NonNull,
22};
23
24use super::*;
25use crate::pointer::{
26 invariant::{Exclusive, Shared, Valid},
27 SizeEq, TransmuteFromPtr,
28};
29
30/// Like [`PhantomData`], but [`Send`] and [`Sync`] regardless of whether the
31/// wrapped `T` is.
32pub(crate) struct SendSyncPhantomData<T: ?Sized>(PhantomData<T>);
33
34// SAFETY: `SendSyncPhantomData` does not enable any behavior which isn't sound
35// to be called from multiple threads.
36unsafe impl<T: ?Sized> Send for SendSyncPhantomData<T> {}
37// SAFETY: `SendSyncPhantomData` does not enable any behavior which isn't sound
38// to be called from multiple threads.
39unsafe impl<T: ?Sized> Sync for SendSyncPhantomData<T> {}
40
41impl<T: ?Sized> Default for SendSyncPhantomData<T> {
42 fn default() -> SendSyncPhantomData<T> {
43 SendSyncPhantomData(PhantomData)
44 }
45}
46
47impl<T: ?Sized> PartialEq for SendSyncPhantomData<T> {
48 fn eq(&self, _other: &Self) -> bool {
49 true
50 }
51}
52
53impl<T: ?Sized> Eq for SendSyncPhantomData<T> {}
54
55impl<T: ?Sized> Clone for SendSyncPhantomData<T> {
56 fn clone(&self) -> Self {
57 SendSyncPhantomData(PhantomData)
58 }
59}
60
61#[cfg(miri)]
62extern "Rust" {
63 /// Miri-provided intrinsic that marks the pointer `ptr` as aligned to
64 /// `align`.
65 ///
66 /// This intrinsic is used to inform Miri's symbolic alignment checker that
67 /// a pointer is aligned, even if Miri cannot statically deduce that fact.
68 /// This is often required when performing raw pointer arithmetic or casts
69 /// where the alignment is guaranteed by runtime checks or invariants that
70 /// Miri is not aware of.
71 pub(crate) fn miri_promise_symbolic_alignment(ptr: *const (), align: usize);
72}
73
74pub(crate) trait AsAddress {
75 fn addr(self) -> usize;
76}
77
78impl<T: ?Sized> AsAddress for &T {
79 #[inline(always)]
80 fn addr(self) -> usize {
81 let ptr: *const T = self;
82 AsAddress::addr(ptr)
83 }
84}
85
86impl<T: ?Sized> AsAddress for &mut T {
87 #[inline(always)]
88 fn addr(self) -> usize {
89 let ptr: *const T = self;
90 AsAddress::addr(ptr)
91 }
92}
93
94impl<T: ?Sized> AsAddress for NonNull<T> {
95 #[inline(always)]
96 fn addr(self) -> usize {
97 AsAddress::addr(self.as_ptr())
98 }
99}
100
101impl<T: ?Sized> AsAddress for *const T {
102 #[inline(always)]
103 fn addr(self) -> usize {
104 // FIXME(#181), FIXME(https://github.com/rust-lang/rust/issues/95228):
105 // Use `.addr()` instead of `as usize` once it's stable, and get rid of
106 // this `allow`. Currently, `as usize` is the only way to accomplish
107 // this.
108 #[allow(clippy::as_conversions)]
109 #[cfg_attr(
110 __ZEROCOPY_INTERNAL_USE_ONLY_NIGHTLY_FEATURES_IN_TESTS,
111 allow(lossy_provenance_casts)
112 )]
113 return self.cast::<()>() as usize;
114 }
115}
116
117impl<T: ?Sized> AsAddress for *mut T {
118 #[inline(always)]
119 fn addr(self) -> usize {
120 let ptr: *const T = self;
121 AsAddress::addr(ptr)
122 }
123}
124
125/// Validates that `t` is aligned to `align_of::<U>()`.
126#[inline(always)]
127pub(crate) fn validate_aligned_to<T: AsAddress, U>(t: T) -> Result<(), AlignmentError<(), U>> {
128 // `mem::align_of::<U>()` is guaranteed to return a non-zero value, which in
129 // turn guarantees that this mod operation will not panic.
130 #[allow(clippy::arithmetic_side_effects)]
131 let remainder = t.addr() % mem::align_of::<U>();
132 if remainder == 0 {
133 Ok(())
134 } else {
135 // SAFETY: We just confirmed that `t.addr() % align_of::<U>() != 0`.
136 // That's only possible if `align_of::<U>() > 1`.
137 Err(unsafe { AlignmentError::new_unchecked(()) })
138 }
139}
140
141/// Returns the bytes needed to pad `len` to the next multiple of `align`.
142///
143/// This function assumes that align is a power of two; there are no guarantees
144/// on the answer it gives if this is not the case.
145#[cfg_attr(
146 kani,
147 kani::requires(len <= DstLayout::MAX_SIZE),
148 kani::requires(align.is_power_of_two()),
149 kani::ensures(|&p| (len + p) % align.get() == 0),
150 // Ensures that we add the minimum required padding.
151 kani::ensures(|&p| p < align.get()),
152)]
153#[cfg_attr(not(zerocopy_inline_always), inline)]
154#[cfg_attr(zerocopy_inline_always, inline(always))]
155pub(crate) const fn padding_needed_for(len: usize, align: NonZeroUsize) -> usize {
156 #[cfg(kani)]
157 #[kani::proof_for_contract(padding_needed_for)]
158 fn proof() {
159 padding_needed_for(kani::any(), kani::any());
160 }
161
162 // Abstractly, we want to compute:
163 // align - (len % align).
164 // Handling the case where len%align is 0.
165 // Because align is a power of two, len % align = len & (align-1).
166 // Guaranteed not to underflow as align is nonzero.
167 #[allow(clippy::arithmetic_side_effects)]
168 let mask = align.get() - 1;
169
170 // To efficiently subtract this value from align, we can use the bitwise
171 // complement.
172 // Note that ((!len) & (align-1)) gives us a number that with (len &
173 // (align-1)) sums to align-1. So subtracting 1 from x before taking the
174 // complement subtracts `len` from `align`. Some quick inspection of
175 // cases shows that this also handles the case where `len % align = 0`
176 // correctly too: len-1 % align then equals align-1, so the complement mod
177 // align will be 0, as desired.
178 //
179 // The following reasoning can be verified quickly by an SMT solver
180 // supporting the theory of bitvectors:
181 // ```smtlib
182 // ; Naive implementation of padding
183 // (define-fun padding1 (
184 // (len (_ BitVec 32))
185 // (align (_ BitVec 32))) (_ BitVec 32)
186 // (ite
187 // (= (_ bv0 32) (bvand len (bvsub align (_ bv1 32))))
188 // (_ bv0 32)
189 // (bvsub align (bvand len (bvsub align (_ bv1 32))))))
190 //
191 // ; The implementation below
192 // (define-fun padding2 (
193 // (len (_ BitVec 32))
194 // (align (_ BitVec 32))) (_ BitVec 32)
195 // (bvand (bvnot (bvsub len (_ bv1 32))) (bvsub align (_ bv1 32))))
196 //
197 // (define-fun is-power-of-two ((x (_ BitVec 32))) Bool
198 // (= (_ bv0 32) (bvand x (bvsub x (_ bv1 32)))))
199 //
200 // (declare-const len (_ BitVec 32))
201 // (declare-const align (_ BitVec 32))
202 // ; Search for a case where align is a power of two and padding2 disagrees
203 // ; with padding1
204 // (assert (and (is-power-of-two align)
205 // (not (= (padding1 len align) (padding2 len align)))))
206 // (simplify (padding1 (_ bv300 32) (_ bv32 32))) ; 20
207 // (simplify (padding2 (_ bv300 32) (_ bv32 32))) ; 20
208 // (simplify (padding1 (_ bv322 32) (_ bv32 32))) ; 30
209 // (simplify (padding2 (_ bv322 32) (_ bv32 32))) ; 30
210 // (simplify (padding1 (_ bv8 32) (_ bv8 32))) ; 0
211 // (simplify (padding2 (_ bv8 32) (_ bv8 32))) ; 0
212 // (check-sat) ; unsat, also works for 64-bit bitvectors
213 // ```
214 !(len.wrapping_sub(1)) & mask
215}
216
217/// Rounds `n` down to the largest value `m` such that `m <= n` and `m % align
218/// == 0`.
219///
220/// # Panics
221///
222/// May panic if `align` is not a power of two. Even if it doesn't panic in this
223/// case, it will produce nonsense results.
224#[inline(always)]
225#[cfg_attr(
226 kani,
227 kani::requires(align.is_power_of_two()),
228 kani::ensures(|&m| m <= n && m % align.get() == 0),
229 // Guarantees that `m` is the *largest* value such that `m % align == 0`.
230 kani::ensures(|&m| {
231 // If this `checked_add` fails, then the next multiple would wrap
232 // around, which trivially satisfies the "largest value" requirement.
233 m.checked_add(align.get()).map(|next_mul| next_mul > n).unwrap_or(true)
234 })
235)]
236pub(crate) const fn round_down_to_next_multiple_of_alignment(
237 n: usize,
238 align: NonZeroUsize,
239) -> usize {
240 #[cfg(kani)]
241 #[kani::proof_for_contract(round_down_to_next_multiple_of_alignment)]
242 fn proof() {
243 round_down_to_next_multiple_of_alignment(kani::any(), kani::any());
244 }
245
246 let align = align.get();
247 #[cfg(not(no_zerocopy_panic_in_const_and_vec_try_reserve_1_57_0))]
248 debug_assert!(align.is_power_of_two());
249
250 // Subtraction can't underflow because `align.get() >= 1`.
251 #[allow(clippy::arithmetic_side_effects)]
252 let mask = !(align - 1);
253 n & mask
254}
255
256#[cfg_attr(not(zerocopy_inline_always), inline)]
257#[cfg_attr(zerocopy_inline_always, inline(always))]
258pub(crate) const fn max(a: NonZeroUsize, b: NonZeroUsize) -> NonZeroUsize {
259 if a.get() < b.get() {
260 b
261 } else {
262 a
263 }
264}
265
266#[cfg_attr(not(zerocopy_inline_always), inline)]
267#[cfg_attr(zerocopy_inline_always, inline(always))]
268pub(crate) const fn min(a: NonZeroUsize, b: NonZeroUsize) -> NonZeroUsize {
269 if a.get() > b.get() {
270 b
271 } else {
272 a
273 }
274}
275
276/// Copies `src` into the prefix of `dst`.
277///
278/// # Safety
279///
280/// The caller guarantees that `src.len() <= dst.len()`.
281#[inline(always)]
282pub(crate) unsafe fn copy_unchecked(src: &[u8], dst: &mut [u8]) {
283 debug_assert!(src.len() <= dst.len());
284 // SAFETY: This invocation satisfies the safety contract of
285 // copy_nonoverlapping [1]:
286 // - `src.as_ptr()` is trivially valid for reads of `src.len()` bytes
287 // - `dst.as_ptr()` is valid for writes of `src.len()` bytes, because the
288 // caller has promised that `src.len() <= dst.len()`
289 // - `src` and `dst` are, trivially, properly aligned
290 // - the region of memory beginning at `src` with a size of `src.len()`
291 // bytes does not overlap with the region of memory beginning at `dst`
292 // with the same size, because `dst` is derived from an exclusive
293 // reference.
294 unsafe {
295 core::ptr::copy_nonoverlapping(src.as_ptr(), dst.as_mut_ptr(), src.len());
296 };
297}
298
299/// Unsafely transmutes the given `src` into a type `Dst`.
300///
301/// # Safety
302///
303/// The value `src` must be a valid instance of `Dst`.
304#[inline(always)]
305pub(crate) const unsafe fn transmute_unchecked<Src, Dst>(src: Src) -> Dst {
306 static_assert!(Src, Dst => core::mem::size_of::<Src>() == core::mem::size_of::<Dst>());
307
308 #[repr(C)]
309 union Transmute<Src, Dst> {
310 src: ManuallyDrop<Src>,
311 dst: ManuallyDrop<Dst>,
312 }
313
314 // SAFETY: Since `Transmute<Src, Dst>` is `#[repr(C)]`, its `src` and `dst`
315 // fields both start at the same offset and the types of those fields are
316 // transparent wrappers around `Src` and `Dst` [1]. Consequently,
317 // initializing `Transmute` with with `src` and then reading out `dst` is
318 // equivalent to transmuting from `Src` to `Dst` [2]. Transmuting from `src`
319 // to `Dst` is valid because — by contract on the caller — `src` is a valid
320 // instance of `Dst`.
321 //
322 // [1] Per https://doc.rust-lang.org/1.82.0/std/mem/struct.ManuallyDrop.html:
323 //
324 // `ManuallyDrop<T>` is guaranteed to have the same layout and bit
325 // validity as `T`, and is subject to the same layout optimizations as
326 // `T`.
327 //
328 // [2] Per https://doc.rust-lang.org/1.82.0/reference/items/unions.html#reading-and-writing-union-fields:
329 //
330 // Effectively, writing to and then reading from a union with the C
331 // representation is analogous to a transmute from the type used for
332 // writing to the type used for reading.
333 unsafe { ManuallyDrop::into_inner(Transmute { src: ManuallyDrop::new(src) }.dst) }
334}
335
336/// # Safety
337///
338/// `Src` must have a greater or equal alignment to `Dst`.
339pub(crate) unsafe fn transmute_ref<Src, Dst, R>(src: &Src) -> &Dst
340where
341 Src: ?Sized,
342 Dst: SizeEq<Src>
343 + TransmuteFromPtr<Src, Shared, Valid, Valid, <Dst as SizeEq<Src>>::CastFrom, R>
344 + ?Sized,
345{
346 let dst = Ptr::from_ref(src).transmute();
347 // SAFETY: The caller promises that `Src`'s alignment is at least as large
348 // as `Dst`'s alignment.
349 let dst = unsafe { dst.assume_alignment() };
350 dst.as_ref()
351}
352
353/// # Safety
354///
355/// `Src` must have a greater or equal alignment to `Dst`.
356pub(crate) unsafe fn transmute_mut<Src, Dst, R>(src: &mut Src) -> &mut Dst
357where
358 Src: ?Sized,
359 Dst: SizeEq<Src>
360 + TransmuteFromPtr<Src, Exclusive, Valid, Valid, <Dst as SizeEq<Src>>::CastFrom, R>
361 + ?Sized,
362{
363 let dst = Ptr::from_mut(src).transmute();
364 // SAFETY: The caller promises that `Src`'s alignment is at least as large
365 // as `Dst`'s alignment.
366 let dst = unsafe { dst.assume_alignment() };
367 dst.as_mut()
368}
369
370/// Uses `allocate` to create a `Box<T>`.
371///
372/// # Errors
373///
374/// Returns an error on allocation failure. Allocation failure is guaranteed
375/// never to cause a panic or an abort.
376///
377/// # Safety
378///
379/// `allocate` must be either `alloc::alloc::alloc` or
380/// `alloc::alloc::alloc_zeroed`. The referent of the box returned by `new_box`
381/// has the same bit-validity as the referent of the pointer returned by the
382/// given `allocate` and sufficient size to store `T` with `meta`.
383#[must_use = "has no side effects (other than allocation)"]
384#[cfg(feature = "alloc")]
385#[inline]
386pub(crate) unsafe fn new_box<T>(
387 meta: T::PointerMetadata,
388 allocate: unsafe fn(core::alloc::Layout) -> *mut u8,
389) -> Result<alloc::boxed::Box<T>, AllocError>
390where
391 T: ?Sized + crate::KnownLayout,
392{
393 let align = T::LAYOUT.align.get();
394 if !T::is_valid_metadata(meta) {
395 return Err(AllocError);
396 }
397 let size = match T::size_for_metadata(meta) {
398 Some(size) => size,
399 // Thanks to the `!T::is_valid_metadata(meta)` check
400 // above, this branch is unreachable. Fortunately, the
401 // optimizer recognizes this, so replacing this branch
402 // with `unreachable_unchecked` produces no codegen
403 // improvements.
404 None => return Err(AllocError),
405 };
406 let ptr = if size != 0 {
407 // SAFETY:
408 // - `align` is derived from a `NonZeroUsize` and is thus non-zero.
409 // - `align` is a power of two because, by invariant on
410 // `KnownLayout::LAYOUT` `<T as KnownLayout>::LAYOUT` accurately
411 // reflects the layout of `T`.
412 // - `size`, by invariant on `size_for_metadata` is well-aligned for
413 // `align` and, by the check on `T::is_valid_metadata(meta)`, is less
414 // than `isize::MAX`.
415 let layout: Layout = unsafe { Layout::from_size_align_unchecked(size, align) };
416 // SAFETY: By contract on the caller, `allocate` is either
417 // `alloc::alloc::alloc` or `alloc::alloc::alloc_zeroed`. The above
418 // check ensures their shared safety precondition: that the supplied
419 // layout is not zero-sized type [1].
420 //
421 // [1] Per https://doc.rust-lang.org/1.81.0/std/alloc/trait.GlobalAlloc.html#tymethod.alloc:
422 //
423 // This function is unsafe because undefined behavior can result if
424 // the caller does not ensure that layout has non-zero size.
425 let ptr = unsafe { allocate(layout) };
426 match NonNull::new(ptr) {
427 Some(ptr) => ptr,
428 None => return Err(AllocError),
429 }
430 } else {
431 // We use `transmute` instead of an `as` cast since Miri (with strict
432 // provenance enabled) notices and complains that an `as` cast creates a
433 // pointer with no provenance. Miri isn't smart enough to realize that
434 // we're only executing this branch when we're constructing a zero-sized
435 // `Box`, which doesn't require provenance.
436 //
437 // SAFETY: any initialized bit sequence is a bit-valid `*mut u8`. All
438 // bits of a `usize` are initialized.
439 //
440 // `#[allow(unknown_lints)]` is for `integer_to_ptr_transmutes`
441 #[allow(unknown_lints)]
442 #[allow(clippy::useless_transmute, integer_to_ptr_transmutes)]
443 let dangling = unsafe { mem::transmute::<usize, *mut u8>(align) };
444 // SAFETY: `dangling` is constructed from `align`, which is derived from
445 // a `NonZeroUsize`, which is guaranteed to be non-zero.
446 //
447 // `Box<[T]>` does not allocate when `T` is zero-sized or when `len` is
448 // zero, but it does require a non-null dangling pointer for its
449 // allocation.
450 //
451 // FIXME(https://github.com/rust-lang/rust/issues/95228): Use
452 // `std::ptr::without_provenance` once it's stable. That may optimize
453 // better. As written, Rust may assume that this consumes "exposed"
454 // provenance, and thus Rust may have to assume that this may consume
455 // provenance from any pointer whose provenance has been exposed.
456 unsafe { NonNull::new_unchecked(dangling) }
457 };
458
459 let ptr = T::raw_from_ptr_len(ptr, meta);
460
461 // FIXME(#429): Add a "SAFETY" comment and remove this `allow`. Make sure to
462 // include a justification that `ptr.as_ptr()` is validly-aligned in the ZST
463 // case (in which we manually construct a dangling pointer) and to justify
464 // why `Box` is safe to drop (it's because `allocate` uses the system
465 // allocator).
466 #[allow(clippy::undocumented_unsafe_blocks)]
467 Ok(unsafe { alloc::boxed::Box::from_raw(ptr.as_ptr()) })
468}
469
470mod len_of {
471 use super::*;
472
473 /// A witness type for metadata of a valid instance of `&T`.
474 pub struct MetadataOf<T: ?Sized + KnownLayout> {
475 /// # Safety
476 ///
477 /// The size of an instance of `&T` with the given metadata is not
478 /// larger than `isize::MAX`.
479 meta: T::PointerMetadata,
480 _p: PhantomData<T>,
481 }
482
483 impl<T: ?Sized + KnownLayout> Copy for MetadataOf<T> {}
484 impl<T: ?Sized + KnownLayout> Clone for MetadataOf<T> {
485 #[inline]
486 fn clone(&self) -> Self {
487 *self
488 }
489 }
490
491 impl<T: ?Sized + KnownLayout> core::fmt::Debug for MetadataOf<T>
492 where
493 T::PointerMetadata: core::fmt::Debug,
494 {
495 #[inline]
496 fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
497 f.debug_struct("MetadataOf").field("meta", &self.meta).finish()
498 }
499 }
500
501 impl<T: ?Sized> MetadataOf<T>
502 where
503 T: KnownLayout,
504 {
505 /// Returns `None` if `meta` is greater than `t`'s metadata.
506 #[inline(always)]
507 pub(crate) fn new_in_bounds(t: &T, meta: usize) -> Option<Self>
508 where
509 T: KnownLayout<PointerMetadata = usize>,
510 {
511 if meta <= Ptr::from_ref(t).len() {
512 // SAFETY: We have checked that `meta` is not greater than `t`'s
513 // metadata, which, by invariant on `&T`, addresses no more than
514 // `isize::MAX` bytes [1][2].
515 //
516 // [1] Per https://doc.rust-lang.org/1.85.0/std/primitive.reference.html#safety:
517 //
518 // For all types, `T: ?Sized`, and for all `t: &T` or `t:
519 // &mut T`, when such values cross an API boundary, the
520 // following invariants must generally be upheld:
521 //
522 // * `t` is non-null
523 // * `t` is aligned to `align_of_val(t)`
524 // * if `size_of_val(t) > 0`, then `t` is dereferenceable for
525 // `size_of_val(t)` many bytes
526 //
527 // If `t` points at address `a`, being "dereferenceable" for
528 // N bytes means that the memory range `[a, a + N)` is all
529 // contained within a single allocated object.
530 //
531 // [2] Per https://doc.rust-lang.org/1.85.0/std/ptr/index.html#allocated-object:
532 //
533 // For any allocated object with `base` address, `size`, and
534 // a set of `addresses`, the following are guaranteed:
535 // - For all addresses `a` in `addresses`, `a` is in the
536 // range `base .. (base + size)` (note that this requires
537 // `a < base + size`, not `a <= base + size`)
538 // - `base` is not equal to [`null()`] (i.e., the address
539 // with the numerical value 0)
540 // - `base + size <= usize::MAX`
541 // - `size <= isize::MAX`
542 Some(unsafe { Self::new_unchecked(meta) })
543 } else {
544 None
545 }
546 }
547
548 /// # Safety
549 ///
550 /// The size of an instance of `&T` with the given metadata is not
551 /// larger than `isize::MAX`.
552 pub(crate) unsafe fn new_unchecked(meta: T::PointerMetadata) -> Self {
553 // SAFETY: The caller has promised that the size of an instance of
554 // `&T` with the given metadata is not larger than `isize::MAX`.
555 Self { meta, _p: PhantomData }
556 }
557
558 pub(crate) fn get(&self) -> T::PointerMetadata
559 where
560 T::PointerMetadata: Copy,
561 {
562 self.meta
563 }
564
565 #[inline]
566 pub(crate) fn padding_needed_for(&self) -> usize
567 where
568 T: KnownLayout<PointerMetadata = usize>,
569 {
570 let trailing_slice_layout = crate::trailing_slice_layout::<T>();
571
572 // FIXME(#67): Remove this allow. See NumExt for more details.
573 #[allow(
574 unstable_name_collisions,
575 clippy::incompatible_msrv,
576 clippy::multiple_unsafe_ops_per_block
577 )]
578 // SAFETY: By invariant on `self`, a `&T` with metadata `self.meta`
579 // describes an object of size `<= isize::MAX`. This computes the
580 // size of such a `&T` without any trailing padding, and so neither
581 // the multiplication nor the addition will overflow.
582 let unpadded_size = unsafe {
583 let trailing_size = self.meta.unchecked_mul(trailing_slice_layout.elem_size);
584 trailing_size.unchecked_add(trailing_slice_layout.offset)
585 };
586
587 util::padding_needed_for(unpadded_size, T::LAYOUT.align)
588 }
589
590 #[inline(always)]
591 pub(crate) fn validate_cast_and_convert_metadata(
592 addr: usize,
593 bytes_len: MetadataOf<[u8]>,
594 cast_type: CastType,
595 meta: Option<T::PointerMetadata>,
596 ) -> Result<(MetadataOf<T>, MetadataOf<[u8]>), MetadataCastError> {
597 let layout = match meta {
598 None => T::LAYOUT,
599 // This can return `Err(MetadataCastError::Size)` if the
600 // metadata describes an object which can't fit in an `isize`.
601 Some(meta) => {
602 if !T::is_valid_metadata(meta) {
603 return Err(MetadataCastError::Size);
604 }
605 let size = match T::size_for_metadata(meta) {
606 Some(size) => size,
607 // Thanks to the `!T::is_valid_metadata(meta)` check
608 // above, this branch is unreachable. Fortunately, the
609 // optimizer recognizes this, so replacing this branch
610 // with `unreachable_unchecked` produces no codegen
611 // improvements.
612 None => return Err(MetadataCastError::Size),
613 };
614 DstLayout {
615 align: T::LAYOUT.align,
616 size_info: crate::SizeInfo::Sized { size },
617 statically_shallow_unpadded: false,
618 }
619 }
620 };
621 // Lemma 0: By contract on `validate_cast_and_convert_metadata`, if
622 // the result is `Ok(..)`, then a `&T` with `elems` trailing slice
623 // elements is no larger in size than `bytes_len.get()`.
624 let (elems, split_at) =
625 layout.validate_cast_and_convert_metadata(addr, bytes_len.get(), cast_type)?;
626 let elems = T::PointerMetadata::from_elem_count(elems);
627
628 // For a slice DST type, if `meta` is `Some(elems)`, then we
629 // synthesize `layout` to describe a sized type whose size is equal
630 // to the size of the instance that we are asked to cast. For sized
631 // types, `validate_cast_and_convert_metadata` returns `elems == 0`.
632 // Thus, in this case, we need to use the `elems` passed by the
633 // caller, not the one returned by
634 // `validate_cast_and_convert_metadata`.
635 //
636 // Lemma 1: A `&T` with `elems` trailing slice elements is no larger
637 // in size than `bytes_len.get()`. Proof:
638 // - If `meta` is `None`, then `elems` satisfies this condition by
639 // Lemma 0.
640 // - If `meta` is `Some(meta)`, then `layout` describes an object
641 // whose size is equal to the size of an `&T` with `meta`
642 // metadata. By Lemma 0, that size is not larger than
643 // `bytes_len.get()`.
644 //
645 // Lemma 2: A `&T` with `elems` trailing slice elements is no larger
646 // than `isize::MAX` bytes. Proof: By Lemma 1, a `&T` with metadata
647 // `elems` is not larger in size than `bytes_len.get()`. By
648 // invariant on `MetadataOf<[u8]>`, a `&[u8]` with metadata
649 // `bytes_len` is not larger than `isize::MAX`. Because
650 // `size_of::<u8>()` is `1`, a `&[u8]` with metadata `bytes_len` has
651 // size `bytes_len.get()` bytes. Therefore, a `&T` with metadata
652 // `elems` has size not larger than `isize::MAX`.
653 let elems = meta.unwrap_or(elems);
654
655 // SAFETY: See Lemma 2.
656 let elems = unsafe { MetadataOf::new_unchecked(elems) };
657
658 // SAFETY: Let `size` be the size of a `&T` with metadata `elems`.
659 // By post-condition on `validate_cast_and_convert_metadata`, one of
660 // the following conditions holds:
661 // - `split_at == size`, in which case, by Lemma 2, `split_at <=
662 // isize::MAX`. Since `size_of::<u8>() == 1`, a `[u8]` with
663 // `split_at` elems has size not larger than `isize::MAX`.
664 // - `split_at == bytes_len - size`. Since `bytes_len:
665 // MetadataOf<u8>`, and since `size` is non-negative, `split_at`
666 // addresses no more bytes than `bytes_len` does. Since
667 // `bytes_len: MetadataOf<u8>`, `bytes_len` describes a `[u8]`
668 // which has no more than `isize::MAX` bytes, and thus so does
669 // `split_at`.
670 let split_at = unsafe { MetadataOf::<[u8]>::new_unchecked(split_at) };
671 Ok((elems, split_at))
672 }
673 }
674}
675
676pub use len_of::MetadataOf;
677
678/// Since we support multiple versions of Rust, there are often features which
679/// have been stabilized in the most recent stable release which do not yet
680/// exist (stably) on our MSRV. This module provides polyfills for those
681/// features so that we can write more "modern" code, and just remove the
682/// polyfill once our MSRV supports the corresponding feature. Without this,
683/// we'd have to write worse/more verbose code and leave FIXME comments
684/// sprinkled throughout the codebase to update to the new pattern once it's
685/// stabilized.
686///
687/// Each trait is imported as `_` at the crate root; each polyfill should "just
688/// work" at usage sites.
689pub(crate) mod polyfills {
690 use core::ptr::{self, NonNull};
691
692 // A polyfill for `NonNull::slice_from_raw_parts` that we can use before our
693 // MSRV is 1.70, when that function was stabilized.
694 //
695 // The `#[allow(unused)]` is necessary because, on sufficiently recent
696 // toolchain versions, `ptr.slice_from_raw_parts()` resolves to the inherent
697 // method rather than to this trait, and so this trait is considered unused.
698 //
699 // FIXME(#67): Once our MSRV is 1.70, remove this.
700 #[allow(unused)]
701 pub(crate) trait NonNullExt<T> {
702 fn slice_from_raw_parts(data: Self, len: usize) -> NonNull<[T]>;
703 }
704
705 impl<T> NonNullExt<T> for NonNull<T> {
706 // NOTE on coverage: this will never be tested in nightly since it's a
707 // polyfill for a feature which has been stabilized on our nightly
708 // toolchain.
709 #[cfg_attr(
710 all(coverage_nightly, __ZEROCOPY_INTERNAL_USE_ONLY_NIGHTLY_FEATURES_IN_TESTS),
711 coverage(off)
712 )]
713 #[inline(always)]
714 fn slice_from_raw_parts(data: Self, len: usize) -> NonNull<[T]> {
715 let ptr = ptr::slice_from_raw_parts_mut(data.as_ptr(), len);
716 // SAFETY: `ptr` is converted from `data`, which is non-null.
717 unsafe { NonNull::new_unchecked(ptr) }
718 }
719 }
720
721 // A polyfill for `Self::unchecked_sub` that we can use until methods like
722 // `usize::unchecked_sub` is stabilized.
723 //
724 // The `#[allow(unused)]` is necessary because, on sufficiently recent
725 // toolchain versions, `ptr.slice_from_raw_parts()` resolves to the inherent
726 // method rather than to this trait, and so this trait is considered unused.
727 //
728 // FIXME(#67): Once our MSRV is high enough, remove this.
729 #[allow(unused)]
730 pub(crate) trait NumExt {
731 /// Add without checking for overflow.
732 ///
733 /// # Safety
734 ///
735 /// The caller promises that the addition will not overflow.
736 unsafe fn unchecked_add(self, rhs: Self) -> Self;
737
738 /// Subtract without checking for underflow.
739 ///
740 /// # Safety
741 ///
742 /// The caller promises that the subtraction will not underflow.
743 unsafe fn unchecked_sub(self, rhs: Self) -> Self;
744
745 /// Multiply without checking for overflow.
746 ///
747 /// # Safety
748 ///
749 /// The caller promises that the multiplication will not overflow.
750 unsafe fn unchecked_mul(self, rhs: Self) -> Self;
751 }
752
753 // NOTE on coverage: these will never be tested in nightly since they're
754 // polyfills for a feature which has been stabilized on our nightly
755 // toolchain.
756 impl NumExt for usize {
757 #[cfg_attr(
758 all(coverage_nightly, __ZEROCOPY_INTERNAL_USE_ONLY_NIGHTLY_FEATURES_IN_TESTS),
759 coverage(off)
760 )]
761 #[inline(always)]
762 unsafe fn unchecked_add(self, rhs: usize) -> usize {
763 match self.checked_add(rhs) {
764 Some(x) => x,
765 None => {
766 // SAFETY: The caller promises that the addition will not
767 // underflow.
768 unsafe { core::hint::unreachable_unchecked() }
769 }
770 }
771 }
772
773 #[cfg_attr(
774 all(coverage_nightly, __ZEROCOPY_INTERNAL_USE_ONLY_NIGHTLY_FEATURES_IN_TESTS),
775 coverage(off)
776 )]
777 #[inline(always)]
778 unsafe fn unchecked_sub(self, rhs: usize) -> usize {
779 match self.checked_sub(rhs) {
780 Some(x) => x,
781 None => {
782 // SAFETY: The caller promises that the subtraction will not
783 // underflow.
784 unsafe { core::hint::unreachable_unchecked() }
785 }
786 }
787 }
788
789 #[cfg_attr(
790 all(coverage_nightly, __ZEROCOPY_INTERNAL_USE_ONLY_NIGHTLY_FEATURES_IN_TESTS),
791 coverage(off)
792 )]
793 #[inline(always)]
794 unsafe fn unchecked_mul(self, rhs: usize) -> usize {
795 match self.checked_mul(rhs) {
796 Some(x) => x,
797 None => {
798 // SAFETY: The caller promises that the multiplication will
799 // not overflow.
800 unsafe { core::hint::unreachable_unchecked() }
801 }
802 }
803 }
804 }
805}
806
807#[cfg(test)]
808pub(crate) mod testutil {
809 use crate::*;
810
811 /// A `T` which is aligned to at least `align_of::<A>()`.
812 #[derive(Default)]
813 pub(crate) struct Align<T, A> {
814 pub(crate) t: T,
815 _a: [A; 0],
816 }
817
818 impl<T: Default, A> Align<T, A> {
819 pub(crate) fn set_default(&mut self) {
820 self.t = T::default();
821 }
822 }
823
824 impl<T, A> Align<T, A> {
825 pub(crate) const fn new(t: T) -> Align<T, A> {
826 Align { t, _a: [] }
827 }
828 }
829
830 /// A `T` which is guaranteed not to satisfy `align_of::<A>()`.
831 ///
832 /// It must be the case that `align_of::<T>() < align_of::<A>()` in order
833 /// for this type to work properly.
834 #[repr(C)]
835 pub(crate) struct ForceUnalign<T: Unaligned, A> {
836 // The outer struct is aligned to `A`, and, thanks to `repr(C)`, `t` is
837 // placed at the minimum offset that guarantees its alignment. If
838 // `align_of::<T>() < align_of::<A>()`, then that offset will be
839 // guaranteed *not* to satisfy `align_of::<A>()`.
840 //
841 // Note that we need `T: Unaligned` in order to guarantee that there is
842 // no padding between `_u` and `t`.
843 _u: u8,
844 pub(crate) t: T,
845 _a: [A; 0],
846 }
847
848 impl<T: Unaligned, A> ForceUnalign<T, A> {
849 pub(crate) fn new(t: T) -> ForceUnalign<T, A> {
850 ForceUnalign { _u: 0, t, _a: [] }
851 }
852 }
853 // A `u64` with alignment 8.
854 //
855 // Though `u64` has alignment 8 on some platforms, it's not guaranteed. By
856 // contrast, `AU64` is guaranteed to have alignment 8 on all platforms.
857 #[derive(
858 KnownLayout,
859 Immutable,
860 FromBytes,
861 IntoBytes,
862 Eq,
863 PartialEq,
864 Ord,
865 PartialOrd,
866 Default,
867 Debug,
868 Copy,
869 Clone,
870 )]
871 #[repr(C, align(8))]
872 pub(crate) struct AU64(pub(crate) u64);
873
874 impl AU64 {
875 // Converts this `AU64` to bytes using this platform's endianness.
876 pub(crate) fn to_bytes(self) -> [u8; 8] {
877 crate::transmute!(self)
878 }
879 }
880
881 impl Display for AU64 {
882 #[cfg_attr(
883 all(coverage_nightly, __ZEROCOPY_INTERNAL_USE_ONLY_NIGHTLY_FEATURES_IN_TESTS),
884 coverage(off)
885 )]
886 fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
887 Display::fmt(&self.0, f)
888 }
889 }
890}
891
892#[cfg(test)]
893mod tests {
894 use super::*;
895
896 #[test]
897 fn test_round_down_to_next_multiple_of_alignment() {
898 fn alt_impl(n: usize, align: NonZeroUsize) -> usize {
899 let mul = n / align.get();
900 mul * align.get()
901 }
902
903 for align in [1, 2, 4, 8, 16] {
904 for n in 0..256 {
905 let align = NonZeroUsize::new(align).unwrap();
906 let want = alt_impl(n, align);
907 let got = round_down_to_next_multiple_of_alignment(n, align);
908 assert_eq!(got, want, "round_down_to_next_multiple_of_alignment({}, {})", n, align);
909 }
910 }
911 }
912
913 #[rustversion::since(1.57.0)]
914 #[test]
915 #[should_panic]
916 fn test_round_down_to_next_multiple_of_alignment_zerocopy_panic_in_const_and_vec_try_reserve() {
917 round_down_to_next_multiple_of_alignment(0, NonZeroUsize::new(3).unwrap());
918 }
919 #[test]
920 fn test_send_sync_phantom_data() {
921 let x = SendSyncPhantomData::<u8>::default();
922 let y = x.clone();
923 assert!(x == y);
924 assert!(x == SendSyncPhantomData::<u8>::default());
925 }
926
927 #[test]
928 #[allow(clippy::as_conversions)]
929 fn test_as_address() {
930 let x = 0u8;
931 let r = &x;
932 let mut x_mut = 0u8;
933 let rm = &mut x_mut;
934 let p = r as *const u8;
935 let pm = rm as *mut u8;
936 let nn = NonNull::new(p as *mut u8).unwrap();
937
938 assert_eq!(AsAddress::addr(r), p as usize);
939 assert_eq!(AsAddress::addr(rm), pm as usize);
940 assert_eq!(AsAddress::addr(p), p as usize);
941 assert_eq!(AsAddress::addr(pm), pm as usize);
942 assert_eq!(AsAddress::addr(nn), p as usize);
943 }
944}