Skip to main content

strided_view/
raw.rs

1//! Borrowed raw strided layout types.
2//!
3//! These are the prepared-replay counterparts to [`crate::StridedView`] and
4//! [`crate::StridedViewMut`]. They borrow shape/stride metadata instead of
5//! owning it, so compiled kernels can reuse already-validated layout
6//! descriptors without rebuilding dynamic-rank view wrappers.
7
8use crate::element_op::Identity;
9use crate::view::validate_bounds;
10use core::marker::PhantomData;
11use core::mem::MaybeUninit;
12use core::ptr::NonNull;
13use num_complex::{Complex32, Complex64};
14
15mod private {
16    pub trait Sealed {}
17}
18
19/// A scalar type that can safely witness storage for an erased kernel view.
20pub trait KernelStorageElement: private::Sealed + Copy + 'static {
21    const DTYPE: KernelDType;
22}
23
24macro_rules! kernel_storage_element {
25    ($($ty:ty => $dtype:ident),* $(,)?) => {$ (
26        impl private::Sealed for $ty {}
27        impl KernelStorageElement for $ty {
28            const DTYPE: KernelDType = KernelDType::$dtype;
29        }
30    )* };
31}
32
33kernel_storage_element! {
34    f32 => F32,
35    f64 => F64,
36    i32 => I32,
37    i64 => I64,
38    bool => Bool,
39    Complex32 => C32,
40    Complex64 => C64,
41}
42
43use crate::{Result, StridedError, StridedView, StridedViewMut};
44
45/// Dtypes supported by dtype-erased kernel entry points.
46///
47/// The enum is intentionally limited to the scalar set currently used by the
48/// tensor runtime callers. Later FFI layers should map their ABI dtype tags to
49/// this enum before dispatching into prepared kernels.
50#[non_exhaustive]
51#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]
52#[repr(u8)]
53pub enum KernelDType {
54    F32 = 1,
55    F64 = 2,
56    I32 = 3,
57    I64 = 4,
58    Bool = 5,
59    C32 = 6,
60    C64 = 7,
61}
62
63impl KernelDType {
64    #[inline]
65    pub const fn label(self) -> &'static str {
66        match self {
67            Self::F32 => "f32",
68            Self::F64 => "f64",
69            Self::I32 => "i32",
70            Self::I64 => "i64",
71            Self::Bool => "bool",
72            Self::C32 => "c32",
73            Self::C64 => "c64",
74        }
75    }
76
77    #[inline]
78    pub const fn size_of(self) -> usize {
79        match self {
80            Self::F32 => core::mem::size_of::<f32>(),
81            Self::F64 => core::mem::size_of::<f64>(),
82            Self::I32 => core::mem::size_of::<i32>(),
83            Self::I64 => core::mem::size_of::<i64>(),
84            Self::Bool => core::mem::size_of::<bool>(),
85            Self::C32 => core::mem::size_of::<Complex32>(),
86            Self::C64 => core::mem::size_of::<Complex64>(),
87        }
88    }
89
90    #[inline]
91    pub const fn alignment(self) -> usize {
92        match self {
93            Self::F32 => core::mem::align_of::<f32>(),
94            Self::F64 => core::mem::align_of::<f64>(),
95            Self::I32 => core::mem::align_of::<i32>(),
96            Self::I64 => core::mem::align_of::<i64>(),
97            Self::Bool => core::mem::align_of::<bool>(),
98            Self::C32 => core::mem::align_of::<Complex32>(),
99            Self::C64 => core::mem::align_of::<Complex64>(),
100        }
101    }
102
103    #[inline]
104    pub const fn requires_valid_byte_values(self) -> bool {
105        matches!(self, Self::Bool)
106    }
107}
108
109fn validate_erased_buffer(dtype: KernelDType, data: &[u8]) -> Result<usize> {
110    let element_count = validate_erased_buffer_extent(dtype, data)?;
111    if element_count != 0 && dtype.requires_valid_byte_values() {
112        validate_bool_bytes(data)?;
113    }
114    Ok(element_count)
115}
116
117/// Validate the length and alignment of erased storage, without reading it.
118///
119/// Typed constructors call this alone: `KernelStorageElement` is sealed, so a
120/// `&[T]` already holds valid values of `T::DTYPE`.
121fn validate_erased_buffer_extent(dtype: KernelDType, data: &[u8]) -> Result<usize> {
122    let element_size = dtype.size_of();
123    if !data.len().is_multiple_of(element_size) {
124        return Err(StridedError::ByteLengthMismatch {
125            dtype: dtype.label(),
126            byte_len: data.len(),
127            element_size,
128        });
129    }
130
131    let element_count = data.len() / element_size;
132    // A zero-length extent still has to be aligned: `data_as` casts this base
133    // to `T`, and `slice::from_raw_parts` requires an aligned pointer even for
134    // length 0, so a misaligned empty buffer is rejected here rather than
135    // reaching the cast.
136    let alignment = dtype.alignment();
137    if !(data.as_ptr() as usize).is_multiple_of(alignment) {
138        return Err(StridedError::DataAlignmentMismatch {
139            dtype: dtype.label(),
140            alignment,
141        });
142    }
143    Ok(element_count)
144}
145
146/// Reject any byte other than 0 or 1.
147///
148/// The OR fold has no early exit, so it vectorizes and runs at memory
149/// bandwidth; a byte by byte `find` does not, and cost more than the select
150/// kernel it guarded. The slow scan runs only to report the offending byte.
151fn validate_bool_bytes(data: &[u8]) -> Result<()> {
152    if data.iter().fold(0u8, |acc, &value| acc | value) <= 1 {
153        return Ok(());
154    }
155    match data.iter().find(|&&value| value > 1) {
156        Some(&value) => Err(StridedError::InvalidBoolByte { value }),
157        None => Ok(()),
158    }
159}
160
161fn validate_erased_buffer_layout(
162    dtype: KernelDType,
163    data: NonNull<u8>,
164    byte_len: usize,
165) -> Result<usize> {
166    let element_size = dtype.size_of();
167    if !byte_len.is_multiple_of(element_size) {
168        return Err(StridedError::ByteLengthMismatch {
169            dtype: dtype.label(),
170            byte_len,
171            element_size,
172        });
173    }
174    if !(data.as_ptr() as usize).is_multiple_of(dtype.alignment()) {
175        return Err(StridedError::DataAlignmentMismatch {
176            dtype: dtype.label(),
177            alignment: dtype.alignment(),
178        });
179    }
180    Ok(byte_len / element_size)
181}
182
183/// Pointer-backed dtype-erased input used by one-shot write entry points.
184///
185/// Unlike [`ErasedRawStridedRef`], this descriptor does not create a shared
186/// Rust reference at construction time. That lets the entry point reject an
187/// input/output overlap before forming references to either side.
188#[derive(Clone, Copy, Debug)]
189pub struct ErasedRawStridedPtr<'a> {
190    dtype: KernelDType,
191    data: NonNull<u8>,
192    byte_len: usize,
193    dims: &'a [usize],
194    strides: &'a [isize],
195    offset: isize,
196    marker: PhantomData<&'a [MaybeUninit<u8>]>,
197}
198
199impl<'a> ErasedRawStridedPtr<'a> {
200    /// Create a pointer-backed descriptor from erased storage.
201    ///
202    /// # Safety
203    ///
204    /// `data` must point into an allocation whose alignment is suitable for
205    /// `dtype`, independently of the observed address. The allocation must
206    /// provide `byte_len` initialized, readable bytes with valid provenance
207    /// for `'a`, and remain alive for `'a`. For `bool`, the bytes may contain
208    /// invalid values temporarily. They must not be read or used to form a
209    /// typed reference until a caller has rejected overlap and
210    /// [`Self::try_as_ref_after_no_overlap`] has validated the complete extent.
211    /// The allocation may overlap a destination.
212    /// The original owner may perform synchronized sequential mutation through
213    /// the same-provenance raw pointer before conversion. No concurrent
214    /// mutation or conflicting access is permitted during overlap checking,
215    /// conversion, or consumer access.
216    ///
217    /// # Errors
218    ///
219    /// Returns [`StridedError::ByteLengthMismatch`] when `byte_len` is not
220    /// a whole number of `dtype` elements, and
221    /// [`StridedError::DataAlignmentMismatch`] when `data` is
222    /// not aligned for `dtype`. Returns
223    /// [`StridedError::StrideLengthMismatch`] when `dims` and `strides`
224    /// have different lengths, and [`StridedError::OffsetOverflow`] when
225    /// a reachable offset is not representable: `dim - 1` of a non-unit
226    /// extent does not fit `isize`, a stride times that last index or the
227    /// accumulated offsets overflow `isize` or go negative, or the highest
228    /// reachable offset is not below the storage length. A layout with a zero
229    /// extent skips this check.
230    /// Bool bytes are not checked here;
231    /// [`Self::try_as_ref_after_no_overlap`] does that.
232    pub unsafe fn from_raw_parts(
233        dtype: KernelDType,
234        data: NonNull<u8>,
235        byte_len: usize,
236        dims: &'a [usize],
237        strides: &'a [isize],
238        offset: isize,
239    ) -> Result<Self> {
240        let element_count = validate_erased_buffer_layout(dtype, data, byte_len)?;
241        validate_bounds(element_count, dims, strides, offset)?;
242        Ok(Self {
243            dtype,
244            data,
245            byte_len,
246            dims,
247            strides,
248            offset,
249            marker: PhantomData,
250        })
251    }
252
253    /// Borrow a safe erased input as a pointer descriptor.
254    pub fn from_ref(input: &ErasedRawStridedRef<'a>) -> Self {
255        Self {
256            dtype: input.dtype,
257            data: NonNull::new(input.data.as_ptr().cast_mut()).unwrap_or_else(NonNull::dangling),
258            byte_len: input.data.len(),
259            dims: input.dims,
260            strides: input.strides,
261            offset: input.offset,
262            marker: PhantomData,
263        }
264    }
265
266    /// Convert this pointer to an initialized descriptor after overlap checks.
267    ///
268    /// # Safety
269    ///
270    /// The caller must have proved that no mutable allocation overlaps this
271    /// pointer's complete byte extent.
272    /// The allocation contract from [`Self::from_raw_parts`] must still hold.
273    /// No mutation or other conflicting access may occur during this conversion
274    /// or for the duration of the returned descriptor's consumer access.
275    /// Bool bytes are validated here, immediately before typed access.
276    ///
277    /// # Errors
278    ///
279    /// Returns [`StridedError::InvalidBoolByte`] when `dtype` is `Bool` and
280    /// a byte of the extent is neither 0 nor 1. It repeats the extent
281    /// length, alignment, stride length and bounds checks
282    /// ([`StridedError::ByteLengthMismatch`],
283    /// [`StridedError::DataAlignmentMismatch`],
284    /// [`StridedError::StrideLengthMismatch`],
285    /// [`StridedError::OffsetOverflow`]) that [`Self::from_raw_parts`] and
286    /// [`Self::from_ref`] already applied to the same values, so those
287    /// cannot fire again here.
288    pub unsafe fn try_as_ref_after_no_overlap(&self) -> Result<ErasedRawStridedRef<'_>> {
289        // SAFETY: this function's contract keeps the `from_raw_parts` contract
290        // (raw.rs:204) in force: `data` (a `NonNull`) heads `byte_len`
291        // initialized, readable bytes alive for `'a`, and the caller has ruled
292        // out a mutable overlap and any concurrent access, so a shared `u8`
293        // slice over them is not aliased by a writer. `u8` accepts every byte
294        // value.
295        let bytes = core::slice::from_raw_parts(self.data.as_ptr(), self.byte_len);
296        validate_erased_buffer(self.dtype, bytes)?;
297        // SAFETY: `ErasedRawStridedRef::from_raw_parts` (raw.rs:451) needs the
298        // start of an allocation aligned for `dtype`, `byte_len` initialized
299        // bytes for `'a`, valid `dtype` values, and no mutable alias. Extent and
300        // lifetime are this pointer's own `from_raw_parts` contract
301        // (raw.rs:204); bool values were just checked by `validate_erased_buffer`
302        // (raw.rs:109) via `validate_bool_bytes` (raw.rs:151); the absence of a
303        // mutable alias is this function's contract. The result borrows `self`,
304        // so it cannot outlive `'a`. The "start of an allocation" wording is
305        // NOT met: the pointer contract (raw.rs:204) only asks that `data`
306        // point *into* an aligned allocation, so an interior pointer is
307        // forwarded as is. What the `Ref` relies on is narrower: `data` aligned
308        // for `dtype` with `byte_len` readable bytes. For a non-empty buffer
309        // that holds, because `validate_erased_buffer_extent` (raw.rs:121)
310        // checked the address; the mismatch is then in the contract text, not
311        // UB. For `byte_len == 0` both validators check alignment too
312        // (raw.rs:137, raw.rs:174; #434), so a misaligned interior pointer is
313        // rejected on the way through instead of being forwarded; what is left
314        // is the wording difference above, not an unsound state.
315        ErasedRawStridedRef::from_raw_parts(
316            self.dtype,
317            self.data,
318            self.byte_len,
319            self.dims,
320            self.strides,
321            self.offset,
322        )
323    }
324
325    #[inline]
326    pub fn dtype(&self) -> KernelDType {
327        self.dtype
328    }
329
330    /// Check overlap with initialized mutable storage without reading it.
331    ///
332    /// # Errors
333    ///
334    /// Never returns `Err` in practice: both byte extents come from valid
335    /// descriptors, whose contracts already guarantee an allocation of that
336    /// size, so the end-address arithmetic cannot wrap. The `Result` only
337    /// carries that arithmetic's [`StridedError::OffsetOverflow`].
338    pub fn overlaps_mut(&self, dest: &ErasedRawStridedMut<'_>) -> Result<bool> {
339        ranges_overlap(
340            self.data.as_ptr() as usize,
341            self.byte_len,
342            dest.data.as_ptr() as usize,
343            dest.data.len(),
344        )
345    }
346
347    /// Check overlap with uninitialized mutable storage without reading it.
348    ///
349    /// # Errors
350    ///
351    /// Never returns `Err` in practice: both byte extents come from valid
352    /// descriptors, whose contracts already guarantee an allocation of that
353    /// size, so the end-address arithmetic cannot wrap. The `Result` only
354    /// carries that arithmetic's [`StridedError::OffsetOverflow`].
355    pub fn overlaps_uninit_mut(&self, dest: &ErasedRawStridedUninitMut<'_>) -> Result<bool> {
356        ranges_overlap(
357            self.data.as_ptr() as usize,
358            self.byte_len,
359            dest.data.as_ptr() as usize,
360            dest.data.len(),
361        )
362    }
363
364    #[inline]
365    pub fn dims(&self) -> &'a [usize] {
366        self.dims
367    }
368
369    #[inline]
370    pub fn strides(&self) -> &'a [isize] {
371        self.strides
372    }
373
374    #[inline]
375    pub fn offset(&self) -> isize {
376        self.offset
377    }
378}
379
380/// Borrowed dtype-erased raw strided input layout.
381///
382/// `dims`, `strides`, and `offset` are expressed in dtype elements, not bytes.
383#[derive(Clone, Copy)]
384pub struct ErasedRawStridedRef<'a> {
385    dtype: KernelDType,
386    data: &'a [u8],
387    dims: &'a [usize],
388    strides: &'a [isize],
389    offset: isize,
390}
391
392// A derived `Debug` would print every byte of the backing buffer; the layout
393// metadata is what a reader wants.
394impl core::fmt::Debug for ErasedRawStridedRef<'_> {
395    fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
396        f.debug_struct("ErasedRawStridedRef")
397            .field("dtype", &self.dtype)
398            .field("byte_len", &self.data.len())
399            .field("dims", &self.dims)
400            .field("strides", &self.strides)
401            .field("offset", &self.offset)
402            .finish()
403    }
404}
405
406impl<'a> ErasedRawStridedRef<'a> {
407    /// Create an erased input descriptor from typed, initialized storage.
408    ///
409    /// # Errors
410    ///
411    /// Returns [`StridedError::StrideLengthMismatch`] when `dims` and
412    /// `strides` have different lengths, and
413    /// [`StridedError::OffsetOverflow`] when a reachable offset is not
414    /// representable: `dim - 1` of a non-unit extent does not fit `isize`, a
415    /// stride times that last index or the accumulated offsets overflow
416    /// `isize` or go negative, or the highest reachable offset is not below
417    /// the buffer length. A layout with a zero extent skips this check. The
418    /// byte-length and alignment checks
419    /// cannot fail for a typed `&[T]`.
420    pub fn from_slice<T: KernelStorageElement>(
421        data: &'a [T],
422        dims: &'a [usize],
423        strides: &'a [isize],
424        offset: isize,
425    ) -> Result<Self> {
426        let dtype = T::DTYPE;
427        let byte_len = data
428            .len()
429            .checked_mul(core::mem::size_of::<T>())
430            .ok_or(StridedError::OffsetOverflow)?;
431        // SAFETY: `data` is a live `&[T]` and `byte_len` is
432        // `data.len() * size_of::<T>()` from the `checked_mul` above, so the
433        // `u8` view spans exactly that initialized allocation for `data`'s
434        // lifetime. `u8` has no invalid bit patterns, and the resulting length
435        // is re-checked by `validate_erased_buffer_extent` (raw.rs:121).
436        let bytes = unsafe { core::slice::from_raw_parts(data.as_ptr().cast::<u8>(), byte_len) };
437        let element_count = validate_erased_buffer_extent(dtype, bytes)?;
438        validate_bounds(element_count, dims, strides, offset)?;
439        Ok(Self {
440            dtype,
441            data: bytes,
442            dims,
443            strides,
444            offset,
445        })
446    }
447
448    /// Construct from erased storage.
449    ///
450    /// # Safety
451    /// `data` must be the start of an allocation aligned for `dtype`; the
452    /// allocation must contain `byte_len` initialized, readable bytes for
453    /// `'a`, and every byte extent must represent valid values for `dtype`.
454    /// The allocation and metadata must outlive `'a`; no mutable alias may
455    /// exist while this descriptor is used. Alignment is an allocation
456    /// property and must not be inferred from the observed address alone.
457    ///
458    /// # Errors
459    ///
460    /// Returns [`StridedError::ByteLengthMismatch`] when `byte_len` is not
461    /// a whole number of `dtype` elements, and
462    /// [`StridedError::DataAlignmentMismatch`] when `data` is
463    /// not aligned for `dtype`. Returns
464    /// [`StridedError::StrideLengthMismatch`] when `dims` and `strides`
465    /// have different lengths, and [`StridedError::OffsetOverflow`] when
466    /// a reachable offset is not representable: `dim - 1` of a non-unit
467    /// extent does not fit `isize`, a stride times that last index or the
468    /// accumulated offsets overflow `isize` or go negative, or the highest
469    /// reachable offset is not below the storage length. A layout with a zero
470    /// extent skips this check.
471    /// Bool bytes are the caller's responsibility here, per the safety
472    /// contract.
473    pub unsafe fn from_raw_parts(
474        dtype: KernelDType,
475        data: NonNull<u8>,
476        byte_len: usize,
477        dims: &'a [usize],
478        strides: &'a [isize],
479        offset: isize,
480    ) -> Result<Self> {
481        let count = validate_erased_buffer_layout(dtype, data, byte_len)?;
482        // SAFETY: `data` is a `NonNull`, and this function's contract
483        // (raw.rs:451) makes `data..data + byte_len` one live allocation of
484        // initialized bytes for `'a` with no mutable alias, which also bounds
485        // `byte_len` by `isize::MAX`. `u8` needs no alignment beyond 1.
486        let bytes = core::slice::from_raw_parts(data.as_ptr(), byte_len);
487        validate_bounds(count, dims, strides, offset)?;
488        Ok(Self {
489            dtype,
490            data: bytes,
491            dims,
492            strides,
493            offset,
494        })
495    }
496
497    /// Borrow the initialized storage as its concrete scalar type.
498    ///
499    /// # Errors
500    ///
501    /// Returns [`StridedError::DTypeMismatch`] when `T` is not the
502    /// descriptor's dtype.
503    pub fn data_as<T: KernelStorageElement>(&self) -> Result<&[T]> {
504        if self.dtype != T::DTYPE {
505            return Err(StridedError::DTypeMismatch {
506                expected: T::DTYPE.label(),
507                actual: self.dtype.label(),
508            });
509        }
510        // SAFETY: the dtype check above proves the stored dtype is `T`, and the
511        // constructor ran `validate_erased_buffer_extent` (raw.rs:121) /
512        // `validate_erased_buffer_layout` (raw.rs:161): the byte length is a
513        // whole multiple of `size_of::<T>()` and the base is `T`-aligned, so
514        // `len / size_of::<T>()` initialized `T` values are in bounds. Both
515        // checks verify alignment unconditionally, including `byte_len == 0`
516        // (raw.rs:137, raw.rs:174; #434), so a misaligned base never reaches
517        // this cast, whether the buffer is empty or not. The `&[T]` origin
518        // (`from_slice`, raw.rs:420) is aligned by the type system, and a
519        // direct `from_raw_parts` caller owes the start of an aligned
520        // allocation (raw.rs:451). A zero-length slice requires only a
521        // non-null aligned pointer, so a dangling aligned base is accepted
522        // here, and so is an aligned interior base; only misaligned ones are rejected.
523        Ok(unsafe {
524            core::slice::from_raw_parts(
525                self.data.as_ptr().cast::<T>(),
526                self.data.len() / core::mem::size_of::<T>(),
527            )
528        })
529    }
530
531    #[inline]
532    pub fn dtype(&self) -> KernelDType {
533        self.dtype
534    }
535
536    #[inline]
537    pub fn dims(&self) -> &'a [usize] {
538        self.dims
539    }
540
541    #[inline]
542    pub fn strides(&self) -> &'a [isize] {
543        self.strides
544    }
545
546    #[inline]
547    pub fn offset(&self) -> isize {
548        self.offset
549    }
550}
551
552/// Borrowed dtype-erased raw strided output layout.
553///
554/// `dims`, `strides`, and `offset` are expressed in dtype elements, not bytes.
555pub struct ErasedRawStridedMut<'a> {
556    dtype: KernelDType,
557    data: &'a mut [u8],
558    dims: &'a [usize],
559    strides: &'a [isize],
560    offset: isize,
561}
562
563// A derived `Debug` would print every byte of the backing buffer; the layout
564// metadata is what a reader wants.
565impl core::fmt::Debug for ErasedRawStridedMut<'_> {
566    fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
567        f.debug_struct("ErasedRawStridedMut")
568            .field("dtype", &self.dtype)
569            .field("byte_len", &self.data.len())
570            .field("dims", &self.dims)
571            .field("strides", &self.strides)
572            .field("offset", &self.offset)
573            .finish()
574    }
575}
576
577impl<'a> ErasedRawStridedMut<'a> {
578    /// Create an erased output descriptor from typed, initialized storage.
579    ///
580    /// # Errors
581    ///
582    /// Returns [`StridedError::StrideLengthMismatch`] when `dims` and
583    /// `strides` have different lengths, and
584    /// [`StridedError::OffsetOverflow`] when a reachable offset is not
585    /// representable: `dim - 1` of a non-unit extent does not fit `isize`, a
586    /// stride times that last index or the accumulated offsets overflow
587    /// `isize` or go negative, or the highest reachable offset is not below
588    /// the buffer length. A layout with a zero extent skips this check. The
589    /// byte-length and alignment checks
590    /// cannot fail for a typed `&mut [T]`.
591    pub fn from_slice_mut<T: KernelStorageElement>(
592        data: &'a mut [T],
593        dims: &'a [usize],
594        strides: &'a [isize],
595        offset: isize,
596    ) -> Result<Self> {
597        let dtype = T::DTYPE;
598        let byte_len = data
599            .len()
600            .checked_mul(core::mem::size_of::<T>())
601            .ok_or(StridedError::OffsetOverflow)?;
602        // SAFETY: `data` is a live, exclusively borrowed `&mut [T]` and
603        // `byte_len` is `data.len() * size_of::<T>()` from the `checked_mul`
604        // above, so the `u8` view spans exactly that initialized allocation for
605        // `'a`. `u8` has no invalid bit patterns, and the resulting length is
606        // re-checked by `validate_erased_buffer_extent` (raw.rs:121).
607        let data =
608            unsafe { core::slice::from_raw_parts_mut(data.as_mut_ptr().cast::<u8>(), byte_len) };
609        let element_count = validate_erased_buffer_extent(dtype, data)?;
610        validate_bounds(element_count, dims, strides, offset)?;
611        Ok(Self {
612            dtype,
613            data,
614            dims,
615            strides,
616            offset,
617        })
618    }
619
620    /// Construct from erased storage.
621    ///
622    /// # Safety
623    /// `data..data + byte_len` must be an aligned, writable allocation valid
624    /// for `'a`, with valid initialized values for `dtype` throughout the
625    /// complete byte extent; its provenance, extent, lifetime, and exclusive
626    /// aliasing must be upheld by the caller.
627    /// The alignment requirement is on the allocation, not merely the observed
628    /// address. The caller must retain exclusive access: no mutable alias or
629    /// concurrent mutation may exist while this descriptor is used.
630    ///
631    /// # Errors
632    ///
633    /// Returns [`StridedError::ByteLengthMismatch`] when `byte_len` is not
634    /// a whole number of `dtype` elements, and
635    /// [`StridedError::DataAlignmentMismatch`] when `data` is
636    /// not aligned for `dtype`. Returns
637    /// [`StridedError::InvalidBoolByte`] when `dtype` is `Bool` and a byte
638    /// of the extent is neither 0 nor 1. Returns
639    /// [`StridedError::StrideLengthMismatch`] when `dims` and `strides`
640    /// have different lengths, and [`StridedError::OffsetOverflow`] when
641    /// a reachable offset is not representable: `dim - 1` of a non-unit
642    /// extent does not fit `isize`, a stride times that last index or the
643    /// accumulated offsets overflow `isize` or go negative, or the highest
644    /// reachable offset is not below the storage length. A layout with a zero
645    /// extent skips this check.
646    pub unsafe fn from_raw_parts(
647        dtype: KernelDType,
648        data: NonNull<u8>,
649        byte_len: usize,
650        dims: &'a [usize],
651        strides: &'a [isize],
652        offset: isize,
653    ) -> Result<Self> {
654        let count = validate_erased_buffer_layout(dtype, data, byte_len)?;
655        // SAFETY: `data` is a `NonNull`, and this function's contract
656        // (raw.rs:623) makes `data..data + byte_len` one live, writable,
657        // initialized allocation for `'a` held exclusively by the caller, which
658        // also bounds `byte_len` by `isize::MAX`. `u8` needs no alignment
659        // beyond 1.
660        let data = core::slice::from_raw_parts_mut(data.as_ptr(), byte_len);
661        validate_erased_buffer(dtype, data)?;
662        validate_bounds(count, dims, strides, offset)?;
663        Ok(Self {
664            dtype,
665            data,
666            dims,
667            strides,
668            offset,
669        })
670    }
671
672    /// Borrow initialized storage as its concrete scalar type.
673    ///
674    /// # Errors
675    ///
676    /// Returns [`StridedError::DTypeMismatch`] when `T` is not the
677    /// descriptor's dtype.
678    pub fn data_as<T: KernelStorageElement>(&self) -> Result<&[T]> {
679        if self.dtype != T::DTYPE {
680            return Err(StridedError::DTypeMismatch {
681                expected: T::DTYPE.label(),
682                actual: self.dtype.label(),
683            });
684        }
685        // SAFETY: the dtype check above proves the stored dtype is `T`, and the
686        // constructor ran `validate_erased_buffer_extent` (raw.rs:121) /
687        // `validate_erased_buffer_layout` (raw.rs:161): the byte length is a
688        // whole multiple of `size_of::<T>()` and the base is `T`-aligned, so
689        // `len / size_of::<T>()` initialized `T` values are in bounds. Both
690        // checks verify alignment unconditionally, including `byte_len == 0`
691        // (raw.rs:137, raw.rs:174; #434), so the base is `T`-aligned whether or
692        // not the buffer is empty. That is the whole requirement here: the
693        // `&mut [T]` (`from_slice_mut`, raw.rs:591) and the `from_raw_parts`
694        // contract (raw.rs:623) make `data..data + byte_len` the writable
695        // allocation, and nothing in this crate builds it from an interior pointer.
696        Ok(unsafe {
697            core::slice::from_raw_parts(
698                self.data.as_ptr().cast::<T>(),
699                self.data.len() / core::mem::size_of::<T>(),
700            )
701        })
702    }
703
704    /// Borrow initialized storage mutably as its concrete scalar type.
705    ///
706    /// # Errors
707    ///
708    /// Returns [`StridedError::DTypeMismatch`] when `T` is not the
709    /// descriptor's dtype.
710    pub fn data_as_mut<T: KernelStorageElement>(&mut self) -> Result<&mut [T]> {
711        if self.dtype != T::DTYPE {
712            return Err(StridedError::DTypeMismatch {
713                expected: T::DTYPE.label(),
714                actual: self.dtype.label(),
715            });
716        }
717        // SAFETY: as `data_as` (raw.rs:678) — the dtype check proves `T`, and the
718        // constructor's extent/layout check (raw.rs:161), or for an empty
719        // buffer the `&mut [T]` origin / `from_raw_parts` contract (aligned
720        // start, no interior-pointer path), makes the base `T`-aligned with a
721        // whole multiple of `size_of::<T>()` bytes.
722        // `&mut self` keeps this reborrow exclusive for its own lifetime.
723        Ok(unsafe {
724            core::slice::from_raw_parts_mut(
725                self.data.as_mut_ptr().cast::<T>(),
726                self.data.len() / core::mem::size_of::<T>(),
727            )
728        })
729    }
730
731    #[inline]
732    pub fn dtype(&self) -> KernelDType {
733        self.dtype
734    }
735
736    #[inline]
737    pub fn dims(&self) -> &'a [usize] {
738        self.dims
739    }
740
741    #[inline]
742    pub fn strides(&self) -> &'a [isize] {
743        self.strides
744    }
745
746    #[inline]
747    pub fn offset(&self) -> isize {
748        self.offset
749    }
750}
751
752/// Borrowed dtype-erased raw strided output whose reachable elements may be
753/// uninitialized.
754///
755/// This descriptor is only accepted by operations that prove and perform a
756/// full overwrite of every reachable logical destination element. The backing
757/// allocation may contain non-reachable holes, which remain uninitialized.
758pub struct ErasedRawStridedUninitMut<'a> {
759    dtype: KernelDType,
760    data: &'a mut [MaybeUninit<u8>],
761    dims: &'a [usize],
762    strides: &'a [isize],
763    offset: isize,
764}
765
766// A derived `Debug` would print one entry per byte of the backing buffer; the
767// layout metadata is what a reader wants.
768impl core::fmt::Debug for ErasedRawStridedUninitMut<'_> {
769    fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
770        f.debug_struct("ErasedRawStridedUninitMut")
771            .field("dtype", &self.dtype)
772            .field("byte_len", &self.data.len())
773            .field("dims", &self.dims)
774            .field("strides", &self.strides)
775            .field("offset", &self.offset)
776            .finish()
777    }
778}
779
780impl<'a> ErasedRawStridedUninitMut<'a> {
781    /// Create an uninitialized erased output descriptor from typed storage.
782    ///
783    /// # Errors
784    ///
785    /// Returns [`StridedError::StrideLengthMismatch`] when `dims` and
786    /// `strides` have different lengths, and
787    /// [`StridedError::OffsetOverflow`] when a reachable offset is not
788    /// representable: `dim - 1` of a non-unit extent does not fit `isize`, a
789    /// stride times that last index or the accumulated offsets overflow
790    /// `isize` or go negative, or the highest reachable offset is not below
791    /// the buffer length. A layout with a zero extent skips this check. The
792    /// byte-length and alignment checks
793    /// cannot fail for a typed `&mut [MaybeUninit<T>]`.
794    pub fn from_uninit_slice<T: KernelStorageElement>(
795        data: &'a mut [MaybeUninit<T>],
796        dims: &'a [usize],
797        strides: &'a [isize],
798        offset: isize,
799    ) -> Result<Self> {
800        let dtype = T::DTYPE;
801        let byte_len = data
802            .len()
803            .checked_mul(core::mem::size_of::<T>())
804            .ok_or(StridedError::OffsetOverflow)?;
805        // SAFETY: `data` is a live, exclusively borrowed `&mut
806        // [MaybeUninit<T>]` and `byte_len` is `data.len() * size_of::<T>()` from
807        // the `checked_mul` above, so the view spans exactly that allocation for
808        // `'a`. `MaybeUninit<u8>` accepts every bit pattern, so no
809        // initialization is asserted here.
810        let data = unsafe {
811            core::slice::from_raw_parts_mut(data.as_mut_ptr().cast::<MaybeUninit<u8>>(), byte_len)
812        };
813        let data_ptr =
814            NonNull::new(data.as_mut_ptr().cast::<u8>()).unwrap_or_else(NonNull::dangling);
815        let element_count = validate_erased_buffer_layout(dtype, data_ptr, byte_len)?;
816        validate_bounds(element_count, dims, strides, offset)?;
817        Ok(Self {
818            dtype,
819            data,
820            dims,
821            strides,
822            offset,
823        })
824    }
825
826    /// Construct from erased uninitialized storage.
827    ///
828    /// # Safety
829    /// `data..data + byte_len` must be an aligned, writable allocation valid
830    /// for `'a`, with provenance and extent sufficient for all reachable
831    /// elements. The caller must ensure every reachable element is completely
832    /// overwritten before it is read or exposed as initialized storage. The
833    /// allocation's alignment is an allocation property independent of the
834    /// observed address, and the descriptor must have exclusive access with no
835    /// mutable alias or concurrent mutation during use.
836    ///
837    /// # Errors
838    ///
839    /// Returns [`StridedError::ByteLengthMismatch`] when `byte_len` is not
840    /// a whole number of `dtype` elements, and
841    /// [`StridedError::DataAlignmentMismatch`] when `data` is
842    /// not aligned for `dtype`. Returns
843    /// [`StridedError::StrideLengthMismatch`] when `dims` and `strides`
844    /// have different lengths, and [`StridedError::OffsetOverflow`] when
845    /// a reachable offset is not representable: `dim - 1` of a non-unit
846    /// extent does not fit `isize`, a stride times that last index or the
847    /// accumulated offsets overflow `isize` or go negative, or the highest
848    /// reachable offset is not below the storage length. A layout with a zero
849    /// extent skips this check.
850    pub unsafe fn from_raw_parts(
851        dtype: KernelDType,
852        data: NonNull<u8>,
853        byte_len: usize,
854        dims: &'a [usize],
855        strides: &'a [isize],
856        offset: isize,
857    ) -> Result<Self> {
858        let count = validate_erased_buffer_layout(dtype, data, byte_len)?;
859        // SAFETY: `data` is a `NonNull`, and this function's contract
860        // (raw.rs:826) makes `data..data + byte_len` one live, writable
861        // allocation for `'a` held exclusively by the caller, which also bounds
862        // `byte_len` by `isize::MAX`. `MaybeUninit<u8>` accepts every bit
863        // pattern, so no initialization is asserted.
864        let data =
865            core::slice::from_raw_parts_mut(data.as_ptr().cast::<MaybeUninit<u8>>(), byte_len);
866        validate_bounds(count, dims, strides, offset)?;
867        Ok(Self {
868            dtype,
869            data,
870            dims,
871            strides,
872            offset,
873        })
874    }
875
876    /// Borrow the uninitialized storage as its concrete scalar type.
877    ///
878    /// # Errors
879    ///
880    /// Returns [`StridedError::DTypeMismatch`] when `T` is not the
881    /// descriptor's dtype.
882    pub fn data_as_uninit_mut<T: KernelStorageElement>(&mut self) -> Result<&mut [MaybeUninit<T>]> {
883        if self.dtype != T::DTYPE {
884            return Err(StridedError::DTypeMismatch {
885                expected: T::DTYPE.label(),
886                actual: self.dtype.label(),
887            });
888        }
889        // SAFETY: the dtype check proves `T`, and
890        // `validate_erased_buffer_layout` (raw.rs:161) proved the base is
891        // `T`-aligned with a whole multiple of `size_of::<T>()` bytes, for an
892        // empty buffer too (raw.rs:137, raw.rs:174; #434): the base comes from
893        // a `&mut [MaybeUninit<T>]` (`from_uninit_slice`, raw.rs:794) or from
894        // `from_raw_parts`, whose contract (raw.rs:826) makes
895        // `data..data + byte_len` the allocation itself. As for `ErasedRawStridedMut`, no
896        // path in this crate builds it from an interior pointer.
897        // `MaybeUninit<T>` accepts every bit pattern, so the partially
898        // initialized extent is not read as `T` here.
899        Ok(unsafe {
900            core::slice::from_raw_parts_mut(
901                self.data.as_mut_ptr().cast::<MaybeUninit<T>>(),
902                self.data.len() / core::mem::size_of::<T>(),
903            )
904        })
905    }
906
907    #[inline]
908    pub fn dtype(&self) -> KernelDType {
909        self.dtype
910    }
911
912    #[inline]
913    pub fn dims(&self) -> &'a [usize] {
914        self.dims
915    }
916
917    #[inline]
918    pub fn strides(&self) -> &'a [isize] {
919        self.strides
920    }
921
922    #[inline]
923    pub fn offset(&self) -> isize {
924        self.offset
925    }
926}
927
928fn ranges_overlap(a_start: usize, a_len: usize, b_start: usize, b_len: usize) -> Result<bool> {
929    let a_end = a_start
930        .checked_add(a_len)
931        .ok_or(StridedError::OffsetOverflow)?;
932    let b_end = b_start
933        .checked_add(b_len)
934        .ok_or(StridedError::OffsetOverflow)?;
935    Ok(a_start < b_end && b_start < a_end)
936}
937
938/// Borrowed raw strided input layout.
939///
940/// Use [`RawStridedRef::new`] for checked construction, or
941/// [`RawStridedRef::new_unchecked`] when a higher-level compiled plan has
942/// already validated bounds.
943#[derive(Clone, Copy)]
944pub struct RawStridedRef<'a, T> {
945    data: &'a [T],
946    dims: &'a [usize],
947    strides: &'a [isize],
948    offset: isize,
949}
950
951// A derived `Debug` would print every element of the backing buffer; the layout
952// metadata is what a reader wants.
953impl<T: core::fmt::Debug> core::fmt::Debug for RawStridedRef<'_, T> {
954    fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
955        f.debug_struct("RawStridedRef")
956            .field("data_len", &self.data.len())
957            .field("dims", &self.dims)
958            .field("strides", &self.strides)
959            .field("offset", &self.offset)
960            .finish()
961    }
962}
963
964impl<'a, T> RawStridedRef<'a, T> {
965    /// Create a raw strided input after validating reachable offsets.
966    ///
967    /// # Errors
968    ///
969    /// Returns [`StridedError::StrideLengthMismatch`] when `dims` and
970    /// `strides` have different lengths, and
971    /// [`StridedError::OffsetOverflow`] when a reachable offset is not
972    /// representable: `dim - 1` of a non-unit extent does not fit `isize`, a
973    /// stride times that last index or the accumulated offsets overflow
974    /// `isize` or go negative, or the highest reachable offset is not below
975    /// the buffer length. A layout with a zero extent skips this check.
976    pub fn new(
977        data: &'a [T],
978        dims: &'a [usize],
979        strides: &'a [isize],
980        offset: isize,
981    ) -> Result<Self> {
982        validate_bounds(data.len(), dims, strides, offset)?;
983        Ok(Self {
984            data,
985            dims,
986            strides,
987            offset,
988        })
989    }
990
991    /// Create a raw strided input without bounds checking.
992    ///
993    /// # Safety
994    /// The caller must ensure every index reachable by `dims`/`strides` from
995    /// `offset` lies inside `data`.
996    pub unsafe fn new_unchecked(
997        data: &'a [T],
998        dims: &'a [usize],
999        strides: &'a [isize],
1000        offset: isize,
1001    ) -> Self {
1002        Self {
1003            data,
1004            dims,
1005            strides,
1006            offset,
1007        }
1008    }
1009
1010    #[inline]
1011    pub fn data(&self) -> &'a [T] {
1012        self.data
1013    }
1014
1015    #[inline]
1016    pub fn dims(&self) -> &'a [usize] {
1017        self.dims
1018    }
1019
1020    #[inline]
1021    pub fn strides(&self) -> &'a [isize] {
1022        self.strides
1023    }
1024
1025    #[inline]
1026    pub fn offset(&self) -> isize {
1027        self.offset
1028    }
1029
1030    #[inline]
1031    pub fn ptr(&self) -> *const T {
1032        if self.dims.contains(&0) {
1033            NonNull::<T>::dangling().as_ptr()
1034        } else {
1035            // SAFETY: `offset` was accepted by `validate_bounds`
1036            // (view.rs:187) in `RawStridedRef::new`, or by the
1037            // caller contract of `new_unchecked`, so `data.as_ptr() + offset`
1038            // stays inside `data`. The empty-layout branch above returns a
1039            // dangling pointer instead of offsetting a pointer no element
1040            // reaches.
1041            unsafe { self.data.as_ptr().offset(self.offset) }
1042        }
1043    }
1044
1045    /// Convert to an immutable view, borrowing this descriptor's metadata.
1046    ///
1047    /// The conversion allocates nothing; hot prepared paths use the raw
1048    /// accessors directly all the same.
1049    #[inline]
1050    pub fn as_view(&self) -> StridedView<'a, T, Identity> {
1051        // SAFETY: `new_unchecked` requires every reachable index to stay inside
1052        // `data`; this descriptor's `offset`/`dims`/`strides` came from
1053        // `validate_bounds` (view.rs:187) in `RawStridedRef::new`,
1054        // or from the same contract on `new_unchecked`.
1055        unsafe { StridedView::new_unchecked(self.data, self.dims, self.strides, self.offset) }
1056    }
1057}
1058
1059/// Borrowed raw strided output layout.
1060///
1061/// This is the mutable counterpart to [`RawStridedRef`]. It avoids allocating
1062/// owned shape/stride metadata in prepared replay paths.
1063pub struct RawStridedMut<'a, T> {
1064    data: &'a mut [T],
1065    dims: &'a [usize],
1066    strides: &'a [isize],
1067    offset: isize,
1068}
1069
1070// A derived `Debug` would print every element of the backing buffer; the layout
1071// metadata is what a reader wants.
1072impl<T: core::fmt::Debug> core::fmt::Debug for RawStridedMut<'_, T> {
1073    fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
1074        f.debug_struct("RawStridedMut")
1075            .field("data_len", &self.data.len())
1076            .field("dims", &self.dims)
1077            .field("strides", &self.strides)
1078            .field("offset", &self.offset)
1079            .finish()
1080    }
1081}
1082
1083impl<'a, T> RawStridedMut<'a, T> {
1084    /// Create a raw strided output after validating reachable offsets.
1085    ///
1086    /// # Errors
1087    ///
1088    /// Returns [`StridedError::StrideLengthMismatch`] when `dims` and
1089    /// `strides` have different lengths, and
1090    /// [`StridedError::OffsetOverflow`] when a reachable offset is not
1091    /// representable: `dim - 1` of a non-unit extent does not fit `isize`, a
1092    /// stride times that last index or the accumulated offsets overflow
1093    /// `isize` or go negative, or the highest reachable offset is not below
1094    /// the buffer length. A layout with a zero extent skips this check.
1095    pub fn new(
1096        data: &'a mut [T],
1097        dims: &'a [usize],
1098        strides: &'a [isize],
1099        offset: isize,
1100    ) -> Result<Self> {
1101        validate_bounds(data.len(), dims, strides, offset)?;
1102        Ok(Self {
1103            data,
1104            dims,
1105            strides,
1106            offset,
1107        })
1108    }
1109
1110    /// Create a raw strided output without bounds checking.
1111    ///
1112    /// # Safety
1113    /// The caller must ensure every index reachable by `dims`/`strides` from
1114    /// `offset` lies inside `data`, and no aliases violate mutable access.
1115    pub unsafe fn new_unchecked(
1116        data: &'a mut [T],
1117        dims: &'a [usize],
1118        strides: &'a [isize],
1119        offset: isize,
1120    ) -> Self {
1121        Self {
1122            data,
1123            dims,
1124            strides,
1125            offset,
1126        }
1127    }
1128
1129    #[inline]
1130    pub fn data(&self) -> &[T] {
1131        self.data
1132    }
1133
1134    #[inline]
1135    pub fn data_mut(&mut self) -> &mut [T] {
1136        self.data
1137    }
1138
1139    #[inline]
1140    pub fn dims(&self) -> &'a [usize] {
1141        self.dims
1142    }
1143
1144    #[inline]
1145    pub fn strides(&self) -> &'a [isize] {
1146        self.strides
1147    }
1148
1149    #[inline]
1150    pub fn offset(&self) -> isize {
1151        self.offset
1152    }
1153
1154    #[inline]
1155    pub fn ptr(&self) -> *const T {
1156        if self.dims.contains(&0) {
1157            NonNull::<T>::dangling().as_ptr()
1158        } else {
1159            // SAFETY: `offset` was accepted by `validate_bounds`
1160            // (view.rs:187) in `RawStridedMut::new`, or by the
1161            // caller contract of `new_unchecked`, so `data.as_ptr() + offset`
1162            // stays inside `data`. The empty-layout branch above returns a
1163            // dangling pointer instead of offsetting a pointer no element
1164            // reaches.
1165            unsafe { self.data.as_ptr().offset(self.offset) }
1166        }
1167    }
1168
1169    #[inline]
1170    pub fn as_mut_ptr(&mut self) -> *mut T {
1171        if self.dims.contains(&0) {
1172            NonNull::<T>::dangling().as_ptr()
1173        } else {
1174            // SAFETY: as `ptr` above — `offset` was accepted by
1175            // `validate_bounds` (view.rs:187) in
1176            // `RawStridedMut::new` or by the `new_unchecked` contract, and
1177            // `&mut self` keeps the write provenance exclusive.
1178            unsafe { self.data.as_mut_ptr().offset(self.offset) }
1179        }
1180    }
1181
1182    /// Convert to an immutable view, borrowing this descriptor's metadata.
1183    ///
1184    /// The conversion allocates nothing; hot prepared paths use the raw
1185    /// accessors directly all the same.
1186    #[inline]
1187    pub fn as_view(&self) -> StridedView<'_, T, Identity> {
1188        // SAFETY: `new_unchecked` requires every reachable index to stay inside
1189        // `data`; this descriptor's metadata came from `validate_bounds`
1190        // (view.rs:187) in `RawStridedMut::new`, or from the
1191        // same contract on `new_unchecked`.
1192        unsafe { StridedView::new_unchecked(self.data, self.dims, self.strides, self.offset) }
1193    }
1194
1195    /// Convert to a mutable view, borrowing this descriptor's metadata.
1196    ///
1197    /// The conversion allocates nothing; hot prepared paths use the raw
1198    /// accessors directly all the same.
1199    #[inline]
1200    pub fn as_view_mut(&mut self) -> StridedViewMut<'_, T> {
1201        // SAFETY: as `as_view` above, and `&mut self` keeps the mutable borrow
1202        // exclusive to the returned view, which `new_unchecked` requires.
1203        unsafe { StridedViewMut::new_unchecked(self.data, self.dims, self.strides, self.offset) }
1204    }
1205}
1206
1207#[cfg(test)]
1208mod tests;