strided_view/raw.rs
1//! Borrowed raw strided layout types.
2//!
3//! These are the prepared-replay counterparts to [`crate::StridedView`] and
4//! [`crate::StridedViewMut`]. They borrow shape/stride metadata instead of
5//! owning it, so compiled kernels can reuse already-validated layout
6//! descriptors without rebuilding dynamic-rank view wrappers.
7
8use crate::element_op::Identity;
9use crate::view::validate_bounds;
10use core::marker::PhantomData;
11use core::mem::MaybeUninit;
12use core::ptr::NonNull;
13use num_complex::{Complex32, Complex64};
14
15mod private {
16 pub trait Sealed {}
17}
18
19/// A scalar type that can safely witness storage for an erased kernel view.
20pub trait KernelStorageElement: private::Sealed + Copy + 'static {
21 const DTYPE: KernelDType;
22}
23
24macro_rules! kernel_storage_element {
25 ($($ty:ty => $dtype:ident),* $(,)?) => {$ (
26 impl private::Sealed for $ty {}
27 impl KernelStorageElement for $ty {
28 const DTYPE: KernelDType = KernelDType::$dtype;
29 }
30 )* };
31}
32
33kernel_storage_element! {
34 f32 => F32,
35 f64 => F64,
36 i32 => I32,
37 i64 => I64,
38 bool => Bool,
39 Complex32 => C32,
40 Complex64 => C64,
41}
42
43use crate::{Result, StridedError, StridedView, StridedViewMut};
44
45/// Dtypes supported by dtype-erased kernel entry points.
46///
47/// The enum is intentionally limited to the scalar set currently used by the
48/// tensor runtime callers. Later FFI layers should map their ABI dtype tags to
49/// this enum before dispatching into prepared kernels.
50#[non_exhaustive]
51#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]
52#[repr(u8)]
53pub enum KernelDType {
54 F32 = 1,
55 F64 = 2,
56 I32 = 3,
57 I64 = 4,
58 Bool = 5,
59 C32 = 6,
60 C64 = 7,
61}
62
63impl KernelDType {
64 #[inline]
65 pub const fn label(self) -> &'static str {
66 match self {
67 Self::F32 => "f32",
68 Self::F64 => "f64",
69 Self::I32 => "i32",
70 Self::I64 => "i64",
71 Self::Bool => "bool",
72 Self::C32 => "c32",
73 Self::C64 => "c64",
74 }
75 }
76
77 #[inline]
78 pub const fn size_of(self) -> usize {
79 match self {
80 Self::F32 => core::mem::size_of::<f32>(),
81 Self::F64 => core::mem::size_of::<f64>(),
82 Self::I32 => core::mem::size_of::<i32>(),
83 Self::I64 => core::mem::size_of::<i64>(),
84 Self::Bool => core::mem::size_of::<bool>(),
85 Self::C32 => core::mem::size_of::<Complex32>(),
86 Self::C64 => core::mem::size_of::<Complex64>(),
87 }
88 }
89
90 #[inline]
91 pub const fn alignment(self) -> usize {
92 match self {
93 Self::F32 => core::mem::align_of::<f32>(),
94 Self::F64 => core::mem::align_of::<f64>(),
95 Self::I32 => core::mem::align_of::<i32>(),
96 Self::I64 => core::mem::align_of::<i64>(),
97 Self::Bool => core::mem::align_of::<bool>(),
98 Self::C32 => core::mem::align_of::<Complex32>(),
99 Self::C64 => core::mem::align_of::<Complex64>(),
100 }
101 }
102
103 #[inline]
104 pub const fn requires_valid_byte_values(self) -> bool {
105 matches!(self, Self::Bool)
106 }
107}
108
109fn validate_erased_buffer(dtype: KernelDType, data: &[u8]) -> Result<usize> {
110 let element_count = validate_erased_buffer_extent(dtype, data)?;
111 if element_count != 0 && dtype.requires_valid_byte_values() {
112 validate_bool_bytes(data)?;
113 }
114 Ok(element_count)
115}
116
117/// Validate the length and alignment of erased storage, without reading it.
118///
119/// Typed constructors call this alone: `KernelStorageElement` is sealed, so a
120/// `&[T]` already holds valid values of `T::DTYPE`.
121fn validate_erased_buffer_extent(dtype: KernelDType, data: &[u8]) -> Result<usize> {
122 let element_size = dtype.size_of();
123 if !data.len().is_multiple_of(element_size) {
124 return Err(StridedError::ByteLengthMismatch {
125 dtype: dtype.label(),
126 byte_len: data.len(),
127 element_size,
128 });
129 }
130
131 let element_count = data.len() / element_size;
132 // A zero-length extent still has to be aligned: `data_as` casts this base
133 // to `T`, and `slice::from_raw_parts` requires an aligned pointer even for
134 // length 0, so a misaligned empty buffer is rejected here rather than
135 // reaching the cast.
136 let alignment = dtype.alignment();
137 if !(data.as_ptr() as usize).is_multiple_of(alignment) {
138 return Err(StridedError::DataAlignmentMismatch {
139 dtype: dtype.label(),
140 alignment,
141 });
142 }
143 Ok(element_count)
144}
145
146/// Reject any byte other than 0 or 1.
147///
148/// The OR fold has no early exit, so it vectorizes and runs at memory
149/// bandwidth; a byte by byte `find` does not, and cost more than the select
150/// kernel it guarded. The slow scan runs only to report the offending byte.
151fn validate_bool_bytes(data: &[u8]) -> Result<()> {
152 if data.iter().fold(0u8, |acc, &value| acc | value) <= 1 {
153 return Ok(());
154 }
155 match data.iter().find(|&&value| value > 1) {
156 Some(&value) => Err(StridedError::InvalidBoolByte { value }),
157 None => Ok(()),
158 }
159}
160
161fn validate_erased_buffer_layout(
162 dtype: KernelDType,
163 data: NonNull<u8>,
164 byte_len: usize,
165) -> Result<usize> {
166 let element_size = dtype.size_of();
167 if !byte_len.is_multiple_of(element_size) {
168 return Err(StridedError::ByteLengthMismatch {
169 dtype: dtype.label(),
170 byte_len,
171 element_size,
172 });
173 }
174 if !(data.as_ptr() as usize).is_multiple_of(dtype.alignment()) {
175 return Err(StridedError::DataAlignmentMismatch {
176 dtype: dtype.label(),
177 alignment: dtype.alignment(),
178 });
179 }
180 Ok(byte_len / element_size)
181}
182
183/// Pointer-backed dtype-erased input used by one-shot write entry points.
184///
185/// Unlike [`ErasedRawStridedRef`], this descriptor does not create a shared
186/// Rust reference at construction time. That lets the entry point reject an
187/// input/output overlap before forming references to either side.
188#[derive(Clone, Copy, Debug)]
189pub struct ErasedRawStridedPtr<'a> {
190 dtype: KernelDType,
191 data: NonNull<u8>,
192 byte_len: usize,
193 dims: &'a [usize],
194 strides: &'a [isize],
195 offset: isize,
196 marker: PhantomData<&'a [MaybeUninit<u8>]>,
197}
198
199impl<'a> ErasedRawStridedPtr<'a> {
200 /// Create a pointer-backed descriptor from erased storage.
201 ///
202 /// # Safety
203 ///
204 /// `data` must point into an allocation whose alignment is suitable for
205 /// `dtype`, independently of the observed address. The allocation must
206 /// provide `byte_len` initialized, readable bytes with valid provenance
207 /// for `'a`, and remain alive for `'a`. For `bool`, the bytes may contain
208 /// invalid values temporarily. They must not be read or used to form a
209 /// typed reference until a caller has rejected overlap and
210 /// [`Self::try_as_ref_after_no_overlap`] has validated the complete extent.
211 /// The allocation may overlap a destination.
212 /// The original owner may perform synchronized sequential mutation through
213 /// the same-provenance raw pointer before conversion. No concurrent
214 /// mutation or conflicting access is permitted during overlap checking,
215 /// conversion, or consumer access.
216 ///
217 /// # Errors
218 ///
219 /// Returns [`StridedError::ByteLengthMismatch`] when `byte_len` is not
220 /// a whole number of `dtype` elements, and
221 /// [`StridedError::DataAlignmentMismatch`] when `data` is
222 /// not aligned for `dtype`. Returns
223 /// [`StridedError::StrideLengthMismatch`] when `dims` and `strides`
224 /// have different lengths, and [`StridedError::OffsetOverflow`] when
225 /// a reachable offset is not representable: `dim - 1` of a non-unit
226 /// extent does not fit `isize`, a stride times that last index or the
227 /// accumulated offsets overflow `isize` or go negative, or the highest
228 /// reachable offset is not below the storage length. A layout with a zero
229 /// extent skips this check.
230 /// Bool bytes are not checked here;
231 /// [`Self::try_as_ref_after_no_overlap`] does that.
232 pub unsafe fn from_raw_parts(
233 dtype: KernelDType,
234 data: NonNull<u8>,
235 byte_len: usize,
236 dims: &'a [usize],
237 strides: &'a [isize],
238 offset: isize,
239 ) -> Result<Self> {
240 let element_count = validate_erased_buffer_layout(dtype, data, byte_len)?;
241 validate_bounds(element_count, dims, strides, offset)?;
242 Ok(Self {
243 dtype,
244 data,
245 byte_len,
246 dims,
247 strides,
248 offset,
249 marker: PhantomData,
250 })
251 }
252
253 /// Borrow a safe erased input as a pointer descriptor.
254 pub fn from_ref(input: &ErasedRawStridedRef<'a>) -> Self {
255 Self {
256 dtype: input.dtype,
257 data: NonNull::new(input.data.as_ptr().cast_mut()).unwrap_or_else(NonNull::dangling),
258 byte_len: input.data.len(),
259 dims: input.dims,
260 strides: input.strides,
261 offset: input.offset,
262 marker: PhantomData,
263 }
264 }
265
266 /// Convert this pointer to an initialized descriptor after overlap checks.
267 ///
268 /// # Safety
269 ///
270 /// The caller must have proved that no mutable allocation overlaps this
271 /// pointer's complete byte extent.
272 /// The allocation contract from [`Self::from_raw_parts`] must still hold.
273 /// No mutation or other conflicting access may occur during this conversion
274 /// or for the duration of the returned descriptor's consumer access.
275 /// Bool bytes are validated here, immediately before typed access.
276 ///
277 /// # Errors
278 ///
279 /// Returns [`StridedError::InvalidBoolByte`] when `dtype` is `Bool` and
280 /// a byte of the extent is neither 0 nor 1. It repeats the extent
281 /// length, alignment, stride length and bounds checks
282 /// ([`StridedError::ByteLengthMismatch`],
283 /// [`StridedError::DataAlignmentMismatch`],
284 /// [`StridedError::StrideLengthMismatch`],
285 /// [`StridedError::OffsetOverflow`]) that [`Self::from_raw_parts`] and
286 /// [`Self::from_ref`] already applied to the same values, so those
287 /// cannot fire again here.
288 pub unsafe fn try_as_ref_after_no_overlap(&self) -> Result<ErasedRawStridedRef<'_>> {
289 // SAFETY: this function's contract keeps the `from_raw_parts` contract
290 // (raw.rs:204) in force: `data` (a `NonNull`) heads `byte_len`
291 // initialized, readable bytes alive for `'a`, and the caller has ruled
292 // out a mutable overlap and any concurrent access, so a shared `u8`
293 // slice over them is not aliased by a writer. `u8` accepts every byte
294 // value.
295 let bytes = core::slice::from_raw_parts(self.data.as_ptr(), self.byte_len);
296 validate_erased_buffer(self.dtype, bytes)?;
297 // SAFETY: `ErasedRawStridedRef::from_raw_parts` (raw.rs:451) needs the
298 // start of an allocation aligned for `dtype`, `byte_len` initialized
299 // bytes for `'a`, valid `dtype` values, and no mutable alias. Extent and
300 // lifetime are this pointer's own `from_raw_parts` contract
301 // (raw.rs:204); bool values were just checked by `validate_erased_buffer`
302 // (raw.rs:109) via `validate_bool_bytes` (raw.rs:151); the absence of a
303 // mutable alias is this function's contract. The result borrows `self`,
304 // so it cannot outlive `'a`. The "start of an allocation" wording is
305 // NOT met: the pointer contract (raw.rs:204) only asks that `data`
306 // point *into* an aligned allocation, so an interior pointer is
307 // forwarded as is. What the `Ref` relies on is narrower: `data` aligned
308 // for `dtype` with `byte_len` readable bytes. For a non-empty buffer
309 // that holds, because `validate_erased_buffer_extent` (raw.rs:121)
310 // checked the address; the mismatch is then in the contract text, not
311 // UB. For `byte_len == 0` both validators check alignment too
312 // (raw.rs:137, raw.rs:174; #434), so a misaligned interior pointer is
313 // rejected on the way through instead of being forwarded; what is left
314 // is the wording difference above, not an unsound state.
315 ErasedRawStridedRef::from_raw_parts(
316 self.dtype,
317 self.data,
318 self.byte_len,
319 self.dims,
320 self.strides,
321 self.offset,
322 )
323 }
324
325 #[inline]
326 pub fn dtype(&self) -> KernelDType {
327 self.dtype
328 }
329
330 /// Check overlap with initialized mutable storage without reading it.
331 ///
332 /// # Errors
333 ///
334 /// Never returns `Err` in practice: both byte extents come from valid
335 /// descriptors, whose contracts already guarantee an allocation of that
336 /// size, so the end-address arithmetic cannot wrap. The `Result` only
337 /// carries that arithmetic's [`StridedError::OffsetOverflow`].
338 pub fn overlaps_mut(&self, dest: &ErasedRawStridedMut<'_>) -> Result<bool> {
339 ranges_overlap(
340 self.data.as_ptr() as usize,
341 self.byte_len,
342 dest.data.as_ptr() as usize,
343 dest.data.len(),
344 )
345 }
346
347 /// Check overlap with uninitialized mutable storage without reading it.
348 ///
349 /// # Errors
350 ///
351 /// Never returns `Err` in practice: both byte extents come from valid
352 /// descriptors, whose contracts already guarantee an allocation of that
353 /// size, so the end-address arithmetic cannot wrap. The `Result` only
354 /// carries that arithmetic's [`StridedError::OffsetOverflow`].
355 pub fn overlaps_uninit_mut(&self, dest: &ErasedRawStridedUninitMut<'_>) -> Result<bool> {
356 ranges_overlap(
357 self.data.as_ptr() as usize,
358 self.byte_len,
359 dest.data.as_ptr() as usize,
360 dest.data.len(),
361 )
362 }
363
364 #[inline]
365 pub fn dims(&self) -> &'a [usize] {
366 self.dims
367 }
368
369 #[inline]
370 pub fn strides(&self) -> &'a [isize] {
371 self.strides
372 }
373
374 #[inline]
375 pub fn offset(&self) -> isize {
376 self.offset
377 }
378}
379
380/// Borrowed dtype-erased raw strided input layout.
381///
382/// `dims`, `strides`, and `offset` are expressed in dtype elements, not bytes.
383#[derive(Clone, Copy)]
384pub struct ErasedRawStridedRef<'a> {
385 dtype: KernelDType,
386 data: &'a [u8],
387 dims: &'a [usize],
388 strides: &'a [isize],
389 offset: isize,
390}
391
392// A derived `Debug` would print every byte of the backing buffer; the layout
393// metadata is what a reader wants.
394impl core::fmt::Debug for ErasedRawStridedRef<'_> {
395 fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
396 f.debug_struct("ErasedRawStridedRef")
397 .field("dtype", &self.dtype)
398 .field("byte_len", &self.data.len())
399 .field("dims", &self.dims)
400 .field("strides", &self.strides)
401 .field("offset", &self.offset)
402 .finish()
403 }
404}
405
406impl<'a> ErasedRawStridedRef<'a> {
407 /// Create an erased input descriptor from typed, initialized storage.
408 ///
409 /// # Errors
410 ///
411 /// Returns [`StridedError::StrideLengthMismatch`] when `dims` and
412 /// `strides` have different lengths, and
413 /// [`StridedError::OffsetOverflow`] when a reachable offset is not
414 /// representable: `dim - 1` of a non-unit extent does not fit `isize`, a
415 /// stride times that last index or the accumulated offsets overflow
416 /// `isize` or go negative, or the highest reachable offset is not below
417 /// the buffer length. A layout with a zero extent skips this check. The
418 /// byte-length and alignment checks
419 /// cannot fail for a typed `&[T]`.
420 pub fn from_slice<T: KernelStorageElement>(
421 data: &'a [T],
422 dims: &'a [usize],
423 strides: &'a [isize],
424 offset: isize,
425 ) -> Result<Self> {
426 let dtype = T::DTYPE;
427 let byte_len = data
428 .len()
429 .checked_mul(core::mem::size_of::<T>())
430 .ok_or(StridedError::OffsetOverflow)?;
431 // SAFETY: `data` is a live `&[T]` and `byte_len` is
432 // `data.len() * size_of::<T>()` from the `checked_mul` above, so the
433 // `u8` view spans exactly that initialized allocation for `data`'s
434 // lifetime. `u8` has no invalid bit patterns, and the resulting length
435 // is re-checked by `validate_erased_buffer_extent` (raw.rs:121).
436 let bytes = unsafe { core::slice::from_raw_parts(data.as_ptr().cast::<u8>(), byte_len) };
437 let element_count = validate_erased_buffer_extent(dtype, bytes)?;
438 validate_bounds(element_count, dims, strides, offset)?;
439 Ok(Self {
440 dtype,
441 data: bytes,
442 dims,
443 strides,
444 offset,
445 })
446 }
447
448 /// Construct from erased storage.
449 ///
450 /// # Safety
451 /// `data` must be the start of an allocation aligned for `dtype`; the
452 /// allocation must contain `byte_len` initialized, readable bytes for
453 /// `'a`, and every byte extent must represent valid values for `dtype`.
454 /// The allocation and metadata must outlive `'a`; no mutable alias may
455 /// exist while this descriptor is used. Alignment is an allocation
456 /// property and must not be inferred from the observed address alone.
457 ///
458 /// # Errors
459 ///
460 /// Returns [`StridedError::ByteLengthMismatch`] when `byte_len` is not
461 /// a whole number of `dtype` elements, and
462 /// [`StridedError::DataAlignmentMismatch`] when `data` is
463 /// not aligned for `dtype`. Returns
464 /// [`StridedError::StrideLengthMismatch`] when `dims` and `strides`
465 /// have different lengths, and [`StridedError::OffsetOverflow`] when
466 /// a reachable offset is not representable: `dim - 1` of a non-unit
467 /// extent does not fit `isize`, a stride times that last index or the
468 /// accumulated offsets overflow `isize` or go negative, or the highest
469 /// reachable offset is not below the storage length. A layout with a zero
470 /// extent skips this check.
471 /// Bool bytes are the caller's responsibility here, per the safety
472 /// contract.
473 pub unsafe fn from_raw_parts(
474 dtype: KernelDType,
475 data: NonNull<u8>,
476 byte_len: usize,
477 dims: &'a [usize],
478 strides: &'a [isize],
479 offset: isize,
480 ) -> Result<Self> {
481 let count = validate_erased_buffer_layout(dtype, data, byte_len)?;
482 // SAFETY: `data` is a `NonNull`, and this function's contract
483 // (raw.rs:451) makes `data..data + byte_len` one live allocation of
484 // initialized bytes for `'a` with no mutable alias, which also bounds
485 // `byte_len` by `isize::MAX`. `u8` needs no alignment beyond 1.
486 let bytes = core::slice::from_raw_parts(data.as_ptr(), byte_len);
487 validate_bounds(count, dims, strides, offset)?;
488 Ok(Self {
489 dtype,
490 data: bytes,
491 dims,
492 strides,
493 offset,
494 })
495 }
496
497 /// Borrow the initialized storage as its concrete scalar type.
498 ///
499 /// # Errors
500 ///
501 /// Returns [`StridedError::DTypeMismatch`] when `T` is not the
502 /// descriptor's dtype.
503 pub fn data_as<T: KernelStorageElement>(&self) -> Result<&[T]> {
504 if self.dtype != T::DTYPE {
505 return Err(StridedError::DTypeMismatch {
506 expected: T::DTYPE.label(),
507 actual: self.dtype.label(),
508 });
509 }
510 // SAFETY: the dtype check above proves the stored dtype is `T`, and the
511 // constructor ran `validate_erased_buffer_extent` (raw.rs:121) /
512 // `validate_erased_buffer_layout` (raw.rs:161): the byte length is a
513 // whole multiple of `size_of::<T>()` and the base is `T`-aligned, so
514 // `len / size_of::<T>()` initialized `T` values are in bounds. Both
515 // checks verify alignment unconditionally, including `byte_len == 0`
516 // (raw.rs:137, raw.rs:174; #434), so a misaligned base never reaches
517 // this cast, whether the buffer is empty or not. The `&[T]` origin
518 // (`from_slice`, raw.rs:420) is aligned by the type system, and a
519 // direct `from_raw_parts` caller owes the start of an aligned
520 // allocation (raw.rs:451). A zero-length slice requires only a
521 // non-null aligned pointer, so a dangling aligned base is accepted
522 // here, and so is an aligned interior base; only misaligned ones are rejected.
523 Ok(unsafe {
524 core::slice::from_raw_parts(
525 self.data.as_ptr().cast::<T>(),
526 self.data.len() / core::mem::size_of::<T>(),
527 )
528 })
529 }
530
531 #[inline]
532 pub fn dtype(&self) -> KernelDType {
533 self.dtype
534 }
535
536 #[inline]
537 pub fn dims(&self) -> &'a [usize] {
538 self.dims
539 }
540
541 #[inline]
542 pub fn strides(&self) -> &'a [isize] {
543 self.strides
544 }
545
546 #[inline]
547 pub fn offset(&self) -> isize {
548 self.offset
549 }
550}
551
552/// Borrowed dtype-erased raw strided output layout.
553///
554/// `dims`, `strides`, and `offset` are expressed in dtype elements, not bytes.
555pub struct ErasedRawStridedMut<'a> {
556 dtype: KernelDType,
557 data: &'a mut [u8],
558 dims: &'a [usize],
559 strides: &'a [isize],
560 offset: isize,
561}
562
563// A derived `Debug` would print every byte of the backing buffer; the layout
564// metadata is what a reader wants.
565impl core::fmt::Debug for ErasedRawStridedMut<'_> {
566 fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
567 f.debug_struct("ErasedRawStridedMut")
568 .field("dtype", &self.dtype)
569 .field("byte_len", &self.data.len())
570 .field("dims", &self.dims)
571 .field("strides", &self.strides)
572 .field("offset", &self.offset)
573 .finish()
574 }
575}
576
577impl<'a> ErasedRawStridedMut<'a> {
578 /// Create an erased output descriptor from typed, initialized storage.
579 ///
580 /// # Errors
581 ///
582 /// Returns [`StridedError::StrideLengthMismatch`] when `dims` and
583 /// `strides` have different lengths, and
584 /// [`StridedError::OffsetOverflow`] when a reachable offset is not
585 /// representable: `dim - 1` of a non-unit extent does not fit `isize`, a
586 /// stride times that last index or the accumulated offsets overflow
587 /// `isize` or go negative, or the highest reachable offset is not below
588 /// the buffer length. A layout with a zero extent skips this check. The
589 /// byte-length and alignment checks
590 /// cannot fail for a typed `&mut [T]`.
591 pub fn from_slice_mut<T: KernelStorageElement>(
592 data: &'a mut [T],
593 dims: &'a [usize],
594 strides: &'a [isize],
595 offset: isize,
596 ) -> Result<Self> {
597 let dtype = T::DTYPE;
598 let byte_len = data
599 .len()
600 .checked_mul(core::mem::size_of::<T>())
601 .ok_or(StridedError::OffsetOverflow)?;
602 // SAFETY: `data` is a live, exclusively borrowed `&mut [T]` and
603 // `byte_len` is `data.len() * size_of::<T>()` from the `checked_mul`
604 // above, so the `u8` view spans exactly that initialized allocation for
605 // `'a`. `u8` has no invalid bit patterns, and the resulting length is
606 // re-checked by `validate_erased_buffer_extent` (raw.rs:121).
607 let data =
608 unsafe { core::slice::from_raw_parts_mut(data.as_mut_ptr().cast::<u8>(), byte_len) };
609 let element_count = validate_erased_buffer_extent(dtype, data)?;
610 validate_bounds(element_count, dims, strides, offset)?;
611 Ok(Self {
612 dtype,
613 data,
614 dims,
615 strides,
616 offset,
617 })
618 }
619
620 /// Construct from erased storage.
621 ///
622 /// # Safety
623 /// `data..data + byte_len` must be an aligned, writable allocation valid
624 /// for `'a`, with valid initialized values for `dtype` throughout the
625 /// complete byte extent; its provenance, extent, lifetime, and exclusive
626 /// aliasing must be upheld by the caller.
627 /// The alignment requirement is on the allocation, not merely the observed
628 /// address. The caller must retain exclusive access: no mutable alias or
629 /// concurrent mutation may exist while this descriptor is used.
630 ///
631 /// # Errors
632 ///
633 /// Returns [`StridedError::ByteLengthMismatch`] when `byte_len` is not
634 /// a whole number of `dtype` elements, and
635 /// [`StridedError::DataAlignmentMismatch`] when `data` is
636 /// not aligned for `dtype`. Returns
637 /// [`StridedError::InvalidBoolByte`] when `dtype` is `Bool` and a byte
638 /// of the extent is neither 0 nor 1. Returns
639 /// [`StridedError::StrideLengthMismatch`] when `dims` and `strides`
640 /// have different lengths, and [`StridedError::OffsetOverflow`] when
641 /// a reachable offset is not representable: `dim - 1` of a non-unit
642 /// extent does not fit `isize`, a stride times that last index or the
643 /// accumulated offsets overflow `isize` or go negative, or the highest
644 /// reachable offset is not below the storage length. A layout with a zero
645 /// extent skips this check.
646 pub unsafe fn from_raw_parts(
647 dtype: KernelDType,
648 data: NonNull<u8>,
649 byte_len: usize,
650 dims: &'a [usize],
651 strides: &'a [isize],
652 offset: isize,
653 ) -> Result<Self> {
654 let count = validate_erased_buffer_layout(dtype, data, byte_len)?;
655 // SAFETY: `data` is a `NonNull`, and this function's contract
656 // (raw.rs:623) makes `data..data + byte_len` one live, writable,
657 // initialized allocation for `'a` held exclusively by the caller, which
658 // also bounds `byte_len` by `isize::MAX`. `u8` needs no alignment
659 // beyond 1.
660 let data = core::slice::from_raw_parts_mut(data.as_ptr(), byte_len);
661 validate_erased_buffer(dtype, data)?;
662 validate_bounds(count, dims, strides, offset)?;
663 Ok(Self {
664 dtype,
665 data,
666 dims,
667 strides,
668 offset,
669 })
670 }
671
672 /// Borrow initialized storage as its concrete scalar type.
673 ///
674 /// # Errors
675 ///
676 /// Returns [`StridedError::DTypeMismatch`] when `T` is not the
677 /// descriptor's dtype.
678 pub fn data_as<T: KernelStorageElement>(&self) -> Result<&[T]> {
679 if self.dtype != T::DTYPE {
680 return Err(StridedError::DTypeMismatch {
681 expected: T::DTYPE.label(),
682 actual: self.dtype.label(),
683 });
684 }
685 // SAFETY: the dtype check above proves the stored dtype is `T`, and the
686 // constructor ran `validate_erased_buffer_extent` (raw.rs:121) /
687 // `validate_erased_buffer_layout` (raw.rs:161): the byte length is a
688 // whole multiple of `size_of::<T>()` and the base is `T`-aligned, so
689 // `len / size_of::<T>()` initialized `T` values are in bounds. Both
690 // checks verify alignment unconditionally, including `byte_len == 0`
691 // (raw.rs:137, raw.rs:174; #434), so the base is `T`-aligned whether or
692 // not the buffer is empty. That is the whole requirement here: the
693 // `&mut [T]` (`from_slice_mut`, raw.rs:591) and the `from_raw_parts`
694 // contract (raw.rs:623) make `data..data + byte_len` the writable
695 // allocation, and nothing in this crate builds it from an interior pointer.
696 Ok(unsafe {
697 core::slice::from_raw_parts(
698 self.data.as_ptr().cast::<T>(),
699 self.data.len() / core::mem::size_of::<T>(),
700 )
701 })
702 }
703
704 /// Borrow initialized storage mutably as its concrete scalar type.
705 ///
706 /// # Errors
707 ///
708 /// Returns [`StridedError::DTypeMismatch`] when `T` is not the
709 /// descriptor's dtype.
710 pub fn data_as_mut<T: KernelStorageElement>(&mut self) -> Result<&mut [T]> {
711 if self.dtype != T::DTYPE {
712 return Err(StridedError::DTypeMismatch {
713 expected: T::DTYPE.label(),
714 actual: self.dtype.label(),
715 });
716 }
717 // SAFETY: as `data_as` (raw.rs:678) — the dtype check proves `T`, and the
718 // constructor's extent/layout check (raw.rs:161), or for an empty
719 // buffer the `&mut [T]` origin / `from_raw_parts` contract (aligned
720 // start, no interior-pointer path), makes the base `T`-aligned with a
721 // whole multiple of `size_of::<T>()` bytes.
722 // `&mut self` keeps this reborrow exclusive for its own lifetime.
723 Ok(unsafe {
724 core::slice::from_raw_parts_mut(
725 self.data.as_mut_ptr().cast::<T>(),
726 self.data.len() / core::mem::size_of::<T>(),
727 )
728 })
729 }
730
731 #[inline]
732 pub fn dtype(&self) -> KernelDType {
733 self.dtype
734 }
735
736 #[inline]
737 pub fn dims(&self) -> &'a [usize] {
738 self.dims
739 }
740
741 #[inline]
742 pub fn strides(&self) -> &'a [isize] {
743 self.strides
744 }
745
746 #[inline]
747 pub fn offset(&self) -> isize {
748 self.offset
749 }
750}
751
752/// Borrowed dtype-erased raw strided output whose reachable elements may be
753/// uninitialized.
754///
755/// This descriptor is only accepted by operations that prove and perform a
756/// full overwrite of every reachable logical destination element. The backing
757/// allocation may contain non-reachable holes, which remain uninitialized.
758pub struct ErasedRawStridedUninitMut<'a> {
759 dtype: KernelDType,
760 data: &'a mut [MaybeUninit<u8>],
761 dims: &'a [usize],
762 strides: &'a [isize],
763 offset: isize,
764}
765
766// A derived `Debug` would print one entry per byte of the backing buffer; the
767// layout metadata is what a reader wants.
768impl core::fmt::Debug for ErasedRawStridedUninitMut<'_> {
769 fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
770 f.debug_struct("ErasedRawStridedUninitMut")
771 .field("dtype", &self.dtype)
772 .field("byte_len", &self.data.len())
773 .field("dims", &self.dims)
774 .field("strides", &self.strides)
775 .field("offset", &self.offset)
776 .finish()
777 }
778}
779
780impl<'a> ErasedRawStridedUninitMut<'a> {
781 /// Create an uninitialized erased output descriptor from typed storage.
782 ///
783 /// # Errors
784 ///
785 /// Returns [`StridedError::StrideLengthMismatch`] when `dims` and
786 /// `strides` have different lengths, and
787 /// [`StridedError::OffsetOverflow`] when a reachable offset is not
788 /// representable: `dim - 1` of a non-unit extent does not fit `isize`, a
789 /// stride times that last index or the accumulated offsets overflow
790 /// `isize` or go negative, or the highest reachable offset is not below
791 /// the buffer length. A layout with a zero extent skips this check. The
792 /// byte-length and alignment checks
793 /// cannot fail for a typed `&mut [MaybeUninit<T>]`.
794 pub fn from_uninit_slice<T: KernelStorageElement>(
795 data: &'a mut [MaybeUninit<T>],
796 dims: &'a [usize],
797 strides: &'a [isize],
798 offset: isize,
799 ) -> Result<Self> {
800 let dtype = T::DTYPE;
801 let byte_len = data
802 .len()
803 .checked_mul(core::mem::size_of::<T>())
804 .ok_or(StridedError::OffsetOverflow)?;
805 // SAFETY: `data` is a live, exclusively borrowed `&mut
806 // [MaybeUninit<T>]` and `byte_len` is `data.len() * size_of::<T>()` from
807 // the `checked_mul` above, so the view spans exactly that allocation for
808 // `'a`. `MaybeUninit<u8>` accepts every bit pattern, so no
809 // initialization is asserted here.
810 let data = unsafe {
811 core::slice::from_raw_parts_mut(data.as_mut_ptr().cast::<MaybeUninit<u8>>(), byte_len)
812 };
813 let data_ptr =
814 NonNull::new(data.as_mut_ptr().cast::<u8>()).unwrap_or_else(NonNull::dangling);
815 let element_count = validate_erased_buffer_layout(dtype, data_ptr, byte_len)?;
816 validate_bounds(element_count, dims, strides, offset)?;
817 Ok(Self {
818 dtype,
819 data,
820 dims,
821 strides,
822 offset,
823 })
824 }
825
826 /// Construct from erased uninitialized storage.
827 ///
828 /// # Safety
829 /// `data..data + byte_len` must be an aligned, writable allocation valid
830 /// for `'a`, with provenance and extent sufficient for all reachable
831 /// elements. The caller must ensure every reachable element is completely
832 /// overwritten before it is read or exposed as initialized storage. The
833 /// allocation's alignment is an allocation property independent of the
834 /// observed address, and the descriptor must have exclusive access with no
835 /// mutable alias or concurrent mutation during use.
836 ///
837 /// # Errors
838 ///
839 /// Returns [`StridedError::ByteLengthMismatch`] when `byte_len` is not
840 /// a whole number of `dtype` elements, and
841 /// [`StridedError::DataAlignmentMismatch`] when `data` is
842 /// not aligned for `dtype`. Returns
843 /// [`StridedError::StrideLengthMismatch`] when `dims` and `strides`
844 /// have different lengths, and [`StridedError::OffsetOverflow`] when
845 /// a reachable offset is not representable: `dim - 1` of a non-unit
846 /// extent does not fit `isize`, a stride times that last index or the
847 /// accumulated offsets overflow `isize` or go negative, or the highest
848 /// reachable offset is not below the storage length. A layout with a zero
849 /// extent skips this check.
850 pub unsafe fn from_raw_parts(
851 dtype: KernelDType,
852 data: NonNull<u8>,
853 byte_len: usize,
854 dims: &'a [usize],
855 strides: &'a [isize],
856 offset: isize,
857 ) -> Result<Self> {
858 let count = validate_erased_buffer_layout(dtype, data, byte_len)?;
859 // SAFETY: `data` is a `NonNull`, and this function's contract
860 // (raw.rs:826) makes `data..data + byte_len` one live, writable
861 // allocation for `'a` held exclusively by the caller, which also bounds
862 // `byte_len` by `isize::MAX`. `MaybeUninit<u8>` accepts every bit
863 // pattern, so no initialization is asserted.
864 let data =
865 core::slice::from_raw_parts_mut(data.as_ptr().cast::<MaybeUninit<u8>>(), byte_len);
866 validate_bounds(count, dims, strides, offset)?;
867 Ok(Self {
868 dtype,
869 data,
870 dims,
871 strides,
872 offset,
873 })
874 }
875
876 /// Borrow the uninitialized storage as its concrete scalar type.
877 ///
878 /// # Errors
879 ///
880 /// Returns [`StridedError::DTypeMismatch`] when `T` is not the
881 /// descriptor's dtype.
882 pub fn data_as_uninit_mut<T: KernelStorageElement>(&mut self) -> Result<&mut [MaybeUninit<T>]> {
883 if self.dtype != T::DTYPE {
884 return Err(StridedError::DTypeMismatch {
885 expected: T::DTYPE.label(),
886 actual: self.dtype.label(),
887 });
888 }
889 // SAFETY: the dtype check proves `T`, and
890 // `validate_erased_buffer_layout` (raw.rs:161) proved the base is
891 // `T`-aligned with a whole multiple of `size_of::<T>()` bytes, for an
892 // empty buffer too (raw.rs:137, raw.rs:174; #434): the base comes from
893 // a `&mut [MaybeUninit<T>]` (`from_uninit_slice`, raw.rs:794) or from
894 // `from_raw_parts`, whose contract (raw.rs:826) makes
895 // `data..data + byte_len` the allocation itself. As for `ErasedRawStridedMut`, no
896 // path in this crate builds it from an interior pointer.
897 // `MaybeUninit<T>` accepts every bit pattern, so the partially
898 // initialized extent is not read as `T` here.
899 Ok(unsafe {
900 core::slice::from_raw_parts_mut(
901 self.data.as_mut_ptr().cast::<MaybeUninit<T>>(),
902 self.data.len() / core::mem::size_of::<T>(),
903 )
904 })
905 }
906
907 #[inline]
908 pub fn dtype(&self) -> KernelDType {
909 self.dtype
910 }
911
912 #[inline]
913 pub fn dims(&self) -> &'a [usize] {
914 self.dims
915 }
916
917 #[inline]
918 pub fn strides(&self) -> &'a [isize] {
919 self.strides
920 }
921
922 #[inline]
923 pub fn offset(&self) -> isize {
924 self.offset
925 }
926}
927
928fn ranges_overlap(a_start: usize, a_len: usize, b_start: usize, b_len: usize) -> Result<bool> {
929 let a_end = a_start
930 .checked_add(a_len)
931 .ok_or(StridedError::OffsetOverflow)?;
932 let b_end = b_start
933 .checked_add(b_len)
934 .ok_or(StridedError::OffsetOverflow)?;
935 Ok(a_start < b_end && b_start < a_end)
936}
937
938/// Borrowed raw strided input layout.
939///
940/// Use [`RawStridedRef::new`] for checked construction, or
941/// [`RawStridedRef::new_unchecked`] when a higher-level compiled plan has
942/// already validated bounds.
943#[derive(Clone, Copy)]
944pub struct RawStridedRef<'a, T> {
945 data: &'a [T],
946 dims: &'a [usize],
947 strides: &'a [isize],
948 offset: isize,
949}
950
951// A derived `Debug` would print every element of the backing buffer; the layout
952// metadata is what a reader wants.
953impl<T: core::fmt::Debug> core::fmt::Debug for RawStridedRef<'_, T> {
954 fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
955 f.debug_struct("RawStridedRef")
956 .field("data_len", &self.data.len())
957 .field("dims", &self.dims)
958 .field("strides", &self.strides)
959 .field("offset", &self.offset)
960 .finish()
961 }
962}
963
964impl<'a, T> RawStridedRef<'a, T> {
965 /// Create a raw strided input after validating reachable offsets.
966 ///
967 /// # Errors
968 ///
969 /// Returns [`StridedError::StrideLengthMismatch`] when `dims` and
970 /// `strides` have different lengths, and
971 /// [`StridedError::OffsetOverflow`] when a reachable offset is not
972 /// representable: `dim - 1` of a non-unit extent does not fit `isize`, a
973 /// stride times that last index or the accumulated offsets overflow
974 /// `isize` or go negative, or the highest reachable offset is not below
975 /// the buffer length. A layout with a zero extent skips this check.
976 pub fn new(
977 data: &'a [T],
978 dims: &'a [usize],
979 strides: &'a [isize],
980 offset: isize,
981 ) -> Result<Self> {
982 validate_bounds(data.len(), dims, strides, offset)?;
983 Ok(Self {
984 data,
985 dims,
986 strides,
987 offset,
988 })
989 }
990
991 /// Create a raw strided input without bounds checking.
992 ///
993 /// # Safety
994 /// The caller must ensure every index reachable by `dims`/`strides` from
995 /// `offset` lies inside `data`.
996 pub unsafe fn new_unchecked(
997 data: &'a [T],
998 dims: &'a [usize],
999 strides: &'a [isize],
1000 offset: isize,
1001 ) -> Self {
1002 Self {
1003 data,
1004 dims,
1005 strides,
1006 offset,
1007 }
1008 }
1009
1010 #[inline]
1011 pub fn data(&self) -> &'a [T] {
1012 self.data
1013 }
1014
1015 #[inline]
1016 pub fn dims(&self) -> &'a [usize] {
1017 self.dims
1018 }
1019
1020 #[inline]
1021 pub fn strides(&self) -> &'a [isize] {
1022 self.strides
1023 }
1024
1025 #[inline]
1026 pub fn offset(&self) -> isize {
1027 self.offset
1028 }
1029
1030 #[inline]
1031 pub fn ptr(&self) -> *const T {
1032 if self.dims.contains(&0) {
1033 NonNull::<T>::dangling().as_ptr()
1034 } else {
1035 // SAFETY: `offset` was accepted by `validate_bounds`
1036 // (view.rs:187) in `RawStridedRef::new`, or by the
1037 // caller contract of `new_unchecked`, so `data.as_ptr() + offset`
1038 // stays inside `data`. The empty-layout branch above returns a
1039 // dangling pointer instead of offsetting a pointer no element
1040 // reaches.
1041 unsafe { self.data.as_ptr().offset(self.offset) }
1042 }
1043 }
1044
1045 /// Convert to an immutable view, borrowing this descriptor's metadata.
1046 ///
1047 /// The conversion allocates nothing; hot prepared paths use the raw
1048 /// accessors directly all the same.
1049 #[inline]
1050 pub fn as_view(&self) -> StridedView<'a, T, Identity> {
1051 // SAFETY: `new_unchecked` requires every reachable index to stay inside
1052 // `data`; this descriptor's `offset`/`dims`/`strides` came from
1053 // `validate_bounds` (view.rs:187) in `RawStridedRef::new`,
1054 // or from the same contract on `new_unchecked`.
1055 unsafe { StridedView::new_unchecked(self.data, self.dims, self.strides, self.offset) }
1056 }
1057}
1058
1059/// Borrowed raw strided output layout.
1060///
1061/// This is the mutable counterpart to [`RawStridedRef`]. It avoids allocating
1062/// owned shape/stride metadata in prepared replay paths.
1063pub struct RawStridedMut<'a, T> {
1064 data: &'a mut [T],
1065 dims: &'a [usize],
1066 strides: &'a [isize],
1067 offset: isize,
1068}
1069
1070// A derived `Debug` would print every element of the backing buffer; the layout
1071// metadata is what a reader wants.
1072impl<T: core::fmt::Debug> core::fmt::Debug for RawStridedMut<'_, T> {
1073 fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
1074 f.debug_struct("RawStridedMut")
1075 .field("data_len", &self.data.len())
1076 .field("dims", &self.dims)
1077 .field("strides", &self.strides)
1078 .field("offset", &self.offset)
1079 .finish()
1080 }
1081}
1082
1083impl<'a, T> RawStridedMut<'a, T> {
1084 /// Create a raw strided output after validating reachable offsets.
1085 ///
1086 /// # Errors
1087 ///
1088 /// Returns [`StridedError::StrideLengthMismatch`] when `dims` and
1089 /// `strides` have different lengths, and
1090 /// [`StridedError::OffsetOverflow`] when a reachable offset is not
1091 /// representable: `dim - 1` of a non-unit extent does not fit `isize`, a
1092 /// stride times that last index or the accumulated offsets overflow
1093 /// `isize` or go negative, or the highest reachable offset is not below
1094 /// the buffer length. A layout with a zero extent skips this check.
1095 pub fn new(
1096 data: &'a mut [T],
1097 dims: &'a [usize],
1098 strides: &'a [isize],
1099 offset: isize,
1100 ) -> Result<Self> {
1101 validate_bounds(data.len(), dims, strides, offset)?;
1102 Ok(Self {
1103 data,
1104 dims,
1105 strides,
1106 offset,
1107 })
1108 }
1109
1110 /// Create a raw strided output without bounds checking.
1111 ///
1112 /// # Safety
1113 /// The caller must ensure every index reachable by `dims`/`strides` from
1114 /// `offset` lies inside `data`, and no aliases violate mutable access.
1115 pub unsafe fn new_unchecked(
1116 data: &'a mut [T],
1117 dims: &'a [usize],
1118 strides: &'a [isize],
1119 offset: isize,
1120 ) -> Self {
1121 Self {
1122 data,
1123 dims,
1124 strides,
1125 offset,
1126 }
1127 }
1128
1129 #[inline]
1130 pub fn data(&self) -> &[T] {
1131 self.data
1132 }
1133
1134 #[inline]
1135 pub fn data_mut(&mut self) -> &mut [T] {
1136 self.data
1137 }
1138
1139 #[inline]
1140 pub fn dims(&self) -> &'a [usize] {
1141 self.dims
1142 }
1143
1144 #[inline]
1145 pub fn strides(&self) -> &'a [isize] {
1146 self.strides
1147 }
1148
1149 #[inline]
1150 pub fn offset(&self) -> isize {
1151 self.offset
1152 }
1153
1154 #[inline]
1155 pub fn ptr(&self) -> *const T {
1156 if self.dims.contains(&0) {
1157 NonNull::<T>::dangling().as_ptr()
1158 } else {
1159 // SAFETY: `offset` was accepted by `validate_bounds`
1160 // (view.rs:187) in `RawStridedMut::new`, or by the
1161 // caller contract of `new_unchecked`, so `data.as_ptr() + offset`
1162 // stays inside `data`. The empty-layout branch above returns a
1163 // dangling pointer instead of offsetting a pointer no element
1164 // reaches.
1165 unsafe { self.data.as_ptr().offset(self.offset) }
1166 }
1167 }
1168
1169 #[inline]
1170 pub fn as_mut_ptr(&mut self) -> *mut T {
1171 if self.dims.contains(&0) {
1172 NonNull::<T>::dangling().as_ptr()
1173 } else {
1174 // SAFETY: as `ptr` above — `offset` was accepted by
1175 // `validate_bounds` (view.rs:187) in
1176 // `RawStridedMut::new` or by the `new_unchecked` contract, and
1177 // `&mut self` keeps the write provenance exclusive.
1178 unsafe { self.data.as_mut_ptr().offset(self.offset) }
1179 }
1180 }
1181
1182 /// Convert to an immutable view, borrowing this descriptor's metadata.
1183 ///
1184 /// The conversion allocates nothing; hot prepared paths use the raw
1185 /// accessors directly all the same.
1186 #[inline]
1187 pub fn as_view(&self) -> StridedView<'_, T, Identity> {
1188 // SAFETY: `new_unchecked` requires every reachable index to stay inside
1189 // `data`; this descriptor's metadata came from `validate_bounds`
1190 // (view.rs:187) in `RawStridedMut::new`, or from the
1191 // same contract on `new_unchecked`.
1192 unsafe { StridedView::new_unchecked(self.data, self.dims, self.strides, self.offset) }
1193 }
1194
1195 /// Convert to a mutable view, borrowing this descriptor's metadata.
1196 ///
1197 /// The conversion allocates nothing; hot prepared paths use the raw
1198 /// accessors directly all the same.
1199 #[inline]
1200 pub fn as_view_mut(&mut self) -> StridedViewMut<'_, T> {
1201 // SAFETY: as `as_view` above, and `&mut self` keeps the mutable borrow
1202 // exclusive to the returned view, which `new_unchecked` requires.
1203 unsafe { StridedViewMut::new_unchecked(self.data, self.dims, self.strides, self.offset) }
1204 }
1205}
1206
1207#[cfg(test)]
1208mod tests;