hermes_simd_core/view/
mod.rs

1//! Safely typed views over slices with static alignment, architecture dispatch, reference typestates, and execution mode.
2
3use crate::align::Alignment;
4use crate::arch::SimdArch;
5use crate::execution::{ExecutionMode, Unmasked};
6use crate::iter;
7use crate::kernel::SimdKernel;
8use crate::scalar::Scalar;
9use core::marker::PhantomData;
10
11/// Vectorized indirect load (gather) operations.
12pub mod gather;
13/// Lane-masked math and compaction/expansion operations on SIMD views.
14pub mod masked;
15/// Standard elementwise and accumulation operations on SIMD views.
16pub mod ops;
17/// Standard exclusive mutable elementwise operations on SIMD views.
18pub mod ops_mut;
19/// Unrolled generic horizontal reductions on SIMD views.
20pub mod reduce;
21/// Inclusive/exclusive prefix scans and running min/max.
22pub mod scan;
23/// Lane-wise conditional select and masked-negate.
24pub mod select;
25/// 2D matrix tile views and operations.
26pub mod tile;
27/// Unary mapping operations on SIMD views.
28pub mod unary;
29
30pub use tile::{TileMatrixMultiply, TileView};
31
32/// Module containing the SIMD mask register wrappers.
33pub mod mask_reg;
34/// Module containing operator overload implementations for SIMD vectors.
35pub mod vector_ops;
36/// Module containing the generic SIMD vector register wrappers.
37pub mod vector_reg;
38
39pub use mask_reg::Mask;
40pub use vector_reg::Vector;
41
42/// Error types for SIMD view operations.
43#[derive(Debug, Clone, Copy, PartialEq, Eq)]
44pub enum SimdError {
45    /// The lengths of the operand views do not match.
46    LengthMismatch,
47    /// The input slice is too small to load the requested vector.
48    InsufficientInputLength,
49    /// The output slice is too small to store the results.
50    InsufficientOutputLength,
51    /// The memory address is not aligned as required.
52    UnalignedAddress,
53    /// An index is out of bounds of the view.
54    IndexOutOfBounds,
55    /// The current host cannot execute the requested SIMD target safely.
56    UnsupportedTarget,
57}
58
59impl core::fmt::Display for SimdError {
60    fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
61        match self {
62            Self::LengthMismatch => write!(f, "Operand views have mismatched lengths"),
63            Self::InsufficientInputLength => write!(f, "Input slice has insufficient length"),
64            Self::InsufficientOutputLength => write!(f, "Output slice has insufficient length"),
65            Self::UnalignedAddress => {
66                write!(f, "Memory address does not satisfy alignment constraints")
67            }
68            Self::IndexOutOfBounds => write!(f, "Index is out of bounds of the view"),
69            Self::UnsupportedTarget => {
70                write!(f, "SIMD target is not supported or enabled on this host")
71            }
72        }
73    }
74}
75
76// Implement standard Error trait if std is available.
77#[cfg(feature = "std")]
78impl std::error::Error for SimdError {}
79
80/// A zero-copy, typed slice view parameterized by architecture, alignment, execution mode, and reference typestates.
81///
82/// # Type Parameters
83/// - `T`: scalar element type
84/// - `Arch`: SIMD architecture ZST marker
85/// - `Align`: alignment typestate
86/// - `Mode`: execution mode (`Unmasked` or `Masked`); defaults to `Unmasked`
87/// - `Ref`: reference typestate; defaults to `&'a [T]`
88///
89/// Guaranteed to have zero runtime overhead and remains `#[repr(transparent)]`.
90#[repr(transparent)]
91pub struct SimdView<
92    'a,
93    T: 'a,
94    Arch: SimdArch,
95    Align: Alignment,
96    Mode: ExecutionMode = Unmasked,
97    Ref: 'a = &'a [T],
98> {
99    ptr: *mut [T],
100    _marker: PhantomData<(&'a T, Arch, Align, Mode, Ref)>,
101}
102
103unsafe impl<'a, T: 'a, Arch: SimdArch, Align: Alignment, Mode: ExecutionMode, Ref: 'a> Send
104    for SimdView<'a, T, Arch, Align, Mode, Ref>
105where
106    Ref: Send,
107{
108}
109
110unsafe impl<'a, T: 'a, Arch: SimdArch, Align: Alignment, Mode: ExecutionMode, Ref: 'a> Sync
111    for SimdView<'a, T, Arch, Align, Mode, Ref>
112where
113    Ref: Sync,
114{
115}
116
117impl<'a, T: 'a, Arch: SimdArch, Align: Alignment, Mode: ExecutionMode> Clone
118    for SimdView<'a, T, Arch, Align, Mode, &'a [T]>
119{
120    #[inline(always)]
121    fn clone(&self) -> Self {
122        *self
123    }
124}
125
126impl<'a, T: 'a, Arch: SimdArch, Align: Alignment, Mode: ExecutionMode> Copy
127    for SimdView<'a, T, Arch, Align, Mode, &'a [T]>
128{
129}
130
131impl<'a, T: 'a, Arch: SimdArch, Align: Alignment, Mode: ExecutionMode>
132    SimdView<'a, T, Arch, Align, Mode, &'a [T]>
133{
134    /// Create a new read-only `SimdView` after verifying that `Arch` runs on
135    /// this host and that the alignment invariants hold.
136    ///
137    /// Returns `None` when the host cannot execute `Arch` — naming a marker the
138    /// CPU does not implement, such as `Avx512` on a machine without it — or
139    /// when the alignment requirements are not met. Every operation on the view
140    /// calls `#[target_feature]`-gated kernels, so a view that existed without
141    /// that guarantee would let safe code execute unsupported instructions.
142    #[inline]
143    pub fn new(data: &'a [T]) -> Option<Self> {
144        if !Arch::is_runtime_supported() {
145            return None;
146        }
147        if Align::IS_ALIGNED {
148            let req_align = Arch::REGISTER_WIDTH_BITS as usize / 8;
149            if req_align > 0 && Align::ALIGN_BYTES < req_align {
150                return None;
151            }
152            let addr = data.as_ptr() as usize;
153            if addr % Align::ALIGN_BYTES != 0 {
154                return None;
155            }
156        }
157        Some(Self {
158            ptr: data as *const [T] as *mut [T],
159            _marker: PhantomData,
160        })
161    }
162}
163
164impl<'a, T: 'a, Arch: SimdArch, Align: Alignment, Mode: ExecutionMode>
165    SimdView<'a, T, Arch, Align, Mode, &'a mut [T]>
166{
167    /// Create a new mutable `SimdView` after verifying that `Arch` runs on this
168    /// host and that the alignment invariants hold.
169    ///
170    /// Returns `None` under the same conditions as [`SimdView::new`].
171    #[inline]
172    pub fn new_mut(data: &'a mut [T]) -> Option<Self> {
173        if !Arch::is_runtime_supported() {
174            return None;
175        }
176        if Align::IS_ALIGNED {
177            let req_align = Arch::REGISTER_WIDTH_BITS as usize / 8;
178            if req_align > 0 && Align::ALIGN_BYTES < req_align {
179                return None;
180            }
181            let addr = data.as_ptr() as usize;
182            if addr % Align::ALIGN_BYTES != 0 {
183                return None;
184            }
185        }
186        Some(Self {
187            ptr: data as *mut [T],
188            _marker: PhantomData,
189        })
190    }
191
192    /// Access the underlying raw mutable slice.
193    #[inline(always)]
194    pub fn as_slice_mut(&mut self) -> &mut [T] {
195        unsafe { &mut *self.ptr }
196    }
197
198    /// Downgrade the exclusive mutable view to a shared read-only view.
199    #[inline(always)]
200    pub fn downgrade(self) -> SimdView<'a, T, Arch, Align, Mode, &'a [T]> {
201        SimdView {
202            ptr: self.ptr,
203            _marker: PhantomData,
204        }
205    }
206}
207
208impl<'a, T: 'a, Arch: SimdArch, Align: Alignment, Mode: ExecutionMode, Ref: 'a>
209    SimdView<'a, T, Arch, Align, Mode, Ref>
210{
211    /// Access the underlying raw slice.
212    #[inline(always)]
213    pub fn as_slice(&self) -> &[T] {
214        unsafe { &*self.ptr }
215    }
216
217    /// Returns the length of the slice.
218    #[inline(always)]
219    pub fn len(&self) -> usize {
220        self.as_slice().len()
221    }
222
223    /// Returns true if the slice is empty.
224    #[inline(always)]
225    pub fn is_empty(&self) -> bool {
226        self.len() == 0
227    }
228
229    /// Strips the static alignment guarantee of this view, returning an unaligned view zero-cost.
230    #[inline(always)]
231    pub fn into_unaligned(self) -> SimdView<'a, T, Arch, crate::align::Unaligned, Mode, Ref> {
232        SimdView {
233            ptr: self.ptr,
234            _marker: PhantomData,
235        }
236    }
237
238    /// Attempts to promote the alignment of this view to boundary `A` bytes.
239    /// Returns `Some(SimdView)` if the start pointer is aligned to `A` bytes, otherwise `None`.
240    #[inline]
241    pub fn try_into_aligned<const A: usize>(
242        self,
243    ) -> Option<SimdView<'a, T, Arch, crate::align::Aligned<A>, Mode, Ref>> {
244        let req_align = Arch::REGISTER_WIDTH_BITS as usize / 8;
245        if req_align > 0 && A < req_align {
246            return None;
247        }
248        let addr = self.as_slice().as_ptr() as usize;
249        if addr % A == 0 {
250            Some(SimdView {
251                ptr: self.ptr,
252                _marker: PhantomData,
253            })
254        } else {
255            None
256        }
257    }
258}
259
260impl<'a, T: 'a, Arch: SimdArch, Align: Alignment, Mode: ExecutionMode>
261    SimdView<'a, T, Arch, Align, Mode, &'a [T]>
262{
263    /// Zero-copy sub-slice over a range of indices, returning an unaligned view.
264    #[inline]
265    pub fn slice_unaligned(
266        self,
267        range: core::ops::Range<usize>,
268    ) -> SimdView<'a, T, Arch, crate::align::Unaligned, Mode, &'a [T]> {
269        let sub = &self.as_slice()[range];
270        SimdView {
271            ptr: sub as *const [T] as *mut [T],
272            _marker: PhantomData,
273        }
274    }
275
276    /// Zero-copy sub-slice over a range of indices, returning an aligned view with boundary `A` bytes.
277    /// Returns `Some(SimdView)` if the sub-slice satisfies the alignment, otherwise `None`.
278    #[inline]
279    pub fn slice_aligned<const A: usize>(
280        self,
281        range: core::ops::Range<usize>,
282    ) -> Option<SimdView<'a, T, Arch, crate::align::Aligned<A>, Mode, &'a [T]>> {
283        let req_align = Arch::REGISTER_WIDTH_BITS as usize / 8;
284        if req_align > 0 && A < req_align {
285            return None;
286        }
287        let sub = &self.as_slice()[range];
288        let addr = sub.as_ptr() as usize;
289        if addr % A == 0 {
290            Some(SimdView {
291                ptr: sub as *const [T] as *mut [T],
292                _marker: PhantomData,
293            })
294        } else {
295            None
296        }
297    }
298}
299
300impl<'a, T: 'a, Arch: SimdArch, Align: Alignment, Mode: ExecutionMode>
301    SimdView<'a, T, Arch, Align, Mode, &'a mut [T]>
302{
303    /// Zero-copy mutable sub-slice over a range of indices, returning an unaligned view.
304    #[inline]
305    pub fn slice_unaligned_mut(
306        mut self,
307        range: core::ops::Range<usize>,
308    ) -> SimdView<'a, T, Arch, crate::align::Unaligned, Mode, &'a mut [T]> {
309        let sub = &mut self.as_slice_mut()[range];
310        SimdView {
311            ptr: sub as *mut [T],
312            _marker: PhantomData,
313        }
314    }
315
316    /// Zero-copy mutable sub-slice over a range of indices, returning an aligned view with boundary `A` bytes.
317    /// Returns `Some(SimdView)` if the sub-slice satisfies the alignment, otherwise `None`.
318    #[inline]
319    pub fn slice_aligned_mut<const A: usize>(
320        mut self,
321        range: core::ops::Range<usize>,
322    ) -> Option<SimdView<'a, T, Arch, crate::align::Aligned<A>, Mode, &'a mut [T]>> {
323        let req_align = Arch::REGISTER_WIDTH_BITS as usize / 8;
324        if req_align > 0 && A < req_align {
325            return None;
326        }
327        let sub = &mut self.as_slice_mut()[range];
328        let addr = sub.as_ptr() as usize;
329        if addr % A == 0 {
330            Some(SimdView {
331                ptr: sub as *mut [T],
332                _marker: PhantomData,
333            })
334        } else {
335            None
336        }
337    }
338}
339
340impl<
341        'a,
342        T: Scalar + 'a,
343        Arch: SimdArch + SimdKernel<T>,
344        Align: Alignment,
345        Mode: ExecutionMode,
346        Ref: 'a,
347    > SimdView<'a, T, Arch, Align, Mode, Ref>
348{
349    /// Return a zero-copy iterator over non-overlapping `LANE_COUNT`-wide sub-views.
350    ///
351    /// Each yielded item is a `SimdView<'a, T, Arch, Align, Mode, &'a [T]>` covering
352    /// exactly `Arch::LANE_COUNT` elements. The scalar tail (elements that do not fill
353    /// a complete vector) is accessible via [`iter::SimdChunks::remainder`].
354    #[inline(always)]
355    pub fn simd_chunks(&self) -> iter::SimdChunks<'a, T, Arch, Align, Mode> {
356        // SAFETY: self.as_slice() is valid for the lifetime 'a (it derives from our ptr).
357        unsafe {
358            iter::SimdChunks::from_raw_parts(self.as_slice().as_ptr(), self.len(), Arch::LANE_COUNT)
359        }
360    }
361
362    /// Return a zero-copy iterator that advances two views in lockstep.
363    ///
364    /// Iterates non-overlapping `LANE_COUNT`-wide pairs of sub-views from `self` and `other`
365    /// until the shorter SIMD prefix is exhausted. Access the tails via
366    /// [`iter::ZipChunks::remainder`].
367    #[inline(always)]
368    pub fn zip_chunks<'b>(
369        &self,
370        other: &'b SimdView<'b, T, Arch, Align, Mode, &'b [T]>,
371    ) -> iter::ZipChunks<'a, 'b, T, Arch, Align, Mode> {
372        // SAFETY: both slice pointers are valid for their respective lifetimes.
373        unsafe {
374            iter::ZipChunks::from_raw_parts(
375                self.as_slice().as_ptr(),
376                self.len(),
377                other.as_slice().as_ptr(),
378                other.len(),
379            )
380        }
381    }
382}
383
384impl<'a, T: Scalar + 'a, Arch: SimdArch + SimdKernel<T>, Align: Alignment, Mode: ExecutionMode>
385    SimdView<'a, T, Arch, Align, Mode, &'a mut [T]>
386{
387    /// Return a zero-copy mutable iterator over non-overlapping `LANE_COUNT`-wide sub-views.
388    ///
389    /// Each yielded item is a `SimdView<'a, T, Arch, Align, Mode, &'a mut [T]>` covering
390    /// exactly `Arch::LANE_COUNT` elements. The scalar tail (elements that do not fill
391    /// a complete vector) is accessible via [`iter::SimdChunksMut::into_remainder`].
392    #[inline(always)]
393    pub fn simd_chunks_mut(self) -> iter::SimdChunksMut<'a, T, Arch, Align, Mode> {
394        // SAFETY: self.ptr is valid for writes of total elements for lifetime 'a.
395        unsafe {
396            iter::SimdChunksMut::from_raw_parts(self.ptr as *mut T, self.len(), Arch::LANE_COUNT)
397        }
398    }
399
400    /// Return a paired mutable/immutable chunk iterator (the SAXPY pattern).
401    ///
402    /// Advances `self` (mutable) and `other` (immutable) in lockstep by `LANE_COUNT` per step.
403    /// The scalar tails are returned by [`iter::ZipChunksMut::into_remainder`].
404    ///
405    /// ```rust,ignore
406    /// let mut chunks = view_a.zip_chunks_mut(&view_b);
407    /// for (mut a_chunk, b_chunk) in &mut chunks {
408    ///     a_chunk.transform_in_place(&b_chunk, Add);
409    /// }
410    /// let (tail_a, tail_b) = chunks.into_remainder();
411    /// for (a, &b) in tail_a.iter_mut().zip(tail_b) { *a = *a + b; }
412    /// ```
413    #[inline(always)]
414    pub fn zip_chunks_mut<'b>(
415        self,
416        other: &'b SimdView<'b, T, Arch, Align, Mode, &'b [T]>,
417    ) -> iter::ZipChunksMut<'a, 'b, T, Arch, Align, Mode> {
418        // SAFETY: self is an exclusive mutable view for 'a; other is a shared view for 'b.
419        // Non-overlap is a caller invariant (enforced by the borrow checker: `self` is `&'a mut`).
420        unsafe {
421            iter::ZipChunksMut::from_raw_parts(
422                self.ptr as *mut T,
423                self.len(),
424                other.as_slice().as_ptr(),
425                other.len(),
426            )
427        }
428    }
429}
430
431impl<'a, T: 'a, Arch: SimdArch, Align: Alignment, Mode: ExecutionMode, Ref: 'a> core::ops::Deref
432    for SimdView<'a, T, Arch, Align, Mode, Ref>
433{
434    type Target = [T];
435
436    #[inline(always)]
437    fn deref(&self) -> &Self::Target {
438        self.as_slice()
439    }
440}
441
442impl<'a, T: 'a, Arch: SimdArch, Align: Alignment, Mode: ExecutionMode> core::ops::DerefMut
443    for SimdView<'a, T, Arch, Align, Mode, &'a mut [T]>
444{
445    #[inline(always)]
446    fn deref_mut(&mut self) -> &mut Self::Target {
447        self.as_slice_mut()
448    }
449}
450
451impl<'a, T: 'a, Arch: SimdArch, Align: Alignment, Mode: ExecutionMode>
452    SimdView<'a, T, Arch, Align, Mode, &'a [T]>
453where
454    T: bytemuck::Pod,
455{
456    /// Safe cast of the underlying data slice to a slice of another Pod type, returning a new `SimdView`.
457    #[inline]
458    pub fn cast<U: bytemuck::Pod>(self) -> Option<SimdView<'a, U, Arch, Align, Mode, &'a [U]>> {
459        let casted = bytemuck::try_cast_slice(unsafe { &*self.ptr }).ok()?;
460        SimdView::new(casted)
461    }
462}
463
464impl<'a, T: 'a, Arch: SimdArch, Align: Alignment, Mode: ExecutionMode>
465    SimdView<'a, T, Arch, Align, Mode, &'a mut [T]>
466where
467    T: bytemuck::Pod,
468{
469    /// Safe cast of the underlying mutable data slice to a mutable slice of another Pod type, returning a new mutable `SimdView`.
470    #[inline]
471    pub fn cast_mut<U: bytemuck::Pod>(
472        self,
473    ) -> Option<SimdView<'a, U, Arch, Align, Mode, &'a mut [U]>> {
474        let casted = bytemuck::try_cast_slice_mut(unsafe { &mut *self.ptr }).ok()?;
475        SimdView::new_mut(casted)
476    }
477}
478
479#[inline(never)]
480pub(crate) fn check_lengths_equal(len1: usize, len2: usize) -> Result<(), SimdError> {
481    if len1 != len2 {
482        return Err(SimdError::LengthMismatch);
483    }
484    Ok(())
485}
486
487#[inline(never)]
488pub(crate) fn check_output_length(input_len: usize, output_len: usize) -> Result<(), SimdError> {
489    if output_len < input_len {
490        return Err(SimdError::InsufficientOutputLength);
491    }
492    Ok(())
493}
hermes_simd_core/view/mod.rs

hermes_simd_core/view/
mod.rs
