generic_simd/
slice.rs

1//! Extensions for slices of vectors.
2
3use crate::arch::Token;
4use crate::{
5    scalar::Scalar,
6    vector::{width, Native, NativeWidth, Vector},
7};
8use core::marker::PhantomData;
9
10/// A slice of scalars.
11pub trait Slice<Token, Width>
12where
13    Token: crate::arch::Token,
14    Width: width::Width,
15{
16    type Vector: Vector<Token = Token, Width = Width>;
17
18    /// Read a vector from a slice without checking the length.
19    ///
20    /// # Safety
21    /// See [`read_unchecked`](../vector/trait.Vector.html#method.read_ptr).
22    unsafe fn read_unchecked(&self, token: Token) -> Self::Vector;
23
24    /// Read a vector from a slice.
25    ///
26    /// See [`read`](../vector/trait.Vector.html#method.read).
27    fn read(&self, token: Token) -> Self::Vector;
28
29    /// Extract a slice of aligned vectors, as if by [`align_to`].
30    ///
31    /// [`align_to`]: https://doc.rust-lang.org/std/primitive.slice.html#method.align_to
32    #[allow(clippy::type_complexity)]
33    fn align(
34        &self,
35        #[allow(unused_variables)] token: Token,
36    ) -> (
37        &[<Self::Vector as Vector>::Scalar],
38        &[Self::Vector],
39        &[<Self::Vector as Vector>::Scalar],
40    );
41
42    /// Extract a slice of aligned mutable vectors, as if by [`align_to_mut`].
43    ///
44    /// [`align_to_mut`]: https://doc.rust-lang.org/std/primitive.slice.html#method.align_to_mut
45    #[allow(clippy::type_complexity)]
46    fn align_mut(
47        &mut self,
48        #[allow(unused_variables)] token: Token,
49    ) -> (
50        &mut [<Self::Vector as Vector>::Scalar],
51        &mut [Self::Vector],
52        &mut [<Self::Vector as Vector>::Scalar],
53    );
54
55    /// Create a slice of overlapping vectors from a slice of scalars.
56    fn overlapping(&self, token: Token) -> Overlapping<'_, Self::Vector>;
57
58    /// Create a mutable slice of overlapping vectors from a slice of scalars.
59    fn overlapping_mut(&mut self, token: Token) -> OverlappingMut<'_, Self::Vector>;
60}
61
62impl<T, Token, Width> Slice<Token, Width> for [T]
63where
64    T: Scalar<Token, Width>,
65    Token: crate::arch::Token,
66    Width: width::Width,
67{
68    type Vector = T::Vector;
69
70    #[inline]
71    unsafe fn read_unchecked(&self, token: Token) -> Self::Vector {
72        Self::Vector::read_unchecked(token, self)
73    }
74
75    #[inline]
76    fn read(&self, token: Token) -> Self::Vector {
77        Self::Vector::read(token, self)
78    }
79
80    #[allow(clippy::type_complexity)]
81    #[inline]
82    fn align(
83        &self,
84        #[allow(unused_variables)] token: Token,
85    ) -> (
86        &[<Self::Vector as Vector>::Scalar],
87        &[Self::Vector],
88        &[<Self::Vector as Vector>::Scalar],
89    ) {
90        unsafe { self.align_to() }
91    }
92
93    #[allow(clippy::type_complexity)]
94    #[inline]
95    fn align_mut(
96        &mut self,
97        #[allow(unused_variables)] token: Token,
98    ) -> (
99        &mut [<Self::Vector as Vector>::Scalar],
100        &mut [Self::Vector],
101        &mut [<Self::Vector as Vector>::Scalar],
102    ) {
103        unsafe { self.align_to_mut() }
104    }
105
106    #[inline]
107    fn overlapping(&self, token: Token) -> Overlapping<'_, Self::Vector> {
108        Overlapping::new(token, self)
109    }
110
111    #[inline]
112    fn overlapping_mut(&mut self, token: Token) -> OverlappingMut<'_, Self::Vector> {
113        OverlappingMut::new(token, self)
114    }
115}
116
117macro_rules! slice_impl {
118    {
119        $width:literal,
120        $width_type:ty,
121        $read_unchecked:ident,
122        $read:ident,
123        $align:ident,
124        $align_mut:ident,
125        $overlapping:ident,
126        $overlapping_mut:ident
127    } => {
128        #[doc = "Read a vector with "]
129        #[doc = $width]
130        #[doc = " from a slice without checking the length.\n\nSee [`read_unchecked`](../vector/trait.Vector.html#method.read_ptr)."]
131        #[inline]
132        unsafe fn $read_unchecked(&self, token: Token) -> <Self as Slice<Token, $width_type>>::Vector {
133            <Self as Slice<Token, $width_type>>::read_unchecked(self, token)
134        }
135
136        #[doc = "Read a vector with "]
137        #[doc = $width]
138        #[doc = " from a slice.\n\nSee [`read`](../vector/trait.Vector.html#method.read)."]
139        #[inline]
140        fn $read(&self, token: Token) -> <Self as Slice<Token, $width_type>>::Vector {
141            <Self as Slice<Token, $width_type>>::read(self, token)
142        }
143
144        #[doc = "Align a slice of scalars to vectors with "]
145        #[doc = $width]
146        #[doc = ".\n\nSee [`align`](trait.Slice.html#tymethod.align)."]
147        #[allow(clippy::type_complexity)]
148        #[inline]
149        fn $align(&self, token: Token) ->
150        (
151            &[<<Self as Slice<Token, $width_type>>::Vector as Vector>::Scalar],
152            &[<Self as Slice<Token, $width_type>>::Vector],
153            &[<<Self as Slice<Token, $width_type>>::Vector as Vector>::Scalar],
154        ) {
155            <Self as Slice<Token, $width_type>>::align(self, token)
156        }
157
158        #[doc = "Align a slice of scalars to vectors with "]
159        #[doc = $width]
160        #[doc = ".\n\nSee [`align_mut`](trait.Slice.html#tymethod.align_mut)."]
161        #[allow(clippy::type_complexity)]
162        #[inline]
163        fn $align_mut(&mut self, token: Token) ->
164        (
165            &mut [<<Self as Slice<Token, $width_type>>::Vector as Vector>::Scalar],
166            &mut [<Self as Slice<Token, $width_type>>::Vector],
167            &mut [<<Self as Slice<Token, $width_type>>::Vector as Vector>::Scalar],
168        ){
169            <Self as Slice<Token, $width_type>>::align_mut(self, token)
170        }
171
172        #[doc = "Create a slice of overlapping vectors of "]
173        #[doc = $width]
174        #[doc = "from a slice of scalars.\n\nSee [`overlapping`](trait.Slice.html#tymethod.overlapping)."]
175        #[inline]
176        fn $overlapping(&self, token: Token) -> Overlapping<'_, <Self as Slice<Token, $width_type>>::Vector> {
177            <Self as Slice<Token, $width_type>>::overlapping(self, token)
178        }
179
180        #[doc = "Create a mutable slice of overlapping vectors of "]
181        #[doc = $width]
182        #[doc = "from a slice of scalars.\n\nSee [`overlapping_mut`](trait.Slice.html#tymethod.overlapping_mut)."]
183        #[inline]
184        fn $overlapping_mut(
185            &mut self,
186            token: Token,
187        ) -> OverlappingMut<'_, <Self as Slice<Token, $width_type>>::Vector> {
188            <Self as Slice<Token, $width_type>>::overlapping_mut(self, token)
189        }
190    }
191}
192
193impl<T, Token> Native<Token> for [T]
194where
195    T: Native<Token>,
196{
197    type Width = T::Width;
198}
199
200/// A slice of scalars, supporting all vector widths.
201pub trait SliceExt<Token>:
202    Native<Token>
203    + Slice<Token, width::W1>
204    + Slice<Token, width::W2>
205    + Slice<Token, width::W4>
206    + Slice<Token, width::W8>
207    + Slice<Token, NativeWidth<Self, Token>>
208where
209    Token: crate::arch::Token,
210{
211    slice_impl! { "the native number of lanes", <Self as Native<Token>>::Width, read_unchecked_native, read_native, align_native, align_native_mut, overlapping_native, overlapping_native_mut }
212    slice_impl! { "1 lane",   width::W1, read_unchecked1, read1, align1, align1_mut, overlapping1, overlapping1_mut }
213    slice_impl! { "2 lanes",  width::W2, read_unchecked2, read2, align2, align2_mut, overlapping2, overlapping2_mut }
214    slice_impl! { "4 lanes",  width::W4, read_unchecked4, read4, align4, align4_mut, overlapping4, overlapping4_mut }
215    slice_impl! { "8 lanes",  width::W8, read_unchecked8, read8, align8, align8_mut, overlapping8, overlapping8_mut }
216}
217
218impl<T, Token> SliceExt<Token> for T
219where
220    T: ?Sized
221        + Native<Token>
222        + Slice<Token, width::W1>
223        + Slice<Token, width::W2>
224        + Slice<Token, width::W4>
225        + Slice<Token, width::W8>
226        + Slice<Token, NativeWidth<Self, Token>>,
227    Token: crate::arch::Token,
228{
229}
230
231/// Wrapper for producing a mutable reference from an unaligned pointer.
232pub struct RefMut<'a, V>
233where
234    V: Vector,
235{
236    source: *mut V::Scalar,
237    temp: V,
238    lifetime: PhantomData<&'a V::Scalar>,
239}
240
241impl<'a, V> RefMut<'a, V>
242where
243    V: Vector,
244{
245    #[inline]
246    fn new(token: V::Token, source: *mut V::Scalar) -> Self {
247        Self {
248            source,
249            temp: V::zeroed(token),
250            lifetime: PhantomData,
251        }
252    }
253}
254
255impl<'a, V> core::ops::Deref for RefMut<'a, V>
256where
257    V: Vector,
258{
259    type Target = V;
260
261    #[inline]
262    fn deref(&self) -> &V {
263        &self.temp
264    }
265}
266
267impl<'a, V> core::ops::DerefMut for RefMut<'a, V>
268where
269    V: Vector,
270{
271    #[inline]
272    fn deref_mut(&mut self) -> &mut V {
273        &mut self.temp
274    }
275}
276
277impl<'a, V> core::ops::Drop for RefMut<'a, V>
278where
279    V: Vector,
280{
281    #[inline]
282    fn drop(&mut self) {
283        unsafe {
284            self.temp.write_ptr(self.source);
285        }
286    }
287}
288
289/// Wrapper for indexing into overlapping vectors.
290pub struct Overlapping<'a, V>
291where
292    V: Vector,
293{
294    slice: &'a [V::Scalar],
295    phantom: PhantomData<V>,
296}
297
298#[allow(clippy::len_without_is_empty)]
299impl<'a, V> Overlapping<'a, V>
300where
301    V: Vector,
302{
303    /// Create a new overlapping vector slice.
304    #[inline]
305    pub fn new(
306        #[allow(unused_variables)] token: impl Into<V::Token>,
307        slice: &'a [V::Scalar],
308    ) -> Self {
309        assert!(
310            slice.len() >= V::width(),
311            "slice must be at least as wide as the vector"
312        );
313        Self {
314            slice,
315            phantom: PhantomData,
316        }
317    }
318
319    /// Returns the number of overlapping vectors.
320    ///
321    /// Equal to `slice.len() - V::width() + 1`.
322    #[inline]
323    pub fn len(&self) -> usize {
324        self.slice.len() - V::width() + 1
325    }
326
327    /// Returns the vector offset `index` into the slice of scalars.
328    #[inline]
329    pub fn get(&self, index: usize) -> Option<V> {
330        if index < self.len() {
331            Some(unsafe { self.get_unchecked(index) })
332        } else {
333            None
334        }
335    }
336
337    /// Returns the vector offset `index` into the slice of scalars.
338    ///
339    /// # Safety
340    /// Index must be less than `len()`, i.e. the underlying slice must be at least `index
341    /// + V::width()` long.
342    #[inline]
343    pub unsafe fn get_unchecked(&self, index: usize) -> V
344    where
345        V: Vector,
346    {
347        V::read_ptr(V::Token::new_unchecked(), self.slice.as_ptr().add(index))
348    }
349}
350
351/// Wrapper for indexing into overlapping mutable vectors.
352pub struct OverlappingMut<'a, V>
353where
354    V: Vector,
355{
356    slice: &'a mut [V::Scalar],
357    phantom: PhantomData<V>,
358}
359
360#[allow(clippy::len_without_is_empty)]
361impl<'a, V> OverlappingMut<'a, V>
362where
363    V: Vector,
364{
365    /// Create a new overlapping vector slice.
366    #[inline]
367    pub fn new(
368        #[allow(unused_variables)] token: impl Into<V::Token>,
369        slice: &'a mut [V::Scalar],
370    ) -> Self {
371        assert!(
372            slice.len() >= V::width(),
373            "slice must be at least as wide as the vector"
374        );
375        Self {
376            slice,
377            phantom: PhantomData,
378        }
379    }
380
381    /// Returns the number of overlapping vectors.
382    ///
383    /// Equal to `slice.len() - V::width() + 1`.
384    #[inline]
385    pub fn len(&self) -> usize {
386        self.slice.len() - V::width() + 1
387    }
388
389    /// Returns the vector offset `index` into the slice of scalars.
390    #[inline]
391    pub fn get(&self, index: usize) -> Option<V> {
392        if index < self.len() {
393            Some(unsafe { self.get_unchecked(index) })
394        } else {
395            None
396        }
397    }
398
399    /// Returns the vector offset `index` into the slice of scalars.
400    ///
401    /// # Safety
402    /// Index must be less than `len()`, i.e. the underlying slice must be at least `index
403    /// + V::width()` long.
404    #[inline]
405    pub unsafe fn get_unchecked(&self, index: usize) -> V {
406        V::read_ptr(V::Token::new_unchecked(), self.slice.as_ptr().add(index))
407    }
408
409    /// Returns the mutable vector offset `index` into the slice of scalars.
410    #[inline]
411    pub fn get_mut(&'a mut self, index: usize) -> Option<RefMut<'a, V>> {
412        if index < self.len() {
413            Some(unsafe { self.get_unchecked_mut(index) })
414        } else {
415            None
416        }
417    }
418
419    /// Returns the mutable vector offset `index` into the slice of scalars.
420    ///
421    /// # Safety
422    /// Index must be less than `len()`, i.e. the underlying slice must be at least `index
423    /// + V::width()` long.
424    #[inline]
425    pub unsafe fn get_unchecked_mut(&'a mut self, index: usize) -> RefMut<'a, V> {
426        RefMut::new(
427            V::Token::new_unchecked(),
428            self.slice.as_mut_ptr().add(index),
429        )
430    }
431}
generic_simd/slice.rs

generic_simd/
slice.rs
