thermite/vector/
ops.rs

1#[rustfmt::skip]
2macro_rules! decl_binary_ops {
3    ($kind:ident $(: $unary:ident)?; $($trait_name:ident::$method_name:ident),*) => {paste::paste! {
4        #[doc = "Combination of masked binary operation traits for " $kind " operations."]
5        pub trait [<Masked $kind Ops>]<Mask, Rhs>: $($unary<Mask> +)? $([<$trait_name Masked>]<Mask, Rhs> + )* {}
6
7        #[doc = "Combination of masked assignment binary operation traits for " $kind " operations."]
8        pub trait [<AssignMasked $kind Ops>]<Mask, Rhs>: $([<$trait_name AssignMasked>]<Mask, Rhs> + )* {}
9
10        impl<V, Mask, Rhs> [<Masked $kind Ops>]<Mask, Rhs> for V
11        where
12            $(V: $unary<Mask>,)?
13            V: $([<$trait_name Masked>]<Mask, Rhs> + )*
14        {}
15
16        impl<V, Mask, Rhs> [<AssignMasked $kind Ops>]<Mask, Rhs> for V
17        where
18            $(V: [<$trait_name AssignMasked>]<Mask, Rhs>),*
19        {}
20
21        $(
22            #[doc = "Masked variants of the [`" $trait_name "`] trait."]
23            pub trait [<$trait_name Masked>]<Mask, Rhs = Self>: $trait_name<Rhs> {
24                #[doc = "Computes [`" $trait_name "`] with `rhs` where `mask` is true."]
25                fn [<$method_name _c>](self, mask: Mask, rhs: Rhs) -> Self::Output;
26
27                #[doc = "Merges [`" $trait_name "`] with `src` using `mask`, returning `src` where mask is false."]
28                fn [<$method_name _m>](self, src: Self, mask: Mask, rhs: Rhs) -> Self::Output;
29
30                #[doc = "Computes [`" $trait_name "`] masked (zeroed where mask is false)."]
31                fn [<$method_name _z>](self, mask: Mask, rhs: Rhs) -> Self::Output;
32            }
33
34            #[doc = "Masked assignment variants of the [`" $trait_name "`] trait."]
35            pub trait [<$trait_name AssignMasked>]<Mask, Rhs = Self>: [<$trait_name Assign>]<Rhs> {
36                #[doc = "Computes [`" $trait_name "Assign`] with `rhs` where `mask` is true."]
37                fn [<$method_name _assign_c>](&mut self, mask: Mask, rhs: Rhs);
38
39                #[doc = "Merges [`" $trait_name "Assign`] with `src` using `mask`, assigning `src` where mask is false."]
40                fn [<$method_name _assign_m>](&mut self, src: Self, mask: Mask, rhs: Rhs);
41
42                #[doc = "Computes [`" $trait_name "Assign`] masked (zeroed where mask is false)."]
43                fn [<$method_name _assign_z>](&mut self, mask: Mask, rhs: Rhs);
44            }
45        )*
46    }};
47}
48
49macro_rules! decl_unary_ops {
50    ($($trait_name:ident::$method_name:ident),*) => {paste::paste! {$(
51        #[doc = "Masked variants of the [`" $trait_name "`] trait."]
52        pub trait [<$trait_name Masked>]<Mask>: $trait_name {
53            #[doc = "Computes [`" $trait_name "`] where `mask` is true, does nothing where false."]
54            fn [<$method_name _c>](self, mask: Mask) -> Self::Output;
55            #[doc = "Merges [`" $trait_name "`] with `src` using `mask`, returning `src` where mask is false."]
56            fn [<$method_name _m>](self, src: Self, mask: Mask) -> Self::Output;
57            #[doc = "Computes [`" $trait_name "`] masked (zeroed where mask is false)."]
58            fn [<$method_name _z>](self, mask: Mask) -> Self::Output;
59        }
60    )*}};
61}
62
63macro_rules! impl_binary_op {
64    ($trait_name:ident::$method_name:ident for $reg:ident, $rhs:ty) => {
65        paste::paste! {
66            impl<R: $reg + Register> $trait_name<$rhs> for Vector<R> {
67                type Output = Self;
68
69                #[inline(always)]
70                fn $method_name(self, rhs: $rhs) -> Self::Output {
71                    Vector(R::$method_name(self.0, rhs.0))
72                }
73            }
74
75            impl<R: $reg + Register> [<$trait_name Masked>]<Mask<R>, $rhs> for Vector<R> {
76                #[inline(always)]
77                fn [<$method_name _c>](self, mask: Mask<R>, rhs: $rhs) -> Self::Output {
78                    Vector(R::[<$method_name _c>](mask.0, self.0, rhs.0))
79                }
80
81                #[inline(always)]
82                fn [<$method_name _m>](self, src: Self, mask: Mask<R>, rhs: $rhs) -> Self::Output {
83                    Vector(R::[<$method_name _m>](src.0, mask.0, self.0, rhs.0))
84                }
85
86                #[inline(always)]
87                fn [<$method_name _z>](self, mask: Mask<R>, rhs: $rhs) -> Self::Output {
88                    Vector(R::[<$method_name _z>](mask.0, self.0, rhs.0))
89                }
90            }
91
92            impl<R: $reg + Register> [<$trait_name Assign>]<$rhs> for Vector<R> {
93                #[inline(always)]
94                fn [<$method_name _assign>](&mut self, rhs: $rhs) {
95                    self.0 = R::$method_name(self.0, rhs.0);
96                }
97            }
98
99            impl<R: $reg + Register> [<$trait_name AssignMasked>]<Mask<R>, $rhs> for Vector<R> {
100                #[inline(always)]
101                fn [<$method_name _assign_c>](&mut self, mask: Mask<R>, rhs: $rhs) {
102                    self.0 = R::[<$method_name _c>](mask.0, self.0, rhs.0);
103                }
104
105                #[inline(always)]
106                fn [<$method_name _assign_m>](&mut self, src: Self, mask: Mask<R>, rhs: $rhs) {
107                    self.0 = R::[<$method_name _m>](src.0, mask.0, self.0, rhs.0);
108                }
109
110                #[inline(always)]
111                fn [<$method_name _assign_z>](&mut self, mask: Mask<R>, rhs: $rhs) {
112                    self.0 = R::[<$method_name _z>](mask.0, self.0, rhs.0);
113                }
114            }
115        }
116    };
117}
118
119use core::ops::{
120    Add, AddAssign, BitAnd, BitAndAssign, BitOr, BitOrAssign, BitXor, BitXorAssign, Div, DivAssign, Mul, MulAssign,
121    Neg, Not, Rem, RemAssign, Shl, ShlAssign, Shr, ShrAssign, Sub, SubAssign,
122};
123
124use crate::{
125    Mask, Vector,
126    register::{BitshiftRegister, BitwiseRegister, FloatRegister, NumericRegister, Register, SignedRegister},
127};
128
129/// Trait for squaring a value: `self * self`
130///
131/// Some types are able to provide optimized implementations of squaring that are
132/// faster or more accurate than a simple multiplication with itself.
133pub trait Square {
134    /// The squared value. Not always `Self`: a type may widen to hold the product.
135    type Output;
136
137    /// Computes `self * self`.
138    fn square(self) -> Self::Output;
139}
140
141decl_binary_ops!(Num;
142    Add::add,
143    Sub::sub,
144    Mul::mul,
145    Div::div,
146    Rem::rem
147);
148
149impl_binary_op!(Add::add for NumericRegister, Self);
150impl_binary_op!(Sub::sub for NumericRegister, Self);
151impl_binary_op!(Mul::mul for NumericRegister, Self);
152impl_binary_op!(Div::div for NumericRegister, Self);
153impl_binary_op!(Rem::rem for NumericRegister, Self);
154
155/// Trait for the bitwise AND NOT operation: `self & !rhs`
156pub trait BitAndNot<Rhs = Self> {
157    /// The result of the AND NOT.
158    type Output;
159
160    /// Computes `self & !rhs`, one instruction on every SIMD backend.
161    ///
162    /// The **second** operand is the negated one here.
163    #[must_use]
164    fn bitandnot(self, rhs: Rhs) -> Self::Output;
165}
166
167/// Trait for the bitwise AND NOT assignment operation: `self &= !rhs`
168pub trait BitAndNotAssign<Rhs = Self> {
169    /// Assigns `self & !rhs` into `self`.
170    fn bitandnot_assign(&mut self, rhs: Rhs);
171}
172
173decl_binary_ops!(Bitwise;
174    BitAnd::bitand,
175    BitAndNot::bitandnot,
176    BitOr::bitor,
177    BitXor::bitxor
178);
179
180impl_binary_op!(BitAnd::bitand for BitwiseRegister, Self);
181impl_binary_op!(BitOr::bitor for BitwiseRegister, Self);
182impl_binary_op!(BitXor::bitxor for BitwiseRegister, Self);
183
184decl_binary_ops!(Bitshift;
185    Shl::shl,
186    Shr::shr
187);
188
189decl_unary_ops!(Not::not, Neg::neg, Square::square);
190
191impl<R: BitwiseRegister + Register> Not for Vector<R> {
192    type Output = Self;
193
194    #[inline(always)]
195    fn not(self) -> Self::Output {
196        Vector(R::not(self.0))
197    }
198}
199
200impl<R: BitwiseRegister + Register> NotMasked<Mask<R>> for Vector<R> {
201    #[inline(always)]
202    fn not_c(self, mask: Mask<R>) -> Self::Output {
203        Vector(R::not_c(mask.0, self.0))
204    }
205
206    #[inline(always)]
207    fn not_m(self, src: Self, mask: Mask<R>) -> Self::Output {
208        Vector(R::not_m(src.0, mask.0, self.0))
209    }
210
211    #[inline(always)]
212    fn not_z(self, mask: Mask<R>) -> Self::Output {
213        Vector(R::not_z(mask.0, self.0))
214    }
215}
216
217impl<R: SignedRegister> Neg for Vector<R> {
218    type Output = Self;
219
220    #[inline(always)]
221    fn neg(self) -> Self::Output {
222        Vector(R::neg(self.0))
223    }
224}
225
226impl<R: SignedRegister> NegMasked<Mask<R>> for Vector<R> {
227    #[inline(always)]
228    fn neg_c(self, mask: Mask<R>) -> Self::Output {
229        Vector(R::neg_c(mask.0, self.0))
230    }
231
232    #[inline(always)]
233    fn neg_m(self, src: Self, mask: Mask<R>) -> Self::Output {
234        Vector(R::neg_m(src.0, mask.0, self.0))
235    }
236
237    #[inline(always)]
238    fn neg_z(self, mask: Mask<R>) -> Self::Output {
239        Vector(R::neg_z(mask.0, self.0))
240    }
241}
242
243// NOTE: BitAndNot is unique in that the BitwiseRegister trait expects !lhs & rhs,
244// but since we want lhs & !rhs, the order of parameters is reversed here.
245impl<R: BitwiseRegister + Register> BitAndNot<Self> for Vector<R> {
246    type Output = Self;
247
248    #[inline(always)]
249    fn bitandnot(self, rhs: Self) -> Self::Output {
250        Vector(R::bitandnot(rhs.0, self.0))
251    }
252}
253
254impl<R: BitwiseRegister + Register> BitAndNotMasked<Mask<R>, Self> for Vector<R> {
255    #[inline(always)]
256    fn bitandnot_c(self, mask: Mask<R>, rhs: Self) -> Self::Output {
257        Vector(R::bitandnot_c(mask.0, rhs.0, self.0))
258    }
259
260    #[inline(always)]
261    fn bitandnot_m(self, src: Self, mask: Mask<R>, rhs: Self) -> Self::Output {
262        Vector(R::bitandnot_m(src.0, mask.0, rhs.0, self.0))
263    }
264
265    #[inline(always)]
266    fn bitandnot_z(self, mask: Mask<R>, rhs: Self) -> Self::Output {
267        Vector(R::bitandnot_z(mask.0, rhs.0, self.0))
268    }
269}
270
271impl<R: BitwiseRegister + Register> BitAndNotAssign<Self> for Vector<R> {
272    #[inline(always)]
273    fn bitandnot_assign(&mut self, rhs: Self) {
274        self.0 = R::bitandnot(rhs.0, self.0);
275    }
276}
277
278impl<R: BitwiseRegister + Register> BitAndNotAssignMasked<Mask<R>, Self> for Vector<R> {
279    #[inline(always)]
280    fn bitandnot_assign_c(&mut self, mask: Mask<R>, rhs: Self) {
281        self.0 = R::bitandnot_c(mask.0, rhs.0, self.0);
282    }
283
284    #[inline(always)]
285    fn bitandnot_assign_m(&mut self, src: Self, mask: Mask<R>, rhs: Self) {
286        self.0 = R::bitandnot_m(src.0, mask.0, rhs.0, self.0);
287    }
288
289    #[inline(always)]
290    fn bitandnot_assign_z(&mut self, mask: Mask<R>, rhs: Self) {
291        self.0 = R::bitandnot_z(mask.0, rhs.0, self.0);
292    }
293}
294
295macro_rules! mul_add_ext {
296    ($($(#[$meta:meta])* $name:ident),*) => {paste::paste! {
297        /// Trait for fused multiply-add operations. This contains all variants of
298        /// fused multiply-add, including guaranteed FMA and maybe FMA versions.
299        ///
300        /// If the platform does _not_ have native FMA support, the guaranteed FMA
301        /// will either use a fast compensated arithmetic algorithm, or fall back to
302        /// slow scalar evaluation to ensure correctness.
303        ///
304        /// This trait is superior to `num_traits::MulAdd`, but does not include
305        /// it as a supertrait due to the potential for multiple
306        /// conflicting implementation warnings.
307        pub trait MulAddExt<A = Self, B = Self> {
308            /// The result of the fused operation.
309            type Output;
310
311            /// Indicates whether the implementation uses true fused-multiply-add instructions.
312            ///
313            /// Non-`e` variants will always be accurate, regardless of this flag, but the `e` variants
314            /// will fallback to separate multiply and add operations if this is false.
315            const HAS_TRUE_FMA: bool;
316
317            $(
318                $(#[$meta])*
319                fn $name(self, a: A, b: B) -> Self::Output;
320            )*
321        }
322
323        impl<R: FloatRegister> MulAddExt<Self, Self> for Vector<R> {
324            type Output = Self;
325
326            const HAS_TRUE_FMA: bool = R::HAS_TRUE_FMA;
327
328            $(#[inline(always)] fn $name(self, a: Self, b: Self) -> Self::Output { Vector(R::$name(self.0, a.0, b.0)) } )*
329        }
330
331        /// Provides assignment variants of the fused multiply-add operations from [`MulAddExt`].
332        ///
333        /// Unlike regular assignment traits, this does require `MulAddExt` as a supertrait,
334        /// so we can access the associated `HAS_TRUE_FMA` constant.
335        pub trait MulAddAssignExt<A = Self, B = Self>: MulAddExt<A, B> {
336            $(
337                $(#[$meta])*
338                fn [<$name _assign>](&mut self, a: A, b: B);
339            )*
340        }
341
342        impl<R: FloatRegister> MulAddAssignExt<Self, Self> for Vector<R> {
343            $(#[inline(always)] fn [<$name _assign>](&mut self, a: Self, b: Self) { self.0 = R::$name(self.0, a.0, b.0); } )*
344        }
345
346        /// Provides masked variants of the fused multiply-add operations from [`MulAddExt`].
347        pub trait MulAddExtMasked<Mask, A = Self, B = Self>: MulAddExt<A, B> {
348            $(
349                $(#[$meta])*
350                #[doc = "\n\nThis variant computes [`" $name "`](MulAddExt::" $name ") with `a` and `b` where `mask` is true."]
351                fn [<$name _c>](self, mask: Mask, a: A, b: B) -> Self::Output;
352
353                $(#[$meta])*
354                #[doc = "\n\nThis variant merges [`" $name "`](MulAddExt::" $name ") with `src` using `mask`, returning `src` where mask is false."]
355                fn [<$name _m>](self, src: Self, mask: Mask, a: A, b: B) -> Self::Output;
356
357                $(#[$meta])*
358                #[doc = "\n\nThis variant computes [`" $name "`](MulAddExt::" $name ") masked (zeroed where mask is false)."]
359                fn [<$name _z>](self, mask: Mask, a: A, b: B) -> Self::Output;
360            )*
361        }
362
363        impl<R: FloatRegister> MulAddExtMasked<Mask<R>, Self, Self> for Vector<R> {
364            $(
365                #[inline(always)] fn [<$name _c>](self, mask: Mask<R>, a: Self, b: Self) -> Self::Output { Vector(R::[<$name _c>](mask.0, self.0, a.0, b.0)) }
366                #[inline(always)] fn [<$name _m>](self, src: Self, mask: Mask<R>, a: Self, b: Self) -> Self::Output { Vector(R::[<$name _m>](src.0, mask.0, self.0, a.0, b.0)) }
367                #[inline(always)] fn [<$name _z>](self, mask: Mask<R>, a: Self, b: Self) -> Self::Output { Vector(R::[<$name _z>](mask.0, self.0, a.0, b.0)) }
368            )*
369        }
370
371        /// Provides masked assignment variants of the fused multiply-add operations from [`MulAddExt`].
372        pub trait MulAddAssignExtMasked<Mask, A = Self, B = Self>: MulAddAssignExt<A, B> {
373            $(
374                $(#[$meta])*
375                #[doc = "\n\nThis variant computes [`" $name "_assign`](MulAddAssignExt::" $name "_assign) with `a` and `b` where `mask` is true."]
376                fn [<$name _assign_c>](&mut self, mask: Mask, a: A, b: B);
377
378                $(#[$meta])*
379                #[doc = "\n\nThis variant merges [`" $name "_assign`](MulAddAssignExt::" $name "_assign) with `src` using `mask`, assigning `src` where mask is false."]
380                fn [<$name _assign_m>](&mut self, src: Self, mask: Mask, a: A, b: B);
381
382                $(#[$meta])*
383                #[doc = "\n\nThis variant computes [`" $name "_assign`](MulAddAssignExt::" $name "_assign) masked (zeroed where mask is false)."]
384                fn [<$name _assign_z>](&mut self, mask: Mask, a: A, b: B);
385            )*
386        }
387
388        impl<R: FloatRegister> MulAddAssignExtMasked<Mask<R>, Self, Self> for Vector<R> {
389            $(
390                #[inline(always)] fn [<$name _assign_c>](&mut self, mask: Mask<R>, a: Self, b: Self) { self.0 = R::[<$name _c>](mask.0, self.0, a.0, b.0); }
391                #[inline(always)] fn [<$name _assign_m>](&mut self, src: Self, mask: Mask<R>, a: Self, b: Self) { self.0 = R::[<$name _m>](src.0, mask.0, self.0, a.0, b.0); }
392                #[inline(always)] fn [<$name _assign_z>](&mut self, mask: Mask<R>, a: Self, b: Self) { self.0 = R::[<$name _z>](mask.0, self.0, a.0, b.0); }
393            )*
394        }
395    }};
396}
397
398mul_add_ext! {
399    /// Guaranteed fused-multiply-add operation.
400    ///
401    /// If the target architecture does not support native FMA, this will use
402    /// either compensated arithmetic or slow scalar evaluation to ensure correctness.
403    mul_add,
404
405    /// Guaranteed fused-multiply-subtract operation.
406    ///
407    /// If the target architecture does not support native FMA, this will use
408    /// either compensated arithmetic or slow scalar evaluation to ensure correctness.
409    mul_sub,
410
411    /// Guaranteed fused-negated-multiply-add operation.
412    ///
413    /// If the target architecture does not support native FMA, this will use
414    /// either compensated arithmetic or slow scalar evaluation to ensure correctness.
415    nmul_add,
416
417    /// Guaranteed fused-negated-multiply-subtract operation.
418    ///
419    /// If the target architecture does not support native FMA, this will use
420    /// either compensated arithmetic or slow scalar evaluation to ensure correctness.
421    nmul_sub,
422
423    /// Fused-multiply-add operation where possible. May gracefully degrade to separate multiply and add
424    /// if the target architecture does not support native FMA.
425    mul_adde,
426
427    /// Fused-multiply-subtract operation where possible. May gracefully degrade to separate multiply and subtract
428    /// if the target architecture does not support native FMA.
429    mul_sube,
430
431    /// Fused-negated-multiply-add operation where possible. May gracefully degrade to separate multiply and add
432    /// if the target architecture does not support native FMA.
433    nmul_adde,
434
435    /// Fused-negated-multiply-subtract operation where possible. May gracefully degrade to separate multiply and subtract
436    /// if the target architecture does not support native FMA.
437    nmul_sube
438}
439
440/// Lane-alternating add/subtract operations for interleaved data (e.g. complex
441/// numbers packed as `[re, im, re, im, ...]`).
442///
443/// All three follow the x86 `ADDSUB`/`FMADDSUB` convention: **even lanes
444/// subtract, odd lanes add** (`fmsubadd` is the opposite parity). See
445/// [`FloatRegister::addsub`] for the
446/// interleaved complex-multiply lowering these are built for.
447pub trait AddSubExt: Sized {
448    /// The result of the lane-alternating operation.
449    type Output;
450
451    /// `[a0 - b0, a1 + b1, a2 - b2, ...]` - even lanes subtract, odd lanes add.
452    fn addsub(self, b: Self) -> Self::Output;
453
454    /// `[a0*b0 - c0, a1*b1 + c1, ...]` - fused multiply then [`addsub`](Self::addsub).
455    fn fmaddsub(self, b: Self, c: Self) -> Self::Output;
456
457    /// `[a0*b0 + c0, a1*b1 - c1, ...]` - fused multiply then subadd (opposite parity).
458    fn fmsubadd(self, b: Self, c: Self) -> Self::Output;
459}
460
461impl<R: FloatRegister> AddSubExt for Vector<R> {
462    type Output = Self;
463
464    #[inline(always)]
465    fn addsub(self, b: Self) -> Self {
466        Vector(R::addsub(self.0, b.0))
467    }
468
469    #[inline(always)]
470    fn fmaddsub(self, b: Self, c: Self) -> Self {
471        Vector(R::fmaddsub(self.0, b.0, c.0))
472    }
473
474    #[inline(always)]
475    fn fmsubadd(self, b: Self, c: Self) -> Self {
476        Vector(R::fmsubadd(self.0, b.0, c.0))
477    }
478}
479
480/// Masked variants of [`AddSubExt`] (mask-first argument order, matching the rest
481/// of Thermite's `_c`/`_m`/`_z` surface).
482pub trait AddSubExtMasked<Mask>: AddSubExt {
483    /// [`addsub`](AddSubExt::addsub) where `mask` is true, else `self`.
484    fn addsub_c(self, mask: Mask, b: Self) -> Self::Output;
485    /// [`addsub`](AddSubExt::addsub) where `mask` is true, else `src`.
486    fn addsub_m(self, src: Self, mask: Mask, b: Self) -> Self::Output;
487    /// [`addsub`](AddSubExt::addsub) where `mask` is true, else zero.
488    fn addsub_z(self, mask: Mask, b: Self) -> Self::Output;
489
490    /// [`fmaddsub`](AddSubExt::fmaddsub) where `mask` is true, else `self`.
491    fn fmaddsub_c(self, mask: Mask, b: Self, c: Self) -> Self::Output;
492    /// [`fmaddsub`](AddSubExt::fmaddsub) where `mask` is true, else `src`.
493    fn fmaddsub_m(self, src: Self, mask: Mask, b: Self, c: Self) -> Self::Output;
494    /// [`fmaddsub`](AddSubExt::fmaddsub) where `mask` is true, else zero.
495    fn fmaddsub_z(self, mask: Mask, b: Self, c: Self) -> Self::Output;
496
497    /// [`fmsubadd`](AddSubExt::fmsubadd) where `mask` is true, else `self`.
498    fn fmsubadd_c(self, mask: Mask, b: Self, c: Self) -> Self::Output;
499    /// [`fmsubadd`](AddSubExt::fmsubadd) where `mask` is true, else `src`.
500    fn fmsubadd_m(self, src: Self, mask: Mask, b: Self, c: Self) -> Self::Output;
501    /// [`fmsubadd`](AddSubExt::fmsubadd) where `mask` is true, else zero.
502    fn fmsubadd_z(self, mask: Mask, b: Self, c: Self) -> Self::Output;
503}
504
505impl<R: FloatRegister> AddSubExtMasked<Mask<R>> for Vector<R> {
506    #[inline(always)]
507    fn addsub_c(self, mask: Mask<R>, b: Self) -> Self {
508        Vector(R::addsub_c(mask.0, self.0, b.0))
509    }
510    #[inline(always)]
511    fn addsub_m(self, src: Self, mask: Mask<R>, b: Self) -> Self {
512        Vector(R::addsub_m(src.0, mask.0, self.0, b.0))
513    }
514    #[inline(always)]
515    fn addsub_z(self, mask: Mask<R>, b: Self) -> Self {
516        Vector(R::addsub_z(mask.0, self.0, b.0))
517    }
518
519    #[inline(always)]
520    fn fmaddsub_c(self, mask: Mask<R>, b: Self, c: Self) -> Self {
521        Vector(R::fmaddsub_c(mask.0, self.0, b.0, c.0))
522    }
523    #[inline(always)]
524    fn fmaddsub_m(self, src: Self, mask: Mask<R>, b: Self, c: Self) -> Self {
525        Vector(R::fmaddsub_m(src.0, mask.0, self.0, b.0, c.0))
526    }
527    #[inline(always)]
528    fn fmaddsub_z(self, mask: Mask<R>, b: Self, c: Self) -> Self {
529        Vector(R::fmaddsub_z(mask.0, self.0, b.0, c.0))
530    }
531
532    #[inline(always)]
533    fn fmsubadd_c(self, mask: Mask<R>, b: Self, c: Self) -> Self {
534        Vector(R::fmsubadd_c(mask.0, self.0, b.0, c.0))
535    }
536    #[inline(always)]
537    fn fmsubadd_m(self, src: Self, mask: Mask<R>, b: Self, c: Self) -> Self {
538        Vector(R::fmsubadd_m(src.0, mask.0, self.0, b.0, c.0))
539    }
540    #[inline(always)]
541    fn fmsubadd_z(self, mask: Mask<R>, b: Self, c: Self) -> Self {
542        Vector(R::fmsubadd_z(mask.0, self.0, b.0, c.0))
543    }
544}
545
546// Vector shifts
547
548impl<R: BitshiftRegister> Shl<Vector<R::Unsigned>> for Vector<R> {
549    type Output = Self;
550
551    #[inline(always)]
552    fn shl(self, rhs: Vector<R::Unsigned>) -> Self::Output {
553        Vector(R::shlv(self.0, rhs.0))
554    }
555}
556
557impl<R: BitshiftRegister> ShlMasked<Mask<R>, Vector<R::Unsigned>> for Vector<R> {
558    #[inline(always)]
559    fn shl_c(self, mask: Mask<R>, rhs: Vector<R::Unsigned>) -> Self::Output {
560        Vector(R::shlv_c(mask.0, self.0, rhs.0))
561    }
562
563    #[inline(always)]
564    fn shl_m(self, src: Self, mask: Mask<R>, rhs: Vector<R::Unsigned>) -> Self::Output {
565        Vector(R::shlv_m(src.0, mask.0, self.0, rhs.0))
566    }
567
568    #[inline(always)]
569    fn shl_z(self, mask: Mask<R>, rhs: Vector<R::Unsigned>) -> Self::Output {
570        Vector(R::shlv_z(mask.0, self.0, rhs.0))
571    }
572}
573
574impl<R: BitshiftRegister> Shr<Vector<R::Unsigned>> for Vector<R> {
575    type Output = Self;
576
577    #[inline(always)]
578    fn shr(self, rhs: Vector<R::Unsigned>) -> Self::Output {
579        Vector(R::shrv(self.0, rhs.0))
580    }
581}
582
583impl<R: BitshiftRegister> ShrMasked<Mask<R>, Vector<R::Unsigned>> for Vector<R> {
584    #[inline(always)]
585    fn shr_c(self, mask: Mask<R>, rhs: Vector<R::Unsigned>) -> Self::Output {
586        Vector(R::shrv_c(mask.0, self.0, rhs.0))
587    }
588
589    #[inline(always)]
590    fn shr_m(self, src: Self, mask: Mask<R>, rhs: Vector<R::Unsigned>) -> Self::Output {
591        Vector(R::shrv_m(src.0, mask.0, self.0, rhs.0))
592    }
593
594    #[inline(always)]
595    fn shr_z(self, mask: Mask<R>, rhs: Vector<R::Unsigned>) -> Self::Output {
596        Vector(R::shrv_z(mask.0, self.0, rhs.0))
597    }
598}
599
600impl<R: BitshiftRegister> ShlAssign<Vector<R::Unsigned>> for Vector<R> {
601    #[inline(always)]
602    fn shl_assign(&mut self, rhs: Vector<R::Unsigned>) {
603        self.0 = R::shlv(self.0, rhs.0);
604    }
605}
606
607impl<R: BitshiftRegister> ShlAssignMasked<Mask<R>, Vector<R::Unsigned>> for Vector<R> {
608    #[inline(always)]
609    fn shl_assign_c(&mut self, mask: Mask<R>, rhs: Vector<R::Unsigned>) {
610        self.0 = R::shlv_c(mask.0, self.0, rhs.0);
611    }
612
613    #[inline(always)]
614    fn shl_assign_m(&mut self, src: Self, mask: Mask<R>, rhs: Vector<R::Unsigned>) {
615        self.0 = R::shlv_m(src.0, mask.0, self.0, rhs.0);
616    }
617
618    #[inline(always)]
619    fn shl_assign_z(&mut self, mask: Mask<R>, rhs: Vector<R::Unsigned>) {
620        self.0 = R::shlv_z(mask.0, self.0, rhs.0);
621    }
622}
623
624impl<R: BitshiftRegister> ShrAssign<Vector<R::Unsigned>> for Vector<R> {
625    #[inline(always)]
626    fn shr_assign(&mut self, rhs: Vector<R::Unsigned>) {
627        self.0 = R::shrv(self.0, rhs.0);
628    }
629}
630
631impl<R: BitshiftRegister> ShrAssignMasked<Mask<R>, Vector<R::Unsigned>> for Vector<R> {
632    #[inline(always)]
633    fn shr_assign_c(&mut self, mask: Mask<R>, rhs: Vector<R::Unsigned>) {
634        self.0 = R::shrv_c(mask.0, self.0, rhs.0);
635    }
636
637    #[inline(always)]
638    fn shr_assign_m(&mut self, src: Self, mask: Mask<R>, rhs: Vector<R::Unsigned>) {
639        self.0 = R::shrv_m(src.0, mask.0, self.0, rhs.0);
640    }
641
642    #[inline(always)]
643    fn shr_assign_z(&mut self, mask: Mask<R>, rhs: Vector<R::Unsigned>) {
644        self.0 = R::shrv_z(mask.0, self.0, rhs.0);
645    }
646}
647
648// Scalar shifts
649
650impl<R: BitshiftRegister> Shl<u32> for Vector<R> {
651    type Output = Self;
652
653    #[inline(always)]
654    fn shl(self, rhs: u32) -> Self::Output {
655        Vector(R::shl(self.0, rhs))
656    }
657}
658
659impl<R: BitshiftRegister> ShlMasked<Mask<R>, u32> for Vector<R> {
660    #[inline(always)]
661    fn shl_c(self, mask: Mask<R>, rhs: u32) -> Self::Output {
662        Vector(R::shl_c(mask.0, self.0, rhs))
663    }
664
665    #[inline(always)]
666    fn shl_m(self, src: Self, mask: Mask<R>, rhs: u32) -> Self::Output {
667        Vector(R::shl_m(src.0, mask.0, self.0, rhs))
668    }
669
670    #[inline(always)]
671    fn shl_z(self, mask: Mask<R>, rhs: u32) -> Self::Output {
672        Vector(R::shl_z(mask.0, self.0, rhs))
673    }
674}
675
676impl<R: BitshiftRegister> Shr<u32> for Vector<R> {
677    type Output = Self;
678
679    #[inline(always)]
680    fn shr(self, rhs: u32) -> Self::Output {
681        Vector(R::shr(self.0, rhs))
682    }
683}
684
685impl<R: BitshiftRegister> ShrMasked<Mask<R>, u32> for Vector<R> {
686    #[inline(always)]
687    fn shr_c(self, mask: Mask<R>, rhs: u32) -> Self::Output {
688        Vector(R::shr_c(mask.0, self.0, rhs))
689    }
690
691    #[inline(always)]
692    fn shr_m(self, src: Self, mask: Mask<R>, rhs: u32) -> Self::Output {
693        Vector(R::shr_m(src.0, mask.0, self.0, rhs))
694    }
695
696    #[inline(always)]
697    fn shr_z(self, mask: Mask<R>, rhs: u32) -> Self::Output {
698        Vector(R::shr_z(mask.0, self.0, rhs))
699    }
700}
701
702impl<R: BitshiftRegister> ShlAssign<u32> for Vector<R> {
703    #[inline(always)]
704    fn shl_assign(&mut self, rhs: u32) {
705        self.0 = R::shl(self.0, rhs);
706    }
707}
708
709impl<R: BitshiftRegister> ShlAssignMasked<Mask<R>, u32> for Vector<R> {
710    #[inline(always)]
711    fn shl_assign_c(&mut self, mask: Mask<R>, rhs: u32) {
712        self.0 = R::shl_c(mask.0, self.0, rhs);
713    }
714
715    #[inline(always)]
716    fn shl_assign_m(&mut self, src: Self, mask: Mask<R>, rhs: u32) {
717        self.0 = R::shl_m(src.0, mask.0, self.0, rhs);
718    }
719
720    #[inline(always)]
721    fn shl_assign_z(&mut self, mask: Mask<R>, rhs: u32) {
722        self.0 = R::shl_z(mask.0, self.0, rhs);
723    }
724}
725
726impl<R: BitshiftRegister> ShrAssign<u32> for Vector<R> {
727    #[inline(always)]
728    fn shr_assign(&mut self, rhs: u32) {
729        self.0 = R::shr(self.0, rhs);
730    }
731}
732
733impl<R: BitshiftRegister> ShrAssignMasked<Mask<R>, u32> for Vector<R> {
734    #[inline(always)]
735    fn shr_assign_c(&mut self, mask: Mask<R>, rhs: u32) {
736        self.0 = R::shr_c(mask.0, self.0, rhs);
737    }
738
739    #[inline(always)]
740    fn shr_assign_m(&mut self, src: Self, mask: Mask<R>, rhs: u32) {
741        self.0 = R::shr_m(src.0, mask.0, self.0, rhs);
742    }
743
744    #[inline(always)]
745    fn shr_assign_z(&mut self, mask: Mask<R>, rhs: u32) {
746        self.0 = R::shr_z(mask.0, self.0, rhs);
747    }
748}
749
750macro_rules! impl_square {
751    ($($ty:ty),*) => {$(
752        impl Square for $ty {
753            type Output = Self;
754
755            #[inline(always)]
756            fn square(self) -> Self::Output {
757                self * self
758            }
759        }
760    )*};
761}
762
763impl_square!(i8, i16, i32, i64, i128, isize, u8, u16, u32, u64, u128, usize, f32, f64);
thermite/vector/ops.rs

thermite/vector/
ops.rs
