macerator/backend/x86/
mod.rs

1#![allow(
2    clippy::missing_transmute_annotations,
3    clippy::useless_transmute,
4    clippy::transmute_int_to_float,
5    unused_unsafe
6)]
7// Lint can't detect the build script version check
8#![cfg_attr(all(avx512, not(avx512_nightly)), allow(incompatible_msrv))]
9
10pub mod v2;
11pub mod v3;
12#[cfg(avx512)]
13pub mod v4;
14
15pub use v2::V2;
16pub use v3::V3;
17#[cfg(avx512)]
18pub use v4::V4;
19#[cfg(fp16)]
20pub use v4::V4FP16;
21
22macro_rules! lanes {
23    ($($bits: literal),*) => {
24        $(paste! {
25            #[inline(always)]
26            fn [<lanes $bits>]() -> usize {
27                WIDTH / $bits
28            }
29        })*
30    };
31}
32pub(crate) use lanes;
33
34macro_rules! impl_binop_scalar {
35    ($func: ident, $intrinsic: path, $($ty: ty),*) => {
36        $(paste! {
37            #[inline(always)]
38            fn [<$func _ $ty>](a: Self::Register, b: Self::Register) -> Self::Register {
39                const LANES: usize = WIDTH / (8 * size_of::<$ty>());
40                let a: [$ty; LANES] = cast!(a);
41                let b: [$ty; LANES] = cast!(b);
42                let mut out = [$ty::default(); LANES];
43
44                for i in 0..LANES {
45                    out[i] = $intrinsic(a[i], b[i]);
46                }
47                cast!(out)
48            }
49            #[inline(always)]
50            fn [<$func _ $ty _supported>]() -> bool {
51                false
52            }
53        })*
54    };
55}
56pub(crate) use impl_binop_scalar;
57
58macro_rules! impl_unop_scalar {
59    ($func: ident, $intrinsic: path, $($ty: ty),*) => {
60        $(paste! {
61            #[inline(always)]
62            fn [<$func _ $ty>](a: Self::Register) -> Self::Register {
63                const LANES: usize = WIDTH / (8 * size_of::<$ty>());
64                let a: [$ty; LANES] = cast!(a);
65                let mut out = [$ty::default(); LANES];
66
67                for i in 0..LANES {
68                    out[i] = a[i].$intrinsic();
69                }
70                cast!(out)
71            }
72            #[inline(always)]
73            fn [<$func _ $ty _supported>]() -> bool {
74                false
75            }
76        })*
77    };
78}
79pub(crate) use impl_unop_scalar;
80
81macro_rules! impl_reduce_scalar {
82    ($func: ident, $intrinsic: path, $($ty: ty),*) => {
83        $(paste! {
84            #[inline(always)]
85            fn [<$func _ $ty>](a: Self::Register) -> $ty {
86                const LANES: usize = WIDTH / (8 * size_of::<$ty>());
87                let a: [$ty; LANES] = cast!(a);
88                let mut out: $ty = a[0];
89
90                for i in 1..LANES {
91                    out = out.$intrinsic(a[i]);
92                }
93                out
94            }
95            #[inline(always)]
96            fn [<$func _ $ty _supported>]() -> bool {
97                false
98            }
99        })*
100    };
101}
102pub(crate) use impl_reduce_scalar;
103
104macro_rules! with_ty {
105    ($func: ident, f16) => {
106        paste!([<$func _ph>])
107    };
108    ($func: ident, f32) => {
109        paste!([<$func _ps>])
110    };
111    ($func: ident, f64) => {
112        paste!([<$func _pd>])
113    };
114    ($func: ident, $ty: ident) => {
115        paste!([<$func _ep $ty>])
116    }
117}
118pub(crate) use with_ty;
119
120macro_rules! with_ty_signless {
121    ($func: ident, u8) => {
122        with_ty!($func, i8)
123    };
124    ($func: ident, u16) => {
125        with_ty!($func, i16)
126    };
127    ($func: ident, u32) => {
128        with_ty!($func, i32)
129    };
130    ($func: ident, u64) => {
131        with_ty!($func, i64)
132    };
133    ($func: ident, $ty: ident) => {
134        with_ty!($func, $ty)
135    };
136}
137pub(crate) use with_ty_signless;
138
139macro_rules! impl_binop_signless {
140    ($func: ident, $intrinsic: ident, $($ty: ty),*) => {
141        $(paste! {
142            #[inline(always)]
143            fn [<$func _ $ty>](a: Self::Register, b: Self::Register) -> Self::Register {
144                cast!(with_ty_signless!($intrinsic, $ty)(cast!(a), cast!(b)))
145            }
146            #[inline(always)]
147            fn [<$func _ $ty _supported>]() -> bool {
148                true
149            }
150        })*
151    };
152}
153pub(crate) use impl_binop_signless;
154
155macro_rules! impl_binop {
156    ($func: ident, $intrinsic: ident, $($ty: ty),*) => {
157        $(paste! {
158            #[inline(always)]
159            fn [<$func _ $ty>](a: Self::Register, b: Self::Register) -> Self::Register {
160                cast!(with_ty!($intrinsic, $ty)(cast!(a), cast!(b)))
161            }
162            #[inline(always)]
163            fn [<$func _ $ty _supported>]() -> bool {
164                true
165            }
166        })*
167    };
168}
169pub(crate) use impl_binop;
170
171macro_rules! impl_binop_untyped {
172    ($func: ident, $intrinsic: ident) => {
173        paste! {
174            #[inline(always)]
175            fn $func(a: Self::Register, b: Self::Register) -> Self::Register {
176                cast!($intrinsic(cast!(a), cast!(b)))
177            }
178            #[inline(always)]
179            fn [<$func _supported>]() -> bool {
180                true
181            }
182        }
183    };
184}
185pub(crate) use impl_binop_untyped;
186
187macro_rules! impl_unop {
188    ($func: ident, $intrinsic: ident, $($ty: ty),*) => {
189        $(paste! {
190            #[inline(always)]
191            fn [<$func _ $ty>](a: Self::Register) -> Self::Register {
192                cast!(with_ty!($intrinsic, $ty)(cast!(a)))
193            }
194            #[inline(always)]
195            fn [<$func _ $ty _supported>]() -> bool {
196                true
197            }
198        })*
199    };
200}
201pub(crate) use impl_unop;
202
203macro_rules! impl_reduce {
204    ($func: ident, $intrinsic: ident, $($ty: ty),*) => {
205        $(paste! {
206            #[inline(always)]
207            fn [<$func _ $ty>](a: Self::Register) -> $ty {
208                unsafe { cast!(with_ty!($intrinsic, $ty)(cast!(a))) }
209            }
210            #[inline(always)]
211            fn [<$func _ $ty _supported>]() -> bool {
212                true
213            }
214        })*
215    };
216}
217pub(crate) use impl_reduce;
218
219macro_rules! impl_reduce_signless {
220    ($func: ident, $intrinsic: ident, $($ty: ty),*) => {
221        $(paste! {
222            #[inline(always)]
223            fn [<$func _ $ty>](a: Self::Register) -> $ty {
224                cast!(with_ty_signless!($intrinsic, $ty)(cast!(a)))
225            }
226            #[inline(always)]
227            fn [<$func _ $ty _supported>]() -> bool {
228                true
229            }
230        })*
231    };
232}
233pub(crate) use impl_reduce_signless;
234
235macro_rules! impl_cmp {
236    ($func: ident, $intrinsic: ident, $($ty: ty),*) => {
237        $(paste! {
238            #[inline(always)]
239            fn [<$func _ $ty>](a: Self::Register, b: Self::Register) -> <$ty as Scalar>::Mask<Self> {
240                cast!(with_ty_signless!($intrinsic, $ty)(cast!(a), cast!(b)))
241            }
242            #[inline(always)]
243            fn [<$func _ $ty _supported>]() -> bool {
244                true
245            }
246        })*
247    };
248}
249pub(crate) use impl_cmp;
macerator/backend/x86/mod.rs

macerator/backend/x86/
mod.rs
