1#[cfg(all(feature = "libm", not(feature = "std")))]
7#[allow(
8 unused_imports,
9 reason = "Generated backends use different subsets of these helpers"
10)]
11use crate::math::FloatExt as _;
12use crate::math::ScalarExt as _;
13use crate::{Level, arch_types::ArchTypes, prelude::*, seal::Seal};
14use crate::{
15 f32x4, f32x8, f32x16, f64x2, f64x4, f64x8, i8x16, i8x32, i8x64, i16x8, i16x16, i16x32, i32x4,
16 i32x8, i32x16, i64x2, i64x4, i64x8, mask8x16, mask8x32, mask8x64, mask16x8, mask16x16,
17 mask16x32, mask32x4, mask32x8, mask32x16, mask64x2, mask64x4, mask64x8, u8x16, u8x32, u8x64,
18 u16x8, u16x16, u16x32, u32x4, u32x8, u32x16, u64x2, u64x4, u64x8,
19};
20use core::ops::*;
21#[doc = "A token for scalar fallback SIMD, representing the \"fallback\" level."]
22#[doc = "# Browsing the documentation"]
23#[doc = "The method list on this struct is very verbose."]
24#[doc = "A better way to browse the docs is by looking at vector types such as [`u32x4`], [`f32x4`] or [`mask32x4`]."]
25#[doc = "They include all the operations listed here, and also provide some additional convenience methods."]
26#[derive(Clone, Copy, Debug)]
27pub struct Fallback {
28 _private: (),
29}
30impl Fallback {
31 #[doc = r" Create a scalar fallback token."]
32 #[inline]
33 pub const fn new() -> Self {
34 Self { _private: () }
35 }
36}
37impl Seal for Fallback {}
38impl ExtractToken for Fallback {
39 type S = Self;
40 #[inline]
41 fn token(&self) -> Self {
42 *self
43 }
44}
45impl ArchTypes for Fallback {
46 type f32x4 = crate::support::Aligned128<[f32; 4usize]>;
47 type i8x16 = crate::support::Aligned128<[i8; 16usize]>;
48 type u8x16 = crate::support::Aligned128<[u8; 16usize]>;
49 type mask8x16 = crate::support::Aligned128<[i8; 16usize]>;
50 type i16x8 = crate::support::Aligned128<[i16; 8usize]>;
51 type u16x8 = crate::support::Aligned128<[u16; 8usize]>;
52 type mask16x8 = crate::support::Aligned128<[i16; 8usize]>;
53 type i32x4 = crate::support::Aligned128<[i32; 4usize]>;
54 type u32x4 = crate::support::Aligned128<[u32; 4usize]>;
55 type mask32x4 = crate::support::Aligned128<[i32; 4usize]>;
56 type f64x2 = crate::support::Aligned128<[f64; 2usize]>;
57 type i64x2 = crate::support::Aligned128<[i64; 2usize]>;
58 type u64x2 = crate::support::Aligned128<[u64; 2usize]>;
59 type mask64x2 = crate::support::Aligned128<[i64; 2usize]>;
60 type f32x8 = crate::support::Aligned256<[f32; 8usize]>;
61 type i8x32 = crate::support::Aligned256<[i8; 32usize]>;
62 type u8x32 = crate::support::Aligned256<[u8; 32usize]>;
63 type mask8x32 = crate::support::Aligned256<[i8; 32usize]>;
64 type i16x16 = crate::support::Aligned256<[i16; 16usize]>;
65 type u16x16 = crate::support::Aligned256<[u16; 16usize]>;
66 type mask16x16 = crate::support::Aligned256<[i16; 16usize]>;
67 type i32x8 = crate::support::Aligned256<[i32; 8usize]>;
68 type u32x8 = crate::support::Aligned256<[u32; 8usize]>;
69 type mask32x8 = crate::support::Aligned256<[i32; 8usize]>;
70 type f64x4 = crate::support::Aligned256<[f64; 4usize]>;
71 type i64x4 = crate::support::Aligned256<[i64; 4usize]>;
72 type u64x4 = crate::support::Aligned256<[u64; 4usize]>;
73 type mask64x4 = crate::support::Aligned256<[i64; 4usize]>;
74 type f32x16 = crate::support::Aligned512<[f32; 16usize]>;
75 type i8x64 = crate::support::Aligned512<[i8; 64usize]>;
76 type u8x64 = crate::support::Aligned512<[u8; 64usize]>;
77 type mask8x64 = crate::support::Aligned512<[i8; 64usize]>;
78 type i16x32 = crate::support::Aligned512<[i16; 32usize]>;
79 type u16x32 = crate::support::Aligned512<[u16; 32usize]>;
80 type mask16x32 = crate::support::Aligned512<[i16; 32usize]>;
81 type i32x16 = crate::support::Aligned512<[i32; 16usize]>;
82 type u32x16 = crate::support::Aligned512<[u32; 16usize]>;
83 type mask32x16 = crate::support::Aligned512<[i32; 16usize]>;
84 type f64x8 = crate::support::Aligned512<[f64; 8usize]>;
85 type i64x8 = crate::support::Aligned512<[i64; 8usize]>;
86 type u64x8 = crate::support::Aligned512<[u64; 8usize]>;
87 type mask64x8 = crate::support::Aligned512<[i64; 8usize]>;
88}
89impl Simd for Fallback {
90 type f32s = f32x4<Self>;
91 type f64s = f64x2<Self>;
92 type u8s = u8x16<Self>;
93 type i8s = i8x16<Self>;
94 type u16s = u16x8<Self>;
95 type i16s = i16x8<Self>;
96 type u32s = u32x4<Self>;
97 type i32s = i32x4<Self>;
98 type u64s = u64x2<Self>;
99 type i64s = i64x2<Self>;
100 type mask8s = mask8x16<Self>;
101 type mask16s = mask16x8<Self>;
102 type mask32s = mask32x4<Self>;
103 type mask64s = mask64x2<Self>;
104 #[inline(always)]
105 fn level(self) -> Level {
106 #[cfg(feature = "force_support_fallback")]
107 return Level::Fallback(self);
108 #[cfg(not(feature = "force_support_fallback"))]
109 Level::baseline()
110 }
111 #[inline]
112 fn vectorize<F: FnOnce() -> R, R>(self, f: F) -> R {
113 #[inline]
114 fn vectorize_inner<F: FnOnce() -> R, R>(f: F) -> R {
115 f()
116 }
117 vectorize_inner(f)
118 }
119 #[inline(always)]
120 fn abs_f32x4(self, a: f32x4<Self>) -> f32x4<Self> {
121 [
122 f32::abs(a[0usize]),
123 f32::abs(a[1usize]),
124 f32::abs(a[2usize]),
125 f32::abs(a[3usize]),
126 ]
127 .simd_into(self)
128 }
129 #[inline(always)]
130 fn splat_f32x4(self, val: f32) -> f32x4<Self> {
131 [val; 4usize].simd_into(self)
132 }
133 #[inline(always)]
134 fn reverse_f32x4(self, a: f32x4<Self>) -> f32x4<Self> {
135 [a[3usize], a[2usize], a[1usize], a[0usize]].simd_into(self)
136 }
137 #[inline(always)]
138 fn slide_f32x4<const SHIFT: usize>(self, a: f32x4<Self>, b: f32x4<Self>) -> f32x4<Self> {
139 let mut dest = [Default::default(); 4usize];
140 dest[..4usize - SHIFT].copy_from_slice(&a.val.0[SHIFT..]);
141 dest[4usize - SHIFT..].copy_from_slice(&b.val.0[..SHIFT]);
142 dest.simd_into(self)
143 }
144 #[inline(always)]
145 fn compress_f32x4(self, values: f32x4<Self>, mask: mask32x4<Self>) -> f32x4<Self> {
146 let mask = mask8x16 {
147 val: crate::transmute::checked_transmute_copy(&mask.val),
148 simd: self,
149 };
150 Bytes::from_bytes(self.compress_u8x16(Bytes::to_bytes(values), mask))
151 }
152 #[inline(always)]
153 fn compress_merge_f32x4(
154 self,
155 values: f32x4<Self>,
156 mask: mask32x4<Self>,
157 merge: f32x4<Self>,
158 ) -> f32x4<Self> {
159 let mask = mask8x16 {
160 val: crate::transmute::checked_transmute_copy(&mask.val),
161 simd: self,
162 };
163 Bytes::from_bytes(self.compress_merge_u8x16(
164 Bytes::to_bytes(values),
165 mask,
166 Bytes::to_bytes(merge),
167 ))
168 }
169 #[inline(always)]
170 fn expand_f32x4(self, values: f32x4<Self>, mask: mask32x4<Self>) -> f32x4<Self> {
171 let mask = mask8x16 {
172 val: crate::transmute::checked_transmute_copy(&mask.val),
173 simd: self,
174 };
175 Bytes::from_bytes(self.expand_u8x16(Bytes::to_bytes(values), mask))
176 }
177 #[inline(always)]
178 fn expand_merge_f32x4(
179 self,
180 values: f32x4<Self>,
181 mask: mask32x4<Self>,
182 merge: f32x4<Self>,
183 ) -> f32x4<Self> {
184 let mask = mask8x16 {
185 val: crate::transmute::checked_transmute_copy(&mask.val),
186 simd: self,
187 };
188 Bytes::from_bytes(self.expand_merge_u8x16(
189 Bytes::to_bytes(values),
190 mask,
191 Bytes::to_bytes(merge),
192 ))
193 }
194 #[inline(always)]
195 fn neg_f32x4(self, a: f32x4<Self>) -> f32x4<Self> {
196 [
197 f32::neg(a[0usize]),
198 f32::neg(a[1usize]),
199 f32::neg(a[2usize]),
200 f32::neg(a[3usize]),
201 ]
202 .simd_into(self)
203 }
204 #[inline(always)]
205 fn sqrt_f32x4(self, a: f32x4<Self>) -> f32x4<Self> {
206 [
207 f32::sqrt(a[0usize]),
208 f32::sqrt(a[1usize]),
209 f32::sqrt(a[2usize]),
210 f32::sqrt(a[3usize]),
211 ]
212 .simd_into(self)
213 }
214 #[inline(always)]
215 fn approximate_recip_f32x4(self, a: f32x4<Self>) -> f32x4<Self> {
216 1.0 / a
217 }
218 #[inline(always)]
219 fn add_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> f32x4<Self> {
220 [
221 f32::add(a[0usize], &b[0usize]),
222 f32::add(a[1usize], &b[1usize]),
223 f32::add(a[2usize], &b[2usize]),
224 f32::add(a[3usize], &b[3usize]),
225 ]
226 .simd_into(self)
227 }
228 #[inline(always)]
229 fn sub_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> f32x4<Self> {
230 [
231 f32::sub(a[0usize], &b[0usize]),
232 f32::sub(a[1usize], &b[1usize]),
233 f32::sub(a[2usize], &b[2usize]),
234 f32::sub(a[3usize], &b[3usize]),
235 ]
236 .simd_into(self)
237 }
238 #[inline(always)]
239 fn mul_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> f32x4<Self> {
240 [
241 f32::mul(a[0usize], &b[0usize]),
242 f32::mul(a[1usize], &b[1usize]),
243 f32::mul(a[2usize], &b[2usize]),
244 f32::mul(a[3usize], &b[3usize]),
245 ]
246 .simd_into(self)
247 }
248 #[inline(always)]
249 fn div_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> f32x4<Self> {
250 [
251 f32::div(a[0usize], &b[0usize]),
252 f32::div(a[1usize], &b[1usize]),
253 f32::div(a[2usize], &b[2usize]),
254 f32::div(a[3usize], &b[3usize]),
255 ]
256 .simd_into(self)
257 }
258 #[inline(always)]
259 fn copysign_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> f32x4<Self> {
260 [
261 f32::copysign(a[0usize], b[0usize]),
262 f32::copysign(a[1usize], b[1usize]),
263 f32::copysign(a[2usize], b[2usize]),
264 f32::copysign(a[3usize], b[3usize]),
265 ]
266 .simd_into(self)
267 }
268 #[inline(always)]
269 fn reduce_max_f32x4(self, a: f32x4<Self>) -> f32 {
270 let reduced: [f32; 2usize] = [
271 f32::max(a[0usize], a[1usize]),
272 f32::max(a[2usize], a[3usize]),
273 ];
274 let reduced: [f32; 1usize] = [f32::max(reduced[0usize], reduced[1usize])];
275 reduced[0]
276 }
277 #[inline(always)]
278 fn reduce_min_f32x4(self, a: f32x4<Self>) -> f32 {
279 let reduced: [f32; 2usize] = [
280 f32::min(a[0usize], a[1usize]),
281 f32::min(a[2usize], a[3usize]),
282 ];
283 let reduced: [f32; 1usize] = [f32::min(reduced[0usize], reduced[1usize])];
284 reduced[0]
285 }
286 #[inline(always)]
287 fn reduce_max_precise_f32x4(self, a: f32x4<Self>) -> f32 {
288 let reduced: [f32; 2usize] = [
289 f32::max(a[0usize], a[1usize]),
290 f32::max(a[2usize], a[3usize]),
291 ];
292 let reduced: [f32; 1usize] = [f32::max(reduced[0usize], reduced[1usize])];
293 reduced[0]
294 }
295 #[inline(always)]
296 fn reduce_min_precise_f32x4(self, a: f32x4<Self>) -> f32 {
297 let reduced: [f32; 2usize] = [
298 f32::min(a[0usize], a[1usize]),
299 f32::min(a[2usize], a[3usize]),
300 ];
301 let reduced: [f32; 1usize] = [f32::min(reduced[0usize], reduced[1usize])];
302 reduced[0]
303 }
304 #[inline(always)]
305 fn reduce_sum_f32x4(self, a: f32x4<Self>) -> f32 {
306 let add_level_0: [f32; 2usize] = [a[0usize] + a[1usize], a[2usize] + a[3usize]];
307 let add_level_1: [f32; 1usize] = [add_level_0[0usize] + add_level_0[1usize]];
308 add_level_1[0]
309 }
310 #[inline(always)]
311 fn reduce_product_f32x4(self, a: f32x4<Self>) -> f32 {
312 let mul_level_0: [f32; 2usize] = [a[0usize] * a[1usize], a[2usize] * a[3usize]];
313 let mul_level_1: [f32; 1usize] = [mul_level_0[0usize] * mul_level_0[1usize]];
314 mul_level_1[0]
315 }
316 #[inline(always)]
317 fn max_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> f32x4<Self> {
318 [
319 f32::max(a[0usize], b[0usize]),
320 f32::max(a[1usize], b[1usize]),
321 f32::max(a[2usize], b[2usize]),
322 f32::max(a[3usize], b[3usize]),
323 ]
324 .simd_into(self)
325 }
326 #[inline(always)]
327 fn min_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> f32x4<Self> {
328 [
329 f32::min(a[0usize], b[0usize]),
330 f32::min(a[1usize], b[1usize]),
331 f32::min(a[2usize], b[2usize]),
332 f32::min(a[3usize], b[3usize]),
333 ]
334 .simd_into(self)
335 }
336 #[inline(always)]
337 fn max_precise_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> f32x4<Self> {
338 [
339 f32::max(a[0usize], b[0usize]),
340 f32::max(a[1usize], b[1usize]),
341 f32::max(a[2usize], b[2usize]),
342 f32::max(a[3usize], b[3usize]),
343 ]
344 .simd_into(self)
345 }
346 #[inline(always)]
347 fn min_precise_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> f32x4<Self> {
348 [
349 f32::min(a[0usize], b[0usize]),
350 f32::min(a[1usize], b[1usize]),
351 f32::min(a[2usize], b[2usize]),
352 f32::min(a[3usize], b[3usize]),
353 ]
354 .simd_into(self)
355 }
356 #[inline(always)]
357 fn clamp_f32x4(self, a: f32x4<Self>, b: f32x4<Self>, c: f32x4<Self>) -> f32x4<Self> {
358 [
359 f32::clamp_debug(a[0usize], b[0usize], c[0usize]),
360 f32::clamp_debug(a[1usize], b[1usize], c[1usize]),
361 f32::clamp_debug(a[2usize], b[2usize], c[2usize]),
362 f32::clamp_debug(a[3usize], b[3usize], c[3usize]),
363 ]
364 .simd_into(self)
365 }
366 #[inline(always)]
367 fn simd_eq_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> mask32x4<Self> {
368 [
369 -(f32::eq(&a[0usize], &b[0usize]) as i32),
370 -(f32::eq(&a[1usize], &b[1usize]) as i32),
371 -(f32::eq(&a[2usize], &b[2usize]) as i32),
372 -(f32::eq(&a[3usize], &b[3usize]) as i32),
373 ]
374 .simd_into(self)
375 }
376 #[inline(always)]
377 fn simd_lt_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> mask32x4<Self> {
378 [
379 -(f32::lt(&a[0usize], &b[0usize]) as i32),
380 -(f32::lt(&a[1usize], &b[1usize]) as i32),
381 -(f32::lt(&a[2usize], &b[2usize]) as i32),
382 -(f32::lt(&a[3usize], &b[3usize]) as i32),
383 ]
384 .simd_into(self)
385 }
386 #[inline(always)]
387 fn simd_le_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> mask32x4<Self> {
388 [
389 -(f32::le(&a[0usize], &b[0usize]) as i32),
390 -(f32::le(&a[1usize], &b[1usize]) as i32),
391 -(f32::le(&a[2usize], &b[2usize]) as i32),
392 -(f32::le(&a[3usize], &b[3usize]) as i32),
393 ]
394 .simd_into(self)
395 }
396 #[inline(always)]
397 fn zip_low_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> f32x4<Self> {
398 [a[0usize], b[0usize], a[1usize], b[1usize]].simd_into(self)
399 }
400 #[inline(always)]
401 fn zip_high_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> f32x4<Self> {
402 [a[2usize], b[2usize], a[3usize], b[3usize]].simd_into(self)
403 }
404 #[inline(always)]
405 fn unzip_low_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> f32x4<Self> {
406 [a[0usize], a[2usize], b[0usize], b[2usize]].simd_into(self)
407 }
408 #[inline(always)]
409 fn unzip_high_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> f32x4<Self> {
410 [a[1usize], a[3usize], b[1usize], b[3usize]].simd_into(self)
411 }
412 #[inline(always)]
413 fn interleave_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> (f32x4<Self>, f32x4<Self>) {
414 (self.zip_low_f32x4(a, b), self.zip_high_f32x4(a, b))
415 }
416 #[inline(always)]
417 fn deinterleave_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> (f32x4<Self>, f32x4<Self>) {
418 (self.unzip_low_f32x4(a, b), self.unzip_high_f32x4(a, b))
419 }
420 #[inline(always)]
421 fn mul_add_f32x4(self, a: f32x4<Self>, b: f32x4<Self>, c: f32x4<Self>) -> f32x4<Self> {
422 a.mul(b).add(c)
423 }
424 #[inline(always)]
425 fn mul_add_precise_f32x4(self, a: f32x4<Self>, b: f32x4<Self>, c: f32x4<Self>) -> f32x4<Self> {
426 crate::math::mul_add_precise_f32x4(self, a, b, c)
427 }
428 #[inline(always)]
429 fn mul_sub_f32x4(self, a: f32x4<Self>, b: f32x4<Self>, c: f32x4<Self>) -> f32x4<Self> {
430 a.mul(b).sub(c)
431 }
432 #[inline(always)]
433 fn mul_sub_precise_f32x4(self, a: f32x4<Self>, b: f32x4<Self>, c: f32x4<Self>) -> f32x4<Self> {
434 self.mul_add_precise_f32x4(a, b, -c)
435 }
436 #[inline(always)]
437 fn floor_f32x4(self, a: f32x4<Self>) -> f32x4<Self> {
438 [
439 f32::floor(a[0usize]),
440 f32::floor(a[1usize]),
441 f32::floor(a[2usize]),
442 f32::floor(a[3usize]),
443 ]
444 .simd_into(self)
445 }
446 #[inline(always)]
447 fn ceil_f32x4(self, a: f32x4<Self>) -> f32x4<Self> {
448 [
449 f32::ceil(a[0usize]),
450 f32::ceil(a[1usize]),
451 f32::ceil(a[2usize]),
452 f32::ceil(a[3usize]),
453 ]
454 .simd_into(self)
455 }
456 #[inline(always)]
457 fn round_ties_even_f32x4(self, a: f32x4<Self>) -> f32x4<Self> {
458 [
459 f32::round_ties_even(a[0usize]),
460 f32::round_ties_even(a[1usize]),
461 f32::round_ties_even(a[2usize]),
462 f32::round_ties_even(a[3usize]),
463 ]
464 .simd_into(self)
465 }
466 #[inline(always)]
467 fn round_f32x4(self, a: f32x4<Self>) -> f32x4<Self> {
468 [
469 f32::round(a[0usize]),
470 f32::round(a[1usize]),
471 f32::round(a[2usize]),
472 f32::round(a[3usize]),
473 ]
474 .simd_into(self)
475 }
476 #[inline(always)]
477 fn fract_f32x4(self, a: f32x4<Self>) -> f32x4<Self> {
478 [
479 f32::fract(a[0usize]),
480 f32::fract(a[1usize]),
481 f32::fract(a[2usize]),
482 f32::fract(a[3usize]),
483 ]
484 .simd_into(self)
485 }
486 #[inline(always)]
487 fn trunc_f32x4(self, a: f32x4<Self>) -> f32x4<Self> {
488 [
489 f32::trunc(a[0usize]),
490 f32::trunc(a[1usize]),
491 f32::trunc(a[2usize]),
492 f32::trunc(a[3usize]),
493 ]
494 .simd_into(self)
495 }
496 #[inline(always)]
497 fn to_degrees_f32x4(self, a: f32x4<Self>) -> f32x4<Self> {
498 [
499 f32::to_degrees(a[0usize]),
500 f32::to_degrees(a[1usize]),
501 f32::to_degrees(a[2usize]),
502 f32::to_degrees(a[3usize]),
503 ]
504 .simd_into(self)
505 }
506 #[inline(always)]
507 fn to_radians_f32x4(self, a: f32x4<Self>) -> f32x4<Self> {
508 [
509 f32::to_radians(a[0usize]),
510 f32::to_radians(a[1usize]),
511 f32::to_radians(a[2usize]),
512 f32::to_radians(a[3usize]),
513 ]
514 .simd_into(self)
515 }
516 #[inline(always)]
517 fn is_nan_f32x4(self, a: f32x4<Self>) -> mask32x4<Self> {
518 a.simd_ne(a)
519 }
520 #[inline(always)]
521 fn is_infinite_f32x4(self, a: f32x4<Self>) -> mask32x4<Self> {
522 a.abs().simd_eq(f32::INFINITY)
523 }
524 #[inline(always)]
525 fn is_finite_f32x4(self, a: f32x4<Self>) -> mask32x4<Self> {
526 a.abs().simd_lt(f32::INFINITY)
527 }
528 #[inline(always)]
529 fn is_subnormal_f32x4(self, a: f32x4<Self>) -> mask32x4<Self> {
530 let i: i32x4<Self> = a.bitcast();
531 (i & 0x7F80_0000_i32).simd_eq(0) & (i & 0x007F_FFFF_i32).simd_ne(0)
532 }
533 #[inline(always)]
534 fn is_normal_f32x4(self, a: f32x4<Self>) -> mask32x4<Self> {
535 let i: i32x4<Self> = a.bitcast();
536 let exp = i & 0x7F80_0000_i32;
537 exp.simd_ne(0) & exp.simd_ne(0x7F80_0000_i32)
538 }
539 #[inline(always)]
540 fn is_sign_positive_f32x4(self, a: f32x4<Self>) -> mask32x4<Self> {
541 let i: i32x4<Self> = a.bitcast();
542 (i & i32::MIN).simd_eq(0)
543 }
544 #[inline(always)]
545 fn is_sign_negative_f32x4(self, a: f32x4<Self>) -> mask32x4<Self> {
546 !a.is_sign_positive()
547 }
548 #[inline(always)]
549 fn select_f32x4(self, a: mask32x4<Self>, b: f32x4<Self>, c: f32x4<Self>) -> f32x4<Self> {
550 [
551 if a.val.0[0usize] != 0 {
552 b[0usize]
553 } else {
554 c[0usize]
555 },
556 if a.val.0[1usize] != 0 {
557 b[1usize]
558 } else {
559 c[1usize]
560 },
561 if a.val.0[2usize] != 0 {
562 b[2usize]
563 } else {
564 c[2usize]
565 },
566 if a.val.0[3usize] != 0 {
567 b[3usize]
568 } else {
569 c[3usize]
570 },
571 ]
572 .simd_into(self)
573 }
574 #[inline(always)]
575 fn combine_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> f32x8<Self> {
576 let mut result = [0.0; 8usize];
577 result[0..4usize].copy_from_slice(&a.val.0);
578 result[4usize..8usize].copy_from_slice(&b.val.0);
579 result.simd_into(self)
580 }
581 #[inline(always)]
582 fn load_four_interleaved_f32x4(self, src: &[f32; 16usize]) -> [f32x4<Self>; 4usize] {
583 [
584 [src[0usize], src[4usize], src[8usize], src[12usize]].simd_into(self),
585 [src[1usize], src[5usize], src[9usize], src[13usize]].simd_into(self),
586 [src[2usize], src[6usize], src[10usize], src[14usize]].simd_into(self),
587 [src[3usize], src[7usize], src[11usize], src[15usize]].simd_into(self),
588 ]
589 }
590 #[inline(always)]
591 fn store_four_interleaved_f32x4(
592 self,
593 vectors: [f32x4<Self>; 4usize],
594 dest: &mut [f32; 16usize],
595 ) -> () {
596 *dest = [
597 vectors[0usize][0usize],
598 vectors[1usize][0usize],
599 vectors[2usize][0usize],
600 vectors[3usize][0usize],
601 vectors[0usize][1usize],
602 vectors[1usize][1usize],
603 vectors[2usize][1usize],
604 vectors[3usize][1usize],
605 vectors[0usize][2usize],
606 vectors[1usize][2usize],
607 vectors[2usize][2usize],
608 vectors[3usize][2usize],
609 vectors[0usize][3usize],
610 vectors[1usize][3usize],
611 vectors[2usize][3usize],
612 vectors[3usize][3usize],
613 ];
614 }
615 #[inline(always)]
616 fn widen_f32x4(self, a: f32x4<Self>) -> (f64x2<Self>, f64x2<Self>) {
617 (
618 [a[0usize] as f64, a[1usize] as f64].simd_into(self),
619 [a[2usize] as f64, a[3usize] as f64].simd_into(self),
620 )
621 }
622 #[inline(always)]
623 fn cvt_u32_f32x4(self, a: f32x4<Self>) -> u32x4<Self> {
624 [
625 a[0usize] as u32,
626 a[1usize] as u32,
627 a[2usize] as u32,
628 a[3usize] as u32,
629 ]
630 .simd_into(self)
631 }
632 #[inline(always)]
633 fn cvt_u32_precise_f32x4(self, a: f32x4<Self>) -> u32x4<Self> {
634 [
635 a[0usize] as u32,
636 a[1usize] as u32,
637 a[2usize] as u32,
638 a[3usize] as u32,
639 ]
640 .simd_into(self)
641 }
642 #[inline(always)]
643 fn cvt_i32_f32x4(self, a: f32x4<Self>) -> i32x4<Self> {
644 [
645 a[0usize] as i32,
646 a[1usize] as i32,
647 a[2usize] as i32,
648 a[3usize] as i32,
649 ]
650 .simd_into(self)
651 }
652 #[inline(always)]
653 fn cvt_i32_precise_f32x4(self, a: f32x4<Self>) -> i32x4<Self> {
654 [
655 a[0usize] as i32,
656 a[1usize] as i32,
657 a[2usize] as i32,
658 a[3usize] as i32,
659 ]
660 .simd_into(self)
661 }
662 #[inline(always)]
663 fn abs_i8x16(self, a: i8x16<Self>) -> i8x16<Self> {
664 [
665 i8::wrapping_abs(a[0usize]),
666 i8::wrapping_abs(a[1usize]),
667 i8::wrapping_abs(a[2usize]),
668 i8::wrapping_abs(a[3usize]),
669 i8::wrapping_abs(a[4usize]),
670 i8::wrapping_abs(a[5usize]),
671 i8::wrapping_abs(a[6usize]),
672 i8::wrapping_abs(a[7usize]),
673 i8::wrapping_abs(a[8usize]),
674 i8::wrapping_abs(a[9usize]),
675 i8::wrapping_abs(a[10usize]),
676 i8::wrapping_abs(a[11usize]),
677 i8::wrapping_abs(a[12usize]),
678 i8::wrapping_abs(a[13usize]),
679 i8::wrapping_abs(a[14usize]),
680 i8::wrapping_abs(a[15usize]),
681 ]
682 .simd_into(self)
683 }
684 #[inline(always)]
685 fn splat_i8x16(self, val: i8) -> i8x16<Self> {
686 [val; 16usize].simd_into(self)
687 }
688 #[inline(always)]
689 fn reverse_i8x16(self, a: i8x16<Self>) -> i8x16<Self> {
690 [
691 a[15usize], a[14usize], a[13usize], a[12usize], a[11usize], a[10usize], a[9usize],
692 a[8usize], a[7usize], a[6usize], a[5usize], a[4usize], a[3usize], a[2usize], a[1usize],
693 a[0usize],
694 ]
695 .simd_into(self)
696 }
697 #[inline(always)]
698 fn slide_i8x16<const SHIFT: usize>(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
699 let mut dest = [Default::default(); 16usize];
700 dest[..16usize - SHIFT].copy_from_slice(&a.val.0[SHIFT..]);
701 dest[16usize - SHIFT..].copy_from_slice(&b.val.0[..SHIFT]);
702 dest.simd_into(self)
703 }
704 #[inline(always)]
705 fn compress_i8x16(self, values: i8x16<Self>, mask: mask8x16<Self>) -> i8x16<Self> {
706 let mask = mask8x16 {
707 val: crate::transmute::checked_transmute_copy(&mask.val),
708 simd: self,
709 };
710 Bytes::from_bytes(self.compress_u8x16(Bytes::to_bytes(values), mask))
711 }
712 #[inline(always)]
713 fn compress_merge_i8x16(
714 self,
715 values: i8x16<Self>,
716 mask: mask8x16<Self>,
717 merge: i8x16<Self>,
718 ) -> i8x16<Self> {
719 let mask = mask8x16 {
720 val: crate::transmute::checked_transmute_copy(&mask.val),
721 simd: self,
722 };
723 Bytes::from_bytes(self.compress_merge_u8x16(
724 Bytes::to_bytes(values),
725 mask,
726 Bytes::to_bytes(merge),
727 ))
728 }
729 #[inline(always)]
730 fn expand_i8x16(self, values: i8x16<Self>, mask: mask8x16<Self>) -> i8x16<Self> {
731 let mask = mask8x16 {
732 val: crate::transmute::checked_transmute_copy(&mask.val),
733 simd: self,
734 };
735 Bytes::from_bytes(self.expand_u8x16(Bytes::to_bytes(values), mask))
736 }
737 #[inline(always)]
738 fn expand_merge_i8x16(
739 self,
740 values: i8x16<Self>,
741 mask: mask8x16<Self>,
742 merge: i8x16<Self>,
743 ) -> i8x16<Self> {
744 let mask = mask8x16 {
745 val: crate::transmute::checked_transmute_copy(&mask.val),
746 simd: self,
747 };
748 Bytes::from_bytes(self.expand_merge_u8x16(
749 Bytes::to_bytes(values),
750 mask,
751 Bytes::to_bytes(merge),
752 ))
753 }
754 #[inline(always)]
755 fn count_ones_i8x16(self, a: i8x16<Self>) -> i8x16<Self> {
756 [
757 i8::try_from(a[0usize].count_ones()).unwrap(),
758 i8::try_from(a[1usize].count_ones()).unwrap(),
759 i8::try_from(a[2usize].count_ones()).unwrap(),
760 i8::try_from(a[3usize].count_ones()).unwrap(),
761 i8::try_from(a[4usize].count_ones()).unwrap(),
762 i8::try_from(a[5usize].count_ones()).unwrap(),
763 i8::try_from(a[6usize].count_ones()).unwrap(),
764 i8::try_from(a[7usize].count_ones()).unwrap(),
765 i8::try_from(a[8usize].count_ones()).unwrap(),
766 i8::try_from(a[9usize].count_ones()).unwrap(),
767 i8::try_from(a[10usize].count_ones()).unwrap(),
768 i8::try_from(a[11usize].count_ones()).unwrap(),
769 i8::try_from(a[12usize].count_ones()).unwrap(),
770 i8::try_from(a[13usize].count_ones()).unwrap(),
771 i8::try_from(a[14usize].count_ones()).unwrap(),
772 i8::try_from(a[15usize].count_ones()).unwrap(),
773 ]
774 .simd_into(self)
775 }
776 #[inline(always)]
777 fn count_zeros_i8x16(self, a: i8x16<Self>) -> i8x16<Self> {
778 [
779 i8::try_from(a[0usize].count_zeros()).unwrap(),
780 i8::try_from(a[1usize].count_zeros()).unwrap(),
781 i8::try_from(a[2usize].count_zeros()).unwrap(),
782 i8::try_from(a[3usize].count_zeros()).unwrap(),
783 i8::try_from(a[4usize].count_zeros()).unwrap(),
784 i8::try_from(a[5usize].count_zeros()).unwrap(),
785 i8::try_from(a[6usize].count_zeros()).unwrap(),
786 i8::try_from(a[7usize].count_zeros()).unwrap(),
787 i8::try_from(a[8usize].count_zeros()).unwrap(),
788 i8::try_from(a[9usize].count_zeros()).unwrap(),
789 i8::try_from(a[10usize].count_zeros()).unwrap(),
790 i8::try_from(a[11usize].count_zeros()).unwrap(),
791 i8::try_from(a[12usize].count_zeros()).unwrap(),
792 i8::try_from(a[13usize].count_zeros()).unwrap(),
793 i8::try_from(a[14usize].count_zeros()).unwrap(),
794 i8::try_from(a[15usize].count_zeros()).unwrap(),
795 ]
796 .simd_into(self)
797 }
798 #[inline(always)]
799 fn add_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
800 [
801 i8::wrapping_add(a[0usize], b[0usize]),
802 i8::wrapping_add(a[1usize], b[1usize]),
803 i8::wrapping_add(a[2usize], b[2usize]),
804 i8::wrapping_add(a[3usize], b[3usize]),
805 i8::wrapping_add(a[4usize], b[4usize]),
806 i8::wrapping_add(a[5usize], b[5usize]),
807 i8::wrapping_add(a[6usize], b[6usize]),
808 i8::wrapping_add(a[7usize], b[7usize]),
809 i8::wrapping_add(a[8usize], b[8usize]),
810 i8::wrapping_add(a[9usize], b[9usize]),
811 i8::wrapping_add(a[10usize], b[10usize]),
812 i8::wrapping_add(a[11usize], b[11usize]),
813 i8::wrapping_add(a[12usize], b[12usize]),
814 i8::wrapping_add(a[13usize], b[13usize]),
815 i8::wrapping_add(a[14usize], b[14usize]),
816 i8::wrapping_add(a[15usize], b[15usize]),
817 ]
818 .simd_into(self)
819 }
820 #[inline(always)]
821 fn saturating_add_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
822 [
823 i8::saturating_add(a[0usize], b[0usize]),
824 i8::saturating_add(a[1usize], b[1usize]),
825 i8::saturating_add(a[2usize], b[2usize]),
826 i8::saturating_add(a[3usize], b[3usize]),
827 i8::saturating_add(a[4usize], b[4usize]),
828 i8::saturating_add(a[5usize], b[5usize]),
829 i8::saturating_add(a[6usize], b[6usize]),
830 i8::saturating_add(a[7usize], b[7usize]),
831 i8::saturating_add(a[8usize], b[8usize]),
832 i8::saturating_add(a[9usize], b[9usize]),
833 i8::saturating_add(a[10usize], b[10usize]),
834 i8::saturating_add(a[11usize], b[11usize]),
835 i8::saturating_add(a[12usize], b[12usize]),
836 i8::saturating_add(a[13usize], b[13usize]),
837 i8::saturating_add(a[14usize], b[14usize]),
838 i8::saturating_add(a[15usize], b[15usize]),
839 ]
840 .simd_into(self)
841 }
842 #[inline(always)]
843 fn sub_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
844 [
845 i8::wrapping_sub(a[0usize], b[0usize]),
846 i8::wrapping_sub(a[1usize], b[1usize]),
847 i8::wrapping_sub(a[2usize], b[2usize]),
848 i8::wrapping_sub(a[3usize], b[3usize]),
849 i8::wrapping_sub(a[4usize], b[4usize]),
850 i8::wrapping_sub(a[5usize], b[5usize]),
851 i8::wrapping_sub(a[6usize], b[6usize]),
852 i8::wrapping_sub(a[7usize], b[7usize]),
853 i8::wrapping_sub(a[8usize], b[8usize]),
854 i8::wrapping_sub(a[9usize], b[9usize]),
855 i8::wrapping_sub(a[10usize], b[10usize]),
856 i8::wrapping_sub(a[11usize], b[11usize]),
857 i8::wrapping_sub(a[12usize], b[12usize]),
858 i8::wrapping_sub(a[13usize], b[13usize]),
859 i8::wrapping_sub(a[14usize], b[14usize]),
860 i8::wrapping_sub(a[15usize], b[15usize]),
861 ]
862 .simd_into(self)
863 }
864 #[inline(always)]
865 fn saturating_sub_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
866 [
867 i8::saturating_sub(a[0usize], b[0usize]),
868 i8::saturating_sub(a[1usize], b[1usize]),
869 i8::saturating_sub(a[2usize], b[2usize]),
870 i8::saturating_sub(a[3usize], b[3usize]),
871 i8::saturating_sub(a[4usize], b[4usize]),
872 i8::saturating_sub(a[5usize], b[5usize]),
873 i8::saturating_sub(a[6usize], b[6usize]),
874 i8::saturating_sub(a[7usize], b[7usize]),
875 i8::saturating_sub(a[8usize], b[8usize]),
876 i8::saturating_sub(a[9usize], b[9usize]),
877 i8::saturating_sub(a[10usize], b[10usize]),
878 i8::saturating_sub(a[11usize], b[11usize]),
879 i8::saturating_sub(a[12usize], b[12usize]),
880 i8::saturating_sub(a[13usize], b[13usize]),
881 i8::saturating_sub(a[14usize], b[14usize]),
882 i8::saturating_sub(a[15usize], b[15usize]),
883 ]
884 .simd_into(self)
885 }
886 #[inline(always)]
887 fn mul_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
888 [
889 i8::wrapping_mul(a[0usize], b[0usize]),
890 i8::wrapping_mul(a[1usize], b[1usize]),
891 i8::wrapping_mul(a[2usize], b[2usize]),
892 i8::wrapping_mul(a[3usize], b[3usize]),
893 i8::wrapping_mul(a[4usize], b[4usize]),
894 i8::wrapping_mul(a[5usize], b[5usize]),
895 i8::wrapping_mul(a[6usize], b[6usize]),
896 i8::wrapping_mul(a[7usize], b[7usize]),
897 i8::wrapping_mul(a[8usize], b[8usize]),
898 i8::wrapping_mul(a[9usize], b[9usize]),
899 i8::wrapping_mul(a[10usize], b[10usize]),
900 i8::wrapping_mul(a[11usize], b[11usize]),
901 i8::wrapping_mul(a[12usize], b[12usize]),
902 i8::wrapping_mul(a[13usize], b[13usize]),
903 i8::wrapping_mul(a[14usize], b[14usize]),
904 i8::wrapping_mul(a[15usize], b[15usize]),
905 ]
906 .simd_into(self)
907 }
908 #[inline(always)]
909 fn and_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
910 [
911 i8::bitand(a[0usize], &b[0usize]),
912 i8::bitand(a[1usize], &b[1usize]),
913 i8::bitand(a[2usize], &b[2usize]),
914 i8::bitand(a[3usize], &b[3usize]),
915 i8::bitand(a[4usize], &b[4usize]),
916 i8::bitand(a[5usize], &b[5usize]),
917 i8::bitand(a[6usize], &b[6usize]),
918 i8::bitand(a[7usize], &b[7usize]),
919 i8::bitand(a[8usize], &b[8usize]),
920 i8::bitand(a[9usize], &b[9usize]),
921 i8::bitand(a[10usize], &b[10usize]),
922 i8::bitand(a[11usize], &b[11usize]),
923 i8::bitand(a[12usize], &b[12usize]),
924 i8::bitand(a[13usize], &b[13usize]),
925 i8::bitand(a[14usize], &b[14usize]),
926 i8::bitand(a[15usize], &b[15usize]),
927 ]
928 .simd_into(self)
929 }
930 #[inline(always)]
931 fn or_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
932 [
933 i8::bitor(a[0usize], &b[0usize]),
934 i8::bitor(a[1usize], &b[1usize]),
935 i8::bitor(a[2usize], &b[2usize]),
936 i8::bitor(a[3usize], &b[3usize]),
937 i8::bitor(a[4usize], &b[4usize]),
938 i8::bitor(a[5usize], &b[5usize]),
939 i8::bitor(a[6usize], &b[6usize]),
940 i8::bitor(a[7usize], &b[7usize]),
941 i8::bitor(a[8usize], &b[8usize]),
942 i8::bitor(a[9usize], &b[9usize]),
943 i8::bitor(a[10usize], &b[10usize]),
944 i8::bitor(a[11usize], &b[11usize]),
945 i8::bitor(a[12usize], &b[12usize]),
946 i8::bitor(a[13usize], &b[13usize]),
947 i8::bitor(a[14usize], &b[14usize]),
948 i8::bitor(a[15usize], &b[15usize]),
949 ]
950 .simd_into(self)
951 }
952 #[inline(always)]
953 fn xor_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
954 [
955 i8::bitxor(a[0usize], &b[0usize]),
956 i8::bitxor(a[1usize], &b[1usize]),
957 i8::bitxor(a[2usize], &b[2usize]),
958 i8::bitxor(a[3usize], &b[3usize]),
959 i8::bitxor(a[4usize], &b[4usize]),
960 i8::bitxor(a[5usize], &b[5usize]),
961 i8::bitxor(a[6usize], &b[6usize]),
962 i8::bitxor(a[7usize], &b[7usize]),
963 i8::bitxor(a[8usize], &b[8usize]),
964 i8::bitxor(a[9usize], &b[9usize]),
965 i8::bitxor(a[10usize], &b[10usize]),
966 i8::bitxor(a[11usize], &b[11usize]),
967 i8::bitxor(a[12usize], &b[12usize]),
968 i8::bitxor(a[13usize], &b[13usize]),
969 i8::bitxor(a[14usize], &b[14usize]),
970 i8::bitxor(a[15usize], &b[15usize]),
971 ]
972 .simd_into(self)
973 }
974 #[inline(always)]
975 fn not_i8x16(self, a: i8x16<Self>) -> i8x16<Self> {
976 [
977 i8::not(a[0usize]),
978 i8::not(a[1usize]),
979 i8::not(a[2usize]),
980 i8::not(a[3usize]),
981 i8::not(a[4usize]),
982 i8::not(a[5usize]),
983 i8::not(a[6usize]),
984 i8::not(a[7usize]),
985 i8::not(a[8usize]),
986 i8::not(a[9usize]),
987 i8::not(a[10usize]),
988 i8::not(a[11usize]),
989 i8::not(a[12usize]),
990 i8::not(a[13usize]),
991 i8::not(a[14usize]),
992 i8::not(a[15usize]),
993 ]
994 .simd_into(self)
995 }
996 #[inline(always)]
997 fn shl_i8x16(self, a: i8x16<Self>, shift: u32) -> i8x16<Self> {
998 [
999 i8::wrapping_shl(a[0usize], shift),
1000 i8::wrapping_shl(a[1usize], shift),
1001 i8::wrapping_shl(a[2usize], shift),
1002 i8::wrapping_shl(a[3usize], shift),
1003 i8::wrapping_shl(a[4usize], shift),
1004 i8::wrapping_shl(a[5usize], shift),
1005 i8::wrapping_shl(a[6usize], shift),
1006 i8::wrapping_shl(a[7usize], shift),
1007 i8::wrapping_shl(a[8usize], shift),
1008 i8::wrapping_shl(a[9usize], shift),
1009 i8::wrapping_shl(a[10usize], shift),
1010 i8::wrapping_shl(a[11usize], shift),
1011 i8::wrapping_shl(a[12usize], shift),
1012 i8::wrapping_shl(a[13usize], shift),
1013 i8::wrapping_shl(a[14usize], shift),
1014 i8::wrapping_shl(a[15usize], shift),
1015 ]
1016 .simd_into(self)
1017 }
1018 #[inline(always)]
1019 fn shlv_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
1020 [
1021 i8::wrapping_shl(a[0usize], b[0usize] as u32),
1022 i8::wrapping_shl(a[1usize], b[1usize] as u32),
1023 i8::wrapping_shl(a[2usize], b[2usize] as u32),
1024 i8::wrapping_shl(a[3usize], b[3usize] as u32),
1025 i8::wrapping_shl(a[4usize], b[4usize] as u32),
1026 i8::wrapping_shl(a[5usize], b[5usize] as u32),
1027 i8::wrapping_shl(a[6usize], b[6usize] as u32),
1028 i8::wrapping_shl(a[7usize], b[7usize] as u32),
1029 i8::wrapping_shl(a[8usize], b[8usize] as u32),
1030 i8::wrapping_shl(a[9usize], b[9usize] as u32),
1031 i8::wrapping_shl(a[10usize], b[10usize] as u32),
1032 i8::wrapping_shl(a[11usize], b[11usize] as u32),
1033 i8::wrapping_shl(a[12usize], b[12usize] as u32),
1034 i8::wrapping_shl(a[13usize], b[13usize] as u32),
1035 i8::wrapping_shl(a[14usize], b[14usize] as u32),
1036 i8::wrapping_shl(a[15usize], b[15usize] as u32),
1037 ]
1038 .simd_into(self)
1039 }
1040 #[inline(always)]
1041 fn shr_i8x16(self, a: i8x16<Self>, shift: u32) -> i8x16<Self> {
1042 [
1043 i8::wrapping_shr(a[0usize], shift),
1044 i8::wrapping_shr(a[1usize], shift),
1045 i8::wrapping_shr(a[2usize], shift),
1046 i8::wrapping_shr(a[3usize], shift),
1047 i8::wrapping_shr(a[4usize], shift),
1048 i8::wrapping_shr(a[5usize], shift),
1049 i8::wrapping_shr(a[6usize], shift),
1050 i8::wrapping_shr(a[7usize], shift),
1051 i8::wrapping_shr(a[8usize], shift),
1052 i8::wrapping_shr(a[9usize], shift),
1053 i8::wrapping_shr(a[10usize], shift),
1054 i8::wrapping_shr(a[11usize], shift),
1055 i8::wrapping_shr(a[12usize], shift),
1056 i8::wrapping_shr(a[13usize], shift),
1057 i8::wrapping_shr(a[14usize], shift),
1058 i8::wrapping_shr(a[15usize], shift),
1059 ]
1060 .simd_into(self)
1061 }
1062 #[inline(always)]
1063 fn shrv_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
1064 [
1065 i8::wrapping_shr(a[0usize], b[0usize] as u32),
1066 i8::wrapping_shr(a[1usize], b[1usize] as u32),
1067 i8::wrapping_shr(a[2usize], b[2usize] as u32),
1068 i8::wrapping_shr(a[3usize], b[3usize] as u32),
1069 i8::wrapping_shr(a[4usize], b[4usize] as u32),
1070 i8::wrapping_shr(a[5usize], b[5usize] as u32),
1071 i8::wrapping_shr(a[6usize], b[6usize] as u32),
1072 i8::wrapping_shr(a[7usize], b[7usize] as u32),
1073 i8::wrapping_shr(a[8usize], b[8usize] as u32),
1074 i8::wrapping_shr(a[9usize], b[9usize] as u32),
1075 i8::wrapping_shr(a[10usize], b[10usize] as u32),
1076 i8::wrapping_shr(a[11usize], b[11usize] as u32),
1077 i8::wrapping_shr(a[12usize], b[12usize] as u32),
1078 i8::wrapping_shr(a[13usize], b[13usize] as u32),
1079 i8::wrapping_shr(a[14usize], b[14usize] as u32),
1080 i8::wrapping_shr(a[15usize], b[15usize] as u32),
1081 ]
1082 .simd_into(self)
1083 }
1084 #[inline(always)]
1085 fn reduce_max_i8x16(self, a: i8x16<Self>) -> i8 {
1086 let reduced: [i8; 8usize] = [
1087 i8::max(a[0usize], a[1usize]),
1088 i8::max(a[2usize], a[3usize]),
1089 i8::max(a[4usize], a[5usize]),
1090 i8::max(a[6usize], a[7usize]),
1091 i8::max(a[8usize], a[9usize]),
1092 i8::max(a[10usize], a[11usize]),
1093 i8::max(a[12usize], a[13usize]),
1094 i8::max(a[14usize], a[15usize]),
1095 ];
1096 let reduced: [i8; 4usize] = [
1097 i8::max(reduced[0usize], reduced[1usize]),
1098 i8::max(reduced[2usize], reduced[3usize]),
1099 i8::max(reduced[4usize], reduced[5usize]),
1100 i8::max(reduced[6usize], reduced[7usize]),
1101 ];
1102 let reduced: [i8; 2usize] = [
1103 i8::max(reduced[0usize], reduced[1usize]),
1104 i8::max(reduced[2usize], reduced[3usize]),
1105 ];
1106 let reduced: [i8; 1usize] = [i8::max(reduced[0usize], reduced[1usize])];
1107 reduced[0]
1108 }
1109 #[inline(always)]
1110 fn reduce_min_i8x16(self, a: i8x16<Self>) -> i8 {
1111 let reduced: [i8; 8usize] = [
1112 i8::min(a[0usize], a[1usize]),
1113 i8::min(a[2usize], a[3usize]),
1114 i8::min(a[4usize], a[5usize]),
1115 i8::min(a[6usize], a[7usize]),
1116 i8::min(a[8usize], a[9usize]),
1117 i8::min(a[10usize], a[11usize]),
1118 i8::min(a[12usize], a[13usize]),
1119 i8::min(a[14usize], a[15usize]),
1120 ];
1121 let reduced: [i8; 4usize] = [
1122 i8::min(reduced[0usize], reduced[1usize]),
1123 i8::min(reduced[2usize], reduced[3usize]),
1124 i8::min(reduced[4usize], reduced[5usize]),
1125 i8::min(reduced[6usize], reduced[7usize]),
1126 ];
1127 let reduced: [i8; 2usize] = [
1128 i8::min(reduced[0usize], reduced[1usize]),
1129 i8::min(reduced[2usize], reduced[3usize]),
1130 ];
1131 let reduced: [i8; 1usize] = [i8::min(reduced[0usize], reduced[1usize])];
1132 reduced[0]
1133 }
1134 #[inline(always)]
1135 fn reduce_sum_i8x16(self, a: i8x16<Self>) -> i8 {
1136 let add_level_0: [i8; 8usize] = [
1137 a[0usize].wrapping_add(a[1usize]),
1138 a[2usize].wrapping_add(a[3usize]),
1139 a[4usize].wrapping_add(a[5usize]),
1140 a[6usize].wrapping_add(a[7usize]),
1141 a[8usize].wrapping_add(a[9usize]),
1142 a[10usize].wrapping_add(a[11usize]),
1143 a[12usize].wrapping_add(a[13usize]),
1144 a[14usize].wrapping_add(a[15usize]),
1145 ];
1146 let add_level_1: [i8; 4usize] = [
1147 add_level_0[0usize].wrapping_add(add_level_0[1usize]),
1148 add_level_0[2usize].wrapping_add(add_level_0[3usize]),
1149 add_level_0[4usize].wrapping_add(add_level_0[5usize]),
1150 add_level_0[6usize].wrapping_add(add_level_0[7usize]),
1151 ];
1152 let add_level_2: [i8; 2usize] = [
1153 add_level_1[0usize].wrapping_add(add_level_1[1usize]),
1154 add_level_1[2usize].wrapping_add(add_level_1[3usize]),
1155 ];
1156 let add_level_3: [i8; 1usize] = [add_level_2[0usize].wrapping_add(add_level_2[1usize])];
1157 add_level_3[0]
1158 }
1159 #[inline(always)]
1160 fn reduce_product_i8x16(self, a: i8x16<Self>) -> i8 {
1161 let mul_level_0: [i8; 8usize] = [
1162 a[0usize].wrapping_mul(a[1usize]),
1163 a[2usize].wrapping_mul(a[3usize]),
1164 a[4usize].wrapping_mul(a[5usize]),
1165 a[6usize].wrapping_mul(a[7usize]),
1166 a[8usize].wrapping_mul(a[9usize]),
1167 a[10usize].wrapping_mul(a[11usize]),
1168 a[12usize].wrapping_mul(a[13usize]),
1169 a[14usize].wrapping_mul(a[15usize]),
1170 ];
1171 let mul_level_1: [i8; 4usize] = [
1172 mul_level_0[0usize].wrapping_mul(mul_level_0[1usize]),
1173 mul_level_0[2usize].wrapping_mul(mul_level_0[3usize]),
1174 mul_level_0[4usize].wrapping_mul(mul_level_0[5usize]),
1175 mul_level_0[6usize].wrapping_mul(mul_level_0[7usize]),
1176 ];
1177 let mul_level_2: [i8; 2usize] = [
1178 mul_level_1[0usize].wrapping_mul(mul_level_1[1usize]),
1179 mul_level_1[2usize].wrapping_mul(mul_level_1[3usize]),
1180 ];
1181 let mul_level_3: [i8; 1usize] = [mul_level_2[0usize].wrapping_mul(mul_level_2[1usize])];
1182 mul_level_3[0]
1183 }
1184 #[inline(always)]
1185 fn max_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
1186 [
1187 i8::max(a[0usize], b[0usize]),
1188 i8::max(a[1usize], b[1usize]),
1189 i8::max(a[2usize], b[2usize]),
1190 i8::max(a[3usize], b[3usize]),
1191 i8::max(a[4usize], b[4usize]),
1192 i8::max(a[5usize], b[5usize]),
1193 i8::max(a[6usize], b[6usize]),
1194 i8::max(a[7usize], b[7usize]),
1195 i8::max(a[8usize], b[8usize]),
1196 i8::max(a[9usize], b[9usize]),
1197 i8::max(a[10usize], b[10usize]),
1198 i8::max(a[11usize], b[11usize]),
1199 i8::max(a[12usize], b[12usize]),
1200 i8::max(a[13usize], b[13usize]),
1201 i8::max(a[14usize], b[14usize]),
1202 i8::max(a[15usize], b[15usize]),
1203 ]
1204 .simd_into(self)
1205 }
1206 #[inline(always)]
1207 fn min_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
1208 [
1209 i8::min(a[0usize], b[0usize]),
1210 i8::min(a[1usize], b[1usize]),
1211 i8::min(a[2usize], b[2usize]),
1212 i8::min(a[3usize], b[3usize]),
1213 i8::min(a[4usize], b[4usize]),
1214 i8::min(a[5usize], b[5usize]),
1215 i8::min(a[6usize], b[6usize]),
1216 i8::min(a[7usize], b[7usize]),
1217 i8::min(a[8usize], b[8usize]),
1218 i8::min(a[9usize], b[9usize]),
1219 i8::min(a[10usize], b[10usize]),
1220 i8::min(a[11usize], b[11usize]),
1221 i8::min(a[12usize], b[12usize]),
1222 i8::min(a[13usize], b[13usize]),
1223 i8::min(a[14usize], b[14usize]),
1224 i8::min(a[15usize], b[15usize]),
1225 ]
1226 .simd_into(self)
1227 }
1228 #[inline(always)]
1229 fn clamp_i8x16(self, a: i8x16<Self>, b: i8x16<Self>, c: i8x16<Self>) -> i8x16<Self> {
1230 [
1231 i8::clamp_debug(a[0usize], b[0usize], c[0usize]),
1232 i8::clamp_debug(a[1usize], b[1usize], c[1usize]),
1233 i8::clamp_debug(a[2usize], b[2usize], c[2usize]),
1234 i8::clamp_debug(a[3usize], b[3usize], c[3usize]),
1235 i8::clamp_debug(a[4usize], b[4usize], c[4usize]),
1236 i8::clamp_debug(a[5usize], b[5usize], c[5usize]),
1237 i8::clamp_debug(a[6usize], b[6usize], c[6usize]),
1238 i8::clamp_debug(a[7usize], b[7usize], c[7usize]),
1239 i8::clamp_debug(a[8usize], b[8usize], c[8usize]),
1240 i8::clamp_debug(a[9usize], b[9usize], c[9usize]),
1241 i8::clamp_debug(a[10usize], b[10usize], c[10usize]),
1242 i8::clamp_debug(a[11usize], b[11usize], c[11usize]),
1243 i8::clamp_debug(a[12usize], b[12usize], c[12usize]),
1244 i8::clamp_debug(a[13usize], b[13usize], c[13usize]),
1245 i8::clamp_debug(a[14usize], b[14usize], c[14usize]),
1246 i8::clamp_debug(a[15usize], b[15usize], c[15usize]),
1247 ]
1248 .simd_into(self)
1249 }
1250 #[inline(always)]
1251 fn simd_eq_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> mask8x16<Self> {
1252 [
1253 -(i8::eq(&a[0usize], &b[0usize]) as i8),
1254 -(i8::eq(&a[1usize], &b[1usize]) as i8),
1255 -(i8::eq(&a[2usize], &b[2usize]) as i8),
1256 -(i8::eq(&a[3usize], &b[3usize]) as i8),
1257 -(i8::eq(&a[4usize], &b[4usize]) as i8),
1258 -(i8::eq(&a[5usize], &b[5usize]) as i8),
1259 -(i8::eq(&a[6usize], &b[6usize]) as i8),
1260 -(i8::eq(&a[7usize], &b[7usize]) as i8),
1261 -(i8::eq(&a[8usize], &b[8usize]) as i8),
1262 -(i8::eq(&a[9usize], &b[9usize]) as i8),
1263 -(i8::eq(&a[10usize], &b[10usize]) as i8),
1264 -(i8::eq(&a[11usize], &b[11usize]) as i8),
1265 -(i8::eq(&a[12usize], &b[12usize]) as i8),
1266 -(i8::eq(&a[13usize], &b[13usize]) as i8),
1267 -(i8::eq(&a[14usize], &b[14usize]) as i8),
1268 -(i8::eq(&a[15usize], &b[15usize]) as i8),
1269 ]
1270 .simd_into(self)
1271 }
1272 #[inline(always)]
1273 fn simd_lt_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> mask8x16<Self> {
1274 [
1275 -(i8::lt(&a[0usize], &b[0usize]) as i8),
1276 -(i8::lt(&a[1usize], &b[1usize]) as i8),
1277 -(i8::lt(&a[2usize], &b[2usize]) as i8),
1278 -(i8::lt(&a[3usize], &b[3usize]) as i8),
1279 -(i8::lt(&a[4usize], &b[4usize]) as i8),
1280 -(i8::lt(&a[5usize], &b[5usize]) as i8),
1281 -(i8::lt(&a[6usize], &b[6usize]) as i8),
1282 -(i8::lt(&a[7usize], &b[7usize]) as i8),
1283 -(i8::lt(&a[8usize], &b[8usize]) as i8),
1284 -(i8::lt(&a[9usize], &b[9usize]) as i8),
1285 -(i8::lt(&a[10usize], &b[10usize]) as i8),
1286 -(i8::lt(&a[11usize], &b[11usize]) as i8),
1287 -(i8::lt(&a[12usize], &b[12usize]) as i8),
1288 -(i8::lt(&a[13usize], &b[13usize]) as i8),
1289 -(i8::lt(&a[14usize], &b[14usize]) as i8),
1290 -(i8::lt(&a[15usize], &b[15usize]) as i8),
1291 ]
1292 .simd_into(self)
1293 }
1294 #[inline(always)]
1295 fn simd_le_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> mask8x16<Self> {
1296 [
1297 -(i8::le(&a[0usize], &b[0usize]) as i8),
1298 -(i8::le(&a[1usize], &b[1usize]) as i8),
1299 -(i8::le(&a[2usize], &b[2usize]) as i8),
1300 -(i8::le(&a[3usize], &b[3usize]) as i8),
1301 -(i8::le(&a[4usize], &b[4usize]) as i8),
1302 -(i8::le(&a[5usize], &b[5usize]) as i8),
1303 -(i8::le(&a[6usize], &b[6usize]) as i8),
1304 -(i8::le(&a[7usize], &b[7usize]) as i8),
1305 -(i8::le(&a[8usize], &b[8usize]) as i8),
1306 -(i8::le(&a[9usize], &b[9usize]) as i8),
1307 -(i8::le(&a[10usize], &b[10usize]) as i8),
1308 -(i8::le(&a[11usize], &b[11usize]) as i8),
1309 -(i8::le(&a[12usize], &b[12usize]) as i8),
1310 -(i8::le(&a[13usize], &b[13usize]) as i8),
1311 -(i8::le(&a[14usize], &b[14usize]) as i8),
1312 -(i8::le(&a[15usize], &b[15usize]) as i8),
1313 ]
1314 .simd_into(self)
1315 }
1316 #[inline(always)]
1317 fn zip_low_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
1318 [
1319 a[0usize], b[0usize], a[1usize], b[1usize], a[2usize], b[2usize], a[3usize], b[3usize],
1320 a[4usize], b[4usize], a[5usize], b[5usize], a[6usize], b[6usize], a[7usize], b[7usize],
1321 ]
1322 .simd_into(self)
1323 }
1324 #[inline(always)]
1325 fn zip_high_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
1326 [
1327 a[8usize], b[8usize], a[9usize], b[9usize], a[10usize], b[10usize], a[11usize],
1328 b[11usize], a[12usize], b[12usize], a[13usize], b[13usize], a[14usize], b[14usize],
1329 a[15usize], b[15usize],
1330 ]
1331 .simd_into(self)
1332 }
1333 #[inline(always)]
1334 fn unzip_low_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
1335 [
1336 a[0usize], a[2usize], a[4usize], a[6usize], a[8usize], a[10usize], a[12usize],
1337 a[14usize], b[0usize], b[2usize], b[4usize], b[6usize], b[8usize], b[10usize],
1338 b[12usize], b[14usize],
1339 ]
1340 .simd_into(self)
1341 }
1342 #[inline(always)]
1343 fn unzip_high_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
1344 [
1345 a[1usize], a[3usize], a[5usize], a[7usize], a[9usize], a[11usize], a[13usize],
1346 a[15usize], b[1usize], b[3usize], b[5usize], b[7usize], b[9usize], b[11usize],
1347 b[13usize], b[15usize],
1348 ]
1349 .simd_into(self)
1350 }
1351 #[inline(always)]
1352 fn interleave_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> (i8x16<Self>, i8x16<Self>) {
1353 (self.zip_low_i8x16(a, b), self.zip_high_i8x16(a, b))
1354 }
1355 #[inline(always)]
1356 fn deinterleave_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> (i8x16<Self>, i8x16<Self>) {
1357 (self.unzip_low_i8x16(a, b), self.unzip_high_i8x16(a, b))
1358 }
1359 #[inline(always)]
1360 fn select_i8x16(self, a: mask8x16<Self>, b: i8x16<Self>, c: i8x16<Self>) -> i8x16<Self> {
1361 [
1362 if a.val.0[0usize] != 0 {
1363 b[0usize]
1364 } else {
1365 c[0usize]
1366 },
1367 if a.val.0[1usize] != 0 {
1368 b[1usize]
1369 } else {
1370 c[1usize]
1371 },
1372 if a.val.0[2usize] != 0 {
1373 b[2usize]
1374 } else {
1375 c[2usize]
1376 },
1377 if a.val.0[3usize] != 0 {
1378 b[3usize]
1379 } else {
1380 c[3usize]
1381 },
1382 if a.val.0[4usize] != 0 {
1383 b[4usize]
1384 } else {
1385 c[4usize]
1386 },
1387 if a.val.0[5usize] != 0 {
1388 b[5usize]
1389 } else {
1390 c[5usize]
1391 },
1392 if a.val.0[6usize] != 0 {
1393 b[6usize]
1394 } else {
1395 c[6usize]
1396 },
1397 if a.val.0[7usize] != 0 {
1398 b[7usize]
1399 } else {
1400 c[7usize]
1401 },
1402 if a.val.0[8usize] != 0 {
1403 b[8usize]
1404 } else {
1405 c[8usize]
1406 },
1407 if a.val.0[9usize] != 0 {
1408 b[9usize]
1409 } else {
1410 c[9usize]
1411 },
1412 if a.val.0[10usize] != 0 {
1413 b[10usize]
1414 } else {
1415 c[10usize]
1416 },
1417 if a.val.0[11usize] != 0 {
1418 b[11usize]
1419 } else {
1420 c[11usize]
1421 },
1422 if a.val.0[12usize] != 0 {
1423 b[12usize]
1424 } else {
1425 c[12usize]
1426 },
1427 if a.val.0[13usize] != 0 {
1428 b[13usize]
1429 } else {
1430 c[13usize]
1431 },
1432 if a.val.0[14usize] != 0 {
1433 b[14usize]
1434 } else {
1435 c[14usize]
1436 },
1437 if a.val.0[15usize] != 0 {
1438 b[15usize]
1439 } else {
1440 c[15usize]
1441 },
1442 ]
1443 .simd_into(self)
1444 }
1445 #[inline(always)]
1446 fn combine_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x32<Self> {
1447 let mut result = [0; 32usize];
1448 result[0..16usize].copy_from_slice(&a.val.0);
1449 result[16usize..32usize].copy_from_slice(&b.val.0);
1450 result.simd_into(self)
1451 }
1452 #[inline(always)]
1453 fn neg_i8x16(self, a: i8x16<Self>) -> i8x16<Self> {
1454 [
1455 i8::wrapping_neg(a[0usize]),
1456 i8::wrapping_neg(a[1usize]),
1457 i8::wrapping_neg(a[2usize]),
1458 i8::wrapping_neg(a[3usize]),
1459 i8::wrapping_neg(a[4usize]),
1460 i8::wrapping_neg(a[5usize]),
1461 i8::wrapping_neg(a[6usize]),
1462 i8::wrapping_neg(a[7usize]),
1463 i8::wrapping_neg(a[8usize]),
1464 i8::wrapping_neg(a[9usize]),
1465 i8::wrapping_neg(a[10usize]),
1466 i8::wrapping_neg(a[11usize]),
1467 i8::wrapping_neg(a[12usize]),
1468 i8::wrapping_neg(a[13usize]),
1469 i8::wrapping_neg(a[14usize]),
1470 i8::wrapping_neg(a[15usize]),
1471 ]
1472 .simd_into(self)
1473 }
1474 #[inline(always)]
1475 fn load_four_interleaved_i8x16(self, src: &[i8; 64usize]) -> [i8x16<Self>; 4usize] {
1476 [
1477 [
1478 src[0usize],
1479 src[4usize],
1480 src[8usize],
1481 src[12usize],
1482 src[16usize],
1483 src[20usize],
1484 src[24usize],
1485 src[28usize],
1486 src[32usize],
1487 src[36usize],
1488 src[40usize],
1489 src[44usize],
1490 src[48usize],
1491 src[52usize],
1492 src[56usize],
1493 src[60usize],
1494 ]
1495 .simd_into(self),
1496 [
1497 src[1usize],
1498 src[5usize],
1499 src[9usize],
1500 src[13usize],
1501 src[17usize],
1502 src[21usize],
1503 src[25usize],
1504 src[29usize],
1505 src[33usize],
1506 src[37usize],
1507 src[41usize],
1508 src[45usize],
1509 src[49usize],
1510 src[53usize],
1511 src[57usize],
1512 src[61usize],
1513 ]
1514 .simd_into(self),
1515 [
1516 src[2usize],
1517 src[6usize],
1518 src[10usize],
1519 src[14usize],
1520 src[18usize],
1521 src[22usize],
1522 src[26usize],
1523 src[30usize],
1524 src[34usize],
1525 src[38usize],
1526 src[42usize],
1527 src[46usize],
1528 src[50usize],
1529 src[54usize],
1530 src[58usize],
1531 src[62usize],
1532 ]
1533 .simd_into(self),
1534 [
1535 src[3usize],
1536 src[7usize],
1537 src[11usize],
1538 src[15usize],
1539 src[19usize],
1540 src[23usize],
1541 src[27usize],
1542 src[31usize],
1543 src[35usize],
1544 src[39usize],
1545 src[43usize],
1546 src[47usize],
1547 src[51usize],
1548 src[55usize],
1549 src[59usize],
1550 src[63usize],
1551 ]
1552 .simd_into(self),
1553 ]
1554 }
1555 #[inline(always)]
1556 fn store_four_interleaved_i8x16(
1557 self,
1558 vectors: [i8x16<Self>; 4usize],
1559 dest: &mut [i8; 64usize],
1560 ) -> () {
1561 *dest = [
1562 vectors[0usize][0usize],
1563 vectors[1usize][0usize],
1564 vectors[2usize][0usize],
1565 vectors[3usize][0usize],
1566 vectors[0usize][1usize],
1567 vectors[1usize][1usize],
1568 vectors[2usize][1usize],
1569 vectors[3usize][1usize],
1570 vectors[0usize][2usize],
1571 vectors[1usize][2usize],
1572 vectors[2usize][2usize],
1573 vectors[3usize][2usize],
1574 vectors[0usize][3usize],
1575 vectors[1usize][3usize],
1576 vectors[2usize][3usize],
1577 vectors[3usize][3usize],
1578 vectors[0usize][4usize],
1579 vectors[1usize][4usize],
1580 vectors[2usize][4usize],
1581 vectors[3usize][4usize],
1582 vectors[0usize][5usize],
1583 vectors[1usize][5usize],
1584 vectors[2usize][5usize],
1585 vectors[3usize][5usize],
1586 vectors[0usize][6usize],
1587 vectors[1usize][6usize],
1588 vectors[2usize][6usize],
1589 vectors[3usize][6usize],
1590 vectors[0usize][7usize],
1591 vectors[1usize][7usize],
1592 vectors[2usize][7usize],
1593 vectors[3usize][7usize],
1594 vectors[0usize][8usize],
1595 vectors[1usize][8usize],
1596 vectors[2usize][8usize],
1597 vectors[3usize][8usize],
1598 vectors[0usize][9usize],
1599 vectors[1usize][9usize],
1600 vectors[2usize][9usize],
1601 vectors[3usize][9usize],
1602 vectors[0usize][10usize],
1603 vectors[1usize][10usize],
1604 vectors[2usize][10usize],
1605 vectors[3usize][10usize],
1606 vectors[0usize][11usize],
1607 vectors[1usize][11usize],
1608 vectors[2usize][11usize],
1609 vectors[3usize][11usize],
1610 vectors[0usize][12usize],
1611 vectors[1usize][12usize],
1612 vectors[2usize][12usize],
1613 vectors[3usize][12usize],
1614 vectors[0usize][13usize],
1615 vectors[1usize][13usize],
1616 vectors[2usize][13usize],
1617 vectors[3usize][13usize],
1618 vectors[0usize][14usize],
1619 vectors[1usize][14usize],
1620 vectors[2usize][14usize],
1621 vectors[3usize][14usize],
1622 vectors[0usize][15usize],
1623 vectors[1usize][15usize],
1624 vectors[2usize][15usize],
1625 vectors[3usize][15usize],
1626 ];
1627 }
1628 #[inline(always)]
1629 fn widen_i8x16(self, a: i8x16<Self>) -> (i16x8<Self>, i16x8<Self>) {
1630 (
1631 [
1632 a[0usize] as i16,
1633 a[1usize] as i16,
1634 a[2usize] as i16,
1635 a[3usize] as i16,
1636 a[4usize] as i16,
1637 a[5usize] as i16,
1638 a[6usize] as i16,
1639 a[7usize] as i16,
1640 ]
1641 .simd_into(self),
1642 [
1643 a[8usize] as i16,
1644 a[9usize] as i16,
1645 a[10usize] as i16,
1646 a[11usize] as i16,
1647 a[12usize] as i16,
1648 a[13usize] as i16,
1649 a[14usize] as i16,
1650 a[15usize] as i16,
1651 ]
1652 .simd_into(self),
1653 )
1654 }
1655 #[inline(always)]
1656 fn widening_mul_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> (i16x8<Self>, i16x8<Self>) {
1657 let (a0, a1) = self.widen_i8x16(a);
1658 let (b0, b1) = self.widen_i8x16(b);
1659 (self.mul_i16x8(a0, b0), self.mul_i16x8(a1, b1))
1660 }
1661 #[inline(always)]
1662 fn widening_add_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> (i16x8<Self>, i16x8<Self>) {
1663 let (a0, a1) = self.widen_i8x16(a);
1664 let (b0, b1) = self.widen_i8x16(b);
1665 (self.add_i16x8(a0, b0), self.add_i16x8(a1, b1))
1666 }
1667 #[inline(always)]
1668 fn pairwise_widening_add_i8x16(self, a: i8x16<Self>) -> i16x8<Self> {
1669 [
1670 (a[0usize] as i16) + (a[1usize] as i16),
1671 (a[2usize] as i16) + (a[3usize] as i16),
1672 (a[4usize] as i16) + (a[5usize] as i16),
1673 (a[6usize] as i16) + (a[7usize] as i16),
1674 (a[8usize] as i16) + (a[9usize] as i16),
1675 (a[10usize] as i16) + (a[11usize] as i16),
1676 (a[12usize] as i16) + (a[13usize] as i16),
1677 (a[14usize] as i16) + (a[15usize] as i16),
1678 ]
1679 .simd_into(self)
1680 }
1681 #[inline(always)]
1682 fn pairwise_dot_product_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i16x8<Self> {
1683 let (low, high) = self.widening_mul_i8x16(a, b);
1684 let (even, odd) = self.deinterleave_i16x8(low, high);
1685 self.add_i16x8(even, odd)
1686 }
1687 #[inline(always)]
1688 fn splat_u8x16(self, val: u8) -> u8x16<Self> {
1689 [val; 16usize].simd_into(self)
1690 }
1691 #[inline(always)]
1692 fn reverse_u8x16(self, a: u8x16<Self>) -> u8x16<Self> {
1693 [
1694 a[15usize], a[14usize], a[13usize], a[12usize], a[11usize], a[10usize], a[9usize],
1695 a[8usize], a[7usize], a[6usize], a[5usize], a[4usize], a[3usize], a[2usize], a[1usize],
1696 a[0usize],
1697 ]
1698 .simd_into(self)
1699 }
1700 #[inline(always)]
1701 fn slide_u8x16<const SHIFT: usize>(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
1702 let mut dest = [Default::default(); 16usize];
1703 dest[..16usize - SHIFT].copy_from_slice(&a.val.0[SHIFT..]);
1704 dest[16usize - SHIFT..].copy_from_slice(&b.val.0[..SHIFT]);
1705 dest.simd_into(self)
1706 }
1707 #[inline(always)]
1708 fn swizzle_dyn_within_blocks_u8x16(self, a: u8x16<Self>, indices: u8x16<Self>) -> u8x16<Self> {
1709 let bytes = Bytes::to_bytes(a);
1710 let result: u8x16<Self> = [
1711 {
1712 let index = indices[0usize] as usize;
1713 bytes[index % 16usize]
1714 },
1715 {
1716 let index = indices[1usize] as usize;
1717 bytes[index % 16usize]
1718 },
1719 {
1720 let index = indices[2usize] as usize;
1721 bytes[index % 16usize]
1722 },
1723 {
1724 let index = indices[3usize] as usize;
1725 bytes[index % 16usize]
1726 },
1727 {
1728 let index = indices[4usize] as usize;
1729 bytes[index % 16usize]
1730 },
1731 {
1732 let index = indices[5usize] as usize;
1733 bytes[index % 16usize]
1734 },
1735 {
1736 let index = indices[6usize] as usize;
1737 bytes[index % 16usize]
1738 },
1739 {
1740 let index = indices[7usize] as usize;
1741 bytes[index % 16usize]
1742 },
1743 {
1744 let index = indices[8usize] as usize;
1745 bytes[index % 16usize]
1746 },
1747 {
1748 let index = indices[9usize] as usize;
1749 bytes[index % 16usize]
1750 },
1751 {
1752 let index = indices[10usize] as usize;
1753 bytes[index % 16usize]
1754 },
1755 {
1756 let index = indices[11usize] as usize;
1757 bytes[index % 16usize]
1758 },
1759 {
1760 let index = indices[12usize] as usize;
1761 bytes[index % 16usize]
1762 },
1763 {
1764 let index = indices[13usize] as usize;
1765 bytes[index % 16usize]
1766 },
1767 {
1768 let index = indices[14usize] as usize;
1769 bytes[index % 16usize]
1770 },
1771 {
1772 let index = indices[15usize] as usize;
1773 bytes[index % 16usize]
1774 },
1775 ]
1776 .simd_into(self);
1777 Bytes::from_bytes(result)
1778 }
1779 #[inline(always)]
1780 fn swizzle_dyn_u8x16(self, a: u8x16<Self>, indices: u8x16<Self>) -> u8x16<Self> {
1781 let bytes = Bytes::to_bytes(a);
1782 let mut output = [0u8; 16usize];
1783 for lane in 0..16usize {
1784 let index = indices[lane] as usize;
1785 output[lane] = bytes[index % 16usize];
1786 }
1787 let result: u8x16<Self> = output.simd_into(self);
1788 Bytes::from_bytes(result)
1789 }
1790 #[inline(always)]
1791 fn swizzle_dyn_precise_u8x16(self, a: u8x16<Self>, indices: u8x16<Self>) -> u8x16<Self> {
1792 let bytes = Bytes::to_bytes(a);
1793 let mut output = [0u8; 16usize];
1794 for lane in 0..16usize {
1795 let index = indices[lane] as usize;
1796 let value = bytes[index % 16usize];
1797 output[lane] = core::hint::select_unpredictable(index < 16usize, value, 0u8);
1798 }
1799 let result: u8x16<Self> = output.simd_into(self);
1800 Bytes::from_bytes(result)
1801 }
1802 #[inline(always)]
1803 fn concat_swizzle_dyn_u8x16(
1804 self,
1805 a: u8x16<Self>,
1806 b: u8x16<Self>,
1807 indices: u8x16<Self>,
1808 ) -> u8x16<Self> {
1809 let first_table = Bytes::to_bytes(a);
1810 let second_table = Bytes::to_bytes(b);
1811 let mut output = [0u8; 16usize];
1812 for lane in 0..16usize {
1813 let index = indices[lane] as usize % 32usize;
1814 output[lane] = if index < 16usize {
1815 first_table[index]
1816 } else {
1817 second_table[index - 16usize]
1818 };
1819 }
1820 let result: u8x16<Self> = output.simd_into(self);
1821 Bytes::from_bytes(result)
1822 }
1823 #[inline(always)]
1824 fn concat_swizzle_dyn_precise_u8x16(
1825 self,
1826 a: u8x16<Self>,
1827 b: u8x16<Self>,
1828 indices: u8x16<Self>,
1829 ) -> u8x16<Self> {
1830 let first_table = Bytes::to_bytes(a);
1831 let second_table = Bytes::to_bytes(b);
1832 let mut output = [0u8; 16usize];
1833 for lane in 0..16usize {
1834 let index = indices[lane] as usize;
1835 let table_index = index % 16usize;
1836 let value = if index < 16usize {
1837 first_table[table_index]
1838 } else {
1839 second_table[table_index]
1840 };
1841 output[lane] = if index < 32usize { value } else { 0 };
1842 }
1843 let result: u8x16<Self> = output.simd_into(self);
1844 Bytes::from_bytes(result)
1845 }
1846 #[inline(always)]
1847 fn compress_u8x16(self, values: u8x16<Self>, mask: mask8x16<Self>) -> u8x16<Self> {
1848 self.compress_merge_u8x16(values, mask, u8x16::splat(self, 0))
1849 }
1850 #[inline(always)]
1851 fn compress_merge_u8x16(
1852 self,
1853 values: u8x16<Self>,
1854 mask: mask8x16<Self>,
1855 merge: u8x16<Self>,
1856 ) -> u8x16<Self> {
1857 let mask = self.to_bitmask_mask8x16(mask);
1858 let mut merge_padded = [0u8; 16 + 1];
1859 merge_padded[..16].copy_from_slice(&*merge);
1860 let mut compacted = merge_padded;
1861 let mut output_lane = 0;
1862 for input_lane in 0..16 {
1863 compacted[output_lane] = values[input_lane];
1864 output_lane += ((mask >> input_lane) & 1) as usize;
1865 }
1866 compacted[output_lane] = merge_padded[output_lane];
1867 let mut result = merge;
1868 result.copy_from_slice(&compacted[..16]);
1869 result
1870 }
1871 #[inline(always)]
1872 fn expand_u8x16(self, values: u8x16<Self>, mask: mask8x16<Self>) -> u8x16<Self> {
1873 self.expand_merge_u8x16(values, mask, u8x16::splat(self, 0))
1874 }
1875 #[inline(always)]
1876 fn expand_merge_u8x16(
1877 self,
1878 values: u8x16<Self>,
1879 mask: mask8x16<Self>,
1880 merge: u8x16<Self>,
1881 ) -> u8x16<Self> {
1882 let mask = self.to_bitmask_mask8x16(mask);
1883 let mut result = merge;
1884 let mut input_lane = 0;
1885 for output_lane in 0..16 {
1886 let bit = ((mask >> output_lane) & 1) as u8;
1887 let keep = 0u8.wrapping_sub(bit);
1888 result[output_lane] = (values[input_lane] & keep) | (result[output_lane] & !keep);
1889 input_lane += usize::from(bit);
1890 }
1891 result
1892 }
1893 #[inline(always)]
1894 fn count_ones_u8x16(self, a: u8x16<Self>) -> u8x16<Self> {
1895 [
1896 u8::try_from(a[0usize].count_ones()).unwrap(),
1897 u8::try_from(a[1usize].count_ones()).unwrap(),
1898 u8::try_from(a[2usize].count_ones()).unwrap(),
1899 u8::try_from(a[3usize].count_ones()).unwrap(),
1900 u8::try_from(a[4usize].count_ones()).unwrap(),
1901 u8::try_from(a[5usize].count_ones()).unwrap(),
1902 u8::try_from(a[6usize].count_ones()).unwrap(),
1903 u8::try_from(a[7usize].count_ones()).unwrap(),
1904 u8::try_from(a[8usize].count_ones()).unwrap(),
1905 u8::try_from(a[9usize].count_ones()).unwrap(),
1906 u8::try_from(a[10usize].count_ones()).unwrap(),
1907 u8::try_from(a[11usize].count_ones()).unwrap(),
1908 u8::try_from(a[12usize].count_ones()).unwrap(),
1909 u8::try_from(a[13usize].count_ones()).unwrap(),
1910 u8::try_from(a[14usize].count_ones()).unwrap(),
1911 u8::try_from(a[15usize].count_ones()).unwrap(),
1912 ]
1913 .simd_into(self)
1914 }
1915 #[inline(always)]
1916 fn count_zeros_u8x16(self, a: u8x16<Self>) -> u8x16<Self> {
1917 [
1918 u8::try_from(a[0usize].count_zeros()).unwrap(),
1919 u8::try_from(a[1usize].count_zeros()).unwrap(),
1920 u8::try_from(a[2usize].count_zeros()).unwrap(),
1921 u8::try_from(a[3usize].count_zeros()).unwrap(),
1922 u8::try_from(a[4usize].count_zeros()).unwrap(),
1923 u8::try_from(a[5usize].count_zeros()).unwrap(),
1924 u8::try_from(a[6usize].count_zeros()).unwrap(),
1925 u8::try_from(a[7usize].count_zeros()).unwrap(),
1926 u8::try_from(a[8usize].count_zeros()).unwrap(),
1927 u8::try_from(a[9usize].count_zeros()).unwrap(),
1928 u8::try_from(a[10usize].count_zeros()).unwrap(),
1929 u8::try_from(a[11usize].count_zeros()).unwrap(),
1930 u8::try_from(a[12usize].count_zeros()).unwrap(),
1931 u8::try_from(a[13usize].count_zeros()).unwrap(),
1932 u8::try_from(a[14usize].count_zeros()).unwrap(),
1933 u8::try_from(a[15usize].count_zeros()).unwrap(),
1934 ]
1935 .simd_into(self)
1936 }
1937 #[inline(always)]
1938 fn add_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
1939 [
1940 u8::wrapping_add(a[0usize], b[0usize]),
1941 u8::wrapping_add(a[1usize], b[1usize]),
1942 u8::wrapping_add(a[2usize], b[2usize]),
1943 u8::wrapping_add(a[3usize], b[3usize]),
1944 u8::wrapping_add(a[4usize], b[4usize]),
1945 u8::wrapping_add(a[5usize], b[5usize]),
1946 u8::wrapping_add(a[6usize], b[6usize]),
1947 u8::wrapping_add(a[7usize], b[7usize]),
1948 u8::wrapping_add(a[8usize], b[8usize]),
1949 u8::wrapping_add(a[9usize], b[9usize]),
1950 u8::wrapping_add(a[10usize], b[10usize]),
1951 u8::wrapping_add(a[11usize], b[11usize]),
1952 u8::wrapping_add(a[12usize], b[12usize]),
1953 u8::wrapping_add(a[13usize], b[13usize]),
1954 u8::wrapping_add(a[14usize], b[14usize]),
1955 u8::wrapping_add(a[15usize], b[15usize]),
1956 ]
1957 .simd_into(self)
1958 }
1959 #[inline(always)]
1960 fn saturating_add_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
1961 [
1962 u8::saturating_add(a[0usize], b[0usize]),
1963 u8::saturating_add(a[1usize], b[1usize]),
1964 u8::saturating_add(a[2usize], b[2usize]),
1965 u8::saturating_add(a[3usize], b[3usize]),
1966 u8::saturating_add(a[4usize], b[4usize]),
1967 u8::saturating_add(a[5usize], b[5usize]),
1968 u8::saturating_add(a[6usize], b[6usize]),
1969 u8::saturating_add(a[7usize], b[7usize]),
1970 u8::saturating_add(a[8usize], b[8usize]),
1971 u8::saturating_add(a[9usize], b[9usize]),
1972 u8::saturating_add(a[10usize], b[10usize]),
1973 u8::saturating_add(a[11usize], b[11usize]),
1974 u8::saturating_add(a[12usize], b[12usize]),
1975 u8::saturating_add(a[13usize], b[13usize]),
1976 u8::saturating_add(a[14usize], b[14usize]),
1977 u8::saturating_add(a[15usize], b[15usize]),
1978 ]
1979 .simd_into(self)
1980 }
1981 #[inline(always)]
1982 fn sub_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
1983 [
1984 u8::wrapping_sub(a[0usize], b[0usize]),
1985 u8::wrapping_sub(a[1usize], b[1usize]),
1986 u8::wrapping_sub(a[2usize], b[2usize]),
1987 u8::wrapping_sub(a[3usize], b[3usize]),
1988 u8::wrapping_sub(a[4usize], b[4usize]),
1989 u8::wrapping_sub(a[5usize], b[5usize]),
1990 u8::wrapping_sub(a[6usize], b[6usize]),
1991 u8::wrapping_sub(a[7usize], b[7usize]),
1992 u8::wrapping_sub(a[8usize], b[8usize]),
1993 u8::wrapping_sub(a[9usize], b[9usize]),
1994 u8::wrapping_sub(a[10usize], b[10usize]),
1995 u8::wrapping_sub(a[11usize], b[11usize]),
1996 u8::wrapping_sub(a[12usize], b[12usize]),
1997 u8::wrapping_sub(a[13usize], b[13usize]),
1998 u8::wrapping_sub(a[14usize], b[14usize]),
1999 u8::wrapping_sub(a[15usize], b[15usize]),
2000 ]
2001 .simd_into(self)
2002 }
2003 #[inline(always)]
2004 fn saturating_sub_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
2005 [
2006 u8::saturating_sub(a[0usize], b[0usize]),
2007 u8::saturating_sub(a[1usize], b[1usize]),
2008 u8::saturating_sub(a[2usize], b[2usize]),
2009 u8::saturating_sub(a[3usize], b[3usize]),
2010 u8::saturating_sub(a[4usize], b[4usize]),
2011 u8::saturating_sub(a[5usize], b[5usize]),
2012 u8::saturating_sub(a[6usize], b[6usize]),
2013 u8::saturating_sub(a[7usize], b[7usize]),
2014 u8::saturating_sub(a[8usize], b[8usize]),
2015 u8::saturating_sub(a[9usize], b[9usize]),
2016 u8::saturating_sub(a[10usize], b[10usize]),
2017 u8::saturating_sub(a[11usize], b[11usize]),
2018 u8::saturating_sub(a[12usize], b[12usize]),
2019 u8::saturating_sub(a[13usize], b[13usize]),
2020 u8::saturating_sub(a[14usize], b[14usize]),
2021 u8::saturating_sub(a[15usize], b[15usize]),
2022 ]
2023 .simd_into(self)
2024 }
2025 #[inline(always)]
2026 fn mul_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
2027 [
2028 u8::wrapping_mul(a[0usize], b[0usize]),
2029 u8::wrapping_mul(a[1usize], b[1usize]),
2030 u8::wrapping_mul(a[2usize], b[2usize]),
2031 u8::wrapping_mul(a[3usize], b[3usize]),
2032 u8::wrapping_mul(a[4usize], b[4usize]),
2033 u8::wrapping_mul(a[5usize], b[5usize]),
2034 u8::wrapping_mul(a[6usize], b[6usize]),
2035 u8::wrapping_mul(a[7usize], b[7usize]),
2036 u8::wrapping_mul(a[8usize], b[8usize]),
2037 u8::wrapping_mul(a[9usize], b[9usize]),
2038 u8::wrapping_mul(a[10usize], b[10usize]),
2039 u8::wrapping_mul(a[11usize], b[11usize]),
2040 u8::wrapping_mul(a[12usize], b[12usize]),
2041 u8::wrapping_mul(a[13usize], b[13usize]),
2042 u8::wrapping_mul(a[14usize], b[14usize]),
2043 u8::wrapping_mul(a[15usize], b[15usize]),
2044 ]
2045 .simd_into(self)
2046 }
2047 #[inline(always)]
2048 fn and_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
2049 [
2050 u8::bitand(a[0usize], &b[0usize]),
2051 u8::bitand(a[1usize], &b[1usize]),
2052 u8::bitand(a[2usize], &b[2usize]),
2053 u8::bitand(a[3usize], &b[3usize]),
2054 u8::bitand(a[4usize], &b[4usize]),
2055 u8::bitand(a[5usize], &b[5usize]),
2056 u8::bitand(a[6usize], &b[6usize]),
2057 u8::bitand(a[7usize], &b[7usize]),
2058 u8::bitand(a[8usize], &b[8usize]),
2059 u8::bitand(a[9usize], &b[9usize]),
2060 u8::bitand(a[10usize], &b[10usize]),
2061 u8::bitand(a[11usize], &b[11usize]),
2062 u8::bitand(a[12usize], &b[12usize]),
2063 u8::bitand(a[13usize], &b[13usize]),
2064 u8::bitand(a[14usize], &b[14usize]),
2065 u8::bitand(a[15usize], &b[15usize]),
2066 ]
2067 .simd_into(self)
2068 }
2069 #[inline(always)]
2070 fn or_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
2071 [
2072 u8::bitor(a[0usize], &b[0usize]),
2073 u8::bitor(a[1usize], &b[1usize]),
2074 u8::bitor(a[2usize], &b[2usize]),
2075 u8::bitor(a[3usize], &b[3usize]),
2076 u8::bitor(a[4usize], &b[4usize]),
2077 u8::bitor(a[5usize], &b[5usize]),
2078 u8::bitor(a[6usize], &b[6usize]),
2079 u8::bitor(a[7usize], &b[7usize]),
2080 u8::bitor(a[8usize], &b[8usize]),
2081 u8::bitor(a[9usize], &b[9usize]),
2082 u8::bitor(a[10usize], &b[10usize]),
2083 u8::bitor(a[11usize], &b[11usize]),
2084 u8::bitor(a[12usize], &b[12usize]),
2085 u8::bitor(a[13usize], &b[13usize]),
2086 u8::bitor(a[14usize], &b[14usize]),
2087 u8::bitor(a[15usize], &b[15usize]),
2088 ]
2089 .simd_into(self)
2090 }
2091 #[inline(always)]
2092 fn xor_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
2093 [
2094 u8::bitxor(a[0usize], &b[0usize]),
2095 u8::bitxor(a[1usize], &b[1usize]),
2096 u8::bitxor(a[2usize], &b[2usize]),
2097 u8::bitxor(a[3usize], &b[3usize]),
2098 u8::bitxor(a[4usize], &b[4usize]),
2099 u8::bitxor(a[5usize], &b[5usize]),
2100 u8::bitxor(a[6usize], &b[6usize]),
2101 u8::bitxor(a[7usize], &b[7usize]),
2102 u8::bitxor(a[8usize], &b[8usize]),
2103 u8::bitxor(a[9usize], &b[9usize]),
2104 u8::bitxor(a[10usize], &b[10usize]),
2105 u8::bitxor(a[11usize], &b[11usize]),
2106 u8::bitxor(a[12usize], &b[12usize]),
2107 u8::bitxor(a[13usize], &b[13usize]),
2108 u8::bitxor(a[14usize], &b[14usize]),
2109 u8::bitxor(a[15usize], &b[15usize]),
2110 ]
2111 .simd_into(self)
2112 }
2113 #[inline(always)]
2114 fn not_u8x16(self, a: u8x16<Self>) -> u8x16<Self> {
2115 [
2116 u8::not(a[0usize]),
2117 u8::not(a[1usize]),
2118 u8::not(a[2usize]),
2119 u8::not(a[3usize]),
2120 u8::not(a[4usize]),
2121 u8::not(a[5usize]),
2122 u8::not(a[6usize]),
2123 u8::not(a[7usize]),
2124 u8::not(a[8usize]),
2125 u8::not(a[9usize]),
2126 u8::not(a[10usize]),
2127 u8::not(a[11usize]),
2128 u8::not(a[12usize]),
2129 u8::not(a[13usize]),
2130 u8::not(a[14usize]),
2131 u8::not(a[15usize]),
2132 ]
2133 .simd_into(self)
2134 }
2135 #[inline(always)]
2136 fn shl_u8x16(self, a: u8x16<Self>, shift: u32) -> u8x16<Self> {
2137 [
2138 u8::wrapping_shl(a[0usize], shift),
2139 u8::wrapping_shl(a[1usize], shift),
2140 u8::wrapping_shl(a[2usize], shift),
2141 u8::wrapping_shl(a[3usize], shift),
2142 u8::wrapping_shl(a[4usize], shift),
2143 u8::wrapping_shl(a[5usize], shift),
2144 u8::wrapping_shl(a[6usize], shift),
2145 u8::wrapping_shl(a[7usize], shift),
2146 u8::wrapping_shl(a[8usize], shift),
2147 u8::wrapping_shl(a[9usize], shift),
2148 u8::wrapping_shl(a[10usize], shift),
2149 u8::wrapping_shl(a[11usize], shift),
2150 u8::wrapping_shl(a[12usize], shift),
2151 u8::wrapping_shl(a[13usize], shift),
2152 u8::wrapping_shl(a[14usize], shift),
2153 u8::wrapping_shl(a[15usize], shift),
2154 ]
2155 .simd_into(self)
2156 }
2157 #[inline(always)]
2158 fn shlv_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
2159 [
2160 u8::wrapping_shl(a[0usize], b[0usize] as u32),
2161 u8::wrapping_shl(a[1usize], b[1usize] as u32),
2162 u8::wrapping_shl(a[2usize], b[2usize] as u32),
2163 u8::wrapping_shl(a[3usize], b[3usize] as u32),
2164 u8::wrapping_shl(a[4usize], b[4usize] as u32),
2165 u8::wrapping_shl(a[5usize], b[5usize] as u32),
2166 u8::wrapping_shl(a[6usize], b[6usize] as u32),
2167 u8::wrapping_shl(a[7usize], b[7usize] as u32),
2168 u8::wrapping_shl(a[8usize], b[8usize] as u32),
2169 u8::wrapping_shl(a[9usize], b[9usize] as u32),
2170 u8::wrapping_shl(a[10usize], b[10usize] as u32),
2171 u8::wrapping_shl(a[11usize], b[11usize] as u32),
2172 u8::wrapping_shl(a[12usize], b[12usize] as u32),
2173 u8::wrapping_shl(a[13usize], b[13usize] as u32),
2174 u8::wrapping_shl(a[14usize], b[14usize] as u32),
2175 u8::wrapping_shl(a[15usize], b[15usize] as u32),
2176 ]
2177 .simd_into(self)
2178 }
2179 #[inline(always)]
2180 fn shr_u8x16(self, a: u8x16<Self>, shift: u32) -> u8x16<Self> {
2181 [
2182 u8::wrapping_shr(a[0usize], shift),
2183 u8::wrapping_shr(a[1usize], shift),
2184 u8::wrapping_shr(a[2usize], shift),
2185 u8::wrapping_shr(a[3usize], shift),
2186 u8::wrapping_shr(a[4usize], shift),
2187 u8::wrapping_shr(a[5usize], shift),
2188 u8::wrapping_shr(a[6usize], shift),
2189 u8::wrapping_shr(a[7usize], shift),
2190 u8::wrapping_shr(a[8usize], shift),
2191 u8::wrapping_shr(a[9usize], shift),
2192 u8::wrapping_shr(a[10usize], shift),
2193 u8::wrapping_shr(a[11usize], shift),
2194 u8::wrapping_shr(a[12usize], shift),
2195 u8::wrapping_shr(a[13usize], shift),
2196 u8::wrapping_shr(a[14usize], shift),
2197 u8::wrapping_shr(a[15usize], shift),
2198 ]
2199 .simd_into(self)
2200 }
2201 #[inline(always)]
2202 fn shrv_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
2203 [
2204 u8::wrapping_shr(a[0usize], b[0usize] as u32),
2205 u8::wrapping_shr(a[1usize], b[1usize] as u32),
2206 u8::wrapping_shr(a[2usize], b[2usize] as u32),
2207 u8::wrapping_shr(a[3usize], b[3usize] as u32),
2208 u8::wrapping_shr(a[4usize], b[4usize] as u32),
2209 u8::wrapping_shr(a[5usize], b[5usize] as u32),
2210 u8::wrapping_shr(a[6usize], b[6usize] as u32),
2211 u8::wrapping_shr(a[7usize], b[7usize] as u32),
2212 u8::wrapping_shr(a[8usize], b[8usize] as u32),
2213 u8::wrapping_shr(a[9usize], b[9usize] as u32),
2214 u8::wrapping_shr(a[10usize], b[10usize] as u32),
2215 u8::wrapping_shr(a[11usize], b[11usize] as u32),
2216 u8::wrapping_shr(a[12usize], b[12usize] as u32),
2217 u8::wrapping_shr(a[13usize], b[13usize] as u32),
2218 u8::wrapping_shr(a[14usize], b[14usize] as u32),
2219 u8::wrapping_shr(a[15usize], b[15usize] as u32),
2220 ]
2221 .simd_into(self)
2222 }
2223 #[inline(always)]
2224 fn reduce_max_u8x16(self, a: u8x16<Self>) -> u8 {
2225 let reduced: [u8; 8usize] = [
2226 u8::max(a[0usize], a[1usize]),
2227 u8::max(a[2usize], a[3usize]),
2228 u8::max(a[4usize], a[5usize]),
2229 u8::max(a[6usize], a[7usize]),
2230 u8::max(a[8usize], a[9usize]),
2231 u8::max(a[10usize], a[11usize]),
2232 u8::max(a[12usize], a[13usize]),
2233 u8::max(a[14usize], a[15usize]),
2234 ];
2235 let reduced: [u8; 4usize] = [
2236 u8::max(reduced[0usize], reduced[1usize]),
2237 u8::max(reduced[2usize], reduced[3usize]),
2238 u8::max(reduced[4usize], reduced[5usize]),
2239 u8::max(reduced[6usize], reduced[7usize]),
2240 ];
2241 let reduced: [u8; 2usize] = [
2242 u8::max(reduced[0usize], reduced[1usize]),
2243 u8::max(reduced[2usize], reduced[3usize]),
2244 ];
2245 let reduced: [u8; 1usize] = [u8::max(reduced[0usize], reduced[1usize])];
2246 reduced[0]
2247 }
2248 #[inline(always)]
2249 fn reduce_min_u8x16(self, a: u8x16<Self>) -> u8 {
2250 let reduced: [u8; 8usize] = [
2251 u8::min(a[0usize], a[1usize]),
2252 u8::min(a[2usize], a[3usize]),
2253 u8::min(a[4usize], a[5usize]),
2254 u8::min(a[6usize], a[7usize]),
2255 u8::min(a[8usize], a[9usize]),
2256 u8::min(a[10usize], a[11usize]),
2257 u8::min(a[12usize], a[13usize]),
2258 u8::min(a[14usize], a[15usize]),
2259 ];
2260 let reduced: [u8; 4usize] = [
2261 u8::min(reduced[0usize], reduced[1usize]),
2262 u8::min(reduced[2usize], reduced[3usize]),
2263 u8::min(reduced[4usize], reduced[5usize]),
2264 u8::min(reduced[6usize], reduced[7usize]),
2265 ];
2266 let reduced: [u8; 2usize] = [
2267 u8::min(reduced[0usize], reduced[1usize]),
2268 u8::min(reduced[2usize], reduced[3usize]),
2269 ];
2270 let reduced: [u8; 1usize] = [u8::min(reduced[0usize], reduced[1usize])];
2271 reduced[0]
2272 }
2273 #[inline(always)]
2274 fn reduce_sum_u8x16(self, a: u8x16<Self>) -> u8 {
2275 let add_level_0: [u8; 8usize] = [
2276 a[0usize].wrapping_add(a[1usize]),
2277 a[2usize].wrapping_add(a[3usize]),
2278 a[4usize].wrapping_add(a[5usize]),
2279 a[6usize].wrapping_add(a[7usize]),
2280 a[8usize].wrapping_add(a[9usize]),
2281 a[10usize].wrapping_add(a[11usize]),
2282 a[12usize].wrapping_add(a[13usize]),
2283 a[14usize].wrapping_add(a[15usize]),
2284 ];
2285 let add_level_1: [u8; 4usize] = [
2286 add_level_0[0usize].wrapping_add(add_level_0[1usize]),
2287 add_level_0[2usize].wrapping_add(add_level_0[3usize]),
2288 add_level_0[4usize].wrapping_add(add_level_0[5usize]),
2289 add_level_0[6usize].wrapping_add(add_level_0[7usize]),
2290 ];
2291 let add_level_2: [u8; 2usize] = [
2292 add_level_1[0usize].wrapping_add(add_level_1[1usize]),
2293 add_level_1[2usize].wrapping_add(add_level_1[3usize]),
2294 ];
2295 let add_level_3: [u8; 1usize] = [add_level_2[0usize].wrapping_add(add_level_2[1usize])];
2296 add_level_3[0]
2297 }
2298 #[inline(always)]
2299 fn reduce_product_u8x16(self, a: u8x16<Self>) -> u8 {
2300 let mul_level_0: [u8; 8usize] = [
2301 a[0usize].wrapping_mul(a[1usize]),
2302 a[2usize].wrapping_mul(a[3usize]),
2303 a[4usize].wrapping_mul(a[5usize]),
2304 a[6usize].wrapping_mul(a[7usize]),
2305 a[8usize].wrapping_mul(a[9usize]),
2306 a[10usize].wrapping_mul(a[11usize]),
2307 a[12usize].wrapping_mul(a[13usize]),
2308 a[14usize].wrapping_mul(a[15usize]),
2309 ];
2310 let mul_level_1: [u8; 4usize] = [
2311 mul_level_0[0usize].wrapping_mul(mul_level_0[1usize]),
2312 mul_level_0[2usize].wrapping_mul(mul_level_0[3usize]),
2313 mul_level_0[4usize].wrapping_mul(mul_level_0[5usize]),
2314 mul_level_0[6usize].wrapping_mul(mul_level_0[7usize]),
2315 ];
2316 let mul_level_2: [u8; 2usize] = [
2317 mul_level_1[0usize].wrapping_mul(mul_level_1[1usize]),
2318 mul_level_1[2usize].wrapping_mul(mul_level_1[3usize]),
2319 ];
2320 let mul_level_3: [u8; 1usize] = [mul_level_2[0usize].wrapping_mul(mul_level_2[1usize])];
2321 mul_level_3[0]
2322 }
2323 #[inline(always)]
2324 fn max_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
2325 [
2326 u8::max(a[0usize], b[0usize]),
2327 u8::max(a[1usize], b[1usize]),
2328 u8::max(a[2usize], b[2usize]),
2329 u8::max(a[3usize], b[3usize]),
2330 u8::max(a[4usize], b[4usize]),
2331 u8::max(a[5usize], b[5usize]),
2332 u8::max(a[6usize], b[6usize]),
2333 u8::max(a[7usize], b[7usize]),
2334 u8::max(a[8usize], b[8usize]),
2335 u8::max(a[9usize], b[9usize]),
2336 u8::max(a[10usize], b[10usize]),
2337 u8::max(a[11usize], b[11usize]),
2338 u8::max(a[12usize], b[12usize]),
2339 u8::max(a[13usize], b[13usize]),
2340 u8::max(a[14usize], b[14usize]),
2341 u8::max(a[15usize], b[15usize]),
2342 ]
2343 .simd_into(self)
2344 }
2345 #[inline(always)]
2346 fn min_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
2347 [
2348 u8::min(a[0usize], b[0usize]),
2349 u8::min(a[1usize], b[1usize]),
2350 u8::min(a[2usize], b[2usize]),
2351 u8::min(a[3usize], b[3usize]),
2352 u8::min(a[4usize], b[4usize]),
2353 u8::min(a[5usize], b[5usize]),
2354 u8::min(a[6usize], b[6usize]),
2355 u8::min(a[7usize], b[7usize]),
2356 u8::min(a[8usize], b[8usize]),
2357 u8::min(a[9usize], b[9usize]),
2358 u8::min(a[10usize], b[10usize]),
2359 u8::min(a[11usize], b[11usize]),
2360 u8::min(a[12usize], b[12usize]),
2361 u8::min(a[13usize], b[13usize]),
2362 u8::min(a[14usize], b[14usize]),
2363 u8::min(a[15usize], b[15usize]),
2364 ]
2365 .simd_into(self)
2366 }
2367 #[inline(always)]
2368 fn clamp_u8x16(self, a: u8x16<Self>, b: u8x16<Self>, c: u8x16<Self>) -> u8x16<Self> {
2369 [
2370 u8::clamp_debug(a[0usize], b[0usize], c[0usize]),
2371 u8::clamp_debug(a[1usize], b[1usize], c[1usize]),
2372 u8::clamp_debug(a[2usize], b[2usize], c[2usize]),
2373 u8::clamp_debug(a[3usize], b[3usize], c[3usize]),
2374 u8::clamp_debug(a[4usize], b[4usize], c[4usize]),
2375 u8::clamp_debug(a[5usize], b[5usize], c[5usize]),
2376 u8::clamp_debug(a[6usize], b[6usize], c[6usize]),
2377 u8::clamp_debug(a[7usize], b[7usize], c[7usize]),
2378 u8::clamp_debug(a[8usize], b[8usize], c[8usize]),
2379 u8::clamp_debug(a[9usize], b[9usize], c[9usize]),
2380 u8::clamp_debug(a[10usize], b[10usize], c[10usize]),
2381 u8::clamp_debug(a[11usize], b[11usize], c[11usize]),
2382 u8::clamp_debug(a[12usize], b[12usize], c[12usize]),
2383 u8::clamp_debug(a[13usize], b[13usize], c[13usize]),
2384 u8::clamp_debug(a[14usize], b[14usize], c[14usize]),
2385 u8::clamp_debug(a[15usize], b[15usize], c[15usize]),
2386 ]
2387 .simd_into(self)
2388 }
2389 #[inline(always)]
2390 fn simd_eq_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> mask8x16<Self> {
2391 [
2392 -(u8::eq(&a[0usize], &b[0usize]) as i8),
2393 -(u8::eq(&a[1usize], &b[1usize]) as i8),
2394 -(u8::eq(&a[2usize], &b[2usize]) as i8),
2395 -(u8::eq(&a[3usize], &b[3usize]) as i8),
2396 -(u8::eq(&a[4usize], &b[4usize]) as i8),
2397 -(u8::eq(&a[5usize], &b[5usize]) as i8),
2398 -(u8::eq(&a[6usize], &b[6usize]) as i8),
2399 -(u8::eq(&a[7usize], &b[7usize]) as i8),
2400 -(u8::eq(&a[8usize], &b[8usize]) as i8),
2401 -(u8::eq(&a[9usize], &b[9usize]) as i8),
2402 -(u8::eq(&a[10usize], &b[10usize]) as i8),
2403 -(u8::eq(&a[11usize], &b[11usize]) as i8),
2404 -(u8::eq(&a[12usize], &b[12usize]) as i8),
2405 -(u8::eq(&a[13usize], &b[13usize]) as i8),
2406 -(u8::eq(&a[14usize], &b[14usize]) as i8),
2407 -(u8::eq(&a[15usize], &b[15usize]) as i8),
2408 ]
2409 .simd_into(self)
2410 }
2411 #[inline(always)]
2412 fn simd_lt_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> mask8x16<Self> {
2413 [
2414 -(u8::lt(&a[0usize], &b[0usize]) as i8),
2415 -(u8::lt(&a[1usize], &b[1usize]) as i8),
2416 -(u8::lt(&a[2usize], &b[2usize]) as i8),
2417 -(u8::lt(&a[3usize], &b[3usize]) as i8),
2418 -(u8::lt(&a[4usize], &b[4usize]) as i8),
2419 -(u8::lt(&a[5usize], &b[5usize]) as i8),
2420 -(u8::lt(&a[6usize], &b[6usize]) as i8),
2421 -(u8::lt(&a[7usize], &b[7usize]) as i8),
2422 -(u8::lt(&a[8usize], &b[8usize]) as i8),
2423 -(u8::lt(&a[9usize], &b[9usize]) as i8),
2424 -(u8::lt(&a[10usize], &b[10usize]) as i8),
2425 -(u8::lt(&a[11usize], &b[11usize]) as i8),
2426 -(u8::lt(&a[12usize], &b[12usize]) as i8),
2427 -(u8::lt(&a[13usize], &b[13usize]) as i8),
2428 -(u8::lt(&a[14usize], &b[14usize]) as i8),
2429 -(u8::lt(&a[15usize], &b[15usize]) as i8),
2430 ]
2431 .simd_into(self)
2432 }
2433 #[inline(always)]
2434 fn simd_le_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> mask8x16<Self> {
2435 [
2436 -(u8::le(&a[0usize], &b[0usize]) as i8),
2437 -(u8::le(&a[1usize], &b[1usize]) as i8),
2438 -(u8::le(&a[2usize], &b[2usize]) as i8),
2439 -(u8::le(&a[3usize], &b[3usize]) as i8),
2440 -(u8::le(&a[4usize], &b[4usize]) as i8),
2441 -(u8::le(&a[5usize], &b[5usize]) as i8),
2442 -(u8::le(&a[6usize], &b[6usize]) as i8),
2443 -(u8::le(&a[7usize], &b[7usize]) as i8),
2444 -(u8::le(&a[8usize], &b[8usize]) as i8),
2445 -(u8::le(&a[9usize], &b[9usize]) as i8),
2446 -(u8::le(&a[10usize], &b[10usize]) as i8),
2447 -(u8::le(&a[11usize], &b[11usize]) as i8),
2448 -(u8::le(&a[12usize], &b[12usize]) as i8),
2449 -(u8::le(&a[13usize], &b[13usize]) as i8),
2450 -(u8::le(&a[14usize], &b[14usize]) as i8),
2451 -(u8::le(&a[15usize], &b[15usize]) as i8),
2452 ]
2453 .simd_into(self)
2454 }
2455 #[inline(always)]
2456 fn zip_low_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
2457 [
2458 a[0usize], b[0usize], a[1usize], b[1usize], a[2usize], b[2usize], a[3usize], b[3usize],
2459 a[4usize], b[4usize], a[5usize], b[5usize], a[6usize], b[6usize], a[7usize], b[7usize],
2460 ]
2461 .simd_into(self)
2462 }
2463 #[inline(always)]
2464 fn zip_high_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
2465 [
2466 a[8usize], b[8usize], a[9usize], b[9usize], a[10usize], b[10usize], a[11usize],
2467 b[11usize], a[12usize], b[12usize], a[13usize], b[13usize], a[14usize], b[14usize],
2468 a[15usize], b[15usize],
2469 ]
2470 .simd_into(self)
2471 }
2472 #[inline(always)]
2473 fn unzip_low_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
2474 [
2475 a[0usize], a[2usize], a[4usize], a[6usize], a[8usize], a[10usize], a[12usize],
2476 a[14usize], b[0usize], b[2usize], b[4usize], b[6usize], b[8usize], b[10usize],
2477 b[12usize], b[14usize],
2478 ]
2479 .simd_into(self)
2480 }
2481 #[inline(always)]
2482 fn unzip_high_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
2483 [
2484 a[1usize], a[3usize], a[5usize], a[7usize], a[9usize], a[11usize], a[13usize],
2485 a[15usize], b[1usize], b[3usize], b[5usize], b[7usize], b[9usize], b[11usize],
2486 b[13usize], b[15usize],
2487 ]
2488 .simd_into(self)
2489 }
2490 #[inline(always)]
2491 fn interleave_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> (u8x16<Self>, u8x16<Self>) {
2492 (self.zip_low_u8x16(a, b), self.zip_high_u8x16(a, b))
2493 }
2494 #[inline(always)]
2495 fn deinterleave_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> (u8x16<Self>, u8x16<Self>) {
2496 (self.unzip_low_u8x16(a, b), self.unzip_high_u8x16(a, b))
2497 }
2498 #[inline(always)]
2499 fn select_u8x16(self, a: mask8x16<Self>, b: u8x16<Self>, c: u8x16<Self>) -> u8x16<Self> {
2500 [
2501 if a.val.0[0usize] != 0 {
2502 b[0usize]
2503 } else {
2504 c[0usize]
2505 },
2506 if a.val.0[1usize] != 0 {
2507 b[1usize]
2508 } else {
2509 c[1usize]
2510 },
2511 if a.val.0[2usize] != 0 {
2512 b[2usize]
2513 } else {
2514 c[2usize]
2515 },
2516 if a.val.0[3usize] != 0 {
2517 b[3usize]
2518 } else {
2519 c[3usize]
2520 },
2521 if a.val.0[4usize] != 0 {
2522 b[4usize]
2523 } else {
2524 c[4usize]
2525 },
2526 if a.val.0[5usize] != 0 {
2527 b[5usize]
2528 } else {
2529 c[5usize]
2530 },
2531 if a.val.0[6usize] != 0 {
2532 b[6usize]
2533 } else {
2534 c[6usize]
2535 },
2536 if a.val.0[7usize] != 0 {
2537 b[7usize]
2538 } else {
2539 c[7usize]
2540 },
2541 if a.val.0[8usize] != 0 {
2542 b[8usize]
2543 } else {
2544 c[8usize]
2545 },
2546 if a.val.0[9usize] != 0 {
2547 b[9usize]
2548 } else {
2549 c[9usize]
2550 },
2551 if a.val.0[10usize] != 0 {
2552 b[10usize]
2553 } else {
2554 c[10usize]
2555 },
2556 if a.val.0[11usize] != 0 {
2557 b[11usize]
2558 } else {
2559 c[11usize]
2560 },
2561 if a.val.0[12usize] != 0 {
2562 b[12usize]
2563 } else {
2564 c[12usize]
2565 },
2566 if a.val.0[13usize] != 0 {
2567 b[13usize]
2568 } else {
2569 c[13usize]
2570 },
2571 if a.val.0[14usize] != 0 {
2572 b[14usize]
2573 } else {
2574 c[14usize]
2575 },
2576 if a.val.0[15usize] != 0 {
2577 b[15usize]
2578 } else {
2579 c[15usize]
2580 },
2581 ]
2582 .simd_into(self)
2583 }
2584 #[inline(always)]
2585 fn combine_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x32<Self> {
2586 let mut result = [0; 32usize];
2587 result[0..16usize].copy_from_slice(&a.val.0);
2588 result[16usize..32usize].copy_from_slice(&b.val.0);
2589 result.simd_into(self)
2590 }
2591 #[inline(always)]
2592 fn load_four_interleaved_u8x16(self, src: &[u8; 64usize]) -> [u8x16<Self>; 4usize] {
2593 [
2594 [
2595 src[0usize],
2596 src[4usize],
2597 src[8usize],
2598 src[12usize],
2599 src[16usize],
2600 src[20usize],
2601 src[24usize],
2602 src[28usize],
2603 src[32usize],
2604 src[36usize],
2605 src[40usize],
2606 src[44usize],
2607 src[48usize],
2608 src[52usize],
2609 src[56usize],
2610 src[60usize],
2611 ]
2612 .simd_into(self),
2613 [
2614 src[1usize],
2615 src[5usize],
2616 src[9usize],
2617 src[13usize],
2618 src[17usize],
2619 src[21usize],
2620 src[25usize],
2621 src[29usize],
2622 src[33usize],
2623 src[37usize],
2624 src[41usize],
2625 src[45usize],
2626 src[49usize],
2627 src[53usize],
2628 src[57usize],
2629 src[61usize],
2630 ]
2631 .simd_into(self),
2632 [
2633 src[2usize],
2634 src[6usize],
2635 src[10usize],
2636 src[14usize],
2637 src[18usize],
2638 src[22usize],
2639 src[26usize],
2640 src[30usize],
2641 src[34usize],
2642 src[38usize],
2643 src[42usize],
2644 src[46usize],
2645 src[50usize],
2646 src[54usize],
2647 src[58usize],
2648 src[62usize],
2649 ]
2650 .simd_into(self),
2651 [
2652 src[3usize],
2653 src[7usize],
2654 src[11usize],
2655 src[15usize],
2656 src[19usize],
2657 src[23usize],
2658 src[27usize],
2659 src[31usize],
2660 src[35usize],
2661 src[39usize],
2662 src[43usize],
2663 src[47usize],
2664 src[51usize],
2665 src[55usize],
2666 src[59usize],
2667 src[63usize],
2668 ]
2669 .simd_into(self),
2670 ]
2671 }
2672 #[inline(always)]
2673 fn store_four_interleaved_u8x16(
2674 self,
2675 vectors: [u8x16<Self>; 4usize],
2676 dest: &mut [u8; 64usize],
2677 ) -> () {
2678 *dest = [
2679 vectors[0usize][0usize],
2680 vectors[1usize][0usize],
2681 vectors[2usize][0usize],
2682 vectors[3usize][0usize],
2683 vectors[0usize][1usize],
2684 vectors[1usize][1usize],
2685 vectors[2usize][1usize],
2686 vectors[3usize][1usize],
2687 vectors[0usize][2usize],
2688 vectors[1usize][2usize],
2689 vectors[2usize][2usize],
2690 vectors[3usize][2usize],
2691 vectors[0usize][3usize],
2692 vectors[1usize][3usize],
2693 vectors[2usize][3usize],
2694 vectors[3usize][3usize],
2695 vectors[0usize][4usize],
2696 vectors[1usize][4usize],
2697 vectors[2usize][4usize],
2698 vectors[3usize][4usize],
2699 vectors[0usize][5usize],
2700 vectors[1usize][5usize],
2701 vectors[2usize][5usize],
2702 vectors[3usize][5usize],
2703 vectors[0usize][6usize],
2704 vectors[1usize][6usize],
2705 vectors[2usize][6usize],
2706 vectors[3usize][6usize],
2707 vectors[0usize][7usize],
2708 vectors[1usize][7usize],
2709 vectors[2usize][7usize],
2710 vectors[3usize][7usize],
2711 vectors[0usize][8usize],
2712 vectors[1usize][8usize],
2713 vectors[2usize][8usize],
2714 vectors[3usize][8usize],
2715 vectors[0usize][9usize],
2716 vectors[1usize][9usize],
2717 vectors[2usize][9usize],
2718 vectors[3usize][9usize],
2719 vectors[0usize][10usize],
2720 vectors[1usize][10usize],
2721 vectors[2usize][10usize],
2722 vectors[3usize][10usize],
2723 vectors[0usize][11usize],
2724 vectors[1usize][11usize],
2725 vectors[2usize][11usize],
2726 vectors[3usize][11usize],
2727 vectors[0usize][12usize],
2728 vectors[1usize][12usize],
2729 vectors[2usize][12usize],
2730 vectors[3usize][12usize],
2731 vectors[0usize][13usize],
2732 vectors[1usize][13usize],
2733 vectors[2usize][13usize],
2734 vectors[3usize][13usize],
2735 vectors[0usize][14usize],
2736 vectors[1usize][14usize],
2737 vectors[2usize][14usize],
2738 vectors[3usize][14usize],
2739 vectors[0usize][15usize],
2740 vectors[1usize][15usize],
2741 vectors[2usize][15usize],
2742 vectors[3usize][15usize],
2743 ];
2744 }
2745 #[inline(always)]
2746 fn widen_u8x16(self, a: u8x16<Self>) -> (u16x8<Self>, u16x8<Self>) {
2747 (
2748 [
2749 a[0usize] as u16,
2750 a[1usize] as u16,
2751 a[2usize] as u16,
2752 a[3usize] as u16,
2753 a[4usize] as u16,
2754 a[5usize] as u16,
2755 a[6usize] as u16,
2756 a[7usize] as u16,
2757 ]
2758 .simd_into(self),
2759 [
2760 a[8usize] as u16,
2761 a[9usize] as u16,
2762 a[10usize] as u16,
2763 a[11usize] as u16,
2764 a[12usize] as u16,
2765 a[13usize] as u16,
2766 a[14usize] as u16,
2767 a[15usize] as u16,
2768 ]
2769 .simd_into(self),
2770 )
2771 }
2772 #[inline(always)]
2773 fn widening_mul_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> (u16x8<Self>, u16x8<Self>) {
2774 let (a0, a1) = self.widen_u8x16(a);
2775 let (b0, b1) = self.widen_u8x16(b);
2776 (self.mul_u16x8(a0, b0), self.mul_u16x8(a1, b1))
2777 }
2778 #[inline(always)]
2779 fn widening_add_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> (u16x8<Self>, u16x8<Self>) {
2780 let (a0, a1) = self.widen_u8x16(a);
2781 let (b0, b1) = self.widen_u8x16(b);
2782 (self.add_u16x8(a0, b0), self.add_u16x8(a1, b1))
2783 }
2784 #[inline(always)]
2785 fn pairwise_widening_add_u8x16(self, a: u8x16<Self>) -> u16x8<Self> {
2786 [
2787 (a[0usize] as u16) + (a[1usize] as u16),
2788 (a[2usize] as u16) + (a[3usize] as u16),
2789 (a[4usize] as u16) + (a[5usize] as u16),
2790 (a[6usize] as u16) + (a[7usize] as u16),
2791 (a[8usize] as u16) + (a[9usize] as u16),
2792 (a[10usize] as u16) + (a[11usize] as u16),
2793 (a[12usize] as u16) + (a[13usize] as u16),
2794 (a[14usize] as u16) + (a[15usize] as u16),
2795 ]
2796 .simd_into(self)
2797 }
2798 #[inline(always)]
2799 fn pairwise_dot_product_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u16x8<Self> {
2800 let (low, high) = self.widening_mul_u8x16(a, b);
2801 let (even, odd) = self.deinterleave_u16x8(low, high);
2802 self.add_u16x8(even, odd)
2803 }
2804 #[inline(always)]
2805 fn splat_mask8x16(self, val: bool) -> mask8x16<Self> {
2806 let val: i8 = if val { !0 } else { 0 };
2807 [val; 16usize].simd_into(self)
2808 }
2809 #[inline(always)]
2810 fn from_bitmask_mask8x16(self, bits: u64) -> mask8x16<Self> {
2811 let lanes: [i8; 16usize] = [
2812 if bits & 1 != 0 { !0 } else { 0 },
2813 if (bits >> 1usize) & 1 != 0 { !0 } else { 0 },
2814 if (bits >> 2usize) & 1 != 0 { !0 } else { 0 },
2815 if (bits >> 3usize) & 1 != 0 { !0 } else { 0 },
2816 if (bits >> 4usize) & 1 != 0 { !0 } else { 0 },
2817 if (bits >> 5usize) & 1 != 0 { !0 } else { 0 },
2818 if (bits >> 6usize) & 1 != 0 { !0 } else { 0 },
2819 if (bits >> 7usize) & 1 != 0 { !0 } else { 0 },
2820 if (bits >> 8usize) & 1 != 0 { !0 } else { 0 },
2821 if (bits >> 9usize) & 1 != 0 { !0 } else { 0 },
2822 if (bits >> 10usize) & 1 != 0 { !0 } else { 0 },
2823 if (bits >> 11usize) & 1 != 0 { !0 } else { 0 },
2824 if (bits >> 12usize) & 1 != 0 { !0 } else { 0 },
2825 if (bits >> 13usize) & 1 != 0 { !0 } else { 0 },
2826 if (bits >> 14usize) & 1 != 0 { !0 } else { 0 },
2827 if (bits >> 15usize) & 1 != 0 { !0 } else { 0 },
2828 ];
2829 lanes.simd_into(self)
2830 }
2831 #[inline(always)]
2832 fn to_bitmask_mask8x16(self, a: mask8x16<Self>) -> u64 {
2833 let lanes: [i8; 16usize] = a.into();
2834 let mut bits = 0u64;
2835 let mut i = 0;
2836 while i < 16usize {
2837 if lanes[i] != 0 {
2838 bits |= 1u64 << i;
2839 }
2840 i += 1;
2841 }
2842 bits
2843 }
2844 #[inline(always)]
2845 fn from_vector_mask8x16(self, vector: i8x16<Self>) -> mask8x16<Self> {
2846 mask8x16 {
2847 val: vector.val,
2848 simd: self,
2849 }
2850 }
2851 #[inline(always)]
2852 fn to_vector_mask8x16(self, a: mask8x16<Self>) -> i8x16<Self> {
2853 i8x16 {
2854 val: a.val,
2855 simd: self,
2856 }
2857 }
2858 #[inline(always)]
2859 fn set_mask8x16(self, a: &mut mask8x16<Self>, index: usize, value: bool) -> () {
2860 assert!(
2861 index < 16usize,
2862 "mask lane index {index} is out of bounds for {} lanes",
2863 16usize
2864 );
2865 let mut lanes: [i8; 16usize] = (*a).into();
2866 lanes[index] = if value { !0 } else { 0 };
2867 *a = lanes.simd_into(self);
2868 }
2869 #[inline(always)]
2870 fn rotate_elements_left_mask8x16<const OFFSET: usize>(
2871 self,
2872 a: mask8x16<Self>,
2873 ) -> mask8x16<Self> {
2874 let int = i8x16 {
2875 val: crate::transmute::checked_transmute_copy(&a.val),
2876 simd: self,
2877 };
2878 let rotated = int.rotate_elements_left::<OFFSET>();
2879 mask8x16 {
2880 val: crate::transmute::checked_transmute_copy(&rotated.val),
2881 simd: self,
2882 }
2883 }
2884 #[inline(always)]
2885 fn rotate_elements_right_mask8x16<const OFFSET: usize>(
2886 self,
2887 a: mask8x16<Self>,
2888 ) -> mask8x16<Self> {
2889 let int = i8x16 {
2890 val: crate::transmute::checked_transmute_copy(&a.val),
2891 simd: self,
2892 };
2893 let rotated = int.rotate_elements_right::<OFFSET>();
2894 mask8x16 {
2895 val: crate::transmute::checked_transmute_copy(&rotated.val),
2896 simd: self,
2897 }
2898 }
2899 #[inline(always)]
2900 fn and_mask8x16(self, a: mask8x16<Self>, b: mask8x16<Self>) -> mask8x16<Self> {
2901 [
2902 i8::bitand(a.val.0[0usize], &b.val.0[0usize]),
2903 i8::bitand(a.val.0[1usize], &b.val.0[1usize]),
2904 i8::bitand(a.val.0[2usize], &b.val.0[2usize]),
2905 i8::bitand(a.val.0[3usize], &b.val.0[3usize]),
2906 i8::bitand(a.val.0[4usize], &b.val.0[4usize]),
2907 i8::bitand(a.val.0[5usize], &b.val.0[5usize]),
2908 i8::bitand(a.val.0[6usize], &b.val.0[6usize]),
2909 i8::bitand(a.val.0[7usize], &b.val.0[7usize]),
2910 i8::bitand(a.val.0[8usize], &b.val.0[8usize]),
2911 i8::bitand(a.val.0[9usize], &b.val.0[9usize]),
2912 i8::bitand(a.val.0[10usize], &b.val.0[10usize]),
2913 i8::bitand(a.val.0[11usize], &b.val.0[11usize]),
2914 i8::bitand(a.val.0[12usize], &b.val.0[12usize]),
2915 i8::bitand(a.val.0[13usize], &b.val.0[13usize]),
2916 i8::bitand(a.val.0[14usize], &b.val.0[14usize]),
2917 i8::bitand(a.val.0[15usize], &b.val.0[15usize]),
2918 ]
2919 .simd_into(self)
2920 }
2921 #[inline(always)]
2922 fn or_mask8x16(self, a: mask8x16<Self>, b: mask8x16<Self>) -> mask8x16<Self> {
2923 [
2924 i8::bitor(a.val.0[0usize], &b.val.0[0usize]),
2925 i8::bitor(a.val.0[1usize], &b.val.0[1usize]),
2926 i8::bitor(a.val.0[2usize], &b.val.0[2usize]),
2927 i8::bitor(a.val.0[3usize], &b.val.0[3usize]),
2928 i8::bitor(a.val.0[4usize], &b.val.0[4usize]),
2929 i8::bitor(a.val.0[5usize], &b.val.0[5usize]),
2930 i8::bitor(a.val.0[6usize], &b.val.0[6usize]),
2931 i8::bitor(a.val.0[7usize], &b.val.0[7usize]),
2932 i8::bitor(a.val.0[8usize], &b.val.0[8usize]),
2933 i8::bitor(a.val.0[9usize], &b.val.0[9usize]),
2934 i8::bitor(a.val.0[10usize], &b.val.0[10usize]),
2935 i8::bitor(a.val.0[11usize], &b.val.0[11usize]),
2936 i8::bitor(a.val.0[12usize], &b.val.0[12usize]),
2937 i8::bitor(a.val.0[13usize], &b.val.0[13usize]),
2938 i8::bitor(a.val.0[14usize], &b.val.0[14usize]),
2939 i8::bitor(a.val.0[15usize], &b.val.0[15usize]),
2940 ]
2941 .simd_into(self)
2942 }
2943 #[inline(always)]
2944 fn xor_mask8x16(self, a: mask8x16<Self>, b: mask8x16<Self>) -> mask8x16<Self> {
2945 [
2946 i8::bitxor(a.val.0[0usize], &b.val.0[0usize]),
2947 i8::bitxor(a.val.0[1usize], &b.val.0[1usize]),
2948 i8::bitxor(a.val.0[2usize], &b.val.0[2usize]),
2949 i8::bitxor(a.val.0[3usize], &b.val.0[3usize]),
2950 i8::bitxor(a.val.0[4usize], &b.val.0[4usize]),
2951 i8::bitxor(a.val.0[5usize], &b.val.0[5usize]),
2952 i8::bitxor(a.val.0[6usize], &b.val.0[6usize]),
2953 i8::bitxor(a.val.0[7usize], &b.val.0[7usize]),
2954 i8::bitxor(a.val.0[8usize], &b.val.0[8usize]),
2955 i8::bitxor(a.val.0[9usize], &b.val.0[9usize]),
2956 i8::bitxor(a.val.0[10usize], &b.val.0[10usize]),
2957 i8::bitxor(a.val.0[11usize], &b.val.0[11usize]),
2958 i8::bitxor(a.val.0[12usize], &b.val.0[12usize]),
2959 i8::bitxor(a.val.0[13usize], &b.val.0[13usize]),
2960 i8::bitxor(a.val.0[14usize], &b.val.0[14usize]),
2961 i8::bitxor(a.val.0[15usize], &b.val.0[15usize]),
2962 ]
2963 .simd_into(self)
2964 }
2965 #[inline(always)]
2966 fn not_mask8x16(self, a: mask8x16<Self>) -> mask8x16<Self> {
2967 [
2968 i8::not(a.val.0[0usize]),
2969 i8::not(a.val.0[1usize]),
2970 i8::not(a.val.0[2usize]),
2971 i8::not(a.val.0[3usize]),
2972 i8::not(a.val.0[4usize]),
2973 i8::not(a.val.0[5usize]),
2974 i8::not(a.val.0[6usize]),
2975 i8::not(a.val.0[7usize]),
2976 i8::not(a.val.0[8usize]),
2977 i8::not(a.val.0[9usize]),
2978 i8::not(a.val.0[10usize]),
2979 i8::not(a.val.0[11usize]),
2980 i8::not(a.val.0[12usize]),
2981 i8::not(a.val.0[13usize]),
2982 i8::not(a.val.0[14usize]),
2983 i8::not(a.val.0[15usize]),
2984 ]
2985 .simd_into(self)
2986 }
2987 #[inline(always)]
2988 fn reverse_mask8x16(self, a: mask8x16<Self>) -> mask8x16<Self> {
2989 [
2990 a.val.0[15usize],
2991 a.val.0[14usize],
2992 a.val.0[13usize],
2993 a.val.0[12usize],
2994 a.val.0[11usize],
2995 a.val.0[10usize],
2996 a.val.0[9usize],
2997 a.val.0[8usize],
2998 a.val.0[7usize],
2999 a.val.0[6usize],
3000 a.val.0[5usize],
3001 a.val.0[4usize],
3002 a.val.0[3usize],
3003 a.val.0[2usize],
3004 a.val.0[1usize],
3005 a.val.0[0usize],
3006 ]
3007 .simd_into(self)
3008 }
3009 #[inline(always)]
3010 fn select_mask8x16(
3011 self,
3012 a: mask8x16<Self>,
3013 b: mask8x16<Self>,
3014 c: mask8x16<Self>,
3015 ) -> mask8x16<Self> {
3016 [
3017 if a.val.0[0usize] != 0 {
3018 b.val.0[0usize]
3019 } else {
3020 c.val.0[0usize]
3021 },
3022 if a.val.0[1usize] != 0 {
3023 b.val.0[1usize]
3024 } else {
3025 c.val.0[1usize]
3026 },
3027 if a.val.0[2usize] != 0 {
3028 b.val.0[2usize]
3029 } else {
3030 c.val.0[2usize]
3031 },
3032 if a.val.0[3usize] != 0 {
3033 b.val.0[3usize]
3034 } else {
3035 c.val.0[3usize]
3036 },
3037 if a.val.0[4usize] != 0 {
3038 b.val.0[4usize]
3039 } else {
3040 c.val.0[4usize]
3041 },
3042 if a.val.0[5usize] != 0 {
3043 b.val.0[5usize]
3044 } else {
3045 c.val.0[5usize]
3046 },
3047 if a.val.0[6usize] != 0 {
3048 b.val.0[6usize]
3049 } else {
3050 c.val.0[6usize]
3051 },
3052 if a.val.0[7usize] != 0 {
3053 b.val.0[7usize]
3054 } else {
3055 c.val.0[7usize]
3056 },
3057 if a.val.0[8usize] != 0 {
3058 b.val.0[8usize]
3059 } else {
3060 c.val.0[8usize]
3061 },
3062 if a.val.0[9usize] != 0 {
3063 b.val.0[9usize]
3064 } else {
3065 c.val.0[9usize]
3066 },
3067 if a.val.0[10usize] != 0 {
3068 b.val.0[10usize]
3069 } else {
3070 c.val.0[10usize]
3071 },
3072 if a.val.0[11usize] != 0 {
3073 b.val.0[11usize]
3074 } else {
3075 c.val.0[11usize]
3076 },
3077 if a.val.0[12usize] != 0 {
3078 b.val.0[12usize]
3079 } else {
3080 c.val.0[12usize]
3081 },
3082 if a.val.0[13usize] != 0 {
3083 b.val.0[13usize]
3084 } else {
3085 c.val.0[13usize]
3086 },
3087 if a.val.0[14usize] != 0 {
3088 b.val.0[14usize]
3089 } else {
3090 c.val.0[14usize]
3091 },
3092 if a.val.0[15usize] != 0 {
3093 b.val.0[15usize]
3094 } else {
3095 c.val.0[15usize]
3096 },
3097 ]
3098 .simd_into(self)
3099 }
3100 #[inline(always)]
3101 fn simd_eq_mask8x16(self, a: mask8x16<Self>, b: mask8x16<Self>) -> mask8x16<Self> {
3102 [
3103 -(i8::eq(&a.val.0[0usize], &b.val.0[0usize]) as i8),
3104 -(i8::eq(&a.val.0[1usize], &b.val.0[1usize]) as i8),
3105 -(i8::eq(&a.val.0[2usize], &b.val.0[2usize]) as i8),
3106 -(i8::eq(&a.val.0[3usize], &b.val.0[3usize]) as i8),
3107 -(i8::eq(&a.val.0[4usize], &b.val.0[4usize]) as i8),
3108 -(i8::eq(&a.val.0[5usize], &b.val.0[5usize]) as i8),
3109 -(i8::eq(&a.val.0[6usize], &b.val.0[6usize]) as i8),
3110 -(i8::eq(&a.val.0[7usize], &b.val.0[7usize]) as i8),
3111 -(i8::eq(&a.val.0[8usize], &b.val.0[8usize]) as i8),
3112 -(i8::eq(&a.val.0[9usize], &b.val.0[9usize]) as i8),
3113 -(i8::eq(&a.val.0[10usize], &b.val.0[10usize]) as i8),
3114 -(i8::eq(&a.val.0[11usize], &b.val.0[11usize]) as i8),
3115 -(i8::eq(&a.val.0[12usize], &b.val.0[12usize]) as i8),
3116 -(i8::eq(&a.val.0[13usize], &b.val.0[13usize]) as i8),
3117 -(i8::eq(&a.val.0[14usize], &b.val.0[14usize]) as i8),
3118 -(i8::eq(&a.val.0[15usize], &b.val.0[15usize]) as i8),
3119 ]
3120 .simd_into(self)
3121 }
3122 #[inline(always)]
3123 fn any_true_mask8x16(self, a: mask8x16<Self>) -> bool {
3124 a.val.0[0usize] != 0
3125 || a.val.0[1usize] != 0
3126 || a.val.0[2usize] != 0
3127 || a.val.0[3usize] != 0
3128 || a.val.0[4usize] != 0
3129 || a.val.0[5usize] != 0
3130 || a.val.0[6usize] != 0
3131 || a.val.0[7usize] != 0
3132 || a.val.0[8usize] != 0
3133 || a.val.0[9usize] != 0
3134 || a.val.0[10usize] != 0
3135 || a.val.0[11usize] != 0
3136 || a.val.0[12usize] != 0
3137 || a.val.0[13usize] != 0
3138 || a.val.0[14usize] != 0
3139 || a.val.0[15usize] != 0
3140 }
3141 #[inline(always)]
3142 fn all_true_mask8x16(self, a: mask8x16<Self>) -> bool {
3143 a.val.0[0usize] != 0
3144 && a.val.0[1usize] != 0
3145 && a.val.0[2usize] != 0
3146 && a.val.0[3usize] != 0
3147 && a.val.0[4usize] != 0
3148 && a.val.0[5usize] != 0
3149 && a.val.0[6usize] != 0
3150 && a.val.0[7usize] != 0
3151 && a.val.0[8usize] != 0
3152 && a.val.0[9usize] != 0
3153 && a.val.0[10usize] != 0
3154 && a.val.0[11usize] != 0
3155 && a.val.0[12usize] != 0
3156 && a.val.0[13usize] != 0
3157 && a.val.0[14usize] != 0
3158 && a.val.0[15usize] != 0
3159 }
3160 #[inline(always)]
3161 fn any_false_mask8x16(self, a: mask8x16<Self>) -> bool {
3162 a.val.0[0usize] == 0
3163 || a.val.0[1usize] == 0
3164 || a.val.0[2usize] == 0
3165 || a.val.0[3usize] == 0
3166 || a.val.0[4usize] == 0
3167 || a.val.0[5usize] == 0
3168 || a.val.0[6usize] == 0
3169 || a.val.0[7usize] == 0
3170 || a.val.0[8usize] == 0
3171 || a.val.0[9usize] == 0
3172 || a.val.0[10usize] == 0
3173 || a.val.0[11usize] == 0
3174 || a.val.0[12usize] == 0
3175 || a.val.0[13usize] == 0
3176 || a.val.0[14usize] == 0
3177 || a.val.0[15usize] == 0
3178 }
3179 #[inline(always)]
3180 fn all_false_mask8x16(self, a: mask8x16<Self>) -> bool {
3181 a.val.0[0usize] == 0
3182 && a.val.0[1usize] == 0
3183 && a.val.0[2usize] == 0
3184 && a.val.0[3usize] == 0
3185 && a.val.0[4usize] == 0
3186 && a.val.0[5usize] == 0
3187 && a.val.0[6usize] == 0
3188 && a.val.0[7usize] == 0
3189 && a.val.0[8usize] == 0
3190 && a.val.0[9usize] == 0
3191 && a.val.0[10usize] == 0
3192 && a.val.0[11usize] == 0
3193 && a.val.0[12usize] == 0
3194 && a.val.0[13usize] == 0
3195 && a.val.0[14usize] == 0
3196 && a.val.0[15usize] == 0
3197 }
3198 #[inline(always)]
3199 fn combine_mask8x16(self, a: mask8x16<Self>, b: mask8x16<Self>) -> mask8x32<Self> {
3200 let mut result = [0; 32usize];
3201 result[0..16usize].copy_from_slice(&a.val.0);
3202 result[16usize..32usize].copy_from_slice(&b.val.0);
3203 result.simd_into(self)
3204 }
3205 #[inline(always)]
3206 fn widen_mask8x16(self, a: mask8x16<Self>) -> (mask16x8<Self>, mask16x8<Self>) {
3207 (
3208 [
3209 a.val.0[0usize] as i16,
3210 a.val.0[1usize] as i16,
3211 a.val.0[2usize] as i16,
3212 a.val.0[3usize] as i16,
3213 a.val.0[4usize] as i16,
3214 a.val.0[5usize] as i16,
3215 a.val.0[6usize] as i16,
3216 a.val.0[7usize] as i16,
3217 ]
3218 .simd_into(self),
3219 [
3220 a.val.0[8usize] as i16,
3221 a.val.0[9usize] as i16,
3222 a.val.0[10usize] as i16,
3223 a.val.0[11usize] as i16,
3224 a.val.0[12usize] as i16,
3225 a.val.0[13usize] as i16,
3226 a.val.0[14usize] as i16,
3227 a.val.0[15usize] as i16,
3228 ]
3229 .simd_into(self),
3230 )
3231 }
3232 #[inline(always)]
3233 fn abs_i16x8(self, a: i16x8<Self>) -> i16x8<Self> {
3234 [
3235 i16::wrapping_abs(a[0usize]),
3236 i16::wrapping_abs(a[1usize]),
3237 i16::wrapping_abs(a[2usize]),
3238 i16::wrapping_abs(a[3usize]),
3239 i16::wrapping_abs(a[4usize]),
3240 i16::wrapping_abs(a[5usize]),
3241 i16::wrapping_abs(a[6usize]),
3242 i16::wrapping_abs(a[7usize]),
3243 ]
3244 .simd_into(self)
3245 }
3246 #[inline(always)]
3247 fn splat_i16x8(self, val: i16) -> i16x8<Self> {
3248 [val; 8usize].simd_into(self)
3249 }
3250 #[inline(always)]
3251 fn reverse_i16x8(self, a: i16x8<Self>) -> i16x8<Self> {
3252 [
3253 a[7usize], a[6usize], a[5usize], a[4usize], a[3usize], a[2usize], a[1usize], a[0usize],
3254 ]
3255 .simd_into(self)
3256 }
3257 #[inline(always)]
3258 fn slide_i16x8<const SHIFT: usize>(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
3259 let mut dest = [Default::default(); 8usize];
3260 dest[..8usize - SHIFT].copy_from_slice(&a.val.0[SHIFT..]);
3261 dest[8usize - SHIFT..].copy_from_slice(&b.val.0[..SHIFT]);
3262 dest.simd_into(self)
3263 }
3264 #[inline(always)]
3265 fn compress_i16x8(self, values: i16x8<Self>, mask: mask16x8<Self>) -> i16x8<Self> {
3266 let mask = mask8x16 {
3267 val: crate::transmute::checked_transmute_copy(&mask.val),
3268 simd: self,
3269 };
3270 Bytes::from_bytes(self.compress_u8x16(Bytes::to_bytes(values), mask))
3271 }
3272 #[inline(always)]
3273 fn compress_merge_i16x8(
3274 self,
3275 values: i16x8<Self>,
3276 mask: mask16x8<Self>,
3277 merge: i16x8<Self>,
3278 ) -> i16x8<Self> {
3279 let mask = mask8x16 {
3280 val: crate::transmute::checked_transmute_copy(&mask.val),
3281 simd: self,
3282 };
3283 Bytes::from_bytes(self.compress_merge_u8x16(
3284 Bytes::to_bytes(values),
3285 mask,
3286 Bytes::to_bytes(merge),
3287 ))
3288 }
3289 #[inline(always)]
3290 fn expand_i16x8(self, values: i16x8<Self>, mask: mask16x8<Self>) -> i16x8<Self> {
3291 let mask = mask8x16 {
3292 val: crate::transmute::checked_transmute_copy(&mask.val),
3293 simd: self,
3294 };
3295 Bytes::from_bytes(self.expand_u8x16(Bytes::to_bytes(values), mask))
3296 }
3297 #[inline(always)]
3298 fn expand_merge_i16x8(
3299 self,
3300 values: i16x8<Self>,
3301 mask: mask16x8<Self>,
3302 merge: i16x8<Self>,
3303 ) -> i16x8<Self> {
3304 let mask = mask8x16 {
3305 val: crate::transmute::checked_transmute_copy(&mask.val),
3306 simd: self,
3307 };
3308 Bytes::from_bytes(self.expand_merge_u8x16(
3309 Bytes::to_bytes(values),
3310 mask,
3311 Bytes::to_bytes(merge),
3312 ))
3313 }
3314 #[inline(always)]
3315 fn count_ones_i16x8(self, a: i16x8<Self>) -> i16x8<Self> {
3316 [
3317 i16::try_from(a[0usize].count_ones()).unwrap(),
3318 i16::try_from(a[1usize].count_ones()).unwrap(),
3319 i16::try_from(a[2usize].count_ones()).unwrap(),
3320 i16::try_from(a[3usize].count_ones()).unwrap(),
3321 i16::try_from(a[4usize].count_ones()).unwrap(),
3322 i16::try_from(a[5usize].count_ones()).unwrap(),
3323 i16::try_from(a[6usize].count_ones()).unwrap(),
3324 i16::try_from(a[7usize].count_ones()).unwrap(),
3325 ]
3326 .simd_into(self)
3327 }
3328 #[inline(always)]
3329 fn count_zeros_i16x8(self, a: i16x8<Self>) -> i16x8<Self> {
3330 [
3331 i16::try_from(a[0usize].count_zeros()).unwrap(),
3332 i16::try_from(a[1usize].count_zeros()).unwrap(),
3333 i16::try_from(a[2usize].count_zeros()).unwrap(),
3334 i16::try_from(a[3usize].count_zeros()).unwrap(),
3335 i16::try_from(a[4usize].count_zeros()).unwrap(),
3336 i16::try_from(a[5usize].count_zeros()).unwrap(),
3337 i16::try_from(a[6usize].count_zeros()).unwrap(),
3338 i16::try_from(a[7usize].count_zeros()).unwrap(),
3339 ]
3340 .simd_into(self)
3341 }
3342 #[inline(always)]
3343 fn add_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
3344 [
3345 i16::wrapping_add(a[0usize], b[0usize]),
3346 i16::wrapping_add(a[1usize], b[1usize]),
3347 i16::wrapping_add(a[2usize], b[2usize]),
3348 i16::wrapping_add(a[3usize], b[3usize]),
3349 i16::wrapping_add(a[4usize], b[4usize]),
3350 i16::wrapping_add(a[5usize], b[5usize]),
3351 i16::wrapping_add(a[6usize], b[6usize]),
3352 i16::wrapping_add(a[7usize], b[7usize]),
3353 ]
3354 .simd_into(self)
3355 }
3356 #[inline(always)]
3357 fn saturating_add_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
3358 [
3359 i16::saturating_add(a[0usize], b[0usize]),
3360 i16::saturating_add(a[1usize], b[1usize]),
3361 i16::saturating_add(a[2usize], b[2usize]),
3362 i16::saturating_add(a[3usize], b[3usize]),
3363 i16::saturating_add(a[4usize], b[4usize]),
3364 i16::saturating_add(a[5usize], b[5usize]),
3365 i16::saturating_add(a[6usize], b[6usize]),
3366 i16::saturating_add(a[7usize], b[7usize]),
3367 ]
3368 .simd_into(self)
3369 }
3370 #[inline(always)]
3371 fn sub_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
3372 [
3373 i16::wrapping_sub(a[0usize], b[0usize]),
3374 i16::wrapping_sub(a[1usize], b[1usize]),
3375 i16::wrapping_sub(a[2usize], b[2usize]),
3376 i16::wrapping_sub(a[3usize], b[3usize]),
3377 i16::wrapping_sub(a[4usize], b[4usize]),
3378 i16::wrapping_sub(a[5usize], b[5usize]),
3379 i16::wrapping_sub(a[6usize], b[6usize]),
3380 i16::wrapping_sub(a[7usize], b[7usize]),
3381 ]
3382 .simd_into(self)
3383 }
3384 #[inline(always)]
3385 fn saturating_sub_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
3386 [
3387 i16::saturating_sub(a[0usize], b[0usize]),
3388 i16::saturating_sub(a[1usize], b[1usize]),
3389 i16::saturating_sub(a[2usize], b[2usize]),
3390 i16::saturating_sub(a[3usize], b[3usize]),
3391 i16::saturating_sub(a[4usize], b[4usize]),
3392 i16::saturating_sub(a[5usize], b[5usize]),
3393 i16::saturating_sub(a[6usize], b[6usize]),
3394 i16::saturating_sub(a[7usize], b[7usize]),
3395 ]
3396 .simd_into(self)
3397 }
3398 #[inline(always)]
3399 fn mul_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
3400 [
3401 i16::wrapping_mul(a[0usize], b[0usize]),
3402 i16::wrapping_mul(a[1usize], b[1usize]),
3403 i16::wrapping_mul(a[2usize], b[2usize]),
3404 i16::wrapping_mul(a[3usize], b[3usize]),
3405 i16::wrapping_mul(a[4usize], b[4usize]),
3406 i16::wrapping_mul(a[5usize], b[5usize]),
3407 i16::wrapping_mul(a[6usize], b[6usize]),
3408 i16::wrapping_mul(a[7usize], b[7usize]),
3409 ]
3410 .simd_into(self)
3411 }
3412 #[inline(always)]
3413 fn and_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
3414 [
3415 i16::bitand(a[0usize], &b[0usize]),
3416 i16::bitand(a[1usize], &b[1usize]),
3417 i16::bitand(a[2usize], &b[2usize]),
3418 i16::bitand(a[3usize], &b[3usize]),
3419 i16::bitand(a[4usize], &b[4usize]),
3420 i16::bitand(a[5usize], &b[5usize]),
3421 i16::bitand(a[6usize], &b[6usize]),
3422 i16::bitand(a[7usize], &b[7usize]),
3423 ]
3424 .simd_into(self)
3425 }
3426 #[inline(always)]
3427 fn or_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
3428 [
3429 i16::bitor(a[0usize], &b[0usize]),
3430 i16::bitor(a[1usize], &b[1usize]),
3431 i16::bitor(a[2usize], &b[2usize]),
3432 i16::bitor(a[3usize], &b[3usize]),
3433 i16::bitor(a[4usize], &b[4usize]),
3434 i16::bitor(a[5usize], &b[5usize]),
3435 i16::bitor(a[6usize], &b[6usize]),
3436 i16::bitor(a[7usize], &b[7usize]),
3437 ]
3438 .simd_into(self)
3439 }
3440 #[inline(always)]
3441 fn xor_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
3442 [
3443 i16::bitxor(a[0usize], &b[0usize]),
3444 i16::bitxor(a[1usize], &b[1usize]),
3445 i16::bitxor(a[2usize], &b[2usize]),
3446 i16::bitxor(a[3usize], &b[3usize]),
3447 i16::bitxor(a[4usize], &b[4usize]),
3448 i16::bitxor(a[5usize], &b[5usize]),
3449 i16::bitxor(a[6usize], &b[6usize]),
3450 i16::bitxor(a[7usize], &b[7usize]),
3451 ]
3452 .simd_into(self)
3453 }
3454 #[inline(always)]
3455 fn not_i16x8(self, a: i16x8<Self>) -> i16x8<Self> {
3456 [
3457 i16::not(a[0usize]),
3458 i16::not(a[1usize]),
3459 i16::not(a[2usize]),
3460 i16::not(a[3usize]),
3461 i16::not(a[4usize]),
3462 i16::not(a[5usize]),
3463 i16::not(a[6usize]),
3464 i16::not(a[7usize]),
3465 ]
3466 .simd_into(self)
3467 }
3468 #[inline(always)]
3469 fn shl_i16x8(self, a: i16x8<Self>, shift: u32) -> i16x8<Self> {
3470 [
3471 i16::wrapping_shl(a[0usize], shift),
3472 i16::wrapping_shl(a[1usize], shift),
3473 i16::wrapping_shl(a[2usize], shift),
3474 i16::wrapping_shl(a[3usize], shift),
3475 i16::wrapping_shl(a[4usize], shift),
3476 i16::wrapping_shl(a[5usize], shift),
3477 i16::wrapping_shl(a[6usize], shift),
3478 i16::wrapping_shl(a[7usize], shift),
3479 ]
3480 .simd_into(self)
3481 }
3482 #[inline(always)]
3483 fn shlv_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
3484 [
3485 i16::wrapping_shl(a[0usize], b[0usize] as u32),
3486 i16::wrapping_shl(a[1usize], b[1usize] as u32),
3487 i16::wrapping_shl(a[2usize], b[2usize] as u32),
3488 i16::wrapping_shl(a[3usize], b[3usize] as u32),
3489 i16::wrapping_shl(a[4usize], b[4usize] as u32),
3490 i16::wrapping_shl(a[5usize], b[5usize] as u32),
3491 i16::wrapping_shl(a[6usize], b[6usize] as u32),
3492 i16::wrapping_shl(a[7usize], b[7usize] as u32),
3493 ]
3494 .simd_into(self)
3495 }
3496 #[inline(always)]
3497 fn shr_i16x8(self, a: i16x8<Self>, shift: u32) -> i16x8<Self> {
3498 [
3499 i16::wrapping_shr(a[0usize], shift),
3500 i16::wrapping_shr(a[1usize], shift),
3501 i16::wrapping_shr(a[2usize], shift),
3502 i16::wrapping_shr(a[3usize], shift),
3503 i16::wrapping_shr(a[4usize], shift),
3504 i16::wrapping_shr(a[5usize], shift),
3505 i16::wrapping_shr(a[6usize], shift),
3506 i16::wrapping_shr(a[7usize], shift),
3507 ]
3508 .simd_into(self)
3509 }
3510 #[inline(always)]
3511 fn shrv_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
3512 [
3513 i16::wrapping_shr(a[0usize], b[0usize] as u32),
3514 i16::wrapping_shr(a[1usize], b[1usize] as u32),
3515 i16::wrapping_shr(a[2usize], b[2usize] as u32),
3516 i16::wrapping_shr(a[3usize], b[3usize] as u32),
3517 i16::wrapping_shr(a[4usize], b[4usize] as u32),
3518 i16::wrapping_shr(a[5usize], b[5usize] as u32),
3519 i16::wrapping_shr(a[6usize], b[6usize] as u32),
3520 i16::wrapping_shr(a[7usize], b[7usize] as u32),
3521 ]
3522 .simd_into(self)
3523 }
3524 #[inline(always)]
3525 fn reduce_max_i16x8(self, a: i16x8<Self>) -> i16 {
3526 let reduced: [i16; 4usize] = [
3527 i16::max(a[0usize], a[1usize]),
3528 i16::max(a[2usize], a[3usize]),
3529 i16::max(a[4usize], a[5usize]),
3530 i16::max(a[6usize], a[7usize]),
3531 ];
3532 let reduced: [i16; 2usize] = [
3533 i16::max(reduced[0usize], reduced[1usize]),
3534 i16::max(reduced[2usize], reduced[3usize]),
3535 ];
3536 let reduced: [i16; 1usize] = [i16::max(reduced[0usize], reduced[1usize])];
3537 reduced[0]
3538 }
3539 #[inline(always)]
3540 fn reduce_min_i16x8(self, a: i16x8<Self>) -> i16 {
3541 let reduced: [i16; 4usize] = [
3542 i16::min(a[0usize], a[1usize]),
3543 i16::min(a[2usize], a[3usize]),
3544 i16::min(a[4usize], a[5usize]),
3545 i16::min(a[6usize], a[7usize]),
3546 ];
3547 let reduced: [i16; 2usize] = [
3548 i16::min(reduced[0usize], reduced[1usize]),
3549 i16::min(reduced[2usize], reduced[3usize]),
3550 ];
3551 let reduced: [i16; 1usize] = [i16::min(reduced[0usize], reduced[1usize])];
3552 reduced[0]
3553 }
3554 #[inline(always)]
3555 fn reduce_sum_i16x8(self, a: i16x8<Self>) -> i16 {
3556 let add_level_0: [i16; 4usize] = [
3557 a[0usize].wrapping_add(a[1usize]),
3558 a[2usize].wrapping_add(a[3usize]),
3559 a[4usize].wrapping_add(a[5usize]),
3560 a[6usize].wrapping_add(a[7usize]),
3561 ];
3562 let add_level_1: [i16; 2usize] = [
3563 add_level_0[0usize].wrapping_add(add_level_0[1usize]),
3564 add_level_0[2usize].wrapping_add(add_level_0[3usize]),
3565 ];
3566 let add_level_2: [i16; 1usize] = [add_level_1[0usize].wrapping_add(add_level_1[1usize])];
3567 add_level_2[0]
3568 }
3569 #[inline(always)]
3570 fn reduce_product_i16x8(self, a: i16x8<Self>) -> i16 {
3571 let mul_level_0: [i16; 4usize] = [
3572 a[0usize].wrapping_mul(a[1usize]),
3573 a[2usize].wrapping_mul(a[3usize]),
3574 a[4usize].wrapping_mul(a[5usize]),
3575 a[6usize].wrapping_mul(a[7usize]),
3576 ];
3577 let mul_level_1: [i16; 2usize] = [
3578 mul_level_0[0usize].wrapping_mul(mul_level_0[1usize]),
3579 mul_level_0[2usize].wrapping_mul(mul_level_0[3usize]),
3580 ];
3581 let mul_level_2: [i16; 1usize] = [mul_level_1[0usize].wrapping_mul(mul_level_1[1usize])];
3582 mul_level_2[0]
3583 }
3584 #[inline(always)]
3585 fn max_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
3586 [
3587 i16::max(a[0usize], b[0usize]),
3588 i16::max(a[1usize], b[1usize]),
3589 i16::max(a[2usize], b[2usize]),
3590 i16::max(a[3usize], b[3usize]),
3591 i16::max(a[4usize], b[4usize]),
3592 i16::max(a[5usize], b[5usize]),
3593 i16::max(a[6usize], b[6usize]),
3594 i16::max(a[7usize], b[7usize]),
3595 ]
3596 .simd_into(self)
3597 }
3598 #[inline(always)]
3599 fn min_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
3600 [
3601 i16::min(a[0usize], b[0usize]),
3602 i16::min(a[1usize], b[1usize]),
3603 i16::min(a[2usize], b[2usize]),
3604 i16::min(a[3usize], b[3usize]),
3605 i16::min(a[4usize], b[4usize]),
3606 i16::min(a[5usize], b[5usize]),
3607 i16::min(a[6usize], b[6usize]),
3608 i16::min(a[7usize], b[7usize]),
3609 ]
3610 .simd_into(self)
3611 }
3612 #[inline(always)]
3613 fn clamp_i16x8(self, a: i16x8<Self>, b: i16x8<Self>, c: i16x8<Self>) -> i16x8<Self> {
3614 [
3615 i16::clamp_debug(a[0usize], b[0usize], c[0usize]),
3616 i16::clamp_debug(a[1usize], b[1usize], c[1usize]),
3617 i16::clamp_debug(a[2usize], b[2usize], c[2usize]),
3618 i16::clamp_debug(a[3usize], b[3usize], c[3usize]),
3619 i16::clamp_debug(a[4usize], b[4usize], c[4usize]),
3620 i16::clamp_debug(a[5usize], b[5usize], c[5usize]),
3621 i16::clamp_debug(a[6usize], b[6usize], c[6usize]),
3622 i16::clamp_debug(a[7usize], b[7usize], c[7usize]),
3623 ]
3624 .simd_into(self)
3625 }
3626 #[inline(always)]
3627 fn simd_eq_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> mask16x8<Self> {
3628 [
3629 -(i16::eq(&a[0usize], &b[0usize]) as i16),
3630 -(i16::eq(&a[1usize], &b[1usize]) as i16),
3631 -(i16::eq(&a[2usize], &b[2usize]) as i16),
3632 -(i16::eq(&a[3usize], &b[3usize]) as i16),
3633 -(i16::eq(&a[4usize], &b[4usize]) as i16),
3634 -(i16::eq(&a[5usize], &b[5usize]) as i16),
3635 -(i16::eq(&a[6usize], &b[6usize]) as i16),
3636 -(i16::eq(&a[7usize], &b[7usize]) as i16),
3637 ]
3638 .simd_into(self)
3639 }
3640 #[inline(always)]
3641 fn simd_lt_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> mask16x8<Self> {
3642 [
3643 -(i16::lt(&a[0usize], &b[0usize]) as i16),
3644 -(i16::lt(&a[1usize], &b[1usize]) as i16),
3645 -(i16::lt(&a[2usize], &b[2usize]) as i16),
3646 -(i16::lt(&a[3usize], &b[3usize]) as i16),
3647 -(i16::lt(&a[4usize], &b[4usize]) as i16),
3648 -(i16::lt(&a[5usize], &b[5usize]) as i16),
3649 -(i16::lt(&a[6usize], &b[6usize]) as i16),
3650 -(i16::lt(&a[7usize], &b[7usize]) as i16),
3651 ]
3652 .simd_into(self)
3653 }
3654 #[inline(always)]
3655 fn simd_le_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> mask16x8<Self> {
3656 [
3657 -(i16::le(&a[0usize], &b[0usize]) as i16),
3658 -(i16::le(&a[1usize], &b[1usize]) as i16),
3659 -(i16::le(&a[2usize], &b[2usize]) as i16),
3660 -(i16::le(&a[3usize], &b[3usize]) as i16),
3661 -(i16::le(&a[4usize], &b[4usize]) as i16),
3662 -(i16::le(&a[5usize], &b[5usize]) as i16),
3663 -(i16::le(&a[6usize], &b[6usize]) as i16),
3664 -(i16::le(&a[7usize], &b[7usize]) as i16),
3665 ]
3666 .simd_into(self)
3667 }
3668 #[inline(always)]
3669 fn zip_low_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
3670 [
3671 a[0usize], b[0usize], a[1usize], b[1usize], a[2usize], b[2usize], a[3usize], b[3usize],
3672 ]
3673 .simd_into(self)
3674 }
3675 #[inline(always)]
3676 fn zip_high_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
3677 [
3678 a[4usize], b[4usize], a[5usize], b[5usize], a[6usize], b[6usize], a[7usize], b[7usize],
3679 ]
3680 .simd_into(self)
3681 }
3682 #[inline(always)]
3683 fn unzip_low_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
3684 [
3685 a[0usize], a[2usize], a[4usize], a[6usize], b[0usize], b[2usize], b[4usize], b[6usize],
3686 ]
3687 .simd_into(self)
3688 }
3689 #[inline(always)]
3690 fn unzip_high_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
3691 [
3692 a[1usize], a[3usize], a[5usize], a[7usize], b[1usize], b[3usize], b[5usize], b[7usize],
3693 ]
3694 .simd_into(self)
3695 }
3696 #[inline(always)]
3697 fn interleave_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> (i16x8<Self>, i16x8<Self>) {
3698 (self.zip_low_i16x8(a, b), self.zip_high_i16x8(a, b))
3699 }
3700 #[inline(always)]
3701 fn deinterleave_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> (i16x8<Self>, i16x8<Self>) {
3702 (self.unzip_low_i16x8(a, b), self.unzip_high_i16x8(a, b))
3703 }
3704 #[inline(always)]
3705 fn select_i16x8(self, a: mask16x8<Self>, b: i16x8<Self>, c: i16x8<Self>) -> i16x8<Self> {
3706 [
3707 if a.val.0[0usize] != 0 {
3708 b[0usize]
3709 } else {
3710 c[0usize]
3711 },
3712 if a.val.0[1usize] != 0 {
3713 b[1usize]
3714 } else {
3715 c[1usize]
3716 },
3717 if a.val.0[2usize] != 0 {
3718 b[2usize]
3719 } else {
3720 c[2usize]
3721 },
3722 if a.val.0[3usize] != 0 {
3723 b[3usize]
3724 } else {
3725 c[3usize]
3726 },
3727 if a.val.0[4usize] != 0 {
3728 b[4usize]
3729 } else {
3730 c[4usize]
3731 },
3732 if a.val.0[5usize] != 0 {
3733 b[5usize]
3734 } else {
3735 c[5usize]
3736 },
3737 if a.val.0[6usize] != 0 {
3738 b[6usize]
3739 } else {
3740 c[6usize]
3741 },
3742 if a.val.0[7usize] != 0 {
3743 b[7usize]
3744 } else {
3745 c[7usize]
3746 },
3747 ]
3748 .simd_into(self)
3749 }
3750 #[inline(always)]
3751 fn combine_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x16<Self> {
3752 let mut result = [0; 16usize];
3753 result[0..8usize].copy_from_slice(&a.val.0);
3754 result[8usize..16usize].copy_from_slice(&b.val.0);
3755 result.simd_into(self)
3756 }
3757 #[inline(always)]
3758 fn neg_i16x8(self, a: i16x8<Self>) -> i16x8<Self> {
3759 [
3760 i16::wrapping_neg(a[0usize]),
3761 i16::wrapping_neg(a[1usize]),
3762 i16::wrapping_neg(a[2usize]),
3763 i16::wrapping_neg(a[3usize]),
3764 i16::wrapping_neg(a[4usize]),
3765 i16::wrapping_neg(a[5usize]),
3766 i16::wrapping_neg(a[6usize]),
3767 i16::wrapping_neg(a[7usize]),
3768 ]
3769 .simd_into(self)
3770 }
3771 #[inline(always)]
3772 fn load_four_interleaved_i16x8(self, src: &[i16; 32usize]) -> [i16x8<Self>; 4usize] {
3773 [
3774 [
3775 src[0usize],
3776 src[4usize],
3777 src[8usize],
3778 src[12usize],
3779 src[16usize],
3780 src[20usize],
3781 src[24usize],
3782 src[28usize],
3783 ]
3784 .simd_into(self),
3785 [
3786 src[1usize],
3787 src[5usize],
3788 src[9usize],
3789 src[13usize],
3790 src[17usize],
3791 src[21usize],
3792 src[25usize],
3793 src[29usize],
3794 ]
3795 .simd_into(self),
3796 [
3797 src[2usize],
3798 src[6usize],
3799 src[10usize],
3800 src[14usize],
3801 src[18usize],
3802 src[22usize],
3803 src[26usize],
3804 src[30usize],
3805 ]
3806 .simd_into(self),
3807 [
3808 src[3usize],
3809 src[7usize],
3810 src[11usize],
3811 src[15usize],
3812 src[19usize],
3813 src[23usize],
3814 src[27usize],
3815 src[31usize],
3816 ]
3817 .simd_into(self),
3818 ]
3819 }
3820 #[inline(always)]
3821 fn store_four_interleaved_i16x8(
3822 self,
3823 vectors: [i16x8<Self>; 4usize],
3824 dest: &mut [i16; 32usize],
3825 ) -> () {
3826 *dest = [
3827 vectors[0usize][0usize],
3828 vectors[1usize][0usize],
3829 vectors[2usize][0usize],
3830 vectors[3usize][0usize],
3831 vectors[0usize][1usize],
3832 vectors[1usize][1usize],
3833 vectors[2usize][1usize],
3834 vectors[3usize][1usize],
3835 vectors[0usize][2usize],
3836 vectors[1usize][2usize],
3837 vectors[2usize][2usize],
3838 vectors[3usize][2usize],
3839 vectors[0usize][3usize],
3840 vectors[1usize][3usize],
3841 vectors[2usize][3usize],
3842 vectors[3usize][3usize],
3843 vectors[0usize][4usize],
3844 vectors[1usize][4usize],
3845 vectors[2usize][4usize],
3846 vectors[3usize][4usize],
3847 vectors[0usize][5usize],
3848 vectors[1usize][5usize],
3849 vectors[2usize][5usize],
3850 vectors[3usize][5usize],
3851 vectors[0usize][6usize],
3852 vectors[1usize][6usize],
3853 vectors[2usize][6usize],
3854 vectors[3usize][6usize],
3855 vectors[0usize][7usize],
3856 vectors[1usize][7usize],
3857 vectors[2usize][7usize],
3858 vectors[3usize][7usize],
3859 ];
3860 }
3861 #[inline(always)]
3862 fn widen_i16x8(self, a: i16x8<Self>) -> (i32x4<Self>, i32x4<Self>) {
3863 (
3864 [
3865 a[0usize] as i32,
3866 a[1usize] as i32,
3867 a[2usize] as i32,
3868 a[3usize] as i32,
3869 ]
3870 .simd_into(self),
3871 [
3872 a[4usize] as i32,
3873 a[5usize] as i32,
3874 a[6usize] as i32,
3875 a[7usize] as i32,
3876 ]
3877 .simd_into(self),
3878 )
3879 }
3880 #[inline(always)]
3881 fn widening_mul_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> (i32x4<Self>, i32x4<Self>) {
3882 let (a0, a1) = self.widen_i16x8(a);
3883 let (b0, b1) = self.widen_i16x8(b);
3884 (self.mul_i32x4(a0, b0), self.mul_i32x4(a1, b1))
3885 }
3886 #[inline(always)]
3887 fn widening_add_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> (i32x4<Self>, i32x4<Self>) {
3888 let (a0, a1) = self.widen_i16x8(a);
3889 let (b0, b1) = self.widen_i16x8(b);
3890 (self.add_i32x4(a0, b0), self.add_i32x4(a1, b1))
3891 }
3892 #[inline(always)]
3893 fn pairwise_widening_add_i16x8(self, a: i16x8<Self>) -> i32x4<Self> {
3894 [
3895 (a[0usize] as i32) + (a[1usize] as i32),
3896 (a[2usize] as i32) + (a[3usize] as i32),
3897 (a[4usize] as i32) + (a[5usize] as i32),
3898 (a[6usize] as i32) + (a[7usize] as i32),
3899 ]
3900 .simd_into(self)
3901 }
3902 #[inline(always)]
3903 fn pairwise_dot_product_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i32x4<Self> {
3904 let (low, high) = self.widening_mul_i16x8(a, b);
3905 let (even, odd) = self.deinterleave_i32x4(low, high);
3906 self.add_i32x4(even, odd)
3907 }
3908 #[inline(always)]
3909 fn narrow_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i8x16<Self> {
3910 [
3911 a[0usize] as i8,
3912 a[1usize] as i8,
3913 a[2usize] as i8,
3914 a[3usize] as i8,
3915 a[4usize] as i8,
3916 a[5usize] as i8,
3917 a[6usize] as i8,
3918 a[7usize] as i8,
3919 b[0usize] as i8,
3920 b[1usize] as i8,
3921 b[2usize] as i8,
3922 b[3usize] as i8,
3923 b[4usize] as i8,
3924 b[5usize] as i8,
3925 b[6usize] as i8,
3926 b[7usize] as i8,
3927 ]
3928 .simd_into(self)
3929 }
3930 #[inline(always)]
3931 fn saturating_narrow_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i8x16<Self> {
3932 [
3933 a[0usize].clamp(i8::MIN as i16, i8::MAX as i16) as i8,
3934 a[1usize].clamp(i8::MIN as i16, i8::MAX as i16) as i8,
3935 a[2usize].clamp(i8::MIN as i16, i8::MAX as i16) as i8,
3936 a[3usize].clamp(i8::MIN as i16, i8::MAX as i16) as i8,
3937 a[4usize].clamp(i8::MIN as i16, i8::MAX as i16) as i8,
3938 a[5usize].clamp(i8::MIN as i16, i8::MAX as i16) as i8,
3939 a[6usize].clamp(i8::MIN as i16, i8::MAX as i16) as i8,
3940 a[7usize].clamp(i8::MIN as i16, i8::MAX as i16) as i8,
3941 b[0usize].clamp(i8::MIN as i16, i8::MAX as i16) as i8,
3942 b[1usize].clamp(i8::MIN as i16, i8::MAX as i16) as i8,
3943 b[2usize].clamp(i8::MIN as i16, i8::MAX as i16) as i8,
3944 b[3usize].clamp(i8::MIN as i16, i8::MAX as i16) as i8,
3945 b[4usize].clamp(i8::MIN as i16, i8::MAX as i16) as i8,
3946 b[5usize].clamp(i8::MIN as i16, i8::MAX as i16) as i8,
3947 b[6usize].clamp(i8::MIN as i16, i8::MAX as i16) as i8,
3948 b[7usize].clamp(i8::MIN as i16, i8::MAX as i16) as i8,
3949 ]
3950 .simd_into(self)
3951 }
3952 #[inline(always)]
3953 fn relaxed_narrow_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i8x16<Self> {
3954 debug_assert!(
3955 a.as_slice()
3956 .iter()
3957 .chain(b.as_slice())
3958 .all(|&value| { value >= i8::MIN as i16 && value <= i8::MAX as i16 }),
3959 "relaxed_narrow inputs must fit in the destination type",
3960 );
3961 self.narrow_i16x8(a, b)
3962 }
3963 #[inline(always)]
3964 fn splat_u16x8(self, val: u16) -> u16x8<Self> {
3965 [val; 8usize].simd_into(self)
3966 }
3967 #[inline(always)]
3968 fn reverse_u16x8(self, a: u16x8<Self>) -> u16x8<Self> {
3969 [
3970 a[7usize], a[6usize], a[5usize], a[4usize], a[3usize], a[2usize], a[1usize], a[0usize],
3971 ]
3972 .simd_into(self)
3973 }
3974 #[inline(always)]
3975 fn slide_u16x8<const SHIFT: usize>(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
3976 let mut dest = [Default::default(); 8usize];
3977 dest[..8usize - SHIFT].copy_from_slice(&a.val.0[SHIFT..]);
3978 dest[8usize - SHIFT..].copy_from_slice(&b.val.0[..SHIFT]);
3979 dest.simd_into(self)
3980 }
3981 #[inline(always)]
3982 fn compress_u16x8(self, values: u16x8<Self>, mask: mask16x8<Self>) -> u16x8<Self> {
3983 let mask = mask8x16 {
3984 val: crate::transmute::checked_transmute_copy(&mask.val),
3985 simd: self,
3986 };
3987 Bytes::from_bytes(self.compress_u8x16(Bytes::to_bytes(values), mask))
3988 }
3989 #[inline(always)]
3990 fn compress_merge_u16x8(
3991 self,
3992 values: u16x8<Self>,
3993 mask: mask16x8<Self>,
3994 merge: u16x8<Self>,
3995 ) -> u16x8<Self> {
3996 let mask = mask8x16 {
3997 val: crate::transmute::checked_transmute_copy(&mask.val),
3998 simd: self,
3999 };
4000 Bytes::from_bytes(self.compress_merge_u8x16(
4001 Bytes::to_bytes(values),
4002 mask,
4003 Bytes::to_bytes(merge),
4004 ))
4005 }
4006 #[inline(always)]
4007 fn expand_u16x8(self, values: u16x8<Self>, mask: mask16x8<Self>) -> u16x8<Self> {
4008 let mask = mask8x16 {
4009 val: crate::transmute::checked_transmute_copy(&mask.val),
4010 simd: self,
4011 };
4012 Bytes::from_bytes(self.expand_u8x16(Bytes::to_bytes(values), mask))
4013 }
4014 #[inline(always)]
4015 fn expand_merge_u16x8(
4016 self,
4017 values: u16x8<Self>,
4018 mask: mask16x8<Self>,
4019 merge: u16x8<Self>,
4020 ) -> u16x8<Self> {
4021 let mask = mask8x16 {
4022 val: crate::transmute::checked_transmute_copy(&mask.val),
4023 simd: self,
4024 };
4025 Bytes::from_bytes(self.expand_merge_u8x16(
4026 Bytes::to_bytes(values),
4027 mask,
4028 Bytes::to_bytes(merge),
4029 ))
4030 }
4031 #[inline(always)]
4032 fn count_ones_u16x8(self, a: u16x8<Self>) -> u16x8<Self> {
4033 [
4034 u16::try_from(a[0usize].count_ones()).unwrap(),
4035 u16::try_from(a[1usize].count_ones()).unwrap(),
4036 u16::try_from(a[2usize].count_ones()).unwrap(),
4037 u16::try_from(a[3usize].count_ones()).unwrap(),
4038 u16::try_from(a[4usize].count_ones()).unwrap(),
4039 u16::try_from(a[5usize].count_ones()).unwrap(),
4040 u16::try_from(a[6usize].count_ones()).unwrap(),
4041 u16::try_from(a[7usize].count_ones()).unwrap(),
4042 ]
4043 .simd_into(self)
4044 }
4045 #[inline(always)]
4046 fn count_zeros_u16x8(self, a: u16x8<Self>) -> u16x8<Self> {
4047 [
4048 u16::try_from(a[0usize].count_zeros()).unwrap(),
4049 u16::try_from(a[1usize].count_zeros()).unwrap(),
4050 u16::try_from(a[2usize].count_zeros()).unwrap(),
4051 u16::try_from(a[3usize].count_zeros()).unwrap(),
4052 u16::try_from(a[4usize].count_zeros()).unwrap(),
4053 u16::try_from(a[5usize].count_zeros()).unwrap(),
4054 u16::try_from(a[6usize].count_zeros()).unwrap(),
4055 u16::try_from(a[7usize].count_zeros()).unwrap(),
4056 ]
4057 .simd_into(self)
4058 }
4059 #[inline(always)]
4060 fn add_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
4061 [
4062 u16::wrapping_add(a[0usize], b[0usize]),
4063 u16::wrapping_add(a[1usize], b[1usize]),
4064 u16::wrapping_add(a[2usize], b[2usize]),
4065 u16::wrapping_add(a[3usize], b[3usize]),
4066 u16::wrapping_add(a[4usize], b[4usize]),
4067 u16::wrapping_add(a[5usize], b[5usize]),
4068 u16::wrapping_add(a[6usize], b[6usize]),
4069 u16::wrapping_add(a[7usize], b[7usize]),
4070 ]
4071 .simd_into(self)
4072 }
4073 #[inline(always)]
4074 fn saturating_add_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
4075 [
4076 u16::saturating_add(a[0usize], b[0usize]),
4077 u16::saturating_add(a[1usize], b[1usize]),
4078 u16::saturating_add(a[2usize], b[2usize]),
4079 u16::saturating_add(a[3usize], b[3usize]),
4080 u16::saturating_add(a[4usize], b[4usize]),
4081 u16::saturating_add(a[5usize], b[5usize]),
4082 u16::saturating_add(a[6usize], b[6usize]),
4083 u16::saturating_add(a[7usize], b[7usize]),
4084 ]
4085 .simd_into(self)
4086 }
4087 #[inline(always)]
4088 fn sub_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
4089 [
4090 u16::wrapping_sub(a[0usize], b[0usize]),
4091 u16::wrapping_sub(a[1usize], b[1usize]),
4092 u16::wrapping_sub(a[2usize], b[2usize]),
4093 u16::wrapping_sub(a[3usize], b[3usize]),
4094 u16::wrapping_sub(a[4usize], b[4usize]),
4095 u16::wrapping_sub(a[5usize], b[5usize]),
4096 u16::wrapping_sub(a[6usize], b[6usize]),
4097 u16::wrapping_sub(a[7usize], b[7usize]),
4098 ]
4099 .simd_into(self)
4100 }
4101 #[inline(always)]
4102 fn saturating_sub_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
4103 [
4104 u16::saturating_sub(a[0usize], b[0usize]),
4105 u16::saturating_sub(a[1usize], b[1usize]),
4106 u16::saturating_sub(a[2usize], b[2usize]),
4107 u16::saturating_sub(a[3usize], b[3usize]),
4108 u16::saturating_sub(a[4usize], b[4usize]),
4109 u16::saturating_sub(a[5usize], b[5usize]),
4110 u16::saturating_sub(a[6usize], b[6usize]),
4111 u16::saturating_sub(a[7usize], b[7usize]),
4112 ]
4113 .simd_into(self)
4114 }
4115 #[inline(always)]
4116 fn mul_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
4117 [
4118 u16::wrapping_mul(a[0usize], b[0usize]),
4119 u16::wrapping_mul(a[1usize], b[1usize]),
4120 u16::wrapping_mul(a[2usize], b[2usize]),
4121 u16::wrapping_mul(a[3usize], b[3usize]),
4122 u16::wrapping_mul(a[4usize], b[4usize]),
4123 u16::wrapping_mul(a[5usize], b[5usize]),
4124 u16::wrapping_mul(a[6usize], b[6usize]),
4125 u16::wrapping_mul(a[7usize], b[7usize]),
4126 ]
4127 .simd_into(self)
4128 }
4129 #[inline(always)]
4130 fn and_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
4131 [
4132 u16::bitand(a[0usize], &b[0usize]),
4133 u16::bitand(a[1usize], &b[1usize]),
4134 u16::bitand(a[2usize], &b[2usize]),
4135 u16::bitand(a[3usize], &b[3usize]),
4136 u16::bitand(a[4usize], &b[4usize]),
4137 u16::bitand(a[5usize], &b[5usize]),
4138 u16::bitand(a[6usize], &b[6usize]),
4139 u16::bitand(a[7usize], &b[7usize]),
4140 ]
4141 .simd_into(self)
4142 }
4143 #[inline(always)]
4144 fn or_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
4145 [
4146 u16::bitor(a[0usize], &b[0usize]),
4147 u16::bitor(a[1usize], &b[1usize]),
4148 u16::bitor(a[2usize], &b[2usize]),
4149 u16::bitor(a[3usize], &b[3usize]),
4150 u16::bitor(a[4usize], &b[4usize]),
4151 u16::bitor(a[5usize], &b[5usize]),
4152 u16::bitor(a[6usize], &b[6usize]),
4153 u16::bitor(a[7usize], &b[7usize]),
4154 ]
4155 .simd_into(self)
4156 }
4157 #[inline(always)]
4158 fn xor_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
4159 [
4160 u16::bitxor(a[0usize], &b[0usize]),
4161 u16::bitxor(a[1usize], &b[1usize]),
4162 u16::bitxor(a[2usize], &b[2usize]),
4163 u16::bitxor(a[3usize], &b[3usize]),
4164 u16::bitxor(a[4usize], &b[4usize]),
4165 u16::bitxor(a[5usize], &b[5usize]),
4166 u16::bitxor(a[6usize], &b[6usize]),
4167 u16::bitxor(a[7usize], &b[7usize]),
4168 ]
4169 .simd_into(self)
4170 }
4171 #[inline(always)]
4172 fn not_u16x8(self, a: u16x8<Self>) -> u16x8<Self> {
4173 [
4174 u16::not(a[0usize]),
4175 u16::not(a[1usize]),
4176 u16::not(a[2usize]),
4177 u16::not(a[3usize]),
4178 u16::not(a[4usize]),
4179 u16::not(a[5usize]),
4180 u16::not(a[6usize]),
4181 u16::not(a[7usize]),
4182 ]
4183 .simd_into(self)
4184 }
4185 #[inline(always)]
4186 fn shl_u16x8(self, a: u16x8<Self>, shift: u32) -> u16x8<Self> {
4187 [
4188 u16::wrapping_shl(a[0usize], shift),
4189 u16::wrapping_shl(a[1usize], shift),
4190 u16::wrapping_shl(a[2usize], shift),
4191 u16::wrapping_shl(a[3usize], shift),
4192 u16::wrapping_shl(a[4usize], shift),
4193 u16::wrapping_shl(a[5usize], shift),
4194 u16::wrapping_shl(a[6usize], shift),
4195 u16::wrapping_shl(a[7usize], shift),
4196 ]
4197 .simd_into(self)
4198 }
4199 #[inline(always)]
4200 fn shlv_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
4201 [
4202 u16::wrapping_shl(a[0usize], b[0usize] as u32),
4203 u16::wrapping_shl(a[1usize], b[1usize] as u32),
4204 u16::wrapping_shl(a[2usize], b[2usize] as u32),
4205 u16::wrapping_shl(a[3usize], b[3usize] as u32),
4206 u16::wrapping_shl(a[4usize], b[4usize] as u32),
4207 u16::wrapping_shl(a[5usize], b[5usize] as u32),
4208 u16::wrapping_shl(a[6usize], b[6usize] as u32),
4209 u16::wrapping_shl(a[7usize], b[7usize] as u32),
4210 ]
4211 .simd_into(self)
4212 }
4213 #[inline(always)]
4214 fn shr_u16x8(self, a: u16x8<Self>, shift: u32) -> u16x8<Self> {
4215 [
4216 u16::wrapping_shr(a[0usize], shift),
4217 u16::wrapping_shr(a[1usize], shift),
4218 u16::wrapping_shr(a[2usize], shift),
4219 u16::wrapping_shr(a[3usize], shift),
4220 u16::wrapping_shr(a[4usize], shift),
4221 u16::wrapping_shr(a[5usize], shift),
4222 u16::wrapping_shr(a[6usize], shift),
4223 u16::wrapping_shr(a[7usize], shift),
4224 ]
4225 .simd_into(self)
4226 }
4227 #[inline(always)]
4228 fn shrv_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
4229 [
4230 u16::wrapping_shr(a[0usize], b[0usize] as u32),
4231 u16::wrapping_shr(a[1usize], b[1usize] as u32),
4232 u16::wrapping_shr(a[2usize], b[2usize] as u32),
4233 u16::wrapping_shr(a[3usize], b[3usize] as u32),
4234 u16::wrapping_shr(a[4usize], b[4usize] as u32),
4235 u16::wrapping_shr(a[5usize], b[5usize] as u32),
4236 u16::wrapping_shr(a[6usize], b[6usize] as u32),
4237 u16::wrapping_shr(a[7usize], b[7usize] as u32),
4238 ]
4239 .simd_into(self)
4240 }
4241 #[inline(always)]
4242 fn reduce_max_u16x8(self, a: u16x8<Self>) -> u16 {
4243 let reduced: [u16; 4usize] = [
4244 u16::max(a[0usize], a[1usize]),
4245 u16::max(a[2usize], a[3usize]),
4246 u16::max(a[4usize], a[5usize]),
4247 u16::max(a[6usize], a[7usize]),
4248 ];
4249 let reduced: [u16; 2usize] = [
4250 u16::max(reduced[0usize], reduced[1usize]),
4251 u16::max(reduced[2usize], reduced[3usize]),
4252 ];
4253 let reduced: [u16; 1usize] = [u16::max(reduced[0usize], reduced[1usize])];
4254 reduced[0]
4255 }
4256 #[inline(always)]
4257 fn reduce_min_u16x8(self, a: u16x8<Self>) -> u16 {
4258 let reduced: [u16; 4usize] = [
4259 u16::min(a[0usize], a[1usize]),
4260 u16::min(a[2usize], a[3usize]),
4261 u16::min(a[4usize], a[5usize]),
4262 u16::min(a[6usize], a[7usize]),
4263 ];
4264 let reduced: [u16; 2usize] = [
4265 u16::min(reduced[0usize], reduced[1usize]),
4266 u16::min(reduced[2usize], reduced[3usize]),
4267 ];
4268 let reduced: [u16; 1usize] = [u16::min(reduced[0usize], reduced[1usize])];
4269 reduced[0]
4270 }
4271 #[inline(always)]
4272 fn reduce_sum_u16x8(self, a: u16x8<Self>) -> u16 {
4273 let add_level_0: [u16; 4usize] = [
4274 a[0usize].wrapping_add(a[1usize]),
4275 a[2usize].wrapping_add(a[3usize]),
4276 a[4usize].wrapping_add(a[5usize]),
4277 a[6usize].wrapping_add(a[7usize]),
4278 ];
4279 let add_level_1: [u16; 2usize] = [
4280 add_level_0[0usize].wrapping_add(add_level_0[1usize]),
4281 add_level_0[2usize].wrapping_add(add_level_0[3usize]),
4282 ];
4283 let add_level_2: [u16; 1usize] = [add_level_1[0usize].wrapping_add(add_level_1[1usize])];
4284 add_level_2[0]
4285 }
4286 #[inline(always)]
4287 fn reduce_product_u16x8(self, a: u16x8<Self>) -> u16 {
4288 let mul_level_0: [u16; 4usize] = [
4289 a[0usize].wrapping_mul(a[1usize]),
4290 a[2usize].wrapping_mul(a[3usize]),
4291 a[4usize].wrapping_mul(a[5usize]),
4292 a[6usize].wrapping_mul(a[7usize]),
4293 ];
4294 let mul_level_1: [u16; 2usize] = [
4295 mul_level_0[0usize].wrapping_mul(mul_level_0[1usize]),
4296 mul_level_0[2usize].wrapping_mul(mul_level_0[3usize]),
4297 ];
4298 let mul_level_2: [u16; 1usize] = [mul_level_1[0usize].wrapping_mul(mul_level_1[1usize])];
4299 mul_level_2[0]
4300 }
4301 #[inline(always)]
4302 fn max_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
4303 [
4304 u16::max(a[0usize], b[0usize]),
4305 u16::max(a[1usize], b[1usize]),
4306 u16::max(a[2usize], b[2usize]),
4307 u16::max(a[3usize], b[3usize]),
4308 u16::max(a[4usize], b[4usize]),
4309 u16::max(a[5usize], b[5usize]),
4310 u16::max(a[6usize], b[6usize]),
4311 u16::max(a[7usize], b[7usize]),
4312 ]
4313 .simd_into(self)
4314 }
4315 #[inline(always)]
4316 fn min_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
4317 [
4318 u16::min(a[0usize], b[0usize]),
4319 u16::min(a[1usize], b[1usize]),
4320 u16::min(a[2usize], b[2usize]),
4321 u16::min(a[3usize], b[3usize]),
4322 u16::min(a[4usize], b[4usize]),
4323 u16::min(a[5usize], b[5usize]),
4324 u16::min(a[6usize], b[6usize]),
4325 u16::min(a[7usize], b[7usize]),
4326 ]
4327 .simd_into(self)
4328 }
4329 #[inline(always)]
4330 fn clamp_u16x8(self, a: u16x8<Self>, b: u16x8<Self>, c: u16x8<Self>) -> u16x8<Self> {
4331 [
4332 u16::clamp_debug(a[0usize], b[0usize], c[0usize]),
4333 u16::clamp_debug(a[1usize], b[1usize], c[1usize]),
4334 u16::clamp_debug(a[2usize], b[2usize], c[2usize]),
4335 u16::clamp_debug(a[3usize], b[3usize], c[3usize]),
4336 u16::clamp_debug(a[4usize], b[4usize], c[4usize]),
4337 u16::clamp_debug(a[5usize], b[5usize], c[5usize]),
4338 u16::clamp_debug(a[6usize], b[6usize], c[6usize]),
4339 u16::clamp_debug(a[7usize], b[7usize], c[7usize]),
4340 ]
4341 .simd_into(self)
4342 }
4343 #[inline(always)]
4344 fn simd_eq_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> mask16x8<Self> {
4345 [
4346 -(u16::eq(&a[0usize], &b[0usize]) as i16),
4347 -(u16::eq(&a[1usize], &b[1usize]) as i16),
4348 -(u16::eq(&a[2usize], &b[2usize]) as i16),
4349 -(u16::eq(&a[3usize], &b[3usize]) as i16),
4350 -(u16::eq(&a[4usize], &b[4usize]) as i16),
4351 -(u16::eq(&a[5usize], &b[5usize]) as i16),
4352 -(u16::eq(&a[6usize], &b[6usize]) as i16),
4353 -(u16::eq(&a[7usize], &b[7usize]) as i16),
4354 ]
4355 .simd_into(self)
4356 }
4357 #[inline(always)]
4358 fn simd_lt_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> mask16x8<Self> {
4359 [
4360 -(u16::lt(&a[0usize], &b[0usize]) as i16),
4361 -(u16::lt(&a[1usize], &b[1usize]) as i16),
4362 -(u16::lt(&a[2usize], &b[2usize]) as i16),
4363 -(u16::lt(&a[3usize], &b[3usize]) as i16),
4364 -(u16::lt(&a[4usize], &b[4usize]) as i16),
4365 -(u16::lt(&a[5usize], &b[5usize]) as i16),
4366 -(u16::lt(&a[6usize], &b[6usize]) as i16),
4367 -(u16::lt(&a[7usize], &b[7usize]) as i16),
4368 ]
4369 .simd_into(self)
4370 }
4371 #[inline(always)]
4372 fn simd_le_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> mask16x8<Self> {
4373 [
4374 -(u16::le(&a[0usize], &b[0usize]) as i16),
4375 -(u16::le(&a[1usize], &b[1usize]) as i16),
4376 -(u16::le(&a[2usize], &b[2usize]) as i16),
4377 -(u16::le(&a[3usize], &b[3usize]) as i16),
4378 -(u16::le(&a[4usize], &b[4usize]) as i16),
4379 -(u16::le(&a[5usize], &b[5usize]) as i16),
4380 -(u16::le(&a[6usize], &b[6usize]) as i16),
4381 -(u16::le(&a[7usize], &b[7usize]) as i16),
4382 ]
4383 .simd_into(self)
4384 }
4385 #[inline(always)]
4386 fn zip_low_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
4387 [
4388 a[0usize], b[0usize], a[1usize], b[1usize], a[2usize], b[2usize], a[3usize], b[3usize],
4389 ]
4390 .simd_into(self)
4391 }
4392 #[inline(always)]
4393 fn zip_high_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
4394 [
4395 a[4usize], b[4usize], a[5usize], b[5usize], a[6usize], b[6usize], a[7usize], b[7usize],
4396 ]
4397 .simd_into(self)
4398 }
4399 #[inline(always)]
4400 fn unzip_low_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
4401 [
4402 a[0usize], a[2usize], a[4usize], a[6usize], b[0usize], b[2usize], b[4usize], b[6usize],
4403 ]
4404 .simd_into(self)
4405 }
4406 #[inline(always)]
4407 fn unzip_high_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
4408 [
4409 a[1usize], a[3usize], a[5usize], a[7usize], b[1usize], b[3usize], b[5usize], b[7usize],
4410 ]
4411 .simd_into(self)
4412 }
4413 #[inline(always)]
4414 fn interleave_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> (u16x8<Self>, u16x8<Self>) {
4415 (self.zip_low_u16x8(a, b), self.zip_high_u16x8(a, b))
4416 }
4417 #[inline(always)]
4418 fn deinterleave_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> (u16x8<Self>, u16x8<Self>) {
4419 (self.unzip_low_u16x8(a, b), self.unzip_high_u16x8(a, b))
4420 }
4421 #[inline(always)]
4422 fn select_u16x8(self, a: mask16x8<Self>, b: u16x8<Self>, c: u16x8<Self>) -> u16x8<Self> {
4423 [
4424 if a.val.0[0usize] != 0 {
4425 b[0usize]
4426 } else {
4427 c[0usize]
4428 },
4429 if a.val.0[1usize] != 0 {
4430 b[1usize]
4431 } else {
4432 c[1usize]
4433 },
4434 if a.val.0[2usize] != 0 {
4435 b[2usize]
4436 } else {
4437 c[2usize]
4438 },
4439 if a.val.0[3usize] != 0 {
4440 b[3usize]
4441 } else {
4442 c[3usize]
4443 },
4444 if a.val.0[4usize] != 0 {
4445 b[4usize]
4446 } else {
4447 c[4usize]
4448 },
4449 if a.val.0[5usize] != 0 {
4450 b[5usize]
4451 } else {
4452 c[5usize]
4453 },
4454 if a.val.0[6usize] != 0 {
4455 b[6usize]
4456 } else {
4457 c[6usize]
4458 },
4459 if a.val.0[7usize] != 0 {
4460 b[7usize]
4461 } else {
4462 c[7usize]
4463 },
4464 ]
4465 .simd_into(self)
4466 }
4467 #[inline(always)]
4468 fn combine_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x16<Self> {
4469 let mut result = [0; 16usize];
4470 result[0..8usize].copy_from_slice(&a.val.0);
4471 result[8usize..16usize].copy_from_slice(&b.val.0);
4472 result.simd_into(self)
4473 }
4474 #[inline(always)]
4475 fn load_four_interleaved_u16x8(self, src: &[u16; 32usize]) -> [u16x8<Self>; 4usize] {
4476 [
4477 [
4478 src[0usize],
4479 src[4usize],
4480 src[8usize],
4481 src[12usize],
4482 src[16usize],
4483 src[20usize],
4484 src[24usize],
4485 src[28usize],
4486 ]
4487 .simd_into(self),
4488 [
4489 src[1usize],
4490 src[5usize],
4491 src[9usize],
4492 src[13usize],
4493 src[17usize],
4494 src[21usize],
4495 src[25usize],
4496 src[29usize],
4497 ]
4498 .simd_into(self),
4499 [
4500 src[2usize],
4501 src[6usize],
4502 src[10usize],
4503 src[14usize],
4504 src[18usize],
4505 src[22usize],
4506 src[26usize],
4507 src[30usize],
4508 ]
4509 .simd_into(self),
4510 [
4511 src[3usize],
4512 src[7usize],
4513 src[11usize],
4514 src[15usize],
4515 src[19usize],
4516 src[23usize],
4517 src[27usize],
4518 src[31usize],
4519 ]
4520 .simd_into(self),
4521 ]
4522 }
4523 #[inline(always)]
4524 fn store_four_interleaved_u16x8(
4525 self,
4526 vectors: [u16x8<Self>; 4usize],
4527 dest: &mut [u16; 32usize],
4528 ) -> () {
4529 *dest = [
4530 vectors[0usize][0usize],
4531 vectors[1usize][0usize],
4532 vectors[2usize][0usize],
4533 vectors[3usize][0usize],
4534 vectors[0usize][1usize],
4535 vectors[1usize][1usize],
4536 vectors[2usize][1usize],
4537 vectors[3usize][1usize],
4538 vectors[0usize][2usize],
4539 vectors[1usize][2usize],
4540 vectors[2usize][2usize],
4541 vectors[3usize][2usize],
4542 vectors[0usize][3usize],
4543 vectors[1usize][3usize],
4544 vectors[2usize][3usize],
4545 vectors[3usize][3usize],
4546 vectors[0usize][4usize],
4547 vectors[1usize][4usize],
4548 vectors[2usize][4usize],
4549 vectors[3usize][4usize],
4550 vectors[0usize][5usize],
4551 vectors[1usize][5usize],
4552 vectors[2usize][5usize],
4553 vectors[3usize][5usize],
4554 vectors[0usize][6usize],
4555 vectors[1usize][6usize],
4556 vectors[2usize][6usize],
4557 vectors[3usize][6usize],
4558 vectors[0usize][7usize],
4559 vectors[1usize][7usize],
4560 vectors[2usize][7usize],
4561 vectors[3usize][7usize],
4562 ];
4563 }
4564 #[inline(always)]
4565 fn widen_u16x8(self, a: u16x8<Self>) -> (u32x4<Self>, u32x4<Self>) {
4566 (
4567 [
4568 a[0usize] as u32,
4569 a[1usize] as u32,
4570 a[2usize] as u32,
4571 a[3usize] as u32,
4572 ]
4573 .simd_into(self),
4574 [
4575 a[4usize] as u32,
4576 a[5usize] as u32,
4577 a[6usize] as u32,
4578 a[7usize] as u32,
4579 ]
4580 .simd_into(self),
4581 )
4582 }
4583 #[inline(always)]
4584 fn widening_mul_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> (u32x4<Self>, u32x4<Self>) {
4585 let (a0, a1) = self.widen_u16x8(a);
4586 let (b0, b1) = self.widen_u16x8(b);
4587 (self.mul_u32x4(a0, b0), self.mul_u32x4(a1, b1))
4588 }
4589 #[inline(always)]
4590 fn widening_add_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> (u32x4<Self>, u32x4<Self>) {
4591 let (a0, a1) = self.widen_u16x8(a);
4592 let (b0, b1) = self.widen_u16x8(b);
4593 (self.add_u32x4(a0, b0), self.add_u32x4(a1, b1))
4594 }
4595 #[inline(always)]
4596 fn pairwise_widening_add_u16x8(self, a: u16x8<Self>) -> u32x4<Self> {
4597 [
4598 (a[0usize] as u32) + (a[1usize] as u32),
4599 (a[2usize] as u32) + (a[3usize] as u32),
4600 (a[4usize] as u32) + (a[5usize] as u32),
4601 (a[6usize] as u32) + (a[7usize] as u32),
4602 ]
4603 .simd_into(self)
4604 }
4605 #[inline(always)]
4606 fn pairwise_dot_product_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u32x4<Self> {
4607 let (low, high) = self.widening_mul_u16x8(a, b);
4608 let (even, odd) = self.deinterleave_u32x4(low, high);
4609 self.add_u32x4(even, odd)
4610 }
4611 #[inline(always)]
4612 fn narrow_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u8x16<Self> {
4613 [
4614 a[0usize] as u8,
4615 a[1usize] as u8,
4616 a[2usize] as u8,
4617 a[3usize] as u8,
4618 a[4usize] as u8,
4619 a[5usize] as u8,
4620 a[6usize] as u8,
4621 a[7usize] as u8,
4622 b[0usize] as u8,
4623 b[1usize] as u8,
4624 b[2usize] as u8,
4625 b[3usize] as u8,
4626 b[4usize] as u8,
4627 b[5usize] as u8,
4628 b[6usize] as u8,
4629 b[7usize] as u8,
4630 ]
4631 .simd_into(self)
4632 }
4633 #[inline(always)]
4634 fn saturating_narrow_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u8x16<Self> {
4635 [
4636 a[0usize].clamp(u8::MIN as u16, u8::MAX as u16) as u8,
4637 a[1usize].clamp(u8::MIN as u16, u8::MAX as u16) as u8,
4638 a[2usize].clamp(u8::MIN as u16, u8::MAX as u16) as u8,
4639 a[3usize].clamp(u8::MIN as u16, u8::MAX as u16) as u8,
4640 a[4usize].clamp(u8::MIN as u16, u8::MAX as u16) as u8,
4641 a[5usize].clamp(u8::MIN as u16, u8::MAX as u16) as u8,
4642 a[6usize].clamp(u8::MIN as u16, u8::MAX as u16) as u8,
4643 a[7usize].clamp(u8::MIN as u16, u8::MAX as u16) as u8,
4644 b[0usize].clamp(u8::MIN as u16, u8::MAX as u16) as u8,
4645 b[1usize].clamp(u8::MIN as u16, u8::MAX as u16) as u8,
4646 b[2usize].clamp(u8::MIN as u16, u8::MAX as u16) as u8,
4647 b[3usize].clamp(u8::MIN as u16, u8::MAX as u16) as u8,
4648 b[4usize].clamp(u8::MIN as u16, u8::MAX as u16) as u8,
4649 b[5usize].clamp(u8::MIN as u16, u8::MAX as u16) as u8,
4650 b[6usize].clamp(u8::MIN as u16, u8::MAX as u16) as u8,
4651 b[7usize].clamp(u8::MIN as u16, u8::MAX as u16) as u8,
4652 ]
4653 .simd_into(self)
4654 }
4655 #[inline(always)]
4656 fn relaxed_narrow_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u8x16<Self> {
4657 debug_assert!(
4658 a.as_slice()
4659 .iter()
4660 .chain(b.as_slice())
4661 .all(|&value| { value >= u8::MIN as u16 && value <= u8::MAX as u16 }),
4662 "relaxed_narrow inputs must fit in the destination type",
4663 );
4664 self.narrow_u16x8(a, b)
4665 }
4666 #[inline(always)]
4667 fn splat_mask16x8(self, val: bool) -> mask16x8<Self> {
4668 let val: i16 = if val { !0 } else { 0 };
4669 [val; 8usize].simd_into(self)
4670 }
4671 #[inline(always)]
4672 fn from_bitmask_mask16x8(self, bits: u64) -> mask16x8<Self> {
4673 let lanes: [i16; 8usize] = [
4674 if bits & 1 != 0 { !0 } else { 0 },
4675 if (bits >> 1usize) & 1 != 0 { !0 } else { 0 },
4676 if (bits >> 2usize) & 1 != 0 { !0 } else { 0 },
4677 if (bits >> 3usize) & 1 != 0 { !0 } else { 0 },
4678 if (bits >> 4usize) & 1 != 0 { !0 } else { 0 },
4679 if (bits >> 5usize) & 1 != 0 { !0 } else { 0 },
4680 if (bits >> 6usize) & 1 != 0 { !0 } else { 0 },
4681 if (bits >> 7usize) & 1 != 0 { !0 } else { 0 },
4682 ];
4683 lanes.simd_into(self)
4684 }
4685 #[inline(always)]
4686 fn to_bitmask_mask16x8(self, a: mask16x8<Self>) -> u64 {
4687 let lanes: [i16; 8usize] = a.into();
4688 let mut bits = 0u64;
4689 let mut i = 0;
4690 while i < 8usize {
4691 if lanes[i] != 0 {
4692 bits |= 1u64 << i;
4693 }
4694 i += 1;
4695 }
4696 bits
4697 }
4698 #[inline(always)]
4699 fn from_vector_mask16x8(self, vector: i16x8<Self>) -> mask16x8<Self> {
4700 mask16x8 {
4701 val: vector.val,
4702 simd: self,
4703 }
4704 }
4705 #[inline(always)]
4706 fn to_vector_mask16x8(self, a: mask16x8<Self>) -> i16x8<Self> {
4707 i16x8 {
4708 val: a.val,
4709 simd: self,
4710 }
4711 }
4712 #[inline(always)]
4713 fn set_mask16x8(self, a: &mut mask16x8<Self>, index: usize, value: bool) -> () {
4714 assert!(
4715 index < 8usize,
4716 "mask lane index {index} is out of bounds for {} lanes",
4717 8usize
4718 );
4719 let mut lanes: [i16; 8usize] = (*a).into();
4720 lanes[index] = if value { !0 } else { 0 };
4721 *a = lanes.simd_into(self);
4722 }
4723 #[inline(always)]
4724 fn rotate_elements_left_mask16x8<const OFFSET: usize>(
4725 self,
4726 a: mask16x8<Self>,
4727 ) -> mask16x8<Self> {
4728 let int = i16x8 {
4729 val: crate::transmute::checked_transmute_copy(&a.val),
4730 simd: self,
4731 };
4732 let rotated = int.rotate_elements_left::<OFFSET>();
4733 mask16x8 {
4734 val: crate::transmute::checked_transmute_copy(&rotated.val),
4735 simd: self,
4736 }
4737 }
4738 #[inline(always)]
4739 fn rotate_elements_right_mask16x8<const OFFSET: usize>(
4740 self,
4741 a: mask16x8<Self>,
4742 ) -> mask16x8<Self> {
4743 let int = i16x8 {
4744 val: crate::transmute::checked_transmute_copy(&a.val),
4745 simd: self,
4746 };
4747 let rotated = int.rotate_elements_right::<OFFSET>();
4748 mask16x8 {
4749 val: crate::transmute::checked_transmute_copy(&rotated.val),
4750 simd: self,
4751 }
4752 }
4753 #[inline(always)]
4754 fn and_mask16x8(self, a: mask16x8<Self>, b: mask16x8<Self>) -> mask16x8<Self> {
4755 [
4756 i16::bitand(a.val.0[0usize], &b.val.0[0usize]),
4757 i16::bitand(a.val.0[1usize], &b.val.0[1usize]),
4758 i16::bitand(a.val.0[2usize], &b.val.0[2usize]),
4759 i16::bitand(a.val.0[3usize], &b.val.0[3usize]),
4760 i16::bitand(a.val.0[4usize], &b.val.0[4usize]),
4761 i16::bitand(a.val.0[5usize], &b.val.0[5usize]),
4762 i16::bitand(a.val.0[6usize], &b.val.0[6usize]),
4763 i16::bitand(a.val.0[7usize], &b.val.0[7usize]),
4764 ]
4765 .simd_into(self)
4766 }
4767 #[inline(always)]
4768 fn or_mask16x8(self, a: mask16x8<Self>, b: mask16x8<Self>) -> mask16x8<Self> {
4769 [
4770 i16::bitor(a.val.0[0usize], &b.val.0[0usize]),
4771 i16::bitor(a.val.0[1usize], &b.val.0[1usize]),
4772 i16::bitor(a.val.0[2usize], &b.val.0[2usize]),
4773 i16::bitor(a.val.0[3usize], &b.val.0[3usize]),
4774 i16::bitor(a.val.0[4usize], &b.val.0[4usize]),
4775 i16::bitor(a.val.0[5usize], &b.val.0[5usize]),
4776 i16::bitor(a.val.0[6usize], &b.val.0[6usize]),
4777 i16::bitor(a.val.0[7usize], &b.val.0[7usize]),
4778 ]
4779 .simd_into(self)
4780 }
4781 #[inline(always)]
4782 fn xor_mask16x8(self, a: mask16x8<Self>, b: mask16x8<Self>) -> mask16x8<Self> {
4783 [
4784 i16::bitxor(a.val.0[0usize], &b.val.0[0usize]),
4785 i16::bitxor(a.val.0[1usize], &b.val.0[1usize]),
4786 i16::bitxor(a.val.0[2usize], &b.val.0[2usize]),
4787 i16::bitxor(a.val.0[3usize], &b.val.0[3usize]),
4788 i16::bitxor(a.val.0[4usize], &b.val.0[4usize]),
4789 i16::bitxor(a.val.0[5usize], &b.val.0[5usize]),
4790 i16::bitxor(a.val.0[6usize], &b.val.0[6usize]),
4791 i16::bitxor(a.val.0[7usize], &b.val.0[7usize]),
4792 ]
4793 .simd_into(self)
4794 }
4795 #[inline(always)]
4796 fn not_mask16x8(self, a: mask16x8<Self>) -> mask16x8<Self> {
4797 [
4798 i16::not(a.val.0[0usize]),
4799 i16::not(a.val.0[1usize]),
4800 i16::not(a.val.0[2usize]),
4801 i16::not(a.val.0[3usize]),
4802 i16::not(a.val.0[4usize]),
4803 i16::not(a.val.0[5usize]),
4804 i16::not(a.val.0[6usize]),
4805 i16::not(a.val.0[7usize]),
4806 ]
4807 .simd_into(self)
4808 }
4809 #[inline(always)]
4810 fn reverse_mask16x8(self, a: mask16x8<Self>) -> mask16x8<Self> {
4811 [
4812 a.val.0[7usize],
4813 a.val.0[6usize],
4814 a.val.0[5usize],
4815 a.val.0[4usize],
4816 a.val.0[3usize],
4817 a.val.0[2usize],
4818 a.val.0[1usize],
4819 a.val.0[0usize],
4820 ]
4821 .simd_into(self)
4822 }
4823 #[inline(always)]
4824 fn select_mask16x8(
4825 self,
4826 a: mask16x8<Self>,
4827 b: mask16x8<Self>,
4828 c: mask16x8<Self>,
4829 ) -> mask16x8<Self> {
4830 [
4831 if a.val.0[0usize] != 0 {
4832 b.val.0[0usize]
4833 } else {
4834 c.val.0[0usize]
4835 },
4836 if a.val.0[1usize] != 0 {
4837 b.val.0[1usize]
4838 } else {
4839 c.val.0[1usize]
4840 },
4841 if a.val.0[2usize] != 0 {
4842 b.val.0[2usize]
4843 } else {
4844 c.val.0[2usize]
4845 },
4846 if a.val.0[3usize] != 0 {
4847 b.val.0[3usize]
4848 } else {
4849 c.val.0[3usize]
4850 },
4851 if a.val.0[4usize] != 0 {
4852 b.val.0[4usize]
4853 } else {
4854 c.val.0[4usize]
4855 },
4856 if a.val.0[5usize] != 0 {
4857 b.val.0[5usize]
4858 } else {
4859 c.val.0[5usize]
4860 },
4861 if a.val.0[6usize] != 0 {
4862 b.val.0[6usize]
4863 } else {
4864 c.val.0[6usize]
4865 },
4866 if a.val.0[7usize] != 0 {
4867 b.val.0[7usize]
4868 } else {
4869 c.val.0[7usize]
4870 },
4871 ]
4872 .simd_into(self)
4873 }
4874 #[inline(always)]
4875 fn simd_eq_mask16x8(self, a: mask16x8<Self>, b: mask16x8<Self>) -> mask16x8<Self> {
4876 [
4877 -(i16::eq(&a.val.0[0usize], &b.val.0[0usize]) as i16),
4878 -(i16::eq(&a.val.0[1usize], &b.val.0[1usize]) as i16),
4879 -(i16::eq(&a.val.0[2usize], &b.val.0[2usize]) as i16),
4880 -(i16::eq(&a.val.0[3usize], &b.val.0[3usize]) as i16),
4881 -(i16::eq(&a.val.0[4usize], &b.val.0[4usize]) as i16),
4882 -(i16::eq(&a.val.0[5usize], &b.val.0[5usize]) as i16),
4883 -(i16::eq(&a.val.0[6usize], &b.val.0[6usize]) as i16),
4884 -(i16::eq(&a.val.0[7usize], &b.val.0[7usize]) as i16),
4885 ]
4886 .simd_into(self)
4887 }
4888 #[inline(always)]
4889 fn any_true_mask16x8(self, a: mask16x8<Self>) -> bool {
4890 a.val.0[0usize] != 0
4891 || a.val.0[1usize] != 0
4892 || a.val.0[2usize] != 0
4893 || a.val.0[3usize] != 0
4894 || a.val.0[4usize] != 0
4895 || a.val.0[5usize] != 0
4896 || a.val.0[6usize] != 0
4897 || a.val.0[7usize] != 0
4898 }
4899 #[inline(always)]
4900 fn all_true_mask16x8(self, a: mask16x8<Self>) -> bool {
4901 a.val.0[0usize] != 0
4902 && a.val.0[1usize] != 0
4903 && a.val.0[2usize] != 0
4904 && a.val.0[3usize] != 0
4905 && a.val.0[4usize] != 0
4906 && a.val.0[5usize] != 0
4907 && a.val.0[6usize] != 0
4908 && a.val.0[7usize] != 0
4909 }
4910 #[inline(always)]
4911 fn any_false_mask16x8(self, a: mask16x8<Self>) -> bool {
4912 a.val.0[0usize] == 0
4913 || a.val.0[1usize] == 0
4914 || a.val.0[2usize] == 0
4915 || a.val.0[3usize] == 0
4916 || a.val.0[4usize] == 0
4917 || a.val.0[5usize] == 0
4918 || a.val.0[6usize] == 0
4919 || a.val.0[7usize] == 0
4920 }
4921 #[inline(always)]
4922 fn all_false_mask16x8(self, a: mask16x8<Self>) -> bool {
4923 a.val.0[0usize] == 0
4924 && a.val.0[1usize] == 0
4925 && a.val.0[2usize] == 0
4926 && a.val.0[3usize] == 0
4927 && a.val.0[4usize] == 0
4928 && a.val.0[5usize] == 0
4929 && a.val.0[6usize] == 0
4930 && a.val.0[7usize] == 0
4931 }
4932 #[inline(always)]
4933 fn combine_mask16x8(self, a: mask16x8<Self>, b: mask16x8<Self>) -> mask16x16<Self> {
4934 let mut result = [0; 16usize];
4935 result[0..8usize].copy_from_slice(&a.val.0);
4936 result[8usize..16usize].copy_from_slice(&b.val.0);
4937 result.simd_into(self)
4938 }
4939 #[inline(always)]
4940 fn widen_mask16x8(self, a: mask16x8<Self>) -> (mask32x4<Self>, mask32x4<Self>) {
4941 (
4942 [
4943 a.val.0[0usize] as i32,
4944 a.val.0[1usize] as i32,
4945 a.val.0[2usize] as i32,
4946 a.val.0[3usize] as i32,
4947 ]
4948 .simd_into(self),
4949 [
4950 a.val.0[4usize] as i32,
4951 a.val.0[5usize] as i32,
4952 a.val.0[6usize] as i32,
4953 a.val.0[7usize] as i32,
4954 ]
4955 .simd_into(self),
4956 )
4957 }
4958 #[inline(always)]
4959 fn narrow_mask16x8(self, a: mask16x8<Self>, b: mask16x8<Self>) -> mask8x16<Self> {
4960 [
4961 a.val.0[0usize] as i8,
4962 a.val.0[1usize] as i8,
4963 a.val.0[2usize] as i8,
4964 a.val.0[3usize] as i8,
4965 a.val.0[4usize] as i8,
4966 a.val.0[5usize] as i8,
4967 a.val.0[6usize] as i8,
4968 a.val.0[7usize] as i8,
4969 b.val.0[0usize] as i8,
4970 b.val.0[1usize] as i8,
4971 b.val.0[2usize] as i8,
4972 b.val.0[3usize] as i8,
4973 b.val.0[4usize] as i8,
4974 b.val.0[5usize] as i8,
4975 b.val.0[6usize] as i8,
4976 b.val.0[7usize] as i8,
4977 ]
4978 .simd_into(self)
4979 }
4980 #[inline(always)]
4981 fn abs_i32x4(self, a: i32x4<Self>) -> i32x4<Self> {
4982 [
4983 i32::wrapping_abs(a[0usize]),
4984 i32::wrapping_abs(a[1usize]),
4985 i32::wrapping_abs(a[2usize]),
4986 i32::wrapping_abs(a[3usize]),
4987 ]
4988 .simd_into(self)
4989 }
4990 #[inline(always)]
4991 fn splat_i32x4(self, val: i32) -> i32x4<Self> {
4992 [val; 4usize].simd_into(self)
4993 }
4994 #[inline(always)]
4995 fn reverse_i32x4(self, a: i32x4<Self>) -> i32x4<Self> {
4996 [a[3usize], a[2usize], a[1usize], a[0usize]].simd_into(self)
4997 }
4998 #[inline(always)]
4999 fn slide_i32x4<const SHIFT: usize>(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
5000 let mut dest = [Default::default(); 4usize];
5001 dest[..4usize - SHIFT].copy_from_slice(&a.val.0[SHIFT..]);
5002 dest[4usize - SHIFT..].copy_from_slice(&b.val.0[..SHIFT]);
5003 dest.simd_into(self)
5004 }
5005 #[inline(always)]
5006 fn compress_i32x4(self, values: i32x4<Self>, mask: mask32x4<Self>) -> i32x4<Self> {
5007 let mask = mask8x16 {
5008 val: crate::transmute::checked_transmute_copy(&mask.val),
5009 simd: self,
5010 };
5011 Bytes::from_bytes(self.compress_u8x16(Bytes::to_bytes(values), mask))
5012 }
5013 #[inline(always)]
5014 fn compress_merge_i32x4(
5015 self,
5016 values: i32x4<Self>,
5017 mask: mask32x4<Self>,
5018 merge: i32x4<Self>,
5019 ) -> i32x4<Self> {
5020 let mask = mask8x16 {
5021 val: crate::transmute::checked_transmute_copy(&mask.val),
5022 simd: self,
5023 };
5024 Bytes::from_bytes(self.compress_merge_u8x16(
5025 Bytes::to_bytes(values),
5026 mask,
5027 Bytes::to_bytes(merge),
5028 ))
5029 }
5030 #[inline(always)]
5031 fn expand_i32x4(self, values: i32x4<Self>, mask: mask32x4<Self>) -> i32x4<Self> {
5032 let mask = mask8x16 {
5033 val: crate::transmute::checked_transmute_copy(&mask.val),
5034 simd: self,
5035 };
5036 Bytes::from_bytes(self.expand_u8x16(Bytes::to_bytes(values), mask))
5037 }
5038 #[inline(always)]
5039 fn expand_merge_i32x4(
5040 self,
5041 values: i32x4<Self>,
5042 mask: mask32x4<Self>,
5043 merge: i32x4<Self>,
5044 ) -> i32x4<Self> {
5045 let mask = mask8x16 {
5046 val: crate::transmute::checked_transmute_copy(&mask.val),
5047 simd: self,
5048 };
5049 Bytes::from_bytes(self.expand_merge_u8x16(
5050 Bytes::to_bytes(values),
5051 mask,
5052 Bytes::to_bytes(merge),
5053 ))
5054 }
5055 #[inline(always)]
5056 fn count_ones_i32x4(self, a: i32x4<Self>) -> i32x4<Self> {
5057 [
5058 a[0usize].count_ones().cast_signed(),
5059 a[1usize].count_ones().cast_signed(),
5060 a[2usize].count_ones().cast_signed(),
5061 a[3usize].count_ones().cast_signed(),
5062 ]
5063 .simd_into(self)
5064 }
5065 #[inline(always)]
5066 fn count_zeros_i32x4(self, a: i32x4<Self>) -> i32x4<Self> {
5067 [
5068 a[0usize].count_zeros().cast_signed(),
5069 a[1usize].count_zeros().cast_signed(),
5070 a[2usize].count_zeros().cast_signed(),
5071 a[3usize].count_zeros().cast_signed(),
5072 ]
5073 .simd_into(self)
5074 }
5075 #[inline(always)]
5076 fn add_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
5077 [
5078 i32::wrapping_add(a[0usize], b[0usize]),
5079 i32::wrapping_add(a[1usize], b[1usize]),
5080 i32::wrapping_add(a[2usize], b[2usize]),
5081 i32::wrapping_add(a[3usize], b[3usize]),
5082 ]
5083 .simd_into(self)
5084 }
5085 #[inline(always)]
5086 fn saturating_add_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
5087 [
5088 i32::saturating_add(a[0usize], b[0usize]),
5089 i32::saturating_add(a[1usize], b[1usize]),
5090 i32::saturating_add(a[2usize], b[2usize]),
5091 i32::saturating_add(a[3usize], b[3usize]),
5092 ]
5093 .simd_into(self)
5094 }
5095 #[inline(always)]
5096 fn sub_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
5097 [
5098 i32::wrapping_sub(a[0usize], b[0usize]),
5099 i32::wrapping_sub(a[1usize], b[1usize]),
5100 i32::wrapping_sub(a[2usize], b[2usize]),
5101 i32::wrapping_sub(a[3usize], b[3usize]),
5102 ]
5103 .simd_into(self)
5104 }
5105 #[inline(always)]
5106 fn saturating_sub_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
5107 [
5108 i32::saturating_sub(a[0usize], b[0usize]),
5109 i32::saturating_sub(a[1usize], b[1usize]),
5110 i32::saturating_sub(a[2usize], b[2usize]),
5111 i32::saturating_sub(a[3usize], b[3usize]),
5112 ]
5113 .simd_into(self)
5114 }
5115 #[inline(always)]
5116 fn mul_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
5117 [
5118 i32::wrapping_mul(a[0usize], b[0usize]),
5119 i32::wrapping_mul(a[1usize], b[1usize]),
5120 i32::wrapping_mul(a[2usize], b[2usize]),
5121 i32::wrapping_mul(a[3usize], b[3usize]),
5122 ]
5123 .simd_into(self)
5124 }
5125 #[inline(always)]
5126 fn and_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
5127 [
5128 i32::bitand(a[0usize], &b[0usize]),
5129 i32::bitand(a[1usize], &b[1usize]),
5130 i32::bitand(a[2usize], &b[2usize]),
5131 i32::bitand(a[3usize], &b[3usize]),
5132 ]
5133 .simd_into(self)
5134 }
5135 #[inline(always)]
5136 fn or_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
5137 [
5138 i32::bitor(a[0usize], &b[0usize]),
5139 i32::bitor(a[1usize], &b[1usize]),
5140 i32::bitor(a[2usize], &b[2usize]),
5141 i32::bitor(a[3usize], &b[3usize]),
5142 ]
5143 .simd_into(self)
5144 }
5145 #[inline(always)]
5146 fn xor_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
5147 [
5148 i32::bitxor(a[0usize], &b[0usize]),
5149 i32::bitxor(a[1usize], &b[1usize]),
5150 i32::bitxor(a[2usize], &b[2usize]),
5151 i32::bitxor(a[3usize], &b[3usize]),
5152 ]
5153 .simd_into(self)
5154 }
5155 #[inline(always)]
5156 fn not_i32x4(self, a: i32x4<Self>) -> i32x4<Self> {
5157 [
5158 i32::not(a[0usize]),
5159 i32::not(a[1usize]),
5160 i32::not(a[2usize]),
5161 i32::not(a[3usize]),
5162 ]
5163 .simd_into(self)
5164 }
5165 #[inline(always)]
5166 fn shl_i32x4(self, a: i32x4<Self>, shift: u32) -> i32x4<Self> {
5167 [
5168 i32::wrapping_shl(a[0usize], shift),
5169 i32::wrapping_shl(a[1usize], shift),
5170 i32::wrapping_shl(a[2usize], shift),
5171 i32::wrapping_shl(a[3usize], shift),
5172 ]
5173 .simd_into(self)
5174 }
5175 #[inline(always)]
5176 fn shlv_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
5177 [
5178 i32::wrapping_shl(a[0usize], b[0usize] as u32),
5179 i32::wrapping_shl(a[1usize], b[1usize] as u32),
5180 i32::wrapping_shl(a[2usize], b[2usize] as u32),
5181 i32::wrapping_shl(a[3usize], b[3usize] as u32),
5182 ]
5183 .simd_into(self)
5184 }
5185 #[inline(always)]
5186 fn shr_i32x4(self, a: i32x4<Self>, shift: u32) -> i32x4<Self> {
5187 [
5188 i32::wrapping_shr(a[0usize], shift),
5189 i32::wrapping_shr(a[1usize], shift),
5190 i32::wrapping_shr(a[2usize], shift),
5191 i32::wrapping_shr(a[3usize], shift),
5192 ]
5193 .simd_into(self)
5194 }
5195 #[inline(always)]
5196 fn shrv_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
5197 [
5198 i32::wrapping_shr(a[0usize], b[0usize] as u32),
5199 i32::wrapping_shr(a[1usize], b[1usize] as u32),
5200 i32::wrapping_shr(a[2usize], b[2usize] as u32),
5201 i32::wrapping_shr(a[3usize], b[3usize] as u32),
5202 ]
5203 .simd_into(self)
5204 }
5205 #[inline(always)]
5206 fn reduce_max_i32x4(self, a: i32x4<Self>) -> i32 {
5207 let reduced: [i32; 2usize] = [
5208 i32::max(a[0usize], a[1usize]),
5209 i32::max(a[2usize], a[3usize]),
5210 ];
5211 let reduced: [i32; 1usize] = [i32::max(reduced[0usize], reduced[1usize])];
5212 reduced[0]
5213 }
5214 #[inline(always)]
5215 fn reduce_min_i32x4(self, a: i32x4<Self>) -> i32 {
5216 let reduced: [i32; 2usize] = [
5217 i32::min(a[0usize], a[1usize]),
5218 i32::min(a[2usize], a[3usize]),
5219 ];
5220 let reduced: [i32; 1usize] = [i32::min(reduced[0usize], reduced[1usize])];
5221 reduced[0]
5222 }
5223 #[inline(always)]
5224 fn reduce_sum_i32x4(self, a: i32x4<Self>) -> i32 {
5225 let add_level_0: [i32; 2usize] = [
5226 a[0usize].wrapping_add(a[1usize]),
5227 a[2usize].wrapping_add(a[3usize]),
5228 ];
5229 let add_level_1: [i32; 1usize] = [add_level_0[0usize].wrapping_add(add_level_0[1usize])];
5230 add_level_1[0]
5231 }
5232 #[inline(always)]
5233 fn reduce_product_i32x4(self, a: i32x4<Self>) -> i32 {
5234 let mul_level_0: [i32; 2usize] = [
5235 a[0usize].wrapping_mul(a[1usize]),
5236 a[2usize].wrapping_mul(a[3usize]),
5237 ];
5238 let mul_level_1: [i32; 1usize] = [mul_level_0[0usize].wrapping_mul(mul_level_0[1usize])];
5239 mul_level_1[0]
5240 }
5241 #[inline(always)]
5242 fn max_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
5243 [
5244 i32::max(a[0usize], b[0usize]),
5245 i32::max(a[1usize], b[1usize]),
5246 i32::max(a[2usize], b[2usize]),
5247 i32::max(a[3usize], b[3usize]),
5248 ]
5249 .simd_into(self)
5250 }
5251 #[inline(always)]
5252 fn min_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
5253 [
5254 i32::min(a[0usize], b[0usize]),
5255 i32::min(a[1usize], b[1usize]),
5256 i32::min(a[2usize], b[2usize]),
5257 i32::min(a[3usize], b[3usize]),
5258 ]
5259 .simd_into(self)
5260 }
5261 #[inline(always)]
5262 fn clamp_i32x4(self, a: i32x4<Self>, b: i32x4<Self>, c: i32x4<Self>) -> i32x4<Self> {
5263 [
5264 i32::clamp_debug(a[0usize], b[0usize], c[0usize]),
5265 i32::clamp_debug(a[1usize], b[1usize], c[1usize]),
5266 i32::clamp_debug(a[2usize], b[2usize], c[2usize]),
5267 i32::clamp_debug(a[3usize], b[3usize], c[3usize]),
5268 ]
5269 .simd_into(self)
5270 }
5271 #[inline(always)]
5272 fn simd_eq_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> mask32x4<Self> {
5273 [
5274 -(i32::eq(&a[0usize], &b[0usize]) as i32),
5275 -(i32::eq(&a[1usize], &b[1usize]) as i32),
5276 -(i32::eq(&a[2usize], &b[2usize]) as i32),
5277 -(i32::eq(&a[3usize], &b[3usize]) as i32),
5278 ]
5279 .simd_into(self)
5280 }
5281 #[inline(always)]
5282 fn simd_lt_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> mask32x4<Self> {
5283 [
5284 -(i32::lt(&a[0usize], &b[0usize]) as i32),
5285 -(i32::lt(&a[1usize], &b[1usize]) as i32),
5286 -(i32::lt(&a[2usize], &b[2usize]) as i32),
5287 -(i32::lt(&a[3usize], &b[3usize]) as i32),
5288 ]
5289 .simd_into(self)
5290 }
5291 #[inline(always)]
5292 fn simd_le_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> mask32x4<Self> {
5293 [
5294 -(i32::le(&a[0usize], &b[0usize]) as i32),
5295 -(i32::le(&a[1usize], &b[1usize]) as i32),
5296 -(i32::le(&a[2usize], &b[2usize]) as i32),
5297 -(i32::le(&a[3usize], &b[3usize]) as i32),
5298 ]
5299 .simd_into(self)
5300 }
5301 #[inline(always)]
5302 fn zip_low_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
5303 [a[0usize], b[0usize], a[1usize], b[1usize]].simd_into(self)
5304 }
5305 #[inline(always)]
5306 fn zip_high_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
5307 [a[2usize], b[2usize], a[3usize], b[3usize]].simd_into(self)
5308 }
5309 #[inline(always)]
5310 fn unzip_low_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
5311 [a[0usize], a[2usize], b[0usize], b[2usize]].simd_into(self)
5312 }
5313 #[inline(always)]
5314 fn unzip_high_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
5315 [a[1usize], a[3usize], b[1usize], b[3usize]].simd_into(self)
5316 }
5317 #[inline(always)]
5318 fn interleave_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> (i32x4<Self>, i32x4<Self>) {
5319 (self.zip_low_i32x4(a, b), self.zip_high_i32x4(a, b))
5320 }
5321 #[inline(always)]
5322 fn deinterleave_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> (i32x4<Self>, i32x4<Self>) {
5323 (self.unzip_low_i32x4(a, b), self.unzip_high_i32x4(a, b))
5324 }
5325 #[inline(always)]
5326 fn select_i32x4(self, a: mask32x4<Self>, b: i32x4<Self>, c: i32x4<Self>) -> i32x4<Self> {
5327 [
5328 if a.val.0[0usize] != 0 {
5329 b[0usize]
5330 } else {
5331 c[0usize]
5332 },
5333 if a.val.0[1usize] != 0 {
5334 b[1usize]
5335 } else {
5336 c[1usize]
5337 },
5338 if a.val.0[2usize] != 0 {
5339 b[2usize]
5340 } else {
5341 c[2usize]
5342 },
5343 if a.val.0[3usize] != 0 {
5344 b[3usize]
5345 } else {
5346 c[3usize]
5347 },
5348 ]
5349 .simd_into(self)
5350 }
5351 #[inline(always)]
5352 fn combine_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x8<Self> {
5353 let mut result = [0; 8usize];
5354 result[0..4usize].copy_from_slice(&a.val.0);
5355 result[4usize..8usize].copy_from_slice(&b.val.0);
5356 result.simd_into(self)
5357 }
5358 #[inline(always)]
5359 fn neg_i32x4(self, a: i32x4<Self>) -> i32x4<Self> {
5360 [
5361 i32::wrapping_neg(a[0usize]),
5362 i32::wrapping_neg(a[1usize]),
5363 i32::wrapping_neg(a[2usize]),
5364 i32::wrapping_neg(a[3usize]),
5365 ]
5366 .simd_into(self)
5367 }
5368 #[inline(always)]
5369 fn load_four_interleaved_i32x4(self, src: &[i32; 16usize]) -> [i32x4<Self>; 4usize] {
5370 [
5371 [src[0usize], src[4usize], src[8usize], src[12usize]].simd_into(self),
5372 [src[1usize], src[5usize], src[9usize], src[13usize]].simd_into(self),
5373 [src[2usize], src[6usize], src[10usize], src[14usize]].simd_into(self),
5374 [src[3usize], src[7usize], src[11usize], src[15usize]].simd_into(self),
5375 ]
5376 }
5377 #[inline(always)]
5378 fn store_four_interleaved_i32x4(
5379 self,
5380 vectors: [i32x4<Self>; 4usize],
5381 dest: &mut [i32; 16usize],
5382 ) -> () {
5383 *dest = [
5384 vectors[0usize][0usize],
5385 vectors[1usize][0usize],
5386 vectors[2usize][0usize],
5387 vectors[3usize][0usize],
5388 vectors[0usize][1usize],
5389 vectors[1usize][1usize],
5390 vectors[2usize][1usize],
5391 vectors[3usize][1usize],
5392 vectors[0usize][2usize],
5393 vectors[1usize][2usize],
5394 vectors[2usize][2usize],
5395 vectors[3usize][2usize],
5396 vectors[0usize][3usize],
5397 vectors[1usize][3usize],
5398 vectors[2usize][3usize],
5399 vectors[3usize][3usize],
5400 ];
5401 }
5402 #[inline(always)]
5403 fn widen_i32x4(self, a: i32x4<Self>) -> (i64x2<Self>, i64x2<Self>) {
5404 (
5405 [a[0usize] as i64, a[1usize] as i64].simd_into(self),
5406 [a[2usize] as i64, a[3usize] as i64].simd_into(self),
5407 )
5408 }
5409 #[inline(always)]
5410 fn widening_mul_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> (i64x2<Self>, i64x2<Self>) {
5411 let (a0, a1) = self.widen_i32x4(a);
5412 let (b0, b1) = self.widen_i32x4(b);
5413 (self.mul_i64x2(a0, b0), self.mul_i64x2(a1, b1))
5414 }
5415 #[inline(always)]
5416 fn widening_add_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> (i64x2<Self>, i64x2<Self>) {
5417 let (a0, a1) = self.widen_i32x4(a);
5418 let (b0, b1) = self.widen_i32x4(b);
5419 (self.add_i64x2(a0, b0), self.add_i64x2(a1, b1))
5420 }
5421 #[inline(always)]
5422 fn pairwise_widening_add_i32x4(self, a: i32x4<Self>) -> i64x2<Self> {
5423 [
5424 (a[0usize] as i64) + (a[1usize] as i64),
5425 (a[2usize] as i64) + (a[3usize] as i64),
5426 ]
5427 .simd_into(self)
5428 }
5429 #[inline(always)]
5430 fn pairwise_dot_product_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i64x2<Self> {
5431 let (low, high) = self.widening_mul_i32x4(a, b);
5432 let (even, odd) = self.deinterleave_i64x2(low, high);
5433 self.add_i64x2(even, odd)
5434 }
5435 #[inline(always)]
5436 fn narrow_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i16x8<Self> {
5437 [
5438 a[0usize] as i16,
5439 a[1usize] as i16,
5440 a[2usize] as i16,
5441 a[3usize] as i16,
5442 b[0usize] as i16,
5443 b[1usize] as i16,
5444 b[2usize] as i16,
5445 b[3usize] as i16,
5446 ]
5447 .simd_into(self)
5448 }
5449 #[inline(always)]
5450 fn saturating_narrow_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i16x8<Self> {
5451 [
5452 a[0usize].clamp(i16::MIN as i32, i16::MAX as i32) as i16,
5453 a[1usize].clamp(i16::MIN as i32, i16::MAX as i32) as i16,
5454 a[2usize].clamp(i16::MIN as i32, i16::MAX as i32) as i16,
5455 a[3usize].clamp(i16::MIN as i32, i16::MAX as i32) as i16,
5456 b[0usize].clamp(i16::MIN as i32, i16::MAX as i32) as i16,
5457 b[1usize].clamp(i16::MIN as i32, i16::MAX as i32) as i16,
5458 b[2usize].clamp(i16::MIN as i32, i16::MAX as i32) as i16,
5459 b[3usize].clamp(i16::MIN as i32, i16::MAX as i32) as i16,
5460 ]
5461 .simd_into(self)
5462 }
5463 #[inline(always)]
5464 fn relaxed_narrow_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i16x8<Self> {
5465 debug_assert!(
5466 a.as_slice()
5467 .iter()
5468 .chain(b.as_slice())
5469 .all(|&value| { value >= i16::MIN as i32 && value <= i16::MAX as i32 }),
5470 "relaxed_narrow inputs must fit in the destination type",
5471 );
5472 self.narrow_i32x4(a, b)
5473 }
5474 #[inline(always)]
5475 fn cvt_f32_i32x4(self, a: i32x4<Self>) -> f32x4<Self> {
5476 [
5477 a[0usize] as f32,
5478 a[1usize] as f32,
5479 a[2usize] as f32,
5480 a[3usize] as f32,
5481 ]
5482 .simd_into(self)
5483 }
5484 #[inline(always)]
5485 fn splat_u32x4(self, val: u32) -> u32x4<Self> {
5486 [val; 4usize].simd_into(self)
5487 }
5488 #[inline(always)]
5489 fn reverse_u32x4(self, a: u32x4<Self>) -> u32x4<Self> {
5490 [a[3usize], a[2usize], a[1usize], a[0usize]].simd_into(self)
5491 }
5492 #[inline(always)]
5493 fn slide_u32x4<const SHIFT: usize>(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
5494 let mut dest = [Default::default(); 4usize];
5495 dest[..4usize - SHIFT].copy_from_slice(&a.val.0[SHIFT..]);
5496 dest[4usize - SHIFT..].copy_from_slice(&b.val.0[..SHIFT]);
5497 dest.simd_into(self)
5498 }
5499 #[inline(always)]
5500 fn compress_u32x4(self, values: u32x4<Self>, mask: mask32x4<Self>) -> u32x4<Self> {
5501 let mask = mask8x16 {
5502 val: crate::transmute::checked_transmute_copy(&mask.val),
5503 simd: self,
5504 };
5505 Bytes::from_bytes(self.compress_u8x16(Bytes::to_bytes(values), mask))
5506 }
5507 #[inline(always)]
5508 fn compress_merge_u32x4(
5509 self,
5510 values: u32x4<Self>,
5511 mask: mask32x4<Self>,
5512 merge: u32x4<Self>,
5513 ) -> u32x4<Self> {
5514 let mask = mask8x16 {
5515 val: crate::transmute::checked_transmute_copy(&mask.val),
5516 simd: self,
5517 };
5518 Bytes::from_bytes(self.compress_merge_u8x16(
5519 Bytes::to_bytes(values),
5520 mask,
5521 Bytes::to_bytes(merge),
5522 ))
5523 }
5524 #[inline(always)]
5525 fn expand_u32x4(self, values: u32x4<Self>, mask: mask32x4<Self>) -> u32x4<Self> {
5526 let mask = mask8x16 {
5527 val: crate::transmute::checked_transmute_copy(&mask.val),
5528 simd: self,
5529 };
5530 Bytes::from_bytes(self.expand_u8x16(Bytes::to_bytes(values), mask))
5531 }
5532 #[inline(always)]
5533 fn expand_merge_u32x4(
5534 self,
5535 values: u32x4<Self>,
5536 mask: mask32x4<Self>,
5537 merge: u32x4<Self>,
5538 ) -> u32x4<Self> {
5539 let mask = mask8x16 {
5540 val: crate::transmute::checked_transmute_copy(&mask.val),
5541 simd: self,
5542 };
5543 Bytes::from_bytes(self.expand_merge_u8x16(
5544 Bytes::to_bytes(values),
5545 mask,
5546 Bytes::to_bytes(merge),
5547 ))
5548 }
5549 #[inline(always)]
5550 fn count_ones_u32x4(self, a: u32x4<Self>) -> u32x4<Self> {
5551 [
5552 a[0usize].count_ones(),
5553 a[1usize].count_ones(),
5554 a[2usize].count_ones(),
5555 a[3usize].count_ones(),
5556 ]
5557 .simd_into(self)
5558 }
5559 #[inline(always)]
5560 fn count_zeros_u32x4(self, a: u32x4<Self>) -> u32x4<Self> {
5561 [
5562 a[0usize].count_zeros(),
5563 a[1usize].count_zeros(),
5564 a[2usize].count_zeros(),
5565 a[3usize].count_zeros(),
5566 ]
5567 .simd_into(self)
5568 }
5569 #[inline(always)]
5570 fn add_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
5571 [
5572 u32::wrapping_add(a[0usize], b[0usize]),
5573 u32::wrapping_add(a[1usize], b[1usize]),
5574 u32::wrapping_add(a[2usize], b[2usize]),
5575 u32::wrapping_add(a[3usize], b[3usize]),
5576 ]
5577 .simd_into(self)
5578 }
5579 #[inline(always)]
5580 fn saturating_add_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
5581 [
5582 u32::saturating_add(a[0usize], b[0usize]),
5583 u32::saturating_add(a[1usize], b[1usize]),
5584 u32::saturating_add(a[2usize], b[2usize]),
5585 u32::saturating_add(a[3usize], b[3usize]),
5586 ]
5587 .simd_into(self)
5588 }
5589 #[inline(always)]
5590 fn sub_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
5591 [
5592 u32::wrapping_sub(a[0usize], b[0usize]),
5593 u32::wrapping_sub(a[1usize], b[1usize]),
5594 u32::wrapping_sub(a[2usize], b[2usize]),
5595 u32::wrapping_sub(a[3usize], b[3usize]),
5596 ]
5597 .simd_into(self)
5598 }
5599 #[inline(always)]
5600 fn saturating_sub_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
5601 [
5602 u32::saturating_sub(a[0usize], b[0usize]),
5603 u32::saturating_sub(a[1usize], b[1usize]),
5604 u32::saturating_sub(a[2usize], b[2usize]),
5605 u32::saturating_sub(a[3usize], b[3usize]),
5606 ]
5607 .simd_into(self)
5608 }
5609 #[inline(always)]
5610 fn mul_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
5611 [
5612 u32::wrapping_mul(a[0usize], b[0usize]),
5613 u32::wrapping_mul(a[1usize], b[1usize]),
5614 u32::wrapping_mul(a[2usize], b[2usize]),
5615 u32::wrapping_mul(a[3usize], b[3usize]),
5616 ]
5617 .simd_into(self)
5618 }
5619 #[inline(always)]
5620 fn and_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
5621 [
5622 u32::bitand(a[0usize], &b[0usize]),
5623 u32::bitand(a[1usize], &b[1usize]),
5624 u32::bitand(a[2usize], &b[2usize]),
5625 u32::bitand(a[3usize], &b[3usize]),
5626 ]
5627 .simd_into(self)
5628 }
5629 #[inline(always)]
5630 fn or_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
5631 [
5632 u32::bitor(a[0usize], &b[0usize]),
5633 u32::bitor(a[1usize], &b[1usize]),
5634 u32::bitor(a[2usize], &b[2usize]),
5635 u32::bitor(a[3usize], &b[3usize]),
5636 ]
5637 .simd_into(self)
5638 }
5639 #[inline(always)]
5640 fn xor_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
5641 [
5642 u32::bitxor(a[0usize], &b[0usize]),
5643 u32::bitxor(a[1usize], &b[1usize]),
5644 u32::bitxor(a[2usize], &b[2usize]),
5645 u32::bitxor(a[3usize], &b[3usize]),
5646 ]
5647 .simd_into(self)
5648 }
5649 #[inline(always)]
5650 fn not_u32x4(self, a: u32x4<Self>) -> u32x4<Self> {
5651 [
5652 u32::not(a[0usize]),
5653 u32::not(a[1usize]),
5654 u32::not(a[2usize]),
5655 u32::not(a[3usize]),
5656 ]
5657 .simd_into(self)
5658 }
5659 #[inline(always)]
5660 fn shl_u32x4(self, a: u32x4<Self>, shift: u32) -> u32x4<Self> {
5661 [
5662 u32::wrapping_shl(a[0usize], shift),
5663 u32::wrapping_shl(a[1usize], shift),
5664 u32::wrapping_shl(a[2usize], shift),
5665 u32::wrapping_shl(a[3usize], shift),
5666 ]
5667 .simd_into(self)
5668 }
5669 #[inline(always)]
5670 fn shlv_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
5671 [
5672 u32::wrapping_shl(a[0usize], b[0usize] as u32),
5673 u32::wrapping_shl(a[1usize], b[1usize] as u32),
5674 u32::wrapping_shl(a[2usize], b[2usize] as u32),
5675 u32::wrapping_shl(a[3usize], b[3usize] as u32),
5676 ]
5677 .simd_into(self)
5678 }
5679 #[inline(always)]
5680 fn shr_u32x4(self, a: u32x4<Self>, shift: u32) -> u32x4<Self> {
5681 [
5682 u32::wrapping_shr(a[0usize], shift),
5683 u32::wrapping_shr(a[1usize], shift),
5684 u32::wrapping_shr(a[2usize], shift),
5685 u32::wrapping_shr(a[3usize], shift),
5686 ]
5687 .simd_into(self)
5688 }
5689 #[inline(always)]
5690 fn shrv_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
5691 [
5692 u32::wrapping_shr(a[0usize], b[0usize] as u32),
5693 u32::wrapping_shr(a[1usize], b[1usize] as u32),
5694 u32::wrapping_shr(a[2usize], b[2usize] as u32),
5695 u32::wrapping_shr(a[3usize], b[3usize] as u32),
5696 ]
5697 .simd_into(self)
5698 }
5699 #[inline(always)]
5700 fn reduce_max_u32x4(self, a: u32x4<Self>) -> u32 {
5701 let reduced: [u32; 2usize] = [
5702 u32::max(a[0usize], a[1usize]),
5703 u32::max(a[2usize], a[3usize]),
5704 ];
5705 let reduced: [u32; 1usize] = [u32::max(reduced[0usize], reduced[1usize])];
5706 reduced[0]
5707 }
5708 #[inline(always)]
5709 fn reduce_min_u32x4(self, a: u32x4<Self>) -> u32 {
5710 let reduced: [u32; 2usize] = [
5711 u32::min(a[0usize], a[1usize]),
5712 u32::min(a[2usize], a[3usize]),
5713 ];
5714 let reduced: [u32; 1usize] = [u32::min(reduced[0usize], reduced[1usize])];
5715 reduced[0]
5716 }
5717 #[inline(always)]
5718 fn reduce_sum_u32x4(self, a: u32x4<Self>) -> u32 {
5719 let add_level_0: [u32; 2usize] = [
5720 a[0usize].wrapping_add(a[1usize]),
5721 a[2usize].wrapping_add(a[3usize]),
5722 ];
5723 let add_level_1: [u32; 1usize] = [add_level_0[0usize].wrapping_add(add_level_0[1usize])];
5724 add_level_1[0]
5725 }
5726 #[inline(always)]
5727 fn reduce_product_u32x4(self, a: u32x4<Self>) -> u32 {
5728 let mul_level_0: [u32; 2usize] = [
5729 a[0usize].wrapping_mul(a[1usize]),
5730 a[2usize].wrapping_mul(a[3usize]),
5731 ];
5732 let mul_level_1: [u32; 1usize] = [mul_level_0[0usize].wrapping_mul(mul_level_0[1usize])];
5733 mul_level_1[0]
5734 }
5735 #[inline(always)]
5736 fn max_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
5737 [
5738 u32::max(a[0usize], b[0usize]),
5739 u32::max(a[1usize], b[1usize]),
5740 u32::max(a[2usize], b[2usize]),
5741 u32::max(a[3usize], b[3usize]),
5742 ]
5743 .simd_into(self)
5744 }
5745 #[inline(always)]
5746 fn min_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
5747 [
5748 u32::min(a[0usize], b[0usize]),
5749 u32::min(a[1usize], b[1usize]),
5750 u32::min(a[2usize], b[2usize]),
5751 u32::min(a[3usize], b[3usize]),
5752 ]
5753 .simd_into(self)
5754 }
5755 #[inline(always)]
5756 fn clamp_u32x4(self, a: u32x4<Self>, b: u32x4<Self>, c: u32x4<Self>) -> u32x4<Self> {
5757 [
5758 u32::clamp_debug(a[0usize], b[0usize], c[0usize]),
5759 u32::clamp_debug(a[1usize], b[1usize], c[1usize]),
5760 u32::clamp_debug(a[2usize], b[2usize], c[2usize]),
5761 u32::clamp_debug(a[3usize], b[3usize], c[3usize]),
5762 ]
5763 .simd_into(self)
5764 }
5765 #[inline(always)]
5766 fn simd_eq_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> mask32x4<Self> {
5767 [
5768 -(u32::eq(&a[0usize], &b[0usize]) as i32),
5769 -(u32::eq(&a[1usize], &b[1usize]) as i32),
5770 -(u32::eq(&a[2usize], &b[2usize]) as i32),
5771 -(u32::eq(&a[3usize], &b[3usize]) as i32),
5772 ]
5773 .simd_into(self)
5774 }
5775 #[inline(always)]
5776 fn simd_lt_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> mask32x4<Self> {
5777 [
5778 -(u32::lt(&a[0usize], &b[0usize]) as i32),
5779 -(u32::lt(&a[1usize], &b[1usize]) as i32),
5780 -(u32::lt(&a[2usize], &b[2usize]) as i32),
5781 -(u32::lt(&a[3usize], &b[3usize]) as i32),
5782 ]
5783 .simd_into(self)
5784 }
5785 #[inline(always)]
5786 fn simd_le_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> mask32x4<Self> {
5787 [
5788 -(u32::le(&a[0usize], &b[0usize]) as i32),
5789 -(u32::le(&a[1usize], &b[1usize]) as i32),
5790 -(u32::le(&a[2usize], &b[2usize]) as i32),
5791 -(u32::le(&a[3usize], &b[3usize]) as i32),
5792 ]
5793 .simd_into(self)
5794 }
5795 #[inline(always)]
5796 fn zip_low_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
5797 [a[0usize], b[0usize], a[1usize], b[1usize]].simd_into(self)
5798 }
5799 #[inline(always)]
5800 fn zip_high_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
5801 [a[2usize], b[2usize], a[3usize], b[3usize]].simd_into(self)
5802 }
5803 #[inline(always)]
5804 fn unzip_low_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
5805 [a[0usize], a[2usize], b[0usize], b[2usize]].simd_into(self)
5806 }
5807 #[inline(always)]
5808 fn unzip_high_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
5809 [a[1usize], a[3usize], b[1usize], b[3usize]].simd_into(self)
5810 }
5811 #[inline(always)]
5812 fn interleave_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> (u32x4<Self>, u32x4<Self>) {
5813 (self.zip_low_u32x4(a, b), self.zip_high_u32x4(a, b))
5814 }
5815 #[inline(always)]
5816 fn deinterleave_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> (u32x4<Self>, u32x4<Self>) {
5817 (self.unzip_low_u32x4(a, b), self.unzip_high_u32x4(a, b))
5818 }
5819 #[inline(always)]
5820 fn select_u32x4(self, a: mask32x4<Self>, b: u32x4<Self>, c: u32x4<Self>) -> u32x4<Self> {
5821 [
5822 if a.val.0[0usize] != 0 {
5823 b[0usize]
5824 } else {
5825 c[0usize]
5826 },
5827 if a.val.0[1usize] != 0 {
5828 b[1usize]
5829 } else {
5830 c[1usize]
5831 },
5832 if a.val.0[2usize] != 0 {
5833 b[2usize]
5834 } else {
5835 c[2usize]
5836 },
5837 if a.val.0[3usize] != 0 {
5838 b[3usize]
5839 } else {
5840 c[3usize]
5841 },
5842 ]
5843 .simd_into(self)
5844 }
5845 #[inline(always)]
5846 fn combine_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x8<Self> {
5847 let mut result = [0; 8usize];
5848 result[0..4usize].copy_from_slice(&a.val.0);
5849 result[4usize..8usize].copy_from_slice(&b.val.0);
5850 result.simd_into(self)
5851 }
5852 #[inline(always)]
5853 fn load_four_interleaved_u32x4(self, src: &[u32; 16usize]) -> [u32x4<Self>; 4usize] {
5854 [
5855 [src[0usize], src[4usize], src[8usize], src[12usize]].simd_into(self),
5856 [src[1usize], src[5usize], src[9usize], src[13usize]].simd_into(self),
5857 [src[2usize], src[6usize], src[10usize], src[14usize]].simd_into(self),
5858 [src[3usize], src[7usize], src[11usize], src[15usize]].simd_into(self),
5859 ]
5860 }
5861 #[inline(always)]
5862 fn store_four_interleaved_u32x4(
5863 self,
5864 vectors: [u32x4<Self>; 4usize],
5865 dest: &mut [u32; 16usize],
5866 ) -> () {
5867 *dest = [
5868 vectors[0usize][0usize],
5869 vectors[1usize][0usize],
5870 vectors[2usize][0usize],
5871 vectors[3usize][0usize],
5872 vectors[0usize][1usize],
5873 vectors[1usize][1usize],
5874 vectors[2usize][1usize],
5875 vectors[3usize][1usize],
5876 vectors[0usize][2usize],
5877 vectors[1usize][2usize],
5878 vectors[2usize][2usize],
5879 vectors[3usize][2usize],
5880 vectors[0usize][3usize],
5881 vectors[1usize][3usize],
5882 vectors[2usize][3usize],
5883 vectors[3usize][3usize],
5884 ];
5885 }
5886 #[inline(always)]
5887 fn widen_u32x4(self, a: u32x4<Self>) -> (u64x2<Self>, u64x2<Self>) {
5888 (
5889 [a[0usize] as u64, a[1usize] as u64].simd_into(self),
5890 [a[2usize] as u64, a[3usize] as u64].simd_into(self),
5891 )
5892 }
5893 #[inline(always)]
5894 fn widening_mul_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> (u64x2<Self>, u64x2<Self>) {
5895 let (a0, a1) = self.widen_u32x4(a);
5896 let (b0, b1) = self.widen_u32x4(b);
5897 (self.mul_u64x2(a0, b0), self.mul_u64x2(a1, b1))
5898 }
5899 #[inline(always)]
5900 fn widening_add_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> (u64x2<Self>, u64x2<Self>) {
5901 let (a0, a1) = self.widen_u32x4(a);
5902 let (b0, b1) = self.widen_u32x4(b);
5903 (self.add_u64x2(a0, b0), self.add_u64x2(a1, b1))
5904 }
5905 #[inline(always)]
5906 fn pairwise_widening_add_u32x4(self, a: u32x4<Self>) -> u64x2<Self> {
5907 [
5908 (a[0usize] as u64) + (a[1usize] as u64),
5909 (a[2usize] as u64) + (a[3usize] as u64),
5910 ]
5911 .simd_into(self)
5912 }
5913 #[inline(always)]
5914 fn pairwise_dot_product_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u64x2<Self> {
5915 let (low, high) = self.widening_mul_u32x4(a, b);
5916 let (even, odd) = self.deinterleave_u64x2(low, high);
5917 self.add_u64x2(even, odd)
5918 }
5919 #[inline(always)]
5920 fn narrow_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u16x8<Self> {
5921 [
5922 a[0usize] as u16,
5923 a[1usize] as u16,
5924 a[2usize] as u16,
5925 a[3usize] as u16,
5926 b[0usize] as u16,
5927 b[1usize] as u16,
5928 b[2usize] as u16,
5929 b[3usize] as u16,
5930 ]
5931 .simd_into(self)
5932 }
5933 #[inline(always)]
5934 fn saturating_narrow_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u16x8<Self> {
5935 [
5936 a[0usize].clamp(u16::MIN as u32, u16::MAX as u32) as u16,
5937 a[1usize].clamp(u16::MIN as u32, u16::MAX as u32) as u16,
5938 a[2usize].clamp(u16::MIN as u32, u16::MAX as u32) as u16,
5939 a[3usize].clamp(u16::MIN as u32, u16::MAX as u32) as u16,
5940 b[0usize].clamp(u16::MIN as u32, u16::MAX as u32) as u16,
5941 b[1usize].clamp(u16::MIN as u32, u16::MAX as u32) as u16,
5942 b[2usize].clamp(u16::MIN as u32, u16::MAX as u32) as u16,
5943 b[3usize].clamp(u16::MIN as u32, u16::MAX as u32) as u16,
5944 ]
5945 .simd_into(self)
5946 }
5947 #[inline(always)]
5948 fn relaxed_narrow_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u16x8<Self> {
5949 debug_assert!(
5950 a.as_slice()
5951 .iter()
5952 .chain(b.as_slice())
5953 .all(|&value| { value >= u16::MIN as u32 && value <= u16::MAX as u32 }),
5954 "relaxed_narrow inputs must fit in the destination type",
5955 );
5956 self.narrow_u32x4(a, b)
5957 }
5958 #[inline(always)]
5959 fn cvt_f32_u32x4(self, a: u32x4<Self>) -> f32x4<Self> {
5960 [
5961 a[0usize] as f32,
5962 a[1usize] as f32,
5963 a[2usize] as f32,
5964 a[3usize] as f32,
5965 ]
5966 .simd_into(self)
5967 }
5968 #[inline(always)]
5969 fn splat_mask32x4(self, val: bool) -> mask32x4<Self> {
5970 let val: i32 = if val { !0 } else { 0 };
5971 [val; 4usize].simd_into(self)
5972 }
5973 #[inline(always)]
5974 fn from_bitmask_mask32x4(self, bits: u64) -> mask32x4<Self> {
5975 let lanes: [i32; 4usize] = [
5976 if bits & 1 != 0 { !0 } else { 0 },
5977 if (bits >> 1usize) & 1 != 0 { !0 } else { 0 },
5978 if (bits >> 2usize) & 1 != 0 { !0 } else { 0 },
5979 if (bits >> 3usize) & 1 != 0 { !0 } else { 0 },
5980 ];
5981 lanes.simd_into(self)
5982 }
5983 #[inline(always)]
5984 fn to_bitmask_mask32x4(self, a: mask32x4<Self>) -> u64 {
5985 let lanes: [i32; 4usize] = a.into();
5986 let mut bits = 0u64;
5987 let mut i = 0;
5988 while i < 4usize {
5989 if lanes[i] != 0 {
5990 bits |= 1u64 << i;
5991 }
5992 i += 1;
5993 }
5994 bits
5995 }
5996 #[inline(always)]
5997 fn from_vector_mask32x4(self, vector: i32x4<Self>) -> mask32x4<Self> {
5998 mask32x4 {
5999 val: vector.val,
6000 simd: self,
6001 }
6002 }
6003 #[inline(always)]
6004 fn to_vector_mask32x4(self, a: mask32x4<Self>) -> i32x4<Self> {
6005 i32x4 {
6006 val: a.val,
6007 simd: self,
6008 }
6009 }
6010 #[inline(always)]
6011 fn set_mask32x4(self, a: &mut mask32x4<Self>, index: usize, value: bool) -> () {
6012 assert!(
6013 index < 4usize,
6014 "mask lane index {index} is out of bounds for {} lanes",
6015 4usize
6016 );
6017 let mut lanes: [i32; 4usize] = (*a).into();
6018 lanes[index] = if value { !0 } else { 0 };
6019 *a = lanes.simd_into(self);
6020 }
6021 #[inline(always)]
6022 fn rotate_elements_left_mask32x4<const OFFSET: usize>(
6023 self,
6024 a: mask32x4<Self>,
6025 ) -> mask32x4<Self> {
6026 let int = i32x4 {
6027 val: crate::transmute::checked_transmute_copy(&a.val),
6028 simd: self,
6029 };
6030 let rotated = int.rotate_elements_left::<OFFSET>();
6031 mask32x4 {
6032 val: crate::transmute::checked_transmute_copy(&rotated.val),
6033 simd: self,
6034 }
6035 }
6036 #[inline(always)]
6037 fn rotate_elements_right_mask32x4<const OFFSET: usize>(
6038 self,
6039 a: mask32x4<Self>,
6040 ) -> mask32x4<Self> {
6041 let int = i32x4 {
6042 val: crate::transmute::checked_transmute_copy(&a.val),
6043 simd: self,
6044 };
6045 let rotated = int.rotate_elements_right::<OFFSET>();
6046 mask32x4 {
6047 val: crate::transmute::checked_transmute_copy(&rotated.val),
6048 simd: self,
6049 }
6050 }
6051 #[inline(always)]
6052 fn and_mask32x4(self, a: mask32x4<Self>, b: mask32x4<Self>) -> mask32x4<Self> {
6053 [
6054 i32::bitand(a.val.0[0usize], &b.val.0[0usize]),
6055 i32::bitand(a.val.0[1usize], &b.val.0[1usize]),
6056 i32::bitand(a.val.0[2usize], &b.val.0[2usize]),
6057 i32::bitand(a.val.0[3usize], &b.val.0[3usize]),
6058 ]
6059 .simd_into(self)
6060 }
6061 #[inline(always)]
6062 fn or_mask32x4(self, a: mask32x4<Self>, b: mask32x4<Self>) -> mask32x4<Self> {
6063 [
6064 i32::bitor(a.val.0[0usize], &b.val.0[0usize]),
6065 i32::bitor(a.val.0[1usize], &b.val.0[1usize]),
6066 i32::bitor(a.val.0[2usize], &b.val.0[2usize]),
6067 i32::bitor(a.val.0[3usize], &b.val.0[3usize]),
6068 ]
6069 .simd_into(self)
6070 }
6071 #[inline(always)]
6072 fn xor_mask32x4(self, a: mask32x4<Self>, b: mask32x4<Self>) -> mask32x4<Self> {
6073 [
6074 i32::bitxor(a.val.0[0usize], &b.val.0[0usize]),
6075 i32::bitxor(a.val.0[1usize], &b.val.0[1usize]),
6076 i32::bitxor(a.val.0[2usize], &b.val.0[2usize]),
6077 i32::bitxor(a.val.0[3usize], &b.val.0[3usize]),
6078 ]
6079 .simd_into(self)
6080 }
6081 #[inline(always)]
6082 fn not_mask32x4(self, a: mask32x4<Self>) -> mask32x4<Self> {
6083 [
6084 i32::not(a.val.0[0usize]),
6085 i32::not(a.val.0[1usize]),
6086 i32::not(a.val.0[2usize]),
6087 i32::not(a.val.0[3usize]),
6088 ]
6089 .simd_into(self)
6090 }
6091 #[inline(always)]
6092 fn reverse_mask32x4(self, a: mask32x4<Self>) -> mask32x4<Self> {
6093 [
6094 a.val.0[3usize],
6095 a.val.0[2usize],
6096 a.val.0[1usize],
6097 a.val.0[0usize],
6098 ]
6099 .simd_into(self)
6100 }
6101 #[inline(always)]
6102 fn select_mask32x4(
6103 self,
6104 a: mask32x4<Self>,
6105 b: mask32x4<Self>,
6106 c: mask32x4<Self>,
6107 ) -> mask32x4<Self> {
6108 [
6109 if a.val.0[0usize] != 0 {
6110 b.val.0[0usize]
6111 } else {
6112 c.val.0[0usize]
6113 },
6114 if a.val.0[1usize] != 0 {
6115 b.val.0[1usize]
6116 } else {
6117 c.val.0[1usize]
6118 },
6119 if a.val.0[2usize] != 0 {
6120 b.val.0[2usize]
6121 } else {
6122 c.val.0[2usize]
6123 },
6124 if a.val.0[3usize] != 0 {
6125 b.val.0[3usize]
6126 } else {
6127 c.val.0[3usize]
6128 },
6129 ]
6130 .simd_into(self)
6131 }
6132 #[inline(always)]
6133 fn simd_eq_mask32x4(self, a: mask32x4<Self>, b: mask32x4<Self>) -> mask32x4<Self> {
6134 [
6135 -(i32::eq(&a.val.0[0usize], &b.val.0[0usize]) as i32),
6136 -(i32::eq(&a.val.0[1usize], &b.val.0[1usize]) as i32),
6137 -(i32::eq(&a.val.0[2usize], &b.val.0[2usize]) as i32),
6138 -(i32::eq(&a.val.0[3usize], &b.val.0[3usize]) as i32),
6139 ]
6140 .simd_into(self)
6141 }
6142 #[inline(always)]
6143 fn any_true_mask32x4(self, a: mask32x4<Self>) -> bool {
6144 a.val.0[0usize] != 0 || a.val.0[1usize] != 0 || a.val.0[2usize] != 0 || a.val.0[3usize] != 0
6145 }
6146 #[inline(always)]
6147 fn all_true_mask32x4(self, a: mask32x4<Self>) -> bool {
6148 a.val.0[0usize] != 0 && a.val.0[1usize] != 0 && a.val.0[2usize] != 0 && a.val.0[3usize] != 0
6149 }
6150 #[inline(always)]
6151 fn any_false_mask32x4(self, a: mask32x4<Self>) -> bool {
6152 a.val.0[0usize] == 0 || a.val.0[1usize] == 0 || a.val.0[2usize] == 0 || a.val.0[3usize] == 0
6153 }
6154 #[inline(always)]
6155 fn all_false_mask32x4(self, a: mask32x4<Self>) -> bool {
6156 a.val.0[0usize] == 0 && a.val.0[1usize] == 0 && a.val.0[2usize] == 0 && a.val.0[3usize] == 0
6157 }
6158 #[inline(always)]
6159 fn combine_mask32x4(self, a: mask32x4<Self>, b: mask32x4<Self>) -> mask32x8<Self> {
6160 let mut result = [0; 8usize];
6161 result[0..4usize].copy_from_slice(&a.val.0);
6162 result[4usize..8usize].copy_from_slice(&b.val.0);
6163 result.simd_into(self)
6164 }
6165 #[inline(always)]
6166 fn widen_mask32x4(self, a: mask32x4<Self>) -> (mask64x2<Self>, mask64x2<Self>) {
6167 (
6168 [a.val.0[0usize] as i64, a.val.0[1usize] as i64].simd_into(self),
6169 [a.val.0[2usize] as i64, a.val.0[3usize] as i64].simd_into(self),
6170 )
6171 }
6172 #[inline(always)]
6173 fn narrow_mask32x4(self, a: mask32x4<Self>, b: mask32x4<Self>) -> mask16x8<Self> {
6174 [
6175 a.val.0[0usize] as i16,
6176 a.val.0[1usize] as i16,
6177 a.val.0[2usize] as i16,
6178 a.val.0[3usize] as i16,
6179 b.val.0[0usize] as i16,
6180 b.val.0[1usize] as i16,
6181 b.val.0[2usize] as i16,
6182 b.val.0[3usize] as i16,
6183 ]
6184 .simd_into(self)
6185 }
6186 #[inline(always)]
6187 fn abs_f64x2(self, a: f64x2<Self>) -> f64x2<Self> {
6188 [f64::abs(a[0usize]), f64::abs(a[1usize])].simd_into(self)
6189 }
6190 #[inline(always)]
6191 fn splat_f64x2(self, val: f64) -> f64x2<Self> {
6192 [val; 2usize].simd_into(self)
6193 }
6194 #[inline(always)]
6195 fn reverse_f64x2(self, a: f64x2<Self>) -> f64x2<Self> {
6196 [a[1usize], a[0usize]].simd_into(self)
6197 }
6198 #[inline(always)]
6199 fn slide_f64x2<const SHIFT: usize>(self, a: f64x2<Self>, b: f64x2<Self>) -> f64x2<Self> {
6200 let mut dest = [Default::default(); 2usize];
6201 dest[..2usize - SHIFT].copy_from_slice(&a.val.0[SHIFT..]);
6202 dest[2usize - SHIFT..].copy_from_slice(&b.val.0[..SHIFT]);
6203 dest.simd_into(self)
6204 }
6205 #[inline(always)]
6206 fn compress_f64x2(self, values: f64x2<Self>, mask: mask64x2<Self>) -> f64x2<Self> {
6207 let mask = mask8x16 {
6208 val: crate::transmute::checked_transmute_copy(&mask.val),
6209 simd: self,
6210 };
6211 Bytes::from_bytes(self.compress_u8x16(Bytes::to_bytes(values), mask))
6212 }
6213 #[inline(always)]
6214 fn compress_merge_f64x2(
6215 self,
6216 values: f64x2<Self>,
6217 mask: mask64x2<Self>,
6218 merge: f64x2<Self>,
6219 ) -> f64x2<Self> {
6220 let mask = mask8x16 {
6221 val: crate::transmute::checked_transmute_copy(&mask.val),
6222 simd: self,
6223 };
6224 Bytes::from_bytes(self.compress_merge_u8x16(
6225 Bytes::to_bytes(values),
6226 mask,
6227 Bytes::to_bytes(merge),
6228 ))
6229 }
6230 #[inline(always)]
6231 fn expand_f64x2(self, values: f64x2<Self>, mask: mask64x2<Self>) -> f64x2<Self> {
6232 let mask = mask8x16 {
6233 val: crate::transmute::checked_transmute_copy(&mask.val),
6234 simd: self,
6235 };
6236 Bytes::from_bytes(self.expand_u8x16(Bytes::to_bytes(values), mask))
6237 }
6238 #[inline(always)]
6239 fn expand_merge_f64x2(
6240 self,
6241 values: f64x2<Self>,
6242 mask: mask64x2<Self>,
6243 merge: f64x2<Self>,
6244 ) -> f64x2<Self> {
6245 let mask = mask8x16 {
6246 val: crate::transmute::checked_transmute_copy(&mask.val),
6247 simd: self,
6248 };
6249 Bytes::from_bytes(self.expand_merge_u8x16(
6250 Bytes::to_bytes(values),
6251 mask,
6252 Bytes::to_bytes(merge),
6253 ))
6254 }
6255 #[inline(always)]
6256 fn neg_f64x2(self, a: f64x2<Self>) -> f64x2<Self> {
6257 [f64::neg(a[0usize]), f64::neg(a[1usize])].simd_into(self)
6258 }
6259 #[inline(always)]
6260 fn sqrt_f64x2(self, a: f64x2<Self>) -> f64x2<Self> {
6261 [f64::sqrt(a[0usize]), f64::sqrt(a[1usize])].simd_into(self)
6262 }
6263 #[inline(always)]
6264 fn approximate_recip_f64x2(self, a: f64x2<Self>) -> f64x2<Self> {
6265 1.0 / a
6266 }
6267 #[inline(always)]
6268 fn add_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f64x2<Self> {
6269 [
6270 f64::add(a[0usize], &b[0usize]),
6271 f64::add(a[1usize], &b[1usize]),
6272 ]
6273 .simd_into(self)
6274 }
6275 #[inline(always)]
6276 fn sub_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f64x2<Self> {
6277 [
6278 f64::sub(a[0usize], &b[0usize]),
6279 f64::sub(a[1usize], &b[1usize]),
6280 ]
6281 .simd_into(self)
6282 }
6283 #[inline(always)]
6284 fn mul_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f64x2<Self> {
6285 [
6286 f64::mul(a[0usize], &b[0usize]),
6287 f64::mul(a[1usize], &b[1usize]),
6288 ]
6289 .simd_into(self)
6290 }
6291 #[inline(always)]
6292 fn div_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f64x2<Self> {
6293 [
6294 f64::div(a[0usize], &b[0usize]),
6295 f64::div(a[1usize], &b[1usize]),
6296 ]
6297 .simd_into(self)
6298 }
6299 #[inline(always)]
6300 fn copysign_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f64x2<Self> {
6301 [
6302 f64::copysign(a[0usize], b[0usize]),
6303 f64::copysign(a[1usize], b[1usize]),
6304 ]
6305 .simd_into(self)
6306 }
6307 #[inline(always)]
6308 fn reduce_max_f64x2(self, a: f64x2<Self>) -> f64 {
6309 let reduced: [f64; 1usize] = [f64::max(a[0usize], a[1usize])];
6310 reduced[0]
6311 }
6312 #[inline(always)]
6313 fn reduce_min_f64x2(self, a: f64x2<Self>) -> f64 {
6314 let reduced: [f64; 1usize] = [f64::min(a[0usize], a[1usize])];
6315 reduced[0]
6316 }
6317 #[inline(always)]
6318 fn reduce_max_precise_f64x2(self, a: f64x2<Self>) -> f64 {
6319 let reduced: [f64; 1usize] = [f64::max(a[0usize], a[1usize])];
6320 reduced[0]
6321 }
6322 #[inline(always)]
6323 fn reduce_min_precise_f64x2(self, a: f64x2<Self>) -> f64 {
6324 let reduced: [f64; 1usize] = [f64::min(a[0usize], a[1usize])];
6325 reduced[0]
6326 }
6327 #[inline(always)]
6328 fn reduce_sum_f64x2(self, a: f64x2<Self>) -> f64 {
6329 let add_level_0: [f64; 1usize] = [a[0usize] + a[1usize]];
6330 add_level_0[0]
6331 }
6332 #[inline(always)]
6333 fn reduce_product_f64x2(self, a: f64x2<Self>) -> f64 {
6334 let mul_level_0: [f64; 1usize] = [a[0usize] * a[1usize]];
6335 mul_level_0[0]
6336 }
6337 #[inline(always)]
6338 fn max_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f64x2<Self> {
6339 [
6340 f64::max(a[0usize], b[0usize]),
6341 f64::max(a[1usize], b[1usize]),
6342 ]
6343 .simd_into(self)
6344 }
6345 #[inline(always)]
6346 fn min_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f64x2<Self> {
6347 [
6348 f64::min(a[0usize], b[0usize]),
6349 f64::min(a[1usize], b[1usize]),
6350 ]
6351 .simd_into(self)
6352 }
6353 #[inline(always)]
6354 fn max_precise_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f64x2<Self> {
6355 [
6356 f64::max(a[0usize], b[0usize]),
6357 f64::max(a[1usize], b[1usize]),
6358 ]
6359 .simd_into(self)
6360 }
6361 #[inline(always)]
6362 fn min_precise_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f64x2<Self> {
6363 [
6364 f64::min(a[0usize], b[0usize]),
6365 f64::min(a[1usize], b[1usize]),
6366 ]
6367 .simd_into(self)
6368 }
6369 #[inline(always)]
6370 fn clamp_f64x2(self, a: f64x2<Self>, b: f64x2<Self>, c: f64x2<Self>) -> f64x2<Self> {
6371 [
6372 f64::clamp_debug(a[0usize], b[0usize], c[0usize]),
6373 f64::clamp_debug(a[1usize], b[1usize], c[1usize]),
6374 ]
6375 .simd_into(self)
6376 }
6377 #[inline(always)]
6378 fn simd_eq_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> mask64x2<Self> {
6379 [
6380 -(f64::eq(&a[0usize], &b[0usize]) as i64),
6381 -(f64::eq(&a[1usize], &b[1usize]) as i64),
6382 ]
6383 .simd_into(self)
6384 }
6385 #[inline(always)]
6386 fn simd_lt_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> mask64x2<Self> {
6387 [
6388 -(f64::lt(&a[0usize], &b[0usize]) as i64),
6389 -(f64::lt(&a[1usize], &b[1usize]) as i64),
6390 ]
6391 .simd_into(self)
6392 }
6393 #[inline(always)]
6394 fn simd_le_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> mask64x2<Self> {
6395 [
6396 -(f64::le(&a[0usize], &b[0usize]) as i64),
6397 -(f64::le(&a[1usize], &b[1usize]) as i64),
6398 ]
6399 .simd_into(self)
6400 }
6401 #[inline(always)]
6402 fn zip_low_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f64x2<Self> {
6403 [a[0usize], b[0usize]].simd_into(self)
6404 }
6405 #[inline(always)]
6406 fn zip_high_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f64x2<Self> {
6407 [a[1usize], b[1usize]].simd_into(self)
6408 }
6409 #[inline(always)]
6410 fn unzip_low_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f64x2<Self> {
6411 [a[0usize], b[0usize]].simd_into(self)
6412 }
6413 #[inline(always)]
6414 fn unzip_high_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f64x2<Self> {
6415 [a[1usize], b[1usize]].simd_into(self)
6416 }
6417 #[inline(always)]
6418 fn interleave_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> (f64x2<Self>, f64x2<Self>) {
6419 (self.zip_low_f64x2(a, b), self.zip_high_f64x2(a, b))
6420 }
6421 #[inline(always)]
6422 fn deinterleave_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> (f64x2<Self>, f64x2<Self>) {
6423 (self.unzip_low_f64x2(a, b), self.unzip_high_f64x2(a, b))
6424 }
6425 #[inline(always)]
6426 fn mul_add_f64x2(self, a: f64x2<Self>, b: f64x2<Self>, c: f64x2<Self>) -> f64x2<Self> {
6427 a.mul(b).add(c)
6428 }
6429 #[inline(always)]
6430 fn mul_add_precise_f64x2(self, a: f64x2<Self>, b: f64x2<Self>, c: f64x2<Self>) -> f64x2<Self> {
6431 crate::math::mul_add_precise_f64x2(self, a, b, c)
6432 }
6433 #[inline(always)]
6434 fn mul_sub_f64x2(self, a: f64x2<Self>, b: f64x2<Self>, c: f64x2<Self>) -> f64x2<Self> {
6435 a.mul(b).sub(c)
6436 }
6437 #[inline(always)]
6438 fn mul_sub_precise_f64x2(self, a: f64x2<Self>, b: f64x2<Self>, c: f64x2<Self>) -> f64x2<Self> {
6439 self.mul_add_precise_f64x2(a, b, -c)
6440 }
6441 #[inline(always)]
6442 fn floor_f64x2(self, a: f64x2<Self>) -> f64x2<Self> {
6443 [f64::floor(a[0usize]), f64::floor(a[1usize])].simd_into(self)
6444 }
6445 #[inline(always)]
6446 fn ceil_f64x2(self, a: f64x2<Self>) -> f64x2<Self> {
6447 [f64::ceil(a[0usize]), f64::ceil(a[1usize])].simd_into(self)
6448 }
6449 #[inline(always)]
6450 fn round_ties_even_f64x2(self, a: f64x2<Self>) -> f64x2<Self> {
6451 [
6452 f64::round_ties_even(a[0usize]),
6453 f64::round_ties_even(a[1usize]),
6454 ]
6455 .simd_into(self)
6456 }
6457 #[inline(always)]
6458 fn round_f64x2(self, a: f64x2<Self>) -> f64x2<Self> {
6459 [f64::round(a[0usize]), f64::round(a[1usize])].simd_into(self)
6460 }
6461 #[inline(always)]
6462 fn fract_f64x2(self, a: f64x2<Self>) -> f64x2<Self> {
6463 [f64::fract(a[0usize]), f64::fract(a[1usize])].simd_into(self)
6464 }
6465 #[inline(always)]
6466 fn trunc_f64x2(self, a: f64x2<Self>) -> f64x2<Self> {
6467 [f64::trunc(a[0usize]), f64::trunc(a[1usize])].simd_into(self)
6468 }
6469 #[inline(always)]
6470 fn to_degrees_f64x2(self, a: f64x2<Self>) -> f64x2<Self> {
6471 [f64::to_degrees(a[0usize]), f64::to_degrees(a[1usize])].simd_into(self)
6472 }
6473 #[inline(always)]
6474 fn to_radians_f64x2(self, a: f64x2<Self>) -> f64x2<Self> {
6475 [f64::to_radians(a[0usize]), f64::to_radians(a[1usize])].simd_into(self)
6476 }
6477 #[inline(always)]
6478 fn is_nan_f64x2(self, a: f64x2<Self>) -> mask64x2<Self> {
6479 a.simd_ne(a)
6480 }
6481 #[inline(always)]
6482 fn is_infinite_f64x2(self, a: f64x2<Self>) -> mask64x2<Self> {
6483 a.abs().simd_eq(f64::INFINITY)
6484 }
6485 #[inline(always)]
6486 fn is_finite_f64x2(self, a: f64x2<Self>) -> mask64x2<Self> {
6487 a.abs().simd_lt(f64::INFINITY)
6488 }
6489 #[inline(always)]
6490 fn is_subnormal_f64x2(self, a: f64x2<Self>) -> mask64x2<Self> {
6491 let i: i64x2<Self> = a.bitcast();
6492 (i & 0x7FF0_0000_0000_0000_i64).simd_eq(0) & (i & 0x000F_FFFF_FFFF_FFFF_i64).simd_ne(0)
6493 }
6494 #[inline(always)]
6495 fn is_normal_f64x2(self, a: f64x2<Self>) -> mask64x2<Self> {
6496 let i: i64x2<Self> = a.bitcast();
6497 let exp = i & 0x7FF0_0000_0000_0000_i64;
6498 exp.simd_ne(0) & exp.simd_ne(0x7FF0_0000_0000_0000_i64)
6499 }
6500 #[inline(always)]
6501 fn is_sign_positive_f64x2(self, a: f64x2<Self>) -> mask64x2<Self> {
6502 let i: i64x2<Self> = a.bitcast();
6503 (i & i64::MIN).simd_eq(0)
6504 }
6505 #[inline(always)]
6506 fn is_sign_negative_f64x2(self, a: f64x2<Self>) -> mask64x2<Self> {
6507 !a.is_sign_positive()
6508 }
6509 #[inline(always)]
6510 fn select_f64x2(self, a: mask64x2<Self>, b: f64x2<Self>, c: f64x2<Self>) -> f64x2<Self> {
6511 [
6512 if a.val.0[0usize] != 0 {
6513 b[0usize]
6514 } else {
6515 c[0usize]
6516 },
6517 if a.val.0[1usize] != 0 {
6518 b[1usize]
6519 } else {
6520 c[1usize]
6521 },
6522 ]
6523 .simd_into(self)
6524 }
6525 #[inline(always)]
6526 fn combine_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f64x4<Self> {
6527 let mut result = [0.0; 4usize];
6528 result[0..2usize].copy_from_slice(&a.val.0);
6529 result[2usize..4usize].copy_from_slice(&b.val.0);
6530 result.simd_into(self)
6531 }
6532 #[inline(always)]
6533 fn load_four_interleaved_f64x2(self, src: &[f64; 8usize]) -> [f64x2<Self>; 4usize] {
6534 [
6535 [src[0usize], src[4usize]].simd_into(self),
6536 [src[1usize], src[5usize]].simd_into(self),
6537 [src[2usize], src[6usize]].simd_into(self),
6538 [src[3usize], src[7usize]].simd_into(self),
6539 ]
6540 }
6541 #[inline(always)]
6542 fn store_four_interleaved_f64x2(
6543 self,
6544 vectors: [f64x2<Self>; 4usize],
6545 dest: &mut [f64; 8usize],
6546 ) -> () {
6547 *dest = [
6548 vectors[0usize][0usize],
6549 vectors[1usize][0usize],
6550 vectors[2usize][0usize],
6551 vectors[3usize][0usize],
6552 vectors[0usize][1usize],
6553 vectors[1usize][1usize],
6554 vectors[2usize][1usize],
6555 vectors[3usize][1usize],
6556 ];
6557 }
6558 #[inline(always)]
6559 fn narrow_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f32x4<Self> {
6560 [
6561 a[0usize] as f32,
6562 a[1usize] as f32,
6563 b[0usize] as f32,
6564 b[1usize] as f32,
6565 ]
6566 .simd_into(self)
6567 }
6568 #[inline(always)]
6569 fn saturating_narrow_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f32x4<Self> {
6570 [
6571 a[0usize] as f32,
6572 a[1usize] as f32,
6573 b[0usize] as f32,
6574 b[1usize] as f32,
6575 ]
6576 .simd_into(self)
6577 }
6578 #[inline(always)]
6579 fn relaxed_narrow_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f32x4<Self> {
6580 self.narrow_f64x2(a, b)
6581 }
6582 #[inline(always)]
6583 fn cvt_u64_f64x2(self, a: f64x2<Self>) -> u64x2<Self> {
6584 [a[0usize] as u64, a[1usize] as u64].simd_into(self)
6585 }
6586 #[inline(always)]
6587 fn cvt_u64_precise_f64x2(self, a: f64x2<Self>) -> u64x2<Self> {
6588 [a[0usize] as u64, a[1usize] as u64].simd_into(self)
6589 }
6590 #[inline(always)]
6591 fn cvt_i64_f64x2(self, a: f64x2<Self>) -> i64x2<Self> {
6592 [a[0usize] as i64, a[1usize] as i64].simd_into(self)
6593 }
6594 #[inline(always)]
6595 fn cvt_i64_precise_f64x2(self, a: f64x2<Self>) -> i64x2<Self> {
6596 [a[0usize] as i64, a[1usize] as i64].simd_into(self)
6597 }
6598 #[inline(always)]
6599 fn abs_i64x2(self, a: i64x2<Self>) -> i64x2<Self> {
6600 [i64::wrapping_abs(a[0usize]), i64::wrapping_abs(a[1usize])].simd_into(self)
6601 }
6602 #[inline(always)]
6603 fn splat_i64x2(self, val: i64) -> i64x2<Self> {
6604 [val; 2usize].simd_into(self)
6605 }
6606 #[inline(always)]
6607 fn reverse_i64x2(self, a: i64x2<Self>) -> i64x2<Self> {
6608 [a[1usize], a[0usize]].simd_into(self)
6609 }
6610 #[inline(always)]
6611 fn slide_i64x2<const SHIFT: usize>(self, a: i64x2<Self>, b: i64x2<Self>) -> i64x2<Self> {
6612 let mut dest = [Default::default(); 2usize];
6613 dest[..2usize - SHIFT].copy_from_slice(&a.val.0[SHIFT..]);
6614 dest[2usize - SHIFT..].copy_from_slice(&b.val.0[..SHIFT]);
6615 dest.simd_into(self)
6616 }
6617 #[inline(always)]
6618 fn compress_i64x2(self, values: i64x2<Self>, mask: mask64x2<Self>) -> i64x2<Self> {
6619 let mask = mask8x16 {
6620 val: crate::transmute::checked_transmute_copy(&mask.val),
6621 simd: self,
6622 };
6623 Bytes::from_bytes(self.compress_u8x16(Bytes::to_bytes(values), mask))
6624 }
6625 #[inline(always)]
6626 fn compress_merge_i64x2(
6627 self,
6628 values: i64x2<Self>,
6629 mask: mask64x2<Self>,
6630 merge: i64x2<Self>,
6631 ) -> i64x2<Self> {
6632 let mask = mask8x16 {
6633 val: crate::transmute::checked_transmute_copy(&mask.val),
6634 simd: self,
6635 };
6636 Bytes::from_bytes(self.compress_merge_u8x16(
6637 Bytes::to_bytes(values),
6638 mask,
6639 Bytes::to_bytes(merge),
6640 ))
6641 }
6642 #[inline(always)]
6643 fn expand_i64x2(self, values: i64x2<Self>, mask: mask64x2<Self>) -> i64x2<Self> {
6644 let mask = mask8x16 {
6645 val: crate::transmute::checked_transmute_copy(&mask.val),
6646 simd: self,
6647 };
6648 Bytes::from_bytes(self.expand_u8x16(Bytes::to_bytes(values), mask))
6649 }
6650 #[inline(always)]
6651 fn expand_merge_i64x2(
6652 self,
6653 values: i64x2<Self>,
6654 mask: mask64x2<Self>,
6655 merge: i64x2<Self>,
6656 ) -> i64x2<Self> {
6657 let mask = mask8x16 {
6658 val: crate::transmute::checked_transmute_copy(&mask.val),
6659 simd: self,
6660 };
6661 Bytes::from_bytes(self.expand_merge_u8x16(
6662 Bytes::to_bytes(values),
6663 mask,
6664 Bytes::to_bytes(merge),
6665 ))
6666 }
6667 #[inline(always)]
6668 fn count_ones_i64x2(self, a: i64x2<Self>) -> i64x2<Self> {
6669 [
6670 i64::from(a[0usize].count_ones()),
6671 i64::from(a[1usize].count_ones()),
6672 ]
6673 .simd_into(self)
6674 }
6675 #[inline(always)]
6676 fn count_zeros_i64x2(self, a: i64x2<Self>) -> i64x2<Self> {
6677 [
6678 i64::from(a[0usize].count_zeros()),
6679 i64::from(a[1usize].count_zeros()),
6680 ]
6681 .simd_into(self)
6682 }
6683 #[inline(always)]
6684 fn add_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> i64x2<Self> {
6685 [
6686 i64::wrapping_add(a[0usize], b[0usize]),
6687 i64::wrapping_add(a[1usize], b[1usize]),
6688 ]
6689 .simd_into(self)
6690 }
6691 #[inline(always)]
6692 fn saturating_add_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> i64x2<Self> {
6693 [
6694 i64::saturating_add(a[0usize], b[0usize]),
6695 i64::saturating_add(a[1usize], b[1usize]),
6696 ]
6697 .simd_into(self)
6698 }
6699 #[inline(always)]
6700 fn sub_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> i64x2<Self> {
6701 [
6702 i64::wrapping_sub(a[0usize], b[0usize]),
6703 i64::wrapping_sub(a[1usize], b[1usize]),
6704 ]
6705 .simd_into(self)
6706 }
6707 #[inline(always)]
6708 fn saturating_sub_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> i64x2<Self> {
6709 [
6710 i64::saturating_sub(a[0usize], b[0usize]),
6711 i64::saturating_sub(a[1usize], b[1usize]),
6712 ]
6713 .simd_into(self)
6714 }
6715 #[inline(always)]
6716 fn mul_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> i64x2<Self> {
6717 [
6718 i64::wrapping_mul(a[0usize], b[0usize]),
6719 i64::wrapping_mul(a[1usize], b[1usize]),
6720 ]
6721 .simd_into(self)
6722 }
6723 #[inline(always)]
6724 fn and_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> i64x2<Self> {
6725 [
6726 i64::bitand(a[0usize], &b[0usize]),
6727 i64::bitand(a[1usize], &b[1usize]),
6728 ]
6729 .simd_into(self)
6730 }
6731 #[inline(always)]
6732 fn or_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> i64x2<Self> {
6733 [
6734 i64::bitor(a[0usize], &b[0usize]),
6735 i64::bitor(a[1usize], &b[1usize]),
6736 ]
6737 .simd_into(self)
6738 }
6739 #[inline(always)]
6740 fn xor_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> i64x2<Self> {
6741 [
6742 i64::bitxor(a[0usize], &b[0usize]),
6743 i64::bitxor(a[1usize], &b[1usize]),
6744 ]
6745 .simd_into(self)
6746 }
6747 #[inline(always)]
6748 fn not_i64x2(self, a: i64x2<Self>) -> i64x2<Self> {
6749 [i64::not(a[0usize]), i64::not(a[1usize])].simd_into(self)
6750 }
6751 #[inline(always)]
6752 fn shl_i64x2(self, a: i64x2<Self>, shift: u32) -> i64x2<Self> {
6753 [
6754 i64::wrapping_shl(a[0usize], shift),
6755 i64::wrapping_shl(a[1usize], shift),
6756 ]
6757 .simd_into(self)
6758 }
6759 #[inline(always)]
6760 fn shlv_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> i64x2<Self> {
6761 [
6762 i64::wrapping_shl(a[0usize], b[0usize] as u32),
6763 i64::wrapping_shl(a[1usize], b[1usize] as u32),
6764 ]
6765 .simd_into(self)
6766 }
6767 #[inline(always)]
6768 fn shr_i64x2(self, a: i64x2<Self>, shift: u32) -> i64x2<Self> {
6769 [
6770 i64::wrapping_shr(a[0usize], shift),
6771 i64::wrapping_shr(a[1usize], shift),
6772 ]
6773 .simd_into(self)
6774 }
6775 #[inline(always)]
6776 fn shrv_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> i64x2<Self> {
6777 [
6778 i64::wrapping_shr(a[0usize], b[0usize] as u32),
6779 i64::wrapping_shr(a[1usize], b[1usize] as u32),
6780 ]
6781 .simd_into(self)
6782 }
6783 #[inline(always)]
6784 fn reduce_max_i64x2(self, a: i64x2<Self>) -> i64 {
6785 let reduced: [i64; 1usize] = [i64::max(a[0usize], a[1usize])];
6786 reduced[0]
6787 }
6788 #[inline(always)]
6789 fn reduce_min_i64x2(self, a: i64x2<Self>) -> i64 {
6790 let reduced: [i64; 1usize] = [i64::min(a[0usize], a[1usize])];
6791 reduced[0]
6792 }
6793 #[inline(always)]
6794 fn reduce_sum_i64x2(self, a: i64x2<Self>) -> i64 {
6795 let add_level_0: [i64; 1usize] = [a[0usize].wrapping_add(a[1usize])];
6796 add_level_0[0]
6797 }
6798 #[inline(always)]
6799 fn reduce_product_i64x2(self, a: i64x2<Self>) -> i64 {
6800 let mul_level_0: [i64; 1usize] = [a[0usize].wrapping_mul(a[1usize])];
6801 mul_level_0[0]
6802 }
6803 #[inline(always)]
6804 fn max_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> i64x2<Self> {
6805 [
6806 i64::max(a[0usize], b[0usize]),
6807 i64::max(a[1usize], b[1usize]),
6808 ]
6809 .simd_into(self)
6810 }
6811 #[inline(always)]
6812 fn min_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> i64x2<Self> {
6813 [
6814 i64::min(a[0usize], b[0usize]),
6815 i64::min(a[1usize], b[1usize]),
6816 ]
6817 .simd_into(self)
6818 }
6819 #[inline(always)]
6820 fn clamp_i64x2(self, a: i64x2<Self>, b: i64x2<Self>, c: i64x2<Self>) -> i64x2<Self> {
6821 [
6822 i64::clamp_debug(a[0usize], b[0usize], c[0usize]),
6823 i64::clamp_debug(a[1usize], b[1usize], c[1usize]),
6824 ]
6825 .simd_into(self)
6826 }
6827 #[inline(always)]
6828 fn simd_eq_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> mask64x2<Self> {
6829 [
6830 -(i64::eq(&a[0usize], &b[0usize]) as i64),
6831 -(i64::eq(&a[1usize], &b[1usize]) as i64),
6832 ]
6833 .simd_into(self)
6834 }
6835 #[inline(always)]
6836 fn simd_lt_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> mask64x2<Self> {
6837 [
6838 -(i64::lt(&a[0usize], &b[0usize]) as i64),
6839 -(i64::lt(&a[1usize], &b[1usize]) as i64),
6840 ]
6841 .simd_into(self)
6842 }
6843 #[inline(always)]
6844 fn simd_le_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> mask64x2<Self> {
6845 [
6846 -(i64::le(&a[0usize], &b[0usize]) as i64),
6847 -(i64::le(&a[1usize], &b[1usize]) as i64),
6848 ]
6849 .simd_into(self)
6850 }
6851 #[inline(always)]
6852 fn zip_low_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> i64x2<Self> {
6853 [a[0usize], b[0usize]].simd_into(self)
6854 }
6855 #[inline(always)]
6856 fn zip_high_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> i64x2<Self> {
6857 [a[1usize], b[1usize]].simd_into(self)
6858 }
6859 #[inline(always)]
6860 fn unzip_low_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> i64x2<Self> {
6861 [a[0usize], b[0usize]].simd_into(self)
6862 }
6863 #[inline(always)]
6864 fn unzip_high_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> i64x2<Self> {
6865 [a[1usize], b[1usize]].simd_into(self)
6866 }
6867 #[inline(always)]
6868 fn interleave_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> (i64x2<Self>, i64x2<Self>) {
6869 (self.zip_low_i64x2(a, b), self.zip_high_i64x2(a, b))
6870 }
6871 #[inline(always)]
6872 fn deinterleave_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> (i64x2<Self>, i64x2<Self>) {
6873 (self.unzip_low_i64x2(a, b), self.unzip_high_i64x2(a, b))
6874 }
6875 #[inline(always)]
6876 fn select_i64x2(self, a: mask64x2<Self>, b: i64x2<Self>, c: i64x2<Self>) -> i64x2<Self> {
6877 [
6878 if a.val.0[0usize] != 0 {
6879 b[0usize]
6880 } else {
6881 c[0usize]
6882 },
6883 if a.val.0[1usize] != 0 {
6884 b[1usize]
6885 } else {
6886 c[1usize]
6887 },
6888 ]
6889 .simd_into(self)
6890 }
6891 #[inline(always)]
6892 fn combine_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> i64x4<Self> {
6893 let mut result = [0; 4usize];
6894 result[0..2usize].copy_from_slice(&a.val.0);
6895 result[2usize..4usize].copy_from_slice(&b.val.0);
6896 result.simd_into(self)
6897 }
6898 #[inline(always)]
6899 fn neg_i64x2(self, a: i64x2<Self>) -> i64x2<Self> {
6900 [i64::wrapping_neg(a[0usize]), i64::wrapping_neg(a[1usize])].simd_into(self)
6901 }
6902 #[inline(always)]
6903 fn load_four_interleaved_i64x2(self, src: &[i64; 8usize]) -> [i64x2<Self>; 4usize] {
6904 [
6905 [src[0usize], src[4usize]].simd_into(self),
6906 [src[1usize], src[5usize]].simd_into(self),
6907 [src[2usize], src[6usize]].simd_into(self),
6908 [src[3usize], src[7usize]].simd_into(self),
6909 ]
6910 }
6911 #[inline(always)]
6912 fn store_four_interleaved_i64x2(
6913 self,
6914 vectors: [i64x2<Self>; 4usize],
6915 dest: &mut [i64; 8usize],
6916 ) -> () {
6917 *dest = [
6918 vectors[0usize][0usize],
6919 vectors[1usize][0usize],
6920 vectors[2usize][0usize],
6921 vectors[3usize][0usize],
6922 vectors[0usize][1usize],
6923 vectors[1usize][1usize],
6924 vectors[2usize][1usize],
6925 vectors[3usize][1usize],
6926 ];
6927 }
6928 #[inline(always)]
6929 fn narrow_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> i32x4<Self> {
6930 [
6931 a[0usize] as i32,
6932 a[1usize] as i32,
6933 b[0usize] as i32,
6934 b[1usize] as i32,
6935 ]
6936 .simd_into(self)
6937 }
6938 #[inline(always)]
6939 fn saturating_narrow_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> i32x4<Self> {
6940 [
6941 a[0usize].clamp(i32::MIN as i64, i32::MAX as i64) as i32,
6942 a[1usize].clamp(i32::MIN as i64, i32::MAX as i64) as i32,
6943 b[0usize].clamp(i32::MIN as i64, i32::MAX as i64) as i32,
6944 b[1usize].clamp(i32::MIN as i64, i32::MAX as i64) as i32,
6945 ]
6946 .simd_into(self)
6947 }
6948 #[inline(always)]
6949 fn relaxed_narrow_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> i32x4<Self> {
6950 debug_assert!(
6951 a.as_slice()
6952 .iter()
6953 .chain(b.as_slice())
6954 .all(|&value| { value >= i32::MIN as i64 && value <= i32::MAX as i64 }),
6955 "relaxed_narrow inputs must fit in the destination type",
6956 );
6957 self.narrow_i64x2(a, b)
6958 }
6959 #[inline(always)]
6960 fn cvt_f64_i64x2(self, a: i64x2<Self>) -> f64x2<Self> {
6961 [a[0usize] as f64, a[1usize] as f64].simd_into(self)
6962 }
6963 #[inline(always)]
6964 fn splat_u64x2(self, val: u64) -> u64x2<Self> {
6965 [val; 2usize].simd_into(self)
6966 }
6967 #[inline(always)]
6968 fn reverse_u64x2(self, a: u64x2<Self>) -> u64x2<Self> {
6969 [a[1usize], a[0usize]].simd_into(self)
6970 }
6971 #[inline(always)]
6972 fn slide_u64x2<const SHIFT: usize>(self, a: u64x2<Self>, b: u64x2<Self>) -> u64x2<Self> {
6973 let mut dest = [Default::default(); 2usize];
6974 dest[..2usize - SHIFT].copy_from_slice(&a.val.0[SHIFT..]);
6975 dest[2usize - SHIFT..].copy_from_slice(&b.val.0[..SHIFT]);
6976 dest.simd_into(self)
6977 }
6978 #[inline(always)]
6979 fn compress_u64x2(self, values: u64x2<Self>, mask: mask64x2<Self>) -> u64x2<Self> {
6980 let mask = mask8x16 {
6981 val: crate::transmute::checked_transmute_copy(&mask.val),
6982 simd: self,
6983 };
6984 Bytes::from_bytes(self.compress_u8x16(Bytes::to_bytes(values), mask))
6985 }
6986 #[inline(always)]
6987 fn compress_merge_u64x2(
6988 self,
6989 values: u64x2<Self>,
6990 mask: mask64x2<Self>,
6991 merge: u64x2<Self>,
6992 ) -> u64x2<Self> {
6993 let mask = mask8x16 {
6994 val: crate::transmute::checked_transmute_copy(&mask.val),
6995 simd: self,
6996 };
6997 Bytes::from_bytes(self.compress_merge_u8x16(
6998 Bytes::to_bytes(values),
6999 mask,
7000 Bytes::to_bytes(merge),
7001 ))
7002 }
7003 #[inline(always)]
7004 fn expand_u64x2(self, values: u64x2<Self>, mask: mask64x2<Self>) -> u64x2<Self> {
7005 let mask = mask8x16 {
7006 val: crate::transmute::checked_transmute_copy(&mask.val),
7007 simd: self,
7008 };
7009 Bytes::from_bytes(self.expand_u8x16(Bytes::to_bytes(values), mask))
7010 }
7011 #[inline(always)]
7012 fn expand_merge_u64x2(
7013 self,
7014 values: u64x2<Self>,
7015 mask: mask64x2<Self>,
7016 merge: u64x2<Self>,
7017 ) -> u64x2<Self> {
7018 let mask = mask8x16 {
7019 val: crate::transmute::checked_transmute_copy(&mask.val),
7020 simd: self,
7021 };
7022 Bytes::from_bytes(self.expand_merge_u8x16(
7023 Bytes::to_bytes(values),
7024 mask,
7025 Bytes::to_bytes(merge),
7026 ))
7027 }
7028 #[inline(always)]
7029 fn count_ones_u64x2(self, a: u64x2<Self>) -> u64x2<Self> {
7030 [
7031 u64::from(a[0usize].count_ones()),
7032 u64::from(a[1usize].count_ones()),
7033 ]
7034 .simd_into(self)
7035 }
7036 #[inline(always)]
7037 fn count_zeros_u64x2(self, a: u64x2<Self>) -> u64x2<Self> {
7038 [
7039 u64::from(a[0usize].count_zeros()),
7040 u64::from(a[1usize].count_zeros()),
7041 ]
7042 .simd_into(self)
7043 }
7044 #[inline(always)]
7045 fn add_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> u64x2<Self> {
7046 [
7047 u64::wrapping_add(a[0usize], b[0usize]),
7048 u64::wrapping_add(a[1usize], b[1usize]),
7049 ]
7050 .simd_into(self)
7051 }
7052 #[inline(always)]
7053 fn saturating_add_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> u64x2<Self> {
7054 [
7055 u64::saturating_add(a[0usize], b[0usize]),
7056 u64::saturating_add(a[1usize], b[1usize]),
7057 ]
7058 .simd_into(self)
7059 }
7060 #[inline(always)]
7061 fn sub_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> u64x2<Self> {
7062 [
7063 u64::wrapping_sub(a[0usize], b[0usize]),
7064 u64::wrapping_sub(a[1usize], b[1usize]),
7065 ]
7066 .simd_into(self)
7067 }
7068 #[inline(always)]
7069 fn saturating_sub_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> u64x2<Self> {
7070 [
7071 u64::saturating_sub(a[0usize], b[0usize]),
7072 u64::saturating_sub(a[1usize], b[1usize]),
7073 ]
7074 .simd_into(self)
7075 }
7076 #[inline(always)]
7077 fn mul_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> u64x2<Self> {
7078 [
7079 u64::wrapping_mul(a[0usize], b[0usize]),
7080 u64::wrapping_mul(a[1usize], b[1usize]),
7081 ]
7082 .simd_into(self)
7083 }
7084 #[inline(always)]
7085 fn and_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> u64x2<Self> {
7086 [
7087 u64::bitand(a[0usize], &b[0usize]),
7088 u64::bitand(a[1usize], &b[1usize]),
7089 ]
7090 .simd_into(self)
7091 }
7092 #[inline(always)]
7093 fn or_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> u64x2<Self> {
7094 [
7095 u64::bitor(a[0usize], &b[0usize]),
7096 u64::bitor(a[1usize], &b[1usize]),
7097 ]
7098 .simd_into(self)
7099 }
7100 #[inline(always)]
7101 fn xor_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> u64x2<Self> {
7102 [
7103 u64::bitxor(a[0usize], &b[0usize]),
7104 u64::bitxor(a[1usize], &b[1usize]),
7105 ]
7106 .simd_into(self)
7107 }
7108 #[inline(always)]
7109 fn not_u64x2(self, a: u64x2<Self>) -> u64x2<Self> {
7110 [u64::not(a[0usize]), u64::not(a[1usize])].simd_into(self)
7111 }
7112 #[inline(always)]
7113 fn shl_u64x2(self, a: u64x2<Self>, shift: u32) -> u64x2<Self> {
7114 [
7115 u64::wrapping_shl(a[0usize], shift),
7116 u64::wrapping_shl(a[1usize], shift),
7117 ]
7118 .simd_into(self)
7119 }
7120 #[inline(always)]
7121 fn shlv_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> u64x2<Self> {
7122 [
7123 u64::wrapping_shl(a[0usize], b[0usize] as u32),
7124 u64::wrapping_shl(a[1usize], b[1usize] as u32),
7125 ]
7126 .simd_into(self)
7127 }
7128 #[inline(always)]
7129 fn shr_u64x2(self, a: u64x2<Self>, shift: u32) -> u64x2<Self> {
7130 [
7131 u64::wrapping_shr(a[0usize], shift),
7132 u64::wrapping_shr(a[1usize], shift),
7133 ]
7134 .simd_into(self)
7135 }
7136 #[inline(always)]
7137 fn shrv_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> u64x2<Self> {
7138 [
7139 u64::wrapping_shr(a[0usize], b[0usize] as u32),
7140 u64::wrapping_shr(a[1usize], b[1usize] as u32),
7141 ]
7142 .simd_into(self)
7143 }
7144 #[inline(always)]
7145 fn reduce_max_u64x2(self, a: u64x2<Self>) -> u64 {
7146 let reduced: [u64; 1usize] = [u64::max(a[0usize], a[1usize])];
7147 reduced[0]
7148 }
7149 #[inline(always)]
7150 fn reduce_min_u64x2(self, a: u64x2<Self>) -> u64 {
7151 let reduced: [u64; 1usize] = [u64::min(a[0usize], a[1usize])];
7152 reduced[0]
7153 }
7154 #[inline(always)]
7155 fn reduce_sum_u64x2(self, a: u64x2<Self>) -> u64 {
7156 let add_level_0: [u64; 1usize] = [a[0usize].wrapping_add(a[1usize])];
7157 add_level_0[0]
7158 }
7159 #[inline(always)]
7160 fn reduce_product_u64x2(self, a: u64x2<Self>) -> u64 {
7161 let mul_level_0: [u64; 1usize] = [a[0usize].wrapping_mul(a[1usize])];
7162 mul_level_0[0]
7163 }
7164 #[inline(always)]
7165 fn max_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> u64x2<Self> {
7166 [
7167 u64::max(a[0usize], b[0usize]),
7168 u64::max(a[1usize], b[1usize]),
7169 ]
7170 .simd_into(self)
7171 }
7172 #[inline(always)]
7173 fn min_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> u64x2<Self> {
7174 [
7175 u64::min(a[0usize], b[0usize]),
7176 u64::min(a[1usize], b[1usize]),
7177 ]
7178 .simd_into(self)
7179 }
7180 #[inline(always)]
7181 fn clamp_u64x2(self, a: u64x2<Self>, b: u64x2<Self>, c: u64x2<Self>) -> u64x2<Self> {
7182 [
7183 u64::clamp_debug(a[0usize], b[0usize], c[0usize]),
7184 u64::clamp_debug(a[1usize], b[1usize], c[1usize]),
7185 ]
7186 .simd_into(self)
7187 }
7188 #[inline(always)]
7189 fn simd_eq_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> mask64x2<Self> {
7190 [
7191 -(u64::eq(&a[0usize], &b[0usize]) as i64),
7192 -(u64::eq(&a[1usize], &b[1usize]) as i64),
7193 ]
7194 .simd_into(self)
7195 }
7196 #[inline(always)]
7197 fn simd_lt_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> mask64x2<Self> {
7198 [
7199 -(u64::lt(&a[0usize], &b[0usize]) as i64),
7200 -(u64::lt(&a[1usize], &b[1usize]) as i64),
7201 ]
7202 .simd_into(self)
7203 }
7204 #[inline(always)]
7205 fn simd_le_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> mask64x2<Self> {
7206 [
7207 -(u64::le(&a[0usize], &b[0usize]) as i64),
7208 -(u64::le(&a[1usize], &b[1usize]) as i64),
7209 ]
7210 .simd_into(self)
7211 }
7212 #[inline(always)]
7213 fn zip_low_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> u64x2<Self> {
7214 [a[0usize], b[0usize]].simd_into(self)
7215 }
7216 #[inline(always)]
7217 fn zip_high_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> u64x2<Self> {
7218 [a[1usize], b[1usize]].simd_into(self)
7219 }
7220 #[inline(always)]
7221 fn unzip_low_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> u64x2<Self> {
7222 [a[0usize], b[0usize]].simd_into(self)
7223 }
7224 #[inline(always)]
7225 fn unzip_high_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> u64x2<Self> {
7226 [a[1usize], b[1usize]].simd_into(self)
7227 }
7228 #[inline(always)]
7229 fn interleave_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> (u64x2<Self>, u64x2<Self>) {
7230 (self.zip_low_u64x2(a, b), self.zip_high_u64x2(a, b))
7231 }
7232 #[inline(always)]
7233 fn deinterleave_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> (u64x2<Self>, u64x2<Self>) {
7234 (self.unzip_low_u64x2(a, b), self.unzip_high_u64x2(a, b))
7235 }
7236 #[inline(always)]
7237 fn select_u64x2(self, a: mask64x2<Self>, b: u64x2<Self>, c: u64x2<Self>) -> u64x2<Self> {
7238 [
7239 if a.val.0[0usize] != 0 {
7240 b[0usize]
7241 } else {
7242 c[0usize]
7243 },
7244 if a.val.0[1usize] != 0 {
7245 b[1usize]
7246 } else {
7247 c[1usize]
7248 },
7249 ]
7250 .simd_into(self)
7251 }
7252 #[inline(always)]
7253 fn combine_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> u64x4<Self> {
7254 let mut result = [0; 4usize];
7255 result[0..2usize].copy_from_slice(&a.val.0);
7256 result[2usize..4usize].copy_from_slice(&b.val.0);
7257 result.simd_into(self)
7258 }
7259 #[inline(always)]
7260 fn load_four_interleaved_u64x2(self, src: &[u64; 8usize]) -> [u64x2<Self>; 4usize] {
7261 [
7262 [src[0usize], src[4usize]].simd_into(self),
7263 [src[1usize], src[5usize]].simd_into(self),
7264 [src[2usize], src[6usize]].simd_into(self),
7265 [src[3usize], src[7usize]].simd_into(self),
7266 ]
7267 }
7268 #[inline(always)]
7269 fn store_four_interleaved_u64x2(
7270 self,
7271 vectors: [u64x2<Self>; 4usize],
7272 dest: &mut [u64; 8usize],
7273 ) -> () {
7274 *dest = [
7275 vectors[0usize][0usize],
7276 vectors[1usize][0usize],
7277 vectors[2usize][0usize],
7278 vectors[3usize][0usize],
7279 vectors[0usize][1usize],
7280 vectors[1usize][1usize],
7281 vectors[2usize][1usize],
7282 vectors[3usize][1usize],
7283 ];
7284 }
7285 #[inline(always)]
7286 fn narrow_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> u32x4<Self> {
7287 [
7288 a[0usize] as u32,
7289 a[1usize] as u32,
7290 b[0usize] as u32,
7291 b[1usize] as u32,
7292 ]
7293 .simd_into(self)
7294 }
7295 #[inline(always)]
7296 fn saturating_narrow_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> u32x4<Self> {
7297 [
7298 a[0usize].clamp(u32::MIN as u64, u32::MAX as u64) as u32,
7299 a[1usize].clamp(u32::MIN as u64, u32::MAX as u64) as u32,
7300 b[0usize].clamp(u32::MIN as u64, u32::MAX as u64) as u32,
7301 b[1usize].clamp(u32::MIN as u64, u32::MAX as u64) as u32,
7302 ]
7303 .simd_into(self)
7304 }
7305 #[inline(always)]
7306 fn relaxed_narrow_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> u32x4<Self> {
7307 debug_assert!(
7308 a.as_slice()
7309 .iter()
7310 .chain(b.as_slice())
7311 .all(|&value| { value >= u32::MIN as u64 && value <= u32::MAX as u64 }),
7312 "relaxed_narrow inputs must fit in the destination type",
7313 );
7314 self.narrow_u64x2(a, b)
7315 }
7316 #[inline(always)]
7317 fn cvt_f64_u64x2(self, a: u64x2<Self>) -> f64x2<Self> {
7318 [a[0usize] as f64, a[1usize] as f64].simd_into(self)
7319 }
7320 #[inline(always)]
7321 fn splat_mask64x2(self, val: bool) -> mask64x2<Self> {
7322 let val: i64 = if val { !0 } else { 0 };
7323 [val; 2usize].simd_into(self)
7324 }
7325 #[inline(always)]
7326 fn from_bitmask_mask64x2(self, bits: u64) -> mask64x2<Self> {
7327 let lanes: [i64; 2usize] = [
7328 if bits & 1 != 0 { !0 } else { 0 },
7329 if (bits >> 1usize) & 1 != 0 { !0 } else { 0 },
7330 ];
7331 lanes.simd_into(self)
7332 }
7333 #[inline(always)]
7334 fn to_bitmask_mask64x2(self, a: mask64x2<Self>) -> u64 {
7335 let lanes: [i64; 2usize] = a.into();
7336 let mut bits = 0u64;
7337 let mut i = 0;
7338 while i < 2usize {
7339 if lanes[i] != 0 {
7340 bits |= 1u64 << i;
7341 }
7342 i += 1;
7343 }
7344 bits
7345 }
7346 #[inline(always)]
7347 fn from_vector_mask64x2(self, vector: i64x2<Self>) -> mask64x2<Self> {
7348 mask64x2 {
7349 val: vector.val,
7350 simd: self,
7351 }
7352 }
7353 #[inline(always)]
7354 fn to_vector_mask64x2(self, a: mask64x2<Self>) -> i64x2<Self> {
7355 i64x2 {
7356 val: a.val,
7357 simd: self,
7358 }
7359 }
7360 #[inline(always)]
7361 fn set_mask64x2(self, a: &mut mask64x2<Self>, index: usize, value: bool) -> () {
7362 assert!(
7363 index < 2usize,
7364 "mask lane index {index} is out of bounds for {} lanes",
7365 2usize
7366 );
7367 let mut lanes: [i64; 2usize] = (*a).into();
7368 lanes[index] = if value { !0 } else { 0 };
7369 *a = lanes.simd_into(self);
7370 }
7371 #[inline(always)]
7372 fn rotate_elements_left_mask64x2<const OFFSET: usize>(
7373 self,
7374 a: mask64x2<Self>,
7375 ) -> mask64x2<Self> {
7376 let int = i64x2 {
7377 val: crate::transmute::checked_transmute_copy(&a.val),
7378 simd: self,
7379 };
7380 let rotated = int.rotate_elements_left::<OFFSET>();
7381 mask64x2 {
7382 val: crate::transmute::checked_transmute_copy(&rotated.val),
7383 simd: self,
7384 }
7385 }
7386 #[inline(always)]
7387 fn rotate_elements_right_mask64x2<const OFFSET: usize>(
7388 self,
7389 a: mask64x2<Self>,
7390 ) -> mask64x2<Self> {
7391 let int = i64x2 {
7392 val: crate::transmute::checked_transmute_copy(&a.val),
7393 simd: self,
7394 };
7395 let rotated = int.rotate_elements_right::<OFFSET>();
7396 mask64x2 {
7397 val: crate::transmute::checked_transmute_copy(&rotated.val),
7398 simd: self,
7399 }
7400 }
7401 #[inline(always)]
7402 fn and_mask64x2(self, a: mask64x2<Self>, b: mask64x2<Self>) -> mask64x2<Self> {
7403 [
7404 i64::bitand(a.val.0[0usize], &b.val.0[0usize]),
7405 i64::bitand(a.val.0[1usize], &b.val.0[1usize]),
7406 ]
7407 .simd_into(self)
7408 }
7409 #[inline(always)]
7410 fn or_mask64x2(self, a: mask64x2<Self>, b: mask64x2<Self>) -> mask64x2<Self> {
7411 [
7412 i64::bitor(a.val.0[0usize], &b.val.0[0usize]),
7413 i64::bitor(a.val.0[1usize], &b.val.0[1usize]),
7414 ]
7415 .simd_into(self)
7416 }
7417 #[inline(always)]
7418 fn xor_mask64x2(self, a: mask64x2<Self>, b: mask64x2<Self>) -> mask64x2<Self> {
7419 [
7420 i64::bitxor(a.val.0[0usize], &b.val.0[0usize]),
7421 i64::bitxor(a.val.0[1usize], &b.val.0[1usize]),
7422 ]
7423 .simd_into(self)
7424 }
7425 #[inline(always)]
7426 fn not_mask64x2(self, a: mask64x2<Self>) -> mask64x2<Self> {
7427 [i64::not(a.val.0[0usize]), i64::not(a.val.0[1usize])].simd_into(self)
7428 }
7429 #[inline(always)]
7430 fn reverse_mask64x2(self, a: mask64x2<Self>) -> mask64x2<Self> {
7431 [a.val.0[1usize], a.val.0[0usize]].simd_into(self)
7432 }
7433 #[inline(always)]
7434 fn select_mask64x2(
7435 self,
7436 a: mask64x2<Self>,
7437 b: mask64x2<Self>,
7438 c: mask64x2<Self>,
7439 ) -> mask64x2<Self> {
7440 [
7441 if a.val.0[0usize] != 0 {
7442 b.val.0[0usize]
7443 } else {
7444 c.val.0[0usize]
7445 },
7446 if a.val.0[1usize] != 0 {
7447 b.val.0[1usize]
7448 } else {
7449 c.val.0[1usize]
7450 },
7451 ]
7452 .simd_into(self)
7453 }
7454 #[inline(always)]
7455 fn simd_eq_mask64x2(self, a: mask64x2<Self>, b: mask64x2<Self>) -> mask64x2<Self> {
7456 [
7457 -(i64::eq(&a.val.0[0usize], &b.val.0[0usize]) as i64),
7458 -(i64::eq(&a.val.0[1usize], &b.val.0[1usize]) as i64),
7459 ]
7460 .simd_into(self)
7461 }
7462 #[inline(always)]
7463 fn any_true_mask64x2(self, a: mask64x2<Self>) -> bool {
7464 a.val.0[0usize] != 0 || a.val.0[1usize] != 0
7465 }
7466 #[inline(always)]
7467 fn all_true_mask64x2(self, a: mask64x2<Self>) -> bool {
7468 a.val.0[0usize] != 0 && a.val.0[1usize] != 0
7469 }
7470 #[inline(always)]
7471 fn any_false_mask64x2(self, a: mask64x2<Self>) -> bool {
7472 a.val.0[0usize] == 0 || a.val.0[1usize] == 0
7473 }
7474 #[inline(always)]
7475 fn all_false_mask64x2(self, a: mask64x2<Self>) -> bool {
7476 a.val.0[0usize] == 0 && a.val.0[1usize] == 0
7477 }
7478 #[inline(always)]
7479 fn combine_mask64x2(self, a: mask64x2<Self>, b: mask64x2<Self>) -> mask64x4<Self> {
7480 let mut result = [0; 4usize];
7481 result[0..2usize].copy_from_slice(&a.val.0);
7482 result[2usize..4usize].copy_from_slice(&b.val.0);
7483 result.simd_into(self)
7484 }
7485 #[inline(always)]
7486 fn narrow_mask64x2(self, a: mask64x2<Self>, b: mask64x2<Self>) -> mask32x4<Self> {
7487 [
7488 a.val.0[0usize] as i32,
7489 a.val.0[1usize] as i32,
7490 b.val.0[0usize] as i32,
7491 b.val.0[1usize] as i32,
7492 ]
7493 .simd_into(self)
7494 }
7495 #[inline(always)]
7496 fn slide_f32x8<const SHIFT: usize>(self, a: f32x8<Self>, b: f32x8<Self>) -> f32x8<Self> {
7497 let mut dest = [Default::default(); 8usize];
7498 dest[..8usize - SHIFT].copy_from_slice(&a.val.0[SHIFT..]);
7499 dest[8usize - SHIFT..].copy_from_slice(&b.val.0[..SHIFT]);
7500 dest.simd_into(self)
7501 }
7502 #[inline(always)]
7503 fn compress_f32x8(self, values: f32x8<Self>, mask: mask32x8<Self>) -> f32x8<Self> {
7504 let mask = mask8x32 {
7505 val: crate::transmute::checked_transmute_copy(&mask.val),
7506 simd: self,
7507 };
7508 Bytes::from_bytes(self.compress_u8x32(Bytes::to_bytes(values), mask))
7509 }
7510 #[inline(always)]
7511 fn compress_merge_f32x8(
7512 self,
7513 values: f32x8<Self>,
7514 mask: mask32x8<Self>,
7515 merge: f32x8<Self>,
7516 ) -> f32x8<Self> {
7517 let mask = mask8x32 {
7518 val: crate::transmute::checked_transmute_copy(&mask.val),
7519 simd: self,
7520 };
7521 Bytes::from_bytes(self.compress_merge_u8x32(
7522 Bytes::to_bytes(values),
7523 mask,
7524 Bytes::to_bytes(merge),
7525 ))
7526 }
7527 #[inline(always)]
7528 fn expand_f32x8(self, values: f32x8<Self>, mask: mask32x8<Self>) -> f32x8<Self> {
7529 let mask = mask8x32 {
7530 val: crate::transmute::checked_transmute_copy(&mask.val),
7531 simd: self,
7532 };
7533 Bytes::from_bytes(self.expand_u8x32(Bytes::to_bytes(values), mask))
7534 }
7535 #[inline(always)]
7536 fn expand_merge_f32x8(
7537 self,
7538 values: f32x8<Self>,
7539 mask: mask32x8<Self>,
7540 merge: f32x8<Self>,
7541 ) -> f32x8<Self> {
7542 let mask = mask8x32 {
7543 val: crate::transmute::checked_transmute_copy(&mask.val),
7544 simd: self,
7545 };
7546 Bytes::from_bytes(self.expand_merge_u8x32(
7547 Bytes::to_bytes(values),
7548 mask,
7549 Bytes::to_bytes(merge),
7550 ))
7551 }
7552 #[inline(always)]
7553 fn is_nan_f32x8(self, a: f32x8<Self>) -> mask32x8<Self> {
7554 a.simd_ne(a)
7555 }
7556 #[inline(always)]
7557 fn is_infinite_f32x8(self, a: f32x8<Self>) -> mask32x8<Self> {
7558 a.abs().simd_eq(f32::INFINITY)
7559 }
7560 #[inline(always)]
7561 fn is_finite_f32x8(self, a: f32x8<Self>) -> mask32x8<Self> {
7562 a.abs().simd_lt(f32::INFINITY)
7563 }
7564 #[inline(always)]
7565 fn is_subnormal_f32x8(self, a: f32x8<Self>) -> mask32x8<Self> {
7566 let i: i32x8<Self> = a.bitcast();
7567 (i & 0x7F80_0000_i32).simd_eq(0) & (i & 0x007F_FFFF_i32).simd_ne(0)
7568 }
7569 #[inline(always)]
7570 fn is_normal_f32x8(self, a: f32x8<Self>) -> mask32x8<Self> {
7571 let i: i32x8<Self> = a.bitcast();
7572 let exp = i & 0x7F80_0000_i32;
7573 exp.simd_ne(0) & exp.simd_ne(0x7F80_0000_i32)
7574 }
7575 #[inline(always)]
7576 fn is_sign_positive_f32x8(self, a: f32x8<Self>) -> mask32x8<Self> {
7577 let i: i32x8<Self> = a.bitcast();
7578 (i & i32::MIN).simd_eq(0)
7579 }
7580 #[inline(always)]
7581 fn is_sign_negative_f32x8(self, a: f32x8<Self>) -> mask32x8<Self> {
7582 !a.is_sign_positive()
7583 }
7584 #[inline(always)]
7585 fn combine_f32x8(self, a: f32x8<Self>, b: f32x8<Self>) -> f32x16<Self> {
7586 let mut result = [0.0; 16usize];
7587 result[0..8usize].copy_from_slice(&a.val.0);
7588 result[8usize..16usize].copy_from_slice(&b.val.0);
7589 result.simd_into(self)
7590 }
7591 #[inline(always)]
7592 fn split_f32x8(self, a: f32x8<Self>) -> (f32x4<Self>, f32x4<Self>) {
7593 let mut b0 = [0.0; 4usize];
7594 let mut b1 = [0.0; 4usize];
7595 b0.copy_from_slice(&a.val.0[0..4usize]);
7596 b1.copy_from_slice(&a.val.0[4usize..8usize]);
7597 (b0.simd_into(self), b1.simd_into(self))
7598 }
7599 #[inline(always)]
7600 fn slide_i8x32<const SHIFT: usize>(self, a: i8x32<Self>, b: i8x32<Self>) -> i8x32<Self> {
7601 let mut dest = [Default::default(); 32usize];
7602 dest[..32usize - SHIFT].copy_from_slice(&a.val.0[SHIFT..]);
7603 dest[32usize - SHIFT..].copy_from_slice(&b.val.0[..SHIFT]);
7604 dest.simd_into(self)
7605 }
7606 #[inline(always)]
7607 fn compress_i8x32(self, values: i8x32<Self>, mask: mask8x32<Self>) -> i8x32<Self> {
7608 let mask = mask8x32 {
7609 val: crate::transmute::checked_transmute_copy(&mask.val),
7610 simd: self,
7611 };
7612 Bytes::from_bytes(self.compress_u8x32(Bytes::to_bytes(values), mask))
7613 }
7614 #[inline(always)]
7615 fn compress_merge_i8x32(
7616 self,
7617 values: i8x32<Self>,
7618 mask: mask8x32<Self>,
7619 merge: i8x32<Self>,
7620 ) -> i8x32<Self> {
7621 let mask = mask8x32 {
7622 val: crate::transmute::checked_transmute_copy(&mask.val),
7623 simd: self,
7624 };
7625 Bytes::from_bytes(self.compress_merge_u8x32(
7626 Bytes::to_bytes(values),
7627 mask,
7628 Bytes::to_bytes(merge),
7629 ))
7630 }
7631 #[inline(always)]
7632 fn expand_i8x32(self, values: i8x32<Self>, mask: mask8x32<Self>) -> i8x32<Self> {
7633 let mask = mask8x32 {
7634 val: crate::transmute::checked_transmute_copy(&mask.val),
7635 simd: self,
7636 };
7637 Bytes::from_bytes(self.expand_u8x32(Bytes::to_bytes(values), mask))
7638 }
7639 #[inline(always)]
7640 fn expand_merge_i8x32(
7641 self,
7642 values: i8x32<Self>,
7643 mask: mask8x32<Self>,
7644 merge: i8x32<Self>,
7645 ) -> i8x32<Self> {
7646 let mask = mask8x32 {
7647 val: crate::transmute::checked_transmute_copy(&mask.val),
7648 simd: self,
7649 };
7650 Bytes::from_bytes(self.expand_merge_u8x32(
7651 Bytes::to_bytes(values),
7652 mask,
7653 Bytes::to_bytes(merge),
7654 ))
7655 }
7656 #[inline(always)]
7657 fn combine_i8x32(self, a: i8x32<Self>, b: i8x32<Self>) -> i8x64<Self> {
7658 let mut result = [0; 64usize];
7659 result[0..32usize].copy_from_slice(&a.val.0);
7660 result[32usize..64usize].copy_from_slice(&b.val.0);
7661 result.simd_into(self)
7662 }
7663 #[inline(always)]
7664 fn split_i8x32(self, a: i8x32<Self>) -> (i8x16<Self>, i8x16<Self>) {
7665 let mut b0 = [0; 16usize];
7666 let mut b1 = [0; 16usize];
7667 b0.copy_from_slice(&a.val.0[0..16usize]);
7668 b1.copy_from_slice(&a.val.0[16usize..32usize]);
7669 (b0.simd_into(self), b1.simd_into(self))
7670 }
7671 #[inline(always)]
7672 fn slide_u8x32<const SHIFT: usize>(self, a: u8x32<Self>, b: u8x32<Self>) -> u8x32<Self> {
7673 let mut dest = [Default::default(); 32usize];
7674 dest[..32usize - SHIFT].copy_from_slice(&a.val.0[SHIFT..]);
7675 dest[32usize - SHIFT..].copy_from_slice(&b.val.0[..SHIFT]);
7676 dest.simd_into(self)
7677 }
7678 #[inline(always)]
7679 fn swizzle_dyn_u8x32(self, a: u8x32<Self>, indices: u8x32<Self>) -> u8x32<Self> {
7680 let bytes = Bytes::to_bytes(a);
7681 let mut output = [0u8; 32usize];
7682 for lane in 0..32usize {
7683 let index = indices[lane] as usize;
7684 output[lane] = bytes[index % 32usize];
7685 }
7686 let result: u8x32<Self> = output.simd_into(self);
7687 Bytes::from_bytes(result)
7688 }
7689 #[inline(always)]
7690 fn swizzle_dyn_precise_u8x32(self, a: u8x32<Self>, indices: u8x32<Self>) -> u8x32<Self> {
7691 let bytes = Bytes::to_bytes(a);
7692 let mut output = [0u8; 32usize];
7693 for lane in 0..32usize {
7694 let index = indices[lane] as usize;
7695 let value = bytes[index % 32usize];
7696 output[lane] = core::hint::select_unpredictable(index < 32usize, value, 0u8);
7697 }
7698 let result: u8x32<Self> = output.simd_into(self);
7699 Bytes::from_bytes(result)
7700 }
7701 #[inline(always)]
7702 fn concat_swizzle_dyn_u8x32(
7703 self,
7704 a: u8x32<Self>,
7705 b: u8x32<Self>,
7706 indices: u8x32<Self>,
7707 ) -> u8x32<Self> {
7708 let first_table = Bytes::to_bytes(a);
7709 let second_table = Bytes::to_bytes(b);
7710 let mut output = [0u8; 32usize];
7711 for lane in 0..32usize {
7712 let index = indices[lane] as usize % 64usize;
7713 output[lane] = if index < 32usize {
7714 first_table[index]
7715 } else {
7716 second_table[index - 32usize]
7717 };
7718 }
7719 let result: u8x32<Self> = output.simd_into(self);
7720 Bytes::from_bytes(result)
7721 }
7722 #[inline(always)]
7723 fn concat_swizzle_dyn_precise_u8x32(
7724 self,
7725 a: u8x32<Self>,
7726 b: u8x32<Self>,
7727 indices: u8x32<Self>,
7728 ) -> u8x32<Self> {
7729 let first_table = Bytes::to_bytes(a);
7730 let second_table = Bytes::to_bytes(b);
7731 let mut output = [0u8; 32usize];
7732 for lane in 0..32usize {
7733 let index = indices[lane] as usize;
7734 let table_index = index % 32usize;
7735 let value = if index < 32usize {
7736 first_table[table_index]
7737 } else {
7738 second_table[table_index]
7739 };
7740 output[lane] = if index < 64usize { value } else { 0 };
7741 }
7742 let result: u8x32<Self> = output.simd_into(self);
7743 Bytes::from_bytes(result)
7744 }
7745 #[inline(always)]
7746 fn compress_u8x32(self, values: u8x32<Self>, mask: mask8x32<Self>) -> u8x32<Self> {
7747 self.compress_merge_u8x32(values, mask, u8x32::splat(self, 0))
7748 }
7749 #[inline(always)]
7750 fn compress_merge_u8x32(
7751 self,
7752 values: u8x32<Self>,
7753 mask: mask8x32<Self>,
7754 merge: u8x32<Self>,
7755 ) -> u8x32<Self> {
7756 let mask = self.to_bitmask_mask8x32(mask);
7757 let mut merge_padded = [0u8; 32 + 1];
7758 merge_padded[..32].copy_from_slice(&*merge);
7759 let mut compacted = merge_padded;
7760 let mut output_lane = 0;
7761 for input_lane in 0..32 {
7762 compacted[output_lane] = values[input_lane];
7763 output_lane += ((mask >> input_lane) & 1) as usize;
7764 }
7765 compacted[output_lane] = merge_padded[output_lane];
7766 let mut result = merge;
7767 result.copy_from_slice(&compacted[..32]);
7768 result
7769 }
7770 #[inline(always)]
7771 fn expand_u8x32(self, values: u8x32<Self>, mask: mask8x32<Self>) -> u8x32<Self> {
7772 self.expand_merge_u8x32(values, mask, u8x32::splat(self, 0))
7773 }
7774 #[inline(always)]
7775 fn expand_merge_u8x32(
7776 self,
7777 values: u8x32<Self>,
7778 mask: mask8x32<Self>,
7779 merge: u8x32<Self>,
7780 ) -> u8x32<Self> {
7781 let mask = self.to_bitmask_mask8x32(mask);
7782 let mut result = merge;
7783 let mut input_lane = 0;
7784 for output_lane in 0..32 {
7785 let bit = ((mask >> output_lane) & 1) as u8;
7786 let keep = 0u8.wrapping_sub(bit);
7787 result[output_lane] = (values[input_lane] & keep) | (result[output_lane] & !keep);
7788 input_lane += usize::from(bit);
7789 }
7790 result
7791 }
7792 #[inline(always)]
7793 fn combine_u8x32(self, a: u8x32<Self>, b: u8x32<Self>) -> u8x64<Self> {
7794 let mut result = [0; 64usize];
7795 result[0..32usize].copy_from_slice(&a.val.0);
7796 result[32usize..64usize].copy_from_slice(&b.val.0);
7797 result.simd_into(self)
7798 }
7799 #[inline(always)]
7800 fn split_u8x32(self, a: u8x32<Self>) -> (u8x16<Self>, u8x16<Self>) {
7801 let mut b0 = [0; 16usize];
7802 let mut b1 = [0; 16usize];
7803 b0.copy_from_slice(&a.val.0[0..16usize]);
7804 b1.copy_from_slice(&a.val.0[16usize..32usize]);
7805 (b0.simd_into(self), b1.simd_into(self))
7806 }
7807 #[inline(always)]
7808 fn from_vector_mask8x32(self, vector: i8x32<Self>) -> mask8x32<Self> {
7809 mask8x32 {
7810 val: vector.val,
7811 simd: self,
7812 }
7813 }
7814 #[inline(always)]
7815 fn to_vector_mask8x32(self, a: mask8x32<Self>) -> i8x32<Self> {
7816 i8x32 {
7817 val: a.val,
7818 simd: self,
7819 }
7820 }
7821 #[inline(always)]
7822 fn set_mask8x32(self, a: &mut mask8x32<Self>, index: usize, value: bool) -> () {
7823 assert!(
7824 index < 32usize,
7825 "mask lane index {index} is out of bounds for {} lanes",
7826 32usize
7827 );
7828 let mut lanes: [i8; 32usize] = (*a).into();
7829 lanes[index] = if value { !0 } else { 0 };
7830 *a = lanes.simd_into(self);
7831 }
7832 #[inline(always)]
7833 fn rotate_elements_left_mask8x32<const OFFSET: usize>(
7834 self,
7835 a: mask8x32<Self>,
7836 ) -> mask8x32<Self> {
7837 let int = i8x32 {
7838 val: crate::transmute::checked_transmute_copy(&a.val),
7839 simd: self,
7840 };
7841 let rotated = int.rotate_elements_left::<OFFSET>();
7842 mask8x32 {
7843 val: crate::transmute::checked_transmute_copy(&rotated.val),
7844 simd: self,
7845 }
7846 }
7847 #[inline(always)]
7848 fn rotate_elements_right_mask8x32<const OFFSET: usize>(
7849 self,
7850 a: mask8x32<Self>,
7851 ) -> mask8x32<Self> {
7852 let int = i8x32 {
7853 val: crate::transmute::checked_transmute_copy(&a.val),
7854 simd: self,
7855 };
7856 let rotated = int.rotate_elements_right::<OFFSET>();
7857 mask8x32 {
7858 val: crate::transmute::checked_transmute_copy(&rotated.val),
7859 simd: self,
7860 }
7861 }
7862 #[inline(always)]
7863 fn combine_mask8x32(self, a: mask8x32<Self>, b: mask8x32<Self>) -> mask8x64<Self> {
7864 let mut result = [0; 64usize];
7865 result[0..32usize].copy_from_slice(&a.val.0);
7866 result[32usize..64usize].copy_from_slice(&b.val.0);
7867 result.simd_into(self)
7868 }
7869 #[inline(always)]
7870 fn split_mask8x32(self, a: mask8x32<Self>) -> (mask8x16<Self>, mask8x16<Self>) {
7871 let mut b0 = [0; 16usize];
7872 let mut b1 = [0; 16usize];
7873 b0.copy_from_slice(&a.val.0[0..16usize]);
7874 b1.copy_from_slice(&a.val.0[16usize..32usize]);
7875 (b0.simd_into(self), b1.simd_into(self))
7876 }
7877 #[inline(always)]
7878 fn slide_i16x16<const SHIFT: usize>(self, a: i16x16<Self>, b: i16x16<Self>) -> i16x16<Self> {
7879 let mut dest = [Default::default(); 16usize];
7880 dest[..16usize - SHIFT].copy_from_slice(&a.val.0[SHIFT..]);
7881 dest[16usize - SHIFT..].copy_from_slice(&b.val.0[..SHIFT]);
7882 dest.simd_into(self)
7883 }
7884 #[inline(always)]
7885 fn compress_i16x16(self, values: i16x16<Self>, mask: mask16x16<Self>) -> i16x16<Self> {
7886 let mask = mask8x32 {
7887 val: crate::transmute::checked_transmute_copy(&mask.val),
7888 simd: self,
7889 };
7890 Bytes::from_bytes(self.compress_u8x32(Bytes::to_bytes(values), mask))
7891 }
7892 #[inline(always)]
7893 fn compress_merge_i16x16(
7894 self,
7895 values: i16x16<Self>,
7896 mask: mask16x16<Self>,
7897 merge: i16x16<Self>,
7898 ) -> i16x16<Self> {
7899 let mask = mask8x32 {
7900 val: crate::transmute::checked_transmute_copy(&mask.val),
7901 simd: self,
7902 };
7903 Bytes::from_bytes(self.compress_merge_u8x32(
7904 Bytes::to_bytes(values),
7905 mask,
7906 Bytes::to_bytes(merge),
7907 ))
7908 }
7909 #[inline(always)]
7910 fn expand_i16x16(self, values: i16x16<Self>, mask: mask16x16<Self>) -> i16x16<Self> {
7911 let mask = mask8x32 {
7912 val: crate::transmute::checked_transmute_copy(&mask.val),
7913 simd: self,
7914 };
7915 Bytes::from_bytes(self.expand_u8x32(Bytes::to_bytes(values), mask))
7916 }
7917 #[inline(always)]
7918 fn expand_merge_i16x16(
7919 self,
7920 values: i16x16<Self>,
7921 mask: mask16x16<Self>,
7922 merge: i16x16<Self>,
7923 ) -> i16x16<Self> {
7924 let mask = mask8x32 {
7925 val: crate::transmute::checked_transmute_copy(&mask.val),
7926 simd: self,
7927 };
7928 Bytes::from_bytes(self.expand_merge_u8x32(
7929 Bytes::to_bytes(values),
7930 mask,
7931 Bytes::to_bytes(merge),
7932 ))
7933 }
7934 #[inline(always)]
7935 fn combine_i16x16(self, a: i16x16<Self>, b: i16x16<Self>) -> i16x32<Self> {
7936 let mut result = [0; 32usize];
7937 result[0..16usize].copy_from_slice(&a.val.0);
7938 result[16usize..32usize].copy_from_slice(&b.val.0);
7939 result.simd_into(self)
7940 }
7941 #[inline(always)]
7942 fn split_i16x16(self, a: i16x16<Self>) -> (i16x8<Self>, i16x8<Self>) {
7943 let mut b0 = [0; 8usize];
7944 let mut b1 = [0; 8usize];
7945 b0.copy_from_slice(&a.val.0[0..8usize]);
7946 b1.copy_from_slice(&a.val.0[8usize..16usize]);
7947 (b0.simd_into(self), b1.simd_into(self))
7948 }
7949 #[inline(always)]
7950 fn slide_u16x16<const SHIFT: usize>(self, a: u16x16<Self>, b: u16x16<Self>) -> u16x16<Self> {
7951 let mut dest = [Default::default(); 16usize];
7952 dest[..16usize - SHIFT].copy_from_slice(&a.val.0[SHIFT..]);
7953 dest[16usize - SHIFT..].copy_from_slice(&b.val.0[..SHIFT]);
7954 dest.simd_into(self)
7955 }
7956 #[inline(always)]
7957 fn compress_u16x16(self, values: u16x16<Self>, mask: mask16x16<Self>) -> u16x16<Self> {
7958 let mask = mask8x32 {
7959 val: crate::transmute::checked_transmute_copy(&mask.val),
7960 simd: self,
7961 };
7962 Bytes::from_bytes(self.compress_u8x32(Bytes::to_bytes(values), mask))
7963 }
7964 #[inline(always)]
7965 fn compress_merge_u16x16(
7966 self,
7967 values: u16x16<Self>,
7968 mask: mask16x16<Self>,
7969 merge: u16x16<Self>,
7970 ) -> u16x16<Self> {
7971 let mask = mask8x32 {
7972 val: crate::transmute::checked_transmute_copy(&mask.val),
7973 simd: self,
7974 };
7975 Bytes::from_bytes(self.compress_merge_u8x32(
7976 Bytes::to_bytes(values),
7977 mask,
7978 Bytes::to_bytes(merge),
7979 ))
7980 }
7981 #[inline(always)]
7982 fn expand_u16x16(self, values: u16x16<Self>, mask: mask16x16<Self>) -> u16x16<Self> {
7983 let mask = mask8x32 {
7984 val: crate::transmute::checked_transmute_copy(&mask.val),
7985 simd: self,
7986 };
7987 Bytes::from_bytes(self.expand_u8x32(Bytes::to_bytes(values), mask))
7988 }
7989 #[inline(always)]
7990 fn expand_merge_u16x16(
7991 self,
7992 values: u16x16<Self>,
7993 mask: mask16x16<Self>,
7994 merge: u16x16<Self>,
7995 ) -> u16x16<Self> {
7996 let mask = mask8x32 {
7997 val: crate::transmute::checked_transmute_copy(&mask.val),
7998 simd: self,
7999 };
8000 Bytes::from_bytes(self.expand_merge_u8x32(
8001 Bytes::to_bytes(values),
8002 mask,
8003 Bytes::to_bytes(merge),
8004 ))
8005 }
8006 #[inline(always)]
8007 fn combine_u16x16(self, a: u16x16<Self>, b: u16x16<Self>) -> u16x32<Self> {
8008 let mut result = [0; 32usize];
8009 result[0..16usize].copy_from_slice(&a.val.0);
8010 result[16usize..32usize].copy_from_slice(&b.val.0);
8011 result.simd_into(self)
8012 }
8013 #[inline(always)]
8014 fn split_u16x16(self, a: u16x16<Self>) -> (u16x8<Self>, u16x8<Self>) {
8015 let mut b0 = [0; 8usize];
8016 let mut b1 = [0; 8usize];
8017 b0.copy_from_slice(&a.val.0[0..8usize]);
8018 b1.copy_from_slice(&a.val.0[8usize..16usize]);
8019 (b0.simd_into(self), b1.simd_into(self))
8020 }
8021 #[inline(always)]
8022 fn from_vector_mask16x16(self, vector: i16x16<Self>) -> mask16x16<Self> {
8023 mask16x16 {
8024 val: vector.val,
8025 simd: self,
8026 }
8027 }
8028 #[inline(always)]
8029 fn to_vector_mask16x16(self, a: mask16x16<Self>) -> i16x16<Self> {
8030 i16x16 {
8031 val: a.val,
8032 simd: self,
8033 }
8034 }
8035 #[inline(always)]
8036 fn set_mask16x16(self, a: &mut mask16x16<Self>, index: usize, value: bool) -> () {
8037 assert!(
8038 index < 16usize,
8039 "mask lane index {index} is out of bounds for {} lanes",
8040 16usize
8041 );
8042 let mut lanes: [i16; 16usize] = (*a).into();
8043 lanes[index] = if value { !0 } else { 0 };
8044 *a = lanes.simd_into(self);
8045 }
8046 #[inline(always)]
8047 fn rotate_elements_left_mask16x16<const OFFSET: usize>(
8048 self,
8049 a: mask16x16<Self>,
8050 ) -> mask16x16<Self> {
8051 let int = i16x16 {
8052 val: crate::transmute::checked_transmute_copy(&a.val),
8053 simd: self,
8054 };
8055 let rotated = int.rotate_elements_left::<OFFSET>();
8056 mask16x16 {
8057 val: crate::transmute::checked_transmute_copy(&rotated.val),
8058 simd: self,
8059 }
8060 }
8061 #[inline(always)]
8062 fn rotate_elements_right_mask16x16<const OFFSET: usize>(
8063 self,
8064 a: mask16x16<Self>,
8065 ) -> mask16x16<Self> {
8066 let int = i16x16 {
8067 val: crate::transmute::checked_transmute_copy(&a.val),
8068 simd: self,
8069 };
8070 let rotated = int.rotate_elements_right::<OFFSET>();
8071 mask16x16 {
8072 val: crate::transmute::checked_transmute_copy(&rotated.val),
8073 simd: self,
8074 }
8075 }
8076 #[inline(always)]
8077 fn combine_mask16x16(self, a: mask16x16<Self>, b: mask16x16<Self>) -> mask16x32<Self> {
8078 let mut result = [0; 32usize];
8079 result[0..16usize].copy_from_slice(&a.val.0);
8080 result[16usize..32usize].copy_from_slice(&b.val.0);
8081 result.simd_into(self)
8082 }
8083 #[inline(always)]
8084 fn split_mask16x16(self, a: mask16x16<Self>) -> (mask16x8<Self>, mask16x8<Self>) {
8085 let mut b0 = [0; 8usize];
8086 let mut b1 = [0; 8usize];
8087 b0.copy_from_slice(&a.val.0[0..8usize]);
8088 b1.copy_from_slice(&a.val.0[8usize..16usize]);
8089 (b0.simd_into(self), b1.simd_into(self))
8090 }
8091 #[inline(always)]
8092 fn slide_i32x8<const SHIFT: usize>(self, a: i32x8<Self>, b: i32x8<Self>) -> i32x8<Self> {
8093 let mut dest = [Default::default(); 8usize];
8094 dest[..8usize - SHIFT].copy_from_slice(&a.val.0[SHIFT..]);
8095 dest[8usize - SHIFT..].copy_from_slice(&b.val.0[..SHIFT]);
8096 dest.simd_into(self)
8097 }
8098 #[inline(always)]
8099 fn compress_i32x8(self, values: i32x8<Self>, mask: mask32x8<Self>) -> i32x8<Self> {
8100 let mask = mask8x32 {
8101 val: crate::transmute::checked_transmute_copy(&mask.val),
8102 simd: self,
8103 };
8104 Bytes::from_bytes(self.compress_u8x32(Bytes::to_bytes(values), mask))
8105 }
8106 #[inline(always)]
8107 fn compress_merge_i32x8(
8108 self,
8109 values: i32x8<Self>,
8110 mask: mask32x8<Self>,
8111 merge: i32x8<Self>,
8112 ) -> i32x8<Self> {
8113 let mask = mask8x32 {
8114 val: crate::transmute::checked_transmute_copy(&mask.val),
8115 simd: self,
8116 };
8117 Bytes::from_bytes(self.compress_merge_u8x32(
8118 Bytes::to_bytes(values),
8119 mask,
8120 Bytes::to_bytes(merge),
8121 ))
8122 }
8123 #[inline(always)]
8124 fn expand_i32x8(self, values: i32x8<Self>, mask: mask32x8<Self>) -> i32x8<Self> {
8125 let mask = mask8x32 {
8126 val: crate::transmute::checked_transmute_copy(&mask.val),
8127 simd: self,
8128 };
8129 Bytes::from_bytes(self.expand_u8x32(Bytes::to_bytes(values), mask))
8130 }
8131 #[inline(always)]
8132 fn expand_merge_i32x8(
8133 self,
8134 values: i32x8<Self>,
8135 mask: mask32x8<Self>,
8136 merge: i32x8<Self>,
8137 ) -> i32x8<Self> {
8138 let mask = mask8x32 {
8139 val: crate::transmute::checked_transmute_copy(&mask.val),
8140 simd: self,
8141 };
8142 Bytes::from_bytes(self.expand_merge_u8x32(
8143 Bytes::to_bytes(values),
8144 mask,
8145 Bytes::to_bytes(merge),
8146 ))
8147 }
8148 #[inline(always)]
8149 fn combine_i32x8(self, a: i32x8<Self>, b: i32x8<Self>) -> i32x16<Self> {
8150 let mut result = [0; 16usize];
8151 result[0..8usize].copy_from_slice(&a.val.0);
8152 result[8usize..16usize].copy_from_slice(&b.val.0);
8153 result.simd_into(self)
8154 }
8155 #[inline(always)]
8156 fn split_i32x8(self, a: i32x8<Self>) -> (i32x4<Self>, i32x4<Self>) {
8157 let mut b0 = [0; 4usize];
8158 let mut b1 = [0; 4usize];
8159 b0.copy_from_slice(&a.val.0[0..4usize]);
8160 b1.copy_from_slice(&a.val.0[4usize..8usize]);
8161 (b0.simd_into(self), b1.simd_into(self))
8162 }
8163 #[inline(always)]
8164 fn slide_u32x8<const SHIFT: usize>(self, a: u32x8<Self>, b: u32x8<Self>) -> u32x8<Self> {
8165 let mut dest = [Default::default(); 8usize];
8166 dest[..8usize - SHIFT].copy_from_slice(&a.val.0[SHIFT..]);
8167 dest[8usize - SHIFT..].copy_from_slice(&b.val.0[..SHIFT]);
8168 dest.simd_into(self)
8169 }
8170 #[inline(always)]
8171 fn compress_u32x8(self, values: u32x8<Self>, mask: mask32x8<Self>) -> u32x8<Self> {
8172 let mask = mask8x32 {
8173 val: crate::transmute::checked_transmute_copy(&mask.val),
8174 simd: self,
8175 };
8176 Bytes::from_bytes(self.compress_u8x32(Bytes::to_bytes(values), mask))
8177 }
8178 #[inline(always)]
8179 fn compress_merge_u32x8(
8180 self,
8181 values: u32x8<Self>,
8182 mask: mask32x8<Self>,
8183 merge: u32x8<Self>,
8184 ) -> u32x8<Self> {
8185 let mask = mask8x32 {
8186 val: crate::transmute::checked_transmute_copy(&mask.val),
8187 simd: self,
8188 };
8189 Bytes::from_bytes(self.compress_merge_u8x32(
8190 Bytes::to_bytes(values),
8191 mask,
8192 Bytes::to_bytes(merge),
8193 ))
8194 }
8195 #[inline(always)]
8196 fn expand_u32x8(self, values: u32x8<Self>, mask: mask32x8<Self>) -> u32x8<Self> {
8197 let mask = mask8x32 {
8198 val: crate::transmute::checked_transmute_copy(&mask.val),
8199 simd: self,
8200 };
8201 Bytes::from_bytes(self.expand_u8x32(Bytes::to_bytes(values), mask))
8202 }
8203 #[inline(always)]
8204 fn expand_merge_u32x8(
8205 self,
8206 values: u32x8<Self>,
8207 mask: mask32x8<Self>,
8208 merge: u32x8<Self>,
8209 ) -> u32x8<Self> {
8210 let mask = mask8x32 {
8211 val: crate::transmute::checked_transmute_copy(&mask.val),
8212 simd: self,
8213 };
8214 Bytes::from_bytes(self.expand_merge_u8x32(
8215 Bytes::to_bytes(values),
8216 mask,
8217 Bytes::to_bytes(merge),
8218 ))
8219 }
8220 #[inline(always)]
8221 fn combine_u32x8(self, a: u32x8<Self>, b: u32x8<Self>) -> u32x16<Self> {
8222 let mut result = [0; 16usize];
8223 result[0..8usize].copy_from_slice(&a.val.0);
8224 result[8usize..16usize].copy_from_slice(&b.val.0);
8225 result.simd_into(self)
8226 }
8227 #[inline(always)]
8228 fn split_u32x8(self, a: u32x8<Self>) -> (u32x4<Self>, u32x4<Self>) {
8229 let mut b0 = [0; 4usize];
8230 let mut b1 = [0; 4usize];
8231 b0.copy_from_slice(&a.val.0[0..4usize]);
8232 b1.copy_from_slice(&a.val.0[4usize..8usize]);
8233 (b0.simd_into(self), b1.simd_into(self))
8234 }
8235 #[inline(always)]
8236 fn from_vector_mask32x8(self, vector: i32x8<Self>) -> mask32x8<Self> {
8237 mask32x8 {
8238 val: vector.val,
8239 simd: self,
8240 }
8241 }
8242 #[inline(always)]
8243 fn to_vector_mask32x8(self, a: mask32x8<Self>) -> i32x8<Self> {
8244 i32x8 {
8245 val: a.val,
8246 simd: self,
8247 }
8248 }
8249 #[inline(always)]
8250 fn set_mask32x8(self, a: &mut mask32x8<Self>, index: usize, value: bool) -> () {
8251 assert!(
8252 index < 8usize,
8253 "mask lane index {index} is out of bounds for {} lanes",
8254 8usize
8255 );
8256 let mut lanes: [i32; 8usize] = (*a).into();
8257 lanes[index] = if value { !0 } else { 0 };
8258 *a = lanes.simd_into(self);
8259 }
8260 #[inline(always)]
8261 fn rotate_elements_left_mask32x8<const OFFSET: usize>(
8262 self,
8263 a: mask32x8<Self>,
8264 ) -> mask32x8<Self> {
8265 let int = i32x8 {
8266 val: crate::transmute::checked_transmute_copy(&a.val),
8267 simd: self,
8268 };
8269 let rotated = int.rotate_elements_left::<OFFSET>();
8270 mask32x8 {
8271 val: crate::transmute::checked_transmute_copy(&rotated.val),
8272 simd: self,
8273 }
8274 }
8275 #[inline(always)]
8276 fn rotate_elements_right_mask32x8<const OFFSET: usize>(
8277 self,
8278 a: mask32x8<Self>,
8279 ) -> mask32x8<Self> {
8280 let int = i32x8 {
8281 val: crate::transmute::checked_transmute_copy(&a.val),
8282 simd: self,
8283 };
8284 let rotated = int.rotate_elements_right::<OFFSET>();
8285 mask32x8 {
8286 val: crate::transmute::checked_transmute_copy(&rotated.val),
8287 simd: self,
8288 }
8289 }
8290 #[inline(always)]
8291 fn combine_mask32x8(self, a: mask32x8<Self>, b: mask32x8<Self>) -> mask32x16<Self> {
8292 let mut result = [0; 16usize];
8293 result[0..8usize].copy_from_slice(&a.val.0);
8294 result[8usize..16usize].copy_from_slice(&b.val.0);
8295 result.simd_into(self)
8296 }
8297 #[inline(always)]
8298 fn split_mask32x8(self, a: mask32x8<Self>) -> (mask32x4<Self>, mask32x4<Self>) {
8299 let mut b0 = [0; 4usize];
8300 let mut b1 = [0; 4usize];
8301 b0.copy_from_slice(&a.val.0[0..4usize]);
8302 b1.copy_from_slice(&a.val.0[4usize..8usize]);
8303 (b0.simd_into(self), b1.simd_into(self))
8304 }
8305 #[inline(always)]
8306 fn slide_f64x4<const SHIFT: usize>(self, a: f64x4<Self>, b: f64x4<Self>) -> f64x4<Self> {
8307 let mut dest = [Default::default(); 4usize];
8308 dest[..4usize - SHIFT].copy_from_slice(&a.val.0[SHIFT..]);
8309 dest[4usize - SHIFT..].copy_from_slice(&b.val.0[..SHIFT]);
8310 dest.simd_into(self)
8311 }
8312 #[inline(always)]
8313 fn compress_f64x4(self, values: f64x4<Self>, mask: mask64x4<Self>) -> f64x4<Self> {
8314 let mask = mask8x32 {
8315 val: crate::transmute::checked_transmute_copy(&mask.val),
8316 simd: self,
8317 };
8318 Bytes::from_bytes(self.compress_u8x32(Bytes::to_bytes(values), mask))
8319 }
8320 #[inline(always)]
8321 fn compress_merge_f64x4(
8322 self,
8323 values: f64x4<Self>,
8324 mask: mask64x4<Self>,
8325 merge: f64x4<Self>,
8326 ) -> f64x4<Self> {
8327 let mask = mask8x32 {
8328 val: crate::transmute::checked_transmute_copy(&mask.val),
8329 simd: self,
8330 };
8331 Bytes::from_bytes(self.compress_merge_u8x32(
8332 Bytes::to_bytes(values),
8333 mask,
8334 Bytes::to_bytes(merge),
8335 ))
8336 }
8337 #[inline(always)]
8338 fn expand_f64x4(self, values: f64x4<Self>, mask: mask64x4<Self>) -> f64x4<Self> {
8339 let mask = mask8x32 {
8340 val: crate::transmute::checked_transmute_copy(&mask.val),
8341 simd: self,
8342 };
8343 Bytes::from_bytes(self.expand_u8x32(Bytes::to_bytes(values), mask))
8344 }
8345 #[inline(always)]
8346 fn expand_merge_f64x4(
8347 self,
8348 values: f64x4<Self>,
8349 mask: mask64x4<Self>,
8350 merge: f64x4<Self>,
8351 ) -> f64x4<Self> {
8352 let mask = mask8x32 {
8353 val: crate::transmute::checked_transmute_copy(&mask.val),
8354 simd: self,
8355 };
8356 Bytes::from_bytes(self.expand_merge_u8x32(
8357 Bytes::to_bytes(values),
8358 mask,
8359 Bytes::to_bytes(merge),
8360 ))
8361 }
8362 #[inline(always)]
8363 fn is_nan_f64x4(self, a: f64x4<Self>) -> mask64x4<Self> {
8364 a.simd_ne(a)
8365 }
8366 #[inline(always)]
8367 fn is_infinite_f64x4(self, a: f64x4<Self>) -> mask64x4<Self> {
8368 a.abs().simd_eq(f64::INFINITY)
8369 }
8370 #[inline(always)]
8371 fn is_finite_f64x4(self, a: f64x4<Self>) -> mask64x4<Self> {
8372 a.abs().simd_lt(f64::INFINITY)
8373 }
8374 #[inline(always)]
8375 fn is_subnormal_f64x4(self, a: f64x4<Self>) -> mask64x4<Self> {
8376 let i: i64x4<Self> = a.bitcast();
8377 (i & 0x7FF0_0000_0000_0000_i64).simd_eq(0) & (i & 0x000F_FFFF_FFFF_FFFF_i64).simd_ne(0)
8378 }
8379 #[inline(always)]
8380 fn is_normal_f64x4(self, a: f64x4<Self>) -> mask64x4<Self> {
8381 let i: i64x4<Self> = a.bitcast();
8382 let exp = i & 0x7FF0_0000_0000_0000_i64;
8383 exp.simd_ne(0) & exp.simd_ne(0x7FF0_0000_0000_0000_i64)
8384 }
8385 #[inline(always)]
8386 fn is_sign_positive_f64x4(self, a: f64x4<Self>) -> mask64x4<Self> {
8387 let i: i64x4<Self> = a.bitcast();
8388 (i & i64::MIN).simd_eq(0)
8389 }
8390 #[inline(always)]
8391 fn is_sign_negative_f64x4(self, a: f64x4<Self>) -> mask64x4<Self> {
8392 !a.is_sign_positive()
8393 }
8394 #[inline(always)]
8395 fn combine_f64x4(self, a: f64x4<Self>, b: f64x4<Self>) -> f64x8<Self> {
8396 let mut result = [0.0; 8usize];
8397 result[0..4usize].copy_from_slice(&a.val.0);
8398 result[4usize..8usize].copy_from_slice(&b.val.0);
8399 result.simd_into(self)
8400 }
8401 #[inline(always)]
8402 fn split_f64x4(self, a: f64x4<Self>) -> (f64x2<Self>, f64x2<Self>) {
8403 let mut b0 = [0.0; 2usize];
8404 let mut b1 = [0.0; 2usize];
8405 b0.copy_from_slice(&a.val.0[0..2usize]);
8406 b1.copy_from_slice(&a.val.0[2usize..4usize]);
8407 (b0.simd_into(self), b1.simd_into(self))
8408 }
8409 #[inline(always)]
8410 fn slide_i64x4<const SHIFT: usize>(self, a: i64x4<Self>, b: i64x4<Self>) -> i64x4<Self> {
8411 let mut dest = [Default::default(); 4usize];
8412 dest[..4usize - SHIFT].copy_from_slice(&a.val.0[SHIFT..]);
8413 dest[4usize - SHIFT..].copy_from_slice(&b.val.0[..SHIFT]);
8414 dest.simd_into(self)
8415 }
8416 #[inline(always)]
8417 fn compress_i64x4(self, values: i64x4<Self>, mask: mask64x4<Self>) -> i64x4<Self> {
8418 let mask = mask8x32 {
8419 val: crate::transmute::checked_transmute_copy(&mask.val),
8420 simd: self,
8421 };
8422 Bytes::from_bytes(self.compress_u8x32(Bytes::to_bytes(values), mask))
8423 }
8424 #[inline(always)]
8425 fn compress_merge_i64x4(
8426 self,
8427 values: i64x4<Self>,
8428 mask: mask64x4<Self>,
8429 merge: i64x4<Self>,
8430 ) -> i64x4<Self> {
8431 let mask = mask8x32 {
8432 val: crate::transmute::checked_transmute_copy(&mask.val),
8433 simd: self,
8434 };
8435 Bytes::from_bytes(self.compress_merge_u8x32(
8436 Bytes::to_bytes(values),
8437 mask,
8438 Bytes::to_bytes(merge),
8439 ))
8440 }
8441 #[inline(always)]
8442 fn expand_i64x4(self, values: i64x4<Self>, mask: mask64x4<Self>) -> i64x4<Self> {
8443 let mask = mask8x32 {
8444 val: crate::transmute::checked_transmute_copy(&mask.val),
8445 simd: self,
8446 };
8447 Bytes::from_bytes(self.expand_u8x32(Bytes::to_bytes(values), mask))
8448 }
8449 #[inline(always)]
8450 fn expand_merge_i64x4(
8451 self,
8452 values: i64x4<Self>,
8453 mask: mask64x4<Self>,
8454 merge: i64x4<Self>,
8455 ) -> i64x4<Self> {
8456 let mask = mask8x32 {
8457 val: crate::transmute::checked_transmute_copy(&mask.val),
8458 simd: self,
8459 };
8460 Bytes::from_bytes(self.expand_merge_u8x32(
8461 Bytes::to_bytes(values),
8462 mask,
8463 Bytes::to_bytes(merge),
8464 ))
8465 }
8466 #[inline(always)]
8467 fn combine_i64x4(self, a: i64x4<Self>, b: i64x4<Self>) -> i64x8<Self> {
8468 let mut result = [0; 8usize];
8469 result[0..4usize].copy_from_slice(&a.val.0);
8470 result[4usize..8usize].copy_from_slice(&b.val.0);
8471 result.simd_into(self)
8472 }
8473 #[inline(always)]
8474 fn split_i64x4(self, a: i64x4<Self>) -> (i64x2<Self>, i64x2<Self>) {
8475 let mut b0 = [0; 2usize];
8476 let mut b1 = [0; 2usize];
8477 b0.copy_from_slice(&a.val.0[0..2usize]);
8478 b1.copy_from_slice(&a.val.0[2usize..4usize]);
8479 (b0.simd_into(self), b1.simd_into(self))
8480 }
8481 #[inline(always)]
8482 fn slide_u64x4<const SHIFT: usize>(self, a: u64x4<Self>, b: u64x4<Self>) -> u64x4<Self> {
8483 let mut dest = [Default::default(); 4usize];
8484 dest[..4usize - SHIFT].copy_from_slice(&a.val.0[SHIFT..]);
8485 dest[4usize - SHIFT..].copy_from_slice(&b.val.0[..SHIFT]);
8486 dest.simd_into(self)
8487 }
8488 #[inline(always)]
8489 fn compress_u64x4(self, values: u64x4<Self>, mask: mask64x4<Self>) -> u64x4<Self> {
8490 let mask = mask8x32 {
8491 val: crate::transmute::checked_transmute_copy(&mask.val),
8492 simd: self,
8493 };
8494 Bytes::from_bytes(self.compress_u8x32(Bytes::to_bytes(values), mask))
8495 }
8496 #[inline(always)]
8497 fn compress_merge_u64x4(
8498 self,
8499 values: u64x4<Self>,
8500 mask: mask64x4<Self>,
8501 merge: u64x4<Self>,
8502 ) -> u64x4<Self> {
8503 let mask = mask8x32 {
8504 val: crate::transmute::checked_transmute_copy(&mask.val),
8505 simd: self,
8506 };
8507 Bytes::from_bytes(self.compress_merge_u8x32(
8508 Bytes::to_bytes(values),
8509 mask,
8510 Bytes::to_bytes(merge),
8511 ))
8512 }
8513 #[inline(always)]
8514 fn expand_u64x4(self, values: u64x4<Self>, mask: mask64x4<Self>) -> u64x4<Self> {
8515 let mask = mask8x32 {
8516 val: crate::transmute::checked_transmute_copy(&mask.val),
8517 simd: self,
8518 };
8519 Bytes::from_bytes(self.expand_u8x32(Bytes::to_bytes(values), mask))
8520 }
8521 #[inline(always)]
8522 fn expand_merge_u64x4(
8523 self,
8524 values: u64x4<Self>,
8525 mask: mask64x4<Self>,
8526 merge: u64x4<Self>,
8527 ) -> u64x4<Self> {
8528 let mask = mask8x32 {
8529 val: crate::transmute::checked_transmute_copy(&mask.val),
8530 simd: self,
8531 };
8532 Bytes::from_bytes(self.expand_merge_u8x32(
8533 Bytes::to_bytes(values),
8534 mask,
8535 Bytes::to_bytes(merge),
8536 ))
8537 }
8538 #[inline(always)]
8539 fn combine_u64x4(self, a: u64x4<Self>, b: u64x4<Self>) -> u64x8<Self> {
8540 let mut result = [0; 8usize];
8541 result[0..4usize].copy_from_slice(&a.val.0);
8542 result[4usize..8usize].copy_from_slice(&b.val.0);
8543 result.simd_into(self)
8544 }
8545 #[inline(always)]
8546 fn split_u64x4(self, a: u64x4<Self>) -> (u64x2<Self>, u64x2<Self>) {
8547 let mut b0 = [0; 2usize];
8548 let mut b1 = [0; 2usize];
8549 b0.copy_from_slice(&a.val.0[0..2usize]);
8550 b1.copy_from_slice(&a.val.0[2usize..4usize]);
8551 (b0.simd_into(self), b1.simd_into(self))
8552 }
8553 #[inline(always)]
8554 fn from_vector_mask64x4(self, vector: i64x4<Self>) -> mask64x4<Self> {
8555 mask64x4 {
8556 val: vector.val,
8557 simd: self,
8558 }
8559 }
8560 #[inline(always)]
8561 fn to_vector_mask64x4(self, a: mask64x4<Self>) -> i64x4<Self> {
8562 i64x4 {
8563 val: a.val,
8564 simd: self,
8565 }
8566 }
8567 #[inline(always)]
8568 fn set_mask64x4(self, a: &mut mask64x4<Self>, index: usize, value: bool) -> () {
8569 assert!(
8570 index < 4usize,
8571 "mask lane index {index} is out of bounds for {} lanes",
8572 4usize
8573 );
8574 let mut lanes: [i64; 4usize] = (*a).into();
8575 lanes[index] = if value { !0 } else { 0 };
8576 *a = lanes.simd_into(self);
8577 }
8578 #[inline(always)]
8579 fn rotate_elements_left_mask64x4<const OFFSET: usize>(
8580 self,
8581 a: mask64x4<Self>,
8582 ) -> mask64x4<Self> {
8583 let int = i64x4 {
8584 val: crate::transmute::checked_transmute_copy(&a.val),
8585 simd: self,
8586 };
8587 let rotated = int.rotate_elements_left::<OFFSET>();
8588 mask64x4 {
8589 val: crate::transmute::checked_transmute_copy(&rotated.val),
8590 simd: self,
8591 }
8592 }
8593 #[inline(always)]
8594 fn rotate_elements_right_mask64x4<const OFFSET: usize>(
8595 self,
8596 a: mask64x4<Self>,
8597 ) -> mask64x4<Self> {
8598 let int = i64x4 {
8599 val: crate::transmute::checked_transmute_copy(&a.val),
8600 simd: self,
8601 };
8602 let rotated = int.rotate_elements_right::<OFFSET>();
8603 mask64x4 {
8604 val: crate::transmute::checked_transmute_copy(&rotated.val),
8605 simd: self,
8606 }
8607 }
8608 #[inline(always)]
8609 fn combine_mask64x4(self, a: mask64x4<Self>, b: mask64x4<Self>) -> mask64x8<Self> {
8610 let mut result = [0; 8usize];
8611 result[0..4usize].copy_from_slice(&a.val.0);
8612 result[4usize..8usize].copy_from_slice(&b.val.0);
8613 result.simd_into(self)
8614 }
8615 #[inline(always)]
8616 fn split_mask64x4(self, a: mask64x4<Self>) -> (mask64x2<Self>, mask64x2<Self>) {
8617 let mut b0 = [0; 2usize];
8618 let mut b1 = [0; 2usize];
8619 b0.copy_from_slice(&a.val.0[0..2usize]);
8620 b1.copy_from_slice(&a.val.0[2usize..4usize]);
8621 (b0.simd_into(self), b1.simd_into(self))
8622 }
8623 #[inline(always)]
8624 fn slide_f32x16<const SHIFT: usize>(self, a: f32x16<Self>, b: f32x16<Self>) -> f32x16<Self> {
8625 let mut dest = [Default::default(); 16usize];
8626 dest[..16usize - SHIFT].copy_from_slice(&a.val.0[SHIFT..]);
8627 dest[16usize - SHIFT..].copy_from_slice(&b.val.0[..SHIFT]);
8628 dest.simd_into(self)
8629 }
8630 #[inline(always)]
8631 fn compress_f32x16(self, values: f32x16<Self>, mask: mask32x16<Self>) -> f32x16<Self> {
8632 let mask = mask8x64 {
8633 val: crate::transmute::checked_transmute_copy(&mask.val),
8634 simd: self,
8635 };
8636 Bytes::from_bytes(self.compress_u8x64(Bytes::to_bytes(values), mask))
8637 }
8638 #[inline(always)]
8639 fn compress_merge_f32x16(
8640 self,
8641 values: f32x16<Self>,
8642 mask: mask32x16<Self>,
8643 merge: f32x16<Self>,
8644 ) -> f32x16<Self> {
8645 let mask = mask8x64 {
8646 val: crate::transmute::checked_transmute_copy(&mask.val),
8647 simd: self,
8648 };
8649 Bytes::from_bytes(self.compress_merge_u8x64(
8650 Bytes::to_bytes(values),
8651 mask,
8652 Bytes::to_bytes(merge),
8653 ))
8654 }
8655 #[inline(always)]
8656 fn expand_f32x16(self, values: f32x16<Self>, mask: mask32x16<Self>) -> f32x16<Self> {
8657 let mask = mask8x64 {
8658 val: crate::transmute::checked_transmute_copy(&mask.val),
8659 simd: self,
8660 };
8661 Bytes::from_bytes(self.expand_u8x64(Bytes::to_bytes(values), mask))
8662 }
8663 #[inline(always)]
8664 fn expand_merge_f32x16(
8665 self,
8666 values: f32x16<Self>,
8667 mask: mask32x16<Self>,
8668 merge: f32x16<Self>,
8669 ) -> f32x16<Self> {
8670 let mask = mask8x64 {
8671 val: crate::transmute::checked_transmute_copy(&mask.val),
8672 simd: self,
8673 };
8674 Bytes::from_bytes(self.expand_merge_u8x64(
8675 Bytes::to_bytes(values),
8676 mask,
8677 Bytes::to_bytes(merge),
8678 ))
8679 }
8680 #[inline(always)]
8681 fn is_nan_f32x16(self, a: f32x16<Self>) -> mask32x16<Self> {
8682 a.simd_ne(a)
8683 }
8684 #[inline(always)]
8685 fn is_infinite_f32x16(self, a: f32x16<Self>) -> mask32x16<Self> {
8686 a.abs().simd_eq(f32::INFINITY)
8687 }
8688 #[inline(always)]
8689 fn is_finite_f32x16(self, a: f32x16<Self>) -> mask32x16<Self> {
8690 a.abs().simd_lt(f32::INFINITY)
8691 }
8692 #[inline(always)]
8693 fn is_subnormal_f32x16(self, a: f32x16<Self>) -> mask32x16<Self> {
8694 let i: i32x16<Self> = a.bitcast();
8695 (i & 0x7F80_0000_i32).simd_eq(0) & (i & 0x007F_FFFF_i32).simd_ne(0)
8696 }
8697 #[inline(always)]
8698 fn is_normal_f32x16(self, a: f32x16<Self>) -> mask32x16<Self> {
8699 let i: i32x16<Self> = a.bitcast();
8700 let exp = i & 0x7F80_0000_i32;
8701 exp.simd_ne(0) & exp.simd_ne(0x7F80_0000_i32)
8702 }
8703 #[inline(always)]
8704 fn is_sign_positive_f32x16(self, a: f32x16<Self>) -> mask32x16<Self> {
8705 let i: i32x16<Self> = a.bitcast();
8706 (i & i32::MIN).simd_eq(0)
8707 }
8708 #[inline(always)]
8709 fn is_sign_negative_f32x16(self, a: f32x16<Self>) -> mask32x16<Self> {
8710 !a.is_sign_positive()
8711 }
8712 #[inline(always)]
8713 fn split_f32x16(self, a: f32x16<Self>) -> (f32x8<Self>, f32x8<Self>) {
8714 let mut b0 = [0.0; 8usize];
8715 let mut b1 = [0.0; 8usize];
8716 b0.copy_from_slice(&a.val.0[0..8usize]);
8717 b1.copy_from_slice(&a.val.0[8usize..16usize]);
8718 (b0.simd_into(self), b1.simd_into(self))
8719 }
8720 #[inline(always)]
8721 fn slide_i8x64<const SHIFT: usize>(self, a: i8x64<Self>, b: i8x64<Self>) -> i8x64<Self> {
8722 let mut dest = [Default::default(); 64usize];
8723 dest[..64usize - SHIFT].copy_from_slice(&a.val.0[SHIFT..]);
8724 dest[64usize - SHIFT..].copy_from_slice(&b.val.0[..SHIFT]);
8725 dest.simd_into(self)
8726 }
8727 #[inline(always)]
8728 fn compress_i8x64(self, values: i8x64<Self>, mask: mask8x64<Self>) -> i8x64<Self> {
8729 let mask = mask8x64 {
8730 val: crate::transmute::checked_transmute_copy(&mask.val),
8731 simd: self,
8732 };
8733 Bytes::from_bytes(self.compress_u8x64(Bytes::to_bytes(values), mask))
8734 }
8735 #[inline(always)]
8736 fn compress_merge_i8x64(
8737 self,
8738 values: i8x64<Self>,
8739 mask: mask8x64<Self>,
8740 merge: i8x64<Self>,
8741 ) -> i8x64<Self> {
8742 let mask = mask8x64 {
8743 val: crate::transmute::checked_transmute_copy(&mask.val),
8744 simd: self,
8745 };
8746 Bytes::from_bytes(self.compress_merge_u8x64(
8747 Bytes::to_bytes(values),
8748 mask,
8749 Bytes::to_bytes(merge),
8750 ))
8751 }
8752 #[inline(always)]
8753 fn expand_i8x64(self, values: i8x64<Self>, mask: mask8x64<Self>) -> i8x64<Self> {
8754 let mask = mask8x64 {
8755 val: crate::transmute::checked_transmute_copy(&mask.val),
8756 simd: self,
8757 };
8758 Bytes::from_bytes(self.expand_u8x64(Bytes::to_bytes(values), mask))
8759 }
8760 #[inline(always)]
8761 fn expand_merge_i8x64(
8762 self,
8763 values: i8x64<Self>,
8764 mask: mask8x64<Self>,
8765 merge: i8x64<Self>,
8766 ) -> i8x64<Self> {
8767 let mask = mask8x64 {
8768 val: crate::transmute::checked_transmute_copy(&mask.val),
8769 simd: self,
8770 };
8771 Bytes::from_bytes(self.expand_merge_u8x64(
8772 Bytes::to_bytes(values),
8773 mask,
8774 Bytes::to_bytes(merge),
8775 ))
8776 }
8777 #[inline(always)]
8778 fn split_i8x64(self, a: i8x64<Self>) -> (i8x32<Self>, i8x32<Self>) {
8779 let mut b0 = [0; 32usize];
8780 let mut b1 = [0; 32usize];
8781 b0.copy_from_slice(&a.val.0[0..32usize]);
8782 b1.copy_from_slice(&a.val.0[32usize..64usize]);
8783 (b0.simd_into(self), b1.simd_into(self))
8784 }
8785 #[inline(always)]
8786 fn slide_u8x64<const SHIFT: usize>(self, a: u8x64<Self>, b: u8x64<Self>) -> u8x64<Self> {
8787 let mut dest = [Default::default(); 64usize];
8788 dest[..64usize - SHIFT].copy_from_slice(&a.val.0[SHIFT..]);
8789 dest[64usize - SHIFT..].copy_from_slice(&b.val.0[..SHIFT]);
8790 dest.simd_into(self)
8791 }
8792 #[inline(always)]
8793 fn swizzle_dyn_u8x64(self, a: u8x64<Self>, indices: u8x64<Self>) -> u8x64<Self> {
8794 let bytes = Bytes::to_bytes(a);
8795 let mut output = [0u8; 64usize];
8796 for lane in 0..64usize {
8797 let index = indices[lane] as usize;
8798 output[lane] = bytes[index % 64usize];
8799 }
8800 let result: u8x64<Self> = output.simd_into(self);
8801 Bytes::from_bytes(result)
8802 }
8803 #[inline(always)]
8804 fn swizzle_dyn_precise_u8x64(self, a: u8x64<Self>, indices: u8x64<Self>) -> u8x64<Self> {
8805 let bytes = Bytes::to_bytes(a);
8806 let mut output = [0u8; 64usize];
8807 for lane in 0..64usize {
8808 let index = indices[lane] as usize;
8809 let value = bytes[index % 64usize];
8810 output[lane] = core::hint::select_unpredictable(index < 64usize, value, 0u8);
8811 }
8812 let result: u8x64<Self> = output.simd_into(self);
8813 Bytes::from_bytes(result)
8814 }
8815 #[inline(always)]
8816 fn concat_swizzle_dyn_u8x64(
8817 self,
8818 a: u8x64<Self>,
8819 b: u8x64<Self>,
8820 indices: u8x64<Self>,
8821 ) -> u8x64<Self> {
8822 let first_table = Bytes::to_bytes(a);
8823 let second_table = Bytes::to_bytes(b);
8824 let mut output = [0u8; 64usize];
8825 for lane in 0..64usize {
8826 let index = indices[lane] as usize % 128usize;
8827 output[lane] = if index < 64usize {
8828 first_table[index]
8829 } else {
8830 second_table[index - 64usize]
8831 };
8832 }
8833 let result: u8x64<Self> = output.simd_into(self);
8834 Bytes::from_bytes(result)
8835 }
8836 #[inline(always)]
8837 fn concat_swizzle_dyn_precise_u8x64(
8838 self,
8839 a: u8x64<Self>,
8840 b: u8x64<Self>,
8841 indices: u8x64<Self>,
8842 ) -> u8x64<Self> {
8843 let first_table = Bytes::to_bytes(a);
8844 let second_table = Bytes::to_bytes(b);
8845 let mut output = [0u8; 64usize];
8846 for lane in 0..64usize {
8847 let index = indices[lane] as usize;
8848 let table_index = index % 64usize;
8849 let value = if index < 64usize {
8850 first_table[table_index]
8851 } else {
8852 second_table[table_index]
8853 };
8854 output[lane] = if index < 128usize { value } else { 0 };
8855 }
8856 let result: u8x64<Self> = output.simd_into(self);
8857 Bytes::from_bytes(result)
8858 }
8859 #[inline(always)]
8860 fn compress_u8x64(self, values: u8x64<Self>, mask: mask8x64<Self>) -> u8x64<Self> {
8861 self.compress_merge_u8x64(values, mask, u8x64::splat(self, 0))
8862 }
8863 #[inline(always)]
8864 fn compress_merge_u8x64(
8865 self,
8866 values: u8x64<Self>,
8867 mask: mask8x64<Self>,
8868 merge: u8x64<Self>,
8869 ) -> u8x64<Self> {
8870 let mask = self.to_bitmask_mask8x64(mask);
8871 let mut merge_padded = [0u8; 64 + 1];
8872 merge_padded[..64].copy_from_slice(&*merge);
8873 let mut compacted = merge_padded;
8874 let mut output_lane = 0;
8875 for input_lane in 0..64 {
8876 compacted[output_lane] = values[input_lane];
8877 output_lane += ((mask >> input_lane) & 1) as usize;
8878 }
8879 compacted[output_lane] = merge_padded[output_lane];
8880 let mut result = merge;
8881 result.copy_from_slice(&compacted[..64]);
8882 result
8883 }
8884 #[inline(always)]
8885 fn expand_u8x64(self, values: u8x64<Self>, mask: mask8x64<Self>) -> u8x64<Self> {
8886 self.expand_merge_u8x64(values, mask, u8x64::splat(self, 0))
8887 }
8888 #[inline(always)]
8889 fn expand_merge_u8x64(
8890 self,
8891 values: u8x64<Self>,
8892 mask: mask8x64<Self>,
8893 merge: u8x64<Self>,
8894 ) -> u8x64<Self> {
8895 let mask = self.to_bitmask_mask8x64(mask);
8896 let mut result = merge;
8897 let mut input_lane = 0;
8898 for output_lane in 0..64 {
8899 let bit = ((mask >> output_lane) & 1) as u8;
8900 let keep = 0u8.wrapping_sub(bit);
8901 result[output_lane] = (values[input_lane] & keep) | (result[output_lane] & !keep);
8902 input_lane += usize::from(bit);
8903 }
8904 result
8905 }
8906 #[inline(always)]
8907 fn split_u8x64(self, a: u8x64<Self>) -> (u8x32<Self>, u8x32<Self>) {
8908 let mut b0 = [0; 32usize];
8909 let mut b1 = [0; 32usize];
8910 b0.copy_from_slice(&a.val.0[0..32usize]);
8911 b1.copy_from_slice(&a.val.0[32usize..64usize]);
8912 (b0.simd_into(self), b1.simd_into(self))
8913 }
8914 #[inline(always)]
8915 fn from_vector_mask8x64(self, vector: i8x64<Self>) -> mask8x64<Self> {
8916 mask8x64 {
8917 val: vector.val,
8918 simd: self,
8919 }
8920 }
8921 #[inline(always)]
8922 fn to_vector_mask8x64(self, a: mask8x64<Self>) -> i8x64<Self> {
8923 i8x64 {
8924 val: a.val,
8925 simd: self,
8926 }
8927 }
8928 #[inline(always)]
8929 fn set_mask8x64(self, a: &mut mask8x64<Self>, index: usize, value: bool) -> () {
8930 assert!(
8931 index < 64usize,
8932 "mask lane index {index} is out of bounds for {} lanes",
8933 64usize
8934 );
8935 let mut lanes: [i8; 64usize] = (*a).into();
8936 lanes[index] = if value { !0 } else { 0 };
8937 *a = lanes.simd_into(self);
8938 }
8939 #[inline(always)]
8940 fn rotate_elements_left_mask8x64<const OFFSET: usize>(
8941 self,
8942 a: mask8x64<Self>,
8943 ) -> mask8x64<Self> {
8944 let int = i8x64 {
8945 val: crate::transmute::checked_transmute_copy(&a.val),
8946 simd: self,
8947 };
8948 let rotated = int.rotate_elements_left::<OFFSET>();
8949 mask8x64 {
8950 val: crate::transmute::checked_transmute_copy(&rotated.val),
8951 simd: self,
8952 }
8953 }
8954 #[inline(always)]
8955 fn rotate_elements_right_mask8x64<const OFFSET: usize>(
8956 self,
8957 a: mask8x64<Self>,
8958 ) -> mask8x64<Self> {
8959 let int = i8x64 {
8960 val: crate::transmute::checked_transmute_copy(&a.val),
8961 simd: self,
8962 };
8963 let rotated = int.rotate_elements_right::<OFFSET>();
8964 mask8x64 {
8965 val: crate::transmute::checked_transmute_copy(&rotated.val),
8966 simd: self,
8967 }
8968 }
8969 #[inline(always)]
8970 fn split_mask8x64(self, a: mask8x64<Self>) -> (mask8x32<Self>, mask8x32<Self>) {
8971 let mut b0 = [0; 32usize];
8972 let mut b1 = [0; 32usize];
8973 b0.copy_from_slice(&a.val.0[0..32usize]);
8974 b1.copy_from_slice(&a.val.0[32usize..64usize]);
8975 (b0.simd_into(self), b1.simd_into(self))
8976 }
8977 #[inline(always)]
8978 fn slide_i16x32<const SHIFT: usize>(self, a: i16x32<Self>, b: i16x32<Self>) -> i16x32<Self> {
8979 let mut dest = [Default::default(); 32usize];
8980 dest[..32usize - SHIFT].copy_from_slice(&a.val.0[SHIFT..]);
8981 dest[32usize - SHIFT..].copy_from_slice(&b.val.0[..SHIFT]);
8982 dest.simd_into(self)
8983 }
8984 #[inline(always)]
8985 fn compress_i16x32(self, values: i16x32<Self>, mask: mask16x32<Self>) -> i16x32<Self> {
8986 let mask = mask8x64 {
8987 val: crate::transmute::checked_transmute_copy(&mask.val),
8988 simd: self,
8989 };
8990 Bytes::from_bytes(self.compress_u8x64(Bytes::to_bytes(values), mask))
8991 }
8992 #[inline(always)]
8993 fn compress_merge_i16x32(
8994 self,
8995 values: i16x32<Self>,
8996 mask: mask16x32<Self>,
8997 merge: i16x32<Self>,
8998 ) -> i16x32<Self> {
8999 let mask = mask8x64 {
9000 val: crate::transmute::checked_transmute_copy(&mask.val),
9001 simd: self,
9002 };
9003 Bytes::from_bytes(self.compress_merge_u8x64(
9004 Bytes::to_bytes(values),
9005 mask,
9006 Bytes::to_bytes(merge),
9007 ))
9008 }
9009 #[inline(always)]
9010 fn expand_i16x32(self, values: i16x32<Self>, mask: mask16x32<Self>) -> i16x32<Self> {
9011 let mask = mask8x64 {
9012 val: crate::transmute::checked_transmute_copy(&mask.val),
9013 simd: self,
9014 };
9015 Bytes::from_bytes(self.expand_u8x64(Bytes::to_bytes(values), mask))
9016 }
9017 #[inline(always)]
9018 fn expand_merge_i16x32(
9019 self,
9020 values: i16x32<Self>,
9021 mask: mask16x32<Self>,
9022 merge: i16x32<Self>,
9023 ) -> i16x32<Self> {
9024 let mask = mask8x64 {
9025 val: crate::transmute::checked_transmute_copy(&mask.val),
9026 simd: self,
9027 };
9028 Bytes::from_bytes(self.expand_merge_u8x64(
9029 Bytes::to_bytes(values),
9030 mask,
9031 Bytes::to_bytes(merge),
9032 ))
9033 }
9034 #[inline(always)]
9035 fn split_i16x32(self, a: i16x32<Self>) -> (i16x16<Self>, i16x16<Self>) {
9036 let mut b0 = [0; 16usize];
9037 let mut b1 = [0; 16usize];
9038 b0.copy_from_slice(&a.val.0[0..16usize]);
9039 b1.copy_from_slice(&a.val.0[16usize..32usize]);
9040 (b0.simd_into(self), b1.simd_into(self))
9041 }
9042 #[inline(always)]
9043 fn slide_u16x32<const SHIFT: usize>(self, a: u16x32<Self>, b: u16x32<Self>) -> u16x32<Self> {
9044 let mut dest = [Default::default(); 32usize];
9045 dest[..32usize - SHIFT].copy_from_slice(&a.val.0[SHIFT..]);
9046 dest[32usize - SHIFT..].copy_from_slice(&b.val.0[..SHIFT]);
9047 dest.simd_into(self)
9048 }
9049 #[inline(always)]
9050 fn compress_u16x32(self, values: u16x32<Self>, mask: mask16x32<Self>) -> u16x32<Self> {
9051 let mask = mask8x64 {
9052 val: crate::transmute::checked_transmute_copy(&mask.val),
9053 simd: self,
9054 };
9055 Bytes::from_bytes(self.compress_u8x64(Bytes::to_bytes(values), mask))
9056 }
9057 #[inline(always)]
9058 fn compress_merge_u16x32(
9059 self,
9060 values: u16x32<Self>,
9061 mask: mask16x32<Self>,
9062 merge: u16x32<Self>,
9063 ) -> u16x32<Self> {
9064 let mask = mask8x64 {
9065 val: crate::transmute::checked_transmute_copy(&mask.val),
9066 simd: self,
9067 };
9068 Bytes::from_bytes(self.compress_merge_u8x64(
9069 Bytes::to_bytes(values),
9070 mask,
9071 Bytes::to_bytes(merge),
9072 ))
9073 }
9074 #[inline(always)]
9075 fn expand_u16x32(self, values: u16x32<Self>, mask: mask16x32<Self>) -> u16x32<Self> {
9076 let mask = mask8x64 {
9077 val: crate::transmute::checked_transmute_copy(&mask.val),
9078 simd: self,
9079 };
9080 Bytes::from_bytes(self.expand_u8x64(Bytes::to_bytes(values), mask))
9081 }
9082 #[inline(always)]
9083 fn expand_merge_u16x32(
9084 self,
9085 values: u16x32<Self>,
9086 mask: mask16x32<Self>,
9087 merge: u16x32<Self>,
9088 ) -> u16x32<Self> {
9089 let mask = mask8x64 {
9090 val: crate::transmute::checked_transmute_copy(&mask.val),
9091 simd: self,
9092 };
9093 Bytes::from_bytes(self.expand_merge_u8x64(
9094 Bytes::to_bytes(values),
9095 mask,
9096 Bytes::to_bytes(merge),
9097 ))
9098 }
9099 #[inline(always)]
9100 fn split_u16x32(self, a: u16x32<Self>) -> (u16x16<Self>, u16x16<Self>) {
9101 let mut b0 = [0; 16usize];
9102 let mut b1 = [0; 16usize];
9103 b0.copy_from_slice(&a.val.0[0..16usize]);
9104 b1.copy_from_slice(&a.val.0[16usize..32usize]);
9105 (b0.simd_into(self), b1.simd_into(self))
9106 }
9107 #[inline(always)]
9108 fn from_vector_mask16x32(self, vector: i16x32<Self>) -> mask16x32<Self> {
9109 mask16x32 {
9110 val: vector.val,
9111 simd: self,
9112 }
9113 }
9114 #[inline(always)]
9115 fn to_vector_mask16x32(self, a: mask16x32<Self>) -> i16x32<Self> {
9116 i16x32 {
9117 val: a.val,
9118 simd: self,
9119 }
9120 }
9121 #[inline(always)]
9122 fn set_mask16x32(self, a: &mut mask16x32<Self>, index: usize, value: bool) -> () {
9123 assert!(
9124 index < 32usize,
9125 "mask lane index {index} is out of bounds for {} lanes",
9126 32usize
9127 );
9128 let mut lanes: [i16; 32usize] = (*a).into();
9129 lanes[index] = if value { !0 } else { 0 };
9130 *a = lanes.simd_into(self);
9131 }
9132 #[inline(always)]
9133 fn rotate_elements_left_mask16x32<const OFFSET: usize>(
9134 self,
9135 a: mask16x32<Self>,
9136 ) -> mask16x32<Self> {
9137 let int = i16x32 {
9138 val: crate::transmute::checked_transmute_copy(&a.val),
9139 simd: self,
9140 };
9141 let rotated = int.rotate_elements_left::<OFFSET>();
9142 mask16x32 {
9143 val: crate::transmute::checked_transmute_copy(&rotated.val),
9144 simd: self,
9145 }
9146 }
9147 #[inline(always)]
9148 fn rotate_elements_right_mask16x32<const OFFSET: usize>(
9149 self,
9150 a: mask16x32<Self>,
9151 ) -> mask16x32<Self> {
9152 let int = i16x32 {
9153 val: crate::transmute::checked_transmute_copy(&a.val),
9154 simd: self,
9155 };
9156 let rotated = int.rotate_elements_right::<OFFSET>();
9157 mask16x32 {
9158 val: crate::transmute::checked_transmute_copy(&rotated.val),
9159 simd: self,
9160 }
9161 }
9162 #[inline(always)]
9163 fn split_mask16x32(self, a: mask16x32<Self>) -> (mask16x16<Self>, mask16x16<Self>) {
9164 let mut b0 = [0; 16usize];
9165 let mut b1 = [0; 16usize];
9166 b0.copy_from_slice(&a.val.0[0..16usize]);
9167 b1.copy_from_slice(&a.val.0[16usize..32usize]);
9168 (b0.simd_into(self), b1.simd_into(self))
9169 }
9170 #[inline(always)]
9171 fn slide_i32x16<const SHIFT: usize>(self, a: i32x16<Self>, b: i32x16<Self>) -> i32x16<Self> {
9172 let mut dest = [Default::default(); 16usize];
9173 dest[..16usize - SHIFT].copy_from_slice(&a.val.0[SHIFT..]);
9174 dest[16usize - SHIFT..].copy_from_slice(&b.val.0[..SHIFT]);
9175 dest.simd_into(self)
9176 }
9177 #[inline(always)]
9178 fn compress_i32x16(self, values: i32x16<Self>, mask: mask32x16<Self>) -> i32x16<Self> {
9179 let mask = mask8x64 {
9180 val: crate::transmute::checked_transmute_copy(&mask.val),
9181 simd: self,
9182 };
9183 Bytes::from_bytes(self.compress_u8x64(Bytes::to_bytes(values), mask))
9184 }
9185 #[inline(always)]
9186 fn compress_merge_i32x16(
9187 self,
9188 values: i32x16<Self>,
9189 mask: mask32x16<Self>,
9190 merge: i32x16<Self>,
9191 ) -> i32x16<Self> {
9192 let mask = mask8x64 {
9193 val: crate::transmute::checked_transmute_copy(&mask.val),
9194 simd: self,
9195 };
9196 Bytes::from_bytes(self.compress_merge_u8x64(
9197 Bytes::to_bytes(values),
9198 mask,
9199 Bytes::to_bytes(merge),
9200 ))
9201 }
9202 #[inline(always)]
9203 fn expand_i32x16(self, values: i32x16<Self>, mask: mask32x16<Self>) -> i32x16<Self> {
9204 let mask = mask8x64 {
9205 val: crate::transmute::checked_transmute_copy(&mask.val),
9206 simd: self,
9207 };
9208 Bytes::from_bytes(self.expand_u8x64(Bytes::to_bytes(values), mask))
9209 }
9210 #[inline(always)]
9211 fn expand_merge_i32x16(
9212 self,
9213 values: i32x16<Self>,
9214 mask: mask32x16<Self>,
9215 merge: i32x16<Self>,
9216 ) -> i32x16<Self> {
9217 let mask = mask8x64 {
9218 val: crate::transmute::checked_transmute_copy(&mask.val),
9219 simd: self,
9220 };
9221 Bytes::from_bytes(self.expand_merge_u8x64(
9222 Bytes::to_bytes(values),
9223 mask,
9224 Bytes::to_bytes(merge),
9225 ))
9226 }
9227 #[inline(always)]
9228 fn split_i32x16(self, a: i32x16<Self>) -> (i32x8<Self>, i32x8<Self>) {
9229 let mut b0 = [0; 8usize];
9230 let mut b1 = [0; 8usize];
9231 b0.copy_from_slice(&a.val.0[0..8usize]);
9232 b1.copy_from_slice(&a.val.0[8usize..16usize]);
9233 (b0.simd_into(self), b1.simd_into(self))
9234 }
9235 #[inline(always)]
9236 fn slide_u32x16<const SHIFT: usize>(self, a: u32x16<Self>, b: u32x16<Self>) -> u32x16<Self> {
9237 let mut dest = [Default::default(); 16usize];
9238 dest[..16usize - SHIFT].copy_from_slice(&a.val.0[SHIFT..]);
9239 dest[16usize - SHIFT..].copy_from_slice(&b.val.0[..SHIFT]);
9240 dest.simd_into(self)
9241 }
9242 #[inline(always)]
9243 fn compress_u32x16(self, values: u32x16<Self>, mask: mask32x16<Self>) -> u32x16<Self> {
9244 let mask = mask8x64 {
9245 val: crate::transmute::checked_transmute_copy(&mask.val),
9246 simd: self,
9247 };
9248 Bytes::from_bytes(self.compress_u8x64(Bytes::to_bytes(values), mask))
9249 }
9250 #[inline(always)]
9251 fn compress_merge_u32x16(
9252 self,
9253 values: u32x16<Self>,
9254 mask: mask32x16<Self>,
9255 merge: u32x16<Self>,
9256 ) -> u32x16<Self> {
9257 let mask = mask8x64 {
9258 val: crate::transmute::checked_transmute_copy(&mask.val),
9259 simd: self,
9260 };
9261 Bytes::from_bytes(self.compress_merge_u8x64(
9262 Bytes::to_bytes(values),
9263 mask,
9264 Bytes::to_bytes(merge),
9265 ))
9266 }
9267 #[inline(always)]
9268 fn expand_u32x16(self, values: u32x16<Self>, mask: mask32x16<Self>) -> u32x16<Self> {
9269 let mask = mask8x64 {
9270 val: crate::transmute::checked_transmute_copy(&mask.val),
9271 simd: self,
9272 };
9273 Bytes::from_bytes(self.expand_u8x64(Bytes::to_bytes(values), mask))
9274 }
9275 #[inline(always)]
9276 fn expand_merge_u32x16(
9277 self,
9278 values: u32x16<Self>,
9279 mask: mask32x16<Self>,
9280 merge: u32x16<Self>,
9281 ) -> u32x16<Self> {
9282 let mask = mask8x64 {
9283 val: crate::transmute::checked_transmute_copy(&mask.val),
9284 simd: self,
9285 };
9286 Bytes::from_bytes(self.expand_merge_u8x64(
9287 Bytes::to_bytes(values),
9288 mask,
9289 Bytes::to_bytes(merge),
9290 ))
9291 }
9292 #[inline(always)]
9293 fn split_u32x16(self, a: u32x16<Self>) -> (u32x8<Self>, u32x8<Self>) {
9294 let mut b0 = [0; 8usize];
9295 let mut b1 = [0; 8usize];
9296 b0.copy_from_slice(&a.val.0[0..8usize]);
9297 b1.copy_from_slice(&a.val.0[8usize..16usize]);
9298 (b0.simd_into(self), b1.simd_into(self))
9299 }
9300 #[inline(always)]
9301 fn from_vector_mask32x16(self, vector: i32x16<Self>) -> mask32x16<Self> {
9302 mask32x16 {
9303 val: vector.val,
9304 simd: self,
9305 }
9306 }
9307 #[inline(always)]
9308 fn to_vector_mask32x16(self, a: mask32x16<Self>) -> i32x16<Self> {
9309 i32x16 {
9310 val: a.val,
9311 simd: self,
9312 }
9313 }
9314 #[inline(always)]
9315 fn set_mask32x16(self, a: &mut mask32x16<Self>, index: usize, value: bool) -> () {
9316 assert!(
9317 index < 16usize,
9318 "mask lane index {index} is out of bounds for {} lanes",
9319 16usize
9320 );
9321 let mut lanes: [i32; 16usize] = (*a).into();
9322 lanes[index] = if value { !0 } else { 0 };
9323 *a = lanes.simd_into(self);
9324 }
9325 #[inline(always)]
9326 fn rotate_elements_left_mask32x16<const OFFSET: usize>(
9327 self,
9328 a: mask32x16<Self>,
9329 ) -> mask32x16<Self> {
9330 let int = i32x16 {
9331 val: crate::transmute::checked_transmute_copy(&a.val),
9332 simd: self,
9333 };
9334 let rotated = int.rotate_elements_left::<OFFSET>();
9335 mask32x16 {
9336 val: crate::transmute::checked_transmute_copy(&rotated.val),
9337 simd: self,
9338 }
9339 }
9340 #[inline(always)]
9341 fn rotate_elements_right_mask32x16<const OFFSET: usize>(
9342 self,
9343 a: mask32x16<Self>,
9344 ) -> mask32x16<Self> {
9345 let int = i32x16 {
9346 val: crate::transmute::checked_transmute_copy(&a.val),
9347 simd: self,
9348 };
9349 let rotated = int.rotate_elements_right::<OFFSET>();
9350 mask32x16 {
9351 val: crate::transmute::checked_transmute_copy(&rotated.val),
9352 simd: self,
9353 }
9354 }
9355 #[inline(always)]
9356 fn split_mask32x16(self, a: mask32x16<Self>) -> (mask32x8<Self>, mask32x8<Self>) {
9357 let mut b0 = [0; 8usize];
9358 let mut b1 = [0; 8usize];
9359 b0.copy_from_slice(&a.val.0[0..8usize]);
9360 b1.copy_from_slice(&a.val.0[8usize..16usize]);
9361 (b0.simd_into(self), b1.simd_into(self))
9362 }
9363 #[inline(always)]
9364 fn slide_f64x8<const SHIFT: usize>(self, a: f64x8<Self>, b: f64x8<Self>) -> f64x8<Self> {
9365 let mut dest = [Default::default(); 8usize];
9366 dest[..8usize - SHIFT].copy_from_slice(&a.val.0[SHIFT..]);
9367 dest[8usize - SHIFT..].copy_from_slice(&b.val.0[..SHIFT]);
9368 dest.simd_into(self)
9369 }
9370 #[inline(always)]
9371 fn compress_f64x8(self, values: f64x8<Self>, mask: mask64x8<Self>) -> f64x8<Self> {
9372 let mask = mask8x64 {
9373 val: crate::transmute::checked_transmute_copy(&mask.val),
9374 simd: self,
9375 };
9376 Bytes::from_bytes(self.compress_u8x64(Bytes::to_bytes(values), mask))
9377 }
9378 #[inline(always)]
9379 fn compress_merge_f64x8(
9380 self,
9381 values: f64x8<Self>,
9382 mask: mask64x8<Self>,
9383 merge: f64x8<Self>,
9384 ) -> f64x8<Self> {
9385 let mask = mask8x64 {
9386 val: crate::transmute::checked_transmute_copy(&mask.val),
9387 simd: self,
9388 };
9389 Bytes::from_bytes(self.compress_merge_u8x64(
9390 Bytes::to_bytes(values),
9391 mask,
9392 Bytes::to_bytes(merge),
9393 ))
9394 }
9395 #[inline(always)]
9396 fn expand_f64x8(self, values: f64x8<Self>, mask: mask64x8<Self>) -> f64x8<Self> {
9397 let mask = mask8x64 {
9398 val: crate::transmute::checked_transmute_copy(&mask.val),
9399 simd: self,
9400 };
9401 Bytes::from_bytes(self.expand_u8x64(Bytes::to_bytes(values), mask))
9402 }
9403 #[inline(always)]
9404 fn expand_merge_f64x8(
9405 self,
9406 values: f64x8<Self>,
9407 mask: mask64x8<Self>,
9408 merge: f64x8<Self>,
9409 ) -> f64x8<Self> {
9410 let mask = mask8x64 {
9411 val: crate::transmute::checked_transmute_copy(&mask.val),
9412 simd: self,
9413 };
9414 Bytes::from_bytes(self.expand_merge_u8x64(
9415 Bytes::to_bytes(values),
9416 mask,
9417 Bytes::to_bytes(merge),
9418 ))
9419 }
9420 #[inline(always)]
9421 fn is_nan_f64x8(self, a: f64x8<Self>) -> mask64x8<Self> {
9422 a.simd_ne(a)
9423 }
9424 #[inline(always)]
9425 fn is_infinite_f64x8(self, a: f64x8<Self>) -> mask64x8<Self> {
9426 a.abs().simd_eq(f64::INFINITY)
9427 }
9428 #[inline(always)]
9429 fn is_finite_f64x8(self, a: f64x8<Self>) -> mask64x8<Self> {
9430 a.abs().simd_lt(f64::INFINITY)
9431 }
9432 #[inline(always)]
9433 fn is_subnormal_f64x8(self, a: f64x8<Self>) -> mask64x8<Self> {
9434 let i: i64x8<Self> = a.bitcast();
9435 (i & 0x7FF0_0000_0000_0000_i64).simd_eq(0) & (i & 0x000F_FFFF_FFFF_FFFF_i64).simd_ne(0)
9436 }
9437 #[inline(always)]
9438 fn is_normal_f64x8(self, a: f64x8<Self>) -> mask64x8<Self> {
9439 let i: i64x8<Self> = a.bitcast();
9440 let exp = i & 0x7FF0_0000_0000_0000_i64;
9441 exp.simd_ne(0) & exp.simd_ne(0x7FF0_0000_0000_0000_i64)
9442 }
9443 #[inline(always)]
9444 fn is_sign_positive_f64x8(self, a: f64x8<Self>) -> mask64x8<Self> {
9445 let i: i64x8<Self> = a.bitcast();
9446 (i & i64::MIN).simd_eq(0)
9447 }
9448 #[inline(always)]
9449 fn is_sign_negative_f64x8(self, a: f64x8<Self>) -> mask64x8<Self> {
9450 !a.is_sign_positive()
9451 }
9452 #[inline(always)]
9453 fn split_f64x8(self, a: f64x8<Self>) -> (f64x4<Self>, f64x4<Self>) {
9454 let mut b0 = [0.0; 4usize];
9455 let mut b1 = [0.0; 4usize];
9456 b0.copy_from_slice(&a.val.0[0..4usize]);
9457 b1.copy_from_slice(&a.val.0[4usize..8usize]);
9458 (b0.simd_into(self), b1.simd_into(self))
9459 }
9460 #[inline(always)]
9461 fn slide_i64x8<const SHIFT: usize>(self, a: i64x8<Self>, b: i64x8<Self>) -> i64x8<Self> {
9462 let mut dest = [Default::default(); 8usize];
9463 dest[..8usize - SHIFT].copy_from_slice(&a.val.0[SHIFT..]);
9464 dest[8usize - SHIFT..].copy_from_slice(&b.val.0[..SHIFT]);
9465 dest.simd_into(self)
9466 }
9467 #[inline(always)]
9468 fn compress_i64x8(self, values: i64x8<Self>, mask: mask64x8<Self>) -> i64x8<Self> {
9469 let mask = mask8x64 {
9470 val: crate::transmute::checked_transmute_copy(&mask.val),
9471 simd: self,
9472 };
9473 Bytes::from_bytes(self.compress_u8x64(Bytes::to_bytes(values), mask))
9474 }
9475 #[inline(always)]
9476 fn compress_merge_i64x8(
9477 self,
9478 values: i64x8<Self>,
9479 mask: mask64x8<Self>,
9480 merge: i64x8<Self>,
9481 ) -> i64x8<Self> {
9482 let mask = mask8x64 {
9483 val: crate::transmute::checked_transmute_copy(&mask.val),
9484 simd: self,
9485 };
9486 Bytes::from_bytes(self.compress_merge_u8x64(
9487 Bytes::to_bytes(values),
9488 mask,
9489 Bytes::to_bytes(merge),
9490 ))
9491 }
9492 #[inline(always)]
9493 fn expand_i64x8(self, values: i64x8<Self>, mask: mask64x8<Self>) -> i64x8<Self> {
9494 let mask = mask8x64 {
9495 val: crate::transmute::checked_transmute_copy(&mask.val),
9496 simd: self,
9497 };
9498 Bytes::from_bytes(self.expand_u8x64(Bytes::to_bytes(values), mask))
9499 }
9500 #[inline(always)]
9501 fn expand_merge_i64x8(
9502 self,
9503 values: i64x8<Self>,
9504 mask: mask64x8<Self>,
9505 merge: i64x8<Self>,
9506 ) -> i64x8<Self> {
9507 let mask = mask8x64 {
9508 val: crate::transmute::checked_transmute_copy(&mask.val),
9509 simd: self,
9510 };
9511 Bytes::from_bytes(self.expand_merge_u8x64(
9512 Bytes::to_bytes(values),
9513 mask,
9514 Bytes::to_bytes(merge),
9515 ))
9516 }
9517 #[inline(always)]
9518 fn split_i64x8(self, a: i64x8<Self>) -> (i64x4<Self>, i64x4<Self>) {
9519 let mut b0 = [0; 4usize];
9520 let mut b1 = [0; 4usize];
9521 b0.copy_from_slice(&a.val.0[0..4usize]);
9522 b1.copy_from_slice(&a.val.0[4usize..8usize]);
9523 (b0.simd_into(self), b1.simd_into(self))
9524 }
9525 #[inline(always)]
9526 fn slide_u64x8<const SHIFT: usize>(self, a: u64x8<Self>, b: u64x8<Self>) -> u64x8<Self> {
9527 let mut dest = [Default::default(); 8usize];
9528 dest[..8usize - SHIFT].copy_from_slice(&a.val.0[SHIFT..]);
9529 dest[8usize - SHIFT..].copy_from_slice(&b.val.0[..SHIFT]);
9530 dest.simd_into(self)
9531 }
9532 #[inline(always)]
9533 fn compress_u64x8(self, values: u64x8<Self>, mask: mask64x8<Self>) -> u64x8<Self> {
9534 let mask = mask8x64 {
9535 val: crate::transmute::checked_transmute_copy(&mask.val),
9536 simd: self,
9537 };
9538 Bytes::from_bytes(self.compress_u8x64(Bytes::to_bytes(values), mask))
9539 }
9540 #[inline(always)]
9541 fn compress_merge_u64x8(
9542 self,
9543 values: u64x8<Self>,
9544 mask: mask64x8<Self>,
9545 merge: u64x8<Self>,
9546 ) -> u64x8<Self> {
9547 let mask = mask8x64 {
9548 val: crate::transmute::checked_transmute_copy(&mask.val),
9549 simd: self,
9550 };
9551 Bytes::from_bytes(self.compress_merge_u8x64(
9552 Bytes::to_bytes(values),
9553 mask,
9554 Bytes::to_bytes(merge),
9555 ))
9556 }
9557 #[inline(always)]
9558 fn expand_u64x8(self, values: u64x8<Self>, mask: mask64x8<Self>) -> u64x8<Self> {
9559 let mask = mask8x64 {
9560 val: crate::transmute::checked_transmute_copy(&mask.val),
9561 simd: self,
9562 };
9563 Bytes::from_bytes(self.expand_u8x64(Bytes::to_bytes(values), mask))
9564 }
9565 #[inline(always)]
9566 fn expand_merge_u64x8(
9567 self,
9568 values: u64x8<Self>,
9569 mask: mask64x8<Self>,
9570 merge: u64x8<Self>,
9571 ) -> u64x8<Self> {
9572 let mask = mask8x64 {
9573 val: crate::transmute::checked_transmute_copy(&mask.val),
9574 simd: self,
9575 };
9576 Bytes::from_bytes(self.expand_merge_u8x64(
9577 Bytes::to_bytes(values),
9578 mask,
9579 Bytes::to_bytes(merge),
9580 ))
9581 }
9582 #[inline(always)]
9583 fn split_u64x8(self, a: u64x8<Self>) -> (u64x4<Self>, u64x4<Self>) {
9584 let mut b0 = [0; 4usize];
9585 let mut b1 = [0; 4usize];
9586 b0.copy_from_slice(&a.val.0[0..4usize]);
9587 b1.copy_from_slice(&a.val.0[4usize..8usize]);
9588 (b0.simd_into(self), b1.simd_into(self))
9589 }
9590 #[inline(always)]
9591 fn from_vector_mask64x8(self, vector: i64x8<Self>) -> mask64x8<Self> {
9592 mask64x8 {
9593 val: vector.val,
9594 simd: self,
9595 }
9596 }
9597 #[inline(always)]
9598 fn to_vector_mask64x8(self, a: mask64x8<Self>) -> i64x8<Self> {
9599 i64x8 {
9600 val: a.val,
9601 simd: self,
9602 }
9603 }
9604 #[inline(always)]
9605 fn set_mask64x8(self, a: &mut mask64x8<Self>, index: usize, value: bool) -> () {
9606 assert!(
9607 index < 8usize,
9608 "mask lane index {index} is out of bounds for {} lanes",
9609 8usize
9610 );
9611 let mut lanes: [i64; 8usize] = (*a).into();
9612 lanes[index] = if value { !0 } else { 0 };
9613 *a = lanes.simd_into(self);
9614 }
9615 #[inline(always)]
9616 fn rotate_elements_left_mask64x8<const OFFSET: usize>(
9617 self,
9618 a: mask64x8<Self>,
9619 ) -> mask64x8<Self> {
9620 let int = i64x8 {
9621 val: crate::transmute::checked_transmute_copy(&a.val),
9622 simd: self,
9623 };
9624 let rotated = int.rotate_elements_left::<OFFSET>();
9625 mask64x8 {
9626 val: crate::transmute::checked_transmute_copy(&rotated.val),
9627 simd: self,
9628 }
9629 }
9630 #[inline(always)]
9631 fn rotate_elements_right_mask64x8<const OFFSET: usize>(
9632 self,
9633 a: mask64x8<Self>,
9634 ) -> mask64x8<Self> {
9635 let int = i64x8 {
9636 val: crate::transmute::checked_transmute_copy(&a.val),
9637 simd: self,
9638 };
9639 let rotated = int.rotate_elements_right::<OFFSET>();
9640 mask64x8 {
9641 val: crate::transmute::checked_transmute_copy(&rotated.val),
9642 simd: self,
9643 }
9644 }
9645 #[inline(always)]
9646 fn split_mask64x8(self, a: mask64x8<Self>) -> (mask64x4<Self>, mask64x4<Self>) {
9647 let mut b0 = [0; 4usize];
9648 let mut b1 = [0; 4usize];
9649 b0.copy_from_slice(&a.val.0[0..4usize]);
9650 b1.copy_from_slice(&a.val.0[4usize..8usize]);
9651 (b0.simd_into(self), b1.simd_into(self))
9652 }
9653}