1use core::arch::x86_64::*;
3use crate::{Simd, has_avx2};
4
5#[derive(Clone, Copy)]
9pub struct Sse2;
10
11#[inline(always)]
12fn chk(b: &[u8], n: usize) {
13 assert!(b.len() >= n, "simd: slice too short");
14}
15
16macro_rules! sse {
19 ($e:expr) => {
20 unsafe { $e }
21 };
22}
23
24impl Simd for Sse2 {
25 type V = __m128i;
26 const BYTES: usize = 16;
27 const PX: usize = 2;
28
29 #[inline(always)]
30 fn zero(self) -> __m128i {
31 sse!(_mm_setzero_si128())
32 }
33 #[inline(always)]
34 fn splat8(self, v: u8) -> __m128i {
35 sse!(_mm_set1_epi8(v as i8))
36 }
37 #[inline(always)]
38 fn splat16(self, v: u16) -> __m128i {
39 sse!(_mm_set1_epi16(v as i16))
40 }
41 #[inline(always)]
42 fn splat32(self, v: u32) -> __m128i {
43 sse!(_mm_set1_epi32(v as i32))
44 }
45 #[inline(always)]
46 fn splat64(self, v: u64) -> __m128i {
47 sse!(_mm_set1_epi64x(v as i64))
48 }
49 #[inline(always)]
50 fn splatf(self, v: f32) -> __m128i {
51 sse!(_mm_castps_si128(_mm_set1_ps(v)))
52 }
53
54 #[inline(always)]
55 fn load(self, b: &[u8]) -> __m128i {
56 chk(b, 16);
57 unsafe { _mm_loadu_si128(b.as_ptr() as *const __m128i) }
59 }
60 #[inline(always)]
61 fn store(self, b: &mut [u8], v: __m128i) {
62 chk(b, 16);
63 unsafe { _mm_storeu_si128(b.as_mut_ptr() as *mut __m128i, v) }
65 }
66 #[inline(always)]
67 fn load_px(self, b: &[u8]) -> __m128i {
68 chk(b, 8);
69 let v = unsafe { _mm_loadl_epi64(b.as_ptr() as *const __m128i) };
71 sse!(_mm_unpacklo_epi8(v, _mm_setzero_si128()))
72 }
73 #[inline(always)]
74 fn store_px(self, b: &mut [u8], v: __m128i) {
75 chk(b, 8);
76 let p = sse!(_mm_packus_epi16(v, v));
77 unsafe { _mm_storel_epi64(b.as_mut_ptr() as *mut __m128i, p) }
79 }
80 #[inline(always)]
81 fn load_raw(self, b: &[u8]) -> __m128i {
82 chk(b, 8);
83 unsafe { _mm_loadl_epi64(b.as_ptr() as *const __m128i) }
85 }
86 #[inline(always)]
87 fn store_raw(self, b: &mut [u8], v: __m128i) {
88 chk(b, 8);
89 unsafe { _mm_storel_epi64(b.as_mut_ptr() as *mut __m128i, v) }
91 }
92 #[inline(always)]
93 fn widen_px(self, v: __m128i) -> __m128i {
94 sse!(_mm_unpacklo_epi8(v, _mm_setzero_si128()))
95 }
96 #[inline(always)]
97 fn load_lane32(self, b: &[u8]) -> __m128i {
98 chk(b, 4);
99 sse!(_mm_cvtsi32_si128(u32::from_le_bytes([b[0], b[1], b[2], b[3]]) as i32))
100 }
101 #[inline(always)]
102 fn load_i32(self, b: &[i32]) -> __m128i {
103 assert!(b.len() >= 4, "simd: slice too short");
104 unsafe { _mm_loadu_si128(b.as_ptr() as *const __m128i) }
106 }
107 #[inline(always)]
108 fn store_i32(self, b: &mut [i32], v: __m128i) {
109 assert!(b.len() >= 4, "simd: slice too short");
110 unsafe { _mm_storeu_si128(b.as_mut_ptr() as *mut __m128i, v) }
112 }
113 #[inline(always)]
114 fn from_i32(self, v: i32) -> __m128i {
115 sse!(_mm_cvtsi32_si128(v))
116 }
117 #[inline(always)]
118 fn to_i32(self, v: __m128i) -> i32 {
119 sse!(_mm_cvtsi128_si32(v))
120 }
121 #[inline(always)]
122 fn bcast_px(self, b: &[u8]) -> __m128i {
123 chk(b, 2);
124 let v = sse!(_mm_cvtsi32_si128(u16::from_le_bytes([b[0], b[1]]) as i32));
125 let v = sse!(_mm_unpacklo_epi8(v, v));
126 let v = sse!(_mm_unpacklo_epi16(v, v));
127 sse!(_mm_unpacklo_epi8(v, _mm_setzero_si128()))
128 }
129
130 #[inline(always)]
131 fn add8(self, a: __m128i, b: __m128i) -> __m128i {
132 sse!(_mm_add_epi8(a, b))
133 }
134 #[inline(always)]
135 fn sub8(self, a: __m128i, b: __m128i) -> __m128i {
136 sse!(_mm_sub_epi8(a, b))
137 }
138 #[inline(always)]
139 fn addus8(self, a: __m128i, b: __m128i) -> __m128i {
140 sse!(_mm_adds_epu8(a, b))
141 }
142 #[inline(always)]
143 fn subus8(self, a: __m128i, b: __m128i) -> __m128i {
144 sse!(_mm_subs_epu8(a, b))
145 }
146 #[inline(always)]
147 fn maxu8(self, a: __m128i, b: __m128i) -> __m128i {
148 sse!(_mm_max_epu8(a, b))
149 }
150 #[inline(always)]
151 fn minu8(self, a: __m128i, b: __m128i) -> __m128i {
152 sse!(_mm_min_epu8(a, b))
153 }
154 #[inline(always)]
155 fn cmpeq8(self, a: __m128i, b: __m128i) -> __m128i {
156 sse!(_mm_cmpeq_epi8(a, b))
157 }
158 #[inline(always)]
159 fn avgu8(self, a: __m128i, b: __m128i) -> __m128i {
160 sse!(_mm_avg_epu8(a, b))
161 }
162 #[inline(always)]
163 fn movemask8(self, a: __m128i) -> u32 {
164 sse!(_mm_movemask_epi8(a) as u32)
165 }
166 #[inline(always)]
167 fn unpacklo8(self, a: __m128i, b: __m128i) -> __m128i {
168 sse!(_mm_unpacklo_epi8(a, b))
169 }
170 #[inline(always)]
171 fn unpackhi8(self, a: __m128i, b: __m128i) -> __m128i {
172 sse!(_mm_unpackhi_epi8(a, b))
173 }
174 #[inline(always)]
175 fn packus16(self, a: __m128i, b: __m128i) -> __m128i {
176 sse!(_mm_packus_epi16(a, b))
177 }
178
179 #[inline(always)]
180 fn add16(self, a: __m128i, b: __m128i) -> __m128i {
181 sse!(_mm_add_epi16(a, b))
182 }
183 #[inline(always)]
184 fn sub16(self, a: __m128i, b: __m128i) -> __m128i {
185 sse!(_mm_sub_epi16(a, b))
186 }
187 #[inline(always)]
188 fn subus16(self, a: __m128i, b: __m128i) -> __m128i {
189 sse!(_mm_subs_epu16(a, b))
190 }
191 #[inline(always)]
192 fn mullo16(self, a: __m128i, b: __m128i) -> __m128i {
193 sse!(_mm_mullo_epi16(a, b))
194 }
195 #[inline(always)]
196 fn mulhiu16(self, a: __m128i, b: __m128i) -> __m128i {
197 sse!(_mm_mulhi_epu16(a, b))
198 }
199 #[inline(always)]
200 fn mins16(self, a: __m128i, b: __m128i) -> __m128i {
201 sse!(_mm_min_epi16(a, b))
202 }
203 #[inline(always)]
204 fn maxs16(self, a: __m128i, b: __m128i) -> __m128i {
205 sse!(_mm_max_epi16(a, b))
206 }
207 #[inline(always)]
208 fn cmpeq16(self, a: __m128i, b: __m128i) -> __m128i {
209 sse!(_mm_cmpeq_epi16(a, b))
210 }
211 #[inline(always)]
212 fn cmpgts16(self, a: __m128i, b: __m128i) -> __m128i {
213 sse!(_mm_cmpgt_epi16(a, b))
214 }
215 #[inline(always)]
216 fn srl16<const N: i32>(self, a: __m128i) -> __m128i {
217 sse!(_mm_srli_epi16::<N>(a))
218 }
219 #[inline(always)]
220 fn sll16<const N: i32>(self, a: __m128i) -> __m128i {
221 sse!(_mm_slli_epi16::<N>(a))
222 }
223 #[inline(always)]
224 fn shuf16<const IMM: i32>(self, a: __m128i) -> __m128i {
225 sse!(_mm_shufflehi_epi16::<IMM>(_mm_shufflelo_epi16::<IMM>(a)))
226 }
227 #[inline(always)]
228 fn unpacklo16(self, a: __m128i, b: __m128i) -> __m128i {
229 sse!(_mm_unpacklo_epi16(a, b))
230 }
231 #[inline(always)]
232 fn unpackhi16(self, a: __m128i, b: __m128i) -> __m128i {
233 sse!(_mm_unpackhi_epi16(a, b))
234 }
235 #[inline(always)]
236 fn packs32(self, a: __m128i, b: __m128i) -> __m128i {
237 sse!(_mm_packs_epi32(a, b))
238 }
239 #[inline(always)]
240 fn madd16(self, a: __m128i, b: __m128i) -> __m128i {
241 sse!(_mm_madd_epi16(a, b))
242 }
243
244 #[inline(always)]
245 fn add32(self, a: __m128i, b: __m128i) -> __m128i {
246 sse!(_mm_add_epi32(a, b))
247 }
248 #[inline(always)]
249 fn sub32(self, a: __m128i, b: __m128i) -> __m128i {
250 sse!(_mm_sub_epi32(a, b))
251 }
252 #[inline(always)]
253 fn cmpeq32(self, a: __m128i, b: __m128i) -> __m128i {
254 sse!(_mm_cmpeq_epi32(a, b))
255 }
256 #[inline(always)]
257 fn cmpgts32(self, a: __m128i, b: __m128i) -> __m128i {
258 sse!(_mm_cmpgt_epi32(a, b))
259 }
260 #[inline(always)]
261 fn movemask32(self, a: __m128i) -> u32 {
262 sse!(_mm_movemask_ps(_mm_castsi128_ps(a))) as u32
263 }
264 #[inline(always)]
265 fn shuf32<const IMM: i32>(self, a: __m128i) -> __m128i {
266 sse!(_mm_shuffle_epi32::<IMM>(a))
267 }
268 #[inline(always)]
269 fn shufps<const IMM: i32>(self, a: __m128i, b: __m128i) -> __m128i {
270 sse!(_mm_castps_si128(_mm_shuffle_ps::<IMM>(_mm_castsi128_ps(a), _mm_castsi128_ps(b))))
271 }
272 #[inline(always)]
273 fn srl32<const N: i32>(self, a: __m128i) -> __m128i {
274 sse!(_mm_srli_epi32::<N>(a))
275 }
276 #[inline(always)]
277 fn sll32<const N: i32>(self, a: __m128i) -> __m128i {
278 sse!(_mm_slli_epi32::<N>(a))
279 }
280 #[inline(always)]
281 fn sra32<const N: i32>(self, a: __m128i) -> __m128i {
282 sse!(_mm_srai_epi32::<N>(a))
283 }
284 #[inline(always)]
285 fn bsll<const N: i32>(self, a: __m128i) -> __m128i {
286 sse!(_mm_slli_si128::<N>(a))
287 }
288 #[inline(always)]
289 fn bsrl<const N: i32>(self, a: __m128i) -> __m128i {
290 sse!(_mm_srli_si128::<N>(a))
291 }
292
293 #[inline(always)]
294 fn and(self, a: __m128i, b: __m128i) -> __m128i {
295 sse!(_mm_and_si128(a, b))
296 }
297 #[inline(always)]
298 fn andnot(self, a: __m128i, b: __m128i) -> __m128i {
299 sse!(_mm_andnot_si128(a, b))
300 }
301 #[inline(always)]
302 fn or(self, a: __m128i, b: __m128i) -> __m128i {
303 sse!(_mm_or_si128(a, b))
304 }
305 #[inline(always)]
306 fn xor(self, a: __m128i, b: __m128i) -> __m128i {
307 sse!(_mm_xor_si128(a, b))
308 }
309
310 #[inline(always)]
311 fn cvt_i32_f32(self, a: __m128i) -> __m128i {
312 sse!(_mm_castps_si128(_mm_cvtepi32_ps(a)))
313 }
314 #[inline(always)]
315 fn cvtt_f32_i32(self, a: __m128i) -> __m128i {
316 sse!(_mm_cvttps_epi32(_mm_castsi128_ps(a)))
317 }
318 #[inline(always)]
319 fn cvtr_f32_i32(self, a: __m128i) -> __m128i {
320 sse!(_mm_cvtps_epi32(_mm_castsi128_ps(a)))
321 }
322 #[inline(always)]
323 fn addf(self, a: __m128i, b: __m128i) -> __m128i {
324 sse!(_mm_castps_si128(_mm_add_ps(_mm_castsi128_ps(a), _mm_castsi128_ps(b))))
325 }
326 #[inline(always)]
327 fn subf(self, a: __m128i, b: __m128i) -> __m128i {
328 sse!(_mm_castps_si128(_mm_sub_ps(_mm_castsi128_ps(a), _mm_castsi128_ps(b))))
329 }
330 #[inline(always)]
331 fn mulf(self, a: __m128i, b: __m128i) -> __m128i {
332 sse!(_mm_castps_si128(_mm_mul_ps(_mm_castsi128_ps(a), _mm_castsi128_ps(b))))
333 }
334 #[inline(always)]
335 fn divf(self, a: __m128i, b: __m128i) -> __m128i {
336 sse!(_mm_castps_si128(_mm_div_ps(_mm_castsi128_ps(a), _mm_castsi128_ps(b))))
337 }
338
339 #[inline(always)]
340 fn splatd(self, v: f64) -> __m128i {
341 sse!(_mm_castpd_si128(_mm_set1_pd(v)))
342 }
343 #[inline(always)]
344 fn addd(self, a: __m128i, b: __m128i) -> __m128i {
345 sse!(_mm_castpd_si128(_mm_add_pd(_mm_castsi128_pd(a), _mm_castsi128_pd(b))))
346 }
347 #[inline(always)]
348 fn subd(self, a: __m128i, b: __m128i) -> __m128i {
349 sse!(_mm_castpd_si128(_mm_sub_pd(_mm_castsi128_pd(a), _mm_castsi128_pd(b))))
350 }
351 #[inline(always)]
352 fn muld(self, a: __m128i, b: __m128i) -> __m128i {
353 sse!(_mm_castpd_si128(_mm_mul_pd(_mm_castsi128_pd(a), _mm_castsi128_pd(b))))
354 }
355 #[inline(always)]
356 fn divd(self, a: __m128i, b: __m128i) -> __m128i {
357 sse!(_mm_castpd_si128(_mm_div_pd(_mm_castsi128_pd(a), _mm_castsi128_pd(b))))
358 }
359 #[inline(always)]
360 fn sqrtd(self, a: __m128i) -> __m128i {
361 sse!(_mm_castpd_si128(_mm_sqrt_pd(_mm_castsi128_pd(a))))
362 }
363 #[inline(always)]
364 fn maxd(self, a: __m128i, b: __m128i) -> __m128i {
365 sse!(_mm_castpd_si128(_mm_max_pd(_mm_castsi128_pd(a), _mm_castsi128_pd(b))))
366 }
367 #[inline(always)]
368 fn mind(self, a: __m128i, b: __m128i) -> __m128i {
369 sse!(_mm_castpd_si128(_mm_min_pd(_mm_castsi128_pd(a), _mm_castsi128_pd(b))))
370 }
371 #[inline(always)]
372 fn cmpltd(self, a: __m128i, b: __m128i) -> __m128i {
373 sse!(_mm_castpd_si128(_mm_cmplt_pd(_mm_castsi128_pd(a), _mm_castsi128_pd(b))))
374 }
375 #[inline(always)]
376 fn cvt_i32_f64(self, a: __m128i) -> __m128i {
377 sse!(_mm_castpd_si128(_mm_cvtepi32_pd(a)))
378 }
379 #[inline(always)]
380 fn cvtt_f64_i32(self, a: __m128i) -> __m128i {
381 sse!(_mm_cvttpd_epi32(_mm_castsi128_pd(a)))
382 }
383 #[inline(always)]
384 fn store_i32_half(self, b: &mut [i32], v: __m128i) {
385 assert!(b.len() >= 2, "simd: slice too short");
386 unsafe { _mm_storel_epi64(b.as_mut_ptr() as *mut __m128i, v) }
388 }
389
390 #[inline(always)]
392 fn sra16<const N: i32>(self, a: __m128i) -> __m128i {
393 sse!(_mm_srai_epi16::<N>(a))
394 }
395 #[inline(always)]
396 fn mulhis16(self, a: __m128i, b: __m128i) -> __m128i {
397 sse!(_mm_mulhi_epi16(a, b))
398 }
399 #[inline(always)]
400 fn packs16(self, a: __m128i, b: __m128i) -> __m128i {
401 sse!(_mm_packs_epi16(a, b))
402 }
403 #[inline(always)]
404 fn load_i16(self, b: &[i16]) -> __m128i {
405 assert!(b.len() >= 8, "simd: slice too short");
406 unsafe { _mm_loadu_si128(b.as_ptr() as *const __m128i) }
408 }
409 #[inline(always)]
410 fn store_i16(self, b: &mut [i16], v: __m128i) {
411 assert!(b.len() >= 8, "simd: slice too short");
412 unsafe { _mm_storeu_si128(b.as_mut_ptr() as *mut __m128i, v) }
414 }
415 #[inline(always)]
416 fn store_lanes2(self, b: &mut [u8], lo: __m128i, hi: __m128i) {
417 chk(b, 32);
418 unsafe {
420 _mm_storeu_si128(b.as_mut_ptr() as *mut __m128i, lo);
421 _mm_storeu_si128(b.as_mut_ptr().add(16) as *mut __m128i, hi);
422 }
423 }
424 #[inline(always)]
425 fn load_lanes2(self, b: &[u8]) -> (__m128i, __m128i) {
426 chk(b, 32);
427 unsafe {
429 (_mm_loadu_si128(b.as_ptr() as *const __m128i), _mm_loadu_si128(b.as_ptr().add(16) as *const __m128i))
430 }
431 }
432 #[inline(always)]
433 fn load_i16_lanes(self, a: &[i16], _b: &[i16]) -> __m128i {
434 self.load_i16(a)
435 }
436 #[inline(always)]
437 fn store_raw_lanes(self, a: &mut [u8], _b: &mut [u8], v: __m128i) {
438 self.store_raw(a, v)
439 }
440 #[inline(always)]
441 fn sadu8(self, a: __m128i, b: __m128i) -> __m128i {
442 sse!(_mm_sad_epu8(a, b))
443 }
444 #[inline(always)]
445 fn add64(self, a: __m128i, b: __m128i) -> __m128i {
446 sse!(_mm_add_epi64(a, b))
447 }
448
449 #[inline(always)]
451 fn maxf(self, a: __m128i, b: __m128i) -> __m128i {
452 sse!(_mm_castps_si128(_mm_max_ps(_mm_castsi128_ps(a), _mm_castsi128_ps(b))))
453 }
454 #[inline(always)]
455 fn minf(self, a: __m128i, b: __m128i) -> __m128i {
456 sse!(_mm_castps_si128(_mm_min_ps(_mm_castsi128_ps(a), _mm_castsi128_ps(b))))
457 }
458 #[inline(always)]
459 fn sqrtf(self, a: __m128i) -> __m128i {
460 sse!(_mm_castps_si128(_mm_sqrt_ps(_mm_castsi128_ps(a))))
461 }
462
463 #[inline(always)]
465 fn load_u16(self, b: &[u16]) -> __m128i {
466 assert!(b.len() >= 8, "simd: slice too short");
467 unsafe { _mm_loadu_si128(b.as_ptr() as *const __m128i) }
469 }
470 #[inline(always)]
471 fn store_u16(self, b: &mut [u16], v: __m128i) {
472 assert!(b.len() >= 8, "simd: slice too short");
473 unsafe { _mm_storeu_si128(b.as_mut_ptr() as *mut __m128i, v) }
475 }
476 #[inline(always)]
477 fn load_u16_lo(self, b: &[u16]) -> __m128i {
478 assert!(b.len() >= 4, "simd: slice too short");
479 unsafe { _mm_loadl_epi64(b.as_ptr() as *const __m128i) }
481 }
482 #[inline(always)]
483 fn store_u16_lo(self, b: &mut [u16], v: __m128i) {
484 assert!(b.len() >= 4, "simd: slice too short");
485 unsafe { _mm_storel_epi64(b.as_mut_ptr() as *mut __m128i, v) }
487 }
488 #[inline(always)]
489 fn srl16x(self, a: __m128i, n: u32) -> __m128i {
490 sse!(_mm_srl_epi16(a, _mm_cvtsi32_si128(n as i32)))
491 }
492
493 #[inline(always)]
495 fn load_f32(self, b: &[f32]) -> __m128i {
496 assert!(b.len() >= 4, "simd: slice too short");
497 unsafe { _mm_castps_si128(_mm_loadu_ps(b.as_ptr())) }
499 }
500 #[inline(always)]
501 fn store_f32(self, b: &mut [f32], v: __m128i) {
502 assert!(b.len() >= 4, "simd: slice too short");
503 unsafe { _mm_storeu_ps(b.as_mut_ptr(), _mm_castsi128_ps(v)) }
505 }
506
507 #[inline(always)]
509 fn adds8(self, a: __m128i, b: __m128i) -> __m128i {
510 sse!(_mm_adds_epi8(a, b))
511 }
512 #[inline(always)]
513 fn subs8(self, a: __m128i, b: __m128i) -> __m128i {
514 sse!(_mm_subs_epi8(a, b))
515 }
516
517 #[inline(always)]
519 fn cmpltf(self, a: __m128i, b: __m128i) -> __m128i {
520 sse!(_mm_castps_si128(_mm_cmplt_ps(_mm_castsi128_ps(a), _mm_castsi128_ps(b))))
521 }
522
523 #[inline(always)]
525 fn load_f32_lanes(self, a: &[f32], _b: &[f32]) -> __m128i {
526 self.load_f32(a)
527 }
528 #[inline(always)]
529 fn store_f32_lanes(self, a: &mut [f32], _b: &mut [f32], v: __m128i) {
530 self.store_f32(a, v)
531 }
532
533 #[inline(always)]
535 fn packssdw(self, a: __m128i, b: __m128i) -> __m128i {
536 sse!(_mm_packs_epi32(a, b))
537 }
538 #[inline(always)]
539 fn packuswb_px(self, v: __m128i) -> __m128i {
540 sse!(_mm_packus_epi16(v, v))
541 }
542 #[inline(always)]
543 fn ptest_raw(self, a: __m128i, b: __m128i) -> u32 {
544 let z = sse!(_mm_movemask_epi8(_mm_cmpeq_epi8(_mm_and_si128(a, b), _mm_setzero_si128()))) & 0xFF == 0xFF;
545 let c = sse!(_mm_movemask_epi8(_mm_cmpeq_epi8(_mm_andnot_si128(a, b), _mm_setzero_si128()))) & 0xFF == 0xFF;
546 if z {
547 1
548 } else if c {
549 2
550 } else {
551 0
552 }
553 }
554 #[inline(always)]
555 fn ptestz_raw(self, a: __m128i, b: __m128i) -> bool {
556 sse!(_mm_movemask_epi8(_mm_cmpeq_epi8(_mm_and_si128(a, b), _mm_setzero_si128()))) & 0xFF == 0xFF
557 }
558}
559
560#[derive(Clone, Copy)]
564pub struct Avx2(());
565
566impl Avx2 {
567 #[inline]
568 pub fn new() -> Option<Avx2> {
569 has_avx2().then_some(Avx2(()))
570 }
571}
572
573macro_rules! avx {
576 ($e:expr) => {
577 unsafe { $e }
578 };
579}
580
581impl Simd for Avx2 {
582 type V = __m256i;
583 const BYTES: usize = 32;
584 const PX: usize = 4;
585
586 #[inline(always)]
587 fn zero(self) -> __m256i {
588 avx!(_mm256_setzero_si256())
589 }
590 #[inline(always)]
591 fn splat8(self, v: u8) -> __m256i {
592 avx!(_mm256_set1_epi8(v as i8))
593 }
594 #[inline(always)]
595 fn splat16(self, v: u16) -> __m256i {
596 avx!(_mm256_set1_epi16(v as i16))
597 }
598 #[inline(always)]
599 fn splat32(self, v: u32) -> __m256i {
600 avx!(_mm256_set1_epi32(v as i32))
601 }
602 #[inline(always)]
603 fn splat64(self, v: u64) -> __m256i {
604 avx!(_mm256_set1_epi64x(v as i64))
605 }
606 #[inline(always)]
607 fn splatf(self, v: f32) -> __m256i {
608 avx!(_mm256_castps_si256(_mm256_set1_ps(v)))
609 }
610
611 #[inline(always)]
612 fn load(self, b: &[u8]) -> __m256i {
613 chk(b, 32);
614 unsafe { _mm256_loadu_si256(b.as_ptr() as *const __m256i) }
616 }
617 #[inline(always)]
618 fn store(self, b: &mut [u8], v: __m256i) {
619 chk(b, 32);
620 unsafe { _mm256_storeu_si256(b.as_mut_ptr() as *mut __m256i, v) }
622 }
623 #[inline(always)]
624 fn load_px(self, b: &[u8]) -> __m256i {
625 chk(b, 16);
626 unsafe { _mm256_cvtepu8_epi16(_mm_loadu_si128(b.as_ptr() as *const __m128i)) }
628 }
629 #[inline(always)]
630 fn store_px(self, b: &mut [u8], v: __m256i) {
631 chk(b, 16);
632 unsafe {
634 let p = _mm256_permute4x64_epi64::<0x08>(_mm256_packus_epi16(v, v));
635 _mm_storeu_si128(b.as_mut_ptr() as *mut __m128i, _mm256_castsi256_si128(p))
636 }
637 }
638 #[inline(always)]
639 fn load_raw(self, b: &[u8]) -> __m256i {
640 chk(b, 16);
641 unsafe { _mm256_zextsi128_si256(_mm_loadu_si128(b.as_ptr() as *const __m128i)) }
643 }
644 #[inline(always)]
645 fn store_raw(self, b: &mut [u8], v: __m256i) {
646 chk(b, 16);
647 unsafe { _mm_storeu_si128(b.as_mut_ptr() as *mut __m128i, _mm256_castsi256_si128(v)) }
649 }
650 #[inline(always)]
651 fn widen_px(self, v: __m256i) -> __m256i {
652 avx!(_mm256_cvtepu8_epi16(_mm256_castsi256_si128(v)))
653 }
654 #[inline(always)]
655 fn load_lane32(self, b: &[u8]) -> __m256i {
656 chk(b, 8);
657 let d = |k: usize| u32::from_le_bytes([b[k], b[k + 1], b[k + 2], b[k + 3]]) as i32;
658 avx!(_mm256_setr_epi32(d(0), 0, 0, 0, d(4), 0, 0, 0))
659 }
660 #[inline(always)]
661 fn load_i32(self, b: &[i32]) -> __m256i {
662 assert!(b.len() >= 8, "simd: slice too short");
663 unsafe { _mm256_loadu_si256(b.as_ptr() as *const __m256i) }
665 }
666 #[inline(always)]
667 fn store_i32(self, b: &mut [i32], v: __m256i) {
668 assert!(b.len() >= 8, "simd: slice too short");
669 unsafe { _mm256_storeu_si256(b.as_mut_ptr() as *mut __m256i, v) }
671 }
672 #[inline(always)]
673 fn from_i32(self, v: i32) -> __m256i {
674 avx!(_mm256_zextsi128_si256(_mm_cvtsi32_si128(v)))
675 }
676 #[inline(always)]
677 fn to_i32(self, v: __m256i) -> i32 {
678 avx!(_mm_cvtsi128_si32(_mm256_castsi256_si128(v)))
679 }
680 #[inline(always)]
681 fn bcast_px(self, b: &[u8]) -> __m256i {
682 chk(b, 4);
683 let v = avx!(_mm_cvtsi32_si128(u32::from_le_bytes([b[0], b[1], b[2], b[3]]) as i32));
684 let v = avx!(_mm_unpacklo_epi8(v, v));
685 let v = avx!(_mm_unpacklo_epi16(v, v));
686 avx!(_mm256_cvtepu8_epi16(v))
687 }
688
689 #[inline(always)]
690 fn add8(self, a: __m256i, b: __m256i) -> __m256i {
691 avx!(_mm256_add_epi8(a, b))
692 }
693 #[inline(always)]
694 fn sub8(self, a: __m256i, b: __m256i) -> __m256i {
695 avx!(_mm256_sub_epi8(a, b))
696 }
697 #[inline(always)]
698 fn addus8(self, a: __m256i, b: __m256i) -> __m256i {
699 avx!(_mm256_adds_epu8(a, b))
700 }
701 #[inline(always)]
702 fn subus8(self, a: __m256i, b: __m256i) -> __m256i {
703 avx!(_mm256_subs_epu8(a, b))
704 }
705 #[inline(always)]
706 fn maxu8(self, a: __m256i, b: __m256i) -> __m256i {
707 avx!(_mm256_max_epu8(a, b))
708 }
709 #[inline(always)]
710 fn minu8(self, a: __m256i, b: __m256i) -> __m256i {
711 avx!(_mm256_min_epu8(a, b))
712 }
713 #[inline(always)]
714 fn cmpeq8(self, a: __m256i, b: __m256i) -> __m256i {
715 avx!(_mm256_cmpeq_epi8(a, b))
716 }
717 #[inline(always)]
718 fn avgu8(self, a: __m256i, b: __m256i) -> __m256i {
719 avx!(_mm256_avg_epu8(a, b))
720 }
721 #[inline(always)]
722 fn movemask8(self, a: __m256i) -> u32 {
723 avx!(_mm256_movemask_epi8(a)) as u32
724 }
725 #[inline(always)]
726 fn unpacklo8(self, a: __m256i, b: __m256i) -> __m256i {
727 avx!(_mm256_unpacklo_epi8(a, b))
728 }
729 #[inline(always)]
730 fn unpackhi8(self, a: __m256i, b: __m256i) -> __m256i {
731 avx!(_mm256_unpackhi_epi8(a, b))
732 }
733 #[inline(always)]
734 fn packus16(self, a: __m256i, b: __m256i) -> __m256i {
735 avx!(_mm256_packus_epi16(a, b))
736 }
737
738 #[inline(always)]
739 fn add16(self, a: __m256i, b: __m256i) -> __m256i {
740 avx!(_mm256_add_epi16(a, b))
741 }
742 #[inline(always)]
743 fn sub16(self, a: __m256i, b: __m256i) -> __m256i {
744 avx!(_mm256_sub_epi16(a, b))
745 }
746 #[inline(always)]
747 fn subus16(self, a: __m256i, b: __m256i) -> __m256i {
748 avx!(_mm256_subs_epu16(a, b))
749 }
750 #[inline(always)]
751 fn mullo16(self, a: __m256i, b: __m256i) -> __m256i {
752 avx!(_mm256_mullo_epi16(a, b))
753 }
754 #[inline(always)]
755 fn mulhiu16(self, a: __m256i, b: __m256i) -> __m256i {
756 avx!(_mm256_mulhi_epu16(a, b))
757 }
758 #[inline(always)]
759 fn mins16(self, a: __m256i, b: __m256i) -> __m256i {
760 avx!(_mm256_min_epi16(a, b))
761 }
762 #[inline(always)]
763 fn maxs16(self, a: __m256i, b: __m256i) -> __m256i {
764 avx!(_mm256_max_epi16(a, b))
765 }
766 #[inline(always)]
767 fn cmpeq16(self, a: __m256i, b: __m256i) -> __m256i {
768 avx!(_mm256_cmpeq_epi16(a, b))
769 }
770 #[inline(always)]
771 fn cmpgts16(self, a: __m256i, b: __m256i) -> __m256i {
772 avx!(_mm256_cmpgt_epi16(a, b))
773 }
774 #[inline(always)]
775 fn srl16<const N: i32>(self, a: __m256i) -> __m256i {
776 avx!(_mm256_srli_epi16::<N>(a))
777 }
778 #[inline(always)]
779 fn sll16<const N: i32>(self, a: __m256i) -> __m256i {
780 avx!(_mm256_slli_epi16::<N>(a))
781 }
782 #[inline(always)]
783 fn shuf16<const IMM: i32>(self, a: __m256i) -> __m256i {
784 avx!(_mm256_shufflehi_epi16::<IMM>(_mm256_shufflelo_epi16::<IMM>(a)))
785 }
786 #[inline(always)]
787 fn unpacklo16(self, a: __m256i, b: __m256i) -> __m256i {
788 avx!(_mm256_unpacklo_epi16(a, b))
789 }
790 #[inline(always)]
791 fn unpackhi16(self, a: __m256i, b: __m256i) -> __m256i {
792 avx!(_mm256_unpackhi_epi16(a, b))
793 }
794 #[inline(always)]
795 fn packs32(self, a: __m256i, b: __m256i) -> __m256i {
796 avx!(_mm256_packs_epi32(a, b))
797 }
798 #[inline(always)]
799 fn madd16(self, a: __m256i, b: __m256i) -> __m256i {
800 avx!(_mm256_madd_epi16(a, b))
801 }
802
803 #[inline(always)]
804 fn add32(self, a: __m256i, b: __m256i) -> __m256i {
805 avx!(_mm256_add_epi32(a, b))
806 }
807 #[inline(always)]
808 fn sub32(self, a: __m256i, b: __m256i) -> __m256i {
809 avx!(_mm256_sub_epi32(a, b))
810 }
811 #[inline(always)]
812 fn cmpeq32(self, a: __m256i, b: __m256i) -> __m256i {
813 avx!(_mm256_cmpeq_epi32(a, b))
814 }
815 #[inline(always)]
816 fn cmpgts32(self, a: __m256i, b: __m256i) -> __m256i {
817 avx!(_mm256_cmpgt_epi32(a, b))
818 }
819 #[inline(always)]
820 fn movemask32(self, a: __m256i) -> u32 {
821 avx!(_mm256_movemask_ps(_mm256_castsi256_ps(a))) as u32
822 }
823 #[inline(always)]
824 fn shuf32<const IMM: i32>(self, a: __m256i) -> __m256i {
825 avx!(_mm256_shuffle_epi32::<IMM>(a))
826 }
827 #[inline(always)]
828 fn shufps<const IMM: i32>(self, a: __m256i, b: __m256i) -> __m256i {
829 avx!(_mm256_castps_si256(_mm256_shuffle_ps::<IMM>(_mm256_castsi256_ps(a), _mm256_castsi256_ps(b))))
830 }
831 #[inline(always)]
832 fn srl32<const N: i32>(self, a: __m256i) -> __m256i {
833 avx!(_mm256_srli_epi32::<N>(a))
834 }
835 #[inline(always)]
836 fn sll32<const N: i32>(self, a: __m256i) -> __m256i {
837 avx!(_mm256_slli_epi32::<N>(a))
838 }
839 #[inline(always)]
840 fn sra32<const N: i32>(self, a: __m256i) -> __m256i {
841 avx!(_mm256_srai_epi32::<N>(a))
842 }
843 #[inline(always)]
844 fn bsll<const N: i32>(self, a: __m256i) -> __m256i {
845 avx!(_mm256_bslli_epi128::<N>(a))
846 }
847 #[inline(always)]
848 fn bsrl<const N: i32>(self, a: __m256i) -> __m256i {
849 avx!(_mm256_bsrli_epi128::<N>(a))
850 }
851
852 #[inline(always)]
853 fn and(self, a: __m256i, b: __m256i) -> __m256i {
854 avx!(_mm256_and_si256(a, b))
855 }
856 #[inline(always)]
857 fn andnot(self, a: __m256i, b: __m256i) -> __m256i {
858 avx!(_mm256_andnot_si256(a, b))
859 }
860 #[inline(always)]
861 fn or(self, a: __m256i, b: __m256i) -> __m256i {
862 avx!(_mm256_or_si256(a, b))
863 }
864 #[inline(always)]
865 fn xor(self, a: __m256i, b: __m256i) -> __m256i {
866 avx!(_mm256_xor_si256(a, b))
867 }
868
869 #[inline(always)]
870 fn cvt_i32_f32(self, a: __m256i) -> __m256i {
871 avx!(_mm256_castps_si256(_mm256_cvtepi32_ps(a)))
872 }
873 #[inline(always)]
874 fn cvtt_f32_i32(self, a: __m256i) -> __m256i {
875 avx!(_mm256_cvttps_epi32(_mm256_castsi256_ps(a)))
876 }
877 #[inline(always)]
878 fn cvtr_f32_i32(self, a: __m256i) -> __m256i {
879 avx!(_mm256_cvtps_epi32(_mm256_castsi256_ps(a)))
880 }
881 #[inline(always)]
882 fn addf(self, a: __m256i, b: __m256i) -> __m256i {
883 avx!(_mm256_castps_si256(_mm256_add_ps(_mm256_castsi256_ps(a), _mm256_castsi256_ps(b))))
884 }
885 #[inline(always)]
886 fn subf(self, a: __m256i, b: __m256i) -> __m256i {
887 avx!(_mm256_castps_si256(_mm256_sub_ps(_mm256_castsi256_ps(a), _mm256_castsi256_ps(b))))
888 }
889 #[inline(always)]
890 fn mulf(self, a: __m256i, b: __m256i) -> __m256i {
891 avx!(_mm256_castps_si256(_mm256_mul_ps(_mm256_castsi256_ps(a), _mm256_castsi256_ps(b))))
892 }
893 #[inline(always)]
894 fn divf(self, a: __m256i, b: __m256i) -> __m256i {
895 avx!(_mm256_castps_si256(_mm256_div_ps(_mm256_castsi256_ps(a), _mm256_castsi256_ps(b))))
896 }
897
898 #[inline(always)]
899 fn splatd(self, v: f64) -> __m256i {
900 avx!(_mm256_castpd_si256(_mm256_set1_pd(v)))
901 }
902 #[inline(always)]
903 fn addd(self, a: __m256i, b: __m256i) -> __m256i {
904 avx!(_mm256_castpd_si256(_mm256_add_pd(_mm256_castsi256_pd(a), _mm256_castsi256_pd(b))))
905 }
906 #[inline(always)]
907 fn subd(self, a: __m256i, b: __m256i) -> __m256i {
908 avx!(_mm256_castpd_si256(_mm256_sub_pd(_mm256_castsi256_pd(a), _mm256_castsi256_pd(b))))
909 }
910 #[inline(always)]
911 fn muld(self, a: __m256i, b: __m256i) -> __m256i {
912 avx!(_mm256_castpd_si256(_mm256_mul_pd(_mm256_castsi256_pd(a), _mm256_castsi256_pd(b))))
913 }
914 #[inline(always)]
915 fn divd(self, a: __m256i, b: __m256i) -> __m256i {
916 avx!(_mm256_castpd_si256(_mm256_div_pd(_mm256_castsi256_pd(a), _mm256_castsi256_pd(b))))
917 }
918 #[inline(always)]
919 fn sqrtd(self, a: __m256i) -> __m256i {
920 avx!(_mm256_castpd_si256(_mm256_sqrt_pd(_mm256_castsi256_pd(a))))
921 }
922 #[inline(always)]
923 fn maxd(self, a: __m256i, b: __m256i) -> __m256i {
924 avx!(_mm256_castpd_si256(_mm256_max_pd(_mm256_castsi256_pd(a), _mm256_castsi256_pd(b))))
925 }
926 #[inline(always)]
927 fn mind(self, a: __m256i, b: __m256i) -> __m256i {
928 avx!(_mm256_castpd_si256(_mm256_min_pd(_mm256_castsi256_pd(a), _mm256_castsi256_pd(b))))
929 }
930 #[inline(always)]
931 fn cmpltd(self, a: __m256i, b: __m256i) -> __m256i {
932 avx!(_mm256_castpd_si256(_mm256_cmp_pd::<_CMP_LT_OS>(_mm256_castsi256_pd(a), _mm256_castsi256_pd(b))))
933 }
934 #[inline(always)]
935 fn cvt_i32_f64(self, a: __m256i) -> __m256i {
936 avx!(_mm256_castpd_si256(_mm256_cvtepi32_pd(_mm256_castsi256_si128(a))))
937 }
938 #[inline(always)]
939 fn cvtt_f64_i32(self, a: __m256i) -> __m256i {
940 avx!(_mm256_zextsi128_si256(_mm256_cvttpd_epi32(_mm256_castsi256_pd(a))))
941 }
942 #[inline(always)]
943 fn store_i32_half(self, b: &mut [i32], v: __m256i) {
944 assert!(b.len() >= 4, "simd: slice too short");
945 unsafe { _mm_storeu_si128(b.as_mut_ptr() as *mut __m128i, _mm256_castsi256_si128(v)) }
947 }
948
949 #[inline(always)]
951 fn sra16<const N: i32>(self, a: __m256i) -> __m256i {
952 avx!(_mm256_srai_epi16::<N>(a))
953 }
954 #[inline(always)]
955 fn mulhis16(self, a: __m256i, b: __m256i) -> __m256i {
956 avx!(_mm256_mulhi_epi16(a, b))
957 }
958 #[inline(always)]
959 fn packs16(self, a: __m256i, b: __m256i) -> __m256i {
960 avx!(_mm256_packs_epi16(a, b))
961 }
962 #[inline(always)]
963 fn load_i16(self, b: &[i16]) -> __m256i {
964 assert!(b.len() >= 16, "simd: slice too short");
965 unsafe { _mm256_loadu_si256(b.as_ptr() as *const __m256i) }
967 }
968 #[inline(always)]
969 fn store_i16(self, b: &mut [i16], v: __m256i) {
970 assert!(b.len() >= 16, "simd: slice too short");
971 unsafe { _mm256_storeu_si256(b.as_mut_ptr() as *mut __m256i, v) }
973 }
974 #[inline(always)]
975 fn store_lanes2(self, b: &mut [u8], lo: __m256i, hi: __m256i) {
976 chk(b, 64);
977 unsafe {
979 _mm256_storeu_si256(b.as_mut_ptr() as *mut __m256i, _mm256_permute2x128_si256::<0x20>(lo, hi));
980 _mm256_storeu_si256(b.as_mut_ptr().add(32) as *mut __m256i, _mm256_permute2x128_si256::<0x31>(lo, hi));
981 }
982 }
983 #[inline(always)]
984 fn load_lanes2(self, b: &[u8]) -> (__m256i, __m256i) {
985 chk(b, 64);
986 unsafe {
988 let a = _mm256_loadu_si256(b.as_ptr() as *const __m256i);
989 let c = _mm256_loadu_si256(b.as_ptr().add(32) as *const __m256i);
990 (_mm256_permute2x128_si256::<0x20>(a, c), _mm256_permute2x128_si256::<0x31>(a, c))
991 }
992 }
993 #[inline(always)]
994 fn load_i16_lanes(self, a: &[i16], b: &[i16]) -> __m256i {
995 assert!(a.len() >= 8 && b.len() >= 8, "simd: slice too short");
996 unsafe {
998 let lo = _mm_loadu_si128(a.as_ptr() as *const __m128i);
999 let hi = _mm_loadu_si128(b.as_ptr() as *const __m128i);
1000 _mm256_inserti128_si256::<1>(_mm256_castsi128_si256(lo), hi)
1001 }
1002 }
1003 #[inline(always)]
1004 fn store_raw_lanes(self, a: &mut [u8], b: &mut [u8], v: __m256i) {
1005 chk(a, 8);
1006 chk(b, 8);
1007 unsafe {
1009 _mm_storel_epi64(a.as_mut_ptr() as *mut __m128i, _mm256_castsi256_si128(v));
1010 _mm_storel_epi64(b.as_mut_ptr() as *mut __m128i, _mm256_extracti128_si256::<1>(v));
1011 }
1012 }
1013 #[inline(always)]
1014 fn sadu8(self, a: __m256i, b: __m256i) -> __m256i {
1015 avx!(_mm256_sad_epu8(a, b))
1016 }
1017 #[inline(always)]
1018 fn add64(self, a: __m256i, b: __m256i) -> __m256i {
1019 avx!(_mm256_add_epi64(a, b))
1020 }
1021
1022 #[inline(always)]
1024 fn maxf(self, a: __m256i, b: __m256i) -> __m256i {
1025 avx!(_mm256_castps_si256(_mm256_max_ps(_mm256_castsi256_ps(a), _mm256_castsi256_ps(b))))
1026 }
1027 #[inline(always)]
1028 fn minf(self, a: __m256i, b: __m256i) -> __m256i {
1029 avx!(_mm256_castps_si256(_mm256_min_ps(_mm256_castsi256_ps(a), _mm256_castsi256_ps(b))))
1030 }
1031 #[inline(always)]
1032 fn sqrtf(self, a: __m256i) -> __m256i {
1033 avx!(_mm256_castps_si256(_mm256_sqrt_ps(_mm256_castsi256_ps(a))))
1034 }
1035
1036 #[inline(always)]
1038 fn load_u16(self, b: &[u16]) -> __m256i {
1039 assert!(b.len() >= 16, "simd: slice too short");
1040 unsafe { _mm256_loadu_si256(b.as_ptr() as *const __m256i) }
1042 }
1043 #[inline(always)]
1044 fn store_u16(self, b: &mut [u16], v: __m256i) {
1045 assert!(b.len() >= 16, "simd: slice too short");
1046 unsafe { _mm256_storeu_si256(b.as_mut_ptr() as *mut __m256i, v) }
1048 }
1049 #[inline(always)]
1050 fn load_u16_lo(self, b: &[u16]) -> __m256i {
1051 assert!(b.len() >= 4, "simd: slice too short");
1052 unsafe { _mm256_zextsi128_si256(_mm_loadl_epi64(b.as_ptr() as *const __m128i)) }
1054 }
1055 #[inline(always)]
1056 fn store_u16_lo(self, b: &mut [u16], v: __m256i) {
1057 assert!(b.len() >= 4, "simd: slice too short");
1058 unsafe { _mm_storel_epi64(b.as_mut_ptr() as *mut __m128i, _mm256_castsi256_si128(v)) }
1060 }
1061 #[inline(always)]
1062 fn srl16x(self, a: __m256i, n: u32) -> __m256i {
1063 avx!(_mm256_srl_epi16(a, _mm_cvtsi32_si128(n as i32)))
1064 }
1065
1066 #[inline(always)]
1068 fn load_f32(self, b: &[f32]) -> __m256i {
1069 assert!(b.len() >= 8, "simd: slice too short");
1070 unsafe { _mm256_castps_si256(_mm256_loadu_ps(b.as_ptr())) }
1072 }
1073 #[inline(always)]
1074 fn store_f32(self, b: &mut [f32], v: __m256i) {
1075 assert!(b.len() >= 8, "simd: slice too short");
1076 unsafe { _mm256_storeu_ps(b.as_mut_ptr(), _mm256_castsi256_ps(v)) }
1078 }
1079
1080 #[inline(always)]
1082 fn adds8(self, a: __m256i, b: __m256i) -> __m256i {
1083 avx!(_mm256_adds_epi8(a, b))
1084 }
1085 #[inline(always)]
1086 fn subs8(self, a: __m256i, b: __m256i) -> __m256i {
1087 avx!(_mm256_subs_epi8(a, b))
1088 }
1089
1090 #[inline(always)]
1092 fn cmpltf(self, a: __m256i, b: __m256i) -> __m256i {
1093 avx!(_mm256_castps_si256(_mm256_cmp_ps::<_CMP_LT_OS>(_mm256_castsi256_ps(a), _mm256_castsi256_ps(b))))
1094 }
1095
1096 #[inline(always)]
1098 fn load_f32_lanes(self, a: &[f32], b: &[f32]) -> __m256i {
1099 assert!(a.len() >= 4 && b.len() >= 4, "simd: slice too short");
1100 unsafe {
1102 let lo = _mm_loadu_ps(a.as_ptr());
1103 let hi = _mm_loadu_ps(b.as_ptr());
1104 _mm256_castps_si256(_mm256_insertf128_ps::<1>(_mm256_castps128_ps256(lo), hi))
1105 }
1106 }
1107 #[inline(always)]
1108 fn store_f32_lanes(self, a: &mut [f32], b: &mut [f32], v: __m256i) {
1109 assert!(a.len() >= 4 && b.len() >= 4, "simd: slice too short");
1110 unsafe {
1112 let f = _mm256_castsi256_ps(v);
1113 _mm_storeu_ps(a.as_mut_ptr(), _mm256_castps256_ps128(f));
1114 _mm_storeu_ps(b.as_mut_ptr(), _mm256_extractf128_ps::<1>(f));
1115 }
1116 }
1117
1118 #[inline(always)]
1120 fn packssdw(self, a: __m256i, b: __m256i) -> __m256i {
1121 unsafe { avx2_packssdw(a, b) }
1123 }
1124 #[inline(always)]
1125 fn packuswb_px(self, v: __m256i) -> __m256i {
1126 let p = unsafe { avx2_packuswb(v, v) };
1128 avx!(_mm256_permute4x64_epi64::<0x08>(p))
1129 }
1130 #[inline(always)]
1131 fn ptest_raw(self, a: __m256i, b: __m256i) -> u32 {
1132 let (a, b) = avx!((_mm256_castsi256_si128(a), _mm256_castsi256_si128(b)));
1133 unsafe {
1136 core::arch::asm!("vptest {a}, {b}", "jz {z}", "jnc {m}", a = in(xmm_reg) a, b = in(xmm_reg) b,
1137 z = label { return 1 }, m = label { return 0 }, options(nomem, nostack))
1138 };
1139 2
1140 }
1141 #[inline(always)]
1142 fn ptestz_raw(self, a: __m256i, b: __m256i) -> bool {
1143 let (a, b) = avx!((_mm256_castsi256_si128(a), _mm256_castsi256_si128(b)));
1144 unsafe {
1146 core::arch::asm!("vptest {a}, {b}", "jz {z}", a = in(xmm_reg) a, b = in(xmm_reg) b,
1147 z = label { return true }, options(nomem, nostack))
1148 };
1149 false
1150 }
1151}
1152
1153#[inline]
1157#[target_feature(enable = "avx2")]
1158fn avx2_packssdw(a: __m256i, b: __m256i) -> __m256i {
1159 let r: __m256i;
1160 unsafe {
1163 core::arch::asm!("vpackssdw {r}, {a}, {b}", r = lateout(ymm_reg) r, a = in(ymm_reg) a, b = in(ymm_reg) b,
1164 options(pure, nomem, nostack, preserves_flags))
1165 };
1166 r
1167}
1168
1169#[inline]
1170#[target_feature(enable = "avx2")]
1171fn avx2_packuswb(a: __m256i, b: __m256i) -> __m256i {
1172 let r: __m256i;
1173 unsafe {
1175 core::arch::asm!("vpackuswb {r}, {a}, {b}", r = lateout(ymm_reg) r, a = in(ymm_reg) a, b = in(ymm_reg) b,
1176 options(pure, nomem, nostack, preserves_flags))
1177 };
1178 r
1179}