Skip to main content

simd/
native.rs

1// Native SSE2/AVX2 backend, kept byte-for-byte in arithmetic and instruction order.
2use core::arch::x86_64::*;
3use crate::{Simd, has_avx2};
4
5// ---------------------------------------------------------------- SSE2
6
7/// SSE2: 128-bit registers (the x86-64 baseline)
8#[derive(Clone, Copy)]
9pub struct Sse2;
10
11#[inline(always)]
12fn chk(b: &[u8], n: usize) {
13    assert!(b.len() >= n, "simd: slice too short");
14}
15
16// SAFETY (every sse! block): SSE2 is part of the x86-64 baseline, enabled on every target
17// this code is built for; the intrinsics only need the feature (no memory access).
18macro_rules! sse {
19    ($e:expr) => {
20        unsafe { $e }
21    };
22}
23
24impl Simd for Sse2 {
25    type V = __m128i;
26    const BYTES: usize = 16;
27    const PX: usize = 2;
28
29    #[inline(always)]
30    fn zero(self) -> __m128i {
31        sse!(_mm_setzero_si128())
32    }
33    #[inline(always)]
34    fn splat8(self, v: u8) -> __m128i {
35        sse!(_mm_set1_epi8(v as i8))
36    }
37    #[inline(always)]
38    fn splat16(self, v: u16) -> __m128i {
39        sse!(_mm_set1_epi16(v as i16))
40    }
41    #[inline(always)]
42    fn splat32(self, v: u32) -> __m128i {
43        sse!(_mm_set1_epi32(v as i32))
44    }
45    #[inline(always)]
46    fn splat64(self, v: u64) -> __m128i {
47        sse!(_mm_set1_epi64x(v as i64))
48    }
49    #[inline(always)]
50    fn splatf(self, v: f32) -> __m128i {
51        sse!(_mm_castps_si128(_mm_set1_ps(v)))
52    }
53
54    #[inline(always)]
55    fn load(self, b: &[u8]) -> __m128i {
56        chk(b, 16);
57        // SAFETY: 16 readable bytes (checked); unaligned load.
58        unsafe { _mm_loadu_si128(b.as_ptr() as *const __m128i) }
59    }
60    #[inline(always)]
61    fn store(self, b: &mut [u8], v: __m128i) {
62        chk(b, 16);
63        // SAFETY: 16 writable bytes (checked); unaligned store.
64        unsafe { _mm_storeu_si128(b.as_mut_ptr() as *mut __m128i, v) }
65    }
66    #[inline(always)]
67    fn load_px(self, b: &[u8]) -> __m128i {
68        chk(b, 8);
69        // SAFETY: 8 readable bytes (checked).
70        let v = unsafe { _mm_loadl_epi64(b.as_ptr() as *const __m128i) };
71        sse!(_mm_unpacklo_epi8(v, _mm_setzero_si128()))
72    }
73    #[inline(always)]
74    fn store_px(self, b: &mut [u8], v: __m128i) {
75        chk(b, 8);
76        let p = sse!(_mm_packus_epi16(v, v));
77        // SAFETY: 8 writable bytes (checked).
78        unsafe { _mm_storel_epi64(b.as_mut_ptr() as *mut __m128i, p) }
79    }
80    #[inline(always)]
81    fn load_raw(self, b: &[u8]) -> __m128i {
82        chk(b, 8);
83        // SAFETY: 8 readable bytes (checked).
84        unsafe { _mm_loadl_epi64(b.as_ptr() as *const __m128i) }
85    }
86    #[inline(always)]
87    fn store_raw(self, b: &mut [u8], v: __m128i) {
88        chk(b, 8);
89        // SAFETY: 8 writable bytes (checked).
90        unsafe { _mm_storel_epi64(b.as_mut_ptr() as *mut __m128i, v) }
91    }
92    #[inline(always)]
93    fn widen_px(self, v: __m128i) -> __m128i {
94        sse!(_mm_unpacklo_epi8(v, _mm_setzero_si128()))
95    }
96    #[inline(always)]
97    fn load_lane32(self, b: &[u8]) -> __m128i {
98        chk(b, 4);
99        sse!(_mm_cvtsi32_si128(u32::from_le_bytes([b[0], b[1], b[2], b[3]]) as i32))
100    }
101    #[inline(always)]
102    fn load_i32(self, b: &[i32]) -> __m128i {
103        assert!(b.len() >= 4, "simd: slice too short");
104        // SAFETY: 16 readable bytes (checked).
105        unsafe { _mm_loadu_si128(b.as_ptr() as *const __m128i) }
106    }
107    #[inline(always)]
108    fn store_i32(self, b: &mut [i32], v: __m128i) {
109        assert!(b.len() >= 4, "simd: slice too short");
110        // SAFETY: 16 writable bytes (checked).
111        unsafe { _mm_storeu_si128(b.as_mut_ptr() as *mut __m128i, v) }
112    }
113    #[inline(always)]
114    fn from_i32(self, v: i32) -> __m128i {
115        sse!(_mm_cvtsi32_si128(v))
116    }
117    #[inline(always)]
118    fn to_i32(self, v: __m128i) -> i32 {
119        sse!(_mm_cvtsi128_si32(v))
120    }
121    #[inline(always)]
122    fn bcast_px(self, b: &[u8]) -> __m128i {
123        chk(b, 2);
124        let v = sse!(_mm_cvtsi32_si128(u16::from_le_bytes([b[0], b[1]]) as i32));
125        let v = sse!(_mm_unpacklo_epi8(v, v));
126        let v = sse!(_mm_unpacklo_epi16(v, v));
127        sse!(_mm_unpacklo_epi8(v, _mm_setzero_si128()))
128    }
129
130    #[inline(always)]
131    fn add8(self, a: __m128i, b: __m128i) -> __m128i {
132        sse!(_mm_add_epi8(a, b))
133    }
134    #[inline(always)]
135    fn sub8(self, a: __m128i, b: __m128i) -> __m128i {
136        sse!(_mm_sub_epi8(a, b))
137    }
138    #[inline(always)]
139    fn addus8(self, a: __m128i, b: __m128i) -> __m128i {
140        sse!(_mm_adds_epu8(a, b))
141    }
142    #[inline(always)]
143    fn subus8(self, a: __m128i, b: __m128i) -> __m128i {
144        sse!(_mm_subs_epu8(a, b))
145    }
146    #[inline(always)]
147    fn maxu8(self, a: __m128i, b: __m128i) -> __m128i {
148        sse!(_mm_max_epu8(a, b))
149    }
150    #[inline(always)]
151    fn minu8(self, a: __m128i, b: __m128i) -> __m128i {
152        sse!(_mm_min_epu8(a, b))
153    }
154    #[inline(always)]
155    fn cmpeq8(self, a: __m128i, b: __m128i) -> __m128i {
156        sse!(_mm_cmpeq_epi8(a, b))
157    }
158    #[inline(always)]
159    fn avgu8(self, a: __m128i, b: __m128i) -> __m128i {
160        sse!(_mm_avg_epu8(a, b))
161    }
162    #[inline(always)]
163    fn movemask8(self, a: __m128i) -> u32 {
164        sse!(_mm_movemask_epi8(a) as u32)
165    }
166    #[inline(always)]
167    fn unpacklo8(self, a: __m128i, b: __m128i) -> __m128i {
168        sse!(_mm_unpacklo_epi8(a, b))
169    }
170    #[inline(always)]
171    fn unpackhi8(self, a: __m128i, b: __m128i) -> __m128i {
172        sse!(_mm_unpackhi_epi8(a, b))
173    }
174    #[inline(always)]
175    fn packus16(self, a: __m128i, b: __m128i) -> __m128i {
176        sse!(_mm_packus_epi16(a, b))
177    }
178
179    #[inline(always)]
180    fn add16(self, a: __m128i, b: __m128i) -> __m128i {
181        sse!(_mm_add_epi16(a, b))
182    }
183    #[inline(always)]
184    fn sub16(self, a: __m128i, b: __m128i) -> __m128i {
185        sse!(_mm_sub_epi16(a, b))
186    }
187    #[inline(always)]
188    fn subus16(self, a: __m128i, b: __m128i) -> __m128i {
189        sse!(_mm_subs_epu16(a, b))
190    }
191    #[inline(always)]
192    fn mullo16(self, a: __m128i, b: __m128i) -> __m128i {
193        sse!(_mm_mullo_epi16(a, b))
194    }
195    #[inline(always)]
196    fn mulhiu16(self, a: __m128i, b: __m128i) -> __m128i {
197        sse!(_mm_mulhi_epu16(a, b))
198    }
199    #[inline(always)]
200    fn mins16(self, a: __m128i, b: __m128i) -> __m128i {
201        sse!(_mm_min_epi16(a, b))
202    }
203    #[inline(always)]
204    fn maxs16(self, a: __m128i, b: __m128i) -> __m128i {
205        sse!(_mm_max_epi16(a, b))
206    }
207    #[inline(always)]
208    fn cmpeq16(self, a: __m128i, b: __m128i) -> __m128i {
209        sse!(_mm_cmpeq_epi16(a, b))
210    }
211    #[inline(always)]
212    fn cmpgts16(self, a: __m128i, b: __m128i) -> __m128i {
213        sse!(_mm_cmpgt_epi16(a, b))
214    }
215    #[inline(always)]
216    fn srl16<const N: i32>(self, a: __m128i) -> __m128i {
217        sse!(_mm_srli_epi16::<N>(a))
218    }
219    #[inline(always)]
220    fn sll16<const N: i32>(self, a: __m128i) -> __m128i {
221        sse!(_mm_slli_epi16::<N>(a))
222    }
223    #[inline(always)]
224    fn shuf16<const IMM: i32>(self, a: __m128i) -> __m128i {
225        sse!(_mm_shufflehi_epi16::<IMM>(_mm_shufflelo_epi16::<IMM>(a)))
226    }
227    #[inline(always)]
228    fn unpacklo16(self, a: __m128i, b: __m128i) -> __m128i {
229        sse!(_mm_unpacklo_epi16(a, b))
230    }
231    #[inline(always)]
232    fn unpackhi16(self, a: __m128i, b: __m128i) -> __m128i {
233        sse!(_mm_unpackhi_epi16(a, b))
234    }
235    #[inline(always)]
236    fn packs32(self, a: __m128i, b: __m128i) -> __m128i {
237        sse!(_mm_packs_epi32(a, b))
238    }
239    #[inline(always)]
240    fn madd16(self, a: __m128i, b: __m128i) -> __m128i {
241        sse!(_mm_madd_epi16(a, b))
242    }
243
244    #[inline(always)]
245    fn add32(self, a: __m128i, b: __m128i) -> __m128i {
246        sse!(_mm_add_epi32(a, b))
247    }
248    #[inline(always)]
249    fn sub32(self, a: __m128i, b: __m128i) -> __m128i {
250        sse!(_mm_sub_epi32(a, b))
251    }
252    #[inline(always)]
253    fn cmpeq32(self, a: __m128i, b: __m128i) -> __m128i {
254        sse!(_mm_cmpeq_epi32(a, b))
255    }
256    #[inline(always)]
257    fn cmpgts32(self, a: __m128i, b: __m128i) -> __m128i {
258        sse!(_mm_cmpgt_epi32(a, b))
259    }
260    #[inline(always)]
261    fn movemask32(self, a: __m128i) -> u32 {
262        sse!(_mm_movemask_ps(_mm_castsi128_ps(a))) as u32
263    }
264    #[inline(always)]
265    fn shuf32<const IMM: i32>(self, a: __m128i) -> __m128i {
266        sse!(_mm_shuffle_epi32::<IMM>(a))
267    }
268    #[inline(always)]
269    fn shufps<const IMM: i32>(self, a: __m128i, b: __m128i) -> __m128i {
270        sse!(_mm_castps_si128(_mm_shuffle_ps::<IMM>(_mm_castsi128_ps(a), _mm_castsi128_ps(b))))
271    }
272    #[inline(always)]
273    fn srl32<const N: i32>(self, a: __m128i) -> __m128i {
274        sse!(_mm_srli_epi32::<N>(a))
275    }
276    #[inline(always)]
277    fn sll32<const N: i32>(self, a: __m128i) -> __m128i {
278        sse!(_mm_slli_epi32::<N>(a))
279    }
280    #[inline(always)]
281    fn sra32<const N: i32>(self, a: __m128i) -> __m128i {
282        sse!(_mm_srai_epi32::<N>(a))
283    }
284    #[inline(always)]
285    fn bsll<const N: i32>(self, a: __m128i) -> __m128i {
286        sse!(_mm_slli_si128::<N>(a))
287    }
288    #[inline(always)]
289    fn bsrl<const N: i32>(self, a: __m128i) -> __m128i {
290        sse!(_mm_srli_si128::<N>(a))
291    }
292
293    #[inline(always)]
294    fn and(self, a: __m128i, b: __m128i) -> __m128i {
295        sse!(_mm_and_si128(a, b))
296    }
297    #[inline(always)]
298    fn andnot(self, a: __m128i, b: __m128i) -> __m128i {
299        sse!(_mm_andnot_si128(a, b))
300    }
301    #[inline(always)]
302    fn or(self, a: __m128i, b: __m128i) -> __m128i {
303        sse!(_mm_or_si128(a, b))
304    }
305    #[inline(always)]
306    fn xor(self, a: __m128i, b: __m128i) -> __m128i {
307        sse!(_mm_xor_si128(a, b))
308    }
309
310    #[inline(always)]
311    fn cvt_i32_f32(self, a: __m128i) -> __m128i {
312        sse!(_mm_castps_si128(_mm_cvtepi32_ps(a)))
313    }
314    #[inline(always)]
315    fn cvtt_f32_i32(self, a: __m128i) -> __m128i {
316        sse!(_mm_cvttps_epi32(_mm_castsi128_ps(a)))
317    }
318    #[inline(always)]
319    fn cvtr_f32_i32(self, a: __m128i) -> __m128i {
320        sse!(_mm_cvtps_epi32(_mm_castsi128_ps(a)))
321    }
322    #[inline(always)]
323    fn addf(self, a: __m128i, b: __m128i) -> __m128i {
324        sse!(_mm_castps_si128(_mm_add_ps(_mm_castsi128_ps(a), _mm_castsi128_ps(b))))
325    }
326    #[inline(always)]
327    fn subf(self, a: __m128i, b: __m128i) -> __m128i {
328        sse!(_mm_castps_si128(_mm_sub_ps(_mm_castsi128_ps(a), _mm_castsi128_ps(b))))
329    }
330    #[inline(always)]
331    fn mulf(self, a: __m128i, b: __m128i) -> __m128i {
332        sse!(_mm_castps_si128(_mm_mul_ps(_mm_castsi128_ps(a), _mm_castsi128_ps(b))))
333    }
334    #[inline(always)]
335    fn divf(self, a: __m128i, b: __m128i) -> __m128i {
336        sse!(_mm_castps_si128(_mm_div_ps(_mm_castsi128_ps(a), _mm_castsi128_ps(b))))
337    }
338
339    #[inline(always)]
340    fn splatd(self, v: f64) -> __m128i {
341        sse!(_mm_castpd_si128(_mm_set1_pd(v)))
342    }
343    #[inline(always)]
344    fn addd(self, a: __m128i, b: __m128i) -> __m128i {
345        sse!(_mm_castpd_si128(_mm_add_pd(_mm_castsi128_pd(a), _mm_castsi128_pd(b))))
346    }
347    #[inline(always)]
348    fn subd(self, a: __m128i, b: __m128i) -> __m128i {
349        sse!(_mm_castpd_si128(_mm_sub_pd(_mm_castsi128_pd(a), _mm_castsi128_pd(b))))
350    }
351    #[inline(always)]
352    fn muld(self, a: __m128i, b: __m128i) -> __m128i {
353        sse!(_mm_castpd_si128(_mm_mul_pd(_mm_castsi128_pd(a), _mm_castsi128_pd(b))))
354    }
355    #[inline(always)]
356    fn divd(self, a: __m128i, b: __m128i) -> __m128i {
357        sse!(_mm_castpd_si128(_mm_div_pd(_mm_castsi128_pd(a), _mm_castsi128_pd(b))))
358    }
359    #[inline(always)]
360    fn sqrtd(self, a: __m128i) -> __m128i {
361        sse!(_mm_castpd_si128(_mm_sqrt_pd(_mm_castsi128_pd(a))))
362    }
363    #[inline(always)]
364    fn maxd(self, a: __m128i, b: __m128i) -> __m128i {
365        sse!(_mm_castpd_si128(_mm_max_pd(_mm_castsi128_pd(a), _mm_castsi128_pd(b))))
366    }
367    #[inline(always)]
368    fn mind(self, a: __m128i, b: __m128i) -> __m128i {
369        sse!(_mm_castpd_si128(_mm_min_pd(_mm_castsi128_pd(a), _mm_castsi128_pd(b))))
370    }
371    #[inline(always)]
372    fn cmpltd(self, a: __m128i, b: __m128i) -> __m128i {
373        sse!(_mm_castpd_si128(_mm_cmplt_pd(_mm_castsi128_pd(a), _mm_castsi128_pd(b))))
374    }
375    #[inline(always)]
376    fn cvt_i32_f64(self, a: __m128i) -> __m128i {
377        sse!(_mm_castpd_si128(_mm_cvtepi32_pd(a)))
378    }
379    #[inline(always)]
380    fn cvtt_f64_i32(self, a: __m128i) -> __m128i {
381        sse!(_mm_cvttpd_epi32(_mm_castsi128_pd(a)))
382    }
383    #[inline(always)]
384    fn store_i32_half(self, b: &mut [i32], v: __m128i) {
385        assert!(b.len() >= 2, "simd: slice too short");
386        // SAFETY: 8 writable bytes (checked).
387        unsafe { _mm_storel_epi64(b.as_mut_ptr() as *mut __m128i, v) }
388    }
389
390    // R6-jpeg, R6-xf
391    #[inline(always)]
392    fn sra16<const N: i32>(self, a: __m128i) -> __m128i {
393        sse!(_mm_srai_epi16::<N>(a))
394    }
395    #[inline(always)]
396    fn mulhis16(self, a: __m128i, b: __m128i) -> __m128i {
397        sse!(_mm_mulhi_epi16(a, b))
398    }
399    #[inline(always)]
400    fn packs16(self, a: __m128i, b: __m128i) -> __m128i {
401        sse!(_mm_packs_epi16(a, b))
402    }
403    #[inline(always)]
404    fn load_i16(self, b: &[i16]) -> __m128i {
405        assert!(b.len() >= 8, "simd: slice too short");
406        // SAFETY: 16 readable bytes (checked).
407        unsafe { _mm_loadu_si128(b.as_ptr() as *const __m128i) }
408    }
409    #[inline(always)]
410    fn store_i16(self, b: &mut [i16], v: __m128i) {
411        assert!(b.len() >= 8, "simd: slice too short");
412        // SAFETY: 16 writable bytes (checked).
413        unsafe { _mm_storeu_si128(b.as_mut_ptr() as *mut __m128i, v) }
414    }
415    #[inline(always)]
416    fn store_lanes2(self, b: &mut [u8], lo: __m128i, hi: __m128i) {
417        chk(b, 32);
418        // SAFETY: 32 writable bytes (checked).
419        unsafe {
420            _mm_storeu_si128(b.as_mut_ptr() as *mut __m128i, lo);
421            _mm_storeu_si128(b.as_mut_ptr().add(16) as *mut __m128i, hi);
422        }
423    }
424    #[inline(always)]
425    fn load_lanes2(self, b: &[u8]) -> (__m128i, __m128i) {
426        chk(b, 32);
427        // SAFETY: 32 readable bytes (checked).
428        unsafe {
429            (_mm_loadu_si128(b.as_ptr() as *const __m128i), _mm_loadu_si128(b.as_ptr().add(16) as *const __m128i))
430        }
431    }
432    #[inline(always)]
433    fn load_i16_lanes(self, a: &[i16], _b: &[i16]) -> __m128i {
434        self.load_i16(a)
435    }
436    #[inline(always)]
437    fn store_raw_lanes(self, a: &mut [u8], _b: &mut [u8], v: __m128i) {
438        self.store_raw(a, v)
439    }
440    #[inline(always)]
441    fn sadu8(self, a: __m128i, b: __m128i) -> __m128i {
442        sse!(_mm_sad_epu8(a, b))
443    }
444    #[inline(always)]
445    fn add64(self, a: __m128i, b: __m128i) -> __m128i {
446        sse!(_mm_add_epi64(a, b))
447    }
448
449    // R6-icc
450    #[inline(always)]
451    fn maxf(self, a: __m128i, b: __m128i) -> __m128i {
452        sse!(_mm_castps_si128(_mm_max_ps(_mm_castsi128_ps(a), _mm_castsi128_ps(b))))
453    }
454    #[inline(always)]
455    fn minf(self, a: __m128i, b: __m128i) -> __m128i {
456        sse!(_mm_castps_si128(_mm_min_ps(_mm_castsi128_ps(a), _mm_castsi128_ps(b))))
457    }
458    #[inline(always)]
459    fn sqrtf(self, a: __m128i) -> __m128i {
460        sse!(_mm_castps_si128(_mm_sqrt_ps(_mm_castsi128_ps(a))))
461    }
462
463    // r9-av1
464    #[inline(always)]
465    fn load_u16(self, b: &[u16]) -> __m128i {
466        assert!(b.len() >= 8, "simd: slice too short");
467        // SAFETY: 16 readable bytes (checked).
468        unsafe { _mm_loadu_si128(b.as_ptr() as *const __m128i) }
469    }
470    #[inline(always)]
471    fn store_u16(self, b: &mut [u16], v: __m128i) {
472        assert!(b.len() >= 8, "simd: slice too short");
473        // SAFETY: 16 writable bytes (checked).
474        unsafe { _mm_storeu_si128(b.as_mut_ptr() as *mut __m128i, v) }
475    }
476    #[inline(always)]
477    fn load_u16_lo(self, b: &[u16]) -> __m128i {
478        assert!(b.len() >= 4, "simd: slice too short");
479        // SAFETY: 8 readable bytes (checked).
480        unsafe { _mm_loadl_epi64(b.as_ptr() as *const __m128i) }
481    }
482    #[inline(always)]
483    fn store_u16_lo(self, b: &mut [u16], v: __m128i) {
484        assert!(b.len() >= 4, "simd: slice too short");
485        // SAFETY: 8 writable bytes (checked).
486        unsafe { _mm_storel_epi64(b.as_mut_ptr() as *mut __m128i, v) }
487    }
488    #[inline(always)]
489    fn srl16x(self, a: __m128i, n: u32) -> __m128i {
490        sse!(_mm_srl_epi16(a, _mm_cvtsi32_si128(n as i32)))
491    }
492
493    // R9-jxl
494    #[inline(always)]
495    fn load_f32(self, b: &[f32]) -> __m128i {
496        assert!(b.len() >= 4, "simd: slice too short");
497        // SAFETY: 16 readable bytes (checked).
498        unsafe { _mm_castps_si128(_mm_loadu_ps(b.as_ptr())) }
499    }
500    #[inline(always)]
501    fn store_f32(self, b: &mut [f32], v: __m128i) {
502        assert!(b.len() >= 4, "simd: slice too short");
503        // SAFETY: 16 writable bytes (checked).
504        unsafe { _mm_storeu_ps(b.as_mut_ptr(), _mm_castsi128_ps(v)) }
505    }
506
507    // R9-webp
508    #[inline(always)]
509    fn adds8(self, a: __m128i, b: __m128i) -> __m128i {
510        sse!(_mm_adds_epi8(a, b))
511    }
512    #[inline(always)]
513    fn subs8(self, a: __m128i, b: __m128i) -> __m128i {
514        sse!(_mm_subs_epi8(a, b))
515    }
516
517    // r9-dds
518    #[inline(always)]
519    fn cmpltf(self, a: __m128i, b: __m128i) -> __m128i {
520        sse!(_mm_castps_si128(_mm_cmplt_ps(_mm_castsi128_ps(a), _mm_castsi128_ps(b))))
521    }
522
523    // jxl-speed
524    #[inline(always)]
525    fn load_f32_lanes(self, a: &[f32], _b: &[f32]) -> __m128i {
526        self.load_f32(a)
527    }
528    #[inline(always)]
529    fn store_f32_lanes(self, a: &mut [f32], _b: &mut [f32], v: __m128i) {
530        self.store_f32(a, v)
531    }
532
533    // perf-blend (the intrinsics: see the trait)
534    #[inline(always)]
535    fn packssdw(self, a: __m128i, b: __m128i) -> __m128i {
536        sse!(_mm_packs_epi32(a, b))
537    }
538    #[inline(always)]
539    fn packuswb_px(self, v: __m128i) -> __m128i {
540        sse!(_mm_packus_epi16(v, v))
541    }
542    #[inline(always)]
543    fn ptest_raw(self, a: __m128i, b: __m128i) -> u32 {
544        let z = sse!(_mm_movemask_epi8(_mm_cmpeq_epi8(_mm_and_si128(a, b), _mm_setzero_si128()))) & 0xFF == 0xFF;
545        let c = sse!(_mm_movemask_epi8(_mm_cmpeq_epi8(_mm_andnot_si128(a, b), _mm_setzero_si128()))) & 0xFF == 0xFF;
546        if z {
547            1
548        } else if c {
549            2
550        } else {
551            0
552        }
553    }
554    #[inline(always)]
555    fn ptestz_raw(self, a: __m128i, b: __m128i) -> bool {
556        sse!(_mm_movemask_epi8(_mm_cmpeq_epi8(_mm_and_si128(a, b), _mm_setzero_si128()))) & 0xFF == 0xFF
557    }
558}
559
560// ---------------------------------------------------------------- AVX2
561
562/// AVX2: 256-bit registers as two 128-bit lanes; only constructible when the CPU has AVX2
563#[derive(Clone, Copy)]
564pub struct Avx2(());
565
566impl Avx2 {
567    #[inline]
568    pub fn new() -> Option<Avx2> {
569        has_avx2().then_some(Avx2(()))
570    }
571}
572
573// SAFETY (every unsafe block below unless stated otherwise): an Avx2 value exists only after
574// Avx2::new found AVX2 support, which is the only precondition of these intrinsics.
575macro_rules! avx {
576    ($e:expr) => {
577        unsafe { $e }
578    };
579}
580
581impl Simd for Avx2 {
582    type V = __m256i;
583    const BYTES: usize = 32;
584    const PX: usize = 4;
585
586    #[inline(always)]
587    fn zero(self) -> __m256i {
588        avx!(_mm256_setzero_si256())
589    }
590    #[inline(always)]
591    fn splat8(self, v: u8) -> __m256i {
592        avx!(_mm256_set1_epi8(v as i8))
593    }
594    #[inline(always)]
595    fn splat16(self, v: u16) -> __m256i {
596        avx!(_mm256_set1_epi16(v as i16))
597    }
598    #[inline(always)]
599    fn splat32(self, v: u32) -> __m256i {
600        avx!(_mm256_set1_epi32(v as i32))
601    }
602    #[inline(always)]
603    fn splat64(self, v: u64) -> __m256i {
604        avx!(_mm256_set1_epi64x(v as i64))
605    }
606    #[inline(always)]
607    fn splatf(self, v: f32) -> __m256i {
608        avx!(_mm256_castps_si256(_mm256_set1_ps(v)))
609    }
610
611    #[inline(always)]
612    fn load(self, b: &[u8]) -> __m256i {
613        chk(b, 32);
614        // SAFETY: 32 readable bytes (checked), AVX2 present.
615        unsafe { _mm256_loadu_si256(b.as_ptr() as *const __m256i) }
616    }
617    #[inline(always)]
618    fn store(self, b: &mut [u8], v: __m256i) {
619        chk(b, 32);
620        // SAFETY: 32 writable bytes (checked), AVX2 present.
621        unsafe { _mm256_storeu_si256(b.as_mut_ptr() as *mut __m256i, v) }
622    }
623    #[inline(always)]
624    fn load_px(self, b: &[u8]) -> __m256i {
625        chk(b, 16);
626        // SAFETY: 16 readable bytes (checked), AVX2 present.
627        unsafe { _mm256_cvtepu8_epi16(_mm_loadu_si128(b.as_ptr() as *const __m128i)) }
628    }
629    #[inline(always)]
630    fn store_px(self, b: &mut [u8], v: __m256i) {
631        chk(b, 16);
632        // SAFETY: 16 writable bytes (checked), AVX2 present.
633        unsafe {
634            let p = _mm256_permute4x64_epi64::<0x08>(_mm256_packus_epi16(v, v));
635            _mm_storeu_si128(b.as_mut_ptr() as *mut __m128i, _mm256_castsi256_si128(p))
636        }
637    }
638    #[inline(always)]
639    fn load_raw(self, b: &[u8]) -> __m256i {
640        chk(b, 16);
641        // SAFETY: 16 readable bytes (checked), AVX2 present.
642        unsafe { _mm256_zextsi128_si256(_mm_loadu_si128(b.as_ptr() as *const __m128i)) }
643    }
644    #[inline(always)]
645    fn store_raw(self, b: &mut [u8], v: __m256i) {
646        chk(b, 16);
647        // SAFETY: 16 writable bytes (checked), AVX2 present.
648        unsafe { _mm_storeu_si128(b.as_mut_ptr() as *mut __m128i, _mm256_castsi256_si128(v)) }
649    }
650    #[inline(always)]
651    fn widen_px(self, v: __m256i) -> __m256i {
652        avx!(_mm256_cvtepu8_epi16(_mm256_castsi256_si128(v)))
653    }
654    #[inline(always)]
655    fn load_lane32(self, b: &[u8]) -> __m256i {
656        chk(b, 8);
657        let d = |k: usize| u32::from_le_bytes([b[k], b[k + 1], b[k + 2], b[k + 3]]) as i32;
658        avx!(_mm256_setr_epi32(d(0), 0, 0, 0, d(4), 0, 0, 0))
659    }
660    #[inline(always)]
661    fn load_i32(self, b: &[i32]) -> __m256i {
662        assert!(b.len() >= 8, "simd: slice too short");
663        // SAFETY: 32 readable bytes (checked), AVX2 present.
664        unsafe { _mm256_loadu_si256(b.as_ptr() as *const __m256i) }
665    }
666    #[inline(always)]
667    fn store_i32(self, b: &mut [i32], v: __m256i) {
668        assert!(b.len() >= 8, "simd: slice too short");
669        // SAFETY: 32 writable bytes (checked), AVX2 present.
670        unsafe { _mm256_storeu_si256(b.as_mut_ptr() as *mut __m256i, v) }
671    }
672    #[inline(always)]
673    fn from_i32(self, v: i32) -> __m256i {
674        avx!(_mm256_zextsi128_si256(_mm_cvtsi32_si128(v)))
675    }
676    #[inline(always)]
677    fn to_i32(self, v: __m256i) -> i32 {
678        avx!(_mm_cvtsi128_si32(_mm256_castsi256_si128(v)))
679    }
680    #[inline(always)]
681    fn bcast_px(self, b: &[u8]) -> __m256i {
682        chk(b, 4);
683        let v = avx!(_mm_cvtsi32_si128(u32::from_le_bytes([b[0], b[1], b[2], b[3]]) as i32));
684        let v = avx!(_mm_unpacklo_epi8(v, v));
685        let v = avx!(_mm_unpacklo_epi16(v, v));
686        avx!(_mm256_cvtepu8_epi16(v))
687    }
688
689    #[inline(always)]
690    fn add8(self, a: __m256i, b: __m256i) -> __m256i {
691        avx!(_mm256_add_epi8(a, b))
692    }
693    #[inline(always)]
694    fn sub8(self, a: __m256i, b: __m256i) -> __m256i {
695        avx!(_mm256_sub_epi8(a, b))
696    }
697    #[inline(always)]
698    fn addus8(self, a: __m256i, b: __m256i) -> __m256i {
699        avx!(_mm256_adds_epu8(a, b))
700    }
701    #[inline(always)]
702    fn subus8(self, a: __m256i, b: __m256i) -> __m256i {
703        avx!(_mm256_subs_epu8(a, b))
704    }
705    #[inline(always)]
706    fn maxu8(self, a: __m256i, b: __m256i) -> __m256i {
707        avx!(_mm256_max_epu8(a, b))
708    }
709    #[inline(always)]
710    fn minu8(self, a: __m256i, b: __m256i) -> __m256i {
711        avx!(_mm256_min_epu8(a, b))
712    }
713    #[inline(always)]
714    fn cmpeq8(self, a: __m256i, b: __m256i) -> __m256i {
715        avx!(_mm256_cmpeq_epi8(a, b))
716    }
717    #[inline(always)]
718    fn avgu8(self, a: __m256i, b: __m256i) -> __m256i {
719        avx!(_mm256_avg_epu8(a, b))
720    }
721    #[inline(always)]
722    fn movemask8(self, a: __m256i) -> u32 {
723        avx!(_mm256_movemask_epi8(a)) as u32
724    }
725    #[inline(always)]
726    fn unpacklo8(self, a: __m256i, b: __m256i) -> __m256i {
727        avx!(_mm256_unpacklo_epi8(a, b))
728    }
729    #[inline(always)]
730    fn unpackhi8(self, a: __m256i, b: __m256i) -> __m256i {
731        avx!(_mm256_unpackhi_epi8(a, b))
732    }
733    #[inline(always)]
734    fn packus16(self, a: __m256i, b: __m256i) -> __m256i {
735        avx!(_mm256_packus_epi16(a, b))
736    }
737
738    #[inline(always)]
739    fn add16(self, a: __m256i, b: __m256i) -> __m256i {
740        avx!(_mm256_add_epi16(a, b))
741    }
742    #[inline(always)]
743    fn sub16(self, a: __m256i, b: __m256i) -> __m256i {
744        avx!(_mm256_sub_epi16(a, b))
745    }
746    #[inline(always)]
747    fn subus16(self, a: __m256i, b: __m256i) -> __m256i {
748        avx!(_mm256_subs_epu16(a, b))
749    }
750    #[inline(always)]
751    fn mullo16(self, a: __m256i, b: __m256i) -> __m256i {
752        avx!(_mm256_mullo_epi16(a, b))
753    }
754    #[inline(always)]
755    fn mulhiu16(self, a: __m256i, b: __m256i) -> __m256i {
756        avx!(_mm256_mulhi_epu16(a, b))
757    }
758    #[inline(always)]
759    fn mins16(self, a: __m256i, b: __m256i) -> __m256i {
760        avx!(_mm256_min_epi16(a, b))
761    }
762    #[inline(always)]
763    fn maxs16(self, a: __m256i, b: __m256i) -> __m256i {
764        avx!(_mm256_max_epi16(a, b))
765    }
766    #[inline(always)]
767    fn cmpeq16(self, a: __m256i, b: __m256i) -> __m256i {
768        avx!(_mm256_cmpeq_epi16(a, b))
769    }
770    #[inline(always)]
771    fn cmpgts16(self, a: __m256i, b: __m256i) -> __m256i {
772        avx!(_mm256_cmpgt_epi16(a, b))
773    }
774    #[inline(always)]
775    fn srl16<const N: i32>(self, a: __m256i) -> __m256i {
776        avx!(_mm256_srli_epi16::<N>(a))
777    }
778    #[inline(always)]
779    fn sll16<const N: i32>(self, a: __m256i) -> __m256i {
780        avx!(_mm256_slli_epi16::<N>(a))
781    }
782    #[inline(always)]
783    fn shuf16<const IMM: i32>(self, a: __m256i) -> __m256i {
784        avx!(_mm256_shufflehi_epi16::<IMM>(_mm256_shufflelo_epi16::<IMM>(a)))
785    }
786    #[inline(always)]
787    fn unpacklo16(self, a: __m256i, b: __m256i) -> __m256i {
788        avx!(_mm256_unpacklo_epi16(a, b))
789    }
790    #[inline(always)]
791    fn unpackhi16(self, a: __m256i, b: __m256i) -> __m256i {
792        avx!(_mm256_unpackhi_epi16(a, b))
793    }
794    #[inline(always)]
795    fn packs32(self, a: __m256i, b: __m256i) -> __m256i {
796        avx!(_mm256_packs_epi32(a, b))
797    }
798    #[inline(always)]
799    fn madd16(self, a: __m256i, b: __m256i) -> __m256i {
800        avx!(_mm256_madd_epi16(a, b))
801    }
802
803    #[inline(always)]
804    fn add32(self, a: __m256i, b: __m256i) -> __m256i {
805        avx!(_mm256_add_epi32(a, b))
806    }
807    #[inline(always)]
808    fn sub32(self, a: __m256i, b: __m256i) -> __m256i {
809        avx!(_mm256_sub_epi32(a, b))
810    }
811    #[inline(always)]
812    fn cmpeq32(self, a: __m256i, b: __m256i) -> __m256i {
813        avx!(_mm256_cmpeq_epi32(a, b))
814    }
815    #[inline(always)]
816    fn cmpgts32(self, a: __m256i, b: __m256i) -> __m256i {
817        avx!(_mm256_cmpgt_epi32(a, b))
818    }
819    #[inline(always)]
820    fn movemask32(self, a: __m256i) -> u32 {
821        avx!(_mm256_movemask_ps(_mm256_castsi256_ps(a))) as u32
822    }
823    #[inline(always)]
824    fn shuf32<const IMM: i32>(self, a: __m256i) -> __m256i {
825        avx!(_mm256_shuffle_epi32::<IMM>(a))
826    }
827    #[inline(always)]
828    fn shufps<const IMM: i32>(self, a: __m256i, b: __m256i) -> __m256i {
829        avx!(_mm256_castps_si256(_mm256_shuffle_ps::<IMM>(_mm256_castsi256_ps(a), _mm256_castsi256_ps(b))))
830    }
831    #[inline(always)]
832    fn srl32<const N: i32>(self, a: __m256i) -> __m256i {
833        avx!(_mm256_srli_epi32::<N>(a))
834    }
835    #[inline(always)]
836    fn sll32<const N: i32>(self, a: __m256i) -> __m256i {
837        avx!(_mm256_slli_epi32::<N>(a))
838    }
839    #[inline(always)]
840    fn sra32<const N: i32>(self, a: __m256i) -> __m256i {
841        avx!(_mm256_srai_epi32::<N>(a))
842    }
843    #[inline(always)]
844    fn bsll<const N: i32>(self, a: __m256i) -> __m256i {
845        avx!(_mm256_bslli_epi128::<N>(a))
846    }
847    #[inline(always)]
848    fn bsrl<const N: i32>(self, a: __m256i) -> __m256i {
849        avx!(_mm256_bsrli_epi128::<N>(a))
850    }
851
852    #[inline(always)]
853    fn and(self, a: __m256i, b: __m256i) -> __m256i {
854        avx!(_mm256_and_si256(a, b))
855    }
856    #[inline(always)]
857    fn andnot(self, a: __m256i, b: __m256i) -> __m256i {
858        avx!(_mm256_andnot_si256(a, b))
859    }
860    #[inline(always)]
861    fn or(self, a: __m256i, b: __m256i) -> __m256i {
862        avx!(_mm256_or_si256(a, b))
863    }
864    #[inline(always)]
865    fn xor(self, a: __m256i, b: __m256i) -> __m256i {
866        avx!(_mm256_xor_si256(a, b))
867    }
868
869    #[inline(always)]
870    fn cvt_i32_f32(self, a: __m256i) -> __m256i {
871        avx!(_mm256_castps_si256(_mm256_cvtepi32_ps(a)))
872    }
873    #[inline(always)]
874    fn cvtt_f32_i32(self, a: __m256i) -> __m256i {
875        avx!(_mm256_cvttps_epi32(_mm256_castsi256_ps(a)))
876    }
877    #[inline(always)]
878    fn cvtr_f32_i32(self, a: __m256i) -> __m256i {
879        avx!(_mm256_cvtps_epi32(_mm256_castsi256_ps(a)))
880    }
881    #[inline(always)]
882    fn addf(self, a: __m256i, b: __m256i) -> __m256i {
883        avx!(_mm256_castps_si256(_mm256_add_ps(_mm256_castsi256_ps(a), _mm256_castsi256_ps(b))))
884    }
885    #[inline(always)]
886    fn subf(self, a: __m256i, b: __m256i) -> __m256i {
887        avx!(_mm256_castps_si256(_mm256_sub_ps(_mm256_castsi256_ps(a), _mm256_castsi256_ps(b))))
888    }
889    #[inline(always)]
890    fn mulf(self, a: __m256i, b: __m256i) -> __m256i {
891        avx!(_mm256_castps_si256(_mm256_mul_ps(_mm256_castsi256_ps(a), _mm256_castsi256_ps(b))))
892    }
893    #[inline(always)]
894    fn divf(self, a: __m256i, b: __m256i) -> __m256i {
895        avx!(_mm256_castps_si256(_mm256_div_ps(_mm256_castsi256_ps(a), _mm256_castsi256_ps(b))))
896    }
897
898    #[inline(always)]
899    fn splatd(self, v: f64) -> __m256i {
900        avx!(_mm256_castpd_si256(_mm256_set1_pd(v)))
901    }
902    #[inline(always)]
903    fn addd(self, a: __m256i, b: __m256i) -> __m256i {
904        avx!(_mm256_castpd_si256(_mm256_add_pd(_mm256_castsi256_pd(a), _mm256_castsi256_pd(b))))
905    }
906    #[inline(always)]
907    fn subd(self, a: __m256i, b: __m256i) -> __m256i {
908        avx!(_mm256_castpd_si256(_mm256_sub_pd(_mm256_castsi256_pd(a), _mm256_castsi256_pd(b))))
909    }
910    #[inline(always)]
911    fn muld(self, a: __m256i, b: __m256i) -> __m256i {
912        avx!(_mm256_castpd_si256(_mm256_mul_pd(_mm256_castsi256_pd(a), _mm256_castsi256_pd(b))))
913    }
914    #[inline(always)]
915    fn divd(self, a: __m256i, b: __m256i) -> __m256i {
916        avx!(_mm256_castpd_si256(_mm256_div_pd(_mm256_castsi256_pd(a), _mm256_castsi256_pd(b))))
917    }
918    #[inline(always)]
919    fn sqrtd(self, a: __m256i) -> __m256i {
920        avx!(_mm256_castpd_si256(_mm256_sqrt_pd(_mm256_castsi256_pd(a))))
921    }
922    #[inline(always)]
923    fn maxd(self, a: __m256i, b: __m256i) -> __m256i {
924        avx!(_mm256_castpd_si256(_mm256_max_pd(_mm256_castsi256_pd(a), _mm256_castsi256_pd(b))))
925    }
926    #[inline(always)]
927    fn mind(self, a: __m256i, b: __m256i) -> __m256i {
928        avx!(_mm256_castpd_si256(_mm256_min_pd(_mm256_castsi256_pd(a), _mm256_castsi256_pd(b))))
929    }
930    #[inline(always)]
931    fn cmpltd(self, a: __m256i, b: __m256i) -> __m256i {
932        avx!(_mm256_castpd_si256(_mm256_cmp_pd::<_CMP_LT_OS>(_mm256_castsi256_pd(a), _mm256_castsi256_pd(b))))
933    }
934    #[inline(always)]
935    fn cvt_i32_f64(self, a: __m256i) -> __m256i {
936        avx!(_mm256_castpd_si256(_mm256_cvtepi32_pd(_mm256_castsi256_si128(a))))
937    }
938    #[inline(always)]
939    fn cvtt_f64_i32(self, a: __m256i) -> __m256i {
940        avx!(_mm256_zextsi128_si256(_mm256_cvttpd_epi32(_mm256_castsi256_pd(a))))
941    }
942    #[inline(always)]
943    fn store_i32_half(self, b: &mut [i32], v: __m256i) {
944        assert!(b.len() >= 4, "simd: slice too short");
945        // SAFETY: 16 writable bytes (checked), AVX2 present.
946        unsafe { _mm_storeu_si128(b.as_mut_ptr() as *mut __m128i, _mm256_castsi256_si128(v)) }
947    }
948
949    // R6-jpeg, R6-xf
950    #[inline(always)]
951    fn sra16<const N: i32>(self, a: __m256i) -> __m256i {
952        avx!(_mm256_srai_epi16::<N>(a))
953    }
954    #[inline(always)]
955    fn mulhis16(self, a: __m256i, b: __m256i) -> __m256i {
956        avx!(_mm256_mulhi_epi16(a, b))
957    }
958    #[inline(always)]
959    fn packs16(self, a: __m256i, b: __m256i) -> __m256i {
960        avx!(_mm256_packs_epi16(a, b))
961    }
962    #[inline(always)]
963    fn load_i16(self, b: &[i16]) -> __m256i {
964        assert!(b.len() >= 16, "simd: slice too short");
965        // SAFETY: 32 readable bytes (checked), AVX2 present.
966        unsafe { _mm256_loadu_si256(b.as_ptr() as *const __m256i) }
967    }
968    #[inline(always)]
969    fn store_i16(self, b: &mut [i16], v: __m256i) {
970        assert!(b.len() >= 16, "simd: slice too short");
971        // SAFETY: 32 writable bytes (checked), AVX2 present.
972        unsafe { _mm256_storeu_si256(b.as_mut_ptr() as *mut __m256i, v) }
973    }
974    #[inline(always)]
975    fn store_lanes2(self, b: &mut [u8], lo: __m256i, hi: __m256i) {
976        chk(b, 64);
977        // SAFETY: 64 writable bytes (checked), AVX2 present.
978        unsafe {
979            _mm256_storeu_si256(b.as_mut_ptr() as *mut __m256i, _mm256_permute2x128_si256::<0x20>(lo, hi));
980            _mm256_storeu_si256(b.as_mut_ptr().add(32) as *mut __m256i, _mm256_permute2x128_si256::<0x31>(lo, hi));
981        }
982    }
983    #[inline(always)]
984    fn load_lanes2(self, b: &[u8]) -> (__m256i, __m256i) {
985        chk(b, 64);
986        // SAFETY: 64 readable bytes (checked), AVX2 present.
987        unsafe {
988            let a = _mm256_loadu_si256(b.as_ptr() as *const __m256i);
989            let c = _mm256_loadu_si256(b.as_ptr().add(32) as *const __m256i);
990            (_mm256_permute2x128_si256::<0x20>(a, c), _mm256_permute2x128_si256::<0x31>(a, c))
991        }
992    }
993    #[inline(always)]
994    fn load_i16_lanes(self, a: &[i16], b: &[i16]) -> __m256i {
995        assert!(a.len() >= 8 && b.len() >= 8, "simd: slice too short");
996        // SAFETY: 16 readable bytes in each source (checked), AVX2 present.
997        unsafe {
998            let lo = _mm_loadu_si128(a.as_ptr() as *const __m128i);
999            let hi = _mm_loadu_si128(b.as_ptr() as *const __m128i);
1000            _mm256_inserti128_si256::<1>(_mm256_castsi128_si256(lo), hi)
1001        }
1002    }
1003    #[inline(always)]
1004    fn store_raw_lanes(self, a: &mut [u8], b: &mut [u8], v: __m256i) {
1005        chk(a, 8);
1006        chk(b, 8);
1007        // SAFETY: 8 writable bytes in each destination (checked), AVX2 present.
1008        unsafe {
1009            _mm_storel_epi64(a.as_mut_ptr() as *mut __m128i, _mm256_castsi256_si128(v));
1010            _mm_storel_epi64(b.as_mut_ptr() as *mut __m128i, _mm256_extracti128_si256::<1>(v));
1011        }
1012    }
1013    #[inline(always)]
1014    fn sadu8(self, a: __m256i, b: __m256i) -> __m256i {
1015        avx!(_mm256_sad_epu8(a, b))
1016    }
1017    #[inline(always)]
1018    fn add64(self, a: __m256i, b: __m256i) -> __m256i {
1019        avx!(_mm256_add_epi64(a, b))
1020    }
1021
1022    // R6-icc
1023    #[inline(always)]
1024    fn maxf(self, a: __m256i, b: __m256i) -> __m256i {
1025        avx!(_mm256_castps_si256(_mm256_max_ps(_mm256_castsi256_ps(a), _mm256_castsi256_ps(b))))
1026    }
1027    #[inline(always)]
1028    fn minf(self, a: __m256i, b: __m256i) -> __m256i {
1029        avx!(_mm256_castps_si256(_mm256_min_ps(_mm256_castsi256_ps(a), _mm256_castsi256_ps(b))))
1030    }
1031    #[inline(always)]
1032    fn sqrtf(self, a: __m256i) -> __m256i {
1033        avx!(_mm256_castps_si256(_mm256_sqrt_ps(_mm256_castsi256_ps(a))))
1034    }
1035
1036    // r9-av1
1037    #[inline(always)]
1038    fn load_u16(self, b: &[u16]) -> __m256i {
1039        assert!(b.len() >= 16, "simd: slice too short");
1040        // SAFETY: 32 readable bytes (checked), AVX2 present.
1041        unsafe { _mm256_loadu_si256(b.as_ptr() as *const __m256i) }
1042    }
1043    #[inline(always)]
1044    fn store_u16(self, b: &mut [u16], v: __m256i) {
1045        assert!(b.len() >= 16, "simd: slice too short");
1046        // SAFETY: 32 writable bytes (checked), AVX2 present.
1047        unsafe { _mm256_storeu_si256(b.as_mut_ptr() as *mut __m256i, v) }
1048    }
1049    #[inline(always)]
1050    fn load_u16_lo(self, b: &[u16]) -> __m256i {
1051        assert!(b.len() >= 4, "simd: slice too short");
1052        // SAFETY: 8 readable bytes (checked), AVX2 present.
1053        unsafe { _mm256_zextsi128_si256(_mm_loadl_epi64(b.as_ptr() as *const __m128i)) }
1054    }
1055    #[inline(always)]
1056    fn store_u16_lo(self, b: &mut [u16], v: __m256i) {
1057        assert!(b.len() >= 4, "simd: slice too short");
1058        // SAFETY: 8 writable bytes (checked), AVX2 present.
1059        unsafe { _mm_storel_epi64(b.as_mut_ptr() as *mut __m128i, _mm256_castsi256_si128(v)) }
1060    }
1061    #[inline(always)]
1062    fn srl16x(self, a: __m256i, n: u32) -> __m256i {
1063        avx!(_mm256_srl_epi16(a, _mm_cvtsi32_si128(n as i32)))
1064    }
1065
1066    // R9-jxl
1067    #[inline(always)]
1068    fn load_f32(self, b: &[f32]) -> __m256i {
1069        assert!(b.len() >= 8, "simd: slice too short");
1070        // SAFETY: 32 readable bytes (checked), AVX2 present.
1071        unsafe { _mm256_castps_si256(_mm256_loadu_ps(b.as_ptr())) }
1072    }
1073    #[inline(always)]
1074    fn store_f32(self, b: &mut [f32], v: __m256i) {
1075        assert!(b.len() >= 8, "simd: slice too short");
1076        // SAFETY: 32 writable bytes (checked), AVX2 present.
1077        unsafe { _mm256_storeu_ps(b.as_mut_ptr(), _mm256_castsi256_ps(v)) }
1078    }
1079
1080    // R9-webp
1081    #[inline(always)]
1082    fn adds8(self, a: __m256i, b: __m256i) -> __m256i {
1083        avx!(_mm256_adds_epi8(a, b))
1084    }
1085    #[inline(always)]
1086    fn subs8(self, a: __m256i, b: __m256i) -> __m256i {
1087        avx!(_mm256_subs_epi8(a, b))
1088    }
1089
1090    // r9-dds
1091    #[inline(always)]
1092    fn cmpltf(self, a: __m256i, b: __m256i) -> __m256i {
1093        avx!(_mm256_castps_si256(_mm256_cmp_ps::<_CMP_LT_OS>(_mm256_castsi256_ps(a), _mm256_castsi256_ps(b))))
1094    }
1095
1096    // jxl-speed
1097    #[inline(always)]
1098    fn load_f32_lanes(self, a: &[f32], b: &[f32]) -> __m256i {
1099        assert!(a.len() >= 4 && b.len() >= 4, "simd: slice too short");
1100        // SAFETY: 16 readable bytes in each source (checked), AVX2 present.
1101        unsafe {
1102            let lo = _mm_loadu_ps(a.as_ptr());
1103            let hi = _mm_loadu_ps(b.as_ptr());
1104            _mm256_castps_si256(_mm256_insertf128_ps::<1>(_mm256_castps128_ps256(lo), hi))
1105        }
1106    }
1107    #[inline(always)]
1108    fn store_f32_lanes(self, a: &mut [f32], b: &mut [f32], v: __m256i) {
1109        assert!(a.len() >= 4 && b.len() >= 4, "simd: slice too short");
1110        // SAFETY: 16 writable bytes in each destination (checked), AVX2 present.
1111        unsafe {
1112            let f = _mm256_castsi256_ps(v);
1113            _mm_storeu_ps(a.as_mut_ptr(), _mm256_castps256_ps128(f));
1114            _mm_storeu_ps(b.as_mut_ptr(), _mm256_extractf128_ps::<1>(f));
1115        }
1116    }
1117
1118    // perf-blend
1119    #[inline(always)]
1120    fn packssdw(self, a: __m256i, b: __m256i) -> __m256i {
1121        // SAFETY: AVX2 present (an Avx2 value exists), the precondition of avx2_packssdw.
1122        unsafe { avx2_packssdw(a, b) }
1123    }
1124    #[inline(always)]
1125    fn packuswb_px(self, v: __m256i) -> __m256i {
1126        // SAFETY: AVX2 present (an Avx2 value exists), the precondition of avx2_packuswb.
1127        let p = unsafe { avx2_packuswb(v, v) };
1128        avx!(_mm256_permute4x64_epi64::<0x08>(p))
1129    }
1130    #[inline(always)]
1131    fn ptest_raw(self, a: __m256i, b: __m256i) -> u32 {
1132        let (a, b) = avx!((_mm256_castsi256_si128(a), _mm256_castsi256_si128(b)));
1133        // SAFETY: vptest reads two registers and sets the flags, the jumps go to the label
1134        // blocks; AVX present (an Avx2 value exists).
1135        unsafe {
1136            core::arch::asm!("vptest {a}, {b}", "jz {z}", "jnc {m}", a = in(xmm_reg) a, b = in(xmm_reg) b,
1137                             z = label { return 1 }, m = label { return 0 }, options(nomem, nostack))
1138        };
1139        2
1140    }
1141    #[inline(always)]
1142    fn ptestz_raw(self, a: __m256i, b: __m256i) -> bool {
1143        let (a, b) = avx!((_mm256_castsi256_si128(a), _mm256_castsi256_si128(b)));
1144        // SAFETY: as ptest_raw.
1145        unsafe {
1146            core::arch::asm!("vptest {a}, {b}", "jz {z}", a = in(xmm_reg) a, b = in(xmm_reg) b,
1147                             z = label { return true }, options(nomem, nostack))
1148        };
1149        false
1150    }
1151}
1152
1153// perf-blend: vpackssdw / vpackuswb as inline assembly, so LLVM cannot rewrite them into
1154// smax / smin + pack (see the trait).  #[inline] functions with the avx2 feature: inlined
1155// into the AVX2 kernels (run_avx2 has the feature), the ymm operands need it.
1156#[inline]
1157#[target_feature(enable = "avx2")]
1158fn avx2_packssdw(a: __m256i, b: __m256i) -> __m256i {
1159    let r: __m256i;
1160    // SAFETY: a register-only AVX2 instruction (no memory, stack or flags); this function
1161    // has the avx2 feature, so its callers checked for AVX2.
1162    unsafe {
1163        core::arch::asm!("vpackssdw {r}, {a}, {b}", r = lateout(ymm_reg) r, a = in(ymm_reg) a, b = in(ymm_reg) b,
1164                         options(pure, nomem, nostack, preserves_flags))
1165    };
1166    r
1167}
1168
1169#[inline]
1170#[target_feature(enable = "avx2")]
1171fn avx2_packuswb(a: __m256i, b: __m256i) -> __m256i {
1172    let r: __m256i;
1173    // SAFETY: as avx2_packssdw.
1174    unsafe {
1175        core::arch::asm!("vpackuswb {r}, {a}, {b}", r = lateout(ymm_reg) r, a = in(ymm_reg) a, b = in(ymm_reg) b,
1176                         options(pure, nomem, nostack, preserves_flags))
1177    };
1178    r
1179}