Skip to main content

simd/
neon.rs

1// NEON backend: part of the ARM64 baseline, so no check at run time and no second register
2// width (lib.rs: Neon128 as Sse2, Scalar256 as Avx2).  Every result is SSE2's bit for bit
3// (apps/portabletest compares them with the scalar code).  Where the obvious instruction
4// computes something else:
5//   float arithmetic   ARM's default NaN is positive, SSE's "indefinite" negative, and a
6//                      signalling second operand wins over a quiet first one.  A result with a
7//                      NaN takes a cold path with SSE's rule, per lane (rt::portable::fx, as
8//                      in the scalar code)
9//   maxf / minf / ...  fmax and fmin differ on NaN and signed zeros: compare and select
10//   float to int       fcvtzs / fcvtns saturate and give 0 for NaN, SSE 0x80000000
11//   srl16<N> / ...     the _n_ intrinsics refuse what SSE allows (N = 0, N > width) at compile
12//                      time, even on a branch not taken: shifts by a constant register, which
13//                      LLVM turns back into the immediate instruction
14//   shuf16<IMM> / ...  no such instructions: tbl with a constant table, which LLVM turns into
15//                      the cheapest shuffle there is
16use crate::Simd;
17use core::arch::aarch64::*;
18use rt::portable::fx;
19
20/// NEON: 128-bit registers (the ARM64 baseline)
21#[derive(Clone, Copy)]
22pub struct Neon128;
23
24type R = uint8x16_t;
25
26#[inline(always)]
27fn chk(b: &[u8], n: usize) {
28    assert!(b.len() >= n, "simd: slice too short");
29}
30
31// SAFETY (every neon! block): NEON is part of the ARM64 baseline, enabled on every target this
32// code is built for; the intrinsics only need the feature (no memory access).
33macro_rules! neon {
34    ($e:expr) => {
35        unsafe { $e }
36    };
37}
38
39// the lanes of a register, and back; no instruction
40#[inline(always)]
41fn i8s(a: R) -> int8x16_t {
42    neon!(vreinterpretq_s8_u8(a))
43}
44#[inline(always)]
45fn u16s(a: R) -> uint16x8_t {
46    neon!(vreinterpretq_u16_u8(a))
47}
48#[inline(always)]
49fn i16s(a: R) -> int16x8_t {
50    neon!(vreinterpretq_s16_u8(a))
51}
52#[inline(always)]
53fn u32s(a: R) -> uint32x4_t {
54    neon!(vreinterpretq_u32_u8(a))
55}
56#[inline(always)]
57fn i32s(a: R) -> int32x4_t {
58    neon!(vreinterpretq_s32_u8(a))
59}
60#[inline(always)]
61fn u64s(a: R) -> uint64x2_t {
62    neon!(vreinterpretq_u64_u8(a))
63}
64#[inline(always)]
65fn f32s(a: R) -> float32x4_t {
66    neon!(vreinterpretq_f32_u8(a))
67}
68#[inline(always)]
69fn f64s(a: R) -> float64x2_t {
70    neon!(vreinterpretq_f64_u8(a))
71}
72trait Lanes: Copy {
73    fn r(self) -> R;
74}
75macro_rules! lanes {
76    ($t:ty, $to:ident) => {
77        impl Lanes for $t {
78            #[inline(always)]
79            fn r(self) -> R {
80                neon!($to(self))
81            }
82        }
83    };
84}
85lanes!(int8x16_t, vreinterpretq_u8_s8);
86lanes!(uint16x8_t, vreinterpretq_u8_u16);
87lanes!(int16x8_t, vreinterpretq_u8_s16);
88lanes!(uint32x4_t, vreinterpretq_u8_u32);
89lanes!(int32x4_t, vreinterpretq_u8_s32);
90lanes!(uint64x2_t, vreinterpretq_u8_u64);
91lanes!(float32x4_t, vreinterpretq_u8_f32);
92lanes!(float64x2_t, vreinterpretq_u8_f64);
93
94// the lanes of a result with a NaN, by SSE's rule; a and b as in the operation
95#[cold]
96#[inline(never)]
97fn sse_nan32(a: float32x4_t, b: float32x4_t, r: float32x4_t) -> float32x4_t {
98    let (mut x, mut y, mut z) = ([0f32; 4], [0f32; 4], [0f32; 4]);
99    // SAFETY: 16 writable and readable bytes in each array.
100    unsafe {
101        vst1q_f32(x.as_mut_ptr(), a);
102        vst1q_f32(y.as_mut_ptr(), b);
103        vst1q_f32(z.as_mut_ptr(), r);
104        for i in 0..4 {
105            z[i] = fx::arithmetic32(x[i], y[i], z[i]);
106        }
107        vld1q_f32(z.as_ptr())
108    }
109}
110#[cold]
111#[inline(never)]
112fn sse_nan64(a: float64x2_t, b: float64x2_t, r: float64x2_t) -> float64x2_t {
113    let (mut x, mut y, mut z) = ([0f64; 2], [0f64; 2], [0f64; 2]);
114    // SAFETY: 16 writable and readable bytes in each array.
115    unsafe {
116        vst1q_f64(x.as_mut_ptr(), a);
117        vst1q_f64(y.as_mut_ptr(), b);
118        vst1q_f64(z.as_mut_ptr(), r);
119        for i in 0..2 {
120            z[i] = fx::arithmetic64(x[i], y[i], z[i]);
121        }
122        vld1q_f64(z.as_ptr())
123    }
124}
125// fmaxv / fmaxp pass on a NaN of any lane; a NaN in an operand always is one in r
126#[inline(always)]
127fn result32(a: float32x4_t, b: float32x4_t, r: float32x4_t) -> R {
128    if neon!(vmaxvq_f32(r)).is_nan() {
129        sse_nan32(a, b, r).r()
130    } else {
131        r.r()
132    }
133}
134#[inline(always)]
135fn result64(a: float64x2_t, b: float64x2_t, r: float64x2_t) -> R {
136    if neon!(vmaxvq_f64(r)).is_nan() {
137        sse_nan64(a, b, r).r()
138    } else {
139        r.r()
140    }
141}
142
143// tbl: bytes of a by index, 0 for an index out of range (255)
144#[inline(always)]
145fn tbl(a: R, index: &[u8; 16]) -> R {
146    // SAFETY: 16 readable bytes.
147    unsafe { vqtbl1q_u8(a, vld1q_u8(index.as_ptr())) }
148}
149const fn shuf16_index(imm: i32) -> [u8; 16] {
150    let mut t = [0; 16];
151    let mut i = 0;
152    while i < 8 {
153        let word = i / 4 * 4 + ((imm >> (i % 4 * 2)) & 3) as usize;
154        t[i * 2] = (word * 2) as u8;
155        t[i * 2 + 1] = (word * 2 + 1) as u8;
156        i += 1;
157    }
158    t
159}
160const fn shuf32_index(imm: i32) -> [u8; 16] {
161    let mut t = [0; 16];
162    let mut i = 0;
163    while i < 16 {
164        t[i] = (((imm >> (i / 4 * 2)) & 3) * 4) as u8 + (i % 4) as u8;
165        i += 1;
166    }
167    t
168}
169const fn bsll_index(n: i32) -> [u8; 16] {
170    let mut t = [255; 16];
171    let mut i = 0;
172    while i < 16 {
173        if (n as u32) <= i {
174            t[i as usize] = (i - n as u32) as u8;
175        }
176        i += 1;
177    }
178    t
179}
180const fn bsrl_index(n: i32) -> [u8; 16] {
181    let mut t = [255; 16];
182    let mut i = 0;
183    while i < 16 {
184        if (n as u32) < 16 - i {
185            t[i as usize] = (i + n as u32) as u8;
186        }
187        i += 1;
188    }
189    t
190}
191
192const TWO31F: f32 = 2147483648.0;
193const TWO31D: f64 = 2147483648.0;
194
195impl Simd for Neon128 {
196    type V = R;
197    const BYTES: usize = 16;
198    const PX: usize = 2;
199
200    #[inline(always)]
201    fn zero(self) -> R {
202        neon!(vdupq_n_u8(0))
203    }
204    #[inline(always)]
205    fn splat8(self, v: u8) -> R {
206        neon!(vdupq_n_u8(v))
207    }
208    #[inline(always)]
209    fn splat16(self, v: u16) -> R {
210        neon!(vdupq_n_u16(v)).r()
211    }
212    #[inline(always)]
213    fn splat32(self, v: u32) -> R {
214        neon!(vdupq_n_u32(v)).r()
215    }
216    #[inline(always)]
217    fn splat64(self, v: u64) -> R {
218        neon!(vdupq_n_u64(v)).r()
219    }
220    #[inline(always)]
221    fn splatf(self, v: f32) -> R {
222        neon!(vdupq_n_f32(v)).r()
223    }
224
225    // ---- memory (vld1 / vst1 on bytes: no alignment asked)
226    #[inline(always)]
227    fn load(self, b: &[u8]) -> R {
228        chk(b, 16);
229        // SAFETY: 16 readable bytes (checked).
230        unsafe { vld1q_u8(b.as_ptr()) }
231    }
232    #[inline(always)]
233    fn store(self, b: &mut [u8], v: R) {
234        chk(b, 16);
235        // SAFETY: 16 writable bytes (checked).
236        unsafe { vst1q_u8(b.as_mut_ptr(), v) }
237    }
238    #[inline(always)]
239    fn load_px(self, b: &[u8]) -> R {
240        self.widen_px(self.load_raw(b))
241    }
242    #[inline(always)]
243    fn store_px(self, b: &mut [u8], v: R) {
244        chk(b, 8);
245        // SAFETY: 8 writable bytes (checked).
246        unsafe { vst1_u8(b.as_mut_ptr(), vqmovun_s16(i16s(v))) }
247    }
248    #[inline(always)]
249    fn bcast_px(self, b: &[u8]) -> R {
250        chk(b, 2);
251        neon!(vcombine_u16(vdup_n_u16(b[0] as u16), vdup_n_u16(b[1] as u16))).r()
252    }
253    #[inline(always)]
254    fn load_raw(self, b: &[u8]) -> R {
255        chk(b, 8);
256        // SAFETY: 8 readable bytes (checked).
257        unsafe { vcombine_u8(vld1_u8(b.as_ptr()), vdup_n_u8(0)) }
258    }
259    #[inline(always)]
260    fn store_raw(self, b: &mut [u8], v: R) {
261        chk(b, 8);
262        // SAFETY: 8 writable bytes (checked).
263        unsafe { vst1_u8(b.as_mut_ptr(), vget_low_u8(v)) }
264    }
265    #[inline(always)]
266    fn widen_px(self, v: R) -> R {
267        neon!(vmovl_u8(vget_low_u8(v))).r()
268    }
269    #[inline(always)]
270    fn load_lane32(self, b: &[u8]) -> R {
271        chk(b, 4);
272        self.from_i32(i32::from_le_bytes([b[0], b[1], b[2], b[3]]))
273    }
274    #[inline(always)]
275    fn load_i32(self, b: &[i32]) -> R {
276        assert!(b.len() >= 4, "simd: slice too short");
277        // SAFETY: 16 readable bytes (checked).
278        unsafe { vld1q_u8(b.as_ptr() as *const u8) }
279    }
280    #[inline(always)]
281    fn store_i32(self, b: &mut [i32], v: R) {
282        assert!(b.len() >= 4, "simd: slice too short");
283        // SAFETY: 16 writable bytes (checked).
284        unsafe { vst1q_u8(b.as_mut_ptr() as *mut u8, v) }
285    }
286    #[inline(always)]
287    fn from_i32(self, v: i32) -> R {
288        neon!(vsetq_lane_s32::<0>(v, vdupq_n_s32(0))).r()
289    }
290    #[inline(always)]
291    fn to_i32(self, v: R) -> i32 {
292        neon!(vgetq_lane_s32::<0>(i32s(v)))
293    }
294
295    // ---- 8-bit lanes
296    #[inline(always)]
297    fn add8(self, a: R, b: R) -> R {
298        neon!(vaddq_u8(a, b))
299    }
300    #[inline(always)]
301    fn sub8(self, a: R, b: R) -> R {
302        neon!(vsubq_u8(a, b))
303    }
304    #[inline(always)]
305    fn addus8(self, a: R, b: R) -> R {
306        neon!(vqaddq_u8(a, b))
307    }
308    #[inline(always)]
309    fn subus8(self, a: R, b: R) -> R {
310        neon!(vqsubq_u8(a, b))
311    }
312    #[inline(always)]
313    fn maxu8(self, a: R, b: R) -> R {
314        neon!(vmaxq_u8(a, b))
315    }
316    #[inline(always)]
317    fn minu8(self, a: R, b: R) -> R {
318        neon!(vminq_u8(a, b))
319    }
320    #[inline(always)]
321    fn cmpeq8(self, a: R, b: R) -> R {
322        neon!(vceqq_u8(a, b))
323    }
324    #[inline(always)]
325    fn avgu8(self, a: R, b: R) -> R {
326        neon!(vrhaddq_u8(a, b))
327    }
328    #[inline(always)]
329    fn movemask8(self, a: R) -> u32 {
330        const BIT: [u8; 16] = [1, 2, 4, 8, 16, 32, 64, 128, 1, 2, 4, 8, 16, 32, 64, 128];
331        // SAFETY: 16 readable bytes.
332        unsafe {
333            let bits = vandq_u8(vcltzq_s8(i8s(a)), vld1q_u8(BIT.as_ptr()));
334            let p = vpaddq_u8(bits, bits);
335            let p = vpaddq_u8(p, p);
336            let p = vpaddq_u8(p, p);
337            vgetq_lane_u16::<0>(u16s(p)) as u32
338        }
339    }
340    #[inline(always)]
341    fn unpacklo8(self, a: R, b: R) -> R {
342        neon!(vzip1q_u8(a, b))
343    }
344    #[inline(always)]
345    fn unpackhi8(self, a: R, b: R) -> R {
346        neon!(vzip2q_u8(a, b))
347    }
348    #[inline(always)]
349    fn packus16(self, a: R, b: R) -> R {
350        neon!(vqmovun_high_s16(vqmovun_s16(i16s(a)), i16s(b)))
351    }
352
353    // ---- 16-bit lanes
354    #[inline(always)]
355    fn add16(self, a: R, b: R) -> R {
356        neon!(vaddq_u16(u16s(a), u16s(b))).r()
357    }
358    #[inline(always)]
359    fn sub16(self, a: R, b: R) -> R {
360        neon!(vsubq_u16(u16s(a), u16s(b))).r()
361    }
362    #[inline(always)]
363    fn subus16(self, a: R, b: R) -> R {
364        neon!(vqsubq_u16(u16s(a), u16s(b))).r()
365    }
366    #[inline(always)]
367    fn mullo16(self, a: R, b: R) -> R {
368        neon!(vmulq_u16(u16s(a), u16s(b))).r()
369    }
370    #[inline(always)]
371    fn mulhiu16(self, a: R, b: R) -> R {
372        let (x, y) = (u16s(a), u16s(b));
373        // SAFETY: see neon!.
374        unsafe {
375            let lo = vmull_u16(vget_low_u16(x), vget_low_u16(y));
376            let hi = vmull_high_u16(x, y);
377            vuzp2q_u16(u16s(lo.r()), u16s(hi.r())).r()
378        }
379    }
380    #[inline(always)]
381    fn mins16(self, a: R, b: R) -> R {
382        neon!(vminq_s16(i16s(a), i16s(b))).r()
383    }
384    #[inline(always)]
385    fn maxs16(self, a: R, b: R) -> R {
386        neon!(vmaxq_s16(i16s(a), i16s(b))).r()
387    }
388    #[inline(always)]
389    fn cmpeq16(self, a: R, b: R) -> R {
390        neon!(vceqq_u16(u16s(a), u16s(b))).r()
391    }
392    #[inline(always)]
393    fn cmpgts16(self, a: R, b: R) -> R {
394        neon!(vcgtq_s16(i16s(a), i16s(b))).r()
395    }
396    #[inline(always)]
397    fn srl16<const N: i32>(self, a: R) -> R {
398        if (N as u32) >= 16 {
399            self.zero()
400        } else {
401            neon!(vshlq_u16(u16s(a), vdupq_n_s16(-(N as i16)))).r()
402        }
403    }
404    #[inline(always)]
405    fn sll16<const N: i32>(self, a: R) -> R {
406        if (N as u32) >= 16 {
407            self.zero()
408        } else {
409            neon!(vshlq_u16(u16s(a), vdupq_n_s16(N as i16))).r()
410        }
411    }
412    #[inline(always)]
413    fn shuf16<const IMM: i32>(self, a: R) -> R {
414        tbl(a, const { &shuf16_index(IMM) })
415    }
416    #[inline(always)]
417    fn unpacklo16(self, a: R, b: R) -> R {
418        neon!(vzip1q_u16(u16s(a), u16s(b))).r()
419    }
420    #[inline(always)]
421    fn unpackhi16(self, a: R, b: R) -> R {
422        neon!(vzip2q_u16(u16s(a), u16s(b))).r()
423    }
424    #[inline(always)]
425    fn packs32(self, a: R, b: R) -> R {
426        neon!(vqmovn_high_s32(vqmovn_s32(i32s(a)), i32s(b))).r()
427    }
428    #[inline(always)]
429    fn madd16(self, a: R, b: R) -> R {
430        let (x, y) = (i16s(a), i16s(b));
431        // SAFETY: see neon!.
432        unsafe {
433            let lo = vmull_s16(vget_low_s16(x), vget_low_s16(y));
434            let hi = vmull_high_s16(x, y);
435            vpaddq_s32(lo, hi).r()
436        }
437    }
438
439    // ---- 32-bit lanes
440    #[inline(always)]
441    fn add32(self, a: R, b: R) -> R {
442        neon!(vaddq_u32(u32s(a), u32s(b))).r()
443    }
444    #[inline(always)]
445    fn sub32(self, a: R, b: R) -> R {
446        neon!(vsubq_u32(u32s(a), u32s(b))).r()
447    }
448    #[inline(always)]
449    fn cmpeq32(self, a: R, b: R) -> R {
450        neon!(vceqq_u32(u32s(a), u32s(b))).r()
451    }
452    #[inline(always)]
453    fn cmpgts32(self, a: R, b: R) -> R {
454        neon!(vcgtq_s32(i32s(a), i32s(b))).r()
455    }
456    #[inline(always)]
457    fn movemask32(self, a: R) -> u32 {
458        const BIT: [u32; 4] = [1, 2, 4, 8];
459        // SAFETY: 16 readable bytes.
460        unsafe { vaddvq_u32(vandq_u32(vcltzq_s32(i32s(a)), vld1q_u32(BIT.as_ptr()))) }
461    }
462    #[inline(always)]
463    fn shuf32<const IMM: i32>(self, a: R) -> R {
464        tbl(a, const { &shuf32_index(IMM) })
465    }
466    #[inline(always)]
467    fn shufps<const IMM: i32>(self, a: R, b: R) -> R {
468        neon!(vcombine_u8(vget_low_u8(self.shuf32::<IMM>(a)), vget_high_u8(self.shuf32::<IMM>(b))))
469    }
470    #[inline(always)]
471    fn srl32<const N: i32>(self, a: R) -> R {
472        if (N as u32) >= 32 {
473            self.zero()
474        } else {
475            neon!(vshlq_u32(u32s(a), vdupq_n_s32(-N))).r()
476        }
477    }
478    #[inline(always)]
479    fn sll32<const N: i32>(self, a: R) -> R {
480        if (N as u32) >= 32 {
481            self.zero()
482        } else {
483            neon!(vshlq_u32(u32s(a), vdupq_n_s32(N))).r()
484        }
485    }
486    #[inline(always)]
487    fn sra32<const N: i32>(self, a: R) -> R {
488        neon!(vshlq_s32(i32s(a), vdupq_n_s32(-((N as u32).min(31) as i32)))).r()
489    }
490    #[inline(always)]
491    fn bsll<const N: i32>(self, a: R) -> R {
492        tbl(a, const { &bsll_index(N) })
493    }
494    #[inline(always)]
495    fn bsrl<const N: i32>(self, a: R) -> R {
496        tbl(a, const { &bsrl_index(N) })
497    }
498
499    // ---- bits
500    #[inline(always)]
501    fn and(self, a: R, b: R) -> R {
502        neon!(vandq_u8(a, b))
503    }
504    #[inline(always)]
505    fn andnot(self, a: R, b: R) -> R {
506        neon!(vbicq_u8(b, a))
507    }
508    #[inline(always)]
509    fn or(self, a: R, b: R) -> R {
510        neon!(vorrq_u8(a, b))
511    }
512    #[inline(always)]
513    fn xor(self, a: R, b: R) -> R {
514        neon!(veorq_u8(a, b))
515    }
516
517    // ---- f32 lanes
518    #[inline(always)]
519    fn cvt_i32_f32(self, a: R) -> R {
520        neon!(vcvtq_f32_s32(i32s(a))).r()
521    }
522    #[inline(always)]
523    fn cvtt_f32_i32(self, a: R) -> R {
524        let x = f32s(a);
525        // (too negative saturates to what SSE gives anyway)
526        neon!(vbslq_s32(vcltq_f32(x, vdupq_n_f32(TWO31F)), vcvtq_s32_f32(x), vdupq_n_s32(i32::MIN))).r()
527    }
528    #[inline(always)]
529    fn cvtr_f32_i32(self, a: R) -> R {
530        let x = f32s(a);
531        neon!(vbslq_s32(vcltq_f32(x, vdupq_n_f32(TWO31F)), vcvtnq_s32_f32(x), vdupq_n_s32(i32::MIN))).r()
532    }
533    #[inline(always)]
534    fn addf(self, a: R, b: R) -> R {
535        let (x, y) = (f32s(a), f32s(b));
536        result32(x, y, neon!(vaddq_f32(x, y)))
537    }
538    #[inline(always)]
539    fn subf(self, a: R, b: R) -> R {
540        let (x, y) = (f32s(a), f32s(b));
541        result32(x, y, neon!(vsubq_f32(x, y)))
542    }
543    #[inline(always)]
544    fn mulf(self, a: R, b: R) -> R {
545        let (x, y) = (f32s(a), f32s(b));
546        result32(x, y, neon!(vmulq_f32(x, y)))
547    }
548    #[inline(always)]
549    fn divf(self, a: R, b: R) -> R {
550        let (x, y) = (f32s(a), f32s(b));
551        result32(x, y, neon!(vdivq_f32(x, y)))
552    }
553
554    // ---- f64 lanes
555    #[inline(always)]
556    fn splatd(self, v: f64) -> R {
557        neon!(vdupq_n_f64(v)).r()
558    }
559    #[inline(always)]
560    fn addd(self, a: R, b: R) -> R {
561        let (x, y) = (f64s(a), f64s(b));
562        result64(x, y, neon!(vaddq_f64(x, y)))
563    }
564    #[inline(always)]
565    fn subd(self, a: R, b: R) -> R {
566        let (x, y) = (f64s(a), f64s(b));
567        result64(x, y, neon!(vsubq_f64(x, y)))
568    }
569    #[inline(always)]
570    fn muld(self, a: R, b: R) -> R {
571        let (x, y) = (f64s(a), f64s(b));
572        result64(x, y, neon!(vmulq_f64(x, y)))
573    }
574    #[inline(always)]
575    fn divd(self, a: R, b: R) -> R {
576        let (x, y) = (f64s(a), f64s(b));
577        result64(x, y, neon!(vdivq_f64(x, y)))
578    }
579    #[inline(always)]
580    fn sqrtd(self, a: R) -> R {
581        let x = f64s(a);
582        result64(x, x, neon!(vsqrtq_f64(x)))
583    }
584    #[inline(always)]
585    fn maxd(self, a: R, b: R) -> R {
586        let (x, y) = (f64s(a), f64s(b));
587        neon!(vbslq_f64(vcgtq_f64(x, y), x, y)).r()
588    }
589    #[inline(always)]
590    fn mind(self, a: R, b: R) -> R {
591        let (x, y) = (f64s(a), f64s(b));
592        neon!(vbslq_f64(vcltq_f64(x, y), x, y)).r()
593    }
594    #[inline(always)]
595    fn cmpltd(self, a: R, b: R) -> R {
596        neon!(vcltq_f64(f64s(a), f64s(b))).r()
597    }
598    #[inline(always)]
599    fn cvt_i32_f64(self, a: R) -> R {
600        neon!(vcvtq_f64_s64(vmovl_s32(vget_low_s32(i32s(a))))).r()
601    }
602    #[inline(always)]
603    fn cvtt_f64_i32(self, a: R) -> R {
604        let x = f64s(a);
605        // SAFETY: see neon!.
606        unsafe {
607            // (too negative saturates twice to what SSE gives anyway)
608            let v = vqmovn_s64(vcvtq_s64_f64(x));
609            let ok = vmovn_u64(vcltq_f64(x, vdupq_n_f64(TWO31D)));
610            vcombine_s32(vbsl_s32(ok, v, vdup_n_s32(i32::MIN)), vdup_n_s32(0)).r()
611        }
612    }
613    #[inline(always)]
614    fn store_i32_half(self, b: &mut [i32], v: R) {
615        assert!(b.len() >= 2, "simd: slice too short");
616        // SAFETY: 8 writable bytes (checked).
617        unsafe { vst1_u8(b.as_mut_ptr() as *mut u8, vget_low_u8(v)) }
618    }
619
620    // R6-jpeg, R6-xf
621    #[inline(always)]
622    fn sra16<const N: i32>(self, a: R) -> R {
623        neon!(vshlq_s16(i16s(a), vdupq_n_s16(-((N as u32).min(15) as i16)))).r()
624    }
625    #[inline(always)]
626    fn mulhis16(self, a: R, b: R) -> R {
627        let (x, y) = (i16s(a), i16s(b));
628        // SAFETY: see neon!.
629        unsafe {
630            let lo = vmull_s16(vget_low_s16(x), vget_low_s16(y));
631            let hi = vmull_high_s16(x, y);
632            vuzp2q_u16(u16s(lo.r()), u16s(hi.r())).r()
633        }
634    }
635    #[inline(always)]
636    fn packs16(self, a: R, b: R) -> R {
637        neon!(vqmovn_high_s16(vqmovn_s16(i16s(a)), i16s(b))).r()
638    }
639    #[inline(always)]
640    fn load_i16(self, b: &[i16]) -> R {
641        assert!(b.len() >= 8, "simd: slice too short");
642        // SAFETY: 16 readable bytes (checked).
643        unsafe { vld1q_u8(b.as_ptr() as *const u8) }
644    }
645    #[inline(always)]
646    fn store_i16(self, b: &mut [i16], v: R) {
647        assert!(b.len() >= 8, "simd: slice too short");
648        // SAFETY: 16 writable bytes (checked).
649        unsafe { vst1q_u8(b.as_mut_ptr() as *mut u8, v) }
650    }
651    #[inline(always)]
652    fn store_lanes2(self, b: &mut [u8], lo: R, hi: R) {
653        chk(b, 32);
654        // SAFETY: 32 writable bytes (checked).
655        unsafe {
656            vst1q_u8(b.as_mut_ptr(), lo);
657            vst1q_u8(b.as_mut_ptr().add(16), hi);
658        }
659    }
660    #[inline(always)]
661    fn load_lanes2(self, b: &[u8]) -> (R, R) {
662        chk(b, 32);
663        // SAFETY: 32 readable bytes (checked).
664        unsafe { (vld1q_u8(b.as_ptr()), vld1q_u8(b.as_ptr().add(16))) }
665    }
666    #[inline(always)]
667    fn load_i16_lanes(self, a: &[i16], _b: &[i16]) -> R {
668        self.load_i16(a)
669    }
670    #[inline(always)]
671    fn store_raw_lanes(self, a: &mut [u8], _b: &mut [u8], v: R) {
672        self.store_raw(a, v);
673    }
674    #[inline(always)]
675    fn sadu8(self, a: R, b: R) -> R {
676        neon!(vpaddlq_u32(vpaddlq_u16(vpaddlq_u8(vabdq_u8(a, b))))).r()
677    }
678    #[inline(always)]
679    fn add64(self, a: R, b: R) -> R {
680        neon!(vaddq_u64(u64s(a), u64s(b))).r()
681    }
682
683    // R6-icc
684    #[inline(always)]
685    fn maxf(self, a: R, b: R) -> R {
686        let (x, y) = (f32s(a), f32s(b));
687        neon!(vbslq_f32(vcgtq_f32(x, y), x, y)).r()
688    }
689    #[inline(always)]
690    fn minf(self, a: R, b: R) -> R {
691        let (x, y) = (f32s(a), f32s(b));
692        neon!(vbslq_f32(vcltq_f32(x, y), x, y)).r()
693    }
694    #[inline(always)]
695    fn sqrtf(self, a: R) -> R {
696        let x = f32s(a);
697        result32(x, x, neon!(vsqrtq_f32(x)))
698    }
699
700    // r9-av1
701    #[inline(always)]
702    fn load_u16(self, b: &[u16]) -> R {
703        assert!(b.len() >= 8, "simd: slice too short");
704        // SAFETY: 16 readable bytes (checked).
705        unsafe { vld1q_u8(b.as_ptr() as *const u8) }
706    }
707    #[inline(always)]
708    fn store_u16(self, b: &mut [u16], v: R) {
709        assert!(b.len() >= 8, "simd: slice too short");
710        // SAFETY: 16 writable bytes (checked).
711        unsafe { vst1q_u8(b.as_mut_ptr() as *mut u8, v) }
712    }
713    #[inline(always)]
714    fn load_u16_lo(self, b: &[u16]) -> R {
715        assert!(b.len() >= 4, "simd: slice too short");
716        // SAFETY: 8 readable bytes (checked).
717        unsafe { vcombine_u8(vld1_u8(b.as_ptr() as *const u8), vdup_n_u8(0)) }
718    }
719    #[inline(always)]
720    fn store_u16_lo(self, b: &mut [u16], v: R) {
721        assert!(b.len() >= 4, "simd: slice too short");
722        // SAFETY: 8 writable bytes (checked).
723        unsafe { vst1_u8(b.as_mut_ptr() as *mut u8, vget_low_u8(v)) }
724    }
725    #[inline(always)]
726    fn srl16x(self, a: R, n: u32) -> R {
727        if n >= 16 {
728            self.zero()
729        } else {
730            neon!(vshlq_u16(u16s(a), vdupq_n_s16(-(n as i16)))).r()
731        }
732    }
733
734    // R9-jxl
735    #[inline(always)]
736    fn load_f32(self, b: &[f32]) -> R {
737        assert!(b.len() >= 4, "simd: slice too short");
738        // SAFETY: 16 readable bytes (checked).
739        unsafe { vld1q_u8(b.as_ptr() as *const u8) }
740    }
741    #[inline(always)]
742    fn store_f32(self, b: &mut [f32], v: R) {
743        assert!(b.len() >= 4, "simd: slice too short");
744        // SAFETY: 16 writable bytes (checked).
745        unsafe { vst1q_u8(b.as_mut_ptr() as *mut u8, v) }
746    }
747
748    // R9-webp
749    #[inline(always)]
750    fn adds8(self, a: R, b: R) -> R {
751        neon!(vqaddq_s8(i8s(a), i8s(b))).r()
752    }
753    #[inline(always)]
754    fn subs8(self, a: R, b: R) -> R {
755        neon!(vqsubq_s8(i8s(a), i8s(b))).r()
756    }
757
758    // r9-dds
759    #[inline(always)]
760    fn cmpltf(self, a: R, b: R) -> R {
761        neon!(vcltq_f32(f32s(a), f32s(b))).r()
762    }
763
764    // jxl-speed
765    #[inline(always)]
766    fn load_f32_lanes(self, a: &[f32], _b: &[f32]) -> R {
767        self.load_f32(a)
768    }
769    #[inline(always)]
770    fn store_f32_lanes(self, a: &mut [f32], _b: &mut [f32], v: R) {
771        self.store_f32(a, v);
772    }
773
774    // perf-blend
775    #[inline(always)]
776    fn packssdw(self, a: R, b: R) -> R {
777        self.packs32(a, b)
778    }
779    #[inline(always)]
780    fn packuswb_px(self, v: R) -> R {
781        // SAFETY: see neon!.
782        unsafe {
783            let p = vqmovun_s16(i16s(v));
784            vcombine_u8(p, p)
785        }
786    }
787    #[inline(always)]
788    fn ptest_raw(self, a: R, b: R) -> u32 {
789        let (a, b) = (neon!(vgetq_lane_u64::<0>(u64s(a))), neon!(vgetq_lane_u64::<0>(u64s(b))));
790        if a & b == 0 {
791            1
792        } else if !a & b == 0 {
793            2
794        } else {
795            0
796        }
797    }
798    #[inline(always)]
799    fn ptestz_raw(self, a: R, b: R) -> bool {
800        neon!(vgetq_lane_u64::<0>(u64s(self.and(a, b)))) == 0
801    }
802}