1use crate::Simd;
17use core::arch::aarch64::*;
18use rt::portable::fx;
19
20#[derive(Clone, Copy)]
22pub struct Neon128;
23
24type R = uint8x16_t;
25
26#[inline(always)]
27fn chk(b: &[u8], n: usize) {
28 assert!(b.len() >= n, "simd: slice too short");
29}
30
31macro_rules! neon {
34 ($e:expr) => {
35 unsafe { $e }
36 };
37}
38
39#[inline(always)]
41fn i8s(a: R) -> int8x16_t {
42 neon!(vreinterpretq_s8_u8(a))
43}
44#[inline(always)]
45fn u16s(a: R) -> uint16x8_t {
46 neon!(vreinterpretq_u16_u8(a))
47}
48#[inline(always)]
49fn i16s(a: R) -> int16x8_t {
50 neon!(vreinterpretq_s16_u8(a))
51}
52#[inline(always)]
53fn u32s(a: R) -> uint32x4_t {
54 neon!(vreinterpretq_u32_u8(a))
55}
56#[inline(always)]
57fn i32s(a: R) -> int32x4_t {
58 neon!(vreinterpretq_s32_u8(a))
59}
60#[inline(always)]
61fn u64s(a: R) -> uint64x2_t {
62 neon!(vreinterpretq_u64_u8(a))
63}
64#[inline(always)]
65fn f32s(a: R) -> float32x4_t {
66 neon!(vreinterpretq_f32_u8(a))
67}
68#[inline(always)]
69fn f64s(a: R) -> float64x2_t {
70 neon!(vreinterpretq_f64_u8(a))
71}
72trait Lanes: Copy {
73 fn r(self) -> R;
74}
75macro_rules! lanes {
76 ($t:ty, $to:ident) => {
77 impl Lanes for $t {
78 #[inline(always)]
79 fn r(self) -> R {
80 neon!($to(self))
81 }
82 }
83 };
84}
85lanes!(int8x16_t, vreinterpretq_u8_s8);
86lanes!(uint16x8_t, vreinterpretq_u8_u16);
87lanes!(int16x8_t, vreinterpretq_u8_s16);
88lanes!(uint32x4_t, vreinterpretq_u8_u32);
89lanes!(int32x4_t, vreinterpretq_u8_s32);
90lanes!(uint64x2_t, vreinterpretq_u8_u64);
91lanes!(float32x4_t, vreinterpretq_u8_f32);
92lanes!(float64x2_t, vreinterpretq_u8_f64);
93
94#[cold]
96#[inline(never)]
97fn sse_nan32(a: float32x4_t, b: float32x4_t, r: float32x4_t) -> float32x4_t {
98 let (mut x, mut y, mut z) = ([0f32; 4], [0f32; 4], [0f32; 4]);
99 unsafe {
101 vst1q_f32(x.as_mut_ptr(), a);
102 vst1q_f32(y.as_mut_ptr(), b);
103 vst1q_f32(z.as_mut_ptr(), r);
104 for i in 0..4 {
105 z[i] = fx::arithmetic32(x[i], y[i], z[i]);
106 }
107 vld1q_f32(z.as_ptr())
108 }
109}
110#[cold]
111#[inline(never)]
112fn sse_nan64(a: float64x2_t, b: float64x2_t, r: float64x2_t) -> float64x2_t {
113 let (mut x, mut y, mut z) = ([0f64; 2], [0f64; 2], [0f64; 2]);
114 unsafe {
116 vst1q_f64(x.as_mut_ptr(), a);
117 vst1q_f64(y.as_mut_ptr(), b);
118 vst1q_f64(z.as_mut_ptr(), r);
119 for i in 0..2 {
120 z[i] = fx::arithmetic64(x[i], y[i], z[i]);
121 }
122 vld1q_f64(z.as_ptr())
123 }
124}
125#[inline(always)]
127fn result32(a: float32x4_t, b: float32x4_t, r: float32x4_t) -> R {
128 if neon!(vmaxvq_f32(r)).is_nan() {
129 sse_nan32(a, b, r).r()
130 } else {
131 r.r()
132 }
133}
134#[inline(always)]
135fn result64(a: float64x2_t, b: float64x2_t, r: float64x2_t) -> R {
136 if neon!(vmaxvq_f64(r)).is_nan() {
137 sse_nan64(a, b, r).r()
138 } else {
139 r.r()
140 }
141}
142
143#[inline(always)]
145fn tbl(a: R, index: &[u8; 16]) -> R {
146 unsafe { vqtbl1q_u8(a, vld1q_u8(index.as_ptr())) }
148}
149const fn shuf16_index(imm: i32) -> [u8; 16] {
150 let mut t = [0; 16];
151 let mut i = 0;
152 while i < 8 {
153 let word = i / 4 * 4 + ((imm >> (i % 4 * 2)) & 3) as usize;
154 t[i * 2] = (word * 2) as u8;
155 t[i * 2 + 1] = (word * 2 + 1) as u8;
156 i += 1;
157 }
158 t
159}
160const fn shuf32_index(imm: i32) -> [u8; 16] {
161 let mut t = [0; 16];
162 let mut i = 0;
163 while i < 16 {
164 t[i] = (((imm >> (i / 4 * 2)) & 3) * 4) as u8 + (i % 4) as u8;
165 i += 1;
166 }
167 t
168}
169const fn bsll_index(n: i32) -> [u8; 16] {
170 let mut t = [255; 16];
171 let mut i = 0;
172 while i < 16 {
173 if (n as u32) <= i {
174 t[i as usize] = (i - n as u32) as u8;
175 }
176 i += 1;
177 }
178 t
179}
180const fn bsrl_index(n: i32) -> [u8; 16] {
181 let mut t = [255; 16];
182 let mut i = 0;
183 while i < 16 {
184 if (n as u32) < 16 - i {
185 t[i as usize] = (i + n as u32) as u8;
186 }
187 i += 1;
188 }
189 t
190}
191
192const TWO31F: f32 = 2147483648.0;
193const TWO31D: f64 = 2147483648.0;
194
195impl Simd for Neon128 {
196 type V = R;
197 const BYTES: usize = 16;
198 const PX: usize = 2;
199
200 #[inline(always)]
201 fn zero(self) -> R {
202 neon!(vdupq_n_u8(0))
203 }
204 #[inline(always)]
205 fn splat8(self, v: u8) -> R {
206 neon!(vdupq_n_u8(v))
207 }
208 #[inline(always)]
209 fn splat16(self, v: u16) -> R {
210 neon!(vdupq_n_u16(v)).r()
211 }
212 #[inline(always)]
213 fn splat32(self, v: u32) -> R {
214 neon!(vdupq_n_u32(v)).r()
215 }
216 #[inline(always)]
217 fn splat64(self, v: u64) -> R {
218 neon!(vdupq_n_u64(v)).r()
219 }
220 #[inline(always)]
221 fn splatf(self, v: f32) -> R {
222 neon!(vdupq_n_f32(v)).r()
223 }
224
225 #[inline(always)]
227 fn load(self, b: &[u8]) -> R {
228 chk(b, 16);
229 unsafe { vld1q_u8(b.as_ptr()) }
231 }
232 #[inline(always)]
233 fn store(self, b: &mut [u8], v: R) {
234 chk(b, 16);
235 unsafe { vst1q_u8(b.as_mut_ptr(), v) }
237 }
238 #[inline(always)]
239 fn load_px(self, b: &[u8]) -> R {
240 self.widen_px(self.load_raw(b))
241 }
242 #[inline(always)]
243 fn store_px(self, b: &mut [u8], v: R) {
244 chk(b, 8);
245 unsafe { vst1_u8(b.as_mut_ptr(), vqmovun_s16(i16s(v))) }
247 }
248 #[inline(always)]
249 fn bcast_px(self, b: &[u8]) -> R {
250 chk(b, 2);
251 neon!(vcombine_u16(vdup_n_u16(b[0] as u16), vdup_n_u16(b[1] as u16))).r()
252 }
253 #[inline(always)]
254 fn load_raw(self, b: &[u8]) -> R {
255 chk(b, 8);
256 unsafe { vcombine_u8(vld1_u8(b.as_ptr()), vdup_n_u8(0)) }
258 }
259 #[inline(always)]
260 fn store_raw(self, b: &mut [u8], v: R) {
261 chk(b, 8);
262 unsafe { vst1_u8(b.as_mut_ptr(), vget_low_u8(v)) }
264 }
265 #[inline(always)]
266 fn widen_px(self, v: R) -> R {
267 neon!(vmovl_u8(vget_low_u8(v))).r()
268 }
269 #[inline(always)]
270 fn load_lane32(self, b: &[u8]) -> R {
271 chk(b, 4);
272 self.from_i32(i32::from_le_bytes([b[0], b[1], b[2], b[3]]))
273 }
274 #[inline(always)]
275 fn load_i32(self, b: &[i32]) -> R {
276 assert!(b.len() >= 4, "simd: slice too short");
277 unsafe { vld1q_u8(b.as_ptr() as *const u8) }
279 }
280 #[inline(always)]
281 fn store_i32(self, b: &mut [i32], v: R) {
282 assert!(b.len() >= 4, "simd: slice too short");
283 unsafe { vst1q_u8(b.as_mut_ptr() as *mut u8, v) }
285 }
286 #[inline(always)]
287 fn from_i32(self, v: i32) -> R {
288 neon!(vsetq_lane_s32::<0>(v, vdupq_n_s32(0))).r()
289 }
290 #[inline(always)]
291 fn to_i32(self, v: R) -> i32 {
292 neon!(vgetq_lane_s32::<0>(i32s(v)))
293 }
294
295 #[inline(always)]
297 fn add8(self, a: R, b: R) -> R {
298 neon!(vaddq_u8(a, b))
299 }
300 #[inline(always)]
301 fn sub8(self, a: R, b: R) -> R {
302 neon!(vsubq_u8(a, b))
303 }
304 #[inline(always)]
305 fn addus8(self, a: R, b: R) -> R {
306 neon!(vqaddq_u8(a, b))
307 }
308 #[inline(always)]
309 fn subus8(self, a: R, b: R) -> R {
310 neon!(vqsubq_u8(a, b))
311 }
312 #[inline(always)]
313 fn maxu8(self, a: R, b: R) -> R {
314 neon!(vmaxq_u8(a, b))
315 }
316 #[inline(always)]
317 fn minu8(self, a: R, b: R) -> R {
318 neon!(vminq_u8(a, b))
319 }
320 #[inline(always)]
321 fn cmpeq8(self, a: R, b: R) -> R {
322 neon!(vceqq_u8(a, b))
323 }
324 #[inline(always)]
325 fn avgu8(self, a: R, b: R) -> R {
326 neon!(vrhaddq_u8(a, b))
327 }
328 #[inline(always)]
329 fn movemask8(self, a: R) -> u32 {
330 const BIT: [u8; 16] = [1, 2, 4, 8, 16, 32, 64, 128, 1, 2, 4, 8, 16, 32, 64, 128];
331 unsafe {
333 let bits = vandq_u8(vcltzq_s8(i8s(a)), vld1q_u8(BIT.as_ptr()));
334 let p = vpaddq_u8(bits, bits);
335 let p = vpaddq_u8(p, p);
336 let p = vpaddq_u8(p, p);
337 vgetq_lane_u16::<0>(u16s(p)) as u32
338 }
339 }
340 #[inline(always)]
341 fn unpacklo8(self, a: R, b: R) -> R {
342 neon!(vzip1q_u8(a, b))
343 }
344 #[inline(always)]
345 fn unpackhi8(self, a: R, b: R) -> R {
346 neon!(vzip2q_u8(a, b))
347 }
348 #[inline(always)]
349 fn packus16(self, a: R, b: R) -> R {
350 neon!(vqmovun_high_s16(vqmovun_s16(i16s(a)), i16s(b)))
351 }
352
353 #[inline(always)]
355 fn add16(self, a: R, b: R) -> R {
356 neon!(vaddq_u16(u16s(a), u16s(b))).r()
357 }
358 #[inline(always)]
359 fn sub16(self, a: R, b: R) -> R {
360 neon!(vsubq_u16(u16s(a), u16s(b))).r()
361 }
362 #[inline(always)]
363 fn subus16(self, a: R, b: R) -> R {
364 neon!(vqsubq_u16(u16s(a), u16s(b))).r()
365 }
366 #[inline(always)]
367 fn mullo16(self, a: R, b: R) -> R {
368 neon!(vmulq_u16(u16s(a), u16s(b))).r()
369 }
370 #[inline(always)]
371 fn mulhiu16(self, a: R, b: R) -> R {
372 let (x, y) = (u16s(a), u16s(b));
373 unsafe {
375 let lo = vmull_u16(vget_low_u16(x), vget_low_u16(y));
376 let hi = vmull_high_u16(x, y);
377 vuzp2q_u16(u16s(lo.r()), u16s(hi.r())).r()
378 }
379 }
380 #[inline(always)]
381 fn mins16(self, a: R, b: R) -> R {
382 neon!(vminq_s16(i16s(a), i16s(b))).r()
383 }
384 #[inline(always)]
385 fn maxs16(self, a: R, b: R) -> R {
386 neon!(vmaxq_s16(i16s(a), i16s(b))).r()
387 }
388 #[inline(always)]
389 fn cmpeq16(self, a: R, b: R) -> R {
390 neon!(vceqq_u16(u16s(a), u16s(b))).r()
391 }
392 #[inline(always)]
393 fn cmpgts16(self, a: R, b: R) -> R {
394 neon!(vcgtq_s16(i16s(a), i16s(b))).r()
395 }
396 #[inline(always)]
397 fn srl16<const N: i32>(self, a: R) -> R {
398 if (N as u32) >= 16 {
399 self.zero()
400 } else {
401 neon!(vshlq_u16(u16s(a), vdupq_n_s16(-(N as i16)))).r()
402 }
403 }
404 #[inline(always)]
405 fn sll16<const N: i32>(self, a: R) -> R {
406 if (N as u32) >= 16 {
407 self.zero()
408 } else {
409 neon!(vshlq_u16(u16s(a), vdupq_n_s16(N as i16))).r()
410 }
411 }
412 #[inline(always)]
413 fn shuf16<const IMM: i32>(self, a: R) -> R {
414 tbl(a, const { &shuf16_index(IMM) })
415 }
416 #[inline(always)]
417 fn unpacklo16(self, a: R, b: R) -> R {
418 neon!(vzip1q_u16(u16s(a), u16s(b))).r()
419 }
420 #[inline(always)]
421 fn unpackhi16(self, a: R, b: R) -> R {
422 neon!(vzip2q_u16(u16s(a), u16s(b))).r()
423 }
424 #[inline(always)]
425 fn packs32(self, a: R, b: R) -> R {
426 neon!(vqmovn_high_s32(vqmovn_s32(i32s(a)), i32s(b))).r()
427 }
428 #[inline(always)]
429 fn madd16(self, a: R, b: R) -> R {
430 let (x, y) = (i16s(a), i16s(b));
431 unsafe {
433 let lo = vmull_s16(vget_low_s16(x), vget_low_s16(y));
434 let hi = vmull_high_s16(x, y);
435 vpaddq_s32(lo, hi).r()
436 }
437 }
438
439 #[inline(always)]
441 fn add32(self, a: R, b: R) -> R {
442 neon!(vaddq_u32(u32s(a), u32s(b))).r()
443 }
444 #[inline(always)]
445 fn sub32(self, a: R, b: R) -> R {
446 neon!(vsubq_u32(u32s(a), u32s(b))).r()
447 }
448 #[inline(always)]
449 fn cmpeq32(self, a: R, b: R) -> R {
450 neon!(vceqq_u32(u32s(a), u32s(b))).r()
451 }
452 #[inline(always)]
453 fn cmpgts32(self, a: R, b: R) -> R {
454 neon!(vcgtq_s32(i32s(a), i32s(b))).r()
455 }
456 #[inline(always)]
457 fn movemask32(self, a: R) -> u32 {
458 const BIT: [u32; 4] = [1, 2, 4, 8];
459 unsafe { vaddvq_u32(vandq_u32(vcltzq_s32(i32s(a)), vld1q_u32(BIT.as_ptr()))) }
461 }
462 #[inline(always)]
463 fn shuf32<const IMM: i32>(self, a: R) -> R {
464 tbl(a, const { &shuf32_index(IMM) })
465 }
466 #[inline(always)]
467 fn shufps<const IMM: i32>(self, a: R, b: R) -> R {
468 neon!(vcombine_u8(vget_low_u8(self.shuf32::<IMM>(a)), vget_high_u8(self.shuf32::<IMM>(b))))
469 }
470 #[inline(always)]
471 fn srl32<const N: i32>(self, a: R) -> R {
472 if (N as u32) >= 32 {
473 self.zero()
474 } else {
475 neon!(vshlq_u32(u32s(a), vdupq_n_s32(-N))).r()
476 }
477 }
478 #[inline(always)]
479 fn sll32<const N: i32>(self, a: R) -> R {
480 if (N as u32) >= 32 {
481 self.zero()
482 } else {
483 neon!(vshlq_u32(u32s(a), vdupq_n_s32(N))).r()
484 }
485 }
486 #[inline(always)]
487 fn sra32<const N: i32>(self, a: R) -> R {
488 neon!(vshlq_s32(i32s(a), vdupq_n_s32(-((N as u32).min(31) as i32)))).r()
489 }
490 #[inline(always)]
491 fn bsll<const N: i32>(self, a: R) -> R {
492 tbl(a, const { &bsll_index(N) })
493 }
494 #[inline(always)]
495 fn bsrl<const N: i32>(self, a: R) -> R {
496 tbl(a, const { &bsrl_index(N) })
497 }
498
499 #[inline(always)]
501 fn and(self, a: R, b: R) -> R {
502 neon!(vandq_u8(a, b))
503 }
504 #[inline(always)]
505 fn andnot(self, a: R, b: R) -> R {
506 neon!(vbicq_u8(b, a))
507 }
508 #[inline(always)]
509 fn or(self, a: R, b: R) -> R {
510 neon!(vorrq_u8(a, b))
511 }
512 #[inline(always)]
513 fn xor(self, a: R, b: R) -> R {
514 neon!(veorq_u8(a, b))
515 }
516
517 #[inline(always)]
519 fn cvt_i32_f32(self, a: R) -> R {
520 neon!(vcvtq_f32_s32(i32s(a))).r()
521 }
522 #[inline(always)]
523 fn cvtt_f32_i32(self, a: R) -> R {
524 let x = f32s(a);
525 neon!(vbslq_s32(vcltq_f32(x, vdupq_n_f32(TWO31F)), vcvtq_s32_f32(x), vdupq_n_s32(i32::MIN))).r()
527 }
528 #[inline(always)]
529 fn cvtr_f32_i32(self, a: R) -> R {
530 let x = f32s(a);
531 neon!(vbslq_s32(vcltq_f32(x, vdupq_n_f32(TWO31F)), vcvtnq_s32_f32(x), vdupq_n_s32(i32::MIN))).r()
532 }
533 #[inline(always)]
534 fn addf(self, a: R, b: R) -> R {
535 let (x, y) = (f32s(a), f32s(b));
536 result32(x, y, neon!(vaddq_f32(x, y)))
537 }
538 #[inline(always)]
539 fn subf(self, a: R, b: R) -> R {
540 let (x, y) = (f32s(a), f32s(b));
541 result32(x, y, neon!(vsubq_f32(x, y)))
542 }
543 #[inline(always)]
544 fn mulf(self, a: R, b: R) -> R {
545 let (x, y) = (f32s(a), f32s(b));
546 result32(x, y, neon!(vmulq_f32(x, y)))
547 }
548 #[inline(always)]
549 fn divf(self, a: R, b: R) -> R {
550 let (x, y) = (f32s(a), f32s(b));
551 result32(x, y, neon!(vdivq_f32(x, y)))
552 }
553
554 #[inline(always)]
556 fn splatd(self, v: f64) -> R {
557 neon!(vdupq_n_f64(v)).r()
558 }
559 #[inline(always)]
560 fn addd(self, a: R, b: R) -> R {
561 let (x, y) = (f64s(a), f64s(b));
562 result64(x, y, neon!(vaddq_f64(x, y)))
563 }
564 #[inline(always)]
565 fn subd(self, a: R, b: R) -> R {
566 let (x, y) = (f64s(a), f64s(b));
567 result64(x, y, neon!(vsubq_f64(x, y)))
568 }
569 #[inline(always)]
570 fn muld(self, a: R, b: R) -> R {
571 let (x, y) = (f64s(a), f64s(b));
572 result64(x, y, neon!(vmulq_f64(x, y)))
573 }
574 #[inline(always)]
575 fn divd(self, a: R, b: R) -> R {
576 let (x, y) = (f64s(a), f64s(b));
577 result64(x, y, neon!(vdivq_f64(x, y)))
578 }
579 #[inline(always)]
580 fn sqrtd(self, a: R) -> R {
581 let x = f64s(a);
582 result64(x, x, neon!(vsqrtq_f64(x)))
583 }
584 #[inline(always)]
585 fn maxd(self, a: R, b: R) -> R {
586 let (x, y) = (f64s(a), f64s(b));
587 neon!(vbslq_f64(vcgtq_f64(x, y), x, y)).r()
588 }
589 #[inline(always)]
590 fn mind(self, a: R, b: R) -> R {
591 let (x, y) = (f64s(a), f64s(b));
592 neon!(vbslq_f64(vcltq_f64(x, y), x, y)).r()
593 }
594 #[inline(always)]
595 fn cmpltd(self, a: R, b: R) -> R {
596 neon!(vcltq_f64(f64s(a), f64s(b))).r()
597 }
598 #[inline(always)]
599 fn cvt_i32_f64(self, a: R) -> R {
600 neon!(vcvtq_f64_s64(vmovl_s32(vget_low_s32(i32s(a))))).r()
601 }
602 #[inline(always)]
603 fn cvtt_f64_i32(self, a: R) -> R {
604 let x = f64s(a);
605 unsafe {
607 let v = vqmovn_s64(vcvtq_s64_f64(x));
609 let ok = vmovn_u64(vcltq_f64(x, vdupq_n_f64(TWO31D)));
610 vcombine_s32(vbsl_s32(ok, v, vdup_n_s32(i32::MIN)), vdup_n_s32(0)).r()
611 }
612 }
613 #[inline(always)]
614 fn store_i32_half(self, b: &mut [i32], v: R) {
615 assert!(b.len() >= 2, "simd: slice too short");
616 unsafe { vst1_u8(b.as_mut_ptr() as *mut u8, vget_low_u8(v)) }
618 }
619
620 #[inline(always)]
622 fn sra16<const N: i32>(self, a: R) -> R {
623 neon!(vshlq_s16(i16s(a), vdupq_n_s16(-((N as u32).min(15) as i16)))).r()
624 }
625 #[inline(always)]
626 fn mulhis16(self, a: R, b: R) -> R {
627 let (x, y) = (i16s(a), i16s(b));
628 unsafe {
630 let lo = vmull_s16(vget_low_s16(x), vget_low_s16(y));
631 let hi = vmull_high_s16(x, y);
632 vuzp2q_u16(u16s(lo.r()), u16s(hi.r())).r()
633 }
634 }
635 #[inline(always)]
636 fn packs16(self, a: R, b: R) -> R {
637 neon!(vqmovn_high_s16(vqmovn_s16(i16s(a)), i16s(b))).r()
638 }
639 #[inline(always)]
640 fn load_i16(self, b: &[i16]) -> R {
641 assert!(b.len() >= 8, "simd: slice too short");
642 unsafe { vld1q_u8(b.as_ptr() as *const u8) }
644 }
645 #[inline(always)]
646 fn store_i16(self, b: &mut [i16], v: R) {
647 assert!(b.len() >= 8, "simd: slice too short");
648 unsafe { vst1q_u8(b.as_mut_ptr() as *mut u8, v) }
650 }
651 #[inline(always)]
652 fn store_lanes2(self, b: &mut [u8], lo: R, hi: R) {
653 chk(b, 32);
654 unsafe {
656 vst1q_u8(b.as_mut_ptr(), lo);
657 vst1q_u8(b.as_mut_ptr().add(16), hi);
658 }
659 }
660 #[inline(always)]
661 fn load_lanes2(self, b: &[u8]) -> (R, R) {
662 chk(b, 32);
663 unsafe { (vld1q_u8(b.as_ptr()), vld1q_u8(b.as_ptr().add(16))) }
665 }
666 #[inline(always)]
667 fn load_i16_lanes(self, a: &[i16], _b: &[i16]) -> R {
668 self.load_i16(a)
669 }
670 #[inline(always)]
671 fn store_raw_lanes(self, a: &mut [u8], _b: &mut [u8], v: R) {
672 self.store_raw(a, v);
673 }
674 #[inline(always)]
675 fn sadu8(self, a: R, b: R) -> R {
676 neon!(vpaddlq_u32(vpaddlq_u16(vpaddlq_u8(vabdq_u8(a, b))))).r()
677 }
678 #[inline(always)]
679 fn add64(self, a: R, b: R) -> R {
680 neon!(vaddq_u64(u64s(a), u64s(b))).r()
681 }
682
683 #[inline(always)]
685 fn maxf(self, a: R, b: R) -> R {
686 let (x, y) = (f32s(a), f32s(b));
687 neon!(vbslq_f32(vcgtq_f32(x, y), x, y)).r()
688 }
689 #[inline(always)]
690 fn minf(self, a: R, b: R) -> R {
691 let (x, y) = (f32s(a), f32s(b));
692 neon!(vbslq_f32(vcltq_f32(x, y), x, y)).r()
693 }
694 #[inline(always)]
695 fn sqrtf(self, a: R) -> R {
696 let x = f32s(a);
697 result32(x, x, neon!(vsqrtq_f32(x)))
698 }
699
700 #[inline(always)]
702 fn load_u16(self, b: &[u16]) -> R {
703 assert!(b.len() >= 8, "simd: slice too short");
704 unsafe { vld1q_u8(b.as_ptr() as *const u8) }
706 }
707 #[inline(always)]
708 fn store_u16(self, b: &mut [u16], v: R) {
709 assert!(b.len() >= 8, "simd: slice too short");
710 unsafe { vst1q_u8(b.as_mut_ptr() as *mut u8, v) }
712 }
713 #[inline(always)]
714 fn load_u16_lo(self, b: &[u16]) -> R {
715 assert!(b.len() >= 4, "simd: slice too short");
716 unsafe { vcombine_u8(vld1_u8(b.as_ptr() as *const u8), vdup_n_u8(0)) }
718 }
719 #[inline(always)]
720 fn store_u16_lo(self, b: &mut [u16], v: R) {
721 assert!(b.len() >= 4, "simd: slice too short");
722 unsafe { vst1_u8(b.as_mut_ptr() as *mut u8, vget_low_u8(v)) }
724 }
725 #[inline(always)]
726 fn srl16x(self, a: R, n: u32) -> R {
727 if n >= 16 {
728 self.zero()
729 } else {
730 neon!(vshlq_u16(u16s(a), vdupq_n_s16(-(n as i16)))).r()
731 }
732 }
733
734 #[inline(always)]
736 fn load_f32(self, b: &[f32]) -> R {
737 assert!(b.len() >= 4, "simd: slice too short");
738 unsafe { vld1q_u8(b.as_ptr() as *const u8) }
740 }
741 #[inline(always)]
742 fn store_f32(self, b: &mut [f32], v: R) {
743 assert!(b.len() >= 4, "simd: slice too short");
744 unsafe { vst1q_u8(b.as_mut_ptr() as *mut u8, v) }
746 }
747
748 #[inline(always)]
750 fn adds8(self, a: R, b: R) -> R {
751 neon!(vqaddq_s8(i8s(a), i8s(b))).r()
752 }
753 #[inline(always)]
754 fn subs8(self, a: R, b: R) -> R {
755 neon!(vqsubq_s8(i8s(a), i8s(b))).r()
756 }
757
758 #[inline(always)]
760 fn cmpltf(self, a: R, b: R) -> R {
761 neon!(vcltq_f32(f32s(a), f32s(b))).r()
762 }
763
764 #[inline(always)]
766 fn load_f32_lanes(self, a: &[f32], _b: &[f32]) -> R {
767 self.load_f32(a)
768 }
769 #[inline(always)]
770 fn store_f32_lanes(self, a: &mut [f32], _b: &mut [f32], v: R) {
771 self.store_f32(a, v);
772 }
773
774 #[inline(always)]
776 fn packssdw(self, a: R, b: R) -> R {
777 self.packs32(a, b)
778 }
779 #[inline(always)]
780 fn packuswb_px(self, v: R) -> R {
781 unsafe {
783 let p = vqmovun_s16(i16s(v));
784 vcombine_u8(p, p)
785 }
786 }
787 #[inline(always)]
788 fn ptest_raw(self, a: R, b: R) -> u32 {
789 let (a, b) = (neon!(vgetq_lane_u64::<0>(u64s(a))), neon!(vgetq_lane_u64::<0>(u64s(b))));
790 if a & b == 0 {
791 1
792 } else if !a & b == 0 {
793 2
794 } else {
795 0
796 }
797 }
798 #[inline(always)]
799 fn ptestz_raw(self, a: R, b: R) -> bool {
800 neon!(vgetq_lane_u64::<0>(u64s(self.and(a, b)))) == 0
801 }
802}