use core::arch::aarch64::*;
#[inline]
#[target_feature(enable = "neon")]
pub(super) fn load_s32x4(src: &[i32]) -> int32x4_t {
debug_assert!(src.len() >= 4);
unsafe { vld1q_s32(src.as_ptr()) }
}
#[inline]
#[target_feature(enable = "neon")]
pub(super) fn load_rows4_s32x4(src: &[i32], stride: usize, x: usize) -> int32x4_t {
debug_assert!(src.len() > 3 * stride + x);
let lanes = [
src[x],
src[stride + x],
src[2 * stride + x],
src[3 * stride + x],
];
unsafe { vld1q_s32(lanes.as_ptr()) }
}
#[inline]
#[target_feature(enable = "neon")]
pub(super) fn store_s32x4(dst: &mut [i32], v: int32x4_t) {
debug_assert!(dst.len() >= 4);
unsafe { vst1q_s32(dst.as_mut_ptr(), v) }
}
#[inline]
#[target_feature(enable = "neon")]
pub(super) fn store_rows4_s32x4(dst: &mut [i32], stride: usize, x: usize, v: int32x4_t) {
debug_assert!(dst.len() > 3 * stride + x);
let mut lanes = [0i32; 4];
unsafe { vst1q_s32(lanes.as_mut_ptr(), v) };
dst[x] = lanes[0];
dst[stride + x] = lanes[1];
dst[2 * stride + x] = lanes[2];
dst[3 * stride + x] = lanes[3];
}
#[inline]
#[target_feature(enable = "neon")]
pub(super) fn ld_i16x4(src: &[i16]) -> int16x4_t {
debug_assert!(src.len() >= 4);
unsafe { vld1_s16(src.as_ptr()) }
}
#[inline]
#[target_feature(enable = "neon")]
pub(super) fn st_i16x4(dst: &mut [i16], v: int16x4_t) {
debug_assert!(dst.len() >= 4);
unsafe { vst1_s16(dst.as_mut_ptr(), v) }
}
#[inline]
#[target_feature(enable = "neon")]
pub(super) fn zero() -> int32x4_t {
vdupq_n_s32(0)
}
#[inline]
#[target_feature(enable = "neon")]
pub(super) fn add(a: int32x4_t, b: int32x4_t) -> int32x4_t {
vaddq_s32(a, b)
}
#[inline]
#[target_feature(enable = "neon")]
pub(super) fn sub(a: int32x4_t, b: int32x4_t) -> int32x4_t {
vsubq_s32(a, b)
}
#[inline]
#[target_feature(enable = "neon")]
pub(super) fn mul_const(v: int32x4_t, c: i32) -> int32x4_t {
vmulq_s32(v, vdupq_n_s32(c))
}
#[inline]
#[target_feature(enable = "neon")]
pub(super) fn madd_const(acc: int32x4_t, v: int32x4_t, c: i32) -> int32x4_t {
add(acc, mul_const(v, c))
}
#[inline]
#[target_feature(enable = "neon")]
pub(super) fn round_shift_s32x4(v: int32x4_t, add: i32, shift: i32) -> int32x4_t {
vshlq_s32(vaddq_s32(v, vdupq_n_s32(add)), vdupq_n_s32(-shift))
}
#[inline]
#[target_feature(enable = "neon")]
pub(super) fn round_shift_clip_i16_s32x4(v: int32x4_t, add: i32, shift: i32) -> int32x4_t {
let v = round_shift_s32x4(v, add, shift);
vmaxq_s32(vminq_s32(v, vdupq_n_s32(32767)), vdupq_n_s32(-32768))
}