use std::arch::aarch64::*;
use crate::gdf_tables::{GDF_ALPHA, GDF_INTER_ERROR, GDF_INTRA_ERROR, GDF_WEIGHT};
const GDF_PREP_COORDS: [[i8; 2]; 18] = [
[6, 0],
[5, 0],
[4, 0],
[3, 0],
[2, 1],
[2, 0],
[2, -1],
[1, 2],
[1, 1],
[1, 0],
[1, -1],
[1, -2],
[0, 6],
[0, 5],
[0, 4],
[0, 3],
[0, 2],
[0, 1],
];
#[inline(always)]
fn load_i32x4(a: &[i32; 4]) -> int32x4_t {
unsafe { vld1q_s32(a.as_ptr()) }
}
#[inline(always)]
fn load_i16x4_i32(a: &[i16; 4]) -> int32x4_t {
unsafe { vmovl_s16(vld1_s16(a.as_ptr())) }
}
#[inline(always)]
fn load_u16x4_i32(a: &[u16; 4]) -> int32x4_t {
unsafe { vreinterpretq_s32_u32(vmovl_u16(vld1_u16(a.as_ptr()))) }
}
#[inline]
#[target_feature(enable = "neon")]
fn load_u8x4_i32(a: &[u8; 4]) -> int32x4_t {
let dup = vreinterpret_u8_u32(vdup_n_u32(u32::from_le_bytes(*a)));
vreinterpretq_s32_u32(vmovl_u16(vget_low_u16(vmovl_u8(dup))))
}
#[inline]
#[target_feature(enable = "neon")]
fn load_i16x8_i32x2(a: &[i16; 8]) -> (int32x4_t, int32x4_t) {
let v = unsafe { vld1q_s16(a.as_ptr()) };
(vmovl_s16(vget_low_s16(v)), vmovl_s16(vget_high_s16(v)))
}
#[inline]
#[target_feature(enable = "neon")]
fn load_i16x8(a: &[i16; 8]) -> int16x8_t {
unsafe { vld1q_s16(a.as_ptr()) }
}
#[inline]
#[target_feature(enable = "neon")]
fn load_u8x8_i16(a: &[u8; 8]) -> int16x8_t {
unsafe { vreinterpretq_s16_u16(vmovl_u8(vld1_u8(a.as_ptr()))) }
}
#[inline]
#[target_feature(enable = "neon")]
fn load_u8x8_u16(a: &[u8; 8]) -> uint16x8_t {
unsafe { vmovl_u8(vld1_u8(a.as_ptr())) }
}
#[inline]
#[target_feature(enable = "neon")]
fn madd_i16x8_const(
a: int16x8_t,
b: int16x8_t,
w1: int16x4_t,
w2: int16x4_t,
) -> (int32x4_t, int32x4_t) {
(
vmlal_s16(vmull_s16(vget_low_s16(a), w1), vget_low_s16(b), w2),
vmlal_s16(vmull_s16(vget_high_s16(a), w1), vget_high_s16(b), w2),
)
}
#[inline]
#[target_feature(enable = "neon")]
fn madd_i16x8(a: int16x8_t, b: int16x8_t, w1: int16x8_t, w2: int16x8_t) -> (int32x4_t, int32x4_t) {
(
vmlal_s16(
vmull_s16(vget_low_s16(a), vget_low_s16(w1)),
vget_low_s16(b),
vget_low_s16(w2),
),
vmlal_s16(
vmull_s16(vget_high_s16(a), vget_high_s16(w1)),
vget_high_s16(b),
vget_high_s16(w2),
),
)
}
#[inline]
#[target_feature(enable = "neon")]
fn load_u16x8_i32x2(a: &[u16; 8]) -> (int32x4_t, int32x4_t) {
let v = unsafe { vld1q_u16(a.as_ptr()) };
(
vreinterpretq_s32_u32(vmovl_u16(vget_low_u16(v))),
vreinterpretq_s32_u32(vmovl_u16(vget_high_u16(v))),
)
}
#[inline]
#[target_feature(enable = "neon")]
fn load_u16x8(a: &[u16; 8]) -> uint16x8_t {
unsafe { vld1q_u16(a.as_ptr()) }
}
#[inline]
#[target_feature(enable = "neon")]
fn load_u16x8_s16(a: &[u16; 8]) -> int16x8_t {
unsafe { vreinterpretq_s16_u16(vld1q_u16(a.as_ptr())) }
}
#[inline]
#[target_feature(enable = "neon")]
fn store_u16x8(a: &mut [u16; 8], v: uint16x8_t) {
unsafe { vst1q_u16(a.as_mut_ptr(), v) };
}
#[inline]
#[target_feature(enable = "neon")]
fn store_i32x4_u16_clip(a: &mut [u16; 4], v: int32x4_t, max_v: int32x4_t) {
let v = vminq_s32(vmaxq_s32(v, vdupq_n_s32(0)), max_v);
unsafe { vst1_u16(a.as_mut_ptr(), vqmovun_s32(v)) };
}
#[inline]
#[target_feature(enable = "neon")]
fn store_i32x8_u16_clip(a: &mut [u16; 8], lo: int32x4_t, hi: int32x4_t, max_v: int32x4_t) {
let zero = vdupq_n_s32(0);
let lo = vminq_s32(vmaxq_s32(lo, zero), max_v);
let hi = vminq_s32(vmaxq_s32(hi, zero), max_v);
unsafe {
vst1q_u16(
a.as_mut_ptr(),
vcombine_u16(vqmovun_s32(lo), vqmovun_s32(hi)),
)
};
}
#[inline]
#[target_feature(enable = "neon")]
fn shr(v: int32x4_t, sh: i32) -> int32x4_t {
vshlq_s32(v, vdupq_n_s32(-sh))
}
#[target_feature(enable = "neon")]
pub(crate) fn residual_add_row_hbd_neon(
dst: &mut [u16],
c: &[i32],
n: usize,
rnd: i32,
shift: i32,
bitdepth_max: i32,
) {
let rnd_v = vdupq_n_s32(rnd);
let max_v = vdupq_n_s32(bitdepth_max);
let f = |d: int32x4_t, cv: int32x4_t| vaddq_s32(d, shr(vaddq_s32(cv, rnd_v), shift));
let (d8, r8) = dst[..n].as_chunks_mut::<8>();
let (c8, _) = c[..n].as_chunks::<8>();
for (d, cv) in d8.iter_mut().zip(c8) {
let (d0, d1) = load_u16x8_i32x2(&*d);
let c0 = load_i32x4((&cv[..4]).try_into().unwrap());
let c1 = load_i32x4((&cv[4..]).try_into().unwrap());
store_i32x8_u16_clip(d, f(d0, c0), f(d1, c1), max_v);
}
let done = d8.len() * 8;
let (d4, r4) = r8.as_chunks_mut::<4>();
let (c4, cr) = c[done..n].as_chunks::<4>();
for (d, cv) in d4.iter_mut().zip(c4) {
store_i32x4_u16_clip(d, f(load_u16x4_i32(&*d), load_i32x4(cv)), max_v);
}
for (d, &cv) in r4.iter_mut().zip(cr) {
*d = ((*d as i32) + ((cv + rnd) >> shift)).clamp(0, bitdepth_max) as u16;
}
}
#[target_feature(enable = "neon")]
pub(crate) fn dc_add_row_hbd_neon(dst: &mut [u16], dc: i32, n: usize, bitdepth_max: i32) {
if dc == 0 {
return;
}
let dc_v = vdupq_n_s32(dc);
let max_v = vdupq_n_s32(bitdepth_max);
let (d32, r32) = dst[..n].as_chunks_mut::<32>();
for d in d32.iter_mut() {
let (d01, d23) = d.split_at_mut(16);
let (d0, d1) = d01.split_at_mut(8);
let (d2, d3) = d23.split_at_mut(8);
let d0: &mut [u16; 8] = d0.try_into().unwrap();
let d1: &mut [u16; 8] = d1.try_into().unwrap();
let d2: &mut [u16; 8] = d2.try_into().unwrap();
let d3: &mut [u16; 8] = d3.try_into().unwrap();
let (d00, d01) = load_u16x8_i32x2(&*d0);
let (d10, d11) = load_u16x8_i32x2(&*d1);
let (d20, d21) = load_u16x8_i32x2(&*d2);
let (d30, d31) = load_u16x8_i32x2(&*d3);
store_i32x8_u16_clip(d0, vaddq_s32(d00, dc_v), vaddq_s32(d01, dc_v), max_v);
store_i32x8_u16_clip(d1, vaddq_s32(d10, dc_v), vaddq_s32(d11, dc_v), max_v);
store_i32x8_u16_clip(d2, vaddq_s32(d20, dc_v), vaddq_s32(d21, dc_v), max_v);
store_i32x8_u16_clip(d3, vaddq_s32(d30, dc_v), vaddq_s32(d31, dc_v), max_v);
}
let (d8, r8) = r32.as_chunks_mut::<8>();
for d in d8.iter_mut() {
let (d0, d1) = load_u16x8_i32x2(&*d);
store_i32x8_u16_clip(d, vaddq_s32(d0, dc_v), vaddq_s32(d1, dc_v), max_v);
}
let (d4, r4) = r8.as_chunks_mut::<4>();
for d in d4.iter_mut() {
store_i32x4_u16_clip(d, vaddq_s32(load_u16x4_i32(&*d), dc_v), max_v);
}
for d in r4.iter_mut() {
*d = ((*d as i32) + dc).clamp(0, bitdepth_max) as u16;
}
}
#[target_feature(enable = "neon")]
pub(crate) fn avg_row_hbd_neon(
dst: &mut [u16],
t1: &[i16],
t2: &[i16],
n: usize,
rnd: i32,
sh: i32,
bitdepth_max: i32,
) {
let rnd_v = vdupq_n_s32(rnd);
let max_v = vdupq_n_s32(bitdepth_max);
let f = |a: int32x4_t, b: int32x4_t| shr(vaddq_s32(vaddq_s32(a, b), rnd_v), sh);
let (d8, r8) = dst[..n].as_chunks_mut::<8>();
let (a8, _) = t1[..n].as_chunks::<8>();
let (b8, _) = t2[..n].as_chunks::<8>();
for ((d, a), b) in d8.iter_mut().zip(a8).zip(b8) {
let (a0, a1) = load_i16x8_i32x2(a);
let (b0, b1) = load_i16x8_i32x2(b);
store_i32x8_u16_clip(d, f(a0, b0), f(a1, b1), max_v);
}
let done = d8.len() * 8;
let (d4, r4) = r8.as_chunks_mut::<4>();
let (a4, ar) = t1[done..n].as_chunks::<4>();
let (b4, br) = t2[done..n].as_chunks::<4>();
for ((d, a), b) in d4.iter_mut().zip(a4).zip(b4) {
store_i32x4_u16_clip(d, f(load_i16x4_i32(a), load_i16x4_i32(b)), max_v);
}
for ((d, &a), &b) in r4.iter_mut().zip(ar).zip(br) {
*d = ((a as i32 + b as i32 + rnd) >> sh).clamp(0, bitdepth_max) as u16;
}
}
#[allow(clippy::too_many_arguments)]
#[target_feature(enable = "neon")]
pub(crate) fn w_avg_row_hbd_neon(
dst: &mut [u16],
t1: &[i16],
t2: &[i16],
n: usize,
weight: i32,
rnd: i32,
sh: i32,
bitdepth_max: i32,
) {
let w1 = vdup_n_s16(weight as i16);
let w2 = vdup_n_s16((16 - weight) as i16);
let rnd_v = vdupq_n_s32(rnd);
let max_v = vdupq_n_s32(bitdepth_max);
let f = |s: int32x4_t| shr(vaddq_s32(s, rnd_v), sh);
let (d8, r8) = dst[..n].as_chunks_mut::<8>();
let (a8, _) = t1[..n].as_chunks::<8>();
let (b8, _) = t2[..n].as_chunks::<8>();
for ((d, a), b) in d8.iter_mut().zip(a8).zip(b8) {
let (s0, s1) = madd_i16x8_const(load_i16x8(a), load_i16x8(b), w1, w2);
store_i32x8_u16_clip(d, f(s0), f(s1), max_v);
}
let done = d8.len() * 8;
let (d4, r4) = r8.as_chunks_mut::<4>();
let (a4, ar) = t1[done..n].as_chunks::<4>();
let (b4, br) = t2[done..n].as_chunks::<4>();
let w1_32 = vdupq_n_s32(weight);
let w2_32 = vdupq_n_s32(16 - weight);
let f4 = |a: int32x4_t, b: int32x4_t| {
shr(
vaddq_s32(vaddq_s32(vmulq_s32(a, w1_32), vmulq_s32(b, w2_32)), rnd_v),
sh,
)
};
for ((d, a), b) in d4.iter_mut().zip(a4).zip(b4) {
store_i32x4_u16_clip(d, f4(load_i16x4_i32(a), load_i16x4_i32(b)), max_v);
}
for ((d, &a), &b) in r4.iter_mut().zip(ar).zip(br) {
*d = ((a as i32 * weight + b as i32 * (16 - weight) + rnd) >> sh).clamp(0, bitdepth_max)
as u16;
}
}
#[allow(clippy::too_many_arguments)]
#[target_feature(enable = "neon")]
pub(crate) fn mask_row_hbd_neon(
dst: &mut [u16],
t1: &[i16],
t2: &[i16],
mask: &[u8],
n: usize,
rnd: i32,
sh: i32,
bitdepth_max: i32,
) {
let rnd_v = vdupq_n_s32(rnd);
let c64_16 = vdupq_n_s16(64);
let max_v = vdupq_n_s32(bitdepth_max);
let f = |s: int32x4_t| shr(vaddq_s32(s, rnd_v), sh);
let (d8, r8) = dst[..n].as_chunks_mut::<8>();
let (a8, _) = t1[..n].as_chunks::<8>();
let (b8, _) = t2[..n].as_chunks::<8>();
let (m8, _) = mask[..n].as_chunks::<8>();
for (((d, a), b), m) in d8.iter_mut().zip(a8).zip(b8).zip(m8) {
let mv = load_u8x8_i16(m);
let (s0, s1) = madd_i16x8(load_i16x8(a), load_i16x8(b), mv, vsubq_s16(c64_16, mv));
store_i32x8_u16_clip(d, f(s0), f(s1), max_v);
}
let done = d8.len() * 8;
let (d4, r4) = r8.as_chunks_mut::<4>();
let (a4, ar) = t1[done..n].as_chunks::<4>();
let (b4, br) = t2[done..n].as_chunks::<4>();
let (m4, mr) = mask[done..n].as_chunks::<4>();
let c64 = vdupq_n_s32(64);
let f4 = |a: int32x4_t, b: int32x4_t, m: int32x4_t| {
shr(
vaddq_s32(
vaddq_s32(vmulq_s32(a, m), vmulq_s32(b, vsubq_s32(c64, m))),
rnd_v,
),
sh,
)
};
for (((d, a), b), m) in d4.iter_mut().zip(a4).zip(b4).zip(m4) {
store_i32x4_u16_clip(
d,
f4(load_i16x4_i32(a), load_i16x4_i32(b), load_u8x4_i32(m)),
max_v,
);
}
for (((d, &a), &b), &m) in r4.iter_mut().zip(ar).zip(br).zip(mr) {
let mk = m as i32;
*d = ((a as i32 * mk + b as i32 * (64 - mk) + rnd) >> sh).clamp(0, bitdepth_max) as u16;
}
}
#[target_feature(enable = "neon")]
pub(crate) fn blend_row_hbd_neon(dst: &mut [u16], tmp: &[u16], mask: &[u8], n: usize) {
let c64 = vdupq_n_u16(64);
let f = |d: uint16x8_t, t: uint16x8_t, m: uint16x8_t| {
let inv_m = vsubq_u16(c64, m);
let lo = vmlal_u16(
vmull_u16(vget_low_u16(t), vget_low_u16(m)),
vget_low_u16(d),
vget_low_u16(inv_m),
);
let hi = vmlal_u16(
vmull_u16(vget_high_u16(t), vget_high_u16(m)),
vget_high_u16(d),
vget_high_u16(inv_m),
);
vcombine_u16(vrshrn_n_u32::<6>(lo), vrshrn_n_u32::<6>(hi))
};
let (d8, r8) = dst[..n].as_chunks_mut::<8>();
let (t8, _) = tmp[..n].as_chunks::<8>();
let (m8, _) = mask[..n].as_chunks::<8>();
for ((d, t), m) in d8.iter_mut().zip(t8).zip(m8) {
store_u16x8(d, f(load_u16x8(&*d), load_u16x8(t), load_u8x8_u16(m)));
}
let done = d8.len() * 8;
let (d4, r4) = r8.as_chunks_mut::<4>();
let (t4, tr) = tmp[done..n].as_chunks::<4>();
let (m4, mr) = mask[done..n].as_chunks::<4>();
let c64_32 = vdupq_n_s32(64);
let rnd_v = vdupq_n_s32(32);
let max_v = vdupq_n_s32(0xffff);
let f4 = |d: int32x4_t, t: int32x4_t, m: int32x4_t| {
vshrq_n_s32::<6>(vaddq_s32(
vaddq_s32(vmulq_s32(d, vsubq_s32(c64_32, m)), vmulq_s32(t, m)),
rnd_v,
))
};
for ((d, t), m) in d4.iter_mut().zip(t4).zip(m4) {
store_i32x4_u16_clip(
d,
f4(load_u16x4_i32(&*d), load_u16x4_i32(t), load_u8x4_i32(m)),
max_v,
);
}
for ((d, &t), &m) in r4.iter_mut().zip(tr).zip(mr) {
let mk = m as i32;
*d = (((*d as i32) * (64 - mk) + (t as i32) * mk + 32) >> 6) as u16;
}
}
#[target_feature(enable = "neon")]
pub(crate) fn morph_row_hbd_neon(
dst: &mut [u16],
alpha: i32,
beta: i32,
n: usize,
bitdepth_max: i32,
) {
if !(i16::MIN as i32..=i16::MAX as i32).contains(&alpha) || bitdepth_max > i16::MAX as i32 {
for d in dst[..n].iter_mut() {
*d = ((alpha * (*d as i32) + beta) >> 8).clamp(0, bitdepth_max) as u16;
}
return;
}
let a_v = vdup_n_s16(alpha as i16);
let b_v = vdupq_n_s32(beta);
let max_v = vdupq_n_s32(bitdepth_max);
let f = |d: int16x8_t| {
(
vshrq_n_s32::<8>(vaddq_s32(vmull_s16(vget_low_s16(d), a_v), b_v)),
vshrq_n_s32::<8>(vaddq_s32(vmull_s16(vget_high_s16(d), a_v), b_v)),
)
};
let (d8, r8) = dst[..n].as_chunks_mut::<8>();
for d in d8.iter_mut() {
let (o0, o1) = f(load_u16x8_s16(&*d));
store_i32x8_u16_clip(d, o0, o1, max_v);
}
let (d4, r4) = r8.as_chunks_mut::<4>();
for d in d4.iter_mut() {
let r = vshrq_n_s32::<8>(vaddq_s32(
vmulq_s32(load_u16x4_i32(&*d), vdupq_n_s32(alpha)),
b_v,
));
store_i32x4_u16_clip(d, r, max_v);
}
for d in r4.iter_mut() {
*d = ((alpha * (*d as i32) + beta) >> 8).clamp(0, bitdepth_max) as u16;
}
}
#[inline]
fn gdf_bitdepth_from_max(bitdepth_max: i32) -> i32 {
if bitdepth_max <= 0xff {
8
} else if bitdepth_max <= 0x3ff {
10
} else {
12
}
}
#[target_feature(enable = "neon")]
pub(crate) fn gdf_add_run_hbd_neon(
dst: &mut [u16],
err: &[i8],
scale: i32,
n: usize,
bitdepth_max: i32,
) {
let bitdepth = gdf_bitdepth_from_max(bitdepth_max);
let shift = 12 - bitdepth;
let sc = vdupq_n_s16(scale as i16);
let rnd = vdupq_n_s16(if shift == 0 { 0 } else { 1 << (shift - 1) });
let nsh = vdupq_n_s16(-(shift as i16));
let zero = vdupq_n_s16(0);
let max_v = vdupq_n_s16(bitdepth_max as i16);
let adj = |e: int16x8_t| {
let diff = vmulq_s16(e, sc);
let mag = vshlq_s16(vaddq_s16(vabsq_s16(diff), rnd), nsh);
vbslq_s16(vcltq_s16(diff, zero), vnegq_s16(mag), mag)
};
let clip = |v: int16x8_t| vreinterpretq_u16_s16(vminq_s16(vmaxq_s16(v, zero), max_v));
let (d8, tail_dst) = dst[..n].as_chunks_mut::<8>();
let (e8, tail_err) = err[..n].as_chunks::<8>();
for (d, e) in d8.iter_mut().zip(e8) {
let d0 = vreinterpretq_s16_u16(unsafe { vld1q_u16(d.as_ptr()) });
let e0 = unsafe { vmovl_s8(vld1_s8(e.as_ptr())) };
unsafe { vst1q_u16(d.as_mut_ptr(), clip(vaddq_s16(d0, adj(e0)))) };
}
let rnd_scalar = if shift == 0 { 0 } else { 1 << (shift - 1) };
for (d, &e) in tail_dst.iter_mut().zip(tail_err) {
let diff = e as i32 * scale;
let mag = (diff.abs() + rnd_scalar) >> shift;
let adj = if diff < 0 { -mag } else { mag };
*d = (*d as i32 + adj).clamp(0, bitdepth_max) as u16;
}
}
#[allow(clippy::too_many_arguments)]
#[target_feature(enable = "neon")]
pub(crate) fn gdf_gradient_group_hbd_neon(
dst: &mut [[u16; 4]],
d: usize,
base_cell: usize,
ncells: usize,
center_rows: [&[u16]; 2],
a_rows: [&[u16]; 2],
c_rows: [&[u16]; 2],
col0: usize,
dx: i32,
shift: u32,
) {
let mut acc = vdupq_n_s16(0);
let nsh = vdupq_n_s16(-(shift as i16));
for y in 0..2 {
let bcol = col0 - 1;
let acol = (bcol as i32 - dx) as usize;
let ccol = (bcol as i32 + dx) as usize;
let b = unsafe { vld1q_u16(center_rows[y].as_ptr().add(bcol)) };
let a = unsafe { vld1q_u16(a_rows[y].as_ptr().add(acol)) };
let c = unsafe { vld1q_u16(c_rows[y].as_ptr().add(ccol)) };
let b = vreinterpretq_s16_u16(vshlq_u16(b, nsh));
let a = vreinterpretq_s16_u16(vshlq_u16(a, nsh));
let c = vreinterpretq_s16_u16(vshlq_u16(c, nsh));
let t = vsubq_s16(vsubq_s16(vaddq_s16(b, b), a), c);
acc = vaddq_s16(acc, vabsq_s16(t));
}
let pair = vpaddq_s16(acc, acc);
store_gdf_gradient_cells_i16x8(dst, d, base_cell, ncells, pair);
}
#[inline]
#[target_feature(enable = "neon")]
fn store_gdf_gradient_cells_i16x8(
dst: &mut [[u16; 4]],
d: usize,
base_cell: usize,
ncells: usize,
v: int16x8_t,
) {
if ncells > 0 {
dst[base_cell][d] = vgetq_lane_s16::<0>(v) as u16;
}
if ncells > 1 {
dst[base_cell + 1][d] = vgetq_lane_s16::<1>(v) as u16;
}
if ncells > 2 {
dst[base_cell + 2][d] = vgetq_lane_s16::<2>(v) as u16;
}
if ncells > 3 {
dst[base_cell + 3][d] = vgetq_lane_s16::<3>(v) as u16;
}
}
#[inline]
fn gdf_prep_apply_sign(v: i32) -> i32 {
if v < 0 {
-((v.wrapping_neg() + (1 << 14)) >> 15)
} else {
(v + (1 << 14)) >> 15
}
}
#[inline]
fn gdf_prep_lookup_error(ref_dst_idx: usize, error_lut_base: usize, full_idx: usize) -> i8 {
if ref_dst_idx == 0 {
GDF_INTRA_ERROR[error_lut_base + full_idx]
} else {
GDF_INTER_ERROR[error_lut_base + full_idx]
}
}
#[inline]
#[target_feature(enable = "neon")]
fn gdf_load_pair_i32(row: &[u16], col: usize, nsh: int32x4_t) -> int32x4_t {
let raw = unsafe { core::ptr::read_unaligned(row.as_ptr().add(col).cast::<u32>()) };
let pair = vreinterpret_u16_u32(vdup_n_u32(raw));
vshlq_s32(vreinterpretq_s32_u32(vmovl_u16(pair)), nsh)
}
#[inline]
#[target_feature(enable = "neon")]
fn gdf_clip_i32x4(v: int32x4_t, lo: int32x4_t, hi: int32x4_t) -> int32x4_t {
vminq_s32(vmaxq_s32(v, lo), hi)
}
#[inline(always)]
fn gdf_prep_full_idx(v0: i32, v1: i32, v2: i32, scale: i32) -> usize {
let scale2 = scale as usize * 2;
let v0 = gdf_prep_apply_sign(v0 * scale);
let v1 = gdf_prep_apply_sign(v1 * scale);
let v2 = gdf_prep_apply_sign(v2 * scale);
let s0 = (v0.clamp(-scale, scale - 1) + scale) as usize;
let s1 = (v1.clamp(-scale, scale - 1) + scale) as usize;
let s2 = (v2.clamp(-scale, scale - 1) + scale) as usize;
((s0 * scale2) + s1) * scale2 + s2
}
#[allow(clippy::too_many_arguments)]
#[target_feature(enable = "neon")]
pub(crate) fn gdf_prep_pair_hbd_neon(
rows: [&[u16]; 13],
col: usize,
cls: usize,
shared_vals: [i32; 3],
alpha_base: usize,
weight_base: usize,
error_lut_base: usize,
scale: i32,
down_shift: u32,
up_scale: i32,
ref_dst_idx: usize,
) -> [i8; 2] {
let nsh = vdupq_n_s32(-(down_shift as i32));
let m = gdf_load_pair_i32(rows[6], col, nsh);
let up_scale_v = vdupq_n_s32(up_scale);
let v_lo = vdupq_n_s32(-512);
let v_hi = vdupq_n_s32(511);
let mut acc0 = vdupq_n_s32(shared_vals[0]);
let mut acc1 = vdupq_n_s32(shared_vals[1]);
let mut acc2 = vdupq_n_s32(shared_vals[2]);
for (k, &[dy, dx]) in GDF_PREP_COORDS.iter().enumerate() {
let dy = dy as i32;
let dx = dx as i32;
let alpha = GDF_ALPHA[alpha_base + k * 4 + cls] as i32;
let alpha_v = vdupq_n_s32(alpha);
let neg_alpha_v = vdupq_n_s32(-alpha);
let a_col = (col as i32 - dx) as usize;
let b_col = (col as i32 + dx) as usize;
let a = gdf_load_pair_i32(rows[(6 - dy) as usize], a_col, nsh);
let b = gdf_load_pair_i32(rows[(6 + dy) as usize], b_col, nsh);
let above = gdf_clip_i32x4(vmulq_s32(vsubq_s32(a, m), up_scale_v), neg_alpha_v, alpha_v);
let below = gdf_clip_i32x4(vmulq_s32(vsubq_s32(b, m), up_scale_v), neg_alpha_v, alpha_v);
let v = gdf_clip_i32x4(vaddq_s32(above, below), v_lo, v_hi);
acc0 = vaddq_s32(
acc0,
vmulq_s32(v, vdupq_n_s32(GDF_WEIGHT[weight_base + k * 4 + cls] as i32)),
);
acc1 = vaddq_s32(
acc1,
vmulq_s32(
v,
vdupq_n_s32(GDF_WEIGHT[weight_base + 88 + k * 4 + cls] as i32),
),
);
acc2 = vaddq_s32(
acc2,
vmulq_s32(
v,
vdupq_n_s32(GDF_WEIGHT[weight_base + 176 + k * 4 + cls] as i32),
),
);
}
[
gdf_prep_lookup_error(
ref_dst_idx,
error_lut_base,
gdf_prep_full_idx(
vgetq_lane_s32::<0>(acc0),
vgetq_lane_s32::<0>(acc1),
vgetq_lane_s32::<0>(acc2),
scale,
),
),
gdf_prep_lookup_error(
ref_dst_idx,
error_lut_base,
gdf_prep_full_idx(
vgetq_lane_s32::<1>(acc0),
vgetq_lane_s32::<1>(acc1),
vgetq_lane_s32::<1>(acc2),
scale,
),
),
]
}
trait GdfPrepPx: Copy {
unsafe fn load8_s16(row: &[Self], col: usize, sh: int16x8_t) -> int16x8_t;
}
impl GdfPrepPx for u8 {
#[inline(always)]
unsafe fn load8_s16(row: &[u8], col: usize, sh: int16x8_t) -> int16x8_t {
let v = &row[col..col + 8];
unsafe { vshlq_s16(vreinterpretq_s16_u16(vmovl_u8(vld1_u8(v.as_ptr()))), sh) }
}
}
impl GdfPrepPx for u16 {
#[inline(always)]
unsafe fn load8_s16(row: &[u16], col: usize, sh: int16x8_t) -> int16x8_t {
let v = &row[col..col + 8];
unsafe { vreinterpretq_s16_u16(vshlq_u16(vld1q_u16(v.as_ptr()), sh)) }
}
}
#[inline(always)]
unsafe fn gdf_class_lookup<T>(table: &[T], tidx: uint8x16_t) -> int16x8_t {
debug_assert!(table.len() >= 4 && core::mem::size_of::<T>() == 2);
unsafe {
let t = vld1_u8(table.as_ptr().cast());
vreinterpretq_s16_u8(vqtbl1q_u8(vcombine_u8(t, t), tidx))
}
}
#[inline(always)]
unsafe fn gdf_sub_idx(v: int32x4_t, scale: i32) -> int32x4_t {
unsafe {
let v = vmulq_n_s32(v, scale);
let r = vshrq_n_s32::<15>(vaddq_s32(vabsq_s32(v), vdupq_n_s32(1 << 14)));
let r = vbslq_s32(vcltzq_s32(v), vnegq_s32(r), r);
let r = vminq_s32(vmaxq_s32(r, vdupq_n_s32(-scale)), vdupq_n_s32(scale - 1));
vaddq_s32(r, vdupq_n_s32(scale))
}
}
#[inline(always)]
unsafe fn gdf_prep_8px<P: GdfPrepPx>(
rows: [&[P]; 13],
col: usize,
cls: [usize; 4],
shared: [[i32; 3]; 4],
p: &crate::rowops_dispatch::GdfPrepRow,
sh: int16x8_t,
dst: &mut [i8],
) {
unsafe {
let mut tb = [0u8; 16];
for (i, pair) in tb.as_chunks_mut::<2>().0.iter_mut().enumerate() {
let c = 2 * cls[i >> 1] as u8;
*pair = [c, c + 1];
}
let tidx = vld1q_u8(tb.as_ptr());
let mut acc = [[vdupq_n_s32(0); 2]; 3];
for (idx, acc) in acc.iter_mut().enumerate() {
let lo = [
shared[0][idx],
shared[0][idx],
shared[1][idx],
shared[1][idx],
];
let hi = [
shared[2][idx],
shared[2][idx],
shared[3][idx],
shared[3][idx],
];
*acc = [vld1q_s32(lo.as_ptr()), vld1q_s32(hi.as_ptr())];
}
let m = P::load8_s16(rows[6], col, sh);
let v_lo = vdupq_n_s16(-512);
let v_hi = vdupq_n_s16(511);
for (k, &[dy, dx]) in GDF_PREP_COORDS.iter().enumerate() {
let alpha = gdf_class_lookup(&GDF_ALPHA[p.alpha_base + k * 4..], tidx);
let neg_alpha = vnegq_s16(alpha);
let a = P::load8_s16(
rows[(6 - dy) as usize],
(col as isize - dx as isize) as usize,
sh,
);
let b = P::load8_s16(
rows[(6 + dy) as usize],
(col as isize + dx as isize) as usize,
sh,
);
let above = vminq_s16(vmaxq_s16(vsubq_s16(a, m), neg_alpha), alpha);
let below = vminq_s16(vmaxq_s16(vsubq_s16(b, m), neg_alpha), alpha);
let v = vminq_s16(vmaxq_s16(vaddq_s16(above, below), v_lo), v_hi);
for (idx, acc) in acc.iter_mut().enumerate() {
let wv = gdf_class_lookup(&GDF_WEIGHT[p.weight_base + idx * 88 + k * 4..], tidx);
acc[0] = vmlal_s16(acc[0], vget_low_s16(v), vget_low_s16(wv));
acc[1] = vmlal_high_s16(acc[1], v, wv);
}
}
let scale2 = vdupq_n_s32(2 * p.scale);
let mut full = [0i32; 8];
for half in 0..2 {
let s0 = gdf_sub_idx(acc[0][half], p.scale);
let s1 = gdf_sub_idx(acc[1][half], p.scale);
let s2 = gdf_sub_idx(acc[2][half], p.scale);
let f = vaddq_s32(vmulq_s32(vaddq_s32(vmulq_s32(s0, scale2), s1), scale2), s2);
vst1q_s32(full[half * 4..].as_mut_ptr(), f);
}
for (d, &f) in dst[..8].iter_mut().zip(full.iter()) {
*d = gdf_prep_lookup_error(p.ref_dst_idx, p.error_lut_base, f as usize);
}
}
}
#[target_feature(enable = "neon")]
pub(crate) fn gdf_prep_row_8bpc_neon(
rows: [&[u8]; 13],
o: usize,
w: usize,
grad: [&[[u16; 4]]; 2],
dst: &mut [i8],
p: &crate::rowops_dispatch::GdfPrepRow,
) {
use crate::rowops_dispatch::gdf_prep_class;
let sh = vdupq_n_s16(2);
let full = w & !7;
let mut x1 = 0usize;
while x1 < full {
let hx = x1 >> 1;
let c = [0, 1, 2, 3].map(|j| gdf_prep_class(grad, hx + j, p));
unsafe {
gdf_prep_8px(
rows,
o + x1,
c.map(|c| c.0),
c.map(|c| c.1),
p,
sh,
&mut dst[x1..],
);
}
x1 += 8;
}
if x1 < w {
crate::rowops_dispatch::gdf_prep_row_8bpc_scalar(
rows.map(|r| &r[x1..]),
o,
w - x1,
[&grad[0][x1 >> 1..], &grad[1][x1 >> 1..]],
&mut dst[x1..],
p,
);
}
}
#[allow(clippy::too_many_arguments)]
#[target_feature(enable = "neon")]
pub(crate) fn gdf_prep_row_hbd_neon(
rows: [&[u16]; 13],
o: usize,
w: usize,
grad: [&[[u16; 4]]; 2],
dst: &mut [i8],
p: &crate::rowops_dispatch::GdfPrepRow,
down_shift: u32,
up_scale: i32,
) {
use crate::rowops_dispatch::gdf_prep_class;
let sh = vdupq_n_s16(-(down_shift as i16));
let full = if up_scale == 1 { w & !7 } else { 0 };
let mut x1 = 0usize;
while x1 < full {
let hx = x1 >> 1;
let c = [0, 1, 2, 3].map(|j| gdf_prep_class(grad, hx + j, p));
unsafe {
gdf_prep_8px(
rows,
o + x1,
c.map(|c| c.0),
c.map(|c| c.1),
p,
sh,
&mut dst[x1..],
);
}
x1 += 8;
}
if x1 < w {
crate::rowops_dispatch::gdf_prep_row_hbd_scalar(
rows.map(|r| &r[x1..]),
o,
w - x1,
[&grad[0][x1 >> 1..], &grad[1][x1 >> 1..]],
&mut dst[x1..],
p,
down_shift,
up_scale,
);
}
}