use crate::ssimulacra2::MapContribution;
use std::arch::aarch64::*;
#[target_feature(enable = "neon")]
pub(crate) unsafe fn ss2_contribution_neon(
map: &[f32],
p: MapContribution,
acc: &mut [f32],
) -> usize {
debug_assert_eq!(map.len(), acc.len());
if p.norm_weight == 0.0 {
let done = unsafe { contribution::<4, false>(map, p, acc) };
done + unsafe { contribution::<1, false>(&map[done..], p, &mut acc[done..]) }
} else {
let done = unsafe { contribution::<4, true>(map, p, acc) };
done + unsafe { contribution::<1, true>(&map[done..], p, &mut acc[done..]) }
}
}
#[inline]
#[target_feature(enable = "neon")]
unsafe fn contribution<const N: usize, const FOURTH: bool>(
map: &[f32],
p: MapContribution,
acc: &mut [f32],
) -> usize {
let done = acc.len() / (4 * N) * (4 * N);
for start in (0..done).step_by(4 * N) {
for lane in 0..N {
let i = start + 4 * lane;
unsafe {
let v = vld1q_f32(map.as_ptr().add(i));
let old = vld1q_f32(acc.as_ptr().add(i));
let out = if FOURTH {
let normalized = vmulq_n_f32(v, p.inv_norm);
let square = vmulq_f32(normalized, normalized);
let weighted = vmulq_n_f32(square, p.norm_weight);
let first = vmulq_n_f32(v, p.linear);
vaddq_f32(old, vfmaq_f32(first, weighted, square))
} else {
vfmaq_n_f32(old, v, p.linear)
};
vst1q_f32(acc.as_mut_ptr().add(i), out);
}
}
}
done
}
#[target_feature(enable = "neon")]
pub(crate) unsafe fn ss2_expand_row_neon<const STEP: usize>(
src: &[f32],
dst: &mut [f32],
share: f32,
) -> usize {
let span = if STEP == 1 {
4
} else if STEP == 2 {
8
} else {
STEP
};
let done = dst.len() / span * span;
for i in (0..done).step_by(span) {
unsafe {
if STEP == 1 {
let value = vld1q_f32(src.as_ptr().add(i));
let old = vld1q_f32(dst.as_ptr().add(i));
vst1q_f32(dst.as_mut_ptr().add(i), vfmaq_n_f32(old, value, share));
} else if STEP == 2 {
let v = vld1q_f32(src.as_ptr().add(i / 2));
let a = vzip1q_f32(v, v);
let b = vzip2q_f32(v, v);
let old_a = vld1q_f32(dst.as_ptr().add(i));
let old_b = vld1q_f32(dst.as_ptr().add(i + 4));
vst1q_f32(dst.as_mut_ptr().add(i), vfmaq_n_f32(old_a, a, share));
vst1q_f32(dst.as_mut_ptr().add(i + 4), vfmaq_n_f32(old_b, b, share));
} else {
let value = vdupq_n_f32(*src.get_unchecked(i / STEP));
for lane in 0..STEP / 4 {
let offset = i + 4 * lane;
let old = vld1q_f32(dst.as_ptr().add(offset));
vst1q_f32(dst.as_mut_ptr().add(offset), vfmaq_n_f32(old, value, share));
}
}
}
}
done
}