use std::arch::aarch64::*;
#[target_feature(enable = "neon")]
pub(crate) fn cfl_rd_cost_neon(
m: &[f32],
s: &[f32],
factor: f32,
weight: f32,
lambda: f32,
bias: crate::group::DequantBias,
) -> f32 {
cfl_rd_cost_bounded_neon(m, s, factor, weight, lambda, bias, 0.0, f32::INFINITY)
}
const BOUND_CHECK_COEFFICIENTS: usize = 64;
#[allow(clippy::too_many_arguments)]
#[target_feature(enable = "neon")]
fn cfl_rd_cost_bounded_neon(
m: &[f32],
s: &[f32],
factor: f32,
weight: f32,
lambda: f32,
bias: crate::group::DequantBias,
rate: f32,
limit: f32,
) -> f32 {
use crate::color_correlation::{CFL_MAGNITUDE_BITS, CFL_NONZERO_BITS};
let len = m.len().min(s.len());
let factor_v = vdupq_n_f32(factor);
let weight_v = vdupq_n_f32(weight);
let zero = vdupq_n_f32(0.0);
let one = vdupq_n_f32(1.0);
let nonzero_bits = vdupq_n_f32(lambda * CFL_NONZERO_BITS);
let magnitude_bits = vdupq_n_f32(lambda * CFL_MAGNITUDE_BITS);
let mut acc = zero;
let (m_chunks, m_tail) = m[..len].as_chunks::<4>();
let (s_chunks, s_tail) = s[..len].as_chunks::<4>();
let spans = m_chunks
.chunks(BOUND_CHECK_COEFFICIENTS / 4)
.zip(s_chunks.chunks(BOUND_CHECK_COEFFICIENTS / 4));
for (m_span, s_span) in spans {
for (m, s) in m_span.iter().zip(s_span) {
let mv = unsafe { vld1q_f32(m.as_ptr()) };
let sv = unsafe { vld1q_f32(s.as_ptr()) };
let residual = vsubq_f32(sv, vmulq_f32(factor_v, mv));
let zero_cost = vmulq_f32(vmulq_f32(weight_v, residual), residual);
let level = vrndaq_f32(residual);
let coded_err = vsubq_f32(
residual,
super::ac_strategy::neon_dequantized_level_f32(level, bias),
);
let abs_level = vabsq_f32(level);
let log2_level =
super::ac_strategy::neon_log2p1_f32(vmaxq_f32(vsubq_f32(abs_level, one), zero));
let bits = vfmaq_f32(nonzero_bits, magnitude_bits, log2_level);
let coded = vaddq_f32(vmulq_f32(vmulq_f32(weight_v, coded_err), coded_err), bits);
let nonzero = vcgtq_f32(abs_level, zero);
let best = vminq_f32(zero_cost, vbslq_f32(nonzero, coded, zero_cost));
acc = vaddq_f32(acc, best);
}
if vaddvq_f32(acc) + rate > limit {
return f32::INFINITY;
}
}
let mut cost = vaddvq_f32(acc);
for (&mv, &sv) in m_tail.iter().zip(s_tail) {
let residual = sv - factor * mv;
let mut best = weight * residual * residual;
let level = residual.round();
if level != 0.0 {
let coded_err = residual - bias.level_f32(level);
let coded = weight * coded_err * coded_err
+ lambda
* (CFL_NONZERO_BITS
+ CFL_MAGNITUDE_BITS * crate::adaptive_quant::dirty_log2f(level.abs()));
best = best.min(coded);
}
cost += best;
}
cost
}
#[inline(never)]
#[target_feature(enable = "neon")]
fn cfl_rd_cost_vectors_neon<const N: usize>(
m: &[f32],
s: &[f32],
factors: &[f32; N],
weight: f32,
lambda: f32,
bias: crate::group::DequantBias,
) -> [f32; N] {
use crate::color_correlation::{CFL_MAGNITUDE_BITS, CFL_NONZERO_BITS};
assert!(N == 2 || N == 4);
let len = m.len().min(s.len());
let factor_v = if N == 2 {
let pair = unsafe { vld1_f32(factors.as_ptr()) };
vcombine_f32(pair, pair)
} else {
unsafe { vld1q_f32(factors.as_ptr()) }
};
let weight_v = vdupq_n_f32(weight);
let zero = vdupq_n_f32(0.0);
let one = vdupq_n_f32(1.0);
let nonzero_bits = vdupq_n_f32(lambda * CFL_NONZERO_BITS);
let magnitude_bits = vdupq_n_f32(lambda * CFL_MAGNITUDE_BITS);
let cost = |mv, sv, factor_v| {
let residual = vsubq_f32(sv, vmulq_f32(factor_v, mv));
let zero_cost = vmulq_f32(vmulq_f32(weight_v, residual), residual);
let level = vrndaq_f32(residual);
let coded_err = vsubq_f32(
residual,
super::ac_strategy::neon_dequantized_level_f32(level, bias),
);
let abs_level = vabsq_f32(level);
let log2_level =
super::ac_strategy::neon_log2p1_f32(vmaxq_f32(vsubq_f32(abs_level, one), zero));
let bits = vfmaq_f32(nonzero_bits, magnitude_bits, log2_level);
let coded = vaddq_f32(vmulq_f32(vmulq_f32(weight_v, coded_err), coded_err), bits);
let nonzero = vcgtq_f32(abs_level, zero);
vminq_f32(zero_cost, vbslq_f32(nonzero, coded, zero_cost))
};
let can_share_zero = weight.is_finite()
&& lambda.is_finite()
&& bias.one.is_finite()
&& bias.numerator.is_finite()
&& vmaxvq_u32(vreinterpretq_u32_f32(vabsq_f32(factor_v))) < f32::INFINITY.to_bits();
let mut a0 = zero;
let mut a1 = zero;
let mut a2 = zero;
let mut a3 = zero;
for (m, s) in m[..len]
.as_chunks::<4>()
.0
.iter()
.zip(s[..len].as_chunks::<4>().0)
{
let mv = unsafe { vld1q_f32(m.as_ptr()) };
let sv = unsafe { vld1q_f32(s.as_ptr()) };
if can_share_zero
&& vmaxvq_u32(vreinterpretq_u32_f32(vabsq_f32(mv))) == 0
&& vmaxvq_u32(vreinterpretq_u32_f32(vabsq_f32(sv))) < f32::INFINITY.to_bits()
{
let best = cost(mv, sv, vdupq_laneq_f32::<0>(factor_v));
if N == 2 {
a0 = vaddq_f32(a0, vzip1q_f32(best, best));
a1 = vaddq_f32(a1, vzip2q_f32(best, best));
} else {
a0 = vaddq_f32(a0, vdupq_laneq_f32::<0>(best));
a1 = vaddq_f32(a1, vdupq_laneq_f32::<1>(best));
a2 = vaddq_f32(a2, vdupq_laneq_f32::<2>(best));
a3 = vaddq_f32(a3, vdupq_laneq_f32::<3>(best));
}
} else {
if N == 2 {
a0 = vaddq_f32(a0, cost(vzip1q_f32(mv, mv), vzip1q_f32(sv, sv), factor_v));
a1 = vaddq_f32(a1, cost(vzip2q_f32(mv, mv), vzip2q_f32(sv, sv), factor_v));
} else {
a0 = vaddq_f32(
a0,
cost(vdupq_laneq_f32::<0>(mv), vdupq_laneq_f32::<0>(sv), factor_v),
);
a1 = vaddq_f32(
a1,
cost(vdupq_laneq_f32::<1>(mv), vdupq_laneq_f32::<1>(sv), factor_v),
);
a2 = vaddq_f32(
a2,
cost(vdupq_laneq_f32::<2>(mv), vdupq_laneq_f32::<2>(sv), factor_v),
);
a3 = vaddq_f32(
a3,
cost(vdupq_laneq_f32::<3>(mv), vdupq_laneq_f32::<3>(sv), factor_v),
);
}
}
}
let mut costs = [0.0; N];
if N == 2 {
let first = vadd_f32(vget_low_f32(a0), vget_high_f32(a0));
let second = vadd_f32(vget_low_f32(a1), vget_high_f32(a1));
unsafe {
vst1_f32(costs.as_mut_ptr(), vadd_f32(first, second));
}
} else {
let reduced = vaddq_f32(vaddq_f32(a0, a1), vaddq_f32(a2, a3));
unsafe {
vst1q_f32(costs.as_mut_ptr(), reduced);
}
}
costs
}
#[target_feature(enable = "neon")]
fn cfl_rd_cost_batch_neon<const N: usize>(
m: &[f32],
s: &[f32],
factors: &[f32; N],
weight: f32,
lambda: f32,
bias: crate::group::DequantBias,
) -> [f32; N] {
use crate::color_correlation::{CFL_MAGNITUDE_BITS, CFL_NONZERO_BITS};
let mut costs = cfl_rd_cost_vectors_neon(m, s, factors, weight, lambda, bias);
let len = m.len().min(s.len());
for (&mv, &sv) in m[..len]
.as_chunks::<4>()
.1
.iter()
.zip(s[..len].as_chunks::<4>().1)
{
for candidate in 0..N {
let residual = sv - factors[candidate] * mv;
let mut best = weight * residual * residual;
let level = residual.round();
if level != 0.0 {
let coded_err = residual - bias.level_f32(level);
let coded = weight * coded_err * coded_err
+ lambda
* (CFL_NONZERO_BITS
+ CFL_MAGNITUDE_BITS * crate::adaptive_quant::dirty_log2f(level.abs()));
best = best.min(coded);
}
costs[candidate] += best;
}
}
costs
}
#[target_feature(enable = "neon")]
fn cfl_rd_costs_neon_impl<const N: usize>(
m: &[f32],
s: &[f32],
factors: &[f32],
weight: f32,
lambda: f32,
bias: crate::group::DequantBias,
costs: &mut [f32],
) {
assert_eq!(factors.len(), costs.len());
let (batches, rest) = factors.as_chunks::<N>();
let (outputs, rest_costs) = costs.as_chunks_mut::<N>();
for (batch, output) in batches.iter().zip(outputs) {
*output = cfl_rd_cost_batch_neon(m, s, batch, weight, lambda, bias);
}
for (&factor, output) in rest.iter().zip(rest_costs) {
*output = cfl_rd_cost_neon(m, s, factor, weight, lambda, bias);
}
}
#[allow(clippy::too_many_arguments)]
#[target_feature(enable = "neon")]
fn cfl_rd_costs_neon_adaptive<const N: usize>(
m: &[f32],
s: &[f32],
factors: &[f32],
weight: f32,
lambda: f32,
bias: crate::group::DequantBias,
costs: &mut [f32],
bound: Option<(&[f32], usize)>,
) {
assert_eq!(factors.len(), costs.len());
let len = m.len().min(s.len());
let use_batch = factors.len() >= N && len >= 64 && {
[0, (len / 8) * 4, (len / 4 - 1) * 4].iter().any(|&i| {
let value = unsafe { vld1q_f32(m.as_ptr().add(i)) };
vmaxvq_u32(vreinterpretq_u32_f32(vabsq_f32(value))) == 0
})
};
if use_batch {
cfl_rd_costs_neon_impl::<N>(m, s, factors, weight, lambda, bias, costs);
} else if let Some((rates, first)) = bound
&& weight >= 0.0
&& lambda >= 0.0
{
assert_eq!(rates.len(), costs.len());
let mut limit = f32::INFINITY;
let order = std::iter::once(first).chain((0..factors.len()).filter(|&i| i != first));
for i in order {
let cost =
cfl_rd_cost_bounded_neon(m, s, factors[i], weight, lambda, bias, rates[i], limit);
costs[i] = cost;
let total = cost + rates[i];
if total < limit {
limit = total;
}
}
} else {
for (&factor, cost) in factors.iter().zip(costs) {
*cost = cfl_rd_cost_neon(m, s, factor, weight, lambda, bias);
}
}
}
#[allow(clippy::too_many_arguments)]
#[target_feature(enable = "neon")]
pub(crate) fn cfl_rd_costs_neon(
m: &[f32],
s: &[f32],
factors: &[f32],
rates: &[f32],
first: usize,
weight: f32,
lambda: f32,
bias: crate::group::DequantBias,
costs: &mut [f32],
) {
cfl_rd_costs_neon_adaptive::<4>(
m,
s,
factors,
weight,
lambda,
bias,
costs,
Some((rates, first)),
);
}
#[target_feature(enable = "neon")]
pub(crate) fn cfl_regression_neon(
y: &[f32; 64],
x: &[f32; 64],
b: &[f32; 64],
qm_x: &[f32; 64],
qm_b: &[f32; 64],
) -> [f32; 4] {
let inv_color = vdupq_n_f32(1.0 / 84.0);
let mut ca_x = vdupq_n_f32(0.0);
let mut cb_x = vdupq_n_f32(0.0);
let mut ca_b = vdupq_n_f32(0.0);
let mut cb_b = vdupq_n_f32(0.0);
let y_chunks = y.as_chunks::<4>().0;
let x_chunks = x.as_chunks::<4>().0;
let b_chunks = b.as_chunks::<4>().0;
let qm_x_chunks = qm_x.as_chunks::<4>().0;
let qm_b_chunks = qm_b.as_chunks::<4>().0;
for ((((y, x), b), qm_x), qm_b) in y_chunks
.iter()
.zip(x_chunks)
.zip(b_chunks)
.zip(qm_x_chunks)
.zip(qm_b_chunks)
{
let yv = unsafe { vld1q_f32(y.as_ptr()) };
let xv = unsafe { vld1q_f32(x.as_ptr()) };
let bv = unsafe { vld1q_f32(b.as_ptr()) };
let qx = unsafe { vld1q_f32(qm_x.as_ptr()) };
let qb = unsafe { vld1q_f32(qm_b.as_ptr()) };
let mx = vmulq_f32(yv, qx);
let sx = vmulq_f32(xv, qx);
let ax = vmulq_f32(inv_color, mx);
ca_x = vfmaq_f32(ca_x, ax, ax);
cb_x = vfmsq_f32(cb_x, ax, sx);
let mb = vmulq_f32(yv, qb);
let sb = vmulq_f32(bv, qb);
let ab = vmulq_f32(inv_color, mb);
let residual_b = vsubq_f32(mb, sb);
ca_b = vfmaq_f32(ca_b, ab, ab);
cb_b = vfmaq_f32(cb_b, ab, residual_b);
}
[
vaddvq_f32(ca_x),
vaddvq_f32(cb_x),
vaddvq_f32(ca_b),
vaddvq_f32(cb_b),
]
}
#[allow(clippy::too_many_arguments)]
#[target_feature(enable = "neon")]
pub(crate) fn cfl_rdo_refill_neon(
m_x: &mut [f32],
s_x: &mut [f32],
m_b: &mut [f32],
s_b: &mut [f32],
y: &[f32],
x: &[f32],
b: &[f32],
qm_x: &[f32],
qm_b: &[f32],
params: crate::color_correlation::CflRdoRefillParams,
) {
let len = m_x.len();
assert_eq!(s_x.len(), len);
assert_eq!(m_b.len(), len);
assert_eq!(s_b.len(), len);
assert_eq!(y.len(), len);
assert_eq!(x.len(), len);
assert_eq!(b.len(), len);
assert_eq!(qm_x.len(), len);
assert_eq!(qm_b.len(), len);
let q_block = vdupq_n_f32(params.q_block);
let x_mul = vdupq_n_f32(params.x_mul);
let b_mul = vdupq_n_f32(params.b_mul);
let shift_x = vdupq_n_f32(params.shift_x);
let shift_b = vdupq_n_f32(params.shift_b);
let mut i = 0;
while i + 4 <= len {
let yv = unsafe { vld1q_f32(y.as_ptr().add(i)) };
let xv = unsafe { vld1q_f32(x.as_ptr().add(i)) };
let bv = unsafe { vld1q_f32(b.as_ptr().add(i)) };
let qx = unsafe { vld1q_f32(qm_x.as_ptr().add(i)) };
let qb = unsafe { vld1q_f32(qm_b.as_ptr().add(i)) };
let qx = vmulq_f32(vmulq_f32(qx, q_block), x_mul);
let qb = vmulq_f32(vmulq_f32(qb, q_block), b_mul);
let sx = vfmaq_f32(xv, shift_x, yv);
let sb = vfmaq_f32(bv, shift_b, yv);
unsafe {
vst1q_f32(m_x.as_mut_ptr().add(i), vmulq_f32(yv, qx));
vst1q_f32(s_x.as_mut_ptr().add(i), vmulq_f32(sx, qx));
vst1q_f32(m_b.as_mut_ptr().add(i), vmulq_f32(yv, qb));
vst1q_f32(s_b.as_mut_ptr().add(i), vmulq_f32(sb, qb));
}
i += 4;
}
let src = y[i..]
.iter()
.zip(&x[i..])
.zip(&b[i..])
.zip(qm_x[i..].iter().zip(&qm_b[i..]));
let dst = m_x[i..]
.iter_mut()
.zip(&mut s_x[i..])
.zip(m_b[i..].iter_mut().zip(&mut s_b[i..]));
for ((((&y, &x), &b), (&qx, &qb)), ((m_x, s_x), (m_b, s_b))) in src.zip(dst) {
let qx = qx * params.q_block * params.x_mul;
let qb = qb * params.q_block * params.b_mul;
*m_x = y * qx;
*s_x = params.shift_x.mul_add(y, x) * qx;
*m_b = y * qb;
*s_b = params.shift_b.mul_add(y, b) * qb;
}
}
#[target_feature(enable = "neon")]
pub(crate) fn cfl_rdo_block_neon(
m_x: &mut [f32; 63],
s_x: &mut [f32; 63],
m_b: &mut [f32; 63],
s_b: &mut [f32; 63],
block_y: &[f32; 64],
block_x: &[f32; 64],
block_b: &[f32; 64],
qm_x: &[f32; 64],
qm_b: &[f32; 64],
q_block: f32,
) {
let q_block_v = vdupq_n_f32(q_block);
for i in (0..60).step_by(4) {
let coeff = i + 1;
let y = unsafe { vld1q_f32(block_y.as_ptr().add(coeff)) };
let x = unsafe { vld1q_f32(block_x.as_ptr().add(coeff)) };
let b = unsafe { vld1q_f32(block_b.as_ptr().add(coeff)) };
let qx = unsafe { vld1q_f32(qm_x.as_ptr().add(coeff)) };
let qb = unsafe { vld1q_f32(qm_b.as_ptr().add(coeff)) };
let qx = vmulq_f32(qx, q_block_v);
let qb = vmulq_f32(qb, q_block_v);
unsafe {
vst1q_f32(m_x.as_mut_ptr().add(i), vmulq_f32(y, qx));
vst1q_f32(s_x.as_mut_ptr().add(i), vmulq_f32(x, qx));
vst1q_f32(m_b.as_mut_ptr().add(i), vmulq_f32(y, qb));
vst1q_f32(s_b.as_mut_ptr().add(i), vmulq_f32(b, qb));
}
}
let i = 59;
let coeff = i + 1;
let y = unsafe { vld1q_f32(block_y.as_ptr().add(coeff)) };
let x = unsafe { vld1q_f32(block_x.as_ptr().add(coeff)) };
let b = unsafe { vld1q_f32(block_b.as_ptr().add(coeff)) };
let qx = unsafe { vld1q_f32(qm_x.as_ptr().add(coeff)) };
let qb = unsafe { vld1q_f32(qm_b.as_ptr().add(coeff)) };
let qx = vmulq_f32(qx, q_block_v);
let qb = vmulq_f32(qb, q_block_v);
unsafe {
vst1q_f32(m_x.as_mut_ptr().add(i), vmulq_f32(y, qx));
vst1q_f32(s_x.as_mut_ptr().add(i), vmulq_f32(x, qx));
vst1q_f32(m_b.as_mut_ptr().add(i), vmulq_f32(y, qb));
vst1q_f32(s_b.as_mut_ptr().add(i), vmulq_f32(b, qb));
}
}
#[target_feature(enable = "neon")]
pub(crate) fn cfl_rdo_stats_neon(m: &[f32], s: &[f32]) -> [f32; 4] {
let len = m.len().min(s.len());
let (m_chunks, m_tail) = m[..len].as_chunks::<4>();
let (s_chunks, s_tail) = s[..len].as_chunks::<4>();
let mut dot_ms = vdupq_n_f32(0.0);
let mut dot_mm = vdupq_n_f32(0.0);
let mut dot_ss = vdupq_n_f32(0.0);
let mut sum_abs_s = vdupq_n_f32(0.0);
for (m, s) in m_chunks.iter().zip(s_chunks) {
let mv = unsafe { vld1q_f32(m.as_ptr()) };
let sv = unsafe { vld1q_f32(s.as_ptr()) };
dot_ms = vfmaq_f32(dot_ms, mv, sv);
dot_mm = vfmaq_f32(dot_mm, mv, mv);
dot_ss = vfmaq_f32(dot_ss, sv, sv);
sum_abs_s = vaddq_f32(sum_abs_s, vabsq_f32(sv));
}
let mut stats = [
vaddvq_f32(dot_ms),
vaddvq_f32(dot_mm),
vaddvq_f32(dot_ss),
vaddvq_f32(sum_abs_s),
];
for (&mv, &sv) in m_tail.iter().zip(s_tail) {
stats[0] = mv.mul_add(sv, stats[0]);
stats[1] = mv.mul_add(mv, stats[1]);
stats[2] = sv.mul_add(sv, stats[2]);
stats[3] += sv.abs();
}
stats
}
#[target_feature(enable = "neon")]
pub(crate) fn apply_cfl_neon(x: &mut [f32], y: &[f32], b: &mut [f32], cmap_factor: [f32; 3]) {
assert_eq!(x.len(), y.len());
assert_eq!(x.len(), b.len());
let cx = vdupq_n_f32(-cmap_factor[0]);
let cb = vdupq_n_f32(-cmap_factor[2]);
let (x_chunks, x_tail) = x.as_chunks_mut::<4>();
let (y_chunks, y_tail) = y.as_chunks::<4>();
let (b_chunks, b_tail) = b.as_chunks_mut::<4>();
let (x_groups, x_chunks_tail) = x_chunks.as_chunks_mut::<4>();
let (y_groups, y_chunks_tail) = y_chunks.as_chunks::<4>();
let (b_groups, b_chunks_tail) = b_chunks.as_chunks_mut::<4>();
for ((x, y), b) in x_groups.iter_mut().zip(y_groups).zip(b_groups) {
let y0 = unsafe { vld1q_f32(y[0].as_ptr()) };
let y1 = unsafe { vld1q_f32(y[1].as_ptr()) };
let y2 = unsafe { vld1q_f32(y[2].as_ptr()) };
let y3 = unsafe { vld1q_f32(y[3].as_ptr()) };
let x0 = unsafe { vld1q_f32(x[0].as_ptr()) };
let x1 = unsafe { vld1q_f32(x[1].as_ptr()) };
let x2 = unsafe { vld1q_f32(x[2].as_ptr()) };
let x3 = unsafe { vld1q_f32(x[3].as_ptr()) };
let b0 = unsafe { vld1q_f32(b[0].as_ptr()) };
let b1 = unsafe { vld1q_f32(b[1].as_ptr()) };
let b2 = unsafe { vld1q_f32(b[2].as_ptr()) };
let b3 = unsafe { vld1q_f32(b[3].as_ptr()) };
let x0 = vfmaq_f32(x0, cx, y0);
let x1 = vfmaq_f32(x1, cx, y1);
let x2 = vfmaq_f32(x2, cx, y2);
let x3 = vfmaq_f32(x3, cx, y3);
let b0 = vfmaq_f32(b0, cb, y0);
let b1 = vfmaq_f32(b1, cb, y1);
let b2 = vfmaq_f32(b2, cb, y2);
let b3 = vfmaq_f32(b3, cb, y3);
unsafe {
vst1q_f32(x[0].as_mut_ptr(), x0);
vst1q_f32(x[1].as_mut_ptr(), x1);
vst1q_f32(x[2].as_mut_ptr(), x2);
vst1q_f32(x[3].as_mut_ptr(), x3);
vst1q_f32(b[0].as_mut_ptr(), b0);
vst1q_f32(b[1].as_mut_ptr(), b1);
vst1q_f32(b[2].as_mut_ptr(), b2);
vst1q_f32(b[3].as_mut_ptr(), b3);
}
}
for ((x, y), b) in x_chunks_tail
.iter_mut()
.zip(y_chunks_tail)
.zip(b_chunks_tail)
{
let yv = unsafe { vld1q_f32(y.as_ptr()) };
let xv = unsafe { vld1q_f32(x.as_ptr()) };
let bv = unsafe { vld1q_f32(b.as_ptr()) };
unsafe {
vst1q_f32(x.as_mut_ptr(), vfmaq_f32(xv, cx, yv));
vst1q_f32(b.as_mut_ptr(), vfmaq_f32(bv, cb, yv));
}
}
for ((x, &y), b) in x_tail.iter_mut().zip(y_tail).zip(b_tail) {
*x = (-cmap_factor[0]).mul_add(y, *x);
*b = (-cmap_factor[2]).mul_add(y, *b);
}
}
#[cfg(test)]
mod tests {
#[test]
fn apply_cfl_neon_matches_scalar() {
if !std::arch::is_aarch64_feature_detected!("neon") {
return;
}
for size in [0, 1, 3, 4, 7, 64, 1024, 4096] {
let y: Vec<f32> = (0..size)
.map(|i| ((i * 37 % 101) as f32 - 50.0) * 0.03125)
.collect();
let mut x: Vec<f32> = (0..size).map(|i| i as f32 * 0.015625 - 2.0).collect();
let mut b: Vec<f32> = (0..size).map(|i| 3.0 - i as f32 * 0.0078125).collect();
let mut want_x = x.clone();
let mut want_b = b.clone();
let factors: [f32; 3] = [0.3125, 0.0, -0.1875];
for i in 0..size {
want_x[i] = (-factors[0]).mul_add(y[i], want_x[i]);
want_b[i] = (-factors[2]).mul_add(y[i], want_b[i]);
}
unsafe { super::apply_cfl_neon(&mut x, &y, &mut b, factors) };
assert_eq!(x, want_x, "X mismatch at size {size}");
assert_eq!(b, want_b, "B mismatch at size {size}");
}
}
}
#[cfg(test)]
mod regression_tests {
use super::cfl_regression_neon;
use crate::color_correlation::cfl_regression_scalar;
fn rng(state: &mut u64) -> f32 {
*state = state
.wrapping_mul(6_364_136_223_846_793_005)
.wrapping_add(1_442_695_040_888_963_407);
((*state >> 33) as f32 / (1u64 << 31) as f32) - 1.0
}
#[test]
fn cfl_regression_neon_matches_scalar() {
let mut state = 0x1234_abcd_u64;
for case in 0..12 {
let mut y = [0.0f32; 64];
let mut x = [0.0f32; 64];
let mut b = [0.0f32; 64];
let mut qm_x = [0.0f32; 64];
let mut qm_b = [0.0f32; 64];
for i in 0..64 {
y[i] = rng(&mut state) * 30.0;
x[i] = if case % 3 == 0 {
0.3 * y[i]
} else {
rng(&mut state) * 10.0
};
b[i] = rng(&mut state) * 20.0;
qm_x[i] = 0.1 + rng(&mut state).abs() * 3.0;
qm_b[i] = 0.1 + rng(&mut state).abs() * 3.0;
}
let want = cfl_regression_scalar(&y, &x, &b, &qm_x, &qm_b);
let got = unsafe { cfl_regression_neon(&y, &x, &b, &qm_x, &qm_b) };
for k in 0..4 {
let scale = want[k].abs().max(1.0);
assert!(
(want[k] - got[k]).abs() <= 1e-4 * scale,
"case {case} sum {k}: neon {} vs scalar {}",
got[k],
want[k]
);
}
}
}
}