use super::ac_strategy::neon_log2p1_f32;
use crate::image::{Image3F, Plane};
use crate::inflated_cost::{
LumaPhase, RateLog2Lut, ReconCost, ReconDistInput, ReconErrorKernels, ReconKernels,
recon_chroma_phase_with_kernels, recon_dist_and_rate_with_kernels,
recon_luma_phase_with_kernels, validate_ssim_inputs,
};
use std::arch::aarch64::*;
#[inline]
#[target_feature(enable = "neon")]
fn clamp01_f32x4(value: float32x4_t) -> float32x4_t {
vminq_f32(vmaxq_f32(value, vdupq_n_f32(0.0)), vdupq_n_f32(1.0))
}
#[inline]
#[target_feature(enable = "neon")]
fn xyb_to_oklab_f32x4(
matrix: &crate::xyb::XybMatrix,
x: float32x4_t,
y: float32x4_t,
b: float32x4_t,
) -> (float32x4_t, float32x4_t, float32x4_t) {
let cube = |value: float32x4_t| {
let value = vsubq_f32(value, vdupq_n_f32(crate::xyb::NEG_BIAS_CBRT));
vsubq_f32(
vmulq_f32(vmulq_f32(value, value), value),
vdupq_n_f32(crate::xyb::OPSIN_BIAS),
)
};
let mixed0 = cube(vaddq_f32(y, x));
let mixed1 = cube(vsubq_f32(y, x));
let mixed2 = cube(b);
let inverse_row = |offset: usize| {
let mut value = vmulq_n_f32(mixed2, matrix.inv[offset + 2]);
value = vfmaq_n_f32(value, mixed1, matrix.inv[offset + 1]);
vfmaq_n_f32(value, mixed0, matrix.inv[offset])
};
let r = inverse_row(0);
let g = inverse_row(3);
let b = inverse_row(6);
let zero = vdupq_n_f32(0.0);
let rgb_row = |cr: f32, cg: f32, cb: f32| {
let mut value = vmulq_n_f32(b, cb);
value = vfmaq_n_f32(value, g, cg);
vmaxq_f32(vfmaq_n_f32(value, r, cr), zero)
};
let l = rgb_row(0.412_221_46, 0.536_332_55, 0.051_445_995);
let m = rgb_row(0.211_903_5, 0.680_699_5, 0.107_396_96);
let s = rgb_row(0.088_302_46, 0.281_718_85, 0.629_978_7);
let (l, m, s) = super::xyb::vcbrtq_fast3_positive_f32(l, m, s);
let lab_row = |cl: f32, cm: f32, cs: f32| {
let mut value = vmulq_n_f32(s, cs);
value = vfmaq_n_f32(value, m, cm);
vfmaq_n_f32(value, l, cl)
};
(
lab_row(0.210_454_26, 0.793_617_8, -0.004_072_047),
lab_row(1.977_998_5, -2.428_592_2, 0.450_593_7),
lab_row(0.025_904_037, 0.782_771_77, -0.808_675_77),
)
}
#[inline]
#[target_feature(enable = "neon")]
fn shifted_right_f32x4(value: float32x4_t) -> float32x4_t {
let shifted = vextq_f32::<1>(value, value);
vsetq_lane_f32::<3>(vgetq_lane_f32::<3>(value), shifted)
}
#[target_feature(enable = "neon")]
pub(crate) fn rgb_hue_chroma_edge_loss_neon(
opsin: &Image3F,
px: usize,
py: usize,
width: usize,
height: usize,
spatial_error: [&[f32]; 3],
matrix: &crate::xyb::XybMatrix,
) -> f32 {
if width == 0 || height == 0 {
return 0.0;
}
let Some(end_x) = px.checked_add(width) else {
return crate::inflated_cost::rgb_hue_chroma_edge_loss_scalar(
opsin,
px,
py,
width,
height,
spatial_error,
matrix,
);
};
let Some(end_y) = py.checked_add(height) else {
return crate::inflated_cost::rgb_hue_chroma_edge_loss_scalar(
opsin,
px,
py,
width,
height,
spatial_error,
matrix,
);
};
if end_x > opsin.xsize() || end_y > opsin.ysize() || !width.is_multiple_of(4) {
return crate::inflated_cost::rgb_hue_chroma_edge_loss_scalar(
opsin,
px,
py,
width,
height,
spatial_error,
matrix,
);
}
let zero = vdupq_n_f32(0.0);
let mut sum = zero;
for y in 0..height {
let rows = [
opsin.plane_row(0, py + y),
opsin.plane_row(1, py + y),
opsin.plane_row(2, py + y),
];
let below = if y + 1 < height {
[
opsin.plane_row(0, py + y + 1),
opsin.plane_row(1, py + y + 1),
opsin.plane_row(2, py + y + 1),
]
} else {
rows
};
let error_offset = y * width;
for x in (0..width).step_by(4) {
let p = px + x;
let sx = unsafe { vld1q_f32(rows[0].as_ptr().add(p)) };
let sy = unsafe { vld1q_f32(rows[1].as_ptr().add(p)) };
let sb = unsafe { vld1q_f32(rows[2].as_ptr().add(p)) };
let (rx, ry, rb) = if x + 4 < width {
(
unsafe { vld1q_f32(rows[0].as_ptr().add(p + 1)) },
unsafe { vld1q_f32(rows[1].as_ptr().add(p + 1)) },
unsafe { vld1q_f32(rows[2].as_ptr().add(p + 1)) },
)
} else {
(
shifted_right_f32x4(sx),
shifted_right_f32x4(sy),
shifted_right_f32x4(sb),
)
};
let bx = unsafe { vld1q_f32(below[0].as_ptr().add(p)) };
let by = unsafe { vld1q_f32(below[1].as_ptr().add(p)) };
let bb = unsafe { vld1q_f32(below[2].as_ptr().add(p)) };
let cb = vsubq_f32(sb, sy);
let horizontal = vaddq_f32(
vabsq_f32(vsubq_f32(rx, sx)),
vabsq_f32(vsubq_f32(vsubq_f32(rb, ry), cb)),
);
let vertical = vaddq_f32(
vabsq_f32(vsubq_f32(bx, sx)),
vabsq_f32(vsubq_f32(vsubq_f32(bb, by), cb)),
);
let edge_risk = clamp01_f32x4(vmulq_n_f32(
vsubq_f32(vmaxq_f32(horizontal, vertical), vdupq_n_f32(0.006)),
1.0 / 0.030,
));
let e = error_offset + x;
let ex = unsafe { vld1q_f32(spatial_error[0].as_ptr().add(e)) };
let ey = unsafe { vld1q_f32(spatial_error[1].as_ptr().add(e)) };
let eb = unsafe { vld1q_f32(spatial_error[2].as_ptr().add(e)) };
let (source_l, source_a, source_b) = xyb_to_oklab_f32x4(matrix, sx, sy, sb);
let (recon_l, recon_a, recon_b) = xyb_to_oklab_f32x4(
matrix,
vsubq_f32(sx, ex),
vsubq_f32(sy, ey),
vsubq_f32(sb, eb),
);
let _ = recon_l;
let source_chroma =
vsqrtq_f32(vfmaq_f32(vmulq_f32(source_b, source_b), source_a, source_a));
let radial_error = vdivq_f32(
vfmaq_f32(
vmulq_f32(source_b, vsubq_f32(source_b, recon_b)),
source_a,
vsubq_f32(source_a, recon_a),
),
vmaxq_f32(source_chroma, vdupq_n_f32(1e-4)),
);
let brightness_risk = clamp01_f32x4(vmulq_n_f32(
vsubq_f32(
vsubq_f32(source_l, vdupq_n_f32(0.35)),
vminq_f32(source_b, zero),
),
1.0 / 0.40,
));
let chroma_risk = clamp01_f32x4(vmulq_n_f32(
vsubq_f32(source_chroma, vdupq_n_f32(0.03)),
1.0 / 0.12,
));
let risk = vmulq_f32(vmulq_f32(edge_risk, brightness_risk), chroma_risk);
let desaturation = vmaxq_f32(radial_error, zero);
let perpendicular = vdivq_f32(
vsubq_f32(vmulq_f32(source_a, recon_b), vmulq_f32(source_b, recon_a)),
vaddq_f32(source_chroma, vdupq_n_f32(1e-4)),
);
let penalty = vfmaq_n_f32(
vmulq_f32(desaturation, desaturation),
vmulq_f32(perpendicular, perpendicular),
0.75,
);
sum = vfmaq_f32(sum, risk, penalty);
}
}
vaddvq_f32(sum)
}
#[inline]
#[target_feature(enable = "neon")]
fn combine_error_x4(spatial: &[f32; 4], luma: &[f32; 4], factor: f32, combined: &mut [f32; 4]) {
let spatial = unsafe { vld1q_f32(spatial.as_ptr()) };
let luma = unsafe { vld1q_f32(luma.as_ptr()) };
let value = vfmaq_n_f32(spatial, luma, factor);
unsafe { vst1q_f32(combined.as_mut_ptr(), value) };
}
#[target_feature(enable = "neon")]
pub(crate) fn combine_error_neon(spatial: &[f32], luma: &[f32], factor: f32, combined: &mut [f32]) {
debug_assert_eq!(spatial.len(), luma.len());
debug_assert_eq!(spatial.len(), combined.len());
let (spatial16, spatial_tail) = spatial.as_chunks::<16>();
let (luma16, luma_tail) = luma.as_chunks::<16>();
let (combined16, combined_tail) = combined.as_chunks_mut::<16>();
for ((spatial, luma), combined) in spatial16.iter().zip(luma16).zip(combined16) {
let [s0, s1, s2, s3] = spatial.as_chunks::<4>().0 else {
unreachable!()
};
let [l0, l1, l2, l3] = luma.as_chunks::<4>().0 else {
unreachable!()
};
let [c0, c1, c2, c3] = combined.as_chunks_mut::<4>().0 else {
unreachable!()
};
combine_error_x4(s0, l0, factor, c0);
combine_error_x4(s1, l1, factor, c1);
combine_error_x4(s2, l2, factor, c2);
combine_error_x4(s3, l3, factor, c3);
}
let (spatial4, spatial_remainder) = spatial_tail.as_chunks::<4>();
let (luma4, luma_remainder) = luma_tail.as_chunks::<4>();
let (combined4, combined_remainder) = combined_tail.as_chunks_mut::<4>();
for ((spatial, luma), combined) in spatial4.iter().zip(luma4).zip(combined4) {
combine_error_x4(spatial, luma, factor, combined);
}
if !spatial_remainder.is_empty() {
let mut spatial = [0.0; 4];
let mut luma = [0.0; 4];
let mut combined = [0.0; 4];
spatial[..spatial_remainder.len()].copy_from_slice(spatial_remainder);
luma[..luma_remainder.len()].copy_from_slice(luma_remainder);
combine_error_x4(&spatial, &luma, factor, &mut combined);
combined_remainder.copy_from_slice(&combined[..combined_remainder.len()]);
}
}
#[target_feature(enable = "neon")]
pub(crate) fn recon_dist_and_rate_neon(
scratch: &mut [[f32; 4096]; 8],
input: &ReconDistInput<'_>,
error: &ReconErrorKernels,
) -> ReconCost {
recon_dist_and_rate_neon_impl::<true>(scratch, input, error)
}
#[target_feature(enable = "neon")]
pub(crate) fn recon_luma_phase_neon(
scratch: &mut [[f32; 4096]; 8],
input: &ReconDistInput<'_>,
error: &ReconErrorKernels,
) -> LumaPhase {
recon_luma_phase_with_kernels(scratch, input, &neon_kernels::<true>(input, error))
}
#[target_feature(enable = "neon")]
pub(crate) fn recon_chroma_phase_neon(
scratch: &mut [[f32; 4096]; 8],
input: &ReconDistInput<'_>,
error: &ReconErrorKernels,
luma: LumaPhase,
) -> ReconCost {
recon_chroma_phase_with_kernels(scratch, input, &neon_kernels::<true>(input, error), luma)
}
fn neon_kernels<'a, const BIASED: bool>(
input: &ReconDistInput<'_>,
error: &'a ReconErrorKernels,
) -> ReconKernels<'a> {
ReconKernels {
quantize: if input.quantization.compute_rate {
recon_quantize_neon::<BIASED, true>
} else {
recon_quantize_neon::<BIASED, false>
},
ssim: ssim_deficit_neon,
prepare: prepare_reconstruction_neon,
error,
}
}
#[target_feature(enable = "neon")]
fn recon_dist_and_rate_neon_impl<const BIASED: bool>(
scratch: &mut [[f32; 4096]; 8],
input: &ReconDistInput<'_>,
error: &ReconErrorKernels,
) -> ReconCost {
recon_dist_and_rate_with_kernels(
scratch,
input,
&ReconKernels {
quantize: if input.quantization.compute_rate {
recon_quantize_neon::<BIASED, true>
} else {
recon_quantize_neon::<BIASED, false>
},
ssim: ssim_deficit_neon,
prepare: prepare_reconstruction_neon,
error,
},
)
}
#[allow(clippy::too_many_arguments)]
#[target_feature(enable = "neon")]
fn recon_quantize_neon<const BIASED: bool, const COMPUTE_RATE: bool>(
coeff: &[f32],
inv: &[f32],
matrix: &[f32],
quant_scale: f32,
bias: crate::group::DequantBias,
thresholds: &[f32; 4],
width: usize,
height: usize,
half: usize,
cx: usize,
cy: usize,
coeff_error: &mut [f32],
_rate_log2_lut: &RateLog2Lut,
scan_pos: &[u32],
) -> f32 {
let n = width
.checked_mul(height)
.expect("coefficient size overflow");
assert!(width.is_multiple_of(4) && half.is_multiple_of(4));
assert!(coeff.len() >= n && inv.len() >= n && matrix.len() >= n && coeff_error.len() >= n);
assert!(!COMPUTE_RATE || scan_pos.len() >= n);
let scale = vdupq_n_f32(quant_scale);
let inv_scale = vdupq_n_f32(1.0 / quant_scale);
let zero = vdupq_n_f32(0.0);
let all = vdupq_n_u32(u32::MAX);
let lanes = unsafe { vld1q_u32([0, 1, 2, 3].as_ptr()) };
let mut nonzero_acc = vdupq_n_u32(0);
let mut magnitude_bits = vdupq_n_f32(0.0);
let mut scan_acc = vdupq_n_u32(0);
for (y, (((coeff_row, inv_row), matrix_row), error_row)) in coeff
.chunks_exact(width)
.zip(inv.chunks_exact(width))
.zip(matrix.chunks_exact(width))
.zip(coeff_error.chunks_exact_mut(width))
.take(height)
.enumerate()
{
let yfix = if y >= height / 2 { 2 } else { 0 };
for (chunk_x, (((coeff4, inv4), matrix4), error4)) in coeff_row
.as_chunks::<4>()
.0
.iter()
.zip(inv_row.as_chunks::<4>().0.iter())
.zip(matrix_row.as_chunks::<4>().0.iter())
.zip(error_row.as_chunks_mut::<4>().0.iter_mut())
.enumerate()
{
let x = chunk_x * 4;
let threshold = vdupq_n_f32(if x >= half {
thresholds[yfix + 1]
} else {
thresholds[yfix]
});
let coeff_v = unsafe { vld1q_f32(coeff4.as_ptr()) };
let inv_v = unsafe { vld1q_f32(inv4.as_ptr()) };
let denominator = vmulq_f32(inv_v, scale);
let scaled = vmulq_f32(denominator, coeff_v);
let keep = vcgeq_f32(vabsq_f32(scaled), threshold);
let quantized =
vreinterpretq_f32_u32(vandq_u32(vreinterpretq_u32_f32(vrndaq_f32(scaled)), keep));
let dequantized = if BIASED {
super::ac_strategy::neon_dequantized_level_f32(quantized, bias)
} else {
quantized
};
let matrix_v = unsafe { vld1q_f32(matrix4.as_ptr()) };
let error = vmulq_f32(
vsubq_f32(scaled, dequantized),
vmulq_f32(matrix_v, inv_scale),
);
let active = if y < cy && x < cx {
vcgeq_u32(
vaddq_u32(vdupq_n_u32(x as u32), lanes),
vdupq_n_u32(cx as u32),
)
} else {
all
};
unsafe { vst1q_f32(error4.as_mut_ptr(), vbslq_f32(active, error, zero)) };
if COMPUTE_RATE {
let active_quantized = vbslq_f32(active, quantized, zero);
let absolute_q = vabsq_f32(active_quantized);
let nonzero_mask = vcgtq_f32(absolute_q, zero);
if vmaxvq_u32(nonzero_mask) != 0 {
nonzero_acc = vaddq_u32(nonzero_acc, vshrq_n_u32::<31>(nonzero_mask));
magnitude_bits = vaddq_f32(
magnitude_bits,
vbslq_f32(nonzero_mask, neon_log2p1_f32(absolute_q), zero),
);
let sv = unsafe { vld1q_u32(scan_pos.as_ptr().add(y * width + x)) };
scan_acc = vmaxq_u32(scan_acc, vandq_u32(sv, nonzero_mask));
}
}
}
}
if !COMPUTE_RATE {
return 0.0;
}
let nonzero = vaddvq_u32(nonzero_acc) as usize;
let header = vgetq_lane_f32::<0>(neon_log2p1_f32(vsetq_lane_f32::<0>(
nonzero as f32,
vdupq_n_f32(0.0),
)));
nonzero as f32 * 1.6
+ vaddvq_f32(magnitude_bits)
+ 0.4 * header
+ crate::inflated_cost::R_ZERO
* crate::inflated_cost::visited_zeros(nonzero, vmaxvq_u32(scan_acc), cx, cy)
}
#[allow(clippy::too_many_arguments)]
#[target_feature(enable = "neon")]
fn prepare_reconstruction_neon(
plane: &Plane<f32>,
px: usize,
py: usize,
pixel_width: usize,
pixel_height: usize,
error: &[f32],
original: &mut [f32],
reconstructed: &mut [f32],
) {
let n = pixel_width
.checked_mul(pixel_height)
.expect("reconstruction size overflow");
assert!(pixel_width.is_multiple_of(4));
assert!(error.len() >= n && original.len() >= n && reconstructed.len() >= n);
let (image_width, image_height) = (plane.xsize(), plane.ysize());
assert!(image_width != 0 && image_height != 0);
let source_x = px.min(image_width - 1);
let copied = pixel_width.min(image_width - source_x);
for (y, ((original_row, reconstructed_row), error_row)) in original
.chunks_exact_mut(pixel_width)
.zip(reconstructed.chunks_exact_mut(pixel_width))
.zip(error.chunks_exact(pixel_width))
.take(pixel_height)
.enumerate()
{
let source = plane.row(py.saturating_add(y).min(image_height - 1));
original_row[..copied].copy_from_slice(&source[source_x..source_x + copied]);
original_row[copied..].fill(source[image_width - 1]);
for ((output, value), delta) in reconstructed_row
.as_chunks_mut::<4>()
.0
.iter_mut()
.zip(original_row.as_chunks::<4>().0.iter())
.zip(error_row.as_chunks::<4>().0.iter())
{
let values = unsafe { vld1q_f32(value.as_ptr()) };
let errors = unsafe { vld1q_f32(delta.as_ptr()) };
unsafe { vst1q_f32(output.as_mut_ptr(), vsubq_f32(values, errors)) };
}
}
}
#[target_feature(enable = "neon")]
pub(crate) fn ssim_deficit_neon(orig: &[f32], recon: &[f32], width: usize, height: usize) -> f32 {
validate_ssim_inputs(orig, recon, width, height);
const C1: f32 = 1e-4;
const C2: f32 = 9e-4;
const INV_64: f32 = 1.0 / 64.0;
let mut deficit = 0.0f32;
for wy in 0..height / 8 {
for wx in 0..width / 8 {
let (x0, y0) = (wx * 8, wy * 8);
let base_index = y0 * width + x0;
let base_o = orig[base_index];
let base_r = recon[base_index];
let base_ov = vdupq_n_f32(base_o);
let base_rv = vdupq_n_f32(base_r);
let (mut sum_o, mut sum_r) = (vdupq_n_f32(0.0), vdupq_n_f32(0.0));
for (orig_row, recon_row) in orig
.chunks_exact(width)
.zip(recon.chunks_exact(width))
.skip(y0)
.take(8)
{
let orig8 = &orig_row.as_chunks::<8>().0[wx];
let recon8 = &recon_row.as_chunks::<8>().0[wx];
for (orig4, recon4) in orig8
.as_chunks::<4>()
.0
.iter()
.zip(recon8.as_chunks::<4>().0)
{
sum_o = vaddq_f32(
sum_o,
vsubq_f32(unsafe { vld1q_f32(orig4.as_ptr()) }, base_ov),
);
sum_r = vaddq_f32(
sum_r,
vsubq_f32(unsafe { vld1q_f32(recon4.as_ptr()) }, base_rv),
);
}
}
let mean_o = f32::mul_add(vaddvq_f32(sum_o), INV_64, base_o);
let mean_r = f32::mul_add(vaddvq_f32(sum_r), INV_64, base_r);
let mean_ov = vdupq_n_f32(mean_o);
let mean_rv = vdupq_n_f32(mean_r);
let (mut var_o, mut var_r, mut cov) =
(vdupq_n_f32(0.0), vdupq_n_f32(0.0), vdupq_n_f32(0.0));
for (orig_row, recon_row) in orig
.chunks_exact(width)
.zip(recon.chunks_exact(width))
.skip(y0)
.take(8)
{
let orig8 = &orig_row.as_chunks::<8>().0[wx];
let recon8 = &recon_row.as_chunks::<8>().0[wx];
for (orig4, recon4) in orig8
.as_chunks::<4>()
.0
.iter()
.zip(recon8.as_chunks::<4>().0)
{
let o = vsubq_f32(unsafe { vld1q_f32(orig4.as_ptr()) }, mean_ov);
let r = vsubq_f32(unsafe { vld1q_f32(recon4.as_ptr()) }, mean_rv);
var_o = vfmaq_f32(var_o, o, o);
var_r = vfmaq_f32(var_r, r, r);
cov = vfmaq_f32(cov, o, r);
}
}
let vo = vaddvq_f32(var_o) * INV_64;
let vr = vaddvq_f32(var_r) * INV_64;
let covariance = vaddvq_f32(cov) * INV_64;
let luminance = (2.0 * mean_o * mean_r + C1) / (mean_o * mean_o + mean_r * mean_r + C1);
let structure = (2.0 * covariance + C2) / (vo + vr + C2);
deficit += (1.0 - luminance * structure) * 64.0;
}
}
deficit
}
#[cfg(test)]
#[allow(clippy::too_many_arguments)]
#[target_feature(enable = "neon")]
fn recon_quantize_neon_reference<const BIASED: bool, const COMPUTE_RATE: bool>(
coeff: &[f32],
inv: &[f32],
matrix: &[f32],
quant_scale: f32,
bias: crate::group::DequantBias,
thresholds: &[f32; 4],
width: usize,
height: usize,
half: usize,
cx: usize,
cy: usize,
coeff_error: &mut [f32],
_rate_log2_lut: &RateLog2Lut,
scan_pos: &[u32],
) -> f32 {
let n = width
.checked_mul(height)
.expect("coefficient size overflow");
assert!(width.is_multiple_of(4) && half.is_multiple_of(4));
assert!(coeff.len() >= n && inv.len() >= n && matrix.len() >= n && coeff_error.len() >= n);
assert!(!COMPUTE_RATE || scan_pos.len() >= n);
let scale = vdupq_n_f32(quant_scale);
let inv_scale = vdupq_n_f32(1.0 / quant_scale);
let zero = vdupq_n_f32(0.0);
let all = vdupq_n_u32(u32::MAX);
let lanes = unsafe { vld1q_u32([0, 1, 2, 3].as_ptr()) };
let mut nonzero = 0usize;
let mut magnitude_bits = vdupq_n_f32(0.0);
let mut scan_acc = vdupq_n_u32(0);
for (y, (((coeff_row, inv_row), matrix_row), error_row)) in coeff
.chunks_exact(width)
.zip(inv.chunks_exact(width))
.zip(matrix.chunks_exact(width))
.zip(coeff_error.chunks_exact_mut(width))
.take(height)
.enumerate()
{
let yfix = if y >= height / 2 { 2 } else { 0 };
for (chunk_x, (((coeff4, inv4), matrix4), error4)) in coeff_row
.as_chunks::<4>()
.0
.iter()
.zip(inv_row.as_chunks::<4>().0.iter())
.zip(matrix_row.as_chunks::<4>().0.iter())
.zip(error_row.as_chunks_mut::<4>().0.iter_mut())
.enumerate()
{
let x = chunk_x * 4;
let threshold = vdupq_n_f32(if x >= half {
thresholds[yfix + 1]
} else {
thresholds[yfix]
});
let coeff_v = unsafe { vld1q_f32(coeff4.as_ptr()) };
let inv_v = unsafe { vld1q_f32(inv4.as_ptr()) };
let denominator = vmulq_f32(inv_v, scale);
let scaled = vmulq_f32(denominator, coeff_v);
let keep = vcgeq_f32(vabsq_f32(scaled), threshold);
let quantized =
vreinterpretq_f32_u32(vandq_u32(vreinterpretq_u32_f32(vrndaq_f32(scaled)), keep));
let dequantized = if BIASED {
super::ac_strategy::neon_dequantized_level_f32(quantized, bias)
} else {
quantized
};
let matrix_v = unsafe { vld1q_f32(matrix4.as_ptr()) };
let error = vmulq_f32(
vsubq_f32(scaled, dequantized),
vmulq_f32(matrix_v, inv_scale),
);
let active = if y < cy && x < cx {
vcgeq_u32(
vaddq_u32(vdupq_n_u32(x as u32), lanes),
vdupq_n_u32(cx as u32),
)
} else {
all
};
unsafe { vst1q_f32(error4.as_mut_ptr(), vbslq_f32(active, error, zero)) };
if COMPUTE_RATE {
let active_quantized = vbslq_f32(active, quantized, zero);
let absolute_q = vabsq_f32(active_quantized);
let nonzero_mask = vcgtq_f32(absolute_q, zero);
nonzero += vaddvq_u32(vshrq_n_u32::<31>(nonzero_mask)) as usize;
magnitude_bits = vaddq_f32(
magnitude_bits,
vbslq_f32(nonzero_mask, neon_log2p1_f32(absolute_q), zero),
);
let sv = unsafe { vld1q_u32(scan_pos.as_ptr().add(y * width + x)) };
scan_acc = vmaxq_u32(scan_acc, vandq_u32(sv, nonzero_mask));
}
}
}
if !COMPUTE_RATE {
return 0.0;
}
let header = vgetq_lane_f32::<0>(neon_log2p1_f32(vsetq_lane_f32::<0>(
nonzero as f32,
vdupq_n_f32(0.0),
)));
nonzero as f32 * 1.6
+ vaddvq_f32(magnitude_bits)
+ 0.4 * header
+ crate::inflated_cost::R_ZERO
* crate::inflated_cost::visited_zeros(nonzero, vmaxvq_u32(scan_acc), cx, cy)
}
#[cfg(test)]
mod tests {
use super::{
combine_error_neon, recon_dist_and_rate_neon_impl, rgb_hue_chroma_edge_loss_neon,
ssim_deficit_neon,
};
use crate::dc_group_data::{
STRATEGY_DCT, STRATEGY_DCT8X16, STRATEGY_DCT16X8, STRATEGY_DCT16X16, STRATEGY_DCT16X32,
STRATEGY_DCT32X16, STRATEGY_DCT32X32,
};
use crate::image::Image3F;
use crate::inflated_cost::{
ReconDistInput, ReconErrorKernels, ReconQuantization, ReconScoring, ReconSource,
ReconTransform, rate_log2_lut, recon_dist_and_rate_scalar, ssim_deficit_scalar,
};
#[test]
fn sparse_reconstruction_rate_preserves_dense_kernel_bits() {
let special = [
0.0,
-0.0,
0.5,
-0.5,
1.0,
-1.0,
1.5,
-1.5,
f32::INFINITY,
f32::NEG_INFINITY,
f32::NAN,
];
let mut rng = 73u32;
for (width, height, cx, cy) in [
(8, 8, 1, 1),
(16, 8, 2, 1),
(8, 16, 1, 2),
(16, 16, 2, 2),
(32, 16, 4, 2),
(32, 32, 4, 4),
] {
let n = width * height;
let scan: Vec<u32> = (0..n).map(|i| ((i * 17) % n) as u32).collect();
for case in 0..64 {
let coeff: Vec<f32> = (0..n)
.map(|_| {
rng = rng.wrapping_mul(1664525).wrapping_add(1013904223);
match case % 4 {
0 => 0.0,
1 => ((rng >> 8) as f32 / (1u32 << 24) as f32 - 0.5) * 0.2,
2 => ((rng >> 8) as f32 / (1u32 << 24) as f32 - 0.5) * 200.0,
_ => special[rng as usize % special.len()],
}
})
.collect();
let inv: Vec<f32> = (0..n).map(|i| 0.01 + (i % 7) as f32 * 0.15).collect();
let forward: Vec<f32> = inv.iter().map(|v| 1.0 / v).collect();
let scale = [0.01, 0.5, 1.0, 7.0, 80.0][case % 5];
let thresholds = [0.0, 0.42, 0.57, 0.91];
for (original, optimized) in [
(
super::recon_quantize_neon_reference::<true, true>
as crate::inflated_cost::ReconQuantizeFn,
super::recon_quantize_neon::<true, true>
as crate::inflated_cost::ReconQuantizeFn,
),
(
super::recon_quantize_neon_reference::<false, true>
as crate::inflated_cost::ReconQuantizeFn,
super::recon_quantize_neon::<false, true>
as crate::inflated_cost::ReconQuantizeFn,
),
] {
let mut expected_error = vec![-123.0; n];
let mut actual_error = expected_error.clone();
let run = |kernel: crate::inflated_cost::ReconQuantizeFn, error: &mut [f32]| unsafe {
kernel(
&coeff,
&inv,
&forward,
scale,
crate::group::DequantBias::DEFAULT[case % 3],
&thresholds,
width,
height,
width / 2,
cx,
cy,
error,
rate_log2_lut(),
&scan,
)
};
let expected = run(original, &mut expected_error);
let actual = run(optimized, &mut actual_error);
assert_eq!(
actual.to_bits(),
expected.to_bits(),
"{width}x{height}, case {case}"
);
for (a, e) in actual_error.iter().zip(expected_error) {
assert_eq!(a.to_bits(), e.to_bits());
}
}
}
}
}
#[test]
fn ssim_neon_matches_scalar() {
let mut state = 91u32;
for (width, height) in [(8, 8), (16, 8), (8, 16), (16, 16), (32, 32)] {
let n = width * height;
let mut orig = vec![0.0f32; n];
let mut recon = vec![0.0f32; n];
for i in 0..n {
state = state.wrapping_mul(1_664_525).wrapping_add(1_013_904_223);
orig[i] = (state >> 8) as f32 / (1u32 << 24) as f32;
recon[i] = orig[i] + ((i % 13) as f32 - 6.0) * 0.0007;
}
let scalar = ssim_deficit_scalar(&orig, &recon, width, height);
let simd = unsafe { ssim_deficit_neon(&orig, &recon, width, height) };
let tolerance = 2e-4f32.max(scalar.abs() * 2e-5);
assert!(
(simd - scalar).abs() <= tolerance,
"{width}x{height}: simd={simd} scalar={scalar}"
);
}
}
#[test]
fn recon_neon_matches_scalar_for_all_standard_transforms() {
let mut image = Image3F::new(40, 40);
for c in 0..3 {
for y in 0..40 {
for x in 0..40 {
image.plane_mut(c).row_mut(y)[x] =
0.15 * c as f32 + 0.003 * x as f32 + 0.002 * y as f32;
}
}
}
let mut coeffs = [[0.0f32; 4096]; 3];
let mut state = 17u32;
for channel in &mut coeffs {
for value in channel {
state = state.wrapping_mul(1_664_525).wrapping_add(1_013_904_223);
*value = ((state >> 8) as f32 / (1u32 << 24) as f32 - 0.5) * 2.0;
}
}
let inv_storage = [vec![1.0f32; 4096], vec![0.9f32; 4096], vec![1.1f32; 4096]];
let forward_storage = inv_storage
.clone()
.map(|m| m.iter().map(|v| 1.0 / v).collect::<Vec<f32>>());
let idct = crate::dct::IdctMethods::scalar();
for (strategy, cx, cy) in [
(STRATEGY_DCT, 1, 1),
(STRATEGY_DCT8X16, 2, 1),
(STRATEGY_DCT16X8, 2, 1),
(STRATEGY_DCT16X16, 2, 2),
(STRATEGY_DCT16X32, 4, 2),
(STRATEGY_DCT32X16, 4, 2),
(STRATEGY_DCT32X32, 4, 4),
(crate::dc_group_data::STRATEGY_DCT64X64, 8, 8),
(crate::dc_group_data::STRATEGY_DCT64X32, 8, 4),
(crate::dc_group_data::STRATEGY_DCT32X64, 8, 4),
] {
let inv = [
&inv_storage[0][..],
&inv_storage[1][..],
&inv_storage[2][..],
];
let forward = [
&forward_storage[0][..],
&forward_storage[1][..],
&forward_storage[2][..],
];
let input = ReconDistInput {
idct: &idct,
quantization: ReconQuantization {
compute_rate: true,
rate_log2_lut: rate_log2_lut(),
coeffs: [&coeffs[0], &coeffs[1], &coeffs[2]],
inverse_matrices: inv,
matrices: forward,
qac: 7.0,
qm_mult_x: 1.2,
qm_mult_b: 1.6,
distance: 1.5,
dequant_bias: crate::group::DequantBias::DEFAULT,
},
transform: ReconTransform {
blocks_x: cx,
blocks_y: cy,
strategy,
},
source: ReconSource {
opsin: &image,
x: 3,
y: 5,
},
scoring: ReconScoring {
factor_x: 0.15,
factor_b: -0.1,
channel_weights: crate::inflated_cost::CHANNEL_WEIGHT,
xyb_matrix: crate::xyb::XybMatrix::SPEC,
rgb_hue_alpha: 0.0,
keep_spatial_errors: false,
},
};
for biased in [false, true] {
let mut scalar_scratch = [[0.0f32; 4096]; 8];
let scalar = if biased {
recon_dist_and_rate_scalar::<true>(&mut scalar_scratch, &input)
} else {
recon_dist_and_rate_scalar::<false>(&mut scalar_scratch, &input)
};
let error_kernels = ReconErrorKernels {
combine: |spatial, luma, factor, combined| unsafe {
combine_error_neon(spatial, luma, factor, combined)
},
rgb_hue_chroma_edge_loss: rgb_hue_chroma_edge_loss_neon,
};
let mut simd_scratch = [[0.0f32; 4096]; 8];
let simd = unsafe {
if biased {
recon_dist_and_rate_neon_impl::<true>(
&mut simd_scratch,
&input,
&error_kernels,
)
} else {
recon_dist_and_rate_neon_impl::<false>(
&mut simd_scratch,
&input,
&error_kernels,
)
}
};
let relative_rate_tolerance = if cx * cy > 16 { 6e-6 } else { 3e-6 };
let rate_tolerance = 2e-4f32.max(scalar.rate.abs() * relative_rate_tolerance);
assert!(
(simd.rate - scalar.rate).abs() <= rate_tolerance,
"strategy {strategy} biased={biased} rate: simd={} scalar={}",
simd.rate,
scalar.rate
);
let tolerance = 5e-4f32.max(scalar.distortion.abs() * 3e-5);
assert!(
(simd.distortion - scalar.distortion).abs() <= tolerance,
"strategy {strategy} biased={biased}: simd={simd:?} scalar={scalar:?}"
);
}
}
}
#[test]
fn ssim_neon_rejects_short_slices_before_loading() {
let result =
std::panic::catch_unwind(|| unsafe { ssim_deficit_neon(&[0.0; 63], &[0.0; 64], 8, 8) });
assert!(result.is_err());
}
}