use crate::avx::ac_strategy::{avx2_log2p1_f32, hmax_u32, hsum256};
use crate::image::{Image3F, Plane};
use crate::inflated_cost::{
RateLog2Lut, ReconDistInput, ReconErrorKernels, ReconKernels, recon_dist_and_rate_with_kernels,
validate_ssim_inputs,
};
use std::arch::x86_64::*;
#[inline]
#[target_feature(enable = "avx2,fma")]
fn clamp01_f32x8(value: __m256) -> __m256 {
_mm256_min_ps(
_mm256_max_ps(value, _mm256_setzero_ps()),
_mm256_set1_ps(1.0),
)
}
#[inline]
#[target_feature(enable = "avx2,fma")]
fn xyb_to_oklab_f32x8(
matrix: &crate::xyb::XybMatrix,
x: __m256,
y: __m256,
b: __m256,
) -> (__m256, __m256, __m256) {
let cube = |value: __m256| {
let value = _mm256_sub_ps(value, _mm256_set1_ps(crate::xyb::NEG_BIAS_CBRT));
_mm256_sub_ps(
_mm256_mul_ps(_mm256_mul_ps(value, value), value),
_mm256_set1_ps(crate::xyb::OPSIN_BIAS),
)
};
let mixed0 = cube(_mm256_add_ps(y, x));
let mixed1 = cube(_mm256_sub_ps(y, x));
let mixed2 = cube(b);
let inverse_row = |offset: usize| {
let value = _mm256_mul_ps(mixed2, _mm256_set1_ps(matrix.inv[offset + 2]));
let value = _mm256_fmadd_ps(mixed1, _mm256_set1_ps(matrix.inv[offset + 1]), value);
_mm256_fmadd_ps(mixed0, _mm256_set1_ps(matrix.inv[offset]), value)
};
let r = inverse_row(0);
let g = inverse_row(3);
let b = inverse_row(6);
let zero = _mm256_setzero_ps();
let rgb_row = |cr: f32, cg: f32, cb: f32| {
let value = _mm256_mul_ps(b, _mm256_set1_ps(cb));
let value = _mm256_fmadd_ps(g, _mm256_set1_ps(cg), value);
_mm256_max_ps(_mm256_fmadd_ps(r, _mm256_set1_ps(cr), value), zero)
};
let l = rgb_row(0.412_221_46, 0.536_332_55, 0.051_445_995);
let m = rgb_row(0.211_903_5, 0.680_699_5, 0.107_396_96);
let s = rgb_row(0.088_302_46, 0.281_718_85, 0.629_978_7);
let (l, m, s) = super::xyb::vcbrt_fast3_positive_avx2(l, m, s);
let lab_row = |cl: f32, cm: f32, cs: f32| {
let value = _mm256_mul_ps(s, _mm256_set1_ps(cs));
let value = _mm256_fmadd_ps(m, _mm256_set1_ps(cm), value);
_mm256_fmadd_ps(l, _mm256_set1_ps(cl), value)
};
(
lab_row(0.210_454_26, 0.793_617_8, -0.004_072_047),
lab_row(1.977_998_5, -2.428_592_2, 0.450_593_7),
lab_row(0.025_904_037, 0.782_771_77, -0.808_675_77),
)
}
#[inline]
#[target_feature(enable = "avx2")]
fn shifted_right_f32x8(value: __m256) -> __m256 {
_mm256_permutevar8x32_ps(value, _mm256_setr_epi32(1, 2, 3, 4, 5, 6, 7, 7))
}
#[target_feature(enable = "avx2,fma")]
pub(crate) fn rgb_hue_chroma_edge_loss_avx2(
opsin: &Image3F,
px: usize,
py: usize,
width: usize,
height: usize,
spatial_error: [&[f32]; 3],
matrix: &crate::xyb::XybMatrix,
) -> f32 {
if width == 0 || height == 0 {
return 0.0;
}
let Some(end_x) = px.checked_add(width) else {
return crate::inflated_cost::rgb_hue_chroma_edge_loss_scalar(
opsin,
px,
py,
width,
height,
spatial_error,
matrix,
);
};
let Some(end_y) = py.checked_add(height) else {
return crate::inflated_cost::rgb_hue_chroma_edge_loss_scalar(
opsin,
px,
py,
width,
height,
spatial_error,
matrix,
);
};
if end_x > opsin.xsize() || end_y > opsin.ysize() || !width.is_multiple_of(8) {
return crate::inflated_cost::rgb_hue_chroma_edge_loss_scalar(
opsin,
px,
py,
width,
height,
spatial_error,
matrix,
);
}
let zero = _mm256_setzero_ps();
let abs_mask = _mm256_castsi256_ps(_mm256_set1_epi32(0x7fff_ffff));
let mut sum = zero;
for y in 0..height {
let rows = [
opsin.plane_row(0, py + y),
opsin.plane_row(1, py + y),
opsin.plane_row(2, py + y),
];
let below = if y + 1 < height {
[
opsin.plane_row(0, py + y + 1),
opsin.plane_row(1, py + y + 1),
opsin.plane_row(2, py + y + 1),
]
} else {
rows
};
let error_offset = y * width;
for x in (0..width).step_by(8) {
let p = px + x;
let sx = unsafe { _mm256_loadu_ps(rows[0].as_ptr().add(p)) };
let sy = unsafe { _mm256_loadu_ps(rows[1].as_ptr().add(p)) };
let sb = unsafe { _mm256_loadu_ps(rows[2].as_ptr().add(p)) };
let (rx, ry, rb) = if x + 8 < width {
(
unsafe { _mm256_loadu_ps(rows[0].as_ptr().add(p + 1)) },
unsafe { _mm256_loadu_ps(rows[1].as_ptr().add(p + 1)) },
unsafe { _mm256_loadu_ps(rows[2].as_ptr().add(p + 1)) },
)
} else {
(
shifted_right_f32x8(sx),
shifted_right_f32x8(sy),
shifted_right_f32x8(sb),
)
};
let bx = unsafe { _mm256_loadu_ps(below[0].as_ptr().add(p)) };
let by = unsafe { _mm256_loadu_ps(below[1].as_ptr().add(p)) };
let bb = unsafe { _mm256_loadu_ps(below[2].as_ptr().add(p)) };
let cb = _mm256_sub_ps(sb, sy);
let horizontal = _mm256_add_ps(
_mm256_and_ps(abs_mask, _mm256_sub_ps(rx, sx)),
_mm256_and_ps(abs_mask, _mm256_sub_ps(_mm256_sub_ps(rb, ry), cb)),
);
let vertical = _mm256_add_ps(
_mm256_and_ps(abs_mask, _mm256_sub_ps(bx, sx)),
_mm256_and_ps(abs_mask, _mm256_sub_ps(_mm256_sub_ps(bb, by), cb)),
);
let edge_risk = clamp01_f32x8(_mm256_mul_ps(
_mm256_sub_ps(_mm256_max_ps(horizontal, vertical), _mm256_set1_ps(0.006)),
_mm256_set1_ps(1.0 / 0.030),
));
let e = error_offset + x;
let ex = unsafe { _mm256_loadu_ps(spatial_error[0].as_ptr().add(e)) };
let ey = unsafe { _mm256_loadu_ps(spatial_error[1].as_ptr().add(e)) };
let eb = unsafe { _mm256_loadu_ps(spatial_error[2].as_ptr().add(e)) };
let (source_l, source_a, source_b) = xyb_to_oklab_f32x8(matrix, sx, sy, sb);
let (_, recon_a, recon_b) = xyb_to_oklab_f32x8(
matrix,
_mm256_sub_ps(sx, ex),
_mm256_sub_ps(sy, ey),
_mm256_sub_ps(sb, eb),
);
let source_chroma = _mm256_sqrt_ps(_mm256_fmadd_ps(
source_a,
source_a,
_mm256_mul_ps(source_b, source_b),
));
let radial_error = _mm256_div_ps(
_mm256_fmadd_ps(
source_a,
_mm256_sub_ps(source_a, recon_a),
_mm256_mul_ps(source_b, _mm256_sub_ps(source_b, recon_b)),
),
_mm256_max_ps(source_chroma, _mm256_set1_ps(1e-4)),
);
let brightness_risk = clamp01_f32x8(_mm256_mul_ps(
_mm256_sub_ps(source_l, _mm256_set1_ps(0.35)),
_mm256_set1_ps(1.0 / 0.40),
));
let chroma_risk = clamp01_f32x8(_mm256_mul_ps(
_mm256_sub_ps(source_chroma, _mm256_set1_ps(0.03)),
_mm256_set1_ps(1.0 / 0.12),
));
let risk = _mm256_mul_ps(_mm256_mul_ps(edge_risk, brightness_risk), chroma_risk);
let desaturation = _mm256_max_ps(radial_error, zero);
let perpendicular = _mm256_div_ps(
_mm256_sub_ps(
_mm256_mul_ps(source_a, recon_b),
_mm256_mul_ps(source_b, recon_a),
),
_mm256_add_ps(source_chroma, _mm256_set1_ps(1e-4)),
);
let penalty = _mm256_fmadd_ps(
perpendicular,
_mm256_mul_ps(perpendicular, _mm256_set1_ps(0.75)),
_mm256_mul_ps(desaturation, desaturation),
);
sum = _mm256_fmadd_ps(risk, penalty, sum);
}
}
hsum256(sum)
}
#[inline]
#[target_feature(enable = "avx2,fma")]
fn accumulate_gradient_vectors_x8(left: __m256, right: __m256, sum: __m256) -> __m256 {
let difference = _mm256_sub_ps(right, left);
_mm256_fmadd_ps(difference, difference, sum)
}
#[inline]
#[target_feature(enable = "avx2,fma")]
fn accumulate_gradient_x8(left: &[f32; 8], right: &[f32; 8], sum: __m256) -> __m256 {
let left = unsafe { _mm256_loadu_ps(left.as_ptr()) };
let right = unsafe { _mm256_loadu_ps(right.as_ptr()) };
accumulate_gradient_vectors_x8(left, right, sum)
}
#[target_feature(enable = "avx2,fma")]
pub(crate) fn error_gradient_energy_avx2(error: &[f32], width: usize, height: usize) -> f32 {
let n = width
.checked_mul(height)
.expect("gradient plane size overflow");
assert!(error.len() >= n);
if width == 0 || height == 0 {
return 0.0;
}
let rows = error[..n].chunks_exact(width);
let mut sum = _mm256_setzero_ps();
for row in rows.clone() {
let (row8, row_tail) = row.as_chunks::<8>();
if row_tail.is_empty() {
let (right8, _) = row[1..].as_chunks::<8>();
for (left, right) in row8.iter().zip(right8) {
sum = accumulate_gradient_x8(left, right, sum);
}
let left = unsafe { _mm256_loadu_ps(row8.last().unwrap().as_ptr()) };
let right = _mm256_permutevar8x32_ps(left, _mm256_setr_epi32(1, 2, 3, 4, 5, 6, 7, 7));
sum = accumulate_gradient_vectors_x8(left, right, sum);
continue;
}
let (left8, tail) = row[..width - 1].as_chunks::<8>();
let (right8, right_tail) = row[1..].as_chunks::<8>();
for (left, right) in left8.iter().zip(right8) {
sum = accumulate_gradient_x8(left, right, sum);
}
if !tail.is_empty() {
let mut left = [0.0; 8];
let mut right = [0.0; 8];
left[..tail.len()].copy_from_slice(tail);
right[..right_tail.len()].copy_from_slice(right_tail);
sum = accumulate_gradient_x8(&left, &right, sum);
}
}
for (top, bottom) in rows.clone().zip(rows.skip(1)) {
let (top8, top_tail) = top.as_chunks::<8>();
let (bottom8, bottom_tail) = bottom.as_chunks::<8>();
for (top, bottom) in top8.iter().zip(bottom8) {
sum = accumulate_gradient_x8(top, bottom, sum);
}
if !top_tail.is_empty() {
let mut top = [0.0; 8];
let mut bottom = [0.0; 8];
top[..top_tail.len()].copy_from_slice(top_tail);
bottom[..bottom_tail.len()].copy_from_slice(bottom_tail);
sum = accumulate_gradient_x8(&top, &bottom, sum);
}
}
hsum256(sum)
}
#[inline]
#[target_feature(enable = "avx2,fma")]
fn peak_excess_x8(
a: __m256,
b: __m256,
source_a: __m256,
source_b: __m256,
floor: __m256,
) -> __m256 {
let sign = _mm256_set1_ps(-0.0);
let error_gradient = _mm256_andnot_ps(sign, _mm256_sub_ps(b, a));
let source_gradient = _mm256_andnot_ps(sign, _mm256_sub_ps(source_b, source_a));
let excess = _mm256_max_ps(
_mm256_setzero_ps(),
_mm256_sub_ps(
_mm256_sub_ps(
error_gradient,
_mm256_mul_ps(source_gradient, _mm256_set1_ps(0.5)),
),
floor,
),
);
_mm256_mul_ps(excess, excess)
}
#[inline]
#[target_feature(enable = "avx2,fma")]
fn pooled_max_x8(value: __m256) -> f32 {
let pair_max = _mm256_max_ps(value, _mm256_permute_ps::<0x4e>(value));
let cell_max = _mm256_max_ps(pair_max, _mm256_permute_ps::<0xb1>(pair_max));
let low = _mm256_castps256_ps128(cell_max);
let high = _mm256_extractf128_ps::<1>(cell_max);
_mm_cvtss_f32(_mm_add_ss(low, high))
}
#[target_feature(enable = "avx2,fma")]
pub(crate) fn error_gradient_peak_energy_avx2(
error: &[f32],
original: &[f32],
width: usize,
height: usize,
floor: f32,
) -> f32 {
let n = width
.checked_mul(height)
.expect("gradient plane size overflow");
assert!(error.len() >= n && original.len() >= n);
assert!(floor.is_finite() && floor >= 0.0);
if width == 0 || height == 0 {
return 0.0;
}
if !width.is_multiple_of(8) {
return crate::inflated_cost::error_gradient_peak_energy_scalar(
error, original, width, height, floor,
);
}
let floor = _mm256_set1_ps(floor);
let zero = _mm256_setzero_ps();
let mut total = 0.0f32;
for cell_y in (0..height).step_by(4) {
for cell_x in (0..width).step_by(8) {
let mut max_x = zero;
let mut max_y = zero;
for y in cell_y..(cell_y + 4).min(height) {
let p = y * width + cell_x;
let current = unsafe { _mm256_loadu_ps(error.as_ptr().add(p)) };
let source = unsafe { _mm256_loadu_ps(original.as_ptr().add(p)) };
let (right, source_right) = if cell_x + 8 < width {
(
unsafe { _mm256_loadu_ps(error.as_ptr().add(p + 1)) },
unsafe { _mm256_loadu_ps(original.as_ptr().add(p + 1)) },
)
} else {
let shift = _mm256_setr_epi32(1, 2, 3, 4, 5, 6, 7, 7);
(
_mm256_permutevar8x32_ps(current, shift),
_mm256_permutevar8x32_ps(source, shift),
)
};
max_x = _mm256_max_ps(
max_x,
peak_excess_x8(current, right, source, source_right, floor),
);
if y + 1 < height {
let below = unsafe { _mm256_loadu_ps(error.as_ptr().add(p + width)) };
let source_below = unsafe { _mm256_loadu_ps(original.as_ptr().add(p + width)) };
max_y = _mm256_max_ps(
max_y,
peak_excess_x8(current, below, source, source_below, floor),
);
}
}
total += pooled_max_x8(max_x) + pooled_max_x8(max_y);
}
}
total
}
#[inline]
#[target_feature(enable = "avx2,fma")]
fn combine_error_x8(spatial: &[f32; 8], luma: &[f32; 8], factor: __m256, combined: &mut [f32; 8]) {
let spatial = unsafe { _mm256_loadu_ps(spatial.as_ptr()) };
let luma = unsafe { _mm256_loadu_ps(luma.as_ptr()) };
let value = _mm256_fmadd_ps(factor, luma, spatial);
unsafe { _mm256_storeu_ps(combined.as_mut_ptr(), value) };
}
#[target_feature(enable = "avx2,fma")]
pub(crate) fn combine_error_avx2(spatial: &[f32], luma: &[f32], factor: f32, combined: &mut [f32]) {
debug_assert_eq!(spatial.len(), luma.len());
debug_assert_eq!(spatial.len(), combined.len());
let (spatial32, spatial_tail) = spatial.as_chunks::<32>();
let (luma32, luma_tail) = luma.as_chunks::<32>();
let (combined32, combined_tail) = combined.as_chunks_mut::<32>();
let factor = _mm256_set1_ps(factor);
for ((spatial, luma), combined) in spatial32.iter().zip(luma32).zip(combined32) {
let [s0, s1, s2, s3] = spatial.as_chunks::<8>().0 else {
unreachable!()
};
let [l0, l1, l2, l3] = luma.as_chunks::<8>().0 else {
unreachable!()
};
let [c0, c1, c2, c3] = combined.as_chunks_mut::<8>().0 else {
unreachable!()
};
combine_error_x8(s0, l0, factor, c0);
combine_error_x8(s1, l1, factor, c1);
combine_error_x8(s2, l2, factor, c2);
combine_error_x8(s3, l3, factor, c3);
}
let (spatial8, spatial_remainder) = spatial_tail.as_chunks::<8>();
let (luma8, luma_remainder) = luma_tail.as_chunks::<8>();
let (combined8, combined_remainder) = combined_tail.as_chunks_mut::<8>();
for ((spatial, luma), combined) in spatial8.iter().zip(luma8).zip(combined8) {
combine_error_x8(spatial, luma, factor, combined);
}
if !spatial_remainder.is_empty() {
let mask = _mm256_cmpgt_epi32(
_mm256_set1_epi32(spatial_remainder.len() as i32),
_mm256_setr_epi32(0, 1, 2, 3, 4, 5, 6, 7),
);
let spatial = unsafe { _mm256_maskload_ps(spatial_remainder.as_ptr(), mask) };
let luma = unsafe { _mm256_maskload_ps(luma_remainder.as_ptr(), mask) };
let value = _mm256_fmadd_ps(factor, luma, spatial);
unsafe { _mm256_maskstore_ps(combined_remainder.as_mut_ptr(), mask, value) };
}
}
#[target_feature(enable = "avx2,fma")]
pub(crate) fn recon_dist_and_rate_avx2(
scratch: &mut [[f32; 1024]; 8],
input: &ReconDistInput<'_>,
error: &ReconErrorKernels,
) -> (f32, f32) {
recon_dist_and_rate_avx2_impl::<true>(scratch, input, error)
}
#[target_feature(enable = "avx2,fma")]
fn recon_dist_and_rate_avx2_impl<const BIASED: bool>(
scratch: &mut [[f32; 1024]; 8],
input: &ReconDistInput<'_>,
error: &ReconErrorKernels,
) -> (f32, f32) {
recon_dist_and_rate_with_kernels(
scratch,
input,
&ReconKernels {
quantize: recon_quantize_avx2::<BIASED>,
ssim: ssim_deficit_avx2,
prepare: prepare_reconstruction_avx2,
error,
},
)
}
#[inline]
#[target_feature(enable = "avx2,fma")]
fn avx_dequantized_level_f32(q: __m256) -> __m256 {
let sign_mask = _mm256_set1_ps(-0.0);
let absq = _mm256_andnot_ps(sign_mask, q);
let big = _mm256_sub_ps(
q,
_mm256_div_ps(_mm256_set1_ps(crate::group::DEFAULT_QUANT_BIAS_3), q),
);
let sign_bits = _mm256_and_ps(sign_mask, q);
let one = _mm256_or_ps(
sign_bits,
_mm256_set1_ps(crate::group::DEFAULT_QUANT_BIAS_1),
);
let use_big = _mm256_cmp_ps::<_CMP_GE_OQ>(absq, _mm256_set1_ps(1.125));
let dq = _mm256_blendv_ps(one, big, use_big);
let nz = _mm256_cmp_ps::<_CMP_GT_OQ>(absq, _mm256_setzero_ps());
_mm256_and_ps(dq, nz)
}
#[allow(clippy::too_many_arguments)]
#[target_feature(enable = "avx2,fma")]
fn recon_quantize_avx2<const BIASED: bool>(
coeff: &[f32],
inv: &[f32],
quant_scale: f32,
thresholds: &[f32; 4],
width: usize,
height: usize,
half: usize,
cx: usize,
cy: usize,
coeff_error: &mut [f32],
_rate_log2_lut: &RateLog2Lut,
scan_pos: &[u32],
) -> f32 {
let n = width
.checked_mul(height)
.expect("coefficient size overflow");
assert!(width.is_multiple_of(8));
assert!(coeff.len() >= n && inv.len() >= n && coeff_error.len() >= n && scan_pos.len() >= n);
let scale = _mm256_set1_ps(quant_scale);
let sign = _mm256_set1_ps(-0.0);
let lane_ids = _mm256_setr_epi32(0, 1, 2, 3, 4, 5, 6, 7);
let all = _mm256_set1_epi32(-1);
const ROUND: i32 = _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC;
let mut nonzero = 0usize;
let mut magnitude_bits = _mm256_setzero_ps();
let mut scan_acc = _mm256_setzero_si256();
for (y, (((coeff_row, inv_row), error_row), scan_row)) in coeff
.chunks_exact(width)
.zip(inv.chunks_exact(width))
.zip(coeff_error.chunks_exact_mut(width))
.zip(scan_pos.chunks_exact(width))
.take(height)
.enumerate()
{
let yfix = if y >= height / 2 { 2 } else { 0 };
for (chunk_x, (((coeff8, inv8), error8), scan8)) in coeff_row
.as_chunks::<8>()
.0
.iter()
.zip(inv_row.as_chunks::<8>().0.iter())
.zip(error_row.as_chunks_mut::<8>().0.iter_mut())
.zip(scan_row.as_chunks::<8>().0.iter())
.enumerate()
{
let x = chunk_x * 8;
let threshold = if x + 8 <= half {
_mm256_set1_ps(thresholds[yfix])
} else if x >= half {
_mm256_set1_ps(thresholds[yfix + 1])
} else {
let lane_x = _mm256_add_epi32(_mm256_set1_epi32(x as i32), lane_ids);
let high = _mm256_cmpgt_epi32(lane_x, _mm256_set1_epi32(half as i32 - 1));
_mm256_blendv_ps(
_mm256_set1_ps(thresholds[yfix]),
_mm256_set1_ps(thresholds[yfix + 1]),
_mm256_castsi256_ps(high),
)
};
let coeff_v = unsafe { _mm256_loadu_ps(coeff8.as_ptr()) };
let inv_v = unsafe { _mm256_loadu_ps(inv8.as_ptr()) };
let denominator = _mm256_mul_ps(inv_v, scale);
let scaled = _mm256_mul_ps(denominator, coeff_v);
let absolute = _mm256_andnot_ps(sign, scaled);
let keep = _mm256_cmp_ps::<_CMP_GE_OQ>(absolute, threshold);
let quantized = _mm256_and_ps(_mm256_round_ps::<ROUND>(scaled), keep);
let dequantized = if BIASED {
avx_dequantized_level_f32(quantized)
} else {
quantized
};
let error = _mm256_div_ps(_mm256_sub_ps(scaled, dequantized), denominator);
let active_i = if y < cy && x < cx {
let lane_x = _mm256_add_epi32(_mm256_set1_epi32(x as i32), lane_ids);
_mm256_cmpgt_epi32(lane_x, _mm256_set1_epi32(cx as i32 - 1))
} else {
all
};
let active = _mm256_castsi256_ps(active_i);
unsafe {
_mm256_storeu_ps(error8.as_mut_ptr(), _mm256_and_ps(error, active));
}
let active_quantized = _mm256_and_ps(quantized, active);
let absolute_q = _mm256_andnot_ps(sign, active_quantized);
let nonzero_mask = _mm256_cmp_ps::<_CMP_GT_OQ>(absolute_q, _mm256_setzero_ps());
nonzero += _mm256_movemask_ps(nonzero_mask).count_ones() as usize;
magnitude_bits = _mm256_add_ps(
magnitude_bits,
_mm256_and_ps(avx2_log2p1_f32(absolute_q), nonzero_mask),
);
let sv = unsafe { _mm256_loadu_si256(scan8.as_ptr().cast()) };
scan_acc = _mm256_max_epu32(
scan_acc,
_mm256_and_si256(sv, _mm256_castps_si256(nonzero_mask)),
);
}
}
let v_max = _mm_max_epu32(
_mm256_castsi256_si128(scan_acc),
_mm256_extracti128_si256::<1>(scan_acc),
);
let max_scan = hmax_u32(v_max);
let header_input = _mm256_setr_ps(nonzero as f32, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0);
let header = _mm_cvtss_f32(_mm256_castps256_ps128(avx2_log2p1_f32(header_input)));
nonzero as f32 * 1.6
+ hsum256(magnitude_bits)
+ 0.4 * header
+ crate::inflated_cost::R_ZERO
* crate::inflated_cost::visited_zeros(nonzero, max_scan, cx, cy)
}
#[allow(clippy::too_many_arguments)]
#[target_feature(enable = "avx2")]
fn prepare_reconstruction_avx2(
plane: &Plane<f32>,
px: usize,
py: usize,
pixel_width: usize,
pixel_height: usize,
error: &[f32],
original: &mut [f32],
reconstructed: &mut [f32],
) {
let n = pixel_width
.checked_mul(pixel_height)
.expect("reconstruction size overflow");
assert!(pixel_width.is_multiple_of(8));
assert!(error.len() >= n && original.len() >= n && reconstructed.len() >= n);
let (image_width, image_height) = (plane.xsize(), plane.ysize());
assert!(image_width != 0 && image_height != 0);
let source_x = px.min(image_width - 1);
let copied = pixel_width.min(image_width - source_x);
for (y, ((original_row, reconstructed_row), error_row)) in original
.chunks_exact_mut(pixel_width)
.zip(reconstructed.chunks_exact_mut(pixel_width))
.zip(error.chunks_exact(pixel_width))
.take(pixel_height)
.enumerate()
{
let source = plane.row(py.saturating_add(y).min(image_height - 1));
original_row[..copied].copy_from_slice(&source[source_x..source_x + copied]);
original_row[copied..].fill(source[image_width - 1]);
for ((output, value), delta) in reconstructed_row
.as_chunks_mut::<8>()
.0
.iter_mut()
.zip(original_row.as_chunks::<8>().0.iter())
.zip(error_row.as_chunks::<8>().0.iter())
{
let values = unsafe { _mm256_loadu_ps(value.as_ptr()) };
let errors = unsafe { _mm256_loadu_ps(delta.as_ptr()) };
unsafe { _mm256_storeu_ps(output.as_mut_ptr(), _mm256_sub_ps(values, errors)) };
}
}
}
#[target_feature(enable = "avx2,fma")]
pub(crate) fn ssim_deficit_avx2(orig: &[f32], recon: &[f32], width: usize, height: usize) -> f32 {
validate_ssim_inputs(orig, recon, width, height);
const C1: f32 = 1e-4;
const C2: f32 = 9e-4;
const INV_64: f32 = 1.0 / 64.0;
let mut deficit = 0.0f32;
for wy in 0..height / 8 {
for wx in 0..width / 8 {
let (x0, y0) = (wx * 8, wy * 8);
let base_index = y0 * width + x0;
let base_o = orig[base_index];
let base_r = recon[base_index];
let base_ov = _mm256_set1_ps(base_o);
let base_rv = _mm256_set1_ps(base_r);
let (mut sum_o, mut sum_r) = (_mm256_setzero_ps(), _mm256_setzero_ps());
for (orig_row, recon_row) in orig
.chunks_exact(width)
.zip(recon.chunks_exact(width))
.skip(y0)
.take(8)
{
let orig8 = &orig_row.as_chunks::<8>().0[wx];
let recon8 = &recon_row.as_chunks::<8>().0[wx];
sum_o = _mm256_add_ps(
sum_o,
_mm256_sub_ps(unsafe { _mm256_loadu_ps(orig8.as_ptr()) }, base_ov),
);
sum_r = _mm256_add_ps(
sum_r,
_mm256_sub_ps(unsafe { _mm256_loadu_ps(recon8.as_ptr()) }, base_rv),
);
}
let mean_o = f32::mul_add(hsum256(sum_o), INV_64, base_o);
let mean_r = f32::mul_add(hsum256(sum_r), INV_64, base_r);
let mean_ov = _mm256_set1_ps(mean_o);
let mean_rv = _mm256_set1_ps(mean_r);
let (mut var_o, mut var_r, mut cov) = (
_mm256_setzero_ps(),
_mm256_setzero_ps(),
_mm256_setzero_ps(),
);
for (orig_row, recon_row) in orig
.chunks_exact(width)
.zip(recon.chunks_exact(width))
.skip(y0)
.take(8)
{
let orig8 = &orig_row.as_chunks::<8>().0[wx];
let recon8 = &recon_row.as_chunks::<8>().0[wx];
let o = _mm256_sub_ps(unsafe { _mm256_loadu_ps(orig8.as_ptr()) }, mean_ov);
let r = _mm256_sub_ps(unsafe { _mm256_loadu_ps(recon8.as_ptr()) }, mean_rv);
var_o = _mm256_fmadd_ps(o, o, var_o);
var_r = _mm256_fmadd_ps(r, r, var_r);
cov = _mm256_fmadd_ps(o, r, cov);
}
let vo = hsum256(var_o) * INV_64;
let vr = hsum256(var_r) * INV_64;
let covariance = hsum256(cov) * INV_64;
let luminance = (2.0 * mean_o * mean_r + C1) / (mean_o * mean_o + mean_r * mean_r + C1);
let structure = (2.0 * covariance + C2) / (vo + vr + C2);
deficit += (1.0 - luminance * structure) * 64.0;
}
}
deficit
}
#[cfg(test)]
mod tests {
use super::{
combine_error_avx2, error_gradient_energy_avx2, error_gradient_peak_energy_avx2,
recon_dist_and_rate_avx2_impl, rgb_hue_chroma_edge_loss_avx2, ssim_deficit_avx2,
};
use crate::dc_group_data::{
STRATEGY_DCT, STRATEGY_DCT8X16, STRATEGY_DCT16X8, STRATEGY_DCT16X16, STRATEGY_DCT16X32,
STRATEGY_DCT32X16, STRATEGY_DCT32X32,
};
use crate::image::Image3F;
use crate::inflated_cost::{
ReconDistInput, ReconErrorKernels, ReconQuantization, ReconScoring, ReconSource,
ReconTransform, rate_log2_lut, recon_dist_and_rate_scalar, ssim_deficit_scalar,
};
fn available() -> bool {
std::is_x86_feature_detected!("avx2") && std::is_x86_feature_detected!("fma")
}
#[test]
fn ssim_avx2_matches_scalar() {
if !available() {
return;
}
let mut state = 91u32;
for (width, height) in [(8, 8), (16, 8), (8, 16), (16, 16), (32, 32)] {
let n = width * height;
let mut orig = vec![0.0f32; n];
let mut recon = vec![0.0f32; n];
for i in 0..n {
state = state.wrapping_mul(1_664_525).wrapping_add(1_013_904_223);
orig[i] = (state >> 8) as f32 / (1u32 << 24) as f32;
recon[i] = orig[i] + ((i % 13) as f32 - 6.0) * 0.0007;
}
let scalar = ssim_deficit_scalar(&orig, &recon, width, height);
let simd = unsafe { ssim_deficit_avx2(&orig, &recon, width, height) };
let tolerance = 2e-4f32.max(scalar.abs() * 2e-5);
assert!(
(simd - scalar).abs() <= tolerance,
"{width}x{height}: simd={simd} scalar={scalar}"
);
}
}
#[test]
fn recon_avx2_matches_scalar_for_all_standard_transforms() {
if !available() {
return;
}
let mut image = Image3F::new(40, 40);
for c in 0..3 {
for y in 0..40 {
for x in 0..40 {
image.plane_mut(c).row_mut(y)[x] =
0.15 * c as f32 + 0.003 * x as f32 + 0.002 * y as f32;
}
}
}
let mut coeffs = [[0.0f32; 1024]; 3];
let mut state = 17u32;
for channel in &mut coeffs {
for value in channel {
state = state.wrapping_mul(1_664_525).wrapping_add(1_013_904_223);
*value = ((state >> 8) as f32 / (1u32 << 24) as f32 - 0.5) * 2.0;
}
}
let inv_storage = [vec![1.0f32; 1024], vec![0.9f32; 1024], vec![1.1f32; 1024]];
let idct = crate::dct::IdctMethods::scalar();
for (strategy, cx, cy) in [
(STRATEGY_DCT, 1, 1),
(STRATEGY_DCT8X16, 2, 1),
(STRATEGY_DCT16X8, 2, 1),
(STRATEGY_DCT16X16, 2, 2),
(STRATEGY_DCT16X32, 4, 2),
(STRATEGY_DCT32X16, 4, 2),
(STRATEGY_DCT32X32, 4, 4),
] {
let inv = [
&inv_storage[0][..],
&inv_storage[1][..],
&inv_storage[2][..],
];
let input = ReconDistInput {
idct: &idct,
quantization: ReconQuantization {
rate_log2_lut: rate_log2_lut(),
coeffs: [&coeffs[0], &coeffs[1], &coeffs[2]],
inverse_matrices: inv,
qac: 7.0,
qm_mult_x: 1.2,
qm_mult_b: 1.6,
distance: 1.5,
},
transform: ReconTransform {
blocks_x: cx,
blocks_y: cy,
strategy,
},
source: ReconSource {
opsin: &image,
x: 3,
y: 5,
},
scoring: ReconScoring {
factor_x: 0.15,
factor_b: -0.1,
channel_weights: crate::inflated_cost::CHANNEL_WEIGHT,
xyb_matrix: crate::xyb::XybMatrix::SPEC,
rgb_hue_alpha: 0.0,
gradient_alpha: 0.0,
gradient_peak_alpha: 3.0,
keep_spatial_errors: false,
},
};
for biased in [false, true] {
let mut scalar_scratch = [[0.0f32; 1024]; 8];
let scalar = if biased {
recon_dist_and_rate_scalar::<true>(&mut scalar_scratch, &input)
} else {
recon_dist_and_rate_scalar::<false>(&mut scalar_scratch, &input)
};
let error_kernels = ReconErrorKernels {
gradient_energy: |error, width, height| unsafe {
error_gradient_energy_avx2(error, width, height)
},
gradient_peak_energy: |error, original, width, height, floor| unsafe {
error_gradient_peak_energy_avx2(error, original, width, height, floor)
},
combine: |spatial, luma, factor, combined| unsafe {
combine_error_avx2(spatial, luma, factor, combined)
},
rgb_hue_chroma_edge_loss: rgb_hue_chroma_edge_loss_avx2,
};
let mut simd_scratch = [[0.0f32; 1024]; 8];
let simd = unsafe {
if biased {
recon_dist_and_rate_avx2_impl::<true>(
&mut simd_scratch,
&input,
&error_kernels,
)
} else {
recon_dist_and_rate_avx2_impl::<false>(
&mut simd_scratch,
&input,
&error_kernels,
)
}
};
let rate_tolerance = 2e-4f32.max(scalar.1.abs() * 3e-6);
assert!(
(simd.1 - scalar.1).abs() <= rate_tolerance,
"strategy {strategy} biased={biased} rate: simd={} scalar={}",
simd.1,
scalar.1
);
let tolerance = 5e-4f32.max(scalar.0.abs() * 3e-5);
assert!(
(simd.0 - scalar.0).abs() <= tolerance,
"strategy {strategy} biased={biased}: simd={simd:?} scalar={scalar:?}"
);
}
}
}
#[test]
fn ssim_avx2_rejects_short_slices_before_loading() {
if !available() {
return;
}
let result =
std::panic::catch_unwind(|| unsafe { ssim_deficit_avx2(&[0.0; 63], &[0.0; 64], 8, 8) });
assert!(result.is_err());
}
}