#[cfg(target_arch = "x86")]
use std::arch::x86::*;
#[cfg(target_arch = "x86_64")]
use std::arch::x86_64::*;
#[inline]
#[target_feature(enable = "sse4.1")]
pub(crate) fn hsum(v: __m128) -> f32 {
let shuf = _mm_movehdup_ps(v); let sums = _mm_add_ps(v, shuf);
let shuf2 = _mm_movehl_ps(shuf, sums);
let sums = _mm_add_ss(sums, shuf2);
_mm_cvtss_f32(sums)
}
#[target_feature(enable = "sse4.1")]
fn gradient_region_stats_impl<const CHROMA: bool>(
opsin: &crate::image::Image3F,
px0: usize,
py0: usize,
w: usize,
h: usize,
eps: f32,
) -> crate::ac_strategy::GradientRegionStats {
if !w.is_multiple_of(4)
|| !h.is_multiple_of(4)
|| px0 > opsin.xsize().saturating_sub(w)
|| py0 > opsin.ysize().saturating_sub(h)
{
return if CHROMA {
crate::ac_strategy::gradient_region_stats_with_chroma_scalar(opsin, px0, py0, w, h, eps)
} else {
crate::ac_strategy::gradient_region_stats_scalar(opsin, px0, py0, w, h, eps)
};
}
let xs = opsin.xsize();
let cw = w / 4;
let ch = h / 4;
let mut means = [0.0f32; 256];
let mut within = 0.0f32;
let mut chroma = _mm_setzero_ps();
let sign = _mm_set1_ps(-0.0);
const ONE_OVER_16: f32 = 1.0 / 16.0;
let row_start = py0 * xs;
let row_end = (py0 + h) * xs;
let x_rows = &opsin.plane_data(0)[row_start..row_end];
let y_rows = &opsin.plane_data(1)[row_start..row_end];
let b_rows = &opsin.plane_data(2)[row_start..row_end];
for (mean_row, ((y_rows4, x_rows4), b_rows4)) in means[..cw * ch].chunks_exact_mut(cw).zip(
y_rows
.chunks_exact(xs * 4)
.zip(x_rows.chunks_exact(xs * 4))
.zip(b_rows.chunks_exact(xs * 4)),
) {
for (cx, mean) in mean_row.iter_mut().enumerate() {
let mut sum = _mm_setzero_ps();
let mut sum2 = _mm_setzero_ps();
if CHROMA {
for ((y_row, x_row), b_row) in y_rows4
.chunks_exact(xs)
.zip(x_rows4.chunks_exact(xs))
.zip(b_rows4.chunks_exact(xs))
{
let y4 = &y_row[px0..px0 + w].as_chunks::<4>().0[cx];
let x4 = &x_row[px0..px0 + w].as_chunks::<4>().0[cx];
let b4 = &b_row[px0..px0 + w].as_chunks::<4>().0[cx];
let yv = unsafe { _mm_loadu_ps(y4.as_ptr()) };
let xv = unsafe { _mm_loadu_ps(x4.as_ptr()) };
let bv = unsafe { _mm_loadu_ps(b4.as_ptr()) };
sum = _mm_add_ps(sum, yv);
sum2 = _mm_add_ps(sum2, _mm_mul_ps(yv, yv));
let xa = _mm_andnot_ps(sign, xv);
let bya = _mm_andnot_ps(sign, _mm_sub_ps(bv, yv));
chroma = _mm_add_ps(chroma, _mm_add_ps(xa, bya));
}
} else {
for y_row in y_rows4.chunks_exact(xs) {
let y4 = &y_row[px0..px0 + w].as_chunks::<4>().0[cx];
let yv = unsafe { _mm_loadu_ps(y4.as_ptr()) };
sum = _mm_add_ps(sum, yv);
sum2 = _mm_add_ps(sum2, _mm_mul_ps(yv, yv));
}
}
let m = hsum(sum) * ONE_OVER_16;
*mean = m;
within += (hsum(sum2) * ONE_OVER_16 - m * m).max(0.0);
}
}
crate::ac_strategy::finish_gradient_region_stats(
&means[..cw * ch],
within,
if CHROMA { hsum(chroma) } else { 0.0 },
w * h,
eps,
)
}
#[target_feature(enable = "sse4.1")]
pub(crate) fn gradient_region_stats_sse41(
opsin: &crate::image::Image3F,
px0: usize,
py0: usize,
w: usize,
h: usize,
eps: f32,
) -> crate::ac_strategy::GradientRegionStats {
gradient_region_stats_impl::<false>(opsin, px0, py0, w, h, eps)
}
#[target_feature(enable = "sse4.1")]
pub(crate) fn gradient_region_stats_with_chroma_sse41(
opsin: &crate::image::Image3F,
px0: usize,
py0: usize,
w: usize,
h: usize,
eps: f32,
) -> crate::ac_strategy::GradientRegionStats {
gradient_region_stats_impl::<true>(opsin, px0, py0, w, h, eps)
}
#[inline]
#[target_feature(enable = "sse4.1")]
pub(super) fn sse_dequantized_level_f32(q: __m128) -> __m128 {
let sign_mask = _mm_set1_ps(-0.0);
let absq = _mm_andnot_ps(sign_mask, q);
let big = _mm_sub_ps(
q,
_mm_div_ps(_mm_set1_ps(crate::group::DEFAULT_QUANT_BIAS_3), q),
);
let sign = _mm_and_ps(sign_mask, q);
let one = _mm_or_ps(sign, _mm_set1_ps(crate::group::DEFAULT_QUANT_BIAS_1));
let use_big = _mm_cmpge_ps(absq, _mm_set1_ps(1.125));
let dq = _mm_blendv_ps(one, big, use_big);
let nz = _mm_cmpgt_ps(absq, _mm_setzero_ps());
_mm_and_ps(dq, nz)
}
#[allow(clippy::too_many_arguments)]
#[target_feature(enable = "sse4.1")]
pub(crate) fn sse_and_rate_sse<const BIASED: bool>(
coeff: &[f32],
inv_matrix: &[f32],
q_scaled: f32,
width: usize,
height: usize,
half: usize,
cx: usize,
cy: usize,
_rate_log2_lut: &crate::inflated_cost::RateLog2Lut,
thr: &[f32; 4],
scan_pos: &[u32],
) -> (f32, usize, f32, u32) {
sse_and_rate_sse_impl::<BIASED, false>(
coeff,
inv_matrix,
q_scaled,
width,
height,
half,
cx,
cy,
_rate_log2_lut,
thr,
scan_pos,
&mut [],
)
}
#[allow(clippy::too_many_arguments)]
#[target_feature(enable = "sse4.1")]
pub(crate) fn sse_and_rate_sse_impl<const BIASED: bool, const SAVE_LEVELS: bool>(
coeff: &[f32],
inv_matrix: &[f32],
q_scaled: f32,
width: usize,
height: usize,
half: usize,
cx: usize,
cy: usize,
_rate_log2_lut: &crate::inflated_cost::RateLog2Lut,
thr: &[f32; 4],
scan_pos: &[u32],
levels: &mut [i32],
) -> (f32, usize, f32, u32) {
let n = width * height;
assert!(coeff.len() >= n && inv_matrix.len() >= n);
assert!(if SAVE_LEVELS {
levels.len() >= n
} else {
scan_pos.len() >= n
});
debug_assert!(width.is_multiple_of(4) && half.is_multiple_of(4));
let qs = _mm_set1_ps(q_scaled);
let mut sse_acc = _mm_setzero_ps();
let mut mag_acc = _mm_setzero_ps();
let mut nonzero_acc = _mm_setzero_si128();
let mut scan_acc = _mm_setzero_si128();
let zero = _mm_setzero_ps();
let sign_mask = _mm_set1_ps(-0.0);
let lane_ids = _mm_setr_epi32(0, 1, 2, 3);
let izero = _mm_setzero_si128();
let all_active_i = _mm_cmpeq_epi32(izero, izero);
const RND: i32 = _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC;
for (y, (coeffs, inv_matrix)) in coeff
.chunks_exact(width)
.zip(inv_matrix.chunks_exact(width))
.take(height)
.enumerate()
{
let yfix = if y >= height / 2 { 2 } else { 0 };
let thr_lo = thr[yfix];
let thr_hi = thr[yfix + 1];
for (x0, (coeff4, inv4)) in coeffs
.as_chunks::<4>()
.0
.iter()
.zip(inv_matrix.as_chunks::<4>().0.iter())
.enumerate()
{
let x = x0 * 4;
let threshold = if x >= half { thr_hi } else { thr_lo };
let thrv = _mm_set1_ps(threshold);
let cv = unsafe { _mm_loadu_ps(coeff4.as_ptr()) };
let mv = unsafe { _mm_loadu_ps(inv4.as_ptr()) };
let a = _mm_mul_ps(_mm_mul_ps(mv, qs), cv);
let absa = _mm_andnot_ps(sign_mask, a);
let keep = _mm_cmpge_ps(absa, thrv);
let rounded = _mm_round_ps::<RND>(a);
let q = _mm_and_ps(rounded, keep);
let d = if BIASED {
_mm_sub_ps(a, sse_dequantized_level_f32(q))
} else {
_mm_sub_ps(a, q)
};
let d2 = _mm_mul_ps(d, d);
let active_i = if y < cy && x < cx {
let lane_x = _mm_add_epi32(_mm_set1_epi32(x as i32), lane_ids);
let cxv = _mm_set1_epi32(cx as i32);
let llf = _mm_cmpgt_epi32(cxv, lane_x);
_mm_andnot_si128(llf, all_active_i)
} else {
all_active_i
};
let active = _mm_castsi128_ps(active_i);
let d2 = _mm_and_ps(d2, active);
sse_acc = _mm_add_ps(sse_acc, d2);
let absq = _mm_andnot_ps(sign_mask, q);
let nz = _mm_cmpgt_ps(absq, zero);
let rate_mask = _mm_and_ps(nz, active);
nonzero_acc = _mm_sub_epi32(nonzero_acc, _mm_castps_si128(rate_mask));
if SAVE_LEVELS {
let truncated = _mm_round_ps::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(a);
let frac = _mm_sub_ps(a, truncated);
let ge_half = _mm_cmpge_ps(_mm_andnot_ps(sign_mask, frac), _mm_set1_ps(0.5));
let signed_one = _mm_or_ps(_mm_set1_ps(1.0), _mm_and_ps(a, sign_mask));
let quantized =
_mm_cvttps_epi32(_mm_add_ps(truncated, _mm_and_ps(signed_one, ge_half)));
let quantized =
_mm_and_si128(quantized, _mm_and_si128(_mm_castps_si128(keep), active_i));
unsafe {
_mm_storeu_si128(levels.as_mut_ptr().add(y * width + x).cast(), quantized)
};
}
if !SAVE_LEVELS && _mm_movemask_ps(rate_mask) != 0 {
let ratev = sse_log2p1_f32(absq);
mag_acc = _mm_add_ps(mag_acc, _mm_and_ps(ratev, rate_mask));
let sv = unsafe {
_mm_loadu_si128(scan_pos.as_ptr().add(y * width + x) as *const __m128i)
};
scan_acc = _mm_max_epu32(scan_acc, _mm_and_si128(sv, _mm_castps_si128(rate_mask)));
}
}
}
let nonzero = _mm_hadd_epi32(nonzero_acc, nonzero_acc);
let nonzero = _mm_hadd_epi32(nonzero, nonzero);
let nzeros = _mm_cvtsi128_si32(nonzero) as usize;
(hsum(sse_acc), nzeros, hsum(mag_acc), hmax_u32(scan_acc))
}
#[cfg(test)]
#[allow(clippy::too_many_arguments)]
#[target_feature(enable = "sse4.1")]
pub(crate) fn sse_and_rate_sse_impl_reference<const BIASED: bool, const SAVE_LEVELS: bool>(
coeff: &[f32],
inv_matrix: &[f32],
q_scaled: f32,
width: usize,
height: usize,
half: usize,
cx: usize,
cy: usize,
_rate_log2_lut: &crate::inflated_cost::RateLog2Lut,
thr: &[f32; 4],
scan_pos: &[u32],
levels: &mut [i32],
) -> (f32, usize, f32, u32) {
let n = width * height;
assert!(coeff.len() >= n && inv_matrix.len() >= n);
assert!(if SAVE_LEVELS {
levels.len() >= n
} else {
scan_pos.len() >= n
});
debug_assert!(width.is_multiple_of(4) && half.is_multiple_of(4));
let qs = _mm_set1_ps(q_scaled);
let mut sse_acc = _mm_setzero_ps();
let mut mag_acc = _mm_setzero_ps();
let mut nzeros = 0usize;
let mut scan_acc = _mm_setzero_si128();
let zero = _mm_setzero_ps();
let sign_mask = _mm_set1_ps(-0.0);
let lane_ids = _mm_setr_epi32(0, 1, 2, 3);
let izero = _mm_setzero_si128();
let all_active_i = _mm_cmpeq_epi32(izero, izero);
const RND: i32 = _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC;
for (y, (coeffs, inv_matrix)) in coeff
.chunks_exact(width)
.zip(inv_matrix.chunks_exact(width))
.take(height)
.enumerate()
{
let yfix = if y >= height / 2 { 2 } else { 0 };
let thr_lo = thr[yfix];
let thr_hi = thr[yfix + 1];
for (x0, (coeff4, inv4)) in coeffs
.as_chunks::<4>()
.0
.iter()
.zip(inv_matrix.as_chunks::<4>().0.iter())
.enumerate()
{
let x = x0 * 4;
let threshold = if x >= half { thr_hi } else { thr_lo };
let thrv = _mm_set1_ps(threshold);
let cv = unsafe { _mm_loadu_ps(coeff4.as_ptr()) };
let mv = unsafe { _mm_loadu_ps(inv4.as_ptr()) };
let a = _mm_mul_ps(_mm_mul_ps(mv, qs), cv);
let absa = _mm_andnot_ps(sign_mask, a);
let keep = _mm_cmpge_ps(absa, thrv);
let rounded = _mm_round_ps::<RND>(a);
let q = _mm_and_ps(rounded, keep);
let d = if BIASED {
_mm_sub_ps(a, sse_dequantized_level_f32(q))
} else {
_mm_sub_ps(a, q)
};
let d2 = _mm_mul_ps(d, d);
let active_i = if y < cy && x < cx {
let lane_x = _mm_add_epi32(_mm_set1_epi32(x as i32), lane_ids);
let cxv = _mm_set1_epi32(cx as i32);
let llf = _mm_cmpgt_epi32(cxv, lane_x);
_mm_andnot_si128(llf, all_active_i)
} else {
all_active_i
};
let active = _mm_castsi128_ps(active_i);
let d2 = _mm_and_ps(d2, active);
sse_acc = _mm_add_ps(sse_acc, d2);
let absq = _mm_andnot_ps(sign_mask, q);
let nz = _mm_cmpgt_ps(absq, zero);
let rate_mask = _mm_and_ps(nz, active);
let rate_bits = _mm_movemask_ps(rate_mask);
nzeros += rate_bits.count_ones() as usize;
if SAVE_LEVELS {
let truncated = _mm_round_ps::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(a);
let frac = _mm_sub_ps(a, truncated);
let ge_half = _mm_cmpge_ps(_mm_andnot_ps(sign_mask, frac), _mm_set1_ps(0.5));
let signed_one = _mm_or_ps(_mm_set1_ps(1.0), _mm_and_ps(a, sign_mask));
let quantized =
_mm_cvttps_epi32(_mm_add_ps(truncated, _mm_and_ps(signed_one, ge_half)));
let quantized =
_mm_and_si128(quantized, _mm_and_si128(_mm_castps_si128(keep), active_i));
unsafe {
_mm_storeu_si128(levels.as_mut_ptr().add(y * width + x).cast(), quantized)
};
}
if !SAVE_LEVELS && rate_bits != 0 {
let ratev = sse_log2p1_f32(absq);
mag_acc = _mm_add_ps(mag_acc, _mm_and_ps(ratev, rate_mask));
let sv = unsafe {
_mm_loadu_si128(scan_pos.as_ptr().add(y * width + x) as *const __m128i)
};
scan_acc = _mm_max_epu32(scan_acc, _mm_and_si128(sv, _mm_castps_si128(rate_mask)));
}
}
}
(hsum(sse_acc), nzeros, hsum(mag_acc), hmax_u32(scan_acc))
}
#[inline]
pub(crate) const fn _mm_shuffle(z: u32, y: u32, x: u32, w: u32) -> i32 {
((z << 6) | (y << 4) | (x << 2) | w) as i32
}
#[inline]
#[target_feature(enable = "sse4.1")]
fn hmax_u32(v: __m128i) -> u32 {
let t1 = _mm_shuffle_epi32::<{ _mm_shuffle(1, 0, 3, 2) }>(v);
let m1 = _mm_max_epu32(v, t1);
let t2 = _mm_shuffle_epi32::<{ _mm_shuffle(2, 3, 0, 1) }>(m1);
let m2 = _mm_max_epu32(m1, t2);
_mm_cvtsi128_si32(m2) as u32
}
#[inline]
#[target_feature(enable = "sse4.1")]
pub(super) fn sse_log2p1_f32(x: __m128) -> __m128 {
let y = _mm_add_ps(x, _mm_set1_ps(1.0));
let bits = _mm_castps_si128(y);
let exp_u = _mm_srli_epi32(bits, 23);
let exp_i = _mm_sub_epi32(exp_u, _mm_set1_epi32(127));
let e = _mm_cvtepi32_ps(exp_i);
let mant_bits = _mm_or_si128(
_mm_and_si128(bits, _mm_set1_epi32(0x007f_ffff)),
_mm_set1_epi32(0x3f80_0000),
);
let m = _mm_castsi128_ps(mant_bits);
let t = _mm_sub_ps(m, _mm_set1_ps(1.0));
let c0 = _mm_set1_ps(1.4426934719085693359375);
let c1 = _mm_set1_ps(-0.721179187297821044921875);
let c2 = _mm_set1_ps(0.477900087833404541015625);
let c3 = _mm_set1_ps(-0.340080082416534423828125);
let c4 = _mm_set1_ps(0.21719777584075927734375);
let c5 = _mm_set1_ps(-9.749893844127655029296875e-2);
let c6 = _mm_set1_ps(2.096841670572757720947265625e-2);
let mut p = c6;
p = _mm_add_ps(c5, _mm_mul_ps(t, p));
p = _mm_add_ps(c4, _mm_mul_ps(t, p));
p = _mm_add_ps(c3, _mm_mul_ps(t, p));
p = _mm_add_ps(c2, _mm_mul_ps(t, p));
p = _mm_add_ps(c1, _mm_mul_ps(t, p));
p = _mm_add_ps(c0, _mm_mul_ps(t, p));
let log2_m = _mm_mul_ps(t, p);
_mm_add_ps(e, log2_m)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_sse_and_rate_sse_vs_reference() {
if !std::is_x86_feature_detected!("sse4.1") {
return;
}
crate::inflated_cost::assert_sse_and_rate_matches_reference(
sse_and_rate_sse::<false>,
false,
);
crate::inflated_cost::assert_sse_and_rate_matches_reference(sse_and_rate_sse::<true>, true);
}
#[test]
fn test_log2p1() {
if !std::is_x86_feature_detected!("sse4.1") {
return;
}
unsafe {
for i in 0..5000 {
let q1 = ((i as f32 / 5000.) + 1.).log2();
let q2 = f32::from_bits(_mm_extract_ps::<0>(sse_log2p1_f32(_mm_set1_ps(
i as f32 / 5000.,
))) as u32);
assert!((q1 - q2).abs() < 1e-5, "q1 {} q2 {}", q1, q2);
}
}
}
}
#[cfg(test)]
mod lane_count_tests {
use super::*;
#[test]
fn lane_counts_preserve_original_rate_distortion_and_coding_levels() {
if !std::is_x86_feature_detected!("sse4.1") {
return;
}
for (original, optimized) in [
(
sse_and_rate_sse_impl_reference::<false, false>
as crate::inflated_cost::SseAndQuantizeFn,
sse_and_rate_sse_impl::<false, false> as crate::inflated_cost::SseAndQuantizeFn,
),
(
sse_and_rate_sse_impl_reference::<false, true>
as crate::inflated_cost::SseAndQuantizeFn,
sse_and_rate_sse_impl::<false, true> as crate::inflated_cost::SseAndQuantizeFn,
),
(
sse_and_rate_sse_impl_reference::<true, false>
as crate::inflated_cost::SseAndQuantizeFn,
sse_and_rate_sse_impl::<true, false> as crate::inflated_cost::SseAndQuantizeFn,
),
(
sse_and_rate_sse_impl_reference::<true, true>
as crate::inflated_cost::SseAndQuantizeFn,
sse_and_rate_sse_impl::<true, true> as crate::inflated_cost::SseAndQuantizeFn,
),
] {
crate::inflated_cost::assert_coefficient_kernel_preserves_original(original, optimized);
}
}
}