#[cfg(target_arch = "x86")]
use std::arch::x86::*;
#[cfg(target_arch = "x86_64")]
use std::arch::x86_64::*;
#[allow(clippy::too_many_arguments)]
#[target_feature(enable = "sse4.1")]
pub(crate) fn quantize_block_ac_sse41(
block_in: &[f32],
c: usize,
qm: &[f32],
quant: i32,
scale: f32,
qm_multiplier: f32,
distance: f32,
xsize: usize,
ysize: usize,
block_out: &mut [i32],
) {
let width = xsize * 8;
let height = ysize * 8;
let n = width * height;
debug_assert_eq!(qm.len(), n);
debug_assert!(block_in.len() >= n, "block_in too small");
debug_assert!(block_out.len() >= n, "block_out too small");
debug_assert!(width.is_multiple_of(4));
let qm = &qm[..n];
let block_in = &block_in[..n];
let block_out = &mut block_out[..n];
let thr = crate::group::quantize_ac_thresholds_scaled(c, xsize, ysize, distance, qm_multiplier);
let q_scaled = crate::group::quantize_ac_q_scaled(quant, scale, qm_multiplier);
let half = width / 2;
let qs = _mm_set1_ps(q_scaled);
let sign = _mm_set1_ps(-0.0);
let zero_i = _mm_setzero_si128();
for (y, ((qm_row, in_row), out_row)) in qm
.chunks_exact(width)
.zip(block_in.chunks_exact(width))
.zip(block_out.chunks_exact_mut(width))
.take(height)
.enumerate()
{
let yfix = if y >= height / 2 { 2 } else { 0 };
let thr_lo = thr[yfix];
let thr_hi = thr[yfix + 1];
for (x0, ((qm4, in4), out4)) in qm_row
.as_chunks::<4>()
.0
.iter()
.zip(in_row.as_chunks::<4>().0.iter())
.zip(out_row.as_chunks_mut::<4>().0.iter_mut())
.enumerate()
{
let x = x0 * 4;
let threshold = if x >= half { thr_hi } else { thr_lo };
let qmv = unsafe { _mm_loadu_ps(qm4.as_ptr()) };
let inv = unsafe { _mm_loadu_ps(in4.as_ptr()) };
let val = _mm_mul_ps(_mm_mul_ps(qmv, qs), inv);
let abs = _mm_andnot_ps(sign, val);
let keep = _mm_cmpge_ps(abs, _mm_set1_ps(threshold));
let truncated = _mm_round_ps::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(val);
let frac = _mm_sub_ps(val, truncated);
let abs_frac = _mm_andnot_ps(sign, frac);
let ge_half = _mm_cmpge_ps(abs_frac, _mm_set1_ps(0.5));
let signed_one = _mm_or_ps(_mm_set1_ps(1.0), _mm_and_ps(val, sign));
let q = _mm_cvttps_epi32(_mm_add_ps(truncated, _mm_and_ps(signed_one, ge_half)));
let q = _mm_blendv_epi8(zero_i, q, _mm_castps_si128(keep));
unsafe { _mm_storeu_si128(out4.as_mut_ptr().cast::<__m128i>(), q) };
}
}
}
#[inline]
#[target_feature(enable = "sse4.1")]
pub(crate) fn round_ties_away_i32x4(value: __m128) -> __m128i {
let sign = _mm_set1_ps(-0.0);
let truncated = _mm_round_ps::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(value);
let fraction = _mm_sub_ps(value, truncated);
let at_least_half = _mm_cmpge_ps(_mm_andnot_ps(sign, fraction), _mm_set1_ps(0.5));
let signed_one = _mm_or_ps(_mm_set1_ps(1.0), _mm_and_ps(value, sign));
let rounded = _mm_add_ps(truncated, _mm_and_ps(signed_one, at_least_half));
let quantized = _mm_cvttps_epi32(rounded);
let positive_overflow = _mm_cmpge_ps(rounded, _mm_set1_ps(2_147_483_648.0));
let quantized = _mm_blendv_epi8(
quantized,
_mm_set1_epi32(i32::MAX),
_mm_castps_si128(positive_overflow),
);
let nan = _mm_cmpunord_ps(value, value);
_mm_blendv_epi8(quantized, _mm_setzero_si128(), _mm_castps_si128(nan))
}
#[inline]
#[target_feature(enable = "sse4.1")]
fn store_i16x4(output: &mut [i16; 4], value: __m128i) {
let packed = _mm_packs_epi32(value, value);
unsafe { _mm_storel_epi64(output.as_mut_ptr().cast(), packed) };
}
#[inline]
#[target_feature(enable = "sse4.1")]
fn store_i16x8(output: &mut [[i16; 4]; 2], value0: __m128i, value1: __m128i) {
let packed = _mm_packs_epi32(value0, value1);
unsafe { _mm_storeu_si128(output.as_mut_ptr().cast(), packed) };
}
#[inline]
#[target_feature(enable = "sse4.1")]
fn quantize_dc_cfl_value_x4(
input: __m128,
y_quant: __m128i,
scale: __m128,
negative_cfl: __m128,
) -> __m128i {
let y = _mm_cvtepi32_ps(_mm_cvtepi16_epi32(y_quant));
let correction = _mm_mul_ps(y, negative_cfl);
let value = _mm_add_ps(_mm_mul_ps(input, scale), correction);
round_ties_away_i32x4(value)
}
#[target_feature(enable = "sse4.1")]
pub(crate) fn quantize_dc_sse41(input: &[f32], scale: f32, output: &mut [i16]) {
debug_assert_eq!(input.len(), output.len());
let (input4, input_tail) = input.as_chunks::<4>();
let (output4, output_tail) = output.as_chunks_mut::<4>();
let scale = _mm_set1_ps(scale);
let (input_pairs, input_single) = input4.as_chunks::<2>();
let (output_pairs, output_single) = output4.as_chunks_mut::<2>();
for ([source0, source1], target) in input_pairs.iter().zip(output_pairs) {
let value0 = unsafe { _mm_loadu_ps(source0.as_ptr()) };
let value1 = unsafe { _mm_loadu_ps(source1.as_ptr()) };
let value0 = round_ties_away_i32x4(_mm_mul_ps(value0, scale));
let value1 = round_ties_away_i32x4(_mm_mul_ps(value1, scale));
store_i16x8(target, value0, value1);
}
for (source, target) in input_single.iter().zip(output_single) {
let value = unsafe { _mm_loadu_ps(source.as_ptr()) };
store_i16x4(target, round_ties_away_i32x4(_mm_mul_ps(value, scale)));
}
if !input_tail.is_empty() {
let mut source = [0.0; 4];
source[..input_tail.len()].copy_from_slice(input_tail);
let value = unsafe { _mm_loadu_ps(source.as_ptr()) };
let mut target = [0i16; 4];
store_i16x4(&mut target, round_ties_away_i32x4(_mm_mul_ps(value, scale)));
output_tail.copy_from_slice(&target[..input_tail.len()]);
}
}
#[target_feature(enable = "sse4.1")]
pub(crate) fn quantize_dc_cfl_sse41(
input: &[f32],
y_quant: &[i16],
scale: f32,
cfl: f32,
output: &mut [i16],
) {
debug_assert_eq!(input.len(), y_quant.len());
debug_assert_eq!(input.len(), output.len());
let (input4, input_tail) = input.as_chunks::<4>();
let (y4, y_tail) = y_quant.as_chunks::<4>();
let (output4, output_tail) = output.as_chunks_mut::<4>();
let scale = _mm_set1_ps(scale);
let negative_cfl = _mm_set1_ps(-cfl);
let (input_pairs, input_single) = input4.as_chunks::<2>();
let (y_pairs, y_single) = y4.as_chunks::<2>();
let (output_pairs, output_single) = output4.as_chunks_mut::<2>();
for (([source0, source1], [y0, y1]), target) in
input_pairs.iter().zip(y_pairs).zip(output_pairs)
{
let value0 = unsafe { _mm_loadu_ps(source0.as_ptr()) };
let value1 = unsafe { _mm_loadu_ps(source1.as_ptr()) };
let y0 = unsafe { _mm_loadl_epi64(y0.as_ptr().cast()) };
let y1 = unsafe { _mm_loadl_epi64(y1.as_ptr().cast()) };
let value0 = quantize_dc_cfl_value_x4(value0, y0, scale, negative_cfl);
let value1 = quantize_dc_cfl_value_x4(value1, y1, scale, negative_cfl);
store_i16x8(target, value0, value1);
}
for ((source, y), target) in input_single.iter().zip(y_single).zip(output_single) {
let value = unsafe { _mm_loadu_ps(source.as_ptr()) };
let y = unsafe { _mm_loadl_epi64(y.as_ptr().cast()) };
store_i16x4(
target,
quantize_dc_cfl_value_x4(value, y, scale, negative_cfl),
);
}
if !input_tail.is_empty() {
let mut source = [0.0; 4];
source[..input_tail.len()].copy_from_slice(input_tail);
let mut y = [0i16; 4];
y[..y_tail.len()].copy_from_slice(y_tail);
let value = unsafe { _mm_loadu_ps(source.as_ptr()) };
let y = unsafe { _mm_loadl_epi64(y.as_ptr().cast()) };
let mut target = [0i16; 4];
store_i16x4(
&mut target,
quantize_dc_cfl_value_x4(value, y, scale, negative_cfl),
);
output_tail.copy_from_slice(&target[..input_tail.len()]);
}
}
#[target_feature(enable = "sse4.1")]
pub(crate) fn quantize_dc_i32_sse41(input: &[f32], scale: f32, output: &mut [i32]) {
debug_assert_eq!(input.len(), output.len());
let (input_chunks, input_tail) = input.as_chunks::<4>();
let (output_chunks, output_tail) = output.as_chunks_mut::<4>();
let scale = _mm_set1_ps(scale);
let (input_pairs, input_single) = input_chunks.as_chunks::<2>();
let (output_pairs, output_single) = output_chunks.as_chunks_mut::<2>();
for ([source0, source1], [target0, target1]) in input_pairs.iter().zip(output_pairs) {
let value0 = unsafe { _mm_loadu_ps(source0.as_ptr()) };
let value1 = unsafe { _mm_loadu_ps(source1.as_ptr()) };
let value0 = round_ties_away_i32x4(_mm_mul_ps(value0, scale));
let value1 = round_ties_away_i32x4(_mm_mul_ps(value1, scale));
unsafe { _mm_storeu_si128(target0.as_mut_ptr().cast(), value0) };
unsafe { _mm_storeu_si128(target1.as_mut_ptr().cast(), value1) };
}
for (source, target) in input_single.iter().zip(output_single) {
let value = unsafe { _mm_loadu_ps(source.as_ptr()) };
let value = round_ties_away_i32x4(_mm_mul_ps(value, scale));
unsafe { _mm_storeu_si128(target.as_mut_ptr().cast(), value) };
}
if !input_tail.is_empty() {
let mut source = [0.0; 4];
source[..input_tail.len()].copy_from_slice(input_tail);
let value = unsafe { _mm_loadu_ps(source.as_ptr()) };
let value = round_ties_away_i32x4(_mm_mul_ps(value, scale));
let mut target = [0i32; 4];
unsafe { _mm_storeu_si128(target.as_mut_ptr().cast(), value) };
output_tail.copy_from_slice(&target[..input_tail.len()]);
}
}
#[inline]
#[target_feature(enable = "sse4.1")]
#[cfg_attr(target_feature = "fma", target_feature(enable = "fma"))]
fn quantize_dc_cfl_i32_value_x4(
input: __m128,
y_quant: __m128i,
scale: __m128,
negative_cfl: __m128,
) -> __m128i {
let correction = _mm_mul_ps(_mm_cvtepi32_ps(y_quant), negative_cfl);
#[cfg(target_feature = "fma")]
let value = _mm_fmadd_ps(input, scale, correction);
#[cfg(not(target_feature = "fma"))]
let value = _mm_add_ps(_mm_mul_ps(input, scale), correction);
round_ties_away_i32x4(value)
}
#[target_feature(enable = "sse4.1")]
#[cfg_attr(target_feature = "fma", target_feature(enable = "fma"))]
pub(crate) fn quantize_dc_cfl_i32_sse41(
input: &[f32],
y_quant: &[i32],
scale: f32,
cfl: f32,
output: &mut [i32],
) {
debug_assert_eq!(input.len(), y_quant.len());
debug_assert_eq!(input.len(), output.len());
let (input_chunks, input_tail) = input.as_chunks::<4>();
let (y_chunks, y_tail) = y_quant.as_chunks::<4>();
let (output_chunks, output_tail) = output.as_chunks_mut::<4>();
let scale = _mm_set1_ps(scale);
let negative_cfl = _mm_set1_ps(-cfl);
let (input_pairs, input_single) = input_chunks.as_chunks::<2>();
let (y_pairs, y_single) = y_chunks.as_chunks::<2>();
let (output_pairs, output_single) = output_chunks.as_chunks_mut::<2>();
for (([source0, source1], [y0, y1]), [target0, target1]) in
input_pairs.iter().zip(y_pairs).zip(output_pairs)
{
let value0 = unsafe { _mm_loadu_ps(source0.as_ptr()) };
let value1 = unsafe { _mm_loadu_ps(source1.as_ptr()) };
let y0 = unsafe { _mm_loadu_si128(y0.as_ptr().cast()) };
let y1 = unsafe { _mm_loadu_si128(y1.as_ptr().cast()) };
let value0 = quantize_dc_cfl_i32_value_x4(value0, y0, scale, negative_cfl);
let value1 = quantize_dc_cfl_i32_value_x4(value1, y1, scale, negative_cfl);
unsafe { _mm_storeu_si128(target0.as_mut_ptr().cast(), value0) };
unsafe { _mm_storeu_si128(target1.as_mut_ptr().cast(), value1) };
}
for ((source, y), target) in input_single.iter().zip(y_single).zip(output_single) {
let value = unsafe { _mm_loadu_ps(source.as_ptr()) };
let y = unsafe { _mm_loadu_si128(y.as_ptr().cast()) };
let value = quantize_dc_cfl_i32_value_x4(value, y, scale, negative_cfl);
unsafe { _mm_storeu_si128(target.as_mut_ptr().cast(), value) };
}
if !input_tail.is_empty() {
let mut source = [0.0; 4];
source[..input_tail.len()].copy_from_slice(input_tail);
let mut y = [0i32; 4];
y[..y_tail.len()].copy_from_slice(y_tail);
let value = unsafe { _mm_loadu_ps(source.as_ptr()) };
let y = unsafe { _mm_loadu_si128(y.as_ptr().cast()) };
let value = quantize_dc_cfl_i32_value_x4(value, y, scale, negative_cfl);
let mut target = [0i32; 4];
unsafe { _mm_storeu_si128(target.as_mut_ptr().cast(), value) };
output_tail.copy_from_slice(&target[..input_tail.len()]);
}
}
#[cfg(test)]
mod tests {
use crate::group::quantize_block_ac_scalar;
#[test]
fn quantize_block_ac_sse41_matches_scalar() {
static EDGES: [f32; 12] = [
0.5, 1.5, 2.5, 3.5, -0.5, -1.5, -2.5, -3.5, 0.75, -0.75, 1.25, -1.25,
];
for &(xsize, ysize) in &[(1usize, 1usize), (2, 2), (4, 4)] {
let n = xsize * 8 * ysize * 8;
for c in 0..3 {
for &distance in &[0.5f32, 2.5] {
let qm: Vec<f32> = vec![0.5; n];
let block: Vec<f32> = (0..n).map(|i| EDGES[i % EDGES.len()] / 0.5).collect();
let mut want = vec![0i32; n];
let mut got = vec![0i32; n];
quantize_block_ac_scalar(
&block, c, &qm, 1, 1.0, 1.0, distance, xsize, ysize, &mut want,
);
unsafe {
super::quantize_block_ac_sse41(
&block, c, &qm, 1, 1.0, 1.0, distance, xsize, ysize, &mut got,
);
}
for i in 0..n {
assert_eq!(
got[i], want[i],
"mismatch at {i} ({xsize}x{ysize}, c={c}, d={distance}): \
scalar {} sse {}, input {}",
want[i], got[i], block[i]
);
}
}
}
}
}
}