use std::arch::x86_64::*;
#[inline]
#[target_feature(enable = "avx")]
pub(crate) fn hsum256(v: __m256) -> f32 {
let lo = _mm256_castps256_ps128(v);
let hi = _mm256_extractf128_ps::<1>(v);
let s = _mm_add_ps(lo, hi);
let shuf = _mm_movehdup_ps(s);
let sums = _mm_add_ps(s, shuf);
let shuf2 = _mm_movehl_ps(shuf, sums);
let sums = _mm_add_ss(sums, shuf2);
_mm_cvtss_f32(sums)
}
#[inline]
#[target_feature(enable = "avx2,fma")]
pub(crate) fn avx_dequantized_level_f32(q: __m256, bias: crate::group::DequantBias) -> __m256 {
let sign_mask = _mm256_set1_ps(-0.0);
let absq = _mm256_andnot_ps(sign_mask, q);
let big = _mm256_sub_ps(q, _mm256_div_ps(_mm256_set1_ps(bias.numerator), q));
let sign = _mm256_and_ps(sign_mask, q);
let one = _mm256_or_ps(sign, _mm256_set1_ps(bias.one));
let use_big = _mm256_cmp_ps::<_CMP_GE_OQ>(absq, _mm256_set1_ps(1.125));
let dq = _mm256_blendv_ps(one, big, use_big);
let nz = _mm256_cmp_ps::<_CMP_GT_OQ>(absq, _mm256_setzero_ps());
_mm256_and_ps(dq, nz)
}
#[allow(clippy::too_many_arguments)]
#[target_feature(enable = "avx2,fma")]
pub(crate) fn sse_and_rate_avx2<const BIASED: bool>(
coeff: &[f32],
inv_matrix: &[f32],
q_scaled: f32,
bias: crate::group::DequantBias,
width: usize,
height: usize,
half: usize,
cx: usize,
cy: usize,
_rate_log2_lut: &crate::inflated_cost::RateLog2Lut,
thr: &[f32; 4],
scan_pos: &[u32],
) -> (f32, usize, f32, u32) {
sse_and_rate_avx2_impl::<BIASED, false>(
coeff,
inv_matrix,
q_scaled,
bias,
width,
height,
half,
cx,
cy,
_rate_log2_lut,
thr,
scan_pos,
&mut [],
)
}
#[allow(clippy::too_many_arguments)]
#[target_feature(enable = "avx2,fma")]
pub(crate) fn sse_and_rate_avx2_impl<const BIASED: bool, const SAVE_LEVELS: bool>(
coeff: &[f32],
inv_matrix: &[f32],
q_scaled: f32,
bias: crate::group::DequantBias,
width: usize,
height: usize,
half: usize,
cx: usize,
cy: usize,
_rate_log2_lut: &crate::inflated_cost::RateLog2Lut,
thr: &[f32; 4],
scan_pos: &[u32],
levels: &mut [i32],
) -> (f32, usize, f32, u32) {
let n = width * height;
assert!(coeff.len() >= n && inv_matrix.len() >= n);
assert!(if SAVE_LEVELS {
levels.len() >= n
} else {
scan_pos.len() >= n
});
debug_assert!(width.is_multiple_of(8));
let qs = _mm256_set1_ps(q_scaled);
let sign = _mm256_set1_ps(-0.0);
let zero = _mm256_setzero_ps();
let mut sse_acc = _mm256_setzero_ps();
let mut mag_acc = _mm256_setzero_ps();
let mut nonzero_acc = _mm256_setzero_si256();
let mut scan_acc = _mm256_setzero_si256();
let lane_ids = _mm256_setr_epi32(0, 1, 2, 3, 4, 5, 6, 7);
let izero = _mm256_setzero_si256();
let all_active_i = _mm256_cmpeq_epi32(izero, izero);
const RND: i32 = _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC;
for (y, (coeffs, inv_matrix)) in coeff
.chunks_exact(width)
.zip(inv_matrix.chunks_exact(width))
.take(height)
.enumerate()
{
let yfix = if y >= height / 2 { 2 } else { 0 };
let thr_lo = thr[yfix];
let thr_hi = thr[yfix + 1];
for (x0, (coeff8, inv8)) in coeffs
.as_chunks::<8>()
.0
.iter()
.zip(inv_matrix.as_chunks::<8>().0.iter())
.enumerate()
{
let x = x0 * 8;
let thr_v = if x + 8 <= half {
_mm256_set1_ps(thr_lo)
} else if x >= half {
_mm256_set1_ps(thr_hi)
} else {
let lane_x = _mm256_add_epi32(_mm256_set1_epi32(x as i32), lane_ids);
let hi_mask_i = _mm256_cmpgt_epi32(lane_x, _mm256_set1_epi32(half as i32 - 1));
let hi_mask = _mm256_castsi256_ps(hi_mask_i);
_mm256_blendv_ps(_mm256_set1_ps(thr_lo), _mm256_set1_ps(thr_hi), hi_mask)
};
let cv = unsafe { _mm256_loadu_ps(coeff8.as_ptr()) };
let mv = unsafe { _mm256_loadu_ps(inv8.as_ptr()) };
let a = _mm256_mul_ps(_mm256_mul_ps(mv, qs), cv);
let absa = _mm256_andnot_ps(sign, a);
let keep = _mm256_cmp_ps::<_CMP_GE_OQ>(absa, thr_v);
let rounded = _mm256_round_ps::<RND>(a);
let q = _mm256_and_ps(rounded, keep);
let active_i = if y < cy && x < cx {
let lane_x = _mm256_add_epi32(_mm256_set1_epi32(x as i32), lane_ids);
let cxv = _mm256_set1_epi32(cx as i32);
let llf = _mm256_cmpgt_epi32(cxv, lane_x);
_mm256_andnot_si256(llf, all_active_i)
} else {
all_active_i
};
let active = _mm256_castsi256_ps(active_i);
let absq = _mm256_andnot_ps(sign, q);
let nz = _mm256_cmp_ps::<_CMP_GT_OQ>(absq, zero);
let rate_mask = _mm256_and_ps(nz, active);
nonzero_acc = _mm256_sub_epi32(nonzero_acc, _mm256_castps_si256(rate_mask));
let d = if _mm256_movemask_ps(rate_mask) != 0 {
let tables = SmallLevelTables::new(&bias);
let index = SmallLevelIndex::new(absq);
let mut dq = if BIASED {
tables.dequantized(index, q)
} else {
q
};
let mut lg = tables.log2p1(index);
if _mm256_movemask_ps(_mm256_castsi256_ps(index.big)) != 0 {
let big = _mm256_castsi256_ps(index.big);
if BIASED {
dq = _mm256_blendv_ps(dq, avx_dequantized_level_f32(q, bias), big);
}
if !SAVE_LEVELS {
lg = _mm256_blendv_ps(lg, avx2_log2p1_f32(absq), big);
}
}
if !SAVE_LEVELS {
mag_acc = _mm256_add_ps(mag_acc, _mm256_and_ps(lg, rate_mask));
let sv =
unsafe { _mm256_loadu_si256(scan_pos.as_ptr().add(y * width + x).cast()) };
scan_acc = _mm256_max_epu32(
scan_acc,
_mm256_and_si256(sv, _mm256_castps_si256(rate_mask)),
);
}
_mm256_sub_ps(a, dq)
} else if BIASED {
a
} else {
_mm256_sub_ps(a, q)
};
let d2 = _mm256_mul_ps(d, d);
sse_acc = _mm256_add_ps(sse_acc, _mm256_and_ps(d2, active));
if SAVE_LEVELS {
let truncated = _mm256_round_ps::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(a);
let frac = _mm256_sub_ps(a, truncated);
let ge_half =
_mm256_cmp_ps::<_CMP_GE_OQ>(_mm256_andnot_ps(sign, frac), _mm256_set1_ps(0.5));
let signed_one = _mm256_or_ps(_mm256_set1_ps(1.0), _mm256_and_ps(a, sign));
let quantized = _mm256_cvttps_epi32(_mm256_add_ps(
truncated,
_mm256_and_ps(signed_one, ge_half),
));
let quantized = _mm256_and_si256(
quantized,
_mm256_and_si256(_mm256_castps_si256(keep), active_i),
);
unsafe {
_mm256_storeu_si256(levels.as_mut_ptr().add(y * width + x).cast(), quantized)
};
}
}
}
let v_max = _mm_max_epu32(
_mm256_castsi256_si128(scan_acc),
_mm256_extracti128_si256::<1>(scan_acc),
);
let max_scan = hmax_u32(v_max);
let nonzero = _mm_add_epi32(
_mm256_castsi256_si128(nonzero_acc),
_mm256_extracti128_si256::<1>(nonzero_acc),
);
let nonzero = _mm_hadd_epi32(nonzero, nonzero);
let nonzero = _mm_hadd_epi32(nonzero, nonzero);
let nzeros = _mm_cvtsi128_si32(nonzero) as usize;
(hsum256(sse_acc), nzeros, hsum256(mag_acc), max_scan)
}
static LOG2P1_SMALL: std::sync::LazyLock<[f32; crate::group::SMALL_LEVELS]> =
std::sync::LazyLock::new(|| {
std::array::from_fn(|k| unsafe {
_mm_cvtss_f32(_mm256_castps256_ps128(avx2_log2p1_f32(_mm256_set1_ps(
k as f32,
))))
})
});
#[derive(Clone, Copy)]
pub(crate) struct SmallLevelIndex {
k: __m256i,
pub(crate) big: __m256i,
upper: __m256,
}
impl SmallLevelIndex {
#[inline]
#[target_feature(enable = "avx2,fma")]
pub(crate) fn new(absq: __m256) -> Self {
const _: () = assert!(crate::group::SMALL_LEVELS == 16);
let sixteen = _mm256_set1_epi32(16);
let k = _mm256_min_epu32(_mm256_cvttps_epi32(absq), sixteen);
Self {
k,
big: _mm256_cmpeq_epi32(k, sixteen),
upper: _mm256_castsi256_ps(_mm256_cmpgt_epi32(k, _mm256_set1_epi32(7))),
}
}
}
#[derive(Clone, Copy)]
pub(crate) struct SmallLevelTables {
dequant: [__m256; 2],
log2p1: [__m256; 2],
}
impl SmallLevelTables {
#[inline]
#[target_feature(enable = "avx2,fma")]
pub(crate) fn new(bias: &crate::group::DequantBias) -> Self {
let halves = |table: &[f32; crate::group::SMALL_LEVELS]| unsafe {
[
_mm256_loadu_ps(table.as_ptr()),
_mm256_loadu_ps(table.as_ptr().add(8)),
]
};
Self {
dequant: halves(&bias.small_levels),
log2p1: halves(&LOG2P1_SMALL),
}
}
#[inline]
#[target_feature(enable = "avx2,fma")]
fn gather(table: [__m256; 2], index: SmallLevelIndex) -> __m256 {
_mm256_blendv_ps(
_mm256_permutevar8x32_ps(table[0], index.k),
_mm256_permutevar8x32_ps(table[1], index.k),
index.upper,
)
}
#[inline]
#[target_feature(enable = "avx2,fma")]
pub(crate) fn dequantized(self, index: SmallLevelIndex, q: __m256) -> __m256 {
let magnitude = Self::gather(self.dequant, index);
_mm256_or_ps(magnitude, _mm256_and_ps(q, _mm256_set1_ps(-0.0)))
}
#[inline]
#[target_feature(enable = "avx2,fma")]
pub(crate) fn log2p1(self, index: SmallLevelIndex) -> __m256 {
Self::gather(self.log2p1, index)
}
}
#[cfg(test)]
#[allow(clippy::too_many_arguments)]
#[target_feature(enable = "avx2,fma")]
pub(crate) fn sse_and_rate_avx2_impl_reference<const BIASED: bool, const SAVE_LEVELS: bool>(
coeff: &[f32],
inv_matrix: &[f32],
q_scaled: f32,
bias: crate::group::DequantBias,
width: usize,
height: usize,
half: usize,
cx: usize,
cy: usize,
_rate_log2_lut: &crate::inflated_cost::RateLog2Lut,
thr: &[f32; 4],
scan_pos: &[u32],
levels: &mut [i32],
) -> (f32, usize, f32, u32) {
let n = width * height;
assert!(coeff.len() >= n && inv_matrix.len() >= n);
assert!(if SAVE_LEVELS {
levels.len() >= n
} else {
scan_pos.len() >= n
});
debug_assert!(width.is_multiple_of(8));
let qs = _mm256_set1_ps(q_scaled);
let sign = _mm256_set1_ps(-0.0);
let zero = _mm256_setzero_ps();
let mut sse_acc = _mm256_setzero_ps();
let mut mag_acc = _mm256_setzero_ps();
let mut nzeros = 0usize;
let mut scan_acc = _mm256_setzero_si256();
let lane_ids = _mm256_setr_epi32(0, 1, 2, 3, 4, 5, 6, 7);
let izero = _mm256_setzero_si256();
let all_active_i = _mm256_cmpeq_epi32(izero, izero);
const RND: i32 = _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC;
for (y, (coeffs, inv_matrix)) in coeff
.chunks_exact(width)
.zip(inv_matrix.chunks_exact(width))
.take(height)
.enumerate()
{
let yfix = if y >= height / 2 { 2 } else { 0 };
let thr_lo = thr[yfix];
let thr_hi = thr[yfix + 1];
for (x0, (coeff8, inv8)) in coeffs
.as_chunks::<8>()
.0
.iter()
.zip(inv_matrix.as_chunks::<8>().0.iter())
.enumerate()
{
let x = x0 * 8;
let thr_v = if x + 8 <= half {
_mm256_set1_ps(thr_lo)
} else if x >= half {
_mm256_set1_ps(thr_hi)
} else {
let lane_x = _mm256_add_epi32(_mm256_set1_epi32(x as i32), lane_ids);
let hi_mask_i = _mm256_cmpgt_epi32(lane_x, _mm256_set1_epi32(half as i32 - 1));
let hi_mask = _mm256_castsi256_ps(hi_mask_i);
_mm256_blendv_ps(_mm256_set1_ps(thr_lo), _mm256_set1_ps(thr_hi), hi_mask)
};
let cv = unsafe { _mm256_loadu_ps(coeff8.as_ptr()) };
let mv = unsafe { _mm256_loadu_ps(inv8.as_ptr()) };
let a = _mm256_mul_ps(_mm256_mul_ps(mv, qs), cv);
let absa = _mm256_andnot_ps(sign, a);
let keep = _mm256_cmp_ps::<_CMP_GE_OQ>(absa, thr_v);
let rounded = _mm256_round_ps::<RND>(a);
let q = _mm256_and_ps(rounded, keep);
let d = if BIASED {
_mm256_sub_ps(a, avx_dequantized_level_f32(q, bias))
} else {
_mm256_sub_ps(a, q)
};
let d2 = _mm256_mul_ps(d, d);
let active_i = if y < cy && x < cx {
let lane_x = _mm256_add_epi32(_mm256_set1_epi32(x as i32), lane_ids);
let cxv = _mm256_set1_epi32(cx as i32);
let llf = _mm256_cmpgt_epi32(cxv, lane_x);
_mm256_andnot_si256(llf, all_active_i)
} else {
all_active_i
};
let active = _mm256_castsi256_ps(active_i);
let d2 = _mm256_and_ps(d2, active);
sse_acc = _mm256_add_ps(sse_acc, d2);
let absq = _mm256_andnot_ps(sign, q);
let nz = _mm256_cmp_ps::<_CMP_GT_OQ>(absq, zero);
let rate_mask = _mm256_and_ps(nz, active);
let rate_bits = _mm256_movemask_ps(rate_mask);
nzeros += rate_bits.count_ones() as usize;
if SAVE_LEVELS {
let truncated = _mm256_round_ps::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(a);
let frac = _mm256_sub_ps(a, truncated);
let ge_half =
_mm256_cmp_ps::<_CMP_GE_OQ>(_mm256_andnot_ps(sign, frac), _mm256_set1_ps(0.5));
let signed_one = _mm256_or_ps(_mm256_set1_ps(1.0), _mm256_and_ps(a, sign));
let quantized = _mm256_cvttps_epi32(_mm256_add_ps(
truncated,
_mm256_and_ps(signed_one, ge_half),
));
let quantized = _mm256_and_si256(
quantized,
_mm256_and_si256(_mm256_castps_si256(keep), active_i),
);
unsafe {
_mm256_storeu_si256(levels.as_mut_ptr().add(y * width + x).cast(), quantized)
};
}
if !SAVE_LEVELS && rate_bits != 0 {
let ratev = avx2_log2p1_f32(absq);
mag_acc = _mm256_add_ps(mag_acc, _mm256_and_ps(ratev, rate_mask));
let sv = unsafe { _mm256_loadu_si256(scan_pos.as_ptr().add(y * width + x).cast()) };
scan_acc = _mm256_max_epu32(
scan_acc,
_mm256_and_si256(sv, _mm256_castps_si256(rate_mask)),
);
}
}
}
let v_max = _mm_max_epu32(
_mm256_castsi256_si128(scan_acc),
_mm256_extracti128_si256::<1>(scan_acc),
);
let max_scan = hmax_u32(v_max);
(hsum256(sse_acc), nzeros, hsum256(mag_acc), max_scan)
}
#[inline]
pub(crate) const fn _mm_shuffle(z: u32, y: u32, x: u32, w: u32) -> i32 {
((z << 6) | (y << 4) | (x << 2) | w) as i32
}
#[inline]
#[target_feature(enable = "avx2")]
pub(crate) fn hmax_u32(v: __m128i) -> u32 {
let t1 = _mm_shuffle_epi32::<{ _mm_shuffle(1, 0, 3, 2) }>(v);
let m1 = _mm_max_epu32(v, t1);
let t2 = _mm_shuffle_epi32::<{ _mm_shuffle(2, 3, 0, 1) }>(m1);
let m2 = _mm_max_epu32(m1, t2);
_mm_cvtsi128_si32(m2) as u32
}
#[inline]
#[target_feature(enable = "avx2,fma")]
pub(crate) fn avx2_log2p1_f32(x: __m256) -> __m256 {
let y = _mm256_add_ps(x, _mm256_set1_ps(1.0));
let bits = _mm256_castps_si256(y);
let exp_u = _mm256_srli_epi32::<23>(bits);
let exp_i = _mm256_sub_epi32(exp_u, _mm256_set1_epi32(127));
let e = _mm256_cvtepi32_ps(exp_i);
let mant_bits = _mm256_or_si256(
_mm256_and_si256(bits, _mm256_set1_epi32(0x007f_ffff)),
_mm256_set1_epi32(0x3f80_0000),
);
let m = _mm256_castsi256_ps(mant_bits);
let t = _mm256_sub_ps(m, _mm256_set1_ps(1.0));
let c0 = _mm256_set1_ps(1.4426934719085693359375);
let c1 = _mm256_set1_ps(-0.721179187297821044921875);
let c2 = _mm256_set1_ps(0.477900087833404541015625);
let c3 = _mm256_set1_ps(-0.340080082416534423828125);
let c4 = _mm256_set1_ps(0.21719777584075927734375);
let c5 = _mm256_set1_ps(-9.749893844127655029296875e-2);
let c6 = _mm256_set1_ps(2.096841670572757720947265625e-2);
let mut p = c6;
p = _mm256_fmadd_ps(t, p, c5);
p = _mm256_fmadd_ps(t, p, c4);
p = _mm256_fmadd_ps(t, p, c3);
p = _mm256_fmadd_ps(t, p, c2);
p = _mm256_fmadd_ps(t, p, c1);
p = _mm256_fmadd_ps(t, p, c0);
_mm256_fmadd_ps(t, p, e)
}
#[cfg(test)]
mod tests {
use super::*;
#[allow(clippy::too_many_arguments)]
fn reference(
coeff: &[f32],
inv: &[f32],
qs: f32,
bias: crate::group::DequantBias,
w: usize,
h: usize,
half: usize,
cx: usize,
cy: usize,
thr: &[f32; 4],
scan_pos: &[u32],
biased: bool,
) -> (f32, usize, f32, u32) {
let (mut sse, mut nz, mut mb) = (0.0f32, 0usize, 0.0f32);
let mut max_scan = 0u32;
for y in 0..h {
let yfix = if y >= h / 2 { 2 } else { 0 };
let (lo, hi) = (thr[yfix], thr[yfix + 1]);
for x in 0..w {
if x < cx && y < cy {
continue;
}
let idx = y * w + x;
let t = if x >= half { hi } else { lo };
let a = (inv[idx] * qs) * coeff[idx];
let q = if a.abs() >= t {
a.round_ties_even()
} else {
0.0
};
let d = if biased { a - bias.level_f32(q) } else { a - q };
sse += d * d;
if q != 0.0 {
nz += 1;
mb += crate::inflated_cost::rate_log2_with_lut(
crate::inflated_cost::rate_log2_lut(),
q.abs(),
);
max_scan = max_scan.max(scan_pos[idx]);
}
}
}
(sse, nz, mb, max_scan)
}
#[test]
fn test_sse_and_rate_avx2_vs_reference() {
if !std::is_x86_feature_detected!("avx") || !std::is_x86_feature_detected!("fma") {
return;
}
let mut st: u64 = 0x5e5e_a11d_0f00_d00d;
let mut rnd = || {
st = st
.wrapping_mul(6364136223846793005)
.wrapping_add(1442695040888963407);
((st >> 33) as u32 as f32) / (u32::MAX as f32)
};
for &(w, h, half, cx, cy) in &[
(8usize, 8usize, 4usize, 1usize, 1usize),
(16, 16, 8, 2, 2),
(32, 32, 16, 4, 4),
(16, 8, 8, 2, 1),
(8, 16, 4, 1, 2),
] {
for case in 0..200 {
let n = w * h;
let coeff_scale = if case % 2 == 0 { 0.2 } else { 200.0 };
let coeff: Vec<f32> = (0..n).map(|_| (rnd() - 0.5) * coeff_scale).collect();
let inv: Vec<f32> = (0..n).map(|_| 0.001 + rnd() * 0.5).collect();
let qs = 0.5 + rnd() * 3.0;
let bias = crate::group::DequantBias::DEFAULT[case % 3];
let thr = [rnd() * 0.6, rnd() * 0.6, rnd() * 0.6, rnd() * 0.6];
let scan_pos = crate::coeff_order::scan_pos_lut(w, h);
for biased in [false, true] {
let r = reference(
&coeff, &inv, qs, bias, w, h, half, cx, cy, &thr, scan_pos, biased,
);
let kernel = if biased {
sse_and_rate_avx2::<true>
} else {
sse_and_rate_avx2::<false>
};
let a = unsafe {
kernel(
&coeff,
&inv,
qs,
bias,
w,
h,
half,
cx,
cy,
crate::inflated_cost::rate_log2_lut(),
&thr,
scan_pos,
)
};
assert_eq!(a.1, r.1, "nzeros mismatch {w}x{h}");
assert_eq!(a.3, r.3, "max-scan mismatch {w}x{h}");
let mag_rel = if r.2.abs() > 1e-3 {
((a.2 - r.2) / r.2).abs()
} else {
(a.2 - r.2).abs()
};
assert!(
mag_rel < 1e-4,
"mag_bits rel diff {mag_rel} too large {w}x{h} biased={biased}"
);
let rel = if r.0.abs() > 1e-3 {
((a.0 - r.0) / r.0).abs()
} else {
(a.0 - r.0).abs()
};
assert!(
rel < 1e-4,
"sse rel diff {rel} too large {w}x{h} biased={biased}"
);
}
}
}
}
#[test]
fn test_log2p1() {
if !std::is_x86_feature_detected!("avx") && !std::is_x86_feature_detected!("fma") {
return;
}
unsafe {
for i in 0..5000 {
let q1 = ((i as f32 / 5000.) + 1.).log2();
let r = avx2_log2p1_f32(_mm256_set1_ps(i as f32 / 5000.));
let q2 = f32::from_bits(_mm_extract_ps::<0>(_mm256_castps256_ps128(r)) as u32);
assert!((q1 - q2).abs() < 1e-5, "q1 {} q2 {}", q1, q2);
}
}
}
}
#[cfg(test)]
mod lane_count_tests {
use super::*;
#[test]
fn lane_counts_preserve_original_rate_distortion_and_coding_levels() {
if !std::is_x86_feature_detected!("avx2") || !std::is_x86_feature_detected!("fma") {
return;
}
for (original, optimized) in [
(
sse_and_rate_avx2_impl_reference::<false, false>
as crate::inflated_cost::SseAndQuantizeFn,
sse_and_rate_avx2_impl::<false, false> as crate::inflated_cost::SseAndQuantizeFn,
),
(
sse_and_rate_avx2_impl_reference::<false, true>
as crate::inflated_cost::SseAndQuantizeFn,
sse_and_rate_avx2_impl::<false, true> as crate::inflated_cost::SseAndQuantizeFn,
),
(
sse_and_rate_avx2_impl_reference::<true, false>
as crate::inflated_cost::SseAndQuantizeFn,
sse_and_rate_avx2_impl::<true, false> as crate::inflated_cost::SseAndQuantizeFn,
),
(
sse_and_rate_avx2_impl_reference::<true, true>
as crate::inflated_cost::SseAndQuantizeFn,
sse_and_rate_avx2_impl::<true, true> as crate::inflated_cost::SseAndQuantizeFn,
),
] {
crate::inflated_cost::assert_coefficient_kernel_preserves_original(original, optimized);
}
}
}