use std::arch::x86_64::*;
use crate::entropy::ALPHABET_SIZE;
#[inline]
#[target_feature(enable = "avx2,fma")]
fn dirty_log2f_x8<const EXACT_DIV: bool>(d: __m256) -> __m256 {
let one = _mm256_set1_ps(1.0);
let mut ix = _mm256_castps_si256(d);
ix = _mm256_add_epi32(
ix,
_mm256_set1_epi32((0x3f80_0000u32 - 0x3f35_04f3u32) as i32),
);
let n = _mm256_sub_epi32(_mm256_srli_epi32::<23>(ix), _mm256_set1_epi32(0x7f));
ix = _mm256_add_epi32(
_mm256_and_si256(ix, _mm256_set1_epi32(0x007f_ffff)),
_mm256_set1_epi32(0x3f35_04f3),
);
let a = _mm256_castsi256_ps(ix);
let numerator = _mm256_sub_ps(a, one);
let denominator = _mm256_add_ps(a, one);
let x = if EXACT_DIV {
_mm256_div_ps(numerator, denominator)
} else {
let reciprocal0 = _mm256_rcp_ps(denominator);
let reciprocal = _mm256_mul_ps(
reciprocal0,
_mm256_fnmadd_ps(denominator, reciprocal0, _mm256_set1_ps(2.0)),
);
_mm256_mul_ps(numerator, reciprocal)
};
let x2 = _mm256_mul_ps(x, x);
let mut u = _mm256_set1_ps(0.412_198_57);
u = _mm256_fmadd_ps(u, x2, _mm256_set1_ps(0.577_078_04));
u = _mm256_fmadd_ps(u, x2, _mm256_set1_ps(0.961_796_7));
let base = _mm256_fmadd_ps(x, _mm256_set1_ps(2.885_390_1), _mm256_cvtepi32_ps(n));
_mm256_fmadd_ps(_mm256_mul_ps(x2, x), u, base)
}
#[inline]
#[target_feature(enable = "avx2")]
fn cvtepu32_ps(v: __m256i) -> __m256 {
let signed = _mm256_cvtepi32_ps(v);
let high = _mm256_cmpgt_epi32(_mm256_setzero_si256(), v);
_mm256_add_ps(
signed,
_mm256_and_ps(_mm256_castsi256_ps(high), _mm256_set1_ps(4_294_967_296.0)),
)
}
#[target_feature(enable = "avx2,fma")]
pub(crate) fn entropy_of_hist_avx2(hist: &[u64], total: u64) -> f32 {
if total == 0 {
return 0.0;
}
let t = _mm256_set1_ps(1.0 / total as f32);
let mut bits = 0.0f32;
let mut counts = hist.iter().copied().filter(|&c| c != 0);
loop {
let mut floats = [1.0f32; 8];
let mut used = 0;
for value in &mut floats {
let Some(c) = counts.next() else {
break;
};
*value = c as f32;
used += 1;
}
if used == 0 {
return bits;
}
let c = unsafe { _mm256_loadu_ps(floats.as_ptr()) };
let p = _mm256_mul_ps(c, t);
let terms = _mm256_mul_ps(c, dirty_log2f_x8::<true>(p));
let mut lanes = [0.0f32; 8];
unsafe { _mm256_storeu_ps(lanes.as_mut_ptr(), terms) };
for &value in &lanes[..used] {
bits -= value;
}
if used < floats.len() {
return bits;
}
}
}
#[target_feature(enable = "avx2,fma")]
pub(crate) fn counts_bit_cost_avx2(counts: &[u32; ALPHABET_SIZE], total_count: u32) -> f32 {
debug_assert_ne!(total_count, 0);
let log_total = _mm256_set1_ps(crate::adaptive_quant::dirty_log2f(total_count as f32));
let one = _mm256_set1_ps(1.0);
let mut cost = _mm256_setzero_ps();
for counts8 in counts.as_chunks::<8>().0 {
let count_i = unsafe { _mm256_loadu_si256(counts8.as_ptr().cast()) };
if _mm256_testz_si256(count_i, count_i) != 0 {
continue;
}
let count = cvtepu32_ps(count_i);
let positive = _mm256_max_ps(count, one);
cost = _mm256_fmadd_ps(
count,
_mm256_sub_ps(log_total, dirty_log2f_x8::<false>(positive)),
cost,
);
}
super::ac_strategy::hsum256(cost).max(0.0)
}
#[cfg(test)]
mod tests {
use super::*;
#[target_feature(enable = "avx2,fma")]
fn dense_cost(counts: &[u32; ALPHABET_SIZE], total_count: u32) -> f32 {
debug_assert_ne!(total_count, 0);
let log_total = _mm256_set1_ps(crate::adaptive_quant::dirty_log2f(total_count as f32));
let one = _mm256_set1_ps(1.0);
let mut cost = _mm256_setzero_ps();
for counts8 in counts.as_chunks::<8>().0 {
let count_i = unsafe { _mm256_loadu_si256(counts8.as_ptr().cast()) };
let count = cvtepu32_ps(count_i);
let positive = _mm256_max_ps(count, one);
cost = _mm256_fmadd_ps(
count,
_mm256_sub_ps(log_total, dirty_log2f_x8::<false>(positive)),
cost,
);
}
super::super::ac_strategy::hsum256(cost).max(0.0)
}
#[test]
fn skipped_empty_chunks_preserve_dense_cost_bits() {
if !(std::is_x86_feature_detected!("avx2") && std::is_x86_feature_detected!("fma")) {
return;
}
let mut state = 0x39e1_268bu32;
for case in 0..if cfg!(miri) { 128 } else { 8192 } {
let mut counts = [0u32; ALPHABET_SIZE];
for (i, count) in counts.iter_mut().enumerate() {
state ^= state << 13;
state ^= state >> 17;
state ^= state << 5;
if case % 5 == 0 || (case >> (i / 8 % 12)) & 1 != 0 {
*count = if state & 3 == 0 {
0
} else {
state % 32_000_000
};
}
}
counts[case % ALPHABET_SIZE] += 1;
let total = counts.iter().sum();
unsafe {
assert_eq!(
counts_bit_cost_avx2(&counts, total).to_bits(),
dense_cost(&counts, total).to_bits(),
"case {case}"
);
}
}
for count in [1, 2, 3, (1 << 24) - 1, 1 << 24, (1 << 24) + 1, u32::MAX] {
for symbol in [0, 7, 8, ALPHABET_SIZE - 1] {
let mut counts = [0; ALPHABET_SIZE];
counts[symbol] = count;
unsafe {
assert_eq!(
counts_bit_cost_avx2(&counts, count).to_bits(),
dense_cost(&counts, count).to_bits()
);
}
}
}
}
}