#![cfg(target_arch = "x86_64")]
use core::convert::TryInto;
use crate::msac::{
MSAC_MIN_PROB, MSAC_RATE, MsacReader, MsacState, msac_load_be64_unchecked, msac_refill_eob,
unary_bypass_kernel,
};
use core::arch::x86_64::*;
pub(crate) struct MsacContextSse<'a, const UPDATE_CDF: bool> {
pub(crate) buf_pos: usize,
pub(crate) buf: &'a [u8],
pub(crate) dif: u64,
pub(crate) rng: u32,
pub(crate) cnt: i32,
#[cfg(not(feature = "adaptive_cdf"))]
pub(crate) update_cdf: bool,
}
impl<'a, const UPDATE_CDF: bool> MsacContextSse<'a, UPDATE_CDF> {
#[inline(always)]
pub(crate) fn new(data: &'a [u8]) -> Self {
Self::new_with_update_cdf(data, true)
}
#[inline(always)]
pub(crate) fn new_with_update_cdf(data: &'a [u8], update_cdf: bool) -> Self {
#[cfg(feature = "adaptive_cdf")]
let _ = update_cdf;
let mut s = Self {
buf_pos: 0,
buf: data,
dif: !0u64 >> 1,
rng: 0x8000,
cnt: -15,
#[cfg(not(feature = "adaptive_cdf"))]
update_cdf,
};
s.ctx_refill();
s
}
#[inline(always)]
pub(crate) fn save(&self) -> MsacState {
MsacState {
buf_pos: self.buf_pos,
dif: self.dif,
rng: self.rng,
cnt: self.cnt,
#[cfg(not(feature = "adaptive_cdf"))]
update_cdf: self.update_cdf,
}
}
#[inline(always)]
pub(crate) fn resume(data: &'a [u8], st: MsacState) -> Self {
Self {
buf_pos: st.buf_pos,
buf: data,
dif: st.dif,
rng: st.rng,
cnt: st.cnt,
#[cfg(not(feature = "adaptive_cdf"))]
update_cdf: st.update_cdf,
}
}
#[inline(always)]
fn should_update_cdf(&self) -> bool {
#[cfg(feature = "adaptive_cdf")]
{
UPDATE_CDF
}
#[cfg(not(feature = "adaptive_cdf"))]
{
self.update_cdf
}
}
#[inline(always)]
pub(crate) fn ctx_refill(&mut self) {
let start = self.buf_pos;
let c = 40 - self.cnt;
debug_assert!(c >= 0);
let c = c as u32;
let n = (c as usize >> 3) + 1;
if start + 8 <= self.buf.len() {
let val = unsafe { msac_load_be64_unchecked(self.buf, start) };
let refill = (val >> (56 - c)) & (u64::MAX << (c & 7));
self.dif ^= refill;
self.buf_pos = start + n;
self.cnt += (n as i32) * 8;
} else {
let (dif, buf_pos, cnt_inc) = msac_refill_eob(self.buf, start, c, self.dif);
self.dif = dif;
self.buf_pos = buf_pos;
self.cnt += cnt_inc;
}
}
#[inline(always)]
pub(crate) fn ctx_norm(&mut self, dif: u64, rng: u32) {
debug_assert!(rng <= 65535 && rng > 0);
let d = rng.leading_zeros() ^ 16;
let cnt = self.cnt;
self.dif = ((dif + 1) << d) - 1;
self.rng = rng << d;
self.cnt = cnt - d as i32;
if (cnt as u32) < d {
self.ctx_refill();
}
}
#[inline(always)]
pub(crate) fn decode_hr_bypass(&mut self, cmax: u32, m: u32) -> i32 {
if (self.cnt as u32) < 40 {
self.ctx_refill();
}
match crate::msac::hr_bypass_kernel(self.dif, self.rng, self.cnt as u32, cmax, m) {
crate::msac::HrDecode::Commit { val, bits, dif } => {
self.dif = dif;
self.cnt -= bits as i32;
val
}
crate::msac::HrDecode::Fallback => {
let q = self.decode_unary_bypass(cmax);
debug_assert_eq!(q, cmax);
let length = self.decode_unary_bypass(21) + m + 1;
let golomb = (1u32 << length) + self.decode_bools_bypass(length) - (1 << (m + 1));
(golomb + (cmax << m)) as i32
}
}
}
#[inline(always)]
pub(crate) fn decode_bools_bypass(&mut self, n_bits: u32) -> u32 {
debug_assert!(n_bits > 0 && n_bits <= 32);
if (self.cnt as u32) < n_bits {
self.ctx_refill();
}
let r = self.rng as u64;
let dif = self.dif;
debug_assert!(r & 1 == 0);
debug_assert!((dif >> 48) < r);
if n_bits < 4 {
let mut dif = dif;
let mut vw = r << 47;
let mut ret: u32 = 0;
for _ in 0..n_bits {
let ge = u32::from(dif >= vw);
let mask = 0u64.wrapping_sub(ge as u64);
dif = dif.wrapping_sub(vw & mask);
ret = (ret << 1) | (ge ^ 1);
vw >>= 1;
}
self.dif = ((dif + 1) << n_bits) - 1;
self.cnt -= n_bits as i32;
return ret;
}
let s = 48 - n_bits;
let q = if n_bits <= 16 {
(((dif >> s) as u32) / self.rng) as u64
} else {
dif / (r << s)
};
self.dif = ((dif - ((q * r) << s) + 1) << n_bits) - 1;
self.cnt -= n_bits as i32;
!(q as u32) & (u32::MAX >> (32 - n_bits))
}
#[inline(always)]
pub(crate) fn decode_bool_bypass(&mut self) -> u32 {
if self.cnt < 1 {
self.ctx_refill();
}
let vw = (self.rng as u64) << 47;
let dif = self.dif;
debug_assert!(self.rng & 1 == 0);
debug_assert!((dif >> 48) < self.rng as u64);
let ge = (dif >= vw) as u64;
let mask = 0u64.wrapping_sub(ge);
let dif = dif - (vw & mask);
self.dif = ((dif + 1) << 1) - 1;
self.cnt -= 1;
(ge as u32) ^ 1
}
#[inline(always)]
pub(crate) fn decode_unary_bypass(&mut self, max_bits: u32) -> u32 {
debug_assert!(max_bits == 5 || max_bits == 6 || max_bits == 21);
if (self.cnt as u32) < max_bits {
self.ctx_refill();
}
let (ret, bits, dif) = unary_bypass_kernel(self.dif, self.rng, max_bits);
self.dif = dif;
self.cnt -= bits as i32;
ret
}
#[inline(always)]
fn decode_bool_raw(&mut self, f: u32) -> u32 {
let r = self.rng;
let dif = self.dif;
debug_assert!((dif >> 48) < r as u64);
let p = ((f >> 7) << 4) + 8;
let v = (((r >> 8) * p) >> 7) << 3;
let vw = (v as u64) << 48;
let ge = u32::from(dif >= vw);
let ge_u64 = ge as u64;
let ge_mask64 = 0u64.wrapping_sub(ge_u64);
let ge_mask32 = 0u32.wrapping_sub(ge);
let new_dif = dif.wrapping_sub(vw & ge_mask64);
let new_rng = (v & !ge_mask32) | ((r - v) & ge_mask32);
self.ctx_norm(new_dif, new_rng);
ge ^ 1
}
#[inline(always)]
pub(crate) fn decode_symbol_adapt_n_scalar<const N: usize>(&mut self, cdf: &mut [u16]) -> u32 {
debug_assert!((1..=7).contains(&N));
if cdf.len() <= N {
return 0;
}
let min_prob = &MSAC_MIN_PROB[N - 1];
let c = (self.dif >> 48) as u32;
let r = self.rng >> 8;
let mut u = self.rng;
let mut v = 0u32;
let mut val_usize = N;
for i in 0..N {
let p_raw = (cdf[i] | 127) as i32 - min_prob[i] as i32;
let p = p_raw.max(0) as u32;
let boundary = ((r * p) >> 10) << 3;
if c >= boundary {
v = boundary;
val_usize = i;
break;
}
u = boundary;
}
if val_usize == N {
let p_raw = (cdf[N] | 127) as i32 - min_prob[N] as i32;
let p = p_raw.max(0) as u32;
v = ((r * p) >> 10) << 3;
}
debug_assert!(u <= self.rng);
debug_assert!(u >= v);
self.ctx_norm(self.dif - ((v as u64) << 48), u - v);
if self.should_update_cdf() {
let pc = cdf[N];
let count = (pc & 0xFF) as u8;
debug_assert!(count <= 32);
let rate =
MSAC_RATE[(pc >> 8) as usize][(count >> 4) as usize] + if N > 2 { 1 } else { 0 };
for cdf_i in cdf[..val_usize].iter_mut() {
*cdf_i = cdf_i.wrapping_add((32768u16 - *cdf_i) >> rate);
}
for cdf_i in cdf[val_usize..N].iter_mut() {
*cdf_i = cdf_i.wrapping_sub(*cdf_i >> rate);
}
cdf[N] = pc + u16::from(count < 32);
}
val_usize as u32
}
#[inline(always)]
pub(crate) fn decode_bool_adapt(&mut self, cdf: &mut [u16]) -> u32 {
let bit = self.decode_bool_raw(cdf[0] as u32);
if self.should_update_cdf() {
let pc = cdf[1];
let count = (pc & 0xFF) as u8;
let rate = MSAC_RATE[(pc >> 8) as usize][(count >> 4) as usize];
let b = bit as i32; let c = cdf[0] as i32;
cdf[0] = (c - b - ((c - 32769 * b) >> rate)) as u16;
cdf[1] = pc + if count < 32 { 1 } else { 0 };
}
bit
}
#[inline(always)]
pub(crate) fn decode_uniform(&mut self, n: u32) -> u32 {
debug_assert!(n > 0);
let l = crate::intops::ulog2(n) + 1;
debug_assert!(l > 1);
let m = (1u32 << l) - n;
let v = self.decode_bools_bypass((l - 1) as u32);
if v < m {
v
} else {
(v << 1) - m + self.decode_bool_bypass()
}
}
#[inline(always)]
pub(crate) fn cnt(&self) -> i32 {
self.cnt
}
#[inline(always)]
fn decode_symbol_adapt_sse2(&mut self, cdf: &mut [u16], n_symbols: usize) -> u32 {
match n_symbols {
1 => self.decode_symbol_adapt_n_sse2::<1>(cdf),
2 => self.decode_symbol_adapt_n_sse2::<2>(cdf),
3 => self.decode_symbol_adapt_n_sse2::<3>(cdf),
4 => self.decode_symbol_adapt_n_sse2::<4>(cdf),
5 => self.decode_symbol_adapt_n_sse2::<5>(cdf),
6 => self.decode_symbol_adapt_n_sse2::<6>(cdf),
7 => self.decode_symbol_adapt_n_sse2::<7>(cdf),
_ => unreachable!("invalid MSAC symbol count"),
}
}
#[inline(always)]
pub(crate) fn decode_symbol_adapt_padded_sse2<const LANES: usize>(
&mut self,
cdf: &mut [u16; LANES],
n_symbols: usize,
) -> u32 {
match n_symbols {
1 => self.decode_symbol_adapt_n_padded_sse2::<1, LANES>(cdf),
2 => self.decode_symbol_adapt_n_padded_sse2::<2, LANES>(cdf),
3 => self.decode_symbol_adapt_n_padded_sse2::<3, LANES>(cdf),
4 => self.decode_symbol_adapt_n_padded_sse2::<4, LANES>(cdf),
5 => self.decode_symbol_adapt_n_padded_sse2::<5, LANES>(cdf),
6 => self.decode_symbol_adapt_n_padded_sse2::<6, LANES>(cdf),
7 => self.decode_symbol_adapt_n_padded_sse2::<7, LANES>(cdf),
_ => unreachable!("invalid MSAC symbol count"),
}
}
#[cfg(feature = "adaptive_cdf")]
#[inline(always)]
fn decode_symbol_adapt3_no_update_scalar(&mut self, cdf: &[u16]) -> u32 {
debug_assert!(!self.should_update_cdf());
let c = (self.dif >> 48) as u32;
let r = self.rng >> 8;
let p0 = ((cdf[0] | 127) as u32) - 31;
let b0 = ((r * p0) >> 10) << 3;
if c >= b0 {
self.ctx_norm(self.dif - ((b0 as u64) << 48), self.rng - b0);
return 0;
}
let p1 = ((cdf[1] | 127) as u32) - 63;
let b1 = ((r * p1) >> 10) << 3;
if c >= b1 {
self.ctx_norm(self.dif - ((b1 as u64) << 48), b0 - b1);
return 1;
}
let p2 = ((cdf[2] | 127) as u32) - 95;
let b2 = ((r * p2) >> 10) << 3;
if c >= b2 {
self.ctx_norm(self.dif - ((b2 as u64) << 48), b1 - b2);
return 2;
}
self.ctx_norm(self.dif, b2);
3
}
#[inline(always)]
pub(crate) fn decode_symbol_adapt_n_sse2<const N: usize>(&mut self, cdf: &mut [u16]) -> u32 {
debug_assert!((1..=7).contains(&N));
#[cfg(feature = "adaptive_cdf")]
if !UPDATE_CDF && N == 3 {
return self.decode_symbol_adapt3_no_update_scalar(cdf);
}
if cdf.len() <= N {
return 0;
}
if N <= 3 && cdf.len() >= 4 {
let cdf = (&mut cdf[..4])
.try_into()
.expect("4-lane CDF accessor must expose 4 values");
return msac_decode_symbol_adapt_sse2::<UPDATE_CDF, N, 4>(self, cdf);
}
if N >= 4 && cdf.len() >= 8 {
let cdf = (&mut cdf[..8])
.try_into()
.expect("8-lane CDF accessor must expose 8 values");
return msac_decode_symbol_adapt_sse2::<UPDATE_CDF, N, 8>(self, cdf);
}
self.decode_symbol_adapt_n_scalar::<N>(cdf)
}
#[inline(always)]
pub(crate) fn decode_symbol_adapt_n_padded_sse2<const N: usize, const LANES: usize>(
&mut self,
cdf: &mut [u16; LANES],
) -> u32 {
debug_assert!((1..=7).contains(&N));
#[cfg(feature = "adaptive_cdf")]
if !UPDATE_CDF && N == 3 {
return self.decode_symbol_adapt3_no_update_scalar(&cdf[..]);
}
debug_assert!(LANES == 2 || LANES == 4 || LANES == 8);
debug_assert!(N < LANES);
if N <= 3 && LANES >= 4 {
let cdf = (&mut cdf[..4])
.try_into()
.expect("4-lane CDF accessor must expose 4 values");
return msac_decode_symbol_adapt_sse2::<UPDATE_CDF, N, 4>(self, cdf);
}
if N >= 4 && LANES >= 8 {
let cdf = (&mut cdf[..8])
.try_into()
.expect("8-lane CDF accessor must expose 8 values");
return msac_decode_symbol_adapt_sse2::<UPDATE_CDF, N, 8>(self, cdf);
}
self.decode_symbol_adapt_n_scalar::<N>(&mut cdf[..])
}
}
impl<'a, const UPDATE_CDF: bool> MsacReader<UPDATE_CDF> for MsacContextSse<'a, UPDATE_CDF> {
#[inline(always)]
fn save(&self) -> MsacState {
MsacContextSse::save(self)
}
#[inline(always)]
fn decode_bools_bypass(&mut self, n_bits: u32) -> u32 {
MsacContextSse::decode_bools_bypass(self, n_bits)
}
#[inline(always)]
fn decode_hr_bypass(&mut self, cmax: u32, m: u32) -> i32 {
MsacContextSse::decode_hr_bypass(self, cmax, m)
}
#[inline(always)]
fn decode_bool_bypass(&mut self) -> u32 {
MsacContextSse::decode_bool_bypass(self)
}
#[inline(always)]
fn decode_symbol_adapt(&mut self, cdf: &mut [u16], n_symbols: usize) -> u32 {
MsacContextSse::decode_symbol_adapt_sse2(self, cdf, n_symbols)
}
#[inline(always)]
fn decode_symbol_adapt_padded<const LANES: usize>(
&mut self,
cdf: &mut [u16; LANES],
n_symbols: usize,
) -> u32 {
MsacContextSse::decode_symbol_adapt_padded_sse2::<LANES>(self, cdf, n_symbols)
}
#[inline(always)]
fn decode_symbol_adapt_n_padded<const N: usize, const LANES: usize>(
&mut self,
cdf: &mut [u16; LANES],
) -> u32 {
MsacContextSse::decode_symbol_adapt_n_padded_sse2::<N, LANES>(self, cdf)
}
#[inline(always)]
fn decode_bool_adapt(&mut self, cdf: &mut [u16]) -> u32 {
MsacContextSse::decode_bool_adapt(self, cdf)
}
#[inline(always)]
fn decode_uniform(&mut self, n: u32) -> u32 {
MsacContextSse::decode_uniform(self, n)
}
#[inline(always)]
fn decode_coefs<C: crate::pixel::Coeff>(
&mut self,
coef: &mut crate::cdf::CdfCoefContext,
mode: &mut crate::cdf::CdfModeContext,
a: &[u8],
l: &[u8],
p: &crate::recon::DecodeCoefParams,
cf: &mut [C],
txtp: &mut u16,
res_ctx: &mut u8,
levels_scratch: &mut [i8; 1089],
nz_scratch: &mut [u32; 1024],
) -> i32 {
unsafe {
crate::recon::decode_coefs_sse2(
self,
coef,
mode,
a,
l,
p,
cf,
txtp,
res_ctx,
levels_scratch,
nz_scratch,
)
}
}
#[inline(always)]
fn cnt(&self) -> i32 {
MsacContextSse::cnt(self)
}
}
#[repr(C, align(16))]
struct AlignedU16x8([u16; 8]);
static PW_127: AlignedU16x8 = AlignedU16x8([127; 8]);
#[repr(C, align(16))]
struct AlignedSseMsac(pub(crate) [u16; 16]);
#[inline(always)]
fn msac_decode_symbol_adapt_sse2<const UPDATE_CDF: bool, const N: usize, const LANES: usize>(
s: &mut MsacContextSse<'_, UPDATE_CDF>,
cdf: &mut [u16; LANES],
) -> u32 {
debug_assert!(LANES == 4 || LANES == 8);
debug_assert!(N < LANES);
unsafe {
let cdf_v = if LANES == 4 {
_mm_loadl_epi64(cdf.as_ptr().cast())
} else {
_mm_loadu_si128(cdf.as_ptr().cast())
};
let mp_ptr = MSAC_MIN_PROB[N - 1].as_ptr().cast::<__m128i>();
let min_prob = _mm_load_si128(mp_ptr);
let rng_x = _mm_cvtsi32_si128(s.rng as i32);
let mut rng_bc = _mm_shufflelo_epi16::<0>(rng_x);
if LANES == 8 {
rng_bc = _mm_unpacklo_epi64(rng_bc, rng_bc);
}
let mut bounds = core::mem::MaybeUninit::<AlignedSseMsac>::uninit();
let bp = bounds.as_mut_ptr().cast::<u16>();
bp.add(7).write(s.rng as u16);
let pw = _mm_load_si128(PW_127.0.as_ptr().cast());
let p = _mm_subs_epu16(_mm_or_si128(cdf_v, pw), min_prob);
let scale = _mm_slli_epi16::<6>(_mm_srli_epi16::<8>(rng_bc));
let bnd = _mm_slli_epi16::<3>(_mm_mulhi_epu16(p, scale));
let dif_x = _mm_cvtsi64_si128(s.dif as i64);
let mut c_bc = _mm_shufflelo_epi16::<0xFF>(dif_x);
if LANES == 8 {
c_bc = _mm_unpacklo_epi64(c_bc, c_bc);
}
_mm_store_si128(bp.add(8).cast(), bnd);
let cmp = _mm_cmpeq_epi16(_mm_subs_epu16(bnd, c_bc), _mm_setzero_si128());
if s.should_update_cdf() {
let pc = cdf[N];
let count = (pc & 0xff) as u8;
debug_assert!(count <= 32);
let rate =
MSAC_RATE[(pc >> 8) as usize][(count >> 4) as usize] + if N > 2 { 1 } else { 0 };
let sh = _mm_cvtsi32_si128(rate as i32);
let ones = _mm_cmpeq_epi16(cmp, cmp);
let sel = _mm_avg_epu16(ones, cmp);
let shifted_mask = _mm_srl_epi16(cmp, sh);
let toward = _mm_sub_epi16(sel, cdf_v);
let base = _mm_sub_epi16(cdf_v, shifted_mask);
let delta = _mm_srl_epi16(toward, sh);
let out = _mm_add_epi16(base, delta);
if LANES == 4 {
_mm_storel_epi64(cdf.as_mut_ptr().cast(), out);
} else {
_mm_storeu_si128(cdf.as_mut_ptr().cast(), out);
}
cdf[N] = pc + u16::from(count < 32);
}
let raw = _mm_movemask_epi8(cmp) as u32;
let mask = raw | (1u32 << (2 * N));
let i = (mask.trailing_zeros() >> 1) as usize;
let u = bp.add(7 + i).read() as u32;
let v = bp.add(8 + i).read() as u32;
debug_assert!(u <= s.rng);
debug_assert!(u >= v);
s.ctx_norm(s.dif - ((v as u64) << 48), u - v);
i as u32
}
}
#[cfg(all(test, feature = "adaptive_cdf"))]
mod no_update_fastpath_tests {
use super::MsacContextSse;
#[test]
fn adapt3_no_update_scalar_matches_simd() {
let mut s: u64 = 0x0dd0_5e1e_c7ed_b055;
let mut next = move || {
s ^= s << 13;
s ^= s >> 7;
s ^= s << 17;
s
};
let mut buf = [0u8; 4096];
for b in buf.iter_mut() {
*b = next() as u8;
}
for _ in 0..2_000 {
let mut v = [0u16; 3];
v[0] = 1 + (next() % 32700) as u16;
v[1] = (v[0] as u64 * (next() % 1000) / 1000) as u16;
v[2] = (v[1] as u64 * (next() % 1000) / 1000) as u16;
let cdf = [v[0], v[1], v[2], 0u16];
let skip = (next() % 64) as u32;
let mut a: MsacContextSse<false> = MsacContextSse::new(&buf);
let mut b: MsacContextSse<false> = MsacContextSse::new(&buf);
for _ in 0..skip {
a.decode_bool_bypass();
b.decode_bool_bypass();
}
let ca = cdf;
let mut cb = cdf;
let ra = a.decode_symbol_adapt3_no_update_scalar(&ca[..]);
let rb = msac_simd_ref(&mut b, &mut cb);
assert_eq!(ra, rb);
assert_eq!(
(a.dif, a.rng, a.cnt, a.buf_pos),
(b.dif, b.rng, b.cnt, b.buf_pos)
);
assert_eq!(ca, cb);
}
}
fn msac_simd_ref(s: &mut MsacContextSse<false>, cdf: &mut [u16; 4]) -> u32 {
super::msac_decode_symbol_adapt_sse2::<false, 3, 4>(s, cdf)
}
}