use crate::cdf::cdf_array_mut;
use crate::headers::PixelLayout;
use crate::intops::{apply_sign, iclip, imax, imin, ulog2, umin};
use crate::levels::{IntraPredMode, Mv, N_BS_SIZES, RefPair, txtp};
use crate::mc::OpflRegressionData;
use crate::msac::MsacReader;
use crate::pixel::Coeff;
use crate::refmvs::{self, INVALID_TRAJ, TemporalBlock};
use crate::scan::SCANS;
use crate::tables::{
BLOCK_DIMENSIONS, DIV_RECIP, MODE_TO_ANGLE_MAP, TXFM_DIMENSIONS, TXTP_FROM_UVMODE, TxfmInfo,
};
use crate::warpmv::resolve_divisor_32;
#[inline(always)]
pub(crate) fn decode_hr<const UPDATE_CDF: bool, M: MsacReader<UPDATE_CDF>>(
msac: &mut M,
hr_avg: i32,
) -> i32 {
let m = ulog2(iclip(hr_avg, 2, 64) as u32) as u32;
let cmax = imin(m as i32 + 4, 6) as u32;
msac.decode_hr_bypass(cmax, m)
}
#[inline]
pub(crate) fn tcq_next_state(state: i32, abs_level: i32) -> i32 {
(((state & 0x4) ^ (((abs_level & 1) ^ (state & 0x1)) << 2))
| ((state & 0x6) >> 1)
| -0x80000000i32)
& (state >> 31)
}
pub(crate) fn wide_angle_remap(
t_dim: &TxfmInfo,
mode: IntraPredMode,
angle: &mut i32,
mrl_idx: i32,
) -> IntraPredMode {
let mode_u8 = mode as u8;
if mode_u8.wrapping_sub(1) > IntraPredMode::VertLeftPred as u8 - 1 {
return mode;
}
let mrl_adj = (mrl_idx == 1) as i32 - (mrl_idx == 2) as i32;
*angle = MODE_TO_ANGLE_MAP[(mode_u8 - 1) as usize] as i32 + *angle * 3 + mrl_adj;
static THRESH: [u8; 4] = [61, 73, 82, 86];
let rect = t_dim.lw as i32 - t_dim.lh as i32;
if rect > 0 {
debug_assert!(rect <= 4);
if *angle > 270 - THRESH[(rect - 1) as usize] as i32 {
*angle -= 180;
return IntraPredMode::DiagDownLeftPred;
}
} else if rect < 0 {
debug_assert!(rect >= -4);
if *angle < THRESH[(-1 - rect) as usize] as i32 {
*angle += 180;
return IntraPredMode::HorUpPred;
}
}
mode
}
pub(crate) fn gen_mask(
mask: &mut [u8],
stride: usize,
bw: i32,
bh: i32,
x0: i32,
y0: i32,
x1: i32,
y1: i32,
fw: u32,
fh: u32,
) {
let mut off = 0;
for y in 0..bh {
for x in 0..bw {
let p0 = ((x0 + x) as u32) < fw && ((y0 + y) as u32) < fh;
let p1 = ((x1 + x) as u32) < fw && ((y1 + y) as u32) < fh;
mask[off + x as usize] = (32 * (p0 as i32 - p1 as i32 + 1)) as u8;
}
off += stride;
}
}
pub(crate) fn derive_alpha(num: i32, den: i32, mut alpha: i32) -> i32 {
let max = (2 << 8) - 1;
if num != 0 && den != 0 {
let num_abs = num.abs();
let shift_n = ulog2(num_abs as u32);
debug_assert!(den >= 0);
let shift_d = ulog2(den as u32);
let e_d = den - (1 << shift_d);
let f_d = if shift_d > 7 {
(e_d + (1 << (shift_d - 8))) >> (shift_d - 7)
} else {
e_d << (7 - shift_d)
};
let f_n = if shift_n > 7 {
(num_abs + (1 << (shift_n - 8))) >> (shift_n - 7)
} else {
num_abs << (7 - shift_n)
};
let shift_add = shift_d - shift_n - 8;
if shift_add <= 1 {
let shift0 = 9 + 7 + shift_add;
let tmp_alpha = if shift0 < 0 {
max
} else {
imin((DIV_RECIP[f_d as usize] as i32 * f_n) >> shift0, max)
};
if tmp_alpha != 0 {
alpha = apply_sign(tmp_alpha, num);
}
}
}
alpha
}
fn read_u16_ne(a: &[u8]) -> u16 {
u16::from_ne_bytes(a[..2].try_into().unwrap())
}
fn read_u32_ne(a: &[u8]) -> u32 {
u32::from_ne_bytes(a[..4].try_into().unwrap())
}
fn read_u64_ne(a: &[u8]) -> u64 {
u64::from_ne_bytes(a[..8].try_into().unwrap())
}
pub(crate) fn get_skip_ctx(
t_dim: &TxfmInfo,
bs: usize,
a: &[u8],
l: &[u8],
plane: i32,
u_has_cf: i32,
ss_hor: bool,
ss_ver: bool,
) -> u32 {
debug_assert!(bs < N_BS_SIZES);
let b_dim = &BLOCK_DIMENSIONS[bs];
if plane != 0 {
let not_one_blk = (b_dim[2] - (b_dim[2] != 0 && ss_hor) as u8 > t_dim.lw)
|| (b_dim[3] - (b_dim[3] != 0 && ss_ver) as u8 > t_dim.lh);
let ca: bool = match t_dim.lw {
0 => a[0] != 0x40,
1 => read_u16_ne(a) != 0x4040,
2 => read_u32_ne(a) != 0x40404040,
3 => read_u64_ne(a) != 0x4040404040404040,
4 => (read_u64_ne(a) | read_u64_ne(&a[8..])) != 0x4040404040404040,
_ => unreachable!(),
};
let cl: bool = match t_dim.lh {
0 => l[0] != 0x40,
1 => read_u16_ne(l) != 0x4040,
2 => read_u32_ne(l) != 0x40404040,
3 => read_u64_ne(l) != 0x4040404040404040,
4 => (read_u64_ne(l) | read_u64_ne(&l[8..])) != 0x4040404040404040,
_ => unreachable!(),
};
let offset = if plane == 1 {
6
} else {
6 * u_has_cf + not_one_blk as i32 * 3
} as u32;
offset + ca as u32 + cl as u32
} else if b_dim[2] == t_dim.lw && b_dim[3] == t_dim.lh {
0
} else {
let merge = |dir: &[u8], tx: u8| -> u32 {
let mut v: u32;
if tx == 4 {
let tmp = read_u64_ne(dir) | read_u64_ne(&dir[8..]);
v = (tmp >> 32) as u32 | tmp as u32;
} else {
v = match tx {
0 => dir[0] as u32,
1 => read_u16_ne(dir) as u32,
2 | 3 => read_u32_ne(dir),
_ => unreachable!(),
};
}
if tx == 3 {
v |= read_u32_ne(&dir[4..]);
}
if tx >= 2 {
v |= v >> 16;
}
if tx >= 1 {
v |= v >> 8;
}
v
};
let la = merge(a, t_dim.lw);
let ll = merge(l, t_dim.lh);
(umin(la & 0x3F, 4) + umin(ll & 0x3F, 4) + 3) >> 1
}
}
pub(crate) fn get_dc_sign_ctx(t_dim: &TxfmInfo, a: &[u8], l: &[u8]) -> u32 {
let mask: u64 = 0xC0C0C0C0C0C0C0C0;
let mul: u64 = 0x0101010101010101;
let mut t: u64 = 0;
for &(edge, len) in &[(a, t_dim.lw), (l, t_dim.lh)] {
match len {
0 => t += (edge[0] >> 6) as u64,
1 => t += (read_u16_ne(edge) as u64 & mask) >> 6,
2 => t += (read_u32_ne(edge) as u64 & mask) >> 6,
3 => t += (read_u64_ne(edge) & mask) >> 6,
4 => {
t += (read_u64_ne(&edge[8..]) & mask) >> 6;
t += (read_u64_ne(edge) & mask) >> 6;
}
_ => unreachable!(),
}
}
t = t.wrapping_mul(mul);
let s = (t >> 56) as i32 - t_dim.w as i32 - t_dim.h as i32;
(s != 0) as u32 + (s > 0) as u32
}
static PACK_LEVEL: [i8; 16] = {
let mut t = [0i8; 16];
let mut i = 0;
while i < 16 {
let t5 = if i < 5 { i as i8 } else { 5 };
let t3 = if t5 < 3 { t5 } else { 3 };
t[i] = t5 | (t3 << 5);
i += 1;
}
t
};
#[inline(always)]
fn pack_level(tok: i32) -> i8 {
debug_assert!((0..=8).contains(&tok));
PACK_LEVEL[(tok & 15) as usize]
}
#[inline(always)]
fn packed(x: i8) -> u32 {
x as u8 as u32
}
#[inline(always)]
fn get_ph_ctx(levels: &[i8], is_2d: bool, stride: usize) -> usize {
assert!(stride >= 4 && levels.len() > 2 * stride);
let mut s = packed(levels[1]) + packed(levels[2]) + packed(levels[stride]);
if is_2d {
s += packed(levels[stride + 1]) + packed(levels[2 * stride]);
} else {
s += packed(levels[3]) + packed(levels[4]);
}
let mag = s >> 5;
if mag < 7 {
((mag + 1) >> 1) as usize
} else {
4
}
}
#[inline(always)]
fn get_lo_ctx_luma_2d(base: &[i8], hi_mag: &mut u32, xy: u32, stride: usize) -> u32 {
assert!(stride >= 4 && base.len() > 2 * stride);
let s3 = packed(base[1]) + packed(base[stride]) + packed(base[stride + 1]);
let s5 = s3 + packed(base[2]) + packed(base[2 * stride]);
let hi = s3 & 31;
let lo_freq = xy < 4;
let lo;
let offset;
let lim;
if lo_freq {
lo = s5 & 31;
if xy == 0 {
offset = 0;
lim = 8;
} else if xy < 2 {
offset = 9;
lim = 6;
} else {
offset = 16;
lim = 4;
}
*hi_mag = if xy > 0 { 7 } else { 0 } + ((hi + 1) >> 1).min(6);
} else {
lo = s5 >> 5;
offset = if xy < 6 {
0
} else if xy < 8 {
5
} else {
10
};
lim = 4;
*hi_mag = ((hi + 1) >> 1).min(6);
}
offset + ((lo + 1) >> 1).min(lim)
}
#[inline(always)]
fn get_lo_ctx_luma_1d(base: &[i8], hi_mag: &mut u32, xy: u32, stride: usize) -> u32 {
assert!(stride >= 4 && base.len() > stride);
let sab = packed(base[1]) + packed(base[stride]);
let scde = packed(base[2]) + packed(base[3]) + packed(base[4]);
let sabc = sab + packed(base[2]);
let hi = sabc & 31;
let lo;
let offset;
let lim;
if xy < 2 {
lo = (sab & 31) + (scde >> 5);
if xy == 0 {
offset = 21;
lim = 6;
} else {
offset = 28;
lim = 4;
}
*hi_mag = 7 + ((hi + 1) >> 1).min(6);
} else {
lo = (sab + scde) >> 5;
offset = 15;
lim = 4;
*hi_mag = ((hi + 1) >> 1).min(6);
}
offset + ((lo + 1) >> 1).min(lim)
}
#[inline(always)]
fn get_lo_ctx_chroma_2d(base: &[i8], hi_mag: &mut u32, xy: u32, plane: i32, stride: usize) -> u32 {
assert!(stride >= 2 && base.len() > stride + 1);
let s = packed(base[1]) + packed(base[stride]) + packed(base[stride + 1]);
let hi = s & 31;
let lo = if xy == 0 { hi } else { s >> 5 };
*hi_mag = ((hi + 1) >> 1).min(3);
let offset = if plane == 1 { 0 } else { 4 };
offset + ((lo + 1) >> 1).min(3)
}
#[inline(always)]
fn get_lo_ctx_chroma_1d(base: &[i8], hi_mag: &mut u32, xy: u32, stride: usize) -> u32 {
assert!(stride >= 2 && base.len() > stride);
let s = packed(base[1]) + packed(base[stride]);
let hi = s & 31;
let lo = if xy == 0 { hi } else { s >> 5 };
*hi_mag = ((hi + 1) >> 1).min(3);
8 + ((lo + 1) >> 1).min(3)
}
#[inline(always)]
pub(crate) fn get_lo_ctx_idtx(levels: &[i8], off: usize, hi_mag: &mut u32, stride: usize) -> u32 {
let v0 = levels[off - 1] as u32;
let v1 = levels[off - stride] as u32;
let lo_mag = v0.min(3) + v1.min(3);
let hi = v0.min(5) + v1.min(5);
*hi_mag = hi.min(6);
lo_mag
}
#[inline(always)]
pub(crate) fn get_sign_ctx_idtx(levels: &[i8], off: usize, stride: usize) -> u32 {
let sum =
levels[off - 1] as i32 + levels[off - stride] as i32 + levels[off - stride - 1] as i32;
let offset = if levels[off] > 3 { 2 } else { 0 };
match sum {
-3 => offset + 6,
-2 | -1 => offset + 2,
0 => 0,
1 | 2 => offset + 1,
3 => offset + 5,
_ => unreachable!(),
}
}
pub(crate) fn get_mask(
mask: &mut [u8],
stride: usize,
bx4: i32,
x4: i32,
by4: i32,
y4: i32,
mv: &[Mv; 2],
h_subpel_bits: i32,
v_subpel_bits: i32,
bw4: i32,
bh4: i32,
iw: i32,
ih: i32,
) -> bool {
let (mv0, mv1) = (mv[0].xy(), mv[1].xy());
let x0 = (bx4 + x4) * 4 + (mv0.x >> h_subpel_bits);
let y0 = (by4 + y4) * 4 + (mv0.y >> v_subpel_bits);
let x1 = (bx4 + x4) * 4 + (mv1.x >> h_subpel_bits);
let y1 = (by4 + y4) * 4 + (mv1.y >> v_subpel_bits);
if x0 < 0
|| x1 < 0
|| y0 < 0
|| y1 < 0
|| x0 + bw4 * 4 >= iw
|| x1 + bw4 * 4 >= iw
|| y0 + bh4 * 4 >= ih
|| y1 + bh4 * 4 >= ih
{
let off = (y4 as usize * stride + x4 as usize) * 4;
gen_mask(
&mut mask[off..],
stride,
bw4 * 4,
bh4 * 4,
x0,
y0,
x1,
y1,
iw as u32,
ih as u32,
);
return true;
}
false
}
#[derive(Clone, Copy, Default)]
#[repr(C)]
pub(crate) struct OpflMvDelta {
pub(crate) x: i8,
pub(crate) y: i8,
}
#[derive(Clone, Copy, Default)]
#[repr(C)]
pub(crate) struct OpflMvDeltaBlock {
pub(crate) d: [OpflMvDelta; 2],
}
pub(crate) fn opfl_mv_adj(r: &OpflRegressionData, dd: &mut OpflMvDeltaBlock, d: [i8; 2]) {
let mut su2 = r.su2;
let mut suv = r.suv;
let mut sv2 = r.sv2;
let mut suw = r.suw;
let mut svw = r.svw;
let nbits_su2 = 1 + ulog2((su2 + (su2 == 0) as i32) as u32);
let nbits_sv2 = 1 + ulog2((sv2 + (sv2 == 0) as i32) as u32);
let nbits_suv = 1 + ulog2((suv.abs() + (suv == 0) as i32) as u32);
let nbits_suw = 1 + ulog2((suw.abs() + (suw == 0) as i32) as u32);
let nbits_svw = 1 + ulog2((svw.abs() + (svw == 0) as i32) as u32);
let nbits_max = imax(
nbits_su2 + nbits_sv2,
imax(
imax(nbits_sv2 + nbits_suw, nbits_suv + nbits_svw),
imax(nbits_su2 + nbits_svw, nbits_suv + nbits_suw),
),
);
let rbits = imax(0, nbits_max - 23) >> 1;
if rbits != 0 {
let rnd = (1 << rbits) >> 1;
su2 = (su2 + rnd) >> rbits;
sv2 = (sv2 + rnd) >> rbits;
suv = (suv + rnd - (suv < 0) as i32) >> rbits;
suw = (suw + rnd - (suw < 0) as i32) >> rbits;
svw = (svw + rnd - (svw < 0) as i32) >> rbits;
}
let det = su2 * sv2 - suv * suv;
if det > 0 {
let mut s = [sv2 * suw - suv * svw, su2 * svw - suv * suw];
let mut shift = 0i32;
let idet = resolve_divisor_32(det as u32, &mut shift);
let idet_bits = ulog2(idet as u32);
for i in 0..2 {
if s[i] == 0 {
continue;
}
let mut abss = s[i].abs();
let rb = imax(0, ulog2(abss as u32) + idet_bits - 22);
if rb > 0 {
abss = (abss + ((1 << rb) >> 1)) >> rb;
}
let ibits = 3 + rb - shift;
if ibits >= 0 {
abss = abss * idet * (1 << ibits);
} else {
abss = (abss * idet + ((1 << -ibits) >> 1)) >> -ibits;
}
s[i] = apply_sign(abss, s[i]);
}
dd.d[0].x = -iclip(d[0] as i32 * s[0], -16, 16) as i8;
dd.d[0].y = -iclip(d[0] as i32 * s[1], -16, 16) as i8;
dd.d[1].x = iclip(d[1] as i32 * s[0], -16, 16) as i8;
dd.d[1].y = iclip(d[1] as i32 * s[1], -16, 16) as i8;
} else {
*dd = OpflMvDeltaBlock::default();
}
}
pub(crate) fn scaledown_16pel_mv_for_chroma(mv: &mut [Mv; 2], layout: PixelLayout) {
match layout {
PixelLayout::I420 => {
for m in mv.iter_mut() {
let y = m.y();
m.set_y((y + (y > 0) as i32) >> 1);
}
for m in mv.iter_mut() {
let x = m.x();
m.set_x((x + (x > 0) as i32) >> 1);
}
}
PixelLayout::I422 => {
for m in mv.iter_mut() {
let x = m.x();
m.set_x((x + (x > 0) as i32) >> 1);
}
}
_ => {}
}
}
pub(crate) fn scaleup_8pel_mv_for_chroma(mv: &mut [Mv; 2], layout: PixelLayout) {
match layout {
PixelLayout::I444 => {
for m in mv.iter_mut() {
m.set_x(m.x() << 1);
}
for m in mv.iter_mut() {
m.set_y(m.y() << 1);
}
}
PixelLayout::I422 => {
for m in mv.iter_mut() {
m.set_y(m.y() << 1);
}
}
_ => {}
}
}
static MD_IDX2TYPE: [[[u8; 7]; 13]; 3] = [
[
[
txtp::DCT_DCT,
txtp::ADST_ADST,
txtp::ADST_DCT,
txtp::DCT_ADST,
txtp::ADST_FLIPADST,
txtp::FLIPADST_ADST,
txtp::H_ADST,
],
[
txtp::DCT_DCT,
txtp::ADST_ADST,
txtp::ADST_DCT,
txtp::DCT_ADST,
txtp::ADST_FLIPADST,
txtp::V_DCT,
txtp::V_ADST,
],
[
txtp::DCT_DCT,
txtp::ADST_ADST,
txtp::ADST_DCT,
txtp::DCT_ADST,
txtp::FLIPADST_ADST,
txtp::H_DCT,
txtp::H_ADST,
],
[
txtp::DCT_DCT,
txtp::ADST_ADST,
txtp::ADST_DCT,
txtp::DCT_ADST,
txtp::FLIPADST_FLIPADST,
txtp::ADST_FLIPADST,
txtp::FLIPADST_ADST,
],
[
txtp::DCT_DCT,
txtp::ADST_ADST,
txtp::ADST_DCT,
txtp::DCT_ADST,
txtp::ADST_FLIPADST,
txtp::FLIPADST_ADST,
txtp::H_ADST,
],
[
txtp::DCT_DCT,
txtp::ADST_ADST,
txtp::ADST_DCT,
txtp::DCT_ADST,
txtp::ADST_FLIPADST,
txtp::V_ADST,
txtp::V_FLIPADST,
],
[
txtp::DCT_DCT,
txtp::ADST_ADST,
txtp::ADST_DCT,
txtp::DCT_ADST,
txtp::ADST_FLIPADST,
txtp::FLIPADST_ADST,
txtp::H_ADST,
],
[
txtp::DCT_DCT,
txtp::ADST_ADST,
txtp::ADST_DCT,
txtp::DCT_ADST,
txtp::FLIPADST_ADST,
txtp::H_DCT,
txtp::H_ADST,
],
[
txtp::DCT_DCT,
txtp::ADST_ADST,
txtp::ADST_DCT,
txtp::DCT_ADST,
txtp::ADST_FLIPADST,
txtp::V_DCT,
txtp::V_ADST,
],
[
txtp::DCT_DCT,
txtp::ADST_ADST,
txtp::ADST_DCT,
txtp::DCT_ADST,
txtp::FLIPADST_FLIPADST,
txtp::ADST_FLIPADST,
txtp::FLIPADST_ADST,
],
[
txtp::DCT_DCT,
txtp::ADST_ADST,
txtp::ADST_DCT,
txtp::DCT_ADST,
txtp::ADST_FLIPADST,
txtp::FLIPADST_ADST,
txtp::V_ADST,
],
[
txtp::DCT_DCT,
txtp::ADST_ADST,
txtp::ADST_DCT,
txtp::DCT_ADST,
txtp::ADST_FLIPADST,
txtp::FLIPADST_ADST,
txtp::H_ADST,
],
[
txtp::DCT_DCT,
txtp::ADST_ADST,
txtp::DCT_ADST,
txtp::V_DCT,
txtp::H_DCT,
txtp::V_ADST,
txtp::H_ADST,
],
],
[
[
txtp::DCT_DCT,
txtp::ADST_ADST,
txtp::ADST_DCT,
txtp::DCT_ADST,
txtp::FLIPADST_DCT,
txtp::ADST_FLIPADST,
txtp::FLIPADST_ADST,
],
[
txtp::DCT_DCT,
txtp::ADST_ADST,
txtp::ADST_DCT,
txtp::DCT_ADST,
txtp::ADST_FLIPADST,
txtp::FLIPADST_FLIPADST,
txtp::FLIPADST_ADST,
],
[
txtp::DCT_DCT,
txtp::ADST_ADST,
txtp::ADST_DCT,
txtp::DCT_ADST,
txtp::FLIPADST_ADST,
txtp::FLIPADST_DCT,
txtp::ADST_FLIPADST,
],
[
txtp::DCT_DCT,
txtp::ADST_ADST,
txtp::ADST_DCT,
txtp::DCT_ADST,
txtp::FLIPADST_FLIPADST,
txtp::ADST_FLIPADST,
txtp::FLIPADST_ADST,
],
[
txtp::DCT_DCT,
txtp::ADST_ADST,
txtp::ADST_DCT,
txtp::DCT_ADST,
txtp::DCT_FLIPADST,
txtp::ADST_FLIPADST,
txtp::FLIPADST_ADST,
],
[
txtp::DCT_DCT,
txtp::ADST_ADST,
txtp::ADST_DCT,
txtp::DCT_ADST,
txtp::DCT_FLIPADST,
txtp::ADST_FLIPADST,
txtp::FLIPADST_ADST,
],
[
txtp::DCT_DCT,
txtp::ADST_ADST,
txtp::ADST_DCT,
txtp::DCT_ADST,
txtp::FLIPADST_DCT,
txtp::ADST_FLIPADST,
txtp::FLIPADST_ADST,
],
[
txtp::DCT_DCT,
txtp::ADST_ADST,
txtp::ADST_DCT,
txtp::DCT_ADST,
txtp::FLIPADST_DCT,
txtp::FLIPADST_ADST,
txtp::ADST_FLIPADST,
],
[
txtp::DCT_DCT,
txtp::ADST_ADST,
txtp::ADST_DCT,
txtp::DCT_ADST,
txtp::DCT_FLIPADST,
txtp::FLIPADST_FLIPADST,
txtp::ADST_FLIPADST,
],
[
txtp::DCT_DCT,
txtp::ADST_ADST,
txtp::ADST_DCT,
txtp::DCT_ADST,
txtp::FLIPADST_FLIPADST,
txtp::ADST_FLIPADST,
txtp::FLIPADST_ADST,
],
[
txtp::DCT_DCT,
txtp::ADST_ADST,
txtp::ADST_DCT,
txtp::DCT_ADST,
txtp::FLIPADST_DCT,
txtp::ADST_FLIPADST,
txtp::FLIPADST_ADST,
],
[
txtp::DCT_DCT,
txtp::ADST_ADST,
txtp::ADST_DCT,
txtp::DCT_ADST,
txtp::DCT_FLIPADST,
txtp::ADST_FLIPADST,
txtp::FLIPADST_ADST,
],
[
txtp::DCT_DCT,
txtp::ADST_ADST,
txtp::ADST_DCT,
txtp::DCT_ADST,
txtp::V_DCT,
txtp::H_DCT,
txtp::H_ADST,
],
],
[
[
txtp::DCT_DCT,
txtp::ADST_ADST,
txtp::ADST_DCT,
txtp::DCT_ADST,
txtp::FLIPADST_DCT,
txtp::ADST_FLIPADST,
txtp::FLIPADST_ADST,
],
[
txtp::DCT_DCT,
txtp::ADST_ADST,
txtp::ADST_DCT,
txtp::DCT_ADST,
txtp::DCT_FLIPADST,
txtp::ADST_FLIPADST,
txtp::FLIPADST_ADST,
],
[
txtp::DCT_DCT,
txtp::ADST_ADST,
txtp::ADST_DCT,
txtp::DCT_ADST,
txtp::FLIPADST_DCT,
txtp::FLIPADST_ADST,
txtp::ADST_FLIPADST,
],
[
txtp::DCT_DCT,
txtp::ADST_ADST,
txtp::ADST_DCT,
txtp::DCT_ADST,
txtp::FLIPADST_DCT,
txtp::ADST_FLIPADST,
txtp::FLIPADST_ADST,
],
[
txtp::DCT_DCT,
txtp::ADST_ADST,
txtp::ADST_DCT,
txtp::DCT_ADST,
txtp::FLIPADST_DCT,
txtp::ADST_FLIPADST,
txtp::FLIPADST_ADST,
],
[
txtp::DCT_DCT,
txtp::ADST_ADST,
txtp::ADST_DCT,
txtp::DCT_ADST,
txtp::DCT_FLIPADST,
txtp::ADST_FLIPADST,
txtp::FLIPADST_ADST,
],
[
txtp::DCT_DCT,
txtp::ADST_ADST,
txtp::ADST_DCT,
txtp::DCT_ADST,
txtp::FLIPADST_DCT,
txtp::ADST_FLIPADST,
txtp::FLIPADST_ADST,
],
[
txtp::DCT_DCT,
txtp::ADST_ADST,
txtp::ADST_DCT,
txtp::DCT_ADST,
txtp::FLIPADST_DCT,
txtp::FLIPADST_FLIPADST,
txtp::FLIPADST_ADST,
],
[
txtp::DCT_DCT,
txtp::ADST_ADST,
txtp::ADST_DCT,
txtp::DCT_ADST,
txtp::DCT_FLIPADST,
txtp::ADST_FLIPADST,
txtp::FLIPADST_ADST,
],
[
txtp::DCT_DCT,
txtp::ADST_ADST,
txtp::ADST_DCT,
txtp::DCT_ADST,
txtp::FLIPADST_DCT,
txtp::ADST_FLIPADST,
txtp::FLIPADST_ADST,
],
[
txtp::DCT_DCT,
txtp::ADST_ADST,
txtp::ADST_DCT,
txtp::DCT_ADST,
txtp::FLIPADST_DCT,
txtp::ADST_FLIPADST,
txtp::FLIPADST_ADST,
],
[
txtp::DCT_DCT,
txtp::ADST_ADST,
txtp::ADST_DCT,
txtp::DCT_ADST,
txtp::DCT_FLIPADST,
txtp::ADST_FLIPADST,
txtp::FLIPADST_ADST,
],
[
txtp::DCT_DCT,
txtp::ADST_ADST,
txtp::ADST_DCT,
txtp::DCT_ADST,
txtp::V_DCT,
txtp::H_DCT,
txtp::V_ADST,
],
],
];
static TXTP_INV_TBL: [[u8; 16]; 2] = [
[
txtp::IDTX,
txtp::V_DCT,
txtp::H_DCT,
txtp::V_ADST,
txtp::H_ADST,
txtp::V_FLIPADST,
txtp::H_FLIPADST,
txtp::DCT_DCT,
txtp::ADST_DCT,
txtp::DCT_ADST,
txtp::FLIPADST_DCT,
txtp::DCT_FLIPADST,
txtp::ADST_ADST,
txtp::FLIPADST_FLIPADST,
txtp::ADST_FLIPADST,
txtp::FLIPADST_ADST,
],
[
txtp::IDTX,
txtp::V_DCT,
txtp::H_DCT,
txtp::DCT_DCT,
txtp::ADST_DCT,
txtp::DCT_ADST,
txtp::FLIPADST_DCT,
txtp::DCT_FLIPADST,
txtp::ADST_ADST,
txtp::FLIPADST_FLIPADST,
txtp::ADST_FLIPADST,
txtp::FLIPADST_ADST,
0,
0,
0,
0,
],
];
static INV_MAP_ADST: [[u8; 4]; 12] = [
[3, 1, 0, 2],
[1, 3, 0, 2],
[1, 3, 0, 2],
[1, 3, 0, 2],
[0, 2, 3, 1],
[2, 1, 0, 3],
[2, 1, 0, 3],
[1, 0, 3, 2],
[1, 0, 3, 2],
[3, 1, 0, 2],
[1, 3, 0, 2],
[1, 3, 0, 2],
];
static INV_MAP: [[u8; 7]; 12] = [
[6, 1, 0, 5, 4, 3, 2],
[1, 6, 0, 4, 2, 5, 3],
[1, 6, 0, 4, 2, 5, 3],
[2, 6, 0, 5, 1, 4, 3],
[3, 4, 6, 1, 0, 2, 5],
[4, 1, 3, 6, 0, 5, 2],
[4, 1, 3, 6, 0, 5, 2],
[5, 0, 6, 2, 1, 4, 3],
[5, 0, 6, 2, 1, 4, 3],
[6, 1, 0, 5, 4, 3, 2],
[1, 6, 0, 4, 2, 5, 3],
[1, 6, 0, 4, 2, 5, 3],
];
pub(crate) fn update_temporal(
t_dst: &mut [TemporalBlock],
t_stride: usize,
w8: usize,
h8: usize,
r: RefPair,
mv: &[Mv; 2],
swap: bool,
) {
let s0 = swap as usize;
let s1 = (!swap) as usize;
let refs = r.refs();
let mut r0 = refs[s0];
let mut r1 = refs[s1];
let mut mv0 = refmvs::quantize_mv(mv[s0]);
let mut mv1 = refmvs::quantize_mv(mv[s1]);
let mut ref_pair = RefPair::from_refs(r0, r1);
let mv0_n = mv0.packed();
let mv1_n = mv1.packed();
if mv0_n == INVALID_TRAJ {
if mv1_n == INVALID_TRAJ {
ref_pair = RefPair::from_pair(-1);
} else {
mv0 = mv1;
r0 = r1;
ref_pair = RefPair::from_refs(r0, r1);
}
} else if mv1_n == INVALID_TRAJ {
mv1 = mv0;
r1 = r0;
ref_pair = RefPair::from_refs(r0, r1);
}
let t_src = TemporalBlock {
mv: refmvs::TemporalBlockMv::from_mvs(mv0, mv1),
reference: ref_pair,
};
for y in 0..h8 {
let row = &mut t_dst[y * t_stride..y * t_stride + w8];
for x in 0..w8 {
row[x] = t_src;
}
}
}
pub(crate) struct DecodeCoefParams<'a> {
pub(crate) tx: usize,
pub(crate) bs: usize,
pub(crate) plane: i32,
pub(crate) intra: bool,
pub(crate) fsc: bool,
pub(crate) lossless: bool,
pub(crate) sdp_active: bool,
pub(crate) y_mode: usize,
pub(crate) uv_mode: usize,
pub(crate) seq_fsc: bool,
pub(crate) seq_ist: [bool; 2],
pub(crate) seq_cctx: bool,
pub(crate) chroma_dctonly: bool,
pub(crate) reduced_txtp_set: i32,
pub(crate) tcq_enabled: bool,
pub(crate) parity_hiding: bool,
pub(crate) layout: PixelLayout,
pub(crate) u_has_cf: i32,
pub(crate) cbx: i32,
pub(crate) cby: i32,
pub(crate) luma_fsc_map: &'a [u8],
pub(crate) dq_tbl: [u32; 2],
pub(crate) bitdepth: u32,
pub(crate) qm: Option<&'a [u8]>,
pub(crate) ss_hor: bool,
pub(crate) ss_ver: bool,
}
use crate::cdf::{CdfCoefContext, CdfModeContext};
pub(crate) enum CoefHeader {
Done(i32),
Decode {
eob: i32,
tx_class: u8,
stx_type: u32,
},
}
#[inline(never)]
fn decode_coefs_header<const UPDATE_CDF: bool, M: MsacReader<UPDATE_CDF>>(
msac: &mut M,
coef: &mut CdfCoefContext,
mode: &mut CdfModeContext,
a: &[u8],
l: &[u8],
p: &DecodeCoefParams,
txtp: &mut u16,
res_ctx: &mut u8,
) -> CoefHeader {
let t_dim = &TXFM_DIMENSIONS[p.tx];
let chroma = p.plane != 0;
let sctx = if p.fsc && !chroma && p.seq_fsc {
9
} else {
get_skip_ctx(t_dim, p.bs, a, l, p.plane, p.u_has_cf, p.ss_hor, p.ss_ver) as usize
};
let all_skip = if p.plane == 2 {
msac.decode_bool_adapt(coef.skip_v(sctx))
} else {
let i = if !p.intra || p.fsc { 1 } else { 0 };
msac.decode_bool_adapt(coef.skip(i, t_dim.ctx as usize, sctx))
};
if all_skip != 0 {
*res_ctx = 0x40;
*txtp = if !chroma && p.fsc {
txtp::IDTX as u16
} else {
(p.lossless as u16) * txtp::WHT_WHT as u16
};
return CoefHeader::Done(-1);
}
let slw = imin(t_dim.lw as i32, 3) as usize;
let slh = imin(t_dim.lh as i32, 3) as usize;
let tx2dszctx = slw + slh;
let eob_ctx = if chroma { 2 } else { (!p.intra) as usize };
let mut eob: i32 = match tx2dszctx {
0 => msac.decode_symbol_adapt_n_padded::<4, 8>(coef.eob_bin_16(eob_ctx)) as i32,
1 => msac.decode_symbol_adapt_n_padded::<5, 8>(coef.eob_bin_32(eob_ctx)) as i32,
2 => msac.decode_symbol_adapt_n_padded::<6, 8>(coef.eob_bin_64(eob_ctx)) as i32,
3 => msac.decode_symbol_adapt_n_padded::<7, 8>(coef.eob_bin_128(eob_ctx)) as i32,
4 => {
let mut e = msac.decode_symbol_adapt_n_padded::<7, 8>(coef.eob_bin_256(eob_ctx)) as i32;
if e == 7 {
e += msac.decode_bool_bypass() as i32;
}
e
}
5 => {
let mut e = msac.decode_symbol_adapt_n_padded::<7, 8>(coef.eob_bin_512(eob_ctx)) as i32;
if e == 7 {
e += msac.decode_bools_bypass(2) as i32;
if e == 10 {
return CoefHeader::Done(i32::MIN);
}
}
e
}
_ => {
let mut e =
msac.decode_symbol_adapt_n_padded::<7, 8>(coef.eob_bin_1024(eob_ctx)) as i32;
if e == 7 {
e += msac.decode_bools_bypass(2) as i32;
}
e
}
};
if eob > 1 {
let eob_hi_bit = msac.decode_bool_adapt(coef.eob_hi_bit()) as i32;
let eob_bin = eob - 2;
eob = eob_hi_bit | 2;
if eob_bin != 0 {
eob = (eob << eob_bin) | msac.decode_bools_bypass(eob_bin as u32) as i32;
}
}
static TXTP_LONG_TBL: [[[u8; 4]; 2]; 2] = [
[
[txtp::V_DCT, txtp::V_ADST, txtp::V_FLIPADST, txtp::IDTX],
[txtp::H_DCT, txtp::H_ADST, txtp::H_FLIPADST, txtp::IDTX],
],
[
[
txtp::DCT_DCT,
txtp::ADST_DCT,
txtp::FLIPADST_DCT,
txtp::H_DCT,
],
[
txtp::DCT_DCT,
txtp::DCT_ADST,
txtp::DCT_FLIPADST,
txtp::V_DCT,
],
],
];
if p.lossless {
if chroma {
if p.intra {
let y_fsc = if !p.sdp_active {
p.fsc
} else {
let idx = (p.cby & 15) as usize * 16 + (p.cbx & 15) as usize;
p.luma_fsc_map[idx] != 0
};
*txtp = if y_fsc {
txtp::IDTX as u16
} else {
txtp::WHT_WHT as u16
};
} else {
*txtp &= 0xe7; }
} else if p.intra {
*txtp = if p.fsc {
txtp::IDTX as u16
} else {
txtp::WHT_WHT as u16
};
} else if t_dim.max == 0 {
*txtp = if msac.decode_bool_adapt(mode.txtp_lossless()) != 0 {
txtp::IDTX as u16
} else {
txtp::WHT_WHT as u16
};
} else {
*txtp = txtp::IDTX as u16;
}
} else if chroma {
if p.chroma_dctonly {
*txtp = txtp::DCT_DCT as u16;
} else {
if p.intra {
*txtp = TXTP_FROM_UVMODE[p.uv_mode] as u16;
}
let t = *txtp as u8;
if (t_dim.w >= 8 && t & 0x02 != 0)
|| (t_dim.h >= 8 && t & 0x40 != 0)
|| (p.tx == 2
&& ((t & 0x47 == 0x41) || (t & 0xe2 == 0x22)))
{
*txtp = txtp::DCT_DCT as u16;
} else if t == txtp::IDTX_INV {
*txtp = txtp::IDTX as u16;
}
}
} else if p.intra {
if t_dim.sub == 3 {
*txtp = txtp::DCT_DCT as u16;
} else if p.fsc {
*txtp = txtp::IDTX as u16;
} else if eob == 0 || p.tx == 3 {
*txtp = txtp::DCT_DCT as u16;
} else if t_dim.max >= 3 {
let long_dct = t_dim.max == 4 || msac.decode_bool_adapt(mode.txtp_long32_dct(0)) != 0;
let short_idx = msac
.decode_symbol_adapt_n_padded::<3, 4>(mode.txtp_intra_short_1d(t_dim.min as usize))
as usize;
let wh = (t_dim.w < t_dim.h) as usize;
*txtp = TXTP_LONG_TBL[long_dct as usize][wh][short_idx] as u16;
} else if p.reduced_txtp_set == 2 {
*txtp = txtp::DCT_DCT as u16;
} else {
let sz_ctx = ((t_dim.lw + t_dim.lh) >> 1) as usize;
let tx_idx = if p.reduced_txtp_set != 0 {
msac.decode_bool_adapt(mode.txtp_ext_reduced(t_dim.min as usize)) as usize
} else {
msac.decode_symbol_adapt_n_padded::<6, 8>(mode.txtp_ext(t_dim.min as usize))
as usize
};
*txtp = MD_IDX2TYPE[sz_ctx][p.y_mode][tx_idx] as u16;
}
} else {
if t_dim.sub == 3 {
*txtp = txtp::DCT_DCT as u16;
} else {
let y = eob >> (2 + slw as i32);
let x = eob & ((4 << slw) - 1);
let xy = x + y;
let ww = imin(8, t_dim.w as i32);
let hh = imin(8, t_dim.h as i32);
let ctx = if xy < 2 {
1usize
} else if xy > 4 * (ww + hh) - 4 {
2
} else {
0
};
if p.tx == 3 {
*txtp = if msac.decode_bool_adapt(mode.txtp_inter_dct_idtx(ctx, 3)) != 0 {
txtp::DCT_DCT as u16
} else {
txtp::IDTX as u16
};
} else if t_dim.max >= 3 {
let long_dct =
t_dim.max == 4 || msac.decode_bool_adapt(mode.txtp_long32_dct(1)) != 0;
let short_idx = msac.decode_symbol_adapt_n_padded::<3, 4>(
mode.txtp_inter_short_1d(ctx, t_dim.min as usize),
) as usize;
let wh = (t_dim.w < t_dim.h) as usize;
*txtp = TXTP_LONG_TBL[long_dct as usize][wh][short_idx] as u16;
} else if p.reduced_txtp_set == 1 || p.reduced_txtp_set == 2 {
*txtp = if msac.decode_bool_adapt(mode.txtp_inter_dct_idtx(ctx, t_dim.min as usize))
!= 0
{
txtp::DCT_DCT as u16
} else {
txtp::IDTX as u16
};
} else if p.reduced_txtp_set == 3 {
let tx_idx = msac.decode_symbol_adapt_n_padded::<3, 4>(
mode.txtp_inter_dct_idtx_iddct(ctx, t_dim.min as usize),
) as usize;
static TXTP_DCT_IDTX_IDDCT: [u8; 4] =
[txtp::DCT_DCT, txtp::V_DCT, txtp::H_DCT, txtp::IDTX];
*txtp = TXTP_DCT_IDTX_IDDCT[tx_idx] as u16;
} else {
let setidx = (p.tx == 2) as usize;
let set =
msac.decode_bool_adapt(mode.txtp_inter_tx_set(setidx, ctx, t_dim.min as usize))
as usize;
let t = if set == 0 {
msac.decode_symbol_adapt_n_padded::<7, 8>(mode.txtp_inter_set0(setidx, ctx))
as usize
} else if setidx != 0 {
msac.decode_symbol_adapt_n_padded::<3, 4>(mode.txtp_inter_set2(ctx)) as usize
+ 8
} else {
msac.decode_symbol_adapt_n_padded::<7, 8>(mode.txtp_inter_set1(ctx)) as usize
+ 8
};
*txtp = TXTP_INV_TBL[setidx][t] as u16;
}
}
}
let tx_class = txtp::class(*txtp as u8);
let mut stx_type: u32 = 0;
if p.seq_ist[(!p.intra) as usize] && !chroma {
if p.intra {
if eob >= 1
&& p.y_mode != IntraPredMode::PaethPred as usize
&& (*txtp as u8 == txtp::DCT_DCT || *txtp as u8 == txtp::ADST_ADST)
{
let lim = if p.tx == 1 && *txtp as u8 == txtp::DCT_DCT {
20
} else if t_dim.min >= 1 {
if *txtp as u8 == txtp::DCT_DCT { 32 } else { 20 }
} else {
8
};
stx_type = (eob < lim) as u32;
}
} else {
stx_type =
(t_dim.min >= 2 && *txtp as u8 == txtp::DCT_DCT && (3..32).contains(&eob)) as u32;
}
if stx_type != 0 {
stx_type = msac.decode_symbol_adapt_n_padded::<3, 4>(
mode.stx((!p.intra) as usize, t_dim.min as usize),
);
if stx_type != 0 && p.intra {
let mut stx_set: u32;
if t_dim.min >= 1 && *txtp as u8 == txtp::ADST_ADST {
let s = msac.decode_symbol_adapt_n_padded::<3, 4>(mode.stx_set_adst()) as usize;
stx_set = INV_MAP_ADST[p.y_mode][s] as u32;
} else {
let s = msac.decode_symbol_adapt_n_padded::<6, 8>(mode.stx_set()) as usize;
stx_set = INV_MAP[p.y_mode][s] as u32;
}
stx_set += 7 * (*txtp as u8 == txtp::ADST_ADST) as u32;
*txtp |= (stx_set << 10) as u16;
}
*txtp |= (stx_type << 8) as u16;
}
} else if p.seq_cctx
&& p.plane == 1
&& eob >= p.intra as i32
&& !p.lossless
&& (p.layout == PixelLayout::I420 || t_dim.min < 3)
{
let cctx = msac.decode_symbol_adapt_n_padded::<6, 8>(mode.cctx());
*txtp |= (cctx << 8) as u16;
}
CoefHeader::Decode {
eob,
tx_class,
stx_type,
}
}
#[allow(clippy::too_many_arguments)]
#[inline(never)]
#[allow(unused_variables)]
fn decode_coefs_idtx<C: Coeff, const UPDATE_CDF: bool, M: MsacReader<UPDATE_CDF>>(
msac: &mut M,
coef: &mut CdfCoefContext,
p: &DecodeCoefParams,
cf: &mut [C],
txtp: &mut u16,
res_ctx: &mut u8,
levels_scratch: &mut [i8; 1089],
nz_scratch: &mut [u32; 1024],
eob: i32,
slw: usize,
slh: usize,
tx2dszctx: usize,
tcq_en: bool,
dq_shift: i32,
) -> i32 {
let t_dim = &TXFM_DIMENSIONS[p.tx];
let cf_max = !((!127u32) << p.bitdepth) as i32;
let scan = SCANS[p.tx];
let mut cul_level: u32 = 0;
let mut hr_avg: i32 = 0;
let mut dc_sign_level: u32 = 1 << 6;
*txtp = txtp::IDTX_INV as u16;
let stride = 1 + (4 << slh);
let level_len = stride * ((4usize << slw) + 1);
let levels = &mut levels_scratch[..level_len];
levels.fill(0);
let sz_ctx = imin(t_dim.ctx as i32, 2) as usize;
let sz = (16 << tx2dszctx) - 1;
let bob = sz - eob;
let ctx = ((bob > 2 << tx2dszctx) as usize) + ((bob > 4 << tx2dszctx) as usize);
let mut tok =
1 + msac.decode_symbol_adapt_n_padded::<2, 4>(coef.bob_base_y_tok(sz_ctx, ctx)) as i32;
if tok == 3 {
tok += msac.decode_symbol_adapt_n_padded::<3, 4>(coef.br_y_tok_idtx(sz_ctx, 0)) as i32;
}
let shift = slh + 2;
let mask = (4 << slh) - 1;
let rc = scan[bob as usize] as usize;
let x = rc >> shift;
let y = rc & mask;
cf[rc] = C::from_i32(tok);
levels[(1 + x) * stride + (y + 1)] = tok as i8;
for i in (bob + 1)..(sz + 1) {
let rc = scan[i as usize] as usize;
let x = rc >> shift;
let y = rc & mask;
let off = (1 + x) * stride + (1 + y);
let mut hr_ctx = 0u32;
let ctx = get_lo_ctx_idtx(levels, off, &mut hr_ctx, stride);
let mut tok = msac
.decode_symbol_adapt_n_padded::<3, 4>(coef.base_y_tok_idtx(sz_ctx, ctx as usize))
as i32;
if tok == 3 {
tok += msac
.decode_symbol_adapt_n_padded::<3, 4>(coef.br_y_tok_idtx(sz_ctx, hr_ctx as usize))
as i32;
}
cf[rc] = C::from_i32(tok);
levels[off] = tok as i8;
}
let dq = p.dq_tbl[1];
let dq_shift = dq_shift - tcq_en as i32;
for i in bob..(sz + 1) {
let rc = scan[i as usize] as usize;
let tok_val = cf[rc].to_i32();
if tok_val == 0 {
continue;
}
let x = rc >> shift;
let y = rc & mask;
let off = (1 + x) * stride + (1 + y);
let ctx = get_sign_ctx_idtx(levels, off, stride);
let sign = msac.decode_bool_adapt(coef.sign_idtx(sz_ctx, ctx as usize));
if i == 0 {
dc_sign_level = ((sign as i32 - 1) & (2 << 6)) as u32;
}
levels[off] = 1 - 2 * sign as i8;
let mut tok = tok_val;
let val: i32;
if tok >= 6 {
let hr = decode_hr(msac, hr_avg);
tok += hr;
hr_avg = (hr_avg + hr) >> 1;
tok &= 0xfffff;
val = imin(
((((tok as u32).wrapping_mul(dq)) & 0xffffff).wrapping_add(4) >> dq_shift) as i32,
cf_max + sign as i32,
);
} else {
val = ((tok as u32).wrapping_mul(dq).wrapping_add(4) >> dq_shift) as i32;
}
cul_level += tok as u32;
cf[rc] = C::from_i32(if sign != 0 { -val } else { val });
}
*res_ctx = (cul_level.min(63) | dc_sign_level) as u8;
eob
}
#[allow(clippy::too_many_arguments)]
#[inline(never)]
fn decode_coefs_dc_tail<C: Coeff, const UPDATE_CDF: bool, M: MsacReader<UPDATE_CDF>>(
msac: &mut M,
coef: &mut CdfCoefContext,
a: &[u8],
l: &[u8],
p: &DecodeCoefParams,
cf: &mut [C],
res_ctx: &mut u8,
eob: i32,
mut dc_tok: i32,
mut cul_level: u32,
mut dc_sign_level: u32,
hr_avg: i32,
tcq_state: i32,
tcq_en: bool,
has_qm: bool,
dq_shift: i32,
ph_state: u32,
) -> i32 {
let t_dim = &TXFM_DIMENSIONS[p.tx];
let chroma = p.plane != 0;
let cf_max = !((!127u32) << p.bitdepth) as i32;
if dc_tok == 0 {
*res_ctx = (cul_level.min(63) | dc_sign_level) as u8;
return eob;
}
let dc_sign: u32;
if chroma {
dc_sign = msac.decode_bool_bypass();
} else {
let dc_sign_ctx = get_dc_sign_ctx(t_dim, a, l) as usize;
dc_sign = msac.decode_bool_adapt(coef.dc_sign(
chroma as usize,
(ph_state >> 18 != 0) as usize,
dc_sign_ctx,
));
}
let mut dc_dq = p.dq_tbl[0] as i32;
dc_sign_level = ((dc_sign as i32 - 1) & (2 << 6)) as u32;
if has_qm {
dc_dq = (dc_dq * p.qm.unwrap()[0] as i32 + 16) >> 5;
}
let max_br = if chroma { 5 } else { 8 };
let tcq_bit = (tcq_state & 2) >> 1;
let dc_val: i32;
if ph_state >> 18 != 0 && dc_tok > 5 {
let hr = decode_hr(msac, hr_avg >> 1) << 1;
dc_tok += hr;
dc_val = ((dc_tok as u32).wrapping_mul(dc_dq as u32).wrapping_add(4) >> dq_shift) as i32;
} else if dc_tok >= max_br - tcq_en as i32 {
let hr = decode_hr(msac, hr_avg);
dc_tok += hr << tcq_en as i32;
dc_tok &= 0xfffff;
let v = (dc_tok << tcq_en as i32) - tcq_bit;
dc_val = imin(
((((v as u32).wrapping_mul(dc_dq as u32)) & 0xffffff).wrapping_add(4) >> dq_shift)
as i32,
cf_max + dc_sign as i32,
);
} else {
let v = (dc_tok << tcq_en as i32) - tcq_bit;
dc_val = (((v as u32).wrapping_mul(dc_dq as u32)).wrapping_add(4) >> dq_shift) as i32;
}
cul_level += dc_tok as u32;
cf[0] = C::from_i32(if dc_sign != 0 { -dc_val } else { dc_val });
*res_ctx = (cul_level.min(63) | dc_sign_level) as u8;
eob
}
macro_rules! decode_coefs_impl {
(
$msac:expr,
$coef:expr,
$mode:expr,
$a:expr,
$l:expr,
$p:expr,
$cf:expr,
$txtp:expr,
$res_ctx:expr,
$levels_scratch:expr,
$nz_scratch:expr
) => {{
let msac = $msac;
let coef = $coef;
let mode = $mode;
let a = $a;
let l = $l;
let p = $p;
let cf = $cf;
let txtp = $txtp;
let res_ctx = $res_ctx;
let levels_scratch = $levels_scratch;
let nz_scratch = $nz_scratch;
let t_dim = &TXFM_DIMENSIONS[p.tx];
let chroma = p.plane != 0;
let cf_max = !((!127u32) << p.bitdepth) as i32;
let (eob, tx_class, stx_type) =
match decode_coefs_header(msac, coef, mode, a, l, p, txtp, res_ctx) {
CoefHeader::Done(v) => return v,
CoefHeader::Decode {
eob,
tx_class,
stx_type,
} => (eob, tx_class, stx_type),
};
let slw = imin(t_dim.lw as i32, 3) as usize;
let slh = imin(t_dim.lh as i32, 3) as usize;
let tx2dszctx = slw + slh;
let mut cul_level: u32 = 0;
let mut dc_tok: i32;
let tcq_en = p.tcq_enabled && !chroma && tx_class == 0 && !p.lossless;
let mut hr_avg: i32 = 0;
let mut tcq_state: i32 = if tcq_en { -0x80000000i32 } else { 0 };
let ph_enabled =
!chroma && p.parity_hiding && *txtp != txtp::IDTX as u16 && !p.lossless && eob > 3;
let mut ph_state: u32 = if ph_enabled { 2 } else { 0 };
let has_qm = p.qm.is_some()
&& *txtp != txtp::IDTX as u16
&& *txtp != txtp::WHT_WHT as u16
&& tx_class != 2
&& tx_class != 3;
let dq_shift = tcq_en as i32 + 3 + imax(0, t_dim.ctx as i32 - 2);
let dc_sign_level: u32 = 1 << 6;
let scan = SCANS[p.tx];
if p.seq_fsc && (!p.intra || p.fsc) && *txtp as u8 == txtp::IDTX && !chroma {
return decode_coefs_idtx(
msac,
coef,
p,
cf,
txtp,
res_ctx,
levels_scratch,
nz_scratch,
eob,
slw,
slh,
tx2dszctx,
tcq_en,
dq_shift,
);
}
if eob != 0 {
let level_len = match tx_class {
0 => {
let w = 4usize << slw;
let h = 4usize << slh;
(w + 2) * h
}
2 => {
let h = 4usize << slh;
(h + 2) * 32
}
3 => {
let w = 4usize << slw;
(w + 2) * 32
}
_ => unreachable!(),
};
let levels = &mut levels_scratch[..level_len];
levels.fill(0);
let is_stx = stx_type != 0 && tx_class == 0;
macro_rules! decode_coefs_class {
($tx_cl:expr, $stride:expr, $shift:expr, $shift2:expr, $mask:expr, $hi_to_low:expr, $xy_expr:ident) => {{
let hi_to_low_tx: i32 = $hi_to_low;
let stride: usize = $stride;
let shift: usize = $shift;
let shift2: usize = $shift2;
let mask: usize = $mask;
let mut nz_n = 0usize;
let (mut lim, mut tok): (i32, i32);
let (mut hi_base, mut hi_stride): (usize, usize);
let mut hi_cdf_valid: bool = true;
let ctx_init = 1 + (eob > 2 << tx2dszctx) as u32 + (eob > 4 << tx2dszctx) as u32;
if eob >= hi_to_low_tx {
lim = 3;
if !chroma {
tok = 1 + msac.decode_symbol_adapt_n_padded::<2, 4>(
coef.eob_base_y_tok_hf(t_dim.ctx as usize, ctx_init as usize),
) as i32;
hi_base = 1252;
hi_stride = 4;
} else {
tok = 1 + msac
.decode_symbol_adapt_n_padded::<2, 4>(coef.eob_base_uv_tok_hf(ctx_init as usize))
as i32;
hi_base = 4508;
hi_stride = 4;
}
hi_cdf_valid = true;
} else {
lim = 5;
if !chroma {
tok = 1 + msac.decode_symbol_adapt_n_padded::<4, 8>(
coef.eob_base_y_tok_lf(t_dim.ctx as usize, ctx_init as usize),
) as i32;
hi_base = 4080;
hi_stride = 4;
} else {
tok = 1 + msac
.decode_symbol_adapt_n_padded::<4, 8>(coef.eob_base_uv_tok_lf(ctx_init as usize))
as i32;
hi_base = 0;
hi_stride = 0;
hi_cdf_valid = false;
}
if chroma {
hi_cdf_valid = false;
}
}
let (mut rc, mut x, mut y): (usize, usize, usize);
if $tx_cl == 0 {
rc = scan[eob as usize] as usize;
x = rc >> shift;
y = rc & mask;
} else if $tx_cl == 1 {
x = eob as usize & mask;
y = eob as usize >> shift;
rc = eob as usize;
} else {
x = eob as usize & mask;
y = eob as usize >> shift;
rc = (x << shift2) | y;
}
if tok == lim && hi_cdf_valid {
let hi_idx = if lim == 5 { 7 } else { 0 };
let o = hi_base + hi_idx * hi_stride;
tok += msac
.decode_symbol_adapt_n_padded::<3, 4>(cdf_array_mut::<4>(
&mut coef.data,
o,
)) as i32;
if ph_state != 0 {
ph_state += 0x10000 + tok as u32 + (tok == 7) as u32;
}
} else if ph_state != 0 {
ph_state += 0x10000 + tok as u32;
}
if tcq_state != 0 {
tcq_state = tcq_next_state(tcq_state, tok);
}
cf[if is_stx { eob as usize } else { rc }] = C::from_i32(tok);
nz_scratch[nz_n] = (if is_stx { eob as usize } else { rc }) as u32
| (((eob < hi_to_low_tx) as u32) << 14)
| ((($tx_cl != 0 && y == 0 && !chroma) as u32) << 15)
| ((tok as u32) << 16);
nz_n += 1;
let level_tok = pack_level(tok);
if $tx_cl == 0 {
levels[rc] = level_tok;
} else {
levels[x * stride + y] = level_tok;
}
macro_rules! ac_tokens {
($first:expr, $last:expr, $seg_lim:literal, $seg_nsym:literal,
$seg_lo_base:expr, $seg_lo_stride:literal, $seg_hi_base:expr,
$seg_hi_valid:expr) => {
let mut i: i32 = $first;
let last: i32 = $last;
while i >= last {
if $tx_cl == 0 {
rc = scan[i as usize] as usize;
x = rc >> shift;
y = rc & mask;
} else if $tx_cl == 1 {
x = i as usize & mask;
y = i as usize >> shift;
rc = i as usize;
} else {
x = i as usize & mask;
y = i as usize >> shift;
rc = (x << shift2) | y;
}
let off = if $tx_cl == 0 { rc } else { x * stride + y };
let mut hr_ctx = 0u32;
let xy_val: u32 = if $tx_cl == 0 {
(x + y) as u32
} else {
y as u32
};
let base = &levels[off..];
let ctx = if !chroma {
if $tx_cl == 0 {
get_lo_ctx_luma_2d(base, &mut hr_ctx, xy_val, stride)
} else {
get_lo_ctx_luma_1d(base, &mut hr_ctx, xy_val, stride)
}
} else if $tx_cl == 0 {
get_lo_ctx_chroma_2d(base, &mut hr_ctx, xy_val, p.plane, stride)
} else {
get_lo_ctx_chroma_1d(base, &mut hr_ctx, xy_val, stride)
};
let tcq_bit = ((tcq_state & 2) >> 1) as u32;
let lo_cdf_idx = (ctx * (2 - chroma as u32) + tcq_bit) as usize;
let o = $seg_lo_base + lo_cdf_idx * $seg_lo_stride;
let mut tok = if $seg_nsym == 3 {
msac.decode_symbol_adapt_n_padded::<3, 4>(cdf_array_mut::<4>(
&mut coef.data,
o,
)) as i32
} else {
msac.decode_symbol_adapt_n_padded::<5, 8>(cdf_array_mut::<8>(
&mut coef.data,
o,
)) as i32
};
if tok == $seg_lim && $seg_hi_valid {
let o2 = $seg_hi_base + hr_ctx as usize * 4;
tok += msac
.decode_symbol_adapt_n_padded::<3, 4>(cdf_array_mut::<4>(
&mut coef.data,
o2,
)) as i32;
if ph_state != 0 {
ph_state += 0x10000 + tok as u32 + (tok == 7) as u32;
}
} else if ph_state != 0 {
ph_state =
ph_state.wrapping_add((tok as u32).wrapping_neg() & 0x10001);
}
if tcq_state != 0 {
tcq_state = tcq_next_state(tcq_state, tok);
}
levels[off] = pack_level(tok);
cf[if is_stx { i as usize } else { rc }] = C::from_i32(tok);
if tok != 0 {
nz_scratch[nz_n] = (if is_stx { i as usize } else { rc }) as u32
| (tcq_bit << 13)
| (((i < hi_to_low_tx) as u32) << 14)
| ((($tx_cl != 0 && y == 0 && !chroma) as u32) << 15)
| ((tok as u32) << 16);
nz_n += 1;
}
i -= 1;
}
};
}
let hf_last = hi_to_low_tx.max(1);
if eob >= hi_to_low_tx {
if !chroma {
ac_tokens!(eob - 1, hf_last, 3, 3, 452 + (t_dim.ctx as usize) * 160, 4, 1252, true);
} else {
ac_tokens!(eob - 1, hf_last, 3, 3, 4460, 4, 4508, true);
}
}
lim = 5;
let (lo_base, lo_stride, lo_nsym): (usize, usize, usize);
if !chroma {
lo_base = 1440 + (t_dim.ctx as usize) * 528;
lo_stride = 8;
lo_nsym = 5;
hi_base = 4080;
hi_stride = 4;
hi_cdf_valid = true;
} else {
lo_base = 4560;
lo_stride = 8;
lo_nsym = 5;
hi_base = 0;
hi_stride = 0;
hi_cdf_valid = false;
}
{
let lf_first = (eob - 1).min(hi_to_low_tx - 1);
if !chroma {
ac_tokens!(lf_first, 1, 5, 5, 1440 + (t_dim.ctx as usize) * 528, 8, 4080, true);
} else {
ac_tokens!(lf_first, 1, 5, 5, 4560, 8, 0, false);
}
}
if ph_state >> 18 != 0 {
let ctx = get_ph_ctx(levels, $tx_cl == 0, stride);
let idx =
msac.decode_symbol_adapt_n_padded::<3, 4>(coef.ph_dc_y_tok(ctx)) as i32;
dc_tok = (idx << 1) + (ph_state & 1) as i32;
} else {
let mut hr_ctx = 0u32;
let ctx = if !chroma {
if $tx_cl == 0 {
get_lo_ctx_luma_2d(levels, &mut hr_ctx, 0, stride)
} else {
get_lo_ctx_luma_1d(levels, &mut hr_ctx, 0, stride)
}
} else if $tx_cl == 0 {
get_lo_ctx_chroma_2d(levels, &mut hr_ctx, 0, p.plane, stride)
} else {
get_lo_ctx_chroma_1d(levels, &mut hr_ctx, 0, stride)
};
let tcq_bit = ((tcq_state & 2) >> 1) as u32;
let lo_cdf_idx = (ctx * (2 - chroma as u32) + tcq_bit) as usize;
let o = lo_base + lo_cdf_idx * lo_stride;
dc_tok = if lo_nsym == 3 {
msac.decode_symbol_adapt_n_padded::<3, 4>(cdf_array_mut::<4>(
&mut coef.data,
o,
)) as i32
} else {
debug_assert_eq!(lo_nsym, 5);
msac.decode_symbol_adapt_n_padded::<5, 8>(cdf_array_mut::<8>(
&mut coef.data,
o,
)) as i32
};
if dc_tok == lim && hi_cdf_valid {
let o2 = hi_base + hr_ctx as usize * hi_stride;
dc_tok += msac
.decode_symbol_adapt_n_padded::<3, 4>(cdf_array_mut::<4>(
&mut coef.data,
o2,
)) as i32;
}
}
let ac_dq = p.dq_tbl[1];
let mut j = 0usize;
while j < nz_n {
let mut k = 0usize;
let mut hr_term = false;
while j + k < nz_n && k < 16 {
let rec = nz_scratch[j + k];
if rec & (1 << 15) != 0 {
break;
}
let tok = (rec >> 16) as i32;
let max_br = if rec & (1 << 14) != 0 {
if chroma { 5 } else { 8 }
} else {
6
};
k += 1;
if tok >= max_br - tcq_en as i32 {
hr_term = true;
break;
}
}
let (n, sign_bits) = if k == 0 {
let rec = nz_scratch[j];
let tok = (rec >> 16) as i32;
let max_br = if rec & (1 << 14) != 0 { 8 } else { 6 };
hr_term = tok >= max_br - tcq_en as i32;
let s = msac.decode_bool_adapt(coef.dc_sign(
chroma as usize,
(ph_state >> 18 != 0) as usize,
0,
));
(1usize, s)
} else {
(k, msac.decode_bools_bypass(k as u32))
};
for t in 0..n {
let rec = nz_scratch[j + t];
let sign = (sign_bits >> (n - 1 - t)) & 1;
let rc = (rec & 0x1fff) as usize;
let mut tok = (rec >> 16) as i32;
let tcq_bit = ((rec >> 13) & 1) as i32;
let ac_dq = match p.qm {
Some(qm) if has_qm => {
let qi = if is_stx { scan[rc] as usize } else { rc };
(ac_dq * qm[qi] as u32 + 16) >> 5
}
_ => ac_dq,
};
let ac_val: i32;
if hr_term && t == n - 1 {
let hr = decode_hr(msac, hr_avg);
tok += hr << tcq_en as i32;
hr_avg = (hr_avg + hr) >> 1;
tok &= 0xfffff;
let v = (tok << tcq_en as i32) - tcq_bit;
ac_val = imin(
((((v as u32).wrapping_mul(ac_dq)) & 0xffffff)
.wrapping_add(4)
>> dq_shift) as i32,
cf_max + sign as i32,
);
} else {
let v = (tok << tcq_en as i32) - tcq_bit;
ac_val = (((v as u32).wrapping_mul(ac_dq)).wrapping_add(4)
>> dq_shift) as i32;
}
cul_level += tok as u32;
cf[rc] = C::from_i32(if sign != 0 { -ac_val } else { ac_val });
}
j += n;
}
}};
}
match tx_class {
0 => {
let stride = (4 << slh) as usize;
let shift = slh + 2;
let mask = (4 << slh) - 1;
let hi_to_low = if chroma { 1i32 } else { 10 };
decode_coefs_class!(0, stride, shift, 0, mask, hi_to_low, xy_2d);
}
2 => {
let stride = 32usize;
let shift = slh + 2;
let mask = (4 << slh) - 1;
let hi_to_low = (8 << slh) >> chroma as usize;
decode_coefs_class!(1, stride, shift, 0, mask, hi_to_low, xy_h);
}
3 => {
let stride = 32usize;
let shift = slw + 2;
let shift2 = slh + 2;
let mask = (4 << slw) - 1;
let hi_to_low = (8 << slw) >> chroma as usize;
decode_coefs_class!(2, stride, shift, shift2, mask, hi_to_low, xy_v);
}
_ => unreachable!(),
}
} else if chroma {
dc_tok = 1 + msac.decode_symbol_adapt_n_padded::<4, 8>(coef.eob_base_uv_tok_lf(0)) as i32;
} else {
dc_tok = 1 + msac.decode_symbol_adapt_n_padded::<4, 8>(coef.eob_base_y_tok_lf(t_dim.ctx as usize, 0))
as i32;
if dc_tok == 5 {
let hi_idx = if tx_class == 0 { 0 } else { 7 };
dc_tok += msac.decode_symbol_adapt_n_padded::<3, 4>(coef.br_y_tok_lf(hi_idx)) as i32;
}
}
decode_coefs_dc_tail(
msac,
coef,
a,
l,
p,
cf,
res_ctx,
eob,
dc_tok,
cul_level,
dc_sign_level,
hr_avg,
tcq_state,
tcq_en,
has_qm,
dq_shift,
ph_state,
)
}};
}
#[allow(dead_code)]
#[inline(always)]
pub(crate) fn decode_coefs<C: Coeff, const UPDATE_CDF: bool, M: MsacReader<UPDATE_CDF>>(
msac: &mut M,
coef: &mut CdfCoefContext,
mode: &mut CdfModeContext,
a: &[u8],
l: &[u8],
p: &DecodeCoefParams,
cf: &mut [C],
txtp: &mut u16,
res_ctx: &mut u8,
levels_scratch: &mut [i8; 1089],
nz_scratch: &mut [u32; 1024],
) -> i32 {
decode_coefs_impl!(
msac,
coef,
mode,
a,
l,
p,
cf,
txtp,
res_ctx,
levels_scratch,
nz_scratch
)
}
#[inline(always)]
pub(crate) fn decode_coefs_scalar<C: Coeff, const UPDATE_CDF: bool>(
msac: &mut crate::msac::MsacContextScalar<'_, UPDATE_CDF>,
coef: &mut CdfCoefContext,
mode: &mut CdfModeContext,
a: &[u8],
l: &[u8],
p: &DecodeCoefParams,
cf: &mut [C],
txtp: &mut u16,
res_ctx: &mut u8,
levels_scratch: &mut [i8; 1089],
nz_scratch: &mut [u32; 1024],
) -> i32 {
decode_coefs_impl!(
msac,
coef,
mode,
a,
l,
p,
cf,
txtp,
res_ctx,
levels_scratch,
nz_scratch
)
}
#[cfg(all(target_arch = "x86_64", feature = "sse"))]
#[target_feature(enable = "sse2")]
pub(crate) fn decode_coefs_sse2<C: Coeff, const UPDATE_CDF: bool>(
msac: &mut crate::sse::MsacContextSse<'_, UPDATE_CDF>,
coef: &mut CdfCoefContext,
mode: &mut CdfModeContext,
a: &[u8],
l: &[u8],
p: &DecodeCoefParams,
cf: &mut [C],
txtp: &mut u16,
res_ctx: &mut u8,
levels_scratch: &mut [i8; 1089],
nz_scratch: &mut [u32; 1024],
) -> i32 {
decode_coefs_impl!(
msac,
coef,
mode,
a,
l,
p,
cf,
txtp,
res_ctx,
levels_scratch,
nz_scratch
)
}
#[allow(clippy::too_many_arguments)]
#[inline]
fn intrabc_put_bilin<BD: crate::pixel::BitDepth>(
bd: BD,
exec: &crate::exec_context::ExecContext,
dst: &mut [BD::Pixel],
dst_stride: usize,
src: &[BD::Pixel],
src_stride: usize,
w: usize,
h: usize,
mx: i32,
my: i32,
) {
use crate::pixel::Pixel;
if BD::BPC == 8 {
if let (Some(dst8), Some(src8)) = (
<BD::Pixel as Pixel>::try_as_u8_slice_mut(dst),
<BD::Pixel as Pixel>::try_as_u8_slice(src),
) {
let mut scratch = Vec::new();
exec.put_bilin_8bpc_with_scratch(
dst8,
dst_stride,
src8,
src_stride,
w,
h,
mx,
my,
&mut scratch,
);
return;
}
} else if BD::BPC == 16 {
if let (Some(dst16), Some(src16)) = (
<BD::Pixel as Pixel>::try_as_u16_slice_mut(dst),
<BD::Pixel as Pixel>::try_as_u16_slice(src),
) {
let mut scratch = Vec::new();
exec.put_bilin_hbd_with_scratch(
dst16,
dst_stride,
src16,
src_stride,
w,
h,
mx,
my,
bd.bitdepth(),
&mut scratch,
);
return;
}
}
crate::mc::put_bilin(bd, dst, dst_stride, src, src_stride, w, h, mx, my);
}
#[allow(clippy::too_many_arguments)]
#[inline]
fn intrabc_gather_src<P: crate::pixel::Pixel>(
plane: &[P],
stride: usize,
srcbuf: &mut [P],
src_stride: usize,
src_w: usize,
src_h: usize,
sx: i32,
sy: i32,
right: i32,
bottom: i32,
) {
debug_assert!(src_stride >= src_w);
debug_assert!(srcbuf.len() >= src_stride * src_h);
let bottom = bottom.max(1);
let src_w_i32 = src_w as i32;
for (ry, dst_row) in srcbuf.chunks_mut(src_stride).take(src_h).enumerate() {
let cy = (sy + ry as i32).clamp(0, bottom - 1) as usize;
let row_start = cy.saturating_mul(stride).min(plane.len());
let row_end = row_start.saturating_add(stride).min(plane.len());
let src_row = &plane[row_start..row_end];
if src_row.is_empty() {
dst_row[..src_w].fill(P::default());
continue;
}
let row_right = right.max(1).min(src_row.len() as i32);
let left_ext = (-sx).clamp(0, src_w_i32) as usize;
let right_ext = (sx + src_w_i32 - row_right).clamp(0, src_w_i32 - left_ext as i32) as usize;
let center_w = src_w - left_ext - right_ext;
if left_ext != 0 {
dst_row[..left_ext].fill(src_row[0]);
}
if center_w != 0 {
let src_x = (sx + left_ext as i32).clamp(0, row_right - 1) as usize;
dst_row[left_ext..left_ext + center_w]
.copy_from_slice(&src_row[src_x..src_x + center_w]);
}
if right_ext != 0 {
dst_row[src_w - right_ext..src_w].fill(src_row[row_right as usize - 1]);
}
}
}
#[allow(clippy::too_many_arguments)]
pub(crate) fn intrabc_pred<BD: crate::pixel::BitDepth>(
bd: BD,
exec: &crate::exec_context::ExecContext,
plane: &mut [BD::Pixel],
stride: usize,
bw4: i32,
bh4: i32,
bx: i32,
by: i32,
mvx: i32,
mvy: i32,
ss_hor: i32,
ss_ver: i32,
right: i32,
bottom: i32,
) {
let h_mul = 4 >> ss_hor;
let v_mul = 4 >> ss_ver;
let mx_lo = mvx & (15 >> (ss_hor == 0) as i32);
let my_lo = mvy & (15 >> (ss_ver == 0) as i32);
let mx = mx_lo << (ss_hor == 0) as i32;
let my = my_lo << (ss_ver == 0) as i32;
let sx = bx * h_mul + (mvx >> (3 + ss_hor));
let sy = by * v_mul + (mvy >> (3 + ss_ver));
let dpx = bx * h_mul;
let dpy = by * v_mul;
let w = (bw4 * h_mul) as usize;
let h = (bh4 * v_mul) as usize;
let src_w = w + usize::from(mx_lo != 0);
let src_h = h + usize::from(my_lo != 0);
let src_stride = src_w;
let mut srcbuf = vec![BD::Pixel::default(); src_stride * src_h];
intrabc_gather_src(
plane,
stride,
&mut srcbuf,
src_stride,
src_w,
src_h,
sx,
sy,
right,
bottom,
);
let dst_off = (dpy as usize) * stride + dpx as usize;
if dst_off >= plane.len() {
return;
}
intrabc_put_bilin(
bd,
exec,
&mut plane[dst_off..],
stride,
&srcbuf,
src_stride,
w,
h,
mx,
my,
);
}
#[cfg(test)]
mod packed_level_ctx_tests {
use super::*;
fn l5(x: u32) -> u32 {
x.min(5)
}
fn l3(x: u32) -> u32 {
x.min(3)
}
fn ref_luma_2d(t: &dyn Fn(usize) -> u32, hm: &mut u32, xy: u32, st: usize) -> u32 {
let (a, b, c, d, e) = (t(1), t(st), t(st + 1), t(2), t(2 * st));
let hi = l5(a) + l5(b) + l5(c);
let (lo, offset, lim);
if xy < 4 {
lo = l5(a) + l5(b) + l5(c) + l5(d) + l5(e);
(offset, lim) = if xy == 0 {
(0, 8)
} else if xy < 2 {
(9, 6)
} else {
(16, 4)
};
*hm = if xy > 0 { 7 } else { 0 } + ((hi + 1) >> 1).min(6);
} else {
lo = l3(a) + l3(b) + l3(c) + l3(d) + l3(e);
offset = if xy < 6 {
0
} else if xy < 8 {
5
} else {
10
};
lim = 4;
*hm = ((hi + 1) >> 1).min(6);
}
offset + ((lo + 1) >> 1).min(lim)
}
fn ref_luma_1d(t: &dyn Fn(usize) -> u32, hm: &mut u32, xy: u32, st: usize) -> u32 {
let (a, b, c, d, e) = (t(1), t(st), t(2), t(3), t(4));
let hi = l5(a) + l5(b) + l5(c);
let (lo, offset, lim);
if xy < 2 {
lo = l5(a) + l5(b) + l3(c) + l3(d) + l3(e);
(offset, lim) = if xy == 0 { (21, 6) } else { (28, 4) };
*hm = 7 + ((hi + 1) >> 1).min(6);
} else {
lo = l3(a) + l3(b) + l3(c) + l3(d) + l3(e);
(offset, lim) = (15, 4);
*hm = ((hi + 1) >> 1).min(6);
}
offset + ((lo + 1) >> 1).min(lim)
}
fn ref_chroma_2d(
t: &dyn Fn(usize) -> u32,
hm: &mut u32,
xy: u32,
plane: i32,
st: usize,
) -> u32 {
let (a, b, c) = (t(1), t(st), t(st + 1));
let hi = l5(a) + l5(b) + l5(c);
let lo = if xy == 0 { hi } else { l3(a) + l3(b) + l3(c) };
*hm = ((hi + 1) >> 1).min(3);
(if plane == 1 { 0 } else { 4 }) + ((lo + 1) >> 1).min(3)
}
fn ref_chroma_1d(t: &dyn Fn(usize) -> u32, hm: &mut u32, xy: u32, st: usize) -> u32 {
let (a, b) = (t(1), t(st));
let hi = l5(a) + l5(b);
let lo = if xy == 0 { hi } else { l3(a) + l3(b) };
*hm = ((hi + 1) >> 1).min(3);
8 + ((lo + 1) >> 1).min(3)
}
fn ref_ph(t: &dyn Fn(usize) -> u32, is_2d: bool, st: usize) -> usize {
let mut m = l3(t(1)) + l3(t(2)) + l3(t(st));
if is_2d {
m += l3(t(st + 1)) + l3(t(2 * st));
} else {
m += l3(t(3)) + l3(t(4));
}
if m < 7 { ((m + 1) >> 1) as usize } else { 4 }
}
#[test]
fn packed_ctx_matches_reference() {
let mut s: u64 = 0xc0ff_ee15_600d;
let mut next = move || {
s ^= s << 13;
s ^= s >> 7;
s ^= s << 17;
s
};
for _ in 0..5_000 {
let st = [4usize, 8, 16, 32][(next() % 4) as usize];
let mut toks = [0u32; 128];
let mut lv = [0i8; 128];
for i in 0..(2 * st + 2).min(128) {
toks[i] = (next() % 9) as u32;
lv[i] = pack_level(toks[i] as i32);
}
let t = |i: usize| toks[i];
for xy in 0..13u32 {
let (mut hm_r, mut hm_n) = (0u32, 0u32);
assert_eq!(
ref_luma_2d(&t, &mut hm_r, xy, st),
get_lo_ctx_luma_2d(&lv, &mut hm_n, xy, st)
);
assert_eq!(hm_r, hm_n);
assert_eq!(
ref_luma_1d(&t, &mut hm_r, xy, st.max(32)),
get_lo_ctx_luma_1d(&lv[..64], &mut hm_n, xy, st.max(32))
);
assert_eq!(hm_r, hm_n);
for plane in 1..=2 {
assert_eq!(
ref_chroma_2d(&t, &mut hm_r, xy, plane, st),
get_lo_ctx_chroma_2d(&lv, &mut hm_n, xy, plane, st)
);
assert_eq!(hm_r, hm_n);
}
assert_eq!(
ref_chroma_1d(&t, &mut hm_r, xy, st),
get_lo_ctx_chroma_1d(&lv, &mut hm_n, xy, st)
);
assert_eq!(hm_r, hm_n);
}
assert_eq!(ref_ph(&t, true, st), get_ph_ctx(&lv, true, st));
assert_eq!(ref_ph(&t, false, st), get_ph_ctx(&lv, false, st));
}
}
}