const NORM_ADJUST: [[i32; 3]; 6] = [
[10, 16, 13],
[11, 18, 14],
[13, 20, 16],
[14, 23, 18],
[16, 25, 20],
[18, 29, 23],
];
const QUANT_MF: [[i32; 3]; 6] = [
[13107, 5243, 8066],
[11916, 4660, 7490],
[10082, 4194, 6554],
[9362, 3647, 5825],
[8192, 3355, 5243],
[7282, 2893, 4559],
];
#[inline]
fn class(i: usize, j: usize) -> usize {
match (i % 2, j % 2) {
(0, 0) => 0,
(1, 1) => 1,
_ => 2,
}
}
#[inline]
fn level_scale(m: usize, i: usize, j: usize) -> i32 {
16 * NORM_ADJUST[m][class(i, j)]
}
pub fn forward4(block: &[i32; 16]) -> [i32; 16] {
let mut t = [0i32; 16];
for r in 0..4 {
let (a, b, c, d) = (
block[r * 4],
block[r * 4 + 1],
block[r * 4 + 2],
block[r * 4 + 3],
);
let (s0, s1, s2, s3) = (a + d, b + c, b - c, a - d);
t[r * 4] = s0 + s1;
t[r * 4 + 1] = 2 * s3 + s2;
t[r * 4 + 2] = s0 - s1;
t[r * 4 + 3] = s3 - 2 * s2;
}
let mut o = [0i32; 16];
for c in 0..4 {
let (a, b, cc, d) = (t[c], t[4 + c], t[8 + c], t[12 + c]);
let (s0, s1, s2, s3) = (a + d, b + cc, b - cc, a - d);
o[c] = s0 + s1;
o[4 + c] = 2 * s3 + s2;
o[8 + c] = s0 - s1;
o[12 + c] = s3 - 2 * s2;
}
o
}
pub fn inverse4(coeff: &[i32; 16]) -> [i32; 16] {
let mut t = [0i32; 16];
for r in 0..4 {
let (d0, d1, d2, d3) = (
coeff[r * 4],
coeff[r * 4 + 1],
coeff[r * 4 + 2],
coeff[r * 4 + 3],
);
let e0 = d0 + d2;
let e1 = d0 - d2;
let e2 = (d1 >> 1) - d3;
let e3 = d1 + (d3 >> 1);
t[r * 4] = e0 + e3;
t[r * 4 + 1] = e1 + e2;
t[r * 4 + 2] = e1 - e2;
t[r * 4 + 3] = e0 - e3;
}
let mut o = [0i32; 16];
for c in 0..4 {
let (d0, d1, d2, d3) = (t[c], t[4 + c], t[8 + c], t[12 + c]);
let e0 = d0 + d2;
let e1 = d0 - d2;
let e2 = (d1 >> 1) - d3;
let e3 = d1 + (d3 >> 1);
o[c] = (e0 + e3 + 32) >> 6;
o[4 + c] = (e1 + e2 + 32) >> 6;
o[8 + c] = (e1 - e2 + 32) >> 6;
o[12 + c] = (e0 - e3 + 32) >> 6;
}
o
}
const MAX_DEQUANT: i64 = 32767 / 4;
fn max_level(qp: i32, i: usize, j: usize) -> i64 {
let ls = level_scale((qp % 6) as usize, i, j) as i64;
let shift = qp / 6;
if shift >= 4 {
(MAX_DEQUANT >> (shift - 4)) / ls
} else {
((MAX_DEQUANT << (4 - shift)) - (1 << (3 - shift))) / ls
}
}
pub fn quant4(coeff: &[i32; 16], qp: i32, intra: bool) -> [i32; 16] {
let m = (qp % 6) as usize;
let qbits = 15 + qp / 6;
let f = (1i64 << qbits) / if intra { 3 } else { 6 };
let mut out = [0i32; 16];
for i in 0..4 {
for j in 0..4 {
let k = i * 4 + j;
let c = coeff[k] as i64;
let level =
((c.abs() * QUANT_MF[m][class(i, j)] as i64 + f) >> qbits).min(max_level(qp, i, j));
out[k] = if c < 0 { -(level as i32) } else { level as i32 };
}
}
out
}
pub fn dequant4(levels: &[i32; 16], qp: i32) -> [i32; 16] {
let m = (qp % 6) as usize;
let shift = qp / 6;
let mut out = [0i32; 16];
for i in 0..4 {
for j in 0..4 {
let k = i * 4 + j;
let ls = level_scale(m, i, j);
out[k] = if shift >= 4 {
(levels[k] * ls) << (shift - 4)
} else {
let add = 1 << (3 - shift);
(levels[k] * ls + add) >> (4 - shift)
};
}
}
out
}
pub fn fit_chroma_dc_levels(levels: &mut [i32; 4], qp: i32) {
for _ in 0..24 {
let worst = chroma_dc_inverse(levels, qp)
.iter()
.map(|v| (*v as i64).abs())
.max()
.unwrap_or(0);
if worst <= MAX_DEQUANT {
return;
}
for l in levels.iter_mut() {
*l -= l.signum();
}
}
}
pub fn luma_dc_hadamard(dc: &[i32; 16]) -> [i32; 16] {
let bfly = |a: i32, b: i32, c: i32, d: i32| {
let (s01, d01, s23, d23) = (a + b, a - b, c + d, c - d);
(s01 + s23, s01 - s23, d01 - d23, d01 + d23)
};
let mut t = [0i32; 16];
for r in 0..4 {
let (o0, o1, o2, o3) = bfly(dc[r * 4], dc[r * 4 + 1], dc[r * 4 + 2], dc[r * 4 + 3]);
t[r * 4] = o0;
t[r * 4 + 1] = o1;
t[r * 4 + 2] = o2;
t[r * 4 + 3] = o3;
}
let mut o = [0i32; 16];
for c in 0..4 {
let (o0, o1, o2, o3) = bfly(t[c], t[4 + c], t[8 + c], t[12 + c]);
o[c] = o0;
o[4 + c] = o1;
o[8 + c] = o2;
o[12 + c] = o3;
}
o
}
pub fn luma_dc_inverse(levels: &[i32; 16], qp: i32) -> [i32; 16] {
let f = luma_dc_hadamard(levels);
let ls = level_scale((qp % 6) as usize, 0, 0);
let shift = qp / 6;
let mut out = [0i32; 16];
for k in 0..16 {
out[k] = if shift >= 6 {
(f[k] * ls) << (shift - 6)
} else {
(f[k] * ls + (1 << (5 - shift))) >> (6 - shift)
};
}
out
}
pub fn fit_luma_dc_levels(levels: &mut [i32; 16], qp: i32) {
for _ in 0..64 {
let worst = luma_dc_inverse(levels, qp)
.iter()
.map(|v| (*v as i64).abs())
.max()
.unwrap_or(0);
if worst <= MAX_DEQUANT {
return;
}
for l in levels.iter_mut() {
*l -= l.signum();
}
}
}
pub fn chroma_dc_forward(dc: &[i32; 4]) -> [i32; 4] {
let (a, b, c, d) = (dc[0], dc[1], dc[2], dc[3]);
[a + b + c + d, a - b + c - d, a + b - c - d, a - b - c + d]
}
pub fn chroma_dc_inverse(levels: &[i32; 4], qp: i32) -> [i32; 4] {
let (a, b, c, d) = (levels[0], levels[1], levels[2], levels[3]);
let f = [a + b + c + d, a - b + c - d, a + b - c - d, a - b - c + d];
let m = (qp % 6) as usize;
let ls = level_scale(m, 0, 0);
let mut out = [0i32; 4];
for k in 0..4 {
out[k] = ((f[k] * ls) << (qp / 6)) >> 5;
}
out
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn inverse_of_zero_is_zero() {
assert_eq!(inverse4(&[0; 16]), [0; 16]);
assert_eq!(quant4(&[0; 16], 26, true), [0; 16]);
assert_eq!(dequant4(&[0; 16], 26), [0; 16]);
}
#[test]
fn dc_only_reconstructs_flat() {
let mut c = [0i32; 16];
c[0] = 64 * 16; let r = inverse4(&c);
assert!(r.iter().all(|&v| v == r[0]), "not flat: {r:?}");
assert_eq!(r[0], 16);
}
#[test]
fn roundtrip_is_close_at_low_qp() {
let orig: [i32; 16] = [12, -7, 3, 0, -5, 9, -2, 1, 4, -1, 6, -3, 0, 2, -4, 8];
let levels = quant4(&forward4(&orig), 10, true);
let back = inverse4(&dequant4(&levels, 10));
for (a, b) in orig.iter().zip(&back) {
assert!((a - b).abs() <= 3, "drifted: {orig:?} vs {back:?}");
}
}
#[test]
fn higher_qp_zeroes_more() {
let orig: [i32; 16] = [40, -18, 9, 2, -14, 22, -6, 3, 11, -4, 15, -8, 1, 5, -9, 20];
let c = forward4(&orig);
let low = quant4(&c, 12, true).iter().filter(|&&v| v != 0).count();
let high = quant4(&c, 40, true).iter().filter(|&&v| v != 0).count();
assert!(high <= low, "qp 40 kept {high}, qp 12 kept {low}");
}
#[test]
fn quantized_levels_never_overflow_dequantization() {
for qp in 0..=51 {
let coeff = [20000i32; 16];
for &c in &[coeff, [-20000; 16], [32767; 16]] {
let levels = quant4(&c, qp, true);
let d = dequant4(&levels, qp);
for (k, &v) in d.iter().enumerate() {
assert!(
(v as i64).abs() <= MAX_DEQUANT,
"qp {qp}, position {k}: dequantized to {v}, over the \
inverse-transform bound {MAX_DEQUANT}"
);
}
for r in 0..4 {
let (d0, d1, d2, d3) = (d[r * 4], d[r * 4 + 1], d[r * 4 + 2], d[r * 4 + 3]);
for v in [d0 + d2 + d1 + (d3 / 2), (d0 - d2) + ((d1 / 2) - d3)] {
assert!(
(-32768..=32767).contains(&v),
"qp {qp}, row {r}: inverse-transform intermediate {v}"
);
}
}
}
}
}
#[test]
fn luma_dc_hadamard_is_an_involution_up_to_scale() {
let dc: [i32; 16] = [5, -3, 2, 7, 11, -8, 0, 4, -6, 9, 1, -2, 3, 3, -5, 10];
let twice = luma_dc_hadamard(&luma_dc_hadamard(&dc));
for k in 0..16 {
assert_eq!(twice[k], 16 * dc[k], "position {k}");
}
}
#[test]
fn luma_dc_matrix_is_the_one_in_the_spec() {
const H: [[i32; 4]; 4] = [[1, 1, 1, 1], [1, 1, -1, -1], [1, -1, -1, 1], [1, -1, 1, -1]];
for r in 0..4 {
for c in 0..4 {
let mut imp = [0i32; 16];
imp[r * 4 + c] = 1;
let got = luma_dc_hadamard(&imp);
for i in 0..4 {
for j in 0..4 {
assert_eq!(
got[i * 4 + j],
H[i][r] * H[c][j],
"impulse at ({r},{c}), output ({i},{j})"
);
}
}
}
}
}
#[test]
fn luma_dc_flat_stays_flat() {
let f = luma_dc_hadamard(&[7; 16]);
assert_eq!(f[0], 16 * 7, "DC of a flat field");
assert!(f[1..].iter().all(|&v| v == 0), "no AC from a flat field");
let rec = luma_dc_inverse(&[3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 28);
assert!(
rec.iter().all(|&v| v == rec[0]),
"flat DC level -> flat output"
);
}
#[test]
fn chroma_dc_hadamard_is_its_own_shape() {
let dc = [5i32, -3, 2, 7];
let f = chroma_dc_forward(&dc);
let again = chroma_dc_forward(&f);
for k in 0..4 {
assert_eq!(again[k], 4 * dc[k]);
}
}
#[test]
fn level_scale_follows_position_class() {
for (m, adj) in NORM_ADJUST.iter().enumerate() {
assert_eq!(level_scale(m, 0, 0), 16 * adj[0]);
assert_eq!(level_scale(m, 1, 1), 16 * adj[1]);
assert_eq!(level_scale(m, 0, 1), 16 * adj[2]);
assert_eq!(level_scale(m, 2, 0), 16 * adj[0]);
}
}
}