use crate::cdf::{
dequant_denom, get_ac_quant, get_dc_quant, QuantizerParams, IDTX, TX_HEIGHT, TX_SIZES_ALL,
TX_WIDTH,
};
use crate::qmatrix;
#[allow(clippy::too_many_arguments)]
pub fn forward_quantize(
coeffs: &[i64],
tx_size: usize,
plane: u8,
segment_id: u8,
plane_tx_type: usize,
seg_qm_level: u8,
quant: &QuantizerParams,
) -> Vec<i32> {
forward_quantize_rounded(
coeffs,
tx_size,
plane,
segment_id,
plane_tx_type,
seg_qm_level,
quant,
QUANT_ROUND_HALF,
)
}
pub const QUANT_ROUND_HALF: i64 = 32;
#[allow(clippy::too_many_arguments)]
pub fn forward_quantize_rounded(
coeffs: &[i64],
tx_size: usize,
plane: u8,
segment_id: u8,
plane_tx_type: usize,
seg_qm_level: u8,
quant: &QuantizerParams,
round_num_64: i64,
) -> Vec<i32> {
assert!(
tx_size < TX_SIZES_ALL,
"oxideav-av1 forward_quantize: tx_size {tx_size} out of range (TX_SIZES_ALL = {TX_SIZES_ALL})"
);
let w = TX_WIDTH[tx_size];
let h = TX_HEIGHT[tx_size];
assert_eq!(
coeffs.len(),
w * h,
"oxideav-av1 forward_quantize: coeffs length {} != Tx_Width[{tx_size}] * Tx_Height[{tx_size}] = {}",
coeffs.len(),
w * h
);
let tw = core::cmp::min(32, w);
let th = core::cmp::min(32, h);
let dq_denom = dequant_denom(tx_size);
let dc_q_val = get_dc_quant(quant, plane, segment_id) as i64;
let ac_q_val = get_ac_quant(quant, plane, segment_id) as i64;
let qm_active = quant.using_qmatrix && plane_tx_type < IDTX && seg_qm_level < 15;
let mut levels = vec![0i32; w * h];
for i in 0..th {
for j in 0..tw {
let q = if i == 0 && j == 0 { dc_q_val } else { ac_q_val };
let q2 = if qm_active {
let qm = qmatrix::qmatrix_value(seg_qm_level, plane, tx_size, i, j) as i64;
(q * qm + 16) >> 5
} else {
q
};
if q2 == 0 {
levels[i * w + j] = 0;
continue;
}
let c = coeffs[i * w + j];
let sign: i64 = if c < 0 { -1 } else { 1 };
let abs_c = c.unsigned_abs() as i64;
let round = if i == 0 && j == 0 {
QUANT_ROUND_HALF
} else {
round_num_64.clamp(0, 64)
};
let num = 64 * abs_c * dq_denom + round * q2;
let den = 64 * q2;
let mag = num / den;
levels[i * w + j] = (sign * mag) as i32;
}
}
levels
}
#[cfg(test)]
mod tests {
use super::*;
use crate::cdf::{dequantize_step1, DCT_DCT, TX_16X8, TX_4X4, TX_4X8, TX_8X16, TX_8X4, TX_8X8};
#[test]
fn forward_quantize_lossless_q_index_zero_tx4x4() {
let qp = QuantizerParams::neutral(0, 8);
let coeffs: [i64; 16] = [4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60, 64];
let levels = forward_quantize(&coeffs, TX_4X4, 0, 0, DCT_DCT, 15, &qp);
assert_eq!(
levels,
vec![1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]
);
let dequant = dequantize_step1(&levels, TX_4X4, 0, 0, DCT_DCT, 15, &qp);
assert_eq!(dequant, coeffs.to_vec());
}
#[test]
fn forward_quantize_preserves_sign_lossless() {
let qp = QuantizerParams::neutral(0, 8);
let coeffs: [i64; 16] = [
-4, 8, -12, 16, -20, 24, -28, 32, -36, 40, -44, 48, -52, 56, -60, 64,
];
let levels = forward_quantize(&coeffs, TX_4X4, 0, 0, DCT_DCT, 15, &qp);
let dequant = dequantize_step1(&levels, TX_4X4, 0, 0, DCT_DCT, 15, &qp);
assert_eq!(dequant, coeffs.to_vec());
}
#[test]
fn forward_quantize_lossy_rounds_half_away_from_zero() {
let qp = QuantizerParams::neutral(0, 8);
let coeffs: [i64; 16] = [
6, 5, 7, -6, -5, -7, 1, 2, 3, 0, 0, 0, 0, 0, 0, 0,
];
let levels = forward_quantize(&coeffs, TX_4X4, 0, 0, DCT_DCT, 15, &qp);
assert_eq!(levels[0], 2);
assert_eq!(levels[1], 1);
assert_eq!(levels[2], 2);
assert_eq!(levels[3], -2);
assert_eq!(levels[4], -1);
assert_eq!(levels[5], -2);
let dequant = dequantize_step1(&levels, TX_4X4, 0, 0, DCT_DCT, 15, &qp);
for (k, (orig, recon)) in coeffs.iter().zip(dequant.iter()).enumerate() {
assert!(
(orig - recon).abs() <= 4,
"k={k}: |orig {orig} - recon {recon}| > 4"
);
}
}
#[test]
fn forward_quantize_higher_q_index_aligned_coeffs_roundtrip() {
let qp = QuantizerParams::neutral(16, 8);
let mut coeffs = [0i64; 16];
coeffs[0] = 40; for c in coeffs.iter_mut().skip(1).take(15) {
*c = 46; }
let levels = forward_quantize(&coeffs, TX_4X4, 0, 0, DCT_DCT, 15, &qp);
assert_eq!(levels[0], 2);
for &lv in &levels[1..] {
assert_eq!(lv, 2);
}
let dequant = dequantize_step1(&levels, TX_4X4, 0, 0, DCT_DCT, 15, &qp);
assert_eq!(dequant, coeffs.to_vec());
}
#[test]
fn forward_quantize_tx8x8_lossless_q_index_zero() {
let qp = QuantizerParams::neutral(0, 8);
let mut coeffs = vec![0i64; 64];
coeffs[0] = 20;
for (k, slot) in coeffs.iter_mut().enumerate().skip(1).take(63) {
*slot = ((k as i64) % 7) * 4; }
let levels = forward_quantize(&coeffs, TX_8X8, 0, 0, DCT_DCT, 15, &qp);
assert_eq!(levels[0], 5);
for (k, &lv) in levels.iter().enumerate().skip(1).take(63) {
let expected = ((k as i64) % 7) as i32;
assert_eq!(lv, expected, "k={k}");
}
let dequant = dequantize_step1(&levels, TX_8X8, 0, 0, DCT_DCT, 15, &qp);
assert_eq!(dequant, coeffs);
}
#[test]
fn forward_quantize_tx4x8_round_trip_lossless_q_index_zero() {
let qp = QuantizerParams::neutral(0, 8);
let mut coeffs = vec![0i64; 32];
coeffs[0] = 32; for (k, slot) in coeffs.iter_mut().enumerate().skip(1) {
*slot = ((k as i64) % 5) * 4; }
let levels = forward_quantize(&coeffs, TX_4X8, 0, 0, DCT_DCT, 15, &qp);
assert_eq!(levels[0], 8, "TX_4X8 DC quant level mismatch");
for (k, &lv) in levels.iter().enumerate().skip(1) {
let expected = ((k as i64) % 5) as i32;
assert_eq!(lv, expected, "TX_4X8 AC quant level mismatch at k = {k}");
}
let dequant = dequantize_step1(&levels, TX_4X8, 0, 0, DCT_DCT, 15, &qp);
assert_eq!(dequant, coeffs, "TX_4X8 dequant did not round-trip");
}
#[test]
fn forward_quantize_tx8x4_round_trip_lossless_q_index_zero() {
let qp = QuantizerParams::neutral(0, 8);
let mut coeffs = vec![0i64; 32];
coeffs[0] = 28;
for (k, slot) in coeffs.iter_mut().enumerate().skip(1) {
*slot = ((k as i64) % 6) * 4;
}
let levels = forward_quantize(&coeffs, TX_8X4, 0, 0, DCT_DCT, 15, &qp);
assert_eq!(levels[0], 7);
for (k, &lv) in levels.iter().enumerate().skip(1) {
let expected = ((k as i64) % 6) as i32;
assert_eq!(lv, expected, "TX_8X4 AC quant level mismatch at k = {k}");
}
let dequant = dequantize_step1(&levels, TX_8X4, 0, 0, DCT_DCT, 15, &qp);
assert_eq!(dequant, coeffs);
}
#[test]
fn forward_quantize_tx8x16_round_trip_lossless_q_index_zero() {
let qp = QuantizerParams::neutral(0, 8);
let mut coeffs = vec![0i64; 128];
coeffs[0] = 40; for (k, slot) in coeffs.iter_mut().enumerate().skip(1) {
*slot = ((k as i64) % 7) * 4; }
let levels = forward_quantize(&coeffs, TX_8X16, 0, 0, DCT_DCT, 15, &qp);
assert_eq!(levels[0], 10, "TX_8X16 DC quant level mismatch");
for (k, &lv) in levels.iter().enumerate().skip(1) {
let expected = ((k as i64) % 7) as i32;
assert_eq!(lv, expected, "TX_8X16 AC quant level mismatch at k = {k}");
}
let dequant = dequantize_step1(&levels, TX_8X16, 0, 0, DCT_DCT, 15, &qp);
assert_eq!(dequant, coeffs, "TX_8X16 dequant did not round-trip");
}
#[test]
fn forward_quantize_tx16x8_round_trip_lossless_q_index_zero() {
let qp = QuantizerParams::neutral(0, 8);
let mut coeffs = vec![0i64; 128];
coeffs[0] = 36;
for (k, slot) in coeffs.iter_mut().enumerate().skip(1) {
*slot = ((k as i64) % 8) * 4;
}
let levels = forward_quantize(&coeffs, TX_16X8, 0, 0, DCT_DCT, 15, &qp);
assert_eq!(levels[0], 9);
for (k, &lv) in levels.iter().enumerate().skip(1) {
let expected = ((k as i64) % 8) as i32;
assert_eq!(lv, expected, "TX_16X8 AC quant level mismatch at k = {k}");
}
let dequant = dequantize_step1(&levels, TX_16X8, 0, 0, DCT_DCT, 15, &qp);
assert_eq!(dequant, coeffs);
}
#[test]
fn forward_quantize_zero_coeffs_yield_zero_levels() {
for &qi in &[0u8, 16, 64, 128, 255] {
let qp = QuantizerParams::neutral(qi, 8);
let coeffs = vec![0i64; 16];
let levels = forward_quantize(&coeffs, TX_4X4, 0, 0, DCT_DCT, 15, &qp);
assert_eq!(levels, vec![0; 16], "q_index = {qi}");
}
}
#[test]
fn forward_quantize_inverts_dequantize_step1_on_exact_levels() {
let qp = QuantizerParams::neutral(0, 8);
let patterns: [[i32; 16]; 4] = [
[10, -3, 7, -2, 0, 5, -1, 4, 0, 0, 8, -6, 3, -9, 2, 11],
[0; 16],
[
100, 50, 25, 10, -100, -50, -25, -10, 1, 2, 3, 4, -1, -2, -3, -4,
],
[
4096, -4096, 2048, -2048, 1024, -1024, 512, -512, 256, -256, 128, -128, 64, -64,
32, -32,
],
];
for (idx, levels) in patterns.iter().enumerate() {
let dequant = dequantize_step1(levels, TX_4X4, 0, 0, DCT_DCT, 15, &qp);
let recovered = forward_quantize(&dequant, TX_4X4, 0, 0, DCT_DCT, 15, &qp);
assert_eq!(
recovered,
levels.to_vec(),
"pattern {idx} did not round-trip: dequant = {dequant:?}"
);
}
}
#[test]
fn forward_quantize_qm_active_branch_roundtrip() {
let mut qp = QuantizerParams::neutral(0, 8);
qp.using_qmatrix = true;
let levels = [3i32, -1, 2, 0, -2, 1, 4, -3, 0, 0, 1, -1, 2, -2, 3, -3];
let seg_qm_level = 0;
let dequant = dequantize_step1(&levels, TX_4X4, 0, 0, DCT_DCT, seg_qm_level, &qp);
let recovered = forward_quantize(&dequant, TX_4X4, 0, 0, DCT_DCT, seg_qm_level, &qp);
assert_eq!(recovered, levels.to_vec());
}
#[test]
#[should_panic(expected = "coeffs length")]
fn forward_quantize_rejects_short_buffer() {
let qp = QuantizerParams::neutral(0, 8);
let coeffs = vec![0i64; 15]; let _ = forward_quantize(&coeffs, TX_4X4, 0, 0, DCT_DCT, 15, &qp);
}
#[test]
#[should_panic(expected = "tx_size")]
fn forward_quantize_rejects_invalid_tx_size() {
let qp = QuantizerParams::neutral(0, 8);
let coeffs = vec![0i64; 16];
let _ = forward_quantize(&coeffs, TX_SIZES_ALL, 0, 0, DCT_DCT, 15, &qp);
}
}