#![allow(clippy::excessive_precision)]
use crate::Arbiter;
use crate::dct::fmla;
use crate::util::{HeapMatrix, f16_bits_to_f32, f32_to_f16_bits, heap_array_from_fn};
pub(crate) fn quant_table_slot_of(strategy: u8) -> Option<usize> {
use crate::dc_group_data::*;
Some(match strategy {
STRATEGY_DCT => 0,
STRATEGY_DCT16X16 => 1,
STRATEGY_DCT32X32 => 2,
STRATEGY_DCT16X8 | STRATEGY_DCT8X16 => 3,
STRATEGY_DCT32X16 | STRATEGY_DCT16X32 => 4,
STRATEGY_DCT4X8 | STRATEGY_DCT8X4 => 5,
STRATEGY_DCT64X32 | STRATEGY_DCT32X64 => 6,
STRATEGY_DCT64X64 => 7,
STRATEGY_IDENTITY => 8,
_ => return None,
})
}
pub(crate) struct MatrixHeaderCost {
pub(crate) payload: [usize; 9],
}
impl MatrixHeaderCost {
pub(crate) fn new(matrices: &DequantMatrices) -> Self {
let mut payload = [0; 9];
for (slot, table) in matrices.custom_tables.iter().enumerate() {
if let Some(table) = table {
payload[slot] = 4 + 3 * 16 * table.num_bands + usize::from(slot == 5) * 48;
}
}
if matrices.identity_weights.is_some() {
payload[8] = 9 * 16;
}
Self { payload }
}
pub(crate) fn custom_mask(&self) -> u16 {
self.payload
.iter()
.enumerate()
.fold(0, |mask, (slot, &bits)| {
mask | (u16::from(bits != 0) << slot)
})
}
pub(crate) fn bits(&self, used: u16) -> usize {
let payload: usize = self
.payload
.iter()
.enumerate()
.filter(|&(slot, _)| used & (1 << slot) != 0)
.map(|(_, &bits)| bits)
.sum();
1 + payload + usize::from(payload != 0) * (17 * 3)
}
}
#[derive(Clone, Copy)]
pub(crate) struct BandOverride {
pub(crate) num_bands: usize,
pub(crate) bands: [[f32; 16]; 3],
}
pub(crate) const QM_SS2_MIN_DISTANCE: f32 = 2.25;
const QM_DCT8_MIN_DISTANCE: f32 = 4.3;
const QM_DCT16_TILT_MAX_DISTANCE: f32 = 8.0;
const ORIGINAL_COARSE_DCT8_DISTANCE: f32 = 3.5;
pub(crate) const QM_FLAT_B8_MIN_DISTANCE: f32 = 0.3;
pub(crate) const QM_FLAT_B8_MID_MIN_DISTANCE: f32 = 1.25;
pub(crate) const PAIR_B_FINE_BAND1: f32 = -0.35;
pub(crate) const PAIR_B_FINE_MIN_DISTANCE: f32 = QM_FLAT_B8_MID_MIN_DISTANCE;
pub(crate) const PAIR_B_FINE_MAX_DISTANCE: f32 = QM_SS2_MIN_DISTANCE;
const QM_SS2_SCALE16: f32 = 0.78;
const QM_SS2_SCALE32: f32 = 0.89;
const QM_SS2_SCALE16X32: f32 = 1.20;
const QM_DCT8_Y_HF_SCALE: f32 = 1.25;
static DCT8_BANDS: [[f32; 6]; 3] = [
[3150.0, 0.0, -0.4, -0.4, -0.4, -2.0],
[560.0, 0.0, -0.3, -0.3, -0.3, -0.3],
[512.0, -2.0, -1.0, 0.0, -1.0, -2.0],
];
static FLAT_B8_BANDS_HQ: [f32; 6] = [512.0, -0.5, -0.25, 0.0, -0.25, -0.5];
static FLAT_B8_BANDS_MID: [f32; 6] = [512.0, -0.13, -0.235, -0.276, -0.609, -0.339];
static SAT_B8_BANDS: [f32; 6] = [512.0, 0.024, -0.517, -0.011, -0.357, -0.674];
const HUE_B8_MIN_DISTANCE: f32 = 0.75;
static HUE_B8_BANDS: [f32; 6] = [512.0, -1.5, -1.0, 0.0, -1.0, -2.0];
fn default_dct8_override(use_coarse: bool, flat_b8: &[f32; 6]) -> BandOverride {
let mut out = if use_coarse {
coarse_dct8_override()
} else {
let mut base = BandOverride {
num_bands: 6,
bands: [[0.0; 16]; 3],
};
for c in 0..3 {
base.bands[c][..6].copy_from_slice(&DCT8_BANDS[c]);
}
base
};
out.bands[2][..6].copy_from_slice(flat_b8);
for c in 0..3 {
for i in 0..6 {
let v = out.bands[c][i];
out.bands[c][i] = if i == 0 {
f16_bits_to_f32(f32_to_f16_bits(v / 64.0)) * 64.0
} else {
f16_bits_to_f32(f32_to_f16_bits(v))
};
}
}
out
}
fn coarse_dct8_override() -> BandOverride {
let hf_step = QM_DCT8_Y_HF_SCALE.powf(1.0 / 5.0);
let mut out = BandOverride {
num_bands: 6,
bands: [[0.0; 16]; 3],
};
for c in 0..3 {
out.bands[c][..6].copy_from_slice(&DCT8_BANDS[c]);
}
for i in 1..6 {
let ratio = band_mult(DCT8_BANDS[1][i]) * hf_step;
out.bands[1][i] = if ratio > 1.0 {
ratio - 1.0
} else {
1.0 - 1.0 / ratio
};
}
for c in 0..3 {
for i in 0..6 {
let v = out.bands[c][i];
out.bands[c][i] = if i == 0 {
f16_bits_to_f32(f32_to_f16_bits(v / 64.0)) * 64.0
} else {
f16_bits_to_f32(f32_to_f16_bits(v))
};
}
}
out
}
fn scaled_override<const N: usize>(defaults: &[[f32; N]; 3], scale: f32) -> BandOverride {
let mut out = BandOverride {
num_bands: N,
bands: [[0.0f32; 16]; 3],
};
for c in 0..3 {
for i in 0..N {
let v = if i == 0 {
defaults[c][0] * scale
} else {
defaults[c][i]
};
out.bands[c][i] = if i == 0 {
f16_bits_to_f32(f32_to_f16_bits(v / 64.0)) * 64.0
} else {
f16_bits_to_f32(f32_to_f16_bits(v))
};
}
}
out
}
fn tilted_override<const N: usize>(
defaults: &[[f32; N]; 3],
scale: f32,
luma_hf: f32,
) -> BandOverride {
let mut out = scaled_override(defaults, scale);
let step = luma_hf.powf(1.0 / (N as f32 - 1.0));
for i in 1..N {
let r = band_mult(defaults[1][i]) * step;
let v = if r > 1.0 { r - 1.0 } else { 1.0 - 1.0 / r };
out.bands[1][i] = f16_bits_to_f32(f32_to_f16_bits(v));
}
out
}
pub(crate) static INV_DC_QUANT: [f32; 3] = [4096.0, 512.0, 256.0];
pub(crate) static DC_QUANT: [f32; 3] = [1.0 / 4096.0, 1.0 / 512.0, 1.0 / 256.0];
pub(crate) static DEQUANT_MATRIX_8X8: [[f32; 64]; 3] = [
[
3.1746033e-04,
3.1746057e-04,
3.1854658e-04,
3.7755401e-04,
4.4749113e-04,
5.3038419e-04,
6.2863121e-04,
7.4507861e-04,
3.1746057e-04,
3.1746062e-04,
3.3158599e-04,
3.8811122e-04,
4.5695182e-04,
5.3938502e-04,
6.3753547e-04,
7.5413194e-04,
3.1854658e-04,
3.3158599e-04,
3.6670428e-04,
4.1847790e-04,
4.8487642e-04,
5.6626293e-04,
6.6427846e-04,
7.8140449e-04,
3.7755401e-04,
3.8811122e-04,
4.1847790e-04,
4.6632939e-04,
5.3038419e-04,
6.1082945e-04,
7.0903177e-04,
8.2727504e-04,
4.4749113e-04,
4.5695182e-04,
4.8487642e-04,
5.3038419e-04,
5.9302151e-04,
6.7320757e-04,
7.7229418e-04,
9.4286882e-04,
5.3038419e-04,
5.3938502e-04,
5.6626293e-04,
6.1082945e-04,
6.7320757e-04,
7.5413194e-04,
8.5507357e-04,
1.2723245e-03,
6.2863121e-04,
6.3753547e-04,
6.6427846e-04,
7.0903177e-04,
7.7229418e-04,
8.5507357e-04,
1.1923184e-03,
1.7919940e-03,
7.4507861e-04,
7.5413194e-04,
7.8140449e-04,
8.2727504e-04,
9.4286882e-04,
1.2723245e-03,
1.7919940e-03,
2.6133191e-03,
],
[
1.7857145e-03,
1.7857157e-03,
1.7904768e-03,
2.0441783e-03,
2.3338278e-03,
2.6645192e-03,
3.0420676e-03,
3.4731133e-03,
1.7857157e-03,
1.7857160e-03,
1.8473724e-03,
2.0886122e-03,
2.3722122e-03,
2.6997121e-03,
3.0756146e-03,
3.5059757e-03,
1.7904768e-03,
1.8473724e-03,
1.9982266e-03,
2.2149722e-03,
2.4845072e-03,
2.8040458e-03,
3.1757555e-03,
3.6044519e-03,
2.0441783e-03,
2.0886122e-03,
2.2149722e-03,
2.4100873e-03,
2.6645192e-03,
2.9746795e-03,
3.3413812e-03,
3.7683977e-03,
2.3338278e-03,
2.3722122e-03,
2.4845072e-03,
2.6645192e-03,
2.9068382e-03,
3.2089923e-03,
3.5716419e-03,
3.9980840e-03,
2.6645192e-03,
2.6997121e-03,
2.8040458e-03,
2.9746795e-03,
3.2089923e-03,
3.5059757e-03,
3.8667743e-03,
4.2947000e-03,
3.0420676e-03,
3.0756146e-03,
3.1757555e-03,
3.3413812e-03,
3.5716419e-03,
3.8667743e-03,
4.2286036e-03,
4.6607289e-03,
3.4731133e-03,
3.5059757e-03,
3.6044519e-03,
3.7683977e-03,
3.9980840e-03,
4.2947000e-03,
4.6607289e-03,
5.1001739e-03,
],
[
1.9531252e-03,
3.4018266e-03,
5.9007513e-03,
8.3743408e-03,
1.1718751e-02,
1.1718759e-02,
1.1968765e-02,
1.6986061e-02,
3.4018266e-03,
4.2808522e-03,
6.4091417e-03,
8.8638803e-03,
1.1718752e-02,
1.1718759e-02,
1.2320629e-02,
1.7413978e-02,
5.9007513e-03,
6.4091417e-03,
7.8861341e-03,
1.0351914e-02,
1.1718754e-02,
1.1718762e-02,
1.3408982e-02,
1.8736197e-02,
8.3743408e-03,
8.8638803e-03,
1.0351914e-02,
1.1718752e-02,
1.1718759e-02,
1.1718766e-02,
1.5336527e-02,
2.1072537e-02,
1.1718751e-02,
1.1718752e-02,
1.1718754e-02,
1.1718759e-02,
1.1718764e-02,
1.3782934e-02,
1.8288977e-02,
2.5368163e-02,
1.1718759e-02,
1.1718759e-02,
1.1718762e-02,
1.1718766e-02,
1.3782934e-02,
1.7413978e-02,
2.2557227e-02,
3.4232263e-02,
1.1968765e-02,
1.2320629e-02,
1.3408982e-02,
1.5336527e-02,
1.8288977e-02,
2.2557227e-02,
3.2079678e-02,
4.8214123e-02,
1.6986061e-02,
1.7413978e-02,
1.8736197e-02,
2.1072537e-02,
2.5368163e-02,
3.4232263e-02,
4.8214123e-02,
7.0312120e-02,
],
];
pub(crate) static DEQUANT_MATRIX_16X8: [[f32; 128]; 3] = [
[
1.3810680e-04,
1.6047071e-04,
1.8645605e-04,
2.1664926e-04,
2.5173181e-04,
2.9249521e-04,
3.3985957e-04,
3.9489369e-04,
4.1871337e-04,
4.4087201e-04,
4.6420316e-04,
4.8876996e-04,
5.1463587e-04,
5.4187077e-04,
5.7054684e-04,
6.0074159e-04,
1.9049694e-04,
1.9694651e-04,
2.1442315e-04,
2.4016941e-04,
2.7289384e-04,
3.1245520e-04,
3.5932945e-04,
4.0429863e-04,
4.2484730e-04,
4.4662904e-04,
4.6966935e-04,
4.9400958e-04,
5.1969837e-04,
5.4679497e-04,
5.7536521e-04,
6.0547784e-04,
2.6276117e-04,
2.6734054e-04,
2.8085473e-04,
3.0283103e-04,
3.3291124e-04,
3.7106971e-04,
4.0540050e-04,
4.2322354e-04,
4.4259510e-04,
4.6344541e-04,
4.8574654e-04,
5.0949713e-04,
5.3471868e-04,
5.6144706e-04,
5.8973121e-04,
6.1962701e-04,
3.6243827e-04,
3.6666830e-04,
3.7935356e-04,
3.9960333e-04,
4.0956112e-04,
4.2183659e-04,
4.3620329e-04,
4.5248115e-04,
4.7053859e-04,
4.9028790e-04,
5.1167433e-04,
5.3467450e-04,
5.5928703e-04,
5.8552966e-04,
6.1343284e-04,
6.4304151e-04,
4.3123538e-04,
4.3253010e-04,
4.3638589e-04,
4.4272337e-04,
4.5142765e-04,
4.6236772e-04,
4.7541404e-04,
4.9045112e-04,
5.0738233e-04,
5.2613625e-04,
5.4666388e-04,
5.6893763e-04,
5.9295003e-04,
6.1870721e-04,
6.4623309e-04,
6.7556335e-04,
4.8162136e-04,
4.8277923e-04,
4.8623976e-04,
4.9196521e-04,
4.9989921e-04,
5.0997391e-04,
5.2211789e-04,
5.3626322e-04,
5.5235048e-04,
5.7033246e-04,
5.9017621e-04,
6.1186077e-04,
6.3538179e-04,
6.6074729e-04,
6.8797835e-04,
7.5214857e-04,
5.3789350e-04,
5.3897168e-04,
5.4219965e-04,
5.4755906e-04,
5.5502116e-04,
5.6455150e-04,
5.7611306e-04,
5.8966759e-04,
6.0518348e-04,
6.2263483e-04,
6.4200454e-04,
6.6328526e-04,
6.8647927e-04,
7.3936180e-04,
8.0337300e-04,
8.7534159e-04,
6.0074159e-04,
6.0177385e-04,
6.0486794e-04,
6.1001495e-04,
6.1720144e-04,
6.2641077e-04,
6.3762552e-04,
6.5082888e-04,
6.6600717e-04,
6.8315107e-04,
7.1794738e-04,
7.6673855e-04,
8.2213664e-04,
8.8475435e-04,
9.5529074e-04,
1.0345384e-03,
],
[
6.9053401e-04,
7.7444571e-04,
8.6855399e-04,
9.7409816e-04,
1.0924696e-03,
1.2252233e-03,
1.3741088e-03,
1.5410866e-03,
1.7283577e-03,
1.9383827e-03,
2.1739292e-03,
2.3783136e-03,
2.5041751e-03,
2.6366978e-03,
2.7762330e-03,
2.9231580e-03,
8.8290084e-04,
9.0565201e-04,
9.6644071e-04,
1.0539154e-03,
1.1619731e-03,
1.2886107e-03,
1.4338633e-03,
1.5988132e-03,
1.7851711e-03,
1.9951246e-03,
2.2312698e-03,
2.4038092e-03,
2.5288085e-03,
2.6606584e-03,
2.7996788e-03,
2.9462043e-03,
1.1288587e-03,
1.1438611e-03,
1.1877866e-03,
1.2581701e-03,
1.3525900e-03,
1.4695247e-03,
1.6085195e-03,
1.7700332e-03,
1.9552717e-03,
2.1660449e-03,
2.3636019e-03,
2.4791702e-03,
2.6018962e-03,
2.7319542e-03,
2.8695827e-03,
3.0150530e-03,
1.4433329e-03,
1.4561869e-03,
1.4945272e-03,
1.5578135e-03,
1.6454635e-03,
1.7571596e-03,
1.8930284e-03,
2.0537286e-03,
2.2404641e-03,
2.3856999e-03,
2.4897642e-03,
2.6016813e-03,
2.7214440e-03,
2.8491386e-03,
2.9849128e-03,
3.1289863e-03,
1.8454153e-03,
1.8577600e-03,
1.8947916e-03,
1.9565322e-03,
2.0431101e-03,
2.1548597e-03,
2.2924175e-03,
2.3864941e-03,
2.4688798e-03,
2.5601350e-03,
2.6600207e-03,
2.7684029e-03,
2.8852450e-03,
3.0105773e-03,
3.1445161e-03,
3.2872346e-03,
2.3435291e-03,
2.3491632e-03,
2.3660017e-03,
2.3938618e-03,
2.4324679e-03,
2.4814904e-03,
2.5405819e-03,
2.6094120e-03,
2.6876912e-03,
2.7751899e-03,
2.8717481e-03,
2.9772632e-03,
3.0917146e-03,
3.2151414e-03,
3.3476453e-03,
3.4893905e-03,
2.6173447e-03,
2.6225911e-03,
2.6382981e-03,
2.6643763e-03,
2.7006865e-03,
2.7470603e-03,
2.8033180e-03,
2.8692731e-03,
2.9447721e-03,
3.0296890e-03,
3.1239407e-03,
3.2274905e-03,
3.3403505e-03,
3.4625907e-03,
3.5943130e-03,
3.7356857e-03,
2.9231580e-03,
2.9281813e-03,
2.9432368e-03,
2.9682817e-03,
3.0032503e-03,
3.0480621e-03,
3.1026325e-03,
3.1668788e-03,
3.2407353e-03,
3.3241559e-03,
3.4171303e-03,
3.5196650e-03,
3.6318223e-03,
3.7536959e-03,
3.8854245e-03,
4.0271855e-03,
],
[
1.9729543e-03,
2.5272998e-03,
3.2374004e-03,
4.1470206e-03,
4.8498721e-03,
5.1065302e-03,
5.3767711e-03,
5.6613120e-03,
6.3208523e-03,
7.0889443e-03,
7.9503711e-03,
8.9164926e-03,
9.9999988e-03,
1.1215170e-02,
1.2578006e-02,
1.5967883e-02,
3.3539708e-03,
3.5433732e-03,
4.0769530e-03,
4.7721486e-03,
4.9862624e-03,
5.2236770e-03,
5.4806760e-03,
5.8470899e-03,
6.5286267e-03,
7.2964565e-03,
8.1600742e-03,
9.1304630e-03,
1.0220082e-02,
1.1443083e-02,
1.2854187e-02,
1.6610704e-02,
4.9218577e-03,
4.9511627e-03,
5.0357706e-03,
5.1678251e-03,
5.3387447e-03,
5.5415547e-03,
5.8825868e-03,
6.4732647e-03,
7.1507092e-03,
7.9215374e-03,
8.7942975e-03,
9.7792931e-03,
1.0888627e-02,
1.2136214e-02,
1.4550334e-02,
1.8655479e-02,
5.4969229e-03,
5.5188821e-03,
5.5837538e-03,
5.6971479e-03,
6.0176966e-03,
6.4261849e-03,
6.9230762e-03,
7.5107799e-03,
8.1936987e-03,
8.9781955e-03,
9.8724691e-03,
1.0886625e-02,
1.2032620e-02,
1.4036770e-02,
1.7736901e-02,
2.2478340e-02,
6.7489492e-03,
6.7940955e-03,
6.9295247e-03,
7.1553192e-03,
7.4719470e-03,
7.8806318e-03,
8.3836997e-03,
8.9848433e-03,
9.6892491e-03,
1.0503774e-02,
1.1436986e-02,
1.2499244e-02,
1.4953869e-02,
1.8516723e-02,
2.3044668e-02,
2.8803868e-02,
8.6290650e-03,
8.6752698e-03,
8.8141672e-03,
9.0466458e-03,
9.3743140e-03,
9.7996546e-03,
1.0326200e-02,
1.0958694e-02,
1.1703256e-02,
1.2567491e-02,
1.4605597e-02,
1.7509630e-02,
2.1164555e-02,
2.5766177e-02,
3.1564441e-02,
4.4291977e-02,
1.1032925e-02,
1.1082166e-02,
1.1230315e-02,
1.1478675e-02,
1.1829470e-02,
1.2285955e-02,
1.2938387e-02,
1.4542448e-02,
1.6570158e-02,
1.9115077e-02,
2.2296766e-02,
2.6267400e-02,
3.1220267e-02,
4.1523870e-02,
5.6756895e-02,
7.8389272e-02,
1.5967883e-02,
1.6106272e-02,
1.6526788e-02,
1.7245775e-02,
1.8291343e-02,
1.9704822e-02,
2.1542856e-02,
2.3880199e-02,
2.6813647e-02,
3.0466938e-02,
3.7175436e-02,
4.7613274e-02,
6.1909460e-02,
8.1609353e-02,
1.0892317e-01,
1.4702357e-01,
],
];
pub(crate) struct FineTables {
pub(crate) dct2: [[f32; 6]; 3],
pub(crate) dct4x4: [[f32; 4]; 3],
pub(crate) afv: [[f32; 9]; 3],
}
pub(crate) struct DequantMatrices {
pub(crate) identity_weights: Option<[[f32; 3]; 3]>,
pub(crate) fine_tables: Option<Box<FineTables>>,
pub(crate) matrix: HeapMatrix<f32, 3, 64>,
pub(crate) inv_matrix: HeapMatrix<f32, 3, 64>,
pub(crate) matrix_identity: HeapMatrix<f32, 3, 64>,
pub(crate) inv_matrix_identity: HeapMatrix<f32, 3, 64>,
pub(crate) matrix_dct2x2: &'static [[f32; 64]; 3],
pub(crate) inv_matrix_dct2x2: &'static [[f32; 64]; 3],
pub(crate) matrix_16x8: HeapMatrix<f32, 3, 128>,
pub(crate) inv_matrix_16x8: HeapMatrix<f32, 3, 128>,
pub(crate) matrix_16x16: &'static [[f32; 256]; 3],
pub(crate) inv_matrix_16x16: &'static [[f32; 256]; 3],
pub(crate) matrix_32x32: &'static [[f32; 1024]; 3],
pub(crate) inv_matrix_32x32: &'static [[f32; 1024]; 3],
pub(crate) matrix_64x64: &'static [[f32; 4096]; 3],
pub(crate) inv_matrix_64x64: &'static [[f32; 4096]; 3],
pub(crate) matrix_64x32: &'static [[f32; 2048]; 3],
pub(crate) inv_matrix_64x32: &'static [[f32; 2048]; 3],
pub(crate) custom_tables: Box<[Option<BandOverride>; 8]>,
pub(crate) matrix_4x4: &'static [[f32; 64]; 3],
pub(crate) inv_matrix_4x4: &'static [[f32; 64]; 3],
pub(crate) matrix_4x8: &'static [[f32; 64]; 3],
pub(crate) inv_matrix_4x8: &'static [[f32; 64]; 3],
pub(crate) matrix_32x16: &'static [[f32; 512]; 3],
pub(crate) inv_matrix_32x16: &'static [[f32; 512]; 3],
pub(crate) matrix_afv: &'static [[f32; 64]; 3],
pub(crate) inv_matrix_afv: &'static [[f32; 64]; 3],
}
static DCT16X16_BANDS: [[f32; 7]; 3] = [
[
8996.872_571_181_412,
-1.300_077_739_335_380_4,
-0.494_245_298_245_712_25,
-0.439_093_774_457_103_44,
-0.635_010_183_269_574_4,
-0.901_772_640_508_276_1,
-1.616_209_923_988_741_4,
],
[
3191.483_662_968_442_3,
-0.674_245_821_041_943_55,
-0.807_458_134_284_710,
-0.449_258_374_848_434_4,
-0.358_654_409_810_334_03,
-0.313_223_891_118_773_05,
-0.376_150_253_157_254_83,
],
[
1157.504_081_454_872,
-2.053_142_316_580_441_4,
-1.4,
-0.506_871_300_333_784,
-0.427_087_306_247_339_04,
-1.485_683_453_929_624_4,
-4.920_914_288_440_160,
],
];
static DCT32X32_BANDS: [[f32; 8]; 3] = [
[
15718.408_309_825_19,
-1.025,
-0.98,
-0.901_2,
-0.4,
-0.488_193_95,
-0.421_064,
-0.27,
],
[
7305.763_681_069_598,
-0.804_195_82,
-0.763_303_65,
-0.556_603_8,
-0.497_853_05,
-0.436_995_93,
-0.401_808_67,
-0.273_216_83,
],
[
3803.531_737_212_15,
-3.060_733_6,
-2.041_327,
-2.023_565,
-0.549_538_95,
-0.4,
-0.4,
-0.3,
],
];
#[inline]
fn band_mult(v: f32) -> f32 {
if v > 0.0 { 1.0 + v } else { 1.0 / (1.0 - v) }
}
static DCT16X8_BANDS: [[f32; 7]; 3] = [
[7240.7734393502, -0.7, -0.7, -0.2, -0.2, -0.2, -0.5],
[1448.15468787004, -0.5, -0.5, -0.5, -0.2, -0.2, -0.2],
[506.854002029, -1.4, -0.2, -0.5, -0.5, -1.5, -3.6],
];
static DCT16X32_BANDS: [[f32; 8]; 3] = [
[
13844.97076442300573,
-0.97113799999999995,
-0.658,
-0.42026,
-0.22712,
-0.2206,
-0.226,
-0.6,
],
[
4798.964084220744293,
-0.61125308982767057,
-0.83770786552491361,
-0.79014862079498627,
-0.2692727459704829,
-0.38272769465388551,
-0.22924222653091453,
-0.20719098826199578,
],
[
1807.236946760964614,
-1.2,
-1.2,
-0.7,
-0.7,
-0.7,
-0.4,
-0.5,
],
];
static DCT4X4_BANDS: [[f32; 4]; 3] = [
[2200.0, 0.0, 0.0, 0.0],
[392.0, 0.0, 0.0, 0.0],
[112.0, -0.25, -0.25, -0.5],
];
static DCT4X8_BANDS: [[f32; 4]; 3] = [
[
2198.050556016380522,
-0.96269623020744692,
-0.76194253026666783,
-0.6551140670773547,
],
[
764.3655248643528689,
-0.92630200888366945,
-0.9675229603596517,
-0.27845290869168118,
],
[
527.107573587542228,
-1.4594385811273854,
-1.450082094097871593,
-1.5843722511996204,
],
];
static DCT64X64_BANDS: [[f32; 8]; 3] = [
[
0.9 * 26629.073922049845,
-1.025,
-0.78,
-0.65012,
-0.19041574084286472,
-0.20819395464,
-0.421064,
-0.3273384553584867,
],
[
0.9 * 9311.323871001005,
-0.3041958212306401,
-0.3633036457487539,
-0.35660379990111464,
-0.3443074455424403,
-0.33699592683512467,
-0.3018086652624211,
-0.27321683125358037,
],
[
0.9 * 4992.248644553863,
-1.2,
-1.2,
-0.8,
-0.7,
-0.7,
-0.4,
-0.5,
],
];
static DCT32X64_BANDS: [[f32; 8]; 3] = [
[
0.65 * 23629.073922049845,
-1.025,
-0.78,
-0.65012,
-0.19041574084286472,
-0.20819395464,
-0.421064,
-0.3273384553584867,
],
[
0.65 * 8611.323871001005,
-0.3041958212306401,
-0.3633036457487539,
-0.35660379990111464,
-0.3443074455424403,
-0.33699592683512467,
-0.3018086652624211,
-0.27321683125358037,
],
[
0.65 * 4492.248644553863,
-1.2,
-1.2,
-0.8,
-0.7,
-0.7,
-0.4,
-0.5,
],
];
static DCT128X128_BANDS: [[f32; 8]; 3] = [
[
1.8 * 26629.073922049845,
-1.025,
-0.78,
-0.65012,
-0.19041574084286472,
-0.20819395464,
-0.421064,
-0.3273384553584867,
],
[
1.8 * 9311.323871001005,
-0.3041958212306401,
-0.3633036457487539,
-0.35660379990111464,
-0.3443074455424403,
-0.33699592683512467,
-0.3018086652624211,
-0.27321683125358037,
],
[
1.8 * 4992.248644553863,
-1.2,
-1.2,
-0.8,
-0.7,
-0.7,
-0.4,
-0.5,
],
];
static DCT64X128_BANDS: [[f32; 8]; 3] = [
[
1.3 * 23629.073922049845,
-1.025,
-0.78,
-0.65012,
-0.19041574084286472,
-0.20819395464,
-0.421064,
-0.3273384553584867,
],
[
1.3 * 8611.323871001005,
-0.3041958212306401,
-0.3633036457487539,
-0.35660379990111464,
-0.3443074455424403,
-0.33699592683512467,
-0.3018086652624211,
-0.27321683125358037,
],
[
1.3 * 4492.248644553863,
-1.2,
-1.2,
-0.8,
-0.7,
-0.7,
-0.4,
-0.5,
],
];
fn compute_band_matrix(src: &[[f32; 8]; 3], rows: usize, cols: usize) -> [Vec<f32>; 3] {
const NUM_BANDS: usize = 8;
std::array::from_fn(|c| {
let mut bands = [0.0f32; NUM_BANDS];
bands[0] = src[c][0];
for i in 1..NUM_BANDS {
bands[i] = bands[i - 1] * band_mult(src[c][i]);
}
let scale = (NUM_BANDS as f32 - 1.0) / (std::f32::consts::SQRT_2 + 1e-6);
let rcprow = scale / (rows - 1) as f32;
let rcpcol = scale / (cols - 1) as f32;
let mut out = vec![0.0f32; rows * cols];
for y in 0..rows {
let dy = y as f32 * rcprow;
for x in 0..cols {
let dx = x as f32 * rcpcol;
out[y * cols + x] =
1.0 / interpolate_vec_bands(fmla(dx, dx, dy * dy).sqrt(), &bands);
}
}
out
})
}
pub(crate) struct Tables128 {
pub(crate) matrix_128x128: [Vec<f32>; 3],
pub(crate) inv_matrix_128x128: [Vec<f32>; 3],
pub(crate) matrix_128x64: [Vec<f32>; 3],
pub(crate) inv_matrix_128x64: [Vec<f32>; 3],
}
pub(crate) fn tables_128() -> &'static Tables128 {
static TABLES: std::sync::OnceLock<Tables128> = std::sync::OnceLock::new();
TABLES.get_or_init(|| {
let reciprocals = |m: &[Vec<f32>; 3]| -> [Vec<f32>; 3] {
std::array::from_fn(|c| {
let mut inv = vec![0.0f32; m[c].len()];
fill_ac_reciprocals(&m[c], &mut inv);
inv
})
};
let matrix_128x128 = compute_band_matrix(&DCT128X128_BANDS, 128, 128);
let matrix_128x64 = compute_band_matrix(&DCT64X128_BANDS, 64, 128);
Tables128 {
inv_matrix_128x128: reciprocals(&matrix_128x128),
inv_matrix_128x64: reciprocals(&matrix_128x64),
matrix_128x128,
matrix_128x64,
}
})
}
fn compute_dct64x64_matrix(override_: Option<&BandOverride>) -> HeapMatrix<f32, 3, 4096> {
const NUM_BANDS: usize = 8;
let mut src = DCT64X64_BANDS;
if let Some(ov) = override_ {
for c in 0..3 {
src[c].copy_from_slice(&ov.bands[c][..NUM_BANDS]);
}
}
let mut out = HeapMatrix::new(0.0f32);
for c in 0..3 {
let mut bands = [0.0f32; NUM_BANDS];
bands[0] = src[c][0];
for i in 1..NUM_BANDS {
bands[i] = bands[i - 1] * band_mult(src[c][i]);
}
let scale = (NUM_BANDS as f32 - 1.0) / (std::f32::consts::SQRT_2 + 1e-6);
let rcp = scale / 63.0;
for y in 0..64 {
let dy = y as f32 * rcp;
for x in 0..64 {
let dx = x as f32 * rcp;
out[c][y * 64 + x] =
1.0 / interpolate_vec_bands(fmla(dx, dx, dy * dy).sqrt(), &bands);
}
}
}
out
}
fn compute_dct64x32_matrix(override_: Option<&BandOverride>) -> HeapMatrix<f32, 3, 2048> {
const NUM_BANDS: usize = 8;
let mut src = DCT32X64_BANDS;
if let Some(ov) = override_ {
for c in 0..3 {
src[c].copy_from_slice(&ov.bands[c][..NUM_BANDS]);
}
}
let mut out = HeapMatrix::new(0.0);
for c in 0..3 {
let mut bands = [0.0f32; NUM_BANDS];
bands[0] = src[c][0];
for i in 1..NUM_BANDS {
bands[i] = bands[i - 1] * band_mult(src[c][i]);
}
let scale = (NUM_BANDS as f32 - 1.0) / (std::f32::consts::SQRT_2 + 1e-6);
let rcprow = scale / 31.0;
let rcpcol = scale / 63.0;
for y in 0..32 {
let dy = y as f32 * rcprow;
for x in 0..64 {
let dx = x as f32 * rcpcol;
out[c][y * 64 + x] =
1.0 / interpolate_vec_bands(fmla(dx, dx, dy * dy).sqrt(), &bands);
}
}
}
out
}
static AFV_BANDS: [[f32; 9]; 3] = [
[3072.0, 3072.0, 256.0, 256.0, 256.0, 414.0, 0.0, 0.0, 0.0],
[1024.0, 1024.0, 50.0, 50.0, 50.0, 58.0, 0.0, 0.0, 0.0],
[384.0, 384.0, 12.0, 12.0, 12.0, 22.0, -0.25, -0.25, -0.25],
];
#[rustfmt::skip]
static AFV_FREQS: [f32; 16] = [
0xBAD as f32, 0xBAD as f32, 0.8517778890324296, 5.37778436506804,
0xBAD as f32, 0xBAD as f32, 4.734747904497923, 5.449245381693219,
1.6598270267479331, 4.0, 7.275749096817861, 10.423227632456525,
2.662932286148962, 7.630657783650829, 8.962388608184032, 12.97166202570235,
];
const AFV_FREQ_LO: f32 = 0.8517778890324296;
const AFV_FREQ_HI: f32 = 12.97166202570235 - AFV_FREQ_LO + 1e-6;
#[inline]
fn interpolate_vec_bands(scaled_pos: f32, bands: &[f32]) -> f32 {
let idx_f = scaled_pos.floor();
let frac = scaled_pos - idx_f;
let idx = idx_f as usize;
let a = bands[idx];
let b = bands[idx + 1];
(b / a).powf(frac) * a
}
fn compute_dct4x8_matrix(override_: Option<&BandOverride>) -> HeapMatrix<f32, 3, 64> {
const NUM_BANDS: usize = 4;
let mut src = DCT4X8_BANDS;
if let Some(o) = override_ {
for c in 0..3 {
src[c].copy_from_slice(&o.bands[c][..NUM_BANDS]);
}
}
let mut out = HeapMatrix::new(0.0f32);
for c in 0..3 {
let mut bands = [0.0f32; NUM_BANDS];
bands[0] = src[c][0];
for i in 1..NUM_BANDS {
bands[i] = bands[i - 1] * band_mult(src[c][i]);
}
let scale = (NUM_BANDS as f32 - 1.0) / (std::f32::consts::SQRT_2 + 1e-6);
let rcprow = scale / 3.0; let rcpcol = scale / 7.0; let mut w4x8 = [0.0f32; 32];
for y in 0..4 {
let dy = y as f32 * rcprow;
let dy2 = dy * dy;
for x in 0..8 {
let dx = x as f32 * rcpcol;
let dist = fmla(dx, dx, dy2).sqrt();
w4x8[y * 8 + x] = interpolate_vec_bands(dist, &bands);
}
}
for y in 0..8 {
for x in 0..8 {
let w = w4x8[(y / 2) * 8 + x];
out[c][y * 8 + x] = 1.0 / w;
}
}
}
out
}
fn compute_dct4x4_matrix() -> HeapMatrix<f32, 3, 64> {
compute_dct4x4_matrix_from(&DCT4X4_BANDS)
}
fn compute_dct4x4_matrix_from(src: &[[f32; 4]; 3]) -> HeapMatrix<f32, 3, 64> {
const NUM_BANDS: usize = 4;
let mut out = HeapMatrix::new(0.0f32);
for c in 0..3 {
let mut bands = [0.0f32; NUM_BANDS];
bands[0] = src[c][0];
for i in 1..NUM_BANDS {
bands[i] = bands[i - 1] * band_mult(src[c][i]);
}
let scale = (NUM_BANDS as f32 - 1.0) / (std::f32::consts::SQRT_2 + 1e-6);
let rcp = scale / 3.0; let mut w4x4 = [0.0f32; 16];
for y in 0..4 {
let dy = y as f32 * rcp;
let dy2 = dy * dy;
for x in 0..4 {
let dx = x as f32 * rcp;
let dist = fmla(dx, dx, dy2).sqrt();
w4x4[y * 4 + x] = interpolate_vec_bands(dist, &bands);
}
}
for y in 0..8 {
for x in 0..8 {
let w = w4x4[(y / 2) * 4 + (x / 2)];
out[c][y * 8 + x] = 1.0 / w;
}
}
}
out
}
fn compute_afv_matrix() -> HeapMatrix<f32, 3, 64> {
compute_afv_matrix_from(&AFV_BANDS, &DCT4X8_BANDS, &DCT4X4_BANDS)
}
fn compute_afv_matrix_from(
afv: &[[f32; 9]; 3],
dct4x8: &[[f32; 4]; 3],
dct4x4: &[[f32; 4]; 3],
) -> HeapMatrix<f32, 3, 64> {
let mut out = HeapMatrix::new(0.0f32);
for c in 0..3 {
let mut bands = [0.0f32; 4];
bands[0] = dct4x8[c][0];
for i in 1..4 {
bands[i] = bands[i - 1] * band_mult(dct4x8[c][i]);
}
let scale = 3.0 / (std::f32::consts::SQRT_2 + 1e-6);
let mut w4x8 = [0.0f32; 32];
for y in 0..4 {
let dy = y as f32 * (scale / 3.0);
let dy2 = dy * dy;
for x in 0..8 {
let dx = x as f32 * (scale / 7.0);
let dist = fmla(dx, dx, dy2).sqrt();
w4x8[y * 8 + x] = interpolate_vec_bands(dist, &bands);
}
}
bands[0] = dct4x4[c][0];
for i in 1..4 {
bands[i] = bands[i - 1] * band_mult(dct4x4[c][i]);
}
let mut w4x4 = [0.0f32; 16];
for y in 0..4 {
let dy = y as f32 * (scale / 3.0);
let dy2 = dy * dy;
for x in 0..4 {
let dx = x as f32 * (scale / 3.0);
let dist = fmla(dx, dx, dy2).sqrt();
w4x4[y * 4 + x] = interpolate_vec_bands(dist, &bands);
}
}
bands[0] = afv[c][5];
for i in 1..4 {
bands[i] = bands[i - 1] * band_mult(afv[c][i + 5]);
}
let mut weights = [0.0f32; 64];
weights[0] = 1.0;
weights[8] = afv[c][0];
weights[1] = afv[c][1];
weights[16] = afv[c][2];
weights[2] = afv[c][3];
weights[18] = afv[c][4];
for y in 0..4 {
for x in 0..4 {
if x < 2 && y < 2 {
continue;
}
let pos = (AFV_FREQS[y * 4 + x] - AFV_FREQ_LO) * 3.0 / AFV_FREQ_HI;
weights[(2 * y) * 8 + 2 * x] = interpolate_vec_bands(pos, &bands);
}
}
for y in 0..4 {
for x in 0..8 {
if x == 0 && y == 0 {
continue;
}
weights[(2 * y + 1) * 8 + x] = w4x8[y * 8 + x];
}
}
for y in 0..4 {
for x in 0..4 {
if x == 0 && y == 0 {
continue;
}
weights[(2 * y) * 8 + 2 * x + 1] = w4x4[y * 4 + x];
}
}
for (o, &w) in out[c].iter_mut().zip(weights.iter()) {
*o = 1.0 / w;
}
}
out
}
fn compute_dct8x8_matrix(override_: &BandOverride) -> HeapMatrix<f32, 3, 64> {
const NUM_BANDS: usize = 6;
let mut out = HeapMatrix::new(0.0f32);
for c in 0..3 {
let mut bands = [0.0f32; NUM_BANDS];
bands[0] = override_.bands[c][0];
for i in 1..NUM_BANDS {
bands[i] = bands[i - 1] * band_mult(override_.bands[c][i]);
}
let scale = (NUM_BANDS as f32 - 1.0) / (std::f32::consts::SQRT_2 + 1e-6);
let rcp = scale / 7.0;
for y in 0..8 {
let dy = y as f32 * rcp;
let dy2 = dy * dy;
for x in 0..8 {
let dx = x as f32 * rcp;
let dist = fmla(dx, dx, dy2).sqrt();
out[c][y * 8 + x] = 1.0 / interpolate_vec_bands(dist, &bands);
}
}
}
out
}
fn compute_dct16x16_matrix(override_: Option<&BandOverride>) -> HeapMatrix<f32, 3, 256> {
const NUM_BANDS: usize = 7;
let mut src = DCT16X16_BANDS;
if let Some(o) = override_ {
for c in 0..3 {
src[c].copy_from_slice(&o.bands[c][..NUM_BANDS]);
}
}
let mut out = HeapMatrix::new(0.0f32);
for c in 0..3 {
let mut bands = [0.0f32; NUM_BANDS];
bands[0] = src[c][0];
for i in 1..NUM_BANDS {
bands[i] = bands[i - 1] * band_mult(src[c][i]);
}
let scale = (NUM_BANDS as f32 - 1.0) / (std::f32::consts::SQRT_2 + 1e-6);
let rcp = scale / 15.0;
for y in 0..16 {
let dy = y as f32 * rcp;
let dy2 = dy * dy;
for x in 0..16 {
let dx = x as f32 * rcp;
let dist = fmla(dx, dx, dy2).sqrt();
let weight = interpolate_vec_bands(dist, &bands);
out[c][y * 16 + x] = 1.0 / weight;
}
}
}
out
}
fn compute_dct32x32_matrix(override_: Option<&BandOverride>) -> HeapMatrix<f32, 3, 1024> {
const NUM_BANDS: usize = 8;
let mut src = DCT32X32_BANDS;
if let Some(o) = override_ {
for c in 0..3 {
src[c].copy_from_slice(&o.bands[c][..NUM_BANDS]);
}
}
let mut out = HeapMatrix::new(0.0f32);
for c in 0..3 {
let mut bands = [0.0f32; NUM_BANDS];
bands[0] = src[c][0];
for i in 1..NUM_BANDS {
bands[i] = bands[i - 1] * band_mult(src[c][i]);
}
let scale = (NUM_BANDS as f32 - 1.0) / (std::f32::consts::SQRT_2 + 1e-6);
let rcp = scale / 31.0;
for y in 0..32 {
let dy = y as f32 * rcp;
let dy2 = dy * dy;
for x in 0..32 {
let dx = x as f32 * rcp;
let dist = fmla(dx, dx, dy2).sqrt();
let weight = interpolate_vec_bands(dist, &bands);
out[c][y * 32 + x] = 1.0 / weight;
}
}
}
out
}
fn compute_dct16x8_matrix(override_: Option<&BandOverride>) -> HeapMatrix<f32, 3, 128> {
const NUM_BANDS: usize = 7;
let mut src = DCT16X8_BANDS;
if let Some(o) = override_ {
for c in 0..3 {
src[c].copy_from_slice(&o.bands[c][..NUM_BANDS]);
}
}
let mut out = HeapMatrix::new(0.0f32);
for c in 0..3 {
let mut bands = [0.0f32; NUM_BANDS];
bands[0] = src[c][0];
for i in 1..NUM_BANDS {
bands[i] = bands[i - 1] * band_mult(src[c][i]);
}
let scale = (NUM_BANDS as f32 - 1.0) / (std::f32::consts::SQRT_2 + 1e-6);
let rcprow = scale / 7.0; let rcpcol = scale / 15.0; for y in 0..8 {
let dy = y as f32 * rcprow;
let dy2 = dy * dy;
for x in 0..16 {
let dx = x as f32 * rcpcol;
let dist = fmla(dx, dx, dy2).sqrt();
let weight = interpolate_vec_bands(dist, &bands);
out[c][y * 16 + x] = 1.0 / weight;
}
}
}
out
}
fn compute_dct32x16_matrix(override_: Option<&BandOverride>) -> HeapMatrix<f32, 3, 512> {
const NUM_BANDS: usize = 8;
let mut src = DCT16X32_BANDS;
if let Some(o) = override_ {
for c in 0..3 {
src[c].copy_from_slice(&o.bands[c][..NUM_BANDS]);
}
}
let mut out = HeapMatrix::new(0.0f32);
for c in 0..3 {
let mut bands = [0.0f32; NUM_BANDS];
bands[0] = src[c][0];
for i in 1..NUM_BANDS {
bands[i] = bands[i - 1] * band_mult(src[c][i]);
}
let scale = (NUM_BANDS as f32 - 1.0) / (std::f32::consts::SQRT_2 + 1e-6);
let rcprow = scale / 15.0; let rcpcol = scale / 31.0; for y in 0..16 {
let dy = y as f32 * rcprow;
let dy2 = dy * dy;
for x in 0..32 {
let dx = x as f32 * rcpcol;
let dist = fmla(dx, dx, dy2).sqrt();
let weight = interpolate_vec_bands(dist, &bands);
out[c][y * 32 + x] = 1.0 / weight;
}
}
}
out
}
static IDENTITY_WEIGHTS: [[f32; 3]; 3] = [
[280.0, 3160.0, 3160.0],
[60.0, 864.0, 864.0],
[18.0, 200.0, 200.0],
];
static DCT2_WEIGHTS: [[f32; 6]; 3] = [
[3840.0, 2560.0, 1280.0, 640.0, 480.0, 300.0],
[960.0, 640.0, 320.0, 180.0, 140.0, 120.0],
[640.0, 320.0, 128.0, 64.0, 32.0, 16.0],
];
type FineMatrixPair = (HeapMatrix<f32, 3, 64>, HeapMatrix<f32, 3, 64>);
static IDENTITY_CHROMA_FINE: [f32; 2] = [1.25, 1.25];
static CHROMATIC_DCT8_ROWS: [f32; 3] = [1.56, 1.07, 0.52];
const ORDINARY_DCT8_B_SCALE: f32 = 0.65;
const SS2_DCT8_B_SCALE: f32 = 0.62;
fn scale_dct8_base_weights(o: &mut BandOverride, mul: [f32; 3]) {
for c in 0..3 {
o.bands[c][0] = f16_bits_to_f32(f32_to_f16_bits(o.bands[c][0] * mul[c] / 64.0)) * 64.0;
}
}
fn scaled_identity_weights(mul: [f32; 2]) -> [[f32; 3]; 3] {
let mut w = IDENTITY_WEIGHTS;
for i in 0..3 {
w[0][i] = f16_bits_to_f32(f32_to_f16_bits(w[0][i] * mul[0] / 64.0)) * 64.0;
w[2][i] = f16_bits_to_f32(f32_to_f16_bits(w[2][i] * mul[1] / 64.0)) * 64.0;
}
w
}
fn identity_pair(id: &[[f32; 3]; 3]) -> FineMatrixPair {
let mut matrix_identity = HeapMatrix::new(0.0);
let mut inv_matrix_identity = HeapMatrix::new(0.0);
for c in 0..3 {
let mut weights = [id[c][0]; 64];
weights[1] = id[c][1];
weights[8] = id[c][1];
weights[9] = id[c][2];
for (k, weight) in weights.into_iter().enumerate() {
matrix_identity[c][k] = 1.0 / weight;
if k != 0 {
inv_matrix_identity[c][k] = weight;
}
}
}
(matrix_identity, inv_matrix_identity)
}
fn dct2_pair(dw: &[[f32; 6]; 3]) -> FineMatrixPair {
let mut matrix_dct2x2 = HeapMatrix::new(0.0);
let mut inv_matrix_dct2x2 = HeapMatrix::new(0.0);
for c in 0..3 {
let w = dw[c];
let mut weights = [0xBAD as f32; 64];
weights[1] = w[0];
weights[8] = w[0];
weights[9] = w[1];
for y in 0..2 {
for x in 0..2 {
weights[y * 8 + x + 2] = w[2];
weights[(y + 2) * 8 + x] = w[2];
weights[(y + 2) * 8 + x + 2] = w[3];
}
}
for y in 0..4 {
for x in 0..4 {
weights[y * 8 + x + 4] = w[4];
weights[(y + 4) * 8 + x] = w[4];
weights[(y + 4) * 8 + x + 4] = w[5];
}
}
for (k, weight) in weights.into_iter().enumerate() {
matrix_dct2x2[c][k] = 1.0 / weight;
if k != 0 {
inv_matrix_dct2x2[c][k] = weight;
}
}
}
(matrix_dct2x2, inv_matrix_dct2x2)
}
fn build_fine_matrices() -> (FineMatrixPair, FineMatrixPair) {
let (matrix_identity, inv_matrix_identity) = identity_pair(&IDENTITY_WEIGHTS);
let (matrix_dct2x2, inv_matrix_dct2x2) = dct2_pair(&DCT2_WEIGHTS);
(
(matrix_identity, inv_matrix_identity),
(matrix_dct2x2, inv_matrix_dct2x2),
)
}
struct SharedTables {
matrix: HeapMatrix<f32, 3, 64>,
inv_matrix: HeapMatrix<f32, 3, 64>,
matrix_identity: HeapMatrix<f32, 3, 64>,
inv_matrix_identity: HeapMatrix<f32, 3, 64>,
matrix_dct2x2: HeapMatrix<f32, 3, 64>,
inv_matrix_dct2x2: HeapMatrix<f32, 3, 64>,
matrix_16x8: HeapMatrix<f32, 3, 128>,
inv_matrix_16x8: HeapMatrix<f32, 3, 128>,
matrix_4x4: HeapMatrix<f32, 3, 64>,
inv_matrix_4x4: HeapMatrix<f32, 3, 64>,
matrix_4x8: HeapMatrix<f32, 3, 64>,
inv_matrix_4x8: HeapMatrix<f32, 3, 64>,
matrix_afv: HeapMatrix<f32, 3, 64>,
inv_matrix_afv: HeapMatrix<f32, 3, 64>,
matrix_64x64: HeapMatrix<f32, 3, 4096>,
inv_matrix_64x64: HeapMatrix<f32, 3, 4096>,
matrix_64x32: HeapMatrix<f32, 3, 2048>,
inv_matrix_64x32: HeapMatrix<f32, 3, 2048>,
}
fn shared_tables() -> &'static SharedTables {
static SHARED: std::sync::OnceLock<Box<SharedTables>> = std::sync::OnceLock::new();
SHARED.get_or_init(|| {
let matrix = HeapMatrix::from_rows(&DEQUANT_MATRIX_8X8);
let mut inv_matrix = HeapMatrix::new(0.);
for c in 0..3 {
fill_ac_reciprocals(&matrix[c], &mut inv_matrix[c]);
}
let matrix_16x8 = HeapMatrix::from_rows(&DEQUANT_MATRIX_16X8);
let mut inv_matrix_16x8 = HeapMatrix::new(0.);
for c in 0..3 {
fill_ac_reciprocals(&matrix_16x8[c], &mut inv_matrix_16x8[c]);
}
let matrix_4x4 = compute_dct4x4_matrix();
let mut inv_matrix_4x4 = HeapMatrix::new(0.);
for c in 0..3 {
fill_ac_reciprocals(&matrix_4x4[c], &mut inv_matrix_4x4[c]);
}
let matrix_4x8 = compute_dct4x8_matrix(None);
let mut inv_matrix_4x8 = HeapMatrix::new(0.);
for c in 0..3 {
fill_ac_reciprocals(&matrix_4x8[c], &mut inv_matrix_4x8[c]);
}
let matrix_afv = compute_afv_matrix();
let mut inv_matrix_afv = HeapMatrix::new(0.);
for c in 0..3 {
fill_ac_reciprocals(&matrix_afv[c], &mut inv_matrix_afv[c]);
}
let matrix_64x64 = compute_dct64x64_matrix(None);
let mut inv_matrix_64x64 = HeapMatrix::new(0.0f32);
for c in 0..3 {
fill_ac_reciprocals(&matrix_64x64[c], &mut inv_matrix_64x64[c]);
}
let matrix_64x32 = compute_dct64x32_matrix(None);
let mut inv_matrix_64x32 = HeapMatrix::new(0.0f32);
for c in 0..3 {
fill_ac_reciprocals(&matrix_64x32[c], &mut inv_matrix_64x32[c]);
}
let ((matrix_identity, inv_matrix_identity), (matrix_dct2x2, inv_matrix_dct2x2)) =
build_fine_matrices();
Box::new(SharedTables {
matrix,
inv_matrix,
matrix_identity,
inv_matrix_identity,
matrix_dct2x2,
inv_matrix_dct2x2,
matrix_16x8,
inv_matrix_16x8,
matrix_4x4,
inv_matrix_4x4,
matrix_4x8,
inv_matrix_4x8,
matrix_afv,
inv_matrix_afv,
matrix_64x64,
inv_matrix_64x64,
matrix_64x32,
inv_matrix_64x32,
})
})
}
struct LargeTables {
matrix_16x16: HeapMatrix<f32, 3, 256>,
inv_matrix_16x16: HeapMatrix<f32, 3, 256>,
matrix_32x32: HeapMatrix<f32, 3, 1024>,
inv_matrix_32x32: HeapMatrix<f32, 3, 1024>,
matrix_32x16: HeapMatrix<f32, 3, 512>,
inv_matrix_32x16: HeapMatrix<f32, 3, 512>,
}
fn dct16_override(use_ss2: bool, tilt: Option<Arbiter>) -> Option<BandOverride> {
match tilt {
Some(a) => Some(tilted_override(
&DCT16X16_BANDS,
QM_SS2_SCALE16,
a.tuning().dct16_luma_hf,
)),
None => use_ss2.then(|| scaled_override(&DCT16X16_BANDS, QM_SS2_SCALE16)),
}
}
fn large_tables(use_ss2: bool, tilt: Option<Arbiter>) -> &'static LargeTables {
static DEFAULT: std::sync::OnceLock<LargeTables> = std::sync::OnceLock::new();
static SS2: std::sync::OnceLock<LargeTables> = std::sync::OnceLock::new();
static TILTED: [std::sync::OnceLock<LargeTables>; 2] =
[std::sync::OnceLock::new(), std::sync::OnceLock::new()];
let cache = match tilt {
Some(a) => &TILTED[a as usize],
None if use_ss2 => &SS2,
None => &DEFAULT,
};
cache.get_or_init(|| {
let o16 = dct16_override(use_ss2, tilt);
let o32 = use_ss2.then(|| scaled_override(&DCT32X32_BANDS, QM_SS2_SCALE32));
let o32x16 = use_ss2.then(|| scaled_override(&DCT16X32_BANDS, QM_SS2_SCALE16X32));
let matrix_32x16 = compute_dct32x16_matrix(o32x16.as_ref());
let mut inv_matrix_32x16 = HeapMatrix::new(0.);
for c in 0..3 {
fill_ac_reciprocals(&matrix_32x16[c], &mut inv_matrix_32x16[c]);
}
let matrix_16x16 = compute_dct16x16_matrix(o16.as_ref());
let mut inv_matrix_16x16 = HeapMatrix::new(0.);
for c in 0..3 {
fill_ac_reciprocals(&matrix_16x16[c], &mut inv_matrix_16x16[c]);
}
let matrix_32x32 = compute_dct32x32_matrix(o32.as_ref());
let mut inv_matrix_32x32 = HeapMatrix::new(0.);
for c in 0..3 {
fill_ac_reciprocals(&matrix_32x32[c], &mut inv_matrix_32x32[c]);
}
LargeTables {
matrix_16x16,
inv_matrix_16x16,
matrix_32x32,
inv_matrix_32x32,
matrix_32x16,
inv_matrix_32x16,
}
})
}
impl DequantMatrices {
pub(crate) fn color_proxy_matrix(distance: f32, c: usize) -> &'static [f32; 64] {
static HQ: std::sync::OnceLock<HeapMatrix<f32, 3, 64>> = std::sync::OnceLock::new();
static MID: std::sync::OnceLock<HeapMatrix<f32, 3, 64>> = std::sync::OnceLock::new();
static COARSE: std::sync::OnceLock<HeapMatrix<f32, 3, 64>> = std::sync::OnceLock::new();
if distance < QM_FLAT_B8_MIN_DISTANCE {
return &DEQUANT_MATRIX_8X8[c];
}
let table = if distance >= ORIGINAL_COARSE_DCT8_DISTANCE {
COARSE.get_or_init(|| {
compute_dct8x8_matrix(&default_dct8_override(true, &FLAT_B8_BANDS_MID))
})
} else if distance >= QM_FLAT_B8_MID_MIN_DISTANCE {
MID.get_or_init(|| {
compute_dct8x8_matrix(&default_dct8_override(false, &FLAT_B8_BANDS_MID))
})
} else {
HQ.get_or_init(|| {
compute_dct8x8_matrix(&default_dct8_override(false, &FLAT_B8_BANDS_HQ))
})
};
&table[c]
}
pub(crate) fn new_x_heavy(distance: f32, arbiter: Arbiter) -> &'static Self {
let ss2_from = arbiter.tuning().qm_ss2_distance;
static HUE: std::sync::OnceLock<DequantMatrices> = std::sync::OnceLock::new();
if (HUE_B8_MIN_DISTANCE..ss2_from).contains(&distance) {
HUE.get_or_init(|| Self::compute(false, false, Some(&HUE_B8_BANDS), false, true))
} else {
Self::new(0.0, arbiter)
}
}
fn dct16_tilt(distance: f32, arbiter: Arbiter) -> Option<Arbiter> {
((QM_DCT8_MIN_DISTANCE..QM_DCT16_TILT_MAX_DISTANCE).contains(&distance)
&& arbiter.tuning().dct16_luma_hf != 1.0)
.then_some(arbiter)
}
pub(crate) fn new(distance: f32, arbiter: Arbiter) -> &'static Self {
static COARSE_TILTED: [std::sync::OnceLock<DequantMatrices>; 2] =
[std::sync::OnceLock::new(), std::sync::OnceLock::new()];
match Self::dct16_tilt(distance, arbiter) {
Some(a) => COARSE_TILTED[a as usize].get_or_init(|| {
Self::compute_tilted(true, true, Some(&FLAT_B8_BANDS_MID), false, false, Some(a))
}),
None => Self::new_untilted(distance, arbiter),
}
}
fn new_untilted(distance: f32, arbiter: Arbiter) -> &'static Self {
let ss2_from = arbiter.tuning().qm_ss2_distance;
static HQ: std::sync::OnceLock<DequantMatrices> = std::sync::OnceLock::new();
static DEFAULT_HQ: std::sync::OnceLock<DequantMatrices> = std::sync::OnceLock::new();
static DEFAULT_MID: std::sync::OnceLock<DequantMatrices> = std::sync::OnceLock::new();
static SS2: std::sync::OnceLock<DequantMatrices> = std::sync::OnceLock::new();
static COARSE: std::sync::OnceLock<DequantMatrices> = std::sync::OnceLock::new();
if distance >= QM_DCT8_MIN_DISTANCE {
COARSE.get_or_init(|| Self::compute(true, true, Some(&FLAT_B8_BANDS_MID), false, false))
} else if distance >= ss2_from {
SS2.get_or_init(|| Self::compute(true, false, Some(&FLAT_B8_BANDS_MID), false, false))
} else if distance >= QM_FLAT_B8_MID_MIN_DISTANCE {
DEFAULT_MID
.get_or_init(|| Self::compute(false, false, Some(&FLAT_B8_BANDS_MID), false, false))
} else if distance >= QM_FLAT_B8_MIN_DISTANCE {
DEFAULT_HQ
.get_or_init(|| Self::compute(false, false, Some(&FLAT_B8_BANDS_HQ), false, false))
} else {
HQ.get_or_init(|| Self::compute(false, false, None, false, false))
}
}
pub(crate) fn new_fast(distance: f32, arbiter: Arbiter) -> &'static Self {
let ss2_from = arbiter.tuning().qm_ss2_distance;
static HQ: std::sync::OnceLock<DequantMatrices> = std::sync::OnceLock::new();
static MID: std::sync::OnceLock<DequantMatrices> = std::sync::OnceLock::new();
static SS2: std::sync::OnceLock<DequantMatrices> = std::sync::OnceLock::new();
static COARSE: std::sync::OnceLock<DequantMatrices> = std::sync::OnceLock::new();
if distance < QM_FLAT_B8_MIN_DISTANCE {
return Self::new(distance, arbiter);
}
if distance >= ss2_from {
let coarse = distance >= ORIGINAL_COARSE_DCT8_DISTANCE;
let cache = if coarse { &COARSE } else { &SS2 };
return cache.get_or_init(|| {
Self::compute_with_ordinary_fit(
true,
coarse,
Some(&FLAT_B8_BANDS_MID),
false,
false,
false,
None,
)
});
}
let (cache, bands) = if distance >= QM_FLAT_B8_MID_MIN_DISTANCE {
(&MID, &FLAT_B8_BANDS_MID)
} else {
(&HQ, &FLAT_B8_BANDS_HQ)
};
cache.get_or_init(|| {
Self::compute_with_ordinary_fit(false, false, Some(bands), false, false, false, None)
})
}
pub(crate) fn new_pair_b(distance: f32, arbiter: Arbiter) -> &'static Self {
static PAIR_MID: std::sync::OnceLock<DequantMatrices> = std::sync::OnceLock::new();
if (PAIR_B_FINE_MIN_DISTANCE..PAIR_B_FINE_MAX_DISTANCE).contains(&distance) {
PAIR_MID
.get_or_init(|| Self::compute(false, false, Some(&FLAT_B8_BANDS_MID), true, false))
} else {
Self::new(distance, arbiter)
}
}
pub(crate) fn new_saturated_pair_b(distance: f32, arbiter: Arbiter) -> &'static Self {
static SAT_PAIR: std::sync::OnceLock<DequantMatrices> = std::sync::OnceLock::new();
if (PAIR_B_FINE_MIN_DISTANCE..PAIR_B_FINE_MAX_DISTANCE).contains(&distance) {
SAT_PAIR.get_or_init(|| Self::compute(false, false, Some(&SAT_B8_BANDS), true, true))
} else {
Self::new_saturated(distance, arbiter)
}
}
pub(crate) fn new_saturated(distance: f32, arbiter: Arbiter) -> &'static Self {
let ss2_from = arbiter.tuning().qm_ss2_distance;
static SAT_DEFAULT: std::sync::OnceLock<DequantMatrices> = std::sync::OnceLock::new();
static SAT_SS2: std::sync::OnceLock<DequantMatrices> = std::sync::OnceLock::new();
static SAT_COARSE: std::sync::OnceLock<DequantMatrices> = std::sync::OnceLock::new();
static SAT_COARSE_TILTED: [std::sync::OnceLock<DequantMatrices>; 2] =
[std::sync::OnceLock::new(), std::sync::OnceLock::new()];
if let Some(a) = Self::dct16_tilt(distance, arbiter) {
SAT_COARSE_TILTED[a as usize].get_or_init(|| {
Self::compute_tilted(true, true, Some(&SAT_B8_BANDS), false, true, Some(a))
})
} else if distance >= QM_DCT8_MIN_DISTANCE {
SAT_COARSE.get_or_init(|| Self::compute(true, true, Some(&SAT_B8_BANDS), false, true))
} else if distance >= ss2_from {
SAT_SS2.get_or_init(|| Self::compute(true, false, Some(&SAT_B8_BANDS), false, true))
} else if distance >= QM_FLAT_B8_MIN_DISTANCE {
SAT_DEFAULT
.get_or_init(|| Self::compute(false, false, Some(&SAT_B8_BANDS), false, true))
} else {
Self::new(distance, arbiter)
}
}
fn compute(
use_ss2: bool,
use_coarse_dct8: bool,
flat_b8: Option<&[f32; 6]>,
pair_b: bool,
chromatic: bool,
) -> Self {
Self::compute_tilted(use_ss2, use_coarse_dct8, flat_b8, pair_b, chromatic, None)
}
fn compute_tilted(
use_ss2: bool,
use_coarse_dct8: bool,
flat_b8: Option<&[f32; 6]>,
pair_b: bool,
chromatic: bool,
tilt: Option<Arbiter>,
) -> Self {
Self::compute_with_ordinary_fit(
use_ss2,
use_coarse_dct8,
flat_b8,
pair_b,
chromatic,
true,
tilt,
)
}
fn compute_with_ordinary_fit(
use_ss2: bool,
use_coarse_dct8: bool,
flat_b8: Option<&[f32; 6]>,
pair_b: bool,
chromatic: bool,
ordinary_fit: bool,
tilt: Option<Arbiter>,
) -> Self {
let identity_weights = chromatic.then(|| scaled_identity_weights(IDENTITY_CHROMA_FINE));
let (matrix_identity, inv_matrix_identity) = match identity_weights.as_ref() {
Some(id) => identity_pair(id),
None => (
shared_tables().matrix_identity.clone(),
shared_tables().inv_matrix_identity.clone(),
),
};
let shared = shared_tables();
let mut o8 = flat_b8.map(|bands| default_dct8_override(use_coarse_dct8, bands));
if chromatic && let Some(o) = o8.as_mut() {
scale_dct8_base_weights(o, CHROMATIC_DCT8_ROWS);
} else if ordinary_fit
&& !use_ss2
&& let Some(o) = o8.as_mut()
{
scale_dct8_base_weights(o, [1.0, 1.0, ORDINARY_DCT8_B_SCALE]);
} else if ordinary_fit
&& use_ss2
&& let Some(o) = o8.as_mut()
{
scale_dct8_base_weights(o, [1.0, 1.0, SS2_DCT8_B_SCALE]);
}
let o16 = dct16_override(use_ss2, tilt);
let o32 = use_ss2.then(|| scaled_override(&DCT32X32_BANDS, QM_SS2_SCALE32));
let o64: Option<BandOverride> = None;
let o64r: Option<BandOverride> = None;
let matrix_64x64: &'static [[f32; 4096]; 3] = &shared.matrix_64x64;
let inv_matrix_64x64: &'static [[f32; 4096]; 3] = &shared.inv_matrix_64x64;
let matrix_64x32: &'static [[f32; 2048]; 3] = &shared.matrix_64x32;
let inv_matrix_64x32: &'static [[f32; 2048]; 3] = &shared.inv_matrix_64x32;
let o3: Option<BandOverride> = pair_b.then(|| {
let mut bands = DCT16X8_BANDS;
bands[2][1] = PAIR_B_FINE_BAND1;
scaled_override(&bands, 1.0)
});
let (matrix_16x8, inv_matrix_16x8) = match o3.as_ref() {
None => (shared.matrix_16x8.clone(), shared.inv_matrix_16x8.clone()),
Some(ov) => {
let m = compute_dct16x8_matrix(Some(ov));
let mut inv = HeapMatrix::new(0.0f32);
for c in 0..3 {
fill_ac_reciprocals(&m[c], &mut inv[c]);
}
(m, inv)
}
};
let o32x16 = use_ss2.then(|| scaled_override(&DCT16X32_BANDS, QM_SS2_SCALE16X32));
let large = large_tables(use_ss2, tilt);
let (matrix, inv_matrix) = if let Some(o8) = o8.as_ref() {
let matrix = compute_dct8x8_matrix(o8);
let mut inv_matrix = HeapMatrix::new(0.);
for c in 0..3 {
fill_ac_reciprocals(&matrix[c], &mut inv_matrix[c]);
}
(matrix, inv_matrix)
} else {
(shared.matrix.clone(), shared.inv_matrix.clone())
};
Self {
identity_weights,
fine_tables: None,
matrix,
inv_matrix,
matrix_identity,
inv_matrix_identity,
matrix_dct2x2: &shared.matrix_dct2x2,
inv_matrix_dct2x2: &shared.inv_matrix_dct2x2,
matrix_16x8,
inv_matrix_16x8,
matrix_16x16: &large.matrix_16x16,
inv_matrix_16x16: &large.inv_matrix_16x16,
matrix_32x32: &large.matrix_32x32,
inv_matrix_32x32: &large.inv_matrix_32x32,
matrix_64x64,
inv_matrix_64x64,
matrix_64x32,
inv_matrix_64x32,
custom_tables: heap_array_from_fn(|i| match i {
0 => o8,
5 => None,
3 => o3,
1 => o16,
2 => o32,
4 => o32x16,
6 => o64r,
7 => o64,
_ => unreachable!(),
}),
matrix_4x4: &shared.matrix_4x4,
inv_matrix_4x4: &shared.inv_matrix_4x4,
matrix_4x8: &shared.matrix_4x8,
inv_matrix_4x8: &shared.inv_matrix_4x8,
matrix_32x16: &large.matrix_32x16,
inv_matrix_32x16: &large.inv_matrix_32x16,
matrix_afv: &shared.matrix_afv,
inv_matrix_afv: &shared.inv_matrix_afv,
}
}
pub(crate) fn new_plain(
distance: f32,
rectangles: bool,
arbiter: Arbiter,
coding: crate::coding::CodingTransform,
plane_scale: [f32; 3],
) -> &'static Self {
type Key = (u32, bool, u8, [u32; 3]);
type Cache = std::sync::Mutex<Vec<(Key, &'static DequantMatrices)>>;
static CACHE: Cache = std::sync::Mutex::new(Vec::new());
let template = if rectangles {
Self::new_untilted(distance, arbiter)
} else {
Self::new_fast(distance, arbiter)
};
#[allow(clippy::identity_op)]
let key = (
(template as *const Self as usize & 0xffff_ffff) as u32,
rectangles,
coding as u8,
plane_scale.map(f32::to_bits),
);
let mut cache = CACHE.lock().unwrap_or_else(|e| e.into_inner());
if let Some(&(_, m)) = cache.iter().find(|(k, _)| *k == key) {
return m;
}
let m: &'static Self = Box::leak(Box::new(Self::compute_plain(
template,
std::array::from_fn(|c| coding.table_scale()[c] * plane_scale[c]),
coding.table_hf_tilt(),
)));
cache.push((key, m));
m
}
fn compute_plain(template: &Self, scale: [f32; 3], hf_tilt: [f32; 3]) -> Self {
fn wire_abs(v: f32) -> f32 {
f16_bits_to_f32(f32_to_f16_bits(v / 64.0)) * 64.0
}
fn wire_rel(v: f32) -> f32 {
f16_bits_to_f32(f32_to_f16_bits(v))
}
let tilt = |v: f32, per_band: f32| -> f32 {
let ratio = band_mult(v) * per_band;
if ratio > 1.0 {
ratio - 1.0
} else {
1.0 - 1.0 / ratio
}
};
let plain = |luma: &[f32]| -> BandOverride {
let mut out = BandOverride {
num_bands: luma.len(),
bands: [[0.0; 16]; 3],
};
let transitions = (luma.len() - 1).max(1) as f32;
for c in 0..3 {
let per_band = hf_tilt[c].powf(1.0 / transitions);
for (i, &v) in luma.iter().enumerate() {
out.bands[c][i] = if i == 0 {
wire_abs(v * scale[c])
} else {
wire_rel(tilt(v, per_band))
};
}
}
out
};
let luma_of = |slot: usize, library: &[f32]| -> BandOverride {
match template.custom_tables[slot].as_ref() {
Some(o) => plain(&o.bands[1][..o.num_bands]),
None => plain(library),
}
};
let o8 = luma_of(0, &DCT8_BANDS[1]);
let o16 = luma_of(1, &DCT16X16_BANDS[1]);
let o32 = luma_of(2, &DCT32X32_BANDS[1]);
let o16x8 = luma_of(3, &DCT16X8_BANDS[1]);
let o32x16 = luma_of(4, &DCT16X32_BANDS[1]);
let o4x8 = luma_of(5, &DCT4X8_BANDS[1]);
let o64r = luma_of(6, &DCT32X64_BANDS[1]);
let o64 = luma_of(7, &DCT64X64_BANDS[1]);
let identity: [[f32; 3]; 3] =
std::array::from_fn(|c| IDENTITY_WEIGHTS[1].map(|v| wire_abs(v * scale[c])));
let dct2: [[f32; 6]; 3] =
std::array::from_fn(|c| DCT2_WEIGHTS[1].map(|v| wire_abs(v * scale[c])));
let dct4x4: [[f32; 4]; 3] = std::array::from_fn(|c| {
std::array::from_fn(|i| {
let v = DCT4X4_BANDS[1][i];
if i == 0 {
wire_abs(v * scale[c])
} else {
wire_rel(v)
}
})
});
let afv: [[f32; 9]; 3] = std::array::from_fn(|c| {
std::array::from_fn(|i| {
let v = AFV_BANDS[1][i];
if i < 6 {
wire_abs(v * scale[c])
} else {
wire_rel(v)
}
})
});
let dct4x8: [[f32; 4]; 3] =
std::array::from_fn(|c| std::array::from_fn(|i| o4x8.bands[c][i]));
fn with_inverse<const N: usize>(
m: HeapMatrix<f32, 3, N>,
) -> (HeapMatrix<f32, 3, N>, HeapMatrix<f32, 3, N>) {
let mut inv = HeapMatrix::new(0.0f32);
for c in 0..3 {
fill_ac_reciprocals(&m[c], &mut inv[c]);
}
(m, inv)
}
fn leak<const N: usize>(
(m, inv): (HeapMatrix<f32, 3, N>, HeapMatrix<f32, 3, N>),
) -> (&'static [[f32; N]; 3], &'static [[f32; N]; 3]) {
(&**Box::leak(Box::new(m)), &**Box::leak(Box::new(inv)))
}
let (matrix, inv_matrix) = with_inverse(compute_dct8x8_matrix(&o8));
let (matrix_identity, inv_matrix_identity) = identity_pair(&identity);
let (matrix_dct2x2, inv_matrix_dct2x2) = leak(dct2_pair(&dct2));
let (matrix_16x8, inv_matrix_16x8) = with_inverse(compute_dct16x8_matrix(Some(&o16x8)));
let (matrix_16x16, inv_matrix_16x16) =
leak(with_inverse(compute_dct16x16_matrix(Some(&o16))));
let (matrix_32x32, inv_matrix_32x32) =
leak(with_inverse(compute_dct32x32_matrix(Some(&o32))));
let (matrix_64x64, inv_matrix_64x64) =
leak(with_inverse(compute_dct64x64_matrix(Some(&o64))));
let (matrix_64x32, inv_matrix_64x32) =
leak(with_inverse(compute_dct64x32_matrix(Some(&o64r))));
let (matrix_4x4, inv_matrix_4x4) = leak(with_inverse(compute_dct4x4_matrix_from(&dct4x4)));
let (matrix_4x8, inv_matrix_4x8) = leak(with_inverse(compute_dct4x8_matrix(Some(&o4x8))));
let (matrix_32x16, inv_matrix_32x16) =
leak(with_inverse(compute_dct32x16_matrix(Some(&o32x16))));
let (matrix_afv, inv_matrix_afv) = leak(with_inverse(compute_afv_matrix_from(
&afv, &dct4x8, &dct4x4,
)));
Self {
identity_weights: Some(identity),
fine_tables: Some(Box::new(FineTables { dct2, dct4x4, afv })),
matrix,
inv_matrix,
matrix_identity,
inv_matrix_identity,
matrix_dct2x2,
inv_matrix_dct2x2,
matrix_16x8,
inv_matrix_16x8,
matrix_16x16,
inv_matrix_16x16,
matrix_32x32,
inv_matrix_32x32,
matrix_64x64,
inv_matrix_64x64,
matrix_64x32,
inv_matrix_64x32,
custom_tables: heap_array_from_fn(|i| {
Some(match i {
0 => o8,
1 => o16,
2 => o32,
3 => o16x8,
4 => o32x16,
5 => o4x8,
6 => o64r,
7 => o64,
_ => unreachable!(),
})
}),
matrix_4x4,
inv_matrix_4x4,
matrix_4x8,
inv_matrix_4x8,
matrix_32x16,
inv_matrix_32x16,
matrix_afv,
inv_matrix_afv,
}
}
#[inline]
pub(crate) fn matrix(&self, c: usize) -> &[f32; 64] {
&self.matrix[c]
}
#[inline]
pub(crate) fn inv_matrix(&self, c: usize) -> &[f32; 64] {
&self.inv_matrix[c]
}
#[inline]
pub(crate) fn matrix_identity(&self, c: usize) -> &[f32; 64] {
&self.matrix_identity[c]
}
#[inline]
pub(crate) fn inv_matrix_identity(&self, c: usize) -> &[f32; 64] {
&self.inv_matrix_identity[c]
}
#[inline]
pub(crate) fn matrix_dct2x2(&self, c: usize) -> &[f32; 64] {
&self.matrix_dct2x2[c]
}
#[inline]
pub(crate) fn inv_matrix_dct2x2(&self, c: usize) -> &[f32; 64] {
&self.inv_matrix_dct2x2[c]
}
#[inline]
pub(crate) fn matrix_16x8(&self, c: usize) -> &[f32; 128] {
&self.matrix_16x8[c]
}
#[inline]
pub(crate) fn inv_matrix_16x8(&self, c: usize) -> &[f32; 128] {
&self.inv_matrix_16x8[c]
}
#[inline]
pub(crate) fn matrix_16x16(&self, c: usize) -> &[f32; 256] {
&self.matrix_16x16[c]
}
#[inline]
pub(crate) fn inv_matrix_16x16(&self, c: usize) -> &[f32; 256] {
&self.inv_matrix_16x16[c]
}
#[inline]
pub(crate) fn matrix_32x32(&self, c: usize) -> &[f32; 1024] {
&self.matrix_32x32[c]
}
#[inline]
pub(crate) fn inv_matrix_32x32(&self, c: usize) -> &[f32; 1024] {
&self.inv_matrix_32x32[c]
}
#[inline]
pub(crate) fn matrix_64x64(&self, c: usize) -> &[f32; 4096] {
&self.matrix_64x64[c]
}
#[inline]
pub(crate) fn inv_matrix_64x64(&self, c: usize) -> &[f32; 4096] {
&self.inv_matrix_64x64[c]
}
#[inline]
pub(crate) fn matrix_128(&self, strategy: u8, c: usize) -> &'static [f32] {
let t = tables_128();
if strategy == crate::dc_group_data::STRATEGY_DCT128X128 {
&t.matrix_128x128[c]
} else {
&t.matrix_128x64[c]
}
}
#[inline]
pub(crate) fn inv_matrix_128(&self, strategy: u8, c: usize) -> &'static [f32] {
let t = tables_128();
if strategy == crate::dc_group_data::STRATEGY_DCT128X128 {
&t.inv_matrix_128x128[c]
} else {
&t.inv_matrix_128x64[c]
}
}
#[inline]
pub(crate) fn matrix_64x32(&self, c: usize) -> &[f32; 2048] {
&self.matrix_64x32[c]
}
#[inline]
pub(crate) fn inv_matrix_64x32(&self, c: usize) -> &[f32; 2048] {
&self.inv_matrix_64x32[c]
}
#[inline]
pub(crate) fn matrix_4x4(&self, c: usize) -> &[f32; 64] {
&self.matrix_4x4[c]
}
#[inline]
pub(crate) fn inv_matrix_4x4(&self, c: usize) -> &[f32; 64] {
&self.inv_matrix_4x4[c]
}
#[inline]
pub(crate) fn matrix_4x8(&self, c: usize) -> &[f32; 64] {
&self.matrix_4x8[c]
}
#[inline]
pub(crate) fn inv_matrix_4x8(&self, c: usize) -> &[f32; 64] {
&self.inv_matrix_4x8[c]
}
#[inline]
pub(crate) fn matrix_32x16(&self, c: usize) -> &[f32; 512] {
&self.matrix_32x16[c]
}
#[inline]
pub(crate) fn inv_matrix_32x16(&self, c: usize) -> &[f32; 512] {
&self.inv_matrix_32x16[c]
}
#[inline]
pub(crate) fn matrix_afv(&self, c: usize) -> &[f32; 64] {
&self.matrix_afv[c]
}
#[inline]
pub(crate) fn inv_matrix_afv(&self, c: usize) -> &[f32; 64] {
&self.inv_matrix_afv[c]
}
}
#[inline(never)]
fn fill_ac_reciprocals(matrix: &[f32], inv: &mut [f32]) {
debug_assert_eq!(matrix.len(), inv.len());
for (&value, out) in matrix[1..].iter().zip(&mut inv[1..]) {
*out = 1.0 / value;
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn generic_band_matrix_matches_the_64_tables() {
let square = compute_band_matrix(&DCT64X64_BANDS, 64, 64);
let rect = compute_band_matrix(&DCT32X64_BANDS, 32, 64);
let m64 = compute_dct64x64_matrix(None);
let m6432 = compute_dct64x32_matrix(None);
for c in 0..3 {
assert_eq!(&square[c][..], &m64[c][..]);
assert_eq!(&rect[c][..], &m6432[c][..]);
}
let t = tables_128();
assert_eq!(t.matrix_128x128[1].len(), 16384);
assert_eq!(t.matrix_128x64[1].len(), 8192);
}
#[test]
fn chromatic_sets_scale_the_dct8_base_weights() {
for d in [1.0, 1.5, 2.0] {
let base = DequantMatrices::new(d, Arbiter::Ssimulacra2).custom_tables[0]
.expect("flat DCT8 table");
for set in [
DequantMatrices::new_saturated(d, Arbiter::Ssimulacra2),
DequantMatrices::new_saturated_pair_b(d, Arbiter::Ssimulacra2),
DequantMatrices::new_x_heavy(d, Arbiter::Ssimulacra2),
] {
let t = set.custom_tables[0].expect("chromatic DCT8 table");
for c in 0..3 {
let spec = if c == 2 {
t.bands[2][0]
} else {
base.bands[c][0] * CHROMATIC_DCT8_ROWS[c]
};
if c != 2 {
assert!((t.bands[c][0] / spec - 1.0).abs() < 2e-3, "d={d} c={c}");
}
assert_eq!(
f16_bits_to_f32(f32_to_f16_bits(t.bands[c][0] / 64.0)) * 64.0,
t.bands[c][0]
);
}
let b_spec = f16_bits_to_f32(f32_to_f16_bits(512.0 / 64.0)) * 64.0;
assert!(
(t.bands[2][0] / (b_spec * CHROMATIC_DCT8_ROWS[2]) - 1.0).abs() < 2e-3,
"d={d}"
);
}
assert!(std::ptr::eq(
DequantMatrices::new_pair_b(d, Arbiter::Ssimulacra2).custom_tables[0]
.as_ref()
.map(|o| &o.bands[1][0])
.unwrap(),
DequantMatrices::new_pair_b(d, Arbiter::Ssimulacra2).custom_tables[0]
.as_ref()
.map(|o| &o.bands[1][0])
.unwrap()
));
}
}
#[test]
fn identity_rows_are_signalled_only_on_chromatic_sets() {
for d in [1.0, 1.5, 2.0] {
assert!(
DequantMatrices::new(d, Arbiter::Ssimulacra2)
.identity_weights
.is_none()
);
assert!(
DequantMatrices::new_pair_b(d, Arbiter::Ssimulacra2)
.identity_weights
.is_none()
);
for set in [
DequantMatrices::new_saturated(d, Arbiter::Ssimulacra2),
DequantMatrices::new_saturated_pair_b(d, Arbiter::Ssimulacra2),
DequantMatrices::new_x_heavy(d, Arbiter::Ssimulacra2),
] {
let id = set.identity_weights.expect("chromatic identity table");
assert_eq!(id[1], IDENTITY_WEIGHTS[1]);
for c in [0, 2] {
for i in 0..3 {
let expected = IDENTITY_WEIGHTS[c][i] * IDENTITY_CHROMA_FINE[c / 2];
assert!(
(id[c][i] / expected - 1.0).abs() < 2e-3,
"{c} {i} {}",
id[c][i]
);
assert_eq!(
f16_bits_to_f32(f32_to_f16_bits(id[c][i] / 64.0)) * 64.0,
id[c][i]
);
}
}
assert_eq!(set.inv_matrix_identity(0)[2], id[0][0]);
assert_eq!(set.inv_matrix_identity(2)[9], id[2][2]);
assert_eq!(set.matrix_identity(0)[1], 1.0 / id[0][1]);
}
}
}
#[test]
fn x_heavy_b_table_matches_its_signaled_bands() {
let spec = DequantMatrices::new(0.0, Arbiter::Ssimulacra2);
for d in [0.0, 0.5, 0.749, QM_SS2_MIN_DISTANCE, 3.0, 4.0] {
assert!(std::ptr::eq(
DequantMatrices::new_x_heavy(d, Arbiter::Ssimulacra2),
spec
));
}
let hue = DequantMatrices::new_x_heavy(1.0, Arbiter::Ssimulacra2);
for d in [HUE_B8_MIN_DISTANCE, 1.25, 2.0, 2.249] {
assert!(std::ptr::eq(
DequantMatrices::new_x_heavy(d, Arbiter::Ssimulacra2),
hue
));
}
let bands = hue.custom_tables[0].unwrap();
let decoded = compute_dct8x8_matrix(&bands);
for c in 0..3 {
for k in 1..64 {
assert_eq!(hue.matrix(c)[k], decoded[c][k]);
assert_eq!(hue.inv_matrix(c)[k], 1.0 / decoded[c][k]);
}
if c != 2 {
let mut expected = default_dct8_override(false, &DCT8_BANDS[2]);
scale_dct8_base_weights(&mut expected, CHROMATIC_DCT8_ROWS);
assert_eq!(bands.bands[c], expected.bands[c]);
}
}
let tail = hue.inv_matrix(2)[63] / spec.inv_matrix(2)[63];
assert!(
(tail / (1.2 * CHROMATIC_DCT8_ROWS[2]) - 1.0).abs() < 3e-3,
"{tail}"
);
assert!(hue.custom_tables[1..].iter().all(Option::is_none));
assert!(!std::ptr::eq(
hue,
DequantMatrices::new(1.0, Arbiter::Ssimulacra2)
));
}
#[test]
fn identity_and_dct2_tables_match_the_jpeg_xl_library() {
let m = DequantMatrices::new(0.0, Arbiter::Ssimulacra2);
let identity = m.inv_matrix_identity(0);
assert_eq!(identity[0], 0.0);
assert_eq!(identity[1], 3160.0);
assert_eq!(identity[8], 3160.0);
assert_eq!(identity[9], 3160.0);
assert_eq!(identity[2], 280.0);
assert_eq!(m.matrix_identity(0)[2], 1.0 / 280.0);
let dct2 = m.inv_matrix_dct2x2(2);
assert_eq!(dct2[0], 0.0);
assert_eq!(dct2[1], 640.0);
assert_eq!(dct2[9], 320.0);
assert_eq!(dct2[2], 128.0);
assert_eq!(dct2[18], 64.0);
assert_eq!(dct2[4], 32.0);
assert_eq!(dct2[36], 16.0);
assert_eq!(m.matrix_dct2x2(2)[36], 1.0 / 16.0);
}
#[test]
fn dequant_matrices_are_only_heap_handles() {
let size = std::mem::size_of::<DequantMatrices>();
assert!(size <= 432, "DequantMatrices grew to {size} bytes");
}
#[test]
fn dequant_matrices_construct_on_a_small_stack() {
std::thread::Builder::new()
.name("small-stack-dequant-test".into())
.stack_size(64 * 1024)
.spawn(|| {
let _matrices =
DequantMatrices::compute(false, false, Some(&FLAT_B8_BANDS_MID), false, false);
})
.unwrap()
.join()
.unwrap();
}
#[test]
fn coarse_dct8_preserves_more_luma_high_frequency() {
assert!(
DequantMatrices::new(QM_DCT8_MIN_DISTANCE - 0.01, Arbiter::Ssimulacra2).custom_tables
[0]
.is_some()
);
assert!(
DequantMatrices::new(QM_DCT8_MIN_DISTANCE, Arbiter::Ssimulacra2).custom_tables[0]
.is_some()
);
let normal = DequantMatrices::compute(true, false, Some(&FLAT_B8_BANDS_MID), false, false);
let coarse = DequantMatrices::compute(true, true, Some(&FLAT_B8_BANDS_MID), false, false);
let hf_ratio = coarse.inv_matrix(1)[63] / normal.inv_matrix(1)[63];
assert!((hf_ratio - QM_DCT8_Y_HF_SCALE).abs() < 0.01, "{hf_ratio}");
let lf_ratio = coarse.inv_matrix(1)[1] / normal.inv_matrix(1)[1];
assert!((1.0..1.03).contains(&lf_ratio), "{lf_ratio}");
}
#[test]
fn saturated_tables_swap_only_above_the_flat_b_gate() {
assert!(std::ptr::eq(
DequantMatrices::new_saturated(QM_FLAT_B8_MIN_DISTANCE - 0.01, Arbiter::Ssimulacra2),
DequantMatrices::new(QM_FLAT_B8_MIN_DISTANCE - 0.01, Arbiter::Ssimulacra2),
));
for d in [0.5, 1.5, 2.5, 4.0] {
let sat = DequantMatrices::new_saturated(d, Arbiter::Ssimulacra2);
let base = DequantMatrices::new(d, Arbiter::Ssimulacra2);
assert!(!std::ptr::eq(sat, base), "d={d}");
let table = sat.custom_tables[0].expect("saturated flat table");
assert_eq!(
table.bands[2][1],
f16_bits_to_f32(f32_to_f16_bits(SAT_B8_BANDS[1])),
"d={d}"
);
}
}
#[test]
fn flat_b8_gated_off_at_very_high_quality() {
assert!(
DequantMatrices::new(QM_FLAT_B8_MIN_DISTANCE - 0.01, Arbiter::Ssimulacra2)
.custom_tables[0]
.is_none()
);
assert!(
DequantMatrices::new(QM_FLAT_B8_MIN_DISTANCE, Arbiter::Ssimulacra2).custom_tables[0]
.is_some()
);
}
#[test]
fn flat_b8_shape_splits_at_mid_distance() {
let below = DequantMatrices::new(QM_FLAT_B8_MID_MIN_DISTANCE - 0.01, Arbiter::Ssimulacra2)
.custom_tables[0]
.expect("hq flat table");
let at = DequantMatrices::new(QM_FLAT_B8_MID_MIN_DISTANCE, Arbiter::Ssimulacra2)
.custom_tables[0]
.expect("mid flat table");
assert_eq!(
below.bands[2][1],
f16_bits_to_f32(f32_to_f16_bits(FLAT_B8_BANDS_HQ[1]))
);
assert_eq!(
at.bands[2][1],
f16_bits_to_f32(f32_to_f16_bits(FLAT_B8_BANDS_MID[1]))
);
for c in [0usize, 1] {
assert_eq!(below.bands[c][..6], at.bands[c][..6]);
}
}
#[test]
fn default_dct8_preserves_flat_blue_shape_with_fitted_rate() {
let m = DequantMatrices::compute(false, false, Some(&FLAT_B8_BANDS_HQ), false, false);
for c in [0usize, 1] {
for k in 1..64 {
let spec = DEQUANT_MATRIX_8X8[c][k];
let got = m.matrix(c)[k];
assert!(
(got - spec).abs() <= 2e-3 * spec.abs(),
"c={c} k={k}: {got} vs spec {spec}"
);
}
}
for k in 1..64 {
let ratio = m.matrix(2)[k] / DEQUANT_MATRIX_8X8[2][k];
assert!(ratio <= 1.001 / ORDINARY_DCT8_B_SCALE, "k={k}: {ratio}");
}
let b_hf = m.matrix(2)[63] / DEQUANT_MATRIX_8X8[2][63];
assert!(
b_hf < 0.35 / ORDINARY_DCT8_B_SCALE,
"B HF step ratio {b_hf}"
);
}
#[test]
fn ordinary_dct8_fit_matches_wire_weights_and_existing_gates() {
for d in [0.3, 0.75, 1.0, 1.25, 2.0, 2.249] {
for m in [
DequantMatrices::new(d, Arbiter::Ssimulacra2),
DequantMatrices::new_pair_b(d, Arbiter::Ssimulacra2),
] {
let table = m.custom_tables[0].expect("existing flat DCT8 table");
assert_eq!(table.num_bands, 6);
assert_eq!(table.bands[2][0], 332.75); let decoded = compute_dct8x8_matrix(&table);
for c in 0..3 {
for k in 1..64 {
assert_eq!(m.matrix(c)[k], decoded[c][k]);
assert_eq!(m.inv_matrix(c)[k], 1.0 / decoded[c][k]);
}
}
}
}
assert!(DequantMatrices::new(0.299, Arbiter::Ssimulacra2).custom_tables[0].is_none());
for d in [2.25, 3.0, 3.5, 5.0] {
assert_eq!(
DequantMatrices::new(d, Arbiter::Ssimulacra2).custom_tables[0]
.unwrap()
.bands[2][0],
317.5 );
assert_eq!(
DequantMatrices::new_fast(d, Arbiter::Ssimulacra2).custom_tables[0]
.unwrap()
.bands[2][0],
512.0
);
}
}
#[test]
fn color_proxy_retains_its_original_matrix_calibration() {
for d in [0.29, 0.3, 0.75, 1.249, 1.25, 2.25, 3.5] {
let expected = if d < QM_FLAT_B8_MIN_DISTANCE {
HeapMatrix::from_rows(&DEQUANT_MATRIX_8X8)
} else {
let b = if d < QM_FLAT_B8_MID_MIN_DISTANCE {
&FLAT_B8_BANDS_HQ
} else {
&FLAT_B8_BANDS_MID
};
compute_dct8x8_matrix(&default_dct8_override(
d >= ORIGINAL_COARSE_DCT8_DISTANCE,
b,
))
};
for c in 0..3 {
assert_eq!(DequantMatrices::color_proxy_matrix(d, c), &expected[c]);
}
}
}
#[test]
fn fast_keeps_original_dct8_weights() {
for d in [0.29, 0.3, 0.75, 1.25, 2.0, 2.249, 2.25, 3.5] {
let fast = DequantMatrices::new_fast(d, Arbiter::Ssimulacra2);
for c in 0..3 {
assert_eq!(fast.matrix(c), DequantMatrices::color_proxy_matrix(d, c));
}
}
}
#[test]
fn dct8_matrix_from_bands_matches_static() {
let identity = BandOverride {
num_bands: 6,
bands: {
let mut b = [[0.0f32; 16]; 3];
for c in 0..3 {
b[c][..6].copy_from_slice(&DCT8_BANDS[c]);
}
b
},
};
let computed = compute_dct8x8_matrix(&identity);
for c in 0..3 {
for k in 0..64 {
let expected = DEQUANT_MATRIX_8X8[c][k];
let got = computed[c][k];
assert!(
(got - expected).abs() <= 2e-6 * expected.abs(),
"c={c} k={k}: computed {got}, static {expected}"
);
}
}
}
#[test]
fn pair_b_tier_exists_only_inside_its_band() {
for d in [0.5, 1.0, 1.24, PAIR_B_FINE_MAX_DISTANCE, 3.0] {
assert!(std::ptr::eq(
DequantMatrices::new_pair_b(d, Arbiter::Ssimulacra2),
DequantMatrices::new(d, Arbiter::Ssimulacra2)
));
assert!(std::ptr::eq(
DequantMatrices::new_saturated_pair_b(d, Arbiter::Ssimulacra2),
DequantMatrices::new_saturated(d, Arbiter::Ssimulacra2)
));
}
for d in [PAIR_B_FINE_MIN_DISTANCE, 1.5, 2.0] {
assert!(DequantMatrices::new(d, Arbiter::Ssimulacra2).custom_tables[3].is_none());
for m in [
DequantMatrices::new_pair_b(d, Arbiter::Ssimulacra2),
DequantMatrices::new_saturated_pair_b(d, Arbiter::Ssimulacra2),
] {
let t = m.custom_tables[3].expect("finer pair-B table");
assert_eq!(t.num_bands, 7);
assert_eq!(
t.bands[2][1],
f16_bits_to_f32(f32_to_f16_bits(PAIR_B_FINE_BAND1))
);
assert_eq!(
t.bands[2][2],
f16_bits_to_f32(f32_to_f16_bits(DCT16X8_BANDS[2][2]))
);
assert_eq!(
t.bands[1][1],
f16_bits_to_f32(f32_to_f16_bits(DCT16X8_BANDS[1][1]))
);
assert!(
m.matrix_16x8(2)[2]
< DequantMatrices::new(d, Arbiter::Ssimulacra2).matrix_16x8(2)[2]
);
}
}
}
#[test]
fn dct16x8_matrix_from_bands_matches_static() {
let identity = BandOverride {
num_bands: 7,
bands: {
let mut b = [[0.0f32; 16]; 3];
for c in 0..3 {
b[c][..7].copy_from_slice(&DCT16X8_BANDS[c]);
}
b
},
};
let computed = compute_dct16x8_matrix(Some(&identity));
for c in 0..3 {
for k in 0..128 {
let expected = DEQUANT_MATRIX_16X8[c][k];
let got = computed[c][k];
assert!(
(got - expected).abs() <= 4e-6 * expected.abs(),
"c={c} k={k}: computed {got}, static {expected}"
);
}
}
}
#[test]
fn rect_tables_default_to_the_library_values() {
let m = DequantMatrices::new(1.0, Arbiter::Ssimulacra2);
for (c, k) in (0..3).flat_map(|c| (0..128).map(move |k| (c, k))) {
let expected = DEQUANT_MATRIX_16X8[c][k];
let got = m.matrix_16x8(c)[k];
assert!(
(got - expected).abs() <= 4e-6 * expected.abs(),
"16x8 c={c} k={k}: {got} vs {expected}"
);
}
let identity = BandOverride {
num_bands: 8,
bands: {
let mut b = [[0.0f32; 16]; 3];
for c in 0..3 {
b[c][..8].copy_from_slice(&DCT16X32_BANDS[c]);
}
b
},
};
let computed = compute_dct32x16_matrix(Some(&identity));
for c in 0..3 {
for k in 0..512 {
assert_eq!(computed[c][k], m.matrix_32x16(c)[k], "32x16 c={c} k={k}");
}
}
}
#[test]
fn afv_matrix_fixed_positions_match_library_weights() {
let m = DequantMatrices::new(1.0, Arbiter::Ssimulacra2);
for c in 0..3 {
let mc = m.matrix_afv(c);
assert_eq!(mc[8], 1.0 / AFV_BANDS[c][0], "c={c} [8]");
assert_eq!(mc[1], 1.0 / AFV_BANDS[c][1], "c={c} [1]");
assert_eq!(mc[16], 1.0 / AFV_BANDS[c][2], "c={c} [16]");
assert_eq!(mc[2], 1.0 / AFV_BANDS[c][3], "c={c} [2]");
assert_eq!(mc[18], 1.0 / AFV_BANDS[c][4], "c={c} [18]");
for x in 1..8 {
assert_eq!(mc[8 + x], m.matrix_4x8(c)[x], "c={c} 4x8 col {x}");
}
for x in 1..4 {
assert_eq!(mc[2 * x + 1], m.matrix_4x4(c)[2 * x], "c={c} 4x4 col {x}");
}
for (k, &v) in mc.iter().enumerate() {
assert!(v > 0.0 && v.is_finite(), "c={c} k={k}: {v}");
}
assert_eq!(m.inv_matrix_afv(c)[0], 0.0);
}
}
#[test]
fn dct16x16_matrix_dc_matches_libjxl_polynomial() {
let m = DequantMatrices::new(1.0, Arbiter::Ssimulacra2);
let x_dc = m.matrix_16x16(0)[0];
assert!(
(x_dc - 1.0 / 8996.872_57).abs() < 1e-7,
"X[0,0]={}, expected ~{}",
x_dc,
1.0 / 8996.872_57
);
let y_dc = m.matrix_16x16(1)[0];
assert!(
(y_dc - 1.0 / 3191.483_66).abs() < 1e-7,
"Y[0,0]={}, expected ~{}",
y_dc,
1.0 / 3191.483_66
);
let b_dc = m.matrix_16x16(2)[0];
assert!(
(b_dc - 1.0 / 1157.504_08).abs() < 1e-7,
"B[0,0]={}, expected ~{}",
b_dc,
1.0 / 1157.504_08
);
for c in 0..3 {
let m_c = m.matrix_16x16(c);
for k in 0..256 {
assert!(
m_c[k] > 0.0,
"matrix[{},{}] = {} not positive",
c,
k,
m_c[k]
);
}
assert!(
m_c[255] > m_c[0],
"matrix[{}] HF should exceed DC: {} <= {}",
c,
m_c[255],
m_c[0]
);
}
}
}
#[cfg(test)]
mod shared_storage_tests {
use super::*;
#[test]
fn invariant_tables_share_storage_across_all_preset_families() {
let shared = shared_tables();
for distance in [0.0, 0.1, 0.3, 0.75, 1.0, 1.5, 2.0, 2.25, 3.0, 4.0, 8.0] {
for set in [
DequantMatrices::new(distance, Arbiter::Ssimulacra2),
DequantMatrices::new_fast(distance, Arbiter::Ssimulacra2),
DequantMatrices::new_x_heavy(distance, Arbiter::Ssimulacra2),
DequantMatrices::new_pair_b(distance, Arbiter::Ssimulacra2),
DequantMatrices::new_saturated(distance, Arbiter::Ssimulacra2),
DequantMatrices::new_saturated_pair_b(distance, Arbiter::Ssimulacra2),
] {
let large = large_tables(set.custom_tables[1].is_some(), None);
assert!(std::ptr::eq(set.matrix_16x16, &*large.matrix_16x16));
assert!(std::ptr::eq(set.inv_matrix_16x16, &*large.inv_matrix_16x16));
assert!(std::ptr::eq(set.matrix_32x32, &*large.matrix_32x32));
assert!(std::ptr::eq(set.inv_matrix_32x32, &*large.inv_matrix_32x32));
assert!(std::ptr::eq(set.matrix_32x16, &*large.matrix_32x16));
assert!(std::ptr::eq(set.inv_matrix_32x16, &*large.inv_matrix_32x16));
assert!(std::ptr::eq(
set.matrix_dct2x2.as_ptr(),
shared.matrix_dct2x2.as_ptr()
));
assert!(std::ptr::eq(
set.inv_matrix_dct2x2.as_ptr(),
shared.inv_matrix_dct2x2.as_ptr()
));
assert!(std::ptr::eq(
set.matrix_4x4.as_ptr(),
shared.matrix_4x4.as_ptr()
));
assert!(std::ptr::eq(
set.inv_matrix_4x4.as_ptr(),
shared.inv_matrix_4x4.as_ptr()
));
assert!(std::ptr::eq(
set.matrix_4x8.as_ptr(),
shared.matrix_4x8.as_ptr()
));
assert!(std::ptr::eq(
set.inv_matrix_4x8.as_ptr(),
shared.inv_matrix_4x8.as_ptr()
));
assert!(std::ptr::eq(
set.matrix_afv.as_ptr(),
shared.matrix_afv.as_ptr()
));
assert!(std::ptr::eq(
set.inv_matrix_afv.as_ptr(),
shared.inv_matrix_afv.as_ptr()
));
assert!(std::ptr::eq(
set.matrix_64x64.as_ptr(),
shared.matrix_64x64.as_ptr()
));
assert!(std::ptr::eq(
set.inv_matrix_64x64.as_ptr(),
shared.inv_matrix_64x64.as_ptr()
));
assert!(std::ptr::eq(
set.matrix_64x32.as_ptr(),
shared.matrix_64x32.as_ptr()
));
assert!(std::ptr::eq(
set.inv_matrix_64x32.as_ptr(),
shared.inv_matrix_64x32.as_ptr()
));
}
}
}
}