use super::*;
use crate::adaptive_quant::{dirty_log2f, fast_exp2};
use crate::entropy::pack_signed;
use crate::frame::ac_metadata_context;
mod layout_dp;
mod leaf_first;
mod recon64;
mod search;
pub(crate) use layout_dp::LayoutDpScratch;
pub(crate) use leaf_first::LeafChoice;
pub(crate) use recon64::Recon64Scratch;
use search::{
CandidateSearch, FINE_RUNNER_UP_RATIO, MEDIUM_CANDIDATES_PER_REGION,
MERGE32_CANDIDATES_PER_REGION, MERGE32_SHORTLIST_RATIO,
};
#[derive(Clone, Copy)]
struct AcStrategyParams<'a> {
ctx: &'a EncodingContext,
opsin: &'a Image3F,
dc_group_px: usize,
dc_group_py: usize,
distance: f32,
scale: f32,
qm_mult_x: f32,
ytox_map: &'a ImageSB,
ytob_map: &'a ImageSB,
prices: Option<&'a std::sync::Arc<RatePrices>>,
}
impl AcStrategyParams<'_> {
fn price<'s>(&self, scratch: &'s mut CoderScratch) -> RatePricesScope<'s> {
RatePricesScope::new(scratch, self.prices.cloned())
}
}
struct SuperBlockContext<'a> {
params: AcStrategyParams<'a>,
meta_r: f32,
scope: SearchScope,
dct8_cost_y0: usize,
dct8_cost_stride: usize,
}
struct SuperBlockInput {
bx: usize,
by: usize,
qac: [[f32; 2]; 2],
}
static MERGE_PARTITIONS: [u8; 8] = [0, 1, 2, 4, 6, 8, 16, 24];
fn partition_cost(children: [[f32; 2]; 2], merges: [f32; 5], mask: u8) -> f32 {
if mask & 1 != 0 {
return merges[0];
}
let mut cost = 0.0;
for (k, value) in merges.iter().enumerate().skip(1) {
if mask & (1 << k) != 0 {
cost += value;
}
}
for (y, row) in children.iter().enumerate() {
for (x, &value) in row.iter().enumerate() {
if mask & ((1 << (1 + x)) | (1 << (3 + y))) == 0 {
cost += value;
}
}
}
cost
}
fn choose_merge_partition(
children: [[f32; 2]; 2],
baseline: [[f32; 2]; 2],
merges: [f32; 5],
accept: [f32; 5],
) -> u8 {
let totals = |c: [[f32; 2]; 2]| {
[
c.iter().flatten().sum(),
c[0][0] + c[1][0],
c[0][1] + c[1][1],
c[0][0] + c[0][1],
c[1][0] + c[1][1],
]
};
let child_costs = totals(children);
let base_costs = totals(baseline);
let mut best = 0;
let mut best_cost = child_costs[0];
for mask in MERGE_PARTITIONS.into_iter().skip(1) {
let (mut merged, mut incumbent, mut limit) = (0.0, 0.0, 0.0);
for k in 0..5 {
if mask & (1 << k) != 0 {
merged += merges[k];
incumbent += child_costs[k];
limit += base_costs[k] * accept[k];
}
}
if !(merged < incumbent && merged < limit) {
continue;
}
let cost = partition_cost(children, merges, mask);
if cost < best_cost {
best = mask;
best_cost = cost;
}
}
best
}
fn merge32_sites(bx: usize, by: usize) -> [(usize, usize, u8); 5] {
[
(bx, by, STRATEGY_DCT32X32),
(bx, by, STRATEGY_DCT32X16),
(bx + 2, by, STRATEGY_DCT32X16),
(bx, by, STRATEGY_DCT16X32),
(bx, by + 2, STRATEGY_DCT16X32),
]
}
struct UpgradeBand {
raw: [f32; 5],
incumbent: [f32; 5],
}
fn record_upgrade_candidates(
upgrades: &mut Vec<MergeUpgradeCandidate>,
ac_strategy: &AcStrategyImage,
bx0: usize,
by0: usize,
band: UpgradeBand,
) {
let sites = [
(bx0, by0, STRATEGY_DCT16X16),
(bx0, by0, STRATEGY_DCT16X8),
(bx0 + 1, by0, STRATEGY_DCT16X8),
(bx0, by0, STRATEGY_DCT8X16),
(bx0, by0 + 1, STRATEGY_DCT8X16),
];
for (k, &(bx, by, strategy)) in sites.iter().enumerate() {
if band.raw[k] < band.incumbent[k] && ac_strategy.can_place_strategy(bx, by, strategy) {
upgrades.push(MergeUpgradeCandidate {
bx: bx as u16,
by: by as u16,
strategy,
relative_cost: band.raw[k] / band.incumbent[k],
});
}
}
}
fn record_upgrade32_candidates(
upgrades: &mut Vec<MergeUpgradeCandidate>,
ac_strategy: &AcStrategyImage,
bx: usize,
by: usize,
raw: [f32; 5],
children: [[f32; 2]; 2],
) {
let incumbents = [
children.iter().flatten().sum(),
children[0][0] + children[1][0],
children[0][1] + children[1][1],
children[0][0] + children[0][1],
children[1][0] + children[1][1],
];
let mut candidates: Vec<_> = merge32_sites(bx, by)
.into_iter()
.enumerate()
.filter_map(|(k, (x, y, strategy))| {
let ratio = raw[k] / incumbents[k];
(ratio.is_finite()
&& ratio <= MERGE32_SHORTLIST_RATIO
&& ac_strategy.can_place_strategy(x, y, strategy))
.then_some(MergeUpgradeCandidate {
bx: x as u16,
by: y as u16,
strategy,
relative_cost: ratio,
})
})
.collect();
candidates.sort_by(|a, b| {
a.relative_cost
.total_cmp(&b.relative_cost)
.then_with(|| (a.by, a.bx, a.strategy).cmp(&(b.by, b.bx, b.strategy)))
});
upgrades.extend(candidates.into_iter().take(MERGE32_CANDIDATES_PER_REGION));
}
fn select_super_block(
context: &SuperBlockContext<'_>,
scratch: &mut CoderScratch,
ac_strategy: &mut AcStrategyImage,
input: SuperBlockInput,
saved: &mut Vec<SavedChild>,
upgrades: &mut Vec<MergeUpgradeCandidate>,
) -> SuperBlockCost {
let params = context.params;
let ctx = params.ctx;
let (bx0, by0) = (input.bx, input.by);
let (px0, py0) = (params.dc_group_px + bx0 * 8, params.dc_group_py + by0 * 8);
let qac = input.qac;
let cmap_factor = cmap_factors(
params.ctx.cfl_frame(),
params.ytox_map,
params.ytob_map,
bx0,
by0,
);
let mut c8 = [[0.0f32; 2]; 2];
for dy in 0..2 {
for dx in 0..2 {
c8[dy][dx] = strategy_cost(
ctx,
scratch,
STRATEGY_DCT,
params.opsin,
px0 + dx * 8,
py0 + dy * 8,
qac[dy][dx],
params.qm_mult_x,
context.meta_r,
params.distance,
cmap_factor,
);
scratch.dct8_costs
[(by0 + dy - context.dct8_cost_y0) * context.dct8_cost_stride + bx0 + dx] =
c8[dy][dx];
}
}
let dct8_left = c8[0][0] + c8[1][0];
let dct8_right = c8[0][1] + c8[1][1];
let dct8_top = c8[0][0] + c8[0][1];
let dct8_bottom = c8[1][0] + c8[1][1];
let total_dct8 = dct8_left + dct8_right;
let prescreened =
!context.scope.merge_busy_quads() && total_dct8 > MERGE_PRESCREEN_BITS * RD_LAMBDA;
if prescreened {
return SuperBlockCost {
chosen: total_dct8,
dct8: total_dct8,
prescreened,
};
}
let merge = ctx.merge;
let mut rect_cost = |px: usize, py: usize, strategy: u8, qac: f32| -> (f32, f32) {
if !context.scope.rectangles() {
return (f32::INFINITY, f32::INFINITY);
}
let raw = strategy_cost(
ctx,
scratch,
strategy,
params.opsin,
px,
py,
qac,
params.qm_mult_x,
context.meta_r,
params.distance,
cmap_factor,
);
(merge.bias_rect * raw, raw)
};
let (v_left, v_left_raw) = rect_cost(px0, py0, STRATEGY_DCT16X8, qac[0][0].max(qac[1][0]));
let (v_right, v_right_raw) =
rect_cost(px0 + 8, py0, STRATEGY_DCT16X8, qac[0][1].max(qac[1][1]));
let (h_top, h_top_raw) = rect_cost(px0, py0, STRATEGY_DCT8X16, qac[0][0].max(qac[0][1]));
let (h_bot, h_bot_raw) = rect_cost(px0, py0 + 8, STRATEGY_DCT8X16, qac[1][0].max(qac[1][1]));
let c16_raw = strategy_cost(
ctx,
scratch,
STRATEGY_DCT16X16,
params.opsin,
px0,
py0,
aggregate_qac_2x2(qac, params.scale, params.distance),
params.qm_mult_x,
context.meta_r,
params.distance,
cmap_factor,
);
let c16 = merge.bias_16x16 * c16_raw;
let use_v_left = ac_strategy.can_place_strategy(bx0, by0, STRATEGY_DCT16X8)
&& merge_beats_dct8(v_left, dct8_left, merge.accept_pair);
let use_v_right = ac_strategy.can_place_strategy(bx0 + 1, by0, STRATEGY_DCT16X8)
&& merge_beats_dct8(v_right, dct8_right, merge.accept_pair);
let use_h_top = ac_strategy.can_place_strategy(bx0, by0, STRATEGY_DCT8X16)
&& merge_beats_dct8(h_top, dct8_top, merge.accept_pair);
let use_h_bottom = ac_strategy.can_place_strategy(bx0, by0 + 1, STRATEGY_DCT8X16)
&& merge_beats_dct8(h_bot, dct8_bottom, merge.accept_pair);
let cost_16x8 = if use_v_left { v_left } else { dct8_left }
+ if use_v_right { v_right } else { dct8_right };
let cost_8x16 =
if use_h_top { h_top } else { dct8_top } + if use_h_bottom { h_bot } else { dct8_bottom };
let best_rect = cost_16x8.min(cost_8x16);
let (q_min, q_max) = qac
.iter()
.flatten()
.fold((f32::INFINITY, 0.0f32), |(mn, mx), &q| {
(mn.min(q), mx.max(q))
});
let pick_16x16 = ac_strategy.can_place_strategy(bx0, by0, STRATEGY_DCT16X16)
&& c16 < best_rect
&& merge_beats_dct8(
c16,
total_dct8,
risk_gated(merge.risk_k, merge.accept_16, q_min, q_max, 1.0),
);
if params.distance >= MERGE_UPGRADE_MIN_DISTANCE {
record_upgrade_candidates(
upgrades,
ac_strategy,
bx0,
by0,
UpgradeBand {
raw: [c16_raw, v_left_raw, v_right_raw, h_top_raw, h_bot_raw],
incumbent: [total_dct8, dct8_left, dct8_right, dct8_top, dct8_bottom],
},
);
}
let chosen = if pick_16x16 {
let mut grid = [NO_CHILD_BLOCK; 16];
let any_rect = if cost_16x8 <= cost_8x16 {
grid[0] = if use_v_left {
STRATEGY_DCT16X8
} else {
STRATEGY_DCT
};
grid[4] = if use_v_left {
NO_CHILD_BLOCK
} else {
STRATEGY_DCT
};
grid[1] = if use_v_right {
STRATEGY_DCT16X8
} else {
STRATEGY_DCT
};
grid[5] = if use_v_right {
NO_CHILD_BLOCK
} else {
STRATEGY_DCT
};
use_v_left || use_v_right
} else {
grid[0] = if use_h_top {
STRATEGY_DCT8X16
} else {
STRATEGY_DCT
};
grid[1] = if use_h_top {
NO_CHILD_BLOCK
} else {
STRATEGY_DCT
};
grid[4] = if use_h_bottom {
STRATEGY_DCT8X16
} else {
STRATEGY_DCT
};
grid[5] = if use_h_bottom {
NO_CHILD_BLOCK
} else {
STRATEGY_DCT
};
use_h_top || use_h_bottom
};
if any_rect {
saved.push(SavedChild {
bx: bx0 as u16,
by: by0 as u16,
grid,
});
}
ac_strategy.set_first(bx0, by0, STRATEGY_DCT16X16);
c16
} else if cost_16x8 <= cost_8x16 {
if use_v_left {
ac_strategy.set_first(bx0, by0, STRATEGY_DCT16X8);
}
if use_v_right {
ac_strategy.set_first(bx0 + 1, by0, STRATEGY_DCT16X8);
}
cost_16x8
} else {
if use_h_top {
ac_strategy.set_first(bx0, by0, STRATEGY_DCT8X16);
}
if use_h_bottom {
ac_strategy.set_first(bx0, by0 + 1, STRATEGY_DCT8X16);
}
cost_8x16
};
SuperBlockCost {
chosen,
dct8: total_dct8,
prescreened,
}
}
#[inline]
fn mean_max_mixer(butteraugli_target: f32) -> f32 {
const K_LIMIT: f32 = 1.54138;
const K_MUL: f32 = 0.56391;
(1.0 - (butteraugli_target - K_LIMIT).max(0.0) * K_MUL).max(0.0)
}
#[inline]
pub(crate) fn aggregate_quant(max_q: u8, sum: u32, covered: usize, butteraugli_target: f32) -> u8 {
if covered < 4 {
return max_q;
}
let mixer = mean_max_mixer(butteraugli_target);
let mean = sum as f32 / covered as f32;
let mixed = max_q as f32 * mixer + mean * (1.0 - mixer);
(mixed + 0.5).clamp(1.0, 255.0) as u8
}
#[inline]
fn aggregate_qac_2x2(qac: [[f32; 2]; 2], scale: f32, butteraugli_target: f32) -> f32 {
let max_q = qac[0][0].max(qac[0][1]).max(qac[1][0]).max(qac[1][1]);
let mixer = mean_max_mixer(butteraugli_target);
let mean = (qac[0][0] + qac[0][1] + qac[1][0] + qac[1][1]) * 0.25;
let mixed_q = (max_q * mixer + mean * (1.0 - mixer)) / scale;
scale * (mixed_q + 0.5).clamp(1.0, 255.0).floor()
}
pub(crate) fn adjust_quant_field(
ac_strategy: &AcStrategyImage,
butteraugli_target: f32,
quant_field: &mut ImageB,
) {
for (x, y, raw_strategy) in ac_strategy.iter_first_blocks() {
let cov_x = AcStrategyImage::covered_blocks_x_of(raw_strategy);
let cov_y = AcStrategyImage::covered_blocks_y_of(raw_strategy);
if cov_x == 1 && cov_y == 1 {
continue;
}
let mut max_q: u8 = 0;
let mut sum: u32 = 0;
for iy in 0..cov_y {
for &q in &quant_field.row(y + iy)[x..x + cov_x] {
max_q = max_q.max(q);
sum += q as u32;
}
}
let val = aggregate_quant(max_q, sum, cov_x * cov_y, butteraugli_target);
for iy in 0..cov_y {
quant_field.row_mut(y + iy)[x..x + cov_x].fill(val);
}
}
}
const QF_PRICE_PRIOR_WEIGHT: f32 = 2.0;
const QF_TRELLIS_ROUNDS: usize = 2;
#[inline]
fn quant_refinement_steps(distance: f32) -> usize {
if distance < 1.5 {
0
} else if distance >= 2.0 {
2
} else {
1
}
}
struct QuantRefinementContext<'a> {
params: AcStrategyParams<'a>,
ac_strategy: &'a AcStrategyImage,
current_costs: &'a [f32],
bands: &'a [(usize, usize)],
num_threads: usize,
}
struct QuantRefinementSearch<'a> {
context: &'a QuantRefinementContext<'a>,
quant_field: &'a ImageB,
}
fn find_quant_refinements(
search: &QuantRefinementSearch<'_>,
scratch: &mut CoderScratch,
band: (usize, usize),
refinements: &mut Vec<QuantRefinement>,
) {
let context = search.context;
let params = context.params;
let quant_field = search.quant_field;
let ac_strategy = context.ac_strategy;
let (y0, y1) = band;
let ctx = params.ctx;
let distance = params.distance;
let steps = quant_refinement_steps(distance);
if steps == 0 {
return;
}
for (bx, by, strategy) in ac_strategy.iter_first_blocks() {
if by < y0 || by >= y1 {
continue;
}
if matches!(
strategy,
STRATEGY_DCT64X64 | STRATEGY_DCT64X32 | STRATEGY_DCT32X64
) {
continue;
}
let current_q = quant_field.row(by)[bx];
if current_q <= 1 {
continue;
}
let cmap = cmap_factors(
params.ctx.cfl_frame(),
params.ytox_map,
params.ytob_map,
bx,
by,
);
let px = params.dc_group_px + bx * 8;
let py = params.dc_group_py + by * 8;
let CoderScratch {
strategy_coeffs: coeffs,
transform_gather,
recon,
rate_prices,
..
} = scratch;
let prices = rate_prices.as_deref();
let (cx, cy, _) = prepare_strategy_coeffs(
ctx,
coeffs,
transform_gather,
strategy,
params.opsin,
px,
py,
cmap,
);
let mut cost = |q: u8| {
let qac = params.scale * q as f32;
let ReconCost {
distortion, rate, ..
} = reconstruction_dist_and_rate(
ctx,
recon,
coeffs,
strategy,
params.opsin,
px,
py,
qac,
params.qm_mult_x,
distance,
cmap,
cx,
cy,
0.0,
0.0,
false,
prices,
);
rd_cost(
DistortionModel::Reconstruction,
distance,
0.0,
distortion,
rate,
)
};
let cached_cost = context
.current_costs
.get(by * ac_strategy.xsize() + bx)
.copied()
.unwrap_or(f32::NAN);
let base_cost = if cached_cost.is_finite() {
cached_cost
} else {
cost(current_q)
};
let up = if ctx.x_heavy() {
current_q.saturating_add(1)
} else {
current_q
};
let candidates = if steps == 2 {
[current_q.saturating_sub(1), current_q.saturating_sub(2), up]
} else {
[current_q.saturating_sub(1), up, current_q]
};
let mut refinement = QuantRefinement {
bx,
by,
states: [(current_q, base_cost); 4],
len: 1,
};
for candidate in candidates {
if candidate == 0
|| refinement.states[..refinement.len]
.iter()
.any(|&(q, _)| q == candidate)
{
continue;
}
refinement.states[refinement.len] = (candidate, cost(candidate));
refinement.len += 1;
}
if refinement.len > 1 {
refinements.push(refinement);
}
}
}
#[inline]
fn independent_choice(refinement: &QuantRefinement) -> u8 {
let mut best = refinement.states[0];
for &state in &refinement.states[1..refinement.len] {
if state.1 < best.1 {
best = state;
}
}
best.0
}
fn qf_residual_prices(first_left: i32, chain: &[u8], prices: &mut [[f32; 512]; 4]) {
let mut counts = [[0u32; 512]; 4];
let mut totals = [0u32; 4];
let mut left = first_left;
for &q in chain {
let qf = q as i32 - 1;
let (c, v) = qf_symbol(left, qf);
counts[c][v] += 1;
totals[c] += 1;
left = qf;
}
for c in 0..4 {
let denom = totals[c] as f32 + QF_PRICE_PRIOR_WEIGHT;
for v in 0..512 {
let prior = QF_PRICE_PRIOR_WEIGHT * fast_exp2(-(v as f32 + 1.0).min(100.0));
prices[c][v] = -dirty_log2f((counts[c][v] as f32 + prior) / denom);
}
}
}
#[inline]
fn qf_symbol(left: i32, qf: i32) -> (usize, usize) {
(
ac_metadata_context(left, 0) as usize,
(pack_signed(qf - left) as usize).min(511),
)
}
fn qf_trellis(
first_left: i32,
fixed: &[u8],
states: &[Option<&QuantRefinement>],
lambda_bits: f32,
prices: &[[f32; 512]; 4],
chain: &mut [u8],
) {
let n = fixed.len();
let mut back = vec![[0u8; 4]; n];
let mut prev_q = [0u8; 4];
let mut prev_cost = [0.0f32; 4];
let mut prev_len = 0usize;
for i in 0..n {
let (cur, len): ([(u8, f32); 4], usize) = match states[i] {
Some(r) => (r.states, r.len),
None => ([(fixed[i], 0.0); 4], 1),
};
let mut cost = [f32::INFINITY; 4];
for s in 0..len {
let qf = cur[s].0 as i32 - 1;
if i == 0 {
let (c, v) = qf_symbol(first_left, qf);
cost[s] = cur[s].1 + lambda_bits * prices[c][v];
continue;
}
for p in 0..prev_len {
let (c, v) = qf_symbol(prev_q[p] as i32 - 1, qf);
let total = prev_cost[p] + cur[s].1 + lambda_bits * prices[c][v];
if total < cost[s] {
cost[s] = total;
back[i][s] = p as u8;
}
}
}
let floor = cost[..len].iter().copied().fold(f32::INFINITY, f32::min);
for (prev, (current, cost)) in prev_q
.iter_mut()
.zip(cur.iter().zip(cost.iter_mut()))
.take(len)
{
*prev = current.0;
*cost -= floor;
}
prev_cost = cost;
prev_len = len;
}
if n == 0 {
return;
}
let mut s = (0..prev_len)
.min_by(|&a, &b| prev_cost[a].total_cmp(&prev_cost[b]))
.unwrap_or(0);
for i in (0..n).rev() {
chain[i] = match states[i] {
Some(r) => r.states[s].0,
None => fixed[i],
};
s = back[i][s] as usize;
}
}
fn refine_quant_field(
context: &QuantRefinementContext<'_>,
scratch: &mut CoderScratch,
quant_field: &mut ImageB,
band_scratch: &mut [AcStrategyBandScratch],
) {
if quant_refinement_steps(context.params.distance) == 0 {
return;
}
{
let search = QuantRefinementSearch {
context,
quant_field,
};
context
.params
.ctx
.thread_pool
.steal_for_each_mut_with_threads(
scratch,
&mut band_scratch[..context.bands.len()],
context.num_threads,
|i, band_scratch, scratch| {
let mut scratch = context.params.price(scratch);
band_scratch.quant_refinements.clear();
find_quant_refinements(
&search,
&mut scratch,
context.bands[i],
&mut band_scratch.quant_refinements,
);
},
);
}
let ac_strategy = context.ac_strategy;
let xsize = ac_strategy.xsize();
let mut slot = vec![usize::MAX; xsize * ac_strategy.ysize()];
let refinements: Vec<&QuantRefinement> = band_scratch[..context.bands.len()]
.iter()
.flat_map(|band| band.quant_refinements.iter())
.collect();
for (i, r) in refinements.iter().enumerate() {
slot[r.by * xsize + r.bx] = i;
}
let first_blocks: Vec<(usize, usize, u8)> = ac_strategy.iter_first_blocks().collect();
let fixed: Vec<u8> = first_blocks
.iter()
.map(|&(bx, by, _)| quant_field.row(by)[bx])
.collect();
let states: Vec<Option<&QuantRefinement>> = first_blocks
.iter()
.map(|&(bx, by, _)| refinements.get(slot[by * xsize + bx]).copied())
.collect();
let mut chain: Vec<u8> = states
.iter()
.zip(&fixed)
.map(|(s, &q)| s.map_or(q, independent_choice))
.collect();
if !first_blocks.is_empty() {
let first_left = ac_strategy.strategy_code(0, 0) as i32;
let lambda_bits = rd_cost(
DistortionModel::Reconstruction,
context.params.distance,
0.0,
0.0,
1.0,
);
let mut prices = [[0.0f32; 512]; 4];
for _ in 0..QF_TRELLIS_ROUNDS {
qf_residual_prices(first_left, &chain, &mut prices);
qf_trellis(
first_left,
&fixed,
&states,
lambda_bits,
&prices,
&mut chain,
);
}
}
for ((&(bx, by, strategy), &q), &old) in first_blocks.iter().zip(&chain).zip(&fixed) {
if q == old {
continue;
}
let cov_x = AcStrategyImage::covered_blocks_x_of(strategy);
let cov_y = AcStrategyImage::covered_blocks_y_of(strategy);
for iy in 0..cov_y {
quant_field.row_mut(by + iy)[bx..bx + cov_x].fill(q);
}
}
}
#[allow(clippy::too_many_arguments)]
#[inline]
fn region_qac(
quant_field: &ImageB,
bx: usize,
by: usize,
w: usize,
h: usize,
scale: f32,
butteraugli_target: f32,
) -> f32 {
scale * region_quant(quant_field, bx, by, w, h, butteraugli_target) as f32
}
pub(super) fn region_quant(
quant_field: &ImageB,
bx: usize,
by: usize,
w: usize,
h: usize,
butteraugli_target: f32,
) -> u8 {
let mut max_q = 1u8;
let mut sum = 0u32;
for y in by..by + h {
for &q in &quant_field.row(y)[bx..bx + w] {
max_q = max_q.max(q);
sum += u32::from(q);
}
}
aggregate_quant(max_q, sum, w * h, butteraugli_target)
}
#[inline]
fn block_qac_2x2(quant_field: &ImageB, bx: usize, by: usize, scale: f32) -> [[f32; 2]; 2] {
[
[
scale * quant_field.row(by)[bx] as f32,
scale * quant_field.row(by)[bx + 1] as f32,
],
[
scale * quant_field.row(by + 1)[bx] as f32,
scale * quant_field.row(by + 1)[bx + 1] as f32,
],
]
}
struct SelectionContext<'a> {
params: AcStrategyParams<'a>,
quant_field: &'a ImageB,
meta_r: f32,
scope: SearchScope,
}
#[derive(Clone, Copy)]
pub(crate) struct Chosen32Cost {
pub(crate) bx: u16,
pub(crate) by: u16,
pub(crate) cost: f32,
}
#[derive(Clone, Copy)]
pub(crate) struct SavedChild {
pub(crate) bx: u16,
pub(crate) by: u16,
pub(crate) grid: [u8; 16],
}
pub(crate) const NO_CHILD_BLOCK: u8 = 0xFF;
fn capture_children(
map: &AcStrategyImage,
saved: &mut Vec<SavedChild>,
bx: usize,
by: usize,
cov_x: usize,
cov_y: usize,
) {
let mut grid = [NO_CHILD_BLOCK; 16];
let mut nontrivial = false;
for iy in 0..cov_y {
for ix in 0..cov_x {
if map.is_first_block(bx + ix, by + iy) {
let s = map.raw_strategy(bx + ix, by + iy);
grid[iy * 4 + ix] = s;
nontrivial |= s != STRATEGY_DCT;
}
}
}
if nontrivial {
saved.push(SavedChild {
bx: bx as u16,
by: by as u16,
grid,
});
}
}
fn select_band(
selection: &SelectionContext<'_>,
scratch: &mut CoderScratch,
ac_strategy: &mut AcStrategyImage,
band: (usize, usize),
chosen32: &mut Vec<Chosen32Cost>,
saved: &mut Vec<SavedChild>,
upgrades: &mut Vec<MergeUpgradeCandidate>,
) -> f32 {
let params = selection.params;
let ctx = params.ctx;
let opsin = params.opsin;
let distance = params.distance;
let scale = params.scale;
let qm_mult_x = params.qm_mult_x;
let quant_field = selection.quant_field;
let ytox_map = params.ytox_map;
let ytob_map = params.ytob_map;
let meta_r = selection.meta_r;
let scope = selection.scope;
let (y_begin, y_end) = band;
let xsize = ac_strategy.xsize();
let ysize = ac_strategy.ysize();
let super_block = SuperBlockContext {
params,
meta_r,
scope,
dct8_cost_y0: y_begin,
dct8_cost_stride: xsize,
};
let merge = ctx.merge;
let costs_size = xsize * (y_end - y_begin);
if scratch.dct8_costs.len() != xsize * (y_end - y_begin) {
scratch.dct8_costs.clear();
scratch.dct8_costs.resize(costs_size, f32::NAN);
} else {
scratch.dct8_costs.fill(f32::NAN);
}
let mut by = y_begin;
while by + 1 < ysize && by < y_end {
let four_row = scope.merge_32() && by.is_multiple_of(4) && by + 4 <= ysize;
let mut bx = 0;
while bx + 1 < xsize {
let four_col = bx % 4 == 0 && bx + 4 <= xsize;
if four_row && four_col && ac_strategy.can_place_strategy(bx, by, STRATEGY_DCT32X32) {
let mut children = [[0.0f32; 2]; 2];
let mut baseline = [[0.0f32; 2]; 2];
let mut prescreened = false;
for sy in 0..2 {
for sx in 0..2 {
let sbx = bx + sx * 2;
let sby = by + sy * 2;
let qac = block_qac_2x2(quant_field, sbx, sby, scale);
let costs = select_super_block(
&super_block,
scratch,
ac_strategy,
SuperBlockInput {
bx: sbx,
by: sby,
qac,
},
saved,
upgrades,
);
children[sy][sx] = costs.chosen;
baseline[sy][sx] = costs.dct8;
prescreened |= costs.prescreened;
}
}
if prescreened {
bx += 4;
continue;
}
let qac32 = region_qac(quant_field, bx, by, 4, 4, scale, distance);
let cmap_factor = cmap_factors(ctx.cfl_frame(), ytox_map, ytob_map, bx, by);
let cost32 = strategy_cost(
ctx,
scratch,
STRATEGY_DCT32X32,
opsin,
params.dc_group_px + bx * 8,
params.dc_group_py + by * 8,
qac32,
qm_mult_x,
meta_r,
distance,
cmap_factor,
);
let mut rect32 =
|bx: usize, by: usize, strategy: u8, cw: usize, ch: usize| -> f32 {
if !scope.rectangles() {
return f32::INFINITY;
}
strategy_cost(
ctx,
scratch,
strategy,
opsin,
params.dc_group_px + bx * 8,
params.dc_group_py + by * 8,
region_qac(quant_field, bx, by, cw, ch, scale, distance),
qm_mult_x,
meta_r,
distance,
cmap_factor,
)
};
let cl = rect32(bx, by, STRATEGY_DCT32X16, 2, 4);
let cr = rect32(bx + 2, by, STRATEGY_DCT32X16, 2, 4);
let ct = rect32(bx, by, STRATEGY_DCT16X32, 4, 2);
let cb = rect32(bx, by + 2, STRATEGY_DCT16X32, 4, 2);
let raw = [cost32, cl, cr, ct, cb];
if CandidateSearch::for_speed(ctx.speed, ctx.arbiter).merge32_upgrades
&& distance >= MERGE_UPGRADE_MIN_DISTANCE
{
record_upgrade32_candidates(upgrades, ac_strategy, bx, by, raw, children);
}
let sites = merge32_sites(bx, by);
let mut costs = raw.map(|c| merge.bias_rect32 * c);
costs[0] = merge.bias_32x32 * cost32;
for (k, &(x, y, strategy)) in sites.iter().enumerate() {
if !ac_strategy.can_place_strategy(x, y, strategy) {
costs[k] = f32::INFINITY;
}
}
let (mut q_min, mut q_max) = (u8::MAX, 0u8);
for iy in 0..4 {
for &q in &quant_field.row(by + iy)[bx..bx + 4] {
q_min = q_min.min(q);
q_max = q_max.max(q);
}
}
let gate =
|accept: f32| risk_gated(merge.risk_k, accept, q_min as f32, q_max as f32, 2.0);
let mask = choose_merge_partition(
children,
baseline,
costs,
[
gate(merge.accept_32),
gate(merge.accept_32_rect),
gate(merge.accept_32_rect),
gate(merge.accept_32_rect),
gate(merge.accept_32_rect),
],
);
for (k, &(x, y, strategy)) in sites.iter().enumerate() {
if mask & (1 << k) == 0 {
continue;
}
let cx = AcStrategyImage::covered_blocks_x_of(strategy);
let cy = AcStrategyImage::covered_blocks_y_of(strategy);
capture_children(ac_strategy, saved, x, y, cx, cy);
ac_strategy.set_first(x, y, strategy);
}
chosen32.push(Chosen32Cost {
bx: bx as u16,
by: by as u16,
cost: partition_cost(children, costs, mask),
});
bx += 4;
} else if four_row {
for sby in [by, by + 2] {
let qac = block_qac_2x2(quant_field, bx, sby, scale);
let _ = select_super_block(
&super_block,
scratch,
ac_strategy,
SuperBlockInput { bx, by: sby, qac },
saved,
upgrades,
);
}
bx += 2;
} else {
let qac = block_qac_2x2(quant_field, bx, by, scale);
let _ = select_super_block(
&super_block,
scratch,
ac_strategy,
SuperBlockInput { bx, by, qac },
saved,
upgrades,
);
bx += 2;
}
}
by += if four_row { 4 } else { 2 };
}
let mut benefit = 0.0f32;
if !scope.rectangles() {
return benefit;
}
let with_dct4 = distance <= SUB8_MAX_DISTANCE;
let with_fine = distance <= FINE_TRANSFORM_MAX_DISTANCE;
if !with_dct4 && distance > AFV_MAX_DISTANCE && !with_fine {
return benefit;
}
let bias_afv = BIAS_AFV.at(distance);
for by in y_begin..y_end {
for bx in 0..xsize {
if ac_strategy.raw_strategy(bx, by) != STRATEGY_DCT {
continue;
}
let cached_dct8 = scratch.dct8_costs[(by - y_begin) * xsize + bx];
if let Some((cand, gain, _)) = evaluate_sub8_candidate(
params,
scratch,
bx,
by,
region_qac(quant_field, bx, by, 1, 1, scale, distance),
meta_r,
cached_dct8.is_finite().then_some(cached_dct8),
with_dct4,
with_fine,
bias_afv,
) {
ac_strategy.set_first(bx, by, cand);
benefit += gain;
}
}
}
benefit
}
#[derive(Clone, Copy)]
struct Sub8Pick {
strategy: u8,
biased_j: f32,
gain: f32,
fallback: u8,
}
#[allow(clippy::too_many_arguments)]
fn evaluate_sub8_candidate(
params: AcStrategyParams<'_>,
scratch: &mut CoderScratch,
bx: usize,
by: usize,
qac: f32,
meta_r: f32,
cached_dct8: Option<f32>,
with_dct4: bool,
with_fine: bool,
bias_afv: f32,
) -> Option<(u8, f32, u8)> {
evaluate_sub8(
params,
scratch,
bx,
by,
qac,
meta_r,
cached_dct8,
with_dct4,
with_fine,
bias_afv,
)
.map(|pick| (pick.strategy, pick.gain, pick.fallback))
}
#[allow(clippy::too_many_arguments)]
fn evaluate_sub8(
params: AcStrategyParams<'_>,
scratch: &mut CoderScratch,
bx: usize,
by: usize,
qac: f32,
meta_r: f32,
cached_dct8: Option<f32>,
with_dct4: bool,
with_fine: bool,
bias_afv: f32,
) -> Option<Sub8Pick> {
let shortlist = sub8_shortlist(
params,
scratch,
bx,
by,
qac,
meta_r,
cached_dct8,
with_dct4,
with_fine,
bias_afv,
);
if let Some(fine) = fine_recon_choose(
params,
scratch,
bx,
by,
qac,
meta_r,
[shortlist.fine, shortlist.fine_alt],
shortlist.structural_cost,
) {
let fallback = shortlist.structural.map_or(STRATEGY_DCT, |p| p.strategy);
return Some(Sub8Pick { fallback, ..fine });
}
shortlist.structural
}
struct Sub8Shortlist {
structural: Option<Sub8Pick>,
structural_cost: f32,
fine: Option<Sub8Pick>,
fine_alt: Option<Sub8Pick>,
}
#[allow(clippy::too_many_arguments)]
fn sub8_shortlist(
params: AcStrategyParams<'_>,
scratch: &mut CoderScratch,
bx: usize,
by: usize,
qac: f32,
meta_r: f32,
cached_dct8: Option<f32>,
with_dct4: bool,
with_fine: bool,
bias_afv: f32,
) -> Sub8Shortlist {
let ctx = params.ctx;
let px = params.dc_group_px + bx * 8;
let py = params.dc_group_py + by * 8;
let cmap_factor = cmap_factors(
params.ctx.cfl_frame(),
params.ytox_map,
params.ytob_map,
bx,
by,
);
let costs = sub8_strategy_costs(
ctx,
scratch,
params.opsin,
px,
py,
qac,
params.qm_mult_x,
meta_r,
params.distance,
cmap_factor,
cached_dct8,
with_dct4,
with_fine,
);
let cost8 = costs.dct8;
let cost_identity = fine_transform_bias(1.042_754_3, params.distance) * costs.identity;
let cost_dct2x2 = fine_transform_bias(0.95, params.distance) * costs.dct2x2;
let cost4 = BIAS_4X4 * costs.dct4x4;
let cost48 = BIAS_4X8 * costs.dct4x8;
let cost84 = BIAS_4X8 * costs.dct8x4;
let (cand, cand_cost) = {
let mut best = STRATEGY_DCT4X4;
let mut bc = cost4;
if cost48 < bc {
best = STRATEGY_DCT4X8;
bc = cost48;
}
if cost84 < bc {
best = STRATEGY_DCT8X4;
bc = cost84;
}
for (kind, &afv_cost) in costs.afv.iter().enumerate() {
let biased = bias_afv * afv_cost;
if biased < bc {
best = STRATEGY_AFV0 + kind as u8;
bc = biased;
}
}
(best, bc)
};
let (fine, fine_coeff_cost, runner, runner_cost) = if cost_identity < cost_dct2x2 {
(
STRATEGY_IDENTITY,
cost_identity,
STRATEGY_DCT2X2,
cost_dct2x2,
)
} else {
(
STRATEGY_DCT2X2,
cost_dct2x2,
STRATEGY_IDENTITY,
cost_identity,
)
};
Sub8Shortlist {
structural: (cand_cost < cost8).then_some(Sub8Pick {
strategy: cand,
biased_j: cand_cost,
gain: cost8 - cand_cost,
fallback: STRATEGY_DCT,
}),
structural_cost: cand_cost,
fine: (fine_coeff_cost < cost8).then_some(Sub8Pick {
strategy: fine,
biased_j: fine_coeff_cost,
gain: 0.0,
fallback: STRATEGY_DCT,
}),
fine_alt: (CandidateSearch::for_speed(ctx.speed, ctx.arbiter).fine_runner_up
&& runner_cost < cost8
&& runner_cost <= fine_coeff_cost * FINE_RUNNER_UP_RATIO)
.then_some(Sub8Pick {
strategy: runner,
biased_j: runner_cost,
gain: 0.0,
fallback: STRATEGY_DCT,
}),
}
}
#[inline]
fn learned_fine_rate_correction(rate: f32) -> f32 {
(FINE_ADMIT_RATE_CORRECTION_SHARE * rate).min(FINE_ADMIT_RATE_CORRECTION_BITS)
}
#[inline]
fn fine_admit_charge(learned: bool, distance: f32, rate: f32) -> (f32, f32) {
if learned {
(
reconstruction_lambda(distance),
learned_fine_rate_correction(rate),
)
} else {
(
fine_mosaic_lambda(distance),
FINE_ADMIT_RATE_CORRECTION_BITS,
)
}
}
fn fine_recon_choose(
params: AcStrategyParams<'_>,
scratch: &mut CoderScratch,
bx: usize,
by: usize,
qac: f32,
meta_r: f32,
candidates: [Option<Sub8Pick>; 2],
limit: f32,
) -> Option<Sub8Pick> {
let candidates = candidates.map(|p| p.filter(|p| p.biased_j < limit));
if candidates.iter().all(Option::is_none) {
return None;
}
let ctx = params.ctx;
let px = params.dc_group_px + bx * 8;
let py = params.dc_group_py + by * 8;
let cmap_factor = cmap_factors(
params.ctx.cfl_frame(),
params.ytox_map,
params.ytob_map,
bx,
by,
);
let reconstruction_cost = |scratch: &mut CoderScratch, strategy| {
reconstruction_strategy_cost_and_base(
ctx,
scratch,
strategy,
params.opsin,
px,
py,
qac,
params.qm_mult_x,
meta_r,
params.distance,
cmap_factor,
0.0,
0.0,
None,
)
};
let recon8 = reconstruction_cost(scratch, STRATEGY_DCT).cost;
let mut best: Option<Sub8Pick> = None;
for candidate in candidates.into_iter().flatten() {
let score = reconstruction_cost(scratch, candidate.strategy);
let (lambda, correction) =
fine_admit_charge(scratch.rate_prices.is_some(), params.distance, score.rate);
let cost = fmla(lambda, correction, score.cost);
if cost < recon8 * FINE_RECON_MARGIN {
let gain = recon8 - cost;
if best.is_none_or(|p| gain > p.gain) {
best = Some(Sub8Pick { gain, ..candidate });
}
}
}
best
}
fn fill_selection_bands(bands: &mut Vec<(usize, usize)>, ysize: usize, n: usize) {
bands.clear();
if n <= 1 || ysize < 8 {
bands.push((0, ysize));
return;
}
let mut previous = 0;
for k in 1..n {
let b = (ysize * k / n) / 4 * 4;
if b > previous && b < ysize {
bands.push((previous, b));
previous = b;
}
}
bands.push((previous, ysize));
}
const PILOT_REGION: usize = 8;
const PILOT_REGIONS: usize = 96;
const PILOT_MIN_REGIONS: usize = 4;
#[derive(Clone, Copy, PartialEq, Eq)]
enum SelectionPass {
Full,
Pilot,
}
#[allow(clippy::too_many_arguments)]
fn learn_rate_prices(
ctx: &EncodingContext,
scratch: &mut CoderScratch,
opsin: &Image3F,
dc_group_px: usize,
dc_group_py: usize,
distance: f32,
scale: f32,
x_qm_scale: u32,
quant_field: &ImageB,
ytox_map: &ImageSB,
ytob_map: &ImageSB,
num_threads: usize,
) -> std::sync::Arc<RatePrices> {
let qm_mult_x = 1.25f32.powf(x_qm_scale as f32 - 2.0);
let blocks_x = quant_field.xsize().min((opsin.xsize() - dc_group_px) / 8);
let blocks_y = quant_field.ysize().min((opsin.ysize() - dc_group_py) / 8);
let (regions_x, regions_y) = (blocks_x / PILOT_REGION, blocks_y / PILOT_REGION);
let total = regions_x * regions_y;
let (mut rollbacks, mut fallbacks) = (Vec::new(), Vec::new());
if total < PILOT_MIN_REGIONS {
let mut quant = quant_field.clone();
let mut layout = AcStrategyImage::new(quant_field.xsize(), quant_field.ysize());
select_transforms(
ctx,
scratch,
opsin,
dc_group_px,
dc_group_py,
distance,
scale,
x_qm_scale,
&mut quant,
ytox_map,
ytob_map,
&mut layout,
&mut rollbacks,
&mut fallbacks,
&mut Vec::new(),
num_threads,
None,
SelectionPass::Pilot,
);
return RatePrices::learn(
ctx,
scratch,
opsin,
dc_group_px,
dc_group_py,
quant_field,
ytox_map,
ytob_map,
scale,
qm_mult_x,
distance,
Some(&layout),
num_threads,
);
}
let count = total.min(PILOT_REGIONS);
let columns = (count as f32).sqrt().ceil() as usize;
let rows = count.div_ceil(columns);
let side = PILOT_REGION * 8;
let mut mosaic = Image3F::new(columns * side, rows * side);
let mut quant = ImageB::new(columns * PILOT_REGION, rows * PILOT_REGION);
let mut ytox = ImageSB::new(columns, rows);
let mut ytob = ImageSB::new(columns, rows);
for cell in 0..columns * rows {
let index = cell.min(count - 1);
let start = index * total / count;
let end = (index + 1) * total / count;
let jitter = ((index as u32).wrapping_mul(0x9e37_79b9) >> 16) as usize;
let region = start + jitter % (end - start).max(1);
let (from_x, from_y) = (region % regions_x, region / regions_x);
let (to_x, to_y) = (cell % columns, cell / columns);
for y in 0..side {
for c in 0..3 {
let from = &opsin.plane_row(c, dc_group_py + from_y * side + y)
[dc_group_px + from_x * side..][..side];
mosaic.plane_row_mut(c, to_y * side + y)[to_x * side..][..side]
.copy_from_slice(from);
}
}
for y in 0..PILOT_REGION {
let from = &quant_field.row(from_y * PILOT_REGION + y)[from_x * PILOT_REGION..]
[..PILOT_REGION];
quant.row_mut(to_y * PILOT_REGION + y)[to_x * PILOT_REGION..][..PILOT_REGION]
.copy_from_slice(from);
}
let tile =
|map: &ImageSB| map.row(from_y.min(map.ysize() - 1))[from_x.min(map.xsize() - 1)];
ytox.row_mut(to_y)[to_x] = tile(ytox_map);
ytob.row_mut(to_y)[to_x] = tile(ytob_map);
}
let sampled = quant.clone();
let mut layout = AcStrategyImage::new(quant.xsize(), quant.ysize());
select_transforms(
ctx,
scratch,
&mosaic,
0,
0,
distance,
scale,
x_qm_scale,
&mut quant,
&ytox,
&ytob,
&mut layout,
&mut rollbacks,
&mut fallbacks,
&mut Vec::new(),
num_threads,
None,
SelectionPass::Pilot,
);
RatePrices::learn(
ctx,
scratch,
&mosaic,
0,
0,
&sampled,
&ytox,
&ytob,
scale,
qm_mult_x,
distance,
Some(&layout),
num_threads,
)
}
pub(crate) fn fill_ac_strategy(
ctx: &EncodingContext,
scratch: &mut CoderScratch,
opsin: &Image3F,
dc_group_px: usize,
dc_group_py: usize,
distance: f32,
scale: f32,
x_qm_scale: u32,
quant_field: &mut ImageB,
ytox_map: &ImageSB,
ytob_map: &ImageSB,
ac_strategy: &mut AcStrategyImage,
fine_rollbacks: &mut Vec<FineMergeRollback>,
fine_fallbacks: &mut Vec<u8>,
dp_regions: &mut Vec<crate::dc_group_data::DpRegion>,
num_threads: usize,
) -> f32 {
let learned = ctx.learned_rate
&& ctx.speed.effort().learned_rate
&& !use_dct8_only_for_content(ctx, distance);
let prices = learned.then(|| {
learn_rate_prices(
ctx,
scratch,
opsin,
dc_group_px,
dc_group_py,
distance,
scale,
x_qm_scale,
quant_field,
ytox_map,
ytob_map,
num_threads,
)
});
select_transforms(
ctx,
scratch,
opsin,
dc_group_px,
dc_group_py,
distance,
scale,
x_qm_scale,
quant_field,
ytox_map,
ytob_map,
ac_strategy,
fine_rollbacks,
fine_fallbacks,
dp_regions,
num_threads,
prices,
SelectionPass::Full,
)
}
const BIAS_8X32: f32 = 1.1;
fn place_dct8x32(
params: &AcStrategyParams<'_>,
scratch: &mut CoderScratch,
quant_field: &ImageB,
ac_strategy: &mut AcStrategyImage,
rollbacks: &[(usize, usize, usize, usize)],
) -> Vec<(usize, usize, usize, usize)> {
let p = *params;
let cost = |scratch: &mut CoderScratch, s: u8, bx: usize, by: usize, w: usize, h: usize| {
let cmap = cmap_factors(p.ctx.cfl_frame(), p.ytox_map, p.ytob_map, bx, by);
strategy_cost(
p.ctx,
scratch,
s,
p.opsin,
p.dc_group_px + bx * 8,
p.dc_group_py + by * 8,
region_qac(quant_field, bx, by, w, h, p.scale, p.distance),
p.qm_mult_x,
META_R,
p.distance,
cmap,
)
};
let strip = |scratch: &mut CoderScratch,
ac: &AcStrategyImage,
bx: usize,
by: usize,
w: usize,
h: usize|
-> Option<f32> {
if rollbacks
.iter()
.any(|&(rx, ry, rw, rh)| bx < rx + rw && rx < bx + w && by < ry + rh && ry < by + h)
{
return None;
}
let mut covered = 0;
let mut total = 0.0f32;
for y in by..by + h {
for x in bx..bx + w {
let s = ac.raw_strategy(x, y);
if !matches!(s, STRATEGY_DCT | STRATEGY_DCT16X8 | STRATEGY_DCT8X16) {
return None;
}
if !ac.is_first_block(x, y) {
continue;
}
let cw = AcStrategyImage::covered_blocks_x_of(s);
let ch = AcStrategyImage::covered_blocks_y_of(s);
if x + cw > bx + w || y + ch > by + h {
return None;
}
covered += cw * ch;
total += cost(scratch, s, x, y, cw, ch);
}
}
(covered == w * h).then_some(total)
};
let mut chosen: Vec<(usize, usize, u8)> = Vec::new();
let mut gain_bits = 0.0f32;
let (xsize, ysize) = (ac_strategy.xsize(), ac_strategy.ysize());
for by in (0..ysize.saturating_sub(3)).step_by(4) {
for bx in (0..xsize.saturating_sub(3)).step_by(4) {
let mut tall = [0.0f32; 4];
let mut wide = [0.0f32; 4];
for i in 0..4 {
if ac_strategy.can_place_strategy(bx + i, by, STRATEGY_DCT32X8)
&& let Some(inc) = strip(scratch, ac_strategy, bx + i, by, 1, 4)
{
let cand = BIAS_8X32 * cost(scratch, STRATEGY_DCT32X8, bx + i, by, 1, 4);
tall[i] = (inc - cand).max(0.0);
}
if ac_strategy.can_place_strategy(bx, by + i, STRATEGY_DCT8X32)
&& let Some(inc) = strip(scratch, ac_strategy, bx, by + i, 4, 1)
{
let cand = BIAS_8X32 * cost(scratch, STRATEGY_DCT8X32, bx, by + i, 4, 1);
wide[i] = (inc - cand).max(0.0);
}
}
let (gt, gw): (f32, f32) = (tall.iter().sum(), wide.iter().sum());
if gt <= 0.0 && gw <= 0.0 {
continue;
}
for i in 0..4 {
if gt >= gw && tall[i] > 0.0 {
chosen.push((bx + i, by, STRATEGY_DCT32X8));
} else if gw > gt && wide[i] > 0.0 {
chosen.push((bx, by + i, STRATEGY_DCT8X32));
}
}
gain_bits += gt.max(gw) / RD_LAMBDA;
}
}
if p.opsin.xsize() <= crate::frame::K_DC_GROUP_DIM
&& p.opsin.ysize() <= crate::frame::K_DC_GROUP_DIM
{
let header = crate::quant_weights::MatrixHeaderCost::new(p.ctx.matrices());
let others = header.custom_mask() & !(1 << 12);
let fee = header.payload[12] as f32 + if others == 0 { 17.0 * 3.0 } else { 0.0 };
if gain_bits < fee {
return Vec::new();
}
}
let mut changed = Vec::new();
for (x, y, st) in chosen {
ac_strategy.set_first(x, y, st);
let (w, h) = if st == STRATEGY_DCT32X8 {
(1, 4)
} else {
(4, 1)
};
changed.push((x, y, w, h));
}
changed
}
#[allow(clippy::too_many_arguments)]
fn select_transforms(
ctx: &EncodingContext,
scratch: &mut CoderScratch,
opsin: &Image3F,
dc_group_px: usize,
dc_group_py: usize,
distance: f32,
scale: f32,
x_qm_scale: u32,
quant_field: &mut ImageB,
ytox_map: &ImageSB,
ytob_map: &ImageSB,
ac_strategy: &mut AcStrategyImage,
fine_rollbacks: &mut Vec<FineMergeRollback>,
fine_fallbacks: &mut Vec<u8>,
dp_regions: &mut Vec<crate::dc_group_data::DpRegion>,
num_threads: usize,
prices: Option<std::sync::Arc<RatePrices>>,
pass: SelectionPass,
) -> f32 {
fine_rollbacks.clear();
fine_fallbacks.clear();
let speed = ctx.speed;
let xsize = ac_strategy.xsize();
let ysize = ac_strategy.ysize();
if use_dct8_only_for_content(ctx, distance) || !speed.effort().square_merges {
return 0.0;
}
let scope = SearchScope::for_speed(speed);
let qm_mult_x = 1.25f32.powf(x_qm_scale as f32 - 2.0);
let mut scratch = RatePricesScope::new(scratch, prices.clone());
let scratch: &mut CoderScratch = &mut scratch;
let meta_r = META_R;
let params = AcStrategyParams {
ctx,
opsin,
dc_group_px,
dc_group_py,
distance,
scale,
qm_mult_x,
ytox_map,
ytob_map,
prices: prices.as_ref(),
};
let mut pipeline = std::mem::take(&mut scratch.ac_strategy);
fill_selection_bands(&mut pipeline.bands, ysize, num_threads);
let parallel_selection = pipeline.bands.len() > 1;
pipeline.prepare_bands(xsize, ysize, parallel_selection);
let selection = SelectionContext {
params,
quant_field,
meta_r,
scope,
};
let leaf_first_max = if prices.is_some() {
LEARNED_LEAF_FIRST_MAX_DISTANCE
} else {
LEAF_FIRST_MAX_DISTANCE
};
let leaf_first = distance <= leaf_first_max;
let run_band = |scratch: &mut CoderScratch,
output: &mut AcStrategyBandScratch,
strategy: &mut AcStrategyImage,
band: (usize, usize)|
-> f32 {
if leaf_first {
leaf_first::select_band_leaf_first(&selection, scratch, strategy, band, output);
0.0
} else {
select_band(
&selection,
scratch,
strategy,
band,
&mut output.chosen32,
&mut output.saved_children,
&mut output.upgrade_candidates,
)
}
};
let mut benefit = if !parallel_selection {
let band0 = &mut pipeline.band_scratch[0];
band0.chosen32.clear();
band0.saved_children.clear();
band0.upgrade_candidates.clear();
run_band(scratch, band0, ac_strategy, (0, ysize))
} else {
let bands = &pipeline.bands;
let band_scratch = &mut pipeline.band_scratch[..bands.len()];
ctx.thread_pool.steal_for_each_mut_with_threads(
scratch,
band_scratch,
num_threads,
|i, output, scratch| {
let mut scratch = RatePricesScope::new(scratch, prices.clone());
let scratch: &mut CoderScratch = &mut scratch;
let (y0, y1) = bands[i];
let mut strategy =
std::mem::replace(&mut output.strategy, AcStrategyImage::new(0, 0));
output.benefit = run_band(scratch, output, &mut strategy, (y0, y1));
output.strategy = strategy;
},
);
let mut benefit = 0.0f32;
for (&(y0, y1), output) in bands.iter().zip(band_scratch) {
ac_strategy.copy_rows_from(&output.strategy, y0, y1);
benefit += output.benefit;
}
benefit
};
if leaf_first {
pipeline.leaf_gain.clear();
pipeline.leaf_gain.resize(xsize * ysize, 0.0);
pipeline.rerank_leaves.clear();
pipeline.rerank_leaves.resize(xsize * ysize, STRATEGY_DCT);
fine_fallbacks.resize(xsize * ysize, STRATEGY_DCT);
for (&(y0, y1), band) in pipeline.bands.iter().zip(&pipeline.band_scratch) {
for (((leaf, gain), fallback), strategy) in band.leaves[..xsize * (y1 - y0)]
.iter()
.zip(&mut pipeline.leaf_gain[y0 * xsize..y1 * xsize])
.zip(&mut fine_fallbacks[y0 * xsize..y1 * xsize])
.zip(&mut pipeline.rerank_leaves[y0 * xsize..y1 * xsize])
{
*gain = leaf.gain;
*fallback = leaf.fallback;
*strategy = leaf.strategy;
}
}
}
let qx = xsize.div_ceil(4);
let qy = ysize.div_ceil(4);
pipeline.chosen32_grid.clear();
pipeline.chosen32_grid.resize(qx * qy, f32::NAN);
for band in &pipeline.band_scratch[..pipeline.bands.len().max(1)] {
for c in &band.chosen32 {
pipeline.chosen32_grid[(c.by as usize / 4) * qx + c.bx as usize / 4] = c.cost;
}
}
if use_dct64(speed, distance) {
let qx = xsize.div_ceil(4);
for by in (0..ysize.saturating_sub(7)).step_by(8) {
for bx in (0..xsize.saturating_sub(7)).step_by(8) {
if !ac_strategy.can_place_strategy(bx, by, STRATEGY_DCT64X64) {
continue;
}
let quad = |sx: usize, sy: usize| -> f32 {
pipeline.chosen32_grid[((by + sy) / 4) * qx + (bx + sx) / 4]
};
let cost32 = [[quad(0, 0), quad(4, 0)], [quad(0, 4), quad(4, 4)]];
if cost32.iter().flatten().any(|c| !c.is_finite()) {
continue;
}
let cmap = cmap_factors(ctx.cfl_frame(), ytox_map, ytob_map, bx, by);
let cost64 = BIAS_64X64
* strategy_cost64(
ctx,
scratch,
STRATEGY_DCT64X64,
opsin,
dc_group_px + bx * 8,
dc_group_py + by * 8,
region_qac(quant_field, bx, by, 8, 8, scale, distance),
qm_mult_x,
meta_r,
distance,
cmap,
);
let tall = [0usize, 4].map(|sx| {
BIAS_64_RECT
* strategy_cost64(
ctx,
scratch,
STRATEGY_DCT64X32,
opsin,
dc_group_px + (bx + sx) * 8,
dc_group_py + by * 8,
region_qac(quant_field, bx + sx, by, 4, 8, scale, distance),
qm_mult_x,
meta_r,
distance,
cmap,
)
});
let wide = [0usize, 4].map(|sy| {
BIAS_64_RECT
* strategy_cost64(
ctx,
scratch,
STRATEGY_DCT32X64,
opsin,
dc_group_px + bx * 8,
dc_group_py + (by + sy) * 8,
region_qac(quant_field, bx, by + sy, 8, 4, scale, distance),
qm_mult_x,
meta_r,
distance,
cmap,
)
});
let accept_rect = ctx.merge.accept_64_rect;
let tall_use = [
merge_beats_dct8(tall[0], cost32[0][0] + cost32[1][0], accept_rect),
merge_beats_dct8(tall[1], cost32[0][1] + cost32[1][1], accept_rect),
];
let wide_use = [
merge_beats_dct8(wide[0], cost32[0][0] + cost32[0][1], accept_rect),
merge_beats_dct8(wide[1], cost32[1][0] + cost32[1][1], accept_rect),
];
let base_cost: f32 = cost32.iter().flatten().sum();
let tall_score = (if tall_use[0] {
tall[0] / accept_rect
} else {
cost32[0][0] + cost32[1][0]
}) + if tall_use[1] {
tall[1] / accept_rect
} else {
cost32[0][1] + cost32[1][1]
};
let wide_score = (if wide_use[0] {
wide[0] / accept_rect
} else {
cost32[0][0] + cost32[0][1]
}) + if wide_use[1] {
wide[1] / accept_rect
} else {
cost32[1][0] + cost32[1][1]
};
let square_score = cost64 / ctx.merge.accept_64;
if square_score < base_cost
&& square_score <= tall_score
&& square_score <= wide_score
{
ac_strategy.set_first(bx, by, STRATEGY_DCT64X64);
} else if tall_score < base_cost && tall_score <= wide_score {
if tall_use[0] {
ac_strategy.set_first(bx, by, STRATEGY_DCT64X32);
}
if tall_use[1] {
ac_strategy.set_first(bx + 4, by, STRATEGY_DCT64X32);
}
} else if wide_score < base_cost {
if wide_use[0] {
ac_strategy.set_first(bx, by, STRATEGY_DCT32X64);
}
if wide_use[1] {
ac_strategy.set_first(bx, by + 4, STRATEGY_DCT32X64);
}
}
}
}
}
if pass == SelectionPass::Full
&& speed.effort().layout_dp_coefficient
&& !speed.effort().layout_retile
&& ctx.coding.is_xyb()
{
let band_count = pipeline.bands.len();
layout_dp::retile_coefficient(
&selection,
scratch,
ac_strategy,
&pipeline.bands,
&mut pipeline.band_scratch[..band_count],
num_threads,
&mut [],
);
}
let reranked = pass == SelectionPass::Full && scope.rerank(distance);
if reranked {
pipeline.prepare_rerank(xsize, ysize);
let rerank = RerankContext {
params,
quant_field,
bands: &pipeline.bands,
num_threads,
leaf_first,
leaves: if leaf_first {
&pipeline.rerank_leaves
} else {
&[]
},
};
let band_count = pipeline.bands.len();
let dp_installs = ctx.speed.effort().layout_retile
&& !ctx.speed.effort().layout_loop()
&& ctx.coding.is_xyb();
if dp_installs {
pipeline.current_costs.fill(f32::NAN);
for band in &mut pipeline.band_scratch[..band_count] {
band.rerank_downgrades.clear();
}
} else {
rerank_large_transforms(
&rerank,
scratch,
ac_strategy,
&mut pipeline.band_scratch[..band_count],
&mut pipeline.current_costs,
);
}
if scope.rectangles() && distance <= AFV_MAX_DISTANCE.max(FINE_TRANSFORM_MAX_DISTANCE) {
let with_dct4 = distance <= SUB8_MAX_DISTANCE;
let with_fine = distance <= FINE_TRANSFORM_MAX_DISTANCE;
let bias_afv = BIAS_AFV.at(distance);
for band_idx in 0..band_count {
for di in 0..pipeline.band_scratch[band_idx].rerank_downgrades.len() {
let downgrade = pipeline.band_scratch[band_idx].rerank_downgrades[di];
if pipeline.band_scratch[band_idx]
.fine_rollbacks
.iter()
.any(|r| r.bx == downgrade.bx && r.by == downgrade.by)
{
continue;
}
for iy in 0..downgrade.cov_y {
for ix in 0..downgrade.cov_x {
if downgrade.restore[iy * 4 + ix] != STRATEGY_DCT {
continue;
}
let (bx, by) = (downgrade.bx + ix, downgrade.by + iy);
if let Some((cand, gain, fallback)) = evaluate_sub8_candidate(
params,
scratch,
bx,
by,
region_qac(quant_field, bx, by, 1, 1, scale, distance),
META_R,
None,
with_dct4,
with_fine,
bias_afv,
) {
ac_strategy.set_first(bx, by, cand);
pipeline.current_costs[by * xsize + bx] = f32::NAN;
benefit += gain;
if leaf_first {
pipeline.leaf_gain[by * xsize + bx] = gain;
fine_fallbacks[by * xsize + bx] = fallback;
}
}
}
}
}
}
}
if distance >= MERGE_UPGRADE_MIN_DISTANCE && !dp_installs {
rerank_upgrade_merges(
&rerank,
scratch,
ac_strategy,
&mut pipeline.band_scratch[..band_count],
&mut pipeline.current_costs,
merge_upgrade_margin(distance),
);
}
}
let medium_upgrades = pass == SelectionPass::Full
&& CandidateSearch::for_speed(speed, ctx.arbiter).medium_upgrades
&& (MERGE_UPGRADE_MIN_DISTANCE..=MERGE_UPGRADE_LOW_DISTANCE).contains(&distance);
if medium_upgrades && !reranked {
pipeline.prepare_rerank(xsize, ysize);
pipeline.current_costs.fill(f32::NAN);
let rerank = RerankContext {
params,
quant_field,
bands: &pipeline.bands,
num_threads,
leaf_first,
leaves: &[],
};
let band_count = pipeline.bands.len();
rerank_upgrade_merges(
&rerank,
scratch,
ac_strategy,
&mut pipeline.band_scratch[..band_count],
&mut pipeline.current_costs,
merge_upgrade_margin(distance),
);
}
if scope.rectangles() && ctx.coding.is_xyb() {
let rollbacks: Vec<(usize, usize, usize, usize)> = pipeline.band_scratch
[..pipeline.bands.len().max(1)]
.iter()
.flat_map(|band| band.fine_rollbacks.iter())
.map(|r| (r.bx, r.by, r.cov_x, r.cov_y))
.collect();
let changed = place_dct8x32(¶ms, scratch, quant_field, ac_strategy, &rollbacks);
if reranked {
for (bx, by, w, h) in changed {
for y in by..by + h {
pipeline.current_costs[y * xsize + bx..y * xsize + bx + w].fill(f32::NAN);
}
}
}
}
if reranked
&& ctx.speed.effort().layout_retile
&& !ctx.speed.effort().layout_loop()
&& ctx.coding.is_xyb()
{
let rerank = RerankContext {
params,
quant_field,
bands: &pipeline.bands,
num_threads,
leaf_first,
leaves: if leaf_first {
&pipeline.rerank_leaves
} else {
&[]
},
};
let band_count = pipeline.bands.len();
layout_dp::retile(
&rerank,
scratch,
ac_strategy,
&mut pipeline.band_scratch[..band_count],
&mut pipeline.current_costs,
true,
);
}
if reranked
&& ctx.speed.effort().layout_retile
&& ctx.speed.effort().layout_loop()
&& ctx.coding.is_xyb()
{
let rerank = RerankContext {
params,
quant_field,
bands: &pipeline.bands,
num_threads,
leaf_first,
leaves: if leaf_first {
&pipeline.rerank_leaves
} else {
&[]
},
};
let band_count = pipeline.bands.len();
layout_dp::retile(
&rerank,
scratch,
ac_strategy,
&mut pipeline.band_scratch[..band_count],
&mut pipeline.current_costs,
false,
);
for band in &pipeline.band_scratch[..band_count] {
dp_regions.extend(band.dp_regions.iter().cloned());
}
}
if leaf_first {
let in_mosaic = |bx: usize, by: usize| {
pipeline.band_scratch[..pipeline.bands.len()]
.iter()
.flat_map(|band| band.fine_rollbacks.iter())
.any(|r| bx >= r.bx && bx < r.bx + r.cov_x && by >= r.by && by < r.by + r.cov_y)
};
benefit = ac_strategy
.iter_first_blocks()
.filter(|&(bx, by, s)| {
crate::dc_group_data::is_gated_sub8_strategy(s) && !in_mosaic(bx, by)
})
.map(|(bx, by, _)| pipeline.leaf_gain[by * xsize + bx])
.fold(0.0, |acc, gain| acc + gain);
}
if reranked {
for band in &pipeline.band_scratch[..pipeline.bands.len()] {
fine_rollbacks.extend_from_slice(&band.fine_rollbacks);
}
}
adjust_quant_field(ac_strategy, distance, quant_field);
let quant_refine = speed.effort().quant_refine && quant_refinement_steps(distance) != 0;
if quant_refine {
pipeline.prepare_refinement(xsize);
}
let current_costs = if reranked {
pipeline.current_costs.as_slice()
} else {
&[]
};
let refinement = QuantRefinementContext {
params,
ac_strategy,
current_costs,
bands: &pipeline.bands,
num_threads,
};
let band_count = pipeline.bands.len();
if quant_refine && pass == SelectionPass::Full {
refine_quant_field(
&refinement,
scratch,
quant_field,
&mut pipeline.band_scratch[..band_count],
);
}
scratch.ac_strategy = pipeline;
benefit
}
struct RerankContext<'a> {
params: AcStrategyParams<'a>,
quant_field: &'a ImageB,
bands: &'a [(usize, usize)],
num_threads: usize,
leaf_first: bool,
leaves: &'a [u8],
}
const FINE_MOSAIC_BOUNDARY_ALPHA: f32 = 32.0;
const FINE_MOSAIC_BOUNDARY_RATIO: f32 = 1.0;
const FINE_MOSAIC_PEAK_RATIO: f32 = 1.0;
const FINE_MOSAIC_MARGIN: f32 = 1.0;
const FINE_MOSAIC_RATE_CORRECTION_BITS: f32 = 8.0;
const FINE_MOSAIC_SEAM_SUBSIDY_BITS: f32 = 8.0;
const FINE_MOSAIC_MAX_DISTANCE: f32 = 3.0;
fn coarse_mosaic_hue_is_costly(hue_distortion: f32, lambda: f32) -> bool {
hue_distortion > lambda * (2.0 * FINE_MOSAIC_RATE_CORRECTION_BITS)
}
fn fine_mosaic_hue_repair_is_safe(
reference_hue: f32,
candidate_hue: f32,
candidate_rd: f32,
incumbent_rd: f32,
seam_regression: f32,
) -> bool {
let hue_gain = reference_hue - candidate_hue;
hue_gain > 0.05 * reference_hue
&& candidate_rd <= 0.95 * incumbent_rd
&& seam_regression <= hue_gain
}
const FINE_MOSAIC_CANDIDATES: [u8; 3] = [STRATEGY_DCT, STRATEGY_IDENTITY, STRATEGY_DCT2X2];
const FINE_MOSAIC_MAX_CHILDREN: usize = 4;
#[cfg(test)]
fn block_boundary_error_energy(
ctx: &EncodingContext,
opsin: &Image3F,
errors: &[[f32; 1024]; 3],
px: usize,
py: usize,
width: usize,
height: usize,
distance: f32,
) -> f32 {
block_boundary_error_stats(ctx, opsin, errors, px, py, width, height, distance).0
}
#[allow(clippy::too_many_arguments)]
fn block_boundary_error_stats(
ctx: &EncodingContext,
opsin: &Image3F,
errors: &[[f32; 1024]; 3],
px: usize,
py: usize,
width: usize,
height: usize,
distance: f32,
) -> (f32, f32) {
let coarse_mix = ((distance - 1.9) / 0.1).clamp(0.0, 1.0);
let floor = distance * fmla(coarse_mix, 0.0045 - 0.0015, 0.0015);
let mut energy = 0.0f32;
let mut peak = 0.0f32;
for (c, error) in errors.iter().enumerate() {
let plane = opsin.plane(c);
let mut channel = 0.0f32;
let weight = ctx.channel_weight(c);
for x in (8..width).step_by(8) {
for y in 0..height {
let i = y * width + x;
let sy = (py + y).min(plane.ysize() - 1);
let left_x = (px + x - 1).min(plane.xsize() - 1);
let right_x = (px + x).min(plane.xsize() - 1);
let source_gradient = (plane.row(sy)[right_x] - plane.row(sy)[left_x]).abs();
let excess =
((error[i] - error[i - 1]).abs() - 0.5 * source_gradient - floor).max(0.0);
channel = fmla(excess, excess, channel);
peak = peak.max(weight * excess * excess);
}
}
for y in (8..height).step_by(8) {
for x in 0..width {
let i = y * width + x;
let sx = (px + x).min(plane.xsize() - 1);
let top_y = (py + y - 1).min(plane.ysize() - 1);
let bottom_y = (py + y).min(plane.ysize() - 1);
let source_gradient = (plane.row(bottom_y)[sx] - plane.row(top_y)[sx]).abs();
let excess =
((error[i] - error[i - width]).abs() - 0.5 * source_gradient - floor).max(0.0);
channel = fmla(excess, excess, channel);
peak = peak.max(weight * excess * excess);
}
}
energy += weight * channel;
}
(energy, peak)
}
pub(crate) struct FineMosaicScratch {
big_error: Box<[[f32; 1024]; 3]>,
cand_costs: Box<[[ReconStrategyCost; FINE_MOSAIC_CANDIDATES.len()]; FINE_MOSAIC_MAX_CHILDREN]>,
cand_planes: Box<[[[[f32; 64]; 3]; FINE_MOSAIC_CANDIDATES.len()]; FINE_MOSAIC_MAX_CHILDREN]>,
}
impl Default for FineMosaicScratch {
fn default() -> Self {
let nan_cost = ReconStrategyCost {
cost: f32::NAN,
base: f32::NAN,
distortion: f32::NAN,
rate: f32::NAN,
hue_distortion: f32::NAN,
};
Self {
big_error: crate::util::heap_array([0.0; 1024]),
cand_costs: crate::util::heap_array([nan_cost; FINE_MOSAIC_CANDIDATES.len()]),
cand_planes: crate::util::heap_array([[[0.0; 64]; 3]; FINE_MOSAIC_CANDIDATES.len()]),
}
}
}
#[derive(Clone, Copy)]
struct PartitionTile {
x: usize,
y: usize,
width: usize,
height: usize,
cost: f32,
}
#[derive(Clone, Copy)]
struct ReconstructionPlan {
cost: f32,
tiles: [u8; 16],
}
fn best_reconstruction_plan(
tiles: &[PartitionTile],
width: usize,
height: usize,
) -> ReconstructionPlan {
let empty = ReconstructionPlan {
cost: f32::INFINITY,
tiles: [NO_CHILD_BLOCK; 16],
};
let mut plans = [empty; 9 * 16];
let index = |lx: usize, ly: usize, x: usize, y: usize| (ly * 3 + lx) * 16 + y * 4 + x;
for (i, tile) in tiles.iter().enumerate() {
if !tile.width.is_power_of_two()
|| !tile.height.is_power_of_two()
|| tile.width > width
|| tile.height > height
|| tile.x >= width
|| tile.y >= height
|| !tile.x.is_multiple_of(tile.width)
|| !tile.y.is_multiple_of(tile.height)
{
continue;
}
let slot = index(
tile.width.ilog2() as usize,
tile.height.ilog2() as usize,
tile.x,
tile.y,
);
if tile.cost < plans[slot].cost {
plans[slot].cost = tile.cost;
plans[slot].tiles[tile.y * 4 + tile.x] = i as u8;
}
}
let join = |a: ReconstructionPlan, b: ReconstructionPlan| {
let mut tiles = a.tiles;
for (i, &tile) in b.tiles.iter().enumerate() {
if tile != NO_CHILD_BLOCK {
tiles[i] = tile;
}
}
ReconstructionPlan {
cost: a.cost + b.cost,
tiles,
}
};
for ly in 0..=height.ilog2() as usize {
let h = 1 << ly;
for lx in 0..=width.ilog2() as usize {
let w = 1 << lx;
for y in (0..height).step_by(h) {
for x in (0..width).step_by(w) {
let mut best = empty;
if lx > 0 {
best = join(
plans[index(lx - 1, ly, x, y)],
plans[index(lx - 1, ly, x + w / 2, y)],
);
}
if ly > 0 {
let split = join(
plans[index(lx, ly - 1, x, y)],
plans[index(lx, ly - 1, x, y + h / 2)],
);
if split.cost < best.cost {
best = split;
}
}
let slot = index(lx, ly, x, y);
if plans[slot].cost < best.cost {
best = plans[slot];
}
plans[slot] = best;
}
}
}
}
plans[index(width.ilog2() as usize, height.ilog2() as usize, 0, 0)]
}
#[cfg(test)]
fn best_reconstruction_plan_reference(
tiles: &[PartitionTile],
width: usize,
height: usize,
) -> ReconstructionPlan {
let empty = ReconstructionPlan {
cost: f32::INFINITY,
tiles: [NO_CHILD_BLOCK; 16],
};
let mut plans = [empty; 9 * 16];
let index = |lx: usize, ly: usize, x: usize, y: usize| (ly * 3 + lx) * 16 + y * 4 + x;
let join = |a: ReconstructionPlan, b: ReconstructionPlan| {
let mut tiles = a.tiles;
for (i, &tile) in b.tiles.iter().enumerate() {
if tile != NO_CHILD_BLOCK {
tiles[i] = tile;
}
}
ReconstructionPlan {
cost: a.cost + b.cost,
tiles,
}
};
for ly in 0..=height.ilog2() as usize {
let h = 1 << ly;
for lx in 0..=width.ilog2() as usize {
let w = 1 << lx;
for y in (0..height).step_by(h) {
for x in (0..width).step_by(w) {
let mut best = empty;
if lx > 0 {
best = join(
plans[index(lx - 1, ly, x, y)],
plans[index(lx - 1, ly, x + w / 2, y)],
);
}
if ly > 0 {
let split = join(
plans[index(lx, ly - 1, x, y)],
plans[index(lx, ly - 1, x, y + h / 2)],
);
if split.cost < best.cost {
best = split;
}
}
for (i, tile) in tiles.iter().enumerate() {
if (tile.x, tile.y, tile.width, tile.height) == (x, y, w, h)
&& tile.cost < best.cost
{
best = empty;
best.cost = tile.cost;
best.tiles[y * 4 + x] = i as u8;
}
}
plans[index(lx, ly, x, y)] = best;
}
}
}
}
plans[index(width.ilog2() as usize, height.ilog2() as usize, 0, 0)]
}
fn find_rerank_downgrades(
rerank: &RerankContext<'_>,
scratch: &mut CoderScratch,
ac_strategy: &AcStrategyImage,
band: (usize, usize),
output: &mut AcStrategyBandScratch,
) {
let params = rerank.params;
let ctx = params.ctx;
let (y0, y1) = band;
let rerank_margin = ctx.merge.rerank_margin;
let meta_r = RERANK_META_R;
let saved_map: std::collections::HashMap<(u16, u16), [u8; 16]> = output
.saved_children
.iter()
.map(|s| ((s.bx, s.by), s.grid))
.collect();
output.rerank_downgrades.clear();
output.rerank64_downgrades.clear();
output.fine_rollbacks.clear();
output.current_costs.clear();
let fine_lambda = fine_mosaic_lambda(params.distance);
let coarse_chromatic = ctx.x_heavy();
let hue_repair_enabled = coarse_chromatic && ctx.xyb.fwd[8] >= 0.8;
let mut partition_tiles = Vec::new();
let mut partition_costs = Vec::new();
for (bx, by, strat) in ac_strategy.iter_first_blocks() {
if by < y0 || by >= y1 {
continue;
}
if matches!(
strat,
STRATEGY_DCT64X64 | STRATEGY_DCT64X32 | STRATEGY_DCT32X64
) {
recon64::find_downgrade(rerank, scratch, ac_strategy, bx, by, strat, output);
continue;
}
let mosaic = &mut output.fine_mosaic;
let cxb = AcStrategyImage::covered_blocks_x_of(strat);
let cyb = AcStrategyImage::covered_blocks_y_of(strat);
if cxb * cyb <= 1 {
continue; }
let (px, py) = (params.dc_group_px + bx * 8, params.dc_group_py + by * 8);
let (gradient_alpha, gradient_peak_alpha) =
rerank_pair_gradient_alphas(rerank, strat, px, py);
let qac_big = region_qac(
rerank.quant_field,
bx,
by,
cxb,
cyb,
params.scale,
params.distance,
);
let effort = ctx.speed.effort();
let with_fine_mosaic = effort.fine_mosaic
&& !(effort.layout_retile && !effort.layout_loop())
&& (params.distance <= FINE_MOSAIC_MAX_DISTANCE || coarse_chromatic)
&& matches!(strat, STRATEGY_DCT16X8 | STRATEGY_DCT8X16);
debug_assert!(!with_fine_mosaic || cxb * cyb <= FINE_MOSAIC_MAX_CHILDREN);
let region_pixels = cxb * cyb * 64;
let big = reconstruction_strategy_cost_and_base(
ctx,
scratch,
strat,
params.opsin,
px,
py,
qac_big,
params.qm_mult_x,
meta_r,
params.distance,
cmap_factors(
params.ctx.cfl_frame(),
params.ytox_map,
params.ytob_map,
bx,
by,
),
gradient_alpha,
gradient_peak_alpha,
if with_fine_mosaic {
let [e0, e1, e2] = &mut *mosaic.big_error;
Some([
&mut e0[..region_pixels],
&mut e1[..region_pixels],
&mut e2[..region_pixels],
])
} else {
None
},
);
let (j_big, big_current_cost) = (big.cost, big.base);
let with_fine_mosaic = with_fine_mosaic
&& (params.distance <= FINE_MOSAIC_MAX_DISTANCE
|| coarse_mosaic_hue_is_costly(big.hue_distortion, fine_lambda));
let (big_boundary, big_peak) = if with_fine_mosaic {
block_boundary_error_stats(
ctx,
params.opsin,
&mosaic.big_error,
px,
py,
cxb * 8,
cyb * 8,
params.distance,
)
} else {
(0.0, 0.0)
};
let with_fine_mosaic = with_fine_mosaic && big_boundary > 0.0 && big_peak > 0.0;
let mut j_dct8 = 0.0f32;
partition_tiles.clear();
partition_costs.clear();
partition_tiles.reserve(48);
partition_costs.reserve(48);
let tiled_costs_start = output.current_costs.len();
for iy in 0..cyb {
for ix in 0..cxb {
let q = region_qac(
rerank.quant_field,
bx + ix,
by + iy,
1,
1,
params.scale,
params.distance,
);
let child = iy * cxb + ix;
let cmap = cmap_factors(
params.ctx.cfl_frame(),
params.ytox_map,
params.ytob_map,
bx + ix,
by + iy,
);
for (ci, &strategy) in FINE_MOSAIC_CANDIDATES.iter().enumerate() {
if ci > 0 && !with_fine_mosaic {
break;
}
let sink = if with_fine_mosaic {
let [e0, e1, e2] = &mut mosaic.cand_planes[child][ci];
Some([&mut e0[..], &mut e1[..], &mut e2[..]])
} else {
None
};
let cand = reconstruction_strategy_cost_and_base(
ctx,
scratch,
strategy,
params.opsin,
px + ix * 8,
py + iy * 8,
q,
params.qm_mult_x,
meta_r,
params.distance,
cmap,
gradient_alpha,
gradient_peak_alpha,
sink,
);
if with_fine_mosaic {
mosaic.cand_costs[child][ci] = cand;
}
if ci == 0 {
partition_tiles.push(PartitionTile {
x: ix,
y: iy,
width: 1,
height: 1,
cost: cand.cost,
});
partition_costs.push((STRATEGY_DCT, cand));
j_dct8 += cand.cost;
output.current_costs.push(CachedQuantCost {
bx: bx + ix,
by: by + iy,
cost: cand.base,
});
}
}
}
}
let child_costs_start = output.current_costs.len();
let saved = saved_map.get(&(bx as u16, by as u16));
let rectangles = SearchScope::for_speed(ctx.speed).rectangles();
for iy in 0..cyb {
for ix in 0..cxb {
let leaf = if rerank.leaf_first {
rerank
.leaves
.get((by + iy) * ac_strategy.xsize() + bx + ix)
.copied()
.unwrap_or(STRATEGY_DCT)
} else {
saved.map_or(NO_CHILD_BLOCK, |grid| grid[iy * 4 + ix])
};
for (candidate_index, s) in [
leaf,
STRATEGY_DCT16X8,
STRATEGY_DCT8X16,
STRATEGY_DCT16X16,
STRATEGY_DCT32X16,
STRATEGY_DCT16X32,
]
.into_iter()
.enumerate()
{
if s == NO_CHILD_BLOCK || s == STRATEGY_DCT {
continue;
}
let ccx = AcStrategyImage::covered_blocks_x_of(s);
let ccy = AcStrategyImage::covered_blocks_y_of(s);
if candidate_index == 0 && ccx * ccy > 1 {
continue;
}
if ix + ccx > cxb
|| iy + ccy > cyb
|| !ix.is_multiple_of(ccx)
|| !iy.is_multiple_of(ccy)
|| (ccx == cxb && ccy == cyb)
|| (!rectangles && ccx != ccy)
|| !ac_strategy.can_place_strategy(bx + ix, by + iy, s)
{
continue;
}
let cost = reconstruction_strategy_cost_and_base(
ctx,
scratch,
s,
params.opsin,
px + ix * 8,
py + iy * 8,
region_qac(
rerank.quant_field,
bx + ix,
by + iy,
ccx,
ccy,
params.scale,
params.distance,
),
params.qm_mult_x,
meta_r,
params.distance,
cmap_factors(
params.ctx.cfl_frame(),
params.ytox_map,
params.ytob_map,
bx + ix,
by + iy,
),
gradient_alpha,
gradient_peak_alpha,
None,
);
let charge = if matches!(s, STRATEGY_IDENTITY | STRATEGY_DCT2X2) {
reconstruction_lambda(params.distance)
* learned_fine_rate_correction(cost.rate)
} else {
0.0
};
partition_tiles.push(PartitionTile {
x: ix,
y: iy,
width: ccx,
height: ccy,
cost: cost.cost + charge,
});
partition_costs.push((s, cost));
}
}
}
let plan = best_reconstruction_plan(&partition_tiles, cxb, cyb);
let j_child = plan.cost;
let mut child_grid = [NO_CHILD_BLOCK; 16];
for (i, &tile) in plan.tiles.iter().enumerate() {
if tile == NO_CHILD_BLOCK {
continue;
}
let (strategy, cost) = partition_costs[tile as usize];
child_grid[i] = strategy;
output.current_costs.push(CachedQuantCost {
bx: bx + i % 4,
by: by + i / 4,
cost: cost.base,
});
}
let j_child_fine = if with_fine_mosaic {
for (tile, (_, cost)) in partition_tiles.iter_mut().zip(&partition_costs) {
tile.cost = fmla(fine_lambda, cost.rate + meta_r, cost.distortion);
}
best_reconstruction_plan(&partition_tiles, cxb, cyb).cost
} else {
f32::INFINITY
};
let child_wins = j_child < j_dct8;
let j_alt = if child_wins { j_child } else { j_dct8 };
let ordinary_wins = j_alt < j_big * rerank_margin;
let mut fine_benefit = 0.0f32;
let mut fine_grid = [STRATEGY_DCT; 16];
let mut fine_sel = [0usize; FINE_MOSAIC_MAX_CHILDREN];
let mut fine_wins = false;
if with_fine_mosaic {
let children = cxb * cyb;
let mut rd_fine = [[0.0f32; FINE_MOSAIC_CANDIDATES.len()]; FINE_MOSAIC_MAX_CHILDREN];
for (k, child_rd) in rd_fine[..children].iter_mut().enumerate() {
for (ci, rd) in child_rd.iter_mut().enumerate() {
let extra = if ci == 0 {
0.0
} else {
FINE_MOSAIC_RATE_CORRECTION_BITS
};
let c = &mosaic.cand_costs[k][ci];
*rd = fmla(fine_lambda, c.rate + meta_r + extra, c.distortion);
}
}
let tiled_rd: f32 = rd_fine[..children].iter().map(|c| c[0]).sum();
let big_rd = fmla(fine_lambda, big.rate + meta_r, big.distortion);
let big_joint = fmla(FINE_MOSAIC_BOUNDARY_ALPHA, big_boundary, big_rd);
let hue_reference = if hue_repair_enabled {
let tiled_hue: f32 = mosaic.cand_costs[..children]
.iter()
.map(|c| c[0].hue_distortion)
.sum();
big.hue_distortion.min(tiled_hue)
} else {
0.0
};
let mut tiled_joint = f32::INFINITY;
let mut best_joint = f32::INFINITY;
let mut best_sel = None;
for code in 0..FINE_MOSAIC_CANDIDATES.len().pow(children as u32) {
let mut sel = [0usize; FINE_MOSAIC_MAX_CHILDREN];
let mut rest = code;
let mut rd_sum = 0.0f32;
let mut n_fine = 0usize;
for (k, s) in sel[..children].iter_mut().enumerate() {
*s = rest % FINE_MOSAIC_CANDIDATES.len();
rest /= FINE_MOSAIC_CANDIDATES.len();
rd_sum += rd_fine[k][*s];
n_fine += usize::from(*s != 0);
}
let subsidy_cap = fmla(
fine_lambda * FINE_MOSAIC_SEAM_SUBSIDY_BITS,
n_fine as f32,
tiled_rd,
);
let within_subsidy = rd_sum <= subsidy_cap;
if code != 0 && !within_subsidy {
continue;
}
let mut selected = [&mosaic.cand_planes[0][0]; FINE_MOSAIC_MAX_CHILDREN];
for (k, s) in selected[..children].iter_mut().enumerate() {
*s = &mosaic.cand_planes[k][sel[k]];
}
let (boundary, peak) = (ctx.mosaic_seam_stats)(
ctx,
params.opsin,
px,
py,
cxb,
cyb,
params.distance,
&selected[..children],
);
let joint = fmla(FINE_MOSAIC_BOUNDARY_ALPHA, boundary, rd_sum);
if code == 0 {
tiled_joint = joint;
continue;
}
if joint < best_joint {
let improves_seams = boundary < big_boundary * FINE_MOSAIC_BOUNDARY_RATIO
&& peak < big_peak * FINE_MOSAIC_PEAK_RATIO;
let repairs_hue = hue_repair_enabled && !improves_seams && {
let hue_sum: f32 = sel[..children]
.iter()
.enumerate()
.map(|(k, &ci)| mosaic.cand_costs[k][ci].hue_distortion)
.sum();
let seam_regression = FINE_MOSAIC_BOUNDARY_ALPHA
* ((boundary - big_boundary).max(0.0) + (peak - big_peak).max(0.0));
fine_mosaic_hue_repair_is_safe(
hue_reference,
hue_sum,
rd_sum,
tiled_rd.min(big_rd).min(j_child_fine),
seam_regression,
)
};
if improves_seams || repairs_hue {
best_joint = joint;
best_sel = Some(sel);
}
}
}
if let Some(sel) = best_sel
&& best_joint < tiled_joint * FINE_MOSAIC_MARGIN
&& best_joint < big_joint * rerank_margin * FINE_MOSAIC_MARGIN
&& best_joint < j_child_fine * FINE_MOSAIC_MARGIN
{
fine_wins = true;
fine_benefit = big_joint - best_joint;
fine_sel = sel;
for iy in 0..cyb {
for ix in 0..cxb {
fine_grid[iy * 4 + ix] = FINE_MOSAIC_CANDIDATES[sel[iy * cxb + ix]];
}
}
}
}
if ordinary_wins || fine_wins {
let restore = if fine_wins {
output.current_costs.truncate(child_costs_start);
for iy in 0..cyb {
for ix in 0..cxb {
let child = iy * cxb + ix;
output.current_costs[tiled_costs_start + child].cost =
mosaic.cand_costs[child][fine_sel[child]].base;
}
}
output.fine_rollbacks.push(FineMergeRollback {
bx,
by,
cov_x: cxb,
cov_y: cyb,
strategy: strat,
quant: region_quant(rerank.quant_field, bx, by, cxb, cyb, params.distance),
fine_grid,
benefit: fine_benefit,
});
fine_grid
} else if child_wins {
output
.current_costs
.drain(tiled_costs_start..child_costs_start);
child_grid
} else {
output.current_costs.truncate(child_costs_start);
let mut g = [NO_CHILD_BLOCK; 16];
for iy in 0..cyb {
for ix in 0..cxb {
g[iy * 4 + ix] = STRATEGY_DCT;
}
}
g
};
output.rerank_downgrades.push(RerankDowngrade {
bx,
by,
cov_x: cxb,
cov_y: cyb,
restore,
});
} else {
output.current_costs.truncate(tiled_costs_start);
output.current_costs.push(CachedQuantCost {
bx,
by,
cost: big_current_cost,
});
}
}
}
fn rerank_pair_gradient_alphas(
rerank: &RerankContext<'_>,
strat: u8,
px: usize,
py: usize,
) -> (f32, f32) {
let params = rerank.params;
let ctx = params.ctx;
if !matches!(strat, STRATEGY_DCT16X8 | STRATEGY_DCT8X16) {
return (0.0, 0.0);
}
let alpha = rerank_pair_gradient_alpha(params.distance);
let peak_alpha = rerank_pair_gradient_peak_alpha(params.distance);
if alpha == 0.0 && peak_alpha == 0.0 {
return (0.0, 0.0);
}
let pair_gradient_fit = &PAIR_GRAD_FIT;
let (width, height) = if strat == STRATEGY_DCT16X8 {
(8, 16)
} else {
(16, 8)
};
let stats = if params.distance < RERANK_PAIR_GRADIENT_PEAK_COARSE_END {
(ctx.gradient_region_stats)(params.opsin, px, py, width, height, 1e-5)
} else {
(ctx.gradient_region_stats_with_chroma)(params.opsin, px, py, width, height, 1e-5)
};
(
if stats.dominance >= RERANK_PAIR_GRADIENT_MIN_DOMINANCE {
alpha
} else {
0.0
},
if stats.dominance >= pair_gradient_fit.peak_min_dominance
&& stats.mean >= pair_gradient_fit.min_luma
&& (params.distance < RERANK_PAIR_GRADIENT_PEAK_COARSE_END
|| stats.chroma <= pair_gradient_fit.chroma_cap)
{
peak_alpha
} else {
0.0
},
)
}
fn footprint_is_self_contained(
map: &AcStrategyImage,
bx: usize,
by: usize,
cov_x: usize,
cov_y: usize,
) -> bool {
let mut area = 0;
for y in by..by + cov_y {
for x in bx..bx + cov_x {
if !map.is_first_block(x, y) {
continue;
}
let s = map.raw_strategy(x, y);
let cx = AcStrategyImage::covered_blocks_x_of(s);
let cy = AcStrategyImage::covered_blocks_y_of(s);
if x + cx > bx + cov_x || y + cy > by + cov_y {
return false;
}
area += cx * cy;
}
}
area == cov_x * cov_y
}
fn cap_medium_upgrade_candidates(indices: &mut Vec<usize>, candidates: &[MergeUpgradeCandidate]) {
indices.sort_by(|&a, &b| {
let (a, b) = (candidates[a], candidates[b]);
(a.by / 4, a.bx / 4)
.cmp(&(b.by / 4, b.bx / 4))
.then_with(|| a.relative_cost.total_cmp(&b.relative_cost))
.then_with(|| (a.by, a.bx, a.strategy).cmp(&(b.by, b.bx, b.strategy)))
});
let mut previous = None;
let mut count = 0;
indices.retain(|&i| {
let c = candidates[i];
let region = (c.by / 4, c.bx / 4);
if previous != Some(region) {
previous = Some(region);
count = 0;
}
count += 1;
count <= MEDIUM_CANDIDATES_PER_REGION
});
}
fn find_merge_upgrades(
rerank: &RerankContext<'_>,
scratch: &mut CoderScratch,
ac_strategy: &AcStrategyImage,
band: (usize, usize),
output: &mut AcStrategyBandScratch,
margin: f32,
) {
find_merge_upgrades_at_level(rerank, scratch, ac_strategy, band, output, margin, false);
}
#[allow(clippy::too_many_arguments)]
fn find_merge_upgrades_at_level(
rerank: &RerankContext<'_>,
scratch: &mut CoderScratch,
ac_strategy: &AcStrategyImage,
band: (usize, usize),
output: &mut AcStrategyBandScratch,
margin: f32,
large: bool,
) {
let params = rerank.params;
let ctx = params.ctx;
let (y0, y1) = band;
let meta_r = RERANK_META_R;
output.merge_upgrades.clear();
let mut groups =
std::collections::BTreeMap::<(usize, usize), [Option<(MergeUpgrade, f32)>; 5]>::new();
let medium = CandidateSearch::for_speed(ctx.speed, ctx.arbiter).medium_upgrades;
let mut indices = Vec::new();
for (ci, cand) in output.upgrade_candidates.iter().enumerate() {
let (x, y, strategy) = (cand.bx as usize, cand.by as usize, cand.strategy);
let w = AcStrategyImage::covered_blocks_x_of(strategy);
let h = AcStrategyImage::covered_blocks_y_of(strategy);
if (w.max(h) > 2) != large
|| y < y0
|| y >= y1
|| !ac_strategy.can_place_strategy(x, y, strategy)
|| (ac_strategy.is_first_block(x, y) && ac_strategy.raw_strategy(x, y) == strategy)
|| !footprint_is_self_contained(ac_strategy, x, y, w, h)
{
continue;
}
if (medium || (large && !rerank.leaf_first))
&& (y..y + h).any(|by| {
(x..x + w).any(|bx| {
let s = ac_strategy.raw_strategy(bx, by);
AcStrategyImage::covered_blocks_x_of(s)
* AcStrategyImage::covered_blocks_y_of(s)
== 1
&& s != STRATEGY_DCT
})
})
{
continue;
}
indices.push(ci);
}
if medium {
cap_medium_upgrade_candidates(&mut indices, &output.upgrade_candidates);
}
for ci in indices {
let cand = output.upgrade_candidates[ci];
let (bx, by, strategy) = (cand.bx as usize, cand.by as usize, cand.strategy);
if by < y0 || by >= y1 {
continue;
}
let cx = AcStrategyImage::covered_blocks_x_of(strategy);
let cy = AcStrategyImage::covered_blocks_y_of(strategy);
let touches_mosaic = output.fine_rollbacks.iter().any(|r| {
bx < r.bx + r.cov_x && r.bx < bx + cx && by < r.by + r.cov_y && r.by < by + cy
});
if touches_mosaic
|| (ac_strategy.is_first_block(bx, by) && ac_strategy.raw_strategy(bx, by) == strategy)
|| !ac_strategy.can_place_strategy(bx, by, strategy)
|| !footprint_is_self_contained(ac_strategy, bx, by, cx, cy)
{
continue;
}
let (px, py) = (params.dc_group_px + bx * 8, params.dc_group_py + by * 8);
let (gradient_alpha, gradient_peak_alpha) =
rerank_pair_gradient_alphas(rerank, strategy, px, py);
let score =
|scratch: &mut CoderScratch, s: u8, x: usize, y: usize, cw: usize, ch: usize| {
reconstruction_strategy_cost_and_base(
ctx,
scratch,
s,
params.opsin,
params.dc_group_px + x * 8,
params.dc_group_py + y * 8,
region_qac(
rerank.quant_field,
x,
y,
cw,
ch,
params.scale,
params.distance,
),
params.qm_mult_x,
meta_r,
params.distance,
cmap_factors(
params.ctx.cfl_frame(),
params.ytox_map,
params.ytob_map,
x,
y,
),
gradient_alpha,
gradient_peak_alpha,
None,
)
};
let big = score(scratch, strategy, bx, by, cx, cy);
let mut j_layout = 0.0f32;
for y in by..by + cy {
for x in bx..bx + cx {
if !ac_strategy.is_first_block(x, y) {
continue;
}
let s = ac_strategy.raw_strategy(x, y);
let scx = AcStrategyImage::covered_blocks_x_of(s);
let scy = AcStrategyImage::covered_blocks_y_of(s);
j_layout += score(scratch, s, x, y, scx, scy).cost;
}
}
let gain = j_layout - big.cost / margin;
if gain > 0.0 {
let (k, side) = if large {
(
match strategy {
STRATEGY_DCT32X32 => 0,
STRATEGY_DCT32X16 => 1 + (bx % 4) / 2,
STRATEGY_DCT16X32 => 3 + (by % 4) / 2,
_ => unreachable!("upgrade outside the 32x32 shortlist"),
},
4,
)
} else {
(
match strategy {
STRATEGY_DCT16X16 => 0,
STRATEGY_DCT16X8 => 1 + bx % 2,
STRATEGY_DCT8X16 => 3 + by % 2,
_ => unreachable!("upgrade outside the 16x16 shortlist"),
},
2,
)
};
groups.entry((by / side, bx / side)).or_insert([None; 5])[k] = Some((
MergeUpgrade {
bx,
by,
strategy,
base: big.base,
},
gain,
));
}
}
for candidates in groups.values() {
let gains = candidates.map(|c| c.map_or(f32::NEG_INFINITY, |(_, gain)| gain));
let mask = choose_upgrade_partition(gains);
for (k, candidate) in candidates.iter().enumerate() {
if mask & (1 << k) != 0 {
output.merge_upgrades.push(candidate.unwrap().0);
}
}
}
}
fn choose_upgrade_partition(gains: [f32; 5]) -> u8 {
let (mut best, mut best_gain) = (0, 0.0);
for mask in MERGE_PARTITIONS.into_iter().skip(1) {
let gain: f32 = gains
.iter()
.enumerate()
.filter(|(k, _)| mask & (1 << k) != 0)
.map(|(_, gain)| gain)
.sum();
if gain > best_gain {
best = mask;
best_gain = gain;
}
}
best
}
fn rerank_upgrade_merges(
rerank: &RerankContext<'_>,
scratch: &mut CoderScratch,
ac_strategy: &mut AcStrategyImage,
band_scratch: &mut [AcStrategyBandScratch],
current_costs: &mut [f32],
margin: f32,
) {
rerank_upgrade_level(
rerank,
scratch,
ac_strategy,
band_scratch,
current_costs,
margin,
false,
);
if CandidateSearch::for_speed(rerank.params.ctx.speed, rerank.params.ctx.arbiter)
.merge32_upgrades
{
rerank_upgrade_level(
rerank,
scratch,
ac_strategy,
band_scratch,
current_costs,
1.0,
true,
);
}
}
#[allow(clippy::too_many_arguments)]
fn rerank_upgrade_level(
rerank: &RerankContext<'_>,
scratch: &mut CoderScratch,
ac_strategy: &mut AcStrategyImage,
band_scratch: &mut [AcStrategyBandScratch],
current_costs: &mut [f32],
margin: f32,
large: bool,
) {
let xsize = ac_strategy.xsize();
let map_ref: &AcStrategyImage = ac_strategy;
rerank
.params
.ctx
.thread_pool
.steal_for_each_mut_with_threads(
scratch,
&mut band_scratch[..rerank.bands.len()],
rerank.num_threads,
|i, output, scratch| {
let mut scratch = rerank.params.price(scratch);
if large {
find_merge_upgrades_at_level(
rerank,
&mut scratch,
map_ref,
rerank.bands[i],
output,
margin,
true,
);
} else {
find_merge_upgrades(
rerank,
&mut scratch,
map_ref,
rerank.bands[i],
output,
margin,
);
}
},
);
for band in &band_scratch[..rerank.bands.len()] {
for up in &band.merge_upgrades {
let cx = AcStrategyImage::covered_blocks_x_of(up.strategy);
let cy = AcStrategyImage::covered_blocks_y_of(up.strategy);
for y in up.by..up.by + cy {
current_costs[y * xsize + up.bx..y * xsize + up.bx + cx].fill(f32::NAN);
}
current_costs[up.by * xsize + up.bx] = up.base;
ac_strategy.set_first(up.bx, up.by, up.strategy);
}
}
}
fn rerank_large_transforms(
rerank: &RerankContext<'_>,
scratch: &mut CoderScratch,
ac_strategy: &mut AcStrategyImage,
band_scratch: &mut [AcStrategyBandScratch],
current_costs: &mut [f32],
) {
let ysize = ac_strategy.ysize();
let ac_strategy_ref: &AcStrategyImage = ac_strategy;
rerank
.params
.ctx
.thread_pool
.steal_for_each_mut_with_threads(
scratch,
&mut band_scratch[..rerank.bands.len()],
rerank.num_threads,
|i, output, scratch| {
let mut scratch = rerank.params.price(scratch);
find_rerank_downgrades(
rerank,
&mut scratch,
ac_strategy_ref,
rerank.bands[i],
output,
);
},
);
debug_assert_eq!(current_costs.len(), ac_strategy.xsize() * ysize);
current_costs.fill(f32::NAN);
for result in &band_scratch[..rerank.bands.len()] {
for cached in &result.current_costs {
current_costs[cached.by * ac_strategy.xsize() + cached.bx] = cached.cost;
}
for downgrade in &result.rerank_downgrades {
for iy in 0..downgrade.cov_y {
for ix in 0..downgrade.cov_x {
let s = downgrade.restore[iy * 4 + ix];
if s != NO_CHILD_BLOCK {
ac_strategy.set_first(downgrade.bx + ix, downgrade.by + iy, s);
}
}
}
}
for downgrade in &result.rerank64_downgrades {
for iy in 0..downgrade.cov_y {
for ix in 0..downgrade.cov_x {
let s = downgrade.restore[iy * 8 + ix];
if s != NO_CHILD_BLOCK {
ac_strategy.set_first(downgrade.bx + ix, downgrade.by + iy, s);
}
}
}
}
}
}
#[cfg(test)]
mod tests {
#[test]
fn qf_trellis_resolves_small_local_gains_on_long_chains() {
let n = 65_536;
let refinement = crate::coder_scratch::QuantRefinement {
bx: 0,
by: 0,
states: [(14, 100.0), (13, 99.99), (0, 0.0), (0, 0.0)],
len: 2,
};
let fixed = vec![14u8; n];
let states = vec![Some(&refinement); n];
let prices = [[0.1f32; 512]; 4];
let mut chain = fixed.clone();
super::qf_trellis(13, &fixed, &states, 1.0, &prices, &mut chain);
assert!(chain.iter().all(|&q| q == 13));
}
#[test]
fn candidate_search_respects_speed_and_arbiter() {
let mut plain = vec![
crate::Speed::UltraFast,
crate::Speed::Fastest,
crate::Speed::Fast,
crate::Speed::Medium,
];
let mut slow_like = vec![crate::Speed::MediumSlow, crate::Speed::Slow];
if cfg!(feature = "extraslow") {
plain.push(crate::Speed::ExtraSlow);
} else {
slow_like.push(crate::Speed::ExtraSlow);
}
for arbiter in [crate::Arbiter::Ssimulacra2, crate::Arbiter::Cvvdp] {
for &speed in &plain {
let p = super::CandidateSearch::for_speed(speed, arbiter);
assert!(!p.fine_runner_up && !p.medium_upgrades && !p.merge32_upgrades);
}
for &speed in &slow_like {
let slow = super::CandidateSearch::for_speed(speed, arbiter);
assert!(!slow.fine_runner_up && !slow.medium_upgrades);
assert_eq!(slow.merge32_upgrades, arbiter == crate::Arbiter::Cvvdp);
}
}
}
#[test]
fn merge32_shortlist_retains_near_winners_with_a_fixed_budget() {
let map = AcStrategyImage::new(4, 4);
let mut candidates = Vec::new();
super::record_upgrade32_candidates(
&mut candidates,
&map,
0,
0,
[10.2, 5.1, 5.2, 5.0, 6.0],
[[2.5; 2]; 2],
);
assert_eq!(candidates.len(), 3);
assert!(
candidates
.iter()
.any(|c| c.strategy == crate::dc_group_data::STRATEGY_DCT32X32)
);
assert!(
candidates
.iter()
.all(|c| c.relative_cost <= super::MERGE32_SHORTLIST_RATIO)
);
assert!(candidates.iter().any(|c| c.relative_cost > 1.0));
}
#[test]
fn medium_shortlist_budget_is_local_and_order_independent() {
let mut candidates = Vec::new();
for region in 0..2u16 {
for k in 0..4u16 {
candidates.push(crate::coder_scratch::MergeUpgradeCandidate {
bx: region * 4 + k % 2 * 2,
by: k / 2 * 2,
strategy: STRATEGY_DCT16X16,
relative_cost: 0.9 - k as f32 * 0.02,
});
}
}
let chosen = |candidates: &[crate::coder_scratch::MergeUpgradeCandidate]| {
let mut indices: Vec<_> = (0..candidates.len()).collect();
super::cap_medium_upgrade_candidates(&mut indices, candidates);
indices
.iter()
.map(|&i| (candidates[i].bx, candidates[i].by))
.collect::<Vec<_>>()
};
let forward = chosen(&candidates);
assert_eq!(forward.len(), 4);
candidates.reverse();
assert_eq!(forward, chosen(&candidates));
}
#[test]
fn fine_reconstruction_can_choose_the_coefficient_runner_up() {
let maps = ImageSB::new_fill(1, 1, 0);
let mut admitted = 0;
let mut runner_wins = 0;
for pattern in 0..4 {
let mut opsin = Image3F::new(8, 8);
for y in 0..8 {
for x in 0..8 {
let ink = match pattern {
0 => x == 3,
1 => x == 3 || y == 3,
2 => (x / 2 + y / 2) % 2 == 0,
_ => x == y,
};
opsin.plane_row_mut(0, y)[x] = if ink { 0.08 } else { 0.0 };
opsin.plane_row_mut(1, y)[x] = if ink { 0.65 } else { 0.12 };
opsin.plane_row_mut(2, y)[x] = if ink { 0.45 } else { 0.12 };
}
}
for distance in [0.5, 1.5, 2.0, 4.0] {
let ctx = EncodingContext::new(
crate::Speed::Slow,
crate::Arbiter::Ssimulacra2,
crate::xyb::XybMatrix::SPEC,
distance,
1,
);
let params = AcStrategyParams {
ctx: &ctx,
opsin: &opsin,
dc_group_px: 0,
dc_group_py: 0,
distance,
scale: 1.0,
qm_mult_x: 1.0,
ytox_map: &maps,
ytob_map: &maps,
prices: None,
};
for q in [0.08, 0.16, 0.32, 0.64] {
let mut scratch = CoderScratch::default();
for order in [
[
crate::dc_group_data::STRATEGY_IDENTITY,
crate::dc_group_data::STRATEGY_DCT2X2,
],
[
crate::dc_group_data::STRATEGY_DCT2X2,
crate::dc_group_data::STRATEGY_IDENTITY,
],
] {
let a = super::Sub8Pick {
strategy: order[0],
biased_j: 1.0,
gain: 0.0,
fallback: STRATEGY_DCT,
};
let b = super::Sub8Pick {
strategy: order[1],
biased_j: 1.01,
gain: 0.0,
fallback: STRATEGY_DCT,
};
let first = super::fine_recon_choose(
params,
&mut scratch,
0,
0,
q,
META_R,
[Some(a), None],
2.0,
);
let second = super::fine_recon_choose(
params,
&mut scratch,
0,
0,
q,
META_R,
[Some(b), None],
2.0,
);
let expected = match (first, second) {
(Some(a), Some(b)) => Some(if a.gain >= b.gain { a } else { b }),
(a, b) => a.or(b),
};
let both = super::fine_recon_choose(
params,
&mut scratch,
0,
0,
q,
META_R,
[Some(a), Some(b)],
2.0,
);
assert_eq!(
both.map(|p| (p.strategy, p.gain.to_bits())),
expected.map(|p| (p.strategy, p.gain.to_bits()))
);
admitted += usize::from(both.is_some());
runner_wins += usize::from(both.is_some_and(|p| p.strategy == b.strategy));
}
}
}
}
assert!(admitted > 0 && runner_wins > 0);
}
#[test]
fn strip_candidates_reach_the_frame_header_gate() {
use super::{adjust_quant_field, place_dct8x32};
use crate::dc_group_data::{STRATEGY_DCT8X32, STRATEGY_DCT32X8};
let distance = 3.0;
let ctx = EncodingContext::new(
crate::Speed::Slow,
crate::Arbiter::Ssimulacra2,
crate::xyb::XybMatrix::SPEC,
distance,
1,
);
let opsin = Image3F::new(crate::frame::K_DC_GROUP_DIM + 32, 32);
let maps = ImageSB::new_fill(1, 1, 0);
let mut group = crate::dc_group_data::DcGroupData::new(4, 4).unwrap();
for y in 0..4 {
group.raw_quant_field.row_mut(y).fill(6);
}
group.block_quant_field = Some(group.raw_quant_field.clone());
let params = AcStrategyParams {
ctx: &ctx,
opsin: &opsin,
dc_group_px: 0,
dc_group_py: 0,
distance,
scale: 1.0,
qm_mult_x: 1.0,
ytox_map: &maps,
ytob_map: &maps,
prices: None,
};
let mut scratch = CoderScratch::default();
let changed = place_dct8x32(
¶ms,
&mut scratch,
&group.raw_quant_field,
&mut group.ac_strategy,
&[],
);
assert_eq!(
changed.len(),
4,
"defer activation until all DC groups are known"
);
adjust_quant_field(&group.ac_strategy, distance, &mut group.raw_quant_field);
super::super::account_matrix_headers(
&ctx,
&mut scratch,
&opsin,
distance,
1.0,
2,
&[(0, 0)],
std::slice::from_mut(&mut group),
);
for (_, _, strategy) in group.ac_strategy.iter_first_blocks() {
assert!(!matches!(strategy, STRATEGY_DCT32X8 | STRATEGY_DCT8X32));
}
for y in 0..4 {
assert_eq!(group.raw_quant_field.row(y), &[6; 4]);
}
assert_map_valid(&group.ac_strategy);
let single_opsin = Image3F::new(32, 32);
let single = AcStrategyParams {
opsin: &single_opsin,
..params
};
assert!(
place_dct8x32(
&single,
&mut scratch,
&group.raw_quant_field,
&mut AcStrategyImage::new(4, 4),
&[],
)
.is_empty()
);
}
#[test]
fn quantizer_cache_excludes_rerank_only_gradient_penalties() {
let distance = 2.5;
let ctx = EncodingContext::new(
crate::Speed::Slow,
crate::Arbiter::Ssimulacra2,
crate::xyb::XybMatrix::SPEC,
distance,
1,
);
let mut opsin = noise_opsin(16, 16, 0x1234_9876);
for c in 0..3 {
for y in 0..16 {
for (x, value) in opsin.plane_mut(c).row_mut(y).iter_mut().enumerate() {
*value = 0.25 + 0.2 * ((x * 13 + y * 19 + c * 7) as f32).sin();
}
}
}
let mut scratch = CoderScratch::default();
let mut saw_penalty = false;
for strategy in [STRATEGY_DCT, STRATEGY_DCT16X8, super::STRATEGY_DCT8X16] {
let mut score = |alpha, peak, metadata| {
super::reconstruction_strategy_cost_and_base(
&ctx,
&mut scratch,
strategy,
&opsin,
0,
0,
5.0,
1.0,
metadata,
distance,
[0.0; 3],
alpha,
peak,
None,
)
};
let plain = score(0.0, 0.0, super::RERANK_META_R);
let fresh_incumbent = score(0.0, 0.0, 0.0);
assert_eq!(plain.base, fresh_incumbent.cost);
for (alpha, peak) in [(32.0, 0.0), (0.0, 32.0), (32.0, 32.0)] {
let penalized = score(alpha, peak, super::RERANK_META_R);
assert!(penalized.cost >= plain.cost);
saw_penalty |= penalized.cost > plain.cost;
assert!(
penalized.base.is_nan(),
"penalized incumbent must be rescored under refinement's objective"
);
}
}
assert!(saw_penalty, "fixture must exercise an actual cost mismatch");
}
#[test]
fn merge_partitions_match_exhaustive_nonoverlapping_subsets() {
let footprints = [0b1111u8, 0b0101, 0b1010, 0b0011, 0b1100];
let mut state = 17u32;
for _ in 0..500 {
let mut next = || {
state = state.wrapping_mul(1664525).wrapping_add(1013904223);
(1 + (state >> 24)) as f32
};
let children = [[next(), next()], [next(), next()]];
let baseline = children.map(|row| row.map(|v| v + next()));
let merges = [next(), next(), next(), next(), next()];
let accept = [0.5, 0.6, 0.7, 0.8, 0.9];
let chosen = super::choose_merge_partition(children, baseline, merges, accept);
let mut optimum: f32 = children.iter().flatten().sum();
for mask in 1u8..32 {
let mut occupied = 0;
let mut cost = 0.0;
let (mut incumbent, mut limit) = (0.0, 0.0);
let mut valid = true;
for k in 0..5 {
if mask & (1 << k) == 0 {
continue;
}
let cells = footprints[k];
let mut child = 0.0;
let mut base = 0.0;
for i in 0..4 {
if cells & (1 << i) != 0 {
child += children[i / 2][i % 2];
base += baseline[i / 2][i % 2];
}
}
valid &= occupied & cells == 0;
incumbent += child;
limit += base * accept[k];
occupied |= cells;
cost += merges[k];
}
valid &= cost < incumbent && cost < limit;
for i in 0..4 {
if occupied & (1 << i) == 0 {
cost += children[i / 2][i % 2];
}
}
if valid {
optimum = optimum.min(cost);
}
}
assert_eq!(super::partition_cost(children, merges, chosen), optimum);
}
}
#[test]
fn rejected_square_does_not_hide_a_single_winning_rectangle() {
let children = [[25.0; 2]; 2];
assert_eq!(
super::choose_merge_partition(
children,
children,
[60.0, 15.0, 90.0, 80.0, 80.0],
[0.5, 0.9, 0.9, 0.9, 0.9]
),
2
);
assert_eq!(
super::partition_cost(children, [60.0, 15.0, 90.0, 80.0, 80.0], 2),
65.0
);
}
#[test]
fn upgrades_compare_both_directions_and_combined_gain() {
assert_eq!(
super::choose_upgrade_partition([7.0, 4.0, 4.0, 3.0, 3.0]),
6
);
assert_eq!(
super::choose_upgrade_partition([7.0, 4.0, 4.0, 5.0, 5.0]),
24
);
assert_eq!(
super::choose_upgrade_partition([11.0, 4.0, 4.0, 5.0, 5.0]),
1
);
assert_eq!(
super::choose_upgrade_partition([-1.0, -1.0, 2.0, -1.0, -1.0]),
4
);
assert_eq!(super::choose_upgrade_partition([f32::NEG_INFINITY; 5]), 0);
assert_eq!(super::choose_upgrade_partition([0.0; 5]), 0);
}
#[test]
fn two_rectangles_keep_their_joint_acceptance_margin() {
let children = [[25.0; 2]; 2];
assert_eq!(
super::choose_merge_partition(
children,
children,
[100.0, 10.0, 40.0, 100.0, 100.0],
[0.7; 5]
),
6
);
}
#[test]
fn indexed_reconstruction_planner_preserves_costs_and_ties() {
use super::PartitionTile;
let special = [
0.0,
-0.0,
1.0,
2.0,
f32::INFINITY,
f32::NEG_INFINITY,
f32::NAN,
];
let mut state = 31u32;
for (width, height) in [(1, 1), (1, 2), (2, 1), (2, 2), (2, 4), (4, 2), (4, 4)] {
for round in 0..128 {
let mut tiles = Vec::new();
for h in [1, 2, 4] {
for w in [1, 2, 4] {
if w > width || h > height {
continue;
}
for y in (0..height).step_by(h) {
for x in (0..width).step_by(w) {
for _ in 0..3 {
state = state.wrapping_mul(1664525).wrapping_add(1013904223);
if state.is_multiple_of(5) {
continue;
}
let cost = if round < 64 {
special[state as usize % special.len()]
} else {
f32::from_bits(0x3f00_0000 + (state >> 10))
};
tiles.push(PartitionTile {
x,
y,
width: w,
height: h,
cost,
});
}
}
}
}
}
for i in (1..tiles.len()).rev() {
state = state.wrapping_mul(1664525).wrapping_add(1013904223);
tiles.swap(i, state as usize % (i + 1));
}
let expected = super::best_reconstruction_plan_reference(&tiles, width, height);
let actual = super::best_reconstruction_plan(&tiles, width, height);
assert_eq!(actual.cost.to_bits(), expected.cost.to_bits());
assert_eq!(actual.tiles, expected.tiles);
}
}
}
#[test]
fn reconstruction_planner_matches_every_exact_cover() {
use super::PartitionTile;
fn oracle(
tiles: &[PartitionTile],
masks: &[u16],
covered: u16,
full: u16,
memo: &mut std::collections::HashMap<u16, f32>,
) -> f32 {
if covered == full {
return 0.0;
}
if let Some(&cost) = memo.get(&covered) {
return cost;
}
let first = 1u16 << (full & !covered).trailing_zeros();
let mut best = f32::INFINITY;
for (tile, &mask) in tiles.iter().zip(masks) {
if mask & first != 0 && mask & covered == 0 {
best = best.min(tile.cost + oracle(tiles, masks, covered | mask, full, memo));
}
}
memo.insert(covered, best);
best
}
let mut state = 31u32;
for (width, height) in [(1, 2), (2, 1), (2, 2), (2, 4), (4, 2), (4, 4)] {
for _ in 0..12 {
let mut tiles = Vec::new();
let mut masks = Vec::new();
for (w, h) in [(1, 1), (1, 2), (2, 1), (2, 2), (2, 4), (4, 2)] {
if w > width || h > height {
continue;
}
for y in (0..height).step_by(h) {
for x in (0..width).step_by(w) {
state = state.wrapping_mul(1664525).wrapping_add(1013904223);
let cost = (1 + (state >> 24)) as f32;
tiles.push(PartitionTile {
x,
y,
width: w,
height: h,
cost,
});
let mut mask = 0u16;
for yy in y..y + h {
for xx in x..x + w {
mask |= 1 << (yy * 4 + xx);
}
}
masks.push(mask);
}
}
}
let full = masks.iter().fold(0, |a, b| a | b);
let expected = oracle(&tiles, &masks, 0, full, &mut Default::default());
let plan = super::best_reconstruction_plan(&tiles, width, height);
assert_eq!(plan.cost, expected, "{width}x{height}");
let mut covered = 0;
for &i in &plan.tiles {
if i == super::NO_CHILD_BLOCK {
continue;
}
assert_eq!(covered & masks[i as usize], 0);
covered |= masks[i as usize];
}
assert_eq!(covered, full);
}
}
}
#[test]
fn reconstruction_can_keep_one_child_and_split_the_other() {
use super::PartitionTile;
let mut tiles = Vec::new();
for y in 0..2 {
for x in 0..2 {
tiles.push(PartitionTile {
x,
y,
width: 1,
height: 1,
cost: 5.0,
});
}
}
tiles.push(PartitionTile {
x: 0,
y: 0,
width: 1,
height: 2,
cost: 3.0,
});
tiles.push(PartitionTile {
x: 1,
y: 0,
width: 1,
height: 2,
cost: 15.0,
});
let plan = super::best_reconstruction_plan(&tiles, 2, 2);
assert_eq!(plan.cost, 13.0);
assert_eq!(plan.tiles[0], 4);
assert_eq!(plan.tiles[1], 1);
assert_eq!(plan.tiles[4], super::NO_CHILD_BLOCK);
assert_eq!(plan.tiles[5], 3);
}
use super::{
AcStrategyParams, Chosen32Cost, META_R, NO_CHILD_BLOCK, SavedChild, SelectionContext,
leaf_first, select_band,
};
use super::{
BIAS_4X4, BIAS_4X8, BIAS_16X16, BIAS_AFV, BIAS_RECT32, DCT8_ONLY_MAX_DISTANCE,
FAST_RERANK_MAX_DISTANCE, FINE_ADMIT_RATE_CORRECTION_BITS, LEAF_FIRST_MAX_DISTANCE,
MERGE_MARGIN_16, MERGE_MARGIN_32, MERGE_MARGIN_PAIR, MERGE_UPGRADE_MARGIN,
MERGE_UPGRADE_MIN_DISTANCE, MergeTuning, RERANK_PAIR_GRADIENT_ALPHA,
RERANK_PAIR_GRADIENT_FADE_IN_END, RERANK_PAIR_GRADIENT_FADE_IN_START,
RERANK_PAIR_GRADIENT_FADE_OUT_END, RERANK_PAIR_GRADIENT_FADE_OUT_START,
RERANK_PAIR_GRADIENT_MIN_DOMINANCE, RERANK_PAIR_GRADIENT_PEAK_ALPHA,
RERANK_PAIR_GRADIENT_PEAK_COARSE_ALPHA, RERANK_PAIR_GRADIENT_PEAK_COARSE_END,
RERANK_PAIR_GRADIENT_PEAK_COARSE_START, RERANK_PAIR_GRADIENT_PEAK_FADE_IN_END,
RERANK_PAIR_GRADIENT_PEAK_FADE_IN_START, RERANK_PAIR_GRADIENT_PEAK_FADE_OUT_END,
RERANK_PAIR_GRADIENT_PEAK_FADE_OUT_START, SUB8_MAX_DISTANCE, SearchScope,
aggregate_qac_2x2, aggregate_quant, block_boundary_error_energy,
block_boundary_error_stats, cmap_factors, fill_ac_strategy, fill_selection_bands,
fine_mosaic_lambda, gradient_region_stats_scalar, gradient_region_stats_with_chroma_scalar,
merge_beats_dct8, merge_margin, merge_upgrade_margin, quant_refinement_steps,
rerank_pair_gradient_peak_alpha, rerank_pair_gradient_scale,
select_gradient_region_stats_fn, select_gradient_region_stats_with_chroma_fn,
strategy_cost, sub8_strategy_costs, use_dct8_only, use_dct8_only_for_content,
};
use crate::coder_scratch::{AcStrategyBandScratch, CoderScratch};
use crate::dc_group_data::{
AcStrategyImage, STRATEGY_DCT, STRATEGY_DCT2X2, STRATEGY_DCT4X4, STRATEGY_DCT4X8,
STRATEGY_DCT8X4, STRATEGY_DCT16X8, STRATEGY_DCT16X16, STRATEGY_DCT16X32, STRATEGY_DCT32X16,
STRATEGY_DCT32X32, STRATEGY_DCT32X64, STRATEGY_DCT64X32, STRATEGY_DCT64X64,
STRATEGY_IDENTITY,
};
use crate::encoding_context::EncodingContext;
use crate::image::{Image3F, ImageB, ImageSB};
use crate::inflated_cost::{
forward_for, forward_matrix, reconstruct_error, strategy_pixel_count,
};
#[test]
fn hue_repair_can_pay_for_seams_but_requires_a_clear_rd_win() {
let accepts = super::fine_mosaic_hue_repair_is_safe;
assert!(accepts(10.0, 8.0, 95.0, 100.0, 1.5));
assert!(!accepts(10.0, 9.6, 95.0, 100.0, 0.0));
assert!(!accepts(10.0, 8.0, 98.0, 100.0, 0.0));
assert!(!accepts(10.0, 8.0, 100.1, 100.0, 0.0));
assert!(!accepts(10.0, 8.0, 95.0, 100.0, 2.1));
assert!(!accepts(0.0, 0.0, 95.0, 100.0, 0.0));
assert!(!accepts(f32::NAN, 8.0, 95.0, 100.0, 0.0));
assert!(!accepts(10.0, f32::NAN, 95.0, 100.0, 0.0));
}
#[test]
fn hue_protection_survives_coarse_quantization_only_for_chroma_structure() {
let ctx = EncodingContext::new(
crate::Speed::Slow,
crate::Arbiter::Ssimulacra2,
crate::xyb::XybMatrix::SPEC,
2.0,
1,
);
let alpha = |distance| super::rerank_rgb_hue_alpha(&ctx, distance);
for distance in [1.0, 2.0, 3.0, 4.0, 10.0] {
assert_eq!(alpha(distance), 0.0);
}
ctx.set_x_heavy(true);
assert_eq!(alpha(0.35), 0.0);
assert!(alpha(0.5) > 0.0 && alpha(0.5) < alpha(1.0));
for distance in [2.0, 2.5, 3.0, 3.25, 4.0, 10.0] {
assert_eq!(alpha(distance), alpha(1.0));
}
}
#[test]
fn mosaic_seam_stats_matches_assembled_boundary_stats() {
let mut seed = 127u32;
let mut random = || {
seed = seed.wrapping_mul(1664525).wrapping_add(1013904223);
((seed >> 8) as f32 / (1u32 << 24) as f32 - 0.5) * 0.25
};
for bias in [crate::xyb::B_BIAS, 0.85] {
let ctx = EncodingContext::new(
crate::Speed::Slow,
crate::Arbiter::Ssimulacra2,
crate::yellow_opsin::matrix_for_bias(bias),
1.0,
1,
);
for (width, height) in [(1, 1), (7, 9), (16, 16), (29, 31)] {
let mut opsin = Image3F::new(width, height);
for c in 0..3 {
for y in 0..height {
for v in opsin.plane_row_mut(c, y) {
*v = random();
}
}
}
let mut planes = [[[[0.0f32; 64]; 3]; 3]; 4];
for child in &mut planes {
for candidate in child {
for channel in candidate {
for v in channel {
*v = random();
}
}
}
}
for (cxb, cyb) in [(1usize, 1usize), (2, 1), (1, 2), (2, 2)] {
for (px, py) in [(0, 0), (width - 1, height - 1)] {
for distance in [0.0, 0.5, 1.9, 1.95, 2.0, 3.0] {
for code in 0..3usize.pow((cxb * cyb) as u32) {
let mut rest = code;
let selected: Vec<_> = planes[..cxb * cyb]
.iter()
.map(|child| {
let i = rest % 3;
rest /= 3;
&child[i]
})
.collect();
let mut assembled = [[0.0f32; 1024]; 3];
for (k, child) in selected.iter().enumerate() {
let (kx, ky) = (k % cxb, k / cxb);
for (dest, source) in assembled.iter_mut().zip(child.iter()) {
for (y, row) in source.as_chunks::<8>().0.iter().enumerate()
{
let offset = (ky * 8 + y) * cxb * 8 + kx * 8;
dest[offset..offset + 8].copy_from_slice(row);
}
}
}
let expected = block_boundary_error_stats(
&ctx,
&opsin,
&assembled,
px,
py,
cxb * 8,
cyb * 8,
distance,
);
let got = (ctx.mosaic_seam_stats)(
&ctx, &opsin, px, py, cxb, cyb, distance, &selected,
);
assert_eq!(
(got.0.to_bits(), got.1.to_bits()),
(expected.0.to_bits(), expected.1.to_bits()),
"{width}x{height}, origin={px},{py}, grid={cxb}x{cyb}, d={distance}, code={code}"
);
}
}
}
}
}
}
}
#[test]
fn boundary_error_energy_detects_artificial_block_seams() {
let ctx = EncodingContext::new(
crate::Speed::Slow,
crate::Arbiter::Ssimulacra2,
crate::xyb::XybMatrix::SPEC,
1.0,
1,
);
let mut opsin = Image3F::new(16, 8);
let smooth = [[0.0f32; 1024]; 3];
assert_eq!(
block_boundary_error_energy(&ctx, &opsin, &smooth, 0, 0, 16, 8, 1.0),
0.0
);
let mut seam = smooth;
for y in 0..8 {
seam[1][y * 16 + 8..y * 16 + 16].fill(0.1);
}
let artificial = block_boundary_error_energy(&ctx, &opsin, &seam, 0, 0, 16, 8, 1.0);
assert!(artificial > 0.0);
for y in 0..8 {
opsin.plane_row_mut(1, y)[8..].fill(0.2);
}
let source_aligned = block_boundary_error_energy(&ctx, &opsin, &seam, 0, 0, 16, 8, 1.0);
assert!(source_aligned < artificial);
}
#[test]
fn dct8_only_covers_the_near_lossless_tail_only() {
assert_eq!(DCT8_ONLY_MAX_DISTANCE, 0.056_713_393);
assert!(use_dct8_only(0.01));
assert!(use_dct8_only(0.05));
assert!(!use_dct8_only(0.06));
assert!(!use_dct8_only(0.2));
assert!(!use_dct8_only(1.0));
for speed in [
crate::Speed::Slow,
crate::Speed::Fast,
crate::Speed::Fastest,
crate::Speed::UltraFast,
] {
let ctx = EncodingContext::new(
speed,
crate::Arbiter::Ssimulacra2,
crate::xyb::XybMatrix::SPEC,
0.05,
1,
);
assert!(use_dct8_only_for_content(&ctx, 0.05));
ctx.set_point_chroma(true);
assert_eq!(
use_dct8_only_for_content(&ctx, 0.05),
speed != crate::Speed::Slow
);
assert!(!use_dct8_only_for_content(&ctx, 0.06));
ctx.set_x_heavy(true);
assert!(use_dct8_only_for_content(&ctx, 0.05));
}
}
#[test]
fn selection_band_storage_is_reused_for_smaller_jobs() {
let mut bands = Vec::new();
fill_selection_bands(&mut bands, 32, 12);
assert_eq!(
bands,
[
(0, 4),
(4, 8),
(8, 12),
(12, 16),
(16, 20),
(20, 24),
(24, 28),
(28, 32),
]
);
let capacity = bands.capacity();
fill_selection_bands(&mut bands, 4, 12);
assert_eq!(bands, [(0, 4)]);
assert_eq!(bands.capacity(), capacity);
}
#[test]
fn banded_knobs_are_flat_outside_the_band() {
assert_eq!(MERGE_MARGIN_16.at(0.1), MERGE_MARGIN_16.at(0.5));
assert_eq!(MERGE_MARGIN_16.at(1.5), MERGE_MARGIN_16.at(6.0));
let mid = MERGE_MARGIN_16.at(1.0);
assert!((mid - 0.5 * (MERGE_MARGIN_16.hq + MERGE_MARGIN_16.base)).abs() < 1e-6);
assert!(MERGE_MARGIN_16.at(0.3) > MERGE_MARGIN_16.at(2.0));
assert!(BIAS_16X16.at(0.3) > BIAS_16X16.at(2.0));
}
#[test]
fn pair_gradient_weights_are_banded() {
const { assert!(RERANK_PAIR_GRADIENT_ALPHA > 0.0) };
assert!((0.0..1.0).contains(&RERANK_PAIR_GRADIENT_MIN_DOMINANCE));
assert_eq!(rerank_pair_gradient_scale(0.35), 0.0);
assert_eq!(
rerank_pair_gradient_scale(RERANK_PAIR_GRADIENT_FADE_IN_START),
0.0
);
assert_eq!(
rerank_pair_gradient_scale(RERANK_PAIR_GRADIENT_FADE_IN_END),
1.0
);
assert_eq!(
rerank_pair_gradient_scale(RERANK_PAIR_GRADIENT_FADE_OUT_START),
1.0
);
assert_eq!(
rerank_pair_gradient_scale(RERANK_PAIR_GRADIENT_FADE_OUT_END),
0.0
);
assert_eq!(rerank_pair_gradient_scale(3.0), 0.0);
let fade_in_mid =
0.5 * (RERANK_PAIR_GRADIENT_FADE_IN_START + RERANK_PAIR_GRADIENT_FADE_IN_END);
assert!((rerank_pair_gradient_scale(fade_in_mid) - 0.5).abs() < 1e-6);
let fade_out_mid =
0.5 * (RERANK_PAIR_GRADIENT_FADE_OUT_START + RERANK_PAIR_GRADIENT_FADE_OUT_END);
assert!((rerank_pair_gradient_scale(fade_out_mid) - 0.5).abs() < 1e-6);
const { assert!(RERANK_PAIR_GRADIENT_PEAK_ALPHA > 0.0) };
assert_eq!(
rerank_pair_gradient_peak_alpha(RERANK_PAIR_GRADIENT_PEAK_FADE_IN_START),
0.0
);
assert_eq!(
rerank_pair_gradient_peak_alpha(RERANK_PAIR_GRADIENT_PEAK_FADE_IN_END),
RERANK_PAIR_GRADIENT_PEAK_ALPHA
);
assert_eq!(
rerank_pair_gradient_peak_alpha(RERANK_PAIR_GRADIENT_PEAK_FADE_OUT_START),
RERANK_PAIR_GRADIENT_PEAK_COARSE_ALPHA
);
assert_eq!(
rerank_pair_gradient_peak_alpha(RERANK_PAIR_GRADIENT_PEAK_COARSE_START),
RERANK_PAIR_GRADIENT_PEAK_ALPHA
);
assert_eq!(
rerank_pair_gradient_peak_alpha(RERANK_PAIR_GRADIENT_PEAK_COARSE_END),
RERANK_PAIR_GRADIENT_PEAK_COARSE_ALPHA
);
assert_eq!(
rerank_pair_gradient_peak_alpha(RERANK_PAIR_GRADIENT_PEAK_FADE_OUT_END),
0.0
);
}
#[test]
fn quant_refinement_does_not_release_at_coarse_distances() {
assert_eq!(quant_refinement_steps(1.0), 0);
assert_eq!(quant_refinement_steps(1.5), 1);
assert_eq!(quant_refinement_steps(1.99), 1);
assert_eq!(quant_refinement_steps(2.0), 2);
assert_eq!(quant_refinement_steps(3.49), 2);
assert_eq!(quant_refinement_steps(3.5), 2);
assert_eq!(quant_refinement_steps(5.0), 2);
assert_eq!(quant_refinement_steps(25.0), 2);
}
#[test]
fn fast_scope_selects_squares_and_no_other_merge_shape() {
let ctx = EncodingContext::new(
crate::Speed::Fast,
crate::Arbiter::Ssimulacra2,
crate::xyb::XybMatrix::SPEC,
1.0,
1,
);
let maps = ImageSB::new_fill(1, 1, 0);
let opsin = Image3F::new(32, 32);
let mut qf = ImageB::new_fill(4, 4, 8);
let mut strategies = AcStrategyImage::new(4, 4);
let mut scratch = CoderScratch::default();
let mut fine_rollbacks = Vec::new();
fill_ac_strategy(
&ctx,
&mut scratch,
&opsin,
0,
0,
3.0,
1.0,
2,
&mut qf,
&maps,
&maps,
&mut strategies,
&mut fine_rollbacks,
&mut Vec::new(),
&mut Vec::new(),
1,
);
assert_eq!(strategies.raw_strategy(0, 0), STRATEGY_DCT32X32);
for (_, _, strat) in strategies.iter_first_blocks() {
assert!(
matches!(strat, STRATEGY_DCT | STRATEGY_DCT16X16 | STRATEGY_DCT32X32),
"Fast produced a non-square merge: {strat}"
);
}
}
#[test]
fn fast_rerank_is_gated_by_distance_but_slow_always_reranks() {
assert!(SearchScope::SQUARES.rerank(FAST_RERANK_MAX_DISTANCE));
assert!(!SearchScope::SQUARES.rerank(FAST_RERANK_MAX_DISTANCE + 0.001));
assert!(SearchScope::FULL.rerank(6.0));
}
#[test]
fn sub8_selection_is_fitted_not_neutral() {
let (b44, b48, gate) = (
std::hint::black_box(BIAS_4X4),
std::hint::black_box(BIAS_4X8),
std::hint::black_box(SUB8_MAX_DISTANCE),
);
assert!(b44 > 1.0, "BIAS_4X4 = {b44}");
assert!(b48 > 1.0, "BIAS_4X8 = {b48}");
assert!(b44 > b48, "4X4 carries the stiffer bar");
assert!(
(0.8..=2.5).contains(&gate),
"gate {gate} outside the band the corpora agree on"
);
assert!(gate > std::hint::black_box(DCT8_ONLY_MAX_DISTANCE));
let afv = std::hint::black_box(BIAS_AFV);
assert!(
afv.hq > 1.0 && afv.base > 1.0,
"AFV bias reverted to neutral"
);
}
#[test]
fn sub8_and_afv_start_at_medium_slow() {
let fastest = SearchScope::for_speed(crate::Speed::Fastest);
assert!(!fastest.rectangles() && !fastest.merge_32() && !fastest.merge_busy_quads());
for speed in [crate::Speed::Fast, crate::Speed::Medium] {
assert_eq!(SearchScope::for_speed(speed), SearchScope::SQUARES);
}
let medium_slow = SearchScope::for_speed(crate::Speed::MediumSlow);
assert!(medium_slow.rectangles());
assert!(medium_slow.rerank(FAST_RERANK_MAX_DISTANCE + 0.001));
assert_eq!(
SearchScope::for_speed(crate::Speed::Slow),
SearchScope::FULL
);
assert!(!SearchScope::SQUARES.rectangles());
assert!(SearchScope::FULL.rectangles());
}
fn noise_opsin(w: usize, h: usize, seed: u32) -> Image3F {
let mut opsin = Image3F::new(w, h);
let mut state = seed;
for c in 0..3 {
for y in 0..h {
for value in opsin.plane_mut(c).row_mut(y) {
state = state.wrapping_mul(1_664_525).wrapping_add(1_013_904_223);
let noise = (state >> 8) as f32 / (1u32 << 24) as f32 - 0.5;
*value = 0.05 * noise * ((y / 7) % 3) as f32 + 0.002 * y as f32;
}
}
}
opsin
}
fn run_selector(speed: crate::Speed, distance: f32, threads: usize) -> (AcStrategyImage, f32) {
let (map, benefit, _) = run_selector_full(speed, distance, threads);
(map, benefit)
}
fn run_selector_full(
speed: crate::Speed,
distance: f32,
threads: usize,
) -> (
AcStrategyImage,
f32,
Vec<crate::coder_scratch::FineMergeRollback>,
) {
let ctx = EncodingContext::new(
speed,
crate::Arbiter::Ssimulacra2,
crate::xyb::XybMatrix::SPEC,
distance,
threads,
);
let (bw, bh) = (13usize, 11usize);
let opsin = noise_opsin(bw * 8, bh * 8, 0x9e37_79b9);
let maps = ImageSB::new_fill(2, 2, 0);
let mut qf = ImageB::new_fill(bw, bh, 6);
for y in 0..bh {
for (x, q) in qf.row_mut(y).iter_mut().enumerate() {
*q = 4 + ((x * 3 + y * 5) % 7) as u8;
}
}
let mut strategies = AcStrategyImage::new(bw, bh);
let mut scratch = CoderScratch::default();
let mut fine_rollbacks = Vec::new();
let benefit = fill_ac_strategy(
&ctx,
&mut scratch,
&opsin,
0,
0,
distance,
1.0,
2,
&mut qf,
&maps,
&maps,
&mut strategies,
&mut fine_rollbacks,
&mut Vec::new(),
&mut Vec::new(),
threads,
);
(strategies, benefit, fine_rollbacks)
}
fn strategy_cells(map: &AcStrategyImage) -> Vec<(usize, usize, u8)> {
map.iter_first_blocks().collect()
}
fn assert_map_valid(map: &AcStrategyImage) {
let (bw, bh) = (map.xsize(), map.ysize());
let mut owner = vec![None; bw * bh];
for (bx, by, s) in map.iter_first_blocks() {
let cx = AcStrategyImage::covered_blocks_x_of(s);
let cy = AcStrategyImage::covered_blocks_y_of(s);
for iy in 0..cy {
for ix in 0..cx {
let cell = &mut owner[(by + iy) * bw + bx + ix];
assert!(
cell.is_none(),
"cell ({},{}) covered twice",
bx + ix,
by + iy
);
*cell = Some((bx, by));
}
}
}
assert!(owner.iter().all(Option::is_some), "uncovered cell");
}
#[test]
fn leaf_first_reproduces_legacy_when_every_leaf_is_dct8() {
for scope in [SearchScope::FULL, SearchScope::SQUARES] {
let distance = 5.5f32;
let ctx = EncodingContext::new(
crate::Speed::Slow,
crate::Arbiter::Ssimulacra2,
crate::xyb::XybMatrix::SPEC,
distance,
1,
);
let (bw, bh) = (13usize, 11usize);
let opsin = noise_opsin(bw * 8, bh * 8, 0x9e37_79b9);
let maps = ImageSB::new_fill(2, 2, 0);
let mut qf = ImageB::new_fill(bw, bh, 6);
for y in 0..bh {
for (x, q) in qf.row_mut(y).iter_mut().enumerate() {
*q = 4 + ((x * 3 + y * 5) % 7) as u8;
}
}
let params = AcStrategyParams {
ctx: &ctx,
opsin: &opsin,
dc_group_px: 0,
dc_group_py: 0,
distance,
scale: 1.0,
qm_mult_x: 1.0,
ytox_map: &maps,
ytob_map: &maps,
prices: None,
};
let selection = SelectionContext {
params,
quant_field: &qf,
meta_r: META_R,
scope,
};
let mut scratch = CoderScratch::default();
let mut legacy_map = AcStrategyImage::new(bw, bh);
let mut legacy = AcStrategyBandScratch::default();
scratch.dct8_costs.clear();
let benefit = select_band(
&selection,
&mut scratch,
&mut legacy_map,
(0, bh),
&mut legacy.chosen32,
&mut legacy.saved_children,
&mut legacy.upgrade_candidates,
);
assert_eq!(benefit, 0.0);
let mut leaf_map = AcStrategyImage::new(bw, bh);
let mut leaf = AcStrategyBandScratch::default();
leaf_first::select_band_leaf_first(
&selection,
&mut scratch,
&mut leaf_map,
(0, bh),
&mut leaf,
);
assert_eq!(
strategy_cells(&legacy_map),
strategy_cells(&leaf_map),
"{scope:?}"
);
let costs = |c: &Vec<Chosen32Cost>| {
c.iter()
.map(|c| (c.bx, c.by, c.cost.to_bits()))
.collect::<Vec<_>>()
};
assert_eq!(costs(&legacy.chosen32), costs(&leaf.chosen32), "{scope:?}");
let effective = |c: &Vec<SavedChild>, map: &AcStrategyImage| {
let mut m = std::collections::BTreeMap::new();
for c in c {
m.insert((c.bx, c.by), c.grid);
}
m.retain(|&(bx, by), _| {
map.is_first_block(bx as usize, by as usize)
&& map.raw_strategy(bx as usize, by as usize) != STRATEGY_DCT
});
m
};
assert_eq!(
effective(&legacy.saved_children, &legacy_map),
effective(&leaf.saved_children, &leaf_map),
"{scope:?}"
);
assert!(
leaf.leaves
.iter()
.all(|l| l.strategy == STRATEGY_DCT && l.gain == 0.0)
);
}
}
#[test]
fn leaf_first_band_lies_inside_the_sub8_gate() {
assert!(std::hint::black_box(LEAF_FIRST_MAX_DISTANCE) <= SUB8_MAX_DISTANCE);
}
#[test]
fn fine_admission_charges_the_fitted_correction() {
let bits = std::hint::black_box(FINE_ADMIT_RATE_CORRECTION_BITS);
assert!((16.0..=32.0).contains(&bits), "bits = {bits}");
assert!(fine_mosaic_lambda(0.5) >= crate::ac_strategy::RD_LAMBDA);
}
#[test]
fn leaf_first_commits_a_valid_map_with_sub8_leaves() {
for threads in [1usize, 3] {
let (map, benefit, rollbacks) = run_selector_full(crate::Speed::Slow, 1.0, threads);
assert_map_valid(&map);
let in_mosaic = |bx: usize, by: usize| {
rollbacks
.iter()
.any(|r| bx >= r.bx && bx < r.bx + r.cov_x && by >= r.by && by < r.by + r.cov_y)
};
let gated = map
.iter_first_blocks()
.filter(|&(bx, by, s)| {
crate::dc_group_data::is_gated_sub8_strategy(s) && !in_mosaic(bx, by)
})
.count();
assert!(benefit.is_finite());
assert!(
gated == 0 || benefit > 0.0,
"fine leaves survived without credit"
);
assert!(gated > 0 || benefit == 0.0, "credit without fine leaves");
}
}
#[test]
fn merge_upgrade_keeps_the_map_valid() {
for distance in [1.5f32, 2.0, 3.0, 6.0] {
let (map, _) = run_selector(crate::Speed::Slow, distance, 3);
assert_map_valid(&map);
}
}
#[test]
fn selection_is_independent_of_worker_count() {
for speed in [crate::Speed::Fast, crate::Speed::Medium, crate::Speed::Slow] {
for distance in [0.5, 1.5, 2.0, 4.0, 6.0] {
let (serial, _) = run_selector(speed, distance, 1);
let (parallel, _) = run_selector(speed, distance, 3);
assert_eq!(
strategy_cells(&serial),
strategy_cells(¶llel),
"{speed:?} d{distance}"
);
assert_map_valid(¶llel);
}
}
}
#[test]
fn upgrade_shortlist_survives_a_committed_square_and_is_order_independent() {
let ctx = EncodingContext::new(
crate::Speed::Slow,
crate::Arbiter::Ssimulacra2,
crate::xyb::XybMatrix::SPEC,
3.0,
1,
);
let opsin = Image3F::new(16, 16);
let maps = ImageSB::new_fill(1, 1, 0);
let qf = ImageB::new_fill(2, 2, 6);
let params = AcStrategyParams {
ctx: &ctx,
opsin: &opsin,
dc_group_px: 0,
dc_group_py: 0,
distance: 3.0,
scale: 1.0,
qm_mult_x: 1.0,
ytox_map: &maps,
ytob_map: &maps,
prices: None,
};
let rerank = super::RerankContext {
params,
quant_field: &qf,
bands: &[(0, 2)],
num_threads: 1,
leaf_first: false,
leaves: &[],
};
let mut map = AcStrategyImage::new(2, 2);
map.set_first(0, 0, STRATEGY_DCT16X16);
let mut output = AcStrategyBandScratch::default();
super::record_upgrade_candidates(
&mut output.upgrade_candidates,
&map,
0,
0,
super::UpgradeBand {
raw: [1.0; 5],
incumbent: [2.0; 5],
},
);
assert_eq!(output.upgrade_candidates.len(), 5);
let map = AcStrategyImage::new(2, 2);
let mut scratch = CoderScratch::default();
super::find_merge_upgrades(&rerank, &mut scratch, &map, (0, 2), &mut output, 1.0);
let chosen = |o: &AcStrategyBandScratch| {
o.merge_upgrades
.iter()
.map(|u| (u.bx, u.by, u.strategy, u.base.to_bits()))
.collect::<Vec<_>>()
};
let forward = chosen(&output);
assert!(!forward.is_empty());
output.upgrade_candidates.reverse();
super::find_merge_upgrades(&rerank, &mut scratch, &map, (0, 2), &mut output, 1.0);
assert_eq!(forward, chosen(&output));
}
#[test]
fn merge_upgrade_is_gated_and_knife_edge() {
let gate = std::hint::black_box(MERGE_UPGRADE_MIN_DISTANCE);
let margin = std::hint::black_box(MERGE_UPGRADE_MARGIN);
assert!((1.5..=2.5).contains(&gate), "gate {gate}");
assert!((0.95..=1.05).contains(&margin), "margin {margin}");
assert_eq!(merge_upgrade_margin(gate), margin);
let low = merge_upgrade_margin(6.0);
assert!(low > margin && low <= 1.1, "low margin {low}");
assert_eq!(low, merge_upgrade_margin(4.0));
assert!(merge_upgrade_margin(3.0) > margin && merge_upgrade_margin(3.0) < low);
}
#[test]
fn leaf_first_saved_children_match_their_transform_footprints() {
let mut seen_rect_halves = 0;
let mut histogram = [0usize; crate::dc_group_data::NUM_STRATEGIES];
for (distance, pattern) in [
(1.0f32, 0u32),
(2.5, 0),
(4.0, 0),
(6.0, 0),
(2.5, 1),
(4.0, 1),
(2.5, 2),
(4.0, 2),
] {
let ctx = EncodingContext::new(
crate::Speed::Slow,
crate::Arbiter::Ssimulacra2,
crate::xyb::XybMatrix::SPEC,
distance,
1,
);
let (bw, bh) = (12usize, 12usize);
let mut opsin = noise_opsin(bw * 8, bh * 8, 0x1234_abcd);
if pattern != 0 {
for c in 0..3 {
for y in 0..bh * 8 {
for (x, v) in opsin.plane_mut(c).row_mut(y).iter_mut().enumerate() {
let t = if pattern == 1 { y } else { x } as f32;
*v = 0.2 * *v + 0.02 * (t * 0.35).sin();
}
}
}
}
let maps = ImageSB::new_fill(2, 2, 0);
let qf = ImageB::new_fill(bw, bh, 5);
let mut map = AcStrategyImage::new(bw, bh);
let mut scratch = CoderScratch::default();
let params = AcStrategyParams {
ctx: &ctx,
opsin: &opsin,
dc_group_px: 0,
dc_group_py: 0,
distance,
scale: 1.0,
qm_mult_x: 1.0,
ytox_map: &maps,
ytob_map: &maps,
prices: None,
};
let selection = SelectionContext {
params,
quant_field: &qf,
meta_r: META_R,
scope: SearchScope::FULL,
};
let mut band = AcStrategyBandScratch::default();
leaf_first::select_band_leaf_first(
&selection,
&mut scratch,
&mut map,
(0, bh),
&mut band,
);
for (_, _, s) in map.iter_first_blocks() {
histogram[s as usize] += 1;
}
for child in &band.saved_children {
let (bx, by) = (child.bx as usize, child.by as usize);
assert!(map.is_first_block(bx, by), "saved child at a covered cell");
let s = map.raw_strategy(bx, by);
let cx = AcStrategyImage::covered_blocks_x_of(s);
let cy = AcStrategyImage::covered_blocks_y_of(s);
assert!(cx * cy > 1, "saved child under a single block");
if matches!(s, STRATEGY_DCT32X16 | STRATEGY_DCT16X32) {
seen_rect_halves += 1;
}
for iy in 0..4 {
for ix in 0..4 {
let g = child.grid[iy * 4 + ix];
assert!(
g == NO_CHILD_BLOCK || (ix < cx && iy < cy),
"child grid cell ({ix},{iy}) outside a {cx}x{cy} footprint at ({bx},{by})"
);
}
}
}
}
let _ = (seen_rect_halves, histogram);
}
#[test]
fn bundled_sub8_costs_match_independent_evaluation_and_cached_dct8() {
let mut opsin = Image3F::new(11, 13);
let mut state = 0x1234_5678u32;
for c in 0..3 {
for y in 0..opsin.ysize() {
for value in opsin.plane_mut(c).row_mut(y) {
state = state.wrapping_mul(1_664_525).wrapping_add(1_013_904_223);
*value = (state >> 8) as f32 / (1u32 << 24) as f32 - 0.5;
}
}
}
let ctx = EncodingContext::default();
let (px, py) = (7usize, 9usize); let qac = 3.75;
let qm_mult_x = 1.25;
let meta_r = 0.4;
let distance = 1.5;
let cmap = [0.25, 0.0, -0.375];
let mut scratch = CoderScratch::default();
let independent = [
strategy_cost(
&ctx,
&mut scratch,
STRATEGY_DCT,
&opsin,
px,
py,
qac,
qm_mult_x,
meta_r,
distance,
cmap,
),
strategy_cost(
&ctx,
&mut scratch,
STRATEGY_IDENTITY,
&opsin,
px,
py,
qac,
qm_mult_x,
meta_r,
distance,
cmap,
),
strategy_cost(
&ctx,
&mut scratch,
STRATEGY_DCT2X2,
&opsin,
px,
py,
qac,
qm_mult_x,
meta_r,
distance,
cmap,
),
strategy_cost(
&ctx,
&mut scratch,
STRATEGY_DCT4X4,
&opsin,
px,
py,
qac,
qm_mult_x,
meta_r,
distance,
cmap,
),
strategy_cost(
&ctx,
&mut scratch,
STRATEGY_DCT4X8,
&opsin,
px,
py,
qac,
qm_mult_x,
meta_r,
distance,
cmap,
),
strategy_cost(
&ctx,
&mut scratch,
STRATEGY_DCT8X4,
&opsin,
px,
py,
qac,
qm_mult_x,
meta_r,
distance,
cmap,
),
];
for cached in [None, Some(independent[0])] {
let bundled = sub8_strategy_costs(
&ctx,
&mut scratch,
&opsin,
px,
py,
qac,
qm_mult_x,
meta_r,
distance,
cmap,
cached,
true,
true,
);
let actual = [
bundled.dct8,
bundled.identity,
bundled.dct2x2,
bundled.dct4x4,
bundled.dct4x8,
bundled.dct8x4,
];
for (a, b) in actual.into_iter().zip(independent) {
assert_eq!(a.to_bits(), b.to_bits());
}
}
}
#[test]
fn reconstruction_round_trips() {
let idct = crate::dct::IdctMethods::scalar();
for strategy in [
STRATEGY_DCT,
STRATEGY_IDENTITY,
STRATEGY_DCT2X2,
STRATEGY_DCT4X4,
STRATEGY_DCT4X8,
STRATEGY_DCT8X4,
crate::ac_strategy::STRATEGY_AFV0,
crate::ac_strategy::STRATEGY_AFV1,
crate::ac_strategy::STRATEGY_AFV2,
crate::ac_strategy::STRATEGY_AFV3,
STRATEGY_DCT16X8,
STRATEGY_DCT16X16,
STRATEGY_DCT32X32,
STRATEGY_DCT64X64,
STRATEGY_DCT64X32,
STRATEGY_DCT32X64,
] {
let n = strategy_pixel_count(strategy);
let mut x = vec![0.0f32; n];
let mut s = 12345u32;
for v in &mut x {
s = s.wrapping_mul(1664525).wrapping_add(1013904223);
*v = (s >> 8) as f32 / (1u32 << 24) as f32 - 0.5;
}
let mut c = vec![0.0f32; n];
forward_for(strategy, &x, &mut c);
let mut recon = vec![0.0f32; n];
reconstruct_error(&idct, strategy, &c, &mut recon);
let max_err = (0..n).map(|i| (recon[i] - x[i]).abs()).fold(0.0, f32::max);
assert!(
max_err < 1e-3,
"strategy {strategy}: max reconstruction err {max_err}"
);
}
}
#[test]
fn forward_matrix_is_orthogonal() {
let n = 64;
let ft = forward_matrix(STRATEGY_DCT); let mut max_off = 0.0f32;
for j in 0..8 {
for k in 0..8 {
if j == k {
continue;
}
let dot: f32 = (0..n).map(|p| ft[p * n + j] * ft[p * n + k]).sum();
max_off = max_off.max(dot.abs());
}
}
println!("max off-diagonal |<f_j,f_k>| = {max_off}");
assert!(max_off < 1e-4, "DCT8 basis not orthogonal: {max_off}");
}
#[test]
fn merge_guard_is_stricter_for_large_transforms_and_high_quality() {
assert!(merge_margin(0.3, MERGE_MARGIN_32) > merge_margin(0.3, MERGE_MARGIN_16));
assert!(merge_margin(0.3, MERGE_MARGIN_16) > merge_margin(0.3, MERGE_MARGIN_PAIR));
assert!(merge_margin(0.3, MERGE_MARGIN_16) > merge_margin(4.0, MERGE_MARGIN_16));
let hq = MergeTuning::new(0.3, crate::Arbiter::Ssimulacra2);
let lowq = MergeTuning::new(4.0, crate::Arbiter::Ssimulacra2);
assert!(!merge_beats_dct8(90.0, 100.0, hq.accept_16));
assert!(merge_beats_dct8(75.0, 100.0, hq.accept_16));
assert!(merge_beats_dct8(95.0, 100.0, lowq.accept_16));
assert_eq!(hq.accept_16, 1.0 - merge_margin(0.3, MERGE_MARGIN_16));
assert_eq!(hq.bias_rect32, BIAS_RECT32.at(0.3));
assert_eq!(lowq.rerank_margin, 1.0);
}
#[test]
fn quant_aggregation_uses_max_for_pairs_and_high_quality() {
assert_eq!(aggregate_quant(40, 50, 2, 1.0), 40);
assert_eq!(aggregate_quant(40, 100, 4, 1.0), 40);
}
#[test]
fn quant_aggregation_matches_scaled_candidate_cost() {
let qac = [[2.5, 5.0], [7.5, 10.0]];
let raw = aggregate_quant(40, 100, 4, 2.0);
assert_eq!(aggregate_qac_2x2(qac, 0.25, 2.0), raw as f32 * 0.25);
}
#[test]
fn strategy_cfl_factors_match_signaled_tile_maps() {
let ytox = ImageSB::new_fill(1, 1, 42);
let ytob = ImageSB::new_fill(1, 1, -42);
assert_eq!(
cmap_factors(crate::color_correlation::CflFrame::XYB, &ytox, &ytob, 0, 0),
[0.5, 0.0, 0.5]
);
}
#[test]
fn luma_only_gradient_stats_match_full_stats() {
let mut opsin = Image3F::new(16, 16);
for c in 0..3 {
for y in 0..16 {
for x in 0..16 {
opsin.plane_mut(c).row_mut(y)[x] =
0.01 * x as f32 + 0.02 * y as f32 + 0.1 * c as f32;
}
}
}
let luma = gradient_region_stats_scalar(&opsin, 0, 0, 16, 8, 1e-5);
let full = gradient_region_stats_with_chroma_scalar(&opsin, 0, 0, 16, 8, 1e-5);
assert_eq!(luma.dominance, full.dominance);
assert_eq!(luma.mean, full.mean);
assert!(full.chroma > 0.0);
}
#[test]
fn dispatched_gradient_stats_match_scalar() {
let mut opsin = Image3F::new(24, 24);
for c in 0..3 {
for y in 0..24 {
for x in 0..24 {
let hash = (x * 37 + y * 61 + c * 17 + x * y * 3) % 101;
opsin.plane_mut(c).row_mut(y)[x] = hash as f32 * 0.007 - 0.2 + c as f32 * 0.03;
}
}
}
let kernels = [
(
select_gradient_region_stats_fn(),
gradient_region_stats_scalar as super::GradientRegionStatsFn,
),
(
select_gradient_region_stats_with_chroma_fn(),
gradient_region_stats_with_chroma_scalar as super::GradientRegionStatsFn,
),
];
for &(simd, scalar) in &kernels {
for &(px, py, w, h) in &[(1, 2, 8, 16), (3, 4, 16, 8), (20, 21, 8, 16)] {
let got = simd(&opsin, px, py, w, h, 1e-5);
let expected = scalar(&opsin, px, py, w, h, 1e-5);
assert!((got.dominance - expected.dominance).abs() < 2e-5);
assert!((got.mean - expected.mean).abs() < 2e-6);
assert!((got.chroma - expected.chroma).abs() < 2e-6);
}
}
}
}