use crate::adaptive_quant::fast_exp2;
use crate::coder_scratch::{
AcStrategyBandScratch, CachedQuantCost, CoderScratch, FineMergeRollback, MergeUpgrade,
MergeUpgradeCandidate, QuantRefinement, RerankDowngrade,
};
use crate::dc_group_data::{
AcStrategyImage, STRATEGY_AFV0, STRATEGY_AFV1, STRATEGY_AFV2, STRATEGY_AFV3, STRATEGY_DCT,
STRATEGY_DCT2X2, STRATEGY_DCT4X4, STRATEGY_DCT4X8, STRATEGY_DCT8X4, STRATEGY_DCT8X16,
STRATEGY_DCT16X8, STRATEGY_DCT16X16, STRATEGY_DCT16X32, STRATEGY_DCT32X16, STRATEGY_DCT32X32,
STRATEGY_DCT32X64, STRATEGY_DCT64X32, STRATEGY_DCT64X64, STRATEGY_IDENTITY,
};
use crate::dct::{DctInput, fmla};
use crate::encoding_context::EncodingContext;
use crate::image::{Image3F, ImageB, ImageSB};
use crate::inflated_cost::{
Footprint, ReconCost, ReconDistInput, ReconQuantization, ReconScoring, ReconSource,
ReconTransform, channel_rd,
};
mod matrix_overhead;
mod rate_prices;
mod selection;
pub(crate) use matrix_overhead::account_matrix_headers;
pub(crate) use rate_prices::RatePrices;
use rate_prices::RatePricesScope;
pub(crate) use selection::{
Chosen32Cost, FineMosaicScratch, LeafChoice, SavedChild, aggregate_quant, fill_ac_strategy,
};
const DCT8_ONLY_MAX_DISTANCE: f32 = 0.056_713_393;
const FAST_RERANK_MAX_DISTANCE: f32 = 1.7090991223128462;
#[inline]
fn use_dct8_only(distance: f32) -> bool {
distance <= DCT8_ONLY_MAX_DISTANCE
}
#[inline]
fn use_dct8_only_for_content(ctx: &EncodingContext, distance: f32) -> bool {
use_dct8_only(distance) && !ctx.point_chroma()
}
const MERGE_BAND_D0: f32 = 0.5;
const MERGE_BAND_D1: f32 = 1.5;
#[derive(Clone, Copy)]
struct Banded {
hq: f32,
base: f32,
}
impl Banded {
const fn new(hq: f32, base: f32) -> Self {
Self { hq, base }
}
#[inline]
fn at(self, distance: f32) -> f32 {
let t = ((distance - MERGE_BAND_D0) / (MERGE_BAND_D1 - MERGE_BAND_D0)).clamp(0.0, 1.0);
fmla(t, self.base - self.hq, self.hq)
}
}
const META_R: f32 = 4.0;
pub(crate) const RD_LAMBDA: f32 = 0.080_867_17;
const MERGE_PRESCREEN_BITS: f32 = 600.0;
const RERANK_META_R: f32 = 2.6465739748323758;
const RERANK_DOWNGRADE_MARGIN: Banded = Banded::new(1.0, 0.891_814_53);
const RERANK_LAMBDA_LO: f32 = 0.25;
const RERANK_LAMBDA_HI: f32 = 10.0;
const RERANK_LAMBDA_D0: f32 = 1.0;
const RERANK_LAMBDA_D1: f32 = 4.0;
const BIAS_RECT: Banded = Banded::new(1.026_200_7, 0.915);
const BIAS_16X16: Banded = Banded::new(1.205_927_5, 1.05);
const BIAS_32X32: Banded = Banded::new(1.205_927_5, 1.06);
const BIAS_32X32_LOOSEN: f32 = 0.94;
const BIAS_RECT32: Banded = Banded::new(1.326_520_2, 1.02);
const MERGE_MARGIN_PAIR: Banded = Banded::new(0.014_500_806, 0.0288);
const MERGE_MARGIN_16: Banded = Banded::new(0.192_438_88, 0.048);
const MERGE_MARGIN_32_RECT: Banded = Banded::new(0.161_916_72, 0.29);
const MERGE_MARGIN_32: Banded = Banded::new(0.213_985_08, 0.234);
const MERGE_MARGIN_LOWQ_FRACTION: f32 = 0.82;
const MERGE_MARGIN_LOWQ3_D: f32 = 3.85;
const MERGE_MARGIN_LOWQ3_SCALE: f32 = 1.10;
const MERGE_MARGIN_FADE_START: f32 = 1.0;
const MERGE_MARGIN_FADE_END: f32 = 4.0;
const BIAS_4X4: f32 = 1.33;
const BIAS_4X8: f32 = 1.09;
const BIAS_AFV: Banded = Banded::new(1.15, 1.25);
const SUB8_MAX_DISTANCE: f32 = 1.5;
const AFV_MAX_DISTANCE: f32 = SUB8_MAX_DISTANCE;
const FINE_TRANSFORM_MAX_DISTANCE: f32 = 5.0;
const FINE_RECON_MARGIN: f32 = 0.98;
const FINE_ADMIT_RATE_CORRECTION_BITS: f32 = 24.0;
const FINE_ADMIT_RATE_CORRECTION_SHARE: f32 = 0.1;
const FINE_FAVOR_FULL_DISTANCE: f32 = 1.75;
#[inline]
fn fine_transform_bias(base: f32, distance: f32) -> f32 {
let favor = if distance < 5.0 {
let t = (5.0 - distance) * 0.2;
0.4 * t * t * (distance / FINE_FAVOR_FULL_DISTANCE).clamp(0.0, 1.0)
} else {
0.0
};
base / 0.8 - favor
}
#[inline]
fn merge_margin(distance: f32, margin: Banded) -> f32 {
let fade = ((distance - MERGE_MARGIN_FADE_START)
/ (MERGE_MARGIN_FADE_END - MERGE_MARGIN_FADE_START))
.clamp(0.0, 1.0);
let lowq3 = if distance >= MERGE_MARGIN_LOWQ3_D {
MERGE_MARGIN_LOWQ3_SCALE
} else {
1.0
};
margin.at(distance) * (1.0 - fade * (1.0 - MERGE_MARGIN_LOWQ_FRACTION)) * lowq3
}
#[inline]
fn merge_beats_dct8(candidate_cost: f32, dct8_cost: f32, accept: f32) -> bool {
candidate_cost < dct8_cost * accept
}
#[derive(Clone, Copy)]
pub(crate) struct MergeTuning {
pub(crate) bias_rect: f32,
pub(crate) bias_16x16: f32,
pub(crate) bias_32x32: f32,
pub(crate) bias_rect32: f32,
pub(crate) accept_pair: f32,
pub(crate) accept_16: f32,
pub(crate) accept_32_rect: f32,
pub(crate) accept_32: f32,
pub(crate) rerank_margin: f32,
pub(crate) risk_k: f32,
pub(crate) accept_64: f32,
pub(crate) accept_64_rect: f32,
}
const VLQ_D0: f32 = 4.0;
const VLQ_D1: f32 = 7.5;
const VLQ_MARGIN_PAIR: f32 = -0.002;
const VLQ_MARGIN_16: f32 = 0.067;
const VLQ_MARGIN_32_RECT: f32 = 0.363;
const VLQ_MARGIN_32: f32 = 0.029_25;
const VLQ_BIAS_RECT: f32 = 1.073;
const VLQ_BIAS_16X16: f32 = 0.880;
const VLQ_BIAS_32X32: f32 = 0.867;
const VLQ_BIAS_RECT32: f32 = 1.180;
const VLQ_RERANK_MARGIN: f32 = 0.7384;
const VLQ_ACCEPT_64: f32 = 0.973;
const VLQ_ACCEPT_64_RECT: f32 = 0.736;
impl MergeTuning {
pub(crate) fn new(distance: f32) -> Self {
let accept = |margin: Banded| 1.0 - merge_margin(distance, margin);
let mut tuning = Self {
bias_rect: BIAS_RECT.at(distance),
bias_16x16: BIAS_16X16.at(distance),
bias_32x32: BIAS_32X32.at(distance),
bias_rect32: BIAS_RECT32.at(distance),
accept_pair: accept(MERGE_MARGIN_PAIR),
accept_16: accept(MERGE_MARGIN_16),
accept_32_rect: accept(MERGE_MARGIN_32_RECT),
accept_32: accept(MERGE_MARGIN_32),
rerank_margin: RERANK_DOWNGRADE_MARGIN.at(distance),
risk_k: if crate::adaptive_quant::aq_dampen(distance) > 0.0 {
MERGE_RISK_K
} else {
0.0
},
accept_64: dct64_accept(),
accept_64_rect: dct64_rect_accept(),
};
let vlq_t = ((distance - VLQ_D0) / (VLQ_D1 - VLQ_D0)).clamp(0.0, 1.0);
if vlq_t > 0.0 {
let lerp = |cur: f32, vlq: f32| fmla(vlq_t, vlq - cur, cur);
tuning.bias_rect = lerp(tuning.bias_rect, VLQ_BIAS_RECT);
tuning.bias_16x16 = lerp(tuning.bias_16x16, VLQ_BIAS_16X16);
tuning.bias_32x32 = lerp(tuning.bias_32x32, VLQ_BIAS_32X32);
tuning.bias_rect32 = lerp(tuning.bias_rect32, VLQ_BIAS_RECT32);
tuning.accept_pair = lerp(tuning.accept_pair, 1.0 - VLQ_MARGIN_PAIR);
tuning.accept_16 = lerp(tuning.accept_16, 1.0 - VLQ_MARGIN_16);
tuning.accept_32_rect = lerp(tuning.accept_32_rect, 1.0 - VLQ_MARGIN_32_RECT);
tuning.accept_32 = lerp(tuning.accept_32, 1.0 - VLQ_MARGIN_32);
tuning.rerank_margin = lerp(tuning.rerank_margin, VLQ_RERANK_MARGIN);
tuning.accept_64 = lerp(tuning.accept_64, VLQ_ACCEPT_64);
tuning.accept_64_rect = lerp(tuning.accept_64_rect, VLQ_ACCEPT_64_RECT);
}
if distance >= MERGE_BAND_D1 {
tuning.bias_32x32 *= BIAS_32X32_LOOSEN;
}
tuning
}
}
#[derive(Clone, Copy, PartialEq, Eq, Debug)]
pub(crate) struct SearchScope {
rectangles: bool,
full_rerank: bool,
merge_32: bool,
busy_quads: bool,
}
impl SearchScope {
#[cfg(test)]
pub(crate) const SQUARES: Self = Self {
rectangles: false,
full_rerank: false,
merge_32: true,
busy_quads: true,
};
#[cfg(test)]
pub(crate) const FULL: Self = Self {
rectangles: true,
full_rerank: true,
merge_32: true,
busy_quads: true,
};
#[inline]
fn for_speed(speed: crate::Speed) -> Self {
let effort = speed.effort();
Self {
rectangles: effort.rectangles,
full_rerank: effort.full_rerank,
merge_32: effort.merge_32,
busy_quads: effort.merge_busy_quads,
}
}
#[inline]
fn rectangles(self) -> bool {
self.rectangles
}
#[inline]
fn merge_32(self) -> bool {
self.merge_32
}
#[inline]
fn merge_busy_quads(self) -> bool {
self.busy_quads
}
#[inline]
fn rerank(self, distance: f32) -> bool {
self.full_rerank || distance <= FAST_RERANK_MAX_DISTANCE
}
}
pub(crate) struct RateCalibration;
impl RateCalibration {
const TABLE: [[f32; 3]; 3] = [[1.0, 0.96, 0.96], [1.0, 0.93, 0.93], [1.0, 0.91, 0.91]];
#[inline]
fn family(strategy: u8) -> Option<usize> {
Some(match strategy {
STRATEGY_DCT16X8 | STRATEGY_DCT8X16 => 0,
STRATEGY_DCT16X16 => 1,
STRATEGY_DCT32X32 | STRATEGY_DCT32X16 | STRATEGY_DCT16X32 => 2,
_ => return None,
})
}
#[inline]
pub(crate) fn scale(strategy: u8, distance: f32) -> f32 {
let Some(f) = Self::family(strategy) else {
return 1.0;
};
let [hq, mid, low] = Self::TABLE[f];
if distance <= 1.5 {
hq
} else if distance <= 2.75 {
fmla((distance - 1.5) / 1.25, mid - hq, hq)
} else if distance <= 4.0 {
fmla((distance - 2.75) / 1.25, low - mid, mid)
} else {
low
}
}
}
const MERGE_UPGRADE_MARGIN: f32 = 1.0;
const MERGE_UPGRADE_MIN_DISTANCE: f32 = 2.0;
const MERGE_UPGRADE_LOW_MARGIN: f32 = 1.05;
const MERGE_UPGRADE_LOW_DISTANCE: f32 = 4.0;
#[inline]
fn merge_upgrade_margin(distance: f32) -> f32 {
let t = ((distance - MERGE_UPGRADE_MIN_DISTANCE)
/ (MERGE_UPGRADE_LOW_DISTANCE - MERGE_UPGRADE_MIN_DISTANCE))
.clamp(0.0, 1.0);
fmla(
t,
MERGE_UPGRADE_LOW_MARGIN - MERGE_UPGRADE_MARGIN,
MERGE_UPGRADE_MARGIN,
)
}
const LEAF_FIRST_MAX_DISTANCE: f32 = SUB8_MAX_DISTANCE;
const LEARNED_LEAF_FIRST_MAX_DISTANCE: f32 = FINE_TRANSFORM_MAX_DISTANCE;
#[derive(Clone, Copy)]
struct SuperBlockCost {
chosen: f32,
dct8: f32,
prescreened: bool,
}
#[inline]
pub(crate) fn gather_pixels(
plane: &crate::image::Plane<f32>,
px: usize,
py: usize,
width: usize,
height: usize,
dst: &mut [f32],
) {
let pw = plane.xsize();
let ph = plane.ysize();
if width <= pw.saturating_sub(px) && height <= ph.saturating_sub(py) {
for (v, dst) in dst.chunks_exact_mut(width).take(height).enumerate() {
dst.copy_from_slice(&plane.row(py + v)[px..px + width]);
}
return;
}
let safe_w = width.min(pw.saturating_sub(px));
let safe_h = height.min(ph.saturating_sub(py));
for v in 0..height {
let sy = if v < safe_h { py + v } else { ph - 1 };
let row = plane.row(sy);
let drow = &mut dst[v * width..v * width + width];
drow[..safe_w].copy_from_slice(&row[px..px + safe_w]);
if safe_w < width {
drow[safe_w..].fill(row[pw - 1]);
}
}
}
#[inline]
fn dct_input<'a, const W: usize, const H: usize>(
plane: &'a crate::image::Plane<f32>,
tmp: &'a mut [f32],
px: usize,
py: usize,
) -> DctInput<'a, W, H> {
if W <= plane.xsize().saturating_sub(px) && H <= plane.ysize().saturating_sub(py) {
let stride = plane.xsize();
DctInput::new(&plane.as_slice()[py * stride + px..], stride)
} else {
gather_pixels(plane, px, py, W, H, &mut tmp[..W * H]);
DctInput::new(&tmp[..W * H], W)
}
}
fn forward_transform(
ctx: &EncodingContext,
tmp: &mut [f32],
strategy: u8,
plane: &crate::image::Plane<f32>,
px: usize,
py: usize,
out: &mut [f32],
) -> (usize, usize) {
match strategy {
STRATEGY_DCT => {
let dst: &mut [f32; 64] = out.first_chunk_mut::<64>().unwrap();
(ctx.dct8x8)(dct_input(plane, tmp, px, py), dst);
(1, 1)
}
STRATEGY_IDENTITY => {
let dst: &mut [f32; 64] = out.first_chunk_mut::<64>().unwrap();
(ctx.identity8x8)(dct_input(plane, tmp, px, py), dst);
(1, 1)
}
STRATEGY_DCT2X2 => {
let dst: &mut [f32; 64] = out.first_chunk_mut::<64>().unwrap();
(ctx.dct2x2_8x8)(dct_input(plane, tmp, px, py), dst);
(1, 1)
}
STRATEGY_DCT16X8 => {
let dst: &mut [f32; 128] = out.first_chunk_mut::<128>().unwrap();
(ctx.dct16x8)(dct_input(plane, tmp, px, py), dst);
(2, 1)
}
STRATEGY_DCT8X16 => {
let dst: &mut [f32; 128] = out.first_chunk_mut::<128>().unwrap();
(ctx.dct8x16)(dct_input(plane, tmp, px, py), dst);
(2, 1)
}
STRATEGY_DCT16X16 => {
let dst: &mut [f32; 256] = out.first_chunk_mut::<256>().unwrap();
(ctx.dct16x16)(dct_input(plane, tmp, px, py), dst);
(2, 2)
}
STRATEGY_DCT32X32 => {
let dst: &mut [f32; 1024] = out.first_chunk_mut::<1024>().unwrap();
(ctx.dct32x32)(dct_input(plane, tmp, px, py), dst);
(4, 4)
}
STRATEGY_DCT32X16 => {
let dst: &mut [f32; 512] = out.first_chunk_mut::<512>().unwrap();
(ctx.dct32x16)(dct_input(plane, tmp, px, py), dst);
(4, 2)
}
STRATEGY_DCT16X32 => {
let dst: &mut [f32; 512] = out.first_chunk_mut::<512>().unwrap();
(ctx.dct16x32)(dct_input(plane, tmp, px, py), dst);
(4, 2)
}
STRATEGY_DCT64X64 => {
let dst = out.first_chunk_mut::<4096>().unwrap();
(ctx.dct64x64)(dct_input(plane, tmp, px, py), dst);
(8, 8)
}
STRATEGY_DCT64X32 => {
let dst = out.first_chunk_mut::<2048>().unwrap();
(ctx.dct64x32)(dct_input(plane, tmp, px, py), dst);
(8, 4)
}
STRATEGY_DCT32X64 => {
let dst = out.first_chunk_mut::<2048>().unwrap();
(ctx.dct32x64)(dct_input(plane, tmp, px, py), dst);
(8, 4)
}
STRATEGY_DCT4X4 => {
let dst: &mut [f32; 64] = out.first_chunk_mut::<64>().unwrap();
(ctx.dct4x4)(dct_input(plane, tmp, px, py), dst);
(1, 1)
}
STRATEGY_DCT4X8 => {
let dst: &mut [f32; 64] = out.first_chunk_mut::<64>().unwrap();
(ctx.dct4x8)(dct_input(plane, tmp, px, py), dst);
(1, 1)
}
STRATEGY_DCT8X4 => {
let dst: &mut [f32; 64] = out.first_chunk_mut::<64>().unwrap();
(ctx.dct8x4)(dct_input(plane, tmp, px, py), dst);
(1, 1)
}
STRATEGY_AFV0 => {
let dst: &mut [f32; 64] = out.first_chunk_mut::<64>().unwrap();
(ctx.afv0)(dct_input(plane, tmp, px, py), dst);
(1, 1)
}
STRATEGY_AFV1 => {
let dst: &mut [f32; 64] = out.first_chunk_mut::<64>().unwrap();
(ctx.afv1)(dct_input(plane, tmp, px, py), dst);
(1, 1)
}
STRATEGY_AFV2 => {
let dst: &mut [f32; 64] = out.first_chunk_mut::<64>().unwrap();
(ctx.afv2)(dct_input(plane, tmp, px, py), dst);
(1, 1)
}
STRATEGY_AFV3 => {
let dst: &mut [f32; 64] = out.first_chunk_mut::<64>().unwrap();
(ctx.afv3)(dct_input(plane, tmp, px, py), dst);
(1, 1)
}
_ => unreachable!("invalid strategy {strategy}"),
}
}
const DCT64_MIN_DISTANCE: f32 = 3.0;
const DCT64_WINDOW_LO: f32 = 1.0;
const DCT64_WINDOW_HI: f32 = 2.25;
const BIAS_64X64: f32 = 1.0;
const BIAS_64_RECT: f32 = 1.0;
const ACCEPT_64: f32 = 0.945;
const ACCEPT_64_RECT: f32 = 0.642;
const fn dct64_accept() -> f32 {
ACCEPT_64
}
const fn dct64_rect_accept() -> f32 {
ACCEPT_64_RECT
}
#[inline]
fn use_dct64(speed: crate::Speed, distance: f32) -> bool {
speed.effort().dct64
&& (distance >= DCT64_MIN_DISTANCE
|| (DCT64_WINDOW_LO..DCT64_WINDOW_HI).contains(&distance))
}
#[allow(clippy::too_many_arguments)]
fn strategy_cost64(
ctx: &EncodingContext,
scratch: &mut CoderScratch,
strategy: u8,
opsin: &Image3F,
px: usize,
py: usize,
qac: f32,
qm_mult_x: f32,
meta_r: f32,
distance: f32,
cmap_factor: [f32; 3],
) -> f32 {
let CoderScratch {
strategy_coeffs: coeffs,
transform_gather: input,
rate_prices,
..
} = scratch;
let (cx, cy, size) =
prepare_strategy_coeffs(ctx, coeffs, input, strategy, opsin, px, py, cmap_factor);
let visible = Footprint::of_strategy(opsin, px, py, strategy).visible_fraction();
if let Some((distortion, rate)) = rate_prices.as_deref().and_then(|prices| {
prices.coefficient_dist_and_rate(ctx, strategy, coeffs, qac, qm_mult_x, distance, cx, cy)
}) {
return distortion * visible + RD_LAMBDA * (rate + meta_r);
}
let mut distortion = 0.0f32;
let mut rate = 0.0f32;
for (c, coeff) in coeffs.iter().enumerate() {
let qm_mult = if c == 0 {
qm_mult_x
} else if c == 2 {
ctx.b_qm_mul()
} else {
1.0
};
let matrix: &[f32] = match strategy {
STRATEGY_DCT64X64 => &ctx.matrices().inv_matrix_64x64(c)[..],
STRATEGY_DCT64X32 | STRATEGY_DCT32X64 => &ctx.matrices().inv_matrix_64x32(c)[..],
_ => unreachable!(),
};
let (d, r) = channel_rd(
ctx.sse_and_rate,
ctx.rate_log2_lut,
&coeff[..size],
matrix,
c,
qac,
qm_mult,
distance,
cx,
cy,
);
distortion += ctx.channel_weight(c) * d;
rate += r;
}
distortion * visible + RD_LAMBDA * (rate + meta_r)
}
fn strategy_cost(
ctx: &EncodingContext,
scratch: &mut CoderScratch,
strategy: u8,
opsin: &Image3F,
px: usize,
py: usize,
qac: f32,
qm_mult_x: f32,
meta_r: f32,
distance: f32,
cmap_factor: [f32; 3],
) -> f32 {
strategy_cost_impl(
ctx,
scratch,
strategy,
opsin,
px,
py,
qac,
qm_mult_x,
meta_r,
distance,
cmap_factor,
DistortionModel::Coefficient,
)
}
#[derive(Clone, Copy)]
struct ReconStrategyCost {
cost: f32,
base: f32,
distortion: f32,
rate: f32,
hue_distortion: f32,
}
#[allow(clippy::too_many_arguments)]
fn reconstruction_strategy_cost_and_base(
ctx: &EncodingContext,
scratch: &mut CoderScratch,
strategy: u8,
opsin: &Image3F,
px: usize,
py: usize,
qac: f32,
qm_mult_x: f32,
meta_r: f32,
distance: f32,
cmap_factor: [f32; 3],
gradient_alpha: f32,
gradient_peak_alpha: f32,
spatial_errors: Option<[&mut [f32]; 3]>,
) -> ReconStrategyCost {
let CoderScratch {
strategy_coeffs: coeffs,
transform_gather,
recon,
rate_prices,
..
} = scratch;
let prices = rate_prices.as_deref();
let (cx, cy, _) = prepare_strategy_coeffs(
ctx,
coeffs,
transform_gather,
strategy,
opsin,
px,
py,
cmap_factor,
);
let keep_spatial_errors = spatial_errors.is_some();
let ReconCost {
distortion,
rate,
hue_distortion,
} = reconstruction_dist_and_rate(
ctx,
recon,
coeffs,
strategy,
opsin,
px,
py,
qac,
qm_mult_x,
distance,
cmap_factor,
cx,
cy,
gradient_alpha,
gradient_peak_alpha,
keep_spatial_errors,
prices,
);
if let Some(output) = spatial_errors {
let n = cx * cy * 64;
for (c, plane) in output.into_iter().enumerate() {
plane.copy_from_slice(&recon[c][..n]);
}
}
ReconStrategyCost {
cost: rd_cost(
DistortionModel::Reconstruction,
distance,
meta_r,
distortion,
rate,
),
base: if gradient_alpha == 0.0 && gradient_peak_alpha == 0.0 {
rd_cost(
DistortionModel::Reconstruction,
distance,
0.0,
distortion,
rate,
)
} else {
f32::NAN
},
distortion,
rate,
hue_distortion,
}
}
#[derive(Clone, Copy)]
enum DistortionModel {
Coefficient,
Reconstruction,
}
struct CflXyb<'a> {
x: &'a mut [f32],
y: &'a mut [f32],
b: &'a mut [f32],
}
pub(crate) type ApplyCflFn = fn(&mut [f32], &[f32], &mut [f32], [f32; 3]);
#[cfg(not(any(
all(target_arch = "aarch64", feature = "neon"),
all(target_arch = "wasm32", target_feature = "simd128", feature = "wasm")
)))]
fn apply_cfl_scalar(x: &mut [f32], y: &[f32], b: &mut [f32], cmap_factor: [f32; 3]) {
assert_eq!(x.len(), y.len());
assert_eq!(x.len(), b.len());
let neg_cx = -cmap_factor[0];
let neg_cb = -cmap_factor[2];
for ((x, b), &y) in x.iter_mut().zip(b).zip(y) {
*x = fmla(neg_cx, y, *x);
*b = fmla(neg_cb, y, *b);
}
}
pub(crate) fn selected_apply_cfl_fn() -> ApplyCflFn {
#[cfg(all(target_arch = "aarch64", feature = "neon"))]
{
|x, y, b, cmap_factor| unsafe { crate::neon::apply_cfl_neon(x, y, b, cmap_factor) }
}
#[cfg(all(target_arch = "x86_64", feature = "avx"))]
if is_x86_feature_detected!("avx2") && is_x86_feature_detected!("fma") {
return |x, y, b, cmap_factor| unsafe { crate::avx::apply_cfl_avx2(x, y, b, cmap_factor) };
}
#[cfg(all(target_arch = "wasm32", feature = "wasm", target_feature = "simd128"))]
return |x, y, b, cmap_factor| crate::wasm::apply_cfl_wasm(x, y, b, cmap_factor);
#[cfg(not(any(
all(target_arch = "aarch64", feature = "neon"),
all(target_arch = "wasm32", target_feature = "simd128", feature = "wasm")
)))]
apply_cfl_scalar
}
fn apply_cfl(ctx: &EncodingContext, coeffs: CflXyb<'_>, size: usize, cmap_factor: [f32; 3]) {
let CflXyb { x, y, b } = coeffs;
assert!(size <= x.len() && size <= y.len() && size <= b.len());
(ctx.apply_cfl)(&mut x[..size], &y[..size], &mut b[..size], cmap_factor);
}
#[inline]
fn inverse_matrix_for(ctx: &EncodingContext, strategy: u8, channel: usize) -> &[f32] {
let matrices = ctx.matrices();
match strategy {
STRATEGY_DCT => &matrices.inv_matrix(channel)[..],
STRATEGY_IDENTITY => &matrices.inv_matrix_identity(channel)[..],
STRATEGY_DCT2X2 => &matrices.inv_matrix_dct2x2(channel)[..],
STRATEGY_DCT4X4 => &matrices.inv_matrix_4x4(channel)[..],
STRATEGY_DCT4X8 | STRATEGY_DCT8X4 => &matrices.inv_matrix_4x8(channel)[..],
STRATEGY_AFV0..=STRATEGY_AFV3 => &matrices.inv_matrix_afv(channel)[..],
STRATEGY_DCT16X16 => &matrices.inv_matrix_16x16(channel)[..],
STRATEGY_DCT32X32 => &matrices.inv_matrix_32x32(channel)[..],
STRATEGY_DCT32X16 | STRATEGY_DCT16X32 => &matrices.inv_matrix_32x16(channel)[..],
_ => &matrices.inv_matrix_16x8(channel)[..],
}
}
#[allow(clippy::too_many_arguments)]
#[inline]
fn coefficient_dist_and_rate(
ctx: &EncodingContext,
strategy: u8,
coeffs: &[[f32; 4096]; 3],
size: usize,
qac: f32,
qm_mult_x: f32,
distance: f32,
cx: usize,
cy: usize,
prices: Option<&RatePrices>,
visible: f32,
) -> (f32, f32) {
if let Some((d, r)) = prices.and_then(|prices| {
prices.coefficient_dist_and_rate(ctx, strategy, coeffs, qac, qm_mult_x, distance, cx, cy)
}) {
return (d * visible, r);
}
let mut d_total = 0.0f32;
let mut r_total = 0.0f32;
for c in 0..3 {
let qm_mult = if c == 0 {
qm_mult_x
} else if c == 2 {
ctx.b_qm_mul()
} else {
1.0
};
let (d, r) = channel_rd(
ctx.sse_and_rate,
ctx.rate_log2_lut,
&coeffs[c][..size],
inverse_matrix_for(ctx, strategy, c),
c,
qac,
qm_mult,
distance,
cx,
cy,
);
d_total += ctx.channel_weight(c) * d;
r_total += r;
}
(
d_total * visible,
r_total * RateCalibration::scale(strategy, distance),
)
}
#[allow(clippy::too_many_arguments)]
fn strategy_cost_impl(
ctx: &EncodingContext,
scratch: &mut CoderScratch,
strategy: u8,
opsin: &Image3F,
px: usize,
py: usize,
qac: f32,
qm_mult_x: f32,
meta_r: f32,
distance: f32,
cmap_factor: [f32; 3],
distortion_model: DistortionModel,
) -> f32 {
let CoderScratch {
strategy_coeffs: coeffs,
transform_gather,
recon,
rate_prices,
..
} = scratch;
let prices = rate_prices.as_deref();
let (cx, cy, size) = prepare_strategy_coeffs(
ctx,
coeffs,
transform_gather,
strategy,
opsin,
px,
py,
cmap_factor,
);
let (d_total, r_total) = match distortion_model {
DistortionModel::Reconstruction => reconstruction_dist_and_rate(
ctx,
recon,
coeffs,
strategy,
opsin,
px,
py,
qac,
qm_mult_x,
distance,
cmap_factor,
cx,
cy,
0.0,
0.0,
false,
prices,
)
.dist_and_rate(),
DistortionModel::Coefficient => coefficient_dist_and_rate(
ctx,
strategy,
coeffs,
size,
qac,
qm_mult_x,
distance,
cx,
cy,
prices,
Footprint::of_strategy(opsin, px, py, strategy).visible_fraction(),
),
};
rd_cost(distortion_model, distance, meta_r, d_total, r_total)
}
#[allow(clippy::too_many_arguments)]
fn prepare_strategy_coeffs(
ctx: &EncodingContext,
coeffs: &mut [[f32; 4096]; 3],
transform_gather: &mut [f32; 4096],
strategy: u8,
opsin: &Image3F,
px: usize,
py: usize,
cmap_factor: [f32; 3],
) -> (usize, usize, usize) {
let mut cxy = (1usize, 1usize);
for c in 0..3 {
cxy = forward_transform(
ctx,
transform_gather,
strategy,
opsin.plane(c),
px,
py,
&mut coeffs[c],
);
}
let (cx, cy) = cxy;
let size = cx * cy * 64;
let [x, y, b] = coeffs;
apply_cfl(ctx, CflXyb { x, y, b }, size, cmap_factor);
(cx, cy, size)
}
#[allow(clippy::too_many_arguments)]
fn reconstruction_dist_and_rate(
ctx: &EncodingContext,
recon: &mut [[f32; 1024]; 8],
coeffs: &[[f32; 4096]; 3],
strategy: u8,
opsin: &Image3F,
px: usize,
py: usize,
qac: f32,
qm_mult_x: f32,
distance: f32,
cmap_factor: [f32; 3],
cx: usize,
cy: usize,
gradient_alpha: f32,
gradient_peak_alpha: f32,
keep_spatial_errors: bool,
prices: Option<&RatePrices>,
) -> ReconCost {
let mut cost = (ctx.recon_dist_and_rate)(
recon,
&ReconDistInput {
idct: ctx.idct,
quantization: ReconQuantization {
compute_rate: !prices.is_some_and(|prices| prices.has_table(strategy)),
rate_log2_lut: ctx.rate_log2_lut,
coeffs: [&coeffs[0], &coeffs[1], &coeffs[2]],
inverse_matrices: [
inverse_matrix_for(ctx, strategy, 0),
inverse_matrix_for(ctx, strategy, 1),
inverse_matrix_for(ctx, strategy, 2),
],
qac,
qm_mult_x,
qm_mult_b: ctx.b_qm_mul(),
distance,
},
transform: ReconTransform {
blocks_x: cx,
blocks_y: cy,
strategy,
},
source: ReconSource {
opsin,
x: px,
y: py,
},
scoring: ReconScoring {
factor_x: cmap_factor[0],
factor_b: cmap_factor[2],
channel_weights: ctx.channel_weights(),
xyb_matrix: ctx.xyb,
rgb_hue_alpha: rerank_rgb_hue_alpha(ctx, distance),
gradient_alpha,
gradient_peak_alpha,
keep_spatial_errors,
},
},
&ctx.recon_error_kernels,
);
cost.rate = prices
.and_then(|prices| {
prices.rate(
ctx, strategy, coeffs, [false; 3], qac, qm_mult_x, distance, cx, cy,
)
})
.unwrap_or_else(|| cost.rate * RateCalibration::scale(strategy, distance));
cost
}
const RERANK_PAIR_GRADIENT_FADE_IN_START: f32 = 0.5;
const RERANK_PAIR_GRADIENT_FADE_IN_END: f32 = 0.8;
const RERANK_PAIR_GRADIENT_FADE_OUT_START: f32 = 1.0;
const RERANK_PAIR_GRADIENT_FADE_OUT_END: f32 = 1.5;
const RERANK_PAIR_GRADIENT_ALPHA: f32 = 8.0;
const RERANK_PAIR_GRADIENT_MIN_DOMINANCE: f32 = 0.4;
const RERANK_PAIR_GRADIENT_PEAK_FADE_IN_START: f32 = 1.2;
const RERANK_PAIR_GRADIENT_PEAK_FADE_IN_END: f32 = 1.8;
const RERANK_PAIR_GRADIENT_PEAK_FADE_OUT_START: f32 = 3.0;
const RERANK_PAIR_GRADIENT_PEAK_FADE_OUT_END: f32 = 3.5;
const RERANK_PAIR_GRADIENT_PEAK_COARSE_START: f32 = 1.9;
const RERANK_PAIR_GRADIENT_PEAK_COARSE_END: f32 = 2.0;
const RERANK_PAIR_GRADIENT_PEAK_ALPHA: f32 = 192.0;
const RERANK_PAIR_GRADIENT_PEAK_COARSE_ALPHA: f32 = 96.0;
const RERANK_PAIR_GRADIENT_PEAK_MIN_DOMINANCE: f32 = 0.4;
const RERANK_PAIR_GRADIENT_PEAK_MIN_LUMA: f32 = 0.45;
const RERANK_PAIR_GRADIENT_PEAK_MAX_COARSE_CHROMA: f32 = 0.01;
const RERANK_RGB_HUE_ALPHA: f32 = 800.0;
fn rerank_rgb_hue_alpha(ctx: &EncodingContext, distance: f32) -> f32 {
if !ctx.x_heavy() {
return 0.0;
}
let fade_in = ((distance - 0.35) / 0.40).clamp(0.0, 1.0);
RERANK_RGB_HUE_ALPHA * fade_in
}
#[inline]
fn rerank_pair_gradient_scale(distance: f32) -> f32 {
let fade_in = ((distance - RERANK_PAIR_GRADIENT_FADE_IN_START)
/ (RERANK_PAIR_GRADIENT_FADE_IN_END - RERANK_PAIR_GRADIENT_FADE_IN_START))
.clamp(0.0, 1.0);
let fade_out = 1.0
- ((distance - RERANK_PAIR_GRADIENT_FADE_OUT_START)
/ (RERANK_PAIR_GRADIENT_FADE_OUT_END - RERANK_PAIR_GRADIENT_FADE_OUT_START))
.clamp(0.0, 1.0);
fade_in * fade_out
}
fn rerank_pair_gradient_alpha(distance: f32) -> f32 {
RERANK_PAIR_GRADIENT_ALPHA * rerank_pair_gradient_scale(distance)
}
#[inline]
fn rerank_pair_gradient_peak_alpha(distance: f32) -> f32 {
let fade_in = ((distance - RERANK_PAIR_GRADIENT_PEAK_FADE_IN_START)
/ (RERANK_PAIR_GRADIENT_PEAK_FADE_IN_END - RERANK_PAIR_GRADIENT_PEAK_FADE_IN_START))
.clamp(0.0, 1.0);
let fade_out = 1.0
- ((distance - RERANK_PAIR_GRADIENT_PEAK_FADE_OUT_START)
/ (RERANK_PAIR_GRADIENT_PEAK_FADE_OUT_END - RERANK_PAIR_GRADIENT_PEAK_FADE_OUT_START))
.clamp(0.0, 1.0);
let coarse_mix = ((distance - RERANK_PAIR_GRADIENT_PEAK_COARSE_START)
/ (RERANK_PAIR_GRADIENT_PEAK_COARSE_END - RERANK_PAIR_GRADIENT_PEAK_COARSE_START))
.clamp(0.0, 1.0);
let fit = &PAIR_GRAD_FIT;
let alpha = fmla(
coarse_mix,
fit.peak_coarse_alpha - fit.peak_alpha,
fit.peak_alpha,
);
alpha * fade_in * fade_out
}
struct PairGradFit {
peak_alpha: f32,
peak_coarse_alpha: f32,
min_luma: f32,
chroma_cap: f32,
peak_min_dominance: f32,
}
static PAIR_GRAD_FIT: PairGradFit = PairGradFit {
peak_alpha: RERANK_PAIR_GRADIENT_PEAK_ALPHA,
peak_coarse_alpha: RERANK_PAIR_GRADIENT_PEAK_COARSE_ALPHA,
min_luma: RERANK_PAIR_GRADIENT_PEAK_MIN_LUMA,
chroma_cap: RERANK_PAIR_GRADIENT_PEAK_MAX_COARSE_CHROMA,
peak_min_dominance: RERANK_PAIR_GRADIENT_PEAK_MIN_DOMINANCE,
};
#[inline]
fn reconstruction_lambda(distance: f32) -> f32 {
const RECIP_RERANKING: f32 = 1. / (RERANK_LAMBDA_D1 - RERANK_LAMBDA_D0);
let ramp = ((distance - RERANK_LAMBDA_D0) * RECIP_RERANKING).clamp(0.0, 1.0);
let multiplier = fmla(ramp, RERANK_LAMBDA_HI - RERANK_LAMBDA_LO, RERANK_LAMBDA_LO);
RD_LAMBDA * multiplier
}
#[inline]
pub(crate) fn fine_mosaic_lambda(distance: f32) -> f32 {
reconstruction_lambda(distance).max(RD_LAMBDA)
}
#[inline]
fn rd_cost(
distortion_model: DistortionModel,
distance: f32,
meta_r: f32,
d_total: f32,
r_total: f32,
) -> f32 {
let lam = match distortion_model {
DistortionModel::Coefficient => RD_LAMBDA,
DistortionModel::Reconstruction => reconstruction_lambda(distance),
};
fmla(lam, r_total + meta_r, d_total)
}
#[derive(Clone, Copy)]
struct Sub8Costs {
dct8: f32,
identity: f32,
dct2x2: f32,
dct4x4: f32,
dct4x8: f32,
dct8x4: f32,
afv: [f32; 4],
}
#[inline]
fn forward_sub8_transform(
ctx: &EncodingContext,
strategy: u8,
input: &[f32; 64],
output: &mut [f32],
) {
let dst: &mut [f32; 64] = output.first_chunk_mut::<64>().unwrap();
let input = DctInput::from_flat(input);
match strategy {
STRATEGY_DCT => (ctx.dct8x8)(input, dst),
STRATEGY_IDENTITY => (ctx.identity8x8)(input, dst),
STRATEGY_DCT2X2 => (ctx.dct2x2_8x8)(input, dst),
STRATEGY_DCT4X4 => (ctx.dct4x4)(input, dst),
STRATEGY_DCT4X8 => (ctx.dct4x8)(input, dst),
STRATEGY_DCT8X4 => (ctx.dct8x4)(input, dst),
STRATEGY_AFV0 => (ctx.afv0)(input, dst),
STRATEGY_AFV1 => (ctx.afv1)(input, dst),
STRATEGY_AFV2 => (ctx.afv2)(input, dst),
STRATEGY_AFV3 => (ctx.afv3)(input, dst),
_ => unreachable!("non-sub8 strategy {strategy}"),
}
}
#[allow(clippy::too_many_arguments)]
fn sub8_strategy_costs(
ctx: &EncodingContext,
scratch: &mut CoderScratch,
opsin: &Image3F,
px: usize,
py: usize,
qac: f32,
qm_mult_x: f32,
meta_r: f32,
distance: f32,
cmap_factor: [f32; 3],
cached_dct8: Option<f32>,
with_dct4: bool,
with_fine: bool,
) -> Sub8Costs {
let mut pixels = [[0.0f32; 64]; 3];
for (c, input) in pixels.iter_mut().enumerate() {
gather_pixels(opsin.plane(c), px, py, 8, 8, input);
}
let visible = Footprint::new(opsin.xsize(), opsin.ysize(), px, py, 8, 8).visible_fraction();
let prices = scratch.rate_prices.as_deref();
let coeffs = &mut scratch.strategy_coeffs;
let mut evaluate = |strategy| {
for c in 0..3 {
forward_sub8_transform(ctx, strategy, &pixels[c], &mut coeffs[c]);
}
let [x, y, b] = &mut ***coeffs;
apply_cfl(ctx, CflXyb { x, y, b }, 64, cmap_factor);
let (distortion, rate) = coefficient_dist_and_rate(
ctx, strategy, coeffs, 64, qac, qm_mult_x, distance, 1, 1, prices, visible,
);
fmla(RD_LAMBDA, rate + meta_r, distortion)
};
let dct8 = if let Some(cost) = cached_dct8 {
cost
} else {
evaluate(STRATEGY_DCT)
};
let identity = if with_fine {
evaluate(STRATEGY_IDENTITY)
} else {
f32::INFINITY
};
let dct2x2 = if with_fine {
evaluate(STRATEGY_DCT2X2)
} else {
f32::INFINITY
};
let mut evaluate_dct4 = |strategy| {
if with_dct4 {
evaluate(strategy)
} else {
f32::INFINITY
}
};
Sub8Costs {
dct8,
identity,
dct2x2,
dct4x4: evaluate_dct4(STRATEGY_DCT4X4),
dct4x8: evaluate_dct4(STRATEGY_DCT4X8),
dct8x4: evaluate_dct4(STRATEGY_DCT8X4),
afv: if with_dct4 || distance <= AFV_MAX_DISTANCE {
std::array::from_fn(|kind| evaluate(STRATEGY_AFV0 + kind as u8))
} else {
[f32::INFINITY; 4]
},
}
}
#[inline]
fn cmap_factors(
cfl: crate::color_correlation::CflFrame,
ytox_map: &ImageSB,
ytob_map: &ImageSB,
bx: usize,
by: usize,
) -> [f32; 3] {
let tx = (bx / 8).min(ytox_map.xsize() - 1);
let ty = (by / 8).min(ytox_map.ysize() - 1);
[
crate::color_correlation::y_to_x_ratio(cfl, ytox_map.row(ty)[tx]),
0.0,
crate::color_correlation::y_to_b_ratio(cfl, ytob_map.row(ty)[tx]),
]
}
const MERGE_RISK_K: f32 = std::f32::consts::LOG2_E;
#[derive(Clone, Copy)]
pub(crate) struct GradientRegionStats {
pub(crate) dominance: f32,
pub(crate) mean: f32,
pub(crate) chroma: f32,
}
pub(crate) type GradientRegionStatsFn =
fn(&Image3F, usize, usize, usize, usize, f32) -> GradientRegionStats;
#[inline]
pub(crate) fn finish_gradient_region_stats(
means: &[f32],
within_sum: f32,
chroma_sum: f32,
pixel_count: usize,
eps: f32,
) -> GradientRegionStats {
let n_cells = means.len() as f32;
let within = within_sum / n_cells;
let grand = means.iter().sum::<f32>() / n_cells;
let lf = means.iter().map(|m| (m - grand) * (m - grand)).sum::<f32>() / n_cells;
GradientRegionStats {
dominance: lf / (lf + within + eps),
mean: grand,
chroma: chroma_sum / pixel_count as f32,
}
}
pub(crate) fn gradient_region_stats_scalar(
opsin: &Image3F,
px0: usize,
py0: usize,
w: usize,
h: usize,
eps: f32,
) -> GradientRegionStats {
let xs = opsin.xsize();
let ys = opsin.ysize();
let cw = w / 4;
let ch = h / 4;
let mut means = [0.0f32; 256]; let mut within = 0.0f32;
for cy in 0..ch {
for cx in 0..cw {
let mut s = 0.0f32;
let mut s2 = 0.0f32;
for dy in 0..4 {
let row = opsin.plane_row(1, (py0 + cy * 4 + dy).min(ys - 1));
for dx in 0..4 {
let sx = (px0 + cx * 4 + dx).min(xs - 1);
let v = row[sx];
s += v;
s2 = fmla(v, v, s2);
}
}
let m = s * (1.0 / 16.0);
means[cy * cw + cx] = m;
within += (s2 * (1.0 / 16.0) - m * m).max(0.0);
}
}
finish_gradient_region_stats(&means[..cw * ch], within, 0.0, w * h, eps)
}
pub(crate) fn gradient_region_stats_with_chroma_scalar(
opsin: &Image3F,
px0: usize,
py0: usize,
w: usize,
h: usize,
eps: f32,
) -> GradientRegionStats {
let xs = opsin.xsize();
let ys = opsin.ysize();
let cw = w / 4;
let ch = h / 4;
let mut means = [0.0f32; 256];
let mut within = 0.0f32;
let mut chroma = 0.0f32;
for cy in 0..ch {
for cx in 0..cw {
let mut s = 0.0f32;
let mut s2 = 0.0f32;
for dy in 0..4 {
let y = (py0 + cy * 4 + dy).min(ys - 1);
let row = opsin.plane_row(1, y);
let xrow = opsin.plane_row(0, y);
let brow = opsin.plane_row(2, y);
for dx in 0..4 {
let sx = (px0 + cx * 4 + dx).min(xs - 1);
let v = row[sx];
s += v;
s2 = fmla(v, v, s2);
chroma += xrow[sx].abs() + (brow[sx] - v).abs();
}
}
let m = s * (1.0 / 16.0);
means[cy * cw + cx] = m;
within += (s2 * (1.0 / 16.0) - m * m).max(0.0);
}
}
finish_gradient_region_stats(&means[..cw * ch], within, chroma, w * h, eps)
}
pub(crate) fn select_gradient_region_stats_fn() -> GradientRegionStatsFn {
#[cfg(all(target_arch = "aarch64", feature = "neon"))]
{
return |opsin, px, py, w, h, eps| unsafe {
crate::neon::gradient_region_stats_neon(opsin, px, py, w, h, eps)
};
}
#[cfg(all(target_arch = "x86_64", feature = "avx"))]
if is_x86_feature_detected!("avx2") && is_x86_feature_detected!("fma") {
return |opsin, px, py, w, h, eps| unsafe {
crate::avx::gradient_region_stats_avx2(opsin, px, py, w, h, eps)
};
}
#[cfg(all(any(target_arch = "x86", target_arch = "x86_64"), feature = "sse"))]
if is_x86_feature_detected!("sse4.1") {
return |opsin, px, py, w, h, eps| unsafe {
crate::sse::gradient_region_stats_sse41(opsin, px, py, w, h, eps)
};
}
#[cfg(all(target_arch = "wasm32", feature = "wasm", target_feature = "simd128"))]
return crate::wasm::gradient_region_stats_wasm;
#[allow(unreachable_code)]
gradient_region_stats_scalar
}
pub(crate) fn select_gradient_region_stats_with_chroma_fn() -> GradientRegionStatsFn {
#[cfg(all(target_arch = "aarch64", feature = "neon"))]
{
return |opsin, px, py, w, h, eps| unsafe {
crate::neon::gradient_region_stats_with_chroma_neon(opsin, px, py, w, h, eps)
};
}
#[cfg(all(target_arch = "x86_64", feature = "avx"))]
if is_x86_feature_detected!("avx2") && is_x86_feature_detected!("fma") {
return |opsin, px, py, w, h, eps| unsafe {
crate::avx::gradient_region_stats_with_chroma_avx2(opsin, px, py, w, h, eps)
};
}
#[cfg(all(any(target_arch = "x86", target_arch = "x86_64"), feature = "sse"))]
if is_x86_feature_detected!("sse4.1") {
return |opsin, px, py, w, h, eps| unsafe {
crate::sse::gradient_region_stats_with_chroma_sse41(opsin, px, py, w, h, eps)
};
}
#[cfg(all(target_arch = "wasm32", feature = "wasm", target_feature = "simd128"))]
return crate::wasm::gradient_region_stats_with_chroma_wasm;
#[allow(unreachable_code)]
gradient_region_stats_with_chroma_scalar
}
#[inline]
fn risk_gated(k: f32, accept: f32, q_min: f32, q_max: f32, area_scale: f32) -> f32 {
if q_min <= 0.0 || k <= 0.0 {
return accept;
}
let spread = q_max / q_min - 1.0;
accept * fast_exp2(-k * area_scale * spread)
}