use super::consts::{COMPACT_INVOKE_TOKENS, COMPACT_SCHEMA_TOKENS};
use super::ledger_store::{CURRENT_ESTIMATOR_VERSION, StoredLedgerRecord, TunedEstimateConstants};
use super::types::{AdmissionDecision, StelLedgerEvent};
pub const TUNING_REVIEW_MIN_EVENTS: usize = 5;
pub const TUNING_MIN_SAMPLES: usize = 12;
pub const TUNING_MIN_CORPUS: usize = 2 * TUNING_MIN_SAMPLES;
pub const SC002_MAE_REDUCTION_MARGIN: f64 = 0.20;
#[derive(Clone, Copy, Debug, PartialEq)]
pub struct PredictionSample {
pub ts_ms: u64,
pub predicted_response: u32,
pub actual_response: u32,
}
impl From<&StoredLedgerRecord> for PredictionSample {
fn from(r: &StoredLedgerRecord) -> Self {
Self {
ts_ms: r.ts_ms,
predicted_response: r.predicted_response_tokens,
actual_response: r.actual_response_tokens,
}
}
}
impl From<&StelLedgerEvent> for PredictionSample {
fn from(e: &StelLedgerEvent) -> Self {
Self {
ts_ms: e.ts_ms,
predicted_response: e.predicted_response_tokens,
actual_response: e.actual_response_tokens,
}
}
}
#[derive(Clone, Debug, PartialEq)]
pub enum CalibrationVerdict {
Deferred,
Accumulating { n: usize, min: usize },
Tuned {
sample_size: usize,
error_before: f64,
error_after: f64,
},
}
pub const CORRECTION_FACTOR_CAP: f64 = 4.0;
pub const NO_CORRECTION_FACTOR: f64 = 1.0;
fn bound_factor(factor: f64) -> f64 {
factor.clamp(1.0 / CORRECTION_FACTOR_CAP, CORRECTION_FACTOR_CAP)
}
pub fn apply_factor(predicted_response: u32, factor: f64) -> u32 {
let scaled = (f64::from(predicted_response) * factor).round();
if scaled <= 0.0 {
0
} else if scaled >= f64::from(u32::MAX) {
u32::MAX
} else {
scaled as u32
}
}
fn median_actual_over_predicted(samples: &[PredictionSample]) -> Option<f64> {
let mut ratios: Vec<f64> = samples
.iter()
.filter(|s| s.predicted_response > 0)
.map(|s| f64::from(s.actual_response) / f64::from(s.predicted_response))
.collect();
if ratios.is_empty() {
return None;
}
ratios.sort_by(f64::total_cmp);
let mid = ratios.len() / 2;
let median = if ratios.len() % 2 == 1 {
ratios[mid]
} else {
(ratios[mid - 1] + ratios[mid]) / 2.0
};
Some(median)
}
fn held_out_mae(samples: &[PredictionSample], factor: f64) -> Option<f64> {
if samples.is_empty() {
return None;
}
let total: f64 = samples
.iter()
.map(|s| {
let corrected = apply_factor(s.predicted_response, factor);
(f64::from(corrected) - f64::from(s.actual_response)).abs()
})
.sum();
Some(total / samples.len() as f64)
}
pub(crate) fn derive_tuning_candidate(
samples: &[PredictionSample],
) -> Option<TunedEstimateConstants> {
if samples.len() < TUNING_MIN_SAMPLES {
return None;
}
let factor = bound_factor(median_actual_over_predicted(samples)?);
Some(TunedEstimateConstants {
response_correction_factor: factor,
estimator_version: CURRENT_ESTIMATOR_VERSION.to_string(),
sample_size: u32::try_from(samples.len()).unwrap_or(u32::MAX),
error_before: 0.0,
error_after: 0.0,
tuned_at_ms: 0,
})
}
pub fn validate_candidate(
candidate: &TunedEstimateConstants,
held_out: &[PredictionSample],
in_force_factor: f64,
) -> bool {
let Some(mae_before) = held_out_mae(held_out, in_force_factor) else {
return false;
};
let Some(mae_after) = held_out_mae(held_out, candidate.response_correction_factor) else {
return false;
};
if mae_before <= 0.0 {
return false;
}
let relative_reduction = (mae_before - mae_after) / mae_before;
relative_reduction >= SC002_MAE_REDUCTION_MARGIN
}
pub fn compute_calibration_verdict(
samples: &[PredictionSample],
in_force_factor: f64,
) -> (CalibrationVerdict, Option<TunedEstimateConstants>) {
let n = samples.len();
if n == 0 {
return (CalibrationVerdict::Deferred, None);
}
let accumulating = || {
(
CalibrationVerdict::Accumulating {
n,
min: TUNING_MIN_CORPUS,
},
None,
)
};
if n < TUNING_MIN_CORPUS {
return accumulating();
}
let mut ordered = samples.to_vec();
ordered.sort_by_key(|s| s.ts_ms);
let split = ordered.len() / 2;
let (train, held_out) = ordered.split_at(split);
let Some(mut candidate) = derive_tuning_candidate(train) else {
return accumulating();
};
if !validate_candidate(&candidate, held_out, in_force_factor) {
return accumulating();
}
let error_before = held_out_mae(held_out, in_force_factor).unwrap_or(0.0);
let error_after = held_out_mae(held_out, candidate.response_correction_factor).unwrap_or(0.0);
candidate.error_before = error_before;
candidate.error_after = error_after;
candidate.sample_size = u32::try_from(n).unwrap_or(u32::MAX);
(
CalibrationVerdict::Tuned {
sample_size: n,
error_before,
error_after,
},
Some(candidate),
)
}
pub fn render_calibration_verdict(verdict: &CalibrationVerdict) -> String {
match verdict {
CalibrationVerdict::Deferred => "deferred".to_string(),
CalibrationVerdict::Accumulating { n, min } => {
format!("accumulating ({n}/{min})")
}
CalibrationVerdict::Tuned {
sample_size,
error_before,
error_after,
} => {
let reduction_pct = if *error_before > 0.0 {
((error_before - error_after) / error_before) * 100.0
} else {
0.0
};
format!(
"tuned (error: {error_before:.1} -> {error_after:.1} tok, -{reduction_pct:.0}% MAE; n={sample_size})"
)
}
}
}
#[derive(Clone, Debug, PartialEq)]
pub struct StelCalibrationSummary {
pub total_events: usize,
pub serve_count: usize,
pub degrade_count: usize,
pub bypass_count: usize,
pub cache_hit_count: usize,
pub pff_bypass_count: usize,
pub legacy_executed_count: usize,
pub total_schema_tokens: u64,
pub total_invoke_tokens: u64,
pub total_predicted_net: i64,
pub total_predicted_response_tokens: u64,
pub total_actual_response_tokens: u64,
pub tuning_note: String,
pub verdict: CalibrationVerdict,
}
pub fn summarize_calibration(events: &[StelLedgerEvent]) -> StelCalibrationSummary {
let mut summary = StelCalibrationSummary {
total_events: events.len(),
serve_count: 0,
degrade_count: 0,
bypass_count: 0,
cache_hit_count: 0,
pff_bypass_count: 0,
legacy_executed_count: 0,
total_schema_tokens: 0,
total_invoke_tokens: 0,
total_predicted_net: 0,
total_predicted_response_tokens: 0,
total_actual_response_tokens: 0,
tuning_note: String::new(),
verdict: CalibrationVerdict::Deferred,
};
for event in events {
match event.decision {
AdmissionDecision::Serve => summary.serve_count += 1,
AdmissionDecision::Degrade => summary.degrade_count += 1,
AdmissionDecision::CacheHit => summary.cache_hit_count += 1,
AdmissionDecision::Bypass => summary.bypass_count += 1,
_ => {}
}
if event.pff_bypass == Some(true) {
summary.pff_bypass_count += 1;
}
if !event.tools_called.is_empty() {
summary.legacy_executed_count += 1;
}
summary.total_schema_tokens += u64::from(COMPACT_SCHEMA_TOKENS);
summary.total_invoke_tokens += u64::from(COMPACT_INVOKE_TOKENS);
summary.total_predicted_net += i64::from(event.net_vs_manual);
summary.total_predicted_response_tokens += u64::from(event.predicted_response_tokens);
summary.total_actual_response_tokens += u64::from(event.actual_response_tokens);
}
let samples: Vec<PredictionSample> = events.iter().map(PredictionSample::from).collect();
let (verdict, _candidate) = compute_calibration_verdict(&samples, NO_CORRECTION_FACTOR);
summary.tuning_note = render_calibration_verdict(&verdict);
summary.verdict = verdict;
summary
}
pub fn format_calibration_section(summary: &StelCalibrationSummary) -> String {
let lines = [
"── calibration (observational) ──".to_string(),
format!("events: {}", summary.total_events),
format!("serve: {}", summary.serve_count),
format!("degrade: {}", summary.degrade_count),
format!("bypass: {}", summary.bypass_count),
format!("cache_hit: {}", summary.cache_hit_count),
format!("pff_bypass: {}", summary.pff_bypass_count),
format!("legacy_executed: {}", summary.legacy_executed_count),
format!("schema_tokens: {}", summary.total_schema_tokens),
format!("invoke_tokens: {}", summary.total_invoke_tokens),
format!("predicted_net_total: {}", summary.total_predicted_net),
format!(
"predicted_response_tokens: {}",
summary.total_predicted_response_tokens
),
format!(
"actual_response_tokens: {}",
summary.total_actual_response_tokens
),
format!(
"calibration: {}",
render_calibration_verdict(&summary.verdict)
),
format!("tuning: {}", summary.tuning_note),
"──".to_string(),
];
lines.join("\n")
}
#[cfg(test)]
mod tests {
use super::*;
fn biased_corpus(n: usize, predicted: u32, bias: f64) -> Vec<PredictionSample> {
(0..n)
.map(|i| PredictionSample {
ts_ms: 1_000 + i as u64,
predicted_response: predicted,
actual_response: (f64::from(predicted) * bias).round() as u32,
})
.collect()
}
fn factor_candidate(factor: f64) -> TunedEstimateConstants {
TunedEstimateConstants {
response_correction_factor: factor,
estimator_version: CURRENT_ESTIMATOR_VERSION.to_string(),
sample_size: 12,
error_before: 0.0,
error_after: 0.0,
tuned_at_ms: 0,
}
}
#[test]
fn derive_returns_none_below_minimum() {
let corpus = biased_corpus(TUNING_MIN_SAMPLES - 1, 400, 2.0);
assert!(derive_tuning_candidate(&corpus).is_none());
}
#[test]
fn derive_recovers_systematic_factor() {
let corpus = biased_corpus(TUNING_MIN_SAMPLES, 400, 2.0);
let candidate = derive_tuning_candidate(&corpus).expect("candidate above minimum");
assert_eq!(candidate.response_correction_factor, 2.0);
assert_eq!(candidate.estimator_version, CURRENT_ESTIMATOR_VERSION);
}
#[test]
fn derive_bounds_factor_to_cap() {
let corpus = biased_corpus(TUNING_MIN_SAMPLES, 100, 10.0);
let candidate = derive_tuning_candidate(&corpus).expect("candidate");
assert_eq!(candidate.response_correction_factor, CORRECTION_FACTOR_CAP);
}
#[test]
fn apply_factor_corrects_a_prediction() {
assert_eq!(apply_factor(400, 2.0), 800);
assert_eq!(apply_factor(400, NO_CORRECTION_FACTOR), 400);
assert_eq!(apply_factor(333, 1.5), 500); }
#[test]
fn derive_is_deterministic() {
let a = derive_tuning_candidate(&biased_corpus(20, 400, 1.7)).unwrap();
let b = derive_tuning_candidate(&biased_corpus(20, 400, 1.7)).unwrap();
assert_eq!(a, b);
}
#[test]
fn validate_accepts_when_held_out_real_residual_drops_by_margin() {
let held_out = biased_corpus(10, 400, 2.0);
let candidate = factor_candidate(2.0);
assert!(validate_candidate(
&candidate,
&held_out,
NO_CORRECTION_FACTOR
));
}
#[test]
fn validate_rejects_worse_candidate() {
let held_out = biased_corpus(10, 400, 1.0);
let worse = factor_candidate(2.0);
assert!(
held_out_mae(&held_out, 2.0).unwrap()
> held_out_mae(&held_out, NO_CORRECTION_FACTOR).unwrap()
);
assert!(!validate_candidate(&worse, &held_out, NO_CORRECTION_FACTOR));
}
#[test]
fn validate_rejects_marginal_candidate() {
let held_out = biased_corpus(10, 400, 1.1); let marginal = factor_candidate(1.0175);
let before = held_out_mae(&held_out, NO_CORRECTION_FACTOR).unwrap();
let after = held_out_mae(&held_out, marginal.response_correction_factor).unwrap();
let relative = (before - after) / before;
assert!(
relative < SC002_MAE_REDUCTION_MARGIN,
"fixture must be below the margin to exercise reject (got {relative:.3})"
);
assert!(!validate_candidate(
&marginal,
&held_out,
NO_CORRECTION_FACTOR
));
}
#[test]
fn verdict_tuned_on_biased_accumulating_on_unbiased() {
let biased = biased_corpus(TUNING_MIN_CORPUS, 400, 2.0);
let (verdict, candidate) = compute_calibration_verdict(&biased, NO_CORRECTION_FACTOR);
match verdict {
CalibrationVerdict::Tuned {
error_before,
error_after,
..
} => assert!(
error_after < error_before,
"tuned must reduce the real held-out residual"
),
other => panic!("biased corpus must reach Tuned, got {other:?}"),
}
assert!(candidate.is_some(), "accepted candidate must be returned");
let unbiased = biased_corpus(TUNING_MIN_CORPUS, 400, 1.0);
let (verdict, candidate) = compute_calibration_verdict(&unbiased, NO_CORRECTION_FACTOR);
assert!(
matches!(verdict, CalibrationVerdict::Accumulating { .. }),
"unbiased corpus must NOT tune, got {verdict:?}"
);
assert!(candidate.is_none());
}
#[test]
fn accumulating_renders_true_threshold_no_n_gt_min() {
for n in TUNING_MIN_SAMPLES..TUNING_MIN_CORPUS {
let corpus = biased_corpus(n, 400, 2.0);
let (verdict, candidate) = compute_calibration_verdict(&corpus, NO_CORRECTION_FACTOR);
match verdict {
CalibrationVerdict::Accumulating { n: got_n, min } => {
assert_eq!(got_n, n);
assert_eq!(
min, TUNING_MIN_CORPUS,
"min must be the true 2*MIN threshold"
);
assert!(got_n <= min, "n ({got_n}) must never exceed min ({min})");
}
other => panic!("n={n} in [MIN, 2*MIN) must be Accumulating, got {other:?}"),
}
assert!(candidate.is_none(), "no tuning below the corpus gate");
}
}
#[test]
fn out_of_time_split_rejects_drifted_correction() {
let mut corpus = Vec::new();
for i in 0..TUNING_MIN_SAMPLES {
corpus.push(PredictionSample {
ts_ms: 1_000 + i as u64, predicted_response: 400,
actual_response: 800, });
}
for i in 0..TUNING_MIN_SAMPLES {
corpus.push(PredictionSample {
ts_ms: 9_000 + i as u64, predicted_response: 400,
actual_response: 400, });
}
let (verdict, candidate) = compute_calibration_verdict(&corpus, NO_CORRECTION_FACTOR);
assert!(
matches!(verdict, CalibrationVerdict::Accumulating { .. }),
"a drifted correction must be rejected by the out-of-time split, got {verdict:?}"
);
assert!(candidate.is_none());
}
#[test]
fn render_never_emits_tuned_without_artifact() {
assert_eq!(
render_calibration_verdict(&CalibrationVerdict::Deferred),
"deferred"
);
assert_eq!(
render_calibration_verdict(&CalibrationVerdict::Accumulating { n: 3, min: 24 }),
"accumulating (3/24)"
);
let tuned = render_calibration_verdict(&CalibrationVerdict::Tuned {
sample_size: 40,
error_before: 400.0,
error_after: 10.0,
});
assert!(tuned.starts_with("tuned (error: 400.0 -> 10.0 tok"));
assert!(tuned.contains("n=40"));
}
}