use crate::events::Event;
use crate::events::EventKind;
use serde::{Deserialize, Serialize};
pub const MIN_SAMPLE_COUNT: u64 = 10;
pub const NEVER_APPROACHES_MARGIN: f64 = 0.1;
pub const NEAR_CONSTANT_VARIANCE_EPSILON: f64 = 1e-6;
#[derive(Debug, Clone, PartialEq)]
pub struct GateScoreSample {
pub gate: String,
pub score: f64,
pub threshold: f64,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "kebab-case")]
pub enum GateScoreFlagKind {
NeverApproachesThreshold,
NearConstant,
}
impl GateScoreFlagKind {
pub fn as_str(&self) -> &'static str {
match self {
Self::NeverApproachesThreshold => "never-approaches-threshold",
Self::NearConstant => "near-constant",
}
}
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct ScoreDistribution {
pub samples: u64,
pub min_score: f64,
pub max_score: f64,
pub mean_score: f64,
pub variance: f64,
pub min_distance: f64,
pub max_distance: f64,
pub closest_approach: f64,
}
impl ScoreDistribution {
fn from_samples(samples: &[&GateScoreSample]) -> Self {
let n = samples.len() as f64;
let mut min_score = f64::INFINITY;
let mut max_score = f64::NEG_INFINITY;
let mut sum = 0.0;
let mut min_distance = f64::INFINITY;
let mut max_distance = f64::NEG_INFINITY;
let mut closest_approach = f64::INFINITY;
for sample in samples {
min_score = min_score.min(sample.score);
max_score = max_score.max(sample.score);
sum += sample.score;
let distance = sample.score - sample.threshold;
min_distance = min_distance.min(distance);
max_distance = max_distance.max(distance);
closest_approach = closest_approach.min(distance.abs());
}
let mean_score = sum / n;
let variance = samples
.iter()
.map(|s| (s.score - mean_score).powi(2))
.sum::<f64>()
/ n;
Self {
samples: samples.len() as u64,
min_score,
max_score,
mean_score,
variance,
min_distance,
max_distance,
closest_approach,
}
}
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct GateScoreFlag {
pub gate: String,
pub kind: GateScoreFlagKind,
pub distribution: ScoreDistribution,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", default)]
pub struct GateScoreFlagsReport {
pub min_samples: u64,
pub never_approaches_margin: f64,
pub near_constant_variance_epsilon: f64,
pub scored_gates: u64,
pub assessed_gates: u64,
pub flags: Vec<GateScoreFlag>,
}
impl Default for GateScoreFlagsReport {
fn default() -> Self {
Self {
min_samples: MIN_SAMPLE_COUNT,
never_approaches_margin: NEVER_APPROACHES_MARGIN,
near_constant_variance_epsilon: NEAR_CONSTANT_VARIANCE_EPSILON,
scored_gates: 0,
assessed_gates: 0,
flags: Vec::new(),
}
}
}
fn never_approaches(closest_approach: f64) -> bool {
closest_approach > NEVER_APPROACHES_MARGIN
}
fn near_constant(variance: f64) -> bool {
variance < NEAR_CONSTANT_VARIANCE_EPSILON
}
pub fn collect_scored_samples(mission_events: &[&Event]) -> Vec<GateScoreSample> {
let mut samples = Vec::new();
for event in mission_events {
let EventKind::GateResult {
gate,
score: Some(score),
threshold: Some(threshold),
..
} = &event.kind
else {
continue;
};
samples.push(GateScoreSample {
gate: gate.clone(),
score: *score,
threshold: *threshold,
});
}
samples
}
pub fn score_distribution_report(samples: &[GateScoreSample]) -> GateScoreFlagsReport {
let mut by_gate: std::collections::BTreeMap<&str, Vec<&GateScoreSample>> =
std::collections::BTreeMap::new();
for sample in samples {
by_gate
.entry(sample.gate.as_str())
.or_default()
.push(sample);
}
let mut assessed_gates = 0;
let mut flags = Vec::new();
for (gate, gate_samples) in &by_gate {
if (gate_samples.len() as u64) < MIN_SAMPLE_COUNT {
continue;
}
assessed_gates += 1;
let distribution = ScoreDistribution::from_samples(gate_samples);
if never_approaches(distribution.closest_approach) {
flags.push(GateScoreFlag {
gate: gate.to_string(),
kind: GateScoreFlagKind::NeverApproachesThreshold,
distribution: distribution.clone(),
});
}
if near_constant(distribution.variance) {
flags.push(GateScoreFlag {
gate: gate.to_string(),
kind: GateScoreFlagKind::NearConstant,
distribution,
});
}
}
GateScoreFlagsReport {
scored_gates: by_gate.len() as u64,
assessed_gates,
flags,
..GateScoreFlagsReport::default()
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::gate::{GateKind, GateSurface, GateVerdict};
use chrono::DateTime;
fn ev(seq: u64, mission_id: &str, ts_ms: i64, kind: EventKind) -> Event {
Event {
seq,
ts: DateTime::from_timestamp_millis(ts_ms).unwrap(),
mission_id: mission_id.to_string(),
kind,
}
}
fn gate_result(gate: &str, verdict: GateVerdict, score: Option<(f64, f64)>) -> EventKind {
EventKind::GateResult {
gate: gate.to_string(),
surface: GateSurface::Approval,
kind: GateKind::Deterministic,
index: 0,
verdict,
artefact_ref: format!("contract gate {gate}"),
artefact_detail: None,
score: score.map(|(score, _)| score),
threshold: score.map(|(_, threshold)| threshold),
rule_ids: Vec::new(),
}
}
fn sample(gate: &str, score: f64, threshold: f64) -> GateScoreSample {
GateScoreSample {
gate: gate.to_string(),
score,
threshold,
}
}
fn repeated(gate: &str, n: usize, score: f64, threshold: f64) -> Vec<GateScoreSample> {
(0..n).map(|_| sample(gate, score, threshold)).collect()
}
fn kinds_of(report: &GateScoreFlagsReport, gate: &str) -> Vec<GateScoreFlagKind> {
report
.flags
.iter()
.filter(|f| f.gate == gate)
.map(|f| f.kind)
.collect()
}
#[test]
fn score_distribution_flag_stats_fold_min_max_mean_variance_distances() {
let mut samples = Vec::new();
for score in [0.0, 0.25, 0.5, 0.75, 1.0] {
samples.push(sample("vacuous-filter", score, 0.5));
samples.push(sample("vacuous-filter", score, 0.5));
}
let report = score_distribution_report(&samples);
assert_eq!(report.scored_gates, 1);
assert_eq!(report.assessed_gates, 1);
assert!(
report.flags.is_empty(),
"this spread approaches the threshold and moves: {report:?}"
);
let distribution = ScoreDistribution::from_samples(&samples.iter().collect::<Vec<_>>());
assert_eq!(distribution.samples, 10);
assert_eq!(distribution.min_score, 0.0);
assert_eq!(distribution.max_score, 1.0);
assert_eq!(distribution.mean_score, 0.5);
assert_eq!(distribution.variance, 0.125);
assert_eq!(distribution.min_distance, -0.5);
assert_eq!(distribution.max_distance, 0.5);
assert_eq!(distribution.closest_approach, 0.0);
}
#[test]
fn score_distribution_flag_never_approaches_fires_and_carries_evidence() {
let report = score_distribution_report(&repeated("vacuous-filter", 10, 0.5, 1.0));
assert_eq!(
kinds_of(&report, "vacuous-filter"),
[
GateScoreFlagKind::NeverApproachesThreshold,
GateScoreFlagKind::NearConstant,
]
);
let flag = &report.flags[0];
assert_eq!(flag.gate, "vacuous-filter");
assert_eq!(flag.kind.as_str(), "never-approaches-threshold");
let d = &flag.distribution;
assert_eq!(d.samples, 10);
assert_eq!(d.closest_approach, 0.5);
assert_eq!(d.min_score, 0.5);
assert_eq!(d.max_score, 0.5);
assert_eq!(d.mean_score, 0.5);
assert_eq!(d.variance, 0.0);
assert_eq!(d.min_distance, -0.5);
assert_eq!(d.max_distance, -0.5);
}
#[test]
fn score_distribution_flag_never_approaches_closest_score_within_margin_clears() {
let mut samples = repeated("vacuous-filter", 9, 0.5, 1.0);
samples.push(sample("vacuous-filter", 0.95, 1.0));
let report = score_distribution_report(&samples);
assert_eq!(report.assessed_gates, 1);
assert!(
!kinds_of(&report, "vacuous-filter")
.contains(&GateScoreFlagKind::NeverApproachesThreshold),
"one approach within the margin clears the smell: {report:?}"
);
assert!(report.flags.is_empty(), "{report:?}");
}
#[test]
fn score_distribution_flag_never_approaches_margin_boundary_is_strict() {
assert!(!never_approaches(NEVER_APPROACHES_MARGIN));
assert!(never_approaches(NEVER_APPROACHES_MARGIN * 2.0));
let at = score_distribution_report(&repeated("gate-a", 10, 0.1, 0.0));
assert!(
!kinds_of(&at, "gate-a").contains(&GateScoreFlagKind::NeverApproachesThreshold),
"at the margin is within reach: {at:?}"
);
let beyond = score_distribution_report(&repeated("gate-a", 10, 0.2, 0.0));
assert!(
kinds_of(&beyond, "gate-a").contains(&GateScoreFlagKind::NeverApproachesThreshold),
"beyond the margin flags: {beyond:?}"
);
}
#[test]
fn score_distribution_flag_near_constant_fires_on_unmoving_scores() {
let report = score_distribution_report(&repeated("vacuous-filter", 10, 0.75, 1.0));
assert_eq!(
kinds_of(&report, "vacuous-filter"),
[
GateScoreFlagKind::NeverApproachesThreshold,
GateScoreFlagKind::NearConstant,
]
);
let flag = report
.flags
.iter()
.find(|f| f.kind == GateScoreFlagKind::NearConstant)
.unwrap();
assert_eq!(flag.gate, "vacuous-filter");
assert_eq!(flag.kind.as_str(), "near-constant");
assert_eq!(flag.distribution.variance, 0.0);
assert_eq!(
report.flags[0].kind,
GateScoreFlagKind::NeverApproachesThreshold
);
assert_eq!(report.flags[1].kind, GateScoreFlagKind::NearConstant);
}
#[test]
fn score_distribution_flag_near_constant_epsilon_boundary_is_strict() {
assert!(!near_constant(NEAR_CONSTANT_VARIANCE_EPSILON));
assert!(near_constant(NEAR_CONSTANT_VARIANCE_EPSILON / 2.0));
let mut samples = Vec::new();
for i in 0..10 {
let score = if i % 2 == 0 { 0.0 } else { 0.5 };
samples.push(sample("gate-b", score, 0.25));
}
let report = score_distribution_report(&samples);
assert_eq!(
kinds_of(&report, "gate-b"),
[GateScoreFlagKind::NeverApproachesThreshold]
);
}
#[test]
fn score_distribution_flag_minimum_sample_boundary_at_and_under() {
let under = score_distribution_report(&repeated(
"vacuous-filter",
(MIN_SAMPLE_COUNT - 1) as usize,
0.5,
1.0,
));
assert_eq!(under.scored_gates, 1, "the gate IS counted as scored");
assert_eq!(under.assessed_gates, 0, "but never assessed");
assert!(under.flags.is_empty(), "no flags below the minimum");
let at = score_distribution_report(&repeated(
"vacuous-filter",
MIN_SAMPLE_COUNT as usize,
0.5,
1.0,
));
assert_eq!(at.assessed_gates, 1);
assert_eq!(at.flags.len(), 2, "both rules fire at the minimum");
}
#[test]
fn score_distribution_flag_unscored_gates_excluded_never_flagged() {
let events = [
ev(
1,
"m-1",
1_000,
gate_result("env-sensitive", GateVerdict::Pass, None),
),
ev(
2,
"m-1",
2_000,
gate_result("env-sensitive", GateVerdict::Fail, None),
),
ev(3, "m-1", 3_000, EventKind::MissionCompleted {}),
];
let refs: Vec<&Event> = events.iter().collect();
let samples = collect_scored_samples(&refs);
assert!(samples.is_empty(), "an unscored gate emits no sample");
let report = score_distribution_report(&samples);
assert_eq!(report.scored_gates, 0);
assert_eq!(report.assessed_gates, 0);
assert!(report.flags.is_empty());
let mut events = Vec::new();
for i in 0..10 {
events.push(ev(
i + 1,
"m-1",
1_000 + i as i64,
gate_result("vacuous-filter", GateVerdict::Pass, Some((0.5, 1.0))),
));
}
events.push(ev(
11,
"m-1",
2_000,
gate_result("env-sensitive", GateVerdict::Pass, None),
));
let refs: Vec<&Event> = events.iter().collect();
let samples = collect_scored_samples(&refs);
assert_eq!(samples.len(), 10);
let report = score_distribution_report(&samples);
assert_eq!(report.scored_gates, 1);
assert_eq!(report.assessed_gates, 1);
assert!(
report.flags.iter().all(|f| f.gate == "vacuous-filter"),
"the unscored gate is never named: {report:?}"
);
}
#[test]
fn score_distribution_flag_pure_fold_repeat_is_byte_identical() {
let mut samples = repeated("vacuous-filter", 10, 0.5, 1.0);
samples.extend(repeated("other-gate", 10, 0.95, 1.0));
let first = score_distribution_report(&samples);
let second = score_distribution_report(&samples);
assert_eq!(first, second);
assert_eq!(
serde_json::to_string(&first).unwrap(),
serde_json::to_string(&second).unwrap(),
);
assert_eq!(first.min_samples, MIN_SAMPLE_COUNT);
assert_eq!(first.never_approaches_margin, NEVER_APPROACHES_MARGIN);
assert_eq!(
first.near_constant_variance_epsilon,
NEAR_CONSTANT_VARIANCE_EPSILON
);
let json = serde_json::to_value(&first).unwrap();
let kinds: Vec<&str> = json["flags"]
.as_array()
.unwrap()
.iter()
.map(|f| f["kind"].as_str().unwrap())
.collect();
assert!(
kinds.contains(&"never-approaches-threshold"),
"kebab-case wire kind: {kinds:?}"
);
}
}