warble_eval_runner/
monitor.rs

1//! Live assertive-eval pair scoring.
2//!
3//! `eval run --record-answers` preserves each raw verdict envelope in the ordinary report. This
4//! module joins a clean report, an injected report, and the generator's injection manifest into the
5//! precision / recall / false-alarm / attribution view that a single report cannot express.
6
7use crate::Report;
8use serde::{Deserialize, Serialize};
9use serde_json::Value;
10use std::collections::{BTreeMap, BTreeSet};
11
12#[derive(Debug, Deserialize)]
13struct InjectionManifest {
14    dataset: String,
15    scenario: String,
16    injections: Vec<Injection>,
17}
18
19#[derive(Debug, Deserialize)]
20struct Injection {
21    id: String,
22    entity: String,
23    expected_verdict: String,
24    attribution_keywords: Vec<String>,
25    #[serde(default)]
26    expected_fresh: Option<bool>,
27    #[serde(default)]
28    expected_severity: Option<String>,
29}
30
31#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
32pub struct MonitorMetric {
33    pub value: f64,
34    pub numerator: u32,
35    pub denominator: u32,
36}
37
38impl MonitorMetric {
39    fn new(numerator: u32, denominator: u32) -> Self {
40        Self {
41            value: if denominator == 0 {
42                0.0
43            } else {
44                numerator as f64 / denominator as f64
45            },
46            numerator,
47            denominator,
48        }
49    }
50}
51
52#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
53pub struct MonitorObservation {
54    pub fresh: bool,
55    #[serde(skip_serializing_if = "Option::is_none")]
56    pub severity: Option<String>,
57    pub cause: String,
58    pub verified: bool,
59    pub runner_pass: bool,
60}
61
62#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
63pub struct MonitorReport {
64    pub dataset: String,
65    pub scenario: String,
66    pub injection_id: String,
67    pub entity: String,
68    pub passed: bool,
69    pub clean: MonitorObservation,
70    pub injected: MonitorObservation,
71    pub by_tag: BTreeMap<String, MonitorMetric>,
72    pub notes: Vec<String>,
73}
74
75/// Score the one currently-live freshness scenario. Other injection kinds deliberately remain a
76/// follow-up: only `stopped_updates` has deterministic freshness labels in the manifest.
77pub fn score_monitor_pair(
78    manifest_yaml: &str,
79    clean_report: &Report,
80    injected_report: &Report,
81) -> Result<MonitorReport, String> {
82    let manifest: InjectionManifest = serde_yaml::from_str(manifest_yaml)
83        .map_err(|e| format!("parse injection manifest: {e}"))?;
84    if manifest.scenario != "stopped_updates" {
85        return Err(format!(
86            "unsupported monitor scenario '{}': only stopped_updates has freshness ground truth",
87            manifest.scenario
88        ));
89    }
90    if manifest.injections.len() != 1 {
91        return Err(format!(
92            "stopped_updates manifest must contain exactly one injection, found {}",
93            manifest.injections.len()
94        ));
95    }
96    let injection = &manifest.injections[0];
97    if injection.expected_verdict != "anomaly" || injection.expected_fresh != Some(false) {
98        return Err("stopped_updates manifest must declare anomaly + expected_fresh=false".into());
99    }
100
101    let clean = observation(clean_report, &injection.id)?;
102    let injected = observation(injected_report, &injection.id)?;
103    let clean_anomaly = !clean.fresh;
104    let injected_anomaly = !injected.fresh;
105    let true_positive = u32::from(injected_anomaly);
106    let anomaly_claims = u32::from(clean_anomaly) + u32::from(injected_anomaly);
107    let attribution_match = u32::from(
108        injected_anomaly && cause_matches(&injected.cause, &injection.attribution_keywords),
109    );
110
111    let mut by_tag = BTreeMap::new();
112    by_tag.insert("recall".into(), MonitorMetric::new(true_positive, 1));
113    by_tag.insert(
114        "precision".into(),
115        MonitorMetric::new(true_positive, anomaly_claims),
116    );
117    by_tag.insert(
118        "false_alarm_rate".into(),
119        MonitorMetric::new(u32::from(clean_anomaly), 1),
120    );
121    by_tag.insert(
122        "attribution_accuracy".into(),
123        MonitorMetric::new(attribution_match, true_positive),
124    );
125
126    let severity_matches = injection
127        .expected_severity
128        .as_deref()
129        .is_none_or(|expected| injected.severity.as_deref() == Some(expected));
130    let passed = clean.runner_pass
131        && injected.runner_pass
132        && clean.verified
133        && injected.verified
134        && !clean_anomaly
135        && injected_anomaly
136        && severity_matches;
137    let mut notes = vec![
138        "false_alarm_rate is measured on the clean half of the same seed-42 pair".into(),
139        "attribution is observational; matcher calibration across other anomaly kinds is deferred"
140            .into(),
141    ];
142    if !severity_matches {
143        notes.push(format!(
144            "injected severity {:?} did not match manifest {:?}",
145            injected.severity, injection.expected_severity
146        ));
147    }
148
149    Ok(MonitorReport {
150        dataset: manifest.dataset,
151        scenario: manifest.scenario,
152        injection_id: injection.id.clone(),
153        entity: injection.entity.clone(),
154        passed,
155        clean,
156        injected,
157        by_tag,
158        notes,
159    })
160}
161
162fn observation(report: &Report, case_id: &str) -> Result<MonitorObservation, String> {
163    if report.configs.len() != 1 {
164        return Err(format!(
165            "monitor pair report requires exactly one model config, found {}",
166            report.configs.len()
167        ));
168    }
169    let config = &report.configs[0];
170    let case = config
171        .cases
172        .iter()
173        .find(|case| case.id == case_id)
174        .ok_or_else(|| format!("report has no case '{case_id}'"))?;
175    let answers = case.answer_dist.as_ref().ok_or_else(|| {
176        format!("case '{case_id}' has no answer_dist; rerun with --record-answers")
177    })?;
178    let (raw, _) = answers
179        .iter()
180        .max_by(|a, b| a.1.cmp(b.1).then_with(|| b.0.cmp(a.0)))
181        .ok_or_else(|| format!("case '{case_id}' recorded no parseable verdict envelope"))?;
182    let envelope: Value = serde_json::from_str(raw)
183        .map_err(|e| format!("parse recorded verdict for '{case_id}': {e}"))?;
184    let verdict = envelope
185        .get("verdict")
186        .ok_or_else(|| format!("case '{case_id}' envelope has no verdict"))?;
187    let fresh = verdict
188        .get("fresh")
189        .and_then(Value::as_bool)
190        .ok_or_else(|| format!("case '{case_id}' verdict has no boolean fresh field"))?;
191    let status = envelope
192        .get("blocks")
193        .and_then(Value::as_array)
194        .and_then(|blocks| {
195            blocks
196                .iter()
197                .find(|block| block.get("type").and_then(Value::as_str) == Some("status"))
198        });
199    let severity = status
200        .and_then(|block| block.get("severity"))
201        .and_then(Value::as_str)
202        .or_else(|| verdict.get("severity").and_then(Value::as_str))
203        .map(str::to_string);
204    let cause = [
205        status
206            .and_then(|block| block.get("detail"))
207            .and_then(Value::as_str),
208        verdict.get("cause").and_then(Value::as_str),
209    ]
210    .into_iter()
211    .flatten()
212    .collect::<Vec<_>>()
213    .join(" ");
214
215    Ok(MonitorObservation {
216        fresh,
217        severity,
218        cause,
219        verified: envelope
220            .get("verified")
221            .and_then(Value::as_bool)
222            .unwrap_or(false),
223        runner_pass: case.pass,
224    })
225}
226
227fn tokens(text: &str) -> BTreeSet<String> {
228    text.split(|c: char| !c.is_alphanumeric())
229        .map(str::to_lowercase)
230        .filter(|token| token.len() > 2)
231        .collect()
232}
233
234/// Match against the exact canonical token set serialized by the generator, rather than
235/// independently reconstructing an oracle that can drift from the manifest producer.
236fn cause_matches(cause: &str, attribution_keywords: &[String]) -> bool {
237    let expected = attribution_keywords
238        .iter()
239        .flat_map(|keyword| tokens(keyword))
240        .collect::<BTreeSet<_>>();
241    if expected.is_empty() {
242        return false;
243    }
244    let actual = tokens(cause);
245    expected.intersection(&actual).count() * 2 >= expected.len()
246}
247
248pub fn format_monitor_report(report: &MonitorReport) -> String {
249    let mut out = String::new();
250    out.push_str("\n=== Warble eval — live monitor pair ===\n");
251    out.push_str(&format!(
252        "dataset={} scenario={} entity={}\n",
253        report.dataset, report.scenario, report.entity
254    ));
255    for (tag, metric) in &report.by_tag {
256        out.push_str(&format!(
257            "{tag}: {:.3} ({}/{})\n",
258            metric.value, metric.numerator, metric.denominator
259        ));
260    }
261    out.push_str(if report.passed {
262        "MONITOR PAIR PASS\n"
263    } else {
264        "MONITOR PAIR FAIL\n"
265    });
266    out
267}
warble_eval_runner/monitor.rs

warble_eval_runner/
monitor.rs
