areev_loop/analyzers/
outcome_review.rs

1//! Outcome review (T0). For applied recommendations past their `review_after`,
2//! the engine re-runs the stored metric query (it owns the `&mut` substrate)
3//! and hands the measured values in as `OutcomeInput`s; this analyzer makes the
4//! deterministic changed/regressed decision and proposes a revert on
5//! regression. Closes the honesty loop — makes approve and auto-apply
6//! accountable to measured history.
7//!
8//! Our built-in metrics are lower-is-better (e.g. `tool_error_rate`), so a
9//! regression is `current > baseline` beyond a small epsilon.
10
11use crate::analyzer::{AnalyzeCtx, Analyzer};
12use crate::error::Result;
13use crate::manifest::*;
14use crate::model::{ActionKind, Severity};
15use crate::recommendation::{Proposal, RecDraft, Summary};
16use serde_json::{json, Map};
17
18/// Minimum relative worsening to call a regression (avoids noise at n=1).
19const REGRESSION_EPSILON: f64 = 1e-9;
20
21pub struct OutcomeReview {
22    manifest: AnalyzerManifest,
23}
24
25impl OutcomeReview {
26    pub fn new() -> Self {
27        OutcomeReview {
28            manifest: AnalyzerManifest {
29                id: "loop.outcome_review/1".into(),
30                title: "Outcome review".into(),
31                description:
32                    "Re-measures applied recommendations and proposes revert on regression.".into(),
33                tier: Tier::T0,
34                cadence: CadenceClass::Fast,
35                requires: vec![],
36                target_classes: vec![TargetClass::Memory, TargetClass::Query],
37                auto_apply: AutoApplyClass::Never,
38                trust_class: TrustClass::Builtin,
39                params: vec![],
40                default_on: true,
41            },
42        }
43    }
44}
45
46impl Default for OutcomeReview {
47    fn default() -> Self {
48        Self::new()
49    }
50}
51
52impl Analyzer for OutcomeReview {
53    fn manifest(&self) -> &AnalyzerManifest {
54        &self.manifest
55    }
56
57    fn analyze(&self, ctx: &AnalyzeCtx) -> Result<Vec<RecDraft>> {
58        let mut drafts = Vec::new();
59        for input in ctx.outcome_inputs() {
60            let regressed = input.current > input.baseline + REGRESSION_EPSILON;
61            if !regressed {
62                continue;
63            }
64            let mut args = Map::new();
65            args.insert("metric".into(), json!(input.metric));
66            args.insert("baseline".into(), json!(round4(input.baseline)));
67            args.insert("current".into(), json!(round4(input.current)));
68
69            let mut data = Map::new();
70            data.insert("revert_of".into(), json!(input.rec_hash));
71            data.insert("metric".into(), json!(input.metric));
72
73            drafts.push(
74                RecDraft::new(
75                    input.target_ref.clone(),
76                    ActionKind::Revert,
77                    Summary::new("outcome.regression", args),
78                    Proposal::Data { data },
79                )
80                .severity(Severity::High)
81                .evidence(vec![input.rec_hash.clone()]),
82            );
83        }
84        drafts.sort_by(|a, b| a.evidence.cmp(&b.evidence));
85        Ok(drafts)
86    }
87}
88
89fn round4(x: f64) -> f64 {
90    (x * 10_000.0).round() / 10_000.0
91}
92
93#[cfg(test)]
94mod tests {
95    use super::*;
96    use crate::analyzer::OutcomeInput;
97    use crate::testkit::TestSubstrate;
98
99    fn input(baseline: f64, current: f64) -> OutcomeInput {
100        OutcomeInput {
101            rec_hash: "ref-1".into(),
102            target_ref: "entity:lessons/stripe_refund".into(),
103            metric: "tool_error_rate".into(),
104            baseline,
105            current,
106            unit: "ratio".into(),
107        }
108    }
109
110    #[test]
111    fn proposes_revert_on_regression() {
112        let mut sub = TestSubstrate::new();
113        sub.set_outcome_inputs(vec![input(0.2, 0.5)]);
114        let drafts = sub.analyze(&OutcomeReview::new(), 10_000);
115        assert_eq!(drafts.len(), 1);
116        assert_eq!(drafts[0].action_kind, ActionKind::Revert);
117    }
118
119    #[test]
120    fn silent_when_improved_or_unchanged() {
121        let mut sub = TestSubstrate::new();
122        sub.set_outcome_inputs(vec![input(0.5, 0.2), input(0.3, 0.3)]);
123        assert!(sub.analyze(&OutcomeReview::new(), 10_000).is_empty());
124    }
125}
areev_loop/analyzers/outcome_review.rs

areev_loop/analyzers/
outcome_review.rs
