Skip to main content

rusty_agent_server/
gate.rs

1//! The wave-4 release-gate composition (R0.12 Operations Plane): core's
2//! [`RevisionGateEvaluator`] seam over `rusty-eval`'s public API — the
3//! [`crate::learn::EvalCandidateEvaluator`] discipline turned toward
4//! deployments.
5//!
6//! The split is the seam's: core owns the contract (what a gate must
7//! answer, and that it must name the declaration it was handed back);
8//! this crate owns the composition, because this is where the
9//! `rusty-eval` dependency is allowed to live. Nothing about scoring,
10//! aggregation, or check evaluation is re-implemented here — the policy
11//! evaluates through [`evaluate_gate`], the experiments through
12//! [`ExperimentRunner`] over the same versioned [`Dataset`] the candidate
13//! plane uses, and a serving baseline adds the paired statistical check
14//! the coarse thresholds cannot speak for.
15//!
16//! Fail-closed throughout, because a gate exists to refuse: an unknown
17//! policy or dataset version is an error, not a pass; an evaluation the
18//! runner cannot complete is an error; and the paired comparison appends
19//! a `statistical_power` check that passes only on
20//! [`StatisticalDecision::NoRegression`] — `InsufficientEvidence` fails
21//! closed (a comparison that cannot speak cannot clear a gate), and
22//! `Regression` fails because it *is* the finding the gate exists to
23//! catch. A comparison that errors at all fails the same way.
24
25use std::path::PathBuf;
26use std::sync::Arc;
27
28use async_trait::async_trait;
29use rusty_agent_runtime::deploy::{
30    DeploymentRevision, GateCheckRecord, GateDeclaration, GateEvaluation, GateVerdict,
31    RevisionGateEvaluator,
32};
33use rusty_agent_runtime::error::{Result as RuntimeResult, RustyError};
34use rusty_agent_runtime::journal::Journal;
35use rusty_eval::statistics::{
36    detect_pass_rate_regression, StatisticalDecision, StatisticalRegressionConfig,
37};
38use rusty_eval::{evaluate_gate, EvalCase, ExperimentConfig, ExperimentRunner, GatePolicy};
39use rusty_eval::{ExperimentReport, PreparedRun};
40use serde_json::json;
41
42use crate::learn::DatasetSource;
43
44/// Gate composition failures use core's `invalid` convention — the same
45/// error shape the learn module's adapters produce, so the control plane
46/// distinguishes composition trouble from gate verdicts the way it
47/// already does.
48fn invalid(message: impl Into<String>) -> RustyError {
49    RustyError::InvalidUpdate(message.into())
50}
51
52/// The application-owned half of a gate evaluation: how to build the
53/// agent one revision runs as. The evaluator owns the evidence
54/// discipline (the versioned dataset, the run counts, the policy, the
55/// statistical config); the agent owns what baseline and candidate
56/// revisions *are* as runnable graphs — the
57/// [`crate::learn::EvaluationAgent`] split, one level up: the revision's
58/// frozen pin set is what the agent applies.
59#[async_trait]
60pub trait RevisionEvaluationAgent: Send + Sync + std::fmt::Debug {
61    /// Build the prepared run for one case repetition under `revision`.
62    /// `journal` is the run's Flight Recorder journal — wire it into
63    /// recording graphs so model and tool calls become assertion
64    /// evidence.
65    fn prepare(
66        &self,
67        case: &EvalCase,
68        journal: &Journal,
69        revision: &DeploymentRevision,
70    ) -> RuntimeResult<PreparedRun>;
71}
72
73/// Where named gate policies come from. A declaration names a policy;
74/// the source resolves it. Policies are immutable per name — re-running
75/// a gate must mean re-reading the same rule, so a source that edits in
76/// place breaks the audit the gate journals.
77pub trait GatePolicySource: Send + Sync + std::fmt::Debug {
78    /// Load policy `name` (`Err` when unknown).
79    fn load(&self, name: &str) -> Result<GatePolicy, String>;
80}
81
82/// Gate policies as JSON files under one directory, named
83/// `{policy_name}.json` — `rusty-eval`'s canonical serialization
84/// ([`GatePolicy::to_json`]), so policies diff cleanly in git and the
85/// file layout is the naming.
86#[derive(Debug)]
87pub struct DirectoryGatePolicySource {
88    root: PathBuf,
89}
90
91impl DirectoryGatePolicySource {
92    /// A source serving `{root}/{policy_name}.json`.
93    pub fn new(root: impl Into<PathBuf>) -> Self {
94        Self { root: root.into() }
95    }
96}
97
98impl GatePolicySource for DirectoryGatePolicySource {
99    fn load(&self, name: &str) -> Result<GatePolicy, String> {
100        let text = std::fs::read_to_string(self.root.join(format!("{name}.json")))
101            .map_err(|e| format!("policy `{name}`: {e}"))?;
102        GatePolicy::from_json(&text).map_err(|e| e.to_string())
103    }
104}
105
106/// The wave-4 gate composition: core's [`RevisionGateEvaluator`] over
107/// `rusty-eval`'s public API. The candidate experiment runs under the
108/// revision being promoted; the baseline experiment runs under the
109/// environment's serving revision when one serves, giving the gate's
110/// comparative checks their evidence and arming the paired statistical
111/// check.
112#[derive(Debug)]
113pub struct EvalRevisionGateEvaluator {
114    datasets: Arc<dyn DatasetSource>,
115    policies: Arc<dyn GatePolicySource>,
116    agent: Arc<dyn RevisionEvaluationAgent>,
117    runs_per_case: usize,
118    statistical: StatisticalRegressionConfig,
119}
120
121impl EvalRevisionGateEvaluator {
122    /// An evaluator reading policies through `policies`, datasets through
123    /// `datasets`, building agents through `agent`, running each case
124    /// `runs_per_case` times (normalized to at least one), and judging
125    /// the paired comparison under `statistical`.
126    pub fn new(
127        datasets: Arc<dyn DatasetSource>,
128        policies: Arc<dyn GatePolicySource>,
129        agent: Arc<dyn RevisionEvaluationAgent>,
130        runs_per_case: usize,
131        statistical: StatisticalRegressionConfig,
132    ) -> Self {
133        Self {
134            datasets,
135            policies,
136            agent,
137            runs_per_case: runs_per_case.max(1),
138            statistical,
139        }
140    }
141
142    async fn run_experiment(
143        &self,
144        dataset: &rusty_eval::Dataset,
145        revision: &DeploymentRevision,
146    ) -> RuntimeResult<ExperimentReport> {
147        let runner =
148            ExperimentRunner::new(ExperimentConfig::new().with_runs_per_case(self.runs_per_case));
149        let agent = self.agent.clone();
150        let revision_id = revision.revision_id.to_string();
151        let revision = revision.clone();
152        runner
153            .run(dataset, move |case, journal| {
154                agent.prepare(case, journal, &revision)
155            })
156            .await
157            .map_err(|e| invalid(format!("experiment under revision `{revision_id}`: {e}")))
158    }
159}
160
161#[async_trait]
162impl RevisionGateEvaluator for EvalRevisionGateEvaluator {
163    async fn evaluate(
164        &self,
165        revision: &DeploymentRevision,
166        baseline: Option<&DeploymentRevision>,
167        gate: &GateDeclaration,
168    ) -> RuntimeResult<GateEvaluation> {
169        let policy = self
170            .policies
171            .load(&gate.policy)
172            .map_err(|e| invalid(format!("gate policy: {e}")))?;
173        let dataset = self
174            .datasets
175            .load(&gate.dataset_version)
176            .map_err(|e| invalid(format!("gate dataset: {e}")))?;
177
178        let candidate_report = self.run_experiment(&dataset, revision).await?;
179        let baseline_report = match baseline {
180            Some(baseline) => Some(self.run_experiment(&dataset, baseline).await?),
181            None => None,
182        };
183
184        let decision = evaluate_gate(&policy, &candidate_report, baseline_report.as_ref())
185            .map_err(|e| invalid(format!("gate evaluation: {e}")))?;
186        let mut checks: Vec<GateCheckRecord> = decision
187            .checks()
188            .iter()
189            .map(|check| GateCheckRecord {
190                // The eval metric's own serde form, carried as a string:
191                // a new eval metric needs no core release to journal.
192                metric: serde_json::to_string(&check.metric)
193                    .unwrap_or_else(|_| "\"unserializable\"".to_owned()),
194                passed: check.passed,
195                observed: check.observed.clone(),
196                required: check.required.clone(),
197                detail: check.detail.clone(),
198            })
199            .collect();
200
201        // The paired check the coarse thresholds cannot speak for.
202        // Passes only on `NoRegression`: `InsufficientEvidence` fails
203        // closed (a comparison that cannot speak cannot clear a gate),
204        // `Regression` fails because it is the finding the gate exists
205        // to catch, and a comparison that errors at all fails the same
206        // way — an unevaluable comparison is not evidence of safety.
207        if let Some(baseline_report) = &baseline_report {
208            let (passed, observed, detail) = match detect_pass_rate_regression(
209                baseline_report,
210                &candidate_report,
211                &self.statistical,
212            ) {
213                Ok(report) => {
214                    let passed = report.decision == StatisticalDecision::NoRegression;
215                    (
216                        passed,
217                        json!({
218                            "decision": report.decision,
219                            "pairs": report.pairs,
220                            "pass_rate_drop": report.pass_rate_drop,
221                        }),
222                        format!(
223                            "paired comparison over {} matched runs decided {:?}",
224                            report.pairs, report.decision
225                        ),
226                    )
227                }
228                Err(e) => (
229                    false,
230                    json!({ "error": e.to_string() }),
231                    format!(
232                        "the paired comparison could not be evaluated ({e}) — an \
233                             unevaluable comparison fails closed"
234                    ),
235                ),
236            };
237            checks.push(GateCheckRecord {
238                metric: "\"statistical_power\"".to_owned(),
239                passed,
240                observed,
241                required: json!({ "minimum_pairs": self.statistical.minimum_pairs }),
242                detail,
243            });
244        }
245
246        let outcome = if checks.iter().all(|check| check.passed) {
247            GateVerdict::Allow
248        } else {
249            GateVerdict::Block
250        };
251        Ok(GateEvaluation {
252            policy: gate.policy.clone(),
253            dataset_version: gate.dataset_version.clone(),
254            outcome,
255            checks,
256        })
257    }
258}