rusty_agent_server/gate.rs
1//! The wave-4 release-gate composition (R0.12 Operations Plane): core's
2//! [`RevisionGateEvaluator`] seam over `rusty-eval`'s public API — the
3//! [`crate::learn::EvalCandidateEvaluator`] discipline turned toward
4//! deployments.
5//!
6//! The split is the seam's: core owns the contract (what a gate must
7//! answer, and that it must name the declaration it was handed back);
8//! this crate owns the composition, because this is where the
9//! `rusty-eval` dependency is allowed to live. Nothing about scoring,
10//! aggregation, or check evaluation is re-implemented here — the policy
11//! evaluates through [`evaluate_gate`], the experiments through
12//! [`ExperimentRunner`] over the same versioned [`Dataset`] the candidate
13//! plane uses, and a serving baseline adds the paired statistical check
14//! the coarse thresholds cannot speak for.
15//!
16//! Fail-closed throughout, because a gate exists to refuse: an unknown
17//! policy or dataset version is an error, not a pass; an evaluation the
18//! runner cannot complete is an error; and the paired comparison appends
19//! a `statistical_power` check that passes only on
20//! [`StatisticalDecision::NoRegression`] — `InsufficientEvidence` fails
21//! closed (a comparison that cannot speak cannot clear a gate), and
22//! `Regression` fails because it *is* the finding the gate exists to
23//! catch. A comparison that errors at all fails the same way.
24
25use std::path::PathBuf;
26use std::sync::Arc;
27
28use async_trait::async_trait;
29use rusty_agent_runtime::deploy::{
30 DeploymentRevision, GateCheckRecord, GateDeclaration, GateEvaluation, GateVerdict,
31 RevisionGateEvaluator,
32};
33use rusty_agent_runtime::error::{Result as RuntimeResult, RustyError};
34use rusty_agent_runtime::journal::Journal;
35use rusty_eval::statistics::{
36 detect_pass_rate_regression, StatisticalDecision, StatisticalRegressionConfig,
37};
38use rusty_eval::{evaluate_gate, EvalCase, ExperimentConfig, ExperimentRunner, GatePolicy};
39use rusty_eval::{ExperimentReport, PreparedRun};
40use serde_json::json;
41
42use crate::learn::DatasetSource;
43
44/// Gate composition failures use core's `invalid` convention — the same
45/// error shape the learn module's adapters produce, so the control plane
46/// distinguishes composition trouble from gate verdicts the way it
47/// already does.
48fn invalid(message: impl Into<String>) -> RustyError {
49 RustyError::InvalidUpdate(message.into())
50}
51
52/// The application-owned half of a gate evaluation: how to build the
53/// agent one revision runs as. The evaluator owns the evidence
54/// discipline (the versioned dataset, the run counts, the policy, the
55/// statistical config); the agent owns what baseline and candidate
56/// revisions *are* as runnable graphs — the
57/// [`crate::learn::EvaluationAgent`] split, one level up: the revision's
58/// frozen pin set is what the agent applies.
59#[async_trait]
60pub trait RevisionEvaluationAgent: Send + Sync + std::fmt::Debug {
61 /// Build the prepared run for one case repetition under `revision`.
62 /// `journal` is the run's Flight Recorder journal — wire it into
63 /// recording graphs so model and tool calls become assertion
64 /// evidence.
65 fn prepare(
66 &self,
67 case: &EvalCase,
68 journal: &Journal,
69 revision: &DeploymentRevision,
70 ) -> RuntimeResult<PreparedRun>;
71}
72
73/// Where named gate policies come from. A declaration names a policy;
74/// the source resolves it. Policies are immutable per name — re-running
75/// a gate must mean re-reading the same rule, so a source that edits in
76/// place breaks the audit the gate journals.
77pub trait GatePolicySource: Send + Sync + std::fmt::Debug {
78 /// Load policy `name` (`Err` when unknown).
79 fn load(&self, name: &str) -> Result<GatePolicy, String>;
80}
81
82/// Gate policies as JSON files under one directory, named
83/// `{policy_name}.json` — `rusty-eval`'s canonical serialization
84/// ([`GatePolicy::to_json`]), so policies diff cleanly in git and the
85/// file layout is the naming.
86#[derive(Debug)]
87pub struct DirectoryGatePolicySource {
88 root: PathBuf,
89}
90
91impl DirectoryGatePolicySource {
92 /// A source serving `{root}/{policy_name}.json`.
93 pub fn new(root: impl Into<PathBuf>) -> Self {
94 Self { root: root.into() }
95 }
96}
97
98impl GatePolicySource for DirectoryGatePolicySource {
99 fn load(&self, name: &str) -> Result<GatePolicy, String> {
100 let text = std::fs::read_to_string(self.root.join(format!("{name}.json")))
101 .map_err(|e| format!("policy `{name}`: {e}"))?;
102 GatePolicy::from_json(&text).map_err(|e| e.to_string())
103 }
104}
105
106/// The wave-4 gate composition: core's [`RevisionGateEvaluator`] over
107/// `rusty-eval`'s public API. The candidate experiment runs under the
108/// revision being promoted; the baseline experiment runs under the
109/// environment's serving revision when one serves, giving the gate's
110/// comparative checks their evidence and arming the paired statistical
111/// check.
112#[derive(Debug)]
113pub struct EvalRevisionGateEvaluator {
114 datasets: Arc<dyn DatasetSource>,
115 policies: Arc<dyn GatePolicySource>,
116 agent: Arc<dyn RevisionEvaluationAgent>,
117 runs_per_case: usize,
118 statistical: StatisticalRegressionConfig,
119}
120
121impl EvalRevisionGateEvaluator {
122 /// An evaluator reading policies through `policies`, datasets through
123 /// `datasets`, building agents through `agent`, running each case
124 /// `runs_per_case` times (normalized to at least one), and judging
125 /// the paired comparison under `statistical`.
126 pub fn new(
127 datasets: Arc<dyn DatasetSource>,
128 policies: Arc<dyn GatePolicySource>,
129 agent: Arc<dyn RevisionEvaluationAgent>,
130 runs_per_case: usize,
131 statistical: StatisticalRegressionConfig,
132 ) -> Self {
133 Self {
134 datasets,
135 policies,
136 agent,
137 runs_per_case: runs_per_case.max(1),
138 statistical,
139 }
140 }
141
142 async fn run_experiment(
143 &self,
144 dataset: &rusty_eval::Dataset,
145 revision: &DeploymentRevision,
146 ) -> RuntimeResult<ExperimentReport> {
147 let runner =
148 ExperimentRunner::new(ExperimentConfig::new().with_runs_per_case(self.runs_per_case));
149 let agent = self.agent.clone();
150 let revision_id = revision.revision_id.to_string();
151 let revision = revision.clone();
152 runner
153 .run(dataset, move |case, journal| {
154 agent.prepare(case, journal, &revision)
155 })
156 .await
157 .map_err(|e| invalid(format!("experiment under revision `{revision_id}`: {e}")))
158 }
159}
160
161#[async_trait]
162impl RevisionGateEvaluator for EvalRevisionGateEvaluator {
163 async fn evaluate(
164 &self,
165 revision: &DeploymentRevision,
166 baseline: Option<&DeploymentRevision>,
167 gate: &GateDeclaration,
168 ) -> RuntimeResult<GateEvaluation> {
169 let policy = self
170 .policies
171 .load(&gate.policy)
172 .map_err(|e| invalid(format!("gate policy: {e}")))?;
173 let dataset = self
174 .datasets
175 .load(&gate.dataset_version)
176 .map_err(|e| invalid(format!("gate dataset: {e}")))?;
177
178 let candidate_report = self.run_experiment(&dataset, revision).await?;
179 let baseline_report = match baseline {
180 Some(baseline) => Some(self.run_experiment(&dataset, baseline).await?),
181 None => None,
182 };
183
184 let decision = evaluate_gate(&policy, &candidate_report, baseline_report.as_ref())
185 .map_err(|e| invalid(format!("gate evaluation: {e}")))?;
186 let mut checks: Vec<GateCheckRecord> = decision
187 .checks()
188 .iter()
189 .map(|check| GateCheckRecord {
190 // The eval metric's own serde form, carried as a string:
191 // a new eval metric needs no core release to journal.
192 metric: serde_json::to_string(&check.metric)
193 .unwrap_or_else(|_| "\"unserializable\"".to_owned()),
194 passed: check.passed,
195 observed: check.observed.clone(),
196 required: check.required.clone(),
197 detail: check.detail.clone(),
198 })
199 .collect();
200
201 // The paired check the coarse thresholds cannot speak for.
202 // Passes only on `NoRegression`: `InsufficientEvidence` fails
203 // closed (a comparison that cannot speak cannot clear a gate),
204 // `Regression` fails because it is the finding the gate exists
205 // to catch, and a comparison that errors at all fails the same
206 // way — an unevaluable comparison is not evidence of safety.
207 if let Some(baseline_report) = &baseline_report {
208 let (passed, observed, detail) = match detect_pass_rate_regression(
209 baseline_report,
210 &candidate_report,
211 &self.statistical,
212 ) {
213 Ok(report) => {
214 let passed = report.decision == StatisticalDecision::NoRegression;
215 (
216 passed,
217 json!({
218 "decision": report.decision,
219 "pairs": report.pairs,
220 "pass_rate_drop": report.pass_rate_drop,
221 }),
222 format!(
223 "paired comparison over {} matched runs decided {:?}",
224 report.pairs, report.decision
225 ),
226 )
227 }
228 Err(e) => (
229 false,
230 json!({ "error": e.to_string() }),
231 format!(
232 "the paired comparison could not be evaluated ({e}) — an \
233 unevaluable comparison fails closed"
234 ),
235 ),
236 };
237 checks.push(GateCheckRecord {
238 metric: "\"statistical_power\"".to_owned(),
239 passed,
240 observed,
241 required: json!({ "minimum_pairs": self.statistical.minimum_pairs }),
242 detail,
243 });
244 }
245
246 let outcome = if checks.iter().all(|check| check.passed) {
247 GateVerdict::Allow
248 } else {
249 GateVerdict::Block
250 };
251 Ok(GateEvaluation {
252 policy: gate.policy.clone(),
253 dataset_version: gate.dataset_version.clone(),
254 outcome,
255 checks,
256 })
257 }
258}