use std::path::PathBuf;
use std::sync::Arc;
use async_trait::async_trait;
use rusty_agent_runtime::deploy::{
DeploymentRevision, GateCheckRecord, GateDeclaration, GateEvaluation, GateVerdict,
RevisionGateEvaluator,
};
use rusty_agent_runtime::error::{Result as RuntimeResult, RustyError};
use rusty_agent_runtime::journal::Journal;
use rusty_eval::statistics::{
detect_pass_rate_regression, StatisticalDecision, StatisticalRegressionConfig,
};
use rusty_eval::{evaluate_gate, EvalCase, ExperimentConfig, ExperimentRunner, GatePolicy};
use rusty_eval::{ExperimentReport, PreparedRun};
use serde_json::json;
use crate::learn::DatasetSource;
fn invalid(message: impl Into<String>) -> RustyError {
RustyError::InvalidUpdate(message.into())
}
#[async_trait]
pub trait RevisionEvaluationAgent: Send + Sync + std::fmt::Debug {
fn prepare(
&self,
case: &EvalCase,
journal: &Journal,
revision: &DeploymentRevision,
) -> RuntimeResult<PreparedRun>;
}
pub trait GatePolicySource: Send + Sync + std::fmt::Debug {
fn load(&self, name: &str) -> Result<GatePolicy, String>;
}
#[derive(Debug)]
pub struct DirectoryGatePolicySource {
root: PathBuf,
}
impl DirectoryGatePolicySource {
pub fn new(root: impl Into<PathBuf>) -> Self {
Self { root: root.into() }
}
}
impl GatePolicySource for DirectoryGatePolicySource {
fn load(&self, name: &str) -> Result<GatePolicy, String> {
let text = std::fs::read_to_string(self.root.join(format!("{name}.json")))
.map_err(|e| format!("policy `{name}`: {e}"))?;
GatePolicy::from_json(&text).map_err(|e| e.to_string())
}
}
#[derive(Debug)]
pub struct EvalRevisionGateEvaluator {
datasets: Arc<dyn DatasetSource>,
policies: Arc<dyn GatePolicySource>,
agent: Arc<dyn RevisionEvaluationAgent>,
runs_per_case: usize,
statistical: StatisticalRegressionConfig,
}
impl EvalRevisionGateEvaluator {
pub fn new(
datasets: Arc<dyn DatasetSource>,
policies: Arc<dyn GatePolicySource>,
agent: Arc<dyn RevisionEvaluationAgent>,
runs_per_case: usize,
statistical: StatisticalRegressionConfig,
) -> Self {
Self {
datasets,
policies,
agent,
runs_per_case: runs_per_case.max(1),
statistical,
}
}
async fn run_experiment(
&self,
dataset: &rusty_eval::Dataset,
revision: &DeploymentRevision,
) -> RuntimeResult<ExperimentReport> {
let runner =
ExperimentRunner::new(ExperimentConfig::new().with_runs_per_case(self.runs_per_case));
let agent = self.agent.clone();
let revision_id = revision.revision_id.to_string();
let revision = revision.clone();
runner
.run(dataset, move |case, journal| {
agent.prepare(case, journal, &revision)
})
.await
.map_err(|e| invalid(format!("experiment under revision `{revision_id}`: {e}")))
}
}
#[async_trait]
impl RevisionGateEvaluator for EvalRevisionGateEvaluator {
async fn evaluate(
&self,
revision: &DeploymentRevision,
baseline: Option<&DeploymentRevision>,
gate: &GateDeclaration,
) -> RuntimeResult<GateEvaluation> {
let policy = self
.policies
.load(&gate.policy)
.map_err(|e| invalid(format!("gate policy: {e}")))?;
let dataset = self
.datasets
.load(&gate.dataset_version)
.map_err(|e| invalid(format!("gate dataset: {e}")))?;
let candidate_report = self.run_experiment(&dataset, revision).await?;
let baseline_report = match baseline {
Some(baseline) => Some(self.run_experiment(&dataset, baseline).await?),
None => None,
};
let decision = evaluate_gate(&policy, &candidate_report, baseline_report.as_ref())
.map_err(|e| invalid(format!("gate evaluation: {e}")))?;
let mut checks: Vec<GateCheckRecord> = decision
.checks()
.iter()
.map(|check| GateCheckRecord {
metric: serde_json::to_string(&check.metric)
.unwrap_or_else(|_| "\"unserializable\"".to_owned()),
passed: check.passed,
observed: check.observed.clone(),
required: check.required.clone(),
detail: check.detail.clone(),
})
.collect();
if let Some(baseline_report) = &baseline_report {
let (passed, observed, detail) = match detect_pass_rate_regression(
baseline_report,
&candidate_report,
&self.statistical,
) {
Ok(report) => {
let passed = report.decision == StatisticalDecision::NoRegression;
(
passed,
json!({
"decision": report.decision,
"pairs": report.pairs,
"pass_rate_drop": report.pass_rate_drop,
}),
format!(
"paired comparison over {} matched runs decided {:?}",
report.pairs, report.decision
),
)
}
Err(e) => (
false,
json!({ "error": e.to_string() }),
format!(
"the paired comparison could not be evaluated ({e}) — an \
unevaluable comparison fails closed"
),
),
};
checks.push(GateCheckRecord {
metric: "\"statistical_power\"".to_owned(),
passed,
observed,
required: json!({ "minimum_pairs": self.statistical.minimum_pairs }),
detail,
});
}
let outcome = if checks.iter().all(|check| check.passed) {
GateVerdict::Allow
} else {
GateVerdict::Block
};
Ok(GateEvaluation {
policy: gate.policy.clone(),
dataset_version: gate.dataset_version.clone(),
outcome,
checks,
})
}
}