Skip to main content

rusty_eval/
lib.rs

1//! # Rusty Eval
2//!
3//! Rusty Eval is the foundation of Rusty's Agent TestOps plane: it turns the
4//! Flight Recorder's run evidence into release decisions. Composable pieces,
5//! usable on its own:
6//!
7//! - **Datasets** ([`dataset`]) — versioned JSONL datasets: a header
8//!   (`name`, `version`, `format_version`) plus one [`dataset::EvalCase`]
9//!   per line, each carrying an input payload, an expected tool-call
10//!   trajectory with argument matchers, final-state predicates, and tags.
11//!   Loading validates the schema version; serialization is canonical, so
12//!   datasets diff cleanly in git.
13//! - **Assertions** ([`assertion`]) — deterministic checks over a recorded
14//!   run's [`evidence::RunEvidence`]: tool-call order (subsequence match),
15//!   tool-call count, state predicates by JSON pointer, tool blacklists,
16//!   cost and latency bounds. Every verdict carries its evidence — expected
17//!   vs observed — into the report.
18//! - **Experiments** ([`experiment`]) — [`experiment::ExperimentRunner`]
19//!   drives one agent over a dataset N times per case through the real
20//!   `rusty-agent-runtime` executor, distills each run's journal into
21//!   evidence, grades it, and aggregates an
22//!   [`experiment::ExperimentReport`]: pass rate per assertion, per-case
23//!   detail, latency percentiles, total cost. Runs are sequential by default
24//!   or bounded-parallel with deterministic report ordering.
25//! - **Comparison** ([`mod@compare`]) — [`compare::compare`] diffs a candidate
26//!   report against a baseline: per-assertion deltas, per-case regressions
27//!   and improvements, and threshold-based regression flags (pass-rate drop,
28//!   p95 latency growth).
29//! - **Failure clustering** ([`clustering`]) — deterministic signatures group
30//!   failed runs by termination, assertions, and judge outcome, preserving
31//!   source evidence under stable cluster ids.
32//! - **Judges** ([`judge`]) — [`judge::RuleBasedJudge`] scores deterministic
33//!   expectations, while [`judge::ModelJudge`] adapts any runtime `ChatModel`
34//!   into a strict structured evaluator with local threshold enforcement.
35//! - **Human feedback** ([`feedback`]) — deterministic pairwise annotation
36//!   queues with reviewer leases, rubric validation, consensus resolution,
37//!   disagreement adjudication, corrections, and promotion into versioned
38//!   evaluation cases.
39//! - **Release gates** ([`gate`]) — versioned policies turn candidate reports
40//!   and baseline comparisons into deterministic allow/block decisions with
41//!   machine-readable evidence for every configured check.
42//!
43//! ## Quick sketch
44//!
45//! ```no_run
46//! use rusty_eval::{Dataset, ExperimentConfig, ExperimentRunner, PreparedRun, compare, CompareThresholds};
47//!
48//! # async fn demo() -> rusty_eval::Result<()> {
49//! let dataset = Dataset::load("evals/math_tools_v1.jsonl")?;
50//!
51//! let runner = ExperimentRunner::new(ExperimentConfig::new().with_runs_per_case(3));
52//! let baseline = runner
53//!     .run(&dataset, |case, journal| {
54//!         // Build the agent under test for this case run; `journal` is the
55//!         // run's Flight Recorder journal (wire it into recording graphs so
56//!         // model/tool calls become assertion evidence).
57//!         # let _ = (case, journal);
58//!         # unimplemented!()
59//!     })
60//!     .await?;
61//!
62//! let candidate = /* run the changed agent the same way */ baseline.clone();
63//! let verdict = compare(&baseline, &candidate, &CompareThresholds::default());
64//! assert!(!verdict.regressed);
65//! # Ok(())
66//! # }
67//! ```
68
69pub mod assertion;
70pub mod clustering;
71pub mod compare;
72pub mod dataset;
73pub mod error;
74pub mod evidence;
75pub mod experiment;
76pub mod feedback;
77pub mod gate;
78pub mod judge;
79pub mod statistics;
80
81pub use assertion::{Assertion, AssertionResult};
82pub use clustering::{
83    cluster_failures, AssertionFailureKey, ExecutionFailureCategory, FailureCause, FailureCluster,
84    FailureClusterReport, FailureEvidenceRef, FailureOccurrence, FailureSignature,
85    FailureTermination, FAILURE_CLUSTER_REPORT_FORMAT_VERSION,
86};
87pub use compare::{
88    compare, AssertionDelta, CaseChange, CaseDelta, CompareThresholds, ComparisonReport,
89    LatencyDelta, Regression,
90};
91pub use dataset::{
92    Dataset, EvalCase, Expectation, ExpectedToolCall, StatePredicate, DATASET_FORMAT_VERSION,
93};
94pub use error::{EvalError, Result};
95pub use evidence::{RunEvidence, RunStatus, ToolCallRecord};
96pub use experiment::{
97    AssertionPassRate, CaseReport, CaseRunReport, ExperimentConfig, ExperimentReport,
98    ExperimentRunner, LatencyStats, PreparedRun, ReportSummary, REPORT_FORMAT_VERSION,
99};
100pub use feedback::{
101    AnnotationQueue, AnnotationStatus, AnnotationTask, ResolutionAuthority, ReviewCandidate,
102    ReviewDecision, ReviewLease, ReviewResolution, ReviewRubric, ReviewSubmission, RubricCriterion,
103    StoredReview, TraceRef, FEEDBACK_FORMAT_VERSION,
104};
105pub use gate::{
106    evaluate_gate, GateCheck, GateDecision, GateMetric, GateOutcome, GatePolicy,
107    GATE_DECISION_FORMAT_VERSION, GATE_POLICY_FORMAT_VERSION,
108};
109pub use judge::{
110    JudgeModel, JudgeRequest, JudgeVerdict, ModelJudge, RuleBasedJudge,
111    DEFAULT_MODEL_JUDGE_MAX_REQUEST_BYTES, DEFAULT_MODEL_JUDGE_MAX_RESPONSE_BYTES,
112    DEFAULT_MODEL_JUDGE_PASS_SCORE, MAX_MODEL_JUDGE_RATIONALE_BYTES,
113};
114pub use statistics::{
115    detect_pass_rate_regression, StatisticalDecision, StatisticalRegressionConfig,
116    StatisticalRegressionReport, STATISTICAL_REGRESSION_FORMAT_VERSION,
117};