rusty-eval 0.1.5

Agent TestOps for Rusty — versioned evaluation datasets, deterministic assertions over recorded runs, experiment reports, and baseline-vs-candidate comparison.
Documentation
use std::cell::Cell;
use std::collections::BTreeSet;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::{Arc, Mutex};
use std::time::Duration;

use rusty_agent_runtime::error::Result as RuntimeResult;
use rusty_agent_runtime::graph::GraphBuilder;
use rusty_agent_runtime::node::NodeOutput;
use rusty_agent_runtime::state::StateSpec;
use rusty_eval::{
    Dataset, EvalCase, EvalError, Expectation, ExperimentConfig, ExperimentRunner, JudgeModel,
    JudgeRequest, JudgeVerdict, PreparedRun,
};
use serde_json::json;

#[derive(Default)]
struct ConcurrencyProbe {
    active: AtomicUsize,
    peak: AtomicUsize,
    completed: AtomicUsize,
}

struct ActiveRun {
    probe: Arc<ConcurrencyProbe>,
}

impl ActiveRun {
    fn start(probe: Arc<ConcurrencyProbe>) -> Self {
        let active = probe.active.fetch_add(1, Ordering::SeqCst) + 1;
        probe.peak.fetch_max(active, Ordering::SeqCst);
        Self { probe }
    }
}

impl Drop for ActiveRun {
    fn drop(&mut self) {
        self.probe.active.fetch_sub(1, Ordering::SeqCst);
        self.probe.completed.fetch_add(1, Ordering::SeqCst);
    }
}

fn dataset(case_count: usize) -> Dataset {
    let cases = (0..case_count)
        .map(|index| EvalCase {
            id: format!("case-{index}"),
            input: json!({"delay_ms": 8 + (case_count - index) as u64 * 4}),
            expect: Expectation::default(),
            tags: Vec::new(),
        })
        .collect();
    Dataset::new("parallel-suite", "1", cases).unwrap()
}

fn prepared_run(delay: Duration, probe: Arc<ConcurrencyProbe>) -> RuntimeResult<PreparedRun> {
    let mut builder = GraphBuilder::new();
    builder.add_node("work", move |_context| {
        let probe = Arc::clone(&probe);
        async move {
            let _active = ActiveRun::start(probe);
            tokio::time::sleep(delay).await;
            Ok(NodeOutput::empty())
        }
    });
    builder.set_entry_point("work");
    Ok(PreparedRun::new(builder.compile()?, StateSpec::new()))
}

#[tokio::test]
async fn parallel_runs_are_bounded_isolated_and_reported_in_source_order() {
    let dataset = dataset(4);
    let probe = Arc::new(ConcurrencyProbe::default());
    let journal_ids = Arc::new(Mutex::new(BTreeSet::new()));
    let runner = ExperimentRunner::new(
        ExperimentConfig::new()
            .with_runs_per_case(2)
            .with_max_concurrency(3),
    );

    let report = runner
        .run(&dataset, |case, journal| {
            assert!(journal_ids
                .lock()
                .unwrap()
                .insert(journal.run_id().to_owned()));
            prepared_run(
                Duration::from_millis(case.input["delay_ms"].as_u64().unwrap()),
                Arc::clone(&probe),
            )
        })
        .await
        .unwrap();

    assert_eq!(probe.peak.load(Ordering::SeqCst), 3);
    assert_eq!(probe.active.load(Ordering::SeqCst), 0);
    assert_eq!(probe.completed.load(Ordering::SeqCst), 8);
    assert_eq!(journal_ids.lock().unwrap().len(), 8);
    assert_eq!(report.max_concurrency, 3);
    assert_eq!(
        report
            .cases
            .iter()
            .map(|case| case.case_id.as_str())
            .collect::<Vec<_>>(),
        vec!["case-0", "case-1", "case-2", "case-3"]
    );
    assert!(report
        .cases
        .iter()
        .all(|case| { case.runs.iter().map(|run| run.repetition).eq([0, 1]) }));
}

#[tokio::test]
async fn sequential_default_never_overlaps_case_runs() {
    let dataset = dataset(3);
    let probe = Arc::new(ConcurrencyProbe::default());
    let runner = ExperimentRunner::new(
        ExperimentConfig::default()
            .with_runs_per_case(2)
            .with_max_concurrency(0),
    );

    let report = runner
        .run(&dataset, |case, _journal| {
            prepared_run(
                Duration::from_millis(case.input["delay_ms"].as_u64().unwrap()),
                Arc::clone(&probe),
            )
        })
        .await
        .unwrap();

    assert_eq!(report.summary.runs, 6);
    assert_eq!(report.max_concurrency, 1);
    assert_eq!(probe.peak.load(Ordering::SeqCst), 1);
    assert_eq!(probe.completed.load(Ordering::SeqCst), 6);
}

#[tokio::test]
async fn preparation_factory_retains_its_non_sync_api() {
    let dataset = dataset(2);
    let builds = Cell::new(0);
    let runner = ExperimentRunner::new(ExperimentConfig::new().with_max_concurrency(2));

    let report = runner
        .run(&dataset, |_case, _journal| {
            builds.set(builds.get() + 1);
            prepared_run(Duration::ZERO, Arc::new(ConcurrencyProbe::default()))
        })
        .await
        .unwrap();

    assert_eq!(builds.get(), 2);
    assert_eq!(report.summary.runs, 2);
}

struct FailingJudge {
    completed: Arc<AtomicUsize>,
}

#[async_trait::async_trait]
impl JudgeModel for FailingJudge {
    async fn judge(&self, request: &JudgeRequest) -> rusty_eval::Result<JudgeVerdict> {
        let delay = if request.case_id == "case-0" { 30 } else { 1 };
        tokio::time::sleep(Duration::from_millis(delay)).await;
        self.completed.fetch_add(1, Ordering::SeqCst);
        Err(EvalError::Judge(format!(
            "{} judge unavailable",
            request.case_id
        )))
    }
}

#[tokio::test]
async fn concurrent_failures_return_the_earliest_source_coordinate() {
    let dataset = dataset(4);
    let completed = Arc::new(AtomicUsize::new(0));
    let judge = Arc::new(FailingJudge {
        completed: Arc::clone(&completed),
    });
    let runner = ExperimentRunner::new(
        ExperimentConfig::new()
            .with_max_concurrency(2)
            .with_judge(judge),
    );

    let error = runner
        .run(&dataset, |_case, _journal| {
            prepared_run(Duration::ZERO, Arc::new(ConcurrencyProbe::default()))
        })
        .await
        .unwrap_err();

    assert_eq!(completed.load(Ordering::SeqCst), 2);
    assert!(error.to_string().contains("case `case-0`"));
    assert!(!error.to_string().contains("case-1 judge unavailable"));
}

#[tokio::test]
async fn sequential_infrastructure_failure_still_fails_fast() {
    let dataset = dataset(2);
    let completed = Arc::new(AtomicUsize::new(0));
    let judge = Arc::new(FailingJudge {
        completed: Arc::clone(&completed),
    });
    let runner = ExperimentRunner::new(ExperimentConfig::new().with_judge(judge));

    let error = runner
        .run(&dataset, |_case, _journal| {
            prepared_run(Duration::ZERO, Arc::new(ConcurrencyProbe::default()))
        })
        .await
        .unwrap_err();

    assert_eq!(completed.load(Ordering::SeqCst), 1);
    assert!(error.to_string().contains("case `case-0`"));
}