agent-infra-cli 0.5.0

//! Content-addressed eval bundles.
//!
//! Registering an evaluation suite means registering evaluators, environment
//! profiles and the suite itself, each under an immutable version. Choosing
//! and bumping those version strings by hand is the single largest source of
//! friction: a forgotten bump silently reuses the old definition, and an
//! unnecessary bump breaks the history that flake and metric-health
//! statistics depend on.
//!
//! A bundle therefore declares resources without versions and derives each
//! one from its own content. Re-pushing an unchanged bundle resolves to the
//! same versions, which the immutable catalog accepts as an identical replay.

use std::collections::{BTreeMap, HashMap};

use agent_infra_sdk::evaluate_contract::{
    EvalBudget, EvalTurn, EvaluatorImplementation, MetricSpec, RegisterEnvironmentProfileRequest,
    RegisterEvalCaseRequest, RegisterEvaluatorVersionRequest, RegisterSuiteRequest, TypedAssertion,
    VerifyCommand, VersionedResourceRef,
};
use anyhow::{Context, Result, bail};
use serde::{Deserialize, Serialize};
use sha2::{Digest, Sha256};

#[derive(Debug, Clone, Deserialize, Serialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct EvalBundle {
    pub suite_id: String,
    #[serde(default)]
    pub environment_profiles: Vec<BundleEnvironmentProfile>,
    #[serde(default)]
    pub evaluators: Vec<BundleEvaluator>,
    pub cases: Vec<BundleCase>,
}

#[derive(Debug, Clone, Deserialize, Serialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct BundleEnvironmentProfile {
    pub profile_id: String,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub space_id: Option<String>,
    #[serde(default)]
    pub metadata: BTreeMap<String, String>,
}

#[derive(Debug, Clone, Deserialize, Serialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct BundleEvaluator {
    pub evaluator_id: String,
    pub implementation: EvaluatorImplementation,
    #[serde(default)]
    pub metric: MetricSpec,
}

#[derive(Debug, Clone, Deserialize, Serialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct BundleCase {
    pub id: String,
    pub category: String,
    #[serde(default)]
    pub turns: Vec<EvalTurn>,
    #[serde(default)]
    pub workspace_files: BTreeMap<String, String>,
    #[serde(default)]
    pub assertions: Vec<TypedAssertion>,
    /// Environment profile id declared in this bundle.
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub environment_profile: Option<String>,
    /// Evaluator ids declared in this bundle.
    #[serde(default)]
    pub evaluators: Vec<String>,
    #[serde(default)]
    pub verify_commands: Vec<VerifyCommand>,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub fixture_ref: Option<String>,
    #[serde(default)]
    pub budget: EvalBudget,
}

/// Registration requests with every version resolved from content.
#[derive(Debug, Clone)]
pub struct ResolvedBundle {
    pub environment_profiles: Vec<RegisterEnvironmentProfileRequest>,
    pub evaluators: Vec<RegisterEvaluatorVersionRequest>,
    pub cases: Vec<RegisterEvalCaseRequest>,
    pub suite: RegisterSuiteRequest,
}

/// Derive versions and rewrite the bundle's id references into versioned ones.
pub fn resolve(bundle: &EvalBundle) -> Result<ResolvedBundle> {
    if bundle.suite_id.trim().is_empty() {
        bail!("bundle requires a suiteId");
    }
    if bundle.cases.is_empty() {
        bail!("bundle requires at least one case");
    }

    let mut profile_versions = HashMap::with_capacity(bundle.environment_profiles.len());
    let environment_profiles = bundle
        .environment_profiles
        .iter()
        .map(|profile| {
            let version = content_version(profile)?;
            if profile_versions
                .insert(profile.profile_id.as_str(), version.clone())
                .is_some()
            {
                bail!("duplicate environment profile '{}'", profile.profile_id);
            }
            Ok(RegisterEnvironmentProfileRequest {
                profile_id: profile.profile_id.clone(),
                version,
                space_id: profile.space_id.clone(),
                metadata: profile.metadata.clone(),
            })
        })
        .collect::<Result<Vec<_>>>()?;

    let mut evaluator_versions = HashMap::with_capacity(bundle.evaluators.len());
    let evaluators = bundle
        .evaluators
        .iter()
        .map(|evaluator| {
            let version = content_version(evaluator)?;
            if evaluator_versions
                .insert(evaluator.evaluator_id.as_str(), version.clone())
                .is_some()
            {
                bail!("duplicate evaluator '{}'", evaluator.evaluator_id);
            }
            Ok(RegisterEvaluatorVersionRequest {
                evaluator_id: evaluator.evaluator_id.clone(),
                version,
                implementation: evaluator.implementation.clone(),
                metric: evaluator.metric.clone(),
            })
        })
        .collect::<Result<Vec<_>>>()?;

    let cases = bundle
        .cases
        .iter()
        .map(|case| {
            let environment_profile = case
                .environment_profile
                .as_ref()
                .map(|id| {
                    profile_versions
                        .get(id.as_str())
                        .map(|version| VersionedResourceRef {
                            id: id.clone(),
                            version: version.clone(),
                        })
                        .with_context(|| {
                            format!(
                                "case '{}' references unknown environment profile '{id}'",
                                case.id
                            )
                        })
                })
                .transpose()?;
            let evaluator_refs = case
                .evaluators
                .iter()
                .map(|id| {
                    evaluator_versions
                        .get(id.as_str())
                        .map(|version| VersionedResourceRef {
                            id: id.clone(),
                            version: version.clone(),
                        })
                        .with_context(|| {
                            format!("case '{}' references unknown evaluator '{id}'", case.id)
                        })
                })
                .collect::<Result<Vec<_>>>()?;
            let mut case = RegisterEvalCaseRequest {
                case_id: case.id.clone(),
                // Replaced below once the resolved content is known.
                version: String::new(),
                category: case.category.clone(),
                turns: case.turns.clone(),
                workspace_files: case.workspace_files.clone(),
                assertions: case.assertions.clone(),
                environment_profile,
                evaluator_versions: evaluator_refs,
                verify_commands: case.verify_commands.clone(),
                fixture_ref: case.fixture_ref.clone(),
                budget: case.budget.clone(),
            };
            case.version = content_version(&case)?;
            Ok(case)
        })
        .collect::<Result<Vec<_>>>()?;

    let mut case_ids = HashMap::with_capacity(cases.len());
    let case_refs = cases
        .iter()
        .map(|case| {
            if case_ids.insert(case.case_id.as_str(), ()).is_some() {
                bail!("duplicate case '{}'", case.case_id);
            }
            Ok(VersionedResourceRef {
                id: case.case_id.clone(),
                version: case.version.clone(),
            })
        })
        .collect::<Result<Vec<_>>>()?;

    // The suite version covers the resolved case references, so a change to
    // any case or evaluator also produces a new suite version.
    let manifest_digest = digest(&case_refs)?;
    let version = short_version(&manifest_digest);
    Ok(ResolvedBundle {
        environment_profiles,
        evaluators,
        cases,
        suite: RegisterSuiteRequest {
            suite_id: bundle.suite_id.clone(),
            version,
            manifest_digest,
            schema_version: 1,
            cases: case_refs,
        },
    })
}

fn content_version(value: &impl Serialize) -> Result<String> {
    Ok(short_version(&digest(value)?))
}

fn digest(value: &impl Serialize) -> Result<String> {
    Ok(format!(
        "sha256:{:x}",
        Sha256::digest(serde_json::to_vec(value)?)
    ))
}

/// 16 hex characters is 64 bits of the digest: collision-free in practice for
/// one project's catalog, and short enough to read in a report.
fn short_version(digest: &str) -> String {
    format!("sha256-{}", &digest["sha256:".len()..][..16])
}

#[cfg(test)]
mod tests {
    use super::*;

    fn bundle() -> EvalBundle {
        EvalBundle {
            suite_id: "core".into(),
            environment_profiles: vec![BundleEnvironmentProfile {
                profile_id: "coding".into(),
                space_id: Some("space-1".into()),
                metadata: BTreeMap::new(),
            }],
            evaluators: vec![BundleEvaluator {
                evaluator_id: "grounded".into(),
                implementation: EvaluatorImplementation::BuiltinAssertions {
                    assertions: vec![TypedAssertion::OutputContains {
                        turn: 0,
                        values: vec!["ok".into()],
                        any: false,
                    }],
                },
                metric: MetricSpec::default(),
            }],
            cases: vec![BundleCase {
                id: "answer".into(),
                category: "correctness".into(),
                turns: vec![EvalTurn {
                    user: "say ok".into(),
                }],
                workspace_files: BTreeMap::new(),
                assertions: Vec::new(),
                environment_profile: Some("coding".into()),
                evaluators: vec!["grounded".into()],
                verify_commands: Vec::new(),
                fixture_ref: None,
                budget: EvalBudget::default(),
            }],
        }
    }

    #[test]
    fn identical_content_resolves_to_identical_versions() {
        let first = resolve(&bundle()).unwrap();
        let second = resolve(&bundle()).unwrap();
        assert_eq!(first.suite.version, second.suite.version);
        assert_eq!(first.evaluators[0].version, second.evaluators[0].version);
    }

    #[test]
    fn changing_an_evaluator_changes_both_its_version_and_the_suite_version() {
        let baseline = resolve(&bundle()).unwrap();
        let mut changed = bundle();
        changed.evaluators[0].implementation = EvaluatorImplementation::BuiltinAssertions {
            assertions: vec![TypedAssertion::OutputContains {
                turn: 0,
                values: vec!["different".into()],
                any: false,
            }],
        };
        let resolved = resolve(&changed).unwrap();
        assert_ne!(
            baseline.evaluators[0].version,
            resolved.evaluators[0].version
        );
        assert_ne!(baseline.suite.version, resolved.suite.version);
    }

    #[test]
    fn changing_a_metric_class_changes_the_evaluator_version() {
        let baseline = resolve(&bundle()).unwrap();
        let mut changed = bundle();
        changed.evaluators[0].metric.class =
            agent_infra_sdk::evaluate_contract::MetricClass::Guardrail;
        let resolved = resolve(&changed).unwrap();
        assert_ne!(
            baseline.evaluators[0].version,
            resolved.evaluators[0].version
        );
    }

    #[test]
    fn case_references_are_rewritten_into_versioned_references() {
        let resolved = resolve(&bundle()).unwrap();
        let case = &resolved.cases[0];
        assert_eq!(
            case.evaluator_versions[0].version,
            resolved.evaluators[0].version
        );
        assert_eq!(
            case.environment_profile.as_ref().unwrap().version,
            resolved.environment_profiles[0].version
        );
        assert_eq!(
            resolved.suite.cases[0],
            VersionedResourceRef {
                id: case.case_id.clone(),
                version: case.version.clone(),
            }
        );
    }

    #[test]
    fn changing_one_case_leaves_the_other_case_versions_untouched() {
        let mut two_cases = bundle();
        let mut second = two_cases.cases[0].clone();
        second.id = "second".into();
        two_cases.cases.push(second);
        let baseline = resolve(&two_cases).unwrap();

        let mut changed = two_cases.clone();
        changed.cases[0].turns = vec![EvalTurn {
            user: "say something else".into(),
        }];
        let resolved = resolve(&changed).unwrap();

        assert_ne!(baseline.cases[0].version, resolved.cases[0].version);
        assert_eq!(baseline.cases[1].version, resolved.cases[1].version);
        assert_ne!(baseline.suite.version, resolved.suite.version);
    }

    #[test]
    fn an_unknown_reference_is_rejected_before_anything_is_registered() {
        let mut invalid = bundle();
        invalid.cases[0].evaluators = vec!["missing".into()];
        let error = resolve(&invalid).unwrap_err().to_string();
        assert!(error.contains("unknown evaluator 'missing'"), "{error}");
    }

    #[test]
    fn duplicate_declarations_are_rejected() {
        let mut invalid = bundle();
        let duplicate = invalid.evaluators[0].clone();
        invalid.evaluators.push(duplicate);
        assert!(
            resolve(&invalid)
                .unwrap_err()
                .to_string()
                .contains("duplicate evaluator")
        );
    }
}