agent_infra_cli/

eval_cli.rs

//! `infra eval` — the Evaluate domain application.
//!
//! Evaluate now ships from its own repository; these commands speak its
//! snapshot-centric contract. The older `infra evaluate` commands remain for
//! the deployment that still runs the previous service.

use std::time::Duration;

use agent_infra_sdk::{
    InfraClient,
    eval_contract::{
        BindingTarget, BoundSnapshot, Capability, CompareReportsRequest, CreateRunRequest,
        EvaluationSubject, ExportBundleRequest, GatePolicy, JudgeAllowlistEntry,
        ListAdminBindingsRequest, ListAdminRunsRequest, ListAdminSnapshotsRequest,
        ListBindingsRequest, ListCaseRunsRequest, ListSnapshotsRequest, OwnerKind,
        PutBindingRequest, RegisterSnapshotRequest, SnapshotBundle, SnapshotRef, SnapshotSelection,
    },
};
use anyhow::{Context, Result, bail};
use clap::{Args, Subcommand};

use crate::{CommandOutput, JsonInput};

/// Polling stays deliberately modest so waiting in CI does not hammer the API.
const POLL_INTERVAL: Duration = Duration::from_secs(3);

#[derive(Debug, Subcommand)]
pub enum EvalCommand {
    /// List evaluation snapshots visible to this tenant.
    Snapshots(SnapshotFilter),
    /// Show one snapshot, e.g. `builtin/tool-use@2026-08-20.1`.
    Snapshot { reference: String },
    /// Validate a snapshot file's JSON schema locally, without contacting the Gateway.
    Validate(JsonInput),
    /// Register a tenant snapshot from a file.
    Publish(JsonInput),
    /// Bind snapshots to an agent or runtime.
    Bind(BindArgs),
    /// List bindings.
    Bindings(PageArgs),
    /// Remove a binding.
    Unbind(TargetArgs),
    /// Create an evaluation run.
    Run(RunArgs),
    /// Read an evaluate task.
    Operation { evaluate_task_id: String },
    /// List a run's case runs.
    Cases {
        #[arg(long)]
        run: String,
        #[command(flatten)]
        page: PageArgs,
    },
    /// Read a sealed report.
    Report {
        #[arg(long)]
        run: String,
        #[arg(long)]
        revision: Option<u32>,
    },
    /// Compare a run against a baseline; exits non-zero when the gate blocks.
    Gate(GateArgs),
    /// Cancel a run.
    Cancel {
        #[arg(long)]
        run: String,
    },
    /// Platform catalogue operations (requires an admin credential).
    Admin {
        #[command(subcommand)]
        command: EvalAdminCommand,
    },
}

#[derive(Debug, Args)]
pub struct SnapshotFilter {
    #[arg(long, value_parser = parse_owner)]
    pub owner: Option<OwnerKind>,
    #[arg(long, value_parser = parse_capability)]
    pub capability: Option<Capability>,
    #[arg(long)]
    pub limit: Option<usize>,
    #[arg(long)]
    pub cursor: Option<String>,
}

#[derive(Debug, Args)]
pub struct PageArgs {
    #[arg(long)]
    pub cursor: Option<String>,
    #[arg(long)]
    pub limit: Option<usize>,
}

#[derive(Debug, Args)]
pub struct TargetArgs {
    /// Agent under evaluation.
    #[arg(long, group = "target")]
    pub agent: Option<String>,
    /// Runtime under evaluation.
    #[arg(long, group = "target")]
    pub runtime: Option<String>,
}

impl TargetArgs {
    fn resolve(&self) -> Result<BindingTarget> {
        match (&self.agent, &self.runtime) {
            (Some(agent), None) => Ok(BindingTarget::agent(agent)),
            (None, Some(runtime)) => Ok(BindingTarget::runtime(runtime)),
            _ => bail!("exactly one of --agent or --runtime is required"),
        }
    }
}

#[derive(Debug, Args)]
pub struct BindArgs {
    #[command(flatten)]
    pub target: TargetArgs,
    /// Snapshot that must pass, e.g. `builtin/tool-use@2026-08-20.1`.
    #[arg(long = "snapshot", required = true)]
    pub snapshots: Vec<String>,
    /// Snapshot that is observed but never gates.
    #[arg(long = "optional")]
    pub optional: Vec<String>,
}

#[derive(Debug, Args)]
pub struct RunArgs {
    #[command(flatten)]
    pub target: TargetArgs,
    /// Run the target's binding instead of naming snapshots.
    #[arg(long)]
    pub use_binding: bool,
    #[arg(long = "snapshot")]
    pub snapshots: Vec<String>,
    /// Runtime that will execute the cases; required for out-of-sandbox runs.
    #[arg(long)]
    pub subject_runtime: Option<String>,
    /// Agent the runtime is expected to be bound to.
    #[arg(long)]
    pub subject_agent: Option<String>,
    #[arg(long, default_value = "")]
    pub subject_version: String,
    #[arg(long, default_value_t = 1)]
    pub samples: u32,
    #[arg(long)]
    pub idempotency_key: Option<String>,
    /// Poll until the run reaches a terminal state.
    #[arg(long)]
    pub wait: bool,
    #[arg(long, default_value_t = 1800)]
    pub wait_timeout: u64,
}

#[derive(Debug, Args)]
pub struct GateArgs {
    #[arg(long)]
    pub run: String,
    #[arg(long)]
    pub baseline: String,
    #[arg(long)]
    pub revision: Option<u32>,
    #[arg(long)]
    pub baseline_revision: Option<u32>,
    #[arg(long, default_value_t = 0.0)]
    pub minimum_pass_rate: f64,
    #[arg(long, default_value_t = 1.0)]
    pub maximum_infra_failure_rate: f64,
}

#[derive(Debug, Subcommand)]
pub enum EvalAdminCommand {
    Overview,
    Snapshots {
        #[arg(long)]
        lifecycle: Option<String>,
        #[arg(long)]
        tenant: Option<String>,
        #[command(flatten)]
        page: PageArgs,
    },
    Runs {
        #[arg(long)]
        tenant: Option<String>,
        #[command(flatten)]
        page: PageArgs,
    },
    Bindings {
        #[arg(long)]
        tenant: Option<String>,
        #[command(flatten)]
        page: PageArgs,
    },
    /// Register a platform snapshot; it starts as a draft.
    Register(JsonInput),
    Publish {
        reference: String,
    },
    Deprecate {
        reference: String,
        #[arg(long)]
        replacement: Option<String>,
    },
    /// Export published platform snapshots as a portable bundle.
    Export,
    /// Import a bundle produced by `export`.
    Import(JsonInput),
    /// Allow a judge model to back gating evaluators.
    ApproveJudge {
        model_ref: String,
        #[arg(long)]
        labeled_samples: u64,
        #[arg(long)]
        human_agreement: f64,
        #[arg(long, default_value = "standard")]
        cost_tier: String,
        #[arg(long, default_value = "")]
        notes: String,
    },
    /// List judge models and their gate approval state.
    Judges,
}

pub async fn execute_eval(client: &InfraClient, command: EvalCommand) -> Result<CommandOutput> {
    let eval = client.eval();
    match command {
        EvalCommand::Snapshots(filter) => CommandOutput::json(
            eval.snapshots(&ListSnapshotsRequest {
                owner: filter.owner,
                capability: filter.capability,
                snapshot_id: None,
                cursor: filter.cursor,
                limit: filter.limit,
            })
            .await?,
        ),
        EvalCommand::Snapshot { reference } => {
            CommandOutput::json(eval.snapshot(&parse_ref(&reference)?).await?)
        }
        EvalCommand::Validate(input) => {
            // Semantic checks that need includes, the judge allowlist or
            // project quota remain authoritative on the server.
            let request: RegisterSnapshotRequest = input.read()?;
            CommandOutput::json(serde_json::json!({
                "snapshotId": request.snapshot_id,
                "version": request.version,
                "cases": request.cases.len(),
                "includes": request.includes.len(),
                "schemaValid": true,
                "serverValidationRequired": true,
            }))
        }
        EvalCommand::Publish(input) => {
            let request: RegisterSnapshotRequest = input.read()?;
            CommandOutput::json(eval.register_snapshot(&request).await?)
        }
        EvalCommand::Bind(args) => {
            let mut snapshots = Vec::new();
            for reference in &args.snapshots {
                snapshots.push(BoundSnapshot {
                    reference: parse_ref(reference)?,
                    required: true,
                });
            }
            for reference in &args.optional {
                snapshots.push(BoundSnapshot {
                    reference: parse_ref(reference)?,
                    required: false,
                });
            }
            let target = args.target.resolve()?;
            // A binding write is create-or-update; read the current version so
            // an edit cannot silently clobber a concurrent one.
            let if_version = eval.binding(&target).await?.map(|binding| binding.version);
            CommandOutput::json(
                eval.put_binding(&PutBindingRequest {
                    target,
                    snapshots,
                    gate_policy: None,
                    triggers: Vec::new(),
                    if_version,
                })
                .await?,
            )
        }
        EvalCommand::Bindings(page) => CommandOutput::json(
            eval.bindings(&ListBindingsRequest {
                cursor: page.cursor,
                limit: page.limit,
                ..ListBindingsRequest::default()
            })
            .await?,
        ),
        EvalCommand::Unbind(target) => {
            CommandOutput::json(eval.delete_binding(&target.resolve()?).await?)
        }
        EvalCommand::Run(args) => run(client, args).await,
        EvalCommand::Operation { evaluate_task_id } => {
            CommandOutput::json(eval.operation(&evaluate_task_id).await?)
        }
        EvalCommand::Cases { run, page } => CommandOutput::json(
            eval.case_runs(&ListCaseRunsRequest {
                run_id: run,
                cursor: page.cursor,
                limit: page.limit,
                ..ListCaseRunsRequest::default()
            })
            .await?,
        ),
        EvalCommand::Report { run, revision } => {
            CommandOutput::json(eval.report(&run, revision).await?)
        }
        EvalCommand::Gate(args) => gate(client, args).await,
        EvalCommand::Cancel { run } => CommandOutput::json(eval.cancel_run(&run).await?),
        EvalCommand::Admin { command } => admin(client, command).await,
    }
}

async fn run(client: &InfraClient, args: RunArgs) -> Result<CommandOutput> {
    let eval = client.eval();
    let target = args.target.resolve()?;

    validate_run_selection(args.use_binding, &args.snapshots)?;

    let mut snapshots = Vec::new();
    for reference in &args.snapshots {
        snapshots.push(SnapshotSelection {
            reference: parse_ref(reference)?,
            case_ids: Vec::new(),
        });
    }

    let subject = resolve_subject(
        &target,
        args.subject_agent.as_deref(),
        args.subject_runtime.as_deref(),
    )?;

    let request = CreateRunRequest {
        target,
        use_binding: args.use_binding,
        snapshots,
        subject,
        subject_version: args.subject_version,
        execution_override: None,
        baseline_report_id: None,
        seed: 0,
        concurrency: 8,
        samples: args.samples,
        experiment_matrix: Vec::new(),
        idempotency_key: args.idempotency_key.unwrap_or_default(),
    };

    let operation = eval.create_run(&request).await?;
    if !args.wait {
        return CommandOutput::json(operation);
    }

    let deadline = std::time::Instant::now() + Duration::from_secs(args.wait_timeout);
    let mut latest = operation;
    while !latest.state.is_terminal() {
        if std::time::Instant::now() >= deadline {
            bail!(
                "run {} did not finish within {}s ({} of {} cases done)",
                latest.run_id,
                args.wait_timeout,
                latest.completed_cases,
                latest.total_cases
            );
        }
        tokio::time::sleep(POLL_INTERVAL).await;
        latest = eval.operation(&latest.id).await?;
    }

    if latest.report_id.is_some() {
        CommandOutput::json(eval.report(&latest.run_id, None).await?)
    } else {
        CommandOutput::json(latest)
    }
}

fn validate_run_selection(use_binding: bool, snapshots: &[String]) -> Result<()> {
    match (use_binding, snapshots.is_empty()) {
        (true, false) => bail!("--use-binding cannot be combined with --snapshot"),
        (false, true) => bail!("one or more --snapshot values or --use-binding is required"),
        _ => Ok(()),
    }
}

fn resolve_subject(
    target: &BindingTarget,
    subject_agent: Option<&str>,
    subject_runtime: Option<&str>,
) -> Result<Option<EvaluationSubject>> {
    match (subject_agent, subject_runtime) {
        (Some(agent_id), Some(runtime_id)) => Ok(Some(EvaluationSubject {
            agent_id: agent_id.to_string(),
            runtime_id: runtime_id.to_string(),
        })),
        (None, Some(runtime_id))
            if target.kind == agent_infra_sdk::eval_contract::TargetKind::Agent =>
        {
            Ok(Some(EvaluationSubject {
                agent_id: target.id.clone(),
                runtime_id: runtime_id.to_string(),
            }))
        }
        (None, None) => Ok(None),
        _ => bail!(
            "--subject-agent and --subject-runtime must be provided together unless the target is an agent"
        ),
    }
}

async fn gate(client: &InfraClient, args: GateArgs) -> Result<CommandOutput> {
    let decision = client
        .eval()
        .compare_reports(&CompareReportsRequest {
            run_id: args.run,
            revision: args.revision,
            baseline_run_id: args.baseline,
            baseline_revision: args.baseline_revision,
            policy: GatePolicy {
                minimum_pass_rate: args.minimum_pass_rate,
                maximum_infra_failure_rate: args.maximum_infra_failure_rate,
                ..GatePolicy::default()
            },
        })
        .await?;

    if !decision.allowed {
        let reasons = decision
            .violations
            .iter()
            .map(|violation| format!("{}: {}", violation.evaluator_id, violation.reason))
            .collect::<Vec<_>>()
            .join("; ");
        bail!("evaluation gate blocked the release: {reasons}");
    }
    CommandOutput::json(decision)
}

async fn admin(client: &InfraClient, command: EvalAdminCommand) -> Result<CommandOutput> {
    let eval = client.eval();
    match command {
        EvalAdminCommand::Overview => CommandOutput::json(eval.admin_overview().await?),
        EvalAdminCommand::Snapshots {
            lifecycle,
            tenant,
            page,
        } => {
            let lifecycle = lifecycle
                .map(|value| {
                    serde_json::from_value(serde_json::Value::String(value.clone()))
                        .with_context(|| format!("unknown lifecycle `{value}`"))
                })
                .transpose()?;
            CommandOutput::json(
                eval.admin_snapshots(&ListAdminSnapshotsRequest {
                    lifecycle,
                    tenant_id: tenant,
                    cursor: page.cursor,
                    limit: page.limit,
                    ..ListAdminSnapshotsRequest::default()
                })
                .await?,
            )
        }
        EvalAdminCommand::Runs { tenant, page } => CommandOutput::json(
            eval.admin_runs(&ListAdminRunsRequest {
                tenant_id: tenant,
                cursor: page.cursor,
                limit: page.limit,
                ..ListAdminRunsRequest::default()
            })
            .await?,
        ),
        EvalAdminCommand::Bindings { tenant, page } => CommandOutput::json(
            eval.admin_bindings(&ListAdminBindingsRequest {
                tenant_id: tenant,
                cursor: page.cursor,
                limit: page.limit,
                ..ListAdminBindingsRequest::default()
            })
            .await?,
        ),
        EvalAdminCommand::Register(input) => {
            let request: RegisterSnapshotRequest = input.read()?;
            CommandOutput::json(eval.admin_register_snapshot(&request).await?)
        }
        EvalAdminCommand::Publish { reference } => {
            CommandOutput::json(eval.admin_publish(&parse_ref(&reference)?).await?)
        }
        EvalAdminCommand::Deprecate {
            reference,
            replacement,
        } => {
            let replacement = replacement.as_deref().map(parse_ref).transpose()?;
            CommandOutput::json(
                eval.admin_deprecate(&parse_ref(&reference)?, replacement)
                    .await?,
            )
        }
        EvalAdminCommand::Export => {
            CommandOutput::json(eval.admin_export(&ExportBundleRequest::default()).await?)
        }
        EvalAdminCommand::Import(input) => {
            let bundle: SnapshotBundle = input.read()?;
            CommandOutput::json(eval.admin_import(&bundle).await?)
        }
        EvalAdminCommand::ApproveJudge {
            model_ref,
            labeled_samples,
            human_agreement,
            cost_tier,
            notes,
        } => CommandOutput::json(
            eval.admin_approve_judge(&JudgeAllowlistEntry {
                model_ref,
                gate_approved: true,
                labeled_samples,
                human_agreement,
                cost_tier,
                notes,
                updated_at_ms: now_ms(),
            })
            .await?,
        ),
        EvalAdminCommand::Judges => CommandOutput::json(eval.admin_judge_allowlist().await?),
    }
}

fn parse_ref(value: &str) -> Result<SnapshotRef> {
    value
        .parse()
        .with_context(|| format!("invalid snapshot reference `{value}`"))
}

fn parse_owner(value: &str) -> Result<OwnerKind, String> {
    match value {
        "builtin" => Ok(OwnerKind::Builtin),
        "tenant" => Ok(OwnerKind::Tenant),
        other => Err(format!(
            "unknown owner `{other}`; expected builtin or tenant"
        )),
    }
}

fn parse_capability(value: &str) -> Result<Capability, String> {
    serde_json::from_value(serde_json::Value::String(value.to_string())).map_err(|_| {
        let known = Capability::ALL
            .iter()
            .map(|capability| capability.as_str())
            .collect::<Vec<_>>()
            .join(", ");
        format!("unknown capability `{value}`; expected one of {known}")
    })
}

fn now_ms() -> i64 {
    std::time::SystemTime::now()
        .duration_since(std::time::UNIX_EPOCH)
        .map(|elapsed| elapsed.as_millis() as i64)
        .unwrap_or_default()
}

#[cfg(test)]
mod tests {
    use super::{
        EvalAdminCommand, EvalCommand, TargetArgs, parse_capability, parse_ref, resolve_subject,
        validate_run_selection,
    };
    use crate::{Cli, Command};
    use agent_infra_sdk::eval_contract::BindingTarget;
    use clap::Parser;

    fn parse(arguments: &[&str]) -> Command {
        Cli::try_parse_from(arguments).expect("parses").command
    }

    #[test]
    fn a_run_can_name_snapshots_or_use_the_binding() {
        let explicit = parse(&[
            "infra",
            "eval",
            "run",
            "--agent",
            "agent-1",
            "--snapshot",
            "builtin/tool-use@1",
            "--subject-runtime",
            "rt-1",
        ]);
        let Command::Eval {
            command: EvalCommand::Run(args),
        } = explicit
        else {
            panic!("expected a run command");
        };
        assert_eq!(args.snapshots, vec!["builtin/tool-use@1"]);
        assert!(!args.use_binding);

        let bound = parse(&[
            "infra",
            "eval",
            "run",
            "--agent",
            "agent-1",
            "--use-binding",
        ]);
        let Command::Eval {
            command: EvalCommand::Run(args),
        } = bound
        else {
            panic!("expected a run command");
        };
        assert!(args.use_binding);
    }

    #[test]
    fn a_target_must_be_exactly_one_of_agent_or_runtime() {
        let neither = TargetArgs {
            agent: None,
            runtime: None,
        };
        assert!(neither.resolve().is_err());

        let both = TargetArgs {
            agent: Some("a".into()),
            runtime: Some("r".into()),
        };
        assert!(both.resolve().is_err());

        assert_eq!(
            TargetArgs {
                agent: Some("a".into()),
                runtime: None,
            }
            .resolve()
            .unwrap()
            .id,
            "a"
        );
    }

    #[test]
    fn a_bad_snapshot_reference_names_itself_in_the_error() {
        let error = parse_ref("tool-use@1").unwrap_err().to_string();
        assert!(error.contains("tool-use@1"), "{error}");
    }

    #[test]
    fn an_unknown_capability_lists_the_accepted_values() {
        let error = parse_capability("vibes").unwrap_err();
        assert!(error.contains("tool_use"), "{error}");
    }

    #[test]
    fn admin_commands_are_reachable() {
        let overview = parse(&["infra", "eval", "admin", "overview"]);
        assert!(matches!(
            overview,
            Command::Eval {
                command: EvalCommand::Admin {
                    command: EvalAdminCommand::Overview
                }
            }
        ));

        let approve = parse(&[
            "infra",
            "eval",
            "admin",
            "approve-judge",
            "judge-a@3",
            "--labeled-samples",
            "80",
            "--human-agreement",
            "0.94",
        ]);
        let Command::Eval {
            command:
                EvalCommand::Admin {
                    command:
                        EvalAdminCommand::ApproveJudge {
                            labeled_samples, ..
                        },
                },
        } = approve
        else {
            panic!("expected approve-judge");
        };
        assert_eq!(labeled_samples, 80);
    }

    #[test]
    fn run_selection_is_exactly_one_of_binding_or_snapshots() {
        assert!(validate_run_selection(true, &[]).is_ok());
        assert!(validate_run_selection(false, &["tenant/suite@1".into()]).is_ok());
        assert!(validate_run_selection(false, &[]).is_err());
        assert!(validate_run_selection(true, &["tenant/suite@1".into()]).is_err());
    }

    #[test]
    fn an_agent_target_supplies_the_subject_agent() {
        let subject = resolve_subject(&BindingTarget::agent("agent-1"), None, Some("runtime-1"))
            .unwrap()
            .unwrap();
        assert_eq!(subject.agent_id, "agent-1");
        assert_eq!(subject.runtime_id, "runtime-1");

        assert!(
            resolve_subject(
                &BindingTarget::runtime("runtime-1"),
                None,
                Some("runtime-1")
            )
            .is_err()
        );
    }
}