agent_infra_cli/

evaluate_cli.rs

//! Evaluate subcommands.
//!
//! Kept apart from the top-level command table because the evaluation
//! workflow has its own vocabulary: bundles, gates, reviews and metrics.

use std::path::PathBuf;
use std::time::Duration;

use agent_infra_sdk::{
    EvaluateClient, InfraClient, WaitOptions,
    evaluate_contract::{
        ClaimReviewRequest, CompareReportsRequest, CompleteReviewRequest, CreateEvalRunRequest,
        GatePolicy, GetOperationRequest, GetReportRequest, ListCaseRunsRequest,
        ListMetricCandidatesRequest, MetricHealthRequest, RegisterEnvironmentProfileRequest,
        RegisterEvalCaseRequest, RegisterEvaluatorVersionRequest, RegisterSuiteRequest,
        ReportComparison, SampleReviewsRequest,
    },
};
use anyhow::{Context, Result, bail};
use clap::Subcommand;

use crate::{CommandOutput, JsonInput, display_source, evaluate_bundle, read_input};

#[derive(Debug, Subcommand)]
pub enum EvaluateCommand {
    RegisterCase(JsonInput),
    RegisterEnvironment(JsonInput),
    RegisterEvaluator(JsonInput),
    RegisterSuite(JsonInput),
    /// Register a bundle whose versions are derived from its content.
    ///
    /// Re-pushing an unchanged bundle is a no-op, so nothing has to be
    /// version-bumped by hand and metric history stays continuous.
    Push {
        #[command(flatten)]
        input: JsonInput,
        /// Print the resolved versions without contacting the Gateway.
        #[arg(long)]
        dry_run: bool,
    },
    /// Create a durable task that a matching Runtime starts through POST /evaluate.
    #[command(alias = "run")]
    TaskCreate(JsonInput),
    /// Create a task, wait for it to finish, and print its report.
    RunWait {
        #[command(flatten)]
        input: JsonInput,
        #[arg(long, default_value_t = 1800, value_parser = clap::value_parser!(u64).range(1..=86_400))]
        wait_timeout: u64,
    },
    Operation {
        operation_id: String,
    },
    Cases(JsonInput),
    Report {
        report_id: String,
    },
    /// Compare two reports and exit non-zero when the gate blocks.
    Diff {
        #[arg(long)]
        report_id: String,
        #[arg(long)]
        baseline_report_id: String,
        /// GatePolicy JSON document. Defaults to the built-in policy.
        #[arg(long)]
        policy: Option<PathBuf>,
        /// Report the gate decision without failing the command.
        #[arg(long)]
        no_fail: bool,
    },
    /// Error-analysis review queue.
    Review {
        #[command(subcommand)]
        command: EvaluateReviewCommand,
    },
    /// Metric health and candidate metrics.
    Metrics {
        #[command(subcommand)]
        command: EvaluateMetricsCommand,
    },
}

#[derive(Debug, Subcommand)]
pub enum EvaluateReviewCommand {
    /// Draw a stratified review batch, passing cases included.
    Sample(JsonInput),
    /// Claim the next queued review, if any.
    Claim {
        #[arg(long, default_value_t = 300_000, value_parser = clap::value_parser!(i64).range(10_000..=3_600_000))]
        lease_ms: i64,
    },
    /// Record a reviewer's note, labels and verdict.
    Complete(JsonInput),
}

#[derive(Debug, Subcommand)]
pub enum EvaluateMetricsCommand {
    /// Per-metric statistics and retirement recommendations.
    Health {
        #[arg(long, default_value_t = 100, value_parser = clap::value_parser!(u32).range(1..=1_000))]
        window_runs: u32,
        #[arg(long)]
        evaluator_id: Option<String>,
    },
    /// Labelled failure modes ranked by how well they predict the verdict.
    Candidates {
        #[arg(long)]
        run_id: Option<String>,
        #[arg(long, default_value_t = 3)]
        minimum_occurrences: u64,
    },
}

pub(crate) async fn execute_evaluate(
    client: &InfraClient,
    command: EvaluateCommand,
) -> Result<CommandOutput> {
    let evaluate = client.evaluate();
    match command {
        EvaluateCommand::RegisterCase(input) => {
            let request: RegisterEvalCaseRequest = input.read()?;
            CommandOutput::json(evaluate.register_eval_case(&request).await?)
        }
        EvaluateCommand::RegisterEnvironment(input) => {
            let request: RegisterEnvironmentProfileRequest = input.read()?;
            CommandOutput::json(evaluate.register_environment_profile(&request).await?)
        }
        EvaluateCommand::RegisterEvaluator(input) => {
            let request: RegisterEvaluatorVersionRequest = input.read()?;
            CommandOutput::json(evaluate.register_evaluator_version(&request).await?)
        }
        EvaluateCommand::RegisterSuite(input) => {
            let request: RegisterSuiteRequest = input.read()?;
            CommandOutput::json(evaluate.register_suite(&request).await?)
        }
        EvaluateCommand::Push { input, dry_run } => {
            let bundle: evaluate_bundle::EvalBundle = input.read()?;
            let resolved = evaluate_bundle::resolve(&bundle)?;
            let summary = serde_json::json!({
                "suiteId": resolved.suite.suite_id,
                "suiteVersion": resolved.suite.version,
                "manifestDigest": resolved.suite.manifest_digest,
                "environmentProfiles": resolved
                    .environment_profiles
                    .iter()
                    .map(|profile| serde_json::json!({
                        "profileId": profile.profile_id,
                        "version": profile.version,
                    }))
                    .collect::<Vec<_>>(),
                "evaluators": resolved
                    .evaluators
                    .iter()
                    .map(|evaluator| serde_json::json!({
                        "evaluatorId": evaluator.evaluator_id,
                        "version": evaluator.version,
                        "class": evaluator.metric.class,
                    }))
                    .collect::<Vec<_>>(),
                "cases": resolved
                    .cases
                    .iter()
                    .map(|case| serde_json::json!({
                        "caseId": case.case_id,
                        "version": case.version,
                    }))
                    .collect::<Vec<_>>(),
                "applied": !dry_run,
            });
            if dry_run {
                return CommandOutput::json(summary);
            }
            // Dependencies first: a case registration is rejected unless its
            // environment and evaluators resolve, and a suite registration is
            // rejected unless every case version resolves.
            for profile in &resolved.environment_profiles {
                evaluate.register_environment_profile(profile).await?;
            }
            for evaluator in &resolved.evaluators {
                evaluate.register_evaluator_version(evaluator).await?;
            }
            for case in &resolved.cases {
                evaluate.register_eval_case(case).await?;
            }
            evaluate.register_suite(&resolved.suite).await?;
            CommandOutput::json(summary)
        }
        EvaluateCommand::TaskCreate(input) => {
            let request: CreateEvalRunRequest = input.read()?;
            let handle = evaluate.create_task(&request).await?;
            let operation = handle.refresh().await?;
            CommandOutput::json(serde_json::json!({
                "evaluateTaskId": operation.id,
                "operation": operation,
            }))
        }
        EvaluateCommand::RunWait {
            input,
            wait_timeout,
        } => {
            let request: CreateEvalRunRequest = input.read()?;
            let report = evaluate
                .run_and_wait(
                    &request,
                    WaitOptions::deadline(Duration::from_secs(wait_timeout)),
                )
                .await?;
            CommandOutput::json(report)
        }
        EvaluateCommand::Operation { operation_id } => {
            let request = GetOperationRequest { operation_id };
            let handle = evaluate.operation(request.operation_id).await?;
            CommandOutput::json(handle.refresh().await?)
        }
        EvaluateCommand::Cases(input) => {
            let request: ListCaseRunsRequest = input.read()?;
            CommandOutput::json(evaluate.list_case_runs(&request).await?)
        }
        EvaluateCommand::Report { report_id } => {
            let request = GetReportRequest { report_id };
            CommandOutput::json(evaluate.report(&request.report_id).await?)
        }
        EvaluateCommand::Diff {
            report_id,
            baseline_report_id,
            policy,
            no_fail,
        } => {
            let policy = policy
                .map(|path| -> Result<GatePolicy> {
                    let bytes = read_input(&path)?;
                    serde_json::from_slice(&bytes)
                        .with_context(|| format!("failed to parse {}", display_source(&path)))
                })
                .transpose()?;
            let comparison = evaluate
                .compare_reports(&CompareReportsRequest {
                    report_id,
                    baseline_report_id,
                    minimum_pass_rate: 0.0,
                    maximum_infra_failure_rate: 1.0,
                    significance_level: 0.05,
                    minimum_effect: 0.0,
                    policy,
                })
                .await?;
            if !comparison.gate_passed && !no_fail {
                // A blocking metric must fail the command so CI stops without
                // the caller having to parse the JSON.
                bail!("{}", describe_gate_failure(&comparison));
            }
            CommandOutput::json(comparison)
        }
        EvaluateCommand::Review { command } => execute_evaluate_review(evaluate, command).await,
        EvaluateCommand::Metrics { command } => execute_evaluate_metrics(evaluate, command).await,
    }
}

fn describe_gate_failure(comparison: &ReportComparison) -> String {
    let blocking = comparison
        .gate
        .blocking_metrics
        .iter()
        .map(|violation| {
            format!(
                "{} [{}] {}",
                violation.evaluator_id,
                violation.class.as_str(),
                violation.detail
            )
        })
        .collect::<Vec<_>>()
        .join("; ");
    if blocking.is_empty() {
        "evaluation gate failed".to_string()
    } else {
        format!("evaluation gate blocked by {blocking}")
    }
}

async fn execute_evaluate_review(
    evaluate: &EvaluateClient,
    command: EvaluateReviewCommand,
) -> Result<CommandOutput> {
    match command {
        EvaluateReviewCommand::Sample(input) => {
            let request: SampleReviewsRequest = input.read()?;
            CommandOutput::json(evaluate.sample_reviews(&request).await?)
        }
        EvaluateReviewCommand::Claim { lease_ms } => {
            match evaluate
                .claim_review(&ClaimReviewRequest { lease_ms })
                .await?
            {
                Some(review) => CommandOutput::json(review),
                None => CommandOutput::json(serde_json::json!({ "queued": false })),
            }
        }
        EvaluateReviewCommand::Complete(input) => {
            let request: CompleteReviewRequest = input.read()?;
            CommandOutput::json(evaluate.complete_review(&request).await?)
        }
    }
}

async fn execute_evaluate_metrics(
    evaluate: &EvaluateClient,
    command: EvaluateMetricsCommand,
) -> Result<CommandOutput> {
    match command {
        EvaluateMetricsCommand::Health {
            window_runs,
            evaluator_id,
        } => CommandOutput::json(
            evaluate
                .metric_health(&MetricHealthRequest {
                    window_runs,
                    evaluator_id,
                })
                .await?,
        ),
        EvaluateMetricsCommand::Candidates {
            run_id,
            minimum_occurrences,
        } => CommandOutput::json(
            evaluate
                .metric_candidates(&ListMetricCandidatesRequest {
                    run_id,
                    minimum_occurrences,
                })
                .await?,
        ),
    }
}