systemprompt_cli/commands/admin/evals/
replay.rs

1//! `admin evals replay` — replay a run's failures against the provider.
2//!
3//! Copyright (c) systemprompt.io — Business Source License 1.1.
4//! See <https://systemprompt.io> for licensing details.
5
6use anyhow::Result;
7use clap::Args;
8use systemprompt_evaluation::SampleFilter;
9use systemprompt_identifiers::EvalRunId;
10
11use super::shared::{eval_context, run_request};
12use crate::context::CommandContext;
13use crate::shared::CommandOutput;
14
15#[derive(Debug, Args)]
16pub struct ReplayArgs {
17    #[arg(help = "Source run id whose failures should be replayed")]
18    pub run_id: String,
19
20    #[arg(long, help = "Rubric name (defaults to the built-in 'default' rubric)")]
21    pub rubric: Option<String>,
22
23    #[arg(
24        long,
25        help = "Judge provider (defaults to the configured default provider)"
26    )]
27    pub judge_provider: Option<String>,
28
29    #[arg(long, help = "Judge model (defaults to the provider's default model)")]
30    pub judge_model: Option<String>,
31
32    #[arg(long, help = "Abort once judge spend reaches this many microdollars")]
33    pub budget_microdollars: Option<i64>,
34}
35
36pub async fn execute(args: ReplayArgs, ctx: &CommandContext) -> Result<CommandOutput> {
37    let eval = eval_context(ctx).await?;
38    let source_run = EvalRunId::new(args.run_id);
39
40    let request = run_request(
41        &eval,
42        super::shared::JudgeOptions {
43            judge_provider: args.judge_provider,
44            judge_model: args.judge_model,
45            rubric: args.rubric,
46            budget_microdollars: args.budget_microdollars,
47        },
48        SampleFilter::default(),
49    );
50    let (run_id, report) = eval
51        .evaluation
52        .replay_failures(&source_run, request)
53        .await?;
54    Ok(CommandOutput::card_value(
55        "Replay run",
56        &super::shared::report_card(&run_id, report),
57    ))
58}
systemprompt_cli/commands/admin/evals/replay.rs

systemprompt_cli/commands/admin/evals/
replay.rs
