systemprompt_cli/commands/admin/evals/
mod.rs

1//! `admin evals` command group: judge runs over production AI traffic.
2//!
3//! Exposes [`EvalsCommands`] — launch an auto-improve judge run, list and
4//! inspect runs, replay a run's failures, and promote a sampled request into
5//! the golden case set. All inference goes through the process-local
6//! [`AiService`](systemprompt_ai::AiService) built from the active profile.
7//!
8//! Copyright (c) systemprompt.io — Business Source License 1.1.
9//! See <https://systemprompt.io> for licensing details.
10
11pub mod list;
12pub mod promote;
13pub mod replay;
14pub mod run;
15mod shared;
16pub mod show;
17
18use anyhow::Result;
19use clap::Subcommand;
20
21use crate::context::CommandContext;
22use crate::shared::render_result;
23
24#[derive(Debug, Subcommand)]
25pub enum EvalsCommands {
26    #[command(about = "Judge a sample of recent AI requests (auto-improve pass)")]
27    Run(run::RunArgs),
28
29    #[command(about = "List evaluation runs")]
30    List(list::ListArgs),
31
32    #[command(about = "Show a run and its results")]
33    Show(show::ShowArgs),
34
35    #[command(about = "Replay a run's failures as a new replay run")]
36    Replay(replay::ReplayArgs),
37
38    #[command(about = "Promote an AI request into the golden case set")]
39    Promote(promote::PromoteArgs),
40}
41
42pub async fn execute(cmd: EvalsCommands, ctx: &CommandContext) -> Result<()> {
43    let result = match cmd {
44        EvalsCommands::Run(args) => run::execute(args, ctx).await?,
45        EvalsCommands::List(args) => list::execute(args, ctx).await?,
46        EvalsCommands::Show(args) => show::execute(args, ctx).await?,
47        EvalsCommands::Replay(args) => replay::execute(args, ctx).await?,
48        EvalsCommands::Promote(args) => promote::execute(args, ctx).await?,
49    };
50    render_result(&result, &ctx.cli);
51    Ok(())
52}
systemprompt_cli/commands/admin/evals/mod.rs

systemprompt_cli/commands/admin/evals/
mod.rs
