systemprompt_cli/commands/admin/evals/
run.rs

1//! `admin evals run` — start an evaluation run.
2//!
3//! Copyright (c) systemprompt.io — Business Source License 1.1.
4//! See <https://systemprompt.io> for licensing details.
5
6use anyhow::Result;
7use chrono::{Duration, Utc};
8use clap::Args;
9use systemprompt_evaluation::{SampleFilter, SampleMode};
10use systemprompt_identifiers::ContextId;
11
12use super::shared::{eval_context, run_request};
13use crate::context::CommandContext;
14use crate::shared::CommandOutput;
15
16#[derive(Debug, Args)]
17pub struct RunArgs {
18    #[arg(
19        long,
20        default_value_t = 20,
21        help = "Number of recent requests to judge"
22    )]
23    pub sample_size: i64,
24
25    #[arg(long, default_value_t = 24, help = "Sampling window in hours")]
26    pub window_hours: i64,
27
28    #[arg(long, help = "Rubric name (defaults to the built-in 'default' rubric)")]
29    pub rubric: Option<String>,
30
31    #[arg(
32        long,
33        help = "Judge provider (defaults to the configured default provider)"
34    )]
35    pub judge_provider: Option<String>,
36
37    #[arg(long, help = "Judge model (defaults to the provider's default model)")]
38    pub judge_model: Option<String>,
39
40    #[arg(long, help = "Only sample requests served by this provider")]
41    pub provider: Option<String>,
42
43    #[arg(long, help = "Only sample requests served by this model")]
44    pub model: Option<String>,
45
46    #[arg(long, help = "Abort once judge spend reaches this many microdollars")]
47    pub budget_microdollars: Option<i64>,
48
49    #[arg(
50        long,
51        help = "Judge whole conversations (latest request per context) instead of individual \
52                requests"
53    )]
54    pub conversations: bool,
55
56    #[arg(long, help = "Only sample requests from this context", value_parser = parse_context_id)]
57    pub context_id: Option<ContextId>,
58}
59
60fn parse_context_id(raw: &str) -> Result<ContextId, String> {
61    ContextId::try_new(raw.to_owned()).map_err(|e| e.to_string())
62}
63
64pub async fn execute(args: RunArgs, ctx: &CommandContext) -> Result<CommandOutput> {
65    let eval = eval_context(ctx).await?;
66
67    let mut filter = SampleFilter::with_limit(args.sample_size)
68        .since(Utc::now() - Duration::hours(args.window_hours));
69    if let Some(provider) = args.provider {
70        filter = filter.provider(provider);
71    }
72    if let Some(model) = args.model {
73        filter = filter.model(model);
74    }
75    if let Some(context_id) = args.context_id {
76        filter = filter.context_id(context_id);
77    }
78    if args.conversations {
79        filter = filter.mode(SampleMode::Conversation);
80    }
81
82    let request = run_request(
83        &eval,
84        super::shared::JudgeOptions {
85            judge_provider: args.judge_provider,
86            judge_model: args.judge_model,
87            rubric: args.rubric,
88            budget_microdollars: args.budget_microdollars,
89        },
90        filter,
91    );
92    let (run_id, report) = eval.evaluation.run_judge(request).await?;
93    Ok(CommandOutput::card_value(
94        "Evaluation run",
95        &super::shared::report_card(&run_id, report),
96    ))
97}
systemprompt_cli/commands/admin/evals/run.rs

systemprompt_cli/commands/admin/evals/
run.rs
