systemprompt_cli/commands/admin/evals/
promote.rs

1//! `admin evals promote` — promote an AI request into the golden case set.
2//!
3//! Copyright (c) systemprompt.io — Business Source License 1.1.
4//! See <https://systemprompt.io> for licensing details.
5
6use anyhow::{Context, Result};
7use clap::Args;
8use systemprompt_evaluation::{NewCaseParams, SampleFilter};
9use systemprompt_identifiers::AiRequestId;
10
11use super::shared::eval_context;
12use crate::context::CommandContext;
13use crate::shared::CommandOutput;
14
15#[derive(Debug, Args)]
16pub struct PromoteArgs {
17    #[arg(help = "AI request id to promote into the golden case set")]
18    pub ai_request_id: String,
19
20    #[arg(long, help = "Case name (defaults to the request id)")]
21    pub name: Option<String>,
22
23    #[arg(long, help = "Expected behaviour the judge should score against")]
24    pub expectation: Option<String>,
25
26    #[arg(long, value_delimiter = ',', help = "Comma-separated tags")]
27    pub tags: Vec<String>,
28}
29
30pub async fn execute(args: PromoteArgs, ctx: &CommandContext) -> Result<CommandOutput> {
31    let eval = eval_context(ctx).await?;
32
33    let filter = SampleFilter::with_limit(1).ids(vec![args.ai_request_id.clone()]);
34    let sampled = eval
35        .evaluation
36        .sample(&filter)
37        .await?
38        .into_iter()
39        .next()
40        .with_context(|| {
41            format!(
42                "AI request {} not found or has no completed transcript",
43                args.ai_request_id
44            )
45        })?;
46
47    let prompt = sampled.canonical_prompt();
48    let case_id = eval
49        .evaluation
50        .promote_case(&NewCaseParams {
51            name: args.name.unwrap_or_else(|| args.ai_request_id.clone()),
52            prompt,
53            source_ai_request_id: Some(AiRequestId::new(args.ai_request_id)),
54            expectation: args.expectation,
55            tags: args.tags,
56            created_by: eval.admin_id.clone(),
57            prepared_body_sha256: sampled.prepared_body_sha256,
58        })
59        .await?;
60
61    Ok(CommandOutput::text(format!("Promoted case {case_id}")))
62}
systemprompt_cli/commands/admin/evals/promote.rs

systemprompt_cli/commands/admin/evals/
promote.rs
