systemprompt_evaluation/repository/
results.rs

1//! Repository for evaluation results.
2//!
3//! Copyright (c) systemprompt.io — Business Source License 1.1.
4//! See <https://systemprompt.io> for licensing details.
5
6use sqlx::PgPool;
7use std::str::FromStr;
8use std::sync::Arc;
9use systemprompt_database::DbPool;
10use systemprompt_identifiers::{AiRequestId, EvalCaseId, EvalResultId, EvalRunId};
11
12use crate::error::{EvaluationError, Result};
13use crate::models::{EvalResult, NewResultParams, Verdict};
14
15#[derive(Debug, Clone)]
16pub struct EvalResultRepository {
17    pool: Arc<PgPool>,
18}
19
20impl EvalResultRepository {
21    pub fn new(db: &DbPool) -> Result<Self> {
22        Ok(Self {
23            pool: db.write_pool_arc()?,
24        })
25    }
26
27    pub async fn insert(&self, params: &NewResultParams) -> Result<EvalResultId> {
28        let id = EvalResultId::generate();
29        sqlx::query!(
30            r#"
31            INSERT INTO eval_results (
32                id, run_id, ai_request_id, case_id, provider, model,
33                overall_score, dimension_scores, verdict, rationale, repair_hint,
34                prompt_excerpt, response_excerpt, judge_cost_microdollars,
35                repaired, replay_of_result_id, judge_ai_request_id
36            )
37            VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17)
38            "#,
39            id.as_str(),
40            params.run_id.as_str(),
41            params.ai_request_id.as_ref().map(AiRequestId::as_str),
42            params.case_id.as_ref().map(EvalCaseId::as_str),
43            params.provider,
44            params.model,
45            params.overall_score,
46            params.dimension_scores,
47            params.verdict.as_str(),
48            params.rationale.as_deref(),
49            params.repair_hint.as_deref(),
50            params.prompt_excerpt.as_deref(),
51            params.response_excerpt.as_deref(),
52            params.judge_cost_microdollars,
53            params.repaired,
54            params
55                .replay_of_result_id
56                .as_ref()
57                .map(EvalResultId::as_str),
58            params.judge_ai_request_id.as_ref().map(AiRequestId::as_str)
59        )
60        .execute(self.pool.as_ref())
61        .await?;
62        Ok(id)
63    }
64
65    pub async fn list_by_run(&self, run_id: &EvalRunId) -> Result<Vec<EvalResult>> {
66        let rows = sqlx::query!(
67            r#"
68            SELECT id, run_id, ai_request_id, case_id, provider, model,
69                   overall_score, dimension_scores, verdict, rationale, repair_hint,
70                   prompt_excerpt, response_excerpt, latency_ms,
71                   cost_microdollars, judge_cost_microdollars, created_at,
72                   repaired, replay_of_result_id, judge_ai_request_id
73            FROM eval_results
74            WHERE run_id = $1
75            ORDER BY created_at
76            "#,
77            run_id.as_str()
78        )
79        .fetch_all(self.pool.as_ref())
80        .await?;
81
82        rows.into_iter()
83            .map(|row| {
84                Ok(EvalResult {
85                    id: EvalResultId::new(row.id),
86                    run_id: EvalRunId::new(row.run_id),
87                    ai_request_id: row.ai_request_id.map(AiRequestId::new),
88                    case_id: row.case_id.map(EvalCaseId::new),
89                    provider: row.provider,
90                    model: row.model,
91                    overall_score: row.overall_score,
92                    dimension_scores: row.dimension_scores,
93                    verdict: Verdict::from_str(&row.verdict)
94                        .map_err(EvaluationError::JudgeParse)?,
95                    rationale: row.rationale,
96                    repair_hint: row.repair_hint,
97                    prompt_excerpt: row.prompt_excerpt,
98                    response_excerpt: row.response_excerpt,
99                    latency_ms: row.latency_ms,
100                    cost_microdollars: row.cost_microdollars,
101                    judge_cost_microdollars: row.judge_cost_microdollars,
102                    created_at: row.created_at,
103                    repaired: row.repaired,
104                    replay_of_result_id: row.replay_of_result_id.map(EvalResultId::new),
105                    judge_ai_request_id: row.judge_ai_request_id.map(AiRequestId::new),
106                })
107            })
108            .collect()
109    }
110
111    pub async fn failures_for_replay(&self, run_id: &EvalRunId) -> Result<Vec<EvalResult>> {
112        let results = self.list_by_run(run_id).await?;
113        Ok(results
114            .into_iter()
115            .filter(|r| matches!(r.verdict, Verdict::Fail | Verdict::Partial) && !r.repaired)
116            .collect())
117    }
118
119    pub async fn mark_repaired(&self, id: &EvalResultId) -> Result<()> {
120        sqlx::query!(
121            "UPDATE eval_results SET repaired = TRUE WHERE id = $1",
122            id.as_str()
123        )
124        .execute(self.pool.as_ref())
125        .await?;
126        Ok(())
127    }
128}
systemprompt_evaluation/repository/results.rs

systemprompt_evaluation/repository/
results.rs
