systemprompt-evaluation 0.29.0

Evaluation framework for systemprompt.io AI governance infrastructure. Samples production AI traffic, scores it against rubrics with an LLM judge, and replays failures with repair hints.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45

//! Repositories over the `eval_*` tables plus the sampling reader over the
//! `ai_requests` trace owned by `systemprompt-ai`.
//!
//! Copyright (c) systemprompt.io — Business Source License 1.1.
//! See <https://systemprompt.io> for licensing details.

mod cases;
mod judge_calls;
mod results;
mod rubrics;
mod runs;
mod sampling;

pub use cases::EvalCaseRepository;
pub use judge_calls::{EvalJudgeCallRepository, JudgeCallRecord};
pub use results::EvalResultRepository;
pub use rubrics::EvalRubricRepository;
pub use runs::EvalRunRepository;
pub use sampling::SamplingRepository;

use crate::error::Result;
use systemprompt_database::DbPool;

/// Bundle of the evaluation repositories, constructed once at a composition
/// root (the scheduler job or a CLI command) and cloned by consumers.
#[derive(Debug, Clone)]
pub struct EvalRepositories {
    pub runs: EvalRunRepository,
    pub cases: EvalCaseRepository,
    pub results: EvalResultRepository,
    pub rubrics: EvalRubricRepository,
    pub sampling: SamplingRepository,
}

impl EvalRepositories {
    pub fn new(db: &DbPool) -> Result<Self> {
        Ok(Self {
            runs: EvalRunRepository::new(db)?,
            cases: EvalCaseRepository::new(db)?,
            results: EvalResultRepository::new(db)?,
            rubrics: EvalRubricRepository::new(db)?,
            sampling: SamplingRepository::new(db)?,
        })
    }
}