systemprompt-evaluation 0.29.0

Evaluation framework for systemprompt.io AI governance infrastructure. Samples production AI traffic, scores it against rubrics with an LLM judge, and replays failures with repair hints.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108

//! Evaluation run model and lifecycle states.
//!
//! Copyright (c) systemprompt.io — Business Source License 1.1.
//! See <https://systemprompt.io> for licensing details.

use chrono::{DateTime, Utc};
use serde::{Deserialize, Serialize};
use systemprompt_identifiers::{EvalRubricId, EvalRunId, UserId};

#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "lowercase")]
pub enum EvalRunKind {
    Judge,
    Replay,
    Pairwise,
}

impl EvalRunKind {
    #[must_use]
    pub const fn as_str(self) -> &'static str {
        match self {
            Self::Judge => "judge",
            Self::Replay => "replay",
            Self::Pairwise => "pairwise",
        }
    }
}

impl std::str::FromStr for EvalRunKind {
    type Err = String;

    fn from_str(s: &str) -> Result<Self, Self::Err> {
        match s {
            "judge" => Ok(Self::Judge),
            "replay" => Ok(Self::Replay),
            "pairwise" => Ok(Self::Pairwise),
            other => Err(format!("unknown eval run kind: {other}")),
        }
    }
}

#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "lowercase")]
pub enum EvalRunStatus {
    Running,
    Completed,
    Failed,
}

impl EvalRunStatus {
    #[must_use]
    pub const fn as_str(self) -> &'static str {
        match self {
            Self::Running => "running",
            Self::Completed => "completed",
            Self::Failed => "failed",
        }
    }
}

#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "lowercase")]
pub enum TriggerSource {
    Scheduled,
    Cli,
    Manual,
}

impl TriggerSource {
    #[must_use]
    pub const fn as_str(self) -> &'static str {
        match self {
            Self::Scheduled => "scheduled",
            Self::Cli => "cli",
            Self::Manual => "manual",
        }
    }
}

#[derive(Debug, Clone)]
pub struct EvalRun {
    pub id: EvalRunId,
    pub kind: EvalRunKind,
    pub status: EvalRunStatus,
    pub judge_provider: String,
    pub judge_model: String,
    pub sample_size: i32,
    pub scored_count: i32,
    pub failed_count: i32,
    pub cost_microdollars: i64,
    pub created_by: UserId,
    pub created_at: DateTime<Utc>,
    pub completed_at: Option<DateTime<Utc>>,
    pub error_message: Option<String>,
    pub rubric_id: Option<EvalRubricId>,
    pub trigger_source: TriggerSource,
}

#[derive(Debug, Clone)]
pub struct NewRunParams {
    pub kind: EvalRunKind,
    pub judge_provider: String,
    pub judge_model: String,
    pub sample_size: i32,
    pub created_by: UserId,
    pub rubric_id: Option<EvalRubricId>,
    pub trigger_source: TriggerSource,
}