systemprompt-evaluation 0.29.0

Evaluation framework for systemprompt.io AI governance infrastructure. Samples production AI traffic, scores it against rubrics with an LLM judge, and replays failures with repair hints.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74

//! Rubric model describing judge scoring criteria.
//!
//! Copyright (c) systemprompt.io — Business Source License 1.1.
//! See <https://systemprompt.io> for licensing details.

use serde::{Deserialize, Serialize};
use systemprompt_identifiers::EvalRubricId;

#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct RubricDimension {
    pub name: String,
    pub description: String,
    #[serde(default = "default_weight")]
    pub weight: f64,
}

const fn default_weight() -> f64 {
    1.0
}

#[derive(Debug, Clone)]
pub struct Rubric {
    pub id: EvalRubricId,
    pub name: String,
    pub dimensions: Vec<RubricDimension>,
    pub pass_threshold: i32,
    pub prompt_template: Option<String>,
    pub enabled: bool,
}

/// Structured output the judge model is constrained to produce.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct JudgeVerdict {
    pub overall_score: i32,
    #[serde(default)]
    pub dimension_scores: Vec<DimensionScore>,
    pub rationale: String,
    #[serde(default)]
    pub repair_hint: Option<String>,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct DimensionScore {
    pub name: String,
    pub score: i32,
}

impl JudgeVerdict {
    /// JSON schema the judge request is constrained with; keep in sync with
    /// the `Deserialize` shape above.
    #[must_use]
    pub fn response_schema() -> serde_json::Value {
        serde_json::json!({
            "type": "object",
            "properties": {
                "overall_score": { "type": "integer", "minimum": 1, "maximum": 5 },
                "dimension_scores": {
                    "type": "array",
                    "items": {
                        "type": "object",
                        "properties": {
                            "name": { "type": "string" },
                            "score": { "type": "integer", "minimum": 1, "maximum": 5 }
                        },
                        "required": ["name", "score"]
                    }
                },
                "rationale": { "type": "string" },
                "repair_hint": { "type": ["string", "null"] }
            },
            "required": ["overall_score", "rationale"]
        })
    }
}