systemprompt-evaluation 0.29.0

Evaluation framework for systemprompt.io AI governance infrastructure. Samples production AI traffic, scores it against rubrics with an LLM judge, and replays failures with repair hints.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83

//! Sampling filters and the sampled-request projection.
//!
//! Copyright (c) systemprompt.io — Business Source License 1.1.
//! See <https://systemprompt.io> for licensing details.

use chrono::{DateTime, Utc};
use serde_json::Value;
use systemprompt_identifiers::AiRequestId;

use super::case::{CanonicalMessage, CanonicalPrompt};

#[derive(Debug, Clone, Default)]
pub struct SampleFilter {
    pub since: Option<DateTime<Utc>>,
    pub until: Option<DateTime<Utc>>,
    pub provider: Option<String>,
    pub model: Option<String>,
    pub ids: Option<Vec<String>>,
    pub limit: i64,
}

impl SampleFilter {
    #[must_use]
    pub fn with_limit(limit: i64) -> Self {
        Self {
            limit,
            ..Self::default()
        }
    }

    #[must_use]
    pub const fn since(mut self, since: DateTime<Utc>) -> Self {
        self.since = Some(since);
        self
    }

    #[must_use]
    pub fn provider(mut self, provider: impl Into<String>) -> Self {
        self.provider = Some(provider.into());
        self
    }

    #[must_use]
    pub fn model(mut self, model: impl Into<String>) -> Self {
        self.model = Some(model.into());
        self
    }

    #[must_use]
    pub fn ids(mut self, ids: Vec<String>) -> Self {
        self.ids = Some(ids);
        self
    }
}

/// A completed production request hydrated with everything the judge needs.
#[derive(Debug, Clone)]
pub struct SampledRequest {
    pub ai_request_id: AiRequestId,
    pub provider: String,
    pub model: String,
    pub system_prompt_override: Option<String>,
    pub messages: Vec<CanonicalMessage>,
    pub response_text: Option<String>,
    pub offered_tools: Option<Value>,
    pub prepared_body_sha256: Option<String>,
    pub latency_ms: Option<i32>,
    pub cost_microdollars: i64,
    pub created_at: DateTime<Utc>,
}

impl SampledRequest {
    #[must_use]
    pub fn canonical_prompt(&self) -> CanonicalPrompt {
        CanonicalPrompt {
            messages: self.messages.clone(),
            system_prompt: self.system_prompt_override.clone(),
            offered_tools: self.offered_tools.clone(),
            provider: self.provider.clone(),
            model: self.model.clone(),
        }
    }
}