systemprompt_evaluation/services/
judge.rs

1//! LLM-judge scoring of sampled request/response pairs against a rubric.
2//!
3//! Copyright (c) systemprompt.io — Business Source License 1.1.
4//! See <https://systemprompt.io> for licensing details.
5
6use systemprompt_identifiers::{
7    Actor, AgentName, AiRequestId, ContextId, SessionId, TraceId, UserId,
8};
9use systemprompt_models::RequestContext;
10use systemprompt_models::ai::{
11    AiMessage, AiRequest, DynAiProvider, ResponseFormat, StructuredOutputOptions,
12};
13
14use crate::error::{EvaluationError, Result};
15use crate::models::{JudgeVerdict, Rubric, Verdict};
16use crate::repository::SamplingRepository;
17
18const JUDGE_ACTOR_JOB: &str = "evaluation_judge";
19const JUDGE_AGENT: &str = "evaluation-judge";
20const JUDGE_MAX_OUTPUT_TOKENS: u32 = 2048;
21const MAX_JUDGE_CHARS: usize = 8_000;
22
23/// What the judge grades: the prompt transcript and the response under test.
24#[derive(Debug, Clone)]
25pub struct JudgeTarget {
26    pub transcript: String,
27    pub response: String,
28    pub expectation: Option<String>,
29}
30
31#[derive(Debug, Clone)]
32pub struct ScoredVerdict {
33    pub verdict: JudgeVerdict,
34    pub outcome: Verdict,
35    pub judge_ai_request_id: AiRequestId,
36    pub judge_cost_microdollars: i64,
37}
38
39#[derive(Debug, Clone)]
40pub struct JudgeSpec {
41    pub provider: String,
42    pub model: String,
43    pub created_by: UserId,
44    pub run_context: ContextId,
45}
46
47#[derive(Clone)]
48pub struct JudgeService {
49    ai: DynAiProvider,
50    sampling: SamplingRepository,
51    judge_provider: String,
52    judge_model: String,
53    created_by: UserId,
54    run_context: ContextId,
55}
56
57impl std::fmt::Debug for JudgeService {
58    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
59        f.debug_struct("JudgeService")
60            .field("judge_provider", &self.judge_provider)
61            .field("judge_model", &self.judge_model)
62            .finish_non_exhaustive()
63    }
64}
65
66impl JudgeService {
67    /// `ai` must be the auditing provider (the `AiService` implementation):
68    /// judge isolation and cost lookup depend on `generate` persisting the
69    /// request to `ai_requests` with the job actor this service sets.
70    pub fn new(ai: DynAiProvider, sampling: SamplingRepository, spec: JudgeSpec) -> Self {
71        Self {
72            ai,
73            sampling,
74            judge_provider: spec.provider,
75            judge_model: spec.model,
76            created_by: spec.created_by,
77            run_context: spec.run_context,
78        }
79    }
80
81    pub async fn score(&self, rubric: &Rubric, target: &JudgeTarget) -> Result<ScoredVerdict> {
82        let request = self.build_request(rubric, target);
83        let response = self
84            .ai
85            .generate(&request)
86            .await
87            .map_err(|e| EvaluationError::Ai(e.to_string()))?;
88
89        let verdict: JudgeVerdict = serde_json::from_str(response.content.trim())
90            .map_err(|e| EvaluationError::JudgeParse(e.to_string()))?;
91        if !(1..=5).contains(&verdict.overall_score) {
92            return Err(EvaluationError::JudgeParse(format!(
93                "overall_score {} outside 1-5",
94                verdict.overall_score
95            )));
96        }
97
98        let request_id = response.request_id.to_string();
99        let judge_cost_microdollars = self.sampling.request_cost(&request_id).await?;
100        Ok(ScoredVerdict {
101            outcome: outcome(verdict.overall_score, rubric.pass_threshold),
102            verdict,
103            judge_ai_request_id: AiRequestId::new(request_id),
104            judge_cost_microdollars,
105        })
106    }
107
108    fn build_request(&self, rubric: &Rubric, target: &JudgeTarget) -> AiRequest {
109        let context = RequestContext::new(
110            SessionId::generate(),
111            TraceId::generate(),
112            self.run_context.clone(),
113            AgentName::new(JUDGE_AGENT),
114        )
115        .with_actor(Actor::job(self.created_by.clone(), JUDGE_ACTOR_JOB));
116
117        let messages = vec![AiMessage::user(judge_prompt(rubric, target))];
118        AiRequest::builder(
119            messages,
120            self.judge_provider.clone(),
121            self.judge_model.clone(),
122            JUDGE_MAX_OUTPUT_TOKENS,
123            context,
124        )
125        .with_system_prompt(system_prompt(rubric))
126        .with_structured_output(StructuredOutputOptions {
127            response_format: Some(ResponseFormat::json_schema(JudgeVerdict::response_schema())),
128            ..StructuredOutputOptions::default()
129        })
130        .build()
131    }
132}
133
134const fn outcome(score: i32, pass_threshold: i32) -> Verdict {
135    if score >= pass_threshold {
136        Verdict::Pass
137    } else if score == pass_threshold - 1 {
138        Verdict::Partial
139    } else {
140        Verdict::Fail
141    }
142}
143
144fn system_prompt(rubric: &Rubric) -> String {
145    rubric.prompt_template.clone().unwrap_or_else(|| {
146        "You are a strict evaluation judge. Score the assistant response \
147         against the rubric dimensions on a 1-5 scale, explain your rationale, \
148         and when the response falls short provide a concrete repair_hint the \
149         assistant could follow to fix it."
150            .to_owned()
151    })
152}
153
154fn judge_prompt(rubric: &Rubric, target: &JudgeTarget) -> String {
155    let dimensions = rubric
156        .dimensions
157        .iter()
158        .map(|d| format!("- {}: {}", d.name, d.description))
159        .collect::<Vec<_>>()
160        .join("\n");
161    let expectation = target
162        .expectation
163        .as_deref()
164        .map(|e| format!("\nExpected behaviour:\n{e}\n"))
165        .unwrap_or_default();
166    format!(
167        "Rubric dimensions:\n{dimensions}\n{expectation}\nConversation:\n{}\n\nResponse under \
168         evaluation:\n{}",
169        truncate(&target.transcript, MAX_JUDGE_CHARS),
170        truncate(&target.response, MAX_JUDGE_CHARS)
171    )
172}
173
174fn truncate(text: &str, max_chars: usize) -> &str {
175    match text.char_indices().nth(max_chars) {
176        Some((idx, _)) => &text[..idx],
177        None => text,
178    }
179}
systemprompt_evaluation/services/judge.rs

systemprompt_evaluation/services/
judge.rs
