systemprompt_evaluation/services/
evaluation_service.rs

1//! Orchestration service for evaluation runs (sample, judge, replay).
2//!
3//! Copyright (c) systemprompt.io — Business Source License 1.1.
4//! See <https://systemprompt.io> for licensing details.
5
6use std::sync::Arc;
7
8use systemprompt_identifiers::{ContextId, EvalCaseId, EvalRunId, UserId};
9use systemprompt_models::ai::DynAiProvider;
10
11use crate::error::Result;
12use crate::models::{
13    EvalResult, EvalRun, EvalRunKind, NewCaseParams, NewRunParams, Rubric, RubricDimension,
14    SampleFilter, TriggerSource,
15};
16use crate::repository::{
17    EvalCaseRepository, EvalRepositories, EvalResultRepository, EvalRubricRepository,
18    EvalRunRepository, SamplingRepository,
19};
20use crate::services::judge::{JudgeService, JudgeSpec};
21use crate::services::loop_runner::{AutoImproveLoop, LoopLimits, LoopReport};
22use crate::services::replay::ReplayService;
23use crate::services::sampler::SamplerService;
24
25const DEFAULT_RUBRIC: &str = "default";
26
27#[derive(Debug, Clone)]
28pub struct RunRequest {
29    pub judge_provider: String,
30    pub judge_model: String,
31    pub rubric_name: Option<String>,
32    pub filter: SampleFilter,
33    pub budget_microdollars: Option<i64>,
34    pub created_by: UserId,
35    pub trigger_source: TriggerSource,
36}
37
38#[derive(Clone)]
39pub struct EvaluationService {
40    ai: DynAiProvider,
41    runs: EvalRunRepository,
42    cases: EvalCaseRepository,
43    results: EvalResultRepository,
44    rubrics: EvalRubricRepository,
45    sampling: SamplingRepository,
46}
47
48impl std::fmt::Debug for EvaluationService {
49    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
50        f.debug_struct("EvaluationService")
51            .field("runs", &self.runs)
52            .finish_non_exhaustive()
53    }
54}
55
56impl EvaluationService {
57    /// `ai` must be the auditing provider (the `AiService` implementation):
58    /// judge isolation and cost accounting depend on `generate` persisting
59    /// every request to `ai_requests` with a job actor.
60    pub fn new(repositories: EvalRepositories, ai: DynAiProvider) -> Self {
61        let EvalRepositories {
62            runs,
63            cases,
64            results,
65            rubrics,
66            sampling,
67        } = repositories;
68        Self {
69            ai,
70            runs,
71            cases,
72            results,
73            rubrics,
74            sampling,
75        }
76    }
77
78    /// One full auto-improve pass; returns the run id and its report.
79    pub async fn run_judge(&self, request: RunRequest) -> Result<(EvalRunId, LoopReport)> {
80        self.run_with_kind(EvalRunKind::Judge, request).await
81    }
82
83    /// Re-runs a previous run's failing requests as a new replay-kind run.
84    pub async fn replay_failures(
85        &self,
86        source_run: &EvalRunId,
87        mut request: RunRequest,
88    ) -> Result<(EvalRunId, LoopReport)> {
89        let failures = self.results.failures_for_replay(source_run).await?;
90        let ids: Vec<String> = failures
91            .iter()
92            .filter_map(|r| r.ai_request_id.as_ref())
93            .map(|id| id.as_str().to_owned())
94            .collect();
95        if ids.is_empty() {
96            return Err(crate::error::EvaluationError::ReplaySource(format!(
97                "run {source_run} has no unrepaired failures with a source request"
98            )));
99        }
100        request.filter.limit = i64::try_from(ids.len()).unwrap_or(i64::MAX);
101        request.filter = request.filter.ids(ids);
102        request.filter.since = None;
103        self.run_with_kind(EvalRunKind::Replay, request).await
104    }
105
106    pub async fn list_runs(&self, limit: i64) -> Result<Vec<EvalRun>> {
107        self.runs.list_recent(limit).await
108    }
109
110    async fn run_with_kind(
111        &self,
112        kind: EvalRunKind,
113        request: RunRequest,
114    ) -> Result<(EvalRunId, LoopReport)> {
115        let rubric = self.resolve_rubric(request.rubric_name.as_deref()).await?;
116        let run_id = self
117            .runs
118            .create(&NewRunParams {
119                kind,
120                judge_provider: request.judge_provider.clone(),
121                judge_model: request.judge_model.clone(),
122                sample_size: i32::try_from(request.filter.limit).unwrap_or(i32::MAX),
123                created_by: request.created_by.clone(),
124                rubric_id: Some(rubric.id.clone()),
125                trigger_source: request.trigger_source,
126            })
127            .await?;
128
129        let run_context = ContextId::derived_from_evaluation_run(&run_id);
130        let judge = JudgeService::new(
131            Arc::clone(&self.ai),
132            self.sampling.clone(),
133            JudgeSpec {
134                provider: request.judge_provider,
135                model: request.judge_model,
136                created_by: request.created_by.clone(),
137                run_context: run_context.clone(),
138            },
139        );
140        let auto_improve = AutoImproveLoop {
141            sampler: SamplerService::new(self.sampling.clone()),
142            judge,
143            replay: ReplayService::new(Arc::clone(&self.ai), request.created_by, run_context),
144            runs: self.runs.clone(),
145            results: self.results.clone(),
146        };
147
148        let outcome = auto_improve
149            .run(
150                &run_id,
151                &rubric,
152                &request.filter,
153                LoopLimits {
154                    budget_microdollars: request.budget_microdollars,
155                },
156            )
157            .await;
158        match outcome {
159            Ok(report) => {
160                self.runs.complete(&run_id).await?;
161                Ok((run_id, report))
162            },
163            Err(e) => {
164                self.runs.fail(&run_id, &e.to_string()).await?;
165                Err(e)
166            },
167        }
168    }
169
170    pub async fn promote_case(&self, params: &NewCaseParams) -> Result<EvalCaseId> {
171        self.cases.create(params).await
172    }
173
174    pub async fn get_run(&self, run_id: &EvalRunId) -> Result<EvalRun> {
175        self.runs.get(run_id).await
176    }
177
178    pub async fn list_results(&self, run_id: &EvalRunId) -> Result<Vec<EvalResult>> {
179        self.results.list_by_run(run_id).await
180    }
181
182    pub async fn sample(
183        &self,
184        filter: &SampleFilter,
185    ) -> Result<Vec<crate::models::SampledRequest>> {
186        SamplerService::new(self.sampling.clone())
187            .sample(filter)
188            .await
189    }
190
191    async fn resolve_rubric(&self, name: Option<&str>) -> Result<Rubric> {
192        let name = name.unwrap_or(DEFAULT_RUBRIC);
193        match self.rubrics.get_by_name(name).await {
194            Ok(rubric) => Ok(rubric),
195            Err(crate::error::EvaluationError::RubricNotFound(_)) if name == DEFAULT_RUBRIC => {
196                let rubric = default_rubric();
197                self.rubrics.upsert(&rubric).await?;
198                Ok(rubric)
199            },
200            Err(e) => Err(e),
201        }
202    }
203}
204
205fn default_rubric() -> Rubric {
206    Rubric {
207        id: systemprompt_identifiers::EvalRubricId::generate(),
208        name: DEFAULT_RUBRIC.to_owned(),
209        dimensions: vec![
210            RubricDimension {
211                name: "correctness".to_owned(),
212                description: "The response is factually and technically accurate.".to_owned(),
213                weight: 1.0,
214            },
215            RubricDimension {
216                name: "helpfulness".to_owned(),
217                description: "The response addresses what the user actually asked.".to_owned(),
218                weight: 1.0,
219            },
220            RubricDimension {
221                name: "completeness".to_owned(),
222                description: "The response covers the request without gaps.".to_owned(),
223                weight: 1.0,
224            },
225        ],
226        pass_threshold: 4,
227        prompt_template: None,
228        enabled: true,
229    }
230}
systemprompt_evaluation/services/evaluation_service.rs

systemprompt_evaluation/services/
evaluation_service.rs
