systemprompt_evaluation/services/
replay.rs

1//! Replay of stored evaluation cases against the live AI provider.
2//!
3//! Copyright (c) systemprompt.io — Business Source License 1.1.
4//! See <https://systemprompt.io> for licensing details.
5
6use systemprompt_identifiers::{Actor, AgentName, ContextId, SessionId, TraceId, UserId};
7use systemprompt_models::RequestContext;
8use systemprompt_models::ai::{AiMessage, AiRequest, AiResponse, DynAiProvider, MessageRole};
9
10use crate::error::{EvaluationError, Result};
11use crate::models::{CanonicalMessage, CanonicalPrompt};
12
13const REPLAY_ACTOR_JOB: &str = "evaluation_replay";
14const REPLAY_AGENT: &str = "evaluation-replay";
15const REPLAY_MAX_OUTPUT_TOKENS: u32 = 8192;
16
17#[derive(Clone)]
18pub struct ReplayService {
19    ai: DynAiProvider,
20    created_by: UserId,
21    run_context: ContextId,
22}
23
24impl std::fmt::Debug for ReplayService {
25    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
26        f.debug_struct("ReplayService")
27            .field("created_by", &self.created_by)
28            .finish_non_exhaustive()
29    }
30}
31
32impl ReplayService {
33    pub const fn new(ai: DynAiProvider, created_by: UserId, run_context: ContextId) -> Self {
34        Self {
35            ai,
36            created_by,
37            run_context,
38        }
39    }
40
41    /// Re-executes a canonical prompt with the repair hint injected ahead of
42    /// the final user turn, so the model sees the corrective instruction in
43    /// the position a system hint would occupy at serve time.
44    pub async fn replay(&self, prompt: &CanonicalPrompt, repair_hint: &str) -> Result<AiResponse> {
45        let messages = build_messages(&prompt.messages, repair_hint)?;
46        let context = RequestContext::new(
47            SessionId::generate(),
48            TraceId::generate(),
49            self.run_context.clone(),
50            AgentName::new(REPLAY_AGENT),
51        )
52        .with_actor(Actor::job(self.created_by.clone(), REPLAY_ACTOR_JOB));
53
54        let mut builder = AiRequest::builder(
55            messages,
56            prompt.provider.clone(),
57            prompt.model.clone(),
58            REPLAY_MAX_OUTPUT_TOKENS,
59            context,
60        );
61        if let Some(system) = &prompt.system_prompt {
62            builder = builder.with_system_prompt(system.clone());
63        }
64        self.ai
65            .generate(&builder.build())
66            .await
67            .map_err(|e| EvaluationError::Ai(e.to_string()))
68    }
69}
70
71fn build_messages(canonical: &[CanonicalMessage], repair_hint: &str) -> Result<Vec<AiMessage>> {
72    if canonical.is_empty() {
73        return Err(EvaluationError::ReplaySource(
74            "canonical prompt has no messages".to_owned(),
75        ));
76    }
77    let last_user = canonical
78        .iter()
79        .rposition(|m| m.role == "user")
80        .ok_or_else(|| {
81            EvaluationError::ReplaySource("canonical prompt has no user turn".to_owned())
82        })?;
83
84    let mut messages = Vec::with_capacity(canonical.len() + 1);
85    for (idx, message) in canonical.iter().enumerate() {
86        if idx == last_user {
87            messages.push(AiMessage::system(format!(
88                "Apply this correction when answering the next user message: {repair_hint}"
89            )));
90        }
91        messages.push(to_ai_message(message));
92    }
93    Ok(messages)
94}
95
96fn to_ai_message(message: &CanonicalMessage) -> AiMessage {
97    let role = match message.role.as_str() {
98        "system" => MessageRole::System,
99        "assistant" => MessageRole::Assistant,
100        _ => MessageRole::User,
101    };
102    AiMessage {
103        role,
104        content: message.content.clone(),
105        parts: Vec::new(),
106    }
107}
systemprompt_evaluation/services/replay.rs

systemprompt_evaluation/services/
replay.rs
