use std::sync::Arc;
use crate::error::Result;
use crate::llm::ChatProvider;
use crate::message::Message;
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum GoalStatus {
Pursuing,
Achieved,
}
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
pub struct GoalState {
pub condition: String,
pub status: GoalStatus,
pub turns: u32,
pub max_turns: u32,
pub last_reason: Option<String>,
}
#[derive(Debug, Clone)]
pub struct GoalVerdict {
pub achieved: bool,
pub reason: String,
}
pub struct GoalEvaluator {
provider: Arc<dyn ChatProvider>,
}
impl GoalEvaluator {
pub fn new(provider: Arc<dyn ChatProvider>) -> Self {
Self { provider }
}
pub async fn evaluate(&self, condition: &str, transcript: &[Message]) -> Result<GoalVerdict> {
let transcript_text: String = transcript
.iter()
.filter_map(|m| {
if m.content.is_empty() {
None
} else {
Some(format!("[{:?}]: {}", m.role, m.content))
}
})
.collect::<Vec<_>>()
.join("\n");
let system_msg = Message::system(
"You are a completion evaluator. Answer YES or NO on the first line, \
then a single short sentence explaining why.",
);
let user_msg = Message::user(format!(
"Condition: {condition}\n\nRecent transcript:\n{transcript_text}\n\n\
Is the condition met? Answer YES or NO on the first line, then a short reason."
));
let messages = vec![system_msg, user_msg];
let completion = self.provider.complete(&messages, &[]).await?;
let text = completion.content.trim().to_string();
let first_line = text.lines().next().unwrap_or("").trim().to_uppercase();
let achieved = first_line.starts_with("YES");
let reason = text
.lines()
.skip(1)
.collect::<Vec<_>>()
.join(" ")
.trim()
.to_string();
let reason = if reason.is_empty() {
text.clone()
} else {
reason
};
Ok(GoalVerdict { achieved, reason })
}
}
#[cfg(test)]
mod tests {
use super::*;
use std::sync::Mutex;
use crate::error::Result;
use crate::llm::{ChatProvider, Completion};
use crate::message::Role;
use async_trait::async_trait;
#[test]
fn goal_state_serializes_and_deserializes() {
let state = GoalState {
condition: "all tests pass".into(),
status: GoalStatus::Pursuing,
turns: 3,
max_turns: 20,
last_reason: Some("still failing".into()),
};
let json = serde_json::to_string(&state).unwrap();
let roundtrip: GoalState = serde_json::from_str(&json).unwrap();
assert_eq!(roundtrip.condition, "all tests pass");
assert_eq!(roundtrip.status, GoalStatus::Pursuing);
assert_eq!(roundtrip.turns, 3);
assert_eq!(roundtrip.max_turns, 20);
assert_eq!(roundtrip.last_reason.as_deref(), Some("still failing"));
}
#[test]
fn goal_status_snake_case_serialization() {
assert_eq!(
serde_json::to_string(&GoalStatus::Pursuing).unwrap(),
r#""pursuing""#
);
assert_eq!(
serde_json::to_string(&GoalStatus::Achieved).unwrap(),
r#""achieved""#
);
}
#[test]
fn goal_verdict_achieved_flag_reflects_yes_logic() {
let text = "YES\nAll tests are passing now.";
let first_line = text.lines().next().unwrap_or("").trim().to_uppercase();
let achieved = first_line.starts_with("YES");
assert!(achieved);
let text_no = "NO\nTests still failing.";
let first_line_no = text_no.lines().next().unwrap_or("").trim().to_uppercase();
let achieved_no = first_line_no.starts_with("YES");
assert!(!achieved_no);
}
struct CapturingProvider {
captured_user_content: Mutex<String>,
}
#[async_trait]
impl ChatProvider for CapturingProvider {
async fn complete(
&self,
messages: &[Message],
_tools: &[crate::llm::ToolSpec],
) -> Result<Completion> {
if let Some(last) = messages.last() {
*self.captured_user_content.lock().unwrap() = last.content.clone();
}
Ok(Completion {
content: "YES\nAll conditions met.".into(),
tool_calls: vec![],
finish_reason: Some("stop".into()),
usage: None,
reasoning_content: None,
})
}
}
struct NoProvider;
#[async_trait]
impl ChatProvider for NoProvider {
async fn complete(
&self,
_messages: &[Message],
_tools: &[crate::llm::ToolSpec],
) -> Result<Completion> {
Ok(Completion {
content: "NO\nCondition not yet met.".into(),
tool_calls: vec![],
finish_reason: Some("stop".into()),
usage: None,
reasoning_content: None,
})
}
}
#[tokio::test]
async fn evaluate_returns_not_achieved_for_no_response() {
let evaluator = GoalEvaluator::new(Arc::new(NoProvider));
let verdict = evaluator
.evaluate("all tests pass", &[Message::user("still failing")])
.await
.unwrap();
assert!(!verdict.achieved, "NO response must produce achieved=false");
assert!(
!verdict.reason.is_empty(),
"reason must be non-empty even for NO"
);
}
#[tokio::test]
async fn evaluate_skips_empty_content_messages() {
let provider = Arc::new(CapturingProvider {
captured_user_content: Mutex::new(String::new()),
});
let evaluator = GoalEvaluator::new(provider.clone());
let transcript = vec![
Message::user("visible"),
Message::user(""), ];
let _ = evaluator.evaluate("cond", &transcript).await.unwrap();
let captured = provider.captured_user_content.lock().unwrap().clone();
assert!(
captured.contains("visible"),
"non-empty message must appear in prompt"
);
let occurrences = captured.matches("[User]:").count();
assert_eq!(
occurrences, 1,
"only 1 non-empty User message should be included; got {occurrences} occurrences in: {captured}"
);
}
struct OneLinerProvider;
#[async_trait]
impl ChatProvider for OneLinerProvider {
async fn complete(
&self,
_messages: &[Message],
_tools: &[crate::llm::ToolSpec],
) -> Result<Completion> {
Ok(Completion {
content: "YES".into(), tool_calls: vec![],
finish_reason: Some("stop".into()),
usage: None,
reasoning_content: None,
})
}
}
#[tokio::test]
async fn evaluate_reason_falls_back_to_full_text_when_single_line() {
let evaluator = GoalEvaluator::new(Arc::new(OneLinerProvider));
let verdict = evaluator
.evaluate("cond", &[Message::user("ok")])
.await
.unwrap();
assert!(verdict.achieved, "single-line YES must be achieved=true");
assert!(
!verdict.reason.is_empty(),
"reason must fall back to the full response"
);
assert_eq!(
verdict.reason, "YES",
"reason should equal the full response when there is no second line"
);
}
#[tokio::test]
async fn evaluate_preserves_all_messages_beyond_old_tail_20() {
let transcript: Vec<Message> = (0..25)
.map(|i| Message {
role: Role::User,
content: format!("message-{i:02}"),
tool_calls: vec![],
tool_call_id: None,
reasoning_content: None,
is_compaction_summary: false,
})
.collect();
let provider = Arc::new(CapturingProvider {
captured_user_content: Mutex::new(String::new()),
});
let evaluator = GoalEvaluator::new(provider.clone());
let verdict = evaluator
.evaluate("test condition", &transcript)
.await
.unwrap();
assert!(verdict.achieved);
let captured = provider.captured_user_content.lock().unwrap().clone();
for i in 0..25 {
let needle = format!("message-{i:02}");
assert!(
captured.contains(&needle),
"transcript message {i} ({needle}) missing from prompt;\n\
prompt was: {captured}"
);
}
}
}