use std::sync::Arc;
use crate::llm::types::{CompletionRequest, ContentBlock, Message, TokenUsage};
use crate::llm::{BoxedProvider, LlmProvider};
pub const DEFAULT_MAX_CONTINUATIONS: u32 = 8;
const JUDGE_MAX_TOKENS: u32 = 256;
const MAX_TRANSCRIPT_CHARS: usize = 12_000;
const JUDGE_SYSTEM_PROMPT: &str = "\
You are an impartial completion judge. You did NOT do the work; you only verify \
it. Given an OBJECTIVE and the WORKING TRANSCRIPT/OUTPUT an agent has produced, \
decide whether the objective is genuinely and verifiably satisfied by the \
evidence shown — not merely claimed. An agent asserting it is done is NOT \
evidence; look for the concrete result the objective requires (e.g. a passing \
test, an exit code, the requested content). Be skeptical: if the evidence is \
absent, ambiguous, or only asserted, the objective is NOT met.\n\n\
Respond with EXACTLY one verdict line, then optionally a brief reason:\n\
GOAL_MET: YES\n\
or\n\
GOAL_MET: NO: <one sentence on what concrete evidence is still missing>";
pub type GoalSlot = Arc<std::sync::RwLock<Option<GoalCondition>>>;
#[derive(Clone)]
pub struct GoalCondition {
objective: String,
judge: Arc<BoxedProvider>,
max_continuations: u32,
per_criterion: bool,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct GoalVerdict {
pub satisfied: bool,
pub reason: String,
}
impl GoalCondition {
pub fn new(objective: impl Into<String>, judge: Arc<BoxedProvider>) -> Self {
Self {
objective: objective.into(),
judge,
max_continuations: DEFAULT_MAX_CONTINUATIONS,
per_criterion: false,
}
}
pub fn with_per_criterion(mut self, on: bool) -> Self {
self.per_criterion = on;
self
}
pub fn with_max_continuations(mut self, n: u32) -> Self {
self.max_continuations = n;
self
}
pub fn max_continuations(&self) -> u32 {
self.max_continuations
}
pub fn objective(&self) -> &str {
&self.objective
}
pub fn per_criterion(&self) -> bool {
self.per_criterion
}
pub(crate) fn continuation_message(&self, reason: &str) -> String {
format!(
"The objective is not yet complete. {reason}\n\nContinue working toward this \
objective and demonstrate the concrete result it requires: {}",
self.objective
)
}
pub(crate) async fn evaluate(&self, transcript: &str) -> (GoalVerdict, TokenUsage) {
let evidence = tail_chars(transcript, MAX_TRANSCRIPT_CHARS);
let user = format!(
"OBJECTIVE:\n{}\n\nWORKING TRANSCRIPT (most recent; tool results are the \
evidence — `[Tool result: ...]` lines show what actually happened):\n{}\n\n\
Is the objective satisfied by the evidence above? Reply with the verdict line.",
self.objective, evidence
);
let system = if self.per_criterion {
format!(
"{JUDGE_SYSTEM_PROMPT}\n\nThe OBJECTIVE is a list of acceptance criteria. \
Evaluate EACH criterion independently against the evidence: output one line \
per criterion, `- [met] <criterion>` or `- [NOT MET] <criterion>: <missing \
evidence>`, BEFORE the verdict line. GOAL_MET: YES only when EVERY criterion \
is met; otherwise GOAL_MET: NO: name the specific unmet criteria."
)
} else {
JUDGE_SYSTEM_PROMPT.to_string()
};
let max_tokens = if self.per_criterion {
JUDGE_MAX_TOKENS * 4
} else {
JUDGE_MAX_TOKENS
};
let request = CompletionRequest {
system,
messages: vec![Message::user(user)],
tools: Vec::new(),
max_tokens,
tool_choice: None,
reasoning_effort: None,
};
match self.judge.complete(request).await {
Ok(response) => (
parse_goal_verdict(&response_text(&response.content)),
response.usage,
),
Err(e) => {
tracing::warn!(error = %e, "goal judge call failed; treating goal as not-met");
(
GoalVerdict {
satisfied: false,
reason: "judge unavailable; continuing".to_string(),
},
TokenUsage::default(),
)
}
}
}
}
fn tail_chars(s: &str, max: usize) -> String {
if s.chars().count() <= max {
return s.to_string();
}
let skip = s.chars().count() - max;
let tail: String = s.chars().skip(skip).collect();
format!("[... earlier turns omitted ...]\n{tail}")
}
impl std::fmt::Debug for GoalCondition {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("GoalCondition")
.field("objective", &self.objective)
.field("max_continuations", &self.max_continuations)
.finish_non_exhaustive()
}
}
fn response_text(content: &[ContentBlock]) -> String {
content
.iter()
.filter_map(|b| match b {
ContentBlock::Text { text } => Some(text.as_str()),
_ => None,
})
.collect::<Vec<_>>()
.join("\n")
}
pub(crate) fn parse_goal_verdict(text: &str) -> GoalVerdict {
for line in text.lines() {
let trimmed = line.trim();
if let Some(rest) = trimmed.strip_prefix("GOAL_MET:") {
let rest = rest.trim();
let verdict_token: String = rest
.chars()
.take_while(|c| c.is_ascii_alphabetic())
.collect();
if verdict_token.eq_ignore_ascii_case("yes") {
return GoalVerdict {
satisfied: true,
reason: String::new(),
};
}
if let Some(reason) = rest
.strip_prefix("NO:")
.or_else(|| rest.strip_prefix("no:"))
.or_else(|| rest.strip_prefix("No:"))
{
let reason = reason.trim();
return GoalVerdict {
satisfied: false,
reason: if reason.is_empty() {
"objective not yet demonstrated".to_string()
} else {
reason.to_string()
},
};
}
if rest.eq_ignore_ascii_case("no") || rest.eq_ignore_ascii_case("no.") {
return GoalVerdict {
satisfied: false,
reason: "objective not yet demonstrated".to_string(),
};
}
}
}
GoalVerdict {
satisfied: false,
reason: "judge did not return a recognized verdict; continuing".to_string(),
}
}
#[cfg(test)]
mod tests {
use super::*;
use std::sync::Mutex;
struct StubJudge {
reply: String,
captured: Mutex<Vec<CompletionRequest>>,
}
impl LlmProvider for StubJudge {
async fn complete(
&self,
request: CompletionRequest,
) -> Result<crate::llm::types::CompletionResponse, crate::error::Error> {
self.captured.lock().expect("lock").push(request);
Ok(crate::llm::types::CompletionResponse {
content: vec![ContentBlock::Text {
text: self.reply.clone(),
}],
stop_reason: crate::llm::types::StopReason::EndTurn,
reasoning: None,
usage: TokenUsage::default(),
model: None,
})
}
}
#[tokio::test]
async fn judge_reports_per_criterion_status() {
let judge = Arc::new(StubJudge {
reply: "- [met] A: endpoint returns 200\n- [NOT MET] B: no test evidence\n\
GOAL_MET: NO: criterion B unmet — tests were never run"
.into(),
captured: Mutex::new(Vec::new()),
});
let goal = GoalCondition::new(
"- A: GET /health returns 200\n- B: cargo test green",
Arc::new(BoxedProvider::from_arc(judge.clone())),
)
.with_per_criterion(true);
let (verdict, _usage) = goal.evaluate("transcript evidence here").await;
assert!(!verdict.satisfied);
assert!(
verdict.reason.contains("criterion B"),
"reason names the unmet criterion: {}",
verdict.reason
);
let cont = goal.continuation_message(&verdict.reason);
assert!(
cont.contains("criterion B"),
"continuation carries the specific gap: {cont}"
);
let reqs = judge.captured.lock().expect("lock");
assert!(
reqs[0].system.contains("EACH criterion"),
"per-criterion instruction missing: {}",
reqs[0].system
);
}
#[tokio::test]
async fn default_judge_prompt_has_no_per_criterion_instruction() {
let judge = Arc::new(StubJudge {
reply: "GOAL_MET: YES".into(),
captured: Mutex::new(Vec::new()),
});
let goal = GoalCondition::new("ship it", Arc::new(BoxedProvider::from_arc(judge.clone())));
let _ = goal.evaluate("evidence").await;
let reqs = judge.captured.lock().expect("lock");
assert!(
!reqs[0].system.contains("EACH criterion"),
"off by default (cost): {}",
reqs[0].system
);
}
#[test]
fn parses_yes() {
let v = parse_goal_verdict("Looks complete.\nGOAL_MET: YES");
assert!(v.satisfied);
assert!(v.reason.is_empty());
}
#[test]
fn parses_no_with_reason() {
let v = parse_goal_verdict("GOAL_MET: NO: tests have not been run yet");
assert!(!v.satisfied);
assert_eq!(v.reason, "tests have not been run yet");
}
#[test]
fn parses_bare_no() {
let v = parse_goal_verdict("GOAL_MET: NO");
assert!(!v.satisfied);
assert!(!v.reason.is_empty());
}
#[test]
fn verdict_value_is_case_insensitive() {
assert!(parse_goal_verdict("GOAL_MET: yes").satisfied);
assert!(parse_goal_verdict("GOAL_MET: YES").satisfied);
assert!(!parse_goal_verdict("GOAL_MET: no: x").satisfied);
assert!(!parse_goal_verdict("GOAL_MET: NO: x").satisfied);
}
#[test]
fn verdict_yes_with_trailing_justification_is_satisfied() {
assert!(parse_goal_verdict("GOAL_MET: YES — all criteria met").satisfied);
assert!(parse_goal_verdict("GOAL_MET: YES (verified against tests)").satisfied);
assert!(parse_goal_verdict("GOAL_MET: yes, the objective is demonstrated").satisfied);
assert!(!parse_goal_verdict("GOAL_MET: yesterday it was incomplete").satisfied);
}
#[test]
fn unrecognized_reply_is_not_satisfied() {
let v = parse_goal_verdict("I have completed everything successfully!");
assert!(!v.satisfied, "no verdict line must not count as met");
assert!(!v.reason.is_empty());
}
#[test]
fn empty_reply_is_not_satisfied() {
assert!(!parse_goal_verdict("").satisfied);
}
#[test]
fn builder_defaults_and_overrides() {
use crate::agent::test_helpers::MockProvider;
let judge = Arc::new(BoxedProvider::new(MockProvider::new(vec![])));
let g = GoalCondition::new("ship it", Arc::clone(&judge));
assert_eq!(g.max_continuations(), DEFAULT_MAX_CONTINUATIONS);
assert_eq!(g.objective(), "ship it");
let g2 = GoalCondition::new("ship it", judge).with_max_continuations(2);
assert_eq!(g2.max_continuations(), 2);
}
#[test]
fn continuation_message_recites_objective_and_reason() {
use crate::agent::test_helpers::MockProvider;
let judge = Arc::new(BoxedProvider::new(MockProvider::new(vec![])));
let g = GoalCondition::new("all tests pass", judge);
let msg = g.continuation_message("tests are still failing");
assert!(msg.contains("tests are still failing"));
assert!(msg.contains("all tests pass"));
}
#[tokio::test]
async fn evaluate_uses_independent_judge_and_parses_yes() {
use crate::agent::test_helpers::MockProvider;
let judge = Arc::new(BoxedProvider::new(MockProvider::new(vec![
MockProvider::text_response("GOAL_MET: YES", 5, 3),
])));
let g = GoalCondition::new("do the thing", judge);
let (v, usage) = g.evaluate("I did the thing, here is the result X.").await;
assert!(v.satisfied);
assert!(usage.input_tokens + usage.output_tokens > 0);
}
#[tokio::test]
async fn evaluate_judge_error_fails_toward_not_met() {
use crate::agent::test_helpers::MockProvider;
let judge = Arc::new(BoxedProvider::new(MockProvider::new(vec![])));
let g = GoalCondition::new("do the thing", judge);
let (v, usage) = g.evaluate("anything").await;
assert!(!v.satisfied, "a judge error must not declare the goal met");
assert_eq!(
usage.input_tokens + usage.output_tokens,
0,
"a failed judge call contributes no usage"
);
}
#[test]
fn transcript_tail_keeps_recent_evidence() {
let long: String = (0..5000).map(|i| format!("line {i}\n")).collect();
let t = tail_chars(&long, 100);
assert!(t.starts_with("[... earlier turns omitted ...]"));
assert!(t.contains("line 4999"));
assert!(!t.contains("line 0\n"));
assert_eq!(tail_chars("short", 100), "short");
}
use std::sync::atomic::{AtomicUsize, Ordering};
use crate::agent::AgentRunner;
use crate::error::Error;
use crate::llm::types::{
CompletionRequest, CompletionResponse, ContentBlock, StopReason, TokenUsage,
};
struct CountingWorker {
text: String,
calls: Arc<AtomicUsize>,
}
impl LlmProvider for CountingWorker {
async fn complete(&self, _request: CompletionRequest) -> Result<CompletionResponse, Error> {
self.calls.fetch_add(1, Ordering::SeqCst);
Ok(CompletionResponse {
content: vec![ContentBlock::Text {
text: self.text.clone(),
}],
stop_reason: StopReason::EndTurn,
reasoning: None,
usage: TokenUsage {
input_tokens: 1,
output_tokens: 1,
..Default::default()
},
model: None,
})
}
fn model_name(&self) -> Option<&str> {
Some("worker-mock")
}
}
struct FixedJudge(&'static str);
impl LlmProvider for FixedJudge {
async fn complete(&self, _request: CompletionRequest) -> Result<CompletionResponse, Error> {
Ok(CompletionResponse {
content: vec![ContentBlock::Text {
text: self.0.to_string(),
}],
stop_reason: StopReason::EndTurn,
reasoning: None,
usage: TokenUsage {
input_tokens: 1,
output_tokens: 1,
..Default::default()
},
model: None,
})
}
fn model_name(&self) -> Option<&str> {
Some("judge-mock")
}
}
fn worker(text: &str) -> (Arc<CountingWorker>, Arc<AtomicUsize>) {
let calls = Arc::new(AtomicUsize::new(0));
let w = Arc::new(CountingWorker {
text: text.to_string(),
calls: Arc::clone(&calls),
});
(w, calls)
}
fn judge(verdict: &'static str) -> Arc<BoxedProvider> {
Arc::new(BoxedProvider::new(FixedJudge(verdict)))
}
#[tokio::test]
async fn goal_satisfied_stops_after_one_turn() {
let (w, calls) = worker("I am done.");
let runner = AgentRunner::builder(w)
.name("w")
.system_prompt("sp")
.max_turns(10)
.goal(GoalCondition::new("obj", judge("GOAL_MET: YES")).with_max_continuations(5))
.build()
.expect("build");
let out = runner.execute("task").await.expect("run ok");
assert_eq!(out.goal_met, Some(true));
assert_eq!(
calls.load(Ordering::SeqCst),
1,
"satisfied → exactly one turn"
);
}
#[tokio::test]
async fn goal_unsatisfied_loops_to_cap_then_reports_false() {
let (w, calls) = worker("I am done, everything works!");
let runner = AgentRunner::builder(w)
.name("w")
.system_prompt("sp")
.max_turns(50)
.goal(
GoalCondition::new("obj", judge("GOAL_MET: NO: not yet")).with_max_continuations(2),
)
.build()
.expect("build");
let out = runner.execute("task").await.expect("run ok");
assert_eq!(out.goal_met, Some(false), "cap exhausted → goal unmet");
assert_eq!(
calls.load(Ordering::SeqCst),
3,
"initial completion + 2 continuations"
);
}
#[tokio::test]
async fn worker_self_claim_does_not_satisfy_an_independent_no_judge() {
let (w, calls) = worker("Done! Everything is complete and verified.");
let runner = AgentRunner::builder(w)
.name("w")
.system_prompt("sp")
.max_turns(50)
.goal(
GoalCondition::new("obj", judge("GOAL_MET: NO: show the test output"))
.with_max_continuations(1),
)
.build()
.expect("build");
let out = runner.execute("task").await.expect("run ok");
assert_eq!(out.goal_met, Some(false));
assert!(
calls.load(Ordering::SeqCst) > 1,
"the worker's 'I am done' claim must NOT stop the run when the judge says no"
);
}
#[tokio::test]
async fn goal_continuations_respect_max_turns() {
let (w, _calls) = worker("not done yet");
let runner = AgentRunner::builder(w)
.name("w")
.system_prompt("sp")
.max_turns(2)
.goal(
GoalCondition::new("obj", judge("GOAL_MET: NO: keep going"))
.with_max_continuations(1000),
)
.build()
.expect("build");
let result = runner.execute("task").await;
let err = result.expect_err("should hit max_turns, not loop forever");
let inner = match err {
Error::WithPartialUsage { source, .. } => *source,
other => other,
};
assert!(
matches!(inner, Error::MaxTurnsExceeded(2)),
"goal continuations must be bounded by max_turns, got {inner:?}"
);
}
#[tokio::test]
async fn no_goal_leaves_goal_met_none_and_one_turn() {
let (w, calls) = worker("done");
let runner = AgentRunner::builder(w)
.name("w")
.system_prompt("sp")
.max_turns(10)
.build()
.expect("build");
let out = runner.execute("task").await.expect("run ok");
assert_eq!(out.goal_met, None);
assert_eq!(calls.load(Ordering::SeqCst), 1);
}
#[tokio::test]
async fn goal_converges_when_judge_flips_to_yes() {
use crate::agent::test_helpers::MockProvider;
let (w, calls) = worker("working on it");
let judge_p = Arc::new(BoxedProvider::new(MockProvider::new(vec![
MockProvider::text_response("GOAL_MET: NO: not yet", 1, 1),
MockProvider::text_response("GOAL_MET: YES", 1, 1),
])));
let runner = AgentRunner::builder(w)
.name("w")
.system_prompt("sp")
.max_turns(10)
.goal(GoalCondition::new("obj", judge_p).with_max_continuations(5))
.build()
.expect("build");
let out = runner.execute("task").await.expect("run ok");
assert_eq!(
out.goal_met,
Some(true),
"the second verdict reaches the goal"
);
assert_eq!(
calls.load(Ordering::SeqCst),
2,
"1 initial completion (judged NO) + 1 continuation (judged YES)"
);
}
#[tokio::test]
async fn zero_continuations_judges_once_no_reprompt() {
let (w, calls) = worker("I am done");
let runner = AgentRunner::builder(w)
.name("w")
.system_prompt("sp")
.max_turns(10)
.goal(GoalCondition::new("obj", judge("GOAL_MET: NO: nope")).with_max_continuations(0))
.build()
.expect("build");
let out = runner.execute("task").await.expect("run ok");
assert_eq!(out.goal_met, Some(false));
assert_eq!(
calls.load(Ordering::SeqCst),
1,
"judged once, no continuation"
);
}
#[test]
fn goal_with_structured_schema_is_rejected_at_build() {
let (w, _calls) = worker("x");
let result = AgentRunner::builder(w)
.name("w")
.system_prompt("sp")
.structured_schema(serde_json::json!({"type": "object"}))
.goal(GoalCondition::new("obj", judge("GOAL_MET: YES")))
.build();
let err = match result {
Err(e) => e,
Ok(_) => panic!("goal + structured_schema must be rejected at build"),
};
assert!(err.to_string().contains("mutually exclusive"), "got: {err}");
}
}