1use super::report::{JudgeCriterionSummary, JudgeSummary};
2use super::types::{Expect, JudgeCriterionSpec, JudgeSpec};
3use crate::TaskRun;
4use crate::agents::truncate_chars;
5use crate::evals::format_transcript;
6use aether_core::events::AgentMessage;
7use futures::StreamExt;
8use llm::types::IsoString;
9use llm::{ChatMessage, ContentBlock, Context, LlmResponse, StreamingModelProvider};
10use schemars::{JsonSchema, Schema, schema_for};
11use serde::Deserialize;
12use std::borrow::Borrow;
13use std::collections::{BTreeMap, BTreeSet};
14use std::fs::read_to_string;
15use thiserror::Error;
16
17const JUDGE_CONTEXT_CHARS: usize = 4_000;
18
19#[derive(Debug, Clone)]
20pub struct Judge {
21 pub prompt: String,
22 pub criteria: Vec<JudgeCriterionSpec>,
23}
24
25#[derive(Debug, Clone, Default)]
26pub struct JudgeBuilder {
27 instructions: Option<String>,
28 task: Option<String>,
29 context: JudgeContext,
30 criteria: Vec<JudgeCriterionSpec>,
31}
32
33#[derive(Debug, Clone, Default)]
34pub struct JudgeContext {
35 pub transcript: Option<Vec<AgentMessage>>,
36 pub diff: Option<String>,
37 pub files: BTreeMap<String, String>,
38}
39
40#[derive(Debug, Error)]
41pub enum JudgeError {
42 #[error("invalid judge input: {0}")]
43 InvalidInput(String),
44
45 #[error("judge LLM stream error: {0}")]
46 Stream(#[from] llm::LlmError),
47
48 #[error("judge returned invalid JSON: {source}\nRaw response: {raw_response}")]
49 InvalidJson {
50 #[source]
51 source: serde_json::Error,
52 raw_response: String,
53 },
54
55 #[error("judge returned invalid judgment: {reason}\nRaw response: {raw_response}")]
56 InvalidJudgment { reason: String, raw_response: String },
57}
58
59pub fn judge() -> JudgeBuilder {
60 JudgeBuilder::default()
61}
62
63impl Judge {
64 pub fn response_schema() -> Schema {
65 JudgeRubricResponse::schema()
66 }
67
68 pub fn summarize(&self, response: JudgeRubricResponse) -> Result<JudgeSummary, JudgeError> {
69 let mut responses = BTreeMap::new();
70 for criterion in response.criteria {
71 let id = criterion.id.clone();
72 if responses.insert(id.clone(), criterion).is_some() {
73 return Err(invalid_judgment(format!("duplicate response criterion id `{id}`"), ""));
74 }
75 }
76
77 let mut summaries = Vec::with_capacity(self.criteria.len());
78 let mut weighted_score = 0.0;
79 let mut total_weight = 0.0;
80 let mut blocking_failed = false;
81
82 for criterion in &self.criteria {
83 let Some(response) = responses.remove(&criterion.id) else {
84 return Err(invalid_judgment(format!("missing response criterion `{}`", criterion.id), ""));
85 };
86 if !response.score.is_finite() || !(0.0..=1.0).contains(&response.score) {
87 return Err(invalid_judgment(
88 format!("criterion `{}` score must be between 0.0 and 1.0", criterion.id),
89 "",
90 ));
91 }
92
93 let passed = response.score >= criterion.threshold;
94 blocking_failed |= criterion.blocking && !passed;
95 weighted_score += response.score * criterion.weight;
96 total_weight += criterion.weight;
97 summaries.push(JudgeCriterionSummary {
98 id: criterion.id.clone(),
99 description: criterion.description.clone(),
100 blocking: criterion.blocking,
101 weight: criterion.weight,
102 threshold: criterion.threshold,
103 score: response.score,
104 passed,
105 reason: response.reason,
106 });
107 }
108
109 if let Some(id) = responses.keys().next() {
110 return Err(invalid_judgment(format!("unknown response criterion `{id}`"), ""));
111 }
112
113 let weighted_score = weighted_score / total_weight;
114 let score = if blocking_failed { 0.0 } else { weighted_score };
115 let reason = if blocking_failed {
116 format!("weighted score {:.2}; one or more blockers failed; {}", weighted_score, response.overall_reason)
117 } else {
118 format!("weighted score {:.2}; all blockers met; {}", weighted_score, response.overall_reason)
119 };
120
121 Ok(JudgeSummary { passed: !blocking_failed, score, reason, criteria: summaries })
122 }
123}
124
125impl JudgeBuilder {
126 pub fn instructions(mut self, instructions: impl Into<String>) -> Self {
127 self.instructions = Some(instructions.into());
128 self
129 }
130
131 pub fn task(mut self, task: impl Into<String>) -> Self {
132 self.task = Some(task.into());
133 self
134 }
135
136 pub fn transcript(mut self, transcript: impl Into<Vec<AgentMessage>>) -> Self {
137 self.context.transcript = Some(transcript.into());
138 self
139 }
140
141 pub fn diff(mut self, diff: impl Into<String>) -> Self {
142 self.context.diff = Some(diff.into());
143 self
144 }
145
146 pub fn file(mut self, path: impl Into<String>, contents: impl Into<String>) -> Self {
147 self.context.files.insert(path.into(), contents.into());
148 self
149 }
150
151 pub fn files<T, U, V>(mut self, files: T) -> Self
152 where
153 T: IntoIterator<Item = (U, V)>,
154 U: Into<String>,
155 V: Into<String>,
156 {
157 self.context.files.extend(files.into_iter().map(|(path, contents)| (path.into(), contents.into())));
158 self
159 }
160
161 pub fn criteria<T, U>(mut self, criteria: T) -> Self
162 where
163 T: IntoIterator<Item = U>,
164 U: Borrow<JudgeCriterionSpec>,
165 {
166 self.criteria = criteria.into_iter().map(|criterion| criterion.borrow().clone()).collect();
167 self
168 }
169
170 pub fn context(mut self, context: JudgeContext) -> Self {
171 self.context = context;
172 self
173 }
174
175 pub fn build(self) -> Result<Judge, JudgeError> {
176 let task = self.task.ok_or_else(|| JudgeError::InvalidInput("judge task must be provided".to_string()))?;
177 let criteria = normalize_criteria(self.criteria)?;
178 let prompt = build_prompt(&self.instructions.unwrap_or_default(), &task, &self.context, &criteria);
179 Ok(Judge { prompt, criteria })
180 }
181}
182
183pub(crate) struct JudgeRunner<'a> {
184 llm: &'a dyn StreamingModelProvider,
185 judge: Judge,
186}
187
188impl<'a> JudgeRunner<'a> {
189 pub(crate) fn from_eval_run(
190 llm: &'a dyn StreamingModelProvider,
191 run: &TaskRun,
192 expect: &Expect,
193 spec: &JudgeSpec,
194 ) -> Result<Self, JudgeError> {
195 let builder = judge()
196 .instructions(spec.instructions.clone().unwrap_or_default())
197 .task(run.prompt().to_string())
198 .transcript(run.transcript().messages().to_vec())
199 .criteria(&spec.criteria)
200 .files(collect_eval_context_files(run, expect, spec));
201
202 let builder = match run.workspace().capture_git_diffs().0 {
203 Some(diff) => builder.diff(truncate_chars(&diff.diff, JUDGE_CONTEXT_CHARS)),
204 None => builder,
205 };
206
207 Ok(Self { llm, judge: builder.build()? })
208 }
209
210 pub(crate) async fn run(&self) -> Result<JudgeSummary, JudgeError> {
211 tracing::info!("Running LLM judge");
212 let raw_response = self.stream_response().await?;
213 let response: JudgeRubricResponse = serde_json::from_str(extract_json_object(&raw_response))
214 .map_err(|source| JudgeError::InvalidJson { source, raw_response: raw_response.clone() })?;
215 self.judge.summarize(response)
216 }
217
218 async fn stream_response(&self) -> Result<String, JudgeError> {
219 let message = ChatMessage::User {
220 content: vec![ContentBlock::text(self.judge.prompt.clone())],
221 timestamp: IsoString::now(),
222 };
223 let mut response_stream = self.llm.stream_response(&Context::new(vec![message], vec![]));
224 let mut raw_response = String::new();
225 while let Some(result) = response_stream.next().await {
226 match result {
227 Ok(LlmResponse::Text { chunk }) => raw_response.push_str(&chunk),
228 Err(error) => return Err(JudgeError::Stream(error)),
229 _ => {}
230 }
231 }
232 Ok(raw_response)
233 }
234}
235
236#[derive(Debug, Deserialize, JsonSchema)]
237#[serde(deny_unknown_fields)]
238pub struct JudgeRubricResponse {
239 pub criteria: Vec<JudgeCriterionResponse>,
240 pub overall_reason: String,
241}
242
243#[derive(Debug, Deserialize, JsonSchema)]
244#[serde(deny_unknown_fields)]
245pub struct JudgeCriterionResponse {
246 pub id: String,
247 pub score: f64,
248 pub reason: String,
249}
250
251impl JudgeRubricResponse {
252 pub fn schema() -> Schema {
253 schema_for!(Self)
254 }
255}
256
257fn build_prompt(instructions: &str, task: &str, context: &JudgeContext, criteria: &[JudgeCriterionSpec]) -> String {
258 let mut sections = vec![
259 format!("## Instructions\n\n{instructions}"),
260 format!("## Task\n\nThe agent you're evaluating was given this task: <task>{task}</task>"),
261 ];
262
263 if let Some(transcript) = &context.transcript
264 && !transcript.is_empty()
265 {
266 sections.push(format!(
267 "## Agent Transcript\n\nTranscript of the agent you're evaluating: <transcript>{}</transcript>",
268 format_transcript(transcript)
269 ));
270 }
271
272 if let Some(diff) = &context.diff
273 && !diff.is_empty()
274 {
275 sections.push(format!("## Git diff\n\nGit diff produced by the agent you're evaluating: <diff>{diff}</diff>"));
276 }
277
278 if !context.files.is_empty() {
279 let blocks = context
280 .files
281 .iter()
282 .map(|(path, contents)| format!("<file><path>{path}</path><contents>{contents}</contents></file>"))
283 .collect::<Vec<_>>()
284 .join("\n");
285 sections.push(format!("## File Contents\n\nFiles under evaluation: <files>{blocks}</files>"));
286 }
287
288 let rubric = criteria
289 .iter()
290 .map(|criterion| {
291 format!(
292 "- id: {}\n blocking: {}\n weight: {}\n threshold: {}\n description: {}",
293 criterion.id, criterion.blocking, criterion.weight, criterion.threshold, criterion.description
294 )
295 })
296 .collect::<Vec<_>>()
297 .join("\n");
298 sections.push(format!("## Rubric criteria\n\n{rubric}"));
299 sections.push(format!(
300 "{}\n{}\n{}\n{}",
301 "Return exactly one result for every criterion ID above and no extra criteria.",
302 "Scores must be normalized numbers from 0.0 to 1.0.",
303 "Respond with ONLY a JSON object matching this schema:",
304 judge_response_schema()
305 ));
306
307 sections.join("\n\n")
308}
309
310fn normalize_criteria(criteria: Vec<JudgeCriterionSpec>) -> Result<Vec<JudgeCriterionSpec>, JudgeError> {
311 if criteria.is_empty() {
312 return Err(JudgeError::InvalidInput("judge criteria must not be empty".to_string()));
313 }
314
315 let mut ids = BTreeSet::new();
316 let mut normalized = Vec::with_capacity(criteria.len());
317 for mut criterion in criteria {
318 criterion.id = criterion.id.trim().to_string();
319 if criterion.id.is_empty() {
320 return Err(JudgeError::InvalidInput("judge criterion id must not be empty".to_string()));
321 }
322 if !ids.insert(criterion.id.clone()) {
323 return Err(JudgeError::InvalidInput(format!("duplicate judge criterion id `{}`", criterion.id)));
324 }
325 if criterion.description.trim().is_empty() {
326 return Err(JudgeError::InvalidInput(format!(
327 "judge criterion `{}` description must not be empty",
328 criterion.id
329 )));
330 }
331 if !criterion.weight.is_finite() || criterion.weight <= 0.0 {
332 return Err(JudgeError::InvalidInput(format!(
333 "judge criterion `{}` weight must be positive and finite",
334 criterion.id
335 )));
336 }
337 if !criterion.threshold.is_finite() || !(0.0..=1.0).contains(&criterion.threshold) {
338 return Err(JudgeError::InvalidInput(format!(
339 "judge criterion `{}` threshold must be between 0.0 and 1.0",
340 criterion.id
341 )));
342 }
343 normalized.push(criterion);
344 }
345 Ok(normalized)
346}
347
348fn collect_eval_context_files(run: &TaskRun, expect: &Expect, spec: &JudgeSpec) -> BTreeMap<String, String> {
349 expect
350 .files
351 .keys()
352 .chain(expect.files_contain.keys())
353 .chain(spec.context_files.iter())
354 .map(|path| {
355 let contents = read_to_string(run.workspace().join(path)).map_or_else(
356 |error| format!("could not read: {error}"),
357 |contents| truncate_chars(&contents, JUDGE_CONTEXT_CHARS),
358 );
359 (path.clone(), contents)
360 })
361 .collect()
362}
363
364fn extract_json_object(response: &str) -> &str {
365 let trimmed = response.trim();
366 match (trimmed.find('{'), trimmed.rfind('}')) {
367 (Some(start), Some(end)) if start <= end => &trimmed[start..=end],
368 _ => trimmed,
369 }
370}
371
372fn invalid_judgment(reason: String, raw_response: &str) -> JudgeError {
373 JudgeError::InvalidJudgment { reason, raw_response: raw_response.to_string() }
374}
375
376fn judge_response_schema() -> String {
377 serde_json::to_string_pretty(&JudgeRubricResponse::schema()).unwrap()
378}
379
380#[cfg(test)]
381mod tests {
382 use std::fs::write;
383 use std::path::Path;
384 use std::process::Command;
385
386 use super::*;
387 use crate::{GitRepoSpec, Transcript, Workspace};
388 use llm::testing::FakeLlmProvider;
389 use llm::{ChatMessage, LlmError, ToolCallRequest};
390
391 const VALID_RESPONSE: &str = r#"{"criteria":[{"id":"behavior","score":1.0,"reason":"correct"},{"id":"clarity","score":0.5,"reason":"brief"}],"overall_reason":"good"}"#;
392
393 #[test]
394 fn judge_builder_builds_prompt_from_context_and_criteria() {
395 let judge = judge()
396 .instructions("be strict")
397 .task("do the thing")
398 .diff("+added line")
399 .file("notes.txt", "beta\n")
400 .criteria([criterion("works", "the task works", true, 2.0, 0.9)])
401 .build()
402 .unwrap();
403
404 assert!(judge.prompt.contains("## Instructions\n\nbe strict"));
405 assert!(judge.prompt.contains("## Task"));
406 assert!(judge.prompt.contains("The agent you're evaluating was given this task: <task>do the thing</task>"));
407 assert!(judge.prompt.contains("## Git diff"));
408 assert!(judge.prompt.contains("Git diff produced by the agent you're evaluating: <diff>+added line</diff>"));
409 assert!(judge.prompt.contains("## File Contents"));
410 assert!(judge.prompt.contains("<path>notes.txt</path>"));
411 assert!(judge.prompt.contains("<contents>beta\n</contents>"));
412 assert!(judge.prompt.contains("## Rubric criteria"));
413 assert!(judge.prompt.contains("blocking: true"));
414 assert!(judge.prompt.contains("threshold: 0.9"));
415 assert!(judge.prompt.contains("weight: 2"));
416 assert!(judge.prompt.contains("Return exactly one result for every criterion ID above and no extra criteria."));
417 assert!(judge.prompt.contains("Respond with ONLY a JSON object matching this schema:"));
418 }
419
420 #[test]
421 fn judge_builder_accepts_slice_criteria() {
422 let criteria = vec![criterion("behavior", "does the thing", true, 1.0, 0.8)];
423 let judge = judge().task("do it").criteria(&criteria).build().unwrap();
424 assert_eq!(judge.criteria[0].id, "behavior");
425 }
426
427 #[test]
428 fn judge_summarizes_weighted_rubric() {
429 let judge = judge().task("prompt").criteria(judge_spec().criteria).build().unwrap();
430
431 let summary = judge.summarize(serde_json::from_str(VALID_RESPONSE).unwrap()).unwrap();
432
433 assert!(summary.passed);
434 assert!((summary.score - 0.875).abs() < f64::EPSILON);
435 assert!((summary.criteria[1].score - 0.5).abs() < f64::EPSILON);
436 assert!(summary.reason.contains("all blockers met"));
437 }
438
439 #[test]
440 fn judge_zeroes_score_when_blocker_fails() {
441 let judge = judge().task("prompt").criteria(judge_spec().criteria).build().unwrap();
442 let response = serde_json::from_str(
443 r#"{"criteria":[{"id":"behavior","score":0.75,"reason":"wrong behavior"},{"id":"clarity","score":1.0,"reason":"clear"}],"overall_reason":"bad"}"#,
444 )
445 .unwrap();
446
447 let summary = judge.summarize(response).unwrap();
448
449 assert!(!summary.passed);
450 assert!(summary.score.abs() < f64::EPSILON);
451 assert!(!summary.criteria[0].passed);
452 assert!(summary.reason.contains("one or more blockers failed"));
453 }
454
455 #[test]
456 fn judge_rejects_invalid_criterion_sets() {
457 let judge = judge().task("prompt").criteria(judge_spec().criteria).build().unwrap();
458 for raw_response in [
459 r#"{"criteria":[],"overall_reason":"missing"}"#,
460 r#"{"criteria":[{"id":"behavior","score":1.0,"reason":"ok"},{"id":"behavior","score":1.0,"reason":"ok"}],"overall_reason":"duplicate"}"#,
461 r#"{"criteria":[{"id":"behavior","score":1.0,"reason":"ok"},{"id":"clarity","score":1.0,"reason":"ok"},{"id":"extra","score":1.0,"reason":"ok"}],"overall_reason":"unknown"}"#,
462 r#"{"criteria":[{"id":"behavior","score":1.5,"reason":"bad"},{"id":"clarity","score":1.0,"reason":"ok"}],"overall_reason":"score"}"#,
463 ] {
464 let response = serde_json::from_str(raw_response).unwrap();
465
466 let error = judge.summarize(response).unwrap_err();
467
468 assert!(matches!(error, JudgeError::InvalidJudgment { .. }), "response: {raw_response}");
469 }
470 }
471
472 #[test]
473 fn judge_builder_rejects_invalid_inputs() {
474 for (builder, expected) in [
475 (judge().criteria([criterion("behavior", "ok", true, 1.0, 0.8)]), "judge task must be provided"),
476 (judge().task("prompt"), "judge criteria must not be empty"),
477 (
478 judge().task("prompt").criteria([criterion("", "ok", true, 1.0, 0.8)]),
479 "judge criterion id must not be empty",
480 ),
481 (
482 judge().task("prompt").criteria([criterion("behavior", "ok", true, 0.0, 0.8)]),
483 "weight must be positive and finite",
484 ),
485 ] {
486 let error = builder.build().unwrap_err();
487 assert!(error.to_string().contains(expected), "got: {error}");
488 }
489 }
490
491 #[tokio::test]
492 async fn judge_runner_extracts_json_object_from_surrounding_prose() {
493 let response = format!("Here is my assessment:\n{VALID_RESPONSE}");
494 let judge_llm = FakeLlmProvider::with_single_response(vec![LlmResponse::text(&response)]);
495
496 let summary = JudgeRunner::from_eval_run(&judge_llm, &run(), &Expect::default(), &judge_spec())
497 .unwrap()
498 .run()
499 .await
500 .unwrap();
501
502 assert!(summary.passed);
503 }
504
505 #[tokio::test]
506 async fn judge_runner_returns_invalid_json_error_with_raw_response() {
507 let judge_llm = FakeLlmProvider::with_single_response(vec![LlmResponse::text("not json")]);
508
509 let error = JudgeRunner::from_eval_run(&judge_llm, &run(), &Expect::default(), &judge_spec())
510 .unwrap()
511 .run()
512 .await
513 .unwrap_err();
514
515 let JudgeError::InvalidJson { raw_response, .. } = error else {
516 panic!("expected InvalidJson, got {error:?}");
517 };
518 assert_eq!(raw_response, "not json");
519 }
520
521 #[tokio::test]
522 async fn judge_runner_returns_stream_error_on_llm_failure() {
523 let judge_llm = FakeLlmProvider::from_results(vec![vec![Err(LlmError::Other("boom".to_string()))]]);
524
525 let error = JudgeRunner::from_eval_run(&judge_llm, &run(), &Expect::default(), &judge_spec())
526 .unwrap()
527 .run()
528 .await
529 .unwrap_err();
530
531 assert!(matches!(error, JudgeError::Stream(_)));
532 assert!(error.to_string().contains("boom"));
533 }
534
535 #[tokio::test]
536 async fn judge_runner_prompt_includes_eval_run_context() {
537 let messages = vec![
538 AgentMessage::ToolCall {
539 request: ToolCallRequest {
540 id: "call_1".to_string(),
541 name: "bash".to_string(),
542 arguments: "{}".to_string(),
543 },
544 model_name: "test".to_string(),
545 },
546 AgentMessage::text("msg_1", "all done", true, "test"),
547 ];
548 let run = TaskRun::new("edit the file".to_string(), Workspace::empty().unwrap(), Transcript::new(messages));
549 let judge_llm = FakeLlmProvider::with_single_response(vec![LlmResponse::text(VALID_RESPONSE)]);
550
551 JudgeRunner::from_eval_run(&judge_llm, &run, &Expect::default(), &judge_spec()).unwrap().run().await.unwrap();
552
553 let prompt = judged_prompt(&judge_llm);
554 assert!(prompt.contains("Grade maintainability."));
555 assert!(prompt.contains("behavior"));
556 assert!(prompt.contains("The behavior is correct."));
557 assert!(prompt.contains("edit the file"));
558 assert!(prompt.contains("[tool-call] bash"));
559 assert!(prompt.contains("[agent] all done"));
560 assert!(prompt.contains("overall_reason"));
561 }
562
563 #[tokio::test]
564 async fn judge_runner_prompt_includes_agent_diff_from_workspace() {
565 let workspace = git_workspace_with_agent_change();
566 let run = TaskRun::new("p".to_string(), workspace, Transcript::new(vec![AgentMessage::Done]));
567 let judge_llm = FakeLlmProvider::with_single_response(vec![LlmResponse::text(VALID_RESPONSE)]);
568
569 JudgeRunner::from_eval_run(&judge_llm, &run, &Expect::default(), &judge_spec()).unwrap().run().await.unwrap();
570
571 let prompt = judged_prompt(&judge_llm);
572 assert!(prompt.contains("## Git diff"));
573 assert!(prompt.contains("+agent change"));
574 }
575
576 #[tokio::test]
577 async fn judge_runner_prompt_includes_final_contents_of_files_under_evaluation() {
578 let workspace =
579 Workspace::from_files([("notes.txt", "beta\nalpha\n"), ("extra.txt", "extra context")]).unwrap();
580 let run = TaskRun::new("p".to_string(), workspace, Transcript::new(vec![AgentMessage::Done]));
581 let expect =
582 Expect { files_contain: [("notes.txt".to_string(), "beta".to_string())].into(), ..Expect::default() };
583 let judge_llm = FakeLlmProvider::with_single_response(vec![LlmResponse::text(VALID_RESPONSE)]);
584
585 JudgeRunner::from_eval_run(&judge_llm, &run, &expect, &judge_spec()).unwrap().run().await.unwrap();
586
587 let prompt = judged_prompt(&judge_llm);
588 assert!(prompt.contains("<path>notes.txt</path>"));
589 assert!(prompt.contains("beta\nalpha"));
590 assert!(prompt.contains("<path>extra.txt</path>"));
591 assert!(prompt.contains("extra context"));
592 }
593
594 fn run() -> TaskRun {
595 TaskRun::new("prompt".to_string(), Workspace::empty().unwrap(), Transcript::new(vec![AgentMessage::Done]))
596 }
597
598 fn criterion(id: &str, description: &str, blocking: bool, weight: f64, threshold: f64) -> JudgeCriterionSpec {
599 JudgeCriterionSpec { id: id.to_string(), description: description.to_string(), blocking, weight, threshold }
600 }
601
602 fn git_workspace_with_agent_change() -> Workspace {
603 let source = tempfile::tempdir().unwrap();
604 git(source.path(), ["init", "--initial-branch", "main"]);
605 git(source.path(), ["config", "user.email", "eval@example.com"]);
606 git(source.path(), ["config", "user.name", "Eval"]);
607 std::fs::write(source.path().join("notes.txt"), "start\n").unwrap();
608 git(source.path(), ["add", "."]);
609 git(source.path(), ["commit", "-m", "start"]);
610 let start_commit = git_output(source.path(), ["rev-parse", "HEAD"]);
611 std::fs::write(source.path().join("notes.txt"), "gold\n").unwrap();
612 git(source.path(), ["commit", "-am", "gold"]);
613 let gold_commit = git_output(source.path(), ["rev-parse", "HEAD"]);
614
615 let workspace = Workspace::from_git_repo(GitRepoSpec {
616 url: source.path().to_string_lossy().to_string(),
617 start_commit,
618 gold_commit,
619 subdir: None,
620 })
621 .unwrap();
622 write(workspace.join("notes.txt"), "start\nagent change\n").unwrap();
623 workspace
624 }
625
626 fn git<const N: usize>(cwd: &Path, args: [&str; N]) {
627 let output = Command::new("git").args(args).current_dir(cwd).output().unwrap();
628 assert!(output.status.success(), "git failed: {}", String::from_utf8_lossy(&output.stderr));
629 }
630
631 fn git_output<const N: usize>(cwd: &Path, args: [&str; N]) -> String {
632 let output = Command::new("git").args(args).current_dir(cwd).output().unwrap();
633 assert!(output.status.success(), "git failed: {}", String::from_utf8_lossy(&output.stderr));
634 String::from_utf8(output.stdout).unwrap().trim().to_string()
635 }
636
637 fn judge_spec() -> JudgeSpec {
638 serde_json::from_str(
639 r#"{
640 "model": "judge:model",
641 "instructions": "Grade maintainability.",
642 "contextFiles": ["extra.txt"],
643 "criteria": [
644 { "id": "behavior", "description": "The behavior is correct.", "blocking": true, "weight": 3.0, "threshold": 1.0 },
645 { "id": "clarity", "description": "The response is clear.", "blocking": false, "weight": 1.0, "threshold": 0.5 }
646 ]
647 }"#,
648 )
649 .unwrap()
650 }
651
652 fn judged_prompt(judge_llm: &FakeLlmProvider) -> String {
653 let contexts = judge_llm.captured_contexts();
654 let contexts = contexts.lock().unwrap();
655 let ChatMessage::User { content, .. } = &contexts[0].messages()[0] else {
656 panic!("expected a user message in the judge context");
657 };
658 ContentBlock::join_text(content)
659 }
660}