1use std::future::Future;
16use std::path::PathBuf;
17use std::pin::Pin;
18use std::time::Duration;
19
20use serde_json::json;
21
22use crate::error::Error;
23use crate::llm::types::ToolDefinition;
24use crate::tool::{Tool, ToolOutput};
25
26const DEFAULT_VERIFY_TIMEOUT: Duration = Duration::from_secs(300);
28const DEFAULT_VERIFY_TAIL: usize = 4_000;
30
31#[cfg(unix)]
43struct ProcessGroupKiller {
44 pgid: Option<u32>,
45 armed: bool,
46}
47
48#[cfg(unix)]
49impl ProcessGroupKiller {
50 fn new(pgid: Option<u32>) -> Self {
51 Self { pgid, armed: true }
52 }
53
54 fn disarm(&mut self) {
55 self.armed = false;
56 }
57}
58
59#[cfg(unix)]
60impl Drop for ProcessGroupKiller {
61 fn drop(&mut self) {
62 if self.armed
63 && let Some(pgid) = self.pgid
64 {
65 unsafe {
69 libc::kill(-(pgid as i32), libc::SIGKILL);
70 }
71 }
72 }
73}
74
75#[derive(Debug, Clone, PartialEq, Eq)]
78pub struct VerifyOutcome {
79 pub passed: bool,
81 pub exit_code: i32,
83 pub command: String,
85}
86
87pub const VERIFY_SENTINEL: &str = "VERIFY_RESULT:";
90
91pub fn render_verify_result(
96 command: &str,
97 exit_code: i32,
98 output: &str,
99 max_tail: usize,
100) -> String {
101 let verdict = if exit_code == 0 { "PASS" } else { "FAIL" };
102 let total = output.len();
103 let tail = tail_chars(output, max_tail);
104 let body_header = if tail.len() < total {
105 format!("--- output (tail, {} of {total} bytes) ---", tail.len())
106 } else {
107 "--- output ---".to_string()
108 };
109 format!(
110 "$ {command}\nexit_code={exit_code}\n{body_header}\n{tail}\n\
111 {VERIFY_SENTINEL} {verdict} exit_code={exit_code} command={command}"
112 )
113}
114
115fn tail_chars(s: &str, max: usize) -> &str {
118 if s.len() <= max {
119 return s;
120 }
121 let mut start = s.len() - max;
122 while start < s.len() && !s.is_char_boundary(start) {
123 start += 1;
124 }
125 &s[start..]
126}
127
128pub fn parse_latest_verify(transcript: &str) -> Option<VerifyOutcome> {
147 transcript.lines().rev().find_map(parse_canonical_line)
148}
149
150fn parse_canonical_line(line: &str) -> Option<VerifyOutcome> {
154 const WRAPPED: &str = "[Tool result: VERIFY_RESULT:";
159 let payload = if line.starts_with(VERIFY_SENTINEL) {
160 line
161 } else {
162 let idx = line.find(WRAPPED)?;
163 &line[idx + WRAPPED.len() - VERIFY_SENTINEL.len()..]
164 };
165 let rest = payload.strip_prefix(VERIFY_SENTINEL)?.strip_prefix(' ')?;
166 let (verdict, rest) = rest.split_once(' ')?;
167 let passed = match verdict {
168 "PASS" => true,
169 "FAIL" => false,
170 _ => return None,
171 };
172 let rest = rest.strip_prefix("exit_code=")?;
173 let (code, rest) = rest.split_once(' ')?;
174 let exit_code = code.parse::<i32>().ok()?;
175 let command = rest.strip_prefix("command=")?.trim().to_string();
176 Some(VerifyOutcome {
177 passed,
178 exit_code,
179 command,
180 })
181}
182
183pub struct VerifyCommandTool {
199 commands: Vec<String>,
200 workspace: PathBuf,
201 timeout: Duration,
202 max_tail: usize,
203}
204
205impl VerifyCommandTool {
206 pub fn new(workspace: impl Into<PathBuf>, commands: Vec<String>) -> Self {
209 Self {
210 commands,
211 workspace: workspace.into(),
212 timeout: DEFAULT_VERIFY_TIMEOUT,
213 max_tail: DEFAULT_VERIFY_TAIL,
214 }
215 }
216
217 pub fn timeout(mut self, timeout: Duration) -> Self {
219 self.timeout = timeout;
220 self
221 }
222
223 pub fn max_tail(mut self, max_tail: usize) -> Self {
225 self.max_tail = max_tail;
226 self
227 }
228}
229
230impl Tool for VerifyCommandTool {
231 fn definition(&self) -> ToolDefinition {
232 ToolDefinition {
233 name: "verify".into(),
234 description: "Run the project's configured verification command(s) (build/test/lint) \
235 in the workspace and report PASS/FAIL with the exit code. Takes no \
236 arguments — the command is fixed by the harness. Call this after making \
237 changes; the task is only complete when this reports `VERIFY_RESULT: PASS`."
238 .into(),
239 input_schema: json!({
240 "type": "object",
241 "properties": {},
242 "additionalProperties": false
243 }),
244 }
245 }
246
247 fn execute(
248 &self,
249 _ctx: &crate::ExecutionContext,
250 _input: serde_json::Value,
251 ) -> Pin<Box<dyn Future<Output = Result<ToolOutput, Error>> + Send + '_>> {
252 Box::pin(async move {
253 if self.commands.is_empty() {
254 return Ok(ToolOutput::error(
255 "no verify command configured for this code harness",
256 ));
257 }
258
259 let mut blocks: Vec<String> = Vec::with_capacity(self.commands.len());
260 for command in &self.commands {
261 let mut cmd = tokio::process::Command::new("bash");
262 cmd.arg("-c")
263 .arg(command)
264 .current_dir(&self.workspace)
265 .stdin(std::process::Stdio::null())
266 .stdout(std::process::Stdio::piped())
267 .stderr(std::process::Stdio::piped())
268 .kill_on_drop(true);
269
270 #[cfg(unix)]
275 {
276 use std::os::unix::process::CommandExt as _;
277 cmd.as_std_mut().process_group(0);
278 }
279
280 let child = match cmd.spawn() {
281 Ok(c) => c,
282 Err(e) => {
285 return Ok(ToolOutput::error(format!(
286 "verify command `{command}` could not be spawned: {e}"
287 )));
288 }
289 };
290
291 #[cfg(unix)]
295 let mut group_killer = ProcessGroupKiller::new(child.id());
296
297 match tokio::time::timeout(self.timeout, child.wait_with_output()).await {
298 Ok(Ok(output)) => {
299 #[cfg(unix)]
302 group_killer.disarm();
303 let exit_code = output.status.code().unwrap_or(-1);
304 let mut combined = String::from_utf8_lossy(&output.stdout).into_owned();
305 let stderr = String::from_utf8_lossy(&output.stderr);
306 if !stderr.is_empty() {
307 if !combined.is_empty() {
308 combined.push('\n');
309 }
310 combined.push_str(&stderr);
311 }
312 blocks.push(render_verify_result(
313 command,
314 exit_code,
315 &combined,
316 self.max_tail,
317 ));
318 if exit_code != 0 {
319 break; }
321 }
322 Ok(Err(e)) => {
323 return Ok(ToolOutput::error(format!(
324 "verify command `{command}` failed to run: {e}"
325 )));
326 }
327 Err(_) => {
328 let note = format!("(timed out after {}s)", self.timeout.as_secs());
334 blocks.push(render_verify_result(command, -1, ¬e, self.max_tail));
335 break;
336 }
337 }
338 }
339
340 Ok(ToolOutput::success(blocks.join("\n\n")))
341 })
342 }
343}
344
345#[cfg(test)]
346mod tests {
347 use super::*;
348
349 #[test]
350 fn render_pass_includes_sentinel_and_header() {
351 let s = render_verify_result("true", 0, "all good", 3000);
352 assert!(s.contains("$ true"), "header missing: {s}");
353 assert!(s.contains("exit_code=0"), "exit code missing: {s}");
354 assert!(s.contains("all good"), "output missing: {s}");
355 assert!(
356 s.contains("VERIFY_RESULT: PASS exit_code=0 command=true"),
357 "sentinel missing/wrong: {s}"
358 );
359 }
360
361 #[test]
362 fn render_fail_sentinel() {
363 let s = render_verify_result("cargo test", 101, "boom", 3000);
364 assert!(
365 s.contains("VERIFY_RESULT: FAIL exit_code=101 command=cargo test"),
366 "fail sentinel wrong: {s}"
367 );
368 }
369
370 #[test]
371 fn render_sentinel_is_last_line() {
372 let s = render_verify_result("x", 0, "noise\nmore", 3000);
373 let last = s.lines().last().unwrap_or("");
374 assert!(
375 last.starts_with("VERIFY_RESULT:"),
376 "last line not sentinel: {last:?}"
377 );
378 }
379
380 #[test]
381 fn render_caps_long_output_tail() {
382 let big = "x".repeat(10_000);
383 let s = render_verify_result("c", 0, &big, 100);
384 assert!(s.contains("of 10000 bytes"), "elision note missing: {s}");
386 assert!(s.len() < 1_000, "tail not capped, len={}", s.len());
387 assert!(
388 s.contains("VERIFY_RESULT: PASS"),
389 "sentinel survived cap: {s}"
390 );
391 }
392
393 #[test]
394 fn parse_roundtrips_render_pass() {
395 let s = render_verify_result("cargo test --workspace", 0, "ok", 3000);
396 let got = parse_latest_verify(&s).expect("should parse");
397 assert_eq!(
398 got,
399 VerifyOutcome {
400 passed: true,
401 exit_code: 0,
402 command: "cargo test --workspace".to_string()
403 }
404 );
405 }
406
407 #[test]
408 fn parse_roundtrips_render_fail() {
409 let s = render_verify_result("npm test", 1, "fail", 3000);
410 let got = parse_latest_verify(&s).expect("should parse");
411 assert!(!got.passed);
412 assert_eq!(got.exit_code, 1);
413 assert_eq!(got.command, "npm test");
414 }
415
416 #[test]
417 fn parse_returns_latest_of_many() {
418 let transcript = format!(
420 "{}\n{}",
421 render_verify_result("cargo test", 101, "first attempt failed", 3000),
422 render_verify_result("cargo test", 0, "now passes", 3000),
423 );
424 let got = parse_latest_verify(&transcript).expect("should parse");
425 assert!(got.passed, "latest should be PASS, got {got:?}");
426 assert_eq!(got.exit_code, 0);
427 }
428
429 #[test]
430 fn parse_none_when_absent() {
431 assert_eq!(parse_latest_verify("no verification happened here"), None);
432 }
433
434 #[test]
435 fn parse_ignores_prose_echo_and_trusts_canonical_sentinel() {
436 let transcript = "User: [Tool result: $ pytest\nexit_code=0\n--- output ---\nok\n\
444 VERIFY_RESULT: PASS exit_code=0 command=pytest]\n\
445 Assistant: Earlier it showed VERIFY_RESULT: FAIL but I fixed it; passes now.";
446 let got = parse_latest_verify(transcript).expect("canonical sentinel present");
447 assert!(
448 got.passed,
449 "must trust the canonical tool sentinel, not the prose 'FAIL': {got:?}"
450 );
451 assert_eq!(got.exit_code, 0);
452 assert!(
455 got.command.starts_with("pytest"),
456 "command: {:?}",
457 got.command
458 );
459 }
460
461 #[test]
466 fn parse_rejects_midline_fullformat_echo() {
467 let transcript =
468 "Assistant: per the docs, look for VERIFY_RESULT: PASS exit_code=0 command=cargo test";
469 assert_eq!(parse_latest_verify(transcript), None);
470 }
471
472 #[test]
475 fn parse_rejects_malformed_sentinel_lines() {
476 assert_eq!(
478 parse_latest_verify("VERIFY_RESULT: pass exit_code=0 command=x"),
479 None
480 );
481 assert_eq!(
483 parse_latest_verify("VERIFY_RESULT: PASS command=x exit_code=0"),
484 None
485 );
486 assert_eq!(
488 parse_latest_verify("VERIFY_RESULT: PASS exit_code=zero command=x"),
489 None
490 );
491 }
492
493 #[test]
498 fn parse_accepts_tool_result_wrapped_single_line_sentinel() {
499 let transcript = "User: [Tool result: VERIFY_RESULT: FAIL exit_code=1 command=cargo test]";
500 let got = parse_latest_verify(transcript).expect("wrapped sentinel must parse");
501 assert!(!got.passed);
502 assert_eq!(got.exit_code, 1);
503 }
504
505 #[test]
507 fn parse_midline_echo_does_not_override_genuine_fail() {
508 let transcript = format!(
509 "{}\nAssistant: fixed! VERIFY_RESULT: PASS exit_code=0 command=cargo test",
510 render_verify_result("cargo test", 101, "boom", 3000)
511 );
512 let got = parse_latest_verify(&transcript).expect("genuine FAIL present");
513 assert!(!got.passed, "mid-line PASS echo must not gate: {got:?}");
514 assert_eq!(got.exit_code, 101);
515 }
516
517 #[test]
518 fn parse_none_when_only_a_prose_echo_exists() {
519 assert_eq!(
521 parse_latest_verify("the docs say to look for VERIFY_RESULT: PASS lines"),
522 None
523 );
524 }
525
526 #[test]
527 fn parse_handles_sentinel_embedded_in_a_tool_result_line() {
528 let transcript =
531 "[Tool result: ...\nVERIFY_RESULT: PASS exit_code=0 command=cargo check\nmore text]";
532 let got = parse_latest_verify(transcript).expect("should parse");
533 assert!(got.passed);
534 assert_eq!(got.command, "cargo check");
535 }
536
537 fn run_verify(tool: &VerifyCommandTool) -> ToolOutput {
540 tokio::runtime::Runtime::new()
543 .unwrap()
544 .block_on(tool.execute(&crate::ExecutionContext::default(), json!({})))
545 .expect("tool execute should not error at the framework level")
546 }
547
548 #[test]
549 fn definition_is_verify_with_no_args() {
550 let tool = VerifyCommandTool::new(std::env::temp_dir(), vec!["true".into()]);
551 let def = tool.definition();
552 assert_eq!(def.name, "verify");
553 assert_eq!(def.input_schema["properties"], json!({}));
555 }
556
557 #[test]
558 fn verify_passes_on_true() {
559 let dir = tempfile::tempdir().unwrap();
560 let tool = VerifyCommandTool::new(dir.path(), vec!["true".into()]);
561 let out = run_verify(&tool);
562 assert!(
563 !out.is_error,
564 "a passing verify is not a tool error: {}",
565 out.content
566 );
567 let parsed = parse_latest_verify(&out.content).expect("sentinel present");
568 assert!(parsed.passed);
569 assert_eq!(parsed.exit_code, 0);
570 }
571
572 #[test]
573 fn verify_reports_fail_without_tool_error() {
574 let dir = tempfile::tempdir().unwrap();
575 let tool = VerifyCommandTool::new(dir.path(), vec!["exit 3".into()]);
576 let out = run_verify(&tool);
577 assert!(
580 !out.is_error,
581 "FAIL must not be a tool error: {}",
582 out.content
583 );
584 let parsed = parse_latest_verify(&out.content).expect("sentinel present");
585 assert!(!parsed.passed);
586 assert_eq!(parsed.exit_code, 3);
587 }
588
589 #[test]
590 fn verify_runs_in_workspace_cwd() {
591 let dir = tempfile::tempdir().unwrap();
592 std::fs::write(dir.path().join("marker_file.txt"), "x").unwrap();
593 let tool = VerifyCommandTool::new(dir.path(), vec!["ls".into()]);
594 let out = run_verify(&tool);
595 assert!(
596 out.content.contains("marker_file.txt"),
597 "verify should run in the workspace cwd: {}",
598 out.content
599 );
600 }
601
602 #[test]
603 fn verify_fail_fast_skips_later_commands() {
604 let dir = tempfile::tempdir().unwrap();
605 let tool = VerifyCommandTool::new(
606 dir.path(),
607 vec!["false".into(), "echo SHOULD_NOT_RUN".into()],
608 );
609 let out = run_verify(&tool);
610 assert!(
611 !out.content.contains("SHOULD_NOT_RUN"),
612 "fail-fast must stop before the second command: {}",
613 out.content
614 );
615 assert!(!parse_latest_verify(&out.content).unwrap().passed);
616 }
617
618 #[test]
619 fn verify_all_pass_overall_pass() {
620 let dir = tempfile::tempdir().unwrap();
621 let tool = VerifyCommandTool::new(dir.path(), vec!["true".into(), "true".into()]);
622 let out = run_verify(&tool);
623 assert!(parse_latest_verify(&out.content).unwrap().passed);
624 }
625
626 #[test]
627 fn verify_no_command_configured_is_tool_error() {
628 let tool = VerifyCommandTool::new(std::env::temp_dir(), vec![]);
629 let out = run_verify(&tool);
630 assert!(out.is_error, "an unconfigured verify is a misconfiguration");
631 assert!(out.content.to_lowercase().contains("no verify"));
632 }
633
634 #[cfg(unix)]
639 #[test]
640 fn verify_timeout_kills_process_group() {
641 let dir = tempfile::tempdir().unwrap();
642 let marker = dir.path().join("orphan_marker");
643 let cmd = format!("( sleep 1; touch {} ) & wait", marker.display());
644 let tool =
645 VerifyCommandTool::new(dir.path(), vec![cmd]).timeout(Duration::from_millis(100));
646 let out = run_verify(&tool);
647 assert!(!out.is_error, "timeout is a verdict: {}", out.content);
649 assert!(!parse_latest_verify(&out.content).expect("sentinel").passed);
650 std::thread::sleep(Duration::from_millis(1500));
652 assert!(
653 !marker.exists(),
654 "timed-out verify must SIGKILL its process group (orphan survived)"
655 );
656 }
657
658 #[cfg(unix)]
662 #[test]
663 fn verify_dropped_future_kills_process_group() {
664 let dir = tempfile::tempdir().unwrap();
665 let marker = dir.path().join("orphan_marker_drop");
666 let cmd = format!("( sleep 1; touch {} ) & wait", marker.display());
667 let tool = VerifyCommandTool::new(dir.path(), vec![cmd]);
668 tokio::runtime::Runtime::new().unwrap().block_on(async {
669 let fut = tool.execute(&crate::ExecutionContext::default(), json!({}));
670 let _ = tokio::time::timeout(Duration::from_millis(300), fut).await;
673 });
674 std::thread::sleep(Duration::from_millis(1500));
675 assert!(
676 !marker.exists(),
677 "dropping the verify future must SIGKILL its process group (orphan survived)"
678 );
679 }
680}