1use crate::collab::{AgentOutput, parse_artifacts_from_output};
2use crate::config::{BuildError, BuildErrorLevel, TestFailure};
3use anyhow::Result;
4use serde_json::Value;
5use uuid::Uuid;
6
7pub struct ValidationOutcome {
9 pub output: AgentOutput,
11 pub status: &'static str,
13 pub error: Option<String>,
15}
16
17fn detect_fatal_agent_error(stdout: &str, stderr: &str) -> Option<&'static str> {
18 let stderr_lower = stderr.to_ascii_lowercase();
22 let stdout_plain: String = stdout
23 .lines()
24 .filter(|line| !line.starts_with('{'))
25 .collect::<Vec<_>>()
26 .join("\n")
27 .to_ascii_lowercase();
28 let combined = format!("{}\n{}", stdout_plain, stderr_lower);
29 let patterns = [
30 ("rate-limited", "provider rate limit exceeded"),
31 ("rate limited", "provider rate limit exceeded"),
32 ("quota exceeded", "provider quota exceeded"),
33 ("quota exhausted", "provider quota exhausted"),
34 ("quota resets in", "provider quota exhausted"),
35 ("authentication failed", "provider authentication failed"),
36 ("invalid api key", "provider authentication failed"),
37 ];
38
39 patterns
40 .iter()
41 .find_map(|(needle, reason)| combined.contains(needle).then_some(*reason))
42}
43
44fn is_strict_phase(phase: &str) -> bool {
45 const SDLC_PHASES: &[&str] = &[
55 "qa_testing",
56 "qa_doc_gen",
57 "ticket_fix",
58 "align_tests",
59 "doc_governance",
60 ];
61 if SDLC_PHASES.contains(&phase) {
62 return false;
63 }
64 const STRICT: &[&str] = &["qa", "fix", "retest", "guard", "adaptive_plan"];
65 STRICT
66 .iter()
67 .any(|s| phase == *s || phase.ends_with(&format!("_{}", s)))
68}
69
70fn is_build_phase(phase: &str) -> bool {
72 matches!(phase, "build" | "lint")
73}
74
75fn is_test_phase(phase: &str) -> bool {
76 phase == "test"
77}
78
79pub fn validate_phase_output(
81 phase: &str,
82 run_id: Uuid,
83 agent_id: &str,
84 exit_code: i64,
85 stdout: &str,
86 stderr: &str,
87) -> Result<ValidationOutcome> {
88 if let Some(reason) = detect_fatal_agent_error(stdout, stderr) {
89 let output = AgentOutput::new(
90 run_id,
91 agent_id.to_string(),
92 phase.to_string(),
93 exit_code,
94 stdout.to_string(),
95 stderr.to_string(),
96 );
97 return Ok(ValidationOutcome {
98 output,
99 status: "failed",
100 error: Some(reason.to_string()),
101 });
102 }
103
104 let strict = is_strict_phase(phase);
105 let parsed_json = serde_json::from_str::<Value>(stdout);
106
107 if strict && parsed_json.is_err() {
108 let output = AgentOutput::new(
109 run_id,
110 agent_id.to_string(),
111 phase.to_string(),
112 exit_code,
113 stdout.to_string(),
114 stderr.to_string(),
115 );
116 return Ok(ValidationOutcome {
117 output,
118 status: "failed",
119 error: Some("strict phase requires JSON stdout".to_string()),
120 });
121 }
122
123 let parsed = parsed_json.ok();
124 let confidence = parsed
125 .as_ref()
126 .and_then(|v| v.get("confidence"))
127 .and_then(|v| v.as_f64())
128 .map(|v| v as f32)
129 .unwrap_or(1.0);
130 let quality_score = parsed
131 .as_ref()
132 .and_then(|v| v.get("quality_score"))
133 .and_then(|v| v.as_f64())
134 .map(|v| v as f32)
135 .unwrap_or(1.0);
136
137 let artifacts = match &parsed {
138 Some(v) => {
139 if let Some(arr) = v.get("artifacts") {
140 parse_artifacts_from_output(&serde_json::to_string(arr).unwrap_or_default())
141 } else {
142 parse_artifacts_from_output(stdout)
143 }
144 }
145 None => parse_artifacts_from_output(stdout),
146 };
147
148 let build_errors = if is_build_phase(phase) {
150 parsed
151 .as_ref()
152 .and_then(|v| v.get("build_errors"))
153 .and_then(|v| serde_json::from_value::<Vec<BuildError>>(v.clone()).ok())
154 .unwrap_or_else(|| parse_build_errors_from_text(stderr, stdout))
155 } else {
156 Vec::new()
157 };
158
159 let test_failures = if is_test_phase(phase) {
161 parsed
162 .as_ref()
163 .and_then(|v| v.get("test_failures"))
164 .and_then(|v| serde_json::from_value::<Vec<TestFailure>>(v.clone()).ok())
165 .unwrap_or_else(|| parse_test_failures_from_text(stderr, stdout))
166 } else {
167 Vec::new()
168 };
169
170 let mut output = AgentOutput::new(
171 run_id,
172 agent_id.to_string(),
173 phase.to_string(),
174 exit_code,
175 stdout.to_string(),
176 stderr.to_string(),
177 )
178 .with_artifacts(artifacts)
179 .with_confidence(confidence)
180 .with_quality_score(quality_score);
181
182 output.build_errors = build_errors;
183 output.test_failures = test_failures;
184
185 Ok(ValidationOutcome {
186 output,
187 status: "passed",
188 error: None,
189 })
190}
191
192trait DiagnosticParser: Default {
196 type Item;
197 fn process_line(&mut self, line: &str);
198 fn finish(self) -> Vec<Self::Item>;
199}
200
201fn parse_diagnostic_output<P: DiagnosticParser>(stderr: &str, stdout: &str) -> Vec<P::Item> {
202 let combined = format!("{}\n{}", stderr, stdout);
203 let mut parser = P::default();
204 for line in combined.lines() {
205 parser.process_line(line);
206 }
207 parser.finish()
208}
209
210fn parse_location_line(line: &str) -> (Option<String>, Option<u32>, Option<u32>) {
212 let trimmed = line.trim_start();
213 if !trimmed.starts_with("--> ") {
214 return (None, None, None);
215 }
216 let location = trimmed.trim_start_matches("--> ");
217 if location.is_empty() {
218 return (None, None, None);
219 }
220 let parts: Vec<&str> = location.rsplitn(3, ':').collect();
221 if parts.len() >= 3 {
222 (
223 Some(parts[2].to_string()),
224 parts[1].parse().ok(),
225 parts[0].parse().ok(),
226 )
227 } else if parts.len() == 2 {
228 (Some(parts[1].to_string()), parts[0].parse().ok(), None)
229 } else {
230 (None, None, None)
231 }
232}
233
234#[derive(Default)]
239struct BuildErrorParser {
240 errors: Vec<BuildError>,
241}
242
243impl DiagnosticParser for BuildErrorParser {
244 type Item = BuildError;
245
246 fn process_line(&mut self, line: &str) {
247 if line.starts_with("error") {
248 self.errors.push(BuildError {
249 file: None,
250 line: None,
251 column: None,
252 message: line.to_string(),
253 level: BuildErrorLevel::Error,
254 });
255 } else if line.starts_with("warning") {
256 self.errors.push(BuildError {
257 file: None,
258 line: None,
259 column: None,
260 message: line.to_string(),
261 level: BuildErrorLevel::Warning,
262 });
263 } else if line.trim_start().starts_with("--> ") {
264 if let Some(last_error) = self.errors.last_mut() {
265 let (file, line_num, col) = parse_location_line(line);
266 last_error.file = file;
267 last_error.line = line_num;
268 last_error.column = col;
269 }
270 }
271 }
272
273 fn finish(self) -> Vec<BuildError> {
274 self.errors
275 }
276}
277
278fn parse_build_errors_from_text(stderr: &str, stdout: &str) -> Vec<BuildError> {
279 parse_diagnostic_output::<BuildErrorParser>(stderr, stdout)
280}
281
282#[derive(Default)]
287struct TestFailureParser {
288 failures: Vec<TestFailure>,
289 in_failure_block: bool,
290 current_test: Option<String>,
291 current_message: String,
292}
293
294impl TestFailureParser {
295 fn flush_current(&mut self) {
296 if let Some(test_name) = self.current_test.take() {
297 self.failures.push(TestFailure {
298 test_name,
299 file: None,
300 line: None,
301 message: self.current_message.trim().to_string(),
302 stdout: None,
303 });
304 }
305 self.current_message.clear();
306 }
307}
308
309impl DiagnosticParser for TestFailureParser {
310 type Item = TestFailure;
311
312 fn process_line(&mut self, line: &str) {
313 if line.starts_with("---- ") && line.ends_with(" stdout ----") {
314 self.flush_current();
315 let name = line
316 .trim_start_matches("---- ")
317 .trim_end_matches(" stdout ----");
318 self.current_test = Some(name.to_string());
319 self.in_failure_block = true;
320 } else if self.in_failure_block {
321 if line.starts_with("---- ") || line.starts_with("failures:") {
322 self.flush_current();
323 self.in_failure_block = false;
324 } else {
325 self.current_message.push_str(line);
326 self.current_message.push('\n');
327 }
328 } else if line.contains("... FAILED") && line.starts_with("test ") {
329 let test_name = line
330 .trim_start_matches("test ")
331 .split(" ...")
332 .next()
333 .unwrap_or("unknown")
334 .to_string();
335 if !self.failures.iter().any(|f| f.test_name == test_name) {
336 self.failures.push(TestFailure {
337 test_name,
338 file: None,
339 line: None,
340 message: String::new(),
341 stdout: None,
342 });
343 }
344 }
345 }
346
347 fn finish(mut self) -> Vec<TestFailure> {
348 self.flush_current();
349 self.failures
350 }
351}
352
353fn parse_test_failures_from_text(stderr: &str, stdout: &str) -> Vec<TestFailure> {
354 parse_diagnostic_output::<TestFailureParser>(stderr, stdout)
355}
356
357#[cfg(test)]
358mod tests {
359 use super::*;
360
361 #[test]
362 fn strict_phase_requires_json() {
363 let outcome = validate_phase_output("qa", Uuid::new_v4(), "agent", 0, "plain-text", "")
364 .expect("validation should return outcome");
365 assert_eq!(outcome.status, "failed");
366 assert!(outcome.error.is_some());
367 }
368
369 #[test]
370 fn strict_phase_suffix_match_requires_json() {
371 let outcome = validate_phase_output("run_qa", Uuid::new_v4(), "agent", 0, "plain-text", "")
373 .expect("validation should return outcome");
374 assert_eq!(
375 outcome.status, "failed",
376 "step ID 'run_qa' should be strict via suffix match"
377 );
378 }
379
380 #[test]
381 fn strict_phase_accepts_json() {
382 let stdout = r#"{"confidence":0.7,"quality_score":0.8,"artifacts":[{"kind":"ticket","severity":"high","category":"bug"}]}"#;
383 let outcome = validate_phase_output("qa", Uuid::new_v4(), "agent", 0, stdout, "")
384 .expect("validation should return outcome");
385 assert_eq!(outcome.status, "passed");
386 assert_eq!(outcome.output.artifacts.len(), 1);
387 }
388
389 #[test]
390 fn build_phase_parses_errors() {
391 let stderr = r#"error[E0308]: mismatched types
392 --> src/main.rs:10:5
393warning: unused variable
394 --> src/lib.rs:3:9"#;
395 let outcome = validate_phase_output("build", Uuid::new_v4(), "agent", 1, "", stderr)
396 .expect("validation should return outcome");
397 assert_eq!(outcome.output.build_errors.len(), 2);
398 assert_eq!(outcome.output.build_errors[0].level, BuildErrorLevel::Error);
399 assert_eq!(
400 outcome.output.build_errors[0].file.as_deref(),
401 Some("src/main.rs")
402 );
403 assert_eq!(outcome.output.build_errors[0].line, Some(10));
404 }
405
406 #[test]
407 fn test_phase_parses_failures() {
408 let stdout = "test my_module::test_foo ... FAILED\ntest my_module::test_bar ... ok\n\n---- my_module::test_foo stdout ----\nthread 'my_module::test_foo' panicked at 'assertion failed'\n\nfailures:\n my_module::test_foo\n";
409 let outcome = validate_phase_output("test", Uuid::new_v4(), "agent", 1, stdout, "")
410 .expect("validation should return outcome");
411 assert!(!outcome.output.test_failures.is_empty());
413 assert!(
414 outcome
415 .output
416 .test_failures
417 .iter()
418 .any(|f| f.test_name == "my_module::test_foo")
419 );
420 }
421
422 #[test]
423 fn non_build_phase_has_no_build_errors() {
424 let outcome = validate_phase_output("implement", Uuid::new_v4(), "agent", 0, "done", "")
425 .expect("validation should return outcome");
426 assert!(outcome.output.build_errors.is_empty());
427 assert!(outcome.output.test_failures.is_empty());
428 }
429
430 #[test]
431 fn fatal_provider_error_marks_run_failed_even_with_zero_exit_code() {
432 let stderr = "Error: All 1 account(s) rate-limited for claude. Quota resets in 116h 44m.";
433 let outcome = validate_phase_output("implement", Uuid::new_v4(), "agent", 0, "", stderr)
434 .expect("validation should return outcome");
435 assert_eq!(outcome.status, "failed");
436 assert_eq!(
437 outcome.error.as_deref(),
438 Some("provider rate limit exceeded")
439 );
440 }
441
442 #[test]
443 fn fatal_provider_auth_error_marks_run_failed() {
444 let stderr = "authentication failed: invalid API key";
445 let outcome = validate_phase_output("align_tests", Uuid::new_v4(), "agent", 0, "", stderr)
446 .expect("validation should return outcome");
447 assert_eq!(outcome.status, "failed");
448 assert_eq!(
449 outcome.error.as_deref(),
450 Some("provider authentication failed")
451 );
452 }
453
454 #[test]
455 fn build_phase_parses_warnings() {
456 let stderr = "warning: unused variable `x`\n --> src/lib.rs:5:13";
457 let outcome = validate_phase_output("build", Uuid::new_v4(), "agent", 0, "", stderr)
458 .expect("validation should return outcome");
459 assert_eq!(outcome.output.build_errors.len(), 1);
460 assert_eq!(
461 outcome.output.build_errors[0].level,
462 BuildErrorLevel::Warning
463 );
464 assert_eq!(
465 outcome.output.build_errors[0].file.as_deref(),
466 Some("src/lib.rs")
467 );
468 assert_eq!(outcome.output.build_errors[0].line, Some(5));
469 }
470
471 #[test]
472 fn sdlc_phases_accept_stream_json_output() {
473 let sdlc_phases = [
476 "qa_testing",
477 "qa_doc_gen",
478 "ticket_fix",
479 "align_tests",
480 "doc_governance",
481 ];
482 let stream_json = concat!(
483 r#"{"type":"system","subtype":"init"}"#,
484 "\n",
485 r#"{"type":"result","result":"done"}"#,
486 "\n",
487 );
488 for phase in sdlc_phases {
489 let outcome = validate_phase_output(phase, Uuid::new_v4(), "agent", 0, stream_json, "")
490 .expect("validation should return outcome");
491 assert_eq!(
492 outcome.status, "passed",
493 "phase {} should accept stream-json",
494 phase
495 );
496 }
497 }
498
499 #[test]
500 fn sdlc_phases_accept_plain_text_output() {
501 let sdlc_phases = [
503 "qa_testing",
504 "qa_doc_gen",
505 "ticket_fix",
506 "align_tests",
507 "doc_governance",
508 ];
509 for phase in sdlc_phases {
510 let outcome =
511 validate_phase_output(phase, Uuid::new_v4(), "agent", 0, "plain text output", "")
512 .expect("validation should return outcome");
513 assert_eq!(
514 outcome.status, "passed",
515 "phase {} should accept plain text",
516 phase
517 );
518 }
519 }
520
521 #[test]
522 fn stream_json_with_embedded_error_patterns_no_false_positive() {
523 let stream_json_stdout = concat!(
528 r#"{"type":"system","subtype":"init","model":"test"}"#,
529 "\n",
530 r#"{"type":"tool_result","content":"(\"authentication failed\", \"provider authentication failed\")"}"#,
531 "\n",
532 r#"{"type":"tool_result","content":"(\"rate-limited\", \"provider rate limit exceeded\")"}"#,
533 "\n",
534 r#"{"type":"result","result":"done"}"#,
535 "\n",
536 );
537 let outcome = validate_phase_output(
538 "implement",
539 Uuid::new_v4(),
540 "agent",
541 0,
542 stream_json_stdout,
543 "",
544 )
545 .expect("validation should return outcome");
546 assert_eq!(outcome.status, "passed");
547 assert!(outcome.error.is_none());
548 }
549
550 #[test]
551 fn plain_text_stdout_with_error_pattern_still_detected() {
552 let stdout = "Error: authentication failed for provider";
554 let outcome = validate_phase_output("implement", Uuid::new_v4(), "agent", 0, stdout, "")
555 .expect("validation should return outcome");
556 assert_eq!(outcome.status, "failed");
557 assert_eq!(
558 outcome.error.as_deref(),
559 Some("provider authentication failed")
560 );
561 }
562
563 #[test]
564 fn diagnostic_parser_trait_build_errors_direct() {
565 let errors = parse_diagnostic_output::<BuildErrorParser>(
566 "error[E0308]: mismatch\n --> src/main.rs:10:5",
567 "",
568 );
569 assert_eq!(errors.len(), 1);
570 assert_eq!(errors[0].file.as_deref(), Some("src/main.rs"));
571 assert_eq!(errors[0].line, Some(10));
572 }
573
574 #[test]
575 fn diagnostic_parser_trait_test_failures_direct() {
576 let failures = parse_diagnostic_output::<TestFailureParser>(
577 "",
578 "---- foo stdout ----\npanicked\nfailures:\n",
579 );
580 assert_eq!(failures.len(), 1);
581 assert_eq!(failures[0].test_name, "foo");
582 assert_eq!(failures[0].message, "panicked");
583 }
584
585 #[test]
586 fn diagnostic_parser_combine_order_consistent() {
587 let errors = parse_build_errors_from_text("error: in stderr", "");
590 assert_eq!(errors.len(), 1);
591 let failures = parse_test_failures_from_text("", "test bar ... FAILED");
593 assert_eq!(failures.len(), 1);
594 assert_eq!(failures[0].test_name, "bar");
595 }
596
597 #[test]
598 fn parse_location_line_full() {
599 let (file, line, col) = parse_location_line(" --> src/main.rs:10:5");
600 assert_eq!(file.as_deref(), Some("src/main.rs"));
601 assert_eq!(line, Some(10));
602 assert_eq!(col, Some(5));
603 }
604
605 #[test]
606 fn parse_location_line_no_column() {
607 let (file, line, col) = parse_location_line(" --> src/lib.rs:3");
608 assert_eq!(file.as_deref(), Some("src/lib.rs"));
609 assert_eq!(line, Some(3));
610 assert_eq!(col, None);
611 }
612
613 #[test]
614 fn parse_location_line_not_a_location() {
615 let (file, line, col) = parse_location_line("not a location line");
616 assert!(file.is_none());
617 assert!(line.is_none());
618 assert!(col.is_none());
619 }
620
621 #[test]
622 fn parse_location_line_empty_after_arrow() {
623 let (file, line, col) = parse_location_line(" --> ");
624 assert!(file.is_none());
625 assert!(line.is_none());
626 assert!(col.is_none());
627 }
628
629 #[test]
630 fn test_failure_parser_combine_order() {
631 let stderr = "Compiling my_crate v0.1.0\nFinished test target";
635 let stdout = "\
636---- foo::bar stdout ----\n\
637thread 'foo::bar' panicked at 'assert_eq failed'\n\
638\n\
639failures:\n\
640 foo::bar\n";
641 let failures = parse_test_failures_from_text(stderr, stdout);
642 assert_eq!(failures.len(), 1, "should find exactly one failure");
643 assert_eq!(failures[0].test_name, "foo::bar");
644 assert!(
645 failures[0].message.contains("panicked"),
646 "message should contain panic text"
647 );
648 }
649
650 #[test]
651 fn build_errors_multiple_interleaved() {
652 let stderr = "\
654error[E0308]: mismatched types\n\
655 --> src/main.rs:10:5\n\
656warning: unused variable `x`\n\
657 --> src/lib.rs:3:9\n\
658error[E0433]: unresolved import\n\
659 --> src/util.rs:1:5";
660 let errors = parse_build_errors_from_text(stderr, "");
661 assert_eq!(errors.len(), 3);
662 assert_eq!(errors[0].level, BuildErrorLevel::Error);
664 assert_eq!(errors[0].file.as_deref(), Some("src/main.rs"));
665 assert_eq!(errors[0].line, Some(10));
666 assert_eq!(errors[0].column, Some(5));
667 assert_eq!(errors[1].level, BuildErrorLevel::Warning);
669 assert_eq!(errors[1].file.as_deref(), Some("src/lib.rs"));
670 assert_eq!(errors[1].line, Some(3));
671 assert_eq!(errors[1].column, Some(9));
672 assert_eq!(errors[2].level, BuildErrorLevel::Error);
674 assert_eq!(errors[2].file.as_deref(), Some("src/util.rs"));
675 assert_eq!(errors[2].line, Some(1));
676 assert_eq!(errors[2].column, Some(5));
677 }
678
679 #[test]
680 fn test_failure_parser_last_block_no_delimiter() {
681 let stdout = "\
684---- my_mod::test_alpha stdout ----\n\
685thread 'my_mod::test_alpha' panicked at 'value was None'\n\
686note: run with `RUST_BACKTRACE=1`";
687 let failures = parse_test_failures_from_text("", stdout);
688 assert_eq!(failures.len(), 1);
689 assert_eq!(failures[0].test_name, "my_mod::test_alpha");
690 assert!(
691 failures[0].message.contains("panicked"),
692 "should capture the panic message"
693 );
694 }
695
696 #[test]
697 fn parse_location_line_windows_path() {
698 let (file, line, col) = parse_location_line(r" --> C:\src\main.rs:10:5");
702 assert_eq!(file.as_deref(), Some(r"C:\src\main.rs"));
703 assert_eq!(line, Some(10));
704 assert_eq!(col, Some(5));
705 }
706}