apr_qa_runner/
diagnostics.rs

1//! Fail-Fast Diagnostic Report Generation (FF-REPORT-001)
2//!
3//! Generates comprehensive diagnostic reports on test failure using apr's rich tooling.
4//! Reports are designed for immediate GitHub issue creation with full reproduction context.
5
6use crate::evidence::Evidence;
7use serde::{Deserialize, Serialize};
8use std::fmt::Write as _;
9use std::path::{Path, PathBuf};
10use std::process::Command;
11use std::time::{Duration, Instant};
12
13/// Timeout for each diagnostic command
14const CHECK_TIMEOUT: Duration = Duration::from_secs(30);
15const INSPECT_TIMEOUT: Duration = Duration::from_secs(10);
16const TRACE_TIMEOUT: Duration = Duration::from_secs(60);
17const TENSORS_TIMEOUT: Duration = Duration::from_secs(10);
18const EXPLAIN_TIMEOUT: Duration = Duration::from_secs(5);
19
20/// Result of a diagnostic command
21#[derive(Debug, Clone, Serialize, Deserialize)]
22pub struct DiagnosticResult {
23    /// Command that was run
24    pub command: String,
25    /// Whether the command succeeded
26    pub success: bool,
27    /// Stdout output
28    pub stdout: String,
29    /// Stderr output
30    pub stderr: String,
31    /// Duration in milliseconds
32    pub duration_ms: u64,
33    /// Whether the command timed out
34    pub timed_out: bool,
35}
36
37/// Environment context for the report
38#[derive(Debug, Clone, Serialize, Deserialize)]
39pub struct EnvironmentContext {
40    /// Operating system (e.g., "linux", "macos", "windows")
41    pub os: String,
42    /// CPU architecture (e.g., "x86_64", "aarch64")
43    pub arch: String,
44    /// apr-qa version
45    pub apr_qa_version: String,
46    /// apr CLI version
47    pub apr_cli_version: String,
48    /// Git commit hash (short form)
49    pub git_commit: String,
50    /// Git branch name
51    pub git_branch: String,
52    /// Whether working directory has uncommitted changes
53    pub git_dirty: bool,
54    /// Rust compiler version
55    pub rustc_version: String,
56}
57
58impl EnvironmentContext {
59    /// Collect environment context
60    #[must_use]
61    pub fn collect() -> Self {
62        Self {
63            os: std::env::consts::OS.to_string(),
64            arch: std::env::consts::ARCH.to_string(),
65            apr_qa_version: env!("CARGO_PKG_VERSION").to_string(),
66            apr_cli_version: get_apr_version(),
67            git_commit: get_git_commit(),
68            git_branch: get_git_branch(),
69            git_dirty: get_git_dirty(),
70            rustc_version: get_rustc_version(),
71        }
72    }
73}
74
75/// Failure details from the evidence
76#[derive(Debug, Clone, Serialize, Deserialize)]
77pub struct FailureDetails {
78    /// Gate ID that failed (e.g., "G3-STABLE")
79    pub gate_id: String,
80    /// Model identifier (HuggingFace repo path)
81    pub model: String,
82    /// Model format (e.g., "Apr", "SafeTensors", "Gguf")
83    pub format: String,
84    /// Backend used (e.g., "Cpu", "Metal", "Cuda")
85    pub backend: String,
86    /// Test outcome (e.g., "Crashed", "Falsified", "Timeout")
87    pub outcome: String,
88    /// Human-readable failure reason
89    pub reason: String,
90    /// Process exit code if available
91    pub exit_code: Option<i32>,
92    /// Test duration in milliseconds
93    pub duration_ms: u64,
94    /// Standard error output if captured
95    pub stderr: Option<String>,
96}
97
98impl From<&Evidence> for FailureDetails {
99    fn from(evidence: &Evidence) -> Self {
100        Self {
101            gate_id: evidence.gate_id.clone(),
102            model: evidence.scenario.model.hf_repo(),
103            format: format!("{:?}", evidence.scenario.format),
104            backend: format!("{:?}", evidence.scenario.backend),
105            outcome: format!("{:?}", evidence.outcome),
106            reason: evidence.reason.clone(),
107            exit_code: evidence.exit_code,
108            duration_ms: evidence.metrics.duration_ms,
109            stderr: evidence.stderr.clone(),
110        }
111    }
112}
113
114/// Complete fail-fast diagnostic report
115#[derive(Debug, Clone, Serialize, Deserialize)]
116pub struct FailFastReport {
117    /// Report version
118    pub version: String,
119    /// Timestamp
120    pub timestamp: String,
121    /// Failure details
122    pub failure: FailureDetails,
123    /// Environment context
124    pub environment: EnvironmentContext,
125    /// Diagnostic results
126    pub diagnostics: DiagnosticsBundle,
127    /// Reproduction information
128    pub reproduction: ReproductionInfo,
129}
130
131/// Bundle of all diagnostic results
132#[derive(Debug, Clone, Serialize, Deserialize)]
133pub struct DiagnosticsBundle {
134    /// Results from `apr check` - pipeline integrity
135    pub check: Option<DiagnosticResult>,
136    /// Results from `apr inspect` - model metadata
137    pub inspect: Option<DiagnosticResult>,
138    /// Results from `apr trace` - layer-by-layer analysis
139    pub trace: Option<DiagnosticResult>,
140    /// Results from `apr tensors` - tensor names and shapes
141    pub tensors: Option<DiagnosticResult>,
142    /// Results from `apr explain` - error code explanation
143    pub explain: Option<DiagnosticResult>,
144}
145
146/// Information for reproducing the failure
147#[derive(Debug, Clone, Serialize, Deserialize)]
148pub struct ReproductionInfo {
149    /// Command to reproduce the failure
150    pub command: String,
151    /// Path to the model file used
152    pub model_path: String,
153    /// Path to the playbook file if applicable
154    pub playbook: Option<String>,
155}
156
157/// Fail-fast diagnostic reporter
158pub struct FailFastReporter {
159    output_dir: PathBuf,
160    binary: String,
161}
162
163impl FailFastReporter {
164    /// Create a new reporter
165    #[must_use]
166    pub fn new(output_dir: &Path) -> Self {
167        Self {
168            output_dir: output_dir.to_path_buf(),
169            binary: "apr".to_string(),
170        }
171    }
172
173    /// Create with custom binary path
174    #[must_use]
175    pub fn with_binary(mut self, binary: &str) -> Self {
176        self.binary = binary.to_string();
177        self
178    }
179
180    /// Generate full diagnostic report on failure
181    ///
182    /// # Errors
183    ///
184    /// Returns an error if report generation fails.
185    pub fn generate_report(
186        &self,
187        evidence: &Evidence,
188        model_path: &Path,
189        playbook: Option<&str>,
190    ) -> std::io::Result<FailFastReport> {
191        let report_dir = self.output_dir.join("fail-fast-report");
192        std::fs::create_dir_all(&report_dir)?;
193
194        eprintln!("[FAIL-FAST] Generating diagnostic report...");
195
196        // Collect diagnostics
197        let check = self.run_check(model_path);
198        let inspect = self.run_inspect(model_path);
199        let trace = self.run_trace(model_path);
200        let tensors = self.run_tensors(model_path);
201        let explain = self.run_explain(&evidence.gate_id);
202
203        // Save individual diagnostic files first (before moving into report)
204        if let Some(ref c) = check {
205            self.save_json(&report_dir.join("check.json"), c)?;
206        }
207        if let Some(ref i) = inspect {
208            self.save_json(&report_dir.join("inspect.json"), i)?;
209        }
210        if let Some(ref t) = trace {
211            self.save_json(&report_dir.join("trace.json"), t)?;
212        }
213        if let Some(ref t) = tensors {
214            self.save_json(&report_dir.join("tensors.json"), t)?;
215        }
216
217        // Build report (moves diagnostic values)
218        let report = FailFastReport {
219            version: "1.0.0".to_string(),
220            timestamp: chrono::Utc::now().to_rfc3339(),
221            failure: FailureDetails::from(evidence),
222            environment: EnvironmentContext::collect(),
223            diagnostics: DiagnosticsBundle {
224                check,
225                inspect,
226                trace,
227                tensors,
228                explain,
229            },
230            reproduction: ReproductionInfo {
231                command: format!(
232                    "apr-qa run {} --fail-fast",
233                    playbook.unwrap_or("playbook.yaml")
234                ),
235                model_path: model_path.to_string_lossy().to_string(),
236                playbook: playbook.map(String::from),
237            },
238        };
239
240        // Save full diagnostics JSON
241        self.save_json(&report_dir.join("diagnostics.json"), &report)?;
242
243        // Save environment
244        self.save_json(&report_dir.join("environment.json"), &report.environment)?;
245
246        // Save stderr log
247        if let Some(ref stderr) = evidence.stderr {
248            std::fs::write(report_dir.join("stderr.log"), stderr)?;
249        }
250
251        // Generate markdown summary
252        let summary = self.generate_markdown(&report);
253        std::fs::write(report_dir.join("summary.md"), &summary)?;
254
255        eprintln!("[FAIL-FAST] Report saved to: {}", report_dir.display());
256        eprintln!("[FAIL-FAST] Summary: {}/summary.md", report_dir.display());
257        eprintln!("[FAIL-FAST] GitHub issue body ready for paste");
258
259        Ok(report)
260    }
261
262    /// Run apr check and capture output
263    fn run_check(&self, model_path: &Path) -> Option<DiagnosticResult> {
264        eprint!("[FAIL-FAST] Running apr check... ");
265        let result = self.run_command_with_timeout(
266            &[
267                &self.binary,
268                "check",
269                &model_path.to_string_lossy(),
270                "--json",
271            ],
272            CHECK_TIMEOUT,
273        );
274        eprintln!(
275            "done ({:.1}s){}",
276            result.duration_ms as f64 / 1000.0,
277            if result.timed_out { " [TIMEOUT]" } else { "" }
278        );
279        Some(result)
280    }
281
282    /// Run apr inspect and capture output
283    fn run_inspect(&self, model_path: &Path) -> Option<DiagnosticResult> {
284        eprint!("[FAIL-FAST] Running apr inspect... ");
285        let result = self.run_command_with_timeout(
286            &[
287                &self.binary,
288                "inspect",
289                &model_path.to_string_lossy(),
290                "--json",
291            ],
292            INSPECT_TIMEOUT,
293        );
294        eprintln!(
295            "done ({:.1}s){}",
296            result.duration_ms as f64 / 1000.0,
297            if result.timed_out { " [TIMEOUT]" } else { "" }
298        );
299        Some(result)
300    }
301
302    /// Run apr trace and capture output
303    fn run_trace(&self, model_path: &Path) -> Option<DiagnosticResult> {
304        // Only run trace for .apr files
305        if model_path.extension().is_none_or(|e| e != "apr") {
306            return None;
307        }
308
309        eprint!("[FAIL-FAST] Running apr trace... ");
310        let result = self.run_command_with_timeout(
311            &[
312                &self.binary,
313                "trace",
314                &model_path.to_string_lossy(),
315                "--payload",
316                "--json",
317            ],
318            TRACE_TIMEOUT,
319        );
320        eprintln!(
321            "done ({:.1}s){}",
322            result.duration_ms as f64 / 1000.0,
323            if result.timed_out { " [TIMEOUT]" } else { "" }
324        );
325        Some(result)
326    }
327
328    /// Run apr tensors and capture output
329    fn run_tensors(&self, model_path: &Path) -> Option<DiagnosticResult> {
330        eprint!("[FAIL-FAST] Running apr tensors... ");
331        let result = self.run_command_with_timeout(
332            &[
333                &self.binary,
334                "tensors",
335                &model_path.to_string_lossy(),
336                "--json",
337            ],
338            TENSORS_TIMEOUT,
339        );
340        eprintln!(
341            "done ({:.1}s){}",
342            result.duration_ms as f64 / 1000.0,
343            if result.timed_out { " [TIMEOUT]" } else { "" }
344        );
345        Some(result)
346    }
347
348    /// Run apr explain for the error code
349    fn run_explain(&self, error_code: &str) -> Option<DiagnosticResult> {
350        // Extract error code pattern (e.g., "G3-STABLE" -> try explaining common errors)
351        eprint!("[FAIL-FAST] Running apr explain... ");
352        let result =
353            self.run_command_with_timeout(&[&self.binary, "explain", error_code], EXPLAIN_TIMEOUT);
354        eprintln!(
355            "done ({:.1}s){}",
356            result.duration_ms as f64 / 1000.0,
357            if result.timed_out { " [TIMEOUT]" } else { "" }
358        );
359        Some(result)
360    }
361
362    /// Run a command with timeout
363    fn run_command_with_timeout(&self, args: &[&str], timeout: Duration) -> DiagnosticResult {
364        let start = Instant::now();
365        let command_str = args.join(" ");
366
367        let output = Command::new(args[0]).args(&args[1..]).output();
368
369        let duration = start.elapsed();
370        let timed_out = duration > timeout;
371
372        match output {
373            Ok(out) => DiagnosticResult {
374                command: command_str,
375                success: out.status.success(),
376                stdout: String::from_utf8_lossy(&out.stdout).to_string(),
377                stderr: String::from_utf8_lossy(&out.stderr).to_string(),
378                duration_ms: duration.as_millis() as u64,
379                timed_out,
380            },
381            Err(e) => DiagnosticResult {
382                command: command_str,
383                success: false,
384                stdout: String::new(),
385                stderr: format!("Failed to execute: {e}"),
386                duration_ms: duration.as_millis() as u64,
387                timed_out,
388            },
389        }
390    }
391
392    /// Save JSON to file
393    fn save_json<T: Serialize>(&self, path: &Path, data: &T) -> std::io::Result<()> {
394        let json = serde_json::to_string_pretty(data).map_err(std::io::Error::other)?;
395        std::fs::write(path, json)
396    }
397
398    /// Generate markdown summary
399    #[must_use]
400    #[allow(clippy::too_many_lines)]
401    pub fn generate_markdown(&self, report: &FailFastReport) -> String {
402        let mut md = String::new();
403
404        // Header
405        let _ = writeln!(md, "# Fail-Fast Report: {}\n", report.failure.gate_id);
406
407        // Failure Summary Table
408        md.push_str("## Failure Summary\n\n");
409        md.push_str("| Field | Value |\n");
410        md.push_str("|-------|-------|\n");
411        let _ = writeln!(md, "| Gate | `{}` |", report.failure.gate_id);
412        let _ = writeln!(md, "| Model | `{}` |", report.failure.model);
413        let _ = writeln!(md, "| Format | {} |", report.failure.format);
414        let _ = writeln!(md, "| Backend | {} |", report.failure.backend);
415        let _ = writeln!(md, "| Outcome | {} |", report.failure.outcome);
416        if let Some(code) = report.failure.exit_code {
417            let _ = writeln!(md, "| Exit Code | {code} |");
418        }
419        let _ = writeln!(md, "| Duration | {}ms |", report.failure.duration_ms);
420        md.push('\n');
421
422        // Reason
423        md.push_str("### Reason\n\n");
424        let _ = writeln!(md, "{}\n", report.failure.reason);
425
426        // Environment Table
427        md.push_str("## Environment\n\n");
428        md.push_str("| Field | Value |\n");
429        md.push_str("|-------|-------|\n");
430        let _ = writeln!(
431            md,
432            "| OS | {} {} |",
433            report.environment.os, report.environment.arch
434        );
435        let _ = writeln!(md, "| apr-qa | {} |", report.environment.apr_qa_version);
436        let _ = writeln!(md, "| apr-cli | {} |", report.environment.apr_cli_version);
437        let _ = writeln!(
438            md,
439            "| Git | {} ({}){}|",
440            report.environment.git_commit,
441            report.environment.git_branch,
442            if report.environment.git_dirty {
443                " [dirty]"
444            } else {
445                ""
446            }
447        );
448        let _ = writeln!(md, "| Rust | {} |", report.environment.rustc_version);
449        md.push('\n');
450
451        // Pipeline Check Results
452        if let Some(ref check) = report.diagnostics.check {
453            md.push_str("## Pipeline Check Results\n\n");
454            if check.success {
455                md.push_str("All pipeline checks passed.\n\n");
456            } else {
457                md.push_str("**Pipeline check failed:**\n\n");
458                md.push_str("```\n");
459                md.push_str(&check.stderr);
460                md.push_str("\n```\n\n");
461            }
462        }
463
464        // Model Metadata
465        if let Some(ref inspect) = report.diagnostics.inspect {
466            md.push_str("## Model Metadata\n\n");
467            md.push_str("<details>\n<summary>apr inspect output</summary>\n\n");
468            md.push_str("```json\n");
469            md.push_str(&inspect.stdout);
470            md.push_str("\n```\n\n");
471            md.push_str("</details>\n\n");
472        }
473
474        // Tensor Info
475        if let Some(ref tensors) = report.diagnostics.tensors {
476            md.push_str("## Tensor Inventory\n\n");
477            md.push_str("<details>\n<summary>apr tensors output</summary>\n\n");
478            md.push_str("```json\n");
479            md.push_str(&tensors.stdout);
480            md.push_str("\n```\n\n");
481            md.push_str("</details>\n\n");
482        }
483
484        // Trace (if available)
485        if let Some(ref trace) = report.diagnostics.trace {
486            md.push_str("## Layer Trace\n\n");
487            md.push_str("<details>\n<summary>apr trace output</summary>\n\n");
488            md.push_str("```json\n");
489            md.push_str(&trace.stdout);
490            md.push_str("\n```\n\n");
491            md.push_str("</details>\n\n");
492        }
493
494        // Error Explanation
495        if let Some(ref explain) = report.diagnostics.explain {
496            if !explain.stdout.is_empty() {
497                md.push_str("## Error Analysis\n\n");
498                md.push_str(&explain.stdout);
499                md.push_str("\n\n");
500            }
501        }
502
503        // Stderr Capture
504        if let Some(ref stderr) = report.failure.stderr {
505            if !stderr.is_empty() {
506                md.push_str("## Stderr Capture\n\n");
507                md.push_str("<details>\n<summary>Full stderr output</summary>\n\n");
508                md.push_str("```\n");
509                md.push_str(stderr);
510                md.push_str("\n```\n\n");
511                md.push_str("</details>\n\n");
512            }
513        }
514
515        // Reproduction
516        md.push_str("## Reproduction\n\n");
517        md.push_str("```bash\n");
518        md.push_str("# Reproduce this failure\n");
519        let _ = writeln!(md, "{}\n", report.reproduction.command);
520        md.push_str("# Run diagnostics manually\n");
521        let _ = writeln!(md, "apr check {}", report.reproduction.model_path);
522        let _ = writeln!(
523            md,
524            "apr trace {} --payload -v",
525            report.reproduction.model_path
526        );
527        let _ = writeln!(md, "apr explain {}", report.failure.gate_id);
528        md.push_str("```\n");
529
530        md
531    }
532}
533
534// Helper functions for environment collection
535
536fn get_apr_version() -> String {
537    Command::new("apr")
538        .arg("--version")
539        .output()
540        .ok()
541        .and_then(|o| {
542            String::from_utf8_lossy(&o.stdout)
543                .lines()
544                .next()
545                .map(|s| s.replace("apr ", "").trim().to_string())
546        })
547        .unwrap_or_else(|| "unknown".to_string())
548}
549
550fn get_git_commit() -> String {
551    Command::new("git")
552        .args(["rev-parse", "--short", "HEAD"])
553        .output()
554        .ok()
555        .map_or_else(
556            || "unknown".to_string(),
557            |o| String::from_utf8_lossy(&o.stdout).trim().to_string(),
558        )
559}
560
561fn get_git_branch() -> String {
562    Command::new("git")
563        .args(["rev-parse", "--abbrev-ref", "HEAD"])
564        .output()
565        .ok()
566        .map_or_else(
567            || "unknown".to_string(),
568            |o| String::from_utf8_lossy(&o.stdout).trim().to_string(),
569        )
570}
571
572fn get_git_dirty() -> bool {
573    Command::new("git")
574        .args(["status", "--porcelain"])
575        .output()
576        .ok()
577        .is_some_and(|o| !o.stdout.is_empty())
578}
579
580fn get_rustc_version() -> String {
581    Command::new("rustc")
582        .arg("--version")
583        .output()
584        .ok()
585        .map_or_else(
586            || "unknown".to_string(),
587            |o| {
588                String::from_utf8_lossy(&o.stdout)
589                    .trim()
590                    .replace("rustc ", "")
591            },
592        )
593}
594
595#[cfg(test)]
596mod tests {
597    use super::*;
598    use crate::evidence::{HostInfo, Outcome, PerformanceMetrics};
599    use apr_qa_gen::{Backend, Format, Modality, ModelId, QaScenario};
600    use chrono::Utc;
601    use std::collections::HashMap;
602
603    fn test_evidence() -> Evidence {
604        Evidence {
605            id: "test-evidence-001".to_string(),
606            gate_id: "G3-STABLE".to_string(),
607            scenario: QaScenario::new(
608                ModelId::new("Qwen", "Qwen2.5-Coder-0.5B-Instruct"),
609                Modality::Run,
610                Backend::Cpu,
611                Format::Apr,
612                "What is 2+2?".to_string(),
613                0,
614            ),
615            outcome: Outcome::Crashed,
616            reason: "Process crashed with exit code -1".to_string(),
617            output: String::new(),
618            stderr: Some("SIGSEGV at 0x12345".to_string()),
619            exit_code: Some(-1),
620            metrics: PerformanceMetrics {
621                duration_ms: 52740,
622                ..Default::default()
623            },
624            timestamp: Utc::now(),
625            host: HostInfo::default(),
626            metadata: HashMap::new(),
627        }
628    }
629
630    #[test]
631    fn test_failure_details_from_evidence() {
632        let evidence = test_evidence();
633        let details = FailureDetails::from(&evidence);
634
635        assert_eq!(details.gate_id, "G3-STABLE");
636        assert_eq!(details.model, "Qwen/Qwen2.5-Coder-0.5B-Instruct");
637        assert_eq!(details.format, "Apr");
638        assert_eq!(details.backend, "Cpu");
639        assert_eq!(details.exit_code, Some(-1));
640    }
641
642    #[test]
643    fn test_environment_context_collect() {
644        let ctx = EnvironmentContext::collect();
645
646        assert!(!ctx.os.is_empty());
647        assert!(!ctx.arch.is_empty());
648        assert!(!ctx.apr_qa_version.is_empty());
649    }
650
651    #[test]
652    fn test_diagnostic_result_serialization() {
653        let result = DiagnosticResult {
654            command: "apr check model.apr".to_string(),
655            success: true,
656            stdout: "{}".to_string(),
657            stderr: String::new(),
658            duration_ms: 1234,
659            timed_out: false,
660        };
661
662        let json = serde_json::to_string(&result).unwrap();
663        assert!(json.contains("apr check"));
664        assert!(json.contains("1234"));
665    }
666
667    #[test]
668    fn test_generate_markdown() {
669        let reporter = FailFastReporter::new(Path::new("output"));
670        let evidence = test_evidence();
671
672        let report = FailFastReport {
673            version: "1.0.0".to_string(),
674            timestamp: "2024-02-04T18:00:00Z".to_string(),
675            failure: FailureDetails::from(&evidence),
676            environment: EnvironmentContext {
677                os: "linux".to_string(),
678                arch: "x86_64".to_string(),
679                apr_qa_version: "0.1.0".to_string(),
680                apr_cli_version: "0.2.12".to_string(),
681                git_commit: "abc123".to_string(),
682                git_branch: "main".to_string(),
683                git_dirty: false,
684                rustc_version: "1.93.0".to_string(),
685            },
686            diagnostics: DiagnosticsBundle {
687                check: None,
688                inspect: None,
689                trace: None,
690                tensors: None,
691                explain: None,
692            },
693            reproduction: ReproductionInfo {
694                command: "apr-qa run playbook.yaml --fail-fast".to_string(),
695                model_path: "/path/to/model.apr".to_string(),
696                playbook: Some("playbook.yaml".to_string()),
697            },
698        };
699
700        let md = reporter.generate_markdown(&report);
701
702        assert!(md.contains("# Fail-Fast Report: G3-STABLE"));
703        assert!(md.contains("| Gate | `G3-STABLE` |"));
704        assert!(md.contains("| Model | `Qwen/Qwen2.5-Coder-0.5B-Instruct` |"));
705        assert!(md.contains("## Reproduction"));
706    }
707
708    #[test]
709    fn test_reporter_new() {
710        let reporter = FailFastReporter::new(Path::new("output"));
711        assert_eq!(reporter.output_dir, PathBuf::from("output"));
712        assert_eq!(reporter.binary, "apr");
713    }
714
715    #[test]
716    fn test_reporter_with_binary() {
717        let reporter = FailFastReporter::new(Path::new("output")).with_binary("/custom/apr");
718        assert_eq!(reporter.binary, "/custom/apr");
719    }
720
721    #[test]
722    fn test_generate_markdown_with_diagnostics() {
723        let reporter = FailFastReporter::new(Path::new("output"));
724        let evidence = test_evidence();
725
726        let check_result = DiagnosticResult {
727            command: "apr check /model.apr --json".to_string(),
728            success: false,
729            stdout: "{}".to_string(),
730            stderr: "Error: failed to load model".to_string(),
731            duration_ms: 500,
732            timed_out: false,
733        };
734
735        let inspect_result = DiagnosticResult {
736            command: "apr inspect /model.apr --json".to_string(),
737            success: true,
738            stdout: r#"{"architecture": "Qwen2"}"#.to_string(),
739            stderr: String::new(),
740            duration_ms: 200,
741            timed_out: false,
742        };
743
744        let tensors_result = DiagnosticResult {
745            command: "apr tensors /model.apr --json".to_string(),
746            success: true,
747            stdout: r#"{"count": 256}"#.to_string(),
748            stderr: String::new(),
749            duration_ms: 150,
750            timed_out: false,
751        };
752
753        let trace_result = DiagnosticResult {
754            command: "apr trace /model.apr --payload --json".to_string(),
755            success: true,
756            stdout: r#"{"layers": []}"#.to_string(),
757            stderr: String::new(),
758            duration_ms: 1000,
759            timed_out: false,
760        };
761
762        let explain_result = DiagnosticResult {
763            command: "apr explain G3-STABLE".to_string(),
764            success: true,
765            stdout: "G3-STABLE: Model stability gate - ensures no crashes".to_string(),
766            stderr: String::new(),
767            duration_ms: 50,
768            timed_out: false,
769        };
770
771        let report = FailFastReport {
772            version: "1.0.0".to_string(),
773            timestamp: "2024-02-04T18:00:00Z".to_string(),
774            failure: FailureDetails::from(&evidence),
775            environment: EnvironmentContext {
776                os: "linux".to_string(),
777                arch: "x86_64".to_string(),
778                apr_qa_version: "0.1.0".to_string(),
779                apr_cli_version: "0.2.12".to_string(),
780                git_commit: "abc123".to_string(),
781                git_branch: "main".to_string(),
782                git_dirty: true,
783                rustc_version: "1.93.0".to_string(),
784            },
785            diagnostics: DiagnosticsBundle {
786                check: Some(check_result),
787                inspect: Some(inspect_result),
788                trace: Some(trace_result),
789                tensors: Some(tensors_result),
790                explain: Some(explain_result),
791            },
792            reproduction: ReproductionInfo {
793                command: "apr-qa run playbook.yaml --fail-fast".to_string(),
794                model_path: "/path/to/model.apr".to_string(),
795                playbook: Some("playbook.yaml".to_string()),
796            },
797        };
798
799        let md = reporter.generate_markdown(&report);
800
801        // Check diagnostic sections are included
802        assert!(md.contains("## Pipeline Check Results"));
803        assert!(md.contains("**Pipeline check failed:**"));
804        assert!(md.contains("Error: failed to load model"));
805        assert!(md.contains("## Model Metadata"));
806        assert!(md.contains("apr inspect output"));
807        assert!(md.contains("## Tensor Inventory"));
808        assert!(md.contains("apr tensors output"));
809        assert!(md.contains("## Layer Trace"));
810        assert!(md.contains("apr trace output"));
811        assert!(md.contains("## Error Analysis"));
812        assert!(md.contains("G3-STABLE: Model stability gate"));
813        assert!(md.contains("[dirty]")); // git dirty flag
814        assert!(md.contains("## Stderr Capture"));
815        assert!(md.contains("SIGSEGV at 0x12345"));
816    }
817
818    #[test]
819    fn test_generate_markdown_successful_check() {
820        let reporter = FailFastReporter::new(Path::new("output"));
821        let evidence = test_evidence();
822
823        let check_result = DiagnosticResult {
824            command: "apr check /model.apr --json".to_string(),
825            success: true,
826            stdout: "{}".to_string(),
827            stderr: String::new(),
828            duration_ms: 500,
829            timed_out: false,
830        };
831
832        let report = FailFastReport {
833            version: "1.0.0".to_string(),
834            timestamp: "2024-02-04T18:00:00Z".to_string(),
835            failure: FailureDetails::from(&evidence),
836            environment: EnvironmentContext {
837                os: "linux".to_string(),
838                arch: "x86_64".to_string(),
839                apr_qa_version: "0.1.0".to_string(),
840                apr_cli_version: "0.2.12".to_string(),
841                git_commit: "abc123".to_string(),
842                git_branch: "main".to_string(),
843                git_dirty: false,
844                rustc_version: "1.93.0".to_string(),
845            },
846            diagnostics: DiagnosticsBundle {
847                check: Some(check_result),
848                inspect: None,
849                trace: None,
850                tensors: None,
851                explain: None,
852            },
853            reproduction: ReproductionInfo {
854                command: "apr-qa run playbook.yaml --fail-fast".to_string(),
855                model_path: "/path/to/model.apr".to_string(),
856                playbook: Some("playbook.yaml".to_string()),
857            },
858        };
859
860        let md = reporter.generate_markdown(&report);
861
862        assert!(md.contains("## Pipeline Check Results"));
863        assert!(md.contains("All pipeline checks passed."));
864    }
865
866    #[test]
867    fn test_run_trace_skips_non_apr() {
868        let reporter = FailFastReporter::new(Path::new("output"));
869        // run_trace should return None for non-.apr files
870        let result = reporter.run_trace(Path::new("/model.safetensors"));
871        assert!(result.is_none());
872    }
873
874    #[test]
875    fn test_diagnostics_bundle_debug() {
876        let bundle = DiagnosticsBundle {
877            check: None,
878            inspect: None,
879            trace: None,
880            tensors: None,
881            explain: None,
882        };
883        // Just ensure Debug trait is implemented
884        let _ = format!("{:?}", bundle);
885    }
886
887    #[test]
888    fn test_reproduction_info_debug() {
889        let info = ReproductionInfo {
890            command: "apr-qa run test.yaml".to_string(),
891            model_path: "/test/model.apr".to_string(),
892            playbook: None,
893        };
894        // Just ensure Debug trait is implemented
895        let _ = format!("{:?}", info);
896    }
897
898    #[test]
899    fn test_fail_fast_report_debug() {
900        let evidence = test_evidence();
901        let report = FailFastReport {
902            version: "1.0.0".to_string(),
903            timestamp: "2024-02-04T18:00:00Z".to_string(),
904            failure: FailureDetails::from(&evidence),
905            environment: EnvironmentContext {
906                os: "linux".to_string(),
907                arch: "x86_64".to_string(),
908                apr_qa_version: "0.1.0".to_string(),
909                apr_cli_version: "0.2.12".to_string(),
910                git_commit: "abc123".to_string(),
911                git_branch: "main".to_string(),
912                git_dirty: false,
913                rustc_version: "1.93.0".to_string(),
914            },
915            diagnostics: DiagnosticsBundle {
916                check: None,
917                inspect: None,
918                trace: None,
919                tensors: None,
920                explain: None,
921            },
922            reproduction: ReproductionInfo {
923                command: "apr-qa run playbook.yaml --fail-fast".to_string(),
924                model_path: "/path/to/model.apr".to_string(),
925                playbook: Some("playbook.yaml".to_string()),
926            },
927        };
928        // Just ensure Debug trait is implemented
929        let _ = format!("{:?}", report);
930    }
931}
apr_qa_runner/diagnostics.rs

apr_qa_runner/
diagnostics.rs
