use anyhow::Result;
use serial_test::serial;
use std::path::PathBuf;
use tempfile::TempDir;
use xchecker::orchestrator::{OrchestratorConfig, PhaseOrchestrator};
use xchecker::types::{PhaseId, Receipt};
#[allow(clippy::duplicate_mod)]
#[path = "test_support/mod.rs"]
mod test_support;
fn normalize_core_yaml_for_determinism(content: &str) -> String {
content
.lines()
.map(|line| {
if line.starts_with("spec_id:") {
"spec_id: \"deterministic\"".to_string()
} else if line.starts_with("generated_at:") {
"generated_at: \"1970-01-01T00:00:00Z\"".to_string()
} else if line.starts_with("# Core requirements data for spec") {
"# Core requirements data for spec deterministic".to_string()
} else {
line.to_string()
}
})
.collect::<Vec<_>>()
.join("\n")
}
struct WorkflowTestEnvironment {
#[allow(dead_code)]
_cwd_guard: test_support::CwdGuard,
temp_dir: TempDir,
orchestrator: PhaseOrchestrator,
spec_id: String,
}
impl WorkflowTestEnvironment {
fn new(test_name: &str) -> Result<Self> {
let temp_dir = TempDir::new()?;
let cwd_guard = test_support::CwdGuard::new(temp_dir.path())?;
std::fs::create_dir_all(temp_dir.path().join(".xchecker/specs"))?;
let spec_id = format!("workflow-{test_name}");
let orchestrator = PhaseOrchestrator::new(&spec_id)?;
Ok(Self {
_cwd_guard: cwd_guard,
temp_dir,
orchestrator,
spec_id,
})
}
fn spec_dir(&self) -> PathBuf {
self.temp_dir
.path()
.join(".xchecker/specs")
.join(&self.spec_id)
}
fn create_success_config(&self) -> OrchestratorConfig {
OrchestratorConfig {
dry_run: false,
config: {
let mut map = std::collections::HashMap::new();
map.insert("runner_mode".to_string(), "native".to_string());
let stub_path = test_support::claude_stub_path()
.expect("claude-stub path is required for workflow tests");
map.insert("claude_cli_path".to_string(), stub_path);
map.insert("claude_scenario".to_string(), "success".to_string());
map.insert("verbose".to_string(), "true".to_string());
map
},
full_config: None,
selectors: None,
strict_validation: false,
redactor: Default::default(),
hooks: None,
}
}
fn create_error_config(&self) -> OrchestratorConfig {
OrchestratorConfig {
dry_run: false,
config: {
let mut map = std::collections::HashMap::new();
map.insert("runner_mode".to_string(), "native".to_string());
let stub_path = test_support::claude_stub_path()
.expect("claude-stub path is required for workflow tests");
map.insert("claude_cli_path".to_string(), stub_path);
map.insert("claude_scenario".to_string(), "error".to_string());
map
},
full_config: None,
selectors: None,
strict_validation: false,
redactor: Default::default(),
hooks: None,
}
}
}
#[tokio::test]
#[serial]
#[ignore = "requires_claude_stub"]
async fn test_complete_spec_generation_flow() -> Result<()> {
let env = WorkflowTestEnvironment::new("complete-flow")?;
let config = env.create_success_config();
println!("Executing Requirements phase...");
let req_result = env.orchestrator.execute_requirements_phase(&config).await?;
assert!(req_result.success, "Requirements phase should succeed");
assert_eq!(req_result.phase, PhaseId::Requirements);
assert_eq!(
req_result.artifact_paths.len(),
2,
"Should create .md and .core.yaml"
);
let artifacts_dir = env.spec_dir().join("artifacts");
let req_md = artifacts_dir.join("00-requirements.md");
let req_yaml = artifacts_dir.join("00-requirements.core.yaml");
assert!(req_md.exists(), "Requirements markdown should exist");
assert!(req_yaml.exists(), "Requirements YAML should exist");
let req_content = std::fs::read_to_string(&req_md)?;
assert!(
req_content.contains("# Requirements Document"),
"Should have proper structure"
);
assert!(
req_content.contains("**User Story:**"),
"Should have user stories"
);
assert!(req_content.contains("WHEN"), "Should have EARS format");
println!("Executing Design phase...");
let design_result = env.orchestrator.execute_design_phase(&config).await?;
assert!(design_result.success, "Design phase should succeed");
assert_eq!(design_result.phase, PhaseId::Design);
assert_eq!(
design_result.artifact_paths.len(),
2,
"Should create .md and .core.yaml"
);
let design_md = artifacts_dir.join("10-design.md");
let design_yaml = artifacts_dir.join("10-design.core.yaml");
assert!(design_md.exists(), "Design markdown should exist");
assert!(design_yaml.exists(), "Design YAML should exist");
let design_content = std::fs::read_to_string(&design_md)?;
assert!(
design_content.contains("# Design Document"),
"Should have proper structure"
);
assert!(
design_content.contains("## Architecture"),
"Should have architecture section"
);
println!("Executing Tasks phase...");
let tasks_result = env.orchestrator.execute_tasks_phase(&config).await?;
assert!(tasks_result.success, "Tasks phase should succeed");
assert_eq!(tasks_result.phase, PhaseId::Tasks);
assert_eq!(
tasks_result.artifact_paths.len(),
2,
"Should create .md and .core.yaml"
);
let tasks_md = artifacts_dir.join("20-tasks.md");
let tasks_yaml = artifacts_dir.join("20-tasks.core.yaml");
assert!(tasks_md.exists(), "Tasks markdown should exist");
assert!(tasks_yaml.exists(), "Tasks YAML should exist");
let tasks_content = std::fs::read_to_string(&tasks_md)?;
assert!(
tasks_content.contains("# Implementation Plan"),
"Should have proper structure"
);
assert!(
tasks_content.contains("- [ ]"),
"Should have task checkboxes"
);
let receipts_dir = env.spec_dir().join("receipts");
assert!(receipts_dir.exists(), "Receipts directory should exist");
let receipt_files: Vec<_> = std::fs::read_dir(&receipts_dir)?
.filter_map(std::result::Result::ok)
.filter(|entry| entry.path().extension().is_some_and(|ext| ext == "json"))
.collect();
assert_eq!(
receipt_files.len(),
3,
"Should have 3 receipts (one per phase)"
);
for receipt_file in receipt_files {
let receipt_content = std::fs::read_to_string(receipt_file.path())?;
let receipt: Receipt = serde_json::from_str(&receipt_content)?;
assert_eq!(receipt.spec_id, env.spec_id);
assert_eq!(receipt.exit_code, 0);
assert!(!receipt.xchecker_version.is_empty());
assert!(!receipt.claude_cli_version.is_empty());
assert!(!receipt.model_full_name.is_empty());
assert!(!receipt.outputs.is_empty());
}
println!("✓ Complete spec generation flow test passed");
Ok(())
}
#[tokio::test]
#[serial]
#[ignore = "requires_claude_stub"]
async fn test_resume_scenarios_and_failure_recovery() -> Result<()> {
let env = WorkflowTestEnvironment::new("resume-recovery")?;
let success_config = env.create_success_config();
let req_result = env
.orchestrator
.execute_requirements_phase(&success_config)
.await?;
assert!(req_result.success, "Requirements phase should succeed");
let error_config = env.create_error_config();
let design_result = env.orchestrator.execute_design_phase(&error_config).await?;
assert!(
!design_result.success,
"Design phase should fail with error config"
);
assert_ne!(design_result.exit_code, 0, "Should have non-zero exit code");
let artifacts_dir = env.spec_dir().join("artifacts");
let partial_files: Vec<_> = std::fs::read_dir(&artifacts_dir)?
.filter_map(std::result::Result::ok)
.filter(|entry| {
entry
.path()
.file_name()
.and_then(|name| name.to_str())
.is_some_and(|name| name.contains(".partial."))
})
.collect();
assert!(
!partial_files.is_empty(),
"Should have created partial artifact on failure"
);
assert!(
design_result.receipt_path.is_some(),
"Should create receipt even on failure"
);
let receipt_path = design_result.receipt_path.unwrap();
let receipt_content = std::fs::read_to_string(&receipt_path)?;
let receipt: Receipt = serde_json::from_str(&receipt_content)?;
assert_ne!(
receipt.exit_code, 0,
"Receipt should record failure exit code"
);
assert!(
receipt.stderr_tail.is_some(),
"Receipt should capture stderr"
);
assert!(!receipt.warnings.is_empty(), "Receipt should have warnings");
println!("Resuming Design phase after failure...");
let resume_result = env
.orchestrator
.resume_from_phase(PhaseId::Design, &success_config)
.await?;
assert!(resume_result.success, "Resume should succeed");
assert_eq!(resume_result.phase, PhaseId::Design);
let design_md = artifacts_dir.join("10-design.md");
let design_yaml = artifacts_dir.join("10-design.core.yaml");
assert!(
design_md.exists(),
"Design markdown should exist after resume"
);
assert!(
design_yaml.exists(),
"Design YAML should exist after resume"
);
let _remaining_partials: Vec<_> = std::fs::read_dir(&artifacts_dir)?
.filter_map(std::result::Result::ok)
.filter(|entry| {
entry
.path()
.file_name()
.and_then(|name| name.to_str())
.is_some_and(|name| name.contains(".partial."))
})
.collect();
println!("✓ Resume scenarios and failure recovery test passed");
Ok(())
}
#[tokio::test]
#[serial]
#[ignore = "requires_claude_stub"]
async fn test_determinism_with_identical_inputs() -> Result<()> {
let env1 = WorkflowTestEnvironment::new("determinism-1")?;
let env2 = WorkflowTestEnvironment::new("determinism-2")?;
let config = OrchestratorConfig {
dry_run: true, config: {
let mut map = std::collections::HashMap::new();
map.insert("model".to_string(), "haiku".to_string());
map.insert("verbose".to_string(), "false".to_string());
map
},
full_config: None,
selectors: None,
strict_validation: false,
redactor: Default::default(),
hooks: None,
};
let req_result1 = env1
.orchestrator
.execute_requirements_phase(&config)
.await?;
let req_result2 = env2
.orchestrator
.execute_requirements_phase(&config)
.await?;
assert!(
req_result1.success && req_result2.success,
"Both executions should succeed"
);
let artifacts1_dir = env1.spec_dir().join("artifacts");
let artifacts2_dir = env2.spec_dir().join("artifacts");
let req_md1 = std::fs::read_to_string(artifacts1_dir.join("00-requirements.md"))?;
let req_md2 = std::fs::read_to_string(artifacts2_dir.join("00-requirements.md"))?;
let req_yaml1 = std::fs::read_to_string(artifacts1_dir.join("00-requirements.core.yaml"))?;
let req_yaml2 = std::fs::read_to_string(artifacts2_dir.join("00-requirements.core.yaml"))?;
let req_yaml1 = normalize_core_yaml_for_determinism(&req_yaml1);
let req_yaml2 = normalize_core_yaml_for_determinism(&req_yaml2);
assert_eq!(
req_md1, req_md2,
"Markdown outputs should be identical with same inputs"
);
assert_eq!(
req_yaml1, req_yaml2,
"YAML outputs should be identical with same inputs"
);
let canonicalizer = xchecker::canonicalization::Canonicalizer::new();
let hash1_md =
canonicalizer.hash_canonicalized(&req_md1, xchecker::types::FileType::Markdown)?;
let hash2_md =
canonicalizer.hash_canonicalized(&req_md2, xchecker::types::FileType::Markdown)?;
let hash1_yaml =
canonicalizer.hash_canonicalized(&req_yaml1, xchecker::types::FileType::Yaml)?;
let hash2_yaml =
canonicalizer.hash_canonicalized(&req_yaml2, xchecker::types::FileType::Yaml)?;
assert_eq!(
hash1_md, hash2_md,
"Canonicalized markdown hashes should be identical"
);
assert_eq!(
hash1_yaml, hash2_yaml,
"Canonicalized YAML hashes should be identical"
);
let receipt1_path = req_result1.receipt_path.unwrap();
let receipt2_path = req_result2.receipt_path.unwrap();
let receipt1_content = std::fs::read_to_string(&receipt1_path)?;
let receipt2_content = std::fs::read_to_string(&receipt2_path)?;
let receipt1: Receipt = serde_json::from_str(&receipt1_content)?;
let receipt2: Receipt = serde_json::from_str(&receipt2_content)?;
assert_eq!(
receipt1.model_full_name, receipt2.model_full_name,
"Model names should be identical"
);
assert_eq!(
receipt1.canonicalization_version, receipt2.canonicalization_version,
"Canonicalization versions should be identical"
);
let outputs1: Vec<_> = receipt1
.outputs
.iter()
.filter(|output| !output.path.ends_with(".core.yaml"))
.collect();
let outputs2: Vec<_> = receipt2
.outputs
.iter()
.filter(|output| !output.path.ends_with(".core.yaml"))
.collect();
assert_eq!(
outputs1.len(),
outputs2.len(),
"Should have same number of deterministic outputs"
);
for (output1, output2) in outputs1.iter().zip(outputs2.iter()) {
assert_eq!(output1.path, output2.path, "Output paths should match");
assert_eq!(
output1.blake3_canonicalized, output2.blake3_canonicalized,
"Canonicalized hashes should be identical for same inputs"
);
}
println!("✓ Determinism with identical inputs test passed");
Ok(())
}
#[tokio::test]
#[serial]
#[ignore = "requires_claude_stub"]
async fn test_multi_phase_workflow_with_dependencies() -> Result<()> {
let env = WorkflowTestEnvironment::new("dependencies")?;
let config = env.create_success_config();
let design_result = env.orchestrator.execute_design_phase(&config).await;
assert!(
design_result.is_err(),
"Design phase should fail without Requirements dependency"
);
let req_result = env.orchestrator.execute_requirements_phase(&config).await?;
assert!(req_result.success, "Requirements phase should succeed");
let design_result = env.orchestrator.execute_design_phase(&config).await?;
assert!(
design_result.success,
"Design phase should succeed after Requirements"
);
let tasks_result = env.orchestrator.execute_tasks_phase(&config).await?;
assert!(
tasks_result.success,
"Tasks phase should succeed after Design"
);
let artifacts_dir = env.spec_dir().join("artifacts");
assert!(
artifacts_dir.join("00-requirements.md").exists(),
"Requirements artifacts should exist"
);
assert!(
artifacts_dir.join("00-requirements.core.yaml").exists(),
"Requirements YAML should exist"
);
assert!(
artifacts_dir.join("10-design.md").exists(),
"Design artifacts should exist"
);
assert!(
artifacts_dir.join("10-design.core.yaml").exists(),
"Design YAML should exist"
);
assert!(
artifacts_dir.join("20-tasks.md").exists(),
"Tasks artifacts should exist"
);
assert!(
artifacts_dir.join("20-tasks.core.yaml").exists(),
"Tasks YAML should exist"
);
let receipts_dir = env.spec_dir().join("receipts");
let receipt_files: Vec<_> = std::fs::read_dir(&receipts_dir)?
.filter_map(std::result::Result::ok)
.filter(|entry| entry.path().extension().is_some_and(|ext| ext == "json"))
.collect();
assert_eq!(
receipt_files.len(),
3,
"Should have receipts for all three phases"
);
let mut receipts = Vec::new();
for receipt_file in receipt_files {
let receipt_content = std::fs::read_to_string(receipt_file.path())?;
let receipt: Receipt = serde_json::from_str(&receipt_content)?;
receipts.push(receipt);
}
receipts.sort_by(|a, b| a.emitted_at.cmp(&b.emitted_at));
assert_eq!(
receipts[0].phase, "requirements",
"First receipt should be requirements"
);
assert_eq!(
receipts[1].phase, "design",
"Second receipt should be design"
);
assert_eq!(receipts[2].phase, "tasks", "Third receipt should be tasks");
println!("✓ Multi-phase workflow with dependencies test passed");
Ok(())
}
#[tokio::test]
#[serial]
#[ignore = "requires_claude_stub"]
async fn test_artifact_content_validation() -> Result<()> {
let env = WorkflowTestEnvironment::new("content-validation")?;
let config = env.create_success_config();
let req_result = env.orchestrator.execute_requirements_phase(&config).await?;
let design_result = env.orchestrator.execute_design_phase(&config).await?;
let tasks_result = env.orchestrator.execute_tasks_phase(&config).await?;
assert!(
req_result.success && design_result.success && tasks_result.success,
"All phases should succeed"
);
let artifacts_dir = env.spec_dir().join("artifacts");
let req_md = std::fs::read_to_string(artifacts_dir.join("00-requirements.md"))?;
assert!(
req_md.contains("# Requirements Document"),
"Requirements should have proper title"
);
assert!(
req_md.contains("## Introduction"),
"Requirements should have introduction"
);
assert!(
req_md.contains("## Requirements"),
"Requirements should have requirements section"
);
assert!(
req_md.contains("**User Story:**"),
"Requirements should have user stories"
);
assert!(
req_md.contains("#### Acceptance Criteria"),
"Requirements should have acceptance criteria"
);
assert!(
req_md.contains("WHEN") && req_md.contains("THEN") && req_md.contains("SHALL"),
"Requirements should use EARS format"
);
let req_yaml = std::fs::read_to_string(artifacts_dir.join("00-requirements.core.yaml"))?;
assert!(
!req_yaml.is_empty(),
"Requirements YAML should not be empty"
);
let design_md = std::fs::read_to_string(artifacts_dir.join("10-design.md"))?;
assert!(
design_md.contains("# Design Document"),
"Design should have proper title"
);
assert!(
design_md.contains("## Overview"),
"Design should have overview"
);
assert!(
design_md.contains("## Architecture"),
"Design should have architecture section"
);
let design_yaml = std::fs::read_to_string(artifacts_dir.join("10-design.core.yaml"))?;
assert!(!design_yaml.is_empty(), "Design YAML should not be empty");
let tasks_md = std::fs::read_to_string(artifacts_dir.join("20-tasks.md"))?;
assert!(
tasks_md.contains("# Implementation Plan"),
"Tasks should have proper title"
);
assert!(tasks_md.contains("- [ ]"), "Tasks should have checkboxes");
assert!(
tasks_md.contains("Milestone"),
"Tasks should have milestones"
);
let tasks_yaml = std::fs::read_to_string(artifacts_dir.join("20-tasks.core.yaml"))?;
assert!(!tasks_yaml.is_empty(), "Tasks YAML should not be empty");
let _req_parsed: serde_yaml::Value = serde_yaml::from_str(&req_yaml)?;
let _design_parsed: serde_yaml::Value = serde_yaml::from_str(&design_yaml)?;
let _tasks_parsed: serde_yaml::Value = serde_yaml::from_str(&tasks_yaml)?;
println!("✓ Artifact content validation test passed");
Ok(())
}
#[tokio::test]
#[serial]
#[ignore = "requires_claude_stub"]
async fn test_error_propagation_and_recovery() -> Result<()> {
if !test_support::should_run_e2e() {
eprintln!("(skipped) E2E test requires Claude CLI or XCHECKER_E2E=1");
return Ok(());
}
let env = WorkflowTestEnvironment::new("error-propagation")?;
let success_config = env.create_success_config();
let req_result = env
.orchestrator
.execute_requirements_phase(&success_config)
.await?;
assert!(req_result.success, "Requirements should succeed");
let error_config = env.create_error_config();
let design_result = env.orchestrator.execute_design_phase(&error_config).await?;
assert!(
!design_result.success,
"Design should fail with error config"
);
let tasks_result = env.orchestrator.execute_tasks_phase(&success_config).await;
assert!(
tasks_result.is_err(),
"Tasks should fail without successful Design"
);
let design_recovery = env
.orchestrator
.resume_from_phase(PhaseId::Design, &success_config)
.await?;
assert!(design_recovery.success, "Design recovery should succeed");
let tasks_result = env
.orchestrator
.execute_tasks_phase(&success_config)
.await?;
assert!(
tasks_result.success,
"Tasks should succeed after Design recovery"
);
let artifacts_dir = env.spec_dir().join("artifacts");
assert!(
artifacts_dir.join("00-requirements.md").exists(),
"Requirements should exist"
);
assert!(
artifacts_dir.join("10-design.md").exists(),
"Design should exist after recovery"
);
assert!(
artifacts_dir.join("20-tasks.md").exists(),
"Tasks should exist after recovery"
);
let receipts_dir = env.spec_dir().join("receipts");
let receipt_files: Vec<_> = std::fs::read_dir(&receipts_dir)?
.filter_map(std::result::Result::ok)
.filter(|entry| entry.path().extension().is_some_and(|ext| ext == "json"))
.collect();
assert!(
receipt_files.len() >= 3,
"Should have multiple receipts including failure and recovery"
);
println!("✓ Error propagation and recovery test passed");
Ok(())
}
pub async fn run_full_workflow_validation() -> Result<()> {
println!("🚀 Full workflow tests require claude-stub binary.");
println!(" Run with: cargo test --test integration_full_workflows -- --include-ignored");
println!();
println!("Full Workflow Requirements Coverage:");
println!(" ✓ R1.1: Complete spec generation flows (Requirements → Design → Tasks)");
println!(" ✓ R2.2: Deterministic outputs with identical inputs");
println!(" ✓ R2.5: Structure determinism for canonicalized outputs");
println!(" ✓ R4.2: Resume scenarios and failure recovery");
println!();
println!("Key Features Covered:");
println!(" ✓ End-to-end workflow execution with proper phase dependencies");
println!(" ✓ Resume capability from failed phases with partial artifact handling");
println!(" ✓ Deterministic behavior with identical inputs producing same outputs");
println!(" ✓ Proper error propagation and recovery across multiple phases");
println!(" ✓ Artifact content validation and structure verification");
println!(" ✓ Receipt generation and audit trail maintenance");
Ok(())
}