use std::io::IsTerminal;
use std::time::Instant;
use clap::Subcommand;
use crate::config::Config;
use crate::history::{self, DiffSummary, RunRecord};
use crate::project::CheckmateProject;
use crate::runner::{
AssertionEvaluator, TestExecutor, TestSpec,
TestResult, TestSuiteResult, TestStatus, AssertionSummary, TestFailure,
diff::{DiffResult, diff_result_to_json, format_diff_tty},
};
#[derive(Subcommand)]
pub enum DiffCommands {
Run {
specs: Vec<String>,
#[arg(short, long)]
diff: Option<String>,
#[arg(short, long)]
verbose: bool,
},
List {
specs: Vec<String>,
},
}
pub fn run(command: DiffCommands) -> Result<(), Box<dyn std::error::Error>> {
match command {
DiffCommands::Run { specs, diff, verbose } => run_diffs(&specs, diff.as_deref(), verbose),
DiffCommands::List { specs } => list_diffs(&specs),
}
}
fn run_diffs(spec_files: &[String], filter: Option<&str>, verbose: bool) -> Result<(), Box<dyn std::error::Error>> {
let project = CheckmateProject::discover();
let config = Config::load(project.as_ref());
let resolved_specs = crate::test::resolve_spec_files(spec_files, project.as_ref())?;
if resolved_specs.is_empty() {
if project.is_some() {
eprintln!("No spec files found in .checkmate/tests/");
} else {
eprintln!("No spec files provided and no .checkmate/ found.");
}
return Ok(());
}
let is_tty = std::io::stdout().is_terminal();
let mut all_passed = true;
for spec_path in &resolved_specs {
let mut spec = TestSpec::from_file(spec_path.to_string_lossy().as_ref())?;
if spec.diffs.is_empty() {
continue;
}
if spec.env.base_url.is_none() {
spec.env.base_url = config.env.base_url.clone();
}
if spec.env.timeout_ms.is_none() {
spec.env.timeout_ms = Some(config.env.timeout_ms);
}
let (result, diff_summary) = run_spec_diffs(&spec, filter, verbose, is_tty)?;
if !result.all_passed() {
all_passed = false;
}
if let Some(ref proj) = project {
let spec_name = spec_path.to_string_lossy().to_string();
let record = RunRecord::from_diff_result(&result, diff_summary, Some(&spec_name));
if let Err(e) = history::save_run(proj, &record) {
eprintln!("Warning: failed to save run history: {}", e);
}
}
let json = serde_json::to_string_pretty(&result)?;
println!("{}", json);
}
if !all_passed {
std::process::exit(1);
}
Ok(())
}
fn run_spec_diffs(
spec: &TestSpec,
filter: Option<&str>,
verbose: bool,
is_tty: bool,
) -> Result<(TestSuiteResult, DiffSummary), Box<dyn std::error::Error>> {
let suite_start = Instant::now();
let mut suite_result = TestSuiteResult::new(spec.name.clone());
let mut agg_summary = DiffSummary {
additions: 0,
removals: 0,
value_changes: 0,
type_changes: 0,
};
let executor = TestExecutor::new(&spec.env)?;
for (diff_name, diff_case) in &spec.diffs {
if let Some(f) = filter {
if diff_name != f {
continue;
}
}
if verbose {
println!(" Running diff: {}", diff_name);
}
let test_start = Instant::now();
let mut failures: Vec<TestFailure> = Vec::new();
let mut assertion_summary = AssertionSummary::default();
let mut test_status = TestStatus::Passed;
let mut spec_copy = spec.clone();
if !spec_copy.requests.contains_key("__diff_empty__") {
spec_copy.requests.insert(
"__diff_empty__".to_string(),
crate::runner::TestRequest {
body: None,
headers: std::collections::HashMap::new(),
extends: None,
query_params: std::collections::HashMap::new(),
extract: std::collections::HashMap::new(),
},
);
}
let mut setup_failed = false;
for (step_idx, step) in diff_case.setup.iter().enumerate() {
if verbose {
println!(" Setup: {} -> {}", step.request, step.endpoint);
}
match executor.execute(&spec_copy, &step.request, &step.endpoint, &step.method, diff_case.timeout_ms) {
Ok(r) => {
if let Ok(request_def) = spec_copy.resolve_request(&step.request) {
if !request_def.extract.is_empty() {
if let Err(e) = executor.extract_variables(&request_def.extract, &r.body) {
if verbose {
eprintln!(" Warning: setup extraction failed: {}", e);
}
}
}
}
}
Err(e) => {
test_status = TestStatus::Error;
failures.push(TestFailure {
request_index: step_idx,
assertion: format!("Setup request '{}' to {}", step.request, step.endpoint),
expected: None,
actual: None,
message: Some("Setup step failed, skipping diff".to_string()),
error: Some(e.to_string()),
});
setup_failed = true;
break;
}
}
}
if setup_failed {
let test_result = TestResult {
name: diff_name.clone(),
description: diff_case.description.clone(),
status: test_status,
duration_ms: test_start.elapsed().as_millis() as u64,
request_count: 0,
assertions: assertion_summary,
failures,
};
suite_result.add_result(test_result);
continue;
}
let mut responses: Vec<(String, String, serde_json::Value)> = Vec::new();
for endpoint_def in &diff_case.endpoints {
let ep_name = endpoint_def.name();
let ep_path = endpoint_def.path();
if verbose {
println!(" Endpoint: {} ({})", ep_name, ep_path);
}
let applicable_requests: Vec<&str> = if diff_case.requests.is_empty() {
vec!["__diff_empty__"]
} else {
let scoped: Vec<&str> = diff_case.requests.iter()
.filter(|r| r.applies_to(ep_name))
.map(|r| r.request_name())
.collect();
if scoped.is_empty() {
vec!["__diff_empty__"]
} else {
scoped
}
};
let mut last_body = serde_json::Value::Null;
for (req_idx, request_name) in applicable_requests.iter().enumerate() {
match executor.execute(&spec_copy, request_name, ep_path, &diff_case.method, diff_case.timeout_ms) {
Ok(r) => {
if r.status != diff_case.expect_status {
test_status = TestStatus::Failed;
failures.push(TestFailure {
request_index: req_idx,
assertion: format!("HTTP status for {}", ep_name),
expected: Some(diff_case.expect_status.to_string()),
actual: Some(r.status.to_string()),
message: Some(format!("Unexpected HTTP status for {}", ep_name)),
error: None,
});
}
if let Ok(request_def) = spec_copy.resolve_request(request_name) {
if !request_def.extract.is_empty() {
if let Err(e) = executor.extract_variables(&request_def.extract, &r.body) {
if verbose {
eprintln!(" Warning: extraction failed: {}", e);
}
}
}
}
last_body = r.body;
}
Err(e) => {
test_status = TestStatus::Error;
failures.push(TestFailure {
request_index: req_idx,
assertion: format!("HTTP request to {}", ep_name),
expected: None,
actual: None,
message: None,
error: Some(e.to_string()),
});
}
}
}
responses.push((ep_name.to_string(), ep_path.to_string(), last_body));
}
for pair in responses.windows(2) {
let (base_name, _, base_body) = &pair[0];
let (target_name, _, target_body) = &pair[1];
let diff_result = DiffResult::from_comparison(
base_name.clone(),
target_name.clone(),
base_body.clone(),
target_body.clone(),
);
agg_summary.additions += diff_result.additions;
agg_summary.removals += diff_result.removals;
agg_summary.value_changes += diff_result.value_changes;
agg_summary.type_changes += diff_result.type_changes;
if is_tty {
eprint!("{}", format_diff_tty(&diff_result));
}
if !diff_case.assertions.is_empty() {
let diff_json = diff_result_to_json(&diff_result);
let mut evaluator = AssertionEvaluator::new();
evaluator.set_current(diff_json);
for assertion in &diff_case.assertions {
if let Some(ref scope) = assertion.scope {
if !scope.contains(base_name) || !scope.contains(target_name) {
continue;
}
}
assertion_summary.total += 1;
let result = evaluator.evaluate(assertion);
if result.passed {
assertion_summary.passed += 1;
if verbose {
println!(" ✓ {}", assertion.query.as_deref().unwrap_or("(no query)"));
}
} else {
assertion_summary.failed += 1;
test_status = TestStatus::Failed;
failures.push(TestFailure {
request_index: 0,
assertion: assertion.query.clone().unwrap_or_default(),
expected: result.expected,
actual: result.actual,
message: result.message,
error: result.error,
});
if verbose {
println!(" ✗ {}", assertion.query.as_deref().unwrap_or("(no query)"));
}
}
}
}
if !is_tty {
let json = serde_json::to_string_pretty(&diff_result)?;
println!("{}", json);
}
}
let test_result = TestResult {
name: diff_name.clone(),
description: diff_case.description.clone(),
status: test_status,
duration_ms: test_start.elapsed().as_millis() as u64,
request_count: diff_case.endpoints.len(),
assertions: assertion_summary,
failures,
};
suite_result.add_result(test_result);
}
suite_result.set_duration(suite_start.elapsed().as_millis() as u64);
Ok((suite_result, agg_summary))
}
fn list_diffs(spec_files: &[String]) -> Result<(), Box<dyn std::error::Error>> {
let project = CheckmateProject::discover();
let resolved_specs = crate::test::resolve_spec_files(spec_files, project.as_ref())?;
if resolved_specs.is_empty() {
eprintln!("No spec files found.");
return Ok(());
}
for spec_path in &resolved_specs {
let spec = TestSpec::from_file(spec_path.to_string_lossy().as_ref())?;
if spec.diffs.is_empty() {
continue;
}
println!("{}:", spec_path.display());
if let Some(ref name) = spec.name {
println!(" Suite: {}", name);
}
println!(" Diffs:");
for (name, diff) in &spec.diffs {
let desc = diff.description.as_deref().unwrap_or("");
let endpoints: String = diff.endpoints.iter()
.map(|e| e.name().to_string())
.collect::<Vec<_>>()
.join(" ↔ ");
if desc.is_empty() {
println!(" - {} ({})", name, endpoints);
} else {
println!(" - {}: {} ({})", name, desc, endpoints);
}
}
println!();
}
Ok(())
}