use crate::cargo_ops;
use crate::config::{Config, ResolvedRepo, WptConfig};
use crate::ui;
use crate::WptArgs;
use serde::{Deserialize, Serialize};
use std::collections::BTreeMap;
use std::path::Path;
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
struct Baseline {
#[serde(default)]
results: BTreeMap<String, SuiteResult>,
}
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
struct SuiteResult {
passed: u32,
failed: u32,
}
const DEFAULT_RUNNER: &str = "cargo test --test wpt";
pub fn run(config: Config, args: &WptArgs) {
let targets: Vec<(&ResolvedRepo, &WptConfig)> = config
.repos
.iter()
.filter_map(|r| r.wpt.as_ref().map(|w| (r, w)))
.filter(|(r, _)| match &args.only {
Some(only) => only.split(',').any(|n| n.trim() == r.name),
None => true,
})
.collect();
if targets.is_empty() {
ui::heading("WPT Semantic Comparison");
println!();
ui::info(
"No WPT suites configured in repos.toml.\n\
\x20 Add `wpt = { suites = [\"<name>\"] }` to a [[repos]] entry, then run `pets wpt <project>`.\n\
\x20 See README for the [project.wpt] / [[repos]].wpt schema.",
);
return;
}
let mut all_ok = true;
let mut ran = 0usize;
for (repo, wpt) in &targets {
let runner = wpt
.runner
.as_deref()
.or_else(|| config.wpt.as_ref().and_then(|w| w.runner.as_deref()))
.unwrap_or(DEFAULT_RUNNER);
let baseline_path = match &wpt.baseline {
Some(b) if b.is_absolute() => b.clone(),
Some(b) => repo.abs_dir.join(b),
None => config.baseline_dir.join(format!("{}.toml", repo.name)),
};
for suite in &wpt.suites {
ui::heading(&format!("{} — suite: {}", repo.name, suite));
let mut cmd = String::with_capacity(runner.len() + suite.len() + 1);
cmd.push_str(runner);
cmd.push(' ');
cmd.push_str(suite);
let extra = [
("WPT_SUITE", suite.as_str()),
("PETS_PROJECT", config.project_name.as_str()),
("PETS_REPO", repo.name.as_str()),
];
match crate::ops::run::run_shell_capture(&repo.abs_dir, &cmd, &extra) {
Ok((status, stdout, stderr)) => {
let (passed, failed) = cargo_ops::parse_test_counts(&stdout, &stderr);
ran += 1;
if !compare_baseline(
&baseline_path,
suite,
passed,
failed,
args.update_baseline,
) {
all_ok = false;
}
if !status.success() {
ui::error(&format!(" runner exited with {}", status));
all_ok = false;
}
}
Err(e) => {
ui::error(&format!(" failed to run: {}", e));
all_ok = false;
}
}
}
}
ui::separator();
if all_ok {
ui::success(&format!("WPT complete — {} suite(s) ran", ran));
} else {
ui::error("WPT completed with failures or regressions.");
std::process::exit(1);
}
}
fn compare_baseline(path: &Path, suite: &str, passed: u32, failed: u32, update: bool) -> bool {
let current = SuiteResult { passed, failed };
if update {
let mut baseline = load_baseline(path);
baseline.results.insert(suite.to_string(), current);
if let Err(e) = write_baseline(path, &baseline) {
ui::error(&format!(
" cannot write baseline {}: {}",
path.display(),
e
));
return false;
}
ui::info(&format!(
" baseline updated: {} passed / {} failed",
passed, failed
));
return true;
}
match load_baseline(path).results.get(suite) {
None => {
ui::warn(&format!(
" no baseline for '{}' at {} — run with --update-baseline to record",
suite,
path.display()
));
ui::info(&format!(" result: {} passed / {} failed", passed, failed));
true
}
Some(base) => {
let delta = failed as i64 - base.failed as i64;
ui::info(&format!(
" result: {} passed / {} failed (baseline {}, Δ{})",
passed, failed, base.failed, delta
));
if failed > base.failed {
ui::error(&format!(
" regression: {} failed (baseline {})",
failed, base.failed
));
false
} else {
true
}
}
}
}
fn load_baseline(path: &Path) -> Baseline {
match std::fs::read_to_string(path) {
Ok(content) => toml::from_str(&content).unwrap_or_default(),
Err(_) => Baseline::default(),
}
}
fn write_baseline(path: &Path, baseline: &Baseline) -> Result<(), String> {
if let Some(parent) = path.parent() {
std::fs::create_dir_all(parent)
.map_err(|e| format!("cannot create {}: {}", parent.display(), e))?;
}
let content = toml::to_string(baseline).map_err(|e| e.to_string())?;
std::fs::write(path, content).map_err(|e| e.to_string())
}
#[cfg(test)]
mod tests {
use super::*;
use std::path::PathBuf;
fn temp_dir(tag: &str) -> PathBuf {
let mut dir = std::env::temp_dir();
dir.push(format!(
"pets-wpt-test-{}-{}-{}",
tag,
std::process::id(),
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_nanos()
));
std::fs::create_dir_all(&dir).unwrap();
dir
}
#[test]
fn missing_baseline_is_not_regression() {
let dir = temp_dir("missing");
let path = dir.join("baseline.toml");
assert!(compare_baseline(&path, "html5lib", 10, 2, false));
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn regression_detected_when_failures_exceed_baseline() {
let dir = temp_dir("regress");
let path = dir.join("baseline.toml");
let baseline = Baseline {
results: [(
"html5lib".to_string(),
SuiteResult {
passed: 10,
failed: 0,
},
)]
.into_iter()
.collect(),
};
write_baseline(&path, &baseline).unwrap();
assert!(!compare_baseline(&path, "html5lib", 9, 2, false));
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn update_baseline_records_and_loads_back() {
let dir = temp_dir("update");
let path = dir.join("baseline.toml");
assert!(compare_baseline(&path, "suite", 5, 0, true));
let baseline = load_baseline(&path);
let r = baseline.results.get("suite").unwrap();
assert_eq!(r.passed, 5);
assert_eq!(r.failed, 0);
let _ = std::fs::remove_dir_all(&dir);
}
}