use crate::image;
use crate::script::{exit, Script, Step};
use crate::tools::ToolSet;
use serde::Serialize;
use serde_json::{json, Value};
use std::path::{Path, PathBuf};
use std::time::Instant;
#[derive(Debug, Clone, Serialize, PartialEq, Eq)]
#[serde(rename_all = "snake_case")]
pub enum Outcome {
Pass,
ExpectationFailed,
ToolError,
Timeout,
}
#[derive(Debug, Clone, Serialize)]
pub struct StepResult {
pub index: usize,
pub tool: String,
pub outcome: Outcome,
pub elapsed_ms: u128,
#[serde(skip_serializing_if = "Vec::is_empty")]
pub failures: Vec<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub image: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub diff_ratio: Option<f64>,
}
#[derive(Debug, Clone, Serialize)]
pub struct RunResult {
pub script: String,
pub passed: bool,
pub exit_code: i32,
pub steps: Vec<StepResult>,
#[serde(skip_serializing_if = "Option::is_none")]
pub failed_step: Option<usize>,
}
pub struct RunOptions {
pub out_dir: PathBuf,
pub baselines_dir: PathBuf,
pub update_baselines: bool,
pub stop_on_fail: bool,
}
pub async fn run(script: &Script, tools: &ToolSet, opts: &RunOptions) -> RunResult {
let _ = std::fs::create_dir_all(&opts.out_dir);
let deadline = Instant::now() + std::time::Duration::from_millis(script.timeout_ms);
let mut steps = Vec::new();
let mut failed_step = None;
for (index, step) in script.steps.iter().enumerate() {
if Instant::now() > deadline {
steps.push(StepResult {
index,
tool: step.tool.clone(),
outcome: Outcome::Timeout,
elapsed_ms: 0,
failures: vec![format!("script timeout ({} ms) reached before step {index}", script.timeout_ms)],
image: None,
diff_ratio: None,
});
failed_step = Some(index);
break;
}
let result = run_step(index, step, tools, opts).await;
let failed = result.outcome != Outcome::Pass;
steps.push(result);
if failed {
failed_step = Some(index);
if opts.stop_on_fail {
break;
}
}
}
let exit_code = steps
.iter()
.map(|s| match s.outcome {
Outcome::Pass => exit::PASS,
Outcome::ExpectationFailed => exit::EXPECTATION_FAILED,
Outcome::ToolError => exit::TOOL_ERROR,
Outcome::Timeout => exit::TIMEOUT,
})
.max()
.unwrap_or(exit::PASS);
let result = RunResult {
script: script.name.clone(),
passed: exit_code == exit::PASS,
exit_code,
steps,
failed_step,
};
if let Ok(text) = serde_json::to_string_pretty(&result) {
let _ = std::fs::write(opts.out_dir.join("result.json"), text);
}
result
}
async fn run_step(index: usize, step: &Step, tools: &ToolSet, opts: &RunOptions) -> StepResult {
let started = Instant::now();
let mut res = StepResult {
index,
tool: step.tool.clone(),
outcome: Outcome::Pass,
elapsed_ms: 0,
failures: Vec::new(),
image: None,
diff_ratio: None,
};
let Some(spec) = tools.get(&step.tool) else {
res.outcome = Outcome::ToolError;
res.failures.push(format!("no tool `{}` in this session", step.tool));
res.elapsed_ms = started.elapsed().as_millis();
return res;
};
let args = substitute_out(if step.args.is_null() { json!({}) } else { step.args.clone() }, &opts.out_dir);
let output = match (spec.handler)(args).await {
Ok(o) => {
if let Some(pattern) = &step.expect_error {
res.outcome = Outcome::ExpectationFailed;
res.failures.push(format!("expected the tool to fail with `{pattern}` but it succeeded"));
res.elapsed_ms = started.elapsed().as_millis();
write_step_json(opts, index, &step.tool, &o.json);
return res;
}
o
}
Err(e) => {
write_step_json(opts, index, &step.tool, &json!({ "error": e }));
if let Some(pattern) = &step.expect_error {
if !crate::script::glob_match(pattern, &e) {
res.outcome = Outcome::ExpectationFailed;
res.failures.push(format!("tool failed as expected but with `{e}`, which does not match `{pattern}`"));
}
res.elapsed_ms = started.elapsed().as_millis();
return res;
}
res.outcome = Outcome::ToolError;
res.failures.push(e);
res.elapsed_ms = started.elapsed().as_millis();
return res;
}
};
write_step_json(opts, index, &step.tool, &output.json);
for e in &step.expect {
if let Err(why) = e.check(&output.json) {
res.failures.push(why);
}
}
if let Some(first) = output.images.first() {
let name = step
.save
.clone()
.unwrap_or_else(|| format!("{index:04}-{}.png", step.tool));
let path = opts.out_dir.join(&name);
let _ = std::fs::write(&path, &first.png);
res.image = Some(name.clone());
if let Some(cmp) = &step.compare {
let baseline = opts.baselines_dir.join(&cmp.baseline);
match (image::decode_png(&first.png), std::fs::read(&baseline)) {
(Ok(actual), Ok(base_bytes)) => match image::decode_png(&base_bytes) {
Ok(base) => {
let d = image::diff(&base, &actual, 8);
res.diff_ratio = Some(d.ratio);
if d.ratio > cmp.max_diff_ratio {
if opts.update_baselines {
let _ = std::fs::create_dir_all(baseline.parent().unwrap_or(Path::new(".")));
let _ = std::fs::write(&baseline, &first.png);
} else {
let _ = std::fs::write(opts.out_dir.join(format!("{name}.diff.png")), image::encode_png(&d.mask).unwrap_or_default());
res.failures.push(format!(
"image differs from baseline `{}` by {:.4} (max {:.4})",
cmp.baseline, d.ratio, cmp.max_diff_ratio
));
}
}
}
Err(e) => res.failures.push(format!("baseline `{}`: {e}", cmp.baseline)),
},
(Ok(_), Err(_)) => {
if opts.update_baselines {
let _ = std::fs::create_dir_all(baseline.parent().unwrap_or(Path::new(".")));
let _ = std::fs::write(&baseline, &first.png);
} else {
res.failures.push(format!(
"no baseline `{}` (run with --update-baselines to create it)",
cmp.baseline
));
}
}
(Err(e), _) => res.failures.push(format!("step image: {e}")),
}
}
} else if step.compare.is_some() || step.save.is_some() {
res.failures.push(format!("step {index} asked to save/compare an image but `{}` produced none", step.tool));
}
if !res.failures.is_empty() {
res.outcome = Outcome::ExpectationFailed;
}
res.elapsed_ms = started.elapsed().as_millis();
res
}
fn substitute_out(v: Value, out: &Path) -> Value {
match v {
Value::String(s) if s.contains("${out}") => Value::String(s.replace("${out}", &out.display().to_string())),
Value::Array(a) => Value::Array(a.into_iter().map(|x| substitute_out(x, out)).collect()),
Value::Object(o) => Value::Object(o.into_iter().map(|(k, x)| (k, substitute_out(x, out))).collect()),
other => other,
}
}
fn write_step_json(opts: &RunOptions, index: usize, tool: &str, value: &Value) {
if let Ok(text) = serde_json::to_string_pretty(value) {
let _ = std::fs::write(opts.out_dir.join(format!("{index:04}-{tool}.json")), text);
}
}