use std::error::Error;
use std::fmt;
use std::fmt::Write as _;
use std::fs;
use std::path::{Path, PathBuf};
use url::Url;
use crate::config::Config;
use crate::contract::{CaseV1, ManifestError, ManifestV1};
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct GeneratedRunner {
pub directory: PathBuf,
pub config: PathBuf,
pub manifest_spec: PathBuf,
pub review: PathBuf,
pub case_count: usize,
}
pub fn fetch_manifest(url: &Url) -> Result<ManifestV1, RunnerError> {
let mut response = ureq::get(url.as_str())
.call()
.map_err(|source| RunnerError::Fetch {
url: url.clone(),
source: Box::new(source),
})?;
let manifest: ManifestV1 =
response
.body_mut()
.read_json()
.map_err(|source| RunnerError::Fetch {
url: url.clone(),
source: Box::new(source),
})?;
manifest.validate().map_err(RunnerError::Manifest)?;
Ok(manifest)
}
pub fn generate(
project_root: &Path,
config: &Config,
manifest: &ManifestV1,
) -> Result<GeneratedRunner, RunnerError> {
manifest.validate().map_err(RunnerError::Manifest)?;
let cases = manifest.cases();
if cases.is_empty() {
return Err(RunnerError::NoCases);
}
let directory = project_root.join(".coatcheck/generated");
fs::create_dir_all(&directory).map_err(|source| RunnerError::Write {
path: directory.clone(),
source,
})?;
let config_path = directory.join("playwright.config.ts");
let spec_path = directory.join("manifest.spec.ts");
let review_path = config.artifacts_dir(project_root).join("review.md");
let playwright = render_playwright_config(project_root, config, &spec_path)?;
let spec = render_manifest_spec(config, manifest, &cases)?;
write(&config_path, &playwright)?;
write(&spec_path, &spec)?;
write_review(&review_path, manifest, &cases, None)?;
Ok(GeneratedRunner {
directory,
config: config_path,
manifest_spec: spec_path,
review: review_path,
case_count: cases.len(),
})
}
pub fn finish_review(
generated: &GeneratedRunner,
manifest: &ManifestV1,
success: bool,
) -> Result<(), RunnerError> {
write_review(
&generated.review,
manifest,
&manifest.cases(),
Some(success),
)
}
fn render_playwright_config(
project_root: &Path,
config: &Config,
generated_spec: &Path,
) -> Result<String, RunnerError> {
let generated_relative = generated_spec
.strip_prefix(project_root)
.map_err(|_| RunnerError::InvalidPath(generated_spec.to_path_buf()))?;
let artifacts = config.artifacts_dir(project_root);
let matches = std::iter::once(path_string(generated_relative))
.chain(config.test.story_globs.iter().cloned())
.collect::<Vec<_>>();
let root_json = json(&path_string(project_root))?;
let matches_json = json(&matches)?;
let artifacts_json = json(&path_string(&artifacts))?;
Ok(format!(
r#"import {{ defineConfig }} from "@playwright/test";
export default defineConfig({{
testDir: {root_json},
testMatch: {matches_json},
timeout: {timeout},
fullyParallel: false,
forbidOnly: Boolean(process.env.CI),
retries: process.env.CI ? 1 : 0,
workers: process.env.CI ? 1 : undefined,
reporter: [
["list"],
["html", {{ outputFolder: {artifacts_json} + "/playwright-report", open: "never" }}],
["json", {{ outputFile: {artifacts_json} + "/results.json" }}],
],
outputDir: {artifacts_json} + "/test-results",
use: {{
browserName: "chromium",
headless: process.env.COATCHECK_HEADED !== "1",
launchOptions: {{
slowMo: Number(process.env.COATCHECK_SLOW_MO ?? 0),
}},
trace: "retain-on-failure",
screenshot: "off",
video: "off",
}},
}});
"#,
timeout = config.test.timeout_ms,
))
}
#[allow(clippy::too_many_lines)]
fn render_manifest_spec(
config: &Config,
manifest: &ManifestV1,
cases: &[CaseV1<'_>],
) -> Result<String, RunnerError> {
let mut output = String::from(
r#"import fs from "node:fs/promises";
import path from "node:path";
import AxeBuilder from "@axe-core/playwright";
import { expect, test } from "@playwright/test";
"#,
);
let artifacts = path_string(&config.test.artifacts);
writeln!(output, "const artifactsRoot = {};", json(&artifacts)?).unwrap();
writeln!(
output,
"const axeImpacts = new Set<string>({});",
json(&config.test.axe_impacts)?
)
.unwrap();
writeln!(
output,
"const rootSelector = {};",
json(&manifest.frame.root_selector)?
)
.unwrap();
writeln!(
output,
"const readySelector = {};",
json(&manifest.frame.ready_selector)?
)
.unwrap();
writeln!(
output,
"const errorSelector = {};\n",
json(&manifest.frame.error_selector)?
)
.unwrap();
output.push_str(
r"type Case = {
title: string;
url: string;
artifact: string;
viewport: { width: number; height: number } | null;
};
const cases: Case[] = [
",
);
let base = Url::parse(&config.host.url).map_err(|_| RunnerError::InvalidHostUrl)?;
for case in cases {
let frame_url = frame_url(&base, manifest, case)?;
let scene_id = case.scene.map_or("default", |scene| scene.id.as_str());
let title = format!("{} [{}]", case.story.title, scene_id);
let artifact = format!("{}--{}", slug(&case.story.id), slug(scene_id));
let viewport = case.scene.map(|scene| {
serde_json::json!({
"width": scene.viewport.width,
"height": scene.viewport.height,
})
});
writeln!(
output,
" {{ title: {}, url: {}, artifact: {}, viewport: {} }},",
json(&title)?,
json(frame_url.as_str())?,
json(&artifact)?,
serde_json::to_string(&viewport).map_err(RunnerError::Json)?,
)
.unwrap();
}
output.push_str(
r#"];
for (const item of cases) {
test(item.title, async ({ page }) => {
if (item.viewport) {
await page.setViewportSize(item.viewport);
}
const consoleErrors: string[] = [];
const pageErrors: string[] = [];
page.on("console", (message) => {
if (message.type() === "error") consoleErrors.push(message.text());
});
page.on("pageerror", (error) => pageErrors.push(error.message));
let navigationError: string | null = null;
let status: number | null = null;
try {
const response = await page.goto(item.url, { waitUntil: "domcontentloaded" });
status = response?.status() ?? null;
if (!response?.ok()) navigationError = `frame returned ${status ?? "no response"}`;
} catch (error) {
navigationError = String(error);
}
let readyError: string | null = null;
try {
await page.locator(readySelector).first().waitFor({ state: "attached" });
} catch (error) {
readyError = String(error);
}
const rootCount = await page.locator(rootSelector).count();
const hostErrorCount = await page.locator(errorSelector).count();
const hasHorizontalOverflow = await page.evaluate(
() => document.documentElement.scrollWidth > document.documentElement.clientWidth + 1,
);
let axeError: string | null = null;
let screenshotError: string | null = null;
let axe = { violations: [] as Array<{ id: string; impact?: string | null; help: string; nodes: unknown[] }> };
try {
axe = await new AxeBuilder({ page }).analyze();
} catch (error) {
axeError = String(error);
}
const blockingAxe = axe.violations.filter(
(violation) => violation.impact && axeImpacts.has(violation.impact),
);
const artifactDir = path.join(artifactsRoot, "cases", item.artifact);
await fs.mkdir(artifactDir, { recursive: true });
await page
.screenshot({ path: path.join(artifactDir, "review.png"), fullPage: true })
.catch((error) => { screenshotError = String(error); });
await Promise.all([
fs.writeFile(path.join(artifactDir, "axe.json"), JSON.stringify(axe, null, 2)),
fs.writeFile(
path.join(artifactDir, "console.json"),
JSON.stringify(
{ consoleErrors, pageErrors, navigationError, readyError, axeError, screenshotError, status },
null,
2,
),
),
fs.writeFile(path.join(artifactDir, "dom.html"), await page.content()),
]);
expect.soft(navigationError, "frame navigation").toBeNull();
expect.soft(readyError, "frame readiness marker").toBeNull();
expect.soft(rootCount, "frame root marker").toBeGreaterThan(0);
expect.soft(hostErrorCount, "host error marker").toBe(0);
expect.soft(hasHorizontalOverflow, "horizontal viewport overflow").toBe(false);
expect.soft(pageErrors, "uncaught page errors").toEqual([]);
expect.soft(consoleErrors, "browser console errors").toEqual([]);
expect.soft(axeError, "axe execution").toBeNull();
expect.soft(screenshotError, "review screenshot capture").toBeNull();
expect.soft(
blockingAxe.map(({ id, impact, help }) => ({ id, impact, help })),
"configured axe impact gate",
).toEqual([]);
});
}
"#,
);
Ok(output)
}
fn frame_url(base: &Url, manifest: &ManifestV1, case: &CaseV1<'_>) -> Result<Url, RunnerError> {
let mut url = match Url::parse(&manifest.frame.path) {
Ok(url) => url,
Err(url::ParseError::RelativeUrlWithoutBase) => base
.join(&manifest.frame.path)
.map_err(|_| RunnerError::InvalidFrameUrl(manifest.frame.path.clone()))?,
Err(_) => return Err(RunnerError::InvalidFrameUrl(manifest.frame.path.clone())),
};
{
let mut query = url.query_pairs_mut();
query.append_pair(&manifest.frame.story_query, &case.story.id);
if let Some(scene) = case.scene {
query.append_pair(&manifest.frame.scene_query, &scene.id);
}
}
Ok(url)
}
fn write_review(
path: &Path,
manifest: &ManifestV1,
cases: &[CaseV1<'_>],
success: Option<bool>,
) -> Result<(), RunnerError> {
let mut review = format!(
"# Coatcheck review\n\nCatalog: **{}**\n\nSemantic result: **{}**\n\n",
manifest.name,
match success {
Some(true) => "passed",
Some(false) => "failed",
None => "pending",
}
);
review.push_str(
"The test gate uses navigation, frame markers, browser errors, and Axe. PNG files are review artifacts only; Coatcheck does not compare pixel baselines.\n\n## Cases\n\n",
);
for case in cases {
let scene_id = case.scene.map_or("default", |scene| scene.id.as_str());
let artifact = format!("{}--{}", slug(&case.story.id), slug(scene_id));
writeln!(
review,
"- `{}` / `{}`: [image](cases/{artifact}/review.png), [DOM](cases/{artifact}/dom.html), [Axe](cases/{artifact}/axe.json), [console](cases/{artifact}/console.json)",
case.story.id, scene_id
)
.unwrap();
}
review.push_str(
"\n## Runner output\n\n- [Playwright HTML report](playwright-report/index.html)\n- [Machine-readable results](results.json)\n",
);
write(path, &review)
}
fn write(path: &Path, contents: &str) -> Result<(), RunnerError> {
if let Some(parent) = path.parent() {
fs::create_dir_all(parent).map_err(|source| RunnerError::Write {
path: parent.to_path_buf(),
source,
})?;
}
fs::write(path, contents).map_err(|source| RunnerError::Write {
path: path.to_path_buf(),
source,
})
}
fn json(value: &(impl serde::Serialize + ?Sized)) -> Result<String, RunnerError> {
serde_json::to_string(value).map_err(RunnerError::Json)
}
fn path_string(path: &Path) -> String {
path.to_string_lossy().replace('\\', "/")
}
fn slug(value: &str) -> String {
let slug = value
.chars()
.map(|character| {
if character.is_ascii_alphanumeric() || matches!(character, '-' | '_') {
character.to_ascii_lowercase()
} else {
'-'
}
})
.collect::<String>();
slug.trim_matches('-').to_owned()
}
#[derive(Debug)]
pub enum RunnerError {
Fetch {
url: Url,
source: Box<ureq::Error>,
},
Manifest(ManifestError),
Json(serde_json::Error),
InvalidHostUrl,
InvalidFrameUrl(String),
InvalidPath(PathBuf),
NoCases,
Write {
path: PathBuf,
source: std::io::Error,
},
}
impl fmt::Display for RunnerError {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::Fetch { url, source } => write!(formatter, "could not fetch {url}: {source}"),
Self::Manifest(error) => write!(formatter, "invalid host manifest: {error}"),
Self::Json(error) => write!(formatter, "could not encode runner data: {error}"),
Self::InvalidHostUrl => formatter.write_str("host.url is invalid"),
Self::InvalidFrameUrl(url) => write!(formatter, "frame URL is invalid: {url}"),
Self::InvalidPath(path) => {
write!(formatter, "{} is outside the project root", path.display())
}
Self::NoCases => formatter.write_str("host manifest contains no runnable cases"),
Self::Write { path, source } => {
write!(formatter, "could not write {}: {source}", path.display())
}
}
}
}
impl Error for RunnerError {
fn source(&self) -> Option<&(dyn Error + 'static)> {
match self {
Self::Fetch { source, .. } => Some(source),
Self::Manifest(source) => Some(source),
Self::Json(source) => Some(source),
Self::Write { source, .. } => Some(source),
Self::InvalidHostUrl
| Self::InvalidFrameUrl(_)
| Self::InvalidPath(_)
| Self::NoCases => None,
}
}
}
#[cfg(test)]
mod tests {
use std::path::Path;
use crate::config::Config;
use crate::contract::{FrameContractV1, ManifestV1, SceneV1, StoryV1, ViewportV1};
use super::generate;
#[test]
fn generates_explicit_cases_and_semantic_gates() {
let root = tempfile::tempdir().unwrap();
let config: Config = toml::from_str(
r#"
version = 1
name = "Example"
[host]
command = ["cargo", "run"]
url = "http://127.0.0.1:3000"
[test]
story_globs = ["coatcheck/**/*.spec.ts"]
artifacts = ".coatcheck/artifacts"
timeout_ms = 30000
axe_impacts = ["critical"]
"#,
)
.unwrap();
let manifest = ManifestV1 {
version: 1,
name: "Example".to_owned(),
frame: FrameContractV1::default(),
scenes: vec![SceneV1 {
id: "mobile".to_owned(),
label: "Mobile".to_owned(),
viewport: ViewportV1 {
width: 393,
height: 852,
},
surface: "canvas".to_owned(),
}],
stories: vec![StoryV1 {
id: "button".to_owned(),
title: "Button".to_owned(),
tier: "Atoms".to_owned(),
kind: "state".to_owned(),
description: String::new(),
source: "src/button.rs".to_owned(),
args: Vec::new(),
scenes: Vec::new(),
}],
};
let generated = generate(root.path(), &config, &manifest).unwrap();
let spec = std::fs::read_to_string(generated.manifest_spec).unwrap();
let playwright = std::fs::read_to_string(generated.config).unwrap();
assert_eq!(generated.case_count, 1);
assert!(spec.contains("button--mobile"));
assert!(spec.contains("new AxeBuilder"));
assert!(spec.contains("review.png"));
assert!(!spec.contains("toHaveScreenshot"));
assert!(playwright.contains("coatcheck/**/*.spec.ts"));
assert!(Path::new(&generated.review).exists());
}
}