use std::path::{Path, PathBuf};
use std::process::Command;
fn repository() -> PathBuf {
Path::new(env!("CARGO_MANIFEST_DIR"))
.join("../../..")
.canonicalize()
.expect("the repository root resolves")
}
fn scratch() -> PathBuf {
let at = Path::new(env!("CARGO_TARGET_TMPDIR")).join("json");
std::fs::create_dir_all(&at).expect("the directory is there");
at
}
fn publish_out() -> PathBuf {
use std::sync::atomic::{AtomicUsize, Ordering};
static NEXT: AtomicUsize = AtomicUsize::new(0);
let at = scratch().join(format!("publish-{}", NEXT.fetch_add(1, Ordering::SeqCst)));
if at.exists() {
std::fs::remove_dir_all(&at).expect("the leftover is removed");
}
at
}
#[derive(Debug)]
struct Ran {
code: Option<i32>,
out: Vec<u8>,
err: Vec<u8>,
}
impl Ran {
fn text(&self) -> String {
String::from_utf8_lossy(&self.out).into_owned()
}
}
fn ran(arguments: &[&str]) -> Ran {
let output = Command::new(env!("CARGO_BIN_EXE_headwater"))
.args(arguments)
.arg("--root")
.arg(repository())
.output()
.expect("the binary runs");
Ran {
code: output.status.code(),
out: output.stdout,
err: output.stderr,
}
}
const ANSWERED: &str = "what does a check know about the front matter of a document";
const UNANSWERED: &str = "xyzzy plugh frobnicate quuxbar";
fn read_set() -> PathBuf {
let path = scratch().join("run.readset");
if !path.exists() {
let written = ran(&["check", "--read-set", path.to_str().expect("a path")]);
assert_eq!(
written.code,
Some(0),
"the read set is written: {written:?}"
);
}
path
}
fn sweep_return() -> PathBuf {
let path = scratch().join("return.yml");
if !path.exists() {
let plan = ran(&["sweep", "plan"]).text();
let taxonomy = plan
.lines()
.find(|line| line.trim_start().starts_with("taxonomy: sha256:"))
.expect("the plan states the taxonomy it was written against")
.trim()
.to_string();
std::fs::write(&path, format!("{taxonomy}\nslice: .\nfindings: []\n"))
.expect("the return file writes");
}
path
}
fn both_spellings() -> Vec<(&'static str, Vec<String>)> {
let returned = sweep_return().to_str().expect("a path").to_string();
vec![
("check", vec!["check".to_string(), "--no-cache".to_string()]),
("capture", vec!["capture".to_string()]),
(
"export",
vec![
"export".to_string(),
"--profile".to_string(),
"site".to_string(),
],
),
(
"sweep report",
vec!["sweep".to_string(), "report".to_string(), returned],
),
]
}
fn documents() -> Vec<(&'static str, Ran)> {
let returned = sweep_return();
let recorded = read_set();
let source = repository().join("taxonomy-source/headwater-standard");
let out = publish_out();
vec![
("check --json", ran(&["check", "--json"])),
("capture --json", ran(&["capture", "--json"])),
(
"export --json",
ran(&["export", "--json", "--profile", "site"]),
),
(
"sweep report --json",
ran(&[
"sweep",
"report",
returned.to_str().expect("a path"),
"--json",
]),
),
(
"taxonomy publish --json",
ran(&[
"taxonomy",
"publish",
"--json",
"--from",
source.to_str().expect("a path"),
"--out",
out.to_str().expect("a path"),
]),
),
("route --json, offered", ran(&["route", "--json", ANSWERED])),
(
"route --json, silent",
ran(&["route", "--json", UNANSWERED]),
),
(
"explain --json",
ran(&["explain", "--json", "docs/spec/12-check-layer.md"]),
),
(
"gate --json",
ran(&[
"gate",
"--json",
"--read-set",
recorded.to_str().expect("a path"),
]),
),
("conformance --json", ran(&["conformance", "--json"])),
(
"conformance --json --level",
ran(&["conformance", "--json", "--level", "L1"]),
),
]
}
#[test]
fn the_two_spellings_of_one_target_write_the_same_bytes() {
for (name, base) in both_spellings() {
let mut with_flag: Vec<&str> = base.iter().map(String::as_str).collect();
with_flag.push("--json");
let mut with_format: Vec<&str> = base.iter().map(String::as_str).collect();
with_format.extend(["--format", "json"]);
let flagged = ran(&with_flag);
let formatted = ran(&with_format);
assert_eq!(
flagged.code, formatted.code,
"`{name} --json` and `{name} --format json` exit alike"
);
assert_eq!(
flagged.out, formatted.out,
"`{name} --json` writes the bytes `{name} --format json` writes"
);
assert_eq!(
flagged.err, formatted.err,
"`{name} --json` accounts for itself as `{name} --format json` does"
);
assert!(
!flagged.out.is_empty(),
"`{name} --json` writes something, so the comparison above is not two empty files"
);
}
}
#[test]
fn a_command_line_that_names_one_target_twice_is_refused() {
for (name, base) in both_spellings() {
let mut arguments: Vec<&str> = base.iter().map(String::as_str).collect();
arguments.extend(["--json", "--format", "json"]);
let refused = ran(&arguments);
assert_eq!(
refused.code,
Some(1),
"`{name} --json --format json` is refused with exit 1: {refused:?}"
);
let says = String::from_utf8_lossy(&refused.err);
assert!(
says.contains("--json") && says.contains("--format"),
"the refusal names both spellings: {says}"
);
assert!(
refused.out.is_empty(),
"and it writes no half-artifact: {}",
refused.text()
);
}
}
fn refusals() -> Vec<(&'static str, Vec<&'static str>)> {
vec![
(
"explain --json, no such document",
vec!["explain", "--json", "docs/spec/no-such-part.md"],
),
("explain --json, no target", vec!["explain", "--json"]),
(
"gate --json, no such read set",
vec!["gate", "--json", "--read-set", "no-such.readset"],
),
("gate --json, no read set", vec!["gate", "--json"]),
(
"conformance --json, no such rung",
vec!["conformance", "--json", "--level", "L99"],
),
("route --json, no task", vec!["route", "--json"]),
(
"sweep report --json, no such file",
vec!["sweep", "report", "no-such.yml", "--json"],
),
(
"sweep report --format json, no such file",
vec!["sweep", "report", "no-such.yml", "--format", "json"],
),
("export --json --check", vec!["export", "--json", "--check"]),
(
"export --format json --check",
vec!["export", "--format", "json", "--check"],
),
("export --json, two profiles", vec!["export", "--json"]),
(
"export --format json, two profiles",
vec!["export", "--format", "json"],
),
(
"taxonomy publish --json, no --out",
vec!["taxonomy", "publish", "--json"],
),
(
"taxonomy publish --json, --package beside --from",
vec![
"taxonomy",
"publish",
"--json",
"--package",
"headwater/standard",
"--from",
"taxonomy-source/headwater-standard",
],
),
(
"check --json --format json",
vec!["check", "--json", "--format", "json"],
),
(
"capture --json --format json",
vec!["capture", "--json", "--format", "json"],
),
]
}
#[test]
fn a_refusal_writes_no_document_and_accounts_for_itself_on_the_other_stream() {
for (name, arguments) in refusals() {
let refused = ran(&arguments);
assert_eq!(
refused.code,
Some(1),
"`{name}` is refused with exit 1: {refused:?}"
);
assert!(
refused.out.is_empty(),
"`{name}` writes nothing to standard output, and it wrote: {}",
refused.text()
);
assert!(
!refused.err.is_empty(),
"`{name}` says why on standard error: {refused:?}"
);
}
}
#[test]
fn a_run_that_completed_and_then_failed_still_wrote_its_document() {
let unwritable = scratch().join("no-such-directory").join("run.readset");
let unwritable = unwritable.to_str().expect("a path");
for (name, arguments) in [
(
"check --json",
vec!["check", "--json", "--read-set", unwritable],
),
(
"check --format json",
vec!["check", "--format", "json", "--read-set", unwritable],
),
] {
let failed = ran(&arguments);
assert_eq!(
failed.code,
Some(1),
"`{name}` with an unwritable read set exits 1: {failed:?}"
);
assert!(
!failed.err.is_empty(),
"`{name}` says which path it could not write: {failed:?}"
);
let artifact = failed.text();
let parsed = headwater_yaml::load(&artifact)
.unwrap_or_else(|_| panic!("`{name}` put a whole JSON report on standard output"));
assert!(
member(&parsed.value, "version").is_some(),
"`{name}` wrote the whole document and not a prefix of one: {artifact}"
);
}
}
#[test]
fn a_refusal_names_the_spelling_the_caller_typed() {
for (name, arguments, typed, untyped) in [
(
"export --json --check",
vec!["export", "--json", "--check"],
"--json",
"--format",
),
(
"export --format json --check",
vec!["export", "--format", "json", "--check"],
"--format",
"--json",
),
(
"export --json, two profiles",
vec!["export", "--json"],
"--json",
"--format",
),
(
"export --format json, two profiles",
vec!["export", "--format", "json"],
"--format",
"--json",
),
] {
let refused = ran(&arguments);
assert_eq!(
refused.code,
Some(1),
"`{name}` is refused with exit 1: {refused:?}"
);
let says = String::from_utf8_lossy(&refused.err);
assert!(
says.contains(typed),
"`{name}` names `{typed}`, the spelling it was given: {says}"
);
assert!(
!says.contains(untyped),
"`{name}` does not name `{untyped}`, which nobody typed: {says}"
);
}
}
#[test]
fn every_document_this_binary_writes_is_read_by_a_parser_that_is_not_this_one() {
let documents = documents();
let expected = documents.len();
let mut outside = 0;
for (name, run) in documents {
assert!(
!run.out.is_empty(),
"`{name}` writes a document at all: {run:?}"
);
let artifact = run.text();
headwater_yaml::load(&artifact)
.unwrap_or_else(|errors| panic!("`{name}` does not parse in tree: {errors:?}"));
if let Some(refusal) = oracle(&artifact) {
panic!("`{name}` is not JSON by the clause's own reading: {refusal}");
}
if std::env::var_os("HEADWATER_JSON_ORACLE").is_some() {
outside += 1;
}
}
if std::env::var_os("HEADWATER_JSON_ORACLE").is_some() {
assert_eq!(
outside, expected,
"every document reached the outside parser"
);
}
}
fn oracle(artifact: &str) -> Option<String> {
let required = std::env::var_os("HEADWATER_JSON_ORACLE").is_some();
let written = scratch().join("artifact.json");
std::fs::write(&written, artifact).expect("the artifact writes");
let ran = Command::new("python3")
.args(["-m", "json.tool", written.to_str().expect("a path")])
.output();
let reason = match ran {
Ok(output) if output.status.success() => return None,
Ok(output) => match String::from_utf8_lossy(&output.stderr).trim() {
"" => format!("it exited {}", output.status),
said => said.to_string(),
},
Err(error) => error.to_string(),
};
if reason.contains("Expecting") || reason.contains("Invalid") || reason.contains("Extra data") {
return Some(reason);
}
assert!(
!required,
"HEADWATER_JSON_ORACLE is set and `python3 -m json.tool` did not run, so nothing outside \
this repository read these bytes: {reason}"
);
eprintln!(
"note: `python3 -m json.tool` did not run ({reason}), so only the in-tree reader ran"
);
None
}
#[test]
fn the_json_form_moves_no_exit_status() {
let recorded = read_set();
let recorded = recorded.to_str().expect("a path");
let pairs: Vec<(&str, Vec<&str>)> = vec![
("route, offered", vec!["route", ANSWERED]),
("route, silent", vec!["route", UNANSWERED]),
(
"explain, a document",
vec!["explain", "docs/spec/12-check-layer.md"],
),
(
"explain, nothing of that name",
vec!["explain", "docs/spec/no-such-part.md"],
),
("gate, a read set", vec!["gate", "--read-set", recorded]),
(
"gate, no such file",
vec!["gate", "--read-set", "no-such.readset"],
),
("conformance", vec!["conformance"]),
("conformance, a rung", vec!["conformance", "--level", "L1"]),
(
"conformance, no such rung",
vec!["conformance", "--level", "L99"],
),
];
for (name, base) in pairs {
let plain = ran(&base);
let mut with_flag = base.clone();
with_flag.push("--json");
let flagged = ran(&with_flag);
assert_eq!(
plain.code, flagged.code,
"`{name}` exits alike with and without `--json`: {plain:?} against {flagged:?}"
);
}
}
#[test]
fn no_escape_byte_reaches_a_document_this_binary_writes() {
for (name, run) in documents() {
assert!(
!run.out.contains(&0x1b),
"`{name}` writes no escape byte on standard output"
);
assert!(
!run.err.contains(&0x1b),
"`{name}` writes no escape byte on standard error"
);
}
}
#[test]
fn every_document_this_binary_writes_names_its_own_shape() {
for (name, run) in documents() {
let value = headwater_yaml::load(&run.text())
.unwrap_or_else(|errors| panic!("`{name}` does not parse: {errors:?}"))
.value;
let version = member(&value, "version")
.unwrap_or_else(|| panic!("`{name}` names its own shape in a `version` member"));
assert_ne!(
version,
headwater_resolve::release::ENGINE,
"`{name}` names its shape and not the engine that wrote it"
);
}
}
#[test]
fn a_silent_route_writes_an_empty_pointer_set_and_says_why() {
let silent = ran(&["route", "--json", UNANSWERED]);
assert_eq!(silent.code, Some(0), "a silence is a result: {silent:?}");
let artifact = silent.text();
assert!(
artifact.contains("\"pointers\": []"),
"the pointer set is there and it is empty:\n{artifact}"
);
assert!(
artifact.contains("\"reason\":"),
"and the silence says which of the four it is:\n{artifact}"
);
let offered = ran(&["route", "--json", ANSWERED]);
assert_eq!(offered.code, Some(0), "{offered:?}");
assert!(
!offered.text().contains("\"pointers\": []"),
"a task this corpus answers offers pointers, so the case above is not vacuous"
);
}
#[test]
fn the_text_a_route_carries_is_the_report_the_same_run_would_print() {
for task in [ANSWERED, UNANSWERED] {
let printed = ran(&["route", task]);
let document = ran(&["route", "--json", task]);
let value = headwater_yaml::load(&document.text())
.expect("the route document parses")
.value;
let carried = member(&value, "text").expect("the route document carries `text`");
assert_eq!(
carried,
printed.text(),
"the `text` member is the report, byte for byte"
);
}
}
fn member(value: &headwater_yaml::Value, key: &str) -> Option<String> {
value
.as_map()
.and_then(|map| map.get(key))
.and_then(|spanned| spanned.value.as_scalar())
.map(headwater_yaml::core_schema::as_str)
.map(str::to_string)
}