use std::collections::BTreeMap;
#[cfg(feature = "json-schema")]
use std::fmt::Write as _;
use serde_json::Value as JsonValue;
use crate::models::{AssertionResult, Recipe};
use crate::pypath::PyPath;
use crate::pyrepr::{repr_json, repr_str, str_json, type_name};
#[cfg(feature = "json-schema")]
use crate::pyschema;
const UNCONDITIONAL_TYPES: [&str; 7] = [
"output_contains",
"output_not_contains",
"screen_contains",
"screen_not_contains",
"exit_code",
"file_exists",
"file_contains",
];
#[cfg(feature = "json-schema")]
pub const BUILTIN_TYPES: [&str; 8] = {
let mut all = [""; 8];
let mut i = 0;
while i < UNCONDITIONAL_TYPES.len() {
all[i] = UNCONDITIONAL_TYPES[i];
i += 1;
}
all[i] = "json_schema";
all
};
#[cfg(not(feature = "json-schema"))]
pub const BUILTIN_TYPES: [&str; 7] = UNCONDITIONAL_TYPES;
pub fn evaluated_list(recipe: &Recipe) -> Vec<JsonValue> {
let mut all = recipe.assertions.clone();
if let Some(expected) = recipe.expect_exit_code {
all.push(serde_json::json!({"type": "exit_code", "value": expected}));
}
all
}
pub fn evaluate_all(
recipe: &Recipe,
screen: &str,
raw_output: &str,
exit_code: Option<i32>,
) -> Vec<AssertionResult> {
evaluated_list(recipe)
.iter()
.map(|assertion| evaluate(recipe, assertion, screen, raw_output, exit_code))
.collect()
}
pub fn evaluate(
recipe: &Recipe,
assertion: &JsonValue,
screen: &str,
raw_output: &str,
exit_code: Option<i32>,
) -> AssertionResult {
let name = name_of(assertion);
match dispatch(recipe, assertion, screen, raw_output, exit_code) {
Ok(outcome) => AssertionResult {
name,
passed: outcome.passed,
detail: outcome.detail,
},
Err(detail) => AssertionResult {
name,
passed: false,
detail,
},
}
}
struct Outcome {
passed: bool,
detail: String,
}
impl Outcome {
fn new(passed: bool, detail: String) -> Self {
Self { passed, detail }
}
}
fn name_of(assertion: &JsonValue) -> String {
if let Some(name) = assertion.get("name").and_then(JsonValue::as_str) {
return name.to_string();
}
match assertion.get("type").and_then(JsonValue::as_str) {
Some(kind) => kind.to_string(),
None => "assertion".to_string(),
}
}
fn dispatch(
recipe: &Recipe,
assertion: &JsonValue,
screen: &str,
raw_output: &str,
exit_code: Option<i32>,
) -> Result<Outcome, String> {
let kind = match assertion.get("type") {
Some(JsonValue::String(kind)) => kind.as_str(),
Some(other) => return Err(format!("unknown assertion type {}", repr_json(other))),
None => return Err("assertion has no 'type'".to_string()),
};
match kind {
"output_contains" => contains(assertion, raw_output, true),
"output_not_contains" => contains(assertion, raw_output, false),
"screen_contains" => contains(assertion, screen, true),
"screen_not_contains" => contains(assertion, screen, false),
"exit_code" => exit_code_matches(assertion, exit_code),
"file_exists" => file_exists(recipe, assertion),
"file_contains" => file_contains(recipe, assertion),
#[cfg(feature = "json-schema")]
"json_schema" => json_schema(recipe, assertion, raw_output),
other => Err(format!("unknown assertion type {}", repr_str(other))),
}
}
fn require<'a>(assertion: &'a JsonValue, key: &str, kind: &str) -> Result<&'a JsonValue, String> {
assertion
.get(key)
.ok_or_else(|| format!("{kind} requires a {}", repr_str(key)))
}
fn require_str<'a>(assertion: &'a JsonValue, key: &str, kind: &str) -> Result<&'a str, String> {
let value = require(assertion, key, kind)?;
value.as_str().ok_or_else(|| {
format!(
"{kind} {} must be a string, got {}",
repr_str(key),
type_name(value)
)
})
}
fn contains(assertion: &JsonValue, haystack: &str, want: bool) -> Result<Outcome, String> {
let kind = assertion["type"].as_str().unwrap_or("assertion");
let needle = require_str(assertion, "value", kind)?;
let found = haystack.contains(needle);
let expectation = if want { "contains" } else { "does not contain" };
Ok(Outcome::new(
found == want,
custom_detail(assertion).unwrap_or_else(|| format!("{expectation} {}", repr_str(needle))),
))
}
fn custom_detail(assertion: &JsonValue) -> Option<String> {
match assertion.get("detail").and_then(JsonValue::as_str) {
Some("") | None => None,
Some(detail) => Some(detail.to_string()),
}
}
fn exit_code_matches(assertion: &JsonValue, exit_code: Option<i32>) -> Result<Outcome, String> {
let expected = require(assertion, "value", "exit_code")?;
let actual = match exit_code {
Some(code) => JsonValue::from(code),
None => JsonValue::Null,
};
Ok(Outcome::new(
python_eq(expected, exit_code),
format!("expected {}, got {}", str_json(expected), str_json(&actual)),
))
}
fn python_eq(expected: &JsonValue, exit_code: Option<i32>) -> bool {
let Some(code) = exit_code else {
return expected.is_null();
};
match expected {
JsonValue::Bool(b) => i64::from(*b) == i64::from(code),
JsonValue::Number(n) => match n.as_i64() {
Some(i) => i == i64::from(code),
None => n.as_f64() == Some(f64::from(code)),
},
_ => false,
}
}
fn recipe_path(recipe: &Recipe, path: &str) -> PyPath {
let candidate = PyPath::parse(path);
if candidate.is_absolute() {
return candidate;
}
PyPath::parse(recipe.command.cwd.as_deref().unwrap_or(".")).join(&candidate)
}
fn file_exists(recipe: &Recipe, assertion: &JsonValue) -> Result<Outcome, String> {
let value = require(assertion, "value", "file_exists")?;
let path = recipe_path(recipe, &str_json(value));
let rendered = path.to_string();
Ok(Outcome::new(
std::path::Path::new(&rendered).exists(),
rendered,
))
}
fn file_contains(recipe: &Recipe, assertion: &JsonValue) -> Result<Outcome, String> {
let path = require(assertion, "path", "file_contains")?;
let needle = require_str(assertion, "value", "file_contains")?;
let resolved = recipe_path(recipe, &str_json(path)).to_string();
let text = std::fs::read(&resolved)
.map(|bytes| String::from_utf8_lossy(&bytes).into_owned())
.unwrap_or_default();
Ok(Outcome::new(
text.contains(needle),
format!("contains {}", repr_str(needle)),
))
}
#[cfg(feature = "json-schema")]
fn json_schema(
recipe: &Recipe,
assertion: &JsonValue,
raw_output: &str,
) -> Result<Outcome, String> {
let schema = resolve_schema(recipe, assertion)?;
let validator = match pyschema::compile(&schema) {
Ok(validator) => validator,
Err(why) => return Ok(Outcome::new(false, format!("invalid schema: {why}"))),
};
let instance: JsonValue = match serde_json::from_str(raw_output.trim()) {
Ok(instance) => instance,
Err(error) => {
return Ok(Outcome::new(
false,
format!(
"invalid JSON output: {}",
describe_json_error(raw_output, &error)
),
))
}
};
match pyschema::validate(&validator, &instance) {
None => Ok(Outcome::new(true, "matches JSON schema".to_string())),
Some((path, why)) => {
let mut detail = String::from("schema validation failed");
if !path.is_empty() {
let _ = write!(detail, " at {path}");
}
let _ = write!(detail, ": {why}");
Ok(Outcome::new(false, detail))
}
}
}
#[cfg(feature = "json-schema")]
fn resolve_schema(recipe: &Recipe, assertion: &JsonValue) -> Result<JsonValue, String> {
match assertion.get("schema_path") {
Some(JsonValue::Null) | None => {}
Some(path) => return read_schema_file(recipe, &str_json(path)),
}
match assertion.get("schema") {
Some(JsonValue::Object(map)) => Ok(JsonValue::Object(map.clone())),
Some(JsonValue::String(path)) => read_schema_file(recipe, path),
_ => Err("json_schema requires an object schema or schema path".to_string()),
}
}
#[cfg(feature = "json-schema")]
fn read_schema_file(recipe: &Recipe, path: &str) -> Result<JsonValue, String> {
let resolved = recipe_path(recipe, path).to_string();
let text = std::fs::read_to_string(&resolved).map_err(|error| {
format!(
"schema file unreadable: {}",
describe_io_error(&error, &resolved)
)
})?;
serde_json::from_str(&text).map_err(|error| {
format!(
"invalid schema JSON: {}",
describe_json_error(&text, &error)
)
})
}
#[cfg(feature = "json-schema")]
fn describe_io_error(error: &std::io::Error, path: &str) -> String {
use std::io::ErrorKind;
let cause = match error.kind() {
ErrorKind::NotFound => "no such file or directory",
ErrorKind::PermissionDenied => "permission denied",
ErrorKind::IsADirectory => "is a directory",
ErrorKind::InvalidData => "not valid UTF-8",
_ => "cannot be read",
};
format!("{cause}: {}", repr_str(path))
}
#[cfg(feature = "json-schema")]
fn describe_json_error(input: &str, error: &serde_json::Error) -> String {
if input.starts_with('\u{feff}') {
return "unexpected UTF-8 byte order mark".to_string();
}
if input.trim().is_empty() {
return "no output to parse".to_string();
}
match error.classify() {
serde_json::error::Category::Eof => "unexpected end of input".to_string(),
serde_json::error::Category::Data => format!(
"unexpected value at line {} column {}",
error.line(),
error.column()
),
_ => format!(
"invalid syntax at line {} column {}",
error.line(),
error.column()
),
}
}
pub fn score(results: &[AssertionResult]) -> f64 {
crate::result::score_of(results.iter().map(|r| r.passed))
}
pub fn builtin_registry() -> BTreeMap<&'static str, ()> {
BUILTIN_TYPES.iter().map(|name| (*name, ())).collect()
}
#[cfg(test)]
mod tests {
use super::*;
use crate::models::{CommandSpec, Recipe};
use serde_json::json;
fn recipe_with_cwd(cwd: Option<&str>) -> Recipe {
Recipe {
name: "t".into(),
command: CommandSpec {
argv: vec!["true".into()],
cwd: cwd.map(str::to_string),
..CommandSpec::default()
},
..Recipe::default()
}
}
fn eval(assertion: JsonValue, screen: &str, raw: &str, code: Option<i32>) -> AssertionResult {
evaluate(&recipe_with_cwd(None), &assertion, screen, raw, code)
}
#[test]
fn contains_searches_only_its_own_haystack() {
let a = eval(
json!({"type": "output_contains", "value": "S"}),
"S",
"R",
None,
);
assert!(!a.passed);
let b = eval(
json!({"type": "screen_contains", "value": "R"}),
"S",
"R",
None,
);
assert!(!b.passed);
}
#[test]
fn the_contains_detail_is_the_same_either_way() {
let hit = eval(
json!({"type": "output_contains", "value": "R"}),
"",
"R",
None,
);
let miss = eval(
json!({"type": "output_contains", "value": "R"}),
"",
"",
None,
);
assert_eq!(hit.detail, "contains 'R'");
assert_eq!(miss.detail, "contains 'R'");
assert!(hit.passed && !miss.passed);
}
#[test]
fn an_empty_needle_is_always_present() {
assert!(
eval(
json!({"type": "output_contains", "value": ""}),
"",
"",
None
)
.passed
);
assert!(
!eval(
json!({"type": "output_not_contains", "value": ""}),
"",
"",
None
)
.passed
);
}
#[test]
fn an_empty_detail_override_falls_through() {
let overridden = eval(
json!({"type": "output_contains", "value": "R", "detail": "mine"}),
"",
"R",
None,
);
assert_eq!(overridden.detail, "mine");
let empty = eval(
json!({"type": "output_contains", "value": "R", "detail": ""}),
"",
"R",
None,
);
assert_eq!(empty.detail, "contains 'R'");
}
#[test]
fn the_name_defaults_to_the_type() {
assert_eq!(
eval(
json!({"type": "screen_contains", "value": ""}),
"",
"",
None
)
.name,
"screen_contains"
);
assert_eq!(
eval(
json!({"type": "screen_contains", "value": "", "name": "mine"}),
"",
"",
None
)
.name,
"mine"
);
}
#[test]
fn exit_code_compares_the_way_python_compares() {
let rows: [(JsonValue, Option<i32>, bool, &str); 8] = [
(json!(0), Some(0), true, "expected 0, got 0"),
(json!(1), Some(0), false, "expected 1, got 0"),
(json!(true), Some(1), true, "expected True, got 1"),
(json!(false), Some(0), true, "expected False, got 0"),
(json!(0.0), Some(0), true, "expected 0.0, got 0"),
(json!("0"), Some(0), false, "expected 0, got 0"),
(json!(0), None, false, "expected 0, got None"),
(json!(null), None, true, "expected None, got None"),
];
for (value, code, passed, detail) in rows {
let result = eval(json!({"type": "exit_code", "value": value}), "", "", code);
assert_eq!(result.passed, passed, "{detail}");
assert_eq!(result.detail, detail);
}
}
#[test]
fn paths_join_the_way_pathlib_joins_them() {
let recipe = recipe_with_cwd(Some("/tmp/fx"));
let cases = [
("/tmp/fx/exists.txt", "/tmp/fx/exists.txt"),
("exists.txt", "/tmp/fx/exists.txt"),
("./exists.txt", "/tmp/fx/exists.txt"),
("sub/../exists.txt", "/tmp/fx/sub/../exists.txt"),
("", "/tmp/fx"),
];
for (input, want) in cases {
let result = evaluate(
&recipe,
&json!({"type": "file_exists", "value": input}),
"",
"",
None,
);
assert_eq!(result.detail, want, "{input}");
}
let trailing = recipe_with_cwd(Some("/tmp/fx//"));
assert_eq!(
evaluate(
&trailing,
&json!({"type": "file_exists", "value": "exists.txt"}),
"",
"",
None
)
.detail,
"/tmp/fx/exists.txt"
);
}
#[test]
fn file_exists_stringifies_whatever_it_is_given() {
let recipe = recipe_with_cwd(Some("/tmp/fx"));
for (value, want) in [
(json!(5), "/tmp/fx/5"),
(json!(true), "/tmp/fx/True"),
(json!(null), "/tmp/fx/None"),
] {
let result = evaluate(
&recipe,
&json!({"type": "file_exists", "value": value}),
"",
"",
None,
);
assert_eq!(result.detail, want);
assert!(!result.passed);
}
}
#[test]
fn file_contains_reads_a_missing_file_as_empty() {
let recipe = recipe_with_cwd(Some("/tmp"));
let result = evaluate(
&recipe,
&json!({"type": "file_contains", "path": "definitely-absent-xyz", "value": "x"}),
"",
"",
None,
);
assert!(!result.passed);
assert_eq!(result.detail, "contains 'x'");
}
#[test]
fn file_contains_ignores_the_detail_override() {
let recipe = recipe_with_cwd(Some("/tmp"));
let result = evaluate(
&recipe,
&json!({"type": "file_contains", "path": "absent", "value": "x", "detail": "mine"}),
"",
"",
None,
);
assert_eq!(result.detail, "contains 'x'");
}
#[cfg(feature = "json-schema")]
#[test]
fn json_schema_reports_the_fixed_message_when_there_is_no_schema() {
for assertion in [
json!({"type": "json_schema"}),
json!({"type": "json_schema", "schema": []}),
json!({"type": "json_schema", "schema": 5}),
json!({"type": "json_schema", "schema": null}),
] {
let result = eval(assertion, "", "{}", None);
assert!(!result.passed);
assert_eq!(
result.detail,
"json_schema requires an object schema or schema path"
);
}
}
#[cfg(feature = "json-schema")]
#[test]
fn json_schema_passes_a_matching_instance() {
let result = eval(
json!({"type": "json_schema", "schema": {"type": "object"}}),
"",
" {\"a\": 1} ",
None,
);
assert!(result.passed);
assert_eq!(result.detail, "matches JSON schema");
}
#[cfg(feature = "json-schema")]
#[test]
fn json_schema_names_where_the_instance_failed() {
let result = eval(
json!({"type": "json_schema", "schema": {"properties": {"a": {"type": "string"}}}}),
"",
"{\"a\": 1}",
None,
);
assert_eq!(
result.detail,
"schema validation failed at a: 1 is not of type 'string'"
);
}
#[cfg(feature = "json-schema")]
#[test]
fn a_null_schema_path_is_not_a_path() {
let result = eval(
json!({"type": "json_schema", "schema_path": null, "schema": {"type": "object"}}),
"",
"{}",
None,
);
assert!(result.passed);
}
#[cfg(feature = "json-schema")]
#[test]
fn a_missing_schema_file_says_so_with_its_path() {
let recipe = recipe_with_cwd(Some("/tmp"));
let result = evaluate(
&recipe,
&json!({"type": "json_schema", "schema_path": "definitely-absent-xyz.json"}),
"",
"{}",
None,
);
assert_eq!(
result.detail,
"schema file unreadable: no such file or directory: '/tmp/definitely-absent-xyz.json'"
);
}
#[test]
fn every_malformed_input_is_contained() {
let malformed = [
json!({"value": "x"}),
json!({"type": "no_such_type"}),
json!({"type": null}),
json!({"type": "output_contains"}),
json!({"type": "exit_code"}),
json!({"type": "file_contains", "value": "x"}),
json!({"type": "output_contains", "value": 5}),
json!({"type": "output_contains", "value": null}),
];
for assertion in malformed {
let result = eval(assertion.clone(), "S", "R", Some(0));
assert!(!result.passed, "{assertion}");
assert!(!result.detail.is_empty(), "{assertion}");
assert!(!result.detail.contains('\n'), "{assertion}");
assert!(!result.name.is_empty(), "{assertion}");
}
}
#[test]
fn a_malformed_assertion_does_not_lose_the_others() {
let mut recipe = recipe_with_cwd(None);
recipe.assertions = vec![
json!({"type": "output_contains", "value": "R"}),
json!({"type": "output_contains"}),
json!({"type": "screen_contains", "value": "S"}),
];
recipe.expect_exit_code = Some(0);
let results = evaluate_all(&recipe, "S", "R", Some(0));
assert_eq!(results.len(), 4);
assert_eq!(
results.iter().map(|r| r.passed).collect::<Vec<_>>(),
vec![true, false, true, true]
);
}
#[test]
fn the_synthetic_exit_code_goes_last_and_is_never_suppressed() {
let mut recipe = recipe_with_cwd(None);
recipe.assertions = vec![json!({"type": "exit_code", "value": 3})];
recipe.expect_exit_code = Some(0);
let results = evaluate_all(&recipe, "", "", Some(0));
assert_eq!(results.len(), 2);
assert_eq!(results[0].detail, "expected 3, got 0");
assert_eq!(results[1].detail, "expected 0, got 0");
recipe.expect_exit_code = None;
assert_eq!(evaluate_all(&recipe, "", "", Some(0)).len(), 1);
}
#[test]
fn the_score_is_the_passing_fraction() {
let pass = AssertionResult {
name: "a".into(),
passed: true,
detail: String::new(),
};
let fail = AssertionResult {
passed: false,
..pass.clone()
};
assert_eq!(score(&[]), 1.0);
assert_eq!(score(std::slice::from_ref(&pass)), 1.0);
assert_eq!(score(std::slice::from_ref(&fail)), 0.0);
assert_eq!(score(&[pass.clone(), fail.clone()]), 0.5);
assert_eq!(score(&[pass, fail.clone(), fail]), 1.0 / 3.0);
}
#[test]
fn the_registry_names_every_builtin() {
assert_eq!(builtin_registry().len(), BUILTIN_TYPES.len());
for name in BUILTIN_TYPES {
let result = eval(json!({"type": name}), "", "", None);
assert!(
!result.detail.starts_with("unknown assertion type"),
"{name} is not dispatched"
);
}
}
#[test]
fn json_schema_is_dispatchable_exactly_when_the_registry_lists_it() {
let listed = BUILTIN_TYPES.contains(&"json_schema");
let detail = eval(json!({"type": "json_schema"}), "", "", None).detail;
let dispatched = !detail.starts_with("unknown assertion type");
assert_eq!(
listed, dispatched,
"registry says {listed}, dispatcher says {dispatched}"
);
assert_eq!(listed, cfg!(feature = "json-schema"));
assert_eq!(
BUILTIN_TYPES.len(),
UNCONDITIONAL_TYPES.len() + usize::from(listed)
);
for name in UNCONDITIONAL_TYPES {
assert!(BUILTIN_TYPES.contains(&name), "{name} was dropped");
}
}
}