use serde::Serialize;
#[derive(Debug, PartialEq, Serialize)]
pub struct TestRun {
pub passed: u32,
pub failed: u32,
#[serde(skip_serializing_if = "is_zero")]
pub skipped: u32,
#[serde(skip_serializing_if = "Option::is_none")]
pub suites: Option<Vec<Suite>>,
#[serde(skip_serializing_if = "Option::is_none")]
pub error: Option<TestError>,
#[serde(skip_serializing_if = "Option::is_none")]
pub coverage: Option<Coverage>,
#[serde(skip_serializing_if = "Option::is_none")]
pub transport: Option<String>,
}
fn is_zero(n: &u32) -> bool {
*n == 0
}
#[derive(Debug, PartialEq, Serialize)]
pub struct Coverage {
pub covered: u32,
pub lines: u32,
pub percent: u32,
pub files: Vec<FileCoverage>,
}
#[derive(Debug, PartialEq, Serialize)]
pub struct FileCoverage {
pub path: String,
pub covered: u32,
pub lines: u32,
pub percent: u32,
pub uncovered: Vec<u32>,
pub covered_lines: Vec<u32>,
}
impl From<&crate::coverage::CoverageReport> for Coverage {
fn from(r: &crate::coverage::CoverageReport) -> Self {
Coverage {
covered: r.total_covered(),
lines: r.total_lines(),
percent: r.total_percent(),
files: r
.files
.iter()
.map(|f| FileCoverage {
path: f.path.clone(),
covered: f.covered,
lines: f.total,
percent: crate::coverage::percent(f.covered, f.total),
uncovered: f.uncovered.clone(),
covered_lines: f.covered_lines.clone(),
})
.collect(),
}
}
}
#[derive(Debug, PartialEq, Serialize)]
pub struct Suite {
pub name: String,
pub kind: String,
pub cases: Vec<Case>,
}
#[derive(Debug, PartialEq, Serialize)]
pub struct Case {
pub name: String,
pub outcome: String,
#[serde(skip_serializing_if = "Option::is_none")]
pub message: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub location: Option<Location>,
#[serde(skip_serializing_if = "Option::is_none")]
pub retries: Option<u32>,
#[serde(skip_serializing_if = "Option::is_none")]
pub tier: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub kind: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub seed: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub counterexample: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub violation: Option<Violation>,
#[serde(skip_serializing_if = "Option::is_none")]
pub refusals: Option<Vec<Refusal>>,
#[serde(skip)]
pub duration_ms: Option<f64>,
}
#[derive(Debug, PartialEq, Serialize)]
pub struct Location {
pub path: String,
pub line: u32,
pub col: u32,
}
#[derive(Debug, PartialEq, Serialize)]
pub struct Violation {
pub kind: String,
pub agent: String,
pub name: String,
pub location: Option<Location>,
}
#[derive(Debug, PartialEq, Serialize)]
pub struct Refusal {
pub index: u32,
pub call: String,
pub kind: String,
pub agent: String,
pub name: String,
pub location: Option<Location>,
}
#[derive(Debug, PartialEq, Serialize)]
pub struct TestError {
pub kind: String,
#[serde(skip_serializing_if = "Option::is_none")]
pub message: Option<String>,
#[serde(skip_serializing_if = "Vec::is_empty")]
pub diagnostics: Vec<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub stderr: Option<String>,
}
impl TestRun {
pub fn empty() -> Self {
TestRun {
passed: 0,
failed: 0,
skipped: 0,
suites: Some(Vec::new()),
error: None,
coverage: None,
transport: None,
}
}
pub fn with_coverage(mut self, report: &crate::coverage::CoverageReport) -> Self {
if !report.is_empty() {
self.coverage = Some(Coverage::from(report));
}
self
}
pub fn discovered(suites: Vec<Suite>) -> Self {
TestRun {
passed: 0,
failed: 0,
skipped: 0,
suites: Some(suites),
error: None,
coverage: None,
transport: None,
}
}
pub fn runtime_error(message: impl Into<String>, stderr: Option<String>) -> Self {
TestRun {
passed: 0,
failed: 0,
skipped: 0,
suites: None,
error: Some(TestError {
kind: "runtime".to_string(),
message: Some(message.into()),
diagnostics: Vec::new(),
stderr: stderr.filter(|s| !s.trim().is_empty()),
}),
coverage: None,
transport: None,
}
}
pub fn compile_error(diagnostics: Vec<String>) -> Self {
TestRun {
passed: 0,
failed: 0,
skipped: 0,
suites: None,
error: Some(TestError {
kind: "compile".to_string(),
message: None,
diagnostics,
stderr: None,
}),
coverage: None,
transport: None,
}
}
pub fn with_tiers(mut self, discovered: &[bynk_emit::project::DiscoveredSuite]) -> Self {
for suite in self.suites.iter_mut().flatten() {
let Some(known) = discovered.iter().find(|d| d.name == suite.name) else {
continue;
};
for case in &mut suite.cases {
if let Some(tier) = known
.cases
.iter()
.find(|c| c.name == case.name)
.and_then(|c| c.tier)
{
case.tier = Some(tier.to_string());
}
}
}
self
}
pub fn with_remote(mut self, transport: String, skipped: &[(String, String, String)]) -> Self {
self.transport = Some(transport);
if skipped.is_empty() {
return self;
}
self.skipped = u32::try_from(skipped.len()).unwrap_or(u32::MAX);
let suites = self.suites.get_or_insert_with(Vec::new);
for (suite, case, reason) in skipped {
let case = Case {
name: case.clone(),
outcome: "skipped".to_string(),
message: Some(reason.clone()),
location: None,
retries: None,
tier: Some("system".to_string()),
kind: None,
seed: None,
counterexample: None,
violation: None,
refusals: None,
duration_ms: None,
};
match suites.iter_mut().find(|s| &s.name == suite) {
Some(s) => s.cases.push(case),
None => suites.push(Suite {
name: suite.clone(),
kind: "integration".to_string(),
cases: vec![case],
}),
}
}
self
}
pub fn render(&self) -> String {
let mut s = serde_json::to_string_pretty(self).expect("TestRun serialises");
s.push('\n');
s
}
}
#[derive(Debug, Default, Clone, PartialEq, Eq)]
pub struct Unmatched {
pub suites: Vec<String>,
pub cases: Vec<String>,
}
impl Unmatched {
pub fn is_empty(&self) -> bool {
self.suites.is_empty() && self.cases.is_empty()
}
pub fn messages(&self) -> Vec<String> {
let suites = self
.suites
.iter()
.map(|s| format!("no suite named --suite {s:?}"));
let cases = self
.cases
.iter()
.map(|c| format!("no case matched --case {c:?}"));
suites.chain(cases).collect()
}
pub fn report(&self, program: &str) {
for message in self.messages() {
eprintln!("{program} test: {message}");
}
}
}
#[derive(Debug, Default, PartialEq)]
pub struct ParsedRun {
pub passed: u32,
pub failed: u32,
pub suites: Vec<Suite>,
pub complete: bool,
pub unmatched: Unmatched,
}
impl ParsedRun {
pub fn into_document(self, stderr: &str) -> TestRun {
if self.complete {
TestRun {
passed: self.passed,
failed: self.failed,
skipped: 0,
suites: Some(self.suites),
error: None,
coverage: None,
transport: None,
}
} else {
let trimmed = stderr.trim();
TestRun {
passed: self.passed,
failed: self.failed,
skipped: 0,
suites: Some(self.suites),
error: Some(TestError {
kind: "runtime".to_string(),
message: Some("the test runner exited before completing".to_string()),
diagnostics: Vec::new(),
stderr: (!trimmed.is_empty()).then(|| trimmed.to_string()),
}),
coverage: None,
transport: None,
}
}
}
}
pub fn parse_ndjson(stdout: &str) -> ParsedRun {
let mut run = ParsedRun::default();
for line in stdout.lines() {
let line = line.trim();
if line.is_empty() {
continue;
}
let Ok(value) = serde_json::from_str::<serde_json::Value>(line) else {
continue;
};
match value.get("type").and_then(|t| t.as_str()) {
Some("suite-begin") => {
run.suites.push(Suite {
name: str_field(&value, "name"),
kind: str_field(&value, "kind"),
cases: Vec::new(),
});
}
Some("case") => {
let case = case_from_event(&value);
if case.outcome == "pass" {
run.passed += 1;
} else {
run.failed += 1;
}
if let Some(suite) = run.suites.last_mut() {
suite.cases.push(case);
}
}
Some("run-end") => {
run.complete = true;
run.unmatched = unmatched_from_event(&value);
}
_ => {}
}
}
run
}
pub(crate) fn case_from_event(value: &serde_json::Value) -> Case {
let message = value
.get("message")
.and_then(|m| m.as_str())
.map(str::to_string);
let location = value
.get("location")
.and_then(|l| l.as_str())
.and_then(parse_location);
let retries = value
.get("retries")
.and_then(serde_json::Value::as_u64)
.filter(|n| *n > 0)
.map(|n| u32::try_from(n).unwrap_or(u32::MAX));
let violation = value.get("violation").map(violation_from_json);
let refusals = refusals_from_json(value.get("refusals"));
Case {
name: str_field(value, "name"),
outcome: str_field(value, "outcome"),
message,
location,
retries,
tier: None,
kind: str_opt(value, "kind"),
seed: str_opt(value, "seed"),
counterexample: str_opt(value, "counterexample"),
violation,
refusals,
duration_ms: value.get("durationMs").and_then(serde_json::Value::as_f64),
}
}
fn location_from_json(value: &serde_json::Value) -> Option<Location> {
value.get("location").and_then(|l| match l {
serde_json::Value::String(s) => parse_location(s),
serde_json::Value::Object(_) => Some(Location {
path: str_field(l, "path"),
line: l
.get("line")
.and_then(serde_json::Value::as_u64)
.unwrap_or(0) as u32,
col: l
.get("col")
.and_then(serde_json::Value::as_u64)
.unwrap_or(0) as u32,
}),
_ => None,
})
}
pub(crate) fn violation_from_json(value: &serde_json::Value) -> Violation {
Violation {
kind: str_field(value, "kind"),
agent: str_field(value, "agent"),
name: str_field(value, "name"),
location: location_from_json(value),
}
}
pub(crate) fn refusals_from_json(value: Option<&serde_json::Value>) -> Option<Vec<Refusal>> {
let refusals: Vec<Refusal> = value?
.as_array()?
.iter()
.filter_map(|r| {
Some(Refusal {
index: u32::try_from(r.get("index")?.as_u64()?).ok()?,
call: r.get("call")?.as_str()?.to_string(),
kind: r.get("kind")?.as_str()?.to_string(),
agent: r.get("agent")?.as_str()?.to_string(),
name: r.get("name")?.as_str()?.to_string(),
location: location_from_json(r),
})
})
.collect();
(!refusals.is_empty()).then_some(refusals)
}
pub(crate) fn unmatched_from_event(value: &serde_json::Value) -> Unmatched {
let Some(unmatched) = value.get("unmatched") else {
return Unmatched::default();
};
let strings = |key: &str| -> Vec<String> {
unmatched
.get(key)
.and_then(|v| v.as_array())
.map(|a| {
a.iter()
.filter_map(|v| v.as_str())
.map(str::to_string)
.collect()
})
.unwrap_or_default()
};
Unmatched {
suites: strings("suites"),
cases: strings("cases"),
}
}
fn str_opt(value: &serde_json::Value, key: &str) -> Option<String> {
value.get(key).and_then(|v| v.as_str()).map(str::to_string)
}
fn str_field(value: &serde_json::Value, key: &str) -> String {
value
.get(key)
.and_then(|v| v.as_str())
.unwrap_or_default()
.to_string()
}
fn parse_location(s: &str) -> Option<Location> {
let (rest, col) = s.rsplit_once(':')?;
let (path, line) = rest.rsplit_once(':')?;
let line: u32 = line.parse().ok()?;
let col: u32 = col.parse().ok()?;
if path.is_empty() {
return None;
}
Some(Location {
path: path.to_string(),
line,
col,
})
}
#[cfg(test)]
mod tests {
use super::*;
fn known(
suite: &str,
cases: &[(&str, Option<&'static str>)],
) -> bynk_emit::project::DiscoveredSuite {
bynk_emit::project::DiscoveredSuite {
name: suite.to_string(),
kind: "unit",
cases: cases
.iter()
.map(|(name, tier)| bynk_emit::project::DiscoveredCase {
name: (*name).to_string(),
location: None,
tier: *tier,
generative: None,
})
.collect(),
skipped: Vec::new(),
envs: Vec::new(),
}
}
#[test]
fn a_cases_duration_is_read_but_never_written_to_the_document() {
let ndjson = concat!(
r#"{"type":"suite-begin","name":"s","kind":"unit","tests":1}"#,
"\n",
r#"{"type":"case","suite":"s","name":"c","outcome":"pass","durationMs":1.5}"#,
"\n",
r#"{"type":"suite-end","name":"s"}"#,
"\n",
r#"{"type":"run-end","passed":1,"failed":0}"#,
"\n",
);
let parsed = parse_ndjson(ndjson);
assert_eq!(parsed.suites[0].cases[0].duration_ms, Some(1.5));
let rendered = parsed.into_document("").render();
assert!(!rendered.contains("durationMs"), "{rendered}");
}
#[test]
fn run_end_reports_the_values_a_filter_selected_nothing_for() {
let ended = parse_ndjson(concat!(
r#"{"type":"run-begin","suites":0}"#,
"\n",
r#"{"type":"run-end","passed":0,"failed":0,"unmatched":{"suites":["nope"],"cases":["typo","other"]}}"#,
"\n",
));
assert!(ended.complete);
assert_eq!(ended.unmatched.suites, vec!["nope".to_string()]);
assert_eq!(
ended.unmatched.cases,
vec!["typo".to_string(), "other".to_string()]
);
assert_eq!(
ended.unmatched.messages(),
vec![
"no suite named --suite \"nope\"".to_string(),
"no case matched --case \"typo\"".to_string(),
"no case matched --case \"other\"".to_string(),
]
);
let plain = parse_ndjson(r#"{"type":"run-end","passed":1,"failed":0}"#);
assert!(plain.unmatched.is_empty());
}
#[test]
fn a_run_takes_each_cases_tier_from_the_case_table() {
let ndjson = concat!(
r#"{"type":"suite-begin","name":"shop","kind":"unit"}"#,
"\n",
r#"{"type":"case","name":"wired","outcome":"pass"}"#,
"\n",
r#"{"type":"case","name":"plain","outcome":"fail","message":"no"}"#,
"\n",
r#"{"type":"case","name":"a property","outcome":"pass"}"#,
"\n",
r#"{"type":"case","name":"unlisted","outcome":"pass"}"#,
"\n",
r#"{"type":"suite-begin","name":"other","kind":"unit"}"#,
"\n",
r#"{"type":"case","name":"wired","outcome":"pass"}"#,
"\n",
r#"{"type":"run-end"}"#,
"\n",
);
let table = [
known(
"shop",
&[
("wired", Some("system")),
("plain", Some("unit")),
("a property", None),
],
),
known("other", &[("wired", Some("integration"))]),
];
let doc = parse_ndjson(ndjson).into_document("").with_tiers(&table);
let suites = doc.suites.unwrap();
let tiers = |i: usize| -> Vec<Option<String>> {
suites[i].cases.iter().map(|c| c.tier.clone()).collect()
};
assert_eq!(
tiers(0),
[Some("system".into()), Some("unit".into()), None, None]
);
assert_eq!(
tiers(1),
[Some("integration".into())],
"the same name in another suite"
);
}
#[test]
fn file_coverage_carries_covered_and_uncovered_lines() {
let report = crate::coverage::CoverageReport {
files: vec![crate::coverage::FileCoverage {
path: "src/demo/m.bynk".into(),
covered: 2,
total: 5,
uncovered: vec![7, 8, 9],
covered_lines: vec![3, 4],
}],
};
assert_eq!(
serde_json::to_string(&Coverage::from(&report)).unwrap(),
r#"{"covered":2,"lines":5,"percent":40,"files":[{"path":"src/demo/m.bynk","covered":2,"lines":5,"percent":40,"uncovered":[7,8,9],"covered_lines":[3,4]}]}"#
);
}
#[test]
fn refusals_read_in_either_shape_and_drop_what_is_malformed() {
let refusal = |location: &str| {
format!(
r#"{{"index":1,"call":"Spend","kind":"transition","agent":"Wallet","name":"never_decreases","location":{location}}}"#
)
};
let as_event = refusals_from_json(
serde_json::from_str(&format!(
r#"[{}]"#,
refusal(r#""src/demo/wallet.bynk:18:14""#)
))
.ok()
.as_ref(),
)
.unwrap();
let as_document = refusals_from_json(
serde_json::from_str(&format!(
r#"[{}]"#,
refusal(r#"{"path":"src/demo/wallet.bynk","line":18,"col":14}"#)
))
.ok()
.as_ref(),
)
.unwrap();
assert_eq!(as_event, as_document);
assert_eq!(as_event[0].index, 1);
assert_eq!(as_event[0].kind, "transition");
assert_eq!(as_event[0].location.as_ref().unwrap().line, 18);
let malformed: serde_json::Value =
serde_json::from_str(r#"[{"index":0,"call":"Spend","kind":"invariant"}]"#).unwrap();
assert_eq!(refusals_from_json(Some(&malformed)), None);
assert_eq!(refusals_from_json(Some(&serde_json::json!([]))), None);
assert_eq!(refusals_from_json(Some(&serde_json::json!("x"))), None);
assert_eq!(refusals_from_json(None), None);
}
#[test]
fn a_violation_reads_its_location_in_either_shape() {
let as_event = serde_json::json!({
"kind": "transition", "agent": "Meter", "name": "same",
"location": "src/demo/m.bynk:28:14"
});
let as_document = serde_json::json!({
"kind": "transition", "agent": "Meter", "name": "same",
"location": { "path": "src/demo/m.bynk", "line": 28, "col": 14 }
});
let expected = Some(Location {
path: "src/demo/m.bynk".to_string(),
line: 28,
col: 14,
});
assert_eq!(violation_from_json(&as_event).location, expected);
assert_eq!(violation_from_json(&as_document).location, expected);
}
}